/* * Copyright 2014-2018 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ #define pr_fmt(fmt) "kfd2kgd: " fmt #include <linux/list.h>#include <linux/pagemap.h>#include <linux/sched/mm.h>#include <linux/dma-buf.h>#include <drm/drmP.h>#include "amdgpu_object.h"#include "amdgpu_vm.h"#include "amdgpu_amdkfd.h" /* Special VM and GART address alignment needed for VI pre-Fiji due to * a HW bug. */#define VI_BO_SIZE_ALIGN (0x8000) /* BO flag to indicate a KFD userptr BO */#define AMDGPU_AMDKFD_USERPTR_BO (1ULL << 63) /* Userptr restore delay, just long enough to allow consecutive VM * changes to accumulate */#define AMDGPU_USERPTR_RESTORE_DELAY_MS 1 /* Impose limit on how much memory KFD can use */static struct { uint64_t max_system_mem_limit; uint64_t max_ttm_mem_limit; int64_t system_mem_used; int64_t ttm_mem_used; spinlock_t mem_limit_lock;} kfd_mem_limit; /* Struct used for amdgpu_amdkfd_bo_validate */struct amdgpu_vm_parser { uint32_t domain; bool wait;}; static const char * const domain_bit_to_string[] = { "CPU", "GTT", "VRAM", "GDS", "GWS", "OA"}; #define domain_string(domain) domain_bit_to_string[ffs(domain)-1] static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work); static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd){ return (struct amdgpu_device *)kgd;} static bool check_if_add_bo_to_vm(struct amdgpu_vm *avm, struct kgd_mem *mem){ struct kfd_bo_va_list *entry; list_for_each_entry(entry, &mem->bo_va_list, bo_list) if (entry->bo_va->base.vm == avm) return false; return true;} /* Set memory usage limits. Current, limits are * System (TTM + userptr) memory - 3/4th System RAM * TTM memory - 3/8th System RAM */void amdgpu_amdkfd_gpuvm_init_mem_limits(void){ struct sysinfo si; uint64_t mem; si_meminfo(&si); mem = si.totalram - si.totalhigh; mem *= si.mem_unit; spin_lock_init(&kfd_mem_limit.mem_limit_lock); kfd_mem_limit.max_system_mem_limit = (mem >> 1) + (mem >> 2); kfd_mem_limit.max_ttm_mem_limit = (mem >> 1) - (mem >> 3); pr_debug("Kernel memory limit %lluM, TTM limit %lluM\n", (kfd_mem_limit.max_system_mem_limit >> 20), (kfd_mem_limit.max_ttm_mem_limit >> 20));} static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, uint64_t size, u32 domain, bool sg){ size_t acc_size, system_mem_needed, ttm_mem_needed, vram_needed; uint64_t reserved_for_pt = amdgpu_amdkfd_total_mem_size >> 9; int ret = 0; acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size, sizeof(struct amdgpu_bo)); vram_needed = 0; if (domain == AMDGPU_GEM_DOMAIN_GTT) { /* TTM GTT memory */ system_mem_needed = acc_size + size; ttm_mem_needed = acc_size + size; } else if (domain == AMDGPU_GEM_DOMAIN_CPU && !sg) { /* Userptr */ system_mem_needed = acc_size + size; ttm_mem_needed = acc_size; } else { /* VRAM and SG */ system_mem_needed = acc_size; ttm_mem_needed = acc_size; if (domain == AMDGPU_GEM_DOMAIN_VRAM) vram_needed = size; } spin_lock(&kfd_mem_limit.mem_limit_lock); if ((kfd_mem_limit.system_mem_used + system_mem_needed > kfd_mem_limit.max_system_mem_limit) || (kfd_mem_limit.ttm_mem_used + ttm_mem_needed > kfd_mem_limit.max_ttm_mem_limit) || (adev->kfd.vram_used + vram_needed > adev->gmc.real_vram_size - reserved_for_pt)) { ret = -ENOMEM; } else { kfd_mem_limit.system_mem_used += system_mem_needed; kfd_mem_limit.ttm_mem_used += ttm_mem_needed; adev->kfd.vram_used += vram_needed; } spin_unlock(&kfd_mem_limit.mem_limit_lock); return ret;} static void unreserve_mem_limit(struct amdgpu_device *adev, uint64_t size, u32 domain, bool sg){ size_t acc_size; acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size, sizeof(struct amdgpu_bo)); spin_lock(&kfd_mem_limit.mem_limit_lock); if (domain == AMDGPU_GEM_DOMAIN_GTT) { kfd_mem_limit.system_mem_used -= (acc_size + size); kfd_mem_limit.ttm_mem_used -= (acc_size + size); } else if (domain == AMDGPU_GEM_DOMAIN_CPU && !sg) { kfd_mem_limit.system_mem_used -= (acc_size + size); kfd_mem_limit.ttm_mem_used -= acc_size; } else { kfd_mem_limit.system_mem_used -= acc_size; kfd_mem_limit.ttm_mem_used -= acc_size; if (domain == AMDGPU_GEM_DOMAIN_VRAM) { adev->kfd.vram_used -= size; WARN_ONCE(adev->kfd.vram_used < 0, "kfd VRAM memory accounting unbalanced"); } } WARN_ONCE(kfd_mem_limit.system_mem_used < 0, "kfd system memory accounting unbalanced"); WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0, "kfd TTM memory accounting unbalanced"); spin_unlock(&kfd_mem_limit.mem_limit_lock);} void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo){ struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); u32 domain = bo->preferred_domains; bool sg = (bo->preferred_domains == AMDGPU_GEM_DOMAIN_CPU); if (bo->flags & AMDGPU_AMDKFD_USERPTR_BO) { domain = AMDGPU_GEM_DOMAIN_CPU; sg = false; } unreserve_mem_limit(adev, amdgpu_bo_size(bo), domain, sg);} /* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's * reservation object. * * @bo: [IN] Remove eviction fence(s) from this BO * @ef: [IN] This eviction fence is removed if it * is present in the shared list. * * NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held. */static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo, struct amdgpu_amdkfd_fence *ef){ struct reservation_object *resv = bo->tbo.resv; struct reservation_object_list *old, *new; unsigned int i, j, k; if (!ef) return -EINVAL; old = reservation_object_get_list(resv); if (!old) return 0; new = kmalloc(offsetof(typeof(*new), shared[old->shared_max]), GFP_KERNEL); if (!new) return -ENOMEM; /* Go through all the shared fences in the resevation object and sort * the interesting ones to the end of the list. */ for (i = 0, j = old->shared_count, k = 0; i < old->shared_count; ++i) { struct dma_fence *f; f = rcu_dereference_protected(old->shared[i], reservation_object_held(resv)); if (f->context == ef->base.context) RCU_INIT_POINTER(new->shared[--j], f); else RCU_INIT_POINTER(new->shared[k++], f); } new->shared_max = old->shared_max; new->shared_count = k; /* Install the new fence list, seqcount provides the barriers */ preempt_disable(); write_seqcount_begin(&resv->seq); RCU_INIT_POINTER(resv->fence, new); write_seqcount_end(&resv->seq); preempt_enable(); /* Drop the references to the removed fences or move them to ef_list */ for (i = j, k = 0; i < old->shared_count; ++i) { struct dma_fence *f; f = rcu_dereference_protected(new->shared[i], reservation_object_held(resv)); dma_fence_put(f); } kfree_rcu(old, rcu); return 0;} static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain, bool wait){ struct ttm_operation_ctx ctx = { false, false }; int ret; if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm), "Called with userptr BO")) return -EINVAL; amdgpu_bo_placement_from_domain(bo, domain); ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); if (ret) goto validate_fail; if (wait) amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false); validate_fail: return ret;} static int amdgpu_amdkfd_validate(void *param, struct amdgpu_bo *bo){ struct amdgpu_vm_parser *p = param; return amdgpu_amdkfd_bo_validate(bo, p->domain, p->wait);} /* vm_validate_pt_pd_bos - Validate page table and directory BOs * * Page directories are not updated here because huge page handling * during page table updates can invalidate page directory entries * again. Page directories are only updated after updating page * tables. */static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm){ struct amdgpu_bo *pd = vm->root.base.bo; struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); struct amdgpu_vm_parser param; int ret; param.domain = AMDGPU_GEM_DOMAIN_VRAM; param.wait = false; ret = amdgpu_vm_validate_pt_bos(adev, vm, amdgpu_amdkfd_validate, ¶m); if (ret) { pr_err("amdgpu: failed to validate PT BOs\n"); return ret; } ret = amdgpu_amdkfd_validate(¶m, pd); if (ret) { pr_err("amdgpu: failed to validate PD\n"); return ret; } vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.base.bo); if (vm->use_cpu_for_update) { ret = amdgpu_bo_kmap(pd, NULL); if (ret) { pr_err("amdgpu: failed to kmap PD, ret=%d\n", ret); return ret; } } return 0;} static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync){ struct amdgpu_bo *pd = vm->root.base.bo; struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); int ret; ret = amdgpu_vm_update_directories(adev, vm); if (ret) return ret; return amdgpu_sync_fence(NULL, sync, vm->last_update, false);} /* add_bo_to_vm - Add a BO to a VM * * Everything that needs to bo done only once when a BO is first added * to a VM. It can later be mapped and unmapped many times without * repeating these steps. * * 1. Allocate and initialize BO VA entry data structure * 2. Add BO to the VM * 3. Determine ASIC-specific PTE flags * 4. Alloc page tables and directories if needed * 4a. Validate new page tables and directories */static int add_bo_to_vm(struct amdgpu_device *adev, struct kgd_mem *mem, struct amdgpu_vm *vm, bool is_aql, struct kfd_bo_va_list **p_bo_va_entry){ int ret; struct kfd_bo_va_list *bo_va_entry; struct amdgpu_bo *bo = mem->bo; uint64_t va = mem->va; struct list_head *list_bo_va = &mem->bo_va_list; unsigned long bo_size = bo->tbo.mem.size; if (!va) { pr_err("Invalid VA when adding BO to VM\n"); return -EINVAL; } if (is_aql) va += bo_size; bo_va_entry = kzalloc(sizeof(*bo_va_entry), GFP_KERNEL); if (!bo_va_entry) return -ENOMEM; pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va, va + bo_size, vm); /* Add BO to VM internal data structures*/ bo_va_entry->bo_va = amdgpu_vm_bo_add(adev, vm, bo); if (!bo_va_entry->bo_va) { ret = -EINVAL; pr_err("Failed to add BO object to VM. ret == %d\n", ret); goto err_vmadd; } bo_va_entry->va = va; bo_va_entry->pte_flags = amdgpu_gmc_get_pte_flags(adev, mem->mapping_flags); bo_va_entry->kgd_dev = (void *)adev; list_add(&bo_va_entry->bo_list, list_bo_va); if (p_bo_va_entry) *p_bo_va_entry = bo_va_entry; /* Allocate validate page tables if needed */ ret = vm_validate_pt_pd_bos(vm); if (ret) { pr_err("validate_pt_pd_bos() failed\n"); goto err_alloc_pts; } return 0; err_alloc_pts: amdgpu_vm_bo_rmv(adev, bo_va_entry->bo_va); list_del(&bo_va_entry->bo_list);err_vmadd: kfree(bo_va_entry); return ret;} static void remove_bo_from_vm(struct amdgpu_device *adev, struct kfd_bo_va_list *entry, unsigned long size){ pr_debug("\t remove VA 0x%llx - 0x%llx in entry %p\n", entry->va, entry->va + size, entry); amdgpu_vm_bo_rmv(adev, entry->bo_va); list_del(&entry->bo_list); kfree(entry);} static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem, struct amdkfd_process_info *process_info, bool userptr){ struct ttm_validate_buffer *entry = &mem->validate_list; struct amdgpu_bo *bo = mem->bo; INIT_LIST_HEAD(&entry->head); entry->num_shared = 1; entry->bo = &bo->tbo; mutex_lock(&process_info->lock); if (userptr) list_add_tail(&entry->head, &process_info->userptr_valid_list); else list_add_tail(&entry->head, &process_info->kfd_bo_list); mutex_unlock(&process_info->lock);} /* Initializes user pages. It registers the MMU notifier and validates * the userptr BO in the GTT domain. * * The BO must already be on the userptr_valid_list. Otherwise an * eviction and restore may happen that leaves the new BO unmapped * with the user mode queues running. * * Takes the process_info->lock to protect against concurrent restore * workers. * * Returns 0 for success, negative errno for errors. */static int init_user_pages(struct kgd_mem *mem, struct mm_struct *mm, uint64_t user_addr){ struct amdkfd_process_info *process_info = mem->process_info; struct amdgpu_bo *bo = mem->bo; struct ttm_operation_ctx ctx = { true, false }; int ret = 0; mutex_lock(&process_info->lock); ret = amdgpu_ttm_tt_set_userptr(bo->tbo.ttm, user_addr, 0); if (ret) { pr_err("%s: Failed to set userptr: %d\n", __func__, ret); goto out; } ret = amdgpu_mn_register(bo, user_addr); if (ret) { pr_err("%s: Failed to register MMU notifier: %d\n", __func__, ret); goto out; } /* If no restore worker is running concurrently, user_pages * should not be allocated */ WARN(mem->user_pages, "Leaking user_pages array"); mem->user_pages = kvmalloc_array(bo->tbo.ttm->num_pages, sizeof(struct page *),Lines 1–500 of 2132. Load more
/* * Copyright 2014-2018 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ #define pr_fmt(fmt) "kfd2kgd: " fmt #include <linux/list.h> #include <linux/pagemap.h> #include <linux/sched/mm.h> #include <linux/dma-buf.h> #include <drm/drmP.h> #include "amdgpu_object.h" #include "amdgpu_vm.h" #include "amdgpu_amdkfd.h" /* Special VM and GART address alignment needed for VI pre-Fiji due to * a HW bug. */ #define VI_BO_SIZE_ALIGN (0x8000) /* BO flag to indicate a KFD userptr BO */ #define AMDGPU_AMDKFD_USERPTR_BO (1ULL << 63) /* Userptr restore delay, just long enough to allow consecutive VM * changes to accumulate */ #define AMDGPU_USERPTR_RESTORE_DELAY_MS 1 /* Impose limit on how much memory KFD can use */ static struct { uint64_t max_system_mem_limit; uint64_t max_ttm_mem_limit; int64_t system_mem_used; int64_t ttm_mem_used; spinlock_t mem_limit_lock; } kfd_mem_limit; /* Struct used for amdgpu_amdkfd_bo_validate */ struct amdgpu_vm_parser { uint32_t domain; bool wait; }; static const char * const domain_bit_to_string[] = { "CPU", "GTT", "VRAM", "GDS", "GWS", "OA" }; #define domain_string(domain) domain_bit_to_string[ffs(domain)-1] static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work); static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd) { return (struct amdgpu_device *)kgd; } static bool check_if_add_bo_to_vm(struct amdgpu_vm *avm, struct kgd_mem *mem) { struct kfd_bo_va_list *entry; list_for_each_entry(entry, &mem->bo_va_list, bo_list) if (entry->bo_va->base.vm == avm) return false; return true; } /* Set memory usage limits. Current, limits are * System (TTM + userptr) memory - 3/4th System RAM * TTM memory - 3/8th System RAM */ void amdgpu_amdkfd_gpuvm_init_mem_limits(void) { struct sysinfo si; uint64_t mem; si_meminfo(&si); mem = si.totalram - si.totalhigh; mem *= si.mem_unit; spin_lock_init(&kfd_mem_limit.mem_limit_lock); kfd_mem_limit.max_system_mem_limit = (mem >> 1) + (mem >> 2); kfd_mem_limit.max_ttm_mem_limit = (mem >> 1) - (mem >> 3); pr_debug("Kernel memory limit %lluM, TTM limit %lluM\n", (kfd_mem_limit.max_system_mem_limit >> 20), (kfd_mem_limit.max_ttm_mem_limit >> 20)); } static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, uint64_t size, u32 domain, bool sg) { size_t acc_size, system_mem_needed, ttm_mem_needed, vram_needed; uint64_t reserved_for_pt = amdgpu_amdkfd_total_mem_size >> 9; int ret = 0; acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size, sizeof(struct amdgpu_bo)); vram_needed = 0; if (domain == AMDGPU_GEM_DOMAIN_GTT) { /* TTM GTT memory */ system_mem_needed = acc_size + size; ttm_mem_needed = acc_size + size; } else if (domain == AMDGPU_GEM_DOMAIN_CPU && !sg) { /* Userptr */ system_mem_needed = acc_size + size; ttm_mem_needed = acc_size; } else { /* VRAM and SG */ system_mem_needed = acc_size; ttm_mem_needed = acc_size; if (domain == AMDGPU_GEM_DOMAIN_VRAM) vram_needed = size; } spin_lock(&kfd_mem_limit.mem_limit_lock); if ((kfd_mem_limit.system_mem_used + system_mem_needed > kfd_mem_limit.max_system_mem_limit) || (kfd_mem_limit.ttm_mem_used + ttm_mem_needed > kfd_mem_limit.max_ttm_mem_limit) || (adev->kfd.vram_used + vram_needed > adev->gmc.real_vram_size - reserved_for_pt)) { ret = -ENOMEM; } else { kfd_mem_limit.system_mem_used += system_mem_needed; kfd_mem_limit.ttm_mem_used += ttm_mem_needed; adev->kfd.vram_used += vram_needed; } spin_unlock(&kfd_mem_limit.mem_limit_lock); return ret; } static void unreserve_mem_limit(struct amdgpu_device *adev, uint64_t size, u32 domain, bool sg) { size_t acc_size; acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size, sizeof(struct amdgpu_bo)); spin_lock(&kfd_mem_limit.mem_limit_lock); if (domain == AMDGPU_GEM_DOMAIN_GTT) { kfd_mem_limit.system_mem_used -= (acc_size + size); kfd_mem_limit.ttm_mem_used -= (acc_size + size); } else if (domain == AMDGPU_GEM_DOMAIN_CPU && !sg) { kfd_mem_limit.system_mem_used -= (acc_size + size); kfd_mem_limit.ttm_mem_used -= acc_size; } else { kfd_mem_limit.system_mem_used -= acc_size; kfd_mem_limit.ttm_mem_used -= acc_size; if (domain == AMDGPU_GEM_DOMAIN_VRAM) { adev->kfd.vram_used -= size; WARN_ONCE(adev->kfd.vram_used < 0, "kfd VRAM memory accounting unbalanced"); } } WARN_ONCE(kfd_mem_limit.system_mem_used < 0, "kfd system memory accounting unbalanced"); WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0, "kfd TTM memory accounting unbalanced"); spin_unlock(&kfd_mem_limit.mem_limit_lock); } void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo) { struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); u32 domain = bo->preferred_domains; bool sg = (bo->preferred_domains == AMDGPU_GEM_DOMAIN_CPU); if (bo->flags & AMDGPU_AMDKFD_USERPTR_BO) { domain = AMDGPU_GEM_DOMAIN_CPU; sg = false; } unreserve_mem_limit(adev, amdgpu_bo_size(bo), domain, sg); } /* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's * reservation object. * * @bo: [IN] Remove eviction fence(s) from this BO * @ef: [IN] This eviction fence is removed if it * is present in the shared list. * * NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held. */ static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo, struct amdgpu_amdkfd_fence *ef) { struct reservation_object *resv = bo->tbo.resv; struct reservation_object_list *old, *new; unsigned int i, j, k; if (!ef) return -EINVAL; old = reservation_object_get_list(resv); if (!old) return 0; new = kmalloc(offsetof(typeof(*new), shared[old->shared_max]), GFP_KERNEL); if (!new) return -ENOMEM; /* Go through all the shared fences in the resevation object and sort * the interesting ones to the end of the list. */ for (i = 0, j = old->shared_count, k = 0; i < old->shared_count; ++i) { struct dma_fence *f; f = rcu_dereference_protected(old->shared[i], reservation_object_held(resv)); if (f->context == ef->base.context) RCU_INIT_POINTER(new->shared[--j], f); else RCU_INIT_POINTER(new->shared[k++], f); } new->shared_max = old->shared_max; new->shared_count = k; /* Install the new fence list, seqcount provides the barriers */ preempt_disable(); write_seqcount_begin(&resv->seq); RCU_INIT_POINTER(resv->fence, new); write_seqcount_end(&resv->seq); preempt_enable(); /* Drop the references to the removed fences or move them to ef_list */ for (i = j, k = 0; i < old->shared_count; ++i) { struct dma_fence *f; f = rcu_dereference_protected(new->shared[i], reservation_object_held(resv)); dma_fence_put(f); } kfree_rcu(old, rcu); return 0; } static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain, bool wait) { struct ttm_operation_ctx ctx = { false, false }; int ret; if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm), "Called with userptr BO")) return -EINVAL; amdgpu_bo_placement_from_domain(bo, domain); ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); if (ret) goto validate_fail; if (wait) amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false); validate_fail: return ret; } static int amdgpu_amdkfd_validate(void *param, struct amdgpu_bo *bo) { struct amdgpu_vm_parser *p = param; return amdgpu_amdkfd_bo_validate(bo, p->domain, p->wait); } /* vm_validate_pt_pd_bos - Validate page table and directory BOs * * Page directories are not updated here because huge page handling * during page table updates can invalidate page directory entries * again. Page directories are only updated after updating page * tables. */ static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm) { struct amdgpu_bo *pd = vm->root.base.bo; struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); struct amdgpu_vm_parser param; int ret; param.domain = AMDGPU_GEM_DOMAIN_VRAM; param.wait = false; ret = amdgpu_vm_validate_pt_bos(adev, vm, amdgpu_amdkfd_validate, ¶m); if (ret) { pr_err("amdgpu: failed to validate PT BOs\n"); return ret; } ret = amdgpu_amdkfd_validate(¶m, pd); if (ret) { pr_err("amdgpu: failed to validate PD\n"); return ret; } vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.base.bo); if (vm->use_cpu_for_update) { ret = amdgpu_bo_kmap(pd, NULL); if (ret) { pr_err("amdgpu: failed to kmap PD, ret=%d\n", ret); return ret; } } return 0; } static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync) { struct amdgpu_bo *pd = vm->root.base.bo; struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); int ret; ret = amdgpu_vm_update_directories(adev, vm); if (ret) return ret; return amdgpu_sync_fence(NULL, sync, vm->last_update, false); } /* add_bo_to_vm - Add a BO to a VM * * Everything that needs to bo done only once when a BO is first added * to a VM. It can later be mapped and unmapped many times without * repeating these steps. * * 1. Allocate and initialize BO VA entry data structure * 2. Add BO to the VM * 3. Determine ASIC-specific PTE flags * 4. Alloc page tables and directories if needed * 4a. Validate new page tables and directories */ static int add_bo_to_vm(struct amdgpu_device *adev, struct kgd_mem *mem, struct amdgpu_vm *vm, bool is_aql, struct kfd_bo_va_list **p_bo_va_entry) { int ret; struct kfd_bo_va_list *bo_va_entry; struct amdgpu_bo *bo = mem->bo; uint64_t va = mem->va; struct list_head *list_bo_va = &mem->bo_va_list; unsigned long bo_size = bo->tbo.mem.size; if (!va) { pr_err("Invalid VA when adding BO to VM\n"); return -EINVAL; } if (is_aql) va += bo_size; bo_va_entry = kzalloc(sizeof(*bo_va_entry), GFP_KERNEL); if (!bo_va_entry) return -ENOMEM; pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va, va + bo_size, vm); /* Add BO to VM internal data structures*/ bo_va_entry->bo_va = amdgpu_vm_bo_add(adev, vm, bo); if (!bo_va_entry->bo_va) { ret = -EINVAL; pr_err("Failed to add BO object to VM. ret == %d\n", ret); goto err_vmadd; } bo_va_entry->va = va; bo_va_entry->pte_flags = amdgpu_gmc_get_pte_flags(adev, mem->mapping_flags); bo_va_entry->kgd_dev = (void *)adev; list_add(&bo_va_entry->bo_list, list_bo_va); if (p_bo_va_entry) *p_bo_va_entry = bo_va_entry; /* Allocate validate page tables if needed */ ret = vm_validate_pt_pd_bos(vm); if (ret) { pr_err("validate_pt_pd_bos() failed\n"); goto err_alloc_pts; } return 0; err_alloc_pts: amdgpu_vm_bo_rmv(adev, bo_va_entry->bo_va); list_del(&bo_va_entry->bo_list); err_vmadd: kfree(bo_va_entry); return ret; } static void remove_bo_from_vm(struct amdgpu_device *adev, struct kfd_bo_va_list *entry, unsigned long size) { pr_debug("\t remove VA 0x%llx - 0x%llx in entry %p\n", entry->va, entry->va + size, entry); amdgpu_vm_bo_rmv(adev, entry->bo_va); list_del(&entry->bo_list); kfree(entry); } static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem, struct amdkfd_process_info *process_info, bool userptr) { struct ttm_validate_buffer *entry = &mem->validate_list; struct amdgpu_bo *bo = mem->bo; INIT_LIST_HEAD(&entry->head); entry->num_shared = 1; entry->bo = &bo->tbo; mutex_lock(&process_info->lock); if (userptr) list_add_tail(&entry->head, &process_info->userptr_valid_list); else list_add_tail(&entry->head, &process_info->kfd_bo_list); mutex_unlock(&process_info->lock); } /* Initializes user pages. It registers the MMU notifier and validates * the userptr BO in the GTT domain. * * The BO must already be on the userptr_valid_list. Otherwise an * eviction and restore may happen that leaves the new BO unmapped * with the user mode queues running. * * Takes the process_info->lock to protect against concurrent restore * workers. * * Returns 0 for success, negative errno for errors. */ static int init_user_pages(struct kgd_mem *mem, struct mm_struct *mm, uint64_t user_addr) { struct amdkfd_process_info *process_info = mem->process_info; struct amdgpu_bo *bo = mem->bo; struct ttm_operation_ctx ctx = { true, false }; int ret = 0; mutex_lock(&process_info->lock); ret = amdgpu_ttm_tt_set_userptr(bo->tbo.ttm, user_addr, 0); if (ret) { pr_err("%s: Failed to set userptr: %d\n", __func__, ret); goto out; } ret = amdgpu_mn_register(bo, user_addr); if (ret) { pr_err("%s: Failed to register MMU notifier: %d\n", __func__, ret); goto out; } /* If no restore worker is running concurrently, user_pages * should not be allocated */ WARN(mem->user_pages, "Leaking user_pages array"); mem->user_pages = kvmalloc_array(bo->tbo.ttm->num_pages, sizeof(struct page *),