/* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. * Copyright 2009 Jerome Glisse. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * * Authors: Dave Airlie * Alex Deucher * Jerome Glisse */#include <linux/dma-fence-array.h>#include <linux/interval_tree_generic.h>#include <linux/idr.h>#include <drm/drmP.h>#include <drm/amdgpu_drm.h>#include "amdgpu.h"#include "amdgpu_trace.h" /* * PASID manager * * PASIDs are global address space identifiers that can be shared * between the GPU, an IOMMU and the driver. VMs on different devices * may use the same PASID if they share the same address * space. Therefore PASIDs are allocated using a global IDA. VMs are * looked up from the PASID per amdgpu_device. */static DEFINE_IDA(amdgpu_vm_pasid_ida); /** * amdgpu_vm_alloc_pasid - Allocate a PASID * @bits: Maximum width of the PASID in bits, must be at least 1 * * Allocates a PASID of the given width while keeping smaller PASIDs * available if possible. * * Returns a positive integer on success. Returns %-EINVAL if bits==0. * Returns %-ENOSPC if no PASID was available. Returns %-ENOMEM on * memory allocation failure. */int amdgpu_vm_alloc_pasid(unsigned int bits){ int pasid = -EINVAL; for (bits = min(bits, 31U); bits > 0; bits--) { pasid = ida_simple_get(&amdgpu_vm_pasid_ida, 1U << (bits - 1), 1U << bits, GFP_KERNEL); if (pasid != -ENOSPC) break; } return pasid;} /** * amdgpu_vm_free_pasid - Free a PASID * @pasid: PASID to free */void amdgpu_vm_free_pasid(unsigned int pasid){ ida_simple_remove(&amdgpu_vm_pasid_ida, pasid);} /* * GPUVM * GPUVM is similar to the legacy gart on older asics, however * rather than there being a single global gart table * for the entire GPU, there are multiple VM page tables active * at any given time. The VM page tables can contain a mix * vram pages and system memory pages and system memory pages * can be mapped as snooped (cached system pages) or unsnooped * (uncached system pages). * Each VM has an ID associated with it and there is a page table * associated with each VMID. When execting a command buffer, * the kernel tells the the ring what VMID to use for that command * buffer. VMIDs are allocated dynamically as commands are submitted. * The userspace drivers maintain their own address space and the kernel * sets up their pages tables accordingly when they submit their * command buffers and a VMID is assigned. * Cayman/Trinity support up to 8 active VMs at any given time; * SI supports 16. */ #define START(node) ((node)->start)#define LAST(node) ((node)->last) INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last, START, LAST, static, amdgpu_vm_it) #undef START#undef LAST /* Local structure. Encapsulate some VM table update parameters to reduce * the number of function parameters */struct amdgpu_pte_update_params { /* amdgpu device we do this update for */ struct amdgpu_device *adev; /* optional amdgpu_vm we do this update for */ struct amdgpu_vm *vm; /* address where to copy page table entries from */ uint64_t src; /* indirect buffer to fill with commands */ struct amdgpu_ib *ib; /* Function which actually does the update */ void (*func)(struct amdgpu_pte_update_params *params, uint64_t pe, uint64_t addr, unsigned count, uint32_t incr, uint64_t flags); /* The next two are used during VM update by CPU * DMA addresses to use for mapping * Kernel pointer of PD/PT BO that needs to be updated */ dma_addr_t *pages_addr; void *kptr;}; /* Helper to disable partial resident texture feature from a fence callback */struct amdgpu_prt_cb { struct amdgpu_device *adev; struct dma_fence_cb cb;}; /** * amdgpu_vm_num_entries - return the number of entries in a PD/PT * * @adev: amdgpu_device pointer * * Calculate the number of entries in a page directory or page table. */static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev, unsigned level){ if (level == 0) /* For the root directory */ return adev->vm_manager.max_pfn >> (adev->vm_manager.block_size * adev->vm_manager.num_level); else if (level == adev->vm_manager.num_level) /* For the page tables on the leaves */ return AMDGPU_VM_PTE_COUNT(adev); else /* Everything in between */ return 1 << adev->vm_manager.block_size;} /** * amdgpu_vm_bo_size - returns the size of the BOs in bytes * * @adev: amdgpu_device pointer * * Calculate the size of the BO for a page directory or page table in bytes. */static unsigned amdgpu_vm_bo_size(struct amdgpu_device *adev, unsigned level){ return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev, level) * 8);} /** * amdgpu_vm_get_pd_bo - add the VM PD to a validation list * * @vm: vm providing the BOs * @validated: head of validation list * @entry: entry to add * * Add the page directory to the list of BOs to * validate for command submission. */void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm, struct list_head *validated, struct amdgpu_bo_list_entry *entry){ entry->robj = vm->root.base.bo; entry->priority = 0; entry->tv.bo = &entry->robj->tbo; entry->tv.shared = true; entry->user_pages = NULL; list_add(&entry->tv.head, validated);} /** * amdgpu_vm_validate_pt_bos - validate the page table BOs * * @adev: amdgpu device pointer * @vm: vm providing the BOs * @validate: callback to do the validation * @param: parameter for the validation callback * * Validate the page table BOs on command submission if neccessary. */int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm, int (*validate)(void *p, struct amdgpu_bo *bo), void *param){ struct ttm_bo_global *glob = adev->mman.bdev.glob; int r; spin_lock(&vm->status_lock); while (!list_empty(&vm->evicted)) { struct amdgpu_vm_bo_base *bo_base; struct amdgpu_bo *bo; bo_base = list_first_entry(&vm->evicted, struct amdgpu_vm_bo_base, vm_status); spin_unlock(&vm->status_lock); bo = bo_base->bo; BUG_ON(!bo); if (bo->parent) { r = validate(param, bo); if (r) return r; spin_lock(&glob->lru_lock); ttm_bo_move_to_lru_tail(&bo->tbo); if (bo->shadow) ttm_bo_move_to_lru_tail(&bo->shadow->tbo); spin_unlock(&glob->lru_lock); } if (bo->tbo.type == ttm_bo_type_kernel && vm->use_cpu_for_update) { r = amdgpu_bo_kmap(bo, NULL); if (r) return r; } spin_lock(&vm->status_lock); if (bo->tbo.type != ttm_bo_type_kernel) list_move(&bo_base->vm_status, &vm->moved); else list_move(&bo_base->vm_status, &vm->relocated); } spin_unlock(&vm->status_lock); return 0;} /** * amdgpu_vm_ready - check VM is ready for updates * * @vm: VM to check * * Check if all VM PDs/PTs are ready for updates */bool amdgpu_vm_ready(struct amdgpu_vm *vm){ bool ready; spin_lock(&vm->status_lock); ready = list_empty(&vm->evicted); spin_unlock(&vm->status_lock); return ready;} /** * amdgpu_vm_alloc_levels - allocate the PD/PT levels * * @adev: amdgpu_device pointer * @vm: requested vm * @saddr: start of the address range * @eaddr: end of the address range * * Make sure the page directories and page tables are allocated */static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_vm_pt *parent, uint64_t saddr, uint64_t eaddr, unsigned level){ unsigned shift = (adev->vm_manager.num_level - level) * adev->vm_manager.block_size; unsigned pt_idx, from, to; int r; u64 flags; uint64_t init_value = 0; if (!parent->entries) { unsigned num_entries = amdgpu_vm_num_entries(adev, level); parent->entries = kvmalloc_array(num_entries, sizeof(struct amdgpu_vm_pt), GFP_KERNEL | __GFP_ZERO); if (!parent->entries) return -ENOMEM; memset(parent->entries, 0 , sizeof(struct amdgpu_vm_pt)); } from = saddr >> shift; to = eaddr >> shift; if (from >= amdgpu_vm_num_entries(adev, level) || to >= amdgpu_vm_num_entries(adev, level)) return -EINVAL; if (to > parent->last_entry_used) parent->last_entry_used = to; ++level; saddr = saddr & ((1 << shift) - 1); eaddr = eaddr & ((1 << shift) - 1); flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS | AMDGPU_GEM_CREATE_VRAM_CLEARED; if (vm->use_cpu_for_update) flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; else flags |= (AMDGPU_GEM_CREATE_NO_CPU_ACCESS | AMDGPU_GEM_CREATE_SHADOW); if (vm->pte_support_ats) { init_value = AMDGPU_PTE_DEFAULT_ATC; if (level != adev->vm_manager.num_level - 1) init_value |= AMDGPU_PDE_PTE; } /* walk over the address space and allocate the page tables */ for (pt_idx = from; pt_idx <= to; ++pt_idx) { struct reservation_object *resv = vm->root.base.bo->tbo.resv; struct amdgpu_vm_pt *entry = &parent->entries[pt_idx]; struct amdgpu_bo *pt; if (!entry->base.bo) { r = amdgpu_bo_create(adev, amdgpu_vm_bo_size(adev, level), AMDGPU_GPU_PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM, flags, NULL, resv, init_value, &pt); if (r) return r; if (vm->use_cpu_for_update) { r = amdgpu_bo_kmap(pt, NULL); if (r) { amdgpu_bo_unref(&pt); return r; } } /* Keep a reference to the root directory to avoid * freeing them up in the wrong order. */ pt->parent = amdgpu_bo_ref(parent->base.bo); entry->base.vm = vm; entry->base.bo = pt; list_add_tail(&entry->base.bo_list, &pt->va); spin_lock(&vm->status_lock); list_add(&entry->base.vm_status, &vm->relocated); spin_unlock(&vm->status_lock); entry->addr = 0; } if (level < adev->vm_manager.num_level) { uint64_t sub_saddr = (pt_idx == from) ? saddr : 0; uint64_t sub_eaddr = (pt_idx == to) ? eaddr : ((1 << shift) - 1); r = amdgpu_vm_alloc_levels(adev, vm, entry, sub_saddr, sub_eaddr, level); if (r) return r; } } return 0;} /** * amdgpu_vm_alloc_pts - Allocate page tables. * * @adev: amdgpu_device pointer * @vm: VM to allocate page tables for * @saddr: Start address which needs to be allocated * @size: Size from start address we need. * * Make sure the page tables are allocated. */int amdgpu_vm_alloc_pts(struct amdgpu_device *adev, struct amdgpu_vm *vm, uint64_t saddr, uint64_t size){ uint64_t last_pfn; uint64_t eaddr; /* validate the parameters */ if (saddr & AMDGPU_GPU_PAGE_MASK || size & AMDGPU_GPU_PAGE_MASK) return -EINVAL; eaddr = saddr + size - 1; last_pfn = eaddr / AMDGPU_GPU_PAGE_SIZE; if (last_pfn >= adev->vm_manager.max_pfn) { dev_err(adev->dev, "va above limit (0x%08llX >= 0x%08llX)\n", last_pfn, adev->vm_manager.max_pfn); return -EINVAL; } saddr /= AMDGPU_GPU_PAGE_SIZE; eaddr /= AMDGPU_GPU_PAGE_SIZE; return amdgpu_vm_alloc_levels(adev, vm, &vm->root, saddr, eaddr, 0);} /** * amdgpu_vm_had_gpu_reset - check if reset occured since last use * * @adev: amdgpu_device pointer * @id: VMID structure * * Check if GPU reset occured since last use of the VMID. */static bool amdgpu_vm_had_gpu_reset(struct amdgpu_device *adev, struct amdgpu_vm_id *id){ return id->current_gpu_reset_count != atomic_read(&adev->gpu_reset_counter);} static bool amdgpu_vm_reserved_vmid_ready(struct amdgpu_vm *vm, unsigned vmhub){ return !!vm->reserved_vmid[vmhub];} /* idr_mgr->lock must be held */static int amdgpu_vm_grab_reserved_vmid_locked(struct amdgpu_vm *vm, struct amdgpu_ring *ring, struct amdgpu_sync *sync, struct dma_fence *fence, struct amdgpu_job *job){ struct amdgpu_device *adev = ring->adev; unsigned vmhub = ring->funcs->vmhub; uint64_t fence_context = adev->fence_context + ring->idx; struct amdgpu_vm_id *id = vm->reserved_vmid[vmhub]; struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub]; struct dma_fence *updates = sync->last_vm_update; int r = 0; struct dma_fence *flushed, *tmp; bool needs_flush = vm->use_cpu_for_update; flushed = id->flushed_updates; if ((amdgpu_vm_had_gpu_reset(adev, id)) || (atomic64_read(&id->owner) != vm->client_id) || (job->vm_pd_addr != id->pd_gpu_addr) || (updates && (!flushed || updates->context != flushed->context || dma_fence_is_later(updates, flushed))) || (!id->last_flush || (id->last_flush->context != fence_context && !dma_fence_is_signaled(id->last_flush)))) { needs_flush = true; /* to prevent one context starved by another context */ id->pd_gpu_addr = 0; tmp = amdgpu_sync_peek_fence(&id->active, ring); if (tmp) { r = amdgpu_sync_fence(adev, sync, tmp); return r; } } /* Good we can use this VMID. Remember this submission as * user of the VMID. */ r = amdgpu_sync_fence(ring->adev, &id->active, fence); if (r) goto out; if (updates && (!flushed || updates->context != flushed->context || dma_fence_is_later(updates, flushed))) { dma_fence_put(id->flushed_updates); id->flushed_updates = dma_fence_get(updates); } id->pd_gpu_addr = job->vm_pd_addr; atomic64_set(&id->owner, vm->client_id); job->vm_needs_flush = needs_flush; if (needs_flush) { dma_fence_put(id->last_flush); id->last_flush = NULL; } job->vm_id = id - id_mgr->ids; trace_amdgpu_vm_grab_id(vm, ring, job);out:Lines 1–500 of 2952. Load more
/* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. * Copyright 2009 Jerome Glisse. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * * Authors: Dave Airlie * Alex Deucher * Jerome Glisse */ #include <linux/dma-fence-array.h> #include <linux/interval_tree_generic.h> #include <linux/idr.h> #include <drm/drmP.h> #include <drm/amdgpu_drm.h> #include "amdgpu.h" #include "amdgpu_trace.h" /* * PASID manager * * PASIDs are global address space identifiers that can be shared * between the GPU, an IOMMU and the driver. VMs on different devices * may use the same PASID if they share the same address * space. Therefore PASIDs are allocated using a global IDA. VMs are * looked up from the PASID per amdgpu_device. */ static DEFINE_IDA(amdgpu_vm_pasid_ida); /** * amdgpu_vm_alloc_pasid - Allocate a PASID * @bits: Maximum width of the PASID in bits, must be at least 1 * * Allocates a PASID of the given width while keeping smaller PASIDs * available if possible. * * Returns a positive integer on success. Returns %-EINVAL if bits==0. * Returns %-ENOSPC if no PASID was available. Returns %-ENOMEM on * memory allocation failure. */ int amdgpu_vm_alloc_pasid(unsigned int bits) { int pasid = -EINVAL; for (bits = min(bits, 31U); bits > 0; bits--) { pasid = ida_simple_get(&amdgpu_vm_pasid_ida, 1U << (bits - 1), 1U << bits, GFP_KERNEL); if (pasid != -ENOSPC) break; } return pasid; } /** * amdgpu_vm_free_pasid - Free a PASID * @pasid: PASID to free */ void amdgpu_vm_free_pasid(unsigned int pasid) { ida_simple_remove(&amdgpu_vm_pasid_ida, pasid); } /* * GPUVM * GPUVM is similar to the legacy gart on older asics, however * rather than there being a single global gart table * for the entire GPU, there are multiple VM page tables active * at any given time. The VM page tables can contain a mix * vram pages and system memory pages and system memory pages * can be mapped as snooped (cached system pages) or unsnooped * (uncached system pages). * Each VM has an ID associated with it and there is a page table * associated with each VMID. When execting a command buffer, * the kernel tells the the ring what VMID to use for that command * buffer. VMIDs are allocated dynamically as commands are submitted. * The userspace drivers maintain their own address space and the kernel * sets up their pages tables accordingly when they submit their * command buffers and a VMID is assigned. * Cayman/Trinity support up to 8 active VMs at any given time; * SI supports 16. */ #define START(node) ((node)->start) #define LAST(node) ((node)->last) INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last, START, LAST, static, amdgpu_vm_it) #undef START #undef LAST /* Local structure. Encapsulate some VM table update parameters to reduce * the number of function parameters */ struct amdgpu_pte_update_params { /* amdgpu device we do this update for */ struct amdgpu_device *adev; /* optional amdgpu_vm we do this update for */ struct amdgpu_vm *vm; /* address where to copy page table entries from */ uint64_t src; /* indirect buffer to fill with commands */ struct amdgpu_ib *ib; /* Function which actually does the update */ void (*func)(struct amdgpu_pte_update_params *params, uint64_t pe, uint64_t addr, unsigned count, uint32_t incr, uint64_t flags); /* The next two are used during VM update by CPU * DMA addresses to use for mapping * Kernel pointer of PD/PT BO that needs to be updated */ dma_addr_t *pages_addr; void *kptr; }; /* Helper to disable partial resident texture feature from a fence callback */ struct amdgpu_prt_cb { struct amdgpu_device *adev; struct dma_fence_cb cb; }; /** * amdgpu_vm_num_entries - return the number of entries in a PD/PT * * @adev: amdgpu_device pointer * * Calculate the number of entries in a page directory or page table. */ static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev, unsigned level) { if (level == 0) /* For the root directory */ return adev->vm_manager.max_pfn >> (adev->vm_manager.block_size * adev->vm_manager.num_level); else if (level == adev->vm_manager.num_level) /* For the page tables on the leaves */ return AMDGPU_VM_PTE_COUNT(adev); else /* Everything in between */ return 1 << adev->vm_manager.block_size; } /** * amdgpu_vm_bo_size - returns the size of the BOs in bytes * * @adev: amdgpu_device pointer * * Calculate the size of the BO for a page directory or page table in bytes. */ static unsigned amdgpu_vm_bo_size(struct amdgpu_device *adev, unsigned level) { return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev, level) * 8); } /** * amdgpu_vm_get_pd_bo - add the VM PD to a validation list * * @vm: vm providing the BOs * @validated: head of validation list * @entry: entry to add * * Add the page directory to the list of BOs to * validate for command submission. */ void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm, struct list_head *validated, struct amdgpu_bo_list_entry *entry) { entry->robj = vm->root.base.bo; entry->priority = 0; entry->tv.bo = &entry->robj->tbo; entry->tv.shared = true; entry->user_pages = NULL; list_add(&entry->tv.head, validated); } /** * amdgpu_vm_validate_pt_bos - validate the page table BOs * * @adev: amdgpu device pointer * @vm: vm providing the BOs * @validate: callback to do the validation * @param: parameter for the validation callback * * Validate the page table BOs on command submission if neccessary. */ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm, int (*validate)(void *p, struct amdgpu_bo *bo), void *param) { struct ttm_bo_global *glob = adev->mman.bdev.glob; int r; spin_lock(&vm->status_lock); while (!list_empty(&vm->evicted)) { struct amdgpu_vm_bo_base *bo_base; struct amdgpu_bo *bo; bo_base = list_first_entry(&vm->evicted, struct amdgpu_vm_bo_base, vm_status); spin_unlock(&vm->status_lock); bo = bo_base->bo; BUG_ON(!bo); if (bo->parent) { r = validate(param, bo); if (r) return r; spin_lock(&glob->lru_lock); ttm_bo_move_to_lru_tail(&bo->tbo); if (bo->shadow) ttm_bo_move_to_lru_tail(&bo->shadow->tbo); spin_unlock(&glob->lru_lock); } if (bo->tbo.type == ttm_bo_type_kernel && vm->use_cpu_for_update) { r = amdgpu_bo_kmap(bo, NULL); if (r) return r; } spin_lock(&vm->status_lock); if (bo->tbo.type != ttm_bo_type_kernel) list_move(&bo_base->vm_status, &vm->moved); else list_move(&bo_base->vm_status, &vm->relocated); } spin_unlock(&vm->status_lock); return 0; } /** * amdgpu_vm_ready - check VM is ready for updates * * @vm: VM to check * * Check if all VM PDs/PTs are ready for updates */ bool amdgpu_vm_ready(struct amdgpu_vm *vm) { bool ready; spin_lock(&vm->status_lock); ready = list_empty(&vm->evicted); spin_unlock(&vm->status_lock); return ready; } /** * amdgpu_vm_alloc_levels - allocate the PD/PT levels * * @adev: amdgpu_device pointer * @vm: requested vm * @saddr: start of the address range * @eaddr: end of the address range * * Make sure the page directories and page tables are allocated */ static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_vm_pt *parent, uint64_t saddr, uint64_t eaddr, unsigned level) { unsigned shift = (adev->vm_manager.num_level - level) * adev->vm_manager.block_size; unsigned pt_idx, from, to; int r; u64 flags; uint64_t init_value = 0; if (!parent->entries) { unsigned num_entries = amdgpu_vm_num_entries(adev, level); parent->entries = kvmalloc_array(num_entries, sizeof(struct amdgpu_vm_pt), GFP_KERNEL | __GFP_ZERO); if (!parent->entries) return -ENOMEM; memset(parent->entries, 0 , sizeof(struct amdgpu_vm_pt)); } from = saddr >> shift; to = eaddr >> shift; if (from >= amdgpu_vm_num_entries(adev, level) || to >= amdgpu_vm_num_entries(adev, level)) return -EINVAL; if (to > parent->last_entry_used) parent->last_entry_used = to; ++level; saddr = saddr & ((1 << shift) - 1); eaddr = eaddr & ((1 << shift) - 1); flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS | AMDGPU_GEM_CREATE_VRAM_CLEARED; if (vm->use_cpu_for_update) flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; else flags |= (AMDGPU_GEM_CREATE_NO_CPU_ACCESS | AMDGPU_GEM_CREATE_SHADOW); if (vm->pte_support_ats) { init_value = AMDGPU_PTE_DEFAULT_ATC; if (level != adev->vm_manager.num_level - 1) init_value |= AMDGPU_PDE_PTE; } /* walk over the address space and allocate the page tables */ for (pt_idx = from; pt_idx <= to; ++pt_idx) { struct reservation_object *resv = vm->root.base.bo->tbo.resv; struct amdgpu_vm_pt *entry = &parent->entries[pt_idx]; struct amdgpu_bo *pt; if (!entry->base.bo) { r = amdgpu_bo_create(adev, amdgpu_vm_bo_size(adev, level), AMDGPU_GPU_PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM, flags, NULL, resv, init_value, &pt); if (r) return r; if (vm->use_cpu_for_update) { r = amdgpu_bo_kmap(pt, NULL); if (r) { amdgpu_bo_unref(&pt); return r; } } /* Keep a reference to the root directory to avoid * freeing them up in the wrong order. */ pt->parent = amdgpu_bo_ref(parent->base.bo); entry->base.vm = vm; entry->base.bo = pt; list_add_tail(&entry->base.bo_list, &pt->va); spin_lock(&vm->status_lock); list_add(&entry->base.vm_status, &vm->relocated); spin_unlock(&vm->status_lock); entry->addr = 0; } if (level < adev->vm_manager.num_level) { uint64_t sub_saddr = (pt_idx == from) ? saddr : 0; uint64_t sub_eaddr = (pt_idx == to) ? eaddr : ((1 << shift) - 1); r = amdgpu_vm_alloc_levels(adev, vm, entry, sub_saddr, sub_eaddr, level); if (r) return r; } } return 0; } /** * amdgpu_vm_alloc_pts - Allocate page tables. * * @adev: amdgpu_device pointer * @vm: VM to allocate page tables for * @saddr: Start address which needs to be allocated * @size: Size from start address we need. * * Make sure the page tables are allocated. */ int amdgpu_vm_alloc_pts(struct amdgpu_device *adev, struct amdgpu_vm *vm, uint64_t saddr, uint64_t size) { uint64_t last_pfn; uint64_t eaddr; /* validate the parameters */ if (saddr & AMDGPU_GPU_PAGE_MASK || size & AMDGPU_GPU_PAGE_MASK) return -EINVAL; eaddr = saddr + size - 1; last_pfn = eaddr / AMDGPU_GPU_PAGE_SIZE; if (last_pfn >= adev->vm_manager.max_pfn) { dev_err(adev->dev, "va above limit (0x%08llX >= 0x%08llX)\n", last_pfn, adev->vm_manager.max_pfn); return -EINVAL; } saddr /= AMDGPU_GPU_PAGE_SIZE; eaddr /= AMDGPU_GPU_PAGE_SIZE; return amdgpu_vm_alloc_levels(adev, vm, &vm->root, saddr, eaddr, 0); } /** * amdgpu_vm_had_gpu_reset - check if reset occured since last use * * @adev: amdgpu_device pointer * @id: VMID structure * * Check if GPU reset occured since last use of the VMID. */ static bool amdgpu_vm_had_gpu_reset(struct amdgpu_device *adev, struct amdgpu_vm_id *id) { return id->current_gpu_reset_count != atomic_read(&adev->gpu_reset_counter); } static bool amdgpu_vm_reserved_vmid_ready(struct amdgpu_vm *vm, unsigned vmhub) { return !!vm->reserved_vmid[vmhub]; } /* idr_mgr->lock must be held */ static int amdgpu_vm_grab_reserved_vmid_locked(struct amdgpu_vm *vm, struct amdgpu_ring *ring, struct amdgpu_sync *sync, struct dma_fence *fence, struct amdgpu_job *job) { struct amdgpu_device *adev = ring->adev; unsigned vmhub = ring->funcs->vmhub; uint64_t fence_context = adev->fence_context + ring->idx; struct amdgpu_vm_id *id = vm->reserved_vmid[vmhub]; struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub]; struct dma_fence *updates = sync->last_vm_update; int r = 0; struct dma_fence *flushed, *tmp; bool needs_flush = vm->use_cpu_for_update; flushed = id->flushed_updates; if ((amdgpu_vm_had_gpu_reset(adev, id)) || (atomic64_read(&id->owner) != vm->client_id) || (job->vm_pd_addr != id->pd_gpu_addr) || (updates && (!flushed || updates->context != flushed->context || dma_fence_is_later(updates, flushed))) || (!id->last_flush || (id->last_flush->context != fence_context && !dma_fence_is_signaled(id->last_flush)))) { needs_flush = true; /* to prevent one context starved by another context */ id->pd_gpu_addr = 0; tmp = amdgpu_sync_peek_fence(&id->active, ring); if (tmp) { r = amdgpu_sync_fence(adev, sync, tmp); return r; } } /* Good we can use this VMID. Remember this submission as * user of the VMID. */ r = amdgpu_sync_fence(ring->adev, &id->active, fence); if (r) goto out; if (updates && (!flushed || updates->context != flushed->context || dma_fence_is_later(updates, flushed))) { dma_fence_put(id->flushed_updates); id->flushed_updates = dma_fence_get(updates); } id->pd_gpu_addr = job->vm_pd_addr; atomic64_set(&id->owner, vm->client_id); job->vm_needs_flush = needs_flush; if (needs_flush) { dma_fence_put(id->last_flush); id->last_flush = NULL; } job->vm_id = id - id_mgr->ids; trace_amdgpu_vm_grab_id(vm, ring, job); out: