/* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. * Copyright 2009 Jerome Glisse. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * * Authors: Dave Airlie * Alex Deucher * Jerome Glisse */#include <linux/dma-fence-array.h>#include <linux/interval_tree_generic.h>#include <drm/drmP.h>#include <drm/amdgpu_drm.h>#include "amdgpu.h"#include "amdgpu_trace.h" /* * GPUVM * GPUVM is similar to the legacy gart on older asics, however * rather than there being a single global gart table * for the entire GPU, there are multiple VM page tables active * at any given time. The VM page tables can contain a mix * vram pages and system memory pages and system memory pages * can be mapped as snooped (cached system pages) or unsnooped * (uncached system pages). * Each VM has an ID associated with it and there is a page table * associated with each VMID. When execting a command buffer, * the kernel tells the the ring what VMID to use for that command * buffer. VMIDs are allocated dynamically as commands are submitted. * The userspace drivers maintain their own address space and the kernel * sets up their pages tables accordingly when they submit their * command buffers and a VMID is assigned. * Cayman/Trinity support up to 8 active VMs at any given time; * SI supports 16. */ #define START(node) ((node)->start)#define LAST(node) ((node)->last) INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last, START, LAST, static, amdgpu_vm_it) #undef START#undef LAST /* Local structure. Encapsulate some VM table update parameters to reduce * the number of function parameters */struct amdgpu_pte_update_params { /* amdgpu device we do this update for */ struct amdgpu_device *adev; /* optional amdgpu_vm we do this update for */ struct amdgpu_vm *vm; /* address where to copy page table entries from */ uint64_t src; /* indirect buffer to fill with commands */ struct amdgpu_ib *ib; /* Function which actually does the update */ void (*func)(struct amdgpu_pte_update_params *params, uint64_t pe, uint64_t addr, unsigned count, uint32_t incr, uint64_t flags); /* indicate update pt or its shadow */ bool shadow; /* The next two are used during VM update by CPU * DMA addresses to use for mapping * Kernel pointer of PD/PT BO that needs to be updated */ dma_addr_t *pages_addr; void *kptr;}; /* Helper to disable partial resident texture feature from a fence callback */struct amdgpu_prt_cb { struct amdgpu_device *adev; struct dma_fence_cb cb;}; /** * amdgpu_vm_num_entries - return the number of entries in a PD/PT * * @adev: amdgpu_device pointer * * Calculate the number of entries in a page directory or page table. */static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev, unsigned level){ if (level == 0) /* For the root directory */ return adev->vm_manager.max_pfn >> (adev->vm_manager.block_size * adev->vm_manager.num_level); else if (level == adev->vm_manager.num_level) /* For the page tables on the leaves */ return AMDGPU_VM_PTE_COUNT(adev); else /* Everything in between */ return 1 << adev->vm_manager.block_size;} /** * amdgpu_vm_bo_size - returns the size of the BOs in bytes * * @adev: amdgpu_device pointer * * Calculate the size of the BO for a page directory or page table in bytes. */static unsigned amdgpu_vm_bo_size(struct amdgpu_device *adev, unsigned level){ return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev, level) * 8);} /** * amdgpu_vm_get_pd_bo - add the VM PD to a validation list * * @vm: vm providing the BOs * @validated: head of validation list * @entry: entry to add * * Add the page directory to the list of BOs to * validate for command submission. */void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm, struct list_head *validated, struct amdgpu_bo_list_entry *entry){ entry->robj = vm->root.bo; entry->priority = 0; entry->tv.bo = &entry->robj->tbo; entry->tv.shared = true; entry->user_pages = NULL; list_add(&entry->tv.head, validated);} /** * amdgpu_vm_validate_layer - validate a single page table level * * @parent: parent page table level * @validate: callback to do the validation * @param: parameter for the validation callback * * Validate the page table BOs on command submission if neccessary. */static int amdgpu_vm_validate_level(struct amdgpu_vm_pt *parent, int (*validate)(void *, struct amdgpu_bo *), void *param){ unsigned i; int r; if (!parent->entries) return 0; for (i = 0; i <= parent->last_entry_used; ++i) { struct amdgpu_vm_pt *entry = &parent->entries[i]; if (!entry->bo) continue; r = validate(param, entry->bo); if (r) return r; /* * Recurse into the sub directory. This is harmless because we * have only a maximum of 5 layers. */ r = amdgpu_vm_validate_level(entry, validate, param); if (r) return r; } return r;} /** * amdgpu_vm_validate_pt_bos - validate the page table BOs * * @adev: amdgpu device pointer * @vm: vm providing the BOs * @validate: callback to do the validation * @param: parameter for the validation callback * * Validate the page table BOs on command submission if neccessary. */int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm, int (*validate)(void *p, struct amdgpu_bo *bo), void *param){ uint64_t num_evictions; /* We only need to validate the page tables * if they aren't already valid. */ num_evictions = atomic64_read(&adev->num_evictions); if (num_evictions == vm->last_eviction_counter) return 0; return amdgpu_vm_validate_level(&vm->root, validate, param);} /** * amdgpu_vm_move_level_in_lru - move one level of PT BOs to the LRU tail * * @adev: amdgpu device instance * @vm: vm providing the BOs * * Move the PT BOs to the tail of the LRU. */static void amdgpu_vm_move_level_in_lru(struct amdgpu_vm_pt *parent){ unsigned i; if (!parent->entries) return; for (i = 0; i <= parent->last_entry_used; ++i) { struct amdgpu_vm_pt *entry = &parent->entries[i]; if (!entry->bo) continue; ttm_bo_move_to_lru_tail(&entry->bo->tbo); amdgpu_vm_move_level_in_lru(entry); }} /** * amdgpu_vm_move_pt_bos_in_lru - move the PT BOs to the LRU tail * * @adev: amdgpu device instance * @vm: vm providing the BOs * * Move the PT BOs to the tail of the LRU. */void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev, struct amdgpu_vm *vm){ struct ttm_bo_global *glob = adev->mman.bdev.glob; spin_lock(&glob->lru_lock); amdgpu_vm_move_level_in_lru(&vm->root); spin_unlock(&glob->lru_lock);} /** * amdgpu_vm_alloc_levels - allocate the PD/PT levels * * @adev: amdgpu_device pointer * @vm: requested vm * @saddr: start of the address range * @eaddr: end of the address range * * Make sure the page directories and page tables are allocated */static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_vm_pt *parent, uint64_t saddr, uint64_t eaddr, unsigned level){ unsigned shift = (adev->vm_manager.num_level - level) * adev->vm_manager.block_size; unsigned pt_idx, from, to; int r; u64 flags; if (!parent->entries) { unsigned num_entries = amdgpu_vm_num_entries(adev, level); parent->entries = kvmalloc_array(num_entries, sizeof(struct amdgpu_vm_pt), GFP_KERNEL | __GFP_ZERO); if (!parent->entries) return -ENOMEM; memset(parent->entries, 0 , sizeof(struct amdgpu_vm_pt)); } from = saddr >> shift; to = eaddr >> shift; if (from >= amdgpu_vm_num_entries(adev, level) || to >= amdgpu_vm_num_entries(adev, level)) return -EINVAL; if (to > parent->last_entry_used) parent->last_entry_used = to; ++level; saddr = saddr & ((1 << shift) - 1); eaddr = eaddr & ((1 << shift) - 1); flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS | AMDGPU_GEM_CREATE_VRAM_CLEARED; if (vm->use_cpu_for_update) flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; else flags |= (AMDGPU_GEM_CREATE_NO_CPU_ACCESS | AMDGPU_GEM_CREATE_SHADOW); /* walk over the address space and allocate the page tables */ for (pt_idx = from; pt_idx <= to; ++pt_idx) { struct reservation_object *resv = vm->root.bo->tbo.resv; struct amdgpu_vm_pt *entry = &parent->entries[pt_idx]; struct amdgpu_bo *pt; if (!entry->bo) { r = amdgpu_bo_create(adev, amdgpu_vm_bo_size(adev, level), AMDGPU_GPU_PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM, flags, NULL, resv, &pt); if (r) return r; /* Keep a reference to the root directory to avoid * freeing them up in the wrong order. */ pt->parent = amdgpu_bo_ref(vm->root.bo); entry->bo = pt; entry->addr = 0; } if (level < adev->vm_manager.num_level) { uint64_t sub_saddr = (pt_idx == from) ? saddr : 0; uint64_t sub_eaddr = (pt_idx == to) ? eaddr : ((1 << shift) - 1); r = amdgpu_vm_alloc_levels(adev, vm, entry, sub_saddr, sub_eaddr, level); if (r) return r; } } return 0;} /** * amdgpu_vm_alloc_pts - Allocate page tables. * * @adev: amdgpu_device pointer * @vm: VM to allocate page tables for * @saddr: Start address which needs to be allocated * @size: Size from start address we need. * * Make sure the page tables are allocated. */int amdgpu_vm_alloc_pts(struct amdgpu_device *adev, struct amdgpu_vm *vm, uint64_t saddr, uint64_t size){ uint64_t last_pfn; uint64_t eaddr; /* validate the parameters */ if (saddr & AMDGPU_GPU_PAGE_MASK || size & AMDGPU_GPU_PAGE_MASK) return -EINVAL; eaddr = saddr + size - 1; last_pfn = eaddr / AMDGPU_GPU_PAGE_SIZE; if (last_pfn >= adev->vm_manager.max_pfn) { dev_err(adev->dev, "va above limit (0x%08llX >= 0x%08llX)\n", last_pfn, adev->vm_manager.max_pfn); return -EINVAL; } saddr /= AMDGPU_GPU_PAGE_SIZE; eaddr /= AMDGPU_GPU_PAGE_SIZE; return amdgpu_vm_alloc_levels(adev, vm, &vm->root, saddr, eaddr, 0);} /** * amdgpu_vm_had_gpu_reset - check if reset occured since last use * * @adev: amdgpu_device pointer * @id: VMID structure * * Check if GPU reset occured since last use of the VMID. */static bool amdgpu_vm_had_gpu_reset(struct amdgpu_device *adev, struct amdgpu_vm_id *id){ return id->current_gpu_reset_count != atomic_read(&adev->gpu_reset_counter);} static bool amdgpu_vm_reserved_vmid_ready(struct amdgpu_vm *vm, unsigned vmhub){ return !!vm->reserved_vmid[vmhub];} /* idr_mgr->lock must be held */static int amdgpu_vm_grab_reserved_vmid_locked(struct amdgpu_vm *vm, struct amdgpu_ring *ring, struct amdgpu_sync *sync, struct dma_fence *fence, struct amdgpu_job *job){ struct amdgpu_device *adev = ring->adev; unsigned vmhub = ring->funcs->vmhub; uint64_t fence_context = adev->fence_context + ring->idx; struct amdgpu_vm_id *id = vm->reserved_vmid[vmhub]; struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub]; struct dma_fence *updates = sync->last_vm_update; int r = 0; struct dma_fence *flushed, *tmp; bool needs_flush = false; flushed = id->flushed_updates; if ((amdgpu_vm_had_gpu_reset(adev, id)) || (atomic64_read(&id->owner) != vm->client_id) || (job->vm_pd_addr != id->pd_gpu_addr) || (updates && (!flushed || updates->context != flushed->context || dma_fence_is_later(updates, flushed))) || (!id->last_flush || (id->last_flush->context != fence_context && !dma_fence_is_signaled(id->last_flush)))) { needs_flush = true; /* to prevent one context starved by another context */ id->pd_gpu_addr = 0; tmp = amdgpu_sync_peek_fence(&id->active, ring); if (tmp) { r = amdgpu_sync_fence(adev, sync, tmp); return r; } } /* Good we can use this VMID. Remember this submission as * user of the VMID. */ r = amdgpu_sync_fence(ring->adev, &id->active, fence); if (r) goto out; if (updates && (!flushed || updates->context != flushed->context || dma_fence_is_later(updates, flushed))) { dma_fence_put(id->flushed_updates); id->flushed_updates = dma_fence_get(updates); } id->pd_gpu_addr = job->vm_pd_addr; atomic64_set(&id->owner, vm->client_id); job->vm_needs_flush = needs_flush; if (needs_flush) { dma_fence_put(id->last_flush); id->last_flush = NULL; } job->vm_id = id - id_mgr->ids; trace_amdgpu_vm_grab_id(vm, ring, job);out: return r;} /** * amdgpu_vm_grab_id - allocate the next free VMID * * @vm: vm to allocate id for * @ring: ring we want to submit job to * @sync: sync object where we add dependencies * @fence: fence protecting ID from reuse * * Allocate an id for the vm, adding fences to the sync obj as necessary. */int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring, struct amdgpu_sync *sync, struct dma_fence *fence, struct amdgpu_job *job){ struct amdgpu_device *adev = ring->adev; unsigned vmhub = ring->funcs->vmhub; struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub]; uint64_t fence_context = adev->fence_context + ring->idx; struct dma_fence *updates = sync->last_vm_update; struct amdgpu_vm_id *id, *idle; struct dma_fence **fences; unsigned i; int r = 0; mutex_lock(&id_mgr->lock); if (amdgpu_vm_reserved_vmid_ready(vm, vmhub)) { r = amdgpu_vm_grab_reserved_vmid_locked(vm, ring, sync, fence, job); mutex_unlock(&id_mgr->lock); return r;Lines 1–500 of 2643. Load more
/* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. * Copyright 2009 Jerome Glisse. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * * Authors: Dave Airlie * Alex Deucher * Jerome Glisse */ #include <linux/dma-fence-array.h> #include <linux/interval_tree_generic.h> #include <drm/drmP.h> #include <drm/amdgpu_drm.h> #include "amdgpu.h" #include "amdgpu_trace.h" /* * GPUVM * GPUVM is similar to the legacy gart on older asics, however * rather than there being a single global gart table * for the entire GPU, there are multiple VM page tables active * at any given time. The VM page tables can contain a mix * vram pages and system memory pages and system memory pages * can be mapped as snooped (cached system pages) or unsnooped * (uncached system pages). * Each VM has an ID associated with it and there is a page table * associated with each VMID. When execting a command buffer, * the kernel tells the the ring what VMID to use for that command * buffer. VMIDs are allocated dynamically as commands are submitted. * The userspace drivers maintain their own address space and the kernel * sets up their pages tables accordingly when they submit their * command buffers and a VMID is assigned. * Cayman/Trinity support up to 8 active VMs at any given time; * SI supports 16. */ #define START(node) ((node)->start) #define LAST(node) ((node)->last) INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last, START, LAST, static, amdgpu_vm_it) #undef START #undef LAST /* Local structure. Encapsulate some VM table update parameters to reduce * the number of function parameters */ struct amdgpu_pte_update_params { /* amdgpu device we do this update for */ struct amdgpu_device *adev; /* optional amdgpu_vm we do this update for */ struct amdgpu_vm *vm; /* address where to copy page table entries from */ uint64_t src; /* indirect buffer to fill with commands */ struct amdgpu_ib *ib; /* Function which actually does the update */ void (*func)(struct amdgpu_pte_update_params *params, uint64_t pe, uint64_t addr, unsigned count, uint32_t incr, uint64_t flags); /* indicate update pt or its shadow */ bool shadow; /* The next two are used during VM update by CPU * DMA addresses to use for mapping * Kernel pointer of PD/PT BO that needs to be updated */ dma_addr_t *pages_addr; void *kptr; }; /* Helper to disable partial resident texture feature from a fence callback */ struct amdgpu_prt_cb { struct amdgpu_device *adev; struct dma_fence_cb cb; }; /** * amdgpu_vm_num_entries - return the number of entries in a PD/PT * * @adev: amdgpu_device pointer * * Calculate the number of entries in a page directory or page table. */ static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev, unsigned level) { if (level == 0) /* For the root directory */ return adev->vm_manager.max_pfn >> (adev->vm_manager.block_size * adev->vm_manager.num_level); else if (level == adev->vm_manager.num_level) /* For the page tables on the leaves */ return AMDGPU_VM_PTE_COUNT(adev); else /* Everything in between */ return 1 << adev->vm_manager.block_size; } /** * amdgpu_vm_bo_size - returns the size of the BOs in bytes * * @adev: amdgpu_device pointer * * Calculate the size of the BO for a page directory or page table in bytes. */ static unsigned amdgpu_vm_bo_size(struct amdgpu_device *adev, unsigned level) { return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev, level) * 8); } /** * amdgpu_vm_get_pd_bo - add the VM PD to a validation list * * @vm: vm providing the BOs * @validated: head of validation list * @entry: entry to add * * Add the page directory to the list of BOs to * validate for command submission. */ void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm, struct list_head *validated, struct amdgpu_bo_list_entry *entry) { entry->robj = vm->root.bo; entry->priority = 0; entry->tv.bo = &entry->robj->tbo; entry->tv.shared = true; entry->user_pages = NULL; list_add(&entry->tv.head, validated); } /** * amdgpu_vm_validate_layer - validate a single page table level * * @parent: parent page table level * @validate: callback to do the validation * @param: parameter for the validation callback * * Validate the page table BOs on command submission if neccessary. */ static int amdgpu_vm_validate_level(struct amdgpu_vm_pt *parent, int (*validate)(void *, struct amdgpu_bo *), void *param) { unsigned i; int r; if (!parent->entries) return 0; for (i = 0; i <= parent->last_entry_used; ++i) { struct amdgpu_vm_pt *entry = &parent->entries[i]; if (!entry->bo) continue; r = validate(param, entry->bo); if (r) return r; /* * Recurse into the sub directory. This is harmless because we * have only a maximum of 5 layers. */ r = amdgpu_vm_validate_level(entry, validate, param); if (r) return r; } return r; } /** * amdgpu_vm_validate_pt_bos - validate the page table BOs * * @adev: amdgpu device pointer * @vm: vm providing the BOs * @validate: callback to do the validation * @param: parameter for the validation callback * * Validate the page table BOs on command submission if neccessary. */ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm, int (*validate)(void *p, struct amdgpu_bo *bo), void *param) { uint64_t num_evictions; /* We only need to validate the page tables * if they aren't already valid. */ num_evictions = atomic64_read(&adev->num_evictions); if (num_evictions == vm->last_eviction_counter) return 0; return amdgpu_vm_validate_level(&vm->root, validate, param); } /** * amdgpu_vm_move_level_in_lru - move one level of PT BOs to the LRU tail * * @adev: amdgpu device instance * @vm: vm providing the BOs * * Move the PT BOs to the tail of the LRU. */ static void amdgpu_vm_move_level_in_lru(struct amdgpu_vm_pt *parent) { unsigned i; if (!parent->entries) return; for (i = 0; i <= parent->last_entry_used; ++i) { struct amdgpu_vm_pt *entry = &parent->entries[i]; if (!entry->bo) continue; ttm_bo_move_to_lru_tail(&entry->bo->tbo); amdgpu_vm_move_level_in_lru(entry); } } /** * amdgpu_vm_move_pt_bos_in_lru - move the PT BOs to the LRU tail * * @adev: amdgpu device instance * @vm: vm providing the BOs * * Move the PT BOs to the tail of the LRU. */ void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev, struct amdgpu_vm *vm) { struct ttm_bo_global *glob = adev->mman.bdev.glob; spin_lock(&glob->lru_lock); amdgpu_vm_move_level_in_lru(&vm->root); spin_unlock(&glob->lru_lock); } /** * amdgpu_vm_alloc_levels - allocate the PD/PT levels * * @adev: amdgpu_device pointer * @vm: requested vm * @saddr: start of the address range * @eaddr: end of the address range * * Make sure the page directories and page tables are allocated */ static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_vm_pt *parent, uint64_t saddr, uint64_t eaddr, unsigned level) { unsigned shift = (adev->vm_manager.num_level - level) * adev->vm_manager.block_size; unsigned pt_idx, from, to; int r; u64 flags; if (!parent->entries) { unsigned num_entries = amdgpu_vm_num_entries(adev, level); parent->entries = kvmalloc_array(num_entries, sizeof(struct amdgpu_vm_pt), GFP_KERNEL | __GFP_ZERO); if (!parent->entries) return -ENOMEM; memset(parent->entries, 0 , sizeof(struct amdgpu_vm_pt)); } from = saddr >> shift; to = eaddr >> shift; if (from >= amdgpu_vm_num_entries(adev, level) || to >= amdgpu_vm_num_entries(adev, level)) return -EINVAL; if (to > parent->last_entry_used) parent->last_entry_used = to; ++level; saddr = saddr & ((1 << shift) - 1); eaddr = eaddr & ((1 << shift) - 1); flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS | AMDGPU_GEM_CREATE_VRAM_CLEARED; if (vm->use_cpu_for_update) flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; else flags |= (AMDGPU_GEM_CREATE_NO_CPU_ACCESS | AMDGPU_GEM_CREATE_SHADOW); /* walk over the address space and allocate the page tables */ for (pt_idx = from; pt_idx <= to; ++pt_idx) { struct reservation_object *resv = vm->root.bo->tbo.resv; struct amdgpu_vm_pt *entry = &parent->entries[pt_idx]; struct amdgpu_bo *pt; if (!entry->bo) { r = amdgpu_bo_create(adev, amdgpu_vm_bo_size(adev, level), AMDGPU_GPU_PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM, flags, NULL, resv, &pt); if (r) return r; /* Keep a reference to the root directory to avoid * freeing them up in the wrong order. */ pt->parent = amdgpu_bo_ref(vm->root.bo); entry->bo = pt; entry->addr = 0; } if (level < adev->vm_manager.num_level) { uint64_t sub_saddr = (pt_idx == from) ? saddr : 0; uint64_t sub_eaddr = (pt_idx == to) ? eaddr : ((1 << shift) - 1); r = amdgpu_vm_alloc_levels(adev, vm, entry, sub_saddr, sub_eaddr, level); if (r) return r; } } return 0; } /** * amdgpu_vm_alloc_pts - Allocate page tables. * * @adev: amdgpu_device pointer * @vm: VM to allocate page tables for * @saddr: Start address which needs to be allocated * @size: Size from start address we need. * * Make sure the page tables are allocated. */ int amdgpu_vm_alloc_pts(struct amdgpu_device *adev, struct amdgpu_vm *vm, uint64_t saddr, uint64_t size) { uint64_t last_pfn; uint64_t eaddr; /* validate the parameters */ if (saddr & AMDGPU_GPU_PAGE_MASK || size & AMDGPU_GPU_PAGE_MASK) return -EINVAL; eaddr = saddr + size - 1; last_pfn = eaddr / AMDGPU_GPU_PAGE_SIZE; if (last_pfn >= adev->vm_manager.max_pfn) { dev_err(adev->dev, "va above limit (0x%08llX >= 0x%08llX)\n", last_pfn, adev->vm_manager.max_pfn); return -EINVAL; } saddr /= AMDGPU_GPU_PAGE_SIZE; eaddr /= AMDGPU_GPU_PAGE_SIZE; return amdgpu_vm_alloc_levels(adev, vm, &vm->root, saddr, eaddr, 0); } /** * amdgpu_vm_had_gpu_reset - check if reset occured since last use * * @adev: amdgpu_device pointer * @id: VMID structure * * Check if GPU reset occured since last use of the VMID. */ static bool amdgpu_vm_had_gpu_reset(struct amdgpu_device *adev, struct amdgpu_vm_id *id) { return id->current_gpu_reset_count != atomic_read(&adev->gpu_reset_counter); } static bool amdgpu_vm_reserved_vmid_ready(struct amdgpu_vm *vm, unsigned vmhub) { return !!vm->reserved_vmid[vmhub]; } /* idr_mgr->lock must be held */ static int amdgpu_vm_grab_reserved_vmid_locked(struct amdgpu_vm *vm, struct amdgpu_ring *ring, struct amdgpu_sync *sync, struct dma_fence *fence, struct amdgpu_job *job) { struct amdgpu_device *adev = ring->adev; unsigned vmhub = ring->funcs->vmhub; uint64_t fence_context = adev->fence_context + ring->idx; struct amdgpu_vm_id *id = vm->reserved_vmid[vmhub]; struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub]; struct dma_fence *updates = sync->last_vm_update; int r = 0; struct dma_fence *flushed, *tmp; bool needs_flush = false; flushed = id->flushed_updates; if ((amdgpu_vm_had_gpu_reset(adev, id)) || (atomic64_read(&id->owner) != vm->client_id) || (job->vm_pd_addr != id->pd_gpu_addr) || (updates && (!flushed || updates->context != flushed->context || dma_fence_is_later(updates, flushed))) || (!id->last_flush || (id->last_flush->context != fence_context && !dma_fence_is_signaled(id->last_flush)))) { needs_flush = true; /* to prevent one context starved by another context */ id->pd_gpu_addr = 0; tmp = amdgpu_sync_peek_fence(&id->active, ring); if (tmp) { r = amdgpu_sync_fence(adev, sync, tmp); return r; } } /* Good we can use this VMID. Remember this submission as * user of the VMID. */ r = amdgpu_sync_fence(ring->adev, &id->active, fence); if (r) goto out; if (updates && (!flushed || updates->context != flushed->context || dma_fence_is_later(updates, flushed))) { dma_fence_put(id->flushed_updates); id->flushed_updates = dma_fence_get(updates); } id->pd_gpu_addr = job->vm_pd_addr; atomic64_set(&id->owner, vm->client_id); job->vm_needs_flush = needs_flush; if (needs_flush) { dma_fence_put(id->last_flush); id->last_flush = NULL; } job->vm_id = id - id_mgr->ids; trace_amdgpu_vm_grab_id(vm, ring, job); out: return r; } /** * amdgpu_vm_grab_id - allocate the next free VMID * * @vm: vm to allocate id for * @ring: ring we want to submit job to * @sync: sync object where we add dependencies * @fence: fence protecting ID from reuse * * Allocate an id for the vm, adding fences to the sync obj as necessary. */ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring, struct amdgpu_sync *sync, struct dma_fence *fence, struct amdgpu_job *job) { struct amdgpu_device *adev = ring->adev; unsigned vmhub = ring->funcs->vmhub; struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub]; uint64_t fence_context = adev->fence_context + ring->idx; struct dma_fence *updates = sync->last_vm_update; struct amdgpu_vm_id *id, *idle; struct dma_fence **fences; unsigned i; int r = 0; mutex_lock(&id_mgr->lock); if (amdgpu_vm_reserved_vmid_ready(vm, vmhub)) { r = amdgpu_vm_grab_reserved_vmid_locked(vm, ring, sync, fence, job); mutex_unlock(&id_mgr->lock); return r;