/* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. * Copyright 2009 Jerome Glisse. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * * Authors: Dave Airlie * Alex Deucher * Jerome Glisse */ #include <linux/dma-fence-array.h>#include <linux/interval_tree_generic.h>#include <linux/idr.h>#include <linux/dma-buf.h> #include <drm/amdgpu_drm.h>#include <drm/drm_drv.h>#include "amdgpu.h"#include "amdgpu_trace.h"#include "amdgpu_amdkfd.h"#include "amdgpu_gmc.h"#include "amdgpu_xgmi.h"#include "amdgpu_dma_buf.h"#include "amdgpu_res_cursor.h"#include "kfd_svm.h" /** * DOC: GPUVM * * GPUVM is the MMU functionality provided on the GPU. * GPUVM is similar to the legacy GART on older asics, however * rather than there being a single global GART table * for the entire GPU, there can be multiple GPUVM page tables active * at any given time. The GPUVM page tables can contain a mix * VRAM pages and system pages (both memory and MMIO) and system pages * can be mapped as snooped (cached system pages) or unsnooped * (uncached system pages). * * Each active GPUVM has an ID associated with it and there is a page table * linked with each VMID. When executing a command buffer, * the kernel tells the engine what VMID to use for that command * buffer. VMIDs are allocated dynamically as commands are submitted. * The userspace drivers maintain their own address space and the kernel * sets up their pages tables accordingly when they submit their * command buffers and a VMID is assigned. * The hardware supports up to 16 active GPUVMs at any given time. * * Each GPUVM is represented by a 1-2 or 1-5 level page table, depending * on the ASIC family. GPUVM supports RWX attributes on each page as well * as other features such as encryption and caching attributes. * * VMID 0 is special. It is the GPUVM used for the kernel driver. In * addition to an aperture managed by a page table, VMID 0 also has * several other apertures. There is an aperture for direct access to VRAM * and there is a legacy AGP aperture which just forwards accesses directly * to the matching system physical addresses (or IOVAs when an IOMMU is * present). These apertures provide direct access to these memories without * incurring the overhead of a page table. VMID 0 is used by the kernel * driver for tasks like memory management. * * GPU clients (i.e., engines on the GPU) use GPUVM VMIDs to access memory. * For user applications, each application can have their own unique GPUVM * address space. The application manages the address space and the kernel * driver manages the GPUVM page tables for each process. If an GPU client * accesses an invalid page, it will generate a GPU page fault, similar to * accessing an invalid page on a CPU. */ #define START(node) ((node)->start)#define LAST(node) ((node)->last) INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last, START, LAST, static, amdgpu_vm_it) #undef START#undef LAST /** * struct amdgpu_prt_cb - Helper to disable partial resident texture feature from a fence callback */struct amdgpu_prt_cb { /** * @adev: amdgpu device */ struct amdgpu_device *adev; /** * @cb: callback */ struct dma_fence_cb cb;}; /** * struct amdgpu_vm_tlb_seq_cb - Helper to increment the TLB flush sequence */struct amdgpu_vm_tlb_seq_cb { /** * @vm: pointer to the amdgpu_vm structure to set the fence sequence on */ struct amdgpu_vm *vm; /** * @cb: callback */ struct dma_fence_cb cb;}; /** * amdgpu_vm_set_pasid - manage pasid and vm ptr mapping * * @adev: amdgpu_device pointer * @vm: amdgpu_vm pointer * @pasid: the pasid the VM is using on this GPU * * Set the pasid this VM is using on this GPU, can also be used to remove the * pasid by passing in zero. * */int amdgpu_vm_set_pasid(struct amdgpu_device *adev, struct amdgpu_vm *vm, u32 pasid){ int r; if (vm->pasid == pasid) return 0; if (vm->pasid) { r = xa_err(xa_erase_irq(&adev->vm_manager.pasids, vm->pasid)); if (r < 0) return r; vm->pasid = 0; } if (pasid) { r = xa_err(xa_store_irq(&adev->vm_manager.pasids, pasid, vm, GFP_KERNEL)); if (r < 0) return r; vm->pasid = pasid; } return 0;} /** * amdgpu_vm_bo_evicted - vm_bo is evicted * * @vm_bo: vm_bo which is evicted * * State for PDs/PTs and per VM BOs which are not at the location they should * be. */static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo){ struct amdgpu_vm *vm = vm_bo->vm; struct amdgpu_bo *bo = vm_bo->bo; vm_bo->moved = true; spin_lock(&vm_bo->vm->status_lock); if (bo->tbo.type == ttm_bo_type_kernel) list_move(&vm_bo->vm_status, &vm->evicted); else list_move_tail(&vm_bo->vm_status, &vm->evicted); spin_unlock(&vm_bo->vm->status_lock);}/** * amdgpu_vm_bo_moved - vm_bo is moved * * @vm_bo: vm_bo which is moved * * State for per VM BOs which are moved, but that change is not yet reflected * in the page tables. */static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo){ spin_lock(&vm_bo->vm->status_lock); list_move(&vm_bo->vm_status, &vm_bo->vm->moved); spin_unlock(&vm_bo->vm->status_lock);} /** * amdgpu_vm_bo_idle - vm_bo is idle * * @vm_bo: vm_bo which is now idle * * State for PDs/PTs and per VM BOs which have gone through the state machine * and are now idle. */static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo){ spin_lock(&vm_bo->vm->status_lock); list_move(&vm_bo->vm_status, &vm_bo->vm->idle); spin_unlock(&vm_bo->vm->status_lock); vm_bo->moved = false;} /** * amdgpu_vm_bo_invalidated - vm_bo is invalidated * * @vm_bo: vm_bo which is now invalidated * * State for normal BOs which are invalidated and that change not yet reflected * in the PTs. */static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo){ spin_lock(&vm_bo->vm->status_lock); list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated); spin_unlock(&vm_bo->vm->status_lock);} /** * amdgpu_vm_bo_relocated - vm_bo is reloacted * * @vm_bo: vm_bo which is relocated * * State for PDs/PTs which needs to update their parent PD. * For the root PD, just move to idle state. */static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo){ if (vm_bo->bo->parent) { spin_lock(&vm_bo->vm->status_lock); list_move(&vm_bo->vm_status, &vm_bo->vm->relocated); spin_unlock(&vm_bo->vm->status_lock); } else { amdgpu_vm_bo_idle(vm_bo); }} /** * amdgpu_vm_bo_done - vm_bo is done * * @vm_bo: vm_bo which is now done * * State for normal BOs which are invalidated and that change has been updated * in the PTs. */static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo){ spin_lock(&vm_bo->vm->status_lock); list_move(&vm_bo->vm_status, &vm_bo->vm->done); spin_unlock(&vm_bo->vm->status_lock);} /** * amdgpu_vm_bo_base_init - Adds bo to the list of bos associated with the vm * * @base: base structure for tracking BO usage in a VM * @vm: vm to which bo is to be added * @bo: amdgpu buffer object * * Initialize a bo_va_base structure and add it to the appropriate lists * */void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base, struct amdgpu_vm *vm, struct amdgpu_bo *bo){ base->vm = vm; base->bo = bo; base->next = NULL; INIT_LIST_HEAD(&base->vm_status); if (!bo) return; base->next = bo->vm_bo; bo->vm_bo = base; if (bo->tbo.base.resv != vm->root.bo->tbo.base.resv) return; dma_resv_assert_held(vm->root.bo->tbo.base.resv); ttm_bo_set_bulk_move(&bo->tbo, &vm->lru_bulk_move); if (bo->tbo.type == ttm_bo_type_kernel && bo->parent) amdgpu_vm_bo_relocated(base); else amdgpu_vm_bo_idle(base); if (bo->preferred_domains & amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type)) return; /* * we checked all the prerequisites, but it looks like this per vm bo * is currently evicted. add the bo to the evicted list to make sure it * is validated on next vm use to avoid fault. * */ amdgpu_vm_bo_evicted(base);} /** * amdgpu_vm_get_pd_bo - add the VM PD to a validation list * * @vm: vm providing the BOs * @validated: head of validation list * @entry: entry to add * * Add the page directory to the list of BOs to * validate for command submission. */void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm, struct list_head *validated, struct amdgpu_bo_list_entry *entry){ entry->priority = 0; entry->tv.bo = &vm->root.bo->tbo; /* Two for VM updates, one for TTM and one for the CS job */ entry->tv.num_shared = 4; entry->user_pages = NULL; list_add(&entry->tv.head, validated);} /** * amdgpu_vm_move_to_lru_tail - move all BOs to the end of LRU * * @adev: amdgpu device pointer * @vm: vm providing the BOs * * Move all BOs to the end of LRU and remember their positions to put them * together. */void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev, struct amdgpu_vm *vm){ spin_lock(&adev->mman.bdev.lru_lock); ttm_lru_bulk_move_tail(&vm->lru_bulk_move); spin_unlock(&adev->mman.bdev.lru_lock);} /** * amdgpu_vm_validate_pt_bos - validate the page table BOs * * @adev: amdgpu device pointer * @vm: vm providing the BOs * @validate: callback to do the validation * @param: parameter for the validation callback * * Validate the page table BOs on command submission if neccessary. * * Returns: * Validation result. */int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm, int (*validate)(void *p, struct amdgpu_bo *bo), void *param){ struct amdgpu_vm_bo_base *bo_base; struct amdgpu_bo *shadow; struct amdgpu_bo *bo; int r; spin_lock(&vm->status_lock); while (!list_empty(&vm->evicted)) { bo_base = list_first_entry(&vm->evicted, struct amdgpu_vm_bo_base, vm_status); spin_unlock(&vm->status_lock); bo = bo_base->bo; shadow = amdgpu_bo_shadowed(bo); r = validate(param, bo); if (r) return r; if (shadow) { r = validate(param, shadow); if (r) return r; } if (bo->tbo.type != ttm_bo_type_kernel) { amdgpu_vm_bo_moved(bo_base); } else { vm->update_funcs->map_table(to_amdgpu_bo_vm(bo)); amdgpu_vm_bo_relocated(bo_base); } spin_lock(&vm->status_lock); } spin_unlock(&vm->status_lock); amdgpu_vm_eviction_lock(vm); vm->evicting = false; amdgpu_vm_eviction_unlock(vm); return 0;} /** * amdgpu_vm_ready - check VM is ready for updates * * @vm: VM to check * * Check if all VM PDs/PTs are ready for updates * * Returns: * True if VM is not evicting. */bool amdgpu_vm_ready(struct amdgpu_vm *vm){ bool empty; bool ret; amdgpu_vm_eviction_lock(vm); ret = !vm->evicting; amdgpu_vm_eviction_unlock(vm); spin_lock(&vm->status_lock); empty = list_empty(&vm->evicted); spin_unlock(&vm->status_lock); return ret && empty;} /** * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug * * @adev: amdgpu_device pointer */void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev){ const struct amdgpu_ip_block *ip_block; bool has_compute_vm_bug; struct amdgpu_ring *ring; int i; has_compute_vm_bug = false; ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX); if (ip_block) { /* Compute has a VM bug for GFX version < 7. Compute has a VM bug for GFX 8 MEC firmware version < 673.*/ if (ip_block->version->major <= 7) has_compute_vm_bug = true; else if (ip_block->version->major == 8) if (adev->gfx.mec_fw_version < 673) has_compute_vm_bug = true; } for (i = 0; i < adev->num_rings; i++) { ring = adev->rings[i]; if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) /* only compute rings */ ring->has_compute_vm_bug = has_compute_vm_bug; else ring->has_compute_vm_bug = false; }} /** * amdgpu_vm_need_pipeline_sync - Check if pipe sync is needed for job. * * @ring: ring on which the job will be submitted * @job: job to submit * * Returns: * True if sync is needed. */bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring, struct amdgpu_job *job){ struct amdgpu_device *adev = ring->adev; unsigned vmhub = ring->funcs->vmhub; struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; if (job->vmid == 0) return false; if (job->vm_needs_flush || ring->has_compute_vm_bug) return true; if (ring->funcs->emit_gds_switch && job->gds_switch_needed) return true; if (amdgpu_vmid_had_gpu_reset(adev, &id_mgr->ids[job->vmid])) return true; return false;Lines 1–500 of 2625. Load more
/* * Copyright 2008 Advanced Micro Devices, Inc. * Copyright 2008 Red Hat Inc. * Copyright 2009 Jerome Glisse. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * * Authors: Dave Airlie * Alex Deucher * Jerome Glisse */ #include <linux/dma-fence-array.h> #include <linux/interval_tree_generic.h> #include <linux/idr.h> #include <linux/dma-buf.h> #include <drm/amdgpu_drm.h> #include <drm/drm_drv.h> #include "amdgpu.h" #include "amdgpu_trace.h" #include "amdgpu_amdkfd.h" #include "amdgpu_gmc.h" #include "amdgpu_xgmi.h" #include "amdgpu_dma_buf.h" #include "amdgpu_res_cursor.h" #include "kfd_svm.h" /** * DOC: GPUVM * * GPUVM is the MMU functionality provided on the GPU. * GPUVM is similar to the legacy GART on older asics, however * rather than there being a single global GART table * for the entire GPU, there can be multiple GPUVM page tables active * at any given time. The GPUVM page tables can contain a mix * VRAM pages and system pages (both memory and MMIO) and system pages * can be mapped as snooped (cached system pages) or unsnooped * (uncached system pages). * * Each active GPUVM has an ID associated with it and there is a page table * linked with each VMID. When executing a command buffer, * the kernel tells the engine what VMID to use for that command * buffer. VMIDs are allocated dynamically as commands are submitted. * The userspace drivers maintain their own address space and the kernel * sets up their pages tables accordingly when they submit their * command buffers and a VMID is assigned. * The hardware supports up to 16 active GPUVMs at any given time. * * Each GPUVM is represented by a 1-2 or 1-5 level page table, depending * on the ASIC family. GPUVM supports RWX attributes on each page as well * as other features such as encryption and caching attributes. * * VMID 0 is special. It is the GPUVM used for the kernel driver. In * addition to an aperture managed by a page table, VMID 0 also has * several other apertures. There is an aperture for direct access to VRAM * and there is a legacy AGP aperture which just forwards accesses directly * to the matching system physical addresses (or IOVAs when an IOMMU is * present). These apertures provide direct access to these memories without * incurring the overhead of a page table. VMID 0 is used by the kernel * driver for tasks like memory management. * * GPU clients (i.e., engines on the GPU) use GPUVM VMIDs to access memory. * For user applications, each application can have their own unique GPUVM * address space. The application manages the address space and the kernel * driver manages the GPUVM page tables for each process. If an GPU client * accesses an invalid page, it will generate a GPU page fault, similar to * accessing an invalid page on a CPU. */ #define START(node) ((node)->start) #define LAST(node) ((node)->last) INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last, START, LAST, static, amdgpu_vm_it) #undef START #undef LAST /** * struct amdgpu_prt_cb - Helper to disable partial resident texture feature from a fence callback */ struct amdgpu_prt_cb { /** * @adev: amdgpu device */ struct amdgpu_device *adev; /** * @cb: callback */ struct dma_fence_cb cb; }; /** * struct amdgpu_vm_tlb_seq_cb - Helper to increment the TLB flush sequence */ struct amdgpu_vm_tlb_seq_cb { /** * @vm: pointer to the amdgpu_vm structure to set the fence sequence on */ struct amdgpu_vm *vm; /** * @cb: callback */ struct dma_fence_cb cb; }; /** * amdgpu_vm_set_pasid - manage pasid and vm ptr mapping * * @adev: amdgpu_device pointer * @vm: amdgpu_vm pointer * @pasid: the pasid the VM is using on this GPU * * Set the pasid this VM is using on this GPU, can also be used to remove the * pasid by passing in zero. * */ int amdgpu_vm_set_pasid(struct amdgpu_device *adev, struct amdgpu_vm *vm, u32 pasid) { int r; if (vm->pasid == pasid) return 0; if (vm->pasid) { r = xa_err(xa_erase_irq(&adev->vm_manager.pasids, vm->pasid)); if (r < 0) return r; vm->pasid = 0; } if (pasid) { r = xa_err(xa_store_irq(&adev->vm_manager.pasids, pasid, vm, GFP_KERNEL)); if (r < 0) return r; vm->pasid = pasid; } return 0; } /** * amdgpu_vm_bo_evicted - vm_bo is evicted * * @vm_bo: vm_bo which is evicted * * State for PDs/PTs and per VM BOs which are not at the location they should * be. */ static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo) { struct amdgpu_vm *vm = vm_bo->vm; struct amdgpu_bo *bo = vm_bo->bo; vm_bo->moved = true; spin_lock(&vm_bo->vm->status_lock); if (bo->tbo.type == ttm_bo_type_kernel) list_move(&vm_bo->vm_status, &vm->evicted); else list_move_tail(&vm_bo->vm_status, &vm->evicted); spin_unlock(&vm_bo->vm->status_lock); } /** * amdgpu_vm_bo_moved - vm_bo is moved * * @vm_bo: vm_bo which is moved * * State for per VM BOs which are moved, but that change is not yet reflected * in the page tables. */ static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo) { spin_lock(&vm_bo->vm->status_lock); list_move(&vm_bo->vm_status, &vm_bo->vm->moved); spin_unlock(&vm_bo->vm->status_lock); } /** * amdgpu_vm_bo_idle - vm_bo is idle * * @vm_bo: vm_bo which is now idle * * State for PDs/PTs and per VM BOs which have gone through the state machine * and are now idle. */ static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo) { spin_lock(&vm_bo->vm->status_lock); list_move(&vm_bo->vm_status, &vm_bo->vm->idle); spin_unlock(&vm_bo->vm->status_lock); vm_bo->moved = false; } /** * amdgpu_vm_bo_invalidated - vm_bo is invalidated * * @vm_bo: vm_bo which is now invalidated * * State for normal BOs which are invalidated and that change not yet reflected * in the PTs. */ static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo) { spin_lock(&vm_bo->vm->status_lock); list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated); spin_unlock(&vm_bo->vm->status_lock); } /** * amdgpu_vm_bo_relocated - vm_bo is reloacted * * @vm_bo: vm_bo which is relocated * * State for PDs/PTs which needs to update their parent PD. * For the root PD, just move to idle state. */ static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo) { if (vm_bo->bo->parent) { spin_lock(&vm_bo->vm->status_lock); list_move(&vm_bo->vm_status, &vm_bo->vm->relocated); spin_unlock(&vm_bo->vm->status_lock); } else { amdgpu_vm_bo_idle(vm_bo); } } /** * amdgpu_vm_bo_done - vm_bo is done * * @vm_bo: vm_bo which is now done * * State for normal BOs which are invalidated and that change has been updated * in the PTs. */ static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo) { spin_lock(&vm_bo->vm->status_lock); list_move(&vm_bo->vm_status, &vm_bo->vm->done); spin_unlock(&vm_bo->vm->status_lock); } /** * amdgpu_vm_bo_base_init - Adds bo to the list of bos associated with the vm * * @base: base structure for tracking BO usage in a VM * @vm: vm to which bo is to be added * @bo: amdgpu buffer object * * Initialize a bo_va_base structure and add it to the appropriate lists * */ void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base, struct amdgpu_vm *vm, struct amdgpu_bo *bo) { base->vm = vm; base->bo = bo; base->next = NULL; INIT_LIST_HEAD(&base->vm_status); if (!bo) return; base->next = bo->vm_bo; bo->vm_bo = base; if (bo->tbo.base.resv != vm->root.bo->tbo.base.resv) return; dma_resv_assert_held(vm->root.bo->tbo.base.resv); ttm_bo_set_bulk_move(&bo->tbo, &vm->lru_bulk_move); if (bo->tbo.type == ttm_bo_type_kernel && bo->parent) amdgpu_vm_bo_relocated(base); else amdgpu_vm_bo_idle(base); if (bo->preferred_domains & amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type)) return; /* * we checked all the prerequisites, but it looks like this per vm bo * is currently evicted. add the bo to the evicted list to make sure it * is validated on next vm use to avoid fault. * */ amdgpu_vm_bo_evicted(base); } /** * amdgpu_vm_get_pd_bo - add the VM PD to a validation list * * @vm: vm providing the BOs * @validated: head of validation list * @entry: entry to add * * Add the page directory to the list of BOs to * validate for command submission. */ void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm, struct list_head *validated, struct amdgpu_bo_list_entry *entry) { entry->priority = 0; entry->tv.bo = &vm->root.bo->tbo; /* Two for VM updates, one for TTM and one for the CS job */ entry->tv.num_shared = 4; entry->user_pages = NULL; list_add(&entry->tv.head, validated); } /** * amdgpu_vm_move_to_lru_tail - move all BOs to the end of LRU * * @adev: amdgpu device pointer * @vm: vm providing the BOs * * Move all BOs to the end of LRU and remember their positions to put them * together. */ void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev, struct amdgpu_vm *vm) { spin_lock(&adev->mman.bdev.lru_lock); ttm_lru_bulk_move_tail(&vm->lru_bulk_move); spin_unlock(&adev->mman.bdev.lru_lock); } /** * amdgpu_vm_validate_pt_bos - validate the page table BOs * * @adev: amdgpu device pointer * @vm: vm providing the BOs * @validate: callback to do the validation * @param: parameter for the validation callback * * Validate the page table BOs on command submission if neccessary. * * Returns: * Validation result. */ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm, int (*validate)(void *p, struct amdgpu_bo *bo), void *param) { struct amdgpu_vm_bo_base *bo_base; struct amdgpu_bo *shadow; struct amdgpu_bo *bo; int r; spin_lock(&vm->status_lock); while (!list_empty(&vm->evicted)) { bo_base = list_first_entry(&vm->evicted, struct amdgpu_vm_bo_base, vm_status); spin_unlock(&vm->status_lock); bo = bo_base->bo; shadow = amdgpu_bo_shadowed(bo); r = validate(param, bo); if (r) return r; if (shadow) { r = validate(param, shadow); if (r) return r; } if (bo->tbo.type != ttm_bo_type_kernel) { amdgpu_vm_bo_moved(bo_base); } else { vm->update_funcs->map_table(to_amdgpu_bo_vm(bo)); amdgpu_vm_bo_relocated(bo_base); } spin_lock(&vm->status_lock); } spin_unlock(&vm->status_lock); amdgpu_vm_eviction_lock(vm); vm->evicting = false; amdgpu_vm_eviction_unlock(vm); return 0; } /** * amdgpu_vm_ready - check VM is ready for updates * * @vm: VM to check * * Check if all VM PDs/PTs are ready for updates * * Returns: * True if VM is not evicting. */ bool amdgpu_vm_ready(struct amdgpu_vm *vm) { bool empty; bool ret; amdgpu_vm_eviction_lock(vm); ret = !vm->evicting; amdgpu_vm_eviction_unlock(vm); spin_lock(&vm->status_lock); empty = list_empty(&vm->evicted); spin_unlock(&vm->status_lock); return ret && empty; } /** * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug * * @adev: amdgpu_device pointer */ void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev) { const struct amdgpu_ip_block *ip_block; bool has_compute_vm_bug; struct amdgpu_ring *ring; int i; has_compute_vm_bug = false; ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX); if (ip_block) { /* Compute has a VM bug for GFX version < 7. Compute has a VM bug for GFX 8 MEC firmware version < 673.*/ if (ip_block->version->major <= 7) has_compute_vm_bug = true; else if (ip_block->version->major == 8) if (adev->gfx.mec_fw_version < 673) has_compute_vm_bug = true; } for (i = 0; i < adev->num_rings; i++) { ring = adev->rings[i]; if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) /* only compute rings */ ring->has_compute_vm_bug = has_compute_vm_bug; else ring->has_compute_vm_bug = false; } } /** * amdgpu_vm_need_pipeline_sync - Check if pipe sync is needed for job. * * @ring: ring on which the job will be submitted * @job: job to submit * * Returns: * True if sync is needed. */ bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring, struct amdgpu_job *job) { struct amdgpu_device *adev = ring->adev; unsigned vmhub = ring->funcs->vmhub; struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; if (job->vmid == 0) return false; if (job->vm_needs_flush || ring->has_compute_vm_bug) return true; if (ring->funcs->emit_gds_switch && job->gds_switch_needed) return true; if (amdgpu_vmid_had_gpu_reset(adev, &id_mgr->ids[job->vmid])) return true; return false;