// SPDX-License-Identifier: GPL-2.0 OR MIT/* * Copyright 2022 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ #include <drm/drm_drv.h> #include "amdgpu.h"#include "amdgpu_trace.h"#include "amdgpu_vm.h" /* * amdgpu_vm_pt_cursor - state for for_each_amdgpu_vm_pt */struct amdgpu_vm_pt_cursor { uint64_t pfn; struct amdgpu_vm_bo_base *parent; struct amdgpu_vm_bo_base *entry; unsigned int level;}; /** * amdgpu_vm_pt_level_shift - return the addr shift for each level * * @adev: amdgpu_device pointer * @level: VMPT level * * Returns: * The number of bits the pfn needs to be right shifted for a level. */static unsigned int amdgpu_vm_pt_level_shift(struct amdgpu_device *adev, unsigned int level){ switch (level) { case AMDGPU_VM_PDB2: case AMDGPU_VM_PDB1: case AMDGPU_VM_PDB0: return 9 * (AMDGPU_VM_PDB0 - level) + adev->vm_manager.block_size; case AMDGPU_VM_PTB: return 0; default: return ~0; }} /** * amdgpu_vm_pt_num_entries - return the number of entries in a PD/PT * * @adev: amdgpu_device pointer * @level: VMPT level * * Returns: * The number of entries in a page directory or page table. */static unsigned int amdgpu_vm_pt_num_entries(struct amdgpu_device *adev, unsigned int level){ unsigned int shift; shift = amdgpu_vm_pt_level_shift(adev, adev->vm_manager.root_level); if (level == adev->vm_manager.root_level) /* For the root directory */ return round_up(adev->vm_manager.max_pfn, 1ULL << shift) >> shift; else if (level != AMDGPU_VM_PTB) /* Everything in between */ return 512; /* For the page tables on the leaves */ return AMDGPU_VM_PTE_COUNT(adev);} /** * amdgpu_vm_pt_entries_mask - the mask to get the entry number of a PD/PT * * @adev: amdgpu_device pointer * @level: VMPT level * * Returns: * The mask to extract the entry number of a PD/PT from an address. */static uint32_t amdgpu_vm_pt_entries_mask(struct amdgpu_device *adev, unsigned int level){ if (level <= adev->vm_manager.root_level) return 0xffffffff; else if (level != AMDGPU_VM_PTB) return 0x1ff; else return AMDGPU_VM_PTE_COUNT(adev) - 1;} /** * amdgpu_vm_pt_size - returns the size of the page table in bytes * * @adev: amdgpu_device pointer * @level: VMPT level * * Returns: * The size of the BO for a page directory or page table in bytes. */static unsigned int amdgpu_vm_pt_size(struct amdgpu_device *adev, unsigned int level){ return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_pt_num_entries(adev, level) * 8);} /** * amdgpu_vm_pt_parent - get the parent page directory * * @pt: child page table * * Helper to get the parent entry for the child page table. NULL if we are at * the root page directory. */static struct amdgpu_vm_bo_base *amdgpu_vm_pt_parent(struct amdgpu_vm_bo_base *pt){ struct amdgpu_bo *parent = pt->bo->parent; if (!parent) return NULL; return parent->vm_bo;} /** * amdgpu_vm_pt_start - start PD/PT walk * * @adev: amdgpu_device pointer * @vm: amdgpu_vm structure * @start: start address of the walk * @cursor: state to initialize * * Initialize a amdgpu_vm_pt_cursor to start a walk. */static void amdgpu_vm_pt_start(struct amdgpu_device *adev, struct amdgpu_vm *vm, uint64_t start, struct amdgpu_vm_pt_cursor *cursor){ cursor->pfn = start; cursor->parent = NULL; cursor->entry = &vm->root; cursor->level = adev->vm_manager.root_level;} /** * amdgpu_vm_pt_descendant - go to child node * * @adev: amdgpu_device pointer * @cursor: current state * * Walk to the child node of the current node. * Returns: * True if the walk was possible, false otherwise. */static bool amdgpu_vm_pt_descendant(struct amdgpu_device *adev, struct amdgpu_vm_pt_cursor *cursor){ unsigned int mask, shift, idx; if ((cursor->level == AMDGPU_VM_PTB) || !cursor->entry || !cursor->entry->bo) return false; mask = amdgpu_vm_pt_entries_mask(adev, cursor->level); shift = amdgpu_vm_pt_level_shift(adev, cursor->level); ++cursor->level; idx = (cursor->pfn >> shift) & mask; cursor->parent = cursor->entry; cursor->entry = &to_amdgpu_bo_vm(cursor->entry->bo)->entries[idx]; return true;} /** * amdgpu_vm_pt_sibling - go to sibling node * * @adev: amdgpu_device pointer * @cursor: current state * * Walk to the sibling node of the current node. * Returns: * True if the walk was possible, false otherwise. */static bool amdgpu_vm_pt_sibling(struct amdgpu_device *adev, struct amdgpu_vm_pt_cursor *cursor){ unsigned int shift, num_entries; struct amdgpu_bo_vm *parent; /* Root doesn't have a sibling */ if (!cursor->parent) return false; /* Go to our parents and see if we got a sibling */ shift = amdgpu_vm_pt_level_shift(adev, cursor->level - 1); num_entries = amdgpu_vm_pt_num_entries(adev, cursor->level - 1); parent = to_amdgpu_bo_vm(cursor->parent->bo); if (cursor->entry == &parent->entries[num_entries - 1]) return false; cursor->pfn += 1ULL << shift; cursor->pfn &= ~((1ULL << shift) - 1); ++cursor->entry; return true;} /** * amdgpu_vm_pt_ancestor - go to parent node * * @cursor: current state * * Walk to the parent node of the current node. * Returns: * True if the walk was possible, false otherwise. */static bool amdgpu_vm_pt_ancestor(struct amdgpu_vm_pt_cursor *cursor){ if (!cursor->parent) return false; --cursor->level; cursor->entry = cursor->parent; cursor->parent = amdgpu_vm_pt_parent(cursor->parent); return true;} /** * amdgpu_vm_pt_next - get next PD/PT in hieratchy * * @adev: amdgpu_device pointer * @cursor: current state * * Walk the PD/PT tree to the next node. */static void amdgpu_vm_pt_next(struct amdgpu_device *adev, struct amdgpu_vm_pt_cursor *cursor){ /* First try a newborn child */ if (amdgpu_vm_pt_descendant(adev, cursor)) return; /* If that didn't worked try to find a sibling */ while (!amdgpu_vm_pt_sibling(adev, cursor)) { /* No sibling, go to our parents and grandparents */ if (!amdgpu_vm_pt_ancestor(cursor)) { cursor->pfn = ~0ll; return; } }} /** * amdgpu_vm_pt_first_dfs - start a deep first search * * @adev: amdgpu_device structure * @vm: amdgpu_vm structure * @start: optional cursor to start with * @cursor: state to initialize * * Starts a deep first traversal of the PD/PT tree. */static void amdgpu_vm_pt_first_dfs(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_vm_pt_cursor *start, struct amdgpu_vm_pt_cursor *cursor){ if (start) *cursor = *start; else amdgpu_vm_pt_start(adev, vm, 0, cursor); while (amdgpu_vm_pt_descendant(adev, cursor)) ;} /** * amdgpu_vm_pt_continue_dfs - check if the deep first search should continue * * @start: starting point for the search * @entry: current entry * * Returns: * True when the search should continue, false otherwise. */static bool amdgpu_vm_pt_continue_dfs(struct amdgpu_vm_pt_cursor *start, struct amdgpu_vm_bo_base *entry){ return entry && (!start || entry != start->entry);} /** * amdgpu_vm_pt_next_dfs - get the next node for a deep first search * * @adev: amdgpu_device structure * @cursor: current state * * Move the cursor to the next node in a deep first search. */static void amdgpu_vm_pt_next_dfs(struct amdgpu_device *adev, struct amdgpu_vm_pt_cursor *cursor){ if (!cursor->entry) return; if (!cursor->parent) cursor->entry = NULL; else if (amdgpu_vm_pt_sibling(adev, cursor)) while (amdgpu_vm_pt_descendant(adev, cursor)) ; else amdgpu_vm_pt_ancestor(cursor);} /* * for_each_amdgpu_vm_pt_dfs_safe - safe deep first search of all PDs/PTs */#define for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry) \ for (amdgpu_vm_pt_first_dfs((adev), (vm), (start), &(cursor)), \ (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor));\ amdgpu_vm_pt_continue_dfs((start), (entry)); \ (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor))) /** * amdgpu_vm_pt_clear - initially clear the PDs/PTs * * @adev: amdgpu_device pointer * @vm: VM to clear BO from * @vmbo: BO to clear * @immediate: use an immediate update * * Root PD needs to be reserved when calling this. * * Returns: * 0 on success, errno otherwise. */int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_bo_vm *vmbo, bool immediate){ unsigned int level = adev->vm_manager.root_level; struct ttm_operation_ctx ctx = { true, false }; struct amdgpu_vm_update_params params; struct amdgpu_bo *ancestor = &vmbo->bo; unsigned int entries; struct amdgpu_bo *bo = &vmbo->bo; uint64_t addr; int r, idx; /* Figure out our place in the hierarchy */ if (ancestor->parent) { ++level; while (ancestor->parent->parent) { ++level; ancestor = ancestor->parent; } } entries = amdgpu_bo_size(bo) / 8; r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); if (r) return r; if (!drm_dev_enter(adev_to_drm(adev), &idx)) return -ENODEV; r = vm->update_funcs->map_table(vmbo); if (r) goto exit; memset(¶ms, 0, sizeof(params)); params.adev = adev; params.vm = vm; params.immediate = immediate; r = vm->update_funcs->prepare(¶ms, NULL); if (r) goto exit; addr = 0; uint64_t value = 0, flags = 0; if (adev->asic_type >= CHIP_VEGA10) { if (level != AMDGPU_VM_PTB) { /* Handle leaf PDEs as PTEs */ flags |= AMDGPU_PDE_PTE_FLAG(adev); amdgpu_gmc_get_vm_pde(adev, level, &value, &flags); } else { /* Workaround for fault priority problem on GMC9 */ flags = AMDGPU_PTE_EXECUTABLE; } } r = vm->update_funcs->update(¶ms, vmbo, addr, 0, entries, value, flags); if (r) goto exit; r = vm->update_funcs->commit(¶ms, NULL);exit: drm_dev_exit(idx); return r;} /** * amdgpu_vm_pt_create - create bo for PD/PT * * @adev: amdgpu_device pointer * @vm: requesting vm * @level: the page table level * @immediate: use a immediate update * @vmbo: pointer to the buffer object pointer * @xcp_id: GPU partition id */int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm, int level, bool immediate, struct amdgpu_bo_vm **vmbo, int32_t xcp_id){ struct amdgpu_bo_param bp; unsigned int num_entries; memset(&bp, 0, sizeof(bp)); bp.size = amdgpu_vm_pt_size(adev, level); bp.byte_align = AMDGPU_GPU_PAGE_SIZE; if (!adev->gmc.is_app_apu) bp.domain = AMDGPU_GEM_DOMAIN_VRAM; else bp.domain = AMDGPU_GEM_DOMAIN_GTT; bp.domain = amdgpu_bo_get_preferred_domain(adev, bp.domain); bp.flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS | AMDGPU_GEM_CREATE_CPU_GTT_USWC; if (level < AMDGPU_VM_PTB) num_entries = amdgpu_vm_pt_num_entries(adev, level); else num_entries = 0; bp.bo_ptr_size = struct_size((*vmbo), entries, num_entries); if (vm->use_cpu_for_update) bp.flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; bp.type = ttm_bo_type_kernel; bp.no_wait_gpu = immediate; bp.xcp_id_plus1 = xcp_id + 1; if (vm->root.bo) bp.resv = vm->root.bo->tbo.base.resv; return amdgpu_bo_create_vm(adev, &bp, vmbo);} /** * amdgpu_vm_pt_alloc - Allocate a specific page table * * @adev: amdgpu_device pointer * @vm: VM to allocate page tables for * @cursor: Which page table to allocate * @immediate: use an immediate update * * Make sure a specific page table or directory is allocated. * * Returns: * 1 if page table needed to be allocated, 0 if page table was already * allocated, negative errno if an error occurred. */static int amdgpu_vm_pt_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_vm_pt_cursor *cursor, bool immediate){ struct amdgpu_vm_bo_base *entry = cursor->entry; struct amdgpu_bo *pt_bo; struct amdgpu_bo_vm *pt;Lines 1–500 of 990. Load more
// SPDX-License-Identifier: GPL-2.0 OR MIT /* * Copyright 2022 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ #include <drm/drm_drv.h> #include "amdgpu.h" #include "amdgpu_trace.h" #include "amdgpu_vm.h" /* * amdgpu_vm_pt_cursor - state for for_each_amdgpu_vm_pt */ struct amdgpu_vm_pt_cursor { uint64_t pfn; struct amdgpu_vm_bo_base *parent; struct amdgpu_vm_bo_base *entry; unsigned int level; }; /** * amdgpu_vm_pt_level_shift - return the addr shift for each level * * @adev: amdgpu_device pointer * @level: VMPT level * * Returns: * The number of bits the pfn needs to be right shifted for a level. */ static unsigned int amdgpu_vm_pt_level_shift(struct amdgpu_device *adev, unsigned int level) { switch (level) { case AMDGPU_VM_PDB2: case AMDGPU_VM_PDB1: case AMDGPU_VM_PDB0: return 9 * (AMDGPU_VM_PDB0 - level) + adev->vm_manager.block_size; case AMDGPU_VM_PTB: return 0; default: return ~0; } } /** * amdgpu_vm_pt_num_entries - return the number of entries in a PD/PT * * @adev: amdgpu_device pointer * @level: VMPT level * * Returns: * The number of entries in a page directory or page table. */ static unsigned int amdgpu_vm_pt_num_entries(struct amdgpu_device *adev, unsigned int level) { unsigned int shift; shift = amdgpu_vm_pt_level_shift(adev, adev->vm_manager.root_level); if (level == adev->vm_manager.root_level) /* For the root directory */ return round_up(adev->vm_manager.max_pfn, 1ULL << shift) >> shift; else if (level != AMDGPU_VM_PTB) /* Everything in between */ return 512; /* For the page tables on the leaves */ return AMDGPU_VM_PTE_COUNT(adev); } /** * amdgpu_vm_pt_entries_mask - the mask to get the entry number of a PD/PT * * @adev: amdgpu_device pointer * @level: VMPT level * * Returns: * The mask to extract the entry number of a PD/PT from an address. */ static uint32_t amdgpu_vm_pt_entries_mask(struct amdgpu_device *adev, unsigned int level) { if (level <= adev->vm_manager.root_level) return 0xffffffff; else if (level != AMDGPU_VM_PTB) return 0x1ff; else return AMDGPU_VM_PTE_COUNT(adev) - 1; } /** * amdgpu_vm_pt_size - returns the size of the page table in bytes * * @adev: amdgpu_device pointer * @level: VMPT level * * Returns: * The size of the BO for a page directory or page table in bytes. */ static unsigned int amdgpu_vm_pt_size(struct amdgpu_device *adev, unsigned int level) { return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_pt_num_entries(adev, level) * 8); } /** * amdgpu_vm_pt_parent - get the parent page directory * * @pt: child page table * * Helper to get the parent entry for the child page table. NULL if we are at * the root page directory. */ static struct amdgpu_vm_bo_base * amdgpu_vm_pt_parent(struct amdgpu_vm_bo_base *pt) { struct amdgpu_bo *parent = pt->bo->parent; if (!parent) return NULL; return parent->vm_bo; } /** * amdgpu_vm_pt_start - start PD/PT walk * * @adev: amdgpu_device pointer * @vm: amdgpu_vm structure * @start: start address of the walk * @cursor: state to initialize * * Initialize a amdgpu_vm_pt_cursor to start a walk. */ static void amdgpu_vm_pt_start(struct amdgpu_device *adev, struct amdgpu_vm *vm, uint64_t start, struct amdgpu_vm_pt_cursor *cursor) { cursor->pfn = start; cursor->parent = NULL; cursor->entry = &vm->root; cursor->level = adev->vm_manager.root_level; } /** * amdgpu_vm_pt_descendant - go to child node * * @adev: amdgpu_device pointer * @cursor: current state * * Walk to the child node of the current node. * Returns: * True if the walk was possible, false otherwise. */ static bool amdgpu_vm_pt_descendant(struct amdgpu_device *adev, struct amdgpu_vm_pt_cursor *cursor) { unsigned int mask, shift, idx; if ((cursor->level == AMDGPU_VM_PTB) || !cursor->entry || !cursor->entry->bo) return false; mask = amdgpu_vm_pt_entries_mask(adev, cursor->level); shift = amdgpu_vm_pt_level_shift(adev, cursor->level); ++cursor->level; idx = (cursor->pfn >> shift) & mask; cursor->parent = cursor->entry; cursor->entry = &to_amdgpu_bo_vm(cursor->entry->bo)->entries[idx]; return true; } /** * amdgpu_vm_pt_sibling - go to sibling node * * @adev: amdgpu_device pointer * @cursor: current state * * Walk to the sibling node of the current node. * Returns: * True if the walk was possible, false otherwise. */ static bool amdgpu_vm_pt_sibling(struct amdgpu_device *adev, struct amdgpu_vm_pt_cursor *cursor) { unsigned int shift, num_entries; struct amdgpu_bo_vm *parent; /* Root doesn't have a sibling */ if (!cursor->parent) return false; /* Go to our parents and see if we got a sibling */ shift = amdgpu_vm_pt_level_shift(adev, cursor->level - 1); num_entries = amdgpu_vm_pt_num_entries(adev, cursor->level - 1); parent = to_amdgpu_bo_vm(cursor->parent->bo); if (cursor->entry == &parent->entries[num_entries - 1]) return false; cursor->pfn += 1ULL << shift; cursor->pfn &= ~((1ULL << shift) - 1); ++cursor->entry; return true; } /** * amdgpu_vm_pt_ancestor - go to parent node * * @cursor: current state * * Walk to the parent node of the current node. * Returns: * True if the walk was possible, false otherwise. */ static bool amdgpu_vm_pt_ancestor(struct amdgpu_vm_pt_cursor *cursor) { if (!cursor->parent) return false; --cursor->level; cursor->entry = cursor->parent; cursor->parent = amdgpu_vm_pt_parent(cursor->parent); return true; } /** * amdgpu_vm_pt_next - get next PD/PT in hieratchy * * @adev: amdgpu_device pointer * @cursor: current state * * Walk the PD/PT tree to the next node. */ static void amdgpu_vm_pt_next(struct amdgpu_device *adev, struct amdgpu_vm_pt_cursor *cursor) { /* First try a newborn child */ if (amdgpu_vm_pt_descendant(adev, cursor)) return; /* If that didn't worked try to find a sibling */ while (!amdgpu_vm_pt_sibling(adev, cursor)) { /* No sibling, go to our parents and grandparents */ if (!amdgpu_vm_pt_ancestor(cursor)) { cursor->pfn = ~0ll; return; } } } /** * amdgpu_vm_pt_first_dfs - start a deep first search * * @adev: amdgpu_device structure * @vm: amdgpu_vm structure * @start: optional cursor to start with * @cursor: state to initialize * * Starts a deep first traversal of the PD/PT tree. */ static void amdgpu_vm_pt_first_dfs(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_vm_pt_cursor *start, struct amdgpu_vm_pt_cursor *cursor) { if (start) *cursor = *start; else amdgpu_vm_pt_start(adev, vm, 0, cursor); while (amdgpu_vm_pt_descendant(adev, cursor)) ; } /** * amdgpu_vm_pt_continue_dfs - check if the deep first search should continue * * @start: starting point for the search * @entry: current entry * * Returns: * True when the search should continue, false otherwise. */ static bool amdgpu_vm_pt_continue_dfs(struct amdgpu_vm_pt_cursor *start, struct amdgpu_vm_bo_base *entry) { return entry && (!start || entry != start->entry); } /** * amdgpu_vm_pt_next_dfs - get the next node for a deep first search * * @adev: amdgpu_device structure * @cursor: current state * * Move the cursor to the next node in a deep first search. */ static void amdgpu_vm_pt_next_dfs(struct amdgpu_device *adev, struct amdgpu_vm_pt_cursor *cursor) { if (!cursor->entry) return; if (!cursor->parent) cursor->entry = NULL; else if (amdgpu_vm_pt_sibling(adev, cursor)) while (amdgpu_vm_pt_descendant(adev, cursor)) ; else amdgpu_vm_pt_ancestor(cursor); } /* * for_each_amdgpu_vm_pt_dfs_safe - safe deep first search of all PDs/PTs */ #define for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry) \ for (amdgpu_vm_pt_first_dfs((adev), (vm), (start), &(cursor)), \ (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor));\ amdgpu_vm_pt_continue_dfs((start), (entry)); \ (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor))) /** * amdgpu_vm_pt_clear - initially clear the PDs/PTs * * @adev: amdgpu_device pointer * @vm: VM to clear BO from * @vmbo: BO to clear * @immediate: use an immediate update * * Root PD needs to be reserved when calling this. * * Returns: * 0 on success, errno otherwise. */ int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_bo_vm *vmbo, bool immediate) { unsigned int level = adev->vm_manager.root_level; struct ttm_operation_ctx ctx = { true, false }; struct amdgpu_vm_update_params params; struct amdgpu_bo *ancestor = &vmbo->bo; unsigned int entries; struct amdgpu_bo *bo = &vmbo->bo; uint64_t addr; int r, idx; /* Figure out our place in the hierarchy */ if (ancestor->parent) { ++level; while (ancestor->parent->parent) { ++level; ancestor = ancestor->parent; } } entries = amdgpu_bo_size(bo) / 8; r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); if (r) return r; if (!drm_dev_enter(adev_to_drm(adev), &idx)) return -ENODEV; r = vm->update_funcs->map_table(vmbo); if (r) goto exit; memset(¶ms, 0, sizeof(params)); params.adev = adev; params.vm = vm; params.immediate = immediate; r = vm->update_funcs->prepare(¶ms, NULL); if (r) goto exit; addr = 0; uint64_t value = 0, flags = 0; if (adev->asic_type >= CHIP_VEGA10) { if (level != AMDGPU_VM_PTB) { /* Handle leaf PDEs as PTEs */ flags |= AMDGPU_PDE_PTE_FLAG(adev); amdgpu_gmc_get_vm_pde(adev, level, &value, &flags); } else { /* Workaround for fault priority problem on GMC9 */ flags = AMDGPU_PTE_EXECUTABLE; } } r = vm->update_funcs->update(¶ms, vmbo, addr, 0, entries, value, flags); if (r) goto exit; r = vm->update_funcs->commit(¶ms, NULL); exit: drm_dev_exit(idx); return r; } /** * amdgpu_vm_pt_create - create bo for PD/PT * * @adev: amdgpu_device pointer * @vm: requesting vm * @level: the page table level * @immediate: use a immediate update * @vmbo: pointer to the buffer object pointer * @xcp_id: GPU partition id */ int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm, int level, bool immediate, struct amdgpu_bo_vm **vmbo, int32_t xcp_id) { struct amdgpu_bo_param bp; unsigned int num_entries; memset(&bp, 0, sizeof(bp)); bp.size = amdgpu_vm_pt_size(adev, level); bp.byte_align = AMDGPU_GPU_PAGE_SIZE; if (!adev->gmc.is_app_apu) bp.domain = AMDGPU_GEM_DOMAIN_VRAM; else bp.domain = AMDGPU_GEM_DOMAIN_GTT; bp.domain = amdgpu_bo_get_preferred_domain(adev, bp.domain); bp.flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS | AMDGPU_GEM_CREATE_CPU_GTT_USWC; if (level < AMDGPU_VM_PTB) num_entries = amdgpu_vm_pt_num_entries(adev, level); else num_entries = 0; bp.bo_ptr_size = struct_size((*vmbo), entries, num_entries); if (vm->use_cpu_for_update) bp.flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; bp.type = ttm_bo_type_kernel; bp.no_wait_gpu = immediate; bp.xcp_id_plus1 = xcp_id + 1; if (vm->root.bo) bp.resv = vm->root.bo->tbo.base.resv; return amdgpu_bo_create_vm(adev, &bp, vmbo); } /** * amdgpu_vm_pt_alloc - Allocate a specific page table * * @adev: amdgpu_device pointer * @vm: VM to allocate page tables for * @cursor: Which page table to allocate * @immediate: use an immediate update * * Make sure a specific page table or directory is allocated. * * Returns: * 1 if page table needed to be allocated, 0 if page table was already * allocated, negative errno if an error occurred. */ static int amdgpu_vm_pt_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_vm_pt_cursor *cursor, bool immediate) { struct amdgpu_vm_bo_base *entry = cursor->entry; struct amdgpu_bo *pt_bo; struct amdgpu_bo_vm *pt;