/* * Copyright 2009 Jerome Glisse. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE * USE OR OTHER DEALINGS IN THE SOFTWARE. * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * *//* * Authors: * Jerome Glisse <glisse@freedesktop.org> * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> * Dave Airlie */#include <linux/list.h>#include <linux/slab.h>#include <drm/drmP.h>#include <drm/amdgpu_drm.h>#include <drm/drm_cache.h>#include "amdgpu.h"#include "amdgpu_trace.h" static u64 amdgpu_get_vis_part_size(struct amdgpu_device *adev, struct ttm_mem_reg *mem){ if (mem->start << PAGE_SHIFT >= adev->mc.visible_vram_size) return 0; return ((mem->start << PAGE_SHIFT) + mem->size) > adev->mc.visible_vram_size ? adev->mc.visible_vram_size - (mem->start << PAGE_SHIFT) : mem->size;} static void amdgpu_update_memory_usage(struct amdgpu_device *adev, struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem){ u64 vis_size; if (!adev) return; if (new_mem) { switch (new_mem->mem_type) { case TTM_PL_TT: atomic64_add(new_mem->size, &adev->gtt_usage); break; case TTM_PL_VRAM: atomic64_add(new_mem->size, &adev->vram_usage); vis_size = amdgpu_get_vis_part_size(adev, new_mem); atomic64_add(vis_size, &adev->vram_vis_usage); break; } } if (old_mem) { switch (old_mem->mem_type) { case TTM_PL_TT: atomic64_sub(old_mem->size, &adev->gtt_usage); break; case TTM_PL_VRAM: atomic64_sub(old_mem->size, &adev->vram_usage); vis_size = amdgpu_get_vis_part_size(adev, old_mem); atomic64_sub(vis_size, &adev->vram_vis_usage); break; } }} static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo){ struct amdgpu_bo *bo; bo = container_of(tbo, struct amdgpu_bo, tbo); amdgpu_update_memory_usage(bo->adev, &bo->tbo.mem, NULL); drm_gem_object_release(&bo->gem_base); amdgpu_bo_unref(&bo->parent); if (!list_empty(&bo->shadow_list)) { mutex_lock(&bo->adev->shadow_list_lock); list_del_init(&bo->shadow_list); mutex_unlock(&bo->adev->shadow_list_lock); } kfree(bo->metadata); kfree(bo);} bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo){ if (bo->destroy == &amdgpu_ttm_bo_destroy) return true; return false;} static void amdgpu_ttm_placement_init(struct amdgpu_device *adev, struct ttm_placement *placement, struct ttm_place *places, u32 domain, u64 flags){ u32 c = 0; if (domain & AMDGPU_GEM_DOMAIN_VRAM) { unsigned visible_pfn = adev->mc.visible_vram_size >> PAGE_SHIFT; if (flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS && !(flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) && adev->mc.visible_vram_size < adev->mc.real_vram_size) { places[c].fpfn = visible_pfn; places[c].lpfn = 0; places[c].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_VRAM | TTM_PL_FLAG_TOPDOWN; c++; } places[c].fpfn = 0; places[c].lpfn = 0; places[c].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_VRAM; if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) places[c].lpfn = visible_pfn; else places[c].flags |= TTM_PL_FLAG_TOPDOWN; c++; } if (domain & AMDGPU_GEM_DOMAIN_GTT) { places[c].fpfn = 0; places[c].lpfn = 0; places[c].flags = TTM_PL_FLAG_TT; if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) places[c].flags |= TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED; else places[c].flags |= TTM_PL_FLAG_CACHED; c++; } if (domain & AMDGPU_GEM_DOMAIN_CPU) { places[c].fpfn = 0; places[c].lpfn = 0; places[c].flags = TTM_PL_FLAG_SYSTEM; if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) places[c].flags |= TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED; else places[c].flags |= TTM_PL_FLAG_CACHED; c++; } if (domain & AMDGPU_GEM_DOMAIN_GDS) { places[c].fpfn = 0; places[c].lpfn = 0; places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GDS; c++; } if (domain & AMDGPU_GEM_DOMAIN_GWS) { places[c].fpfn = 0; places[c].lpfn = 0; places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GWS; c++; } if (domain & AMDGPU_GEM_DOMAIN_OA) { places[c].fpfn = 0; places[c].lpfn = 0; places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_OA; c++; } if (!c) { places[c].fpfn = 0; places[c].lpfn = 0; places[c].flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM; c++; } placement->num_placement = c; placement->placement = places; placement->num_busy_placement = c; placement->busy_placement = places;} void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain){ amdgpu_ttm_placement_init(abo->adev, &abo->placement, abo->placements, domain, abo->flags);} static void amdgpu_fill_placement_to_bo(struct amdgpu_bo *bo, struct ttm_placement *placement){ BUG_ON(placement->num_placement > (AMDGPU_GEM_DOMAIN_MAX + 1)); memcpy(bo->placements, placement->placement, placement->num_placement * sizeof(struct ttm_place)); bo->placement.num_placement = placement->num_placement; bo->placement.num_busy_placement = placement->num_busy_placement; bo->placement.placement = bo->placements; bo->placement.busy_placement = bo->placements;} /** * amdgpu_bo_create_kernel - create BO for kernel use * * @adev: amdgpu device object * @size: size for the new BO * @align: alignment for the new BO * @domain: where to place it * @bo_ptr: resulting BO * @gpu_addr: GPU addr of the pinned BO * @cpu_addr: optional CPU address mapping * * Allocates and pins a BO for kernel internal use. * * Returns 0 on success, negative error code otherwise. */int amdgpu_bo_create_kernel(struct amdgpu_device *adev, unsigned long size, int align, u32 domain, struct amdgpu_bo **bo_ptr, u64 *gpu_addr, void **cpu_addr){ int r; r = amdgpu_bo_create(adev, size, align, true, domain, AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED, NULL, NULL, bo_ptr); if (r) { dev_err(adev->dev, "(%d) failed to allocate kernel bo\n", r); return r; } r = amdgpu_bo_reserve(*bo_ptr, false); if (r) { dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r); goto error_free; } r = amdgpu_bo_pin(*bo_ptr, domain, gpu_addr); if (r) { dev_err(adev->dev, "(%d) kernel bo pin failed\n", r); goto error_unreserve; } if (cpu_addr) { r = amdgpu_bo_kmap(*bo_ptr, cpu_addr); if (r) { dev_err(adev->dev, "(%d) kernel bo map failed\n", r); goto error_unreserve; } } amdgpu_bo_unreserve(*bo_ptr); return 0; error_unreserve: amdgpu_bo_unreserve(*bo_ptr); error_free: amdgpu_bo_unref(bo_ptr); return r;} /** * amdgpu_bo_free_kernel - free BO for kernel use * * @bo: amdgpu BO to free * * unmaps and unpin a BO for kernel internal use. */void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr, void **cpu_addr){ if (*bo == NULL) return; if (likely(amdgpu_bo_reserve(*bo, false) == 0)) { if (cpu_addr) amdgpu_bo_kunmap(*bo); amdgpu_bo_unpin(*bo); amdgpu_bo_unreserve(*bo); } amdgpu_bo_unref(bo); if (gpu_addr) *gpu_addr = 0; if (cpu_addr) *cpu_addr = NULL;} int amdgpu_bo_create_restricted(struct amdgpu_device *adev, unsigned long size, int byte_align, bool kernel, u32 domain, u64 flags, struct sg_table *sg, struct ttm_placement *placement, struct reservation_object *resv, struct amdgpu_bo **bo_ptr){ struct amdgpu_bo *bo; enum ttm_bo_type type; unsigned long page_align; size_t acc_size; int r; page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT; size = ALIGN(size, PAGE_SIZE); if (kernel) { type = ttm_bo_type_kernel; } else if (sg) { type = ttm_bo_type_sg; } else { type = ttm_bo_type_device; } *bo_ptr = NULL; acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size, sizeof(struct amdgpu_bo)); bo = kzalloc(sizeof(struct amdgpu_bo), GFP_KERNEL); if (bo == NULL) return -ENOMEM; r = drm_gem_object_init(adev->ddev, &bo->gem_base, size); if (unlikely(r)) { kfree(bo); return r; } bo->adev = adev; INIT_LIST_HEAD(&bo->shadow_list); INIT_LIST_HEAD(&bo->va); bo->prefered_domains = domain & (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT | AMDGPU_GEM_DOMAIN_CPU | AMDGPU_GEM_DOMAIN_GDS | AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA); bo->allowed_domains = bo->prefered_domains; if (!kernel && bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM) bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT; bo->flags = flags; /* For architectures that don't support WC memory, * mask out the WC flag from the BO */ if (!drm_arch_can_wc_memory()) bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; amdgpu_fill_placement_to_bo(bo, placement); /* Kernel allocation are uninterruptible */ r = ttm_bo_init(&adev->mman.bdev, &bo->tbo, size, type, &bo->placement, page_align, !kernel, NULL, acc_size, sg, resv, &amdgpu_ttm_bo_destroy); if (unlikely(r != 0)) { return r; } if (flags & AMDGPU_GEM_CREATE_VRAM_CLEARED && bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) { struct fence *fence; if (adev->mman.buffer_funcs_ring == NULL || !adev->mman.buffer_funcs_ring->ready) { r = -EBUSY; goto fail_free; } r = amdgpu_bo_reserve(bo, false); if (unlikely(r != 0)) goto fail_free; amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_VRAM); r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false); if (unlikely(r != 0)) goto fail_unreserve; amdgpu_fill_buffer(bo, 0, bo->tbo.resv, &fence); amdgpu_bo_fence(bo, fence, false); amdgpu_bo_unreserve(bo); fence_put(bo->tbo.moving); bo->tbo.moving = fence_get(fence); fence_put(fence); } *bo_ptr = bo; trace_amdgpu_bo_create(bo); return 0; fail_unreserve: amdgpu_bo_unreserve(bo);fail_free: amdgpu_bo_unref(&bo); return r;} static int amdgpu_bo_create_shadow(struct amdgpu_device *adev, unsigned long size, int byte_align, struct amdgpu_bo *bo){ struct ttm_placement placement = {0}; struct ttm_place placements[AMDGPU_GEM_DOMAIN_MAX + 1]; int r; if (bo->shadow) return 0; bo->flags |= AMDGPU_GEM_CREATE_SHADOW; memset(&placements, 0, (AMDGPU_GEM_DOMAIN_MAX + 1) * sizeof(struct ttm_place)); amdgpu_ttm_placement_init(adev, &placement, placements, AMDGPU_GEM_DOMAIN_GTT, AMDGPU_GEM_CREATE_CPU_GTT_USWC); r = amdgpu_bo_create_restricted(adev, size, byte_align, true, AMDGPU_GEM_DOMAIN_GTT, AMDGPU_GEM_CREATE_CPU_GTT_USWC, NULL, &placement, bo->tbo.resv, &bo->shadow); if (!r) { bo->shadow->parent = amdgpu_bo_ref(bo); mutex_lock(&adev->shadow_list_lock); list_add_tail(&bo->shadow_list, &adev->shadow_list); mutex_unlock(&adev->shadow_list_lock); } return r;} int amdgpu_bo_create(struct amdgpu_device *adev, unsigned long size, int byte_align, bool kernel, u32 domain, u64 flags, struct sg_table *sg, struct reservation_object *resv, struct amdgpu_bo **bo_ptr){ struct ttm_placement placement = {0}; struct ttm_place placements[AMDGPU_GEM_DOMAIN_MAX + 1]; int r; memset(&placements, 0, (AMDGPU_GEM_DOMAIN_MAX + 1) * sizeof(struct ttm_place)); amdgpu_ttm_placement_init(adev, &placement, placements, domain, flags); r = amdgpu_bo_create_restricted(adev, size, byte_align, kernel, domain, flags, sg, &placement, resv, bo_ptr); if (r) return r; if (amdgpu_need_backup(adev) && (flags & AMDGPU_GEM_CREATE_SHADOW)) { r = amdgpu_bo_create_shadow(adev, size, byte_align, (*bo_ptr)); if (r) amdgpu_bo_unref(bo_ptr); } return r;} int amdgpu_bo_backup_to_shadow(struct amdgpu_device *adev, struct amdgpu_ring *ring, struct amdgpu_bo *bo, struct reservation_object *resv, struct fence **fence, bool direct) { struct amdgpu_bo *shadow = bo->shadow; uint64_t bo_addr, shadow_addr; int r;Lines 1–500 of 964. Load more
/* * Copyright 2009 Jerome Glisse. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE * USE OR OTHER DEALINGS IN THE SOFTWARE. * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * */ /* * Authors: * Jerome Glisse <glisse@freedesktop.org> * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> * Dave Airlie */ #include <linux/list.h> #include <linux/slab.h> #include <drm/drmP.h> #include <drm/amdgpu_drm.h> #include <drm/drm_cache.h> #include "amdgpu.h" #include "amdgpu_trace.h" static u64 amdgpu_get_vis_part_size(struct amdgpu_device *adev, struct ttm_mem_reg *mem) { if (mem->start << PAGE_SHIFT >= adev->mc.visible_vram_size) return 0; return ((mem->start << PAGE_SHIFT) + mem->size) > adev->mc.visible_vram_size ? adev->mc.visible_vram_size - (mem->start << PAGE_SHIFT) : mem->size; } static void amdgpu_update_memory_usage(struct amdgpu_device *adev, struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem) { u64 vis_size; if (!adev) return; if (new_mem) { switch (new_mem->mem_type) { case TTM_PL_TT: atomic64_add(new_mem->size, &adev->gtt_usage); break; case TTM_PL_VRAM: atomic64_add(new_mem->size, &adev->vram_usage); vis_size = amdgpu_get_vis_part_size(adev, new_mem); atomic64_add(vis_size, &adev->vram_vis_usage); break; } } if (old_mem) { switch (old_mem->mem_type) { case TTM_PL_TT: atomic64_sub(old_mem->size, &adev->gtt_usage); break; case TTM_PL_VRAM: atomic64_sub(old_mem->size, &adev->vram_usage); vis_size = amdgpu_get_vis_part_size(adev, old_mem); atomic64_sub(vis_size, &adev->vram_vis_usage); break; } } } static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo) { struct amdgpu_bo *bo; bo = container_of(tbo, struct amdgpu_bo, tbo); amdgpu_update_memory_usage(bo->adev, &bo->tbo.mem, NULL); drm_gem_object_release(&bo->gem_base); amdgpu_bo_unref(&bo->parent); if (!list_empty(&bo->shadow_list)) { mutex_lock(&bo->adev->shadow_list_lock); list_del_init(&bo->shadow_list); mutex_unlock(&bo->adev->shadow_list_lock); } kfree(bo->metadata); kfree(bo); } bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo) { if (bo->destroy == &amdgpu_ttm_bo_destroy) return true; return false; } static void amdgpu_ttm_placement_init(struct amdgpu_device *adev, struct ttm_placement *placement, struct ttm_place *places, u32 domain, u64 flags) { u32 c = 0; if (domain & AMDGPU_GEM_DOMAIN_VRAM) { unsigned visible_pfn = adev->mc.visible_vram_size >> PAGE_SHIFT; if (flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS && !(flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) && adev->mc.visible_vram_size < adev->mc.real_vram_size) { places[c].fpfn = visible_pfn; places[c].lpfn = 0; places[c].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_VRAM | TTM_PL_FLAG_TOPDOWN; c++; } places[c].fpfn = 0; places[c].lpfn = 0; places[c].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_VRAM; if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) places[c].lpfn = visible_pfn; else places[c].flags |= TTM_PL_FLAG_TOPDOWN; c++; } if (domain & AMDGPU_GEM_DOMAIN_GTT) { places[c].fpfn = 0; places[c].lpfn = 0; places[c].flags = TTM_PL_FLAG_TT; if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) places[c].flags |= TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED; else places[c].flags |= TTM_PL_FLAG_CACHED; c++; } if (domain & AMDGPU_GEM_DOMAIN_CPU) { places[c].fpfn = 0; places[c].lpfn = 0; places[c].flags = TTM_PL_FLAG_SYSTEM; if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) places[c].flags |= TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED; else places[c].flags |= TTM_PL_FLAG_CACHED; c++; } if (domain & AMDGPU_GEM_DOMAIN_GDS) { places[c].fpfn = 0; places[c].lpfn = 0; places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GDS; c++; } if (domain & AMDGPU_GEM_DOMAIN_GWS) { places[c].fpfn = 0; places[c].lpfn = 0; places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GWS; c++; } if (domain & AMDGPU_GEM_DOMAIN_OA) { places[c].fpfn = 0; places[c].lpfn = 0; places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_OA; c++; } if (!c) { places[c].fpfn = 0; places[c].lpfn = 0; places[c].flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM; c++; } placement->num_placement = c; placement->placement = places; placement->num_busy_placement = c; placement->busy_placement = places; } void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain) { amdgpu_ttm_placement_init(abo->adev, &abo->placement, abo->placements, domain, abo->flags); } static void amdgpu_fill_placement_to_bo(struct amdgpu_bo *bo, struct ttm_placement *placement) { BUG_ON(placement->num_placement > (AMDGPU_GEM_DOMAIN_MAX + 1)); memcpy(bo->placements, placement->placement, placement->num_placement * sizeof(struct ttm_place)); bo->placement.num_placement = placement->num_placement; bo->placement.num_busy_placement = placement->num_busy_placement; bo->placement.placement = bo->placements; bo->placement.busy_placement = bo->placements; } /** * amdgpu_bo_create_kernel - create BO for kernel use * * @adev: amdgpu device object * @size: size for the new BO * @align: alignment for the new BO * @domain: where to place it * @bo_ptr: resulting BO * @gpu_addr: GPU addr of the pinned BO * @cpu_addr: optional CPU address mapping * * Allocates and pins a BO for kernel internal use. * * Returns 0 on success, negative error code otherwise. */ int amdgpu_bo_create_kernel(struct amdgpu_device *adev, unsigned long size, int align, u32 domain, struct amdgpu_bo **bo_ptr, u64 *gpu_addr, void **cpu_addr) { int r; r = amdgpu_bo_create(adev, size, align, true, domain, AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED, NULL, NULL, bo_ptr); if (r) { dev_err(adev->dev, "(%d) failed to allocate kernel bo\n", r); return r; } r = amdgpu_bo_reserve(*bo_ptr, false); if (r) { dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r); goto error_free; } r = amdgpu_bo_pin(*bo_ptr, domain, gpu_addr); if (r) { dev_err(adev->dev, "(%d) kernel bo pin failed\n", r); goto error_unreserve; } if (cpu_addr) { r = amdgpu_bo_kmap(*bo_ptr, cpu_addr); if (r) { dev_err(adev->dev, "(%d) kernel bo map failed\n", r); goto error_unreserve; } } amdgpu_bo_unreserve(*bo_ptr); return 0; error_unreserve: amdgpu_bo_unreserve(*bo_ptr); error_free: amdgpu_bo_unref(bo_ptr); return r; } /** * amdgpu_bo_free_kernel - free BO for kernel use * * @bo: amdgpu BO to free * * unmaps and unpin a BO for kernel internal use. */ void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr, void **cpu_addr) { if (*bo == NULL) return; if (likely(amdgpu_bo_reserve(*bo, false) == 0)) { if (cpu_addr) amdgpu_bo_kunmap(*bo); amdgpu_bo_unpin(*bo); amdgpu_bo_unreserve(*bo); } amdgpu_bo_unref(bo); if (gpu_addr) *gpu_addr = 0; if (cpu_addr) *cpu_addr = NULL; } int amdgpu_bo_create_restricted(struct amdgpu_device *adev, unsigned long size, int byte_align, bool kernel, u32 domain, u64 flags, struct sg_table *sg, struct ttm_placement *placement, struct reservation_object *resv, struct amdgpu_bo **bo_ptr) { struct amdgpu_bo *bo; enum ttm_bo_type type; unsigned long page_align; size_t acc_size; int r; page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT; size = ALIGN(size, PAGE_SIZE); if (kernel) { type = ttm_bo_type_kernel; } else if (sg) { type = ttm_bo_type_sg; } else { type = ttm_bo_type_device; } *bo_ptr = NULL; acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size, sizeof(struct amdgpu_bo)); bo = kzalloc(sizeof(struct amdgpu_bo), GFP_KERNEL); if (bo == NULL) return -ENOMEM; r = drm_gem_object_init(adev->ddev, &bo->gem_base, size); if (unlikely(r)) { kfree(bo); return r; } bo->adev = adev; INIT_LIST_HEAD(&bo->shadow_list); INIT_LIST_HEAD(&bo->va); bo->prefered_domains = domain & (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT | AMDGPU_GEM_DOMAIN_CPU | AMDGPU_GEM_DOMAIN_GDS | AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA); bo->allowed_domains = bo->prefered_domains; if (!kernel && bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM) bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT; bo->flags = flags; /* For architectures that don't support WC memory, * mask out the WC flag from the BO */ if (!drm_arch_can_wc_memory()) bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; amdgpu_fill_placement_to_bo(bo, placement); /* Kernel allocation are uninterruptible */ r = ttm_bo_init(&adev->mman.bdev, &bo->tbo, size, type, &bo->placement, page_align, !kernel, NULL, acc_size, sg, resv, &amdgpu_ttm_bo_destroy); if (unlikely(r != 0)) { return r; } if (flags & AMDGPU_GEM_CREATE_VRAM_CLEARED && bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) { struct fence *fence; if (adev->mman.buffer_funcs_ring == NULL || !adev->mman.buffer_funcs_ring->ready) { r = -EBUSY; goto fail_free; } r = amdgpu_bo_reserve(bo, false); if (unlikely(r != 0)) goto fail_free; amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_VRAM); r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false); if (unlikely(r != 0)) goto fail_unreserve; amdgpu_fill_buffer(bo, 0, bo->tbo.resv, &fence); amdgpu_bo_fence(bo, fence, false); amdgpu_bo_unreserve(bo); fence_put(bo->tbo.moving); bo->tbo.moving = fence_get(fence); fence_put(fence); } *bo_ptr = bo; trace_amdgpu_bo_create(bo); return 0; fail_unreserve: amdgpu_bo_unreserve(bo); fail_free: amdgpu_bo_unref(&bo); return r; } static int amdgpu_bo_create_shadow(struct amdgpu_device *adev, unsigned long size, int byte_align, struct amdgpu_bo *bo) { struct ttm_placement placement = {0}; struct ttm_place placements[AMDGPU_GEM_DOMAIN_MAX + 1]; int r; if (bo->shadow) return 0; bo->flags |= AMDGPU_GEM_CREATE_SHADOW; memset(&placements, 0, (AMDGPU_GEM_DOMAIN_MAX + 1) * sizeof(struct ttm_place)); amdgpu_ttm_placement_init(adev, &placement, placements, AMDGPU_GEM_DOMAIN_GTT, AMDGPU_GEM_CREATE_CPU_GTT_USWC); r = amdgpu_bo_create_restricted(adev, size, byte_align, true, AMDGPU_GEM_DOMAIN_GTT, AMDGPU_GEM_CREATE_CPU_GTT_USWC, NULL, &placement, bo->tbo.resv, &bo->shadow); if (!r) { bo->shadow->parent = amdgpu_bo_ref(bo); mutex_lock(&adev->shadow_list_lock); list_add_tail(&bo->shadow_list, &adev->shadow_list); mutex_unlock(&adev->shadow_list_lock); } return r; } int amdgpu_bo_create(struct amdgpu_device *adev, unsigned long size, int byte_align, bool kernel, u32 domain, u64 flags, struct sg_table *sg, struct reservation_object *resv, struct amdgpu_bo **bo_ptr) { struct ttm_placement placement = {0}; struct ttm_place placements[AMDGPU_GEM_DOMAIN_MAX + 1]; int r; memset(&placements, 0, (AMDGPU_GEM_DOMAIN_MAX + 1) * sizeof(struct ttm_place)); amdgpu_ttm_placement_init(adev, &placement, placements, domain, flags); r = amdgpu_bo_create_restricted(adev, size, byte_align, kernel, domain, flags, sg, &placement, resv, bo_ptr); if (r) return r; if (amdgpu_need_backup(adev) && (flags & AMDGPU_GEM_CREATE_SHADOW)) { r = amdgpu_bo_create_shadow(adev, size, byte_align, (*bo_ptr)); if (r) amdgpu_bo_unref(bo_ptr); } return r; } int amdgpu_bo_backup_to_shadow(struct amdgpu_device *adev, struct amdgpu_ring *ring, struct amdgpu_bo *bo, struct reservation_object *resv, struct fence **fence, bool direct) { struct amdgpu_bo *shadow = bo->shadow; uint64_t bo_addr, shadow_addr; int r;