/* * Copyright 2009 Jerome Glisse. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE * USE OR OTHER DEALINGS IN THE SOFTWARE. * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * *//* * Authors: * Jerome Glisse <glisse@freedesktop.org> * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> * Dave Airlie */#include <drm/ttm/ttm_bo_api.h>#include <drm/ttm/ttm_bo_driver.h>#include <drm/ttm/ttm_placement.h>#include <drm/ttm/ttm_module.h>#include <drm/ttm/ttm_page_alloc.h>#include <drm/drmP.h>#include <drm/amdgpu_drm.h>#include <linux/seq_file.h>#include <linux/slab.h>#include <linux/swiotlb.h>#include <linux/swap.h>#include <linux/pagemap.h>#include <linux/debugfs.h>#include "amdgpu.h"#include "bif/bif_4_1_d.h" #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT) static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev);static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev); /* * Global memory. */static int amdgpu_ttm_mem_global_init(struct drm_global_reference *ref){ return ttm_mem_global_init(ref->object);} static void amdgpu_ttm_mem_global_release(struct drm_global_reference *ref){ ttm_mem_global_release(ref->object);} static int amdgpu_ttm_global_init(struct amdgpu_device *adev){ struct drm_global_reference *global_ref; struct amdgpu_ring *ring; struct amd_sched_rq *rq; int r; adev->mman.mem_global_referenced = false; global_ref = &adev->mman.mem_global_ref; global_ref->global_type = DRM_GLOBAL_TTM_MEM; global_ref->size = sizeof(struct ttm_mem_global); global_ref->init = &amdgpu_ttm_mem_global_init; global_ref->release = &amdgpu_ttm_mem_global_release; r = drm_global_item_ref(global_ref); if (r) { DRM_ERROR("Failed setting up TTM memory accounting " "subsystem.\n"); goto error_mem; } adev->mman.bo_global_ref.mem_glob = adev->mman.mem_global_ref.object; global_ref = &adev->mman.bo_global_ref.ref; global_ref->global_type = DRM_GLOBAL_TTM_BO; global_ref->size = sizeof(struct ttm_bo_global); global_ref->init = &ttm_bo_global_init; global_ref->release = &ttm_bo_global_release; r = drm_global_item_ref(global_ref); if (r) { DRM_ERROR("Failed setting up TTM BO subsystem.\n"); goto error_bo; } ring = adev->mman.buffer_funcs_ring; rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_KERNEL]; r = amd_sched_entity_init(&ring->sched, &adev->mman.entity, rq, amdgpu_sched_jobs); if (r) { DRM_ERROR("Failed setting up TTM BO move run queue.\n"); goto error_entity; } adev->mman.mem_global_referenced = true; return 0; error_entity: drm_global_item_unref(&adev->mman.bo_global_ref.ref);error_bo: drm_global_item_unref(&adev->mman.mem_global_ref);error_mem: return r;} static void amdgpu_ttm_global_fini(struct amdgpu_device *adev){ if (adev->mman.mem_global_referenced) { amd_sched_entity_fini(adev->mman.entity.sched, &adev->mman.entity); drm_global_item_unref(&adev->mman.bo_global_ref.ref); drm_global_item_unref(&adev->mman.mem_global_ref); adev->mman.mem_global_referenced = false; }} static int amdgpu_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags){ return 0;} static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, struct ttm_mem_type_manager *man){ struct amdgpu_device *adev; adev = amdgpu_ttm_adev(bdev); switch (type) { case TTM_PL_SYSTEM: /* System memory */ man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; man->available_caching = TTM_PL_MASK_CACHING; man->default_caching = TTM_PL_FLAG_CACHED; break; case TTM_PL_TT: man->func = &amdgpu_gtt_mgr_func; man->gpu_offset = adev->mc.gtt_start; man->available_caching = TTM_PL_MASK_CACHING; man->default_caching = TTM_PL_FLAG_CACHED; man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA; break; case TTM_PL_VRAM: /* "On-card" video ram */ man->func = &amdgpu_vram_mgr_func; man->gpu_offset = adev->mc.vram_start; man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_MAPPABLE; man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC; man->default_caching = TTM_PL_FLAG_WC; break; case AMDGPU_PL_GDS: case AMDGPU_PL_GWS: case AMDGPU_PL_OA: /* On-chip GDS memory*/ man->func = &ttm_bo_manager_func; man->gpu_offset = 0; man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_CMA; man->available_caching = TTM_PL_FLAG_UNCACHED; man->default_caching = TTM_PL_FLAG_UNCACHED; break; default: DRM_ERROR("Unsupported memory type %u\n", (unsigned)type); return -EINVAL; } return 0;} static void amdgpu_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *placement){ struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); struct amdgpu_bo *abo; static struct ttm_place placements = { .fpfn = 0, .lpfn = 0, .flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM }; unsigned i; if (!amdgpu_ttm_bo_is_amdgpu_bo(bo)) { placement->placement = &placements; placement->busy_placement = &placements; placement->num_placement = 1; placement->num_busy_placement = 1; return; } abo = container_of(bo, struct amdgpu_bo, tbo); switch (bo->mem.mem_type) { case TTM_PL_VRAM: if (adev->mman.buffer_funcs && adev->mman.buffer_funcs_ring && adev->mman.buffer_funcs_ring->ready == false) { amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU); } else { amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT); for (i = 0; i < abo->placement.num_placement; ++i) { if (!(abo->placements[i].flags & TTM_PL_FLAG_TT)) continue; if (abo->placements[i].lpfn) continue; /* set an upper limit to force directly * allocating address space for the BO. */ abo->placements[i].lpfn = adev->mc.gtt_size >> PAGE_SHIFT; } } break; case TTM_PL_TT: default: amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU); } *placement = abo->placement;} static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp){ struct amdgpu_bo *abo = container_of(bo, struct amdgpu_bo, tbo); if (amdgpu_ttm_tt_get_usermm(bo->ttm)) return -EPERM; return drm_vma_node_verify_access(&abo->gem_base.vma_node, filp->private_data);} static void amdgpu_move_null(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem){ struct ttm_mem_reg *old_mem = &bo->mem; BUG_ON(old_mem->mm_node != NULL); *old_mem = *new_mem; new_mem->mm_node = NULL;} static int amdgpu_mm_node_addr(struct ttm_buffer_object *bo, struct drm_mm_node *mm_node, struct ttm_mem_reg *mem, uint64_t *addr){ int r; switch (mem->mem_type) { case TTM_PL_TT: r = amdgpu_ttm_bind(bo, mem); if (r) return r; case TTM_PL_VRAM: *addr = mm_node->start << PAGE_SHIFT; *addr += bo->bdev->man[mem->mem_type].gpu_offset; break; default: DRM_ERROR("Unknown placement %d\n", mem->mem_type); return -EINVAL; } return 0;} static int amdgpu_move_blit(struct ttm_buffer_object *bo, bool evict, bool no_wait_gpu, struct ttm_mem_reg *new_mem, struct ttm_mem_reg *old_mem){ struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; struct drm_mm_node *old_mm, *new_mm; uint64_t old_start, old_size, new_start, new_size; unsigned long num_pages; struct dma_fence *fence = NULL; int r; BUILD_BUG_ON((PAGE_SIZE % AMDGPU_GPU_PAGE_SIZE) != 0); if (!ring->ready) { DRM_ERROR("Trying to move memory with ring turned off.\n"); return -EINVAL; } old_mm = old_mem->mm_node; r = amdgpu_mm_node_addr(bo, old_mm, old_mem, &old_start); if (r) return r; old_size = old_mm->size; new_mm = new_mem->mm_node; r = amdgpu_mm_node_addr(bo, new_mm, new_mem, &new_start); if (r) return r; new_size = new_mm->size; num_pages = new_mem->num_pages; while (num_pages) { unsigned long cur_pages = min(old_size, new_size); struct dma_fence *next; r = amdgpu_copy_buffer(ring, old_start, new_start, cur_pages * PAGE_SIZE, bo->resv, &next, false); if (r) goto error; dma_fence_put(fence); fence = next; num_pages -= cur_pages; if (!num_pages) break; old_size -= cur_pages; if (!old_size) { r = amdgpu_mm_node_addr(bo, ++old_mm, old_mem, &old_start); if (r) goto error; old_size = old_mm->size; } else { old_start += cur_pages * PAGE_SIZE; } new_size -= cur_pages; if (!new_size) { r = amdgpu_mm_node_addr(bo, ++new_mm, new_mem, &new_start); if (r) goto error; new_size = new_mm->size; } else { new_start += cur_pages * PAGE_SIZE; } } r = ttm_bo_pipeline_move(bo, fence, evict, new_mem); dma_fence_put(fence); return r; error: if (fence) dma_fence_wait(fence, false); dma_fence_put(fence); return r;} static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, bool evict, bool interruptible, bool no_wait_gpu, struct ttm_mem_reg *new_mem){ struct amdgpu_device *adev; struct ttm_mem_reg *old_mem = &bo->mem; struct ttm_mem_reg tmp_mem; struct ttm_place placements; struct ttm_placement placement; int r; adev = amdgpu_ttm_adev(bo->bdev); tmp_mem = *new_mem; tmp_mem.mm_node = NULL; placement.num_placement = 1; placement.placement = &placements; placement.num_busy_placement = 1; placement.busy_placement = &placements; placements.fpfn = 0; placements.lpfn = adev->mc.gtt_size >> PAGE_SHIFT; placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; r = ttm_bo_mem_space(bo, &placement, &tmp_mem, interruptible, no_wait_gpu); if (unlikely(r)) { return r; } r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement); if (unlikely(r)) { goto out_cleanup; } r = ttm_tt_bind(bo->ttm, &tmp_mem); if (unlikely(r)) { goto out_cleanup; } r = amdgpu_move_blit(bo, true, no_wait_gpu, &tmp_mem, old_mem); if (unlikely(r)) { goto out_cleanup; } r = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, new_mem);out_cleanup: ttm_bo_mem_put(bo, &tmp_mem); return r;} static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, bool evict, bool interruptible, bool no_wait_gpu, struct ttm_mem_reg *new_mem){ struct amdgpu_device *adev; struct ttm_mem_reg *old_mem = &bo->mem; struct ttm_mem_reg tmp_mem; struct ttm_placement placement; struct ttm_place placements; int r; adev = amdgpu_ttm_adev(bo->bdev); tmp_mem = *new_mem; tmp_mem.mm_node = NULL; placement.num_placement = 1; placement.placement = &placements; placement.num_busy_placement = 1; placement.busy_placement = &placements; placements.fpfn = 0; placements.lpfn = adev->mc.gtt_size >> PAGE_SHIFT; placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; r = ttm_bo_mem_space(bo, &placement, &tmp_mem, interruptible, no_wait_gpu); if (unlikely(r)) { return r; } r = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, &tmp_mem); if (unlikely(r)) { goto out_cleanup; } r = amdgpu_move_blit(bo, true, no_wait_gpu, new_mem, old_mem); if (unlikely(r)) { goto out_cleanup; }out_cleanup: ttm_bo_mem_put(bo, &tmp_mem); return r;} static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict, bool interruptible, bool no_wait_gpu, struct ttm_mem_reg *new_mem){ struct amdgpu_device *adev; struct amdgpu_bo *abo; struct ttm_mem_reg *old_mem = &bo->mem; int r; /* Can't move a pinned BO */ abo = container_of(bo, struct amdgpu_bo, tbo); if (WARN_ON_ONCE(abo->pin_count > 0)) return -EINVAL; adev = amdgpu_ttm_adev(bo->bdev); if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { amdgpu_move_null(bo, new_mem); return 0; } if ((old_mem->mem_type == TTM_PL_TT && new_mem->mem_type == TTM_PL_SYSTEM) || (old_mem->mem_type == TTM_PL_SYSTEM && new_mem->mem_type == TTM_PL_TT)) { /* bind is enough */ amdgpu_move_null(bo, new_mem); return 0; } if (adev->mman.buffer_funcs == NULL || adev->mman.buffer_funcs_ring == NULL || !adev->mman.buffer_funcs_ring->ready) { /* use memcpy */ goto memcpy; } if (old_mem->mem_type == TTM_PL_VRAM && new_mem->mem_type == TTM_PL_SYSTEM) { r = amdgpu_move_vram_ram(bo, evict, interruptible, no_wait_gpu, new_mem); } else if (old_mem->mem_type == TTM_PL_SYSTEM && new_mem->mem_type == TTM_PL_VRAM) { r = amdgpu_move_ram_vram(bo, evict, interruptible, no_wait_gpu, new_mem); } else { r = amdgpu_move_blit(bo, evict, no_wait_gpu, new_mem, old_mem); }Lines 1–500 of 1608. Load more
/* * Copyright 2009 Jerome Glisse. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE * USE OR OTHER DEALINGS IN THE SOFTWARE. * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * */ /* * Authors: * Jerome Glisse <glisse@freedesktop.org> * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> * Dave Airlie */ #include <drm/ttm/ttm_bo_api.h> #include <drm/ttm/ttm_bo_driver.h> #include <drm/ttm/ttm_placement.h> #include <drm/ttm/ttm_module.h> #include <drm/ttm/ttm_page_alloc.h> #include <drm/drmP.h> #include <drm/amdgpu_drm.h> #include <linux/seq_file.h> #include <linux/slab.h> #include <linux/swiotlb.h> #include <linux/swap.h> #include <linux/pagemap.h> #include <linux/debugfs.h> #include "amdgpu.h" #include "bif/bif_4_1_d.h" #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT) static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev); static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev); /* * Global memory. */ static int amdgpu_ttm_mem_global_init(struct drm_global_reference *ref) { return ttm_mem_global_init(ref->object); } static void amdgpu_ttm_mem_global_release(struct drm_global_reference *ref) { ttm_mem_global_release(ref->object); } static int amdgpu_ttm_global_init(struct amdgpu_device *adev) { struct drm_global_reference *global_ref; struct amdgpu_ring *ring; struct amd_sched_rq *rq; int r; adev->mman.mem_global_referenced = false; global_ref = &adev->mman.mem_global_ref; global_ref->global_type = DRM_GLOBAL_TTM_MEM; global_ref->size = sizeof(struct ttm_mem_global); global_ref->init = &amdgpu_ttm_mem_global_init; global_ref->release = &amdgpu_ttm_mem_global_release; r = drm_global_item_ref(global_ref); if (r) { DRM_ERROR("Failed setting up TTM memory accounting " "subsystem.\n"); goto error_mem; } adev->mman.bo_global_ref.mem_glob = adev->mman.mem_global_ref.object; global_ref = &adev->mman.bo_global_ref.ref; global_ref->global_type = DRM_GLOBAL_TTM_BO; global_ref->size = sizeof(struct ttm_bo_global); global_ref->init = &ttm_bo_global_init; global_ref->release = &ttm_bo_global_release; r = drm_global_item_ref(global_ref); if (r) { DRM_ERROR("Failed setting up TTM BO subsystem.\n"); goto error_bo; } ring = adev->mman.buffer_funcs_ring; rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_KERNEL]; r = amd_sched_entity_init(&ring->sched, &adev->mman.entity, rq, amdgpu_sched_jobs); if (r) { DRM_ERROR("Failed setting up TTM BO move run queue.\n"); goto error_entity; } adev->mman.mem_global_referenced = true; return 0; error_entity: drm_global_item_unref(&adev->mman.bo_global_ref.ref); error_bo: drm_global_item_unref(&adev->mman.mem_global_ref); error_mem: return r; } static void amdgpu_ttm_global_fini(struct amdgpu_device *adev) { if (adev->mman.mem_global_referenced) { amd_sched_entity_fini(adev->mman.entity.sched, &adev->mman.entity); drm_global_item_unref(&adev->mman.bo_global_ref.ref); drm_global_item_unref(&adev->mman.mem_global_ref); adev->mman.mem_global_referenced = false; } } static int amdgpu_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags) { return 0; } static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, struct ttm_mem_type_manager *man) { struct amdgpu_device *adev; adev = amdgpu_ttm_adev(bdev); switch (type) { case TTM_PL_SYSTEM: /* System memory */ man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; man->available_caching = TTM_PL_MASK_CACHING; man->default_caching = TTM_PL_FLAG_CACHED; break; case TTM_PL_TT: man->func = &amdgpu_gtt_mgr_func; man->gpu_offset = adev->mc.gtt_start; man->available_caching = TTM_PL_MASK_CACHING; man->default_caching = TTM_PL_FLAG_CACHED; man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA; break; case TTM_PL_VRAM: /* "On-card" video ram */ man->func = &amdgpu_vram_mgr_func; man->gpu_offset = adev->mc.vram_start; man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_MAPPABLE; man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC; man->default_caching = TTM_PL_FLAG_WC; break; case AMDGPU_PL_GDS: case AMDGPU_PL_GWS: case AMDGPU_PL_OA: /* On-chip GDS memory*/ man->func = &ttm_bo_manager_func; man->gpu_offset = 0; man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_CMA; man->available_caching = TTM_PL_FLAG_UNCACHED; man->default_caching = TTM_PL_FLAG_UNCACHED; break; default: DRM_ERROR("Unsupported memory type %u\n", (unsigned)type); return -EINVAL; } return 0; } static void amdgpu_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *placement) { struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); struct amdgpu_bo *abo; static struct ttm_place placements = { .fpfn = 0, .lpfn = 0, .flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM }; unsigned i; if (!amdgpu_ttm_bo_is_amdgpu_bo(bo)) { placement->placement = &placements; placement->busy_placement = &placements; placement->num_placement = 1; placement->num_busy_placement = 1; return; } abo = container_of(bo, struct amdgpu_bo, tbo); switch (bo->mem.mem_type) { case TTM_PL_VRAM: if (adev->mman.buffer_funcs && adev->mman.buffer_funcs_ring && adev->mman.buffer_funcs_ring->ready == false) { amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU); } else { amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT); for (i = 0; i < abo->placement.num_placement; ++i) { if (!(abo->placements[i].flags & TTM_PL_FLAG_TT)) continue; if (abo->placements[i].lpfn) continue; /* set an upper limit to force directly * allocating address space for the BO. */ abo->placements[i].lpfn = adev->mc.gtt_size >> PAGE_SHIFT; } } break; case TTM_PL_TT: default: amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU); } *placement = abo->placement; } static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp) { struct amdgpu_bo *abo = container_of(bo, struct amdgpu_bo, tbo); if (amdgpu_ttm_tt_get_usermm(bo->ttm)) return -EPERM; return drm_vma_node_verify_access(&abo->gem_base.vma_node, filp->private_data); } static void amdgpu_move_null(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem) { struct ttm_mem_reg *old_mem = &bo->mem; BUG_ON(old_mem->mm_node != NULL); *old_mem = *new_mem; new_mem->mm_node = NULL; } static int amdgpu_mm_node_addr(struct ttm_buffer_object *bo, struct drm_mm_node *mm_node, struct ttm_mem_reg *mem, uint64_t *addr) { int r; switch (mem->mem_type) { case TTM_PL_TT: r = amdgpu_ttm_bind(bo, mem); if (r) return r; case TTM_PL_VRAM: *addr = mm_node->start << PAGE_SHIFT; *addr += bo->bdev->man[mem->mem_type].gpu_offset; break; default: DRM_ERROR("Unknown placement %d\n", mem->mem_type); return -EINVAL; } return 0; } static int amdgpu_move_blit(struct ttm_buffer_object *bo, bool evict, bool no_wait_gpu, struct ttm_mem_reg *new_mem, struct ttm_mem_reg *old_mem) { struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; struct drm_mm_node *old_mm, *new_mm; uint64_t old_start, old_size, new_start, new_size; unsigned long num_pages; struct dma_fence *fence = NULL; int r; BUILD_BUG_ON((PAGE_SIZE % AMDGPU_GPU_PAGE_SIZE) != 0); if (!ring->ready) { DRM_ERROR("Trying to move memory with ring turned off.\n"); return -EINVAL; } old_mm = old_mem->mm_node; r = amdgpu_mm_node_addr(bo, old_mm, old_mem, &old_start); if (r) return r; old_size = old_mm->size; new_mm = new_mem->mm_node; r = amdgpu_mm_node_addr(bo, new_mm, new_mem, &new_start); if (r) return r; new_size = new_mm->size; num_pages = new_mem->num_pages; while (num_pages) { unsigned long cur_pages = min(old_size, new_size); struct dma_fence *next; r = amdgpu_copy_buffer(ring, old_start, new_start, cur_pages * PAGE_SIZE, bo->resv, &next, false); if (r) goto error; dma_fence_put(fence); fence = next; num_pages -= cur_pages; if (!num_pages) break; old_size -= cur_pages; if (!old_size) { r = amdgpu_mm_node_addr(bo, ++old_mm, old_mem, &old_start); if (r) goto error; old_size = old_mm->size; } else { old_start += cur_pages * PAGE_SIZE; } new_size -= cur_pages; if (!new_size) { r = amdgpu_mm_node_addr(bo, ++new_mm, new_mem, &new_start); if (r) goto error; new_size = new_mm->size; } else { new_start += cur_pages * PAGE_SIZE; } } r = ttm_bo_pipeline_move(bo, fence, evict, new_mem); dma_fence_put(fence); return r; error: if (fence) dma_fence_wait(fence, false); dma_fence_put(fence); return r; } static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, bool evict, bool interruptible, bool no_wait_gpu, struct ttm_mem_reg *new_mem) { struct amdgpu_device *adev; struct ttm_mem_reg *old_mem = &bo->mem; struct ttm_mem_reg tmp_mem; struct ttm_place placements; struct ttm_placement placement; int r; adev = amdgpu_ttm_adev(bo->bdev); tmp_mem = *new_mem; tmp_mem.mm_node = NULL; placement.num_placement = 1; placement.placement = &placements; placement.num_busy_placement = 1; placement.busy_placement = &placements; placements.fpfn = 0; placements.lpfn = adev->mc.gtt_size >> PAGE_SHIFT; placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; r = ttm_bo_mem_space(bo, &placement, &tmp_mem, interruptible, no_wait_gpu); if (unlikely(r)) { return r; } r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement); if (unlikely(r)) { goto out_cleanup; } r = ttm_tt_bind(bo->ttm, &tmp_mem); if (unlikely(r)) { goto out_cleanup; } r = amdgpu_move_blit(bo, true, no_wait_gpu, &tmp_mem, old_mem); if (unlikely(r)) { goto out_cleanup; } r = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, new_mem); out_cleanup: ttm_bo_mem_put(bo, &tmp_mem); return r; } static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, bool evict, bool interruptible, bool no_wait_gpu, struct ttm_mem_reg *new_mem) { struct amdgpu_device *adev; struct ttm_mem_reg *old_mem = &bo->mem; struct ttm_mem_reg tmp_mem; struct ttm_placement placement; struct ttm_place placements; int r; adev = amdgpu_ttm_adev(bo->bdev); tmp_mem = *new_mem; tmp_mem.mm_node = NULL; placement.num_placement = 1; placement.placement = &placements; placement.num_busy_placement = 1; placement.busy_placement = &placements; placements.fpfn = 0; placements.lpfn = adev->mc.gtt_size >> PAGE_SHIFT; placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; r = ttm_bo_mem_space(bo, &placement, &tmp_mem, interruptible, no_wait_gpu); if (unlikely(r)) { return r; } r = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, &tmp_mem); if (unlikely(r)) { goto out_cleanup; } r = amdgpu_move_blit(bo, true, no_wait_gpu, new_mem, old_mem); if (unlikely(r)) { goto out_cleanup; } out_cleanup: ttm_bo_mem_put(bo, &tmp_mem); return r; } static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict, bool interruptible, bool no_wait_gpu, struct ttm_mem_reg *new_mem) { struct amdgpu_device *adev; struct amdgpu_bo *abo; struct ttm_mem_reg *old_mem = &bo->mem; int r; /* Can't move a pinned BO */ abo = container_of(bo, struct amdgpu_bo, tbo); if (WARN_ON_ONCE(abo->pin_count > 0)) return -EINVAL; adev = amdgpu_ttm_adev(bo->bdev); if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { amdgpu_move_null(bo, new_mem); return 0; } if ((old_mem->mem_type == TTM_PL_TT && new_mem->mem_type == TTM_PL_SYSTEM) || (old_mem->mem_type == TTM_PL_SYSTEM && new_mem->mem_type == TTM_PL_TT)) { /* bind is enough */ amdgpu_move_null(bo, new_mem); return 0; } if (adev->mman.buffer_funcs == NULL || adev->mman.buffer_funcs_ring == NULL || !adev->mman.buffer_funcs_ring->ready) { /* use memcpy */ goto memcpy; } if (old_mem->mem_type == TTM_PL_VRAM && new_mem->mem_type == TTM_PL_SYSTEM) { r = amdgpu_move_vram_ram(bo, evict, interruptible, no_wait_gpu, new_mem); } else if (old_mem->mem_type == TTM_PL_SYSTEM && new_mem->mem_type == TTM_PL_VRAM) { r = amdgpu_move_ram_vram(bo, evict, interruptible, no_wait_gpu, new_mem); } else { r = amdgpu_move_blit(bo, evict, no_wait_gpu, new_mem, old_mem); }