/* * Copyright 2009 Jerome Glisse. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE * USE OR OTHER DEALINGS IN THE SOFTWARE. * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * *//* * Authors: *    Jerome Glisse <glisse@freedesktop.org> *    Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> *    Dave Airlie */#include <drm/ttm/ttm_bo_api.h>#include <drm/ttm/ttm_bo_driver.h>#include <drm/ttm/ttm_placement.h>#include <drm/ttm/ttm_module.h>#include <drm/ttm/ttm_page_alloc.h>#include <drm/drmP.h>#include <drm/amdgpu_drm.h>#include <linux/seq_file.h>#include <linux/slab.h>#include <linux/swiotlb.h>#include <linux/swap.h>#include <linux/pagemap.h>#include <linux/debugfs.h>#include <linux/iommu.h>#include "amdgpu.h"#include "amdgpu_object.h"#include "amdgpu_trace.h"#include "amdgpu_amdkfd.h"#include "amdgpu_sdma.h"#include "bif/bif_4_1_d.h" static int amdgpu_map_buffer(struct ttm_buffer_object *bo,			     struct ttm_mem_reg *mem, unsigned num_pages,			     uint64_t offset, unsigned window,			     struct amdgpu_ring *ring,			     uint64_t *addr); static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev);static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev); static int amdgpu_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags){	return 0;} /** * amdgpu_init_mem_type - Initialize a memory manager for a specific type of * memory request. * * @bdev: The TTM BO device object (contains a reference to amdgpu_device) * @type: The type of memory requested * @man: The memory type manager for each domain * * This is called by ttm_bo_init_mm() when a buffer object is being * initialized. */static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,				struct ttm_mem_type_manager *man){	struct amdgpu_device *adev; 	adev = amdgpu_ttm_adev(bdev); 	switch (type) {	case TTM_PL_SYSTEM:		/* System memory */		man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;		man->available_caching = TTM_PL_MASK_CACHING;		man->default_caching = TTM_PL_FLAG_CACHED;		break;	case TTM_PL_TT:		/* GTT memory  */		man->func = &amdgpu_gtt_mgr_func;		man->gpu_offset = adev->gmc.gart_start;		man->available_caching = TTM_PL_MASK_CACHING;		man->default_caching = TTM_PL_FLAG_CACHED;		man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA;		break;	case TTM_PL_VRAM:		/* "On-card" video ram */		man->func = &amdgpu_vram_mgr_func;		man->gpu_offset = adev->gmc.vram_start;		man->flags = TTM_MEMTYPE_FLAG_FIXED |			     TTM_MEMTYPE_FLAG_MAPPABLE;		man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC;		man->default_caching = TTM_PL_FLAG_WC;		break;	case AMDGPU_PL_GDS:	case AMDGPU_PL_GWS:	case AMDGPU_PL_OA:		/* On-chip GDS memory*/		man->func = &ttm_bo_manager_func;		man->gpu_offset = 0;		man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_CMA;		man->available_caching = TTM_PL_FLAG_UNCACHED;		man->default_caching = TTM_PL_FLAG_UNCACHED;		break;	default:		DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);		return -EINVAL;	}	return 0;} /** * amdgpu_evict_flags - Compute placement flags * * @bo: The buffer object to evict * @placement: Possible destination(s) for evicted BO * * Fill in placement data when ttm_bo_evict() is called */static void amdgpu_evict_flags(struct ttm_buffer_object *bo,				struct ttm_placement *placement){	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);	struct amdgpu_bo *abo;	static const struct ttm_place placements = {		.fpfn = 0,		.lpfn = 0,		.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM	}; 	/* Don't handle scatter gather BOs */	if (bo->type == ttm_bo_type_sg) {		placement->num_placement = 0;		placement->num_busy_placement = 0;		return;	} 	/* Object isn't an AMDGPU object so ignore */	if (!amdgpu_bo_is_amdgpu_bo(bo)) {		placement->placement = &placements;		placement->busy_placement = &placements;		placement->num_placement = 1;		placement->num_busy_placement = 1;		return;	} 	abo = ttm_to_amdgpu_bo(bo);	switch (bo->mem.mem_type) {	case AMDGPU_PL_GDS:	case AMDGPU_PL_GWS:	case AMDGPU_PL_OA:		placement->num_placement = 0;		placement->num_busy_placement = 0;		return; 	case TTM_PL_VRAM:		if (!adev->mman.buffer_funcs_enabled) {			/* Move to system memory */			amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);		} else if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&			   !(abo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) &&			   amdgpu_bo_in_cpu_visible_vram(abo)) { 			/* Try evicting to the CPU inaccessible part of VRAM			 * first, but only set GTT as busy placement, so this			 * BO will be evicted to GTT rather than causing other			 * BOs to be evicted from VRAM			 */			amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |							 AMDGPU_GEM_DOMAIN_GTT);			abo->placements[0].fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;			abo->placements[0].lpfn = 0;			abo->placement.busy_placement = &abo->placements[1];			abo->placement.num_busy_placement = 1;		} else {			/* Move to GTT memory */			amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT);		}		break;	case TTM_PL_TT:	default:		amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);		break;	}	*placement = abo->placement;} /** * amdgpu_verify_access - Verify access for a mmap call * * @bo:	The buffer object to map * @filp: The file pointer from the process performing the mmap * * This is called by ttm_bo_mmap() to verify whether a process * has the right to mmap a BO to their process space. */static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp){	struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo); 	/*	 * Don't verify access for KFD BOs. They don't have a GEM	 * object associated with them.	 */	if (abo->kfd_bo)		return 0; 	if (amdgpu_ttm_tt_get_usermm(bo->ttm))		return -EPERM;	return drm_vma_node_verify_access(&abo->gem_base.vma_node,					  filp->private_data);} /** * amdgpu_move_null - Register memory for a buffer object * * @bo: The bo to assign the memory to * @new_mem: The memory to be assigned. * * Assign the memory from new_mem to the memory of the buffer object bo. */static void amdgpu_move_null(struct ttm_buffer_object *bo,			     struct ttm_mem_reg *new_mem){	struct ttm_mem_reg *old_mem = &bo->mem; 	BUG_ON(old_mem->mm_node != NULL);	*old_mem = *new_mem;	new_mem->mm_node = NULL;} /** * amdgpu_mm_node_addr - Compute the GPU relative offset of a GTT buffer. * * @bo: The bo to assign the memory to. * @mm_node: Memory manager node for drm allocator. * @mem: The region where the bo resides. * */static uint64_t amdgpu_mm_node_addr(struct ttm_buffer_object *bo,				    struct drm_mm_node *mm_node,				    struct ttm_mem_reg *mem){	uint64_t addr = 0; 	if (mm_node->start != AMDGPU_BO_INVALID_OFFSET) {		addr = mm_node->start << PAGE_SHIFT;		addr += bo->bdev->man[mem->mem_type].gpu_offset;	}	return addr;} /** * amdgpu_find_mm_node - Helper function finds the drm_mm_node corresponding to * @offset. It also modifies the offset to be within the drm_mm_node returned * * @mem: The region where the bo resides. * @offset: The offset that drm_mm_node is used for finding. * */static struct drm_mm_node *amdgpu_find_mm_node(struct ttm_mem_reg *mem,					       unsigned long *offset){	struct drm_mm_node *mm_node = mem->mm_node; 	while (*offset >= (mm_node->size << PAGE_SHIFT)) {		*offset -= (mm_node->size << PAGE_SHIFT);		++mm_node;	}	return mm_node;} /** * amdgpu_copy_ttm_mem_to_mem - Helper function for copy * * The function copies @size bytes from {src->mem + src->offset} to * {dst->mem + dst->offset}. src->bo and dst->bo could be same BO for a * move and different for a BO to BO copy. * * @f: Returns the last fence if multiple jobs are submitted. */int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,			       struct amdgpu_copy_mem *src,			       struct amdgpu_copy_mem *dst,			       uint64_t size,			       struct reservation_object *resv,			       struct dma_fence **f){	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;	struct drm_mm_node *src_mm, *dst_mm;	uint64_t src_node_start, dst_node_start, src_node_size,		 dst_node_size, src_page_offset, dst_page_offset;	struct dma_fence *fence = NULL;	int r = 0;	const uint64_t GTT_MAX_BYTES = (AMDGPU_GTT_MAX_TRANSFER_SIZE *					AMDGPU_GPU_PAGE_SIZE); 	if (!adev->mman.buffer_funcs_enabled) {		DRM_ERROR("Trying to move memory with ring turned off.\n");		return -EINVAL;	} 	src_mm = amdgpu_find_mm_node(src->mem, &src->offset);	src_node_start = amdgpu_mm_node_addr(src->bo, src_mm, src->mem) +					     src->offset;	src_node_size = (src_mm->size << PAGE_SHIFT) - src->offset;	src_page_offset = src_node_start & (PAGE_SIZE - 1); 	dst_mm = amdgpu_find_mm_node(dst->mem, &dst->offset);	dst_node_start = amdgpu_mm_node_addr(dst->bo, dst_mm, dst->mem) +					     dst->offset;	dst_node_size = (dst_mm->size << PAGE_SHIFT) - dst->offset;	dst_page_offset = dst_node_start & (PAGE_SIZE - 1); 	mutex_lock(&adev->mman.gtt_window_lock); 	while (size) {		unsigned long cur_size;		uint64_t from = src_node_start, to = dst_node_start;		struct dma_fence *next; 		/* Copy size cannot exceed GTT_MAX_BYTES. So if src or dst		 * begins at an offset, then adjust the size accordingly		 */		cur_size = min3(min(src_node_size, dst_node_size), size,				GTT_MAX_BYTES);		if (cur_size + src_page_offset > GTT_MAX_BYTES ||		    cur_size + dst_page_offset > GTT_MAX_BYTES)			cur_size -= max(src_page_offset, dst_page_offset); 		/* Map only what needs to be accessed. Map src to window 0 and		 * dst to window 1		 */		if (src->mem->start == AMDGPU_BO_INVALID_OFFSET) {			r = amdgpu_map_buffer(src->bo, src->mem,					PFN_UP(cur_size + src_page_offset),					src_node_start, 0, ring,					&from);			if (r)				goto error;			/* Adjust the offset because amdgpu_map_buffer returns			 * start of mapped page			 */			from += src_page_offset;		} 		if (dst->mem->start == AMDGPU_BO_INVALID_OFFSET) {			r = amdgpu_map_buffer(dst->bo, dst->mem,					PFN_UP(cur_size + dst_page_offset),					dst_node_start, 1, ring,					&to);			if (r)				goto error;			to += dst_page_offset;		} 		r = amdgpu_copy_buffer(ring, from, to, cur_size,				       resv, &next, false, true);		if (r)			goto error; 		dma_fence_put(fence);		fence = next; 		size -= cur_size;		if (!size)			break; 		src_node_size -= cur_size;		if (!src_node_size) {			src_node_start = amdgpu_mm_node_addr(src->bo, ++src_mm,							     src->mem);			src_node_size = (src_mm->size << PAGE_SHIFT);		} else {			src_node_start += cur_size;			src_page_offset = src_node_start & (PAGE_SIZE - 1);		}		dst_node_size -= cur_size;		if (!dst_node_size) {			dst_node_start = amdgpu_mm_node_addr(dst->bo, ++dst_mm,							     dst->mem);			dst_node_size = (dst_mm->size << PAGE_SHIFT);		} else {			dst_node_start += cur_size;			dst_page_offset = dst_node_start & (PAGE_SIZE - 1);		}	}error:	mutex_unlock(&adev->mman.gtt_window_lock);	if (f)		*f = dma_fence_get(fence);	dma_fence_put(fence);	return r;} /** * amdgpu_move_blit - Copy an entire buffer to another buffer * * This is a helper called by amdgpu_bo_move() and amdgpu_move_vram_ram() to * help move buffers to and from VRAM. */static int amdgpu_move_blit(struct ttm_buffer_object *bo,			    bool evict, bool no_wait_gpu,			    struct ttm_mem_reg *new_mem,			    struct ttm_mem_reg *old_mem){	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);	struct amdgpu_copy_mem src, dst;	struct dma_fence *fence = NULL;	int r; 	src.bo = bo;	dst.bo = bo;	src.mem = old_mem;	dst.mem = new_mem;	src.offset = 0;	dst.offset = 0; 	r = amdgpu_ttm_copy_mem_to_mem(adev, &src, &dst,				       new_mem->num_pages << PAGE_SHIFT,				       bo->resv, &fence);	if (r)		goto error; 	/* Always block for VM page tables before committing the new location */	if (bo->type == ttm_bo_type_kernel)		r = ttm_bo_move_accel_cleanup(bo, fence, true, new_mem);	else		r = ttm_bo_pipeline_move(bo, fence, evict, new_mem);	dma_fence_put(fence);	return r; error:	if (fence)		dma_fence_wait(fence, false);	dma_fence_put(fence);	return r;} /** * amdgpu_move_vram_ram - Copy VRAM buffer to RAM buffer * * Called by amdgpu_bo_move(). */static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, bool evict,				struct ttm_operation_ctx *ctx,				struct ttm_mem_reg *new_mem){	struct amdgpu_device *adev;	struct ttm_mem_reg *old_mem = &bo->mem;	struct ttm_mem_reg tmp_mem;	struct ttm_place placements;	struct ttm_placement placement;	int r; 	adev = amdgpu_ttm_adev(bo->bdev); 	/* create space/pages for new_mem in GTT space */	tmp_mem = *new_mem;	tmp_mem.mm_node = NULL;	placement.num_placement = 1;	placement.placement = &placements;	placement.num_busy_placement = 1;	placement.busy_placement = &placements;	placements.fpfn = 0;	placements.lpfn = 0;	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;	r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx);	if (unlikely(r)) {		return r;	} 	/* set caching flags */	r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement);	if (unlikely(r)) {		goto out_cleanup;	} 	/* Bind the memory to the GTT space */	r = ttm_tt_bind(bo->ttm, &tmp_mem, ctx);	if (unlikely(r)) {		goto out_cleanup;	} 	/* blit VRAM to GTT */	r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu, &tmp_mem, old_mem);

Lines 1–500 of 2470. Load more