/* * Copyright 2009 Jerome Glisse. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE * USE OR OTHER DEALINGS IN THE SOFTWARE. * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * *//* * Authors: *    Jerome Glisse <glisse@freedesktop.org> *    Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> *    Dave Airlie */#include <ttm/ttm_bo_api.h>#include <ttm/ttm_bo_driver.h>#include <ttm/ttm_placement.h>#include <ttm/ttm_module.h>#include <ttm/ttm_page_alloc.h>#include <drm/drmP.h>#include <drm/amdgpu_drm.h>#include <linux/seq_file.h>#include <linux/slab.h>#include <linux/swiotlb.h>#include <linux/swap.h>#include <linux/pagemap.h>#include <linux/debugfs.h>#include "amdgpu.h"#include "bif/bif_4_1_d.h" #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT) static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev);static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev); static struct amdgpu_device *amdgpu_get_adev(struct ttm_bo_device *bdev){	struct amdgpu_mman *mman;	struct amdgpu_device *adev; 	mman = container_of(bdev, struct amdgpu_mman, bdev);	adev = container_of(mman, struct amdgpu_device, mman);	return adev;}  /* * Global memory. */static int amdgpu_ttm_mem_global_init(struct drm_global_reference *ref){	return ttm_mem_global_init(ref->object);} static void amdgpu_ttm_mem_global_release(struct drm_global_reference *ref){	ttm_mem_global_release(ref->object);} static int amdgpu_ttm_global_init(struct amdgpu_device *adev){	struct drm_global_reference *global_ref;	int r; 	adev->mman.mem_global_referenced = false;	global_ref = &adev->mman.mem_global_ref;	global_ref->global_type = DRM_GLOBAL_TTM_MEM;	global_ref->size = sizeof(struct ttm_mem_global);	global_ref->init = &amdgpu_ttm_mem_global_init;	global_ref->release = &amdgpu_ttm_mem_global_release;	r = drm_global_item_ref(global_ref);	if (r != 0) {		DRM_ERROR("Failed setting up TTM memory accounting "			  "subsystem.\n");		return r;	} 	adev->mman.bo_global_ref.mem_glob =		adev->mman.mem_global_ref.object;	global_ref = &adev->mman.bo_global_ref.ref;	global_ref->global_type = DRM_GLOBAL_TTM_BO;	global_ref->size = sizeof(struct ttm_bo_global);	global_ref->init = &ttm_bo_global_init;	global_ref->release = &ttm_bo_global_release;	r = drm_global_item_ref(global_ref);	if (r != 0) {		DRM_ERROR("Failed setting up TTM BO subsystem.\n");		drm_global_item_unref(&adev->mman.mem_global_ref);		return r;	} 	adev->mman.mem_global_referenced = true;	return 0;} static void amdgpu_ttm_global_fini(struct amdgpu_device *adev){	if (adev->mman.mem_global_referenced) {		drm_global_item_unref(&adev->mman.bo_global_ref.ref);		drm_global_item_unref(&adev->mman.mem_global_ref);		adev->mman.mem_global_referenced = false;	}} static int amdgpu_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags){	return 0;} static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,				struct ttm_mem_type_manager *man){	struct amdgpu_device *adev; 	adev = amdgpu_get_adev(bdev); 	switch (type) {	case TTM_PL_SYSTEM:		/* System memory */		man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;		man->available_caching = TTM_PL_MASK_CACHING;		man->default_caching = TTM_PL_FLAG_CACHED;		break;	case TTM_PL_TT:		man->func = &ttm_bo_manager_func;		man->gpu_offset = adev->mc.gtt_start;		man->available_caching = TTM_PL_MASK_CACHING;		man->default_caching = TTM_PL_FLAG_CACHED;		man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA;		break;	case TTM_PL_VRAM:		/* "On-card" video ram */		man->func = &ttm_bo_manager_func;		man->gpu_offset = adev->mc.vram_start;		man->flags = TTM_MEMTYPE_FLAG_FIXED |			     TTM_MEMTYPE_FLAG_MAPPABLE;		man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC;		man->default_caching = TTM_PL_FLAG_WC;		break;	case AMDGPU_PL_GDS:	case AMDGPU_PL_GWS:	case AMDGPU_PL_OA:		/* On-chip GDS memory*/		man->func = &ttm_bo_manager_func;		man->gpu_offset = 0;		man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_CMA;		man->available_caching = TTM_PL_FLAG_UNCACHED;		man->default_caching = TTM_PL_FLAG_UNCACHED;		break;	default:		DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);		return -EINVAL;	}	return 0;} static void amdgpu_evict_flags(struct ttm_buffer_object *bo,				struct ttm_placement *placement){	struct amdgpu_bo *rbo;	static struct ttm_place placements = {		.fpfn = 0,		.lpfn = 0,		.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM	}; 	if (!amdgpu_ttm_bo_is_amdgpu_bo(bo)) {		placement->placement = &placements;		placement->busy_placement = &placements;		placement->num_placement = 1;		placement->num_busy_placement = 1;		return;	}	rbo = container_of(bo, struct amdgpu_bo, tbo);	switch (bo->mem.mem_type) {	case TTM_PL_VRAM:		if (rbo->adev->mman.buffer_funcs_ring->ready == false)			amdgpu_ttm_placement_from_domain(rbo, AMDGPU_GEM_DOMAIN_CPU);		else			amdgpu_ttm_placement_from_domain(rbo, AMDGPU_GEM_DOMAIN_GTT);		break;	case TTM_PL_TT:	default:		amdgpu_ttm_placement_from_domain(rbo, AMDGPU_GEM_DOMAIN_CPU);	}	*placement = rbo->placement;} static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp){	struct amdgpu_bo *rbo = container_of(bo, struct amdgpu_bo, tbo); 	return drm_vma_node_verify_access(&rbo->gem_base.vma_node, filp);} static void amdgpu_move_null(struct ttm_buffer_object *bo,			     struct ttm_mem_reg *new_mem){	struct ttm_mem_reg *old_mem = &bo->mem; 	BUG_ON(old_mem->mm_node != NULL);	*old_mem = *new_mem;	new_mem->mm_node = NULL;} static int amdgpu_move_blit(struct ttm_buffer_object *bo,			bool evict, bool no_wait_gpu,			struct ttm_mem_reg *new_mem,			struct ttm_mem_reg *old_mem){	struct amdgpu_device *adev;	struct amdgpu_ring *ring;	uint64_t old_start, new_start;	struct amdgpu_fence *fence;	int r; 	adev = amdgpu_get_adev(bo->bdev);	ring = adev->mman.buffer_funcs_ring;	old_start = old_mem->start << PAGE_SHIFT;	new_start = new_mem->start << PAGE_SHIFT; 	switch (old_mem->mem_type) {	case TTM_PL_VRAM:		old_start += adev->mc.vram_start;		break;	case TTM_PL_TT:		old_start += adev->mc.gtt_start;		break;	default:		DRM_ERROR("Unknown placement %d\n", old_mem->mem_type);		return -EINVAL;	}	switch (new_mem->mem_type) {	case TTM_PL_VRAM:		new_start += adev->mc.vram_start;		break;	case TTM_PL_TT:		new_start += adev->mc.gtt_start;		break;	default:		DRM_ERROR("Unknown placement %d\n", old_mem->mem_type);		return -EINVAL;	}	if (!ring->ready) {		DRM_ERROR("Trying to move memory with ring turned off.\n");		return -EINVAL;	} 	BUILD_BUG_ON((PAGE_SIZE % AMDGPU_GPU_PAGE_SIZE) != 0); 	r = amdgpu_copy_buffer(ring, old_start, new_start,			       new_mem->num_pages * PAGE_SIZE, /* bytes */			       bo->resv, &fence);	/* FIXME: handle copy error */	r = ttm_bo_move_accel_cleanup(bo, &fence->base,				      evict, no_wait_gpu, new_mem);	amdgpu_fence_unref(&fence);	return r;} static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo,				bool evict, bool interruptible,				bool no_wait_gpu,				struct ttm_mem_reg *new_mem){	struct amdgpu_device *adev;	struct ttm_mem_reg *old_mem = &bo->mem;	struct ttm_mem_reg tmp_mem;	struct ttm_place placements;	struct ttm_placement placement;	int r; 	adev = amdgpu_get_adev(bo->bdev);	tmp_mem = *new_mem;	tmp_mem.mm_node = NULL;	placement.num_placement = 1;	placement.placement = &placements;	placement.num_busy_placement = 1;	placement.busy_placement = &placements;	placements.fpfn = 0;	placements.lpfn = 0;	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;	r = ttm_bo_mem_space(bo, &placement, &tmp_mem,			     interruptible, no_wait_gpu);	if (unlikely(r)) {		return r;	} 	r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement);	if (unlikely(r)) {		goto out_cleanup;	} 	r = ttm_tt_bind(bo->ttm, &tmp_mem);	if (unlikely(r)) {		goto out_cleanup;	}	r = amdgpu_move_blit(bo, true, no_wait_gpu, &tmp_mem, old_mem);	if (unlikely(r)) {		goto out_cleanup;	}	r = ttm_bo_move_ttm(bo, true, no_wait_gpu, new_mem);out_cleanup:	ttm_bo_mem_put(bo, &tmp_mem);	return r;} static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo,				bool evict, bool interruptible,				bool no_wait_gpu,				struct ttm_mem_reg *new_mem){	struct amdgpu_device *adev;	struct ttm_mem_reg *old_mem = &bo->mem;	struct ttm_mem_reg tmp_mem;	struct ttm_placement placement;	struct ttm_place placements;	int r; 	adev = amdgpu_get_adev(bo->bdev);	tmp_mem = *new_mem;	tmp_mem.mm_node = NULL;	placement.num_placement = 1;	placement.placement = &placements;	placement.num_busy_placement = 1;	placement.busy_placement = &placements;	placements.fpfn = 0;	placements.lpfn = 0;	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;	r = ttm_bo_mem_space(bo, &placement, &tmp_mem,			     interruptible, no_wait_gpu);	if (unlikely(r)) {		return r;	}	r = ttm_bo_move_ttm(bo, true, no_wait_gpu, &tmp_mem);	if (unlikely(r)) {		goto out_cleanup;	}	r = amdgpu_move_blit(bo, true, no_wait_gpu, new_mem, old_mem);	if (unlikely(r)) {		goto out_cleanup;	}out_cleanup:	ttm_bo_mem_put(bo, &tmp_mem);	return r;} static int amdgpu_bo_move(struct ttm_buffer_object *bo,			bool evict, bool interruptible,			bool no_wait_gpu,			struct ttm_mem_reg *new_mem){	struct amdgpu_device *adev;	struct ttm_mem_reg *old_mem = &bo->mem;	int r; 	adev = amdgpu_get_adev(bo->bdev);	if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {		amdgpu_move_null(bo, new_mem);		return 0;	}	if ((old_mem->mem_type == TTM_PL_TT &&	     new_mem->mem_type == TTM_PL_SYSTEM) ||	    (old_mem->mem_type == TTM_PL_SYSTEM &&	     new_mem->mem_type == TTM_PL_TT)) {		/* bind is enough */		amdgpu_move_null(bo, new_mem);		return 0;	}	if (adev->mman.buffer_funcs == NULL ||	    adev->mman.buffer_funcs_ring == NULL ||	    !adev->mman.buffer_funcs_ring->ready) {		/* use memcpy */		goto memcpy;	} 	if (old_mem->mem_type == TTM_PL_VRAM &&	    new_mem->mem_type == TTM_PL_SYSTEM) {		r = amdgpu_move_vram_ram(bo, evict, interruptible,					no_wait_gpu, new_mem);	} else if (old_mem->mem_type == TTM_PL_SYSTEM &&		   new_mem->mem_type == TTM_PL_VRAM) {		r = amdgpu_move_ram_vram(bo, evict, interruptible,					    no_wait_gpu, new_mem);	} else {		r = amdgpu_move_blit(bo, evict, no_wait_gpu, new_mem, old_mem);	} 	if (r) {memcpy:		r = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem);		if (r) {			return r;		}	} 	/* update statistics */	atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &adev->num_bytes_moved);	return 0;} static int amdgpu_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem){	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];	struct amdgpu_device *adev = amdgpu_get_adev(bdev); 	mem->bus.addr = NULL;	mem->bus.offset = 0;	mem->bus.size = mem->num_pages << PAGE_SHIFT;	mem->bus.base = 0;	mem->bus.is_iomem = false;	if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))		return -EINVAL;	switch (mem->mem_type) {	case TTM_PL_SYSTEM:		/* system memory */		return 0;	case TTM_PL_TT:		break;	case TTM_PL_VRAM:		mem->bus.offset = mem->start << PAGE_SHIFT;		/* check if it's visible */		if ((mem->bus.offset + mem->bus.size) > adev->mc.visible_vram_size)			return -EINVAL;		mem->bus.base = adev->mc.aper_base;		mem->bus.is_iomem = true;#ifdef __alpha__		/*		 * Alpha: use bus.addr to hold the ioremap() return,		 * so we can modify bus.base below.		 */		if (mem->placement & TTM_PL_FLAG_WC)			mem->bus.addr =				ioremap_wc(mem->bus.base + mem->bus.offset,					   mem->bus.size);		else			mem->bus.addr =				ioremap_nocache(mem->bus.base + mem->bus.offset,						mem->bus.size); 		/*		 * Alpha: Use just the bus offset plus		 * the hose/domain memory base for bus.base.		 * It then can be used to build PTEs for VRAM		 * access, as done in ttm_bo_vm_fault().		 */		mem->bus.base = (mem->bus.base & 0x0ffffffffUL) +			adev->ddev->hose->dense_mem_base;#endif		break;	default:		return -EINVAL;	}	return 0;} static void amdgpu_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem){} /* * TTM backend functions. */struct amdgpu_ttm_tt {	struct ttm_dma_tt		ttm;	struct amdgpu_device		*adev;	u64				offset;	uint64_t			userptr;	struct mm_struct		*usermm;	uint32_t			userflags;}; /* prepare the sg table with the user pages */static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm){	struct amdgpu_device *adev = amdgpu_get_adev(ttm->bdev);	struct amdgpu_ttm_tt *gtt = (void *)ttm;	unsigned pinned = 0, nents;	int r; 	int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);	enum dma_data_direction direction = write ?		DMA_BIDIRECTIONAL : DMA_TO_DEVICE;

Lines 1–500 of 1215. Load more