/* * Copyright 2009 Jerome Glisse. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE * USE OR OTHER DEALINGS IN THE SOFTWARE. * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * *//* * Authors: *    Jerome Glisse <glisse@freedesktop.org> *    Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> *    Dave Airlie */#include <linux/list.h>#include <linux/slab.h>#include <drm/drmP.h>#include <drm/amdgpu_drm.h>#include <drm/drm_cache.h>#include "amdgpu.h"#include "amdgpu_trace.h"#include "amdgpu_amdkfd.h" static bool amdgpu_need_backup(struct amdgpu_device *adev){	if (adev->flags & AMD_IS_APU)		return false; 	if (amdgpu_gpu_recovery == 0 ||	    (amdgpu_gpu_recovery == -1  && !amdgpu_sriov_vf(adev)))		return false; 	return true;} static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo){	struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);	struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); 	if (bo->kfd_bo)		amdgpu_amdkfd_unreserve_system_memory_limit(bo); 	amdgpu_bo_kunmap(bo); 	if (bo->gem_base.import_attach)		drm_prime_gem_destroy(&bo->gem_base, bo->tbo.sg);	drm_gem_object_release(&bo->gem_base);	amdgpu_bo_unref(&bo->parent);	if (!list_empty(&bo->shadow_list)) {		mutex_lock(&adev->shadow_list_lock);		list_del_init(&bo->shadow_list);		mutex_unlock(&adev->shadow_list_lock);	}	kfree(bo->metadata);	kfree(bo);} bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo){	if (bo->destroy == &amdgpu_ttm_bo_destroy)		return true;	return false;} void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain){	struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);	struct ttm_placement *placement = &abo->placement;	struct ttm_place *places = abo->placements;	u64 flags = abo->flags;	u32 c = 0; 	if (domain & AMDGPU_GEM_DOMAIN_VRAM) {		unsigned visible_pfn = adev->gmc.visible_vram_size >> PAGE_SHIFT; 		places[c].fpfn = 0;		places[c].lpfn = 0;		places[c].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |			TTM_PL_FLAG_VRAM; 		if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)			places[c].lpfn = visible_pfn;		else			places[c].flags |= TTM_PL_FLAG_TOPDOWN; 		if (flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)			places[c].flags |= TTM_PL_FLAG_CONTIGUOUS;		c++;	} 	if (domain & AMDGPU_GEM_DOMAIN_GTT) {		places[c].fpfn = 0;		if (flags & AMDGPU_GEM_CREATE_SHADOW)			places[c].lpfn = adev->gmc.gart_size >> PAGE_SHIFT;		else			places[c].lpfn = 0;		places[c].flags = TTM_PL_FLAG_TT;		if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)			places[c].flags |= TTM_PL_FLAG_WC |				TTM_PL_FLAG_UNCACHED;		else			places[c].flags |= TTM_PL_FLAG_CACHED;		c++;	} 	if (domain & AMDGPU_GEM_DOMAIN_CPU) {		places[c].fpfn = 0;		places[c].lpfn = 0;		places[c].flags = TTM_PL_FLAG_SYSTEM;		if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)			places[c].flags |= TTM_PL_FLAG_WC |				TTM_PL_FLAG_UNCACHED;		else			places[c].flags |= TTM_PL_FLAG_CACHED;		c++;	} 	if (domain & AMDGPU_GEM_DOMAIN_GDS) {		places[c].fpfn = 0;		places[c].lpfn = 0;		places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GDS;		c++;	} 	if (domain & AMDGPU_GEM_DOMAIN_GWS) {		places[c].fpfn = 0;		places[c].lpfn = 0;		places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GWS;		c++;	} 	if (domain & AMDGPU_GEM_DOMAIN_OA) {		places[c].fpfn = 0;		places[c].lpfn = 0;		places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_OA;		c++;	} 	if (!c) {		places[c].fpfn = 0;		places[c].lpfn = 0;		places[c].flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM;		c++;	} 	placement->num_placement = c;	placement->placement = places; 	placement->num_busy_placement = c;	placement->busy_placement = places;} /** * amdgpu_bo_create_reserved - create reserved BO for kernel use * * @adev: amdgpu device object * @size: size for the new BO * @align: alignment for the new BO * @domain: where to place it * @bo_ptr: used to initialize BOs in structures * @gpu_addr: GPU addr of the pinned BO * @cpu_addr: optional CPU address mapping * * Allocates and pins a BO for kernel internal use, and returns it still * reserved. * * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. * * Returns 0 on success, negative error code otherwise. */int amdgpu_bo_create_reserved(struct amdgpu_device *adev,			      unsigned long size, int align,			      u32 domain, struct amdgpu_bo **bo_ptr,			      u64 *gpu_addr, void **cpu_addr){	struct amdgpu_bo_param bp;	bool free = false;	int r; 	memset(&bp, 0, sizeof(bp));	bp.size = size;	bp.byte_align = align;	bp.domain = domain;	bp.flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |		AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;	bp.type = ttm_bo_type_kernel;	bp.resv = NULL; 	if (!*bo_ptr) {		r = amdgpu_bo_create(adev, &bp, bo_ptr);		if (r) {			dev_err(adev->dev, "(%d) failed to allocate kernel bo\n",				r);			return r;		}		free = true;	} 	r = amdgpu_bo_reserve(*bo_ptr, false);	if (r) {		dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r);		goto error_free;	} 	r = amdgpu_bo_pin(*bo_ptr, domain, gpu_addr);	if (r) {		dev_err(adev->dev, "(%d) kernel bo pin failed\n", r);		goto error_unreserve;	} 	if (cpu_addr) {		r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);		if (r) {			dev_err(adev->dev, "(%d) kernel bo map failed\n", r);			goto error_unreserve;		}	} 	return 0; error_unreserve:	amdgpu_bo_unreserve(*bo_ptr); error_free:	if (free)		amdgpu_bo_unref(bo_ptr); 	return r;} /** * amdgpu_bo_create_kernel - create BO for kernel use * * @adev: amdgpu device object * @size: size for the new BO * @align: alignment for the new BO * @domain: where to place it * @bo_ptr:  used to initialize BOs in structures * @gpu_addr: GPU addr of the pinned BO * @cpu_addr: optional CPU address mapping * * Allocates and pins a BO for kernel internal use. * * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. * * Returns 0 on success, negative error code otherwise. */int amdgpu_bo_create_kernel(struct amdgpu_device *adev,			    unsigned long size, int align,			    u32 domain, struct amdgpu_bo **bo_ptr,			    u64 *gpu_addr, void **cpu_addr){	int r; 	r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr,				      gpu_addr, cpu_addr); 	if (r)		return r; 	amdgpu_bo_unreserve(*bo_ptr); 	return 0;} /** * amdgpu_bo_free_kernel - free BO for kernel use * * @bo: amdgpu BO to free * * unmaps and unpin a BO for kernel internal use. */void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,			   void **cpu_addr){	if (*bo == NULL)		return; 	if (likely(amdgpu_bo_reserve(*bo, true) == 0)) {		if (cpu_addr)			amdgpu_bo_kunmap(*bo); 		amdgpu_bo_unpin(*bo);		amdgpu_bo_unreserve(*bo);	}	amdgpu_bo_unref(bo); 	if (gpu_addr)		*gpu_addr = 0; 	if (cpu_addr)		*cpu_addr = NULL;} /* Validate bo size is bit bigger then the request domain */static bool amdgpu_bo_validate_size(struct amdgpu_device *adev,					  unsigned long size, u32 domain){	struct ttm_mem_type_manager *man = NULL; 	/*	 * If GTT is part of requested domains the check must succeed to	 * allow fall back to GTT	 */	if (domain & AMDGPU_GEM_DOMAIN_GTT) {		man = &adev->mman.bdev.man[TTM_PL_TT]; 		if (size < (man->size << PAGE_SHIFT))			return true;		else			goto fail;	} 	if (domain & AMDGPU_GEM_DOMAIN_VRAM) {		man = &adev->mman.bdev.man[TTM_PL_VRAM]; 		if (size < (man->size << PAGE_SHIFT))			return true;		else			goto fail;	}  	/* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU */	return true; fail:	DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size,		  man->size << PAGE_SHIFT);	return false;} static int amdgpu_bo_do_create(struct amdgpu_device *adev,			       struct amdgpu_bo_param *bp,			       struct amdgpu_bo **bo_ptr){	struct ttm_operation_ctx ctx = {		.interruptible = (bp->type != ttm_bo_type_kernel),		.no_wait_gpu = false,		.resv = bp->resv,		.flags = TTM_OPT_FLAG_ALLOW_RES_EVICT	};	struct amdgpu_bo *bo;	unsigned long page_align, size = bp->size;	size_t acc_size;	int r; 	page_align = roundup(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT;	size = ALIGN(size, PAGE_SIZE); 	if (!amdgpu_bo_validate_size(adev, size, bp->domain))		return -ENOMEM; 	*bo_ptr = NULL; 	acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size,				       sizeof(struct amdgpu_bo)); 	bo = kzalloc(sizeof(struct amdgpu_bo), GFP_KERNEL);	if (bo == NULL)		return -ENOMEM;	drm_gem_private_object_init(adev->ddev, &bo->gem_base, size);	INIT_LIST_HEAD(&bo->shadow_list);	INIT_LIST_HEAD(&bo->va);	bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain :		bp->domain;	bo->allowed_domains = bo->preferred_domains;	if (bp->type != ttm_bo_type_kernel &&	    bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)		bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT; 	bo->flags = bp->flags; #ifdef CONFIG_X86_32	/* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit	 * See https://bugs.freedesktop.org/show_bug.cgi?id=84627	 */	bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;#elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT)	/* Don't try to enable write-combining when it can't work, or things	 * may be slow	 * See https://bugs.freedesktop.org/show_bug.cgi?id=88758	 */ #ifndef CONFIG_COMPILE_TEST#warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \	 thanks to write-combining#endif 	if (bo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)		DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for "			      "better performance thanks to write-combining\n");	bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;#else	/* For architectures that don't support WC memory,	 * mask out the WC flag from the BO	 */	if (!drm_arch_can_wc_memory())		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;#endif 	bo->tbo.bdev = &adev->mman.bdev;	amdgpu_ttm_placement_from_domain(bo, bp->domain);	if (bp->type == ttm_bo_type_kernel)		bo->tbo.priority = 1; 	r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, bp->type,				 &bo->placement, page_align, &ctx, acc_size,				 NULL, bp->resv, &amdgpu_ttm_bo_destroy);	if (unlikely(r != 0))		return r; 	if (adev->gmc.visible_vram_size < adev->gmc.real_vram_size &&	    bo->tbo.mem.mem_type == TTM_PL_VRAM &&	    bo->tbo.mem.start < adev->gmc.visible_vram_size >> PAGE_SHIFT)		amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved,					     ctx.bytes_moved);	else		amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0); 	if (bp->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED &&	    bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) {		struct dma_fence *fence; 		r = amdgpu_fill_buffer(bo, 0, bo->tbo.resv, &fence);		if (unlikely(r))			goto fail_unreserve; 		amdgpu_bo_fence(bo, fence, false);		dma_fence_put(bo->tbo.moving);		bo->tbo.moving = dma_fence_get(fence);		dma_fence_put(fence);	}	if (!bp->resv)		amdgpu_bo_unreserve(bo);	*bo_ptr = bo; 	trace_amdgpu_bo_create(bo); 	/* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */	if (bp->type == ttm_bo_type_device)		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 	return 0; fail_unreserve:	if (!bp->resv)		ww_mutex_unlock(&bo->tbo.resv->lock);	amdgpu_bo_unref(&bo);	return r;} static int amdgpu_bo_create_shadow(struct amdgpu_device *adev,				   unsigned long size, int byte_align,				   struct amdgpu_bo *bo){	struct amdgpu_bo_param bp;	int r; 	if (bo->shadow)		return 0; 	memset(&bp, 0, sizeof(bp));	bp.size = size;	bp.byte_align = byte_align;	bp.domain = AMDGPU_GEM_DOMAIN_GTT;	bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC |		AMDGPU_GEM_CREATE_SHADOW;	bp.type = ttm_bo_type_kernel;	bp.resv = bo->tbo.resv; 	r = amdgpu_bo_do_create(adev, &bp, &bo->shadow);	if (!r) {		bo->shadow->parent = amdgpu_bo_ref(bo);		mutex_lock(&adev->shadow_list_lock);		list_add_tail(&bo->shadow_list, &adev->shadow_list);		mutex_unlock(&adev->shadow_list_lock);	} 	return r;} 

Lines 1–500 of 1071. Load more