/* * Copyright 2009 Jerome Glisse. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE * USE OR OTHER DEALINGS IN THE SOFTWARE. * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * *//* * Authors: *    Jerome Glisse <glisse@freedesktop.org> *    Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> *    Dave Airlie */#include <linux/list.h>#include <linux/slab.h>#include <drm/drmP.h>#include <drm/amdgpu_drm.h>#include <drm/drm_cache.h>#include "amdgpu.h"#include "amdgpu_trace.h" static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo){	struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);	struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); 	amdgpu_bo_kunmap(bo); 	drm_gem_object_release(&bo->gem_base);	amdgpu_bo_unref(&bo->parent);	if (!list_empty(&bo->shadow_list)) {		mutex_lock(&adev->shadow_list_lock);		list_del_init(&bo->shadow_list);		mutex_unlock(&adev->shadow_list_lock);	}	kfree(bo->metadata);	kfree(bo);} bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo){	if (bo->destroy == &amdgpu_ttm_bo_destroy)		return true;	return false;} void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain){	struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);	struct ttm_placement *placement = &abo->placement;	struct ttm_place *places = abo->placements;	u64 flags = abo->flags;	u32 c = 0; 	if (domain & AMDGPU_GEM_DOMAIN_VRAM) {		unsigned visible_pfn = adev->mc.visible_vram_size >> PAGE_SHIFT; 		places[c].fpfn = 0;		places[c].lpfn = 0;		places[c].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |			TTM_PL_FLAG_VRAM; 		if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)			places[c].lpfn = visible_pfn;		else			places[c].flags |= TTM_PL_FLAG_TOPDOWN; 		if (flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)			places[c].flags |= TTM_PL_FLAG_CONTIGUOUS;		c++;	} 	if (domain & AMDGPU_GEM_DOMAIN_GTT) {		places[c].fpfn = 0;		if (flags & AMDGPU_GEM_CREATE_SHADOW)			places[c].lpfn = adev->mc.gart_size >> PAGE_SHIFT;		else			places[c].lpfn = 0;		places[c].flags = TTM_PL_FLAG_TT;		if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)			places[c].flags |= TTM_PL_FLAG_WC |				TTM_PL_FLAG_UNCACHED;		else			places[c].flags |= TTM_PL_FLAG_CACHED;		c++;	} 	if (domain & AMDGPU_GEM_DOMAIN_CPU) {		places[c].fpfn = 0;		places[c].lpfn = 0;		places[c].flags = TTM_PL_FLAG_SYSTEM;		if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)			places[c].flags |= TTM_PL_FLAG_WC |				TTM_PL_FLAG_UNCACHED;		else			places[c].flags |= TTM_PL_FLAG_CACHED;		c++;	} 	if (domain & AMDGPU_GEM_DOMAIN_GDS) {		places[c].fpfn = 0;		places[c].lpfn = 0;		places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GDS;		c++;	} 	if (domain & AMDGPU_GEM_DOMAIN_GWS) {		places[c].fpfn = 0;		places[c].lpfn = 0;		places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GWS;		c++;	} 	if (domain & AMDGPU_GEM_DOMAIN_OA) {		places[c].fpfn = 0;		places[c].lpfn = 0;		places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_OA;		c++;	} 	if (!c) {		places[c].fpfn = 0;		places[c].lpfn = 0;		places[c].flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM;		c++;	} 	placement->num_placement = c;	placement->placement = places; 	placement->num_busy_placement = c;	placement->busy_placement = places;} /** * amdgpu_bo_create_reserved - create reserved BO for kernel use * * @adev: amdgpu device object * @size: size for the new BO * @align: alignment for the new BO * @domain: where to place it * @bo_ptr: resulting BO * @gpu_addr: GPU addr of the pinned BO * @cpu_addr: optional CPU address mapping * * Allocates and pins a BO for kernel internal use, and returns it still * reserved. * * Returns 0 on success, negative error code otherwise. */int amdgpu_bo_create_reserved(struct amdgpu_device *adev,			      unsigned long size, int align,			      u32 domain, struct amdgpu_bo **bo_ptr,			      u64 *gpu_addr, void **cpu_addr){	bool free = false;	int r; 	if (!*bo_ptr) {		r = amdgpu_bo_create(adev, size, align, true, domain,				     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |				     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,				     NULL, NULL, 0, bo_ptr);		if (r) {			dev_err(adev->dev, "(%d) failed to allocate kernel bo\n",				r);			return r;		}		free = true;	} 	r = amdgpu_bo_reserve(*bo_ptr, false);	if (r) {		dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r);		goto error_free;	} 	r = amdgpu_bo_pin(*bo_ptr, domain, gpu_addr);	if (r) {		dev_err(adev->dev, "(%d) kernel bo pin failed\n", r);		goto error_unreserve;	} 	if (cpu_addr) {		r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);		if (r) {			dev_err(adev->dev, "(%d) kernel bo map failed\n", r);			goto error_unreserve;		}	} 	return 0; error_unreserve:	amdgpu_bo_unreserve(*bo_ptr); error_free:	if (free)		amdgpu_bo_unref(bo_ptr); 	return r;} /** * amdgpu_bo_create_kernel - create BO for kernel use * * @adev: amdgpu device object * @size: size for the new BO * @align: alignment for the new BO * @domain: where to place it * @bo_ptr: resulting BO * @gpu_addr: GPU addr of the pinned BO * @cpu_addr: optional CPU address mapping * * Allocates and pins a BO for kernel internal use. * * Returns 0 on success, negative error code otherwise. */int amdgpu_bo_create_kernel(struct amdgpu_device *adev,			    unsigned long size, int align,			    u32 domain, struct amdgpu_bo **bo_ptr,			    u64 *gpu_addr, void **cpu_addr){	int r; 	r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr,				      gpu_addr, cpu_addr); 	if (r)		return r; 	amdgpu_bo_unreserve(*bo_ptr); 	return 0;} /** * amdgpu_bo_free_kernel - free BO for kernel use * * @bo: amdgpu BO to free * * unmaps and unpin a BO for kernel internal use. */void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,			   void **cpu_addr){	if (*bo == NULL)		return; 	if (likely(amdgpu_bo_reserve(*bo, true) == 0)) {		if (cpu_addr)			amdgpu_bo_kunmap(*bo); 		amdgpu_bo_unpin(*bo);		amdgpu_bo_unreserve(*bo);	}	amdgpu_bo_unref(bo); 	if (gpu_addr)		*gpu_addr = 0; 	if (cpu_addr)		*cpu_addr = NULL;} static int amdgpu_bo_do_create(struct amdgpu_device *adev,			       unsigned long size, int byte_align,			       bool kernel, u32 domain, u64 flags,			       struct sg_table *sg,			       struct reservation_object *resv,			       uint64_t init_value,			       struct amdgpu_bo **bo_ptr){	struct amdgpu_bo *bo;	enum ttm_bo_type type;	unsigned long page_align;	u64 initial_bytes_moved, bytes_moved;	size_t acc_size;	int r; 	page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT;	size = ALIGN(size, PAGE_SIZE); 	if (kernel) {		type = ttm_bo_type_kernel;	} else if (sg) {		type = ttm_bo_type_sg;	} else {		type = ttm_bo_type_device;	}	*bo_ptr = NULL; 	acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size,				       sizeof(struct amdgpu_bo)); 	bo = kzalloc(sizeof(struct amdgpu_bo), GFP_KERNEL);	if (bo == NULL)		return -ENOMEM;	r = drm_gem_object_init(adev->ddev, &bo->gem_base, size);	if (unlikely(r)) {		kfree(bo);		return r;	}	INIT_LIST_HEAD(&bo->shadow_list);	INIT_LIST_HEAD(&bo->va);	bo->preferred_domains = domain & (AMDGPU_GEM_DOMAIN_VRAM |					 AMDGPU_GEM_DOMAIN_GTT |					 AMDGPU_GEM_DOMAIN_CPU |					 AMDGPU_GEM_DOMAIN_GDS |					 AMDGPU_GEM_DOMAIN_GWS |					 AMDGPU_GEM_DOMAIN_OA);	bo->allowed_domains = bo->preferred_domains;	if (!kernel && bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)		bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT; 	bo->flags = flags; #ifdef CONFIG_X86_32	/* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit	 * See https://bugs.freedesktop.org/show_bug.cgi?id=84627	 */	bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;#elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT)	/* Don't try to enable write-combining when it can't work, or things	 * may be slow	 * See https://bugs.freedesktop.org/show_bug.cgi?id=88758	 */ #ifndef CONFIG_COMPILE_TEST#warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \	 thanks to write-combining#endif 	if (bo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)		DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for "			      "better performance thanks to write-combining\n");	bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;#else	/* For architectures that don't support WC memory,	 * mask out the WC flag from the BO	 */	if (!drm_arch_can_wc_memory())		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;#endif 	bo->tbo.bdev = &adev->mman.bdev;	amdgpu_ttm_placement_from_domain(bo, domain); 	initial_bytes_moved = atomic64_read(&adev->num_bytes_moved);	/* Kernel allocation are uninterruptible */	r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, type,				 &bo->placement, page_align, !kernel, NULL,				 acc_size, sg, resv, &amdgpu_ttm_bo_destroy);	if (unlikely(r != 0))		return r; 	bytes_moved = atomic64_read(&adev->num_bytes_moved) -		      initial_bytes_moved;	if (adev->mc.visible_vram_size < adev->mc.real_vram_size &&	    bo->tbo.mem.mem_type == TTM_PL_VRAM &&	    bo->tbo.mem.start < adev->mc.visible_vram_size >> PAGE_SHIFT)		amdgpu_cs_report_moved_bytes(adev, bytes_moved, bytes_moved);	else		amdgpu_cs_report_moved_bytes(adev, bytes_moved, 0); 	if (kernel)		bo->tbo.priority = 1; 	if (flags & AMDGPU_GEM_CREATE_VRAM_CLEARED &&	    bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) {		struct dma_fence *fence; 		r = amdgpu_fill_buffer(bo, init_value, bo->tbo.resv, &fence);		if (unlikely(r))			goto fail_unreserve; 		amdgpu_bo_fence(bo, fence, false);		dma_fence_put(bo->tbo.moving);		bo->tbo.moving = dma_fence_get(fence);		dma_fence_put(fence);	}	if (!resv)		amdgpu_bo_unreserve(bo);	*bo_ptr = bo; 	trace_amdgpu_bo_create(bo); 	/* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */	if (type == ttm_bo_type_device)		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 	return 0; fail_unreserve:	if (!resv)		ww_mutex_unlock(&bo->tbo.resv->lock);	amdgpu_bo_unref(&bo);	return r;} static int amdgpu_bo_create_shadow(struct amdgpu_device *adev,				   unsigned long size, int byte_align,				   struct amdgpu_bo *bo){	int r; 	if (bo->shadow)		return 0; 	r = amdgpu_bo_do_create(adev, size, byte_align, true,				AMDGPU_GEM_DOMAIN_GTT,				AMDGPU_GEM_CREATE_CPU_GTT_USWC |				AMDGPU_GEM_CREATE_SHADOW,				NULL, bo->tbo.resv, 0,				&bo->shadow);	if (!r) {		bo->shadow->parent = amdgpu_bo_ref(bo);		mutex_lock(&adev->shadow_list_lock);		list_add_tail(&bo->shadow_list, &adev->shadow_list);		mutex_unlock(&adev->shadow_list_lock);	} 	return r;} /* init_value will only take effect when flags contains * AMDGPU_GEM_CREATE_VRAM_CLEARED. */int amdgpu_bo_create(struct amdgpu_device *adev,		     unsigned long size, int byte_align,		     bool kernel, u32 domain, u64 flags,		     struct sg_table *sg,		     struct reservation_object *resv,		     uint64_t init_value,		     struct amdgpu_bo **bo_ptr){	uint64_t parent_flags = flags & ~AMDGPU_GEM_CREATE_SHADOW;	int r; 	r = amdgpu_bo_do_create(adev, size, byte_align, kernel, domain,				parent_flags, sg, resv, init_value, bo_ptr);	if (r)		return r; 	if ((flags & AMDGPU_GEM_CREATE_SHADOW) && amdgpu_need_backup(adev)) {		if (!resv)			WARN_ON(reservation_object_lock((*bo_ptr)->tbo.resv,							NULL)); 		r = amdgpu_bo_create_shadow(adev, size, byte_align, (*bo_ptr)); 		if (!resv)			reservation_object_unlock((*bo_ptr)->tbo.resv); 		if (r)			amdgpu_bo_unref(bo_ptr);	} 	return r;} int amdgpu_bo_backup_to_shadow(struct amdgpu_device *adev,			       struct amdgpu_ring *ring,			       struct amdgpu_bo *bo,			       struct reservation_object *resv,			       struct dma_fence **fence,			       bool direct) {	struct amdgpu_bo *shadow = bo->shadow;	uint64_t bo_addr, shadow_addr;	int r; 	if (!shadow)		return -EINVAL; 	bo_addr = amdgpu_bo_gpu_offset(bo);	shadow_addr = amdgpu_bo_gpu_offset(bo->shadow); 	r = reservation_object_reserve_shared(bo->tbo.resv);	if (r)		goto err;

Lines 1–500 of 991. Load more