/* * Copyright 2019 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * * based on nouveau_prime.c * * Authors: Alex Deucher */ /** * DOC: PRIME Buffer Sharing * * The following callback implementations are used for :ref:`sharing GEM buffer * objects between different devices via PRIME <prime_buffer_sharing>`. */ #include "amdgpu.h"#include "amdgpu_display.h"#include "amdgpu_gem.h"#include "amdgpu_dma_buf.h"#include "amdgpu_xgmi.h"#include "amdgpu_vm.h"#include "amdgpu_ttm.h"#include <drm/amdgpu_drm.h>#include <drm/ttm/ttm_tt.h>#include <linux/dma-buf.h>#include <linux/dma-fence-array.h>#include <linux/pci-p2pdma.h> static const struct dma_buf_attach_ops amdgpu_dma_buf_attach_ops; /** * dma_buf_attach_adev - Helper to get adev of an attachment * * @attach: attachment * * Returns: * A struct amdgpu_device * if the attaching device is an amdgpu device or * partition, NULL otherwise. */static struct amdgpu_device *dma_buf_attach_adev(struct dma_buf_attachment *attach){	if (attach->importer_ops == &amdgpu_dma_buf_attach_ops) {		struct drm_gem_object *obj = attach->importer_priv;		struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 		return amdgpu_ttm_adev(bo->tbo.bdev);	} 	return NULL;} /** * amdgpu_dma_buf_attach - &dma_buf_ops.attach implementation * * @dmabuf: DMA-buf where we attach to * @attach: attachment to add * * Add the attachment as user to the exported DMA-buf. */static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf,				 struct dma_buf_attachment *attach){	struct amdgpu_device *attach_adev = dma_buf_attach_adev(attach);	struct drm_gem_object *obj = dmabuf->priv;	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);	int r; 	/*	 * Disable peer-to-peer access for DCC-enabled VRAM surfaces on GFX12+.	 * Such buffers cannot be safely accessed over P2P due to device-local	 * compression metadata. Fallback to system-memory path instead.	 * Device supports GFX12 (GC 12.x or newer)	 * BO was created with the AMDGPU_GEM_CREATE_GFX12_DCC flag	 *	 */	if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(12, 0, 0) &&	    bo->flags & AMDGPU_GEM_CREATE_GFX12_DCC)		attach->peer2peer = false; 	if (!amdgpu_dmabuf_is_xgmi_accessible(attach_adev, bo) &&	    pci_p2pdma_distance(adev->pdev, attach->dev, false) < 0)		attach->peer2peer = false; 	r = dma_resv_lock(bo->tbo.base.resv, NULL);	if (r)		return r; 	amdgpu_vm_bo_update_shared(bo); 	dma_resv_unlock(bo->tbo.base.resv); 	return 0;} /** * amdgpu_dma_buf_pin - &dma_buf_ops.pin implementation * * @attach: attachment to pin down * * Pin the BO which is backing the DMA-buf so that it can't move any more. */static int amdgpu_dma_buf_pin(struct dma_buf_attachment *attach){	struct dma_buf *dmabuf = attach->dmabuf;	struct amdgpu_bo *bo = gem_to_amdgpu_bo(dmabuf->priv);	u32 domains = bo->allowed_domains; 	dma_resv_assert_held(dmabuf->resv); 	/* Try pinning into VRAM to allow P2P with RDMA NICs without ODP	 * support if all attachments can do P2P. If any attachment can't do	 * P2P just pin into GTT instead.	 *	 * To avoid with conflicting pinnings between GPUs and RDMA when move	 * notifiers are disabled, only allow pinning in VRAM when move	 * notiers are enabled.	 */	list_for_each_entry(attach, &dmabuf->attachments, node)		if (!attach->peer2peer)			domains &= ~AMDGPU_GEM_DOMAIN_VRAM; 	if (domains & AMDGPU_GEM_DOMAIN_VRAM)		bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 	if (WARN_ON(!domains))		return -EINVAL; 	return amdgpu_bo_pin(bo, domains);} /** * amdgpu_dma_buf_unpin - &dma_buf_ops.unpin implementation * * @attach: attachment to unpin * * Unpin a previously pinned BO to make it movable again. */static void amdgpu_dma_buf_unpin(struct dma_buf_attachment *attach){	struct drm_gem_object *obj = attach->dmabuf->priv;	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 	amdgpu_bo_unpin(bo);} /** * amdgpu_dma_buf_map - &dma_buf_ops.map_dma_buf implementation * @attach: DMA-buf attachment * @dir: DMA direction * * Makes sure that the shared DMA buffer can be accessed by the target device. * For now, simply pins it to the GTT domain, where it should be accessible by * all DMA devices. * * Returns: * sg_table filled with the DMA addresses to use or ERR_PRT with negative error * code. */static struct sg_table *amdgpu_dma_buf_map(struct dma_buf_attachment *attach,					   enum dma_data_direction dir){	struct dma_buf *dma_buf = attach->dmabuf;	struct drm_gem_object *obj = dma_buf->priv;	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);	struct sg_table *sgt;	long r; 	if (!bo->tbo.pin_count) {		/* move buffer into GTT or VRAM */		struct ttm_operation_ctx ctx = { false, false };		unsigned int domains = AMDGPU_GEM_DOMAIN_GTT; 		if (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM &&		    attach->peer2peer) {			bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;			domains |= AMDGPU_GEM_DOMAIN_VRAM;		}		amdgpu_bo_placement_from_domain(bo, domains);		r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);		if (r)			return ERR_PTR(r);	} 	switch (bo->tbo.resource->mem_type) {	case TTM_PL_TT:		sgt = drm_prime_pages_to_sg(obj->dev,					    bo->tbo.ttm->pages,					    bo->tbo.ttm->num_pages);		if (IS_ERR(sgt))			return sgt; 		if (dma_map_sgtable(attach->dev, sgt, dir,				    DMA_ATTR_SKIP_CPU_SYNC))			goto error_free;		break; 	case TTM_PL_VRAM:		/* XGMI-accessible memory should never be DMA-mapped */		if (WARN_ON(amdgpu_dmabuf_is_xgmi_accessible(				dma_buf_attach_adev(attach), bo)))			return ERR_PTR(-EINVAL); 		r = amdgpu_vram_mgr_alloc_sgt(adev, bo->tbo.resource, 0,					      bo->tbo.base.size, attach->dev,					      dir, &sgt);		if (r)			return ERR_PTR(r);		break; 	case AMDGPU_PL_MMIO_REMAP:		r = amdgpu_ttm_mmio_remap_alloc_sgt(adev, bo->tbo.resource,						    attach->dev, dir, &sgt);		if (r)			return ERR_PTR(r);		break; 	default:		return ERR_PTR(-EINVAL);	} 	return sgt; error_free:	sg_free_table(sgt);	kfree(sgt);	return ERR_PTR(-EBUSY);} /** * amdgpu_dma_buf_unmap - &dma_buf_ops.unmap_dma_buf implementation * @attach: DMA-buf attachment * @sgt: sg_table to unmap * @dir: DMA direction * * This is called when a shared DMA buffer no longer needs to be accessible by * another device. For now, simply unpins the buffer from GTT. */static void amdgpu_dma_buf_unmap(struct dma_buf_attachment *attach,				 struct sg_table *sgt,				 enum dma_data_direction dir){	struct drm_gem_object *obj = attach->dmabuf->priv;	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 	if (bo->tbo.resource &&	    bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP) {		amdgpu_ttm_mmio_remap_free_sgt(attach->dev, dir, sgt);		return;	} 	if (sg_page(sgt->sgl)) {		dma_unmap_sgtable(attach->dev, sgt, dir, 0);		sg_free_table(sgt);		kfree(sgt);	} else {		amdgpu_vram_mgr_free_sgt(attach->dev, dir, sgt);	}} /** * amdgpu_dma_buf_begin_cpu_access - &dma_buf_ops.begin_cpu_access implementation * @dma_buf: Shared DMA buffer * @direction: Direction of DMA transfer * * This is called before CPU access to the shared DMA buffer's memory. If it's * a read access, the buffer is moved to the GTT domain if possible, for optimal * CPU read performance. * * Returns: * 0 on success or a negative error code on failure. */static int amdgpu_dma_buf_begin_cpu_access(struct dma_buf *dma_buf,					   enum dma_data_direction direction){	struct amdgpu_bo *bo = gem_to_amdgpu_bo(dma_buf->priv);	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);	struct ttm_operation_ctx ctx = { true, false };	u32 domain = amdgpu_display_supported_domains(adev, bo->flags);	int ret;	bool reads = (direction == DMA_BIDIRECTIONAL ||		      direction == DMA_FROM_DEVICE); 	if (!reads || !(domain & AMDGPU_GEM_DOMAIN_GTT))		return 0; 	/* move to gtt */	ret = amdgpu_bo_reserve(bo, false);	if (unlikely(ret != 0))		return ret; 	if (!bo->tbo.pin_count &&	    (bo->allowed_domains & AMDGPU_GEM_DOMAIN_GTT)) {		amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);		ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);	} 	amdgpu_bo_unreserve(bo);	return ret;} static int amdgpu_dma_buf_vmap(struct dma_buf *dma_buf, struct iosys_map *map){	struct drm_gem_object *obj = dma_buf->priv;	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);	int ret; 	/*	 * Pin to keep buffer in place while it's vmap'ed. The actual	 * domain is not that important as long as it's mapable. Using	 * GTT and VRAM should be compatible with most use cases.	 */	ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT | AMDGPU_GEM_DOMAIN_VRAM);	if (ret)		return ret;	ret = drm_gem_dmabuf_vmap(dma_buf, map);	if (ret)		amdgpu_bo_unpin(bo); 	return ret;} static void amdgpu_dma_buf_vunmap(struct dma_buf *dma_buf, struct iosys_map *map){	struct drm_gem_object *obj = dma_buf->priv;	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 	drm_gem_dmabuf_vunmap(dma_buf, map);	amdgpu_bo_unpin(bo);} const struct dma_buf_ops amdgpu_dmabuf_ops = {	.attach = amdgpu_dma_buf_attach,	.pin = amdgpu_dma_buf_pin,	.unpin = amdgpu_dma_buf_unpin,	.map_dma_buf = amdgpu_dma_buf_map,	.unmap_dma_buf = amdgpu_dma_buf_unmap,	.release = drm_gem_dmabuf_release,	.begin_cpu_access = amdgpu_dma_buf_begin_cpu_access,	.mmap = drm_gem_dmabuf_mmap,	.vmap = amdgpu_dma_buf_vmap,	.vunmap = amdgpu_dma_buf_vunmap,}; /** * amdgpu_gem_prime_export - &drm_driver.gem_prime_export implementation * @gobj: GEM BO * @flags: Flags such as DRM_CLOEXEC and DRM_RDWR. * * The main work is done by the &drm_gem_prime_export helper. * * Returns: * Shared DMA buffer representing the GEM BO from the given device. */struct dma_buf *amdgpu_gem_prime_export(struct drm_gem_object *gobj,					int flags){	struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj);	struct dma_buf *buf;	struct ttm_operation_ctx ctx = {		.interruptible = true,		.no_wait_gpu = true,		/* We opt to avoid OOM on system pages allocations */		.gfp_retry_mayfail = true,		.allow_res_evict = false,	};	int ret; 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) ||	    bo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID)		return ERR_PTR(-EPERM); 	ret = ttm_bo_setup_export(&bo->tbo, &ctx);	if (ret)		return ERR_PTR(ret); 	buf = drm_gem_prime_export(gobj, flags);	if (!IS_ERR(buf))		buf->ops = &amdgpu_dmabuf_ops; 	return buf;} /** * amdgpu_dma_buf_create_obj - create BO for DMA-buf import * * @dev: DRM device * @dma_buf: DMA-buf * * Creates an empty SG BO for DMA-buf import. * * Returns: * A new GEM BO of the given DRM device, representing the memory * described by the given DMA-buf attachment and scatter/gather table. */static struct drm_gem_object *amdgpu_dma_buf_create_obj(struct drm_device *dev, struct dma_buf *dma_buf){	struct dma_resv *resv = dma_buf->resv;	struct amdgpu_device *adev = drm_to_adev(dev);	struct drm_gem_object *gobj;	struct amdgpu_bo *bo;	uint64_t flags = 0;	int ret; 	dma_resv_lock(resv, NULL); 	if (dma_buf->ops == &amdgpu_dmabuf_ops) {		struct amdgpu_bo *other = gem_to_amdgpu_bo(dma_buf->priv); 		flags |= other->flags & (AMDGPU_GEM_CREATE_CPU_GTT_USWC |					 AMDGPU_GEM_CREATE_COHERENT |					 AMDGPU_GEM_CREATE_EXT_COHERENT |					 AMDGPU_GEM_CREATE_UNCACHED);	} 	ret = amdgpu_gem_object_create(adev, dma_buf->size, PAGE_SIZE,				       AMDGPU_GEM_DOMAIN_CPU, flags,				       ttm_bo_type_sg, resv, &gobj, 0);	if (ret)		goto error; 	bo = gem_to_amdgpu_bo(gobj);	bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;	bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT; 	dma_resv_unlock(resv);	return gobj; error:	dma_resv_unlock(resv);	return ERR_PTR(ret);} /** * amdgpu_dma_buf_move_notify - &attach.invalidate_mappings implementation * * @attach: the DMA-buf attachment * * Invalidate the DMA-buf attachment, making sure that the we re-create the * mapping before the next use. */static voidamdgpu_dma_buf_move_notify(struct dma_buf_attachment *attach){	struct drm_gem_object *obj = attach->importer_priv;	struct ww_acquire_ctx *ticket = dma_resv_locking_ctx(obj->resv);	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);	struct ttm_operation_ctx ctx = { false, false };	struct ttm_placement placement = {};	struct amdgpu_vm_bo_base *bo_base;	int r; 	/* FIXME: This should be after the "if", but needs a fix to make sure	 * DMABuf imports are initialized in the right VM list.	 */	amdgpu_vm_bo_invalidate(bo, false);	if (!bo->tbo.resource || bo->tbo.resource->mem_type == TTM_PL_SYSTEM)		return; 	r = ttm_bo_validate(&bo->tbo, &placement, &ctx);	if (r) {		DRM_ERROR("Failed to invalidate DMA-buf import (%d))\n", r);		return;	} 	for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {		struct amdgpu_vm *vm = bo_base->vm;		struct dma_resv *resv = vm->root.bo->tbo.base.resv; 		if (ticket) {			/* When we get an error here it means that somebody			 * else is holding the VM lock and updating page tables			 * So we can just continue here.			 */			r = dma_resv_lock(resv, ticket);			if (r)				continue; 		} else {

Lines 1–500 of 614. Load more