// SPDX-License-Identifier: GPL-2.0-only/* * NVIDIA Tegra DRM GEM helper functions * * Copyright (C) 2012 Sascha Hauer, Pengutronix * Copyright (C) 2013-2015 NVIDIA CORPORATION, All rights reserved. * * Based on the GEM/CMA helpers * * Copyright (c) 2011 Samsung Electronics Co., Ltd. */ #include <linux/dma-buf.h>#include <linux/iommu.h>#include <linux/module.h>#include <linux/pagemap.h>#include <linux/vmalloc.h> #include <drm/drm_drv.h>#include <drm/drm_dumb_buffers.h>#include <drm/drm_prime.h> #include "drm.h"#include "gem.h" MODULE_IMPORT_NS("DMA_BUF"); static unsigned int sg_dma_count_chunks(struct scatterlist *sgl, unsigned int nents){	dma_addr_t next = ~(dma_addr_t)0;	unsigned int count = 0, i;	struct scatterlist *s; 	for_each_sg(sgl, s, nents, i) {		/* sg_dma_address(s) is only valid for entries that have sg_dma_len(s) != 0. */		if (!sg_dma_len(s))			continue; 		if (sg_dma_address(s) != next) {			next = sg_dma_address(s) + sg_dma_len(s);			count++;		}	} 	return count;} static inline unsigned int sgt_dma_count_chunks(struct sg_table *sgt){	return sg_dma_count_chunks(sgt->sgl, sgt->nents);} static void tegra_bo_put(struct host1x_bo *bo){	struct tegra_bo *obj = host1x_to_tegra_bo(bo); 	drm_gem_object_put(&obj->gem);} static struct host1x_bo_mapping *tegra_bo_pin(struct device *dev, struct host1x_bo *bo,					      enum dma_data_direction direction){	struct tegra_bo *obj = host1x_to_tegra_bo(bo);	struct drm_gem_object *gem = &obj->gem;	struct host1x_bo_mapping *map;	int err; 	map = kzalloc_obj(*map);	if (!map)		return ERR_PTR(-ENOMEM); 	kref_init(&map->ref);	map->bo = bo;	map->direction = direction;	map->dev = dev; 	/*	 * Imported buffers need special treatment to satisfy the semantics of DMA-BUF.	 */	if (obj->dma_buf) {		struct dma_buf *buf = obj->dma_buf; 		map->attach = dma_buf_attach(buf, dev);		if (IS_ERR(map->attach)) {			err = PTR_ERR(map->attach);			goto free;		} 		map->sgt = dma_buf_map_attachment_unlocked(map->attach, direction);		if (IS_ERR(map->sgt)) {			dma_buf_detach(buf, map->attach);			err = PTR_ERR(map->sgt);			map->sgt = NULL;			goto free;		} 		err = sgt_dma_count_chunks(map->sgt);		map->size = gem->size; 		goto out;	} 	/*	 * If we don't have a mapping for this buffer yet, return an SG table	 * so that host1x can do the mapping for us via the DMA API.	 */	map->sgt = kzalloc_obj(*map->sgt);	if (!map->sgt) {		err = -ENOMEM;		goto free;	} 	if (obj->pages) {		/*		 * If the buffer object was allocated from the explicit IOMMU		 * API code paths, construct an SG table from the pages.		 */		err = sg_alloc_table_from_pages(map->sgt, obj->pages, obj->num_pages, 0, gem->size,						GFP_KERNEL);		if (err < 0)			goto free;	} else {		/*		 * If the buffer object had no pages allocated and if it was		 * not imported, it had to be allocated with the DMA API, so		 * the DMA API helper can be used.		 */		err = dma_get_sgtable(dev, map->sgt, obj->vaddr, obj->iova, gem->size);		if (err < 0)			goto free;	} 	err = dma_map_sgtable(dev, map->sgt, direction, 0);	if (err)		goto free_sgt; out:	/*	 * If we've manually mapped the buffer object through the IOMMU, make sure to return the	 * existing IOVA address of our mapping.	 */	if (!obj->mm) {		map->phys = sg_dma_address(map->sgt->sgl);		map->chunks = err;	} else {		map->phys = obj->iova;		map->chunks = 1;	} 	map->size = gem->size; 	return map; free_sgt:	sg_free_table(map->sgt);free:	kfree(map->sgt);	kfree(map);	return ERR_PTR(err);} static void tegra_bo_unpin(struct host1x_bo_mapping *map){	if (map->attach) {		dma_buf_unmap_attachment_unlocked(map->attach, map->sgt,						  map->direction);		dma_buf_detach(map->attach->dmabuf, map->attach);	} else {		dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0);		sg_free_table(map->sgt);		kfree(map->sgt);	} 	kfree(map);} static void *tegra_bo_mmap(struct host1x_bo *bo){	struct tegra_bo *obj = host1x_to_tegra_bo(bo);	struct iosys_map map = { 0 };	void *vaddr;	int ret; 	if (obj->vaddr)		return obj->vaddr; 	if (obj->dma_buf) {		ret = dma_buf_vmap_unlocked(obj->dma_buf, &map);		if (ret < 0)			return ERR_PTR(ret); 		return map.vaddr;	} 	vaddr = vmap(obj->pages, obj->num_pages, VM_MAP,		     pgprot_writecombine(PAGE_KERNEL));	if (!vaddr)		return ERR_PTR(-ENOMEM); 	return vaddr;} static void tegra_bo_munmap(struct host1x_bo *bo, void *addr){	struct tegra_bo *obj = host1x_to_tegra_bo(bo);	struct iosys_map map = IOSYS_MAP_INIT_VADDR(addr); 	if (obj->vaddr)		return; 	if (obj->dma_buf)		return dma_buf_vunmap_unlocked(obj->dma_buf, &map); 	vunmap(addr);} static struct host1x_bo *tegra_bo_get(struct host1x_bo *bo){	struct tegra_bo *obj = host1x_to_tegra_bo(bo); 	drm_gem_object_get(&obj->gem); 	return bo;} static const struct host1x_bo_ops tegra_bo_ops = {	.get = tegra_bo_get,	.put = tegra_bo_put,	.pin = tegra_bo_pin,	.unpin = tegra_bo_unpin,	.mmap = tegra_bo_mmap,	.munmap = tegra_bo_munmap,}; static int tegra_bo_iommu_map(struct tegra_drm *tegra, struct tegra_bo *bo){	int prot = IOMMU_READ | IOMMU_WRITE;	ssize_t size;	int err; 	if (bo->mm)		return -EBUSY; 	bo->mm = kzalloc_obj(*bo->mm);	if (!bo->mm)		return -ENOMEM; 	mutex_lock(&tegra->mm_lock); 	err = drm_mm_insert_node_generic(&tegra->mm,					 bo->mm, bo->gem.size, PAGE_SIZE, 0, 0);	if (err < 0) {		dev_err(tegra->drm->dev, "out of I/O virtual memory: %d\n",			err);		goto unlock;	} 	bo->iova = bo->mm->start; 	size = iommu_map_sgtable(tegra->domain, bo->iova, bo->sgt, prot);	if (size < 0) {		dev_err(tegra->drm->dev, "failed to map buffer\n");		err = size;		goto remove;	} 	bo->size = size; 	mutex_unlock(&tegra->mm_lock); 	return 0; remove:	drm_mm_remove_node(bo->mm);unlock:	mutex_unlock(&tegra->mm_lock);	kfree(bo->mm);	return err;} static int tegra_bo_iommu_unmap(struct tegra_drm *tegra, struct tegra_bo *bo){	if (!bo->mm)		return 0; 	mutex_lock(&tegra->mm_lock);	iommu_unmap(tegra->domain, bo->iova, bo->size);	drm_mm_remove_node(bo->mm);	mutex_unlock(&tegra->mm_lock); 	kfree(bo->mm); 	return 0;} static const struct drm_gem_object_funcs tegra_gem_object_funcs = {	.free = tegra_bo_free_object,	.export = tegra_gem_prime_export,	.vm_ops = &tegra_bo_vm_ops,}; static struct tegra_bo *tegra_bo_alloc_object(struct drm_device *drm,					      size_t size){	struct tegra_bo *bo;	int err; 	bo = kzalloc_obj(*bo);	if (!bo)		return ERR_PTR(-ENOMEM); 	bo->gem.funcs = &tegra_gem_object_funcs; 	host1x_bo_init(&bo->base, &tegra_bo_ops);	size = round_up(size, PAGE_SIZE); 	err = drm_gem_object_init(drm, &bo->gem, size);	if (err < 0)		goto free; 	err = drm_gem_create_mmap_offset(&bo->gem);	if (err < 0)		goto release; 	return bo; release:	drm_gem_object_release(&bo->gem);free:	kfree(bo);	return ERR_PTR(err);} static void tegra_bo_free(struct drm_device *drm, struct tegra_bo *bo){	if (bo->pages) {		dma_unmap_sgtable(drm->dev, bo->sgt, DMA_FROM_DEVICE, 0);		drm_gem_put_pages(&bo->gem, bo->pages, true, true);		sg_free_table(bo->sgt);		kfree(bo->sgt);	} else if (bo->vaddr) {		dma_free_wc(drm->dev, bo->gem.size, bo->vaddr, bo->iova);	}} static int tegra_bo_get_pages(struct drm_device *drm, struct tegra_bo *bo){	int err; 	bo->pages = drm_gem_get_pages(&bo->gem);	if (IS_ERR(bo->pages))		return PTR_ERR(bo->pages); 	bo->num_pages = bo->gem.size >> PAGE_SHIFT; 	bo->sgt = drm_prime_pages_to_sg(bo->gem.dev, bo->pages, bo->num_pages);	if (IS_ERR(bo->sgt)) {		err = PTR_ERR(bo->sgt);		goto put_pages;	} 	err = dma_map_sgtable(drm->dev, bo->sgt, DMA_FROM_DEVICE, 0);	if (err)		goto free_sgt; 	return 0; free_sgt:	sg_free_table(bo->sgt);	kfree(bo->sgt);put_pages:	drm_gem_put_pages(&bo->gem, bo->pages, false, false);	return err;} static int tegra_bo_alloc(struct drm_device *drm, struct tegra_bo *bo){	struct tegra_drm *tegra = drm->dev_private;	int err; 	if (tegra->domain) {		err = tegra_bo_get_pages(drm, bo);		if (err < 0)			return err; 		err = tegra_bo_iommu_map(tegra, bo);		if (err < 0) {			tegra_bo_free(drm, bo);			return err;		}	} else {		size_t size = bo->gem.size; 		bo->vaddr = dma_alloc_wc(drm->dev, size, &bo->iova,					 GFP_KERNEL | __GFP_NOWARN);		if (!bo->vaddr) {			dev_err(drm->dev,				"failed to allocate buffer of size %zu\n",				size);			return -ENOMEM;		}	} 	return 0;} struct tegra_bo *tegra_bo_create(struct drm_device *drm, size_t size,				 unsigned long flags){	struct tegra_bo *bo;	int err; 	bo = tegra_bo_alloc_object(drm, size);	if (IS_ERR(bo))		return bo; 	err = tegra_bo_alloc(drm, bo);	if (err < 0)		goto release; 	if (flags & DRM_TEGRA_GEM_CREATE_TILED)		bo->tiling.mode = TEGRA_BO_TILING_MODE_TILED; 	if (flags & DRM_TEGRA_GEM_CREATE_BOTTOM_UP)		bo->flags |= TEGRA_BO_BOTTOM_UP; 	return bo; release:	drm_gem_object_release(&bo->gem);	kfree(bo);	return ERR_PTR(err);} struct tegra_bo *tegra_bo_create_with_handle(struct drm_file *file,					     struct drm_device *drm,					     size_t size,					     unsigned long flags,					     u32 *handle){	struct tegra_bo *bo;	int err; 	bo = tegra_bo_create(drm, size, flags);	if (IS_ERR(bo))		return bo; 	err = drm_gem_handle_create(file, &bo->gem, handle);	if (err) {		tegra_bo_free_object(&bo->gem);		return ERR_PTR(err);	} 	drm_gem_object_put(&bo->gem); 	return bo;} static struct tegra_bo *tegra_bo_import(struct drm_device *drm,					struct dma_buf *buf){	struct tegra_drm *tegra = drm->dev_private;	struct dma_buf_attachment *attach;	struct tegra_bo *bo;	int err; 	bo = tegra_bo_alloc_object(drm, buf->size);	if (IS_ERR(bo))		return bo; 	/*	 * If we need to use IOMMU API to map the dma-buf into the internally managed	 * domain, map it first to the DRM device to get an sgt.	 */	if (tegra->domain) {		attach = dma_buf_attach(buf, drm->dev);		if (IS_ERR(attach)) {			err = PTR_ERR(attach);			goto free;		} 		bo->sgt = dma_buf_map_attachment_unlocked(attach, DMA_TO_DEVICE);		if (IS_ERR(bo->sgt)) {			err = PTR_ERR(bo->sgt);			goto detach;		} 		err = tegra_bo_iommu_map(tegra, bo);		if (err < 0)			goto detach; 		bo->gem.import_attach = attach;	} 	get_dma_buf(buf);	bo->dma_buf = buf; 	return bo; detach:

Lines 1–500 of 800. Load more