// SPDX-License-Identifier: GPL-2.0// Copyright (c) 2011-2018, The Linux Foundation. All rights reserved.// Copyright (c) 2018, Linaro Limited #include <linux/completion.h>#include <linux/device.h>#include <linux/dma-buf.h>#include <linux/dma-mapping.h>#include <linux/dma-resv.h>#include <linux/idr.h>#include <linux/list.h>#include <linux/miscdevice.h>#include <linux/module.h>#include <linux/of_address.h>#include <linux/of.h>#include <linux/platform_device.h>#include <linux/sort.h>#include <linux/of_platform.h>#include <linux/rpmsg.h>#include <linux/scatterlist.h>#include <linux/slab.h>#include <linux/firmware/qcom/qcom_scm.h>#include <uapi/misc/fastrpc.h>#include <linux/of_reserved_mem.h> #define ADSP_DOMAIN_ID (0)#define MDSP_DOMAIN_ID (1)#define SDSP_DOMAIN_ID (2)#define CDSP_DOMAIN_ID (3)#define CDSP1_DOMAIN_ID (4)#define FASTRPC_DEV_MAX 5 /* adsp, mdsp, slpi, cdsp, cdsp1 */#define FASTRPC_MAX_SESSIONS 14#define FASTRPC_MAX_VMIDS 16#define FASTRPC_ALIGN 128#define FASTRPC_MAX_FDLIST 16#define FASTRPC_MAX_CRCLIST 64#define FASTRPC_PHYS(p) ((p) & 0xffffffff)#define FASTRPC_CTX_MAX (256)#define FASTRPC_INIT_HANDLE 1#define FASTRPC_DSP_UTILITIES_HANDLE 2#define FASTRPC_CTXID_MASK (0xFF0)#define INIT_FILELEN_MAX (2 * 1024 * 1024)#define INIT_FILE_NAMELEN_MAX (128)#define FASTRPC_DEVICE_NAME "fastrpc" /* Add memory to static PD pool, protection thru XPU */#define ADSP_MMAP_HEAP_ADDR 4/* MAP static DMA buffer on DSP User PD */#define ADSP_MMAP_DMA_BUFFER 6/* Add memory to static PD pool protection thru hypervisor */#define ADSP_MMAP_REMOTE_HEAP_ADDR 8/* Add memory to userPD pool, for user heap */#define ADSP_MMAP_ADD_PAGES 0x1000/* Add memory to userPD pool, for LLC heap */#define ADSP_MMAP_ADD_PAGES_LLC 0x3000, #define DSP_UNSUPPORTED_API (0x80000414)/* MAX NUMBER of DSP ATTRIBUTES SUPPORTED */#define FASTRPC_MAX_DSP_ATTRIBUTES (256)#define FASTRPC_MAX_DSP_ATTRIBUTES_LEN (sizeof(u32) * FASTRPC_MAX_DSP_ATTRIBUTES) /* Retrives number of input buffers from the scalars parameter */#define REMOTE_SCALARS_INBUFS(sc) (((sc) >> 16) & 0x0ff) /* Retrives number of output buffers from the scalars parameter */#define REMOTE_SCALARS_OUTBUFS(sc) (((sc) >> 8) & 0x0ff) /* Retrives number of input handles from the scalars parameter */#define REMOTE_SCALARS_INHANDLES(sc) (((sc) >> 4) & 0x0f) /* Retrives number of output handles from the scalars parameter */#define REMOTE_SCALARS_OUTHANDLES(sc) ((sc) & 0x0f) #define REMOTE_SCALARS_LENGTH(sc) (REMOTE_SCALARS_INBUFS(sc) + \ REMOTE_SCALARS_OUTBUFS(sc) + \ REMOTE_SCALARS_INHANDLES(sc)+ \ REMOTE_SCALARS_OUTHANDLES(sc))#define FASTRPC_BUILD_SCALARS(attr, method, in, out, oin, oout) \ (((attr & 0x07) << 29) | \ ((method & 0x1f) << 24) | \ ((in & 0xff) << 16) | \ ((out & 0xff) << 8) | \ ((oin & 0x0f) << 4) | \ (oout & 0x0f)) #define FASTRPC_SCALARS(method, in, out) \ FASTRPC_BUILD_SCALARS(0, method, in, out, 0, 0) #define FASTRPC_CREATE_PROCESS_NARGS 6#define FASTRPC_CREATE_STATIC_PROCESS_NARGS 3/* Remote Method id table */#define FASTRPC_RMID_INIT_ATTACH 0#define FASTRPC_RMID_INIT_RELEASE 1#define FASTRPC_RMID_INIT_MMAP 4#define FASTRPC_RMID_INIT_MUNMAP 5#define FASTRPC_RMID_INIT_CREATE 6#define FASTRPC_RMID_INIT_CREATE_ATTR 7#define FASTRPC_RMID_INIT_CREATE_STATIC 8#define FASTRPC_RMID_INIT_MEM_MAP 10#define FASTRPC_RMID_INIT_MEM_UNMAP 11 /* Protection Domain(PD) ids */#define ROOT_PD (0)#define USER_PD (1)#define SENSORS_PD (2) #define miscdev_to_fdevice(d) container_of(d, struct fastrpc_device, miscdev) static const char *domains[FASTRPC_DEV_MAX] = { "adsp", "mdsp", "sdsp", "cdsp", "cdsp1" };struct fastrpc_phy_page { u64 addr; /* physical address */ u64 size; /* size of contiguous region */}; struct fastrpc_invoke_buf { u32 num; /* number of contiguous regions */ u32 pgidx; /* index to start of contiguous region */}; struct fastrpc_remote_dmahandle { s32 fd; /* dma handle fd */ u32 offset; /* dma handle offset */ u32 len; /* dma handle length */}; struct fastrpc_remote_buf { u64 pv; /* buffer pointer */ u64 len; /* length of buffer */}; union fastrpc_remote_arg { struct fastrpc_remote_buf buf; struct fastrpc_remote_dmahandle dma;}; struct fastrpc_mmap_rsp_msg { u64 vaddr;}; struct fastrpc_mmap_req_msg { s32 pgid; u32 flags; u64 vaddr; s32 num;}; struct fastrpc_mem_map_req_msg { s32 pgid; s32 fd; s32 offset; u32 flags; u64 vaddrin; s32 num; s32 data_len;}; struct fastrpc_munmap_req_msg { s32 pgid; u64 vaddr; u64 size;}; struct fastrpc_mem_unmap_req_msg { s32 pgid; s32 fd; u64 vaddrin; u64 len;}; struct fastrpc_msg { int pid; /* process group id */ int tid; /* thread id */ u64 ctx; /* invoke caller context */ u32 handle; /* handle to invoke */ u32 sc; /* scalars structure describing the data */ u64 addr; /* physical address */ u64 size; /* size of contiguous region */}; struct fastrpc_invoke_rsp { u64 ctx; /* invoke caller context */ int retval; /* invoke return value */}; struct fastrpc_buf_overlap { u64 start; u64 end; int raix; u64 mstart; u64 mend; u64 offset;}; struct fastrpc_buf { struct fastrpc_user *fl; struct dma_buf *dmabuf; struct device *dev; void *virt; u64 phys; u64 size; /* Lock for dma buf attachments */ struct mutex lock; struct list_head attachments; /* mmap support */ struct list_head node; /* list of user requested mmaps */ uintptr_t raddr;}; struct fastrpc_dma_buf_attachment { struct device *dev; struct sg_table sgt; struct list_head node;}; struct fastrpc_map { struct list_head node; struct fastrpc_user *fl; int fd; struct dma_buf *buf; struct sg_table *table; struct dma_buf_attachment *attach; u64 phys; u64 size; void *va; u64 len; u64 raddr; u32 attr; struct kref refcount;}; struct fastrpc_invoke_ctx { int nscalars; int nbufs; int retval; int pid; int tgid; u32 sc; u32 *crc; u64 ctxid; u64 msg_sz; struct kref refcount; struct list_head node; /* list of ctxs */ struct completion work; struct work_struct put_work; struct fastrpc_msg msg; struct fastrpc_user *fl; union fastrpc_remote_arg *rpra; struct fastrpc_map **maps; struct fastrpc_buf *buf; struct fastrpc_invoke_args *args; struct fastrpc_buf_overlap *olaps; struct fastrpc_channel_ctx *cctx;}; struct fastrpc_session_ctx { struct device *dev; int sid; bool used; bool valid;}; struct fastrpc_channel_ctx { int domain_id; int sesscount; int vmcount; struct qcom_scm_vmperm vmperms[FASTRPC_MAX_VMIDS]; struct rpmsg_device *rpdev; struct fastrpc_session_ctx session[FASTRPC_MAX_SESSIONS]; spinlock_t lock; struct idr ctx_idr; struct list_head users; struct kref refcount; /* Flag if dsp attributes are cached */ bool valid_attributes; u32 dsp_attributes[FASTRPC_MAX_DSP_ATTRIBUTES]; struct fastrpc_device *secure_fdevice; struct fastrpc_device *fdevice; struct fastrpc_buf *remote_heap; struct list_head invoke_interrupted_mmaps; bool secure; bool unsigned_support; u64 dma_mask;}; struct fastrpc_device { struct fastrpc_channel_ctx *cctx; struct miscdevice miscdev; bool secure;}; struct fastrpc_user { struct list_head user; struct list_head maps; struct list_head pending; struct list_head mmaps; struct fastrpc_channel_ctx *cctx; struct fastrpc_session_ctx *sctx; struct fastrpc_buf *init_mem; int tgid; int pd; bool is_secure_dev; /* Lock for lists */ spinlock_t lock; /* lock for allocations */ struct mutex mutex;}; static void fastrpc_free_map(struct kref *ref){ struct fastrpc_map *map; map = container_of(ref, struct fastrpc_map, refcount); if (map->table) { if (map->attr & FASTRPC_ATTR_SECUREMAP) { struct qcom_scm_vmperm perm; int vmid = map->fl->cctx->vmperms[0].vmid; u64 src_perms = BIT(QCOM_SCM_VMID_HLOS) | BIT(vmid); int err = 0; perm.vmid = QCOM_SCM_VMID_HLOS; perm.perm = QCOM_SCM_PERM_RWX; err = qcom_scm_assign_mem(map->phys, map->size, &src_perms, &perm, 1); if (err) { dev_err(map->fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d\n", map->phys, map->size, err); return; } } dma_buf_unmap_attachment_unlocked(map->attach, map->table, DMA_BIDIRECTIONAL); dma_buf_detach(map->buf, map->attach); dma_buf_put(map->buf); } if (map->fl) { spin_lock(&map->fl->lock); list_del(&map->node); spin_unlock(&map->fl->lock); map->fl = NULL; } kfree(map);} static void fastrpc_map_put(struct fastrpc_map *map){ if (map) kref_put(&map->refcount, fastrpc_free_map);} static int fastrpc_map_get(struct fastrpc_map *map){ if (!map) return -ENOENT; return kref_get_unless_zero(&map->refcount) ? 0 : -ENOENT;} static int fastrpc_map_lookup(struct fastrpc_user *fl, int fd, struct fastrpc_map **ppmap, bool take_ref){ struct fastrpc_session_ctx *sess = fl->sctx; struct fastrpc_map *map = NULL; int ret = -ENOENT; spin_lock(&fl->lock); list_for_each_entry(map, &fl->maps, node) { if (map->fd != fd) continue; if (take_ref) { ret = fastrpc_map_get(map); if (ret) { dev_dbg(sess->dev, "%s: Failed to get map fd=%d ret=%d\n", __func__, fd, ret); break; } } *ppmap = map; ret = 0; break; } spin_unlock(&fl->lock); return ret;} static void fastrpc_buf_free(struct fastrpc_buf *buf){ dma_free_coherent(buf->dev, buf->size, buf->virt, FASTRPC_PHYS(buf->phys)); kfree(buf);} static int __fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev, u64 size, struct fastrpc_buf **obuf){ struct fastrpc_buf *buf; buf = kzalloc(sizeof(*buf), GFP_KERNEL); if (!buf) return -ENOMEM; INIT_LIST_HEAD(&buf->attachments); INIT_LIST_HEAD(&buf->node); mutex_init(&buf->lock); buf->fl = fl; buf->virt = NULL; buf->phys = 0; buf->size = size; buf->dev = dev; buf->raddr = 0; buf->virt = dma_alloc_coherent(dev, buf->size, (dma_addr_t *)&buf->phys, GFP_KERNEL); if (!buf->virt) { mutex_destroy(&buf->lock); kfree(buf); return -ENOMEM; } *obuf = buf; return 0;} static int fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev, u64 size, struct fastrpc_buf **obuf){ int ret; struct fastrpc_buf *buf; ret = __fastrpc_buf_alloc(fl, dev, size, obuf); if (ret) return ret; buf = *obuf; if (fl->sctx && fl->sctx->sid) buf->phys += ((u64)fl->sctx->sid << 32); return 0;} static int fastrpc_remote_heap_alloc(struct fastrpc_user *fl, struct device *dev, u64 size, struct fastrpc_buf **obuf){ struct device *rdev = &fl->cctx->rpdev->dev; return __fastrpc_buf_alloc(fl, rdev, size, obuf);} static void fastrpc_channel_ctx_free(struct kref *ref){ struct fastrpc_channel_ctx *cctx; cctx = container_of(ref, struct fastrpc_channel_ctx, refcount); kfree(cctx);} static void fastrpc_channel_ctx_get(struct fastrpc_channel_ctx *cctx){ kref_get(&cctx->refcount);} static void fastrpc_channel_ctx_put(struct fastrpc_channel_ctx *cctx){ kref_put(&cctx->refcount, fastrpc_channel_ctx_free);} static void fastrpc_context_free(struct kref *ref){ struct fastrpc_invoke_ctx *ctx; struct fastrpc_channel_ctx *cctx; unsigned long flags; int i; ctx = container_of(ref, struct fastrpc_invoke_ctx, refcount); cctx = ctx->cctx; for (i = 0; i < ctx->nbufs; i++) fastrpc_map_put(ctx->maps[i]); if (ctx->buf) fastrpc_buf_free(ctx->buf); spin_lock_irqsave(&cctx->lock, flags); idr_remove(&cctx->ctx_idr, ctx->ctxid >> 4); spin_unlock_irqrestore(&cctx->lock, flags); kfree(ctx->maps);Lines 1–500 of 2508. Load more
// SPDX-License-Identifier: GPL-2.0 // Copyright (c) 2011-2018, The Linux Foundation. All rights reserved. // Copyright (c) 2018, Linaro Limited #include <linux/completion.h> #include <linux/device.h> #include <linux/dma-buf.h> #include <linux/dma-mapping.h> #include <linux/dma-resv.h> #include <linux/idr.h> #include <linux/list.h> #include <linux/miscdevice.h> #include <linux/module.h> #include <linux/of_address.h> #include <linux/of.h> #include <linux/platform_device.h> #include <linux/sort.h> #include <linux/of_platform.h> #include <linux/rpmsg.h> #include <linux/scatterlist.h> #include <linux/slab.h> #include <linux/firmware/qcom/qcom_scm.h> #include <uapi/misc/fastrpc.h> #include <linux/of_reserved_mem.h> #define ADSP_DOMAIN_ID (0) #define MDSP_DOMAIN_ID (1) #define SDSP_DOMAIN_ID (2) #define CDSP_DOMAIN_ID (3) #define CDSP1_DOMAIN_ID (4) #define FASTRPC_DEV_MAX 5 /* adsp, mdsp, slpi, cdsp, cdsp1 */ #define FASTRPC_MAX_SESSIONS 14 #define FASTRPC_MAX_VMIDS 16 #define FASTRPC_ALIGN 128 #define FASTRPC_MAX_FDLIST 16 #define FASTRPC_MAX_CRCLIST 64 #define FASTRPC_PHYS(p) ((p) & 0xffffffff) #define FASTRPC_CTX_MAX (256) #define FASTRPC_INIT_HANDLE 1 #define FASTRPC_DSP_UTILITIES_HANDLE 2 #define FASTRPC_CTXID_MASK (0xFF0) #define INIT_FILELEN_MAX (2 * 1024 * 1024) #define INIT_FILE_NAMELEN_MAX (128) #define FASTRPC_DEVICE_NAME "fastrpc" /* Add memory to static PD pool, protection thru XPU */ #define ADSP_MMAP_HEAP_ADDR 4 /* MAP static DMA buffer on DSP User PD */ #define ADSP_MMAP_DMA_BUFFER 6 /* Add memory to static PD pool protection thru hypervisor */ #define ADSP_MMAP_REMOTE_HEAP_ADDR 8 /* Add memory to userPD pool, for user heap */ #define ADSP_MMAP_ADD_PAGES 0x1000 /* Add memory to userPD pool, for LLC heap */ #define ADSP_MMAP_ADD_PAGES_LLC 0x3000, #define DSP_UNSUPPORTED_API (0x80000414) /* MAX NUMBER of DSP ATTRIBUTES SUPPORTED */ #define FASTRPC_MAX_DSP_ATTRIBUTES (256) #define FASTRPC_MAX_DSP_ATTRIBUTES_LEN (sizeof(u32) * FASTRPC_MAX_DSP_ATTRIBUTES) /* Retrives number of input buffers from the scalars parameter */ #define REMOTE_SCALARS_INBUFS(sc) (((sc) >> 16) & 0x0ff) /* Retrives number of output buffers from the scalars parameter */ #define REMOTE_SCALARS_OUTBUFS(sc) (((sc) >> 8) & 0x0ff) /* Retrives number of input handles from the scalars parameter */ #define REMOTE_SCALARS_INHANDLES(sc) (((sc) >> 4) & 0x0f) /* Retrives number of output handles from the scalars parameter */ #define REMOTE_SCALARS_OUTHANDLES(sc) ((sc) & 0x0f) #define REMOTE_SCALARS_LENGTH(sc) (REMOTE_SCALARS_INBUFS(sc) + \ REMOTE_SCALARS_OUTBUFS(sc) + \ REMOTE_SCALARS_INHANDLES(sc)+ \ REMOTE_SCALARS_OUTHANDLES(sc)) #define FASTRPC_BUILD_SCALARS(attr, method, in, out, oin, oout) \ (((attr & 0x07) << 29) | \ ((method & 0x1f) << 24) | \ ((in & 0xff) << 16) | \ ((out & 0xff) << 8) | \ ((oin & 0x0f) << 4) | \ (oout & 0x0f)) #define FASTRPC_SCALARS(method, in, out) \ FASTRPC_BUILD_SCALARS(0, method, in, out, 0, 0) #define FASTRPC_CREATE_PROCESS_NARGS 6 #define FASTRPC_CREATE_STATIC_PROCESS_NARGS 3 /* Remote Method id table */ #define FASTRPC_RMID_INIT_ATTACH 0 #define FASTRPC_RMID_INIT_RELEASE 1 #define FASTRPC_RMID_INIT_MMAP 4 #define FASTRPC_RMID_INIT_MUNMAP 5 #define FASTRPC_RMID_INIT_CREATE 6 #define FASTRPC_RMID_INIT_CREATE_ATTR 7 #define FASTRPC_RMID_INIT_CREATE_STATIC 8 #define FASTRPC_RMID_INIT_MEM_MAP 10 #define FASTRPC_RMID_INIT_MEM_UNMAP 11 /* Protection Domain(PD) ids */ #define ROOT_PD (0) #define USER_PD (1) #define SENSORS_PD (2) #define miscdev_to_fdevice(d) container_of(d, struct fastrpc_device, miscdev) static const char *domains[FASTRPC_DEV_MAX] = { "adsp", "mdsp", "sdsp", "cdsp", "cdsp1" }; struct fastrpc_phy_page { u64 addr; /* physical address */ u64 size; /* size of contiguous region */ }; struct fastrpc_invoke_buf { u32 num; /* number of contiguous regions */ u32 pgidx; /* index to start of contiguous region */ }; struct fastrpc_remote_dmahandle { s32 fd; /* dma handle fd */ u32 offset; /* dma handle offset */ u32 len; /* dma handle length */ }; struct fastrpc_remote_buf { u64 pv; /* buffer pointer */ u64 len; /* length of buffer */ }; union fastrpc_remote_arg { struct fastrpc_remote_buf buf; struct fastrpc_remote_dmahandle dma; }; struct fastrpc_mmap_rsp_msg { u64 vaddr; }; struct fastrpc_mmap_req_msg { s32 pgid; u32 flags; u64 vaddr; s32 num; }; struct fastrpc_mem_map_req_msg { s32 pgid; s32 fd; s32 offset; u32 flags; u64 vaddrin; s32 num; s32 data_len; }; struct fastrpc_munmap_req_msg { s32 pgid; u64 vaddr; u64 size; }; struct fastrpc_mem_unmap_req_msg { s32 pgid; s32 fd; u64 vaddrin; u64 len; }; struct fastrpc_msg { int pid; /* process group id */ int tid; /* thread id */ u64 ctx; /* invoke caller context */ u32 handle; /* handle to invoke */ u32 sc; /* scalars structure describing the data */ u64 addr; /* physical address */ u64 size; /* size of contiguous region */ }; struct fastrpc_invoke_rsp { u64 ctx; /* invoke caller context */ int retval; /* invoke return value */ }; struct fastrpc_buf_overlap { u64 start; u64 end; int raix; u64 mstart; u64 mend; u64 offset; }; struct fastrpc_buf { struct fastrpc_user *fl; struct dma_buf *dmabuf; struct device *dev; void *virt; u64 phys; u64 size; /* Lock for dma buf attachments */ struct mutex lock; struct list_head attachments; /* mmap support */ struct list_head node; /* list of user requested mmaps */ uintptr_t raddr; }; struct fastrpc_dma_buf_attachment { struct device *dev; struct sg_table sgt; struct list_head node; }; struct fastrpc_map { struct list_head node; struct fastrpc_user *fl; int fd; struct dma_buf *buf; struct sg_table *table; struct dma_buf_attachment *attach; u64 phys; u64 size; void *va; u64 len; u64 raddr; u32 attr; struct kref refcount; }; struct fastrpc_invoke_ctx { int nscalars; int nbufs; int retval; int pid; int tgid; u32 sc; u32 *crc; u64 ctxid; u64 msg_sz; struct kref refcount; struct list_head node; /* list of ctxs */ struct completion work; struct work_struct put_work; struct fastrpc_msg msg; struct fastrpc_user *fl; union fastrpc_remote_arg *rpra; struct fastrpc_map **maps; struct fastrpc_buf *buf; struct fastrpc_invoke_args *args; struct fastrpc_buf_overlap *olaps; struct fastrpc_channel_ctx *cctx; }; struct fastrpc_session_ctx { struct device *dev; int sid; bool used; bool valid; }; struct fastrpc_channel_ctx { int domain_id; int sesscount; int vmcount; struct qcom_scm_vmperm vmperms[FASTRPC_MAX_VMIDS]; struct rpmsg_device *rpdev; struct fastrpc_session_ctx session[FASTRPC_MAX_SESSIONS]; spinlock_t lock; struct idr ctx_idr; struct list_head users; struct kref refcount; /* Flag if dsp attributes are cached */ bool valid_attributes; u32 dsp_attributes[FASTRPC_MAX_DSP_ATTRIBUTES]; struct fastrpc_device *secure_fdevice; struct fastrpc_device *fdevice; struct fastrpc_buf *remote_heap; struct list_head invoke_interrupted_mmaps; bool secure; bool unsigned_support; u64 dma_mask; }; struct fastrpc_device { struct fastrpc_channel_ctx *cctx; struct miscdevice miscdev; bool secure; }; struct fastrpc_user { struct list_head user; struct list_head maps; struct list_head pending; struct list_head mmaps; struct fastrpc_channel_ctx *cctx; struct fastrpc_session_ctx *sctx; struct fastrpc_buf *init_mem; int tgid; int pd; bool is_secure_dev; /* Lock for lists */ spinlock_t lock; /* lock for allocations */ struct mutex mutex; }; static void fastrpc_free_map(struct kref *ref) { struct fastrpc_map *map; map = container_of(ref, struct fastrpc_map, refcount); if (map->table) { if (map->attr & FASTRPC_ATTR_SECUREMAP) { struct qcom_scm_vmperm perm; int vmid = map->fl->cctx->vmperms[0].vmid; u64 src_perms = BIT(QCOM_SCM_VMID_HLOS) | BIT(vmid); int err = 0; perm.vmid = QCOM_SCM_VMID_HLOS; perm.perm = QCOM_SCM_PERM_RWX; err = qcom_scm_assign_mem(map->phys, map->size, &src_perms, &perm, 1); if (err) { dev_err(map->fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d\n", map->phys, map->size, err); return; } } dma_buf_unmap_attachment_unlocked(map->attach, map->table, DMA_BIDIRECTIONAL); dma_buf_detach(map->buf, map->attach); dma_buf_put(map->buf); } if (map->fl) { spin_lock(&map->fl->lock); list_del(&map->node); spin_unlock(&map->fl->lock); map->fl = NULL; } kfree(map); } static void fastrpc_map_put(struct fastrpc_map *map) { if (map) kref_put(&map->refcount, fastrpc_free_map); } static int fastrpc_map_get(struct fastrpc_map *map) { if (!map) return -ENOENT; return kref_get_unless_zero(&map->refcount) ? 0 : -ENOENT; } static int fastrpc_map_lookup(struct fastrpc_user *fl, int fd, struct fastrpc_map **ppmap, bool take_ref) { struct fastrpc_session_ctx *sess = fl->sctx; struct fastrpc_map *map = NULL; int ret = -ENOENT; spin_lock(&fl->lock); list_for_each_entry(map, &fl->maps, node) { if (map->fd != fd) continue; if (take_ref) { ret = fastrpc_map_get(map); if (ret) { dev_dbg(sess->dev, "%s: Failed to get map fd=%d ret=%d\n", __func__, fd, ret); break; } } *ppmap = map; ret = 0; break; } spin_unlock(&fl->lock); return ret; } static void fastrpc_buf_free(struct fastrpc_buf *buf) { dma_free_coherent(buf->dev, buf->size, buf->virt, FASTRPC_PHYS(buf->phys)); kfree(buf); } static int __fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev, u64 size, struct fastrpc_buf **obuf) { struct fastrpc_buf *buf; buf = kzalloc(sizeof(*buf), GFP_KERNEL); if (!buf) return -ENOMEM; INIT_LIST_HEAD(&buf->attachments); INIT_LIST_HEAD(&buf->node); mutex_init(&buf->lock); buf->fl = fl; buf->virt = NULL; buf->phys = 0; buf->size = size; buf->dev = dev; buf->raddr = 0; buf->virt = dma_alloc_coherent(dev, buf->size, (dma_addr_t *)&buf->phys, GFP_KERNEL); if (!buf->virt) { mutex_destroy(&buf->lock); kfree(buf); return -ENOMEM; } *obuf = buf; return 0; } static int fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev, u64 size, struct fastrpc_buf **obuf) { int ret; struct fastrpc_buf *buf; ret = __fastrpc_buf_alloc(fl, dev, size, obuf); if (ret) return ret; buf = *obuf; if (fl->sctx && fl->sctx->sid) buf->phys += ((u64)fl->sctx->sid << 32); return 0; } static int fastrpc_remote_heap_alloc(struct fastrpc_user *fl, struct device *dev, u64 size, struct fastrpc_buf **obuf) { struct device *rdev = &fl->cctx->rpdev->dev; return __fastrpc_buf_alloc(fl, rdev, size, obuf); } static void fastrpc_channel_ctx_free(struct kref *ref) { struct fastrpc_channel_ctx *cctx; cctx = container_of(ref, struct fastrpc_channel_ctx, refcount); kfree(cctx); } static void fastrpc_channel_ctx_get(struct fastrpc_channel_ctx *cctx) { kref_get(&cctx->refcount); } static void fastrpc_channel_ctx_put(struct fastrpc_channel_ctx *cctx) { kref_put(&cctx->refcount, fastrpc_channel_ctx_free); } static void fastrpc_context_free(struct kref *ref) { struct fastrpc_invoke_ctx *ctx; struct fastrpc_channel_ctx *cctx; unsigned long flags; int i; ctx = container_of(ref, struct fastrpc_invoke_ctx, refcount); cctx = ctx->cctx; for (i = 0; i < ctx->nbufs; i++) fastrpc_map_put(ctx->maps[i]); if (ctx->buf) fastrpc_buf_free(ctx->buf); spin_lock_irqsave(&cctx->lock, flags); idr_remove(&cctx->ctx_idr, ctx->ctxid >> 4); spin_unlock_irqrestore(&cctx->lock, flags); kfree(ctx->maps);