// SPDX-License-Identifier: GPL-2.0// Copyright (c) 2011-2018, The Linux Foundation. All rights reserved.// Copyright (c) 2018, Linaro Limited #include <linux/completion.h>#include <linux/device.h>#include <linux/dma-buf.h>#include <linux/dma-mapping.h>#include <linux/dma-resv.h>#include <linux/idr.h>#include <linux/list.h>#include <linux/miscdevice.h>#include <linux/module.h>#include <linux/of_address.h>#include <linux/of.h>#include <linux/platform_device.h>#include <linux/sort.h>#include <linux/of_platform.h>#include <linux/rpmsg.h>#include <linux/scatterlist.h>#include <linux/slab.h>#include <linux/firmware/qcom/qcom_scm.h>#include <uapi/misc/fastrpc.h>#include <linux/of_reserved_mem.h>#include <linux/bitfield.h>#include <linux/bits.h>#include <linux/bitops.h>#include <linux/compiler.h>#include <linux/iopoll.h> #define ADSP_DOMAIN_ID (0)#define MDSP_DOMAIN_ID (1)#define SDSP_DOMAIN_ID (2)#define CDSP_DOMAIN_ID (3)#define GDSP_DOMAIN_ID (4)#define FASTRPC_MAX_SESSIONS 14#define FASTRPC_MAX_VMIDS 16#define FASTRPC_ALIGN 128#define FASTRPC_MAX_FDLIST 16#define FASTRPC_MAX_CRCLIST 64#define FASTRPC_CTX_MAX (256)#define FASTRPC_INIT_HANDLE 1#define FASTRPC_DSP_UTILITIES_HANDLE 2/* * Maximum handle value for static handles. * Static handles are pre-defined, fixed numeric values statically assigned * in the IDL file or FastRPC framework. */#define FASTRPC_MAX_STATIC_HANDLE (20)#define FASTRPC_CTXID_MASK GENMASK(15, 8)/* Sequence number occupies bits 63:16 of the ctxid / message context */#define FASTRPC_CTXID_SEQ_SHIFT 16#define FASTRPC_CTXID_SEQ_MASK GENMASK_ULL(63, 16) /* * Newer DSP firmware implements a PD (Protection Domain) notification * framework that sends PD state notifications upon request. The PD exit * notification is unconditionally sent by the DSP with this fixed sentinel * in the context field rather than the context of an outstanding invocation. * Since the fastrpc driver does not support the DSP PD notification framework, * this message must be dropped rather than matched against the context idr. */#define FASTRPC_DSP_PD_NOTIFY_CTX 0xABCDABCD #define INIT_FILELEN_MAX (2 * 1024 * 1024)#define INIT_FILE_NAMELEN_MAX (128)#define FASTRPC_DEVICE_NAME "fastrpc" /* Add memory to static PD pool, protection thru XPU */#define ADSP_MMAP_HEAP_ADDR 4/* MAP static DMA buffer on DSP User PD */#define ADSP_MMAP_DMA_BUFFER 6/* Add memory to static PD pool protection thru hypervisor */#define ADSP_MMAP_REMOTE_HEAP_ADDR 8/* Add memory to userPD pool, for user heap */#define ADSP_MMAP_ADD_PAGES 0x1000/* Add memory to userPD pool, for LLC heap */#define ADSP_MMAP_ADD_PAGES_LLC 0x3000, #define DSP_UNSUPPORTED_API (0x80000414)/* MAX NUMBER of DSP ATTRIBUTES SUPPORTED */#define FASTRPC_MAX_DSP_ATTRIBUTES (256)#define FASTRPC_MAX_DSP_ATTRIBUTES_LEN (sizeof(u32) * FASTRPC_MAX_DSP_ATTRIBUTES) /* Retrives number of input buffers from the scalars parameter */#define REMOTE_SCALARS_INBUFS(sc) (((sc) >> 16) & 0x0ff) /* Retrives number of output buffers from the scalars parameter */#define REMOTE_SCALARS_OUTBUFS(sc) (((sc) >> 8) & 0x0ff) /* Retrives number of input handles from the scalars parameter */#define REMOTE_SCALARS_INHANDLES(sc) (((sc) >> 4) & 0x0f) /* Retrives number of output handles from the scalars parameter */#define REMOTE_SCALARS_OUTHANDLES(sc) ((sc) & 0x0f) #define REMOTE_SCALARS_LENGTH(sc) (REMOTE_SCALARS_INBUFS(sc) + \ REMOTE_SCALARS_OUTBUFS(sc) + \ REMOTE_SCALARS_INHANDLES(sc)+ \ REMOTE_SCALARS_OUTHANDLES(sc))#define FASTRPC_BUILD_SCALARS(attr, method, in, out, oin, oout) \ (((attr & 0x07) << 29) | \ ((method & 0x1f) << 24) | \ ((in & 0xff) << 16) | \ ((out & 0xff) << 8) | \ ((oin & 0x0f) << 4) | \ (oout & 0x0f)) #define FASTRPC_SCALARS(method, in, out) \ FASTRPC_BUILD_SCALARS(0, method, in, out, 0, 0) #define FASTRPC_CREATE_PROCESS_NARGS 6#define FASTRPC_CREATE_STATIC_PROCESS_NARGS 3/* Remote Method id table */#define FASTRPC_RMID_INIT_ATTACH 0#define FASTRPC_RMID_INIT_RELEASE 1#define FASTRPC_RMID_INIT_MMAP 4#define FASTRPC_RMID_INIT_MUNMAP 5#define FASTRPC_RMID_INIT_CREATE 6#define FASTRPC_RMID_INIT_CREATE_ATTR 7#define FASTRPC_RMID_INIT_CREATE_STATIC 8#define FASTRPC_RMID_INIT_MEM_MAP 10#define FASTRPC_RMID_INIT_MEM_UNMAP 11 /* Protection Domain(PD) ids */#define ROOT_PD (0)#define USER_PD (1)#define SENSORS_PD (2) #define miscdev_to_fdevice(d) container_of(d, struct fastrpc_device, miscdev) /* Poll response number from remote processor for call completion */#define FASTRPC_POLL_RESPONSE (0xdecaf) /* Polling mode timeout limit */#define FASTRPC_POLL_MAX_TIMEOUT_US (10000) struct fastrpc_phy_page { dma_addr_t addr; /* dma address */ u64 size; /* size of contiguous region */}; struct fastrpc_invoke_buf { u32 num; /* number of contiguous regions */ u32 pgidx; /* index to start of contiguous region */}; struct fastrpc_remote_dmahandle { s32 fd; /* dma handle fd */ u32 offset; /* dma handle offset */ u32 len; /* dma handle length */}; struct fastrpc_remote_buf { u64 pv; /* buffer pointer */ u64 len; /* length of buffer */}; union fastrpc_remote_arg { struct fastrpc_remote_buf buf; struct fastrpc_remote_dmahandle dma;}; struct fastrpc_mmap_rsp_msg { u64 vaddr;}; struct fastrpc_mmap_req_msg { s32 client_id; u32 flags; u64 vaddr; s32 num;}; struct fastrpc_mem_map_req_msg { s32 client_id; s32 fd; s32 offset; u32 flags; u64 vaddrin; s32 num; s32 data_len;}; struct fastrpc_munmap_req_msg { s32 client_id; u64 vaddr; u64 size;}; struct fastrpc_mem_unmap_req_msg { s32 client_id; s32 fd; u64 vaddrin; u64 len;}; struct fastrpc_msg { int client_id; /* process client id */ int tid; /* thread id */ u64 ctx; /* invoke caller context */ u32 handle; /* handle to invoke */ u32 sc; /* scalars structure describing the data */ dma_addr_t addr; /* dma address */ u64 size; /* size of contiguous region */}; struct fastrpc_invoke_rsp { u64 ctx; /* invoke caller context */ int retval; /* invoke return value */}; struct fastrpc_buf_overlap { u64 start; u64 end; int raix; u64 mstart; u64 mend; u64 offset;}; struct fastrpc_buf { struct fastrpc_user *fl; struct dma_buf *dmabuf; struct device *dev; void *virt; dma_addr_t dma_addr; u64 size; /* Lock for dma buf attachments */ struct mutex lock; struct list_head attachments; /* mmap support */ struct list_head node; /* list of user requested mmaps */ uintptr_t raddr;}; struct fastrpc_dma_buf_attachment { struct device *dev; struct sg_table sgt; struct list_head node;}; struct fastrpc_map { struct list_head node; struct fastrpc_user *fl; int fd; struct dma_buf *buf; struct sg_table *table; struct dma_buf_attachment *attach; dma_addr_t dma_addr; u64 size; void *va; u64 len; u64 raddr; u32 attr; struct kref refcount;}; struct fastrpc_invoke_ctx { int nscalars; int nbufs; int retval; int pid; int client_id; u32 sc; u64 *fdlist; u32 *crc; /* Poll memory that DSP updates */ u32 *poll_addr; u64 ctxid; u64 msg_sz; /* work done status flag */ bool is_work_done; /* process updates poll memory instead of glink response */ bool is_polled; struct kref refcount; struct list_head node; /* list of ctxs */ struct completion work; struct work_struct put_work; struct fastrpc_msg msg; struct fastrpc_user *fl; union fastrpc_remote_arg *rpra; struct fastrpc_map **maps; struct fastrpc_buf *buf; struct fastrpc_invoke_args *args; struct fastrpc_buf_overlap *olaps; struct fastrpc_channel_ctx *cctx;}; struct fastrpc_session_ctx { struct device *dev; int sid; bool used; bool valid;}; struct fastrpc_soc_data { u32 sid_pos; u32 dma_addr_bits_cdsp; u32 dma_addr_bits_default; bool poll_mode_supported;}; struct fastrpc_channel_ctx { int domain_id; int sesscount; int vmcount; struct qcom_scm_vmperm vmperms[FASTRPC_MAX_VMIDS]; struct rpmsg_device *rpdev; struct fastrpc_session_ctx session[FASTRPC_MAX_SESSIONS]; spinlock_t lock; struct idr ctx_idr; struct list_head users; struct kref refcount; /* Flag if dsp attributes are cached */ bool valid_attributes; u32 dsp_attributes[FASTRPC_MAX_DSP_ATTRIBUTES]; struct fastrpc_device *secure_fdevice; struct fastrpc_device *fdevice; struct fastrpc_buf *remote_heap; struct list_head invoke_interrupted_mmaps; bool secure; bool unsigned_support; bool poll_mode_supported; /* Per-channel sequence counter; incremented on every context allocation */ atomic_t ctx_seq; u64 dma_mask; const struct fastrpc_soc_data *soc_data;}; struct fastrpc_device { struct fastrpc_channel_ctx *cctx; struct miscdevice miscdev; bool secure;}; struct fastrpc_user { struct list_head user; struct list_head maps; struct list_head pending; struct list_head mmaps; struct fastrpc_channel_ctx *cctx; struct fastrpc_session_ctx *sctx; struct fastrpc_buf *init_mem; int client_id; int pd; bool is_secure_dev; /* Flags poll mode state */ bool poll_mode; /* Lock for lists */ spinlock_t lock; /* lock for allocations */ struct mutex mutex; /* Reference count */ struct kref refcount;}; /* Extract SMMU PA from consolidated IOVA */static inline dma_addr_t fastrpc_ipa_to_dma_addr(struct fastrpc_channel_ctx *cctx, dma_addr_t iova){ if (!cctx->soc_data->sid_pos) return 0; return iova & GENMASK_ULL(cctx->soc_data->sid_pos - 1, 0);} /* * Prepare the consolidated iova to send to DSP by prepending the SID * to smmu PA at the appropriate position */static inline u64 fastrpc_sid_offset(struct fastrpc_channel_ctx *cctx, struct fastrpc_session_ctx *sctx){ return (u64)sctx->sid << cctx->soc_data->sid_pos;} static void fastrpc_free_map(struct kref *ref){ struct fastrpc_map *map; map = container_of(ref, struct fastrpc_map, refcount); if (map->table) { if (map->attr & FASTRPC_ATTR_SECUREMAP) { struct qcom_scm_vmperm perm; int vmid = map->fl->cctx->vmperms[0].vmid; u64 src_perms = BIT(QCOM_SCM_VMID_HLOS) | BIT(vmid); int err = 0; perm.vmid = QCOM_SCM_VMID_HLOS; perm.perm = QCOM_SCM_PERM_RWX; err = qcom_scm_assign_mem(map->dma_addr, map->len, &src_perms, &perm, 1); if (err) { dev_err(map->fl->sctx->dev, "Failed to assign memory dma_addr %pad size 0x%llx err %d\n", &map->dma_addr, map->len, err); return; } } dma_buf_unmap_attachment_unlocked(map->attach, map->table, DMA_BIDIRECTIONAL); dma_buf_detach(map->buf, map->attach); dma_buf_put(map->buf); } if (map->fl) { spin_lock(&map->fl->lock); list_del(&map->node); spin_unlock(&map->fl->lock); map->fl = NULL; } kfree(map);} static void fastrpc_map_put(struct fastrpc_map *map){ if (map) kref_put(&map->refcount, fastrpc_free_map);} static int fastrpc_map_get(struct fastrpc_map *map){ if (!map) return -ENOENT; return kref_get_unless_zero(&map->refcount) ? 0 : -ENOENT;} static int fastrpc_map_lookup(struct fastrpc_user *fl, int fd, struct fastrpc_map **ppmap, bool take_ref){ struct fastrpc_map *map = NULL; struct dma_buf *buf; int ret = -ENOENT; buf = dma_buf_get(fd); if (IS_ERR(buf)) return PTR_ERR(buf); spin_lock(&fl->lock); list_for_each_entry(map, &fl->maps, node) { if (map->fd != fd || map->buf != buf) continue; if (take_ref) { ret = fastrpc_map_get(map); if (ret) break; } *ppmap = map; ret = 0; break; } spin_unlock(&fl->lock); dma_buf_put(buf); return ret;} static void fastrpc_buf_free(struct fastrpc_buf *buf){ if (!buf) return; dma_free_coherent(buf->dev, buf->size, buf->virt, fastrpc_ipa_to_dma_addr(buf->fl->cctx, buf->dma_addr)); kfree(buf);} static int __fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev, u64 size, struct fastrpc_buf **obuf){ struct fastrpc_buf *buf; buf = kzalloc_obj(*buf); if (!buf) return -ENOMEM; INIT_LIST_HEAD(&buf->attachments); INIT_LIST_HEAD(&buf->node); mutex_init(&buf->lock); buf->fl = fl; buf->virt = NULL; buf->dma_addr = 0; buf->size = size; buf->dev = dev; buf->raddr = 0; buf->virt = dma_alloc_coherent(dev, buf->size, &buf->dma_addr, GFP_KERNEL); if (!buf->virt) { mutex_destroy(&buf->lock); kfree(buf);Lines 1–500 of 2817. Load more
// SPDX-License-Identifier: GPL-2.0 // Copyright (c) 2011-2018, The Linux Foundation. All rights reserved. // Copyright (c) 2018, Linaro Limited #include <linux/completion.h> #include <linux/device.h> #include <linux/dma-buf.h> #include <linux/dma-mapping.h> #include <linux/dma-resv.h> #include <linux/idr.h> #include <linux/list.h> #include <linux/miscdevice.h> #include <linux/module.h> #include <linux/of_address.h> #include <linux/of.h> #include <linux/platform_device.h> #include <linux/sort.h> #include <linux/of_platform.h> #include <linux/rpmsg.h> #include <linux/scatterlist.h> #include <linux/slab.h> #include <linux/firmware/qcom/qcom_scm.h> #include <uapi/misc/fastrpc.h> #include <linux/of_reserved_mem.h> #include <linux/bitfield.h> #include <linux/bits.h> #include <linux/bitops.h> #include <linux/compiler.h> #include <linux/iopoll.h> #define ADSP_DOMAIN_ID (0) #define MDSP_DOMAIN_ID (1) #define SDSP_DOMAIN_ID (2) #define CDSP_DOMAIN_ID (3) #define GDSP_DOMAIN_ID (4) #define FASTRPC_MAX_SESSIONS 14 #define FASTRPC_MAX_VMIDS 16 #define FASTRPC_ALIGN 128 #define FASTRPC_MAX_FDLIST 16 #define FASTRPC_MAX_CRCLIST 64 #define FASTRPC_CTX_MAX (256) #define FASTRPC_INIT_HANDLE 1 #define FASTRPC_DSP_UTILITIES_HANDLE 2 /* * Maximum handle value for static handles. * Static handles are pre-defined, fixed numeric values statically assigned * in the IDL file or FastRPC framework. */ #define FASTRPC_MAX_STATIC_HANDLE (20) #define FASTRPC_CTXID_MASK GENMASK(15, 8) /* Sequence number occupies bits 63:16 of the ctxid / message context */ #define FASTRPC_CTXID_SEQ_SHIFT 16 #define FASTRPC_CTXID_SEQ_MASK GENMASK_ULL(63, 16) /* * Newer DSP firmware implements a PD (Protection Domain) notification * framework that sends PD state notifications upon request. The PD exit * notification is unconditionally sent by the DSP with this fixed sentinel * in the context field rather than the context of an outstanding invocation. * Since the fastrpc driver does not support the DSP PD notification framework, * this message must be dropped rather than matched against the context idr. */ #define FASTRPC_DSP_PD_NOTIFY_CTX 0xABCDABCD #define INIT_FILELEN_MAX (2 * 1024 * 1024) #define INIT_FILE_NAMELEN_MAX (128) #define FASTRPC_DEVICE_NAME "fastrpc" /* Add memory to static PD pool, protection thru XPU */ #define ADSP_MMAP_HEAP_ADDR 4 /* MAP static DMA buffer on DSP User PD */ #define ADSP_MMAP_DMA_BUFFER 6 /* Add memory to static PD pool protection thru hypervisor */ #define ADSP_MMAP_REMOTE_HEAP_ADDR 8 /* Add memory to userPD pool, for user heap */ #define ADSP_MMAP_ADD_PAGES 0x1000 /* Add memory to userPD pool, for LLC heap */ #define ADSP_MMAP_ADD_PAGES_LLC 0x3000, #define DSP_UNSUPPORTED_API (0x80000414) /* MAX NUMBER of DSP ATTRIBUTES SUPPORTED */ #define FASTRPC_MAX_DSP_ATTRIBUTES (256) #define FASTRPC_MAX_DSP_ATTRIBUTES_LEN (sizeof(u32) * FASTRPC_MAX_DSP_ATTRIBUTES) /* Retrives number of input buffers from the scalars parameter */ #define REMOTE_SCALARS_INBUFS(sc) (((sc) >> 16) & 0x0ff) /* Retrives number of output buffers from the scalars parameter */ #define REMOTE_SCALARS_OUTBUFS(sc) (((sc) >> 8) & 0x0ff) /* Retrives number of input handles from the scalars parameter */ #define REMOTE_SCALARS_INHANDLES(sc) (((sc) >> 4) & 0x0f) /* Retrives number of output handles from the scalars parameter */ #define REMOTE_SCALARS_OUTHANDLES(sc) ((sc) & 0x0f) #define REMOTE_SCALARS_LENGTH(sc) (REMOTE_SCALARS_INBUFS(sc) + \ REMOTE_SCALARS_OUTBUFS(sc) + \ REMOTE_SCALARS_INHANDLES(sc)+ \ REMOTE_SCALARS_OUTHANDLES(sc)) #define FASTRPC_BUILD_SCALARS(attr, method, in, out, oin, oout) \ (((attr & 0x07) << 29) | \ ((method & 0x1f) << 24) | \ ((in & 0xff) << 16) | \ ((out & 0xff) << 8) | \ ((oin & 0x0f) << 4) | \ (oout & 0x0f)) #define FASTRPC_SCALARS(method, in, out) \ FASTRPC_BUILD_SCALARS(0, method, in, out, 0, 0) #define FASTRPC_CREATE_PROCESS_NARGS 6 #define FASTRPC_CREATE_STATIC_PROCESS_NARGS 3 /* Remote Method id table */ #define FASTRPC_RMID_INIT_ATTACH 0 #define FASTRPC_RMID_INIT_RELEASE 1 #define FASTRPC_RMID_INIT_MMAP 4 #define FASTRPC_RMID_INIT_MUNMAP 5 #define FASTRPC_RMID_INIT_CREATE 6 #define FASTRPC_RMID_INIT_CREATE_ATTR 7 #define FASTRPC_RMID_INIT_CREATE_STATIC 8 #define FASTRPC_RMID_INIT_MEM_MAP 10 #define FASTRPC_RMID_INIT_MEM_UNMAP 11 /* Protection Domain(PD) ids */ #define ROOT_PD (0) #define USER_PD (1) #define SENSORS_PD (2) #define miscdev_to_fdevice(d) container_of(d, struct fastrpc_device, miscdev) /* Poll response number from remote processor for call completion */ #define FASTRPC_POLL_RESPONSE (0xdecaf) /* Polling mode timeout limit */ #define FASTRPC_POLL_MAX_TIMEOUT_US (10000) struct fastrpc_phy_page { dma_addr_t addr; /* dma address */ u64 size; /* size of contiguous region */ }; struct fastrpc_invoke_buf { u32 num; /* number of contiguous regions */ u32 pgidx; /* index to start of contiguous region */ }; struct fastrpc_remote_dmahandle { s32 fd; /* dma handle fd */ u32 offset; /* dma handle offset */ u32 len; /* dma handle length */ }; struct fastrpc_remote_buf { u64 pv; /* buffer pointer */ u64 len; /* length of buffer */ }; union fastrpc_remote_arg { struct fastrpc_remote_buf buf; struct fastrpc_remote_dmahandle dma; }; struct fastrpc_mmap_rsp_msg { u64 vaddr; }; struct fastrpc_mmap_req_msg { s32 client_id; u32 flags; u64 vaddr; s32 num; }; struct fastrpc_mem_map_req_msg { s32 client_id; s32 fd; s32 offset; u32 flags; u64 vaddrin; s32 num; s32 data_len; }; struct fastrpc_munmap_req_msg { s32 client_id; u64 vaddr; u64 size; }; struct fastrpc_mem_unmap_req_msg { s32 client_id; s32 fd; u64 vaddrin; u64 len; }; struct fastrpc_msg { int client_id; /* process client id */ int tid; /* thread id */ u64 ctx; /* invoke caller context */ u32 handle; /* handle to invoke */ u32 sc; /* scalars structure describing the data */ dma_addr_t addr; /* dma address */ u64 size; /* size of contiguous region */ }; struct fastrpc_invoke_rsp { u64 ctx; /* invoke caller context */ int retval; /* invoke return value */ }; struct fastrpc_buf_overlap { u64 start; u64 end; int raix; u64 mstart; u64 mend; u64 offset; }; struct fastrpc_buf { struct fastrpc_user *fl; struct dma_buf *dmabuf; struct device *dev; void *virt; dma_addr_t dma_addr; u64 size; /* Lock for dma buf attachments */ struct mutex lock; struct list_head attachments; /* mmap support */ struct list_head node; /* list of user requested mmaps */ uintptr_t raddr; }; struct fastrpc_dma_buf_attachment { struct device *dev; struct sg_table sgt; struct list_head node; }; struct fastrpc_map { struct list_head node; struct fastrpc_user *fl; int fd; struct dma_buf *buf; struct sg_table *table; struct dma_buf_attachment *attach; dma_addr_t dma_addr; u64 size; void *va; u64 len; u64 raddr; u32 attr; struct kref refcount; }; struct fastrpc_invoke_ctx { int nscalars; int nbufs; int retval; int pid; int client_id; u32 sc; u64 *fdlist; u32 *crc; /* Poll memory that DSP updates */ u32 *poll_addr; u64 ctxid; u64 msg_sz; /* work done status flag */ bool is_work_done; /* process updates poll memory instead of glink response */ bool is_polled; struct kref refcount; struct list_head node; /* list of ctxs */ struct completion work; struct work_struct put_work; struct fastrpc_msg msg; struct fastrpc_user *fl; union fastrpc_remote_arg *rpra; struct fastrpc_map **maps; struct fastrpc_buf *buf; struct fastrpc_invoke_args *args; struct fastrpc_buf_overlap *olaps; struct fastrpc_channel_ctx *cctx; }; struct fastrpc_session_ctx { struct device *dev; int sid; bool used; bool valid; }; struct fastrpc_soc_data { u32 sid_pos; u32 dma_addr_bits_cdsp; u32 dma_addr_bits_default; bool poll_mode_supported; }; struct fastrpc_channel_ctx { int domain_id; int sesscount; int vmcount; struct qcom_scm_vmperm vmperms[FASTRPC_MAX_VMIDS]; struct rpmsg_device *rpdev; struct fastrpc_session_ctx session[FASTRPC_MAX_SESSIONS]; spinlock_t lock; struct idr ctx_idr; struct list_head users; struct kref refcount; /* Flag if dsp attributes are cached */ bool valid_attributes; u32 dsp_attributes[FASTRPC_MAX_DSP_ATTRIBUTES]; struct fastrpc_device *secure_fdevice; struct fastrpc_device *fdevice; struct fastrpc_buf *remote_heap; struct list_head invoke_interrupted_mmaps; bool secure; bool unsigned_support; bool poll_mode_supported; /* Per-channel sequence counter; incremented on every context allocation */ atomic_t ctx_seq; u64 dma_mask; const struct fastrpc_soc_data *soc_data; }; struct fastrpc_device { struct fastrpc_channel_ctx *cctx; struct miscdevice miscdev; bool secure; }; struct fastrpc_user { struct list_head user; struct list_head maps; struct list_head pending; struct list_head mmaps; struct fastrpc_channel_ctx *cctx; struct fastrpc_session_ctx *sctx; struct fastrpc_buf *init_mem; int client_id; int pd; bool is_secure_dev; /* Flags poll mode state */ bool poll_mode; /* Lock for lists */ spinlock_t lock; /* lock for allocations */ struct mutex mutex; /* Reference count */ struct kref refcount; }; /* Extract SMMU PA from consolidated IOVA */ static inline dma_addr_t fastrpc_ipa_to_dma_addr(struct fastrpc_channel_ctx *cctx, dma_addr_t iova) { if (!cctx->soc_data->sid_pos) return 0; return iova & GENMASK_ULL(cctx->soc_data->sid_pos - 1, 0); } /* * Prepare the consolidated iova to send to DSP by prepending the SID * to smmu PA at the appropriate position */ static inline u64 fastrpc_sid_offset(struct fastrpc_channel_ctx *cctx, struct fastrpc_session_ctx *sctx) { return (u64)sctx->sid << cctx->soc_data->sid_pos; } static void fastrpc_free_map(struct kref *ref) { struct fastrpc_map *map; map = container_of(ref, struct fastrpc_map, refcount); if (map->table) { if (map->attr & FASTRPC_ATTR_SECUREMAP) { struct qcom_scm_vmperm perm; int vmid = map->fl->cctx->vmperms[0].vmid; u64 src_perms = BIT(QCOM_SCM_VMID_HLOS) | BIT(vmid); int err = 0; perm.vmid = QCOM_SCM_VMID_HLOS; perm.perm = QCOM_SCM_PERM_RWX; err = qcom_scm_assign_mem(map->dma_addr, map->len, &src_perms, &perm, 1); if (err) { dev_err(map->fl->sctx->dev, "Failed to assign memory dma_addr %pad size 0x%llx err %d\n", &map->dma_addr, map->len, err); return; } } dma_buf_unmap_attachment_unlocked(map->attach, map->table, DMA_BIDIRECTIONAL); dma_buf_detach(map->buf, map->attach); dma_buf_put(map->buf); } if (map->fl) { spin_lock(&map->fl->lock); list_del(&map->node); spin_unlock(&map->fl->lock); map->fl = NULL; } kfree(map); } static void fastrpc_map_put(struct fastrpc_map *map) { if (map) kref_put(&map->refcount, fastrpc_free_map); } static int fastrpc_map_get(struct fastrpc_map *map) { if (!map) return -ENOENT; return kref_get_unless_zero(&map->refcount) ? 0 : -ENOENT; } static int fastrpc_map_lookup(struct fastrpc_user *fl, int fd, struct fastrpc_map **ppmap, bool take_ref) { struct fastrpc_map *map = NULL; struct dma_buf *buf; int ret = -ENOENT; buf = dma_buf_get(fd); if (IS_ERR(buf)) return PTR_ERR(buf); spin_lock(&fl->lock); list_for_each_entry(map, &fl->maps, node) { if (map->fd != fd || map->buf != buf) continue; if (take_ref) { ret = fastrpc_map_get(map); if (ret) break; } *ppmap = map; ret = 0; break; } spin_unlock(&fl->lock); dma_buf_put(buf); return ret; } static void fastrpc_buf_free(struct fastrpc_buf *buf) { if (!buf) return; dma_free_coherent(buf->dev, buf->size, buf->virt, fastrpc_ipa_to_dma_addr(buf->fl->cctx, buf->dma_addr)); kfree(buf); } static int __fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev, u64 size, struct fastrpc_buf **obuf) { struct fastrpc_buf *buf; buf = kzalloc_obj(*buf); if (!buf) return -ENOMEM; INIT_LIST_HEAD(&buf->attachments); INIT_LIST_HEAD(&buf->node); mutex_init(&buf->lock); buf->fl = fl; buf->virt = NULL; buf->dma_addr = 0; buf->size = size; buf->dev = dev; buf->raddr = 0; buf->virt = dma_alloc_coherent(dev, buf->size, &buf->dma_addr, GFP_KERNEL); if (!buf->virt) { mutex_destroy(&buf->lock); kfree(buf);