// SPDX-License-Identifier: GPL-2.0// Copyright (c) 2011-2018, The Linux Foundation. All rights reserved.// Copyright (c) 2018, Linaro Limited #include <linux/completion.h>#include <linux/device.h>#include <linux/dma-buf.h>#include <linux/dma-mapping.h>#include <linux/idr.h>#include <linux/list.h>#include <linux/miscdevice.h>#include <linux/module.h>#include <linux/of_address.h>#include <linux/of.h>#include <linux/sort.h>#include <linux/of_platform.h>#include <linux/rpmsg.h>#include <linux/scatterlist.h>#include <linux/slab.h>#include <uapi/misc/fastrpc.h> #define ADSP_DOMAIN_ID (0)#define MDSP_DOMAIN_ID (1)#define SDSP_DOMAIN_ID (2)#define CDSP_DOMAIN_ID (3)#define FASTRPC_DEV_MAX 4 /* adsp, mdsp, slpi, cdsp*/#define FASTRPC_MAX_SESSIONS 9 /*8 compute, 1 cpz*/#define FASTRPC_ALIGN 128#define FASTRPC_MAX_FDLIST 16#define FASTRPC_MAX_CRCLIST 64#define FASTRPC_PHYS(p) ((p) & 0xffffffff)#define FASTRPC_CTX_MAX (256)#define FASTRPC_INIT_HANDLE 1#define FASTRPC_CTXID_MASK (0xFF0)#define INIT_FILELEN_MAX (2 * 1024 * 1024)#define FASTRPC_DEVICE_NAME "fastrpc"#define ADSP_MMAP_ADD_PAGES 0x1000 /* Retrives number of input buffers from the scalars parameter */#define REMOTE_SCALARS_INBUFS(sc) (((sc) >> 16) & 0x0ff) /* Retrives number of output buffers from the scalars parameter */#define REMOTE_SCALARS_OUTBUFS(sc) (((sc) >> 8) & 0x0ff) /* Retrives number of input handles from the scalars parameter */#define REMOTE_SCALARS_INHANDLES(sc) (((sc) >> 4) & 0x0f) /* Retrives number of output handles from the scalars parameter */#define REMOTE_SCALARS_OUTHANDLES(sc) ((sc) & 0x0f) #define REMOTE_SCALARS_LENGTH(sc) (REMOTE_SCALARS_INBUFS(sc) + \ REMOTE_SCALARS_OUTBUFS(sc) + \ REMOTE_SCALARS_INHANDLES(sc)+ \ REMOTE_SCALARS_OUTHANDLES(sc))#define FASTRPC_BUILD_SCALARS(attr, method, in, out, oin, oout) \ (((attr & 0x07) << 29) | \ ((method & 0x1f) << 24) | \ ((in & 0xff) << 16) | \ ((out & 0xff) << 8) | \ ((oin & 0x0f) << 4) | \ (oout & 0x0f)) #define FASTRPC_SCALARS(method, in, out) \ FASTRPC_BUILD_SCALARS(0, method, in, out, 0, 0) #define FASTRPC_CREATE_PROCESS_NARGS 6/* Remote Method id table */#define FASTRPC_RMID_INIT_ATTACH 0#define FASTRPC_RMID_INIT_RELEASE 1#define FASTRPC_RMID_INIT_MMAP 4#define FASTRPC_RMID_INIT_MUNMAP 5#define FASTRPC_RMID_INIT_CREATE 6#define FASTRPC_RMID_INIT_CREATE_ATTR 7#define FASTRPC_RMID_INIT_CREATE_STATIC 8 #define miscdev_to_cctx(d) container_of(d, struct fastrpc_channel_ctx, miscdev) static const char *domains[FASTRPC_DEV_MAX] = { "adsp", "mdsp", "sdsp", "cdsp"};struct fastrpc_phy_page { u64 addr; /* physical address */ u64 size; /* size of contiguous region */}; struct fastrpc_invoke_buf { u32 num; /* number of contiguous regions */ u32 pgidx; /* index to start of contiguous region */}; struct fastrpc_remote_arg { u64 pv; u64 len;}; struct fastrpc_mmap_rsp_msg { u64 vaddr;}; struct fastrpc_mmap_req_msg { s32 pgid; u32 flags; u64 vaddr; s32 num;}; struct fastrpc_munmap_req_msg { s32 pgid; u64 vaddr; u64 size;}; struct fastrpc_msg { int pid; /* process group id */ int tid; /* thread id */ u64 ctx; /* invoke caller context */ u32 handle; /* handle to invoke */ u32 sc; /* scalars structure describing the data */ u64 addr; /* physical address */ u64 size; /* size of contiguous region */}; struct fastrpc_invoke_rsp { u64 ctx; /* invoke caller context */ int retval; /* invoke return value */}; struct fastrpc_buf_overlap { u64 start; u64 end; int raix; u64 mstart; u64 mend; u64 offset;}; struct fastrpc_buf { struct fastrpc_user *fl; struct dma_buf *dmabuf; struct device *dev; void *virt; u64 phys; u64 size; /* Lock for dma buf attachments */ struct mutex lock; struct list_head attachments; /* mmap support */ struct list_head node; /* list of user requested mmaps */ uintptr_t raddr;}; struct fastrpc_dma_buf_attachment { struct device *dev; struct sg_table sgt; struct list_head node;}; struct fastrpc_map { struct list_head node; struct fastrpc_user *fl; int fd; struct dma_buf *buf; struct sg_table *table; struct dma_buf_attachment *attach; u64 phys; u64 size; void *va; u64 len; struct kref refcount;}; struct fastrpc_invoke_ctx { int nscalars; int nbufs; int retval; int pid; int tgid; u32 sc; u32 *crc; u64 ctxid; u64 msg_sz; struct kref refcount; struct list_head node; /* list of ctxs */ struct completion work; struct work_struct put_work; struct fastrpc_msg msg; struct fastrpc_user *fl; struct fastrpc_remote_arg *rpra; struct fastrpc_map **maps; struct fastrpc_buf *buf; struct fastrpc_invoke_args *args; struct fastrpc_buf_overlap *olaps; struct fastrpc_channel_ctx *cctx;}; struct fastrpc_session_ctx { struct device *dev; int sid; bool used; bool valid;}; struct fastrpc_channel_ctx { int domain_id; int sesscount; struct rpmsg_device *rpdev; struct fastrpc_session_ctx session[FASTRPC_MAX_SESSIONS]; spinlock_t lock; struct idr ctx_idr; struct list_head users; struct miscdevice miscdev; struct kref refcount;}; struct fastrpc_user { struct list_head user; struct list_head maps; struct list_head pending; struct list_head mmaps; struct fastrpc_channel_ctx *cctx; struct fastrpc_session_ctx *sctx; struct fastrpc_buf *init_mem; int tgid; int pd; /* Lock for lists */ spinlock_t lock; /* lock for allocations */ struct mutex mutex;}; static void fastrpc_free_map(struct kref *ref){ struct fastrpc_map *map; map = container_of(ref, struct fastrpc_map, refcount); if (map->table) { dma_buf_unmap_attachment(map->attach, map->table, DMA_BIDIRECTIONAL); dma_buf_detach(map->buf, map->attach); dma_buf_put(map->buf); } kfree(map);} static void fastrpc_map_put(struct fastrpc_map *map){ if (map) kref_put(&map->refcount, fastrpc_free_map);} static void fastrpc_map_get(struct fastrpc_map *map){ if (map) kref_get(&map->refcount);} static int fastrpc_map_find(struct fastrpc_user *fl, int fd, struct fastrpc_map **ppmap){ struct fastrpc_map *map = NULL; mutex_lock(&fl->mutex); list_for_each_entry(map, &fl->maps, node) { if (map->fd == fd) { fastrpc_map_get(map); *ppmap = map; mutex_unlock(&fl->mutex); return 0; } } mutex_unlock(&fl->mutex); return -ENOENT;} static void fastrpc_buf_free(struct fastrpc_buf *buf){ dma_free_coherent(buf->dev, buf->size, buf->virt, FASTRPC_PHYS(buf->phys)); kfree(buf);} static int fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev, u64 size, struct fastrpc_buf **obuf){ struct fastrpc_buf *buf; buf = kzalloc(sizeof(*buf), GFP_KERNEL); if (!buf) return -ENOMEM; INIT_LIST_HEAD(&buf->attachments); INIT_LIST_HEAD(&buf->node); mutex_init(&buf->lock); buf->fl = fl; buf->virt = NULL; buf->phys = 0; buf->size = size; buf->dev = dev; buf->raddr = 0; buf->virt = dma_alloc_coherent(dev, buf->size, (dma_addr_t *)&buf->phys, GFP_KERNEL); if (!buf->virt) { mutex_destroy(&buf->lock); kfree(buf); return -ENOMEM; } if (fl->sctx && fl->sctx->sid) buf->phys += ((u64)fl->sctx->sid << 32); *obuf = buf; return 0;} static void fastrpc_channel_ctx_free(struct kref *ref){ struct fastrpc_channel_ctx *cctx; cctx = container_of(ref, struct fastrpc_channel_ctx, refcount); kfree(cctx);} static void fastrpc_channel_ctx_get(struct fastrpc_channel_ctx *cctx){ kref_get(&cctx->refcount);} static void fastrpc_channel_ctx_put(struct fastrpc_channel_ctx *cctx){ kref_put(&cctx->refcount, fastrpc_channel_ctx_free);} static void fastrpc_context_free(struct kref *ref){ struct fastrpc_invoke_ctx *ctx; struct fastrpc_channel_ctx *cctx; unsigned long flags; int i; ctx = container_of(ref, struct fastrpc_invoke_ctx, refcount); cctx = ctx->cctx; for (i = 0; i < ctx->nscalars; i++) fastrpc_map_put(ctx->maps[i]); if (ctx->buf) fastrpc_buf_free(ctx->buf); spin_lock_irqsave(&cctx->lock, flags); idr_remove(&cctx->ctx_idr, ctx->ctxid >> 4); spin_unlock_irqrestore(&cctx->lock, flags); kfree(ctx->maps); kfree(ctx->olaps); kfree(ctx); fastrpc_channel_ctx_put(cctx);} static void fastrpc_context_get(struct fastrpc_invoke_ctx *ctx){ kref_get(&ctx->refcount);} static void fastrpc_context_put(struct fastrpc_invoke_ctx *ctx){ kref_put(&ctx->refcount, fastrpc_context_free);} static void fastrpc_context_put_wq(struct work_struct *work){ struct fastrpc_invoke_ctx *ctx = container_of(work, struct fastrpc_invoke_ctx, put_work); fastrpc_context_put(ctx);} #define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1)static int olaps_cmp(const void *a, const void *b){ struct fastrpc_buf_overlap *pa = (struct fastrpc_buf_overlap *)a; struct fastrpc_buf_overlap *pb = (struct fastrpc_buf_overlap *)b; /* sort with lowest starting buffer first */ int st = CMP(pa->start, pb->start); /* sort with highest ending buffer first */ int ed = CMP(pb->end, pa->end); return st == 0 ? ed : st;} static void fastrpc_get_buff_overlaps(struct fastrpc_invoke_ctx *ctx){ u64 max_end = 0; int i; for (i = 0; i < ctx->nbufs; ++i) { ctx->olaps[i].start = ctx->args[i].ptr; ctx->olaps[i].end = ctx->olaps[i].start + ctx->args[i].length; ctx->olaps[i].raix = i; } sort(ctx->olaps, ctx->nbufs, sizeof(*ctx->olaps), olaps_cmp, NULL); for (i = 0; i < ctx->nbufs; ++i) { /* Falling inside previous range */ if (ctx->olaps[i].start < max_end) { ctx->olaps[i].mstart = max_end; ctx->olaps[i].mend = ctx->olaps[i].end; ctx->olaps[i].offset = max_end - ctx->olaps[i].start; if (ctx->olaps[i].end > max_end) { max_end = ctx->olaps[i].end; } else { ctx->olaps[i].mend = 0; ctx->olaps[i].mstart = 0; } } else { ctx->olaps[i].mend = ctx->olaps[i].end; ctx->olaps[i].mstart = ctx->olaps[i].start; ctx->olaps[i].offset = 0; max_end = ctx->olaps[i].end; } }} static struct fastrpc_invoke_ctx *fastrpc_context_alloc( struct fastrpc_user *user, u32 kernel, u32 sc, struct fastrpc_invoke_args *args){ struct fastrpc_channel_ctx *cctx = user->cctx; struct fastrpc_invoke_ctx *ctx = NULL; unsigned long flags; int ret; ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); if (!ctx) return ERR_PTR(-ENOMEM); INIT_LIST_HEAD(&ctx->node); ctx->fl = user; ctx->nscalars = REMOTE_SCALARS_LENGTH(sc); ctx->nbufs = REMOTE_SCALARS_INBUFS(sc) + REMOTE_SCALARS_OUTBUFS(sc); if (ctx->nscalars) { ctx->maps = kcalloc(ctx->nscalars, sizeof(*ctx->maps), GFP_KERNEL); if (!ctx->maps) { kfree(ctx); return ERR_PTR(-ENOMEM); } ctx->olaps = kcalloc(ctx->nscalars, sizeof(*ctx->olaps), GFP_KERNEL); if (!ctx->olaps) { kfree(ctx->maps); kfree(ctx); return ERR_PTR(-ENOMEM); } ctx->args = args; fastrpc_get_buff_overlaps(ctx); } /* Released in fastrpc_context_put() */ fastrpc_channel_ctx_get(cctx); ctx->sc = sc; ctx->retval = -1; ctx->pid = current->pid; ctx->tgid = user->tgid; ctx->cctx = cctx; init_completion(&ctx->work); INIT_WORK(&ctx->put_work, fastrpc_context_put_wq); spin_lock(&user->lock); list_add_tail(&ctx->node, &user->pending); spin_unlock(&user->lock); spin_lock_irqsave(&cctx->lock, flags); ret = idr_alloc_cyclic(&cctx->ctx_idr, ctx, 1, FASTRPC_CTX_MAX, GFP_ATOMIC); if (ret < 0) { spin_unlock_irqrestore(&cctx->lock, flags); goto err_idr; } ctx->ctxid = ret << 4; spin_unlock_irqrestore(&cctx->lock, flags); kref_init(&ctx->refcount); return ctx;err_idr:Lines 1–500 of 1749. Load more
// SPDX-License-Identifier: GPL-2.0 // Copyright (c) 2011-2018, The Linux Foundation. All rights reserved. // Copyright (c) 2018, Linaro Limited #include <linux/completion.h> #include <linux/device.h> #include <linux/dma-buf.h> #include <linux/dma-mapping.h> #include <linux/idr.h> #include <linux/list.h> #include <linux/miscdevice.h> #include <linux/module.h> #include <linux/of_address.h> #include <linux/of.h> #include <linux/sort.h> #include <linux/of_platform.h> #include <linux/rpmsg.h> #include <linux/scatterlist.h> #include <linux/slab.h> #include <uapi/misc/fastrpc.h> #define ADSP_DOMAIN_ID (0) #define MDSP_DOMAIN_ID (1) #define SDSP_DOMAIN_ID (2) #define CDSP_DOMAIN_ID (3) #define FASTRPC_DEV_MAX 4 /* adsp, mdsp, slpi, cdsp*/ #define FASTRPC_MAX_SESSIONS 9 /*8 compute, 1 cpz*/ #define FASTRPC_ALIGN 128 #define FASTRPC_MAX_FDLIST 16 #define FASTRPC_MAX_CRCLIST 64 #define FASTRPC_PHYS(p) ((p) & 0xffffffff) #define FASTRPC_CTX_MAX (256) #define FASTRPC_INIT_HANDLE 1 #define FASTRPC_CTXID_MASK (0xFF0) #define INIT_FILELEN_MAX (2 * 1024 * 1024) #define FASTRPC_DEVICE_NAME "fastrpc" #define ADSP_MMAP_ADD_PAGES 0x1000 /* Retrives number of input buffers from the scalars parameter */ #define REMOTE_SCALARS_INBUFS(sc) (((sc) >> 16) & 0x0ff) /* Retrives number of output buffers from the scalars parameter */ #define REMOTE_SCALARS_OUTBUFS(sc) (((sc) >> 8) & 0x0ff) /* Retrives number of input handles from the scalars parameter */ #define REMOTE_SCALARS_INHANDLES(sc) (((sc) >> 4) & 0x0f) /* Retrives number of output handles from the scalars parameter */ #define REMOTE_SCALARS_OUTHANDLES(sc) ((sc) & 0x0f) #define REMOTE_SCALARS_LENGTH(sc) (REMOTE_SCALARS_INBUFS(sc) + \ REMOTE_SCALARS_OUTBUFS(sc) + \ REMOTE_SCALARS_INHANDLES(sc)+ \ REMOTE_SCALARS_OUTHANDLES(sc)) #define FASTRPC_BUILD_SCALARS(attr, method, in, out, oin, oout) \ (((attr & 0x07) << 29) | \ ((method & 0x1f) << 24) | \ ((in & 0xff) << 16) | \ ((out & 0xff) << 8) | \ ((oin & 0x0f) << 4) | \ (oout & 0x0f)) #define FASTRPC_SCALARS(method, in, out) \ FASTRPC_BUILD_SCALARS(0, method, in, out, 0, 0) #define FASTRPC_CREATE_PROCESS_NARGS 6 /* Remote Method id table */ #define FASTRPC_RMID_INIT_ATTACH 0 #define FASTRPC_RMID_INIT_RELEASE 1 #define FASTRPC_RMID_INIT_MMAP 4 #define FASTRPC_RMID_INIT_MUNMAP 5 #define FASTRPC_RMID_INIT_CREATE 6 #define FASTRPC_RMID_INIT_CREATE_ATTR 7 #define FASTRPC_RMID_INIT_CREATE_STATIC 8 #define miscdev_to_cctx(d) container_of(d, struct fastrpc_channel_ctx, miscdev) static const char *domains[FASTRPC_DEV_MAX] = { "adsp", "mdsp", "sdsp", "cdsp"}; struct fastrpc_phy_page { u64 addr; /* physical address */ u64 size; /* size of contiguous region */ }; struct fastrpc_invoke_buf { u32 num; /* number of contiguous regions */ u32 pgidx; /* index to start of contiguous region */ }; struct fastrpc_remote_arg { u64 pv; u64 len; }; struct fastrpc_mmap_rsp_msg { u64 vaddr; }; struct fastrpc_mmap_req_msg { s32 pgid; u32 flags; u64 vaddr; s32 num; }; struct fastrpc_munmap_req_msg { s32 pgid; u64 vaddr; u64 size; }; struct fastrpc_msg { int pid; /* process group id */ int tid; /* thread id */ u64 ctx; /* invoke caller context */ u32 handle; /* handle to invoke */ u32 sc; /* scalars structure describing the data */ u64 addr; /* physical address */ u64 size; /* size of contiguous region */ }; struct fastrpc_invoke_rsp { u64 ctx; /* invoke caller context */ int retval; /* invoke return value */ }; struct fastrpc_buf_overlap { u64 start; u64 end; int raix; u64 mstart; u64 mend; u64 offset; }; struct fastrpc_buf { struct fastrpc_user *fl; struct dma_buf *dmabuf; struct device *dev; void *virt; u64 phys; u64 size; /* Lock for dma buf attachments */ struct mutex lock; struct list_head attachments; /* mmap support */ struct list_head node; /* list of user requested mmaps */ uintptr_t raddr; }; struct fastrpc_dma_buf_attachment { struct device *dev; struct sg_table sgt; struct list_head node; }; struct fastrpc_map { struct list_head node; struct fastrpc_user *fl; int fd; struct dma_buf *buf; struct sg_table *table; struct dma_buf_attachment *attach; u64 phys; u64 size; void *va; u64 len; struct kref refcount; }; struct fastrpc_invoke_ctx { int nscalars; int nbufs; int retval; int pid; int tgid; u32 sc; u32 *crc; u64 ctxid; u64 msg_sz; struct kref refcount; struct list_head node; /* list of ctxs */ struct completion work; struct work_struct put_work; struct fastrpc_msg msg; struct fastrpc_user *fl; struct fastrpc_remote_arg *rpra; struct fastrpc_map **maps; struct fastrpc_buf *buf; struct fastrpc_invoke_args *args; struct fastrpc_buf_overlap *olaps; struct fastrpc_channel_ctx *cctx; }; struct fastrpc_session_ctx { struct device *dev; int sid; bool used; bool valid; }; struct fastrpc_channel_ctx { int domain_id; int sesscount; struct rpmsg_device *rpdev; struct fastrpc_session_ctx session[FASTRPC_MAX_SESSIONS]; spinlock_t lock; struct idr ctx_idr; struct list_head users; struct miscdevice miscdev; struct kref refcount; }; struct fastrpc_user { struct list_head user; struct list_head maps; struct list_head pending; struct list_head mmaps; struct fastrpc_channel_ctx *cctx; struct fastrpc_session_ctx *sctx; struct fastrpc_buf *init_mem; int tgid; int pd; /* Lock for lists */ spinlock_t lock; /* lock for allocations */ struct mutex mutex; }; static void fastrpc_free_map(struct kref *ref) { struct fastrpc_map *map; map = container_of(ref, struct fastrpc_map, refcount); if (map->table) { dma_buf_unmap_attachment(map->attach, map->table, DMA_BIDIRECTIONAL); dma_buf_detach(map->buf, map->attach); dma_buf_put(map->buf); } kfree(map); } static void fastrpc_map_put(struct fastrpc_map *map) { if (map) kref_put(&map->refcount, fastrpc_free_map); } static void fastrpc_map_get(struct fastrpc_map *map) { if (map) kref_get(&map->refcount); } static int fastrpc_map_find(struct fastrpc_user *fl, int fd, struct fastrpc_map **ppmap) { struct fastrpc_map *map = NULL; mutex_lock(&fl->mutex); list_for_each_entry(map, &fl->maps, node) { if (map->fd == fd) { fastrpc_map_get(map); *ppmap = map; mutex_unlock(&fl->mutex); return 0; } } mutex_unlock(&fl->mutex); return -ENOENT; } static void fastrpc_buf_free(struct fastrpc_buf *buf) { dma_free_coherent(buf->dev, buf->size, buf->virt, FASTRPC_PHYS(buf->phys)); kfree(buf); } static int fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev, u64 size, struct fastrpc_buf **obuf) { struct fastrpc_buf *buf; buf = kzalloc(sizeof(*buf), GFP_KERNEL); if (!buf) return -ENOMEM; INIT_LIST_HEAD(&buf->attachments); INIT_LIST_HEAD(&buf->node); mutex_init(&buf->lock); buf->fl = fl; buf->virt = NULL; buf->phys = 0; buf->size = size; buf->dev = dev; buf->raddr = 0; buf->virt = dma_alloc_coherent(dev, buf->size, (dma_addr_t *)&buf->phys, GFP_KERNEL); if (!buf->virt) { mutex_destroy(&buf->lock); kfree(buf); return -ENOMEM; } if (fl->sctx && fl->sctx->sid) buf->phys += ((u64)fl->sctx->sid << 32); *obuf = buf; return 0; } static void fastrpc_channel_ctx_free(struct kref *ref) { struct fastrpc_channel_ctx *cctx; cctx = container_of(ref, struct fastrpc_channel_ctx, refcount); kfree(cctx); } static void fastrpc_channel_ctx_get(struct fastrpc_channel_ctx *cctx) { kref_get(&cctx->refcount); } static void fastrpc_channel_ctx_put(struct fastrpc_channel_ctx *cctx) { kref_put(&cctx->refcount, fastrpc_channel_ctx_free); } static void fastrpc_context_free(struct kref *ref) { struct fastrpc_invoke_ctx *ctx; struct fastrpc_channel_ctx *cctx; unsigned long flags; int i; ctx = container_of(ref, struct fastrpc_invoke_ctx, refcount); cctx = ctx->cctx; for (i = 0; i < ctx->nscalars; i++) fastrpc_map_put(ctx->maps[i]); if (ctx->buf) fastrpc_buf_free(ctx->buf); spin_lock_irqsave(&cctx->lock, flags); idr_remove(&cctx->ctx_idr, ctx->ctxid >> 4); spin_unlock_irqrestore(&cctx->lock, flags); kfree(ctx->maps); kfree(ctx->olaps); kfree(ctx); fastrpc_channel_ctx_put(cctx); } static void fastrpc_context_get(struct fastrpc_invoke_ctx *ctx) { kref_get(&ctx->refcount); } static void fastrpc_context_put(struct fastrpc_invoke_ctx *ctx) { kref_put(&ctx->refcount, fastrpc_context_free); } static void fastrpc_context_put_wq(struct work_struct *work) { struct fastrpc_invoke_ctx *ctx = container_of(work, struct fastrpc_invoke_ctx, put_work); fastrpc_context_put(ctx); } #define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1) static int olaps_cmp(const void *a, const void *b) { struct fastrpc_buf_overlap *pa = (struct fastrpc_buf_overlap *)a; struct fastrpc_buf_overlap *pb = (struct fastrpc_buf_overlap *)b; /* sort with lowest starting buffer first */ int st = CMP(pa->start, pb->start); /* sort with highest ending buffer first */ int ed = CMP(pb->end, pa->end); return st == 0 ? ed : st; } static void fastrpc_get_buff_overlaps(struct fastrpc_invoke_ctx *ctx) { u64 max_end = 0; int i; for (i = 0; i < ctx->nbufs; ++i) { ctx->olaps[i].start = ctx->args[i].ptr; ctx->olaps[i].end = ctx->olaps[i].start + ctx->args[i].length; ctx->olaps[i].raix = i; } sort(ctx->olaps, ctx->nbufs, sizeof(*ctx->olaps), olaps_cmp, NULL); for (i = 0; i < ctx->nbufs; ++i) { /* Falling inside previous range */ if (ctx->olaps[i].start < max_end) { ctx->olaps[i].mstart = max_end; ctx->olaps[i].mend = ctx->olaps[i].end; ctx->olaps[i].offset = max_end - ctx->olaps[i].start; if (ctx->olaps[i].end > max_end) { max_end = ctx->olaps[i].end; } else { ctx->olaps[i].mend = 0; ctx->olaps[i].mstart = 0; } } else { ctx->olaps[i].mend = ctx->olaps[i].end; ctx->olaps[i].mstart = ctx->olaps[i].start; ctx->olaps[i].offset = 0; max_end = ctx->olaps[i].end; } } } static struct fastrpc_invoke_ctx *fastrpc_context_alloc( struct fastrpc_user *user, u32 kernel, u32 sc, struct fastrpc_invoke_args *args) { struct fastrpc_channel_ctx *cctx = user->cctx; struct fastrpc_invoke_ctx *ctx = NULL; unsigned long flags; int ret; ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); if (!ctx) return ERR_PTR(-ENOMEM); INIT_LIST_HEAD(&ctx->node); ctx->fl = user; ctx->nscalars = REMOTE_SCALARS_LENGTH(sc); ctx->nbufs = REMOTE_SCALARS_INBUFS(sc) + REMOTE_SCALARS_OUTBUFS(sc); if (ctx->nscalars) { ctx->maps = kcalloc(ctx->nscalars, sizeof(*ctx->maps), GFP_KERNEL); if (!ctx->maps) { kfree(ctx); return ERR_PTR(-ENOMEM); } ctx->olaps = kcalloc(ctx->nscalars, sizeof(*ctx->olaps), GFP_KERNEL); if (!ctx->olaps) { kfree(ctx->maps); kfree(ctx); return ERR_PTR(-ENOMEM); } ctx->args = args; fastrpc_get_buff_overlaps(ctx); } /* Released in fastrpc_context_put() */ fastrpc_channel_ctx_get(cctx); ctx->sc = sc; ctx->retval = -1; ctx->pid = current->pid; ctx->tgid = user->tgid; ctx->cctx = cctx; init_completion(&ctx->work); INIT_WORK(&ctx->put_work, fastrpc_context_put_wq); spin_lock(&user->lock); list_add_tail(&ctx->node, &user->pending); spin_unlock(&user->lock); spin_lock_irqsave(&cctx->lock, flags); ret = idr_alloc_cyclic(&cctx->ctx_idr, ctx, 1, FASTRPC_CTX_MAX, GFP_ATOMIC); if (ret < 0) { spin_unlock_irqrestore(&cctx->lock, flags); goto err_idr; } ctx->ctxid = ret << 4; spin_unlock_irqrestore(&cctx->lock, flags); kref_init(&ctx->refcount); return ctx; err_idr: