History

misc: fastrpc: Add polling mode support for fastRPC driver

For any remote call to DSP, after sending an invocation message, the fastRPC driver waits for a glink response, during which the CPU can enter low power modes. This adds latency to the fastRPC call due to CPU wakeup and scheduling overhead. Add polling mode support where the fastRPC driver polls a shared memory location for completion after sending the invocation, avoiding CPU wakeup and scheduling latency and reducing fastRPC overhead. If the poll times out, the call falls back to the normal interrupt/glink-based completion path. Poll mode is only applied to dynamic modules running in a user PD (handle > FASTRPC_MAX_STATIC_HANDLE), since static/root-PD handles are not expected to benefit from, or require, this optimization. Support is advertised per SoC via fastrpc_soc_data, with a closed exception list for older platforms whose DSP firmware is known to support polling but which otherwise use the default soc_data. Poll mode can be enabled by userspace via the FASTRPC_IOCTL_SET_OPTION ioctl with the FASTRPC_POLL_MODE request id. Since context IDs (ctxid) are allocated from a fixed-size, per-channel cyclic IDR shared by all processes on a DSP, a context ID can be recycled for a new request soon after it is freed. In poll mode the context can be considered complete (and released) as soon as the poll memory is updated, while the corresponding glink COMPLETE response from the DSP may still be in flight. If that response arrives after the ctxid has been reused, it would otherwise match the new context and incorrectly signal completion for it while the DSP may still be operating on the new context's buffers. To prevent this, embed a monotonically increasing per-channel sequence number in the unused upper bits of the ctxid/message context and validate it in the rpmsg callback, dropping any response whose sequence number does not match the current owner of that ctxid slot.

Ekansh Gupta committed 3 months ago

Commit 96e62cc1 parent 62373b72 files · +215 −8Browse files

Unified · Split

2 files
  1. drivers/misc/fastrpc.c +186 −8
  2. include/uapi/misc/fastrpc.h +29 −0
drivers/misc/fastrpc.c+186−8
@@ -25,6 +25,8 @@ · 24 unmodified lines
25#include <linux/bitfield.h>25#include <linux/bitfield.h>
26#include <linux/bits.h>26#include <linux/bits.h>
27#include <linux/bitops.h>27#include <linux/bitops.h>
28#include <linux/compiler.h>
29#include <linux/iopoll.h>
28 30
29#define ADSP_DOMAIN_ID (0)31#define ADSP_DOMAIN_ID (0)
30#define MDSP_DOMAIN_ID (1)32#define MDSP_DOMAIN_ID (1)
@@ -39,7 +41,16 @@ · 8 unmodified lines
39#define FASTRPC_CTX_MAX (256)41#define FASTRPC_CTX_MAX (256)
40#define FASTRPC_INIT_HANDLE 142#define FASTRPC_INIT_HANDLE 1
41#define FASTRPC_DSP_UTILITIES_HANDLE 243#define FASTRPC_DSP_UTILITIES_HANDLE 2
44/*
45 * Maximum handle value for static handles.
46 * Static handles are pre-defined, fixed numeric values statically assigned
47 * in the IDL file or FastRPC framework.
48 */
49#define FASTRPC_MAX_STATIC_HANDLE (20)
42#define FASTRPC_CTXID_MASK GENMASK(15, 8)50#define FASTRPC_CTXID_MASK GENMASK(15, 8)
51/* Sequence number occupies bits 63:16 of the ctxid / message context */
52#define FASTRPC_CTXID_SEQ_SHIFT 16
53#define FASTRPC_CTXID_SEQ_MASK GENMASK_ULL(63, 16)
43#define INIT_FILELEN_MAX (2 * 1024 * 1024)54#define INIT_FILELEN_MAX (2 * 1024 * 1024)
44#define INIT_FILE_NAMELEN_MAX (128)55#define INIT_FILE_NAMELEN_MAX (128)
45#define FASTRPC_DEVICE_NAME "fastrpc"56#define FASTRPC_DEVICE_NAME "fastrpc"
@@ -107,6 +118,12 @@ · 61 unmodified lines
107 118
108#define miscdev_to_fdevice(d) container_of(d, struct fastrpc_device, miscdev)119#define miscdev_to_fdevice(d) container_of(d, struct fastrpc_device, miscdev)
109 120
121/* Poll response number from remote processor for call completion */
122#define FASTRPC_POLL_RESPONSE (0xdecaf)
123
124/* Polling mode timeout limit */
125#define FASTRPC_POLL_MAX_TIMEOUT_US (10000)
126
110struct fastrpc_phy_page {127struct fastrpc_phy_page {
111 dma_addr_t addr; /* dma address */128 dma_addr_t addr; /* dma address */
112 u64 size; /* size of contiguous region */129 u64 size; /* size of contiguous region */
@@ -237,8 +254,14 @@ struct fastrpc_invoke_ctx { · 124 unmodified lines
237 u32 sc;254 u32 sc;
238 u64 *fdlist;255 u64 *fdlist;
239 u32 *crc;256 u32 *crc;
257 /* Poll memory that DSP updates */
258 u32 *poll_addr;
240 u64 ctxid;259 u64 ctxid;
241 u64 msg_sz;260 u64 msg_sz;
261 /* work done status flag */
262 bool is_work_done;
263 /* process updates poll memory instead of glink response */
264 bool is_polled;
242 struct kref refcount;265 struct kref refcount;
243 struct list_head node; /* list of ctxs */266 struct list_head node; /* list of ctxs */
244 struct completion work;267 struct completion work;
@@ -264,6 +287,7 @@ struct fastrpc_soc_data { · 19 unmodified lines
264 u32 sid_pos;287 u32 sid_pos;
265 u32 dma_addr_bits_cdsp;288 u32 dma_addr_bits_cdsp;
266 u32 dma_addr_bits_default;289 u32 dma_addr_bits_default;
290 bool poll_mode_supported;
267};291};
268 292
269struct fastrpc_channel_ctx {293struct fastrpc_channel_ctx {
@@ -286,6 +310,9 @@ struct fastrpc_channel_ctx { · 16 unmodified lines
286 struct list_head invoke_interrupted_mmaps;310 struct list_head invoke_interrupted_mmaps;
287 bool secure;311 bool secure;
288 bool unsigned_support;312 bool unsigned_support;
313 bool poll_mode_supported;
314 /* Per-channel sequence counter; incremented on every context allocation */
315 atomic_t ctx_seq;
289 u64 dma_mask;316 u64 dma_mask;
290 const struct fastrpc_soc_data *soc_data;317 const struct fastrpc_soc_data *soc_data;
291};318};
@@ -309,6 +336,8 @@ struct fastrpc_user { · 17 unmodified lines
309 int client_id;336 int client_id;
310 int pd;337 int pd;
311 bool is_secure_dev;338 bool is_secure_dev;
339 /* Flags poll mode state */
340 bool poll_mode;
312 /* Lock for lists */341 /* Lock for lists */
313 spinlock_t lock;342 spinlock_t lock;
314 /* lock for allocations */343 /* lock for allocations */
@@ -706,7 +735,9 @@ static struct fastrpc_invoke_ctx *fastrpc_context_alloc( · 391 unmodified lines
706 spin_unlock_irqrestore(&cctx->lock, flags);735 spin_unlock_irqrestore(&cctx->lock, flags);
707 goto err_idr;736 goto err_idr;
708 }737 }
709 ctx->ctxid = FIELD_PREP(FASTRPC_CTXID_MASK, ret);738 ctx->ctxid = FIELD_PREP(FASTRPC_CTXID_MASK, ret) |
739 FIELD_PREP(FASTRPC_CTXID_SEQ_MASK,
740 (u64)atomic_inc_return(&cctx->ctx_seq));
710 spin_unlock_irqrestore(&cctx->lock, flags);741 spin_unlock_irqrestore(&cctx->lock, flags);
711 742
712 kref_init(&ctx->refcount);743 kref_init(&ctx->refcount);
@@ -971,7 +1002,8 @@ static int fastrpc_get_meta_size(struct fastrpc_invoke_ctx *ctx) · 258 unmodified lines
971 sizeof(struct fastrpc_invoke_buf) +1002 sizeof(struct fastrpc_invoke_buf) +
972 sizeof(struct fastrpc_phy_page)) * ctx->nscalars +1003 sizeof(struct fastrpc_phy_page)) * ctx->nscalars +
973 sizeof(u64) * FASTRPC_MAX_FDLIST +1004 sizeof(u64) * FASTRPC_MAX_FDLIST +
974 sizeof(u32) * FASTRPC_MAX_CRCLIST;1005 sizeof(u32) * FASTRPC_MAX_CRCLIST +
1006 sizeof(u32);
975 1007
976 return size;1008 return size;
977}1009}
@@ -1067,6 +1099,9 @@ static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx) · 89 unmodified lines
1067 list = fastrpc_invoke_buf_start(rpra, ctx->nscalars);1099 list = fastrpc_invoke_buf_start(rpra, ctx->nscalars);
1068 pages = fastrpc_phy_page_start(list, ctx->nscalars);1100 pages = fastrpc_phy_page_start(list, ctx->nscalars);
1069 ctx->fdlist = (u64 *)(pages + ctx->nscalars);1101 ctx->fdlist = (u64 *)(pages + ctx->nscalars);
1102 ctx->poll_addr = (u32 *)((uintptr_t)ctx->fdlist + sizeof(u64) * FASTRPC_MAX_FDLIST +
1103 sizeof(u32) * FASTRPC_MAX_CRCLIST);
1104
1070 args = (uintptr_t)ctx->buf->virt + metalen;1105 args = (uintptr_t)ctx->buf->virt + metalen;
1071 rlen = pkt_size - metalen;1106 rlen = pkt_size - metalen;
1072 ctx->rpra = rpra;1107 ctx->rpra = rpra;
@@ -1236,6 +1271,71 @@ static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx, · 163 unmodified lines
1236 1271
1237}1272}
1238 1273
1274static u32 fastrpc_read_poll_addr(struct fastrpc_invoke_ctx *ctx)
1275{
1276 dma_rmb();
1277 return READ_ONCE(*ctx->poll_addr);
1278}
1279
1280static int poll_for_remote_response(struct fastrpc_invoke_ctx *ctx)
1281{
1282 u32 val;
1283 int ret;
1284
1285 /*
1286 * Poll until DSP writes FASTRPC_POLL_RESPONSE into *ctx->poll_addr
1287 * or until another path marks the work done.
1288 */
1289 ret = read_poll_timeout_atomic(fastrpc_read_poll_addr, val,
1290 (val == FASTRPC_POLL_RESPONSE) || ctx->is_work_done, 1,
1291 FASTRPC_POLL_MAX_TIMEOUT_US, false, ctx);
1292
1293 if (!ret && val == FASTRPC_POLL_RESPONSE) {
1294 /*
1295 * DSP writes FASTRPC_POLL_RESPONSE to signal successful
1296 * completion via the poll path.
1297 */
1298 ctx->is_work_done = true;
1299 ctx->retval = 0;
1300 }
1301
1302 if (ret == -ETIMEDOUT)
1303 ret = -EIO;
1304
1305 return ret;
1306}
1307
1308static inline int fastrpc_wait_for_response(struct fastrpc_invoke_ctx *ctx,
1309 u32 kernel)
1310{
1311 int err = 0;
1312
1313 if (kernel) {
1314 if (!wait_for_completion_timeout(&ctx->work, 10 * HZ))
1315 err = -ETIMEDOUT;
1316 } else {
1317 err = wait_for_completion_interruptible(&ctx->work);
1318 }
1319
1320 return err;
1321}
1322
1323static int fastrpc_wait_for_completion(struct fastrpc_invoke_ctx *ctx,
1324 u32 kernel)
1325{
1326 int err;
1327
1328 if (ctx->is_polled) {
1329 err = poll_for_remote_response(ctx);
1330 if (!err)
1331 return 0;
1332 /* If polling timed out or failed, move to normal response mode */
1333 ctx->is_polled = false;
1334 }
1335
1336 return fastrpc_wait_for_response(ctx, kernel);
1337}
1338
1239static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel,1339static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel,
1240 u32 handle, u32 sc,1340 u32 handle, u32 sc,
1241 struct fastrpc_invoke_args *args)1341 struct fastrpc_invoke_args *args)
@@ -1271,13 +1371,14 @@ static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel, · 29 unmodified lines
1271 if (err)1371 if (err)
1272 goto bail;1372 goto bail;
1273 1373
1274 if (kernel) {1374 /*
1275 if (!wait_for_completion_timeout(&ctx->work, 10 * HZ))1375 * Set message context as polled if the call is for a user PD
1276 err = -ETIMEDOUT;1376 * dynamic module and user has enabled poll mode.
1277 } else {1377 */
1278 err = wait_for_completion_interruptible(&ctx->work);1378 if (handle > FASTRPC_MAX_STATIC_HANDLE && fl->pd == USER_PD && fl->poll_mode)
1279 }1379 ctx->is_polled = true;
1280 1380
1381 err = fastrpc_wait_for_completion(ctx, kernel);
1281 if (err)1382 if (err)
1282 goto bail;1383 goto bail;
1283 1384
@@ -1841,6 +1942,35 @@ done: · 557 unmodified lines
1841 return 0;1942 return 0;
1842}1943}
1843 1944
1945static int fastrpc_set_option(struct fastrpc_user *fl, char __user *argp)
1946{
1947 struct fastrpc_ioctl_set_option opt = {0};
1948 int i;
1949
1950 if (copy_from_user(&opt, argp, sizeof(opt)))
1951 return -EFAULT;
1952
1953 for (i = 0; i < ARRAY_SIZE(opt.reserved); i++) {
1954 if (opt.reserved[i] != 0)
1955 return -EINVAL;
1956 }
1957
1958 if (opt.request_id != FASTRPC_POLL_MODE)
1959 return -EINVAL;
1960
1961 if (!fl->cctx->poll_mode_supported)
1962 return -EOPNOTSUPP;
1963
1964 if (opt.value == FASTRPC_POLL_MODE_ENABLE)
1965 fl->poll_mode = true;
1966 else if (opt.value == FASTRPC_POLL_MODE_DISABLE)
1967 fl->poll_mode = false;
1968 else
1969 return -EINVAL;
1970
1971 return 0;
1972}
1973
1844static int fastrpc_get_dsp_info(struct fastrpc_user *fl, char __user *argp)1974static int fastrpc_get_dsp_info(struct fastrpc_user *fl, char __user *argp)
1845{1975{
1846 struct fastrpc_ioctl_capability cap = {0};1976 struct fastrpc_ioctl_capability cap = {0};
@@ -2196,6 +2326,9 @@ static long fastrpc_device_ioctl(struct file *file, unsigned int cmd, · 349 unmodified lines
2196 case FASTRPC_IOCTL_MEM_UNMAP:2326 case FASTRPC_IOCTL_MEM_UNMAP:
2197 err = fastrpc_req_mem_unmap(fl, argp);2327 err = fastrpc_req_mem_unmap(fl, argp);
2198 break;2328 break;
2329 case FASTRPC_IOCTL_SET_OPTION:
2330 err = fastrpc_set_option(fl, argp);
2331 break;
2199 case FASTRPC_IOCTL_GET_DSP_INFO:2332 case FASTRPC_IOCTL_GET_DSP_INFO:
2200 err = fastrpc_get_dsp_info(fl, argp);2333 err = fastrpc_get_dsp_info(fl, argp);
2201 break;2334 break;
@@ -2352,6 +2485,7 @@ static const struct fastrpc_soc_data kaanapali_soc_data = { · 150 unmodified lines
2352 .sid_pos = 56,2485 .sid_pos = 56,
2353 .dma_addr_bits_cdsp = 34,2486 .dma_addr_bits_cdsp = 34,
2354 .dma_addr_bits_default = 32,2487 .dma_addr_bits_default = 32,
2488 .poll_mode_supported = true,
2355};2489};
2356 2490
2357static const struct fastrpc_soc_data default_soc_data = {2491static const struct fastrpc_soc_data default_soc_data = {
@@ -2360,6 +2494,29 @@ static const struct fastrpc_soc_data default_soc_data = { · 2 unmodified lines
2360 .dma_addr_bits_default = 32,2494 .dma_addr_bits_default = 32,
2361};2495};
2362 2496
2497/*
2498 * Exception list for older platforms that use default_soc_data but whose
2499 * DSP firmware supports FastRPC polling mode.
2500 *
2501 * NOTE: This list is intentionally closed.
2502 * Do NOT add new platforms here. New SoCs must advertise polling mode
2503 * support via their soc_data.
2504 */
2505
2506static const struct of_device_id fastrpc_poll_supported_machines[] __maybe_unused = {
2507 { .compatible = "qcom,milos" },
2508 { .compatible = "qcom,qcs8300" },
2509 { .compatible = "qcom,sa8775p" },
2510 { .compatible = "qcom,sar2130p" },
2511 { .compatible = "qcom,sm8450" },
2512 { .compatible = "qcom,sm8550" },
2513 { .compatible = "qcom,sm8650" },
2514 { .compatible = "qcom,sm8750" },
2515 { .compatible = "qcom,x1e80100" },
2516 { .compatible = "qcom,x1p42100" },
2517 {},
2518};
2519
2363static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev)2520static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev)
2364{2521{
2365 struct device *rdev = &rpdev->dev;2522 struct device *rdev = &rpdev->dev;
@@ -2426,6 +2583,8 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev) · 60 unmodified lines
2426 secure_dsp = !(of_property_read_bool(rdev->of_node, "qcom,non-secure-domain"));2583 secure_dsp = !(of_property_read_bool(rdev->of_node, "qcom,non-secure-domain"));
2427 data->secure = secure_dsp;2584 data->secure = secure_dsp;
2428 data->soc_data = soc_data;2585 data->soc_data = soc_data;
2586 data->poll_mode_supported = soc_data->poll_mode_supported ||
2587 of_machine_get_match(fastrpc_poll_supported_machines);
2429 2588
2430 switch (domain_id) {2589 switch (domain_id) {
2431 case ADSP_DOMAIN_ID:2590 case ADSP_DOMAIN_ID:
@@ -2455,6 +2614,7 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev) · 23 unmodified lines
2455 }2614 }
2456 2615
2457 kref_init(&data->refcount);2616 kref_init(&data->refcount);
2617 atomic_set(&data->ctx_seq, 0);
2458 2618
2459 rdev->dma_mask = &data->dma_mask;2619 rdev->dma_mask = &data->dma_mask;
2460 dma_set_mask_and_coherent(rdev, DMA_BIT_MASK(32));2620 dma_set_mask_and_coherent(rdev, DMA_BIT_MASK(32));
@@ -2552,7 +2712,25 @@ static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data, · 91 unmodified lines
2552 return -ENOENT;2712 return -ENOENT;
2553 }2713 }
2554 2714
2715 /*
2716 * Validate the sequence number embedded in the upper bits of the
2717 * context ID. Under high concurrency the IDR slot can be recycled
2718 * for a new request before a late (or duplicate) glink COMPLETE
2719 * response for the previous request arrives. Without this check the
2720 * driver would call complete() on the wrong context, waking a thread
2721 * whose buffers are still being accessed by the DSP.
2722 */
2723 if (FIELD_GET(FASTRPC_CTXID_SEQ_MASK, rsp->ctx) !=
2724 FIELD_GET(FASTRPC_CTXID_SEQ_MASK, ctx->ctxid)) {
2725 dev_dbg(&rpdev->dev,
2726 "Stale glink response ctx 0x%llx (expected seq 0x%llx), dropping\n",
2727 rsp->ctx,
2728 FIELD_GET(FASTRPC_CTXID_SEQ_MASK, ctx->ctxid));
2729 return 0;
2730 }
2731
2555 ctx->retval = rsp->retval;2732 ctx->retval = rsp->retval;
2733 ctx->is_work_done = true;
2556 complete(&ctx->work);2734 complete(&ctx->work);
2557 2735
2558 /*2736 /*
include/uapi/misc/fastrpc.h+29−0
@@ -16,6 +16,7 @@ · 15 unmodified lines
16#define FASTRPC_IOCTL_INIT_CREATE_STATIC _IOWR('R', 9, struct fastrpc_init_create_static)16#define FASTRPC_IOCTL_INIT_CREATE_STATIC _IOWR('R', 9, struct fastrpc_init_create_static)
17#define FASTRPC_IOCTL_MEM_MAP _IOWR('R', 10, struct fastrpc_mem_map)17#define FASTRPC_IOCTL_MEM_MAP _IOWR('R', 10, struct fastrpc_mem_map)
18#define FASTRPC_IOCTL_MEM_UNMAP _IOWR('R', 11, struct fastrpc_mem_unmap)18#define FASTRPC_IOCTL_MEM_UNMAP _IOWR('R', 11, struct fastrpc_mem_unmap)
19#define FASTRPC_IOCTL_SET_OPTION _IOWR('R', 12, struct fastrpc_ioctl_set_option)
19#define FASTRPC_IOCTL_GET_DSP_INFO _IOWR('R', 13, struct fastrpc_ioctl_capability)20#define FASTRPC_IOCTL_GET_DSP_INFO _IOWR('R', 13, struct fastrpc_ioctl_capability)
20 21
21/**22/**
@@ -67,6 +68,28 @@ enum fastrpc_proc_attr { · 45 unmodified lines
67/* Fastrpc attribute for memory protection of buffers */68/* Fastrpc attribute for memory protection of buffers */
68#define FASTRPC_ATTR_SECUREMAP (1)69#define FASTRPC_ATTR_SECUREMAP (1)
69 70
71/**
72 * FASTRPC_POLL_MODE - Enable/disable poll mode for FastRPC invocations
73 *
74 * Poll mode is an optimization that allows the CPU to poll shared memory
75 * for completion instead of waiting for an interrupt-based response.
76 * This reduces latency for fast-completing operations.
77 *
78 * Restrictions:
79 * - Only supported for USER_PD (User Protection Domain)
80 * - Only applies to dynamic modules (handle > 20)
81 * - Static modules always use interrupt-based completion
82 *
83 * Values:
84 * - 0: Disable poll mode (use interrupt-based completion)
85 * - 1: Enable poll mode (poll shared memory for completion)
86 */
87#define FASTRPC_POLL_MODE (1)
88
89/* Values for FASTRPC_POLL_MODE request */
90#define FASTRPC_POLL_MODE_DISABLE 0
91#define FASTRPC_POLL_MODE_ENABLE 1
92
70struct fastrpc_invoke_args {93struct fastrpc_invoke_args {
71 __u64 ptr;94 __u64 ptr;
72 __u64 length;95 __u64 length;
@@ -133,6 +156,12 @@ struct fastrpc_mem_unmap { · 60 unmodified lines
133 __s32 reserved[5];156 __s32 reserved[5];
134};157};
135 158
159struct fastrpc_ioctl_set_option {
160 __u32 request_id; /* Request type (e.g., FASTRPC_POLL_MODE) */
161 __u32 value; /* Request-specific value */
162 __s32 reserved[6];
163};
164
136struct fastrpc_ioctl_capability {165struct fastrpc_ioctl_capability {
137 __u32 unused; /* deprecated, ignored by the kernel */166 __u32 unused; /* deprecated, ignored by the kernel */
138 __u32 attribute_id;167 __u32 attribute_id;