History

misc: fastrpc: Add polling mode support for fastRPC driver

For any remote call to DSP, after sending an invocation message, the fastRPC driver waits for a glink response, during which the CPU can enter low power modes. This adds latency to the fastRPC call due to CPU wakeup and scheduling overhead. Add polling mode support where the fastRPC driver polls a shared memory location for completion after sending the invocation, avoiding CPU wakeup and scheduling latency and reducing fastRPC overhead. If the poll times out, the call falls back to the normal interrupt/glink-based completion path. Poll mode is only applied to dynamic modules running in a user PD (handle > FASTRPC_MAX_STATIC_HANDLE), since static/root-PD handles are not expected to benefit from, or require, this optimization. Support is advertised per SoC via fastrpc_soc_data, with a closed exception list for older platforms whose DSP firmware is known to support polling but which otherwise use the default soc_data. Poll mode can be enabled by userspace via the FASTRPC_IOCTL_SET_OPTION ioctl with the FASTRPC_POLL_MODE request id. Since context IDs (ctxid) are allocated from a fixed-size, per-channel cyclic IDR shared by all processes on a DSP, a context ID can be recycled for a new request soon after it is freed. In poll mode the context can be considered complete (and released) as soon as the poll memory is updated, while the corresponding glink COMPLETE response from the DSP may still be in flight. If that response arrives after the ctxid has been reused, it would otherwise match the new context and incorrectly signal completion for it while the DSP may still be operating on the new context's buffers. To prevent this, embed a monotonically increasing per-channel sequence number in the unused upper bits of the ctxid/message context and validate it in the rpmsg callback, dropping any response whose sequence number does not match the current owner of that ctxid slot.

Ekansh Gupta committed 3 months ago

Commit 96e62cc1 parent 62373b72 files · +215 −8Browse files

Unified · Split

2 files
  1. drivers/misc/fastrpc.c +186 −8
  2. include/uapi/misc/fastrpc.h +29 −0
drivers/misc/fastrpc.c+186−8
@@ -25,6 +25,8 @@ · 24 unmodified lines
2525 #include <linux/bitfield.h>
2626 #include <linux/bits.h>
2727 #include <linux/bitops.h>
28+#include <linux/compiler.h>
29+#include <linux/iopoll.h>
2830
2931 #define ADSP_DOMAIN_ID (0)
3032 #define MDSP_DOMAIN_ID (1)
@@ -39,7 +41,16 @@ · 8 unmodified lines
3941 #define FASTRPC_CTX_MAX (256)
4042 #define FASTRPC_INIT_HANDLE 1
4143 #define FASTRPC_DSP_UTILITIES_HANDLE 2
44+/*
45+ * Maximum handle value for static handles.
46+ * Static handles are pre-defined, fixed numeric values statically assigned
47+ * in the IDL file or FastRPC framework.
48+ */
49+#define FASTRPC_MAX_STATIC_HANDLE (20)
4250 #define FASTRPC_CTXID_MASK GENMASK(15, 8)
51+/* Sequence number occupies bits 63:16 of the ctxid / message context */
52+#define FASTRPC_CTXID_SEQ_SHIFT 16
53+#define FASTRPC_CTXID_SEQ_MASK GENMASK_ULL(63, 16)
4354 #define INIT_FILELEN_MAX (2 * 1024 * 1024)
4455 #define INIT_FILE_NAMELEN_MAX (128)
4556 #define FASTRPC_DEVICE_NAME "fastrpc"
@@ -107,6 +118,12 @@ · 61 unmodified lines
107118
108119 #define miscdev_to_fdevice(d) container_of(d, struct fastrpc_device, miscdev)
109120
121+/* Poll response number from remote processor for call completion */
122+#define FASTRPC_POLL_RESPONSE (0xdecaf)
123+
124+/* Polling mode timeout limit */
125+#define FASTRPC_POLL_MAX_TIMEOUT_US (10000)
126+
110127 struct fastrpc_phy_page {
111128 dma_addr_t addr; /* dma address */
112129 u64 size; /* size of contiguous region */
@@ -237,8 +254,14 @@ struct fastrpc_invoke_ctx { · 124 unmodified lines
237254 u32 sc;
238255 u64 *fdlist;
239256 u32 *crc;
257+ /* Poll memory that DSP updates */
258+ u32 *poll_addr;
240259 u64 ctxid;
241260 u64 msg_sz;
261+ /* work done status flag */
262+ bool is_work_done;
263+ /* process updates poll memory instead of glink response */
264+ bool is_polled;
242265 struct kref refcount;
243266 struct list_head node; /* list of ctxs */
244267 struct completion work;
@@ -264,6 +287,7 @@ struct fastrpc_soc_data { · 19 unmodified lines
264287 u32 sid_pos;
265288 u32 dma_addr_bits_cdsp;
266289 u32 dma_addr_bits_default;
290+ bool poll_mode_supported;
267291 };
268292
269293 struct fastrpc_channel_ctx {
@@ -286,6 +310,9 @@ struct fastrpc_channel_ctx { · 16 unmodified lines
286310 struct list_head invoke_interrupted_mmaps;
287311 bool secure;
288312 bool unsigned_support;
313+ bool poll_mode_supported;
314+ /* Per-channel sequence counter; incremented on every context allocation */
315+ atomic_t ctx_seq;
289316 u64 dma_mask;
290317 const struct fastrpc_soc_data *soc_data;
291318 };
@@ -309,6 +336,8 @@ struct fastrpc_user { · 17 unmodified lines
309336 int client_id;
310337 int pd;
311338 bool is_secure_dev;
339+ /* Flags poll mode state */
340+ bool poll_mode;
312341 /* Lock for lists */
313342 spinlock_t lock;
314343 /* lock for allocations */
@@ -706,7 +735,9 @@ static struct fastrpc_invoke_ctx *fastrpc_context_alloc( · 391 unmodified lines
706735 spin_unlock_irqrestore(&cctx->lock, flags);
707736 goto err_idr;
708737 }
709− ctx->ctxid = FIELD_PREP(FASTRPC_CTXID_MASK, ret);
738+ ctx->ctxid = FIELD_PREP(FASTRPC_CTXID_MASK, ret) |
739+ FIELD_PREP(FASTRPC_CTXID_SEQ_MASK,
740+ (u64)atomic_inc_return(&cctx->ctx_seq));
710741 spin_unlock_irqrestore(&cctx->lock, flags);
711742
712743 kref_init(&ctx->refcount);
@@ -971,7 +1002,8 @@ static int fastrpc_get_meta_size(struct fastrpc_invoke_ctx *ctx) · 258 unmodified lines
9711002 sizeof(struct fastrpc_invoke_buf) +
9721003 sizeof(struct fastrpc_phy_page)) * ctx->nscalars +
9731004 sizeof(u64) * FASTRPC_MAX_FDLIST +
974− sizeof(u32) * FASTRPC_MAX_CRCLIST;
1005+ sizeof(u32) * FASTRPC_MAX_CRCLIST +
1006+ sizeof(u32);
9751007
9761008 return size;
9771009 }
@@ -1067,6 +1099,9 @@ static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx) · 89 unmodified lines
10671099 list = fastrpc_invoke_buf_start(rpra, ctx->nscalars);
10681100 pages = fastrpc_phy_page_start(list, ctx->nscalars);
10691101 ctx->fdlist = (u64 *)(pages + ctx->nscalars);
1102+ ctx->poll_addr = (u32 *)((uintptr_t)ctx->fdlist + sizeof(u64) * FASTRPC_MAX_FDLIST +
1103+ sizeof(u32) * FASTRPC_MAX_CRCLIST);
1104+
10701105 args = (uintptr_t)ctx->buf->virt + metalen;
10711106 rlen = pkt_size - metalen;
10721107 ctx->rpra = rpra;
@@ -1236,6 +1271,71 @@ static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx, · 163 unmodified lines
12361271
12371272 }
12381273
1274+static u32 fastrpc_read_poll_addr(struct fastrpc_invoke_ctx *ctx)
1275+{
1276+ dma_rmb();
1277+ return READ_ONCE(*ctx->poll_addr);
1278+}
1279+
1280+static int poll_for_remote_response(struct fastrpc_invoke_ctx *ctx)
1281+{
1282+ u32 val;
1283+ int ret;
1284+
1285+ /*
1286+ * Poll until DSP writes FASTRPC_POLL_RESPONSE into *ctx->poll_addr
1287+ * or until another path marks the work done.
1288+ */
1289+ ret = read_poll_timeout_atomic(fastrpc_read_poll_addr, val,
1290+ (val == FASTRPC_POLL_RESPONSE) || ctx->is_work_done, 1,
1291+ FASTRPC_POLL_MAX_TIMEOUT_US, false, ctx);
1292+
1293+ if (!ret && val == FASTRPC_POLL_RESPONSE) {
1294+ /*
1295+ * DSP writes FASTRPC_POLL_RESPONSE to signal successful
1296+ * completion via the poll path.
1297+ */
1298+ ctx->is_work_done = true;
1299+ ctx->retval = 0;
1300+ }
1301+
1302+ if (ret == -ETIMEDOUT)
1303+ ret = -EIO;
1304+
1305+ return ret;
1306+}
1307+
1308+static inline int fastrpc_wait_for_response(struct fastrpc_invoke_ctx *ctx,
1309+ u32 kernel)
1310+{
1311+ int err = 0;
1312+
1313+ if (kernel) {
1314+ if (!wait_for_completion_timeout(&ctx->work, 10 * HZ))
1315+ err = -ETIMEDOUT;
1316+ } else {
1317+ err = wait_for_completion_interruptible(&ctx->work);
1318+ }
1319+
1320+ return err;
1321+}
1322+
1323+static int fastrpc_wait_for_completion(struct fastrpc_invoke_ctx *ctx,
1324+ u32 kernel)
1325+{
1326+ int err;
1327+
1328+ if (ctx->is_polled) {
1329+ err = poll_for_remote_response(ctx);
1330+ if (!err)
1331+ return 0;
1332+ /* If polling timed out or failed, move to normal response mode */
1333+ ctx->is_polled = false;
1334+ }
1335+
1336+ return fastrpc_wait_for_response(ctx, kernel);
1337+}
1338+
12391339 static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel,
12401340 u32 handle, u32 sc,
12411341 struct fastrpc_invoke_args *args)
@@ -1271,13 +1371,14 @@ static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel, · 29 unmodified lines
12711371 if (err)
12721372 goto bail;
12731373
1274− if (kernel) {
1275− if (!wait_for_completion_timeout(&ctx->work, 10 * HZ))
1276− err = -ETIMEDOUT;
1277− } else {
1278− err = wait_for_completion_interruptible(&ctx->work);
1279− }
1374+ /*
1375+ * Set message context as polled if the call is for a user PD
1376+ * dynamic module and user has enabled poll mode.
1377+ */
1378+ if (handle > FASTRPC_MAX_STATIC_HANDLE && fl->pd == USER_PD && fl->poll_mode)
1379+ ctx->is_polled = true;
12801380
1381+ err = fastrpc_wait_for_completion(ctx, kernel);
12811382 if (err)
12821383 goto bail;
12831384
@@ -1841,6 +1942,35 @@ done: · 557 unmodified lines
18411942 return 0;
18421943 }
18431944
1945+static int fastrpc_set_option(struct fastrpc_user *fl, char __user *argp)
1946+{
1947+ struct fastrpc_ioctl_set_option opt = {0};
1948+ int i;
1949+
1950+ if (copy_from_user(&opt, argp, sizeof(opt)))
1951+ return -EFAULT;
1952+
1953+ for (i = 0; i < ARRAY_SIZE(opt.reserved); i++) {
1954+ if (opt.reserved[i] != 0)
1955+ return -EINVAL;
1956+ }
1957+
1958+ if (opt.request_id != FASTRPC_POLL_MODE)
1959+ return -EINVAL;
1960+
1961+ if (!fl->cctx->poll_mode_supported)
1962+ return -EOPNOTSUPP;
1963+
1964+ if (opt.value == FASTRPC_POLL_MODE_ENABLE)
1965+ fl->poll_mode = true;
1966+ else if (opt.value == FASTRPC_POLL_MODE_DISABLE)
1967+ fl->poll_mode = false;
1968+ else
1969+ return -EINVAL;
1970+
1971+ return 0;
1972+}
1973+
18441974 static int fastrpc_get_dsp_info(struct fastrpc_user *fl, char __user *argp)
18451975 {
18461976 struct fastrpc_ioctl_capability cap = {0};
@@ -2196,6 +2326,9 @@ static long fastrpc_device_ioctl(struct file *file, unsigned int cmd, · 349 unmodified lines
21962326 case FASTRPC_IOCTL_MEM_UNMAP:
21972327 err = fastrpc_req_mem_unmap(fl, argp);
21982328 break;
2329+ case FASTRPC_IOCTL_SET_OPTION:
2330+ err = fastrpc_set_option(fl, argp);
2331+ break;
21992332 case FASTRPC_IOCTL_GET_DSP_INFO:
22002333 err = fastrpc_get_dsp_info(fl, argp);
22012334 break;
@@ -2352,6 +2485,7 @@ static const struct fastrpc_soc_data kaanapali_soc_data = { · 150 unmodified lines
23522485 .sid_pos = 56,
23532486 .dma_addr_bits_cdsp = 34,
23542487 .dma_addr_bits_default = 32,
2488+ .poll_mode_supported = true,
23552489 };
23562490
23572491 static const struct fastrpc_soc_data default_soc_data = {
@@ -2360,6 +2494,29 @@ static const struct fastrpc_soc_data default_soc_data = { · 2 unmodified lines
23602494 .dma_addr_bits_default = 32,
23612495 };
23622496
2497+/*
2498+ * Exception list for older platforms that use default_soc_data but whose
2499+ * DSP firmware supports FastRPC polling mode.
2500+ *
2501+ * NOTE: This list is intentionally closed.
2502+ * Do NOT add new platforms here. New SoCs must advertise polling mode
2503+ * support via their soc_data.
2504+ */
2505+
2506+static const struct of_device_id fastrpc_poll_supported_machines[] __maybe_unused = {
2507+ { .compatible = "qcom,milos" },
2508+ { .compatible = "qcom,qcs8300" },
2509+ { .compatible = "qcom,sa8775p" },
2510+ { .compatible = "qcom,sar2130p" },
2511+ { .compatible = "qcom,sm8450" },
2512+ { .compatible = "qcom,sm8550" },
2513+ { .compatible = "qcom,sm8650" },
2514+ { .compatible = "qcom,sm8750" },
2515+ { .compatible = "qcom,x1e80100" },
2516+ { .compatible = "qcom,x1p42100" },
2517+ {},
2518+};
2519+
23632520 static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev)
23642521 {
23652522 struct device *rdev = &rpdev->dev;
@@ -2426,6 +2583,8 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev) · 60 unmodified lines
24262583 secure_dsp = !(of_property_read_bool(rdev->of_node, "qcom,non-secure-domain"));
24272584 data->secure = secure_dsp;
24282585 data->soc_data = soc_data;
2586+ data->poll_mode_supported = soc_data->poll_mode_supported ||
2587+ of_machine_get_match(fastrpc_poll_supported_machines);
24292588
24302589 switch (domain_id) {
24312590 case ADSP_DOMAIN_ID:
@@ -2455,6 +2614,7 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev) · 23 unmodified lines
24552614 }
24562615
24572616 kref_init(&data->refcount);
2617+ atomic_set(&data->ctx_seq, 0);
24582618
24592619 rdev->dma_mask = &data->dma_mask;
24602620 dma_set_mask_and_coherent(rdev, DMA_BIT_MASK(32));
@@ -2552,7 +2712,25 @@ static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data, · 91 unmodified lines
25522712 return -ENOENT;
25532713 }
25542714
2715+ /*
2716+ * Validate the sequence number embedded in the upper bits of the
2717+ * context ID. Under high concurrency the IDR slot can be recycled
2718+ * for a new request before a late (or duplicate) glink COMPLETE
2719+ * response for the previous request arrives. Without this check the
2720+ * driver would call complete() on the wrong context, waking a thread
2721+ * whose buffers are still being accessed by the DSP.
2722+ */
2723+ if (FIELD_GET(FASTRPC_CTXID_SEQ_MASK, rsp->ctx) !=
2724+ FIELD_GET(FASTRPC_CTXID_SEQ_MASK, ctx->ctxid)) {
2725+ dev_dbg(&rpdev->dev,
2726+ "Stale glink response ctx 0x%llx (expected seq 0x%llx), dropping\n",
2727+ rsp->ctx,
2728+ FIELD_GET(FASTRPC_CTXID_SEQ_MASK, ctx->ctxid));
2729+ return 0;
2730+ }
2731+
25552732 ctx->retval = rsp->retval;
2733+ ctx->is_work_done = true;
25562734 complete(&ctx->work);
25572735
25582736 /*
include/uapi/misc/fastrpc.h+29−0
@@ -16,6 +16,7 @@ · 15 unmodified lines
1616 #define FASTRPC_IOCTL_INIT_CREATE_STATIC _IOWR('R', 9, struct fastrpc_init_create_static)
1717 #define FASTRPC_IOCTL_MEM_MAP _IOWR('R', 10, struct fastrpc_mem_map)
1818 #define FASTRPC_IOCTL_MEM_UNMAP _IOWR('R', 11, struct fastrpc_mem_unmap)
19+#define FASTRPC_IOCTL_SET_OPTION _IOWR('R', 12, struct fastrpc_ioctl_set_option)
1920 #define FASTRPC_IOCTL_GET_DSP_INFO _IOWR('R', 13, struct fastrpc_ioctl_capability)
2021
2122 /**
@@ -67,6 +68,28 @@ enum fastrpc_proc_attr { · 45 unmodified lines
6768 /* Fastrpc attribute for memory protection of buffers */
6869 #define FASTRPC_ATTR_SECUREMAP (1)
6970
71+/**
72+ * FASTRPC_POLL_MODE - Enable/disable poll mode for FastRPC invocations
73+ *
74+ * Poll mode is an optimization that allows the CPU to poll shared memory
75+ * for completion instead of waiting for an interrupt-based response.
76+ * This reduces latency for fast-completing operations.
77+ *
78+ * Restrictions:
79+ * - Only supported for USER_PD (User Protection Domain)
80+ * - Only applies to dynamic modules (handle > 20)
81+ * - Static modules always use interrupt-based completion
82+ *
83+ * Values:
84+ * - 0: Disable poll mode (use interrupt-based completion)
85+ * - 1: Enable poll mode (poll shared memory for completion)
86+ */
87+#define FASTRPC_POLL_MODE (1)
88+
89+/* Values for FASTRPC_POLL_MODE request */
90+#define FASTRPC_POLL_MODE_DISABLE 0
91+#define FASTRPC_POLL_MODE_ENABLE 1
92+
7093 struct fastrpc_invoke_args {
7194 __u64 ptr;
7295 __u64 length;
@@ -133,6 +156,12 @@ struct fastrpc_mem_unmap { · 60 unmodified lines
133156 __s32 reserved[5];
134157 };
135158
159+struct fastrpc_ioctl_set_option {
160+ __u32 request_id; /* Request type (e.g., FASTRPC_POLL_MODE) */
161+ __u32 value; /* Request-specific value */
162+ __s32 reserved[6];
163+};
164+
136165 struct fastrpc_ioctl_capability {
137166 __u32 unused; /* deprecated, ignored by the kernel */
138167 __u32 attribute_id;