net/fungible: Add service module for Fungible drivers

Fungible cards have a number of different PCI functions and thus different drivers, all of which use a common method to initialize and interact with the device. This commit adds a library module that collects these common mechanisms. They mainly deal with device initialization, setting up and destroying queues, and operating an admin queue. A subset of the FW interface is also included here.

Dimitris Michailidis committed 4 years ago

Commit e1ffcc61 parent e8eb9e36 files · +2976 −0Browse files

Unified · Split

6 files

drivers/net/ethernet/fungible/funcore/Makefile +5 −0
drivers/net/ethernet/fungible/funcore/fun_dev.c +843 −0
drivers/net/ethernet/fungible/funcore/fun_dev.h +150 −0
drivers/net/ethernet/fungible/funcore/fun_hci.h +1202 −0
drivers/net/ethernet/fungible/funcore/fun_queue.c +601 −0
drivers/net/ethernet/fungible/funcore/fun_queue.h +175 −0

drivers/net/ethernet/fungible/funcore/Makefile+5−0

@@ -0,0 +1,5 @@
	1	+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
	2	+
	3	+obj-$(CONFIG_FUN_CORE) += funcore.o
	4	+
	5	+funcore-y := fun_dev.o fun_queue.o

drivers/net/ethernet/fungible/funcore/fun_dev.c+843−0

@@ -0,0 +1,843 @@
	1	+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
	2	+
	3	+#include <linux/aer.h>
	4	+#include <linux/bitmap.h>
	5	+#include <linux/delay.h>
	6	+#include <linux/interrupt.h>
	7	+#include <linux/io.h>
	8	+#include <linux/io-64-nonatomic-lo-hi.h>
	9	+#include <linux/mm.h>
	10	+#include <linux/module.h>
	11	+#include <linux/nvme.h>
	12	+#include <linux/pci.h>
	13	+#include <linux/wait.h>
	14	+#include <linux/sched/signal.h>
	15	+
	16	+#include "fun_queue.h"
	17	+#include "fun_dev.h"
	18	+
	19	+#define FUN_ADMIN_CMD_TO_MS 3000
	20	+
	21	+enum {
	22	+ AQA_ASQS_SHIFT = 0,
	23	+ AQA_ACQS_SHIFT = 16,
	24	+ AQA_MIN_QUEUE_SIZE = 2,
	25	+ AQA_MAX_QUEUE_SIZE = 4096
	26	+};
	27	+
	28	+/* context for admin commands */
	29	+struct fun_cmd_ctx {
	30	+ fun_admin_callback_t cb; /* callback to invoke on completion */
	31	+ void cb_data; / user data provided to callback */
	32	+ int cpu; /* CPU where the cmd's tag was allocated */
	33	+};
	34	+
	35	+/* Context for synchronous admin commands. */
	36	+struct fun_sync_cmd_ctx {
	37	+ struct completion compl;
	38	+ u8 rsp_buf; / caller provided response buffer */
	39	+ unsigned int rsp_len; /* response buffer size */
	40	+ u8 rsp_status; /* command response status */
	41	+};
	42	+
	43	+/* Wait for the CSTS.RDY bit to match @enabled. */
	44	+static int fun_wait_ready(struct fun_dev *fdev, bool enabled)
	45	+{
	46	+ unsigned int cap_to = NVME_CAP_TIMEOUT(fdev->cap_reg);
	47	+ u32 bit = enabled ? NVME_CSTS_RDY : 0;
	48	+ unsigned long deadline;
	49	+
	50	+ deadline = ((cap_to + 1) * HZ / 2) + jiffies; /* CAP.TO is in 500ms */
	51	+
	52	+ for (;;) {
	53	+ u32 csts = readl(fdev->bar + NVME_REG_CSTS);
	54	+
	55	+ if (csts == ~0) {
	56	+ dev_err(fdev->dev, "CSTS register read %#x\n", csts);
	57	+ return -EIO;
	58	+ }
	59	+
	60	+ if ((csts & NVME_CSTS_RDY) == bit)
	61	+ return 0;
	62	+
	63	+ if (time_is_before_jiffies(deadline))
	64	+ break;
	65	+
	66	+ msleep(100);
	67	+ }
	68	+
	69	+ dev_err(fdev->dev,
	70	+ "Timed out waiting for device to indicate RDY %u; aborting %s\n",
	71	+ enabled, enabled ? "initialization" : "reset");
	72	+ return -ETIMEDOUT;
	73	+}
	74	+
	75	+/* Check CSTS and return an error if it is unreadable or has unexpected
	76	+ * RDY value.
	77	+ */
	78	+static int fun_check_csts_rdy(struct fun_dev *fdev, unsigned int expected_rdy)
	79	+{
	80	+ u32 csts = readl(fdev->bar + NVME_REG_CSTS);
	81	+ u32 actual_rdy = csts & NVME_CSTS_RDY;
	82	+
	83	+ if (csts == ~0) {
	84	+ dev_err(fdev->dev, "CSTS register read %#x\n", csts);
	85	+ return -EIO;
	86	+ }
	87	+ if (actual_rdy != expected_rdy) {
	88	+ dev_err(fdev->dev, "Unexpected CSTS RDY %u\n", actual_rdy);
	89	+ return -EINVAL;
	90	+ }
	91	+ return 0;
	92	+}
	93	+
	94	+/* Check that CSTS RDY has the expected value. Then write a new value to the CC
	95	+ * register and wait for CSTS RDY to match the new CC ENABLE state.
	96	+ */
	97	+static int fun_update_cc_enable(struct fun_dev *fdev, unsigned int initial_rdy)
	98	+{
	99	+ int rc = fun_check_csts_rdy(fdev, initial_rdy);
	100	+
	101	+ if (rc)
	102	+ return rc;
	103	+ writel(fdev->cc_reg, fdev->bar + NVME_REG_CC);
	104	+ return fun_wait_ready(fdev, !!(fdev->cc_reg & NVME_CC_ENABLE));
	105	+}
	106	+
	107	+static int fun_disable_ctrl(struct fun_dev *fdev)
	108	+{
	109	+ fdev->cc_reg &= ~(NVME_CC_SHN_MASK \| NVME_CC_ENABLE);
	110	+ return fun_update_cc_enable(fdev, 1);
	111	+}
	112	+
	113	+static int fun_enable_ctrl(struct fun_dev *fdev, u32 admin_cqesz_log2,
	114	+ u32 admin_sqesz_log2)
	115	+{
	116	+ fdev->cc_reg = (admin_cqesz_log2 << NVME_CC_IOCQES_SHIFT) \|
	117	+ (admin_sqesz_log2 << NVME_CC_IOSQES_SHIFT) \|
	118	+ ((PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT) \|
	119	+ NVME_CC_ENABLE;
	120	+
	121	+ return fun_update_cc_enable(fdev, 0);
	122	+}
	123	+
	124	+static int fun_map_bars(struct fun_dev fdev, const char name)
	125	+{
	126	+ struct pci_dev *pdev = to_pci_dev(fdev->dev);
	127	+ int err;
	128	+
	129	+ err = pci_request_mem_regions(pdev, name);
	130	+ if (err) {
	131	+ dev_err(&pdev->dev,
	132	+ "Couldn't get PCI memory resources, err %d\n", err);
	133	+ return err;
	134	+ }
	135	+
	136	+ fdev->bar = pci_ioremap_bar(pdev, 0);
	137	+ if (!fdev->bar) {
	138	+ dev_err(&pdev->dev, "Couldn't map BAR 0\n");
	139	+ pci_release_mem_regions(pdev);
	140	+ return -ENOMEM;
	141	+ }
	142	+
	143	+ return 0;
	144	+}
	145	+
	146	+static void fun_unmap_bars(struct fun_dev *fdev)
	147	+{
	148	+ struct pci_dev *pdev = to_pci_dev(fdev->dev);
	149	+
	150	+ if (fdev->bar) {
	151	+ iounmap(fdev->bar);
	152	+ fdev->bar = NULL;
	153	+ pci_release_mem_regions(pdev);
	154	+ }
	155	+}
	156	+
	157	+static int fun_set_dma_masks(struct device *dev)
	158	+{
	159	+ int err;
	160	+
	161	+ err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
	162	+ if (err)
	163	+ dev_err(dev, "DMA mask configuration failed, err %d\n", err);
	164	+ return err;
	165	+}
	166	+
	167	+static irqreturn_t fun_admin_irq(int irq, void *data)
	168	+{
	169	+ struct fun_queue *funq = data;
	170	+
	171	+ return fun_process_cq(funq, 0) ? IRQ_HANDLED : IRQ_NONE;
	172	+}
	173	+
	174	+static void fun_complete_admin_cmd(struct fun_queue funq, void data,
	175	+ void entry, const struct fun_cqe_info info)
	176	+{
	177	+ const struct fun_admin_rsp_common *rsp_common = entry;
	178	+ struct fun_dev *fdev = funq->fdev;
	179	+ struct fun_cmd_ctx *cmd_ctx;
	180	+ int cpu;
	181	+ u16 cid;
	182	+
	183	+ if (info->sqhd == cpu_to_be16(0xffff)) {
	184	+ dev_dbg(fdev->dev, "adminq event");
	185	+ if (fdev->adminq_cb)
	186	+ fdev->adminq_cb(fdev, entry);
	187	+ return;
	188	+ }
	189	+
	190	+ cid = be16_to_cpu(rsp_common->cid);
	191	+ dev_dbg(fdev->dev, "admin CQE cid %u, op %u, ret %u\n", cid,
	192	+ rsp_common->op, rsp_common->ret);
	193	+
	194	+ cmd_ctx = &fdev->cmd_ctx[cid];
	195	+ if (cmd_ctx->cpu < 0) {
	196	+ dev_err(fdev->dev,
	197	+ "admin CQE with CID=%u, op=%u does not match a pending command\n",
	198	+ cid, rsp_common->op);
	199	+ return;
	200	+ }
	201	+
	202	+ if (cmd_ctx->cb)
	203	+ cmd_ctx->cb(fdev, entry, xchg(&cmd_ctx->cb_data, NULL));
	204	+
	205	+ cpu = cmd_ctx->cpu;
	206	+ cmd_ctx->cpu = -1;
	207	+ sbitmap_queue_clear(&fdev->admin_sbq, cid, cpu);
	208	+}
	209	+
	210	+static int fun_init_cmd_ctx(struct fun_dev *fdev, unsigned int ntags)
	211	+{
	212	+ unsigned int i;
	213	+
	214	+ fdev->cmd_ctx = kvcalloc(ntags, sizeof(*fdev->cmd_ctx), GFP_KERNEL);
	215	+ if (!fdev->cmd_ctx)
	216	+ return -ENOMEM;
	217	+
	218	+ for (i = 0; i < ntags; i++)
	219	+ fdev->cmd_ctx[i].cpu = -1;
	220	+
	221	+ return 0;
	222	+}
	223	+
	224	+/* Allocate and enable an admin queue and assign it the first IRQ vector. */
	225	+static int fun_enable_admin_queue(struct fun_dev *fdev,
	226	+ const struct fun_dev_params *areq)
	227	+{
	228	+ struct fun_queue_alloc_req qreq = {
	229	+ .cqe_size_log2 = areq->cqe_size_log2,
	230	+ .sqe_size_log2 = areq->sqe_size_log2,
	231	+ .cq_depth = areq->cq_depth,
	232	+ .sq_depth = areq->sq_depth,
	233	+ .rq_depth = areq->rq_depth,
	234	+ };
	235	+ unsigned int ntags = areq->sq_depth - 1;
	236	+ struct fun_queue *funq;
	237	+ int rc;
	238	+
	239	+ if (fdev->admin_q)
	240	+ return -EEXIST;
	241	+
	242	+ if (areq->sq_depth < AQA_MIN_QUEUE_SIZE \|\|
	243	+ areq->sq_depth > AQA_MAX_QUEUE_SIZE \|\|
	244	+ areq->cq_depth < AQA_MIN_QUEUE_SIZE \|\|
	245	+ areq->cq_depth > AQA_MAX_QUEUE_SIZE)
	246	+ return -EINVAL;
	247	+
	248	+ fdev->admin_q = fun_alloc_queue(fdev, 0, &qreq);
	249	+ if (!fdev->admin_q)
	250	+ return -ENOMEM;
	251	+
	252	+ rc = fun_init_cmd_ctx(fdev, ntags);
	253	+ if (rc)
	254	+ goto free_q;
	255	+
	256	+ rc = sbitmap_queue_init_node(&fdev->admin_sbq, ntags, -1, false,
	257	+ GFP_KERNEL, dev_to_node(fdev->dev));
	258	+ if (rc)
	259	+ goto free_cmd_ctx;
	260	+
	261	+ funq = fdev->admin_q;
	262	+ funq->cq_vector = 0;
	263	+ rc = fun_request_irq(funq, dev_name(fdev->dev), fun_admin_irq, funq);
	264	+ if (rc)
	265	+ goto free_sbq;
	266	+
	267	+ fun_set_cq_callback(funq, fun_complete_admin_cmd, NULL);
	268	+ fdev->adminq_cb = areq->event_cb;
	269	+
	270	+ writel((funq->sq_depth - 1) << AQA_ASQS_SHIFT \|
	271	+ (funq->cq_depth - 1) << AQA_ACQS_SHIFT,
	272	+ fdev->bar + NVME_REG_AQA);
	273	+
	274	+ writeq(funq->sq_dma_addr, fdev->bar + NVME_REG_ASQ);
	275	+ writeq(funq->cq_dma_addr, fdev->bar + NVME_REG_ACQ);
	276	+
	277	+ rc = fun_enable_ctrl(fdev, areq->cqe_size_log2, areq->sqe_size_log2);
	278	+ if (rc)
	279	+ goto free_irq;
	280	+
	281	+ if (areq->rq_depth) {
	282	+ rc = fun_create_rq(funq);
	283	+ if (rc)
	284	+ goto disable_ctrl;
	285	+
	286	+ funq_rq_post(funq);
	287	+ }
	288	+
	289	+ return 0;
	290	+
	291	+disable_ctrl:
	292	+ fun_disable_ctrl(fdev);
	293	+free_irq:
	294	+ fun_free_irq(funq);
	295	+free_sbq:
	296	+ sbitmap_queue_free(&fdev->admin_sbq);
	297	+free_cmd_ctx:
	298	+ kvfree(fdev->cmd_ctx);
	299	+ fdev->cmd_ctx = NULL;
	300	+free_q:
	301	+ fun_free_queue(fdev->admin_q);
	302	+ fdev->admin_q = NULL;
	303	+ return rc;
	304	+}
	305	+
	306	+static void fun_disable_admin_queue(struct fun_dev *fdev)
	307	+{
	308	+ struct fun_queue *admq = fdev->admin_q;
	309	+
	310	+ if (!admq)
	311	+ return;
	312	+
	313	+ fun_disable_ctrl(fdev);
	314	+
	315	+ fun_free_irq(admq);
	316	+ __fun_process_cq(admq, 0);
	317	+
	318	+ sbitmap_queue_free(&fdev->admin_sbq);
	319	+
	320	+ kvfree(fdev->cmd_ctx);
	321	+ fdev->cmd_ctx = NULL;
	322	+
	323	+ fun_free_queue(admq);
	324	+ fdev->admin_q = NULL;
	325	+}
	326	+
	327	+/* Return %true if the admin queue has stopped servicing commands as can be
	328	+ * detected through registers. This isn't exhaustive and may provide false
	329	+ * negatives.
	330	+ */
	331	+static bool fun_adminq_stopped(struct fun_dev *fdev)
	332	+{
	333	+ u32 csts = readl(fdev->bar + NVME_REG_CSTS);
	334	+
	335	+ return (csts & (NVME_CSTS_CFS \| NVME_CSTS_RDY)) != NVME_CSTS_RDY;
	336	+}
	337	+
	338	+static int fun_wait_for_tag(struct fun_dev fdev, int cpup)
	339	+{
	340	+ struct sbitmap_queue *sbq = &fdev->admin_sbq;
	341	+ struct sbq_wait_state *ws = &sbq->ws[0];
	342	+ DEFINE_SBQ_WAIT(wait);
	343	+ int tag;
	344	+
	345	+ for (;;) {
	346	+ sbitmap_prepare_to_wait(sbq, ws, &wait, TASK_UNINTERRUPTIBLE);
	347	+ if (fdev->suppress_cmds) {
	348	+ tag = -ESHUTDOWN;
	349	+ break;
	350	+ }
	351	+ tag = sbitmap_queue_get(sbq, cpup);
	352	+ if (tag >= 0)
	353	+ break;
	354	+ schedule();
	355	+ }
	356	+
	357	+ sbitmap_finish_wait(sbq, ws, &wait);
	358	+ return tag;
	359	+}
	360	+
	361	+/* Submit an asynchronous admin command. Caller is responsible for implementing
	362	+ * any waiting or timeout. Upon command completion the callback @cb is called.
	363	+ */
	364	+int fun_submit_admin_cmd(struct fun_dev fdev, struct fun_admin_req_common cmd,
	365	+ fun_admin_callback_t cb, void *cb_data, bool wait_ok)
	366	+{
	367	+ struct fun_queue *funq = fdev->admin_q;
	368	+ unsigned int cmdsize = cmd->len8 * 8;
	369	+ struct fun_cmd_ctx *cmd_ctx;
	370	+ int tag, cpu, rc = 0;
	371	+
	372	+ if (WARN_ON(cmdsize > (1 << funq->sqe_size_log2)))
	373	+ return -EMSGSIZE;
	374	+
	375	+ tag = sbitmap_queue_get(&fdev->admin_sbq, &cpu);
	376	+ if (tag < 0) {
	377	+ if (!wait_ok)
	378	+ return -EAGAIN;
	379	+ tag = fun_wait_for_tag(fdev, &cpu);
	380	+ if (tag < 0)
	381	+ return tag;
	382	+ }
	383	+
	384	+ cmd->cid = cpu_to_be16(tag);
	385	+
	386	+ cmd_ctx = &fdev->cmd_ctx[tag];
	387	+ cmd_ctx->cb = cb;
	388	+ cmd_ctx->cb_data = cb_data;
	389	+
	390	+ spin_lock(&funq->sq_lock);
	391	+
	392	+ if (unlikely(fdev->suppress_cmds)) {
	393	+ rc = -ESHUTDOWN;
	394	+ sbitmap_queue_clear(&fdev->admin_sbq, tag, cpu);
	395	+ } else {
	396	+ cmd_ctx->cpu = cpu;
	397	+ memcpy(fun_sqe_at(funq, funq->sq_tail), cmd, cmdsize);
	398	+
	399	+ dev_dbg(fdev->dev, "admin cmd @ %u: %8ph\n", funq->sq_tail,
	400	+ cmd);
	401	+
	402	+ if (++funq->sq_tail == funq->sq_depth)
	403	+ funq->sq_tail = 0;
	404	+ writel(funq->sq_tail, funq->sq_db);
	405	+ }
	406	+ spin_unlock(&funq->sq_lock);
	407	+ return rc;
	408	+}
	409	+
	410	+/* Abandon a pending admin command by clearing the issuer's callback data.
	411	+ * Failure indicates that the command either has already completed or its
	412	+ * completion is racing with this call.
	413	+ */
	414	+static bool fun_abandon_admin_cmd(struct fun_dev *fd,
	415	+ const struct fun_admin_req_common *cmd,
	416	+ void *cb_data)
	417	+{
	418	+ u16 cid = be16_to_cpu(cmd->cid);
	419	+ struct fun_cmd_ctx *cmd_ctx = &fd->cmd_ctx[cid];
	420	+
	421	+ return cmpxchg(&cmd_ctx->cb_data, cb_data, NULL) == cb_data;
	422	+}
	423	+
	424	+/* Stop submission of new admin commands and wake up any processes waiting for
	425	+ * tags. Already submitted commands are left to complete or time out.
	426	+ */
	427	+static void fun_admin_stop(struct fun_dev *fdev)
	428	+{
	429	+ spin_lock(&fdev->admin_q->sq_lock);
	430	+ fdev->suppress_cmds = true;
	431	+ spin_unlock(&fdev->admin_q->sq_lock);
	432	+ sbitmap_queue_wake_all(&fdev->admin_sbq);
	433	+}
	434	+
	435	+/* The callback for synchronous execution of admin commands. It copies the
	436	+ * command response to the caller's buffer and signals completion.
	437	+ */
	438	+static void fun_admin_cmd_sync_cb(struct fun_dev fd, void rsp, void *cb_data)
	439	+{
	440	+ const struct fun_admin_rsp_common *rsp_common = rsp;
	441	+ struct fun_sync_cmd_ctx *ctx = cb_data;
	442	+
	443	+ if (!ctx)
	444	+ return; /* command issuer timed out and left */
	445	+ if (ctx->rsp_buf) {
	446	+ unsigned int rsp_len = rsp_common->len8 * 8;
	447	+
	448	+ if (unlikely(rsp_len > ctx->rsp_len)) {
	449	+ dev_err(fd->dev,
	450	+ "response for op %u is %uB > response buffer %uB\n",
	451	+ rsp_common->op, rsp_len, ctx->rsp_len);
	452	+ rsp_len = ctx->rsp_len;
	453	+ }
	454	+ memcpy(ctx->rsp_buf, rsp, rsp_len);
	455	+ }
	456	+ ctx->rsp_status = rsp_common->ret;
	457	+ complete(&ctx->compl);
	458	+}
	459	+
	460	+/* Submit a synchronous admin command. */
	461	+int fun_submit_admin_sync_cmd(struct fun_dev *fdev,
	462	+ struct fun_admin_req_common cmd, void rsp,
	463	+ size_t rspsize, unsigned int timeout)
	464	+{
	465	+ struct fun_sync_cmd_ctx ctx = {
	466	+ .compl = COMPLETION_INITIALIZER_ONSTACK(ctx.compl),
	467	+ .rsp_buf = rsp,
	468	+ .rsp_len = rspsize,
	469	+ };
	470	+ unsigned int cmdlen = cmd->len8 * 8;
	471	+ unsigned long jiffies_left;
	472	+ int ret;
	473	+
	474	+ ret = fun_submit_admin_cmd(fdev, cmd, fun_admin_cmd_sync_cb, &ctx,
	475	+ true);
	476	+ if (ret)
	477	+ return ret;
	478	+
	479	+ if (!timeout)
	480	+ timeout = FUN_ADMIN_CMD_TO_MS;
	481	+
	482	+ jiffies_left = wait_for_completion_timeout(&ctx.compl,
	483	+ msecs_to_jiffies(timeout));
	484	+ if (!jiffies_left) {
	485	+ /* The command timed out. Attempt to cancel it so we can return.
	486	+ * But if the command is in the process of completing we'll
	487	+ * wait for it.
	488	+ */
	489	+ if (fun_abandon_admin_cmd(fdev, cmd, &ctx)) {
	490	+ dev_err(fdev->dev, "admin command timed out: %*ph\n",
	491	+ cmdlen, cmd);
	492	+ fun_admin_stop(fdev);
	493	+ /* see if the timeout was due to a queue failure */
	494	+ if (fun_adminq_stopped(fdev))
	495	+ dev_err(fdev->dev,
	496	+ "device does not accept admin commands\n");
	497	+
	498	+ return -ETIMEDOUT;
	499	+ }
	500	+ wait_for_completion(&ctx.compl);
	501	+ }
	502	+
	503	+ if (ctx.rsp_status) {
	504	+ dev_err(fdev->dev, "admin command failed, err %d: %*ph\n",
	505	+ ctx.rsp_status, cmdlen, cmd);
	506	+ }
	507	+
	508	+ return -ctx.rsp_status;
	509	+}
	510	+EXPORT_SYMBOL_GPL(fun_submit_admin_sync_cmd);
	511	+
	512	+/* Return the number of device resources of the requested type. */
	513	+int fun_get_res_count(struct fun_dev *fdev, enum fun_admin_op res)
	514	+{
	515	+ union {
	516	+ struct fun_admin_res_count_req req;
	517	+ struct fun_admin_res_count_rsp rsp;
	518	+ } cmd;
	519	+ int rc;
	520	+
	521	+ cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(res, sizeof(cmd.req));
	522	+ cmd.req.count = FUN_ADMIN_SIMPLE_SUBOP_INIT(FUN_ADMIN_SUBOP_RES_COUNT,
	523	+ 0, 0);
	524	+
	525	+ rc = fun_submit_admin_sync_cmd(fdev, &cmd.req.common, &cmd.rsp,
	526	+ sizeof(cmd), 0);
	527	+ return rc ? rc : be32_to_cpu(cmd.rsp.count.data);
	528	+}
	529	+EXPORT_SYMBOL_GPL(fun_get_res_count);
	530	+
	531	+/* Request that the instance of resource @res with the given id be deleted. */
	532	+int fun_res_destroy(struct fun_dev *fdev, enum fun_admin_op res,
	533	+ unsigned int flags, u32 id)
	534	+{
	535	+ struct fun_admin_generic_destroy_req req = {
	536	+ .common = FUN_ADMIN_REQ_COMMON_INIT2(res, sizeof(req)),
	537	+ .destroy = FUN_ADMIN_SIMPLE_SUBOP_INIT(FUN_ADMIN_SUBOP_DESTROY,
	538	+ flags, id)
	539	+ };
	540	+
	541	+ return fun_submit_admin_sync_cmd(fdev, &req.common, NULL, 0, 0);
	542	+}
	543	+EXPORT_SYMBOL_GPL(fun_res_destroy);
	544	+
	545	+/* Bind two entities of the given types and IDs. */
	546	+int fun_bind(struct fun_dev *fdev, enum fun_admin_bind_type type0,
	547	+ unsigned int id0, enum fun_admin_bind_type type1,
	548	+ unsigned int id1)
	549	+{
	550	+ struct {
	551	+ struct fun_admin_bind_req req;
	552	+ struct fun_admin_bind_entry entry[2];
	553	+ } cmd = {
	554	+ .req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_BIND,
	555	+ sizeof(cmd)),
	556	+ .entry[0] = FUN_ADMIN_BIND_ENTRY_INIT(type0, id0),
	557	+ .entry[1] = FUN_ADMIN_BIND_ENTRY_INIT(type1, id1),
	558	+ };
	559	+
	560	+ return fun_submit_admin_sync_cmd(fdev, &cmd.req.common, NULL, 0, 0);
	561	+}
	562	+EXPORT_SYMBOL_GPL(fun_bind);
	563	+
	564	+static int fun_get_dev_limits(struct fun_dev *fdev)
	565	+{
	566	+ struct pci_dev *pdev = to_pci_dev(fdev->dev);
	567	+ unsigned int cq_count, sq_count, num_dbs;
	568	+ int rc;
	569	+
	570	+ rc = fun_get_res_count(fdev, FUN_ADMIN_OP_EPCQ);
	571	+ if (rc < 0)
	572	+ return rc;
	573	+ cq_count = rc;
	574	+
	575	+ rc = fun_get_res_count(fdev, FUN_ADMIN_OP_EPSQ);
	576	+ if (rc < 0)
	577	+ return rc;
	578	+ sq_count = rc;
	579	+
	580	+ /* The admin queue consumes 1 CQ and at least 1 SQ. To be usable the
	581	+ * device must provide additional queues.
	582	+ */
	583	+ if (cq_count < 2 \|\| sq_count < 2 + !!fdev->admin_q->rq_depth)
	584	+ return -EINVAL;
	585	+
	586	+ /* Calculate the max QID based on SQ/CQ/doorbell counts.
	587	+ * SQ/CQ doorbells alternate.
	588	+ */
	589	+ num_dbs = (pci_resource_len(pdev, 0) - NVME_REG_DBS) /
	590	+ (fdev->db_stride * 4);
	591	+ fdev->max_qid = min3(cq_count, sq_count, num_dbs / 2) - 1;
	592	+ fdev->kern_end_qid = fdev->max_qid + 1;
	593	+ return 0;
	594	+}
	595	+
	596	+/* Allocate all MSI-X vectors available on a function and at least @min_vecs. */
	597	+static int fun_alloc_irqs(struct pci_dev *pdev, unsigned int min_vecs)
	598	+{
	599	+ int vecs, num_msix = pci_msix_vec_count(pdev);
	600	+
	601	+ if (num_msix < 0)
	602	+ return num_msix;
	603	+ if (min_vecs > num_msix)
	604	+ return -ERANGE;
	605	+
	606	+ vecs = pci_alloc_irq_vectors(pdev, min_vecs, num_msix, PCI_IRQ_MSIX);
	607	+ if (vecs > 0) {
	608	+ dev_info(&pdev->dev,
	609	+ "Allocated %d IRQ vectors of %d requested\n",
	610	+ vecs, num_msix);
	611	+ } else {
	612	+ dev_err(&pdev->dev,
	613	+ "Unable to allocate at least %u IRQ vectors\n",
	614	+ min_vecs);
	615	+ }
	616	+ return vecs;
	617	+}
	618	+
	619	+/* Allocate and initialize the IRQ manager state. */
	620	+static int fun_alloc_irq_mgr(struct fun_dev *fdev)
	621	+{
	622	+ fdev->irq_map = bitmap_zalloc(fdev->num_irqs, GFP_KERNEL);
	623	+ if (!fdev->irq_map)
	624	+ return -ENOMEM;
	625	+
	626	+ spin_lock_init(&fdev->irqmgr_lock);
	627	+ /* mark IRQ 0 allocated, it is used by the admin queue */
	628	+ __set_bit(0, fdev->irq_map);
	629	+ fdev->irqs_avail = fdev->num_irqs - 1;
	630	+ return 0;
	631	+}
	632	+
	633	+/* Reserve @nirqs of the currently available IRQs and return their indices. */
	634	+int fun_reserve_irqs(struct fun_dev fdev, unsigned int nirqs, u16 irq_indices)
	635	+{
	636	+ unsigned int b, n = 0;
	637	+ int err = -ENOSPC;
	638	+
	639	+ if (!nirqs)
	640	+ return 0;
	641	+
	642	+ spin_lock(&fdev->irqmgr_lock);
	643	+ if (nirqs > fdev->irqs_avail)
	644	+ goto unlock;
	645	+
	646	+ for_each_clear_bit(b, fdev->irq_map, fdev->num_irqs) {
	647	+ __set_bit(b, fdev->irq_map);
	648	+ irq_indices[n++] = b;
	649	+ if (n >= nirqs)
	650	+ break;
	651	+ }
	652	+
	653	+ WARN_ON(n < nirqs);
	654	+ fdev->irqs_avail -= n;
	655	+ err = n;
	656	+unlock:
	657	+ spin_unlock(&fdev->irqmgr_lock);
	658	+ return err;
	659	+}
	660	+EXPORT_SYMBOL(fun_reserve_irqs);
	661	+
	662	+/* Release @nirqs previously allocated IRQS with the supplied indices. */
	663	+void fun_release_irqs(struct fun_dev *fdev, unsigned int nirqs,
	664	+ u16 *irq_indices)
	665	+{
	666	+ unsigned int i;
	667	+
	668	+ spin_lock(&fdev->irqmgr_lock);
	669	+ for (i = 0; i < nirqs; i++)
	670	+ __clear_bit(irq_indices[i], fdev->irq_map);
	671	+ fdev->irqs_avail += nirqs;
	672	+ spin_unlock(&fdev->irqmgr_lock);
	673	+}
	674	+EXPORT_SYMBOL(fun_release_irqs);
	675	+
	676	+static void fun_serv_handler(struct work_struct *work)
	677	+{
	678	+ struct fun_dev *fd = container_of(work, struct fun_dev, service_task);
	679	+
	680	+ if (test_bit(FUN_SERV_DISABLED, &fd->service_flags))
	681	+ return;
	682	+ if (fd->serv_cb)
	683	+ fd->serv_cb(fd);
	684	+}
	685	+
	686	+void fun_serv_stop(struct fun_dev *fd)
	687	+{
	688	+ set_bit(FUN_SERV_DISABLED, &fd->service_flags);
	689	+ cancel_work_sync(&fd->service_task);
	690	+}
	691	+EXPORT_SYMBOL_GPL(fun_serv_stop);
	692	+
	693	+void fun_serv_restart(struct fun_dev *fd)
	694	+{
	695	+ clear_bit(FUN_SERV_DISABLED, &fd->service_flags);
	696	+ if (fd->service_flags)
	697	+ schedule_work(&fd->service_task);
	698	+}
	699	+EXPORT_SYMBOL_GPL(fun_serv_restart);
	700	+
	701	+void fun_serv_sched(struct fun_dev *fd)
	702	+{
	703	+ if (!test_bit(FUN_SERV_DISABLED, &fd->service_flags))
	704	+ schedule_work(&fd->service_task);
	705	+}
	706	+EXPORT_SYMBOL_GPL(fun_serv_sched);
	707	+
	708	+/* Check and try to get the device into a proper state for initialization,
	709	+ * i.e., CSTS.RDY = CC.EN = 0.
	710	+ */
	711	+static int sanitize_dev(struct fun_dev *fdev)
	712	+{
	713	+ int rc;
	714	+
	715	+ fdev->cap_reg = readq(fdev->bar + NVME_REG_CAP);
	716	+ fdev->cc_reg = readl(fdev->bar + NVME_REG_CC);
	717	+
	718	+ /* First get RDY to agree with the current EN. Give RDY the opportunity
	719	+ * to complete a potential recent EN change.
	720	+ */
	721	+ rc = fun_wait_ready(fdev, fdev->cc_reg & NVME_CC_ENABLE);
	722	+ if (rc)
	723	+ return rc;
	724	+
	725	+ /* Next, reset the device if EN is currently 1. */
	726	+ if (fdev->cc_reg & NVME_CC_ENABLE)
	727	+ rc = fun_disable_ctrl(fdev);
	728	+
	729	+ return rc;
	730	+}
	731	+
	732	+/* Undo the device initialization of fun_dev_enable(). */
	733	+void fun_dev_disable(struct fun_dev *fdev)
	734	+{
	735	+ struct pci_dev *pdev = to_pci_dev(fdev->dev);
	736	+
	737	+ pci_set_drvdata(pdev, NULL);
	738	+
	739	+ if (fdev->fw_handle != FUN_HCI_ID_INVALID) {
	740	+ fun_res_destroy(fdev, FUN_ADMIN_OP_SWUPGRADE, 0,
	741	+ fdev->fw_handle);
	742	+ fdev->fw_handle = FUN_HCI_ID_INVALID;
	743	+ }
	744	+
	745	+ fun_disable_admin_queue(fdev);
	746	+
	747	+ bitmap_free(fdev->irq_map);
	748	+ pci_free_irq_vectors(pdev);
	749	+
	750	+ pci_clear_master(pdev);
	751	+ pci_disable_pcie_error_reporting(pdev);
	752	+ pci_disable_device(pdev);
	753	+
	754	+ fun_unmap_bars(fdev);
	755	+}
	756	+EXPORT_SYMBOL(fun_dev_disable);
	757	+
	758	+/* Perform basic initialization of a device, including
	759	+ * - PCI config space setup and BAR0 mapping
	760	+ * - interrupt management initialization
	761	+ * - 1 admin queue setup
	762	+ * - determination of some device limits, such as number of queues.
	763	+ */
	764	+int fun_dev_enable(struct fun_dev fdev, struct pci_dev pdev,
	765	+ const struct fun_dev_params areq, const char name)
	766	+{
	767	+ int rc;
	768	+
	769	+ fdev->dev = &pdev->dev;
	770	+ rc = fun_map_bars(fdev, name);
	771	+ if (rc)
	772	+ return rc;
	773	+
	774	+ rc = fun_set_dma_masks(fdev->dev);
	775	+ if (rc)
	776	+ goto unmap;
	777	+
	778	+ rc = pci_enable_device_mem(pdev);
	779	+ if (rc) {
	780	+ dev_err(&pdev->dev, "Couldn't enable device, err %d\n", rc);
	781	+ goto unmap;
	782	+ }
	783	+
	784	+ pci_enable_pcie_error_reporting(pdev);
	785	+
	786	+ rc = sanitize_dev(fdev);
	787	+ if (rc)
	788	+ goto disable_dev;
	789	+
	790	+ fdev->fw_handle = FUN_HCI_ID_INVALID;
	791	+ fdev->q_depth = NVME_CAP_MQES(fdev->cap_reg) + 1;
	792	+ fdev->db_stride = 1 << NVME_CAP_STRIDE(fdev->cap_reg);
	793	+ fdev->dbs = fdev->bar + NVME_REG_DBS;
	794	+
	795	+ INIT_WORK(&fdev->service_task, fun_serv_handler);
	796	+ fdev->service_flags = FUN_SERV_DISABLED;
	797	+ fdev->serv_cb = areq->serv_cb;
	798	+
	799	+ rc = fun_alloc_irqs(pdev, areq->min_msix + 1); /* +1 for admin CQ */
	800	+ if (rc < 0)
	801	+ goto disable_dev;
	802	+ fdev->num_irqs = rc;
	803	+
	804	+ rc = fun_alloc_irq_mgr(fdev);
	805	+ if (rc)
	806	+ goto free_irqs;
	807	+
	808	+ pci_set_master(pdev);
	809	+ rc = fun_enable_admin_queue(fdev, areq);
	810	+ if (rc)
	811	+ goto free_irq_mgr;
	812	+
	813	+ rc = fun_get_dev_limits(fdev);
	814	+ if (rc < 0)
	815	+ goto disable_admin;
	816	+
	817	+ pci_save_state(pdev);
	818	+ pci_set_drvdata(pdev, fdev);
	819	+ pcie_print_link_status(pdev);
	820	+ dev_dbg(fdev->dev, "q_depth %u, db_stride %u, max qid %d kern_end_qid %d\n",
	821	+ fdev->q_depth, fdev->db_stride, fdev->max_qid,
	822	+ fdev->kern_end_qid);
	823	+ return 0;
	824	+
	825	+disable_admin:
	826	+ fun_disable_admin_queue(fdev);
	827	+free_irq_mgr:
	828	+ pci_clear_master(pdev);
	829	+ bitmap_free(fdev->irq_map);
	830	+free_irqs:
	831	+ pci_free_irq_vectors(pdev);
	832	+disable_dev:
	833	+ pci_disable_pcie_error_reporting(pdev);
	834	+ pci_disable_device(pdev);
	835	+unmap:
	836	+ fun_unmap_bars(fdev);
	837	+ return rc;
	838	+}
	839	+EXPORT_SYMBOL(fun_dev_enable);
	840	+
	841	+MODULE_AUTHOR("Dimitris Michailidis <dmichail@fungible.com>");
	842	+MODULE_DESCRIPTION("Core services driver for Fungible devices");
	843	+MODULE_LICENSE("Dual BSD/GPL");

drivers/net/ethernet/fungible/funcore/fun_dev.h+150−0

@@ -0,0 +1,150 @@
	1	+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) */
	2	+
	3	+#ifndef _FUNDEV_H
	4	+#define _FUNDEV_H
	5	+
	6	+#include <linux/sbitmap.h>
	7	+#include <linux/spinlock_types.h>
	8	+#include <linux/workqueue.h>
	9	+#include "fun_hci.h"
	10	+
	11	+struct pci_dev;
	12	+struct fun_dev;
	13	+struct fun_queue;
	14	+struct fun_cmd_ctx;
	15	+struct fun_queue_alloc_req;
	16	+
	17	+/* doorbell fields */
	18	+enum {
	19	+ FUN_DB_QIDX_S = 0,
	20	+ FUN_DB_INTCOAL_ENTRIES_S = 16,
	21	+ FUN_DB_INTCOAL_ENTRIES_M = 0x7f,
	22	+ FUN_DB_INTCOAL_USEC_S = 23,
	23	+ FUN_DB_INTCOAL_USEC_M = 0x7f,
	24	+ FUN_DB_IRQ_S = 30,
	25	+ FUN_DB_IRQ_F = 1 << FUN_DB_IRQ_S,
	26	+ FUN_DB_IRQ_ARM_S = 31,
	27	+ FUN_DB_IRQ_ARM_F = 1U << FUN_DB_IRQ_ARM_S
	28	+};
	29	+
	30	+/* Callback for asynchronous admin commands.
	31	+ * Invoked on reception of command response.
	32	+ */
	33	+typedef void (fun_admin_callback_t)(struct fun_dev fdev, void *rsp,
	34	+ void *cb_data);
	35	+
	36	+/* Callback for events/notifications received by an admin queue. */
	37	+typedef void (fun_admin_event_cb)(struct fun_dev fdev, void *cqe);
	38	+
	39	+/* Callback for pending work handled by the service task. */
	40	+typedef void (fun_serv_cb)(struct fun_dev fd);
	41	+
	42	+/* service task flags */
	43	+enum {
	44	+ FUN_SERV_DISABLED, /* service task is disabled */
	45	+ FUN_SERV_FIRST_AVAIL
	46	+};
	47	+
	48	+/* Driver state associated with a PCI function. */
	49	+struct fun_dev {
	50	+ struct device *dev;
	51	+
	52	+ void __iomem bar; / start of BAR0 mapping */
	53	+ u32 __iomem dbs; / start of doorbells in BAR0 mapping */
	54	+
	55	+ /* admin queue */
	56	+ struct fun_queue *admin_q;
	57	+ struct sbitmap_queue admin_sbq;
	58	+ struct fun_cmd_ctx *cmd_ctx;
	59	+ fun_admin_event_cb adminq_cb;
	60	+ bool suppress_cmds; /* if set don't write commands to SQ */
	61	+
	62	+ /* address increment between consecutive doorbells, in 4B units */
	63	+ unsigned int db_stride;
	64	+
	65	+ /* SW versions of device registers */
	66	+ u32 cc_reg; /* CC register */
	67	+ u64 cap_reg; /* CAPability register */
	68	+
	69	+ unsigned int q_depth; /* max queue depth supported by device */
	70	+ unsigned int max_qid; /* = #queues - 1, separately for SQs and CQs */
	71	+ unsigned int kern_end_qid; /* last qid in the kernel range + 1 */
	72	+
	73	+ unsigned int fw_handle;
	74	+
	75	+ /* IRQ manager */
	76	+ unsigned int num_irqs;
	77	+ unsigned int irqs_avail;
	78	+ spinlock_t irqmgr_lock;
	79	+ unsigned long *irq_map;
	80	+
	81	+ /* The service task handles work that needs a process context */
	82	+ struct work_struct service_task;
	83	+ unsigned long service_flags;
	84	+ fun_serv_cb serv_cb;
	85	+};
	86	+
	87	+struct fun_dev_params {
	88	+ u8 cqe_size_log2; /* admin q CQE size */
	89	+ u8 sqe_size_log2; /* admin q SQE size */
	90	+
	91	+ /* admin q depths */
	92	+ u16 cq_depth;
	93	+ u16 sq_depth;
	94	+ u16 rq_depth;
	95	+
	96	+ u16 min_msix; /* min vectors needed by requesting driver */
	97	+
	98	+ fun_admin_event_cb event_cb;
	99	+ fun_serv_cb serv_cb;
	100	+};
	101	+
	102	+/* Return the BAR address of a doorbell. */
	103	+static inline u32 __iomem fun_db_addr(const struct fun_dev fdev,
	104	+ unsigned int db_index)
	105	+{
	106	+ return &fdev->dbs[db_index * fdev->db_stride];
	107	+}
	108	+
	109	+/* Return the BAR address of an SQ doorbell. SQ and CQ DBs alternate,
	110	+ * SQs have even DB indices.
	111	+ */
	112	+static inline u32 __iomem fun_sq_db_addr(const struct fun_dev fdev,
	113	+ unsigned int sqid)
	114	+{
	115	+ return fun_db_addr(fdev, sqid * 2);
	116	+}
	117	+
	118	+static inline u32 __iomem fun_cq_db_addr(const struct fun_dev fdev,
	119	+ unsigned int cqid)
	120	+{
	121	+ return fun_db_addr(fdev, cqid * 2 + 1);
	122	+}
	123	+
	124	+int fun_get_res_count(struct fun_dev *fdev, enum fun_admin_op res);
	125	+int fun_res_destroy(struct fun_dev *fdev, enum fun_admin_op res,
	126	+ unsigned int flags, u32 id);
	127	+int fun_bind(struct fun_dev *fdev, enum fun_admin_bind_type type0,
	128	+ unsigned int id0, enum fun_admin_bind_type type1,
	129	+ unsigned int id1);
	130	+
	131	+int fun_submit_admin_cmd(struct fun_dev fdev, struct fun_admin_req_common cmd,
	132	+ fun_admin_callback_t cb, void *cb_data, bool wait_ok);
	133	+int fun_submit_admin_sync_cmd(struct fun_dev *fdev,
	134	+ struct fun_admin_req_common cmd, void rsp,
	135	+ size_t rspsize, unsigned int timeout);
	136	+
	137	+int fun_dev_enable(struct fun_dev fdev, struct pci_dev pdev,
	138	+ const struct fun_dev_params areq, const char name);
	139	+void fun_dev_disable(struct fun_dev *fdev);
	140	+
	141	+int fun_reserve_irqs(struct fun_dev *fdev, unsigned int nirqs,
	142	+ u16 *irq_indices);
	143	+void fun_release_irqs(struct fun_dev *fdev, unsigned int nirqs,
	144	+ u16 *irq_indices);
	145	+
	146	+void fun_serv_stop(struct fun_dev *fd);
	147	+void fun_serv_restart(struct fun_dev *fd);
	148	+void fun_serv_sched(struct fun_dev *fd);
	149	+
	150	+#endif /* _FUNDEV_H */

drivers/net/ethernet/fungible/funcore/fun_hci.h+1202−0

@@ -0,0 +1,1202 @@
	1	+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) */
	2	+
	3	+#ifndef __FUN_HCI_H
	4	+#define __FUN_HCI_H
	5	+
	6	+enum {
	7	+ FUN_HCI_ID_INVALID = 0xffffffff,
	8	+};
	9	+
	10	+enum fun_admin_op {
	11	+ FUN_ADMIN_OP_BIND = 0x1,
	12	+ FUN_ADMIN_OP_EPCQ = 0x11,
	13	+ FUN_ADMIN_OP_EPSQ = 0x12,
	14	+ FUN_ADMIN_OP_PORT = 0x13,
	15	+ FUN_ADMIN_OP_ETH = 0x14,
	16	+ FUN_ADMIN_OP_VI = 0x15,
	17	+ FUN_ADMIN_OP_SWUPGRADE = 0x1f,
	18	+ FUN_ADMIN_OP_RSS = 0x21,
	19	+ FUN_ADMIN_OP_ADI = 0x25,
	20	+ FUN_ADMIN_OP_KTLS = 0x26,
	21	+};
	22	+
	23	+enum {
	24	+ FUN_REQ_COMMON_FLAG_RSP = 0x1,
	25	+ FUN_REQ_COMMON_FLAG_HEAD_WB = 0x2,
	26	+ FUN_REQ_COMMON_FLAG_INT = 0x4,
	27	+ FUN_REQ_COMMON_FLAG_CQE_IN_RQBUF = 0x8,
	28	+};
	29	+
	30	+struct fun_admin_req_common {
	31	+ __u8 op;
	32	+ __u8 len8;
	33	+ __be16 flags;
	34	+ __u8 suboff8;
	35	+ __u8 rsvd0;
	36	+ __be16 cid;
	37	+};
	38	+
	39	+#define FUN_ADMIN_REQ_COMMON_INIT(_op, _len8, _flags, _suboff8, _cid) \
	40	+ (struct fun_admin_req_common) { \
	41	+ .op = (_op), .len8 = (_len8), .flags = cpu_to_be16(_flags), \
	42	+ .suboff8 = (_suboff8), .cid = cpu_to_be16(_cid), \
	43	+ }
	44	+
	45	+#define FUN_ADMIN_REQ_COMMON_INIT2(_op, _len) \
	46	+ (struct fun_admin_req_common) { \
	47	+ .op = (_op), .len8 = (_len) / 8, \
	48	+ }
	49	+
	50	+struct fun_admin_rsp_common {
	51	+ __u8 op;
	52	+ __u8 len8;
	53	+ __be16 flags;
	54	+ __u8 suboff8;
	55	+ __u8 ret;
	56	+ __be16 cid;
	57	+};
	58	+
	59	+struct fun_admin_write48_req {
	60	+ __be64 key_to_data;
	61	+};
	62	+
	63	+#define FUN_ADMIN_WRITE48_REQ_KEY_S 56U
	64	+#define FUN_ADMIN_WRITE48_REQ_KEY_M 0xff
	65	+#define FUN_ADMIN_WRITE48_REQ_KEY_P_NOSWAP(x) \
	66	+ (((__u64)x) << FUN_ADMIN_WRITE48_REQ_KEY_S)
	67	+
	68	+#define FUN_ADMIN_WRITE48_REQ_DATA_S 0U
	69	+#define FUN_ADMIN_WRITE48_REQ_DATA_M 0xffffffffffff
	70	+#define FUN_ADMIN_WRITE48_REQ_DATA_P_NOSWAP(x) \
	71	+ (((__u64)x) << FUN_ADMIN_WRITE48_REQ_DATA_S)
	72	+
	73	+#define FUN_ADMIN_WRITE48_REQ_INIT(key, data) \
	74	+ (struct fun_admin_write48_req) { \
	75	+ .key_to_data = cpu_to_be64( \
	76	+ FUN_ADMIN_WRITE48_REQ_KEY_P_NOSWAP(key) \| \
	77	+ FUN_ADMIN_WRITE48_REQ_DATA_P_NOSWAP(data)), \
	78	+ }
	79	+
	80	+struct fun_admin_write48_rsp {
	81	+ __be64 key_to_data;
	82	+};
	83	+
	84	+struct fun_admin_read48_req {
	85	+ __be64 key_pack;
	86	+};
	87	+
	88	+#define FUN_ADMIN_READ48_REQ_KEY_S 56U
	89	+#define FUN_ADMIN_READ48_REQ_KEY_M 0xff
	90	+#define FUN_ADMIN_READ48_REQ_KEY_P_NOSWAP(x) \
	91	+ (((__u64)x) << FUN_ADMIN_READ48_REQ_KEY_S)
	92	+
	93	+#define FUN_ADMIN_READ48_REQ_INIT(key) \
	94	+ (struct fun_admin_read48_req) { \
	95	+ .key_pack = \
	96	+ cpu_to_be64(FUN_ADMIN_READ48_REQ_KEY_P_NOSWAP(key)), \
	97	+ }
	98	+
	99	+struct fun_admin_read48_rsp {
	100	+ __be64 key_to_data;
	101	+};
	102	+
	103	+#define FUN_ADMIN_READ48_RSP_KEY_S 56U
	104	+#define FUN_ADMIN_READ48_RSP_KEY_M 0xff
	105	+#define FUN_ADMIN_READ48_RSP_KEY_G(x) \
	106	+ ((be64_to_cpu(x) >> FUN_ADMIN_READ48_RSP_KEY_S) & \
	107	+ FUN_ADMIN_READ48_RSP_KEY_M)
	108	+
	109	+#define FUN_ADMIN_READ48_RSP_RET_S 48U
	110	+#define FUN_ADMIN_READ48_RSP_RET_M 0xff
	111	+#define FUN_ADMIN_READ48_RSP_RET_G(x) \
	112	+ ((be64_to_cpu(x) >> FUN_ADMIN_READ48_RSP_RET_S) & \
	113	+ FUN_ADMIN_READ48_RSP_RET_M)
	114	+
	115	+#define FUN_ADMIN_READ48_RSP_DATA_S 0U
	116	+#define FUN_ADMIN_READ48_RSP_DATA_M 0xffffffffffff
	117	+#define FUN_ADMIN_READ48_RSP_DATA_G(x) \
	118	+ ((be64_to_cpu(x) >> FUN_ADMIN_READ48_RSP_DATA_S) & \
	119	+ FUN_ADMIN_READ48_RSP_DATA_M)
	120	+
	121	+enum fun_admin_bind_type {
	122	+ FUN_ADMIN_BIND_TYPE_EPCQ = 0x1,
	123	+ FUN_ADMIN_BIND_TYPE_EPSQ = 0x2,
	124	+ FUN_ADMIN_BIND_TYPE_PORT = 0x3,
	125	+ FUN_ADMIN_BIND_TYPE_RSS = 0x4,
	126	+ FUN_ADMIN_BIND_TYPE_VI = 0x5,
	127	+ FUN_ADMIN_BIND_TYPE_ETH = 0x6,
	128	+};
	129	+
	130	+struct fun_admin_bind_entry {
	131	+ __u8 type;
	132	+ __u8 rsvd0[3];
	133	+ __be32 id;
	134	+};
	135	+
	136	+#define FUN_ADMIN_BIND_ENTRY_INIT(_type, _id) \
	137	+ (struct fun_admin_bind_entry) { \
	138	+ .type = (_type), .id = cpu_to_be32(_id), \
	139	+ }
	140	+
	141	+struct fun_admin_bind_req {
	142	+ struct fun_admin_req_common common;
	143	+ struct fun_admin_bind_entry entry[];
	144	+};
	145	+
	146	+struct fun_admin_bind_rsp {
	147	+ struct fun_admin_rsp_common bind_rsp_common;
	148	+};
	149	+
	150	+struct fun_admin_simple_subop {
	151	+ __u8 subop;
	152	+ __u8 rsvd0;
	153	+ __be16 flags;
	154	+ __be32 data;
	155	+};
	156	+
	157	+#define FUN_ADMIN_SIMPLE_SUBOP_INIT(_subop, _flags, _data) \
	158	+ (struct fun_admin_simple_subop) { \
	159	+ .subop = (_subop), .flags = cpu_to_be16(_flags), \
	160	+ .data = cpu_to_be32(_data), \
	161	+ }
	162	+
	163	+enum fun_admin_subop {
	164	+ FUN_ADMIN_SUBOP_CREATE = 0x10,
	165	+ FUN_ADMIN_SUBOP_DESTROY = 0x11,
	166	+ FUN_ADMIN_SUBOP_MODIFY = 0x12,
	167	+ FUN_ADMIN_SUBOP_RES_COUNT = 0x14,
	168	+ FUN_ADMIN_SUBOP_READ = 0x15,
	169	+ FUN_ADMIN_SUBOP_WRITE = 0x16,
	170	+ FUN_ADMIN_SUBOP_NOTIFY = 0x17,
	171	+};
	172	+
	173	+enum {
	174	+ FUN_ADMIN_RES_CREATE_FLAG_ALLOCATOR = 0x1,
	175	+};
	176	+
	177	+struct fun_admin_generic_destroy_req {
	178	+ struct fun_admin_req_common common;
	179	+ struct fun_admin_simple_subop destroy;
	180	+};
	181	+
	182	+struct fun_admin_generic_create_rsp {
	183	+ struct fun_admin_rsp_common common;
	184	+
	185	+ __u8 subop;
	186	+ __u8 rsvd0;
	187	+ __be16 flags;
	188	+ __be32 id;
	189	+};
	190	+
	191	+struct fun_admin_res_count_req {
	192	+ struct fun_admin_req_common common;
	193	+ struct fun_admin_simple_subop count;
	194	+};
	195	+
	196	+struct fun_admin_res_count_rsp {
	197	+ struct fun_admin_rsp_common common;
	198	+ struct fun_admin_simple_subop count;
	199	+};
	200	+
	201	+enum {
	202	+ FUN_ADMIN_EPCQ_CREATE_FLAG_INT_EPCQ = 0x2,
	203	+ FUN_ADMIN_EPCQ_CREATE_FLAG_ENTRY_WR_TPH = 0x4,
	204	+ FUN_ADMIN_EPCQ_CREATE_FLAG_SL_WR_TPH = 0x8,
	205	+ FUN_ADMIN_EPCQ_CREATE_FLAG_RQ = 0x80,
	206	+ FUN_ADMIN_EPCQ_CREATE_FLAG_INT_IQ = 0x100,
	207	+ FUN_ADMIN_EPCQ_CREATE_FLAG_INT_NOARM = 0x200,
	208	+ FUN_ADMIN_EPCQ_CREATE_FLAG_DROP_ON_OVERFLOW = 0x400,
	209	+};
	210	+
	211	+struct fun_admin_epcq_req {
	212	+ struct fun_admin_req_common common;
	213	+ union epcq_req_subop {
	214	+ struct fun_admin_epcq_create_req {
	215	+ __u8 subop;
	216	+ __u8 rsvd0;
	217	+ __be16 flags;
	218	+ __be32 id;
	219	+
	220	+ __be32 epsqid;
	221	+ __u8 rsvd1;
	222	+ __u8 entry_size_log2;
	223	+ __be16 nentries;
	224	+
	225	+ __be64 address;
	226	+
	227	+ __be16 tailroom; /* per packet tailroom in bytes */
	228	+ __u8 headroom; /* per packet headroom in 2B units */
	229	+ __u8 intcoal_kbytes;
	230	+ __u8 intcoal_holdoff_nentries;
	231	+ __u8 intcoal_holdoff_usecs;
	232	+ __be16 intid;
	233	+
	234	+ __be32 scan_start_id;
	235	+ __be32 scan_end_id;
	236	+
	237	+ __be16 tph_cpuid;
	238	+ __u8 rsvd3[6];
	239	+ } create;
	240	+
	241	+ struct fun_admin_epcq_modify_req {
	242	+ __u8 subop;
	243	+ __u8 rsvd0;
	244	+ __be16 flags;
	245	+ __be32 id;
	246	+
	247	+ __be16 headroom; /* headroom in bytes */
	248	+ __u8 rsvd1[6];
	249	+ } modify;
	250	+ } u;
	251	+};
	252	+
	253	+#define FUN_ADMIN_EPCQ_CREATE_REQ_INIT( \
	254	+ _subop, _flags, _id, _epsqid, _entry_size_log2, _nentries, _address, \
	255	+ _tailroom, _headroom, _intcoal_kbytes, _intcoal_holdoff_nentries, \
	256	+ _intcoal_holdoff_usecs, _intid, _scan_start_id, _scan_end_id, \
	257	+ _tph_cpuid) \
	258	+ (struct fun_admin_epcq_create_req) { \
	259	+ .subop = (_subop), .flags = cpu_to_be16(_flags), \
	260	+ .id = cpu_to_be32(_id), .epsqid = cpu_to_be32(_epsqid), \
	261	+ .entry_size_log2 = _entry_size_log2, \
	262	+ .nentries = cpu_to_be16(_nentries), \
	263	+ .address = cpu_to_be64(_address), \
	264	+ .tailroom = cpu_to_be16(_tailroom), .headroom = _headroom, \
	265	+ .intcoal_kbytes = _intcoal_kbytes, \
	266	+ .intcoal_holdoff_nentries = _intcoal_holdoff_nentries, \
	267	+ .intcoal_holdoff_usecs = _intcoal_holdoff_usecs, \
	268	+ .intid = cpu_to_be16(_intid), \
	269	+ .scan_start_id = cpu_to_be32(_scan_start_id), \
	270	+ .scan_end_id = cpu_to_be32(_scan_end_id), \
	271	+ .tph_cpuid = cpu_to_be16(_tph_cpuid), \
	272	+ }
	273	+
	274	+#define FUN_ADMIN_EPCQ_MODIFY_REQ_INIT(_subop, _flags, _id, _headroom) \
	275	+ (struct fun_admin_epcq_modify_req) { \
	276	+ .subop = (_subop), .flags = cpu_to_be16(_flags), \
	277	+ .id = cpu_to_be32(_id), .headroom = cpu_to_be16(_headroom), \
	278	+ }
	279	+
	280	+enum {
	281	+ FUN_ADMIN_EPSQ_CREATE_FLAG_INT_EPSQ = 0x2,
	282	+ FUN_ADMIN_EPSQ_CREATE_FLAG_ENTRY_RD_TPH = 0x4,
	283	+ FUN_ADMIN_EPSQ_CREATE_FLAG_GL_RD_TPH = 0x8,
	284	+ FUN_ADMIN_EPSQ_CREATE_FLAG_HEAD_WB_ADDRESS = 0x10,
	285	+ FUN_ADMIN_EPSQ_CREATE_FLAG_HEAD_WB_ADDRESS_TPH = 0x20,
	286	+ FUN_ADMIN_EPSQ_CREATE_FLAG_HEAD_WB_EPCQ = 0x40,
	287	+ FUN_ADMIN_EPSQ_CREATE_FLAG_RQ = 0x80,
	288	+ FUN_ADMIN_EPSQ_CREATE_FLAG_INT_IQ = 0x100,
	289	+ FUN_ADMIN_EPSQ_CREATE_FLAG_NO_CMPL = 0x200,
	290	+};
	291	+
	292	+struct fun_admin_epsq_req {
	293	+ struct fun_admin_req_common common;
	294	+
	295	+ union epsq_req_subop {
	296	+ struct fun_admin_epsq_create_req {
	297	+ __u8 subop;
	298	+ __u8 rsvd0;
	299	+ __be16 flags;
	300	+ __be32 id;
	301	+
	302	+ __be32 epcqid;
	303	+ __u8 rsvd1;
	304	+ __u8 entry_size_log2;
	305	+ __be16 nentries;
	306	+
	307	+ __be64 address; /* DMA address of epsq */
	308	+
	309	+ __u8 rsvd2[3];
	310	+ __u8 intcoal_kbytes;
	311	+ __u8 intcoal_holdoff_nentries;
	312	+ __u8 intcoal_holdoff_usecs;
	313	+ __be16 intid;
	314	+
	315	+ __be32 scan_start_id;
	316	+ __be32 scan_end_id;
	317	+
	318	+ __u8 rsvd3[4];
	319	+ __be16 tph_cpuid;
	320	+ __u8 buf_size_log2; /* log2 of RQ buffer size */
	321	+ __u8 head_wb_size_log2; /* log2 of head write back size */
	322	+
	323	+ __be64 head_wb_address; /* DMA address for head writeback */
	324	+ } create;
	325	+ } u;
	326	+};
	327	+
	328	+#define FUN_ADMIN_EPSQ_CREATE_REQ_INIT( \
	329	+ _subop, _flags, _id, _epcqid, _entry_size_log2, _nentries, _address, \
	330	+ _intcoal_kbytes, _intcoal_holdoff_nentries, _intcoal_holdoff_usecs, \
	331	+ _intid, _scan_start_id, _scan_end_id, _tph_cpuid, _buf_size_log2, \
	332	+ _head_wb_size_log2, _head_wb_address) \
	333	+ (struct fun_admin_epsq_create_req) { \
	334	+ .subop = (_subop), .flags = cpu_to_be16(_flags), \
	335	+ .id = cpu_to_be32(_id), .epcqid = cpu_to_be32(_epcqid), \
	336	+ .entry_size_log2 = _entry_size_log2, \
	337	+ .nentries = cpu_to_be16(_nentries), \
	338	+ .address = cpu_to_be64(_address), \
	339	+ .intcoal_kbytes = _intcoal_kbytes, \
	340	+ .intcoal_holdoff_nentries = _intcoal_holdoff_nentries, \
	341	+ .intcoal_holdoff_usecs = _intcoal_holdoff_usecs, \
	342	+ .intid = cpu_to_be16(_intid), \
	343	+ .scan_start_id = cpu_to_be32(_scan_start_id), \
	344	+ .scan_end_id = cpu_to_be32(_scan_end_id), \
	345	+ .tph_cpuid = cpu_to_be16(_tph_cpuid), \
	346	+ .buf_size_log2 = _buf_size_log2, \
	347	+ .head_wb_size_log2 = _head_wb_size_log2, \
	348	+ .head_wb_address = cpu_to_be64(_head_wb_address), \
	349	+ }
	350	+
	351	+enum {
	352	+ FUN_PORT_CAP_OFFLOADS = 0x1,
	353	+ FUN_PORT_CAP_STATS = 0x2,
	354	+ FUN_PORT_CAP_LOOPBACK = 0x4,
	355	+ FUN_PORT_CAP_VPORT = 0x8,
	356	+ FUN_PORT_CAP_TX_PAUSE = 0x10,
	357	+ FUN_PORT_CAP_RX_PAUSE = 0x20,
	358	+ FUN_PORT_CAP_AUTONEG = 0x40,
	359	+ FUN_PORT_CAP_RSS = 0x80,
	360	+ FUN_PORT_CAP_VLAN_OFFLOADS = 0x100,
	361	+ FUN_PORT_CAP_ENCAP_OFFLOADS = 0x200,
	362	+ FUN_PORT_CAP_1000_X = 0x1000,
	363	+ FUN_PORT_CAP_10G_R = 0x2000,
	364	+ FUN_PORT_CAP_40G_R4 = 0x4000,
	365	+ FUN_PORT_CAP_25G_R = 0x8000,
	366	+ FUN_PORT_CAP_50G_R2 = 0x10000,
	367	+ FUN_PORT_CAP_50G_R = 0x20000,
	368	+ FUN_PORT_CAP_100G_R4 = 0x40000,
	369	+ FUN_PORT_CAP_100G_R2 = 0x80000,
	370	+ FUN_PORT_CAP_200G_R4 = 0x100000,
	371	+ FUN_PORT_CAP_FEC_NONE = 0x10000000,
	372	+ FUN_PORT_CAP_FEC_FC = 0x20000000,
	373	+ FUN_PORT_CAP_FEC_RS = 0x40000000,
	374	+};
	375	+
	376	+enum fun_port_brkout_mode {
	377	+ FUN_PORT_BRKMODE_NA = 0x0,
	378	+ FUN_PORT_BRKMODE_NONE = 0x1,
	379	+ FUN_PORT_BRKMODE_2X = 0x2,
	380	+ FUN_PORT_BRKMODE_4X = 0x3,
	381	+};
	382	+
	383	+enum {
	384	+ FUN_PORT_SPEED_AUTO = 0x0,
	385	+ FUN_PORT_SPEED_10M = 0x1,
	386	+ FUN_PORT_SPEED_100M = 0x2,
	387	+ FUN_PORT_SPEED_1G = 0x4,
	388	+ FUN_PORT_SPEED_10G = 0x8,
	389	+ FUN_PORT_SPEED_25G = 0x10,
	390	+ FUN_PORT_SPEED_40G = 0x20,
	391	+ FUN_PORT_SPEED_50G = 0x40,
	392	+ FUN_PORT_SPEED_100G = 0x80,
	393	+ FUN_PORT_SPEED_200G = 0x100,
	394	+};
	395	+
	396	+enum fun_port_duplex_mode {
	397	+ FUN_PORT_FULL_DUPLEX = 0x0,
	398	+ FUN_PORT_HALF_DUPLEX = 0x1,
	399	+};
	400	+
	401	+enum {
	402	+ FUN_PORT_FEC_NA = 0x0,
	403	+ FUN_PORT_FEC_OFF = 0x1,
	404	+ FUN_PORT_FEC_RS = 0x2,
	405	+ FUN_PORT_FEC_FC = 0x4,
	406	+ FUN_PORT_FEC_AUTO = 0x8,
	407	+};
	408	+
	409	+enum fun_port_link_status {
	410	+ FUN_PORT_LINK_UP = 0x0,
	411	+ FUN_PORT_LINK_UP_WITH_ERR = 0x1,
	412	+ FUN_PORT_LINK_DOWN = 0x2,
	413	+};
	414	+
	415	+enum fun_port_led_type {
	416	+ FUN_PORT_LED_OFF = 0x0,
	417	+ FUN_PORT_LED_AMBER = 0x1,
	418	+ FUN_PORT_LED_GREEN = 0x2,
	419	+ FUN_PORT_LED_BEACON_ON = 0x3,
	420	+ FUN_PORT_LED_BEACON_OFF = 0x4,
	421	+};
	422	+
	423	+enum {
	424	+ FUN_PORT_FLAG_MAC_DOWN = 0x1,
	425	+ FUN_PORT_FLAG_MAC_UP = 0x2,
	426	+ FUN_PORT_FLAG_NH_DOWN = 0x4,
	427	+ FUN_PORT_FLAG_NH_UP = 0x8,
	428	+};
	429	+
	430	+enum {
	431	+ FUN_PORT_FLAG_ENABLE_NOTIFY = 0x1,
	432	+};
	433	+
	434	+enum fun_port_lane_attr {
	435	+ FUN_PORT_LANE_1 = 0x1,
	436	+ FUN_PORT_LANE_2 = 0x2,
	437	+ FUN_PORT_LANE_4 = 0x4,
	438	+ FUN_PORT_LANE_SPEED_10G = 0x100,
	439	+ FUN_PORT_LANE_SPEED_25G = 0x200,
	440	+ FUN_PORT_LANE_SPEED_50G = 0x400,
	441	+ FUN_PORT_LANE_SPLIT = 0x8000,
	442	+};
	443	+
	444	+enum fun_admin_port_subop {
	445	+ FUN_ADMIN_PORT_SUBOP_INETADDR_EVENT = 0x24,
	446	+};
	447	+
	448	+enum fun_admin_port_key {
	449	+ FUN_ADMIN_PORT_KEY_ILLEGAL = 0x0,
	450	+ FUN_ADMIN_PORT_KEY_MTU = 0x1,
	451	+ FUN_ADMIN_PORT_KEY_FEC = 0x2,
	452	+ FUN_ADMIN_PORT_KEY_SPEED = 0x3,
	453	+ FUN_ADMIN_PORT_KEY_DEBOUNCE = 0x4,
	454	+ FUN_ADMIN_PORT_KEY_DUPLEX = 0x5,
	455	+ FUN_ADMIN_PORT_KEY_MACADDR = 0x6,
	456	+ FUN_ADMIN_PORT_KEY_LINKMODE = 0x7,
	457	+ FUN_ADMIN_PORT_KEY_BREAKOUT = 0x8,
	458	+ FUN_ADMIN_PORT_KEY_ENABLE = 0x9,
	459	+ FUN_ADMIN_PORT_KEY_DISABLE = 0xa,
	460	+ FUN_ADMIN_PORT_KEY_ERR_DISABLE = 0xb,
	461	+ FUN_ADMIN_PORT_KEY_CAPABILITIES = 0xc,
	462	+ FUN_ADMIN_PORT_KEY_LP_CAPABILITIES = 0xd,
	463	+ FUN_ADMIN_PORT_KEY_STATS_DMA_LOW = 0xe,
	464	+ FUN_ADMIN_PORT_KEY_STATS_DMA_HIGH = 0xf,
	465	+ FUN_ADMIN_PORT_KEY_LANE_ATTRS = 0x10,
	466	+ FUN_ADMIN_PORT_KEY_LED = 0x11,
	467	+ FUN_ADMIN_PORT_KEY_ADVERT = 0x12,
	468	+};
	469	+
	470	+struct fun_subop_imm {
	471	+ __u8 subop; /* see fun_data_subop enum */
	472	+ __u8 flags;
	473	+ __u8 nsgl;
	474	+ __u8 rsvd0;
	475	+ __be32 len;
	476	+
	477	+ __u8 data[];
	478	+};
	479	+
	480	+enum fun_subop_sgl_flags {
	481	+ FUN_SUBOP_SGL_USE_OFF8 = 0x1,
	482	+ FUN_SUBOP_FLAG_FREE_BUF = 0x2,
	483	+ FUN_SUBOP_FLAG_IS_REFBUF = 0x4,
	484	+ FUN_SUBOP_SGL_FLAG_LOCAL = 0x8,
	485	+};
	486	+
	487	+enum fun_data_op {
	488	+ FUN_DATAOP_INVALID = 0x0,
	489	+ FUN_DATAOP_SL = 0x1, /* scatter */
	490	+ FUN_DATAOP_GL = 0x2, /* gather */
	491	+ FUN_DATAOP_SGL = 0x3, /* scatter-gather */
	492	+ FUN_DATAOP_IMM = 0x4, /* immediate data */
	493	+ FUN_DATAOP_RQBUF = 0x8, /* rq buffer */
	494	+};
	495	+
	496	+struct fun_dataop_gl {
	497	+ __u8 subop;
	498	+ __u8 flags;
	499	+ __be16 sgl_off;
	500	+ __be32 sgl_len;
	501	+
	502	+ __be64 sgl_data;
	503	+};
	504	+
	505	+static inline void fun_dataop_gl_init(struct fun_dataop_gl *s, u8 flags,
	506	+ u16 sgl_off, u32 sgl_len, u64 sgl_data)
	507	+{
	508	+ s->subop = FUN_DATAOP_GL;
	509	+ s->flags = flags;
	510	+ s->sgl_off = cpu_to_be16(sgl_off);
	511	+ s->sgl_len = cpu_to_be32(sgl_len);
	512	+ s->sgl_data = cpu_to_be64(sgl_data);
	513	+}
	514	+
	515	+struct fun_dataop_imm {
	516	+ __u8 subop;
	517	+ __u8 flags;
	518	+ __be16 rsvd0;
	519	+ __be32 sgl_len;
	520	+};
	521	+
	522	+struct fun_subop_sgl {
	523	+ __u8 subop;
	524	+ __u8 flags;
	525	+ __u8 nsgl;
	526	+ __u8 rsvd0;
	527	+ __be32 sgl_len;
	528	+
	529	+ __be64 sgl_data;
	530	+};
	531	+
	532	+#define FUN_SUBOP_SGL_INIT(_subop, _flags, _nsgl, _sgl_len, _sgl_data) \
	533	+ (struct fun_subop_sgl) { \
	534	+ .subop = (_subop), .flags = (_flags), .nsgl = (_nsgl), \
	535	+ .sgl_len = cpu_to_be32(_sgl_len), \
	536	+ .sgl_data = cpu_to_be64(_sgl_data), \
	537	+ }
	538	+
	539	+struct fun_dataop_rqbuf {
	540	+ __u8 subop;
	541	+ __u8 rsvd0;
	542	+ __be16 cid;
	543	+ __be32 bufoff;
	544	+};
	545	+
	546	+struct fun_dataop_hdr {
	547	+ __u8 nsgl;
	548	+ __u8 flags;
	549	+ __u8 ngather;
	550	+ __u8 nscatter;
	551	+ __be32 total_len;
	552	+
	553	+ struct fun_dataop_imm imm[];
	554	+};
	555	+
	556	+#define FUN_DATAOP_HDR_INIT(_nsgl, _flags, _ngather, _nscatter, _total_len) \
	557	+ (struct fun_dataop_hdr) { \
	558	+ .nsgl = _nsgl, .flags = _flags, .ngather = _ngather, \
	559	+ .nscatter = _nscatter, .total_len = cpu_to_be32(_total_len), \
	560	+ }
	561	+
	562	+enum fun_port_inetaddr_event_type {
	563	+ FUN_PORT_INETADDR_ADD = 0x1,
	564	+ FUN_PORT_INETADDR_DEL = 0x2,
	565	+};
	566	+
	567	+enum fun_port_inetaddr_addr_family {
	568	+ FUN_PORT_INETADDR_IPV4 = 0x1,
	569	+ FUN_PORT_INETADDR_IPV6 = 0x2,
	570	+};
	571	+
	572	+struct fun_admin_port_req {
	573	+ struct fun_admin_req_common common;
	574	+
	575	+ union port_req_subop {
	576	+ struct fun_admin_port_create_req {
	577	+ __u8 subop;
	578	+ __u8 rsvd0;
	579	+ __be16 flags;
	580	+ __be32 id;
	581	+ } create;
	582	+ struct fun_admin_port_write_req {
	583	+ __u8 subop;
	584	+ __u8 rsvd0;
	585	+ __be16 flags;
	586	+ __be32 id; /* portid */
	587	+
	588	+ struct fun_admin_write48_req write48[];
	589	+ } write;
	590	+ struct fun_admin_port_read_req {
	591	+ __u8 subop;
	592	+ __u8 rsvd0;
	593	+ __be16 flags;
	594	+ __be32 id; /* portid */
	595	+
	596	+ struct fun_admin_read48_req read48[];
	597	+ } read;
	598	+ struct fun_admin_port_inetaddr_event_req {
	599	+ __u8 subop;
	600	+ __u8 rsvd0;
	601	+ __u8 event_type;
	602	+ __u8 addr_family;
	603	+ __be32 id;
	604	+
	605	+ __u8 addr[];
	606	+ } inetaddr_event;
	607	+ } u;
	608	+};
	609	+
	610	+#define FUN_ADMIN_PORT_CREATE_REQ_INIT(_subop, _flags, _id) \
	611	+ (struct fun_admin_port_create_req) { \
	612	+ .subop = (_subop), .flags = cpu_to_be16(_flags), \
	613	+ .id = cpu_to_be32(_id), \
	614	+ }
	615	+
	616	+#define FUN_ADMIN_PORT_WRITE_REQ_INIT(_subop, _flags, _id) \
	617	+ (struct fun_admin_port_write_req) { \
	618	+ .subop = (_subop), .flags = cpu_to_be16(_flags), \
	619	+ .id = cpu_to_be32(_id), \
	620	+ }
	621	+
	622	+#define FUN_ADMIN_PORT_READ_REQ_INIT(_subop, _flags, _id) \
	623	+ (struct fun_admin_port_read_req) { \
	624	+ .subop = (_subop), .flags = cpu_to_be16(_flags), \
	625	+ .id = cpu_to_be32(_id), \
	626	+ }
	627	+
	628	+struct fun_admin_port_rsp {
	629	+ struct fun_admin_rsp_common common;
	630	+
	631	+ union port_rsp_subop {
	632	+ struct fun_admin_port_create_rsp {
	633	+ __u8 subop;
	634	+ __u8 rsvd0[3];
	635	+ __be32 id;
	636	+
	637	+ __be16 lport;
	638	+ __u8 rsvd1[6];
	639	+ } create;
	640	+ struct fun_admin_port_write_rsp {
	641	+ __u8 subop;
	642	+ __u8 rsvd0[3];
	643	+ __be32 id; /* portid */
	644	+
	645	+ struct fun_admin_write48_rsp write48[];
	646	+ } write;
	647	+ struct fun_admin_port_read_rsp {
	648	+ __u8 subop;
	649	+ __u8 rsvd0[3];
	650	+ __be32 id; /* portid */
	651	+
	652	+ struct fun_admin_read48_rsp read48[];
	653	+ } read;
	654	+ struct fun_admin_port_inetaddr_event_rsp {
	655	+ __u8 subop;
	656	+ __u8 rsvd0[3];
	657	+ __be32 id; /* portid */
	658	+ } inetaddr_event;
	659	+ } u;
	660	+};
	661	+
	662	+enum fun_xcvr_type {
	663	+ FUN_XCVR_BASET = 0x0,
	664	+ FUN_XCVR_CU = 0x1,
	665	+ FUN_XCVR_SMF = 0x2,
	666	+ FUN_XCVR_MMF = 0x3,
	667	+ FUN_XCVR_AOC = 0x4,
	668	+ FUN_XCVR_SFPP = 0x10, /* SFP+ or later */
	669	+ FUN_XCVR_QSFPP = 0x11, /* QSFP+ or later */
	670	+ FUN_XCVR_QSFPDD = 0x12, /* QSFP-DD */
	671	+};
	672	+
	673	+struct fun_admin_port_notif {
	674	+ struct fun_admin_rsp_common common;
	675	+
	676	+ __u8 subop;
	677	+ __u8 rsvd0;
	678	+ __be16 id;
	679	+ __be32 speed; /* in 10 Mbps units */
	680	+
	681	+ __u8 link_state;
	682	+ __u8 missed_events;
	683	+ __u8 link_down_reason;
	684	+ __u8 xcvr_type;
	685	+ __u8 flow_ctrl;
	686	+ __u8 fec;
	687	+ __u8 active_lanes;
	688	+ __u8 rsvd1;
	689	+
	690	+ __be64 advertising;
	691	+
	692	+ __be64 lp_advertising;
	693	+};
	694	+
	695	+enum fun_eth_rss_const {
	696	+ FUN_ETH_RSS_MAX_KEY_SIZE = 0x28,
	697	+ FUN_ETH_RSS_MAX_INDIR_ENT = 0x40,
	698	+};
	699	+
	700	+enum fun_eth_hash_alg {
	701	+ FUN_ETH_RSS_ALG_INVALID = 0x0,
	702	+ FUN_ETH_RSS_ALG_TOEPLITZ = 0x1,
	703	+ FUN_ETH_RSS_ALG_CRC32 = 0x2,
	704	+};
	705	+
	706	+struct fun_admin_rss_req {
	707	+ struct fun_admin_req_common common;
	708	+
	709	+ union rss_req_subop {
	710	+ struct fun_admin_rss_create_req {
	711	+ __u8 subop;
	712	+ __u8 rsvd0;
	713	+ __be16 flags;
	714	+ __be32 id;
	715	+
	716	+ __be32 rsvd1;
	717	+ __be32 viid; /* VI flow id */
	718	+
	719	+ __be64 metadata[1];
	720	+
	721	+ __u8 alg;
	722	+ __u8 keylen;
	723	+ __u8 indir_nent;
	724	+ __u8 rsvd2;
	725	+ __be16 key_off;
	726	+ __be16 indir_off;
	727	+
	728	+ struct fun_dataop_hdr dataop;
	729	+ } create;
	730	+ } u;
	731	+};
	732	+
	733	+#define FUN_ADMIN_RSS_CREATE_REQ_INIT(_subop, _flags, _id, _viid, _alg, \
	734	+ _keylen, _indir_nent, _key_off, \
	735	+ _indir_off) \
	736	+ (struct fun_admin_rss_create_req) { \
	737	+ .subop = (_subop), .flags = cpu_to_be16(_flags), \
	738	+ .id = cpu_to_be32(_id), .viid = cpu_to_be32(_viid), \
	739	+ .alg = _alg, .keylen = _keylen, .indir_nent = _indir_nent, \
	740	+ .key_off = cpu_to_be16(_key_off), \
	741	+ .indir_off = cpu_to_be16(_indir_off), \
	742	+ }
	743	+
	744	+struct fun_admin_vi_req {
	745	+ struct fun_admin_req_common common;
	746	+
	747	+ union vi_req_subop {
	748	+ struct fun_admin_vi_create_req {
	749	+ __u8 subop;
	750	+ __u8 rsvd0;
	751	+ __be16 flags;
	752	+ __be32 id;
	753	+
	754	+ __be32 rsvd1;
	755	+ __be32 portid; /* port flow id */
	756	+ } create;
	757	+ } u;
	758	+};
	759	+
	760	+#define FUN_ADMIN_VI_CREATE_REQ_INIT(_subop, _flags, _id, _portid) \
	761	+ (struct fun_admin_vi_create_req) { \
	762	+ .subop = (_subop), .flags = cpu_to_be16(_flags), \
	763	+ .id = cpu_to_be32(_id), .portid = cpu_to_be32(_portid), \
	764	+ }
	765	+
	766	+struct fun_admin_eth_req {
	767	+ struct fun_admin_req_common common;
	768	+
	769	+ union eth_req_subop {
	770	+ struct fun_admin_eth_create_req {
	771	+ __u8 subop;
	772	+ __u8 rsvd0;
	773	+ __be16 flags;
	774	+ __be32 id;
	775	+
	776	+ __be32 rsvd1;
	777	+ __be32 portid; /* port flow id */
	778	+ } create;
	779	+ } u;
	780	+};
	781	+
	782	+#define FUN_ADMIN_ETH_CREATE_REQ_INIT(_subop, _flags, _id, _portid) \
	783	+ (struct fun_admin_eth_create_req) { \
	784	+ .subop = (_subop), .flags = cpu_to_be16(_flags), \
	785	+ .id = cpu_to_be32(_id), .portid = cpu_to_be32(_portid), \
	786	+ }
	787	+
	788	+enum {
	789	+ FUN_ADMIN_SWU_UPGRADE_FLAG_INIT = 0x10,
	790	+ FUN_ADMIN_SWU_UPGRADE_FLAG_COMPLETE = 0x20,
	791	+ FUN_ADMIN_SWU_UPGRADE_FLAG_DOWNGRADE = 0x40,
	792	+ FUN_ADMIN_SWU_UPGRADE_FLAG_ACTIVE_IMAGE = 0x80,
	793	+ FUN_ADMIN_SWU_UPGRADE_FLAG_ASYNC = 0x1,
	794	+};
	795	+
	796	+enum fun_admin_swu_subop {
	797	+ FUN_ADMIN_SWU_SUBOP_GET_VERSION = 0x20,
	798	+ FUN_ADMIN_SWU_SUBOP_UPGRADE = 0x21,
	799	+ FUN_ADMIN_SWU_SUBOP_UPGRADE_DATA = 0x22,
	800	+ FUN_ADMIN_SWU_SUBOP_GET_ALL_VERSIONS = 0x23,
	801	+};
	802	+
	803	+struct fun_admin_swu_req {
	804	+ struct fun_admin_req_common common;
	805	+
	806	+ union swu_req_subop {
	807	+ struct fun_admin_swu_create_req {
	808	+ __u8 subop;
	809	+ __u8 rsvd0;
	810	+ __be16 flags;
	811	+ __be32 id;
	812	+ } create;
	813	+ struct fun_admin_swu_upgrade_req {
	814	+ __u8 subop;
	815	+ __u8 rsvd0;
	816	+ __be16 flags;
	817	+ __be32 id;
	818	+
	819	+ __be32 fourcc;
	820	+ __be32 rsvd1;
	821	+
	822	+ __be64 image_size; /* upgrade image length */
	823	+ } upgrade;
	824	+ struct fun_admin_swu_upgrade_data_req {
	825	+ __u8 subop;
	826	+ __u8 rsvd0;
	827	+ __be16 flags;
	828	+ __be32 id;
	829	+
	830	+ __be32 offset; /* offset of data in this command */
	831	+ __be32 size; /* total size of data in this command */
	832	+ } upgrade_data;
	833	+ } u;
	834	+
	835	+ struct fun_subop_sgl sgl[]; /* in, out buffers through sgl */
	836	+};
	837	+
	838	+#define FUN_ADMIN_SWU_CREATE_REQ_INIT(_subop, _flags, _id) \
	839	+ (struct fun_admin_swu_create_req) { \
	840	+ .subop = (_subop), .flags = cpu_to_be16(_flags), \
	841	+ .id = cpu_to_be32(_id), \
	842	+ }
	843	+
	844	+#define FUN_ADMIN_SWU_UPGRADE_REQ_INIT(_subop, _flags, _id, _fourcc, \
	845	+ _image_size) \
	846	+ (struct fun_admin_swu_upgrade_req) { \
	847	+ .subop = (_subop), .flags = cpu_to_be16(_flags), \
	848	+ .id = cpu_to_be32(_id), .fourcc = cpu_to_be32(_fourcc), \
	849	+ .image_size = cpu_to_be64(_image_size), \
	850	+ }
	851	+
	852	+#define FUN_ADMIN_SWU_UPGRADE_DATA_REQ_INIT(_subop, _flags, _id, _offset, \
	853	+ _size) \
	854	+ (struct fun_admin_swu_upgrade_data_req) { \
	855	+ .subop = (_subop), .flags = cpu_to_be16(_flags), \
	856	+ .id = cpu_to_be32(_id), .offset = cpu_to_be32(_offset), \
	857	+ .size = cpu_to_be32(_size), \
	858	+ }
	859	+
	860	+struct fun_admin_swu_rsp {
	861	+ struct fun_admin_rsp_common common;
	862	+
	863	+ union swu_rsp_subop {
	864	+ struct fun_admin_swu_create_rsp {
	865	+ __u8 subop;
	866	+ __u8 rsvd0;
	867	+ __be16 flags;
	868	+ __be32 id;
	869	+ } create;
	870	+ struct fun_admin_swu_upgrade_rsp {
	871	+ __u8 subop;
	872	+ __u8 rsvd0[3];
	873	+ __be32 id;
	874	+
	875	+ __be32 fourcc;
	876	+ __be32 status;
	877	+
	878	+ __be32 progress;
	879	+ __be32 unused;
	880	+ } upgrade;
	881	+ struct fun_admin_swu_upgrade_data_rsp {
	882	+ __u8 subop;
	883	+ __u8 rsvd0;
	884	+ __be16 flags;
	885	+ __be32 id;
	886	+
	887	+ __be32 offset;
	888	+ __be32 size;
	889	+ } upgrade_data;
	890	+ } u;
	891	+};
	892	+
	893	+enum fun_ktls_version {
	894	+ FUN_KTLS_TLSV2 = 0x20,
	895	+ FUN_KTLS_TLSV3 = 0x30,
	896	+};
	897	+
	898	+enum fun_ktls_cipher {
	899	+ FUN_KTLS_CIPHER_AES_GCM_128 = 0x33,
	900	+ FUN_KTLS_CIPHER_AES_GCM_256 = 0x34,
	901	+ FUN_KTLS_CIPHER_AES_CCM_128 = 0x35,
	902	+ FUN_KTLS_CIPHER_CHACHA20_POLY1305 = 0x36,
	903	+};
	904	+
	905	+enum fun_ktls_modify_flags {
	906	+ FUN_KTLS_MODIFY_REMOVE = 0x1,
	907	+};
	908	+
	909	+struct fun_admin_ktls_create_req {
	910	+ struct fun_admin_req_common common;
	911	+
	912	+ __u8 subop;
	913	+ __u8 rsvd0;
	914	+ __be16 flags;
	915	+ __be32 id;
	916	+};
	917	+
	918	+#define FUN_ADMIN_KTLS_CREATE_REQ_INIT(_subop, _flags, _id) \
	919	+ (struct fun_admin_ktls_create_req) { \
	920	+ .subop = (_subop), .flags = cpu_to_be16(_flags), \
	921	+ .id = cpu_to_be32(_id), \
	922	+ }
	923	+
	924	+struct fun_admin_ktls_create_rsp {
	925	+ struct fun_admin_rsp_common common;
	926	+
	927	+ __u8 subop;
	928	+ __u8 rsvd0[3];
	929	+ __be32 id;
	930	+};
	931	+
	932	+struct fun_admin_ktls_modify_req {
	933	+ struct fun_admin_req_common common;
	934	+
	935	+ __u8 subop;
	936	+ __u8 rsvd0;
	937	+ __be16 flags;
	938	+ __be32 id;
	939	+
	940	+ __be64 tlsid;
	941	+
	942	+ __be32 tcp_seq;
	943	+ __u8 version;
	944	+ __u8 cipher;
	945	+ __u8 rsvd1[2];
	946	+
	947	+ __u8 record_seq[8];
	948	+
	949	+ __u8 key[32];
	950	+
	951	+ __u8 iv[16];
	952	+
	953	+ __u8 salt[8];
	954	+};
	955	+
	956	+#define FUN_ADMIN_KTLS_MODIFY_REQ_INIT(_subop, _flags, _id, _tlsid, _tcp_seq, \
	957	+ _version, _cipher) \
	958	+ (struct fun_admin_ktls_modify_req) { \
	959	+ .subop = (_subop), .flags = cpu_to_be16(_flags), \
	960	+ .id = cpu_to_be32(_id), .tlsid = cpu_to_be64(_tlsid), \
	961	+ .tcp_seq = cpu_to_be32(_tcp_seq), .version = _version, \
	962	+ .cipher = _cipher, \
	963	+ }
	964	+
	965	+struct fun_admin_ktls_modify_rsp {
	966	+ struct fun_admin_rsp_common common;
	967	+
	968	+ __u8 subop;
	969	+ __u8 rsvd0[3];
	970	+ __be32 id;
	971	+
	972	+ __be64 tlsid;
	973	+};
	974	+
	975	+struct fun_req_common {
	976	+ __u8 op;
	977	+ __u8 len8;
	978	+ __be16 flags;
	979	+ __u8 suboff8;
	980	+ __u8 rsvd0;
	981	+ __be16 cid;
	982	+};
	983	+
	984	+struct fun_rsp_common {
	985	+ __u8 op;
	986	+ __u8 len8;
	987	+ __be16 flags;
	988	+ __u8 suboff8;
	989	+ __u8 ret;
	990	+ __be16 cid;
	991	+};
	992	+
	993	+struct fun_cqe_info {
	994	+ __be16 sqhd;
	995	+ __be16 sqid;
	996	+ __be16 cid;
	997	+ __be16 sf_p;
	998	+};
	999	+
	1000	+enum fun_eprq_def {
	1001	+ FUN_EPRQ_PKT_ALIGN = 0x80,
	1002	+};
	1003	+
	1004	+struct fun_eprq_rqbuf {
	1005	+ __be64 bufaddr;
	1006	+};
	1007	+
	1008	+#define FUN_EPRQ_RQBUF_INIT(_bufaddr) \
	1009	+ (struct fun_eprq_rqbuf) { \
	1010	+ .bufaddr = cpu_to_be64(_bufaddr), \
	1011	+ }
	1012	+
	1013	+enum fun_eth_op {
	1014	+ FUN_ETH_OP_TX = 0x1,
	1015	+ FUN_ETH_OP_RX = 0x2,
	1016	+};
	1017	+
	1018	+enum {
	1019	+ FUN_ETH_OFFLOAD_EN = 0x8000,
	1020	+ FUN_ETH_OUTER_EN = 0x4000,
	1021	+ FUN_ETH_INNER_LSO = 0x2000,
	1022	+ FUN_ETH_INNER_TSO = 0x1000,
	1023	+ FUN_ETH_OUTER_IPV6 = 0x800,
	1024	+ FUN_ETH_OUTER_UDP = 0x400,
	1025	+ FUN_ETH_INNER_IPV6 = 0x200,
	1026	+ FUN_ETH_INNER_UDP = 0x100,
	1027	+ FUN_ETH_UPDATE_OUTER_L3_LEN = 0x80,
	1028	+ FUN_ETH_UPDATE_OUTER_L3_CKSUM = 0x40,
	1029	+ FUN_ETH_UPDATE_OUTER_L4_LEN = 0x20,
	1030	+ FUN_ETH_UPDATE_OUTER_L4_CKSUM = 0x10,
	1031	+ FUN_ETH_UPDATE_INNER_L3_LEN = 0x8,
	1032	+ FUN_ETH_UPDATE_INNER_L3_CKSUM = 0x4,
	1033	+ FUN_ETH_UPDATE_INNER_L4_LEN = 0x2,
	1034	+ FUN_ETH_UPDATE_INNER_L4_CKSUM = 0x1,
	1035	+};
	1036	+
	1037	+struct fun_eth_offload {
	1038	+ __be16 flags; /* combination of above flags */
	1039	+ __be16 mss; /* TSO max seg size */
	1040	+ __be16 tcp_doff_flags; /* TCP data offset + flags 16b word */
	1041	+ __be16 vlan;
	1042	+
	1043	+ __be16 inner_l3_off; /* Inner L3 header offset */
	1044	+ __be16 inner_l4_off; /* Inner L4 header offset */
	1045	+ __be16 outer_l3_off; /* Outer L3 header offset */
	1046	+ __be16 outer_l4_off; /* Outer L4 header offset */
	1047	+};
	1048	+
	1049	+static inline void fun_eth_offload_init(struct fun_eth_offload *s, u16 flags,
	1050	+ u16 mss, __be16 tcp_doff_flags,
	1051	+ __be16 vlan, u16 inner_l3_off,
	1052	+ u16 inner_l4_off, u16 outer_l3_off,
	1053	+ u16 outer_l4_off)
	1054	+{
	1055	+ s->flags = cpu_to_be16(flags);
	1056	+ s->mss = cpu_to_be16(mss);
	1057	+ s->tcp_doff_flags = tcp_doff_flags;
	1058	+ s->vlan = vlan;
	1059	+ s->inner_l3_off = cpu_to_be16(inner_l3_off);
	1060	+ s->inner_l4_off = cpu_to_be16(inner_l4_off);
	1061	+ s->outer_l3_off = cpu_to_be16(outer_l3_off);
	1062	+ s->outer_l4_off = cpu_to_be16(outer_l4_off);
	1063	+}
	1064	+
	1065	+struct fun_eth_tls {
	1066	+ __be64 tlsid;
	1067	+};
	1068	+
	1069	+enum {
	1070	+ FUN_ETH_TX_TLS = 0x8000,
	1071	+};
	1072	+
	1073	+struct fun_eth_tx_req {
	1074	+ __u8 op;
	1075	+ __u8 len8;
	1076	+ __be16 flags;
	1077	+ __u8 suboff8;
	1078	+ __u8 repr_idn;
	1079	+ __be16 encap_proto;
	1080	+
	1081	+ struct fun_eth_offload offload;
	1082	+
	1083	+ struct fun_dataop_hdr dataop;
	1084	+};
	1085	+
	1086	+struct fun_eth_rx_cv {
	1087	+ __be16 il4_prot_to_l2_type;
	1088	+};
	1089	+
	1090	+#define FUN_ETH_RX_CV_IL4_PROT_S 13U
	1091	+#define FUN_ETH_RX_CV_IL4_PROT_M 0x3
	1092	+
	1093	+#define FUN_ETH_RX_CV_IL3_PROT_S 11U
	1094	+#define FUN_ETH_RX_CV_IL3_PROT_M 0x3
	1095	+
	1096	+#define FUN_ETH_RX_CV_OL4_PROT_S 8U
	1097	+#define FUN_ETH_RX_CV_OL4_PROT_M 0x7
	1098	+
	1099	+#define FUN_ETH_RX_CV_ENCAP_TYPE_S 6U
	1100	+#define FUN_ETH_RX_CV_ENCAP_TYPE_M 0x3
	1101	+
	1102	+#define FUN_ETH_RX_CV_OL3_PROT_S 4U
	1103	+#define FUN_ETH_RX_CV_OL3_PROT_M 0x3
	1104	+
	1105	+#define FUN_ETH_RX_CV_VLAN_TYPE_S 3U
	1106	+#define FUN_ETH_RX_CV_VLAN_TYPE_M 0x1
	1107	+
	1108	+#define FUN_ETH_RX_CV_L2_TYPE_S 2U
	1109	+#define FUN_ETH_RX_CV_L2_TYPE_M 0x1
	1110	+
	1111	+enum fun_rx_cv {
	1112	+ FUN_RX_CV_NONE = 0x0,
	1113	+ FUN_RX_CV_IP = 0x2,
	1114	+ FUN_RX_CV_IP6 = 0x3,
	1115	+ FUN_RX_CV_TCP = 0x2,
	1116	+ FUN_RX_CV_UDP = 0x3,
	1117	+ FUN_RX_CV_VXLAN = 0x2,
	1118	+ FUN_RX_CV_MPLS = 0x3,
	1119	+};
	1120	+
	1121	+struct fun_eth_cqe {
	1122	+ __u8 op;
	1123	+ __u8 len8;
	1124	+ __u8 nsgl;
	1125	+ __u8 repr_idn;
	1126	+ __be32 pkt_len;
	1127	+
	1128	+ __be64 timestamp;
	1129	+
	1130	+ __be16 pkt_cv;
	1131	+ __be16 rsvd0;
	1132	+ __be32 hash;
	1133	+
	1134	+ __be16 encap_proto;
	1135	+ __be16 vlan;
	1136	+ __be32 rsvd1;
	1137	+
	1138	+ __be32 buf_offset;
	1139	+ __be16 headroom;
	1140	+ __be16 csum;
	1141	+};
	1142	+
	1143	+enum fun_admin_adi_attr {
	1144	+ FUN_ADMIN_ADI_ATTR_MACADDR = 0x1,
	1145	+ FUN_ADMIN_ADI_ATTR_VLAN = 0x2,
	1146	+ FUN_ADMIN_ADI_ATTR_RATE = 0x3,
	1147	+};
	1148	+
	1149	+struct fun_adi_param {
	1150	+ union adi_param {
	1151	+ struct fun_adi_mac {
	1152	+ __be64 addr;
	1153	+ } mac;
	1154	+ struct fun_adi_vlan {
	1155	+ __be32 rsvd;
	1156	+ __be16 eth_type;
	1157	+ __be16 tci;
	1158	+ } vlan;
	1159	+ struct fun_adi_rate {
	1160	+ __be32 rsvd;
	1161	+ __be32 tx_mbps;
	1162	+ } rate;
	1163	+ } u;
	1164	+};
	1165	+
	1166	+#define FUN_ADI_MAC_INIT(_addr) \
	1167	+ (struct fun_adi_mac) { \
	1168	+ .addr = cpu_to_be64(_addr), \
	1169	+ }
	1170	+
	1171	+#define FUN_ADI_VLAN_INIT(_eth_type, _tci) \
	1172	+ (struct fun_adi_vlan) { \
	1173	+ .eth_type = cpu_to_be16(_eth_type), .tci = cpu_to_be16(_tci), \
	1174	+ }
	1175	+
	1176	+#define FUN_ADI_RATE_INIT(_tx_mbps) \
	1177	+ (struct fun_adi_rate) { \
	1178	+ .tx_mbps = cpu_to_be32(_tx_mbps), \
	1179	+ }
	1180	+
	1181	+struct fun_admin_adi_req {
	1182	+ struct fun_admin_req_common common;
	1183	+
	1184	+ union adi_req_subop {
	1185	+ struct fun_admin_adi_write_req {
	1186	+ __u8 subop;
	1187	+ __u8 attribute;
	1188	+ __be16 rsvd;
	1189	+ __be32 id;
	1190	+
	1191	+ struct fun_adi_param param;
	1192	+ } write;
	1193	+ } u;
	1194	+};
	1195	+
	1196	+#define FUN_ADMIN_ADI_WRITE_REQ_INIT(_subop, _attribute, _id) \
	1197	+ (struct fun_admin_adi_write_req) { \
	1198	+ .subop = (_subop), .attribute = (_attribute), \
	1199	+ .id = cpu_to_be32(_id), \
	1200	+ }
	1201	+
	1202	+#endif /* __FUN_HCI_H */

drivers/net/ethernet/fungible/funcore/fun_queue.c+601−0

@@ -0,0 +1,601 @@
	1	+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
	2	+
	3	+#include <linux/dma-mapping.h>
	4	+#include <linux/interrupt.h>
	5	+#include <linux/log2.h>
	6	+#include <linux/mm.h>
	7	+#include <linux/netdevice.h>
	8	+#include <linux/pci.h>
	9	+#include <linux/slab.h>
	10	+
	11	+#include "fun_dev.h"
	12	+#include "fun_queue.h"
	13	+
	14	+/* Allocate memory for a queue. This includes the memory for the HW descriptor
	15	+ * ring, an optional 64b HW write-back area, and an optional SW state ring.
	16	+ * Returns the virtual and DMA addresses of the HW ring, the VA of the SW ring,
	17	+ * and the VA of the write-back area.
	18	+ */
	19	+void fun_alloc_ring_mem(struct device dma_dev, size_t depth,
	20	+ size_t hw_desc_sz, size_t sw_desc_sz, bool wb,
	21	+ int numa_node, dma_addr_t dma_addr, void *sw_va,
	22	+ volatile __be64 **wb_va)
	23	+{
	24	+ int dev_node = dev_to_node(dma_dev);
	25	+ size_t dma_sz;
	26	+ void *va;
	27	+
	28	+ if (numa_node == NUMA_NO_NODE)
	29	+ numa_node = dev_node;
	30	+
	31	+ /* Place optional write-back area at end of descriptor ring. */
	32	+ dma_sz = hw_desc_sz * depth;
	33	+ if (wb)
	34	+ dma_sz += sizeof(u64);
	35	+
	36	+ set_dev_node(dma_dev, numa_node);
	37	+ va = dma_alloc_coherent(dma_dev, dma_sz, dma_addr, GFP_KERNEL);
	38	+ set_dev_node(dma_dev, dev_node);
	39	+ if (!va)
	40	+ return NULL;
	41	+
	42	+ if (sw_desc_sz) {
	43	+ sw_va = kvzalloc_node(sw_desc_sz depth, GFP_KERNEL,
	44	+ numa_node);
	45	+ if (!*sw_va) {
	46	+ dma_free_coherent(dma_dev, dma_sz, va, *dma_addr);
	47	+ return NULL;
	48	+ }
	49	+ }
	50	+
	51	+ if (wb)
	52	+ *wb_va = va + dma_sz - sizeof(u64);
	53	+ return va;
	54	+}
	55	+EXPORT_SYMBOL_GPL(fun_alloc_ring_mem);
	56	+
	57	+void fun_free_ring_mem(struct device *dma_dev, size_t depth, size_t hw_desc_sz,
	58	+ bool wb, void hw_va, dma_addr_t dma_addr, void sw_va)
	59	+{
	60	+ if (hw_va) {
	61	+ size_t sz = depth * hw_desc_sz;
	62	+
	63	+ if (wb)
	64	+ sz += sizeof(u64);
	65	+ dma_free_coherent(dma_dev, sz, hw_va, dma_addr);
	66	+ }
	67	+ kvfree(sw_va);
	68	+}
	69	+EXPORT_SYMBOL_GPL(fun_free_ring_mem);
	70	+
	71	+/* Prepare and issue an admin command to create an SQ on the device with the
	72	+ * provided parameters. If the queue ID is auto-allocated by the device it is
	73	+ * returned in *sqidp.
	74	+ */
	75	+int fun_sq_create(struct fun_dev *fdev, u16 flags, u32 sqid, u32 cqid,
	76	+ u8 sqe_size_log2, u32 sq_depth, dma_addr_t dma_addr,
	77	+ u8 coal_nentries, u8 coal_usec, u32 irq_num,
	78	+ u32 scan_start_id, u32 scan_end_id,
	79	+ u32 rq_buf_size_log2, u32 sqidp, u32 __iomem *dbp)
	80	+{
	81	+ union {
	82	+ struct fun_admin_epsq_req req;
	83	+ struct fun_admin_generic_create_rsp rsp;
	84	+ } cmd;
	85	+ dma_addr_t wb_addr;
	86	+ u32 hw_qid;
	87	+ int rc;
	88	+
	89	+ if (sq_depth > fdev->q_depth)
	90	+ return -EINVAL;
	91	+ if (flags & FUN_ADMIN_EPSQ_CREATE_FLAG_RQ)
	92	+ sqe_size_log2 = ilog2(sizeof(struct fun_eprq_rqbuf));
	93	+
	94	+ wb_addr = dma_addr + (sq_depth << sqe_size_log2);
	95	+
	96	+ cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_EPSQ,
	97	+ sizeof(cmd.req));
	98	+ cmd.req.u.create =
	99	+ FUN_ADMIN_EPSQ_CREATE_REQ_INIT(FUN_ADMIN_SUBOP_CREATE, flags,
	100	+ sqid, cqid, sqe_size_log2,
	101	+ sq_depth - 1, dma_addr, 0,
	102	+ coal_nentries, coal_usec,
	103	+ irq_num, scan_start_id,
	104	+ scan_end_id, 0,
	105	+ rq_buf_size_log2,
	106	+ ilog2(sizeof(u64)), wb_addr);
	107	+
	108	+ rc = fun_submit_admin_sync_cmd(fdev, &cmd.req.common,
	109	+ &cmd.rsp, sizeof(cmd.rsp), 0);
	110	+ if (rc)
	111	+ return rc;
	112	+
	113	+ hw_qid = be32_to_cpu(cmd.rsp.id);
	114	+ *dbp = fun_sq_db_addr(fdev, hw_qid);
	115	+ if (flags & FUN_ADMIN_RES_CREATE_FLAG_ALLOCATOR)
	116	+ *sqidp = hw_qid;
	117	+ return rc;
	118	+}
	119	+EXPORT_SYMBOL_GPL(fun_sq_create);
	120	+
	121	+/* Prepare and issue an admin command to create a CQ on the device with the
	122	+ * provided parameters. If the queue ID is auto-allocated by the device it is
	123	+ * returned in *cqidp.
	124	+ */
	125	+int fun_cq_create(struct fun_dev *fdev, u16 flags, u32 cqid, u32 rqid,
	126	+ u8 cqe_size_log2, u32 cq_depth, dma_addr_t dma_addr,
	127	+ u16 headroom, u16 tailroom, u8 coal_nentries, u8 coal_usec,
	128	+ u32 irq_num, u32 scan_start_id, u32 scan_end_id, u32 *cqidp,
	129	+ u32 __iomem **dbp)
	130	+{
	131	+ union {
	132	+ struct fun_admin_epcq_req req;
	133	+ struct fun_admin_generic_create_rsp rsp;
	134	+ } cmd;
	135	+ u32 hw_qid;
	136	+ int rc;
	137	+
	138	+ if (cq_depth > fdev->q_depth)
	139	+ return -EINVAL;
	140	+
	141	+ cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_EPCQ,
	142	+ sizeof(cmd.req));
	143	+ cmd.req.u.create =
	144	+ FUN_ADMIN_EPCQ_CREATE_REQ_INIT(FUN_ADMIN_SUBOP_CREATE, flags,
	145	+ cqid, rqid, cqe_size_log2,
	146	+ cq_depth - 1, dma_addr, tailroom,
	147	+ headroom / 2, 0, coal_nentries,
	148	+ coal_usec, irq_num,
	149	+ scan_start_id, scan_end_id, 0);
	150	+
	151	+ rc = fun_submit_admin_sync_cmd(fdev, &cmd.req.common,
	152	+ &cmd.rsp, sizeof(cmd.rsp), 0);
	153	+ if (rc)
	154	+ return rc;
	155	+
	156	+ hw_qid = be32_to_cpu(cmd.rsp.id);
	157	+ *dbp = fun_cq_db_addr(fdev, hw_qid);
	158	+ if (flags & FUN_ADMIN_RES_CREATE_FLAG_ALLOCATOR)
	159	+ *cqidp = hw_qid;
	160	+ return rc;
	161	+}
	162	+EXPORT_SYMBOL_GPL(fun_cq_create);
	163	+
	164	+static bool fun_sq_is_head_wb(const struct fun_queue *funq)
	165	+{
	166	+ return funq->sq_flags & FUN_ADMIN_EPSQ_CREATE_FLAG_HEAD_WB_ADDRESS;
	167	+}
	168	+
	169	+static void fun_clean_rq(struct fun_queue *funq)
	170	+{
	171	+ struct fun_dev *fdev = funq->fdev;
	172	+ struct fun_rq_info *rqinfo;
	173	+ unsigned int i;
	174	+
	175	+ for (i = 0; i < funq->rq_depth; i++) {
	176	+ rqinfo = &funq->rq_info[i];
	177	+ if (rqinfo->page) {
	178	+ dma_unmap_page(fdev->dev, rqinfo->dma, PAGE_SIZE,
	179	+ DMA_FROM_DEVICE);
	180	+ put_page(rqinfo->page);
	181	+ rqinfo->page = NULL;
	182	+ }
	183	+ }
	184	+}
	185	+
	186	+static int fun_fill_rq(struct fun_queue *funq)
	187	+{
	188	+ struct device *dev = funq->fdev->dev;
	189	+ int i, node = dev_to_node(dev);
	190	+ struct fun_rq_info *rqinfo;
	191	+
	192	+ for (i = 0; i < funq->rq_depth; i++) {
	193	+ rqinfo = &funq->rq_info[i];
	194	+ rqinfo->page = alloc_pages_node(node, GFP_KERNEL, 0);
	195	+ if (unlikely(!rqinfo->page))
	196	+ return -ENOMEM;
	197	+
	198	+ rqinfo->dma = dma_map_page(dev, rqinfo->page, 0,
	199	+ PAGE_SIZE, DMA_FROM_DEVICE);
	200	+ if (unlikely(dma_mapping_error(dev, rqinfo->dma))) {
	201	+ put_page(rqinfo->page);
	202	+ rqinfo->page = NULL;
	203	+ return -ENOMEM;
	204	+ }
	205	+
	206	+ funq->rqes[i] = FUN_EPRQ_RQBUF_INIT(rqinfo->dma);
	207	+ }
	208	+
	209	+ funq->rq_tail = funq->rq_depth - 1;
	210	+ return 0;
	211	+}
	212	+
	213	+static void fun_rq_update_pos(struct fun_queue *funq, int buf_offset)
	214	+{
	215	+ if (buf_offset <= funq->rq_buf_offset) {
	216	+ struct fun_rq_info *rqinfo = &funq->rq_info[funq->rq_buf_idx];
	217	+ struct device *dev = funq->fdev->dev;
	218	+
	219	+ dma_sync_single_for_device(dev, rqinfo->dma, PAGE_SIZE,
	220	+ DMA_FROM_DEVICE);
	221	+ funq->num_rqe_to_fill++;
	222	+ if (++funq->rq_buf_idx == funq->rq_depth)
	223	+ funq->rq_buf_idx = 0;
	224	+ }
	225	+ funq->rq_buf_offset = buf_offset;
	226	+}
	227	+
	228	+/* Given a command response with data scattered across >= 1 RQ buffers return
	229	+ * a pointer to a contiguous buffer containing all the data. If the data is in
	230	+ * one RQ buffer the start address within that buffer is returned, otherwise a
	231	+ * new buffer is allocated and the data is gathered into it.
	232	+ */
	233	+static void fun_data_from_rq(struct fun_queue funq,
	234	+ const struct fun_rsp_common rsp, bool need_free)
	235	+{
	236	+ u32 bufoff, total_len, remaining, fragsize, dataoff;
	237	+ struct device *dma_dev = funq->fdev->dev;
	238	+ const struct fun_dataop_rqbuf *databuf;
	239	+ const struct fun_dataop_hdr *dataop;
	240	+ const struct fun_rq_info *rqinfo;
	241	+ void *data;
	242	+
	243	+ dataop = (void )rsp + rsp->suboff8 8;
	244	+ total_len = be32_to_cpu(dataop->total_len);
	245	+
	246	+ if (likely(dataop->nsgl == 1)) {
	247	+ databuf = (struct fun_dataop_rqbuf *)dataop->imm;
	248	+ bufoff = be32_to_cpu(databuf->bufoff);
	249	+ fun_rq_update_pos(funq, bufoff);
	250	+ rqinfo = &funq->rq_info[funq->rq_buf_idx];
	251	+ dma_sync_single_for_cpu(dma_dev, rqinfo->dma + bufoff,
	252	+ total_len, DMA_FROM_DEVICE);
	253	+ *need_free = false;
	254	+ return page_address(rqinfo->page) + bufoff;
	255	+ }
	256	+
	257	+ /* For scattered completions gather the fragments into one buffer. */
	258	+
	259	+ data = kmalloc(total_len, GFP_ATOMIC);
	260	+ /* NULL is OK here. In case of failure we still need to consume the data
	261	+ * for proper buffer accounting but indicate an error in the response.
	262	+ */
	263	+ if (likely(data))
	264	+ *need_free = true;
	265	+
	266	+ dataoff = 0;
	267	+ for (remaining = total_len; remaining; remaining -= fragsize) {
	268	+ fun_rq_update_pos(funq, 0);
	269	+ fragsize = min_t(unsigned int, PAGE_SIZE, remaining);
	270	+ if (data) {
	271	+ rqinfo = &funq->rq_info[funq->rq_buf_idx];
	272	+ dma_sync_single_for_cpu(dma_dev, rqinfo->dma, fragsize,
	273	+ DMA_FROM_DEVICE);
	274	+ memcpy(data + dataoff, page_address(rqinfo->page),
	275	+ fragsize);
	276	+ dataoff += fragsize;
	277	+ }
	278	+ }
	279	+ return data;
	280	+}
	281	+
	282	+unsigned int __fun_process_cq(struct fun_queue *funq, unsigned int max)
	283	+{
	284	+ const struct fun_cqe_info *info;
	285	+ struct fun_rsp_common *rsp;
	286	+ unsigned int new_cqes;
	287	+ u16 sf_p, flags;
	288	+ bool need_free;
	289	+ void *cqe;
	290	+
	291	+ if (!max)
	292	+ max = funq->cq_depth - 1;
	293	+
	294	+ for (new_cqes = 0; new_cqes < max; new_cqes++) {
	295	+ cqe = funq->cqes + (funq->cq_head << funq->cqe_size_log2);
	296	+ info = funq_cqe_info(funq, cqe);
	297	+ sf_p = be16_to_cpu(info->sf_p);
	298	+
	299	+ if ((sf_p & 1) != funq->cq_phase)
	300	+ break;
	301	+
	302	+ /* ensure the phase tag is read before other CQE fields */
	303	+ dma_rmb();
	304	+
	305	+ if (++funq->cq_head == funq->cq_depth) {
	306	+ funq->cq_head = 0;
	307	+ funq->cq_phase = !funq->cq_phase;
	308	+ }
	309	+
	310	+ rsp = cqe;
	311	+ flags = be16_to_cpu(rsp->flags);
	312	+
	313	+ need_free = false;
	314	+ if (unlikely(flags & FUN_REQ_COMMON_FLAG_CQE_IN_RQBUF)) {
	315	+ rsp = fun_data_from_rq(funq, rsp, &need_free);
	316	+ if (!rsp) {
	317	+ rsp = cqe;
	318	+ rsp->len8 = 1;
	319	+ if (rsp->ret == 0)
	320	+ rsp->ret = ENOMEM;
	321	+ }
	322	+ }
	323	+
	324	+ if (funq->cq_cb)
	325	+ funq->cq_cb(funq, funq->cb_data, rsp, info);
	326	+ if (need_free)
	327	+ kfree(rsp);
	328	+ }
	329	+
	330	+ dev_dbg(funq->fdev->dev, "CQ %u, new CQEs %u/%u, head %u, phase %u\n",
	331	+ funq->cqid, new_cqes, max, funq->cq_head, funq->cq_phase);
	332	+ return new_cqes;
	333	+}
	334	+
	335	+unsigned int fun_process_cq(struct fun_queue *funq, unsigned int max)
	336	+{
	337	+ unsigned int processed;
	338	+ u32 db;
	339	+
	340	+ processed = __fun_process_cq(funq, max);
	341	+
	342	+ if (funq->num_rqe_to_fill) {
	343	+ funq->rq_tail = (funq->rq_tail + funq->num_rqe_to_fill) %
	344	+ funq->rq_depth;
	345	+ funq->num_rqe_to_fill = 0;
	346	+ writel(funq->rq_tail, funq->rq_db);
	347	+ }
	348	+
	349	+ db = funq->cq_head \| FUN_DB_IRQ_ARM_F;
	350	+ writel(db, funq->cq_db);
	351	+ return processed;
	352	+}
	353	+
	354	+static int fun_alloc_sqes(struct fun_queue *funq)
	355	+{
	356	+ funq->sq_cmds = fun_alloc_ring_mem(funq->fdev->dev, funq->sq_depth,
	357	+ 1 << funq->sqe_size_log2, 0,
	358	+ fun_sq_is_head_wb(funq),
	359	+ NUMA_NO_NODE, &funq->sq_dma_addr,
	360	+ NULL, &funq->sq_head);
	361	+ return funq->sq_cmds ? 0 : -ENOMEM;
	362	+}
	363	+
	364	+static int fun_alloc_cqes(struct fun_queue *funq)
	365	+{
	366	+ funq->cqes = fun_alloc_ring_mem(funq->fdev->dev, funq->cq_depth,
	367	+ 1 << funq->cqe_size_log2, 0, false,
	368	+ NUMA_NO_NODE, &funq->cq_dma_addr, NULL,
	369	+ NULL);
	370	+ return funq->cqes ? 0 : -ENOMEM;
	371	+}
	372	+
	373	+static int fun_alloc_rqes(struct fun_queue *funq)
	374	+{
	375	+ funq->rqes = fun_alloc_ring_mem(funq->fdev->dev, funq->rq_depth,
	376	+ sizeof(*funq->rqes),
	377	+ sizeof(*funq->rq_info), false,
	378	+ NUMA_NO_NODE, &funq->rq_dma_addr,
	379	+ (void **)&funq->rq_info, NULL);
	380	+ return funq->rqes ? 0 : -ENOMEM;
	381	+}
	382	+
	383	+/* Free a queue's structures. */
	384	+void fun_free_queue(struct fun_queue *funq)
	385	+{
	386	+ struct device *dev = funq->fdev->dev;
	387	+
	388	+ fun_free_ring_mem(dev, funq->cq_depth, 1 << funq->cqe_size_log2, false,
	389	+ funq->cqes, funq->cq_dma_addr, NULL);
	390	+ fun_free_ring_mem(dev, funq->sq_depth, 1 << funq->sqe_size_log2,
	391	+ fun_sq_is_head_wb(funq), funq->sq_cmds,
	392	+ funq->sq_dma_addr, NULL);
	393	+
	394	+ if (funq->rqes) {
	395	+ fun_clean_rq(funq);
	396	+ fun_free_ring_mem(dev, funq->rq_depth, sizeof(*funq->rqes),
	397	+ false, funq->rqes, funq->rq_dma_addr,
	398	+ funq->rq_info);
	399	+ }
	400	+
	401	+ kfree(funq);
	402	+}
	403	+
	404	+/* Allocate and initialize a funq's structures. */
	405	+struct fun_queue fun_alloc_queue(struct fun_dev fdev, int qid,
	406	+ const struct fun_queue_alloc_req *req)
	407	+{
	408	+ struct fun_queue funq = kzalloc(sizeof(funq), GFP_KERNEL);
	409	+
	410	+ if (!funq)
	411	+ return NULL;
	412	+
	413	+ funq->fdev = fdev;
	414	+ spin_lock_init(&funq->sq_lock);
	415	+
	416	+ funq->qid = qid;
	417	+
	418	+ /* Initial CQ/SQ/RQ ids */
	419	+ if (req->rq_depth) {
	420	+ funq->cqid = 2 * qid;
	421	+ if (funq->qid) {
	422	+ /* I/O Q: use rqid = cqid, sqid = +1 */
	423	+ funq->rqid = funq->cqid;
	424	+ funq->sqid = funq->rqid + 1;
	425	+ } else {
	426	+ /* Admin Q: sqid is always 0, use ID 1 for RQ */
	427	+ funq->sqid = 0;
	428	+ funq->rqid = 1;
	429	+ }
	430	+ } else {
	431	+ funq->cqid = qid;
	432	+ funq->sqid = qid;
	433	+ }
	434	+
	435	+ funq->cq_flags = req->cq_flags;
	436	+ funq->sq_flags = req->sq_flags;
	437	+
	438	+ funq->cqe_size_log2 = req->cqe_size_log2;
	439	+ funq->sqe_size_log2 = req->sqe_size_log2;
	440	+
	441	+ funq->cq_depth = req->cq_depth;
	442	+ funq->sq_depth = req->sq_depth;
	443	+
	444	+ funq->cq_intcoal_nentries = req->cq_intcoal_nentries;
	445	+ funq->cq_intcoal_usec = req->cq_intcoal_usec;
	446	+
	447	+ funq->sq_intcoal_nentries = req->sq_intcoal_nentries;
	448	+ funq->sq_intcoal_usec = req->sq_intcoal_usec;
	449	+
	450	+ if (fun_alloc_cqes(funq))
	451	+ goto free_funq;
	452	+
	453	+ funq->cq_phase = 1;
	454	+
	455	+ if (fun_alloc_sqes(funq))
	456	+ goto free_funq;
	457	+
	458	+ if (req->rq_depth) {
	459	+ funq->rq_flags = req->rq_flags \| FUN_ADMIN_EPSQ_CREATE_FLAG_RQ;
	460	+ funq->rq_depth = req->rq_depth;
	461	+ funq->rq_buf_offset = -1;
	462	+
	463	+ if (fun_alloc_rqes(funq) \|\| fun_fill_rq(funq))
	464	+ goto free_funq;
	465	+ }
	466	+
	467	+ funq->cq_vector = -1;
	468	+ funq->cqe_info_offset = (1 << funq->cqe_size_log2) - sizeof(struct fun_cqe_info);
	469	+
	470	+ /* SQ/CQ 0 are implicitly created, assign their doorbells now.
	471	+ * Other queues are assigned doorbells at their explicit creation.
	472	+ */
	473	+ if (funq->sqid == 0)
	474	+ funq->sq_db = fun_sq_db_addr(fdev, 0);
	475	+ if (funq->cqid == 0)
	476	+ funq->cq_db = fun_cq_db_addr(fdev, 0);
	477	+
	478	+ return funq;
	479	+
	480	+free_funq:
	481	+ fun_free_queue(funq);
	482	+ return NULL;
	483	+}
	484	+
	485	+/* Create a funq's CQ on the device. */
	486	+static int fun_create_cq(struct fun_queue *funq)
	487	+{
	488	+ struct fun_dev *fdev = funq->fdev;
	489	+ unsigned int rqid;
	490	+ int rc;
	491	+
	492	+ rqid = funq->cq_flags & FUN_ADMIN_EPCQ_CREATE_FLAG_RQ ?
	493	+ funq->rqid : FUN_HCI_ID_INVALID;
	494	+ rc = fun_cq_create(fdev, funq->cq_flags, funq->cqid, rqid,
	495	+ funq->cqe_size_log2, funq->cq_depth,
	496	+ funq->cq_dma_addr, 0, 0, funq->cq_intcoal_nentries,
	497	+ funq->cq_intcoal_usec, funq->cq_vector, 0, 0,
	498	+ &funq->cqid, &funq->cq_db);
	499	+ if (!rc)
	500	+ dev_dbg(fdev->dev, "created CQ %u\n", funq->cqid);
	501	+
	502	+ return rc;
	503	+}
	504	+
	505	+/* Create a funq's SQ on the device. */
	506	+static int fun_create_sq(struct fun_queue *funq)
	507	+{
	508	+ struct fun_dev *fdev = funq->fdev;
	509	+ int rc;
	510	+
	511	+ rc = fun_sq_create(fdev, funq->sq_flags, funq->sqid, funq->cqid,
	512	+ funq->sqe_size_log2, funq->sq_depth,
	513	+ funq->sq_dma_addr, funq->sq_intcoal_nentries,
	514	+ funq->sq_intcoal_usec, funq->cq_vector, 0, 0,
	515	+ 0, &funq->sqid, &funq->sq_db);
	516	+ if (!rc)
	517	+ dev_dbg(fdev->dev, "created SQ %u\n", funq->sqid);
	518	+
	519	+ return rc;
	520	+}
	521	+
	522	+/* Create a funq's RQ on the device. */
	523	+int fun_create_rq(struct fun_queue *funq)
	524	+{
	525	+ struct fun_dev *fdev = funq->fdev;
	526	+ int rc;
	527	+
	528	+ rc = fun_sq_create(fdev, funq->rq_flags, funq->rqid, funq->cqid, 0,
	529	+ funq->rq_depth, funq->rq_dma_addr, 0, 0,
	530	+ funq->cq_vector, 0, 0, PAGE_SHIFT, &funq->rqid,
	531	+ &funq->rq_db);
	532	+ if (!rc)
	533	+ dev_dbg(fdev->dev, "created RQ %u\n", funq->rqid);
	534	+
	535	+ return rc;
	536	+}
	537	+
	538	+static unsigned int funq_irq(struct fun_queue *funq)
	539	+{
	540	+ return pci_irq_vector(to_pci_dev(funq->fdev->dev), funq->cq_vector);
	541	+}
	542	+
	543	+int fun_request_irq(struct fun_queue funq, const char devname,
	544	+ irq_handler_t handler, void *data)
	545	+{
	546	+ int rc;
	547	+
	548	+ if (funq->cq_vector < 0)
	549	+ return -EINVAL;
	550	+
	551	+ funq->irq_handler = handler;
	552	+ funq->irq_data = data;
	553	+
	554	+ snprintf(funq->irqname, sizeof(funq->irqname),
	555	+ funq->qid ? "%s-q[%d]" : "%s-adminq", devname, funq->qid);
	556	+
	557	+ rc = request_irq(funq_irq(funq), handler, 0, funq->irqname, data);
	558	+ if (rc)
	559	+ funq->irq_handler = NULL;
	560	+
	561	+ return rc;
	562	+}
	563	+
	564	+/* Create all component queues of a funq on the device. */
	565	+int fun_create_queue(struct fun_queue *funq)
	566	+{
	567	+ int rc;
	568	+
	569	+ rc = fun_create_cq(funq);
	570	+ if (rc)
	571	+ return rc;
	572	+
	573	+ if (funq->rq_depth) {
	574	+ rc = fun_create_rq(funq);
	575	+ if (rc)
	576	+ goto release_cq;
	577	+ }
	578	+
	579	+ rc = fun_create_sq(funq);
	580	+ if (rc)
	581	+ goto release_rq;
	582	+
	583	+ return 0;
	584	+
	585	+release_rq:
	586	+ fun_destroy_sq(funq->fdev, funq->rqid);
	587	+release_cq:
	588	+ fun_destroy_cq(funq->fdev, funq->cqid);
	589	+ return rc;
	590	+}
	591	+
	592	+void fun_free_irq(struct fun_queue *funq)
	593	+{
	594	+ if (funq->irq_handler) {
	595	+ unsigned int vector = funq_irq(funq);
	596	+
	597	+ free_irq(vector, funq->irq_data);
	598	+ funq->irq_handler = NULL;
	599	+ funq->irq_data = NULL;
	600	+ }
	601	+}

drivers/net/ethernet/fungible/funcore/fun_queue.h+175−0

@@ -0,0 +1,175 @@
	1	+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) */
	2	+
	3	+#ifndef _FUN_QEUEUE_H
	4	+#define _FUN_QEUEUE_H
	5	+
	6	+#include <linux/interrupt.h>
	7	+#include <linux/io.h>
	8	+
	9	+struct device;
	10	+struct fun_dev;
	11	+struct fun_queue;
	12	+struct fun_cqe_info;
	13	+struct fun_rsp_common;
	14	+
	15	+typedef void (cq_callback_t)(struct fun_queue funq, void data, void msg,
	16	+ const struct fun_cqe_info *info);
	17	+
	18	+struct fun_rq_info {
	19	+ dma_addr_t dma;
	20	+ struct page *page;
	21	+};
	22	+
	23	+/* A queue group consisting of an SQ, a CQ, and an optional RQ. */
	24	+struct fun_queue {
	25	+ struct fun_dev *fdev;
	26	+ spinlock_t sq_lock;
	27	+
	28	+ dma_addr_t cq_dma_addr;
	29	+ dma_addr_t sq_dma_addr;
	30	+ dma_addr_t rq_dma_addr;
	31	+
	32	+ u32 __iomem *cq_db;
	33	+ u32 __iomem *sq_db;
	34	+ u32 __iomem *rq_db;
	35	+
	36	+ void *cqes;
	37	+ void *sq_cmds;
	38	+ struct fun_eprq_rqbuf *rqes;
	39	+ struct fun_rq_info *rq_info;
	40	+
	41	+ u32 cqid;
	42	+ u32 sqid;
	43	+ u32 rqid;
	44	+
	45	+ u32 cq_depth;
	46	+ u32 sq_depth;
	47	+ u32 rq_depth;
	48	+
	49	+ u16 cq_head;
	50	+ u16 sq_tail;
	51	+ u16 rq_tail;
	52	+
	53	+ u8 cqe_size_log2;
	54	+ u8 sqe_size_log2;
	55	+
	56	+ u16 cqe_info_offset;
	57	+
	58	+ u16 rq_buf_idx;
	59	+ int rq_buf_offset;
	60	+ u16 num_rqe_to_fill;
	61	+
	62	+ u8 cq_intcoal_usec;
	63	+ u8 cq_intcoal_nentries;
	64	+ u8 sq_intcoal_usec;
	65	+ u8 sq_intcoal_nentries;
	66	+
	67	+ u16 cq_flags;
	68	+ u16 sq_flags;
	69	+ u16 rq_flags;
	70	+
	71	+ /* SQ head writeback */
	72	+ u16 sq_comp;
	73	+
	74	+ volatile __be64 *sq_head;
	75	+
	76	+ cq_callback_t cq_cb;
	77	+ void *cb_data;
	78	+
	79	+ irq_handler_t irq_handler;
	80	+ void *irq_data;
	81	+ s16 cq_vector;
	82	+ u8 cq_phase;
	83	+
	84	+ /* I/O q index */
	85	+ u16 qid;
	86	+
	87	+ char irqname[24];
	88	+};
	89	+
	90	+static inline void fun_sqe_at(const struct fun_queue funq, unsigned int pos)
	91	+{
	92	+ return funq->sq_cmds + (pos << funq->sqe_size_log2);
	93	+}
	94	+
	95	+static inline void funq_sq_post_tail(struct fun_queue *funq, u16 tail)
	96	+{
	97	+ if (++tail == funq->sq_depth)
	98	+ tail = 0;
	99	+ funq->sq_tail = tail;
	100	+ writel(tail, funq->sq_db);
	101	+}
	102	+
	103	+static inline struct fun_cqe_info funq_cqe_info(const struct fun_queue funq,
	104	+ void *cqe)
	105	+{
	106	+ return cqe + funq->cqe_info_offset;
	107	+}
	108	+
	109	+static inline void funq_rq_post(struct fun_queue *funq)
	110	+{
	111	+ writel(funq->rq_tail, funq->rq_db);
	112	+}
	113	+
	114	+struct fun_queue_alloc_req {
	115	+ u8 cqe_size_log2;
	116	+ u8 sqe_size_log2;
	117	+
	118	+ u16 cq_flags;
	119	+ u16 sq_flags;
	120	+ u16 rq_flags;
	121	+
	122	+ u32 cq_depth;
	123	+ u32 sq_depth;
	124	+ u32 rq_depth;
	125	+
	126	+ u8 cq_intcoal_usec;
	127	+ u8 cq_intcoal_nentries;
	128	+ u8 sq_intcoal_usec;
	129	+ u8 sq_intcoal_nentries;
	130	+};
	131	+
	132	+int fun_sq_create(struct fun_dev *fdev, u16 flags, u32 sqid, u32 cqid,
	133	+ u8 sqe_size_log2, u32 sq_depth, dma_addr_t dma_addr,
	134	+ u8 coal_nentries, u8 coal_usec, u32 irq_num,
	135	+ u32 scan_start_id, u32 scan_end_id,
	136	+ u32 rq_buf_size_log2, u32 sqidp, u32 __iomem *dbp);
	137	+int fun_cq_create(struct fun_dev *fdev, u16 flags, u32 cqid, u32 rqid,
	138	+ u8 cqe_size_log2, u32 cq_depth, dma_addr_t dma_addr,
	139	+ u16 headroom, u16 tailroom, u8 coal_nentries, u8 coal_usec,
	140	+ u32 irq_num, u32 scan_start_id, u32 scan_end_id,
	141	+ u32 cqidp, u32 __iomem *dbp);
	142	+void fun_alloc_ring_mem(struct device dma_dev, size_t depth,
	143	+ size_t hw_desc_sz, size_t sw_desc_size, bool wb,
	144	+ int numa_node, dma_addr_t dma_addr, void *sw_va,
	145	+ volatile __be64 **wb_va);
	146	+void fun_free_ring_mem(struct device *dma_dev, size_t depth, size_t hw_desc_sz,
	147	+ bool wb, void hw_va, dma_addr_t dma_addr, void sw_va);
	148	+
	149	+#define fun_destroy_sq(fdev, sqid) \
	150	+ fun_res_destroy((fdev), FUN_ADMIN_OP_EPSQ, 0, (sqid))
	151	+#define fun_destroy_cq(fdev, cqid) \
	152	+ fun_res_destroy((fdev), FUN_ADMIN_OP_EPCQ, 0, (cqid))
	153	+
	154	+struct fun_queue fun_alloc_queue(struct fun_dev fdev, int qid,
	155	+ const struct fun_queue_alloc_req *req);
	156	+void fun_free_queue(struct fun_queue *funq);
	157	+
	158	+static inline void fun_set_cq_callback(struct fun_queue *funq, cq_callback_t cb,
	159	+ void *cb_data)
	160	+{
	161	+ funq->cq_cb = cb;
	162	+ funq->cb_data = cb_data;
	163	+}
	164	+
	165	+int fun_create_rq(struct fun_queue *funq);
	166	+int fun_create_queue(struct fun_queue *funq);
	167	+
	168	+void fun_free_irq(struct fun_queue *funq);
	169	+int fun_request_irq(struct fun_queue funq, const char devname,
	170	+ irq_handler_t handler, void *data);
	171	+
	172	+unsigned int __fun_process_cq(struct fun_queue *funq, unsigned int max);
	173	+unsigned int fun_process_cq(struct fun_queue *funq, unsigned int max);
	174	+
	175	+#endif /* _FUN_QEUEUE_H */