net/fungible: Add service module for Fungible drivers

Fungible cards have a number of different PCI functions and thus different drivers, all of which use a common method to initialize and interact with the device. This commit adds a library module that collects these common mechanisms. They mainly deal with device initialization, setting up and destroying queues, and operating an admin queue. A subset of the FW interface is also included here.

Dimitris Michailidis committed 4 years ago

Commit e1ffcc61 parent e8eb9e36 files · +2976 −0Browse files

Unified · Split

6 files

drivers/net/ethernet/fungible/funcore/Makefile +5 −0
drivers/net/ethernet/fungible/funcore/fun_dev.c +843 −0
drivers/net/ethernet/fungible/funcore/fun_dev.h +150 −0
drivers/net/ethernet/fungible/funcore/fun_hci.h +1202 −0
drivers/net/ethernet/fungible/funcore/fun_queue.c +601 −0
drivers/net/ethernet/fungible/funcore/fun_queue.h +175 −0

drivers/net/ethernet/fungible/funcore/Makefile+5−0

@@ -0,0 +1,5 @@
	1	# SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
	2
	3	obj-$(CONFIG_FUN_CORE) += funcore.o
	4
	5	funcore-y := fun_dev.o fun_queue.o

drivers/net/ethernet/fungible/funcore/fun_dev.c+843−0

@@ -0,0 +1,843 @@
	1	// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
	2
	3	#include <linux/aer.h>
	4	#include <linux/bitmap.h>
	5	#include <linux/delay.h>
	6	#include <linux/interrupt.h>
	7	#include <linux/io.h>
	8	#include <linux/io-64-nonatomic-lo-hi.h>
	9	#include <linux/mm.h>
	10	#include <linux/module.h>
	11	#include <linux/nvme.h>
	12	#include <linux/pci.h>
	13	#include <linux/wait.h>
	14	#include <linux/sched/signal.h>
	15
	16	#include "fun_queue.h"
	17	#include "fun_dev.h"
	18
	19	#define FUN_ADMIN_CMD_TO_MS 3000
	20
	21	enum {
	22	AQA_ASQS_SHIFT = 0,
	23	AQA_ACQS_SHIFT = 16,
	24	AQA_MIN_QUEUE_SIZE = 2,
	25	AQA_MAX_QUEUE_SIZE = 4096
	26	};
	27
	28	/* context for admin commands */
	29	struct fun_cmd_ctx {
	30	fun_admin_callback_t cb; /* callback to invoke on completion */
	31	void cb_data; / user data provided to callback */
	32	int cpu; /* CPU where the cmd's tag was allocated */
	33	};
	34
	35	/* Context for synchronous admin commands. */
	36	struct fun_sync_cmd_ctx {
	37	struct completion compl;
	38	u8 rsp_buf; / caller provided response buffer */
	39	unsigned int rsp_len; /* response buffer size */
	40	u8 rsp_status; /* command response status */
	41	};
	42
	43	/* Wait for the CSTS.RDY bit to match @enabled. */
	44	static int fun_wait_ready(struct fun_dev *fdev, bool enabled)
	45	{
	46	unsigned int cap_to = NVME_CAP_TIMEOUT(fdev->cap_reg);
	47	u32 bit = enabled ? NVME_CSTS_RDY : 0;
	48	unsigned long deadline;
	49
	50	deadline = ((cap_to + 1) * HZ / 2) + jiffies; /* CAP.TO is in 500ms */
	51
	52	for (;;) {
	53	u32 csts = readl(fdev->bar + NVME_REG_CSTS);
	54
	55	if (csts == ~0) {
	56	dev_err(fdev->dev, "CSTS register read %#x\n", csts);
	57	return -EIO;
	58	}
	59
	60	if ((csts & NVME_CSTS_RDY) == bit)
	61	return 0;
	62
	63	if (time_is_before_jiffies(deadline))
	64	break;
	65
	66	msleep(100);
	67	}
	68
	69	dev_err(fdev->dev,
	70	"Timed out waiting for device to indicate RDY %u; aborting %s\n",
	71	enabled, enabled ? "initialization" : "reset");
	72	return -ETIMEDOUT;
	73	}
	74
	75	/* Check CSTS and return an error if it is unreadable or has unexpected
	76	* RDY value.
	77	*/
	78	static int fun_check_csts_rdy(struct fun_dev *fdev, unsigned int expected_rdy)
	79	{
	80	u32 csts = readl(fdev->bar + NVME_REG_CSTS);
	81	u32 actual_rdy = csts & NVME_CSTS_RDY;
	82
	83	if (csts == ~0) {
	84	dev_err(fdev->dev, "CSTS register read %#x\n", csts);
	85	return -EIO;
	86	}
	87	if (actual_rdy != expected_rdy) {
	88	dev_err(fdev->dev, "Unexpected CSTS RDY %u\n", actual_rdy);
	89	return -EINVAL;
	90	}
	91	return 0;
	92	}
	93
	94	/* Check that CSTS RDY has the expected value. Then write a new value to the CC
	95	* register and wait for CSTS RDY to match the new CC ENABLE state.
	96	*/
	97	static int fun_update_cc_enable(struct fun_dev *fdev, unsigned int initial_rdy)
	98	{
	99	int rc = fun_check_csts_rdy(fdev, initial_rdy);
	100
	101	if (rc)
	102	return rc;
	103	writel(fdev->cc_reg, fdev->bar + NVME_REG_CC);
	104	return fun_wait_ready(fdev, !!(fdev->cc_reg & NVME_CC_ENABLE));
	105	}
	106
	107	static int fun_disable_ctrl(struct fun_dev *fdev)
	108	{
	109	fdev->cc_reg &= ~(NVME_CC_SHN_MASK \| NVME_CC_ENABLE);
	110	return fun_update_cc_enable(fdev, 1);
	111	}
	112
	113	static int fun_enable_ctrl(struct fun_dev *fdev, u32 admin_cqesz_log2,
	114	u32 admin_sqesz_log2)
	115	{
	116	fdev->cc_reg = (admin_cqesz_log2 << NVME_CC_IOCQES_SHIFT) \|
	117	(admin_sqesz_log2 << NVME_CC_IOSQES_SHIFT) \|
	118	((PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT) \|
	119	NVME_CC_ENABLE;
	120
	121	return fun_update_cc_enable(fdev, 0);
	122	}
	123
	124	static int fun_map_bars(struct fun_dev fdev, const char name)
	125	{
	126	struct pci_dev *pdev = to_pci_dev(fdev->dev);
	127	int err;
	128
	129	err = pci_request_mem_regions(pdev, name);
	130	if (err) {
	131	dev_err(&pdev->dev,
	132	"Couldn't get PCI memory resources, err %d\n", err);
	133	return err;
	134	}
	135
	136	fdev->bar = pci_ioremap_bar(pdev, 0);
	137	if (!fdev->bar) {
	138	dev_err(&pdev->dev, "Couldn't map BAR 0\n");
	139	pci_release_mem_regions(pdev);
	140	return -ENOMEM;
	141	}
	142
	143	return 0;
	144	}
	145
	146	static void fun_unmap_bars(struct fun_dev *fdev)
	147	{
	148	struct pci_dev *pdev = to_pci_dev(fdev->dev);
	149
	150	if (fdev->bar) {
	151	iounmap(fdev->bar);
	152	fdev->bar = NULL;
	153	pci_release_mem_regions(pdev);
	154	}
	155	}
	156
	157	static int fun_set_dma_masks(struct device *dev)
	158	{
	159	int err;
	160
	161	err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
	162	if (err)
	163	dev_err(dev, "DMA mask configuration failed, err %d\n", err);
	164	return err;
	165	}
	166
	167	static irqreturn_t fun_admin_irq(int irq, void *data)
	168	{
	169	struct fun_queue *funq = data;
	170
	171	return fun_process_cq(funq, 0) ? IRQ_HANDLED : IRQ_NONE;
	172	}
	173
	174	static void fun_complete_admin_cmd(struct fun_queue funq, void data,
	175	void entry, const struct fun_cqe_info info)
	176	{
	177	const struct fun_admin_rsp_common *rsp_common = entry;
	178	struct fun_dev *fdev = funq->fdev;
	179	struct fun_cmd_ctx *cmd_ctx;
	180	int cpu;
	181	u16 cid;
	182
	183	if (info->sqhd == cpu_to_be16(0xffff)) {
	184	dev_dbg(fdev->dev, "adminq event");
	185	if (fdev->adminq_cb)
	186	fdev->adminq_cb(fdev, entry);
	187	return;
	188	}
	189
	190	cid = be16_to_cpu(rsp_common->cid);
	191	dev_dbg(fdev->dev, "admin CQE cid %u, op %u, ret %u\n", cid,
	192	rsp_common->op, rsp_common->ret);
	193
	194	cmd_ctx = &fdev->cmd_ctx[cid];
	195	if (cmd_ctx->cpu < 0) {
	196	dev_err(fdev->dev,
	197	"admin CQE with CID=%u, op=%u does not match a pending command\n",
	198	cid, rsp_common->op);
	199	return;
	200	}
	201
	202	if (cmd_ctx->cb)
	203	cmd_ctx->cb(fdev, entry, xchg(&cmd_ctx->cb_data, NULL));
	204
	205	cpu = cmd_ctx->cpu;
	206	cmd_ctx->cpu = -1;
	207	sbitmap_queue_clear(&fdev->admin_sbq, cid, cpu);
	208	}
	209
	210	static int fun_init_cmd_ctx(struct fun_dev *fdev, unsigned int ntags)
	211	{
	212	unsigned int i;
	213
	214	fdev->cmd_ctx = kvcalloc(ntags, sizeof(*fdev->cmd_ctx), GFP_KERNEL);
	215	if (!fdev->cmd_ctx)
	216	return -ENOMEM;
	217
	218	for (i = 0; i < ntags; i++)
	219	fdev->cmd_ctx[i].cpu = -1;
	220
	221	return 0;
	222	}
	223
	224	/* Allocate and enable an admin queue and assign it the first IRQ vector. */
	225	static int fun_enable_admin_queue(struct fun_dev *fdev,
	226	const struct fun_dev_params *areq)
	227	{
	228	struct fun_queue_alloc_req qreq = {
	229	.cqe_size_log2 = areq->cqe_size_log2,
	230	.sqe_size_log2 = areq->sqe_size_log2,
	231	.cq_depth = areq->cq_depth,
	232	.sq_depth = areq->sq_depth,
	233	.rq_depth = areq->rq_depth,
	234	};
	235	unsigned int ntags = areq->sq_depth - 1;
	236	struct fun_queue *funq;
	237	int rc;
	238
	239	if (fdev->admin_q)
	240	return -EEXIST;
	241
	242	if (areq->sq_depth < AQA_MIN_QUEUE_SIZE \|\|
	243	areq->sq_depth > AQA_MAX_QUEUE_SIZE \|\|
	244	areq->cq_depth < AQA_MIN_QUEUE_SIZE \|\|
	245	areq->cq_depth > AQA_MAX_QUEUE_SIZE)
	246	return -EINVAL;
	247
	248	fdev->admin_q = fun_alloc_queue(fdev, 0, &qreq);
	249	if (!fdev->admin_q)
	250	return -ENOMEM;
	251
	252	rc = fun_init_cmd_ctx(fdev, ntags);
	253	if (rc)
	254	goto free_q;
	255
	256	rc = sbitmap_queue_init_node(&fdev->admin_sbq, ntags, -1, false,
	257	GFP_KERNEL, dev_to_node(fdev->dev));
	258	if (rc)
	259	goto free_cmd_ctx;
	260
	261	funq = fdev->admin_q;
	262	funq->cq_vector = 0;
	263	rc = fun_request_irq(funq, dev_name(fdev->dev), fun_admin_irq, funq);
	264	if (rc)
	265	goto free_sbq;
	266
	267	fun_set_cq_callback(funq, fun_complete_admin_cmd, NULL);
	268	fdev->adminq_cb = areq->event_cb;
	269
	270	writel((funq->sq_depth - 1) << AQA_ASQS_SHIFT \|
	271	(funq->cq_depth - 1) << AQA_ACQS_SHIFT,
	272	fdev->bar + NVME_REG_AQA);
	273
	274	writeq(funq->sq_dma_addr, fdev->bar + NVME_REG_ASQ);
	275	writeq(funq->cq_dma_addr, fdev->bar + NVME_REG_ACQ);
	276
	277	rc = fun_enable_ctrl(fdev, areq->cqe_size_log2, areq->sqe_size_log2);
	278	if (rc)
	279	goto free_irq;
	280
	281	if (areq->rq_depth) {
	282	rc = fun_create_rq(funq);
	283	if (rc)
	284	goto disable_ctrl;
	285
	286	funq_rq_post(funq);
	287	}
	288
	289	return 0;
	290
	291	disable_ctrl:
	292	fun_disable_ctrl(fdev);
	293	free_irq:
	294	fun_free_irq(funq);
	295	free_sbq:
	296	sbitmap_queue_free(&fdev->admin_sbq);
	297	free_cmd_ctx:
	298	kvfree(fdev->cmd_ctx);
	299	fdev->cmd_ctx = NULL;
	300	free_q:
	301	fun_free_queue(fdev->admin_q);
	302	fdev->admin_q = NULL;
	303	return rc;
	304	}
	305
	306	static void fun_disable_admin_queue(struct fun_dev *fdev)
	307	{
	308	struct fun_queue *admq = fdev->admin_q;
	309
	310	if (!admq)
	311	return;
	312
	313	fun_disable_ctrl(fdev);
	314
	315	fun_free_irq(admq);
	316	__fun_process_cq(admq, 0);
	317
	318	sbitmap_queue_free(&fdev->admin_sbq);
	319
	320	kvfree(fdev->cmd_ctx);
	321	fdev->cmd_ctx = NULL;
	322
	323	fun_free_queue(admq);
	324	fdev->admin_q = NULL;
	325	}
	326
	327	/* Return %true if the admin queue has stopped servicing commands as can be
	328	* detected through registers. This isn't exhaustive and may provide false
	329	* negatives.
	330	*/
	331	static bool fun_adminq_stopped(struct fun_dev *fdev)
	332	{
	333	u32 csts = readl(fdev->bar + NVME_REG_CSTS);
	334
	335	return (csts & (NVME_CSTS_CFS \| NVME_CSTS_RDY)) != NVME_CSTS_RDY;
	336	}
	337
	338	static int fun_wait_for_tag(struct fun_dev fdev, int cpup)
	339	{
	340	struct sbitmap_queue *sbq = &fdev->admin_sbq;
	341	struct sbq_wait_state *ws = &sbq->ws[0];
	342	DEFINE_SBQ_WAIT(wait);
	343	int tag;
	344
	345	for (;;) {
	346	sbitmap_prepare_to_wait(sbq, ws, &wait, TASK_UNINTERRUPTIBLE);
	347	if (fdev->suppress_cmds) {
	348	tag = -ESHUTDOWN;
	349	break;
	350	}
	351	tag = sbitmap_queue_get(sbq, cpup);
	352	if (tag >= 0)
	353	break;
	354	schedule();
	355	}
	356
	357	sbitmap_finish_wait(sbq, ws, &wait);
	358	return tag;
	359	}
	360
	361	/* Submit an asynchronous admin command. Caller is responsible for implementing
	362	* any waiting or timeout. Upon command completion the callback @cb is called.
	363	*/
	364	int fun_submit_admin_cmd(struct fun_dev fdev, struct fun_admin_req_common cmd,
	365	fun_admin_callback_t cb, void *cb_data, bool wait_ok)
	366	{
	367	struct fun_queue *funq = fdev->admin_q;
	368	unsigned int cmdsize = cmd->len8 * 8;
	369	struct fun_cmd_ctx *cmd_ctx;
	370	int tag, cpu, rc = 0;
	371
	372	if (WARN_ON(cmdsize > (1 << funq->sqe_size_log2)))
	373	return -EMSGSIZE;
	374
	375	tag = sbitmap_queue_get(&fdev->admin_sbq, &cpu);
	376	if (tag < 0) {
	377	if (!wait_ok)
	378	return -EAGAIN;
	379	tag = fun_wait_for_tag(fdev, &cpu);
	380	if (tag < 0)
	381	return tag;
	382	}
	383
	384	cmd->cid = cpu_to_be16(tag);
	385
	386	cmd_ctx = &fdev->cmd_ctx[tag];
	387	cmd_ctx->cb = cb;
	388	cmd_ctx->cb_data = cb_data;
	389
	390	spin_lock(&funq->sq_lock);
	391
	392	if (unlikely(fdev->suppress_cmds)) {
	393	rc = -ESHUTDOWN;
	394	sbitmap_queue_clear(&fdev->admin_sbq, tag, cpu);
	395	} else {
	396	cmd_ctx->cpu = cpu;
	397	memcpy(fun_sqe_at(funq, funq->sq_tail), cmd, cmdsize);
	398
	399	dev_dbg(fdev->dev, "admin cmd @ %u: %8ph\n", funq->sq_tail,
	400	cmd);
	401
	402	if (++funq->sq_tail == funq->sq_depth)
	403	funq->sq_tail = 0;
	404	writel(funq->sq_tail, funq->sq_db);
	405	}
	406	spin_unlock(&funq->sq_lock);
	407	return rc;
	408	}
	409
	410	/* Abandon a pending admin command by clearing the issuer's callback data.
	411	* Failure indicates that the command either has already completed or its
	412	* completion is racing with this call.
	413	*/
	414	static bool fun_abandon_admin_cmd(struct fun_dev *fd,
	415	const struct fun_admin_req_common *cmd,
	416	void *cb_data)
	417	{
	418	u16 cid = be16_to_cpu(cmd->cid);
	419	struct fun_cmd_ctx *cmd_ctx = &fd->cmd_ctx[cid];
	420
	421	return cmpxchg(&cmd_ctx->cb_data, cb_data, NULL) == cb_data;
	422	}
	423
	424	/* Stop submission of new admin commands and wake up any processes waiting for
	425	* tags. Already submitted commands are left to complete or time out.
	426	*/
	427	static void fun_admin_stop(struct fun_dev *fdev)
	428	{
	429	spin_lock(&fdev->admin_q->sq_lock);
	430	fdev->suppress_cmds = true;
	431	spin_unlock(&fdev->admin_q->sq_lock);
	432	sbitmap_queue_wake_all(&fdev->admin_sbq);
	433	}
	434
	435	/* The callback for synchronous execution of admin commands. It copies the
	436	* command response to the caller's buffer and signals completion.
	437	*/
	438	static void fun_admin_cmd_sync_cb(struct fun_dev fd, void rsp, void *cb_data)
	439	{
	440	const struct fun_admin_rsp_common *rsp_common = rsp;
	441	struct fun_sync_cmd_ctx *ctx = cb_data;
	442
	443	if (!ctx)
	444	return; /* command issuer timed out and left */
	445	if (ctx->rsp_buf) {
	446	unsigned int rsp_len = rsp_common->len8 * 8;
	447
	448	if (unlikely(rsp_len > ctx->rsp_len)) {
	449	dev_err(fd->dev,
	450	"response for op %u is %uB > response buffer %uB\n",
	451	rsp_common->op, rsp_len, ctx->rsp_len);
	452	rsp_len = ctx->rsp_len;
	453	}
	454	memcpy(ctx->rsp_buf, rsp, rsp_len);
	455	}
	456	ctx->rsp_status = rsp_common->ret;
	457	complete(&ctx->compl);
	458	}
	459
	460	/* Submit a synchronous admin command. */
	461	int fun_submit_admin_sync_cmd(struct fun_dev *fdev,
	462	struct fun_admin_req_common cmd, void rsp,
	463	size_t rspsize, unsigned int timeout)
	464	{
	465	struct fun_sync_cmd_ctx ctx = {
	466	.compl = COMPLETION_INITIALIZER_ONSTACK(ctx.compl),
	467	.rsp_buf = rsp,
	468	.rsp_len = rspsize,
	469	};
	470	unsigned int cmdlen = cmd->len8 * 8;
	471	unsigned long jiffies_left;
	472	int ret;
	473
	474	ret = fun_submit_admin_cmd(fdev, cmd, fun_admin_cmd_sync_cb, &ctx,
	475	true);
	476	if (ret)
	477	return ret;
	478
	479	if (!timeout)
	480	timeout = FUN_ADMIN_CMD_TO_MS;
	481
	482	jiffies_left = wait_for_completion_timeout(&ctx.compl,
	483	msecs_to_jiffies(timeout));
	484	if (!jiffies_left) {
	485	/* The command timed out. Attempt to cancel it so we can return.
	486	* But if the command is in the process of completing we'll
	487	* wait for it.
	488	*/
	489	if (fun_abandon_admin_cmd(fdev, cmd, &ctx)) {
	490	dev_err(fdev->dev, "admin command timed out: %*ph\n",
	491	cmdlen, cmd);
	492	fun_admin_stop(fdev);
	493	/* see if the timeout was due to a queue failure */
	494	if (fun_adminq_stopped(fdev))
	495	dev_err(fdev->dev,
	496	"device does not accept admin commands\n");
	497
	498	return -ETIMEDOUT;
	499	}
	500	wait_for_completion(&ctx.compl);
	501	}
	502
	503	if (ctx.rsp_status) {
	504	dev_err(fdev->dev, "admin command failed, err %d: %*ph\n",
	505	ctx.rsp_status, cmdlen, cmd);
	506	}
	507
	508	return -ctx.rsp_status;
	509	}
	510	EXPORT_SYMBOL_GPL(fun_submit_admin_sync_cmd);
	511
	512	/* Return the number of device resources of the requested type. */
	513	int fun_get_res_count(struct fun_dev *fdev, enum fun_admin_op res)
	514	{
	515	union {
	516	struct fun_admin_res_count_req req;
	517	struct fun_admin_res_count_rsp rsp;
	518	} cmd;
	519	int rc;
	520
	521	cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(res, sizeof(cmd.req));
	522	cmd.req.count = FUN_ADMIN_SIMPLE_SUBOP_INIT(FUN_ADMIN_SUBOP_RES_COUNT,
	523	0, 0);
	524
	525	rc = fun_submit_admin_sync_cmd(fdev, &cmd.req.common, &cmd.rsp,
	526	sizeof(cmd), 0);
	527	return rc ? rc : be32_to_cpu(cmd.rsp.count.data);
	528	}
	529	EXPORT_SYMBOL_GPL(fun_get_res_count);
	530
	531	/* Request that the instance of resource @res with the given id be deleted. */
	532	int fun_res_destroy(struct fun_dev *fdev, enum fun_admin_op res,
	533	unsigned int flags, u32 id)
	534	{
	535	struct fun_admin_generic_destroy_req req = {
	536	.common = FUN_ADMIN_REQ_COMMON_INIT2(res, sizeof(req)),
	537	.destroy = FUN_ADMIN_SIMPLE_SUBOP_INIT(FUN_ADMIN_SUBOP_DESTROY,
	538	flags, id)
	539	};
	540
	541	return fun_submit_admin_sync_cmd(fdev, &req.common, NULL, 0, 0);
	542	}
	543	EXPORT_SYMBOL_GPL(fun_res_destroy);
	544
	545	/* Bind two entities of the given types and IDs. */
	546	int fun_bind(struct fun_dev *fdev, enum fun_admin_bind_type type0,
	547	unsigned int id0, enum fun_admin_bind_type type1,
	548	unsigned int id1)
	549	{
	550	struct {
	551	struct fun_admin_bind_req req;
	552	struct fun_admin_bind_entry entry[2];
	553	} cmd = {
	554	.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_BIND,
	555	sizeof(cmd)),
	556	.entry[0] = FUN_ADMIN_BIND_ENTRY_INIT(type0, id0),
	557	.entry[1] = FUN_ADMIN_BIND_ENTRY_INIT(type1, id1),
	558	};
	559
	560	return fun_submit_admin_sync_cmd(fdev, &cmd.req.common, NULL, 0, 0);
	561	}
	562	EXPORT_SYMBOL_GPL(fun_bind);
	563
	564	static int fun_get_dev_limits(struct fun_dev *fdev)
	565	{
	566	struct pci_dev *pdev = to_pci_dev(fdev->dev);
	567	unsigned int cq_count, sq_count, num_dbs;
	568	int rc;
	569
	570	rc = fun_get_res_count(fdev, FUN_ADMIN_OP_EPCQ);
	571	if (rc < 0)
	572	return rc;
	573	cq_count = rc;
	574
	575	rc = fun_get_res_count(fdev, FUN_ADMIN_OP_EPSQ);
	576	if (rc < 0)
	577	return rc;
	578	sq_count = rc;
	579
	580	/* The admin queue consumes 1 CQ and at least 1 SQ. To be usable the
	581	* device must provide additional queues.
	582	*/
	583	if (cq_count < 2 \|\| sq_count < 2 + !!fdev->admin_q->rq_depth)
	584	return -EINVAL;
	585
	586	/* Calculate the max QID based on SQ/CQ/doorbell counts.
	587	* SQ/CQ doorbells alternate.
	588	*/
	589	num_dbs = (pci_resource_len(pdev, 0) - NVME_REG_DBS) /
	590	(fdev->db_stride * 4);
	591	fdev->max_qid = min3(cq_count, sq_count, num_dbs / 2) - 1;
	592	fdev->kern_end_qid = fdev->max_qid + 1;
	593	return 0;
	594	}
	595
	596	/* Allocate all MSI-X vectors available on a function and at least @min_vecs. */
	597	static int fun_alloc_irqs(struct pci_dev *pdev, unsigned int min_vecs)
	598	{
	599	int vecs, num_msix = pci_msix_vec_count(pdev);
	600
	601	if (num_msix < 0)
	602	return num_msix;
	603	if (min_vecs > num_msix)
	604	return -ERANGE;
	605
	606	vecs = pci_alloc_irq_vectors(pdev, min_vecs, num_msix, PCI_IRQ_MSIX);
	607	if (vecs > 0) {
	608	dev_info(&pdev->dev,
	609	"Allocated %d IRQ vectors of %d requested\n",
	610	vecs, num_msix);
	611	} else {
	612	dev_err(&pdev->dev,
	613	"Unable to allocate at least %u IRQ vectors\n",
	614	min_vecs);
	615	}
	616	return vecs;
	617	}
	618
	619	/* Allocate and initialize the IRQ manager state. */
	620	static int fun_alloc_irq_mgr(struct fun_dev *fdev)
	621	{
	622	fdev->irq_map = bitmap_zalloc(fdev->num_irqs, GFP_KERNEL);
	623	if (!fdev->irq_map)
	624	return -ENOMEM;
	625
	626	spin_lock_init(&fdev->irqmgr_lock);
	627	/* mark IRQ 0 allocated, it is used by the admin queue */
	628	__set_bit(0, fdev->irq_map);
	629	fdev->irqs_avail = fdev->num_irqs - 1;
	630	return 0;
	631	}
	632
	633	/* Reserve @nirqs of the currently available IRQs and return their indices. */
	634	int fun_reserve_irqs(struct fun_dev fdev, unsigned int nirqs, u16 irq_indices)
	635	{
	636	unsigned int b, n = 0;
	637	int err = -ENOSPC;
	638
	639	if (!nirqs)
	640	return 0;
	641
	642	spin_lock(&fdev->irqmgr_lock);
	643	if (nirqs > fdev->irqs_avail)
	644	goto unlock;
	645
	646	for_each_clear_bit(b, fdev->irq_map, fdev->num_irqs) {
	647	__set_bit(b, fdev->irq_map);
	648	irq_indices[n++] = b;
	649	if (n >= nirqs)
	650	break;
	651	}
	652
	653	WARN_ON(n < nirqs);
	654	fdev->irqs_avail -= n;
	655	err = n;
	656	unlock:
	657	spin_unlock(&fdev->irqmgr_lock);
	658	return err;
	659	}
	660	EXPORT_SYMBOL(fun_reserve_irqs);
	661
	662	/* Release @nirqs previously allocated IRQS with the supplied indices. */
	663	void fun_release_irqs(struct fun_dev *fdev, unsigned int nirqs,
	664	u16 *irq_indices)
	665	{
	666	unsigned int i;
	667
	668	spin_lock(&fdev->irqmgr_lock);
	669	for (i = 0; i < nirqs; i++)
	670	__clear_bit(irq_indices[i], fdev->irq_map);
	671	fdev->irqs_avail += nirqs;
	672	spin_unlock(&fdev->irqmgr_lock);
	673	}
	674	EXPORT_SYMBOL(fun_release_irqs);
	675
	676	static void fun_serv_handler(struct work_struct *work)
	677	{
	678	struct fun_dev *fd = container_of(work, struct fun_dev, service_task);
	679
	680	if (test_bit(FUN_SERV_DISABLED, &fd->service_flags))
	681	return;
	682	if (fd->serv_cb)
	683	fd->serv_cb(fd);
	684	}
	685
	686	void fun_serv_stop(struct fun_dev *fd)
	687	{
	688	set_bit(FUN_SERV_DISABLED, &fd->service_flags);
	689	cancel_work_sync(&fd->service_task);
	690	}
	691	EXPORT_SYMBOL_GPL(fun_serv_stop);
	692
	693	void fun_serv_restart(struct fun_dev *fd)
	694	{
	695	clear_bit(FUN_SERV_DISABLED, &fd->service_flags);
	696	if (fd->service_flags)
	697	schedule_work(&fd->service_task);
	698	}
	699	EXPORT_SYMBOL_GPL(fun_serv_restart);
	700
	701	void fun_serv_sched(struct fun_dev *fd)
	702	{
	703	if (!test_bit(FUN_SERV_DISABLED, &fd->service_flags))
	704	schedule_work(&fd->service_task);
	705	}
	706	EXPORT_SYMBOL_GPL(fun_serv_sched);
	707
	708	/* Check and try to get the device into a proper state for initialization,
	709	* i.e., CSTS.RDY = CC.EN = 0.
	710	*/
	711	static int sanitize_dev(struct fun_dev *fdev)
	712	{
	713	int rc;
	714
	715	fdev->cap_reg = readq(fdev->bar + NVME_REG_CAP);
	716	fdev->cc_reg = readl(fdev->bar + NVME_REG_CC);
	717
	718	/* First get RDY to agree with the current EN. Give RDY the opportunity
	719	* to complete a potential recent EN change.
	720	*/
	721	rc = fun_wait_ready(fdev, fdev->cc_reg & NVME_CC_ENABLE);
	722	if (rc)
	723	return rc;
	724
	725	/* Next, reset the device if EN is currently 1. */
	726	if (fdev->cc_reg & NVME_CC_ENABLE)
	727	rc = fun_disable_ctrl(fdev);
	728
	729	return rc;
	730	}
	731
	732	/* Undo the device initialization of fun_dev_enable(). */
	733	void fun_dev_disable(struct fun_dev *fdev)
	734	{
	735	struct pci_dev *pdev = to_pci_dev(fdev->dev);
	736
	737	pci_set_drvdata(pdev, NULL);
	738
	739	if (fdev->fw_handle != FUN_HCI_ID_INVALID) {
	740	fun_res_destroy(fdev, FUN_ADMIN_OP_SWUPGRADE, 0,
	741	fdev->fw_handle);
	742	fdev->fw_handle = FUN_HCI_ID_INVALID;
	743	}
	744
	745	fun_disable_admin_queue(fdev);
	746
	747	bitmap_free(fdev->irq_map);
	748	pci_free_irq_vectors(pdev);
	749
	750	pci_clear_master(pdev);
	751	pci_disable_pcie_error_reporting(pdev);
	752	pci_disable_device(pdev);
	753
	754	fun_unmap_bars(fdev);
	755	}
	756	EXPORT_SYMBOL(fun_dev_disable);
	757
	758	/* Perform basic initialization of a device, including
	759	* - PCI config space setup and BAR0 mapping
	760	* - interrupt management initialization
	761	* - 1 admin queue setup
	762	* - determination of some device limits, such as number of queues.
	763	*/
	764	int fun_dev_enable(struct fun_dev fdev, struct pci_dev pdev,
	765	const struct fun_dev_params areq, const char name)
	766	{
	767	int rc;
	768
	769	fdev->dev = &pdev->dev;
	770	rc = fun_map_bars(fdev, name);
	771	if (rc)
	772	return rc;
	773
	774	rc = fun_set_dma_masks(fdev->dev);
	775	if (rc)
	776	goto unmap;
	777
	778	rc = pci_enable_device_mem(pdev);
	779	if (rc) {
	780	dev_err(&pdev->dev, "Couldn't enable device, err %d\n", rc);
	781	goto unmap;
	782	}
	783
	784	pci_enable_pcie_error_reporting(pdev);
	785
	786	rc = sanitize_dev(fdev);
	787	if (rc)
	788	goto disable_dev;
	789
	790	fdev->fw_handle = FUN_HCI_ID_INVALID;
	791	fdev->q_depth = NVME_CAP_MQES(fdev->cap_reg) + 1;
	792	fdev->db_stride = 1 << NVME_CAP_STRIDE(fdev->cap_reg);
	793	fdev->dbs = fdev->bar + NVME_REG_DBS;
	794
	795	INIT_WORK(&fdev->service_task, fun_serv_handler);
	796	fdev->service_flags = FUN_SERV_DISABLED;
	797	fdev->serv_cb = areq->serv_cb;
	798
	799	rc = fun_alloc_irqs(pdev, areq->min_msix + 1); /* +1 for admin CQ */
	800	if (rc < 0)
	801	goto disable_dev;
	802	fdev->num_irqs = rc;
	803
	804	rc = fun_alloc_irq_mgr(fdev);
	805	if (rc)
	806	goto free_irqs;
	807
	808	pci_set_master(pdev);
	809	rc = fun_enable_admin_queue(fdev, areq);
	810	if (rc)
	811	goto free_irq_mgr;
	812
	813	rc = fun_get_dev_limits(fdev);
	814	if (rc < 0)
	815	goto disable_admin;
	816
	817	pci_save_state(pdev);
	818	pci_set_drvdata(pdev, fdev);
	819	pcie_print_link_status(pdev);
	820	dev_dbg(fdev->dev, "q_depth %u, db_stride %u, max qid %d kern_end_qid %d\n",
	821	fdev->q_depth, fdev->db_stride, fdev->max_qid,
	822	fdev->kern_end_qid);
	823	return 0;
	824
	825	disable_admin:
	826	fun_disable_admin_queue(fdev);
	827	free_irq_mgr:
	828	pci_clear_master(pdev);
	829	bitmap_free(fdev->irq_map);
	830	free_irqs:
	831	pci_free_irq_vectors(pdev);
	832	disable_dev:
	833	pci_disable_pcie_error_reporting(pdev);
	834	pci_disable_device(pdev);
	835	unmap:
	836	fun_unmap_bars(fdev);
	837	return rc;
	838	}
	839	EXPORT_SYMBOL(fun_dev_enable);
	840
	841	MODULE_AUTHOR("Dimitris Michailidis <dmichail@fungible.com>");
	842	MODULE_DESCRIPTION("Core services driver for Fungible devices");
	843	MODULE_LICENSE("Dual BSD/GPL");

drivers/net/ethernet/fungible/funcore/fun_dev.h+150−0

@@ -0,0 +1,150 @@
	1	/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) */
	2
	3	#ifndef _FUNDEV_H
	4	#define _FUNDEV_H
	5
	6	#include <linux/sbitmap.h>
	7	#include <linux/spinlock_types.h>
	8	#include <linux/workqueue.h>
	9	#include "fun_hci.h"
	10
	11	struct pci_dev;
	12	struct fun_dev;
	13	struct fun_queue;
	14	struct fun_cmd_ctx;
	15	struct fun_queue_alloc_req;
	16
	17	/* doorbell fields */
	18	enum {
	19	FUN_DB_QIDX_S = 0,
	20	FUN_DB_INTCOAL_ENTRIES_S = 16,
	21	FUN_DB_INTCOAL_ENTRIES_M = 0x7f,
	22	FUN_DB_INTCOAL_USEC_S = 23,
	23	FUN_DB_INTCOAL_USEC_M = 0x7f,
	24	FUN_DB_IRQ_S = 30,
	25	FUN_DB_IRQ_F = 1 << FUN_DB_IRQ_S,
	26	FUN_DB_IRQ_ARM_S = 31,
	27	FUN_DB_IRQ_ARM_F = 1U << FUN_DB_IRQ_ARM_S
	28	};
	29
	30	/* Callback for asynchronous admin commands.
	31	* Invoked on reception of command response.
	32	*/
	33	typedef void (fun_admin_callback_t)(struct fun_dev fdev, void *rsp,
	34	void *cb_data);
	35
	36	/* Callback for events/notifications received by an admin queue. */
	37	typedef void (fun_admin_event_cb)(struct fun_dev fdev, void *cqe);
	38
	39	/* Callback for pending work handled by the service task. */
	40	typedef void (fun_serv_cb)(struct fun_dev fd);
	41
	42	/* service task flags */
	43	enum {
	44	FUN_SERV_DISABLED, /* service task is disabled */
	45	FUN_SERV_FIRST_AVAIL
	46	};
	47
	48	/* Driver state associated with a PCI function. */
	49	struct fun_dev {
	50	struct device *dev;
	51
	52	void __iomem bar; / start of BAR0 mapping */
	53	u32 __iomem dbs; / start of doorbells in BAR0 mapping */
	54
	55	/* admin queue */
	56	struct fun_queue *admin_q;
	57	struct sbitmap_queue admin_sbq;
	58	struct fun_cmd_ctx *cmd_ctx;
	59	fun_admin_event_cb adminq_cb;
	60	bool suppress_cmds; /* if set don't write commands to SQ */
	61
	62	/* address increment between consecutive doorbells, in 4B units */
	63	unsigned int db_stride;
	64
	65	/* SW versions of device registers */
	66	u32 cc_reg; /* CC register */
	67	u64 cap_reg; /* CAPability register */
	68
	69	unsigned int q_depth; /* max queue depth supported by device */
	70	unsigned int max_qid; /* = #queues - 1, separately for SQs and CQs */
	71	unsigned int kern_end_qid; /* last qid in the kernel range + 1 */
	72
	73	unsigned int fw_handle;
	74
	75	/* IRQ manager */
	76	unsigned int num_irqs;
	77	unsigned int irqs_avail;
	78	spinlock_t irqmgr_lock;
	79	unsigned long *irq_map;
	80
	81	/* The service task handles work that needs a process context */
	82	struct work_struct service_task;
	83	unsigned long service_flags;
	84	fun_serv_cb serv_cb;
	85	};
	86
	87	struct fun_dev_params {
	88	u8 cqe_size_log2; /* admin q CQE size */
	89	u8 sqe_size_log2; /* admin q SQE size */
	90
	91	/* admin q depths */
	92	u16 cq_depth;
	93	u16 sq_depth;
	94	u16 rq_depth;
	95
	96	u16 min_msix; /* min vectors needed by requesting driver */
	97
	98	fun_admin_event_cb event_cb;
	99	fun_serv_cb serv_cb;
	100	};
	101
	102	/* Return the BAR address of a doorbell. */
	103	static inline u32 __iomem fun_db_addr(const struct fun_dev fdev,
	104	unsigned int db_index)
	105	{
	106	return &fdev->dbs[db_index * fdev->db_stride];
	107	}
	108
	109	/* Return the BAR address of an SQ doorbell. SQ and CQ DBs alternate,
	110	* SQs have even DB indices.
	111	*/
	112	static inline u32 __iomem fun_sq_db_addr(const struct fun_dev fdev,
	113	unsigned int sqid)
	114	{
	115	return fun_db_addr(fdev, sqid * 2);
	116	}
	117
	118	static inline u32 __iomem fun_cq_db_addr(const struct fun_dev fdev,
	119	unsigned int cqid)
	120	{
	121	return fun_db_addr(fdev, cqid * 2 + 1);
	122	}
	123
	124	int fun_get_res_count(struct fun_dev *fdev, enum fun_admin_op res);
	125	int fun_res_destroy(struct fun_dev *fdev, enum fun_admin_op res,
	126	unsigned int flags, u32 id);
	127	int fun_bind(struct fun_dev *fdev, enum fun_admin_bind_type type0,
	128	unsigned int id0, enum fun_admin_bind_type type1,
	129	unsigned int id1);
	130
	131	int fun_submit_admin_cmd(struct fun_dev fdev, struct fun_admin_req_common cmd,
	132	fun_admin_callback_t cb, void *cb_data, bool wait_ok);
	133	int fun_submit_admin_sync_cmd(struct fun_dev *fdev,
	134	struct fun_admin_req_common cmd, void rsp,
	135	size_t rspsize, unsigned int timeout);
	136
	137	int fun_dev_enable(struct fun_dev fdev, struct pci_dev pdev,
	138	const struct fun_dev_params areq, const char name);
	139	void fun_dev_disable(struct fun_dev *fdev);
	140
	141	int fun_reserve_irqs(struct fun_dev *fdev, unsigned int nirqs,
	142	u16 *irq_indices);
	143	void fun_release_irqs(struct fun_dev *fdev, unsigned int nirqs,
	144	u16 *irq_indices);
	145
	146	void fun_serv_stop(struct fun_dev *fd);
	147	void fun_serv_restart(struct fun_dev *fd);
	148	void fun_serv_sched(struct fun_dev *fd);
	149
	150	#endif /* _FUNDEV_H */

drivers/net/ethernet/fungible/funcore/fun_hci.h+1202−0

@@ -0,0 +1,1202 @@
	1	/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) */
	2
	3	#ifndef __FUN_HCI_H
	4	#define __FUN_HCI_H
	5
	6	enum {
	7	FUN_HCI_ID_INVALID = 0xffffffff,
	8	};
	9
	10	enum fun_admin_op {
	11	FUN_ADMIN_OP_BIND = 0x1,
	12	FUN_ADMIN_OP_EPCQ = 0x11,
	13	FUN_ADMIN_OP_EPSQ = 0x12,
	14	FUN_ADMIN_OP_PORT = 0x13,
	15	FUN_ADMIN_OP_ETH = 0x14,
	16	FUN_ADMIN_OP_VI = 0x15,
	17	FUN_ADMIN_OP_SWUPGRADE = 0x1f,
	18	FUN_ADMIN_OP_RSS = 0x21,
	19	FUN_ADMIN_OP_ADI = 0x25,
	20	FUN_ADMIN_OP_KTLS = 0x26,
	21	};
	22
	23	enum {
	24	FUN_REQ_COMMON_FLAG_RSP = 0x1,
	25	FUN_REQ_COMMON_FLAG_HEAD_WB = 0x2,
	26	FUN_REQ_COMMON_FLAG_INT = 0x4,
	27	FUN_REQ_COMMON_FLAG_CQE_IN_RQBUF = 0x8,
	28	};
	29
	30	struct fun_admin_req_common {
	31	__u8 op;
	32	__u8 len8;
	33	__be16 flags;
	34	__u8 suboff8;
	35	__u8 rsvd0;
	36	__be16 cid;
	37	};
	38
	39	#define FUN_ADMIN_REQ_COMMON_INIT(_op, _len8, _flags, _suboff8, _cid) \
	40	(struct fun_admin_req_common) { \
	41	.op = (_op), .len8 = (_len8), .flags = cpu_to_be16(_flags), \
	42	.suboff8 = (_suboff8), .cid = cpu_to_be16(_cid), \
	43	}
	44
	45	#define FUN_ADMIN_REQ_COMMON_INIT2(_op, _len) \
	46	(struct fun_admin_req_common) { \
	47	.op = (_op), .len8 = (_len) / 8, \
	48	}
	49
	50	struct fun_admin_rsp_common {
	51	__u8 op;
	52	__u8 len8;
	53	__be16 flags;
	54	__u8 suboff8;
	55	__u8 ret;
	56	__be16 cid;
	57	};
	58
	59	struct fun_admin_write48_req {
	60	__be64 key_to_data;
	61	};
	62
	63	#define FUN_ADMIN_WRITE48_REQ_KEY_S 56U
	64	#define FUN_ADMIN_WRITE48_REQ_KEY_M 0xff
	65	#define FUN_ADMIN_WRITE48_REQ_KEY_P_NOSWAP(x) \
	66	(((__u64)x) << FUN_ADMIN_WRITE48_REQ_KEY_S)
	67
	68	#define FUN_ADMIN_WRITE48_REQ_DATA_S 0U
	69	#define FUN_ADMIN_WRITE48_REQ_DATA_M 0xffffffffffff
	70	#define FUN_ADMIN_WRITE48_REQ_DATA_P_NOSWAP(x) \
	71	(((__u64)x) << FUN_ADMIN_WRITE48_REQ_DATA_S)
	72
	73	#define FUN_ADMIN_WRITE48_REQ_INIT(key, data) \
	74	(struct fun_admin_write48_req) { \
	75	.key_to_data = cpu_to_be64( \
	76	FUN_ADMIN_WRITE48_REQ_KEY_P_NOSWAP(key) \| \
	77	FUN_ADMIN_WRITE48_REQ_DATA_P_NOSWAP(data)), \
	78	}
	79
	80	struct fun_admin_write48_rsp {
	81	__be64 key_to_data;
	82	};
	83
	84	struct fun_admin_read48_req {
	85	__be64 key_pack;
	86	};
	87
	88	#define FUN_ADMIN_READ48_REQ_KEY_S 56U
	89	#define FUN_ADMIN_READ48_REQ_KEY_M 0xff
	90	#define FUN_ADMIN_READ48_REQ_KEY_P_NOSWAP(x) \
	91	(((__u64)x) << FUN_ADMIN_READ48_REQ_KEY_S)
	92
	93	#define FUN_ADMIN_READ48_REQ_INIT(key) \
	94	(struct fun_admin_read48_req) { \
	95	.key_pack = \
	96	cpu_to_be64(FUN_ADMIN_READ48_REQ_KEY_P_NOSWAP(key)), \
	97	}
	98
	99	struct fun_admin_read48_rsp {
	100	__be64 key_to_data;
	101	};
	102
	103	#define FUN_ADMIN_READ48_RSP_KEY_S 56U
	104	#define FUN_ADMIN_READ48_RSP_KEY_M 0xff
	105	#define FUN_ADMIN_READ48_RSP_KEY_G(x) \
	106	((be64_to_cpu(x) >> FUN_ADMIN_READ48_RSP_KEY_S) & \
	107	FUN_ADMIN_READ48_RSP_KEY_M)
	108
	109	#define FUN_ADMIN_READ48_RSP_RET_S 48U
	110	#define FUN_ADMIN_READ48_RSP_RET_M 0xff
	111	#define FUN_ADMIN_READ48_RSP_RET_G(x) \
	112	((be64_to_cpu(x) >> FUN_ADMIN_READ48_RSP_RET_S) & \
	113	FUN_ADMIN_READ48_RSP_RET_M)
	114
	115	#define FUN_ADMIN_READ48_RSP_DATA_S 0U
	116	#define FUN_ADMIN_READ48_RSP_DATA_M 0xffffffffffff
	117	#define FUN_ADMIN_READ48_RSP_DATA_G(x) \
	118	((be64_to_cpu(x) >> FUN_ADMIN_READ48_RSP_DATA_S) & \
	119	FUN_ADMIN_READ48_RSP_DATA_M)
	120
	121	enum fun_admin_bind_type {
	122	FUN_ADMIN_BIND_TYPE_EPCQ = 0x1,
	123	FUN_ADMIN_BIND_TYPE_EPSQ = 0x2,
	124	FUN_ADMIN_BIND_TYPE_PORT = 0x3,
	125	FUN_ADMIN_BIND_TYPE_RSS = 0x4,
	126	FUN_ADMIN_BIND_TYPE_VI = 0x5,
	127	FUN_ADMIN_BIND_TYPE_ETH = 0x6,
	128	};
	129
	130	struct fun_admin_bind_entry {
	131	__u8 type;
	132	__u8 rsvd0[3];
	133	__be32 id;
	134	};
	135
	136	#define FUN_ADMIN_BIND_ENTRY_INIT(_type, _id) \
	137	(struct fun_admin_bind_entry) { \
	138	.type = (_type), .id = cpu_to_be32(_id), \
	139	}
	140
	141	struct fun_admin_bind_req {
	142	struct fun_admin_req_common common;
	143	struct fun_admin_bind_entry entry[];
	144	};
	145
	146	struct fun_admin_bind_rsp {
	147	struct fun_admin_rsp_common bind_rsp_common;
	148	};
	149
	150	struct fun_admin_simple_subop {
	151	__u8 subop;
	152	__u8 rsvd0;
	153	__be16 flags;
	154	__be32 data;
	155	};
	156
	157	#define FUN_ADMIN_SIMPLE_SUBOP_INIT(_subop, _flags, _data) \
	158	(struct fun_admin_simple_subop) { \
	159	.subop = (_subop), .flags = cpu_to_be16(_flags), \
	160	.data = cpu_to_be32(_data), \
	161	}
	162
	163	enum fun_admin_subop {
	164	FUN_ADMIN_SUBOP_CREATE = 0x10,
	165	FUN_ADMIN_SUBOP_DESTROY = 0x11,
	166	FUN_ADMIN_SUBOP_MODIFY = 0x12,
	167	FUN_ADMIN_SUBOP_RES_COUNT = 0x14,
	168	FUN_ADMIN_SUBOP_READ = 0x15,
	169	FUN_ADMIN_SUBOP_WRITE = 0x16,
	170	FUN_ADMIN_SUBOP_NOTIFY = 0x17,
	171	};
	172
	173	enum {
	174	FUN_ADMIN_RES_CREATE_FLAG_ALLOCATOR = 0x1,
	175	};
	176
	177	struct fun_admin_generic_destroy_req {
	178	struct fun_admin_req_common common;
	179	struct fun_admin_simple_subop destroy;
	180	};
	181
	182	struct fun_admin_generic_create_rsp {
	183	struct fun_admin_rsp_common common;
	184
	185	__u8 subop;
	186	__u8 rsvd0;
	187	__be16 flags;
	188	__be32 id;
	189	};
	190
	191	struct fun_admin_res_count_req {
	192	struct fun_admin_req_common common;
	193	struct fun_admin_simple_subop count;
	194	};
	195
	196	struct fun_admin_res_count_rsp {
	197	struct fun_admin_rsp_common common;
	198	struct fun_admin_simple_subop count;
	199	};
	200
	201	enum {
	202	FUN_ADMIN_EPCQ_CREATE_FLAG_INT_EPCQ = 0x2,
	203	FUN_ADMIN_EPCQ_CREATE_FLAG_ENTRY_WR_TPH = 0x4,
	204	FUN_ADMIN_EPCQ_CREATE_FLAG_SL_WR_TPH = 0x8,
	205	FUN_ADMIN_EPCQ_CREATE_FLAG_RQ = 0x80,
	206	FUN_ADMIN_EPCQ_CREATE_FLAG_INT_IQ = 0x100,
	207	FUN_ADMIN_EPCQ_CREATE_FLAG_INT_NOARM = 0x200,
	208	FUN_ADMIN_EPCQ_CREATE_FLAG_DROP_ON_OVERFLOW = 0x400,
	209	};
	210
	211	struct fun_admin_epcq_req {
	212	struct fun_admin_req_common common;
	213	union epcq_req_subop {
	214	struct fun_admin_epcq_create_req {
	215	__u8 subop;
	216	__u8 rsvd0;
	217	__be16 flags;
	218	__be32 id;
	219
	220	__be32 epsqid;
	221	__u8 rsvd1;
	222	__u8 entry_size_log2;
	223	__be16 nentries;
	224
	225	__be64 address;
	226
	227	__be16 tailroom; /* per packet tailroom in bytes */
	228	__u8 headroom; /* per packet headroom in 2B units */
	229	__u8 intcoal_kbytes;
	230	__u8 intcoal_holdoff_nentries;
	231	__u8 intcoal_holdoff_usecs;
	232	__be16 intid;
	233
	234	__be32 scan_start_id;
	235	__be32 scan_end_id;
	236
	237	__be16 tph_cpuid;
	238	__u8 rsvd3[6];
	239	} create;
	240
	241	struct fun_admin_epcq_modify_req {
	242	__u8 subop;
	243	__u8 rsvd0;
	244	__be16 flags;
	245	__be32 id;
	246
	247	__be16 headroom; /* headroom in bytes */
	248	__u8 rsvd1[6];
	249	} modify;
	250	} u;
	251	};
	252
	253	#define FUN_ADMIN_EPCQ_CREATE_REQ_INIT( \
	254	_subop, _flags, _id, _epsqid, _entry_size_log2, _nentries, _address, \
	255	_tailroom, _headroom, _intcoal_kbytes, _intcoal_holdoff_nentries, \
	256	_intcoal_holdoff_usecs, _intid, _scan_start_id, _scan_end_id, \
	257	_tph_cpuid) \
	258	(struct fun_admin_epcq_create_req) { \
	259	.subop = (_subop), .flags = cpu_to_be16(_flags), \
	260	.id = cpu_to_be32(_id), .epsqid = cpu_to_be32(_epsqid), \
	261	.entry_size_log2 = _entry_size_log2, \
	262	.nentries = cpu_to_be16(_nentries), \
	263	.address = cpu_to_be64(_address), \
	264	.tailroom = cpu_to_be16(_tailroom), .headroom = _headroom, \
	265	.intcoal_kbytes = _intcoal_kbytes, \
	266	.intcoal_holdoff_nentries = _intcoal_holdoff_nentries, \
	267	.intcoal_holdoff_usecs = _intcoal_holdoff_usecs, \
	268	.intid = cpu_to_be16(_intid), \
	269	.scan_start_id = cpu_to_be32(_scan_start_id), \
	270	.scan_end_id = cpu_to_be32(_scan_end_id), \
	271	.tph_cpuid = cpu_to_be16(_tph_cpuid), \
	272	}
	273
	274	#define FUN_ADMIN_EPCQ_MODIFY_REQ_INIT(_subop, _flags, _id, _headroom) \
	275	(struct fun_admin_epcq_modify_req) { \
	276	.subop = (_subop), .flags = cpu_to_be16(_flags), \
	277	.id = cpu_to_be32(_id), .headroom = cpu_to_be16(_headroom), \
	278	}
	279
	280	enum {
	281	FUN_ADMIN_EPSQ_CREATE_FLAG_INT_EPSQ = 0x2,
	282	FUN_ADMIN_EPSQ_CREATE_FLAG_ENTRY_RD_TPH = 0x4,
	283	FUN_ADMIN_EPSQ_CREATE_FLAG_GL_RD_TPH = 0x8,
	284	FUN_ADMIN_EPSQ_CREATE_FLAG_HEAD_WB_ADDRESS = 0x10,
	285	FUN_ADMIN_EPSQ_CREATE_FLAG_HEAD_WB_ADDRESS_TPH = 0x20,
	286	FUN_ADMIN_EPSQ_CREATE_FLAG_HEAD_WB_EPCQ = 0x40,
	287	FUN_ADMIN_EPSQ_CREATE_FLAG_RQ = 0x80,
	288	FUN_ADMIN_EPSQ_CREATE_FLAG_INT_IQ = 0x100,
	289	FUN_ADMIN_EPSQ_CREATE_FLAG_NO_CMPL = 0x200,
	290	};
	291
	292	struct fun_admin_epsq_req {
	293	struct fun_admin_req_common common;
	294
	295	union epsq_req_subop {
	296	struct fun_admin_epsq_create_req {
	297	__u8 subop;
	298	__u8 rsvd0;
	299	__be16 flags;
	300	__be32 id;
	301
	302	__be32 epcqid;
	303	__u8 rsvd1;
	304	__u8 entry_size_log2;
	305	__be16 nentries;
	306
	307	__be64 address; /* DMA address of epsq */
	308
	309	__u8 rsvd2[3];
	310	__u8 intcoal_kbytes;
	311	__u8 intcoal_holdoff_nentries;
	312	__u8 intcoal_holdoff_usecs;
	313	__be16 intid;
	314
	315	__be32 scan_start_id;
	316	__be32 scan_end_id;
	317
	318	__u8 rsvd3[4];
	319	__be16 tph_cpuid;
	320	__u8 buf_size_log2; /* log2 of RQ buffer size */
	321	__u8 head_wb_size_log2; /* log2 of head write back size */
	322
	323	__be64 head_wb_address; /* DMA address for head writeback */
	324	} create;
	325	} u;
	326	};
	327
	328	#define FUN_ADMIN_EPSQ_CREATE_REQ_INIT( \
	329	_subop, _flags, _id, _epcqid, _entry_size_log2, _nentries, _address, \
	330	_intcoal_kbytes, _intcoal_holdoff_nentries, _intcoal_holdoff_usecs, \
	331	_intid, _scan_start_id, _scan_end_id, _tph_cpuid, _buf_size_log2, \
	332	_head_wb_size_log2, _head_wb_address) \
	333	(struct fun_admin_epsq_create_req) { \
	334	.subop = (_subop), .flags = cpu_to_be16(_flags), \
	335	.id = cpu_to_be32(_id), .epcqid = cpu_to_be32(_epcqid), \
	336	.entry_size_log2 = _entry_size_log2, \
	337	.nentries = cpu_to_be16(_nentries), \
	338	.address = cpu_to_be64(_address), \
	339	.intcoal_kbytes = _intcoal_kbytes, \
	340	.intcoal_holdoff_nentries = _intcoal_holdoff_nentries, \
	341	.intcoal_holdoff_usecs = _intcoal_holdoff_usecs, \
	342	.intid = cpu_to_be16(_intid), \
	343	.scan_start_id = cpu_to_be32(_scan_start_id), \
	344	.scan_end_id = cpu_to_be32(_scan_end_id), \
	345	.tph_cpuid = cpu_to_be16(_tph_cpuid), \
	346	.buf_size_log2 = _buf_size_log2, \
	347	.head_wb_size_log2 = _head_wb_size_log2, \
	348	.head_wb_address = cpu_to_be64(_head_wb_address), \
	349	}
	350
	351	enum {
	352	FUN_PORT_CAP_OFFLOADS = 0x1,
	353	FUN_PORT_CAP_STATS = 0x2,
	354	FUN_PORT_CAP_LOOPBACK = 0x4,
	355	FUN_PORT_CAP_VPORT = 0x8,
	356	FUN_PORT_CAP_TX_PAUSE = 0x10,
	357	FUN_PORT_CAP_RX_PAUSE = 0x20,
	358	FUN_PORT_CAP_AUTONEG = 0x40,
	359	FUN_PORT_CAP_RSS = 0x80,
	360	FUN_PORT_CAP_VLAN_OFFLOADS = 0x100,
	361	FUN_PORT_CAP_ENCAP_OFFLOADS = 0x200,
	362	FUN_PORT_CAP_1000_X = 0x1000,
	363	FUN_PORT_CAP_10G_R = 0x2000,
	364	FUN_PORT_CAP_40G_R4 = 0x4000,
	365	FUN_PORT_CAP_25G_R = 0x8000,
	366	FUN_PORT_CAP_50G_R2 = 0x10000,
	367	FUN_PORT_CAP_50G_R = 0x20000,
	368	FUN_PORT_CAP_100G_R4 = 0x40000,
	369	FUN_PORT_CAP_100G_R2 = 0x80000,
	370	FUN_PORT_CAP_200G_R4 = 0x100000,
	371	FUN_PORT_CAP_FEC_NONE = 0x10000000,
	372	FUN_PORT_CAP_FEC_FC = 0x20000000,
	373	FUN_PORT_CAP_FEC_RS = 0x40000000,
	374	};
	375
	376	enum fun_port_brkout_mode {
	377	FUN_PORT_BRKMODE_NA = 0x0,
	378	FUN_PORT_BRKMODE_NONE = 0x1,
	379	FUN_PORT_BRKMODE_2X = 0x2,
	380	FUN_PORT_BRKMODE_4X = 0x3,
	381	};
	382
	383	enum {
	384	FUN_PORT_SPEED_AUTO = 0x0,
	385	FUN_PORT_SPEED_10M = 0x1,
	386	FUN_PORT_SPEED_100M = 0x2,
	387	FUN_PORT_SPEED_1G = 0x4,
	388	FUN_PORT_SPEED_10G = 0x8,
	389	FUN_PORT_SPEED_25G = 0x10,
	390	FUN_PORT_SPEED_40G = 0x20,
	391	FUN_PORT_SPEED_50G = 0x40,
	392	FUN_PORT_SPEED_100G = 0x80,
	393	FUN_PORT_SPEED_200G = 0x100,
	394	};
	395
	396	enum fun_port_duplex_mode {
	397	FUN_PORT_FULL_DUPLEX = 0x0,
	398	FUN_PORT_HALF_DUPLEX = 0x1,
	399	};
	400
	401	enum {
	402	FUN_PORT_FEC_NA = 0x0,
	403	FUN_PORT_FEC_OFF = 0x1,
	404	FUN_PORT_FEC_RS = 0x2,
	405	FUN_PORT_FEC_FC = 0x4,
	406	FUN_PORT_FEC_AUTO = 0x8,
	407	};
	408
	409	enum fun_port_link_status {
	410	FUN_PORT_LINK_UP = 0x0,
	411	FUN_PORT_LINK_UP_WITH_ERR = 0x1,
	412	FUN_PORT_LINK_DOWN = 0x2,
	413	};
	414
	415	enum fun_port_led_type {
	416	FUN_PORT_LED_OFF = 0x0,
	417	FUN_PORT_LED_AMBER = 0x1,
	418	FUN_PORT_LED_GREEN = 0x2,
	419	FUN_PORT_LED_BEACON_ON = 0x3,
	420	FUN_PORT_LED_BEACON_OFF = 0x4,
	421	};
	422
	423	enum {
	424	FUN_PORT_FLAG_MAC_DOWN = 0x1,
	425	FUN_PORT_FLAG_MAC_UP = 0x2,
	426	FUN_PORT_FLAG_NH_DOWN = 0x4,
	427	FUN_PORT_FLAG_NH_UP = 0x8,
	428	};
	429
	430	enum {
	431	FUN_PORT_FLAG_ENABLE_NOTIFY = 0x1,
	432	};
	433
	434	enum fun_port_lane_attr {
	435	FUN_PORT_LANE_1 = 0x1,
	436	FUN_PORT_LANE_2 = 0x2,
	437	FUN_PORT_LANE_4 = 0x4,
	438	FUN_PORT_LANE_SPEED_10G = 0x100,
	439	FUN_PORT_LANE_SPEED_25G = 0x200,
	440	FUN_PORT_LANE_SPEED_50G = 0x400,
	441	FUN_PORT_LANE_SPLIT = 0x8000,
	442	};
	443
	444	enum fun_admin_port_subop {
	445	FUN_ADMIN_PORT_SUBOP_INETADDR_EVENT = 0x24,
	446	};
	447
	448	enum fun_admin_port_key {
	449	FUN_ADMIN_PORT_KEY_ILLEGAL = 0x0,
	450	FUN_ADMIN_PORT_KEY_MTU = 0x1,
	451	FUN_ADMIN_PORT_KEY_FEC = 0x2,
	452	FUN_ADMIN_PORT_KEY_SPEED = 0x3,
	453	FUN_ADMIN_PORT_KEY_DEBOUNCE = 0x4,
	454	FUN_ADMIN_PORT_KEY_DUPLEX = 0x5,
	455	FUN_ADMIN_PORT_KEY_MACADDR = 0x6,
	456	FUN_ADMIN_PORT_KEY_LINKMODE = 0x7,
	457	FUN_ADMIN_PORT_KEY_BREAKOUT = 0x8,
	458	FUN_ADMIN_PORT_KEY_ENABLE = 0x9,
	459	FUN_ADMIN_PORT_KEY_DISABLE = 0xa,
	460	FUN_ADMIN_PORT_KEY_ERR_DISABLE = 0xb,
	461	FUN_ADMIN_PORT_KEY_CAPABILITIES = 0xc,
	462	FUN_ADMIN_PORT_KEY_LP_CAPABILITIES = 0xd,
	463	FUN_ADMIN_PORT_KEY_STATS_DMA_LOW = 0xe,
	464	FUN_ADMIN_PORT_KEY_STATS_DMA_HIGH = 0xf,
	465	FUN_ADMIN_PORT_KEY_LANE_ATTRS = 0x10,
	466	FUN_ADMIN_PORT_KEY_LED = 0x11,
	467	FUN_ADMIN_PORT_KEY_ADVERT = 0x12,
	468	};
	469
	470	struct fun_subop_imm {
	471	__u8 subop; /* see fun_data_subop enum */
	472	__u8 flags;
	473	__u8 nsgl;
	474	__u8 rsvd0;
	475	__be32 len;
	476
	477	__u8 data[];
	478	};
	479
	480	enum fun_subop_sgl_flags {
	481	FUN_SUBOP_SGL_USE_OFF8 = 0x1,
	482	FUN_SUBOP_FLAG_FREE_BUF = 0x2,
	483	FUN_SUBOP_FLAG_IS_REFBUF = 0x4,
	484	FUN_SUBOP_SGL_FLAG_LOCAL = 0x8,
	485	};
	486
	487	enum fun_data_op {
	488	FUN_DATAOP_INVALID = 0x0,
	489	FUN_DATAOP_SL = 0x1, /* scatter */
	490	FUN_DATAOP_GL = 0x2, /* gather */
	491	FUN_DATAOP_SGL = 0x3, /* scatter-gather */
	492	FUN_DATAOP_IMM = 0x4, /* immediate data */
	493	FUN_DATAOP_RQBUF = 0x8, /* rq buffer */
	494	};
	495
	496	struct fun_dataop_gl {
	497	__u8 subop;
	498	__u8 flags;
	499	__be16 sgl_off;
	500	__be32 sgl_len;
	501
	502	__be64 sgl_data;
	503	};
	504
	505	static inline void fun_dataop_gl_init(struct fun_dataop_gl *s, u8 flags,
	506	u16 sgl_off, u32 sgl_len, u64 sgl_data)
	507	{
	508	s->subop = FUN_DATAOP_GL;
	509	s->flags = flags;
	510	s->sgl_off = cpu_to_be16(sgl_off);
	511	s->sgl_len = cpu_to_be32(sgl_len);
	512	s->sgl_data = cpu_to_be64(sgl_data);
	513	}
	514
	515	struct fun_dataop_imm {
	516	__u8 subop;
	517	__u8 flags;
	518	__be16 rsvd0;
	519	__be32 sgl_len;
	520	};
	521
	522	struct fun_subop_sgl {
	523	__u8 subop;
	524	__u8 flags;
	525	__u8 nsgl;
	526	__u8 rsvd0;
	527	__be32 sgl_len;
	528
	529	__be64 sgl_data;
	530	};
	531
	532	#define FUN_SUBOP_SGL_INIT(_subop, _flags, _nsgl, _sgl_len, _sgl_data) \
	533	(struct fun_subop_sgl) { \
	534	.subop = (_subop), .flags = (_flags), .nsgl = (_nsgl), \
	535	.sgl_len = cpu_to_be32(_sgl_len), \
	536	.sgl_data = cpu_to_be64(_sgl_data), \
	537	}
	538
	539	struct fun_dataop_rqbuf {
	540	__u8 subop;
	541	__u8 rsvd0;
	542	__be16 cid;
	543	__be32 bufoff;
	544	};
	545
	546	struct fun_dataop_hdr {
	547	__u8 nsgl;
	548	__u8 flags;
	549	__u8 ngather;
	550	__u8 nscatter;
	551	__be32 total_len;
	552
	553	struct fun_dataop_imm imm[];
	554	};
	555
	556	#define FUN_DATAOP_HDR_INIT(_nsgl, _flags, _ngather, _nscatter, _total_len) \
	557	(struct fun_dataop_hdr) { \
	558	.nsgl = _nsgl, .flags = _flags, .ngather = _ngather, \
	559	.nscatter = _nscatter, .total_len = cpu_to_be32(_total_len), \
	560	}
	561
	562	enum fun_port_inetaddr_event_type {
	563	FUN_PORT_INETADDR_ADD = 0x1,
	564	FUN_PORT_INETADDR_DEL = 0x2,
	565	};
	566
	567	enum fun_port_inetaddr_addr_family {
	568	FUN_PORT_INETADDR_IPV4 = 0x1,
	569	FUN_PORT_INETADDR_IPV6 = 0x2,
	570	};
	571
	572	struct fun_admin_port_req {
	573	struct fun_admin_req_common common;
	574
	575	union port_req_subop {
	576	struct fun_admin_port_create_req {
	577	__u8 subop;
	578	__u8 rsvd0;
	579	__be16 flags;
	580	__be32 id;
	581	} create;
	582	struct fun_admin_port_write_req {
	583	__u8 subop;
	584	__u8 rsvd0;
	585	__be16 flags;
	586	__be32 id; /* portid */
	587
	588	struct fun_admin_write48_req write48[];
	589	} write;
	590	struct fun_admin_port_read_req {
	591	__u8 subop;
	592	__u8 rsvd0;
	593	__be16 flags;
	594	__be32 id; /* portid */
	595
	596	struct fun_admin_read48_req read48[];
	597	} read;
	598	struct fun_admin_port_inetaddr_event_req {
	599	__u8 subop;
	600	__u8 rsvd0;
	601	__u8 event_type;
	602	__u8 addr_family;
	603	__be32 id;
	604
	605	__u8 addr[];
	606	} inetaddr_event;
	607	} u;
	608	};
	609
	610	#define FUN_ADMIN_PORT_CREATE_REQ_INIT(_subop, _flags, _id) \
	611	(struct fun_admin_port_create_req) { \
	612	.subop = (_subop), .flags = cpu_to_be16(_flags), \
	613	.id = cpu_to_be32(_id), \
	614	}
	615
	616	#define FUN_ADMIN_PORT_WRITE_REQ_INIT(_subop, _flags, _id) \
	617	(struct fun_admin_port_write_req) { \
	618	.subop = (_subop), .flags = cpu_to_be16(_flags), \
	619	.id = cpu_to_be32(_id), \
	620	}
	621
	622	#define FUN_ADMIN_PORT_READ_REQ_INIT(_subop, _flags, _id) \
	623	(struct fun_admin_port_read_req) { \
	624	.subop = (_subop), .flags = cpu_to_be16(_flags), \
	625	.id = cpu_to_be32(_id), \
	626	}
	627
	628	struct fun_admin_port_rsp {
	629	struct fun_admin_rsp_common common;
	630
	631	union port_rsp_subop {
	632	struct fun_admin_port_create_rsp {
	633	__u8 subop;
	634	__u8 rsvd0[3];
	635	__be32 id;
	636
	637	__be16 lport;
	638	__u8 rsvd1[6];
	639	} create;
	640	struct fun_admin_port_write_rsp {
	641	__u8 subop;
	642	__u8 rsvd0[3];
	643	__be32 id; /* portid */
	644
	645	struct fun_admin_write48_rsp write48[];
	646	} write;
	647	struct fun_admin_port_read_rsp {
	648	__u8 subop;
	649	__u8 rsvd0[3];
	650	__be32 id; /* portid */
	651
	652	struct fun_admin_read48_rsp read48[];
	653	} read;
	654	struct fun_admin_port_inetaddr_event_rsp {
	655	__u8 subop;
	656	__u8 rsvd0[3];
	657	__be32 id; /* portid */
	658	} inetaddr_event;
	659	} u;
	660	};
	661
	662	enum fun_xcvr_type {
	663	FUN_XCVR_BASET = 0x0,
	664	FUN_XCVR_CU = 0x1,
	665	FUN_XCVR_SMF = 0x2,
	666	FUN_XCVR_MMF = 0x3,
	667	FUN_XCVR_AOC = 0x4,
	668	FUN_XCVR_SFPP = 0x10, /* SFP+ or later */
	669	FUN_XCVR_QSFPP = 0x11, /* QSFP+ or later */
	670	FUN_XCVR_QSFPDD = 0x12, /* QSFP-DD */
	671	};
	672
	673	struct fun_admin_port_notif {
	674	struct fun_admin_rsp_common common;
	675
	676	__u8 subop;
	677	__u8 rsvd0;
	678	__be16 id;
	679	__be32 speed; /* in 10 Mbps units */
	680
	681	__u8 link_state;
	682	__u8 missed_events;
	683	__u8 link_down_reason;
	684	__u8 xcvr_type;
	685	__u8 flow_ctrl;
	686	__u8 fec;
	687	__u8 active_lanes;
	688	__u8 rsvd1;
	689
	690	__be64 advertising;
	691
	692	__be64 lp_advertising;
	693	};
	694
	695	enum fun_eth_rss_const {
	696	FUN_ETH_RSS_MAX_KEY_SIZE = 0x28,
	697	FUN_ETH_RSS_MAX_INDIR_ENT = 0x40,
	698	};
	699
	700	enum fun_eth_hash_alg {
	701	FUN_ETH_RSS_ALG_INVALID = 0x0,
	702	FUN_ETH_RSS_ALG_TOEPLITZ = 0x1,
	703	FUN_ETH_RSS_ALG_CRC32 = 0x2,
	704	};
	705
	706	struct fun_admin_rss_req {
	707	struct fun_admin_req_common common;
	708
	709	union rss_req_subop {
	710	struct fun_admin_rss_create_req {
	711	__u8 subop;
	712	__u8 rsvd0;
	713	__be16 flags;
	714	__be32 id;
	715
	716	__be32 rsvd1;
	717	__be32 viid; /* VI flow id */
	718
	719	__be64 metadata[1];
	720
	721	__u8 alg;
	722	__u8 keylen;
	723	__u8 indir_nent;
	724	__u8 rsvd2;
	725	__be16 key_off;
	726	__be16 indir_off;
	727
	728	struct fun_dataop_hdr dataop;
	729	} create;
	730	} u;
	731	};
	732
	733	#define FUN_ADMIN_RSS_CREATE_REQ_INIT(_subop, _flags, _id, _viid, _alg, \
	734	_keylen, _indir_nent, _key_off, \
	735	_indir_off) \
	736	(struct fun_admin_rss_create_req) { \
	737	.subop = (_subop), .flags = cpu_to_be16(_flags), \
	738	.id = cpu_to_be32(_id), .viid = cpu_to_be32(_viid), \
	739	.alg = _alg, .keylen = _keylen, .indir_nent = _indir_nent, \
	740	.key_off = cpu_to_be16(_key_off), \
	741	.indir_off = cpu_to_be16(_indir_off), \
	742	}
	743
	744	struct fun_admin_vi_req {
	745	struct fun_admin_req_common common;
	746
	747	union vi_req_subop {
	748	struct fun_admin_vi_create_req {
	749	__u8 subop;
	750	__u8 rsvd0;
	751	__be16 flags;
	752	__be32 id;
	753
	754	__be32 rsvd1;
	755	__be32 portid; /* port flow id */
	756	} create;
	757	} u;
	758	};
	759
	760	#define FUN_ADMIN_VI_CREATE_REQ_INIT(_subop, _flags, _id, _portid) \
	761	(struct fun_admin_vi_create_req) { \
	762	.subop = (_subop), .flags = cpu_to_be16(_flags), \
	763	.id = cpu_to_be32(_id), .portid = cpu_to_be32(_portid), \
	764	}
	765
	766	struct fun_admin_eth_req {
	767	struct fun_admin_req_common common;
	768
	769	union eth_req_subop {
	770	struct fun_admin_eth_create_req {
	771	__u8 subop;
	772	__u8 rsvd0;
	773	__be16 flags;
	774	__be32 id;
	775
	776	__be32 rsvd1;
	777	__be32 portid; /* port flow id */
	778	} create;
	779	} u;
	780	};
	781
	782	#define FUN_ADMIN_ETH_CREATE_REQ_INIT(_subop, _flags, _id, _portid) \
	783	(struct fun_admin_eth_create_req) { \
	784	.subop = (_subop), .flags = cpu_to_be16(_flags), \
	785	.id = cpu_to_be32(_id), .portid = cpu_to_be32(_portid), \
	786	}
	787
	788	enum {
	789	FUN_ADMIN_SWU_UPGRADE_FLAG_INIT = 0x10,
	790	FUN_ADMIN_SWU_UPGRADE_FLAG_COMPLETE = 0x20,
	791	FUN_ADMIN_SWU_UPGRADE_FLAG_DOWNGRADE = 0x40,
	792	FUN_ADMIN_SWU_UPGRADE_FLAG_ACTIVE_IMAGE = 0x80,
	793	FUN_ADMIN_SWU_UPGRADE_FLAG_ASYNC = 0x1,
	794	};
	795
	796	enum fun_admin_swu_subop {
	797	FUN_ADMIN_SWU_SUBOP_GET_VERSION = 0x20,
	798	FUN_ADMIN_SWU_SUBOP_UPGRADE = 0x21,
	799	FUN_ADMIN_SWU_SUBOP_UPGRADE_DATA = 0x22,
	800	FUN_ADMIN_SWU_SUBOP_GET_ALL_VERSIONS = 0x23,
	801	};
	802
	803	struct fun_admin_swu_req {
	804	struct fun_admin_req_common common;
	805
	806	union swu_req_subop {
	807	struct fun_admin_swu_create_req {
	808	__u8 subop;
	809	__u8 rsvd0;
	810	__be16 flags;
	811	__be32 id;
	812	} create;
	813	struct fun_admin_swu_upgrade_req {
	814	__u8 subop;
	815	__u8 rsvd0;
	816	__be16 flags;
	817	__be32 id;
	818
	819	__be32 fourcc;
	820	__be32 rsvd1;
	821
	822	__be64 image_size; /* upgrade image length */
	823	} upgrade;
	824	struct fun_admin_swu_upgrade_data_req {
	825	__u8 subop;
	826	__u8 rsvd0;
	827	__be16 flags;
	828	__be32 id;
	829
	830	__be32 offset; /* offset of data in this command */
	831	__be32 size; /* total size of data in this command */
	832	} upgrade_data;
	833	} u;
	834
	835	struct fun_subop_sgl sgl[]; /* in, out buffers through sgl */
	836	};
	837
	838	#define FUN_ADMIN_SWU_CREATE_REQ_INIT(_subop, _flags, _id) \
	839	(struct fun_admin_swu_create_req) { \
	840	.subop = (_subop), .flags = cpu_to_be16(_flags), \
	841	.id = cpu_to_be32(_id), \
	842	}
	843
	844	#define FUN_ADMIN_SWU_UPGRADE_REQ_INIT(_subop, _flags, _id, _fourcc, \
	845	_image_size) \
	846	(struct fun_admin_swu_upgrade_req) { \
	847	.subop = (_subop), .flags = cpu_to_be16(_flags), \
	848	.id = cpu_to_be32(_id), .fourcc = cpu_to_be32(_fourcc), \
	849	.image_size = cpu_to_be64(_image_size), \
	850	}
	851
	852	#define FUN_ADMIN_SWU_UPGRADE_DATA_REQ_INIT(_subop, _flags, _id, _offset, \
	853	_size) \
	854	(struct fun_admin_swu_upgrade_data_req) { \
	855	.subop = (_subop), .flags = cpu_to_be16(_flags), \
	856	.id = cpu_to_be32(_id), .offset = cpu_to_be32(_offset), \
	857	.size = cpu_to_be32(_size), \
	858	}
	859
	860	struct fun_admin_swu_rsp {
	861	struct fun_admin_rsp_common common;
	862
	863	union swu_rsp_subop {
	864	struct fun_admin_swu_create_rsp {
	865	__u8 subop;
	866	__u8 rsvd0;
	867	__be16 flags;
	868	__be32 id;
	869	} create;
	870	struct fun_admin_swu_upgrade_rsp {
	871	__u8 subop;
	872	__u8 rsvd0[3];
	873	__be32 id;
	874
	875	__be32 fourcc;
	876	__be32 status;
	877
	878	__be32 progress;
	879	__be32 unused;
	880	} upgrade;
	881	struct fun_admin_swu_upgrade_data_rsp {
	882	__u8 subop;
	883	__u8 rsvd0;
	884	__be16 flags;
	885	__be32 id;
	886
	887	__be32 offset;
	888	__be32 size;
	889	} upgrade_data;
	890	} u;
	891	};
	892
	893	enum fun_ktls_version {
	894	FUN_KTLS_TLSV2 = 0x20,
	895	FUN_KTLS_TLSV3 = 0x30,
	896	};
	897
	898	enum fun_ktls_cipher {
	899	FUN_KTLS_CIPHER_AES_GCM_128 = 0x33,
	900	FUN_KTLS_CIPHER_AES_GCM_256 = 0x34,
	901	FUN_KTLS_CIPHER_AES_CCM_128 = 0x35,
	902	FUN_KTLS_CIPHER_CHACHA20_POLY1305 = 0x36,
	903	};
	904
	905	enum fun_ktls_modify_flags {
	906	FUN_KTLS_MODIFY_REMOVE = 0x1,
	907	};
	908
	909	struct fun_admin_ktls_create_req {
	910	struct fun_admin_req_common common;
	911
	912	__u8 subop;
	913	__u8 rsvd0;
	914	__be16 flags;
	915	__be32 id;
	916	};
	917
	918	#define FUN_ADMIN_KTLS_CREATE_REQ_INIT(_subop, _flags, _id) \
	919	(struct fun_admin_ktls_create_req) { \
	920	.subop = (_subop), .flags = cpu_to_be16(_flags), \
	921	.id = cpu_to_be32(_id), \
	922	}
	923
	924	struct fun_admin_ktls_create_rsp {
	925	struct fun_admin_rsp_common common;
	926
	927	__u8 subop;
	928	__u8 rsvd0[3];
	929	__be32 id;
	930	};
	931
	932	struct fun_admin_ktls_modify_req {
	933	struct fun_admin_req_common common;
	934
	935	__u8 subop;
	936	__u8 rsvd0;
	937	__be16 flags;
	938	__be32 id;
	939
	940	__be64 tlsid;
	941
	942	__be32 tcp_seq;
	943	__u8 version;
	944	__u8 cipher;
	945	__u8 rsvd1[2];
	946
	947	__u8 record_seq[8];
	948
	949	__u8 key[32];
	950
	951	__u8 iv[16];
	952
	953	__u8 salt[8];
	954	};
	955
	956	#define FUN_ADMIN_KTLS_MODIFY_REQ_INIT(_subop, _flags, _id, _tlsid, _tcp_seq, \
	957	_version, _cipher) \
	958	(struct fun_admin_ktls_modify_req) { \
	959	.subop = (_subop), .flags = cpu_to_be16(_flags), \
	960	.id = cpu_to_be32(_id), .tlsid = cpu_to_be64(_tlsid), \
	961	.tcp_seq = cpu_to_be32(_tcp_seq), .version = _version, \
	962	.cipher = _cipher, \
	963	}
	964
	965	struct fun_admin_ktls_modify_rsp {
	966	struct fun_admin_rsp_common common;
	967
	968	__u8 subop;
	969	__u8 rsvd0[3];
	970	__be32 id;
	971
	972	__be64 tlsid;
	973	};
	974
	975	struct fun_req_common {
	976	__u8 op;
	977	__u8 len8;
	978	__be16 flags;
	979	__u8 suboff8;
	980	__u8 rsvd0;
	981	__be16 cid;
	982	};
	983
	984	struct fun_rsp_common {
	985	__u8 op;
	986	__u8 len8;
	987	__be16 flags;
	988	__u8 suboff8;
	989	__u8 ret;
	990	__be16 cid;
	991	};
	992
	993	struct fun_cqe_info {
	994	__be16 sqhd;
	995	__be16 sqid;
	996	__be16 cid;
	997	__be16 sf_p;
	998	};
	999
	1000	enum fun_eprq_def {
	1001	FUN_EPRQ_PKT_ALIGN = 0x80,
	1002	};
	1003
	1004	struct fun_eprq_rqbuf {
	1005	__be64 bufaddr;
	1006	};
	1007
	1008	#define FUN_EPRQ_RQBUF_INIT(_bufaddr) \
	1009	(struct fun_eprq_rqbuf) { \
	1010	.bufaddr = cpu_to_be64(_bufaddr), \
	1011	}
	1012
	1013	enum fun_eth_op {
	1014	FUN_ETH_OP_TX = 0x1,
	1015	FUN_ETH_OP_RX = 0x2,
	1016	};
	1017
	1018	enum {
	1019	FUN_ETH_OFFLOAD_EN = 0x8000,
	1020	FUN_ETH_OUTER_EN = 0x4000,
	1021	FUN_ETH_INNER_LSO = 0x2000,
	1022	FUN_ETH_INNER_TSO = 0x1000,
	1023	FUN_ETH_OUTER_IPV6 = 0x800,
	1024	FUN_ETH_OUTER_UDP = 0x400,
	1025	FUN_ETH_INNER_IPV6 = 0x200,
	1026	FUN_ETH_INNER_UDP = 0x100,
	1027	FUN_ETH_UPDATE_OUTER_L3_LEN = 0x80,
	1028	FUN_ETH_UPDATE_OUTER_L3_CKSUM = 0x40,
	1029	FUN_ETH_UPDATE_OUTER_L4_LEN = 0x20,
	1030	FUN_ETH_UPDATE_OUTER_L4_CKSUM = 0x10,
	1031	FUN_ETH_UPDATE_INNER_L3_LEN = 0x8,
	1032	FUN_ETH_UPDATE_INNER_L3_CKSUM = 0x4,
	1033	FUN_ETH_UPDATE_INNER_L4_LEN = 0x2,
	1034	FUN_ETH_UPDATE_INNER_L4_CKSUM = 0x1,
	1035	};
	1036
	1037	struct fun_eth_offload {
	1038	__be16 flags; /* combination of above flags */
	1039	__be16 mss; /* TSO max seg size */
	1040	__be16 tcp_doff_flags; /* TCP data offset + flags 16b word */
	1041	__be16 vlan;
	1042
	1043	__be16 inner_l3_off; /* Inner L3 header offset */
	1044	__be16 inner_l4_off; /* Inner L4 header offset */
	1045	__be16 outer_l3_off; /* Outer L3 header offset */
	1046	__be16 outer_l4_off; /* Outer L4 header offset */
	1047	};
	1048
	1049	static inline void fun_eth_offload_init(struct fun_eth_offload *s, u16 flags,
	1050	u16 mss, __be16 tcp_doff_flags,
	1051	__be16 vlan, u16 inner_l3_off,
	1052	u16 inner_l4_off, u16 outer_l3_off,
	1053	u16 outer_l4_off)
	1054	{
	1055	s->flags = cpu_to_be16(flags);
	1056	s->mss = cpu_to_be16(mss);
	1057	s->tcp_doff_flags = tcp_doff_flags;
	1058	s->vlan = vlan;
	1059	s->inner_l3_off = cpu_to_be16(inner_l3_off);
	1060	s->inner_l4_off = cpu_to_be16(inner_l4_off);
	1061	s->outer_l3_off = cpu_to_be16(outer_l3_off);
	1062	s->outer_l4_off = cpu_to_be16(outer_l4_off);
	1063	}
	1064
	1065	struct fun_eth_tls {
	1066	__be64 tlsid;
	1067	};
	1068
	1069	enum {
	1070	FUN_ETH_TX_TLS = 0x8000,
	1071	};
	1072
	1073	struct fun_eth_tx_req {
	1074	__u8 op;
	1075	__u8 len8;
	1076	__be16 flags;
	1077	__u8 suboff8;
	1078	__u8 repr_idn;
	1079	__be16 encap_proto;
	1080
	1081	struct fun_eth_offload offload;
	1082
	1083	struct fun_dataop_hdr dataop;
	1084	};
	1085
	1086	struct fun_eth_rx_cv {
	1087	__be16 il4_prot_to_l2_type;
	1088	};
	1089
	1090	#define FUN_ETH_RX_CV_IL4_PROT_S 13U
	1091	#define FUN_ETH_RX_CV_IL4_PROT_M 0x3
	1092
	1093	#define FUN_ETH_RX_CV_IL3_PROT_S 11U
	1094	#define FUN_ETH_RX_CV_IL3_PROT_M 0x3
	1095
	1096	#define FUN_ETH_RX_CV_OL4_PROT_S 8U
	1097	#define FUN_ETH_RX_CV_OL4_PROT_M 0x7
	1098
	1099	#define FUN_ETH_RX_CV_ENCAP_TYPE_S 6U
	1100	#define FUN_ETH_RX_CV_ENCAP_TYPE_M 0x3
	1101
	1102	#define FUN_ETH_RX_CV_OL3_PROT_S 4U
	1103	#define FUN_ETH_RX_CV_OL3_PROT_M 0x3
	1104
	1105	#define FUN_ETH_RX_CV_VLAN_TYPE_S 3U
	1106	#define FUN_ETH_RX_CV_VLAN_TYPE_M 0x1
	1107
	1108	#define FUN_ETH_RX_CV_L2_TYPE_S 2U
	1109	#define FUN_ETH_RX_CV_L2_TYPE_M 0x1
	1110
	1111	enum fun_rx_cv {
	1112	FUN_RX_CV_NONE = 0x0,
	1113	FUN_RX_CV_IP = 0x2,
	1114	FUN_RX_CV_IP6 = 0x3,
	1115	FUN_RX_CV_TCP = 0x2,
	1116	FUN_RX_CV_UDP = 0x3,
	1117	FUN_RX_CV_VXLAN = 0x2,
	1118	FUN_RX_CV_MPLS = 0x3,
	1119	};
	1120
	1121	struct fun_eth_cqe {
	1122	__u8 op;
	1123	__u8 len8;
	1124	__u8 nsgl;
	1125	__u8 repr_idn;
	1126	__be32 pkt_len;
	1127
	1128	__be64 timestamp;
	1129
	1130	__be16 pkt_cv;
	1131	__be16 rsvd0;
	1132	__be32 hash;
	1133
	1134	__be16 encap_proto;
	1135	__be16 vlan;
	1136	__be32 rsvd1;
	1137
	1138	__be32 buf_offset;
	1139	__be16 headroom;
	1140	__be16 csum;
	1141	};
	1142
	1143	enum fun_admin_adi_attr {
	1144	FUN_ADMIN_ADI_ATTR_MACADDR = 0x1,
	1145	FUN_ADMIN_ADI_ATTR_VLAN = 0x2,
	1146	FUN_ADMIN_ADI_ATTR_RATE = 0x3,
	1147	};
	1148
	1149	struct fun_adi_param {
	1150	union adi_param {
	1151	struct fun_adi_mac {
	1152	__be64 addr;
	1153	} mac;
	1154	struct fun_adi_vlan {
	1155	__be32 rsvd;
	1156	__be16 eth_type;
	1157	__be16 tci;
	1158	} vlan;
	1159	struct fun_adi_rate {
	1160	__be32 rsvd;
	1161	__be32 tx_mbps;
	1162	} rate;
	1163	} u;
	1164	};
	1165
	1166	#define FUN_ADI_MAC_INIT(_addr) \
	1167	(struct fun_adi_mac) { \
	1168	.addr = cpu_to_be64(_addr), \
	1169	}
	1170
	1171	#define FUN_ADI_VLAN_INIT(_eth_type, _tci) \
	1172	(struct fun_adi_vlan) { \
	1173	.eth_type = cpu_to_be16(_eth_type), .tci = cpu_to_be16(_tci), \
	1174	}
	1175
	1176	#define FUN_ADI_RATE_INIT(_tx_mbps) \
	1177	(struct fun_adi_rate) { \
	1178	.tx_mbps = cpu_to_be32(_tx_mbps), \
	1179	}
	1180
	1181	struct fun_admin_adi_req {
	1182	struct fun_admin_req_common common;
	1183
	1184	union adi_req_subop {
	1185	struct fun_admin_adi_write_req {
	1186	__u8 subop;
	1187	__u8 attribute;
	1188	__be16 rsvd;
	1189	__be32 id;
	1190
	1191	struct fun_adi_param param;
	1192	} write;
	1193	} u;
	1194	};
	1195
	1196	#define FUN_ADMIN_ADI_WRITE_REQ_INIT(_subop, _attribute, _id) \
	1197	(struct fun_admin_adi_write_req) { \
	1198	.subop = (_subop), .attribute = (_attribute), \
	1199	.id = cpu_to_be32(_id), \
	1200	}
	1201
	1202	#endif /* __FUN_HCI_H */

drivers/net/ethernet/fungible/funcore/fun_queue.c+601−0

@@ -0,0 +1,601 @@
	1	// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
	2
	3	#include <linux/dma-mapping.h>
	4	#include <linux/interrupt.h>
	5	#include <linux/log2.h>
	6	#include <linux/mm.h>
	7	#include <linux/netdevice.h>
	8	#include <linux/pci.h>
	9	#include <linux/slab.h>
	10
	11	#include "fun_dev.h"
	12	#include "fun_queue.h"
	13
	14	/* Allocate memory for a queue. This includes the memory for the HW descriptor
	15	* ring, an optional 64b HW write-back area, and an optional SW state ring.
	16	* Returns the virtual and DMA addresses of the HW ring, the VA of the SW ring,
	17	* and the VA of the write-back area.
	18	*/
	19	void fun_alloc_ring_mem(struct device dma_dev, size_t depth,
	20	size_t hw_desc_sz, size_t sw_desc_sz, bool wb,
	21	int numa_node, dma_addr_t dma_addr, void *sw_va,
	22	volatile __be64 **wb_va)
	23	{
	24	int dev_node = dev_to_node(dma_dev);
	25	size_t dma_sz;
	26	void *va;
	27
	28	if (numa_node == NUMA_NO_NODE)
	29	numa_node = dev_node;
	30
	31	/* Place optional write-back area at end of descriptor ring. */
	32	dma_sz = hw_desc_sz * depth;
	33	if (wb)
	34	dma_sz += sizeof(u64);
	35
	36	set_dev_node(dma_dev, numa_node);
	37	va = dma_alloc_coherent(dma_dev, dma_sz, dma_addr, GFP_KERNEL);
	38	set_dev_node(dma_dev, dev_node);
	39	if (!va)
	40	return NULL;
	41
	42	if (sw_desc_sz) {
	43	sw_va = kvzalloc_node(sw_desc_sz depth, GFP_KERNEL,
	44	numa_node);
	45	if (!*sw_va) {
	46	dma_free_coherent(dma_dev, dma_sz, va, *dma_addr);
	47	return NULL;
	48	}
	49	}
	50
	51	if (wb)
	52	*wb_va = va + dma_sz - sizeof(u64);
	53	return va;
	54	}
	55	EXPORT_SYMBOL_GPL(fun_alloc_ring_mem);
	56
	57	void fun_free_ring_mem(struct device *dma_dev, size_t depth, size_t hw_desc_sz,
	58	bool wb, void hw_va, dma_addr_t dma_addr, void sw_va)
	59	{
	60	if (hw_va) {
	61	size_t sz = depth * hw_desc_sz;
	62
	63	if (wb)
	64	sz += sizeof(u64);
	65	dma_free_coherent(dma_dev, sz, hw_va, dma_addr);
	66	}
	67	kvfree(sw_va);
	68	}
	69	EXPORT_SYMBOL_GPL(fun_free_ring_mem);
	70
	71	/* Prepare and issue an admin command to create an SQ on the device with the
	72	* provided parameters. If the queue ID is auto-allocated by the device it is
	73	* returned in *sqidp.
	74	*/
	75	int fun_sq_create(struct fun_dev *fdev, u16 flags, u32 sqid, u32 cqid,
	76	u8 sqe_size_log2, u32 sq_depth, dma_addr_t dma_addr,
	77	u8 coal_nentries, u8 coal_usec, u32 irq_num,
	78	u32 scan_start_id, u32 scan_end_id,
	79	u32 rq_buf_size_log2, u32 sqidp, u32 __iomem *dbp)
	80	{
	81	union {
	82	struct fun_admin_epsq_req req;
	83	struct fun_admin_generic_create_rsp rsp;
	84	} cmd;
	85	dma_addr_t wb_addr;
	86	u32 hw_qid;
	87	int rc;
	88
	89	if (sq_depth > fdev->q_depth)
	90	return -EINVAL;
	91	if (flags & FUN_ADMIN_EPSQ_CREATE_FLAG_RQ)
	92	sqe_size_log2 = ilog2(sizeof(struct fun_eprq_rqbuf));
	93
	94	wb_addr = dma_addr + (sq_depth << sqe_size_log2);
	95
	96	cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_EPSQ,
	97	sizeof(cmd.req));
	98	cmd.req.u.create =
	99	FUN_ADMIN_EPSQ_CREATE_REQ_INIT(FUN_ADMIN_SUBOP_CREATE, flags,
	100	sqid, cqid, sqe_size_log2,
	101	sq_depth - 1, dma_addr, 0,
	102	coal_nentries, coal_usec,
	103	irq_num, scan_start_id,
	104	scan_end_id, 0,
	105	rq_buf_size_log2,
	106	ilog2(sizeof(u64)), wb_addr);
	107
	108	rc = fun_submit_admin_sync_cmd(fdev, &cmd.req.common,
	109	&cmd.rsp, sizeof(cmd.rsp), 0);
	110	if (rc)
	111	return rc;
	112
	113	hw_qid = be32_to_cpu(cmd.rsp.id);
	114	*dbp = fun_sq_db_addr(fdev, hw_qid);
	115	if (flags & FUN_ADMIN_RES_CREATE_FLAG_ALLOCATOR)
	116	*sqidp = hw_qid;
	117	return rc;
	118	}
	119	EXPORT_SYMBOL_GPL(fun_sq_create);
	120
	121	/* Prepare and issue an admin command to create a CQ on the device with the
	122	* provided parameters. If the queue ID is auto-allocated by the device it is
	123	* returned in *cqidp.
	124	*/
	125	int fun_cq_create(struct fun_dev *fdev, u16 flags, u32 cqid, u32 rqid,
	126	u8 cqe_size_log2, u32 cq_depth, dma_addr_t dma_addr,
	127	u16 headroom, u16 tailroom, u8 coal_nentries, u8 coal_usec,
	128	u32 irq_num, u32 scan_start_id, u32 scan_end_id, u32 *cqidp,
	129	u32 __iomem **dbp)
	130	{
	131	union {
	132	struct fun_admin_epcq_req req;
	133	struct fun_admin_generic_create_rsp rsp;
	134	} cmd;
	135	u32 hw_qid;
	136	int rc;
	137
	138	if (cq_depth > fdev->q_depth)
	139	return -EINVAL;
	140
	141	cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_EPCQ,
	142	sizeof(cmd.req));
	143	cmd.req.u.create =
	144	FUN_ADMIN_EPCQ_CREATE_REQ_INIT(FUN_ADMIN_SUBOP_CREATE, flags,
	145	cqid, rqid, cqe_size_log2,
	146	cq_depth - 1, dma_addr, tailroom,
	147	headroom / 2, 0, coal_nentries,
	148	coal_usec, irq_num,
	149	scan_start_id, scan_end_id, 0);
	150
	151	rc = fun_submit_admin_sync_cmd(fdev, &cmd.req.common,
	152	&cmd.rsp, sizeof(cmd.rsp), 0);
	153	if (rc)
	154	return rc;
	155
	156	hw_qid = be32_to_cpu(cmd.rsp.id);
	157	*dbp = fun_cq_db_addr(fdev, hw_qid);
	158	if (flags & FUN_ADMIN_RES_CREATE_FLAG_ALLOCATOR)
	159	*cqidp = hw_qid;
	160	return rc;
	161	}
	162	EXPORT_SYMBOL_GPL(fun_cq_create);
	163
	164	static bool fun_sq_is_head_wb(const struct fun_queue *funq)
	165	{
	166	return funq->sq_flags & FUN_ADMIN_EPSQ_CREATE_FLAG_HEAD_WB_ADDRESS;
	167	}
	168
	169	static void fun_clean_rq(struct fun_queue *funq)
	170	{
	171	struct fun_dev *fdev = funq->fdev;
	172	struct fun_rq_info *rqinfo;
	173	unsigned int i;
	174
	175	for (i = 0; i < funq->rq_depth; i++) {
	176	rqinfo = &funq->rq_info[i];
	177	if (rqinfo->page) {
	178	dma_unmap_page(fdev->dev, rqinfo->dma, PAGE_SIZE,
	179	DMA_FROM_DEVICE);
	180	put_page(rqinfo->page);
	181	rqinfo->page = NULL;
	182	}
	183	}
	184	}
	185
	186	static int fun_fill_rq(struct fun_queue *funq)
	187	{
	188	struct device *dev = funq->fdev->dev;
	189	int i, node = dev_to_node(dev);
	190	struct fun_rq_info *rqinfo;
	191
	192	for (i = 0; i < funq->rq_depth; i++) {
	193	rqinfo = &funq->rq_info[i];
	194	rqinfo->page = alloc_pages_node(node, GFP_KERNEL, 0);
	195	if (unlikely(!rqinfo->page))
	196	return -ENOMEM;
	197
	198	rqinfo->dma = dma_map_page(dev, rqinfo->page, 0,
	199	PAGE_SIZE, DMA_FROM_DEVICE);
	200	if (unlikely(dma_mapping_error(dev, rqinfo->dma))) {
	201	put_page(rqinfo->page);
	202	rqinfo->page = NULL;
	203	return -ENOMEM;
	204	}
	205
	206	funq->rqes[i] = FUN_EPRQ_RQBUF_INIT(rqinfo->dma);
	207	}
	208
	209	funq->rq_tail = funq->rq_depth - 1;
	210	return 0;
	211	}
	212
	213	static void fun_rq_update_pos(struct fun_queue *funq, int buf_offset)
	214	{
	215	if (buf_offset <= funq->rq_buf_offset) {
	216	struct fun_rq_info *rqinfo = &funq->rq_info[funq->rq_buf_idx];
	217	struct device *dev = funq->fdev->dev;
	218
	219	dma_sync_single_for_device(dev, rqinfo->dma, PAGE_SIZE,
	220	DMA_FROM_DEVICE);
	221	funq->num_rqe_to_fill++;
	222	if (++funq->rq_buf_idx == funq->rq_depth)
	223	funq->rq_buf_idx = 0;
	224	}
	225	funq->rq_buf_offset = buf_offset;
	226	}
	227
	228	/* Given a command response with data scattered across >= 1 RQ buffers return
	229	* a pointer to a contiguous buffer containing all the data. If the data is in
	230	* one RQ buffer the start address within that buffer is returned, otherwise a
	231	* new buffer is allocated and the data is gathered into it.
	232	*/
	233	static void fun_data_from_rq(struct fun_queue funq,
	234	const struct fun_rsp_common rsp, bool need_free)
	235	{
	236	u32 bufoff, total_len, remaining, fragsize, dataoff;
	237	struct device *dma_dev = funq->fdev->dev;
	238	const struct fun_dataop_rqbuf *databuf;
	239	const struct fun_dataop_hdr *dataop;
	240	const struct fun_rq_info *rqinfo;
	241	void *data;
	242
	243	dataop = (void )rsp + rsp->suboff8 8;
	244	total_len = be32_to_cpu(dataop->total_len);
	245
	246	if (likely(dataop->nsgl == 1)) {
	247	databuf = (struct fun_dataop_rqbuf *)dataop->imm;
	248	bufoff = be32_to_cpu(databuf->bufoff);
	249	fun_rq_update_pos(funq, bufoff);
	250	rqinfo = &funq->rq_info[funq->rq_buf_idx];
	251	dma_sync_single_for_cpu(dma_dev, rqinfo->dma + bufoff,
	252	total_len, DMA_FROM_DEVICE);
	253	*need_free = false;
	254	return page_address(rqinfo->page) + bufoff;
	255	}
	256
	257	/* For scattered completions gather the fragments into one buffer. */
	258
	259	data = kmalloc(total_len, GFP_ATOMIC);
	260	/* NULL is OK here. In case of failure we still need to consume the data
	261	* for proper buffer accounting but indicate an error in the response.
	262	*/
	263	if (likely(data))
	264	*need_free = true;
	265
	266	dataoff = 0;
	267	for (remaining = total_len; remaining; remaining -= fragsize) {
	268	fun_rq_update_pos(funq, 0);
	269	fragsize = min_t(unsigned int, PAGE_SIZE, remaining);
	270	if (data) {
	271	rqinfo = &funq->rq_info[funq->rq_buf_idx];
	272	dma_sync_single_for_cpu(dma_dev, rqinfo->dma, fragsize,
	273	DMA_FROM_DEVICE);
	274	memcpy(data + dataoff, page_address(rqinfo->page),
	275	fragsize);
	276	dataoff += fragsize;
	277	}
	278	}
	279	return data;
	280	}
	281
	282	unsigned int __fun_process_cq(struct fun_queue *funq, unsigned int max)
	283	{
	284	const struct fun_cqe_info *info;
	285	struct fun_rsp_common *rsp;
	286	unsigned int new_cqes;
	287	u16 sf_p, flags;
	288	bool need_free;
	289	void *cqe;
	290
	291	if (!max)
	292	max = funq->cq_depth - 1;
	293
	294	for (new_cqes = 0; new_cqes < max; new_cqes++) {
	295	cqe = funq->cqes + (funq->cq_head << funq->cqe_size_log2);
	296	info = funq_cqe_info(funq, cqe);
	297	sf_p = be16_to_cpu(info->sf_p);
	298
	299	if ((sf_p & 1) != funq->cq_phase)
	300	break;
	301
	302	/* ensure the phase tag is read before other CQE fields */
	303	dma_rmb();
	304
	305	if (++funq->cq_head == funq->cq_depth) {
	306	funq->cq_head = 0;
	307	funq->cq_phase = !funq->cq_phase;
	308	}
	309
	310	rsp = cqe;
	311	flags = be16_to_cpu(rsp->flags);
	312
	313	need_free = false;
	314	if (unlikely(flags & FUN_REQ_COMMON_FLAG_CQE_IN_RQBUF)) {
	315	rsp = fun_data_from_rq(funq, rsp, &need_free);
	316	if (!rsp) {
	317	rsp = cqe;
	318	rsp->len8 = 1;
	319	if (rsp->ret == 0)
	320	rsp->ret = ENOMEM;
	321	}
	322	}
	323
	324	if (funq->cq_cb)
	325	funq->cq_cb(funq, funq->cb_data, rsp, info);
	326	if (need_free)
	327	kfree(rsp);
	328	}
	329
	330	dev_dbg(funq->fdev->dev, "CQ %u, new CQEs %u/%u, head %u, phase %u\n",
	331	funq->cqid, new_cqes, max, funq->cq_head, funq->cq_phase);
	332	return new_cqes;
	333	}
	334
	335	unsigned int fun_process_cq(struct fun_queue *funq, unsigned int max)
	336	{
	337	unsigned int processed;
	338	u32 db;
	339
	340	processed = __fun_process_cq(funq, max);
	341
	342	if (funq->num_rqe_to_fill) {
	343	funq->rq_tail = (funq->rq_tail + funq->num_rqe_to_fill) %
	344	funq->rq_depth;
	345	funq->num_rqe_to_fill = 0;
	346	writel(funq->rq_tail, funq->rq_db);
	347	}
	348
	349	db = funq->cq_head \| FUN_DB_IRQ_ARM_F;
	350	writel(db, funq->cq_db);
	351	return processed;
	352	}
	353
	354	static int fun_alloc_sqes(struct fun_queue *funq)
	355	{
	356	funq->sq_cmds = fun_alloc_ring_mem(funq->fdev->dev, funq->sq_depth,
	357	1 << funq->sqe_size_log2, 0,
	358	fun_sq_is_head_wb(funq),
	359	NUMA_NO_NODE, &funq->sq_dma_addr,
	360	NULL, &funq->sq_head);
	361	return funq->sq_cmds ? 0 : -ENOMEM;
	362	}
	363
	364	static int fun_alloc_cqes(struct fun_queue *funq)
	365	{
	366	funq->cqes = fun_alloc_ring_mem(funq->fdev->dev, funq->cq_depth,
	367	1 << funq->cqe_size_log2, 0, false,
	368	NUMA_NO_NODE, &funq->cq_dma_addr, NULL,
	369	NULL);
	370	return funq->cqes ? 0 : -ENOMEM;
	371	}
	372
	373	static int fun_alloc_rqes(struct fun_queue *funq)
	374	{
	375	funq->rqes = fun_alloc_ring_mem(funq->fdev->dev, funq->rq_depth,
	376	sizeof(*funq->rqes),
	377	sizeof(*funq->rq_info), false,
	378	NUMA_NO_NODE, &funq->rq_dma_addr,
	379	(void **)&funq->rq_info, NULL);
	380	return funq->rqes ? 0 : -ENOMEM;
	381	}
	382
	383	/* Free a queue's structures. */
	384	void fun_free_queue(struct fun_queue *funq)
	385	{
	386	struct device *dev = funq->fdev->dev;
	387
	388	fun_free_ring_mem(dev, funq->cq_depth, 1 << funq->cqe_size_log2, false,
	389	funq->cqes, funq->cq_dma_addr, NULL);
	390	fun_free_ring_mem(dev, funq->sq_depth, 1 << funq->sqe_size_log2,
	391	fun_sq_is_head_wb(funq), funq->sq_cmds,
	392	funq->sq_dma_addr, NULL);
	393
	394	if (funq->rqes) {
	395	fun_clean_rq(funq);
	396	fun_free_ring_mem(dev, funq->rq_depth, sizeof(*funq->rqes),
	397	false, funq->rqes, funq->rq_dma_addr,
	398	funq->rq_info);
	399	}
	400
	401	kfree(funq);
	402	}
	403
	404	/* Allocate and initialize a funq's structures. */
	405	struct fun_queue fun_alloc_queue(struct fun_dev fdev, int qid,
	406	const struct fun_queue_alloc_req *req)
	407	{
	408	struct fun_queue funq = kzalloc(sizeof(funq), GFP_KERNEL);
	409
	410	if (!funq)
	411	return NULL;
	412
	413	funq->fdev = fdev;
	414	spin_lock_init(&funq->sq_lock);
	415
	416	funq->qid = qid;
	417
	418	/* Initial CQ/SQ/RQ ids */
	419	if (req->rq_depth) {
	420	funq->cqid = 2 * qid;
	421	if (funq->qid) {
	422	/* I/O Q: use rqid = cqid, sqid = +1 */
	423	funq->rqid = funq->cqid;
	424	funq->sqid = funq->rqid + 1;
	425	} else {
	426	/* Admin Q: sqid is always 0, use ID 1 for RQ */
	427	funq->sqid = 0;
	428	funq->rqid = 1;
	429	}
	430	} else {
	431	funq->cqid = qid;
	432	funq->sqid = qid;
	433	}
	434
	435	funq->cq_flags = req->cq_flags;
	436	funq->sq_flags = req->sq_flags;
	437
	438	funq->cqe_size_log2 = req->cqe_size_log2;
	439	funq->sqe_size_log2 = req->sqe_size_log2;
	440
	441	funq->cq_depth = req->cq_depth;
	442	funq->sq_depth = req->sq_depth;
	443
	444	funq->cq_intcoal_nentries = req->cq_intcoal_nentries;
	445	funq->cq_intcoal_usec = req->cq_intcoal_usec;
	446
	447	funq->sq_intcoal_nentries = req->sq_intcoal_nentries;
	448	funq->sq_intcoal_usec = req->sq_intcoal_usec;
	449
	450	if (fun_alloc_cqes(funq))
	451	goto free_funq;
	452
	453	funq->cq_phase = 1;
	454
	455	if (fun_alloc_sqes(funq))
	456	goto free_funq;
	457
	458	if (req->rq_depth) {
	459	funq->rq_flags = req->rq_flags \| FUN_ADMIN_EPSQ_CREATE_FLAG_RQ;
	460	funq->rq_depth = req->rq_depth;
	461	funq->rq_buf_offset = -1;
	462
	463	if (fun_alloc_rqes(funq) \|\| fun_fill_rq(funq))
	464	goto free_funq;
	465	}
	466
	467	funq->cq_vector = -1;
	468	funq->cqe_info_offset = (1 << funq->cqe_size_log2) - sizeof(struct fun_cqe_info);
	469
	470	/* SQ/CQ 0 are implicitly created, assign their doorbells now.
	471	* Other queues are assigned doorbells at their explicit creation.
	472	*/
	473	if (funq->sqid == 0)
	474	funq->sq_db = fun_sq_db_addr(fdev, 0);
	475	if (funq->cqid == 0)
	476	funq->cq_db = fun_cq_db_addr(fdev, 0);
	477
	478	return funq;
	479
	480	free_funq:
	481	fun_free_queue(funq);
	482	return NULL;
	483	}
	484
	485	/* Create a funq's CQ on the device. */
	486	static int fun_create_cq(struct fun_queue *funq)
	487	{
	488	struct fun_dev *fdev = funq->fdev;
	489	unsigned int rqid;
	490	int rc;
	491
	492	rqid = funq->cq_flags & FUN_ADMIN_EPCQ_CREATE_FLAG_RQ ?
	493	funq->rqid : FUN_HCI_ID_INVALID;
	494	rc = fun_cq_create(fdev, funq->cq_flags, funq->cqid, rqid,
	495	funq->cqe_size_log2, funq->cq_depth,
	496	funq->cq_dma_addr, 0, 0, funq->cq_intcoal_nentries,
	497	funq->cq_intcoal_usec, funq->cq_vector, 0, 0,
	498	&funq->cqid, &funq->cq_db);
	499	if (!rc)
	500	dev_dbg(fdev->dev, "created CQ %u\n", funq->cqid);
	501
	502	return rc;
	503	}
	504
	505	/* Create a funq's SQ on the device. */
	506	static int fun_create_sq(struct fun_queue *funq)
	507	{
	508	struct fun_dev *fdev = funq->fdev;
	509	int rc;
	510
	511	rc = fun_sq_create(fdev, funq->sq_flags, funq->sqid, funq->cqid,
	512	funq->sqe_size_log2, funq->sq_depth,
	513	funq->sq_dma_addr, funq->sq_intcoal_nentries,
	514	funq->sq_intcoal_usec, funq->cq_vector, 0, 0,
	515	0, &funq->sqid, &funq->sq_db);
	516	if (!rc)
	517	dev_dbg(fdev->dev, "created SQ %u\n", funq->sqid);
	518
	519	return rc;
	520	}
	521
	522	/* Create a funq's RQ on the device. */
	523	int fun_create_rq(struct fun_queue *funq)
	524	{
	525	struct fun_dev *fdev = funq->fdev;
	526	int rc;
	527
	528	rc = fun_sq_create(fdev, funq->rq_flags, funq->rqid, funq->cqid, 0,
	529	funq->rq_depth, funq->rq_dma_addr, 0, 0,
	530	funq->cq_vector, 0, 0, PAGE_SHIFT, &funq->rqid,
	531	&funq->rq_db);
	532	if (!rc)
	533	dev_dbg(fdev->dev, "created RQ %u\n", funq->rqid);
	534
	535	return rc;
	536	}
	537
	538	static unsigned int funq_irq(struct fun_queue *funq)
	539	{
	540	return pci_irq_vector(to_pci_dev(funq->fdev->dev), funq->cq_vector);
	541	}
	542
	543	int fun_request_irq(struct fun_queue funq, const char devname,
	544	irq_handler_t handler, void *data)
	545	{
	546	int rc;
	547
	548	if (funq->cq_vector < 0)
	549	return -EINVAL;
	550
	551	funq->irq_handler = handler;
	552	funq->irq_data = data;
	553
	554	snprintf(funq->irqname, sizeof(funq->irqname),
	555	funq->qid ? "%s-q[%d]" : "%s-adminq", devname, funq->qid);
	556
	557	rc = request_irq(funq_irq(funq), handler, 0, funq->irqname, data);
	558	if (rc)
	559	funq->irq_handler = NULL;
	560
	561	return rc;
	562	}
	563
	564	/* Create all component queues of a funq on the device. */
	565	int fun_create_queue(struct fun_queue *funq)
	566	{
	567	int rc;
	568
	569	rc = fun_create_cq(funq);
	570	if (rc)
	571	return rc;
	572
	573	if (funq->rq_depth) {
	574	rc = fun_create_rq(funq);
	575	if (rc)
	576	goto release_cq;
	577	}
	578
	579	rc = fun_create_sq(funq);
	580	if (rc)
	581	goto release_rq;
	582
	583	return 0;
	584
	585	release_rq:
	586	fun_destroy_sq(funq->fdev, funq->rqid);
	587	release_cq:
	588	fun_destroy_cq(funq->fdev, funq->cqid);
	589	return rc;
	590	}
	591
	592	void fun_free_irq(struct fun_queue *funq)
	593	{
	594	if (funq->irq_handler) {
	595	unsigned int vector = funq_irq(funq);
	596
	597	free_irq(vector, funq->irq_data);
	598	funq->irq_handler = NULL;
	599	funq->irq_data = NULL;
	600	}
	601	}

drivers/net/ethernet/fungible/funcore/fun_queue.h+175−0

@@ -0,0 +1,175 @@
	1	/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) */
	2
	3	#ifndef _FUN_QEUEUE_H
	4	#define _FUN_QEUEUE_H
	5
	6	#include <linux/interrupt.h>
	7	#include <linux/io.h>
	8
	9	struct device;
	10	struct fun_dev;
	11	struct fun_queue;
	12	struct fun_cqe_info;
	13	struct fun_rsp_common;
	14
	15	typedef void (cq_callback_t)(struct fun_queue funq, void data, void msg,
	16	const struct fun_cqe_info *info);
	17
	18	struct fun_rq_info {
	19	dma_addr_t dma;
	20	struct page *page;
	21	};
	22
	23	/* A queue group consisting of an SQ, a CQ, and an optional RQ. */
	24	struct fun_queue {
	25	struct fun_dev *fdev;
	26	spinlock_t sq_lock;
	27
	28	dma_addr_t cq_dma_addr;
	29	dma_addr_t sq_dma_addr;
	30	dma_addr_t rq_dma_addr;
	31
	32	u32 __iomem *cq_db;
	33	u32 __iomem *sq_db;
	34	u32 __iomem *rq_db;
	35
	36	void *cqes;
	37	void *sq_cmds;
	38	struct fun_eprq_rqbuf *rqes;
	39	struct fun_rq_info *rq_info;
	40
	41	u32 cqid;
	42	u32 sqid;
	43	u32 rqid;
	44
	45	u32 cq_depth;
	46	u32 sq_depth;
	47	u32 rq_depth;
	48
	49	u16 cq_head;
	50	u16 sq_tail;
	51	u16 rq_tail;
	52
	53	u8 cqe_size_log2;
	54	u8 sqe_size_log2;
	55
	56	u16 cqe_info_offset;
	57
	58	u16 rq_buf_idx;
	59	int rq_buf_offset;
	60	u16 num_rqe_to_fill;
	61
	62	u8 cq_intcoal_usec;
	63	u8 cq_intcoal_nentries;
	64	u8 sq_intcoal_usec;
	65	u8 sq_intcoal_nentries;
	66
	67	u16 cq_flags;
	68	u16 sq_flags;
	69	u16 rq_flags;
	70
	71	/* SQ head writeback */
	72	u16 sq_comp;
	73
	74	volatile __be64 *sq_head;
	75
	76	cq_callback_t cq_cb;
	77	void *cb_data;
	78
	79	irq_handler_t irq_handler;
	80	void *irq_data;
	81	s16 cq_vector;
	82	u8 cq_phase;
	83
	84	/* I/O q index */
	85	u16 qid;
	86
	87	char irqname[24];
	88	};
	89
	90	static inline void fun_sqe_at(const struct fun_queue funq, unsigned int pos)
	91	{
	92	return funq->sq_cmds + (pos << funq->sqe_size_log2);
	93	}
	94
	95	static inline void funq_sq_post_tail(struct fun_queue *funq, u16 tail)
	96	{
	97	if (++tail == funq->sq_depth)
	98	tail = 0;
	99	funq->sq_tail = tail;
	100	writel(tail, funq->sq_db);
	101	}
	102
	103	static inline struct fun_cqe_info funq_cqe_info(const struct fun_queue funq,
	104	void *cqe)
	105	{
	106	return cqe + funq->cqe_info_offset;
	107	}
	108
	109	static inline void funq_rq_post(struct fun_queue *funq)
	110	{
	111	writel(funq->rq_tail, funq->rq_db);
	112	}
	113
	114	struct fun_queue_alloc_req {
	115	u8 cqe_size_log2;
	116	u8 sqe_size_log2;
	117
	118	u16 cq_flags;
	119	u16 sq_flags;
	120	u16 rq_flags;
	121
	122	u32 cq_depth;
	123	u32 sq_depth;
	124	u32 rq_depth;
	125
	126	u8 cq_intcoal_usec;
	127	u8 cq_intcoal_nentries;
	128	u8 sq_intcoal_usec;
	129	u8 sq_intcoal_nentries;
	130	};
	131
	132	int fun_sq_create(struct fun_dev *fdev, u16 flags, u32 sqid, u32 cqid,
	133	u8 sqe_size_log2, u32 sq_depth, dma_addr_t dma_addr,
	134	u8 coal_nentries, u8 coal_usec, u32 irq_num,
	135	u32 scan_start_id, u32 scan_end_id,
	136	u32 rq_buf_size_log2, u32 sqidp, u32 __iomem *dbp);
	137	int fun_cq_create(struct fun_dev *fdev, u16 flags, u32 cqid, u32 rqid,
	138	u8 cqe_size_log2, u32 cq_depth, dma_addr_t dma_addr,
	139	u16 headroom, u16 tailroom, u8 coal_nentries, u8 coal_usec,
	140	u32 irq_num, u32 scan_start_id, u32 scan_end_id,
	141	u32 cqidp, u32 __iomem *dbp);
	142	void fun_alloc_ring_mem(struct device dma_dev, size_t depth,
	143	size_t hw_desc_sz, size_t sw_desc_size, bool wb,
	144	int numa_node, dma_addr_t dma_addr, void *sw_va,
	145	volatile __be64 **wb_va);
	146	void fun_free_ring_mem(struct device *dma_dev, size_t depth, size_t hw_desc_sz,
	147	bool wb, void hw_va, dma_addr_t dma_addr, void sw_va);
	148
	149	#define fun_destroy_sq(fdev, sqid) \
	150	fun_res_destroy((fdev), FUN_ADMIN_OP_EPSQ, 0, (sqid))
	151	#define fun_destroy_cq(fdev, cqid) \
	152	fun_res_destroy((fdev), FUN_ADMIN_OP_EPCQ, 0, (cqid))
	153
	154	struct fun_queue fun_alloc_queue(struct fun_dev fdev, int qid,
	155	const struct fun_queue_alloc_req *req);
	156	void fun_free_queue(struct fun_queue *funq);
	157
	158	static inline void fun_set_cq_callback(struct fun_queue *funq, cq_callback_t cb,
	159	void *cb_data)
	160	{
	161	funq->cq_cb = cb;
	162	funq->cb_data = cb_data;
	163	}
	164
	165	int fun_create_rq(struct fun_queue *funq);
	166	int fun_create_queue(struct fun_queue *funq);
	167
	168	void fun_free_irq(struct fun_queue *funq);
	169	int fun_request_irq(struct fun_queue funq, const char devname,
	170	irq_handler_t handler, void *data);
	171
	172	unsigned int __fun_process_cq(struct fun_queue *funq, unsigned int max);
	173	unsigned int fun_process_cq(struct fun_queue *funq, unsigned int max);
	174
	175	#endif /* _FUN_QEUEUE_H */