rdma/siw: application interface

Broken up commit to add the Soft iWarp RDMA driver.

Bernard Metzler committed 7 years ago

Commit 303ae1c1 parent 6c52fdc4 files · +2037 −0Browse files

Unified · Split

4 files

drivers/infiniband/sw/siw/siw_verbs.c +1760 −0
drivers/infiniband/sw/siw/siw_verbs.h +91 −0
include/uapi/rdma/rdma_user_ioctl_cmds.h +1 −0
include/uapi/rdma/siw-abi.h +185 −0

drivers/infiniband/sw/siw/siw_verbs.c+1760−0

@@ -0,0 +1,1760 @@
	1	+// SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
	2	+
	3	+/* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
	4	+/* Copyright (c) 2008-2019, IBM Corporation */
	5	+
	6	+#include <linux/errno.h>
	7	+#include <linux/types.h>
	8	+#include <linux/uaccess.h>
	9	+#include <linux/vmalloc.h>
	10	+#include <linux/xarray.h>
	11	+
	12	+#include <rdma/iw_cm.h>
	13	+#include <rdma/ib_verbs.h>
	14	+#include <rdma/ib_user_verbs.h>
	15	+#include <rdma/uverbs_ioctl.h>
	16	+
	17	+#include "siw.h"
	18	+#include "siw_verbs.h"
	19	+#include "siw_mem.h"
	20	+
	21	+static int ib_qp_state_to_siw_qp_state[IB_QPS_ERR + 1] = {
	22	+ [IB_QPS_RESET] = SIW_QP_STATE_IDLE,
	23	+ [IB_QPS_INIT] = SIW_QP_STATE_IDLE,
	24	+ [IB_QPS_RTR] = SIW_QP_STATE_RTR,
	25	+ [IB_QPS_RTS] = SIW_QP_STATE_RTS,
	26	+ [IB_QPS_SQD] = SIW_QP_STATE_CLOSING,
	27	+ [IB_QPS_SQE] = SIW_QP_STATE_TERMINATE,
	28	+ [IB_QPS_ERR] = SIW_QP_STATE_ERROR
	29	+};
	30	+
	31	+static char ib_qp_state_to_string[IB_QPS_ERR + 1][sizeof("RESET")] = {
	32	+ [IB_QPS_RESET] = "RESET", [IB_QPS_INIT] = "INIT", [IB_QPS_RTR] = "RTR",
	33	+ [IB_QPS_RTS] = "RTS", [IB_QPS_SQD] = "SQD", [IB_QPS_SQE] = "SQE",
	34	+ [IB_QPS_ERR] = "ERR"
	35	+};
	36	+
	37	+static u32 siw_create_uobj(struct siw_ucontext uctx, void vaddr, u32 size)
	38	+{
	39	+ struct siw_uobj *uobj;
	40	+ struct xa_limit limit = XA_LIMIT(0, SIW_UOBJ_MAX_KEY);
	41	+ u32 key;
	42	+
	43	+ uobj = kzalloc(sizeof(*uobj), GFP_KERNEL);
	44	+ if (!uobj)
	45	+ return SIW_INVAL_UOBJ_KEY;
	46	+
	47	+ if (xa_alloc_cyclic(&uctx->xa, &key, uobj, limit, &uctx->uobj_nextkey,
	48	+ GFP_KERNEL) < 0) {
	49	+ kfree(uobj);
	50	+ return SIW_INVAL_UOBJ_KEY;
	51	+ }
	52	+ uobj->size = PAGE_ALIGN(size);
	53	+ uobj->addr = vaddr;
	54	+
	55	+ return key;
	56	+}
	57	+
	58	+static struct siw_uobj siw_get_uobj(struct siw_ucontext uctx,
	59	+ unsigned long off, u32 size)
	60	+{
	61	+ struct siw_uobj *uobj = xa_load(&uctx->xa, off);
	62	+
	63	+ if (uobj && uobj->size == size)
	64	+ return uobj;
	65	+
	66	+ return NULL;
	67	+}
	68	+
	69	+int siw_mmap(struct ib_ucontext ctx, struct vm_area_struct vma)
	70	+{
	71	+ struct siw_ucontext *uctx = to_siw_ctx(ctx);
	72	+ struct siw_uobj *uobj;
	73	+ unsigned long off = vma->vm_pgoff;
	74	+ int size = vma->vm_end - vma->vm_start;
	75	+ int rv = -EINVAL;
	76	+
	77	+ /*
	78	+ * Must be page aligned
	79	+ */
	80	+ if (vma->vm_start & (PAGE_SIZE - 1)) {
	81	+ pr_warn("siw: mmap not page aligned\n");
	82	+ goto out;
	83	+ }
	84	+ uobj = siw_get_uobj(uctx, off, size);
	85	+ if (!uobj) {
	86	+ siw_dbg(&uctx->sdev->base_dev, "mmap lookup failed: %lu, %u\n",
	87	+ off, size);
	88	+ goto out;
	89	+ }
	90	+ rv = remap_vmalloc_range(vma, uobj->addr, 0);
	91	+ if (rv)
	92	+ pr_warn("remap_vmalloc_range failed: %lu, %u\n", off, size);
	93	+out:
	94	+ return rv;
	95	+}
	96	+
	97	+int siw_alloc_ucontext(struct ib_ucontext base_ctx, struct ib_udata udata)
	98	+{
	99	+ struct siw_device *sdev = to_siw_dev(base_ctx->device);
	100	+ struct siw_ucontext *ctx = to_siw_ctx(base_ctx);
	101	+ struct siw_uresp_alloc_ctx uresp = {};
	102	+ int rv;
	103	+
	104	+ if (atomic_inc_return(&sdev->num_ctx) > SIW_MAX_CONTEXT) {
	105	+ rv = -ENOMEM;
	106	+ goto err_out;
	107	+ }
	108	+ xa_init_flags(&ctx->xa, XA_FLAGS_ALLOC);
	109	+ ctx->uobj_nextkey = 0;
	110	+ ctx->sdev = sdev;
	111	+
	112	+ uresp.dev_id = sdev->vendor_part_id;
	113	+
	114	+ if (udata->outlen < sizeof(uresp)) {
	115	+ rv = -EINVAL;
	116	+ goto err_out;
	117	+ }
	118	+ rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
	119	+ if (rv)
	120	+ goto err_out;
	121	+
	122	+ siw_dbg(base_ctx->device, "success. now %d context(s)\n",
	123	+ atomic_read(&sdev->num_ctx));
	124	+
	125	+ return 0;
	126	+
	127	+err_out:
	128	+ atomic_dec(&sdev->num_ctx);
	129	+ siw_dbg(base_ctx->device, "failure %d. now %d context(s)\n", rv,
	130	+ atomic_read(&sdev->num_ctx));
	131	+
	132	+ return rv;
	133	+}
	134	+
	135	+void siw_dealloc_ucontext(struct ib_ucontext *base_ctx)
	136	+{
	137	+ struct siw_ucontext *uctx = to_siw_ctx(base_ctx);
	138	+ void *entry;
	139	+ unsigned long index;
	140	+
	141	+ /*
	142	+ * Make sure all user mmap objects are gone. Since QP, CQ
	143	+ * and SRQ destroy routines destroy related objects, nothing
	144	+ * should be found here.
	145	+ */
	146	+ xa_for_each(&uctx->xa, index, entry) {
	147	+ kfree(xa_erase(&uctx->xa, index));
	148	+ pr_warn("siw: dropping orphaned uobj at %lu\n", index);
	149	+ }
	150	+ xa_destroy(&uctx->xa);
	151	+ atomic_dec(&uctx->sdev->num_ctx);
	152	+}
	153	+
	154	+int siw_query_device(struct ib_device base_dev, struct ib_device_attr attr,
	155	+ struct ib_udata *udata)
	156	+{
	157	+ struct siw_device *sdev = to_siw_dev(base_dev);
	158	+
	159	+ if (udata->inlen \|\| udata->outlen)
	160	+ return -EINVAL;
	161	+
	162	+ memset(attr, 0, sizeof(*attr));
	163	+
	164	+ /* Revisit atomic caps if RFC 7306 gets supported */
	165	+ attr->atomic_cap = 0;
	166	+ attr->device_cap_flags =
	167	+ IB_DEVICE_MEM_MGT_EXTENSIONS \| IB_DEVICE_ALLOW_USER_UNREG;
	168	+ attr->max_cq = sdev->attrs.max_cq;
	169	+ attr->max_cqe = sdev->attrs.max_cqe;
	170	+ attr->max_fast_reg_page_list_len = SIW_MAX_SGE_PBL;
	171	+ attr->max_fmr = sdev->attrs.max_fmr;
	172	+ attr->max_mr = sdev->attrs.max_mr;
	173	+ attr->max_mw = sdev->attrs.max_mw;
	174	+ attr->max_mr_size = ~0ull;
	175	+ attr->max_pd = sdev->attrs.max_pd;
	176	+ attr->max_qp = sdev->attrs.max_qp;
	177	+ attr->max_qp_init_rd_atom = sdev->attrs.max_ird;
	178	+ attr->max_qp_rd_atom = sdev->attrs.max_ord;
	179	+ attr->max_qp_wr = sdev->attrs.max_qp_wr;
	180	+ attr->max_recv_sge = sdev->attrs.max_sge;
	181	+ attr->max_res_rd_atom = sdev->attrs.max_qp * sdev->attrs.max_ird;
	182	+ attr->max_send_sge = sdev->attrs.max_sge;
	183	+ attr->max_sge_rd = sdev->attrs.max_sge_rd;
	184	+ attr->max_srq = sdev->attrs.max_srq;
	185	+ attr->max_srq_sge = sdev->attrs.max_srq_sge;
	186	+ attr->max_srq_wr = sdev->attrs.max_srq_wr;
	187	+ attr->page_size_cap = PAGE_SIZE;
	188	+ attr->vendor_id = SIW_VENDOR_ID;
	189	+ attr->vendor_part_id = sdev->vendor_part_id;
	190	+
	191	+ memcpy(&attr->sys_image_guid, sdev->netdev->dev_addr, 6);
	192	+
	193	+ return 0;
	194	+}
	195	+
	196	+int siw_query_port(struct ib_device *base_dev, u8 port,
	197	+ struct ib_port_attr *attr)
	198	+{
	199	+ struct siw_device *sdev = to_siw_dev(base_dev);
	200	+
	201	+ memset(attr, 0, sizeof(*attr));
	202	+
	203	+ attr->active_mtu = attr->max_mtu;
	204	+ attr->active_speed = 2;
	205	+ attr->active_width = 2;
	206	+ attr->gid_tbl_len = 1;
	207	+ attr->max_msg_sz = -1;
	208	+ attr->max_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
	209	+ attr->phys_state = sdev->state == IB_PORT_ACTIVE ? 5 : 3;
	210	+ attr->pkey_tbl_len = 1;
	211	+ attr->port_cap_flags = IB_PORT_CM_SUP \| IB_PORT_DEVICE_MGMT_SUP;
	212	+ attr->state = sdev->state;
	213	+ /*
	214	+ * All zero
	215	+ *
	216	+ * attr->lid = 0;
	217	+ * attr->bad_pkey_cntr = 0;
	218	+ * attr->qkey_viol_cntr = 0;
	219	+ * attr->sm_lid = 0;
	220	+ * attr->lmc = 0;
	221	+ * attr->max_vl_num = 0;
	222	+ * attr->sm_sl = 0;
	223	+ * attr->subnet_timeout = 0;
	224	+ * attr->init_type_repy = 0;
	225	+ */
	226	+ return 0;
	227	+}
	228	+
	229	+int siw_get_port_immutable(struct ib_device *base_dev, u8 port,
	230	+ struct ib_port_immutable *port_immutable)
	231	+{
	232	+ struct ib_port_attr attr;
	233	+ int rv = siw_query_port(base_dev, port, &attr);
	234	+
	235	+ if (rv)
	236	+ return rv;
	237	+
	238	+ port_immutable->pkey_tbl_len = attr.pkey_tbl_len;
	239	+ port_immutable->gid_tbl_len = attr.gid_tbl_len;
	240	+ port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
	241	+
	242	+ return 0;
	243	+}
	244	+
	245	+int siw_query_pkey(struct ib_device base_dev, u8 port, u16 idx, u16 pkey)
	246	+{
	247	+ /* Report the default pkey */
	248	+ *pkey = 0xffff;
	249	+ return 0;
	250	+}
	251	+
	252	+int siw_query_gid(struct ib_device *base_dev, u8 port, int idx,
	253	+ union ib_gid *gid)
	254	+{
	255	+ struct siw_device *sdev = to_siw_dev(base_dev);
	256	+
	257	+ /* subnet_prefix == interface_id == 0; */
	258	+ memset(gid, 0, sizeof(*gid));
	259	+ memcpy(&gid->raw[0], sdev->netdev->dev_addr, 6);
	260	+
	261	+ return 0;
	262	+}
	263	+
	264	+int siw_alloc_pd(struct ib_pd pd, struct ib_udata udata)
	265	+{
	266	+ struct siw_device *sdev = to_siw_dev(pd->device);
	267	+
	268	+ if (atomic_inc_return(&sdev->num_pd) > SIW_MAX_PD) {
	269	+ atomic_dec(&sdev->num_pd);
	270	+ return -ENOMEM;
	271	+ }
	272	+ siw_dbg_pd(pd, "now %d PD's(s)\n", atomic_read(&sdev->num_pd));
	273	+
	274	+ return 0;
	275	+}
	276	+
	277	+void siw_dealloc_pd(struct ib_pd pd, struct ib_udata udata)
	278	+{
	279	+ struct siw_device *sdev = to_siw_dev(pd->device);
	280	+
	281	+ siw_dbg_pd(pd, "free PD\n");
	282	+ atomic_dec(&sdev->num_pd);
	283	+}
	284	+
	285	+void siw_qp_get_ref(struct ib_qp *base_qp)
	286	+{
	287	+ siw_qp_get(to_siw_qp(base_qp));
	288	+}
	289	+
	290	+void siw_qp_put_ref(struct ib_qp *base_qp)
	291	+{
	292	+ siw_qp_put(to_siw_qp(base_qp));
	293	+}
	294	+
	295	+/*
	296	+ * siw_create_qp()
	297	+ *
	298	+ * Create QP of requested size on given device.
	299	+ *
	300	+ * @pd: Protection Domain
	301	+ * @attrs: Initial QP attributes.
	302	+ * @udata: used to provide QP ID, SQ and RQ size back to user.
	303	+ */
	304	+
	305	+struct ib_qp siw_create_qp(struct ib_pd pd,
	306	+ struct ib_qp_init_attr *attrs,
	307	+ struct ib_udata *udata)
	308	+{
	309	+ struct siw_qp *qp = NULL;
	310	+ struct siw_base_qp *siw_base_qp = NULL;
	311	+ struct ib_device *base_dev = pd->device;
	312	+ struct siw_device *sdev = to_siw_dev(base_dev);
	313	+ struct siw_ucontext *uctx =
	314	+ rdma_udata_to_drv_context(udata, struct siw_ucontext,
	315	+ base_ucontext);
	316	+ struct siw_cq scq = NULL, rcq = NULL;
	317	+ unsigned long flags;
	318	+ int num_sqe, num_rqe, rv = 0;
	319	+
	320	+ siw_dbg(base_dev, "create new QP\n");
	321	+
	322	+ if (atomic_inc_return(&sdev->num_qp) > SIW_MAX_QP) {
	323	+ siw_dbg(base_dev, "too many QP's\n");
	324	+ rv = -ENOMEM;
	325	+ goto err_out;
	326	+ }
	327	+ if (attrs->qp_type != IB_QPT_RC) {
	328	+ siw_dbg(base_dev, "only RC QP's supported\n");
	329	+ rv = -EINVAL;
	330	+ goto err_out;
	331	+ }
	332	+ if ((attrs->cap.max_send_wr > SIW_MAX_QP_WR) \|\|
	333	+ (attrs->cap.max_recv_wr > SIW_MAX_QP_WR) \|\|
	334	+ (attrs->cap.max_send_sge > SIW_MAX_SGE) \|\|
	335	+ (attrs->cap.max_recv_sge > SIW_MAX_SGE)) {
	336	+ siw_dbg(base_dev, "QP size error\n");
	337	+ rv = -EINVAL;
	338	+ goto err_out;
	339	+ }
	340	+ if (attrs->cap.max_inline_data > SIW_MAX_INLINE) {
	341	+ siw_dbg(base_dev, "max inline send: %d > %d\n",
	342	+ attrs->cap.max_inline_data, (int)SIW_MAX_INLINE);
	343	+ rv = -EINVAL;
	344	+ goto err_out;
	345	+ }
	346	+ /*
	347	+ * NOTE: we allow for zero element SQ and RQ WQE's SGL's
	348	+ * but not for a QP unable to hold any WQE (SQ + RQ)
	349	+ */
	350	+ if (attrs->cap.max_send_wr + attrs->cap.max_recv_wr == 0) {
	351	+ siw_dbg(base_dev, "QP must have send or receive queue\n");
	352	+ rv = -EINVAL;
	353	+ goto err_out;
	354	+ }
	355	+ scq = to_siw_cq(attrs->send_cq);
	356	+ rcq = to_siw_cq(attrs->recv_cq);
	357	+
	358	+ if (!scq \|\| (!rcq && !attrs->srq)) {
	359	+ siw_dbg(base_dev, "send CQ or receive CQ invalid\n");
	360	+ rv = -EINVAL;
	361	+ goto err_out;
	362	+ }
	363	+ siw_base_qp = kzalloc(sizeof(*siw_base_qp), GFP_KERNEL);
	364	+ if (!siw_base_qp) {
	365	+ rv = -ENOMEM;
	366	+ goto err_out;
	367	+ }
	368	+ qp = kzalloc(sizeof(*qp), GFP_KERNEL);
	369	+ if (!qp) {
	370	+ rv = -ENOMEM;
	371	+ goto err_out;
	372	+ }
	373	+ siw_base_qp->qp = qp;
	374	+ qp->ib_qp = &siw_base_qp->base_qp;
	375	+
	376	+ init_rwsem(&qp->state_lock);
	377	+ spin_lock_init(&qp->sq_lock);
	378	+ spin_lock_init(&qp->rq_lock);
	379	+ spin_lock_init(&qp->orq_lock);
	380	+
	381	+ qp->kernel_verbs = !udata;
	382	+ qp->xa_sq_index = SIW_INVAL_UOBJ_KEY;
	383	+ qp->xa_rq_index = SIW_INVAL_UOBJ_KEY;
	384	+
	385	+ rv = siw_qp_add(sdev, qp);
	386	+ if (rv)
	387	+ goto err_out;
	388	+
	389	+ /* All queue indices are derived from modulo operations
	390	+ * on a free running 'get' (consumer) and 'put' (producer)
	391	+ * unsigned counter. Having queue sizes at power of two
	392	+ * avoids handling counter wrap around.
	393	+ */
	394	+ num_sqe = roundup_pow_of_two(attrs->cap.max_send_wr);
	395	+ num_rqe = roundup_pow_of_two(attrs->cap.max_recv_wr);
	396	+
	397	+ if (qp->kernel_verbs)
	398	+ qp->sendq = vzalloc(num_sqe * sizeof(struct siw_sqe));
	399	+ else
	400	+ qp->sendq = vmalloc_user(num_sqe * sizeof(struct siw_sqe));
	401	+
	402	+ if (qp->sendq == NULL) {
	403	+ siw_dbg(base_dev, "SQ size %d alloc failed\n", num_sqe);
	404	+ rv = -ENOMEM;
	405	+ goto err_out_xa;
	406	+ }
	407	+ if (attrs->sq_sig_type != IB_SIGNAL_REQ_WR) {
	408	+ if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
	409	+ qp->attrs.flags \|= SIW_SIGNAL_ALL_WR;
	410	+ else {
	411	+ rv = -EINVAL;
	412	+ goto err_out_xa;
	413	+ }
	414	+ }
	415	+ qp->pd = pd;
	416	+ qp->scq = scq;
	417	+ qp->rcq = rcq;
	418	+
	419	+ if (attrs->srq) {
	420	+ /*
	421	+ * SRQ support.
	422	+ * Verbs 6.3.7: ignore RQ size, if SRQ present
	423	+ * Verbs 6.3.5: do not check PD of SRQ against PD of QP
	424	+ */
	425	+ qp->srq = to_siw_srq(attrs->srq);
	426	+ qp->attrs.rq_size = 0;
	427	+ siw_dbg(base_dev, "QP [%u]: [SRQ 0x%p] attached\n",
	428	+ qp->qp_num, qp->srq);
	429	+ } else if (num_rqe) {
	430	+ if (qp->kernel_verbs)
	431	+ qp->recvq = vzalloc(num_rqe * sizeof(struct siw_rqe));
	432	+ else
	433	+ qp->recvq =
	434	+ vmalloc_user(num_rqe * sizeof(struct siw_rqe));
	435	+
	436	+ if (qp->recvq == NULL) {
	437	+ siw_dbg(base_dev, "RQ size %d alloc failed\n", num_rqe);
	438	+ rv = -ENOMEM;
	439	+ goto err_out_xa;
	440	+ }
	441	+ qp->attrs.rq_size = num_rqe;
	442	+ }
	443	+ qp->attrs.sq_size = num_sqe;
	444	+ qp->attrs.sq_max_sges = attrs->cap.max_send_sge;
	445	+ qp->attrs.rq_max_sges = attrs->cap.max_recv_sge;
	446	+
	447	+ /* Make those two tunables fixed for now. */
	448	+ qp->tx_ctx.gso_seg_limit = 1;
	449	+ qp->tx_ctx.zcopy_tx = zcopy_tx;
	450	+
	451	+ qp->attrs.state = SIW_QP_STATE_IDLE;
	452	+
	453	+ if (udata) {
	454	+ struct siw_uresp_create_qp uresp = {};
	455	+
	456	+ uresp.num_sqe = num_sqe;
	457	+ uresp.num_rqe = num_rqe;
	458	+ uresp.qp_id = qp_id(qp);
	459	+
	460	+ if (qp->sendq) {
	461	+ qp->xa_sq_index =
	462	+ siw_create_uobj(uctx, qp->sendq,
	463	+ num_sqe * sizeof(struct siw_sqe));
	464	+ }
	465	+ if (qp->recvq) {
	466	+ qp->xa_rq_index =
	467	+ siw_create_uobj(uctx, qp->recvq,
	468	+ num_rqe * sizeof(struct siw_rqe));
	469	+ }
	470	+ if (qp->xa_sq_index == SIW_INVAL_UOBJ_KEY \|\|
	471	+ qp->xa_rq_index == SIW_INVAL_UOBJ_KEY) {
	472	+ rv = -ENOMEM;
	473	+ goto err_out_xa;
	474	+ }
	475	+ uresp.sq_key = qp->xa_sq_index << PAGE_SHIFT;
	476	+ uresp.rq_key = qp->xa_rq_index << PAGE_SHIFT;
	477	+
	478	+ if (udata->outlen < sizeof(uresp)) {
	479	+ rv = -EINVAL;
	480	+ goto err_out_xa;
	481	+ }
	482	+ rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
	483	+ if (rv)
	484	+ goto err_out_xa;
	485	+ }
	486	+ qp->tx_cpu = siw_get_tx_cpu(sdev);
	487	+ if (qp->tx_cpu < 0) {
	488	+ rv = -EINVAL;
	489	+ goto err_out_xa;
	490	+ }
	491	+ INIT_LIST_HEAD(&qp->devq);
	492	+ spin_lock_irqsave(&sdev->lock, flags);
	493	+ list_add_tail(&qp->devq, &sdev->qp_list);
	494	+ spin_unlock_irqrestore(&sdev->lock, flags);
	495	+
	496	+ return qp->ib_qp;
	497	+
	498	+err_out_xa:
	499	+ xa_erase(&sdev->qp_xa, qp_id(qp));
	500	+err_out:
	501	+ kfree(siw_base_qp);
	502	+
	503	+ if (qp) {
	504	+ if (qp->xa_sq_index != SIW_INVAL_UOBJ_KEY)
	505	+ kfree(xa_erase(&uctx->xa, qp->xa_sq_index));
	506	+ if (qp->xa_rq_index != SIW_INVAL_UOBJ_KEY)
	507	+ kfree(xa_erase(&uctx->xa, qp->xa_rq_index));
	508	+
	509	+ vfree(qp->sendq);
	510	+ vfree(qp->recvq);
	511	+ kfree(qp);
	512	+ }
	513	+ atomic_dec(&sdev->num_qp);
	514	+
	515	+ return ERR_PTR(rv);
	516	+}
	517	+
	518	+/*
	519	+ * Minimum siw_query_qp() verb interface.
	520	+ *
	521	+ * @qp_attr_mask is not used but all available information is provided
	522	+ */
	523	+int siw_query_qp(struct ib_qp base_qp, struct ib_qp_attr qp_attr,
	524	+ int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
	525	+{
	526	+ struct siw_qp *qp;
	527	+ struct siw_device *sdev;
	528	+
	529	+ if (base_qp && qp_attr && qp_init_attr) {
	530	+ qp = to_siw_qp(base_qp);
	531	+ sdev = to_siw_dev(base_qp->device);
	532	+ } else {
	533	+ return -EINVAL;
	534	+ }
	535	+ qp_attr->cap.max_inline_data = SIW_MAX_INLINE;
	536	+ qp_attr->cap.max_send_wr = qp->attrs.sq_size;
	537	+ qp_attr->cap.max_send_sge = qp->attrs.sq_max_sges;
	538	+ qp_attr->cap.max_recv_wr = qp->attrs.rq_size;
	539	+ qp_attr->cap.max_recv_sge = qp->attrs.rq_max_sges;
	540	+ qp_attr->path_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
	541	+ qp_attr->max_rd_atomic = qp->attrs.irq_size;
	542	+ qp_attr->max_dest_rd_atomic = qp->attrs.orq_size;
	543	+
	544	+ qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE \|
	545	+ IB_ACCESS_REMOTE_WRITE \|
	546	+ IB_ACCESS_REMOTE_READ;
	547	+
	548	+ qp_init_attr->qp_type = base_qp->qp_type;
	549	+ qp_init_attr->send_cq = base_qp->send_cq;
	550	+ qp_init_attr->recv_cq = base_qp->recv_cq;
	551	+ qp_init_attr->srq = base_qp->srq;
	552	+
	553	+ qp_init_attr->cap = qp_attr->cap;
	554	+
	555	+ return 0;
	556	+}
	557	+
	558	+int siw_verbs_modify_qp(struct ib_qp base_qp, struct ib_qp_attr attr,
	559	+ int attr_mask, struct ib_udata *udata)
	560	+{
	561	+ struct siw_qp_attrs new_attrs;
	562	+ enum siw_qp_attr_mask siw_attr_mask = 0;
	563	+ struct siw_qp *qp = to_siw_qp(base_qp);
	564	+ int rv = 0;
	565	+
	566	+ if (!attr_mask)
	567	+ return 0;
	568	+
	569	+ memset(&new_attrs, 0, sizeof(new_attrs));
	570	+
	571	+ if (attr_mask & IB_QP_ACCESS_FLAGS) {
	572	+ siw_attr_mask = SIW_QP_ATTR_ACCESS_FLAGS;
	573	+
	574	+ if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
	575	+ new_attrs.flags \|= SIW_RDMA_READ_ENABLED;
	576	+ if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
	577	+ new_attrs.flags \|= SIW_RDMA_WRITE_ENABLED;
	578	+ if (attr->qp_access_flags & IB_ACCESS_MW_BIND)
	579	+ new_attrs.flags \|= SIW_RDMA_BIND_ENABLED;
	580	+ }
	581	+ if (attr_mask & IB_QP_STATE) {
	582	+ siw_dbg_qp(qp, "desired IB QP state: %s\n",
	583	+ ib_qp_state_to_string[attr->qp_state]);
	584	+
	585	+ new_attrs.state = ib_qp_state_to_siw_qp_state[attr->qp_state];
	586	+
	587	+ if (new_attrs.state > SIW_QP_STATE_RTS)
	588	+ qp->tx_ctx.tx_suspend = 1;
	589	+
	590	+ siw_attr_mask \|= SIW_QP_ATTR_STATE;
	591	+ }
	592	+ if (!siw_attr_mask)
	593	+ goto out;
	594	+
	595	+ down_write(&qp->state_lock);
	596	+
	597	+ rv = siw_qp_modify(qp, &new_attrs, siw_attr_mask);
	598	+
	599	+ up_write(&qp->state_lock);
	600	+out:
	601	+ return rv;
	602	+}
	603	+
	604	+int siw_destroy_qp(struct ib_qp base_qp, struct ib_udata udata)
	605	+{
	606	+ struct siw_qp *qp = to_siw_qp(base_qp);
	607	+ struct siw_base_qp *siw_base_qp = to_siw_base_qp(base_qp);
	608	+ struct siw_ucontext *uctx =
	609	+ rdma_udata_to_drv_context(udata, struct siw_ucontext,
	610	+ base_ucontext);
	611	+ struct siw_qp_attrs qp_attrs;
	612	+
	613	+ siw_dbg_qp(qp, "state %d, cep 0x%p\n", qp->attrs.state, qp->cep);
	614	+
	615	+ /*
	616	+ * Mark QP as in process of destruction to prevent from
	617	+ * any async callbacks to RDMA core
	618	+ */
	619	+ qp->attrs.flags \|= SIW_QP_IN_DESTROY;
	620	+ qp->rx_stream.rx_suspend = 1;
	621	+
	622	+ if (uctx && qp->xa_sq_index != SIW_INVAL_UOBJ_KEY)
	623	+ kfree(xa_erase(&uctx->xa, qp->xa_sq_index));
	624	+ if (uctx && qp->xa_rq_index != SIW_INVAL_UOBJ_KEY)
	625	+ kfree(xa_erase(&uctx->xa, qp->xa_rq_index));
	626	+
	627	+ down_write(&qp->state_lock);
	628	+
	629	+ qp_attrs.state = SIW_QP_STATE_ERROR;
	630	+ siw_qp_modify(qp, &qp_attrs, SIW_QP_ATTR_STATE);
	631	+
	632	+ if (qp->cep) {
	633	+ siw_cep_put(qp->cep);
	634	+ qp->cep = NULL;
	635	+ }
	636	+ up_write(&qp->state_lock);
	637	+
	638	+ kfree(qp->tx_ctx.mpa_crc_hd);
	639	+ kfree(qp->rx_stream.mpa_crc_hd);
	640	+
	641	+ qp->scq = qp->rcq = NULL;
	642	+
	643	+ siw_qp_put(qp);
	644	+ kfree(siw_base_qp);
	645	+
	646	+ return 0;
	647	+}
	648	+
	649	+/*
	650	+ * siw_copy_inline_sgl()
	651	+ *
	652	+ * Prepare sgl of inlined data for sending. For userland callers
	653	+ * function checks if given buffer addresses and len's are within
	654	+ * process context bounds.
	655	+ * Data from all provided sge's are copied together into the wqe,
	656	+ * referenced by a single sge.
	657	+ */
	658	+static int siw_copy_inline_sgl(const struct ib_send_wr *core_wr,
	659	+ struct siw_sqe *sqe)
	660	+{
	661	+ struct ib_sge *core_sge = core_wr->sg_list;
	662	+ void *kbuf = &sqe->sge[1];
	663	+ int num_sge = core_wr->num_sge, bytes = 0;
	664	+
	665	+ sqe->sge[0].laddr = (u64)kbuf;
	666	+ sqe->sge[0].lkey = 0;
	667	+
	668	+ while (num_sge--) {
	669	+ if (!core_sge->length) {
	670	+ core_sge++;
	671	+ continue;
	672	+ }
	673	+ bytes += core_sge->length;
	674	+ if (bytes > SIW_MAX_INLINE) {
	675	+ bytes = -EINVAL;
	676	+ break;
	677	+ }
	678	+ memcpy(kbuf, (void *)(uintptr_t)core_sge->addr,
	679	+ core_sge->length);
	680	+
	681	+ kbuf += core_sge->length;
	682	+ core_sge++;
	683	+ }
	684	+ sqe->sge[0].length = bytes > 0 ? bytes : 0;
	685	+ sqe->num_sge = bytes > 0 ? 1 : 0;
	686	+
	687	+ return bytes;
	688	+}
	689	+
	690	+/*
	691	+ * siw_post_send()
	692	+ *
	693	+ * Post a list of S-WR's to a SQ.
	694	+ *
	695	+ * @base_qp: Base QP contained in siw QP
	696	+ * @wr: Null terminated list of user WR's
	697	+ * @bad_wr: Points to failing WR in case of synchronous failure.
	698	+ */
	699	+int siw_post_send(struct ib_qp base_qp, const struct ib_send_wr wr,
	700	+ const struct ib_send_wr **bad_wr)
	701	+{
	702	+ struct siw_qp *qp = to_siw_qp(base_qp);
	703	+ struct siw_wqe *wqe = tx_wqe(qp);
	704	+
	705	+ unsigned long flags;
	706	+ int rv = 0;
	707	+
	708	+ /*
	709	+ * Try to acquire QP state lock. Must be non-blocking
	710	+ * to accommodate kernel clients needs.
	711	+ */
	712	+ if (!down_read_trylock(&qp->state_lock)) {
	713	+ *bad_wr = wr;
	714	+ siw_dbg_qp(qp, "QP locked, state %d\n", qp->attrs.state);
	715	+ return -ENOTCONN;
	716	+ }
	717	+ if (unlikely(qp->attrs.state != SIW_QP_STATE_RTS)) {
	718	+ up_read(&qp->state_lock);
	719	+ *bad_wr = wr;
	720	+ siw_dbg_qp(qp, "QP out of state %d\n", qp->attrs.state);
	721	+ return -ENOTCONN;
	722	+ }
	723	+ if (wr && !qp->kernel_verbs) {
	724	+ siw_dbg_qp(qp, "wr must be empty for user mapped sq\n");
	725	+ up_read(&qp->state_lock);
	726	+ *bad_wr = wr;
	727	+ return -EINVAL;
	728	+ }
	729	+ spin_lock_irqsave(&qp->sq_lock, flags);
	730	+
	731	+ while (wr) {
	732	+ u32 idx = qp->sq_put % qp->attrs.sq_size;
	733	+ struct siw_sqe *sqe = &qp->sendq[idx];
	734	+
	735	+ if (sqe->flags) {
	736	+ siw_dbg_qp(qp, "sq full\n");
	737	+ rv = -ENOMEM;
	738	+ break;
	739	+ }
	740	+ if (wr->num_sge > qp->attrs.sq_max_sges) {
	741	+ siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge);
	742	+ rv = -EINVAL;
	743	+ break;
	744	+ }
	745	+ sqe->id = wr->wr_id;
	746	+
	747	+ if ((wr->send_flags & IB_SEND_SIGNALED) \|\|
	748	+ (qp->attrs.flags & SIW_SIGNAL_ALL_WR))
	749	+ sqe->flags \|= SIW_WQE_SIGNALLED;
	750	+
	751	+ if (wr->send_flags & IB_SEND_FENCE)
	752	+ sqe->flags \|= SIW_WQE_READ_FENCE;
	753	+
	754	+ switch (wr->opcode) {
	755	+ case IB_WR_SEND:
	756	+ case IB_WR_SEND_WITH_INV:
	757	+ if (wr->send_flags & IB_SEND_SOLICITED)
	758	+ sqe->flags \|= SIW_WQE_SOLICITED;
	759	+
	760	+ if (!(wr->send_flags & IB_SEND_INLINE)) {
	761	+ siw_copy_sgl(wr->sg_list, sqe->sge,
	762	+ wr->num_sge);
	763	+ sqe->num_sge = wr->num_sge;
	764	+ } else {
	765	+ rv = siw_copy_inline_sgl(wr, sqe);
	766	+ if (rv <= 0) {
	767	+ rv = -EINVAL;
	768	+ break;
	769	+ }
	770	+ sqe->flags \|= SIW_WQE_INLINE;
	771	+ sqe->num_sge = 1;
	772	+ }
	773	+ if (wr->opcode == IB_WR_SEND)
	774	+ sqe->opcode = SIW_OP_SEND;
	775	+ else {
	776	+ sqe->opcode = SIW_OP_SEND_REMOTE_INV;
	777	+ sqe->rkey = wr->ex.invalidate_rkey;
	778	+ }
	779	+ break;
	780	+
	781	+ case IB_WR_RDMA_READ_WITH_INV:
	782	+ case IB_WR_RDMA_READ:
	783	+ /*
	784	+ * iWarp restricts RREAD sink to SGL containing
	785	+ * 1 SGE only. we could relax to SGL with multiple
	786	+ * elements referring the SAME ltag or even sending
	787	+ * a private per-rreq tag referring to a checked
	788	+ * local sgl with MULTIPLE ltag's.
	789	+ */
	790	+ if (unlikely(wr->num_sge != 1)) {
	791	+ rv = -EINVAL;
	792	+ break;
	793	+ }
	794	+ siw_copy_sgl(wr->sg_list, &sqe->sge[0], 1);
	795	+ /*
	796	+ * NOTE: zero length RREAD is allowed!
	797	+ */
	798	+ sqe->raddr = rdma_wr(wr)->remote_addr;
	799	+ sqe->rkey = rdma_wr(wr)->rkey;
	800	+ sqe->num_sge = 1;
	801	+
	802	+ if (wr->opcode == IB_WR_RDMA_READ)
	803	+ sqe->opcode = SIW_OP_READ;
	804	+ else
	805	+ sqe->opcode = SIW_OP_READ_LOCAL_INV;
	806	+ break;
	807	+
	808	+ case IB_WR_RDMA_WRITE:
	809	+ if (!(wr->send_flags & IB_SEND_INLINE)) {
	810	+ siw_copy_sgl(wr->sg_list, &sqe->sge[0],
	811	+ wr->num_sge);
	812	+ sqe->num_sge = wr->num_sge;
	813	+ } else {
	814	+ rv = siw_copy_inline_sgl(wr, sqe);
	815	+ if (unlikely(rv < 0)) {
	816	+ rv = -EINVAL;
	817	+ break;
	818	+ }
	819	+ sqe->flags \|= SIW_WQE_INLINE;
	820	+ sqe->num_sge = 1;
	821	+ }
	822	+ sqe->raddr = rdma_wr(wr)->remote_addr;
	823	+ sqe->rkey = rdma_wr(wr)->rkey;
	824	+ sqe->opcode = SIW_OP_WRITE;
	825	+ break;
	826	+
	827	+ case IB_WR_REG_MR:
	828	+ sqe->base_mr = (uint64_t)reg_wr(wr)->mr;
	829	+ sqe->rkey = reg_wr(wr)->key;
	830	+ sqe->access = reg_wr(wr)->access & IWARP_ACCESS_MASK;
	831	+ sqe->opcode = SIW_OP_REG_MR;
	832	+ break;
	833	+
	834	+ case IB_WR_LOCAL_INV:
	835	+ sqe->rkey = wr->ex.invalidate_rkey;
	836	+ sqe->opcode = SIW_OP_INVAL_STAG;
	837	+ break;
	838	+
	839	+ default:
	840	+ siw_dbg_qp(qp, "ib wr type %d unsupported\n",
	841	+ wr->opcode);
	842	+ rv = -EINVAL;
	843	+ break;
	844	+ }
	845	+ siw_dbg_qp(qp, "opcode %d, flags 0x%x, wr_id 0x%p\n",
	846	+ sqe->opcode, sqe->flags, (void *)sqe->id);
	847	+
	848	+ if (unlikely(rv < 0))
	849	+ break;
	850	+
	851	+ /* make SQE only valid after completely written */
	852	+ smp_wmb();
	853	+ sqe->flags \|= SIW_WQE_VALID;
	854	+
	855	+ qp->sq_put++;
	856	+ wr = wr->next;
	857	+ }
	858	+
	859	+ /*
	860	+ * Send directly if SQ processing is not in progress.
	861	+ * Eventual immediate errors (rv < 0) do not affect the involved
	862	+ * RI resources (Verbs, 8.3.1) and thus do not prevent from SQ
	863	+ * processing, if new work is already pending. But rv must be passed
	864	+ * to caller.
	865	+ */
	866	+ if (wqe->wr_status != SIW_WR_IDLE) {
	867	+ spin_unlock_irqrestore(&qp->sq_lock, flags);
	868	+ goto skip_direct_sending;
	869	+ }
	870	+ rv = siw_activate_tx(qp);
	871	+ spin_unlock_irqrestore(&qp->sq_lock, flags);
	872	+
	873	+ if (rv <= 0)
	874	+ goto skip_direct_sending;
	875	+
	876	+ if (qp->kernel_verbs) {
	877	+ rv = siw_sq_start(qp);
	878	+ } else {
	879	+ qp->tx_ctx.in_syscall = 1;
	880	+
	881	+ if (siw_qp_sq_process(qp) != 0 && !(qp->tx_ctx.tx_suspend))
	882	+ siw_qp_cm_drop(qp, 0);
	883	+
	884	+ qp->tx_ctx.in_syscall = 0;
	885	+ }
	886	+skip_direct_sending:
	887	+
	888	+ up_read(&qp->state_lock);
	889	+
	890	+ if (rv >= 0)
	891	+ return 0;
	892	+ /*
	893	+ * Immediate error
	894	+ */
	895	+ siw_dbg_qp(qp, "error %d\n", rv);
	896	+
	897	+ *bad_wr = wr;
	898	+ return rv;
	899	+}
	900	+
	901	+/*
	902	+ * siw_post_receive()
	903	+ *
	904	+ * Post a list of R-WR's to a RQ.
	905	+ *
	906	+ * @base_qp: Base QP contained in siw QP
	907	+ * @wr: Null terminated list of user WR's
	908	+ * @bad_wr: Points to failing WR in case of synchronous failure.
	909	+ */
	910	+int siw_post_receive(struct ib_qp base_qp, const struct ib_recv_wr wr,
	911	+ const struct ib_recv_wr **bad_wr)
	912	+{
	913	+ struct siw_qp *qp = to_siw_qp(base_qp);
	914	+ unsigned long flags;
	915	+ int rv = 0;
	916	+
	917	+ if (qp->srq) {
	918	+ *bad_wr = wr;
	919	+ return -EOPNOTSUPP; /* what else from errno.h? */
	920	+ }
	921	+ /*
	922	+ * Try to acquire QP state lock. Must be non-blocking
	923	+ * to accommodate kernel clients needs.
	924	+ */
	925	+ if (!down_read_trylock(&qp->state_lock)) {
	926	+ *bad_wr = wr;
	927	+ return -ENOTCONN;
	928	+ }
	929	+ if (!qp->kernel_verbs) {
	930	+ siw_dbg_qp(qp, "no kernel post_recv for user mapped sq\n");
	931	+ up_read(&qp->state_lock);
	932	+ *bad_wr = wr;
	933	+ return -EINVAL;
	934	+ }
	935	+ if (qp->attrs.state > SIW_QP_STATE_RTS) {
	936	+ up_read(&qp->state_lock);
	937	+ *bad_wr = wr;
	938	+ return -EINVAL;
	939	+ }
	940	+ /*
	941	+ * Serialize potentially multiple producers.
	942	+ * Not needed for single threaded consumer side.
	943	+ */
	944	+ spin_lock_irqsave(&qp->rq_lock, flags);
	945	+
	946	+ while (wr) {
	947	+ u32 idx = qp->rq_put % qp->attrs.rq_size;
	948	+ struct siw_rqe *rqe = &qp->recvq[idx];
	949	+
	950	+ if (rqe->flags) {
	951	+ siw_dbg_qp(qp, "RQ full\n");
	952	+ rv = -ENOMEM;
	953	+ break;
	954	+ }
	955	+ if (wr->num_sge > qp->attrs.rq_max_sges) {
	956	+ siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge);
	957	+ rv = -EINVAL;
	958	+ break;
	959	+ }
	960	+ rqe->id = wr->wr_id;
	961	+ rqe->num_sge = wr->num_sge;
	962	+ siw_copy_sgl(wr->sg_list, rqe->sge, wr->num_sge);
	963	+
	964	+ /* make sure RQE is completely written before valid */
	965	+ smp_wmb();
	966	+
	967	+ rqe->flags = SIW_WQE_VALID;
	968	+
	969	+ qp->rq_put++;
	970	+ wr = wr->next;
	971	+ }
	972	+ spin_unlock_irqrestore(&qp->rq_lock, flags);
	973	+
	974	+ up_read(&qp->state_lock);
	975	+
	976	+ if (rv < 0) {
	977	+ siw_dbg_qp(qp, "error %d\n", rv);
	978	+ *bad_wr = wr;
	979	+ }
	980	+ return rv > 0 ? 0 : rv;
	981	+}
	982	+
	983	+void siw_destroy_cq(struct ib_cq base_cq, struct ib_udata udata)
	984	+{
	985	+ struct siw_cq *cq = to_siw_cq(base_cq);
	986	+ struct siw_device *sdev = to_siw_dev(base_cq->device);
	987	+ struct siw_ucontext *ctx =
	988	+ rdma_udata_to_drv_context(udata, struct siw_ucontext,
	989	+ base_ucontext);
	990	+
	991	+ siw_dbg_cq(cq, "free CQ resources\n");
	992	+
	993	+ siw_cq_flush(cq);
	994	+
	995	+ if (ctx && cq->xa_cq_index != SIW_INVAL_UOBJ_KEY)
	996	+ kfree(xa_erase(&ctx->xa, cq->xa_cq_index));
	997	+
	998	+ atomic_dec(&sdev->num_cq);
	999	+
	1000	+ vfree(cq->queue);
	1001	+}
	1002	+
	1003	+/*
	1004	+ * siw_create_cq()
	1005	+ *
	1006	+ * Populate CQ of requested size
	1007	+ *
	1008	+ * @base_cq: CQ as allocated by RDMA midlayer
	1009	+ * @attr: Initial CQ attributes
	1010	+ * @udata: relates to user context
	1011	+ */
	1012	+
	1013	+int siw_create_cq(struct ib_cq base_cq, const struct ib_cq_init_attr attr,
	1014	+ struct ib_udata *udata)
	1015	+{
	1016	+ struct siw_device *sdev = to_siw_dev(base_cq->device);
	1017	+ struct siw_cq *cq = to_siw_cq(base_cq);
	1018	+ int rv, size = attr->cqe;
	1019	+
	1020	+ if (atomic_inc_return(&sdev->num_cq) > SIW_MAX_CQ) {
	1021	+ siw_dbg(base_cq->device, "too many CQ's\n");
	1022	+ rv = -ENOMEM;
	1023	+ goto err_out;
	1024	+ }
	1025	+ if (size < 1 \|\| size > sdev->attrs.max_cqe) {
	1026	+ siw_dbg(base_cq->device, "CQ size error: %d\n", size);
	1027	+ rv = -EINVAL;
	1028	+ goto err_out;
	1029	+ }
	1030	+ size = roundup_pow_of_two(size);
	1031	+ cq->base_cq.cqe = size;
	1032	+ cq->num_cqe = size;
	1033	+ cq->xa_cq_index = SIW_INVAL_UOBJ_KEY;
	1034	+
	1035	+ if (!udata) {
	1036	+ cq->kernel_verbs = 1;
	1037	+ cq->queue = vzalloc(size * sizeof(struct siw_cqe) +
	1038	+ sizeof(struct siw_cq_ctrl));
	1039	+ } else {
	1040	+ cq->queue = vmalloc_user(size * sizeof(struct siw_cqe) +
	1041	+ sizeof(struct siw_cq_ctrl));
	1042	+ }
	1043	+ if (cq->queue == NULL) {
	1044	+ rv = -ENOMEM;
	1045	+ goto err_out;
	1046	+ }
	1047	+ get_random_bytes(&cq->id, 4);
	1048	+ siw_dbg(base_cq->device, "new CQ [%u]\n", cq->id);
	1049	+
	1050	+ spin_lock_init(&cq->lock);
	1051	+
	1052	+ cq->notify = &((struct siw_cq_ctrl *)&cq->queue[size])->notify;
	1053	+
	1054	+ if (udata) {
	1055	+ struct siw_uresp_create_cq uresp = {};
	1056	+ struct siw_ucontext *ctx =
	1057	+ rdma_udata_to_drv_context(udata, struct siw_ucontext,
	1058	+ base_ucontext);
	1059	+
	1060	+ cq->xa_cq_index =
	1061	+ siw_create_uobj(ctx, cq->queue,
	1062	+ size * sizeof(struct siw_cqe) +
	1063	+ sizeof(struct siw_cq_ctrl));
	1064	+ if (cq->xa_cq_index == SIW_INVAL_UOBJ_KEY) {
	1065	+ rv = -ENOMEM;
	1066	+ goto err_out;
	1067	+ }
	1068	+ uresp.cq_key = cq->xa_cq_index << PAGE_SHIFT;
	1069	+ uresp.cq_id = cq->id;
	1070	+ uresp.num_cqe = size;
	1071	+
	1072	+ if (udata->outlen < sizeof(uresp)) {
	1073	+ rv = -EINVAL;
	1074	+ goto err_out;
	1075	+ }
	1076	+ rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
	1077	+ if (rv)
	1078	+ goto err_out;
	1079	+ }
	1080	+ return 0;
	1081	+
	1082	+err_out:
	1083	+ siw_dbg(base_cq->device, "CQ creation failed: %d", rv);
	1084	+
	1085	+ if (cq && cq->queue) {
	1086	+ struct siw_ucontext *ctx =
	1087	+ rdma_udata_to_drv_context(udata, struct siw_ucontext,
	1088	+ base_ucontext);
	1089	+ if (cq->xa_cq_index != SIW_INVAL_UOBJ_KEY)
	1090	+ kfree(xa_erase(&ctx->xa, cq->xa_cq_index));
	1091	+ vfree(cq->queue);
	1092	+ }
	1093	+ atomic_dec(&sdev->num_cq);
	1094	+
	1095	+ return rv;
	1096	+}
	1097	+
	1098	+/*
	1099	+ * siw_poll_cq()
	1100	+ *
	1101	+ * Reap CQ entries if available and copy work completion status into
	1102	+ * array of WC's provided by caller. Returns number of reaped CQE's.
	1103	+ *
	1104	+ * @base_cq: Base CQ contained in siw CQ.
	1105	+ * @num_cqe: Maximum number of CQE's to reap.
	1106	+ * @wc: Array of work completions to be filled by siw.
	1107	+ */
	1108	+int siw_poll_cq(struct ib_cq base_cq, int num_cqe, struct ib_wc wc)
	1109	+{
	1110	+ struct siw_cq *cq = to_siw_cq(base_cq);
	1111	+ int i;
	1112	+
	1113	+ for (i = 0; i < num_cqe; i++) {
	1114	+ if (!siw_reap_cqe(cq, wc))
	1115	+ break;
	1116	+ wc++;
	1117	+ }
	1118	+ return i;
	1119	+}
	1120	+
	1121	+/*
	1122	+ * siw_req_notify_cq()
	1123	+ *
	1124	+ * Request notification for new CQE's added to that CQ.
	1125	+ * Defined flags:
	1126	+ * o SIW_CQ_NOTIFY_SOLICITED lets siw trigger a notification
	1127	+ * event if a WQE with notification flag set enters the CQ
	1128	+ * o SIW_CQ_NOTIFY_NEXT_COMP lets siw trigger a notification
	1129	+ * event if a WQE enters the CQ.
	1130	+ * o IB_CQ_REPORT_MISSED_EVENTS: return value will provide the
	1131	+ * number of not reaped CQE's regardless of its notification
	1132	+ * type and current or new CQ notification settings.
	1133	+ *
	1134	+ * @base_cq: Base CQ contained in siw CQ.
	1135	+ * @flags: Requested notification flags.
	1136	+ */
	1137	+int siw_req_notify_cq(struct ib_cq *base_cq, enum ib_cq_notify_flags flags)
	1138	+{
	1139	+ struct siw_cq *cq = to_siw_cq(base_cq);
	1140	+
	1141	+ siw_dbg_cq(cq, "flags: 0x%02x\n", flags);
	1142	+
	1143	+ if ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED)
	1144	+ /* CQ event for next solicited completion */
	1145	+ smp_store_mb(*cq->notify, SIW_NOTIFY_SOLICITED);
	1146	+ else
	1147	+ /* CQ event for any signalled completion */
	1148	+ smp_store_mb(*cq->notify, SIW_NOTIFY_ALL);
	1149	+
	1150	+ if (flags & IB_CQ_REPORT_MISSED_EVENTS)
	1151	+ return cq->cq_put - cq->cq_get;
	1152	+
	1153	+ return 0;
	1154	+}
	1155	+
	1156	+/*
	1157	+ * siw_dereg_mr()
	1158	+ *
	1159	+ * Release Memory Region.
	1160	+ *
	1161	+ * @base_mr: Base MR contained in siw MR.
	1162	+ * @udata: points to user context, unused.
	1163	+ */
	1164	+int siw_dereg_mr(struct ib_mr base_mr, struct ib_udata udata)
	1165	+{
	1166	+ struct siw_mr *mr = to_siw_mr(base_mr);
	1167	+ struct siw_device *sdev = to_siw_dev(base_mr->device);
	1168	+
	1169	+ siw_dbg_mem(mr->mem, "deregister MR\n");
	1170	+
	1171	+ atomic_dec(&sdev->num_mr);
	1172	+
	1173	+ siw_mr_drop_mem(mr);
	1174	+ kfree_rcu(mr, rcu);
	1175	+
	1176	+ return 0;
	1177	+}
	1178	+
	1179	+/*
	1180	+ * siw_reg_user_mr()
	1181	+ *
	1182	+ * Register Memory Region.
	1183	+ *
	1184	+ * @pd: Protection Domain
	1185	+ * @start: starting address of MR (virtual address)
	1186	+ * @len: len of MR
	1187	+ * @rnic_va: not used by siw
	1188	+ * @rights: MR access rights
	1189	+ * @udata: user buffer to communicate STag and Key.
	1190	+ */
	1191	+struct ib_mr siw_reg_user_mr(struct ib_pd pd, u64 start, u64 len,
	1192	+ u64 rnic_va, int rights, struct ib_udata *udata)
	1193	+{
	1194	+ struct siw_mr *mr = NULL;
	1195	+ struct siw_umem *umem = NULL;
	1196	+ struct siw_ureq_reg_mr ureq;
	1197	+ struct siw_device *sdev = to_siw_dev(pd->device);
	1198	+
	1199	+ unsigned long mem_limit = rlimit(RLIMIT_MEMLOCK);
	1200	+ int rv;
	1201	+
	1202	+ siw_dbg_pd(pd, "start: 0x%016llx, va: 0x%016llx, len: %llu\n",
	1203	+ (unsigned long long)start, (unsigned long long)rnic_va,
	1204	+ (unsigned long long)len);
	1205	+
	1206	+ if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
	1207	+ siw_dbg_pd(pd, "too many mr's\n");
	1208	+ rv = -ENOMEM;
	1209	+ goto err_out;
	1210	+ }
	1211	+ if (!len) {
	1212	+ rv = -EINVAL;
	1213	+ goto err_out;
	1214	+ }
	1215	+ if (mem_limit != RLIM_INFINITY) {
	1216	+ unsigned long num_pages =
	1217	+ (PAGE_ALIGN(len + (start & ~PAGE_MASK))) >> PAGE_SHIFT;
	1218	+ mem_limit >>= PAGE_SHIFT;
	1219	+
	1220	+ if (num_pages > mem_limit - current->mm->locked_vm) {
	1221	+ siw_dbg_pd(pd, "pages req %lu, max %lu, lock %lu\n",
	1222	+ num_pages, mem_limit,
	1223	+ current->mm->locked_vm);
	1224	+ rv = -ENOMEM;
	1225	+ goto err_out;
	1226	+ }
	1227	+ }
	1228	+ umem = siw_umem_get(start, len, ib_access_writable(rights));
	1229	+ if (IS_ERR(umem)) {
	1230	+ rv = PTR_ERR(umem);
	1231	+ siw_dbg_pd(pd, "getting user memory failed: %d\n", rv);
	1232	+ umem = NULL;
	1233	+ goto err_out;
	1234	+ }
	1235	+ mr = kzalloc(sizeof(*mr), GFP_KERNEL);
	1236	+ if (!mr) {
	1237	+ rv = -ENOMEM;
	1238	+ goto err_out;
	1239	+ }
	1240	+ rv = siw_mr_add_mem(mr, pd, umem, start, len, rights);
	1241	+ if (rv)
	1242	+ goto err_out;
	1243	+
	1244	+ if (udata) {
	1245	+ struct siw_uresp_reg_mr uresp = {};
	1246	+ struct siw_mem *mem = mr->mem;
	1247	+
	1248	+ if (udata->inlen < sizeof(ureq)) {
	1249	+ rv = -EINVAL;
	1250	+ goto err_out;
	1251	+ }
	1252	+ rv = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
	1253	+ if (rv)
	1254	+ goto err_out;
	1255	+
	1256	+ mr->base_mr.lkey \|= ureq.stag_key;
	1257	+ mr->base_mr.rkey \|= ureq.stag_key;
	1258	+ mem->stag \|= ureq.stag_key;
	1259	+ uresp.stag = mem->stag;
	1260	+
	1261	+ if (udata->outlen < sizeof(uresp)) {
	1262	+ rv = -EINVAL;
	1263	+ goto err_out;
	1264	+ }
	1265	+ rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
	1266	+ if (rv)
	1267	+ goto err_out;
	1268	+ }
	1269	+ mr->mem->stag_valid = 1;
	1270	+
	1271	+ return &mr->base_mr;
	1272	+
	1273	+err_out:
	1274	+ atomic_dec(&sdev->num_mr);
	1275	+ if (mr) {
	1276	+ if (mr->mem)
	1277	+ siw_mr_drop_mem(mr);
	1278	+ kfree_rcu(mr, rcu);
	1279	+ } else {
	1280	+ if (umem)
	1281	+ siw_umem_release(umem, false);
	1282	+ }
	1283	+ return ERR_PTR(rv);
	1284	+}
	1285	+
	1286	+struct ib_mr siw_alloc_mr(struct ib_pd pd, enum ib_mr_type mr_type,
	1287	+ u32 max_sge, struct ib_udata *udata)
	1288	+{
	1289	+ struct siw_device *sdev = to_siw_dev(pd->device);
	1290	+ struct siw_mr *mr = NULL;
	1291	+ struct siw_pbl *pbl = NULL;
	1292	+ int rv;
	1293	+
	1294	+ if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
	1295	+ siw_dbg_pd(pd, "too many mr's\n");
	1296	+ rv = -ENOMEM;
	1297	+ goto err_out;
	1298	+ }
	1299	+ if (mr_type != IB_MR_TYPE_MEM_REG) {
	1300	+ siw_dbg_pd(pd, "mr type %d unsupported\n", mr_type);
	1301	+ rv = -EOPNOTSUPP;
	1302	+ goto err_out;
	1303	+ }
	1304	+ if (max_sge > SIW_MAX_SGE_PBL) {
	1305	+ siw_dbg_pd(pd, "too many sge's: %d\n", max_sge);
	1306	+ rv = -ENOMEM;
	1307	+ goto err_out;
	1308	+ }
	1309	+ pbl = siw_pbl_alloc(max_sge);
	1310	+ if (IS_ERR(pbl)) {
	1311	+ rv = PTR_ERR(pbl);
	1312	+ siw_dbg_pd(pd, "pbl allocation failed: %d\n", rv);
	1313	+ pbl = NULL;
	1314	+ goto err_out;
	1315	+ }
	1316	+ mr = kzalloc(sizeof(*mr), GFP_KERNEL);
	1317	+ if (!mr) {
	1318	+ rv = -ENOMEM;
	1319	+ goto err_out;
	1320	+ }
	1321	+ rv = siw_mr_add_mem(mr, pd, pbl, 0, max_sge * PAGE_SIZE, 0);
	1322	+ if (rv)
	1323	+ goto err_out;
	1324	+
	1325	+ mr->mem->is_pbl = 1;
	1326	+
	1327	+ siw_dbg_pd(pd, "[MEM %u]: success\n", mr->mem->stag);
	1328	+
	1329	+ return &mr->base_mr;
	1330	+
	1331	+err_out:
	1332	+ atomic_dec(&sdev->num_mr);
	1333	+
	1334	+ if (!mr) {
	1335	+ kfree(pbl);
	1336	+ } else {
	1337	+ if (mr->mem)
	1338	+ siw_mr_drop_mem(mr);
	1339	+ kfree_rcu(mr, rcu);
	1340	+ }
	1341	+ siw_dbg_pd(pd, "failed: %d\n", rv);
	1342	+
	1343	+ return ERR_PTR(rv);
	1344	+}
	1345	+
	1346	+/* Just used to count number of pages being mapped */
	1347	+static int siw_set_pbl_page(struct ib_mr *base_mr, u64 buf_addr)
	1348	+{
	1349	+ return 0;
	1350	+}
	1351	+
	1352	+int siw_map_mr_sg(struct ib_mr base_mr, struct scatterlist sl, int num_sle,
	1353	+ unsigned int *sg_off)
	1354	+{
	1355	+ struct scatterlist *slp;
	1356	+ struct siw_mr *mr = to_siw_mr(base_mr);
	1357	+ struct siw_mem *mem = mr->mem;
	1358	+ struct siw_pbl *pbl = mem->pbl;
	1359	+ struct siw_pble *pble;
	1360	+ u64 pbl_size;
	1361	+ int i, rv;
	1362	+
	1363	+ if (!pbl) {
	1364	+ siw_dbg_mem(mem, "no PBL allocated\n");
	1365	+ return -EINVAL;
	1366	+ }
	1367	+ pble = pbl->pbe;
	1368	+
	1369	+ if (pbl->max_buf < num_sle) {
	1370	+ siw_dbg_mem(mem, "too many SGE's: %d > %d\n",
	1371	+ mem->pbl->max_buf, num_sle);
	1372	+ return -ENOMEM;
	1373	+ }
	1374	+ for_each_sg(sl, slp, num_sle, i) {
	1375	+ if (sg_dma_len(slp) == 0) {
	1376	+ siw_dbg_mem(mem, "empty SGE\n");
	1377	+ return -EINVAL;
	1378	+ }
	1379	+ if (i == 0) {
	1380	+ pble->addr = sg_dma_address(slp);
	1381	+ pble->size = sg_dma_len(slp);
	1382	+ pble->pbl_off = 0;
	1383	+ pbl_size = pble->size;
	1384	+ pbl->num_buf = 1;
	1385	+ } else {
	1386	+ /* Merge PBL entries if adjacent */
	1387	+ if (pble->addr + pble->size == sg_dma_address(slp)) {
	1388	+ pble->size += sg_dma_len(slp);
	1389	+ } else {
	1390	+ pble++;
	1391	+ pbl->num_buf++;
	1392	+ pble->addr = sg_dma_address(slp);
	1393	+ pble->size = sg_dma_len(slp);
	1394	+ pble->pbl_off = pbl_size;
	1395	+ }
	1396	+ pbl_size += sg_dma_len(slp);
	1397	+ }
	1398	+ siw_dbg_mem(mem,
	1399	+ "sge[%d], size %llu, addr 0x%016llx, total %llu\n",
	1400	+ i, pble->size, pble->addr, pbl_size);
	1401	+ }
	1402	+ rv = ib_sg_to_pages(base_mr, sl, num_sle, sg_off, siw_set_pbl_page);
	1403	+ if (rv > 0) {
	1404	+ mem->len = base_mr->length;
	1405	+ mem->va = base_mr->iova;
	1406	+ siw_dbg_mem(mem,
	1407	+ "%llu bytes, start 0x%016llx, %u SLE to %u entries\n",
	1408	+ mem->len, mem->va, num_sle, pbl->num_buf);
	1409	+ }
	1410	+ return rv;
	1411	+}
	1412	+
	1413	+/*
	1414	+ * siw_get_dma_mr()
	1415	+ *
	1416	+ * Create a (empty) DMA memory region, where no umem is attached.
	1417	+ */
	1418	+struct ib_mr siw_get_dma_mr(struct ib_pd pd, int rights)
	1419	+{
	1420	+ struct siw_device *sdev = to_siw_dev(pd->device);
	1421	+ struct siw_mr *mr = NULL;
	1422	+ int rv;
	1423	+
	1424	+ if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
	1425	+ siw_dbg_pd(pd, "too many mr's\n");
	1426	+ rv = -ENOMEM;
	1427	+ goto err_out;
	1428	+ }
	1429	+ mr = kzalloc(sizeof(*mr), GFP_KERNEL);
	1430	+ if (!mr) {
	1431	+ rv = -ENOMEM;
	1432	+ goto err_out;
	1433	+ }
	1434	+ rv = siw_mr_add_mem(mr, pd, NULL, 0, ULONG_MAX, rights);
	1435	+ if (rv)
	1436	+ goto err_out;
	1437	+
	1438	+ mr->mem->stag_valid = 1;
	1439	+
	1440	+ siw_dbg_pd(pd, "[MEM %u]: success\n", mr->mem->stag);
	1441	+
	1442	+ return &mr->base_mr;
	1443	+
	1444	+err_out:
	1445	+ if (rv)
	1446	+ kfree(mr);
	1447	+
	1448	+ atomic_dec(&sdev->num_mr);
	1449	+
	1450	+ return ERR_PTR(rv);
	1451	+}
	1452	+
	1453	+/*
	1454	+ * siw_create_srq()
	1455	+ *
	1456	+ * Create Shared Receive Queue of attributes @init_attrs
	1457	+ * within protection domain given by @pd.
	1458	+ *
	1459	+ * @base_srq: Base SRQ contained in siw SRQ.
	1460	+ * @init_attrs: SRQ init attributes.
	1461	+ * @udata: points to user context
	1462	+ */
	1463	+int siw_create_srq(struct ib_srq *base_srq,
	1464	+ struct ib_srq_init_attr init_attrs, struct ib_udata udata)
	1465	+{
	1466	+ struct siw_srq *srq = to_siw_srq(base_srq);
	1467	+ struct ib_srq_attr *attrs = &init_attrs->attr;
	1468	+ struct siw_device *sdev = to_siw_dev(base_srq->device);
	1469	+ struct siw_ucontext *ctx =
	1470	+ rdma_udata_to_drv_context(udata, struct siw_ucontext,
	1471	+ base_ucontext);
	1472	+ int rv;
	1473	+
	1474	+ if (atomic_inc_return(&sdev->num_srq) > SIW_MAX_SRQ) {
	1475	+ siw_dbg_pd(base_srq->pd, "too many SRQ's\n");
	1476	+ rv = -ENOMEM;
	1477	+ goto err_out;
	1478	+ }
	1479	+ if (attrs->max_wr == 0 \|\| attrs->max_wr > SIW_MAX_SRQ_WR \|\|
	1480	+ attrs->max_sge > SIW_MAX_SGE \|\| attrs->srq_limit > attrs->max_wr) {
	1481	+ rv = -EINVAL;
	1482	+ goto err_out;
	1483	+ }
	1484	+ srq->max_sge = attrs->max_sge;
	1485	+ srq->num_rqe = roundup_pow_of_two(attrs->max_wr);
	1486	+ srq->xa_srq_index = SIW_INVAL_UOBJ_KEY;
	1487	+ srq->limit = attrs->srq_limit;
	1488	+ if (srq->limit)
	1489	+ srq->armed = 1;
	1490	+
	1491	+ srq->kernel_verbs = !udata;
	1492	+
	1493	+ if (udata)
	1494	+ srq->recvq =
	1495	+ vmalloc_user(srq->num_rqe * sizeof(struct siw_rqe));
	1496	+ else
	1497	+ srq->recvq = vzalloc(srq->num_rqe * sizeof(struct siw_rqe));
	1498	+
	1499	+ if (srq->recvq == NULL) {
	1500	+ rv = -ENOMEM;
	1501	+ goto err_out;
	1502	+ }
	1503	+ if (udata) {
	1504	+ struct siw_uresp_create_srq uresp = {};
	1505	+
	1506	+ srq->xa_srq_index = siw_create_uobj(
	1507	+ ctx, srq->recvq, srq->num_rqe * sizeof(struct siw_rqe));
	1508	+
	1509	+ if (srq->xa_srq_index == SIW_INVAL_UOBJ_KEY) {
	1510	+ rv = -ENOMEM;
	1511	+ goto err_out;
	1512	+ }
	1513	+ uresp.srq_key = srq->xa_srq_index;
	1514	+ uresp.num_rqe = srq->num_rqe;
	1515	+
	1516	+ if (udata->outlen < sizeof(uresp)) {
	1517	+ rv = -EINVAL;
	1518	+ goto err_out;
	1519	+ }
	1520	+ rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
	1521	+ if (rv)
	1522	+ goto err_out;
	1523	+ }
	1524	+ spin_lock_init(&srq->lock);
	1525	+
	1526	+ siw_dbg_pd(base_srq->pd, "[SRQ 0x%p]: success\n", srq);
	1527	+
	1528	+ return 0;
	1529	+
	1530	+err_out:
	1531	+ if (srq->recvq) {
	1532	+ if (ctx && srq->xa_srq_index != SIW_INVAL_UOBJ_KEY)
	1533	+ kfree(xa_erase(&ctx->xa, srq->xa_srq_index));
	1534	+ vfree(srq->recvq);
	1535	+ }
	1536	+ atomic_dec(&sdev->num_srq);
	1537	+
	1538	+ return rv;
	1539	+}
	1540	+
	1541	+/*
	1542	+ * siw_modify_srq()
	1543	+ *
	1544	+ * Modify SRQ. The caller may resize SRQ and/or set/reset notification
	1545	+ * limit and (re)arm IB_EVENT_SRQ_LIMIT_REACHED notification.
	1546	+ *
	1547	+ * NOTE: it is unclear if RDMA core allows for changing the MAX_SGE
	1548	+ * parameter. siw_modify_srq() does not check the attrs->max_sge param.
	1549	+ */
	1550	+int siw_modify_srq(struct ib_srq base_srq, struct ib_srq_attr attrs,
	1551	+ enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
	1552	+{
	1553	+ struct siw_srq *srq = to_siw_srq(base_srq);
	1554	+ unsigned long flags;
	1555	+ int rv = 0;
	1556	+
	1557	+ spin_lock_irqsave(&srq->lock, flags);
	1558	+
	1559	+ if (attr_mask & IB_SRQ_MAX_WR) {
	1560	+ /* resize request not yet supported */
	1561	+ rv = -EOPNOTSUPP;
	1562	+ goto out;
	1563	+ }
	1564	+ if (attr_mask & IB_SRQ_LIMIT) {
	1565	+ if (attrs->srq_limit) {
	1566	+ if (unlikely(attrs->srq_limit > srq->num_rqe)) {
	1567	+ rv = -EINVAL;
	1568	+ goto out;
	1569	+ }
	1570	+ srq->armed = 1;
	1571	+ } else {
	1572	+ srq->armed = 0;
	1573	+ }
	1574	+ srq->limit = attrs->srq_limit;
	1575	+ }
	1576	+out:
	1577	+ spin_unlock_irqrestore(&srq->lock, flags);
	1578	+
	1579	+ return rv;
	1580	+}
	1581	+
	1582	+/*
	1583	+ * siw_query_srq()
	1584	+ *
	1585	+ * Query SRQ attributes.
	1586	+ */
	1587	+int siw_query_srq(struct ib_srq base_srq, struct ib_srq_attr attrs)
	1588	+{
	1589	+ struct siw_srq *srq = to_siw_srq(base_srq);
	1590	+ unsigned long flags;
	1591	+
	1592	+ spin_lock_irqsave(&srq->lock, flags);
	1593	+
	1594	+ attrs->max_wr = srq->num_rqe;
	1595	+ attrs->max_sge = srq->max_sge;
	1596	+ attrs->srq_limit = srq->limit;
	1597	+
	1598	+ spin_unlock_irqrestore(&srq->lock, flags);
	1599	+
	1600	+ return 0;
	1601	+}
	1602	+
	1603	+/*
	1604	+ * siw_destroy_srq()
	1605	+ *
	1606	+ * Destroy SRQ.
	1607	+ * It is assumed that the SRQ is not referenced by any
	1608	+ * QP anymore - the code trusts the RDMA core environment to keep track
	1609	+ * of QP references.
	1610	+ */
	1611	+void siw_destroy_srq(struct ib_srq base_srq, struct ib_udata udata)
	1612	+{
	1613	+ struct siw_srq *srq = to_siw_srq(base_srq);
	1614	+ struct siw_device *sdev = to_siw_dev(base_srq->device);
	1615	+ struct siw_ucontext *ctx =
	1616	+ rdma_udata_to_drv_context(udata, struct siw_ucontext,
	1617	+ base_ucontext);
	1618	+
	1619	+ if (ctx && srq->xa_srq_index != SIW_INVAL_UOBJ_KEY)
	1620	+ kfree(xa_erase(&ctx->xa, srq->xa_srq_index));
	1621	+
	1622	+ vfree(srq->recvq);
	1623	+ atomic_dec(&sdev->num_srq);
	1624	+}
	1625	+
	1626	+/*
	1627	+ * siw_post_srq_recv()
	1628	+ *
	1629	+ * Post a list of receive queue elements to SRQ.
	1630	+ * NOTE: The function does not check or lock a certain SRQ state
	1631	+ * during the post operation. The code simply trusts the
	1632	+ * RDMA core environment.
	1633	+ *
	1634	+ * @base_srq: Base SRQ contained in siw SRQ
	1635	+ * @wr: List of R-WR's
	1636	+ * @bad_wr: Updated to failing WR if posting fails.
	1637	+ */
	1638	+int siw_post_srq_recv(struct ib_srq base_srq, const struct ib_recv_wr wr,
	1639	+ const struct ib_recv_wr **bad_wr)
	1640	+{
	1641	+ struct siw_srq *srq = to_siw_srq(base_srq);
	1642	+ unsigned long flags;
	1643	+ int rv = 0;
	1644	+
	1645	+ if (unlikely(!srq->kernel_verbs)) {
	1646	+ siw_dbg_pd(base_srq->pd,
	1647	+ "[SRQ 0x%p]: no kernel post_recv for mapped srq\n",
	1648	+ srq);
	1649	+ rv = -EINVAL;
	1650	+ goto out;
	1651	+ }
	1652	+ /*
	1653	+ * Serialize potentially multiple producers.
	1654	+ * Also needed to serialize potentially multiple
	1655	+ * consumers.
	1656	+ */
	1657	+ spin_lock_irqsave(&srq->lock, flags);
	1658	+
	1659	+ while (wr) {
	1660	+ u32 idx = srq->rq_put % srq->num_rqe;
	1661	+ struct siw_rqe *rqe = &srq->recvq[idx];
	1662	+
	1663	+ if (rqe->flags) {
	1664	+ siw_dbg_pd(base_srq->pd, "SRQ full\n");
	1665	+ rv = -ENOMEM;
	1666	+ break;
	1667	+ }
	1668	+ if (unlikely(wr->num_sge > srq->max_sge)) {
	1669	+ siw_dbg_pd(base_srq->pd,
	1670	+ "[SRQ 0x%p]: too many sge's: %d\n", srq,
	1671	+ wr->num_sge);
	1672	+ rv = -EINVAL;
	1673	+ break;
	1674	+ }
	1675	+ rqe->id = wr->wr_id;
	1676	+ rqe->num_sge = wr->num_sge;
	1677	+ siw_copy_sgl(wr->sg_list, rqe->sge, wr->num_sge);
	1678	+
	1679	+ /* Make sure S-RQE is completely written before valid */
	1680	+ smp_wmb();
	1681	+
	1682	+ rqe->flags = SIW_WQE_VALID;
	1683	+
	1684	+ srq->rq_put++;
	1685	+ wr = wr->next;
	1686	+ }
	1687	+ spin_unlock_irqrestore(&srq->lock, flags);
	1688	+out:
	1689	+ if (unlikely(rv < 0)) {
	1690	+ siw_dbg_pd(base_srq->pd, "[SRQ 0x%p]: error %d\n", srq, rv);
	1691	+ *bad_wr = wr;
	1692	+ }
	1693	+ return rv;
	1694	+}
	1695	+
	1696	+void siw_qp_event(struct siw_qp *qp, enum ib_event_type etype)
	1697	+{
	1698	+ struct ib_event event;
	1699	+ struct ib_qp *base_qp = qp->ib_qp;
	1700	+
	1701	+ /*
	1702	+ * Do not report asynchronous errors on QP which gets
	1703	+ * destroyed via verbs interface (siw_destroy_qp())
	1704	+ */
	1705	+ if (qp->attrs.flags & SIW_QP_IN_DESTROY)
	1706	+ return;
	1707	+
	1708	+ event.event = etype;
	1709	+ event.device = base_qp->device;
	1710	+ event.element.qp = base_qp;
	1711	+
	1712	+ if (base_qp->event_handler) {
	1713	+ siw_dbg_qp(qp, "reporting event %d\n", etype);
	1714	+ base_qp->event_handler(&event, base_qp->qp_context);
	1715	+ }
	1716	+}
	1717	+
	1718	+void siw_cq_event(struct siw_cq *cq, enum ib_event_type etype)
	1719	+{
	1720	+ struct ib_event event;
	1721	+ struct ib_cq *base_cq = &cq->base_cq;
	1722	+
	1723	+ event.event = etype;
	1724	+ event.device = base_cq->device;
	1725	+ event.element.cq = base_cq;
	1726	+
	1727	+ if (base_cq->event_handler) {
	1728	+ siw_dbg_cq(cq, "reporting CQ event %d\n", etype);
	1729	+ base_cq->event_handler(&event, base_cq->cq_context);
	1730	+ }
	1731	+}
	1732	+
	1733	+void siw_srq_event(struct siw_srq *srq, enum ib_event_type etype)
	1734	+{
	1735	+ struct ib_event event;
	1736	+ struct ib_srq *base_srq = &srq->base_srq;
	1737	+
	1738	+ event.event = etype;
	1739	+ event.device = base_srq->device;
	1740	+ event.element.srq = base_srq;
	1741	+
	1742	+ if (base_srq->event_handler) {
	1743	+ siw_dbg_pd(srq->base_srq.pd,
	1744	+ "reporting SRQ event %d\n", etype);
	1745	+ base_srq->event_handler(&event, base_srq->srq_context);
	1746	+ }
	1747	+}
	1748	+
	1749	+void siw_port_event(struct siw_device *sdev, u8 port, enum ib_event_type etype)
	1750	+{
	1751	+ struct ib_event event;
	1752	+
	1753	+ event.event = etype;
	1754	+ event.device = &sdev->base_dev;
	1755	+ event.element.port_num = port;
	1756	+
	1757	+ siw_dbg(&sdev->base_dev, "reporting port event %d\n", etype);
	1758	+
	1759	+ ib_dispatch_event(&event);
	1760	+}

drivers/infiniband/sw/siw/siw_verbs.h+91−0

@@ -0,0 +1,91 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */
	2	+
	3	+/* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
	4	+/* Copyright (c) 2008-2019, IBM Corporation */
	5	+
	6	+#ifndef _SIW_VERBS_H
	7	+#define _SIW_VERBS_H
	8	+
	9	+#include <linux/errno.h>
	10	+
	11	+#include <rdma/iw_cm.h>
	12	+#include <rdma/ib_verbs.h>
	13	+#include <rdma/ib_user_verbs.h>
	14	+
	15	+#include "siw.h"
	16	+#include "siw_cm.h"
	17	+
	18	+/*
	19	+ * siw_copy_sgl()
	20	+ *
	21	+ * Copy SGL from RDMA core representation to local
	22	+ * representation.
	23	+ */
	24	+static inline void siw_copy_sgl(struct ib_sge sge, struct siw_sge siw_sge,
	25	+ int num_sge)
	26	+{
	27	+ while (num_sge--) {
	28	+ siw_sge->laddr = sge->addr;
	29	+ siw_sge->length = sge->length;
	30	+ siw_sge->lkey = sge->lkey;
	31	+
	32	+ siw_sge++;
	33	+ sge++;
	34	+ }
	35	+}
	36	+
	37	+int siw_alloc_ucontext(struct ib_ucontext base_ctx, struct ib_udata udata);
	38	+void siw_dealloc_ucontext(struct ib_ucontext *base_ctx);
	39	+int siw_query_port(struct ib_device *base_dev, u8 port,
	40	+ struct ib_port_attr *attr);
	41	+int siw_get_port_immutable(struct ib_device *base_dev, u8 port,
	42	+ struct ib_port_immutable *port_immutable);
	43	+int siw_query_device(struct ib_device base_dev, struct ib_device_attr attr,
	44	+ struct ib_udata *udata);
	45	+int siw_create_cq(struct ib_cq base_cq, const struct ib_cq_init_attr attr,
	46	+ struct ib_udata *udata);
	47	+int siw_query_port(struct ib_device *base_dev, u8 port,
	48	+ struct ib_port_attr *attr);
	49	+int siw_query_pkey(struct ib_device base_dev, u8 port, u16 idx, u16 pkey);
	50	+int siw_query_gid(struct ib_device *base_dev, u8 port, int idx,
	51	+ union ib_gid *gid);
	52	+int siw_alloc_pd(struct ib_pd base_pd, struct ib_udata udata);
	53	+void siw_dealloc_pd(struct ib_pd base_pd, struct ib_udata udata);
	54	+struct ib_qp siw_create_qp(struct ib_pd base_pd,
	55	+ struct ib_qp_init_attr *attr,
	56	+ struct ib_udata *udata);
	57	+int siw_query_qp(struct ib_qp base_qp, struct ib_qp_attr qp_attr,
	58	+ int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);
	59	+int siw_verbs_modify_qp(struct ib_qp base_qp, struct ib_qp_attr attr,
	60	+ int attr_mask, struct ib_udata *udata);
	61	+int siw_destroy_qp(struct ib_qp base_qp, struct ib_udata udata);
	62	+int siw_post_send(struct ib_qp base_qp, const struct ib_send_wr wr,
	63	+ const struct ib_send_wr **bad_wr);
	64	+int siw_post_receive(struct ib_qp base_qp, const struct ib_recv_wr wr,
	65	+ const struct ib_recv_wr **bad_wr);
	66	+void siw_destroy_cq(struct ib_cq base_cq, struct ib_udata udata);
	67	+int siw_poll_cq(struct ib_cq base_cq, int num_entries, struct ib_wc wc);
	68	+int siw_req_notify_cq(struct ib_cq *base_cq, enum ib_cq_notify_flags flags);
	69	+struct ib_mr siw_reg_user_mr(struct ib_pd base_pd, u64 start, u64 len,
	70	+ u64 rnic_va, int rights, struct ib_udata *udata);
	71	+struct ib_mr siw_alloc_mr(struct ib_pd base_pd, enum ib_mr_type mr_type,
	72	+ u32 max_sge, struct ib_udata *udata);
	73	+struct ib_mr siw_get_dma_mr(struct ib_pd base_pd, int rights);
	74	+int siw_map_mr_sg(struct ib_mr base_mr, struct scatterlist sl, int num_sle,
	75	+ unsigned int *sg_off);
	76	+int siw_dereg_mr(struct ib_mr base_mr, struct ib_udata udata);
	77	+int siw_create_srq(struct ib_srq base_srq, struct ib_srq_init_attr attr,
	78	+ struct ib_udata *udata);
	79	+int siw_modify_srq(struct ib_srq base_srq, struct ib_srq_attr attr,
	80	+ enum ib_srq_attr_mask mask, struct ib_udata *udata);
	81	+int siw_query_srq(struct ib_srq base_srq, struct ib_srq_attr attr);
	82	+void siw_destroy_srq(struct ib_srq base_srq, struct ib_udata udata);
	83	+int siw_post_srq_recv(struct ib_srq base_srq, const struct ib_recv_wr wr,
	84	+ const struct ib_recv_wr **bad_wr);
	85	+int siw_mmap(struct ib_ucontext ctx, struct vm_area_struct vma);
	86	+void siw_qp_event(struct siw_qp *qp, enum ib_event_type type);
	87	+void siw_cq_event(struct siw_cq *cq, enum ib_event_type type);
	88	+void siw_srq_event(struct siw_srq *srq, enum ib_event_type type);
	89	+void siw_port_event(struct siw_device *dev, u8 port, enum ib_event_type type);
	90	+
	91	+#endif

include/uapi/rdma/rdma_user_ioctl_cmds.h+1−0

@@ -103,6 +103,7 @@ enum rdma_driver_id { · 102 unmodified lines
103	103	RDMA_DRIVER_HFI1,
104	104	RDMA_DRIVER_QIB,
105	105	RDMA_DRIVER_EFA,
	106	+ RDMA_DRIVER_SIW,
106	107	};
107	108
108	109	#endif

include/uapi/rdma/siw-abi.h+185−0

@@ -0,0 +1,185 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */
	2	+
	3	+/* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
	4	+/* Copyright (c) 2008-2019, IBM Corporation */
	5	+
	6	+#ifndef _SIW_USER_H
	7	+#define _SIW_USER_H
	8	+
	9	+#include <linux/types.h>
	10	+
	11	+#define SIW_NODE_DESC_COMMON "Software iWARP stack"
	12	+#define SIW_ABI_VERSION 1
	13	+#define SIW_MAX_SGE 6
	14	+#define SIW_UOBJ_MAX_KEY 0x08FFFF
	15	+#define SIW_INVAL_UOBJ_KEY (SIW_UOBJ_MAX_KEY + 1)
	16	+
	17	+struct siw_uresp_create_cq {
	18	+ __u32 cq_id;
	19	+ __u32 num_cqe;
	20	+ __aligned_u64 cq_key;
	21	+};
	22	+
	23	+struct siw_uresp_create_qp {
	24	+ __u32 qp_id;
	25	+ __u32 num_sqe;
	26	+ __u32 num_rqe;
	27	+ __u32 pad;
	28	+ __aligned_u64 sq_key;
	29	+ __aligned_u64 rq_key;
	30	+};
	31	+
	32	+struct siw_ureq_reg_mr {
	33	+ __u8 stag_key;
	34	+ __u8 reserved[3];
	35	+ __u32 pad;
	36	+};
	37	+
	38	+struct siw_uresp_reg_mr {
	39	+ __u32 stag;
	40	+ __u32 pad;
	41	+};
	42	+
	43	+struct siw_uresp_create_srq {
	44	+ __u32 num_rqe;
	45	+ __u32 pad;
	46	+ __aligned_u64 srq_key;
	47	+};
	48	+
	49	+struct siw_uresp_alloc_ctx {
	50	+ __u32 dev_id;
	51	+ __u32 pad;
	52	+};
	53	+
	54	+enum siw_opcode {
	55	+ SIW_OP_WRITE,
	56	+ SIW_OP_READ,
	57	+ SIW_OP_READ_LOCAL_INV,
	58	+ SIW_OP_SEND,
	59	+ SIW_OP_SEND_WITH_IMM,
	60	+ SIW_OP_SEND_REMOTE_INV,
	61	+
	62	+ /* Unsupported */
	63	+ SIW_OP_FETCH_AND_ADD,
	64	+ SIW_OP_COMP_AND_SWAP,
	65	+
	66	+ SIW_OP_RECEIVE,
	67	+ /* provider internal SQE */
	68	+ SIW_OP_READ_RESPONSE,
	69	+ /*
	70	+ * below opcodes valid for
	71	+ * in-kernel clients only
	72	+ */
	73	+ SIW_OP_INVAL_STAG,
	74	+ SIW_OP_REG_MR,
	75	+ SIW_NUM_OPCODES
	76	+};
	77	+
	78	+/* Keep it same as ibv_sge to allow for memcpy */
	79	+struct siw_sge {
	80	+ __aligned_u64 laddr;
	81	+ __u32 length;
	82	+ __u32 lkey;
	83	+};
	84	+
	85	+/*
	86	+ * Inline data are kept within the work request itself occupying
	87	+ * the space of sge[1] .. sge[n]. Therefore, inline data cannot be
	88	+ * supported if SIW_MAX_SGE is below 2 elements.
	89	+ */
	90	+#define SIW_MAX_INLINE (sizeof(struct siw_sge) * (SIW_MAX_SGE - 1))
	91	+
	92	+#if SIW_MAX_SGE < 2
	93	+#error "SIW_MAX_SGE must be at least 2"
	94	+#endif
	95	+
	96	+enum siw_wqe_flags {
	97	+ SIW_WQE_VALID = 1,
	98	+ SIW_WQE_INLINE = (1 << 1),
	99	+ SIW_WQE_SIGNALLED = (1 << 2),
	100	+ SIW_WQE_SOLICITED = (1 << 3),
	101	+ SIW_WQE_READ_FENCE = (1 << 4),
	102	+ SIW_WQE_REM_INVAL = (1 << 5),
	103	+ SIW_WQE_COMPLETED = (1 << 6)
	104	+};
	105	+
	106	+/* Send Queue Element */
	107	+struct siw_sqe {
	108	+ __aligned_u64 id;
	109	+ __u16 flags;
	110	+ __u8 num_sge;
	111	+ /* Contains enum siw_opcode values */
	112	+ __u8 opcode;
	113	+ __u32 rkey;
	114	+ union {
	115	+ __aligned_u64 raddr;
	116	+ __aligned_u64 base_mr;
	117	+ };
	118	+ union {
	119	+ struct siw_sge sge[SIW_MAX_SGE];
	120	+ __aligned_u64 access;
	121	+ };
	122	+};
	123	+
	124	+/* Receive Queue Element */
	125	+struct siw_rqe {
	126	+ __aligned_u64 id;
	127	+ __u16 flags;
	128	+ __u8 num_sge;
	129	+ /*
	130	+ * only used by kernel driver,
	131	+ * ignored if set by user
	132	+ */
	133	+ __u8 opcode;
	134	+ __u32 unused;
	135	+ struct siw_sge sge[SIW_MAX_SGE];
	136	+};
	137	+
	138	+enum siw_notify_flags {
	139	+ SIW_NOTIFY_NOT = (0),
	140	+ SIW_NOTIFY_SOLICITED = (1 << 0),
	141	+ SIW_NOTIFY_NEXT_COMPLETION = (1 << 1),
	142	+ SIW_NOTIFY_MISSED_EVENTS = (1 << 2),
	143	+ SIW_NOTIFY_ALL = SIW_NOTIFY_SOLICITED \| SIW_NOTIFY_NEXT_COMPLETION \|
	144	+ SIW_NOTIFY_MISSED_EVENTS
	145	+};
	146	+
	147	+enum siw_wc_status {
	148	+ SIW_WC_SUCCESS,
	149	+ SIW_WC_LOC_LEN_ERR,
	150	+ SIW_WC_LOC_PROT_ERR,
	151	+ SIW_WC_LOC_QP_OP_ERR,
	152	+ SIW_WC_WR_FLUSH_ERR,
	153	+ SIW_WC_BAD_RESP_ERR,
	154	+ SIW_WC_LOC_ACCESS_ERR,
	155	+ SIW_WC_REM_ACCESS_ERR,
	156	+ SIW_WC_REM_INV_REQ_ERR,
	157	+ SIW_WC_GENERAL_ERR,
	158	+ SIW_NUM_WC_STATUS
	159	+};
	160	+
	161	+struct siw_cqe {
	162	+ __aligned_u64 id;
	163	+ __u8 flags;
	164	+ __u8 opcode;
	165	+ __u16 status;
	166	+ __u32 bytes;
	167	+ union {
	168	+ __aligned_u64 imm_data;
	169	+ __u32 inval_stag;
	170	+ };
	171	+ /* QP number or QP pointer */
	172	+ union {
	173	+ struct ib_qp *base_qp;
	174	+ __aligned_u64 qp_id;
	175	+ };
	176	+};
	177	+
	178	+/*
	179	+ * Shared structure between user and kernel
	180	+ * to control CQ arming.
	181	+ */
	182	+struct siw_cq_ctrl {
	183	+ __aligned_u64 notify;
	184	+};
	185	+#endif