Merge tag 'nfs-rdma-for-4.18-1' of git://git.linux-nfs.org/projects/anna/linux-nfs

NFS-over-RDMA client updates for Linux 4.18 Stable patches: - xprtrdma: Return -ENOBUFS when no pages are available New features: - Add ->alloc_slot() and ->free_slot() functions Bugfixes and cleanups: - Add missing SPDX tags to some files - Try to fail mount quickly if client has no RDMA devices - Create transport IDs in the correct network namespace - Fix max_send_wr computation - Clean up receive tracepoints - Refactor receive handling - Remove unused functions

Trond Myklebust committed 8 years ago

Commit fcda3d52 parents 3f0b3cf 11d0ac116 files · +359 −355Browse files

Unified · Split

16 files

include/linux/sunrpc/rpc_rdma.h +1 −0
include/linux/sunrpc/xprt.h +5 −1
include/linux/sunrpc/xprtrdma.h +1 −0
include/trace/events/rpcrdma.h +53 −23
net/sunrpc/clnt.c +1 −0
net/sunrpc/xprt.c +10 −7
net/sunrpc/xprtrdma/backchannel.c +39 −66
net/sunrpc/xprtrdma/fmr_ops.c +23 −0
net/sunrpc/xprtrdma/frwr_ops.c +27 −4
net/sunrpc/xprtrdma/module.c +1 −0
net/sunrpc/xprtrdma/rpc_rdma.c +22 −44
net/sunrpc/xprtrdma/svc_rdma_backchannel.c +1 −0
net/sunrpc/xprtrdma/transport.c +47 −17
net/sunrpc/xprtrdma/verbs.c +112 −179
net/sunrpc/xprtrdma/xprt_rdma.h +12 −14
net/sunrpc/xprtsock.c +4 −0

include/linux/sunrpc/rpc_rdma.h+1−0

@@ -1,3 +1,4 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
1	2	/*
2	3	* Copyright (c) 2015-2017 Oracle. All rights reserved.
3	4	* Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.

include/linux/sunrpc/xprt.h+5−1

@@ -84,7 +84,6 @@ struct rpc_rqst { · 83 unmodified lines
84	84	void (rq_release_snd_buf)(struct rpc_rqst ); /* release rq_enc_pages */
85	85	struct list_head rq_list;
86	86
87		− void rq_xprtdata; / Per-xprt private data */
88	87	void rq_buffer; / Call XDR encode buffer */
89	88	size_t rq_callsize;
90	89	void rq_rbuffer; / Reply XDR decode buffer */
@@ -127,6 +126,8 @@ struct rpc_xprt_ops { · 36 unmodified lines
127	126	int (reserve_xprt)(struct rpc_xprt xprt, struct rpc_task *task);
128	127	void (release_xprt)(struct rpc_xprt xprt, struct rpc_task *task);
129	128	void (alloc_slot)(struct rpc_xprt xprt, struct rpc_task *task);
	129	+ void (free_slot)(struct rpc_xprt xprt,
	130	+ struct rpc_rqst *req);
130	131	void (rpcbind)(struct rpc_task task);
131	132	void (set_port)(struct rpc_xprt xprt, unsigned short port);
132	133	void (connect)(struct rpc_xprt xprt, struct rpc_task *task);
@@ -324,10 +325,13 @@ struct xprt_class { · 191 unmodified lines
324	325	struct rpc_xprt xprt_create_transport(struct xprt_create args);
325	326	void xprt_connect(struct rpc_task *task);
326	327	void xprt_reserve(struct rpc_task *task);
	328	+void xprt_request_init(struct rpc_task *task);
327	329	void xprt_retry_reserve(struct rpc_task *task);
328	330	int xprt_reserve_xprt(struct rpc_xprt xprt, struct rpc_task task);
329	331	int xprt_reserve_xprt_cong(struct rpc_xprt xprt, struct rpc_task task);
330	332	void xprt_alloc_slot(struct rpc_xprt xprt, struct rpc_task task);
	333	+void xprt_free_slot(struct rpc_xprt *xprt,
	334	+ struct rpc_rqst *req);
331	335	void xprt_lock_and_alloc_slot(struct rpc_xprt xprt, struct rpc_task task);
332	336	bool xprt_prepare_transmit(struct rpc_task *task);
333	337	void xprt_transmit(struct rpc_task *task);

include/linux/sunrpc/xprtrdma.h+1−0

@@ -1,3 +1,4 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
1	2	/*
2	3	* Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
3	4	*

include/trace/events/rpcrdma.h+53−23

@@ -528,24 +528,54 @@ TRACE_EVENT(xprtrdma_post_send, · 527 unmodified lines
528	528
529	529	TRACE_EVENT(xprtrdma_post_recv,
530	530	TP_PROTO(
531		− const struct rpcrdma_rep *rep,
	531	+ const struct ib_cqe *cqe
	532	+ ),
	533	+
	534	+ TP_ARGS(cqe),
	535	+
	536	+ TP_STRUCT__entry(
	537	+ __field(const void *, cqe)
	538	+ ),
	539	+
	540	+ TP_fast_assign(
	541	+ __entry->cqe = cqe;
	542	+ ),
	543	+
	544	+ TP_printk("cqe=%p",
	545	+ __entry->cqe
	546	+ )
	547	+);
	548	+
	549	+TRACE_EVENT(xprtrdma_post_recvs,
	550	+ TP_PROTO(
	551	+ const struct rpcrdma_xprt *r_xprt,
	552	+ unsigned int count,
532	553	int status
533	554	),
534	555
535		− TP_ARGS(rep, status),
	556	+ TP_ARGS(r_xprt, count, status),
536	557
537	558	TP_STRUCT__entry(
538		− __field(const void *, rep)
	559	+ __field(const void *, r_xprt)
	560	+ __field(unsigned int, count)
539	561	__field(int, status)
	562	+ __field(int, posted)
	563	+ __string(addr, rpcrdma_addrstr(r_xprt))
	564	+ __string(port, rpcrdma_portstr(r_xprt))
540	565	),
541	566
542	567	TP_fast_assign(
543		− __entry->rep = rep;
	568	+ __entry->r_xprt = r_xprt;
	569	+ __entry->count = count;
544	570	__entry->status = status;
	571	+ __entry->posted = r_xprt->rx_buf.rb_posted_receives;
	572	+ __assign_str(addr, rpcrdma_addrstr(r_xprt));
	573	+ __assign_str(port, rpcrdma_portstr(r_xprt));
545	574	),
546	575
547		− TP_printk("rep=%p status=%d",
548		− __entry->rep, __entry->status
	576	+ TP_printk("peer=[%s]:%s r_xprt=%p: %u new recvs, %d active (rc %d)",
	577	+ __get_str(addr), __get_str(port), __entry->r_xprt,
	578	+ __entry->count, __entry->posted, __entry->status
549	579	)
550	580	);
551	581
@@ -584,28 +614,32 @@ TRACE_EVENT(xprtrdma_wc_send, · 32 unmodified lines
584	614
585	615	TRACE_EVENT(xprtrdma_wc_receive,
586	616	TP_PROTO(
587		− const struct rpcrdma_rep *rep,
588	617	const struct ib_wc *wc
589	618	),
590	619
591		− TP_ARGS(rep, wc),
	620	+ TP_ARGS(wc),
592	621
593	622	TP_STRUCT__entry(
594		− __field(const void *, rep)
595		− __field(unsigned int, byte_len)
	623	+ __field(const void *, cqe)
	624	+ __field(u32, byte_len)
596	625	__field(unsigned int, status)
597		− __field(unsigned int, vendor_err)
	626	+ __field(u32, vendor_err)
598	627	),
599	628
600	629	TP_fast_assign(
601		− __entry->rep = rep;
602		− __entry->byte_len = wc->byte_len;
	630	+ __entry->cqe = wc->wr_cqe;
603	631	__entry->status = wc->status;
604		− __entry->vendor_err = __entry->status ? wc->vendor_err : 0;
	632	+ if (wc->status) {
	633	+ __entry->byte_len = 0;
	634	+ __entry->vendor_err = wc->vendor_err;
	635	+ } else {
	636	+ __entry->byte_len = wc->byte_len;
	637	+ __entry->vendor_err = 0;
	638	+ }
605	639	),
606	640
607		− TP_printk("rep=%p, %u bytes: %s (%u/0x%x)",
608		− __entry->rep, __entry->byte_len,
	641	+ TP_printk("cqe=%p %u bytes: %s (%u/0x%x)",
	642	+ __entry->cqe, __entry->byte_len,
609	643	rdma_show_wc_status(__entry->status),
610	644	__entry->status, __entry->vendor_err
611	645	)
@@ -616,6 +650,7 @@ DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li); · 4 unmodified lines
616	650	DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li_wake);
617	651
618	652	DEFINE_MR_EVENT(xprtrdma_localinv);
	653	+DEFINE_MR_EVENT(xprtrdma_dma_map);
619	654	DEFINE_MR_EVENT(xprtrdma_dma_unmap);
620	655	DEFINE_MR_EVENT(xprtrdma_remoteinv);
621	656	DEFINE_MR_EVENT(xprtrdma_recover_mr);
@@ -799,7 +834,6 @@ TRACE_EVENT(xprtrdma_allocate, · 177 unmodified lines
799	834	__field(unsigned int, task_id)
800	835	__field(unsigned int, client_id)
801	836	__field(const void *, req)
802		− __field(const void *, rep)
803	837	__field(size_t, callsize)
804	838	__field(size_t, rcvsize)
805	839	),
@@ -808,15 +842,13 @@ TRACE_EVENT(xprtrdma_allocate, · 2 unmodified lines
808	842	__entry->task_id = task->tk_pid;
809	843	__entry->client_id = task->tk_client->cl_clid;
810	844	__entry->req = req;
811		− __entry->rep = req ? req->rl_reply : NULL;
812	845	__entry->callsize = task->tk_rqstp->rq_callsize;
813	846	__entry->rcvsize = task->tk_rqstp->rq_rcvsize;
814	847	),
815	848
816		− TP_printk("task:%u@%u req=%p rep=%p (%zu, %zu)",
	849	+ TP_printk("task:%u@%u req=%p (%zu, %zu)",
817	850	__entry->task_id, __entry->client_id,
818		− __entry->req, __entry->rep,
819		− __entry->callsize, __entry->rcvsize
	851	+ __entry->req, __entry->callsize, __entry->rcvsize
820	852	)
821	853	);
822	854
@@ -848,8 +880,6 @@ TRACE_EVENT(xprtrdma_rpc_done, · 25 unmodified lines
848	880	)
849	881	);
850	882
851		−DEFINE_RXPRT_EVENT(xprtrdma_noreps);
852		−
853	883	/**
854	884	** Callback events
855	885	**/

net/sunrpc/clnt.c+1−0

@@ -1546,6 +1546,7 @@ call_reserveresult(struct rpc_task *task) · 1545 unmodified lines
1546	1546	task->tk_status = 0;
1547	1547	if (status >= 0) {
1548	1548	if (task->tk_rqstp) {
	1549	+ xprt_request_init(task);
1549	1550	task->tk_action = call_refresh;
1550	1551	return;
1551	1552	}

net/sunrpc/xprt.c+10−7

@@ -66,7 +66,7 @@ · 65 unmodified lines
66	66	* Local functions
67	67	*/
68	68	static void xprt_init(struct rpc_xprt xprt, struct net net);
69		−static void xprt_request_init(struct rpc_task , struct rpc_xprt );
	69	+static __be32 xprt_alloc_xid(struct rpc_xprt *xprt);
70	70	static void xprt_connect_status(struct rpc_task *task);
71	71	static int __xprt_get_cong(struct rpc_xprt , struct rpc_task );
72	72	static void __xprt_put_cong(struct rpc_xprt , struct rpc_rqst );
@@ -987,6 +987,8 @@ bool xprt_prepare_transmit(struct rpc_task *task) · 914 unmodified lines
987	987	task->tk_status = -EAGAIN;
988	988	goto out_unlock;
989	989	}
	990	+ if (!bc_prealloc(req) && !req->rq_xmit_bytes_sent)
	991	+ req->rq_xid = xprt_alloc_xid(xprt);
990	992	ret = true;
991	993	out_unlock:
992	994	spin_unlock_bh(&xprt->transport_lock);
@@ -1163,10 +1165,10 @@ void xprt_alloc_slot(struct rpc_xprt xprt, struct rpc_task task) · 170 unmodified lines
1163	1165	out_init_req:
1164	1166	xprt->stat.max_slots = max_t(unsigned int, xprt->stat.max_slots,
1165	1167	xprt->num_reqs);
	1168	+ spin_unlock(&xprt->reserve_lock);
	1169	+
1166	1170	task->tk_status = 0;
1167	1171	task->tk_rqstp = req;
1168		− xprt_request_init(task, xprt);
1169		− spin_unlock(&xprt->reserve_lock);
1170	1172	}
1171	1173	EXPORT_SYMBOL_GPL(xprt_alloc_slot);
1172	1174
@@ -1184,7 +1186,7 @@ void xprt_lock_and_alloc_slot(struct rpc_xprt xprt, struct rpc_task task) · 11 unmodified lines
1184	1186	}
1185	1187	EXPORT_SYMBOL_GPL(xprt_lock_and_alloc_slot);
1186	1188
1187		−static void xprt_free_slot(struct rpc_xprt xprt, struct rpc_rqst req)
	1189	+void xprt_free_slot(struct rpc_xprt xprt, struct rpc_rqst req)
1188	1190	{
1189	1191	spin_lock(&xprt->reserve_lock);
1190	1192	if (!xprt_dynamic_free_slot(xprt, req)) {
@@ -1194,6 +1196,7 @@ static void xprt_free_slot(struct rpc_xprt xprt, struct rpc_rqst req) · 3 unmodified lines
1194	1196	xprt_wake_up_backlog(xprt);
1195	1197	spin_unlock(&xprt->reserve_lock);
1196	1198	}
	1199	+EXPORT_SYMBOL_GPL(xprt_free_slot);
1197	1200
1198	1201	static void xprt_free_all_slots(struct rpc_xprt *xprt)
1199	1202	{
@@ -1303,8 +1306,9 @@ static inline void xprt_init_xid(struct rpc_xprt *xprt) · 103 unmodified lines
1303	1306	xprt->xid = prandom_u32();
1304	1307	}
1305	1308
1306		−static void xprt_request_init(struct rpc_task task, struct rpc_xprt xprt)
	1309	+void xprt_request_init(struct rpc_task *task)
1307	1310	{
	1311	+ struct rpc_xprt *xprt = task->tk_xprt;
1308	1312	struct rpc_rqst *req = task->tk_rqstp;
1309	1313
1310	1314	INIT_LIST_HEAD(&req->rq_list);
@@ -1312,7 +1316,6 @@ static void xprt_request_init(struct rpc_task task, struct rpc_xprt xprt) · 1 unmodified line
1312	1316	req->rq_task = task;
1313	1317	req->rq_xprt = xprt;
1314	1318	req->rq_buffer = NULL;
1315		− req->rq_xid = xprt_alloc_xid(xprt);
1316	1319	req->rq_connect_cookie = xprt->connect_cookie - 1;
1317	1320	req->rq_bytes_sent = 0;
1318	1321	req->rq_snd_buf.len = 0;
@@ -1373,7 +1376,7 @@ void xprt_release(struct rpc_task *task) · 54 unmodified lines
1373	1376
1374	1377	dprintk("RPC: %5u release request %p\n", task->tk_pid, req);
1375	1378	if (likely(!bc_prealloc(req)))
1376		− xprt_free_slot(xprt, req);
	1379	+ xprt->ops->free_slot(xprt, req);
1377	1380	else
1378	1381	xprt_free_bc_request(req);
1379	1382	}

net/sunrpc/xprtrdma/backchannel.c+39−66

@@ -29,29 +29,41 @@ static void rpcrdma_bc_free_rqst(struct rpcrdma_xprt *r_xprt, · 28 unmodified lines
29	29	spin_unlock(&buf->rb_reqslock);
30	30
31	31	rpcrdma_destroy_req(req);
32		−
33		− kfree(rqst);
34	32	}
35	33
36		−static int rpcrdma_bc_setup_rqst(struct rpcrdma_xprt *r_xprt,
37		− struct rpc_rqst *rqst)
	34	+static int rpcrdma_bc_setup_reqs(struct rpcrdma_xprt *r_xprt,
	35	+ unsigned int count)
38	36	{
39		− struct rpcrdma_regbuf *rb;
40		− struct rpcrdma_req *req;
41		− size_t size;
	37	+ struct rpc_xprt *xprt = &r_xprt->rx_xprt;
	38	+ struct rpc_rqst *rqst;
	39	+ unsigned int i;
	40	+
	41	+ for (i = 0; i < (count << 1); i++) {
	42	+ struct rpcrdma_regbuf *rb;
	43	+ struct rpcrdma_req *req;
	44	+ size_t size;
	45	+
	46	+ req = rpcrdma_create_req(r_xprt);
	47	+ if (IS_ERR(req))
	48	+ return PTR_ERR(req);
	49	+ rqst = &req->rl_slot;
	50	+
	51	+ rqst->rq_xprt = xprt;
	52	+ INIT_LIST_HEAD(&rqst->rq_list);
	53	+ INIT_LIST_HEAD(&rqst->rq_bc_list);
	54	+ __set_bit(RPC_BC_PA_IN_USE, &rqst->rq_bc_pa_state);
	55	+ spin_lock_bh(&xprt->bc_pa_lock);
	56	+ list_add(&rqst->rq_bc_pa_list, &xprt->bc_pa_list);
	57	+ spin_unlock_bh(&xprt->bc_pa_lock);
42	58
43		− req = rpcrdma_create_req(r_xprt);
44		− if (IS_ERR(req))
45		− return PTR_ERR(req);
46		−
47		− size = r_xprt->rx_data.inline_rsize;
48		− rb = rpcrdma_alloc_regbuf(size, DMA_TO_DEVICE, GFP_KERNEL);
49		− if (IS_ERR(rb))
50		− goto out_fail;
51		− req->rl_sendbuf = rb;
52		− xdr_buf_init(&rqst->rq_snd_buf, rb->rg_base,
53		− min_t(size_t, size, PAGE_SIZE));
54		− rpcrdma_set_xprtdata(rqst, req);
	59	+ size = r_xprt->rx_data.inline_rsize;
	60	+ rb = rpcrdma_alloc_regbuf(size, DMA_TO_DEVICE, GFP_KERNEL);
	61	+ if (IS_ERR(rb))
	62	+ goto out_fail;
	63	+ req->rl_sendbuf = rb;
	64	+ xdr_buf_init(&rqst->rq_snd_buf, rb->rg_base,
	65	+ min_t(size_t, size, PAGE_SIZE));
	66	+ }
55	67	return 0;
56	68
57	69	out_fail:
@@ -59,23 +71,6 @@ out_fail: · 1 unmodified line
59	71	return -ENOMEM;
60	72	}
61	73
62		−/* Allocate and add receive buffers to the rpcrdma_buffer's
63		− * existing list of rep's. These are released when the
64		− * transport is destroyed.
65		− */
66		−static int rpcrdma_bc_setup_reps(struct rpcrdma_xprt *r_xprt,
67		− unsigned int count)
68		−{
69		− int rc = 0;
70		−
71		− while (count--) {
72		− rc = rpcrdma_create_rep(r_xprt);
73		− if (rc)
74		− break;
75		− }
76		− return rc;
77		−}
78		−
79	74	/**
80	75	* xprt_rdma_bc_setup - Pre-allocate resources for handling backchannel requests
81	76	* @xprt: transport associated with these backchannel resources
@@ -86,9 +81,6 @@ static int rpcrdma_bc_setup_reps(struct rpcrdma_xprt *r_xprt, · 4 unmodified lines
86	81	int xprt_rdma_bc_setup(struct rpc_xprt *xprt, unsigned int reqs)
87	82	{
88	83	struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);
89		− struct rpcrdma_buffer *buffer = &r_xprt->rx_buf;
90		− struct rpc_rqst *rqst;
91		− unsigned int i;
92	84	int rc;
93	85
94	86	/* The backchannel reply path returns each rpc_rqst to the
@@ -103,35 +95,11 @@ int xprt_rdma_bc_setup(struct rpc_xprt *xprt, unsigned int reqs) · 8 unmodified lines
103	95	if (reqs > RPCRDMA_BACKWARD_WRS >> 1)
104	96	goto out_err;
105	97
106		− for (i = 0; i < (reqs << 1); i++) {
107		− rqst = kzalloc(sizeof(*rqst), GFP_KERNEL);
108		− if (!rqst)
109		− goto out_free;
110		−
111		− dprintk("RPC: %s: new rqst %p\n", __func__, rqst);
112		−
113		− rqst->rq_xprt = &r_xprt->rx_xprt;
114		− INIT_LIST_HEAD(&rqst->rq_list);
115		− INIT_LIST_HEAD(&rqst->rq_bc_list);
116		− __set_bit(RPC_BC_PA_IN_USE, &rqst->rq_bc_pa_state);
117		−
118		− if (rpcrdma_bc_setup_rqst(r_xprt, rqst))
119		− goto out_free;
120		−
121		− spin_lock_bh(&xprt->bc_pa_lock);
122		− list_add(&rqst->rq_bc_pa_list, &xprt->bc_pa_list);
123		− spin_unlock_bh(&xprt->bc_pa_lock);
124		− }
125		−
126		− rc = rpcrdma_bc_setup_reps(r_xprt, reqs);
	98	+ rc = rpcrdma_bc_setup_reqs(r_xprt, reqs);
127	99	if (rc)
128	100	goto out_free;
129	101
130		− rc = rpcrdma_ep_post_extra_recv(r_xprt, reqs);
131		− if (rc)
132		− goto out_free;
133		−
134		− buffer->rb_bc_srv_max_requests = reqs;
	102	+ r_xprt->rx_buf.rb_bc_srv_max_requests = reqs;
135	103	request_module("svcrdma");
136	104	trace_xprtrdma_cb_setup(r_xprt, reqs);
137	105	return 0;
@@ -235,6 +203,7 @@ int xprt_rdma_bc_send_reply(struct rpc_rqst *rqst) · 97 unmodified lines
235	203	if (rc < 0)
236	204	goto failed_marshal;
237	205
	206	+ rpcrdma_post_recvs(r_xprt, true);
238	207	if (rpcrdma_ep_post(&r_xprt->rx_ia, &r_xprt->rx_ep, req))
239	208	goto drop_connection;
240	209	return 0;
@@ -275,10 +244,14 @@ void xprt_rdma_bc_destroy(struct rpc_xprt *xprt, unsigned int reqs) · 34 unmodified lines
275	244	*/
276	245	void xprt_rdma_bc_free_rqst(struct rpc_rqst *rqst)
277	246	{
	247	+ struct rpcrdma_req *req = rpcr_to_rdmar(rqst);
278	248	struct rpc_xprt *xprt = rqst->rq_xprt;
279	249
280	250	dprintk("RPC: %s: freeing rqst %p (req %p)\n",
281		− __func__, rqst, rpcr_to_rdmar(rqst));
	251	+ __func__, rqst, req);
	252	+
	253	+ rpcrdma_recv_buffer_put(req->rl_reply);
	254	+ req->rl_reply = NULL;
282	255
283	256	spin_lock_bh(&xprt->bc_pa_lock);
284	257	list_add_tail(&rqst->rq_bc_pa_list, &xprt->bc_pa_list);

net/sunrpc/xprtrdma/fmr_ops.c+23−0

@@ -156,10 +156,32 @@ out_release: · 155 unmodified lines
156	156	fmr_op_release_mr(mr);
157	157	}
158	158
	159	+/* On success, sets:
	160	+ * ep->rep_attr.cap.max_send_wr
	161	+ * ep->rep_attr.cap.max_recv_wr
	162	+ * cdata->max_requests
	163	+ * ia->ri_max_segs
	164	+ */
159	165	static int
160	166	fmr_op_open(struct rpcrdma_ia ia, struct rpcrdma_ep ep,
161	167	struct rpcrdma_create_data_internal *cdata)
162	168	{
	169	+ int max_qp_wr;
	170	+
	171	+ max_qp_wr = ia->ri_device->attrs.max_qp_wr;
	172	+ max_qp_wr -= RPCRDMA_BACKWARD_WRS;
	173	+ max_qp_wr -= 1;
	174	+ if (max_qp_wr < RPCRDMA_MIN_SLOT_TABLE)
	175	+ return -ENOMEM;
	176	+ if (cdata->max_requests > max_qp_wr)
	177	+ cdata->max_requests = max_qp_wr;
	178	+ ep->rep_attr.cap.max_send_wr = cdata->max_requests;
	179	+ ep->rep_attr.cap.max_send_wr += RPCRDMA_BACKWARD_WRS;
	180	+ ep->rep_attr.cap.max_send_wr += 1; /* for ib_drain_sq */
	181	+ ep->rep_attr.cap.max_recv_wr = cdata->max_requests;
	182	+ ep->rep_attr.cap.max_recv_wr += RPCRDMA_BACKWARD_WRS;
	183	+ ep->rep_attr.cap.max_recv_wr += 1; /* for ib_drain_rq */
	184	+
163	185	ia->ri_max_segs = max_t(unsigned int, 1, RPCRDMA_MAX_DATA_SEGS /
164	186	RPCRDMA_MAX_FMR_SGES);
165	187	return 0;
@@ -219,6 +241,7 @@ fmr_op_map(struct rpcrdma_xprt r_xprt, struct rpcrdma_mr_seg seg, · 53 unmodified lines
219	241	mr->mr_sg, i, mr->mr_dir);
220	242	if (!mr->mr_nents)
221	243	goto out_dmamap_err;
	244	+ trace_xprtrdma_dma_map(mr);
222	245
223	246	for (i = 0, dma_pages = mr->fmr.fm_physaddrs; i < mr->mr_nents; i++)
224	247	dma_pages[i] = sg_dma_address(&mr->mr_sg[i]);

net/sunrpc/xprtrdma/frwr_ops.c+27−4

@@ -202,12 +202,22 @@ out_release: · 201 unmodified lines
202	202	frwr_op_release_mr(mr);
203	203	}
204	204
	205	+/* On success, sets:
	206	+ * ep->rep_attr.cap.max_send_wr
	207	+ * ep->rep_attr.cap.max_recv_wr
	208	+ * cdata->max_requests
	209	+ * ia->ri_max_segs
	210	+ *
	211	+ * And these FRWR-related fields:
	212	+ * ia->ri_max_frwr_depth
	213	+ * ia->ri_mrtype
	214	+ */
205	215	static int
206	216	frwr_op_open(struct rpcrdma_ia ia, struct rpcrdma_ep ep,
207	217	struct rpcrdma_create_data_internal *cdata)
208	218	{
209	219	struct ib_device_attr *attrs = &ia->ri_device->attrs;
210		− int depth, delta;
	220	+ int max_qp_wr, depth, delta;
211	221
212	222	ia->ri_mrtype = IB_MR_TYPE_MEM_REG;
213	223	if (attrs->device_cap_flags & IB_DEVICE_SG_GAPS_REG)
@@ -241,14 +251,26 @@ frwr_op_open(struct rpcrdma_ia ia, struct rpcrdma_ep ep, · 27 unmodified lines
241	251	} while (delta > 0);
242	252	}
243	253
244		− ep->rep_attr.cap.max_send_wr *= depth;
245		− if (ep->rep_attr.cap.max_send_wr > attrs->max_qp_wr) {
246		− cdata->max_requests = attrs->max_qp_wr / depth;
	254	+ max_qp_wr = ia->ri_device->attrs.max_qp_wr;
	255	+ max_qp_wr -= RPCRDMA_BACKWARD_WRS;
	256	+ max_qp_wr -= 1;
	257	+ if (max_qp_wr < RPCRDMA_MIN_SLOT_TABLE)
	258	+ return -ENOMEM;
	259	+ if (cdata->max_requests > max_qp_wr)
	260	+ cdata->max_requests = max_qp_wr;
	261	+ ep->rep_attr.cap.max_send_wr = cdata->max_requests * depth;
	262	+ if (ep->rep_attr.cap.max_send_wr > max_qp_wr) {
	263	+ cdata->max_requests = max_qp_wr / depth;
247	264	if (!cdata->max_requests)
248	265	return -EINVAL;
249	266	ep->rep_attr.cap.max_send_wr = cdata->max_requests *
250	267	depth;
251	268	}
	269	+ ep->rep_attr.cap.max_send_wr += RPCRDMA_BACKWARD_WRS;
	270	+ ep->rep_attr.cap.max_send_wr += 1; /* for ib_drain_sq */
	271	+ ep->rep_attr.cap.max_recv_wr = cdata->max_requests;
	272	+ ep->rep_attr.cap.max_recv_wr += RPCRDMA_BACKWARD_WRS;
	273	+ ep->rep_attr.cap.max_recv_wr += 1; /* for ib_drain_rq */
252	274
253	275	ia->ri_max_segs = max_t(unsigned int, 1, RPCRDMA_MAX_DATA_SEGS /
254	276	ia->ri_max_frwr_depth);
@@ -393,6 +415,7 @@ frwr_op_map(struct rpcrdma_xprt r_xprt, struct rpcrdma_mr_seg seg, · 138 unmodified lines
393	415	mr->mr_nents = ib_dma_map_sg(ia->ri_device, mr->mr_sg, i, mr->mr_dir);
394	416	if (!mr->mr_nents)
395	417	goto out_dmamap_err;
	418	+ trace_xprtrdma_dma_map(mr);
396	419
397	420	ibmr = frwr->fr_mr;
398	421	n = ib_map_mr_sg(ibmr, mr->mr_sg, mr->mr_nents, NULL, PAGE_SIZE);

net/sunrpc/xprtrdma/module.c+1−0

@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
1	2	/*
2	3	* Copyright (c) 2015, 2017 Oracle. All rights reserved.
3	4	*/

net/sunrpc/xprtrdma/rpc_rdma.c+22−44

@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
1	2	/*
2	3	* Copyright (c) 2014-2017 Oracle. All rights reserved.
3	4	* Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
@@ -54,14 +55,6 @@ · 50 unmodified lines
54	55	# define RPCDBG_FACILITY RPCDBG_TRANS
55	56	#endif
56	57
57		−static const char transfertypes[][12] = {
58		− "inline", /* no chunks */
59		− "read list", /* some argument via rdma read */
60		− "read list", / entire request via rdma read */
61		− "write list", /* some result via rdma write */
62		− "reply chunk" /* entire reply via rdma write */
63		−};
64		−
65	58	/* Returns size of largest RPC-over-RDMA header in a Call message
66	59	*
67	60	* The largest Call header contains a full-size Read list and a
@@ -230,7 +223,7 @@ rpcrdma_convert_iovs(struct rpcrdma_xprt r_xprt, struct xdr_buf xdrbuf, · 162 unmodified lines
230	223	*/
231	224	*ppages = alloc_page(GFP_ATOMIC);
232	225	if (!*ppages)
233		− return -EAGAIN;
	226	+ return -ENOBUFS;
234	227	}
235	228	seg->mr_page = *ppages;
236	229	seg->mr_offset = (char *)page_base;
@@ -365,7 +358,7 @@ rpcrdma_encode_read_list(struct rpcrdma_xprt r_xprt, struct rpcrdma_req req, · 128 unmodified lines
365	358	seg = r_xprt->rx_ia.ri_ops->ro_map(r_xprt, seg, nsegs,
366	359	false, &mr);
367	360	if (IS_ERR(seg))
368		− goto out_maperr;
	361	+ return PTR_ERR(seg);
369	362	rpcrdma_mr_push(mr, &req->rl_registered);
370	363
371	364	if (encode_read_segment(xdr, mr, pos) < 0)
@@ -377,11 +370,6 @@ rpcrdma_encode_read_list(struct rpcrdma_xprt r_xprt, struct rpcrdma_req req, · 5 unmodified lines
377	370	} while (nsegs);
378	371
379	372	return 0;
380		−
381		−out_maperr:
382		− if (PTR_ERR(seg) == -EAGAIN)
383		− xprt_wait_for_buffer_space(rqst->rq_task, NULL);
384		− return PTR_ERR(seg);
385	373	}
386	374
387	375	/* Register and XDR encode the Write list. Supports encoding a list
@@ -428,7 +416,7 @@ rpcrdma_encode_write_list(struct rpcrdma_xprt r_xprt, struct rpcrdma_req req, · 40 unmodified lines
428	416	seg = r_xprt->rx_ia.ri_ops->ro_map(r_xprt, seg, nsegs,
429	417	true, &mr);
430	418	if (IS_ERR(seg))
431		− goto out_maperr;
	419	+ return PTR_ERR(seg);
432	420	rpcrdma_mr_push(mr, &req->rl_registered);
433	421
434	422	if (encode_rdma_segment(xdr, mr) < 0)
@@ -445,11 +433,6 @@ rpcrdma_encode_write_list(struct rpcrdma_xprt r_xprt, struct rpcrdma_req req, · 10 unmodified lines
445	433	*segcount = cpu_to_be32(nchunks);
446	434
447	435	return 0;
448		−
449		−out_maperr:
450		− if (PTR_ERR(seg) == -EAGAIN)
451		− xprt_wait_for_buffer_space(rqst->rq_task, NULL);
452		− return PTR_ERR(seg);
453	436	}
454	437
455	438	/* Register and XDR encode the Reply chunk. Supports encoding an array
@@ -491,7 +474,7 @@ rpcrdma_encode_reply_chunk(struct rpcrdma_xprt r_xprt, struct rpcrdma_req req, · 35 unmodified lines
491	474	seg = r_xprt->rx_ia.ri_ops->ro_map(r_xprt, seg, nsegs,
492	475	true, &mr);
493	476	if (IS_ERR(seg))
494		− goto out_maperr;
	477	+ return PTR_ERR(seg);
495	478	rpcrdma_mr_push(mr, &req->rl_registered);
496	479
497	480	if (encode_rdma_segment(xdr, mr) < 0)
@@ -508,11 +491,6 @@ rpcrdma_encode_reply_chunk(struct rpcrdma_xprt r_xprt, struct rpcrdma_req req, · 10 unmodified lines
508	491	*segcount = cpu_to_be32(nchunks);
509	492
510	493	return 0;
511		−
512		−out_maperr:
513		− if (PTR_ERR(seg) == -EAGAIN)
514		− xprt_wait_for_buffer_space(rqst->rq_task, NULL);
515		− return PTR_ERR(seg);
516	494	}
517	495
518	496	/**
@@ -709,7 +687,7 @@ rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt, · 190 unmodified lines
709	687	{
710	688	req->rl_sendctx = rpcrdma_sendctx_get_locked(&r_xprt->rx_buf);
711	689	if (!req->rl_sendctx)
712		− return -ENOBUFS;
	690	+ return -EAGAIN;
713	691	req->rl_sendctx->sc_wr.num_sge = 0;
714	692	req->rl_sendctx->sc_unmap_count = 0;
715	693	req->rl_sendctx->sc_req = req;
@@ -883,7 +861,15 @@ rpcrdma_marshal_req(struct rpcrdma_xprt r_xprt, struct rpc_rqst rqst) · 167 unmodified lines
883	861	return 0;
884	862
885	863	out_err:
886		− r_xprt->rx_stats.failed_marshal_count++;
	864	+ switch (ret) {
	865	+ case -EAGAIN:
	866	+ xprt_wait_for_buffer_space(rqst->rq_task, NULL);
	867	+ break;
	868	+ case -ENOBUFS:
	869	+ break;
	870	+ default:
	871	+ r_xprt->rx_stats.failed_marshal_count++;
	872	+ }
887	873	return ret;
888	874	}
889	875
@@ -1026,8 +1012,6 @@ rpcrdma_is_bcall(struct rpcrdma_xprt r_xprt, struct rpcrdma_rep rep) · 136 unmodified lines
1026	1012
1027	1013	out_short:
1028	1014	pr_warn("RPC/RDMA short backward direction call\n");
1029		− if (rpcrdma_ep_post_recv(&r_xprt->rx_ia, rep))
1030		− xprt_disconnect_done(&r_xprt->rx_xprt);
1031	1015	return true;
1032	1016	}
1033	1017	#else /* CONFIG_SUNRPC_BACKCHANNEL */
@@ -1333,13 +1317,14 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *rep) · 299 unmodified lines
1333	1317	u32 credits;
1334	1318	__be32 *p;
1335	1319
	1320	+ --buf->rb_posted_receives;
	1321	+
1336	1322	if (rep->rr_hdrbuf.head[0].iov_len == 0)
1337	1323	goto out_badstatus;
1338	1324
	1325	+ /* Fixed transport header fields */
1339	1326	xdr_init_decode(&rep->rr_stream, &rep->rr_hdrbuf,
1340	1327	rep->rr_hdrbuf.head[0].iov_base);
1341		−
1342		− /* Fixed transport header fields */
1343	1328	p = xdr_inline_decode(&rep->rr_stream, 4 * sizeof(*p));
1344	1329	if (unlikely(!p))
1345	1330	goto out_shortreply;
@@ -1378,17 +1363,10 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *rep) · 32 unmodified lines
1378	1363
1379	1364	trace_xprtrdma_reply(rqst->rq_task, rep, req, credits);
1380	1365
	1366	+ rpcrdma_post_recvs(r_xprt, false);
1381	1367	queue_work(rpcrdma_receive_wq, &rep->rr_work);
1382	1368	return;
1383	1369
1384		−out_badstatus:
1385		− rpcrdma_recv_buffer_put(rep);
1386		− if (r_xprt->rx_ep.rep_connected == 1) {
1387		− r_xprt->rx_ep.rep_connected = -EIO;
1388		− rpcrdma_conn_func(&r_xprt->rx_ep);
1389		− }
1390		− return;
1391		−
1392	1370	out_badversion:
1393	1371	trace_xprtrdma_reply_vers(rep);
1394	1372	goto repost;
@@ -1408,7 +1386,7 @@ out_shortreply: · 13 unmodified lines
1408	1386	* receive buffer before returning.
1409	1387	*/
1410	1388	repost:
1411		− r_xprt->rx_stats.bad_reply_count++;
1412		− if (rpcrdma_ep_post_recv(&r_xprt->rx_ia, rep))
1413		− rpcrdma_recv_buffer_put(rep);
	1389	+ rpcrdma_post_recvs(r_xprt, false);
	1390	+out_badstatus:
	1391	+ rpcrdma_recv_buffer_put(rep);
1414	1392	}

net/sunrpc/xprtrdma/svc_rdma_backchannel.c+1−0

@@ -273,6 +273,7 @@ static const struct rpc_xprt_ops xprt_rdma_bc_procs = { · 272 unmodified lines
273	273	.reserve_xprt = xprt_reserve_xprt_cong,
274	274	.release_xprt = xprt_release_xprt_cong,
275	275	.alloc_slot = xprt_alloc_slot,
	276	+ .free_slot = xprt_free_slot,
276	277	.release_request = xprt_release_rqst_cong,
277	278	.buf_alloc = xprt_rdma_bc_allocate,
278	279	.buf_free = xprt_rdma_bc_free,

net/sunrpc/xprtrdma/transport.c+47−17

@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
1	2	/*
2	3	* Copyright (c) 2014-2017 Oracle. All rights reserved.
3	4	* Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
@@ -330,9 +331,7 @@ xprt_setup_rdma(struct xprt_create *args) · 326 unmodified lines
330	331	return ERR_PTR(-EBADF);
331	332	}
332	333
333		− xprt = xprt_alloc(args->net, sizeof(struct rpcrdma_xprt),
334		− xprt_rdma_slot_table_entries,
335		− xprt_rdma_slot_table_entries);
	334	+ xprt = xprt_alloc(args->net, sizeof(struct rpcrdma_xprt), 0, 0);
336	335	if (xprt == NULL) {
337	336	dprintk("RPC: %s: couldn't allocate rpcrdma_xprt\n",
338	337	__func__);
@@ -364,7 +363,7 @@ xprt_setup_rdma(struct xprt_create *args) · 25 unmodified lines
364	363	xprt_set_bound(xprt);
365	364	xprt_rdma_format_addresses(xprt, sap);
366	365
367		− cdata.max_requests = xprt->max_reqs;
	366	+ cdata.max_requests = xprt_rdma_slot_table_entries;
368	367
369	368	cdata.rsize = RPCRDMA_MAX_SEGS * PAGE_SIZE; /* RDMA write max */
370	369	cdata.wsize = RPCRDMA_MAX_SEGS * PAGE_SIZE; /* RDMA read max */
@@ -537,6 +536,47 @@ xprt_rdma_connect(struct rpc_xprt xprt, struct rpc_task task) · 166 unmodified lines
537	536	}
538	537	}
539	538
	539	+/**
	540	+ * xprt_rdma_alloc_slot - allocate an rpc_rqst
	541	+ * @xprt: controlling RPC transport
	542	+ * @task: RPC task requesting a fresh rpc_rqst
	543	+ *
	544	+ * tk_status values:
	545	+ * %0 if task->tk_rqstp points to a fresh rpc_rqst
	546	+ * %-EAGAIN if no rpc_rqst is available; queued on backlog
	547	+ */
	548	+static void
	549	+xprt_rdma_alloc_slot(struct rpc_xprt xprt, struct rpc_task task)
	550	+{
	551	+ struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);
	552	+ struct rpcrdma_req *req;
	553	+
	554	+ req = rpcrdma_buffer_get(&r_xprt->rx_buf);
	555	+ if (!req)
	556	+ goto out_sleep;
	557	+ task->tk_rqstp = &req->rl_slot;
	558	+ task->tk_status = 0;
	559	+ return;
	560	+
	561	+out_sleep:
	562	+ rpc_sleep_on(&xprt->backlog, task, NULL);
	563	+ task->tk_status = -EAGAIN;
	564	+}
	565	+
	566	+/**
	567	+ * xprt_rdma_free_slot - release an rpc_rqst
	568	+ * @xprt: controlling RPC transport
	569	+ * @rqst: rpc_rqst to release
	570	+ *
	571	+ */
	572	+static void
	573	+xprt_rdma_free_slot(struct rpc_xprt xprt, struct rpc_rqst rqst)
	574	+{
	575	+ memset(rqst, 0, sizeof(*rqst));
	576	+ rpcrdma_buffer_put(rpcr_to_rdmar(rqst));
	577	+ rpc_wake_up_next(&xprt->backlog);
	578	+}
	579	+
540	580	static bool
541	581	rpcrdma_get_sendbuf(struct rpcrdma_xprt r_xprt, struct rpcrdma_req req,
542	582	size_t size, gfp_t flags)
@@ -607,13 +647,9 @@ xprt_rdma_allocate(struct rpc_task *task) · 64 unmodified lines
607	647	{
608	648	struct rpc_rqst *rqst = task->tk_rqstp;
609	649	struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(rqst->rq_xprt);
610		− struct rpcrdma_req *req;
	650	+ struct rpcrdma_req *req = rpcr_to_rdmar(rqst);
611	651	gfp_t flags;
612	652
613		− req = rpcrdma_buffer_get(&r_xprt->rx_buf);
614		− if (req == NULL)
615		− goto out_get;
616		−
617	653	flags = RPCRDMA_DEF_GFP;
618	654	if (RPC_IS_SWAPPER(task))
619	655	flags = __GFP_MEMALLOC \| GFP_NOWAIT \| __GFP_NOWARN;
@@ -623,15 +659,12 @@ xprt_rdma_allocate(struct rpc_task *task) · 3 unmodified lines
623	659	if (!rpcrdma_get_recvbuf(r_xprt, req, rqst->rq_rcvsize, flags))
624	660	goto out_fail;
625	661
626		− rpcrdma_set_xprtdata(rqst, req);
627	662	rqst->rq_buffer = req->rl_sendbuf->rg_base;
628	663	rqst->rq_rbuffer = req->rl_recvbuf->rg_base;
629	664	trace_xprtrdma_allocate(task, req);
630	665	return 0;
631	666
632	667	out_fail:
633		− rpcrdma_buffer_put(req);
634		−out_get:
635	668	trace_xprtrdma_allocate(task, NULL);
636	669	return -ENOMEM;
637	670	}
@@ -652,7 +685,6 @@ xprt_rdma_free(struct rpc_task *task) · 14 unmodified lines
652	685	if (test_bit(RPCRDMA_REQ_F_PENDING, &req->rl_flags))
653	686	rpcrdma_release_rqst(r_xprt, req);
654	687	trace_xprtrdma_rpc_done(task, req);
655		− rpcrdma_buffer_put(req);
656	688	}
657	689
658	690	/**
@@ -690,9 +722,6 @@ xprt_rdma_send_request(struct rpc_task *task) · 31 unmodified lines
690	722	if (rc < 0)
691	723	goto failed_marshal;
692	724
693		− if (req->rl_reply == NULL) /* e.g. reconnection */
694		− rpcrdma_recv_buffer_get(req);
695		−
696	725	/* Must suppress retransmit to maintain credits */
697	726	if (rqst->rq_connect_cookie == xprt->connect_cookie)
698	727	goto drop_connection;
@@ -779,7 +808,8 @@ xprt_rdma_disable_swap(struct rpc_xprt *xprt) · 80 unmodified lines
779	808	static const struct rpc_xprt_ops xprt_rdma_procs = {
780	809	.reserve_xprt = xprt_reserve_xprt_cong,
781	810	.release_xprt = xprt_release_xprt_cong, /* sunrpc/xprt.c */
782		− .alloc_slot = xprt_alloc_slot,
	811	+ .alloc_slot = xprt_rdma_alloc_slot,
	812	+ .free_slot = xprt_rdma_free_slot,
783	813	.release_request = xprt_release_rqst_cong, /* ditto */
784	814	.set_retrans_timeout = xprt_set_retrans_timeout_def, /* ditto */
785	815	.timer = xprt_rdma_timer,

net/sunrpc/xprtrdma/verbs.c+112−179

@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
1	2	/*
2	3	* Copyright (c) 2014-2017 Oracle. All rights reserved.
3	4	* Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
@@ -71,8 +72,10 @@ · 67 unmodified lines
71	72	/*
72	73	* internal functions
73	74	*/
	75	+static void rpcrdma_sendctx_put_locked(struct rpcrdma_sendctx *sc);
74	76	static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt);
75	77	static void rpcrdma_mrs_destroy(struct rpcrdma_buffer *buf);
	78	+static int rpcrdma_create_rep(struct rpcrdma_xprt *r_xprt, bool temp);
76	79	static void rpcrdma_dma_unmap_regbuf(struct rpcrdma_regbuf *rb);
77	80
78	81	struct workqueue_struct *rpcrdma_receive_wq __read_mostly;
@@ -159,7 +162,7 @@ rpcrdma_wc_receive(struct ib_cq cq, struct ib_wc wc) · 80 unmodified lines
159	162	rr_cqe);
160	163
161	164	/* WARNING: Only wr_id and status are reliable at this point */
162		− trace_xprtrdma_wc_receive(rep, wc);
	165	+ trace_xprtrdma_wc_receive(wc);
163	166	if (wc->status != IB_WC_SUCCESS)
164	167	goto out_fail;
165	168
@@ -231,7 +234,7 @@ rpcrdma_conn_upcall(struct rdma_cm_id id, struct rdma_cm_event event) · 65 unmodified lines
231	234	complete(&ia->ri_done);
232	235	break;
233	236	case RDMA_CM_EVENT_ADDR_ERROR:
234		− ia->ri_async_rc = -EHOSTUNREACH;
	237	+ ia->ri_async_rc = -EPROTO;
235	238	complete(&ia->ri_done);
236	239	break;
237	240	case RDMA_CM_EVENT_ROUTE_ERROR:
@@ -262,7 +265,7 @@ rpcrdma_conn_upcall(struct rdma_cm_id id, struct rdma_cm_event event) · 24 unmodified lines
262	265	connstate = -ENOTCONN;
263	266	goto connected;
264	267	case RDMA_CM_EVENT_UNREACHABLE:
265		− connstate = -ENETDOWN;
	268	+ connstate = -ENETUNREACH;
266	269	goto connected;
267	270	case RDMA_CM_EVENT_REJECTED:
268	271	dprintk("rpcrdma: connection to %s:%s rejected: %s\n",
@@ -305,8 +308,8 @@ rpcrdma_create_id(struct rpcrdma_xprt xprt, struct rpcrdma_ia ia) · 36 unmodified lines
305	308	init_completion(&ia->ri_done);
306	309	init_completion(&ia->ri_remove_done);
307	310
308		− id = rdma_create_id(&init_net, rpcrdma_conn_upcall, xprt, RDMA_PS_TCP,
309		− IB_QPT_RC);
	311	+ id = rdma_create_id(xprt->rx_xprt.xprt_net, rpcrdma_conn_upcall,
	312	+ xprt, RDMA_PS_TCP, IB_QPT_RC);
310	313	if (IS_ERR(id)) {
311	314	rc = PTR_ERR(id);
312	315	dprintk("RPC: %s: rdma_create_id() failed %i\n",
@@ -500,8 +503,8 @@ rpcrdma_ep_create(struct rpcrdma_ep ep, struct rpcrdma_ia ia, · 187 unmodified lines
500	503	struct rpcrdma_create_data_internal *cdata)
501	504	{
502	505	struct rpcrdma_connect_private *pmsg = &ep->rep_cm_private;
503		− unsigned int max_qp_wr, max_sge;
504	506	struct ib_cq sendcq, recvcq;
	507	+ unsigned int max_sge;
505	508	int rc;
506	509
507	510	max_sge = min_t(unsigned int, ia->ri_device->attrs.max_sge,
@@ -512,29 +515,13 @@ rpcrdma_ep_create(struct rpcrdma_ep ep, struct rpcrdma_ia ia, · 4 unmodified lines
512	515	}
513	516	ia->ri_max_send_sges = max_sge;
514	517
515		− if (ia->ri_device->attrs.max_qp_wr <= RPCRDMA_BACKWARD_WRS) {
516		− dprintk("RPC: %s: insufficient wqe's available\n",
517		− __func__);
518		− return -ENOMEM;
519		− }
520		− max_qp_wr = ia->ri_device->attrs.max_qp_wr - RPCRDMA_BACKWARD_WRS - 1;
521		−
522		− /* check provider's send/recv wr limits */
523		− if (cdata->max_requests > max_qp_wr)
524		− cdata->max_requests = max_qp_wr;
	518	+ rc = ia->ri_ops->ro_open(ia, ep, cdata);
	519	+ if (rc)
	520	+ return rc;
525	521
526	522	ep->rep_attr.event_handler = rpcrdma_qp_async_error_upcall;
527	523	ep->rep_attr.qp_context = ep;
528	524	ep->rep_attr.srq = NULL;
529		− ep->rep_attr.cap.max_send_wr = cdata->max_requests;
530		− ep->rep_attr.cap.max_send_wr += RPCRDMA_BACKWARD_WRS;
531		− ep->rep_attr.cap.max_send_wr += 1; /* drain cqe */
532		− rc = ia->ri_ops->ro_open(ia, ep, cdata);
533		− if (rc)
534		− return rc;
535		− ep->rep_attr.cap.max_recv_wr = cdata->max_requests;
536		− ep->rep_attr.cap.max_recv_wr += RPCRDMA_BACKWARD_WRS;
537		− ep->rep_attr.cap.max_recv_wr += 1; /* drain cqe */
538	525	ep->rep_attr.cap.max_send_sge = max_sge;
539	526	ep->rep_attr.cap.max_recv_sge = 1;
540	527	ep->rep_attr.cap.max_inline_data = 0;
@@ -741,7 +728,6 @@ rpcrdma_ep_connect(struct rpcrdma_ep ep, struct rpcrdma_ia ia) · 200 unmodified lines
741	728	{
742	729	struct rpcrdma_xprt *r_xprt = container_of(ia, struct rpcrdma_xprt,
743	730	rx_ia);
744		− unsigned int extras;
745	731	int rc;
746	732
747	733	retry:
@@ -785,9 +771,8 @@ retry: · 37 unmodified lines
785	771	}
786	772
787	773	dprintk("RPC: %s: connected\n", __func__);
788		− extras = r_xprt->rx_buf.rb_bc_srv_max_requests;
789		− if (extras)
790		− rpcrdma_ep_post_extra_recv(r_xprt, extras);
	774	+
	775	+ rpcrdma_post_recvs(r_xprt, true);
791	776
792	777	out:
793	778	if (rc)
@@ -893,6 +878,7 @@ static int rpcrdma_sendctxs_create(struct rpcrdma_xprt *r_xprt) · 99 unmodified lines
893	878	sc->sc_xprt = r_xprt;
894	879	buf->rb_sc_ctxs[i] = sc;
895	880	}
	881	+ buf->rb_flags = 0;
896	882
897	883	return 0;
898	884
@@ -950,7 +936,7 @@ out_emptyq: · 51 unmodified lines
950	936	* completions recently. This is a sign the Send Queue is
951	937	* backing up. Cause the caller to pause and try again.
952	938	*/
953		− dprintk("RPC: %s: empty sendctx queue\n", __func__);
	939	+ set_bit(RPCRDMA_BUF_F_EMPTY_SCQ, &buf->rb_flags);
954	940	r_xprt = container_of(buf, struct rpcrdma_xprt, rx_buf);
955	941	r_xprt->rx_stats.empty_sendctx_q++;
956	942	return NULL;
@@ -965,7 +951,8 @@ out_emptyq: · 8 unmodified lines
965	951	*
966	952	* The caller serializes calls to this function (per rpcrdma_buffer).
967	953	*/
968		−void rpcrdma_sendctx_put_locked(struct rpcrdma_sendctx *sc)
	954	+static void
	955	+rpcrdma_sendctx_put_locked(struct rpcrdma_sendctx *sc)
969	956	{
970	957	struct rpcrdma_buffer *buf = &sc->sc_xprt->rx_buf;
971	958	unsigned long next_tail;
@@ -984,6 +971,11 @@ void rpcrdma_sendctx_put_locked(struct rpcrdma_sendctx *sc) · 12 unmodified lines
984	971
985	972	/* Paired with READ_ONCE */
986	973	smp_store_release(&buf->rb_sc_tail, next_tail);
	974	+
	975	+ if (test_and_clear_bit(RPCRDMA_BUF_F_EMPTY_SCQ, &buf->rb_flags)) {
	976	+ smp_mb__after_atomic();
	977	+ xprt_write_space(&sc->sc_xprt->rx_xprt);
	978	+ }
987	979	}
988	980
989	981	static void
@@ -1097,14 +1089,8 @@ rpcrdma_create_req(struct rpcrdma_xprt *r_xprt) · 107 unmodified lines
1097	1089	return req;
1098	1090	}
1099	1091
1100		−/**
1101		− * rpcrdma_create_rep - Allocate an rpcrdma_rep object
1102		− * @r_xprt: controlling transport
1103		− *
1104		− * Returns 0 on success or a negative errno on failure.
1105		− */
1106		−int
1107		−rpcrdma_create_rep(struct rpcrdma_xprt *r_xprt)
	1092	+static int
	1093	+rpcrdma_create_rep(struct rpcrdma_xprt *r_xprt, bool temp)
1108	1094	{
1109	1095	struct rpcrdma_create_data_internal *cdata = &r_xprt->rx_data;
1110	1096	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
@@ -1132,6 +1118,7 @@ rpcrdma_create_rep(struct rpcrdma_xprt *r_xprt) · 21 unmodified lines
1132	1118	rep->rr_recv_wr.wr_cqe = &rep->rr_cqe;
1133	1119	rep->rr_recv_wr.sg_list = &rep->rr_rdmabuf->rg_iov;
1134	1120	rep->rr_recv_wr.num_sge = 1;
	1121	+ rep->rr_temp = temp;
1135	1122
1136	1123	spin_lock(&buf->rb_lock);
1137	1124	list_add(&rep->rr_list, &buf->rb_recv_bufs);
@@ -1183,12 +1170,8 @@ rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt) · 45 unmodified lines
1183	1170	list_add(&req->rl_list, &buf->rb_send_bufs);
1184	1171	}
1185	1172
	1173	+ buf->rb_posted_receives = 0;
1186	1174	INIT_LIST_HEAD(&buf->rb_recv_bufs);
1187		− for (i = 0; i <= buf->rb_max_requests; i++) {
1188		− rc = rpcrdma_create_rep(r_xprt);
1189		− if (rc)
1190		− goto out;
1191		− }
1192	1175
1193	1176	rc = rpcrdma_sendctxs_create(r_xprt);
1194	1177	if (rc)
@@ -1200,28 +1183,6 @@ out: · 5 unmodified lines
1200	1183	return rc;
1201	1184	}
1202	1185
1203		−static struct rpcrdma_req *
1204		−rpcrdma_buffer_get_req_locked(struct rpcrdma_buffer *buf)
1205		−{
1206		− struct rpcrdma_req *req;
1207		−
1208		− req = list_first_entry(&buf->rb_send_bufs,
1209		− struct rpcrdma_req, rl_list);
1210		− list_del_init(&req->rl_list);
1211		− return req;
1212		−}
1213		−
1214		−static struct rpcrdma_rep *
1215		−rpcrdma_buffer_get_rep_locked(struct rpcrdma_buffer *buf)
1216		−{
1217		− struct rpcrdma_rep *rep;
1218		−
1219		− rep = list_first_entry(&buf->rb_recv_bufs,
1220		− struct rpcrdma_rep, rr_list);
1221		− list_del(&rep->rr_list);
1222		− return rep;
1223		−}
1224		−
1225	1186	static void
1226	1187	rpcrdma_destroy_rep(struct rpcrdma_rep *rep)
1227	1188	{
@@ -1280,10 +1241,11 @@ rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf) · 52 unmodified lines
1280	1241	while (!list_empty(&buf->rb_recv_bufs)) {
1281	1242	struct rpcrdma_rep *rep;
1282	1243
1283		− rep = rpcrdma_buffer_get_rep_locked(buf);
	1244	+ rep = list_first_entry(&buf->rb_recv_bufs,
	1245	+ struct rpcrdma_rep, rr_list);
	1246	+ list_del(&rep->rr_list);
1284	1247	rpcrdma_destroy_rep(rep);
1285	1248	}
1286		− buf->rb_send_count = 0;
1287	1249
1288	1250	spin_lock(&buf->rb_reqslock);
1289	1251	while (!list_empty(&buf->rb_allreqs)) {
@@ -1298,7 +1260,6 @@ rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf) · 8 unmodified lines
1298	1260	spin_lock(&buf->rb_reqslock);
1299	1261	}
1300	1262	spin_unlock(&buf->rb_reqslock);
1301		− buf->rb_recv_count = 0;
1302	1263
1303	1264	rpcrdma_mrs_destroy(buf);
1304	1265	}
@@ -1371,27 +1332,11 @@ rpcrdma_mr_unmap_and_put(struct rpcrdma_mr *mr) · 66 unmodified lines
1371	1332	__rpcrdma_mr_put(&r_xprt->rx_buf, mr);
1372	1333	}
1373	1334
1374		−static struct rpcrdma_rep *
1375		−rpcrdma_buffer_get_rep(struct rpcrdma_buffer *buffers)
1376		−{
1377		− /* If an RPC previously completed without a reply (say, a
1378		− * credential problem or a soft timeout occurs) then hold off
1379		− * on supplying more Receive buffers until the number of new
1380		− * pending RPCs catches up to the number of posted Receives.
1381		− */
1382		− if (unlikely(buffers->rb_send_count < buffers->rb_recv_count))
1383		− return NULL;
1384		−
1385		− if (unlikely(list_empty(&buffers->rb_recv_bufs)))
1386		− return NULL;
1387		− buffers->rb_recv_count++;
1388		− return rpcrdma_buffer_get_rep_locked(buffers);
1389		−}
1390		−
1391		−/*
1392		− * Get a set of request/reply buffers.
	1335	+/**
	1336	+ * rpcrdma_buffer_get - Get a request buffer
	1337	+ * @buffers: Buffer pool from which to obtain a buffer
1393	1338	*
1394		− * Reply buffer (if available) is attached to send buffer upon return.
	1339	+ * Returns a fresh rpcrdma_req, or NULL if none are available.
1395	1340	*/
1396	1341	struct rpcrdma_req *
1397	1342	rpcrdma_buffer_get(struct rpcrdma_buffer *buffers)
@@ -1399,23 +1344,18 @@ rpcrdma_buffer_get(struct rpcrdma_buffer *buffers) · 1 unmodified line
1399	1344	struct rpcrdma_req *req;
1400	1345
1401	1346	spin_lock(&buffers->rb_lock);
1402		− if (list_empty(&buffers->rb_send_bufs))
1403		− goto out_reqbuf;
1404		− buffers->rb_send_count++;
1405		− req = rpcrdma_buffer_get_req_locked(buffers);
1406		− req->rl_reply = rpcrdma_buffer_get_rep(buffers);
	1347	+ req = list_first_entry_or_null(&buffers->rb_send_bufs,
	1348	+ struct rpcrdma_req, rl_list);
	1349	+ if (req)
	1350	+ list_del_init(&req->rl_list);
1407	1351	spin_unlock(&buffers->rb_lock);
1408		−
1409	1352	return req;
1410		−
1411		−out_reqbuf:
1412		− spin_unlock(&buffers->rb_lock);
1413		− return NULL;
1414	1353	}
1415	1354
1416		−/*
1417		− * Put request/reply buffers back into pool.
1418		− * Pre-decrement counter/array index.
	1355	+/**
	1356	+ * rpcrdma_buffer_put - Put request/reply buffers back into pool
	1357	+ * @req: object to return
	1358	+ *
1419	1359	*/
1420	1360	void
1421	1361	rpcrdma_buffer_put(struct rpcrdma_req *req)
@@ -1426,27 +1366,16 @@ rpcrdma_buffer_put(struct rpcrdma_req *req) · 4 unmodified lines
1426	1366	req->rl_reply = NULL;
1427	1367
1428	1368	spin_lock(&buffers->rb_lock);
1429		− buffers->rb_send_count--;
1430		− list_add_tail(&req->rl_list, &buffers->rb_send_bufs);
	1369	+ list_add(&req->rl_list, &buffers->rb_send_bufs);
1431	1370	if (rep) {
1432		− buffers->rb_recv_count--;
1433		− list_add_tail(&rep->rr_list, &buffers->rb_recv_bufs);
	1371	+ if (!rep->rr_temp) {
	1372	+ list_add(&rep->rr_list, &buffers->rb_recv_bufs);
	1373	+ rep = NULL;
	1374	+ }
1434	1375	}
1435	1376	spin_unlock(&buffers->rb_lock);
1436		−}
1437		−
1438		−/*
1439		− * Recover reply buffers from pool.
1440		− * This happens when recovering from disconnect.
1441		− */
1442		−void
1443		−rpcrdma_recv_buffer_get(struct rpcrdma_req *req)
1444		−{
1445		− struct rpcrdma_buffer *buffers = req->rl_buffer;
1446		−
1447		− spin_lock(&buffers->rb_lock);
1448		− req->rl_reply = rpcrdma_buffer_get_rep(buffers);
1449		− spin_unlock(&buffers->rb_lock);
	1377	+ if (rep)
	1378	+ rpcrdma_destroy_rep(rep);
1450	1379	}
1451	1380
1452	1381	/*
@@ -1458,10 +1387,13 @@ rpcrdma_recv_buffer_put(struct rpcrdma_rep *rep) · 5 unmodified lines
1458	1387	{
1459	1388	struct rpcrdma_buffer *buffers = &rep->rr_rxprt->rx_buf;
1460	1389
1461		− spin_lock(&buffers->rb_lock);
1462		− buffers->rb_recv_count--;
1463		− list_add_tail(&rep->rr_list, &buffers->rb_recv_bufs);
1464		− spin_unlock(&buffers->rb_lock);
	1390	+ if (!rep->rr_temp) {
	1391	+ spin_lock(&buffers->rb_lock);
	1392	+ list_add(&rep->rr_list, &buffers->rb_recv_bufs);
	1393	+ spin_unlock(&buffers->rb_lock);
	1394	+ } else {
	1395	+ rpcrdma_destroy_rep(rep);
	1396	+ }
1465	1397	}
1466	1398
1467	1399	/**
@@ -1557,13 +1489,6 @@ rpcrdma_ep_post(struct rpcrdma_ia *ia, · 89 unmodified lines
1557	1489	struct ib_send_wr *send_wr = &req->rl_sendctx->sc_wr;
1558	1490	int rc;
1559	1491
1560		− if (req->rl_reply) {
1561		− rc = rpcrdma_ep_post_recv(ia, req->rl_reply);
1562		− if (rc)
1563		− return rc;
1564		− req->rl_reply = NULL;
1565		− }
1566		−
1567	1492	if (!ep->rep_send_count \|\|
1568	1493	test_bit(RPCRDMA_REQ_F_TX_RESOURCES, &req->rl_flags)) {
1569	1494	send_wr->send_flags \|= IB_SEND_SIGNALED;
@@ -1580,61 +1505,69 @@ rpcrdma_ep_post(struct rpcrdma_ia *ia, · 10 unmodified lines
1580	1505	return 0;
1581	1506	}
1582	1507
1583		−int
1584		−rpcrdma_ep_post_recv(struct rpcrdma_ia *ia,
1585		− struct rpcrdma_rep *rep)
1586		−{
1587		− struct ib_recv_wr *recv_wr_fail;
1588		− int rc;
1589		−
1590		− if (!rpcrdma_dma_map_regbuf(ia, rep->rr_rdmabuf))
1591		− goto out_map;
1592		− rc = ib_post_recv(ia->ri_id->qp, &rep->rr_recv_wr, &recv_wr_fail);
1593		− trace_xprtrdma_post_recv(rep, rc);
1594		− if (rc)
1595		− return -ENOTCONN;
1596		− return 0;
1597		−
1598		−out_map:
1599		− pr_err("rpcrdma: failed to DMA map the Receive buffer\n");
1600		− return -EIO;
1601		−}
1602		−
1603	1508	/**
1604		− * rpcrdma_ep_post_extra_recv - Post buffers for incoming backchannel requests
1605		− * @r_xprt: transport associated with these backchannel resources
1606		− * @count: minimum number of incoming requests expected
	1509	+ * rpcrdma_post_recvs - Maybe post some Receive buffers
	1510	+ * @r_xprt: controlling transport
	1511	+ * @temp: when true, allocate temp rpcrdma_rep objects
1607	1512	*
1608		− * Returns zero if all requested buffers were posted, or a negative errno.
1609	1513	*/
1610		−int
1611		−rpcrdma_ep_post_extra_recv(struct rpcrdma_xprt *r_xprt, unsigned int count)
	1514	+void
	1515	+rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp)
1612	1516	{
1613		− struct rpcrdma_buffer *buffers = &r_xprt->rx_buf;
1614		− struct rpcrdma_ia *ia = &r_xprt->rx_ia;
1615		− struct rpcrdma_rep *rep;
1616		− int rc;
	1517	+ struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
	1518	+ struct ib_recv_wr wr, bad_wr;
	1519	+ int needed, count, rc;
1617	1520
1618		− while (count--) {
1619		− spin_lock(&buffers->rb_lock);
1620		− if (list_empty(&buffers->rb_recv_bufs))
1621		− goto out_reqbuf;
1622		− rep = rpcrdma_buffer_get_rep_locked(buffers);
1623		− spin_unlock(&buffers->rb_lock);
	1521	+ needed = buf->rb_credits + (buf->rb_bc_srv_max_requests << 1);
	1522	+ if (buf->rb_posted_receives > needed)
	1523	+ return;
	1524	+ needed -= buf->rb_posted_receives;
1624	1525
1625		− rc = rpcrdma_ep_post_recv(ia, rep);
1626		− if (rc)
1627		− goto out_rc;
1628		− }
	1526	+ count = 0;
	1527	+ wr = NULL;
	1528	+ while (needed) {
	1529	+ struct rpcrdma_regbuf *rb;
	1530	+ struct rpcrdma_rep *rep;
1629	1531
1630		− return 0;
	1532	+ spin_lock(&buf->rb_lock);
	1533	+ rep = list_first_entry_or_null(&buf->rb_recv_bufs,
	1534	+ struct rpcrdma_rep, rr_list);
	1535	+ if (likely(rep))
	1536	+ list_del(&rep->rr_list);
	1537	+ spin_unlock(&buf->rb_lock);
	1538	+ if (!rep) {
	1539	+ if (rpcrdma_create_rep(r_xprt, temp))
	1540	+ break;
	1541	+ continue;
	1542	+ }
1631	1543
1632		−out_reqbuf:
1633		− spin_unlock(&buffers->rb_lock);
1634		− trace_xprtrdma_noreps(r_xprt);
1635		− return -ENOMEM;
	1544	+ rb = rep->rr_rdmabuf;
	1545	+ if (!rpcrdma_regbuf_is_mapped(rb)) {
	1546	+ if (!__rpcrdma_dma_map_regbuf(&r_xprt->rx_ia, rb)) {
	1547	+ rpcrdma_recv_buffer_put(rep);
	1548	+ break;
	1549	+ }
	1550	+ }
1636	1551
1637		−out_rc:
1638		− rpcrdma_recv_buffer_put(rep);
1639		− return rc;
	1552	+ trace_xprtrdma_post_recv(rep->rr_recv_wr.wr_cqe);
	1553	+ rep->rr_recv_wr.next = wr;
	1554	+ wr = &rep->rr_recv_wr;
	1555	+ ++count;
	1556	+ --needed;
	1557	+ }
	1558	+ if (!count)
	1559	+ return;
	1560	+
	1561	+ rc = ib_post_recv(r_xprt->rx_ia.ri_id->qp, wr, &bad_wr);
	1562	+ if (rc) {
	1563	+ for (wr = bad_wr; wr; wr = wr->next) {
	1564	+ struct rpcrdma_rep *rep;
	1565	+
	1566	+ rep = container_of(wr, struct rpcrdma_rep, rr_recv_wr);
	1567	+ rpcrdma_recv_buffer_put(rep);
	1568	+ --count;
	1569	+ }
	1570	+ }
	1571	+ buf->rb_posted_receives += count;
	1572	+ trace_xprtrdma_post_recvs(r_xprt, count, rc);
1640	1573	}

net/sunrpc/xprtrdma/xprt_rdma.h+12−14

@@ -1,3 +1,4 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
1	2	/*
2	3	* Copyright (c) 2014-2017 Oracle. All rights reserved.
3	4	* Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
@@ -196,6 +197,7 @@ struct rpcrdma_rep { · 192 unmodified lines
196	197	__be32 rr_proc;
197	198	int rr_wc_flags;
198	199	u32 rr_inv_rkey;
	200	+ bool rr_temp;
199	201	struct rpcrdma_regbuf *rr_rdmabuf;
200	202	struct rpcrdma_xprt *rr_rxprt;
201	203	struct work_struct rr_work;
@@ -334,6 +336,7 @@ enum { · 132 unmodified lines
334	336	struct rpcrdma_buffer;
335	337	struct rpcrdma_req {
336	338	struct list_head rl_list;
	339	+ struct rpc_rqst rl_slot;
337	340	struct rpcrdma_buffer *rl_buffer;
338	341	struct rpcrdma_rep *rl_reply;
339	342	struct xdr_stream rl_stream;
@@ -356,16 +359,10 @@ enum { · 16 unmodified lines
356	359	RPCRDMA_REQ_F_TX_RESOURCES,
357	360	};
358	361
359		−static inline void
360		−rpcrdma_set_xprtdata(struct rpc_rqst rqst, struct rpcrdma_req req)
361		−{
362		− rqst->rq_xprtdata = req;
363		−}
364		−
365	362	static inline struct rpcrdma_req *
366	363	rpcr_to_rdmar(const struct rpc_rqst *rqst)
367	364	{
368		− return rqst->rq_xprtdata;
	365	+ return container_of(rqst, struct rpcrdma_req, rl_slot);
369	366	}
370	367
371	368	static inline void
@@ -401,11 +398,12 @@ struct rpcrdma_buffer { · 29 unmodified lines
401	398	struct rpcrdma_sendctx **rb_sc_ctxs;
402	399
403	400	spinlock_t rb_lock; /* protect buf lists */
404		− int rb_send_count, rb_recv_count;
405	401	struct list_head rb_send_bufs;
406	402	struct list_head rb_recv_bufs;
	403	+ unsigned long rb_flags;
407	404	u32 rb_max_requests;
408	405	u32 rb_credits; /* most recent credit grant */
	406	+ int rb_posted_receives;
409	407
410	408	u32 rb_bc_srv_max_requests;
411	409	spinlock_t rb_reqslock; /* protect rb_allreqs */
@@ -420,6 +418,11 @@ struct rpcrdma_buffer { · 8 unmodified lines
420	418	};
421	419	#define rdmab_to_ia(b) (&container_of((b), struct rpcrdma_xprt, rx_buf)->rx_ia)
422	420
	421	+/* rb_flags */
	422	+enum {
	423	+ RPCRDMA_BUF_F_EMPTY_SCQ = 0,
	424	+};
	425	+
423	426	/*
424	427	* Internal structure for transport instance creation. This
425	428	* exists primarily for modularity.
@@ -561,18 +564,16 @@ void rpcrdma_ep_disconnect(struct rpcrdma_ep , struct rpcrdma_ia ); · 135 unmodified lines
561	564
562	565	int rpcrdma_ep_post(struct rpcrdma_ia , struct rpcrdma_ep ,
563	566	struct rpcrdma_req *);
564		−int rpcrdma_ep_post_recv(struct rpcrdma_ia , struct rpcrdma_rep );
	567	+void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp);
565	568
566	569	/*
567	570	* Buffer calls - xprtrdma/verbs.c
568	571	*/
569	572	struct rpcrdma_req rpcrdma_create_req(struct rpcrdma_xprt );
570	573	void rpcrdma_destroy_req(struct rpcrdma_req *);
571		−int rpcrdma_create_rep(struct rpcrdma_xprt *r_xprt);
572	574	int rpcrdma_buffer_create(struct rpcrdma_xprt *);
573	575	void rpcrdma_buffer_destroy(struct rpcrdma_buffer *);
574	576	struct rpcrdma_sendctx rpcrdma_sendctx_get_locked(struct rpcrdma_buffer buf);
575		−void rpcrdma_sendctx_put_locked(struct rpcrdma_sendctx *sc);
576	577
577	578	struct rpcrdma_mr rpcrdma_mr_get(struct rpcrdma_xprt r_xprt);
578	579	void rpcrdma_mr_put(struct rpcrdma_mr *mr);
@@ -581,7 +582,6 @@ void rpcrdma_mr_defer_recovery(struct rpcrdma_mr *mr); · 2 unmodified lines
581	582
582	583	struct rpcrdma_req rpcrdma_buffer_get(struct rpcrdma_buffer );
583	584	void rpcrdma_buffer_put(struct rpcrdma_req *);
584		−void rpcrdma_recv_buffer_get(struct rpcrdma_req *);
585	585	void rpcrdma_recv_buffer_put(struct rpcrdma_rep *);
586	586
587	587	struct rpcrdma_regbuf *rpcrdma_alloc_regbuf(size_t, enum dma_data_direction,
@@ -603,8 +603,6 @@ rpcrdma_dma_map_regbuf(struct rpcrdma_ia ia, struct rpcrdma_regbuf rb) · 15 unmodified lines
603	603	return __rpcrdma_dma_map_regbuf(ia, rb);
604	604	}
605	605
606		−int rpcrdma_ep_post_extra_recv(struct rpcrdma_xprt *, unsigned int);
607		−
608	606	int rpcrdma_alloc_wq(void);
609	607	void rpcrdma_destroy_wq(void);
610	608

net/sunrpc/xprtsock.c+4−0

@@ -2763,6 +2763,7 @@ static const struct rpc_xprt_ops xs_local_ops = { · 2762 unmodified lines
2763	2763	.reserve_xprt = xprt_reserve_xprt,
2764	2764	.release_xprt = xs_tcp_release_xprt,
2765	2765	.alloc_slot = xprt_alloc_slot,
	2766	+ .free_slot = xprt_free_slot,
2766	2767	.rpcbind = xs_local_rpcbind,
2767	2768	.set_port = xs_local_set_port,
2768	2769	.connect = xs_local_connect,
@@ -2782,6 +2783,7 @@ static const struct rpc_xprt_ops xs_udp_ops = { · 13 unmodified lines
2782	2783	.reserve_xprt = xprt_reserve_xprt_cong,
2783	2784	.release_xprt = xprt_release_xprt_cong,
2784	2785	.alloc_slot = xprt_alloc_slot,
	2786	+ .free_slot = xprt_free_slot,
2785	2787	.rpcbind = rpcb_getport_async,
2786	2788	.set_port = xs_set_port,
2787	2789	.connect = xs_connect,
@@ -2803,6 +2805,7 @@ static const struct rpc_xprt_ops xs_tcp_ops = { · 15 unmodified lines
2803	2805	.reserve_xprt = xprt_reserve_xprt,
2804	2806	.release_xprt = xs_tcp_release_xprt,
2805	2807	.alloc_slot = xprt_lock_and_alloc_slot,
	2808	+ .free_slot = xprt_free_slot,
2806	2809	.rpcbind = rpcb_getport_async,
2807	2810	.set_port = xs_set_port,
2808	2811	.connect = xs_connect,
@@ -2834,6 +2837,7 @@ static const struct rpc_xprt_ops bc_tcp_ops = { · 25 unmodified lines
2834	2837	.reserve_xprt = xprt_reserve_xprt,
2835	2838	.release_xprt = xprt_release_xprt,
2836	2839	.alloc_slot = xprt_alloc_slot,
	2840	+ .free_slot = xprt_free_slot,
2837	2841	.buf_alloc = bc_malloc,
2838	2842	.buf_free = bc_free,
2839	2843	.send_request = bc_send_request,