Merge tag 'nfs-rdma-for-5.11-1' of git://git.linux-nfs.org/projects/anna/linux-nfs into linux-next

NFSoRDmA Client updates for Linux 5.11 Cleanups and improvements: - Remove use of raw kernel memory addresses in tracepoints - Replace dprintk() call sites in ERR_CHUNK path - Trace unmap sync calls - Optimize MR DMA-unmapping

Trond Myklebust committed 5 years ago

Commit edffb842 parents 5c3485b 7a03aeb7 files · +302 −313Browse files

Unified · Split

7 files

include/trace/events/rpcrdma.h +212 −238
net/sunrpc/xprtrdma/backchannel.c +3 −3
net/sunrpc/xprtrdma/frwr_ops.c +55 −26
net/sunrpc/xprtrdma/rpc_rdma.c +11 −21
net/sunrpc/xprtrdma/transport.c +4 −3
net/sunrpc/xprtrdma/verbs.c +9 −21
net/sunrpc/xprtrdma/xprt_rdma.h +8 −1

include/trace/events/rpcrdma.h+212−238

@@ -60,7 +60,7 @@ DECLARE_EVENT_CLASS(rpcrdma_completion_class, · 59 unmodified lines
60	60	), \
61	61	TP_ARGS(wc, cid))
62	62
63		−DECLARE_EVENT_CLASS(xprtrdma_reply_event,
	63	+DECLARE_EVENT_CLASS(xprtrdma_reply_class,
64	64	TP_PROTO(
65	65	const struct rpcrdma_rep *rep
66	66	),
@@ -68,29 +68,30 @@ DECLARE_EVENT_CLASS(xprtrdma_reply_event, · 1 unmodified line
68	68	TP_ARGS(rep),
69	69
70	70	TP_STRUCT__entry(
71		− __field(const void *, rep)
72		− __field(const void *, r_xprt)
73	71	__field(u32, xid)
74	72	__field(u32, version)
75	73	__field(u32, proc)
	74	+ __string(addr, rpcrdma_addrstr(rep->rr_rxprt))
	75	+ __string(port, rpcrdma_portstr(rep->rr_rxprt))
76	76	),
77	77
78	78	TP_fast_assign(
79		− __entry->rep = rep;
80		− __entry->r_xprt = rep->rr_rxprt;
81	79	__entry->xid = be32_to_cpu(rep->rr_xid);
82	80	__entry->version = be32_to_cpu(rep->rr_vers);
83	81	__entry->proc = be32_to_cpu(rep->rr_proc);
	82	+ __assign_str(addr, rpcrdma_addrstr(rep->rr_rxprt));
	83	+ __assign_str(port, rpcrdma_portstr(rep->rr_rxprt));
84	84	),
85	85
86		− TP_printk("rxprt %p xid=0x%08x rep=%p: version %u proc %u",
87		− __entry->r_xprt, __entry->xid, __entry->rep,
88		− __entry->version, __entry->proc
	86	+ TP_printk("peer=[%s]:%s xid=0x%08x version=%u proc=%u",
	87	+ __get_str(addr), __get_str(port),
	88	+ __entry->xid, __entry->version, __entry->proc
89	89	)
90	90	);
91	91
92	92	#define DEFINE_REPLY_EVENT(name) \
93		− DEFINE_EVENT(xprtrdma_reply_event, name, \
	93	+ DEFINE_EVENT(xprtrdma_reply_class, \
	94	+ xprtrdma_reply_##name##_err, \
94	95	TP_PROTO( \
95	96	const struct rpcrdma_rep *rep \
96	97	), \
@@ -261,54 +262,67 @@ DECLARE_EVENT_CLASS(xprtrdma_wrch_event, · 164 unmodified lines
261	262	), \
262	263	TP_ARGS(task, mr, nsegs))
263	264
264		−DECLARE_EVENT_CLASS(xprtrdma_frwr_done,
	265	+TRACE_DEFINE_ENUM(DMA_BIDIRECTIONAL);
	266	+TRACE_DEFINE_ENUM(DMA_TO_DEVICE);
	267	+TRACE_DEFINE_ENUM(DMA_FROM_DEVICE);
	268	+TRACE_DEFINE_ENUM(DMA_NONE);
	269	+
	270	+#define xprtrdma_show_direction(x) \
	271	+ __print_symbolic(x, \
	272	+ { DMA_BIDIRECTIONAL, "BIDIR" }, \
	273	+ { DMA_TO_DEVICE, "TO_DEVICE" }, \
	274	+ { DMA_FROM_DEVICE, "FROM_DEVICE" }, \
	275	+ { DMA_NONE, "NONE" })
	276	+
	277	+DECLARE_EVENT_CLASS(xprtrdma_mr_class,
265	278	TP_PROTO(
266		− const struct ib_wc *wc,
267		− const struct rpcrdma_frwr *frwr
	279	+ const struct rpcrdma_mr *mr
268	280	),
269	281
270		− TP_ARGS(wc, frwr),
	282	+ TP_ARGS(mr),
271	283
272	284	TP_STRUCT__entry(
	285	+ __field(unsigned int, task_id)
	286	+ __field(unsigned int, client_id)
273	287	__field(u32, mr_id)
274		− __field(unsigned int, status)
275		− __field(unsigned int, vendor_err)
	288	+ __field(int, nents)
	289	+ __field(u32, handle)
	290	+ __field(u32, length)
	291	+ __field(u64, offset)
	292	+ __field(u32, dir)
276	293	),
277	294
278	295	TP_fast_assign(
279		− __entry->mr_id = frwr->fr_mr->res.id;
280		− __entry->status = wc->status;
281		− __entry->vendor_err = __entry->status ? wc->vendor_err : 0;
	296	+ const struct rpcrdma_req *req = mr->mr_req;
	297	+ const struct rpc_task *task = req->rl_slot.rq_task;
	298	+
	299	+ __entry->task_id = task->tk_pid;
	300	+ __entry->client_id = task->tk_client->cl_clid;
	301	+ __entry->mr_id = mr->frwr.fr_mr->res.id;
	302	+ __entry->nents = mr->mr_nents;
	303	+ __entry->handle = mr->mr_handle;
	304	+ __entry->length = mr->mr_length;
	305	+ __entry->offset = mr->mr_offset;
	306	+ __entry->dir = mr->mr_dir;
282	307	),
283	308
284		− TP_printk(
285		− "mr.id=%u: %s (%u/0x%x)",
286		− __entry->mr_id, rdma_show_wc_status(__entry->status),
287		− __entry->status, __entry->vendor_err
	309	+ TP_printk("task:%u@%u mr.id=%u nents=%d %u@0x%016llx:0x%08x (%s)",
	310	+ __entry->task_id, __entry->client_id,
	311	+ __entry->mr_id, __entry->nents, __entry->length,
	312	+ (unsigned long long)__entry->offset, __entry->handle,
	313	+ xprtrdma_show_direction(__entry->dir)
288	314	)
289	315	);
290	316
291		−#define DEFINE_FRWR_DONE_EVENT(name) \
292		− DEFINE_EVENT(xprtrdma_frwr_done, name, \
	317	+#define DEFINE_MR_EVENT(name) \
	318	+ DEFINE_EVENT(xprtrdma_mr_class, \
	319	+ xprtrdma_mr_##name, \
293	320	TP_PROTO( \
294		− const struct ib_wc *wc, \
295		− const struct rpcrdma_frwr *frwr \
	321	+ const struct rpcrdma_mr *mr \
296	322	), \
297		− TP_ARGS(wc, frwr))
298		−
299		−TRACE_DEFINE_ENUM(DMA_BIDIRECTIONAL);
300		−TRACE_DEFINE_ENUM(DMA_TO_DEVICE);
301		−TRACE_DEFINE_ENUM(DMA_FROM_DEVICE);
302		−TRACE_DEFINE_ENUM(DMA_NONE);
303		−
304		−#define xprtrdma_show_direction(x) \
305		− __print_symbolic(x, \
306		− { DMA_BIDIRECTIONAL, "BIDIR" }, \
307		− { DMA_TO_DEVICE, "TO_DEVICE" }, \
308		− { DMA_FROM_DEVICE, "FROM_DEVICE" }, \
309		− { DMA_NONE, "NONE" })
	323	+ TP_ARGS(mr))
310	324
311		−DECLARE_EVENT_CLASS(xprtrdma_mr,
	325	+DECLARE_EVENT_CLASS(xprtrdma_anonymous_mr_class,
312	326	TP_PROTO(
313	327	const struct rpcrdma_mr *mr
314	328	),
@@ -340,45 +354,47 @@ DECLARE_EVENT_CLASS(xprtrdma_mr, · 25 unmodified lines
340	354	)
341	355	);
342	356
343		−#define DEFINE_MR_EVENT(name) \
344		− DEFINE_EVENT(xprtrdma_mr, xprtrdma_mr_##name, \
345		− TP_PROTO( \
346		− const struct rpcrdma_mr *mr \
347		− ), \
	357	+#define DEFINE_ANON_MR_EVENT(name) \
	358	+ DEFINE_EVENT(xprtrdma_anonymous_mr_class, \
	359	+ xprtrdma_mr_##name, \
	360	+ TP_PROTO( \
	361	+ const struct rpcrdma_mr *mr \
	362	+ ), \
348	363	TP_ARGS(mr))
349	364
350		−DECLARE_EVENT_CLASS(xprtrdma_cb_event,
	365	+DECLARE_EVENT_CLASS(xprtrdma_callback_class,
351	366	TP_PROTO(
	367	+ const struct rpcrdma_xprt *r_xprt,
352	368	const struct rpc_rqst *rqst
353	369	),
354	370
355		− TP_ARGS(rqst),
	371	+ TP_ARGS(r_xprt, rqst),
356	372
357	373	TP_STRUCT__entry(
358		− __field(const void *, rqst)
359		− __field(const void *, rep)
360		− __field(const void *, req)
361	374	__field(u32, xid)
	375	+ __string(addr, rpcrdma_addrstr(r_xprt))
	376	+ __string(port, rpcrdma_portstr(r_xprt))
362	377	),
363	378
364	379	TP_fast_assign(
365		− __entry->rqst = rqst;
366		− __entry->req = rpcr_to_rdmar(rqst);
367		− __entry->rep = rpcr_to_rdmar(rqst)->rl_reply;
368	380	__entry->xid = be32_to_cpu(rqst->rq_xid);
	381	+ __assign_str(addr, rpcrdma_addrstr(r_xprt));
	382	+ __assign_str(port, rpcrdma_portstr(r_xprt));
369	383	),
370	384
371		− TP_printk("xid=0x%08x, rqst=%p req=%p rep=%p",
372		− __entry->xid, __entry->rqst, __entry->req, __entry->rep
	385	+ TP_printk("peer=[%s]:%s xid=0x%08x",
	386	+ __get_str(addr), __get_str(port), __entry->xid
373	387	)
374	388	);
375	389
376		−#define DEFINE_CB_EVENT(name) \
377		− DEFINE_EVENT(xprtrdma_cb_event, name, \
	390	+#define DEFINE_CALLBACK_EVENT(name) \
	391	+ DEFINE_EVENT(xprtrdma_callback_class, \
	392	+ xprtrdma_cb_##name, \
378	393	TP_PROTO( \
	394	+ const struct rpcrdma_xprt *r_xprt, \
379	395	const struct rpc_rqst *rqst \
380	396	), \
381		− TP_ARGS(rqst))
	397	+ TP_ARGS(r_xprt, rqst))
382	398
383	399	/**
384	400	** Connection events
@@ -549,61 +565,33 @@ TRACE_EVENT(xprtrdma_createmrs, · 164 unmodified lines
549	565	)
550	566	);
551	567
552		−TRACE_EVENT(xprtrdma_mr_get,
553		− TP_PROTO(
554		− const struct rpcrdma_req *req
555		− ),
556		−
557		− TP_ARGS(req),
558		−
559		− TP_STRUCT__entry(
560		− __field(const void *, req)
561		− __field(unsigned int, task_id)
562		− __field(unsigned int, client_id)
563		− __field(u32, xid)
564		− ),
565		−
566		− TP_fast_assign(
567		− const struct rpc_rqst *rqst = &req->rl_slot;
568		−
569		− __entry->req = req;
570		− __entry->task_id = rqst->rq_task->tk_pid;
571		− __entry->client_id = rqst->rq_task->tk_client->cl_clid;
572		− __entry->xid = be32_to_cpu(rqst->rq_xid);
573		− ),
574		−
575		− TP_printk("task:%u@%u xid=0x%08x req=%p",
576		− __entry->task_id, __entry->client_id, __entry->xid,
577		− __entry->req
578		− )
579		−);
580		−
581		−TRACE_EVENT(xprtrdma_nomrs,
	568	+TRACE_EVENT(xprtrdma_nomrs_err,
582	569	TP_PROTO(
	570	+ const struct rpcrdma_xprt *r_xprt,
583	571	const struct rpcrdma_req *req
584	572	),
585	573
586		− TP_ARGS(req),
	574	+ TP_ARGS(r_xprt, req),
587	575
588	576	TP_STRUCT__entry(
589		− __field(const void *, req)
590	577	__field(unsigned int, task_id)
591	578	__field(unsigned int, client_id)
592		− __field(u32, xid)
	579	+ __string(addr, rpcrdma_addrstr(r_xprt))
	580	+ __string(port, rpcrdma_portstr(r_xprt))
593	581	),
594	582
595	583	TP_fast_assign(
596	584	const struct rpc_rqst *rqst = &req->rl_slot;
597	585
598		− __entry->req = req;
599	586	__entry->task_id = rqst->rq_task->tk_pid;
600	587	__entry->client_id = rqst->rq_task->tk_client->cl_clid;
601		− __entry->xid = be32_to_cpu(rqst->rq_xid);
	588	+ __assign_str(addr, rpcrdma_addrstr(r_xprt));
	589	+ __assign_str(port, rpcrdma_portstr(r_xprt));
602	590	),
603	591
604		− TP_printk("task:%u@%u xid=0x%08x req=%p",
605		− __entry->task_id, __entry->client_id, __entry->xid,
606		− __entry->req
	592	+ TP_printk("peer=[%s]:%s task:%u@%u",
	593	+ __get_str(addr), __get_str(port),
	594	+ __entry->task_id, __entry->client_id
607	595	)
608	596	);
609	597
@@ -735,8 +723,8 @@ TRACE_EVENT(xprtrdma_post_send, · 125 unmodified lines
735	723	TP_ARGS(req),
736	724
737	725	TP_STRUCT__entry(
738		− __field(const void *, req)
739		− __field(const void *, sc)
	726	+ __field(u32, cq_id)
	727	+ __field(int, completion_id)
740	728	__field(unsigned int, task_id)
741	729	__field(unsigned int, client_id)
742	730	__field(int, num_sge)
@@ -745,20 +733,21 @@ TRACE_EVENT(xprtrdma_post_send, · 2 unmodified lines
745	733
746	734	TP_fast_assign(
747	735	const struct rpc_rqst *rqst = &req->rl_slot;
	736	+ const struct rpcrdma_sendctx *sc = req->rl_sendctx;
748	737
	738	+ __entry->cq_id = sc->sc_cid.ci_queue_id;
	739	+ __entry->completion_id = sc->sc_cid.ci_completion_id;
749	740	__entry->task_id = rqst->rq_task->tk_pid;
750	741	__entry->client_id = rqst->rq_task->tk_client ?
751	742	rqst->rq_task->tk_client->cl_clid : -1;
752		− __entry->req = req;
753		− __entry->sc = req->rl_sendctx;
754	743	__entry->num_sge = req->rl_wr.num_sge;
755	744	__entry->signaled = req->rl_wr.send_flags & IB_SEND_SIGNALED;
756	745	),
757	746
758		− TP_printk("task:%u@%u req=%p sc=%p (%d SGE%s) %s",
	747	+ TP_printk("task:%u@%u cq.id=%u cid=%d (%d SGE%s) %s",
759	748	__entry->task_id, __entry->client_id,
760		− __entry->req, __entry->sc, __entry->num_sge,
761		− (__entry->num_sge == 1 ? "" : "s"),
	749	+ __entry->cq_id, __entry->completion_id,
	750	+ __entry->num_sge, (__entry->num_sge == 1 ? "" : "s"),
762	751	(__entry->signaled ? "signaled" : "")
763	752	)
764	753	);
@@ -771,15 +760,17 @@ TRACE_EVENT(xprtrdma_post_recv, · 6 unmodified lines
771	760	TP_ARGS(rep),
772	761
773	762	TP_STRUCT__entry(
774		− __field(const void *, rep)
	763	+ __field(u32, cq_id)
	764	+ __field(int, completion_id)
775	765	),
776	766
777	767	TP_fast_assign(
778		− __entry->rep = rep;
	768	+ __entry->cq_id = rep->rr_cid.ci_queue_id;
	769	+ __entry->completion_id = rep->rr_cid.ci_completion_id;
779	770	),
780	771
781		− TP_printk("rep=%p",
782		− __entry->rep
	772	+ TP_printk("cq.id=%d cid=%d",
	773	+ __entry->cq_id, __entry->completion_id
783	774	)
784	775	);
785	776
@@ -816,7 +807,7 @@ TRACE_EVENT(xprtrdma_post_recvs, · 30 unmodified lines
816	807	)
817	808	);
818	809
819		−TRACE_EVENT(xprtrdma_post_linv,
	810	+TRACE_EVENT(xprtrdma_post_linv_err,
820	811	TP_PROTO(
821	812	const struct rpcrdma_req *req,
822	813	int status
@@ -825,19 +816,21 @@ TRACE_EVENT(xprtrdma_post_linv, · 2 unmodified lines
825	816	TP_ARGS(req, status),
826	817
827	818	TP_STRUCT__entry(
828		− __field(const void *, req)
	819	+ __field(unsigned int, task_id)
	820	+ __field(unsigned int, client_id)
829	821	__field(int, status)
830		− __field(u32, xid)
831	822	),
832	823
833	824	TP_fast_assign(
834		− __entry->req = req;
	825	+ const struct rpc_task *task = req->rl_slot.rq_task;
	826	+
	827	+ __entry->task_id = task->tk_pid;
	828	+ __entry->client_id = task->tk_client->cl_clid;
835	829	__entry->status = status;
836		− __entry->xid = be32_to_cpu(req->rl_slot.rq_xid);
837	830	),
838	831
839		− TP_printk("req=%p xid=0x%08x status=%d",
840		− __entry->req, __entry->xid, __entry->status
	832	+ TP_printk("task:%u@%u status=%d",
	833	+ __entry->task_id, __entry->client_id, __entry->status
841	834	)
842	835	);
843	836
@@ -845,75 +838,12 @@ TRACE_EVENT(xprtrdma_post_linv, · 1 unmodified line
845	838	** Completion events
846	839	**/
847	840
848		−TRACE_EVENT(xprtrdma_wc_send,
849		− TP_PROTO(
850		− const struct rpcrdma_sendctx *sc,
851		− const struct ib_wc *wc
852		− ),
853		−
854		− TP_ARGS(sc, wc),
855		−
856		− TP_STRUCT__entry(
857		− __field(const void *, req)
858		− __field(const void *, sc)
859		− __field(unsigned int, unmap_count)
860		− __field(unsigned int, status)
861		− __field(unsigned int, vendor_err)
862		− ),
863		−
864		− TP_fast_assign(
865		− __entry->req = sc->sc_req;
866		− __entry->sc = sc;
867		− __entry->unmap_count = sc->sc_unmap_count;
868		− __entry->status = wc->status;
869		− __entry->vendor_err = __entry->status ? wc->vendor_err : 0;
870		− ),
871		−
872		− TP_printk("req=%p sc=%p unmapped=%u: %s (%u/0x%x)",
873		− __entry->req, __entry->sc, __entry->unmap_count,
874		− rdma_show_wc_status(__entry->status),
875		− __entry->status, __entry->vendor_err
876		− )
877		−);
878		−
879		−TRACE_EVENT(xprtrdma_wc_receive,
880		− TP_PROTO(
881		− const struct ib_wc *wc
882		− ),
883		−
884		− TP_ARGS(wc),
885		−
886		− TP_STRUCT__entry(
887		− __field(const void *, rep)
888		− __field(u32, byte_len)
889		− __field(unsigned int, status)
890		− __field(u32, vendor_err)
891		− ),
892		−
893		− TP_fast_assign(
894		− __entry->rep = container_of(wc->wr_cqe, struct rpcrdma_rep,
895		− rr_cqe);
896		− __entry->status = wc->status;
897		− if (wc->status) {
898		− __entry->byte_len = 0;
899		− __entry->vendor_err = wc->vendor_err;
900		− } else {
901		− __entry->byte_len = wc->byte_len;
902		− __entry->vendor_err = 0;
903		− }
904		− ),
905		−
906		− TP_printk("rep=%p %u bytes: %s (%u/0x%x)",
907		− __entry->rep, __entry->byte_len,
908		− rdma_show_wc_status(__entry->status),
909		− __entry->status, __entry->vendor_err
910		− )
911		−);
912		−
913		−DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_fastreg);
914		−DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li);
915		−DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li_wake);
916		−DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li_done);
	841	+DEFINE_COMPLETION_EVENT(xprtrdma_wc_receive);
	842	+DEFINE_COMPLETION_EVENT(xprtrdma_wc_send);
	843	+DEFINE_COMPLETION_EVENT(xprtrdma_wc_fastreg);
	844	+DEFINE_COMPLETION_EVENT(xprtrdma_wc_li);
	845	+DEFINE_COMPLETION_EVENT(xprtrdma_wc_li_wake);
	846	+DEFINE_COMPLETION_EVENT(xprtrdma_wc_li_done);
917	847
918	848	TRACE_EVENT(xprtrdma_frwr_alloc,
919	849	TP_PROTO(
@@ -1036,9 +966,9 @@ TRACE_EVENT(xprtrdma_frwr_maperr, · 116 unmodified lines
1036	966
1037	967	DEFINE_MR_EVENT(localinv);
1038	968	DEFINE_MR_EVENT(map);
1039		−DEFINE_MR_EVENT(unmap);
1040		−DEFINE_MR_EVENT(reminv);
1041		−DEFINE_MR_EVENT(recycle);
	969	+
	970	+DEFINE_ANON_MR_EVENT(unmap);
	971	+DEFINE_ANON_MR_EVENT(recycle);
1042	972
1043	973	TRACE_EVENT(xprtrdma_dma_maperr,
1044	974	TP_PROTO(
@@ -1066,17 +996,14 @@ TRACE_EVENT(xprtrdma_reply, · 21 unmodified lines
1066	996	TP_PROTO(
1067	997	const struct rpc_task *task,
1068	998	const struct rpcrdma_rep *rep,
1069		− const struct rpcrdma_req *req,
1070	999	unsigned int credits
1071	1000	),
1072	1001
1073		− TP_ARGS(task, rep, req, credits),
	1002	+ TP_ARGS(task, rep, credits),
1074	1003
1075	1004	TP_STRUCT__entry(
1076	1005	__field(unsigned int, task_id)
1077	1006	__field(unsigned int, client_id)
1078		− __field(const void *, rep)
1079		− __field(const void *, req)
1080	1007	__field(u32, xid)
1081	1008	__field(unsigned int, credits)
1082	1009	),
@@ -1084,49 +1011,102 @@ TRACE_EVENT(xprtrdma_reply, · 1 unmodified line
1084	1011	TP_fast_assign(
1085	1012	__entry->task_id = task->tk_pid;
1086	1013	__entry->client_id = task->tk_client->cl_clid;
1087		− __entry->rep = rep;
1088		− __entry->req = req;
1089	1014	__entry->xid = be32_to_cpu(rep->rr_xid);
1090	1015	__entry->credits = credits;
1091	1016	),
1092	1017
1093		− TP_printk("task:%u@%u xid=0x%08x, %u credits, rep=%p -> req=%p",
	1018	+ TP_printk("task:%u@%u xid=0x%08x credits=%u",
1094	1019	__entry->task_id, __entry->client_id, __entry->xid,
1095		− __entry->credits, __entry->rep, __entry->req
	1020	+ __entry->credits
1096	1021	)
1097	1022	);
1098	1023
1099		−TRACE_EVENT(xprtrdma_defer_cmp,
	1024	+DEFINE_REPLY_EVENT(vers);
	1025	+DEFINE_REPLY_EVENT(rqst);
	1026	+DEFINE_REPLY_EVENT(short);
	1027	+DEFINE_REPLY_EVENT(hdr);
	1028	+
	1029	+TRACE_EVENT(xprtrdma_err_vers,
1100	1030	TP_PROTO(
1101		− const struct rpcrdma_rep *rep
	1031	+ const struct rpc_rqst *rqst,
	1032	+ __be32 *min,
	1033	+ __be32 *max
1102	1034	),
1103	1035
1104		− TP_ARGS(rep),
	1036	+ TP_ARGS(rqst, min, max),
1105	1037
1106	1038	TP_STRUCT__entry(
1107	1039	__field(unsigned int, task_id)
1108	1040	__field(unsigned int, client_id)
1109		− __field(const void *, rep)
1110	1041	__field(u32, xid)
	1042	+ __field(u32, min)
	1043	+ __field(u32, max)
1111	1044	),
1112	1045
1113	1046	TP_fast_assign(
1114		− __entry->task_id = rep->rr_rqst->rq_task->tk_pid;
1115		− __entry->client_id = rep->rr_rqst->rq_task->tk_client->cl_clid;
1116		− __entry->rep = rep;
1117		− __entry->xid = be32_to_cpu(rep->rr_xid);
	1047	+ __entry->task_id = rqst->rq_task->tk_pid;
	1048	+ __entry->client_id = rqst->rq_task->tk_client->cl_clid;
	1049	+ __entry->xid = be32_to_cpu(rqst->rq_xid);
	1050	+ __entry->min = be32_to_cpup(min);
	1051	+ __entry->max = be32_to_cpup(max);
1118	1052	),
1119	1053
1120		− TP_printk("task:%u@%u xid=0x%08x rep=%p",
	1054	+ TP_printk("task:%u@%u xid=0x%08x versions=[%u, %u]",
1121	1055	__entry->task_id, __entry->client_id, __entry->xid,
1122		− __entry->rep
	1056	+ __entry->min, __entry->max
	1057	+ )
	1058	+);
	1059	+
	1060	+TRACE_EVENT(xprtrdma_err_chunk,
	1061	+ TP_PROTO(
	1062	+ const struct rpc_rqst *rqst
	1063	+ ),
	1064	+
	1065	+ TP_ARGS(rqst),
	1066	+
	1067	+ TP_STRUCT__entry(
	1068	+ __field(unsigned int, task_id)
	1069	+ __field(unsigned int, client_id)
	1070	+ __field(u32, xid)
	1071	+ ),
	1072	+
	1073	+ TP_fast_assign(
	1074	+ __entry->task_id = rqst->rq_task->tk_pid;
	1075	+ __entry->client_id = rqst->rq_task->tk_client->cl_clid;
	1076	+ __entry->xid = be32_to_cpu(rqst->rq_xid);
	1077	+ ),
	1078	+
	1079	+ TP_printk("task:%u@%u xid=0x%08x",
	1080	+ __entry->task_id, __entry->client_id, __entry->xid
1123	1081	)
1124	1082	);
1125	1083
1126		−DEFINE_REPLY_EVENT(xprtrdma_reply_vers);
1127		−DEFINE_REPLY_EVENT(xprtrdma_reply_rqst);
1128		−DEFINE_REPLY_EVENT(xprtrdma_reply_short);
1129		−DEFINE_REPLY_EVENT(xprtrdma_reply_hdr);
	1084	+TRACE_EVENT(xprtrdma_err_unrecognized,
	1085	+ TP_PROTO(
	1086	+ const struct rpc_rqst *rqst,
	1087	+ __be32 *procedure
	1088	+ ),
	1089	+
	1090	+ TP_ARGS(rqst, procedure),
	1091	+
	1092	+ TP_STRUCT__entry(
	1093	+ __field(unsigned int, task_id)
	1094	+ __field(unsigned int, client_id)
	1095	+ __field(u32, xid)
	1096	+ __field(u32, procedure)
	1097	+ ),
	1098	+
	1099	+ TP_fast_assign(
	1100	+ __entry->task_id = rqst->rq_task->tk_pid;
	1101	+ __entry->client_id = rqst->rq_task->tk_client->cl_clid;
	1102	+ __entry->procedure = be32_to_cpup(procedure);
	1103	+ ),
	1104	+
	1105	+ TP_printk("task:%u@%u xid=0x%08x procedure=%u",
	1106	+ __entry->task_id, __entry->client_id, __entry->xid,
	1107	+ __entry->procedure
	1108	+ )
	1109	+);
1130	1110
1131	1111	TRACE_EVENT(xprtrdma_fixup,
1132	1112	TP_PROTO(
@@ -1187,6 +1167,28 @@ TRACE_EVENT(xprtrdma_decode_seg, · 54 unmodified lines
1187	1167	)
1188	1168	);
1189	1169
	1170	+TRACE_EVENT(xprtrdma_mrs_zap,
	1171	+ TP_PROTO(
	1172	+ const struct rpc_task *task
	1173	+ ),
	1174	+
	1175	+ TP_ARGS(task),
	1176	+
	1177	+ TP_STRUCT__entry(
	1178	+ __field(unsigned int, task_id)
	1179	+ __field(unsigned int, client_id)
	1180	+ ),
	1181	+
	1182	+ TP_fast_assign(
	1183	+ __entry->task_id = task->tk_pid;
	1184	+ __entry->client_id = task->tk_client->cl_clid;
	1185	+ ),
	1186	+
	1187	+ TP_printk("task:%u@%u",
	1188	+ __entry->task_id, __entry->client_id
	1189	+ )
	1190	+);
	1191	+
1190	1192	/**
1191	1193	** Callback events
1192	1194	**/
@@ -1219,36 +1221,8 @@ TRACE_EVENT(xprtrdma_cb_setup, · 26 unmodified lines
1219	1221	)
1220	1222	);
1221	1223
1222		−DEFINE_CB_EVENT(xprtrdma_cb_call);
1223		−DEFINE_CB_EVENT(xprtrdma_cb_reply);
1224		−
1225		−TRACE_EVENT(xprtrdma_leaked_rep,
1226		− TP_PROTO(
1227		− const struct rpc_rqst *rqst,
1228		− const struct rpcrdma_rep *rep
1229		− ),
1230		−
1231		− TP_ARGS(rqst, rep),
1232		−
1233		− TP_STRUCT__entry(
1234		− __field(unsigned int, task_id)
1235		− __field(unsigned int, client_id)
1236		− __field(u32, xid)
1237		− __field(const void *, rep)
1238		− ),
1239		−
1240		− TP_fast_assign(
1241		− __entry->task_id = rqst->rq_task->tk_pid;
1242		− __entry->client_id = rqst->rq_task->tk_client->cl_clid;
1243		− __entry->xid = be32_to_cpu(rqst->rq_xid);
1244		− __entry->rep = rep;
1245		− ),
1246		−
1247		− TP_printk("task:%u@%u xid=0x%08x rep=%p",
1248		− __entry->task_id, __entry->client_id, __entry->xid,
1249		− __entry->rep
1250		− )
1251		−);
	1224	+DEFINE_CALLBACK_EVENT(call);
	1225	+DEFINE_CALLBACK_EVENT(reply);
1252	1226
1253	1227	/**
1254	1228	** Server-side RPC/RDMA events

net/sunrpc/xprtrdma/backchannel.c+3−3

@@ -1,6 +1,6 @@
1	1	// SPDX-License-Identifier: GPL-2.0
2	2	/*
3		− * Copyright (c) 2015 Oracle. All rights reserved.
	3	+ * Copyright (c) 2015-2020, Oracle and/or its affiliates.
4	4	*
5	5	* Support for backward direction RPCs on RPC/RDMA.
6	6	*/
@@ -82,7 +82,7 @@ static int rpcrdma_bc_marshal_reply(struct rpc_rqst *rqst) · 75 unmodified lines
82	82	&rqst->rq_snd_buf, rpcrdma_noch_pullup))
83	83	return -EIO;
84	84
85		− trace_xprtrdma_cb_reply(rqst);
	85	+ trace_xprtrdma_cb_reply(r_xprt, rqst);
86	86	return 0;
87	87	}
88	88
@@ -260,7 +260,7 @@ void rpcrdma_bc_receive_call(struct rpcrdma_xprt *r_xprt, · 171 unmodified lines
260	260	*/
261	261	req = rpcr_to_rdmar(rqst);
262	262	req->rl_reply = rep;
263		− trace_xprtrdma_cb_call(rqst);
	263	+ trace_xprtrdma_cb_call(r_xprt, rqst);
264	264
265	265	/* Queue rqst for ULP's callback service */
266	266	bc_serv = xprt->bc_serv;

net/sunrpc/xprtrdma/frwr_ops.c+55−26

@@ -65,18 +65,23 @@ void frwr_release_mr(struct rpcrdma_mr *mr) · 64 unmodified lines
65	65	kfree(mr);
66	66	}
67	67
	68	+static void frwr_mr_unmap(struct rpcrdma_xprt r_xprt, struct rpcrdma_mr mr)
	69	+{
	70	+ if (mr->mr_device) {
	71	+ trace_xprtrdma_mr_unmap(mr);
	72	+ ib_dma_unmap_sg(mr->mr_device, mr->mr_sg, mr->mr_nents,
	73	+ mr->mr_dir);
	74	+ mr->mr_device = NULL;
	75	+ }
	76	+}
	77	+
68	78	static void frwr_mr_recycle(struct rpcrdma_mr *mr)
69	79	{
70	80	struct rpcrdma_xprt *r_xprt = mr->mr_xprt;
71	81
72	82	trace_xprtrdma_mr_recycle(mr);
73	83
74		− if (mr->mr_dir != DMA_NONE) {
75		− trace_xprtrdma_mr_unmap(mr);
76		− ib_dma_unmap_sg(r_xprt->rx_ep->re_id->device,
77		− mr->mr_sg, mr->mr_nents, mr->mr_dir);
78		− mr->mr_dir = DMA_NONE;
79		− }
	84	+ frwr_mr_unmap(r_xprt, mr);
80	85
81	86	spin_lock(&r_xprt->rx_buf.rb_lock);
82	87	list_del(&mr->mr_all);
@@ -86,6 +91,16 @@ static void frwr_mr_recycle(struct rpcrdma_mr *mr) · 3 unmodified lines
86	91	frwr_release_mr(mr);
87	92	}
88	93
	94	+static void frwr_mr_put(struct rpcrdma_mr *mr)
	95	+{
	96	+ frwr_mr_unmap(mr->mr_xprt, mr);
	97	+
	98	+ /* The MR is returned to the req's MR free list instead
	99	+ * of to the xprt's MR free list. No spinlock is needed.
	100	+ */
	101	+ rpcrdma_mr_push(mr, &mr->mr_req->rl_free_mrs);
	102	+}
	103	+
89	104	/* frwr_reset - Place MRs back on the free list
90	105	* @req: request to reset
91	106	*
@@ -101,7 +116,7 @@ void frwr_reset(struct rpcrdma_req *req) · 9 unmodified lines
101	116	struct rpcrdma_mr *mr;
102	117
103	118	while ((mr = rpcrdma_mr_pop(&req->rl_registered)))
104		− rpcrdma_mr_put(mr);
	119	+ frwr_mr_put(mr);
105	120	}
106	121
107	122	/**
@@ -130,7 +145,7 @@ int frwr_mr_init(struct rpcrdma_xprt r_xprt, struct rpcrdma_mr mr) · 22 unmodified lines
130	145
131	146	mr->mr_xprt = r_xprt;
132	147	mr->frwr.fr_mr = frmr;
133		− mr->mr_dir = DMA_NONE;
	148	+ mr->mr_device = NULL;
134	149	INIT_LIST_HEAD(&mr->mr_list);
135	150	init_completion(&mr->frwr.fr_linv_done);
136	151
@@ -315,6 +330,7 @@ struct rpcrdma_mr_seg frwr_map(struct rpcrdma_xprt r_xprt, · 178 unmodified lines
315	330	mr->mr_dir);
316	331	if (!dma_nents)
317	332	goto out_dmamap_err;
	333	+ mr->mr_device = ep->re_id->device;
318	334
319	335	ibmr = mr->frwr.fr_mr;
320	336	n = ib_map_mr_sg(ibmr, mr->mr_sg, dma_nents, NULL, PAGE_SIZE);
@@ -341,7 +357,6 @@ struct rpcrdma_mr_seg frwr_map(struct rpcrdma_xprt r_xprt, · 20 unmodified lines
341	357	return seg;
342	358
343	359	out_dmamap_err:
344		− mr->mr_dir = DMA_NONE;
345	360	trace_xprtrdma_frwr_sgerr(mr, i);
346	361	return ERR_PTR(-EIO);
347	362
@@ -363,12 +378,21 @@ static void frwr_wc_fastreg(struct ib_cq cq, struct ib_wc wc) · 15 unmodified lines
363	378	container_of(cqe, struct rpcrdma_frwr, fr_cqe);
364	379
365	380	/* WARNING: Only wr_cqe and status are reliable at this point */
366		− trace_xprtrdma_wc_fastreg(wc, frwr);
	381	+ trace_xprtrdma_wc_fastreg(wc, &frwr->fr_cid);
367	382	/* The MR will get recycled when the associated req is retransmitted */
368	383
369	384	rpcrdma_flush_disconnect(cq->cq_context, wc);
370	385	}
371	386
	387	+static void frwr_cid_init(struct rpcrdma_ep *ep,
	388	+ struct rpcrdma_frwr *frwr)
	389	+{
	390	+ struct rpc_rdma_cid *cid = &frwr->fr_cid;
	391	+
	392	+ cid->ci_queue_id = ep->re_attr.send_cq->res.id;
	393	+ cid->ci_completion_id = frwr->fr_mr->res.id;
	394	+}
	395	+
372	396	/**
373	397	* frwr_send - post Send WRs containing the RPC Call message
374	398	* @r_xprt: controlling transport instance
@@ -385,6 +409,7 @@ static void frwr_wc_fastreg(struct ib_cq cq, struct ib_wc wc) · 10 unmodified lines
385	409	*/
386	410	int frwr_send(struct rpcrdma_xprt r_xprt, struct rpcrdma_req req)
387	411	{
	412	+ struct rpcrdma_ep *ep = r_xprt->rx_ep;
388	413	struct ib_send_wr *post_wr;
389	414	struct rpcrdma_mr *mr;
390	415
@@ -395,6 +420,7 @@ int frwr_send(struct rpcrdma_xprt r_xprt, struct rpcrdma_req req) · 4 unmodified lines
395	420	frwr = &mr->frwr;
396	421
397	422	frwr->fr_cqe.done = frwr_wc_fastreg;
	423	+ frwr_cid_init(ep, frwr);
398	424	frwr->fr_regwr.wr.next = post_wr;
399	425	frwr->fr_regwr.wr.wr_cqe = &frwr->fr_cqe;
400	426	frwr->fr_regwr.wr.num_sge = 0;
@@ -404,7 +430,7 @@ int frwr_send(struct rpcrdma_xprt r_xprt, struct rpcrdma_req req) · 3 unmodified lines
404	430	post_wr = &frwr->fr_regwr.wr;
405	431	}
406	432
407		− return ib_post_send(r_xprt->rx_ep->re_id->qp, post_wr, NULL);
	433	+ return ib_post_send(ep->re_id->qp, post_wr, NULL);
408	434	}
409	435
410	436	/**
@@ -420,18 +446,17 @@ void frwr_reminv(struct rpcrdma_rep rep, struct list_head mrs) · 9 unmodified lines
420	446	list_for_each_entry(mr, mrs, mr_list)
421	447	if (mr->mr_handle == rep->rr_inv_rkey) {
422	448	list_del_init(&mr->mr_list);
423		− trace_xprtrdma_mr_reminv(mr);
424		− rpcrdma_mr_put(mr);
	449	+ frwr_mr_put(mr);
425	450	break; /* only one invalidated MR per RPC */
426	451	}
427	452	}
428	453
429		−static void __frwr_release_mr(struct ib_wc wc, struct rpcrdma_mr mr)
	454	+static void frwr_mr_done(struct ib_wc wc, struct rpcrdma_mr mr)
430	455	{
431	456	if (wc->status != IB_WC_SUCCESS)
432	457	frwr_mr_recycle(mr);
433	458	else
434		− rpcrdma_mr_put(mr);
	459	+ frwr_mr_put(mr);
435	460	}
436	461
437	462	/**
@@ -448,8 +473,8 @@ static void frwr_wc_localinv(struct ib_cq cq, struct ib_wc wc) · 10 unmodified lines
448	473	struct rpcrdma_mr *mr = container_of(frwr, struct rpcrdma_mr, frwr);
449	474
450	475	/* WARNING: Only wr_cqe and status are reliable at this point */
451		− trace_xprtrdma_wc_li(wc, frwr);
452		− __frwr_release_mr(wc, mr);
	476	+ trace_xprtrdma_wc_li(wc, &frwr->fr_cid);
	477	+ frwr_mr_done(wc, mr);
453	478
454	479	rpcrdma_flush_disconnect(cq->cq_context, wc);
455	480	}
@@ -469,8 +494,8 @@ static void frwr_wc_localinv_wake(struct ib_cq cq, struct ib_wc wc) · 13 unmodified lines
469	494	struct rpcrdma_mr *mr = container_of(frwr, struct rpcrdma_mr, frwr);
470	495
471	496	/* WARNING: Only wr_cqe and status are reliable at this point */
472		− trace_xprtrdma_wc_li_wake(wc, frwr);
473		− __frwr_release_mr(wc, mr);
	497	+ trace_xprtrdma_wc_li_wake(wc, &frwr->fr_cid);
	498	+ frwr_mr_done(wc, mr);
474	499	complete(&frwr->fr_linv_done);
475	500
476	501	rpcrdma_flush_disconnect(cq->cq_context, wc);
@@ -490,6 +515,7 @@ static void frwr_wc_localinv_wake(struct ib_cq cq, struct ib_wc wc) · 13 unmodified lines
490	515	void frwr_unmap_sync(struct rpcrdma_xprt r_xprt, struct rpcrdma_req req)
491	516	{
492	517	struct ib_send_wr first, prev, last;
	518	+ struct rpcrdma_ep *ep = r_xprt->rx_ep;
493	519	const struct ib_send_wr *bad_wr;
494	520	struct rpcrdma_frwr *frwr;
495	521	struct rpcrdma_mr *mr;
@@ -509,6 +535,7 @@ void frwr_unmap_sync(struct rpcrdma_xprt r_xprt, struct rpcrdma_req req) · 13 unmodified lines
509	535
510	536	frwr = &mr->frwr;
511	537	frwr->fr_cqe.done = frwr_wc_localinv;
	538	+ frwr_cid_init(ep, frwr);
512	539	last = &frwr->fr_invwr;
513	540	last->next = NULL;
514	541	last->wr_cqe = &frwr->fr_cqe;
@@ -534,7 +561,7 @@ void frwr_unmap_sync(struct rpcrdma_xprt r_xprt, struct rpcrdma_req req) · 19 unmodified lines
534	561	* unless re_id->qp is a valid pointer.
535	562	*/
536	563	bad_wr = NULL;
537		− rc = ib_post_send(r_xprt->rx_ep->re_id->qp, first, &bad_wr);
	564	+ rc = ib_post_send(ep->re_id->qp, first, &bad_wr);
538	565
539	566	/* The final LOCAL_INV WR in the chain is supposed to
540	567	* do the wake. If it was never posted, the wake will
@@ -547,7 +574,7 @@ void frwr_unmap_sync(struct rpcrdma_xprt r_xprt, struct rpcrdma_req req) · 6 unmodified lines
547	574
548	575	/* Recycle MRs in the LOCAL_INV chain that did not get posted.
549	576	*/
550		− trace_xprtrdma_post_linv(req, rc);
	577	+ trace_xprtrdma_post_linv_err(req, rc);
551	578	while (bad_wr) {
552	579	frwr = container_of(bad_wr, struct rpcrdma_frwr,
553	580	fr_invwr);
@@ -574,10 +601,10 @@ static void frwr_wc_localinv_done(struct ib_cq cq, struct ib_wc wc) · 20 unmodified lines
574	601	struct rpcrdma_rep *rep = mr->mr_req->rl_reply;
575	602
576	603	/* WARNING: Only wr_cqe and status are reliable at this point */
577		− trace_xprtrdma_wc_li_done(wc, frwr);
578		− __frwr_release_mr(wc, mr);
	604	+ trace_xprtrdma_wc_li_done(wc, &frwr->fr_cid);
	605	+ frwr_mr_done(wc, mr);
579	606
580		− /* Ensure @rep is generated before __frwr_release_mr */
	607	+ /* Ensure @rep is generated before frwr_mr_done */
581	608	smp_rmb();
582	609	rpcrdma_complete_rqst(rep);
583	610
@@ -597,6 +624,7 @@ static void frwr_wc_localinv_done(struct ib_cq cq, struct ib_wc wc) · 13 unmodified lines
597	624	void frwr_unmap_async(struct rpcrdma_xprt r_xprt, struct rpcrdma_req req)
598	625	{
599	626	struct ib_send_wr first, last, **prev;
	627	+ struct rpcrdma_ep *ep = r_xprt->rx_ep;
600	628	const struct ib_send_wr *bad_wr;
601	629	struct rpcrdma_frwr *frwr;
602	630	struct rpcrdma_mr *mr;
@@ -614,6 +642,7 @@ void frwr_unmap_async(struct rpcrdma_xprt r_xprt, struct rpcrdma_req req) · 11 unmodified lines
614	642
615	643	frwr = &mr->frwr;
616	644	frwr->fr_cqe.done = frwr_wc_localinv;
	645	+ frwr_cid_init(ep, frwr);
617	646	last = &frwr->fr_invwr;
618	647	last->next = NULL;
619	648	last->wr_cqe = &frwr->fr_cqe;
@@ -639,13 +668,13 @@ void frwr_unmap_async(struct rpcrdma_xprt r_xprt, struct rpcrdma_req req) · 19 unmodified lines
639	668	* unless re_id->qp is a valid pointer.
640	669	*/
641	670	bad_wr = NULL;
642		− rc = ib_post_send(r_xprt->rx_ep->re_id->qp, first, &bad_wr);
	671	+ rc = ib_post_send(ep->re_id->qp, first, &bad_wr);
643	672	if (!rc)
644	673	return;
645	674
646	675	/* Recycle MRs in the LOCAL_INV chain that did not get posted.
647	676	*/
648		− trace_xprtrdma_post_linv(req, rc);
	677	+ trace_xprtrdma_post_linv_err(req, rc);
649	678	while (bad_wr) {
650	679	frwr = container_of(bad_wr, struct rpcrdma_frwr, fr_invwr);
651	680	mr = container_of(frwr, struct rpcrdma_mr, frwr);

net/sunrpc/xprtrdma/rpc_rdma.c+11−21

@@ -1,6 +1,6 @@
1	1	// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2	2	/*
3		− * Copyright (c) 2014-2017 Oracle. All rights reserved.
	3	+ * Copyright (c) 2014-2020, Oracle and/or its affiliates.
4	4	* Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
5	5	*
6	6	* This software is available to you under a choice of one of two
@@ -331,7 +331,6 @@ static struct rpcrdma_mr_seg rpcrdma_mr_prepare(struct rpcrdma_xprt r_xprt, · 324 unmodified lines
331	331	*mr = rpcrdma_mr_get(r_xprt);
332	332	if (!*mr)
333	333	goto out_getmr_err;
334		− trace_xprtrdma_mr_get(req);
335	334	(*mr)->mr_req = req;
336	335	}
337	336
@@ -339,7 +338,7 @@ static struct rpcrdma_mr_seg rpcrdma_mr_prepare(struct rpcrdma_xprt r_xprt, · 1 unmodified line
339	338	return frwr_map(r_xprt, seg, nsegs, writing, req->rl_slot.rq_xid, *mr);
340	339
341	340	out_getmr_err:
342		− trace_xprtrdma_nomrs(req);
	341	+ trace_xprtrdma_nomrs_err(r_xprt, req);
343	342	xprt_wait_for_buffer_space(&r_xprt->rx_xprt);
344	343	rpcrdma_mrs_refresh(r_xprt);
345	344	return ERR_PTR(-EAGAIN);
@@ -1344,20 +1343,13 @@ rpcrdma_decode_error(struct rpcrdma_xprt r_xprt, struct rpcrdma_rep rep, · 998 unmodified lines
1344	1343	p = xdr_inline_decode(xdr, 2 * sizeof(*p));
1345	1344	if (!p)
1346	1345	break;
1347		− dprintk("RPC: %s: server reports "
1348		− "version error (%u-%u), xid %08x\n", __func__,
1349		− be32_to_cpup(p), be32_to_cpu(*(p + 1)),
1350		− be32_to_cpu(rep->rr_xid));
	1346	+ trace_xprtrdma_err_vers(rqst, p, p + 1);
1351	1347	break;
1352	1348	case err_chunk:
1353		− dprintk("RPC: %s: server reports "
1354		− "header decoding error, xid %08x\n", __func__,
1355		− be32_to_cpu(rep->rr_xid));
	1349	+ trace_xprtrdma_err_chunk(rqst);
1356	1350	break;
1357	1351	default:
1358		− dprintk("RPC: %s: server reports "
1359		− "unrecognized error %d, xid %08x\n", __func__,
1360		− be32_to_cpup(p), be32_to_cpu(rep->rr_xid));
	1352	+ trace_xprtrdma_err_unrecognized(rqst, p);
1361	1353	}
1362	1354
1363	1355	return -EIO;
@@ -1398,7 +1390,7 @@ out: · 34 unmodified lines
1398	1390	return;
1399	1391
1400	1392	out_badheader:
1401		− trace_xprtrdma_reply_hdr(rep);
	1393	+ trace_xprtrdma_reply_hdr_err(rep);
1402	1394	r_xprt->rx_stats.bad_reply_count++;
1403	1395	rqst->rq_task->tk_status = status;
1404	1396	status = 0;
@@ -1472,14 +1464,12 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *rep) · 67 unmodified lines
1472	1464	rpcrdma_post_recvs(r_xprt, false);
1473	1465
1474	1466	req = rpcr_to_rdmar(rqst);
1475		− if (req->rl_reply) {
1476		− trace_xprtrdma_leaked_rep(rqst, req->rl_reply);
	1467	+ if (unlikely(req->rl_reply))
1477	1468	rpcrdma_recv_buffer_put(req->rl_reply);
1478		− }
1479	1469	req->rl_reply = rep;
1480	1470	rep->rr_rqst = rqst;
1481	1471
1482		− trace_xprtrdma_reply(rqst->rq_task, rep, req, credits);
	1472	+ trace_xprtrdma_reply(rqst->rq_task, rep, credits);
1483	1473
1484	1474	if (rep->rr_wc_flags & IB_WC_WITH_INVALIDATE)
1485	1475	frwr_reminv(rep, &req->rl_registered);
@@ -1491,16 +1481,16 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *rep) · 5 unmodified lines
1491	1481	return;
1492	1482
1493	1483	out_badversion:
1494		− trace_xprtrdma_reply_vers(rep);
	1484	+ trace_xprtrdma_reply_vers_err(rep);
1495	1485	goto out;
1496	1486
1497	1487	out_norqst:
1498	1488	spin_unlock(&xprt->queue_lock);
1499		− trace_xprtrdma_reply_rqst(rep);
	1489	+ trace_xprtrdma_reply_rqst_err(rep);
1500	1490	goto out;
1501	1491
1502	1492	out_shortreply:
1503		− trace_xprtrdma_reply_short(rep);
	1493	+ trace_xprtrdma_reply_short_err(rep);
1504	1494
1505	1495	out:
1506	1496	rpcrdma_recv_buffer_put(rep);

net/sunrpc/xprtrdma/transport.c+4−3

@@ -599,11 +599,12 @@ static void · 598 unmodified lines
599	599	xprt_rdma_free(struct rpc_task *task)
600	600	{
601	601	struct rpc_rqst *rqst = task->tk_rqstp;
602		− struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(rqst->rq_xprt);
603	602	struct rpcrdma_req *req = rpcr_to_rdmar(rqst);
604	603
605		− if (!list_empty(&req->rl_registered))
606		− frwr_unmap_sync(r_xprt, req);
	604	+ if (unlikely(!list_empty(&req->rl_registered))) {
	605	+ trace_xprtrdma_mrs_zap(task);
	606	+ frwr_unmap_sync(rpcx_to_rdmax(rqst->rq_xprt), req);
	607	+ }
607	608
608	609	/* XXX: If the RPC is completing because of a signal and
609	610	* not because a reply was received, we ought to ensure

net/sunrpc/xprtrdma/verbs.c+9−21

@@ -167,7 +167,7 @@ static void rpcrdma_wc_send(struct ib_cq cq, struct ib_wc wc) · 166 unmodified lines
167	167	struct rpcrdma_xprt *r_xprt = cq->cq_context;
168	168
169	169	/* WARNING: Only wr_cqe and status are reliable at this point */
170		− trace_xprtrdma_wc_send(sc, wc);
	170	+ trace_xprtrdma_wc_send(wc, &sc->sc_cid);
171	171	rpcrdma_sendctx_put_locked(r_xprt, sc);
172	172	rpcrdma_flush_disconnect(r_xprt, wc);
173	173	}
@@ -186,7 +186,7 @@ static void rpcrdma_wc_receive(struct ib_cq cq, struct ib_wc wc) · 12 unmodified lines
186	186	struct rpcrdma_xprt *r_xprt = cq->cq_context;
187	187
188	188	/* WARNING: Only wr_cqe and status are reliable at this point */
189		− trace_xprtrdma_wc_receive(wc);
	189	+ trace_xprtrdma_wc_receive(wc, &rep->rr_cid);
190	190	--r_xprt->rx_ep->re_receive_count;
191	191	if (wc->status != IB_WC_SUCCESS)
192	192	goto out_flushed;
@@ -643,6 +643,9 @@ static struct rpcrdma_sendctx rpcrdma_sendctx_create(struct rpcrdma_ep ep) · 450 unmodified lines
643	643	return NULL;
644	644
645	645	sc->sc_cqe.done = rpcrdma_wc_send;
	646	+ sc->sc_cid.ci_queue_id = ep->re_attr.send_cq->res.id;
	647	+ sc->sc_cid.ci_completion_id =
	648	+ atomic_inc_return(&ep->re_completion_ids);
646	649	return sc;
647	650	}
648	651
@@ -972,6 +975,9 @@ struct rpcrdma_rep rpcrdma_rep_create(struct rpcrdma_xprt r_xprt, · 323 unmodified lines
972	975	if (!rpcrdma_regbuf_dma_map(r_xprt, rep->rr_rdmabuf))
973	976	goto out_free_regbuf;
974	977
	978	+ rep->rr_cid.ci_completion_id =
	979	+ atomic_inc_return(&r_xprt->rx_ep->re_completion_ids);
	980	+
975	981	xdr_buf_init(&rep->rr_hdrbuf, rdmab_data(rep->rr_rdmabuf),
976	982	rdmab_length(rep->rr_rdmabuf));
977	983	rep->rr_cqe.done = rpcrdma_wc_receive;
@@ -1178,25 +1184,6 @@ rpcrdma_mr_get(struct rpcrdma_xprt *r_xprt) · 200 unmodified lines
1178	1184	return mr;
1179	1185	}
1180	1186
1181		−/**
1182		− * rpcrdma_mr_put - DMA unmap an MR and release it
1183		− * @mr: MR to release
1184		− *
1185		− */
1186		−void rpcrdma_mr_put(struct rpcrdma_mr *mr)
1187		−{
1188		− struct rpcrdma_xprt *r_xprt = mr->mr_xprt;
1189		−
1190		− if (mr->mr_dir != DMA_NONE) {
1191		− trace_xprtrdma_mr_unmap(mr);
1192		− ib_dma_unmap_sg(r_xprt->rx_ep->re_id->device,
1193		− mr->mr_sg, mr->mr_nents, mr->mr_dir);
1194		− mr->mr_dir = DMA_NONE;
1195		− }
1196		−
1197		− rpcrdma_mr_push(mr, &mr->mr_req->rl_free_mrs);
1198		−}
1199		−
1200	1187	/**
1201	1188	* rpcrdma_buffer_get - Get a request buffer
1202	1189	* @buffers: Buffer pool from which to obtain a buffer
@@ -1411,6 +1398,7 @@ void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp) · 208 unmodified lines
1411	1398	if (!rep)
1412	1399	break;
1413	1400
	1401	+ rep->rr_cid.ci_queue_id = ep->re_attr.recv_cq->res.id;
1414	1402	trace_xprtrdma_post_recv(rep);
1415	1403	rep->rr_recv_wr.next = wr;
1416	1404	wr = &rep->rr_recv_wr;

net/sunrpc/xprtrdma/xprt_rdma.h+8−1

@@ -53,6 +53,7 @@ · 52 unmodified lines
53	53	#include <rdma/ib_verbs.h> /* RDMA verbs api */
54	54
55	55	#include <linux/sunrpc/clnt.h> /* rpc_xprt */
	56	+#include <linux/sunrpc/rpc_rdma_cid.h> /* completion IDs */
56	57	#include <linux/sunrpc/rpc_rdma.h> /* RPC/RDMA protocol */
57	58	#include <linux/sunrpc/xprtrdma.h> /* xprt parameters */
58	59
@@ -93,6 +94,8 @@ struct rpcrdma_ep { · 34 unmodified lines
93	94	unsigned int re_max_requests; /* depends on device */
94	95	unsigned int re_inline_send; /* negotiated */
95	96	unsigned int re_inline_recv; /* negotiated */
	97	+
	98	+ atomic_t re_completion_ids;
96	99	};
97	100
98	101	/* Pre-allocate extra Work Requests for handling backward receives
@@ -180,6 +183,8 @@ enum { · 81 unmodified lines
180	183
181	184	struct rpcrdma_rep {
182	185	struct ib_cqe rr_cqe;
	186	+ struct rpc_rdma_cid rr_cid;
	187	+
183	188	__be32 rr_xid;
184	189	__be32 rr_vers;
185	190	__be32 rr_proc;
@@ -211,6 +216,7 @@ enum { · 25 unmodified lines
211	216	struct rpcrdma_req;
212	217	struct rpcrdma_sendctx {
213	218	struct ib_cqe sc_cqe;
	219	+ struct rpc_rdma_cid sc_cid;
214	220	struct rpcrdma_req *sc_req;
215	221	unsigned int sc_unmap_count;
216	222	struct ib_sge sc_sges[];
@@ -225,6 +231,7 @@ struct rpcrdma_sendctx { · 8 unmodified lines
225	231	struct rpcrdma_frwr {
226	232	struct ib_mr *fr_mr;
227	233	struct ib_cqe fr_cqe;
	234	+ struct rpc_rdma_cid fr_cid;
228	235	struct completion fr_linv_done;
229	236	union {
230	237	struct ib_reg_wr fr_regwr;
@@ -236,6 +243,7 @@ struct rpcrdma_req; · 5 unmodified lines
236	243	struct rpcrdma_mr {
237	244	struct list_head mr_list;
238	245	struct rpcrdma_req *mr_req;
	246	+ struct ib_device *mr_device;
239	247	struct scatterlist *mr_sg;
240	248	int mr_nents;
241	249	enum dma_data_direction mr_dir;
@@ -466,7 +474,6 @@ void rpcrdma_buffer_destroy(struct rpcrdma_buffer *); · 224 unmodified lines
466	474	struct rpcrdma_sendctx rpcrdma_sendctx_get_locked(struct rpcrdma_xprt r_xprt);
467	475
468	476	struct rpcrdma_mr rpcrdma_mr_get(struct rpcrdma_xprt r_xprt);
469		−void rpcrdma_mr_put(struct rpcrdma_mr *mr);
470	477	void rpcrdma_mrs_refresh(struct rpcrdma_xprt *r_xprt);
471	478
472	479	struct rpcrdma_req rpcrdma_buffer_get(struct rpcrdma_buffer );

Merge tag 'nfs-rdma-for-5.11-1' of git://git.linux-nfs.org/projects/anna/linux-nfs into linux-next

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code