History

Merge tag 'nfs-rdma-for-5.11-1' of git://git.linux-nfs.org/projects/anna/linux-nfs into linux-next

NFSoRDmA Client updates for Linux 5.11 Cleanups and improvements: - Remove use of raw kernel memory addresses in tracepoints - Replace dprintk() call sites in ERR_CHUNK path - Trace unmap sync calls - Optimize MR DMA-unmapping

Trond Myklebust committed 5 years ago

Commit edffb842 parents 5c3485b 7a03aeb7 files · +302 −313Browse files

Unified · Split

7 files
  1. include/trace/events/rpcrdma.h +212 −238
  2. net/sunrpc/xprtrdma/backchannel.c +3 −3
  3. net/sunrpc/xprtrdma/frwr_ops.c +55 −26
  4. net/sunrpc/xprtrdma/rpc_rdma.c +11 −21
  5. net/sunrpc/xprtrdma/transport.c +4 −3
  6. net/sunrpc/xprtrdma/verbs.c +9 −21
  7. net/sunrpc/xprtrdma/xprt_rdma.h +8 −1
include/trace/events/rpcrdma.h+212−238
@@ -60,7 +60,7 @@ DECLARE_EVENT_CLASS(rpcrdma_completion_class, · 59 unmodified lines
60 ), \60 ), \
61 TP_ARGS(wc, cid))61 TP_ARGS(wc, cid))
62 62
63DECLARE_EVENT_CLASS(xprtrdma_reply_event,63DECLARE_EVENT_CLASS(xprtrdma_reply_class,
64 TP_PROTO(64 TP_PROTO(
65 const struct rpcrdma_rep *rep65 const struct rpcrdma_rep *rep
66 ),66 ),
@@ -68,29 +68,30 @@ DECLARE_EVENT_CLASS(xprtrdma_reply_event, · 1 unmodified line
68 TP_ARGS(rep),68 TP_ARGS(rep),
69 69
70 TP_STRUCT__entry(70 TP_STRUCT__entry(
71 __field(const void *, rep)
72 __field(const void *, r_xprt)
73 __field(u32, xid)71 __field(u32, xid)
74 __field(u32, version)72 __field(u32, version)
75 __field(u32, proc)73 __field(u32, proc)
74 __string(addr, rpcrdma_addrstr(rep->rr_rxprt))
75 __string(port, rpcrdma_portstr(rep->rr_rxprt))
76 ),76 ),
77 77
78 TP_fast_assign(78 TP_fast_assign(
79 __entry->rep = rep;
80 __entry->r_xprt = rep->rr_rxprt;
81 __entry->xid = be32_to_cpu(rep->rr_xid);79 __entry->xid = be32_to_cpu(rep->rr_xid);
82 __entry->version = be32_to_cpu(rep->rr_vers);80 __entry->version = be32_to_cpu(rep->rr_vers);
83 __entry->proc = be32_to_cpu(rep->rr_proc);81 __entry->proc = be32_to_cpu(rep->rr_proc);
82 __assign_str(addr, rpcrdma_addrstr(rep->rr_rxprt));
83 __assign_str(port, rpcrdma_portstr(rep->rr_rxprt));
84 ),84 ),
85 85
86 TP_printk("rxprt %p xid=0x%08x rep=%p: version %u proc %u",86 TP_printk("peer=[%s]:%s xid=0x%08x version=%u proc=%u",
87 __entry->r_xprt, __entry->xid, __entry->rep,87 __get_str(addr), __get_str(port),
88 __entry->version, __entry->proc88 __entry->xid, __entry->version, __entry->proc
89 )89 )
90);90);
91 91
92#define DEFINE_REPLY_EVENT(name) \92#define DEFINE_REPLY_EVENT(name) \
93 DEFINE_EVENT(xprtrdma_reply_event, name, \93 DEFINE_EVENT(xprtrdma_reply_class, \
94 xprtrdma_reply_##name##_err, \
94 TP_PROTO( \95 TP_PROTO( \
95 const struct rpcrdma_rep *rep \96 const struct rpcrdma_rep *rep \
96 ), \97 ), \
@@ -261,54 +262,67 @@ DECLARE_EVENT_CLASS(xprtrdma_wrch_event, · 164 unmodified lines
261 ), \262 ), \
262 TP_ARGS(task, mr, nsegs))263 TP_ARGS(task, mr, nsegs))
263 264
264DECLARE_EVENT_CLASS(xprtrdma_frwr_done,265TRACE_DEFINE_ENUM(DMA_BIDIRECTIONAL);
266TRACE_DEFINE_ENUM(DMA_TO_DEVICE);
267TRACE_DEFINE_ENUM(DMA_FROM_DEVICE);
268TRACE_DEFINE_ENUM(DMA_NONE);
269
270#define xprtrdma_show_direction(x) \
271 __print_symbolic(x, \
272 { DMA_BIDIRECTIONAL, "BIDIR" }, \
273 { DMA_TO_DEVICE, "TO_DEVICE" }, \
274 { DMA_FROM_DEVICE, "FROM_DEVICE" }, \
275 { DMA_NONE, "NONE" })
276
277DECLARE_EVENT_CLASS(xprtrdma_mr_class,
265 TP_PROTO(278 TP_PROTO(
266 const struct ib_wc *wc,279 const struct rpcrdma_mr *mr
267 const struct rpcrdma_frwr *frwr
268 ),280 ),
269 281
270 TP_ARGS(wc, frwr),282 TP_ARGS(mr),
271 283
272 TP_STRUCT__entry(284 TP_STRUCT__entry(
285 __field(unsigned int, task_id)
286 __field(unsigned int, client_id)
273 __field(u32, mr_id)287 __field(u32, mr_id)
274 __field(unsigned int, status)288 __field(int, nents)
275 __field(unsigned int, vendor_err)289 __field(u32, handle)
290 __field(u32, length)
291 __field(u64, offset)
292 __field(u32, dir)
276 ),293 ),
277 294
278 TP_fast_assign(295 TP_fast_assign(
279 __entry->mr_id = frwr->fr_mr->res.id;296 const struct rpcrdma_req *req = mr->mr_req;
280 __entry->status = wc->status;297 const struct rpc_task *task = req->rl_slot.rq_task;
281 __entry->vendor_err = __entry->status ? wc->vendor_err : 0;298
299 __entry->task_id = task->tk_pid;
300 __entry->client_id = task->tk_client->cl_clid;
301 __entry->mr_id = mr->frwr.fr_mr->res.id;
302 __entry->nents = mr->mr_nents;
303 __entry->handle = mr->mr_handle;
304 __entry->length = mr->mr_length;
305 __entry->offset = mr->mr_offset;
306 __entry->dir = mr->mr_dir;
282 ),307 ),
283 308
284 TP_printk(309 TP_printk("task:%u@%u mr.id=%u nents=%d %u@0x%016llx:0x%08x (%s)",
285 "mr.id=%u: %s (%u/0x%x)",310 __entry->task_id, __entry->client_id,
286 __entry->mr_id, rdma_show_wc_status(__entry->status),311 __entry->mr_id, __entry->nents, __entry->length,
287 __entry->status, __entry->vendor_err312 (unsigned long long)__entry->offset, __entry->handle,
313 xprtrdma_show_direction(__entry->dir)
288 )314 )
289);315);
290 316
291#define DEFINE_FRWR_DONE_EVENT(name) \317#define DEFINE_MR_EVENT(name) \
292 DEFINE_EVENT(xprtrdma_frwr_done, name, \318 DEFINE_EVENT(xprtrdma_mr_class, \
319 xprtrdma_mr_##name, \
293 TP_PROTO( \320 TP_PROTO( \
294 const struct ib_wc *wc, \321 const struct rpcrdma_mr *mr \
295 const struct rpcrdma_frwr *frwr \
296 ), \322 ), \
297 TP_ARGS(wc, frwr))323 TP_ARGS(mr))
298
299TRACE_DEFINE_ENUM(DMA_BIDIRECTIONAL);
300TRACE_DEFINE_ENUM(DMA_TO_DEVICE);
301TRACE_DEFINE_ENUM(DMA_FROM_DEVICE);
302TRACE_DEFINE_ENUM(DMA_NONE);
303
304#define xprtrdma_show_direction(x) \
305 __print_symbolic(x, \
306 { DMA_BIDIRECTIONAL, "BIDIR" }, \
307 { DMA_TO_DEVICE, "TO_DEVICE" }, \
308 { DMA_FROM_DEVICE, "FROM_DEVICE" }, \
309 { DMA_NONE, "NONE" })
310 324
311DECLARE_EVENT_CLASS(xprtrdma_mr,325DECLARE_EVENT_CLASS(xprtrdma_anonymous_mr_class,
312 TP_PROTO(326 TP_PROTO(
313 const struct rpcrdma_mr *mr327 const struct rpcrdma_mr *mr
314 ),328 ),
@@ -340,45 +354,47 @@ DECLARE_EVENT_CLASS(xprtrdma_mr, · 25 unmodified lines
340 )354 )
341);355);
342 356
343#define DEFINE_MR_EVENT(name) \357#define DEFINE_ANON_MR_EVENT(name) \
344 DEFINE_EVENT(xprtrdma_mr, xprtrdma_mr_##name, \358 DEFINE_EVENT(xprtrdma_anonymous_mr_class, \
345 TP_PROTO( \359 xprtrdma_mr_##name, \
346 const struct rpcrdma_mr *mr \360 TP_PROTO( \
347 ), \361 const struct rpcrdma_mr *mr \
362 ), \
348 TP_ARGS(mr))363 TP_ARGS(mr))
349 364
350DECLARE_EVENT_CLASS(xprtrdma_cb_event,365DECLARE_EVENT_CLASS(xprtrdma_callback_class,
351 TP_PROTO(366 TP_PROTO(
367 const struct rpcrdma_xprt *r_xprt,
352 const struct rpc_rqst *rqst368 const struct rpc_rqst *rqst
353 ),369 ),
354 370
355 TP_ARGS(rqst),371 TP_ARGS(r_xprt, rqst),
356 372
357 TP_STRUCT__entry(373 TP_STRUCT__entry(
358 __field(const void *, rqst)
359 __field(const void *, rep)
360 __field(const void *, req)
361 __field(u32, xid)374 __field(u32, xid)
375 __string(addr, rpcrdma_addrstr(r_xprt))
376 __string(port, rpcrdma_portstr(r_xprt))
362 ),377 ),
363 378
364 TP_fast_assign(379 TP_fast_assign(
365 __entry->rqst = rqst;
366 __entry->req = rpcr_to_rdmar(rqst);
367 __entry->rep = rpcr_to_rdmar(rqst)->rl_reply;
368 __entry->xid = be32_to_cpu(rqst->rq_xid);380 __entry->xid = be32_to_cpu(rqst->rq_xid);
381 __assign_str(addr, rpcrdma_addrstr(r_xprt));
382 __assign_str(port, rpcrdma_portstr(r_xprt));
369 ),383 ),
370 384
371 TP_printk("xid=0x%08x, rqst=%p req=%p rep=%p",385 TP_printk("peer=[%s]:%s xid=0x%08x",
372 __entry->xid, __entry->rqst, __entry->req, __entry->rep386 __get_str(addr), __get_str(port), __entry->xid
373 )387 )
374);388);
375 389
376#define DEFINE_CB_EVENT(name) \390#define DEFINE_CALLBACK_EVENT(name) \
377 DEFINE_EVENT(xprtrdma_cb_event, name, \391 DEFINE_EVENT(xprtrdma_callback_class, \
392 xprtrdma_cb_##name, \
378 TP_PROTO( \393 TP_PROTO( \
394 const struct rpcrdma_xprt *r_xprt, \
379 const struct rpc_rqst *rqst \395 const struct rpc_rqst *rqst \
380 ), \396 ), \
381 TP_ARGS(rqst))397 TP_ARGS(r_xprt, rqst))
382 398
383/**399/**
384 ** Connection events400 ** Connection events
@@ -549,61 +565,33 @@ TRACE_EVENT(xprtrdma_createmrs, · 164 unmodified lines
549 )565 )
550);566);
551 567
552TRACE_EVENT(xprtrdma_mr_get,568TRACE_EVENT(xprtrdma_nomrs_err,
553 TP_PROTO(
554 const struct rpcrdma_req *req
555 ),
556
557 TP_ARGS(req),
558
559 TP_STRUCT__entry(
560 __field(const void *, req)
561 __field(unsigned int, task_id)
562 __field(unsigned int, client_id)
563 __field(u32, xid)
564 ),
565
566 TP_fast_assign(
567 const struct rpc_rqst *rqst = &req->rl_slot;
568
569 __entry->req = req;
570 __entry->task_id = rqst->rq_task->tk_pid;
571 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
572 __entry->xid = be32_to_cpu(rqst->rq_xid);
573 ),
574
575 TP_printk("task:%u@%u xid=0x%08x req=%p",
576 __entry->task_id, __entry->client_id, __entry->xid,
577 __entry->req
578 )
579);
580
581TRACE_EVENT(xprtrdma_nomrs,
582 TP_PROTO(569 TP_PROTO(
570 const struct rpcrdma_xprt *r_xprt,
583 const struct rpcrdma_req *req571 const struct rpcrdma_req *req
584 ),572 ),
585 573
586 TP_ARGS(req),574 TP_ARGS(r_xprt, req),
587 575
588 TP_STRUCT__entry(576 TP_STRUCT__entry(
589 __field(const void *, req)
590 __field(unsigned int, task_id)577 __field(unsigned int, task_id)
591 __field(unsigned int, client_id)578 __field(unsigned int, client_id)
592 __field(u32, xid)579 __string(addr, rpcrdma_addrstr(r_xprt))
580 __string(port, rpcrdma_portstr(r_xprt))
593 ),581 ),
594 582
595 TP_fast_assign(583 TP_fast_assign(
596 const struct rpc_rqst *rqst = &req->rl_slot;584 const struct rpc_rqst *rqst = &req->rl_slot;
597 585
598 __entry->req = req;
599 __entry->task_id = rqst->rq_task->tk_pid;586 __entry->task_id = rqst->rq_task->tk_pid;
600 __entry->client_id = rqst->rq_task->tk_client->cl_clid;587 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
601 __entry->xid = be32_to_cpu(rqst->rq_xid);588 __assign_str(addr, rpcrdma_addrstr(r_xprt));
589 __assign_str(port, rpcrdma_portstr(r_xprt));
602 ),590 ),
603 591
604 TP_printk("task:%u@%u xid=0x%08x req=%p",592 TP_printk("peer=[%s]:%s task:%u@%u",
605 __entry->task_id, __entry->client_id, __entry->xid,593 __get_str(addr), __get_str(port),
606 __entry->req594 __entry->task_id, __entry->client_id
607 )595 )
608);596);
609 597
@@ -735,8 +723,8 @@ TRACE_EVENT(xprtrdma_post_send, · 125 unmodified lines
735 TP_ARGS(req),723 TP_ARGS(req),
736 724
737 TP_STRUCT__entry(725 TP_STRUCT__entry(
738 __field(const void *, req)726 __field(u32, cq_id)
739 __field(const void *, sc)727 __field(int, completion_id)
740 __field(unsigned int, task_id)728 __field(unsigned int, task_id)
741 __field(unsigned int, client_id)729 __field(unsigned int, client_id)
742 __field(int, num_sge)730 __field(int, num_sge)
@@ -745,20 +733,21 @@ TRACE_EVENT(xprtrdma_post_send, · 2 unmodified lines
745 733
746 TP_fast_assign(734 TP_fast_assign(
747 const struct rpc_rqst *rqst = &req->rl_slot;735 const struct rpc_rqst *rqst = &req->rl_slot;
736 const struct rpcrdma_sendctx *sc = req->rl_sendctx;
748 737
738 __entry->cq_id = sc->sc_cid.ci_queue_id;
739 __entry->completion_id = sc->sc_cid.ci_completion_id;
749 __entry->task_id = rqst->rq_task->tk_pid;740 __entry->task_id = rqst->rq_task->tk_pid;
750 __entry->client_id = rqst->rq_task->tk_client ?741 __entry->client_id = rqst->rq_task->tk_client ?
751 rqst->rq_task->tk_client->cl_clid : -1;742 rqst->rq_task->tk_client->cl_clid : -1;
752 __entry->req = req;
753 __entry->sc = req->rl_sendctx;
754 __entry->num_sge = req->rl_wr.num_sge;743 __entry->num_sge = req->rl_wr.num_sge;
755 __entry->signaled = req->rl_wr.send_flags & IB_SEND_SIGNALED;744 __entry->signaled = req->rl_wr.send_flags & IB_SEND_SIGNALED;
756 ),745 ),
757 746
758 TP_printk("task:%u@%u req=%p sc=%p (%d SGE%s) %s",747 TP_printk("task:%u@%u cq.id=%u cid=%d (%d SGE%s) %s",
759 __entry->task_id, __entry->client_id,748 __entry->task_id, __entry->client_id,
760 __entry->req, __entry->sc, __entry->num_sge,749 __entry->cq_id, __entry->completion_id,
761 (__entry->num_sge == 1 ? "" : "s"),750 __entry->num_sge, (__entry->num_sge == 1 ? "" : "s"),
762 (__entry->signaled ? "signaled" : "")751 (__entry->signaled ? "signaled" : "")
763 )752 )
764);753);
@@ -771,15 +760,17 @@ TRACE_EVENT(xprtrdma_post_recv, · 6 unmodified lines
771 TP_ARGS(rep),760 TP_ARGS(rep),
772 761
773 TP_STRUCT__entry(762 TP_STRUCT__entry(
774 __field(const void *, rep)763 __field(u32, cq_id)
764 __field(int, completion_id)
775 ),765 ),
776 766
777 TP_fast_assign(767 TP_fast_assign(
778 __entry->rep = rep;768 __entry->cq_id = rep->rr_cid.ci_queue_id;
769 __entry->completion_id = rep->rr_cid.ci_completion_id;
779 ),770 ),
780 771
781 TP_printk("rep=%p",772 TP_printk("cq.id=%d cid=%d",
782 __entry->rep773 __entry->cq_id, __entry->completion_id
783 )774 )
784);775);
785 776
@@ -816,7 +807,7 @@ TRACE_EVENT(xprtrdma_post_recvs, · 30 unmodified lines
816 )807 )
817);808);
818 809
819TRACE_EVENT(xprtrdma_post_linv,810TRACE_EVENT(xprtrdma_post_linv_err,
820 TP_PROTO(811 TP_PROTO(
821 const struct rpcrdma_req *req,812 const struct rpcrdma_req *req,
822 int status813 int status
@@ -825,19 +816,21 @@ TRACE_EVENT(xprtrdma_post_linv, · 2 unmodified lines
825 TP_ARGS(req, status),816 TP_ARGS(req, status),
826 817
827 TP_STRUCT__entry(818 TP_STRUCT__entry(
828 __field(const void *, req)819 __field(unsigned int, task_id)
820 __field(unsigned int, client_id)
829 __field(int, status)821 __field(int, status)
830 __field(u32, xid)
831 ),822 ),
832 823
833 TP_fast_assign(824 TP_fast_assign(
834 __entry->req = req;825 const struct rpc_task *task = req->rl_slot.rq_task;
826
827 __entry->task_id = task->tk_pid;
828 __entry->client_id = task->tk_client->cl_clid;
835 __entry->status = status;829 __entry->status = status;
836 __entry->xid = be32_to_cpu(req->rl_slot.rq_xid);
837 ),830 ),
838 831
839 TP_printk("req=%p xid=0x%08x status=%d",832 TP_printk("task:%u@%u status=%d",
840 __entry->req, __entry->xid, __entry->status833 __entry->task_id, __entry->client_id, __entry->status
841 )834 )
842);835);
843 836
@@ -845,75 +838,12 @@ TRACE_EVENT(xprtrdma_post_linv, · 1 unmodified line
845 ** Completion events838 ** Completion events
846 **/839 **/
847 840
848TRACE_EVENT(xprtrdma_wc_send,841DEFINE_COMPLETION_EVENT(xprtrdma_wc_receive);
849 TP_PROTO(842DEFINE_COMPLETION_EVENT(xprtrdma_wc_send);
850 const struct rpcrdma_sendctx *sc,843DEFINE_COMPLETION_EVENT(xprtrdma_wc_fastreg);
851 const struct ib_wc *wc844DEFINE_COMPLETION_EVENT(xprtrdma_wc_li);
852 ),845DEFINE_COMPLETION_EVENT(xprtrdma_wc_li_wake);
853 846DEFINE_COMPLETION_EVENT(xprtrdma_wc_li_done);
854 TP_ARGS(sc, wc),
855
856 TP_STRUCT__entry(
857 __field(const void *, req)
858 __field(const void *, sc)
859 __field(unsigned int, unmap_count)
860 __field(unsigned int, status)
861 __field(unsigned int, vendor_err)
862 ),
863
864 TP_fast_assign(
865 __entry->req = sc->sc_req;
866 __entry->sc = sc;
867 __entry->unmap_count = sc->sc_unmap_count;
868 __entry->status = wc->status;
869 __entry->vendor_err = __entry->status ? wc->vendor_err : 0;
870 ),
871
872 TP_printk("req=%p sc=%p unmapped=%u: %s (%u/0x%x)",
873 __entry->req, __entry->sc, __entry->unmap_count,
874 rdma_show_wc_status(__entry->status),
875 __entry->status, __entry->vendor_err
876 )
877);
878
879TRACE_EVENT(xprtrdma_wc_receive,
880 TP_PROTO(
881 const struct ib_wc *wc
882 ),
883
884 TP_ARGS(wc),
885
886 TP_STRUCT__entry(
887 __field(const void *, rep)
888 __field(u32, byte_len)
889 __field(unsigned int, status)
890 __field(u32, vendor_err)
891 ),
892
893 TP_fast_assign(
894 __entry->rep = container_of(wc->wr_cqe, struct rpcrdma_rep,
895 rr_cqe);
896 __entry->status = wc->status;
897 if (wc->status) {
898 __entry->byte_len = 0;
899 __entry->vendor_err = wc->vendor_err;
900 } else {
901 __entry->byte_len = wc->byte_len;
902 __entry->vendor_err = 0;
903 }
904 ),
905
906 TP_printk("rep=%p %u bytes: %s (%u/0x%x)",
907 __entry->rep, __entry->byte_len,
908 rdma_show_wc_status(__entry->status),
909 __entry->status, __entry->vendor_err
910 )
911);
912
913DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_fastreg);
914DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li);
915DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li_wake);
916DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li_done);
917 847
918TRACE_EVENT(xprtrdma_frwr_alloc,848TRACE_EVENT(xprtrdma_frwr_alloc,
919 TP_PROTO(849 TP_PROTO(
@@ -1036,9 +966,9 @@ TRACE_EVENT(xprtrdma_frwr_maperr, · 116 unmodified lines
1036 966
1037DEFINE_MR_EVENT(localinv);967DEFINE_MR_EVENT(localinv);
1038DEFINE_MR_EVENT(map);968DEFINE_MR_EVENT(map);
1039DEFINE_MR_EVENT(unmap);969
1040DEFINE_MR_EVENT(reminv);970DEFINE_ANON_MR_EVENT(unmap);
1041DEFINE_MR_EVENT(recycle);971DEFINE_ANON_MR_EVENT(recycle);
1042 972
1043TRACE_EVENT(xprtrdma_dma_maperr,973TRACE_EVENT(xprtrdma_dma_maperr,
1044 TP_PROTO(974 TP_PROTO(
@@ -1066,17 +996,14 @@ TRACE_EVENT(xprtrdma_reply, · 21 unmodified lines
1066 TP_PROTO(996 TP_PROTO(
1067 const struct rpc_task *task,997 const struct rpc_task *task,
1068 const struct rpcrdma_rep *rep,998 const struct rpcrdma_rep *rep,
1069 const struct rpcrdma_req *req,
1070 unsigned int credits999 unsigned int credits
1071 ),1000 ),
1072 1001
1073 TP_ARGS(task, rep, req, credits),1002 TP_ARGS(task, rep, credits),
1074 1003
1075 TP_STRUCT__entry(1004 TP_STRUCT__entry(
1076 __field(unsigned int, task_id)1005 __field(unsigned int, task_id)
1077 __field(unsigned int, client_id)1006 __field(unsigned int, client_id)
1078 __field(const void *, rep)
1079 __field(const void *, req)
1080 __field(u32, xid)1007 __field(u32, xid)
1081 __field(unsigned int, credits)1008 __field(unsigned int, credits)
1082 ),1009 ),
@@ -1084,49 +1011,102 @@ TRACE_EVENT(xprtrdma_reply, · 1 unmodified line
1084 TP_fast_assign(1011 TP_fast_assign(
1085 __entry->task_id = task->tk_pid;1012 __entry->task_id = task->tk_pid;
1086 __entry->client_id = task->tk_client->cl_clid;1013 __entry->client_id = task->tk_client->cl_clid;
1087 __entry->rep = rep;
1088 __entry->req = req;
1089 __entry->xid = be32_to_cpu(rep->rr_xid);1014 __entry->xid = be32_to_cpu(rep->rr_xid);
1090 __entry->credits = credits;1015 __entry->credits = credits;
1091 ),1016 ),
1092 1017
1093 TP_printk("task:%u@%u xid=0x%08x, %u credits, rep=%p -> req=%p",1018 TP_printk("task:%u@%u xid=0x%08x credits=%u",
1094 __entry->task_id, __entry->client_id, __entry->xid,1019 __entry->task_id, __entry->client_id, __entry->xid,
1095 __entry->credits, __entry->rep, __entry->req1020 __entry->credits
1096 )1021 )
1097);1022);
1098 1023
1099TRACE_EVENT(xprtrdma_defer_cmp,1024DEFINE_REPLY_EVENT(vers);
1025DEFINE_REPLY_EVENT(rqst);
1026DEFINE_REPLY_EVENT(short);
1027DEFINE_REPLY_EVENT(hdr);
1028
1029TRACE_EVENT(xprtrdma_err_vers,
1100 TP_PROTO(1030 TP_PROTO(
1101 const struct rpcrdma_rep *rep1031 const struct rpc_rqst *rqst,
1032 __be32 *min,
1033 __be32 *max
1102 ),1034 ),
1103 1035
1104 TP_ARGS(rep),1036 TP_ARGS(rqst, min, max),
1105 1037
1106 TP_STRUCT__entry(1038 TP_STRUCT__entry(
1107 __field(unsigned int, task_id)1039 __field(unsigned int, task_id)
1108 __field(unsigned int, client_id)1040 __field(unsigned int, client_id)
1109 __field(const void *, rep)
1110 __field(u32, xid)1041 __field(u32, xid)
1042 __field(u32, min)
1043 __field(u32, max)
1111 ),1044 ),
1112 1045
1113 TP_fast_assign(1046 TP_fast_assign(
1114 __entry->task_id = rep->rr_rqst->rq_task->tk_pid;1047 __entry->task_id = rqst->rq_task->tk_pid;
1115 __entry->client_id = rep->rr_rqst->rq_task->tk_client->cl_clid;1048 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
1116 __entry->rep = rep;1049 __entry->xid = be32_to_cpu(rqst->rq_xid);
1117 __entry->xid = be32_to_cpu(rep->rr_xid);1050 __entry->min = be32_to_cpup(min);
1051 __entry->max = be32_to_cpup(max);
1118 ),1052 ),
1119 1053
1120 TP_printk("task:%u@%u xid=0x%08x rep=%p",1054 TP_printk("task:%u@%u xid=0x%08x versions=[%u, %u]",
1121 __entry->task_id, __entry->client_id, __entry->xid,1055 __entry->task_id, __entry->client_id, __entry->xid,
1122 __entry->rep1056 __entry->min, __entry->max
1057 )
1058);
1059
1060TRACE_EVENT(xprtrdma_err_chunk,
1061 TP_PROTO(
1062 const struct rpc_rqst *rqst
1063 ),
1064
1065 TP_ARGS(rqst),
1066
1067 TP_STRUCT__entry(
1068 __field(unsigned int, task_id)
1069 __field(unsigned int, client_id)
1070 __field(u32, xid)
1071 ),
1072
1073 TP_fast_assign(
1074 __entry->task_id = rqst->rq_task->tk_pid;
1075 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
1076 __entry->xid = be32_to_cpu(rqst->rq_xid);
1077 ),
1078
1079 TP_printk("task:%u@%u xid=0x%08x",
1080 __entry->task_id, __entry->client_id, __entry->xid
1123 )1081 )
1124);1082);
1125 1083
1126DEFINE_REPLY_EVENT(xprtrdma_reply_vers);1084TRACE_EVENT(xprtrdma_err_unrecognized,
1127DEFINE_REPLY_EVENT(xprtrdma_reply_rqst);1085 TP_PROTO(
1128DEFINE_REPLY_EVENT(xprtrdma_reply_short);1086 const struct rpc_rqst *rqst,
1129DEFINE_REPLY_EVENT(xprtrdma_reply_hdr);1087 __be32 *procedure
1088 ),
1089
1090 TP_ARGS(rqst, procedure),
1091
1092 TP_STRUCT__entry(
1093 __field(unsigned int, task_id)
1094 __field(unsigned int, client_id)
1095 __field(u32, xid)
1096 __field(u32, procedure)
1097 ),
1098
1099 TP_fast_assign(
1100 __entry->task_id = rqst->rq_task->tk_pid;
1101 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
1102 __entry->procedure = be32_to_cpup(procedure);
1103 ),
1104
1105 TP_printk("task:%u@%u xid=0x%08x procedure=%u",
1106 __entry->task_id, __entry->client_id, __entry->xid,
1107 __entry->procedure
1108 )
1109);
1130 1110
1131TRACE_EVENT(xprtrdma_fixup,1111TRACE_EVENT(xprtrdma_fixup,
1132 TP_PROTO(1112 TP_PROTO(
@@ -1187,6 +1167,28 @@ TRACE_EVENT(xprtrdma_decode_seg, · 54 unmodified lines
1187 )1167 )
1188);1168);
1189 1169
1170TRACE_EVENT(xprtrdma_mrs_zap,
1171 TP_PROTO(
1172 const struct rpc_task *task
1173 ),
1174
1175 TP_ARGS(task),
1176
1177 TP_STRUCT__entry(
1178 __field(unsigned int, task_id)
1179 __field(unsigned int, client_id)
1180 ),
1181
1182 TP_fast_assign(
1183 __entry->task_id = task->tk_pid;
1184 __entry->client_id = task->tk_client->cl_clid;
1185 ),
1186
1187 TP_printk("task:%u@%u",
1188 __entry->task_id, __entry->client_id
1189 )
1190);
1191
1190/**1192/**
1191 ** Callback events1193 ** Callback events
1192 **/1194 **/
@@ -1219,36 +1221,8 @@ TRACE_EVENT(xprtrdma_cb_setup, · 26 unmodified lines
1219 )1221 )
1220);1222);
1221 1223
1222DEFINE_CB_EVENT(xprtrdma_cb_call);1224DEFINE_CALLBACK_EVENT(call);
1223DEFINE_CB_EVENT(xprtrdma_cb_reply);1225DEFINE_CALLBACK_EVENT(reply);
1224
1225TRACE_EVENT(xprtrdma_leaked_rep,
1226 TP_PROTO(
1227 const struct rpc_rqst *rqst,
1228 const struct rpcrdma_rep *rep
1229 ),
1230
1231 TP_ARGS(rqst, rep),
1232
1233 TP_STRUCT__entry(
1234 __field(unsigned int, task_id)
1235 __field(unsigned int, client_id)
1236 __field(u32, xid)
1237 __field(const void *, rep)
1238 ),
1239
1240 TP_fast_assign(
1241 __entry->task_id = rqst->rq_task->tk_pid;
1242 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
1243 __entry->xid = be32_to_cpu(rqst->rq_xid);
1244 __entry->rep = rep;
1245 ),
1246
1247 TP_printk("task:%u@%u xid=0x%08x rep=%p",
1248 __entry->task_id, __entry->client_id, __entry->xid,
1249 __entry->rep
1250 )
1251);
1252 1226
1253/**1227/**
1254 ** Server-side RPC/RDMA events1228 ** Server-side RPC/RDMA events
net/sunrpc/xprtrdma/backchannel.c+3−3
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/*2/*
3 * Copyright (c) 2015 Oracle. All rights reserved.3 * Copyright (c) 2015-2020, Oracle and/or its affiliates.
4 *4 *
5 * Support for backward direction RPCs on RPC/RDMA.5 * Support for backward direction RPCs on RPC/RDMA.
6 */6 */
@@ -82,7 +82,7 @@ static int rpcrdma_bc_marshal_reply(struct rpc_rqst *rqst) · 75 unmodified lines
82 &rqst->rq_snd_buf, rpcrdma_noch_pullup))82 &rqst->rq_snd_buf, rpcrdma_noch_pullup))
83 return -EIO;83 return -EIO;
84 84
85 trace_xprtrdma_cb_reply(rqst);85 trace_xprtrdma_cb_reply(r_xprt, rqst);
86 return 0;86 return 0;
87}87}
88 88
@@ -260,7 +260,7 @@ void rpcrdma_bc_receive_call(struct rpcrdma_xprt *r_xprt, · 171 unmodified lines
260 */260 */
261 req = rpcr_to_rdmar(rqst);261 req = rpcr_to_rdmar(rqst);
262 req->rl_reply = rep;262 req->rl_reply = rep;
263 trace_xprtrdma_cb_call(rqst);263 trace_xprtrdma_cb_call(r_xprt, rqst);
264 264
265 /* Queue rqst for ULP's callback service */265 /* Queue rqst for ULP's callback service */
266 bc_serv = xprt->bc_serv;266 bc_serv = xprt->bc_serv;
net/sunrpc/xprtrdma/frwr_ops.c+55−26
@@ -65,18 +65,23 @@ void frwr_release_mr(struct rpcrdma_mr *mr) · 64 unmodified lines
65 kfree(mr);65 kfree(mr);
66}66}
67 67
68static void frwr_mr_unmap(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr *mr)
69{
70 if (mr->mr_device) {
71 trace_xprtrdma_mr_unmap(mr);
72 ib_dma_unmap_sg(mr->mr_device, mr->mr_sg, mr->mr_nents,
73 mr->mr_dir);
74 mr->mr_device = NULL;
75 }
76}
77
68static void frwr_mr_recycle(struct rpcrdma_mr *mr)78static void frwr_mr_recycle(struct rpcrdma_mr *mr)
69{79{
70 struct rpcrdma_xprt *r_xprt = mr->mr_xprt;80 struct rpcrdma_xprt *r_xprt = mr->mr_xprt;
71 81
72 trace_xprtrdma_mr_recycle(mr);82 trace_xprtrdma_mr_recycle(mr);
73 83
74 if (mr->mr_dir != DMA_NONE) {84 frwr_mr_unmap(r_xprt, mr);
75 trace_xprtrdma_mr_unmap(mr);
76 ib_dma_unmap_sg(r_xprt->rx_ep->re_id->device,
77 mr->mr_sg, mr->mr_nents, mr->mr_dir);
78 mr->mr_dir = DMA_NONE;
79 }
80 85
81 spin_lock(&r_xprt->rx_buf.rb_lock);86 spin_lock(&r_xprt->rx_buf.rb_lock);
82 list_del(&mr->mr_all);87 list_del(&mr->mr_all);
@@ -86,6 +91,16 @@ static void frwr_mr_recycle(struct rpcrdma_mr *mr) · 3 unmodified lines
86 frwr_release_mr(mr);91 frwr_release_mr(mr);
87}92}
88 93
94static void frwr_mr_put(struct rpcrdma_mr *mr)
95{
96 frwr_mr_unmap(mr->mr_xprt, mr);
97
98 /* The MR is returned to the req's MR free list instead
99 * of to the xprt's MR free list. No spinlock is needed.
100 */
101 rpcrdma_mr_push(mr, &mr->mr_req->rl_free_mrs);
102}
103
89/* frwr_reset - Place MRs back on the free list104/* frwr_reset - Place MRs back on the free list
90 * @req: request to reset105 * @req: request to reset
91 *106 *
@@ -101,7 +116,7 @@ void frwr_reset(struct rpcrdma_req *req) · 9 unmodified lines
101 struct rpcrdma_mr *mr;116 struct rpcrdma_mr *mr;
102 117
103 while ((mr = rpcrdma_mr_pop(&req->rl_registered)))118 while ((mr = rpcrdma_mr_pop(&req->rl_registered)))
104 rpcrdma_mr_put(mr);119 frwr_mr_put(mr);
105}120}
106 121
107/**122/**
@@ -130,7 +145,7 @@ int frwr_mr_init(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr *mr) · 22 unmodified lines
130 145
131 mr->mr_xprt = r_xprt;146 mr->mr_xprt = r_xprt;
132 mr->frwr.fr_mr = frmr;147 mr->frwr.fr_mr = frmr;
133 mr->mr_dir = DMA_NONE;148 mr->mr_device = NULL;
134 INIT_LIST_HEAD(&mr->mr_list);149 INIT_LIST_HEAD(&mr->mr_list);
135 init_completion(&mr->frwr.fr_linv_done);150 init_completion(&mr->frwr.fr_linv_done);
136 151
@@ -315,6 +330,7 @@ struct rpcrdma_mr_seg *frwr_map(struct rpcrdma_xprt *r_xprt, · 178 unmodified lines
315 mr->mr_dir);330 mr->mr_dir);
316 if (!dma_nents)331 if (!dma_nents)
317 goto out_dmamap_err;332 goto out_dmamap_err;
333 mr->mr_device = ep->re_id->device;
318 334
319 ibmr = mr->frwr.fr_mr;335 ibmr = mr->frwr.fr_mr;
320 n = ib_map_mr_sg(ibmr, mr->mr_sg, dma_nents, NULL, PAGE_SIZE);336 n = ib_map_mr_sg(ibmr, mr->mr_sg, dma_nents, NULL, PAGE_SIZE);
@@ -341,7 +357,6 @@ struct rpcrdma_mr_seg *frwr_map(struct rpcrdma_xprt *r_xprt, · 20 unmodified lines
341 return seg;357 return seg;
342 358
343out_dmamap_err:359out_dmamap_err:
344 mr->mr_dir = DMA_NONE;
345 trace_xprtrdma_frwr_sgerr(mr, i);360 trace_xprtrdma_frwr_sgerr(mr, i);
346 return ERR_PTR(-EIO);361 return ERR_PTR(-EIO);
347 362
@@ -363,12 +378,21 @@ static void frwr_wc_fastreg(struct ib_cq *cq, struct ib_wc *wc) · 15 unmodified lines
363 container_of(cqe, struct rpcrdma_frwr, fr_cqe);378 container_of(cqe, struct rpcrdma_frwr, fr_cqe);
364 379
365 /* WARNING: Only wr_cqe and status are reliable at this point */380 /* WARNING: Only wr_cqe and status are reliable at this point */
366 trace_xprtrdma_wc_fastreg(wc, frwr);381 trace_xprtrdma_wc_fastreg(wc, &frwr->fr_cid);
367 /* The MR will get recycled when the associated req is retransmitted */382 /* The MR will get recycled when the associated req is retransmitted */
368 383
369 rpcrdma_flush_disconnect(cq->cq_context, wc);384 rpcrdma_flush_disconnect(cq->cq_context, wc);
370}385}
371 386
387static void frwr_cid_init(struct rpcrdma_ep *ep,
388 struct rpcrdma_frwr *frwr)
389{
390 struct rpc_rdma_cid *cid = &frwr->fr_cid;
391
392 cid->ci_queue_id = ep->re_attr.send_cq->res.id;
393 cid->ci_completion_id = frwr->fr_mr->res.id;
394}
395
372/**396/**
373 * frwr_send - post Send WRs containing the RPC Call message397 * frwr_send - post Send WRs containing the RPC Call message
374 * @r_xprt: controlling transport instance398 * @r_xprt: controlling transport instance
@@ -385,6 +409,7 @@ static void frwr_wc_fastreg(struct ib_cq *cq, struct ib_wc *wc) · 10 unmodified lines
385 */409 */
386int frwr_send(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)410int frwr_send(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)
387{411{
412 struct rpcrdma_ep *ep = r_xprt->rx_ep;
388 struct ib_send_wr *post_wr;413 struct ib_send_wr *post_wr;
389 struct rpcrdma_mr *mr;414 struct rpcrdma_mr *mr;
390 415
@@ -395,6 +420,7 @@ int frwr_send(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) · 4 unmodified lines
395 frwr = &mr->frwr;420 frwr = &mr->frwr;
396 421
397 frwr->fr_cqe.done = frwr_wc_fastreg;422 frwr->fr_cqe.done = frwr_wc_fastreg;
423 frwr_cid_init(ep, frwr);
398 frwr->fr_regwr.wr.next = post_wr;424 frwr->fr_regwr.wr.next = post_wr;
399 frwr->fr_regwr.wr.wr_cqe = &frwr->fr_cqe;425 frwr->fr_regwr.wr.wr_cqe = &frwr->fr_cqe;
400 frwr->fr_regwr.wr.num_sge = 0;426 frwr->fr_regwr.wr.num_sge = 0;
@@ -404,7 +430,7 @@ int frwr_send(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) · 3 unmodified lines
404 post_wr = &frwr->fr_regwr.wr;430 post_wr = &frwr->fr_regwr.wr;
405 }431 }
406 432
407 return ib_post_send(r_xprt->rx_ep->re_id->qp, post_wr, NULL);433 return ib_post_send(ep->re_id->qp, post_wr, NULL);
408}434}
409 435
410/**436/**
@@ -420,18 +446,17 @@ void frwr_reminv(struct rpcrdma_rep *rep, struct list_head *mrs) · 9 unmodified lines
420 list_for_each_entry(mr, mrs, mr_list)446 list_for_each_entry(mr, mrs, mr_list)
421 if (mr->mr_handle == rep->rr_inv_rkey) {447 if (mr->mr_handle == rep->rr_inv_rkey) {
422 list_del_init(&mr->mr_list);448 list_del_init(&mr->mr_list);
423 trace_xprtrdma_mr_reminv(mr);449 frwr_mr_put(mr);
424 rpcrdma_mr_put(mr);
425 break; /* only one invalidated MR per RPC */450 break; /* only one invalidated MR per RPC */
426 }451 }
427}452}
428 453
429static void __frwr_release_mr(struct ib_wc *wc, struct rpcrdma_mr *mr)454static void frwr_mr_done(struct ib_wc *wc, struct rpcrdma_mr *mr)
430{455{
431 if (wc->status != IB_WC_SUCCESS)456 if (wc->status != IB_WC_SUCCESS)
432 frwr_mr_recycle(mr);457 frwr_mr_recycle(mr);
433 else458 else
434 rpcrdma_mr_put(mr);459 frwr_mr_put(mr);
435}460}
436 461
437/**462/**
@@ -448,8 +473,8 @@ static void frwr_wc_localinv(struct ib_cq *cq, struct ib_wc *wc) · 10 unmodified lines
448 struct rpcrdma_mr *mr = container_of(frwr, struct rpcrdma_mr, frwr);473 struct rpcrdma_mr *mr = container_of(frwr, struct rpcrdma_mr, frwr);
449 474
450 /* WARNING: Only wr_cqe and status are reliable at this point */475 /* WARNING: Only wr_cqe and status are reliable at this point */
451 trace_xprtrdma_wc_li(wc, frwr);476 trace_xprtrdma_wc_li(wc, &frwr->fr_cid);
452 __frwr_release_mr(wc, mr);477 frwr_mr_done(wc, mr);
453 478
454 rpcrdma_flush_disconnect(cq->cq_context, wc);479 rpcrdma_flush_disconnect(cq->cq_context, wc);
455}480}
@@ -469,8 +494,8 @@ static void frwr_wc_localinv_wake(struct ib_cq *cq, struct ib_wc *wc) · 13 unmodified lines
469 struct rpcrdma_mr *mr = container_of(frwr, struct rpcrdma_mr, frwr);494 struct rpcrdma_mr *mr = container_of(frwr, struct rpcrdma_mr, frwr);
470 495
471 /* WARNING: Only wr_cqe and status are reliable at this point */496 /* WARNING: Only wr_cqe and status are reliable at this point */
472 trace_xprtrdma_wc_li_wake(wc, frwr);497 trace_xprtrdma_wc_li_wake(wc, &frwr->fr_cid);
473 __frwr_release_mr(wc, mr);498 frwr_mr_done(wc, mr);
474 complete(&frwr->fr_linv_done);499 complete(&frwr->fr_linv_done);
475 500
476 rpcrdma_flush_disconnect(cq->cq_context, wc);501 rpcrdma_flush_disconnect(cq->cq_context, wc);
@@ -490,6 +515,7 @@ static void frwr_wc_localinv_wake(struct ib_cq *cq, struct ib_wc *wc) · 13 unmodified lines
490void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)515void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)
491{516{
492 struct ib_send_wr *first, **prev, *last;517 struct ib_send_wr *first, **prev, *last;
518 struct rpcrdma_ep *ep = r_xprt->rx_ep;
493 const struct ib_send_wr *bad_wr;519 const struct ib_send_wr *bad_wr;
494 struct rpcrdma_frwr *frwr;520 struct rpcrdma_frwr *frwr;
495 struct rpcrdma_mr *mr;521 struct rpcrdma_mr *mr;
@@ -509,6 +535,7 @@ void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) · 13 unmodified lines
509 535
510 frwr = &mr->frwr;536 frwr = &mr->frwr;
511 frwr->fr_cqe.done = frwr_wc_localinv;537 frwr->fr_cqe.done = frwr_wc_localinv;
538 frwr_cid_init(ep, frwr);
512 last = &frwr->fr_invwr;539 last = &frwr->fr_invwr;
513 last->next = NULL;540 last->next = NULL;
514 last->wr_cqe = &frwr->fr_cqe;541 last->wr_cqe = &frwr->fr_cqe;
@@ -534,7 +561,7 @@ void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) · 19 unmodified lines
534 * unless re_id->qp is a valid pointer.561 * unless re_id->qp is a valid pointer.
535 */562 */
536 bad_wr = NULL;563 bad_wr = NULL;
537 rc = ib_post_send(r_xprt->rx_ep->re_id->qp, first, &bad_wr);564 rc = ib_post_send(ep->re_id->qp, first, &bad_wr);
538 565
539 /* The final LOCAL_INV WR in the chain is supposed to566 /* The final LOCAL_INV WR in the chain is supposed to
540 * do the wake. If it was never posted, the wake will567 * do the wake. If it was never posted, the wake will
@@ -547,7 +574,7 @@ void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) · 6 unmodified lines
547 574
548 /* Recycle MRs in the LOCAL_INV chain that did not get posted.575 /* Recycle MRs in the LOCAL_INV chain that did not get posted.
549 */576 */
550 trace_xprtrdma_post_linv(req, rc);577 trace_xprtrdma_post_linv_err(req, rc);
551 while (bad_wr) {578 while (bad_wr) {
552 frwr = container_of(bad_wr, struct rpcrdma_frwr,579 frwr = container_of(bad_wr, struct rpcrdma_frwr,
553 fr_invwr);580 fr_invwr);
@@ -574,10 +601,10 @@ static void frwr_wc_localinv_done(struct ib_cq *cq, struct ib_wc *wc) · 20 unmodified lines
574 struct rpcrdma_rep *rep = mr->mr_req->rl_reply;601 struct rpcrdma_rep *rep = mr->mr_req->rl_reply;
575 602
576 /* WARNING: Only wr_cqe and status are reliable at this point */603 /* WARNING: Only wr_cqe and status are reliable at this point */
577 trace_xprtrdma_wc_li_done(wc, frwr);604 trace_xprtrdma_wc_li_done(wc, &frwr->fr_cid);
578 __frwr_release_mr(wc, mr);605 frwr_mr_done(wc, mr);
579 606
580 /* Ensure @rep is generated before __frwr_release_mr */607 /* Ensure @rep is generated before frwr_mr_done */
581 smp_rmb();608 smp_rmb();
582 rpcrdma_complete_rqst(rep);609 rpcrdma_complete_rqst(rep);
583 610
@@ -597,6 +624,7 @@ static void frwr_wc_localinv_done(struct ib_cq *cq, struct ib_wc *wc) · 13 unmodified lines
597void frwr_unmap_async(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)624void frwr_unmap_async(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)
598{625{
599 struct ib_send_wr *first, *last, **prev;626 struct ib_send_wr *first, *last, **prev;
627 struct rpcrdma_ep *ep = r_xprt->rx_ep;
600 const struct ib_send_wr *bad_wr;628 const struct ib_send_wr *bad_wr;
601 struct rpcrdma_frwr *frwr;629 struct rpcrdma_frwr *frwr;
602 struct rpcrdma_mr *mr;630 struct rpcrdma_mr *mr;
@@ -614,6 +642,7 @@ void frwr_unmap_async(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) · 11 unmodified lines
614 642
615 frwr = &mr->frwr;643 frwr = &mr->frwr;
616 frwr->fr_cqe.done = frwr_wc_localinv;644 frwr->fr_cqe.done = frwr_wc_localinv;
645 frwr_cid_init(ep, frwr);
617 last = &frwr->fr_invwr;646 last = &frwr->fr_invwr;
618 last->next = NULL;647 last->next = NULL;
619 last->wr_cqe = &frwr->fr_cqe;648 last->wr_cqe = &frwr->fr_cqe;
@@ -639,13 +668,13 @@ void frwr_unmap_async(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) · 19 unmodified lines
639 * unless re_id->qp is a valid pointer.668 * unless re_id->qp is a valid pointer.
640 */669 */
641 bad_wr = NULL;670 bad_wr = NULL;
642 rc = ib_post_send(r_xprt->rx_ep->re_id->qp, first, &bad_wr);671 rc = ib_post_send(ep->re_id->qp, first, &bad_wr);
643 if (!rc)672 if (!rc)
644 return;673 return;
645 674
646 /* Recycle MRs in the LOCAL_INV chain that did not get posted.675 /* Recycle MRs in the LOCAL_INV chain that did not get posted.
647 */676 */
648 trace_xprtrdma_post_linv(req, rc);677 trace_xprtrdma_post_linv_err(req, rc);
649 while (bad_wr) {678 while (bad_wr) {
650 frwr = container_of(bad_wr, struct rpcrdma_frwr, fr_invwr);679 frwr = container_of(bad_wr, struct rpcrdma_frwr, fr_invwr);
651 mr = container_of(frwr, struct rpcrdma_mr, frwr);680 mr = container_of(frwr, struct rpcrdma_mr, frwr);
net/sunrpc/xprtrdma/rpc_rdma.c+11−21
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2/*2/*
3 * Copyright (c) 2014-2017 Oracle. All rights reserved.3 * Copyright (c) 2014-2020, Oracle and/or its affiliates.
4 * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.4 * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
5 *5 *
6 * This software is available to you under a choice of one of two6 * This software is available to you under a choice of one of two
@@ -331,7 +331,6 @@ static struct rpcrdma_mr_seg *rpcrdma_mr_prepare(struct rpcrdma_xprt *r_xprt, · 324 unmodified lines
331 *mr = rpcrdma_mr_get(r_xprt);331 *mr = rpcrdma_mr_get(r_xprt);
332 if (!*mr)332 if (!*mr)
333 goto out_getmr_err;333 goto out_getmr_err;
334 trace_xprtrdma_mr_get(req);
335 (*mr)->mr_req = req;334 (*mr)->mr_req = req;
336 }335 }
337 336
@@ -339,7 +338,7 @@ static struct rpcrdma_mr_seg *rpcrdma_mr_prepare(struct rpcrdma_xprt *r_xprt, · 1 unmodified line
339 return frwr_map(r_xprt, seg, nsegs, writing, req->rl_slot.rq_xid, *mr);338 return frwr_map(r_xprt, seg, nsegs, writing, req->rl_slot.rq_xid, *mr);
340 339
341out_getmr_err:340out_getmr_err:
342 trace_xprtrdma_nomrs(req);341 trace_xprtrdma_nomrs_err(r_xprt, req);
343 xprt_wait_for_buffer_space(&r_xprt->rx_xprt);342 xprt_wait_for_buffer_space(&r_xprt->rx_xprt);
344 rpcrdma_mrs_refresh(r_xprt);343 rpcrdma_mrs_refresh(r_xprt);
345 return ERR_PTR(-EAGAIN);344 return ERR_PTR(-EAGAIN);
@@ -1344,20 +1343,13 @@ rpcrdma_decode_error(struct rpcrdma_xprt *r_xprt, struct rpcrdma_rep *rep, · 998 unmodified lines
1344 p = xdr_inline_decode(xdr, 2 * sizeof(*p));1343 p = xdr_inline_decode(xdr, 2 * sizeof(*p));
1345 if (!p)1344 if (!p)
1346 break;1345 break;
1347 dprintk("RPC: %s: server reports "1346 trace_xprtrdma_err_vers(rqst, p, p + 1);
1348 "version error (%u-%u), xid %08x\n", __func__,
1349 be32_to_cpup(p), be32_to_cpu(*(p + 1)),
1350 be32_to_cpu(rep->rr_xid));
1351 break;1347 break;
1352 case err_chunk:1348 case err_chunk:
1353 dprintk("RPC: %s: server reports "1349 trace_xprtrdma_err_chunk(rqst);
1354 "header decoding error, xid %08x\n", __func__,
1355 be32_to_cpu(rep->rr_xid));
1356 break;1350 break;
1357 default:1351 default:
1358 dprintk("RPC: %s: server reports "1352 trace_xprtrdma_err_unrecognized(rqst, p);
1359 "unrecognized error %d, xid %08x\n", __func__,
1360 be32_to_cpup(p), be32_to_cpu(rep->rr_xid));
1361 }1353 }
1362 1354
1363 return -EIO;1355 return -EIO;
@@ -1398,7 +1390,7 @@ out: · 34 unmodified lines
1398 return;1390 return;
1399 1391
1400out_badheader:1392out_badheader:
1401 trace_xprtrdma_reply_hdr(rep);1393 trace_xprtrdma_reply_hdr_err(rep);
1402 r_xprt->rx_stats.bad_reply_count++;1394 r_xprt->rx_stats.bad_reply_count++;
1403 rqst->rq_task->tk_status = status;1395 rqst->rq_task->tk_status = status;
1404 status = 0;1396 status = 0;
@@ -1472,14 +1464,12 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *rep) · 67 unmodified lines
1472 rpcrdma_post_recvs(r_xprt, false);1464 rpcrdma_post_recvs(r_xprt, false);
1473 1465
1474 req = rpcr_to_rdmar(rqst);1466 req = rpcr_to_rdmar(rqst);
1475 if (req->rl_reply) {1467 if (unlikely(req->rl_reply))
1476 trace_xprtrdma_leaked_rep(rqst, req->rl_reply);
1477 rpcrdma_recv_buffer_put(req->rl_reply);1468 rpcrdma_recv_buffer_put(req->rl_reply);
1478 }
1479 req->rl_reply = rep;1469 req->rl_reply = rep;
1480 rep->rr_rqst = rqst;1470 rep->rr_rqst = rqst;
1481 1471
1482 trace_xprtrdma_reply(rqst->rq_task, rep, req, credits);1472 trace_xprtrdma_reply(rqst->rq_task, rep, credits);
1483 1473
1484 if (rep->rr_wc_flags & IB_WC_WITH_INVALIDATE)1474 if (rep->rr_wc_flags & IB_WC_WITH_INVALIDATE)
1485 frwr_reminv(rep, &req->rl_registered);1475 frwr_reminv(rep, &req->rl_registered);
@@ -1491,16 +1481,16 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *rep) · 5 unmodified lines
1491 return;1481 return;
1492 1482
1493out_badversion:1483out_badversion:
1494 trace_xprtrdma_reply_vers(rep);1484 trace_xprtrdma_reply_vers_err(rep);
1495 goto out;1485 goto out;
1496 1486
1497out_norqst:1487out_norqst:
1498 spin_unlock(&xprt->queue_lock);1488 spin_unlock(&xprt->queue_lock);
1499 trace_xprtrdma_reply_rqst(rep);1489 trace_xprtrdma_reply_rqst_err(rep);
1500 goto out;1490 goto out;
1501 1491
1502out_shortreply:1492out_shortreply:
1503 trace_xprtrdma_reply_short(rep);1493 trace_xprtrdma_reply_short_err(rep);
1504 1494
1505out:1495out:
1506 rpcrdma_recv_buffer_put(rep);1496 rpcrdma_recv_buffer_put(rep);
net/sunrpc/xprtrdma/transport.c+4−3
@@ -599,11 +599,12 @@ static void · 598 unmodified lines
599xprt_rdma_free(struct rpc_task *task)599xprt_rdma_free(struct rpc_task *task)
600{600{
601 struct rpc_rqst *rqst = task->tk_rqstp;601 struct rpc_rqst *rqst = task->tk_rqstp;
602 struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(rqst->rq_xprt);
603 struct rpcrdma_req *req = rpcr_to_rdmar(rqst);602 struct rpcrdma_req *req = rpcr_to_rdmar(rqst);
604 603
605 if (!list_empty(&req->rl_registered))604 if (unlikely(!list_empty(&req->rl_registered))) {
606 frwr_unmap_sync(r_xprt, req);605 trace_xprtrdma_mrs_zap(task);
606 frwr_unmap_sync(rpcx_to_rdmax(rqst->rq_xprt), req);
607 }
607 608
608 /* XXX: If the RPC is completing because of a signal and609 /* XXX: If the RPC is completing because of a signal and
609 * not because a reply was received, we ought to ensure610 * not because a reply was received, we ought to ensure
net/sunrpc/xprtrdma/verbs.c+9−21
@@ -167,7 +167,7 @@ static void rpcrdma_wc_send(struct ib_cq *cq, struct ib_wc *wc) · 166 unmodified lines
167 struct rpcrdma_xprt *r_xprt = cq->cq_context;167 struct rpcrdma_xprt *r_xprt = cq->cq_context;
168 168
169 /* WARNING: Only wr_cqe and status are reliable at this point */169 /* WARNING: Only wr_cqe and status are reliable at this point */
170 trace_xprtrdma_wc_send(sc, wc);170 trace_xprtrdma_wc_send(wc, &sc->sc_cid);
171 rpcrdma_sendctx_put_locked(r_xprt, sc);171 rpcrdma_sendctx_put_locked(r_xprt, sc);
172 rpcrdma_flush_disconnect(r_xprt, wc);172 rpcrdma_flush_disconnect(r_xprt, wc);
173}173}
@@ -186,7 +186,7 @@ static void rpcrdma_wc_receive(struct ib_cq *cq, struct ib_wc *wc) · 12 unmodified lines
186 struct rpcrdma_xprt *r_xprt = cq->cq_context;186 struct rpcrdma_xprt *r_xprt = cq->cq_context;
187 187
188 /* WARNING: Only wr_cqe and status are reliable at this point */188 /* WARNING: Only wr_cqe and status are reliable at this point */
189 trace_xprtrdma_wc_receive(wc);189 trace_xprtrdma_wc_receive(wc, &rep->rr_cid);
190 --r_xprt->rx_ep->re_receive_count;190 --r_xprt->rx_ep->re_receive_count;
191 if (wc->status != IB_WC_SUCCESS)191 if (wc->status != IB_WC_SUCCESS)
192 goto out_flushed;192 goto out_flushed;
@@ -643,6 +643,9 @@ static struct rpcrdma_sendctx *rpcrdma_sendctx_create(struct rpcrdma_ep *ep) · 450 unmodified lines
643 return NULL;643 return NULL;
644 644
645 sc->sc_cqe.done = rpcrdma_wc_send;645 sc->sc_cqe.done = rpcrdma_wc_send;
646 sc->sc_cid.ci_queue_id = ep->re_attr.send_cq->res.id;
647 sc->sc_cid.ci_completion_id =
648 atomic_inc_return(&ep->re_completion_ids);
646 return sc;649 return sc;
647}650}
648 651
@@ -972,6 +975,9 @@ struct rpcrdma_rep *rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt, · 323 unmodified lines
972 if (!rpcrdma_regbuf_dma_map(r_xprt, rep->rr_rdmabuf))975 if (!rpcrdma_regbuf_dma_map(r_xprt, rep->rr_rdmabuf))
973 goto out_free_regbuf;976 goto out_free_regbuf;
974 977
978 rep->rr_cid.ci_completion_id =
979 atomic_inc_return(&r_xprt->rx_ep->re_completion_ids);
980
975 xdr_buf_init(&rep->rr_hdrbuf, rdmab_data(rep->rr_rdmabuf),981 xdr_buf_init(&rep->rr_hdrbuf, rdmab_data(rep->rr_rdmabuf),
976 rdmab_length(rep->rr_rdmabuf));982 rdmab_length(rep->rr_rdmabuf));
977 rep->rr_cqe.done = rpcrdma_wc_receive;983 rep->rr_cqe.done = rpcrdma_wc_receive;
@@ -1178,25 +1184,6 @@ rpcrdma_mr_get(struct rpcrdma_xprt *r_xprt) · 200 unmodified lines
1178 return mr;1184 return mr;
1179}1185}
1180 1186
1181/**
1182 * rpcrdma_mr_put - DMA unmap an MR and release it
1183 * @mr: MR to release
1184 *
1185 */
1186void rpcrdma_mr_put(struct rpcrdma_mr *mr)
1187{
1188 struct rpcrdma_xprt *r_xprt = mr->mr_xprt;
1189
1190 if (mr->mr_dir != DMA_NONE) {
1191 trace_xprtrdma_mr_unmap(mr);
1192 ib_dma_unmap_sg(r_xprt->rx_ep->re_id->device,
1193 mr->mr_sg, mr->mr_nents, mr->mr_dir);
1194 mr->mr_dir = DMA_NONE;
1195 }
1196
1197 rpcrdma_mr_push(mr, &mr->mr_req->rl_free_mrs);
1198}
1199
1200/**1187/**
1201 * rpcrdma_buffer_get - Get a request buffer1188 * rpcrdma_buffer_get - Get a request buffer
1202 * @buffers: Buffer pool from which to obtain a buffer1189 * @buffers: Buffer pool from which to obtain a buffer
@@ -1411,6 +1398,7 @@ void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp) · 208 unmodified lines
1411 if (!rep)1398 if (!rep)
1412 break;1399 break;
1413 1400
1401 rep->rr_cid.ci_queue_id = ep->re_attr.recv_cq->res.id;
1414 trace_xprtrdma_post_recv(rep);1402 trace_xprtrdma_post_recv(rep);
1415 rep->rr_recv_wr.next = wr;1403 rep->rr_recv_wr.next = wr;
1416 wr = &rep->rr_recv_wr;1404 wr = &rep->rr_recv_wr;
net/sunrpc/xprtrdma/xprt_rdma.h+8−1
@@ -53,6 +53,7 @@ · 52 unmodified lines
53#include <rdma/ib_verbs.h> /* RDMA verbs api */53#include <rdma/ib_verbs.h> /* RDMA verbs api */
54 54
55#include <linux/sunrpc/clnt.h> /* rpc_xprt */55#include <linux/sunrpc/clnt.h> /* rpc_xprt */
56#include <linux/sunrpc/rpc_rdma_cid.h> /* completion IDs */
56#include <linux/sunrpc/rpc_rdma.h> /* RPC/RDMA protocol */57#include <linux/sunrpc/rpc_rdma.h> /* RPC/RDMA protocol */
57#include <linux/sunrpc/xprtrdma.h> /* xprt parameters */58#include <linux/sunrpc/xprtrdma.h> /* xprt parameters */
58 59
@@ -93,6 +94,8 @@ struct rpcrdma_ep { · 34 unmodified lines
93 unsigned int re_max_requests; /* depends on device */94 unsigned int re_max_requests; /* depends on device */
94 unsigned int re_inline_send; /* negotiated */95 unsigned int re_inline_send; /* negotiated */
95 unsigned int re_inline_recv; /* negotiated */96 unsigned int re_inline_recv; /* negotiated */
97
98 atomic_t re_completion_ids;
96};99};
97 100
98/* Pre-allocate extra Work Requests for handling backward receives101/* Pre-allocate extra Work Requests for handling backward receives
@@ -180,6 +183,8 @@ enum { · 81 unmodified lines
180 183
181struct rpcrdma_rep {184struct rpcrdma_rep {
182 struct ib_cqe rr_cqe;185 struct ib_cqe rr_cqe;
186 struct rpc_rdma_cid rr_cid;
187
183 __be32 rr_xid;188 __be32 rr_xid;
184 __be32 rr_vers;189 __be32 rr_vers;
185 __be32 rr_proc;190 __be32 rr_proc;
@@ -211,6 +216,7 @@ enum { · 25 unmodified lines
211struct rpcrdma_req;216struct rpcrdma_req;
212struct rpcrdma_sendctx {217struct rpcrdma_sendctx {
213 struct ib_cqe sc_cqe;218 struct ib_cqe sc_cqe;
219 struct rpc_rdma_cid sc_cid;
214 struct rpcrdma_req *sc_req;220 struct rpcrdma_req *sc_req;
215 unsigned int sc_unmap_count;221 unsigned int sc_unmap_count;
216 struct ib_sge sc_sges[];222 struct ib_sge sc_sges[];
@@ -225,6 +231,7 @@ struct rpcrdma_sendctx { · 8 unmodified lines
225struct rpcrdma_frwr {231struct rpcrdma_frwr {
226 struct ib_mr *fr_mr;232 struct ib_mr *fr_mr;
227 struct ib_cqe fr_cqe;233 struct ib_cqe fr_cqe;
234 struct rpc_rdma_cid fr_cid;
228 struct completion fr_linv_done;235 struct completion fr_linv_done;
229 union {236 union {
230 struct ib_reg_wr fr_regwr;237 struct ib_reg_wr fr_regwr;
@@ -236,6 +243,7 @@ struct rpcrdma_req; · 5 unmodified lines
236struct rpcrdma_mr {243struct rpcrdma_mr {
237 struct list_head mr_list;244 struct list_head mr_list;
238 struct rpcrdma_req *mr_req;245 struct rpcrdma_req *mr_req;
246 struct ib_device *mr_device;
239 struct scatterlist *mr_sg;247 struct scatterlist *mr_sg;
240 int mr_nents;248 int mr_nents;
241 enum dma_data_direction mr_dir;249 enum dma_data_direction mr_dir;
@@ -466,7 +474,6 @@ void rpcrdma_buffer_destroy(struct rpcrdma_buffer *); · 224 unmodified lines
466struct rpcrdma_sendctx *rpcrdma_sendctx_get_locked(struct rpcrdma_xprt *r_xprt);474struct rpcrdma_sendctx *rpcrdma_sendctx_get_locked(struct rpcrdma_xprt *r_xprt);
467 475
468struct rpcrdma_mr *rpcrdma_mr_get(struct rpcrdma_xprt *r_xprt);476struct rpcrdma_mr *rpcrdma_mr_get(struct rpcrdma_xprt *r_xprt);
469void rpcrdma_mr_put(struct rpcrdma_mr *mr);
470void rpcrdma_mrs_refresh(struct rpcrdma_xprt *r_xprt);477void rpcrdma_mrs_refresh(struct rpcrdma_xprt *r_xprt);
471 478
472struct rpcrdma_req *rpcrdma_buffer_get(struct rpcrdma_buffer *);479struct rpcrdma_req *rpcrdma_buffer_get(struct rpcrdma_buffer *);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t