History

RDMA/core: Provide RDMA DIM support for ULPs

Added the interface in the infiniband driver that applies the rdma_dim adaptive moderation. There is now a special function for allocating an ib_cq that uses rdma_dim. Performance improvement (ConnectX-5 100GbE, x86) running FIO benchmark over NVMf between two equal end-hosts with 56 cores across a Mellanox switch using null_blk device: READS without DIM: blk size | BW | IOPS | 99th percentile latency | 99.99th latency 512B | 3.8GiB/s | 7.7M | 1401 usec | 2442 usec 4k | 7.0GiB/s | 1.8M | 4817 usec | 6587 usec 64k | 10.7GiB/s| 175k | 9896 usec | 10028 usec IO WRITES without DIM: blk size | BW | IOPS | 99th percentile latency | 99.99th latency 512B | 3.6GiB/s | 7.5M | 1434 usec | 2474 usec 4k | 6.3GiB/s | 1.6M | 938 usec | 1221 usec 64k | 10.7GiB/s| 175k | 8979 usec | 12780 usec IO READS with DIM: blk size | BW | IOPS | 99th percentile latency | 99.99th latency 512B | 4GiB/s | 8.2M | 816 usec | 889 usec 4k | 10.1GiB/s| 2.65M| 3359 usec | 5080 usec 64k | 10.7GiB/s| 175k | 9896 usec | 10028 usec IO WRITES with DIM: blk size | BW | IOPS | 99th percentile latency | 99.99th latency 512B | 3.9GiB/s | 8.1M | 799 usec | 922 usec 4k | 9.6GiB/s | 2.5M | 717 usec | 1004 usec 64k | 10.7GiB/s| 176k | 8586 usec | 12256 usec The rdma_dim algorithm was designed to measure the effectiveness of moderation on the flow in a general way and thus should be appropriate for all RDMA storage protocols. rdma_dim is configured to be the default option based on performance improvement seen after extensive tests.

Yamin Friedman committed 7 years ago

Commit da662971 parent f4915452 files · +49 −0Browse files

Unified · Split

2 files
  1. drivers/infiniband/core/cq.c +45 −0
  2. include/rdma/ib_verbs.h +4 −0
drivers/infiniband/core/cq.c+45−0
@@ -18,6 +18,40 @@ · 17 unmodified lines
18#define IB_POLL_FLAGS \18#define IB_POLL_FLAGS \
19 (IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS)19 (IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS)
20 20
21static void ib_cq_rdma_dim_work(struct work_struct *w)
22{
23 struct dim *dim = container_of(w, struct dim, work);
24 struct ib_cq *cq = dim->priv;
25
26 u16 usec = rdma_dim_prof[dim->profile_ix].usec;
27 u16 comps = rdma_dim_prof[dim->profile_ix].comps;
28
29 dim->state = DIM_START_MEASURE;
30
31 cq->device->ops.modify_cq(cq, comps, usec);
32}
33
34static void rdma_dim_init(struct ib_cq *cq)
35{
36 struct dim *dim;
37
38 if (!cq->device->ops.modify_cq || !cq->device->use_cq_dim ||
39 cq->poll_ctx == IB_POLL_DIRECT)
40 return;
41
42 dim = kzalloc(sizeof(struct dim), GFP_KERNEL);
43 if (!dim)
44 return;
45
46 dim->state = DIM_START_MEASURE;
47 dim->tune_state = DIM_GOING_RIGHT;
48 dim->profile_ix = RDMA_DIM_START_PROFILE;
49 dim->priv = cq;
50 cq->dim = dim;
51
52 INIT_WORK(&dim->work, ib_cq_rdma_dim_work);
53}
54
21static int __ib_process_cq(struct ib_cq *cq, int budget, struct ib_wc *wcs,55static int __ib_process_cq(struct ib_cq *cq, int budget, struct ib_wc *wcs,
22 int batch)56 int batch)
23{57{
@@ -78,6 +112,7 @@ static void ib_cq_completion_direct(struct ib_cq *cq, void *private) · 54 unmodified lines
78static int ib_poll_handler(struct irq_poll *iop, int budget)112static int ib_poll_handler(struct irq_poll *iop, int budget)
79{113{
80 struct ib_cq *cq = container_of(iop, struct ib_cq, iop);114 struct ib_cq *cq = container_of(iop, struct ib_cq, iop);
115 struct dim *dim = cq->dim;
81 int completed;116 int completed;
82 117
83 completed = __ib_process_cq(cq, budget, cq->wc, IB_POLL_BATCH);118 completed = __ib_process_cq(cq, budget, cq->wc, IB_POLL_BATCH);
@@ -87,6 +122,9 @@ static int ib_poll_handler(struct irq_poll *iop, int budget) · 3 unmodified lines
87 irq_poll_sched(&cq->iop);122 irq_poll_sched(&cq->iop);
88 }123 }
89 124
125 if (dim)
126 rdma_dim(dim, completed);
127
90 return completed;128 return completed;
91}129}
92 130
@@ -105,6 +143,8 @@ static void ib_cq_poll_work(struct work_struct *work) · 12 unmodified lines
105 if (completed >= IB_POLL_BUDGET_WORKQUEUE ||143 if (completed >= IB_POLL_BUDGET_WORKQUEUE ||
106 ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0)144 ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0)
107 queue_work(cq->comp_wq, &cq->work);145 queue_work(cq->comp_wq, &cq->work);
146 else if (cq->dim)
147 rdma_dim(cq->dim, completed);
108}148}
109 149
110static void ib_cq_completion_workqueue(struct ib_cq *cq, void *private)150static void ib_cq_completion_workqueue(struct ib_cq *cq, void *private)
@@ -161,6 +201,8 @@ struct ib_cq *__ib_alloc_cq_user(struct ib_device *dev, void *private, · 50 unmodified lines
161 201
162 rdma_restrack_kadd(&cq->res);202 rdma_restrack_kadd(&cq->res);
163 203
204 rdma_dim_init(cq);
205
164 switch (cq->poll_ctx) {206 switch (cq->poll_ctx) {
165 case IB_POLL_DIRECT:207 case IB_POLL_DIRECT:
166 cq->comp_handler = ib_cq_completion_direct;208 cq->comp_handler = ib_cq_completion_direct;
@@ -223,6 +265,9 @@ void ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata) · 56 unmodified lines
223 265
224 rdma_restrack_del(&cq->res);266 rdma_restrack_del(&cq->res);
225 cq->device->ops.destroy_cq(cq, udata);267 cq->device->ops.destroy_cq(cq, udata);
268 if (cq->dim)
269 cancel_work_sync(&cq->dim->work);
270 kfree(cq->dim);
226 kfree(cq->wc);271 kfree(cq->wc);
227 kfree(cq);272 kfree(cq);
228}273}
include/rdma/ib_verbs.h+4−0
@@ -61,6 +61,7 @@ · 60 unmodified lines
61#include <linux/cgroup_rdma.h>61#include <linux/cgroup_rdma.h>
62#include <linux/irqflags.h>62#include <linux/irqflags.h>
63#include <linux/preempt.h>63#include <linux/preempt.h>
64#include <linux/dim.h>
64#include <uapi/rdma/ib_user_verbs.h>65#include <uapi/rdma/ib_user_verbs.h>
65#include <rdma/rdma_counter.h>66#include <rdma/rdma_counter.h>
66#include <rdma/restrack.h>67#include <rdma/restrack.h>
@@ -1509,6 +1510,7 @@ struct ib_cq { · 1442 unmodified lines
1509 struct work_struct work;1510 struct work_struct work;
1510 };1511 };
1511 struct workqueue_struct *comp_wq;1512 struct workqueue_struct *comp_wq;
1513 struct dim *dim;
1512 /*1514 /*
1513 * Implementation details of the RDMA core, don't use in drivers:1515 * Implementation details of the RDMA core, don't use in drivers:
1514 */1516 */
@@ -2576,6 +2578,8 @@ struct ib_device { · 1061 unmodified lines
2576 u16 is_switch:1;2578 u16 is_switch:1;
2577 /* Indicates kernel verbs support, should not be used in drivers */2579 /* Indicates kernel verbs support, should not be used in drivers */
2578 u16 kverbs_provider:1;2580 u16 kverbs_provider:1;
2581 /* CQ adaptive moderation (RDMA DIM) */
2582 u16 use_cq_dim:1;
2579 u8 node_type;2583 u8 node_type;
2580 u8 phys_port_cnt;2584 u8 phys_port_cnt;
2581 struct ib_device_attr attrs;2585 struct ib_device_attr attrs;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t