History

net: devmem: allow rx-page-size > PAGE_SIZE per dmabuf binding

Every devmem dmabuf binding today hands the page_pool PAGE_SIZE niovs. This caps a single RX descriptor at PAGE_SIZE, burning CPU on buffer churn for large flows. Add a bind-time netlink attribute, NETDEV_A_DMABUF_RX_PAGE_SIZE, that lets userspace request a larger niov size. The value must be a power of two >= PAGE_SIZE. The TX path is changed to always pass PAGE_SIZE. Measurements: Setup: kperf in devmem RX/TX cuda mode, 4 flows, 64 MB messages, 60s, dctcp, num-rx-queues=4, dmabuf-rx/tx-size-mb=2048, 10 runs per niov size, mlx5. CPU Util: niov net sirq % net idle % app sys % app idle % ----- ---------------- ---------------- ---------------- ---------------- 4K 62.38 +/- 8.27 33.40 +/- 7.51 54.15 +/- 10.23 43.67 +/- 10.53 16K 58.91 +/- 5.35 35.23 +/- 5.88 41.05 +/- 8.87 56.42 +/- 9.24 32K 64.12 +/- 0.68 31.09 +/- 1.48 44.54 +/- 3.51 52.63 +/- 3.65 64K 54.69 +/- 5.54 39.67 +/- 5.81 35.47 +/- 3.11 61.97 +/- 3.27 RX app sys % drops ~19% from 4K to 64K. Throughput: niov RX dev Gbps RX flow avg Gbps ----- ---------------- ----------------- 4K 300.63 +/- 53.21 75.16 +/- 13.30 16K 321.35 +/- 28.20 80.34 +/- 7.05 32K 347.63 +/- 2.20 86.91 +/- 0.55 64K 332.11 +/- 14.26 83.03 +/- 3.56 Throughput seems to increase, but the stdev is pretty wide so could just be noise. kperf support (not yet merged): https://github.com/facebookexperimental/kperf/commit/8837577f920876bce6986ec18869ac04439ebcd2

Bobby Eshleman committed 3 months ago

Commit b27a8561 parent 55d20f58 files · +78 −23Browse files

Unified · Split

8 files
  1. Documentation/netlink/specs/netdev.yaml +19 −0
  2. include/uapi/linux/netdev.h +1 −0
  3. net/core/devmem.c +22 −15
  4. net/core/devmem.h +9 −4
  5. net/core/netdev-genl-gen.c +9 −2
  6. net/core/netdev-genl-gen.h +1 −0
  7. net/core/netdev-genl.c +16 −2
  8. tools/include/uapi/linux/netdev.h +1 −0
Documentation/netlink/specs/netdev.yaml+19−0
@@ -6,6 +6,14 @@ doc: >- · 5 unmodified lines
66 netdev configuration over generic netlink.
77
88 definitions:
9+ -
10+ type: const
11+ name: page-size
12+ # Dummy value, codegen needs a number. The real value comes from
13+ # the PAGE_SIZE macro in the header below.
14+ value: 0
15+ header: asm/page.h
16+ scope: kernel
917 -
1018 type: flags
1119 name: xdp-act
@@ -598,6 +606,16 @@ attribute-sets: · 586 unmodified lines
598606 type: u32
599607 checks:
600608 min: 1
609+ -
610+ name: rx-page-size
611+ doc: |
612+ Size in bytes of each device page the NIC writes into from the bound
613+ dmabuf. Must be a power of two and >= PAGE_SIZE; defaults to
614+ PAGE_SIZE.
615+ type: u32
616+ checks:
617+ min: page-size
618+ max: u32-max
601619
602620 operations:
603621 list:
@@ -812,6 +830,7 @@ operations: · 208 unmodified lines
812830 - ifindex
813831 - fd
814832 - queues
833+ - rx-page-size
815834 reply:
816835 attributes:
817836 - id
include/uapi/linux/netdev.h+1−0
@@ -219,6 +219,7 @@ enum { · 218 unmodified lines
219219 NETDEV_A_DMABUF_QUEUES,
220220 NETDEV_A_DMABUF_FD,
221221 NETDEV_A_DMABUF_ID,
222+ NETDEV_A_DMABUF_RX_PAGE_SIZE,
222223
223224 __NETDEV_A_DMABUF_MAX,
224225 NETDEV_A_DMABUF_MAX = (__NETDEV_A_DMABUF_MAX - 1)
net/core/devmem.c+22−15
@@ -46,7 +46,7 @@ static dma_addr_t net_devmem_get_dma_addr(const struct net_iov *niov) · 45 unmodified lines
4646
4747 owner = net_devmem_iov_to_chunk_owner(niov);
4848 return owner->base_dma_addr +
49− ((dma_addr_t)net_iov_idx(niov) << PAGE_SHIFT);
49+ ((dma_addr_t)net_iov_idx(niov) << owner->binding->niov_shift);
5050 }
5151
5252 static void net_devmem_dmabuf_binding_release(struct percpu_ref *ref)
@@ -93,13 +93,14 @@ net_devmem_alloc_dmabuf(struct net_devmem_dmabuf_binding *binding) · 40 unmodified lines
9393 ssize_t offset;
9494 ssize_t index;
9595
96− dma_addr = gen_pool_alloc_owner(binding->chunk_pool, PAGE_SIZE,
96+ dma_addr = gen_pool_alloc_owner(binding->chunk_pool,
97+ 1UL << binding->niov_shift,
9798 (void **)&owner);
9899 if (!dma_addr)
99100 return NULL;
100101
101102 offset = dma_addr - owner->base_dma_addr;
102− index = offset / PAGE_SIZE;
103+ index = offset >> binding->niov_shift;
103104 niov = &owner->area.niovs[index];
104105
105106 niov->desc.pp_magic = 0;
@@ -113,12 +114,13 @@ void net_devmem_free_dmabuf(struct net_iov *niov) · 7 unmodified lines
113114 {
114115 struct net_devmem_dmabuf_binding *binding = net_devmem_iov_binding(niov);
115116 unsigned long dma_addr = net_devmem_get_dma_addr(niov);
117+ size_t niov_size = 1UL << binding->niov_shift;
116118
117119 if (WARN_ON(!gen_pool_has_addr(binding->chunk_pool, dma_addr,
118− PAGE_SIZE)))
120+ niov_size)))
119121 return;
120122
121− gen_pool_free(binding->chunk_pool, dma_addr, PAGE_SIZE);
123+ gen_pool_free(binding->chunk_pool, dma_addr, niov_size);
122124 }
123125
124126 void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding)
@@ -163,6 +165,9 @@ int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx, · 38 unmodified lines
163165 u32 xa_idx;
164166 int err;
165167
168+ if (binding->niov_shift != PAGE_SHIFT)
169+ mp_params.rx_page_size = 1U << binding->niov_shift;
170+
166171 err = netif_mp_open_rxq(dev, rxq_idx, &mp_params, extack);
167172 if (err)
168173 return err;
@@ -184,10 +189,12 @@ struct net_devmem_dmabuf_binding * · 15 unmodified lines
184189 net_devmem_bind_dmabuf(struct net_device *dev, void *vdev,
185190 struct device *dma_dev,
186191 enum dma_data_direction direction,
187− unsigned int dmabuf_fd, struct netdev_nl_sock *priv,
192+ unsigned int dmabuf_fd, unsigned int niov_shift,
193+ struct netdev_nl_sock *priv,
188194 struct netlink_ext_ack *extack)
189195 {
190196 struct net_devmem_dmabuf_binding *binding;
197+ size_t niov_size = 1UL << niov_shift;
191198 static u32 id_alloc_next;
192199 struct scatterlist *sg;
193200 struct dma_buf *dmabuf;
@@ -213,6 +220,7 @@ net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, · 19 unmodified lines
213220
214221 binding->dev = dev;
215222 binding->vdev = vdev;
223+ binding->niov_shift = niov_shift;
216224 xa_init_flags(&binding->bound_rxqs, XA_FLAGS_ALLOC);
217225
218226 err = percpu_ref_init(&binding->ref,
@@ -255,11 +263,7 @@ net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, · 36 unmodified lines
255263 }
256264 }
257265
258− /* For simplicity we expect to make PAGE_SIZE allocations, but the
259− * binding can be much more flexible than that. We may be able to
260− * allocate MTU sized chunks here. Leave that for future work...
261− */
262− binding->chunk_pool = gen_pool_create(PAGE_SHIFT,
266+ binding->chunk_pool = gen_pool_create(niov_shift,
263267 dev_to_node(&dev->dev));
264268 if (!binding->chunk_pool) {
265269 err = -ENOMEM;
@@ -273,9 +277,12 @@ net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, · 7 unmodified lines
273277 size_t len = sg_dma_len(sg);
274278 struct net_iov *niov;
275279
276− if (!IS_ALIGNED(len, PAGE_SIZE)) {
280+ if (!IS_ALIGNED(dma_addr, niov_size) ||
281+ !IS_ALIGNED(len, niov_size)) {
277282 err = -EINVAL;
278− NL_SET_ERR_MSG(extack, "dma-buf SG length must be PAGE_SIZE aligned");
283+ NL_SET_ERR_MSG_FMT(extack,
284+ "dmabuf sg entry (addr=%pad, len=%zu) not aligned to niov size %zu",
285+ &dma_addr, len, niov_size);
279286 goto err_free_chunks;
280287 }
281288
@@ -288,7 +295,7 @@ net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, · 6 unmodified lines
288295
289296 owner->area.base_virtual = virtual;
290297 owner->base_dma_addr = dma_addr;
291− owner->area.num_niovs = len / PAGE_SIZE;
298+ owner->area.num_niovs = len >> niov_shift;
292299 owner->binding = binding;
293300
294301 err = gen_pool_add_owner(binding->chunk_pool, dma_addr,
@@ -454,7 +461,7 @@ int mp_dmabuf_devmem_init(struct page_pool *pool) · 159 unmodified lines
454461 pool->dma_sync = false;
455462 pool->dma_sync_for_cpu = false;
456463
457− if (pool->p.order != 0)
464+ if (pool->p.order != binding->niov_shift - PAGE_SHIFT)
458465 return -E2BIG;
459466
460467 net_devmem_dmabuf_binding_get(binding);
net/core/devmem.h+9−4
@@ -71,6 +71,8 @@ struct net_devmem_dmabuf_binding { · 70 unmodified lines
7171 */
7272 struct net_iov **tx_vec;
7373
74+ unsigned int niov_shift;
75+
7476 struct work_struct unbind_w;
7577 };
7678
@@ -93,7 +95,8 @@ struct net_devmem_dmabuf_binding * · 16 unmodified lines
9395 net_devmem_bind_dmabuf(struct net_device *dev, void *vdev,
9496 struct device *dma_dev,
9597 enum dma_data_direction direction,
96− unsigned int dmabuf_fd, struct netdev_nl_sock *priv,
98+ unsigned int dmabuf_fd, unsigned int niov_shift,
99+ struct netdev_nl_sock *priv,
97100 struct netlink_ext_ack *extack);
98101 struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id);
99102 void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding);
@@ -122,10 +125,11 @@ static inline u32 net_devmem_iov_binding_id(const struct net_iov *niov) · 22 unmodified lines
122125
123126 static inline unsigned long net_iov_virtual_addr(const struct net_iov *niov)
124127 {
125− struct net_iov_area *owner = net_iov_owner(niov);
128+ struct dmabuf_genpool_chunk_owner *co =
129+ net_devmem_iov_to_chunk_owner(niov);
126130
127− return owner->base_virtual +
128− ((unsigned long)net_iov_idx(niov) << PAGE_SHIFT);
131+ return net_iov_owner(niov)->base_virtual +
132+ ((unsigned long)net_iov_idx(niov) << co->binding->niov_shift);
129133 }
130134
131135 static inline bool
@@ -175,6 +179,7 @@ net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, · 43 unmodified lines
175179 struct device *dma_dev,
176180 enum dma_data_direction direction,
177181 unsigned int dmabuf_fd,
182+ unsigned int niov_shift,
178183 struct netdev_nl_sock *priv,
179184 struct netlink_ext_ack *extack)
180185 {
net/core/netdev-genl-gen.c+9−2
@@ -11,6 +11,7 @@ · 10 unmodified lines
1111
1212 #include <uapi/linux/netdev.h>
1313 #include <net/netdev_netlink.h>
14+#include <asm/page.h>
1415
1516 /* Integer value ranges */
1617 static const struct netlink_range_validation netdev_a_page_pool_id_range = {
@@ -27,6 +28,11 @@ static const struct netlink_range_validation netdev_a_napi_defer_hard_irqs_range · 10 unmodified lines
2728 .max = S32_MAX,
2829 };
2930
31+static const struct netlink_range_validation netdev_a_dmabuf_rx_page_size_range = {
32+ .min = PAGE_SIZE,
33+ .max = U32_MAX,
34+};
35+
3036 /* Common nested types */
3137 const struct nla_policy netdev_lease_nl_policy[NETDEV_A_LEASE_NETNS_ID + 1] = {
3238 [NETDEV_A_LEASE_IFINDEX] = NLA_POLICY_MIN(NLA_U32, 1),
@@ -106,10 +112,11 @@ static const struct nla_policy netdev_qstats_get_nl_policy[NETDEV_A_QSTATS_SCOPE · 73 unmodified lines
106112 };
107113
108114 /* NETDEV_CMD_BIND_RX - do */
109−static const struct nla_policy netdev_bind_rx_nl_policy[NETDEV_A_DMABUF_FD + 1] = {
115+static const struct nla_policy netdev_bind_rx_nl_policy[NETDEV_A_DMABUF_RX_PAGE_SIZE + 1] = {
110116 [NETDEV_A_DMABUF_IFINDEX] = NLA_POLICY_MIN(NLA_U32, 1),
111117 [NETDEV_A_DMABUF_FD] = { .type = NLA_U32, },
112118 [NETDEV_A_DMABUF_QUEUES] = NLA_POLICY_NESTED(netdev_queue_id_nl_policy),
119+ [NETDEV_A_DMABUF_RX_PAGE_SIZE] = NLA_POLICY_FULL_RANGE(NLA_U32, &netdev_a_dmabuf_rx_page_size_range),
113120 };
114121
115122 /* NETDEV_CMD_NAPI_SET - do */
@@ -219,7 +226,7 @@ static const struct genl_split_ops netdev_nl_ops[] = { · 103 unmodified lines
219226 .cmd = NETDEV_CMD_BIND_RX,
220227 .doit = netdev_nl_bind_rx_doit,
221228 .policy = netdev_bind_rx_nl_policy,
222− .maxattr = NETDEV_A_DMABUF_FD,
229+ .maxattr = NETDEV_A_DMABUF_RX_PAGE_SIZE,
223230 .flags = GENL_UNS_ADMIN_PERM | GENL_CMD_CAP_DO,
224231 },
225232 {
net/core/netdev-genl-gen.h+1−0
@@ -12,6 +12,7 @@ · 11 unmodified lines
1212
1313 #include <uapi/linux/netdev.h>
1414 #include <net/netdev_netlink.h>
15+#include <asm/page.h>
1516
1617 /* Common nested types */
1718 extern const struct nla_policy netdev_lease_nl_policy[NETDEV_A_LEASE_NETNS_ID + 1];
net/core/netdev-genl.c+16−2
@@ -1013,6 +1013,7 @@ netdev_nl_get_dma_dev(struct net_device *netdev, unsigned long *rxq_bitmap, · 1012 unmodified lines
10131013 int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info)
10141014 {
10151015 struct net_devmem_dmabuf_binding *binding;
1016+ unsigned int niov_shift = PAGE_SHIFT;
10161017 u32 ifindex, dmabuf_fd, rxq_idx;
10171018 struct netdev_nl_sock *priv;
10181019 struct net_device *netdev;
@@ -1030,6 +1031,18 @@ int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info) · 11 unmodified lines
10301031 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]);
10311032 dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]);
10321033
1034+ if (info->attrs[NETDEV_A_DMABUF_RX_PAGE_SIZE]) {
1035+ u32 rx_page_size = nla_get_u32(info->attrs[NETDEV_A_DMABUF_RX_PAGE_SIZE]);
1036+
1037+ if (!is_power_of_2(rx_page_size)) {
1038+ NL_SET_ERR_MSG_ATTR(info->extack,
1039+ info->attrs[NETDEV_A_DMABUF_RX_PAGE_SIZE],
1040+ "rx-page-size must be a power of 2");
1041+ return -EINVAL;
1042+ }
1043+ niov_shift = ilog2(rx_page_size);
1044+ }
1045+
10331046 priv = genl_sk_priv_get(&netdev_nl_family, NETLINK_CB(skb).sk);
10341047 if (IS_ERR(priv))
10351048 return PTR_ERR(priv);
@@ -1080,7 +1093,8 @@ int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info) · 44 unmodified lines
10801093 }
10811094
10821095 binding = net_devmem_bind_dmabuf(netdev, NULL, dma_dev, DMA_FROM_DEVICE,
1083− dmabuf_fd, priv, info->extack);
1096+ dmabuf_fd, niov_shift, priv,
1097+ info->extack);
10841098 if (IS_ERR(binding)) {
10851099 err = PTR_ERR(binding);
10861100 goto err_rxq_bitmap;
@@ -1221,7 +1235,7 @@ int netdev_nl_bind_tx_doit(struct sk_buff *skb, struct genl_info *info) · 134 unmodified lines
12211235 binding = net_devmem_bind_dmabuf(bind_dev,
12221236 bind_dev != netdev ? netdev : NULL,
12231237 dma_dev, DMA_TO_DEVICE, dmabuf_fd,
1224− priv, info->extack);
1238+ PAGE_SHIFT, priv, info->extack);
12251239 if (IS_ERR(binding)) {
12261240 err = PTR_ERR(binding);
12271241 goto err_unlock_bind_dev;
tools/include/uapi/linux/netdev.h+1−0
@@ -219,6 +219,7 @@ enum { · 218 unmodified lines
219219 NETDEV_A_DMABUF_QUEUES,
220220 NETDEV_A_DMABUF_FD,
221221 NETDEV_A_DMABUF_ID,
222+ NETDEV_A_DMABUF_RX_PAGE_SIZE,
222223
223224 __NETDEV_A_DMABUF_MAX,
224225 NETDEV_A_DMABUF_MAX = (__NETDEV_A_DMABUF_MAX - 1)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t