net: devmem: allow rx-page-size > PAGE_SIZE per dmabuf binding

Every devmem dmabuf binding today hands the page_pool PAGE_SIZE niovs. This caps a single RX descriptor at PAGE_SIZE, burning CPU on buffer churn for large flows. Add a bind-time netlink attribute, NETDEV_A_DMABUF_RX_PAGE_SIZE, that lets userspace request a larger niov size. The value must be a power of two >= PAGE_SIZE. The TX path is changed to always pass PAGE_SIZE. Measurements: Setup: kperf in devmem RX/TX cuda mode, 4 flows, 64 MB messages, 60s, dctcp, num-rx-queues=4, dmabuf-rx/tx-size-mb=2048, 10 runs per niov size, mlx5. CPU Util: niov net sirq % net idle % app sys % app idle % ----- ---------------- ---------------- ---------------- ---------------- 4K 62.38 +/- 8.27 33.40 +/- 7.51 54.15 +/- 10.23 43.67 +/- 10.53 16K 58.91 +/- 5.35 35.23 +/- 5.88 41.05 +/- 8.87 56.42 +/- 9.24 32K 64.12 +/- 0.68 31.09 +/- 1.48 44.54 +/- 3.51 52.63 +/- 3.65 64K 54.69 +/- 5.54 39.67 +/- 5.81 35.47 +/- 3.11 61.97 +/- 3.27 RX app sys % drops ~19% from 4K to 64K. Throughput: niov RX dev Gbps RX flow avg Gbps ----- ---------------- ----------------- 4K 300.63 +/- 53.21 75.16 +/- 13.30 16K 321.35 +/- 28.20 80.34 +/- 7.05 32K 347.63 +/- 2.20 86.91 +/- 0.55 64K 332.11 +/- 14.26 83.03 +/- 3.56 Throughput seems to increase, but the stdev is pretty wide so could just be noise. kperf support (not yet merged): https://github.com/facebookexperimental/kperf/commit/8837577f920876bce6986ec18869ac04439ebcd2

Bobby Eshleman committed 3 months ago

Commit b27a8561 parent 55d20f58 files · +78 −23Browse files

Unified · Split

8 files

Documentation/netlink/specs/netdev.yaml +19 −0
include/uapi/linux/netdev.h +1 −0
net/core/devmem.c +22 −15
net/core/devmem.h +9 −4
net/core/netdev-genl-gen.c +9 −2
net/core/netdev-genl-gen.h +1 −0
net/core/netdev-genl.c +16 −2
tools/include/uapi/linux/netdev.h +1 −0

Documentation/netlink/specs/netdev.yaml+19−0

@@ -6,6 +6,14 @@ doc: >- · 5 unmodified lines
6	netdev configuration over generic netlink.	6	netdev configuration over generic netlink.
7		7
8	definitions:	8	definitions:
		9	-
		10	type: const
		11	name: page-size
		12	# Dummy value, codegen needs a number. The real value comes from
		13	# the PAGE_SIZE macro in the header below.
		14	value: 0
		15	header: asm/page.h
		16	scope: kernel
9	-	17	-
10	type: flags	18	type: flags
11	name: xdp-act	19	name: xdp-act
@@ -598,6 +606,16 @@ attribute-sets: · 586 unmodified lines
598	type: u32	606	type: u32
599	checks:	607	checks:
600	min: 1	608	min: 1
		609	-
		610	name: rx-page-size
		611	doc: \|
		612	Size in bytes of each device page the NIC writes into from the bound
		613	dmabuf. Must be a power of two and >= PAGE_SIZE; defaults to
		614	PAGE_SIZE.
		615	type: u32
		616	checks:
		617	min: page-size
		618	max: u32-max
601		619
602	operations:	620	operations:
603	list:	621	list:
@@ -812,6 +830,7 @@ operations: · 208 unmodified lines
812	- ifindex	830	- ifindex
813	- fd	831	- fd
814	- queues	832	- queues
		833	- rx-page-size
815	reply:	834	reply:
816	attributes:	835	attributes:
817	- id	836	- id

include/uapi/linux/netdev.h+1−0

@@ -219,6 +219,7 @@ enum { · 218 unmodified lines
219	NETDEV_A_DMABUF_QUEUES,	219	NETDEV_A_DMABUF_QUEUES,
220	NETDEV_A_DMABUF_FD,	220	NETDEV_A_DMABUF_FD,
221	NETDEV_A_DMABUF_ID,	221	NETDEV_A_DMABUF_ID,
		222	NETDEV_A_DMABUF_RX_PAGE_SIZE,
222		223
223	__NETDEV_A_DMABUF_MAX,	224	__NETDEV_A_DMABUF_MAX,
224	NETDEV_A_DMABUF_MAX = (__NETDEV_A_DMABUF_MAX - 1)	225	NETDEV_A_DMABUF_MAX = (__NETDEV_A_DMABUF_MAX - 1)

net/core/devmem.c+22−15

 	owner = net_devmem_iov_to_chunk_owner(niov);
 	return owner->base_dma_addr +
-	       ((dma_addr_t)net_iov_idx(niov) << PAGE_SHIFT);
+	       ((dma_addr_t)net_iov_idx(niov) << owner->binding->niov_shift);
 }
 static void net_devmem_dmabuf_binding_release(struct percpu_ref *ref)
 	ssize_t offset;
 	ssize_t index;
-	dma_addr = gen_pool_alloc_owner(binding->chunk_pool, PAGE_SIZE,
+	dma_addr = gen_pool_alloc_owner(binding->chunk_pool,
+					1UL << binding->niov_shift,
 					(void **)&owner);
 	if (!dma_addr)
 		return NULL;
 	offset = dma_addr - owner->base_dma_addr;
-	index = offset / PAGE_SIZE;
+	index = offset >> binding->niov_shift;
 	niov = &owner->area.niovs[index];
 	niov->desc.pp_magic = 0;
 {
 	struct net_devmem_dmabuf_binding *binding = net_devmem_iov_binding(niov);
 	unsigned long dma_addr = net_devmem_get_dma_addr(niov);
+	size_t niov_size = 1UL << binding->niov_shift;
 	if (WARN_ON(!gen_pool_has_addr(binding->chunk_pool, dma_addr,
-				       PAGE_SIZE)))
+				       niov_size)))
 		return;
-	gen_pool_free(binding->chunk_pool, dma_addr, PAGE_SIZE);
+	gen_pool_free(binding->chunk_pool, dma_addr, niov_size);
 }
 void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding)
 	u32 xa_idx;
 	int err;
+	if (binding->niov_shift != PAGE_SHIFT)
+		mp_params.rx_page_size = 1U << binding->niov_shift;
 	err = netif_mp_open_rxq(dev, rxq_idx, &mp_params, extack);
 	if (err)
 		return err;
 net_devmem_bind_dmabuf(struct net_device *dev, void *vdev,
 		       struct device *dma_dev,
 		       enum dma_data_direction direction,
-		       unsigned int dmabuf_fd, struct netdev_nl_sock *priv,
+		       unsigned int dmabuf_fd, unsigned int niov_shift,
+		       struct netdev_nl_sock *priv,
 		       struct netlink_ext_ack *extack)
 {
 	struct net_devmem_dmabuf_binding *binding;
+	size_t niov_size = 1UL << niov_shift;
 	static u32 id_alloc_next;
 	struct scatterlist *sg;
 	struct dma_buf *dmabuf;
 	binding->dev = dev;
 	binding->vdev = vdev;
+	binding->niov_shift = niov_shift;
 	xa_init_flags(&binding->bound_rxqs, XA_FLAGS_ALLOC);
 	err = percpu_ref_init(&binding->ref,
 		}
 	}
-	/* For simplicity we expect to make PAGE_SIZE allocations, but the
+	binding->chunk_pool = gen_pool_create(niov_shift,
-	 * binding can be much more flexible than that. We may be able to
-	 * allocate MTU sized chunks here. Leave that for future work...
-	 */
-	binding->chunk_pool = gen_pool_create(PAGE_SHIFT,
 					      dev_to_node(&dev->dev));
 	if (!binding->chunk_pool) {
 		err = -ENOMEM;
 		size_t len = sg_dma_len(sg);
 		struct net_iov *niov;
-		if (!IS_ALIGNED(len, PAGE_SIZE)) {
+		if (!IS_ALIGNED(dma_addr, niov_size) ||
+		    !IS_ALIGNED(len, niov_size)) {
 			err = -EINVAL;
-			NL_SET_ERR_MSG(extack, "dma-buf SG length must be PAGE_SIZE aligned");
+			NL_SET_ERR_MSG_FMT(extack,
+					   "dmabuf sg entry (addr=%pad, len=%zu) not aligned to niov size %zu",
+					   &dma_addr, len, niov_size);
 			goto err_free_chunks;
 		}
 		owner->area.base_virtual = virtual;
 		owner->base_dma_addr = dma_addr;
-		owner->area.num_niovs = len / PAGE_SIZE;
+		owner->area.num_niovs = len >> niov_shift;
 		owner->binding = binding;
 		err = gen_pool_add_owner(binding->chunk_pool, dma_addr,
 	pool->dma_sync = false;
 	pool->dma_sync_for_cpu = false;
-	if (pool->p.order != 0)
+	if (pool->p.order != binding->niov_shift - PAGE_SHIFT)
 		return -E2BIG;
 	net_devmem_dmabuf_binding_get(binding);

net/core/devmem.h+9−4

 	 */
 	struct net_iov **tx_vec;
+	unsigned int niov_shift;
 	struct work_struct unbind_w;
 };
 net_devmem_bind_dmabuf(struct net_device *dev, void *vdev,
 		       struct device *dma_dev,
 		       enum dma_data_direction direction,
-		       unsigned int dmabuf_fd, struct netdev_nl_sock *priv,
+		       unsigned int dmabuf_fd, unsigned int niov_shift,
+		       struct netdev_nl_sock *priv,
 		       struct netlink_ext_ack *extack);
 struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id);
 void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding);
 static inline unsigned long net_iov_virtual_addr(const struct net_iov *niov)
 {
-	struct net_iov_area *owner = net_iov_owner(niov);
+	struct dmabuf_genpool_chunk_owner *co =
+		net_devmem_iov_to_chunk_owner(niov);
-	return owner->base_virtual +
+	return net_iov_owner(niov)->base_virtual +
-	       ((unsigned long)net_iov_idx(niov) << PAGE_SHIFT);
+	       ((unsigned long)net_iov_idx(niov) << co->binding->niov_shift);
 }
 static inline bool
 		       struct device *dma_dev,
 		       enum dma_data_direction direction,
 		       unsigned int dmabuf_fd,
+		       unsigned int niov_shift,
 		       struct netdev_nl_sock *priv,
 		       struct netlink_ext_ack *extack)
 {

net/core/netdev-genl-gen.c+9−2

 #include <uapi/linux/netdev.h>
 #include <net/netdev_netlink.h>
+#include <asm/page.h>
 /* Integer value ranges */
 static const struct netlink_range_validation netdev_a_page_pool_id_range = {
 	.max	= S32_MAX,
 };
+static const struct netlink_range_validation netdev_a_dmabuf_rx_page_size_range = {
+	.min	= PAGE_SIZE,
+	.max	= U32_MAX,
+};
 /* Common nested types */
 const struct nla_policy netdev_lease_nl_policy[NETDEV_A_LEASE_NETNS_ID + 1] = {
 	[NETDEV_A_LEASE_IFINDEX] = NLA_POLICY_MIN(NLA_U32, 1),
 };
 /* NETDEV_CMD_BIND_RX - do */
-static const struct nla_policy netdev_bind_rx_nl_policy[NETDEV_A_DMABUF_FD + 1] = {
+static const struct nla_policy netdev_bind_rx_nl_policy[NETDEV_A_DMABUF_RX_PAGE_SIZE + 1] = {
 	[NETDEV_A_DMABUF_IFINDEX] = NLA_POLICY_MIN(NLA_U32, 1),
 	[NETDEV_A_DMABUF_FD] = { .type = NLA_U32, },
 	[NETDEV_A_DMABUF_QUEUES] = NLA_POLICY_NESTED(netdev_queue_id_nl_policy),
+	[NETDEV_A_DMABUF_RX_PAGE_SIZE] = NLA_POLICY_FULL_RANGE(NLA_U32, &netdev_a_dmabuf_rx_page_size_range),
 };
 /* NETDEV_CMD_NAPI_SET - do */
 		.cmd		= NETDEV_CMD_BIND_RX,
 		.doit		= netdev_nl_bind_rx_doit,
 		.policy		= netdev_bind_rx_nl_policy,
-		.maxattr	= NETDEV_A_DMABUF_FD,
+		.maxattr	= NETDEV_A_DMABUF_RX_PAGE_SIZE,
 		.flags		= GENL_UNS_ADMIN_PERM | GENL_CMD_CAP_DO,
 	},
 	{

net/core/netdev-genl-gen.h+1−0

@@ -12,6 +12,7 @@ · 11 unmodified lines
12		12
13	#include <uapi/linux/netdev.h>	13	#include <uapi/linux/netdev.h>
14	#include <net/netdev_netlink.h>	14	#include <net/netdev_netlink.h>
		15	#include <asm/page.h>
15		16
16	/* Common nested types */	17	/* Common nested types */
17	extern const struct nla_policy netdev_lease_nl_policy[NETDEV_A_LEASE_NETNS_ID + 1];	18	extern const struct nla_policy netdev_lease_nl_policy[NETDEV_A_LEASE_NETNS_ID + 1];

net/core/netdev-genl.c+16−2

 int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct net_devmem_dmabuf_binding *binding;
+	unsigned int niov_shift = PAGE_SHIFT;
 	u32 ifindex, dmabuf_fd, rxq_idx;
 	struct netdev_nl_sock *priv;
 	struct net_device *netdev;
 	ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]);
 	dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]);
+	if (info->attrs[NETDEV_A_DMABUF_RX_PAGE_SIZE]) {
+		u32 rx_page_size = nla_get_u32(info->attrs[NETDEV_A_DMABUF_RX_PAGE_SIZE]);
+		if (!is_power_of_2(rx_page_size)) {
+			NL_SET_ERR_MSG_ATTR(info->extack,
+					    info->attrs[NETDEV_A_DMABUF_RX_PAGE_SIZE],
+					    "rx-page-size must be a power of 2");
+			return -EINVAL;
+		}
+		niov_shift = ilog2(rx_page_size);
+	}
 	priv = genl_sk_priv_get(&netdev_nl_family, NETLINK_CB(skb).sk);
 	if (IS_ERR(priv))
 		return PTR_ERR(priv);
 	}
 	binding = net_devmem_bind_dmabuf(netdev, NULL, dma_dev, DMA_FROM_DEVICE,
-					 dmabuf_fd, priv, info->extack);
+					 dmabuf_fd, niov_shift, priv,
+					 info->extack);
 	if (IS_ERR(binding)) {
 		err = PTR_ERR(binding);
 		goto err_rxq_bitmap;
 	binding = net_devmem_bind_dmabuf(bind_dev,
 					 bind_dev != netdev ? netdev : NULL,
 					 dma_dev, DMA_TO_DEVICE, dmabuf_fd,
-					 priv, info->extack);
+					 PAGE_SHIFT, priv, info->extack);
 	if (IS_ERR(binding)) {
 		err = PTR_ERR(binding);
 		goto err_unlock_bind_dev;

tools/include/uapi/linux/netdev.h+1−0

@@ -219,6 +219,7 @@ enum { · 218 unmodified lines
219	NETDEV_A_DMABUF_QUEUES,	219	NETDEV_A_DMABUF_QUEUES,
220	NETDEV_A_DMABUF_FD,	220	NETDEV_A_DMABUF_FD,
221	NETDEV_A_DMABUF_ID,	221	NETDEV_A_DMABUF_ID,
		222	NETDEV_A_DMABUF_RX_PAGE_SIZE,
222		223
223	__NETDEV_A_DMABUF_MAX,	224	__NETDEV_A_DMABUF_MAX,
224	NETDEV_A_DMABUF_MAX = (__NETDEV_A_DMABUF_MAX - 1)	225	NETDEV_A_DMABUF_MAX = (__NETDEV_A_DMABUF_MAX - 1)

net: devmem: allow rx-page-size > PAGE_SIZE per dmabuf binding

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code