Merge branch 'io_uring-zero-copy-rx'

David Wei says: ==================== io_uring zero copy rx This patchset contains net/ patches needed by a new io_uring request implementing zero copy rx into userspace pages, eliminating a kernel to user copy. We configure a page pool that a driver uses to fill a hw rx queue to hand out user pages instead of kernel pages. Any data that ends up hitting this hw rx queue will thus be dma'd into userspace memory directly, without needing to be bounced through kernel memory. 'Reading' data out of a socket instead becomes a _notification_ mechanism, where the kernel tells userspace where the data is. The overall approach is similar to the devmem TCP proposal. This relies on hw header/data split, flow steering and RSS to ensure packet headers remain in kernel memory and only desired flows hit a hw rx queue configured for zero copy. Configuring this is outside of the scope of this patchset. We share netdev core infra with devmem TCP. The main difference is that io_uring is used for the uAPI and the lifetime of all objects are bound to an io_uring instance. Data is 'read' using a new io_uring request type. When done, data is returned via a new shared refill queue. A zero copy page pool refills a hw rx queue from this refill queue directly. Of course, the lifetime of these data buffers are managed by io_uring rather than the networking stack, with different refcounting rules. This patchset is the first step adding basic zero copy support. We will extend this iteratively with new features e.g. dynamically allocated zero copy areas, THP support, dmabuf support, improved copy fallback, general optimisations and more. In terms of netdev support, we're first targeting Broadcom bnxt. Patches aren't included since Taehee Yoo has already sent a more comprehensive patchset adding support in [1]. Google gve should already support this, and Mellanox mlx5 support is WIP pending driver changes. =========== Performance =========== Note: Comparison with epoll + TCP_ZEROCOPY_RECEIVE isn't done yet. Test setup: * AMD EPYC 9454 * Broadcom BCM957508 200G * Kernel v6.11 base [2] * liburing fork [3] * kperf fork [4] * 4K MTU * Single TCP flow With application thread + net rx softirq pinned to _different_ cores: +-------------------------------+ | epoll | io_uring | |-----------|-------------------| | 82.2 Gbps | 116.2 Gbps (+41%) | +-------------------------------+ Pinned to _same_ core: +-------------------------------+ | epoll | io_uring | |-----------|-------------------| | 62.6 Gbps | 80.9 Gbps (+29%) | +-------------------------------+ ===== Links ===== Broadcom bnxt support: [1]: https://lore.kernel.org/20241003160620.1521626-8-ap420073@gmail.com Linux kernel branch including io_uring bits: [2]: https://github.com/isilence/linux.git zcrx/v13 liburing for testing: [3]: https://github.com/isilence/liburing.git zcrx/next kperf for testing: [4]: https://git.kernel.dk/kperf.git ====================

Jakub Kicinski committed 1 year ago

Commit 71f0dd52 parents ba6ec09 6e18ed914 files · +321 −81Browse files

Unified · Split

14 files

Documentation/netlink/specs/netdev.yaml +15 −0
include/net/netmem.h +20 −1
include/net/page_pool/memory_provider.h +45 −0
include/net/page_pool/types.h +4 −0
include/uapi/linux/netdev.h +7 −0
net/core/dev.c +15 −1
net/core/devmem.c +59 −34
net/core/devmem.h +22 −27
net/core/netdev-genl.c +6 −5
net/core/netdev_rx_queue.c +69 −0
net/core/page_pool.c +43 −8
net/core/page_pool_user.c +3 −4
net/ipv4/tcp.c +6 −1
tools/include/uapi/linux/netdev.h +7 −0

Documentation/netlink/specs/netdev.yaml+15−0

         doc: Bitmask of enabled AF_XDP features.
         type: u64
         enum: xsk-flags
+  -
+    name: io-uring-provider-info
+    attributes: []
   -
     name: page-pool
     attributes:
         name: dmabuf
         doc: ID of the dmabuf this page-pool is attached to.
         type: u32
+      -
+        name: io-uring
+        doc: io-uring memory provider information.
+        type: nest
+        nested-attributes: io-uring-provider-info
   -
     name: page-pool-info
     subset-of: page-pool
         name: dmabuf
         doc: ID of the dmabuf attached to this queue, if any.
         type: u32
+      -
+        name: io-uring
+        doc: io_uring memory provider information.
+        type: nest
+        nested-attributes: io-uring-provider-info
   -
     name: qstats
             - inflight-mem
             - detach-time
             - dmabuf
+            - io-uring
       dump:
         reply: *pp-reply
       config-cond: page-pool
             - napi-id
             - ifindex
             - dmabuf
+            - io-uring
       dump:
         request:
           attributes:

include/net/netmem.h+20−1

@@ -24,11 +24,20 @@ struct net_iov { · 23 unmodified lines
24	unsigned long __unused_padding;	24	unsigned long __unused_padding;
25	unsigned long pp_magic;	25	unsigned long pp_magic;
26	struct page_pool *pp;	26	struct page_pool *pp;
27	struct dmabuf_genpool_chunk_owner *owner;	27	struct net_iov_area *owner;
28	unsigned long dma_addr;	28	unsigned long dma_addr;
29	atomic_long_t pp_ref_count;	29	atomic_long_t pp_ref_count;
30	};	30	};
31		31
		32	struct net_iov_area {
		33	/* Array of net_iovs for this area. */
		34	struct net_iov *niovs;
		35	size_t num_niovs;
		36
		37	/* Offset into the dma-buf where this chunk starts. */
		38	unsigned long base_virtual;
		39	};
		40
32	/* These fields in struct page are used by the page_pool and net stack:	41	/* These fields in struct page are used by the page_pool and net stack:
33	*	42	*
34	* struct {	43	* struct {
@@ -54,6 +63,16 @@ NET_IOV_ASSERT_OFFSET(dma_addr, dma_addr); · 19 unmodified lines
54	NET_IOV_ASSERT_OFFSET(pp_ref_count, pp_ref_count);	63	NET_IOV_ASSERT_OFFSET(pp_ref_count, pp_ref_count);
55	#undef NET_IOV_ASSERT_OFFSET	64	#undef NET_IOV_ASSERT_OFFSET
56		65
		66	static inline struct net_iov_area net_iov_owner(const struct net_iov niov)
		67	{
		68	return niov->owner;
		69	}
		70
		71	static inline unsigned int net_iov_idx(const struct net_iov *niov)
		72	{
		73	return niov - net_iov_owner(niov)->niovs;
		74	}
		75
57	/* netmem */	76	/* netmem */
58		77
59	/**	78	/**

include/net/page_pool/memory_provider.h+45−0

@@ -0,0 +1,45 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#ifndef _NET_PAGE_POOL_MEMORY_PROVIDER_H
	3	#define _NET_PAGE_POOL_MEMORY_PROVIDER_H
	4
	5	#include <net/netmem.h>
	6	#include <net/page_pool/types.h>
	7
	8	struct netdev_rx_queue;
	9	struct sk_buff;
	10
	11	struct memory_provider_ops {
	12	netmem_ref (alloc_netmems)(struct page_pool pool, gfp_t gfp);
	13	bool (release_netmem)(struct page_pool pool, netmem_ref netmem);
	14	int (init)(struct page_pool pool);
	15	void (destroy)(struct page_pool pool);
	16	int (nl_fill)(void mp_priv, struct sk_buff *rsp,
	17	struct netdev_rx_queue *rxq);
	18	void (uninstall)(void mp_priv, struct netdev_rx_queue *rxq);
	19	};
	20
	21	bool net_mp_niov_set_dma_addr(struct net_iov *niov, dma_addr_t addr);
	22	void net_mp_niov_set_page_pool(struct page_pool pool, struct net_iov niov);
	23	void net_mp_niov_clear_page_pool(struct net_iov *niov);
	24
	25	int net_mp_open_rxq(struct net_device *dev, unsigned ifq_idx,
	26	struct pp_memory_provider_params *p);
	27	void net_mp_close_rxq(struct net_device *dev, unsigned ifq_idx,
	28	struct pp_memory_provider_params *old_p);
	29
	30	/**
	31	* net_mp_netmem_place_in_cache() - give a netmem to a page pool
	32	* @pool: the page pool to place the netmem into
	33	* @netmem: netmem to give
	34	*
	35	* Push an accounted netmem into the page pool's allocation cache. The caller
	36	* must ensure that there is space in the cache. It should only be called off
	37	* the mp_ops->alloc_netmems() path.
	38	*/
	39	static inline void net_mp_netmem_place_in_cache(struct page_pool *pool,
	40	netmem_ref netmem)
	41	{
	42	pool->alloc.cache[pool->alloc.count++] = netmem;
	43	}
	44
	45	#endif

include/net/page_pool/types.h+4−0

  */
 #define PAGE_POOL_FRAG_GROUP_ALIGN	(4 * sizeof(long))
+struct memory_provider_ops;
 struct pp_memory_provider_params {
 	void *mp_priv;
+	const struct memory_provider_ops *mp_ops;
 };
 struct page_pool {
 	struct ptr_ring ring;
 	void *mp_priv;
+	const struct memory_provider_ops *mp_ops;
 #ifdef CONFIG_PAGE_POOL_STATS
 	/* recycle stats are per-cpu to avoid locking */

include/uapi/linux/netdev.h+7−0

 	NETDEV_A_DEV_MAX = (__NETDEV_A_DEV_MAX - 1)
 };
+enum {
+	__NETDEV_A_IO_URING_PROVIDER_INFO_MAX,
+	NETDEV_A_IO_URING_PROVIDER_INFO_MAX = (__NETDEV_A_IO_URING_PROVIDER_INFO_MAX - 1)
+};
 enum {
 	NETDEV_A_PAGE_POOL_ID = 1,
 	NETDEV_A_PAGE_POOL_IFINDEX,
 	NETDEV_A_PAGE_POOL_INFLIGHT_MEM,
 	NETDEV_A_PAGE_POOL_DETACH_TIME,
 	NETDEV_A_PAGE_POOL_DMABUF,
+	NETDEV_A_PAGE_POOL_IO_URING,
 	__NETDEV_A_PAGE_POOL_MAX,
 	NETDEV_A_PAGE_POOL_MAX = (__NETDEV_A_PAGE_POOL_MAX - 1)
 	NETDEV_A_QUEUE_TYPE,
 	NETDEV_A_QUEUE_NAPI_ID,
 	NETDEV_A_QUEUE_DMABUF,
+	NETDEV_A_QUEUE_IO_URING,
 	__NETDEV_A_QUEUE_MAX,
 	NETDEV_A_QUEUE_MAX = (__NETDEV_A_QUEUE_MAX - 1)

net/core/dev.c+15−1

 #include <net/netdev_rx_queue.h>
 #include <net/page_pool/types.h>
 #include <net/page_pool/helpers.h>
+#include <net/page_pool/memory_provider.h>
 #include <net/rps.h>
 #include <linux/phy_link_topology.h>
 }
 EXPORT_SYMBOL(unregister_netdevice_queue);
+static void dev_memory_provider_uninstall(struct net_device *dev)
+{
+	unsigned int i;
+	for (i = 0; i < dev->real_num_rx_queues; i++) {
+		struct netdev_rx_queue *rxq = &dev->_rx[i];
+		struct pp_memory_provider_params *p = &rxq->mp_params;
+		if (p->mp_ops && p->mp_ops->uninstall)
+			p->mp_ops->uninstall(rxq->mp_params.mp_priv, rxq);
+	}
+}
 void unregister_netdevice_many_notify(struct list_head *head,
 				      u32 portid, const struct nlmsghdr *nlh)
 {
 		dev_tcx_uninstall(dev);
 		dev_xdp_uninstall(dev);
 		bpf_dev_bound_netdev_unregister(dev);
-		dev_dmabuf_uninstall(dev);
+		dev_memory_provider_uninstall(dev);
 		netdev_offload_xstats_disable_all(dev);

net/core/devmem.c+59−34

 #include <net/netdev_queues.h>
 #include <net/netdev_rx_queue.h>
 #include <net/page_pool/helpers.h>
+#include <net/page_pool/memory_provider.h>
 #include <trace/events/page_pool.h>
 #include "devmem.h"
 /* Protected by rtnl_lock() */
 static DEFINE_XARRAY_FLAGS(net_devmem_dmabuf_bindings, XA_FLAGS_ALLOC1);
+static const struct memory_provider_ops dmabuf_devmem_ops;
+bool net_is_devmem_iov(struct net_iov *niov)
+{
+	return niov->pp->mp_ops == &dmabuf_devmem_ops;
+}
 static void net_devmem_dmabuf_free_chunk_owner(struct gen_pool *genpool,
 					       struct gen_pool_chunk *chunk,
 					       void *not_used)
 {
 	struct dmabuf_genpool_chunk_owner *owner = chunk->owner;
-	kvfree(owner->niovs);
+	kvfree(owner->area.niovs);
 	kfree(owner);
 }
 static dma_addr_t net_devmem_get_dma_addr(const struct net_iov *niov)
 {
-	struct dmabuf_genpool_chunk_owner *owner = net_iov_owner(niov);
+	struct dmabuf_genpool_chunk_owner *owner;
+	owner = net_devmem_iov_to_chunk_owner(niov);
 	return owner->base_dma_addr +
 	       ((dma_addr_t)net_iov_idx(niov) << PAGE_SHIFT);
 }
 	offset = dma_addr - owner->base_dma_addr;
 	index = offset / PAGE_SIZE;
-	niov = &owner->niovs[index];
+	niov = &owner->area.niovs[index];
 	niov->pp_magic = 0;
 	niov->pp = NULL;
 void net_devmem_free_dmabuf(struct net_iov *niov)
 {
-	struct net_devmem_dmabuf_binding *binding = net_iov_binding(niov);
+	struct net_devmem_dmabuf_binding *binding = net_devmem_iov_binding(niov);
 	unsigned long dma_addr = net_devmem_get_dma_addr(niov);
 	if (WARN_ON(!gen_pool_has_addr(binding->chunk_pool, dma_addr,
 		WARN_ON(rxq->mp_params.mp_priv != binding);
 		rxq->mp_params.mp_priv = NULL;
+		rxq->mp_params.mp_ops = NULL;
 		rxq_idx = get_netdev_rx_queue_index(rxq);
 	}
 	rxq = __netif_get_rx_queue(dev, rxq_idx);
-	if (rxq->mp_params.mp_priv) {
+	if (rxq->mp_params.mp_ops) {
 		NL_SET_ERR_MSG(extack, "designated queue already memory provider bound");
 		return -EEXIST;
 	}
 		return err;
 	rxq->mp_params.mp_priv = binding;
+	rxq->mp_params.mp_ops = &dmabuf_devmem_ops;
 	err = netdev_rx_queue_restart(dev, rxq_idx);
 	if (err)
 err_xa_erase:
 	rxq->mp_params.mp_priv = NULL;
+	rxq->mp_params.mp_ops = NULL;
 	xa_erase(&binding->bound_rxqs, xa_idx);
 	return err;
 			goto err_free_chunks;
 		}
-		owner->base_virtual = virtual;
+		owner->area.base_virtual = virtual;
 		owner->base_dma_addr = dma_addr;
-		owner->num_niovs = len / PAGE_SIZE;
+		owner->area.num_niovs = len / PAGE_SIZE;
 		owner->binding = binding;
 		err = gen_pool_add_owner(binding->chunk_pool, dma_addr,
 			goto err_free_chunks;
 		}
-		owner->niovs = kvmalloc_array(owner->num_niovs,
+		owner->area.niovs = kvmalloc_array(owner->area.num_niovs,
-					      sizeof(*owner->niovs),
+						   sizeof(*owner->area.niovs),
-					      GFP_KERNEL);
+						   GFP_KERNEL);
-		if (!owner->niovs) {
+		if (!owner->area.niovs) {
 			err = -ENOMEM;
 			goto err_free_chunks;
 		}
-		for (i = 0; i < owner->num_niovs; i++) {
+		for (i = 0; i < owner->area.num_niovs; i++) {
-			niov = &owner->niovs[i];
+			niov = &owner->area.niovs[i];
-			niov->owner = owner;
+			niov->owner = &owner->area;
 			page_pool_set_dma_addr_netmem(net_iov_to_netmem(niov),
 						      net_devmem_get_dma_addr(niov));
 		}
 	return ERR_PTR(err);
 }
-void dev_dmabuf_uninstall(struct net_device *dev)
-{
-	struct net_devmem_dmabuf_binding *binding;
-	struct netdev_rx_queue *rxq;
-	unsigned long xa_idx;
-	unsigned int i;
-	for (i = 0; i < dev->real_num_rx_queues; i++) {
-		binding = dev->_rx[i].mp_params.mp_priv;
-		if (!binding)
-			continue;
-		xa_for_each(&binding->bound_rxqs, xa_idx, rxq)
-			if (rxq == &dev->_rx[i]) {
-				xa_erase(&binding->bound_rxqs, xa_idx);
-				break;
-			}
-	}
-}
 /*** "Dmabuf devmem memory provider" ***/
 int mp_dmabuf_devmem_init(struct page_pool *pool)
 	/* We don't want the page pool put_page()ing our net_iovs. */
 	return false;
 }
+static int mp_dmabuf_devmem_nl_fill(void *mp_priv, struct sk_buff *rsp,
+				    struct netdev_rx_queue *rxq)
+{
+	const struct net_devmem_dmabuf_binding *binding = mp_priv;
+	int type = rxq ? NETDEV_A_QUEUE_DMABUF : NETDEV_A_PAGE_POOL_DMABUF;
+	return nla_put_u32(rsp, type, binding->id);
+}
+static void mp_dmabuf_devmem_uninstall(void *mp_priv,
+				       struct netdev_rx_queue *rxq)
+{
+	struct net_devmem_dmabuf_binding *binding = mp_priv;
+	struct netdev_rx_queue *bound_rxq;
+	unsigned long xa_idx;
+	xa_for_each(&binding->bound_rxqs, xa_idx, bound_rxq) {
+		if (bound_rxq == rxq) {
+			xa_erase(&binding->bound_rxqs, xa_idx);
+			break;
+		}
+	}
+}
+static const struct memory_provider_ops dmabuf_devmem_ops = {
+	.init			= mp_dmabuf_devmem_init,
+	.destroy		= mp_dmabuf_devmem_destroy,
+	.alloc_netmems		= mp_dmabuf_devmem_alloc_netmems,
+	.release_netmem		= mp_dmabuf_devmem_release_page,
+	.nl_fill		= mp_dmabuf_devmem_nl_fill,
+	.uninstall		= mp_dmabuf_devmem_uninstall,
+};

net/core/devmem.h+22−27

 #ifndef _NET_DEVMEM_H
 #define _NET_DEVMEM_H
+#include <net/netmem.h>
 struct netlink_ext_ack;
 struct net_devmem_dmabuf_binding {
  * allocations from this chunk.
  */
 struct dmabuf_genpool_chunk_owner {
-	/* Offset into the dma-buf where this chunk starts.  */
+	struct net_iov_area area;
-	unsigned long base_virtual;
+	struct net_devmem_dmabuf_binding *binding;
 	/* dma_addr of the start of the chunk.  */
 	dma_addr_t base_dma_addr;
-	/* Array of net_iovs for this chunk. */
-	struct net_iov *niovs;
-	size_t num_niovs;
-	struct net_devmem_dmabuf_binding *binding;
 };
 void __net_devmem_dmabuf_binding_free(struct net_devmem_dmabuf_binding *binding);
 int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx,
 				    struct net_devmem_dmabuf_binding *binding,
 				    struct netlink_ext_ack *extack);
-void dev_dmabuf_uninstall(struct net_device *dev);
 static inline struct dmabuf_genpool_chunk_owner *
-net_iov_owner(const struct net_iov *niov)
+net_devmem_iov_to_chunk_owner(const struct net_iov *niov)
 {
-	return niov->owner;
+	struct net_iov_area *owner = net_iov_owner(niov);
+	return container_of(owner, struct dmabuf_genpool_chunk_owner, area);
 }
-static inline unsigned int net_iov_idx(const struct net_iov *niov)
+static inline struct net_devmem_dmabuf_binding *
+net_devmem_iov_binding(const struct net_iov *niov)
 {
-	return niov - net_iov_owner(niov)->niovs;
+	return net_devmem_iov_to_chunk_owner(niov)->binding;
 }
-static inline struct net_devmem_dmabuf_binding *
+static inline u32 net_devmem_iov_binding_id(const struct net_iov *niov)
-net_iov_binding(const struct net_iov *niov)
 {
-	return net_iov_owner(niov)->binding;
+	return net_devmem_iov_binding(niov)->id;
 }
 static inline unsigned long net_iov_virtual_addr(const struct net_iov *niov)
 {
-	struct dmabuf_genpool_chunk_owner *owner = net_iov_owner(niov);
+	struct net_iov_area *owner = net_iov_owner(niov);
 	return owner->base_virtual +
 	       ((unsigned long)net_iov_idx(niov) << PAGE_SHIFT);
 }
-static inline u32 net_iov_binding_id(const struct net_iov *niov)
-{
-	return net_iov_owner(niov)->binding->id;
-}
 static inline void
 net_devmem_dmabuf_binding_get(struct net_devmem_dmabuf_binding *binding)
 {
 net_devmem_alloc_dmabuf(struct net_devmem_dmabuf_binding *binding);
 void net_devmem_free_dmabuf(struct net_iov *ppiov);
+bool net_is_devmem_iov(struct net_iov *niov);
 #else
 struct net_devmem_dmabuf_binding;
 	return -EOPNOTSUPP;
 }
-static inline void dev_dmabuf_uninstall(struct net_device *dev)
-{
-}
 static inline struct net_iov *
 net_devmem_alloc_dmabuf(struct net_devmem_dmabuf_binding *binding)
 {
 	return 0;
 }
-static inline u32 net_iov_binding_id(const struct net_iov *niov)
+static inline u32 net_devmem_iov_binding_id(const struct net_iov *niov)
 {
 	return 0;
 }
+static inline bool net_is_devmem_iov(struct net_iov *niov)
+{
+	return false;
+}
 #endif
 #endif /* _NET_DEVMEM_H */

net/core/netdev-genl.c+6−5

 #include <net/sock.h>
 #include <net/xdp.h>
 #include <net/xdp_sock.h>
+#include <net/page_pool/memory_provider.h>
 #include "dev.h"
 #include "devmem.h"
 netdev_nl_queue_fill_one(struct sk_buff *rsp, struct net_device *netdev,
 			 u32 q_idx, u32 q_type, const struct genl_info *info)
 {
-	struct net_devmem_dmabuf_binding *binding;
+	struct pp_memory_provider_params *params;
 	struct netdev_rx_queue *rxq;
 	struct netdev_queue *txq;
 	void *hdr;
 	switch (q_type) {
 	case NETDEV_QUEUE_TYPE_RX:
 		rxq = __netif_get_rx_queue(netdev, q_idx);
 		if (rxq->napi && nla_put_u32(rsp, NETDEV_A_QUEUE_NAPI_ID,
 					     rxq->napi->napi_id))
 			goto nla_put_failure;
-		binding = rxq->mp_params.mp_priv;
+		params = &rxq->mp_params;
-		if (binding &&
+		if (params->mp_ops &&
-		    nla_put_u32(rsp, NETDEV_A_QUEUE_DMABUF, binding->id))
+		    params->mp_ops->nl_fill(params->mp_priv, rsp, rxq))
 			goto nla_put_failure;
 		break;
 	case NETDEV_QUEUE_TYPE_TX:
 		txq = netdev_get_tx_queue(netdev, q_idx);

net/core/netdev_rx_queue.c+69−0

@@ -3,6 +3,7 @@ · 2 unmodified lines
3	#include <linux/netdevice.h>	3	#include <linux/netdevice.h>
4	#include <net/netdev_queues.h>	4	#include <net/netdev_queues.h>
5	#include <net/netdev_rx_queue.h>	5	#include <net/netdev_rx_queue.h>
		6	#include <net/page_pool/memory_provider.h>
6		7
7	#include "page_pool_priv.h"	8	#include "page_pool_priv.h"
8		9
@@ -80,3 +81,71 @@ err_free_new_mem: · 71 unmodified lines
80	return err;	81	return err;
81	}	82	}
82	EXPORT_SYMBOL_NS_GPL(netdev_rx_queue_restart, "NETDEV_INTERNAL");	83	EXPORT_SYMBOL_NS_GPL(netdev_rx_queue_restart, "NETDEV_INTERNAL");
		84
		85	static int __net_mp_open_rxq(struct net_device *dev, unsigned ifq_idx,
		86	struct pp_memory_provider_params *p)
		87	{
		88	struct netdev_rx_queue *rxq;
		89	int ret;
		90
		91	if (ifq_idx >= dev->real_num_rx_queues)
		92	return -EINVAL;
		93	ifq_idx = array_index_nospec(ifq_idx, dev->real_num_rx_queues);
		94
		95	rxq = __netif_get_rx_queue(dev, ifq_idx);
		96	if (rxq->mp_params.mp_ops)
		97	return -EEXIST;
		98
		99	rxq->mp_params = *p;
		100	ret = netdev_rx_queue_restart(dev, ifq_idx);
		101	if (ret) {
		102	rxq->mp_params.mp_ops = NULL;
		103	rxq->mp_params.mp_priv = NULL;
		104	}
		105	return ret;
		106	}
		107
		108	int net_mp_open_rxq(struct net_device *dev, unsigned ifq_idx,
		109	struct pp_memory_provider_params *p)
		110	{
		111	int ret;
		112
		113	rtnl_lock();
		114	ret = __net_mp_open_rxq(dev, ifq_idx, p);
		115	rtnl_unlock();
		116	return ret;
		117	}
		118
		119	static void __net_mp_close_rxq(struct net_device *dev, unsigned ifq_idx,
		120	struct pp_memory_provider_params *old_p)
		121	{
		122	struct netdev_rx_queue *rxq;
		123
		124	if (WARN_ON_ONCE(ifq_idx >= dev->real_num_rx_queues))
		125	return;
		126
		127	rxq = __netif_get_rx_queue(dev, ifq_idx);
		128
		129	/* Callers holding a netdev ref may get here after we already
		130	* went thru shutdown via dev_memory_provider_uninstall().
		131	*/
		132	if (dev->reg_state > NETREG_REGISTERED &&
		133	!rxq->mp_params.mp_ops)
		134	return;
		135
		136	if (WARN_ON_ONCE(rxq->mp_params.mp_ops != old_p->mp_ops \|\|
		137	rxq->mp_params.mp_priv != old_p->mp_priv))
		138	return;
		139
		140	rxq->mp_params.mp_ops = NULL;
		141	rxq->mp_params.mp_priv = NULL;
		142	WARN_ON(netdev_rx_queue_restart(dev, ifq_idx));
		143	}
		144
		145	void net_mp_close_rxq(struct net_device *dev, unsigned ifq_idx,
		146	struct pp_memory_provider_params *old_p)
		147	{
		148	rtnl_lock();
		149	__net_mp_close_rxq(dev, ifq_idx, old_p);
		150	rtnl_unlock();
		151	}

net/core/page_pool.c+43−8

 #include <net/netdev_rx_queue.h>
 #include <net/page_pool/helpers.h>
+#include <net/page_pool/memory_provider.h>
 #include <net/xdp.h>
 #include <linux/dma-direction.h>
 		rxq = __netif_get_rx_queue(pool->slow.netdev,
 					   pool->slow.queue_idx);
 		pool->mp_priv = rxq->mp_params.mp_priv;
+		pool->mp_ops = rxq->mp_params.mp_ops;
 	}
-	if (pool->mp_priv) {
+	if (pool->mp_ops) {
 		if (!pool->dma_map || !pool->dma_sync)
 			return -EOPNOTSUPP;
-		err = mp_dmabuf_devmem_init(pool);
+		if (WARN_ON(!is_kernel_rodata((unsigned long)pool->mp_ops))) {
+			err = -EFAULT;
+			goto free_ptr_ring;
+		}
+		err = pool->mp_ops->init(pool);
 		if (err) {
 			pr_warn("%s() mem-provider init failed %d\n", __func__,
 				err);
 		return netmem;
 	/* Slow-path: cache empty, do real allocation */
-	if (static_branch_unlikely(&page_pool_mem_providers) && pool->mp_priv)
+	if (static_branch_unlikely(&page_pool_mem_providers) && pool->mp_ops)
-		netmem = mp_dmabuf_devmem_alloc_netmems(pool, gfp);
+		netmem = pool->mp_ops->alloc_netmems(pool, gfp);
 	else
 		netmem = __page_pool_alloc_pages_slow(pool, gfp);
 	return netmem;
 	bool put;
 	put = true;
-	if (static_branch_unlikely(&page_pool_mem_providers) && pool->mp_priv)
+	if (static_branch_unlikely(&page_pool_mem_providers) && pool->mp_ops)
-		put = mp_dmabuf_devmem_release_page(pool, netmem);
+		put = pool->mp_ops->release_netmem(pool, netmem);
 	else
 		__page_pool_release_page_dma(pool, netmem);
 	page_pool_unlist(pool);
 	page_pool_uninit(pool);
-	if (pool->mp_priv) {
+	if (pool->mp_ops) {
-		mp_dmabuf_devmem_destroy(pool);
+		pool->mp_ops->destroy(pool);
 		static_branch_dec(&page_pool_mem_providers);
 	}
 	}
 }
 EXPORT_SYMBOL(page_pool_update_nid);
+bool net_mp_niov_set_dma_addr(struct net_iov *niov, dma_addr_t addr)
+{
+	return page_pool_set_dma_addr_netmem(net_iov_to_netmem(niov), addr);
+}
+/* Associate a niov with a page pool. Should follow with a matching
+ * net_mp_niov_clear_page_pool()
+ */
+void net_mp_niov_set_page_pool(struct page_pool *pool, struct net_iov *niov)
+{
+	netmem_ref netmem = net_iov_to_netmem(niov);
+	page_pool_set_pp_info(pool, netmem);
+	pool->pages_state_hold_cnt++;
+	trace_page_pool_state_hold(pool, netmem, pool->pages_state_hold_cnt);
+}
+/* Disassociate a niov from a page pool. Should only be used in the
+ * ->release_netmem() path.
+ */
+void net_mp_niov_clear_page_pool(struct net_iov *niov)
+{
+	netmem_ref netmem = net_iov_to_netmem(niov);
+	page_pool_clear_pp_info(netmem);
+}

net/core/page_pool_user.c+3−4

 #include <net/netdev_rx_queue.h>
 #include <net/page_pool/helpers.h>
 #include <net/page_pool/types.h>
+#include <net/page_pool/memory_provider.h>
 #include <net/sock.h>
-#include "devmem.h"
 #include "page_pool_priv.h"
 #include "netdev-genl-gen.h"
 page_pool_nl_fill(struct sk_buff *rsp, const struct page_pool *pool,
 		  const struct genl_info *info)
 {
-	struct net_devmem_dmabuf_binding *binding = pool->mp_priv;
 	size_t inflight, refsz;
 	unsigned int napi_id;
 	void *hdr;
 			 pool->user.detach_time))
 		goto err_cancel;
-	if (binding && nla_put_u32(rsp, NETDEV_A_PAGE_POOL_DMABUF, binding->id))
+	if (pool->mp_ops && pool->mp_ops->nl_fill(pool->mp_priv, rsp, NULL))
 		goto err_cancel;
 	genlmsg_end(rsp, hdr);
 int page_pool_check_memory_provider(struct net_device *dev,
 				    struct netdev_rx_queue *rxq)
 {
-	struct net_devmem_dmabuf_binding *binding = rxq->mp_params.mp_priv;
+	void *binding = rxq->mp_params.mp_priv;
 	struct page_pool *pool;
 	struct hlist_node *n;

net/ipv4/tcp.c+6−1

 			}
 			niov = skb_frag_net_iov(frag);
+			if (!net_is_devmem_iov(niov)) {
+				err = -ENODEV;
+				goto out;
+			}
 			end = start + skb_frag_size(frag);
 			copy = end - offset;
 				/* Will perform the exchange later */
 				dmabuf_cmsg.frag_token = tcp_xa_pool.tokens[tcp_xa_pool.idx];
-				dmabuf_cmsg.dmabuf_id = net_iov_binding_id(niov);
+				dmabuf_cmsg.dmabuf_id = net_devmem_iov_binding_id(niov);
 				offset += copy;
 				remaining_len -= copy;

tools/include/uapi/linux/netdev.h+7−0

 	NETDEV_A_DEV_MAX = (__NETDEV_A_DEV_MAX - 1)
 };
+enum {
+	__NETDEV_A_IO_URING_PROVIDER_INFO_MAX,
+	NETDEV_A_IO_URING_PROVIDER_INFO_MAX = (__NETDEV_A_IO_URING_PROVIDER_INFO_MAX - 1)
+};
 enum {
 	NETDEV_A_PAGE_POOL_ID = 1,
 	NETDEV_A_PAGE_POOL_IFINDEX,
 	NETDEV_A_PAGE_POOL_INFLIGHT_MEM,
 	NETDEV_A_PAGE_POOL_DETACH_TIME,
 	NETDEV_A_PAGE_POOL_DMABUF,
+	NETDEV_A_PAGE_POOL_IO_URING,
 	__NETDEV_A_PAGE_POOL_MAX,
 	NETDEV_A_PAGE_POOL_MAX = (__NETDEV_A_PAGE_POOL_MAX - 1)
 	NETDEV_A_QUEUE_TYPE,
 	NETDEV_A_QUEUE_NAPI_ID,
 	NETDEV_A_QUEUE_DMABUF,
+	NETDEV_A_QUEUE_IO_URING,
 	__NETDEV_A_QUEUE_MAX,
 	NETDEV_A_QUEUE_MAX = (__NETDEV_A_QUEUE_MAX - 1)

Merge branch 'io_uring-zero-copy-rx'

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code