History

netdev: support binding dma-buf to netdevice

Add a netdev_dmabuf_binding struct which represents the dma-buf-to-netdevice binding. The netlink API will bind the dma-buf to rx queues on the netdevice. On the binding, the dma_buf_attach & dma_buf_map_attachment will occur. The entries in the sg_table from mapping will be inserted into a genpool to make it ready for allocation. The chunks in the genpool are owned by a dmabuf_chunk_owner struct which holds the dma-buf offset of the base of the chunk and the dma_addr of the chunk. Both are needed to use allocations that come from this chunk. We create a new type that represents an allocation from the genpool: net_iov. We setup the net_iov allocation size in the genpool to PAGE_SIZE for simplicity: to match the PAGE_SIZE normally allocated by the page pool and given to the drivers. The user can unbind the dmabuf from the netdevice by closing the netlink socket that established the binding. We do this so that the binding is automatically unbound even if the userspace process crashes. The binding and unbinding leaves an indicator in struct netdev_rx_queue that the given queue is bound, and the binding is actuated by resetting the rx queue using the queue API. The netdev_dmabuf_binding struct is refcounted, and releases its resources only when all the refs are released.

Mina Almasry committed 2 years ago

Commit 170aafe1 parent 3efd7ab15 files · +596 −5Browse files

Unified · Split

15 files
  1. Documentation/netlink/specs/netdev.yaml +4 −0
  2. include/linux/netdevice.h +2 −0
  3. include/net/netdev_rx_queue.h +2 −0
  4. include/net/netmem.h +8 −0
  5. include/net/page_pool/types.h +6 −0
  6. net/Kconfig +5 −0
  7. net/core/Makefile +1 −0
  8. net/core/dev.c +26 −0
  9. net/core/devmem.c +274 −0
  10. net/core/devmem.h +124 −0
  11. net/core/netdev-genl-gen.c +4 −0
  12. net/core/netdev-genl-gen.h +4 −0
  13. net/core/netdev-genl.c +123 −5
  14. net/ethtool/common.c +8 −0
  15. net/xdp/xsk_buff_pool.c +5 −0
Documentation/netlink/specs/netdev.yaml+4−0
@@ -667,6 +667,10 @@ operations: · 666 unmodified lines
667 attributes:667 attributes:
668 - id668 - id
669 669
670kernel-family:
671 headers: [ "linux/list.h"]
672 sock-priv: struct list_head
673
670mcast-groups:674mcast-groups:
671 list:675 list:
672 -676 -
include/linux/netdevice.h+2−0
@@ -3953,6 +3953,8 @@ u8 dev_xdp_prog_count(struct net_device *dev); · 3952 unmodified lines
3953int dev_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf);3953int dev_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf);
3954u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode);3954u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode);
3955 3955
3956u32 dev_get_min_mp_channel_count(const struct net_device *dev);
3957
3956int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb);3958int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
3957int dev_forward_skb(struct net_device *dev, struct sk_buff *skb);3959int dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
3958int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb);3960int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb);
include/net/netdev_rx_queue.h+2−0
@@ -6,6 +6,7 @@ · 5 unmodified lines
6#include <linux/netdevice.h>6#include <linux/netdevice.h>
7#include <linux/sysfs.h>7#include <linux/sysfs.h>
8#include <net/xdp.h>8#include <net/xdp.h>
9#include <net/page_pool/types.h>
9 10
10/* This structure contains an instance of an RX queue. */11/* This structure contains an instance of an RX queue. */
11struct netdev_rx_queue {12struct netdev_rx_queue {
@@ -25,6 +26,7 @@ struct netdev_rx_queue { · 13 unmodified lines
25 * Readers and writers must hold RTNL26 * Readers and writers must hold RTNL
26 */27 */
27 struct napi_struct *napi;28 struct napi_struct *napi;
29 struct pp_memory_provider_params mp_params;
28} ____cacheline_aligned_in_smp;30} ____cacheline_aligned_in_smp;
29 31
30/*32/*
include/net/netmem.h+8−0
@@ -8,6 +8,14 @@ · 7 unmodified lines
8#ifndef _NET_NETMEM_H8#ifndef _NET_NETMEM_H
9#define _NET_NETMEM_H9#define _NET_NETMEM_H
10 10
11/* net_iov */
12
13struct net_iov {
14 struct dmabuf_genpool_chunk_owner *owner;
15};
16
17/* netmem */
18
11/**19/**
12 * typedef netmem_ref - a nonexistent type marking a reference to generic20 * typedef netmem_ref - a nonexistent type marking a reference to generic
13 * network memory.21 * network memory.
include/net/page_pool/types.h+6−0
@@ -139,6 +139,10 @@ struct page_pool_stats { · 138 unmodified lines
139 */139 */
140#define PAGE_POOL_FRAG_GROUP_ALIGN (4 * sizeof(long))140#define PAGE_POOL_FRAG_GROUP_ALIGN (4 * sizeof(long))
141 141
142struct pp_memory_provider_params {
143 void *mp_priv;
144};
145
142struct page_pool {146struct page_pool {
143 struct page_pool_params_fast p;147 struct page_pool_params_fast p;
144 148
@@ -197,6 +201,8 @@ struct page_pool { · 52 unmodified lines
197 */201 */
198 struct ptr_ring ring;202 struct ptr_ring ring;
199 203
204 void *mp_priv;
205
200#ifdef CONFIG_PAGE_POOL_STATS206#ifdef CONFIG_PAGE_POOL_STATS
201 /* recycle stats are per-cpu to avoid locking */207 /* recycle stats are per-cpu to avoid locking */
202 struct page_pool_recycle_stats __percpu *recycle_stats;208 struct page_pool_recycle_stats __percpu *recycle_stats;
net/Kconfig+5−0
@@ -66,6 +66,11 @@ config SKB_DECRYPTED · 65 unmodified lines
66config SKB_EXTENSIONS66config SKB_EXTENSIONS
67 bool67 bool
68 68
69config NET_DEVMEM
70 def_bool y
71 depends on DMA_SHARED_BUFFER
72 depends on GENERIC_ALLOCATOR
73
69menu "Networking options"74menu "Networking options"
70 75
71source "net/packet/Kconfig"76source "net/packet/Kconfig"
net/core/Makefile+1−0
@@ -44,3 +44,4 @@ obj-$(CONFIG_BPF_SYSCALL) += sock_map.o · 43 unmodified lines
44obj-$(CONFIG_BPF_SYSCALL) += bpf_sk_storage.o44obj-$(CONFIG_BPF_SYSCALL) += bpf_sk_storage.o
45obj-$(CONFIG_OF) += of_net.o45obj-$(CONFIG_OF) += of_net.o
46obj-$(CONFIG_NET_TEST) += net_test.o46obj-$(CONFIG_NET_TEST) += net_test.o
47obj-$(CONFIG_NET_DEVMEM) += devmem.o
net/core/dev.c+26−0
@@ -161,6 +161,7 @@ · 160 unmodified lines
161#include <linux/phy_link_topology.h>161#include <linux/phy_link_topology.h>
162 162
163#include "dev.h"163#include "dev.h"
164#include "devmem.h"
164#include "net-sysfs.h"165#include "net-sysfs.h"
165 166
166static DEFINE_SPINLOCK(ptype_lock);167static DEFINE_SPINLOCK(ptype_lock);
@@ -9367,6 +9368,11 @@ int dev_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf) · 9200 unmodified lines
9367 if (!dev->netdev_ops->ndo_bpf)9368 if (!dev->netdev_ops->ndo_bpf)
9368 return -EOPNOTSUPP;9369 return -EOPNOTSUPP;
9369 9370
9371 if (dev_get_min_mp_channel_count(dev)) {
9372 NL_SET_ERR_MSG(bpf->extack, "unable to propagate XDP to device using memory provider");
9373 return -EBUSY;
9374 }
9375
9370 return dev->netdev_ops->ndo_bpf(dev, bpf);9376 return dev->netdev_ops->ndo_bpf(dev, bpf);
9371}9377}
9372EXPORT_SYMBOL_GPL(dev_xdp_propagate);9378EXPORT_SYMBOL_GPL(dev_xdp_propagate);
@@ -9399,6 +9405,11 @@ static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode, · 26 unmodified lines
9399 struct netdev_bpf xdp;9405 struct netdev_bpf xdp;
9400 int err;9406 int err;
9401 9407
9408 if (dev_get_min_mp_channel_count(dev)) {
9409 NL_SET_ERR_MSG(extack, "unable to install XDP to device using memory provider");
9410 return -EBUSY;
9411 }
9412
9402 memset(&xdp, 0, sizeof(xdp));9413 memset(&xdp, 0, sizeof(xdp));
9403 xdp.command = mode == XDP_MODE_HW ? XDP_SETUP_PROG_HW : XDP_SETUP_PROG;9414 xdp.command = mode == XDP_MODE_HW ? XDP_SETUP_PROG_HW : XDP_SETUP_PROG;
9404 xdp.extack = extack;9415 xdp.extack = extack;
@@ -9823,6 +9834,20 @@ err_out: · 418 unmodified lines
9823 return err;9834 return err;
9824}9835}
9825 9836
9837u32 dev_get_min_mp_channel_count(const struct net_device *dev)
9838{
9839 int i;
9840
9841 ASSERT_RTNL();
9842
9843 for (i = dev->real_num_rx_queues - 1; i >= 0; i--)
9844 if (dev->_rx[i].mp_params.mp_priv)
9845 /* The channel count is the idx plus 1. */
9846 return i + 1;
9847
9848 return 0;
9849}
9850
9826/**9851/**
9827 * dev_index_reserve() - allocate an ifindex in a namespace9852 * dev_index_reserve() - allocate an ifindex in a namespace
9828 * @net: the applicable net namespace9853 * @net: the applicable net namespace
@@ -11359,6 +11384,7 @@ void unregister_netdevice_many_notify(struct list_head *head, · 1530 unmodified lines
11359 dev_tcx_uninstall(dev);11384 dev_tcx_uninstall(dev);
11360 dev_xdp_uninstall(dev);11385 dev_xdp_uninstall(dev);
11361 bpf_dev_bound_netdev_unregister(dev);11386 bpf_dev_bound_netdev_unregister(dev);
11387 dev_dmabuf_uninstall(dev);
11362 11388
11363 netdev_offload_xstats_disable_all(dev);11389 netdev_offload_xstats_disable_all(dev);
11364 11390
net/core/devmem.c+274−0
@@ -0,0 +1,274 @@
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Devmem TCP
4 *
5 * Authors: Mina Almasry <almasrymina@google.com>
6 * Willem de Bruijn <willemdebruijn.kernel@gmail.com>
7 * Kaiyuan Zhang <kaiyuanz@google.com
8 */
9
10#include <linux/dma-buf.h>
11#include <linux/genalloc.h>
12#include <linux/mm.h>
13#include <linux/netdevice.h>
14#include <linux/types.h>
15#include <net/netdev_queues.h>
16#include <net/netdev_rx_queue.h>
17#include <net/page_pool/helpers.h>
18#include <trace/events/page_pool.h>
19
20#include "devmem.h"
21
22/* Device memory support */
23
24/* Protected by rtnl_lock() */
25static DEFINE_XARRAY_FLAGS(net_devmem_dmabuf_bindings, XA_FLAGS_ALLOC1);
26
27static void net_devmem_dmabuf_free_chunk_owner(struct gen_pool *genpool,
28 struct gen_pool_chunk *chunk,
29 void *not_used)
30{
31 struct dmabuf_genpool_chunk_owner *owner = chunk->owner;
32
33 kvfree(owner->niovs);
34 kfree(owner);
35}
36
37void __net_devmem_dmabuf_binding_free(struct net_devmem_dmabuf_binding *binding)
38{
39 size_t size, avail;
40
41 gen_pool_for_each_chunk(binding->chunk_pool,
42 net_devmem_dmabuf_free_chunk_owner, NULL);
43
44 size = gen_pool_size(binding->chunk_pool);
45 avail = gen_pool_avail(binding->chunk_pool);
46
47 if (!WARN(size != avail, "can't destroy genpool. size=%zu, avail=%zu",
48 size, avail))
49 gen_pool_destroy(binding->chunk_pool);
50
51 dma_buf_unmap_attachment_unlocked(binding->attachment, binding->sgt,
52 DMA_FROM_DEVICE);
53 dma_buf_detach(binding->dmabuf, binding->attachment);
54 dma_buf_put(binding->dmabuf);
55 xa_destroy(&binding->bound_rxqs);
56 kfree(binding);
57}
58
59void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding)
60{
61 struct netdev_rx_queue *rxq;
62 unsigned long xa_idx;
63 unsigned int rxq_idx;
64
65 if (binding->list.next)
66 list_del(&binding->list);
67
68 xa_for_each(&binding->bound_rxqs, xa_idx, rxq) {
69 WARN_ON(rxq->mp_params.mp_priv != binding);
70
71 rxq->mp_params.mp_priv = NULL;
72
73 rxq_idx = get_netdev_rx_queue_index(rxq);
74
75 WARN_ON(netdev_rx_queue_restart(binding->dev, rxq_idx));
76 }
77
78 xa_erase(&net_devmem_dmabuf_bindings, binding->id);
79
80 net_devmem_dmabuf_binding_put(binding);
81}
82
83int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx,
84 struct net_devmem_dmabuf_binding *binding,
85 struct netlink_ext_ack *extack)
86{
87 struct netdev_rx_queue *rxq;
88 u32 xa_idx;
89 int err;
90
91 if (rxq_idx >= dev->real_num_rx_queues) {
92 NL_SET_ERR_MSG(extack, "rx queue index out of range");
93 return -ERANGE;
94 }
95
96 rxq = __netif_get_rx_queue(dev, rxq_idx);
97 if (rxq->mp_params.mp_priv) {
98 NL_SET_ERR_MSG(extack, "designated queue already memory provider bound");
99 return -EEXIST;
100 }
101
102#ifdef CONFIG_XDP_SOCKETS
103 if (rxq->pool) {
104 NL_SET_ERR_MSG(extack, "designated queue already in use by AF_XDP");
105 return -EBUSY;
106 }
107#endif
108
109 err = xa_alloc(&binding->bound_rxqs, &xa_idx, rxq, xa_limit_32b,
110 GFP_KERNEL);
111 if (err)
112 return err;
113
114 rxq->mp_params.mp_priv = binding;
115
116 err = netdev_rx_queue_restart(dev, rxq_idx);
117 if (err)
118 goto err_xa_erase;
119
120 return 0;
121
122err_xa_erase:
123 rxq->mp_params.mp_priv = NULL;
124 xa_erase(&binding->bound_rxqs, xa_idx);
125
126 return err;
127}
128
129struct net_devmem_dmabuf_binding *
130net_devmem_bind_dmabuf(struct net_device *dev, unsigned int dmabuf_fd,
131 struct netlink_ext_ack *extack)
132{
133 struct net_devmem_dmabuf_binding *binding;
134 static u32 id_alloc_next;
135 struct scatterlist *sg;
136 struct dma_buf *dmabuf;
137 unsigned int sg_idx, i;
138 unsigned long virtual;
139 int err;
140
141 dmabuf = dma_buf_get(dmabuf_fd);
142 if (IS_ERR(dmabuf))
143 return ERR_CAST(dmabuf);
144
145 binding = kzalloc_node(sizeof(*binding), GFP_KERNEL,
146 dev_to_node(&dev->dev));
147 if (!binding) {
148 err = -ENOMEM;
149 goto err_put_dmabuf;
150 }
151
152 binding->dev = dev;
153
154 err = xa_alloc_cyclic(&net_devmem_dmabuf_bindings, &binding->id,
155 binding, xa_limit_32b, &id_alloc_next,
156 GFP_KERNEL);
157 if (err < 0)
158 goto err_free_binding;
159
160 xa_init_flags(&binding->bound_rxqs, XA_FLAGS_ALLOC);
161
162 refcount_set(&binding->ref, 1);
163
164 binding->dmabuf = dmabuf;
165
166 binding->attachment = dma_buf_attach(binding->dmabuf, dev->dev.parent);
167 if (IS_ERR(binding->attachment)) {
168 err = PTR_ERR(binding->attachment);
169 NL_SET_ERR_MSG(extack, "Failed to bind dmabuf to device");
170 goto err_free_id;
171 }
172
173 binding->sgt = dma_buf_map_attachment_unlocked(binding->attachment,
174 DMA_FROM_DEVICE);
175 if (IS_ERR(binding->sgt)) {
176 err = PTR_ERR(binding->sgt);
177 NL_SET_ERR_MSG(extack, "Failed to map dmabuf attachment");
178 goto err_detach;
179 }
180
181 /* For simplicity we expect to make PAGE_SIZE allocations, but the
182 * binding can be much more flexible than that. We may be able to
183 * allocate MTU sized chunks here. Leave that for future work...
184 */
185 binding->chunk_pool =
186 gen_pool_create(PAGE_SHIFT, dev_to_node(&dev->dev));
187 if (!binding->chunk_pool) {
188 err = -ENOMEM;
189 goto err_unmap;
190 }
191
192 virtual = 0;
193 for_each_sgtable_dma_sg(binding->sgt, sg, sg_idx) {
194 dma_addr_t dma_addr = sg_dma_address(sg);
195 struct dmabuf_genpool_chunk_owner *owner;
196 size_t len = sg_dma_len(sg);
197 struct net_iov *niov;
198
199 owner = kzalloc_node(sizeof(*owner), GFP_KERNEL,
200 dev_to_node(&dev->dev));
201 if (!owner) {
202 err = -ENOMEM;
203 goto err_free_chunks;
204 }
205
206 owner->base_virtual = virtual;
207 owner->base_dma_addr = dma_addr;
208 owner->num_niovs = len / PAGE_SIZE;
209 owner->binding = binding;
210
211 err = gen_pool_add_owner(binding->chunk_pool, dma_addr,
212 dma_addr, len, dev_to_node(&dev->dev),
213 owner);
214 if (err) {
215 kfree(owner);
216 err = -EINVAL;
217 goto err_free_chunks;
218 }
219
220 owner->niovs = kvmalloc_array(owner->num_niovs,
221 sizeof(*owner->niovs),
222 GFP_KERNEL);
223 if (!owner->niovs) {
224 err = -ENOMEM;
225 goto err_free_chunks;
226 }
227
228 for (i = 0; i < owner->num_niovs; i++) {
229 niov = &owner->niovs[i];
230 niov->owner = owner;
231 }
232
233 virtual += len;
234 }
235
236 return binding;
237
238err_free_chunks:
239 gen_pool_for_each_chunk(binding->chunk_pool,
240 net_devmem_dmabuf_free_chunk_owner, NULL);
241 gen_pool_destroy(binding->chunk_pool);
242err_unmap:
243 dma_buf_unmap_attachment_unlocked(binding->attachment, binding->sgt,
244 DMA_FROM_DEVICE);
245err_detach:
246 dma_buf_detach(dmabuf, binding->attachment);
247err_free_id:
248 xa_erase(&net_devmem_dmabuf_bindings, binding->id);
249err_free_binding:
250 kfree(binding);
251err_put_dmabuf:
252 dma_buf_put(dmabuf);
253 return ERR_PTR(err);
254}
255
256void dev_dmabuf_uninstall(struct net_device *dev)
257{
258 struct net_devmem_dmabuf_binding *binding;
259 struct netdev_rx_queue *rxq;
260 unsigned long xa_idx;
261 unsigned int i;
262
263 for (i = 0; i < dev->real_num_rx_queues; i++) {
264 binding = dev->_rx[i].mp_params.mp_priv;
265 if (!binding)
266 continue;
267
268 xa_for_each(&binding->bound_rxqs, xa_idx, rxq)
269 if (rxq == &dev->_rx[i]) {
270 xa_erase(&binding->bound_rxqs, xa_idx);
271 break;
272 }
273 }
274}
net/core/devmem.h+124−0
@@ -0,0 +1,124 @@
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * Device memory TCP support
4 *
5 * Authors: Mina Almasry <almasrymina@google.com>
6 * Willem de Bruijn <willemb@google.com>
7 * Kaiyuan Zhang <kaiyuanz@google.com>
8 *
9 */
10#ifndef _NET_DEVMEM_H
11#define _NET_DEVMEM_H
12
13struct netlink_ext_ack;
14
15struct net_devmem_dmabuf_binding {
16 struct dma_buf *dmabuf;
17 struct dma_buf_attachment *attachment;
18 struct sg_table *sgt;
19 struct net_device *dev;
20 struct gen_pool *chunk_pool;
21
22 /* The user holds a ref (via the netlink API) for as long as they want
23 * the binding to remain alive. Each page pool using this binding holds
24 * a ref to keep the binding alive. Each allocated net_iov holds a
25 * ref.
26 *
27 * The binding undos itself and unmaps the underlying dmabuf once all
28 * those refs are dropped and the binding is no longer desired or in
29 * use.
30 */
31 refcount_t ref;
32
33 /* The list of bindings currently active. Used for netlink to notify us
34 * of the user dropping the bind.
35 */
36 struct list_head list;
37
38 /* rxq's this binding is active on. */
39 struct xarray bound_rxqs;
40
41 /* ID of this binding. Globally unique to all bindings currently
42 * active.
43 */
44 u32 id;
45};
46
47#if defined(CONFIG_NET_DEVMEM)
48/* Owner of the dma-buf chunks inserted into the gen pool. Each scatterlist
49 * entry from the dmabuf is inserted into the genpool as a chunk, and needs
50 * this owner struct to keep track of some metadata necessary to create
51 * allocations from this chunk.
52 */
53struct dmabuf_genpool_chunk_owner {
54 /* Offset into the dma-buf where this chunk starts. */
55 unsigned long base_virtual;
56
57 /* dma_addr of the start of the chunk. */
58 dma_addr_t base_dma_addr;
59
60 /* Array of net_iovs for this chunk. */
61 struct net_iov *niovs;
62 size_t num_niovs;
63
64 struct net_devmem_dmabuf_binding *binding;
65};
66
67void __net_devmem_dmabuf_binding_free(struct net_devmem_dmabuf_binding *binding);
68struct net_devmem_dmabuf_binding *
69net_devmem_bind_dmabuf(struct net_device *dev, unsigned int dmabuf_fd,
70 struct netlink_ext_ack *extack);
71void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding);
72int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx,
73 struct net_devmem_dmabuf_binding *binding,
74 struct netlink_ext_ack *extack);
75void dev_dmabuf_uninstall(struct net_device *dev);
76
77static inline void
78net_devmem_dmabuf_binding_get(struct net_devmem_dmabuf_binding *binding)
79{
80 refcount_inc(&binding->ref);
81}
82
83static inline void
84net_devmem_dmabuf_binding_put(struct net_devmem_dmabuf_binding *binding)
85{
86 if (!refcount_dec_and_test(&binding->ref))
87 return;
88
89 __net_devmem_dmabuf_binding_free(binding);
90}
91
92#else
93static inline void
94__net_devmem_dmabuf_binding_free(struct net_devmem_dmabuf_binding *binding)
95{
96}
97
98static inline struct net_devmem_dmabuf_binding *
99net_devmem_bind_dmabuf(struct net_device *dev, unsigned int dmabuf_fd,
100 struct netlink_ext_ack *extack)
101{
102 return ERR_PTR(-EOPNOTSUPP);
103}
104
105static inline void
106net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding)
107{
108}
109
110static inline int
111net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx,
112 struct net_devmem_dmabuf_binding *binding,
113 struct netlink_ext_ack *extack)
114
115{
116 return -EOPNOTSUPP;
117}
118
119static inline void dev_dmabuf_uninstall(struct net_device *dev)
120{
121}
122#endif
123
124#endif /* _NET_DEVMEM_H */
net/core/netdev-genl-gen.c+4−0
@@ -9,6 +9,7 @@ · 8 unmodified lines
9#include "netdev-genl-gen.h"9#include "netdev-genl-gen.h"
10 10
11#include <uapi/linux/netdev.h>11#include <uapi/linux/netdev.h>
12#include <linux/list.h>
12 13
13/* Integer value ranges */14/* Integer value ranges */
14static const struct netlink_range_validation netdev_a_page_pool_id_range = {15static const struct netlink_range_validation netdev_a_page_pool_id_range = {
@@ -187,4 +188,7 @@ struct genl_family netdev_nl_family __ro_after_init = { · 172 unmodified lines
187 .n_split_ops = ARRAY_SIZE(netdev_nl_ops),188 .n_split_ops = ARRAY_SIZE(netdev_nl_ops),
188 .mcgrps = netdev_nl_mcgrps,189 .mcgrps = netdev_nl_mcgrps,
189 .n_mcgrps = ARRAY_SIZE(netdev_nl_mcgrps),190 .n_mcgrps = ARRAY_SIZE(netdev_nl_mcgrps),
191 .sock_priv_size = sizeof(struct list_head),
192 .sock_priv_init = (void *)netdev_nl_sock_priv_init,
193 .sock_priv_destroy = (void *)netdev_nl_sock_priv_destroy,
190};194};
net/core/netdev-genl-gen.h+4−0
@@ -10,6 +10,7 @@ · 9 unmodified lines
10#include <net/genetlink.h>10#include <net/genetlink.h>
11 11
12#include <uapi/linux/netdev.h>12#include <uapi/linux/netdev.h>
13#include <linux/list.h>
13 14
14/* Common nested types */15/* Common nested types */
15extern const struct nla_policy netdev_page_pool_info_nl_policy[NETDEV_A_PAGE_POOL_IFINDEX + 1];16extern const struct nla_policy netdev_page_pool_info_nl_policy[NETDEV_A_PAGE_POOL_IFINDEX + 1];
@@ -40,4 +41,7 @@ enum { · 24 unmodified lines
40 41
41extern struct genl_family netdev_nl_family;42extern struct genl_family netdev_nl_family;
42 43
44void netdev_nl_sock_priv_init(struct list_head *priv);
45void netdev_nl_sock_priv_destroy(struct list_head *priv);
46
43#endif /* _LINUX_NETDEV_GEN_H */47#endif /* _LINUX_NETDEV_GEN_H */
net/core/netdev-genl.c+123−5
@@ -3,16 +3,17 @@ · 2 unmodified lines
3#include <linux/netdevice.h>3#include <linux/netdevice.h>
4#include <linux/notifier.h>4#include <linux/notifier.h>
5#include <linux/rtnetlink.h>5#include <linux/rtnetlink.h>
6#include <net/busy_poll.h>
6#include <net/net_namespace.h>7#include <net/net_namespace.h>
8#include <net/netdev_queues.h>
9#include <net/netdev_rx_queue.h>
7#include <net/sock.h>10#include <net/sock.h>
8#include <net/xdp.h>11#include <net/xdp.h>
9#include <net/xdp_sock.h>12#include <net/xdp_sock.h>
10#include <net/netdev_rx_queue.h>
11#include <net/netdev_queues.h>
12#include <net/busy_poll.h>
13 13
14#include "netdev-genl-gen.h"
15#include "dev.h"14#include "dev.h"
15#include "devmem.h"
16#include "netdev-genl-gen.h"
16 17
17struct netdev_nl_dump_ctx {18struct netdev_nl_dump_ctx {
18 unsigned long ifindex;19 unsigned long ifindex;
@@ -723,10 +724,127 @@ int netdev_nl_qstats_get_dumpit(struct sk_buff *skb, · 704 unmodified lines
723 return err;724 return err;
724}725}
725 726
726/* Stub */
727int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info)727int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info)
728{728{
729 struct nlattr *tb[ARRAY_SIZE(netdev_queue_id_nl_policy)];
730 struct net_devmem_dmabuf_binding *binding;
731 struct list_head *sock_binding_list;
732 u32 ifindex, dmabuf_fd, rxq_idx;
733 struct net_device *netdev;
734 struct sk_buff *rsp;
735 struct nlattr *attr;
736 int rem, err = 0;
737 void *hdr;
738
739 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX) ||
740 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_FD) ||
741 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_QUEUES))
742 return -EINVAL;
743
744 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]);
745 dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]);
746
747 sock_binding_list = genl_sk_priv_get(&netdev_nl_family,
748 NETLINK_CB(skb).sk);
749 if (IS_ERR(sock_binding_list))
750 return PTR_ERR(sock_binding_list);
751
752 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
753 if (!rsp)
754 return -ENOMEM;
755
756 hdr = genlmsg_iput(rsp, info);
757 if (!hdr) {
758 err = -EMSGSIZE;
759 goto err_genlmsg_free;
760 }
761
762 rtnl_lock();
763
764 netdev = __dev_get_by_index(genl_info_net(info), ifindex);
765 if (!netdev || !netif_device_present(netdev)) {
766 err = -ENODEV;
767 goto err_unlock;
768 }
769
770 if (dev_xdp_prog_count(netdev)) {
771 NL_SET_ERR_MSG(info->extack, "unable to bind dmabuf to device with XDP program attached");
772 err = -EEXIST;
773 goto err_unlock;
774 }
775
776 binding = net_devmem_bind_dmabuf(netdev, dmabuf_fd, info->extack);
777 if (IS_ERR(binding)) {
778 err = PTR_ERR(binding);
779 goto err_unlock;
780 }
781
782 nla_for_each_attr_type(attr, NETDEV_A_DMABUF_QUEUES,
783 genlmsg_data(info->genlhdr),
784 genlmsg_len(info->genlhdr), rem) {
785 err = nla_parse_nested(
786 tb, ARRAY_SIZE(netdev_queue_id_nl_policy) - 1, attr,
787 netdev_queue_id_nl_policy, info->extack);
788 if (err < 0)
789 goto err_unbind;
790
791 if (NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_ID) ||
792 NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_TYPE)) {
793 err = -EINVAL;
794 goto err_unbind;
795 }
796
797 if (nla_get_u32(tb[NETDEV_A_QUEUE_TYPE]) != NETDEV_QUEUE_TYPE_RX) {
798 NL_SET_BAD_ATTR(info->extack, tb[NETDEV_A_QUEUE_TYPE]);
799 err = -EINVAL;
800 goto err_unbind;
801 }
802
803 rxq_idx = nla_get_u32(tb[NETDEV_A_QUEUE_ID]);
804
805 err = net_devmem_bind_dmabuf_to_queue(netdev, rxq_idx, binding,
806 info->extack);
807 if (err)
808 goto err_unbind;
809 }
810
811 list_add(&binding->list, sock_binding_list);
812
813 nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id);
814 genlmsg_end(rsp, hdr);
815
816 err = genlmsg_reply(rsp, info);
817 if (err)
818 goto err_unbind;
819
820 rtnl_unlock();
821
729 return 0;822 return 0;
823
824err_unbind:
825 net_devmem_unbind_dmabuf(binding);
826err_unlock:
827 rtnl_unlock();
828err_genlmsg_free:
829 nlmsg_free(rsp);
830 return err;
831}
832
833void netdev_nl_sock_priv_init(struct list_head *priv)
834{
835 INIT_LIST_HEAD(priv);
836}
837
838void netdev_nl_sock_priv_destroy(struct list_head *priv)
839{
840 struct net_devmem_dmabuf_binding *binding;
841 struct net_devmem_dmabuf_binding *temp;
842
843 list_for_each_entry_safe(binding, temp, priv, list) {
844 rtnl_lock();
845 net_devmem_unbind_dmabuf(binding);
846 rtnl_unlock();
847 }
730}848}
731 849
732static int netdev_genl_netdevice_event(struct notifier_block *nb,850static int netdev_genl_netdevice_event(struct notifier_block *nb,
net/ethtool/common.c+8−0
@@ -655,6 +655,7 @@ int ethtool_check_max_channel(struct net_device *dev, · 654 unmodified lines
655{655{
656 u64 max_rxnfc_in_use;656 u64 max_rxnfc_in_use;
657 u32 max_rxfh_in_use;657 u32 max_rxfh_in_use;
658 int max_mp_in_use;
658 659
659 /* ensure the new Rx count fits within the configured Rx flow660 /* ensure the new Rx count fits within the configured Rx flow
660 * indirection table/rxnfc settings661 * indirection table/rxnfc settings
@@ -673,6 +674,13 @@ int ethtool_check_max_channel(struct net_device *dev, · 12 unmodified lines
673 return -EINVAL;674 return -EINVAL;
674 }675 }
675 676
677 max_mp_in_use = dev_get_min_mp_channel_count(dev);
678 if (channels.combined_count + channels.rx_count <= max_mp_in_use) {
679 if (info)
680 GENL_SET_ERR_MSG_FMT(info, "requested channel counts are too low for existing memory provider setting (%d)", max_mp_in_use);
681 return -EINVAL;
682 }
683
676 return 0;684 return 0;
677}685}
678 686
net/xdp/xsk_buff_pool.c+5−0
@@ -211,6 +211,11 @@ int xp_assign_dev(struct xsk_buff_pool *pool, · 210 unmodified lines
211 goto err_unreg_pool;211 goto err_unreg_pool;
212 }212 }
213 213
214 if (dev_get_min_mp_channel_count(netdev)) {
215 err = -EBUSY;
216 goto err_unreg_pool;
217 }
218
214 bpf.command = XDP_SETUP_XSK_POOL;219 bpf.command = XDP_SETUP_XSK_POOL;
215 bpf.xsk.pool = pool;220 bpf.xsk.pool = pool;
216 bpf.xsk.queue_id = queue_id;221 bpf.xsk.queue_id = queue_id;