History

xsk: Add TX timestamp and TX checksum offload support

This change actually defines the (initial) metadata layout that should be used by AF_XDP userspace (xsk_tx_metadata). The first field is flags which requests appropriate offloads, followed by the offload-specific fields. The supported per-device offloads are exported via netlink (new xsk-flags). The offloads themselves are still implemented in a bit of a framework-y fashion that's left from my initial kfunc attempt. I'm introducing new xsk_tx_metadata_ops which drivers are supposed to implement. The drivers are also supposed to call xsk_tx_metadata_request/xsk_tx_metadata_complete in the right places. Since xsk_tx_metadata_{request,_complete} are static inline, we don't incur any extra overhead doing indirect calls. The benefit of this scheme is as follows: - keeps all metadata layout parsing away from driver code - makes it easy to grep and see which drivers implement what - don't need any extra flags to maintain to keep track of what offloads are implemented; if the callback is implemented - the offload is supported (used by netlink reporting code) Two offloads are defined right now: 1. XDP_TXMD_FLAGS_CHECKSUM: skb-style csum_start+csum_offset 2. XDP_TXMD_FLAGS_TIMESTAMP: writes TX timestamp back into metadata area upon completion (tx_timestamp field) XDP_TXMD_FLAGS_TIMESTAMP is also implemented for XDP_COPY mode: it writes SW timestamp from the skb destructor (note I'm reusing hwtstamps to pass metadata pointer). The struct is forward-compatible and can be extended in the future by appending more fields.

Stanislav Fomichev committed 2 years ago

Commit 48eb03d1 parent 341ac9815 files · +348 −9Browse files

Unified · Split

15 files
  1. Documentation/netlink/specs/netdev.yaml +18 −1
  2. include/linux/netdevice.h +2 −0
  3. include/linux/skbuff.h +13 −1
  4. include/net/xdp_sock.h +110 −0
  5. include/net/xdp_sock_drv.h +13 −0
  6. include/net/xsk_buff_pool.h +6 −0
  7. include/uapi/linux/if_xdp.h +38 −0
  8. include/uapi/linux/netdev.h +16 −0
  9. net/core/netdev-genl.c +12 −1
  10. net/xdp/xsk.c +34 −0
  11. net/xdp/xsk_queue.h +1 −1
  12. tools/include/uapi/linux/if_xdp.h +47 −5
  13. tools/include/uapi/linux/netdev.h +16 −0
  14. tools/net/ynl/generated/netdev-user.c +19 −0
  15. tools/net/ynl/generated/netdev-user.h +3 −0
Documentation/netlink/specs/netdev.yaml+18−1
@@ -45,7 +45,6 @@ definitions: · 44 unmodified lines
45 -45 -
46 type: flags46 type: flags
47 name: xdp-rx-metadata47 name: xdp-rx-metadata
48 render-max: true
49 entries:48 entries:
50 -49 -
51 name: timestamp50 name: timestamp
@@ -55,6 +54,18 @@ definitions: · 3 unmodified lines
55 name: hash54 name: hash
56 doc:55 doc:
57 Device is capable of exposing receive packet hash via bpf_xdp_metadata_rx_hash().56 Device is capable of exposing receive packet hash via bpf_xdp_metadata_rx_hash().
57 -
58 type: flags
59 name: xsk-flags
60 entries:
61 -
62 name: tx-timestamp
63 doc:
64 HW timestamping egress packets is supported by the driver.
65 -
66 name: tx-checksum
67 doc:
68 L3 checksum HW offload is supported by the driver.
58 69
59attribute-sets:70attribute-sets:
60 -71 -
@@ -86,6 +97,11 @@ attribute-sets: · 25 unmodified lines
86 See Documentation/networking/xdp-rx-metadata.rst for more details.97 See Documentation/networking/xdp-rx-metadata.rst for more details.
87 type: u6498 type: u64
88 enum: xdp-rx-metadata99 enum: xdp-rx-metadata
100 -
101 name: xsk-features
102 doc: Bitmask of enabled AF_XDP features.
103 type: u64
104 enum: xsk-flags
89 105
90operations:106operations:
91 list:107 list:
@@ -103,6 +119,7 @@ operations: · 11 unmodified lines
103 - xdp-features119 - xdp-features
104 - xdp-zc-max-segs120 - xdp-zc-max-segs
105 - xdp-rx-metadata-features121 - xdp-rx-metadata-features
122 - xsk-features
106 dump:123 dump:
107 reply: *dev-all124 reply: *dev-all
108 -125 -
include/linux/netdevice.h+2−0
@@ -1865,6 +1865,7 @@ enum netdev_stat_type { · 1864 unmodified lines
1865 * @netdev_ops: Includes several pointers to callbacks,1865 * @netdev_ops: Includes several pointers to callbacks,
1866 * if one wants to override the ndo_*() functions1866 * if one wants to override the ndo_*() functions
1867 * @xdp_metadata_ops: Includes pointers to XDP metadata callbacks.1867 * @xdp_metadata_ops: Includes pointers to XDP metadata callbacks.
1868 * @xsk_tx_metadata_ops: Includes pointers to AF_XDP TX metadata callbacks.
1868 * @ethtool_ops: Management operations1869 * @ethtool_ops: Management operations
1869 * @l3mdev_ops: Layer 3 master device operations1870 * @l3mdev_ops: Layer 3 master device operations
1870 * @ndisc_ops: Includes callbacks for different IPv6 neighbour1871 * @ndisc_ops: Includes callbacks for different IPv6 neighbour
@@ -2128,6 +2129,7 @@ struct net_device { · 257 unmodified lines
2128 unsigned long long priv_flags;2129 unsigned long long priv_flags;
2129 const struct net_device_ops *netdev_ops;2130 const struct net_device_ops *netdev_ops;
2130 const struct xdp_metadata_ops *xdp_metadata_ops;2131 const struct xdp_metadata_ops *xdp_metadata_ops;
2132 const struct xsk_tx_metadata_ops *xsk_tx_metadata_ops;
2131 int ifindex;2133 int ifindex;
2132 unsigned short gflags;2134 unsigned short gflags;
2133 unsigned short hard_header_len;2135 unsigned short hard_header_len;
include/linux/skbuff.h+13−1
@@ -566,6 +566,15 @@ struct ubuf_info_msgzc { · 565 unmodified lines
566int mm_account_pinned_pages(struct mmpin *mmp, size_t size);566int mm_account_pinned_pages(struct mmpin *mmp, size_t size);
567void mm_unaccount_pinned_pages(struct mmpin *mmp);567void mm_unaccount_pinned_pages(struct mmpin *mmp);
568 568
569/* Preserve some data across TX submission and completion.
570 *
571 * Note, this state is stored in the driver. Extending the layout
572 * might need some special care.
573 */
574struct xsk_tx_metadata_compl {
575 __u64 *tx_timestamp;
576};
577
569/* This data is invariant across clones and lives at578/* This data is invariant across clones and lives at
570 * the end of the header data, ie. at skb->end.579 * the end of the header data, ie. at skb->end.
571 */580 */
@@ -578,7 +587,10 @@ struct skb_shared_info { · 6 unmodified lines
578 /* Warning: this field is not always filled in (UFO)! */587 /* Warning: this field is not always filled in (UFO)! */
579 unsigned short gso_segs;588 unsigned short gso_segs;
580 struct sk_buff *frag_list;589 struct sk_buff *frag_list;
581 struct skb_shared_hwtstamps hwtstamps;590 union {
591 struct skb_shared_hwtstamps hwtstamps;
592 struct xsk_tx_metadata_compl xsk_meta;
593 };
582 unsigned int gso_type;594 unsigned int gso_type;
583 u32 tskey;595 u32 tskey;
584 596
include/net/xdp_sock.h+110−0
@@ -93,12 +93,105 @@ struct xdp_sock { · 92 unmodified lines
93 struct xsk_queue *cq_tmp; /* Only as tmp storage before bind */93 struct xsk_queue *cq_tmp; /* Only as tmp storage before bind */
94};94};
95 95
96/*
97 * AF_XDP TX metadata hooks for network devices.
98 * The following hooks can be defined; unless noted otherwise, they are
99 * optional and can be filled with a null pointer.
100 *
101 * void (*tmo_request_timestamp)(void *priv)
102 * Called when AF_XDP frame requested egress timestamp.
103 *
104 * u64 (*tmo_fill_timestamp)(void *priv)
105 * Called when AF_XDP frame, that had requested egress timestamp,
106 * received a completion. The hook needs to return the actual HW timestamp.
107 *
108 * void (*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv)
109 * Called when AF_XDP frame requested HW checksum offload. csum_start
110 * indicates position where checksumming should start.
111 * csum_offset indicates position where checksum should be stored.
112 *
113 */
114struct xsk_tx_metadata_ops {
115 void (*tmo_request_timestamp)(void *priv);
116 u64 (*tmo_fill_timestamp)(void *priv);
117 void (*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv);
118};
119
96#ifdef CONFIG_XDP_SOCKETS120#ifdef CONFIG_XDP_SOCKETS
97 121
98int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);122int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);
99int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp);123int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp);
100void __xsk_map_flush(void);124void __xsk_map_flush(void);
101 125
126/**
127 * xsk_tx_metadata_to_compl - Save enough relevant metadata information
128 * to perform tx completion in the future.
129 * @meta: pointer to AF_XDP metadata area
130 * @compl: pointer to output struct xsk_tx_metadata_to_compl
131 *
132 * This function should be called by the networking device when
133 * it prepares AF_XDP egress packet. The value of @compl should be stored
134 * and passed to xsk_tx_metadata_complete upon TX completion.
135 */
136static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
137 struct xsk_tx_metadata_compl *compl)
138{
139 if (!meta)
140 return;
141
142 if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
143 compl->tx_timestamp = &meta->completion.tx_timestamp;
144 else
145 compl->tx_timestamp = NULL;
146}
147
148/**
149 * xsk_tx_metadata_request - Evaluate AF_XDP TX metadata at submission
150 * and call appropriate xsk_tx_metadata_ops operation.
151 * @meta: pointer to AF_XDP metadata area
152 * @ops: pointer to struct xsk_tx_metadata_ops
153 * @priv: pointer to driver-private aread
154 *
155 * This function should be called by the networking device when
156 * it prepares AF_XDP egress packet.
157 */
158static inline void xsk_tx_metadata_request(const struct xsk_tx_metadata *meta,
159 const struct xsk_tx_metadata_ops *ops,
160 void *priv)
161{
162 if (!meta)
163 return;
164
165 if (ops->tmo_request_timestamp)
166 if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
167 ops->tmo_request_timestamp(priv);
168
169 if (ops->tmo_request_checksum)
170 if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM)
171 ops->tmo_request_checksum(meta->request.csum_start,
172 meta->request.csum_offset, priv);
173}
174
175/**
176 * xsk_tx_metadata_complete - Evaluate AF_XDP TX metadata at completion
177 * and call appropriate xsk_tx_metadata_ops operation.
178 * @compl: pointer to completion metadata produced from xsk_tx_metadata_to_compl
179 * @ops: pointer to struct xsk_tx_metadata_ops
180 * @priv: pointer to driver-private aread
181 *
182 * This function should be called by the networking device upon
183 * AF_XDP egress completion.
184 */
185static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl,
186 const struct xsk_tx_metadata_ops *ops,
187 void *priv)
188{
189 if (!compl)
190 return;
191
192 *compl->tx_timestamp = ops->tmo_fill_timestamp(priv);
193}
194
102#else195#else
103 196
104static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)197static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
@@ -115,6 +208,23 @@ static inline void __xsk_map_flush(void) · 10 unmodified lines
115{208{
116}209}
117 210
211static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
212 struct xsk_tx_metadata_compl *compl)
213{
214}
215
216static inline void xsk_tx_metadata_request(struct xsk_tx_metadata *meta,
217 const struct xsk_tx_metadata_ops *ops,
218 void *priv)
219{
220}
221
222static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl,
223 const struct xsk_tx_metadata_ops *ops,
224 void *priv)
225{
226}
227
118#endif /* CONFIG_XDP_SOCKETS */228#endif /* CONFIG_XDP_SOCKETS */
119 229
120#if defined(CONFIG_XDP_SOCKETS) && defined(CONFIG_DEBUG_NET)230#if defined(CONFIG_XDP_SOCKETS) && defined(CONFIG_DEBUG_NET)
include/net/xdp_sock_drv.h+13−0
@@ -165,6 +165,14 @@ static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr) · 164 unmodified lines
165 return xp_raw_get_data(pool, addr);165 return xp_raw_get_data(pool, addr);
166}166}
167 167
168static inline struct xsk_tx_metadata *xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr)
169{
170 if (!pool->tx_metadata_len)
171 return NULL;
172
173 return xp_raw_get_data(pool, addr) - pool->tx_metadata_len;
174}
175
168static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp, struct xsk_buff_pool *pool)176static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp, struct xsk_buff_pool *pool)
169{177{
170 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);178 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
@@ -324,6 +332,11 @@ static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr) · 153 unmodified lines
324 return NULL;332 return NULL;
325}333}
326 334
335static inline struct xsk_tx_metadata *xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr)
336{
337 return NULL;
338}
339
327static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp, struct xsk_buff_pool *pool)340static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp, struct xsk_buff_pool *pool)
328{341{
329}342}
include/net/xsk_buff_pool.h+6−0
@@ -33,6 +33,7 @@ struct xdp_buff_xsk { · 32 unmodified lines
33};33};
34 34
35#define XSK_CHECK_PRIV_TYPE(t) BUILD_BUG_ON(sizeof(t) > offsetofend(struct xdp_buff_xsk, cb))35#define XSK_CHECK_PRIV_TYPE(t) BUILD_BUG_ON(sizeof(t) > offsetofend(struct xdp_buff_xsk, cb))
36#define XSK_TX_COMPL_FITS(t) BUILD_BUG_ON(sizeof(struct xsk_tx_metadata_compl) > sizeof(t))
36 37
37struct xsk_dma_map {38struct xsk_dma_map {
38 dma_addr_t *dma_pages;39 dma_addr_t *dma_pages;
@@ -234,4 +235,9 @@ static inline u64 xp_get_handle(struct xdp_buff_xsk *xskb) · 195 unmodified lines
234 return xskb->orig_addr + (offset << XSK_UNALIGNED_BUF_OFFSET_SHIFT);235 return xskb->orig_addr + (offset << XSK_UNALIGNED_BUF_OFFSET_SHIFT);
235}236}
236 237
238static inline bool xp_tx_metadata_enabled(const struct xsk_buff_pool *pool)
239{
240 return pool->tx_metadata_len > 0;
241}
242
237#endif /* XSK_BUFF_POOL_H_ */243#endif /* XSK_BUFF_POOL_H_ */
include/uapi/linux/if_xdp.h+38−0
@@ -106,6 +106,41 @@ struct xdp_options { · 105 unmodified lines
106#define XSK_UNALIGNED_BUF_ADDR_MASK \106#define XSK_UNALIGNED_BUF_ADDR_MASK \
107 ((1ULL << XSK_UNALIGNED_BUF_OFFSET_SHIFT) - 1)107 ((1ULL << XSK_UNALIGNED_BUF_OFFSET_SHIFT) - 1)
108 108
109/* Request transmit timestamp. Upon completion, put it into tx_timestamp
110 * field of struct xsk_tx_metadata.
111 */
112#define XDP_TXMD_FLAGS_TIMESTAMP (1 << 0)
113
114/* Request transmit checksum offload. Checksum start position and offset
115 * are communicated via csum_start and csum_offset fields of struct
116 * xsk_tx_metadata.
117 */
118#define XDP_TXMD_FLAGS_CHECKSUM (1 << 1)
119
120/* AF_XDP offloads request. 'request' union member is consumed by the driver
121 * when the packet is being transmitted. 'completion' union member is
122 * filled by the driver when the transmit completion arrives.
123 */
124struct xsk_tx_metadata {
125 __u64 flags;
126
127 union {
128 struct {
129 /* XDP_TXMD_FLAGS_CHECKSUM */
130
131 /* Offset from desc->addr where checksumming should start. */
132 __u16 csum_start;
133 /* Offset from csum_start where checksum should be stored. */
134 __u16 csum_offset;
135 } request;
136
137 struct {
138 /* XDP_TXMD_FLAGS_TIMESTAMP */
139 __u64 tx_timestamp;
140 } completion;
141 };
142};
143
109/* Rx/Tx descriptor */144/* Rx/Tx descriptor */
110struct xdp_desc {145struct xdp_desc {
111 __u64 addr;146 __u64 addr;
@@ -122,4 +157,7 @@ struct xdp_desc { · 10 unmodified lines
122 */157 */
123#define XDP_PKT_CONTD (1 << 0)158#define XDP_PKT_CONTD (1 << 0)
124 159
160/* TX packet carries valid metadata. */
161#define XDP_TX_METADATA (1 << 1)
162
125#endif /* _LINUX_IF_XDP_H */163#endif /* _LINUX_IF_XDP_H */
include/uapi/linux/netdev.h+16−0
@@ -53,12 +53,28 @@ enum netdev_xdp_rx_metadata { · 52 unmodified lines
53 NETDEV_XDP_RX_METADATA_MASK = 3,53 NETDEV_XDP_RX_METADATA_MASK = 3,
54};54};
55 55
56/**
57 * enum netdev_xsk_flags
58 * @NETDEV_XSK_FLAGS_TX_TIMESTAMP: HW timestamping egress packets is supported
59 * by the driver.
60 * @NETDEV_XSK_FLAGS_TX_CHECKSUM: L3 checksum HW offload is supported by the
61 * driver.
62 */
63enum netdev_xsk_flags {
64 NETDEV_XSK_FLAGS_TX_TIMESTAMP = 1,
65 NETDEV_XSK_FLAGS_TX_CHECKSUM = 2,
66
67 /* private: */
68 NETDEV_XSK_FLAGS_MASK = 3,
69};
70
56enum {71enum {
57 NETDEV_A_DEV_IFINDEX = 1,72 NETDEV_A_DEV_IFINDEX = 1,
58 NETDEV_A_DEV_PAD,73 NETDEV_A_DEV_PAD,
59 NETDEV_A_DEV_XDP_FEATURES,74 NETDEV_A_DEV_XDP_FEATURES,
60 NETDEV_A_DEV_XDP_ZC_MAX_SEGS,75 NETDEV_A_DEV_XDP_ZC_MAX_SEGS,
61 NETDEV_A_DEV_XDP_RX_METADATA_FEATURES,76 NETDEV_A_DEV_XDP_RX_METADATA_FEATURES,
77 NETDEV_A_DEV_XSK_FEATURES,
62 78
63 __NETDEV_A_DEV_MAX,79 __NETDEV_A_DEV_MAX,
64 NETDEV_A_DEV_MAX = (__NETDEV_A_DEV_MAX - 1)80 NETDEV_A_DEV_MAX = (__NETDEV_A_DEV_MAX - 1)
net/core/netdev-genl.c+12−1
@@ -6,6 +6,7 @@ · 5 unmodified lines
6#include <net/net_namespace.h>6#include <net/net_namespace.h>
7#include <net/sock.h>7#include <net/sock.h>
8#include <net/xdp.h>8#include <net/xdp.h>
9#include <net/xdp_sock.h>
9 10
10#include "netdev-genl-gen.h"11#include "netdev-genl-gen.h"
11 12
@@ -13,6 +14,7 @@ static int · 1 unmodified line
13netdev_nl_dev_fill(struct net_device *netdev, struct sk_buff *rsp,14netdev_nl_dev_fill(struct net_device *netdev, struct sk_buff *rsp,
14 const struct genl_info *info)15 const struct genl_info *info)
15{16{
17 u64 xsk_features = 0;
16 u64 xdp_rx_meta = 0;18 u64 xdp_rx_meta = 0;
17 void *hdr;19 void *hdr;
18 20
@@ -26,11 +28,20 @@ netdev_nl_dev_fill(struct net_device *netdev, struct sk_buff *rsp, · 7 unmodified lines
26XDP_METADATA_KFUNC_xxx28XDP_METADATA_KFUNC_xxx
27#undef XDP_METADATA_KFUNC29#undef XDP_METADATA_KFUNC
28 30
31 if (netdev->xsk_tx_metadata_ops) {
32 if (netdev->xsk_tx_metadata_ops->tmo_fill_timestamp)
33 xsk_features |= NETDEV_XSK_FLAGS_TX_TIMESTAMP;
34 if (netdev->xsk_tx_metadata_ops->tmo_request_checksum)
35 xsk_features |= NETDEV_XSK_FLAGS_TX_CHECKSUM;
36 }
37
29 if (nla_put_u32(rsp, NETDEV_A_DEV_IFINDEX, netdev->ifindex) ||38 if (nla_put_u32(rsp, NETDEV_A_DEV_IFINDEX, netdev->ifindex) ||
30 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_FEATURES,39 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_FEATURES,
31 netdev->xdp_features, NETDEV_A_DEV_PAD) ||40 netdev->xdp_features, NETDEV_A_DEV_PAD) ||
32 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_RX_METADATA_FEATURES,41 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_RX_METADATA_FEATURES,
33 xdp_rx_meta, NETDEV_A_DEV_PAD)) {42 xdp_rx_meta, NETDEV_A_DEV_PAD) ||
43 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XSK_FEATURES,
44 xsk_features, NETDEV_A_DEV_PAD)) {
34 genlmsg_cancel(rsp, hdr);45 genlmsg_cancel(rsp, hdr);
35 return -EINVAL;46 return -EINVAL;
36 }47 }
net/xdp/xsk.c+34−0
@@ -571,6 +571,13 @@ static u32 xsk_get_num_desc(struct sk_buff *skb) · 570 unmodified lines
571 571
572static void xsk_destruct_skb(struct sk_buff *skb)572static void xsk_destruct_skb(struct sk_buff *skb)
573{573{
574 struct xsk_tx_metadata_compl *compl = &skb_shinfo(skb)->xsk_meta;
575
576 if (compl->tx_timestamp) {
577 /* sw completion timestamp, not a real one */
578 *compl->tx_timestamp = ktime_get_tai_fast_ns();
579 }
580
574 xsk_cq_submit_locked(xdp_sk(skb->sk), xsk_get_num_desc(skb));581 xsk_cq_submit_locked(xdp_sk(skb->sk), xsk_get_num_desc(skb));
575 sock_wfree(skb);582 sock_wfree(skb);
576}583}
@@ -655,8 +662,10 @@ static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs, · 78 unmodified lines
655static struct sk_buff *xsk_build_skb(struct xdp_sock *xs,662static struct sk_buff *xsk_build_skb(struct xdp_sock *xs,
656 struct xdp_desc *desc)663 struct xdp_desc *desc)
657{664{
665 struct xsk_tx_metadata *meta = NULL;
658 struct net_device *dev = xs->dev;666 struct net_device *dev = xs->dev;
659 struct sk_buff *skb = xs->skb;667 struct sk_buff *skb = xs->skb;
668 bool first_frag = false;
660 int err;669 int err;
661 670
662 if (dev->priv_flags & IFF_TX_SKB_NO_LINEAR) {671 if (dev->priv_flags & IFF_TX_SKB_NO_LINEAR) {
@@ -687,6 +696,8 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, · 24 unmodified lines
687 kfree_skb(skb);696 kfree_skb(skb);
688 goto free_err;697 goto free_err;
689 }698 }
699
700 first_frag = true;
690 } else {701 } else {
691 int nr_frags = skb_shinfo(skb)->nr_frags;702 int nr_frags = skb_shinfo(skb)->nr_frags;
692 struct page *page;703 struct page *page;
@@ -709,12 +720,35 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, · 16 unmodified lines
709 720
710 skb_add_rx_frag(skb, nr_frags, page, 0, len, 0);721 skb_add_rx_frag(skb, nr_frags, page, 0, len, 0);
711 }722 }
723
724 if (first_frag && desc->options & XDP_TX_METADATA) {
725 if (unlikely(xs->pool->tx_metadata_len == 0)) {
726 err = -EINVAL;
727 goto free_err;
728 }
729
730 meta = buffer - xs->pool->tx_metadata_len;
731
732 if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM) {
733 if (unlikely(meta->request.csum_start +
734 meta->request.csum_offset +
735 sizeof(__sum16) > len)) {
736 err = -EINVAL;
737 goto free_err;
738 }
739
740 skb->csum_start = hr + meta->request.csum_start;
741 skb->csum_offset = meta->request.csum_offset;
742 skb->ip_summed = CHECKSUM_PARTIAL;
743 }
744 }
712 }745 }
713 746
714 skb->dev = dev;747 skb->dev = dev;
715 skb->priority = READ_ONCE(xs->sk.sk_priority);748 skb->priority = READ_ONCE(xs->sk.sk_priority);
716 skb->mark = READ_ONCE(xs->sk.sk_mark);749 skb->mark = READ_ONCE(xs->sk.sk_mark);
717 skb->destructor = xsk_destruct_skb;750 skb->destructor = xsk_destruct_skb;
751 xsk_tx_metadata_to_compl(meta, &skb_shinfo(skb)->xsk_meta);
718 xsk_set_destructor_arg(skb);752 xsk_set_destructor_arg(skb);
719 753
720 return skb;754 return skb;
net/xdp/xsk_queue.h+1−1
@@ -137,7 +137,7 @@ static inline bool xskq_cons_read_addr_unchecked(struct xsk_queue *q, u64 *addr) · 136 unmodified lines
137 137
138static inline bool xp_unused_options_set(u32 options)138static inline bool xp_unused_options_set(u32 options)
139{139{
140 return options & ~XDP_PKT_CONTD;140 return options & ~(XDP_PKT_CONTD | XDP_TX_METADATA);
141}141}
142 142
143static inline bool xp_aligned_validate_desc(struct xsk_buff_pool *pool,143static inline bool xp_aligned_validate_desc(struct xsk_buff_pool *pool,
tools/include/uapi/linux/if_xdp.h+47−5
@@ -26,11 +26,11 @@ · 25 unmodified lines
26 */26 */
27#define XDP_USE_NEED_WAKEUP (1 << 3)27#define XDP_USE_NEED_WAKEUP (1 << 3)
28/* By setting this option, userspace application indicates that it can28/* By setting this option, userspace application indicates that it can
29 * handle multiple descriptors per packet thus enabling xsk core to split29 * handle multiple descriptors per packet thus enabling AF_XDP to split
30 * multi-buffer XDP frames into multiple Rx descriptors. Without this set30 * multi-buffer XDP frames into multiple Rx descriptors. Without this set
31 * such frames will be dropped by xsk.31 * such frames will be dropped.
32 */32 */
33#define XDP_USE_SG (1 << 4)33#define XDP_USE_SG (1 << 4)
34 34
35/* Flags for xsk_umem_config flags */35/* Flags for xsk_umem_config flags */
36#define XDP_UMEM_UNALIGNED_CHUNK_FLAG (1 << 0)36#define XDP_UMEM_UNALIGNED_CHUNK_FLAG (1 << 0)
@@ -106,6 +106,41 @@ struct xdp_options { · 69 unmodified lines
106#define XSK_UNALIGNED_BUF_ADDR_MASK \106#define XSK_UNALIGNED_BUF_ADDR_MASK \
107 ((1ULL << XSK_UNALIGNED_BUF_OFFSET_SHIFT) - 1)107 ((1ULL << XSK_UNALIGNED_BUF_OFFSET_SHIFT) - 1)
108 108
109/* Request transmit timestamp. Upon completion, put it into tx_timestamp
110 * field of union xsk_tx_metadata.
111 */
112#define XDP_TXMD_FLAGS_TIMESTAMP (1 << 0)
113
114/* Request transmit checksum offload. Checksum start position and offset
115 * are communicated via csum_start and csum_offset fields of union
116 * xsk_tx_metadata.
117 */
118#define XDP_TXMD_FLAGS_CHECKSUM (1 << 1)
119
120/* AF_XDP offloads request. 'request' union member is consumed by the driver
121 * when the packet is being transmitted. 'completion' union member is
122 * filled by the driver when the transmit completion arrives.
123 */
124struct xsk_tx_metadata {
125 __u64 flags;
126
127 union {
128 struct {
129 /* XDP_TXMD_FLAGS_CHECKSUM */
130
131 /* Offset from desc->addr where checksumming should start. */
132 __u16 csum_start;
133 /* Offset from csum_start where checksum should be stored. */
134 __u16 csum_offset;
135 } request;
136
137 struct {
138 /* XDP_TXMD_FLAGS_TIMESTAMP */
139 __u64 tx_timestamp;
140 } completion;
141 };
142};
143
109/* Rx/Tx descriptor */144/* Rx/Tx descriptor */
110struct xdp_desc {145struct xdp_desc {
111 __u64 addr;146 __u64 addr;
@@ -113,9 +148,16 @@ struct xdp_desc { · 1 unmodified line
113 __u32 options;148 __u32 options;
114};149};
115 150
116/* Flag indicating packet constitutes of multiple buffers*/151/* UMEM descriptor is __u64 */
152
153/* Flag indicating that the packet continues with the buffer pointed out by the
154 * next frame in the ring. The end of the packet is signalled by setting this
155 * bit to zero. For single buffer packets, every descriptor has 'options' set
156 * to 0 and this maintains backward compatibility.
157 */
117#define XDP_PKT_CONTD (1 << 0)158#define XDP_PKT_CONTD (1 << 0)
118 159
119/* UMEM descriptor is __u64 */160/* TX packet carries valid metadata. */
161#define XDP_TX_METADATA (1 << 1)
120 162
121#endif /* _LINUX_IF_XDP_H */163#endif /* _LINUX_IF_XDP_H */
tools/include/uapi/linux/netdev.h+16−0
@@ -53,12 +53,28 @@ enum netdev_xdp_rx_metadata { · 52 unmodified lines
53 NETDEV_XDP_RX_METADATA_MASK = 3,53 NETDEV_XDP_RX_METADATA_MASK = 3,
54};54};
55 55
56/**
57 * enum netdev_xsk_flags
58 * @NETDEV_XSK_FLAGS_TX_TIMESTAMP: HW timestamping egress packets is supported
59 * by the driver.
60 * @NETDEV_XSK_FLAGS_TX_CHECKSUM: L3 checksum HW offload is supported by the
61 * driver.
62 */
63enum netdev_xsk_flags {
64 NETDEV_XSK_FLAGS_TX_TIMESTAMP = 1,
65 NETDEV_XSK_FLAGS_TX_CHECKSUM = 2,
66
67 /* private: */
68 NETDEV_XSK_FLAGS_MASK = 3,
69};
70
56enum {71enum {
57 NETDEV_A_DEV_IFINDEX = 1,72 NETDEV_A_DEV_IFINDEX = 1,
58 NETDEV_A_DEV_PAD,73 NETDEV_A_DEV_PAD,
59 NETDEV_A_DEV_XDP_FEATURES,74 NETDEV_A_DEV_XDP_FEATURES,
60 NETDEV_A_DEV_XDP_ZC_MAX_SEGS,75 NETDEV_A_DEV_XDP_ZC_MAX_SEGS,
61 NETDEV_A_DEV_XDP_RX_METADATA_FEATURES,76 NETDEV_A_DEV_XDP_RX_METADATA_FEATURES,
77 NETDEV_A_DEV_XSK_FEATURES,
62 78
63 __NETDEV_A_DEV_MAX,79 __NETDEV_A_DEV_MAX,
64 NETDEV_A_DEV_MAX = (__NETDEV_A_DEV_MAX - 1)80 NETDEV_A_DEV_MAX = (__NETDEV_A_DEV_MAX - 1)
tools/net/ynl/generated/netdev-user.c+19−0
@@ -58,6 +58,19 @@ const char *netdev_xdp_rx_metadata_str(enum netdev_xdp_rx_metadata value) · 57 unmodified lines
58 return netdev_xdp_rx_metadata_strmap[value];58 return netdev_xdp_rx_metadata_strmap[value];
59}59}
60 60
61static const char * const netdev_xsk_flags_strmap[] = {
62 [0] = "tx-timestamp",
63 [1] = "tx-checksum",
64};
65
66const char *netdev_xsk_flags_str(enum netdev_xsk_flags value)
67{
68 value = ffs(value) - 1;
69 if (value < 0 || value >= (int)MNL_ARRAY_SIZE(netdev_xsk_flags_strmap))
70 return NULL;
71 return netdev_xsk_flags_strmap[value];
72}
73
61/* Policies */74/* Policies */
62struct ynl_policy_attr netdev_dev_policy[NETDEV_A_DEV_MAX + 1] = {75struct ynl_policy_attr netdev_dev_policy[NETDEV_A_DEV_MAX + 1] = {
63 [NETDEV_A_DEV_IFINDEX] = { .name = "ifindex", .type = YNL_PT_U32, },76 [NETDEV_A_DEV_IFINDEX] = { .name = "ifindex", .type = YNL_PT_U32, },
@@ -65,6 +78,7 @@ struct ynl_policy_attr netdev_dev_policy[NETDEV_A_DEV_MAX + 1] = { · 1 unmodified line
65 [NETDEV_A_DEV_XDP_FEATURES] = { .name = "xdp-features", .type = YNL_PT_U64, },78 [NETDEV_A_DEV_XDP_FEATURES] = { .name = "xdp-features", .type = YNL_PT_U64, },
66 [NETDEV_A_DEV_XDP_ZC_MAX_SEGS] = { .name = "xdp-zc-max-segs", .type = YNL_PT_U32, },79 [NETDEV_A_DEV_XDP_ZC_MAX_SEGS] = { .name = "xdp-zc-max-segs", .type = YNL_PT_U32, },
67 [NETDEV_A_DEV_XDP_RX_METADATA_FEATURES] = { .name = "xdp-rx-metadata-features", .type = YNL_PT_U64, },80 [NETDEV_A_DEV_XDP_RX_METADATA_FEATURES] = { .name = "xdp-rx-metadata-features", .type = YNL_PT_U64, },
81 [NETDEV_A_DEV_XSK_FEATURES] = { .name = "xsk-features", .type = YNL_PT_U64, },
68};82};
69 83
70struct ynl_policy_nest netdev_dev_nest = {84struct ynl_policy_nest netdev_dev_nest = {
@@ -116,6 +130,11 @@ int netdev_dev_get_rsp_parse(const struct nlmsghdr *nlh, void *data) · 45 unmodified lines
116 return MNL_CB_ERROR;130 return MNL_CB_ERROR;
117 dst->_present.xdp_rx_metadata_features = 1;131 dst->_present.xdp_rx_metadata_features = 1;
118 dst->xdp_rx_metadata_features = mnl_attr_get_u64(attr);132 dst->xdp_rx_metadata_features = mnl_attr_get_u64(attr);
133 } else if (type == NETDEV_A_DEV_XSK_FEATURES) {
134 if (ynl_attr_validate(yarg, attr))
135 return MNL_CB_ERROR;
136 dst->_present.xsk_features = 1;
137 dst->xsk_features = mnl_attr_get_u64(attr);
119 }138 }
120 }139 }
121 140
tools/net/ynl/generated/netdev-user.h+3−0
@@ -19,6 +19,7 @@ extern const struct ynl_family ynl_netdev_family; · 18 unmodified lines
19const char *netdev_op_str(int op);19const char *netdev_op_str(int op);
20const char *netdev_xdp_act_str(enum netdev_xdp_act value);20const char *netdev_xdp_act_str(enum netdev_xdp_act value);
21const char *netdev_xdp_rx_metadata_str(enum netdev_xdp_rx_metadata value);21const char *netdev_xdp_rx_metadata_str(enum netdev_xdp_rx_metadata value);
22const char *netdev_xsk_flags_str(enum netdev_xsk_flags value);
22 23
23/* Common nested types */24/* Common nested types */
24/* ============== NETDEV_CMD_DEV_GET ============== */25/* ============== NETDEV_CMD_DEV_GET ============== */
@@ -50,12 +51,14 @@ struct netdev_dev_get_rsp { · 25 unmodified lines
50 __u32 xdp_features:1;51 __u32 xdp_features:1;
51 __u32 xdp_zc_max_segs:1;52 __u32 xdp_zc_max_segs:1;
52 __u32 xdp_rx_metadata_features:1;53 __u32 xdp_rx_metadata_features:1;
54 __u32 xsk_features:1;
53 } _present;55 } _present;
54 56
55 __u32 ifindex;57 __u32 ifindex;
56 __u64 xdp_features;58 __u64 xdp_features;
57 __u32 xdp_zc_max_segs;59 __u32 xdp_zc_max_segs;
58 __u64 xdp_rx_metadata_features;60 __u64 xdp_rx_metadata_features;
61 __u64 xsk_features;
59};62};
60 63
61void netdev_dev_get_rsp_free(struct netdev_dev_get_rsp *rsp);64void netdev_dev_get_rsp_free(struct netdev_dev_get_rsp *rsp);