History

xsk: Add TX timestamp and TX checksum offload support

This change actually defines the (initial) metadata layout that should be used by AF_XDP userspace (xsk_tx_metadata). The first field is flags which requests appropriate offloads, followed by the offload-specific fields. The supported per-device offloads are exported via netlink (new xsk-flags). The offloads themselves are still implemented in a bit of a framework-y fashion that's left from my initial kfunc attempt. I'm introducing new xsk_tx_metadata_ops which drivers are supposed to implement. The drivers are also supposed to call xsk_tx_metadata_request/xsk_tx_metadata_complete in the right places. Since xsk_tx_metadata_{request,_complete} are static inline, we don't incur any extra overhead doing indirect calls. The benefit of this scheme is as follows: - keeps all metadata layout parsing away from driver code - makes it easy to grep and see which drivers implement what - don't need any extra flags to maintain to keep track of what offloads are implemented; if the callback is implemented - the offload is supported (used by netlink reporting code) Two offloads are defined right now: 1. XDP_TXMD_FLAGS_CHECKSUM: skb-style csum_start+csum_offset 2. XDP_TXMD_FLAGS_TIMESTAMP: writes TX timestamp back into metadata area upon completion (tx_timestamp field) XDP_TXMD_FLAGS_TIMESTAMP is also implemented for XDP_COPY mode: it writes SW timestamp from the skb destructor (note I'm reusing hwtstamps to pass metadata pointer). The struct is forward-compatible and can be extended in the future by appending more fields.

Stanislav Fomichev committed 2 years ago

Commit 48eb03d1 parent 341ac9815 files · +348 −9Browse files

Showing the changes to include (7 of 15 files). Show all

Unified · Split

7 files
  1. include/linux/netdevice.h +2 −0
  2. include/linux/skbuff.h +13 −1
  3. include/net/xdp_sock.h +110 −0
  4. include/net/xdp_sock_drv.h +13 −0
  5. include/net/xsk_buff_pool.h +6 −0
  6. include/uapi/linux/if_xdp.h +38 −0
  7. include/uapi/linux/netdev.h +16 −0
include/linux/netdevice.h+2−0
@@ -1865,6 +1865,7 @@ enum netdev_stat_type { · 1864 unmodified lines
1865 * @netdev_ops: Includes several pointers to callbacks,1865 * @netdev_ops: Includes several pointers to callbacks,
1866 * if one wants to override the ndo_*() functions1866 * if one wants to override the ndo_*() functions
1867 * @xdp_metadata_ops: Includes pointers to XDP metadata callbacks.1867 * @xdp_metadata_ops: Includes pointers to XDP metadata callbacks.
1868 * @xsk_tx_metadata_ops: Includes pointers to AF_XDP TX metadata callbacks.
1868 * @ethtool_ops: Management operations1869 * @ethtool_ops: Management operations
1869 * @l3mdev_ops: Layer 3 master device operations1870 * @l3mdev_ops: Layer 3 master device operations
1870 * @ndisc_ops: Includes callbacks for different IPv6 neighbour1871 * @ndisc_ops: Includes callbacks for different IPv6 neighbour
@@ -2128,6 +2129,7 @@ struct net_device { · 257 unmodified lines
2128 unsigned long long priv_flags;2129 unsigned long long priv_flags;
2129 const struct net_device_ops *netdev_ops;2130 const struct net_device_ops *netdev_ops;
2130 const struct xdp_metadata_ops *xdp_metadata_ops;2131 const struct xdp_metadata_ops *xdp_metadata_ops;
2132 const struct xsk_tx_metadata_ops *xsk_tx_metadata_ops;
2131 int ifindex;2133 int ifindex;
2132 unsigned short gflags;2134 unsigned short gflags;
2133 unsigned short hard_header_len;2135 unsigned short hard_header_len;
include/linux/skbuff.h+13−1
@@ -566,6 +566,15 @@ struct ubuf_info_msgzc { · 565 unmodified lines
566int mm_account_pinned_pages(struct mmpin *mmp, size_t size);566int mm_account_pinned_pages(struct mmpin *mmp, size_t size);
567void mm_unaccount_pinned_pages(struct mmpin *mmp);567void mm_unaccount_pinned_pages(struct mmpin *mmp);
568 568
569/* Preserve some data across TX submission and completion.
570 *
571 * Note, this state is stored in the driver. Extending the layout
572 * might need some special care.
573 */
574struct xsk_tx_metadata_compl {
575 __u64 *tx_timestamp;
576};
577
569/* This data is invariant across clones and lives at578/* This data is invariant across clones and lives at
570 * the end of the header data, ie. at skb->end.579 * the end of the header data, ie. at skb->end.
571 */580 */
@@ -578,7 +587,10 @@ struct skb_shared_info { · 6 unmodified lines
578 /* Warning: this field is not always filled in (UFO)! */587 /* Warning: this field is not always filled in (UFO)! */
579 unsigned short gso_segs;588 unsigned short gso_segs;
580 struct sk_buff *frag_list;589 struct sk_buff *frag_list;
581 struct skb_shared_hwtstamps hwtstamps;590 union {
591 struct skb_shared_hwtstamps hwtstamps;
592 struct xsk_tx_metadata_compl xsk_meta;
593 };
582 unsigned int gso_type;594 unsigned int gso_type;
583 u32 tskey;595 u32 tskey;
584 596
include/net/xdp_sock.h+110−0
@@ -93,12 +93,105 @@ struct xdp_sock { · 92 unmodified lines
93 struct xsk_queue *cq_tmp; /* Only as tmp storage before bind */93 struct xsk_queue *cq_tmp; /* Only as tmp storage before bind */
94};94};
95 95
96/*
97 * AF_XDP TX metadata hooks for network devices.
98 * The following hooks can be defined; unless noted otherwise, they are
99 * optional and can be filled with a null pointer.
100 *
101 * void (*tmo_request_timestamp)(void *priv)
102 * Called when AF_XDP frame requested egress timestamp.
103 *
104 * u64 (*tmo_fill_timestamp)(void *priv)
105 * Called when AF_XDP frame, that had requested egress timestamp,
106 * received a completion. The hook needs to return the actual HW timestamp.
107 *
108 * void (*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv)
109 * Called when AF_XDP frame requested HW checksum offload. csum_start
110 * indicates position where checksumming should start.
111 * csum_offset indicates position where checksum should be stored.
112 *
113 */
114struct xsk_tx_metadata_ops {
115 void (*tmo_request_timestamp)(void *priv);
116 u64 (*tmo_fill_timestamp)(void *priv);
117 void (*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv);
118};
119
96#ifdef CONFIG_XDP_SOCKETS120#ifdef CONFIG_XDP_SOCKETS
97 121
98int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);122int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);
99int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp);123int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp);
100void __xsk_map_flush(void);124void __xsk_map_flush(void);
101 125
126/**
127 * xsk_tx_metadata_to_compl - Save enough relevant metadata information
128 * to perform tx completion in the future.
129 * @meta: pointer to AF_XDP metadata area
130 * @compl: pointer to output struct xsk_tx_metadata_to_compl
131 *
132 * This function should be called by the networking device when
133 * it prepares AF_XDP egress packet. The value of @compl should be stored
134 * and passed to xsk_tx_metadata_complete upon TX completion.
135 */
136static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
137 struct xsk_tx_metadata_compl *compl)
138{
139 if (!meta)
140 return;
141
142 if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
143 compl->tx_timestamp = &meta->completion.tx_timestamp;
144 else
145 compl->tx_timestamp = NULL;
146}
147
148/**
149 * xsk_tx_metadata_request - Evaluate AF_XDP TX metadata at submission
150 * and call appropriate xsk_tx_metadata_ops operation.
151 * @meta: pointer to AF_XDP metadata area
152 * @ops: pointer to struct xsk_tx_metadata_ops
153 * @priv: pointer to driver-private aread
154 *
155 * This function should be called by the networking device when
156 * it prepares AF_XDP egress packet.
157 */
158static inline void xsk_tx_metadata_request(const struct xsk_tx_metadata *meta,
159 const struct xsk_tx_metadata_ops *ops,
160 void *priv)
161{
162 if (!meta)
163 return;
164
165 if (ops->tmo_request_timestamp)
166 if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
167 ops->tmo_request_timestamp(priv);
168
169 if (ops->tmo_request_checksum)
170 if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM)
171 ops->tmo_request_checksum(meta->request.csum_start,
172 meta->request.csum_offset, priv);
173}
174
175/**
176 * xsk_tx_metadata_complete - Evaluate AF_XDP TX metadata at completion
177 * and call appropriate xsk_tx_metadata_ops operation.
178 * @compl: pointer to completion metadata produced from xsk_tx_metadata_to_compl
179 * @ops: pointer to struct xsk_tx_metadata_ops
180 * @priv: pointer to driver-private aread
181 *
182 * This function should be called by the networking device upon
183 * AF_XDP egress completion.
184 */
185static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl,
186 const struct xsk_tx_metadata_ops *ops,
187 void *priv)
188{
189 if (!compl)
190 return;
191
192 *compl->tx_timestamp = ops->tmo_fill_timestamp(priv);
193}
194
102#else195#else
103 196
104static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)197static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
@@ -115,6 +208,23 @@ static inline void __xsk_map_flush(void) · 10 unmodified lines
115{208{
116}209}
117 210
211static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
212 struct xsk_tx_metadata_compl *compl)
213{
214}
215
216static inline void xsk_tx_metadata_request(struct xsk_tx_metadata *meta,
217 const struct xsk_tx_metadata_ops *ops,
218 void *priv)
219{
220}
221
222static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl,
223 const struct xsk_tx_metadata_ops *ops,
224 void *priv)
225{
226}
227
118#endif /* CONFIG_XDP_SOCKETS */228#endif /* CONFIG_XDP_SOCKETS */
119 229
120#if defined(CONFIG_XDP_SOCKETS) && defined(CONFIG_DEBUG_NET)230#if defined(CONFIG_XDP_SOCKETS) && defined(CONFIG_DEBUG_NET)
include/net/xdp_sock_drv.h+13−0
@@ -165,6 +165,14 @@ static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr) · 164 unmodified lines
165 return xp_raw_get_data(pool, addr);165 return xp_raw_get_data(pool, addr);
166}166}
167 167
168static inline struct xsk_tx_metadata *xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr)
169{
170 if (!pool->tx_metadata_len)
171 return NULL;
172
173 return xp_raw_get_data(pool, addr) - pool->tx_metadata_len;
174}
175
168static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp, struct xsk_buff_pool *pool)176static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp, struct xsk_buff_pool *pool)
169{177{
170 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);178 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
@@ -324,6 +332,11 @@ static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr) · 153 unmodified lines
324 return NULL;332 return NULL;
325}333}
326 334
335static inline struct xsk_tx_metadata *xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr)
336{
337 return NULL;
338}
339
327static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp, struct xsk_buff_pool *pool)340static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp, struct xsk_buff_pool *pool)
328{341{
329}342}
include/net/xsk_buff_pool.h+6−0
@@ -33,6 +33,7 @@ struct xdp_buff_xsk { · 32 unmodified lines
33};33};
34 34
35#define XSK_CHECK_PRIV_TYPE(t) BUILD_BUG_ON(sizeof(t) > offsetofend(struct xdp_buff_xsk, cb))35#define XSK_CHECK_PRIV_TYPE(t) BUILD_BUG_ON(sizeof(t) > offsetofend(struct xdp_buff_xsk, cb))
36#define XSK_TX_COMPL_FITS(t) BUILD_BUG_ON(sizeof(struct xsk_tx_metadata_compl) > sizeof(t))
36 37
37struct xsk_dma_map {38struct xsk_dma_map {
38 dma_addr_t *dma_pages;39 dma_addr_t *dma_pages;
@@ -234,4 +235,9 @@ static inline u64 xp_get_handle(struct xdp_buff_xsk *xskb) · 195 unmodified lines
234 return xskb->orig_addr + (offset << XSK_UNALIGNED_BUF_OFFSET_SHIFT);235 return xskb->orig_addr + (offset << XSK_UNALIGNED_BUF_OFFSET_SHIFT);
235}236}
236 237
238static inline bool xp_tx_metadata_enabled(const struct xsk_buff_pool *pool)
239{
240 return pool->tx_metadata_len > 0;
241}
242
237#endif /* XSK_BUFF_POOL_H_ */243#endif /* XSK_BUFF_POOL_H_ */
include/uapi/linux/if_xdp.h+38−0
@@ -106,6 +106,41 @@ struct xdp_options { · 105 unmodified lines
106#define XSK_UNALIGNED_BUF_ADDR_MASK \106#define XSK_UNALIGNED_BUF_ADDR_MASK \
107 ((1ULL << XSK_UNALIGNED_BUF_OFFSET_SHIFT) - 1)107 ((1ULL << XSK_UNALIGNED_BUF_OFFSET_SHIFT) - 1)
108 108
109/* Request transmit timestamp. Upon completion, put it into tx_timestamp
110 * field of struct xsk_tx_metadata.
111 */
112#define XDP_TXMD_FLAGS_TIMESTAMP (1 << 0)
113
114/* Request transmit checksum offload. Checksum start position and offset
115 * are communicated via csum_start and csum_offset fields of struct
116 * xsk_tx_metadata.
117 */
118#define XDP_TXMD_FLAGS_CHECKSUM (1 << 1)
119
120/* AF_XDP offloads request. 'request' union member is consumed by the driver
121 * when the packet is being transmitted. 'completion' union member is
122 * filled by the driver when the transmit completion arrives.
123 */
124struct xsk_tx_metadata {
125 __u64 flags;
126
127 union {
128 struct {
129 /* XDP_TXMD_FLAGS_CHECKSUM */
130
131 /* Offset from desc->addr where checksumming should start. */
132 __u16 csum_start;
133 /* Offset from csum_start where checksum should be stored. */
134 __u16 csum_offset;
135 } request;
136
137 struct {
138 /* XDP_TXMD_FLAGS_TIMESTAMP */
139 __u64 tx_timestamp;
140 } completion;
141 };
142};
143
109/* Rx/Tx descriptor */144/* Rx/Tx descriptor */
110struct xdp_desc {145struct xdp_desc {
111 __u64 addr;146 __u64 addr;
@@ -122,4 +157,7 @@ struct xdp_desc { · 10 unmodified lines
122 */157 */
123#define XDP_PKT_CONTD (1 << 0)158#define XDP_PKT_CONTD (1 << 0)
124 159
160/* TX packet carries valid metadata. */
161#define XDP_TX_METADATA (1 << 1)
162
125#endif /* _LINUX_IF_XDP_H */163#endif /* _LINUX_IF_XDP_H */
include/uapi/linux/netdev.h+16−0
@@ -53,12 +53,28 @@ enum netdev_xdp_rx_metadata { · 52 unmodified lines
53 NETDEV_XDP_RX_METADATA_MASK = 3,53 NETDEV_XDP_RX_METADATA_MASK = 3,
54};54};
55 55
56/**
57 * enum netdev_xsk_flags
58 * @NETDEV_XSK_FLAGS_TX_TIMESTAMP: HW timestamping egress packets is supported
59 * by the driver.
60 * @NETDEV_XSK_FLAGS_TX_CHECKSUM: L3 checksum HW offload is supported by the
61 * driver.
62 */
63enum netdev_xsk_flags {
64 NETDEV_XSK_FLAGS_TX_TIMESTAMP = 1,
65 NETDEV_XSK_FLAGS_TX_CHECKSUM = 2,
66
67 /* private: */
68 NETDEV_XSK_FLAGS_MASK = 3,
69};
70
56enum {71enum {
57 NETDEV_A_DEV_IFINDEX = 1,72 NETDEV_A_DEV_IFINDEX = 1,
58 NETDEV_A_DEV_PAD,73 NETDEV_A_DEV_PAD,
59 NETDEV_A_DEV_XDP_FEATURES,74 NETDEV_A_DEV_XDP_FEATURES,
60 NETDEV_A_DEV_XDP_ZC_MAX_SEGS,75 NETDEV_A_DEV_XDP_ZC_MAX_SEGS,
61 NETDEV_A_DEV_XDP_RX_METADATA_FEATURES,76 NETDEV_A_DEV_XDP_RX_METADATA_FEATURES,
77 NETDEV_A_DEV_XSK_FEATURES,
62 78
63 __NETDEV_A_DEV_MAX,79 __NETDEV_A_DEV_MAX,
64 NETDEV_A_DEV_MAX = (__NETDEV_A_DEV_MAX - 1)80 NETDEV_A_DEV_MAX = (__NETDEV_A_DEV_MAX - 1)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t