xsk: Add TX timestamp and TX checksum offload support

This change actually defines the (initial) metadata layout that should be used by AF_XDP userspace (xsk_tx_metadata). The first field is flags which requests appropriate offloads, followed by the offload-specific fields. The supported per-device offloads are exported via netlink (new xsk-flags). The offloads themselves are still implemented in a bit of a framework-y fashion that's left from my initial kfunc attempt. I'm introducing new xsk_tx_metadata_ops which drivers are supposed to implement. The drivers are also supposed to call xsk_tx_metadata_request/xsk_tx_metadata_complete in the right places. Since xsk_tx_metadata_{request,_complete} are static inline, we don't incur any extra overhead doing indirect calls. The benefit of this scheme is as follows: - keeps all metadata layout parsing away from driver code - makes it easy to grep and see which drivers implement what - don't need any extra flags to maintain to keep track of what offloads are implemented; if the callback is implemented - the offload is supported (used by netlink reporting code) Two offloads are defined right now: 1. XDP_TXMD_FLAGS_CHECKSUM: skb-style csum_start+csum_offset 2. XDP_TXMD_FLAGS_TIMESTAMP: writes TX timestamp back into metadata area upon completion (tx_timestamp field) XDP_TXMD_FLAGS_TIMESTAMP is also implemented for XDP_COPY mode: it writes SW timestamp from the skb destructor (note I'm reusing hwtstamps to pass metadata pointer). The struct is forward-compatible and can be extended in the future by appending more fields.

Stanislav Fomichev committed 2 years ago

Commit 48eb03d1 parent 341ac9815 files · +348 −9Browse files

Showing the changes to include (7 of 15 files). Show all

Unified · Split

7 files

include/linux/netdevice.h +2 −0
include/linux/skbuff.h +13 −1
include/net/xdp_sock.h +110 −0
include/net/xdp_sock_drv.h +13 −0
include/net/xsk_buff_pool.h +6 −0
include/uapi/linux/if_xdp.h +38 −0
include/uapi/linux/netdev.h +16 −0

include/linux/netdevice.h+2−0

@@ -1865,6 +1865,7 @@ enum netdev_stat_type { · 1864 unmodified lines
1865	1865	* @netdev_ops: Includes several pointers to callbacks,
1866	1866	* if one wants to override the ndo_*() functions
1867	1867	* @xdp_metadata_ops: Includes pointers to XDP metadata callbacks.
	1868	+ * @xsk_tx_metadata_ops: Includes pointers to AF_XDP TX metadata callbacks.
1868	1869	* @ethtool_ops: Management operations
1869	1870	* @l3mdev_ops: Layer 3 master device operations
1870	1871	* @ndisc_ops: Includes callbacks for different IPv6 neighbour
@@ -2128,6 +2129,7 @@ struct net_device { · 257 unmodified lines
2128	2129	unsigned long long priv_flags;
2129	2130	const struct net_device_ops *netdev_ops;
2130	2131	const struct xdp_metadata_ops *xdp_metadata_ops;
	2132	+ const struct xsk_tx_metadata_ops *xsk_tx_metadata_ops;
2131	2133	int ifindex;
2132	2134	unsigned short gflags;
2133	2135	unsigned short hard_header_len;

include/linux/skbuff.h+13−1

@@ -566,6 +566,15 @@ struct ubuf_info_msgzc { · 565 unmodified lines
566	566	int mm_account_pinned_pages(struct mmpin *mmp, size_t size);
567	567	void mm_unaccount_pinned_pages(struct mmpin *mmp);
568	568
	569	+/* Preserve some data across TX submission and completion.
	570	+ *
	571	+ * Note, this state is stored in the driver. Extending the layout
	572	+ * might need some special care.
	573	+ */
	574	+struct xsk_tx_metadata_compl {
	575	+ __u64 *tx_timestamp;
	576	+};
	577	+
569	578	/* This data is invariant across clones and lives at
570	579	* the end of the header data, ie. at skb->end.
571	580	*/
@@ -578,7 +587,10 @@ struct skb_shared_info { · 6 unmodified lines
578	587	/* Warning: this field is not always filled in (UFO)! */
579	588	unsigned short gso_segs;
580	589	struct sk_buff *frag_list;
581		− struct skb_shared_hwtstamps hwtstamps;
	590	+ union {
	591	+ struct skb_shared_hwtstamps hwtstamps;
	592	+ struct xsk_tx_metadata_compl xsk_meta;
	593	+ };
582	594	unsigned int gso_type;
583	595	u32 tskey;
584	596

include/net/xdp_sock.h+110−0

@@ -93,12 +93,105 @@ struct xdp_sock { · 92 unmodified lines
93	93	struct xsk_queue cq_tmp; / Only as tmp storage before bind */
94	94	};
95	95
	96	+/*
	97	+ * AF_XDP TX metadata hooks for network devices.
	98	+ * The following hooks can be defined; unless noted otherwise, they are
	99	+ * optional and can be filled with a null pointer.
	100	+ *
	101	+ * void (tmo_request_timestamp)(void priv)
	102	+ * Called when AF_XDP frame requested egress timestamp.
	103	+ *
	104	+ * u64 (tmo_fill_timestamp)(void priv)
	105	+ * Called when AF_XDP frame, that had requested egress timestamp,
	106	+ * received a completion. The hook needs to return the actual HW timestamp.
	107	+ *
	108	+ * void (tmo_request_checksum)(u16 csum_start, u16 csum_offset, void priv)
	109	+ * Called when AF_XDP frame requested HW checksum offload. csum_start
	110	+ * indicates position where checksumming should start.
	111	+ * csum_offset indicates position where checksum should be stored.
	112	+ *
	113	+ */
	114	+struct xsk_tx_metadata_ops {
	115	+ void (tmo_request_timestamp)(void priv);
	116	+ u64 (tmo_fill_timestamp)(void priv);
	117	+ void (tmo_request_checksum)(u16 csum_start, u16 csum_offset, void priv);
	118	+};
	119	+
96	120	#ifdef CONFIG_XDP_SOCKETS
97	121
98	122	int xsk_generic_rcv(struct xdp_sock xs, struct xdp_buff xdp);
99	123	int __xsk_map_redirect(struct xdp_sock xs, struct xdp_buff xdp);
100	124	void __xsk_map_flush(void);
101	125
	126	+/**
	127	+ * xsk_tx_metadata_to_compl - Save enough relevant metadata information
	128	+ * to perform tx completion in the future.
	129	+ * @meta: pointer to AF_XDP metadata area
	130	+ * @compl: pointer to output struct xsk_tx_metadata_to_compl
	131	+ *
	132	+ * This function should be called by the networking device when
	133	+ * it prepares AF_XDP egress packet. The value of @compl should be stored
	134	+ * and passed to xsk_tx_metadata_complete upon TX completion.
	135	+ */
	136	+static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
	137	+ struct xsk_tx_metadata_compl *compl)
	138	+{
	139	+ if (!meta)
	140	+ return;
	141	+
	142	+ if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
	143	+ compl->tx_timestamp = &meta->completion.tx_timestamp;
	144	+ else
	145	+ compl->tx_timestamp = NULL;
	146	+}
	147	+
	148	+/**
	149	+ * xsk_tx_metadata_request - Evaluate AF_XDP TX metadata at submission
	150	+ * and call appropriate xsk_tx_metadata_ops operation.
	151	+ * @meta: pointer to AF_XDP metadata area
	152	+ * @ops: pointer to struct xsk_tx_metadata_ops
	153	+ * @priv: pointer to driver-private aread
	154	+ *
	155	+ * This function should be called by the networking device when
	156	+ * it prepares AF_XDP egress packet.
	157	+ */
	158	+static inline void xsk_tx_metadata_request(const struct xsk_tx_metadata *meta,
	159	+ const struct xsk_tx_metadata_ops *ops,
	160	+ void *priv)
	161	+{
	162	+ if (!meta)
	163	+ return;
	164	+
	165	+ if (ops->tmo_request_timestamp)
	166	+ if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
	167	+ ops->tmo_request_timestamp(priv);
	168	+
	169	+ if (ops->tmo_request_checksum)
	170	+ if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM)
	171	+ ops->tmo_request_checksum(meta->request.csum_start,
	172	+ meta->request.csum_offset, priv);
	173	+}
	174	+
	175	+/**
	176	+ * xsk_tx_metadata_complete - Evaluate AF_XDP TX metadata at completion
	177	+ * and call appropriate xsk_tx_metadata_ops operation.
	178	+ * @compl: pointer to completion metadata produced from xsk_tx_metadata_to_compl
	179	+ * @ops: pointer to struct xsk_tx_metadata_ops
	180	+ * @priv: pointer to driver-private aread
	181	+ *
	182	+ * This function should be called by the networking device upon
	183	+ * AF_XDP egress completion.
	184	+ */
	185	+static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl,
	186	+ const struct xsk_tx_metadata_ops *ops,
	187	+ void *priv)
	188	+{
	189	+ if (!compl)
	190	+ return;
	191	+
	192	+ *compl->tx_timestamp = ops->tmo_fill_timestamp(priv);
	193	+}
	194	+
102	195	#else
103	196
104	197	static inline int xsk_generic_rcv(struct xdp_sock xs, struct xdp_buff xdp)
@@ -115,6 +208,23 @@ static inline void __xsk_map_flush(void) · 10 unmodified lines
115	208	{
116	209	}
117	210
	211	+static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
	212	+ struct xsk_tx_metadata_compl *compl)
	213	+{
	214	+}
	215	+
	216	+static inline void xsk_tx_metadata_request(struct xsk_tx_metadata *meta,
	217	+ const struct xsk_tx_metadata_ops *ops,
	218	+ void *priv)
	219	+{
	220	+}
	221	+
	222	+static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl,
	223	+ const struct xsk_tx_metadata_ops *ops,
	224	+ void *priv)
	225	+{
	226	+}
	227	+
118	228	#endif /* CONFIG_XDP_SOCKETS */
119	229
120	230	#if defined(CONFIG_XDP_SOCKETS) && defined(CONFIG_DEBUG_NET)

include/net/xdp_sock_drv.h+13−0

@@ -165,6 +165,14 @@ static inline void xsk_buff_raw_get_data(struct xsk_buff_pool pool, u64 addr) · 164 unmodified lines
165	165	return xp_raw_get_data(pool, addr);
166	166	}
167	167
	168	+static inline struct xsk_tx_metadata xsk_buff_get_metadata(struct xsk_buff_pool pool, u64 addr)
	169	+{
	170	+ if (!pool->tx_metadata_len)
	171	+ return NULL;
	172	+
	173	+ return xp_raw_get_data(pool, addr) - pool->tx_metadata_len;
	174	+}
	175	+
168	176	static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff xdp, struct xsk_buff_pool pool)
169	177	{
170	178	struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
@@ -324,6 +332,11 @@ static inline void xsk_buff_raw_get_data(struct xsk_buff_pool pool, u64 addr) · 153 unmodified lines
324	332	return NULL;
325	333	}
326	334
	335	+static inline struct xsk_tx_metadata xsk_buff_get_metadata(struct xsk_buff_pool pool, u64 addr)
	336	+{
	337	+ return NULL;
	338	+}
	339	+
327	340	static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff xdp, struct xsk_buff_pool pool)
328	341	{
329	342	}

include/net/xsk_buff_pool.h+6−0

@@ -33,6 +33,7 @@ struct xdp_buff_xsk { · 32 unmodified lines
33	33	};
34	34
35	35	#define XSK_CHECK_PRIV_TYPE(t) BUILD_BUG_ON(sizeof(t) > offsetofend(struct xdp_buff_xsk, cb))
	36	+#define XSK_TX_COMPL_FITS(t) BUILD_BUG_ON(sizeof(struct xsk_tx_metadata_compl) > sizeof(t))
36	37
37	38	struct xsk_dma_map {
38	39	dma_addr_t *dma_pages;
@@ -234,4 +235,9 @@ static inline u64 xp_get_handle(struct xdp_buff_xsk *xskb) · 195 unmodified lines
234	235	return xskb->orig_addr + (offset << XSK_UNALIGNED_BUF_OFFSET_SHIFT);
235	236	}
236	237
	238	+static inline bool xp_tx_metadata_enabled(const struct xsk_buff_pool *pool)
	239	+{
	240	+ return pool->tx_metadata_len > 0;
	241	+}
	242	+
237	243	#endif /* XSK_BUFF_POOL_H_ */

include/uapi/linux/if_xdp.h+38−0

@@ -106,6 +106,41 @@ struct xdp_options { · 105 unmodified lines
106	106	#define XSK_UNALIGNED_BUF_ADDR_MASK \
107	107	((1ULL << XSK_UNALIGNED_BUF_OFFSET_SHIFT) - 1)
108	108
	109	+/* Request transmit timestamp. Upon completion, put it into tx_timestamp
	110	+ * field of struct xsk_tx_metadata.
	111	+ */
	112	+#define XDP_TXMD_FLAGS_TIMESTAMP (1 << 0)
	113	+
	114	+/* Request transmit checksum offload. Checksum start position and offset
	115	+ * are communicated via csum_start and csum_offset fields of struct
	116	+ * xsk_tx_metadata.
	117	+ */
	118	+#define XDP_TXMD_FLAGS_CHECKSUM (1 << 1)
	119	+
	120	+/* AF_XDP offloads request. 'request' union member is consumed by the driver
	121	+ * when the packet is being transmitted. 'completion' union member is
	122	+ * filled by the driver when the transmit completion arrives.
	123	+ */
	124	+struct xsk_tx_metadata {
	125	+ __u64 flags;
	126	+
	127	+ union {
	128	+ struct {
	129	+ /* XDP_TXMD_FLAGS_CHECKSUM */
	130	+
	131	+ /* Offset from desc->addr where checksumming should start. */
	132	+ __u16 csum_start;
	133	+ /* Offset from csum_start where checksum should be stored. */
	134	+ __u16 csum_offset;
	135	+ } request;
	136	+
	137	+ struct {
	138	+ /* XDP_TXMD_FLAGS_TIMESTAMP */
	139	+ __u64 tx_timestamp;
	140	+ } completion;
	141	+ };
	142	+};
	143	+
109	144	/* Rx/Tx descriptor */
110	145	struct xdp_desc {
111	146	__u64 addr;
@@ -122,4 +157,7 @@ struct xdp_desc { · 10 unmodified lines
122	157	*/
123	158	#define XDP_PKT_CONTD (1 << 0)
124	159
	160	+/* TX packet carries valid metadata. */
	161	+#define XDP_TX_METADATA (1 << 1)
	162	+
125	163	#endif /* _LINUX_IF_XDP_H */

include/uapi/linux/netdev.h+16−0

@@ -53,12 +53,28 @@ enum netdev_xdp_rx_metadata { · 52 unmodified lines
53	53	NETDEV_XDP_RX_METADATA_MASK = 3,
54	54	};
55	55
	56	+/**
	57	+ * enum netdev_xsk_flags
	58	+ * @NETDEV_XSK_FLAGS_TX_TIMESTAMP: HW timestamping egress packets is supported
	59	+ * by the driver.
	60	+ * @NETDEV_XSK_FLAGS_TX_CHECKSUM: L3 checksum HW offload is supported by the
	61	+ * driver.
	62	+ */
	63	+enum netdev_xsk_flags {
	64	+ NETDEV_XSK_FLAGS_TX_TIMESTAMP = 1,
	65	+ NETDEV_XSK_FLAGS_TX_CHECKSUM = 2,
	66	+
	67	+ /* private: */
	68	+ NETDEV_XSK_FLAGS_MASK = 3,
	69	+};
	70	+
56	71	enum {
57	72	NETDEV_A_DEV_IFINDEX = 1,
58	73	NETDEV_A_DEV_PAD,
59	74	NETDEV_A_DEV_XDP_FEATURES,
60	75	NETDEV_A_DEV_XDP_ZC_MAX_SEGS,
61	76	NETDEV_A_DEV_XDP_RX_METADATA_FEATURES,
	77	+ NETDEV_A_DEV_XSK_FEATURES,
62	78
63	79	__NETDEV_A_DEV_MAX,
64	80	NETDEV_A_DEV_MAX = (__NETDEV_A_DEV_MAX - 1)

xsk: Add TX timestamp and TX checksum offload support

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code