xsk: Add TX timestamp and TX checksum offload support

This change actually defines the (initial) metadata layout that should be used by AF_XDP userspace (xsk_tx_metadata). The first field is flags which requests appropriate offloads, followed by the offload-specific fields. The supported per-device offloads are exported via netlink (new xsk-flags). The offloads themselves are still implemented in a bit of a framework-y fashion that's left from my initial kfunc attempt. I'm introducing new xsk_tx_metadata_ops which drivers are supposed to implement. The drivers are also supposed to call xsk_tx_metadata_request/xsk_tx_metadata_complete in the right places. Since xsk_tx_metadata_{request,_complete} are static inline, we don't incur any extra overhead doing indirect calls. The benefit of this scheme is as follows: - keeps all metadata layout parsing away from driver code - makes it easy to grep and see which drivers implement what - don't need any extra flags to maintain to keep track of what offloads are implemented; if the callback is implemented - the offload is supported (used by netlink reporting code) Two offloads are defined right now: 1. XDP_TXMD_FLAGS_CHECKSUM: skb-style csum_start+csum_offset 2. XDP_TXMD_FLAGS_TIMESTAMP: writes TX timestamp back into metadata area upon completion (tx_timestamp field) XDP_TXMD_FLAGS_TIMESTAMP is also implemented for XDP_COPY mode: it writes SW timestamp from the skb destructor (note I'm reusing hwtstamps to pass metadata pointer). The struct is forward-compatible and can be extended in the future by appending more fields.

Stanislav Fomichev committed 2 years ago

Commit 48eb03d1 parent 341ac9815 files · +348 −9Browse files

Unified · Split

15 files

Documentation/netlink/specs/netdev.yaml +18 −1
include/linux/netdevice.h +2 −0
include/linux/skbuff.h +13 −1
include/net/xdp_sock.h +110 −0
include/net/xdp_sock_drv.h +13 −0
include/net/xsk_buff_pool.h +6 −0
include/uapi/linux/if_xdp.h +38 −0
include/uapi/linux/netdev.h +16 −0
net/core/netdev-genl.c +12 −1
net/xdp/xsk.c +34 −0
net/xdp/xsk_queue.h +1 −1
tools/include/uapi/linux/if_xdp.h +47 −5
tools/include/uapi/linux/netdev.h +16 −0
tools/net/ynl/generated/netdev-user.c +19 −0
tools/net/ynl/generated/netdev-user.h +3 −0

Documentation/netlink/specs/netdev.yaml+18−1

@@ -45,7 +45,6 @@ definitions: · 44 unmodified lines
45	45	-
46	46	type: flags
47	47	name: xdp-rx-metadata
48		− render-max: true
49	48	entries:
50	49	-
51	50	name: timestamp
@@ -55,6 +54,18 @@ definitions: · 3 unmodified lines
55	54	name: hash
56	55	doc:
57	56	Device is capable of exposing receive packet hash via bpf_xdp_metadata_rx_hash().
	57	+ -
	58	+ type: flags
	59	+ name: xsk-flags
	60	+ entries:
	61	+ -
	62	+ name: tx-timestamp
	63	+ doc:
	64	+ HW timestamping egress packets is supported by the driver.
	65	+ -
	66	+ name: tx-checksum
	67	+ doc:
	68	+ L3 checksum HW offload is supported by the driver.
58	69
59	70	attribute-sets:
60	71	-
@@ -86,6 +97,11 @@ attribute-sets: · 25 unmodified lines
86	97	See Documentation/networking/xdp-rx-metadata.rst for more details.
87	98	type: u64
88	99	enum: xdp-rx-metadata
	100	+ -
	101	+ name: xsk-features
	102	+ doc: Bitmask of enabled AF_XDP features.
	103	+ type: u64
	104	+ enum: xsk-flags
89	105
90	106	operations:
91	107	list:
@@ -103,6 +119,7 @@ operations: · 11 unmodified lines
103	119	- xdp-features
104	120	- xdp-zc-max-segs
105	121	- xdp-rx-metadata-features
	122	+ - xsk-features
106	123	dump:
107	124	reply: *dev-all
108	125	-

include/linux/netdevice.h+2−0

@@ -1865,6 +1865,7 @@ enum netdev_stat_type { · 1864 unmodified lines
1865	1865	* @netdev_ops: Includes several pointers to callbacks,
1866	1866	* if one wants to override the ndo_*() functions
1867	1867	* @xdp_metadata_ops: Includes pointers to XDP metadata callbacks.
	1868	+ * @xsk_tx_metadata_ops: Includes pointers to AF_XDP TX metadata callbacks.
1868	1869	* @ethtool_ops: Management operations
1869	1870	* @l3mdev_ops: Layer 3 master device operations
1870	1871	* @ndisc_ops: Includes callbacks for different IPv6 neighbour
@@ -2128,6 +2129,7 @@ struct net_device { · 257 unmodified lines
2128	2129	unsigned long long priv_flags;
2129	2130	const struct net_device_ops *netdev_ops;
2130	2131	const struct xdp_metadata_ops *xdp_metadata_ops;
	2132	+ const struct xsk_tx_metadata_ops *xsk_tx_metadata_ops;
2131	2133	int ifindex;
2132	2134	unsigned short gflags;
2133	2135	unsigned short hard_header_len;

include/linux/skbuff.h+13−1

@@ -566,6 +566,15 @@ struct ubuf_info_msgzc { · 565 unmodified lines
566	566	int mm_account_pinned_pages(struct mmpin *mmp, size_t size);
567	567	void mm_unaccount_pinned_pages(struct mmpin *mmp);
568	568
	569	+/* Preserve some data across TX submission and completion.
	570	+ *
	571	+ * Note, this state is stored in the driver. Extending the layout
	572	+ * might need some special care.
	573	+ */
	574	+struct xsk_tx_metadata_compl {
	575	+ __u64 *tx_timestamp;
	576	+};
	577	+
569	578	/* This data is invariant across clones and lives at
570	579	* the end of the header data, ie. at skb->end.
571	580	*/
@@ -578,7 +587,10 @@ struct skb_shared_info { · 6 unmodified lines
578	587	/* Warning: this field is not always filled in (UFO)! */
579	588	unsigned short gso_segs;
580	589	struct sk_buff *frag_list;
581		− struct skb_shared_hwtstamps hwtstamps;
	590	+ union {
	591	+ struct skb_shared_hwtstamps hwtstamps;
	592	+ struct xsk_tx_metadata_compl xsk_meta;
	593	+ };
582	594	unsigned int gso_type;
583	595	u32 tskey;
584	596

include/net/xdp_sock.h+110−0

@@ -93,12 +93,105 @@ struct xdp_sock { · 92 unmodified lines
93	93	struct xsk_queue cq_tmp; / Only as tmp storage before bind */
94	94	};
95	95
	96	+/*
	97	+ * AF_XDP TX metadata hooks for network devices.
	98	+ * The following hooks can be defined; unless noted otherwise, they are
	99	+ * optional and can be filled with a null pointer.
	100	+ *
	101	+ * void (tmo_request_timestamp)(void priv)
	102	+ * Called when AF_XDP frame requested egress timestamp.
	103	+ *
	104	+ * u64 (tmo_fill_timestamp)(void priv)
	105	+ * Called when AF_XDP frame, that had requested egress timestamp,
	106	+ * received a completion. The hook needs to return the actual HW timestamp.
	107	+ *
	108	+ * void (tmo_request_checksum)(u16 csum_start, u16 csum_offset, void priv)
	109	+ * Called when AF_XDP frame requested HW checksum offload. csum_start
	110	+ * indicates position where checksumming should start.
	111	+ * csum_offset indicates position where checksum should be stored.
	112	+ *
	113	+ */
	114	+struct xsk_tx_metadata_ops {
	115	+ void (tmo_request_timestamp)(void priv);
	116	+ u64 (tmo_fill_timestamp)(void priv);
	117	+ void (tmo_request_checksum)(u16 csum_start, u16 csum_offset, void priv);
	118	+};
	119	+
96	120	#ifdef CONFIG_XDP_SOCKETS
97	121
98	122	int xsk_generic_rcv(struct xdp_sock xs, struct xdp_buff xdp);
99	123	int __xsk_map_redirect(struct xdp_sock xs, struct xdp_buff xdp);
100	124	void __xsk_map_flush(void);
101	125
	126	+/**
	127	+ * xsk_tx_metadata_to_compl - Save enough relevant metadata information
	128	+ * to perform tx completion in the future.
	129	+ * @meta: pointer to AF_XDP metadata area
	130	+ * @compl: pointer to output struct xsk_tx_metadata_to_compl
	131	+ *
	132	+ * This function should be called by the networking device when
	133	+ * it prepares AF_XDP egress packet. The value of @compl should be stored
	134	+ * and passed to xsk_tx_metadata_complete upon TX completion.
	135	+ */
	136	+static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
	137	+ struct xsk_tx_metadata_compl *compl)
	138	+{
	139	+ if (!meta)
	140	+ return;
	141	+
	142	+ if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
	143	+ compl->tx_timestamp = &meta->completion.tx_timestamp;
	144	+ else
	145	+ compl->tx_timestamp = NULL;
	146	+}
	147	+
	148	+/**
	149	+ * xsk_tx_metadata_request - Evaluate AF_XDP TX metadata at submission
	150	+ * and call appropriate xsk_tx_metadata_ops operation.
	151	+ * @meta: pointer to AF_XDP metadata area
	152	+ * @ops: pointer to struct xsk_tx_metadata_ops
	153	+ * @priv: pointer to driver-private aread
	154	+ *
	155	+ * This function should be called by the networking device when
	156	+ * it prepares AF_XDP egress packet.
	157	+ */
	158	+static inline void xsk_tx_metadata_request(const struct xsk_tx_metadata *meta,
	159	+ const struct xsk_tx_metadata_ops *ops,
	160	+ void *priv)
	161	+{
	162	+ if (!meta)
	163	+ return;
	164	+
	165	+ if (ops->tmo_request_timestamp)
	166	+ if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
	167	+ ops->tmo_request_timestamp(priv);
	168	+
	169	+ if (ops->tmo_request_checksum)
	170	+ if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM)
	171	+ ops->tmo_request_checksum(meta->request.csum_start,
	172	+ meta->request.csum_offset, priv);
	173	+}
	174	+
	175	+/**
	176	+ * xsk_tx_metadata_complete - Evaluate AF_XDP TX metadata at completion
	177	+ * and call appropriate xsk_tx_metadata_ops operation.
	178	+ * @compl: pointer to completion metadata produced from xsk_tx_metadata_to_compl
	179	+ * @ops: pointer to struct xsk_tx_metadata_ops
	180	+ * @priv: pointer to driver-private aread
	181	+ *
	182	+ * This function should be called by the networking device upon
	183	+ * AF_XDP egress completion.
	184	+ */
	185	+static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl,
	186	+ const struct xsk_tx_metadata_ops *ops,
	187	+ void *priv)
	188	+{
	189	+ if (!compl)
	190	+ return;
	191	+
	192	+ *compl->tx_timestamp = ops->tmo_fill_timestamp(priv);
	193	+}
	194	+
102	195	#else
103	196
104	197	static inline int xsk_generic_rcv(struct xdp_sock xs, struct xdp_buff xdp)
@@ -115,6 +208,23 @@ static inline void __xsk_map_flush(void) · 10 unmodified lines
115	208	{
116	209	}
117	210
	211	+static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
	212	+ struct xsk_tx_metadata_compl *compl)
	213	+{
	214	+}
	215	+
	216	+static inline void xsk_tx_metadata_request(struct xsk_tx_metadata *meta,
	217	+ const struct xsk_tx_metadata_ops *ops,
	218	+ void *priv)
	219	+{
	220	+}
	221	+
	222	+static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl,
	223	+ const struct xsk_tx_metadata_ops *ops,
	224	+ void *priv)
	225	+{
	226	+}
	227	+
118	228	#endif /* CONFIG_XDP_SOCKETS */
119	229
120	230	#if defined(CONFIG_XDP_SOCKETS) && defined(CONFIG_DEBUG_NET)

include/net/xdp_sock_drv.h+13−0

@@ -165,6 +165,14 @@ static inline void xsk_buff_raw_get_data(struct xsk_buff_pool pool, u64 addr) · 164 unmodified lines
165	165	return xp_raw_get_data(pool, addr);
166	166	}
167	167
	168	+static inline struct xsk_tx_metadata xsk_buff_get_metadata(struct xsk_buff_pool pool, u64 addr)
	169	+{
	170	+ if (!pool->tx_metadata_len)
	171	+ return NULL;
	172	+
	173	+ return xp_raw_get_data(pool, addr) - pool->tx_metadata_len;
	174	+}
	175	+
168	176	static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff xdp, struct xsk_buff_pool pool)
169	177	{
170	178	struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
@@ -324,6 +332,11 @@ static inline void xsk_buff_raw_get_data(struct xsk_buff_pool pool, u64 addr) · 153 unmodified lines
324	332	return NULL;
325	333	}
326	334
	335	+static inline struct xsk_tx_metadata xsk_buff_get_metadata(struct xsk_buff_pool pool, u64 addr)
	336	+{
	337	+ return NULL;
	338	+}
	339	+
327	340	static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff xdp, struct xsk_buff_pool pool)
328	341	{
329	342	}

include/net/xsk_buff_pool.h+6−0

@@ -33,6 +33,7 @@ struct xdp_buff_xsk { · 32 unmodified lines
33	33	};
34	34
35	35	#define XSK_CHECK_PRIV_TYPE(t) BUILD_BUG_ON(sizeof(t) > offsetofend(struct xdp_buff_xsk, cb))
	36	+#define XSK_TX_COMPL_FITS(t) BUILD_BUG_ON(sizeof(struct xsk_tx_metadata_compl) > sizeof(t))
36	37
37	38	struct xsk_dma_map {
38	39	dma_addr_t *dma_pages;
@@ -234,4 +235,9 @@ static inline u64 xp_get_handle(struct xdp_buff_xsk *xskb) · 195 unmodified lines
234	235	return xskb->orig_addr + (offset << XSK_UNALIGNED_BUF_OFFSET_SHIFT);
235	236	}
236	237
	238	+static inline bool xp_tx_metadata_enabled(const struct xsk_buff_pool *pool)
	239	+{
	240	+ return pool->tx_metadata_len > 0;
	241	+}
	242	+
237	243	#endif /* XSK_BUFF_POOL_H_ */

include/uapi/linux/if_xdp.h+38−0

@@ -106,6 +106,41 @@ struct xdp_options { · 105 unmodified lines
106	106	#define XSK_UNALIGNED_BUF_ADDR_MASK \
107	107	((1ULL << XSK_UNALIGNED_BUF_OFFSET_SHIFT) - 1)
108	108
	109	+/* Request transmit timestamp. Upon completion, put it into tx_timestamp
	110	+ * field of struct xsk_tx_metadata.
	111	+ */
	112	+#define XDP_TXMD_FLAGS_TIMESTAMP (1 << 0)
	113	+
	114	+/* Request transmit checksum offload. Checksum start position and offset
	115	+ * are communicated via csum_start and csum_offset fields of struct
	116	+ * xsk_tx_metadata.
	117	+ */
	118	+#define XDP_TXMD_FLAGS_CHECKSUM (1 << 1)
	119	+
	120	+/* AF_XDP offloads request. 'request' union member is consumed by the driver
	121	+ * when the packet is being transmitted. 'completion' union member is
	122	+ * filled by the driver when the transmit completion arrives.
	123	+ */
	124	+struct xsk_tx_metadata {
	125	+ __u64 flags;
	126	+
	127	+ union {
	128	+ struct {
	129	+ /* XDP_TXMD_FLAGS_CHECKSUM */
	130	+
	131	+ /* Offset from desc->addr where checksumming should start. */
	132	+ __u16 csum_start;
	133	+ /* Offset from csum_start where checksum should be stored. */
	134	+ __u16 csum_offset;
	135	+ } request;
	136	+
	137	+ struct {
	138	+ /* XDP_TXMD_FLAGS_TIMESTAMP */
	139	+ __u64 tx_timestamp;
	140	+ } completion;
	141	+ };
	142	+};
	143	+
109	144	/* Rx/Tx descriptor */
110	145	struct xdp_desc {
111	146	__u64 addr;
@@ -122,4 +157,7 @@ struct xdp_desc { · 10 unmodified lines
122	157	*/
123	158	#define XDP_PKT_CONTD (1 << 0)
124	159
	160	+/* TX packet carries valid metadata. */
	161	+#define XDP_TX_METADATA (1 << 1)
	162	+
125	163	#endif /* _LINUX_IF_XDP_H */

include/uapi/linux/netdev.h+16−0

@@ -53,12 +53,28 @@ enum netdev_xdp_rx_metadata { · 52 unmodified lines
53	53	NETDEV_XDP_RX_METADATA_MASK = 3,
54	54	};
55	55
	56	+/**
	57	+ * enum netdev_xsk_flags
	58	+ * @NETDEV_XSK_FLAGS_TX_TIMESTAMP: HW timestamping egress packets is supported
	59	+ * by the driver.
	60	+ * @NETDEV_XSK_FLAGS_TX_CHECKSUM: L3 checksum HW offload is supported by the
	61	+ * driver.
	62	+ */
	63	+enum netdev_xsk_flags {
	64	+ NETDEV_XSK_FLAGS_TX_TIMESTAMP = 1,
	65	+ NETDEV_XSK_FLAGS_TX_CHECKSUM = 2,
	66	+
	67	+ /* private: */
	68	+ NETDEV_XSK_FLAGS_MASK = 3,
	69	+};
	70	+
56	71	enum {
57	72	NETDEV_A_DEV_IFINDEX = 1,
58	73	NETDEV_A_DEV_PAD,
59	74	NETDEV_A_DEV_XDP_FEATURES,
60	75	NETDEV_A_DEV_XDP_ZC_MAX_SEGS,
61	76	NETDEV_A_DEV_XDP_RX_METADATA_FEATURES,
	77	+ NETDEV_A_DEV_XSK_FEATURES,
62	78
63	79	__NETDEV_A_DEV_MAX,
64	80	NETDEV_A_DEV_MAX = (__NETDEV_A_DEV_MAX - 1)

net/core/netdev-genl.c+12−1

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	6	#include <net/net_namespace.h>
7	7	#include <net/sock.h>
8	8	#include <net/xdp.h>
	9	+#include <net/xdp_sock.h>
9	10
10	11	#include "netdev-genl-gen.h"
11	12
@@ -13,6 +14,7 @@ static int · 1 unmodified line
13	14	netdev_nl_dev_fill(struct net_device netdev, struct sk_buff rsp,
14	15	const struct genl_info *info)
15	16	{
	17	+ u64 xsk_features = 0;
16	18	u64 xdp_rx_meta = 0;
17	19	void *hdr;
18	20
@@ -26,11 +28,20 @@ netdev_nl_dev_fill(struct net_device netdev, struct sk_buff rsp, · 7 unmodified lines
26	28	XDP_METADATA_KFUNC_xxx
27	29	#undef XDP_METADATA_KFUNC
28	30
	31	+ if (netdev->xsk_tx_metadata_ops) {
	32	+ if (netdev->xsk_tx_metadata_ops->tmo_fill_timestamp)
	33	+ xsk_features \|= NETDEV_XSK_FLAGS_TX_TIMESTAMP;
	34	+ if (netdev->xsk_tx_metadata_ops->tmo_request_checksum)
	35	+ xsk_features \|= NETDEV_XSK_FLAGS_TX_CHECKSUM;
	36	+ }
	37	+
29	38	if (nla_put_u32(rsp, NETDEV_A_DEV_IFINDEX, netdev->ifindex) \|\|
30	39	nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_FEATURES,
31	40	netdev->xdp_features, NETDEV_A_DEV_PAD) \|\|
32	41	nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_RX_METADATA_FEATURES,
33		− xdp_rx_meta, NETDEV_A_DEV_PAD)) {
	42	+ xdp_rx_meta, NETDEV_A_DEV_PAD) \|\|
	43	+ nla_put_u64_64bit(rsp, NETDEV_A_DEV_XSK_FEATURES,
	44	+ xsk_features, NETDEV_A_DEV_PAD)) {
34	45	genlmsg_cancel(rsp, hdr);
35	46	return -EINVAL;
36	47	}

net/xdp/xsk.c+34−0

@@ -571,6 +571,13 @@ static u32 xsk_get_num_desc(struct sk_buff *skb) · 570 unmodified lines
571	571
572	572	static void xsk_destruct_skb(struct sk_buff *skb)
573	573	{
	574	+ struct xsk_tx_metadata_compl *compl = &skb_shinfo(skb)->xsk_meta;
	575	+
	576	+ if (compl->tx_timestamp) {
	577	+ /* sw completion timestamp, not a real one */
	578	+ *compl->tx_timestamp = ktime_get_tai_fast_ns();
	579	+ }
	580	+
574	581	xsk_cq_submit_locked(xdp_sk(skb->sk), xsk_get_num_desc(skb));
575	582	sock_wfree(skb);
576	583	}
@@ -655,8 +662,10 @@ static struct sk_buff xsk_build_skb_zerocopy(struct xdp_sock xs, · 78 unmodified lines
655	662	static struct sk_buff xsk_build_skb(struct xdp_sock xs,
656	663	struct xdp_desc *desc)
657	664	{
	665	+ struct xsk_tx_metadata *meta = NULL;
658	666	struct net_device *dev = xs->dev;
659	667	struct sk_buff *skb = xs->skb;
	668	+ bool first_frag = false;
660	669	int err;
661	670
662	671	if (dev->priv_flags & IFF_TX_SKB_NO_LINEAR) {
@@ -687,6 +696,8 @@ static struct sk_buff xsk_build_skb(struct xdp_sock xs, · 24 unmodified lines
687	696	kfree_skb(skb);
688	697	goto free_err;
689	698	}
	699	+
	700	+ first_frag = true;
690	701	} else {
691	702	int nr_frags = skb_shinfo(skb)->nr_frags;
692	703	struct page *page;
@@ -709,12 +720,35 @@ static struct sk_buff xsk_build_skb(struct xdp_sock xs, · 16 unmodified lines
709	720
710	721	skb_add_rx_frag(skb, nr_frags, page, 0, len, 0);
711	722	}
	723	+
	724	+ if (first_frag && desc->options & XDP_TX_METADATA) {
	725	+ if (unlikely(xs->pool->tx_metadata_len == 0)) {
	726	+ err = -EINVAL;
	727	+ goto free_err;
	728	+ }
	729	+
	730	+ meta = buffer - xs->pool->tx_metadata_len;
	731	+
	732	+ if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM) {
	733	+ if (unlikely(meta->request.csum_start +
	734	+ meta->request.csum_offset +
	735	+ sizeof(__sum16) > len)) {
	736	+ err = -EINVAL;
	737	+ goto free_err;
	738	+ }
	739	+
	740	+ skb->csum_start = hr + meta->request.csum_start;
	741	+ skb->csum_offset = meta->request.csum_offset;
	742	+ skb->ip_summed = CHECKSUM_PARTIAL;
	743	+ }
	744	+ }
712	745	}
713	746
714	747	skb->dev = dev;
715	748	skb->priority = READ_ONCE(xs->sk.sk_priority);
716	749	skb->mark = READ_ONCE(xs->sk.sk_mark);
717	750	skb->destructor = xsk_destruct_skb;
	751	+ xsk_tx_metadata_to_compl(meta, &skb_shinfo(skb)->xsk_meta);
718	752	xsk_set_destructor_arg(skb);
719	753
720	754	return skb;

net/xdp/xsk_queue.h+1−1

@@ -137,7 +137,7 @@ static inline bool xskq_cons_read_addr_unchecked(struct xsk_queue q, u64 addr) · 136 unmodified lines
137	137
138	138	static inline bool xp_unused_options_set(u32 options)
139	139	{
140		− return options & ~XDP_PKT_CONTD;
	140	+ return options & ~(XDP_PKT_CONTD \| XDP_TX_METADATA);
141	141	}
142	142
143	143	static inline bool xp_aligned_validate_desc(struct xsk_buff_pool *pool,

tools/include/uapi/linux/if_xdp.h+47−5

@@ -26,11 +26,11 @@ · 25 unmodified lines
26	26	*/
27	27	#define XDP_USE_NEED_WAKEUP (1 << 3)
28	28	/* By setting this option, userspace application indicates that it can
29		− * handle multiple descriptors per packet thus enabling xsk core to split
	29	+ * handle multiple descriptors per packet thus enabling AF_XDP to split
30	30	* multi-buffer XDP frames into multiple Rx descriptors. Without this set
31		− * such frames will be dropped by xsk.
	31	+ * such frames will be dropped.
32	32	*/
33		−#define XDP_USE_SG (1 << 4)
	33	+#define XDP_USE_SG (1 << 4)
34	34
35	35	/* Flags for xsk_umem_config flags */
36	36	#define XDP_UMEM_UNALIGNED_CHUNK_FLAG (1 << 0)
@@ -106,6 +106,41 @@ struct xdp_options { · 69 unmodified lines
106	106	#define XSK_UNALIGNED_BUF_ADDR_MASK \
107	107	((1ULL << XSK_UNALIGNED_BUF_OFFSET_SHIFT) - 1)
108	108
	109	+/* Request transmit timestamp. Upon completion, put it into tx_timestamp
	110	+ * field of union xsk_tx_metadata.
	111	+ */
	112	+#define XDP_TXMD_FLAGS_TIMESTAMP (1 << 0)
	113	+
	114	+/* Request transmit checksum offload. Checksum start position and offset
	115	+ * are communicated via csum_start and csum_offset fields of union
	116	+ * xsk_tx_metadata.
	117	+ */
	118	+#define XDP_TXMD_FLAGS_CHECKSUM (1 << 1)
	119	+
	120	+/* AF_XDP offloads request. 'request' union member is consumed by the driver
	121	+ * when the packet is being transmitted. 'completion' union member is
	122	+ * filled by the driver when the transmit completion arrives.
	123	+ */
	124	+struct xsk_tx_metadata {
	125	+ __u64 flags;
	126	+
	127	+ union {
	128	+ struct {
	129	+ /* XDP_TXMD_FLAGS_CHECKSUM */
	130	+
	131	+ /* Offset from desc->addr where checksumming should start. */
	132	+ __u16 csum_start;
	133	+ /* Offset from csum_start where checksum should be stored. */
	134	+ __u16 csum_offset;
	135	+ } request;
	136	+
	137	+ struct {
	138	+ /* XDP_TXMD_FLAGS_TIMESTAMP */
	139	+ __u64 tx_timestamp;
	140	+ } completion;
	141	+ };
	142	+};
	143	+
109	144	/* Rx/Tx descriptor */
110	145	struct xdp_desc {
111	146	__u64 addr;
@@ -113,9 +148,16 @@ struct xdp_desc { · 1 unmodified line
113	148	__u32 options;
114	149	};
115	150
116		−/* Flag indicating packet constitutes of multiple buffers*/
	151	+/* UMEM descriptor is __u64 */
	152	+
	153	+/* Flag indicating that the packet continues with the buffer pointed out by the
	154	+ * next frame in the ring. The end of the packet is signalled by setting this
	155	+ * bit to zero. For single buffer packets, every descriptor has 'options' set
	156	+ * to 0 and this maintains backward compatibility.
	157	+ */
117	158	#define XDP_PKT_CONTD (1 << 0)
118	159
119		−/* UMEM descriptor is __u64 */
	160	+/* TX packet carries valid metadata. */
	161	+#define XDP_TX_METADATA (1 << 1)
120	162
121	163	#endif /* _LINUX_IF_XDP_H */

tools/include/uapi/linux/netdev.h+16−0

@@ -53,12 +53,28 @@ enum netdev_xdp_rx_metadata { · 52 unmodified lines
53	53	NETDEV_XDP_RX_METADATA_MASK = 3,
54	54	};
55	55
	56	+/**
	57	+ * enum netdev_xsk_flags
	58	+ * @NETDEV_XSK_FLAGS_TX_TIMESTAMP: HW timestamping egress packets is supported
	59	+ * by the driver.
	60	+ * @NETDEV_XSK_FLAGS_TX_CHECKSUM: L3 checksum HW offload is supported by the
	61	+ * driver.
	62	+ */
	63	+enum netdev_xsk_flags {
	64	+ NETDEV_XSK_FLAGS_TX_TIMESTAMP = 1,
	65	+ NETDEV_XSK_FLAGS_TX_CHECKSUM = 2,
	66	+
	67	+ /* private: */
	68	+ NETDEV_XSK_FLAGS_MASK = 3,
	69	+};
	70	+
56	71	enum {
57	72	NETDEV_A_DEV_IFINDEX = 1,
58	73	NETDEV_A_DEV_PAD,
59	74	NETDEV_A_DEV_XDP_FEATURES,
60	75	NETDEV_A_DEV_XDP_ZC_MAX_SEGS,
61	76	NETDEV_A_DEV_XDP_RX_METADATA_FEATURES,
	77	+ NETDEV_A_DEV_XSK_FEATURES,
62	78
63	79	__NETDEV_A_DEV_MAX,
64	80	NETDEV_A_DEV_MAX = (__NETDEV_A_DEV_MAX - 1)

tools/net/ynl/generated/netdev-user.c+19−0

@@ -58,6 +58,19 @@ const char *netdev_xdp_rx_metadata_str(enum netdev_xdp_rx_metadata value) · 57 unmodified lines
58	58	return netdev_xdp_rx_metadata_strmap[value];
59	59	}
60	60
	61	+static const char * const netdev_xsk_flags_strmap[] = {
	62	+ [0] = "tx-timestamp",
	63	+ [1] = "tx-checksum",
	64	+};
	65	+
	66	+const char *netdev_xsk_flags_str(enum netdev_xsk_flags value)
	67	+{
	68	+ value = ffs(value) - 1;
	69	+ if (value < 0 \|\| value >= (int)MNL_ARRAY_SIZE(netdev_xsk_flags_strmap))
	70	+ return NULL;
	71	+ return netdev_xsk_flags_strmap[value];
	72	+}
	73	+
61	74	/* Policies */
62	75	struct ynl_policy_attr netdev_dev_policy[NETDEV_A_DEV_MAX + 1] = {
63	76	[NETDEV_A_DEV_IFINDEX] = { .name = "ifindex", .type = YNL_PT_U32, },
@@ -65,6 +78,7 @@ struct ynl_policy_attr netdev_dev_policy[NETDEV_A_DEV_MAX + 1] = { · 1 unmodified line
65	78	[NETDEV_A_DEV_XDP_FEATURES] = { .name = "xdp-features", .type = YNL_PT_U64, },
66	79	[NETDEV_A_DEV_XDP_ZC_MAX_SEGS] = { .name = "xdp-zc-max-segs", .type = YNL_PT_U32, },
67	80	[NETDEV_A_DEV_XDP_RX_METADATA_FEATURES] = { .name = "xdp-rx-metadata-features", .type = YNL_PT_U64, },
	81	+ [NETDEV_A_DEV_XSK_FEATURES] = { .name = "xsk-features", .type = YNL_PT_U64, },
68	82	};
69	83
70	84	struct ynl_policy_nest netdev_dev_nest = {
@@ -116,6 +130,11 @@ int netdev_dev_get_rsp_parse(const struct nlmsghdr nlh, void data) · 45 unmodified lines
116	130	return MNL_CB_ERROR;
117	131	dst->_present.xdp_rx_metadata_features = 1;
118	132	dst->xdp_rx_metadata_features = mnl_attr_get_u64(attr);
	133	+ } else if (type == NETDEV_A_DEV_XSK_FEATURES) {
	134	+ if (ynl_attr_validate(yarg, attr))
	135	+ return MNL_CB_ERROR;
	136	+ dst->_present.xsk_features = 1;
	137	+ dst->xsk_features = mnl_attr_get_u64(attr);
119	138	}
120	139	}
121	140

tools/net/ynl/generated/netdev-user.h+3−0

@@ -19,6 +19,7 @@ extern const struct ynl_family ynl_netdev_family; · 18 unmodified lines
19	19	const char *netdev_op_str(int op);
20	20	const char *netdev_xdp_act_str(enum netdev_xdp_act value);
21	21	const char *netdev_xdp_rx_metadata_str(enum netdev_xdp_rx_metadata value);
	22	+const char *netdev_xsk_flags_str(enum netdev_xsk_flags value);
22	23
23	24	/* Common nested types */
24	25	/* ============== NETDEV_CMD_DEV_GET ============== */
@@ -50,12 +51,14 @@ struct netdev_dev_get_rsp { · 25 unmodified lines
50	51	__u32 xdp_features:1;
51	52	__u32 xdp_zc_max_segs:1;
52	53	__u32 xdp_rx_metadata_features:1;
	54	+ __u32 xsk_features:1;
53	55	} _present;
54	56
55	57	__u32 ifindex;
56	58	__u64 xdp_features;
57	59	__u32 xdp_zc_max_segs;
58	60	__u64 xdp_rx_metadata_features;
	61	+ __u64 xsk_features;
59	62	};
60	63
61	64	void netdev_dev_get_rsp_free(struct netdev_dev_get_rsp *rsp);