/* SPDX-License-Identifier: GPL-2.0-only *//* Copyright (C) 2025 Intel Corporation */ #ifndef __LIBETH_XDP_H#define __LIBETH_XDP_H #include <linux/bpf_trace.h>#include <linux/unroll.h> #include <net/libeth/rx.h>#include <net/libeth/tx.h>#include <net/xsk_buff_pool.h> /* * Defined as bits to be able to use them as a mask on Rx. * Also used as internal return values on Tx. */enum {	LIBETH_XDP_PASS			= 0U,	LIBETH_XDP_DROP			= BIT(0),	LIBETH_XDP_ABORTED		= BIT(1),	LIBETH_XDP_TX			= BIT(2),	LIBETH_XDP_REDIRECT		= BIT(3),}; /* * &xdp_buff_xsk is the largest structure &libeth_xdp_buff gets casted to, * pick maximum pointer-compatible alignment. */#define __LIBETH_XDP_BUFF_ALIGN						      \	(IS_ALIGNED(sizeof(struct xdp_buff_xsk), 16) ? 16 :		      \	 IS_ALIGNED(sizeof(struct xdp_buff_xsk), 8) ? 8 :		      \	 sizeof(long)) /** * struct libeth_xdp_buff - libeth extension over &xdp_buff * @base: main &xdp_buff * @data: shortcut for @base.data * @desc: RQ descriptor containing metadata for this buffer * @priv: driver-private scratchspace * * The main reason for this is to have a pointer to the descriptor to be able * to quickly get frame metadata from xdpmo and driver buff-to-xdp callbacks * (as well as bigger alignment). * Pointer/layout-compatible with &xdp_buff and &xdp_buff_xsk. */struct libeth_xdp_buff {	union {		struct xdp_buff		base;		void			*data;	}; 	const void			*desc;	unsigned long			priv[]					__aligned(__LIBETH_XDP_BUFF_ALIGN);} __aligned(__LIBETH_XDP_BUFF_ALIGN);static_assert(offsetof(struct libeth_xdp_buff, data) ==	      offsetof(struct xdp_buff_xsk, xdp.data));static_assert(offsetof(struct libeth_xdp_buff, desc) ==	      offsetof(struct xdp_buff_xsk, cb));static_assert(IS_ALIGNED(sizeof(struct xdp_buff_xsk),			 __alignof(struct libeth_xdp_buff))); /** * __LIBETH_XDP_ONSTACK_BUFF - declare a &libeth_xdp_buff on the stack * @name: name of the variable to declare * @...: sizeof() of the driver-private data */#define __LIBETH_XDP_ONSTACK_BUFF(name, ...)				      \	___LIBETH_XDP_ONSTACK_BUFF(name, ##__VA_ARGS__)/** * LIBETH_XDP_ONSTACK_BUFF - declare a &libeth_xdp_buff on the stack * @name: name of the variable to declare * @...: type or variable name of the driver-private data */#define LIBETH_XDP_ONSTACK_BUFF(name, ...)				      \	__LIBETH_XDP_ONSTACK_BUFF(name, __libeth_xdp_priv_sz(__VA_ARGS__)) #define ___LIBETH_XDP_ONSTACK_BUFF(name, ...)				      \	__DEFINE_FLEX(struct libeth_xdp_buff, name, priv,		      \		      LIBETH_XDP_PRIV_SZ(__VA_ARGS__ + 0),		      \		      __uninitialized);					      \	LIBETH_XDP_ASSERT_PRIV_SZ(__VA_ARGS__ + 0) #define __libeth_xdp_priv_sz(...)					      \	CONCATENATE(__libeth_xdp_psz, COUNT_ARGS(__VA_ARGS__))(__VA_ARGS__) #define __libeth_xdp_psz0(...)#define __libeth_xdp_psz1(...)		sizeof(__VA_ARGS__) #define LIBETH_XDP_PRIV_SZ(sz)						      \	(ALIGN(sz, __alignof(struct libeth_xdp_buff)) / sizeof(long)) /* Performs XSK_CHECK_PRIV_TYPE() */#define LIBETH_XDP_ASSERT_PRIV_SZ(sz)					      \	static_assert(offsetofend(struct xdp_buff_xsk, cb) >=		      \		      struct_size_t(struct libeth_xdp_buff, priv,	      \				    LIBETH_XDP_PRIV_SZ(sz))) /* XDPSQ sharing */ DECLARE_STATIC_KEY_FALSE(libeth_xdpsq_share); /** * libeth_xdpsq_num - calculate optimal number of XDPSQs for this device + sys * @rxq: current number of active Rx queues * @txq: current number of active Tx queues * @max: maximum number of Tx queues * * Each RQ must have its own XDPSQ for XSk pairs, each CPU must have own XDPSQ * for lockless sending (``XDP_TX``, .ndo_xdp_xmit()). Cap the maximum of these * two with the number of SQs the device can have (minus used ones). * * Return: number of XDP Tx queues the device needs to use. */static inline u32 libeth_xdpsq_num(u32 rxq, u32 txq, u32 max){	return min(max(nr_cpu_ids, rxq), max - txq);} /** * libeth_xdpsq_shared - whether XDPSQs can be shared between several CPUs * @num: number of active XDPSQs * * Return: true if there's no 1:1 XDPSQ/CPU association, false otherwise. */static inline bool libeth_xdpsq_shared(u32 num){	return num < nr_cpu_ids;} /** * libeth_xdpsq_id - get XDPSQ index corresponding to this CPU * @num: number of active XDPSQs * * Helper for libeth_xdp routines, do not use in drivers directly. * * Return: XDPSQ index needs to be used on this CPU. */static inline u32 libeth_xdpsq_id(u32 num){	u32 ret = raw_smp_processor_id(); 	if (static_branch_unlikely(&libeth_xdpsq_share) &&	    libeth_xdpsq_shared(num))		ret %= num; 	return ret;} void __libeth_xdpsq_get(struct libeth_xdpsq_lock *lock,			const struct net_device *dev);void __libeth_xdpsq_put(struct libeth_xdpsq_lock *lock,			const struct net_device *dev); /** * libeth_xdpsq_get - initialize &libeth_xdpsq_lock * @lock: lock to initialize * @dev: netdev which this lock belongs to * @share: whether XDPSQs can be shared * * Tracks the current XDPSQ association and enables the static lock * if needed. */static inline void libeth_xdpsq_get(struct libeth_xdpsq_lock *lock,				    const struct net_device *dev,				    bool share){	if (unlikely(share))		__libeth_xdpsq_get(lock, dev);} /** * libeth_xdpsq_put - deinitialize &libeth_xdpsq_lock * @lock: lock to deinitialize * @dev: netdev which this lock belongs to * * Tracks the current XDPSQ association and disables the static lock * if needed. */static inline void libeth_xdpsq_put(struct libeth_xdpsq_lock *lock,				    const struct net_device *dev){	if (static_branch_unlikely(&libeth_xdpsq_share) && lock->share)		__libeth_xdpsq_put(lock, dev);} void __libeth_xdpsq_lock(struct libeth_xdpsq_lock *lock);void __libeth_xdpsq_unlock(struct libeth_xdpsq_lock *lock); /** * libeth_xdpsq_lock - grab &libeth_xdpsq_lock if needed * @lock: lock to take * * Touches the underlying spinlock only if the static key is enabled * and the queue itself is marked as shareable. */static inline void libeth_xdpsq_lock(struct libeth_xdpsq_lock *lock){	if (static_branch_unlikely(&libeth_xdpsq_share) && lock->share)		__libeth_xdpsq_lock(lock);} /** * libeth_xdpsq_unlock - free &libeth_xdpsq_lock if needed * @lock: lock to free * * Touches the underlying spinlock only if the static key is enabled * and the queue itself is marked as shareable. */static inline void libeth_xdpsq_unlock(struct libeth_xdpsq_lock *lock){	if (static_branch_unlikely(&libeth_xdpsq_share) && lock->share)		__libeth_xdpsq_unlock(lock);} /* XDPSQ clean-up timers */ void libeth_xdpsq_init_timer(struct libeth_xdpsq_timer *timer, void *xdpsq,			     struct libeth_xdpsq_lock *lock,			     void (*poll)(struct work_struct *work)); /** * libeth_xdpsq_deinit_timer - deinitialize &libeth_xdpsq_timer * @timer: timer to deinitialize * * Flush and disable the underlying workqueue. */static inline void libeth_xdpsq_deinit_timer(struct libeth_xdpsq_timer *timer){	cancel_delayed_work_sync(&timer->dwork);} /** * libeth_xdpsq_queue_timer - run &libeth_xdpsq_timer * @timer: timer to queue * * Should be called after the queue was filled and the transmission was run * to complete the pending buffers if no further sending will be done in a * second (-> lazy cleaning won't happen). * If the timer was already run, it will be requeued back to one second * timeout again. */static inline void libeth_xdpsq_queue_timer(struct libeth_xdpsq_timer *timer){	mod_delayed_work_on(raw_smp_processor_id(), system_bh_highpri_wq,			    &timer->dwork, HZ);} /** * libeth_xdpsq_run_timer - wrapper to run a queue clean-up on a timer event * @work: workqueue belonging to the corresponding timer * @poll: driver-specific completion queue poll function * * Run the polling function on the locked queue and requeue the timer if * there's more work to do. * Designed to be used via LIBETH_XDP_DEFINE_TIMER() below. */static __always_inline voidlibeth_xdpsq_run_timer(struct work_struct *work,		       u32 (*poll)(void *xdpsq, u32 budget)){	struct libeth_xdpsq_timer *timer = container_of(work, typeof(*timer),							dwork.work); 	libeth_xdpsq_lock(timer->lock); 	if (poll(timer->xdpsq, U32_MAX))		libeth_xdpsq_queue_timer(timer); 	libeth_xdpsq_unlock(timer->lock);} /* Common Tx bits */ /** * enum - libeth_xdp internal Tx flags * @LIBETH_XDP_TX_BULK: one bulk size at which it will be flushed to the queue * @LIBETH_XDP_TX_BATCH: batch size for which the queue fill loop is unrolled * @LIBETH_XDP_TX_DROP: indicates the send function must drop frames not sent * @LIBETH_XDP_TX_NDO: whether the send function is called from .ndo_xdp_xmit() * @LIBETH_XDP_TX_XSK: whether the function is called for ``XDP_TX`` for XSk */enum {	LIBETH_XDP_TX_BULK		= DEV_MAP_BULK_SIZE,	LIBETH_XDP_TX_BATCH		= 8, 	LIBETH_XDP_TX_DROP		= BIT(0),	LIBETH_XDP_TX_NDO		= BIT(1),	LIBETH_XDP_TX_XSK		= BIT(2),}; /** * enum - &libeth_xdp_tx_frame and &libeth_xdp_tx_desc flags * @LIBETH_XDP_TX_LEN: only for ``XDP_TX``, [15:0] of ::len_fl is actual length * @LIBETH_XDP_TX_CSUM: for XSk xmit, enable checksum offload * @LIBETH_XDP_TX_XSKMD: for XSk xmit, mask of the metadata bits * @LIBETH_XDP_TX_FIRST: indicates the frag is the first one of the frame * @LIBETH_XDP_TX_LAST: whether the frag is the last one of the frame * @LIBETH_XDP_TX_MULTI: whether the frame contains several frags * @LIBETH_XDP_TX_FLAGS: only for ``XDP_TX``, [31:16] of ::len_fl is flags */enum {	LIBETH_XDP_TX_LEN		= GENMASK(15, 0), 	LIBETH_XDP_TX_CSUM		= XDP_TXMD_FLAGS_CHECKSUM,	LIBETH_XDP_TX_XSKMD		= LIBETH_XDP_TX_LEN, 	LIBETH_XDP_TX_FIRST		= BIT(16),	LIBETH_XDP_TX_LAST		= BIT(17),	LIBETH_XDP_TX_MULTI		= BIT(18), 	LIBETH_XDP_TX_FLAGS		= GENMASK(31, 16),}; /** * struct libeth_xdp_tx_frame - represents one XDP Tx element * @data: frame start pointer for ``XDP_TX`` * @len_fl: ``XDP_TX``, combined flags [31:16] and len [15:0] field for speed * @soff: ``XDP_TX``, offset from @data to the start of &skb_shared_info * @frag: one (non-head) frag for ``XDP_TX`` * @xdpf: &xdp_frame for the head frag for .ndo_xdp_xmit() * @dma: DMA address of the non-head frag for .ndo_xdp_xmit() * @xsk: ``XDP_TX`` for XSk, XDP buffer for any frag * @len: frag length for XSk ``XDP_TX`` and .ndo_xdp_xmit() * @flags: Tx flags for the above * @opts: combined @len + @flags for the above for speed * @desc: XSk xmit descriptor for direct casting */struct libeth_xdp_tx_frame {	union {		/* ``XDP_TX`` */		struct {			void				*data;			u32				len_fl;			u32				soff;		}; 		/* ``XDP_TX`` frag */		skb_frag_t			frag; 		/* .ndo_xdp_xmit(), XSk ``XDP_TX`` */		struct {			union {				struct xdp_frame		*xdpf;				dma_addr_t			dma; 				struct libeth_xdp_buff		*xsk;			};			union {				struct {					u32				len;					u32				flags;				};				aligned_u64			opts;			};		}; 		/* XSk xmit */		struct xdp_desc			desc;	};} __aligned(sizeof(struct xdp_desc));static_assert(offsetof(struct libeth_xdp_tx_frame, frag.len) ==	      offsetof(struct libeth_xdp_tx_frame, len_fl));static_assert(sizeof(struct libeth_xdp_tx_frame) == sizeof(struct xdp_desc)); /** * struct libeth_xdp_tx_bulk - XDP Tx frame bulk for bulk sending * @prog: corresponding active XDP program, %NULL for .ndo_xdp_xmit() * @dev: &net_device which the frames are transmitted on * @xdpsq: shortcut to the corresponding driver-specific XDPSQ structure * @act_mask: Rx only, mask of all the XDP prog verdicts for that NAPI session * @count: current number of frames in @bulk * @bulk: array of queued frames for bulk Tx * * All XDP Tx operations except XSk xmit queue each frame to the bulk first * and flush it when @count reaches the array end. Bulk is always placed on * the stack for performance. One bulk element contains all the data necessary * for sending a frame and then freeing it on completion. * For XSk xmit, Tx descriptor array from &xsk_buff_pool is casted directly * to &libeth_xdp_tx_frame as they are compatible and the bulk structure is * not used. */struct libeth_xdp_tx_bulk {	const struct bpf_prog		*prog;	struct net_device		*dev;	void				*xdpsq; 	u32				act_mask;	u32				count;	struct libeth_xdp_tx_frame	bulk[LIBETH_XDP_TX_BULK];} __aligned(sizeof(struct libeth_xdp_tx_frame)); /** * LIBETH_XDP_ONSTACK_BULK - declare &libeth_xdp_tx_bulk on the stack * @bq: name of the variable to declare * * Helper to declare a bulk on the stack with a compiler hint that it should * not be initialized automatically (with `CONFIG_INIT_STACK_ALL_*`) for * performance reasons. */#define LIBETH_XDP_ONSTACK_BULK(bq)					      \	struct libeth_xdp_tx_bulk bq __uninitialized /** * struct libeth_xdpsq - abstraction for an XDPSQ * @pool: XSk buffer pool for XSk ``XDP_TX`` and xmit * @sqes: array of Tx buffers from the actual queue struct * @descs: opaque pointer to the HW descriptor array * @ntu: pointer to the next free descriptor index * @count: number of descriptors on that queue * @pending: pointer to the number of sent-not-completed descs on that queue * @xdp_tx: pointer to the above, but only for non-XSk-xmit frames * @lock: corresponding XDPSQ lock * * Abstraction for driver-independent implementation of Tx. Placed on the stack * and filled by the driver before the transmission, so that the generic * functions can access and modify driver-specific resources. */struct libeth_xdpsq {	struct xsk_buff_pool		*pool;	struct libeth_sqe		*sqes;	void				*descs; 	u32				*ntu;	u32				count; 	u32				*pending;	u32				*xdp_tx;	struct libeth_xdpsq_lock	*lock;}; /** * struct libeth_xdp_tx_desc - abstraction for an XDP Tx descriptor * @addr: DMA address of the frame * @len: length of the frame * @flags: XDP Tx flags * @opts: combined @len + @flags for speed * * Filled by the generic functions and then passed to driver-specific functions * to fill a HW Tx descriptor, always placed on the [function] stack. */struct libeth_xdp_tx_desc {	dma_addr_t			addr;	union {		struct {			u32				len;			u32				flags;		};		aligned_u64			opts;	};} __aligned_largest; /** * libeth_xdp_ptr_to_priv - convert pointer to a libeth_xdp u64 priv * @ptr: pointer to convert * * The main sending function passes private data as the largest scalar, u64. * Use this helper when you want to pass a pointer there. */#define libeth_xdp_ptr_to_priv(ptr) ({					      \	typecheck_pointer(ptr);						      \	((u64)(uintptr_t)(ptr));					      \})/** * libeth_xdp_priv_to_ptr - convert libeth_xdp u64 priv to a pointer * @priv: private data to convert * * The main sending function passes private data as the largest scalar, u64. * Use this helper when your callback takes this u64 and you want to convert * it back to a pointer. */#define libeth_xdp_priv_to_ptr(priv) ({					      \	static_assert(__same_type(priv, u64));				      \	((const void *)(uintptr_t)(priv));				      \}) /* * On 64-bit systems, assigning one u64 is faster than two u32s. When ::len * occupies lowest 32 bits (LE), whole ::opts can be assigned directly instead. */#ifdef __LITTLE_ENDIAN#define __LIBETH_WORD_ACCESS		1#endif#ifdef __LIBETH_WORD_ACCESS#define __libeth_xdp_tx_len(flen, ...)					      \	.opts = ((flen) | FIELD_PREP(GENMASK_ULL(63, 32), (__VA_ARGS__ + 0)))#else#define __libeth_xdp_tx_len(flen, ...)					      \	.len = (flen), .flags = (__VA_ARGS__ + 0)#endif /** * libeth_xdp_tx_xmit_bulk - main XDP Tx function * @bulk: array of frames to send * @xdpsq: pointer to the driver-specific XDPSQ struct * @n: number of frames to send * @unroll: whether to unroll the queue filling loop for speed * @priv: driver-specific private data * @prep: callback for cleaning the queue and filling abstract &libeth_xdpsq

Lines 1–500 of 1870. Load more