lael-labs/linux

Documentation/bpf/index.rst+1−0

@@ -42,6 +42,7 @@ Program types · 41 unmodified lines
42	.. toctree::	42	.. toctree::
43	:maxdepth: 1	43	:maxdepth: 1
44		44
		45	prog_cgroup_sockopt
45	prog_cgroup_sysctl	46	prog_cgroup_sysctl
46	prog_flow_dissector	47	prog_flow_dissector
47		48

Documentation/bpf/prog_cgroup_sockopt.rst+93−0

@@ -0,0 +1,93 @@
	1	.. SPDX-License-Identifier: GPL-2.0
	2
	3	============================
	4	BPF_PROG_TYPE_CGROUP_SOCKOPT
	5	============================
	6
	7	``BPF_PROG_TYPE_CGROUP_SOCKOPT`` program type can be attached to two
	8	cgroup hooks:
	9
	10	* ``BPF_CGROUP_GETSOCKOPT`` - called every time process executes ``getsockopt``
	11	system call.
	12	* ``BPF_CGROUP_SETSOCKOPT`` - called every time process executes ``setsockopt``
	13	system call.
	14
	15	The context (``struct bpf_sockopt``) has associated socket (``sk``) and
	16	all input arguments: ``level``, ``optname``, ``optval`` and ``optlen``.
	17
	18	BPF_CGROUP_SETSOCKOPT
	19	=====================
	20
	21	``BPF_CGROUP_SETSOCKOPT`` is triggered before the kernel handling of
	22	sockopt and it has writable context: it can modify the supplied arguments
	23	before passing them down to the kernel. This hook has access to the cgroup
	24	and socket local storage.
	25
	26	If BPF program sets ``optlen`` to -1, the control will be returned
	27	back to the userspace after all other BPF programs in the cgroup
	28	chain finish (i.e. kernel ``setsockopt`` handling will not be executed).
	29
	30	Note, that ``optlen`` can not be increased beyond the user-supplied
	31	value. It can only be decreased or set to -1. Any other value will
	32	trigger ``EFAULT``.
	33
	34	Return Type
	35	-----------
	36
	37	* ``0`` - reject the syscall, ``EPERM`` will be returned to the userspace.
	38	* ``1`` - success, continue with next BPF program in the cgroup chain.
	39
	40	BPF_CGROUP_GETSOCKOPT
	41	=====================
	42
	43	``BPF_CGROUP_GETSOCKOPT`` is triggered after the kernel handing of
	44	sockopt. The BPF hook can observe ``optval``, ``optlen`` and ``retval``
	45	if it's interested in whatever kernel has returned. BPF hook can override
	46	the values above, adjust ``optlen`` and reset ``retval`` to 0. If ``optlen``
	47	has been increased above initial ``getsockopt`` value (i.e. userspace
	48	buffer is too small), ``EFAULT`` is returned.
	49
	50	This hook has access to the cgroup and socket local storage.
	51
	52	Note, that the only acceptable value to set to ``retval`` is 0 and the
	53	original value that the kernel returned. Any other value will trigger
	54	``EFAULT``.
	55
	56	Return Type
	57	-----------
	58
	59	* ``0`` - reject the syscall, ``EPERM`` will be returned to the userspace.
	60	* ``1`` - success: copy ``optval`` and ``optlen`` to userspace, return
	61	``retval`` from the syscall (note that this can be overwritten by
	62	the BPF program from the parent cgroup).
	63
	64	Cgroup Inheritance
	65	==================
	66
	67	Suppose, there is the following cgroup hierarchy where each cgroup
	68	has ``BPF_CGROUP_GETSOCKOPT`` attached at each level with
	69	``BPF_F_ALLOW_MULTI`` flag::
	70
	71	A (root, parent)
	72	\
	73	B (child)
	74
	75	When the application calls ``getsockopt`` syscall from the cgroup B,
	76	the programs are executed from the bottom up: B, A. First program
	77	(B) sees the result of kernel's ``getsockopt``. It can optionally
	78	adjust ``optval``, ``optlen`` and reset ``retval`` to 0. After that
	79	control will be passed to the second (A) program which will see the
	80	same context as B including any potential modifications.
	81
	82	Same for ``BPF_CGROUP_SETSOCKOPT``: if the program is attached to
	83	A and B, the trigger order is B, then A. If B does any changes
	84	to the input arguments (``level``, ``optname``, ``optval``, ``optlen``),
	85	then the next program in the chain (A) will see those changes,
	86	not the original input ``setsockopt`` arguments. The potentially
	87	modified values will be then passed down to the kernel.
	88
	89	Example
	90	=======
	91
	92	See ``tools/testing/selftests/bpf/progs/sockopt_sk.c`` for an example
	93	of BPF program that handles socket options.

Documentation/networking/af_xdp.rst+15−1

@@ -220,7 +220,21 @@ Usage · 219 unmodified lines
220	In order to use AF_XDP sockets there are two parts needed. The	220	In order to use AF_XDP sockets there are two parts needed. The
221	user-space application and the XDP program. For a complete setup and	221	user-space application and the XDP program. For a complete setup and
222	usage example, please refer to the sample application. The user-space	222	usage example, please refer to the sample application. The user-space
223	side is xdpsock_user.c and the XDP side xdpsock_kern.c.	223	side is xdpsock_user.c and the XDP side is part of libbpf.
		224
		225	The XDP code sample included in tools/lib/bpf/xsk.c is the following::
		226
		227	SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)
		228	{
		229	int index = ctx->rx_queue_index;
		230
		231	// A set entry here means that the correspnding queue_id
		232	// has an active AF_XDP socket bound to it.
		233	if (bpf_map_lookup_elem(&xsks_map, &index))
		234	return bpf_redirect_map(&xsks_map, index, 0);
		235
		236	return XDP_PASS;
		237	}
224		238
225	Naive ring dequeue and enqueue could look like this::	239	Naive ring dequeue and enqueue could look like this::
226		240

drivers/net/ethernet/intel/i40e/i40e_xsk.c+7−5

 	struct i40e_tx_desc *tx_desc = NULL;
 	struct i40e_tx_buffer *tx_bi;
 	bool work_done = true;
+	struct xdp_desc desc;
 	dma_addr_t dma;
-	u32 len;
 	while (budget-- > 0) {
 		if (!unlikely(I40E_DESC_UNUSED(xdp_ring))) {
 			break;
 		}
-		if (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &dma, &len))
+		if (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &desc))
 			break;
-		dma_sync_single_for_device(xdp_ring->dev, dma, len,
+		dma = xdp_umem_get_dma(xdp_ring->xsk_umem, desc.addr);
+		dma_sync_single_for_device(xdp_ring->dev, dma, desc.len,
 					   DMA_BIDIRECTIONAL);
 		tx_bi = &xdp_ring->tx_bi[xdp_ring->next_to_use];
-		tx_bi->bytecount = len;
+		tx_bi->bytecount = desc.len;
 		tx_desc = I40E_TX_DESC(xdp_ring, xdp_ring->next_to_use);
 		tx_desc->buffer_addr = cpu_to_le64(dma);
 		tx_desc->cmd_type_offset_bsz =
 			build_ctob(I40E_TX_DESC_CMD_ICRC
 				   | I40E_TX_DESC_CMD_EOP,
-				   0, len, 0);
+				   0, desc.len, 0);
 		xdp_ring->next_to_use++;
 		if (xdp_ring->next_to_use == xdp_ring->count)

drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c+9−6

 	union ixgbe_adv_tx_desc *tx_desc = NULL;
 	struct ixgbe_tx_buffer *tx_bi;
 	bool work_done = true;
-	u32 len, cmd_type;
+	struct xdp_desc desc;
 	dma_addr_t dma;
+	u32 cmd_type;
 	while (budget-- > 0) {
 		if (unlikely(!ixgbe_desc_unused(xdp_ring)) ||
 			break;
 		}
-		if (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &dma, &len))
+		if (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &desc))
 			break;
-		dma_sync_single_for_device(xdp_ring->dev, dma, len,
+		dma = xdp_umem_get_dma(xdp_ring->xsk_umem, desc.addr);
+		dma_sync_single_for_device(xdp_ring->dev, dma, desc.len,
 					   DMA_BIDIRECTIONAL);
 		tx_bi = &xdp_ring->tx_buffer_info[xdp_ring->next_to_use];
-		tx_bi->bytecount = len;
+		tx_bi->bytecount = desc.len;
 		tx_bi->xdpf = NULL;
 		tx_bi->gso_segs = 1;
 		cmd_type = IXGBE_ADVTXD_DTYP_DATA |
 			   IXGBE_ADVTXD_DCMD_DEXT |
 			   IXGBE_ADVTXD_DCMD_IFCS;
-		cmd_type |= len | IXGBE_TXD_CMD;
+		cmd_type |= desc.len | IXGBE_TXD_CMD;
 		tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
 		tx_desc->read.olinfo_status =
-			cpu_to_le32(len << IXGBE_ADVTXD_PAYLEN_SHIFT);
+			cpu_to_le32(desc.len << IXGBE_ADVTXD_PAYLEN_SHIFT);
 		xdp_ring->next_to_use++;
 		if (xdp_ring->next_to_use == xdp_ring->count)

drivers/net/ethernet/mellanox/mlx5/core/Makefile+1−1

@@ -24,7 +24,7 @@ mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \ · 23 unmodified lines
24	mlx5_core-$(CONFIG_MLX5_CORE_EN) += en_main.o en_common.o en_fs.o en_ethtool.o \	24	mlx5_core-$(CONFIG_MLX5_CORE_EN) += en_main.o en_common.o en_fs.o en_ethtool.o \
25	en_tx.o en_rx.o en_dim.o en_txrx.o en/xdp.o en_stats.o \	25	en_tx.o en_rx.o en_dim.o en_txrx.o en/xdp.o en_stats.o \
26	en_selftest.o en/port.o en/monitor_stats.o en/reporter_tx.o \	26	en_selftest.o en/port.o en/monitor_stats.o en/reporter_tx.o \
27	en/params.o	27	en/params.o en/xsk/umem.o en/xsk/setup.o en/xsk/rx.o en/xsk/tx.o
28		28
29	#	29	#
30	# Netdev extra	30	# Netdev extra

drivers/net/ethernet/mellanox/mlx5/core/en.h+140−15

 #define MLX5E_MAX_NUM_CHANNELS         (MLX5E_INDIR_RQT_SIZE >> 1)
 #define MLX5E_MAX_NUM_SQS              (MLX5E_MAX_NUM_CHANNELS * MLX5E_MAX_NUM_TC)
 #define MLX5E_TX_CQ_POLL_BUDGET        128
+#define MLX5E_TX_XSK_POLL_BUDGET       64
 #define MLX5E_SQ_RECOVER_MIN_INTERVAL  500 /* msecs */
 #define MLX5E_UMR_WQE_INLINE_SZ \
 			    ##__VA_ARGS__);                     \
 } while (0)
+enum mlx5e_rq_group {
+	MLX5E_RQ_GROUP_REGULAR,
+	MLX5E_RQ_GROUP_XSK,
+	MLX5E_NUM_RQ_GROUPS /* Keep last. */
+};
 static inline u16 mlx5_min_rx_wqes(int wq_type, u32 wq_size)
 {
 /* Use this function to get max num channels after netdev was created */
 static inline int mlx5e_get_netdev_max_channels(struct net_device *netdev)
 {
-	return min_t(unsigned int, netdev->num_rx_queues,
+	return min_t(unsigned int,
+		     netdev->num_rx_queues / MLX5E_NUM_RQ_GROUPS,
 		     netdev->num_tx_queues);
 }
 	u32 lro_timeout;
 	u32 pflags;
 	struct bpf_prog *xdp_prog;
+	struct mlx5e_xsk *xsk;
 	unsigned int sw_mtu;
 	int hard_mtu;
 };
 struct mlx5e_sq_wqe_info {
 	u8  opcode;
+	/* Auxiliary data for different opcodes. */
+	union {
+		struct {
+			struct mlx5e_rq *rq;
+		} umr;
+	};
 };
 struct mlx5e_txqsq {
 } ____cacheline_aligned_in_smp;
 struct mlx5e_dma_info {
-	struct page     *page;
+	dma_addr_t addr;
-	dma_addr_t      addr;
+	union {
+		struct page *page;
+		struct {
+			u64 handle;
+			void *data;
+		} xsk;
+	};
+};
+/* XDP packets can be transmitted in different ways. On completion, we need to
+ * distinguish between them to clean up things in a proper way.
+ */
+enum mlx5e_xdp_xmit_mode {
+	/* An xdp_frame was transmitted due to either XDP_REDIRECT from another
+	 * device or XDP_TX from an XSK RQ. The frame has to be unmapped and
+	 * returned.
+	 */
+	MLX5E_XDP_XMIT_MODE_FRAME,
+	/* The xdp_frame was created in place as a result of XDP_TX from a
+	 * regular RQ. No DMA remapping happened, and the page belongs to us.
+	 */
+	MLX5E_XDP_XMIT_MODE_PAGE,
+	/* No xdp_frame was created at all, the transmit happened from a UMEM
+	 * page. The UMEM Completion Ring producer pointer has to be increased.
+	 */
+	MLX5E_XDP_XMIT_MODE_XSK,
 };
 struct mlx5e_xdp_info {
-	struct xdp_frame      *xdpf;
+	enum mlx5e_xdp_xmit_mode mode;
-	dma_addr_t            dma_addr;
+	union {
-	struct mlx5e_dma_info di;
+		struct {
+			struct xdp_frame *xdpf;
+			dma_addr_t dma_addr;
+		} frame;
+		struct {
+			struct mlx5e_rq *rq;
+			struct mlx5e_dma_info di;
+		} page;
+	};
+};
+struct mlx5e_xdp_xmit_data {
+	dma_addr_t  dma_addr;
+	void       *data;
+	u32         len;
 };
 struct mlx5e_xdp_info_fifo {
 };
 struct mlx5e_xdpsq;
-typedef bool (*mlx5e_fp_xmit_xdp_frame)(struct mlx5e_xdpsq*,
+typedef int (*mlx5e_fp_xmit_xdp_frame_check)(struct mlx5e_xdpsq *);
-					struct mlx5e_xdp_info*);
+typedef bool (*mlx5e_fp_xmit_xdp_frame)(struct mlx5e_xdpsq *,
+					struct mlx5e_xdp_xmit_data *,
+					struct mlx5e_xdp_info *,
+					int);
 struct mlx5e_xdpsq {
 	/* data path */
 	struct mlx5e_cq            cq;
 	/* read only */
+	struct xdp_umem           *umem;
 	struct mlx5_wq_cyc         wq;
 	struct mlx5e_xdpsq_stats  *stats;
+	mlx5e_fp_xmit_xdp_frame_check xmit_xdp_frame_check;
 	mlx5e_fp_xmit_xdp_frame    xmit_xdp_frame;
 	struct {
 		struct mlx5e_xdp_wqe_info *wqe_info;
 			u8                     log_stride_sz;
 			u8                     umr_in_progress;
 			u8                     umr_last_bulk;
+			u8                     umr_completed;
 		} mpwqe;
 	};
 	struct {
+		u16            umem_headroom;
 		u16            headroom;
 		u8             map_dir;   /* dma map direction */
 	} buff;
 	/* XDP */
 	struct bpf_prog       *xdp_prog;
-	struct mlx5e_xdpsq     xdpsq;
+	struct mlx5e_xdpsq    *xdpsq;
 	DECLARE_BITMAP(flags, 8);
 	struct page_pool      *page_pool;
+	/* AF_XDP zero-copy */
+	struct zero_copy_allocator zca;
+	struct xdp_umem       *umem;
 	/* control */
 	struct mlx5_wq_ctrl    wq_ctrl;
 	__be32                 mkey_be;
 	struct xdp_rxq_info    xdp_rxq;
 } ____cacheline_aligned_in_smp;
+enum mlx5e_channel_state {
+	MLX5E_CHANNEL_STATE_XSK,
+	MLX5E_CHANNEL_NUM_STATES
+};
 struct mlx5e_channel {
 	/* data path */
 	struct mlx5e_rq            rq;
+	struct mlx5e_xdpsq         rq_xdpsq;
 	struct mlx5e_txqsq         sq[MLX5E_MAX_NUM_TC];
 	struct mlx5e_icosq         icosq;   /* internal control operations */
 	bool                       xdp;
 	/* XDP_REDIRECT */
 	struct mlx5e_xdpsq         xdpsq;
+	/* AF_XDP zero-copy */
+	struct mlx5e_rq            xskrq;
+	struct mlx5e_xdpsq         xsksq;
+	struct mlx5e_icosq         xskicosq;
+	/* xskicosq can be accessed from any CPU - the spinlock protects it. */
+	spinlock_t                 xskicosq_lock;
 	/* data path - accessed per napi poll */
 	struct irq_desc *irq_desc;
 	struct mlx5e_ch_stats     *stats;
 	struct mlx5e_priv         *priv;
 	struct mlx5_core_dev      *mdev;
 	struct hwtstamp_config    *tstamp;
+	DECLARE_BITMAP(state, MLX5E_CHANNEL_NUM_STATES);
 	int                        ix;
 	int                        cpu;
 	cpumask_var_t              xps_cpumask;
 	struct mlx5e_ch_stats ch;
 	struct mlx5e_sq_stats sq[MLX5E_MAX_NUM_TC];
 	struct mlx5e_rq_stats rq;
+	struct mlx5e_rq_stats xskrq;
 	struct mlx5e_xdpsq_stats rq_xdpsq;
 	struct mlx5e_xdpsq_stats xdpsq;
+	struct mlx5e_xdpsq_stats xsksq;
 } ____cacheline_aligned_in_smp;
 enum {
 	MLX5E_STATE_OPENED,
 	MLX5E_STATE_DESTROYING,
 	MLX5E_STATE_XDP_TX_ENABLED,
+	MLX5E_STATE_XDP_OPEN,
 };
 struct mlx5e_rqt {
 	int rl_index;
 };
+struct mlx5e_xsk {
+	/* UMEMs are stored separately from channels, because we don't want to
+	 * lose them when channels are recreated. The kernel also stores UMEMs,
+	 * but it doesn't distinguish between zero-copy and non-zero-copy UMEMs,
+	 * so rely on our mechanism.
+	 */
+	struct xdp_umem **umems;
+	u16 refcnt;
+	bool ever_used;
+};
 struct mlx5e_priv {
 	/* priv data path fields - start */
 	struct mlx5e_txqsq *txq2sq[MLX5E_MAX_NUM_CHANNELS * MLX5E_MAX_NUM_TC];
 	struct mlx5e_tir           indir_tir[MLX5E_NUM_INDIR_TIRS];
 	struct mlx5e_tir           inner_indir_tir[MLX5E_NUM_INDIR_TIRS];
 	struct mlx5e_tir           direct_tir[MLX5E_MAX_NUM_CHANNELS];
+	struct mlx5e_tir           xsk_tir[MLX5E_MAX_NUM_CHANNELS];
 	struct mlx5e_rss_params    rss_params;
 	u32                        tx_rates[MLX5E_MAX_NUM_SQS];
 	struct mlx5e_tls          *tls;
 #endif
 	struct devlink_health_reporter *tx_reporter;
+	struct mlx5e_xsk           xsk;
 };
 struct mlx5e_profile {
 				struct mlx5e_params *params);
 void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info);
-void mlx5e_page_release(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info,
+void mlx5e_page_release_dynamic(struct mlx5e_rq *rq,
-			bool recycle);
+				struct mlx5e_dma_info *dma_info,
+				bool recycle);
 void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
 void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
 bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq);
+void mlx5e_poll_ico_cq(struct mlx5e_cq *cq);
 bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq);
 void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix);
 void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix);
 void mlx5e_modify_tirs_hash(struct mlx5e_priv *priv, void *in, int inlen);
 struct mlx5e_tirc_config mlx5e_tirc_get_default_config(enum mlx5e_traffic_types tt);
+struct mlx5e_xsk_param;
+struct mlx5e_rq_param;
+int mlx5e_open_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
+		  struct mlx5e_rq_param *param, struct mlx5e_xsk_param *xsk,
+		  struct xdp_umem *umem, struct mlx5e_rq *rq);
+int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time);
+void mlx5e_deactivate_rq(struct mlx5e_rq *rq);
+void mlx5e_close_rq(struct mlx5e_rq *rq);
+struct mlx5e_sq_param;
+int mlx5e_open_icosq(struct mlx5e_channel *c, struct mlx5e_params *params,
+		     struct mlx5e_sq_param *param, struct mlx5e_icosq *sq);
+void mlx5e_close_icosq(struct mlx5e_icosq *sq);
+int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params,
+		     struct mlx5e_sq_param *param, struct xdp_umem *umem,
+		     struct mlx5e_xdpsq *sq, bool is_redirect);
+void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq);
+struct mlx5e_cq_param;
+int mlx5e_open_cq(struct mlx5e_channel *c, struct dim_cq_moder moder,
+		  struct mlx5e_cq_param *param, struct mlx5e_cq *cq);
+void mlx5e_close_cq(struct mlx5e_cq *cq);
 int mlx5e_open_locked(struct net_device *netdev);
 int mlx5e_close_locked(struct net_device *netdev);
 int mlx5e_create_indirect_tirs(struct mlx5e_priv *priv, bool inner_ttc);
 void mlx5e_destroy_indirect_tirs(struct mlx5e_priv *priv, bool inner_ttc);
-int mlx5e_create_direct_rqts(struct mlx5e_priv *priv);
+int mlx5e_create_direct_rqts(struct mlx5e_priv *priv, struct mlx5e_tir *tirs);
-void mlx5e_destroy_direct_rqts(struct mlx5e_priv *priv);
+void mlx5e_destroy_direct_rqts(struct mlx5e_priv *priv, struct mlx5e_tir *tirs);
-int mlx5e_create_direct_tirs(struct mlx5e_priv *priv);
+int mlx5e_create_direct_tirs(struct mlx5e_priv *priv, struct mlx5e_tir *tirs);
-void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv);
+void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv, struct mlx5e_tir *tirs);
 void mlx5e_destroy_rqt(struct mlx5e_priv *priv, struct mlx5e_rqt *rqt);
 int mlx5e_create_tis(struct mlx5_core_dev *mdev, int tc,
 void mlx5e_destroy_netdev(struct mlx5e_priv *priv);
 void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv);
 void mlx5e_build_nic_params(struct mlx5_core_dev *mdev,
+			    struct mlx5e_xsk *xsk,
 			    struct mlx5e_rss_params *rss_params,
 			    struct mlx5e_params *params,
 			    u16 max_channels, u16 mtu);

drivers/net/ethernet/mellanox/mlx5/core/en/params.c+71−37

@@ -3,65 +3,102 @@ · 2 unmodified lines
3		3
4	#include "en/params.h"	4	#include "en/params.h"
5		5
6	u32 mlx5e_rx_get_linear_frag_sz(struct mlx5e_params *params)	6	static inline bool mlx5e_rx_is_xdp(struct mlx5e_params *params,
		7	struct mlx5e_xsk_param *xsk)
7	{	8	{
8	u16 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);	9	return params->xdp_prog \|\| xsk;
9	u16 linear_rq_headroom = params->xdp_prog ?	10	}
10	XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM;	11
11	u32 frag_sz;	12	u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params,
		13	struct mlx5e_xsk_param *xsk)
		14	{
		15	u16 headroom = NET_IP_ALIGN;
		16
		17	if (mlx5e_rx_is_xdp(params, xsk)) {
		18	headroom += XDP_PACKET_HEADROOM;
		19	if (xsk)
		20	headroom += xsk->headroom;
		21	} else {
		22	headroom += MLX5_RX_HEADROOM;
		23	}
		24
		25	return headroom;
		26	}
		27
		28	u32 mlx5e_rx_get_linear_frag_sz(struct mlx5e_params *params,
		29	struct mlx5e_xsk_param *xsk)
		30	{
		31	u32 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
		32	u16 linear_rq_headroom = mlx5e_get_linear_rq_headroom(params, xsk);
		33	u32 frag_sz = linear_rq_headroom + hw_mtu;
12		34
13	linear_rq_headroom += NET_IP_ALIGN;	35	/* AF_XDP doesn't build SKBs in place. */
		36	if (!xsk)
		37	frag_sz = MLX5_SKB_FRAG_SZ(frag_sz);
14		38
15	frag_sz = MLX5_SKB_FRAG_SZ(linear_rq_headroom + hw_mtu);	39	/* XDP in mlx5e doesn't support multiple packets per page. */
		40	if (mlx5e_rx_is_xdp(params, xsk))
		41	frag_sz = max_t(u32, frag_sz, PAGE_SIZE);
16		42
17	if (params->xdp_prog && frag_sz < PAGE_SIZE)	43	/* Even if we can go with a smaller fragment size, we must not put
18	frag_sz = PAGE_SIZE;	44	* multiple packets into a single frame.
		45	*/
		46	if (xsk)
		47	frag_sz = max_t(u32, frag_sz, xsk->chunk_size);
19		48
20	return frag_sz;	49	return frag_sz;
21	}	50	}
22		51
23	u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params)	52	u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params,
		53	struct mlx5e_xsk_param *xsk)
24	{	54	{
25	u32 linear_frag_sz = mlx5e_rx_get_linear_frag_sz(params);	55	u32 linear_frag_sz = mlx5e_rx_get_linear_frag_sz(params, xsk);
26		56
27	return MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(linear_frag_sz);	57	return MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(linear_frag_sz);
28	}	58	}
29		59
30	bool mlx5e_rx_is_linear_skb(struct mlx5e_params *params)	60	bool mlx5e_rx_is_linear_skb(struct mlx5e_params *params,
		61	struct mlx5e_xsk_param *xsk)
31	{	62	{
32	u32 frag_sz = mlx5e_rx_get_linear_frag_sz(params);	63	/* AF_XDP allocates SKBs on XDP_PASS - ensure they don't occupy more
		64	* than one page. For this, check both with and without xsk.
		65	*/
		66	u32 linear_frag_sz = max(mlx5e_rx_get_linear_frag_sz(params, xsk),
		67	mlx5e_rx_get_linear_frag_sz(params, NULL));
33		68
34	return !params->lro_en && frag_sz <= PAGE_SIZE;	69	return !params->lro_en && linear_frag_sz <= PAGE_SIZE;
35	}	70	}
36		71
37	#define MLX5_MAX_MPWQE_LOG_WQE_STRIDE_SZ ((BIT(__mlx5_bit_sz(wq, log_wqe_stride_size)) - 1) + \	72	#define MLX5_MAX_MPWQE_LOG_WQE_STRIDE_SZ ((BIT(__mlx5_bit_sz(wq, log_wqe_stride_size)) - 1) + \
38	MLX5_MPWQE_LOG_STRIDE_SZ_BASE)	73	MLX5_MPWQE_LOG_STRIDE_SZ_BASE)
39	bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,	74	bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
40	struct mlx5e_params *params)	75	struct mlx5e_params *params,
		76	struct mlx5e_xsk_param *xsk)
41	{	77	{
42	u32 frag_sz = mlx5e_rx_get_linear_frag_sz(params);	78	u32 linear_frag_sz = mlx5e_rx_get_linear_frag_sz(params, xsk);
43	s8 signed_log_num_strides_param;	79	s8 signed_log_num_strides_param;
44	u8 log_num_strides;	80	u8 log_num_strides;
45		81
46	if (!mlx5e_rx_is_linear_skb(params))	82	if (!mlx5e_rx_is_linear_skb(params, xsk))
47	return false;	83	return false;
48		84
49	if (order_base_2(frag_sz) > MLX5_MAX_MPWQE_LOG_WQE_STRIDE_SZ)	85	if (order_base_2(linear_frag_sz) > MLX5_MAX_MPWQE_LOG_WQE_STRIDE_SZ)
50	return false;	86	return false;
51		87
52	if (MLX5_CAP_GEN(mdev, ext_stride_num_range))	88	if (MLX5_CAP_GEN(mdev, ext_stride_num_range))
53	return true;	89	return true;
54		90
55	log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(frag_sz);	91	log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(linear_frag_sz);
56	signed_log_num_strides_param =	92	signed_log_num_strides_param =
57	(s8)log_num_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE;	93	(s8)log_num_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE;
58		94
59	return signed_log_num_strides_param >= 0;	95	return signed_log_num_strides_param >= 0;
60	}	96	}
61		97
62	u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params)	98	u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params,
		99	struct mlx5e_xsk_param *xsk)
63	{	100	{
64	u8 log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(params);	101	u8 log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(params, xsk);
65		102
66	/* Numbers are unsigned, don't subtract to avoid underflow. */	103	/* Numbers are unsigned, don't subtract to avoid underflow. */
67	if (params->log_rq_mtu_frames <	104	if (params->log_rq_mtu_frames <
@@ -72,33 +109,30 @@ u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params) · 4 unmodified lines
72	}	109	}
73		110
74	u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,	111	u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,
75	struct mlx5e_params *params)	112	struct mlx5e_params *params,
		113	struct mlx5e_xsk_param *xsk)
76	{	114	{
77	if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params))	115	if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk))
78	return order_base_2(mlx5e_rx_get_linear_frag_sz(params));	116	return order_base_2(mlx5e_rx_get_linear_frag_sz(params, xsk));
79		117
80	return MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev);	118	return MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev);
81	}	119	}
82		120
83	u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,	121	u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,
84	struct mlx5e_params *params)	122	struct mlx5e_params *params,
		123	struct mlx5e_xsk_param *xsk)
85	{	124	{
86	return MLX5_MPWRQ_LOG_WQE_SZ -	125	return MLX5_MPWRQ_LOG_WQE_SZ -
87	mlx5e_mpwqe_get_log_stride_size(mdev, params);	126	mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
88	}	127	}
89		128
90	u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,	129	u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,
91	struct mlx5e_params *params)	130	struct mlx5e_params *params,
		131	struct mlx5e_xsk_param *xsk)
92	{	132	{
93	u16 linear_rq_headroom = params->xdp_prog ?	133	bool is_linear_skb = (params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC) ?
94	XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM;	134	mlx5e_rx_is_linear_skb(params, xsk) :
95	bool is_linear_skb;	135	mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk);
96
97	linear_rq_headroom += NET_IP_ALIGN;
98
99	is_linear_skb = (params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC) ?
100	mlx5e_rx_is_linear_skb(params) :
101	mlx5e_rx_mpwqe_is_linear_skb(mdev, params);
102		136
103	return is_linear_skb ? linear_rq_headroom : 0;	137	return is_linear_skb ? mlx5e_get_linear_rq_headroom(params, xsk) : 0;
104	}	138	}

drivers/net/ethernet/mellanox/mlx5/core/en/params.h+110−8

@@ -6,17 +6,119 @@ · 5 unmodified lines
6		6
7	#include "en.h"	7	#include "en.h"
8		8
9	u32 mlx5e_rx_get_linear_frag_sz(struct mlx5e_params *params);	9	struct mlx5e_xsk_param {
10	u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params);	10	u16 headroom;
11	bool mlx5e_rx_is_linear_skb(struct mlx5e_params *params);	11	u16 chunk_size;
		12	};
		13
		14	struct mlx5e_rq_param {
		15	u32 rqc[MLX5_ST_SZ_DW(rqc)];
		16	struct mlx5_wq_param wq;
		17	struct mlx5e_rq_frags_info frags_info;
		18	};
		19
		20	struct mlx5e_sq_param {
		21	u32 sqc[MLX5_ST_SZ_DW(sqc)];
		22	struct mlx5_wq_param wq;
		23	bool is_mpw;
		24	};
		25
		26	struct mlx5e_cq_param {
		27	u32 cqc[MLX5_ST_SZ_DW(cqc)];
		28	struct mlx5_wq_param wq;
		29	u16 eq_ix;
		30	u8 cq_period_mode;
		31	};
		32
		33	struct mlx5e_channel_param {
		34	struct mlx5e_rq_param rq;
		35	struct mlx5e_sq_param sq;
		36	struct mlx5e_sq_param xdp_sq;
		37	struct mlx5e_sq_param icosq;
		38	struct mlx5e_cq_param rx_cq;
		39	struct mlx5e_cq_param tx_cq;
		40	struct mlx5e_cq_param icosq_cq;
		41	};
		42
		43	static inline bool mlx5e_qid_get_ch_if_in_group(struct mlx5e_params *params,
		44	u16 qid,
		45	enum mlx5e_rq_group group,
		46	u16 *ix)
		47	{
		48	int nch = params->num_channels;
		49	int ch = qid - nch * group;
		50
		51	if (ch < 0 \|\| ch >= nch)
		52	return false;
		53
		54	*ix = ch;
		55	return true;
		56	}
		57
		58	static inline void mlx5e_qid_get_ch_and_group(struct mlx5e_params *params,
		59	u16 qid,
		60	u16 *ix,
		61	enum mlx5e_rq_group *group)
		62	{
		63	u16 nch = params->num_channels;
		64
		65	*ix = qid % nch;
		66	*group = qid / nch;
		67	}
		68
		69	static inline bool mlx5e_qid_validate(struct mlx5e_params *params, u64 qid)
		70	{
		71	return qid < params->num_channels * MLX5E_NUM_RQ_GROUPS;
		72	}
		73
		74	/* Parameter calculations */
		75
		76	u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params,
		77	struct mlx5e_xsk_param *xsk);
		78	u32 mlx5e_rx_get_linear_frag_sz(struct mlx5e_params *params,
		79	struct mlx5e_xsk_param *xsk);
		80	u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params,
		81	struct mlx5e_xsk_param *xsk);
		82	bool mlx5e_rx_is_linear_skb(struct mlx5e_params *params,
		83	struct mlx5e_xsk_param *xsk);
12	bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,	84	bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
13	struct mlx5e_params *params);	85	struct mlx5e_params *params,
14	u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params);	86	struct mlx5e_xsk_param *xsk);
		87	u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params,
		88	struct mlx5e_xsk_param *xsk);
15	u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,	89	u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,
16	struct mlx5e_params *params);	90	struct mlx5e_params *params,
		91	struct mlx5e_xsk_param *xsk);
17	u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,	92	u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,
18	struct mlx5e_params *params);	93	struct mlx5e_params *params,
		94	struct mlx5e_xsk_param *xsk);
19	u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,	95	u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,
20	struct mlx5e_params *params);	96	struct mlx5e_params *params,
		97	struct mlx5e_xsk_param *xsk);
		98
		99	/* Build queue parameters */
		100
		101	void mlx5e_build_rq_param(struct mlx5e_priv *priv,
		102	struct mlx5e_params *params,
		103	struct mlx5e_xsk_param *xsk,
		104	struct mlx5e_rq_param *param);
		105	void mlx5e_build_sq_param_common(struct mlx5e_priv *priv,
		106	struct mlx5e_sq_param *param);
		107	void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
		108	struct mlx5e_params *params,
		109	struct mlx5e_xsk_param *xsk,
		110	struct mlx5e_cq_param *param);
		111	void mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
		112	struct mlx5e_params *params,
		113	struct mlx5e_cq_param *param);
		114	void mlx5e_build_ico_cq_param(struct mlx5e_priv *priv,
		115	u8 log_wq_size,
		116	struct mlx5e_cq_param *param);
		117	void mlx5e_build_icosq_param(struct mlx5e_priv *priv,
		118	u8 log_wq_size,
		119	struct mlx5e_sq_param *param);
		120	void mlx5e_build_xdpsq_param(struct mlx5e_priv *priv,
		121	struct mlx5e_params *params,
		122	struct mlx5e_sq_param *param);
21		123
22	#endif /* __MLX5_EN_PARAMS_H__ */	124	#endif /* __MLX5_EN_PARAMS_H__ */

drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c+172−59

  */
 #include <linux/bpf_trace.h>
+#include <net/xdp_sock.h>
 #include "en/xdp.h"
+#include "en/params.h"
-int mlx5e_xdp_max_mtu(struct mlx5e_params *params)
+int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk)
 {
-	int hr = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
+	int hr = mlx5e_get_linear_rq_headroom(params, xsk);
 	/* Let S := SKB_DATA_ALIGN(sizeof(struct skb_shared_info)).
 	 * The condition checked in mlx5e_rx_is_linear_skb is:
 }
 static inline bool
-mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_dma_info *di,
+mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq,
-		    struct xdp_buff *xdp)
+		    struct mlx5e_dma_info *di, struct xdp_buff *xdp)
 {
+	struct mlx5e_xdp_xmit_data xdptxd;
 	struct mlx5e_xdp_info xdpi;
+	struct xdp_frame *xdpf;
+	dma_addr_t dma_addr;
-	xdpi.xdpf = convert_to_xdp_frame(xdp);
+	xdpf = convert_to_xdp_frame(xdp);
-	if (unlikely(!xdpi.xdpf))
+	if (unlikely(!xdpf))
 		return false;
-	xdpi.dma_addr = di->addr + (xdpi.xdpf->data - (void *)xdpi.xdpf);
-	dma_sync_single_for_device(sq->pdev, xdpi.dma_addr,
-				   xdpi.xdpf->len, PCI_DMA_TODEVICE);
-	xdpi.di = *di;
-	return sq->xmit_xdp_frame(sq, &xdpi);
+	xdptxd.data = xdpf->data;
+	xdptxd.len  = xdpf->len;
+	if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY) {
+		/* The xdp_buff was in the UMEM and was copied into a newly
+		 * allocated page. The UMEM page was returned via the ZCA, and
+		 * this new page has to be mapped at this point and has to be
+		 * unmapped and returned via xdp_return_frame on completion.
+		 */
+		/* Prevent double recycling of the UMEM page. Even in case this
+		 * function returns false, the xdp_buff shouldn't be recycled,
+		 * as it was already done in xdp_convert_zc_to_xdp_frame.
+		 */
+		__set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags); /* non-atomic */
+		xdpi.mode = MLX5E_XDP_XMIT_MODE_FRAME;
+		dma_addr = dma_map_single(sq->pdev, xdptxd.data, xdptxd.len,
+					  DMA_TO_DEVICE);
+		if (dma_mapping_error(sq->pdev, dma_addr)) {
+			xdp_return_frame(xdpf);
+			return false;
+		}
+		xdptxd.dma_addr     = dma_addr;
+		xdpi.frame.xdpf     = xdpf;
+		xdpi.frame.dma_addr = dma_addr;
+	} else {
+		/* Driver assumes that convert_to_xdp_frame returns an xdp_frame
+		 * that points to the same memory region as the original
+		 * xdp_buff. It allows to map the memory only once and to use
+		 * the DMA_BIDIRECTIONAL mode.
+		 */
+		xdpi.mode = MLX5E_XDP_XMIT_MODE_PAGE;
+		dma_addr = di->addr + (xdpf->data - (void *)xdpf);
+		dma_sync_single_for_device(sq->pdev, dma_addr, xdptxd.len,
+					   DMA_TO_DEVICE);
+		xdptxd.dma_addr = dma_addr;
+		xdpi.page.rq    = rq;
+		xdpi.page.di    = *di;
+	}
+	return sq->xmit_xdp_frame(sq, &xdptxd, &xdpi, 0);
 }
 /* returns true if packet was consumed by xdp */
 bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct mlx5e_dma_info *di,
-		      void *va, u16 *rx_headroom, u32 *len)
+		      void *va, u16 *rx_headroom, u32 *len, bool xsk)
 {
 	struct bpf_prog *prog = READ_ONCE(rq->xdp_prog);
 	struct xdp_buff xdp;
 	xdp_set_data_meta_invalid(&xdp);
 	xdp.data_end = xdp.data + *len;
 	xdp.data_hard_start = va;
+	if (xsk)
+		xdp.handle = di->xsk.handle;
 	xdp.rxq = &rq->xdp_rxq;
 	act = bpf_prog_run_xdp(prog, &xdp);
+	if (xsk)
+		xdp.handle += xdp.data - xdp.data_hard_start;
 	switch (act) {
 	case XDP_PASS:
 		*rx_headroom = xdp.data - xdp.data_hard_start;
 		*len = xdp.data_end - xdp.data;
 		return false;
 	case XDP_TX:
-		if (unlikely(!mlx5e_xmit_xdp_buff(&rq->xdpsq, di, &xdp)))
+		if (unlikely(!mlx5e_xmit_xdp_buff(rq->xdpsq, rq, di, &xdp)))
 			goto xdp_abort;
 		__set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags); /* non-atomic */
 		return true;
 			goto xdp_abort;
 		__set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags);
 		__set_bit(MLX5E_RQ_FLAG_XDP_REDIRECT, rq->flags);
-		mlx5e_page_dma_unmap(rq, di);
+		if (!xsk)
+			mlx5e_page_dma_unmap(rq, di);
 		rq->stats->xdp_redirect++;
 		return true;
 	default:
 	stats->mpwqe++;
 }
-static void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq)
+void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq)
 {
 	struct mlx5_wq_cyc       *wq    = &sq->wq;
 	struct mlx5e_xdp_mpwqe *session = &sq->mpwqe;
 	session->wqe = NULL; /* Close session */
 }
+enum {
+	MLX5E_XDP_CHECK_OK = 1,
+	MLX5E_XDP_CHECK_START_MPWQE = 2,
+};
+static int mlx5e_xmit_xdp_frame_check_mpwqe(struct mlx5e_xdpsq *sq)
+{
+	if (unlikely(!sq->mpwqe.wqe)) {
+		if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc,
+						     MLX5_SEND_WQE_MAX_WQEBBS))) {
+			/* SQ is full, ring doorbell */
+			mlx5e_xmit_xdp_doorbell(sq);
+			sq->stats->full++;
+			return -EBUSY;
+		}
+		return MLX5E_XDP_CHECK_START_MPWQE;
+	}
+	return MLX5E_XDP_CHECK_OK;
+}
 static bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq,
-				       struct mlx5e_xdp_info *xdpi)
+				       struct mlx5e_xdp_xmit_data *xdptxd,
+				       struct mlx5e_xdp_info *xdpi,
+				       int check_result)
 {
 	struct mlx5e_xdp_mpwqe *session = &sq->mpwqe;
 	struct mlx5e_xdpsq_stats *stats = sq->stats;
-	struct xdp_frame *xdpf = xdpi->xdpf;
+	if (unlikely(xdptxd->len > sq->hw_mtu)) {
-	if (unlikely(sq->hw_mtu < xdpf->len)) {
 		stats->err++;
 		return false;
 	}
-	if (unlikely(!session->wqe)) {
+	if (!check_result)
-		if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc,
+		check_result = mlx5e_xmit_xdp_frame_check_mpwqe(sq);
-						     MLX5_SEND_WQE_MAX_WQEBBS))) {
+	if (unlikely(check_result < 0))
-			/* SQ is full, ring doorbell */
+		return false;
-			mlx5e_xmit_xdp_doorbell(sq);
-			stats->full++;
-			return false;
-		}
+	if (check_result == MLX5E_XDP_CHECK_START_MPWQE) {
+		/* Start the session when nothing can fail, so it's guaranteed
+		 * that if there is an active session, it has at least one dseg,
+		 * and it's safe to complete it at any time.
+		 */
 		mlx5e_xdp_mpwqe_session_start(sq);
 	}
-	mlx5e_xdp_mpwqe_add_dseg(sq, xdpi, stats);
+	mlx5e_xdp_mpwqe_add_dseg(sq, xdptxd, stats);
 	if (unlikely(session->complete ||
 		     session->ds_count == session->max_ds_count))
 	return true;
 }
-static bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xdp_info *xdpi)
+static int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq *sq)
+{
+	if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, 1))) {
+		/* SQ is full, ring doorbell */
+		mlx5e_xmit_xdp_doorbell(sq);
+		sq->stats->full++;
+		return -EBUSY;
+	}
+	return MLX5E_XDP_CHECK_OK;
+}
+static bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq,
+				 struct mlx5e_xdp_xmit_data *xdptxd,
+				 struct mlx5e_xdp_info *xdpi,
+				 int check_result)
 {
 	struct mlx5_wq_cyc       *wq   = &sq->wq;
 	u16                       pi   = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
 	struct mlx5_wqe_eth_seg  *eseg = &wqe->eth;
 	struct mlx5_wqe_data_seg *dseg = wqe->data;
-	struct xdp_frame *xdpf = xdpi->xdpf;
+	dma_addr_t dma_addr = xdptxd->dma_addr;
-	dma_addr_t dma_addr  = xdpi->dma_addr;
+	u32 dma_len = xdptxd->len;
-	unsigned int dma_len = xdpf->len;
 	struct mlx5e_xdpsq_stats *stats = sq->stats;
 		return false;
 	}
-	if (unlikely(!mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1))) {
+	if (!check_result)
-		/* SQ is full, ring doorbell */
+		check_result = mlx5e_xmit_xdp_frame_check(sq);
-		mlx5e_xmit_xdp_doorbell(sq);
+	if (unlikely(check_result < 0))
-		stats->full++;
 		return false;
-	}
 	cseg->fm_ce_se = 0;
 	/* copy the inline part if required */
 	if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) {
-		memcpy(eseg->inline_hdr.start, xdpf->data, MLX5E_XDP_MIN_INLINE);
+		memcpy(eseg->inline_hdr.start, xdptxd->data, MLX5E_XDP_MIN_INLINE);
 		eseg->inline_hdr.sz = cpu_to_be16(MLX5E_XDP_MIN_INLINE);
 		dma_len  -= MLX5E_XDP_MIN_INLINE;
 		dma_addr += MLX5E_XDP_MIN_INLINE;
 static void mlx5e_free_xdpsq_desc(struct mlx5e_xdpsq *sq,
 				  struct mlx5e_xdp_wqe_info *wi,
-				  struct mlx5e_rq *rq,
+				  u32 *xsk_frames,
 				  bool recycle)
 {
 	struct mlx5e_xdp_info_fifo *xdpi_fifo = &sq->db.xdpi_fifo;
 	for (i = 0; i < wi->num_pkts; i++) {
 		struct mlx5e_xdp_info xdpi = mlx5e_xdpi_fifo_pop(xdpi_fifo);
-		if (rq) {
+		switch (xdpi.mode) {
-			/* XDP_TX */
+		case MLX5E_XDP_XMIT_MODE_FRAME:
-			mlx5e_page_release(rq, &xdpi.di, recycle);
+			/* XDP_TX from the XSK RQ and XDP_REDIRECT */
-		} else {
+			dma_unmap_single(sq->pdev, xdpi.frame.dma_addr,
-			/* XDP_REDIRECT */
+					 xdpi.frame.xdpf->len, DMA_TO_DEVICE);
-			dma_unmap_single(sq->pdev, xdpi.dma_addr,
+			xdp_return_frame(xdpi.frame.xdpf);
-					 xdpi.xdpf->len, DMA_TO_DEVICE);
+			break;
-			xdp_return_frame(xdpi.xdpf);
+		case MLX5E_XDP_XMIT_MODE_PAGE:
+			/* XDP_TX from the regular RQ */
+			mlx5e_page_release_dynamic(xdpi.page.rq, &xdpi.page.di, recycle);
+			break;
+		case MLX5E_XDP_XMIT_MODE_XSK:
+			/* AF_XDP send */
+			(*xsk_frames)++;
+			break;
+		default:
+			WARN_ON_ONCE(true);
 		}
 	}
 }
-bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq)
+bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
 {
 	struct mlx5e_xdpsq *sq;
 	struct mlx5_cqe64 *cqe;
+	u32 xsk_frames = 0;
 	u16 sqcc;
 	int i;
 			sqcc += wi->num_wqebbs;
-			mlx5e_free_xdpsq_desc(sq, wi, rq, true);
+			mlx5e_free_xdpsq_desc(sq, wi, &xsk_frames, true);
 		} while (!last_wqe);
 	} while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
+	if (xsk_frames)
+		xsk_umem_complete_tx(sq->umem, xsk_frames);
 	sq->stats->cqes += i;
 	mlx5_cqwq_update_db_record(&cq->wq);
 	return (i == MLX5E_TX_CQ_POLL_BUDGET);
 }
-void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq)
+void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq)
 {
+	u32 xsk_frames = 0;
 	while (sq->cc != sq->pc) {
 		struct mlx5e_xdp_wqe_info *wi;
 		u16 ci;
 		sq->cc += wi->num_wqebbs;
-		mlx5e_free_xdpsq_desc(sq, wi, rq, false);
+		mlx5e_free_xdpsq_desc(sq, wi, &xsk_frames, false);
 	}
+	if (xsk_frames)
+		xsk_umem_complete_tx(sq->umem, xsk_frames);
 }
 int mlx5e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,
 	for (i = 0; i < n; i++) {
 		struct xdp_frame *xdpf = frames[i];
+		struct mlx5e_xdp_xmit_data xdptxd;
 		struct mlx5e_xdp_info xdpi;
-		xdpi.dma_addr = dma_map_single(sq->pdev, xdpf->data, xdpf->len,
+		xdptxd.data = xdpf->data;
-					       DMA_TO_DEVICE);
+		xdptxd.len = xdpf->len;
-		if (unlikely(dma_mapping_error(sq->pdev, xdpi.dma_addr))) {
+		xdptxd.dma_addr = dma_map_single(sq->pdev, xdptxd.data,
+						 xdptxd.len, DMA_TO_DEVICE);
+		if (unlikely(dma_mapping_error(sq->pdev, xdptxd.dma_addr))) {
 			xdp_return_frame_rx_napi(xdpf);
 			drops++;
 			continue;
 		}
-		xdpi.xdpf = xdpf;
+		xdpi.mode           = MLX5E_XDP_XMIT_MODE_FRAME;
+		xdpi.frame.xdpf     = xdpf;
+		xdpi.frame.dma_addr = xdptxd.dma_addr;
-		if (unlikely(!sq->xmit_xdp_frame(sq, &xdpi))) {
+		if (unlikely(!sq->xmit_xdp_frame(sq, &xdptxd, &xdpi, 0))) {
-			dma_unmap_single(sq->pdev, xdpi.dma_addr,
+			dma_unmap_single(sq->pdev, xdptxd.dma_addr,
-					 xdpf->len, DMA_TO_DEVICE);
+					 xdptxd.len, DMA_TO_DEVICE);
 			xdp_return_frame_rx_napi(xdpf);
 			drops++;
 		}
 void mlx5e_xdp_rx_poll_complete(struct mlx5e_rq *rq)
 {
-	struct mlx5e_xdpsq *xdpsq = &rq->xdpsq;
+	struct mlx5e_xdpsq *xdpsq = rq->xdpsq;
 	if (xdpsq->mpwqe.wqe)
 		mlx5e_xdp_mpwqe_complete(xdpsq);
 void mlx5e_set_xmit_fp(struct mlx5e_xdpsq *sq, bool is_mpw)
 {
+	sq->xmit_xdp_frame_check = is_mpw ?
+		mlx5e_xmit_xdp_frame_check_mpwqe : mlx5e_xmit_xdp_frame_check;
 	sq->xmit_xdp_frame = is_mpw ?
 		mlx5e_xmit_xdp_frame_mpwqe : mlx5e_xmit_xdp_frame;
 }

drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h+26−10

 	(sizeof(struct mlx5e_tx_wqe) / MLX5_SEND_WQE_DS)
 #define MLX5E_XDP_TX_DS_COUNT (MLX5E_XDP_TX_EMPTY_DS_COUNT + 1 /* SG DS */)
-int mlx5e_xdp_max_mtu(struct mlx5e_params *params);
+struct mlx5e_xsk_param;
+int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk);
 bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct mlx5e_dma_info *di,
-		      void *va, u16 *rx_headroom, u32 *len);
+		      void *va, u16 *rx_headroom, u32 *len, bool xsk);
-bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq);
+void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq);
-void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq);
+bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq);
+void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq);
 void mlx5e_set_xmit_fp(struct mlx5e_xdpsq *sq, bool is_mpw);
 void mlx5e_xdp_rx_poll_complete(struct mlx5e_rq *rq);
 int mlx5e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,
 	return test_bit(MLX5E_STATE_XDP_TX_ENABLED, &priv->state);
 }
+static inline void mlx5e_xdp_set_open(struct mlx5e_priv *priv)
+{
+	set_bit(MLX5E_STATE_XDP_OPEN, &priv->state);
+}
+static inline void mlx5e_xdp_set_closed(struct mlx5e_priv *priv)
+{
+	clear_bit(MLX5E_STATE_XDP_OPEN, &priv->state);
+}
+static inline bool mlx5e_xdp_is_open(struct mlx5e_priv *priv)
+{
+	return test_bit(MLX5E_STATE_XDP_OPEN, &priv->state);
+}
 static inline void mlx5e_xmit_xdp_doorbell(struct mlx5e_xdpsq *sq)
 {
 	if (sq->doorbell_cseg) {
 }
 static inline void
-mlx5e_xdp_mpwqe_add_dseg(struct mlx5e_xdpsq *sq, struct mlx5e_xdp_info *xdpi,
+mlx5e_xdp_mpwqe_add_dseg(struct mlx5e_xdpsq *sq,
+			 struct mlx5e_xdp_xmit_data *xdptxd,
 			 struct mlx5e_xdpsq_stats *stats)
 {
 	struct mlx5e_xdp_mpwqe *session = &sq->mpwqe;
-	dma_addr_t dma_addr    = xdpi->dma_addr;
-	struct xdp_frame *xdpf = xdpi->xdpf;
 	struct mlx5_wqe_data_seg *dseg =
 		(struct mlx5_wqe_data_seg *)session->wqe + session->ds_count;
-	u16 dma_len = xdpf->len;
+	u32 dma_len = xdptxd->len;
 	session->pkt_count++;
 		}
 		inline_dseg->byte_count = cpu_to_be32(dma_len | MLX5_INLINE_SEG);
-		memcpy(inline_dseg->data, xdpf->data, dma_len);
+		memcpy(inline_dseg->data, xdptxd->data, dma_len);
 		session->ds_count += ds_cnt;
 		stats->inlnw++;
 	}
 no_inline:
-	dseg->addr       = cpu_to_be64(dma_addr);
+	dseg->addr       = cpu_to_be64(xdptxd->dma_addr);
 	dseg->byte_count = cpu_to_be32(dma_len);
 	dseg->lkey       = sq->mkey_be;
 	session->ds_count++;

drivers/net/ethernet/mellanox/mlx5/core/en/xsk/Makefile+1−0

@@ -0,0 +1 @@
		1	subdir-ccflags-y += -I$(src)/../..

drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c+192−0

@@ -0,0 +1,192 @@
	1	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
	2	/* Copyright (c) 2019 Mellanox Technologies. */
	3
	4	#include "rx.h"
	5	#include "en/xdp.h"
	6	#include <net/xdp_sock.h>
	7
	8	/* RX data path */
	9
	10	bool mlx5e_xsk_pages_enough_umem(struct mlx5e_rq *rq, int count)
	11	{
	12	/* Check in advance that we have enough frames, instead of allocating
	13	* one-by-one, failing and moving frames to the Reuse Ring.
	14	*/
	15	return xsk_umem_has_addrs_rq(rq->umem, count);
	16	}
	17
	18	int mlx5e_xsk_page_alloc_umem(struct mlx5e_rq *rq,
	19	struct mlx5e_dma_info *dma_info)
	20	{
	21	struct xdp_umem *umem = rq->umem;
	22	u64 handle;
	23
	24	if (!xsk_umem_peek_addr_rq(umem, &handle))
	25	return -ENOMEM;
	26
	27	dma_info->xsk.handle = handle + rq->buff.umem_headroom;
	28	dma_info->xsk.data = xdp_umem_get_data(umem, dma_info->xsk.handle);
	29
	30	/* No need to add headroom to the DMA address. In striding RQ case, we
	31	* just provide pages for UMR, and headroom is counted at the setup
	32	* stage when creating a WQE. In non-striding RQ case, headroom is
	33	* accounted in mlx5e_alloc_rx_wqe.
	34	*/
	35	dma_info->addr = xdp_umem_get_dma(umem, handle);
	36
	37	xsk_umem_discard_addr_rq(umem);
	38
	39	dma_sync_single_for_device(rq->pdev, dma_info->addr, PAGE_SIZE,
	40	DMA_BIDIRECTIONAL);
	41
	42	return 0;
	43	}
	44
	45	static inline void mlx5e_xsk_recycle_frame(struct mlx5e_rq *rq, u64 handle)
	46	{
	47	xsk_umem_fq_reuse(rq->umem, handle & rq->umem->chunk_mask);
	48	}
	49
	50	/* XSKRQ uses pages from UMEM, they must not be released. They are returned to
	51	* the userspace if possible, and if not, this function is called to reuse them
	52	* in the driver.
	53	*/
	54	void mlx5e_xsk_page_release(struct mlx5e_rq *rq,
	55	struct mlx5e_dma_info *dma_info)
	56	{
	57	mlx5e_xsk_recycle_frame(rq, dma_info->xsk.handle);
	58	}
	59
	60	/* Return a frame back to the hardware to fill in again. It is used by XDP when
	61	* the XDP program returns XDP_TX or XDP_REDIRECT not to an XSKMAP.
	62	*/
	63	void mlx5e_xsk_zca_free(struct zero_copy_allocator *zca, unsigned long handle)
	64	{
	65	struct mlx5e_rq *rq = container_of(zca, struct mlx5e_rq, zca);
	66
	67	mlx5e_xsk_recycle_frame(rq, handle);
	68	}
	69
	70	static struct sk_buff mlx5e_xsk_construct_skb(struct mlx5e_rq rq, void *data,
	71	u32 cqe_bcnt)
	72	{
	73	struct sk_buff *skb;
	74
	75	skb = napi_alloc_skb(rq->cq.napi, cqe_bcnt);
	76	if (unlikely(!skb)) {
	77	rq->stats->buff_alloc_err++;
	78	return NULL;
	79	}
	80
	81	skb_put_data(skb, data, cqe_bcnt);
	82
	83	return skb;
	84	}
	85
	86	struct sk_buff mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq rq,
	87	struct mlx5e_mpw_info *wi,
	88	u16 cqe_bcnt,
	89	u32 head_offset,
	90	u32 page_idx)
	91	{
	92	struct mlx5e_dma_info *di = &wi->umr.dma_info[page_idx];
	93	u16 rx_headroom = rq->buff.headroom - rq->buff.umem_headroom;
	94	u32 cqe_bcnt32 = cqe_bcnt;
	95	void va, data;
	96	u32 frag_size;
	97	bool consumed;
	98
	99	/* Check packet size. Note LRO doesn't use linear SKB */
	100	if (unlikely(cqe_bcnt > rq->hw_mtu)) {
	101	rq->stats->oversize_pkts_sw_drop++;
	102	return NULL;
	103	}
	104
	105	/* head_offset is not used in this function, because di->xsk.data and
	106	* di->addr point directly to the necessary place. Furthermore, in the
	107	* current implementation, one page = one packet = one frame, so
	108	* head_offset should always be 0.
	109	*/
	110	WARN_ON_ONCE(head_offset);
	111
	112	va = di->xsk.data;
	113	data = va + rx_headroom;
	114	frag_size = rq->buff.headroom + cqe_bcnt32;
	115
	116	dma_sync_single_for_cpu(rq->pdev, di->addr, frag_size, DMA_BIDIRECTIONAL);
	117	prefetch(data);
	118
	119	rcu_read_lock();
	120	consumed = mlx5e_xdp_handle(rq, di, va, &rx_headroom, &cqe_bcnt32, true);
	121	rcu_read_unlock();
	122
	123	/* Possible flows:
	124	* - XDP_REDIRECT to XSKMAP:
	125	* The page is owned by the userspace from now.
	126	* - XDP_TX and other XDP_REDIRECTs:
	127	* The page was returned by ZCA and recycled.
	128	* - XDP_DROP:
	129	* Recycle the page.
	130	* - XDP_PASS:
	131	* Allocate an SKB, copy the data and recycle the page.
	132	*
	133	* Pages to be recycled go to the Reuse Ring on MPWQE deallocation. Its
	134	* size is the same as the Driver RX Ring's size, and pages for WQEs are
	135	* allocated first from the Reuse Ring, so it has enough space.
	136	*/
	137
	138	if (likely(consumed)) {
	139	if (likely(__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)))
	140	__set_bit(page_idx, wi->xdp_xmit_bitmap); /* non-atomic */
	141	return NULL; /* page/packet was consumed by XDP */
	142	}
	143
	144	/* XDP_PASS: copy the data from the UMEM to a new SKB and reuse the
	145	* frame. On SKB allocation failure, NULL is returned.
	146	*/
	147	return mlx5e_xsk_construct_skb(rq, data, cqe_bcnt32);
	148	}
	149
	150	struct sk_buff mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq rq,
	151	struct mlx5_cqe64 *cqe,
	152	struct mlx5e_wqe_frag_info *wi,
	153	u32 cqe_bcnt)
	154	{
	155	struct mlx5e_dma_info *di = wi->di;
	156	u16 rx_headroom = rq->buff.headroom - rq->buff.umem_headroom;
	157	void va, data;
	158	bool consumed;
	159	u32 frag_size;
	160
	161	/* wi->offset is not used in this function, because di->xsk.data and
	162	* di->addr point directly to the necessary place. Furthermore, in the
	163	* current implementation, one page = one packet = one frame, so
	164	* wi->offset should always be 0.
	165	*/
	166	WARN_ON_ONCE(wi->offset);
	167
	168	va = di->xsk.data;
	169	data = va + rx_headroom;
	170	frag_size = rq->buff.headroom + cqe_bcnt;
	171
	172	dma_sync_single_for_cpu(rq->pdev, di->addr, frag_size, DMA_BIDIRECTIONAL);
	173	prefetch(data);
	174
	175	if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_RESP_SEND)) {
	176	rq->stats->wqe_err++;
	177	return NULL;
	178	}
	179
	180	rcu_read_lock();
	181	consumed = mlx5e_xdp_handle(rq, di, va, &rx_headroom, &cqe_bcnt, true);
	182	rcu_read_unlock();
	183
	184	if (likely(consumed))
	185	return NULL; /* page/packet was consumed by XDP */
	186
	187	/* XDP_PASS: copy the data from the UMEM to a new SKB. The frame reuse
	188	* will be handled by mlx5e_put_rx_frag.
	189	* On SKB allocation failure, NULL is returned.
	190	*/
	191	return mlx5e_xsk_construct_skb(rq, data, cqe_bcnt);
	192	}

drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h+27−0

@@ -0,0 +1,27 @@
	1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
	2	/* Copyright (c) 2019 Mellanox Technologies. */
	3
	4	#ifndef __MLX5_EN_XSK_RX_H__
	5	#define __MLX5_EN_XSK_RX_H__
	6
	7	#include "en.h"
	8
	9	/* RX data path */
	10
	11	bool mlx5e_xsk_pages_enough_umem(struct mlx5e_rq *rq, int count);
	12	int mlx5e_xsk_page_alloc_umem(struct mlx5e_rq *rq,
	13	struct mlx5e_dma_info *dma_info);
	14	void mlx5e_xsk_page_release(struct mlx5e_rq *rq,
	15	struct mlx5e_dma_info *dma_info);
	16	void mlx5e_xsk_zca_free(struct zero_copy_allocator *zca, unsigned long handle);
	17	struct sk_buff mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq rq,
	18	struct mlx5e_mpw_info *wi,
	19	u16 cqe_bcnt,
	20	u32 head_offset,
	21	u32 page_idx);
	22	struct sk_buff mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq rq,
	23	struct mlx5_cqe64 *cqe,
	24	struct mlx5e_wqe_frag_info *wi,
	25	u32 cqe_bcnt);
	26
	27	#endif /* __MLX5_EN_XSK_RX_H__ */

drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c+223−0

@@ -0,0 +1,223 @@
	1	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
	2	/* Copyright (c) 2019 Mellanox Technologies. */
	3
	4	#include "setup.h"
	5	#include "en/params.h"
	6
	7	bool mlx5e_validate_xsk_param(struct mlx5e_params *params,
	8	struct mlx5e_xsk_param *xsk,
	9	struct mlx5_core_dev *mdev)
	10	{
	11	/* AF_XDP doesn't support frames larger than PAGE_SIZE, and the current
	12	* mlx5e XDP implementation doesn't support multiple packets per page.
	13	*/
	14	if (xsk->chunk_size != PAGE_SIZE)
	15	return false;
	16
	17	/* Current MTU and XSK headroom don't allow packets to fit the frames. */
	18	if (mlx5e_rx_get_linear_frag_sz(params, xsk) > xsk->chunk_size)
	19	return false;
	20
	21	/* frag_sz is different for regular and XSK RQs, so ensure that linear
	22	* SKB mode is possible.
	23	*/
	24	switch (params->rq_wq_type) {
	25	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
	26	return mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk);
	27	default: /* MLX5_WQ_TYPE_CYCLIC */
	28	return mlx5e_rx_is_linear_skb(params, xsk);
	29	}
	30	}
	31
	32	static void mlx5e_build_xskicosq_param(struct mlx5e_priv *priv,
	33	u8 log_wq_size,
	34	struct mlx5e_sq_param *param)
	35	{
	36	void *sqc = param->sqc;
	37	void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
	38
	39	mlx5e_build_sq_param_common(priv, param);
	40
	41	MLX5_SET(wq, wq, log_wq_sz, log_wq_size);
	42	}
	43
	44	static void mlx5e_build_xsk_cparam(struct mlx5e_priv *priv,
	45	struct mlx5e_params *params,
	46	struct mlx5e_xsk_param *xsk,
	47	struct mlx5e_channel_param *cparam)
	48	{
	49	const u8 xskicosq_size = MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
	50
	51	mlx5e_build_rq_param(priv, params, xsk, &cparam->rq);
	52	mlx5e_build_xdpsq_param(priv, params, &cparam->xdp_sq);
	53	mlx5e_build_xskicosq_param(priv, xskicosq_size, &cparam->icosq);
	54	mlx5e_build_rx_cq_param(priv, params, xsk, &cparam->rx_cq);
	55	mlx5e_build_tx_cq_param(priv, params, &cparam->tx_cq);
	56	mlx5e_build_ico_cq_param(priv, xskicosq_size, &cparam->icosq_cq);
	57	}
	58
	59	int mlx5e_open_xsk(struct mlx5e_priv priv, struct mlx5e_params params,
	60	struct mlx5e_xsk_param xsk, struct xdp_umem umem,
	61	struct mlx5e_channel *c)
	62	{
	63	struct mlx5e_channel_param cparam = {};
	64	struct dim_cq_moder icocq_moder = {};
	65	int err;
	66
	67	if (!mlx5e_validate_xsk_param(params, xsk, priv->mdev))
	68	return -EINVAL;
	69
	70	mlx5e_build_xsk_cparam(priv, params, xsk, &cparam);
	71
	72	err = mlx5e_open_cq(c, params->rx_cq_moderation, &cparam.rx_cq, &c->xskrq.cq);
	73	if (unlikely(err))
	74	return err;
	75
	76	err = mlx5e_open_rq(c, params, &cparam.rq, xsk, umem, &c->xskrq);
	77	if (unlikely(err))
	78	goto err_close_rx_cq;
	79
	80	err = mlx5e_open_cq(c, params->tx_cq_moderation, &cparam.tx_cq, &c->xsksq.cq);
	81	if (unlikely(err))
	82	goto err_close_rq;
	83
	84	/* Create a separate SQ, so that when the UMEM is disabled, we could
	85	* close this SQ safely and stop receiving CQEs. In other case, e.g., if
	86	* the XDPSQ was used instead, we might run into trouble when the UMEM
	87	* is disabled and then reenabled, but the SQ continues receiving CQEs
	88	* from the old UMEM.
	89	*/
	90	err = mlx5e_open_xdpsq(c, params, &cparam.xdp_sq, umem, &c->xsksq, true);
	91	if (unlikely(err))
	92	goto err_close_tx_cq;
	93
	94	err = mlx5e_open_cq(c, icocq_moder, &cparam.icosq_cq, &c->xskicosq.cq);
	95	if (unlikely(err))
	96	goto err_close_sq;
	97
	98	/* Create a dedicated SQ for posting NOPs whenever we need an IRQ to be
	99	* triggered and NAPI to be called on the correct CPU.
	100	*/
	101	err = mlx5e_open_icosq(c, params, &cparam.icosq, &c->xskicosq);
	102	if (unlikely(err))
	103	goto err_close_icocq;
	104
	105	spin_lock_init(&c->xskicosq_lock);
	106
	107	set_bit(MLX5E_CHANNEL_STATE_XSK, c->state);
	108
	109	return 0;
	110
	111	err_close_icocq:
	112	mlx5e_close_cq(&c->xskicosq.cq);
	113
	114	err_close_sq:
	115	mlx5e_close_xdpsq(&c->xsksq);
	116
	117	err_close_tx_cq:
	118	mlx5e_close_cq(&c->xsksq.cq);
	119
	120	err_close_rq:
	121	mlx5e_close_rq(&c->xskrq);
	122
	123	err_close_rx_cq:
	124	mlx5e_close_cq(&c->xskrq.cq);
	125
	126	return err;
	127	}
	128
	129	void mlx5e_close_xsk(struct mlx5e_channel *c)
	130	{
	131	clear_bit(MLX5E_CHANNEL_STATE_XSK, c->state);
	132	napi_synchronize(&c->napi);
	133
	134	mlx5e_close_rq(&c->xskrq);
	135	mlx5e_close_cq(&c->xskrq.cq);
	136	mlx5e_close_icosq(&c->xskicosq);
	137	mlx5e_close_cq(&c->xskicosq.cq);
	138	mlx5e_close_xdpsq(&c->xsksq);
	139	mlx5e_close_cq(&c->xsksq.cq);
	140	}
	141
	142	void mlx5e_activate_xsk(struct mlx5e_channel *c)
	143	{
	144	set_bit(MLX5E_RQ_STATE_ENABLED, &c->xskrq.state);
	145	/* TX queue is created active. */
	146	mlx5e_trigger_irq(&c->xskicosq);
	147	}
	148
	149	void mlx5e_deactivate_xsk(struct mlx5e_channel *c)
	150	{
	151	mlx5e_deactivate_rq(&c->xskrq);
	152	/* TX queue is disabled on close. */
	153	}
	154
	155	static int mlx5e_redirect_xsk_rqt(struct mlx5e_priv *priv, u16 ix, u32 rqn)
	156	{
	157	struct mlx5e_redirect_rqt_param direct_rrp = {
	158	.is_rss = false,
	159	{
	160	.rqn = rqn,
	161	},
	162	};
	163
	164	u32 rqtn = priv->xsk_tir[ix].rqt.rqtn;
	165
	166	return mlx5e_redirect_rqt(priv, rqtn, 1, direct_rrp);
	167	}
	168
	169	int mlx5e_xsk_redirect_rqt_to_channel(struct mlx5e_priv priv, struct mlx5e_channel c)
	170	{
	171	return mlx5e_redirect_xsk_rqt(priv, c->ix, c->xskrq.rqn);
	172	}
	173
	174	int mlx5e_xsk_redirect_rqt_to_drop(struct mlx5e_priv *priv, u16 ix)
	175	{
	176	return mlx5e_redirect_xsk_rqt(priv, ix, priv->drop_rq.rqn);
	177	}
	178
	179	int mlx5e_xsk_redirect_rqts_to_channels(struct mlx5e_priv priv, struct mlx5e_channels chs)
	180	{
	181	int err, i;
	182
	183	if (!priv->xsk.refcnt)
	184	return 0;
	185
	186	for (i = 0; i < chs->num; i++) {
	187	struct mlx5e_channel *c = chs->c[i];
	188
	189	if (!test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
	190	continue;
	191
	192	err = mlx5e_xsk_redirect_rqt_to_channel(priv, c);
	193	if (unlikely(err))
	194	goto err_stop;
	195	}
	196
	197	return 0;
	198
	199	err_stop:
	200	for (i--; i >= 0; i--) {
	201	if (!test_bit(MLX5E_CHANNEL_STATE_XSK, chs->c[i]->state))
	202	continue;
	203
	204	mlx5e_xsk_redirect_rqt_to_drop(priv, i);
	205	}
	206
	207	return err;
	208	}
	209
	210	void mlx5e_xsk_redirect_rqts_to_drop(struct mlx5e_priv priv, struct mlx5e_channels chs)
	211	{
	212	int i;
	213
	214	if (!priv->xsk.refcnt)
	215	return;
	216
	217	for (i = 0; i < chs->num; i++) {
	218	if (!test_bit(MLX5E_CHANNEL_STATE_XSK, chs->c[i]->state))
	219	continue;
	220
	221	mlx5e_xsk_redirect_rqt_to_drop(priv, i);
	222	}
	223	}

drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.h+25−0

@@ -0,0 +1,25 @@
	1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
	2	/* Copyright (c) 2019 Mellanox Technologies. */
	3
	4	#ifndef __MLX5_EN_XSK_SETUP_H__
	5	#define __MLX5_EN_XSK_SETUP_H__
	6
	7	#include "en.h"
	8
	9	struct mlx5e_xsk_param;
	10
	11	bool mlx5e_validate_xsk_param(struct mlx5e_params *params,
	12	struct mlx5e_xsk_param *xsk,
	13	struct mlx5_core_dev *mdev);
	14	int mlx5e_open_xsk(struct mlx5e_priv priv, struct mlx5e_params params,
	15	struct mlx5e_xsk_param xsk, struct xdp_umem umem,
	16	struct mlx5e_channel *c);
	17	void mlx5e_close_xsk(struct mlx5e_channel *c);
	18	void mlx5e_activate_xsk(struct mlx5e_channel *c);
	19	void mlx5e_deactivate_xsk(struct mlx5e_channel *c);
	20	int mlx5e_xsk_redirect_rqt_to_channel(struct mlx5e_priv priv, struct mlx5e_channel c);
	21	int mlx5e_xsk_redirect_rqt_to_drop(struct mlx5e_priv *priv, u16 ix);
	22	int mlx5e_xsk_redirect_rqts_to_channels(struct mlx5e_priv priv, struct mlx5e_channels chs);
	23	void mlx5e_xsk_redirect_rqts_to_drop(struct mlx5e_priv priv, struct mlx5e_channels chs);
	24
	25	#endif /* __MLX5_EN_XSK_SETUP_H__ */

drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c+111−0

@@ -0,0 +1,111 @@
	1	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
	2	/* Copyright (c) 2019 Mellanox Technologies. */
	3
	4	#include "tx.h"
	5	#include "umem.h"
	6	#include "en/xdp.h"
	7	#include "en/params.h"
	8	#include <net/xdp_sock.h>
	9
	10	int mlx5e_xsk_async_xmit(struct net_device *dev, u32 qid)
	11	{
	12	struct mlx5e_priv *priv = netdev_priv(dev);
	13	struct mlx5e_params *params = &priv->channels.params;
	14	struct mlx5e_channel *c;
	15	u16 ix;
	16
	17	if (unlikely(!mlx5e_xdp_is_open(priv)))
	18	return -ENETDOWN;
	19
	20	if (unlikely(!mlx5e_qid_get_ch_if_in_group(params, qid, MLX5E_RQ_GROUP_XSK, &ix)))
	21	return -EINVAL;
	22
	23	c = priv->channels.c[ix];
	24
	25	if (unlikely(!test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)))
	26	return -ENXIO;
	27
	28	if (!napi_if_scheduled_mark_missed(&c->napi)) {
	29	spin_lock(&c->xskicosq_lock);
	30	mlx5e_trigger_irq(&c->xskicosq);
	31	spin_unlock(&c->xskicosq_lock);
	32	}
	33
	34	return 0;
	35	}
	36
	37	/* When TX fails (because of the size of the packet), we need to get completions
	38	* in order, so post a NOP to get a CQE. Since AF_XDP doesn't distinguish
	39	* between successful TX and errors, handling in mlx5e_poll_xdpsq_cq is the
	40	* same.
	41	*/
	42	static void mlx5e_xsk_tx_post_err(struct mlx5e_xdpsq *sq,
	43	struct mlx5e_xdp_info *xdpi)
	44	{
	45	u16 pi = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->pc);
	46	struct mlx5e_xdp_wqe_info *wi = &sq->db.wqe_info[pi];
	47	struct mlx5e_tx_wqe *nopwqe;
	48
	49	wi->num_wqebbs = 1;
	50	wi->num_pkts = 1;
	51
	52	nopwqe = mlx5e_post_nop(&sq->wq, sq->sqn, &sq->pc);
	53	mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, xdpi);
	54	sq->doorbell_cseg = &nopwqe->ctrl;
	55	}
	56
	57	bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget)
	58	{
	59	struct xdp_umem *umem = sq->umem;
	60	struct mlx5e_xdp_info xdpi;
	61	struct mlx5e_xdp_xmit_data xdptxd;
	62	bool work_done = true;
	63	bool flush = false;
	64
	65	xdpi.mode = MLX5E_XDP_XMIT_MODE_XSK;
	66
	67	for (; budget; budget--) {
	68	int check_result = sq->xmit_xdp_frame_check(sq);
	69	struct xdp_desc desc;
	70
	71	if (unlikely(check_result < 0)) {
	72	work_done = false;
	73	break;
	74	}
	75
	76	if (!xsk_umem_consume_tx(umem, &desc)) {
	77	/* TX will get stuck until something wakes it up by
	78	* triggering NAPI. Currently it's expected that the
	79	* application calls sendto() if there are consumed, but
	80	* not completed frames.
	81	*/
	82	break;
	83	}
	84
	85	xdptxd.dma_addr = xdp_umem_get_dma(umem, desc.addr);
	86	xdptxd.data = xdp_umem_get_data(umem, desc.addr);
	87	xdptxd.len = desc.len;
	88
	89	dma_sync_single_for_device(sq->pdev, xdptxd.dma_addr,
	90	xdptxd.len, DMA_BIDIRECTIONAL);
	91
	92	if (unlikely(!sq->xmit_xdp_frame(sq, &xdptxd, &xdpi, check_result))) {
	93	if (sq->mpwqe.wqe)
	94	mlx5e_xdp_mpwqe_complete(sq);
	95
	96	mlx5e_xsk_tx_post_err(sq, &xdpi);
	97	}
	98
	99	flush = true;
	100	}
	101
	102	if (flush) {
	103	if (sq->mpwqe.wqe)
	104	mlx5e_xdp_mpwqe_complete(sq);
	105	mlx5e_xmit_xdp_doorbell(sq);
	106
	107	xsk_umem_consume_tx_done(umem);
	108	}
	109
	110	return !(budget && work_done);
	111	}

drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.h+15−0

@@ -0,0 +1,15 @@
	1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
	2	/* Copyright (c) 2019 Mellanox Technologies. */
	3
	4	#ifndef __MLX5_EN_XSK_TX_H__
	5	#define __MLX5_EN_XSK_TX_H__
	6
	7	#include "en.h"
	8
	9	/* TX data path */
	10
	11	int mlx5e_xsk_async_xmit(struct net_device *dev, u32 qid);
	12
	13	bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget);
	14
	15	#endif /* __MLX5_EN_XSK_TX_H__ */

drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c+267−0

@@ -0,0 +1,267 @@
	1	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
	2	/* Copyright (c) 2019 Mellanox Technologies. */
	3
	4	#include <net/xdp_sock.h>
	5	#include "umem.h"
	6	#include "setup.h"
	7	#include "en/params.h"
	8
	9	static int mlx5e_xsk_map_umem(struct mlx5e_priv *priv,
	10	struct xdp_umem *umem)
	11	{
	12	struct device *dev = priv->mdev->device;
	13	u32 i;
	14
	15	for (i = 0; i < umem->npgs; i++) {
	16	dma_addr_t dma = dma_map_page(dev, umem->pgs[i], 0, PAGE_SIZE,
	17	DMA_BIDIRECTIONAL);
	18
	19	if (unlikely(dma_mapping_error(dev, dma)))
	20	goto err_unmap;
	21	umem->pages[i].dma = dma;
	22	}
	23
	24	return 0;
	25
	26	err_unmap:
	27	while (i--) {
	28	dma_unmap_page(dev, umem->pages[i].dma, PAGE_SIZE,
	29	DMA_BIDIRECTIONAL);
	30	umem->pages[i].dma = 0;
	31	}
	32
	33	return -ENOMEM;
	34	}
	35
	36	static void mlx5e_xsk_unmap_umem(struct mlx5e_priv *priv,
	37	struct xdp_umem *umem)
	38	{
	39	struct device *dev = priv->mdev->device;
	40	u32 i;
	41
	42	for (i = 0; i < umem->npgs; i++) {
	43	dma_unmap_page(dev, umem->pages[i].dma, PAGE_SIZE,
	44	DMA_BIDIRECTIONAL);
	45	umem->pages[i].dma = 0;
	46	}
	47	}
	48
	49	static int mlx5e_xsk_get_umems(struct mlx5e_xsk *xsk)
	50	{
	51	if (!xsk->umems) {
	52	xsk->umems = kcalloc(MLX5E_MAX_NUM_CHANNELS,
	53	sizeof(*xsk->umems), GFP_KERNEL);
	54	if (unlikely(!xsk->umems))
	55	return -ENOMEM;
	56	}
	57
	58	xsk->refcnt++;
	59	xsk->ever_used = true;
	60
	61	return 0;
	62	}
	63
	64	static void mlx5e_xsk_put_umems(struct mlx5e_xsk *xsk)
	65	{
	66	if (!--xsk->refcnt) {
	67	kfree(xsk->umems);
	68	xsk->umems = NULL;
	69	}
	70	}
	71
	72	static int mlx5e_xsk_add_umem(struct mlx5e_xsk xsk, struct xdp_umem umem, u16 ix)
	73	{
	74	int err;
	75
	76	err = mlx5e_xsk_get_umems(xsk);
	77	if (unlikely(err))
	78	return err;
	79
	80	xsk->umems[ix] = umem;
	81	return 0;
	82	}
	83
	84	static void mlx5e_xsk_remove_umem(struct mlx5e_xsk *xsk, u16 ix)
	85	{
	86	xsk->umems[ix] = NULL;
	87
	88	mlx5e_xsk_put_umems(xsk);
	89	}
	90
	91	static bool mlx5e_xsk_is_umem_sane(struct xdp_umem *umem)
	92	{
	93	return umem->headroom <= 0xffff && umem->chunk_size_nohr <= 0xffff;
	94	}
	95
	96	void mlx5e_build_xsk_param(struct xdp_umem umem, struct mlx5e_xsk_param xsk)
	97	{
	98	xsk->headroom = umem->headroom;
	99	xsk->chunk_size = umem->chunk_size_nohr + umem->headroom;
	100	}
	101
	102	static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
	103	struct xdp_umem *umem, u16 ix)
	104	{
	105	struct mlx5e_params *params = &priv->channels.params;
	106	struct mlx5e_xsk_param xsk;
	107	struct mlx5e_channel *c;
	108	int err;
	109
	110	if (unlikely(mlx5e_xsk_get_umem(&priv->channels.params, &priv->xsk, ix)))
	111	return -EBUSY;
	112
	113	if (unlikely(!mlx5e_xsk_is_umem_sane(umem)))
	114	return -EINVAL;
	115
	116	err = mlx5e_xsk_map_umem(priv, umem);
	117	if (unlikely(err))
	118	return err;
	119
	120	err = mlx5e_xsk_add_umem(&priv->xsk, umem, ix);
	121	if (unlikely(err))
	122	goto err_unmap_umem;
	123
	124	mlx5e_build_xsk_param(umem, &xsk);
	125
	126	if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
	127	/* XSK objects will be created on open. */
	128	goto validate_closed;
	129	}
	130
	131	if (!params->xdp_prog) {
	132	/* XSK objects will be created when an XDP program is set,
	133	* and the channels are reopened.
	134	*/
	135	goto validate_closed;
	136	}
	137
	138	c = priv->channels.c[ix];
	139
	140	err = mlx5e_open_xsk(priv, params, &xsk, umem, c);
	141	if (unlikely(err))
	142	goto err_remove_umem;
	143
	144	mlx5e_activate_xsk(c);
	145
	146	/* Don't wait for WQEs, because the newer xdpsock sample doesn't provide
	147	* any Fill Ring entries at the setup stage.
	148	*/
	149
	150	err = mlx5e_xsk_redirect_rqt_to_channel(priv, priv->channels.c[ix]);
	151	if (unlikely(err))
	152	goto err_deactivate;
	153
	154	return 0;
	155
	156	err_deactivate:
	157	mlx5e_deactivate_xsk(c);
	158	mlx5e_close_xsk(c);
	159
	160	err_remove_umem:
	161	mlx5e_xsk_remove_umem(&priv->xsk, ix);
	162
	163	err_unmap_umem:
	164	mlx5e_xsk_unmap_umem(priv, umem);
	165
	166	return err;
	167
	168	validate_closed:
	169	/* Check the configuration in advance, rather than fail at a later stage
	170	* (in mlx5e_xdp_set or on open) and end up with no channels.
	171	*/
	172	if (!mlx5e_validate_xsk_param(params, &xsk, priv->mdev)) {
	173	err = -EINVAL;
	174	goto err_remove_umem;
	175	}
	176
	177	return 0;
	178	}
	179
	180	static int mlx5e_xsk_disable_locked(struct mlx5e_priv *priv, u16 ix)
	181	{
	182	struct xdp_umem *umem = mlx5e_xsk_get_umem(&priv->channels.params,
	183	&priv->xsk, ix);
	184	struct mlx5e_channel *c;
	185
	186	if (unlikely(!umem))
	187	return -EINVAL;
	188
	189	if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
	190	goto remove_umem;
	191
	192	/* XSK RQ and SQ are only created if XDP program is set. */
	193	if (!priv->channels.params.xdp_prog)
	194	goto remove_umem;
	195
	196	c = priv->channels.c[ix];
	197	mlx5e_xsk_redirect_rqt_to_drop(priv, ix);
	198	mlx5e_deactivate_xsk(c);
	199	mlx5e_close_xsk(c);
	200
	201	remove_umem:
	202	mlx5e_xsk_remove_umem(&priv->xsk, ix);
	203	mlx5e_xsk_unmap_umem(priv, umem);
	204
	205	return 0;
	206	}
	207
	208	static int mlx5e_xsk_enable_umem(struct mlx5e_priv priv, struct xdp_umem umem,
	209	u16 ix)
	210	{
	211	int err;
	212
	213	mutex_lock(&priv->state_lock);
	214	err = mlx5e_xsk_enable_locked(priv, umem, ix);
	215	mutex_unlock(&priv->state_lock);
	216
	217	return err;
	218	}
	219
	220	static int mlx5e_xsk_disable_umem(struct mlx5e_priv *priv, u16 ix)
	221	{
	222	int err;
	223
	224	mutex_lock(&priv->state_lock);
	225	err = mlx5e_xsk_disable_locked(priv, ix);
	226	mutex_unlock(&priv->state_lock);
	227
	228	return err;
	229	}
	230
	231	int mlx5e_xsk_setup_umem(struct net_device dev, struct xdp_umem umem, u16 qid)
	232	{
	233	struct mlx5e_priv *priv = netdev_priv(dev);
	234	struct mlx5e_params *params = &priv->channels.params;
	235	u16 ix;
	236
	237	if (unlikely(!mlx5e_qid_get_ch_if_in_group(params, qid, MLX5E_RQ_GROUP_XSK, &ix)))
	238	return -EINVAL;
	239
	240	return umem ? mlx5e_xsk_enable_umem(priv, umem, ix) :
	241	mlx5e_xsk_disable_umem(priv, ix);
	242	}
	243
	244	int mlx5e_xsk_resize_reuseq(struct xdp_umem *umem, u32 nentries)
	245	{
	246	struct xdp_umem_fq_reuse *reuseq;
	247
	248	reuseq = xsk_reuseq_prepare(nentries);
	249	if (unlikely(!reuseq))
	250	return -ENOMEM;
	251	xsk_reuseq_free(xsk_reuseq_swap(umem, reuseq));
	252
	253	return 0;
	254	}
	255
	256	u16 mlx5e_xsk_first_unused_channel(struct mlx5e_params params, struct mlx5e_xsk xsk)
	257	{
	258	u16 res = xsk->refcnt ? params->num_channels : 0;
	259
	260	while (res) {
	261	if (mlx5e_xsk_get_umem(params, xsk, res - 1))
	262	break;
	263	--res;
	264	}
	265
	266	return res;
	267	}

drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.h+31−0

@@ -0,0 +1,31 @@
	1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
	2	/* Copyright (c) 2019 Mellanox Technologies. */
	3
	4	#ifndef __MLX5_EN_XSK_UMEM_H__
	5	#define __MLX5_EN_XSK_UMEM_H__
	6
	7	#include "en.h"
	8
	9	static inline struct xdp_umem mlx5e_xsk_get_umem(struct mlx5e_params params,
	10	struct mlx5e_xsk *xsk, u16 ix)
	11	{
	12	if (!xsk \|\| !xsk->umems)
	13	return NULL;
	14
	15	if (unlikely(ix >= params->num_channels))
	16	return NULL;
	17
	18	return xsk->umems[ix];
	19	}
	20
	21	struct mlx5e_xsk_param;
	22	void mlx5e_build_xsk_param(struct xdp_umem umem, struct mlx5e_xsk_param xsk);
	23
	24	/* .ndo_bpf callback. */
	25	int mlx5e_xsk_setup_umem(struct net_device dev, struct xdp_umem umem, u16 qid);
	26
	27	int mlx5e_xsk_resize_reuseq(struct xdp_umem *umem, u32 nentries);
	28
	29	u16 mlx5e_xsk_first_unused_channel(struct mlx5e_params params, struct mlx5e_xsk xsk);
	30
	31	#endif /* __MLX5_EN_XSK_UMEM_H__ */

drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c+23−2

 #include "en.h"
 #include "en/port.h"
+#include "en/xsk/umem.h"
 #include "lib/clock.h"
 void mlx5e_ethtool_get_drvinfo(struct mlx5e_priv *priv,
 void mlx5e_ethtool_get_channels(struct mlx5e_priv *priv,
 				struct ethtool_channels *ch)
 {
+	mutex_lock(&priv->state_lock);
 	ch->max_combined   = mlx5e_get_netdev_max_channels(priv->netdev);
 	ch->combined_count = priv->channels.params.num_channels;
+	if (priv->xsk.refcnt) {
+		/* The upper half are XSK queues. */
+		ch->max_combined *= 2;
+		ch->combined_count *= 2;
+	}
+	mutex_unlock(&priv->state_lock);
 }
 static void mlx5e_get_channels(struct net_device *dev,
 int mlx5e_ethtool_set_channels(struct mlx5e_priv *priv,
 			       struct ethtool_channels *ch)
 {
+	struct mlx5e_params *cur_params = &priv->channels.params;
 	unsigned int count = ch->combined_count;
 	struct mlx5e_channels new_channels = {};
 	bool arfs_enabled;
 		return -EINVAL;
 	}
-	if (priv->channels.params.num_channels == count)
+	if (cur_params->num_channels == count)
 		return 0;
 	mutex_lock(&priv->state_lock);
+	/* Don't allow changing the number of channels if there is an active
+	 * XSK, because the numeration of the XSK and regular RQs will change.
+	 */
+	if (priv->xsk.refcnt) {
+		err = -EINVAL;
+		netdev_err(priv->netdev, "%s: AF_XDP is active, cannot change the number of channels\n",
+			   __func__);
+		goto out;
+	}
 	new_channels.params = priv->channels.params;
 	new_channels.params.num_channels = count;
 	if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
-		priv->channels.params = new_channels.params;
+		*cur_params = new_channels.params;
 		if (!netif_is_rxfh_configured(priv->netdev))
 			mlx5e_build_default_indir_rqt(priv->rss_params.indirection_rqt,
 						      MLX5E_INDIR_RQT_SIZE, count);

drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c+14−4

 #include <linux/mlx5/fs.h>
 #include "en.h"
+#include "en/params.h"
+#include "en/xsk/umem.h"
 struct mlx5e_ethtool_rule {
 	struct list_head             list;
 	if (fs->ring_cookie == RX_CLS_FLOW_DISC) {
 		flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP;
 	} else {
+		struct mlx5e_params *params = &priv->channels.params;
+		enum mlx5e_rq_group group;
+		struct mlx5e_tir *tir;
+		u16 ix;
+		mlx5e_qid_get_ch_and_group(params, fs->ring_cookie, &ix, &group);
+		tir = group == MLX5E_RQ_GROUP_XSK ? priv->xsk_tir : priv->direct_tir;
 		dst = kzalloc(sizeof(*dst), GFP_KERNEL);
 		if (!dst) {
 			err = -ENOMEM;
 		}
 		dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR;
-		dst->tir_num = priv->direct_tir[fs->ring_cookie].tirn;
+		dst->tir_num = tir[ix].tirn;
 		flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
 	}
 	if (fs->location >= MAX_NUM_OF_ETHTOOL_RULES)
 		return -ENOSPC;
-	if (fs->ring_cookie >= priv->channels.params.num_channels &&
+	if (fs->ring_cookie != RX_CLS_FLOW_DISC)
-	    fs->ring_cookie != RX_CLS_FLOW_DISC)
+		if (!mlx5e_qid_validate(&priv->channels.params, fs->ring_cookie))
-		return -EINVAL;
+			return -EINVAL;
 	switch (fs->flow_type & ~(FLOW_EXT | FLOW_MAC_EXT)) {
 	case ETHER_FLOW:

drivers/net/ethernet/mellanox/mlx5/core/en_main.c+473−257

@@ -38,6 +38,7 @@ · 37 unmodified lines
38	#include <linux/bpf.h>	38	#include <linux/bpf.h>
39	#include <linux/if_bridge.h>	39	#include <linux/if_bridge.h>
40	#include <net/page_pool.h>	40	#include <net/page_pool.h>
		41	#include <net/xdp_sock.h>
41	#include "eswitch.h"	42	#include "eswitch.h"
42	#include "en.h"	43	#include "en.h"
43	#include "en_tc.h"	44	#include "en_tc.h"
@@ -56,35 +57,10 @@ · 12 unmodified lines
56	#include "en/monitor_stats.h"	57	#include "en/monitor_stats.h"
57	#include "en/reporter.h"	58	#include "en/reporter.h"
58	#include "en/params.h"	59	#include "en/params.h"
59		60	#include "en/xsk/umem.h"
60	struct mlx5e_rq_param {	61	#include "en/xsk/setup.h"
61	u32 rqc[MLX5_ST_SZ_DW(rqc)];	62	#include "en/xsk/rx.h"
62	struct mlx5_wq_param wq;	63	#include "en/xsk/tx.h"
63	struct mlx5e_rq_frags_info frags_info;
64	};
65
66	struct mlx5e_sq_param {
67	u32 sqc[MLX5_ST_SZ_DW(sqc)];
68	struct mlx5_wq_param wq;
69	bool is_mpw;
70	};
71
72	struct mlx5e_cq_param {
73	u32 cqc[MLX5_ST_SZ_DW(cqc)];
74	struct mlx5_wq_param wq;
75	u16 eq_ix;
76	u8 cq_period_mode;
77	};
78
79	struct mlx5e_channel_param {
80	struct mlx5e_rq_param rq;
81	struct mlx5e_sq_param sq;
82	struct mlx5e_sq_param xdp_sq;
83	struct mlx5e_sq_param icosq;
84	struct mlx5e_cq_param rx_cq;
85	struct mlx5e_cq_param tx_cq;
86	struct mlx5e_cq_param icosq_cq;
87	};
88		64
89	bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev)	65	bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev)
90	{	66	{
@@ -114,18 +90,31 @@ void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev, · 23 unmodified lines
114	mlx5_core_info(mdev, "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n",	90	mlx5_core_info(mdev, "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n",
115	params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ,	91	params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ,
116	params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ ?	92	params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ ?
117	BIT(mlx5e_mpwqe_get_log_rq_size(params)) :	93	BIT(mlx5e_mpwqe_get_log_rq_size(params, NULL)) :
118	BIT(params->log_rq_mtu_frames),	94	BIT(params->log_rq_mtu_frames),
119	BIT(mlx5e_mpwqe_get_log_stride_size(mdev, params)),	95	BIT(mlx5e_mpwqe_get_log_stride_size(mdev, params, NULL)),
120	MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS));	96	MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS));
121	}	97	}
122		98
123	bool mlx5e_striding_rq_possible(struct mlx5_core_dev *mdev,	99	bool mlx5e_striding_rq_possible(struct mlx5_core_dev *mdev,
124	struct mlx5e_params *params)	100	struct mlx5e_params *params)
125	{	101	{
126	return mlx5e_check_fragmented_striding_rq_cap(mdev) &&	102	if (!mlx5e_check_fragmented_striding_rq_cap(mdev))
127	!MLX5_IPSEC_DEV(mdev) &&	103	return false;
128	!(params->xdp_prog && !mlx5e_rx_mpwqe_is_linear_skb(mdev, params));	104
		105	if (MLX5_IPSEC_DEV(mdev))
		106	return false;
		107
		108	if (params->xdp_prog) {
		109	/* XSK params are not considered here. If striding RQ is in use,
		110	* and an XSK is being opened, mlx5e_rx_mpwqe_is_linear_skb will
		111	* be called with the known XSK params.
		112	*/
		113	if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL))
		114	return false;
		115	}
		116
		117	return true;
129	}	118	}
130		119
131	void mlx5e_set_rq_type(struct mlx5_core_dev mdev, struct mlx5e_params params)	120	void mlx5e_set_rq_type(struct mlx5_core_dev mdev, struct mlx5e_params params)
@@ -394,6 +383,8 @@ static void mlx5e_free_di_list(struct mlx5e_rq *rq) · 262 unmodified lines
394		383
395	static int mlx5e_alloc_rq(struct mlx5e_channel *c,	384	static int mlx5e_alloc_rq(struct mlx5e_channel *c,
396	struct mlx5e_params *params,	385	struct mlx5e_params *params,
		386	struct mlx5e_xsk_param *xsk,
		387	struct xdp_umem *umem,
397	struct mlx5e_rq_param *rqp,	388	struct mlx5e_rq_param *rqp,
398	struct mlx5e_rq *rq)	389	struct mlx5e_rq *rq)
399	{	390	{
@@ -401,6 +392,8 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, · 1 unmodified line
401	struct mlx5_core_dev *mdev = c->mdev;	392	struct mlx5_core_dev *mdev = c->mdev;
402	void *rqc = rqp->rqc;	393	void *rqc = rqp->rqc;
403	void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);	394	void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
		395	u32 num_xsk_frames = 0;
		396	u32 rq_xdp_ix;
404	u32 pool_size;	397	u32 pool_size;
405	int wq_sz;	398	int wq_sz;
406	int err;	399	int err;
@@ -417,7 +410,13 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, · 10 unmodified lines
417	rq->ix = c->ix;	410	rq->ix = c->ix;
418	rq->mdev = mdev;	411	rq->mdev = mdev;
419	rq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);	412	rq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
420	rq->stats = &c->priv->channel_stats[c->ix].rq;	413	rq->xdpsq = &c->rq_xdpsq;
		414	rq->umem = umem;
		415
		416	if (rq->umem)
		417	rq->stats = &c->priv->channel_stats[c->ix].xskrq;
		418	else
		419	rq->stats = &c->priv->channel_stats[c->ix].rq;
421		420
422	rq->xdp_prog = params->xdp_prog ? bpf_prog_inc(params->xdp_prog) : NULL;	421	rq->xdp_prog = params->xdp_prog ? bpf_prog_inc(params->xdp_prog) : NULL;
423	if (IS_ERR(rq->xdp_prog)) {	422	if (IS_ERR(rq->xdp_prog)) {
@@ -426,12 +425,16 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, · 2 unmodified lines
426	goto err_rq_wq_destroy;	425	goto err_rq_wq_destroy;
427	}	426	}
428		427
429	err = xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq->ix);	428	rq_xdp_ix = rq->ix;
		429	if (xsk)
		430	rq_xdp_ix += params->num_channels * MLX5E_RQ_GROUP_XSK;
		431	err = xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq_xdp_ix);
430	if (err < 0)	432	if (err < 0)
431	goto err_rq_wq_destroy;	433	goto err_rq_wq_destroy;
432		434
433	rq->buff.map_dir = rq->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;	435	rq->buff.map_dir = rq->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
434	rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params);	436	rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, xsk);
		437	rq->buff.umem_headroom = xsk ? xsk->headroom : 0;
435	pool_size = 1 << params->log_rq_mtu_frames;	438	pool_size = 1 << params->log_rq_mtu_frames;
436		439
437	switch (rq->wq_type) {	440	switch (rq->wq_type) {
@@ -445,7 +448,12 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, · 7 unmodified lines
445		448
446	wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);	449	wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
447		450
448	pool_size = MLX5_MPWRQ_PAGES_PER_WQE << mlx5e_mpwqe_get_log_rq_size(params);	451	if (xsk)
		452	num_xsk_frames = wq_sz <<
		453	mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk);
		454
		455	pool_size = MLX5_MPWRQ_PAGES_PER_WQE <<
		456	mlx5e_mpwqe_get_log_rq_size(params, xsk);
449		457
450	rq->post_wqes = mlx5e_post_rx_mpwqes;	458	rq->post_wqes = mlx5e_post_rx_mpwqes;
451	rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe;	459	rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe;
@@ -464,12 +472,15 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, · 12 unmodified lines
464	goto err_rq_wq_destroy;	472	goto err_rq_wq_destroy;
465	}	473	}
466		474
467	rq->mpwqe.skb_from_cqe_mpwrq =	475	rq->mpwqe.skb_from_cqe_mpwrq = xsk ?
468	mlx5e_rx_mpwqe_is_linear_skb(mdev, params) ?	476	mlx5e_xsk_skb_from_cqe_mpwrq_linear :
469	mlx5e_skb_from_cqe_mpwrq_linear :	477	mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL) ?
470	mlx5e_skb_from_cqe_mpwrq_nonlinear;	478	mlx5e_skb_from_cqe_mpwrq_linear :
471	rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params);	479	mlx5e_skb_from_cqe_mpwrq_nonlinear;
472	rq->mpwqe.num_strides = BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params));	480
		481	rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
		482	rq->mpwqe.num_strides =
		483	BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk));
473		484
474	err = mlx5e_create_rq_umr_mkey(mdev, rq);	485	err = mlx5e_create_rq_umr_mkey(mdev, rq);
475	if (err)	486	if (err)
@@ -490,6 +501,9 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, · 14 unmodified lines
490		501
491	wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq);	502	wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq);
492		503
		504	if (xsk)
		505	num_xsk_frames = wq_sz << rq->wqe.info.log_num_frags;
		506
493	rq->wqe.info = rqp->frags_info;	507	rq->wqe.info = rqp->frags_info;
494	rq->wqe.frags =	508	rq->wqe.frags =
495	kvzalloc_node(array_size(sizeof(*rq->wqe.frags),	509	kvzalloc_node(array_size(sizeof(*rq->wqe.frags),
@@ -503,6 +517,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, · 7 unmodified lines
503	err = mlx5e_init_di_list(rq, wq_sz, c->cpu);	517	err = mlx5e_init_di_list(rq, wq_sz, c->cpu);
504	if (err)	518	if (err)
505	goto err_free;	519	goto err_free;
		520
506	rq->post_wqes = mlx5e_post_rx_wqes;	521	rq->post_wqes = mlx5e_post_rx_wqes;
507	rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;	522	rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;
508		523
@@ -518,37 +533,53 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, · 9 unmodified lines
518	goto err_free;	533	goto err_free;
519	}	534	}
520		535
521	rq->wqe.skb_from_cqe = mlx5e_rx_is_linear_skb(params) ?	536	rq->wqe.skb_from_cqe = xsk ?
522	mlx5e_skb_from_cqe_linear :	537	mlx5e_xsk_skb_from_cqe_linear :
523	mlx5e_skb_from_cqe_nonlinear;	538	mlx5e_rx_is_linear_skb(params, NULL) ?
		539	mlx5e_skb_from_cqe_linear :
		540	mlx5e_skb_from_cqe_nonlinear;
524	rq->mkey_be = c->mkey_be;	541	rq->mkey_be = c->mkey_be;
525	}	542	}
526		543
527	/* Create a page_pool and register it with rxq */	544	if (xsk) {
528	pp_params.order = 0;	545	err = mlx5e_xsk_resize_reuseq(umem, num_xsk_frames);
529	pp_params.flags = 0; /* No-internal DMA mapping in page_pool */	546	if (unlikely(err)) {
530	pp_params.pool_size = pool_size;	547	mlx5_core_err(mdev, "Unable to allocate the Reuse Ring for %u frames\n",
531	pp_params.nid = cpu_to_node(c->cpu);	548	num_xsk_frames);
532	pp_params.dev = c->pdev;	549	goto err_free;
533	pp_params.dma_dir = rq->buff.map_dir;	550	}
534		551
535	/* page_pool can be used even when there is no rq->xdp_prog,	552	rq->zca.free = mlx5e_xsk_zca_free;
536	* given page_pool does not handle DMA mapping there is no	553	err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
537	* required state to clear. And page_pool gracefully handle	554	MEM_TYPE_ZERO_COPY,
538	* elevated refcnt.	555	&rq->zca);
539	*/	556	} else {
540	rq->page_pool = page_pool_create(&pp_params);	557	/* Create a page_pool and register it with rxq */
541	if (IS_ERR(rq->page_pool)) {	558	pp_params.order = 0;
542	err = PTR_ERR(rq->page_pool);	559	pp_params.flags = 0; /* No-internal DMA mapping in page_pool */
543	rq->page_pool = NULL;	560	pp_params.pool_size = pool_size;
544	goto err_free;	561	pp_params.nid = cpu_to_node(c->cpu);
		562	pp_params.dev = c->pdev;
		563	pp_params.dma_dir = rq->buff.map_dir;
		564
		565	/* page_pool can be used even when there is no rq->xdp_prog,
		566	* given page_pool does not handle DMA mapping there is no
		567	* required state to clear. And page_pool gracefully handle
		568	* elevated refcnt.
		569	*/
		570	rq->page_pool = page_pool_create(&pp_params);
		571	if (IS_ERR(rq->page_pool)) {
		572	err = PTR_ERR(rq->page_pool);
		573	rq->page_pool = NULL;
		574	goto err_free;
		575	}
		576	err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
		577	MEM_TYPE_PAGE_POOL, rq->page_pool);
		578	if (err)
		579	page_pool_free(rq->page_pool);
545	}	580	}
546	err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,	581	if (err)
547	MEM_TYPE_PAGE_POOL, rq->page_pool);
548	if (err) {
549	page_pool_free(rq->page_pool);
550	goto err_free;	582	goto err_free;
551	}
552		583
553	for (i = 0; i < wq_sz; i++) {	584	for (i = 0; i < wq_sz; i++) {
554	if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {	585	if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
@@ -639,7 +670,11 @@ static void mlx5e_free_rq(struct mlx5e_rq *rq) · 84 unmodified lines
639	i = (i + 1) & (MLX5E_CACHE_SIZE - 1)) {	670	i = (i + 1) & (MLX5E_CACHE_SIZE - 1)) {
640	struct mlx5e_dma_info *dma_info = &rq->page_cache.page_cache[i];	671	struct mlx5e_dma_info *dma_info = &rq->page_cache.page_cache[i];
641		672
642	mlx5e_page_release(rq, dma_info, false);	673	/* With AF_XDP, page_cache is not used, so this loop is not
		674	* entered, and it's safe to call mlx5e_page_release_dynamic
		675	* directly.
		676	*/
		677	mlx5e_page_release_dynamic(rq, dma_info, false);
643	}	678	}
644		679
645	xdp_rxq_info_unreg(&rq->xdp_rxq);	680	xdp_rxq_info_unreg(&rq->xdp_rxq);
@@ -776,7 +811,7 @@ static void mlx5e_destroy_rq(struct mlx5e_rq *rq) · 130 unmodified lines
776	mlx5_core_destroy_rq(rq->mdev, rq->rqn);	811	mlx5_core_destroy_rq(rq->mdev, rq->rqn);
777	}	812	}
778		813
779	static int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time)	814	int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time)
780	{	815	{
781	unsigned long exp_time = jiffies + msecs_to_jiffies(wait_time);	816	unsigned long exp_time = jiffies + msecs_to_jiffies(wait_time);
782	struct mlx5e_channel *c = rq->channel;	817	struct mlx5e_channel *c = rq->channel;
@@ -834,14 +869,13 @@ static void mlx5e_free_rx_descs(struct mlx5e_rq *rq) · 51 unmodified lines
834		869
835	}	870	}
836		871
837	static int mlx5e_open_rq(struct mlx5e_channel *c,	872	int mlx5e_open_rq(struct mlx5e_channel c, struct mlx5e_params params,
838	struct mlx5e_params *params,	873	struct mlx5e_rq_param param, struct mlx5e_xsk_param xsk,
839	struct mlx5e_rq_param *param,	874	struct xdp_umem umem, struct mlx5e_rq rq)
840	struct mlx5e_rq *rq)
841	{	875	{
842	int err;	876	int err;
843		877
844	err = mlx5e_alloc_rq(c, params, param, rq);	878	err = mlx5e_alloc_rq(c, params, xsk, umem, param, rq);
845	if (err)	879	if (err)
846	return err;	880	return err;
847		881
@@ -879,13 +913,13 @@ static void mlx5e_activate_rq(struct mlx5e_rq *rq) · 31 unmodified lines
879	mlx5e_trigger_irq(&rq->channel->icosq);	913	mlx5e_trigger_irq(&rq->channel->icosq);
880	}	914	}
881		915
882	static void mlx5e_deactivate_rq(struct mlx5e_rq *rq)	916	void mlx5e_deactivate_rq(struct mlx5e_rq *rq)
883	{	917	{
884	clear_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);	918	clear_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
885	napi_synchronize(&rq->channel->napi); /* prevent mlx5e_post_rx_wqes */	919	napi_synchronize(&rq->channel->napi); /* prevent mlx5e_post_rx_wqes */
886	}	920	}
887		921
888	static void mlx5e_close_rq(struct mlx5e_rq *rq)	922	void mlx5e_close_rq(struct mlx5e_rq *rq)
889	{	923	{
890	cancel_work_sync(&rq->dim.work);	924	cancel_work_sync(&rq->dim.work);
891	mlx5e_destroy_rq(rq);	925	mlx5e_destroy_rq(rq);
@@ -938,6 +972,7 @@ static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq *sq, int numa) · 46 unmodified lines
938		972
939	static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c,	973	static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c,
940	struct mlx5e_params *params,	974	struct mlx5e_params *params,
		975	struct xdp_umem *umem,
941	struct mlx5e_sq_param *param,	976	struct mlx5e_sq_param *param,
942	struct mlx5e_xdpsq *sq,	977	struct mlx5e_xdpsq *sq,
943	bool is_redirect)	978	bool is_redirect)
@@ -953,9 +988,13 @@ static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c, · 9 unmodified lines
953	sq->uar_map = mdev->mlx5e_res.bfreg.map;	988	sq->uar_map = mdev->mlx5e_res.bfreg.map;
954	sq->min_inline_mode = params->tx_min_inline_mode;	989	sq->min_inline_mode = params->tx_min_inline_mode;
955	sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);	990	sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
956	sq->stats = is_redirect ?	991	sq->umem = umem;
957	&c->priv->channel_stats[c->ix].xdpsq :	992
958	&c->priv->channel_stats[c->ix].rq_xdpsq;	993	sq->stats = sq->umem ?
		994	&c->priv->channel_stats[c->ix].xsksq :
		995	is_redirect ?
		996	&c->priv->channel_stats[c->ix].xdpsq :
		997	&c->priv->channel_stats[c->ix].rq_xdpsq;
959		998
960	param->wq.db_numa_node = cpu_to_node(c->cpu);	999	param->wq.db_numa_node = cpu_to_node(c->cpu);
961	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);	1000	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
@@ -1335,10 +1374,8 @@ static void mlx5e_tx_err_cqe_work(struct work_struct *recover_work) · 373 unmodified lines
1335	mlx5e_tx_reporter_err_cqe(sq);	1374	mlx5e_tx_reporter_err_cqe(sq);
1336	}	1375	}
1337		1376
1338	static int mlx5e_open_icosq(struct mlx5e_channel *c,	1377	int mlx5e_open_icosq(struct mlx5e_channel c, struct mlx5e_params params,
1339	struct mlx5e_params *params,	1378	struct mlx5e_sq_param param, struct mlx5e_icosq sq)
1340	struct mlx5e_sq_param *param,
1341	struct mlx5e_icosq *sq)
1342	{	1379	{
1343	struct mlx5e_create_sq_param csp = {};	1380	struct mlx5e_create_sq_param csp = {};
1344	int err;	1381	int err;
@@ -1364,7 +1401,7 @@ err_free_icosq: · 19 unmodified lines
1364	return err;	1401	return err;
1365	}	1402	}
1366		1403
1367	static void mlx5e_close_icosq(struct mlx5e_icosq *sq)	1404	void mlx5e_close_icosq(struct mlx5e_icosq *sq)
1368	{	1405	{
1369	struct mlx5e_channel *c = sq->channel;	1406	struct mlx5e_channel *c = sq->channel;
1370		1407
@@ -1375,16 +1412,14 @@ static void mlx5e_close_icosq(struct mlx5e_icosq *sq) · 4 unmodified lines
1375	mlx5e_free_icosq(sq);	1412	mlx5e_free_icosq(sq);
1376	}	1413	}
1377		1414
1378	static int mlx5e_open_xdpsq(struct mlx5e_channel *c,	1415	int mlx5e_open_xdpsq(struct mlx5e_channel c, struct mlx5e_params params,
1379	struct mlx5e_params *params,	1416	struct mlx5e_sq_param param, struct xdp_umem umem,
1380	struct mlx5e_sq_param *param,	1417	struct mlx5e_xdpsq *sq, bool is_redirect)
1381	struct mlx5e_xdpsq *sq,
1382	bool is_redirect)
1383	{	1418	{
1384	struct mlx5e_create_sq_param csp = {};	1419	struct mlx5e_create_sq_param csp = {};
1385	int err;	1420	int err;
1386		1421
1387	err = mlx5e_alloc_xdpsq(c, params, param, sq, is_redirect);	1422	err = mlx5e_alloc_xdpsq(c, params, umem, param, sq, is_redirect);
1388	if (err)	1423	if (err)
1389	return err;	1424	return err;
1390		1425
@@ -1438,7 +1473,7 @@ err_free_xdpsq: · 47 unmodified lines
1438	return err;	1473	return err;
1439	}	1474	}
1440		1475
1441	static void mlx5e_close_xdpsq(struct mlx5e_xdpsq sq, struct mlx5e_rq rq)	1476	void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq)
1442	{	1477	{
1443	struct mlx5e_channel *c = sq->channel;	1478	struct mlx5e_channel *c = sq->channel;
1444		1479
@@ -1446,7 +1481,7 @@ static void mlx5e_close_xdpsq(struct mlx5e_xdpsq sq, struct mlx5e_rq rq) · 1 unmodified line
1446	napi_synchronize(&c->napi);	1481	napi_synchronize(&c->napi);
1447		1482
1448	mlx5e_destroy_sq(c->mdev, sq->sqn);	1483	mlx5e_destroy_sq(c->mdev, sq->sqn);
1449	mlx5e_free_xdpsq_descs(sq, rq);	1484	mlx5e_free_xdpsq_descs(sq);
1450	mlx5e_free_xdpsq(sq);	1485	mlx5e_free_xdpsq(sq);
1451	}	1486	}
1452		1487
@@ -1567,10 +1602,8 @@ static void mlx5e_destroy_cq(struct mlx5e_cq *cq) · 114 unmodified lines
1567	mlx5_core_destroy_cq(cq->mdev, &cq->mcq);	1602	mlx5_core_destroy_cq(cq->mdev, &cq->mcq);
1568	}	1603	}
1569		1604
1570	static int mlx5e_open_cq(struct mlx5e_channel *c,	1605	int mlx5e_open_cq(struct mlx5e_channel *c, struct dim_cq_moder moder,
1571	struct dim_cq_moder moder,	1606	struct mlx5e_cq_param param, struct mlx5e_cq cq)
1572	struct mlx5e_cq_param *param,
1573	struct mlx5e_cq *cq)
1574	{	1607	{
1575	struct mlx5_core_dev *mdev = c->mdev;	1608	struct mlx5_core_dev *mdev = c->mdev;
1576	int err;	1609	int err;
@@ -1593,7 +1626,7 @@ err_free_cq: · 16 unmodified lines
1593	return err;	1626	return err;
1594	}	1627	}
1595		1628
1596	static void mlx5e_close_cq(struct mlx5e_cq *cq)	1629	void mlx5e_close_cq(struct mlx5e_cq *cq)
1597	{	1630	{
1598	mlx5e_destroy_cq(cq);	1631	mlx5e_destroy_cq(cq);
1599	mlx5e_free_cq(cq);	1632	mlx5e_free_cq(cq);
@@ -1767,49 +1800,16 @@ static void mlx5e_free_xps_cpumask(struct mlx5e_channel *c) · 167 unmodified lines
1767	free_cpumask_var(c->xps_cpumask);	1800	free_cpumask_var(c->xps_cpumask);
1768	}	1801	}
1769		1802
1770	static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,	1803	static int mlx5e_open_queues(struct mlx5e_channel *c,
1771	struct mlx5e_params *params,	1804	struct mlx5e_params *params,
1772	struct mlx5e_channel_param *cparam,	1805	struct mlx5e_channel_param *cparam)
1773	struct mlx5e_channel **cp)
1774	{	1806	{
1775	int cpu = cpumask_first(mlx5_comp_irq_get_affinity_mask(priv->mdev, ix));
1776	struct dim_cq_moder icocq_moder = {0, 0};	1807	struct dim_cq_moder icocq_moder = {0, 0};
1777	struct net_device *netdev = priv->netdev;
1778	struct mlx5e_channel *c;
1779	unsigned int irq;
1780	int err;	1808	int err;
1781	int eqn;
1782
1783	err = mlx5_vector2eqn(priv->mdev, ix, &eqn, &irq);
1784	if (err)
1785	return err;
1786
1787	c = kvzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu));
1788	if (!c)
1789	return -ENOMEM;
1790
1791	c->priv = priv;
1792	c->mdev = priv->mdev;
1793	c->tstamp = &priv->tstamp;
1794	c->ix = ix;
1795	c->cpu = cpu;
1796	c->pdev = priv->mdev->device;
1797	c->netdev = priv->netdev;
1798	c->mkey_be = cpu_to_be32(priv->mdev->mlx5e_res.mkey.key);
1799	c->num_tc = params->num_tc;
1800	c->xdp = !!params->xdp_prog;
1801	c->stats = &priv->channel_stats[ix].ch;
1802	c->irq_desc = irq_to_desc(irq);
1803
1804	err = mlx5e_alloc_xps_cpumask(c, params);
1805	if (err)
1806	goto err_free_channel;
1807
1808	netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);
1809		1809
1810	err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->icosq.cq);	1810	err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->icosq.cq);
1811	if (err)	1811	if (err)
1812	goto err_napi_del;	1812	return err;
1813		1813
1814	err = mlx5e_open_tx_cqs(c, params, cparam);	1814	err = mlx5e_open_tx_cqs(c, params, cparam);
1815	if (err)	1815	if (err)
@@ -1825,7 +1825,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, · 9 unmodified lines
1825		1825
1826	/* XDP SQ CQ params are same as normal TXQ sq CQ params */	1826	/* XDP SQ CQ params are same as normal TXQ sq CQ params */
1827	err = c->xdp ? mlx5e_open_cq(c, params->tx_cq_moderation,	1827	err = c->xdp ? mlx5e_open_cq(c, params->tx_cq_moderation,
1828	&cparam->tx_cq, &c->rq.xdpsq.cq) : 0;	1828	&cparam->tx_cq, &c->rq_xdpsq.cq) : 0;
1829	if (err)	1829	if (err)
1830	goto err_close_rx_cq;	1830	goto err_close_rx_cq;
1831		1831
@@ -1839,20 +1839,21 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, · 7 unmodified lines
1839	if (err)	1839	if (err)
1840	goto err_close_icosq;	1840	goto err_close_icosq;
1841		1841
1842	err = c->xdp ? mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, &c->rq.xdpsq, false) : 0;	1842	if (c->xdp) {
1843	if (err)	1843	err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL,
1844	goto err_close_sqs;	1844	&c->rq_xdpsq, false);
		1845	if (err)
		1846	goto err_close_sqs;
		1847	}
1845		1848
1846	err = mlx5e_open_rq(c, params, &cparam->rq, &c->rq);	1849	err = mlx5e_open_rq(c, params, &cparam->rq, NULL, NULL, &c->rq);
1847	if (err)	1850	if (err)
1848	goto err_close_xdp_sq;	1851	goto err_close_xdp_sq;
1849		1852
1850	err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, &c->xdpsq, true);	1853	err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL, &c->xdpsq, true);
1851	if (err)	1854	if (err)
1852	goto err_close_rq;	1855	goto err_close_rq;
1853		1856
1854	*cp = c;
1855
1856	return 0;	1857	return 0;
1857		1858
1858	err_close_rq:	1859	err_close_rq:
@@ -1860,7 +1861,7 @@ err_close_rq: · 1 unmodified line
1860		1861
1861	err_close_xdp_sq:	1862	err_close_xdp_sq:
1862	if (c->xdp)	1863	if (c->xdp)
1863	mlx5e_close_xdpsq(&c->rq.xdpsq, &c->rq);	1864	mlx5e_close_xdpsq(&c->rq_xdpsq);
1864		1865
1865	err_close_sqs:	1866	err_close_sqs:
1866	mlx5e_close_sqs(c);	1867	mlx5e_close_sqs(c);
@@ -1870,8 +1871,9 @@ err_close_icosq: · 3 unmodified lines
1870		1871
1871	err_disable_napi:	1872	err_disable_napi:
1872	napi_disable(&c->napi);	1873	napi_disable(&c->napi);
		1874
1873	if (c->xdp)	1875	if (c->xdp)
1874	mlx5e_close_cq(&c->rq.xdpsq.cq);	1876	mlx5e_close_cq(&c->rq_xdpsq.cq);
1875		1877
1876	err_close_rx_cq:	1878	err_close_rx_cq:
1877	mlx5e_close_cq(&c->rq.cq);	1879	mlx5e_close_cq(&c->rq.cq);
@@ -1885,6 +1887,85 @@ err_close_tx_cqs: · 7 unmodified lines
1885	err_close_icosq_cq:	1887	err_close_icosq_cq:
1886	mlx5e_close_cq(&c->icosq.cq);	1888	mlx5e_close_cq(&c->icosq.cq);
1887		1889
		1890	return err;
		1891	}
		1892
		1893	static void mlx5e_close_queues(struct mlx5e_channel *c)
		1894	{
		1895	mlx5e_close_xdpsq(&c->xdpsq);
		1896	mlx5e_close_rq(&c->rq);
		1897	if (c->xdp)
		1898	mlx5e_close_xdpsq(&c->rq_xdpsq);
		1899	mlx5e_close_sqs(c);
		1900	mlx5e_close_icosq(&c->icosq);
		1901	napi_disable(&c->napi);
		1902	if (c->xdp)
		1903	mlx5e_close_cq(&c->rq_xdpsq.cq);
		1904	mlx5e_close_cq(&c->rq.cq);
		1905	mlx5e_close_cq(&c->xdpsq.cq);
		1906	mlx5e_close_tx_cqs(c);
		1907	mlx5e_close_cq(&c->icosq.cq);
		1908	}
		1909
		1910	static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
		1911	struct mlx5e_params *params,
		1912	struct mlx5e_channel_param *cparam,
		1913	struct xdp_umem *umem,
		1914	struct mlx5e_channel **cp)
		1915	{
		1916	int cpu = cpumask_first(mlx5_comp_irq_get_affinity_mask(priv->mdev, ix));
		1917	struct net_device *netdev = priv->netdev;
		1918	struct mlx5e_xsk_param xsk;
		1919	struct mlx5e_channel *c;
		1920	unsigned int irq;
		1921	int err;
		1922	int eqn;
		1923
		1924	err = mlx5_vector2eqn(priv->mdev, ix, &eqn, &irq);
		1925	if (err)
		1926	return err;
		1927
		1928	c = kvzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu));
		1929	if (!c)
		1930	return -ENOMEM;
		1931
		1932	c->priv = priv;
		1933	c->mdev = priv->mdev;
		1934	c->tstamp = &priv->tstamp;
		1935	c->ix = ix;
		1936	c->cpu = cpu;
		1937	c->pdev = priv->mdev->device;
		1938	c->netdev = priv->netdev;
		1939	c->mkey_be = cpu_to_be32(priv->mdev->mlx5e_res.mkey.key);
		1940	c->num_tc = params->num_tc;
		1941	c->xdp = !!params->xdp_prog;
		1942	c->stats = &priv->channel_stats[ix].ch;
		1943	c->irq_desc = irq_to_desc(irq);
		1944
		1945	err = mlx5e_alloc_xps_cpumask(c, params);
		1946	if (err)
		1947	goto err_free_channel;
		1948
		1949	netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);
		1950
		1951	err = mlx5e_open_queues(c, params, cparam);
		1952	if (unlikely(err))
		1953	goto err_napi_del;
		1954
		1955	if (umem) {
		1956	mlx5e_build_xsk_param(umem, &xsk);
		1957	err = mlx5e_open_xsk(priv, params, &xsk, umem, c);
		1958	if (unlikely(err))
		1959	goto err_close_queues;
		1960	}
		1961
		1962	*cp = c;
		1963
		1964	return 0;
		1965
		1966	err_close_queues:
		1967	mlx5e_close_queues(c);
		1968
1888	err_napi_del:	1969	err_napi_del:
1889	netif_napi_del(&c->napi);	1970	netif_napi_del(&c->napi);
1890	mlx5e_free_xps_cpumask(c);	1971	mlx5e_free_xps_cpumask(c);
@@ -1903,12 +1984,18 @@ static void mlx5e_activate_channel(struct mlx5e_channel *c) · 12 unmodified lines
1903	mlx5e_activate_txqsq(&c->sq[tc]);	1984	mlx5e_activate_txqsq(&c->sq[tc]);
1904	mlx5e_activate_rq(&c->rq);	1985	mlx5e_activate_rq(&c->rq);
1905	netif_set_xps_queue(c->netdev, c->xps_cpumask, c->ix);	1986	netif_set_xps_queue(c->netdev, c->xps_cpumask, c->ix);
		1987
		1988	if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
		1989	mlx5e_activate_xsk(c);
1906	}	1990	}
1907		1991
1908	static void mlx5e_deactivate_channel(struct mlx5e_channel *c)	1992	static void mlx5e_deactivate_channel(struct mlx5e_channel *c)
1909	{	1993	{
1910	int tc;	1994	int tc;
1911		1995
		1996	if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
		1997	mlx5e_deactivate_xsk(c);
		1998
1912	mlx5e_deactivate_rq(&c->rq);	1999	mlx5e_deactivate_rq(&c->rq);
1913	for (tc = 0; tc < c->num_tc; tc++)	2000	for (tc = 0; tc < c->num_tc; tc++)
1914	mlx5e_deactivate_txqsq(&c->sq[tc]);	2001	mlx5e_deactivate_txqsq(&c->sq[tc]);
@@ -1916,19 +2003,9 @@ static void mlx5e_deactivate_channel(struct mlx5e_channel *c) · 1 unmodified line
1916		2003
1917	static void mlx5e_close_channel(struct mlx5e_channel *c)	2004	static void mlx5e_close_channel(struct mlx5e_channel *c)
1918	{	2005	{
1919	mlx5e_close_xdpsq(&c->xdpsq, NULL);	2006	if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
1920	mlx5e_close_rq(&c->rq);	2007	mlx5e_close_xsk(c);
1921	if (c->xdp)	2008	mlx5e_close_queues(c);
1922	mlx5e_close_xdpsq(&c->rq.xdpsq, &c->rq);
1923	mlx5e_close_sqs(c);
1924	mlx5e_close_icosq(&c->icosq);
1925	napi_disable(&c->napi);
1926	if (c->xdp)
1927	mlx5e_close_cq(&c->rq.xdpsq.cq);
1928	mlx5e_close_cq(&c->rq.cq);
1929	mlx5e_close_cq(&c->xdpsq.cq);
1930	mlx5e_close_tx_cqs(c);
1931	mlx5e_close_cq(&c->icosq.cq);
1932	netif_napi_del(&c->napi);	2009	netif_napi_del(&c->napi);
1933	mlx5e_free_xps_cpumask(c);	2010	mlx5e_free_xps_cpumask(c);
1934		2011
@@ -1939,6 +2016,7 @@ static void mlx5e_close_channel(struct mlx5e_channel *c) · 4 unmodified lines
1939		2016
1940	static void mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev,	2017	static void mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev,
1941	struct mlx5e_params *params,	2018	struct mlx5e_params *params,
		2019	struct mlx5e_xsk_param *xsk,
1942	struct mlx5e_rq_frags_info *info)	2020	struct mlx5e_rq_frags_info *info)
1943	{	2021	{
1944	u32 byte_count = MLX5E_SW2HW_MTU(params, params->sw_mtu);	2022	u32 byte_count = MLX5E_SW2HW_MTU(params, params->sw_mtu);
@@ -1951,10 +2029,10 @@ static void mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev, · 6 unmodified lines
1951	byte_count += MLX5E_METADATA_ETHER_LEN;	2029	byte_count += MLX5E_METADATA_ETHER_LEN;
1952	#endif	2030	#endif
1953		2031
1954	if (mlx5e_rx_is_linear_skb(params)) {	2032	if (mlx5e_rx_is_linear_skb(params, xsk)) {
1955	int frag_stride;	2033	int frag_stride;
1956		2034
1957	frag_stride = mlx5e_rx_get_linear_frag_sz(params);	2035	frag_stride = mlx5e_rx_get_linear_frag_sz(params, xsk);
1958	frag_stride = roundup_pow_of_two(frag_stride);	2036	frag_stride = roundup_pow_of_two(frag_stride);
1959		2037
1960	info->arr[0].frag_size = byte_count;	2038	info->arr[0].frag_size = byte_count;
@@ -2012,9 +2090,10 @@ static u8 mlx5e_get_rq_log_wq_sz(void *rqc) · 51 unmodified lines
2012	return MLX5_GET(wq, wq, log_wq_sz);	2090	return MLX5_GET(wq, wq, log_wq_sz);
2013	}	2091	}
2014		2092
2015	static void mlx5e_build_rq_param(struct mlx5e_priv *priv,	2093	void mlx5e_build_rq_param(struct mlx5e_priv *priv,
2016	struct mlx5e_params *params,	2094	struct mlx5e_params *params,
2017	struct mlx5e_rq_param *param)	2095	struct mlx5e_xsk_param *xsk,
		2096	struct mlx5e_rq_param *param)
2018	{	2097	{
2019	struct mlx5_core_dev *mdev = priv->mdev;	2098	struct mlx5_core_dev *mdev = priv->mdev;
2020	void *rqc = param->rqc;	2099	void *rqc = param->rqc;
@@ -2024,16 +2103,16 @@ static void mlx5e_build_rq_param(struct mlx5e_priv *priv, · 3 unmodified lines
2024	switch (params->rq_wq_type) {	2103	switch (params->rq_wq_type) {
2025	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:	2104	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
2026	MLX5_SET(wq, wq, log_wqe_num_of_strides,	2105	MLX5_SET(wq, wq, log_wqe_num_of_strides,
2027	mlx5e_mpwqe_get_log_num_strides(mdev, params) -	2106	mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk) -
2028	MLX5_MPWQE_LOG_NUM_STRIDES_BASE);	2107	MLX5_MPWQE_LOG_NUM_STRIDES_BASE);
2029	MLX5_SET(wq, wq, log_wqe_stride_size,	2108	MLX5_SET(wq, wq, log_wqe_stride_size,
2030	mlx5e_mpwqe_get_log_stride_size(mdev, params) -	2109	mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk) -
2031	MLX5_MPWQE_LOG_STRIDE_SZ_BASE);	2110	MLX5_MPWQE_LOG_STRIDE_SZ_BASE);
2032	MLX5_SET(wq, wq, log_wq_sz, mlx5e_mpwqe_get_log_rq_size(params));	2111	MLX5_SET(wq, wq, log_wq_sz, mlx5e_mpwqe_get_log_rq_size(params, xsk));
2033	break;	2112	break;
2034	default: /* MLX5_WQ_TYPE_CYCLIC */	2113	default: /* MLX5_WQ_TYPE_CYCLIC */
2035	MLX5_SET(wq, wq, log_wq_sz, params->log_rq_mtu_frames);	2114	MLX5_SET(wq, wq, log_wq_sz, params->log_rq_mtu_frames);
2036	mlx5e_build_rq_frags_info(mdev, params, &param->frags_info);	2115	mlx5e_build_rq_frags_info(mdev, params, xsk, &param->frags_info);
2037	ndsegs = param->frags_info.num_frags;	2116	ndsegs = param->frags_info.num_frags;
2038	}	2117	}
2039		2118
@@ -2064,8 +2143,8 @@ static void mlx5e_build_drop_rq_param(struct mlx5e_priv *priv, · 24 unmodified lines
2064	param->wq.buf_numa_node = dev_to_node(mdev->device);	2143	param->wq.buf_numa_node = dev_to_node(mdev->device);
2065	}	2144	}
2066		2145
2067	static void mlx5e_build_sq_param_common(struct mlx5e_priv *priv,	2146	void mlx5e_build_sq_param_common(struct mlx5e_priv *priv,
2068	struct mlx5e_sq_param *param)	2147	struct mlx5e_sq_param *param)
2069	{	2148	{
2070	void *sqc = param->sqc;	2149	void *sqc = param->sqc;
2071	void *wq = MLX5_ADDR_OF(sqc, sqc, wq);	2150	void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
@@ -2101,9 +2180,10 @@ static void mlx5e_build_common_cq_param(struct mlx5e_priv *priv, · 29 unmodified lines
2101	MLX5_SET(cqc, cqc, cqe_sz, CQE_STRIDE_128_PAD);	2180	MLX5_SET(cqc, cqc, cqe_sz, CQE_STRIDE_128_PAD);
2102	}	2181	}
2103		2182
2104	static void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,	2183	void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
2105	struct mlx5e_params *params,	2184	struct mlx5e_params *params,
2106	struct mlx5e_cq_param *param)	2185	struct mlx5e_xsk_param *xsk,
		2186	struct mlx5e_cq_param *param)
2107	{	2187	{
2108	struct mlx5_core_dev *mdev = priv->mdev;	2188	struct mlx5_core_dev *mdev = priv->mdev;
2109	void *cqc = param->cqc;	2189	void *cqc = param->cqc;
@@ -2111,8 +2191,8 @@ static void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv, · 1 unmodified line
2111		2191
2112	switch (params->rq_wq_type) {	2192	switch (params->rq_wq_type) {
2113	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:	2193	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
2114	log_cq_size = mlx5e_mpwqe_get_log_rq_size(params) +	2194	log_cq_size = mlx5e_mpwqe_get_log_rq_size(params, xsk) +
2115	mlx5e_mpwqe_get_log_num_strides(mdev, params);	2195	mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk);
2116	break;	2196	break;
2117	default: /* MLX5_WQ_TYPE_CYCLIC */	2197	default: /* MLX5_WQ_TYPE_CYCLIC */
2118	log_cq_size = params->log_rq_mtu_frames;	2198	log_cq_size = params->log_rq_mtu_frames;
@@ -2128,9 +2208,9 @@ static void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv, · 9 unmodified lines
2128	param->cq_period_mode = params->rx_cq_moderation.cq_period_mode;	2208	param->cq_period_mode = params->rx_cq_moderation.cq_period_mode;
2129	}	2209	}
2130		2210
2131	static void mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,	2211	void mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
2132	struct mlx5e_params *params,	2212	struct mlx5e_params *params,
2133	struct mlx5e_cq_param *param)	2213	struct mlx5e_cq_param *param)
2134	{	2214	{
2135	void *cqc = param->cqc;	2215	void *cqc = param->cqc;
2136		2216
@@ -2140,9 +2220,9 @@ static void mlx5e_build_tx_cq_param(struct mlx5e_priv *priv, · 3 unmodified lines
2140	param->cq_period_mode = params->tx_cq_moderation.cq_period_mode;	2220	param->cq_period_mode = params->tx_cq_moderation.cq_period_mode;
2141	}	2221	}
2142		2222
2143	static void mlx5e_build_ico_cq_param(struct mlx5e_priv *priv,	2223	void mlx5e_build_ico_cq_param(struct mlx5e_priv *priv,
2144	u8 log_wq_size,	2224	u8 log_wq_size,
2145	struct mlx5e_cq_param *param)	2225	struct mlx5e_cq_param *param)
2146	{	2226	{
2147	void *cqc = param->cqc;	2227	void *cqc = param->cqc;
2148		2228
@@ -2153,9 +2233,9 @@ static void mlx5e_build_ico_cq_param(struct mlx5e_priv *priv, · 4 unmodified lines
2153	param->cq_period_mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;	2233	param->cq_period_mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
2154	}	2234	}
2155		2235
2156	static void mlx5e_build_icosq_param(struct mlx5e_priv *priv,	2236	void mlx5e_build_icosq_param(struct mlx5e_priv *priv,
2157	u8 log_wq_size,	2237	u8 log_wq_size,
2158	struct mlx5e_sq_param *param)	2238	struct mlx5e_sq_param *param)
2159	{	2239	{
2160	void *sqc = param->sqc;	2240	void *sqc = param->sqc;
2161	void *wq = MLX5_ADDR_OF(sqc, sqc, wq);	2241	void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
@@ -2166,9 +2246,9 @@ static void mlx5e_build_icosq_param(struct mlx5e_priv *priv, · 4 unmodified lines
2166	MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(priv->mdev, reg_umr_sq));	2246	MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(priv->mdev, reg_umr_sq));
2167	}	2247	}
2168		2248
2169	static void mlx5e_build_xdpsq_param(struct mlx5e_priv *priv,	2249	void mlx5e_build_xdpsq_param(struct mlx5e_priv *priv,
2170	struct mlx5e_params *params,	2250	struct mlx5e_params *params,
2171	struct mlx5e_sq_param *param)	2251	struct mlx5e_sq_param *param)
2172	{	2252	{
2173	void *sqc = param->sqc;	2253	void *sqc = param->sqc;
2174	void *wq = MLX5_ADDR_OF(sqc, sqc, wq);	2254	void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
@@ -2196,14 +2276,14 @@ static void mlx5e_build_channel_param(struct mlx5e_priv *priv, · 21 unmodified lines
2196	{	2276	{
2197	u8 icosq_log_wq_sz;	2277	u8 icosq_log_wq_sz;
2198		2278
2199	mlx5e_build_rq_param(priv, params, &cparam->rq);	2279	mlx5e_build_rq_param(priv, params, NULL, &cparam->rq);
2200		2280
2201	icosq_log_wq_sz = mlx5e_build_icosq_log_wq_sz(params, &cparam->rq);	2281	icosq_log_wq_sz = mlx5e_build_icosq_log_wq_sz(params, &cparam->rq);
2202		2282
2203	mlx5e_build_sq_param(priv, params, &cparam->sq);	2283	mlx5e_build_sq_param(priv, params, &cparam->sq);
2204	mlx5e_build_xdpsq_param(priv, params, &cparam->xdp_sq);	2284	mlx5e_build_xdpsq_param(priv, params, &cparam->xdp_sq);
2205	mlx5e_build_icosq_param(priv, icosq_log_wq_sz, &cparam->icosq);	2285	mlx5e_build_icosq_param(priv, icosq_log_wq_sz, &cparam->icosq);
2206	mlx5e_build_rx_cq_param(priv, params, &cparam->rx_cq);	2286	mlx5e_build_rx_cq_param(priv, params, NULL, &cparam->rx_cq);
2207	mlx5e_build_tx_cq_param(priv, params, &cparam->tx_cq);	2287	mlx5e_build_tx_cq_param(priv, params, &cparam->tx_cq);
2208	mlx5e_build_ico_cq_param(priv, icosq_log_wq_sz, &cparam->icosq_cq);	2288	mlx5e_build_ico_cq_param(priv, icosq_log_wq_sz, &cparam->icosq_cq);
2209	}	2289	}
@@ -2224,7 +2304,12 @@ int mlx5e_open_channels(struct mlx5e_priv *priv, · 14 unmodified lines
2224		2304
2225	mlx5e_build_channel_param(priv, &chs->params, cparam);	2305	mlx5e_build_channel_param(priv, &chs->params, cparam);
2226	for (i = 0; i < chs->num; i++) {	2306	for (i = 0; i < chs->num; i++) {
2227	err = mlx5e_open_channel(priv, i, &chs->params, cparam, &chs->c[i]);	2307	struct xdp_umem *umem = NULL;
		2308
		2309	if (chs->params.xdp_prog)
		2310	umem = mlx5e_xsk_get_umem(&chs->params, chs->params.xsk, i);
		2311
		2312	err = mlx5e_open_channel(priv, i, &chs->params, cparam, umem, &chs->c[i]);
2228	if (err)	2313	if (err)
2229	goto err_close_channels;	2314	goto err_close_channels;
2230	}	2315	}
@@ -2266,6 +2351,10 @@ static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs) · 35 unmodified lines
2266	int timeout = err ? 0 : MLX5E_RQ_WQES_TIMEOUT;	2351	int timeout = err ? 0 : MLX5E_RQ_WQES_TIMEOUT;
2267		2352
2268	err \|= mlx5e_wait_for_min_rx_wqes(&chs->c[i]->rq, timeout);	2353	err \|= mlx5e_wait_for_min_rx_wqes(&chs->c[i]->rq, timeout);
		2354
		2355	/* Don't wait on the XSK RQ, because the newer xdpsock sample
		2356	* doesn't provide any Fill Ring entries at the setup stage.
		2357	*/
2269	}	2358	}
2270		2359
2271	return err ? -ETIMEDOUT : 0;	2360	return err ? -ETIMEDOUT : 0;
@@ -2338,35 +2427,35 @@ int mlx5e_create_indirect_rqt(struct mlx5e_priv *priv) · 66 unmodified lines
2338	return err;	2427	return err;
2339	}	2428	}
2340		2429
2341	int mlx5e_create_direct_rqts(struct mlx5e_priv *priv)	2430	int mlx5e_create_direct_rqts(struct mlx5e_priv priv, struct mlx5e_tir tirs)
2342	{	2431	{
2343	struct mlx5e_rqt *rqt;	2432	const int max_nch = mlx5e_get_netdev_max_channels(priv->netdev);
2344	int err;	2433	int err;
2345	int ix;	2434	int ix;
2346		2435
2347	for (ix = 0; ix < mlx5e_get_netdev_max_channels(priv->netdev); ix++) {	2436	for (ix = 0; ix < max_nch; ix++) {
2348	rqt = &priv->direct_tir[ix].rqt;	2437	err = mlx5e_create_rqt(priv, 1 /size /, &tirs[ix].rqt);
2349	err = mlx5e_create_rqt(priv, 1 /size /, rqt);	2438	if (unlikely(err))
2350	if (err)
2351	goto err_destroy_rqts;	2439	goto err_destroy_rqts;
2352	}	2440	}
2353		2441
2354	return 0;	2442	return 0;
2355		2443
2356	err_destroy_rqts:	2444	err_destroy_rqts:
2357	mlx5_core_warn(priv->mdev, "create direct rqts failed, %d\n", err);	2445	mlx5_core_warn(priv->mdev, "create rqts failed, %d\n", err);
2358	for (ix--; ix >= 0; ix--)	2446	for (ix--; ix >= 0; ix--)
2359	mlx5e_destroy_rqt(priv, &priv->direct_tir[ix].rqt);	2447	mlx5e_destroy_rqt(priv, &tirs[ix].rqt);
2360		2448
2361	return err;	2449	return err;
2362	}	2450	}
2363		2451
2364	void mlx5e_destroy_direct_rqts(struct mlx5e_priv *priv)	2452	void mlx5e_destroy_direct_rqts(struct mlx5e_priv priv, struct mlx5e_tir tirs)
2365	{	2453	{
		2454	const int max_nch = mlx5e_get_netdev_max_channels(priv->netdev);
2366	int i;	2455	int i;
2367		2456
2368	for (i = 0; i < mlx5e_get_netdev_max_channels(priv->netdev); i++)	2457	for (i = 0; i < max_nch; i++)
2369	mlx5e_destroy_rqt(priv, &priv->direct_tir[i].rqt);	2458	mlx5e_destroy_rqt(priv, &tirs[i].rqt);
2370	}	2459	}
2371		2460
2372	static int mlx5e_rx_hash_fn(int hfunc)	2461	static int mlx5e_rx_hash_fn(int hfunc)
@@ -2786,11 +2875,12 @@ static void mlx5e_build_tx2sq_maps(struct mlx5e_priv *priv) · 413 unmodified lines
2786	void mlx5e_activate_priv_channels(struct mlx5e_priv *priv)	2875	void mlx5e_activate_priv_channels(struct mlx5e_priv *priv)
2787	{	2876	{
2788	int num_txqs = priv->channels.num * priv->channels.params.num_tc;	2877	int num_txqs = priv->channels.num * priv->channels.params.num_tc;
		2878	int num_rxqs = priv->channels.num * MLX5E_NUM_RQ_GROUPS;
2789	struct net_device *netdev = priv->netdev;	2879	struct net_device *netdev = priv->netdev;
2790		2880
2791	mlx5e_netdev_set_tcs(netdev);	2881	mlx5e_netdev_set_tcs(netdev);
2792	netif_set_real_num_tx_queues(netdev, num_txqs);	2882	netif_set_real_num_tx_queues(netdev, num_txqs);
2793	netif_set_real_num_rx_queues(netdev, priv->channels.num);	2883	netif_set_real_num_rx_queues(netdev, num_rxqs);
2794		2884
2795	mlx5e_build_tx2sq_maps(priv);	2885	mlx5e_build_tx2sq_maps(priv);
2796	mlx5e_activate_channels(&priv->channels);	2886	mlx5e_activate_channels(&priv->channels);
@@ -2802,10 +2892,14 @@ void mlx5e_activate_priv_channels(struct mlx5e_priv *priv) · 5 unmodified lines
2802		2892
2803	mlx5e_wait_channels_min_rx_wqes(&priv->channels);	2893	mlx5e_wait_channels_min_rx_wqes(&priv->channels);
2804	mlx5e_redirect_rqts_to_channels(priv, &priv->channels);	2894	mlx5e_redirect_rqts_to_channels(priv, &priv->channels);
		2895
		2896	mlx5e_xsk_redirect_rqts_to_channels(priv, &priv->channels);
2805	}	2897	}
2806		2898
2807	void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv)	2899	void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv)
2808	{	2900	{
		2901	mlx5e_xsk_redirect_rqts_to_drop(priv, &priv->channels);
		2902
2809	mlx5e_redirect_rqts_to_drop(priv);	2903	mlx5e_redirect_rqts_to_drop(priv);
2810		2904
2811	if (mlx5e_is_vport_rep(priv))	2905	if (mlx5e_is_vport_rep(priv))
@@ -2884,9 +2978,12 @@ void mlx5e_timestamp_init(struct mlx5e_priv *priv) · 72 unmodified lines
2884	int mlx5e_open_locked(struct net_device *netdev)	2978	int mlx5e_open_locked(struct net_device *netdev)
2885	{	2979	{
2886	struct mlx5e_priv *priv = netdev_priv(netdev);	2980	struct mlx5e_priv *priv = netdev_priv(netdev);
		2981	bool is_xdp = priv->channels.params.xdp_prog;
2887	int err;	2982	int err;
2888		2983
2889	set_bit(MLX5E_STATE_OPENED, &priv->state);	2984	set_bit(MLX5E_STATE_OPENED, &priv->state);
		2985	if (is_xdp)
		2986	mlx5e_xdp_set_open(priv);
2890		2987
2891	err = mlx5e_open_channels(priv, &priv->channels);	2988	err = mlx5e_open_channels(priv, &priv->channels);
2892	if (err)	2989	if (err)
@@ -2901,6 +2998,8 @@ int mlx5e_open_locked(struct net_device *netdev) · 8 unmodified lines
2901	return 0;	2998	return 0;
2902		2999
2903	err_clear_state_opened_flag:	3000	err_clear_state_opened_flag:
		3001	if (is_xdp)
		3002	mlx5e_xdp_set_closed(priv);
2904	clear_bit(MLX5E_STATE_OPENED, &priv->state);	3003	clear_bit(MLX5E_STATE_OPENED, &priv->state);
2905	return err;	3004	return err;
2906	}	3005	}
@@ -2932,6 +3031,8 @@ int mlx5e_close_locked(struct net_device *netdev) · 25 unmodified lines
2932	if (!test_bit(MLX5E_STATE_OPENED, &priv->state))	3031	if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
2933	return 0;	3032	return 0;
2934		3033
		3034	if (priv->channels.params.xdp_prog)
		3035	mlx5e_xdp_set_closed(priv);
2935	clear_bit(MLX5E_STATE_OPENED, &priv->state);	3036	clear_bit(MLX5E_STATE_OPENED, &priv->state);
2936		3037
2937	netif_carrier_off(priv->netdev);	3038	netif_carrier_off(priv->netdev);
@@ -3188,13 +3289,13 @@ err_destroy_inner_tirs: · 250 unmodified lines
3188	return err;	3289	return err;
3189	}	3290	}
3190		3291
3191	int mlx5e_create_direct_tirs(struct mlx5e_priv *priv)	3292	int mlx5e_create_direct_tirs(struct mlx5e_priv priv, struct mlx5e_tir tirs)
3192	{	3293	{
3193	int nch = mlx5e_get_netdev_max_channels(priv->netdev);	3294	const int max_nch = mlx5e_get_netdev_max_channels(priv->netdev);
3194	struct mlx5e_tir *tir;	3295	struct mlx5e_tir *tir;
3195	void *tirc;	3296	void *tirc;
3196	int inlen;	3297	int inlen;
3197	int err;	3298	int err = 0;
3198	u32 *in;	3299	u32 *in;
3199	int ix;	3300	int ix;
3200		3301
@@ -3203,25 +3304,24 @@ int mlx5e_create_direct_tirs(struct mlx5e_priv *priv) · 2 unmodified lines
3203	if (!in)	3304	if (!in)
3204	return -ENOMEM;	3305	return -ENOMEM;
3205		3306
3206	for (ix = 0; ix < nch; ix++) {	3307	for (ix = 0; ix < max_nch; ix++) {
3207	memset(in, 0, inlen);	3308	memset(in, 0, inlen);
3208	tir = &priv->direct_tir[ix];	3309	tir = &tirs[ix];
3209	tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);	3310	tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
3210	mlx5e_build_direct_tir_ctx(priv, priv->direct_tir[ix].rqt.rqtn, tirc);	3311	mlx5e_build_direct_tir_ctx(priv, tir->rqt.rqtn, tirc);
3211	err = mlx5e_create_tir(priv->mdev, tir, in, inlen);	3312	err = mlx5e_create_tir(priv->mdev, tir, in, inlen);
3212	if (err)	3313	if (unlikely(err))
3213	goto err_destroy_ch_tirs;	3314	goto err_destroy_ch_tirs;
3214	}	3315	}
3215		3316
3216	kvfree(in);	3317	goto out;
3217
3218	return 0;
3219		3318
3220	err_destroy_ch_tirs:	3319	err_destroy_ch_tirs:
3221	mlx5_core_warn(priv->mdev, "create direct tirs failed, %d\n", err);	3320	mlx5_core_warn(priv->mdev, "create tirs failed, %d\n", err);
3222	for (ix--; ix >= 0; ix--)	3321	for (ix--; ix >= 0; ix--)
3223	mlx5e_destroy_tir(priv->mdev, &priv->direct_tir[ix]);	3322	mlx5e_destroy_tir(priv->mdev, &tirs[ix]);
3224		3323
		3324	out:
3225	kvfree(in);	3325	kvfree(in);
3226		3326
3227	return err;	3327	return err;
@@ -3241,13 +3341,13 @@ void mlx5e_destroy_indirect_tirs(struct mlx5e_priv *priv, bool inner_ttc) · 13 unmodified lines
3241	mlx5e_destroy_tir(priv->mdev, &priv->inner_indir_tir[i]);	3341	mlx5e_destroy_tir(priv->mdev, &priv->inner_indir_tir[i]);
3242	}	3342	}
3243		3343
3244	void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv)	3344	void mlx5e_destroy_direct_tirs(struct mlx5e_priv priv, struct mlx5e_tir tirs)
3245	{	3345	{
3246	int nch = mlx5e_get_netdev_max_channels(priv->netdev);	3346	const int max_nch = mlx5e_get_netdev_max_channels(priv->netdev);
3247	int i;	3347	int i;
3248		3348
3249	for (i = 0; i < nch; i++)	3349	for (i = 0; i < max_nch; i++)
3250	mlx5e_destroy_tir(priv->mdev, &priv->direct_tir[i]);	3350	mlx5e_destroy_tir(priv->mdev, &tirs[i]);
3251	}	3351	}
3252		3352
3253	static int mlx5e_modify_channels_scatter_fcs(struct mlx5e_channels *chs, bool enable)	3353	static int mlx5e_modify_channels_scatter_fcs(struct mlx5e_channels *chs, bool enable)
@@ -3389,11 +3489,12 @@ void mlx5e_fold_sw_stats64(struct mlx5e_priv priv, struct rtnl_link_stats64 s) · 135 unmodified lines
3389		3489
3390	for (i = 0; i < mlx5e_get_netdev_max_channels(priv->netdev); i++) {	3490	for (i = 0; i < mlx5e_get_netdev_max_channels(priv->netdev); i++) {
3391	struct mlx5e_channel_stats *channel_stats = &priv->channel_stats[i];	3491	struct mlx5e_channel_stats *channel_stats = &priv->channel_stats[i];
		3492	struct mlx5e_rq_stats *xskrq_stats = &channel_stats->xskrq;
3392	struct mlx5e_rq_stats *rq_stats = &channel_stats->rq;	3493	struct mlx5e_rq_stats *rq_stats = &channel_stats->rq;
3393	int j;	3494	int j;
3394		3495
3395	s->rx_packets += rq_stats->packets;	3496	s->rx_packets += rq_stats->packets + xskrq_stats->packets;
3396	s->rx_bytes += rq_stats->bytes;	3497	s->rx_bytes += rq_stats->bytes + xskrq_stats->bytes;
3397		3498
3398	for (j = 0; j < priv->max_opened_tc; j++) {	3499	for (j = 0; j < priv->max_opened_tc; j++) {
3399	struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[j];	3500	struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[j];
@@ -3492,6 +3593,13 @@ static int set_feature_lro(struct net_device *netdev, bool enable) · 92 unmodified lines
3492		3593
3493	mutex_lock(&priv->state_lock);	3594	mutex_lock(&priv->state_lock);
3494		3595
		3596	if (enable && priv->xsk.refcnt) {
		3597	netdev_warn(netdev, "LRO is incompatible with AF_XDP (%hu XSKs are active)\n",
		3598	priv->xsk.refcnt);
		3599	err = -EINVAL;
		3600	goto out;
		3601	}
		3602
3495	old_params = &priv->channels.params;	3603	old_params = &priv->channels.params;
3496	if (enable && !MLX5E_GET_PFLAG(old_params, MLX5E_PFLAG_RX_STRIDING_RQ)) {	3604	if (enable && !MLX5E_GET_PFLAG(old_params, MLX5E_PFLAG_RX_STRIDING_RQ)) {
3497	netdev_warn(netdev, "can't set LRO with legacy RQ\n");	3605	netdev_warn(netdev, "can't set LRO with legacy RQ\n");
@@ -3505,8 +3613,8 @@ static int set_feature_lro(struct net_device *netdev, bool enable) · 7 unmodified lines
3505	new_channels.params.lro_en = enable;	3613	new_channels.params.lro_en = enable;
3506		3614
3507	if (old_params->rq_wq_type != MLX5_WQ_TYPE_CYCLIC) {	3615	if (old_params->rq_wq_type != MLX5_WQ_TYPE_CYCLIC) {
3508	if (mlx5e_rx_mpwqe_is_linear_skb(mdev, old_params) ==	3616	if (mlx5e_rx_mpwqe_is_linear_skb(mdev, old_params, NULL) ==
3509	mlx5e_rx_mpwqe_is_linear_skb(mdev, &new_channels.params))	3617	mlx5e_rx_mpwqe_is_linear_skb(mdev, &new_channels.params, NULL))
3510	reset = false;	3618	reset = false;
3511	}	3619	}
3512		3620
@@ -3696,6 +3804,43 @@ static netdev_features_t mlx5e_fix_features(struct net_device *netdev, · 183 unmodified lines
3696	return features;	3804	return features;
3697	}	3805	}
3698		3806
		3807	static bool mlx5e_xsk_validate_mtu(struct net_device *netdev,
		3808	struct mlx5e_channels *chs,
		3809	struct mlx5e_params *new_params,
		3810	struct mlx5_core_dev *mdev)
		3811	{
		3812	u16 ix;
		3813
		3814	for (ix = 0; ix < chs->params.num_channels; ix++) {
		3815	struct xdp_umem *umem = mlx5e_xsk_get_umem(&chs->params, chs->params.xsk, ix);
		3816	struct mlx5e_xsk_param xsk;
		3817
		3818	if (!umem)
		3819	continue;
		3820
		3821	mlx5e_build_xsk_param(umem, &xsk);
		3822
		3823	if (!mlx5e_validate_xsk_param(new_params, &xsk, mdev)) {
		3824	u32 hr = mlx5e_get_linear_rq_headroom(new_params, &xsk);
		3825	int max_mtu_frame, max_mtu_page, max_mtu;
		3826
		3827	/* Two criteria must be met:
		3828	* 1. HW MTU + all headrooms <= XSK frame size.
		3829	* 2. Size of SKBs allocated on XDP_PASS <= PAGE_SIZE.
		3830	*/
		3831	max_mtu_frame = MLX5E_HW2SW_MTU(new_params, xsk.chunk_size - hr);
		3832	max_mtu_page = mlx5e_xdp_max_mtu(new_params, &xsk);
		3833	max_mtu = min(max_mtu_frame, max_mtu_page);
		3834
		3835	netdev_err(netdev, "MTU %d is too big for an XSK running on channel %hu. Try MTU <= %d\n",
		3836	new_params->sw_mtu, ix, max_mtu);
		3837	return false;
		3838	}
		3839	}
		3840
		3841	return true;
		3842	}
		3843
3699	int mlx5e_change_mtu(struct net_device *netdev, int new_mtu,	3844	int mlx5e_change_mtu(struct net_device *netdev, int new_mtu,
3700	change_hw_mtu_cb set_mtu_cb)	3845	change_hw_mtu_cb set_mtu_cb)
3701	{	3846	{
@@ -3716,18 +3861,31 @@ int mlx5e_change_mtu(struct net_device *netdev, int new_mtu, · 14 unmodified lines
3716	new_channels.params.sw_mtu = new_mtu;	3861	new_channels.params.sw_mtu = new_mtu;
3717		3862
3718	if (params->xdp_prog &&	3863	if (params->xdp_prog &&
3719	!mlx5e_rx_is_linear_skb(&new_channels.params)) {	3864	!mlx5e_rx_is_linear_skb(&new_channels.params, NULL)) {
3720	netdev_err(netdev, "MTU(%d) > %d is not allowed while XDP enabled\n",	3865	netdev_err(netdev, "MTU(%d) > %d is not allowed while XDP enabled\n",
3721	new_mtu, mlx5e_xdp_max_mtu(params));	3866	new_mtu, mlx5e_xdp_max_mtu(params, NULL));
		3867	err = -EINVAL;
		3868	goto out;
		3869	}
		3870
		3871	if (priv->xsk.refcnt &&
		3872	!mlx5e_xsk_validate_mtu(netdev, &priv->channels,
		3873	&new_channels.params, priv->mdev)) {
3722	err = -EINVAL;	3874	err = -EINVAL;
3723	goto out;	3875	goto out;
3724	}	3876	}
3725		3877
3726	if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {	3878	if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
3727	bool is_linear = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev, &new_channels.params);	3879	bool is_linear = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev,
3728	u8 ppw_old = mlx5e_mpwqe_log_pkts_per_wqe(params);	3880	&new_channels.params,
3729	u8 ppw_new = mlx5e_mpwqe_log_pkts_per_wqe(&new_channels.params);	3881	NULL);
		3882	u8 ppw_old = mlx5e_mpwqe_log_pkts_per_wqe(params, NULL);
		3883	u8 ppw_new = mlx5e_mpwqe_log_pkts_per_wqe(&new_channels.params, NULL);
		3884
		3885	/* If XSK is active, XSK RQs are linear. */
		3886	is_linear \|= priv->xsk.refcnt;
3730		3887
		3888	/* Always reset in linear mode - hw_mtu is used in data path. */
3731	reset = reset && (is_linear \|\| (ppw_old != ppw_new));	3889	reset = reset && (is_linear \|\| (ppw_old != ppw_new));
3732	}	3890	}
3733		3891
@@ -4160,16 +4318,29 @@ static int mlx5e_xdp_allowed(struct mlx5e_priv priv, struct bpf_prog prog) · 426 unmodified lines
4160	new_channels.params = priv->channels.params;	4318	new_channels.params = priv->channels.params;
4161	new_channels.params.xdp_prog = prog;	4319	new_channels.params.xdp_prog = prog;
4162		4320
4163	if (!mlx5e_rx_is_linear_skb(&new_channels.params)) {	4321	/* No XSK params: AF_XDP can't be enabled yet at the point of setting
		4322	* the XDP program.
		4323	*/
		4324	if (!mlx5e_rx_is_linear_skb(&new_channels.params, NULL)) {
4164	netdev_warn(netdev, "XDP is not allowed with MTU(%d) > %d\n",	4325	netdev_warn(netdev, "XDP is not allowed with MTU(%d) > %d\n",
4165	new_channels.params.sw_mtu,	4326	new_channels.params.sw_mtu,
4166	mlx5e_xdp_max_mtu(&new_channels.params));	4327	mlx5e_xdp_max_mtu(&new_channels.params, NULL));
4167	return -EINVAL;	4328	return -EINVAL;
4168	}	4329	}
4169		4330
4170	return 0;	4331	return 0;
4171	}	4332	}
4172		4333
		4334	static int mlx5e_xdp_update_state(struct mlx5e_priv *priv)
		4335	{
		4336	if (priv->channels.params.xdp_prog)
		4337	mlx5e_xdp_set_open(priv);
		4338	else
		4339	mlx5e_xdp_set_closed(priv);
		4340
		4341	return 0;
		4342	}
		4343
4173	static int mlx5e_xdp_set(struct net_device netdev, struct bpf_prog prog)	4344	static int mlx5e_xdp_set(struct net_device netdev, struct bpf_prog prog)
4174	{	4345	{
4175	struct mlx5e_priv *priv = netdev_priv(netdev);	4346	struct mlx5e_priv *priv = netdev_priv(netdev);
@@ -4190,8 +4361,6 @@ static int mlx5e_xdp_set(struct net_device netdev, struct bpf_prog prog) · 14 unmodified lines
4190	/* no need for full reset when exchanging programs */	4361	/* no need for full reset when exchanging programs */
4191	reset = (!priv->channels.params.xdp_prog \|\| !prog);	4362	reset = (!priv->channels.params.xdp_prog \|\| !prog);
4192		4363
4193	if (was_opened && reset)
4194	mlx5e_close_locked(netdev);
4195	if (was_opened && !reset) {	4364	if (was_opened && !reset) {
4196	/* num_channels is invariant here, so we can take the	4365	/* num_channels is invariant here, so we can take the
4197	* batched reference right upfront.	4366	* batched reference right upfront.
@@ -4203,20 +4372,31 @@ static int mlx5e_xdp_set(struct net_device netdev, struct bpf_prog prog) · 5 unmodified lines
4203	}	4372	}
4204	}	4373	}
4205		4374
4206	/* exchange programs, extra prog reference we got from caller	4375	if (was_opened && reset) {
4207	* as long as we don't fail from this point onwards.	4376	struct mlx5e_channels new_channels = {};
4208	*/	4377
4209	old_prog = xchg(&priv->channels.params.xdp_prog, prog);	4378	new_channels.params = priv->channels.params;
		4379	new_channels.params.xdp_prog = prog;
		4380	mlx5e_set_rq_type(priv->mdev, &new_channels.params);
		4381	old_prog = priv->channels.params.xdp_prog;
		4382
		4383	err = mlx5e_safe_switch_channels(priv, &new_channels, mlx5e_xdp_update_state);
		4384	if (err)
		4385	goto unlock;
		4386	} else {
		4387	/* exchange programs, extra prog reference we got from caller
		4388	* as long as we don't fail from this point onwards.
		4389	*/
		4390	old_prog = xchg(&priv->channels.params.xdp_prog, prog);
		4391	}
		4392
4210	if (old_prog)	4393	if (old_prog)
4211	bpf_prog_put(old_prog);	4394	bpf_prog_put(old_prog);
4212		4395
4213	if (reset) /* change RQ type according to priv->xdp_prog */	4396	if (!was_opened && reset) /* change RQ type according to priv->xdp_prog */
4214	mlx5e_set_rq_type(priv->mdev, &priv->channels.params);	4397	mlx5e_set_rq_type(priv->mdev, &priv->channels.params);
4215		4398
4216	if (was_opened && reset)	4399	if (!was_opened \|\| reset)
4217	err = mlx5e_open_locked(netdev);
4218
4219	if (!test_bit(MLX5E_STATE_OPENED, &priv->state) \|\| reset)
4220	goto unlock;	4400	goto unlock;
4221		4401
4222	/* exchanging programs w/o reset, we update ref counts on behalf	4402	/* exchanging programs w/o reset, we update ref counts on behalf
@@ -4224,19 +4404,29 @@ static int mlx5e_xdp_set(struct net_device netdev, struct bpf_prog prog) · 1 unmodified line
4224	*/	4404	*/
4225	for (i = 0; i < priv->channels.num; i++) {	4405	for (i = 0; i < priv->channels.num; i++) {
4226	struct mlx5e_channel *c = priv->channels.c[i];	4406	struct mlx5e_channel *c = priv->channels.c[i];
		4407	bool xsk_open = test_bit(MLX5E_CHANNEL_STATE_XSK, c->state);
4227		4408
4228	clear_bit(MLX5E_RQ_STATE_ENABLED, &c->rq.state);	4409	clear_bit(MLX5E_RQ_STATE_ENABLED, &c->rq.state);
		4410	if (xsk_open)
		4411	clear_bit(MLX5E_RQ_STATE_ENABLED, &c->xskrq.state);
4229	napi_synchronize(&c->napi);	4412	napi_synchronize(&c->napi);
4230	/* prevent mlx5e_poll_rx_cq from accessing rq->xdp_prog */	4413	/* prevent mlx5e_poll_rx_cq from accessing rq->xdp_prog */
4231		4414
4232	old_prog = xchg(&c->rq.xdp_prog, prog);	4415	old_prog = xchg(&c->rq.xdp_prog, prog);
		4416	if (old_prog)
		4417	bpf_prog_put(old_prog);
		4418
		4419	if (xsk_open) {
		4420	old_prog = xchg(&c->xskrq.xdp_prog, prog);
		4421	if (old_prog)
		4422	bpf_prog_put(old_prog);
		4423	}
4233		4424
4234	set_bit(MLX5E_RQ_STATE_ENABLED, &c->rq.state);	4425	set_bit(MLX5E_RQ_STATE_ENABLED, &c->rq.state);
		4426	if (xsk_open)
		4427	set_bit(MLX5E_RQ_STATE_ENABLED, &c->xskrq.state);
4235	/* napi_schedule in case we have missed anything */	4428	/* napi_schedule in case we have missed anything */
4236	napi_schedule(&c->napi);	4429	napi_schedule(&c->napi);
4237
4238	if (old_prog)
4239	bpf_prog_put(old_prog);
4240	}	4430	}
4241		4431
4242	unlock:	4432	unlock:
@@ -4267,6 +4457,9 @@ static int mlx5e_xdp(struct net_device dev, struct netdev_bpf xdp) · 24 unmodified lines
4267	case XDP_QUERY_PROG:	4457	case XDP_QUERY_PROG:
4268	xdp->prog_id = mlx5e_xdp_query(dev);	4458	xdp->prog_id = mlx5e_xdp_query(dev);
4269	return 0;	4459	return 0;
		4460	case XDP_SETUP_XSK_UMEM:
		4461	return mlx5e_xsk_setup_umem(dev, xdp->xsk.umem,
		4462	xdp->xsk.queue_id);
4270	default:	4463	default:
4271	return -EINVAL;	4464	return -EINVAL;
4272	}	4465	}
@@ -4349,6 +4542,7 @@ const struct net_device_ops mlx5e_netdev_ops = { · 76 unmodified lines
4349	.ndo_tx_timeout = mlx5e_tx_timeout,	4542	.ndo_tx_timeout = mlx5e_tx_timeout,
4350	.ndo_bpf = mlx5e_xdp,	4543	.ndo_bpf = mlx5e_xdp,
4351	.ndo_xdp_xmit = mlx5e_xdp_xmit,	4544	.ndo_xdp_xmit = mlx5e_xdp_xmit,
		4545	.ndo_xsk_async_xmit = mlx5e_xsk_async_xmit,
4352	#ifdef CONFIG_MLX5_EN_ARFS	4546	#ifdef CONFIG_MLX5_EN_ARFS
4353	.ndo_rx_flow_steer = mlx5e_rx_flow_steer,	4547	.ndo_rx_flow_steer = mlx5e_rx_flow_steer,
4354	#endif	4548	#endif
@@ -4500,11 +4694,13 @@ void mlx5e_build_rq_params(struct mlx5_core_dev *mdev, · 145 unmodified lines
4500	* - Striding RQ configuration is not possible/supported.	4694	* - Striding RQ configuration is not possible/supported.
4501	* - Slow PCI heuristic.	4695	* - Slow PCI heuristic.
4502	* - Legacy RQ would use linear SKB while Striding RQ would use non-linear.	4696	* - Legacy RQ would use linear SKB while Striding RQ would use non-linear.
		4697	*
		4698	* No XSK params: checking the availability of striding RQ in general.
4503	*/	4699	*/
4504	if (!slow_pci_heuristic(mdev) &&	4700	if (!slow_pci_heuristic(mdev) &&
4505	mlx5e_striding_rq_possible(mdev, params) &&	4701	mlx5e_striding_rq_possible(mdev, params) &&
4506	(mlx5e_rx_mpwqe_is_linear_skb(mdev, params) \|\|	4702	(mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL) \|\|
4507	!mlx5e_rx_is_linear_skb(params)))	4703	!mlx5e_rx_is_linear_skb(params, NULL)))
4508	MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ, true);	4704	MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ, true);
4509	mlx5e_set_rq_type(mdev, params);	4705	mlx5e_set_rq_type(mdev, params);
4510	mlx5e_init_rq_type_params(mdev, params);	4706	mlx5e_init_rq_type_params(mdev, params);
@@ -4526,6 +4722,7 @@ void mlx5e_build_rss_params(struct mlx5e_rss_params *rss_params, · 15 unmodified lines
4526	}	4722	}
4527		4723
4528	void mlx5e_build_nic_params(struct mlx5_core_dev *mdev,	4724	void mlx5e_build_nic_params(struct mlx5_core_dev *mdev,
		4725	struct mlx5e_xsk *xsk,
4529	struct mlx5e_rss_params *rss_params,	4726	struct mlx5e_rss_params *rss_params,
4530	struct mlx5e_params *params,	4727	struct mlx5e_params *params,
4531	u16 max_channels, u16 mtu)	4728	u16 max_channels, u16 mtu)
@@ -4561,9 +4758,11 @@ void mlx5e_build_nic_params(struct mlx5_core_dev *mdev, · 29 unmodified lines
4561	/* HW LRO */	4758	/* HW LRO */
4562		4759
4563	/* TODO: && MLX5_CAP_ETH(mdev, lro_cap) */	4760	/* TODO: && MLX5_CAP_ETH(mdev, lro_cap) */
4564	if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)	4761	if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
4565	if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params))	4762	/* No XSK params: checking the availability of striding RQ in general. */
		4763	if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL))
4566	params->lro_en = !slow_pci_heuristic(mdev);	4764	params->lro_en = !slow_pci_heuristic(mdev);
		4765	}
4567	params->lro_timeout = mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT);	4766	params->lro_timeout = mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT);
4568		4767
4569	/* CQ moderation params */	4768	/* CQ moderation params */
@@ -4582,6 +4781,9 @@ void mlx5e_build_nic_params(struct mlx5_core_dev *mdev, · 12 unmodified lines
4582	mlx5e_build_rss_params(rss_params, params->num_channels);	4781	mlx5e_build_rss_params(rss_params, params->num_channels);
4583	params->tunneled_offload_en =	4782	params->tunneled_offload_en =
4584	mlx5e_tunnel_inner_ft_supported(mdev);	4783	mlx5e_tunnel_inner_ft_supported(mdev);
		4784
		4785	/* AF_XDP */
		4786	params->xsk = xsk;
4585	}	4787	}
4586		4788
4587	static void mlx5e_set_netdev_dev_addr(struct net_device *netdev)	4789	static void mlx5e_set_netdev_dev_addr(struct net_device *netdev)
@@ -4757,7 +4959,7 @@ static int mlx5e_nic_init(struct mlx5_core_dev *mdev, · 169 unmodified lines
4757	if (err)	4959	if (err)
4758	return err;	4960	return err;
4759		4961
4760	mlx5e_build_nic_params(mdev, rss, &priv->channels.params,	4962	mlx5e_build_nic_params(mdev, &priv->xsk, rss, &priv->channels.params,
4761	mlx5e_get_netdev_max_channels(netdev),	4963	mlx5e_get_netdev_max_channels(netdev),
4762	netdev->mtu);	4964	netdev->mtu);
4763		4965
@@ -4799,7 +5001,7 @@ static int mlx5e_init_nic_rx(struct mlx5e_priv *priv) · 35 unmodified lines
4799	if (err)	5001	if (err)
4800	goto err_close_drop_rq;	5002	goto err_close_drop_rq;
4801		5003
4802	err = mlx5e_create_direct_rqts(priv);	5004	err = mlx5e_create_direct_rqts(priv, priv->direct_tir);
4803	if (err)	5005	if (err)
4804	goto err_destroy_indirect_rqts;	5006	goto err_destroy_indirect_rqts;
4805		5007
@@ -4807,14 +5009,22 @@ static int mlx5e_init_nic_rx(struct mlx5e_priv *priv) · 1 unmodified line
4807	if (err)	5009	if (err)
4808	goto err_destroy_direct_rqts;	5010	goto err_destroy_direct_rqts;
4809		5011
4810	err = mlx5e_create_direct_tirs(priv);	5012	err = mlx5e_create_direct_tirs(priv, priv->direct_tir);
4811	if (err)	5013	if (err)
4812	goto err_destroy_indirect_tirs;	5014	goto err_destroy_indirect_tirs;
4813		5015
		5016	err = mlx5e_create_direct_rqts(priv, priv->xsk_tir);
		5017	if (unlikely(err))
		5018	goto err_destroy_direct_tirs;
		5019
		5020	err = mlx5e_create_direct_tirs(priv, priv->xsk_tir);
		5021	if (unlikely(err))
		5022	goto err_destroy_xsk_rqts;
		5023
4814	err = mlx5e_create_flow_steering(priv);	5024	err = mlx5e_create_flow_steering(priv);
4815	if (err) {	5025	if (err) {
4816	mlx5_core_warn(mdev, "create flow steering failed, %d\n", err);	5026	mlx5_core_warn(mdev, "create flow steering failed, %d\n", err);
4817	goto err_destroy_direct_tirs;	5027	goto err_destroy_xsk_tirs;
4818	}	5028	}
4819		5029
4820	err = mlx5e_tc_nic_init(priv);	5030	err = mlx5e_tc_nic_init(priv);
@@ -4825,12 +5035,16 @@ static int mlx5e_init_nic_rx(struct mlx5e_priv *priv) · 4 unmodified lines
4825		5035
4826	err_destroy_flow_steering:	5036	err_destroy_flow_steering:
4827	mlx5e_destroy_flow_steering(priv);	5037	mlx5e_destroy_flow_steering(priv);
		5038	err_destroy_xsk_tirs:
		5039	mlx5e_destroy_direct_tirs(priv, priv->xsk_tir);
		5040	err_destroy_xsk_rqts:
		5041	mlx5e_destroy_direct_rqts(priv, priv->xsk_tir);
4828	err_destroy_direct_tirs:	5042	err_destroy_direct_tirs:
4829	mlx5e_destroy_direct_tirs(priv);	5043	mlx5e_destroy_direct_tirs(priv, priv->direct_tir);
4830	err_destroy_indirect_tirs:	5044	err_destroy_indirect_tirs:
4831	mlx5e_destroy_indirect_tirs(priv, true);	5045	mlx5e_destroy_indirect_tirs(priv, true);
4832	err_destroy_direct_rqts:	5046	err_destroy_direct_rqts:
4833	mlx5e_destroy_direct_rqts(priv);	5047	mlx5e_destroy_direct_rqts(priv, priv->direct_tir);
4834	err_destroy_indirect_rqts:	5048	err_destroy_indirect_rqts:
4835	mlx5e_destroy_rqt(priv, &priv->indir_rqt);	5049	mlx5e_destroy_rqt(priv, &priv->indir_rqt);
4836	err_close_drop_rq:	5050	err_close_drop_rq:
@@ -4844,9 +5058,11 @@ static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv) · 7 unmodified lines
4844	{	5058	{
4845	mlx5e_tc_nic_cleanup(priv);	5059	mlx5e_tc_nic_cleanup(priv);
4846	mlx5e_destroy_flow_steering(priv);	5060	mlx5e_destroy_flow_steering(priv);
4847	mlx5e_destroy_direct_tirs(priv);	5061	mlx5e_destroy_direct_tirs(priv, priv->xsk_tir);
		5062	mlx5e_destroy_direct_rqts(priv, priv->xsk_tir);
		5063	mlx5e_destroy_direct_tirs(priv, priv->direct_tir);
4848	mlx5e_destroy_indirect_tirs(priv, true);	5064	mlx5e_destroy_indirect_tirs(priv, true);
4849	mlx5e_destroy_direct_rqts(priv);	5065	mlx5e_destroy_direct_rqts(priv, priv->direct_tir);
4850	mlx5e_destroy_rqt(priv, &priv->indir_rqt);	5066	mlx5e_destroy_rqt(priv, &priv->indir_rqt);
4851	mlx5e_close_drop_rq(&priv->drop_rq);	5067	mlx5e_close_drop_rq(&priv->drop_rq);
4852	mlx5e_destroy_q_counters(priv);	5068	mlx5e_destroy_q_counters(priv);
@@ -5002,7 +5218,7 @@ struct net_device mlx5e_create_netdev(struct mlx5_core_dev mdev, · 149 unmodified lines
5002		5218
5003	netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv),	5219	netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv),
5004	nch * profile->max_tc,	5220	nch * profile->max_tc,
5005	nch);	5221	nch * MLX5E_NUM_RQ_GROUPS);
5006	if (!netdev) {	5222	if (!netdev) {
5007	mlx5_core_err(mdev, "alloc_etherdev_mqs() failed\n");	5223	mlx5_core_err(mdev, "alloc_etherdev_mqs() failed\n");
5008	return NULL;	5224	return NULL;

drivers/net/ethernet/mellanox/mlx5/core/en_rep.c+6−6

@@ -1510,7 +1510,7 @@ static int mlx5e_init_rep_rx(struct mlx5e_priv *priv) · 1509 unmodified lines
1510	if (err)	1510	if (err)
1511	goto err_close_drop_rq;	1511	goto err_close_drop_rq;
1512		1512
1513	err = mlx5e_create_direct_rqts(priv);	1513	err = mlx5e_create_direct_rqts(priv, priv->direct_tir);
1514	if (err)	1514	if (err)
1515	goto err_destroy_indirect_rqts;	1515	goto err_destroy_indirect_rqts;
1516		1516
@@ -1518,7 +1518,7 @@ static int mlx5e_init_rep_rx(struct mlx5e_priv *priv) · 1 unmodified line
1518	if (err)	1518	if (err)
1519	goto err_destroy_direct_rqts;	1519	goto err_destroy_direct_rqts;
1520		1520
1521	err = mlx5e_create_direct_tirs(priv);	1521	err = mlx5e_create_direct_tirs(priv, priv->direct_tir);
1522	if (err)	1522	if (err)
1523	goto err_destroy_indirect_tirs;	1523	goto err_destroy_indirect_tirs;
1524		1524
@@ -1535,11 +1535,11 @@ static int mlx5e_init_rep_rx(struct mlx5e_priv *priv) · 10 unmodified lines
1535	err_destroy_ttc_table:	1535	err_destroy_ttc_table:
1536	mlx5e_destroy_ttc_table(priv, &priv->fs.ttc);	1536	mlx5e_destroy_ttc_table(priv, &priv->fs.ttc);
1537	err_destroy_direct_tirs:	1537	err_destroy_direct_tirs:
1538	mlx5e_destroy_direct_tirs(priv);	1538	mlx5e_destroy_direct_tirs(priv, priv->direct_tir);
1539	err_destroy_indirect_tirs:	1539	err_destroy_indirect_tirs:
1540	mlx5e_destroy_indirect_tirs(priv, false);	1540	mlx5e_destroy_indirect_tirs(priv, false);
1541	err_destroy_direct_rqts:	1541	err_destroy_direct_rqts:
1542	mlx5e_destroy_direct_rqts(priv);	1542	mlx5e_destroy_direct_rqts(priv, priv->direct_tir);
1543	err_destroy_indirect_rqts:	1543	err_destroy_indirect_rqts:
1544	mlx5e_destroy_rqt(priv, &priv->indir_rqt);	1544	mlx5e_destroy_rqt(priv, &priv->indir_rqt);
1545	err_close_drop_rq:	1545	err_close_drop_rq:
@@ -1553,9 +1553,9 @@ static void mlx5e_cleanup_rep_rx(struct mlx5e_priv *priv) · 7 unmodified lines
1553		1553
1554	mlx5_del_flow_rules(rpriv->vport_rx_rule);	1554	mlx5_del_flow_rules(rpriv->vport_rx_rule);
1555	mlx5e_destroy_ttc_table(priv, &priv->fs.ttc);	1555	mlx5e_destroy_ttc_table(priv, &priv->fs.ttc);
1556	mlx5e_destroy_direct_tirs(priv);	1556	mlx5e_destroy_direct_tirs(priv, priv->direct_tir);
1557	mlx5e_destroy_indirect_tirs(priv, false);	1557	mlx5e_destroy_indirect_tirs(priv, false);
1558	mlx5e_destroy_direct_rqts(priv);	1558	mlx5e_destroy_direct_rqts(priv, priv->direct_tir);
1559	mlx5e_destroy_rqt(priv, &priv->indir_rqt);	1559	mlx5e_destroy_rqt(priv, &priv->indir_rqt);
1560	mlx5e_close_drop_rq(&priv->drop_rq);	1560	mlx5e_close_drop_rq(&priv->drop_rq);
1561	}	1561	}

drivers/net/ethernet/mellanox/mlx5/core/en_rx.c+76−28

 #include "en_accel/tls_rxtx.h"
 #include "lib/clock.h"
 #include "en/xdp.h"
+#include "en/xsk/rx.h"
 static inline bool mlx5e_rx_hw_stamp(struct hwtstamp_config *config)
 {
 	return true;
 }
-static inline int mlx5e_page_alloc_mapped(struct mlx5e_rq *rq,
+static inline int mlx5e_page_alloc_pool(struct mlx5e_rq *rq,
-					  struct mlx5e_dma_info *dma_info)
+					struct mlx5e_dma_info *dma_info)
 {
 	if (mlx5e_rx_cache_get(rq, dma_info))
 		return 0;
 	return 0;
 }
+static inline int mlx5e_page_alloc(struct mlx5e_rq *rq,
+				   struct mlx5e_dma_info *dma_info)
+{
+	if (rq->umem)
+		return mlx5e_xsk_page_alloc_umem(rq, dma_info);
+	else
+		return mlx5e_page_alloc_pool(rq, dma_info);
+}
 void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info)
 {
 	dma_unmap_page(rq->pdev, dma_info->addr, PAGE_SIZE, rq->buff.map_dir);
 }
-void mlx5e_page_release(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info,
+void mlx5e_page_release_dynamic(struct mlx5e_rq *rq,
-			bool recycle)
+				struct mlx5e_dma_info *dma_info,
+				bool recycle)
 {
 	if (likely(recycle)) {
 		if (mlx5e_rx_cache_put(rq, dma_info))
 	}
 }
+static inline void mlx5e_page_release(struct mlx5e_rq *rq,
+				      struct mlx5e_dma_info *dma_info,
+				      bool recycle)
+{
+	if (rq->umem)
+		/* The `recycle` parameter is ignored, and the page is always
+		 * put into the Reuse Ring, because there is no way to return
+		 * the page to the userspace when the interface goes down.
+		 */
+		mlx5e_xsk_page_release(rq, dma_info);
+	else
+		mlx5e_page_release_dynamic(rq, dma_info, recycle);
+}
 static inline int mlx5e_get_rx_frag(struct mlx5e_rq *rq,
 				    struct mlx5e_wqe_frag_info *frag)
 {
 		 * offset) should just use the new one without replenishing again
 		 * by themselves.
 		 */
-		err = mlx5e_page_alloc_mapped(rq, frag->di);
+		err = mlx5e_page_alloc(rq, frag->di);
 	return err;
 }
 	int err;
 	int i;
+	if (rq->umem) {
+		int pages_desired = wqe_bulk << rq->wqe.info.log_num_frags;
+		if (unlikely(!mlx5e_xsk_pages_enough_umem(rq, pages_desired)))
+			return -ENOMEM;
+	}
 	for (i = 0; i < wqe_bulk; i++) {
 		struct mlx5e_rx_wqe_cyc *wqe = mlx5_wq_cyc_get_wqe(wq, ix + i);
 static void
 mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, bool recycle)
 {
-	const bool no_xdp_xmit =
+	bool no_xdp_xmit;
-		bitmap_empty(wi->xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE);
 	struct mlx5e_dma_info *dma_info = wi->umr.dma_info;
 	int i;
+	/* A common case for AF_XDP. */
+	if (bitmap_full(wi->xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE))
+		return;
+	no_xdp_xmit = bitmap_empty(wi->xdp_xmit_bitmap,
+				   MLX5_MPWRQ_PAGES_PER_WQE);
 	for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++)
 		if (no_xdp_xmit || !test_bit(i, wi->xdp_xmit_bitmap))
 			mlx5e_page_release(rq, &dma_info[i], recycle);
 	mlx5_wq_ll_update_db_record(wq);
 }
-static inline u16 mlx5e_icosq_wrap_cnt(struct mlx5e_icosq *sq)
-{
-	return mlx5_wq_cyc_get_ctr_wrap_cnt(&sq->wq, sq->pc);
-}
 static inline void mlx5e_fill_icosq_frag_edge(struct mlx5e_icosq *sq,
 					      struct mlx5_wq_cyc *wq,
 					      u16 pi, u16 nnops)
 	int err;
 	int i;
+	if (rq->umem &&
+	    unlikely(!mlx5e_xsk_pages_enough_umem(rq, MLX5_MPWRQ_PAGES_PER_WQE))) {
+		err = -ENOMEM;
+		goto err;
+	}
 	pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
 	contig_wqebbs_room = mlx5_wq_cyc_get_contig_wqebbs(wq, pi);
 	if (unlikely(contig_wqebbs_room < MLX5E_UMR_WQEBBS)) {
 	}
 	umr_wqe = mlx5_wq_cyc_get_wqe(wq, pi);
-	if (unlikely(mlx5e_icosq_wrap_cnt(sq) < 2))
+	memcpy(umr_wqe, &rq->mpwqe.umr_wqe, offsetof(struct mlx5e_umr_wqe, inline_mtts));
-		memcpy(umr_wqe, &rq->mpwqe.umr_wqe,
-		       offsetof(struct mlx5e_umr_wqe, inline_mtts));
 	for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++, dma_info++) {
-		err = mlx5e_page_alloc_mapped(rq, dma_info);
+		err = mlx5e_page_alloc(rq, dma_info);
 		if (unlikely(err))
 			goto err_unmap;
 		umr_wqe->inline_mtts[i].ptag = cpu_to_be64(dma_info->addr | MLX5_EN_WR);
 	umr_wqe->uctrl.xlt_offset = cpu_to_be16(xlt_offset);
 	sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_UMR;
+	sq->db.ico_wqe[pi].umr.rq = rq;
 	sq->pc += MLX5E_UMR_WQEBBS;
 	sq->doorbell_cseg = &umr_wqe->ctrl;
 		dma_info--;
 		mlx5e_page_release(rq, dma_info, true);
 	}
+err:
 	rq->stats->buff_alloc_err++;
 	return err;
 	return !!err;
 }
-static void mlx5e_poll_ico_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq)
+void mlx5e_poll_ico_cq(struct mlx5e_cq *cq)
 {
 	struct mlx5e_icosq *sq = container_of(cq, struct mlx5e_icosq, cq);
 	struct mlx5_cqe64 *cqe;
-	u8  completed_umr = 0;
 	u16 sqcc;
 	int i;
 			if (likely(wi->opcode == MLX5_OPCODE_UMR)) {
 				sqcc += MLX5E_UMR_WQEBBS;
-				completed_umr++;
+				wi->umr.rq->mpwqe.umr_completed++;
 			} else if (likely(wi->opcode == MLX5_OPCODE_NOP)) {
 				sqcc++;
 			} else {
 	sq->cc = sqcc;
 	mlx5_cqwq_update_db_record(&cq->wq);
-	if (likely(completed_umr)) {
-		mlx5e_post_rx_mpwqe(rq, completed_umr);
-		rq->mpwqe.umr_in_progress -= completed_umr;
-	}
 }
 bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq)
 {
 	struct mlx5e_icosq *sq = &rq->channel->icosq;
 	struct mlx5_wq_ll *wq = &rq->mpwqe.wq;
+	u8  umr_completed = rq->mpwqe.umr_completed;
+	int alloc_err = 0;
 	u8  missing, i;
 	u16 head;
 	if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state)))
 		return false;
-	mlx5e_poll_ico_cq(&sq->cq, rq);
+	if (umr_completed) {
+		mlx5e_post_rx_mpwqe(rq, umr_completed);
+		rq->mpwqe.umr_in_progress -= umr_completed;
+		rq->mpwqe.umr_completed = 0;
+	}
 	missing = mlx5_wq_ll_missing(wq) - rq->mpwqe.umr_in_progress;
 	head = rq->mpwqe.actual_wq_head;
 	i = missing;
 	do {
-		if (unlikely(mlx5e_alloc_rx_mpwqe(rq, head)))
+		alloc_err = mlx5e_alloc_rx_mpwqe(rq, head);
+		if (unlikely(alloc_err))
 			break;
 		head = mlx5_wq_ll_get_wqe_next_ix(wq, head);
 	} while (--i);
 	rq->mpwqe.umr_in_progress += rq->mpwqe.umr_last_bulk;
 	rq->mpwqe.actual_wq_head   = head;
+	/* If XSK Fill Ring doesn't have enough frames, busy poll by
+	 * rescheduling the NAPI poll.
+	 */
+	if (unlikely(alloc_err == -ENOMEM && rq->umem))
+		return true;
 	return false;
 }
 	}
 	rcu_read_lock();
-	consumed = mlx5e_xdp_handle(rq, di, va, &rx_headroom, &cqe_bcnt);
+	consumed = mlx5e_xdp_handle(rq, di, va, &rx_headroom, &cqe_bcnt, false);
 	rcu_read_unlock();
 	if (consumed)
 		return NULL; /* page/packet was consumed by XDP */
 	prefetch(data);
 	rcu_read_lock();
-	consumed = mlx5e_xdp_handle(rq, di, va, &rx_headroom, &cqe_bcnt32);
+	consumed = mlx5e_xdp_handle(rq, di, va, &rx_headroom, &cqe_bcnt32, false);
 	rcu_read_unlock();
 	if (consumed) {
 		if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags))

drivers/net/ethernet/mellanox/mlx5/core/en_stats.c+112−3

@@ -104,7 +104,33 @@ static const struct counter_desc sw_stats_desc[] = { · 103 unmodified lines
104	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_poll) },	104	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_poll) },
105	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_arm) },	105	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_arm) },
106	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_aff_change) },	106	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_aff_change) },
		107	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_force_irq) },
107	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_eq_rearm) },	108	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_eq_rearm) },
		109	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_packets) },
		110	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_bytes) },
		111	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_csum_complete) },
		112	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_csum_unnecessary) },
		113	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_csum_unnecessary_inner) },
		114	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_csum_none) },
		115	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_ecn_mark) },
		116	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_removed_vlan_packets) },
		117	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_xdp_drop) },
		118	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_xdp_redirect) },
		119	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_wqe_err) },
		120	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_mpwqe_filler_cqes) },
		121	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_mpwqe_filler_strides) },
		122	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_oversize_pkts_sw_drop) },
		123	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_buff_alloc_err) },
		124	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_cqe_compress_blks) },
		125	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_cqe_compress_pkts) },
		126	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_congst_umr) },
		127	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_arfs_err) },
		128	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xsk_xmit) },
		129	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xsk_mpwqe) },
		130	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xsk_inlnw) },
		131	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xsk_full) },
		132	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xsk_err) },
		133	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xsk_cqes) },
108	};	134	};
109		135
110	#define NUM_SW_COUNTERS ARRAY_SIZE(sw_stats_desc)	136	#define NUM_SW_COUNTERS ARRAY_SIZE(sw_stats_desc)
@@ -144,6 +170,8 @@ static void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv) · 33 unmodified lines
144	&priv->channel_stats[i];	170	&priv->channel_stats[i];
145	struct mlx5e_xdpsq_stats *xdpsq_red_stats = &channel_stats->xdpsq;	171	struct mlx5e_xdpsq_stats *xdpsq_red_stats = &channel_stats->xdpsq;
146	struct mlx5e_xdpsq_stats *xdpsq_stats = &channel_stats->rq_xdpsq;	172	struct mlx5e_xdpsq_stats *xdpsq_stats = &channel_stats->rq_xdpsq;
		173	struct mlx5e_xdpsq_stats *xsksq_stats = &channel_stats->xsksq;
		174	struct mlx5e_rq_stats *xskrq_stats = &channel_stats->xskrq;
147	struct mlx5e_rq_stats *rq_stats = &channel_stats->rq;	175	struct mlx5e_rq_stats *rq_stats = &channel_stats->rq;
148	struct mlx5e_ch_stats *ch_stats = &channel_stats->ch;	176	struct mlx5e_ch_stats *ch_stats = &channel_stats->ch;
149	int j;	177	int j;
@@ -186,6 +214,7 @@ static void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv) · 36 unmodified lines
186	s->ch_poll += ch_stats->poll;	214	s->ch_poll += ch_stats->poll;
187	s->ch_arm += ch_stats->arm;	215	s->ch_arm += ch_stats->arm;
188	s->ch_aff_change += ch_stats->aff_change;	216	s->ch_aff_change += ch_stats->aff_change;
		217	s->ch_force_irq += ch_stats->force_irq;
189	s->ch_eq_rearm += ch_stats->eq_rearm;	218	s->ch_eq_rearm += ch_stats->eq_rearm;
190	/* xdp redirect */	219	/* xdp redirect */
191	s->tx_xdp_xmit += xdpsq_red_stats->xmit;	220	s->tx_xdp_xmit += xdpsq_red_stats->xmit;
@@ -194,6 +223,32 @@ static void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv) · 2 unmodified lines
194	s->tx_xdp_full += xdpsq_red_stats->full;	223	s->tx_xdp_full += xdpsq_red_stats->full;
195	s->tx_xdp_err += xdpsq_red_stats->err;	224	s->tx_xdp_err += xdpsq_red_stats->err;
196	s->tx_xdp_cqes += xdpsq_red_stats->cqes;	225	s->tx_xdp_cqes += xdpsq_red_stats->cqes;
		226	/* AF_XDP zero-copy */
		227	s->rx_xsk_packets += xskrq_stats->packets;
		228	s->rx_xsk_bytes += xskrq_stats->bytes;
		229	s->rx_xsk_csum_complete += xskrq_stats->csum_complete;
		230	s->rx_xsk_csum_unnecessary += xskrq_stats->csum_unnecessary;
		231	s->rx_xsk_csum_unnecessary_inner += xskrq_stats->csum_unnecessary_inner;
		232	s->rx_xsk_csum_none += xskrq_stats->csum_none;
		233	s->rx_xsk_ecn_mark += xskrq_stats->ecn_mark;
		234	s->rx_xsk_removed_vlan_packets += xskrq_stats->removed_vlan_packets;
		235	s->rx_xsk_xdp_drop += xskrq_stats->xdp_drop;
		236	s->rx_xsk_xdp_redirect += xskrq_stats->xdp_redirect;
		237	s->rx_xsk_wqe_err += xskrq_stats->wqe_err;
		238	s->rx_xsk_mpwqe_filler_cqes += xskrq_stats->mpwqe_filler_cqes;
		239	s->rx_xsk_mpwqe_filler_strides += xskrq_stats->mpwqe_filler_strides;
		240	s->rx_xsk_oversize_pkts_sw_drop += xskrq_stats->oversize_pkts_sw_drop;
		241	s->rx_xsk_buff_alloc_err += xskrq_stats->buff_alloc_err;
		242	s->rx_xsk_cqe_compress_blks += xskrq_stats->cqe_compress_blks;
		243	s->rx_xsk_cqe_compress_pkts += xskrq_stats->cqe_compress_pkts;
		244	s->rx_xsk_congst_umr += xskrq_stats->congst_umr;
		245	s->rx_xsk_arfs_err += xskrq_stats->arfs_err;
		246	s->tx_xsk_xmit += xsksq_stats->xmit;
		247	s->tx_xsk_mpwqe += xsksq_stats->mpwqe;
		248	s->tx_xsk_inlnw += xsksq_stats->inlnw;
		249	s->tx_xsk_full += xsksq_stats->full;
		250	s->tx_xsk_err += xsksq_stats->err;
		251	s->tx_xsk_cqes += xsksq_stats->cqes;
197		252
198	for (j = 0; j < priv->max_opened_tc; j++) {	253	for (j = 0; j < priv->max_opened_tc; j++) {
199	struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[j];	254	struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[j];
@@ -1266,11 +1321,43 @@ static const struct counter_desc xdpsq_stats_desc[] = { · 1066 unmodified lines
1266	{ MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, cqes) },	1321	{ MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, cqes) },
1267	};	1322	};
1268		1323
		1324	static const struct counter_desc xskrq_stats_desc[] = {
		1325	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, packets) },
		1326	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, bytes) },
		1327	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, csum_complete) },
		1328	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, csum_unnecessary) },
		1329	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, csum_unnecessary_inner) },
		1330	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, csum_none) },
		1331	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, ecn_mark) },
		1332	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, removed_vlan_packets) },
		1333	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, xdp_drop) },
		1334	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, xdp_redirect) },
		1335	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, wqe_err) },
		1336	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, mpwqe_filler_cqes) },
		1337	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, mpwqe_filler_strides) },
		1338	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, oversize_pkts_sw_drop) },
		1339	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, buff_alloc_err) },
		1340	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, cqe_compress_blks) },
		1341	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, cqe_compress_pkts) },
		1342	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, congst_umr) },
		1343	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, arfs_err) },
		1344	};
		1345
		1346	static const struct counter_desc xsksq_stats_desc[] = {
		1347	{ MLX5E_DECLARE_XSKSQ_STAT(struct mlx5e_xdpsq_stats, xmit) },
		1348	{ MLX5E_DECLARE_XSKSQ_STAT(struct mlx5e_xdpsq_stats, mpwqe) },
		1349	{ MLX5E_DECLARE_XSKSQ_STAT(struct mlx5e_xdpsq_stats, inlnw) },
		1350	{ MLX5E_DECLARE_XSKSQ_STAT(struct mlx5e_xdpsq_stats, full) },
		1351	{ MLX5E_DECLARE_XSKSQ_STAT(struct mlx5e_xdpsq_stats, err) },
		1352	{ MLX5E_DECLARE_XSKSQ_STAT(struct mlx5e_xdpsq_stats, cqes) },
		1353	};
		1354
1269	static const struct counter_desc ch_stats_desc[] = {	1355	static const struct counter_desc ch_stats_desc[] = {
1270	{ MLX5E_DECLARE_CH_STAT(struct mlx5e_ch_stats, events) },	1356	{ MLX5E_DECLARE_CH_STAT(struct mlx5e_ch_stats, events) },
1271	{ MLX5E_DECLARE_CH_STAT(struct mlx5e_ch_stats, poll) },	1357	{ MLX5E_DECLARE_CH_STAT(struct mlx5e_ch_stats, poll) },
1272	{ MLX5E_DECLARE_CH_STAT(struct mlx5e_ch_stats, arm) },	1358	{ MLX5E_DECLARE_CH_STAT(struct mlx5e_ch_stats, arm) },
1273	{ MLX5E_DECLARE_CH_STAT(struct mlx5e_ch_stats, aff_change) },	1359	{ MLX5E_DECLARE_CH_STAT(struct mlx5e_ch_stats, aff_change) },
		1360	{ MLX5E_DECLARE_CH_STAT(struct mlx5e_ch_stats, force_irq) },
1274	{ MLX5E_DECLARE_CH_STAT(struct mlx5e_ch_stats, eq_rearm) },	1361	{ MLX5E_DECLARE_CH_STAT(struct mlx5e_ch_stats, eq_rearm) },
1275	};	1362	};
1276		1363
@@ -1278,6 +1365,8 @@ static const struct counter_desc ch_stats_desc[] = { · 1 unmodified line
1278	#define NUM_SQ_STATS ARRAY_SIZE(sq_stats_desc)	1365	#define NUM_SQ_STATS ARRAY_SIZE(sq_stats_desc)
1279	#define NUM_XDPSQ_STATS ARRAY_SIZE(xdpsq_stats_desc)	1366	#define NUM_XDPSQ_STATS ARRAY_SIZE(xdpsq_stats_desc)
1280	#define NUM_RQ_XDPSQ_STATS ARRAY_SIZE(rq_xdpsq_stats_desc)	1367	#define NUM_RQ_XDPSQ_STATS ARRAY_SIZE(rq_xdpsq_stats_desc)
		1368	#define NUM_XSKRQ_STATS ARRAY_SIZE(xskrq_stats_desc)
		1369	#define NUM_XSKSQ_STATS ARRAY_SIZE(xsksq_stats_desc)
1281	#define NUM_CH_STATS ARRAY_SIZE(ch_stats_desc)	1370	#define NUM_CH_STATS ARRAY_SIZE(ch_stats_desc)
1282		1371
1283	static int mlx5e_grp_channels_get_num_stats(struct mlx5e_priv *priv)	1372	static int mlx5e_grp_channels_get_num_stats(struct mlx5e_priv *priv)
@@ -1288,13 +1377,16 @@ static int mlx5e_grp_channels_get_num_stats(struct mlx5e_priv *priv) · 4 unmodified lines
1288	(NUM_CH_STATS * max_nch) +	1377	(NUM_CH_STATS * max_nch) +
1289	(NUM_SQ_STATS * max_nch * priv->max_opened_tc) +	1378	(NUM_SQ_STATS * max_nch * priv->max_opened_tc) +
1290	(NUM_RQ_XDPSQ_STATS * max_nch) +	1379	(NUM_RQ_XDPSQ_STATS * max_nch) +
1291	(NUM_XDPSQ_STATS * max_nch);	1380	(NUM_XDPSQ_STATS * max_nch) +
		1381	(NUM_XSKRQ_STATS * max_nch * priv->xsk.ever_used) +
		1382	(NUM_XSKSQ_STATS * max_nch * priv->xsk.ever_used);
1292	}	1383	}
1293		1384
1294	static int mlx5e_grp_channels_fill_strings(struct mlx5e_priv priv, u8 data,	1385	static int mlx5e_grp_channels_fill_strings(struct mlx5e_priv priv, u8 data,
1295	int idx)	1386	int idx)
1296	{	1387	{
1297	int max_nch = mlx5e_get_netdev_max_channels(priv->netdev);	1388	int max_nch = mlx5e_get_netdev_max_channels(priv->netdev);
		1389	bool is_xsk = priv->xsk.ever_used;
1298	int i, j, tc;	1390	int i, j, tc;
1299		1391
1300	for (i = 0; i < max_nch; i++)	1392	for (i = 0; i < max_nch; i++)
@@ -1306,6 +1398,9 @@ static int mlx5e_grp_channels_fill_strings(struct mlx5e_priv priv, u8 data, · 5 unmodified lines
1306	for (j = 0; j < NUM_RQ_STATS; j++)	1398	for (j = 0; j < NUM_RQ_STATS; j++)
1307	sprintf(data + (idx++) * ETH_GSTRING_LEN,	1399	sprintf(data + (idx++) * ETH_GSTRING_LEN,
1308	rq_stats_desc[j].format, i);	1400	rq_stats_desc[j].format, i);
		1401	for (j = 0; j < NUM_XSKRQ_STATS * is_xsk; j++)
		1402	sprintf(data + (idx++) * ETH_GSTRING_LEN,
		1403	xskrq_stats_desc[j].format, i);
1309	for (j = 0; j < NUM_RQ_XDPSQ_STATS; j++)	1404	for (j = 0; j < NUM_RQ_XDPSQ_STATS; j++)
1310	sprintf(data + (idx++) * ETH_GSTRING_LEN,	1405	sprintf(data + (idx++) * ETH_GSTRING_LEN,
1311	rq_xdpsq_stats_desc[j].format, i);	1406	rq_xdpsq_stats_desc[j].format, i);
@@ -1318,10 +1413,14 @@ static int mlx5e_grp_channels_fill_strings(struct mlx5e_priv priv, u8 data, · 6 unmodified lines
1318	sq_stats_desc[j].format,	1413	sq_stats_desc[j].format,
1319	priv->channel_tc2txq[i][tc]);	1414	priv->channel_tc2txq[i][tc]);
1320		1415
1321	for (i = 0; i < max_nch; i++)	1416	for (i = 0; i < max_nch; i++) {
		1417	for (j = 0; j < NUM_XSKSQ_STATS * is_xsk; j++)
		1418	sprintf(data + (idx++) * ETH_GSTRING_LEN,
		1419	xsksq_stats_desc[j].format, i);
1322	for (j = 0; j < NUM_XDPSQ_STATS; j++)	1420	for (j = 0; j < NUM_XDPSQ_STATS; j++)
1323	sprintf(data + (idx++) * ETH_GSTRING_LEN,	1421	sprintf(data + (idx++) * ETH_GSTRING_LEN,
1324	xdpsq_stats_desc[j].format, i);	1422	xdpsq_stats_desc[j].format, i);
		1423	}
1325		1424
1326	return idx;	1425	return idx;
1327	}	1426	}
@@ -1330,6 +1429,7 @@ static int mlx5e_grp_channels_fill_stats(struct mlx5e_priv priv, u64 data, · 2 unmodified lines
1330	int idx)	1429	int idx)
1331	{	1430	{
1332	int max_nch = mlx5e_get_netdev_max_channels(priv->netdev);	1431	int max_nch = mlx5e_get_netdev_max_channels(priv->netdev);
		1432	bool is_xsk = priv->xsk.ever_used;
1333	int i, j, tc;	1433	int i, j, tc;
1334		1434
1335	for (i = 0; i < max_nch; i++)	1435	for (i = 0; i < max_nch; i++)
@@ -1343,6 +1443,10 @@ static int mlx5e_grp_channels_fill_stats(struct mlx5e_priv priv, u64 data, · 7 unmodified lines
1343	data[idx++] =	1443	data[idx++] =
1344	MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].rq,	1444	MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].rq,
1345	rq_stats_desc, j);	1445	rq_stats_desc, j);
		1446	for (j = 0; j < NUM_XSKRQ_STATS * is_xsk; j++)
		1447	data[idx++] =
		1448	MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].xskrq,
		1449	xskrq_stats_desc, j);
1346	for (j = 0; j < NUM_RQ_XDPSQ_STATS; j++)	1450	for (j = 0; j < NUM_RQ_XDPSQ_STATS; j++)
1347	data[idx++] =	1451	data[idx++] =
1348	MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].rq_xdpsq,	1452	MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].rq_xdpsq,
@@ -1356,11 +1460,16 @@ static int mlx5e_grp_channels_fill_stats(struct mlx5e_priv priv, u64 data, · 7 unmodified lines
1356	MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].sq[tc],	1460	MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].sq[tc],
1357	sq_stats_desc, j);	1461	sq_stats_desc, j);
1358		1462
1359	for (i = 0; i < max_nch; i++)	1463	for (i = 0; i < max_nch; i++) {
		1464	for (j = 0; j < NUM_XSKSQ_STATS * is_xsk; j++)
		1465	data[idx++] =
		1466	MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].xsksq,
		1467	xsksq_stats_desc, j);
1360	for (j = 0; j < NUM_XDPSQ_STATS; j++)	1468	for (j = 0; j < NUM_XDPSQ_STATS; j++)
1361	data[idx++] =	1469	data[idx++] =
1362	MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].xdpsq,	1470	MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].xdpsq,
1363	xdpsq_stats_desc, j);	1471	xdpsq_stats_desc, j);
		1472	}
1364		1473
1365	return idx;	1474	return idx;
1366	}	1475	}

drivers/net/ethernet/mellanox/mlx5/core/en_stats.h+30−0

@@ -46,6 +46,8 @@ · 45 unmodified lines
46	#define MLX5E_DECLARE_TX_STAT(type, fld) "tx%d_"#fld, offsetof(type, fld)	46	#define MLX5E_DECLARE_TX_STAT(type, fld) "tx%d_"#fld, offsetof(type, fld)
47	#define MLX5E_DECLARE_XDPSQ_STAT(type, fld) "tx%d_xdp_"#fld, offsetof(type, fld)	47	#define MLX5E_DECLARE_XDPSQ_STAT(type, fld) "tx%d_xdp_"#fld, offsetof(type, fld)
48	#define MLX5E_DECLARE_RQ_XDPSQ_STAT(type, fld) "rx%d_xdp_tx_"#fld, offsetof(type, fld)	48	#define MLX5E_DECLARE_RQ_XDPSQ_STAT(type, fld) "rx%d_xdp_tx_"#fld, offsetof(type, fld)
		49	#define MLX5E_DECLARE_XSKRQ_STAT(type, fld) "rx%d_xsk_"#fld, offsetof(type, fld)
		50	#define MLX5E_DECLARE_XSKSQ_STAT(type, fld) "tx%d_xsk_"#fld, offsetof(type, fld)
49	#define MLX5E_DECLARE_CH_STAT(type, fld) "ch%d_"#fld, offsetof(type, fld)	51	#define MLX5E_DECLARE_CH_STAT(type, fld) "ch%d_"#fld, offsetof(type, fld)
50		52
51	struct counter_desc {	53	struct counter_desc {
@@ -116,12 +118,39 @@ struct mlx5e_sw_stats { · 64 unmodified lines
116	u64 ch_poll;	118	u64 ch_poll;
117	u64 ch_arm;	119	u64 ch_arm;
118	u64 ch_aff_change;	120	u64 ch_aff_change;
		121	u64 ch_force_irq;
119	u64 ch_eq_rearm;	122	u64 ch_eq_rearm;
120		123
121	#ifdef CONFIG_MLX5_EN_TLS	124	#ifdef CONFIG_MLX5_EN_TLS
122	u64 tx_tls_ooo;	125	u64 tx_tls_ooo;
123	u64 tx_tls_resync_bytes;	126	u64 tx_tls_resync_bytes;
124	#endif	127	#endif
		128
		129	u64 rx_xsk_packets;
		130	u64 rx_xsk_bytes;
		131	u64 rx_xsk_csum_complete;
		132	u64 rx_xsk_csum_unnecessary;
		133	u64 rx_xsk_csum_unnecessary_inner;
		134	u64 rx_xsk_csum_none;
		135	u64 rx_xsk_ecn_mark;
		136	u64 rx_xsk_removed_vlan_packets;
		137	u64 rx_xsk_xdp_drop;
		138	u64 rx_xsk_xdp_redirect;
		139	u64 rx_xsk_wqe_err;
		140	u64 rx_xsk_mpwqe_filler_cqes;
		141	u64 rx_xsk_mpwqe_filler_strides;
		142	u64 rx_xsk_oversize_pkts_sw_drop;
		143	u64 rx_xsk_buff_alloc_err;
		144	u64 rx_xsk_cqe_compress_blks;
		145	u64 rx_xsk_cqe_compress_pkts;
		146	u64 rx_xsk_congst_umr;
		147	u64 rx_xsk_arfs_err;
		148	u64 tx_xsk_xmit;
		149	u64 tx_xsk_mpwqe;
		150	u64 tx_xsk_inlnw;
		151	u64 tx_xsk_full;
		152	u64 tx_xsk_err;
		153	u64 tx_xsk_cqes;
125	};	154	};
126		155
127	struct mlx5e_qcounter_stats {	156	struct mlx5e_qcounter_stats {
@@ -256,6 +285,7 @@ struct mlx5e_ch_stats { · 128 unmodified lines
256	u64 poll;	285	u64 poll;
257	u64 arm;	286	u64 arm;
258	u64 aff_change;	287	u64 aff_change;
		288	u64 force_irq;
259	u64 eq_rearm;	289	u64 eq_rearm;
260	};	290	};
261		291

drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c+38−4

 #include <linux/irq.h>
 #include "en.h"
 #include "en/xdp.h"
+#include "en/xsk/tx.h"
 static inline bool mlx5e_channel_no_affinity_change(struct mlx5e_channel *c)
 {
 	struct mlx5e_channel *c = container_of(napi, struct mlx5e_channel,
 					       napi);
 	struct mlx5e_ch_stats *ch_stats = c->stats;
+	struct mlx5e_xdpsq *xsksq = &c->xsksq;
+	struct mlx5e_rq *xskrq = &c->xskrq;
 	struct mlx5e_rq *rq = &c->rq;
+	bool xsk_open = test_bit(MLX5E_CHANNEL_STATE_XSK, c->state);
+	bool aff_change = false;
+	bool busy_xsk = false;
 	bool busy = false;
 	int work_done = 0;
 	int i;
 	for (i = 0; i < c->num_tc; i++)
 		busy |= mlx5e_poll_tx_cq(&c->sq[i].cq, budget);
-	busy |= mlx5e_poll_xdpsq_cq(&c->xdpsq.cq, NULL);
+	busy |= mlx5e_poll_xdpsq_cq(&c->xdpsq.cq);
 	if (c->xdp)
-		busy |= mlx5e_poll_xdpsq_cq(&rq->xdpsq.cq, rq);
+		busy |= mlx5e_poll_xdpsq_cq(&c->rq_xdpsq.cq);
 	if (likely(budget)) { /* budget=0 means: don't poll rx rings */
-		work_done = mlx5e_poll_rx_cq(&rq->cq, budget);
+		if (xsk_open)
+			work_done = mlx5e_poll_rx_cq(&xskrq->cq, budget);
+		if (likely(budget - work_done))
+			work_done += mlx5e_poll_rx_cq(&rq->cq, budget - work_done);
 		busy |= work_done == budget;
 	}
-	busy |= c->rq.post_wqes(rq);
+	mlx5e_poll_ico_cq(&c->icosq.cq);
+	busy |= rq->post_wqes(rq);
+	if (xsk_open) {
+		mlx5e_poll_ico_cq(&c->xskicosq.cq);
+		busy |= mlx5e_poll_xdpsq_cq(&xsksq->cq);
+		busy_xsk |= mlx5e_xsk_tx(xsksq, MLX5E_TX_XSK_POLL_BUDGET);
+		busy_xsk |= xskrq->post_wqes(xskrq);
+	}
+	busy |= busy_xsk;
 	if (busy) {
 		if (likely(mlx5e_channel_no_affinity_change(c)))
 			return budget;
 		ch_stats->aff_change++;
+		aff_change = true;
 		if (budget && work_done == budget)
 			work_done--;
 	}
 	mlx5e_cq_arm(&c->icosq.cq);
 	mlx5e_cq_arm(&c->xdpsq.cq);
+	if (xsk_open) {
+		mlx5e_handle_rx_dim(xskrq);
+		mlx5e_cq_arm(&c->xskicosq.cq);
+		mlx5e_cq_arm(&xsksq->cq);
+		mlx5e_cq_arm(&xskrq->cq);
+	}
+	if (unlikely(aff_change && busy_xsk)) {
+		mlx5e_trigger_irq(&c->icosq);
+		ch_stats->force_irq++;
+	}
 	return work_done;
 }

drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c+7−7

@@ -87,7 +87,7 @@ int mlx5i_init(struct mlx5_core_dev *mdev, · 86 unmodified lines
87	mlx5e_set_netdev_mtu_boundaries(priv);	87	mlx5e_set_netdev_mtu_boundaries(priv);
88	netdev->mtu = netdev->max_mtu;	88	netdev->mtu = netdev->max_mtu;
89		89
90	mlx5e_build_nic_params(mdev, &priv->rss_params, &priv->channels.params,	90	mlx5e_build_nic_params(mdev, NULL, &priv->rss_params, &priv->channels.params,
91	mlx5e_get_netdev_max_channels(netdev),	91	mlx5e_get_netdev_max_channels(netdev),
92	netdev->mtu);	92	netdev->mtu);
93	mlx5i_build_nic_params(mdev, &priv->channels.params);	93	mlx5i_build_nic_params(mdev, &priv->channels.params);
@@ -365,7 +365,7 @@ static int mlx5i_init_rx(struct mlx5e_priv *priv) · 271 unmodified lines
365	if (err)	365	if (err)
366	goto err_close_drop_rq;	366	goto err_close_drop_rq;
367		367
368	err = mlx5e_create_direct_rqts(priv);	368	err = mlx5e_create_direct_rqts(priv, priv->direct_tir);
369	if (err)	369	if (err)
370	goto err_destroy_indirect_rqts;	370	goto err_destroy_indirect_rqts;
371		371
@@ -373,7 +373,7 @@ static int mlx5i_init_rx(struct mlx5e_priv *priv) · 1 unmodified line
373	if (err)	373	if (err)
374	goto err_destroy_direct_rqts;	374	goto err_destroy_direct_rqts;
375		375
376	err = mlx5e_create_direct_tirs(priv);	376	err = mlx5e_create_direct_tirs(priv, priv->direct_tir);
377	if (err)	377	if (err)
378	goto err_destroy_indirect_tirs;	378	goto err_destroy_indirect_tirs;
379		379
@@ -384,11 +384,11 @@ static int mlx5i_init_rx(struct mlx5e_priv *priv) · 4 unmodified lines
384	return 0;	384	return 0;
385		385
386	err_destroy_direct_tirs:	386	err_destroy_direct_tirs:
387	mlx5e_destroy_direct_tirs(priv);	387	mlx5e_destroy_direct_tirs(priv, priv->direct_tir);
388	err_destroy_indirect_tirs:	388	err_destroy_indirect_tirs:
389	mlx5e_destroy_indirect_tirs(priv, true);	389	mlx5e_destroy_indirect_tirs(priv, true);
390	err_destroy_direct_rqts:	390	err_destroy_direct_rqts:
391	mlx5e_destroy_direct_rqts(priv);	391	mlx5e_destroy_direct_rqts(priv, priv->direct_tir);
392	err_destroy_indirect_rqts:	392	err_destroy_indirect_rqts:
393	mlx5e_destroy_rqt(priv, &priv->indir_rqt);	393	mlx5e_destroy_rqt(priv, &priv->indir_rqt);
394	err_close_drop_rq:	394	err_close_drop_rq:
@@ -401,9 +401,9 @@ err_destroy_q_counters: · 6 unmodified lines
401	static void mlx5i_cleanup_rx(struct mlx5e_priv *priv)	401	static void mlx5i_cleanup_rx(struct mlx5e_priv *priv)
402	{	402	{
403	mlx5i_destroy_flow_steering(priv);	403	mlx5i_destroy_flow_steering(priv);
404	mlx5e_destroy_direct_tirs(priv);	404	mlx5e_destroy_direct_tirs(priv, priv->direct_tir);
405	mlx5e_destroy_indirect_tirs(priv, true);	405	mlx5e_destroy_indirect_tirs(priv, true);
406	mlx5e_destroy_direct_rqts(priv);	406	mlx5e_destroy_direct_rqts(priv, priv->direct_tir);
407	mlx5e_destroy_rqt(priv, &priv->indir_rqt);	407	mlx5e_destroy_rqt(priv, &priv->indir_rqt);
408	mlx5e_close_drop_rq(&priv->drop_rq);	408	mlx5e_close_drop_rq(&priv->drop_rq);
409	mlx5e_destroy_q_counters(priv);	409	mlx5e_destroy_q_counters(priv);

drivers/net/ethernet/mellanox/mlx5/core/wq.h+0−5

 	*wq->db = cpu_to_be32(wq->wqe_ctr);
 }
-static inline u16 mlx5_wq_cyc_get_ctr_wrap_cnt(struct mlx5_wq_cyc *wq, u16 ctr)
-{
-	return ctr >> wq->fbc.log_sz;
-}
 static inline u16 mlx5_wq_cyc_ctr2ix(struct mlx5_wq_cyc *wq, u16 ctr)
 {
 	return ctr & wq->fbc.sz_m1;

drivers/net/veth.c+48−12

 #define VETH_XDP_TX		BIT(0)
 #define VETH_XDP_REDIR		BIT(1)
+#define VETH_XDP_TX_BULK_SIZE	16
 struct veth_rq_stats {
 	u64			xdp_packets;
 	u64			xdp_bytes;
 	unsigned int		requested_headroom;
 };
+struct veth_xdp_tx_bq {
+	struct xdp_frame *q[VETH_XDP_TX_BULK_SIZE];
+	unsigned int count;
+};
 /*
  * ethtool interface
  */
 	return ret;
 }
-static void veth_xdp_flush(struct net_device *dev)
+static void veth_xdp_flush_bq(struct net_device *dev, struct veth_xdp_tx_bq *bq)
+{
+	int sent, i, err = 0;
+	sent = veth_xdp_xmit(dev, bq->count, bq->q, 0);
+	if (sent < 0) {
+		err = sent;
+		sent = 0;
+		for (i = 0; i < bq->count; i++)
+			xdp_return_frame(bq->q[i]);
+	}
+	trace_xdp_bulk_tx(dev, sent, bq->count - sent, err);
+	bq->count = 0;
+}
+static void veth_xdp_flush(struct net_device *dev, struct veth_xdp_tx_bq *bq)
 {
 	struct veth_priv *rcv_priv, *priv = netdev_priv(dev);
 	struct net_device *rcv;
 	struct veth_rq *rq;
 	rcu_read_lock();
+	veth_xdp_flush_bq(dev, bq);
 	rcv = rcu_dereference(priv->peer);
 	if (unlikely(!rcv))
 		goto out;
 	rcu_read_unlock();
 }
-static int veth_xdp_tx(struct net_device *dev, struct xdp_buff *xdp)
+static int veth_xdp_tx(struct net_device *dev, struct xdp_buff *xdp,
+		       struct veth_xdp_tx_bq *bq)
 {
 	struct xdp_frame *frame = convert_to_xdp_frame(xdp);
 	if (unlikely(!frame))
 		return -EOVERFLOW;
-	return veth_xdp_xmit(dev, 1, &frame, 0);
+	if (unlikely(bq->count == VETH_XDP_TX_BULK_SIZE))
+		veth_xdp_flush_bq(dev, bq);
+	bq->q[bq->count++] = frame;
+	return 0;
 }
 static struct sk_buff *veth_xdp_rcv_one(struct veth_rq *rq,
 					struct xdp_frame *frame,
-					unsigned int *xdp_xmit)
+					unsigned int *xdp_xmit,
+					struct veth_xdp_tx_bq *bq)
 {
 	void *hard_start = frame->data - frame->headroom;
 	void *head = hard_start - sizeof(struct xdp_frame);
 			orig_frame = *frame;
 			xdp.data_hard_start = head;
 			xdp.rxq->mem = frame->mem;
-			if (unlikely(veth_xdp_tx(rq->dev, &xdp) < 0)) {
+			if (unlikely(veth_xdp_tx(rq->dev, &xdp, bq) < 0)) {
 				trace_xdp_exception(rq->dev, xdp_prog, act);
 				frame = &orig_frame;
 				goto err_xdp;
 }
 static struct sk_buff *veth_xdp_rcv_skb(struct veth_rq *rq, struct sk_buff *skb,
-					unsigned int *xdp_xmit)
+					unsigned int *xdp_xmit,
+					struct veth_xdp_tx_bq *bq)
 {
 	u32 pktlen, headroom, act, metalen;
 	void *orig_data, *orig_data_end;
 		get_page(virt_to_page(xdp.data));
 		consume_skb(skb);
 		xdp.rxq->mem = rq->xdp_mem;
-		if (unlikely(veth_xdp_tx(rq->dev, &xdp) < 0)) {
+		if (unlikely(veth_xdp_tx(rq->dev, &xdp, bq) < 0)) {
 			trace_xdp_exception(rq->dev, xdp_prog, act);
 			goto err_xdp;
 		}
 	return NULL;
 }
-static int veth_xdp_rcv(struct veth_rq *rq, int budget, unsigned int *xdp_xmit)
+static int veth_xdp_rcv(struct veth_rq *rq, int budget, unsigned int *xdp_xmit,
+			struct veth_xdp_tx_bq *bq)
 {
 	int i, done = 0, drops = 0, bytes = 0;
 			struct xdp_frame *frame = veth_ptr_to_xdp(ptr);
 			bytes += frame->len;
-			skb = veth_xdp_rcv_one(rq, frame, &xdp_xmit_one);
+			skb = veth_xdp_rcv_one(rq, frame, &xdp_xmit_one, bq);
 		} else {
 			skb = ptr;
 			bytes += skb->len;
-			skb = veth_xdp_rcv_skb(rq, skb, &xdp_xmit_one);
+			skb = veth_xdp_rcv_skb(rq, skb, &xdp_xmit_one, bq);
 		}
 		*xdp_xmit |= xdp_xmit_one;
 	struct veth_rq *rq =
 		container_of(napi, struct veth_rq, xdp_napi);
 	unsigned int xdp_xmit = 0;
+	struct veth_xdp_tx_bq bq;
 	int done;
+	bq.count = 0;
 	xdp_set_return_frame_no_direct();
-	done = veth_xdp_rcv(rq, budget, &xdp_xmit);
+	done = veth_xdp_rcv(rq, budget, &xdp_xmit, &bq);
 	if (done < budget && napi_complete_done(napi, done)) {
 		/* Write rx_notify_masked before reading ptr_ring */
 	}
 	if (xdp_xmit & VETH_XDP_TX)
-		veth_xdp_flush(rq->dev);
+		veth_xdp_flush(rq->dev, &bq);
 	if (xdp_xmit & VETH_XDP_REDIR)
 		xdp_do_flush_map();
 	xdp_clear_return_frame_no_direct();

include/linux/bpf-cgroup.h+45−0

@@ -124,6 +124,14 @@ int __cgroup_bpf_run_filter_sysctl(struct ctl_table_header *head, · 123 unmodified lines
124	loff_t ppos, void *new_buf,	124	loff_t ppos, void *new_buf,
125	enum bpf_attach_type type);	125	enum bpf_attach_type type);
126		126
		127	int __cgroup_bpf_run_filter_setsockopt(struct sock sock, int level,
		128	int optname, char __user optval,
		129	int optlen, char *kernel_optval);
		130	int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level,
		131	int optname, char __user *optval,
		132	int __user *optlen, int max_optlen,
		133	int retval);
		134
127	static inline enum bpf_cgroup_storage_type cgroup_storage_type(	135	static inline enum bpf_cgroup_storage_type cgroup_storage_type(
128	struct bpf_map *map)	136	struct bpf_map *map)
129	{	137	{
@@ -286,6 +294,38 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map map, void key, · 156 unmodified lines
286	__ret; \	294	__ret; \
287	})	295	})
288		296
		297	#define BPF_CGROUP_RUN_PROG_SETSOCKOPT(sock, level, optname, optval, optlen, \
		298	kernel_optval) \
		299	({ \
		300	int __ret = 0; \
		301	if (cgroup_bpf_enabled) \
		302	__ret = __cgroup_bpf_run_filter_setsockopt(sock, level, \
		303	optname, optval, \
		304	optlen, \
		305	kernel_optval); \
		306	__ret; \
		307	})
		308
		309	#define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen) \
		310	({ \
		311	int __ret = 0; \
		312	if (cgroup_bpf_enabled) \
		313	get_user(__ret, optlen); \
		314	__ret; \
		315	})
		316
		317	#define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, optlen, \
		318	max_optlen, retval) \
		319	({ \
		320	int __ret = retval; \
		321	if (cgroup_bpf_enabled) \
		322	__ret = __cgroup_bpf_run_filter_getsockopt(sock, level, \
		323	optname, optval, \
		324	optlen, max_optlen, \
		325	retval); \
		326	__ret; \
		327	})
		328
289	int cgroup_bpf_prog_attach(const union bpf_attr *attr,	329	int cgroup_bpf_prog_attach(const union bpf_attr *attr,
290	enum bpf_prog_type ptype, struct bpf_prog *prog);	330	enum bpf_prog_type ptype, struct bpf_prog *prog);
291	int cgroup_bpf_prog_detach(const union bpf_attr *attr,	331	int cgroup_bpf_prog_detach(const union bpf_attr *attr,
@@ -357,6 +397,11 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map, · 65 unmodified lines
357	#define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) ({ 0; })	397	#define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) ({ 0; })
358	#define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(type,major,minor,access) ({ 0; })	398	#define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(type,major,minor,access) ({ 0; })
359	#define BPF_CGROUP_RUN_PROG_SYSCTL(head,table,write,buf,count,pos,nbuf) ({ 0; })	399	#define BPF_CGROUP_RUN_PROG_SYSCTL(head,table,write,buf,count,pos,nbuf) ({ 0; })
		400	#define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen) ({ 0; })
		401	#define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, \
		402	optlen, max_optlen, retval) ({ retval; })
		403	#define BPF_CGROUP_RUN_PROG_SETSOCKOPT(sock, level, optname, optval, optlen, \
		404	kernel_optval) ({ 0; })
360		405
361	#define for_each_cgroup_storage_type(stype) for (; false; )	406	#define for_each_cgroup_storage_type(stype) for (; false; )
362		407

include/linux/bpf.h+2−0

 struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags);
 void bpf_prog_array_free(struct bpf_prog_array *progs);
 int bpf_prog_array_length(struct bpf_prog_array *progs);
+bool bpf_prog_array_is_empty(struct bpf_prog_array *array);
 int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs,
 				__u32 __user *prog_ids, u32 cnt);
 extern const struct bpf_func_proto bpf_get_local_storage_proto;
 extern const struct bpf_func_proto bpf_strtol_proto;
 extern const struct bpf_func_proto bpf_strtoul_proto;
+extern const struct bpf_func_proto bpf_tcp_sock_proto;
 /* Shared helpers among cBPF and eBPF. */
 void bpf_user_rnd_init_once(void);

include/linux/bpf_types.h+1−0

@@ -30,6 +30,7 @@ BPF_PROG_TYPE(BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, raw_tracepoint_writable) · 29 unmodified lines
30	#ifdef CONFIG_CGROUP_BPF	30	#ifdef CONFIG_CGROUP_BPF
31	BPF_PROG_TYPE(BPF_PROG_TYPE_CGROUP_DEVICE, cg_dev)	31	BPF_PROG_TYPE(BPF_PROG_TYPE_CGROUP_DEVICE, cg_dev)
32	BPF_PROG_TYPE(BPF_PROG_TYPE_CGROUP_SYSCTL, cg_sysctl)	32	BPF_PROG_TYPE(BPF_PROG_TYPE_CGROUP_SYSCTL, cg_sysctl)
		33	BPF_PROG_TYPE(BPF_PROG_TYPE_CGROUP_SOCKOPT, cg_sockopt)
33	#endif	34	#endif
34	#ifdef CONFIG_BPF_LIRC_MODE2	35	#ifdef CONFIG_BPF_LIRC_MODE2
35	BPF_PROG_TYPE(BPF_PROG_TYPE_LIRC_MODE2, lirc_mode2)	36	BPF_PROG_TYPE(BPF_PROG_TYPE_LIRC_MODE2, lirc_mode2)

include/linux/filter.h+12−1

 };
 struct bpf_redirect_info {
-	u32 ifindex;
 	u32 flags;
+	u32 tgt_index;
+	void *tgt_value;
 	struct bpf_map *map;
 	struct bpf_map *map_to_flush;
 	u32 kern_flags;
 	u64 tmp_reg;
 };
+struct bpf_sockopt_kern {
+	struct sock	*sk;
+	u8		*optval;
+	u8		*optval_end;
+	s32		level;
+	s32		optname;
+	s32		optlen;
+	s32		retval;
+};
 #endif /* __LINUX_FILTER_H__ */

include/linux/list.h+14−0

@@ -106,6 +106,20 @@ static inline void __list_del(struct list_head * prev, struct list_head * next) · 105 unmodified lines
106	WRITE_ONCE(prev->next, next);	106	WRITE_ONCE(prev->next, next);
107	}	107	}
108		108
		109	/*
		110	* Delete a list entry and clear the 'prev' pointer.
		111	*
		112	* This is a special-purpose list clearing method used in the networking code
		113	* for lists allocated as per-cpu, where we don't want to incur the extra
		114	* WRITE_ONCE() overhead of a regular list_del_init(). The code that uses this
		115	* needs to check the node 'prev' pointer instead of calling list_empty().
		116	*/
		117	static inline void __list_del_clearprev(struct list_head *entry)
		118	{
		119	__list_del(entry->prev, entry->next);
		120	entry->prev = NULL;
		121	}
		122
109	/**	123	/**
110	* list_del - deletes entry from list.	124	* list_del - deletes entry from list.
111	* @entry: the element to delete from the list.	125	* @entry: the element to delete from the list.

include/net/tcp.h+8−0

@@ -2221,6 +2221,14 @@ static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk) · 2220 unmodified lines
2221	return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN, 0, NULL) == 1);	2221	return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN, 0, NULL) == 1);
2222	}	2222	}
2223		2223
		2224	static inline void tcp_bpf_rtt(struct sock *sk)
		2225	{
		2226	struct tcp_sock *tp = tcp_sk(sk);
		2227
		2228	if (BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_RTT_CB_FLAG))
		2229	tcp_call_bpf(sk, BPF_SOCK_OPS_RTT_CB, 0, NULL);
		2230	}
		2231
2224	#if IS_ENABLED(CONFIG_SMC)	2232	#if IS_ENABLED(CONFIG_SMC)
2225	extern struct static_key_false tcp_have_smc;	2233	extern struct static_key_false tcp_have_smc;
2226	#endif	2234	#endif

include/net/xdp_sock.h+24−3

 void xsk_flush(struct xdp_sock *xs);
 bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs);
 /* Used from netdev driver */
+bool xsk_umem_has_addrs(struct xdp_umem *umem, u32 cnt);
 u64 *xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr);
 void xsk_umem_discard_addr(struct xdp_umem *umem);
 void xsk_umem_complete_tx(struct xdp_umem *umem, u32 nb_entries);
-bool xsk_umem_consume_tx(struct xdp_umem *umem, dma_addr_t *dma, u32 *len);
+bool xsk_umem_consume_tx(struct xdp_umem *umem, struct xdp_desc *desc);
 void xsk_umem_consume_tx_done(struct xdp_umem *umem);
 struct xdp_umem_fq_reuse *xsk_reuseq_prepare(u32 nentries);
 struct xdp_umem_fq_reuse *xsk_reuseq_swap(struct xdp_umem *umem,
 }
 /* Reuse-queue aware version of FILL queue helpers */
+static inline bool xsk_umem_has_addrs_rq(struct xdp_umem *umem, u32 cnt)
+{
+	struct xdp_umem_fq_reuse *rq = umem->fq_reuse;
+	if (rq->length >= cnt)
+		return true;
+	return xsk_umem_has_addrs(umem, cnt - rq->length);
+}
 static inline u64 *xsk_umem_peek_addr_rq(struct xdp_umem *umem, u64 *addr)
 {
 	struct xdp_umem_fq_reuse *rq = umem->fq_reuse;
 	return false;
 }
+static inline bool xsk_umem_has_addrs(struct xdp_umem *umem, u32 cnt)
+{
+	return false;
+}
 static inline u64 *xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr)
 {
 	return NULL;
 {
 }
-static inline bool xsk_umem_consume_tx(struct xdp_umem *umem, dma_addr_t *dma,
+static inline bool xsk_umem_consume_tx(struct xdp_umem *umem,
-				       u32 *len)
+				       struct xdp_desc *desc)
 {
 	return false;
 }
 	return 0;
 }
+static inline bool xsk_umem_has_addrs_rq(struct xdp_umem *umem, u32 cnt)
+{
+	return false;
+}
 static inline u64 *xsk_umem_peek_addr_rq(struct xdp_umem *umem, u64 *addr)
 {
 	return NULL;

include/trace/events/xdp.h+31−3

@@ -50,6 +50,35 @@ TRACE_EVENT(xdp_exception, · 49 unmodified lines
50	__entry->ifindex)	50	__entry->ifindex)
51	);	51	);
52		52
		53	TRACE_EVENT(xdp_bulk_tx,
		54
		55	TP_PROTO(const struct net_device *dev,
		56	int sent, int drops, int err),
		57
		58	TP_ARGS(dev, sent, drops, err),
		59
		60	TP_STRUCT__entry(
		61	__field(int, ifindex)
		62	__field(u32, act)
		63	__field(int, drops)
		64	__field(int, sent)
		65	__field(int, err)
		66	),
		67
		68	TP_fast_assign(
		69	__entry->ifindex = dev->ifindex;
		70	__entry->act = XDP_TX;
		71	__entry->drops = drops;
		72	__entry->sent = sent;
		73	__entry->err = err;
		74	),
		75
		76	TP_printk("ifindex=%d action=%s sent=%d drops=%d err=%d",
		77	__entry->ifindex,
		78	__print_symbolic(__entry->act, __XDP_ACT_SYM_TAB),
		79	__entry->sent, __entry->drops, __entry->err)
		80	);
		81
53	DECLARE_EVENT_CLASS(xdp_redirect_template,	82	DECLARE_EVENT_CLASS(xdp_redirect_template,
54		83
55	TP_PROTO(const struct net_device *dev,	84	TP_PROTO(const struct net_device *dev,
@@ -146,9 +175,8 @@ struct _bpf_dtab_netdev { · 90 unmodified lines
146	#endif /* __DEVMAP_OBJ_TYPE */	175	#endif /* __DEVMAP_OBJ_TYPE */
147		176
148	#define devmap_ifindex(fwd, map) \	177	#define devmap_ifindex(fwd, map) \
149	(!fwd ? 0 : \	178	((map->map_type == BPF_MAP_TYPE_DEVMAP) ? \
150	((map->map_type == BPF_MAP_TYPE_DEVMAP) ? \	179	((struct _bpf_dtab_netdev *)fwd)->dev->ifindex : 0)
151	((struct _bpf_dtab_netdev *)fwd)->dev->ifindex : 0))
152		180
153	#define _trace_xdp_redirect_map(dev, xdp, fwd, map, idx) \	181	#define _trace_xdp_redirect_map(dev, xdp, fwd, map, idx) \
154	trace_xdp_redirect_map(dev, xdp, devmap_ifindex(fwd, map), \	182	trace_xdp_redirect_map(dev, xdp, devmap_ifindex(fwd, map), \

include/uapi/linux/bpf.h+30−3

 	BPF_PROG_TYPE_FLOW_DISSECTOR,
 	BPF_PROG_TYPE_CGROUP_SYSCTL,
 	BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE,
+	BPF_PROG_TYPE_CGROUP_SOCKOPT,
 };
 enum bpf_attach_type {
 	BPF_CGROUP_SYSCTL,
 	BPF_CGROUP_UDP4_RECVMSG,
 	BPF_CGROUP_UDP6_RECVMSG,
+	BPF_CGROUP_GETSOCKOPT,
+	BPF_CGROUP_SETSOCKOPT,
 	__MAX_BPF_ATTACH_TYPE
 };
  * 		but this is only implemented for native XDP (with driver
  * 		support) as of this writing).
  *
- * 		All values for *flags* are reserved for future usage, and must
+ * 		The lower two bits of *flags* are used as the return code if
- * 		be left at zero.
+ * 		the map lookup fails. This is so that the return value can be
+ * 		one of the XDP program return codes up to XDP_TX, as chosen by
+ * 		the caller. Any higher bits in the *flags* argument must be
+ * 		unset.
  *
  * 		When used to redirect packets to net devices, this helper
  * 		provides a high performance increase over **bpf_redirect**\ ().
  * 		* **BPF_SOCK_OPS_RTO_CB_FLAG** (retransmission time out)
  * 		* **BPF_SOCK_OPS_RETRANS_CB_FLAG** (retransmission)
  * 		* **BPF_SOCK_OPS_STATE_CB_FLAG** (TCP state change)
+ * 		* **BPF_SOCK_OPS_RTT_CB_FLAG** (every RTT)
  *
  * 		Therefore, this function can be used to clear a callback flag by
  * 		setting the appropriate bit to zero. e.g. to disable the RTO
 				 * sum(delta(snd_una)), or how many bytes
 				 * were acked.
 				 */
+	__u32 dsack_dups;	/* RFC4898 tcpEStatsStackDSACKDups
+				 * total number of DSACK blocks received
+				 */
+	__u32 delivered;	/* Total data packets delivered incl. rexmits */
+	__u32 delivered_ce;	/* Like the above but only ECE marked packets */
+	__u32 icsk_retransmits;	/* Number of unrecovered [RTO] timeouts */
 };
 struct bpf_sock_tuple {
 #define BPF_SOCK_OPS_RTO_CB_FLAG	(1<<0)
 #define BPF_SOCK_OPS_RETRANS_CB_FLAG	(1<<1)
 #define BPF_SOCK_OPS_STATE_CB_FLAG	(1<<2)
-#define BPF_SOCK_OPS_ALL_CB_FLAGS       0x7		/* Mask of all currently
+#define BPF_SOCK_OPS_RTT_CB_FLAG	(1<<3)
+#define BPF_SOCK_OPS_ALL_CB_FLAGS       0xF		/* Mask of all currently
 							 * supported cb flags
 							 */
 	BPF_SOCK_OPS_TCP_LISTEN_CB,	/* Called on listen(2), right after
 					 * socket transition to LISTEN state.
 					 */
+	BPF_SOCK_OPS_RTT_CB,		/* Called on every RTT.
+					 */
 };
 /* List of TCP states. There is a build check in net/ipv4/tcp.c to detect
 				 */
 };
+struct bpf_sockopt {
+	__bpf_md_ptr(struct bpf_sock *, sk);
+	__bpf_md_ptr(void *, optval);
+	__bpf_md_ptr(void *, optval_end);
+	__s32	level;
+	__s32	optname;
+	__s32	optlen;
+	__s32	retval;
+};
 #endif /* _UAPI__LINUX_BPF_H__ */

include/uapi/linux/if_xdp.h+8−0

 #define XDP_UMEM_FILL_RING		5
 #define XDP_UMEM_COMPLETION_RING	6
 #define XDP_STATISTICS			7
+#define XDP_OPTIONS			8
 struct xdp_umem_reg {
 	__u64 addr; /* Start of packet data area */
 	__u64 tx_invalid_descs; /* Dropped due to invalid descriptor */
 };
+struct xdp_options {
+	__u32 flags;
+};
+/* Flags for the flags field of struct xdp_options */
+#define XDP_OPTIONS_ZEROCOPY (1 << 0)
 /* Pgoff for mmaping the rings */
 #define XDP_PGOFF_RX_RING			  0
 #define XDP_PGOFF_TX_RING		 0x80000000

kernel/bpf/cgroup.c+351−1

@@ -15,6 +15,9 @@ · 14 unmodified lines
15	#include <linux/bpf.h>	15	#include <linux/bpf.h>
16	#include <linux/bpf-cgroup.h>	16	#include <linux/bpf-cgroup.h>
17	#include <net/sock.h>	17	#include <net/sock.h>
		18	#include <net/bpf_sk_storage.h>
		19
		20	#include "../cgroup/cgroup-internal.h"
18		21
19	DEFINE_STATIC_KEY_FALSE(cgroup_bpf_enabled_key);	22	DEFINE_STATIC_KEY_FALSE(cgroup_bpf_enabled_key);
20	EXPORT_SYMBOL(cgroup_bpf_enabled_key);	23	EXPORT_SYMBOL(cgroup_bpf_enabled_key);
@@ -38,6 +41,8 @@ static void cgroup_bpf_release(struct work_struct *work) · 17 unmodified lines
38	struct bpf_prog_array *old_array;	41	struct bpf_prog_array *old_array;
39	unsigned int type;	42	unsigned int type;
40		43
		44	mutex_lock(&cgroup_mutex);
		45
41	for (type = 0; type < ARRAY_SIZE(cgrp->bpf.progs); type++) {	46	for (type = 0; type < ARRAY_SIZE(cgrp->bpf.progs); type++) {
42	struct list_head *progs = &cgrp->bpf.progs[type];	47	struct list_head *progs = &cgrp->bpf.progs[type];
43	struct bpf_prog_list pl, tmp;	48	struct bpf_prog_list pl, tmp;
@@ -54,10 +59,12 @@ static void cgroup_bpf_release(struct work_struct *work) · 10 unmodified lines
54	}	59	}
55	old_array = rcu_dereference_protected(	60	old_array = rcu_dereference_protected(
56	cgrp->bpf.effective[type],	61	cgrp->bpf.effective[type],
57	percpu_ref_is_dying(&cgrp->bpf.refcnt));	62	lockdep_is_held(&cgroup_mutex));
58	bpf_prog_array_free(old_array);	63	bpf_prog_array_free(old_array);
59	}	64	}
60		65
		66	mutex_unlock(&cgroup_mutex);
		67
61	percpu_ref_exit(&cgrp->bpf.refcnt);	68	percpu_ref_exit(&cgrp->bpf.refcnt);
62	cgroup_put(cgrp);	69	cgroup_put(cgrp);
63	}	70	}
@@ -229,6 +236,9 @@ static int update_effective_progs(struct cgroup *cgrp, · 165 unmodified lines
229	css_for_each_descendant_pre(css, &cgrp->self) {	236	css_for_each_descendant_pre(css, &cgrp->self) {
230	struct cgroup *desc = container_of(css, struct cgroup, self);	237	struct cgroup *desc = container_of(css, struct cgroup, self);
231		238
		239	if (percpu_ref_is_zero(&desc->bpf.refcnt))
		240	continue;
		241
232	err = compute_effective_progs(desc, type, &desc->bpf.inactive);	242	err = compute_effective_progs(desc, type, &desc->bpf.inactive);
233	if (err)	243	if (err)
234	goto cleanup;	244	goto cleanup;
@@ -238,6 +248,14 @@ static int update_effective_progs(struct cgroup *cgrp, · 3 unmodified lines
238	css_for_each_descendant_pre(css, &cgrp->self) {	248	css_for_each_descendant_pre(css, &cgrp->self) {
239	struct cgroup *desc = container_of(css, struct cgroup, self);	249	struct cgroup *desc = container_of(css, struct cgroup, self);
240		250
		251	if (percpu_ref_is_zero(&desc->bpf.refcnt)) {
		252	if (unlikely(desc->bpf.inactive)) {
		253	bpf_prog_array_free(desc->bpf.inactive);
		254	desc->bpf.inactive = NULL;
		255	}
		256	continue;
		257	}
		258
241	activate_effective_progs(desc, type, desc->bpf.inactive);	259	activate_effective_progs(desc, type, desc->bpf.inactive);
242	desc->bpf.inactive = NULL;	260	desc->bpf.inactive = NULL;
243	}	261	}
@@ -921,6 +939,188 @@ int __cgroup_bpf_run_filter_sysctl(struct ctl_table_header *head, · 677 unmodified lines
921	}	939	}
922	EXPORT_SYMBOL(__cgroup_bpf_run_filter_sysctl);	940	EXPORT_SYMBOL(__cgroup_bpf_run_filter_sysctl);
923		941
		942	static bool __cgroup_bpf_prog_array_is_empty(struct cgroup *cgrp,
		943	enum bpf_attach_type attach_type)
		944	{
		945	struct bpf_prog_array *prog_array;
		946	bool empty;
		947
		948	rcu_read_lock();
		949	prog_array = rcu_dereference(cgrp->bpf.effective[attach_type]);
		950	empty = bpf_prog_array_is_empty(prog_array);
		951	rcu_read_unlock();
		952
		953	return empty;
		954	}
		955
		956	static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen)
		957	{
		958	if (unlikely(max_optlen > PAGE_SIZE) \|\| max_optlen < 0)
		959	return -EINVAL;
		960
		961	ctx->optval = kzalloc(max_optlen, GFP_USER);
		962	if (!ctx->optval)
		963	return -ENOMEM;
		964
		965	ctx->optval_end = ctx->optval + max_optlen;
		966	ctx->optlen = max_optlen;
		967
		968	return 0;
		969	}
		970
		971	static void sockopt_free_buf(struct bpf_sockopt_kern *ctx)
		972	{
		973	kfree(ctx->optval);
		974	}
		975
		976	int __cgroup_bpf_run_filter_setsockopt(struct sock sk, int level,
		977	int optname, char __user optval,
		978	int optlen, char *kernel_optval)
		979	{
		980	struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
		981	struct bpf_sockopt_kern ctx = {
		982	.sk = sk,
		983	.level = *level,
		984	.optname = *optname,
		985	};
		986	int ret;
		987
		988	/* Opportunistic check to see whether we have any BPF program
		989	* attached to the hook so we don't waste time allocating
		990	* memory and locking the socket.
		991	*/
		992	if (!cgroup_bpf_enabled \|\|
		993	__cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_SETSOCKOPT))
		994	return 0;
		995
		996	ret = sockopt_alloc_buf(&ctx, *optlen);
		997	if (ret)
		998	return ret;
		999
		1000	if (copy_from_user(ctx.optval, optval, *optlen) != 0) {
		1001	ret = -EFAULT;
		1002	goto out;
		1003	}
		1004
		1005	lock_sock(sk);
		1006	ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[BPF_CGROUP_SETSOCKOPT],
		1007	&ctx, BPF_PROG_RUN);
		1008	release_sock(sk);
		1009
		1010	if (!ret) {
		1011	ret = -EPERM;
		1012	goto out;
		1013	}
		1014
		1015	if (ctx.optlen == -1) {
		1016	/* optlen set to -1, bypass kernel */
		1017	ret = 1;
		1018	} else if (ctx.optlen > *optlen \|\| ctx.optlen < -1) {
		1019	/* optlen is out of bounds */
		1020	ret = -EFAULT;
		1021	} else {
		1022	/* optlen within bounds, run kernel handler */
		1023	ret = 0;
		1024
		1025	/* export any potential modifications */
		1026	*level = ctx.level;
		1027	*optname = ctx.optname;
		1028	*optlen = ctx.optlen;
		1029	*kernel_optval = ctx.optval;
		1030	}
		1031
		1032	out:
		1033	if (ret)
		1034	sockopt_free_buf(&ctx);
		1035	return ret;
		1036	}
		1037	EXPORT_SYMBOL(__cgroup_bpf_run_filter_setsockopt);
		1038
		1039	int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level,
		1040	int optname, char __user *optval,
		1041	int __user *optlen, int max_optlen,
		1042	int retval)
		1043	{
		1044	struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
		1045	struct bpf_sockopt_kern ctx = {
		1046	.sk = sk,
		1047	.level = level,
		1048	.optname = optname,
		1049	.retval = retval,
		1050	};
		1051	int ret;
		1052
		1053	/* Opportunistic check to see whether we have any BPF program
		1054	* attached to the hook so we don't waste time allocating
		1055	* memory and locking the socket.
		1056	*/
		1057	if (!cgroup_bpf_enabled \|\|
		1058	__cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_GETSOCKOPT))
		1059	return retval;
		1060
		1061	ret = sockopt_alloc_buf(&ctx, max_optlen);
		1062	if (ret)
		1063	return ret;
		1064
		1065	if (!retval) {
		1066	/* If kernel getsockopt finished successfully,
		1067	* copy whatever was returned to the user back
		1068	* into our temporary buffer. Set optlen to the
		1069	* one that kernel returned as well to let
		1070	* BPF programs inspect the value.
		1071	*/
		1072
		1073	if (get_user(ctx.optlen, optlen)) {
		1074	ret = -EFAULT;
		1075	goto out;
		1076	}
		1077
		1078	if (ctx.optlen > max_optlen)
		1079	ctx.optlen = max_optlen;
		1080
		1081	if (copy_from_user(ctx.optval, optval, ctx.optlen) != 0) {
		1082	ret = -EFAULT;
		1083	goto out;
		1084	}
		1085	}
		1086
		1087	lock_sock(sk);
		1088	ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[BPF_CGROUP_GETSOCKOPT],
		1089	&ctx, BPF_PROG_RUN);
		1090	release_sock(sk);
		1091
		1092	if (!ret) {
		1093	ret = -EPERM;
		1094	goto out;
		1095	}
		1096
		1097	if (ctx.optlen > max_optlen) {
		1098	ret = -EFAULT;
		1099	goto out;
		1100	}
		1101
		1102	/* BPF programs only allowed to set retval to 0, not some
		1103	* arbitrary value.
		1104	*/
		1105	if (ctx.retval != 0 && ctx.retval != retval) {
		1106	ret = -EFAULT;
		1107	goto out;
		1108	}
		1109
		1110	if (copy_to_user(optval, ctx.optval, ctx.optlen) \|\|
		1111	put_user(ctx.optlen, optlen)) {
		1112	ret = -EFAULT;
		1113	goto out;
		1114	}
		1115
		1116	ret = ctx.retval;
		1117
		1118	out:
		1119	sockopt_free_buf(&ctx);
		1120	return ret;
		1121	}
		1122	EXPORT_SYMBOL(__cgroup_bpf_run_filter_getsockopt);
		1123
924	static ssize_t sysctl_cpy_dir(const struct ctl_dir dir, char *bufp,	1124	static ssize_t sysctl_cpy_dir(const struct ctl_dir dir, char *bufp,
925	size_t *lenp)	1125	size_t *lenp)
926	{	1126	{
@@ -1181,3 +1381,153 @@ const struct bpf_verifier_ops cg_sysctl_verifier_ops = { · 254 unmodified lines
1181		1381
1182	const struct bpf_prog_ops cg_sysctl_prog_ops = {	1382	const struct bpf_prog_ops cg_sysctl_prog_ops = {
1183	};	1383	};
		1384
		1385	static const struct bpf_func_proto *
		1386	cg_sockopt_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
		1387	{
		1388	switch (func_id) {
		1389	case BPF_FUNC_sk_storage_get:
		1390	return &bpf_sk_storage_get_proto;
		1391	case BPF_FUNC_sk_storage_delete:
		1392	return &bpf_sk_storage_delete_proto;
		1393	#ifdef CONFIG_INET
		1394	case BPF_FUNC_tcp_sock:
		1395	return &bpf_tcp_sock_proto;
		1396	#endif
		1397	default:
		1398	return cgroup_base_func_proto(func_id, prog);
		1399	}
		1400	}
		1401
		1402	static bool cg_sockopt_is_valid_access(int off, int size,
		1403	enum bpf_access_type type,
		1404	const struct bpf_prog *prog,
		1405	struct bpf_insn_access_aux *info)
		1406	{
		1407	const int size_default = sizeof(__u32);
		1408
		1409	if (off < 0 \|\| off >= sizeof(struct bpf_sockopt))
		1410	return false;
		1411
		1412	if (off % size != 0)
		1413	return false;
		1414
		1415	if (type == BPF_WRITE) {
		1416	switch (off) {
		1417	case offsetof(struct bpf_sockopt, retval):
		1418	if (size != size_default)
		1419	return false;
		1420	return prog->expected_attach_type ==
		1421	BPF_CGROUP_GETSOCKOPT;
		1422	case offsetof(struct bpf_sockopt, optname):
		1423	/* fallthrough */
		1424	case offsetof(struct bpf_sockopt, level):
		1425	if (size != size_default)
		1426	return false;
		1427	return prog->expected_attach_type ==
		1428	BPF_CGROUP_SETSOCKOPT;
		1429	case offsetof(struct bpf_sockopt, optlen):
		1430	return size == size_default;
		1431	default:
		1432	return false;
		1433	}
		1434	}
		1435
		1436	switch (off) {
		1437	case offsetof(struct bpf_sockopt, sk):
		1438	if (size != sizeof(__u64))
		1439	return false;
		1440	info->reg_type = PTR_TO_SOCKET;
		1441	break;
		1442	case offsetof(struct bpf_sockopt, optval):
		1443	if (size != sizeof(__u64))
		1444	return false;
		1445	info->reg_type = PTR_TO_PACKET;
		1446	break;
		1447	case offsetof(struct bpf_sockopt, optval_end):
		1448	if (size != sizeof(__u64))
		1449	return false;
		1450	info->reg_type = PTR_TO_PACKET_END;
		1451	break;
		1452	case offsetof(struct bpf_sockopt, retval):
		1453	if (size != size_default)
		1454	return false;
		1455	return prog->expected_attach_type == BPF_CGROUP_GETSOCKOPT;
		1456	default:
		1457	if (size != size_default)
		1458	return false;
		1459	break;
		1460	}
		1461	return true;
		1462	}
		1463
		1464	#define CG_SOCKOPT_ACCESS_FIELD(T, F) \
		1465	T(BPF_FIELD_SIZEOF(struct bpf_sockopt_kern, F), \
		1466	si->dst_reg, si->src_reg, \
		1467	offsetof(struct bpf_sockopt_kern, F))
		1468
		1469	static u32 cg_sockopt_convert_ctx_access(enum bpf_access_type type,
		1470	const struct bpf_insn *si,
		1471	struct bpf_insn *insn_buf,
		1472	struct bpf_prog *prog,
		1473	u32 *target_size)
		1474	{
		1475	struct bpf_insn *insn = insn_buf;
		1476
		1477	switch (si->off) {
		1478	case offsetof(struct bpf_sockopt, sk):
		1479	*insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, sk);
		1480	break;
		1481	case offsetof(struct bpf_sockopt, level):
		1482	if (type == BPF_WRITE)
		1483	*insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_STX_MEM, level);
		1484	else
		1485	*insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, level);
		1486	break;
		1487	case offsetof(struct bpf_sockopt, optname):
		1488	if (type == BPF_WRITE)
		1489	*insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_STX_MEM, optname);
		1490	else
		1491	*insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optname);
		1492	break;
		1493	case offsetof(struct bpf_sockopt, optlen):
		1494	if (type == BPF_WRITE)
		1495	*insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_STX_MEM, optlen);
		1496	else
		1497	*insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optlen);
		1498	break;
		1499	case offsetof(struct bpf_sockopt, retval):
		1500	if (type == BPF_WRITE)
		1501	*insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_STX_MEM, retval);
		1502	else
		1503	*insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, retval);
		1504	break;
		1505	case offsetof(struct bpf_sockopt, optval):
		1506	*insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optval);
		1507	break;
		1508	case offsetof(struct bpf_sockopt, optval_end):
		1509	*insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optval_end);
		1510	break;
		1511	}
		1512
		1513	return insn - insn_buf;
		1514	}
		1515
		1516	static int cg_sockopt_get_prologue(struct bpf_insn *insn_buf,
		1517	bool direct_write,
		1518	const struct bpf_prog *prog)
		1519	{
		1520	/* Nothing to do for sockopt argument. The data is kzalloc'ated.
		1521	*/
		1522	return 0;
		1523	}
		1524
		1525	const struct bpf_verifier_ops cg_sockopt_verifier_ops = {
		1526	.get_func_proto = cg_sockopt_func_proto,
		1527	.is_valid_access = cg_sockopt_is_valid_access,
		1528	.convert_ctx_access = cg_sockopt_convert_ctx_access,
		1529	.gen_prologue = cg_sockopt_get_prologue,
		1530	};
		1531
		1532	const struct bpf_prog_ops cg_sockopt_prog_ops = {
		1533	};

kernel/bpf/core.c+10−0

@@ -1809,6 +1809,15 @@ int bpf_prog_array_length(struct bpf_prog_array *array) · 1808 unmodified lines
1809	return cnt;	1809	return cnt;
1810	}	1810	}
1811		1811
		1812	bool bpf_prog_array_is_empty(struct bpf_prog_array *array)
		1813	{
		1814	struct bpf_prog_array_item *item;
		1815
		1816	for (item = array->items; item->prog; item++)
		1817	if (item->prog != &dummy_bpf_prog.prog)
		1818	return false;
		1819	return true;
		1820	}
1812		1821
1813	static bool bpf_prog_array_copy_core(struct bpf_prog_array *array,	1822	static bool bpf_prog_array_copy_core(struct bpf_prog_array *array,
1814	u32 *prog_ids,	1823	u32 *prog_ids,
@@ -2101,3 +2110,4 @@ EXPORT_SYMBOL(bpf_stats_enabled_key); · 286 unmodified lines
2101	#include <linux/bpf_trace.h>	2110	#include <linux/bpf_trace.h>
2102		2111
2103	EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_exception);	2112	EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_exception);
		2113	EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_bulk_tx);

kernel/bpf/cpumap.c+48−57

 /* General idea: XDP packets getting XDP redirected to another CPU,
  * will maximum be stored/queued for one driver ->poll() call.  It is
- * guaranteed that setting flush bit and flush operation happen on
+ * guaranteed that queueing the frame and the flush operation happen on
  * same CPU.  Thus, cpu_map_flush operation can deduct via this_cpu_ptr()
  * which queue in bpf_cpu_map_entry contains packets.
  */
 #define CPU_MAP_BULK_SIZE 8  /* 8 == one cacheline on 64-bit archs */
+struct bpf_cpu_map_entry;
+struct bpf_cpu_map;
 struct xdp_bulk_queue {
 	void *q[CPU_MAP_BULK_SIZE];
+	struct list_head flush_node;
+	struct bpf_cpu_map_entry *obj;
 	unsigned int count;
 };
 	/* XDP can run multiple RX-ring queues, need __percpu enqueue store */
 	struct xdp_bulk_queue __percpu *bulkq;
+	struct bpf_cpu_map *cmap;
 	/* Queue with potential multi-producers, and single-consumer kthread */
 	struct ptr_ring *queue;
 	struct task_struct *kthread;
 	struct bpf_map map;
 	/* Below members specific for map type */
 	struct bpf_cpu_map_entry **cpu_map;
-	unsigned long __percpu *flush_needed;
+	struct list_head __percpu *flush_list;
 };
-static int bq_flush_to_queue(struct bpf_cpu_map_entry *rcpu,
+static int bq_flush_to_queue(struct xdp_bulk_queue *bq, bool in_napi_ctx);
-			     struct xdp_bulk_queue *bq, bool in_napi_ctx);
-static u64 cpu_map_bitmap_size(const union bpf_attr *attr)
-{
-	return BITS_TO_LONGS(attr->max_entries) * sizeof(unsigned long);
-}
 static struct bpf_map *cpu_map_alloc(union bpf_attr *attr)
 {
 	struct bpf_cpu_map *cmap;
 	int err = -ENOMEM;
+	int ret, cpu;
 	u64 cost;
-	int ret;
 	if (!capable(CAP_SYS_ADMIN))
 		return ERR_PTR(-EPERM);
 	/* make sure page count doesn't overflow */
 	cost = (u64) cmap->map.max_entries * sizeof(struct bpf_cpu_map_entry *);
-	cost += cpu_map_bitmap_size(attr) * num_possible_cpus();
+	cost += sizeof(struct list_head) * num_possible_cpus();
 	/* Notice returns -EPERM on if map size is larger than memlock limit */
 	ret = bpf_map_charge_init(&cmap->map.memory, cost);
 		goto free_cmap;
 	}
-	/* A per cpu bitfield with a bit per possible CPU in map  */
+	cmap->flush_list = alloc_percpu(struct list_head);
-	cmap->flush_needed = __alloc_percpu(cpu_map_bitmap_size(attr),
+	if (!cmap->flush_list)
-					    __alignof__(unsigned long));
-	if (!cmap->flush_needed)
 		goto free_charge;
+	for_each_possible_cpu(cpu)
+		INIT_LIST_HEAD(per_cpu_ptr(cmap->flush_list, cpu));
 	/* Alloc array for possible remote "destination" CPUs */
 	cmap->cpu_map = bpf_map_area_alloc(cmap->map.max_entries *
 					   sizeof(struct bpf_cpu_map_entry *),
 	return &cmap->map;
 free_percpu:
-	free_percpu(cmap->flush_needed);
+	free_percpu(cmap->flush_list);
 free_charge:
 	bpf_map_charge_finish(&cmap->map.memory);
 free_cmap:
 {
 	gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
 	struct bpf_cpu_map_entry *rcpu;
-	int numa, err;
+	struct xdp_bulk_queue *bq;
+	int numa, err, i;
 	/* Have map->numa_node, but choose node of redirect target CPU */
 	numa = cpu_to_node(cpu);
 	if (!rcpu->bulkq)
 		goto free_rcu;
+	for_each_possible_cpu(i) {
+		bq = per_cpu_ptr(rcpu->bulkq, i);
+		bq->obj = rcpu;
+	}
 	/* Alloc queue */
 	rcpu->queue = kzalloc_node(sizeof(*rcpu->queue), gfp, numa);
 	if (!rcpu->queue)
 		struct xdp_bulk_queue *bq = per_cpu_ptr(rcpu->bulkq, cpu);
 		/* No concurrent bq_enqueue can run at this point */
-		bq_flush_to_queue(rcpu, bq, false);
+		bq_flush_to_queue(bq, false);
 	}
 	free_percpu(rcpu->bulkq);
 	/* Cannot kthread_stop() here, last put free rcpu resources */
 		rcpu = __cpu_map_entry_alloc(qsize, key_cpu, map->id);
 		if (!rcpu)
 			return -ENOMEM;
+		rcpu->cmap = cmap;
 	}
 	rcu_read_lock();
 	__cpu_map_entry_replace(cmap, key_cpu, rcpu);
 	synchronize_rcu();
 	/* To ensure all pending flush operations have completed wait for flush
-	 * bitmap to indicate all flush_needed bits to be zero on _all_ cpus.
+	 * list be empty on _all_ cpus. Because the above synchronize_rcu()
-	 * Because the above synchronize_rcu() ensures the map is disconnected
+	 * ensures the map is disconnected from the program we can assume no new
-	 * from the program we can assume no new bits will be set.
+	 * items will be added to the list.
 	 */
 	for_each_online_cpu(cpu) {
-		unsigned long *bitmap = per_cpu_ptr(cmap->flush_needed, cpu);
+		struct list_head *flush_list = per_cpu_ptr(cmap->flush_list, cpu);
-		while (!bitmap_empty(bitmap, cmap->map.max_entries))
+		while (!list_empty(flush_list))
 			cond_resched();
 	}
 		/* bq flush and cleanup happens after RCU graze-period */
 		__cpu_map_entry_replace(cmap, i, NULL); /* call_rcu */
 	}
-	free_percpu(cmap->flush_needed);
+	free_percpu(cmap->flush_list);
 	bpf_map_area_free(cmap->cpu_map);
 	kfree(cmap);
 }
 	.map_check_btf		= map_check_no_btf,
 };
-static int bq_flush_to_queue(struct bpf_cpu_map_entry *rcpu,
+static int bq_flush_to_queue(struct xdp_bulk_queue *bq, bool in_napi_ctx)
-			     struct xdp_bulk_queue *bq, bool in_napi_ctx)
 {
+	struct bpf_cpu_map_entry *rcpu = bq->obj;
 	unsigned int processed = 0, drops = 0;
 	const int to_cpu = rcpu->cpu;
 	struct ptr_ring *q;
 	bq->count = 0;
 	spin_unlock(&q->producer_lock);
+	__list_del_clearprev(&bq->flush_node);
 	/* Feedback loop via tracepoints */
 	trace_xdp_cpumap_enqueue(rcpu->map_id, processed, drops, to_cpu);
 	return 0;
  */
 static int bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf)
 {
+	struct list_head *flush_list = this_cpu_ptr(rcpu->cmap->flush_list);
 	struct xdp_bulk_queue *bq = this_cpu_ptr(rcpu->bulkq);
 	if (unlikely(bq->count == CPU_MAP_BULK_SIZE))
-		bq_flush_to_queue(rcpu, bq, true);
+		bq_flush_to_queue(bq, true);
 	/* Notice, xdp_buff/page MUST be queued here, long enough for
 	 * driver to code invoking us to finished, due to driver
 	 * operation, when completing napi->poll call.
 	 */
 	bq->q[bq->count++] = xdpf;
+	if (!bq->flush_node.prev)
+		list_add(&bq->flush_node, flush_list);
 	return 0;
 }
 	return 0;
 }
-void __cpu_map_insert_ctx(struct bpf_map *map, u32 bit)
-{
-	struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
-	unsigned long *bitmap = this_cpu_ptr(cmap->flush_needed);
-	__set_bit(bit, bitmap);
-}
 void __cpu_map_flush(struct bpf_map *map)
 {
 	struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
-	unsigned long *bitmap = this_cpu_ptr(cmap->flush_needed);
+	struct list_head *flush_list = this_cpu_ptr(cmap->flush_list);
-	u32 bit;
+	struct xdp_bulk_queue *bq, *tmp;
-	/* The napi->poll softirq makes sure __cpu_map_insert_ctx()
-	 * and __cpu_map_flush() happen on same CPU. Thus, the percpu
-	 * bitmap indicate which percpu bulkq have packets.
-	 */
-	for_each_set_bit(bit, bitmap, map->max_entries) {
-		struct bpf_cpu_map_entry *rcpu = READ_ONCE(cmap->cpu_map[bit]);
-		struct xdp_bulk_queue *bq;
-		/* This is possible if entry is removed by user space
-		 * between xdp redirect and flush op.
-		 */
-		if (unlikely(!rcpu))
-			continue;
-		__clear_bit(bit, bitmap);
-		/* Flush all frames in bulkq to real queue */
+	list_for_each_entry_safe(bq, tmp, flush_list, flush_node) {
-		bq = this_cpu_ptr(rcpu->bulkq);
+		bq_flush_to_queue(bq, true);
-		bq_flush_to_queue(rcpu, bq, true);
 		/* If already running, costs spin_lock_irqsave + smb_mb */
-		wake_up_process(rcpu->kthread);
+		wake_up_process(bq->obj->kthread);
 	}
 }

kernel/bpf/devmap.c+52−60

  * datapath always has a valid copy. However, the datapath does a "flush"
  * operation that pushes any pending packets in the driver outside the RCU
  * critical section. Each bpf_dtab_netdev tracks these pending operations using
- * an atomic per-cpu bitmap. The bpf_dtab_netdev object will not be destroyed
+ * a per-cpu flush list. The bpf_dtab_netdev object will not be destroyed  until
- * until all bits are cleared indicating outstanding flush operations have
+ * this list is empty, indicating outstanding flush operations have completed.
- * completed.
  *
  * BPF syscalls may race with BPF program calls on any of the update, delete
  * or lookup operations. As noted above the xchg() operation also keep the
 	(BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY)
 #define DEV_MAP_BULK_SIZE 16
+struct bpf_dtab_netdev;
 struct xdp_bulk_queue {
 	struct xdp_frame *q[DEV_MAP_BULK_SIZE];
+	struct list_head flush_node;
 	struct net_device *dev_rx;
+	struct bpf_dtab_netdev *obj;
 	unsigned int count;
 };
 struct bpf_dtab {
 	struct bpf_map map;
 	struct bpf_dtab_netdev **netdev_map;
-	unsigned long __percpu *flush_needed;
+	struct list_head __percpu *flush_list;
 	struct list_head list;
 };
 static DEFINE_SPINLOCK(dev_map_lock);
 static LIST_HEAD(dev_map_list);
-static u64 dev_map_bitmap_size(const union bpf_attr *attr)
-{
-	return BITS_TO_LONGS((u64) attr->max_entries) * sizeof(unsigned long);
-}
 static struct bpf_map *dev_map_alloc(union bpf_attr *attr)
 {
 	struct bpf_dtab *dtab;
+	int err, cpu;
 	u64 cost;
-	int err;
 	if (!capable(CAP_NET_ADMIN))
 		return ERR_PTR(-EPERM);
 	    attr->value_size != 4 || attr->map_flags & ~DEV_CREATE_FLAG_MASK)
 		return ERR_PTR(-EINVAL);
+	/* Lookup returns a pointer straight to dev->ifindex, so make sure the
+	 * verifier prevents writes from the BPF side
+	 */
+	attr->map_flags |= BPF_F_RDONLY_PROG;
 	dtab = kzalloc(sizeof(*dtab), GFP_USER);
 	if (!dtab)
 		return ERR_PTR(-ENOMEM);
 	/* make sure page count doesn't overflow */
 	cost = (u64) dtab->map.max_entries * sizeof(struct bpf_dtab_netdev *);
-	cost += dev_map_bitmap_size(attr) * num_possible_cpus();
+	cost += sizeof(struct list_head) * num_possible_cpus();
 	/* if map size is larger than memlock limit, reject it */
 	err = bpf_map_charge_init(&dtab->map.memory, cost);
 	err = -ENOMEM;
-	/* A per cpu bitfield with a bit per possible net device */
+	dtab->flush_list = alloc_percpu(struct list_head);
-	dtab->flush_needed = __alloc_percpu_gfp(dev_map_bitmap_size(attr),
+	if (!dtab->flush_list)
-						__alignof__(unsigned long),
-						GFP_KERNEL | __GFP_NOWARN);
-	if (!dtab->flush_needed)
 		goto free_charge;
+	for_each_possible_cpu(cpu)
+		INIT_LIST_HEAD(per_cpu_ptr(dtab->flush_list, cpu));
 	dtab->netdev_map = bpf_map_area_alloc(dtab->map.max_entries *
 					      sizeof(struct bpf_dtab_netdev *),
 					      dtab->map.numa_node);
 	if (!dtab->netdev_map)
-		goto free_charge;
+		goto free_percpu;
 	spin_lock(&dev_map_lock);
 	list_add_tail_rcu(&dtab->list, &dev_map_list);
 	spin_unlock(&dev_map_lock);
 	return &dtab->map;
+free_percpu:
+	free_percpu(dtab->flush_list);
 free_charge:
 	bpf_map_charge_finish(&dtab->map.memory);
 free_dtab:
-	free_percpu(dtab->flush_needed);
 	kfree(dtab);
 	return ERR_PTR(err);
 }
 	rcu_barrier();
 	/* To ensure all pending flush operations have completed wait for flush
-	 * bitmap to indicate all flush_needed bits to be zero on _all_ cpus.
+	 * list to empty on _all_ cpus.
 	 * Because the above synchronize_rcu() ensures the map is disconnected
-	 * from the program we can assume no new bits will be set.
+	 * from the program we can assume no new items will be added.
 	 */
 	for_each_online_cpu(cpu) {
-		unsigned long *bitmap = per_cpu_ptr(dtab->flush_needed, cpu);
+		struct list_head *flush_list = per_cpu_ptr(dtab->flush_list, cpu);
-		while (!bitmap_empty(bitmap, dtab->map.max_entries))
+		while (!list_empty(flush_list))
 			cond_resched();
 	}
 		kfree(dev);
 	}
-	free_percpu(dtab->flush_needed);
+	free_percpu(dtab->flush_list);
 	bpf_map_area_free(dtab->netdev_map);
 	kfree(dtab);
 }
 	return 0;
 }
-void __dev_map_insert_ctx(struct bpf_map *map, u32 bit)
+static int bq_xmit_all(struct xdp_bulk_queue *bq, u32 flags,
-{
-	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
-	unsigned long *bitmap = this_cpu_ptr(dtab->flush_needed);
-	__set_bit(bit, bitmap);
-}
-static int bq_xmit_all(struct bpf_dtab_netdev *obj,
-		       struct xdp_bulk_queue *bq, u32 flags,
 		       bool in_napi_ctx)
 {
+	struct bpf_dtab_netdev *obj = bq->obj;
 	struct net_device *dev = obj->dev;
 	int sent = 0, drops = 0, err = 0;
 	int i;
 	trace_xdp_devmap_xmit(&obj->dtab->map, obj->bit,
 			      sent, drops, bq->dev_rx, dev, err);
 	bq->dev_rx = NULL;
+	__list_del_clearprev(&bq->flush_node);
 	return 0;
 error:
 	/* If ndo_xdp_xmit fails with an errno, no frames have been
  * from the driver before returning from its napi->poll() routine. The poll()
  * routine is called either from busy_poll context or net_rx_action signaled
  * from NET_RX_SOFTIRQ. Either way the poll routine must complete before the
- * net device can be torn down. On devmap tear down we ensure the ctx bitmap
+ * net device can be torn down. On devmap tear down we ensure the flush list
- * is zeroed before completing to ensure all flush operations have completed.
+ * is empty before completing to ensure all flush operations have completed.
  */
 void __dev_map_flush(struct bpf_map *map)
 {
 	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
-	unsigned long *bitmap = this_cpu_ptr(dtab->flush_needed);
+	struct list_head *flush_list = this_cpu_ptr(dtab->flush_list);
-	u32 bit;
+	struct xdp_bulk_queue *bq, *tmp;
 	rcu_read_lock();
-	for_each_set_bit(bit, bitmap, map->max_entries) {
+	list_for_each_entry_safe(bq, tmp, flush_list, flush_node)
-		struct bpf_dtab_netdev *dev = READ_ONCE(dtab->netdev_map[bit]);
+		bq_xmit_all(bq, XDP_XMIT_FLUSH, true);
-		struct xdp_bulk_queue *bq;
-		/* This is possible if the dev entry is removed by user space
-		 * between xdp redirect and flush op.
-		 */
-		if (unlikely(!dev))
-			continue;
-		bq = this_cpu_ptr(dev->bulkq);
-		bq_xmit_all(dev, bq, XDP_XMIT_FLUSH, true);
-		__clear_bit(bit, bitmap);
-	}
 	rcu_read_unlock();
 }
 		      struct net_device *dev_rx)
 {
+	struct list_head *flush_list = this_cpu_ptr(obj->dtab->flush_list);
 	struct xdp_bulk_queue *bq = this_cpu_ptr(obj->bulkq);
 	if (unlikely(bq->count == DEV_MAP_BULK_SIZE))
-		bq_xmit_all(obj, bq, 0, true);
+		bq_xmit_all(bq, 0, true);
 	/* Ingress dev_rx will be the same for all xdp_frame's in
 	 * bulk_queue, because bq stored per-CPU and must be flushed
 		bq->dev_rx = dev_rx;
 	bq->q[bq->count++] = xdpf;
+	if (!bq->flush_node.prev)
+		list_add(&bq->flush_node, flush_list);
 	return 0;
 }
 {
 	if (dev->dev->netdev_ops->ndo_xdp_xmit) {
 		struct xdp_bulk_queue *bq;
-		unsigned long *bitmap;
 		int cpu;
 		rcu_read_lock();
 		for_each_online_cpu(cpu) {
-			bitmap = per_cpu_ptr(dev->dtab->flush_needed, cpu);
-			__clear_bit(dev->bit, bitmap);
 			bq = per_cpu_ptr(dev->bulkq, cpu);
-			bq_xmit_all(dev, bq, XDP_XMIT_FLUSH, false);
+			bq_xmit_all(bq, XDP_XMIT_FLUSH, false);
 		}
 		rcu_read_unlock();
 	}
 	struct net *net = current->nsproxy->net_ns;
 	gfp_t gfp = GFP_ATOMIC | __GFP_NOWARN;
 	struct bpf_dtab_netdev *dev, *old_dev;
-	u32 i = *(u32 *)key;
 	u32 ifindex = *(u32 *)value;
+	struct xdp_bulk_queue *bq;
+	u32 i = *(u32 *)key;
+	int cpu;
 	if (unlikely(map_flags > BPF_EXIST))
 		return -EINVAL;
 			return -ENOMEM;
 		}
+		for_each_possible_cpu(cpu) {
+			bq = per_cpu_ptr(dev->bulkq, cpu);
+			bq->obj = dev;
+		}
 		dev->dev = dev_get_by_index(net, ifindex);
 		if (!dev->dev) {
 			free_percpu(dev->bulkq);

kernel/bpf/syscall.c+19−0

 		default:
 			return -EINVAL;
 		}
+	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
+		switch (expected_attach_type) {
+		case BPF_CGROUP_SETSOCKOPT:
+		case BPF_CGROUP_GETSOCKOPT:
+			return 0;
+		default:
+			return -EINVAL;
+		}
 	default:
 		return 0;
 	}
 	switch (prog->type) {
 	case BPF_PROG_TYPE_CGROUP_SOCK:
 	case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
+	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
 		return attach_type == prog->expected_attach_type ? 0 : -EINVAL;
 	case BPF_PROG_TYPE_CGROUP_SKB:
 		return prog->enforce_expected_attach_type &&
 	case BPF_CGROUP_SYSCTL:
 		ptype = BPF_PROG_TYPE_CGROUP_SYSCTL;
 		break;
+	case BPF_CGROUP_GETSOCKOPT:
+	case BPF_CGROUP_SETSOCKOPT:
+		ptype = BPF_PROG_TYPE_CGROUP_SOCKOPT;
+		break;
 	default:
 		return -EINVAL;
 	}
 	case BPF_CGROUP_SYSCTL:
 		ptype = BPF_PROG_TYPE_CGROUP_SYSCTL;
 		break;
+	case BPF_CGROUP_GETSOCKOPT:
+	case BPF_CGROUP_SETSOCKOPT:
+		ptype = BPF_PROG_TYPE_CGROUP_SOCKOPT;
+		break;
 	default:
 		return -EINVAL;
 	}
 	case BPF_CGROUP_SOCK_OPS:
 	case BPF_CGROUP_DEVICE:
 	case BPF_CGROUP_SYSCTL:
+	case BPF_CGROUP_GETSOCKOPT:
+	case BPF_CGROUP_SETSOCKOPT:
 		break;
 	case BPF_LIRC_MODE2:
 		return lirc_prog_query(attr, uattr);

kernel/bpf/verifier.c+117−19

@@ -1659,16 +1659,18 @@ static void mark_all_scalars_precise(struct bpf_verifier_env *env, · 1658 unmodified lines
1659	}	1659	}
1660	}	1660	}
1661		1661
1662	static int mark_chain_precision(struct bpf_verifier_env *env, int regno)	1662	static int __mark_chain_precision(struct bpf_verifier_env *env, int regno,
		1663	int spi)
1663	{	1664	{
1664	struct bpf_verifier_state *st = env->cur_state;	1665	struct bpf_verifier_state *st = env->cur_state;
1665	int first_idx = st->first_insn_idx;	1666	int first_idx = st->first_insn_idx;
1666	int last_idx = env->insn_idx;	1667	int last_idx = env->insn_idx;
1667	struct bpf_func_state *func;	1668	struct bpf_func_state *func;
1668	struct bpf_reg_state *reg;	1669	struct bpf_reg_state *reg;
1669	u32 reg_mask = 1u << regno;	1670	u32 reg_mask = regno >= 0 ? 1u << regno : 0;
1670	u64 stack_mask = 0;	1671	u64 stack_mask = spi >= 0 ? 1ull << spi : 0;
1671	bool skip_first = true;	1672	bool skip_first = true;
		1673	bool new_marks = false;
1672	int i, err;	1674	int i, err;
1673		1675
1674	if (!env->allow_ptr_leaks)	1676	if (!env->allow_ptr_leaks)
@@ -1676,18 +1678,43 @@ static int mark_chain_precision(struct bpf_verifier_env *env, int regno) · 1 unmodified line
1676	return 0;	1678	return 0;
1677		1679
1678	func = st->frame[st->curframe];	1680	func = st->frame[st->curframe];
1679	reg = &func->regs[regno];	1681	if (regno >= 0) {
1680	if (reg->type != SCALAR_VALUE) {	1682	reg = &func->regs[regno];
1681	WARN_ONCE(1, "backtracing misuse");	1683	if (reg->type != SCALAR_VALUE) {
1682	return -EFAULT;	1684	WARN_ONCE(1, "backtracing misuse");
		1685	return -EFAULT;
		1686	}
		1687	if (!reg->precise)
		1688	new_marks = true;
		1689	else
		1690	reg_mask = 0;
		1691	reg->precise = true;
		1692	}
		1693
		1694	while (spi >= 0) {
		1695	if (func->stack[spi].slot_type[0] != STACK_SPILL) {
		1696	stack_mask = 0;
		1697	break;
		1698	}
		1699	reg = &func->stack[spi].spilled_ptr;
		1700	if (reg->type != SCALAR_VALUE) {
		1701	stack_mask = 0;
		1702	break;
		1703	}
		1704	if (!reg->precise)
		1705	new_marks = true;
		1706	else
		1707	stack_mask = 0;
		1708	reg->precise = true;
		1709	break;
1683	}	1710	}
1684	if (reg->precise)
1685	return 0;
1686	func->regs[regno].precise = true;
1687		1711
		1712	if (!new_marks)
		1713	return 0;
		1714	if (!reg_mask && !stack_mask)
		1715	return 0;
1688	for (;;) {	1716	for (;;) {
1689	DECLARE_BITMAP(mask, 64);	1717	DECLARE_BITMAP(mask, 64);
1690	bool new_marks = false;
1691	u32 history = st->jmp_history_cnt;	1718	u32 history = st->jmp_history_cnt;
1692		1719
1693	if (env->log.level & BPF_LOG_LEVEL)	1720	if (env->log.level & BPF_LOG_LEVEL)
@@ -1730,12 +1757,15 @@ static int mark_chain_precision(struct bpf_verifier_env *env, int regno) · 36 unmodified lines
1730	if (!st)	1757	if (!st)
1731	break;	1758	break;
1732		1759
		1760	new_marks = false;
1733	func = st->frame[st->curframe];	1761	func = st->frame[st->curframe];
1734	bitmap_from_u64(mask, reg_mask);	1762	bitmap_from_u64(mask, reg_mask);
1735	for_each_set_bit(i, mask, 32) {	1763	for_each_set_bit(i, mask, 32) {
1736	reg = &func->regs[i];	1764	reg = &func->regs[i];
1737	if (reg->type != SCALAR_VALUE)	1765	if (reg->type != SCALAR_VALUE) {
		1766	reg_mask &= ~(1u << i);
1738	continue;	1767	continue;
		1768	}
1739	if (!reg->precise)	1769	if (!reg->precise)
1740	new_marks = true;	1770	new_marks = true;
1741	reg->precise = true;	1771	reg->precise = true;
@@ -1756,11 +1786,15 @@ static int mark_chain_precision(struct bpf_verifier_env *env, int regno) · 14 unmodified lines
1756	return -EFAULT;	1786	return -EFAULT;
1757	}	1787	}
1758		1788
1759	if (func->stack[i].slot_type[0] != STACK_SPILL)	1789	if (func->stack[i].slot_type[0] != STACK_SPILL) {
		1790	stack_mask &= ~(1ull << i);
1760	continue;	1791	continue;
		1792	}
1761	reg = &func->stack[i].spilled_ptr;	1793	reg = &func->stack[i].spilled_ptr;
1762	if (reg->type != SCALAR_VALUE)	1794	if (reg->type != SCALAR_VALUE) {
		1795	stack_mask &= ~(1ull << i);
1763	continue;	1796	continue;
		1797	}
1764	if (!reg->precise)	1798	if (!reg->precise)
1765	new_marks = true;	1799	new_marks = true;
1766	reg->precise = true;	1800	reg->precise = true;
@@ -1772,6 +1806,8 @@ static int mark_chain_precision(struct bpf_verifier_env *env, int regno) · 5 unmodified lines
1772	reg_mask, stack_mask);	1806	reg_mask, stack_mask);
1773	}	1807	}
1774		1808
		1809	if (!reg_mask && !stack_mask)
		1810	break;
1775	if (!new_marks)	1811	if (!new_marks)
1776	break;	1812	break;
1777		1813
@@ -1781,6 +1817,15 @@ static int mark_chain_precision(struct bpf_verifier_env *env, int regno) · 3 unmodified lines
1781	return 0;	1817	return 0;
1782	}	1818	}
1783		1819
		1820	static int mark_chain_precision(struct bpf_verifier_env *env, int regno)
		1821	{
		1822	return __mark_chain_precision(env, regno, -1);
		1823	}
		1824
		1825	static int mark_chain_precision_stack(struct bpf_verifier_env *env, int spi)
		1826	{
		1827	return __mark_chain_precision(env, -1, spi);
		1828	}
1784		1829
1785	static bool is_spillable_regtype(enum bpf_reg_type type)	1830	static bool is_spillable_regtype(enum bpf_reg_type type)
1786	{	1831	{
@@ -2215,6 +2260,13 @@ static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, · 428 unmodified lines
2215		2260
2216	env->seen_direct_write = true;	2261	env->seen_direct_write = true;
2217	return true;	2262	return true;
		2263
		2264	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
		2265	if (t == BPF_WRITE)
		2266	env->seen_direct_write = true;
		2267
		2268	return true;
		2269
2218	default:	2270	default:
2219	return false;	2271	return false;
2220	}	2272	}
@@ -3407,12 +3459,9 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, · 1186 unmodified lines
3407	if (func_id != BPF_FUNC_get_local_storage)	3459	if (func_id != BPF_FUNC_get_local_storage)
3408	goto error;	3460	goto error;
3409	break;	3461	break;
3410	/* devmap returns a pointer to a live net_device ifindex that we cannot
3411	* allow to be modified from bpf side. So do not allow lookup elements
3412	* for now.
3413	*/
3414	case BPF_MAP_TYPE_DEVMAP:	3462	case BPF_MAP_TYPE_DEVMAP:
3415	if (func_id != BPF_FUNC_redirect_map)	3463	if (func_id != BPF_FUNC_redirect_map &&
		3464	func_id != BPF_FUNC_map_lookup_elem)
3416	goto error;	3465	goto error;
3417	break;	3466	break;
3418	/* Restrict bpf side of cpumap and xskmap, open when use-cases	3467	/* Restrict bpf side of cpumap and xskmap, open when use-cases
@@ -6066,6 +6115,7 @@ static int check_return_code(struct bpf_verifier_env *env) · 2647 unmodified lines
6066	case BPF_PROG_TYPE_SOCK_OPS:	6115	case BPF_PROG_TYPE_SOCK_OPS:
6067	case BPF_PROG_TYPE_CGROUP_DEVICE:	6116	case BPF_PROG_TYPE_CGROUP_DEVICE:
6068	case BPF_PROG_TYPE_CGROUP_SYSCTL:	6117	case BPF_PROG_TYPE_CGROUP_SYSCTL:
		6118	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
6069	break;	6119	break;
6070	default:	6120	default:
6071	return 0;	6121	return 0;
@@ -7106,6 +7156,46 @@ static int propagate_liveness(struct bpf_verifier_env *env, · 1034 unmodified lines
7106	return 0;	7156	return 0;
7107	}	7157	}
7108		7158
		7159	/* find precise scalars in the previous equivalent state and
		7160	* propagate them into the current state
		7161	*/
		7162	static int propagate_precision(struct bpf_verifier_env *env,
		7163	const struct bpf_verifier_state *old)
		7164	{
		7165	struct bpf_reg_state *state_reg;
		7166	struct bpf_func_state *state;
		7167	int i, err = 0;
		7168
		7169	state = old->frame[old->curframe];
		7170	state_reg = state->regs;
		7171	for (i = 0; i < BPF_REG_FP; i++, state_reg++) {
		7172	if (state_reg->type != SCALAR_VALUE \|\|
		7173	!state_reg->precise)
		7174	continue;
		7175	if (env->log.level & BPF_LOG_LEVEL2)
		7176	verbose(env, "propagating r%d\n", i);
		7177	err = mark_chain_precision(env, i);
		7178	if (err < 0)
		7179	return err;
		7180	}
		7181
		7182	for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
		7183	if (state->stack[i].slot_type[0] != STACK_SPILL)
		7184	continue;
		7185	state_reg = &state->stack[i].spilled_ptr;
		7186	if (state_reg->type != SCALAR_VALUE \|\|
		7187	!state_reg->precise)
		7188	continue;
		7189	if (env->log.level & BPF_LOG_LEVEL2)
		7190	verbose(env, "propagating fp%d\n",
		7191	(-i - 1) * BPF_REG_SIZE);
		7192	err = mark_chain_precision_stack(env, i);
		7193	if (err < 0)
		7194	return err;
		7195	}
		7196	return 0;
		7197	}
		7198
7109	static bool states_maybe_looping(struct bpf_verifier_state *old,	7199	static bool states_maybe_looping(struct bpf_verifier_state *old,
7110	struct bpf_verifier_state *cur)	7200	struct bpf_verifier_state *cur)
7111	{	7201	{
@@ -7198,6 +7288,14 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) · 86 unmodified lines
7198	* this state and will pop a new one.	7288	* this state and will pop a new one.
7199	*/	7289	*/
7200	err = propagate_liveness(env, &sl->state, cur);	7290	err = propagate_liveness(env, &sl->state, cur);
		7291
		7292	/* if previous state reached the exit with precision and
		7293	* current state is equivalent to it (except precsion marks)
		7294	* the precision needs to be propagated back in
		7295	* the current state.
		7296	*/
		7297	err = err ? : push_jmp_history(env, cur);
		7298	err = err ? : propagate_precision(env, &sl->state);
7201	if (err)	7299	if (err)
7202	return err;	7300	return err;
7203	return 1;	7301	return 1;

kernel/bpf/xskmap.c+1−2

@@ -145,8 +145,7 @@ void __xsk_map_flush(struct bpf_map *map) · 144 unmodified lines
145		145
146	list_for_each_entry_safe(xs, tmp, flush_list, flush_node) {	146	list_for_each_entry_safe(xs, tmp, flush_list, flush_node) {
147	xsk_flush(xs);	147	xsk_flush(xs);
148	__list_del(xs->flush_node.prev, xs->flush_node.next);	148	__list_del_clearprev(&xs->flush_node);
149	xs->flush_node.prev = NULL;
150	}	149	}
151	}	150	}
152		151

kernel/trace/bpf_trace.c+14−13

 	return err;
 }
+static int __init send_signal_irq_work_init(void)
+{
+	int cpu;
+	struct send_signal_irq_work *work;
+	for_each_possible_cpu(cpu) {
+		work = per_cpu_ptr(&send_signal_work, cpu);
+		init_irq_work(&work->irq_work, do_bpf_send_signal);
+	}
+	return 0;
+}
+subsys_initcall(send_signal_irq_work_init);
 #ifdef CONFIG_MODULES
 static int bpf_event_notify(struct notifier_block *nb, unsigned long op,
 			    void *module)
 	return 0;
 }
-static int __init send_signal_irq_work_init(void)
-{
-	int cpu;
-	struct send_signal_irq_work *work;
-	for_each_possible_cpu(cpu) {
-		work = per_cpu_ptr(&send_signal_work, cpu);
-		init_irq_work(&work->irq_work, do_bpf_send_signal);
-	}
-	return 0;
-}
 fs_initcall(bpf_event_init);
-subsys_initcall(send_signal_irq_work_init);
 #endif /* CONFIG_MODULES */

net/core/filter.c+185−84

 	if (unlikely(flags & ~(BPF_F_INGRESS)))
 		return TC_ACT_SHOT;
-	ri->ifindex = ifindex;
 	ri->flags = flags;
+	ri->tgt_index = ifindex;
 	return TC_ACT_REDIRECT;
 }
 	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
 	struct net_device *dev;
-	dev = dev_get_by_index_rcu(dev_net(skb->dev), ri->ifindex);
+	dev = dev_get_by_index_rcu(dev_net(skb->dev), ri->tgt_index);
-	ri->ifindex = 0;
+	ri->tgt_index = 0;
 	if (unlikely(!dev)) {
 		kfree_skb(skb);
 		return -EINVAL;
 		     struct bpf_prog *xdp_prog, struct bpf_redirect_info *ri)
 {
 	struct net_device *fwd;
-	u32 index = ri->ifindex;
+	u32 index = ri->tgt_index;
 	int err;
 	fwd = dev_get_by_index_rcu(dev_net(dev), index);
-	ri->ifindex = 0;
+	ri->tgt_index = 0;
 	if (unlikely(!fwd)) {
 		err = -EINVAL;
 		goto err;
 		err = dev_map_enqueue(dst, xdp, dev_rx);
 		if (unlikely(err))
 			return err;
-		__dev_map_insert_ctx(map, index);
 		break;
 	}
 	case BPF_MAP_TYPE_CPUMAP: {
 		err = cpu_map_enqueue(rcpu, xdp, dev_rx);
 		if (unlikely(err))
 			return err;
-		__cpu_map_insert_ctx(map, index);
 		break;
 	}
 	case BPF_MAP_TYPE_XSKMAP: {
 			       struct bpf_prog *xdp_prog, struct bpf_map *map,
 			       struct bpf_redirect_info *ri)
 {
-	u32 index = ri->ifindex;
+	u32 index = ri->tgt_index;
-	void *fwd = NULL;
+	void *fwd = ri->tgt_value;
 	int err;
-	ri->ifindex = 0;
+	ri->tgt_index = 0;
+	ri->tgt_value = NULL;
 	WRITE_ONCE(ri->map, NULL);
-	fwd = __xdp_map_lookup_elem(map, index);
-	if (unlikely(!fwd)) {
-		err = -EINVAL;
-		goto err;
-	}
 	if (ri->map_to_flush && unlikely(ri->map_to_flush != map))
 		xdp_do_flush_map();
 				       struct bpf_map *map)
 {
 	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
-	u32 index = ri->ifindex;
+	u32 index = ri->tgt_index;
-	void *fwd = NULL;
+	void *fwd = ri->tgt_value;
 	int err = 0;
-	ri->ifindex = 0;
+	ri->tgt_index = 0;
+	ri->tgt_value = NULL;
 	WRITE_ONCE(ri->map, NULL);
-	fwd = __xdp_map_lookup_elem(map, index);
-	if (unlikely(!fwd)) {
-		err = -EINVAL;
-		goto err;
-	}
 	if (map->map_type == BPF_MAP_TYPE_DEVMAP) {
 		struct bpf_dtab_netdev *dst = fwd;
 {
 	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
 	struct bpf_map *map = READ_ONCE(ri->map);
-	u32 index = ri->ifindex;
+	u32 index = ri->tgt_index;
 	struct net_device *fwd;
 	int err = 0;
 	if (map)
 		return xdp_do_generic_redirect_map(dev, skb, xdp, xdp_prog,
 						   map);
-	ri->ifindex = 0;
+	ri->tgt_index = 0;
 	fwd = dev_get_by_index_rcu(dev_net(dev), index);
 	if (unlikely(!fwd)) {
 		err = -EINVAL;
 	if (unlikely(flags))
 		return XDP_ABORTED;
-	ri->ifindex = ifindex;
 	ri->flags = flags;
+	ri->tgt_index = ifindex;
+	ri->tgt_value = NULL;
 	WRITE_ONCE(ri->map, NULL);
 	return XDP_REDIRECT;
 {
 	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
-	if (unlikely(flags))
+	/* Lower bits of the flags are used as return code on lookup failure */
+	if (unlikely(flags > XDP_TX))
 		return XDP_ABORTED;
-	ri->ifindex = ifindex;
+	ri->tgt_value = __xdp_map_lookup_elem(map, ifindex);
+	if (unlikely(!ri->tgt_value)) {
+		/* If the lookup fails we want to clear out the state in the
+		 * redirect_info struct completely, so that if an eBPF program
+		 * performs multiple lookups, the last one always takes
+		 * precedence.
+		 */
+		WRITE_ONCE(ri->map, NULL);
+		return flags;
+	}
 	ri->flags = flags;
+	ri->tgt_index = ifindex;
 	WRITE_ONCE(ri->map, map);
 	return XDP_REDIRECT;
 };
 #endif /* CONFIG_IPV6_SEG6_BPF */
-#define CONVERT_COMMON_TCP_SOCK_FIELDS(md_type, CONVERT)		\
-do {									\
-	switch (si->off) {						\
-	case offsetof(md_type, snd_cwnd):				\
-		CONVERT(snd_cwnd); break;				\
-	case offsetof(md_type, srtt_us):				\
-		CONVERT(srtt_us); break;				\
-	case offsetof(md_type, snd_ssthresh):				\
-		CONVERT(snd_ssthresh); break;				\
-	case offsetof(md_type, rcv_nxt):				\
-		CONVERT(rcv_nxt); break;				\
-	case offsetof(md_type, snd_nxt):				\
-		CONVERT(snd_nxt); break;				\
-	case offsetof(md_type, snd_una):				\
-		CONVERT(snd_una); break;				\
-	case offsetof(md_type, mss_cache):				\
-		CONVERT(mss_cache); break;				\
-	case offsetof(md_type, ecn_flags):				\
-		CONVERT(ecn_flags); break;				\
-	case offsetof(md_type, rate_delivered):				\
-		CONVERT(rate_delivered); break;				\
-	case offsetof(md_type, rate_interval_us):			\
-		CONVERT(rate_interval_us); break;			\
-	case offsetof(md_type, packets_out):				\
-		CONVERT(packets_out); break;				\
-	case offsetof(md_type, retrans_out):				\
-		CONVERT(retrans_out); break;				\
-	case offsetof(md_type, total_retrans):				\
-		CONVERT(total_retrans); break;				\
-	case offsetof(md_type, segs_in):				\
-		CONVERT(segs_in); break;				\
-	case offsetof(md_type, data_segs_in):				\
-		CONVERT(data_segs_in); break;				\
-	case offsetof(md_type, segs_out):				\
-		CONVERT(segs_out); break;				\
-	case offsetof(md_type, data_segs_out):				\
-		CONVERT(data_segs_out); break;				\
-	case offsetof(md_type, lost_out):				\
-		CONVERT(lost_out); break;				\
-	case offsetof(md_type, sacked_out):				\
-		CONVERT(sacked_out); break;				\
-	case offsetof(md_type, bytes_received):				\
-		CONVERT(bytes_received); break;				\
-	case offsetof(md_type, bytes_acked):				\
-		CONVERT(bytes_acked); break;				\
-	}								\
-} while (0)
 #ifdef CONFIG_INET
 static struct sock *sk_lookup(struct net *net, struct bpf_sock_tuple *tuple,
 			      int dif, int sdif, u8 family, u8 proto)
 bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
 				  struct bpf_insn_access_aux *info)
 {
-	if (off < 0 || off >= offsetofend(struct bpf_tcp_sock, bytes_acked))
+	if (off < 0 || off >= offsetofend(struct bpf_tcp_sock,
+					  icsk_retransmits))
 		return false;
 	if (off % size != 0)
 				      offsetof(struct tcp_sock, FIELD)); \
 	} while (0)
-	CONVERT_COMMON_TCP_SOCK_FIELDS(struct bpf_tcp_sock,
+#define BPF_INET_SOCK_GET_COMMON(FIELD)					\
-				       BPF_TCP_SOCK_GET_COMMON);
+	do {								\
+		BUILD_BUG_ON(FIELD_SIZEOF(struct inet_connection_sock,	\
+					  FIELD) >			\
+			     FIELD_SIZEOF(struct bpf_tcp_sock, FIELD));	\
+		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(			\
+					struct inet_connection_sock,	\
+					FIELD),				\
+				      si->dst_reg, si->src_reg,		\
+				      offsetof(				\
+					struct inet_connection_sock,	\
+					FIELD));			\
+	} while (0)
 	if (insn > insn_buf)
 		return insn - insn_buf;
 				      offsetof(struct tcp_sock, rtt_min) +
 				      offsetof(struct minmax_sample, v));
 		break;
+	case offsetof(struct bpf_tcp_sock, snd_cwnd):
+		BPF_TCP_SOCK_GET_COMMON(snd_cwnd);
+		break;
+	case offsetof(struct bpf_tcp_sock, srtt_us):
+		BPF_TCP_SOCK_GET_COMMON(srtt_us);
+		break;
+	case offsetof(struct bpf_tcp_sock, snd_ssthresh):
+		BPF_TCP_SOCK_GET_COMMON(snd_ssthresh);
+		break;
+	case offsetof(struct bpf_tcp_sock, rcv_nxt):
+		BPF_TCP_SOCK_GET_COMMON(rcv_nxt);
+		break;
+	case offsetof(struct bpf_tcp_sock, snd_nxt):
+		BPF_TCP_SOCK_GET_COMMON(snd_nxt);
+		break;
+	case offsetof(struct bpf_tcp_sock, snd_una):
+		BPF_TCP_SOCK_GET_COMMON(snd_una);
+		break;
+	case offsetof(struct bpf_tcp_sock, mss_cache):
+		BPF_TCP_SOCK_GET_COMMON(mss_cache);
+		break;
+	case offsetof(struct bpf_tcp_sock, ecn_flags):
+		BPF_TCP_SOCK_GET_COMMON(ecn_flags);
+		break;
+	case offsetof(struct bpf_tcp_sock, rate_delivered):
+		BPF_TCP_SOCK_GET_COMMON(rate_delivered);
+		break;
+	case offsetof(struct bpf_tcp_sock, rate_interval_us):
+		BPF_TCP_SOCK_GET_COMMON(rate_interval_us);
+		break;
+	case offsetof(struct bpf_tcp_sock, packets_out):
+		BPF_TCP_SOCK_GET_COMMON(packets_out);
+		break;
+	case offsetof(struct bpf_tcp_sock, retrans_out):
+		BPF_TCP_SOCK_GET_COMMON(retrans_out);
+		break;
+	case offsetof(struct bpf_tcp_sock, total_retrans):
+		BPF_TCP_SOCK_GET_COMMON(total_retrans);
+		break;
+	case offsetof(struct bpf_tcp_sock, segs_in):
+		BPF_TCP_SOCK_GET_COMMON(segs_in);
+		break;
+	case offsetof(struct bpf_tcp_sock, data_segs_in):
+		BPF_TCP_SOCK_GET_COMMON(data_segs_in);
+		break;
+	case offsetof(struct bpf_tcp_sock, segs_out):
+		BPF_TCP_SOCK_GET_COMMON(segs_out);
+		break;
+	case offsetof(struct bpf_tcp_sock, data_segs_out):
+		BPF_TCP_SOCK_GET_COMMON(data_segs_out);
+		break;
+	case offsetof(struct bpf_tcp_sock, lost_out):
+		BPF_TCP_SOCK_GET_COMMON(lost_out);
+		break;
+	case offsetof(struct bpf_tcp_sock, sacked_out):
+		BPF_TCP_SOCK_GET_COMMON(sacked_out);
+		break;
+	case offsetof(struct bpf_tcp_sock, bytes_received):
+		BPF_TCP_SOCK_GET_COMMON(bytes_received);
+		break;
+	case offsetof(struct bpf_tcp_sock, bytes_acked):
+		BPF_TCP_SOCK_GET_COMMON(bytes_acked);
+		break;
+	case offsetof(struct bpf_tcp_sock, dsack_dups):
+		BPF_TCP_SOCK_GET_COMMON(dsack_dups);
+		break;
+	case offsetof(struct bpf_tcp_sock, delivered):
+		BPF_TCP_SOCK_GET_COMMON(delivered);
+		break;
+	case offsetof(struct bpf_tcp_sock, delivered_ce):
+		BPF_TCP_SOCK_GET_COMMON(delivered_ce);
+		break;
+	case offsetof(struct bpf_tcp_sock, icsk_retransmits):
+		BPF_INET_SOCK_GET_COMMON(icsk_retransmits);
+		break;
 	}
 	return insn - insn_buf;
 	return (unsigned long)NULL;
 }
-static const struct bpf_func_proto bpf_tcp_sock_proto = {
+const struct bpf_func_proto bpf_tcp_sock_proto = {
 	.func		= bpf_tcp_sock,
 	.gpl_only	= false,
 	.ret_type	= RET_PTR_TO_TCP_SOCK_OR_NULL,
 			SOCK_OPS_GET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ);	      \
 	} while (0)
-	CONVERT_COMMON_TCP_SOCK_FIELDS(struct bpf_sock_ops,
-				       SOCK_OPS_GET_TCP_SOCK_FIELD);
 	if (insn > insn_buf)
 		return insn - insn_buf;
 		SOCK_OPS_GET_OR_SET_FIELD(sk_txhash, sk_txhash,
 					  struct sock, type);
 		break;
+	case offsetof(struct bpf_sock_ops, snd_cwnd):
+		SOCK_OPS_GET_TCP_SOCK_FIELD(snd_cwnd);
+		break;
+	case offsetof(struct bpf_sock_ops, srtt_us):
+		SOCK_OPS_GET_TCP_SOCK_FIELD(srtt_us);
+		break;
+	case offsetof(struct bpf_sock_ops, snd_ssthresh):
+		SOCK_OPS_GET_TCP_SOCK_FIELD(snd_ssthresh);
+		break;
+	case offsetof(struct bpf_sock_ops, rcv_nxt):
+		SOCK_OPS_GET_TCP_SOCK_FIELD(rcv_nxt);
+		break;
+	case offsetof(struct bpf_sock_ops, snd_nxt):
+		SOCK_OPS_GET_TCP_SOCK_FIELD(snd_nxt);
+		break;
+	case offsetof(struct bpf_sock_ops, snd_una):
+		SOCK_OPS_GET_TCP_SOCK_FIELD(snd_una);
+		break;
+	case offsetof(struct bpf_sock_ops, mss_cache):
+		SOCK_OPS_GET_TCP_SOCK_FIELD(mss_cache);
+		break;
+	case offsetof(struct bpf_sock_ops, ecn_flags):
+		SOCK_OPS_GET_TCP_SOCK_FIELD(ecn_flags);
+		break;
+	case offsetof(struct bpf_sock_ops, rate_delivered):
+		SOCK_OPS_GET_TCP_SOCK_FIELD(rate_delivered);
+		break;
+	case offsetof(struct bpf_sock_ops, rate_interval_us):
+		SOCK_OPS_GET_TCP_SOCK_FIELD(rate_interval_us);
+		break;
+	case offsetof(struct bpf_sock_ops, packets_out):
+		SOCK_OPS_GET_TCP_SOCK_FIELD(packets_out);
+		break;
+	case offsetof(struct bpf_sock_ops, retrans_out):
+		SOCK_OPS_GET_TCP_SOCK_FIELD(retrans_out);
+		break;
+	case offsetof(struct bpf_sock_ops, total_retrans):
+		SOCK_OPS_GET_TCP_SOCK_FIELD(total_retrans);
+		break;
+	case offsetof(struct bpf_sock_ops, segs_in):
+		SOCK_OPS_GET_TCP_SOCK_FIELD(segs_in);
+		break;
+	case offsetof(struct bpf_sock_ops, data_segs_in):
+		SOCK_OPS_GET_TCP_SOCK_FIELD(data_segs_in);
+		break;
+	case offsetof(struct bpf_sock_ops, segs_out):
+		SOCK_OPS_GET_TCP_SOCK_FIELD(segs_out);
+		break;
+	case offsetof(struct bpf_sock_ops, data_segs_out):
+		SOCK_OPS_GET_TCP_SOCK_FIELD(data_segs_out);
+		break;
+	case offsetof(struct bpf_sock_ops, lost_out):
+		SOCK_OPS_GET_TCP_SOCK_FIELD(lost_out);
+		break;
+	case offsetof(struct bpf_sock_ops, sacked_out):
+		SOCK_OPS_GET_TCP_SOCK_FIELD(sacked_out);
+		break;
+	case offsetof(struct bpf_sock_ops, bytes_received):
+		SOCK_OPS_GET_TCP_SOCK_FIELD(bytes_received);
+		break;
+	case offsetof(struct bpf_sock_ops, bytes_acked):
+		SOCK_OPS_GET_TCP_SOCK_FIELD(bytes_acked);
+		break;
 	case offsetof(struct bpf_sock_ops, sk):
 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
 						struct bpf_sock_ops_kern,

net/core/xdp.c+1−1

@@ -85,7 +85,7 @@ static void __xdp_mem_allocator_rcu_free(struct rcu_head *rcu) · 84 unmodified lines
85	kfree(xa);	85	kfree(xa);
86	}	86	}
87		87
88	bool __mem_id_disconnect(int id, bool force)	88	static bool __mem_id_disconnect(int id, bool force)
89	{	89	{
90	struct xdp_mem_allocator *xa;	90	struct xdp_mem_allocator *xa;
91	bool safe_to_remove = true;	91	bool safe_to_remove = true;

net/ipv4/tcp_input.c+4−0

 				tp->rttvar_us -= (tp->rttvar_us - tp->mdev_max_us) >> 2;
 			tp->rtt_seq = tp->snd_nxt;
 			tp->mdev_max_us = tcp_rto_min_us(sk);
+			tcp_bpf_rtt(sk);
 		}
 	} else {
 		/* no previous measure. */
 		tp->rttvar_us = max(tp->mdev_us, tcp_rto_min_us(sk));
 		tp->mdev_max_us = tp->rttvar_us;
 		tp->rtt_seq = tp->snd_nxt;
+		tcp_bpf_rtt(sk);
 	}
 	tp->srtt_us = max(1U, srtt);
 }

net/socket.c+30−0

 static int __sys_setsockopt(int fd, int level, int optname,
 			    char __user *optval, int optlen)
 {
+	mm_segment_t oldfs = get_fs();
+	char *kernel_optval = NULL;
 	int err, fput_needed;
 	struct socket *sock;
 		if (err)
 			goto out_put;
+		err = BPF_CGROUP_RUN_PROG_SETSOCKOPT(sock->sk, &level,
+						     &optname, optval, &optlen,
+						     &kernel_optval);
+		if (err < 0) {
+			goto out_put;
+		} else if (err > 0) {
+			err = 0;
+			goto out_put;
+		}
+		if (kernel_optval) {
+			set_fs(KERNEL_DS);
+			optval = (char __user __force *)kernel_optval;
+		}
 		if (level == SOL_SOCKET)
 			err =
 			    sock_setsockopt(sock, level, optname, optval,
 			err =
 			    sock->ops->setsockopt(sock, level, optname, optval,
 						  optlen);
+		if (kernel_optval) {
+			set_fs(oldfs);
+			kfree(kernel_optval);
+		}
 out_put:
 		fput_light(sock->file, fput_needed);
 	}
 {
 	int err, fput_needed;
 	struct socket *sock;
+	int max_optlen;
 	sock = sockfd_lookup_light(fd, &err, &fput_needed);
 	if (sock != NULL) {
 		if (err)
 			goto out_put;
+		max_optlen = BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen);
 		if (level == SOL_SOCKET)
 			err =
 			    sock_getsockopt(sock, level, optname, optval,
 			err =
 			    sock->ops->getsockopt(sock, level, optname, optval,
 						  optlen);
+		err = BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock->sk, level, optname,
+						     optval, optlen,
+						     max_optlen, err);
 out_put:
 		fput_light(sock->file, fput_needed);
 	}

net/xdp/xsk.c+29−7

 		READ_ONCE(xs->umem->fq);
 }
+bool xsk_umem_has_addrs(struct xdp_umem *umem, u32 cnt)
+{
+	return xskq_has_addrs(umem->fq, cnt);
+}
+EXPORT_SYMBOL(xsk_umem_has_addrs);
 u64 *xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr)
 {
 	return xskq_peek_addr(umem->fq, addr);
 }
 EXPORT_SYMBOL(xsk_umem_consume_tx_done);
-bool xsk_umem_consume_tx(struct xdp_umem *umem, dma_addr_t *dma, u32 *len)
+bool xsk_umem_consume_tx(struct xdp_umem *umem, struct xdp_desc *desc)
 {
-	struct xdp_desc desc;
 	struct xdp_sock *xs;
 	rcu_read_lock();
 	list_for_each_entry_rcu(xs, &umem->xsk_list, list) {
-		if (!xskq_peek_desc(xs->tx, &desc))
+		if (!xskq_peek_desc(xs->tx, desc))
 			continue;
-		if (xskq_produce_addr_lazy(umem->cq, desc.addr))
+		if (xskq_produce_addr_lazy(umem->cq, desc->addr))
 			goto out;
-		*dma = xdp_umem_get_dma(umem, desc.addr);
-		*len = desc.len;
 		xskq_discard_desc(xs->tx);
 		rcu_read_unlock();
 		return true;
 		return 0;
 	}
+	case XDP_OPTIONS:
+	{
+		struct xdp_options opts = {};
+		if (len < sizeof(opts))
+			return -EINVAL;
+		mutex_lock(&xs->mutex);
+		if (xs->zc)
+			opts.flags |= XDP_OPTIONS_ZEROCOPY;
+		mutex_unlock(&xs->mutex);
+		len = sizeof(opts);
+		if (copy_to_user(optval, &opts, len))
+			return -EFAULT;
+		if (put_user(len, optlen))
+			return -EFAULT;
+		return 0;
+	}
 	default:
 		break;
 	}

net/xdp/xsk_queue.h+14−0

@@ -117,6 +117,20 @@ static inline u32 xskq_nb_free(struct xsk_queue *q, u32 producer, u32 dcnt) · 116 unmodified lines
117	return q->nentries - (producer - q->cons_tail);	117	return q->nentries - (producer - q->cons_tail);
118	}	118	}
119		119
		120	static inline bool xskq_has_addrs(struct xsk_queue *q, u32 cnt)
		121	{
		122	u32 entries = q->prod_tail - q->cons_tail;
		123
		124	if (entries >= cnt)
		125	return true;
		126
		127	/* Refresh the local pointer. */
		128	q->prod_tail = READ_ONCE(q->ring->producer);
		129	entries = q->prod_tail - q->cons_tail;
		130
		131	return entries >= cnt;
		132	}
		133
120	/* UMEM queue */	134	/* UMEM queue */
121		135
122	static inline bool xskq_is_valid_addr(struct xsk_queue *q, u64 addr)	136	static inline bool xskq_is_valid_addr(struct xsk_queue *q, u64 addr)

samples/bpf/Makefile+3−0

 always += tcp_clamp_kern.o
 always += tcp_basertt_kern.o
 always += tcp_tos_reflect_kern.o
+always += tcp_dumpstats_kern.o
 always += xdp_redirect_kern.o
 always += xdp_redirect_map_kern.o
 always += xdp_redirect_cpu_kern.o
 always += xdp_sample_pkts_kern.o
 always += ibumad_kern.o
 always += hbm_out_kern.o
+always += hbm_edt_kern.o
 KBUILD_HOSTCFLAGS += -I$(objtree)/usr/include
 KBUILD_HOSTCFLAGS += -I$(srctree)/tools/lib/bpf/
 $(obj)/tracex5_kern.o: $(obj)/syscall_nrs.h
 $(obj)/hbm_out_kern.o: $(src)/hbm.h $(src)/hbm_kern.h
 $(obj)/hbm.o: $(src)/hbm.h
+$(obj)/hbm_edt_kern.o: $(src)/hbm.h $(src)/hbm_kern.h
 # asm/sysreg.h - inline assembly used by it is incompatible with llvm.
 # But, there is no easy way to fix it, so just exclude it since it is

samples/bpf/do_hbm_test.sh+12−10

@@ -14,7 +14,7 @@ Usage() { · 13 unmodified lines
14	echo "loads. The output is the goodput in Mbps (unless -D was used)."	14	echo "loads. The output is the goodput in Mbps (unless -D was used)."
15	echo ""	15	echo ""
16	echo "USAGE: $name [out] [-b=<prog>\|--bpf=<prog>] [-c=<cc>\|--cc=<cc>]"	16	echo "USAGE: $name [out] [-b=<prog>\|--bpf=<prog>] [-c=<cc>\|--cc=<cc>]"
17	echo " [-D] [-d=<delay>\|--delay=<delay>] [--debug] [-E]"	17	echo " [-D] [-d=<delay>\|--delay=<delay>] [--debug] [-E] [--edt]"
18	echo " [-f=<#flows>\|--flows=<#flows>] [-h] [-i=<id>\|--id=<id >]"	18	echo " [-f=<#flows>\|--flows=<#flows>] [-h] [-i=<id>\|--id=<id >]"
19	echo " [-l] [-N] [--no_cn] [-p=<port>\|--port=<port>] [-P]"	19	echo " [-l] [-N] [--no_cn] [-p=<port>\|--port=<port>] [-P]"
20	echo " [-q=<qdisc>] [-R] [-s=<server>\|--server=<server]"	20	echo " [-q=<qdisc>] [-R] [-s=<server>\|--server=<server]"
@@ -30,6 +30,7 @@ Usage() { · 9 unmodified lines
30	echo " other detailed information. This information is"	30	echo " other detailed information. This information is"
31	echo " test dependent (i.e. iperf3 or netperf)."	31	echo " test dependent (i.e. iperf3 or netperf)."
32	echo " -E enable ECN (not required for dctcp)"	32	echo " -E enable ECN (not required for dctcp)"
		33	echo " --edt use fq's Earliest Departure Time (requires fq)"
33	echo " -f or --flows number of concurrent flows (default=1)"	34	echo " -f or --flows number of concurrent flows (default=1)"
34	echo " -i or --id cgroup id (an integer, default is 1)"	35	echo " -i or --id cgroup id (an integer, default is 1)"
35	echo " -N use netperf instead of iperf3"	36	echo " -N use netperf instead of iperf3"
@@ -130,13 +131,12 @@ processArgs () { · 94 unmodified lines
130	details=1	131	details=1
131	;;	132	;;
132	-E)	133	-E)
133	ecn=1	134	ecn=1
		135	;;
		136	--edt)
		137	flags="$flags --edt"
		138	qdisc="fq"
134	;;	139	;;
135	# Support for upcomming fq Early Departure Time egress rate limiting
136	#--edt)
137	# prog="hbm_out_edt_kern.o"
138	# qdisc="fq"
139	# ;;
140	-f=\|--flows=)	140	-f=\|--flows=)
141	flows="${i#*=}"	141	flows="${i#*=}"
142	;;	142	;;
@@ -228,8 +228,8 @@ if [ "$netem" -ne "0" ] ; then · 85 unmodified lines
228	tc qdisc del dev lo root > /dev/null 2>&1	228	tc qdisc del dev lo root > /dev/null 2>&1
229	tc qdisc add dev lo root netem delay $netem\ms > /dev/null 2>&1	229	tc qdisc add dev lo root netem delay $netem\ms > /dev/null 2>&1
230	elif [ "$qdisc" != "" ] ; then	230	elif [ "$qdisc" != "" ] ; then
231	tc qdisc del dev lo root > /dev/null 2>&1	231	tc qdisc del dev eth0 root > /dev/null 2>&1
232	tc qdisc add dev lo root $qdisc > /dev/null 2>&1	232	tc qdisc add dev eth0 root $qdisc > /dev/null 2>&1
233	fi	233	fi
234		234
235	n=0	235	n=0
@@ -399,7 +399,9 @@ fi · 163 unmodified lines
399	if [ "$netem" -ne "0" ] ; then	399	if [ "$netem" -ne "0" ] ; then
400	tc qdisc del dev lo root > /dev/null 2>&1	400	tc qdisc del dev lo root > /dev/null 2>&1
401	fi	401	fi
402		402	if [ "$qdisc" != "" ] ; then
		403	tc qdisc del dev eth0 root > /dev/null 2>&1
		404	fi
403	sleep 2	405	sleep 2
404		406
405	hbmPid=`ps ax \| grep "hbm " \| grep --invert-match "grep" \| awk '{ print $1 }'`	407	hbmPid=`ps ax \| grep "hbm " \| grep --invert-match "grep" \| awk '{ print $1 }'`

samples/bpf/hbm.c+15−3

 bool debugFlag;
 bool work_conserving_flag;
 bool no_cn_flag;
+bool edt_flag;
 static void Usage(void);
 static void read_trace_pipe2(void);
 		fprintf(fout, "avg rtt:%d\n",
 			(int)(qstats.sum_rtt / (qstats.pkts_total + 1)));
 		// Average credit
-		fprintf(fout, "avg credit:%d\n",
+		if (edt_flag)
-			(int)(qstats.sum_credit /
+			fprintf(fout, "avg credit_ms:%.03f\n",
-			      (1500 * ((int)qstats.pkts_total) + 1)));
+				(qstats.sum_credit /
+				 (qstats.pkts_total + 1.0)) / 1000000.0);
+		else
+			fprintf(fout, "avg credit:%d\n",
+				(int)(qstats.sum_credit /
+				      (1500 * ((int)qstats.pkts_total ) + 1)));
 		// Return values stats
 		for (k = 0; k < RET_VAL_COUNT; k++) {
 	       "  Where:\n"
 	       "    -o         indicates egress direction (default)\n"
 	       "    -d         print BPF trace debug buffer\n"
+	       "    --edt      use fq's Earliest Departure Time\n"
 	       "    -l         also limit flows using loopback\n"
 	       "    -n <#>     to create cgroup \"/hbm#\" and attach prog\n"
 	       "               Default is /hbm1\n"
 	char *optstring = "iodln:r:st:wh";
 	struct option loptions[] = {
 		{"no_cn", 0, NULL, 1},
+		{"edt", 0, NULL, 2},
 		{NULL, 0, NULL, 0}
 	};
 		case 1:
 			no_cn_flag = true;
 			break;
+		case 2:
+			prog = "hbm_edt_kern.o";
+			edt_flag = true;
+			break;
 		case'o':
 			break;
 		case 'd':

samples/bpf/hbm_edt_kern.c+168−0

@@ -0,0 +1,168 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2019 Facebook
	3	*
	4	* This program is free software; you can redistribute it and/or
	5	* modify it under the terms of version 2 of the GNU General Public
	6	* License as published by the Free Software Foundation.
	7	*
	8	* Sample Host Bandwidth Manager (HBM) BPF program.
	9	*
	10	* A cgroup skb BPF egress program to limit cgroup output bandwidth.
	11	* It uses a modified virtual token bucket queue to limit average
	12	* egress bandwidth. The implementation uses credits instead of tokens.
	13	* Negative credits imply that queueing would have happened (this is
	14	* a virtual queue, so no queueing is done by it. However, queueing may
	15	* occur at the actual qdisc (which is not used for rate limiting).
	16	*
	17	* This implementation uses 3 thresholds, one to start marking packets and
	18	* the other two to drop packets:
	19	* CREDIT
	20	* - <--------------------------\|------------------------> +
	21	* \| \| \| 0
	22	* \| Large pkt \|
	23	* \| drop thresh \|
	24	* Small pkt drop Mark threshold
	25	* thresh
	26	*
	27	* The effect of marking depends on the type of packet:
	28	* a) If the packet is ECN enabled and it is a TCP packet, then the packet
	29	* is ECN marked.
	30	* b) If the packet is a TCP packet, then we probabilistically call tcp_cwr
	31	* to reduce the congestion window. The current implementation uses a linear
	32	* distribution (0% probability at marking threshold, 100% probability
	33	* at drop threshold).
	34	* c) If the packet is not a TCP packet, then it is dropped.
	35	*
	36	* If the credit is below the drop threshold, the packet is dropped. If it
	37	* is a TCP packet, then it also calls tcp_cwr since packets dropped by
	38	* by a cgroup skb BPF program do not automatically trigger a call to
	39	* tcp_cwr in the current kernel code.
	40	*
	41	* This BPF program actually uses 2 drop thresholds, one threshold
	42	* for larger packets (>= 120 bytes) and another for smaller packets. This
	43	* protects smaller packets such as SYNs, ACKs, etc.
	44	*
	45	* The default bandwidth limit is set at 1Gbps but this can be changed by
	46	* a user program through a shared BPF map. In addition, by default this BPF
	47	* program does not limit connections using loopback. This behavior can be
	48	* overwritten by the user program. There is also an option to calculate
	49	* some statistics, such as percent of packets marked or dropped, which
	50	* a user program, such as hbm, can access.
	51	*/
	52
	53	#include "hbm_kern.h"
	54
	55	SEC("cgroup_skb/egress")
	56	int _hbm_out_cg(struct __sk_buff *skb)
	57	{
	58	long long delta = 0, delta_send;
	59	unsigned long long curtime, sendtime;
	60	struct hbm_queue_stats *qsp = NULL;
	61	unsigned int queue_index = 0;
	62	bool congestion_flag = false;
	63	bool ecn_ce_flag = false;
	64	struct hbm_pkt_info pkti = {};
	65	struct hbm_vqueue *qdp;
	66	bool drop_flag = false;
	67	bool cwr_flag = false;
	68	int len = skb->len;
	69	int rv = ALLOW_PKT;
	70
	71	qsp = bpf_map_lookup_elem(&queue_stats, &queue_index);
	72
	73	// Check if we should ignore loopback traffic
	74	if (qsp != NULL && !qsp->loopback && (skb->ifindex == 1))
	75	return ALLOW_PKT;
	76
	77	hbm_get_pkt_info(skb, &pkti);
	78
	79	// We may want to account for the length of headers in len
	80	// calculation, like ETH header + overhead, specially if it
	81	// is a gso packet. But I am not doing it right now.
	82
	83	qdp = bpf_get_local_storage(&queue_state, 0);
	84	if (!qdp)
	85	return ALLOW_PKT;
	86	if (qdp->lasttime == 0)
	87	hbm_init_edt_vqueue(qdp, 1024);
	88
	89	curtime = bpf_ktime_get_ns();
	90
	91	// Begin critical section
	92	bpf_spin_lock(&qdp->lock);
	93	delta = qdp->lasttime - curtime;
	94	// bound bursts to 100us
	95	if (delta < -BURST_SIZE_NS) {
	96	// negative delta is a credit that allows bursts
	97	qdp->lasttime = curtime - BURST_SIZE_NS;
	98	delta = -BURST_SIZE_NS;
	99	}
	100	sendtime = qdp->lasttime;
	101	delta_send = BYTES_TO_NS(len, qdp->rate);
	102	__sync_add_and_fetch(&(qdp->lasttime), delta_send);
	103	bpf_spin_unlock(&qdp->lock);
	104	// End critical section
	105
	106	// Set EDT of packet
	107	skb->tstamp = sendtime;
	108
	109	// Check if we should update rate
	110	if (qsp != NULL && (qsp->rate * 128) != qdp->rate)
	111	qdp->rate = qsp->rate * 128;
	112
	113	// Set flags (drop, congestion, cwr)
	114	// last packet will be sent in the future, bound latency
	115	if (delta > DROP_THRESH_NS \|\| (delta > LARGE_PKT_DROP_THRESH_NS &&
	116	len > LARGE_PKT_THRESH)) {
	117	drop_flag = true;
	118	if (pkti.is_tcp && pkti.ecn == 0)
	119	cwr_flag = true;
	120	} else if (delta > MARK_THRESH_NS) {
	121	if (pkti.is_tcp)
	122	congestion_flag = true;
	123	else
	124	drop_flag = true;
	125	}
	126
	127	if (congestion_flag) {
	128	if (bpf_skb_ecn_set_ce(skb)) {
	129	ecn_ce_flag = true;
	130	} else {
	131	if (pkti.is_tcp) {
	132	unsigned int rand = bpf_get_prandom_u32();
	133
	134	if (delta >= MARK_THRESH_NS +
	135	(rand % MARK_REGION_SIZE_NS)) {
	136	// Do congestion control
	137	cwr_flag = true;
	138	}
	139	} else if (len > LARGE_PKT_THRESH) {
	140	// Problem if too many small packets?
	141	drop_flag = true;
	142	congestion_flag = false;
	143	}
	144	}
	145	}
	146
	147	if (pkti.is_tcp && drop_flag && pkti.packets_out <= 1) {
	148	drop_flag = false;
	149	cwr_flag = true;
	150	congestion_flag = false;
	151	}
	152
	153	if (qsp != NULL && qsp->no_cn)
	154	cwr_flag = false;
	155
	156	hbm_update_stats(qsp, len, curtime, congestion_flag, drop_flag,
	157	cwr_flag, ecn_ce_flag, &pkti, (int) delta);
	158
	159	if (drop_flag) {
	160	__sync_add_and_fetch(&(qdp->lasttime), -delta_send);
	161	rv = DROP_PKT;
	162	}
	163
	164	if (cwr_flag)
	165	rv \|= CWR;
	166	return rv;
	167	}
	168	char _license[] SEC("license") = "GPL";

samples/bpf/hbm_kern.h+34−6

 #define DROP_PKT	0
 #define ALLOW_PKT	1
 #define TCP_ECN_OK	1
+#define CWR		2
 #ifndef HBM_DEBUG  // Define HBM_DEBUG to enable debugging
 #undef bpf_printk
 #define MAX_CREDIT		(100 * MAX_BYTES_PER_PACKET)
 #define INIT_CREDIT		(INITIAL_CREDIT_PACKETS * MAX_BYTES_PER_PACKET)
+// Time base accounting for fq's EDT
+#define BURST_SIZE_NS		100000 // 100us
+#define MARK_THRESH_NS		50000 // 50us
+#define DROP_THRESH_NS		500000 // 500us
+// Reserve 20us of queuing for small packets (less than 120 bytes)
+#define LARGE_PKT_DROP_THRESH_NS (DROP_THRESH_NS - 20000)
+#define MARK_REGION_SIZE_NS	(LARGE_PKT_DROP_THRESH_NS - MARK_THRESH_NS)
 // rate in bytes per ns << 20
 #define CREDIT_PER_NS(delta, rate) ((((u64)(delta)) * (rate)) >> 20)
+#define BYTES_PER_NS(delta, rate) ((((u64)(delta)) * (rate)) >> 20)
+#define BYTES_TO_NS(bytes, rate) div64_u64(((u64)(bytes)) << 20, (u64)(rate))
 struct bpf_map_def SEC("maps") queue_state = {
 	.type = BPF_MAP_TYPE_CGROUP_STORAGE,
 struct hbm_pkt_info {
 	int	cwnd;
 	int	rtt;
+	int	packets_out;
 	bool	is_ip;
 	bool	is_tcp;
 	short	ecn;
 				if (tp) {
 					pkti->cwnd = tp->snd_cwnd;
 					pkti->rtt = tp->srtt_us >> 3;
+					pkti->packets_out = tp->packets_out;
 					return 0;
 				}
 			}
 		}
 	}
+	pkti->cwnd = 0;
+	pkti->rtt = 0;
+	pkti->packets_out = 0;
 	return 1;
 }
-static __always_inline void hbm_get_pkt_info(struct __sk_buff *skb,
+static void hbm_get_pkt_info(struct __sk_buff *skb,
-					     struct hbm_pkt_info *pkti)
+			     struct hbm_pkt_info *pkti)
 {
 	struct iphdr iph;
 	struct ipv6hdr *ip6h;
 static __always_inline void hbm_init_vqueue(struct hbm_vqueue *qdp, int rate)
 {
-		bpf_printk("Initializing queue_state, rate:%d\n", rate * 128);
+	bpf_printk("Initializing queue_state, rate:%d\n", rate * 128);
-		qdp->lasttime = bpf_ktime_get_ns();
+	qdp->lasttime = bpf_ktime_get_ns();
-		qdp->credit = INIT_CREDIT;
+	qdp->credit = INIT_CREDIT;
-		qdp->rate = rate * 128;
+	qdp->rate = rate * 128;
+}
+static __always_inline void hbm_init_edt_vqueue(struct hbm_vqueue *qdp,
+						int rate)
+{
+	unsigned long long curtime;
+	curtime = bpf_ktime_get_ns();
+	bpf_printk("Initializing queue_state, rate:%d\n", rate * 128);
+	qdp->lasttime = curtime - BURST_SIZE_NS;	// support initial burst
+	qdp->credit = 0;				// not used
+	qdp->rate = rate * 128;
 }
 static __always_inline void hbm_update_stats(struct hbm_queue_stats *qsp,

samples/bpf/ibumad_kern.c+6−12

 };
 #undef DEBUG
-#ifdef DEBUG
+#ifndef DEBUG
-#define bpf_debug(fmt, ...)                         \
+#undef bpf_printk
-({                                                  \
+#define bpf_printk(fmt, ...)
-	char ____fmt[] = fmt;                       \
-	bpf_trace_printk(____fmt, sizeof(____fmt),  \
-			 ##__VA_ARGS__);            \
-})
-#else
-#define bpf_debug(fmt, ...)
 #endif
 /* Taken from the current format defined in
 	u64 zero = 0, *val;
 	u8 class = ctx->mgmt_class;
-	bpf_debug("ib_umad read recv : class 0x%x\n", class);
+	bpf_printk("ib_umad read recv : class 0x%x\n", class);
 	val = bpf_map_lookup_elem(&read_count, &class);
 	if (!val) {
 	u64 zero = 0, *val;
 	u8 class = ctx->mgmt_class;
-	bpf_debug("ib_umad read send : class 0x%x\n", class);
+	bpf_printk("ib_umad read send : class 0x%x\n", class);
 	val = bpf_map_lookup_elem(&read_count, &class);
 	if (!val) {
 	u64 zero = 0, *val;
 	u8 class = ctx->mgmt_class;
-	bpf_debug("ib_umad write : class 0x%x\n", class);
+	bpf_printk("ib_umad write : class 0x%x\n", class);
 	val = bpf_map_lookup_elem(&write_count, &class);
 	if (!val) {

samples/bpf/tcp_bpf.readme+1−1

@@ -25,4 +25,4 @@ attached to the cgroupv2). · 24 unmodified lines
25		25
26	To remove (unattach) a socket_ops BPF program from a cgroupv2:	26	To remove (unattach) a socket_ops BPF program from a cgroupv2:
27		27
28	bpftool cgroup attach /tmp/cgroupv2/foo sock_ops pinned /sys/fs/bpf/tcp_prog	28	bpftool cgroup detach /tmp/cgroupv2/foo sock_ops pinned /sys/fs/bpf/tcp_prog

samples/bpf/tcp_dumpstats_kern.c+68−0

@@ -0,0 +1,68 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Refer to samples/bpf/tcp_bpf.readme for the instructions on
	3	* how to run this sample program.
	4	*/
	5	#include <linux/bpf.h>
	6
	7	#include "bpf_helpers.h"
	8	#include "bpf_endian.h"
	9
	10	#define INTERVAL 1000000000ULL
	11
	12	int _version SEC("version") = 1;
	13	char _license[] SEC("license") = "GPL";
	14
	15	struct {
	16	__u32 type;
	17	__u32 map_flags;
	18	int *key;
	19	__u64 *value;
	20	} bpf_next_dump SEC(".maps") = {
	21	.type = BPF_MAP_TYPE_SK_STORAGE,
	22	.map_flags = BPF_F_NO_PREALLOC,
	23	};
	24
	25	SEC("sockops")
	26	int _sockops(struct bpf_sock_ops *ctx)
	27	{
	28	struct bpf_tcp_sock *tcp_sk;
	29	struct bpf_sock *sk;
	30	__u64 *next_dump;
	31	__u64 now;
	32
	33	switch (ctx->op) {
	34	case BPF_SOCK_OPS_TCP_CONNECT_CB:
	35	bpf_sock_ops_cb_flags_set(ctx, BPF_SOCK_OPS_RTT_CB_FLAG);
	36	return 1;
	37	case BPF_SOCK_OPS_RTT_CB:
	38	break;
	39	default:
	40	return 1;
	41	}
	42
	43	sk = ctx->sk;
	44	if (!sk)
	45	return 1;
	46
	47	next_dump = bpf_sk_storage_get(&bpf_next_dump, sk, 0,
	48	BPF_SK_STORAGE_GET_F_CREATE);
	49	if (!next_dump)
	50	return 1;
	51
	52	now = bpf_ktime_get_ns();
	53	if (now < *next_dump)
	54	return 1;
	55
	56	tcp_sk = bpf_tcp_sock(sk);
	57	if (!tcp_sk)
	58	return 1;
	59
	60	*next_dump = now + INTERVAL;
	61
	62	bpf_printk("dsack_dups=%u delivered=%u\n",
	63	tcp_sk->dsack_dups, tcp_sk->delivered);
	64	bpf_printk("delivered_ce=%u icsk_retransmits=%u\n",
	65	tcp_sk->delivered_ce, tcp_sk->icsk_retransmits);
	66
	67	return 1;
	68	}

samples/bpf/xdp_adjust_tail_user.c+10−2

 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <net/if.h>
 #include <sys/resource.h>
 #include <arpa/inet.h>
 #include <netinet/ether.h>
 	printf("Start a XDP prog which send ICMP \"packet too big\" \n"
 		"messages if ingress packet is bigger then MAX_SIZE bytes\n");
 	printf("Usage: %s [...]\n", cmd);
-	printf("    -i <ifindex> Interface Index\n");
+	printf("    -i <ifname|ifindex> Interface\n");
 	printf("    -T <stop-after-X-seconds> Default: 0 (forever)\n");
 	printf("    -S use skb-mode\n");
 	printf("    -N enforce native mode\n");
 		switch (opt) {
 		case 'i':
-			ifindex = atoi(optarg);
+			ifindex = if_nametoindex(optarg);
+			if (!ifindex)
+				ifindex = atoi(optarg);
 			break;
 		case 'T':
 			kill_after_s = atoi(optarg);
 		return 1;
 	}
+	if (!ifindex) {
+		fprintf(stderr, "Invalid ifname\n");
+		return 1;
+	}
 	snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]);
 	prog_load_attr.file = filename;

samples/bpf/xdp_redirect_map_user.c+11−4

 #include <stdlib.h>
 #include <stdbool.h>
 #include <string.h>
+#include <net/if.h>
 #include <unistd.h>
 #include <libgen.h>
 #include <sys/resource.h>
 static void usage(const char *prog)
 {
 	fprintf(stderr,
-		"usage: %s [OPTS] IFINDEX_IN IFINDEX_OUT\n\n"
+		"usage: %s [OPTS] <IFNAME|IFINDEX>_IN <IFNAME|IFINDEX>_OUT\n\n"
 		"OPTS:\n"
 		"    -S    use skb-mode\n"
 		"    -N    enforce native mode\n"
 	}
 	if (optind == argc) {
-		printf("usage: %s IFINDEX_IN IFINDEX_OUT\n", argv[0]);
+		printf("usage: %s <IFNAME|IFINDEX>_IN <IFNAME|IFINDEX>_OUT\n", argv[0]);
 		return 1;
 	}
 		return 1;
 	}
-	ifindex_in = strtoul(argv[optind], NULL, 0);
+	ifindex_in = if_nametoindex(argv[optind]);
-	ifindex_out = strtoul(argv[optind + 1], NULL, 0);
+	if (!ifindex_in)
+		ifindex_in = strtoul(argv[optind], NULL, 0);
+	ifindex_out = if_nametoindex(argv[optind + 1]);
+	if (!ifindex_out)
+		ifindex_out = strtoul(argv[optind + 1], NULL, 0);
 	printf("input: %d output: %d\n", ifindex_in, ifindex_out);
 	snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]);

samples/bpf/xdp_redirect_user.c+11−4

 #include <stdlib.h>
 #include <stdbool.h>
 #include <string.h>
+#include <net/if.h>
 #include <unistd.h>
 #include <libgen.h>
 #include <sys/resource.h>
 static void usage(const char *prog)
 {
 	fprintf(stderr,
-		"usage: %s [OPTS] IFINDEX_IN IFINDEX_OUT\n\n"
+		"usage: %s [OPTS] <IFNAME|IFINDEX>_IN <IFNAME|IFINDEX>_OUT\n\n"
 		"OPTS:\n"
 		"    -S    use skb-mode\n"
 		"    -N    enforce native mode\n"
 	}
 	if (optind == argc) {
-		printf("usage: %s IFINDEX_IN IFINDEX_OUT\n", argv[0]);
+		printf("usage: %s <IFNAME|IFINDEX>_IN <IFNAME|IFINDEX>_OUT\n", argv[0]);
 		return 1;
 	}
 		return 1;
 	}
-	ifindex_in = strtoul(argv[optind], NULL, 0);
+	ifindex_in = if_nametoindex(argv[optind]);
-	ifindex_out = strtoul(argv[optind + 1], NULL, 0);
+	if (!ifindex_in)
+		ifindex_in = strtoul(argv[optind], NULL, 0);
+	ifindex_out = if_nametoindex(argv[optind + 1]);
+	if (!ifindex_out)
+		ifindex_out = strtoul(argv[optind + 1], NULL, 0);
 	printf("input: %d output: %d\n", ifindex_in, ifindex_out);
 	snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]);

samples/bpf/xdp_tx_iptunnel_user.c+10−2

 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <net/if.h>
 #include <sys/resource.h>
 #include <arpa/inet.h>
 #include <netinet/ether.h>
 	       "in an IPv4/v6 header and XDP_TX it out.  The dst <VIP:PORT>\n"
 	       "is used to select packets to encapsulate\n\n");
 	printf("Usage: %s [...]\n", cmd);
-	printf("    -i <ifindex> Interface Index\n");
+	printf("    -i <ifname|ifindex> Interface\n");
 	printf("    -a <vip-service-address> IPv4 or IPv6\n");
 	printf("    -p <vip-service-port> A port range (e.g. 433-444) is also allowed\n");
 	printf("    -s <source-ip> Used in the IPTunnel header\n");
 		switch (opt) {
 		case 'i':
-			ifindex = atoi(optarg);
+			ifindex = if_nametoindex(optarg);
+			if (!ifindex)
+				ifindex = atoi(optarg);
 			break;
 		case 'a':
 			vip.family = parse_ipstr(optarg, vip.daddr.v6);
 		return 1;
 	}
+	if (!ifindex) {
+		fprintf(stderr, "Invalid ifname\n");
+		return 1;
+	}
 	snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]);
 	prog_load_attr.file = filename;

samples/bpf/xdpsock_user.c+28−16

 static int opt_poll;
 static int opt_interval = 1;
 static u32 opt_xdp_bind_flags;
+static int opt_xsk_frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
 static __u32 prog_id;
 struct xsk_umem_info {
 static struct xsk_umem_info *xsk_configure_umem(void *buffer, u64 size)
 {
 	struct xsk_umem_info *umem;
+	struct xsk_umem_config cfg = {
+		.fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,
+		.comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,
+		.frame_size = opt_xsk_frame_size,
+		.frame_headroom = XSK_UMEM__DEFAULT_FRAME_HEADROOM,
+	};
 	int ret;
 	umem = calloc(1, sizeof(*umem));
 		exit_with_error(errno);
 	ret = xsk_umem__create(&umem->umem, buffer, size, &umem->fq, &umem->cq,
-			       NULL);
+			       &cfg);
 	if (ret)
 		exit_with_error(-ret);
 				     &idx);
 	if (ret != XSK_RING_PROD__DEFAULT_NUM_DESCS)
 		exit_with_error(-ret);
-	for (i = 0;
+	for (i = 0; i < XSK_RING_PROD__DEFAULT_NUM_DESCS; i++)
-	     i < XSK_RING_PROD__DEFAULT_NUM_DESCS *
+		*xsk_ring_prod__fill_addr(&xsk->umem->fq, idx++) =
-		     XSK_UMEM__DEFAULT_FRAME_SIZE;
+			i * opt_xsk_frame_size;
-	     i += XSK_UMEM__DEFAULT_FRAME_SIZE)
-		*xsk_ring_prod__fill_addr(&xsk->umem->fq, idx++) = i;
 	xsk_ring_prod__submit(&xsk->umem->fq,
 			      XSK_RING_PROD__DEFAULT_NUM_DESCS);
 	{"interval", required_argument, 0, 'n'},
 	{"zero-copy", no_argument, 0, 'z'},
 	{"copy", no_argument, 0, 'c'},
+	{"frame-size", required_argument, 0, 'f'},
 	{0, 0, 0, 0}
 };
 		"  -n, --interval=n	Specify statistics update interval (default 1 sec).\n"
 		"  -z, --zero-copy      Force zero-copy mode.\n"
 		"  -c, --copy           Force copy mode.\n"
+		"  -f, --frame-size=n   Set the frame size (must be a power of two, default is %d).\n"
 		"\n";
-	fprintf(stderr, str, prog);
+	fprintf(stderr, str, prog, XSK_UMEM__DEFAULT_FRAME_SIZE);
 	exit(EXIT_FAILURE);
 }
 	opterr = 0;
 	for (;;) {
-		c = getopt_long(argc, argv, "Frtli:q:psSNn:cz", long_options,
+		c = getopt_long(argc, argv, "Frtli:q:psSNn:czf:", long_options,
 				&option_index);
 		if (c == -1)
 			break;
 		case 'F':
 			opt_xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST;
 			break;
+		case 'f':
+			opt_xsk_frame_size = atoi(optarg);
+			break;
 		default:
 			usage(basename(argv[0]));
 		}
 		usage(basename(argv[0]));
 	}
+	if (opt_xsk_frame_size & (opt_xsk_frame_size - 1)) {
+		fprintf(stderr, "--frame-size=%d is not a power of two\n",
+			opt_xsk_frame_size);
+		usage(basename(argv[0]));
+	}
 }
 static void kick_tx(struct xsk_socket_info *xsk)
 			for (i = 0; i < BATCH_SIZE; i++) {
 				xsk_ring_prod__tx_desc(&xsk->tx, idx + i)->addr
-					= (frame_nb + i) <<
+					= (frame_nb + i) * opt_xsk_frame_size;
-					XSK_UMEM__DEFAULT_FRAME_SHIFT;
 				xsk_ring_prod__tx_desc(&xsk->tx, idx + i)->len =
 					sizeof(pkt_data) - 1;
 			}
 	}
 	ret = posix_memalign(&bufs, getpagesize(), /* PAGE_SIZE aligned */
-			     NUM_FRAMES * XSK_UMEM__DEFAULT_FRAME_SIZE);
+			     NUM_FRAMES * opt_xsk_frame_size);
 	if (ret)
 		exit_with_error(ret);
        /* Create sockets... */
-	umem = xsk_configure_umem(bufs,
+	umem = xsk_configure_umem(bufs, NUM_FRAMES * opt_xsk_frame_size);
-				  NUM_FRAMES * XSK_UMEM__DEFAULT_FRAME_SIZE);
 	xsks[num_socks++] = xsk_configure_socket(umem);
 	if (opt_bench == BENCH_TXONLY) {
 		int i;
-		for (i = 0; i < NUM_FRAMES * XSK_UMEM__DEFAULT_FRAME_SIZE;
+		for (i = 0; i < NUM_FRAMES; i++)
-		     i += XSK_UMEM__DEFAULT_FRAME_SIZE)
+			(void)gen_eth_frame(umem, i * opt_xsk_frame_size);
-			(void)gen_eth_frame(umem, i);
 	}
 	signal(SIGINT, int_exit);

tools/bpf/bpftool/Documentation/bpftool-cgroup.rst+5−2

 |	*PROG* := { **id** *PROG_ID* | **pinned** *FILE* | **tag** *PROG_TAG* }
 |	*ATTACH_TYPE* := { **ingress** | **egress** | **sock_create** | **sock_ops** | **device** |
 |		**bind4** | **bind6** | **post_bind4** | **post_bind6** | **connect4** | **connect6** |
-|		**sendmsg4** | **sendmsg6** | **recvmsg4** | **recvmsg6** | **sysctl** }
+|		**sendmsg4** | **sendmsg6** | **recvmsg4** | **recvmsg6** | **sysctl** |
+|		**getsockopt** | **setsockopt** }
 |	*ATTACH_FLAGS* := { **multi** | **override** }
 DESCRIPTION
                   an unconnected udp4 socket (since 5.2);
 		  **recvmsg6** call to recvfrom(2), recvmsg(2), recvmmsg(2) for
                   an unconnected udp6 socket (since 5.2);
-		  **sysctl** sysctl access (since 5.2).
+		  **sysctl** sysctl access (since 5.2);
+		  **getsockopt** call to getsockopt (since 5.3);
+		  **setsockopt** call to setsockopt (since 5.3).
 	**bpftool cgroup detach** *CGROUP* *ATTACH_TYPE* *PROG*
 		  Detach *PROG* from the cgroup *CGROUP* and attach type

tools/bpf/bpftool/Documentation/bpftool-prog.rst+2−1

 |		**lwt_seg6local** | **sockops** | **sk_skb** | **sk_msg** | **lirc_mode2** |
 |		**cgroup/bind4** | **cgroup/bind6** | **cgroup/post_bind4** | **cgroup/post_bind6** |
 |		**cgroup/connect4** | **cgroup/connect6** | **cgroup/sendmsg4** | **cgroup/sendmsg6** |
-|		**cgroup/recvmsg4** | **cgroup/recvmsg6** | **cgroup/sysctl**
+|		**cgroup/recvmsg4** | **cgroup/recvmsg6** | **cgroup/sysctl** |
+|		**cgroup/getsockopt** | **cgroup/setsockopt**
 |	}
 |       *ATTACH_TYPE* := {
 |		**msg_verdict** | **stream_verdict** | **stream_parser** | **flow_dissector**

tools/bpf/bpftool/bash-completion/bpftool+6−3

                                 cgroup/sendmsg4 cgroup/sendmsg6 \
                                 cgroup/recvmsg4 cgroup/recvmsg6 \
                                 cgroup/post_bind4 cgroup/post_bind6 \
-                                cgroup/sysctl" -- \
+                                cgroup/sysctl cgroup/getsockopt \
+                                cgroup/setsockopt" -- \
                                                    "$cur" ) )
                             return 0
                             ;;
                 attach|detach)
                     local ATTACH_TYPES='ingress egress sock_create sock_ops \
                         device bind4 bind6 post_bind4 post_bind6 connect4 \
-                        connect6 sendmsg4 sendmsg6 recvmsg4 recvmsg6 sysctl'
+                        connect6 sendmsg4 sendmsg6 recvmsg4 recvmsg6 sysctl \
+                        getsockopt setsockopt'
                     local ATTACH_FLAGS='multi override'
                     local PROG_TYPE='id pinned tag'
                     case $prev in
                             ;;
                         ingress|egress|sock_create|sock_ops|device|bind4|bind6|\
                         post_bind4|post_bind6|connect4|connect6|sendmsg4|\
-                        sendmsg6|recvmsg4|recvmsg6|sysctl)
+                        sendmsg6|recvmsg4|recvmsg6|sysctl|getsockopt|\
+                        setsockopt)
                             COMPREPLY=( $( compgen -W "$PROG_TYPE" -- \
                                 "$cur" ) )
                             return 0

tools/bpf/bpftool/cgroup.c+4−1

 	"                        sock_ops | device | bind4 | bind6 |\n"	       \
 	"                        post_bind4 | post_bind6 | connect4 |\n"       \
 	"                        connect6 | sendmsg4 | sendmsg6 |\n"           \
-	"                        recvmsg4 | recvmsg6 | sysctl }"
+	"                        recvmsg4 | recvmsg6 | sysctl |\n"	       \
+	"                        getsockopt | setsockopt }"
 static const char * const attach_type_strings[] = {
 	[BPF_CGROUP_INET_INGRESS] = "ingress",
 	[BPF_CGROUP_SYSCTL] = "sysctl",
 	[BPF_CGROUP_UDP4_RECVMSG] = "recvmsg4",
 	[BPF_CGROUP_UDP6_RECVMSG] = "recvmsg6",
+	[BPF_CGROUP_GETSOCKOPT] = "getsockopt",
+	[BPF_CGROUP_SETSOCKOPT] = "setsockopt",
 	[__MAX_BPF_ATTACH_TYPE] = NULL,
 };

tools/bpf/bpftool/main.h+1−0

@@ -74,6 +74,7 @@ static const char * const prog_type_name[] = { · 73 unmodified lines
74	[BPF_PROG_TYPE_SK_REUSEPORT] = "sk_reuseport",	74	[BPF_PROG_TYPE_SK_REUSEPORT] = "sk_reuseport",
75	[BPF_PROG_TYPE_FLOW_DISSECTOR] = "flow_dissector",	75	[BPF_PROG_TYPE_FLOW_DISSECTOR] = "flow_dissector",
76	[BPF_PROG_TYPE_CGROUP_SYSCTL] = "cgroup_sysctl",	76	[BPF_PROG_TYPE_CGROUP_SYSCTL] = "cgroup_sysctl",
		77	[BPF_PROG_TYPE_CGROUP_SOCKOPT] = "cgroup_sockopt",
77	};	78	};
78		79
79	extern const char * const map_type_name[];	80	extern const char * const map_type_name[];

tools/bpf/bpftool/prog.c+2−1

 		"                 cgroup/bind4 | cgroup/bind6 | cgroup/post_bind4 |\n"
 		"                 cgroup/post_bind6 | cgroup/connect4 | cgroup/connect6 |\n"
 		"                 cgroup/sendmsg4 | cgroup/sendmsg6 | cgroup/recvmsg4 |\n"
-		"                 cgroup/recvmsg6 }\n"
+		"                 cgroup/recvmsg6 | cgroup/getsockopt |\n"
+		"                 cgroup/setsockopt }\n"
 		"       ATTACH_TYPE := { msg_verdict | stream_verdict | stream_parser |\n"
 		"                        flow_dissector }\n"
 		"       " HELP_SPEC_OPTIONS "\n"

tools/include/uapi/linux/bpf.h+25−1

 	BPF_PROG_TYPE_FLOW_DISSECTOR,
 	BPF_PROG_TYPE_CGROUP_SYSCTL,
 	BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE,
+	BPF_PROG_TYPE_CGROUP_SOCKOPT,
 };
 enum bpf_attach_type {
 	BPF_CGROUP_SYSCTL,
 	BPF_CGROUP_UDP4_RECVMSG,
 	BPF_CGROUP_UDP6_RECVMSG,
+	BPF_CGROUP_GETSOCKOPT,
+	BPF_CGROUP_SETSOCKOPT,
 	__MAX_BPF_ATTACH_TYPE
 };
  * 		* **BPF_SOCK_OPS_RTO_CB_FLAG** (retransmission time out)
  * 		* **BPF_SOCK_OPS_RETRANS_CB_FLAG** (retransmission)
  * 		* **BPF_SOCK_OPS_STATE_CB_FLAG** (TCP state change)
+ * 		* **BPF_SOCK_OPS_RTT_CB_FLAG** (every RTT)
  *
  * 		Therefore, this function can be used to clear a callback flag by
  * 		setting the appropriate bit to zero. e.g. to disable the RTO
 				 * sum(delta(snd_una)), or how many bytes
 				 * were acked.
 				 */
+	__u32 dsack_dups;	/* RFC4898 tcpEStatsStackDSACKDups
+				 * total number of DSACK blocks received
+				 */
+	__u32 delivered;	/* Total data packets delivered incl. rexmits */
+	__u32 delivered_ce;	/* Like the above but only ECE marked packets */
+	__u32 icsk_retransmits;	/* Number of unrecovered [RTO] timeouts */
 };
 struct bpf_sock_tuple {
 #define BPF_SOCK_OPS_RTO_CB_FLAG	(1<<0)
 #define BPF_SOCK_OPS_RETRANS_CB_FLAG	(1<<1)
 #define BPF_SOCK_OPS_STATE_CB_FLAG	(1<<2)
-#define BPF_SOCK_OPS_ALL_CB_FLAGS       0x7		/* Mask of all currently
+#define BPF_SOCK_OPS_RTT_CB_FLAG	(1<<3)
+#define BPF_SOCK_OPS_ALL_CB_FLAGS       0xF		/* Mask of all currently
 							 * supported cb flags
 							 */
 	BPF_SOCK_OPS_TCP_LISTEN_CB,	/* Called on listen(2), right after
 					 * socket transition to LISTEN state.
 					 */
+	BPF_SOCK_OPS_RTT_CB,		/* Called on every RTT.
+					 */
 };
 /* List of TCP states. There is a build check in net/ipv4/tcp.c to detect
 				 */
 };
+struct bpf_sockopt {
+	__bpf_md_ptr(struct bpf_sock *, sk);
+	__bpf_md_ptr(void *, optval);
+	__bpf_md_ptr(void *, optval_end);
+	__s32	level;
+	__s32	optname;
+	__s32	optlen;
+	__s32	retval;
+};
 #endif /* _UAPI__LINUX_BPF_H__ */

tools/include/uapi/linux/if_xdp.h+8−0

 #define XDP_UMEM_FILL_RING		5
 #define XDP_UMEM_COMPLETION_RING	6
 #define XDP_STATISTICS			7
+#define XDP_OPTIONS			8
 struct xdp_umem_reg {
 	__u64 addr; /* Start of packet data area */
 	__u64 tx_invalid_descs; /* Dropped due to invalid descriptor */
 };
+struct xdp_options {
+	__u32 flags;
+};
+/* Flags for the flags field of struct xdp_options */
+#define XDP_OPTIONS_ZEROCOPY (1 << 0)
 /* Pgoff for mmaping the rings */
 #define XDP_PGOFF_RX_RING			  0
 #define XDP_PGOFF_TX_RING		 0x80000000

tools/lib/bpf/libbpf.c+14−9

 	if (obj->nr_maps < obj->maps_cap)
 		return &obj->maps[obj->nr_maps++];
-	new_cap = max(4ul, obj->maps_cap * 3 / 2);
+	new_cap = max((size_t)4, obj->maps_cap * 3 / 2);
 	new_maps = realloc(obj->maps, new_cap * sizeof(*obj->maps));
 	if (!new_maps) {
 		pr_warning("alloc maps for object failed\n");
 			pr_debug("map '%s': found key_size = %u.\n",
 				 map_name, sz);
 			if (map->def.key_size && map->def.key_size != sz) {
-				pr_warning("map '%s': conflictling key size %u != %u.\n",
+				pr_warning("map '%s': conflicting key size %u != %u.\n",
 					   map_name, map->def.key_size, sz);
 				return -EINVAL;
 			}
 			pr_debug("map '%s': found key [%u], sz = %lld.\n",
 				 map_name, t->type, sz);
 			if (map->def.key_size && map->def.key_size != sz) {
-				pr_warning("map '%s': conflictling key size %u != %lld.\n",
+				pr_warning("map '%s': conflicting key size %u != %lld.\n",
 					   map_name, map->def.key_size, sz);
 				return -EINVAL;
 			}
 			pr_debug("map '%s': found value_size = %u.\n",
 				 map_name, sz);
 			if (map->def.value_size && map->def.value_size != sz) {
-				pr_warning("map '%s': conflictling value size %u != %u.\n",
+				pr_warning("map '%s': conflicting value size %u != %u.\n",
 					   map_name, map->def.value_size, sz);
 				return -EINVAL;
 			}
 			pr_debug("map '%s': found value [%u], sz = %lld.\n",
 				 map_name, t->type, sz);
 			if (map->def.value_size && map->def.value_size != sz) {
-				pr_warning("map '%s': conflictling value size %u != %lld.\n",
+				pr_warning("map '%s': conflicting value size %u != %lld.\n",
 					   map_name, map->def.value_size, sz);
 				return -EINVAL;
 			}
 	case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
 	case BPF_PROG_TYPE_PERF_EVENT:
 	case BPF_PROG_TYPE_CGROUP_SYSCTL:
+	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
 		return false;
 	case BPF_PROG_TYPE_KPROBE:
 	default:
 						BPF_CGROUP_UDP6_RECVMSG),
 	BPF_EAPROG_SEC("cgroup/sysctl",		BPF_PROG_TYPE_CGROUP_SYSCTL,
 						BPF_CGROUP_SYSCTL),
+	BPF_EAPROG_SEC("cgroup/getsockopt",	BPF_PROG_TYPE_CGROUP_SOCKOPT,
+						BPF_CGROUP_GETSOCKOPT),
+	BPF_EAPROG_SEC("cgroup/setsockopt",	BPF_PROG_TYPE_CGROUP_SOCKOPT,
+						BPF_CGROUP_SETSOCKOPT),
 };
 #undef BPF_PROG_SEC_IMPL
 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr,
 			struct bpf_object **pobj, int *prog_fd)
 {
-	struct bpf_object_open_attr open_attr = {
+	struct bpf_object_open_attr open_attr = {};
-		.file		= attr->file,
-		.prog_type	= attr->prog_type,
-	};
 	struct bpf_program *prog, *first_prog = NULL;
 	enum bpf_attach_type expected_attach_type;
 	enum bpf_prog_type prog_type;
 	if (!attr->file)
 		return -EINVAL;
+	open_attr.file = attr->file;
+	open_attr.prog_type = attr->prog_type;
 	obj = bpf_object__open_xattr(&open_attr);
 	if (IS_ERR_OR_NULL(obj))
 		return -ENOENT;

tools/lib/bpf/libbpf_probes.c+1−0

@@ -101,6 +101,7 @@ probe_load(enum bpf_prog_type prog_type, const struct bpf_insn *insns, · 100 unmodified lines
101	case BPF_PROG_TYPE_SK_REUSEPORT:	101	case BPF_PROG_TYPE_SK_REUSEPORT:
102	case BPF_PROG_TYPE_FLOW_DISSECTOR:	102	case BPF_PROG_TYPE_FLOW_DISSECTOR:
103	case BPF_PROG_TYPE_CGROUP_SYSCTL:	103	case BPF_PROG_TYPE_CGROUP_SYSCTL:
		104	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
104	default:	105	default:
105	break;	106	break;
106	}	107	}

tools/lib/bpf/xsk.c+14−1

 	int xsks_map_fd;
 	__u32 queue_id;
 	char ifname[IFNAMSIZ];
+	bool zc;
 };
 struct xsk_nl_info {
 	channels.cmd = ETHTOOL_GCHANNELS;
 	ifr.ifr_data = (void *)&channels;
-	strncpy(ifr.ifr_name, xsk->ifname, IFNAMSIZ);
+	strncpy(ifr.ifr_name, xsk->ifname, IFNAMSIZ - 1);
+	ifr.ifr_name[IFNAMSIZ - 1] = '\0';
 	err = ioctl(fd, SIOCETHTOOL, &ifr);
 	if (err && errno != EOPNOTSUPP) {
 		ret = -errno;
 	void *rx_map = NULL, *tx_map = NULL;
 	struct sockaddr_xdp sxdp = {};
 	struct xdp_mmap_offsets off;
+	struct xdp_options opts;
 	struct xsk_socket *xsk;
 	socklen_t optlen;
 	int err;
 	}
 	xsk->prog_fd = -1;
+	optlen = sizeof(opts);
+	err = getsockopt(xsk->fd, SOL_XDP, XDP_OPTIONS, &opts, &optlen);
+	if (err) {
+		err = -errno;
+		goto out_mmap_tx;
+	}
+	xsk->zc = opts.flags & XDP_OPTIONS_ZEROCOPY;
 	if (!(xsk->config.libbpf_flags & XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD)) {
 		err = xsk_setup_xdp_prog(xsk);
 		if (err)

tools/lib/bpf/xsk.h+1−1

@@ -167,7 +167,7 @@ LIBBPF_API int xsk_socket__fd(const struct xsk_socket *xsk); · 166 unmodified lines
167		167
168	#define XSK_RING_CONS__DEFAULT_NUM_DESCS 2048	168	#define XSK_RING_CONS__DEFAULT_NUM_DESCS 2048
169	#define XSK_RING_PROD__DEFAULT_NUM_DESCS 2048	169	#define XSK_RING_PROD__DEFAULT_NUM_DESCS 2048
170	#define XSK_UMEM__DEFAULT_FRAME_SHIFT 11 /* 2048 bytes */	170	#define XSK_UMEM__DEFAULT_FRAME_SHIFT 12 /* 4096 bytes */
171	#define XSK_UMEM__DEFAULT_FRAME_SIZE (1 << XSK_UMEM__DEFAULT_FRAME_SHIFT)	171	#define XSK_UMEM__DEFAULT_FRAME_SIZE (1 << XSK_UMEM__DEFAULT_FRAME_SHIFT)
172	#define XSK_UMEM__DEFAULT_FRAME_HEADROOM 0	172	#define XSK_UMEM__DEFAULT_FRAME_HEADROOM 0
173		173

tools/testing/selftests/bpf/.gitignore+3−0

@@ -39,3 +39,6 @@ libbpf.so.* · 38 unmodified lines
39	test_hashmap	39	test_hashmap
40	test_btf_dump	40	test_btf_dump
41	xdping	41	xdping
		42	test_sockopt
		43	test_sockopt_sk
		44	test_sockopt_multi

tools/testing/selftests/bpf/Makefile+8−2

 LLVM_OBJCOPY	?= llvm-objcopy
 LLVM_READELF	?= llvm-readelf
 BTF_PAHOLE	?= pahole
-CFLAGS += -Wall -O2 -I$(APIDIR) -I$(LIBDIR) -I$(BPFDIR) -I$(GENDIR) $(GENFLAGS) -I../../../include \
+CFLAGS += -g -Wall -O2 -I$(APIDIR) -I$(LIBDIR) -I$(BPFDIR) -I$(GENDIR) $(GENFLAGS) -I../../../include \
 	  -Dbpf_prog_load=bpf_prog_test_load \
 	  -Dbpf_load_program=bpf_test_load_program
 LDLIBS += -lcap -lelf -lrt -lpthread
 	test_sock test_btf test_sockmap get_cgroup_id_user test_socket_cookie \
 	test_cgroup_storage test_select_reuseport test_section_names \
 	test_netcnt test_tcpnotify_user test_sock_fields test_sysctl test_hashmap \
-	test_btf_dump test_cgroup_attach xdping
+	test_btf_dump test_cgroup_attach xdping test_sockopt test_sockopt_sk \
+	test_sockopt_multi test_tcp_rtt
 BPF_OBJ_FILES = $(patsubst %.c,%.o, $(notdir $(wildcard progs/*.c)))
 TEST_GEN_FILES = $(BPF_OBJ_FILES)
 	test_libbpf.sh \
 	test_xdp_redirect.sh \
 	test_xdp_meta.sh \
+	test_xdp_veth.sh \
 	test_offload.py \
 	test_sock_addr.sh \
 	test_tunnel.sh \
 $(OUTPUT)/test_sock_fields: cgroup_helpers.c
 $(OUTPUT)/test_sysctl: cgroup_helpers.c
 $(OUTPUT)/test_cgroup_attach: cgroup_helpers.c
+$(OUTPUT)/test_sockopt: cgroup_helpers.c
+$(OUTPUT)/test_sockopt_sk: cgroup_helpers.c
+$(OUTPUT)/test_sockopt_multi: cgroup_helpers.c
+$(OUTPUT)/test_tcp_rtt: cgroup_helpers.c
 .PHONY: force

tools/testing/selftests/bpf/progs/pyperf.h+4−5

 	void* co_name; // PyCodeObject.co_name
 } FrameData;
-static inline __attribute__((__always_inline__)) void*
+static __always_inline void *get_thread_state(void *tls_base, PidData *pidData)
-get_thread_state(void* tls_base, PidData* pidData)
 {
 	void* thread_state;
 	int key;
 	return thread_state;
 }
-static inline __attribute__((__always_inline__)) bool
+static __always_inline bool get_frame_data(void *frame_ptr, PidData *pidData,
-get_frame_data(void* frame_ptr, PidData* pidData, FrameData* frame, Symbol* symbol)
+					   FrameData *frame, Symbol *symbol)
 {
 	// read data from PyFrameObject
 	bpf_probe_read(&frame->f_back,
 	.max_elem = 1000,
 };
-static inline __attribute__((__always_inline__)) int __on_event(struct pt_regs *ctx)
+static __always_inline int __on_event(struct pt_regs *ctx)
 {
 	uint64_t pid_tgid = bpf_get_current_pid_tgid();
 	pid_t pid = (pid_t)(pid_tgid >> 32);

tools/testing/selftests/bpf/progs/sockopt_multi.c+71−0

@@ -0,0 +1,71 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	#include <netinet/in.h>
	3	#include <linux/bpf.h>
	4	#include "bpf_helpers.h"
	5
	6	char _license[] SEC("license") = "GPL";
	7	__u32 _version SEC("version") = 1;
	8
	9	SEC("cgroup/getsockopt/child")
	10	int _getsockopt_child(struct bpf_sockopt *ctx)
	11	{
	12	__u8 *optval_end = ctx->optval_end;
	13	__u8 *optval = ctx->optval;
	14
	15	if (ctx->level != SOL_IP \|\| ctx->optname != IP_TOS)
	16	return 1;
	17
	18	if (optval + 1 > optval_end)
	19	return 0; /* EPERM, bounds check */
	20
	21	if (optval[0] != 0x80)
	22	return 0; /* EPERM, unexpected optval from the kernel */
	23
	24	ctx->retval = 0; /* Reset system call return value to zero */
	25
	26	optval[0] = 0x90;
	27	ctx->optlen = 1;
	28
	29	return 1;
	30	}
	31
	32	SEC("cgroup/getsockopt/parent")
	33	int _getsockopt_parent(struct bpf_sockopt *ctx)
	34	{
	35	__u8 *optval_end = ctx->optval_end;
	36	__u8 *optval = ctx->optval;
	37
	38	if (ctx->level != SOL_IP \|\| ctx->optname != IP_TOS)
	39	return 1;
	40
	41	if (optval + 1 > optval_end)
	42	return 0; /* EPERM, bounds check */
	43
	44	if (optval[0] != 0x90)
	45	return 0; /* EPERM, unexpected optval from the kernel */
	46
	47	ctx->retval = 0; /* Reset system call return value to zero */
	48
	49	optval[0] = 0xA0;
	50	ctx->optlen = 1;
	51
	52	return 1;
	53	}
	54
	55	SEC("cgroup/setsockopt")
	56	int _setsockopt(struct bpf_sockopt *ctx)
	57	{
	58	__u8 *optval_end = ctx->optval_end;
	59	__u8 *optval = ctx->optval;
	60
	61	if (ctx->level != SOL_IP \|\| ctx->optname != IP_TOS)
	62	return 1;
	63
	64	if (optval + 1 > optval_end)
	65	return 0; /* EPERM, bounds check */
	66
	67	optval[0] += 0x10;
	68	ctx->optlen = 1;
	69
	70	return 1;
	71	}

tools/testing/selftests/bpf/progs/sockopt_sk.c+111−0

@@ -0,0 +1,111 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	#include <netinet/in.h>
	3	#include <linux/bpf.h>
	4	#include "bpf_helpers.h"
	5
	6	char _license[] SEC("license") = "GPL";
	7	__u32 _version SEC("version") = 1;
	8
	9	#define SOL_CUSTOM 0xdeadbeef
	10
	11	struct sockopt_sk {
	12	__u8 val;
	13	};
	14
	15	struct bpf_map_def SEC("maps") socket_storage_map = {
	16	.type = BPF_MAP_TYPE_SK_STORAGE,
	17	.key_size = sizeof(int),
	18	.value_size = sizeof(struct sockopt_sk),
	19	.map_flags = BPF_F_NO_PREALLOC,
	20	};
	21	BPF_ANNOTATE_KV_PAIR(socket_storage_map, int, struct sockopt_sk);
	22
	23	SEC("cgroup/getsockopt")
	24	int _getsockopt(struct bpf_sockopt *ctx)
	25	{
	26	__u8 *optval_end = ctx->optval_end;
	27	__u8 *optval = ctx->optval;
	28	struct sockopt_sk *storage;
	29
	30	if (ctx->level == SOL_IP && ctx->optname == IP_TOS)
	31	/* Not interested in SOL_IP:IP_TOS;
	32	* let next BPF program in the cgroup chain or kernel
	33	* handle it.
	34	*/
	35	return 1;
	36
	37	if (ctx->level == SOL_SOCKET && ctx->optname == SO_SNDBUF) {
	38	/* Not interested in SOL_SOCKET:SO_SNDBUF;
	39	* let next BPF program in the cgroup chain or kernel
	40	* handle it.
	41	*/
	42	return 1;
	43	}
	44
	45	if (ctx->level != SOL_CUSTOM)
	46	return 0; /* EPERM, deny everything except custom level */
	47
	48	if (optval + 1 > optval_end)
	49	return 0; /* EPERM, bounds check */
	50
	51	storage = bpf_sk_storage_get(&socket_storage_map, ctx->sk, 0,
	52	BPF_SK_STORAGE_GET_F_CREATE);
	53	if (!storage)
	54	return 0; /* EPERM, couldn't get sk storage */
	55
	56	if (!ctx->retval)
	57	return 0; /* EPERM, kernel should not have handled
	58	* SOL_CUSTOM, something is wrong!
	59	*/
	60	ctx->retval = 0; /* Reset system call return value to zero */
	61
	62	optval[0] = storage->val;
	63	ctx->optlen = 1;
	64
	65	return 1;
	66	}
	67
	68	SEC("cgroup/setsockopt")
	69	int _setsockopt(struct bpf_sockopt *ctx)
	70	{
	71	__u8 *optval_end = ctx->optval_end;
	72	__u8 *optval = ctx->optval;
	73	struct sockopt_sk *storage;
	74
	75	if (ctx->level == SOL_IP && ctx->optname == IP_TOS)
	76	/* Not interested in SOL_IP:IP_TOS;
	77	* let next BPF program in the cgroup chain or kernel
	78	* handle it.
	79	*/
	80	return 1;
	81
	82	if (ctx->level == SOL_SOCKET && ctx->optname == SO_SNDBUF) {
	83	/* Overwrite SO_SNDBUF value */
	84
	85	if (optval + sizeof(__u32) > optval_end)
	86	return 0; /* EPERM, bounds check */
	87
	88	(__u32 )optval = 0x55AA;
	89	ctx->optlen = 4;
	90
	91	return 1;
	92	}
	93
	94	if (ctx->level != SOL_CUSTOM)
	95	return 0; /* EPERM, deny everything except custom level */
	96
	97	if (optval + 1 > optval_end)
	98	return 0; /* EPERM, bounds check */
	99
	100	storage = bpf_sk_storage_get(&socket_storage_map, ctx->sk, 0,
	101	BPF_SK_STORAGE_GET_F_CREATE);
	102	if (!storage)
	103	return 0; /* EPERM, couldn't get sk storage */
	104
	105	storage->val = optval[0];
	106	ctx->optlen = -1; /* BPF has consumed this option, don't call kernel
	107	* setsockopt handler.
	108	*/
	109
	110	return 1;
	111	}

tools/testing/selftests/bpf/progs/strobemeta.h+19−17

 	uint64_t offset;
 };
-static inline __attribute__((always_inline))
+static __always_inline void *calc_location(struct strobe_value_loc *loc,
-void *calc_location(struct strobe_value_loc *loc, void *tls_base)
+					   void *tls_base)
 {
 	/*
 	 * tls_mode value is:
 		: NULL;
 }
-static inline __attribute__((always_inline))
+static __always_inline void read_int_var(struct strobemeta_cfg *cfg,
-void read_int_var(struct strobemeta_cfg *cfg, size_t idx, void *tls_base,
+					 size_t idx, void *tls_base,
-		  struct strobe_value_generic *value,
+					 struct strobe_value_generic *value,
-		  struct strobemeta_payload *data)
+					 struct strobemeta_payload *data)
 {
 	void *location = calc_location(&cfg->int_locs[idx], tls_base);
 	if (!location)
 		data->int_vals_set_mask |= (1 << idx);
 }
-static inline __attribute__((always_inline))
+static __always_inline uint64_t read_str_var(struct strobemeta_cfg *cfg,
-uint64_t read_str_var(struct strobemeta_cfg* cfg, size_t idx, void *tls_base,
+					     size_t idx, void *tls_base,
-		      struct strobe_value_generic *value,
+					     struct strobe_value_generic *value,
-		      struct strobemeta_payload *data, void *payload)
+					     struct strobemeta_payload *data,
+					     void *payload)
 {
 	void *location;
 	uint32_t len;
 	return len;
 }
-static inline __attribute__((always_inline))
+static __always_inline void *read_map_var(struct strobemeta_cfg *cfg,
-void *read_map_var(struct strobemeta_cfg *cfg, size_t idx, void *tls_base,
+					  size_t idx, void *tls_base,
-		   struct strobe_value_generic *value,
+					  struct strobe_value_generic *value,
-		   struct strobemeta_payload* data, void *payload)
+					  struct strobemeta_payload *data,
+					  void *payload)
 {
 	struct strobe_map_descr* descr = &data->map_descrs[idx];
 	struct strobe_map_raw map;
  * read_strobe_meta returns NULL, if no metadata was read; otherwise returns
  * pointer to *right after* payload ends
  */
-static inline __attribute__((always_inline))
+static __always_inline void *read_strobe_meta(struct task_struct *task,
-void *read_strobe_meta(struct task_struct* task,
+					      struct strobemeta_payload *data)
-		       struct strobemeta_payload* data) {
+{
 	pid_t pid = bpf_get_current_pid_tgid() >> 32;
 	struct strobe_value_generic value = {0};
 	struct strobemeta_cfg *cfg;

tools/testing/selftests/bpf/progs/tcp_rtt.c+61−0

@@ -0,0 +1,61 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	#include <linux/bpf.h>
	3	#include "bpf_helpers.h"
	4
	5	char _license[] SEC("license") = "GPL";
	6	__u32 _version SEC("version") = 1;
	7
	8	struct tcp_rtt_storage {
	9	__u32 invoked;
	10	__u32 dsack_dups;
	11	__u32 delivered;
	12	__u32 delivered_ce;
	13	__u32 icsk_retransmits;
	14	};
	15
	16	struct bpf_map_def SEC("maps") socket_storage_map = {
	17	.type = BPF_MAP_TYPE_SK_STORAGE,
	18	.key_size = sizeof(int),
	19	.value_size = sizeof(struct tcp_rtt_storage),
	20	.map_flags = BPF_F_NO_PREALLOC,
	21	};
	22	BPF_ANNOTATE_KV_PAIR(socket_storage_map, int, struct tcp_rtt_storage);
	23
	24	SEC("sockops")
	25	int _sockops(struct bpf_sock_ops *ctx)
	26	{
	27	struct tcp_rtt_storage *storage;
	28	struct bpf_tcp_sock *tcp_sk;
	29	int op = (int) ctx->op;
	30	struct bpf_sock *sk;
	31
	32	sk = ctx->sk;
	33	if (!sk)
	34	return 1;
	35
	36	storage = bpf_sk_storage_get(&socket_storage_map, sk, 0,
	37	BPF_SK_STORAGE_GET_F_CREATE);
	38	if (!storage)
	39	return 1;
	40
	41	if (op == BPF_SOCK_OPS_TCP_CONNECT_CB) {
	42	bpf_sock_ops_cb_flags_set(ctx, BPF_SOCK_OPS_RTT_CB_FLAG);
	43	return 1;
	44	}
	45
	46	if (op != BPF_SOCK_OPS_RTT_CB)
	47	return 1;
	48
	49	tcp_sk = bpf_tcp_sock(sk);
	50	if (!tcp_sk)
	51	return 1;
	52
	53	storage->invoked++;
	54
	55	storage->dsack_dups = tcp_sk->dsack_dups;
	56	storage->delivered = tcp_sk->delivered;
	57	storage->delivered_ce = tcp_sk->delivered_ce;
	58	storage->icsk_retransmits = tcp_sk->icsk_retransmits;
	59
	60	return 1;
	61	}

tools/testing/selftests/bpf/progs/test_jhash.h+2−1

 // SPDX-License-Identifier: GPL-2.0
 // Copyright (c) 2019 Facebook
+#include <features.h>
 typedef unsigned int u32;
-static __attribute__((always_inline)) u32 rol32(u32 word, unsigned int shift)
+static __always_inline u32 rol32(u32 word, unsigned int shift)
 {
 	return (word << shift) | (word >> ((-shift) & 31));
 }

tools/testing/selftests/bpf/progs/test_seg6_loop.c+12−11

 	unsigned char value[0];
 } BPF_PACKET_HEADER;
-static __attribute__((always_inline)) struct ip6_srh_t *get_srh(struct __sk_buff *skb)
+static __always_inline struct ip6_srh_t *get_srh(struct __sk_buff *skb)
 {
 	void *cursor, *data_end;
 	struct ip6_srh_t *srh;
 	return srh;
 }
-static __attribute__((always_inline))
+static __always_inline int update_tlv_pad(struct __sk_buff *skb,
-int update_tlv_pad(struct __sk_buff *skb, uint32_t new_pad,
+					  uint32_t new_pad, uint32_t old_pad,
-		   uint32_t old_pad, uint32_t pad_off)
+					  uint32_t pad_off)
 {
 	int err;
 	return 0;
 }
-static __attribute__((always_inline))
+static __always_inline int is_valid_tlv_boundary(struct __sk_buff *skb,
-int is_valid_tlv_boundary(struct __sk_buff *skb, struct ip6_srh_t *srh,
+						 struct ip6_srh_t *srh,
-			  uint32_t *tlv_off, uint32_t *pad_size,
+						 uint32_t *tlv_off,
-			  uint32_t *pad_off)
+						 uint32_t *pad_size,
+						 uint32_t *pad_off)
 {
 	uint32_t srh_off, cur_off;
 	int offset_valid = 0;
 	return 0;
 }
-static __attribute__((always_inline))
+static __always_inline int add_tlv(struct __sk_buff *skb,
-int add_tlv(struct __sk_buff *skb, struct ip6_srh_t *srh, uint32_t tlv_off,
+				   struct ip6_srh_t *srh, uint32_t tlv_off,
-	    struct sr6_tlv_t *itlv, uint8_t tlv_size)
+				   struct sr6_tlv_t *itlv, uint8_t tlv_size)
 {
 	uint32_t srh_off = (char *)srh - (char *)(long)skb->data;
 	uint8_t len_remaining, new_pad;

tools/testing/selftests/bpf/progs/test_verif_scale2.c+1−1

@@ -2,7 +2,7 @@ · 1 unmodified line
2	// Copyright (c) 2019 Facebook	2	// Copyright (c) 2019 Facebook
3	#include <linux/bpf.h>	3	#include <linux/bpf.h>
4	#include "bpf_helpers.h"	4	#include "bpf_helpers.h"
5	#define ATTR __attribute__((always_inline))	5	#define ATTR __always_inline
6	#include "test_jhash.h"	6	#include "test_jhash.h"
7		7
8	SEC("scale90_inline")	8	SEC("scale90_inline")

tools/testing/selftests/bpf/progs/xdp_redirect_map.c+31−0

@@ -0,0 +1,31 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	#include <linux/bpf.h>
	4	#include "bpf_helpers.h"
	5
	6	struct bpf_map_def SEC("maps") tx_port = {
	7	.type = BPF_MAP_TYPE_DEVMAP,
	8	.key_size = sizeof(int),
	9	.value_size = sizeof(int),
	10	.max_entries = 8,
	11	};
	12
	13	SEC("redirect_map_0")
	14	int xdp_redirect_map_0(struct xdp_md *xdp)
	15	{
	16	return bpf_redirect_map(&tx_port, 0, 0);
	17	}
	18
	19	SEC("redirect_map_1")
	20	int xdp_redirect_map_1(struct xdp_md *xdp)
	21	{
	22	return bpf_redirect_map(&tx_port, 1, 0);
	23	}
	24
	25	SEC("redirect_map_2")
	26	int xdp_redirect_map_2(struct xdp_md *xdp)
	27	{
	28	return bpf_redirect_map(&tx_port, 2, 0);
	29	}
	30
	31	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/xdp_tx.c+12−0

@@ -0,0 +1,12 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	#include <linux/bpf.h>
	4	#include "bpf_helpers.h"
	5
	6	SEC("tx")
	7	int xdp_tx(struct xdp_md *xdp)
	8	{
	9	return XDP_TX;
	10	}
	11
	12	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/test_section_names.c+10−0

@@ -134,6 +134,16 @@ static struct sec_name_test tests[] = { · 133 unmodified lines
134	{0, BPF_PROG_TYPE_CGROUP_SYSCTL, BPF_CGROUP_SYSCTL},	134	{0, BPF_PROG_TYPE_CGROUP_SYSCTL, BPF_CGROUP_SYSCTL},
135	{0, BPF_CGROUP_SYSCTL},	135	{0, BPF_CGROUP_SYSCTL},
136	},	136	},
		137	{
		138	"cgroup/getsockopt",
		139	{0, BPF_PROG_TYPE_CGROUP_SOCKOPT, BPF_CGROUP_GETSOCKOPT},
		140	{0, BPF_CGROUP_GETSOCKOPT},
		141	},
		142	{
		143	"cgroup/setsockopt",
		144	{0, BPF_PROG_TYPE_CGROUP_SOCKOPT, BPF_CGROUP_SETSOCKOPT},
		145	{0, BPF_CGROUP_SETSOCKOPT},
		146	},
137	};	147	};
138		148
139	static int test_prog_type_by_name(const struct sec_name_test *test)	149	static int test_prog_type_by_name(const struct sec_name_test *test)

tools/testing/selftests/bpf/test_sockopt.c+1021−0

@@ -0,0 +1,1021 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	#include <errno.h>
	4	#include <stdio.h>
	5	#include <unistd.h>
	6	#include <sys/types.h>
	7	#include <sys/socket.h>
	8	#include <netinet/in.h>
	9
	10	#include <linux/filter.h>
	11	#include <bpf/bpf.h>
	12	#include <bpf/libbpf.h>
	13
	14	#include "bpf_rlimit.h"
	15	#include "bpf_util.h"
	16	#include "cgroup_helpers.h"
	17
	18	#define CG_PATH "/sockopt"
	19
	20	static char bpf_log_buf[4096];
	21	static bool verbose;
	22
	23	enum sockopt_test_error {
	24	OK = 0,
	25	DENY_LOAD,
	26	DENY_ATTACH,
	27	EPERM_GETSOCKOPT,
	28	EFAULT_GETSOCKOPT,
	29	EPERM_SETSOCKOPT,
	30	EFAULT_SETSOCKOPT,
	31	};
	32
	33	static struct sockopt_test {
	34	const char *descr;
	35	const struct bpf_insn insns[64];
	36	enum bpf_attach_type attach_type;
	37	enum bpf_attach_type expected_attach_type;
	38
	39	int set_optname;
	40	int set_level;
	41	const char set_optval[64];
	42	socklen_t set_optlen;
	43
	44	int get_optname;
	45	int get_level;
	46	const char get_optval[64];
	47	socklen_t get_optlen;
	48	socklen_t get_optlen_ret;
	49
	50	enum sockopt_test_error error;
	51	} tests[] = {
	52
	53	/* ==================== getsockopt ==================== */
	54
	55	{
	56	.descr = "getsockopt: no expected_attach_type",
	57	.insns = {
	58	/* return 1 */
	59	BPF_MOV64_IMM(BPF_REG_0, 1),
	60	BPF_EXIT_INSN(),
	61
	62	},
	63	.attach_type = BPF_CGROUP_GETSOCKOPT,
	64	.expected_attach_type = 0,
	65	.error = DENY_LOAD,
	66	},
	67	{
	68	.descr = "getsockopt: wrong expected_attach_type",
	69	.insns = {
	70	/* return 1 */
	71	BPF_MOV64_IMM(BPF_REG_0, 1),
	72	BPF_EXIT_INSN(),
	73
	74	},
	75	.attach_type = BPF_CGROUP_GETSOCKOPT,
	76	.expected_attach_type = BPF_CGROUP_SETSOCKOPT,
	77	.error = DENY_ATTACH,
	78	},
	79	{
	80	.descr = "getsockopt: bypass bpf hook",
	81	.insns = {
	82	/* return 1 */
	83	BPF_MOV64_IMM(BPF_REG_0, 1),
	84	BPF_EXIT_INSN(),
	85	},
	86	.attach_type = BPF_CGROUP_GETSOCKOPT,
	87	.expected_attach_type = BPF_CGROUP_GETSOCKOPT,
	88
	89	.get_level = SOL_IP,
	90	.set_level = SOL_IP,
	91
	92	.get_optname = IP_TOS,
	93	.set_optname = IP_TOS,
	94
	95	.set_optval = { 1 << 3 },
	96	.set_optlen = 1,
	97
	98	.get_optval = { 1 << 3 },
	99	.get_optlen = 1,
	100	},
	101	{
	102	.descr = "getsockopt: return EPERM from bpf hook",
	103	.insns = {
	104	BPF_MOV64_IMM(BPF_REG_0, 0),
	105	BPF_EXIT_INSN(),
	106	},
	107	.attach_type = BPF_CGROUP_GETSOCKOPT,
	108	.expected_attach_type = BPF_CGROUP_GETSOCKOPT,
	109
	110	.get_level = SOL_IP,
	111	.get_optname = IP_TOS,
	112
	113	.get_optlen = 1,
	114	.error = EPERM_GETSOCKOPT,
	115	},
	116	{
	117	.descr = "getsockopt: no optval bounds check, deny loading",
	118	.insns = {
	119	/* r6 = ctx->optval */
	120	BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1,
	121	offsetof(struct bpf_sockopt, optval)),
	122
	123	/* ctx->optval[0] = 0x80 */
	124	BPF_MOV64_IMM(BPF_REG_0, 0x80),
	125	BPF_STX_MEM(BPF_W, BPF_REG_6, BPF_REG_0, 0),
	126
	127	/* return 1 */
	128	BPF_MOV64_IMM(BPF_REG_0, 1),
	129	BPF_EXIT_INSN(),
	130	},
	131	.attach_type = BPF_CGROUP_GETSOCKOPT,
	132	.expected_attach_type = BPF_CGROUP_GETSOCKOPT,
	133	.error = DENY_LOAD,
	134	},
	135	{
	136	.descr = "getsockopt: read ctx->level",
	137	.insns = {
	138	/* r6 = ctx->level */
	139	BPF_LDX_MEM(BPF_W, BPF_REG_6, BPF_REG_1,
	140	offsetof(struct bpf_sockopt, level)),
	141
	142	/* if (ctx->level == 123) { */
	143	BPF_JMP_IMM(BPF_JNE, BPF_REG_6, 123, 4),
	144	/* ctx->retval = 0 */
	145	BPF_MOV64_IMM(BPF_REG_0, 0),
	146	BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0,
	147	offsetof(struct bpf_sockopt, retval)),
	148	/* return 1 */
	149	BPF_MOV64_IMM(BPF_REG_0, 1),
	150	BPF_JMP_A(1),
	151	/* } else { */
	152	/* return 0 */
	153	BPF_MOV64_IMM(BPF_REG_0, 0),
	154	/* } */
	155	BPF_EXIT_INSN(),
	156	},
	157	.attach_type = BPF_CGROUP_GETSOCKOPT,
	158	.expected_attach_type = BPF_CGROUP_GETSOCKOPT,
	159
	160	.get_level = 123,
	161
	162	.get_optlen = 1,
	163	},
	164	{
	165	.descr = "getsockopt: deny writing to ctx->level",
	166	.insns = {
	167	/* ctx->level = 1 */
	168	BPF_MOV64_IMM(BPF_REG_0, 1),
	169	BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0,
	170	offsetof(struct bpf_sockopt, level)),
	171	BPF_EXIT_INSN(),
	172	},
	173	.attach_type = BPF_CGROUP_GETSOCKOPT,
	174	.expected_attach_type = BPF_CGROUP_GETSOCKOPT,
	175
	176	.error = DENY_LOAD,
	177	},
	178	{
	179	.descr = "getsockopt: read ctx->optname",
	180	.insns = {
	181	/* r6 = ctx->optname */
	182	BPF_LDX_MEM(BPF_W, BPF_REG_6, BPF_REG_1,
	183	offsetof(struct bpf_sockopt, optname)),
	184
	185	/* if (ctx->optname == 123) { */
	186	BPF_JMP_IMM(BPF_JNE, BPF_REG_6, 123, 4),
	187	/* ctx->retval = 0 */
	188	BPF_MOV64_IMM(BPF_REG_0, 0),
	189	BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0,
	190	offsetof(struct bpf_sockopt, retval)),
	191	/* return 1 */
	192	BPF_MOV64_IMM(BPF_REG_0, 1),
	193	BPF_JMP_A(1),
	194	/* } else { */
	195	/* return 0 */
	196	BPF_MOV64_IMM(BPF_REG_0, 0),
	197	/* } */
	198	BPF_EXIT_INSN(),
	199	},
	200	.attach_type = BPF_CGROUP_GETSOCKOPT,
	201	.expected_attach_type = BPF_CGROUP_GETSOCKOPT,
	202
	203	.get_optname = 123,
	204
	205	.get_optlen = 1,
	206	},
	207	{
	208	.descr = "getsockopt: read ctx->retval",
	209	.insns = {
	210	/* r6 = ctx->retval */
	211	BPF_LDX_MEM(BPF_W, BPF_REG_6, BPF_REG_1,
	212	offsetof(struct bpf_sockopt, retval)),
	213
	214	/* return 1 */
	215	BPF_MOV64_IMM(BPF_REG_0, 1),
	216	BPF_EXIT_INSN(),
	217	},
	218	.attach_type = BPF_CGROUP_GETSOCKOPT,
	219	.expected_attach_type = BPF_CGROUP_GETSOCKOPT,
	220
	221	.get_level = SOL_IP,
	222	.get_optname = IP_TOS,
	223	.get_optlen = 1,
	224	},
	225	{
	226	.descr = "getsockopt: deny writing to ctx->optname",
	227	.insns = {
	228	/* ctx->optname = 1 */
	229	BPF_MOV64_IMM(BPF_REG_0, 1),
	230	BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0,
	231	offsetof(struct bpf_sockopt, optname)),
	232	BPF_EXIT_INSN(),
	233	},
	234	.attach_type = BPF_CGROUP_GETSOCKOPT,
	235	.expected_attach_type = BPF_CGROUP_GETSOCKOPT,
	236
	237	.error = DENY_LOAD,
	238	},
	239	{
	240	.descr = "getsockopt: read ctx->optlen",
	241	.insns = {
	242	/* r6 = ctx->optlen */
	243	BPF_LDX_MEM(BPF_W, BPF_REG_6, BPF_REG_1,
	244	offsetof(struct bpf_sockopt, optlen)),
	245
	246	/* if (ctx->optlen == 64) { */
	247	BPF_JMP_IMM(BPF_JNE, BPF_REG_6, 64, 4),
	248	/* ctx->retval = 0 */
	249	BPF_MOV64_IMM(BPF_REG_0, 0),
	250	BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0,
	251	offsetof(struct bpf_sockopt, retval)),
	252	/* return 1 */
	253	BPF_MOV64_IMM(BPF_REG_0, 1),
	254	BPF_JMP_A(1),
	255	/* } else { */
	256	/* return 0 */
	257	BPF_MOV64_IMM(BPF_REG_0, 0),
	258	/* } */
	259	BPF_EXIT_INSN(),
	260	},
	261	.attach_type = BPF_CGROUP_GETSOCKOPT,
	262	.expected_attach_type = BPF_CGROUP_GETSOCKOPT,
	263
	264	.get_optlen = 64,
	265	},
	266	{
	267	.descr = "getsockopt: deny bigger ctx->optlen",
	268	.insns = {
	269	/* ctx->optlen = 65 */
	270	BPF_MOV64_IMM(BPF_REG_0, 65),
	271	BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0,
	272	offsetof(struct bpf_sockopt, optlen)),
	273
	274	/* ctx->retval = 0 */
	275	BPF_MOV64_IMM(BPF_REG_0, 0),
	276	BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0,
	277	offsetof(struct bpf_sockopt, retval)),
	278
	279	/* return 1 */
	280	BPF_MOV64_IMM(BPF_REG_0, 1),
	281	BPF_EXIT_INSN(),
	282	},
	283	.attach_type = BPF_CGROUP_GETSOCKOPT,
	284	.expected_attach_type = BPF_CGROUP_GETSOCKOPT,
	285
	286	.get_optlen = 64,
	287
	288	.error = EFAULT_GETSOCKOPT,
	289	},
	290	{
	291	.descr = "getsockopt: deny arbitrary ctx->retval",
	292	.insns = {
	293	/* ctx->retval = 123 */
	294	BPF_MOV64_IMM(BPF_REG_0, 123),
	295	BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0,
	296	offsetof(struct bpf_sockopt, retval)),
	297
	298	/* return 1 */
	299	BPF_MOV64_IMM(BPF_REG_0, 1),
	300	BPF_EXIT_INSN(),
	301	},
	302	.attach_type = BPF_CGROUP_GETSOCKOPT,
	303	.expected_attach_type = BPF_CGROUP_GETSOCKOPT,
	304
	305	.get_optlen = 64,
	306
	307	.error = EFAULT_GETSOCKOPT,
	308	},
	309	{
	310	.descr = "getsockopt: support smaller ctx->optlen",
	311	.insns = {
	312	/* ctx->optlen = 32 */
	313	BPF_MOV64_IMM(BPF_REG_0, 32),
	314	BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0,
	315	offsetof(struct bpf_sockopt, optlen)),
	316	/* ctx->retval = 0 */
	317	BPF_MOV64_IMM(BPF_REG_0, 0),
	318	BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0,
	319	offsetof(struct bpf_sockopt, retval)),
	320	/* return 1 */
	321	BPF_MOV64_IMM(BPF_REG_0, 1),
	322	BPF_EXIT_INSN(),
	323	},
	324	.attach_type = BPF_CGROUP_GETSOCKOPT,
	325	.expected_attach_type = BPF_CGROUP_GETSOCKOPT,
	326
	327	.get_optlen = 64,
	328	.get_optlen_ret = 32,
	329	},
	330	{
	331	.descr = "getsockopt: deny writing to ctx->optval",
	332	.insns = {
	333	/* ctx->optval = 1 */
	334	BPF_MOV64_IMM(BPF_REG_0, 1),
	335	BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0,
	336	offsetof(struct bpf_sockopt, optval)),
	337	BPF_EXIT_INSN(),
	338	},
	339	.attach_type = BPF_CGROUP_GETSOCKOPT,
	340	.expected_attach_type = BPF_CGROUP_GETSOCKOPT,
	341
	342	.error = DENY_LOAD,
	343	},
	344	{
	345	.descr = "getsockopt: deny writing to ctx->optval_end",
	346	.insns = {
	347	/* ctx->optval_end = 1 */
	348	BPF_MOV64_IMM(BPF_REG_0, 1),
	349	BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0,
	350	offsetof(struct bpf_sockopt, optval_end)),
	351	BPF_EXIT_INSN(),
	352	},
	353	.attach_type = BPF_CGROUP_GETSOCKOPT,
	354	.expected_attach_type = BPF_CGROUP_GETSOCKOPT,
	355
	356	.error = DENY_LOAD,
	357	},
	358	{
	359	.descr = "getsockopt: rewrite value",
	360	.insns = {
	361	/* r6 = ctx->optval */
	362	BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1,
	363	offsetof(struct bpf_sockopt, optval)),
	364	/* r2 = ctx->optval */
	365	BPF_MOV64_REG(BPF_REG_2, BPF_REG_6),
	366	/* r6 = ctx->optval + 1 */
	367	BPF_ALU64_IMM(BPF_ADD, BPF_REG_6, 1),
	368
	369	/* r7 = ctx->optval_end */
	370	BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_1,
	371	offsetof(struct bpf_sockopt, optval_end)),
	372
	373	/* if (ctx->optval + 1 <= ctx->optval_end) { */
	374	BPF_JMP_REG(BPF_JGT, BPF_REG_6, BPF_REG_7, 1),
	375	/* ctx->optval[0] = 0xF0 */
	376	BPF_ST_MEM(BPF_B, BPF_REG_2, 0, 0xF0),
	377	/* } */
	378
	379	/* ctx->retval = 0 */
	380	BPF_MOV64_IMM(BPF_REG_0, 0),
	381	BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0,
	382	offsetof(struct bpf_sockopt, retval)),
	383
	384	/* return 1*/
	385	BPF_MOV64_IMM(BPF_REG_0, 1),
	386	BPF_EXIT_INSN(),
	387	},
	388	.attach_type = BPF_CGROUP_GETSOCKOPT,
	389	.expected_attach_type = BPF_CGROUP_GETSOCKOPT,
	390
	391	.get_level = SOL_IP,
	392	.get_optname = IP_TOS,
	393
	394	.get_optval = { 0xF0 },
	395	.get_optlen = 1,
	396	},
	397
	398	/* ==================== setsockopt ==================== */
	399
	400	{
	401	.descr = "setsockopt: no expected_attach_type",
	402	.insns = {
	403	/* return 1 */
	404	BPF_MOV64_IMM(BPF_REG_0, 1),
	405	BPF_EXIT_INSN(),
	406
	407	},
	408	.attach_type = BPF_CGROUP_SETSOCKOPT,
	409	.expected_attach_type = 0,
	410	.error = DENY_LOAD,
	411	},
	412	{
	413	.descr = "setsockopt: wrong expected_attach_type",
	414	.insns = {
	415	/* return 1 */
	416	BPF_MOV64_IMM(BPF_REG_0, 1),
	417	BPF_EXIT_INSN(),
	418
	419	},
	420	.attach_type = BPF_CGROUP_SETSOCKOPT,
	421	.expected_attach_type = BPF_CGROUP_GETSOCKOPT,
	422	.error = DENY_ATTACH,
	423	},
	424	{
	425	.descr = "setsockopt: bypass bpf hook",
	426	.insns = {
	427	/* return 1 */
	428	BPF_MOV64_IMM(BPF_REG_0, 1),
	429	BPF_EXIT_INSN(),
	430	},
	431	.attach_type = BPF_CGROUP_SETSOCKOPT,
	432	.expected_attach_type = BPF_CGROUP_SETSOCKOPT,
	433
	434	.get_level = SOL_IP,
	435	.set_level = SOL_IP,
	436
	437	.get_optname = IP_TOS,
	438	.set_optname = IP_TOS,
	439
	440	.set_optval = { 1 << 3 },
	441	.set_optlen = 1,
	442
	443	.get_optval = { 1 << 3 },
	444	.get_optlen = 1,
	445	},
	446	{
	447	.descr = "setsockopt: return EPERM from bpf hook",
	448	.insns = {
	449	/* return 0 */
	450	BPF_MOV64_IMM(BPF_REG_0, 0),
	451	BPF_EXIT_INSN(),
	452	},
	453	.attach_type = BPF_CGROUP_SETSOCKOPT,
	454	.expected_attach_type = BPF_CGROUP_SETSOCKOPT,
	455
	456	.set_level = SOL_IP,
	457	.set_optname = IP_TOS,
	458
	459	.set_optlen = 1,
	460	.error = EPERM_SETSOCKOPT,
	461	},
	462	{
	463	.descr = "setsockopt: no optval bounds check, deny loading",
	464	.insns = {
	465	/* r6 = ctx->optval */
	466	BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1,
	467	offsetof(struct bpf_sockopt, optval)),
	468
	469	/* r0 = ctx->optval[0] */
	470	BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6, 0),
	471
	472	/* return 1 */
	473	BPF_MOV64_IMM(BPF_REG_0, 1),
	474	BPF_EXIT_INSN(),
	475	},
	476	.attach_type = BPF_CGROUP_SETSOCKOPT,
	477	.expected_attach_type = BPF_CGROUP_SETSOCKOPT,
	478	.error = DENY_LOAD,
	479	},
	480	{
	481	.descr = "setsockopt: read ctx->level",
	482	.insns = {
	483	/* r6 = ctx->level */
	484	BPF_LDX_MEM(BPF_W, BPF_REG_6, BPF_REG_1,
	485	offsetof(struct bpf_sockopt, level)),
	486
	487	/* if (ctx->level == 123) { */
	488	BPF_JMP_IMM(BPF_JNE, BPF_REG_6, 123, 4),
	489	/* ctx->optlen = -1 */
	490	BPF_MOV64_IMM(BPF_REG_0, -1),
	491	BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0,
	492	offsetof(struct bpf_sockopt, optlen)),
	493	/* return 1 */
	494	BPF_MOV64_IMM(BPF_REG_0, 1),
	495	BPF_JMP_A(1),
	496	/* } else { */
	497	/* return 0 */
	498	BPF_MOV64_IMM(BPF_REG_0, 0),
	499	/* } */
	500	BPF_EXIT_INSN(),
	501	},
	502	.attach_type = BPF_CGROUP_SETSOCKOPT,
	503	.expected_attach_type = BPF_CGROUP_SETSOCKOPT,
	504
	505	.set_level = 123,
	506
	507	.set_optlen = 1,
	508	},
	509	{
	510	.descr = "setsockopt: allow changing ctx->level",
	511	.insns = {
	512	/* ctx->level = SOL_IP */
	513	BPF_MOV64_IMM(BPF_REG_0, SOL_IP),
	514	BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0,
	515	offsetof(struct bpf_sockopt, level)),
	516	/* return 1 */
	517	BPF_MOV64_IMM(BPF_REG_0, 1),
	518	BPF_EXIT_INSN(),
	519	},
	520	.attach_type = BPF_CGROUP_SETSOCKOPT,
	521	.expected_attach_type = BPF_CGROUP_SETSOCKOPT,
	522
	523	.get_level = SOL_IP,
	524	.set_level = 234, /* should be rewritten to SOL_IP */
	525
	526	.get_optname = IP_TOS,
	527	.set_optname = IP_TOS,
	528
	529	.set_optval = { 1 << 3 },
	530	.set_optlen = 1,
	531	.get_optval = { 1 << 3 },
	532	.get_optlen = 1,
	533	},
	534	{
	535	.descr = "setsockopt: read ctx->optname",
	536	.insns = {
	537	/* r6 = ctx->optname */
	538	BPF_LDX_MEM(BPF_W, BPF_REG_6, BPF_REG_1,
	539	offsetof(struct bpf_sockopt, optname)),
	540
	541	/* if (ctx->optname == 123) { */
	542	BPF_JMP_IMM(BPF_JNE, BPF_REG_6, 123, 4),
	543	/* ctx->optlen = -1 */
	544	BPF_MOV64_IMM(BPF_REG_0, -1),
	545	BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0,
	546	offsetof(struct bpf_sockopt, optlen)),
	547	/* return 1 */
	548	BPF_MOV64_IMM(BPF_REG_0, 1),
	549	BPF_JMP_A(1),
	550	/* } else { */
	551	/* return 0 */
	552	BPF_MOV64_IMM(BPF_REG_0, 0),
	553	/* } */
	554	BPF_EXIT_INSN(),
	555	},
	556	.attach_type = BPF_CGROUP_SETSOCKOPT,
	557	.expected_attach_type = BPF_CGROUP_SETSOCKOPT,
	558
	559	.set_optname = 123,
	560
	561	.set_optlen = 1,
	562	},
	563	{
	564	.descr = "setsockopt: allow changing ctx->optname",
	565	.insns = {
	566	/* ctx->optname = IP_TOS */
	567	BPF_MOV64_IMM(BPF_REG_0, IP_TOS),
	568	BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0,
	569	offsetof(struct bpf_sockopt, optname)),
	570	/* return 1 */
	571	BPF_MOV64_IMM(BPF_REG_0, 1),
	572	BPF_EXIT_INSN(),
	573	},
	574	.attach_type = BPF_CGROUP_SETSOCKOPT,
	575	.expected_attach_type = BPF_CGROUP_SETSOCKOPT,
	576
	577	.get_level = SOL_IP,
	578	.set_level = SOL_IP,
	579
	580	.get_optname = IP_TOS,
	581	.set_optname = 456, /* should be rewritten to IP_TOS */
	582
	583	.set_optval = { 1 << 3 },
	584	.set_optlen = 1,
	585	.get_optval = { 1 << 3 },
	586	.get_optlen = 1,
	587	},
	588	{
	589	.descr = "setsockopt: read ctx->optlen",
	590	.insns = {
	591	/* r6 = ctx->optlen */
	592	BPF_LDX_MEM(BPF_W, BPF_REG_6, BPF_REG_1,
	593	offsetof(struct bpf_sockopt, optlen)),
	594
	595	/* if (ctx->optlen == 64) { */
	596	BPF_JMP_IMM(BPF_JNE, BPF_REG_6, 64, 4),
	597	/* ctx->optlen = -1 */
	598	BPF_MOV64_IMM(BPF_REG_0, -1),
	599	BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0,
	600	offsetof(struct bpf_sockopt, optlen)),
	601	/* return 1 */
	602	BPF_MOV64_IMM(BPF_REG_0, 1),
	603	BPF_JMP_A(1),
	604	/* } else { */
	605	/* return 0 */
	606	BPF_MOV64_IMM(BPF_REG_0, 0),
	607	/* } */
	608	BPF_EXIT_INSN(),
	609	},
	610	.attach_type = BPF_CGROUP_SETSOCKOPT,
	611	.expected_attach_type = BPF_CGROUP_SETSOCKOPT,
	612
	613	.set_optlen = 64,
	614	},
	615	{
	616	.descr = "setsockopt: ctx->optlen == -1 is ok",
	617	.insns = {
	618	/* ctx->optlen = -1 */
	619	BPF_MOV64_IMM(BPF_REG_0, -1),
	620	BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0,
	621	offsetof(struct bpf_sockopt, optlen)),
	622	/* return 1 */
	623	BPF_MOV64_IMM(BPF_REG_0, 1),
	624	BPF_EXIT_INSN(),
	625	},
	626	.attach_type = BPF_CGROUP_SETSOCKOPT,
	627	.expected_attach_type = BPF_CGROUP_SETSOCKOPT,
	628
	629	.set_optlen = 64,
	630	},
	631	{
	632	.descr = "setsockopt: deny ctx->optlen < 0 (except -1)",
	633	.insns = {
	634	/* ctx->optlen = -2 */
	635	BPF_MOV64_IMM(BPF_REG_0, -2),
	636	BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0,
	637	offsetof(struct bpf_sockopt, optlen)),
	638	/* return 1 */
	639	BPF_MOV64_IMM(BPF_REG_0, 1),
	640	BPF_EXIT_INSN(),
	641	},
	642	.attach_type = BPF_CGROUP_SETSOCKOPT,
	643	.expected_attach_type = BPF_CGROUP_SETSOCKOPT,
	644
	645	.set_optlen = 4,
	646
	647	.error = EFAULT_SETSOCKOPT,
	648	},
	649	{
	650	.descr = "setsockopt: deny ctx->optlen > input optlen",
	651	.insns = {
	652	/* ctx->optlen = 65 */
	653	BPF_MOV64_IMM(BPF_REG_0, 65),
	654	BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0,
	655	offsetof(struct bpf_sockopt, optlen)),
	656	BPF_MOV64_IMM(BPF_REG_0, 1),
	657	BPF_EXIT_INSN(),
	658	},
	659	.attach_type = BPF_CGROUP_SETSOCKOPT,
	660	.expected_attach_type = BPF_CGROUP_SETSOCKOPT,
	661
	662	.set_optlen = 64,
	663
	664	.error = EFAULT_SETSOCKOPT,
	665	},
	666	{
	667	.descr = "setsockopt: allow changing ctx->optlen within bounds",
	668	.insns = {
	669	/* r6 = ctx->optval */
	670	BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1,
	671	offsetof(struct bpf_sockopt, optval)),
	672	/* r2 = ctx->optval */
	673	BPF_MOV64_REG(BPF_REG_2, BPF_REG_6),
	674	/* r6 = ctx->optval + 1 */
	675	BPF_ALU64_IMM(BPF_ADD, BPF_REG_6, 1),
	676
	677	/* r7 = ctx->optval_end */
	678	BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_1,
	679	offsetof(struct bpf_sockopt, optval_end)),
	680
	681	/* if (ctx->optval + 1 <= ctx->optval_end) { */
	682	BPF_JMP_REG(BPF_JGT, BPF_REG_6, BPF_REG_7, 1),
	683	/* ctx->optval[0] = 1 << 3 */
	684	BPF_ST_MEM(BPF_B, BPF_REG_2, 0, 1 << 3),
	685	/* } */
	686
	687	/* ctx->optlen = 1 */
	688	BPF_MOV64_IMM(BPF_REG_0, 1),
	689	BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0,
	690	offsetof(struct bpf_sockopt, optlen)),
	691
	692	/* return 1*/
	693	BPF_MOV64_IMM(BPF_REG_0, 1),
	694	BPF_EXIT_INSN(),
	695	},
	696	.attach_type = BPF_CGROUP_SETSOCKOPT,
	697	.expected_attach_type = BPF_CGROUP_SETSOCKOPT,
	698
	699	.get_level = SOL_IP,
	700	.set_level = SOL_IP,
	701
	702	.get_optname = IP_TOS,
	703	.set_optname = IP_TOS,
	704
	705	.set_optval = { 1, 1, 1, 1 },
	706	.set_optlen = 4,
	707	.get_optval = { 1 << 3 },
	708	.get_optlen = 1,
	709	},
	710	{
	711	.descr = "setsockopt: deny write ctx->retval",
	712	.insns = {
	713	/* ctx->retval = 0 */
	714	BPF_MOV64_IMM(BPF_REG_0, 0),
	715	BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0,
	716	offsetof(struct bpf_sockopt, retval)),
	717
	718	/* return 1 */
	719	BPF_MOV64_IMM(BPF_REG_0, 1),
	720	BPF_EXIT_INSN(),
	721	},
	722	.attach_type = BPF_CGROUP_SETSOCKOPT,
	723	.expected_attach_type = BPF_CGROUP_SETSOCKOPT,
	724
	725	.error = DENY_LOAD,
	726	},
	727	{
	728	.descr = "setsockopt: deny read ctx->retval",
	729	.insns = {
	730	/* r6 = ctx->retval */
	731	BPF_LDX_MEM(BPF_W, BPF_REG_6, BPF_REG_1,
	732	offsetof(struct bpf_sockopt, retval)),
	733
	734	/* return 1 */
	735	BPF_MOV64_IMM(BPF_REG_0, 1),
	736	BPF_EXIT_INSN(),
	737	},
	738	.attach_type = BPF_CGROUP_SETSOCKOPT,
	739	.expected_attach_type = BPF_CGROUP_SETSOCKOPT,
	740
	741	.error = DENY_LOAD,
	742	},
	743	{
	744	.descr = "setsockopt: deny writing to ctx->optval",
	745	.insns = {
	746	/* ctx->optval = 1 */
	747	BPF_MOV64_IMM(BPF_REG_0, 1),
	748	BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0,
	749	offsetof(struct bpf_sockopt, optval)),
	750	BPF_EXIT_INSN(),
	751	},
	752	.attach_type = BPF_CGROUP_SETSOCKOPT,
	753	.expected_attach_type = BPF_CGROUP_SETSOCKOPT,
	754
	755	.error = DENY_LOAD,
	756	},
	757	{
	758	.descr = "setsockopt: deny writing to ctx->optval_end",
	759	.insns = {
	760	/* ctx->optval_end = 1 */
	761	BPF_MOV64_IMM(BPF_REG_0, 1),
	762	BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0,
	763	offsetof(struct bpf_sockopt, optval_end)),
	764	BPF_EXIT_INSN(),
	765	},
	766	.attach_type = BPF_CGROUP_SETSOCKOPT,
	767	.expected_attach_type = BPF_CGROUP_SETSOCKOPT,
	768
	769	.error = DENY_LOAD,
	770	},
	771	{
	772	.descr = "setsockopt: allow IP_TOS <= 128",
	773	.insns = {
	774	/* r6 = ctx->optval */
	775	BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1,
	776	offsetof(struct bpf_sockopt, optval)),
	777	/* r7 = ctx->optval + 1 */
	778	BPF_MOV64_REG(BPF_REG_7, BPF_REG_6),
	779	BPF_ALU64_IMM(BPF_ADD, BPF_REG_7, 1),
	780
	781	/* r8 = ctx->optval_end */
	782	BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_1,
	783	offsetof(struct bpf_sockopt, optval_end)),
	784
	785	/* if (ctx->optval + 1 <= ctx->optval_end) { */
	786	BPF_JMP_REG(BPF_JGT, BPF_REG_7, BPF_REG_8, 4),
	787
	788	/* r9 = ctx->optval[0] */
	789	BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_6, 0),
	790
	791	/* if (ctx->optval[0] < 128) */
	792	BPF_JMP_IMM(BPF_JGT, BPF_REG_9, 128, 2),
	793	BPF_MOV64_IMM(BPF_REG_0, 1),
	794	BPF_JMP_A(1),
	795	/* } */
	796
	797	/* } else { */
	798	BPF_MOV64_IMM(BPF_REG_0, 0),
	799	/* } */
	800
	801	BPF_EXIT_INSN(),
	802	},
	803	.attach_type = BPF_CGROUP_SETSOCKOPT,
	804	.expected_attach_type = BPF_CGROUP_SETSOCKOPT,
	805
	806	.get_level = SOL_IP,
	807	.set_level = SOL_IP,
	808
	809	.get_optname = IP_TOS,
	810	.set_optname = IP_TOS,
	811
	812	.set_optval = { 0x80 },
	813	.set_optlen = 1,
	814	.get_optval = { 0x80 },
	815	.get_optlen = 1,
	816	},
	817	{
	818	.descr = "setsockopt: deny IP_TOS > 128",
	819	.insns = {
	820	/* r6 = ctx->optval */
	821	BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1,
	822	offsetof(struct bpf_sockopt, optval)),
	823	/* r7 = ctx->optval + 1 */
	824	BPF_MOV64_REG(BPF_REG_7, BPF_REG_6),
	825	BPF_ALU64_IMM(BPF_ADD, BPF_REG_7, 1),
	826
	827	/* r8 = ctx->optval_end */
	828	BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_1,
	829	offsetof(struct bpf_sockopt, optval_end)),
	830
	831	/* if (ctx->optval + 1 <= ctx->optval_end) { */
	832	BPF_JMP_REG(BPF_JGT, BPF_REG_7, BPF_REG_8, 4),
	833
	834	/* r9 = ctx->optval[0] */
	835	BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_6, 0),
	836
	837	/* if (ctx->optval[0] < 128) */
	838	BPF_JMP_IMM(BPF_JGT, BPF_REG_9, 128, 2),
	839	BPF_MOV64_IMM(BPF_REG_0, 1),
	840	BPF_JMP_A(1),
	841	/* } */
	842
	843	/* } else { */
	844	BPF_MOV64_IMM(BPF_REG_0, 0),
	845	/* } */
	846
	847	BPF_EXIT_INSN(),
	848	},
	849	.attach_type = BPF_CGROUP_SETSOCKOPT,
	850	.expected_attach_type = BPF_CGROUP_SETSOCKOPT,
	851
	852	.get_level = SOL_IP,
	853	.set_level = SOL_IP,
	854
	855	.get_optname = IP_TOS,
	856	.set_optname = IP_TOS,
	857
	858	.set_optval = { 0x81 },
	859	.set_optlen = 1,
	860	.get_optval = { 0x00 },
	861	.get_optlen = 1,
	862
	863	.error = EPERM_SETSOCKOPT,
	864	},
	865	};
	866
	867	static int load_prog(const struct bpf_insn *insns,
	868	enum bpf_attach_type expected_attach_type)
	869	{
	870	struct bpf_load_program_attr attr = {
	871	.prog_type = BPF_PROG_TYPE_CGROUP_SOCKOPT,
	872	.expected_attach_type = expected_attach_type,
	873	.insns = insns,
	874	.license = "GPL",
	875	.log_level = 2,
	876	};
	877	int fd;
	878
	879	for (;
	880	insns[attr.insns_cnt].code != (BPF_JMP \| BPF_EXIT);
	881	attr.insns_cnt++) {
	882	}
	883	attr.insns_cnt++;
	884
	885	fd = bpf_load_program_xattr(&attr, bpf_log_buf, sizeof(bpf_log_buf));
	886	if (verbose && fd < 0)
	887	fprintf(stderr, "%s\n", bpf_log_buf);
	888
	889	return fd;
	890	}
	891
	892	static int run_test(int cgroup_fd, struct sockopt_test *test)
	893	{
	894	int sock_fd, err, prog_fd;
	895	void *optval = NULL;
	896	int ret = 0;
	897
	898	prog_fd = load_prog(test->insns, test->expected_attach_type);
	899	if (prog_fd < 0) {
	900	if (test->error == DENY_LOAD)
	901	return 0;
	902
	903	log_err("Failed to load BPF program");
	904	return -1;
	905	}
	906
	907	err = bpf_prog_attach(prog_fd, cgroup_fd, test->attach_type, 0);
	908	if (err < 0) {
	909	if (test->error == DENY_ATTACH)
	910	goto close_prog_fd;
	911
	912	log_err("Failed to attach BPF program");
	913	ret = -1;
	914	goto close_prog_fd;
	915	}
	916
	917	sock_fd = socket(AF_INET, SOCK_STREAM, 0);
	918	if (sock_fd < 0) {
	919	log_err("Failed to create AF_INET socket");
	920	ret = -1;
	921	goto detach_prog;
	922	}
	923
	924	if (test->set_optlen) {
	925	err = setsockopt(sock_fd, test->set_level, test->set_optname,
	926	test->set_optval, test->set_optlen);
	927	if (err) {
	928	if (errno == EPERM && test->error == EPERM_SETSOCKOPT)
	929	goto close_sock_fd;
	930	if (errno == EFAULT && test->error == EFAULT_SETSOCKOPT)
	931	goto free_optval;
	932
	933	log_err("Failed to call setsockopt");
	934	ret = -1;
	935	goto close_sock_fd;
	936	}
	937	}
	938
	939	if (test->get_optlen) {
	940	optval = malloc(test->get_optlen);
	941	socklen_t optlen = test->get_optlen;
	942	socklen_t expected_get_optlen = test->get_optlen_ret ?:
	943	test->get_optlen;
	944
	945	err = getsockopt(sock_fd, test->get_level, test->get_optname,
	946	optval, &optlen);
	947	if (err) {
	948	if (errno == EPERM && test->error == EPERM_GETSOCKOPT)
	949	goto free_optval;
	950	if (errno == EFAULT && test->error == EFAULT_GETSOCKOPT)
	951	goto free_optval;
	952
	953	log_err("Failed to call getsockopt");
	954	ret = -1;
	955	goto free_optval;
	956	}
	957
	958	if (optlen != expected_get_optlen) {
	959	errno = 0;
	960	log_err("getsockopt returned unexpected optlen");
	961	ret = -1;
	962	goto free_optval;
	963	}
	964
	965	if (memcmp(optval, test->get_optval, optlen) != 0) {
	966	errno = 0;
	967	log_err("getsockopt returned unexpected optval");
	968	ret = -1;
	969	goto free_optval;
	970	}
	971	}
	972
	973	ret = test->error != OK;
	974
	975	free_optval:
	976	free(optval);
	977	close_sock_fd:
	978	close(sock_fd);
	979	detach_prog:
	980	bpf_prog_detach2(prog_fd, cgroup_fd, test->attach_type);
	981	close_prog_fd:
	982	close(prog_fd);
	983	return ret;
	984	}
	985
	986	int main(int args, char **argv)
	987	{
	988	int err = EXIT_FAILURE, error_cnt = 0;
	989	int cgroup_fd, i;
	990
	991	if (setup_cgroup_environment())
	992	goto cleanup_obj;
	993
	994	cgroup_fd = create_and_get_cgroup(CG_PATH);
	995	if (cgroup_fd < 0)
	996	goto cleanup_cgroup_env;
	997
	998	if (join_cgroup(CG_PATH))
	999	goto cleanup_cgroup;
	1000
	1001	for (i = 0; i < ARRAY_SIZE(tests); i++) {
	1002	int err = run_test(cgroup_fd, &tests[i]);
	1003
	1004	if (err)
	1005	error_cnt++;
	1006
	1007	printf("#%d %s: %s\n", i, err ? "FAIL" : "PASS",
	1008	tests[i].descr);
	1009	}
	1010
	1011	printf("Summary: %ld PASSED, %d FAILED\n",
	1012	ARRAY_SIZE(tests) - error_cnt, error_cnt);
	1013	err = error_cnt ? EXIT_FAILURE : EXIT_SUCCESS;
	1014
	1015	cleanup_cgroup:
	1016	close(cgroup_fd);
	1017	cleanup_cgroup_env:
	1018	cleanup_cgroup_environment();
	1019	cleanup_obj:
	1020	return err;
	1021	}

tools/testing/selftests/bpf/test_sockopt_multi.c+374−0

@@ -0,0 +1,374 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	#include <error.h>
	4	#include <errno.h>
	5	#include <stdio.h>
	6	#include <unistd.h>
	7	#include <sys/types.h>
	8	#include <sys/socket.h>
	9	#include <netinet/in.h>
	10
	11	#include <linux/filter.h>
	12	#include <bpf/bpf.h>
	13	#include <bpf/libbpf.h>
	14
	15	#include "bpf_rlimit.h"
	16	#include "bpf_util.h"
	17	#include "cgroup_helpers.h"
	18
	19	static int prog_attach(struct bpf_object obj, int cgroup_fd, const char title)
	20	{
	21	enum bpf_attach_type attach_type;
	22	enum bpf_prog_type prog_type;
	23	struct bpf_program *prog;
	24	int err;
	25
	26	err = libbpf_prog_type_by_name(title, &prog_type, &attach_type);
	27	if (err) {
	28	log_err("Failed to deduct types for %s BPF program", title);
	29	return -1;
	30	}
	31
	32	prog = bpf_object__find_program_by_title(obj, title);
	33	if (!prog) {
	34	log_err("Failed to find %s BPF program", title);
	35	return -1;
	36	}
	37
	38	err = bpf_prog_attach(bpf_program__fd(prog), cgroup_fd,
	39	attach_type, BPF_F_ALLOW_MULTI);
	40	if (err) {
	41	log_err("Failed to attach %s BPF program", title);
	42	return -1;
	43	}
	44
	45	return 0;
	46	}
	47
	48	static int prog_detach(struct bpf_object obj, int cgroup_fd, const char title)
	49	{
	50	enum bpf_attach_type attach_type;
	51	enum bpf_prog_type prog_type;
	52	struct bpf_program *prog;
	53	int err;
	54
	55	err = libbpf_prog_type_by_name(title, &prog_type, &attach_type);
	56	if (err)
	57	return -1;
	58
	59	prog = bpf_object__find_program_by_title(obj, title);
	60	if (!prog)
	61	return -1;
	62
	63	err = bpf_prog_detach2(bpf_program__fd(prog), cgroup_fd,
	64	attach_type);
	65	if (err)
	66	return -1;
	67
	68	return 0;
	69	}
	70
	71	static int run_getsockopt_test(struct bpf_object *obj, int cg_parent,
	72	int cg_child, int sock_fd)
	73	{
	74	socklen_t optlen;
	75	__u8 buf;
	76	int err;
	77
	78	/* Set IP_TOS to the expected value (0x80). */
	79
	80	buf = 0x80;
	81	err = setsockopt(sock_fd, SOL_IP, IP_TOS, &buf, 1);
	82	if (err < 0) {
	83	log_err("Failed to call setsockopt(IP_TOS)");
	84	goto detach;
	85	}
	86
	87	buf = 0x00;
	88	optlen = 1;
	89	err = getsockopt(sock_fd, SOL_IP, IP_TOS, &buf, &optlen);
	90	if (err) {
	91	log_err("Failed to call getsockopt(IP_TOS)");
	92	goto detach;
	93	}
	94
	95	if (buf != 0x80) {
	96	log_err("Unexpected getsockopt 0x%x != 0x80 without BPF", buf);
	97	err = -1;
	98	goto detach;
	99	}
	100
	101	/* Attach child program and make sure it returns new value:
	102	* - kernel: -> 0x80
	103	* - child: 0x80 -> 0x90
	104	*/
	105
	106	err = prog_attach(obj, cg_child, "cgroup/getsockopt/child");
	107	if (err)
	108	goto detach;
	109
	110	buf = 0x00;
	111	optlen = 1;
	112	err = getsockopt(sock_fd, SOL_IP, IP_TOS, &buf, &optlen);
	113	if (err) {
	114	log_err("Failed to call getsockopt(IP_TOS)");
	115	goto detach;
	116	}
	117
	118	if (buf != 0x90) {
	119	log_err("Unexpected getsockopt 0x%x != 0x90", buf);
	120	err = -1;
	121	goto detach;
	122	}
	123
	124	/* Attach parent program and make sure it returns new value:
	125	* - kernel: -> 0x80
	126	* - child: 0x80 -> 0x90
	127	* - parent: 0x90 -> 0xA0
	128	*/
	129
	130	err = prog_attach(obj, cg_parent, "cgroup/getsockopt/parent");
	131	if (err)
	132	goto detach;
	133
	134	buf = 0x00;
	135	optlen = 1;
	136	err = getsockopt(sock_fd, SOL_IP, IP_TOS, &buf, &optlen);
	137	if (err) {
	138	log_err("Failed to call getsockopt(IP_TOS)");
	139	goto detach;
	140	}
	141
	142	if (buf != 0xA0) {
	143	log_err("Unexpected getsockopt 0x%x != 0xA0", buf);
	144	err = -1;
	145	goto detach;
	146	}
	147
	148	/* Setting unexpected initial sockopt should return EPERM:
	149	* - kernel: -> 0x40
	150	* - child: unexpected 0x40, EPERM
	151	* - parent: unexpected 0x40, EPERM
	152	*/
	153
	154	buf = 0x40;
	155	if (setsockopt(sock_fd, SOL_IP, IP_TOS, &buf, 1) < 0) {
	156	log_err("Failed to call setsockopt(IP_TOS)");
	157	goto detach;
	158	}
	159
	160	buf = 0x00;
	161	optlen = 1;
	162	err = getsockopt(sock_fd, SOL_IP, IP_TOS, &buf, &optlen);
	163	if (!err) {
	164	log_err("Unexpected success from getsockopt(IP_TOS)");
	165	goto detach;
	166	}
	167
	168	/* Detach child program and make sure we still get EPERM:
	169	* - kernel: -> 0x40
	170	* - parent: unexpected 0x40, EPERM
	171	*/
	172
	173	err = prog_detach(obj, cg_child, "cgroup/getsockopt/child");
	174	if (err) {
	175	log_err("Failed to detach child program");
	176	goto detach;
	177	}
	178
	179	buf = 0x00;
	180	optlen = 1;
	181	err = getsockopt(sock_fd, SOL_IP, IP_TOS, &buf, &optlen);
	182	if (!err) {
	183	log_err("Unexpected success from getsockopt(IP_TOS)");
	184	goto detach;
	185	}
	186
	187	/* Set initial value to the one the parent program expects:
	188	* - kernel: -> 0x90
	189	* - parent: 0x90 -> 0xA0
	190	*/
	191
	192	buf = 0x90;
	193	err = setsockopt(sock_fd, SOL_IP, IP_TOS, &buf, 1);
	194	if (err < 0) {
	195	log_err("Failed to call setsockopt(IP_TOS)");
	196	goto detach;
	197	}
	198
	199	buf = 0x00;
	200	optlen = 1;
	201	err = getsockopt(sock_fd, SOL_IP, IP_TOS, &buf, &optlen);
	202	if (err) {
	203	log_err("Failed to call getsockopt(IP_TOS)");
	204	goto detach;
	205	}
	206
	207	if (buf != 0xA0) {
	208	log_err("Unexpected getsockopt 0x%x != 0xA0", buf);
	209	err = -1;
	210	goto detach;
	211	}
	212
	213	detach:
	214	prog_detach(obj, cg_child, "cgroup/getsockopt/child");
	215	prog_detach(obj, cg_parent, "cgroup/getsockopt/parent");
	216
	217	return err;
	218	}
	219
	220	static int run_setsockopt_test(struct bpf_object *obj, int cg_parent,
	221	int cg_child, int sock_fd)
	222	{
	223	socklen_t optlen;
	224	__u8 buf;
	225	int err;
	226
	227	/* Set IP_TOS to the expected value (0x80). */
	228
	229	buf = 0x80;
	230	err = setsockopt(sock_fd, SOL_IP, IP_TOS, &buf, 1);
	231	if (err < 0) {
	232	log_err("Failed to call setsockopt(IP_TOS)");
	233	goto detach;
	234	}
	235
	236	buf = 0x00;
	237	optlen = 1;
	238	err = getsockopt(sock_fd, SOL_IP, IP_TOS, &buf, &optlen);
	239	if (err) {
	240	log_err("Failed to call getsockopt(IP_TOS)");
	241	goto detach;
	242	}
	243
	244	if (buf != 0x80) {
	245	log_err("Unexpected getsockopt 0x%x != 0x80 without BPF", buf);
	246	err = -1;
	247	goto detach;
	248	}
	249
	250	/* Attach child program and make sure it adds 0x10. */
	251
	252	err = prog_attach(obj, cg_child, "cgroup/setsockopt");
	253	if (err)
	254	goto detach;
	255
	256	buf = 0x80;
	257	err = setsockopt(sock_fd, SOL_IP, IP_TOS, &buf, 1);
	258	if (err < 0) {
	259	log_err("Failed to call setsockopt(IP_TOS)");
	260	goto detach;
	261	}
	262
	263	buf = 0x00;
	264	optlen = 1;
	265	err = getsockopt(sock_fd, SOL_IP, IP_TOS, &buf, &optlen);
	266	if (err) {
	267	log_err("Failed to call getsockopt(IP_TOS)");
	268	goto detach;
	269	}
	270
	271	if (buf != 0x80 + 0x10) {
	272	log_err("Unexpected getsockopt 0x%x != 0x80 + 0x10", buf);
	273	err = -1;
	274	goto detach;
	275	}
	276
	277	/* Attach parent program and make sure it adds another 0x10. */
	278
	279	err = prog_attach(obj, cg_parent, "cgroup/setsockopt");
	280	if (err)
	281	goto detach;
	282
	283	buf = 0x80;
	284	err = setsockopt(sock_fd, SOL_IP, IP_TOS, &buf, 1);
	285	if (err < 0) {
	286	log_err("Failed to call setsockopt(IP_TOS)");
	287	goto detach;
	288	}
	289
	290	buf = 0x00;
	291	optlen = 1;
	292	err = getsockopt(sock_fd, SOL_IP, IP_TOS, &buf, &optlen);
	293	if (err) {
	294	log_err("Failed to call getsockopt(IP_TOS)");
	295	goto detach;
	296	}
	297
	298	if (buf != 0x80 + 2 * 0x10) {
	299	log_err("Unexpected getsockopt 0x%x != 0x80 + 2 * 0x10", buf);
	300	err = -1;
	301	goto detach;
	302	}
	303
	304	detach:
	305	prog_detach(obj, cg_child, "cgroup/setsockopt");
	306	prog_detach(obj, cg_parent, "cgroup/setsockopt");
	307
	308	return err;
	309	}
	310
	311	int main(int argc, char **argv)
	312	{
	313	struct bpf_prog_load_attr attr = {
	314	.file = "./sockopt_multi.o",
	315	};
	316	int cg_parent = -1, cg_child = -1;
	317	struct bpf_object *obj = NULL;
	318	int sock_fd = -1;
	319	int err = -1;
	320	int ignored;
	321
	322	if (setup_cgroup_environment()) {
	323	log_err("Failed to setup cgroup environment\n");
	324	goto out;
	325	}
	326
	327	cg_parent = create_and_get_cgroup("/parent");
	328	if (cg_parent < 0) {
	329	log_err("Failed to create cgroup /parent\n");
	330	goto out;
	331	}
	332
	333	cg_child = create_and_get_cgroup("/parent/child");
	334	if (cg_child < 0) {
	335	log_err("Failed to create cgroup /parent/child\n");
	336	goto out;
	337	}
	338
	339	if (join_cgroup("/parent/child")) {
	340	log_err("Failed to join cgroup /parent/child\n");
	341	goto out;
	342	}
	343
	344	err = bpf_prog_load_xattr(&attr, &obj, &ignored);
	345	if (err) {
	346	log_err("Failed to load BPF object");
	347	goto out;
	348	}
	349
	350	sock_fd = socket(AF_INET, SOCK_STREAM, 0);
	351	if (sock_fd < 0) {
	352	log_err("Failed to create socket");
	353	goto out;
	354	}
	355
	356	if (run_getsockopt_test(obj, cg_parent, cg_child, sock_fd))
	357	err = -1;
	358	printf("test_sockopt_multi: getsockopt %s\n",
	359	err ? "FAILED" : "PASSED");
	360
	361	if (run_setsockopt_test(obj, cg_parent, cg_child, sock_fd))
	362	err = -1;
	363	printf("test_sockopt_multi: setsockopt %s\n",
	364	err ? "FAILED" : "PASSED");
	365
	366	out:
	367	close(sock_fd);
	368	bpf_object__close(obj);
	369	close(cg_child);
	370	close(cg_parent);
	371
	372	printf("test_sockopt_multi: %s\n", err ? "FAILED" : "PASSED");
	373	return err ? EXIT_FAILURE : EXIT_SUCCESS;
	374	}

tools/testing/selftests/bpf/test_sockopt_sk.c+211−0

@@ -0,0 +1,211 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	#include <errno.h>
	4	#include <stdio.h>
	5	#include <unistd.h>
	6	#include <sys/types.h>
	7	#include <sys/socket.h>
	8	#include <netinet/in.h>
	9
	10	#include <linux/filter.h>
	11	#include <bpf/bpf.h>
	12	#include <bpf/libbpf.h>
	13
	14	#include "bpf_rlimit.h"
	15	#include "bpf_util.h"
	16	#include "cgroup_helpers.h"
	17
	18	#define CG_PATH "/sockopt"
	19
	20	#define SOL_CUSTOM 0xdeadbeef
	21
	22	static int getsetsockopt(void)
	23	{
	24	int fd, err;
	25	union {
	26	char u8[4];
	27	__u32 u32;
	28	} buf = {};
	29	socklen_t optlen;
	30
	31	fd = socket(AF_INET, SOCK_STREAM, 0);
	32	if (fd < 0) {
	33	log_err("Failed to create socket");
	34	return -1;
	35	}
	36
	37	/* IP_TOS - BPF bypass */
	38
	39	buf.u8[0] = 0x08;
	40	err = setsockopt(fd, SOL_IP, IP_TOS, &buf, 1);
	41	if (err) {
	42	log_err("Failed to call setsockopt(IP_TOS)");
	43	goto err;
	44	}
	45
	46	buf.u8[0] = 0x00;
	47	optlen = 1;
	48	err = getsockopt(fd, SOL_IP, IP_TOS, &buf, &optlen);
	49	if (err) {
	50	log_err("Failed to call getsockopt(IP_TOS)");
	51	goto err;
	52	}
	53
	54	if (buf.u8[0] != 0x08) {
	55	log_err("Unexpected getsockopt(IP_TOS) buf[0] 0x%02x != 0x08",
	56	buf.u8[0]);
	57	goto err;
	58	}
	59
	60	/* IP_TTL - EPERM */
	61
	62	buf.u8[0] = 1;
	63	err = setsockopt(fd, SOL_IP, IP_TTL, &buf, 1);
	64	if (!err \|\| errno != EPERM) {
	65	log_err("Unexpected success from setsockopt(IP_TTL)");
	66	goto err;
	67	}
	68
	69	/* SOL_CUSTOM - handled by BPF */
	70
	71	buf.u8[0] = 0x01;
	72	err = setsockopt(fd, SOL_CUSTOM, 0, &buf, 1);
	73	if (err) {
	74	log_err("Failed to call setsockopt");
	75	goto err;
	76	}
	77
	78	buf.u32 = 0x00;
	79	optlen = 4;
	80	err = getsockopt(fd, SOL_CUSTOM, 0, &buf, &optlen);
	81	if (err) {
	82	log_err("Failed to call getsockopt");
	83	goto err;
	84	}
	85
	86	if (optlen != 1) {
	87	log_err("Unexpected optlen %d != 1", optlen);
	88	goto err;
	89	}
	90	if (buf.u8[0] != 0x01) {
	91	log_err("Unexpected buf[0] 0x%02x != 0x01", buf.u8[0]);
	92	goto err;
	93	}
	94
	95	/* SO_SNDBUF is overwritten */
	96
	97	buf.u32 = 0x01010101;
	98	err = setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &buf, 4);
	99	if (err) {
	100	log_err("Failed to call setsockopt(SO_SNDBUF)");
	101	goto err;
	102	}
	103
	104	buf.u32 = 0x00;
	105	optlen = 4;
	106	err = getsockopt(fd, SOL_SOCKET, SO_SNDBUF, &buf, &optlen);
	107	if (err) {
	108	log_err("Failed to call getsockopt(SO_SNDBUF)");
	109	goto err;
	110	}
	111
	112	if (buf.u32 != 0x55AA*2) {
	113	log_err("Unexpected getsockopt(SO_SNDBUF) 0x%x != 0x55AA*2",
	114	buf.u32);
	115	goto err;
	116	}
	117
	118	close(fd);
	119	return 0;
	120	err:
	121	close(fd);
	122	return -1;
	123	}
	124
	125	static int prog_attach(struct bpf_object obj, int cgroup_fd, const char title)
	126	{
	127	enum bpf_attach_type attach_type;
	128	enum bpf_prog_type prog_type;
	129	struct bpf_program *prog;
	130	int err;
	131
	132	err = libbpf_prog_type_by_name(title, &prog_type, &attach_type);
	133	if (err) {
	134	log_err("Failed to deduct types for %s BPF program", title);
	135	return -1;
	136	}
	137
	138	prog = bpf_object__find_program_by_title(obj, title);
	139	if (!prog) {
	140	log_err("Failed to find %s BPF program", title);
	141	return -1;
	142	}
	143
	144	err = bpf_prog_attach(bpf_program__fd(prog), cgroup_fd,
	145	attach_type, 0);
	146	if (err) {
	147	log_err("Failed to attach %s BPF program", title);
	148	return -1;
	149	}
	150
	151	return 0;
	152	}
	153
	154	static int run_test(int cgroup_fd)
	155	{
	156	struct bpf_prog_load_attr attr = {
	157	.file = "./sockopt_sk.o",
	158	};
	159	struct bpf_object *obj;
	160	int ignored;
	161	int err;
	162
	163	err = bpf_prog_load_xattr(&attr, &obj, &ignored);
	164	if (err) {
	165	log_err("Failed to load BPF object");
	166	return -1;
	167	}
	168
	169	err = prog_attach(obj, cgroup_fd, "cgroup/getsockopt");
	170	if (err)
	171	goto close_bpf_object;
	172
	173	err = prog_attach(obj, cgroup_fd, "cgroup/setsockopt");
	174	if (err)
	175	goto close_bpf_object;
	176
	177	err = getsetsockopt();
	178
	179	close_bpf_object:
	180	bpf_object__close(obj);
	181	return err;
	182	}
	183
	184	int main(int args, char **argv)
	185	{
	186	int cgroup_fd;
	187	int err = EXIT_SUCCESS;
	188
	189	if (setup_cgroup_environment())
	190	goto cleanup_obj;
	191
	192	cgroup_fd = create_and_get_cgroup(CG_PATH);
	193	if (cgroup_fd < 0)
	194	goto cleanup_cgroup_env;
	195
	196	if (join_cgroup(CG_PATH))
	197	goto cleanup_cgroup;
	198
	199	if (run_test(cgroup_fd))
	200	err = EXIT_FAILURE;
	201
	202	printf("test_sockopt_sk: %s\n",
	203	err == EXIT_SUCCESS ? "PASSED" : "FAILED");
	204
	205	cleanup_cgroup:
	206	close(cgroup_fd);
	207	cleanup_cgroup_env:
	208	cleanup_cgroup_environment();
	209	cleanup_obj:
	210	return err;
	211	}

tools/testing/selftests/bpf/test_tcp_rtt.c+254−0

@@ -0,0 +1,254 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	#include <error.h>
	3	#include <errno.h>
	4	#include <stdio.h>
	5	#include <unistd.h>
	6	#include <sys/types.h>
	7	#include <sys/socket.h>
	8	#include <netinet/in.h>
	9	#include <pthread.h>
	10
	11	#include <linux/filter.h>
	12	#include <bpf/bpf.h>
	13	#include <bpf/libbpf.h>
	14
	15	#include "bpf_rlimit.h"
	16	#include "bpf_util.h"
	17	#include "cgroup_helpers.h"
	18
	19	#define CG_PATH "/tcp_rtt"
	20
	21	struct tcp_rtt_storage {
	22	__u32 invoked;
	23	__u32 dsack_dups;
	24	__u32 delivered;
	25	__u32 delivered_ce;
	26	__u32 icsk_retransmits;
	27	};
	28
	29	static void send_byte(int fd)
	30	{
	31	char b = 0x55;
	32
	33	if (write(fd, &b, sizeof(b)) != 1)
	34	error(1, errno, "Failed to send single byte");
	35	}
	36
	37	static int verify_sk(int map_fd, int client_fd, const char *msg, __u32 invoked,
	38	__u32 dsack_dups, __u32 delivered, __u32 delivered_ce,
	39	__u32 icsk_retransmits)
	40	{
	41	int err = 0;
	42	struct tcp_rtt_storage val;
	43
	44	if (bpf_map_lookup_elem(map_fd, &client_fd, &val) < 0)
	45	error(1, errno, "Failed to read socket storage");
	46
	47	if (val.invoked != invoked) {
	48	log_err("%s: unexpected bpf_tcp_sock.invoked %d != %d",
	49	msg, val.invoked, invoked);
	50	err++;
	51	}
	52
	53	if (val.dsack_dups != dsack_dups) {
	54	log_err("%s: unexpected bpf_tcp_sock.dsack_dups %d != %d",
	55	msg, val.dsack_dups, dsack_dups);
	56	err++;
	57	}
	58
	59	if (val.delivered != delivered) {
	60	log_err("%s: unexpected bpf_tcp_sock.delivered %d != %d",
	61	msg, val.delivered, delivered);
	62	err++;
	63	}
	64
	65	if (val.delivered_ce != delivered_ce) {
	66	log_err("%s: unexpected bpf_tcp_sock.delivered_ce %d != %d",
	67	msg, val.delivered_ce, delivered_ce);
	68	err++;
	69	}
	70
	71	if (val.icsk_retransmits != icsk_retransmits) {
	72	log_err("%s: unexpected bpf_tcp_sock.icsk_retransmits %d != %d",
	73	msg, val.icsk_retransmits, icsk_retransmits);
	74	err++;
	75	}
	76
	77	return err;
	78	}
	79
	80	static int connect_to_server(int server_fd)
	81	{
	82	struct sockaddr_storage addr;
	83	socklen_t len = sizeof(addr);
	84	int fd;
	85
	86	fd = socket(AF_INET, SOCK_STREAM, 0);
	87	if (fd < 0) {
	88	log_err("Failed to create client socket");
	89	return -1;
	90	}
	91
	92	if (getsockname(server_fd, (struct sockaddr *)&addr, &len)) {
	93	log_err("Failed to get server addr");
	94	goto out;
	95	}
	96
	97	if (connect(fd, (const struct sockaddr *)&addr, len) < 0) {
	98	log_err("Fail to connect to server");
	99	goto out;
	100	}
	101
	102	return fd;
	103
	104	out:
	105	close(fd);
	106	return -1;
	107	}
	108
	109	static int run_test(int cgroup_fd, int server_fd)
	110	{
	111	struct bpf_prog_load_attr attr = {
	112	.prog_type = BPF_PROG_TYPE_SOCK_OPS,
	113	.file = "./tcp_rtt.o",
	114	.expected_attach_type = BPF_CGROUP_SOCK_OPS,
	115	};
	116	struct bpf_object *obj;
	117	struct bpf_map *map;
	118	int client_fd;
	119	int prog_fd;
	120	int map_fd;
	121	int err;
	122
	123	err = bpf_prog_load_xattr(&attr, &obj, &prog_fd);
	124	if (err) {
	125	log_err("Failed to load BPF object");
	126	return -1;
	127	}
	128
	129	map = bpf_map__next(NULL, obj);
	130	map_fd = bpf_map__fd(map);
	131
	132	err = bpf_prog_attach(prog_fd, cgroup_fd, BPF_CGROUP_SOCK_OPS, 0);
	133	if (err) {
	134	log_err("Failed to attach BPF program");
	135	goto close_bpf_object;
	136	}
	137
	138	client_fd = connect_to_server(server_fd);
	139	if (client_fd < 0) {
	140	err = -1;
	141	goto close_bpf_object;
	142	}
	143
	144	err += verify_sk(map_fd, client_fd, "syn-ack",
	145	/invoked=/1,
	146	/dsack_dups=/0,
	147	/delivered=/1,
	148	/delivered_ce=/0,
	149	/icsk_retransmits=/0);
	150
	151	send_byte(client_fd);
	152
	153	err += verify_sk(map_fd, client_fd, "first payload byte",
	154	/invoked=/2,
	155	/dsack_dups=/0,
	156	/delivered=/2,
	157	/delivered_ce=/0,
	158	/icsk_retransmits=/0);
	159
	160	close(client_fd);
	161
	162	close_bpf_object:
	163	bpf_object__close(obj);
	164	return err;
	165	}
	166
	167	static int start_server(void)
	168	{
	169	struct sockaddr_in addr = {
	170	.sin_family = AF_INET,
	171	.sin_addr.s_addr = htonl(INADDR_LOOPBACK),
	172	};
	173	int fd;
	174
	175	fd = socket(AF_INET, SOCK_STREAM, 0);
	176	if (fd < 0) {
	177	log_err("Failed to create server socket");
	178	return -1;
	179	}
	180
	181	if (bind(fd, (const struct sockaddr *)&addr, sizeof(addr)) < 0) {
	182	log_err("Failed to bind socket");
	183	close(fd);
	184	return -1;
	185	}
	186
	187	return fd;
	188	}
	189
	190	static void server_thread(void arg)
	191	{
	192	struct sockaddr_storage addr;
	193	socklen_t len = sizeof(addr);
	194	int fd = (int )arg;
	195	int client_fd;
	196
	197	if (listen(fd, 1) < 0)
	198	error(1, errno, "Failed to listed on socket");
	199
	200	client_fd = accept(fd, (struct sockaddr *)&addr, &len);
	201	if (client_fd < 0)
	202	error(1, errno, "Failed to accept client");
	203
	204	/* Wait for the next connection (that never arrives)
	205	* to keep this thread alive to prevent calling
	206	* close() on client_fd.
	207	*/
	208	if (accept(fd, (struct sockaddr *)&addr, &len) >= 0)
	209	error(1, errno, "Unexpected success in second accept");
	210
	211	close(client_fd);
	212
	213	return NULL;
	214	}
	215
	216	int main(int args, char **argv)
	217	{
	218	int server_fd, cgroup_fd;
	219	int err = EXIT_SUCCESS;
	220	pthread_t tid;
	221
	222	if (setup_cgroup_environment())
	223	goto cleanup_obj;
	224
	225	cgroup_fd = create_and_get_cgroup(CG_PATH);
	226	if (cgroup_fd < 0)
	227	goto cleanup_cgroup_env;
	228
	229	if (join_cgroup(CG_PATH))
	230	goto cleanup_cgroup;
	231
	232	server_fd = start_server();
	233	if (server_fd < 0) {
	234	err = EXIT_FAILURE;
	235	goto cleanup_cgroup;
	236	}
	237
	238	pthread_create(&tid, NULL, server_thread, (void *)&server_fd);
	239
	240	if (run_test(cgroup_fd, server_fd))
	241	err = EXIT_FAILURE;
	242
	243	close(server_fd);
	244
	245	printf("test_sockopt_sk: %s\n",
	246	err == EXIT_SUCCESS ? "PASSED" : "FAILED");
	247
	248	cleanup_cgroup:
	249	close(cgroup_fd);
	250	cleanup_cgroup_env:
	251	cleanup_cgroup_environment();
	252	cleanup_obj:
	253	return err;
	254	}

tools/testing/selftests/bpf/test_xdp_veth.sh+118−0

@@ -0,0 +1,118 @@
	1	#!/bin/sh
	2	# SPDX-License-Identifier: GPL-2.0
	3	#
	4	# Create 3 namespaces with 3 veth peers, and
	5	# forward packets in-between using native XDP
	6	#
	7	# XDP_TX
	8	# NS1(veth11) NS2(veth22) NS3(veth33)
	9	# \| \| \|
	10	# \| \| \|
	11	# (veth1, (veth2, (veth3,
	12	# id:111) id:122) id:133)
	13	# ^ \| ^ \| ^ \|
	14	# \| \| XDP_REDIRECT \| \| XDP_REDIRECT \| \|
	15	# \| ------------------ ------------------ \|
	16	# -----------------------------------------
	17	# XDP_REDIRECT
	18
	19	# Kselftest framework requirement - SKIP code is 4.
	20	ksft_skip=4
	21
	22	TESTNAME=xdp_veth
	23	BPF_FS=$(awk '$3 == "bpf" {print $2; exit}' /proc/mounts)
	24	BPF_DIR=$BPF_FS/test_$TESTNAME
	25
	26	_cleanup()
	27	{
	28	set +e
	29	ip link del veth1 2> /dev/null
	30	ip link del veth2 2> /dev/null
	31	ip link del veth3 2> /dev/null
	32	ip netns del ns1 2> /dev/null
	33	ip netns del ns2 2> /dev/null
	34	ip netns del ns3 2> /dev/null
	35	rm -rf $BPF_DIR 2> /dev/null
	36	}
	37
	38	cleanup_skip()
	39	{
	40	echo "selftests: $TESTNAME [SKIP]"
	41	_cleanup
	42
	43	exit $ksft_skip
	44	}
	45
	46	cleanup()
	47	{
	48	if [ "$?" = 0 ]; then
	49	echo "selftests: $TESTNAME [PASS]"
	50	else
	51	echo "selftests: $TESTNAME [FAILED]"
	52	fi
	53	_cleanup
	54	}
	55
	56	if [ $(id -u) -ne 0 ]; then
	57	echo "selftests: $TESTNAME [SKIP] Need root privileges"
	58	exit $ksft_skip
	59	fi
	60
	61	if ! ip link set dev lo xdp off > /dev/null 2>&1; then
	62	echo "selftests: $TESTNAME [SKIP] Could not run test without the ip xdp support"
	63	exit $ksft_skip
	64	fi
	65
	66	if [ -z "$BPF_FS" ]; then
	67	echo "selftests: $TESTNAME [SKIP] Could not run test without bpffs mounted"
	68	exit $ksft_skip
	69	fi
	70
	71	if ! bpftool version > /dev/null 2>&1; then
	72	echo "selftests: $TESTNAME [SKIP] Could not run test without bpftool"
	73	exit $ksft_skip
	74	fi
	75
	76	set -e
	77
	78	trap cleanup_skip EXIT
	79
	80	ip netns add ns1
	81	ip netns add ns2
	82	ip netns add ns3
	83
	84	ip link add veth1 index 111 type veth peer name veth11 netns ns1
	85	ip link add veth2 index 122 type veth peer name veth22 netns ns2
	86	ip link add veth3 index 133 type veth peer name veth33 netns ns3
	87
	88	ip link set veth1 up
	89	ip link set veth2 up
	90	ip link set veth3 up
	91
	92	ip -n ns1 addr add 10.1.1.11/24 dev veth11
	93	ip -n ns3 addr add 10.1.1.33/24 dev veth33
	94
	95	ip -n ns1 link set dev veth11 up
	96	ip -n ns2 link set dev veth22 up
	97	ip -n ns3 link set dev veth33 up
	98
	99	mkdir $BPF_DIR
	100	bpftool prog loadall \
	101	xdp_redirect_map.o $BPF_DIR/progs type xdp \
	102	pinmaps $BPF_DIR/maps
	103	bpftool map update pinned $BPF_DIR/maps/tx_port key 0 0 0 0 value 122 0 0 0
	104	bpftool map update pinned $BPF_DIR/maps/tx_port key 1 0 0 0 value 133 0 0 0
	105	bpftool map update pinned $BPF_DIR/maps/tx_port key 2 0 0 0 value 111 0 0 0
	106	ip link set dev veth1 xdp pinned $BPF_DIR/progs/redirect_map_0
	107	ip link set dev veth2 xdp pinned $BPF_DIR/progs/redirect_map_1
	108	ip link set dev veth3 xdp pinned $BPF_DIR/progs/redirect_map_2
	109
	110	ip -n ns1 link set dev veth11 xdp obj xdp_dummy.o sec xdp_dummy
	111	ip -n ns2 link set dev veth22 xdp obj xdp_tx.o sec tx
	112	ip -n ns3 link set dev veth33 xdp obj xdp_dummy.o sec xdp_dummy
	113
	114	trap cleanup EXIT
	115
	116	ip netns exec ns1 ping -c 1 -W 1 10.1.1.33
	117
	118	exit 0