Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf

Daniel Borkmann says: ==================== pull-request: bpf 2019-12-19 The following pull-request contains BPF updates for your *net* tree. We've added 10 non-merge commits during the last 8 day(s) which contain a total of 21 files changed, 269 insertions(+), 108 deletions(-). The main changes are: 1) Fix lack of synchronization between xsk wakeup and destroying resources used by xsk wakeup, from Maxim Mikityanskiy. 2) Fix pruning with tail call patching, untrack programs in case of verifier error and fix a cgroup local storage tracking bug, from Daniel Borkmann. 3) Fix clearing skb->tstamp in bpf_redirect() when going from ingress to egress which otherwise cause issues e.g. on fq qdisc, from Lorenz Bauer. 4) Fix compile warning of unused proc_dointvec_minmax_bpf_restricted() when only cBPF is present, from Alexander Lobakin. ====================

David S. Miller committed 6 years ago

Commit 0fd26002 parents 6fa9a11 3123d8021 files · +269 −108Browse files

Unified · Split

21 files

drivers/net/ethernet/intel/i40e/i40e.h +1 −1
drivers/net/ethernet/intel/i40e/i40e_main.c +7 −3
drivers/net/ethernet/intel/i40e/i40e_xsk.c +4 −0
drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +6 −1
drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c +6 −2
drivers/net/ethernet/mellanox/mlx5/core/en.h +1 −1
drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h +9 −13
drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c +1 −0
drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c +1 −1
drivers/net/ethernet/mellanox/mlx5/core/en_main.c +1 −18
include/linux/bpf-cgroup.h +4 −4
include/linux/bpf.h +2 −0
kernel/bpf/core.c +11 −6
kernel/bpf/local_storage.c +12 −12
kernel/bpf/verifier.c +10 −14
net/core/filter.c +1 −0
net/core/sysctl_net_core.c +2 −0
net/xdp/xsk.c +14 −8
tools/testing/selftests/bpf/test_verifier.c +22 −21
tools/testing/selftests/bpf/verifier/ref_tracking.c +3 −3
tools/testing/selftests/bpf/verifier/runtime_jit.c +151 −0

drivers/net/ethernet/intel/i40e/i40e.h+1−1

@@ -1152,7 +1152,7 @@ void i40e_set_fec_in_flags(u8 fec_cfg, u32 *flags); · 1151 unmodified lines
1152	1152
1153	1153	static inline bool i40e_enabled_xdp_vsi(struct i40e_vsi *vsi)
1154	1154	{
1155		− return !!vsi->xdp_prog;
	1155	+ return !!READ_ONCE(vsi->xdp_prog);
1156	1156	}
1157	1157
1158	1158	int i40e_create_queue_channel(struct i40e_vsi vsi, struct i40e_channel ch);

drivers/net/ethernet/intel/i40e/i40e_main.c+7−3

@@ -6823,8 +6823,8 @@ void i40e_down(struct i40e_vsi *vsi) · 6822 unmodified lines
6823	6823	for (i = 0; i < vsi->num_queue_pairs; i++) {
6824	6824	i40e_clean_tx_ring(vsi->tx_rings[i]);
6825	6825	if (i40e_enabled_xdp_vsi(vsi)) {
6826		− /* Make sure that in-progress ndo_xdp_xmit
6827		− * calls are completed.
	6826	+ /* Make sure that in-progress ndo_xdp_xmit and
	6827	+ * ndo_xsk_wakeup calls are completed.
6828	6828	*/
6829	6829	synchronize_rcu();
6830	6830	i40e_clean_tx_ring(vsi->xdp_rings[i]);
@@ -12546,8 +12546,12 @@ static int i40e_xdp_setup(struct i40e_vsi *vsi, · 5715 unmodified lines
12546	12546
12547	12547	old_prog = xchg(&vsi->xdp_prog, prog);
12548	12548
12549		− if (need_reset)
	12549	+ if (need_reset) {
	12550	+ if (!prog)
	12551	+ /* Wait until ndo_xsk_wakeup completes. */
	12552	+ synchronize_rcu();
12550	12553	i40e_reset_and_rebuild(pf, true, true);
	12554	+ }
12551	12555
12552	12556	for (i = 0; i < vsi->num_queue_pairs; i++)
12553	12557	WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);

drivers/net/ethernet/intel/i40e/i40e_xsk.c+4−0

@@ -787,8 +787,12 @@ int i40e_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) · 786 unmodified lines
787	787	{
788	788	struct i40e_netdev_priv *np = netdev_priv(dev);
789	789	struct i40e_vsi *vsi = np->vsi;
	790	+ struct i40e_pf *pf = vsi->back;
790	791	struct i40e_ring *ring;
791	792
	793	+ if (test_bit(__I40E_CONFIG_BUSY, pf->state))
	794	+ return -ENETDOWN;
	795	+
792	796	if (test_bit(__I40E_VSI_DOWN, vsi->state))
793	797	return -ENETDOWN;
794	798

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c+6−1

@@ -10261,7 +10261,12 @@ static int ixgbe_xdp_setup(struct net_device dev, struct bpf_prog prog) · 10260 unmodified lines
10261	10261
10262	10262	/* If transitioning XDP modes reconfigure rings */
10263	10263	if (need_reset) {
10264		− int err = ixgbe_setup_tc(dev, adapter->hw_tcs);
	10264	+ int err;
	10265	+
	10266	+ if (!prog)
	10267	+ /* Wait until ndo_xsk_wakeup completes. */
	10268	+ synchronize_rcu();
	10269	+ err = ixgbe_setup_tc(dev, adapter->hw_tcs);
10265	10270
10266	10271	if (err) {
10267	10272	rcu_assign_pointer(adapter->xdp_prog, old_prog);

drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c+6−2

@@ -709,10 +709,14 @@ int ixgbe_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) · 708 unmodified lines
709	709	if (qid >= adapter->num_xdp_queues)
710	710	return -ENXIO;
711	711
712		− if (!adapter->xdp_ring[qid]->xsk_umem)
	712	+ ring = adapter->xdp_ring[qid];
	713	+
	714	+ if (test_bit(__IXGBE_TX_DISABLED, &ring->state))
	715	+ return -ENETDOWN;
	716	+
	717	+ if (!ring->xsk_umem)
713	718	return -ENXIO;
714	719
715		− ring = adapter->xdp_ring[qid];
716	720	if (!napi_if_scheduled_mark_missed(&ring->q_vector->napi)) {
717	721	u64 eics = BIT_ULL(ring->q_vector->v_idx);
718	722

drivers/net/ethernet/mellanox/mlx5/core/en.h+1−1

@@ -760,7 +760,7 @@ enum { · 759 unmodified lines
760	760	MLX5E_STATE_OPENED,
761	761	MLX5E_STATE_DESTROYING,
762	762	MLX5E_STATE_XDP_TX_ENABLED,
763		− MLX5E_STATE_XDP_OPEN,
	763	+ MLX5E_STATE_XDP_ACTIVE,
764	764	};
765	765
766	766	struct mlx5e_rqt {

drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h+9−13

@@ -75,12 +75,18 @@ int mlx5e_xdp_xmit(struct net_device dev, int n, struct xdp_frame *frames, · 74 unmodified lines
75	75	static inline void mlx5e_xdp_tx_enable(struct mlx5e_priv *priv)
76	76	{
77	77	set_bit(MLX5E_STATE_XDP_TX_ENABLED, &priv->state);
	78	+
	79	+ if (priv->channels.params.xdp_prog)
	80	+ set_bit(MLX5E_STATE_XDP_ACTIVE, &priv->state);
78	81	}
79	82
80	83	static inline void mlx5e_xdp_tx_disable(struct mlx5e_priv *priv)
81	84	{
	85	+ if (priv->channels.params.xdp_prog)
	86	+ clear_bit(MLX5E_STATE_XDP_ACTIVE, &priv->state);
	87	+
82	88	clear_bit(MLX5E_STATE_XDP_TX_ENABLED, &priv->state);
83		− /* let other device's napi(s) see our new state */
	89	+ /* Let other device's napi(s) and XSK wakeups see our new state. */
84	90	synchronize_rcu();
85	91	}
86	92
@@ -89,19 +95,9 @@ static inline bool mlx5e_xdp_tx_is_enabled(struct mlx5e_priv *priv) · 2 unmodified lines
89	95	return test_bit(MLX5E_STATE_XDP_TX_ENABLED, &priv->state);
90	96	}
91	97
92		−static inline void mlx5e_xdp_set_open(struct mlx5e_priv *priv)
93		−{
94		− set_bit(MLX5E_STATE_XDP_OPEN, &priv->state);
95		−}
96		−
97		−static inline void mlx5e_xdp_set_closed(struct mlx5e_priv *priv)
98		−{
99		− clear_bit(MLX5E_STATE_XDP_OPEN, &priv->state);
100		−}
101		−
102		−static inline bool mlx5e_xdp_is_open(struct mlx5e_priv *priv)
	98	+static inline bool mlx5e_xdp_is_active(struct mlx5e_priv *priv)
103	99	{
104		− return test_bit(MLX5E_STATE_XDP_OPEN, &priv->state);
	100	+ return test_bit(MLX5E_STATE_XDP_ACTIVE, &priv->state);
105	101	}
106	102
107	103	static inline void mlx5e_xmit_xdp_doorbell(struct mlx5e_xdpsq *sq)

drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c+1−0

@@ -144,6 +144,7 @@ void mlx5e_close_xsk(struct mlx5e_channel *c) · 143 unmodified lines
144	144	{
145	145	clear_bit(MLX5E_CHANNEL_STATE_XSK, c->state);
146	146	napi_synchronize(&c->napi);
	147	+ synchronize_rcu(); /* Sync with the XSK wakeup. */
147	148
148	149	mlx5e_close_rq(&c->xskrq);
149	150	mlx5e_close_cq(&c->xskrq.cq);

drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c+1−1

@@ -14,7 +14,7 @@ int mlx5e_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) · 13 unmodified lines
14	14	struct mlx5e_channel *c;
15	15	u16 ix;
16	16
17		− if (unlikely(!mlx5e_xdp_is_open(priv)))
	17	+ if (unlikely(!mlx5e_xdp_is_active(priv)))
18	18	return -ENETDOWN;
19	19
20	20	if (unlikely(!mlx5e_qid_get_ch_if_in_group(params, qid, MLX5E_RQ_GROUP_XSK, &ix)))

drivers/net/ethernet/mellanox/mlx5/core/en_main.c+1−18

@@ -3000,12 +3000,9 @@ void mlx5e_timestamp_init(struct mlx5e_priv *priv) · 2999 unmodified lines
3000	3000	int mlx5e_open_locked(struct net_device *netdev)
3001	3001	{
3002	3002	struct mlx5e_priv *priv = netdev_priv(netdev);
3003		− bool is_xdp = priv->channels.params.xdp_prog;
3004	3003	int err;
3005	3004
3006	3005	set_bit(MLX5E_STATE_OPENED, &priv->state);
3007		− if (is_xdp)
3008		− mlx5e_xdp_set_open(priv);
3009	3006
3010	3007	err = mlx5e_open_channels(priv, &priv->channels);
3011	3008	if (err)
@@ -3020,8 +3017,6 @@ int mlx5e_open_locked(struct net_device *netdev) · 8 unmodified lines
3020	3017	return 0;
3021	3018
3022	3019	err_clear_state_opened_flag:
3023		− if (is_xdp)
3024		− mlx5e_xdp_set_closed(priv);
3025	3020	clear_bit(MLX5E_STATE_OPENED, &priv->state);
3026	3021	return err;
3027	3022	}
@@ -3053,8 +3048,6 @@ int mlx5e_close_locked(struct net_device *netdev) · 25 unmodified lines
3053	3048	if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
3054	3049	return 0;
3055	3050
3056		− if (priv->channels.params.xdp_prog)
3057		− mlx5e_xdp_set_closed(priv);
3058	3051	clear_bit(MLX5E_STATE_OPENED, &priv->state);
3059	3052
3060	3053	netif_carrier_off(priv->netdev);
@@ -4371,16 +4364,6 @@ static int mlx5e_xdp_allowed(struct mlx5e_priv priv, struct bpf_prog prog) · 1310 unmodified lines
4371	4364	return 0;
4372	4365	}
4373	4366
4374		−static int mlx5e_xdp_update_state(struct mlx5e_priv *priv)
4375		−{
4376		− if (priv->channels.params.xdp_prog)
4377		− mlx5e_xdp_set_open(priv);
4378		− else
4379		− mlx5e_xdp_set_closed(priv);
4380		−
4381		− return 0;
4382		−}
4383		−
4384	4367	static int mlx5e_xdp_set(struct net_device netdev, struct bpf_prog prog)
4385	4368	{
4386	4369	struct mlx5e_priv *priv = netdev_priv(netdev);
@@ -4415,7 +4398,7 @@ static int mlx5e_xdp_set(struct net_device netdev, struct bpf_prog prog) · 28 unmodified lines
4415	4398	mlx5e_set_rq_type(priv->mdev, &new_channels.params);
4416	4399	old_prog = priv->channels.params.xdp_prog;
4417	4400
4418		− err = mlx5e_safe_switch_channels(priv, &new_channels, mlx5e_xdp_update_state);
	4401	+ err = mlx5e_safe_switch_channels(priv, &new_channels, NULL);
4419	4402	if (err)
4420	4403	goto unlock;
4421	4404	} else {

include/linux/bpf-cgroup.h+4−4

@@ -157,8 +157,8 @@ void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage, · 156 unmodified lines
157	157	struct cgroup *cgroup,
158	158	enum bpf_attach_type type);
159	159	void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage);
160		−int bpf_cgroup_storage_assign(struct bpf_prog prog, struct bpf_map map);
161		−void bpf_cgroup_storage_release(struct bpf_prog prog, struct bpf_map map);
	160	+int bpf_cgroup_storage_assign(struct bpf_prog_aux aux, struct bpf_map map);
	161	+void bpf_cgroup_storage_release(struct bpf_prog_aux aux, struct bpf_map map);
162	162
163	163	int bpf_percpu_cgroup_storage_copy(struct bpf_map map, void key, void *value);
164	164	int bpf_percpu_cgroup_storage_update(struct bpf_map map, void key,
@@ -360,9 +360,9 @@ static inline int cgroup_bpf_prog_query(const union bpf_attr *attr, · 195 unmodified lines
360	360
361	361	static inline void bpf_cgroup_storage_set(
362	362	struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE]) {}
363		−static inline int bpf_cgroup_storage_assign(struct bpf_prog *prog,
	363	+static inline int bpf_cgroup_storage_assign(struct bpf_prog_aux *aux,
364	364	struct bpf_map *map) { return 0; }
365		−static inline void bpf_cgroup_storage_release(struct bpf_prog *prog,
	365	+static inline void bpf_cgroup_storage_release(struct bpf_prog_aux *aux,
366	366	struct bpf_map *map) {}
367	367	static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(
368	368	struct bpf_prog *prog, enum bpf_cgroup_storage_type stype) { return NULL; }

include/linux/bpf.h+2−0

@@ -818,6 +818,8 @@ struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog); · 817 unmodified lines
818	818	void bpf_prog_put(struct bpf_prog *prog);
819	819	int __bpf_prog_charge(struct user_struct *user, u32 pages);
820	820	void __bpf_prog_uncharge(struct user_struct *user, u32 pages);
	821	+void __bpf_free_used_maps(struct bpf_prog_aux *aux,
	822	+ struct bpf_map **used_maps, u32 len);
821	823
822	824	void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock);
823	825	void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock);

kernel/bpf/core.c+11−6

@@ -2043,23 +2043,28 @@ static void bpf_free_cgroup_storage(struct bpf_prog_aux *aux) · 2042 unmodified lines
2043	2043	for_each_cgroup_storage_type(stype) {
2044	2044	if (!aux->cgroup_storage[stype])
2045	2045	continue;
2046		− bpf_cgroup_storage_release(aux->prog,
2047		− aux->cgroup_storage[stype]);
	2046	+ bpf_cgroup_storage_release(aux, aux->cgroup_storage[stype]);
2048	2047	}
2049	2048	}
2050	2049
2051		−static void bpf_free_used_maps(struct bpf_prog_aux *aux)
	2050	+void __bpf_free_used_maps(struct bpf_prog_aux *aux,
	2051	+ struct bpf_map **used_maps, u32 len)
2052	2052	{
2053	2053	struct bpf_map *map;
2054		− int i;
	2054	+ u32 i;
2055	2055
2056	2056	bpf_free_cgroup_storage(aux);
2057		− for (i = 0; i < aux->used_map_cnt; i++) {
2058		− map = aux->used_maps[i];
	2057	+ for (i = 0; i < len; i++) {
	2058	+ map = used_maps[i];
2059	2059	if (map->ops->map_poke_untrack)
2060	2060	map->ops->map_poke_untrack(map, aux);
2061	2061	bpf_map_put(map);
2062	2062	}
	2063	+}
	2064	+
	2065	+static void bpf_free_used_maps(struct bpf_prog_aux *aux)
	2066	+{
	2067	+ __bpf_free_used_maps(aux, aux->used_maps, aux->used_map_cnt);
2063	2068	kfree(aux->used_maps);
2064	2069	}
2065	2070

kernel/bpf/local_storage.c+12−12

@@ -20,7 +20,7 @@ struct bpf_cgroup_storage_map { · 19 unmodified lines
20	20	struct bpf_map map;
21	21
22	22	spinlock_t lock;
23		− struct bpf_prog *prog;
	23	+ struct bpf_prog_aux *aux;
24	24	struct rb_root root;
25	25	struct list_head list;
26	26	};
@@ -420,7 +420,7 @@ const struct bpf_map_ops cgroup_storage_map_ops = { · 393 unmodified lines
420	420	.map_seq_show_elem = cgroup_storage_seq_show_elem,
421	421	};
422	422
423		−int bpf_cgroup_storage_assign(struct bpf_prog prog, struct bpf_map _map)
	423	+int bpf_cgroup_storage_assign(struct bpf_prog_aux aux, struct bpf_map _map)
424	424	{
425	425	enum bpf_cgroup_storage_type stype = cgroup_storage_type(_map);
426	426	struct bpf_cgroup_storage_map *map = map_to_storage(_map);
@@ -428,14 +428,14 @@ int bpf_cgroup_storage_assign(struct bpf_prog prog, struct bpf_map _map) · 1 unmodified line
428	428
429	429	spin_lock_bh(&map->lock);
430	430
431		− if (map->prog && map->prog != prog)
	431	+ if (map->aux && map->aux != aux)
432	432	goto unlock;
433		− if (prog->aux->cgroup_storage[stype] &&
434		− prog->aux->cgroup_storage[stype] != _map)
	433	+ if (aux->cgroup_storage[stype] &&
	434	+ aux->cgroup_storage[stype] != _map)
435	435	goto unlock;
436	436
437		− map->prog = prog;
438		− prog->aux->cgroup_storage[stype] = _map;
	437	+ map->aux = aux;
	438	+ aux->cgroup_storage[stype] = _map;
439	439	ret = 0;
440	440	unlock:
441	441	spin_unlock_bh(&map->lock);
@@ -443,16 +443,16 @@ unlock: · 1 unmodified line
443	443	return ret;
444	444	}
445	445
446		−void bpf_cgroup_storage_release(struct bpf_prog prog, struct bpf_map _map)
	446	+void bpf_cgroup_storage_release(struct bpf_prog_aux aux, struct bpf_map _map)
447	447	{
448	448	enum bpf_cgroup_storage_type stype = cgroup_storage_type(_map);
449	449	struct bpf_cgroup_storage_map *map = map_to_storage(_map);
450	450
451	451	spin_lock_bh(&map->lock);
452		− if (map->prog == prog) {
453		− WARN_ON(prog->aux->cgroup_storage[stype] != _map);
454		− map->prog = NULL;
455		− prog->aux->cgroup_storage[stype] = NULL;
	452	+ if (map->aux == aux) {
	453	+ WARN_ON(aux->cgroup_storage[stype] != _map);
	454	+ map->aux = NULL;
	455	+ aux->cgroup_storage[stype] = NULL;
456	456	}
457	457	spin_unlock_bh(&map->lock);
458	458	}

kernel/bpf/verifier.c+10−14

@@ -4134,6 +4134,7 @@ record_func_key(struct bpf_verifier_env env, struct bpf_call_arg_meta meta, · 4133 unmodified lines
4134	4134	struct bpf_map *map = meta->map_ptr;
4135	4135	struct tnum range;
4136	4136	u64 val;
	4137	+ int err;
4137	4138
4138	4139	if (func_id != BPF_FUNC_tail_call)
4139	4140	return 0;
@@ -4150,6 +4151,10 @@ record_func_key(struct bpf_verifier_env env, struct bpf_call_arg_meta meta, · 10 unmodified lines
4150	4151	return 0;
4151	4152	}
4152	4153
	4154	+ err = mark_chain_precision(env, BPF_REG_3);
	4155	+ if (err)
	4156	+ return err;
	4157	+
4153	4158	val = reg->var_off.value;
4154	4159	if (bpf_map_key_unseen(aux))
4155	4160	bpf_map_key_store(aux, val);
@@ -8268,7 +8273,7 @@ static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env) · 4112 unmodified lines
8268	8273	env->used_maps[env->used_map_cnt++] = map;
8269	8274
8270	8275	if (bpf_map_is_cgroup_storage(map) &&
8271		− bpf_cgroup_storage_assign(env->prog, map)) {
	8276	+ bpf_cgroup_storage_assign(env->prog->aux, map)) {
8272	8277	verbose(env, "only one cgroup storage of each type is allowed\n");
8273	8278	fdput(f);
8274	8279	return -EBUSY;
@@ -8298,18 +8303,8 @@ next_insn: · 23 unmodified lines
8298	8303	/* drop refcnt of maps used by the rejected program */
8299	8304	static void release_maps(struct bpf_verifier_env *env)
8300	8305	{
8301		− enum bpf_cgroup_storage_type stype;
8302		− int i;
8303		−
8304		− for_each_cgroup_storage_type(stype) {
8305		− if (!env->prog->aux->cgroup_storage[stype])
8306		− continue;
8307		− bpf_cgroup_storage_release(env->prog,
8308		− env->prog->aux->cgroup_storage[stype]);
8309		− }
8310		−
8311		− for (i = 0; i < env->used_map_cnt; i++)
8312		− bpf_map_put(env->used_maps[i]);
	8306	+ __bpf_free_used_maps(env->prog->aux, env->used_maps,
	8307	+ env->used_map_cnt);
8313	8308	}
8314	8309
8315	8310	/* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */
@@ -9282,7 +9277,8 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env) · 966 unmodified lines
9282	9277	insn->code = BPF_JMP \| BPF_TAIL_CALL;
9283	9278
9284	9279	aux = &env->insn_aux_data[i + delta];
9285		− if (prog->jit_requested && !expect_blinding &&
	9280	+ if (env->allow_ptr_leaks && !expect_blinding &&
	9281	+ prog->jit_requested &&
9286	9282	!bpf_map_key_poisoned(aux) &&
9287	9283	!bpf_map_ptr_poisoned(aux) &&
9288	9284	!bpf_map_ptr_unpriv(aux)) {

net/core/filter.c+1−0

@@ -2055,6 +2055,7 @@ static inline int __bpf_tx_skb(struct net_device dev, struct sk_buff skb) · 2054 unmodified lines
2055	2055	}
2056	2056
2057	2057	skb->dev = dev;
	2058	+ skb->tstamp = 0;
2058	2059
2059	2060	dev_xmit_recursion_inc();
2060	2061	ret = dev_queue_xmit(skb);

net/core/sysctl_net_core.c+2−0

@@ -288,6 +288,7 @@ static int proc_dointvec_minmax_bpf_enable(struct ctl_table *table, int write, · 287 unmodified lines
288	288	return ret;
289	289	}
290	290
	291	+# ifdef CONFIG_HAVE_EBPF_JIT
291	292	static int
292	293	proc_dointvec_minmax_bpf_restricted(struct ctl_table *table, int write,
293	294	void __user buffer, size_t lenp,
@@ -298,6 +299,7 @@ proc_dointvec_minmax_bpf_restricted(struct ctl_table *table, int write, · 4 unmodified lines
298	299
299	300	return proc_dointvec_minmax(table, write, buffer, lenp, ppos);
300	301	}
	302	+# endif /* CONFIG_HAVE_EBPF_JIT */
301	303
302	304	static int
303	305	proc_dolongvec_minmax_bpf_restricted(struct ctl_table *table, int write,

net/xdp/xsk.c+14−8

@@ -334,12 +334,21 @@ out: · 333 unmodified lines
334	334	}
335	335	EXPORT_SYMBOL(xsk_umem_consume_tx);
336	336
337		−static int xsk_zc_xmit(struct xdp_sock *xs)
	337	+static int xsk_wakeup(struct xdp_sock *xs, u8 flags)
338	338	{
339	339	struct net_device *dev = xs->dev;
	340	+ int err;
	341	+
	342	+ rcu_read_lock();
	343	+ err = dev->netdev_ops->ndo_xsk_wakeup(dev, xs->queue_id, flags);
	344	+ rcu_read_unlock();
	345	+
	346	+ return err;
	347	+}
340	348
341		− return dev->netdev_ops->ndo_xsk_wakeup(dev, xs->queue_id,
342		− XDP_WAKEUP_TX);
	349	+static int xsk_zc_xmit(struct xdp_sock *xs)
	350	+{
	351	+ return xsk_wakeup(xs, XDP_WAKEUP_TX);
343	352	}
344	353
345	354	static void xsk_destruct_skb(struct sk_buff *skb)
@@ -453,19 +462,16 @@ static __poll_t xsk_poll(struct file file, struct socket sock, · 107 unmodified lines
453	462	__poll_t mask = datagram_poll(file, sock, wait);
454	463	struct sock *sk = sock->sk;
455	464	struct xdp_sock *xs = xdp_sk(sk);
456		− struct net_device *dev;
457	465	struct xdp_umem *umem;
458	466
459	467	if (unlikely(!xsk_is_bound(xs)))
460	468	return mask;
461	469
462		− dev = xs->dev;
463	470	umem = xs->umem;
464	471
465	472	if (umem->need_wakeup) {
466		− if (dev->netdev_ops->ndo_xsk_wakeup)
467		− dev->netdev_ops->ndo_xsk_wakeup(dev, xs->queue_id,
468		− umem->need_wakeup);
	473	+ if (xs->zc)
	474	+ xsk_wakeup(xs, umem->need_wakeup);
469	475	else
470	476	/* Poll needs to drive Tx also in copy mode */
471	477	__xsk_sendmsg(sk);

tools/testing/selftests/bpf/test_verifier.c+22−21

@@ -408,10 +408,10 @@ static void update_map(int fd, int index) · 407 unmodified lines
408	408	assert(!bpf_map_update_elem(fd, &index, &value, 0));
409	409	}
410	410
411		−static int create_prog_dummy1(enum bpf_prog_type prog_type)
	411	+static int create_prog_dummy_simple(enum bpf_prog_type prog_type, int ret)
412	412	{
413	413	struct bpf_insn prog[] = {
414		− BPF_MOV64_IMM(BPF_REG_0, 42),
	414	+ BPF_MOV64_IMM(BPF_REG_0, ret),
415	415	BPF_EXIT_INSN(),
416	416	};
417	417
@@ -419,14 +419,15 @@ static int create_prog_dummy1(enum bpf_prog_type prog_type) · 1 unmodified line
419	419	ARRAY_SIZE(prog), "GPL", 0, NULL, 0);
420	420	}
421	421
422		−static int create_prog_dummy2(enum bpf_prog_type prog_type, int mfd, int idx)
	422	+static int create_prog_dummy_loop(enum bpf_prog_type prog_type, int mfd,
	423	+ int idx, int ret)
423	424	{
424	425	struct bpf_insn prog[] = {
425	426	BPF_MOV64_IMM(BPF_REG_3, idx),
426	427	BPF_LD_MAP_FD(BPF_REG_2, mfd),
427	428	BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 0, 0,
428	429	BPF_FUNC_tail_call),
429		− BPF_MOV64_IMM(BPF_REG_0, 41),
	430	+ BPF_MOV64_IMM(BPF_REG_0, ret),
430	431	BPF_EXIT_INSN(),
431	432	};
432	433
@@ -435,10 +436,9 @@ static int create_prog_dummy2(enum bpf_prog_type prog_type, int mfd, int idx) · 2 unmodified lines
435	436	}
436	437
437	438	static int create_prog_array(enum bpf_prog_type prog_type, uint32_t max_elem,
438		− int p1key)
	439	+ int p1key, int p2key, int p3key)
439	440	{
440		− int p2key = 1;
441		− int mfd, p1fd, p2fd;
	441	+ int mfd, p1fd, p2fd, p3fd;
442	442
443	443	mfd = bpf_create_map(BPF_MAP_TYPE_PROG_ARRAY, sizeof(int),
444	444	sizeof(int), max_elem, 0);
@@ -449,23 +449,24 @@ static int create_prog_array(enum bpf_prog_type prog_type, uint32_t max_elem, · 4 unmodified lines
449	449	return -1;
450	450	}
451	451
452		− p1fd = create_prog_dummy1(prog_type);
453		− p2fd = create_prog_dummy2(prog_type, mfd, p2key);
454		− if (p1fd < 0 \|\| p2fd < 0)
455		− goto out;
	452	+ p1fd = create_prog_dummy_simple(prog_type, 42);
	453	+ p2fd = create_prog_dummy_loop(prog_type, mfd, p2key, 41);
	454	+ p3fd = create_prog_dummy_simple(prog_type, 24);
	455	+ if (p1fd < 0 \|\| p2fd < 0 \|\| p3fd < 0)
	456	+ goto err;
456	457	if (bpf_map_update_elem(mfd, &p1key, &p1fd, BPF_ANY) < 0)
457		− goto out;
	458	+ goto err;
458	459	if (bpf_map_update_elem(mfd, &p2key, &p2fd, BPF_ANY) < 0)
459		− goto out;
	460	+ goto err;
	461	+ if (bpf_map_update_elem(mfd, &p3key, &p3fd, BPF_ANY) < 0) {
	462	+err:
	463	+ close(mfd);
	464	+ mfd = -1;
	465	+ }
	466	+ close(p3fd);
460	467	close(p2fd);
461	468	close(p1fd);
462		−
463	469	return mfd;
464		−out:
465		− close(p2fd);
466		− close(p1fd);
467		− close(mfd);
468		− return -1;
469	470	}
470	471
471	472	static int create_map_in_map(void)
@@ -684,7 +685,7 @@ static void do_test_fixup(struct bpf_test *test, enum bpf_prog_type prog_type, · 212 unmodified lines
684	685	}
685	686
686	687	if (*fixup_prog1) {
687		− map_fds[4] = create_prog_array(prog_type, 4, 0);
	688	+ map_fds[4] = create_prog_array(prog_type, 4, 0, 1, 2);
688	689	do {
689	690	prog[*fixup_prog1].imm = map_fds[4];
690	691	fixup_prog1++;
@@ -692,7 +693,7 @@ static void do_test_fixup(struct bpf_test *test, enum bpf_prog_type prog_type, · 1 unmodified line
692	693	}
693	694
694	695	if (*fixup_prog2) {
695		− map_fds[5] = create_prog_array(prog_type, 8, 7);
	696	+ map_fds[5] = create_prog_array(prog_type, 8, 7, 1, 2);
696	697	do {
697	698	prog[*fixup_prog2].imm = map_fds[5];
698	699	fixup_prog2++;

tools/testing/selftests/bpf/verifier/ref_tracking.c+3−3

@@ -455,7 +455,7 @@ · 454 unmodified lines
455	455	BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
456	456	BPF_JMP_IMM(BPF_JNE, BPF_REG_1, 0, 7),
457	457	/* bpf_tail_call() */
458		− BPF_MOV64_IMM(BPF_REG_3, 2),
	458	+ BPF_MOV64_IMM(BPF_REG_3, 3),
459	459	BPF_LD_MAP_FD(BPF_REG_2, 0),
460	460	BPF_MOV64_REG(BPF_REG_1, BPF_REG_7),
461	461	BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 0, 0, BPF_FUNC_tail_call),
@@ -478,7 +478,7 @@ · 16 unmodified lines
478	478	BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 1),
479	479	BPF_EMIT_CALL(BPF_FUNC_sk_release),
480	480	/* bpf_tail_call() */
481		− BPF_MOV64_IMM(BPF_REG_3, 2),
	481	+ BPF_MOV64_IMM(BPF_REG_3, 3),
482	482	BPF_LD_MAP_FD(BPF_REG_2, 0),
483	483	BPF_MOV64_REG(BPF_REG_1, BPF_REG_7),
484	484	BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 0, 0, BPF_FUNC_tail_call),
@@ -497,7 +497,7 @@ · 12 unmodified lines
497	497	BPF_SK_LOOKUP(sk_lookup_tcp),
498	498	/* bpf_tail_call() */
499	499	BPF_MOV64_REG(BPF_REG_6, BPF_REG_0),
500		− BPF_MOV64_IMM(BPF_REG_3, 2),
	500	+ BPF_MOV64_IMM(BPF_REG_3, 3),
501	501	BPF_LD_MAP_FD(BPF_REG_2, 0),
502	502	BPF_MOV64_REG(BPF_REG_1, BPF_REG_7),
503	503	BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 0, 0, BPF_FUNC_tail_call),

tools/testing/selftests/bpf/verifier/runtime_jit.c+151−0

@@ -27,6 +27,19 @@ · 26 unmodified lines
27	27	{
28	28	"runtime/jit: tail_call within bounds, no prog",
29	29	.insns = {
	30	+ BPF_MOV64_IMM(BPF_REG_3, 3),
	31	+ BPF_LD_MAP_FD(BPF_REG_2, 0),
	32	+ BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 0, 0, BPF_FUNC_tail_call),
	33	+ BPF_MOV64_IMM(BPF_REG_0, 1),
	34	+ BPF_EXIT_INSN(),
	35	+ },
	36	+ .fixup_prog1 = { 1 },
	37	+ .result = ACCEPT,
	38	+ .retval = 1,
	39	+},
	40	+{
	41	+ "runtime/jit: tail_call within bounds, key 2",
	42	+ .insns = {
30	43	BPF_MOV64_IMM(BPF_REG_3, 2),
31	44	BPF_LD_MAP_FD(BPF_REG_2, 0),
32	45	BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 0, 0, BPF_FUNC_tail_call),
@@ -35,8 +48,146 @@ · 2 unmodified lines
35	48	},
36	49	.fixup_prog1 = { 1 },
37	50	.result = ACCEPT,
	51	+ .retval = 24,
	52	+},
	53	+{
	54	+ "runtime/jit: tail_call within bounds, key 2 / key 2, first branch",
	55	+ .insns = {
	56	+ BPF_MOV64_IMM(BPF_REG_0, 13),
	57	+ BPF_STX_MEM(BPF_B, BPF_REG_1, BPF_REG_0,
	58	+ offsetof(struct __sk_buff, cb[0])),
	59	+ BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_1,
	60	+ offsetof(struct __sk_buff, cb[0])),
	61	+ BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 13, 4),
	62	+ BPF_MOV64_IMM(BPF_REG_3, 2),
	63	+ BPF_LD_MAP_FD(BPF_REG_2, 0),
	64	+ BPF_JMP_IMM(BPF_JA, 0, 0, 3),
	65	+ BPF_MOV64_IMM(BPF_REG_3, 2),
	66	+ BPF_LD_MAP_FD(BPF_REG_2, 0),
	67	+ BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 0, 0, BPF_FUNC_tail_call),
	68	+ BPF_MOV64_IMM(BPF_REG_0, 1),
	69	+ BPF_EXIT_INSN(),
	70	+ },
	71	+ .fixup_prog1 = { 5, 9 },
	72	+ .result = ACCEPT,
	73	+ .retval = 24,
	74	+},
	75	+{
	76	+ "runtime/jit: tail_call within bounds, key 2 / key 2, second branch",
	77	+ .insns = {
	78	+ BPF_MOV64_IMM(BPF_REG_0, 14),
	79	+ BPF_STX_MEM(BPF_B, BPF_REG_1, BPF_REG_0,
	80	+ offsetof(struct __sk_buff, cb[0])),
	81	+ BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_1,
	82	+ offsetof(struct __sk_buff, cb[0])),
	83	+ BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 13, 4),
	84	+ BPF_MOV64_IMM(BPF_REG_3, 2),
	85	+ BPF_LD_MAP_FD(BPF_REG_2, 0),
	86	+ BPF_JMP_IMM(BPF_JA, 0, 0, 3),
	87	+ BPF_MOV64_IMM(BPF_REG_3, 2),
	88	+ BPF_LD_MAP_FD(BPF_REG_2, 0),
	89	+ BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 0, 0, BPF_FUNC_tail_call),
	90	+ BPF_MOV64_IMM(BPF_REG_0, 1),
	91	+ BPF_EXIT_INSN(),
	92	+ },
	93	+ .fixup_prog1 = { 5, 9 },
	94	+ .result = ACCEPT,
	95	+ .retval = 24,
	96	+},
	97	+{
	98	+ "runtime/jit: tail_call within bounds, key 0 / key 2, first branch",
	99	+ .insns = {
	100	+ BPF_MOV64_IMM(BPF_REG_0, 13),
	101	+ BPF_STX_MEM(BPF_B, BPF_REG_1, BPF_REG_0,
	102	+ offsetof(struct __sk_buff, cb[0])),
	103	+ BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_1,
	104	+ offsetof(struct __sk_buff, cb[0])),
	105	+ BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 13, 4),
	106	+ BPF_MOV64_IMM(BPF_REG_3, 0),
	107	+ BPF_LD_MAP_FD(BPF_REG_2, 0),
	108	+ BPF_JMP_IMM(BPF_JA, 0, 0, 3),
	109	+ BPF_MOV64_IMM(BPF_REG_3, 2),
	110	+ BPF_LD_MAP_FD(BPF_REG_2, 0),
	111	+ BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 0, 0, BPF_FUNC_tail_call),
	112	+ BPF_MOV64_IMM(BPF_REG_0, 1),
	113	+ BPF_EXIT_INSN(),
	114	+ },
	115	+ .fixup_prog1 = { 5, 9 },
	116	+ .result = ACCEPT,
	117	+ .retval = 24,
	118	+},
	119	+{
	120	+ "runtime/jit: tail_call within bounds, key 0 / key 2, second branch",
	121	+ .insns = {
	122	+ BPF_MOV64_IMM(BPF_REG_0, 14),
	123	+ BPF_STX_MEM(BPF_B, BPF_REG_1, BPF_REG_0,
	124	+ offsetof(struct __sk_buff, cb[0])),
	125	+ BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_1,
	126	+ offsetof(struct __sk_buff, cb[0])),
	127	+ BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 13, 4),
	128	+ BPF_MOV64_IMM(BPF_REG_3, 0),
	129	+ BPF_LD_MAP_FD(BPF_REG_2, 0),
	130	+ BPF_JMP_IMM(BPF_JA, 0, 0, 3),
	131	+ BPF_MOV64_IMM(BPF_REG_3, 2),
	132	+ BPF_LD_MAP_FD(BPF_REG_2, 0),
	133	+ BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 0, 0, BPF_FUNC_tail_call),
	134	+ BPF_MOV64_IMM(BPF_REG_0, 1),
	135	+ BPF_EXIT_INSN(),
	136	+ },
	137	+ .fixup_prog1 = { 5, 9 },
	138	+ .result = ACCEPT,
	139	+ .retval = 42,
	140	+},
	141	+{
	142	+ "runtime/jit: tail_call within bounds, different maps, first branch",
	143	+ .insns = {
	144	+ BPF_MOV64_IMM(BPF_REG_0, 13),
	145	+ BPF_STX_MEM(BPF_B, BPF_REG_1, BPF_REG_0,
	146	+ offsetof(struct __sk_buff, cb[0])),
	147	+ BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_1,
	148	+ offsetof(struct __sk_buff, cb[0])),
	149	+ BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 13, 4),
	150	+ BPF_MOV64_IMM(BPF_REG_3, 0),
	151	+ BPF_LD_MAP_FD(BPF_REG_2, 0),
	152	+ BPF_JMP_IMM(BPF_JA, 0, 0, 3),
	153	+ BPF_MOV64_IMM(BPF_REG_3, 0),
	154	+ BPF_LD_MAP_FD(BPF_REG_2, 0),
	155	+ BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 0, 0, BPF_FUNC_tail_call),
	156	+ BPF_MOV64_IMM(BPF_REG_0, 1),
	157	+ BPF_EXIT_INSN(),
	158	+ },
	159	+ .fixup_prog1 = { 5 },
	160	+ .fixup_prog2 = { 9 },
	161	+ .result_unpriv = REJECT,
	162	+ .errstr_unpriv = "tail_call abusing map_ptr",
	163	+ .result = ACCEPT,
38	164	.retval = 1,
39	165	},
	166	+{
	167	+ "runtime/jit: tail_call within bounds, different maps, second branch",
	168	+ .insns = {
	169	+ BPF_MOV64_IMM(BPF_REG_0, 14),
	170	+ BPF_STX_MEM(BPF_B, BPF_REG_1, BPF_REG_0,
	171	+ offsetof(struct __sk_buff, cb[0])),
	172	+ BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_1,
	173	+ offsetof(struct __sk_buff, cb[0])),
	174	+ BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 13, 4),
	175	+ BPF_MOV64_IMM(BPF_REG_3, 0),
	176	+ BPF_LD_MAP_FD(BPF_REG_2, 0),
	177	+ BPF_JMP_IMM(BPF_JA, 0, 0, 3),
	178	+ BPF_MOV64_IMM(BPF_REG_3, 0),
	179	+ BPF_LD_MAP_FD(BPF_REG_2, 0),
	180	+ BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 0, 0, BPF_FUNC_tail_call),
	181	+ BPF_MOV64_IMM(BPF_REG_0, 1),
	182	+ BPF_EXIT_INSN(),
	183	+ },
	184	+ .fixup_prog1 = { 5 },
	185	+ .fixup_prog2 = { 9 },
	186	+ .result_unpriv = REJECT,
	187	+ .errstr_unpriv = "tail_call abusing map_ptr",
	188	+ .result = ACCEPT,
	189	+ .retval = 42,
	190	+},
40	191	{
41	192	"runtime/jit: tail_call out of bounds",
42	193	.insns = {