return -EOPNOTSUPP; 	if (unlikely(!(dev->xdp_features & NETDEV_XDP_ACT_NDO_XMIT_SG) &&		     xdp_frame_has_frags(xdpf)))		return -EOPNOTSUPP; 	err = xdp_ok_fwd_dev(dev, xdp_get_frame_len(xdpf));	if (unlikely(err))		return err; 	bq_enqueue(dev, xdpf, dev_rx, xdp_prog);	return 0;} static u32 dev_map_bpf_prog_run_skb(struct sk_buff *skb, struct bpf_dtab_netdev *dst){	struct xdp_txq_info txq = { .dev = dst->dev };	struct xdp_buff xdp;	u32 act; 	if (!dst->xdp_prog)		return XDP_PASS; 	__skb_pull(skb, skb->mac_len);	xdp.txq = &txq; 	act = bpf_prog_run_generic_xdp(skb, &xdp, dst->xdp_prog);	switch (act) {	case XDP_PASS:		__skb_push(skb, skb->mac_len);		break;	default:		bpf_warn_invalid_xdp_action(NULL, dst->xdp_prog, act);		fallthrough;	case XDP_ABORTED:		trace_xdp_exception(dst->dev, dst->xdp_prog, act);		fallthrough;	case XDP_DROP:		kfree_skb(skb);		break;	} 	return act;} int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,		    struct net_device *dev_rx){	return __xdp_enqueue(dev, xdpf, dev_rx, NULL);} int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,		    struct net_device *dev_rx){	struct net_device *dev = dst->dev; 	return __xdp_enqueue(dev, xdpf, dev_rx, dst->xdp_prog);} static bool is_valid_dst(struct bpf_dtab_netdev *obj, struct xdp_frame *xdpf){	if (!obj)		return false; 	if (!(obj->dev->xdp_features & NETDEV_XDP_ACT_NDO_XMIT))		return false; 	if (unlikely(!(obj->dev->xdp_features & NETDEV_XDP_ACT_NDO_XMIT_SG) &&		     xdp_frame_has_frags(xdpf)))		return false; 	if (xdp_ok_fwd_dev(obj->dev, xdp_get_frame_len(xdpf)))		return false; 	return true;} static int dev_map_enqueue_clone(struct bpf_dtab_netdev *obj,				 struct net_device *dev_rx,				 struct xdp_frame *xdpf){	struct xdp_frame *nxdpf; 	/* Frags live outside the linear frame and cannot be cloned safely. */	if (unlikely(xdp_frame_has_frags(xdpf)))		return -EOPNOTSUPP; 	nxdpf = xdpf_clone(xdpf);	if (!nxdpf)		return -ENOMEM; 	bq_enqueue(obj->dev, nxdpf, dev_rx, obj->xdp_prog); 	return 0;} static inline bool is_ifindex_excluded(int *excluded, int num_excluded, int ifindex){	while (num_excluded--) {		if (ifindex == excluded[num_excluded])			return true;	}	return false;} /* Get ifindex of each upper device. 'indexes' must be able to hold at * least 'max' elements. * Returns the number of ifindexes added, or -EOVERFLOW if there are too * many upper devices. */static int get_upper_ifindexes(struct net_device *dev, int *indexes, int max){	struct net_device *upper;	struct list_head *iter;	int n = 0; 	netdev_for_each_upper_dev_rcu(dev, upper, iter) {		if (n >= max)			return -EOVERFLOW;		indexes[n++] = upper->ifindex;	} 	return n;} int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,			  struct bpf_map *map, bool exclude_ingress){	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);	struct bpf_dtab_netdev *dst, *last_dst = NULL;	int excluded_devices[1+MAX_NEST_DEV];	struct hlist_head *head;	int num_excluded = 0;	unsigned int i;	int err; 	if (exclude_ingress) {		num_excluded = get_upper_ifindexes(dev_rx, excluded_devices,						   ARRAY_SIZE(excluded_devices) - 1);		if (num_excluded < 0)			return num_excluded; 		excluded_devices[num_excluded++] = dev_rx->ifindex;	} 	if (map->map_type == BPF_MAP_TYPE_DEVMAP) {		for (i = 0; i < map->max_entries; i++) {			dst = rcu_dereference_check(dtab->netdev_map[i],						    rcu_read_lock_bh_held());			if (!is_valid_dst(dst, xdpf))				continue; 			if (is_ifindex_excluded(excluded_devices, num_excluded, dst->dev->ifindex))				continue; 			/* we only need n-1 clones; last_dst enqueued below */			if (!last_dst) {				last_dst = dst;				continue;			} 			err = dev_map_enqueue_clone(last_dst, dev_rx, xdpf);			if (err)				return err; 			last_dst = dst;		}	} else { /* BPF_MAP_TYPE_DEVMAP_HASH */		for (i = 0; i < dtab->n_buckets; i++) {			head = dev_map_index_hash(dtab, i);			hlist_for_each_entry_rcu(dst, head, index_hlist,						rcu_read_lock_bh_held()) {				if (!is_valid_dst(dst, xdpf))					continue; 				if (is_ifindex_excluded(excluded_devices, num_excluded,							dst->dev->ifindex))					continue; 				/* we only need n-1 clones; last_dst enqueued below */				if (!last_dst) {					last_dst = dst;					continue;				} 				err = dev_map_enqueue_clone(last_dst, dev_rx, xdpf);				if (err)					return err; 				last_dst = dst;			}		}	} 	/* consume the last copy of the frame */	if (last_dst)		bq_enqueue(last_dst->dev, xdpf, dev_rx, last_dst->xdp_prog);	else		xdp_return_frame_rx_napi(xdpf); /* dtab is empty */ 	return 0;} int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb,			     const struct bpf_prog *xdp_prog){	int err; 	err = xdp_ok_fwd_dev(dst->dev, skb->len);	if (unlikely(err))		return err; 	if (dst->xdp_prog && skb_cloned(skb)) {		struct sk_buff *nskb; 		nskb = skb_copy(skb, GFP_ATOMIC);		if (!nskb)			return -ENOMEM; 		nskb->mac_len = skb->mac_len;		consume_skb(skb);		skb = nskb;	} 	/* Redirect has already succeeded semantically at this point, so we just	 * return 0 even if packet is dropped. Helper below takes care of	 * freeing skb.	 */	if (dev_map_bpf_prog_run_skb(skb, dst) != XDP_PASS)		return 0; 	skb->dev = dst->dev;	generic_xdp_tx(skb, xdp_prog); 	return 0;} static int dev_map_redirect_clone(struct bpf_dtab_netdev *dst,				  struct sk_buff *skb,				  const struct bpf_prog *xdp_prog){	struct sk_buff *nskb;	int err; 	if (unlikely(skb_is_nonlinear(skb)))		return -EOPNOTSUPP; 	nskb = skb_clone(skb, GFP_ATOMIC);	if (!nskb)		return -ENOMEM; 	err = dev_map_generic_redirect(dst, nskb, xdp_prog);	if (unlikely(err)) {		consume_skb(nskb);		return err;	} 	return 0;} int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb,			   const struct bpf_prog *xdp_prog,			   struct bpf_map *map, bool exclude_ingress){	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);	struct bpf_dtab_netdev *dst, *last_dst = NULL;	int excluded_devices[1+MAX_NEST_DEV];	struct hlist_head *head;	int num_excluded = 0;	unsigned int i;	int err; 	if (exclude_ingress) {		num_excluded = get_upper_ifindexes(dev, excluded_devices,						   ARRAY_SIZE(excluded_devices) - 1);		if (num_excluded < 0)			return num_excluded; 		excluded_devices[num_excluded++] = dev->ifindex;	} 	if (map->map_type == BPF_MAP_TYPE_DEVMAP) {		for (i = 0; i < map->max_entries; i++) {			dst = rcu_dereference_check(dtab->netdev_map[i],						    rcu_read_lock_bh_held());			if (!dst)				continue; 			if (is_ifindex_excluded(excluded_devices, num_excluded, dst->dev->ifindex))				continue; 			/* we only need n-1 clones; last_dst enqueued below */			if (!last_dst) {				last_dst = dst;				continue;			} 			err = dev_map_redirect_clone(last_dst, skb, xdp_prog);			if (err)				return err; 			last_dst = dst; 		}	} else { /* BPF_MAP_TYPE_DEVMAP_HASH */		for (i = 0; i < dtab->n_buckets; i++) {			head = dev_map_index_hash(dtab, i);			hlist_for_each_entry_rcu(dst, head, index_hlist, rcu_read_lock_bh_held()) {				if (is_ifindex_excluded(excluded_devices, num_excluded,							dst->dev->ifindex))					continue; 				/* we only need n-1 clones; last_dst enqueued below */				if (!last_dst) {					last_dst = dst;					continue;				} 				err = dev_map_redirect_clone(last_dst, skb, xdp_prog);				if (err)					return err; 				last_dst = dst;			}		}	} 	/* consume the first skb and return */	if (last_dst)		return dev_map_generic_redirect(last_dst, skb, xdp_prog); 	/* dtab is empty */	consume_skb(skb);	return 0;} static void *dev_map_lookup_elem(struct bpf_map *map, void *key){	struct bpf_dtab_netdev *obj = __dev_map_lookup_elem(map, *(u32 *)key); 	return obj ? &obj->val : NULL;} static void *dev_map_hash_lookup_elem(struct bpf_map *map, void *key){	struct bpf_dtab_netdev *obj = __dev_map_hash_lookup_elem(map,								*(u32 *)key);	return obj ? &obj->val : NULL;} static void __dev_map_entry_free(struct rcu_head *rcu){	struct bpf_dtab_netdev *dev; 	dev = container_of(rcu, struct bpf_dtab_netdev, rcu);	if (dev->xdp_prog)		bpf_prog_put(dev->xdp_prog);	dev_put(dev->dev);	kfree(dev);} static long dev_map_delete_elem(struct bpf_map *map, void *key){	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);	struct bpf_dtab_netdev *old_dev;	u32 k = *(u32 *)key; 	if (k >= map->max_entries)		return -EINVAL; 	old_dev = unrcu_pointer(xchg(&dtab->netdev_map[k], NULL));	if (old_dev) {		call_rcu(&old_dev->rcu, __dev_map_entry_free);		atomic_dec((atomic_t *)&dtab->items);	}	return 0;} static long dev_map_hash_delete_elem(struct bpf_map *map, void *key){	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);	struct bpf_dtab_netdev *old_dev;	u32 k = *(u32 *)key;	unsigned long flags;	int ret = -ENOENT; 	spin_lock_irqsave(&dtab->index_lock, flags); 	old_dev = __dev_map_hash_lookup_elem(map, k);	if (old_dev) {		dtab->items--;		hlist_del_init_rcu(&old_dev->index_hlist);		call_rcu(&old_dev->rcu, __dev_map_entry_free);		ret = 0;	}	spin_unlock_irqrestore(&dtab->index_lock, flags); 	return ret;} static struct bpf_dtab_netdev *__dev_map_alloc_node(struct net *net,						    struct bpf_dtab *dtab,						    struct bpf_devmap_val *val,						    unsigned int idx){	struct bpf_prog *prog = NULL;	struct bpf_dtab_netdev *dev; 	dev = bpf_map_kmalloc_node(&dtab->map, sizeof(*dev),				   GFP_NOWAIT,				   dtab->map.numa_node);	if (!dev)		return ERR_PTR(-ENOMEM); 	dev->dev = dev_get_by_index(net, val->ifindex);	if (!dev->dev)		goto err_out; 	if (val->bpf_prog.fd > 0) {		prog = bpf_prog_get_type_dev(val->bpf_prog.fd,					     BPF_PROG_TYPE_XDP, false);		if (IS_ERR(prog))			goto err_put_dev;		if (prog->expected_attach_type != BPF_XDP_DEVMAP ||		    !bpf_prog_map_compatible(&dtab->map, prog))			goto err_put_prog;	} 	dev->idx = idx;	if (prog) {		dev->xdp_prog = prog;		dev->val.bpf_prog.id = prog->aux->id;	} else {		dev->xdp_prog = NULL;		dev->val.bpf_prog.id = 0;	}	dev->val.ifindex = val->ifindex; 	return dev;err_put_prog:	bpf_prog_put(prog);err_put_dev:	dev_put(dev->dev);err_out:	kfree(dev);	return ERR_PTR(-EINVAL);} static long __dev_map_update_elem(struct net *net, struct bpf_map *map,				  void *key, void *value, u64 map_flags){	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);	struct bpf_dtab_netdev *dev, *old_dev;	struct bpf_devmap_val val = {};	u32 i = *(u32 *)key; 	if (unlikely(map_flags > BPF_EXIST))		return -EINVAL;	if (unlikely(i >= dtab->map.max_entries))		return -E2BIG;	if (unlikely(map_flags == BPF_NOEXIST))		return -EEXIST; 	/* already verified value_size <= sizeof val */	memcpy(&val, value, map->value_size); 	if (!val.ifindex) {		dev = NULL;		/* can not specify fd if ifindex is 0 */		if (val.bpf_prog.fd > 0)			return -EINVAL;	} else {		dev = __dev_map_alloc_node(net, dtab, &val, i);		if (IS_ERR(dev))			return PTR_ERR(dev);	} 	/* Use call_rcu() here to ensure rcu critical sections have completed	 * Remembering the driver side flush operation will happen before the	 * net device is removed.	 */	old_dev = unrcu_pointer(xchg(&dtab->netdev_map[i], RCU_INITIALIZER(dev)));	if (old_dev)		call_rcu(&old_dev->rcu, __dev_map_entry_free);	else		atomic_inc((atomic_t *)&dtab->items); 	return 0;} static long dev_map_update_elem(struct bpf_map *map, void *key, void *value,				u64 map_flags){	return __dev_map_update_elem(current->nsproxy->net_ns,				     map, key, value, map_flags);} static long __dev_map_hash_update_elem(struct net *net, struct bpf_map *map,				       void *key, void *value, u64 map_flags){

Lines 501–1000 of 1217. Previous lines Load more