History

bpf: Add support to attach bpf program to a devmap entry

Add BPF_XDP_DEVMAP attach type for use with programs associated with a DEVMAP entry. Allow DEVMAPs to associate a program with a device entry by adding a bpf_prog.fd to 'struct bpf_devmap_val'. Values read show the program id, so the fd and id are a union. bpf programs can get access to the struct via vmlinux.h. The program associated with the fd must have type XDP with expected attach type BPF_XDP_DEVMAP. When a program is associated with a device index, the program is run on an XDP_REDIRECT and before the buffer is added to the per-cpu queue. At this point rxq data is still valid; the next patch adds tx device information allowing the prorgam to see both ingress and egress device indices. XDP generic is skb based and XDP programs do not work with skb's. Block the use case by walking maps used by a program that is to be attached via xdpgeneric and fail if any of them are DEVMAP / DEVMAP_HASH with Block attach of BPF_XDP_DEVMAP programs to devices.

David Ahern committed 6 years ago

Commit fbee97f1 parent 7f1c0425 files · +109 −4Browse files

Unified · Split

5 files
  1. include/linux/bpf.h +5 −0
  2. include/uapi/linux/bpf.h +1 −0
  3. kernel/bpf/devmap.c +84 −4
  4. net/core/dev.c +18 −0
  5. tools/include/uapi/linux/bpf.h +1 −0
include/linux/bpf.h+5−0
@@ -1250,6 +1250,7 @@ int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, · 1249 unmodified lines
1250 struct net_device *dev_rx);1250 struct net_device *dev_rx);
1251int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb,1251int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb,
1252 struct bpf_prog *xdp_prog);1252 struct bpf_prog *xdp_prog);
1253bool dev_map_can_have_prog(struct bpf_map *map);
1253 1254
1254struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key);1255struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key);
1255void __cpu_map_flush(void);1256void __cpu_map_flush(void);
@@ -1363,6 +1364,10 @@ static inline struct net_device *__dev_map_hash_lookup_elem(struct bpf_map *map · 107 unmodified lines
1363{1364{
1364 return NULL;1365 return NULL;
1365}1366}
1367static inline bool dev_map_can_have_prog(struct bpf_map *map)
1368{
1369 return false;
1370}
1366 1371
1367static inline void __dev_flush(void)1372static inline void __dev_flush(void)
1368{1373{
include/uapi/linux/bpf.h+1−0
@@ -225,6 +225,7 @@ enum bpf_attach_type { · 224 unmodified lines
225 BPF_CGROUP_INET6_GETPEERNAME,225 BPF_CGROUP_INET6_GETPEERNAME,
226 BPF_CGROUP_INET4_GETSOCKNAME,226 BPF_CGROUP_INET4_GETSOCKNAME,
227 BPF_CGROUP_INET6_GETSOCKNAME,227 BPF_CGROUP_INET6_GETSOCKNAME,
228 BPF_XDP_DEVMAP,
228 __MAX_BPF_ATTACH_TYPE229 __MAX_BPF_ATTACH_TYPE
229};230};
230 231
kernel/bpf/devmap.c+84−4
@@ -63,12 +63,17 @@ struct xdp_dev_bulk_queue { · 62 unmodified lines
63/* DEVMAP values */63/* DEVMAP values */
64struct bpf_devmap_val {64struct bpf_devmap_val {
65 u32 ifindex; /* device index */65 u32 ifindex; /* device index */
66 union {
67 int fd; /* prog fd on map write */
68 u32 id; /* prog id on map read */
69 } bpf_prog;
66};70};
67 71
68struct bpf_dtab_netdev {72struct bpf_dtab_netdev {
69 struct net_device *dev; /* must be first member, due to tracepoint */73 struct net_device *dev; /* must be first member, due to tracepoint */
70 struct hlist_node index_hlist;74 struct hlist_node index_hlist;
71 struct bpf_dtab *dtab;75 struct bpf_dtab *dtab;
76 struct bpf_prog *xdp_prog;
72 struct rcu_head rcu;77 struct rcu_head rcu;
73 unsigned int idx;78 unsigned int idx;
74 struct bpf_devmap_val val;79 struct bpf_devmap_val val;
@@ -111,12 +116,18 @@ static inline struct hlist_head *dev_map_index_hash(struct bpf_dtab *dtab, · 36 unmodified lines
111 116
112static int dev_map_init_map(struct bpf_dtab *dtab, union bpf_attr *attr)117static int dev_map_init_map(struct bpf_dtab *dtab, union bpf_attr *attr)
113{118{
119 u32 valsize = attr->value_size;
114 u64 cost = 0;120 u64 cost = 0;
115 int err;121 int err;
116 122
117 /* check sanity of attributes */123 /* check sanity of attributes. 2 value sizes supported:
124 * 4 bytes: ifindex
125 * 8 bytes: ifindex + prog fd
126 */
118 if (attr->max_entries == 0 || attr->key_size != 4 ||127 if (attr->max_entries == 0 || attr->key_size != 4 ||
119 attr->value_size != 4 || attr->map_flags & ~DEV_CREATE_FLAG_MASK)128 (valsize != offsetofend(struct bpf_devmap_val, ifindex) &&
129 valsize != offsetofend(struct bpf_devmap_val, bpf_prog.fd)) ||
130 attr->map_flags & ~DEV_CREATE_FLAG_MASK)
120 return -EINVAL;131 return -EINVAL;
121 132
122 /* Lookup returns a pointer straight to dev->ifindex, so make sure the133 /* Lookup returns a pointer straight to dev->ifindex, so make sure the
@@ -223,6 +234,8 @@ static void dev_map_free(struct bpf_map *map) · 100 unmodified lines
223 234
224 hlist_for_each_entry_safe(dev, next, head, index_hlist) {235 hlist_for_each_entry_safe(dev, next, head, index_hlist) {
225 hlist_del_rcu(&dev->index_hlist);236 hlist_del_rcu(&dev->index_hlist);
237 if (dev->xdp_prog)
238 bpf_prog_put(dev->xdp_prog);
226 dev_put(dev->dev);239 dev_put(dev->dev);
227 kfree(dev);240 kfree(dev);
228 }241 }
@@ -237,6 +250,8 @@ static void dev_map_free(struct bpf_map *map) · 8 unmodified lines
237 if (!dev)250 if (!dev)
238 continue;251 continue;
239 252
253 if (dev->xdp_prog)
254 bpf_prog_put(dev->xdp_prog);
240 dev_put(dev->dev);255 dev_put(dev->dev);
241 kfree(dev);256 kfree(dev);
242 }257 }
@@ -323,6 +338,16 @@ static int dev_map_hash_get_next_key(struct bpf_map *map, void *key, · 80 unmodified lines
323 return -ENOENT;338 return -ENOENT;
324}339}
325 340
341bool dev_map_can_have_prog(struct bpf_map *map)
342{
343 if ((map->map_type == BPF_MAP_TYPE_DEVMAP ||
344 map->map_type == BPF_MAP_TYPE_DEVMAP_HASH) &&
345 map->value_size != offsetofend(struct bpf_devmap_val, ifindex))
346 return true;
347
348 return false;
349}
350
326static int bq_xmit_all(struct xdp_dev_bulk_queue *bq, u32 flags)351static int bq_xmit_all(struct xdp_dev_bulk_queue *bq, u32 flags)
327{352{
328 struct net_device *dev = bq->dev;353 struct net_device *dev = bq->dev;
@@ -447,6 +472,30 @@ static inline int __xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp, · 118 unmodified lines
447 return bq_enqueue(dev, xdpf, dev_rx);472 return bq_enqueue(dev, xdpf, dev_rx);
448}473}
449 474
475static struct xdp_buff *dev_map_run_prog(struct net_device *dev,
476 struct xdp_buff *xdp,
477 struct bpf_prog *xdp_prog)
478{
479 u32 act;
480
481 act = bpf_prog_run_xdp(xdp_prog, xdp);
482 switch (act) {
483 case XDP_PASS:
484 return xdp;
485 case XDP_DROP:
486 break;
487 default:
488 bpf_warn_invalid_xdp_action(act);
489 fallthrough;
490 case XDP_ABORTED:
491 trace_xdp_exception(dev, xdp_prog, act);
492 break;
493 }
494
495 xdp_return_buff(xdp);
496 return NULL;
497}
498
450int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp,499int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp,
451 struct net_device *dev_rx)500 struct net_device *dev_rx)
452{501{
@@ -458,6 +507,11 @@ int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, · 5 unmodified lines
458{507{
459 struct net_device *dev = dst->dev;508 struct net_device *dev = dst->dev;
460 509
510 if (dst->xdp_prog) {
511 xdp = dev_map_run_prog(dev, xdp, dst->xdp_prog);
512 if (!xdp)
513 return 0;
514 }
461 return __xdp_enqueue(dev, xdp, dev_rx);515 return __xdp_enqueue(dev, xdp, dev_rx);
462}516}
463 517
@@ -494,6 +548,8 @@ static void __dev_map_entry_free(struct rcu_head *rcu) · 30 unmodified lines
494 struct bpf_dtab_netdev *dev;548 struct bpf_dtab_netdev *dev;
495 549
496 dev = container_of(rcu, struct bpf_dtab_netdev, rcu);550 dev = container_of(rcu, struct bpf_dtab_netdev, rcu);
551 if (dev->xdp_prog)
552 bpf_prog_put(dev->xdp_prog);
497 dev_put(dev->dev);553 dev_put(dev->dev);
498 kfree(dev);554 kfree(dev);
499}555}
@@ -547,6 +603,7 @@ static struct bpf_dtab_netdev *__dev_map_alloc_node(struct net *net, · 47 unmodified lines
547 struct bpf_devmap_val *val,603 struct bpf_devmap_val *val,
548 unsigned int idx)604 unsigned int idx)
549{605{
606 struct bpf_prog *prog = NULL;
550 struct bpf_dtab_netdev *dev;607 struct bpf_dtab_netdev *dev;
551 608
552 dev = kmalloc_node(sizeof(*dev), GFP_ATOMIC | __GFP_NOWARN,609 dev = kmalloc_node(sizeof(*dev), GFP_ATOMIC | __GFP_NOWARN,
@@ -558,11 +615,31 @@ static struct bpf_dtab_netdev *__dev_map_alloc_node(struct net *net, · 5 unmodified lines
558 if (!dev->dev)615 if (!dev->dev)
559 goto err_out;616 goto err_out;
560 617
618 if (val->bpf_prog.fd >= 0) {
619 prog = bpf_prog_get_type_dev(val->bpf_prog.fd,
620 BPF_PROG_TYPE_XDP, false);
621 if (IS_ERR(prog))
622 goto err_put_dev;
623 if (prog->expected_attach_type != BPF_XDP_DEVMAP)
624 goto err_put_prog;
625 }
626
561 dev->idx = idx;627 dev->idx = idx;
562 dev->dtab = dtab;628 dev->dtab = dtab;
629 if (prog) {
630 dev->xdp_prog = prog;
631 dev->val.bpf_prog.id = prog->aux->id;
632 } else {
633 dev->xdp_prog = NULL;
634 dev->val.bpf_prog.id = 0;
635 }
563 dev->val.ifindex = val->ifindex;636 dev->val.ifindex = val->ifindex;
564 637
565 return dev;638 return dev;
639err_put_prog:
640 bpf_prog_put(prog);
641err_put_dev:
642 dev_put(dev->dev);
566err_out:643err_out:
567 kfree(dev);644 kfree(dev);
568 return ERR_PTR(-EINVAL);645 return ERR_PTR(-EINVAL);
@@ -572,8 +649,8 @@ static int __dev_map_update_elem(struct net *net, struct bpf_map *map, · 3 unmodified lines
572 void *key, void *value, u64 map_flags)649 void *key, void *value, u64 map_flags)
573{650{
574 struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);651 struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
652 struct bpf_devmap_val val = { .bpf_prog.fd = -1 };
575 struct bpf_dtab_netdev *dev, *old_dev;653 struct bpf_dtab_netdev *dev, *old_dev;
576 struct bpf_devmap_val val = { };
577 u32 i = *(u32 *)key;654 u32 i = *(u32 *)key;
578 655
579 if (unlikely(map_flags > BPF_EXIST))656 if (unlikely(map_flags > BPF_EXIST))
@@ -588,6 +665,9 @@ static int __dev_map_update_elem(struct net *net, struct bpf_map *map, · 8 unmodified lines
588 665
589 if (!val.ifindex) {666 if (!val.ifindex) {
590 dev = NULL;667 dev = NULL;
668 /* can not specify fd if ifindex is 0 */
669 if (val.bpf_prog.fd != -1)
670 return -EINVAL;
591 } else {671 } else {
592 dev = __dev_map_alloc_node(net, dtab, &val, i);672 dev = __dev_map_alloc_node(net, dtab, &val, i);
593 if (IS_ERR(dev))673 if (IS_ERR(dev))
@@ -616,8 +696,8 @@ static int __dev_map_hash_update_elem(struct net *net, struct bpf_map *map, · 22 unmodified lines
616 void *key, void *value, u64 map_flags)696 void *key, void *value, u64 map_flags)
617{697{
618 struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);698 struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
699 struct bpf_devmap_val val = { .bpf_prog.fd = -1 };
619 struct bpf_dtab_netdev *dev, *old_dev;700 struct bpf_dtab_netdev *dev, *old_dev;
620 struct bpf_devmap_val val = { };
621 u32 idx = *(u32 *)key;701 u32 idx = *(u32 *)key;
622 unsigned long flags;702 unsigned long flags;
623 int err = -EEXIST;703 int err = -EEXIST;
net/core/dev.c+18−0
@@ -5420,6 +5420,18 @@ static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp) · 5419 unmodified lines
5420 struct bpf_prog *new = xdp->prog;5420 struct bpf_prog *new = xdp->prog;
5421 int ret = 0;5421 int ret = 0;
5422 5422
5423 if (new) {
5424 u32 i;
5425
5426 /* generic XDP does not work with DEVMAPs that can
5427 * have a bpf_prog installed on an entry
5428 */
5429 for (i = 0; i < new->aux->used_map_cnt; i++) {
5430 if (dev_map_can_have_prog(new->aux->used_maps[i]))
5431 return -EINVAL;
5432 }
5433 }
5434
5423 switch (xdp->command) {5435 switch (xdp->command) {
5424 case XDP_SETUP_PROG:5436 case XDP_SETUP_PROG:
5425 rcu_assign_pointer(dev->xdp_prog, new);5437 rcu_assign_pointer(dev->xdp_prog, new);
@@ -8835,6 +8847,12 @@ int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack, · 3409 unmodified lines
8835 return -EINVAL;8847 return -EINVAL;
8836 }8848 }
8837 8849
8850 if (prog->expected_attach_type == BPF_XDP_DEVMAP) {
8851 NL_SET_ERR_MSG(extack, "BPF_XDP_DEVMAP programs can not be attached to a device");
8852 bpf_prog_put(prog);
8853 return -EINVAL;
8854 }
8855
8838 /* prog->aux->id may be 0 for orphaned device-bound progs */8856 /* prog->aux->id may be 0 for orphaned device-bound progs */
8839 if (prog->aux->id && prog->aux->id == prog_id) {8857 if (prog->aux->id && prog->aux->id == prog_id) {
8840 bpf_prog_put(prog);8858 bpf_prog_put(prog);
tools/include/uapi/linux/bpf.h+1−0
@@ -225,6 +225,7 @@ enum bpf_attach_type { · 224 unmodified lines
225 BPF_CGROUP_INET6_GETPEERNAME,225 BPF_CGROUP_INET6_GETPEERNAME,
226 BPF_CGROUP_INET4_GETSOCKNAME,226 BPF_CGROUP_INET4_GETSOCKNAME,
227 BPF_CGROUP_INET6_GETSOCKNAME,227 BPF_CGROUP_INET6_GETSOCKNAME,
228 BPF_XDP_DEVMAP,
228 __MAX_BPF_ATTACH_TYPE229 __MAX_BPF_ATTACH_TYPE
229};230};
230 231