History

bpf: introduce new bpf AF_XDP map type BPF_MAP_TYPE_XSKMAP

The xskmap is yet another BPF map, very much inspired by dev/cpu/sockmap, and is a holder of AF_XDP sockets. A user application adds AF_XDP sockets into the map, and by using the bpf_redirect_map helper, an XDP program can redirect XDP frames to an AF_XDP socket. Note that a socket that is bound to certain ifindex/queue index will *only* accept XDP frames from that netdev/queue index. If an XDP program tries to redirect from a netdev/queue index other than what the socket is bound to, the frame will not be received on the socket. A socket can reside in multiple maps. v3: Fixed race and simplified code. v2: Removed one indirection in map lookup.

Björn Töpel committed 8 years ago

Commit fbfc5041 parent c4971768 files · +289 −2Browse files

Unified · Split

8 files
  1. include/linux/bpf.h +25 −0
  2. include/linux/bpf_types.h +3 −0
  3. include/net/xdp_sock.h +7 −0
  4. include/uapi/linux/bpf.h +1 −0
  5. kernel/bpf/Makefile +3 −0
  6. kernel/bpf/verifier.c +6 −2
  7. kernel/bpf/xskmap.c +239 −0
  8. net/xdp/xsk.c +5 −0
include/linux/bpf.h+25−0
@@ -676,6 +676,31 @@ static inline int sock_map_prog(struct bpf_map *map, · 675 unmodified lines
676}676}
677#endif677#endif
678 678
679#if defined(CONFIG_XDP_SOCKETS)
680struct xdp_sock;
681struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map, u32 key);
682int __xsk_map_redirect(struct bpf_map *map, struct xdp_buff *xdp,
683 struct xdp_sock *xs);
684void __xsk_map_flush(struct bpf_map *map);
685#else
686struct xdp_sock;
687static inline struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map,
688 u32 key)
689{
690 return NULL;
691}
692
693static inline int __xsk_map_redirect(struct bpf_map *map, struct xdp_buff *xdp,
694 struct xdp_sock *xs)
695{
696 return -EOPNOTSUPP;
697}
698
699static inline void __xsk_map_flush(struct bpf_map *map)
700{
701}
702#endif
703
679/* verifier prototypes for helper functions called from eBPF programs */704/* verifier prototypes for helper functions called from eBPF programs */
680extern const struct bpf_func_proto bpf_map_lookup_elem_proto;705extern const struct bpf_func_proto bpf_map_lookup_elem_proto;
681extern const struct bpf_func_proto bpf_map_update_elem_proto;706extern const struct bpf_func_proto bpf_map_update_elem_proto;
include/linux/bpf_types.h+3−0
@@ -49,4 +49,7 @@ BPF_MAP_TYPE(BPF_MAP_TYPE_DEVMAP, dev_map_ops) · 48 unmodified lines
49BPF_MAP_TYPE(BPF_MAP_TYPE_SOCKMAP, sock_map_ops)49BPF_MAP_TYPE(BPF_MAP_TYPE_SOCKMAP, sock_map_ops)
50#endif50#endif
51BPF_MAP_TYPE(BPF_MAP_TYPE_CPUMAP, cpu_map_ops)51BPF_MAP_TYPE(BPF_MAP_TYPE_CPUMAP, cpu_map_ops)
52#if defined(CONFIG_XDP_SOCKETS)
53BPF_MAP_TYPE(BPF_MAP_TYPE_XSKMAP, xsk_map_ops)
54#endif
52#endif55#endif
include/net/xdp_sock.h+7−0
@@ -28,6 +28,7 @@ struct xdp_sock { · 27 unmodified lines
28 struct xsk_queue *rx;28 struct xsk_queue *rx;
29 struct net_device *dev;29 struct net_device *dev;
30 struct xdp_umem *umem;30 struct xdp_umem *umem;
31 struct list_head flush_node;
31 u16 queue_id;32 u16 queue_id;
32 /* Protects multiple processes in the control path */33 /* Protects multiple processes in the control path */
33 struct mutex mutex;34 struct mutex mutex;
@@ -39,6 +40,7 @@ struct xdp_buff; · 5 unmodified lines
39int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);40int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);
40int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);41int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);
41void xsk_flush(struct xdp_sock *xs);42void xsk_flush(struct xdp_sock *xs);
43bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs);
42#else44#else
43static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)45static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
44{46{
@@ -53,6 +55,11 @@ static inline int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) · 8 unmodified lines
53static inline void xsk_flush(struct xdp_sock *xs)55static inline void xsk_flush(struct xdp_sock *xs)
54{56{
55}57}
58
59static inline bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs)
60{
61 return false;
62}
56#endif /* CONFIG_XDP_SOCKETS */63#endif /* CONFIG_XDP_SOCKETS */
57 64
58#endif /* _LINUX_XDP_SOCK_H */65#endif /* _LINUX_XDP_SOCK_H */
include/uapi/linux/bpf.h+1−0
@@ -116,6 +116,7 @@ enum bpf_map_type { · 115 unmodified lines
116 BPF_MAP_TYPE_DEVMAP,116 BPF_MAP_TYPE_DEVMAP,
117 BPF_MAP_TYPE_SOCKMAP,117 BPF_MAP_TYPE_SOCKMAP,
118 BPF_MAP_TYPE_CPUMAP,118 BPF_MAP_TYPE_CPUMAP,
119 BPF_MAP_TYPE_XSKMAP,
119};120};
120 121
121enum bpf_prog_type {122enum bpf_prog_type {
kernel/bpf/Makefile+3−0
@@ -8,6 +8,9 @@ obj-$(CONFIG_BPF_SYSCALL) += btf.o · 7 unmodified lines
8ifeq ($(CONFIG_NET),y)8ifeq ($(CONFIG_NET),y)
9obj-$(CONFIG_BPF_SYSCALL) += devmap.o9obj-$(CONFIG_BPF_SYSCALL) += devmap.o
10obj-$(CONFIG_BPF_SYSCALL) += cpumap.o10obj-$(CONFIG_BPF_SYSCALL) += cpumap.o
11ifeq ($(CONFIG_XDP_SOCKETS),y)
12obj-$(CONFIG_BPF_SYSCALL) += xskmap.o
13endif
11obj-$(CONFIG_BPF_SYSCALL) += offload.o14obj-$(CONFIG_BPF_SYSCALL) += offload.o
12ifeq ($(CONFIG_STREAM_PARSER),y)15ifeq ($(CONFIG_STREAM_PARSER),y)
13ifeq ($(CONFIG_INET),y)16ifeq ($(CONFIG_INET),y)
kernel/bpf/verifier.c+6−2
@@ -2070,8 +2070,11 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, · 2069 unmodified lines
2070 if (func_id != BPF_FUNC_redirect_map)2070 if (func_id != BPF_FUNC_redirect_map)
2071 goto error;2071 goto error;
2072 break;2072 break;
2073 /* Restrict bpf side of cpumap, open when use-cases appear */2073 /* Restrict bpf side of cpumap and xskmap, open when use-cases
2074 * appear.
2075 */
2074 case BPF_MAP_TYPE_CPUMAP:2076 case BPF_MAP_TYPE_CPUMAP:
2077 case BPF_MAP_TYPE_XSKMAP:
2075 if (func_id != BPF_FUNC_redirect_map)2078 if (func_id != BPF_FUNC_redirect_map)
2076 goto error;2079 goto error;
2077 break;2080 break;
@@ -2118,7 +2121,8 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, · 40 unmodified lines
2118 break;2121 break;
2119 case BPF_FUNC_redirect_map:2122 case BPF_FUNC_redirect_map:
2120 if (map->map_type != BPF_MAP_TYPE_DEVMAP &&2123 if (map->map_type != BPF_MAP_TYPE_DEVMAP &&
2121 map->map_type != BPF_MAP_TYPE_CPUMAP)2124 map->map_type != BPF_MAP_TYPE_CPUMAP &&
2125 map->map_type != BPF_MAP_TYPE_XSKMAP)
2122 goto error;2126 goto error;
2123 break;2127 break;
2124 case BPF_FUNC_sk_redirect_map:2128 case BPF_FUNC_sk_redirect_map:
kernel/bpf/xskmap.c+239−0
@@ -0,0 +1,239 @@
1// SPDX-License-Identifier: GPL-2.0
2/* XSKMAP used for AF_XDP sockets
3 * Copyright(c) 2018 Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 */
14
15#include <linux/bpf.h>
16#include <linux/capability.h>
17#include <net/xdp_sock.h>
18#include <linux/slab.h>
19#include <linux/sched.h>
20
21struct xsk_map {
22 struct bpf_map map;
23 struct xdp_sock **xsk_map;
24 struct list_head __percpu *flush_list;
25};
26
27static struct bpf_map *xsk_map_alloc(union bpf_attr *attr)
28{
29 int cpu, err = -EINVAL;
30 struct xsk_map *m;
31 u64 cost;
32
33 if (!capable(CAP_NET_ADMIN))
34 return ERR_PTR(-EPERM);
35
36 if (attr->max_entries == 0 || attr->key_size != 4 ||
37 attr->value_size != 4 ||
38 attr->map_flags & ~(BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY))
39 return ERR_PTR(-EINVAL);
40
41 m = kzalloc(sizeof(*m), GFP_USER);
42 if (!m)
43 return ERR_PTR(-ENOMEM);
44
45 bpf_map_init_from_attr(&m->map, attr);
46
47 cost = (u64)m->map.max_entries * sizeof(struct xdp_sock *);
48 cost += sizeof(struct list_head) * num_possible_cpus();
49 if (cost >= U32_MAX - PAGE_SIZE)
50 goto free_m;
51
52 m->map.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;
53
54 /* Notice returns -EPERM on if map size is larger than memlock limit */
55 err = bpf_map_precharge_memlock(m->map.pages);
56 if (err)
57 goto free_m;
58
59 m->flush_list = alloc_percpu(struct list_head);
60 if (!m->flush_list)
61 goto free_m;
62
63 for_each_possible_cpu(cpu)
64 INIT_LIST_HEAD(per_cpu_ptr(m->flush_list, cpu));
65
66 m->xsk_map = bpf_map_area_alloc(m->map.max_entries *
67 sizeof(struct xdp_sock *),
68 m->map.numa_node);
69 if (!m->xsk_map)
70 goto free_percpu;
71 return &m->map;
72
73free_percpu:
74 free_percpu(m->flush_list);
75free_m:
76 kfree(m);
77 return ERR_PTR(err);
78}
79
80static void xsk_map_free(struct bpf_map *map)
81{
82 struct xsk_map *m = container_of(map, struct xsk_map, map);
83 int i;
84
85 synchronize_net();
86
87 for (i = 0; i < map->max_entries; i++) {
88 struct xdp_sock *xs;
89
90 xs = m->xsk_map[i];
91 if (!xs)
92 continue;
93
94 sock_put((struct sock *)xs);
95 }
96
97 free_percpu(m->flush_list);
98 bpf_map_area_free(m->xsk_map);
99 kfree(m);
100}
101
102static int xsk_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
103{
104 struct xsk_map *m = container_of(map, struct xsk_map, map);
105 u32 index = key ? *(u32 *)key : U32_MAX;
106 u32 *next = next_key;
107
108 if (index >= m->map.max_entries) {
109 *next = 0;
110 return 0;
111 }
112
113 if (index == m->map.max_entries - 1)
114 return -ENOENT;
115 *next = index + 1;
116 return 0;
117}
118
119struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map, u32 key)
120{
121 struct xsk_map *m = container_of(map, struct xsk_map, map);
122 struct xdp_sock *xs;
123
124 if (key >= map->max_entries)
125 return NULL;
126
127 xs = READ_ONCE(m->xsk_map[key]);
128 return xs;
129}
130
131int __xsk_map_redirect(struct bpf_map *map, struct xdp_buff *xdp,
132 struct xdp_sock *xs)
133{
134 struct xsk_map *m = container_of(map, struct xsk_map, map);
135 struct list_head *flush_list = this_cpu_ptr(m->flush_list);
136 int err;
137
138 err = xsk_rcv(xs, xdp);
139 if (err)
140 return err;
141
142 if (!xs->flush_node.prev)
143 list_add(&xs->flush_node, flush_list);
144
145 return 0;
146}
147
148void __xsk_map_flush(struct bpf_map *map)
149{
150 struct xsk_map *m = container_of(map, struct xsk_map, map);
151 struct list_head *flush_list = this_cpu_ptr(m->flush_list);
152 struct xdp_sock *xs, *tmp;
153
154 list_for_each_entry_safe(xs, tmp, flush_list, flush_node) {
155 xsk_flush(xs);
156 __list_del(xs->flush_node.prev, xs->flush_node.next);
157 xs->flush_node.prev = NULL;
158 }
159}
160
161static void *xsk_map_lookup_elem(struct bpf_map *map, void *key)
162{
163 return NULL;
164}
165
166static int xsk_map_update_elem(struct bpf_map *map, void *key, void *value,
167 u64 map_flags)
168{
169 struct xsk_map *m = container_of(map, struct xsk_map, map);
170 u32 i = *(u32 *)key, fd = *(u32 *)value;
171 struct xdp_sock *xs, *old_xs;
172 struct socket *sock;
173 int err;
174
175 if (unlikely(map_flags > BPF_EXIST))
176 return -EINVAL;
177 if (unlikely(i >= m->map.max_entries))
178 return -E2BIG;
179 if (unlikely(map_flags == BPF_NOEXIST))
180 return -EEXIST;
181
182 sock = sockfd_lookup(fd, &err);
183 if (!sock)
184 return err;
185
186 if (sock->sk->sk_family != PF_XDP) {
187 sockfd_put(sock);
188 return -EOPNOTSUPP;
189 }
190
191 xs = (struct xdp_sock *)sock->sk;
192
193 if (!xsk_is_setup_for_bpf_map(xs)) {
194 sockfd_put(sock);
195 return -EOPNOTSUPP;
196 }
197
198 sock_hold(sock->sk);
199
200 old_xs = xchg(&m->xsk_map[i], xs);
201 if (old_xs) {
202 /* Make sure we've flushed everything. */
203 synchronize_net();
204 sock_put((struct sock *)old_xs);
205 }
206
207 sockfd_put(sock);
208 return 0;
209}
210
211static int xsk_map_delete_elem(struct bpf_map *map, void *key)
212{
213 struct xsk_map *m = container_of(map, struct xsk_map, map);
214 struct xdp_sock *old_xs;
215 int k = *(u32 *)key;
216
217 if (k >= map->max_entries)
218 return -EINVAL;
219
220 old_xs = xchg(&m->xsk_map[k], NULL);
221 if (old_xs) {
222 /* Make sure we've flushed everything. */
223 synchronize_net();
224 sock_put((struct sock *)old_xs);
225 }
226
227 return 0;
228}
229
230const struct bpf_map_ops xsk_map_ops = {
231 .map_alloc = xsk_map_alloc,
232 .map_free = xsk_map_free,
233 .map_get_next_key = xsk_map_get_next_key,
234 .map_lookup_elem = xsk_map_lookup_elem,
235 .map_update_elem = xsk_map_update_elem,
236 .map_delete_elem = xsk_map_delete_elem,
237};
238
239
net/xdp/xsk.c+5−0
@@ -41,6 +41,11 @@ static struct xdp_sock *xdp_sk(struct sock *sk) · 40 unmodified lines
41 return (struct xdp_sock *)sk;41 return (struct xdp_sock *)sk;
42}42}
43 43
44bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs)
45{
46 return !!xs->rx;
47}
48
44static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)49static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
45{50{
46 u32 *id, len = xdp->data_end - xdp->data;51 u32 *id, len = xdp->data_end - xdp->data;