History

bpf: Add "live packet" mode for XDP in BPF_PROG_RUN

This adds support for running XDP programs through BPF_PROG_RUN in a mode that enables live packet processing of the resulting frames. Previous uses of BPF_PROG_RUN for XDP returned the XDP program return code and the modified packet data to userspace, which is useful for unit testing of XDP programs. The existing BPF_PROG_RUN for XDP allows userspace to set the ingress ifindex and RXQ number as part of the context object being passed to the kernel. This patch reuses that code, but adds a new mode with different semantics, which can be selected with the new BPF_F_TEST_XDP_LIVE_FRAMES flag. When running BPF_PROG_RUN in this mode, the XDP program return codes will be honoured: returning XDP_PASS will result in the frame being injected into the networking stack as if it came from the selected networking interface, while returning XDP_TX and XDP_REDIRECT will result in the frame being transmitted out that interface. XDP_TX is translated into an XDP_REDIRECT operation to the same interface, since the real XDP_TX action is only possible from within the network drivers themselves, not from the process context where BPF_PROG_RUN is executed. Internally, this new mode of operation creates a page pool instance while setting up the test run, and feeds pages from that into the XDP program. The setup cost of this is amortised over the number of repetitions specified by userspace. To support the performance testing use case, we further optimise the setup step so that all pages in the pool are pre-initialised with the packet data, and pre-computed context and xdp_frame objects stored at the start of each page. This makes it possible to entirely avoid touching the page content on each XDP program invocation, and enables sending up to 9 Mpps/core on my test box. Because the data pages are recycled by the page pool, and the test runner doesn't re-initialise them for each run, subsequent invocations of the XDP program will see the packet data in the state it was after the last time it ran on that particular page. This means that an XDP program that modifies the packet before redirecting it has to be careful about which assumptions it makes about the packet content, but that is only an issue for the most naively written programs. Enabling the new flag is only allowed when not setting ctx_out and data_out in the test specification, since using it means frames will be redirected somewhere else, so they can't be returned.

Toke Høiland-Jørgensen committed 4 years ago

Commit b530e9e1 parent 3399dd95 files · +328 −15Browse files

Unified · Split

5 files
  1. include/uapi/linux/bpf.h +3 −0
  2. kernel/bpf/Kconfig +1 −0
  3. kernel/bpf/syscall.c +1 −1
  4. net/bpf/test_run.c +320 −14
  5. tools/include/uapi/linux/bpf.h +3 −0
include/uapi/linux/bpf.h+3−0
@@ -1232,6 +1232,8 @@ enum { · 1231 unmodified lines
1232 1232
1233/* If set, run the test on the cpu specified by bpf_attr.test.cpu */1233/* If set, run the test on the cpu specified by bpf_attr.test.cpu */
1234#define BPF_F_TEST_RUN_ON_CPU (1U << 0)1234#define BPF_F_TEST_RUN_ON_CPU (1U << 0)
1235/* If set, XDP frames will be transmitted after processing */
1236#define BPF_F_TEST_XDP_LIVE_FRAMES (1U << 1)
1235 1237
1236/* type for BPF_ENABLE_STATS */1238/* type for BPF_ENABLE_STATS */
1237enum bpf_stats_type {1239enum bpf_stats_type {
@@ -1393,6 +1395,7 @@ union bpf_attr { · 155 unmodified lines
1393 __aligned_u64 ctx_out;1395 __aligned_u64 ctx_out;
1394 __u32 flags;1396 __u32 flags;
1395 __u32 cpu;1397 __u32 cpu;
1398 __u32 batch_size;
1396 } test;1399 } test;
1397 1400
1398 struct { /* anonymous struct used by BPF_*_GET_*_ID */1401 struct { /* anonymous struct used by BPF_*_GET_*_ID */
kernel/bpf/Kconfig+1−0
@@ -30,6 +30,7 @@ config BPF_SYSCALL · 29 unmodified lines
30 select TASKS_TRACE_RCU30 select TASKS_TRACE_RCU
31 select BINARY_PRINTF31 select BINARY_PRINTF
32 select NET_SOCK_MSG if NET32 select NET_SOCK_MSG if NET
33 select PAGE_POOL if NET
33 default n34 default n
34 help35 help
35 Enable the bpf() system call that allows to manipulate BPF programs36 Enable the bpf() system call that allows to manipulate BPF programs
kernel/bpf/syscall.c+1−1
@@ -3336,7 +3336,7 @@ static int bpf_prog_query(const union bpf_attr *attr, · 3335 unmodified lines
3336 }3336 }
3337}3337}
3338 3338
3339#define BPF_PROG_TEST_RUN_LAST_FIELD test.cpu3339#define BPF_PROG_TEST_RUN_LAST_FIELD test.batch_size
3340 3340
3341static int bpf_prog_test_run(const union bpf_attr *attr,3341static int bpf_prog_test_run(const union bpf_attr *attr,
3342 union bpf_attr __user *uattr)3342 union bpf_attr __user *uattr)
net/bpf/test_run.c+320−14
@@ -15,6 +15,7 @@ · 14 unmodified lines
15#include <net/sock.h>15#include <net/sock.h>
16#include <net/tcp.h>16#include <net/tcp.h>
17#include <net/net_namespace.h>17#include <net/net_namespace.h>
18#include <net/page_pool.h>
18#include <linux/error-injection.h>19#include <linux/error-injection.h>
19#include <linux/smp.h>20#include <linux/smp.h>
20#include <linux/sock_diag.h>21#include <linux/sock_diag.h>
@@ -53,10 +54,11 @@ static void bpf_test_timer_leave(struct bpf_test_timer *t) · 32 unmodified lines
53 rcu_read_unlock();54 rcu_read_unlock();
54}55}
55 56
56static bool bpf_test_timer_continue(struct bpf_test_timer *t, u32 repeat, int *err, u32 *duration)57static bool bpf_test_timer_continue(struct bpf_test_timer *t, int iterations,
58 u32 repeat, int *err, u32 *duration)
57 __must_hold(rcu)59 __must_hold(rcu)
58{60{
59 t->i++;61 t->i += iterations;
60 if (t->i >= repeat) {62 if (t->i >= repeat) {
61 /* We're done. */63 /* We're done. */
62 t->time_spent += ktime_get_ns() - t->time_start;64 t->time_spent += ktime_get_ns() - t->time_start;
@@ -88,6 +90,286 @@ reset: · 25 unmodified lines
88 return false;90 return false;
89}91}
90 92
93/* We put this struct at the head of each page with a context and frame
94 * initialised when the page is allocated, so we don't have to do this on each
95 * repetition of the test run.
96 */
97struct xdp_page_head {
98 struct xdp_buff orig_ctx;
99 struct xdp_buff ctx;
100 struct xdp_frame frm;
101 u8 data[];
102};
103
104struct xdp_test_data {
105 struct xdp_buff *orig_ctx;
106 struct xdp_rxq_info rxq;
107 struct net_device *dev;
108 struct page_pool *pp;
109 struct xdp_frame **frames;
110 struct sk_buff **skbs;
111 u32 batch_size;
112 u32 frame_cnt;
113};
114
115#define TEST_XDP_FRAME_SIZE (PAGE_SIZE - sizeof(struct xdp_page_head) \
116 - sizeof(struct skb_shared_info))
117#define TEST_XDP_MAX_BATCH 256
118
119static void xdp_test_run_init_page(struct page *page, void *arg)
120{
121 struct xdp_page_head *head = phys_to_virt(page_to_phys(page));
122 struct xdp_buff *new_ctx, *orig_ctx;
123 u32 headroom = XDP_PACKET_HEADROOM;
124 struct xdp_test_data *xdp = arg;
125 size_t frm_len, meta_len;
126 struct xdp_frame *frm;
127 void *data;
128
129 orig_ctx = xdp->orig_ctx;
130 frm_len = orig_ctx->data_end - orig_ctx->data_meta;
131 meta_len = orig_ctx->data - orig_ctx->data_meta;
132 headroom -= meta_len;
133
134 new_ctx = &head->ctx;
135 frm = &head->frm;
136 data = &head->data;
137 memcpy(data + headroom, orig_ctx->data_meta, frm_len);
138
139 xdp_init_buff(new_ctx, TEST_XDP_FRAME_SIZE, &xdp->rxq);
140 xdp_prepare_buff(new_ctx, data, headroom, frm_len, true);
141 new_ctx->data = new_ctx->data_meta + meta_len;
142
143 xdp_update_frame_from_buff(new_ctx, frm);
144 frm->mem = new_ctx->rxq->mem;
145
146 memcpy(&head->orig_ctx, new_ctx, sizeof(head->orig_ctx));
147}
148
149static int xdp_test_run_setup(struct xdp_test_data *xdp, struct xdp_buff *orig_ctx)
150{
151 struct xdp_mem_info mem = {};
152 struct page_pool *pp;
153 int err = -ENOMEM;
154 struct page_pool_params pp_params = {
155 .order = 0,
156 .flags = 0,
157 .pool_size = xdp->batch_size,
158 .nid = NUMA_NO_NODE,
159 .max_len = TEST_XDP_FRAME_SIZE,
160 .init_callback = xdp_test_run_init_page,
161 .init_arg = xdp,
162 };
163
164 xdp->frames = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL);
165 if (!xdp->frames)
166 return -ENOMEM;
167
168 xdp->skbs = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL);
169 if (!xdp->skbs)
170 goto err_skbs;
171
172 pp = page_pool_create(&pp_params);
173 if (IS_ERR(pp)) {
174 err = PTR_ERR(pp);
175 goto err_pp;
176 }
177
178 /* will copy 'mem.id' into pp->xdp_mem_id */
179 err = xdp_reg_mem_model(&mem, MEM_TYPE_PAGE_POOL, pp);
180 if (err)
181 goto err_mmodel;
182
183 xdp->pp = pp;
184
185 /* We create a 'fake' RXQ referencing the original dev, but with an
186 * xdp_mem_info pointing to our page_pool
187 */
188 xdp_rxq_info_reg(&xdp->rxq, orig_ctx->rxq->dev, 0, 0);
189 xdp->rxq.mem.type = MEM_TYPE_PAGE_POOL;
190 xdp->rxq.mem.id = pp->xdp_mem_id;
191 xdp->dev = orig_ctx->rxq->dev;
192 xdp->orig_ctx = orig_ctx;
193
194 return 0;
195
196err_mmodel:
197 page_pool_destroy(pp);
198err_pp:
199 kfree(xdp->skbs);
200err_skbs:
201 kfree(xdp->frames);
202 return err;
203}
204
205static void xdp_test_run_teardown(struct xdp_test_data *xdp)
206{
207 page_pool_destroy(xdp->pp);
208 kfree(xdp->frames);
209 kfree(xdp->skbs);
210}
211
212static bool ctx_was_changed(struct xdp_page_head *head)
213{
214 return head->orig_ctx.data != head->ctx.data ||
215 head->orig_ctx.data_meta != head->ctx.data_meta ||
216 head->orig_ctx.data_end != head->ctx.data_end;
217}
218
219static void reset_ctx(struct xdp_page_head *head)
220{
221 if (likely(!ctx_was_changed(head)))
222 return;
223
224 head->ctx.data = head->orig_ctx.data;
225 head->ctx.data_meta = head->orig_ctx.data_meta;
226 head->ctx.data_end = head->orig_ctx.data_end;
227 xdp_update_frame_from_buff(&head->ctx, &head->frm);
228}
229
230static int xdp_recv_frames(struct xdp_frame **frames, int nframes,
231 struct sk_buff **skbs,
232 struct net_device *dev)
233{
234 gfp_t gfp = __GFP_ZERO | GFP_ATOMIC;
235 int i, n;
236 LIST_HEAD(list);
237
238 n = kmem_cache_alloc_bulk(skbuff_head_cache, gfp, nframes, (void **)skbs);
239 if (unlikely(n == 0)) {
240 for (i = 0; i < nframes; i++)
241 xdp_return_frame(frames[i]);
242 return -ENOMEM;
243 }
244
245 for (i = 0; i < nframes; i++) {
246 struct xdp_frame *xdpf = frames[i];
247 struct sk_buff *skb = skbs[i];
248
249 skb = __xdp_build_skb_from_frame(xdpf, skb, dev);
250 if (!skb) {
251 xdp_return_frame(xdpf);
252 continue;
253 }
254
255 list_add_tail(&skb->list, &list);
256 }
257 netif_receive_skb_list(&list);
258
259 return 0;
260}
261
262static int xdp_test_run_batch(struct xdp_test_data *xdp, struct bpf_prog *prog,
263 u32 repeat)
264{
265 struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
266 int err = 0, act, ret, i, nframes = 0, batch_sz;
267 struct xdp_frame **frames = xdp->frames;
268 struct xdp_page_head *head;
269 struct xdp_frame *frm;
270 bool redirect = false;
271 struct xdp_buff *ctx;
272 struct page *page;
273
274 batch_sz = min_t(u32, repeat, xdp->batch_size);
275
276 local_bh_disable();
277 xdp_set_return_frame_no_direct();
278
279 for (i = 0; i < batch_sz; i++) {
280 page = page_pool_dev_alloc_pages(xdp->pp);
281 if (!page) {
282 err = -ENOMEM;
283 goto out;
284 }
285
286 head = phys_to_virt(page_to_phys(page));
287 reset_ctx(head);
288 ctx = &head->ctx;
289 frm = &head->frm;
290 xdp->frame_cnt++;
291
292 act = bpf_prog_run_xdp(prog, ctx);
293
294 /* if program changed pkt bounds we need to update the xdp_frame */
295 if (unlikely(ctx_was_changed(head))) {
296 ret = xdp_update_frame_from_buff(ctx, frm);
297 if (ret) {
298 xdp_return_buff(ctx);
299 continue;
300 }
301 }
302
303 switch (act) {
304 case XDP_TX:
305 /* we can't do a real XDP_TX since we're not in the
306 * driver, so turn it into a REDIRECT back to the same
307 * index
308 */
309 ri->tgt_index = xdp->dev->ifindex;
310 ri->map_id = INT_MAX;
311 ri->map_type = BPF_MAP_TYPE_UNSPEC;
312 fallthrough;
313 case XDP_REDIRECT:
314 redirect = true;
315 ret = xdp_do_redirect_frame(xdp->dev, ctx, frm, prog);
316 if (ret)
317 xdp_return_buff(ctx);
318 break;
319 case XDP_PASS:
320 frames[nframes++] = frm;
321 break;
322 default:
323 bpf_warn_invalid_xdp_action(NULL, prog, act);
324 fallthrough;
325 case XDP_DROP:
326 xdp_return_buff(ctx);
327 break;
328 }
329 }
330
331out:
332 if (redirect)
333 xdp_do_flush();
334 if (nframes) {
335 ret = xdp_recv_frames(frames, nframes, xdp->skbs, xdp->dev);
336 if (ret)
337 err = ret;
338 }
339
340 xdp_clear_return_frame_no_direct();
341 local_bh_enable();
342 return err;
343}
344
345static int bpf_test_run_xdp_live(struct bpf_prog *prog, struct xdp_buff *ctx,
346 u32 repeat, u32 batch_size, u32 *time)
347
348{
349 struct xdp_test_data xdp = { .batch_size = batch_size };
350 struct bpf_test_timer t = { .mode = NO_MIGRATE };
351 int ret;
352
353 if (!repeat)
354 repeat = 1;
355
356 ret = xdp_test_run_setup(&xdp, ctx);
357 if (ret)
358 return ret;
359
360 bpf_test_timer_enter(&t);
361 do {
362 xdp.frame_cnt = 0;
363 ret = xdp_test_run_batch(&xdp, prog, repeat - t.i);
364 if (unlikely(ret < 0))
365 break;
366 } while (bpf_test_timer_continue(&t, xdp.frame_cnt, repeat, &ret, time));
367 bpf_test_timer_leave(&t);
368
369 xdp_test_run_teardown(&xdp);
370 return ret;
371}
372
91static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat,373static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat,
92 u32 *retval, u32 *time, bool xdp)374 u32 *retval, u32 *time, bool xdp)
93{375{
@@ -119,7 +401,7 @@ static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, · 25 unmodified lines
119 *retval = bpf_prog_run_xdp(prog, ctx);401 *retval = bpf_prog_run_xdp(prog, ctx);
120 else402 else
121 *retval = bpf_prog_run(prog, ctx);403 *retval = bpf_prog_run(prog, ctx);
122 } while (bpf_test_timer_continue(&t, repeat, &ret, time));404 } while (bpf_test_timer_continue(&t, 1, repeat, &ret, time));
123 bpf_reset_run_ctx(old_ctx);405 bpf_reset_run_ctx(old_ctx);
124 bpf_test_timer_leave(&t);406 bpf_test_timer_leave(&t);
125 407
@@ -446,7 +728,7 @@ int bpf_prog_test_run_tracing(struct bpf_prog *prog, · 320 unmodified lines
446 int b = 2, err = -EFAULT;728 int b = 2, err = -EFAULT;
447 u32 retval = 0;729 u32 retval = 0;
448 730
449 if (kattr->test.flags || kattr->test.cpu)731 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)
450 return -EINVAL;732 return -EINVAL;
451 733
452 switch (prog->expected_attach_type) {734 switch (prog->expected_attach_type) {
@@ -510,7 +792,7 @@ int bpf_prog_test_run_raw_tp(struct bpf_prog *prog, · 57 unmodified lines
510 /* doesn't support data_in/out, ctx_out, duration, or repeat */792 /* doesn't support data_in/out, ctx_out, duration, or repeat */
511 if (kattr->test.data_in || kattr->test.data_out ||793 if (kattr->test.data_in || kattr->test.data_out ||
512 kattr->test.ctx_out || kattr->test.duration ||794 kattr->test.ctx_out || kattr->test.duration ||
513 kattr->test.repeat)795 kattr->test.repeat || kattr->test.batch_size)
514 return -EINVAL;796 return -EINVAL;
515 797
516 if (ctx_size_in < prog->aux->max_ctx_offset ||798 if (ctx_size_in < prog->aux->max_ctx_offset ||
@@ -741,7 +1023,7 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, · 224 unmodified lines
741 void *data;1023 void *data;
742 int ret;1024 int ret;
743 1025
744 if (kattr->test.flags || kattr->test.cpu)1026 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)
745 return -EINVAL;1027 return -EINVAL;
746 1028
747 data = bpf_test_init(kattr, kattr->test.data_size_in,1029 data = bpf_test_init(kattr, kattr->test.data_size_in,
@@ -922,7 +1204,9 @@ static void xdp_convert_buff_to_md(struct xdp_buff *xdp, struct xdp_md *xdp_md) · 174 unmodified lines
922int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,1204int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
923 union bpf_attr __user *uattr)1205 union bpf_attr __user *uattr)
924{1206{
1207 bool do_live = (kattr->test.flags & BPF_F_TEST_XDP_LIVE_FRAMES);
925 u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));1208 u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1209 u32 batch_size = kattr->test.batch_size;
926 u32 size = kattr->test.data_size_in;1210 u32 size = kattr->test.data_size_in;
927 u32 headroom = XDP_PACKET_HEADROOM;1211 u32 headroom = XDP_PACKET_HEADROOM;
928 u32 retval, duration, max_data_sz;1212 u32 retval, duration, max_data_sz;
@@ -938,6 +1222,18 @@ int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, · 9 unmodified lines
938 prog->expected_attach_type == BPF_XDP_CPUMAP)1222 prog->expected_attach_type == BPF_XDP_CPUMAP)
939 return -EINVAL;1223 return -EINVAL;
940 1224
1225 if (kattr->test.flags & ~BPF_F_TEST_XDP_LIVE_FRAMES)
1226 return -EINVAL;
1227
1228 if (do_live) {
1229 if (!batch_size)
1230 batch_size = NAPI_POLL_WEIGHT;
1231 else if (batch_size > TEST_XDP_MAX_BATCH)
1232 return -E2BIG;
1233 } else if (batch_size) {
1234 return -EINVAL;
1235 }
1236
941 ctx = bpf_ctx_init(kattr, sizeof(struct xdp_md));1237 ctx = bpf_ctx_init(kattr, sizeof(struct xdp_md));
942 if (IS_ERR(ctx))1238 if (IS_ERR(ctx))
943 return PTR_ERR(ctx);1239 return PTR_ERR(ctx);
@@ -946,14 +1242,20 @@ int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, · 2 unmodified lines
946 /* There can't be user provided data before the meta data */1242 /* There can't be user provided data before the meta data */
947 if (ctx->data_meta || ctx->data_end != size ||1243 if (ctx->data_meta || ctx->data_end != size ||
948 ctx->data > ctx->data_end ||1244 ctx->data > ctx->data_end ||
949 unlikely(xdp_metalen_invalid(ctx->data)))1245 unlikely(xdp_metalen_invalid(ctx->data)) ||
1246 (do_live && (kattr->test.data_out || kattr->test.ctx_out)))
950 goto free_ctx;1247 goto free_ctx;
951 /* Meta data is allocated from the headroom */1248 /* Meta data is allocated from the headroom */
952 headroom -= ctx->data;1249 headroom -= ctx->data;
953 }1250 }
954 1251
955 max_data_sz = 4096 - headroom - tailroom;1252 max_data_sz = 4096 - headroom - tailroom;
956 size = min_t(u32, size, max_data_sz);1253 if (size > max_data_sz) {
1254 /* disallow live data mode for jumbo frames */
1255 if (do_live)
1256 goto free_ctx;
1257 size = max_data_sz;
1258 }
957 1259
958 data = bpf_test_init(kattr, size, max_data_sz, headroom, tailroom);1260 data = bpf_test_init(kattr, size, max_data_sz, headroom, tailroom);
959 if (IS_ERR(data)) {1261 if (IS_ERR(data)) {
@@ -1011,7 +1313,10 @@ int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, · 51 unmodified lines
1011 if (repeat > 1)1313 if (repeat > 1)
1012 bpf_prog_change_xdp(NULL, prog);1314 bpf_prog_change_xdp(NULL, prog);
1013 1315
1014 ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true);1316 if (do_live)
1317 ret = bpf_test_run_xdp_live(prog, &xdp, repeat, batch_size, &duration);
1318 else
1319 ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true);
1015 /* We convert the xdp_buff back to an xdp_md before checking the return1320 /* We convert the xdp_buff back to an xdp_md before checking the return
1016 * code so the reference count of any held netdevice will be decremented1321 * code so the reference count of any held netdevice will be decremented
1017 * even if the test run failed.1322 * even if the test run failed.
@@ -1073,7 +1378,7 @@ int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, · 55 unmodified lines
1073 if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR)1378 if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR)
1074 return -EINVAL;1379 return -EINVAL;
1075 1380
1076 if (kattr->test.flags || kattr->test.cpu)1381 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)
1077 return -EINVAL;1382 return -EINVAL;
1078 1383
1079 if (size < ETH_HLEN)1384 if (size < ETH_HLEN)
@@ -1108,7 +1413,7 @@ int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, · 28 unmodified lines
1108 do {1413 do {
1109 retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN,1414 retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN,
1110 size, flags);1415 size, flags);
1111 } while (bpf_test_timer_continue(&t, repeat, &ret, &duration));1416 } while (bpf_test_timer_continue(&t, 1, repeat, &ret, &duration));
1112 bpf_test_timer_leave(&t);1417 bpf_test_timer_leave(&t);
1113 1418
1114 if (ret < 0)1419 if (ret < 0)
@@ -1140,7 +1445,7 @@ int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kat · 25 unmodified lines
1140 if (prog->type != BPF_PROG_TYPE_SK_LOOKUP)1445 if (prog->type != BPF_PROG_TYPE_SK_LOOKUP)
1141 return -EINVAL;1446 return -EINVAL;
1142 1447
1143 if (kattr->test.flags || kattr->test.cpu)1448 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)
1144 return -EINVAL;1449 return -EINVAL;
1145 1450
1146 if (kattr->test.data_in || kattr->test.data_size_in || kattr->test.data_out ||1451 if (kattr->test.data_in || kattr->test.data_size_in || kattr->test.data_out ||
@@ -1203,7 +1508,7 @@ int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kat · 56 unmodified lines
1203 do {1508 do {
1204 ctx.selected_sk = NULL;1509 ctx.selected_sk = NULL;
1205 retval = BPF_PROG_SK_LOOKUP_RUN_ARRAY(progs, ctx, bpf_prog_run);1510 retval = BPF_PROG_SK_LOOKUP_RUN_ARRAY(progs, ctx, bpf_prog_run);
1206 } while (bpf_test_timer_continue(&t, repeat, &ret, &duration));1511 } while (bpf_test_timer_continue(&t, 1, repeat, &ret, &duration));
1207 bpf_test_timer_leave(&t);1512 bpf_test_timer_leave(&t);
1208 1513
1209 if (ret < 0)1514 if (ret < 0)
@@ -1242,7 +1547,8 @@ int bpf_prog_test_run_syscall(struct bpf_prog *prog, · 32 unmodified lines
1242 /* doesn't support data_in/out, ctx_out, duration, or repeat or flags */1547 /* doesn't support data_in/out, ctx_out, duration, or repeat or flags */
1243 if (kattr->test.data_in || kattr->test.data_out ||1548 if (kattr->test.data_in || kattr->test.data_out ||
1244 kattr->test.ctx_out || kattr->test.duration ||1549 kattr->test.ctx_out || kattr->test.duration ||
1245 kattr->test.repeat || kattr->test.flags)1550 kattr->test.repeat || kattr->test.flags ||
1551 kattr->test.batch_size)
1246 return -EINVAL;1552 return -EINVAL;
1247 1553
1248 if (ctx_size_in < prog->aux->max_ctx_offset ||1554 if (ctx_size_in < prog->aux->max_ctx_offset ||
tools/include/uapi/linux/bpf.h+3−0
@@ -1232,6 +1232,8 @@ enum { · 1231 unmodified lines
1232 1232
1233/* If set, run the test on the cpu specified by bpf_attr.test.cpu */1233/* If set, run the test on the cpu specified by bpf_attr.test.cpu */
1234#define BPF_F_TEST_RUN_ON_CPU (1U << 0)1234#define BPF_F_TEST_RUN_ON_CPU (1U << 0)
1235/* If set, XDP frames will be transmitted after processing */
1236#define BPF_F_TEST_XDP_LIVE_FRAMES (1U << 1)
1235 1237
1236/* type for BPF_ENABLE_STATS */1238/* type for BPF_ENABLE_STATS */
1237enum bpf_stats_type {1239enum bpf_stats_type {
@@ -1393,6 +1395,7 @@ union bpf_attr { · 155 unmodified lines
1393 __aligned_u64 ctx_out;1395 __aligned_u64 ctx_out;
1394 __u32 flags;1396 __u32 flags;
1395 __u32 cpu;1397 __u32 cpu;
1398 __u32 batch_size;
1396 } test;1399 } test;
1397 1400
1398 struct { /* anonymous struct used by BPF_*_GET_*_ID */1401 struct { /* anonymous struct used by BPF_*_GET_*_ID */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t