| @@ -15,6 +15,7 @@ · 14 unmodified lines |
| 15 | #include <net/sock.h> | 15 | #include <net/sock.h> |
| 16 | #include <net/tcp.h> | 16 | #include <net/tcp.h> |
| 17 | #include <net/net_namespace.h> | 17 | #include <net/net_namespace.h> |
| | 18 | #include <net/page_pool.h> |
| 18 | #include <linux/error-injection.h> | 19 | #include <linux/error-injection.h> |
| 19 | #include <linux/smp.h> | 20 | #include <linux/smp.h> |
| 20 | #include <linux/sock_diag.h> | 21 | #include <linux/sock_diag.h> |
| @@ -53,10 +54,11 @@ static void bpf_test_timer_leave(struct bpf_test_timer *t) · 32 unmodified lines |
| 53 | rcu_read_unlock(); | 54 | rcu_read_unlock(); |
| 54 | } | 55 | } |
| 55 | | 56 | |
| 56 | static bool bpf_test_timer_continue(struct bpf_test_timer *t, u32 repeat, int *err, u32 *duration) | 57 | static bool bpf_test_timer_continue(struct bpf_test_timer *t, int iterations, |
| | 58 | u32 repeat, int *err, u32 *duration) |
| 57 | __must_hold(rcu) | 59 | __must_hold(rcu) |
| 58 | { | 60 | { |
| 59 | t->i++; | 61 | t->i += iterations; |
| 60 | if (t->i >= repeat) { | 62 | if (t->i >= repeat) { |
| 61 | /* We're done. */ | 63 | /* We're done. */ |
| 62 | t->time_spent += ktime_get_ns() - t->time_start; | 64 | t->time_spent += ktime_get_ns() - t->time_start; |
| @@ -88,6 +90,286 @@ reset: · 25 unmodified lines |
| 88 | return false; | 90 | return false; |
| 89 | } | 91 | } |
| 90 | | 92 | |
| | 93 | /* We put this struct at the head of each page with a context and frame |
| | 94 | * initialised when the page is allocated, so we don't have to do this on each |
| | 95 | * repetition of the test run. |
| | 96 | */ |
| | 97 | struct xdp_page_head { |
| | 98 | struct xdp_buff orig_ctx; |
| | 99 | struct xdp_buff ctx; |
| | 100 | struct xdp_frame frm; |
| | 101 | u8 data[]; |
| | 102 | }; |
| | 103 | |
| | 104 | struct xdp_test_data { |
| | 105 | struct xdp_buff *orig_ctx; |
| | 106 | struct xdp_rxq_info rxq; |
| | 107 | struct net_device *dev; |
| | 108 | struct page_pool *pp; |
| | 109 | struct xdp_frame **frames; |
| | 110 | struct sk_buff **skbs; |
| | 111 | u32 batch_size; |
| | 112 | u32 frame_cnt; |
| | 113 | }; |
| | 114 | |
| | 115 | #define TEST_XDP_FRAME_SIZE (PAGE_SIZE - sizeof(struct xdp_page_head) \ |
| | 116 | - sizeof(struct skb_shared_info)) |
| | 117 | #define TEST_XDP_MAX_BATCH 256 |
| | 118 | |
| | 119 | static void xdp_test_run_init_page(struct page *page, void *arg) |
| | 120 | { |
| | 121 | struct xdp_page_head *head = phys_to_virt(page_to_phys(page)); |
| | 122 | struct xdp_buff *new_ctx, *orig_ctx; |
| | 123 | u32 headroom = XDP_PACKET_HEADROOM; |
| | 124 | struct xdp_test_data *xdp = arg; |
| | 125 | size_t frm_len, meta_len; |
| | 126 | struct xdp_frame *frm; |
| | 127 | void *data; |
| | 128 | |
| | 129 | orig_ctx = xdp->orig_ctx; |
| | 130 | frm_len = orig_ctx->data_end - orig_ctx->data_meta; |
| | 131 | meta_len = orig_ctx->data - orig_ctx->data_meta; |
| | 132 | headroom -= meta_len; |
| | 133 | |
| | 134 | new_ctx = &head->ctx; |
| | 135 | frm = &head->frm; |
| | 136 | data = &head->data; |
| | 137 | memcpy(data + headroom, orig_ctx->data_meta, frm_len); |
| | 138 | |
| | 139 | xdp_init_buff(new_ctx, TEST_XDP_FRAME_SIZE, &xdp->rxq); |
| | 140 | xdp_prepare_buff(new_ctx, data, headroom, frm_len, true); |
| | 141 | new_ctx->data = new_ctx->data_meta + meta_len; |
| | 142 | |
| | 143 | xdp_update_frame_from_buff(new_ctx, frm); |
| | 144 | frm->mem = new_ctx->rxq->mem; |
| | 145 | |
| | 146 | memcpy(&head->orig_ctx, new_ctx, sizeof(head->orig_ctx)); |
| | 147 | } |
| | 148 | |
| | 149 | static int xdp_test_run_setup(struct xdp_test_data *xdp, struct xdp_buff *orig_ctx) |
| | 150 | { |
| | 151 | struct xdp_mem_info mem = {}; |
| | 152 | struct page_pool *pp; |
| | 153 | int err = -ENOMEM; |
| | 154 | struct page_pool_params pp_params = { |
| | 155 | .order = 0, |
| | 156 | .flags = 0, |
| | 157 | .pool_size = xdp->batch_size, |
| | 158 | .nid = NUMA_NO_NODE, |
| | 159 | .max_len = TEST_XDP_FRAME_SIZE, |
| | 160 | .init_callback = xdp_test_run_init_page, |
| | 161 | .init_arg = xdp, |
| | 162 | }; |
| | 163 | |
| | 164 | xdp->frames = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL); |
| | 165 | if (!xdp->frames) |
| | 166 | return -ENOMEM; |
| | 167 | |
| | 168 | xdp->skbs = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL); |
| | 169 | if (!xdp->skbs) |
| | 170 | goto err_skbs; |
| | 171 | |
| | 172 | pp = page_pool_create(&pp_params); |
| | 173 | if (IS_ERR(pp)) { |
| | 174 | err = PTR_ERR(pp); |
| | 175 | goto err_pp; |
| | 176 | } |
| | 177 | |
| | 178 | /* will copy 'mem.id' into pp->xdp_mem_id */ |
| | 179 | err = xdp_reg_mem_model(&mem, MEM_TYPE_PAGE_POOL, pp); |
| | 180 | if (err) |
| | 181 | goto err_mmodel; |
| | 182 | |
| | 183 | xdp->pp = pp; |
| | 184 | |
| | 185 | /* We create a 'fake' RXQ referencing the original dev, but with an |
| | 186 | * xdp_mem_info pointing to our page_pool |
| | 187 | */ |
| | 188 | xdp_rxq_info_reg(&xdp->rxq, orig_ctx->rxq->dev, 0, 0); |
| | 189 | xdp->rxq.mem.type = MEM_TYPE_PAGE_POOL; |
| | 190 | xdp->rxq.mem.id = pp->xdp_mem_id; |
| | 191 | xdp->dev = orig_ctx->rxq->dev; |
| | 192 | xdp->orig_ctx = orig_ctx; |
| | 193 | |
| | 194 | return 0; |
| | 195 | |
| | 196 | err_mmodel: |
| | 197 | page_pool_destroy(pp); |
| | 198 | err_pp: |
| | 199 | kfree(xdp->skbs); |
| | 200 | err_skbs: |
| | 201 | kfree(xdp->frames); |
| | 202 | return err; |
| | 203 | } |
| | 204 | |
| | 205 | static void xdp_test_run_teardown(struct xdp_test_data *xdp) |
| | 206 | { |
| | 207 | page_pool_destroy(xdp->pp); |
| | 208 | kfree(xdp->frames); |
| | 209 | kfree(xdp->skbs); |
| | 210 | } |
| | 211 | |
| | 212 | static bool ctx_was_changed(struct xdp_page_head *head) |
| | 213 | { |
| | 214 | return head->orig_ctx.data != head->ctx.data || |
| | 215 | head->orig_ctx.data_meta != head->ctx.data_meta || |
| | 216 | head->orig_ctx.data_end != head->ctx.data_end; |
| | 217 | } |
| | 218 | |
| | 219 | static void reset_ctx(struct xdp_page_head *head) |
| | 220 | { |
| | 221 | if (likely(!ctx_was_changed(head))) |
| | 222 | return; |
| | 223 | |
| | 224 | head->ctx.data = head->orig_ctx.data; |
| | 225 | head->ctx.data_meta = head->orig_ctx.data_meta; |
| | 226 | head->ctx.data_end = head->orig_ctx.data_end; |
| | 227 | xdp_update_frame_from_buff(&head->ctx, &head->frm); |
| | 228 | } |
| | 229 | |
| | 230 | static int xdp_recv_frames(struct xdp_frame **frames, int nframes, |
| | 231 | struct sk_buff **skbs, |
| | 232 | struct net_device *dev) |
| | 233 | { |
| | 234 | gfp_t gfp = __GFP_ZERO | GFP_ATOMIC; |
| | 235 | int i, n; |
| | 236 | LIST_HEAD(list); |
| | 237 | |
| | 238 | n = kmem_cache_alloc_bulk(skbuff_head_cache, gfp, nframes, (void **)skbs); |
| | 239 | if (unlikely(n == 0)) { |
| | 240 | for (i = 0; i < nframes; i++) |
| | 241 | xdp_return_frame(frames[i]); |
| | 242 | return -ENOMEM; |
| | 243 | } |
| | 244 | |
| | 245 | for (i = 0; i < nframes; i++) { |
| | 246 | struct xdp_frame *xdpf = frames[i]; |
| | 247 | struct sk_buff *skb = skbs[i]; |
| | 248 | |
| | 249 | skb = __xdp_build_skb_from_frame(xdpf, skb, dev); |
| | 250 | if (!skb) { |
| | 251 | xdp_return_frame(xdpf); |
| | 252 | continue; |
| | 253 | } |
| | 254 | |
| | 255 | list_add_tail(&skb->list, &list); |
| | 256 | } |
| | 257 | netif_receive_skb_list(&list); |
| | 258 | |
| | 259 | return 0; |
| | 260 | } |
| | 261 | |
| | 262 | static int xdp_test_run_batch(struct xdp_test_data *xdp, struct bpf_prog *prog, |
| | 263 | u32 repeat) |
| | 264 | { |
| | 265 | struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); |
| | 266 | int err = 0, act, ret, i, nframes = 0, batch_sz; |
| | 267 | struct xdp_frame **frames = xdp->frames; |
| | 268 | struct xdp_page_head *head; |
| | 269 | struct xdp_frame *frm; |
| | 270 | bool redirect = false; |
| | 271 | struct xdp_buff *ctx; |
| | 272 | struct page *page; |
| | 273 | |
| | 274 | batch_sz = min_t(u32, repeat, xdp->batch_size); |
| | 275 | |
| | 276 | local_bh_disable(); |
| | 277 | xdp_set_return_frame_no_direct(); |
| | 278 | |
| | 279 | for (i = 0; i < batch_sz; i++) { |
| | 280 | page = page_pool_dev_alloc_pages(xdp->pp); |
| | 281 | if (!page) { |
| | 282 | err = -ENOMEM; |
| | 283 | goto out; |
| | 284 | } |
| | 285 | |
| | 286 | head = phys_to_virt(page_to_phys(page)); |
| | 287 | reset_ctx(head); |
| | 288 | ctx = &head->ctx; |
| | 289 | frm = &head->frm; |
| | 290 | xdp->frame_cnt++; |
| | 291 | |
| | 292 | act = bpf_prog_run_xdp(prog, ctx); |
| | 293 | |
| | 294 | /* if program changed pkt bounds we need to update the xdp_frame */ |
| | 295 | if (unlikely(ctx_was_changed(head))) { |
| | 296 | ret = xdp_update_frame_from_buff(ctx, frm); |
| | 297 | if (ret) { |
| | 298 | xdp_return_buff(ctx); |
| | 299 | continue; |
| | 300 | } |
| | 301 | } |
| | 302 | |
| | 303 | switch (act) { |
| | 304 | case XDP_TX: |
| | 305 | /* we can't do a real XDP_TX since we're not in the |
| | 306 | * driver, so turn it into a REDIRECT back to the same |
| | 307 | * index |
| | 308 | */ |
| | 309 | ri->tgt_index = xdp->dev->ifindex; |
| | 310 | ri->map_id = INT_MAX; |
| | 311 | ri->map_type = BPF_MAP_TYPE_UNSPEC; |
| | 312 | fallthrough; |
| | 313 | case XDP_REDIRECT: |
| | 314 | redirect = true; |
| | 315 | ret = xdp_do_redirect_frame(xdp->dev, ctx, frm, prog); |
| | 316 | if (ret) |
| | 317 | xdp_return_buff(ctx); |
| | 318 | break; |
| | 319 | case XDP_PASS: |
| | 320 | frames[nframes++] = frm; |
| | 321 | break; |
| | 322 | default: |
| | 323 | bpf_warn_invalid_xdp_action(NULL, prog, act); |
| | 324 | fallthrough; |
| | 325 | case XDP_DROP: |
| | 326 | xdp_return_buff(ctx); |
| | 327 | break; |
| | 328 | } |
| | 329 | } |
| | 330 | |
| | 331 | out: |
| | 332 | if (redirect) |
| | 333 | xdp_do_flush(); |
| | 334 | if (nframes) { |
| | 335 | ret = xdp_recv_frames(frames, nframes, xdp->skbs, xdp->dev); |
| | 336 | if (ret) |
| | 337 | err = ret; |
| | 338 | } |
| | 339 | |
| | 340 | xdp_clear_return_frame_no_direct(); |
| | 341 | local_bh_enable(); |
| | 342 | return err; |
| | 343 | } |
| | 344 | |
| | 345 | static int bpf_test_run_xdp_live(struct bpf_prog *prog, struct xdp_buff *ctx, |
| | 346 | u32 repeat, u32 batch_size, u32 *time) |
| | 347 | |
| | 348 | { |
| | 349 | struct xdp_test_data xdp = { .batch_size = batch_size }; |
| | 350 | struct bpf_test_timer t = { .mode = NO_MIGRATE }; |
| | 351 | int ret; |
| | 352 | |
| | 353 | if (!repeat) |
| | 354 | repeat = 1; |
| | 355 | |
| | 356 | ret = xdp_test_run_setup(&xdp, ctx); |
| | 357 | if (ret) |
| | 358 | return ret; |
| | 359 | |
| | 360 | bpf_test_timer_enter(&t); |
| | 361 | do { |
| | 362 | xdp.frame_cnt = 0; |
| | 363 | ret = xdp_test_run_batch(&xdp, prog, repeat - t.i); |
| | 364 | if (unlikely(ret < 0)) |
| | 365 | break; |
| | 366 | } while (bpf_test_timer_continue(&t, xdp.frame_cnt, repeat, &ret, time)); |
| | 367 | bpf_test_timer_leave(&t); |
| | 368 | |
| | 369 | xdp_test_run_teardown(&xdp); |
| | 370 | return ret; |
| | 371 | } |
| | 372 | |
| 91 | static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, | 373 | static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, |
| 92 | u32 *retval, u32 *time, bool xdp) | 374 | u32 *retval, u32 *time, bool xdp) |
| 93 | { | 375 | { |
| @@ -119,7 +401,7 @@ static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, · 25 unmodified lines |
| 119 | *retval = bpf_prog_run_xdp(prog, ctx); | 401 | *retval = bpf_prog_run_xdp(prog, ctx); |
| 120 | else | 402 | else |
| 121 | *retval = bpf_prog_run(prog, ctx); | 403 | *retval = bpf_prog_run(prog, ctx); |
| 122 | } while (bpf_test_timer_continue(&t, repeat, &ret, time)); | 404 | } while (bpf_test_timer_continue(&t, 1, repeat, &ret, time)); |
| 123 | bpf_reset_run_ctx(old_ctx); | 405 | bpf_reset_run_ctx(old_ctx); |
| 124 | bpf_test_timer_leave(&t); | 406 | bpf_test_timer_leave(&t); |
| 125 | | 407 | |
| @@ -446,7 +728,7 @@ int bpf_prog_test_run_tracing(struct bpf_prog *prog, · 320 unmodified lines |
| 446 | int b = 2, err = -EFAULT; | 728 | int b = 2, err = -EFAULT; |
| 447 | u32 retval = 0; | 729 | u32 retval = 0; |
| 448 | | 730 | |
| 449 | if (kattr->test.flags || kattr->test.cpu) | 731 | if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) |
| 450 | return -EINVAL; | 732 | return -EINVAL; |
| 451 | | 733 | |
| 452 | switch (prog->expected_attach_type) { | 734 | switch (prog->expected_attach_type) { |
| @@ -510,7 +792,7 @@ int bpf_prog_test_run_raw_tp(struct bpf_prog *prog, · 57 unmodified lines |
| 510 | /* doesn't support data_in/out, ctx_out, duration, or repeat */ | 792 | /* doesn't support data_in/out, ctx_out, duration, or repeat */ |
| 511 | if (kattr->test.data_in || kattr->test.data_out || | 793 | if (kattr->test.data_in || kattr->test.data_out || |
| 512 | kattr->test.ctx_out || kattr->test.duration || | 794 | kattr->test.ctx_out || kattr->test.duration || |
| 513 | kattr->test.repeat) | 795 | kattr->test.repeat || kattr->test.batch_size) |
| 514 | return -EINVAL; | 796 | return -EINVAL; |
| 515 | | 797 | |
| 516 | if (ctx_size_in < prog->aux->max_ctx_offset || | 798 | if (ctx_size_in < prog->aux->max_ctx_offset || |
| @@ -741,7 +1023,7 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, · 224 unmodified lines |
| 741 | void *data; | 1023 | void *data; |
| 742 | int ret; | 1024 | int ret; |
| 743 | | 1025 | |
| 744 | if (kattr->test.flags || kattr->test.cpu) | 1026 | if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) |
| 745 | return -EINVAL; | 1027 | return -EINVAL; |
| 746 | | 1028 | |
| 747 | data = bpf_test_init(kattr, kattr->test.data_size_in, | 1029 | data = bpf_test_init(kattr, kattr->test.data_size_in, |
| @@ -922,7 +1204,9 @@ static void xdp_convert_buff_to_md(struct xdp_buff *xdp, struct xdp_md *xdp_md) · 174 unmodified lines |
| 922 | int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, | 1204 | int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, |
| 923 | union bpf_attr __user *uattr) | 1205 | union bpf_attr __user *uattr) |
| 924 | { | 1206 | { |
| | 1207 | bool do_live = (kattr->test.flags & BPF_F_TEST_XDP_LIVE_FRAMES); |
| 925 | u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); | 1208 | u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); |
| | 1209 | u32 batch_size = kattr->test.batch_size; |
| 926 | u32 size = kattr->test.data_size_in; | 1210 | u32 size = kattr->test.data_size_in; |
| 927 | u32 headroom = XDP_PACKET_HEADROOM; | 1211 | u32 headroom = XDP_PACKET_HEADROOM; |
| 928 | u32 retval, duration, max_data_sz; | 1212 | u32 retval, duration, max_data_sz; |
| @@ -938,6 +1222,18 @@ int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, · 9 unmodified lines |
| 938 | prog->expected_attach_type == BPF_XDP_CPUMAP) | 1222 | prog->expected_attach_type == BPF_XDP_CPUMAP) |
| 939 | return -EINVAL; | 1223 | return -EINVAL; |
| 940 | | 1224 | |
| | 1225 | if (kattr->test.flags & ~BPF_F_TEST_XDP_LIVE_FRAMES) |
| | 1226 | return -EINVAL; |
| | 1227 | |
| | 1228 | if (do_live) { |
| | 1229 | if (!batch_size) |
| | 1230 | batch_size = NAPI_POLL_WEIGHT; |
| | 1231 | else if (batch_size > TEST_XDP_MAX_BATCH) |
| | 1232 | return -E2BIG; |
| | 1233 | } else if (batch_size) { |
| | 1234 | return -EINVAL; |
| | 1235 | } |
| | 1236 | |
| 941 | ctx = bpf_ctx_init(kattr, sizeof(struct xdp_md)); | 1237 | ctx = bpf_ctx_init(kattr, sizeof(struct xdp_md)); |
| 942 | if (IS_ERR(ctx)) | 1238 | if (IS_ERR(ctx)) |
| 943 | return PTR_ERR(ctx); | 1239 | return PTR_ERR(ctx); |
| @@ -946,14 +1242,20 @@ int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, · 2 unmodified lines |
| 946 | /* There can't be user provided data before the meta data */ | 1242 | /* There can't be user provided data before the meta data */ |
| 947 | if (ctx->data_meta || ctx->data_end != size || | 1243 | if (ctx->data_meta || ctx->data_end != size || |
| 948 | ctx->data > ctx->data_end || | 1244 | ctx->data > ctx->data_end || |
| 949 | unlikely(xdp_metalen_invalid(ctx->data))) | 1245 | unlikely(xdp_metalen_invalid(ctx->data)) || |
| | 1246 | (do_live && (kattr->test.data_out || kattr->test.ctx_out))) |
| 950 | goto free_ctx; | 1247 | goto free_ctx; |
| 951 | /* Meta data is allocated from the headroom */ | 1248 | /* Meta data is allocated from the headroom */ |
| 952 | headroom -= ctx->data; | 1249 | headroom -= ctx->data; |
| 953 | } | 1250 | } |
| 954 | | 1251 | |
| 955 | max_data_sz = 4096 - headroom - tailroom; | 1252 | max_data_sz = 4096 - headroom - tailroom; |
| 956 | size = min_t(u32, size, max_data_sz); | 1253 | if (size > max_data_sz) { |
| | 1254 | /* disallow live data mode for jumbo frames */ |
| | 1255 | if (do_live) |
| | 1256 | goto free_ctx; |
| | 1257 | size = max_data_sz; |
| | 1258 | } |
| 957 | | 1259 | |
| 958 | data = bpf_test_init(kattr, size, max_data_sz, headroom, tailroom); | 1260 | data = bpf_test_init(kattr, size, max_data_sz, headroom, tailroom); |
| 959 | if (IS_ERR(data)) { | 1261 | if (IS_ERR(data)) { |
| @@ -1011,7 +1313,10 @@ int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, · 51 unmodified lines |
| 1011 | if (repeat > 1) | 1313 | if (repeat > 1) |
| 1012 | bpf_prog_change_xdp(NULL, prog); | 1314 | bpf_prog_change_xdp(NULL, prog); |
| 1013 | | 1315 | |
| 1014 | ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true); | 1316 | if (do_live) |
| | 1317 | ret = bpf_test_run_xdp_live(prog, &xdp, repeat, batch_size, &duration); |
| | 1318 | else |
| | 1319 | ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true); |
| 1015 | /* We convert the xdp_buff back to an xdp_md before checking the return | 1320 | /* We convert the xdp_buff back to an xdp_md before checking the return |
| 1016 | * code so the reference count of any held netdevice will be decremented | 1321 | * code so the reference count of any held netdevice will be decremented |
| 1017 | * even if the test run failed. | 1322 | * even if the test run failed. |
| @@ -1073,7 +1378,7 @@ int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, · 55 unmodified lines |
| 1073 | if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR) | 1378 | if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR) |
| 1074 | return -EINVAL; | 1379 | return -EINVAL; |
| 1075 | | 1380 | |
| 1076 | if (kattr->test.flags || kattr->test.cpu) | 1381 | if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) |
| 1077 | return -EINVAL; | 1382 | return -EINVAL; |
| 1078 | | 1383 | |
| 1079 | if (size < ETH_HLEN) | 1384 | if (size < ETH_HLEN) |
| @@ -1108,7 +1413,7 @@ int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, · 28 unmodified lines |
| 1108 | do { | 1413 | do { |
| 1109 | retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN, | 1414 | retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN, |
| 1110 | size, flags); | 1415 | size, flags); |
| 1111 | } while (bpf_test_timer_continue(&t, repeat, &ret, &duration)); | 1416 | } while (bpf_test_timer_continue(&t, 1, repeat, &ret, &duration)); |
| 1112 | bpf_test_timer_leave(&t); | 1417 | bpf_test_timer_leave(&t); |
| 1113 | | 1418 | |
| 1114 | if (ret < 0) | 1419 | if (ret < 0) |
| @@ -1140,7 +1445,7 @@ int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kat · 25 unmodified lines |
| 1140 | if (prog->type != BPF_PROG_TYPE_SK_LOOKUP) | 1445 | if (prog->type != BPF_PROG_TYPE_SK_LOOKUP) |
| 1141 | return -EINVAL; | 1446 | return -EINVAL; |
| 1142 | | 1447 | |
| 1143 | if (kattr->test.flags || kattr->test.cpu) | 1448 | if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) |
| 1144 | return -EINVAL; | 1449 | return -EINVAL; |
| 1145 | | 1450 | |
| 1146 | if (kattr->test.data_in || kattr->test.data_size_in || kattr->test.data_out || | 1451 | if (kattr->test.data_in || kattr->test.data_size_in || kattr->test.data_out || |
| @@ -1203,7 +1508,7 @@ int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kat · 56 unmodified lines |
| 1203 | do { | 1508 | do { |
| 1204 | ctx.selected_sk = NULL; | 1509 | ctx.selected_sk = NULL; |
| 1205 | retval = BPF_PROG_SK_LOOKUP_RUN_ARRAY(progs, ctx, bpf_prog_run); | 1510 | retval = BPF_PROG_SK_LOOKUP_RUN_ARRAY(progs, ctx, bpf_prog_run); |
| 1206 | } while (bpf_test_timer_continue(&t, repeat, &ret, &duration)); | 1511 | } while (bpf_test_timer_continue(&t, 1, repeat, &ret, &duration)); |
| 1207 | bpf_test_timer_leave(&t); | 1512 | bpf_test_timer_leave(&t); |
| 1208 | | 1513 | |
| 1209 | if (ret < 0) | 1514 | if (ret < 0) |
| @@ -1242,7 +1547,8 @@ int bpf_prog_test_run_syscall(struct bpf_prog *prog, · 32 unmodified lines |
| 1242 | /* doesn't support data_in/out, ctx_out, duration, or repeat or flags */ | 1547 | /* doesn't support data_in/out, ctx_out, duration, or repeat or flags */ |
| 1243 | if (kattr->test.data_in || kattr->test.data_out || | 1548 | if (kattr->test.data_in || kattr->test.data_out || |
| 1244 | kattr->test.ctx_out || kattr->test.duration || | 1549 | kattr->test.ctx_out || kattr->test.duration || |
| 1245 | kattr->test.repeat || kattr->test.flags) | 1550 | kattr->test.repeat || kattr->test.flags || |
| | 1551 | kattr->test.batch_size) |
| 1246 | return -EINVAL; | 1552 | return -EINVAL; |
| 1247 | | 1553 | |
| 1248 | if (ctx_size_in < prog->aux->max_ctx_offset || | 1554 | if (ctx_size_in < prog->aux->max_ctx_offset || |