Allow eBPF programs attached to TC qdiscs call skb_vlan_push/pop via helper functions. These functions may change skb->data/hlen which are cached by some JITs to improve performance of ld_abs/ld_ind instructions. Therefore JITs need to recognize bpf_skb_vlan_push/pop() calls, re-compute header len and re-cache skb->data/hlen back into cpu registers. Note, skb->data/hlen are not directly accessible from the programs, so any changes to skb->data done either by these helpers or by other TC actions are safe. eBPF JIT supported by three architectures: - arm64 JIT is using bpf_load_pointer() without caching, so it's ok as-is. - x64 JIT re-caches skb->data/hlen unconditionally after vlan_push/pop calls (experiments showed that conditional re-caching is slower). - s390 JIT falls back to interpreter for now when bpf_skb_vlan_push() is present in the program (re-caching is tbd). These helpers allow more scalable handling of vlan from the programs. Instead of creating thousands of vlan netdevs on top of eth0 and attaching TC+ingress+bpf to all of them, the program can be attached to eth0 directly and manipulate vlans as necessary.
Unified · Split
| @@ -973,6 +973,10 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i · 972 unmodified lines | ||
| 973 | 973 | */ |
| 974 | 974 | const u64 func = (u64)__bpf_call_base + imm; |
| 975 | 975 | |
| 976 | + if (bpf_helper_changes_skb_data((void *)func)) | |
| 977 | + /* TODO reload skb->data, hlen */ | |
| 978 | + return -1; | |
| 979 | + | |
| 976 | 980 | REG_SET_SEEN(BPF_REG_5); |
| 977 | 981 | jit->seen |= SEEN_FUNC; |
| 978 | 982 | /* lg %w1,<d(imm)>(%l) */ |
| @@ -315,6 +315,26 @@ static void emit_bpf_tail_call(u8 **pprog) · 314 unmodified lines | ||
| 315 | 315 | *pprog = prog; |
| 316 | 316 | } |
| 317 | 317 | |
| 318 | + | |
| 319 | +static void emit_load_skb_data_hlen(u8 **pprog) | |
| 320 | +{ | |
| 321 | + u8 *prog = *pprog; | |
| 322 | + int cnt = 0; | |
| 323 | + | |
| 324 | + /* r9d = skb->len - skb->data_len (headlen) | |
| 325 | + * r10 = skb->data | |
| 326 | + */ | |
| 327 | + /* mov %r9d, off32(%rdi) */ | |
| 328 | + EMIT3_off32(0x44, 0x8b, 0x8f, offsetof(struct sk_buff, len)); | |
| 329 | + | |
| 330 | + /* sub %r9d, off32(%rdi) */ | |
| 331 | + EMIT3_off32(0x44, 0x2b, 0x8f, offsetof(struct sk_buff, data_len)); | |
| 332 | + | |
| 333 | + /* mov %r10, off32(%rdi) */ | |
| 334 | + EMIT3_off32(0x4c, 0x8b, 0x97, offsetof(struct sk_buff, data)); | |
| 335 | + *pprog = prog; | |
| 336 | +} | |
| 337 | + | |
| 318 | 338 | static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, |
| 319 | 339 | int oldproglen, struct jit_context *ctx) |
| 320 | 340 | { |
| @@ -329,36 +349,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, · 8 unmodified lines | ||
| 329 | 349 | |
| 330 | 350 | emit_prologue(&prog); |
| 331 | 351 | |
| 332 | − if (seen_ld_abs) { | |
| 333 | − /* r9d : skb->len - skb->data_len (headlen) | |
| 334 | − * r10 : skb->data | |
| 335 | − */ | |
| 336 | − if (is_imm8(offsetof(struct sk_buff, len))) | |
| 337 | − /* mov %r9d, off8(%rdi) */ | |
| 338 | − EMIT4(0x44, 0x8b, 0x4f, | |
| 339 | − offsetof(struct sk_buff, len)); | |
| 340 | − else | |
| 341 | − /* mov %r9d, off32(%rdi) */ | |
| 342 | − EMIT3_off32(0x44, 0x8b, 0x8f, | |
| 343 | − offsetof(struct sk_buff, len)); | |
| 344 | − | |
| 345 | − if (is_imm8(offsetof(struct sk_buff, data_len))) | |
| 346 | − /* sub %r9d, off8(%rdi) */ | |
| 347 | − EMIT4(0x44, 0x2b, 0x4f, | |
| 348 | − offsetof(struct sk_buff, data_len)); | |
| 349 | − else | |
| 350 | − EMIT3_off32(0x44, 0x2b, 0x8f, | |
| 351 | − offsetof(struct sk_buff, data_len)); | |
| 352 | − | |
| 353 | − if (is_imm8(offsetof(struct sk_buff, data))) | |
| 354 | − /* mov %r10, off8(%rdi) */ | |
| 355 | − EMIT4(0x4c, 0x8b, 0x57, | |
| 356 | − offsetof(struct sk_buff, data)); | |
| 357 | − else | |
| 358 | − /* mov %r10, off32(%rdi) */ | |
| 359 | − EMIT3_off32(0x4c, 0x8b, 0x97, | |
| 360 | − offsetof(struct sk_buff, data)); | |
| 361 | − } | |
| 352 | + if (seen_ld_abs) | |
| 353 | + emit_load_skb_data_hlen(&prog); | |
| 362 | 354 | |
| 363 | 355 | for (i = 0; i < insn_cnt; i++, insn++) { |
| 364 | 356 | const s32 imm32 = insn->imm; |
| @@ -367,6 +359,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, · 2 unmodified lines | ||
| 367 | 359 | u8 b1 = 0, b2 = 0, b3 = 0; |
| 368 | 360 | s64 jmp_offset; |
| 369 | 361 | u8 jmp_cond; |
| 362 | + bool reload_skb_data; | |
| 370 | 363 | int ilen; |
| 371 | 364 | u8 *func; |
| 372 | 365 | |
| @@ -818,12 +811,18 @@ xadd: if (is_imm8(insn->off)) · 445 unmodified lines | ||
| 818 | 811 | func = (u8 *) __bpf_call_base + imm32; |
| 819 | 812 | jmp_offset = func - (image + addrs[i]); |
| 820 | 813 | if (seen_ld_abs) { |
| 821 | − EMIT2(0x41, 0x52); /* push %r10 */ | |
| 822 | − EMIT2(0x41, 0x51); /* push %r9 */ | |
| 823 | − /* need to adjust jmp offset, since | |
| 824 | − * pop %r9, pop %r10 take 4 bytes after call insn | |
| 825 | − */ | |
| 826 | − jmp_offset += 4; | |
| 814 | + reload_skb_data = bpf_helper_changes_skb_data(func); | |
| 815 | + if (reload_skb_data) { | |
| 816 | + EMIT1(0x57); /* push %rdi */ | |
| 817 | + jmp_offset += 22; /* pop, mov, sub, mov */ | |
| 818 | + } else { | |
| 819 | + EMIT2(0x41, 0x52); /* push %r10 */ | |
| 820 | + EMIT2(0x41, 0x51); /* push %r9 */ | |
| 821 | + /* need to adjust jmp offset, since | |
| 822 | + * pop %r9, pop %r10 take 4 bytes after call insn | |
| 823 | + */ | |
| 824 | + jmp_offset += 4; | |
| 825 | + } | |
| 827 | 826 | } |
| 828 | 827 | if (!imm32 || !is_simm32(jmp_offset)) { |
| 829 | 828 | pr_err("unsupported bpf func %d addr %p image %p\n", |
| @@ -832,8 +831,13 @@ xadd: if (is_imm8(insn->off)) · 2 unmodified lines | ||
| 832 | 831 | } |
| 833 | 832 | EMIT1_off32(0xE8, jmp_offset); |
| 834 | 833 | if (seen_ld_abs) { |
| 835 | − EMIT2(0x41, 0x59); /* pop %r9 */ | |
| 836 | − EMIT2(0x41, 0x5A); /* pop %r10 */ | |
| 834 | + if (reload_skb_data) { | |
| 835 | + EMIT1(0x5F); /* pop %rdi */ | |
| 836 | + emit_load_skb_data_hlen(&prog); | |
| 837 | + } else { | |
| 838 | + EMIT2(0x41, 0x59); /* pop %r9 */ | |
| 839 | + EMIT2(0x41, 0x5A); /* pop %r10 */ | |
| 840 | + } | |
| 837 | 841 | } |
| 838 | 842 | break; |
| 839 | 843 | |
| @@ -192,5 +192,7 @@ extern const struct bpf_func_proto bpf_ktime_get_ns_proto; · 191 unmodified lines | ||
| 192 | 192 | extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto; |
| 193 | 193 | extern const struct bpf_func_proto bpf_get_current_uid_gid_proto; |
| 194 | 194 | extern const struct bpf_func_proto bpf_get_current_comm_proto; |
| 195 | +extern const struct bpf_func_proto bpf_skb_vlan_push_proto; | |
| 196 | +extern const struct bpf_func_proto bpf_skb_vlan_pop_proto; | |
| 195 | 197 | |
| 196 | 198 | #endif /* _LINUX_BPF_H */ |
| @@ -411,6 +411,7 @@ void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp); · 410 unmodified lines | ||
| 411 | 411 | |
| 412 | 412 | u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); |
| 413 | 413 | void bpf_int_jit_compile(struct bpf_prog *fp); |
| 414 | +bool bpf_helper_changes_skb_data(void *func); | |
| 414 | 415 | |
| 415 | 416 | #ifdef CONFIG_BPF_JIT |
| 416 | 417 | typedef void (*bpf_jit_fill_hole_t)(void *area, unsigned int size); |
| @@ -256,6 +256,8 @@ enum bpf_func_id { · 255 unmodified lines | ||
| 256 | 256 | * Return: classid if != 0 |
| 257 | 257 | */ |
| 258 | 258 | BPF_FUNC_get_cgroup_classid, |
| 259 | + BPF_FUNC_skb_vlan_push, /* bpf_skb_vlan_push(skb, vlan_proto, vlan_tci) */ | |
| 260 | + BPF_FUNC_skb_vlan_pop, /* bpf_skb_vlan_pop(skb) */ | |
| 259 | 261 | __BPF_FUNC_MAX_ID, |
| 260 | 262 | }; |
| 261 | 263 | |
| @@ -1437,6 +1437,50 @@ static const struct bpf_func_proto bpf_get_cgroup_classid_proto = { · 1436 unmodified lines | ||
| 1437 | 1437 | .arg1_type = ARG_PTR_TO_CTX, |
| 1438 | 1438 | }; |
| 1439 | 1439 | |
| 1440 | +static u64 bpf_skb_vlan_push(u64 r1, u64 r2, u64 vlan_tci, u64 r4, u64 r5) | |
| 1441 | +{ | |
| 1442 | + struct sk_buff *skb = (struct sk_buff *) (long) r1; | |
| 1443 | + __be16 vlan_proto = (__force __be16) r2; | |
| 1444 | + | |
| 1445 | + if (unlikely(vlan_proto != htons(ETH_P_8021Q) && | |
| 1446 | + vlan_proto != htons(ETH_P_8021AD))) | |
| 1447 | + vlan_proto = htons(ETH_P_8021Q); | |
| 1448 | + | |
| 1449 | + return skb_vlan_push(skb, vlan_proto, vlan_tci); | |
| 1450 | +} | |
| 1451 | + | |
| 1452 | +const struct bpf_func_proto bpf_skb_vlan_push_proto = { | |
| 1453 | + .func = bpf_skb_vlan_push, | |
| 1454 | + .gpl_only = false, | |
| 1455 | + .ret_type = RET_INTEGER, | |
| 1456 | + .arg1_type = ARG_PTR_TO_CTX, | |
| 1457 | + .arg2_type = ARG_ANYTHING, | |
| 1458 | + .arg3_type = ARG_ANYTHING, | |
| 1459 | +}; | |
| 1460 | + | |
| 1461 | +static u64 bpf_skb_vlan_pop(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) | |
| 1462 | +{ | |
| 1463 | + struct sk_buff *skb = (struct sk_buff *) (long) r1; | |
| 1464 | + | |
| 1465 | + return skb_vlan_pop(skb); | |
| 1466 | +} | |
| 1467 | + | |
| 1468 | +const struct bpf_func_proto bpf_skb_vlan_pop_proto = { | |
| 1469 | + .func = bpf_skb_vlan_pop, | |
| 1470 | + .gpl_only = false, | |
| 1471 | + .ret_type = RET_INTEGER, | |
| 1472 | + .arg1_type = ARG_PTR_TO_CTX, | |
| 1473 | +}; | |
| 1474 | + | |
| 1475 | +bool bpf_helper_changes_skb_data(void *func) | |
| 1476 | +{ | |
| 1477 | + if (func == bpf_skb_vlan_push) | |
| 1478 | + return true; | |
| 1479 | + if (func == bpf_skb_vlan_pop) | |
| 1480 | + return true; | |
| 1481 | + return false; | |
| 1482 | +} | |
| 1483 | + | |
| 1440 | 1484 | static const struct bpf_func_proto * |
| 1441 | 1485 | sk_filter_func_proto(enum bpf_func_id func_id) |
| 1442 | 1486 | { |
| @@ -1476,6 +1520,10 @@ tc_cls_act_func_proto(enum bpf_func_id func_id) · 33 unmodified lines | ||
| 1476 | 1520 | return &bpf_clone_redirect_proto; |
| 1477 | 1521 | case BPF_FUNC_get_cgroup_classid: |
| 1478 | 1522 | return &bpf_get_cgroup_classid_proto; |
| 1523 | + case BPF_FUNC_skb_vlan_push: | |
| 1524 | + return &bpf_skb_vlan_push_proto; | |
| 1525 | + case BPF_FUNC_skb_vlan_pop: | |
| 1526 | + return &bpf_skb_vlan_pop_proto; | |
| 1479 | 1527 | default: |
| 1480 | 1528 | return sk_filter_func_proto(func_id); |
| 1481 | 1529 | } |