History

bpf: introduce bpf_skb_vlan_push/pop() helpers

Allow eBPF programs attached to TC qdiscs call skb_vlan_push/pop via helper functions. These functions may change skb->data/hlen which are cached by some JITs to improve performance of ld_abs/ld_ind instructions. Therefore JITs need to recognize bpf_skb_vlan_push/pop() calls, re-compute header len and re-cache skb->data/hlen back into cpu registers. Note, skb->data/hlen are not directly accessible from the programs, so any changes to skb->data done either by these helpers or by other TC actions are safe. eBPF JIT supported by three architectures: - arm64 JIT is using bpf_load_pointer() without caching, so it's ok as-is. - x64 JIT re-caches skb->data/hlen unconditionally after vlan_push/pop calls (experiments showed that conditional re-caching is slower). - s390 JIT falls back to interpreter for now when bpf_skb_vlan_push() is present in the program (re-caching is tbd). These helpers allow more scalable handling of vlan from the programs. Instead of creating thousands of vlan netdevs on top of eth0 and attaching TC+ingress+bpf to all of them, the program can be attached to eth0 directly and manipulate vlans as necessary.

Alexei Starovoitov committed 11 years ago

Commit 4e10df91 parent f3120ac6 files · +99 −38Browse files

Unified · Split

6 files
  1. arch/s390/net/bpf_jit_comp.c +4 −0
  2. arch/x86/net/bpf_jit_comp.c +42 −38
  3. include/linux/bpf.h +2 −0
  4. include/linux/filter.h +1 −0
  5. include/uapi/linux/bpf.h +2 −0
  6. net/core/filter.c +48 −0
arch/s390/net/bpf_jit_comp.c+4−0
@@ -973,6 +973,10 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i · 972 unmodified lines
973973 */
974974 const u64 func = (u64)__bpf_call_base + imm;
975975
976+ if (bpf_helper_changes_skb_data((void *)func))
977+ /* TODO reload skb->data, hlen */
978+ return -1;
979+
976980 REG_SET_SEEN(BPF_REG_5);
977981 jit->seen |= SEEN_FUNC;
978982 /* lg %w1,<d(imm)>(%l) */
arch/x86/net/bpf_jit_comp.c+42−38
@@ -315,6 +315,26 @@ static void emit_bpf_tail_call(u8 **pprog) · 314 unmodified lines
315315 *pprog = prog;
316316 }
317317
318+
319+static void emit_load_skb_data_hlen(u8 **pprog)
320+{
321+ u8 *prog = *pprog;
322+ int cnt = 0;
323+
324+ /* r9d = skb->len - skb->data_len (headlen)
325+ * r10 = skb->data
326+ */
327+ /* mov %r9d, off32(%rdi) */
328+ EMIT3_off32(0x44, 0x8b, 0x8f, offsetof(struct sk_buff, len));
329+
330+ /* sub %r9d, off32(%rdi) */
331+ EMIT3_off32(0x44, 0x2b, 0x8f, offsetof(struct sk_buff, data_len));
332+
333+ /* mov %r10, off32(%rdi) */
334+ EMIT3_off32(0x4c, 0x8b, 0x97, offsetof(struct sk_buff, data));
335+ *pprog = prog;
336+}
337+
318338 static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
319339 int oldproglen, struct jit_context *ctx)
320340 {
@@ -329,36 +349,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, · 8 unmodified lines
329349
330350 emit_prologue(&prog);
331351
332− if (seen_ld_abs) {
333− /* r9d : skb->len - skb->data_len (headlen)
334− * r10 : skb->data
335− */
336− if (is_imm8(offsetof(struct sk_buff, len)))
337− /* mov %r9d, off8(%rdi) */
338− EMIT4(0x44, 0x8b, 0x4f,
339− offsetof(struct sk_buff, len));
340− else
341− /* mov %r9d, off32(%rdi) */
342− EMIT3_off32(0x44, 0x8b, 0x8f,
343− offsetof(struct sk_buff, len));
344−
345− if (is_imm8(offsetof(struct sk_buff, data_len)))
346− /* sub %r9d, off8(%rdi) */
347− EMIT4(0x44, 0x2b, 0x4f,
348− offsetof(struct sk_buff, data_len));
349− else
350− EMIT3_off32(0x44, 0x2b, 0x8f,
351− offsetof(struct sk_buff, data_len));
352−
353− if (is_imm8(offsetof(struct sk_buff, data)))
354− /* mov %r10, off8(%rdi) */
355− EMIT4(0x4c, 0x8b, 0x57,
356− offsetof(struct sk_buff, data));
357− else
358− /* mov %r10, off32(%rdi) */
359− EMIT3_off32(0x4c, 0x8b, 0x97,
360− offsetof(struct sk_buff, data));
361− }
352+ if (seen_ld_abs)
353+ emit_load_skb_data_hlen(&prog);
362354
363355 for (i = 0; i < insn_cnt; i++, insn++) {
364356 const s32 imm32 = insn->imm;
@@ -367,6 +359,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, · 2 unmodified lines
367359 u8 b1 = 0, b2 = 0, b3 = 0;
368360 s64 jmp_offset;
369361 u8 jmp_cond;
362+ bool reload_skb_data;
370363 int ilen;
371364 u8 *func;
372365
@@ -818,12 +811,18 @@ xadd: if (is_imm8(insn->off)) · 445 unmodified lines
818811 func = (u8 *) __bpf_call_base + imm32;
819812 jmp_offset = func - (image + addrs[i]);
820813 if (seen_ld_abs) {
821− EMIT2(0x41, 0x52); /* push %r10 */
822− EMIT2(0x41, 0x51); /* push %r9 */
823− /* need to adjust jmp offset, since
824− * pop %r9, pop %r10 take 4 bytes after call insn
825− */
826− jmp_offset += 4;
814+ reload_skb_data = bpf_helper_changes_skb_data(func);
815+ if (reload_skb_data) {
816+ EMIT1(0x57); /* push %rdi */
817+ jmp_offset += 22; /* pop, mov, sub, mov */
818+ } else {
819+ EMIT2(0x41, 0x52); /* push %r10 */
820+ EMIT2(0x41, 0x51); /* push %r9 */
821+ /* need to adjust jmp offset, since
822+ * pop %r9, pop %r10 take 4 bytes after call insn
823+ */
824+ jmp_offset += 4;
825+ }
827826 }
828827 if (!imm32 || !is_simm32(jmp_offset)) {
829828 pr_err("unsupported bpf func %d addr %p image %p\n",
@@ -832,8 +831,13 @@ xadd: if (is_imm8(insn->off)) · 2 unmodified lines
832831 }
833832 EMIT1_off32(0xE8, jmp_offset);
834833 if (seen_ld_abs) {
835− EMIT2(0x41, 0x59); /* pop %r9 */
836− EMIT2(0x41, 0x5A); /* pop %r10 */
834+ if (reload_skb_data) {
835+ EMIT1(0x5F); /* pop %rdi */
836+ emit_load_skb_data_hlen(&prog);
837+ } else {
838+ EMIT2(0x41, 0x59); /* pop %r9 */
839+ EMIT2(0x41, 0x5A); /* pop %r10 */
840+ }
837841 }
838842 break;
839843
include/linux/bpf.h+2−0
@@ -192,5 +192,7 @@ extern const struct bpf_func_proto bpf_ktime_get_ns_proto; · 191 unmodified lines
192192 extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto;
193193 extern const struct bpf_func_proto bpf_get_current_uid_gid_proto;
194194 extern const struct bpf_func_proto bpf_get_current_comm_proto;
195+extern const struct bpf_func_proto bpf_skb_vlan_push_proto;
196+extern const struct bpf_func_proto bpf_skb_vlan_pop_proto;
195197
196198 #endif /* _LINUX_BPF_H */
include/linux/filter.h+1−0
@@ -411,6 +411,7 @@ void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp); · 410 unmodified lines
411411
412412 u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
413413 void bpf_int_jit_compile(struct bpf_prog *fp);
414+bool bpf_helper_changes_skb_data(void *func);
414415
415416 #ifdef CONFIG_BPF_JIT
416417 typedef void (*bpf_jit_fill_hole_t)(void *area, unsigned int size);
include/uapi/linux/bpf.h+2−0
@@ -256,6 +256,8 @@ enum bpf_func_id { · 255 unmodified lines
256256 * Return: classid if != 0
257257 */
258258 BPF_FUNC_get_cgroup_classid,
259+ BPF_FUNC_skb_vlan_push, /* bpf_skb_vlan_push(skb, vlan_proto, vlan_tci) */
260+ BPF_FUNC_skb_vlan_pop, /* bpf_skb_vlan_pop(skb) */
259261 __BPF_FUNC_MAX_ID,
260262 };
261263
net/core/filter.c+48−0
@@ -1437,6 +1437,50 @@ static const struct bpf_func_proto bpf_get_cgroup_classid_proto = { · 1436 unmodified lines
14371437 .arg1_type = ARG_PTR_TO_CTX,
14381438 };
14391439
1440+static u64 bpf_skb_vlan_push(u64 r1, u64 r2, u64 vlan_tci, u64 r4, u64 r5)
1441+{
1442+ struct sk_buff *skb = (struct sk_buff *) (long) r1;
1443+ __be16 vlan_proto = (__force __be16) r2;
1444+
1445+ if (unlikely(vlan_proto != htons(ETH_P_8021Q) &&
1446+ vlan_proto != htons(ETH_P_8021AD)))
1447+ vlan_proto = htons(ETH_P_8021Q);
1448+
1449+ return skb_vlan_push(skb, vlan_proto, vlan_tci);
1450+}
1451+
1452+const struct bpf_func_proto bpf_skb_vlan_push_proto = {
1453+ .func = bpf_skb_vlan_push,
1454+ .gpl_only = false,
1455+ .ret_type = RET_INTEGER,
1456+ .arg1_type = ARG_PTR_TO_CTX,
1457+ .arg2_type = ARG_ANYTHING,
1458+ .arg3_type = ARG_ANYTHING,
1459+};
1460+
1461+static u64 bpf_skb_vlan_pop(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
1462+{
1463+ struct sk_buff *skb = (struct sk_buff *) (long) r1;
1464+
1465+ return skb_vlan_pop(skb);
1466+}
1467+
1468+const struct bpf_func_proto bpf_skb_vlan_pop_proto = {
1469+ .func = bpf_skb_vlan_pop,
1470+ .gpl_only = false,
1471+ .ret_type = RET_INTEGER,
1472+ .arg1_type = ARG_PTR_TO_CTX,
1473+};
1474+
1475+bool bpf_helper_changes_skb_data(void *func)
1476+{
1477+ if (func == bpf_skb_vlan_push)
1478+ return true;
1479+ if (func == bpf_skb_vlan_pop)
1480+ return true;
1481+ return false;
1482+}
1483+
14401484 static const struct bpf_func_proto *
14411485 sk_filter_func_proto(enum bpf_func_id func_id)
14421486 {
@@ -1476,6 +1520,10 @@ tc_cls_act_func_proto(enum bpf_func_id func_id) · 33 unmodified lines
14761520 return &bpf_clone_redirect_proto;
14771521 case BPF_FUNC_get_cgroup_classid:
14781522 return &bpf_get_cgroup_classid_proto;
1523+ case BPF_FUNC_skb_vlan_push:
1524+ return &bpf_skb_vlan_push_proto;
1525+ case BPF_FUNC_skb_vlan_pop:
1526+ return &bpf_skb_vlan_pop_proto;
14791527 default:
14801528 return sk_filter_func_proto(func_id);
14811529 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t