History

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Alexei Starovoitov says: ==================== pull-request: bpf-next 2021-04-01 The following pull-request contains BPF updates for your *net-next* tree. We've added 68 non-merge commits during the last 7 day(s) which contain a total of 70 files changed, 2944 insertions(+), 1139 deletions(-). The main changes are: 1) UDP support for sockmap, from Cong. 2) Verifier merge conflict resolution fix, from Daniel. 3) xsk selftests enhancements, from Maciej. 4) Unstable helpers aka kernel func calling, from Martin. 5) Batches ops for LPM map, from Pedro. 6) Fix race in bpf_get_local_storage, from Yonghong. ====================

David S. Miller committed 5 years ago

Commit c2bcb4c2 parents bd78980 89d69c570 files · +2944 −1139Browse files

Unified · Split

70 files
  1. Documentation/bpf/bpf_design_QA.rst +15 −0
  2. arch/x86/net/bpf_jit_comp.c +5 −0
  3. arch/x86/net/bpf_jit_comp32.c +198 −0
  4. drivers/net/veth.c +12 −0
  5. include/linux/bpf-cgroup.h +49 −8
  6. include/linux/bpf.h +51 −7
  7. include/linux/btf.h +6 −0
  8. include/linux/filter.h +2 −11
  9. include/linux/skbuff.h +1 −0
  10. include/linux/skmsg.h +58 −19
  11. include/net/bpf_sk_storage.h +0 −1
  12. include/net/sock.h +3 −0
  13. include/net/tcp.h +1 −2
  14. include/net/udp.h +3 −0
  15. include/uapi/linux/bpf.h +5 −0
  16. kernel/bpf/btf.c +143 −76
  17. kernel/bpf/core.c +24 −23
  18. kernel/bpf/disasm.c +10 −3
  19. kernel/bpf/helpers.c +11 −4
  20. kernel/bpf/local_storage.c +3 −2
  21. kernel/bpf/lpm_trie.c +3 −0
  22. kernel/bpf/syscall.c +4 −1
  23. kernel/bpf/verifier.c +348 −42
  24. net/bpf/test_run.c +33 −1
  25. net/core/filter.c +1 −0
  26. net/core/skbuff.c +48 −7
  27. net/core/skmsg.c +143 −34
  28. net/core/sock_map.c +55 −63
  29. net/ipv4/af_inet.c +1 −0
  30. net/ipv4/bpf_tcp_ca.c +43 −0
  31. net/ipv4/tcp_bpf.c +23 −107
  32. net/ipv4/tcp_cubic.c +12 −12
  33. net/ipv4/tcp_ipv4.c +3 −0
  34. net/ipv4/udp.c +32 −0
  35. net/ipv4/udp_bpf.c +76 −3
  36. net/ipv6/af_inet6.c +1 −0
  37. net/ipv6/tcp_ipv6.c +3 −0
  38. net/ipv6/udp.c +3 −0
  39. net/tls/tls_sw.c +2 −2
  40. samples/bpf/sampleip_kern.c +0 −1
  41. samples/bpf/trace_event_kern.c +0 −1
  42. samples/bpf/xdpsock_user.c +14 −41
  43. tools/bpf/bpftool/common.c +1 −0
  44. tools/bpf/bpftool/prog.c +1 −0
  45. tools/bpf/resolve_btfids/main.c +5 −6
  46. tools/include/uapi/linux/bpf.h +5 −0
  47. tools/lib/bpf/libbpf.c +300 −103
  48. tools/lib/bpf/libbpf.h +3 −2
  49. tools/lib/bpf/libbpf.map +1 −0
  50. tools/lib/bpf/linker.c +28 −9
  51. tools/lib/bpf/xsk.c +213 −45
  52. tools/testing/selftests/bpf/README.rst +14 −0
  53. tools/testing/selftests/bpf/bpf_tcp_helpers.h +2 −27
  54. tools/testing/selftests/bpf/map_tests/lpm_trie_map_batch_ops.c +158 −0
  55. tools/testing/selftests/bpf/prog_tests/kfunc_call.c +59 −0
  56. tools/testing/selftests/bpf/prog_tests/sockmap_basic.c +40 −0
  57. tools/testing/selftests/bpf/prog_tests/sockmap_listen.c +136 −0
  58. tools/testing/selftests/bpf/prog_tests/test_ima.c +4 −2
  59. tools/testing/selftests/bpf/progs/bpf_cubic.c +18 −18
  60. tools/testing/selftests/bpf/progs/bpf_dctcp.c +6 −16
  61. tools/testing/selftests/bpf/progs/kfunc_call_test.c +47 −0
  62. tools/testing/selftests/bpf/progs/kfunc_call_test_subprog.c +42 −0
  63. tools/testing/selftests/bpf/progs/test_sockmap_listen.c +22 −0
  64. tools/testing/selftests/bpf/progs/test_sockmap_skb_verdict_attach.c +18 −0
  65. tools/testing/selftests/bpf/test_xsk.sh +2 −1
  66. tools/testing/selftests/bpf/verifier/calls.c +6 −6
  67. tools/testing/selftests/bpf/verifier/dead_code.c +5 −5
  68. tools/testing/selftests/bpf/vmtest.sh +28 −11
  69. tools/testing/selftests/bpf/xdpxceiver.c +317 −383
  70. tools/testing/selftests/bpf/xdpxceiver.h +15 −34
Documentation/bpf/bpf_design_QA.rst+15−0
@@ -258,3 +258,18 @@ Q: Can BPF functionality such as new program or map types, new · 257 unmodified lines
258258 helpers, etc be added out of kernel module code?
259259
260260 A: NO.
261+
262+Q: Directly calling kernel function is an ABI?
263+----------------------------------------------
264+Q: Some kernel functions (e.g. tcp_slow_start) can be called
265+by BPF programs. Do these kernel functions become an ABI?
266+
267+A: NO.
268+
269+The kernel function protos will change and the bpf programs will be
270+rejected by the verifier. Also, for example, some of the bpf-callable
271+kernel functions have already been used by other kernel tcp
272+cc (congestion-control) implementations. If any of these kernel
273+functions has changed, both the in-tree and out-of-tree kernel tcp cc
274+implementations have to be changed. The same goes for the bpf
275+programs and they have to be adjusted accordingly.
arch/x86/net/bpf_jit_comp.c+5−0
@@ -2346,3 +2346,8 @@ out: · 2345 unmodified lines
23462346 tmp : orig_prog);
23472347 return prog;
23482348 }
2349+
2350+bool bpf_jit_supports_kfunc_call(void)
2351+{
2352+ return true;
2353+}
arch/x86/net/bpf_jit_comp32.c+198−0
@@ -1390,6 +1390,19 @@ static inline void emit_push_r64(const u8 src[], u8 **pprog) · 1389 unmodified lines
13901390 *pprog = prog;
13911391 }
13921392
1393+static void emit_push_r32(const u8 src[], u8 **pprog)
1394+{
1395+ u8 *prog = *pprog;
1396+ int cnt = 0;
1397+
1398+ /* mov ecx,dword ptr [ebp+off] */
1399+ EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src_lo));
1400+ /* push ecx */
1401+ EMIT1(0x51);
1402+
1403+ *pprog = prog;
1404+}
1405+
13931406 static u8 get_cond_jmp_opcode(const u8 op, bool is_cmp_lo)
13941407 {
13951408 u8 jmp_cond;
@@ -1459,6 +1472,174 @@ static u8 get_cond_jmp_opcode(const u8 op, bool is_cmp_lo) · 63 unmodified lines
14591472 return jmp_cond;
14601473 }
14611474
1475+/* i386 kernel compiles with "-mregparm=3". From gcc document:
1476+ *
1477+ * ==== snippet ====
1478+ * regparm (number)
1479+ * On x86-32 targets, the regparm attribute causes the compiler
1480+ * to pass arguments number one to (number) if they are of integral
1481+ * type in registers EAX, EDX, and ECX instead of on the stack.
1482+ * Functions that take a variable number of arguments continue
1483+ * to be passed all of their arguments on the stack.
1484+ * ==== snippet ====
1485+ *
1486+ * The first three args of a function will be considered for
1487+ * putting into the 32bit register EAX, EDX, and ECX.
1488+ *
1489+ * Two 32bit registers are used to pass a 64bit arg.
1490+ *
1491+ * For example,
1492+ * void foo(u32 a, u32 b, u32 c, u32 d):
1493+ * u32 a: EAX
1494+ * u32 b: EDX
1495+ * u32 c: ECX
1496+ * u32 d: stack
1497+ *
1498+ * void foo(u64 a, u32 b, u32 c):
1499+ * u64 a: EAX (lo32) EDX (hi32)
1500+ * u32 b: ECX
1501+ * u32 c: stack
1502+ *
1503+ * void foo(u32 a, u64 b, u32 c):
1504+ * u32 a: EAX
1505+ * u64 b: EDX (lo32) ECX (hi32)
1506+ * u32 c: stack
1507+ *
1508+ * void foo(u32 a, u32 b, u64 c):
1509+ * u32 a: EAX
1510+ * u32 b: EDX
1511+ * u64 c: stack
1512+ *
1513+ * The return value will be stored in the EAX (and EDX for 64bit value).
1514+ *
1515+ * For example,
1516+ * u32 foo(u32 a, u32 b, u32 c):
1517+ * return value: EAX
1518+ *
1519+ * u64 foo(u32 a, u32 b, u32 c):
1520+ * return value: EAX (lo32) EDX (hi32)
1521+ *
1522+ * Notes:
1523+ * The verifier only accepts function having integer and pointers
1524+ * as its args and return value, so it does not have
1525+ * struct-by-value.
1526+ *
1527+ * emit_kfunc_call() finds out the btf_func_model by calling
1528+ * bpf_jit_find_kfunc_model(). A btf_func_model
1529+ * has the details about the number of args, size of each arg,
1530+ * and the size of the return value.
1531+ *
1532+ * It first decides how many args can be passed by EAX, EDX, and ECX.
1533+ * That will decide what args should be pushed to the stack:
1534+ * [first_stack_regno, last_stack_regno] are the bpf regnos
1535+ * that should be pushed to the stack.
1536+ *
1537+ * It will first push all args to the stack because the push
1538+ * will need to use ECX. Then, it moves
1539+ * [BPF_REG_1, first_stack_regno) to EAX, EDX, and ECX.
1540+ *
1541+ * When emitting a call (0xE8), it needs to figure out
1542+ * the jmp_offset relative to the jit-insn address immediately
1543+ * following the call (0xE8) instruction. At this point, it knows
1544+ * the end of the jit-insn address after completely translated the
1545+ * current (BPF_JMP | BPF_CALL) bpf-insn. It is passed as "end_addr"
1546+ * to the emit_kfunc_call(). Thus, it can learn the "immediate-follow-call"
1547+ * address by figuring out how many jit-insn is generated between
1548+ * the call (0xE8) and the end_addr:
1549+ * - 0-1 jit-insn (3 bytes each) to restore the esp pointer if there
1550+ * is arg pushed to the stack.
1551+ * - 0-2 jit-insns (3 bytes each) to handle the return value.
1552+ */
1553+static int emit_kfunc_call(const struct bpf_prog *bpf_prog, u8 *end_addr,
1554+ const struct bpf_insn *insn, u8 **pprog)
1555+{
1556+ const u8 arg_regs[] = { IA32_EAX, IA32_EDX, IA32_ECX };
1557+ int i, cnt = 0, first_stack_regno, last_stack_regno;
1558+ int free_arg_regs = ARRAY_SIZE(arg_regs);
1559+ const struct btf_func_model *fm;
1560+ int bytes_in_stack = 0;
1561+ const u8 *cur_arg_reg;
1562+ u8 *prog = *pprog;
1563+ s64 jmp_offset;
1564+
1565+ fm = bpf_jit_find_kfunc_model(bpf_prog, insn);
1566+ if (!fm)
1567+ return -EINVAL;
1568+
1569+ first_stack_regno = BPF_REG_1;
1570+ for (i = 0; i < fm->nr_args; i++) {
1571+ int regs_needed = fm->arg_size[i] > sizeof(u32) ? 2 : 1;
1572+
1573+ if (regs_needed > free_arg_regs)
1574+ break;
1575+
1576+ free_arg_regs -= regs_needed;
1577+ first_stack_regno++;
1578+ }
1579+
1580+ /* Push the args to the stack */
1581+ last_stack_regno = BPF_REG_0 + fm->nr_args;
1582+ for (i = last_stack_regno; i >= first_stack_regno; i--) {
1583+ if (fm->arg_size[i - 1] > sizeof(u32)) {
1584+ emit_push_r64(bpf2ia32[i], &prog);
1585+ bytes_in_stack += 8;
1586+ } else {
1587+ emit_push_r32(bpf2ia32[i], &prog);
1588+ bytes_in_stack += 4;
1589+ }
1590+ }
1591+
1592+ cur_arg_reg = &arg_regs[0];
1593+ for (i = BPF_REG_1; i < first_stack_regno; i++) {
1594+ /* mov e[adc]x,dword ptr [ebp+off] */
1595+ EMIT3(0x8B, add_2reg(0x40, IA32_EBP, *cur_arg_reg++),
1596+ STACK_VAR(bpf2ia32[i][0]));
1597+ if (fm->arg_size[i - 1] > sizeof(u32))
1598+ /* mov e[adc]x,dword ptr [ebp+off] */
1599+ EMIT3(0x8B, add_2reg(0x40, IA32_EBP, *cur_arg_reg++),
1600+ STACK_VAR(bpf2ia32[i][1]));
1601+ }
1602+
1603+ if (bytes_in_stack)
1604+ /* add esp,"bytes_in_stack" */
1605+ end_addr -= 3;
1606+
1607+ /* mov dword ptr [ebp+off],edx */
1608+ if (fm->ret_size > sizeof(u32))
1609+ end_addr -= 3;
1610+
1611+ /* mov dword ptr [ebp+off],eax */
1612+ if (fm->ret_size)
1613+ end_addr -= 3;
1614+
1615+ jmp_offset = (u8 *)__bpf_call_base + insn->imm - end_addr;
1616+ if (!is_simm32(jmp_offset)) {
1617+ pr_err("unsupported BPF kernel function jmp_offset:%lld\n",
1618+ jmp_offset);
1619+ return -EINVAL;
1620+ }
1621+
1622+ EMIT1_off32(0xE8, jmp_offset);
1623+
1624+ if (fm->ret_size)
1625+ /* mov dword ptr [ebp+off],eax */
1626+ EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX),
1627+ STACK_VAR(bpf2ia32[BPF_REG_0][0]));
1628+
1629+ if (fm->ret_size > sizeof(u32))
1630+ /* mov dword ptr [ebp+off],edx */
1631+ EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX),
1632+ STACK_VAR(bpf2ia32[BPF_REG_0][1]));
1633+
1634+ if (bytes_in_stack)
1635+ /* add esp,"bytes_in_stack" */
1636+ EMIT3(0x83, add_1reg(0xC0, IA32_ESP), bytes_in_stack);
1637+
1638+ *pprog = prog;
1639+
1640+ return 0;
1641+}
1642+
14621643 static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
14631644 int oldproglen, struct jit_context *ctx)
14641645 {
@@ -1888,6 +2069,18 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, · 423 unmodified lines
18882069 if (insn->src_reg == BPF_PSEUDO_CALL)
18892070 goto notyet;
18902071
2072+ if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
2073+ int err;
2074+
2075+ err = emit_kfunc_call(bpf_prog,
2076+ image + addrs[i],
2077+ insn, &prog);
2078+
2079+ if (err)
2080+ return err;
2081+ break;
2082+ }
2083+
18912084 func = (u8 *) __bpf_call_base + imm32;
18922085 jmp_offset = func - (image + addrs[i]);
18932086
@@ -2393,3 +2586,8 @@ out: · 499 unmodified lines
23932586 tmp : orig_prog);
23942587 return prog;
23952588 }
2589+
2590+bool bpf_jit_supports_kfunc_call(void)
2591+{
2592+ return true;
2593+}
drivers/net/veth.c+12−0
@@ -218,6 +218,17 @@ static void veth_get_ethtool_stats(struct net_device *dev, · 217 unmodified lines
218218 }
219219 }
220220
221+static void veth_get_channels(struct net_device *dev,
222+ struct ethtool_channels *channels)
223+{
224+ channels->tx_count = dev->real_num_tx_queues;
225+ channels->rx_count = dev->real_num_rx_queues;
226+ channels->max_tx = dev->real_num_tx_queues;
227+ channels->max_rx = dev->real_num_rx_queues;
228+ channels->combined_count = min(dev->real_num_rx_queues, dev->real_num_tx_queues);
229+ channels->max_combined = min(dev->real_num_rx_queues, dev->real_num_tx_queues);
230+}
231+
221232 static const struct ethtool_ops veth_ethtool_ops = {
222233 .get_drvinfo = veth_get_drvinfo,
223234 .get_link = ethtool_op_get_link,
@@ -226,6 +237,7 @@ static const struct ethtool_ops veth_ethtool_ops = { · 2 unmodified lines
226237 .get_ethtool_stats = veth_get_ethtool_stats,
227238 .get_link_ksettings = veth_get_link_ksettings,
228239 .get_ts_info = ethtool_op_get_ts_info,
240+ .get_channels = veth_get_channels,
229241 };
230242
231243 /* general routines */
include/linux/bpf-cgroup.h+49−8
@@ -20,14 +20,25 @@ struct bpf_sock_ops_kern; · 19 unmodified lines
2020 struct bpf_cgroup_storage;
2121 struct ctl_table;
2222 struct ctl_table_header;
23+struct task_struct;
2324
2425 #ifdef CONFIG_CGROUP_BPF
2526
2627 extern struct static_key_false cgroup_bpf_enabled_key[MAX_BPF_ATTACH_TYPE];
2728 #define cgroup_bpf_enabled(type) static_branch_unlikely(&cgroup_bpf_enabled_key[type])
2829
29−DECLARE_PER_CPU(struct bpf_cgroup_storage*,
30− bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);
30+#define BPF_CGROUP_STORAGE_NEST_MAX 8
31+
32+struct bpf_cgroup_storage_info {
33+ struct task_struct *task;
34+ struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE];
35+};
36+
37+/* For each cpu, permit maximum BPF_CGROUP_STORAGE_NEST_MAX number of tasks
38+ * to use bpf cgroup storage simultaneously.
39+ */
40+DECLARE_PER_CPU(struct bpf_cgroup_storage_info,
41+ bpf_cgroup_storage_info[BPF_CGROUP_STORAGE_NEST_MAX]);
3142
3243 #define for_each_cgroup_storage_type(stype) \
3344 for (stype = 0; stype < MAX_BPF_CGROUP_STORAGE_TYPE; stype++)
@@ -161,13 +172,42 @@ static inline enum bpf_cgroup_storage_type cgroup_storage_type( · 127 unmodified lines
161172 return BPF_CGROUP_STORAGE_SHARED;
162173 }
163174
164−static inline void bpf_cgroup_storage_set(struct bpf_cgroup_storage
165− *storage[MAX_BPF_CGROUP_STORAGE_TYPE])
175+static inline int bpf_cgroup_storage_set(struct bpf_cgroup_storage
176+ *storage[MAX_BPF_CGROUP_STORAGE_TYPE])
166177 {
167178 enum bpf_cgroup_storage_type stype;
179+ int i, err = 0;
180+
181+ preempt_disable();
182+ for (i = 0; i < BPF_CGROUP_STORAGE_NEST_MAX; i++) {
183+ if (unlikely(this_cpu_read(bpf_cgroup_storage_info[i].task) != NULL))
184+ continue;
185+
186+ this_cpu_write(bpf_cgroup_storage_info[i].task, current);
187+ for_each_cgroup_storage_type(stype)
188+ this_cpu_write(bpf_cgroup_storage_info[i].storage[stype],
189+ storage[stype]);
190+ goto out;
191+ }
192+ err = -EBUSY;
193+ WARN_ON_ONCE(1);
194+
195+out:
196+ preempt_enable();
197+ return err;
198+}
199+
200+static inline void bpf_cgroup_storage_unset(void)
201+{
202+ int i;
203+
204+ for (i = 0; i < BPF_CGROUP_STORAGE_NEST_MAX; i++) {
205+ if (unlikely(this_cpu_read(bpf_cgroup_storage_info[i].task) != current))
206+ continue;
168207
169− for_each_cgroup_storage_type(stype)
170− this_cpu_write(bpf_cgroup_storage[stype], storage[stype]);
208+ this_cpu_write(bpf_cgroup_storage_info[i].task, NULL);
209+ return;
210+ }
171211 }
172212
173213 struct bpf_cgroup_storage *
@@ -448,8 +488,9 @@ static inline int cgroup_bpf_prog_query(const union bpf_attr *attr, · 274 unmodified lines
448488 return -EINVAL;
449489 }
450490
451−static inline void bpf_cgroup_storage_set(
452− struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE]) {}
491+static inline int bpf_cgroup_storage_set(
492+ struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE]) { return 0; }
493+static inline void bpf_cgroup_storage_unset(void) {}
453494 static inline int bpf_cgroup_storage_assign(struct bpf_prog_aux *aux,
454495 struct bpf_map *map) { return 0; }
455496 static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(
include/linux/bpf.h+51−7
@@ -56,7 +56,7 @@ struct bpf_iter_seq_info { · 55 unmodified lines
5656 u32 seq_priv_size;
5757 };
5858
59−/* map is generic key/value storage optionally accesible by eBPF programs */
59+/* map is generic key/value storage optionally accessible by eBPF programs */
6060 struct bpf_map_ops {
6161 /* funcs callable from userspace (via syscall) */
6262 int (*map_alloc_check)(union bpf_attr *attr);
@@ -427,6 +427,7 @@ enum bpf_reg_type { · 364 unmodified lines
427427 PTR_TO_PERCPU_BTF_ID, /* reg points to a percpu kernel variable */
428428 PTR_TO_FUNC, /* reg points to a bpf program function */
429429 PTR_TO_MAP_KEY, /* reg points to a map element key */
430+ __BPF_REG_TYPE_MAX,
430431 };
431432
432433 /* The information passed from prog-specific *_is_valid_access
@@ -480,6 +481,7 @@ struct bpf_verifier_ops { · 47 unmodified lines
480481 const struct btf_type *t, int off, int size,
481482 enum bpf_access_type atype,
482483 u32 *next_btf_id);
484+ bool (*check_kfunc_call)(u32 kfunc_btf_id);
483485 };
484486
485487 struct bpf_prog_offload_ops {
@@ -796,6 +798,8 @@ struct btf_mod_pair { · 310 unmodified lines
796798 struct module *module;
797799 };
798800
801+struct bpf_kfunc_desc_tab;
802+
799803 struct bpf_prog_aux {
800804 atomic64_t refcnt;
801805 u32 used_map_cnt;
@@ -832,6 +836,7 @@ struct bpf_prog_aux { · 30 unmodified lines
832836 struct bpf_prog **func;
833837 void *jit_data; /* JIT specific data. arch dependent */
834838 struct bpf_jit_poke_descriptor *poke_tab;
839+ struct bpf_kfunc_desc_tab *kfunc_tab;
835840 u32 size_poke_tab;
836841 struct bpf_ksym ksym;
837842 const struct bpf_prog_ops *ops;
@@ -1106,6 +1111,13 @@ int bpf_prog_array_copy(struct bpf_prog_array *old_array, · 268 unmodified lines
11061111 /* BPF program asks to set CN on the packet. */
11071112 #define BPF_RET_SET_CN (1 << 0)
11081113
1114+/* For BPF_PROG_RUN_ARRAY_FLAGS and __BPF_PROG_RUN_ARRAY,
1115+ * if bpf_cgroup_storage_set() failed, the rest of programs
1116+ * will not execute. This should be a really rare scenario
1117+ * as it requires BPF_CGROUP_STORAGE_NEST_MAX number of
1118+ * preemptions all between bpf_cgroup_storage_set() and
1119+ * bpf_cgroup_storage_unset() on the same cpu.
1120+ */
11091121 #define BPF_PROG_RUN_ARRAY_FLAGS(array, ctx, func, ret_flags) \
11101122 ({ \
11111123 struct bpf_prog_array_item *_item; \
@@ -1118,10 +1130,12 @@ int bpf_prog_array_copy(struct bpf_prog_array *old_array, · 6 unmodified lines
11181130 _array = rcu_dereference(array); \
11191131 _item = &_array->items[0]; \
11201132 while ((_prog = READ_ONCE(_item->prog))) { \
1121− bpf_cgroup_storage_set(_item->cgroup_storage); \
1133+ if (unlikely(bpf_cgroup_storage_set(_item->cgroup_storage))) \
1134+ break; \
11221135 func_ret = func(_prog, ctx); \
11231136 _ret &= (func_ret & 1); \
11241137 *(ret_flags) |= (func_ret >> 1); \
1138+ bpf_cgroup_storage_unset(); \
11251139 _item++; \
11261140 } \
11271141 rcu_read_unlock(); \
@@ -1142,9 +1156,14 @@ int bpf_prog_array_copy(struct bpf_prog_array *old_array, · 14 unmodified lines
11421156 goto _out; \
11431157 _item = &_array->items[0]; \
11441158 while ((_prog = READ_ONCE(_item->prog))) { \
1145− if (set_cg_storage) \
1146− bpf_cgroup_storage_set(_item->cgroup_storage); \
1147− _ret &= func(_prog, ctx); \
1159+ if (!set_cg_storage) { \
1160+ _ret &= func(_prog, ctx); \
1161+ } else { \
1162+ if (unlikely(bpf_cgroup_storage_set(_item->cgroup_storage))) \
1163+ break; \
1164+ _ret &= func(_prog, ctx); \
1165+ bpf_cgroup_storage_unset(); \
1166+ } \
11481167 _item++; \
11491168 } \
11501169 _out: \
@@ -1513,6 +1532,7 @@ int bpf_prog_test_run_raw_tp(struct bpf_prog *prog, · 362 unmodified lines
15131532 int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog,
15141533 const union bpf_attr *kattr,
15151534 union bpf_attr __user *uattr);
1535+bool bpf_prog_test_check_kfunc_call(u32 kfunc_id);
15161536 bool btf_ctx_access(int off, int size, enum bpf_access_type type,
15171537 const struct bpf_prog *prog,
15181538 struct bpf_insn_access_aux *info);
@@ -1531,8 +1551,11 @@ int btf_distill_func_proto(struct bpf_verifier_log *log, · 12 unmodified lines
15311551 struct btf_func_model *m);
15321552
15331553 struct bpf_reg_state;
1534−int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
1535− struct bpf_reg_state *regs);
1554+int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog,
1555+ struct bpf_reg_state *regs);
1556+int btf_check_kfunc_arg_match(struct bpf_verifier_env *env,
1557+ const struct btf *btf, u32 func_id,
1558+ struct bpf_reg_state *regs);
15361559 int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
15371560 struct bpf_reg_state *reg);
15381561 int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog,
@@ -1543,6 +1566,10 @@ struct bpf_link *bpf_link_by_id(u32 id); · 4 unmodified lines
15431566
15441567 const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id);
15451568 void bpf_task_storage_free(struct task_struct *task);
1569+bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog);
1570+const struct btf_func_model *
1571+bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
1572+ const struct bpf_insn *insn);
15461573 #else /* !CONFIG_BPF_SYSCALL */
15471574 static inline struct bpf_prog *bpf_prog_get(u32 ufd)
15481575 {
@@ -1705,6 +1732,11 @@ static inline int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, · 156 unmodified lines
17051732 return -ENOTSUPP;
17061733 }
17071734
1735+static inline bool bpf_prog_test_check_kfunc_call(u32 kfunc_id)
1736+{
1737+ return false;
1738+}
1739+
17081740 static inline void bpf_map_put(struct bpf_map *map)
17091741 {
17101742 }
@@ -1723,6 +1755,18 @@ bpf_base_func_proto(enum bpf_func_id func_id) · 12 unmodified lines
17231755 static inline void bpf_task_storage_free(struct task_struct *task)
17241756 {
17251757 }
1758+
1759+static inline bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
1760+{
1761+ return false;
1762+}
1763+
1764+static inline const struct btf_func_model *
1765+bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
1766+ const struct bpf_insn *insn)
1767+{
1768+ return NULL;
1769+}
17261770 #endif /* CONFIG_BPF_SYSCALL */
17271771
17281772 void __bpf_free_used_btfs(struct bpf_prog_aux *aux,
include/linux/btf.h+6−0
@@ -110,6 +110,7 @@ const struct btf_type *btf_type_resolve_func_ptr(const struct btf *btf, · 109 unmodified lines
110110 const struct btf_type *
111111 btf_resolve_size(const struct btf *btf, const struct btf_type *type,
112112 u32 *type_size);
113+const char *btf_type_str(const struct btf_type *t);
113114
114115 #define for_each_member(i, struct_type, member) \
115116 for (i = 0, member = btf_type_member(struct_type); \
@@ -141,6 +142,11 @@ static inline bool btf_type_is_enum(const struct btf_type *t) · 25 unmodified lines
141142 return BTF_INFO_KIND(t->info) == BTF_KIND_ENUM;
142143 }
143144
145+static inline bool btf_type_is_scalar(const struct btf_type *t)
146+{
147+ return btf_type_is_int(t) || btf_type_is_enum(t);
148+}
149+
144150 static inline bool btf_type_is_typedef(const struct btf_type *t)
145151 {
146152 return BTF_INFO_KIND(t->info) == BTF_KIND_TYPEDEF;
include/linux/filter.h+2−11
@@ -877,8 +877,7 @@ void bpf_prog_free_linfo(struct bpf_prog *prog); · 876 unmodified lines
877877 void bpf_prog_fill_jited_linfo(struct bpf_prog *prog,
878878 const u32 *insn_to_jit_off);
879879 int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog);
880−void bpf_prog_free_jited_linfo(struct bpf_prog *prog);
881−void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog);
880+void bpf_prog_jit_attempt_done(struct bpf_prog *prog);
882881
883882 struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags);
884883 struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flags);
@@ -919,6 +918,7 @@ u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); · 34 unmodified lines
919918 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog);
920919 void bpf_jit_compile(struct bpf_prog *prog);
921920 bool bpf_jit_needs_zext(void);
921+bool bpf_jit_supports_kfunc_call(void);
922922 bool bpf_helper_changes_pkt_data(void *func);
923923
924924 static inline bool bpf_dump_raw_ok(const struct cred *cred)
@@ -1246,15 +1246,6 @@ static inline u16 bpf_anc_helper(const struct sock_filter *ftest) · 321 unmodified lines
12461246 void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb,
12471247 int k, unsigned int size);
12481248
1249−static inline void *bpf_load_pointer(const struct sk_buff *skb, int k,
1250− unsigned int size, void *buffer)
1251−{
1252− if (k >= 0)
1253− return skb_header_pointer(skb, k, size, buffer);
1254−
1255− return bpf_internal_load_pointer_neg_helper(skb, k, size);
1256−}
1257−
12581249 static inline int bpf_tell_extensions(void)
12591250 {
12601251 return SKF_AD_MAX;
include/linux/skbuff.h+1−0
@@ -3626,6 +3626,7 @@ int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset, · 3625 unmodified lines
36263626 unsigned int flags);
36273627 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
36283628 int len);
3629+int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len);
36293630 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to);
36303631 unsigned int skb_zerocopy_headlen(const struct sk_buff *from);
36313632 int skb_zerocopy(struct sk_buff *to, struct sk_buff *from,
include/linux/skmsg.h+58−19
@@ -58,6 +58,7 @@ struct sk_psock_progs { · 57 unmodified lines
5858 struct bpf_prog *msg_parser;
5959 struct bpf_prog *stream_parser;
6060 struct bpf_prog *stream_verdict;
61+ struct bpf_prog *skb_verdict;
6162 };
6263
6364 enum sk_psock_state_bits {
@@ -89,6 +90,7 @@ struct sk_psock { · 25 unmodified lines
8990 #endif
9091 struct sk_buff_head ingress_skb;
9192 struct list_head ingress_msg;
93+ spinlock_t ingress_lock;
9294 unsigned long state;
9395 struct list_head link;
9496 spinlock_t link_lock;
@@ -97,13 +99,12 @@ struct sk_psock { · 2 unmodified lines
9799 void (*saved_close)(struct sock *sk, long timeout);
98100 void (*saved_write_space)(struct sock *sk);
99101 void (*saved_data_ready)(struct sock *sk);
102+ int (*psock_update_sk_prot)(struct sock *sk, bool restore);
100103 struct proto *sk_proto;
104+ struct mutex work_mutex;
101105 struct sk_psock_work_state work_state;
102106 struct work_struct work;
103− union {
104− struct rcu_head rcu;
105− struct work_struct gc;
106− };
107+ struct rcu_work rwork;
107108 };
108109
109110 int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,
@@ -124,6 +125,10 @@ int sk_msg_zerocopy_from_iter(struct sock *sk, struct iov_iter *from, · 14 unmodified lines
124125 struct sk_msg *msg, u32 bytes);
125126 int sk_msg_memcopy_from_iter(struct sock *sk, struct iov_iter *from,
126127 struct sk_msg *msg, u32 bytes);
128+int sk_msg_wait_data(struct sock *sk, struct sk_psock *psock, int flags,
129+ long timeo, int *err);
130+int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,
131+ int len, int flags);
127132
128133 static inline void sk_msg_check_to_free(struct sk_msg *msg, u32 i, u32 bytes)
129134 {
@@ -284,7 +289,45 @@ static inline struct sk_psock *sk_psock(const struct sock *sk) · 154 unmodified lines
284289 static inline void sk_psock_queue_msg(struct sk_psock *psock,
285290 struct sk_msg *msg)
286291 {
292+ spin_lock_bh(&psock->ingress_lock);
287293 list_add_tail(&msg->list, &psock->ingress_msg);
294+ spin_unlock_bh(&psock->ingress_lock);
295+}
296+
297+static inline struct sk_msg *sk_psock_dequeue_msg(struct sk_psock *psock)
298+{
299+ struct sk_msg *msg;
300+
301+ spin_lock_bh(&psock->ingress_lock);
302+ msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list);
303+ if (msg)
304+ list_del(&msg->list);
305+ spin_unlock_bh(&psock->ingress_lock);
306+ return msg;
307+}
308+
309+static inline struct sk_msg *sk_psock_peek_msg(struct sk_psock *psock)
310+{
311+ struct sk_msg *msg;
312+
313+ spin_lock_bh(&psock->ingress_lock);
314+ msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list);
315+ spin_unlock_bh(&psock->ingress_lock);
316+ return msg;
317+}
318+
319+static inline struct sk_msg *sk_psock_next_msg(struct sk_psock *psock,
320+ struct sk_msg *msg)
321+{
322+ struct sk_msg *ret;
323+
324+ spin_lock_bh(&psock->ingress_lock);
325+ if (list_is_last(&msg->list, &psock->ingress_msg))
326+ ret = NULL;
327+ else
328+ ret = list_next_entry(msg, list);
329+ spin_unlock_bh(&psock->ingress_lock);
330+ return ret;
288331 }
289332
290333 static inline bool sk_psock_queue_empty(const struct sk_psock *psock)
@@ -292,6 +335,13 @@ static inline bool sk_psock_queue_empty(const struct sk_psock *psock) · 1 unmodified line
292335 return psock ? list_empty(&psock->ingress_msg) : true;
293336 }
294337
338+static inline void kfree_sk_msg(struct sk_msg *msg)
339+{
340+ if (msg->skb)
341+ consume_skb(msg->skb);
342+ kfree(msg);
343+}
344+
295345 static inline void sk_psock_report_error(struct sk_psock *psock, int err)
296346 {
297347 struct sock *sk = psock->sk;
@@ -301,6 +351,7 @@ static inline void sk_psock_report_error(struct sk_psock *psock, int err) · 3 unmodified lines
301351 }
302352
303353 struct sk_psock *sk_psock_init(struct sock *sk, int node);
354+void sk_psock_stop(struct sk_psock *psock, bool wait);
304355
305356 #if IS_ENABLED(CONFIG_BPF_STREAM_PARSER)
306357 int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock);
@@ -349,25 +400,12 @@ static inline void sk_psock_cork_free(struct sk_psock *psock) · 42 unmodified lines
349400 }
350401 }
351402
352−static inline void sk_psock_update_proto(struct sock *sk,
353− struct sk_psock *psock,
354− struct proto *ops)
355−{
356− /* Pairs with lockless read in sk_clone_lock() */
357− WRITE_ONCE(sk->sk_prot, ops);
358−}
359−
360403 static inline void sk_psock_restore_proto(struct sock *sk,
361404 struct sk_psock *psock)
362405 {
363406 sk->sk_prot->unhash = psock->saved_unhash;
364− if (inet_csk_has_ulp(sk)) {
365− tcp_update_ulp(sk, psock->sk_proto, psock->saved_write_space);
366− } else {
367− sk->sk_write_space = psock->saved_write_space;
368− /* Pairs with lockless read in sk_clone_lock() */
369− WRITE_ONCE(sk->sk_prot, psock->sk_proto);
370− }
407+ if (psock->psock_update_sk_prot)
408+ psock->psock_update_sk_prot(sk, true);
371409 }
372410
373411 static inline void sk_psock_set_state(struct sk_psock *psock,
@@ -442,6 +480,7 @@ static inline void psock_progs_drop(struct sk_psock_progs *progs) · 68 unmodified lines
442480 psock_set_prog(&progs->msg_parser, NULL);
443481 psock_set_prog(&progs->stream_parser, NULL);
444482 psock_set_prog(&progs->stream_verdict, NULL);
483+ psock_set_prog(&progs->skb_verdict, NULL);
445484 }
446485
447486 int sk_psock_tls_strp_read(struct sk_psock *psock, struct sk_buff *skb);
include/net/bpf_sk_storage.h+0−1
@@ -27,7 +27,6 @@ struct bpf_local_storage_elem; · 26 unmodified lines
2727 struct bpf_sk_storage_diag;
2828 struct sk_buff;
2929 struct nlattr;
30−struct sock;
3130
3231 #ifdef CONFIG_BPF_SYSCALL
3332 int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk);
include/net/sock.h+3−0
@@ -1184,6 +1184,9 @@ struct proto { · 1183 unmodified lines
11841184 void (*unhash)(struct sock *sk);
11851185 void (*rehash)(struct sock *sk);
11861186 int (*get_port)(struct sock *sk, unsigned short snum);
1187+#ifdef CONFIG_BPF_SYSCALL
1188+ int (*psock_update_sk_prot)(struct sock *sk, bool restore);
1189+#endif
11871190
11881191 /* Keeping track of sockets in use */
11891192 #ifdef CONFIG_PROC_FS
include/net/tcp.h+1−2
@@ -2203,13 +2203,12 @@ struct sk_psock; · 2202 unmodified lines
22032203
22042204 #ifdef CONFIG_BPF_SYSCALL
22052205 struct proto *tcp_bpf_get_proto(struct sock *sk, struct sk_psock *psock);
2206+int tcp_bpf_update_proto(struct sock *sk, bool restore);
22062207 void tcp_bpf_clone(const struct sock *sk, struct sock *newsk);
22072208 #endif /* CONFIG_BPF_SYSCALL */
22082209
22092210 int tcp_bpf_sendmsg_redir(struct sock *sk, struct sk_msg *msg, u32 bytes,
22102211 int flags);
2211−int __tcp_bpf_recvmsg(struct sock *sk, struct sk_psock *psock,
2212− struct msghdr *msg, int len, int flags);
22132212 #endif /* CONFIG_NET_SOCK_MSG */
22142213
22152214 #if !defined(CONFIG_BPF_SYSCALL) || !defined(CONFIG_NET_SOCK_MSG)
include/net/udp.h+3−0
@@ -329,6 +329,8 @@ struct sock *__udp6_lib_lookup(struct net *net, · 328 unmodified lines
329329 struct sk_buff *skb);
330330 struct sock *udp6_lib_lookup_skb(const struct sk_buff *skb,
331331 __be16 sport, __be16 dport);
332+int udp_read_sock(struct sock *sk, read_descriptor_t *desc,
333+ sk_read_actor_t recv_actor);
332334
333335 /* UDP uses skb->dev_scratch to cache as much information as possible and avoid
334336 * possibly multiple cache miss on dequeue()
@@ -541,6 +543,7 @@ static inline void udp_post_segment_fix_csum(struct sk_buff *skb) · 206 unmodified lines
541543 #ifdef CONFIG_BPF_SYSCALL
542544 struct sk_psock;
543545 struct proto *udp_bpf_get_proto(struct sock *sk, struct sk_psock *psock);
546+int udp_bpf_update_proto(struct sock *sk, bool restore);
544547 #endif
545548
546549 #endif /* _UDP_H */
include/uapi/linux/bpf.h+5−0
@@ -957,6 +957,7 @@ enum bpf_attach_type { · 956 unmodified lines
957957 BPF_XDP_CPUMAP,
958958 BPF_SK_LOOKUP,
959959 BPF_XDP,
960+ BPF_SK_SKB_VERDICT,
960961 __MAX_BPF_ATTACH_TYPE
961962 };
962963
@@ -1117,6 +1118,10 @@ enum bpf_link_type { · 154 unmodified lines
11171118 * offset to another bpf function
11181119 */
11191120 #define BPF_PSEUDO_CALL 1
1121+/* when bpf_call->src_reg == BPF_PSEUDO_KFUNC_CALL,
1122+ * bpf_call->imm == btf_id of a BTF_KIND_FUNC in the running kernel
1123+ */
1124+#define BPF_PSEUDO_KFUNC_CALL 2
11201125
11211126 /* flags for BPF_MAP_UPDATE_ELEM command */
11221127 enum {
kernel/bpf/btf.c+143−76
@@ -283,7 +283,7 @@ static const char * const btf_kind_str[NR_BTF_KINDS] = { · 282 unmodified lines
283283 [BTF_KIND_FLOAT] = "FLOAT",
284284 };
285285
286−static const char *btf_type_str(const struct btf_type *t)
286+const char *btf_type_str(const struct btf_type *t)
287287 {
288288 return btf_kind_str[BTF_INFO_KIND(t->info)];
289289 }
@@ -789,7 +789,6 @@ static const struct btf_type *btf_type_skip_qualifiers(const struct btf *btf, · 499 unmodified lines
789789
790790 while (btf_type_is_modifier(t) &&
791791 BTF_INFO_KIND(t->info) != BTF_KIND_TYPEDEF) {
792− id = t->type;
793792 t = btf_type_by_id(btf, t->type);
794793 }
795794
@@ -4377,7 +4376,7 @@ static u8 bpf_ctx_convert_map[] = { · 3581 unmodified lines
43774376 #undef BPF_LINK_TYPE
43784377
43794378 static const struct btf_member *
4380−btf_get_prog_ctx_type(struct bpf_verifier_log *log, struct btf *btf,
4379+btf_get_prog_ctx_type(struct bpf_verifier_log *log, const struct btf *btf,
43814380 const struct btf_type *t, enum bpf_prog_type prog_type,
43824381 int arg)
43834382 {
@@ -5362,122 +5361,190 @@ int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *pr · 978 unmodified lines
53625361 return btf_check_func_type_match(log, btf1, t1, btf2, t2);
53635362 }
53645363
5365−/* Compare BTF of a function with given bpf_reg_state.
5366− * Returns:
5367− * EFAULT - there is a verifier bug. Abort verification.
5368− * EINVAL - there is a type mismatch or BTF is not available.
5369− * 0 - BTF matches with what bpf_reg_state expects.
5370− * Only PTR_TO_CTX and SCALAR_VALUE states are recognized.
5371− */
5372−int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
5373− struct bpf_reg_state *regs)
5364+static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = {
5365+#ifdef CONFIG_NET
5366+ [PTR_TO_SOCKET] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK],
5367+ [PTR_TO_SOCK_COMMON] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
5368+ [PTR_TO_TCP_SOCK] = &btf_sock_ids[BTF_SOCK_TYPE_TCP],
5369+#endif
5370+};
5371+
5372+static int btf_check_func_arg_match(struct bpf_verifier_env *env,
5373+ const struct btf *btf, u32 func_id,
5374+ struct bpf_reg_state *regs,
5375+ bool ptr_to_mem_ok)
53745376 {
53755377 struct bpf_verifier_log *log = &env->log;
5376− struct bpf_prog *prog = env->prog;
5377− struct btf *btf = prog->aux->btf;
5378− const struct btf_param *args;
5378+ const char *func_name, *ref_tname;
53795379 const struct btf_type *t, *ref_t;
5380− u32 i, nargs, btf_id, type_size;
5381− const char *tname;
5382− bool is_global;
5383−
5384− if (!prog->aux->func_info)
5385− return -EINVAL;
5386−
5387− btf_id = prog->aux->func_info[subprog].type_id;
5388− if (!btf_id)
5389− return -EFAULT;
5390−
5391− if (prog->aux->func_info_aux[subprog].unreliable)
5392− return -EINVAL;
5380+ const struct btf_param *args;
5381+ u32 i, nargs, ref_id;
53935382
5394− t = btf_type_by_id(btf, btf_id);
5383+ t = btf_type_by_id(btf, func_id);
53955384 if (!t || !btf_type_is_func(t)) {
53965385 /* These checks were already done by the verifier while loading
5397− * struct bpf_func_info
5386+ * struct bpf_func_info or in add_kfunc_call().
53985387 */
5399− bpf_log(log, "BTF of func#%d doesn't point to KIND_FUNC\n",
5400− subprog);
5388+ bpf_log(log, "BTF of func_id %u doesn't point to KIND_FUNC\n",
5389+ func_id);
54015390 return -EFAULT;
54025391 }
5403− tname = btf_name_by_offset(btf, t->name_off);
5392+ func_name = btf_name_by_offset(btf, t->name_off);
54045393
54055394 t = btf_type_by_id(btf, t->type);
54065395 if (!t || !btf_type_is_func_proto(t)) {
5407− bpf_log(log, "Invalid BTF of func %s\n", tname);
5396+ bpf_log(log, "Invalid BTF of func %s\n", func_name);
54085397 return -EFAULT;
54095398 }
54105399 args = (const struct btf_param *)(t + 1);
54115400 nargs = btf_type_vlen(t);
54125401 if (nargs > MAX_BPF_FUNC_REG_ARGS) {
5413− bpf_log(log, "Function %s has %d > %d args\n", tname, nargs,
5402+ bpf_log(log, "Function %s has %d > %d args\n", func_name, nargs,
54145403 MAX_BPF_FUNC_REG_ARGS);
5415− goto out;
5404+ return -EINVAL;
54165405 }
54175406
5418− is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
54195407 /* check that BTF function arguments match actual types that the
54205408 * verifier sees.
54215409 */
54225410 for (i = 0; i < nargs; i++) {
5423− struct bpf_reg_state *reg = &regs[i + 1];
5411+ u32 regno = i + 1;
5412+ struct bpf_reg_state *reg = &regs[regno];
54245413
5425− t = btf_type_by_id(btf, args[i].type);
5426− while (btf_type_is_modifier(t))
5427− t = btf_type_by_id(btf, t->type);
5428− if (btf_type_is_int(t) || btf_type_is_enum(t)) {
5414+ t = btf_type_skip_modifiers(btf, args[i].type, NULL);
5415+ if (btf_type_is_scalar(t)) {
54295416 if (reg->type == SCALAR_VALUE)
54305417 continue;
5431− bpf_log(log, "R%d is not a scalar\n", i + 1);
5432− goto out;
5418+ bpf_log(log, "R%d is not a scalar\n", regno);
5419+ return -EINVAL;
54335420 }
5434− if (btf_type_is_ptr(t)) {
5421+
5422+ if (!btf_type_is_ptr(t)) {
5423+ bpf_log(log, "Unrecognized arg#%d type %s\n",
5424+ i, btf_type_str(t));
5425+ return -EINVAL;
5426+ }
5427+
5428+ ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id);
5429+ ref_tname = btf_name_by_offset(btf, ref_t->name_off);
5430+ if (btf_is_kernel(btf)) {
5431+ const struct btf_type *reg_ref_t;
5432+ const struct btf *reg_btf;
5433+ const char *reg_ref_tname;
5434+ u32 reg_ref_id;
5435+
5436+ if (!btf_type_is_struct(ref_t)) {
5437+ bpf_log(log, "kernel function %s args#%d pointer type %s %s is not supported\n",
5438+ func_name, i, btf_type_str(ref_t),
5439+ ref_tname);
5440+ return -EINVAL;
5441+ }
5442+
5443+ if (reg->type == PTR_TO_BTF_ID) {
5444+ reg_btf = reg->btf;
5445+ reg_ref_id = reg->btf_id;
5446+ } else if (reg2btf_ids[reg->type]) {
5447+ reg_btf = btf_vmlinux;
5448+ reg_ref_id = *reg2btf_ids[reg->type];
5449+ } else {
5450+ bpf_log(log, "kernel function %s args#%d expected pointer to %s %s but R%d is not a pointer to btf_id\n",
5451+ func_name, i,
5452+ btf_type_str(ref_t), ref_tname, regno);
5453+ return -EINVAL;
5454+ }
5455+
5456+ reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id,
5457+ &reg_ref_id);
5458+ reg_ref_tname = btf_name_by_offset(reg_btf,
5459+ reg_ref_t->name_off);
5460+ if (!btf_struct_ids_match(log, reg_btf, reg_ref_id,
5461+ reg->off, btf, ref_id)) {
5462+ bpf_log(log, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\n",
5463+ func_name, i,
5464+ btf_type_str(ref_t), ref_tname,
5465+ regno, btf_type_str(reg_ref_t),
5466+ reg_ref_tname);
5467+ return -EINVAL;
5468+ }
5469+ } else if (btf_get_prog_ctx_type(log, btf, t,
5470+ env->prog->type, i)) {
54355471 /* If function expects ctx type in BTF check that caller
54365472 * is passing PTR_TO_CTX.
54375473 */
5438− if (btf_get_prog_ctx_type(log, btf, t, prog->type, i)) {
5439− if (reg->type != PTR_TO_CTX) {
5440− bpf_log(log,
5441− "arg#%d expected pointer to ctx, but got %s\n",
5442− i, btf_kind_str[BTF_INFO_KIND(t->info)]);
5443− goto out;
5444− }
5445− if (check_ctx_reg(env, reg, i + 1))
5446− goto out;
5447− continue;
5474+ if (reg->type != PTR_TO_CTX) {
5475+ bpf_log(log,
5476+ "arg#%d expected pointer to ctx, but got %s\n",
5477+ i, btf_type_str(t));
5478+ return -EINVAL;
54485479 }
5480+ if (check_ctx_reg(env, reg, regno))
5481+ return -EINVAL;
5482+ } else if (ptr_to_mem_ok) {
5483+ const struct btf_type *resolve_ret;
5484+ u32 type_size;
54495485
5450− if (!is_global)
5451− goto out;
5452−
5453− t = btf_type_skip_modifiers(btf, t->type, NULL);
5454−
5455− ref_t = btf_resolve_size(btf, t, &type_size);
5456− if (IS_ERR(ref_t)) {
5486+ resolve_ret = btf_resolve_size(btf, ref_t, &type_size);
5487+ if (IS_ERR(resolve_ret)) {
54575488 bpf_log(log,
5458− "arg#%d reference type('%s %s') size cannot be determined: %ld\n",
5459− i, btf_type_str(t), btf_name_by_offset(btf, t->name_off),
5460− PTR_ERR(ref_t));
5461− goto out;
5489+ "arg#%d reference type('%s %s') size cannot be determined: %ld\n",
5490+ i, btf_type_str(ref_t), ref_tname,
5491+ PTR_ERR(resolve_ret));
5492+ return -EINVAL;
54625493 }
54635494
5464− if (check_mem_reg(env, reg, i + 1, type_size))
5465− goto out;
5466−
5467− continue;
5495+ if (check_mem_reg(env, reg, regno, type_size))
5496+ return -EINVAL;
5497+ } else {
5498+ return -EINVAL;
54685499 }
5469− bpf_log(log, "Unrecognized arg#%d type %s\n",
5470− i, btf_kind_str[BTF_INFO_KIND(t->info)]);
5471− goto out;
54725500 }
5501+
54735502 return 0;
5474−out:
5503+}
5504+
5505+/* Compare BTF of a function with given bpf_reg_state.
5506+ * Returns:
5507+ * EFAULT - there is a verifier bug. Abort verification.
5508+ * EINVAL - there is a type mismatch or BTF is not available.
5509+ * 0 - BTF matches with what bpf_reg_state expects.
5510+ * Only PTR_TO_CTX and SCALAR_VALUE states are recognized.
5511+ */
5512+int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog,
5513+ struct bpf_reg_state *regs)
5514+{
5515+ struct bpf_prog *prog = env->prog;
5516+ struct btf *btf = prog->aux->btf;
5517+ bool is_global;
5518+ u32 btf_id;
5519+ int err;
5520+
5521+ if (!prog->aux->func_info)
5522+ return -EINVAL;
5523+
5524+ btf_id = prog->aux->func_info[subprog].type_id;
5525+ if (!btf_id)
5526+ return -EFAULT;
5527+
5528+ if (prog->aux->func_info_aux[subprog].unreliable)
5529+ return -EINVAL;
5530+
5531+ is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
5532+ err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global);
5533+
54755534 /* Compiler optimizations can remove arguments from static functions
54765535 * or mismatched type can be passed into a global function.
54775536 * In such cases mark the function as unreliable from BTF point of view.
54785537 */
5479− prog->aux->func_info_aux[subprog].unreliable = true;
5480− return -EINVAL;
5538+ if (err)
5539+ prog->aux->func_info_aux[subprog].unreliable = true;
5540+ return err;
5541+}
5542+
5543+int btf_check_kfunc_arg_match(struct bpf_verifier_env *env,
5544+ const struct btf *btf, u32 func_id,
5545+ struct bpf_reg_state *regs)
5546+{
5547+ return btf_check_func_arg_match(env, btf, func_id, regs, false);
54815548 }
54825549
54835550 /* Convert BTF of a function into bpf_reg_state if possible
kernel/bpf/core.c+24−23
@@ -143,25 +143,25 @@ int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog) · 142 unmodified lines
143143 if (!prog->aux->nr_linfo || !prog->jit_requested)
144144 return 0;
145145
146− prog->aux->jited_linfo = kcalloc(prog->aux->nr_linfo,
147− sizeof(*prog->aux->jited_linfo),
148− GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
146+ prog->aux->jited_linfo = kvcalloc(prog->aux->nr_linfo,
147+ sizeof(*prog->aux->jited_linfo),
148+ GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
149149 if (!prog->aux->jited_linfo)
150150 return -ENOMEM;
151151
152152 return 0;
153153 }
154154
155−void bpf_prog_free_jited_linfo(struct bpf_prog *prog)
155+void bpf_prog_jit_attempt_done(struct bpf_prog *prog)
156156 {
157− kfree(prog->aux->jited_linfo);
158− prog->aux->jited_linfo = NULL;
159−}
157+ if (prog->aux->jited_linfo &&
158+ (!prog->jited || !prog->aux->jited_linfo[0])) {
159+ kvfree(prog->aux->jited_linfo);
160+ prog->aux->jited_linfo = NULL;
161+ }
160162
161−void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog)
162−{
163− if (prog->aux->jited_linfo && !prog->aux->jited_linfo[0])
164− bpf_prog_free_jited_linfo(prog);
163+ kfree(prog->aux->kfunc_tab);
164+ prog->aux->kfunc_tab = NULL;
165165 }
166166
167167 /* The jit engine is responsible to provide an array
@@ -217,12 +217,6 @@ void bpf_prog_fill_jited_linfo(struct bpf_prog *prog, · 49 unmodified lines
217217 insn_to_jit_off[linfo[i].insn_off - insn_start - 1];
218218 }
219219
220−void bpf_prog_free_linfo(struct bpf_prog *prog)
221−{
222− bpf_prog_free_jited_linfo(prog);
223− kvfree(prog->aux->linfo);
224−}
225−
226220 struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,
227221 gfp_t gfp_extra_flags)
228222 {
@@ -1849,9 +1843,15 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) · 1620 unmodified lines
18491843 /* In case of BPF to BPF calls, verifier did all the prep
18501844 * work with regards to JITing, etc.
18511845 */
1846+ bool jit_needed = false;
1847+
18521848 if (fp->bpf_func)
18531849 goto finalize;
18541850
1851+ if (IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON) ||
1852+ bpf_prog_has_kfunc_call(fp))
1853+ jit_needed = true;
1854+
18551855 bpf_prog_select_func(fp);
18561856
18571857 /* eBPF JITs can rewrite the program in case constant
@@ -1866,14 +1866,10 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) · 8 unmodified lines
18661866 return fp;
18671867
18681868 fp = bpf_int_jit_compile(fp);
1869− if (!fp->jited) {
1870− bpf_prog_free_jited_linfo(fp);
1871−#ifdef CONFIG_BPF_JIT_ALWAYS_ON
1869+ bpf_prog_jit_attempt_done(fp);
1870+ if (!fp->jited && jit_needed) {
18721871 *err = -ENOTSUPP;
18731872 return fp;
1874−#endif
1875− } else {
1876− bpf_prog_free_unused_jited_linfo(fp);
18771873 }
18781874 } else {
18791875 *err = bpf_prog_offload_compile(fp);
@@ -2354,6 +2350,11 @@ bool __weak bpf_jit_needs_zext(void) · 474 unmodified lines
23542350 return false;
23552351 }
23562352
2353+bool __weak bpf_jit_supports_kfunc_call(void)
2354+{
2355+ return false;
2356+}
2357+
23572358 /* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call
23582359 * skb_copy_bits(), so provide a weak definition of it for NET-less config.
23592360 */
kernel/bpf/disasm.c+10−3
@@ -19,16 +19,23 @@ static const char *__func_get_name(const struct bpf_insn_cbs *cbs, · 18 unmodified lines
1919 {
2020 BUILD_BUG_ON(ARRAY_SIZE(func_id_str) != __BPF_FUNC_MAX_ID);
2121
22− if (insn->src_reg != BPF_PSEUDO_CALL &&
22+ if (!insn->src_reg &&
2323 insn->imm >= 0 && insn->imm < __BPF_FUNC_MAX_ID &&
2424 func_id_str[insn->imm])
2525 return func_id_str[insn->imm];
2626
27− if (cbs && cbs->cb_call)
28− return cbs->cb_call(cbs->private_data, insn);
27+ if (cbs && cbs->cb_call) {
28+ const char *res;
29+
30+ res = cbs->cb_call(cbs->private_data, insn);
31+ if (res)
32+ return res;
33+ }
2934
3035 if (insn->src_reg == BPF_PSEUDO_CALL)
3136 snprintf(buff, len, "%+d", insn->imm);
37+ else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL)
38+ snprintf(buff, len, "kernel-function");
3239
3340 return buff;
3441 }
kernel/bpf/helpers.c+11−4
@@ -382,8 +382,8 @@ const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto = { · 381 unmodified lines
382382 };
383383
384384 #ifdef CONFIG_CGROUP_BPF
385−DECLARE_PER_CPU(struct bpf_cgroup_storage*,
386− bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);
385+DECLARE_PER_CPU(struct bpf_cgroup_storage_info,
386+ bpf_cgroup_storage_info[BPF_CGROUP_STORAGE_NEST_MAX]);
387387
388388 BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags)
389389 {
@@ -392,10 +392,17 @@ BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags) · 2 unmodified lines
392392 * verifier checks that its value is correct.
393393 */
394394 enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);
395− struct bpf_cgroup_storage *storage;
395+ struct bpf_cgroup_storage *storage = NULL;
396396 void *ptr;
397+ int i;
397398
398− storage = this_cpu_read(bpf_cgroup_storage[stype]);
399+ for (i = 0; i < BPF_CGROUP_STORAGE_NEST_MAX; i++) {
400+ if (unlikely(this_cpu_read(bpf_cgroup_storage_info[i].task) != current))
401+ continue;
402+
403+ storage = this_cpu_read(bpf_cgroup_storage_info[i].storage[stype]);
404+ break;
405+ }
399406
400407 if (stype == BPF_CGROUP_STORAGE_SHARED)
401408 ptr = &READ_ONCE(storage->buf)->data[0];
kernel/bpf/local_storage.c+3−2
@@ -9,10 +9,11 @@ · 8 unmodified lines
99 #include <linux/slab.h>
1010 #include <uapi/linux/btf.h>
1111
12−DEFINE_PER_CPU(struct bpf_cgroup_storage*, bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);
13−
1412 #ifdef CONFIG_CGROUP_BPF
1513
14+DEFINE_PER_CPU(struct bpf_cgroup_storage_info,
15+ bpf_cgroup_storage_info[BPF_CGROUP_STORAGE_NEST_MAX]);
16+
1617 #include "../cgroup/cgroup-internal.h"
1718
1819 #define LOCAL_STORAGE_CREATE_FLAG_MASK \
kernel/bpf/lpm_trie.c+3−0
@@ -726,6 +726,9 @@ const struct bpf_map_ops trie_map_ops = { · 725 unmodified lines
726726 .map_lookup_elem = trie_lookup_elem,
727727 .map_update_elem = trie_update_elem,
728728 .map_delete_elem = trie_delete_elem,
729+ .map_lookup_batch = generic_map_lookup_batch,
730+ .map_update_batch = generic_map_update_batch,
731+ .map_delete_batch = generic_map_delete_batch,
729732 .map_check_btf = trie_check_btf,
730733 .map_btf_name = "lpm_trie",
731734 .map_btf_id = &trie_map_btf_id,
kernel/bpf/syscall.c+4−1
@@ -1694,7 +1694,9 @@ static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) · 1693 unmodified lines
16941694 {
16951695 bpf_prog_kallsyms_del_all(prog);
16961696 btf_put(prog->aux->btf);
1697− bpf_prog_free_linfo(prog);
1697+ kvfree(prog->aux->jited_linfo);
1698+ kvfree(prog->aux->linfo);
1699+ kfree(prog->aux->kfunc_tab);
16981700 if (prog->aux->attach_btf)
16991701 btf_put(prog->aux->attach_btf);
17001702
@@ -2946,6 +2948,7 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type) · 1245 unmodified lines
29462948 return BPF_PROG_TYPE_SK_MSG;
29472949 case BPF_SK_SKB_STREAM_PARSER:
29482950 case BPF_SK_SKB_STREAM_VERDICT:
2951+ case BPF_SK_SKB_VERDICT:
29492952 return BPF_PROG_TYPE_SK_SKB;
29502953 case BPF_LIRC_MODE2:
29512954 return BPF_PROG_TYPE_LIRC_MODE2;
kernel/bpf/verifier.c+348−42
@@ -234,6 +234,12 @@ static bool bpf_pseudo_call(const struct bpf_insn *insn) · 233 unmodified lines
234234 insn->src_reg == BPF_PSEUDO_CALL;
235235 }
236236
237+static bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn)
238+{
239+ return insn->code == (BPF_JMP | BPF_CALL) &&
240+ insn->src_reg == BPF_PSEUDO_KFUNC_CALL;
241+}
242+
237243 static bool bpf_pseudo_func(const struct bpf_insn *insn)
238244 {
239245 return insn->code == (BPF_LD | BPF_IMM | BPF_DW) &&
@@ -1554,47 +1560,205 @@ static int add_subprog(struct bpf_verifier_env *env, int off) · 1314 unmodified lines
15541560 verbose(env, "too many subprograms\n");
15551561 return -E2BIG;
15561562 }
1563+ /* determine subprog starts. The end is one before the next starts */
15571564 env->subprog_info[env->subprog_cnt++].start = off;
15581565 sort(env->subprog_info, env->subprog_cnt,
15591566 sizeof(env->subprog_info[0]), cmp_subprogs, NULL);
15601567 return env->subprog_cnt - 1;
15611568 }
15621569
1563−static int check_subprogs(struct bpf_verifier_env *env)
1570+struct bpf_kfunc_desc {
1571+ struct btf_func_model func_model;
1572+ u32 func_id;
1573+ s32 imm;
1574+};
1575+
1576+#define MAX_KFUNC_DESCS 256
1577+struct bpf_kfunc_desc_tab {
1578+ struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS];
1579+ u32 nr_descs;
1580+};
1581+
1582+static int kfunc_desc_cmp_by_id(const void *a, const void *b)
1583+{
1584+ const struct bpf_kfunc_desc *d0 = a;
1585+ const struct bpf_kfunc_desc *d1 = b;
1586+
1587+ /* func_id is not greater than BTF_MAX_TYPE */
1588+ return d0->func_id - d1->func_id;
1589+}
1590+
1591+static const struct bpf_kfunc_desc *
1592+find_kfunc_desc(const struct bpf_prog *prog, u32 func_id)
1593+{
1594+ struct bpf_kfunc_desc desc = {
1595+ .func_id = func_id,
1596+ };
1597+ struct bpf_kfunc_desc_tab *tab;
1598+
1599+ tab = prog->aux->kfunc_tab;
1600+ return bsearch(&desc, tab->descs, tab->nr_descs,
1601+ sizeof(tab->descs[0]), kfunc_desc_cmp_by_id);
1602+}
1603+
1604+static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id)
1605+{
1606+ const struct btf_type *func, *func_proto;
1607+ struct bpf_kfunc_desc_tab *tab;
1608+ struct bpf_prog_aux *prog_aux;
1609+ struct bpf_kfunc_desc *desc;
1610+ const char *func_name;
1611+ unsigned long addr;
1612+ int err;
1613+
1614+ prog_aux = env->prog->aux;
1615+ tab = prog_aux->kfunc_tab;
1616+ if (!tab) {
1617+ if (!btf_vmlinux) {
1618+ verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n");
1619+ return -ENOTSUPP;
1620+ }
1621+
1622+ if (!env->prog->jit_requested) {
1623+ verbose(env, "JIT is required for calling kernel function\n");
1624+ return -ENOTSUPP;
1625+ }
1626+
1627+ if (!bpf_jit_supports_kfunc_call()) {
1628+ verbose(env, "JIT does not support calling kernel function\n");
1629+ return -ENOTSUPP;
1630+ }
1631+
1632+ if (!env->prog->gpl_compatible) {
1633+ verbose(env, "cannot call kernel function from non-GPL compatible program\n");
1634+ return -EINVAL;
1635+ }
1636+
1637+ tab = kzalloc(sizeof(*tab), GFP_KERNEL);
1638+ if (!tab)
1639+ return -ENOMEM;
1640+ prog_aux->kfunc_tab = tab;
1641+ }
1642+
1643+ if (find_kfunc_desc(env->prog, func_id))
1644+ return 0;
1645+
1646+ if (tab->nr_descs == MAX_KFUNC_DESCS) {
1647+ verbose(env, "too many different kernel function calls\n");
1648+ return -E2BIG;
1649+ }
1650+
1651+ func = btf_type_by_id(btf_vmlinux, func_id);
1652+ if (!func || !btf_type_is_func(func)) {
1653+ verbose(env, "kernel btf_id %u is not a function\n",
1654+ func_id);
1655+ return -EINVAL;
1656+ }
1657+ func_proto = btf_type_by_id(btf_vmlinux, func->type);
1658+ if (!func_proto || !btf_type_is_func_proto(func_proto)) {
1659+ verbose(env, "kernel function btf_id %u does not have a valid func_proto\n",
1660+ func_id);
1661+ return -EINVAL;
1662+ }
1663+
1664+ func_name = btf_name_by_offset(btf_vmlinux, func->name_off);
1665+ addr = kallsyms_lookup_name(func_name);
1666+ if (!addr) {
1667+ verbose(env, "cannot find address for kernel function %s\n",
1668+ func_name);
1669+ return -EINVAL;
1670+ }
1671+
1672+ desc = &tab->descs[tab->nr_descs++];
1673+ desc->func_id = func_id;
1674+ desc->imm = BPF_CAST_CALL(addr) - __bpf_call_base;
1675+ err = btf_distill_func_proto(&env->log, btf_vmlinux,
1676+ func_proto, func_name,
1677+ &desc->func_model);
1678+ if (!err)
1679+ sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
1680+ kfunc_desc_cmp_by_id, NULL);
1681+ return err;
1682+}
1683+
1684+static int kfunc_desc_cmp_by_imm(const void *a, const void *b)
1685+{
1686+ const struct bpf_kfunc_desc *d0 = a;
1687+ const struct bpf_kfunc_desc *d1 = b;
1688+
1689+ if (d0->imm > d1->imm)
1690+ return 1;
1691+ else if (d0->imm < d1->imm)
1692+ return -1;
1693+ return 0;
1694+}
1695+
1696+static void sort_kfunc_descs_by_imm(struct bpf_prog *prog)
1697+{
1698+ struct bpf_kfunc_desc_tab *tab;
1699+
1700+ tab = prog->aux->kfunc_tab;
1701+ if (!tab)
1702+ return;
1703+
1704+ sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
1705+ kfunc_desc_cmp_by_imm, NULL);
1706+}
1707+
1708+bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
1709+{
1710+ return !!prog->aux->kfunc_tab;
1711+}
1712+
1713+const struct btf_func_model *
1714+bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
1715+ const struct bpf_insn *insn)
1716+{
1717+ const struct bpf_kfunc_desc desc = {
1718+ .imm = insn->imm,
1719+ };
1720+ const struct bpf_kfunc_desc *res;
1721+ struct bpf_kfunc_desc_tab *tab;
1722+
1723+ tab = prog->aux->kfunc_tab;
1724+ res = bsearch(&desc, tab->descs, tab->nr_descs,
1725+ sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm);
1726+
1727+ return res ? &res->func_model : NULL;
1728+}
1729+
1730+static int add_subprog_and_kfunc(struct bpf_verifier_env *env)
15641731 {
1565− int i, ret, subprog_start, subprog_end, off, cur_subprog = 0;
15661732 struct bpf_subprog_info *subprog = env->subprog_info;
15671733 struct bpf_insn *insn = env->prog->insnsi;
1568− int insn_cnt = env->prog->len;
1734+ int i, ret, insn_cnt = env->prog->len;
15691735
15701736 /* Add entry function. */
15711737 ret = add_subprog(env, 0);
1572− if (ret < 0)
1738+ if (ret)
15731739 return ret;
15741740
1575− /* determine subprog starts. The end is one before the next starts */
1576− for (i = 0; i < insn_cnt; i++) {
1577− if (bpf_pseudo_func(insn + i)) {
1578− if (!env->bpf_capable) {
1579− verbose(env,
1580− "function pointers are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
1581− return -EPERM;
1582− }
1583− ret = add_subprog(env, i + insn[i].imm + 1);
1584− if (ret < 0)
1585− return ret;
1586− /* remember subprog */
1587− insn[i + 1].imm = ret;
1588− continue;
1589− }
1590− if (!bpf_pseudo_call(insn + i))
1741+ for (i = 0; i < insn_cnt; i++, insn++) {
1742+ if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) &&
1743+ !bpf_pseudo_kfunc_call(insn))
15911744 continue;
1745+
15921746 if (!env->bpf_capable) {
1593− verbose(env,
1594− "function calls to other bpf functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
1747+ verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
15951748 return -EPERM;
15961749 }
1597− ret = add_subprog(env, i + insn[i].imm + 1);
1750+
1751+ if (bpf_pseudo_func(insn)) {
1752+ ret = add_subprog(env, i + insn->imm + 1);
1753+ if (ret >= 0)
1754+ /* remember subprog */
1755+ insn[1].imm = ret;
1756+ } else if (bpf_pseudo_call(insn)) {
1757+ ret = add_subprog(env, i + insn->imm + 1);
1758+ } else {
1759+ ret = add_kfunc_call(env, insn->imm);
1760+ }
1761+
15981762 if (ret < 0)
15991763 return ret;
16001764 }
@@ -1608,6 +1772,16 @@ static int check_subprogs(struct bpf_verifier_env *env) · 7 unmodified lines
16081772 for (i = 0; i < env->subprog_cnt; i++)
16091773 verbose(env, "func#%d @%d\n", i, subprog[i].start);
16101774
1775+ return 0;
1776+}
1777+
1778+static int check_subprogs(struct bpf_verifier_env *env)
1779+{
1780+ int i, subprog_start, subprog_end, off, cur_subprog = 0;
1781+ struct bpf_subprog_info *subprog = env->subprog_info;
1782+ struct bpf_insn *insn = env->prog->insnsi;
1783+ int insn_cnt = env->prog->len;
1784+
16111785 /* now check that all jumps are within the same subprog */
16121786 subprog_start = subprog[cur_subprog].start;
16131787 subprog_end = subprog[cur_subprog + 1].start;
@@ -1916,6 +2090,17 @@ static int get_prev_insn_idx(struct bpf_verifier_state *st, int i, · 302 unmodified lines
19162090 return i;
19172091 }
19182092
2093+static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn)
2094+{
2095+ const struct btf_type *func;
2096+
2097+ if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL)
2098+ return NULL;
2099+
2100+ func = btf_type_by_id(btf_vmlinux, insn->imm);
2101+ return btf_name_by_offset(btf_vmlinux, func->name_off);
2102+}
2103+
19192104 /* For given verifier state backtrack_insn() is called from the last insn to
19202105 * the first insn. Its purpose is to compute a bitmask of registers and
19212106 * stack slots that needs precision in the parent verifier state.
@@ -1924,6 +2109,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, · 2 unmodified lines
19242109 u32 *reg_mask, u64 *stack_mask)
19252110 {
19262111 const struct bpf_insn_cbs cbs = {
2112+ .cb_call = disasm_kfunc_name,
19272113 .cb_print = verbose,
19282114 .private_data = env,
19292115 };
@@ -5365,7 +5551,7 @@ static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn · 3435 unmodified lines
53655551 func_info_aux = env->prog->aux->func_info_aux;
53665552 if (func_info_aux)
53675553 is_global = func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
5368− err = btf_check_func_arg_match(env, subprog, caller->regs);
5554+ err = btf_check_subprog_arg_match(env, subprog, caller->regs);
53695555 if (err == -EFAULT)
53705556 return err;
53715557 if (is_global) {
@@ -5960,6 +6146,98 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn · 588 unmodified lines
59606146 return 0;
59616147 }
59626148
6149+/* mark_btf_func_reg_size() is used when the reg size is determined by
6150+ * the BTF func_proto's return value size and argument.
6151+ */
6152+static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno,
6153+ size_t reg_size)
6154+{
6155+ struct bpf_reg_state *reg = &cur_regs(env)[regno];
6156+
6157+ if (regno == BPF_REG_0) {
6158+ /* Function return value */
6159+ reg->live |= REG_LIVE_WRITTEN;
6160+ reg->subreg_def = reg_size == sizeof(u64) ?
6161+ DEF_NOT_SUBREG : env->insn_idx + 1;
6162+ } else {
6163+ /* Function argument */
6164+ if (reg_size == sizeof(u64)) {
6165+ mark_insn_zext(env, reg);
6166+ mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
6167+ } else {
6168+ mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32);
6169+ }
6170+ }
6171+}
6172+
6173+static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn)
6174+{
6175+ const struct btf_type *t, *func, *func_proto, *ptr_type;
6176+ struct bpf_reg_state *regs = cur_regs(env);
6177+ const char *func_name, *ptr_type_name;
6178+ u32 i, nargs, func_id, ptr_type_id;
6179+ const struct btf_param *args;
6180+ int err;
6181+
6182+ func_id = insn->imm;
6183+ func = btf_type_by_id(btf_vmlinux, func_id);
6184+ func_name = btf_name_by_offset(btf_vmlinux, func->name_off);
6185+ func_proto = btf_type_by_id(btf_vmlinux, func->type);
6186+
6187+ if (!env->ops->check_kfunc_call ||
6188+ !env->ops->check_kfunc_call(func_id)) {
6189+ verbose(env, "calling kernel function %s is not allowed\n",
6190+ func_name);
6191+ return -EACCES;
6192+ }
6193+
6194+ /* Check the arguments */
6195+ err = btf_check_kfunc_arg_match(env, btf_vmlinux, func_id, regs);
6196+ if (err)
6197+ return err;
6198+
6199+ for (i = 0; i < CALLER_SAVED_REGS; i++)
6200+ mark_reg_not_init(env, regs, caller_saved[i]);
6201+
6202+ /* Check return type */
6203+ t = btf_type_skip_modifiers(btf_vmlinux, func_proto->type, NULL);
6204+ if (btf_type_is_scalar(t)) {
6205+ mark_reg_unknown(env, regs, BPF_REG_0);
6206+ mark_btf_func_reg_size(env, BPF_REG_0, t->size);
6207+ } else if (btf_type_is_ptr(t)) {
6208+ ptr_type = btf_type_skip_modifiers(btf_vmlinux, t->type,
6209+ &ptr_type_id);
6210+ if (!btf_type_is_struct(ptr_type)) {
6211+ ptr_type_name = btf_name_by_offset(btf_vmlinux,
6212+ ptr_type->name_off);
6213+ verbose(env, "kernel function %s returns pointer type %s %s is not supported\n",
6214+ func_name, btf_type_str(ptr_type),
6215+ ptr_type_name);
6216+ return -EINVAL;
6217+ }
6218+ mark_reg_known_zero(env, regs, BPF_REG_0);
6219+ regs[BPF_REG_0].btf = btf_vmlinux;
6220+ regs[BPF_REG_0].type = PTR_TO_BTF_ID;
6221+ regs[BPF_REG_0].btf_id = ptr_type_id;
6222+ mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *));
6223+ } /* else { add_kfunc_call() ensures it is btf_type_is_void(t) } */
6224+
6225+ nargs = btf_type_vlen(func_proto);
6226+ args = (const struct btf_param *)(func_proto + 1);
6227+ for (i = 0; i < nargs; i++) {
6228+ u32 regno = i + 1;
6229+
6230+ t = btf_type_skip_modifiers(btf_vmlinux, args[i].type, NULL);
6231+ if (btf_type_is_ptr(t))
6232+ mark_btf_func_reg_size(env, regno, sizeof(void *));
6233+ else
6234+ /* scalar. ensured by btf_check_kfunc_arg_match() */
6235+ mark_btf_func_reg_size(env, regno, t->size);
6236+ }
6237+
6238+ return 0;
6239+}
6240+
59636241 static bool signed_add_overflows(s64 a, s64 b)
59646242 {
59656243 /* Do the add in u64, where overflow is well-defined */
@@ -6062,19 +6340,6 @@ static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, · 96 unmodified lines
60626340 else
60636341 *ptr_limit = -off - 1;
60646342 return *ptr_limit >= max ? -ERANGE : 0;
6065− case PTR_TO_MAP_KEY:
6066− /* Currently, this code is not exercised as the only use
6067− * is bpf_for_each_map_elem() helper which requires
6068− * bpf_capble. The code has been tested manually for
6069− * future use.
6070− */
6071− if (mask_to_left) {
6072− *ptr_limit = ptr_reg->umax_value + ptr_reg->off;
6073− } else {
6074− off = ptr_reg->smin_value + ptr_reg->off;
6075− *ptr_limit = ptr_reg->map_ptr->key_size - off;
6076− }
6077− return 0;
60786343 case PTR_TO_MAP_VALUE:
60796344 max = ptr_reg->map_ptr->value_size;
60806345 if (mask_to_left) {
@@ -6281,7 +6546,6 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, · 200 unmodified lines
62816546 verbose(env, "R%d pointer arithmetic on %s prohibited\n",
62826547 dst, reg_type_str[ptr_reg->type]);
62836548 return -EACCES;
6284− case PTR_TO_MAP_KEY:
62856549 case PTR_TO_MAP_VALUE:
62866550 if (!env->allow_ptr_leaks && !known && (smin_val < 0) != (smax_val < 0)) {
62876551 verbose(env, "R%d has unknown scalar with mixed signed bounds, pointer arithmetic with it prohibited for !root\n",
@@ -10176,6 +10440,7 @@ static int do_check(struct bpf_verifier_env *env) · 3888 unmodified lines
1017610440
1017710441 if (env->log.level & BPF_LOG_LEVEL) {
1017810442 const struct bpf_insn_cbs cbs = {
10443+ .cb_call = disasm_kfunc_name,
1017910444 .cb_print = verbose,
1018010445 .private_data = env,
1018110446 };
@@ -10323,7 +10588,8 @@ static int do_check(struct bpf_verifier_env *env) · 141 unmodified lines
1032310588 if (BPF_SRC(insn->code) != BPF_K ||
1032410589 insn->off != 0 ||
1032510590 (insn->src_reg != BPF_REG_0 &&
10326− insn->src_reg != BPF_PSEUDO_CALL) ||
10591+ insn->src_reg != BPF_PSEUDO_CALL &&
10592+ insn->src_reg != BPF_PSEUDO_KFUNC_CALL) ||
1032710593 insn->dst_reg != BPF_REG_0 ||
1032810594 class == BPF_JMP32) {
1032910595 verbose(env, "BPF_CALL uses reserved fields\n");
@@ -10338,6 +10604,8 @@ static int do_check(struct bpf_verifier_env *env) · 8 unmodified lines
1033810604 }
1033910605 if (insn->src_reg == BPF_PSEUDO_CALL)
1034010606 err = check_func_call(env, insn, &env->insn_idx);
10607+ else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL)
10608+ err = check_kfunc_call(env, insn);
1034110609 else
1034210610 err = check_helper_call(env, insn, &env->insn_idx);
1034310611 if (err)
@@ -11648,6 +11916,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 1304 unmodified lines
1164811916 func[i]->aux->name[0] = 'F';
1164911917 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
1165011918 func[i]->jit_requested = 1;
11919+ func[i]->aux->kfunc_tab = prog->aux->kfunc_tab;
1165111920 func[i]->aux->linfo = prog->aux->linfo;
1165211921 func[i]->aux->nr_linfo = prog->aux->nr_linfo;
1165311922 func[i]->aux->jited_linfo = prog->aux->jited_linfo;
@@ -11755,7 +12024,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 101 unmodified lines
1175512024 prog->bpf_func = func[0]->bpf_func;
1175612025 prog->aux->func = func;
1175712026 prog->aux->func_cnt = env->subprog_cnt;
11758− bpf_prog_free_unused_jited_linfo(prog);
12027+ bpf_prog_jit_attempt_done(prog);
1175912028 return 0;
1176012029 out_free:
1176112030 for (i = 0; i < env->subprog_cnt; i++) {
@@ -11778,7 +12047,7 @@ out_undo_insn: · 16 unmodified lines
1177812047 insn->off = 0;
1177912048 insn->imm = env->insn_aux_data[i].call_imm;
1178012049 }
11781− bpf_prog_free_jited_linfo(prog);
12050+ bpf_prog_jit_attempt_done(prog);
1178212051 return err;
1178312052 }
1178412053
@@ -11787,6 +12056,7 @@ static int fixup_call_args(struct bpf_verifier_env *env) · 2 unmodified lines
1178712056 #ifndef CONFIG_BPF_JIT_ALWAYS_ON
1178812057 struct bpf_prog *prog = env->prog;
1178912058 struct bpf_insn *insn = prog->insnsi;
12059+ bool has_kfunc_call = bpf_prog_has_kfunc_call(prog);
1179012060 int i, depth;
1179112061 #endif
1179212062 int err = 0;
@@ -11800,6 +12070,10 @@ static int fixup_call_args(struct bpf_verifier_env *env) · 7 unmodified lines
1180012070 return err;
1180112071 }
1180212072 #ifndef CONFIG_BPF_JIT_ALWAYS_ON
12073+ if (has_kfunc_call) {
12074+ verbose(env, "calling kernel functions are not allowed in non-JITed programs\n");
12075+ return -EINVAL;
12076+ }
1180312077 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {
1180412078 /* When JIT fails the progs with bpf2bpf calls and tail_calls
1180512079 * have to be rejected, since interpreter doesn't support them yet.
@@ -11828,6 +12102,26 @@ static int fixup_call_args(struct bpf_verifier_env *env) · 22 unmodified lines
1182812102 return err;
1182912103 }
1183012104
12105+static int fixup_kfunc_call(struct bpf_verifier_env *env,
12106+ struct bpf_insn *insn)
12107+{
12108+ const struct bpf_kfunc_desc *desc;
12109+
12110+ /* insn->imm has the btf func_id. Replace it with
12111+ * an address (relative to __bpf_base_call).
12112+ */
12113+ desc = find_kfunc_desc(env->prog, insn->imm);
12114+ if (!desc) {
12115+ verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n",
12116+ insn->imm);
12117+ return -EFAULT;
12118+ }
12119+
12120+ insn->imm = desc->imm;
12121+
12122+ return 0;
12123+}
12124+
1183112125 /* Do various post-verification rewrites in a single program pass.
1183212126 * These rewrites simplify JIT and interpreter implementations.
1183312127 */
@@ -11963,6 +12257,12 @@ static int do_misc_fixups(struct bpf_verifier_env *env) · 129 unmodified lines
1196312257 continue;
1196412258 if (insn->src_reg == BPF_PSEUDO_CALL)
1196512259 continue;
12260+ if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
12261+ ret = fixup_kfunc_call(env, insn);
12262+ if (ret)
12263+ return ret;
12264+ continue;
12265+ }
1196612266
1196712267 if (insn->imm == BPF_FUNC_get_route_realm)
1196812268 prog->dst_needed = 1;
@@ -12192,6 +12492,8 @@ patch_call_imm: · 223 unmodified lines
1219212492 }
1219312493 }
1219412494
12495+ sort_kfunc_descs_by_imm(env->prog);
12496+
1219512497 return 0;
1219612498 }
1219712499
@@ -12302,7 +12604,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) · 104 unmodified lines
1230212604 /* 1st arg to a function */
1230312605 regs[BPF_REG_1].type = PTR_TO_CTX;
1230412606 mark_reg_known_zero(env, regs, BPF_REG_1);
12305− ret = btf_check_func_arg_match(env, subprog, regs);
12607+ ret = btf_check_subprog_arg_match(env, subprog, regs);
1230612608 if (ret == -EFAULT)
1230712609 /* unlikely verifier bug. abort.
1230812610 * ret == 0 and ret < 0 are sadly acceptable for
@@ -12897,6 +13199,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, · 588 unmodified lines
1289713199 if (!env->explored_states)
1289813200 goto skip_full_check;
1289913201
13202+ ret = add_subprog_and_kfunc(env);
13203+ if (ret < 0)
13204+ goto skip_full_check;
13205+
1290013206 ret = check_subprogs(env);
1290113207 if (ret < 0)
1290213208 goto skip_full_check;
net/bpf/test_run.c+33−1
@@ -2,6 +2,7 @@ · 1 unmodified line
22 /* Copyright (c) 2017 Facebook
33 */
44 #include <linux/bpf.h>
5+#include <linux/btf_ids.h>
56 #include <linux/slab.h>
67 #include <linux/vmalloc.h>
78 #include <linux/etherdevice.h>
@@ -106,12 +107,16 @@ static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, · 98 unmodified lines
106107
107108 bpf_test_timer_enter(&t);
108109 do {
109− bpf_cgroup_storage_set(storage);
110+ ret = bpf_cgroup_storage_set(storage);
111+ if (ret)
112+ break;
110113
111114 if (xdp)
112115 *retval = bpf_prog_run_xdp(prog, ctx);
113116 else
114117 *retval = BPF_PROG_RUN(prog, ctx);
118+
119+ bpf_cgroup_storage_unset();
115120 } while (bpf_test_timer_continue(&t, repeat, &ret, time));
116121 bpf_test_timer_leave(&t);
117122
@@ -209,10 +214,37 @@ int noinline bpf_modify_return_test(int a, int *b) · 91 unmodified lines
209214 *b += 1;
210215 return a + *b;
211216 }
217+
218+u64 noinline bpf_kfunc_call_test1(struct sock *sk, u32 a, u64 b, u32 c, u64 d)
219+{
220+ return a + b + c + d;
221+}
222+
223+int noinline bpf_kfunc_call_test2(struct sock *sk, u32 a, u32 b)
224+{
225+ return a + b;
226+}
227+
228+struct sock * noinline bpf_kfunc_call_test3(struct sock *sk)
229+{
230+ return sk;
231+}
232+
212233 __diag_pop();
213234
214235 ALLOW_ERROR_INJECTION(bpf_modify_return_test, ERRNO);
215236
237+BTF_SET_START(test_sk_kfunc_ids)
238+BTF_ID(func, bpf_kfunc_call_test1)
239+BTF_ID(func, bpf_kfunc_call_test2)
240+BTF_ID(func, bpf_kfunc_call_test3)
241+BTF_SET_END(test_sk_kfunc_ids)
242+
243+bool bpf_prog_test_check_kfunc_call(u32 kfunc_id)
244+{
245+ return btf_id_set_contains(&test_sk_kfunc_ids, kfunc_id);
246+}
247+
216248 static void *bpf_test_init(const union bpf_attr *kattr, u32 size,
217249 u32 headroom, u32 tailroom)
218250 {
net/core/filter.c+1−0
@@ -9813,6 +9813,7 @@ const struct bpf_verifier_ops tc_cls_act_verifier_ops = { · 9812 unmodified lines
98139813 .convert_ctx_access = tc_cls_act_convert_ctx_access,
98149814 .gen_prologue = tc_cls_act_prologue,
98159815 .gen_ld_abs = bpf_gen_ld_abs,
9816+ .check_kfunc_call = bpf_prog_test_check_kfunc_call,
98169817 };
98179818
98189819 const struct bpf_prog_ops tc_cls_act_prog_ops = {
net/core/skbuff.c+48−7
@@ -2500,9 +2500,32 @@ int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset, · 2499 unmodified lines
25002500 }
25012501 EXPORT_SYMBOL_GPL(skb_splice_bits);
25022502
2503−/* Send skb data on a socket. Socket must be locked. */
2504−int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
2505− int len)
2503+static int sendmsg_unlocked(struct sock *sk, struct msghdr *msg,
2504+ struct kvec *vec, size_t num, size_t size)
2505+{
2506+ struct socket *sock = sk->sk_socket;
2507+
2508+ if (!sock)
2509+ return -EINVAL;
2510+ return kernel_sendmsg(sock, msg, vec, num, size);
2511+}
2512+
2513+static int sendpage_unlocked(struct sock *sk, struct page *page, int offset,
2514+ size_t size, int flags)
2515+{
2516+ struct socket *sock = sk->sk_socket;
2517+
2518+ if (!sock)
2519+ return -EINVAL;
2520+ return kernel_sendpage(sock, page, offset, size, flags);
2521+}
2522+
2523+typedef int (*sendmsg_func)(struct sock *sk, struct msghdr *msg,
2524+ struct kvec *vec, size_t num, size_t size);
2525+typedef int (*sendpage_func)(struct sock *sk, struct page *page, int offset,
2526+ size_t size, int flags);
2527+static int __skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset,
2528+ int len, sendmsg_func sendmsg, sendpage_func sendpage)
25062529 {
25072530 unsigned int orig_len = len;
25082531 struct sk_buff *head = skb;
@@ -2522,7 +2545,8 @@ do_frag_list: · 13 unmodified lines
25222545 memset(&msg, 0, sizeof(msg));
25232546 msg.msg_flags = MSG_DONTWAIT;
25242547
2525− ret = kernel_sendmsg_locked(sk, &msg, &kv, 1, slen);
2548+ ret = INDIRECT_CALL_2(sendmsg, kernel_sendmsg_locked,
2549+ sendmsg_unlocked, sk, &msg, &kv, 1, slen);
25262550 if (ret <= 0)
25272551 goto error;
25282552
@@ -2553,9 +2577,11 @@ do_frag_list: · 24 unmodified lines
25532577 slen = min_t(size_t, len, skb_frag_size(frag) - offset);
25542578
25552579 while (slen) {
2556− ret = kernel_sendpage_locked(sk, skb_frag_page(frag),
2557− skb_frag_off(frag) + offset,
2558− slen, MSG_DONTWAIT);
2580+ ret = INDIRECT_CALL_2(sendpage, kernel_sendpage_locked,
2581+ sendpage_unlocked, sk,
2582+ skb_frag_page(frag),
2583+ skb_frag_off(frag) + offset,
2584+ slen, MSG_DONTWAIT);
25592585 if (ret <= 0)
25602586 goto error;
25612587
@@ -2587,8 +2613,23 @@ out: · 25 unmodified lines
25872613 error:
25882614 return orig_len == len ? ret : orig_len - len;
25892615 }
2616+
2617+/* Send skb data on a socket. Socket must be locked. */
2618+int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
2619+ int len)
2620+{
2621+ return __skb_send_sock(sk, skb, offset, len, kernel_sendmsg_locked,
2622+ kernel_sendpage_locked);
2623+}
25902624 EXPORT_SYMBOL_GPL(skb_send_sock_locked);
25912625
2626+/* Send skb data on a socket. Socket must be unlocked. */
2627+int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len)
2628+{
2629+ return __skb_send_sock(sk, skb, offset, len, sendmsg_unlocked,
2630+ sendpage_unlocked);
2631+}
2632+
25922633 /**
25932634 * skb_store_bits - store bits from kernel buffer to skb
25942635 * @skb: destination buffer
net/core/skmsg.c+143−34
@@ -399,6 +399,104 @@ out: · 398 unmodified lines
399399 }
400400 EXPORT_SYMBOL_GPL(sk_msg_memcopy_from_iter);
401401
402+int sk_msg_wait_data(struct sock *sk, struct sk_psock *psock, int flags,
403+ long timeo, int *err)
404+{
405+ DEFINE_WAIT_FUNC(wait, woken_wake_function);
406+ int ret = 0;
407+
408+ if (sk->sk_shutdown & RCV_SHUTDOWN)
409+ return 1;
410+
411+ if (!timeo)
412+ return ret;
413+
414+ add_wait_queue(sk_sleep(sk), &wait);
415+ sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
416+ ret = sk_wait_event(sk, &timeo,
417+ !list_empty(&psock->ingress_msg) ||
418+ !skb_queue_empty(&sk->sk_receive_queue), &wait);
419+ sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
420+ remove_wait_queue(sk_sleep(sk), &wait);
421+ return ret;
422+}
423+EXPORT_SYMBOL_GPL(sk_msg_wait_data);
424+
425+/* Receive sk_msg from psock->ingress_msg to @msg. */
426+int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,
427+ int len, int flags)
428+{
429+ struct iov_iter *iter = &msg->msg_iter;
430+ int peek = flags & MSG_PEEK;
431+ struct sk_msg *msg_rx;
432+ int i, copied = 0;
433+
434+ msg_rx = sk_psock_peek_msg(psock);
435+ while (copied != len) {
436+ struct scatterlist *sge;
437+
438+ if (unlikely(!msg_rx))
439+ break;
440+
441+ i = msg_rx->sg.start;
442+ do {
443+ struct page *page;
444+ int copy;
445+
446+ sge = sk_msg_elem(msg_rx, i);
447+ copy = sge->length;
448+ page = sg_page(sge);
449+ if (copied + copy > len)
450+ copy = len - copied;
451+ copy = copy_page_to_iter(page, sge->offset, copy, iter);
452+ if (!copy)
453+ return copied ? copied : -EFAULT;
454+
455+ copied += copy;
456+ if (likely(!peek)) {
457+ sge->offset += copy;
458+ sge->length -= copy;
459+ if (!msg_rx->skb)
460+ sk_mem_uncharge(sk, copy);
461+ msg_rx->sg.size -= copy;
462+
463+ if (!sge->length) {
464+ sk_msg_iter_var_next(i);
465+ if (!msg_rx->skb)
466+ put_page(page);
467+ }
468+ } else {
469+ /* Lets not optimize peek case if copy_page_to_iter
470+ * didn't copy the entire length lets just break.
471+ */
472+ if (copy != sge->length)
473+ return copied;
474+ sk_msg_iter_var_next(i);
475+ }
476+
477+ if (copied == len)
478+ break;
479+ } while (i != msg_rx->sg.end);
480+
481+ if (unlikely(peek)) {
482+ msg_rx = sk_psock_next_msg(psock, msg_rx);
483+ if (!msg_rx)
484+ break;
485+ continue;
486+ }
487+
488+ msg_rx->sg.start = i;
489+ if (!sge->length && msg_rx->sg.start == msg_rx->sg.end) {
490+ msg_rx = sk_psock_dequeue_msg(psock);
491+ kfree_sk_msg(msg_rx);
492+ }
493+ msg_rx = sk_psock_peek_msg(psock);
494+ }
495+
496+ return copied;
497+}
498+EXPORT_SYMBOL_GPL(sk_msg_recvmsg);
499+
402500 static struct sk_msg *sk_psock_create_ingress_msg(struct sock *sk,
403501 struct sk_buff *skb)
404502 {
@@ -410,7 +508,7 @@ static struct sk_msg *sk_psock_create_ingress_msg(struct sock *sk, · 5 unmodified lines
410508 if (!sk_rmem_schedule(sk, skb, skb->truesize))
411509 return NULL;
412510
413− msg = kzalloc(sizeof(*msg), __GFP_NOWARN | GFP_ATOMIC);
511+ msg = kzalloc(sizeof(*msg), __GFP_NOWARN | GFP_KERNEL);
414512 if (unlikely(!msg))
415513 return NULL;
416514
@@ -497,7 +595,7 @@ static int sk_psock_handle_skb(struct sk_psock *psock, struct sk_buff *skb, · 80 unmodified lines
497595 if (!ingress) {
498596 if (!sock_writeable(psock->sk))
499597 return -EAGAIN;
500− return skb_send_sock_locked(psock->sk, skb, off, len);
598+ return skb_send_sock(psock->sk, skb, off, len);
501599 }
502600 return sk_psock_skb_ingress(psock, skb);
503601 }
@@ -511,8 +609,7 @@ static void sk_psock_backlog(struct work_struct *work) · 7 unmodified lines
511609 u32 len, off;
512610 int ret;
513611
514− /* Lock sock to avoid losing sk_socket during loop. */
515− lock_sock(psock->sk);
612+ mutex_lock(&psock->work_mutex);
516613 if (state->skb) {
517614 skb = state->skb;
518615 len = state->len;
@@ -529,7 +626,7 @@ start: · 10 unmodified lines
529626 skb_bpf_redirect_clear(skb);
530627 do {
531628 ret = -EIO;
532− if (likely(psock->sk->sk_socket))
629+ if (!sock_flag(psock->sk, SOCK_DEAD))
533630 ret = sk_psock_handle_skb(psock, skb, off,
534631 len, ingress);
535632 if (ret <= 0) {
@@ -553,7 +650,7 @@ start: · 17 unmodified lines
553650 kfree_skb(skb);
554651 }
555652 end:
556− release_sock(psock->sk);
653+ mutex_unlock(&psock->work_mutex);
557654 }
558655
559656 struct sk_psock *sk_psock_init(struct sock *sk, int node)
@@ -563,11 +660,6 @@ struct sk_psock *sk_psock_init(struct sock *sk, int node) · 3 unmodified lines
563660
564661 write_lock_bh(&sk->sk_callback_lock);
565662
566− if (inet_csk_has_ulp(sk)) {
567− psock = ERR_PTR(-EINVAL);
568− goto out;
569− }
570−
571663 if (sk->sk_user_data) {
572664 psock = ERR_PTR(-EBUSY);
573665 goto out;
@@ -591,7 +683,9 @@ struct sk_psock *sk_psock_init(struct sock *sk, int node) · 17 unmodified lines
591683 spin_lock_init(&psock->link_lock);
592684
593685 INIT_WORK(&psock->work, sk_psock_backlog);
686+ mutex_init(&psock->work_mutex);
594687 INIT_LIST_HEAD(&psock->ingress_msg);
688+ spin_lock_init(&psock->ingress_lock);
595689 skb_queue_head_init(&psock->ingress_skb);
596690
597691 sk_psock_set_state(psock, SK_PSOCK_TX_ENABLED);
@@ -630,11 +724,11 @@ static void __sk_psock_purge_ingress_msg(struct sk_psock *psock) · 32 unmodified lines
630724 }
631725 }
632726
633−static void sk_psock_zap_ingress(struct sk_psock *psock)
727+static void __sk_psock_zap_ingress(struct sk_psock *psock)
634728 {
635729 struct sk_buff *skb;
636730
637− while ((skb = __skb_dequeue(&psock->ingress_skb)) != NULL) {
731+ while ((skb = skb_dequeue(&psock->ingress_skb)) != NULL) {
638732 skb_bpf_redirect_clear(skb);
639733 kfree_skb(skb);
640734 }
@@ -651,23 +745,35 @@ static void sk_psock_link_destroy(struct sk_psock *psock) · 10 unmodified lines
651745 }
652746 }
653747
748+void sk_psock_stop(struct sk_psock *psock, bool wait)
749+{
750+ spin_lock_bh(&psock->ingress_lock);
751+ sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);
752+ sk_psock_cork_free(psock);
753+ __sk_psock_zap_ingress(psock);
754+ spin_unlock_bh(&psock->ingress_lock);
755+
756+ if (wait)
757+ cancel_work_sync(&psock->work);
758+}
759+
654760 static void sk_psock_done_strp(struct sk_psock *psock);
655761
656−static void sk_psock_destroy_deferred(struct work_struct *gc)
762+static void sk_psock_destroy(struct work_struct *work)
657763 {
658− struct sk_psock *psock = container_of(gc, struct sk_psock, gc);
659−
764+ struct sk_psock *psock = container_of(to_rcu_work(work),
765+ struct sk_psock, rwork);
660766 /* No sk_callback_lock since already detached. */
661767
662768 sk_psock_done_strp(psock);
663769
664770 cancel_work_sync(&psock->work);
771+ mutex_destroy(&psock->work_mutex);
665772
666773 psock_progs_drop(&psock->progs);
667774
668775 sk_psock_link_destroy(psock);
669776 sk_psock_cork_free(psock);
670− sk_psock_zap_ingress(psock);
671777
672778 if (psock->sk_redir)
673779 sock_put(psock->sk_redir);
@@ -675,30 +781,21 @@ static void sk_psock_destroy_deferred(struct work_struct *gc) · 1 unmodified line
675781 kfree(psock);
676782 }
677783
678−static void sk_psock_destroy(struct rcu_head *rcu)
679−{
680− struct sk_psock *psock = container_of(rcu, struct sk_psock, rcu);
681−
682− INIT_WORK(&psock->gc, sk_psock_destroy_deferred);
683− schedule_work(&psock->gc);
684−}
685−
686784 void sk_psock_drop(struct sock *sk, struct sk_psock *psock)
687785 {
688− sk_psock_cork_free(psock);
689− sk_psock_zap_ingress(psock);
786+ sk_psock_stop(psock, false);
690787
691788 write_lock_bh(&sk->sk_callback_lock);
692789 sk_psock_restore_proto(sk, psock);
693790 rcu_assign_sk_user_data(sk, NULL);
694791 if (psock->progs.stream_parser)
695792 sk_psock_stop_strp(sk, psock);
696− else if (psock->progs.stream_verdict)
793+ else if (psock->progs.stream_verdict || psock->progs.skb_verdict)
697794 sk_psock_stop_verdict(sk, psock);
698795 write_unlock_bh(&sk->sk_callback_lock);
699− sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);
700796
701− call_rcu(&psock->rcu, sk_psock_destroy);
797+ INIT_RCU_WORK(&psock->rwork, sk_psock_destroy);
798+ queue_rcu_work(system_wq, &psock->rwork);
702799 }
703800 EXPORT_SYMBOL_GPL(sk_psock_drop);
704801
@@ -767,14 +864,20 @@ static void sk_psock_skb_redirect(struct sk_buff *skb) · 62 unmodified lines
767864 * error that caused the pipe to break. We can't send a packet on
768865 * a socket that is in this state so we drop the skb.
769866 */
770− if (!psock_other || sock_flag(sk_other, SOCK_DEAD) ||
771− !sk_psock_test_state(psock_other, SK_PSOCK_TX_ENABLED)) {
867+ if (!psock_other || sock_flag(sk_other, SOCK_DEAD)) {
868+ kfree_skb(skb);
869+ return;
870+ }
871+ spin_lock_bh(&psock_other->ingress_lock);
872+ if (!sk_psock_test_state(psock_other, SK_PSOCK_TX_ENABLED)) {
873+ spin_unlock_bh(&psock_other->ingress_lock);
772874 kfree_skb(skb);
773875 return;
774876 }
775877
776878 skb_queue_tail(&psock_other->ingress_skb, skb);
777879 schedule_work(&psock_other->work);
880+ spin_unlock_bh(&psock_other->ingress_lock);
778881 }
779882
780883 static void sk_psock_tls_verdict_apply(struct sk_buff *skb, struct sock *sk, int verdict)
@@ -842,8 +945,12 @@ static void sk_psock_verdict_apply(struct sk_psock *psock, · 61 unmodified lines
842945 err = sk_psock_skb_ingress_self(psock, skb);
843946 }
844947 if (err < 0) {
845− skb_queue_tail(&psock->ingress_skb, skb);
846− schedule_work(&psock->work);
948+ spin_lock_bh(&psock->ingress_lock);
949+ if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) {
950+ skb_queue_tail(&psock->ingress_skb, skb);
951+ schedule_work(&psock->work);
952+ }
953+ spin_unlock_bh(&psock->ingress_lock);
847954 }
848955 break;
849956 case __SK_REDIRECT:
@@ -1010,6 +1117,8 @@ static int sk_psock_verdict_recv(read_descriptor_t *desc, struct sk_buff *skb, · 160 unmodified lines
10101117 }
10111118 skb_set_owner_r(skb, sk);
10121119 prog = READ_ONCE(psock->progs.stream_verdict);
1120+ if (!prog)
1121+ prog = READ_ONCE(psock->progs.skb_verdict);
10131122 if (likely(prog)) {
10141123 skb_dst_drop(skb);
10151124 skb_bpf_redirect_clear(skb);
net/core/sock_map.c+55−63
@@ -26,6 +26,7 @@ struct bpf_stab { · 25 unmodified lines
2626
2727 static int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog,
2828 struct bpf_prog *old, u32 which);
29+static struct sk_psock_progs *sock_map_progs(struct bpf_map *map);
2930
3031 static struct bpf_map *sock_map_alloc(union bpf_attr *attr)
3132 {
@@ -155,6 +156,8 @@ static void sock_map_del_link(struct sock *sk, · 123 unmodified lines
155156 strp_stop = true;
156157 if (psock->saved_data_ready && stab->progs.stream_verdict)
157158 verdict_stop = true;
159+ if (psock->saved_data_ready && stab->progs.skb_verdict)
160+ verdict_stop = true;
158161 list_del(&link->list);
159162 sk_psock_free_link(link);
160163 }
@@ -182,26 +185,10 @@ static void sock_map_unref(struct sock *sk, void *link_raw) · 21 unmodified lines
182185
183186 static int sock_map_init_proto(struct sock *sk, struct sk_psock *psock)
184187 {
185− struct proto *prot;
186−
187− switch (sk->sk_type) {
188− case SOCK_STREAM:
189− prot = tcp_bpf_get_proto(sk, psock);
190− break;
191−
192− case SOCK_DGRAM:
193− prot = udp_bpf_get_proto(sk, psock);
194− break;
195−
196− default:
188+ if (!sk->sk_prot->psock_update_sk_prot)
197189 return -EINVAL;
198− }
199−
200− if (IS_ERR(prot))
201− return PTR_ERR(prot);
202−
203− sk_psock_update_proto(sk, psock, prot);
204− return 0;
190+ psock->psock_update_sk_prot = sk->sk_prot->psock_update_sk_prot;
191+ return sk->sk_prot->psock_update_sk_prot(sk, false);
205192 }
206193
207194 static struct sk_psock *sock_map_psock_get_checked(struct sock *sk)
@@ -224,13 +211,25 @@ out: · 16 unmodified lines
224211 return psock;
225212 }
226213
227−static int sock_map_link(struct bpf_map *map, struct sk_psock_progs *progs,
228− struct sock *sk)
214+static bool sock_map_redirect_allowed(const struct sock *sk);
215+
216+static int sock_map_link(struct bpf_map *map, struct sock *sk)
229217 {
230− struct bpf_prog *msg_parser, *stream_parser, *stream_verdict;
218+ struct sk_psock_progs *progs = sock_map_progs(map);
219+ struct bpf_prog *stream_verdict = NULL;
220+ struct bpf_prog *stream_parser = NULL;
221+ struct bpf_prog *skb_verdict = NULL;
222+ struct bpf_prog *msg_parser = NULL;
231223 struct sk_psock *psock;
232224 int ret;
233225
226+ /* Only sockets we can redirect into/from in BPF need to hold
227+ * refs to parser/verdict progs and have their sk_data_ready
228+ * and sk_write_space callbacks overridden.
229+ */
230+ if (!sock_map_redirect_allowed(sk))
231+ goto no_progs;
232+
234233 stream_verdict = READ_ONCE(progs->stream_verdict);
235234 if (stream_verdict) {
236235 stream_verdict = bpf_prog_inc_not_zero(stream_verdict);
@@ -256,6 +255,16 @@ static int sock_map_link(struct bpf_map *map, struct sk_psock_progs *progs, · 19 unmodified lines
256255 }
257256 }
258257
258+ skb_verdict = READ_ONCE(progs->skb_verdict);
259+ if (skb_verdict) {
260+ skb_verdict = bpf_prog_inc_not_zero(skb_verdict);
261+ if (IS_ERR(skb_verdict)) {
262+ ret = PTR_ERR(skb_verdict);
263+ goto out_put_msg_parser;
264+ }
265+ }
266+
267+no_progs:
259268 psock = sock_map_psock_get_checked(sk);
260269 if (IS_ERR(psock)) {
261270 ret = PTR_ERR(psock);
@@ -265,6 +274,9 @@ static int sock_map_link(struct bpf_map *map, struct sk_psock_progs *progs, · 3 unmodified lines
265274 if (psock) {
266275 if ((msg_parser && READ_ONCE(psock->progs.msg_parser)) ||
267276 (stream_parser && READ_ONCE(psock->progs.stream_parser)) ||
277+ (skb_verdict && READ_ONCE(psock->progs.skb_verdict)) ||
278+ (skb_verdict && READ_ONCE(psock->progs.stream_verdict)) ||
279+ (stream_verdict && READ_ONCE(psock->progs.skb_verdict)) ||
268280 (stream_verdict && READ_ONCE(psock->progs.stream_verdict))) {
269281 sk_psock_put(sk, psock);
270282 ret = -EBUSY;
@@ -296,6 +308,9 @@ static int sock_map_link(struct bpf_map *map, struct sk_psock_progs *progs, · 25 unmodified lines
296308 } else if (!stream_parser && stream_verdict && !psock->saved_data_ready) {
297309 psock_set_prog(&psock->progs.stream_verdict, stream_verdict);
298310 sk_psock_start_verdict(sk,psock);
311+ } else if (!stream_verdict && skb_verdict && !psock->saved_data_ready) {
312+ psock_set_prog(&psock->progs.skb_verdict, skb_verdict);
313+ sk_psock_start_verdict(sk, psock);
299314 }
300315 write_unlock_bh(&sk->sk_callback_lock);
301316 return 0;
@@ -304,6 +319,9 @@ out_unlock_drop: · 2 unmodified lines
304319 out_drop:
305320 sk_psock_put(sk, psock);
306321 out_progs:
322+ if (skb_verdict)
323+ bpf_prog_put(skb_verdict);
324+out_put_msg_parser:
307325 if (msg_parser)
308326 bpf_prog_put(msg_parser);
309327 out_put_stream_parser:
@@ -315,27 +333,6 @@ out_put_stream_verdict: · 5 unmodified lines
315333 return ret;
316334 }
317335
318−static int sock_map_link_no_progs(struct bpf_map *map, struct sock *sk)
319−{
320− struct sk_psock *psock;
321− int ret;
322−
323− psock = sock_map_psock_get_checked(sk);
324− if (IS_ERR(psock))
325− return PTR_ERR(psock);
326−
327− if (!psock) {
328− psock = sk_psock_init(sk, map->numa_node);
329− if (IS_ERR(psock))
330− return PTR_ERR(psock);
331− }
332−
333− ret = sock_map_init_proto(sk, psock);
334− if (ret < 0)
335− sk_psock_put(sk, psock);
336− return ret;
337−}
338−
339336 static void sock_map_free(struct bpf_map *map)
340337 {
341338 struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
@@ -466,8 +463,6 @@ static int sock_map_get_next_key(struct bpf_map *map, void *key, void *next) · 124 unmodified lines
466463 return 0;
467464 }
468465
469−static bool sock_map_redirect_allowed(const struct sock *sk);
470−
471466 static int sock_map_update_common(struct bpf_map *map, u32 idx,
472467 struct sock *sk, u64 flags)
473468 {
@@ -487,14 +482,7 @@ static int sock_map_update_common(struct bpf_map *map, u32 idx, · 13 unmodified lines
487482 if (!link)
488483 return -ENOMEM;
489484
490− /* Only sockets we can redirect into/from in BPF need to hold
491− * refs to parser/verdict progs and have their sk_data_ready
492− * and sk_write_space callbacks overridden.
493− */
494− if (sock_map_redirect_allowed(sk))
495− ret = sock_map_link(map, &stab->progs, sk);
496− else
497− ret = sock_map_link_no_progs(map, sk);
485+ ret = sock_map_link(map, sk);
498486 if (ret < 0)
499487 goto out_free;
500488
@@ -547,12 +535,15 @@ static bool sk_is_udp(const struct sock *sk) · 46 unmodified lines
547535
548536 static bool sock_map_redirect_allowed(const struct sock *sk)
549537 {
550− return sk_is_tcp(sk) && sk->sk_state != TCP_LISTEN;
538+ if (sk_is_tcp(sk))
539+ return sk->sk_state != TCP_LISTEN;
540+ else
541+ return sk->sk_state == TCP_ESTABLISHED;
551542 }
552543
553544 static bool sock_map_sk_is_suitable(const struct sock *sk)
554545 {
555− return sk_is_tcp(sk) || sk_is_udp(sk);
546+ return !!sk->sk_prot->psock_update_sk_prot;
556547 }
557548
558549 static bool sock_map_sk_state_allowed(const struct sock *sk)
@@ -999,14 +990,7 @@ static int sock_hash_update_common(struct bpf_map *map, void *key, · 440 unmodified lines
999990 if (!link)
1000991 return -ENOMEM;
1001992
1002− /* Only sockets we can redirect into/from in BPF need to hold
1003− * refs to parser/verdict progs and have their sk_data_ready
1004− * and sk_write_space callbacks overridden.
1005− */
1006− if (sock_map_redirect_allowed(sk))
1007− ret = sock_map_link(map, &htab->progs, sk);
1008− else
1009− ret = sock_map_link_no_progs(map, sk);
993+ ret = sock_map_link(map, sk);
1010994 if (ret < 0)
1011995 goto out_free;
1012996
@@ -1466,8 +1450,15 @@ static int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog, · 453 unmodified lines
14661450 break;
14671451 #endif
14681452 case BPF_SK_SKB_STREAM_VERDICT:
1453+ if (progs->skb_verdict)
1454+ return -EBUSY;
14691455 pprog = &progs->stream_verdict;
14701456 break;
1457+ case BPF_SK_SKB_VERDICT:
1458+ if (progs->stream_verdict)
1459+ return -EBUSY;
1460+ pprog = &progs->skb_verdict;
1461+ break;
14711462 default:
14721463 return -EOPNOTSUPP;
14731464 }
@@ -1540,6 +1531,7 @@ void sock_map_close(struct sock *sk, long timeout) · 66 unmodified lines
15401531 saved_close = psock->saved_close;
15411532 sock_map_remove_links(sk, psock);
15421533 rcu_read_unlock();
1534+ sk_psock_stop(psock, true);
15431535 release_sock(sk);
15441536 saved_close(sk, timeout);
15451537 }
net/ipv4/af_inet.c+1−0
@@ -1070,6 +1070,7 @@ const struct proto_ops inet_dgram_ops = { · 1069 unmodified lines
10701070 .setsockopt = sock_common_setsockopt,
10711071 .getsockopt = sock_common_getsockopt,
10721072 .sendmsg = inet_sendmsg,
1073+ .read_sock = udp_read_sock,
10731074 .recvmsg = inet_recvmsg,
10741075 .mmap = sock_no_mmap,
10751076 .sendpage = inet_sendpage,
net/ipv4/bpf_tcp_ca.c+43−0
@@ -5,6 +5,7 @@ · 4 unmodified lines
55 #include <linux/bpf_verifier.h>
66 #include <linux/bpf.h>
77 #include <linux/btf.h>
8+#include <linux/btf_ids.h>
89 #include <linux/filter.h>
910 #include <net/tcp.h>
1011 #include <net/bpf_sk_storage.h>
@@ -178,10 +179,52 @@ bpf_tcp_ca_get_func_proto(enum bpf_func_id func_id, · 167 unmodified lines
178179 }
179180 }
180181
182+BTF_SET_START(bpf_tcp_ca_kfunc_ids)
183+BTF_ID(func, tcp_reno_ssthresh)
184+BTF_ID(func, tcp_reno_cong_avoid)
185+BTF_ID(func, tcp_reno_undo_cwnd)
186+BTF_ID(func, tcp_slow_start)
187+BTF_ID(func, tcp_cong_avoid_ai)
188+#ifdef CONFIG_DYNAMIC_FTRACE
189+#if IS_BUILTIN(CONFIG_TCP_CONG_CUBIC)
190+BTF_ID(func, cubictcp_init)
191+BTF_ID(func, cubictcp_recalc_ssthresh)
192+BTF_ID(func, cubictcp_cong_avoid)
193+BTF_ID(func, cubictcp_state)
194+BTF_ID(func, cubictcp_cwnd_event)
195+BTF_ID(func, cubictcp_acked)
196+#endif
197+#if IS_BUILTIN(CONFIG_TCP_CONG_DCTCP)
198+BTF_ID(func, dctcp_init)
199+BTF_ID(func, dctcp_update_alpha)
200+BTF_ID(func, dctcp_cwnd_event)
201+BTF_ID(func, dctcp_ssthresh)
202+BTF_ID(func, dctcp_cwnd_undo)
203+BTF_ID(func, dctcp_state)
204+#endif
205+#if IS_BUILTIN(CONFIG_TCP_CONG_BBR)
206+BTF_ID(func, bbr_init)
207+BTF_ID(func, bbr_main)
208+BTF_ID(func, bbr_sndbuf_expand)
209+BTF_ID(func, bbr_undo_cwnd)
210+BTF_ID(func, bbr_cwnd_event)
211+BTF_ID(func, bbr_ssthresh)
212+BTF_ID(func, bbr_min_tso_segs)
213+BTF_ID(func, bbr_set_state)
214+#endif
215+#endif /* CONFIG_DYNAMIC_FTRACE */
216+BTF_SET_END(bpf_tcp_ca_kfunc_ids)
217+
218+static bool bpf_tcp_ca_check_kfunc_call(u32 kfunc_btf_id)
219+{
220+ return btf_id_set_contains(&bpf_tcp_ca_kfunc_ids, kfunc_btf_id);
221+}
222+
181223 static const struct bpf_verifier_ops bpf_tcp_ca_verifier_ops = {
182224 .get_func_proto = bpf_tcp_ca_get_func_proto,
183225 .is_valid_access = bpf_tcp_ca_is_valid_access,
184226 .btf_struct_access = bpf_tcp_ca_btf_struct_access,
227+ .check_kfunc_call = bpf_tcp_ca_check_kfunc_call,
185228 };
186229
187230 static int bpf_tcp_ca_init_member(const struct btf_type *t,
net/ipv4/tcp_bpf.c+23−107
@@ -10,86 +10,6 @@ · 9 unmodified lines
1010 #include <net/inet_common.h>
1111 #include <net/tls.h>
1212
13−int __tcp_bpf_recvmsg(struct sock *sk, struct sk_psock *psock,
14− struct msghdr *msg, int len, int flags)
15−{
16− struct iov_iter *iter = &msg->msg_iter;
17− int peek = flags & MSG_PEEK;
18− struct sk_msg *msg_rx;
19− int i, copied = 0;
20−
21− msg_rx = list_first_entry_or_null(&psock->ingress_msg,
22− struct sk_msg, list);
23−
24− while (copied != len) {
25− struct scatterlist *sge;
26−
27− if (unlikely(!msg_rx))
28− break;
29−
30− i = msg_rx->sg.start;
31− do {
32− struct page *page;
33− int copy;
34−
35− sge = sk_msg_elem(msg_rx, i);
36− copy = sge->length;
37− page = sg_page(sge);
38− if (copied + copy > len)
39− copy = len - copied;
40− copy = copy_page_to_iter(page, sge->offset, copy, iter);
41− if (!copy)
42− return copied ? copied : -EFAULT;
43−
44− copied += copy;
45− if (likely(!peek)) {
46− sge->offset += copy;
47− sge->length -= copy;
48− if (!msg_rx->skb)
49− sk_mem_uncharge(sk, copy);
50− msg_rx->sg.size -= copy;
51−
52− if (!sge->length) {
53− sk_msg_iter_var_next(i);
54− if (!msg_rx->skb)
55− put_page(page);
56− }
57− } else {
58− /* Lets not optimize peek case if copy_page_to_iter
59− * didn't copy the entire length lets just break.
60− */
61− if (copy != sge->length)
62− return copied;
63− sk_msg_iter_var_next(i);
64− }
65−
66− if (copied == len)
67− break;
68− } while (i != msg_rx->sg.end);
69−
70− if (unlikely(peek)) {
71− if (msg_rx == list_last_entry(&psock->ingress_msg,
72− struct sk_msg, list))
73− break;
74− msg_rx = list_next_entry(msg_rx, list);
75− continue;
76− }
77−
78− msg_rx->sg.start = i;
79− if (!sge->length && msg_rx->sg.start == msg_rx->sg.end) {
80− list_del(&msg_rx->list);
81− if (msg_rx->skb)
82− consume_skb(msg_rx->skb);
83− kfree(msg_rx);
84− }
85− msg_rx = list_first_entry_or_null(&psock->ingress_msg,
86− struct sk_msg, list);
87− }
88−
89− return copied;
90−}
91−EXPORT_SYMBOL_GPL(__tcp_bpf_recvmsg);
92−
9313 static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock,
9414 struct sk_msg *msg, u32 apply_bytes, int flags)
9515 {
@@ -243,28 +163,6 @@ static bool tcp_bpf_stream_read(const struct sock *sk) · 147 unmodified lines
243163 return !empty;
244164 }
245165
246−static int tcp_bpf_wait_data(struct sock *sk, struct sk_psock *psock,
247− int flags, long timeo, int *err)
248−{
249− DEFINE_WAIT_FUNC(wait, woken_wake_function);
250− int ret = 0;
251−
252− if (sk->sk_shutdown & RCV_SHUTDOWN)
253− return 1;
254−
255− if (!timeo)
256− return ret;
257−
258− add_wait_queue(sk_sleep(sk), &wait);
259− sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
260− ret = sk_wait_event(sk, &timeo,
261− !list_empty(&psock->ingress_msg) ||
262− !skb_queue_empty(&sk->sk_receive_queue), &wait);
263− sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
264− remove_wait_queue(sk_sleep(sk), &wait);
265− return ret;
266−}
267−
268166 static int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
269167 int nonblock, int flags, int *addr_len)
270168 {
@@ -284,13 +182,13 @@ static int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, · 13 unmodified lines
284182 }
285183 lock_sock(sk);
286184 msg_bytes_ready:
287− copied = __tcp_bpf_recvmsg(sk, psock, msg, len, flags);
185+ copied = sk_msg_recvmsg(sk, psock, msg, len, flags);
288186 if (!copied) {
289187 int data, err = 0;
290188 long timeo;
291189
292190 timeo = sock_rcvtimeo(sk, nonblock);
293− data = tcp_bpf_wait_data(sk, psock, flags, timeo, &err);
191+ data = sk_msg_wait_data(sk, psock, flags, timeo, &err);
294192 if (data) {
295193 if (!sk_psock_queue_empty(psock))
296194 goto msg_bytes_ready;
@@ -601,20 +499,38 @@ static int tcp_bpf_assert_proto_ops(struct proto *ops) · 304 unmodified lines
601499 ops->sendpage == tcp_sendpage ? 0 : -ENOTSUPP;
602500 }
603501
604−struct proto *tcp_bpf_get_proto(struct sock *sk, struct sk_psock *psock)
502+int tcp_bpf_update_proto(struct sock *sk, bool restore)
605503 {
504+ struct sk_psock *psock = sk_psock(sk);
606505 int family = sk->sk_family == AF_INET6 ? TCP_BPF_IPV6 : TCP_BPF_IPV4;
607506 int config = psock->progs.msg_parser ? TCP_BPF_TX : TCP_BPF_BASE;
608507
508+ if (restore) {
509+ if (inet_csk_has_ulp(sk)) {
510+ tcp_update_ulp(sk, psock->sk_proto, psock->saved_write_space);
511+ } else {
512+ sk->sk_write_space = psock->saved_write_space;
513+ /* Pairs with lockless read in sk_clone_lock() */
514+ WRITE_ONCE(sk->sk_prot, psock->sk_proto);
515+ }
516+ return 0;
517+ }
518+
519+ if (inet_csk_has_ulp(sk))
520+ return -EINVAL;
521+
609522 if (sk->sk_family == AF_INET6) {
610523 if (tcp_bpf_assert_proto_ops(psock->sk_proto))
611− return ERR_PTR(-EINVAL);
524+ return -EINVAL;
612525
613526 tcp_bpf_check_v6_needs_rebuild(psock->sk_proto);
614527 }
615528
616− return &tcp_bpf_prots[family][config];
529+ /* Pairs with lockless read in sk_clone_lock() */
530+ WRITE_ONCE(sk->sk_prot, &tcp_bpf_prots[family][config]);
531+ return 0;
617532 }
533+EXPORT_SYMBOL_GPL(tcp_bpf_update_proto);
618534
619535 /* If a child got cloned from a listening socket that had tcp_bpf
620536 * protocol callbacks installed, we need to restore the callbacks to
net/ipv4/tcp_cubic.c+12−12
@@ -124,7 +124,7 @@ static inline void bictcp_hystart_reset(struct sock *sk) · 123 unmodified lines
124124 ca->sample_cnt = 0;
125125 }
126126
127−static void bictcp_init(struct sock *sk)
127+static void cubictcp_init(struct sock *sk)
128128 {
129129 struct bictcp *ca = inet_csk_ca(sk);
130130
@@ -137,7 +137,7 @@ static void bictcp_init(struct sock *sk) · 6 unmodified lines
137137 tcp_sk(sk)->snd_ssthresh = initial_ssthresh;
138138 }
139139
140−static void bictcp_cwnd_event(struct sock *sk, enum tcp_ca_event event)
140+static void cubictcp_cwnd_event(struct sock *sk, enum tcp_ca_event event)
141141 {
142142 if (event == CA_EVENT_TX_START) {
143143 struct bictcp *ca = inet_csk_ca(sk);
@@ -319,7 +319,7 @@ tcp_friendliness: · 175 unmodified lines
319319 ca->cnt = max(ca->cnt, 2U);
320320 }
321321
322−static void bictcp_cong_avoid(struct sock *sk, u32 ack, u32 acked)
322+static void cubictcp_cong_avoid(struct sock *sk, u32 ack, u32 acked)
323323 {
324324 struct tcp_sock *tp = tcp_sk(sk);
325325 struct bictcp *ca = inet_csk_ca(sk);
@@ -338,7 +338,7 @@ static void bictcp_cong_avoid(struct sock *sk, u32 ack, u32 acked) · 12 unmodified lines
338338 tcp_cong_avoid_ai(tp, ca->cnt, acked);
339339 }
340340
341−static u32 bictcp_recalc_ssthresh(struct sock *sk)
341+static u32 cubictcp_recalc_ssthresh(struct sock *sk)
342342 {
343343 const struct tcp_sock *tp = tcp_sk(sk);
344344 struct bictcp *ca = inet_csk_ca(sk);
@@ -355,7 +355,7 @@ static u32 bictcp_recalc_ssthresh(struct sock *sk) · 10 unmodified lines
355355 return max((tp->snd_cwnd * beta) / BICTCP_BETA_SCALE, 2U);
356356 }
357357
358−static void bictcp_state(struct sock *sk, u8 new_state)
358+static void cubictcp_state(struct sock *sk, u8 new_state)
359359 {
360360 if (new_state == TCP_CA_Loss) {
361361 bictcp_reset(inet_csk_ca(sk));
@@ -442,7 +442,7 @@ static void hystart_update(struct sock *sk, u32 delay) · 80 unmodified lines
442442 }
443443 }
444444
445−static void bictcp_acked(struct sock *sk, const struct ack_sample *sample)
445+static void cubictcp_acked(struct sock *sk, const struct ack_sample *sample)
446446 {
447447 const struct tcp_sock *tp = tcp_sk(sk);
448448 struct bictcp *ca = inet_csk_ca(sk);
@@ -471,13 +471,13 @@ static void bictcp_acked(struct sock *sk, const struct ack_sample *sample) · 22 unmodified lines
471471 }
472472
473473 static struct tcp_congestion_ops cubictcp __read_mostly = {
474− .init = bictcp_init,
475− .ssthresh = bictcp_recalc_ssthresh,
476− .cong_avoid = bictcp_cong_avoid,
477− .set_state = bictcp_state,
474+ .init = cubictcp_init,
475+ .ssthresh = cubictcp_recalc_ssthresh,
476+ .cong_avoid = cubictcp_cong_avoid,
477+ .set_state = cubictcp_state,
478478 .undo_cwnd = tcp_reno_undo_cwnd,
479− .cwnd_event = bictcp_cwnd_event,
480− .pkts_acked = bictcp_acked,
479+ .cwnd_event = cubictcp_cwnd_event,
480+ .pkts_acked = cubictcp_acked,
481481 .owner = THIS_MODULE,
482482 .name = "cubic",
483483 };
net/ipv4/tcp_ipv4.c+3−0
@@ -2806,6 +2806,9 @@ struct proto tcp_prot = { · 2805 unmodified lines
28062806 .hash = inet_hash,
28072807 .unhash = inet_unhash,
28082808 .get_port = inet_csk_get_port,
2809+#ifdef CONFIG_BPF_SYSCALL
2810+ .psock_update_sk_prot = tcp_bpf_update_proto,
2811+#endif
28092812 .enter_memory_pressure = tcp_enter_memory_pressure,
28102813 .leave_memory_pressure = tcp_leave_memory_pressure,
28112814 .stream_memory_free = tcp_stream_memory_free,
net/ipv4/udp.c+32−0
@@ -1782,6 +1782,35 @@ busy_check: · 1781 unmodified lines
17821782 }
17831783 EXPORT_SYMBOL(__skb_recv_udp);
17841784
1785+int udp_read_sock(struct sock *sk, read_descriptor_t *desc,
1786+ sk_read_actor_t recv_actor)
1787+{
1788+ int copied = 0;
1789+
1790+ while (1) {
1791+ struct sk_buff *skb;
1792+ int err, used;
1793+
1794+ skb = skb_recv_udp(sk, 0, 1, &err);
1795+ if (!skb)
1796+ return err;
1797+ used = recv_actor(desc, skb, 0, skb->len);
1798+ if (used <= 0) {
1799+ if (!copied)
1800+ copied = used;
1801+ break;
1802+ } else if (used <= skb->len) {
1803+ copied += used;
1804+ }
1805+
1806+ if (!desc->count)
1807+ break;
1808+ }
1809+
1810+ return copied;
1811+}
1812+EXPORT_SYMBOL(udp_read_sock);
1813+
17851814 /*
17861815 * This should be easy, if there is something there we
17871816 * return it, otherwise we block.
@@ -2854,6 +2883,9 @@ struct proto udp_prot = { · 1066 unmodified lines
28542883 .unhash = udp_lib_unhash,
28552884 .rehash = udp_v4_rehash,
28562885 .get_port = udp_v4_get_port,
2886+#ifdef CONFIG_BPF_SYSCALL
2887+ .psock_update_sk_prot = udp_bpf_update_proto,
2888+#endif
28572889 .memory_allocated = &udp_memory_allocated,
28582890 .sysctl_mem = sysctl_udp_mem,
28592891 .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_udp_wmem_min),
net/ipv4/udp_bpf.c+76−3
@@ -4,6 +4,68 @@ · 3 unmodified lines
44 #include <linux/skmsg.h>
55 #include <net/sock.h>
66 #include <net/udp.h>
7+#include <net/inet_common.h>
8+
9+#include "udp_impl.h"
10+
11+static struct proto *udpv6_prot_saved __read_mostly;
12+
13+static int sk_udp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
14+ int noblock, int flags, int *addr_len)
15+{
16+#if IS_ENABLED(CONFIG_IPV6)
17+ if (sk->sk_family == AF_INET6)
18+ return udpv6_prot_saved->recvmsg(sk, msg, len, noblock, flags,
19+ addr_len);
20+#endif
21+ return udp_prot.recvmsg(sk, msg, len, noblock, flags, addr_len);
22+}
23+
24+static int udp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
25+ int nonblock, int flags, int *addr_len)
26+{
27+ struct sk_psock *psock;
28+ int copied, ret;
29+
30+ if (unlikely(flags & MSG_ERRQUEUE))
31+ return inet_recv_error(sk, msg, len, addr_len);
32+
33+ psock = sk_psock_get(sk);
34+ if (unlikely(!psock))
35+ return sk_udp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
36+
37+ lock_sock(sk);
38+ if (sk_psock_queue_empty(psock)) {
39+ ret = sk_udp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
40+ goto out;
41+ }
42+
43+msg_bytes_ready:
44+ copied = sk_msg_recvmsg(sk, psock, msg, len, flags);
45+ if (!copied) {
46+ int data, err = 0;
47+ long timeo;
48+
49+ timeo = sock_rcvtimeo(sk, nonblock);
50+ data = sk_msg_wait_data(sk, psock, flags, timeo, &err);
51+ if (data) {
52+ if (!sk_psock_queue_empty(psock))
53+ goto msg_bytes_ready;
54+ ret = sk_udp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
55+ goto out;
56+ }
57+ if (err) {
58+ ret = err;
59+ goto out;
60+ }
61+ copied = -EAGAIN;
62+ }
63+ ret = copied;
64+out:
65+ release_sock(sk);
66+ sk_psock_put(sk, psock);
67+ return ret;
68+}
769
870 enum {
971 UDP_BPF_IPV4,
@@ -11,7 +73,6 @@ enum { · 1 unmodified line
1173 UDP_BPF_NUM_PROTS,
1274 };
1375
14−static struct proto *udpv6_prot_saved __read_mostly;
1576 static DEFINE_SPINLOCK(udpv6_prot_lock);
1677 static struct proto udp_bpf_prots[UDP_BPF_NUM_PROTS];
1778
@@ -20,6 +81,7 @@ static void udp_bpf_rebuild_protos(struct proto *prot, const struct proto *base) · 2 unmodified lines
2081 *prot = *base;
2182 prot->unhash = sock_map_unhash;
2283 prot->close = sock_map_close;
84+ prot->recvmsg = udp_bpf_recvmsg;
2385 }
2486
2587 static void udp_bpf_check_v6_needs_rebuild(struct proto *ops)
@@ -41,12 +103,23 @@ static int __init udp_bpf_v4_build_proto(void) · 15 unmodified lines
41103 }
42104 core_initcall(udp_bpf_v4_build_proto);
43105
44−struct proto *udp_bpf_get_proto(struct sock *sk, struct sk_psock *psock)
106+int udp_bpf_update_proto(struct sock *sk, bool restore)
45107 {
46108 int family = sk->sk_family == AF_INET ? UDP_BPF_IPV4 : UDP_BPF_IPV6;
109+ struct sk_psock *psock = sk_psock(sk);
110+
111+ if (restore) {
112+ sk->sk_write_space = psock->saved_write_space;
113+ /* Pairs with lockless read in sk_clone_lock() */
114+ WRITE_ONCE(sk->sk_prot, psock->sk_proto);
115+ return 0;
116+ }
47117
48118 if (sk->sk_family == AF_INET6)
49119 udp_bpf_check_v6_needs_rebuild(psock->sk_proto);
50120
51− return &udp_bpf_prots[family];
121+ /* Pairs with lockless read in sk_clone_lock() */
122+ WRITE_ONCE(sk->sk_prot, &udp_bpf_prots[family]);
123+ return 0;
52124 }
125+EXPORT_SYMBOL_GPL(udp_bpf_update_proto);
net/ipv6/af_inet6.c+1−0
@@ -714,6 +714,7 @@ const struct proto_ops inet6_dgram_ops = { · 713 unmodified lines
714714 .getsockopt = sock_common_getsockopt, /* ok */
715715 .sendmsg = inet6_sendmsg, /* retpoline's sake */
716716 .recvmsg = inet6_recvmsg, /* retpoline's sake */
717+ .read_sock = udp_read_sock,
717718 .mmap = sock_no_mmap,
718719 .sendpage = sock_no_sendpage,
719720 .set_peek_off = sk_set_peek_off,
net/ipv6/tcp_ipv6.c+3−0
@@ -2139,6 +2139,9 @@ struct proto tcpv6_prot = { · 2138 unmodified lines
21392139 .hash = inet6_hash,
21402140 .unhash = inet_unhash,
21412141 .get_port = inet_csk_get_port,
2142+#ifdef CONFIG_BPF_SYSCALL
2143+ .psock_update_sk_prot = tcp_bpf_update_proto,
2144+#endif
21422145 .enter_memory_pressure = tcp_enter_memory_pressure,
21432146 .leave_memory_pressure = tcp_leave_memory_pressure,
21442147 .stream_memory_free = tcp_stream_memory_free,
net/ipv6/udp.c+3−0
@@ -1714,6 +1714,9 @@ struct proto udpv6_prot = { · 1713 unmodified lines
17141714 .unhash = udp_lib_unhash,
17151715 .rehash = udp_v6_rehash,
17161716 .get_port = udp_v6_get_port,
1717+#ifdef CONFIG_BPF_SYSCALL
1718+ .psock_update_sk_prot = udp_bpf_update_proto,
1719+#endif
17171720 .memory_allocated = &udp_memory_allocated,
17181721 .sysctl_mem = sysctl_udp_mem,
17191722 .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_udp_wmem_min),
net/tls/tls_sw.c+2−2
@@ -1789,8 +1789,8 @@ int tls_sw_recvmsg(struct sock *sk, · 1788 unmodified lines
17891789 skb = tls_wait_data(sk, psock, flags, timeo, &err);
17901790 if (!skb) {
17911791 if (psock) {
1792− int ret = __tcp_bpf_recvmsg(sk, psock,
1793− msg, len, flags);
1792+ int ret = sk_msg_recvmsg(sk, psock, msg, len,
1793+ flags);
17941794
17951795 if (ret > 0) {
17961796 decrypted += ret;
samples/bpf/sampleip_kern.c+0−1
@@ -4,7 +4,6 @@ · 3 unmodified lines
44 * modify it under the terms of version 2 of the GNU General Public
55 * License as published by the Free Software Foundation.
66 */
7−#include <linux/version.h>
87 #include <linux/ptrace.h>
98 #include <uapi/linux/bpf.h>
109 #include <uapi/linux/bpf_perf_event.h>
samples/bpf/trace_event_kern.c+0−1
@@ -5,7 +5,6 @@ · 4 unmodified lines
55 * License as published by the Free Software Foundation.
66 */
77 #include <linux/ptrace.h>
8−#include <linux/version.h>
98 #include <uapi/linux/bpf.h>
109 #include <uapi/linux/bpf_perf_event.h>
1110 #include <uapi/linux/perf_event.h>
samples/bpf/xdpsock_user.c+14−41
@@ -96,7 +96,6 @@ static int opt_xsk_frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; · 95 unmodified lines
9696 static int opt_timeout = 1000;
9797 static bool opt_need_wakeup = true;
9898 static u32 opt_num_xsks = 1;
99−static u32 prog_id;
10099 static bool opt_busy_poll;
101100 static bool opt_reduced_cap;
102101
@@ -462,59 +461,37 @@ static void *poller(void *arg) · 359 unmodified lines
462461 return NULL;
463462 }
464463
465−static void remove_xdp_program(void)
464+static void int_exit(int sig)
466465 {
467− u32 curr_prog_id = 0;
468− int cmd = CLOSE_CONN;
469−
470− if (bpf_get_link_xdp_id(opt_ifindex, &curr_prog_id, opt_xdp_flags)) {
471− printf("bpf_get_link_xdp_id failed\n");
472− exit(EXIT_FAILURE);
473− }
474− if (prog_id == curr_prog_id)
475− bpf_set_link_xdp_fd(opt_ifindex, -1, opt_xdp_flags);
476− else if (!curr_prog_id)
477− printf("couldn't find a prog id on a given interface\n");
478− else
479− printf("program on interface changed, not removing\n");
480−
481− if (opt_reduced_cap) {
482− if (write(sock, &cmd, sizeof(int)) < 0) {
483− fprintf(stderr, "Error writing into stream socket: %s", strerror(errno));
484− exit(EXIT_FAILURE);
485− }
486− }
466+ benchmark_done = true;
487467 }
488468
489−static void int_exit(int sig)
469+static void __exit_with_error(int error, const char *file, const char *func,
470+ int line)
490471 {
491− benchmark_done = true;
472+ fprintf(stderr, "%s:%s:%i: errno: %d/\"%s\"\n", file, func,
473+ line, error, strerror(error));
474+ exit(EXIT_FAILURE);
492475 }
493476
477+#define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__)
478+
494479 static void xdpsock_cleanup(void)
495480 {
496481 struct xsk_umem *umem = xsks[0]->umem->umem;
497− int i;
482+ int i, cmd = CLOSE_CONN;
498483
499484 dump_stats();
500485 for (i = 0; i < num_socks; i++)
501486 xsk_socket__delete(xsks[i]->xsk);
502487 (void)xsk_umem__delete(umem);
503− remove_xdp_program();
504−}
505488
506−static void __exit_with_error(int error, const char *file, const char *func,
507− int line)
508−{
509− fprintf(stderr, "%s:%s:%i: errno: %d/\"%s\"\n", file, func,
510− line, error, strerror(error));
511− dump_stats();
512− remove_xdp_program();
513− exit(EXIT_FAILURE);
489+ if (opt_reduced_cap) {
490+ if (write(sock, &cmd, sizeof(int)) < 0)
491+ exit_with_error(errno);
492+ }
514493 }
515494
516−#define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, \
517− __LINE__)
518495 static void swap_mac_addresses(void *data)
519496 {
520497 struct ether_header *eth = (struct ether_header *)data;
@@ -880,10 +857,6 @@ static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem, · 359 unmodified lines
880857 if (ret)
881858 exit_with_error(-ret);
882859
883− ret = bpf_get_link_xdp_id(opt_ifindex, &prog_id, opt_xdp_flags);
884− if (ret)
885− exit_with_error(-ret);
886−
887860 xsk->app_stats.rx_empty_polls = 0;
888861 xsk->app_stats.fill_fail_polls = 0;
889862 xsk->app_stats.copy_tx_sendtos = 0;
tools/bpf/bpftool/common.c+1−0
@@ -57,6 +57,7 @@ const char * const attach_type_name[__MAX_BPF_ATTACH_TYPE] = { · 56 unmodified lines
5757
5858 [BPF_SK_SKB_STREAM_PARSER] = "sk_skb_stream_parser",
5959 [BPF_SK_SKB_STREAM_VERDICT] = "sk_skb_stream_verdict",
60+ [BPF_SK_SKB_VERDICT] = "sk_skb_verdict",
6061 [BPF_SK_MSG_VERDICT] = "sk_msg_verdict",
6162 [BPF_LIRC_MODE2] = "lirc_mode2",
6263 [BPF_FLOW_DISSECTOR] = "flow_dissector",
tools/bpf/bpftool/prog.c+1−0
@@ -76,6 +76,7 @@ enum dump_mode { · 75 unmodified lines
7676 static const char * const attach_type_strings[] = {
7777 [BPF_SK_SKB_STREAM_PARSER] = "stream_parser",
7878 [BPF_SK_SKB_STREAM_VERDICT] = "stream_verdict",
79+ [BPF_SK_SKB_VERDICT] = "skb_verdict",
7980 [BPF_SK_MSG_VERDICT] = "msg_verdict",
8081 [BPF_FLOW_DISSECTOR] = "flow_dissector",
8182 [__MAX_BPF_ATTACH_TYPE] = NULL,
tools/bpf/resolve_btfids/main.c+5−6
@@ -115,10 +115,10 @@ struct object { · 114 unmodified lines
115115
116116 static int verbose;
117117
118−int eprintf(int level, int var, const char *fmt, ...)
118+static int eprintf(int level, int var, const char *fmt, ...)
119119 {
120120 va_list args;
121− int ret;
121+ int ret = 0;
122122
123123 if (var >= level) {
124124 va_start(args, fmt);
@@ -385,7 +385,7 @@ static int elf_collect(struct object *obj) · 260 unmodified lines
385385 static int symbols_collect(struct object *obj)
386386 {
387387 Elf_Scn *scn = NULL;
388− int n, i, err = 0;
388+ int n, i;
389389 GElf_Shdr sh;
390390 char *name;
391391
@@ -402,11 +402,10 @@ static int symbols_collect(struct object *obj) · 10 unmodified lines
402402 * Scan symbols and look for the ones starting with
403403 * __BTF_ID__* over .BTF_ids section.
404404 */
405− for (i = 0; !err && i < n; i++) {
406− char *tmp, *prefix;
405+ for (i = 0; i < n; i++) {
406+ char *prefix;
407407 struct btf_id *id;
408408 GElf_Sym sym;
409− int err = -1;
410409
411410 if (!gelf_getsym(obj->efile.symbols, i, &sym))
412411 return -1;
tools/include/uapi/linux/bpf.h+5−0
@@ -957,6 +957,7 @@ enum bpf_attach_type { · 956 unmodified lines
957957 BPF_XDP_CPUMAP,
958958 BPF_SK_LOOKUP,
959959 BPF_XDP,
960+ BPF_SK_SKB_VERDICT,
960961 __MAX_BPF_ATTACH_TYPE
961962 };
962963
@@ -1117,6 +1118,10 @@ enum bpf_link_type { · 154 unmodified lines
11171118 * offset to another bpf function
11181119 */
11191120 #define BPF_PSEUDO_CALL 1
1121+/* when bpf_call->src_reg == BPF_PSEUDO_KFUNC_CALL,
1122+ * bpf_call->imm == btf_id of a BTF_KIND_FUNC in the running kernel
1123+ */
1124+#define BPF_PSEUDO_KFUNC_CALL 2
11201125
11211126 /* flags for BPF_MAP_UPDATE_ELEM command */
11221127 enum {
tools/lib/bpf/libbpf.c+300−103
@@ -185,7 +185,8 @@ enum reloc_type { · 184 unmodified lines
185185 RELO_LD64,
186186 RELO_CALL,
187187 RELO_DATA,
188− RELO_EXTERN,
188+ RELO_EXTERN_VAR,
189+ RELO_EXTERN_FUNC,
189190 RELO_SUBPROG_ADDR,
190191 };
191192
@@ -573,14 +574,19 @@ static bool insn_is_subprog_call(const struct bpf_insn *insn) · 381 unmodified lines
573574 insn->off == 0;
574575 }
575576
576−static bool is_ldimm64(struct bpf_insn *insn)
577+static bool is_ldimm64_insn(struct bpf_insn *insn)
577578 {
578579 return insn->code == (BPF_LD | BPF_IMM | BPF_DW);
579580 }
580581
582+static bool is_call_insn(const struct bpf_insn *insn)
583+{
584+ return insn->code == (BPF_JMP | BPF_CALL);
585+}
586+
581587 static bool insn_is_pseudo_func(struct bpf_insn *insn)
582588 {
583− return is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
589+ return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
584590 }
585591
586592 static int
@@ -1921,9 +1927,9 @@ resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id) · 1334 unmodified lines
19211927 return btf_is_func_proto(t) ? t : NULL;
19221928 }
19231929
1924−static const char *btf_kind_str(const struct btf_type *t)
1930+static const char *__btf_kind_str(__u16 kind)
19251931 {
1926− switch (btf_kind(t)) {
1932+ switch (kind) {
19271933 case BTF_KIND_UNKN: return "void";
19281934 case BTF_KIND_INT: return "int";
19291935 case BTF_KIND_PTR: return "ptr";
@@ -1945,6 +1951,16 @@ static const char *btf_kind_str(const struct btf_type *t) · 15 unmodified lines
19451951 }
19461952 }
19471953
1954+static const char *btf_kind_str(const struct btf_type *t)
1955+{
1956+ return __btf_kind_str(btf_kind(t));
1957+}
1958+
1959+static enum btf_func_linkage btf_func_linkage(const struct btf_type *t)
1960+{
1961+ return (enum btf_func_linkage)BTF_INFO_VLEN(t->info);
1962+}
1963+
19481964 /*
19491965 * Fetch integer attribute of BTF map definition. Such attributes are
19501966 * represented using a pointer to an array, in which dimensionality of array
@@ -3009,7 +3025,7 @@ static bool sym_is_subprog(const GElf_Sym *sym, int text_shndx) · 1058 unmodified lines
30093025 static int find_extern_btf_id(const struct btf *btf, const char *ext_name)
30103026 {
30113027 const struct btf_type *t;
3012− const char *var_name;
3028+ const char *tname;
30133029 int i, n;
30143030
30153031 if (!btf)
@@ -3019,14 +3035,18 @@ static int find_extern_btf_id(const struct btf *btf, const char *ext_name) · 3 unmodified lines
30193035 for (i = 1; i <= n; i++) {
30203036 t = btf__type_by_id(btf, i);
30213037
3022− if (!btf_is_var(t))
3038+ if (!btf_is_var(t) && !btf_is_func(t))
30233039 continue;
30243040
3025− var_name = btf__name_by_offset(btf, t->name_off);
3026− if (strcmp(var_name, ext_name))
3041+ tname = btf__name_by_offset(btf, t->name_off);
3042+ if (strcmp(tname, ext_name))
30273043 continue;
30283044
3029− if (btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
3045+ if (btf_is_var(t) &&
3046+ btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
3047+ return -EINVAL;
3048+
3049+ if (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_EXTERN)
30303050 return -EINVAL;
30313051
30323052 return i;
@@ -3139,12 +3159,48 @@ static int find_int_btf_id(const struct btf *btf) · 106 unmodified lines
31393159 return 0;
31403160 }
31413161
3162+static int add_dummy_ksym_var(struct btf *btf)
3163+{
3164+ int i, int_btf_id, sec_btf_id, dummy_var_btf_id;
3165+ const struct btf_var_secinfo *vs;
3166+ const struct btf_type *sec;
3167+
3168+ sec_btf_id = btf__find_by_name_kind(btf, KSYMS_SEC,
3169+ BTF_KIND_DATASEC);
3170+ if (sec_btf_id < 0)
3171+ return 0;
3172+
3173+ sec = btf__type_by_id(btf, sec_btf_id);
3174+ vs = btf_var_secinfos(sec);
3175+ for (i = 0; i < btf_vlen(sec); i++, vs++) {
3176+ const struct btf_type *vt;
3177+
3178+ vt = btf__type_by_id(btf, vs->type);
3179+ if (btf_is_func(vt))
3180+ break;
3181+ }
3182+
3183+ /* No func in ksyms sec. No need to add dummy var. */
3184+ if (i == btf_vlen(sec))
3185+ return 0;
3186+
3187+ int_btf_id = find_int_btf_id(btf);
3188+ dummy_var_btf_id = btf__add_var(btf,
3189+ "dummy_ksym",
3190+ BTF_VAR_GLOBAL_ALLOCATED,
3191+ int_btf_id);
3192+ if (dummy_var_btf_id < 0)
3193+ pr_warn("cannot create a dummy_ksym var\n");
3194+
3195+ return dummy_var_btf_id;
3196+}
3197+
31423198 static int bpf_object__collect_externs(struct bpf_object *obj)
31433199 {
31443200 struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL;
31453201 const struct btf_type *t;
31463202 struct extern_desc *ext;
3147− int i, n, off;
3203+ int i, n, off, dummy_var_btf_id;
31483204 const char *ext_name, *sec_name;
31493205 Elf_Scn *scn;
31503206 GElf_Shdr sh;
@@ -3156,6 +3212,10 @@ static int bpf_object__collect_externs(struct bpf_object *obj) · 5 unmodified lines
31563212 if (elf_sec_hdr(obj, scn, &sh))
31573213 return -LIBBPF_ERRNO__FORMAT;
31583214
3215+ dummy_var_btf_id = add_dummy_ksym_var(obj->btf);
3216+ if (dummy_var_btf_id < 0)
3217+ return dummy_var_btf_id;
3218+
31593219 n = sh.sh_size / sh.sh_entsize;
31603220 pr_debug("looking for externs among %d symbols...\n", n);
31613221
@@ -3200,6 +3260,11 @@ static int bpf_object__collect_externs(struct bpf_object *obj) · 38 unmodified lines
32003260 sec_name = btf__name_by_offset(obj->btf, sec->name_off);
32013261
32023262 if (strcmp(sec_name, KCONFIG_SEC) == 0) {
3263+ if (btf_is_func(t)) {
3264+ pr_warn("extern function %s is unsupported under %s section\n",
3265+ ext->name, KCONFIG_SEC);
3266+ return -ENOTSUP;
3267+ }
32033268 kcfg_sec = sec;
32043269 ext->type = EXT_KCFG;
32053270 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);
@@ -3221,6 +3286,11 @@ static int bpf_object__collect_externs(struct bpf_object *obj) · 15 unmodified lines
32213286 return -ENOTSUP;
32223287 }
32233288 } else if (strcmp(sec_name, KSYMS_SEC) == 0) {
3289+ if (btf_is_func(t) && ext->is_weak) {
3290+ pr_warn("extern weak function %s is unsupported\n",
3291+ ext->name);
3292+ return -ENOTSUP;
3293+ }
32243294 ksym_sec = sec;
32253295 ext->type = EXT_KSYM;
32263296 skip_mods_and_typedefs(obj->btf, t->type,
@@ -3247,7 +3317,14 @@ static int bpf_object__collect_externs(struct bpf_object *obj) · 20 unmodified lines
32473317 * extern variables in DATASEC
32483318 */
32493319 int int_btf_id = find_int_btf_id(obj->btf);
3320+ /* For extern function, a dummy_var added earlier
3321+ * will be used to replace the vs->type and
3322+ * its name string will be used to refill
3323+ * the missing param's name.
3324+ */
3325+ const struct btf_type *dummy_var;
32503326
3327+ dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id);
32513328 for (i = 0; i < obj->nr_extern; i++) {
32523329 ext = &obj->externs[i];
32533330 if (ext->type != EXT_KSYM)
@@ -3266,12 +3343,32 @@ static int bpf_object__collect_externs(struct bpf_object *obj) · 12 unmodified lines
32663343 ext_name = btf__name_by_offset(obj->btf, vt->name_off);
32673344 ext = find_extern_by_name(obj, ext_name);
32683345 if (!ext) {
3269− pr_warn("failed to find extern definition for BTF var '%s'\n",
3270− ext_name);
3346+ pr_warn("failed to find extern definition for BTF %s '%s'\n",
3347+ btf_kind_str(vt), ext_name);
32713348 return -ESRCH;
32723349 }
3273− btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
3274− vt->type = int_btf_id;
3350+ if (btf_is_func(vt)) {
3351+ const struct btf_type *func_proto;
3352+ struct btf_param *param;
3353+ int j;
3354+
3355+ func_proto = btf__type_by_id(obj->btf,
3356+ vt->type);
3357+ param = btf_params(func_proto);
3358+ /* Reuse the dummy_var string if the
3359+ * func proto does not have param name.
3360+ */
3361+ for (j = 0; j < btf_vlen(func_proto); j++)
3362+ if (param[j].type && !param[j].name_off)
3363+ param[j].name_off =
3364+ dummy_var->name_off;
3365+ vs->type = dummy_var_btf_id;
3366+ vt->info &= ~0xffff;
3367+ vt->info |= BTF_FUNC_GLOBAL;
3368+ } else {
3369+ btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
3370+ vt->type = int_btf_id;
3371+ }
32753372 vs->offset = off;
32763373 vs->size = sizeof(int);
32773374 }
@@ -3403,31 +3500,7 @@ static int bpf_program__record_reloc(struct bpf_program *prog, · 125 unmodified lines
34033500
34043501 reloc_desc->processed = false;
34053502
3406− /* sub-program call relocation */
3407− if (insn->code == (BPF_JMP | BPF_CALL)) {
3408− if (insn->src_reg != BPF_PSEUDO_CALL) {
3409− pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
3410− return -LIBBPF_ERRNO__RELOC;
3411− }
3412− /* text_shndx can be 0, if no default "main" program exists */
3413− if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
3414− sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
3415− pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n",
3416− prog->name, sym_name, sym_sec_name);
3417− return -LIBBPF_ERRNO__RELOC;
3418− }
3419− if (sym->st_value % BPF_INSN_SZ) {
3420− pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n",
3421− prog->name, sym_name, (size_t)sym->st_value);
3422− return -LIBBPF_ERRNO__RELOC;
3423− }
3424− reloc_desc->type = RELO_CALL;
3425− reloc_desc->insn_idx = insn_idx;
3426− reloc_desc->sym_off = sym->st_value;
3427− return 0;
3428− }
3429−
3430− if (!is_ldimm64(insn)) {
3503+ if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) {
34313504 pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n",
34323505 prog->name, sym_name, insn_idx, insn->code);
34333506 return -LIBBPF_ERRNO__RELOC;
@@ -3450,12 +3523,39 @@ static int bpf_program__record_reloc(struct bpf_program *prog, · 16 unmodified lines
34503523 }
34513524 pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n",
34523525 prog->name, i, ext->name, ext->sym_idx, insn_idx);
3453− reloc_desc->type = RELO_EXTERN;
3526+ if (insn->code == (BPF_JMP | BPF_CALL))
3527+ reloc_desc->type = RELO_EXTERN_FUNC;
3528+ else
3529+ reloc_desc->type = RELO_EXTERN_VAR;
34543530 reloc_desc->insn_idx = insn_idx;
34553531 reloc_desc->sym_off = i; /* sym_off stores extern index */
34563532 return 0;
34573533 }
34583534
3535+ /* sub-program call relocation */
3536+ if (is_call_insn(insn)) {
3537+ if (insn->src_reg != BPF_PSEUDO_CALL) {
3538+ pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
3539+ return -LIBBPF_ERRNO__RELOC;
3540+ }
3541+ /* text_shndx can be 0, if no default "main" program exists */
3542+ if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
3543+ sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
3544+ pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n",
3545+ prog->name, sym_name, sym_sec_name);
3546+ return -LIBBPF_ERRNO__RELOC;
3547+ }
3548+ if (sym->st_value % BPF_INSN_SZ) {
3549+ pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n",
3550+ prog->name, sym_name, (size_t)sym->st_value);
3551+ return -LIBBPF_ERRNO__RELOC;
3552+ }
3553+ reloc_desc->type = RELO_CALL;
3554+ reloc_desc->insn_idx = insn_idx;
3555+ reloc_desc->sym_off = sym->st_value;
3556+ return 0;
3557+ }
3558+
34593559 if (!shdr_idx || shdr_idx >= SHN_LORESERVE) {
34603560 pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n",
34613561 prog->name, sym_name, shdr_idx);
@@ -5695,7 +5795,7 @@ poison: · 2233 unmodified lines
56955795 /* poison second part of ldimm64 to avoid confusing error from
56965796 * verifier about "unknown opcode 00"
56975797 */
5698− if (is_ldimm64(insn))
5798+ if (is_ldimm64_insn(insn))
56995799 bpf_core_poison_insn(prog, relo_idx, insn_idx + 1, insn + 1);
57005800 bpf_core_poison_insn(prog, relo_idx, insn_idx, insn);
57015801 return 0;
@@ -5771,7 +5871,7 @@ poison: · 69 unmodified lines
57715871 case BPF_LD: {
57725872 __u64 imm;
57735873
5774− if (!is_ldimm64(insn) ||
5874+ if (!is_ldimm64_insn(insn) ||
57755875 insn[0].src_reg != 0 || insn[0].off != 0 ||
57765876 insn_idx + 1 >= prog->insns_cnt ||
57775877 insn[1].code != 0 || insn[1].dst_reg != 0 ||
@@ -6213,7 +6313,7 @@ bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) · 435 unmodified lines
62136313 insn[0].imm = obj->maps[relo->map_idx].fd;
62146314 relo->processed = true;
62156315 break;
6216− case RELO_EXTERN:
6316+ case RELO_EXTERN_VAR:
62176317 ext = &obj->externs[relo->sym_off];
62186318 if (ext->type == EXT_KCFG) {
62196319 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
@@ -6231,6 +6331,12 @@ bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) · 11 unmodified lines
62316331 }
62326332 relo->processed = true;
62336333 break;
6334+ case RELO_EXTERN_FUNC:
6335+ ext = &obj->externs[relo->sym_off];
6336+ insn[0].src_reg = BPF_PSEUDO_KFUNC_CALL;
6337+ insn[0].imm = ext->ksym.kernel_btf_id;
6338+ relo->processed = true;
6339+ break;
62346340 case RELO_SUBPROG_ADDR:
62356341 insn[0].src_reg = BPF_PSEUDO_FUNC;
62366342 /* will be handled as a follow up pass */
@@ -7351,6 +7457,7 @@ static int bpf_object__read_kallsyms_file(struct bpf_object *obj) · 1114 unmodified lines
73517457 {
73527458 char sym_type, sym_name[500];
73537459 unsigned long long sym_addr;
7460+ const struct btf_type *t;
73547461 struct extern_desc *ext;
73557462 int ret, err = 0;
73567463 FILE *f;
@@ -7377,6 +7484,10 @@ static int bpf_object__read_kallsyms_file(struct bpf_object *obj) · 20 unmodified lines
73777484 if (!ext || ext->type != EXT_KSYM)
73787485 continue;
73797486
7487+ t = btf__type_by_id(obj->btf, ext->btf_id);
7488+ if (!btf_is_var(t))
7489+ continue;
7490+
73807491 if (ext->is_set && ext->ksym.addr != sym_addr) {
73817492 pr_warn("extern (ksym) '%s' resolution is ambiguous: 0x%llx or 0x%llx\n",
73827493 sym_name, ext->ksym.addr, sym_addr);
@@ -7395,75 +7506,151 @@ out: · 12 unmodified lines
73957506 return err;
73967507 }
73977508
7398−static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj)
7509+static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name,
7510+ __u16 kind, struct btf **res_btf,
7511+ int *res_btf_fd)
73997512 {
7400− struct extern_desc *ext;
7513+ int i, id, btf_fd, err;
74017514 struct btf *btf;
7402− int i, j, id, btf_fd, err;
74037515
7404− for (i = 0; i < obj->nr_extern; i++) {
7405− const struct btf_type *targ_var, *targ_type;
7406− __u32 targ_type_id, local_type_id;
7407− const char *targ_var_name;
7408− int ret;
7516+ btf = obj->btf_vmlinux;
7517+ btf_fd = 0;
7518+ id = btf__find_by_name_kind(btf, ksym_name, kind);
74097519
7410− ext = &obj->externs[i];
7411− if (ext->type != EXT_KSYM || !ext->ksym.type_id)
7412− continue;
7413−
7414− btf = obj->btf_vmlinux;
7415− btf_fd = 0;
7416− id = btf__find_by_name_kind(btf, ext->name, BTF_KIND_VAR);
7417− if (id == -ENOENT) {
7418− err = load_module_btfs(obj);
7419− if (err)
7420− return err;
7520+ if (id == -ENOENT) {
7521+ err = load_module_btfs(obj);
7522+ if (err)
7523+ return err;
74217524
7422− for (j = 0; j < obj->btf_module_cnt; j++) {
7423− btf = obj->btf_modules[j].btf;
7424− /* we assume module BTF FD is always >0 */
7425− btf_fd = obj->btf_modules[j].fd;
7426− id = btf__find_by_name_kind(btf, ext->name, BTF_KIND_VAR);
7427− if (id != -ENOENT)
7428− break;
7429− }
7430− }
7431− if (id <= 0) {
7432− pr_warn("extern (ksym) '%s': failed to find BTF ID in kernel BTF(s).\n",
7433− ext->name);
7434− return -ESRCH;
7525+ for (i = 0; i < obj->btf_module_cnt; i++) {
7526+ btf = obj->btf_modules[i].btf;
7527+ /* we assume module BTF FD is always >0 */
7528+ btf_fd = obj->btf_modules[i].fd;
7529+ id = btf__find_by_name_kind(btf, ksym_name, kind);
7530+ if (id != -ENOENT)
7531+ break;
74357532 }
7533+ }
7534+ if (id <= 0) {
7535+ pr_warn("extern (%s ksym) '%s': failed to find BTF ID in kernel BTF(s).\n",
7536+ __btf_kind_str(kind), ksym_name);
7537+ return -ESRCH;
7538+ }
74367539
7437− /* find local type_id */
7438− local_type_id = ext->ksym.type_id;
7540+ *res_btf = btf;
7541+ *res_btf_fd = btf_fd;
7542+ return id;
7543+}
74397544
7440− /* find target type_id */
7441− targ_var = btf__type_by_id(btf, id);
7442− targ_var_name = btf__name_by_offset(btf, targ_var->name_off);
7443− targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id);
7545+static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj,
7546+ struct extern_desc *ext)
7547+{
7548+ const struct btf_type *targ_var, *targ_type;
7549+ __u32 targ_type_id, local_type_id;
7550+ const char *targ_var_name;
7551+ int id, btf_fd = 0, err;
7552+ struct btf *btf = NULL;
74447553
7445− ret = bpf_core_types_are_compat(obj->btf, local_type_id,
7446− btf, targ_type_id);
7447− if (ret <= 0) {
7448− const struct btf_type *local_type;
7449− const char *targ_name, *local_name;
7554+ id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &btf_fd);
7555+ if (id < 0)
7556+ return id;
74507557
7451− local_type = btf__type_by_id(obj->btf, local_type_id);
7452− local_name = btf__name_by_offset(obj->btf, local_type->name_off);
7453− targ_name = btf__name_by_offset(btf, targ_type->name_off);
7558+ /* find local type_id */
7559+ local_type_id = ext->ksym.type_id;
74547560
7455− pr_warn("extern (ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n",
7456− ext->name, local_type_id,
7457− btf_kind_str(local_type), local_name, targ_type_id,
7458− btf_kind_str(targ_type), targ_name);
7459− return -EINVAL;
7460− }
7561+ /* find target type_id */
7562+ targ_var = btf__type_by_id(btf, id);
7563+ targ_var_name = btf__name_by_offset(btf, targ_var->name_off);
7564+ targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id);
7565+
7566+ err = bpf_core_types_are_compat(obj->btf, local_type_id,
7567+ btf, targ_type_id);
7568+ if (err <= 0) {
7569+ const struct btf_type *local_type;
7570+ const char *targ_name, *local_name;
74617571
7462− ext->is_set = true;
7463− ext->ksym.kernel_btf_obj_fd = btf_fd;
7464− ext->ksym.kernel_btf_id = id;
7465− pr_debug("extern (ksym) '%s': resolved to [%d] %s %s\n",
7466− ext->name, id, btf_kind_str(targ_var), targ_var_name);
7572+ local_type = btf__type_by_id(obj->btf, local_type_id);
7573+ local_name = btf__name_by_offset(obj->btf, local_type->name_off);
7574+ targ_name = btf__name_by_offset(btf, targ_type->name_off);
7575+
7576+ pr_warn("extern (var ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n",
7577+ ext->name, local_type_id,
7578+ btf_kind_str(local_type), local_name, targ_type_id,
7579+ btf_kind_str(targ_type), targ_name);
7580+ return -EINVAL;
7581+ }
7582+
7583+ ext->is_set = true;
7584+ ext->ksym.kernel_btf_obj_fd = btf_fd;
7585+ ext->ksym.kernel_btf_id = id;
7586+ pr_debug("extern (var ksym) '%s': resolved to [%d] %s %s\n",
7587+ ext->name, id, btf_kind_str(targ_var), targ_var_name);
7588+
7589+ return 0;
7590+}
7591+
7592+static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj,
7593+ struct extern_desc *ext)
7594+{
7595+ int local_func_proto_id, kfunc_proto_id, kfunc_id;
7596+ const struct btf_type *kern_func;
7597+ struct btf *kern_btf = NULL;
7598+ int ret, kern_btf_fd = 0;
7599+
7600+ local_func_proto_id = ext->ksym.type_id;
7601+
7602+ kfunc_id = find_ksym_btf_id(obj, ext->name, BTF_KIND_FUNC,
7603+ &kern_btf, &kern_btf_fd);
7604+ if (kfunc_id < 0) {
7605+ pr_warn("extern (func ksym) '%s': not found in kernel BTF\n",
7606+ ext->name);
7607+ return kfunc_id;
7608+ }
7609+
7610+ if (kern_btf != obj->btf_vmlinux) {
7611+ pr_warn("extern (func ksym) '%s': function in kernel module is not supported\n",
7612+ ext->name);
7613+ return -ENOTSUP;
7614+ }
7615+
7616+ kern_func = btf__type_by_id(kern_btf, kfunc_id);
7617+ kfunc_proto_id = kern_func->type;
7618+
7619+ ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id,
7620+ kern_btf, kfunc_proto_id);
7621+ if (ret <= 0) {
7622+ pr_warn("extern (func ksym) '%s': func_proto [%d] incompatible with kernel [%d]\n",
7623+ ext->name, local_func_proto_id, kfunc_proto_id);
7624+ return -EINVAL;
7625+ }
7626+
7627+ ext->is_set = true;
7628+ ext->ksym.kernel_btf_obj_fd = kern_btf_fd;
7629+ ext->ksym.kernel_btf_id = kfunc_id;
7630+ pr_debug("extern (func ksym) '%s': resolved to kernel [%d]\n",
7631+ ext->name, kfunc_id);
7632+
7633+ return 0;
7634+}
7635+
7636+static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj)
7637+{
7638+ const struct btf_type *t;
7639+ struct extern_desc *ext;
7640+ int i, err;
7641+
7642+ for (i = 0; i < obj->nr_extern; i++) {
7643+ ext = &obj->externs[i];
7644+ if (ext->type != EXT_KSYM || !ext->ksym.type_id)
7645+ continue;
7646+
7647+ t = btf__type_by_id(obj->btf, ext->btf_id);
7648+ if (btf_is_var(t))
7649+ err = bpf_object__resolve_ksym_var_btf_id(obj, ext);
7650+ else
7651+ err = bpf_object__resolve_ksym_func_btf_id(obj, ext);
7652+ if (err)
7653+ return err;
74677654 }
74687655 return 0;
74697656 }
@@ -8270,6 +8457,16 @@ int bpf_object__btf_fd(const struct bpf_object *obj) · 800 unmodified lines
82708457 return obj->btf ? btf__fd(obj->btf) : -1;
82718458 }
82728459
8460+int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version)
8461+{
8462+ if (obj->loaded)
8463+ return -EINVAL;
8464+
8465+ obj->kern_version = kern_version;
8466+
8467+ return 0;
8468+}
8469+
82738470 int bpf_object__set_priv(struct bpf_object *obj, void *priv,
82748471 bpf_object_clear_priv_t clear_priv)
82758472 {
@@ -8458,7 +8655,7 @@ int bpf_program__nth_fd(const struct bpf_program *prog, int n) · 182 unmodified lines
84588655 return fd;
84598656 }
84608657
8461−enum bpf_prog_type bpf_program__get_type(struct bpf_program *prog)
8658+enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog)
84628659 {
84638660 return prog->type;
84648661 }
@@ -8503,7 +8700,7 @@ BPF_PROG_TYPE_FNS(extension, BPF_PROG_TYPE_EXT); · 38 unmodified lines
85038700 BPF_PROG_TYPE_FNS(sk_lookup, BPF_PROG_TYPE_SK_LOOKUP);
85048701
85058702 enum bpf_attach_type
8506−bpf_program__get_expected_attach_type(struct bpf_program *prog)
8703+bpf_program__get_expected_attach_type(const struct bpf_program *prog)
85078704 {
85088705 return prog->expected_attach_type;
85098706 }
tools/lib/bpf/libbpf.h+3−2
@@ -143,6 +143,7 @@ LIBBPF_API int bpf_object__unload(struct bpf_object *obj); · 142 unmodified lines
143143
144144 LIBBPF_API const char *bpf_object__name(const struct bpf_object *obj);
145145 LIBBPF_API unsigned int bpf_object__kversion(const struct bpf_object *obj);
146+LIBBPF_API int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version);
146147
147148 struct btf;
148149 LIBBPF_API struct btf *bpf_object__btf(const struct bpf_object *obj);
@@ -361,12 +362,12 @@ LIBBPF_API int bpf_program__set_struct_ops(struct bpf_program *prog); · 212 unmodified lines
361362 LIBBPF_API int bpf_program__set_extension(struct bpf_program *prog);
362363 LIBBPF_API int bpf_program__set_sk_lookup(struct bpf_program *prog);
363364
364−LIBBPF_API enum bpf_prog_type bpf_program__get_type(struct bpf_program *prog);
365+LIBBPF_API enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog);
365366 LIBBPF_API void bpf_program__set_type(struct bpf_program *prog,
366367 enum bpf_prog_type type);
367368
368369 LIBBPF_API enum bpf_attach_type
369−bpf_program__get_expected_attach_type(struct bpf_program *prog);
370+bpf_program__get_expected_attach_type(const struct bpf_program *prog);
370371 LIBBPF_API void
371372 bpf_program__set_expected_attach_type(struct bpf_program *prog,
372373 enum bpf_attach_type type);
tools/lib/bpf/libbpf.map+1−0
@@ -359,4 +359,5 @@ LIBBPF_0.4.0 { · 358 unmodified lines
359359 bpf_linker__finalize;
360360 bpf_linker__free;
361361 bpf_linker__new;
362+ bpf_object__set_kversion;
362363 } LIBBPF_0.3.0;
tools/lib/bpf/linker.c+28−9
@@ -94,6 +94,7 @@ struct dst_sec { · 93 unmodified lines
9494 int sec_sym_idx;
9595
9696 /* section's DATASEC variable info, emitted on BTF finalization */
97+ bool has_btf;
9798 int sec_var_cnt;
9899 struct btf_var_secinfo *sec_vars;
99100
@@ -1436,6 +1437,16 @@ static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj) · 1336 unmodified lines
14361437 continue;
14371438 dst_sec = &linker->secs[src_sec->dst_id];
14381439
1440+ /* Mark section as having BTF regardless of the presence of
1441+ * variables. In some cases compiler might generate empty BTF
1442+ * with no variables information. E.g., when promoting local
1443+ * array/structure variable initial values and BPF object
1444+ * file otherwise has no read-only static variables in
1445+ * .rodata. We need to preserve such empty BTF and just set
1446+ * correct section size.
1447+ */
1448+ dst_sec->has_btf = true;
1449+
14391450 t = btf__type_by_id(obj->btf, src_sec->sec_type_id);
14401451 src_var = btf_var_secinfos(t);
14411452 n = btf_vlen(t);
@@ -1717,7 +1728,7 @@ static int finalize_btf(struct bpf_linker *linker) · 275 unmodified lines
17171728 for (i = 1; i < linker->sec_cnt; i++) {
17181729 struct dst_sec *sec = &linker->secs[i];
17191730
1720− if (!sec->sec_var_cnt)
1731+ if (!sec->has_btf)
17211732 continue;
17221733
17231734 id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz);
@@ -1895,8 +1906,10 @@ static int finalize_btf_ext(struct bpf_linker *linker) · 171 unmodified lines
18951906 struct dst_sec *sec = &linker->secs[i];
18961907
18971908 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info);
1898− if (sz < 0)
1899− return sz;
1909+ if (sz < 0) {
1910+ err = sz;
1911+ goto out;
1912+ }
19001913
19011914 cur += sz;
19021915 }
@@ -1910,8 +1923,10 @@ static int finalize_btf_ext(struct bpf_linker *linker) · 7 unmodified lines
19101923 struct dst_sec *sec = &linker->secs[i];
19111924
19121925 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info);
1913− if (sz < 0)
1914− return sz;
1926+ if (sz < 0) {
1927+ err = sz;
1928+ goto out;
1929+ }
19151930
19161931 cur += sz;
19171932 }
@@ -1925,8 +1940,10 @@ static int finalize_btf_ext(struct bpf_linker *linker) · 7 unmodified lines
19251940 struct dst_sec *sec = &linker->secs[i];
19261941
19271942 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info);
1928− if (sz < 0)
1929− return sz;
1943+ if (sz < 0) {
1944+ err = sz;
1945+ goto out;
1946+ }
19301947
19311948 cur += sz;
19321949 }
@@ -1937,8 +1954,10 @@ static int finalize_btf_ext(struct bpf_linker *linker) · 4 unmodified lines
19371954 if (err) {
19381955 linker->btf_ext = NULL;
19391956 pr_warn("failed to parse final .BTF.ext data: %d\n", err);
1940− return err;
1957+ goto out;
19411958 }
19421959
1943− return 0;
1960+out:
1961+ free(data);
1962+ return err;
19441963 }
tools/lib/bpf/xsk.c+213−45
@@ -28,6 +28,7 @@ · 27 unmodified lines
2828 #include <sys/mman.h>
2929 #include <sys/socket.h>
3030 #include <sys/types.h>
31+#include <linux/if_link.h>
3132
3233 #include "bpf.h"
3334 #include "libbpf.h"
@@ -70,8 +71,10 @@ struct xsk_ctx { · 36 unmodified lines
7071 int ifindex;
7172 struct list_head list;
7273 int prog_fd;
74+ int link_fd;
7375 int xsks_map_fd;
7476 char ifname[IFNAMSIZ];
77+ bool has_bpf_link;
7578 };
7679
7780 struct xsk_socket {
@@ -409,7 +412,7 @@ static int xsk_load_xdp_prog(struct xsk_socket *xsk) · 331 unmodified lines
409412 static const int log_buf_size = 16 * 1024;
410413 struct xsk_ctx *ctx = xsk->ctx;
411414 char log_buf[log_buf_size];
412− int err, prog_fd;
415+ int prog_fd;
413416
414417 /* This is the fallback C-program:
415418 * SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)
@@ -499,14 +502,41 @@ static int xsk_load_xdp_prog(struct xsk_socket *xsk) · 83 unmodified lines
499502 return prog_fd;
500503 }
501504
502− err = bpf_set_link_xdp_fd(xsk->ctx->ifindex, prog_fd,
503− xsk->config.xdp_flags);
505+ ctx->prog_fd = prog_fd;
506+ return 0;
507+}
508+
509+static int xsk_create_bpf_link(struct xsk_socket *xsk)
510+{
511+ DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts);
512+ struct xsk_ctx *ctx = xsk->ctx;
513+ __u32 prog_id = 0;
514+ int link_fd;
515+ int err;
516+
517+ err = bpf_get_link_xdp_id(ctx->ifindex, &prog_id, xsk->config.xdp_flags);
504518 if (err) {
505− close(prog_fd);
519+ pr_warn("getting XDP prog id failed\n");
506520 return err;
507521 }
508522
509− ctx->prog_fd = prog_fd;
523+ /* if there's a netlink-based XDP prog loaded on interface, bail out
524+ * and ask user to do the removal by himself
525+ */
526+ if (prog_id) {
527+ pr_warn("Netlink-based XDP prog detected, please unload it in order to launch AF_XDP prog\n");
528+ return -EINVAL;
529+ }
530+
531+ opts.flags = xsk->config.xdp_flags & ~(XDP_FLAGS_UPDATE_IF_NOEXIST | XDP_FLAGS_REPLACE);
532+
533+ link_fd = bpf_link_create(ctx->prog_fd, ctx->ifindex, BPF_XDP, &opts);
534+ if (link_fd < 0) {
535+ pr_warn("bpf_link_create failed: %s\n", strerror(errno));
536+ return link_fd;
537+ }
538+
539+ ctx->link_fd = link_fd;
510540 return 0;
511541 }
512542
@@ -625,7 +655,6 @@ static int xsk_lookup_bpf_maps(struct xsk_socket *xsk) · 112 unmodified lines
625655 close(fd);
626656 }
627657
628− err = 0;
629658 if (ctx->xsks_map_fd == -1)
630659 err = -ENOENT;
631660
@@ -642,6 +671,98 @@ static int xsk_set_bpf_maps(struct xsk_socket *xsk) · 10 unmodified lines
642671 &xsk->fd, 0);
643672 }
644673
674+static int xsk_link_lookup(int ifindex, __u32 *prog_id, int *link_fd)
675+{
676+ struct bpf_link_info link_info;
677+ __u32 link_len;
678+ __u32 id = 0;
679+ int err;
680+ int fd;
681+
682+ while (true) {
683+ err = bpf_link_get_next_id(id, &id);
684+ if (err) {
685+ if (errno == ENOENT) {
686+ err = 0;
687+ break;
688+ }
689+ pr_warn("can't get next link: %s\n", strerror(errno));
690+ break;
691+ }
692+
693+ fd = bpf_link_get_fd_by_id(id);
694+ if (fd < 0) {
695+ if (errno == ENOENT)
696+ continue;
697+ pr_warn("can't get link by id (%u): %s\n", id, strerror(errno));
698+ err = -errno;
699+ break;
700+ }
701+
702+ link_len = sizeof(struct bpf_link_info);
703+ memset(&link_info, 0, link_len);
704+ err = bpf_obj_get_info_by_fd(fd, &link_info, &link_len);
705+ if (err) {
706+ pr_warn("can't get link info: %s\n", strerror(errno));
707+ close(fd);
708+ break;
709+ }
710+ if (link_info.type == BPF_LINK_TYPE_XDP) {
711+ if (link_info.xdp.ifindex == ifindex) {
712+ *link_fd = fd;
713+ if (prog_id)
714+ *prog_id = link_info.prog_id;
715+ break;
716+ }
717+ }
718+ close(fd);
719+ }
720+
721+ return err;
722+}
723+
724+static bool xsk_probe_bpf_link(void)
725+{
726+ DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts,
727+ .flags = XDP_FLAGS_SKB_MODE);
728+ struct bpf_load_program_attr prog_attr;
729+ struct bpf_insn insns[2] = {
730+ BPF_MOV64_IMM(BPF_REG_0, XDP_PASS),
731+ BPF_EXIT_INSN()
732+ };
733+ int prog_fd, link_fd = -1;
734+ int ifindex_lo = 1;
735+ bool ret = false;
736+ int err;
737+
738+ err = xsk_link_lookup(ifindex_lo, NULL, &link_fd);
739+ if (err)
740+ return ret;
741+
742+ if (link_fd >= 0)
743+ return true;
744+
745+ memset(&prog_attr, 0, sizeof(prog_attr));
746+ prog_attr.prog_type = BPF_PROG_TYPE_XDP;
747+ prog_attr.insns = insns;
748+ prog_attr.insns_cnt = ARRAY_SIZE(insns);
749+ prog_attr.license = "GPL";
750+
751+ prog_fd = bpf_load_program_xattr(&prog_attr, NULL, 0);
752+ if (prog_fd < 0)
753+ return ret;
754+
755+ link_fd = bpf_link_create(prog_fd, ifindex_lo, BPF_XDP, &opts);
756+ close(prog_fd);
757+
758+ if (link_fd >= 0) {
759+ ret = true;
760+ close(link_fd);
761+ }
762+
763+ return ret;
764+}
765+
645766 static int xsk_create_xsk_struct(int ifindex, struct xsk_socket *xsk)
646767 {
647768 char ifname[IFNAMSIZ];
@@ -663,64 +784,108 @@ static int xsk_create_xsk_struct(int ifindex, struct xsk_socket *xsk) · 15 unmodified lines
663784 ctx->ifname[IFNAMSIZ - 1] = 0;
664785
665786 xsk->ctx = ctx;
787+ xsk->ctx->has_bpf_link = xsk_probe_bpf_link();
666788
667789 return 0;
668790 }
669791
670−static int __xsk_setup_xdp_prog(struct xsk_socket *_xdp,
671− int *xsks_map_fd)
792+static int xsk_init_xdp_res(struct xsk_socket *xsk,
793+ int *xsks_map_fd)
672794 {
673− struct xsk_socket *xsk = _xdp;
674795 struct xsk_ctx *ctx = xsk->ctx;
675− __u32 prog_id = 0;
676796 int err;
677797
678− err = bpf_get_link_xdp_id(ctx->ifindex, &prog_id,
679− xsk->config.xdp_flags);
798+ err = xsk_create_bpf_maps(xsk);
680799 if (err)
681800 return err;
682801
683− if (!prog_id) {
684− err = xsk_create_bpf_maps(xsk);
685− if (err)
686− return err;
802+ err = xsk_load_xdp_prog(xsk);
803+ if (err)
804+ goto err_load_xdp_prog;
687805
688− err = xsk_load_xdp_prog(xsk);
689− if (err) {
690− goto err_load_xdp_prog;
691− }
692− } else {
693− ctx->prog_fd = bpf_prog_get_fd_by_id(prog_id);
694− if (ctx->prog_fd < 0)
695− return -errno;
696− err = xsk_lookup_bpf_maps(xsk);
697− if (err) {
698− close(ctx->prog_fd);
699− return err;
700− }
701− }
806+ if (ctx->has_bpf_link)
807+ err = xsk_create_bpf_link(xsk);
808+ else
809+ err = bpf_set_link_xdp_fd(xsk->ctx->ifindex, ctx->prog_fd,
810+ xsk->config.xdp_flags);
702811
703− if (xsk->rx) {
704− err = xsk_set_bpf_maps(xsk);
705− if (err) {
706− if (!prog_id) {
707− goto err_set_bpf_maps;
708− } else {
709− close(ctx->prog_fd);
710− return err;
711− }
712− }
713− }
714− if (xsks_map_fd)
715− *xsks_map_fd = ctx->xsks_map_fd;
812+ if (err)
813+ goto err_attach_xdp_prog;
716814
717− return 0;
815+ if (!xsk->rx)
816+ return err;
817+
818+ err = xsk_set_bpf_maps(xsk);
819+ if (err)
820+ goto err_set_bpf_maps;
821+
822+ return err;
718823
719824 err_set_bpf_maps:
825+ if (ctx->has_bpf_link)
826+ close(ctx->link_fd);
827+ else
828+ bpf_set_link_xdp_fd(ctx->ifindex, -1, 0);
829+err_attach_xdp_prog:
720830 close(ctx->prog_fd);
721− bpf_set_link_xdp_fd(ctx->ifindex, -1, 0);
722831 err_load_xdp_prog:
723832 xsk_delete_bpf_maps(xsk);
833+ return err;
834+}
835+
836+static int xsk_lookup_xdp_res(struct xsk_socket *xsk, int *xsks_map_fd, int prog_id)
837+{
838+ struct xsk_ctx *ctx = xsk->ctx;
839+ int err;
840+
841+ ctx->prog_fd = bpf_prog_get_fd_by_id(prog_id);
842+ if (ctx->prog_fd < 0) {
843+ err = -errno;
844+ goto err_prog_fd;
845+ }
846+ err = xsk_lookup_bpf_maps(xsk);
847+ if (err)
848+ goto err_lookup_maps;
849+
850+ if (!xsk->rx)
851+ return err;
852+
853+ err = xsk_set_bpf_maps(xsk);
854+ if (err)
855+ goto err_set_maps;
856+
857+ return err;
858+
859+err_set_maps:
860+ close(ctx->xsks_map_fd);
861+err_lookup_maps:
862+ close(ctx->prog_fd);
863+err_prog_fd:
864+ if (ctx->has_bpf_link)
865+ close(ctx->link_fd);
866+ return err;
867+}
868+
869+static int __xsk_setup_xdp_prog(struct xsk_socket *_xdp, int *xsks_map_fd)
870+{
871+ struct xsk_socket *xsk = _xdp;
872+ struct xsk_ctx *ctx = xsk->ctx;
873+ __u32 prog_id = 0;
874+ int err;
875+
876+ if (ctx->has_bpf_link)
877+ err = xsk_link_lookup(ctx->ifindex, &prog_id, &ctx->link_fd);
878+ else
879+ err = bpf_get_link_xdp_id(ctx->ifindex, &prog_id, xsk->config.xdp_flags);
880+
881+ if (err)
882+ return err;
883+
884+ err = !prog_id ? xsk_init_xdp_res(xsk, xsks_map_fd) :
885+ xsk_lookup_xdp_res(xsk, xsks_map_fd, prog_id);
886+
887+ if (!err && xsks_map_fd)
888+ *xsks_map_fd = ctx->xsks_map_fd;
724889
725890 return err;
726891 }
@@ -898,6 +1063,7 @@ int xsk_socket__create_shared(struct xsk_socket **xsk_ptr, · 171 unmodified lines
8981063 }
8991064 }
9001065 xsk->ctx = ctx;
1066+ xsk->ctx->has_bpf_link = xsk_probe_bpf_link();
9011067
9021068 if (rx) {
9031069 err = setsockopt(xsk->fd, SOL_XDP, XDP_RX_RING,
@@ -1054,6 +1220,8 @@ void xsk_socket__delete(struct xsk_socket *xsk) · 150 unmodified lines
10541220 if (ctx->prog_fd != -1) {
10551221 xsk_delete_bpf_maps(xsk);
10561222 close(ctx->prog_fd);
1223+ if (ctx->has_bpf_link)
1224+ close(ctx->link_fd);
10571225 }
10581226
10591227 err = xsk_get_mmap_offsets(xsk->fd, &off);
tools/testing/selftests/bpf/README.rst+14−0
@@ -179,3 +179,17 @@ types, which was introduced in `Clang 13`__. The older Clang versions will · 178 unmodified lines
179179 either crash when compiling these tests, or generate an incorrect BTF.
180180
181181 __ https://reviews.llvm.org/D83289
182+
183+Kernel function call test and Clang version
184+===========================================
185+
186+Some selftests (e.g. kfunc_call and bpf_tcp_ca) require a LLVM support
187+to generate extern function in BTF. It was introduced in `Clang 13`__.
188+
189+Without it, the error from compiling bpf selftests looks like:
190+
191+.. code-block:: console
192+
193+ libbpf: failed to find BTF for extern 'tcp_slow_start' [25] section: -2
194+
195+__ https://reviews.llvm.org/D93563
tools/testing/selftests/bpf/bpf_tcp_helpers.h+2−27
@@ -187,16 +187,6 @@ struct tcp_congestion_ops { · 186 unmodified lines
187187 typeof(y) __y = (y); \
188188 __x == 0 ? __y : ((__y == 0) ? __x : min(__x, __y)); })
189189
190−static __always_inline __u32 tcp_slow_start(struct tcp_sock *tp, __u32 acked)
191−{
192− __u32 cwnd = min(tp->snd_cwnd + acked, tp->snd_ssthresh);
193−
194− acked -= cwnd - tp->snd_cwnd;
195− tp->snd_cwnd = min(cwnd, tp->snd_cwnd_clamp);
196−
197− return acked;
198−}
199−
200190 static __always_inline bool tcp_in_slow_start(const struct tcp_sock *tp)
201191 {
202192 return tp->snd_cwnd < tp->snd_ssthresh;
@@ -213,22 +203,7 @@ static __always_inline bool tcp_is_cwnd_limited(const struct sock *sk) · 10 unmodified lines
213203 return !!BPF_CORE_READ_BITFIELD(tp, is_cwnd_limited);
214204 }
215205
216−static __always_inline void tcp_cong_avoid_ai(struct tcp_sock *tp, __u32 w, __u32 acked)
217−{
218− /* If credits accumulated at a higher w, apply them gently now. */
219− if (tp->snd_cwnd_cnt >= w) {
220− tp->snd_cwnd_cnt = 0;
221− tp->snd_cwnd++;
222− }
223−
224− tp->snd_cwnd_cnt += acked;
225− if (tp->snd_cwnd_cnt >= w) {
226− __u32 delta = tp->snd_cwnd_cnt / w;
227−
228− tp->snd_cwnd_cnt -= delta * w;
229− tp->snd_cwnd += delta;
230− }
231− tp->snd_cwnd = min(tp->snd_cwnd, tp->snd_cwnd_clamp);
232−}
206+extern __u32 tcp_slow_start(struct tcp_sock *tp, __u32 acked) __ksym;
207+extern void tcp_cong_avoid_ai(struct tcp_sock *tp, __u32 w, __u32 acked) __ksym;
233208
234209 #endif
tools/testing/selftests/bpf/map_tests/lpm_trie_map_batch_ops.c+158−0
@@ -0,0 +1,158 @@
1+// SPDX-License-Identifier: GPL-2.0
2+
3+#include <arpa/inet.h>
4+#include <linux/bpf.h>
5+#include <netinet/in.h>
6+#include <stdio.h>
7+#include <errno.h>
8+#include <string.h>
9+#include <stdlib.h>
10+
11+#include <bpf/bpf.h>
12+#include <bpf/libbpf.h>
13+
14+#include <test_maps.h>
15+
16+struct test_lpm_key {
17+ __u32 prefix;
18+ struct in_addr ipv4;
19+};
20+
21+static void map_batch_update(int map_fd, __u32 max_entries,
22+ struct test_lpm_key *keys, int *values)
23+{
24+ __u32 i;
25+ int err;
26+ char buff[16] = { 0 };
27+ DECLARE_LIBBPF_OPTS(bpf_map_batch_opts, opts,
28+ .elem_flags = 0,
29+ .flags = 0,
30+ );
31+
32+ for (i = 0; i < max_entries; i++) {
33+ keys[i].prefix = 32;
34+ snprintf(buff, 16, "192.168.1.%d", i + 1);
35+ inet_pton(AF_INET, buff, &keys[i].ipv4);
36+ values[i] = i + 1;
37+ }
38+
39+ err = bpf_map_update_batch(map_fd, keys, values, &max_entries, &opts);
40+ CHECK(err, "bpf_map_update_batch()", "error:%s\n", strerror(errno));
41+}
42+
43+static void map_batch_verify(int *visited, __u32 max_entries,
44+ struct test_lpm_key *keys, int *values)
45+{
46+ char buff[16] = { 0 };
47+ int lower_byte = 0;
48+ __u32 i;
49+
50+ memset(visited, 0, max_entries * sizeof(*visited));
51+ for (i = 0; i < max_entries; i++) {
52+ inet_ntop(AF_INET, &keys[i].ipv4, buff, 32);
53+ CHECK(sscanf(buff, "192.168.1.%d", &lower_byte) == EOF,
54+ "sscanf()", "error: i %d\n", i);
55+ CHECK(lower_byte != values[i], "key/value checking",
56+ "error: i %d key %s value %d\n", i, buff, values[i]);
57+ visited[i] = 1;
58+ }
59+ for (i = 0; i < max_entries; i++) {
60+ CHECK(visited[i] != 1, "visited checking",
61+ "error: keys array at index %d missing\n", i);
62+ }
63+}
64+
65+void test_lpm_trie_map_batch_ops(void)
66+{
67+ struct bpf_create_map_attr xattr = {
68+ .name = "lpm_trie_map",
69+ .map_type = BPF_MAP_TYPE_LPM_TRIE,
70+ .key_size = sizeof(struct test_lpm_key),
71+ .value_size = sizeof(int),
72+ .map_flags = BPF_F_NO_PREALLOC,
73+ };
74+ struct test_lpm_key *keys, key;
75+ int map_fd, *values, *visited;
76+ __u32 step, count, total, total_success;
77+ const __u32 max_entries = 10;
78+ __u64 batch = 0;
79+ int err;
80+ DECLARE_LIBBPF_OPTS(bpf_map_batch_opts, opts,
81+ .elem_flags = 0,
82+ .flags = 0,
83+ );
84+
85+ xattr.max_entries = max_entries;
86+ map_fd = bpf_create_map_xattr(&xattr);
87+ CHECK(map_fd == -1, "bpf_create_map_xattr()", "error:%s\n",
88+ strerror(errno));
89+
90+ keys = malloc(max_entries * sizeof(struct test_lpm_key));
91+ values = malloc(max_entries * sizeof(int));
92+ visited = malloc(max_entries * sizeof(int));
93+ CHECK(!keys || !values || !visited, "malloc()", "error:%s\n",
94+ strerror(errno));
95+
96+ total_success = 0;
97+ for (step = 1; step < max_entries; step++) {
98+ map_batch_update(map_fd, max_entries, keys, values);
99+ map_batch_verify(visited, max_entries, keys, values);
100+ memset(keys, 0, max_entries * sizeof(*keys));
101+ memset(values, 0, max_entries * sizeof(*values));
102+ batch = 0;
103+ total = 0;
104+ /* iteratively lookup/delete elements with 'step'
105+ * elements each.
106+ */
107+ count = step;
108+ while (true) {
109+ err = bpf_map_lookup_batch(map_fd,
110+ total ? &batch : NULL, &batch,
111+ keys + total, values + total, &count, &opts);
112+
113+ CHECK((err && errno != ENOENT), "lookup with steps",
114+ "error: %s\n", strerror(errno));
115+
116+ total += count;
117+ if (err)
118+ break;
119+ }
120+
121+ CHECK(total != max_entries, "lookup with steps",
122+ "total = %u, max_entries = %u\n", total, max_entries);
123+
124+ map_batch_verify(visited, max_entries, keys, values);
125+
126+ total = 0;
127+ count = step;
128+ while (total < max_entries) {
129+ if (max_entries - total < step)
130+ count = max_entries - total;
131+ err = bpf_map_delete_batch(map_fd, keys + total, &count,
132+ &opts);
133+ CHECK((err && errno != ENOENT), "delete batch",
134+ "error: %s\n", strerror(errno));
135+ total += count;
136+ if (err)
137+ break;
138+ }
139+ CHECK(total != max_entries, "delete with steps",
140+ "total = %u, max_entries = %u\n", total, max_entries);
141+
142+ /* check map is empty, errono == ENOENT */
143+ err = bpf_map_get_next_key(map_fd, NULL, &key);
144+ CHECK(!err || errno != ENOENT, "bpf_map_get_next_key()",
145+ "error: %s\n", strerror(errno));
146+
147+ total_success++;
148+ }
149+
150+ CHECK(total_success == 0, "check total_success",
151+ "unexpected failure\n");
152+
153+ printf("%s:PASS\n", __func__);
154+
155+ free(keys);
156+ free(values);
157+ free(visited);
158+}
tools/testing/selftests/bpf/prog_tests/kfunc_call.c+59−0
@@ -0,0 +1,59 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2021 Facebook */
3+#include <test_progs.h>
4+#include <network_helpers.h>
5+#include "kfunc_call_test.skel.h"
6+#include "kfunc_call_test_subprog.skel.h"
7+
8+static void test_main(void)
9+{
10+ struct kfunc_call_test *skel;
11+ int prog_fd, retval, err;
12+
13+ skel = kfunc_call_test__open_and_load();
14+ if (!ASSERT_OK_PTR(skel, "skel"))
15+ return;
16+
17+ prog_fd = bpf_program__fd(skel->progs.kfunc_call_test1);
18+ err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
19+ NULL, NULL, (__u32 *)&retval, NULL);
20+ ASSERT_OK(err, "bpf_prog_test_run(test1)");
21+ ASSERT_EQ(retval, 12, "test1-retval");
22+
23+ prog_fd = bpf_program__fd(skel->progs.kfunc_call_test2);
24+ err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
25+ NULL, NULL, (__u32 *)&retval, NULL);
26+ ASSERT_OK(err, "bpf_prog_test_run(test2)");
27+ ASSERT_EQ(retval, 3, "test2-retval");
28+
29+ kfunc_call_test__destroy(skel);
30+}
31+
32+static void test_subprog(void)
33+{
34+ struct kfunc_call_test_subprog *skel;
35+ int prog_fd, retval, err;
36+
37+ skel = kfunc_call_test_subprog__open_and_load();
38+ if (!ASSERT_OK_PTR(skel, "skel"))
39+ return;
40+
41+ prog_fd = bpf_program__fd(skel->progs.kfunc_call_test1);
42+ err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
43+ NULL, NULL, (__u32 *)&retval, NULL);
44+ ASSERT_OK(err, "bpf_prog_test_run(test1)");
45+ ASSERT_EQ(retval, 10, "test1-retval");
46+ ASSERT_NEQ(skel->data->active_res, -1, "active_res");
47+ ASSERT_EQ(skel->data->sk_state, BPF_TCP_CLOSE, "sk_state");
48+
49+ kfunc_call_test_subprog__destroy(skel);
50+}
51+
52+void test_kfunc_call(void)
53+{
54+ if (test__start_subtest("main"))
55+ test_main();
56+
57+ if (test__start_subtest("subprog"))
58+ test_subprog();
59+}
tools/testing/selftests/bpf/prog_tests/sockmap_basic.c+40−0
@@ -7,6 +7,7 @@ · 6 unmodified lines
77 #include "test_skmsg_load_helpers.skel.h"
88 #include "test_sockmap_update.skel.h"
99 #include "test_sockmap_invalid_update.skel.h"
10+#include "test_sockmap_skb_verdict_attach.skel.h"
1011 #include "bpf_iter_sockmap.skel.h"
1112
1213 #define TCP_REPAIR 19 /* TCP sock is under repair right now */
@@ -281,6 +282,39 @@ out: · 268 unmodified lines
281282 bpf_iter_sockmap__destroy(skel);
282283 }
283284
285+static void test_sockmap_skb_verdict_attach(enum bpf_attach_type first,
286+ enum bpf_attach_type second)
287+{
288+ struct test_sockmap_skb_verdict_attach *skel;
289+ int err, map, verdict;
290+
291+ skel = test_sockmap_skb_verdict_attach__open_and_load();
292+ if (CHECK_FAIL(!skel)) {
293+ perror("test_sockmap_skb_verdict_attach__open_and_load");
294+ return;
295+ }
296+
297+ verdict = bpf_program__fd(skel->progs.prog_skb_verdict);
298+ map = bpf_map__fd(skel->maps.sock_map);
299+
300+ err = bpf_prog_attach(verdict, map, first, 0);
301+ if (CHECK_FAIL(err)) {
302+ perror("bpf_prog_attach");
303+ goto out;
304+ }
305+
306+ err = bpf_prog_attach(verdict, map, second, 0);
307+ assert(err == -1 && errno == EBUSY);
308+
309+ err = bpf_prog_detach2(verdict, map, first);
310+ if (CHECK_FAIL(err)) {
311+ perror("bpf_prog_detach2");
312+ goto out;
313+ }
314+out:
315+ test_sockmap_skb_verdict_attach__destroy(skel);
316+}
317+
284318 void test_sockmap_basic(void)
285319 {
286320 if (test__start_subtest("sockmap create_update_free"))
@@ -301,4 +335,10 @@ void test_sockmap_basic(void) · 14 unmodified lines
301335 test_sockmap_copy(BPF_MAP_TYPE_SOCKMAP);
302336 if (test__start_subtest("sockhash copy"))
303337 test_sockmap_copy(BPF_MAP_TYPE_SOCKHASH);
338+ if (test__start_subtest("sockmap skb_verdict attach")) {
339+ test_sockmap_skb_verdict_attach(BPF_SK_SKB_VERDICT,
340+ BPF_SK_SKB_STREAM_VERDICT);
341+ test_sockmap_skb_verdict_attach(BPF_SK_SKB_STREAM_VERDICT,
342+ BPF_SK_SKB_VERDICT);
343+ }
304344 }
tools/testing/selftests/bpf/prog_tests/sockmap_listen.c+136−0
@@ -1603,6 +1603,141 @@ static void test_reuseport(struct test_sockmap_listen *skel, · 1602 unmodified lines
16031603 }
16041604 }
16051605
1606+static void udp_redir_to_connected(int family, int sotype, int sock_mapfd,
1607+ int verd_mapfd, enum redir_mode mode)
1608+{
1609+ const char *log_prefix = redir_mode_str(mode);
1610+ struct sockaddr_storage addr;
1611+ int c0, c1, p0, p1;
1612+ unsigned int pass;
1613+ socklen_t len;
1614+ int err, n;
1615+ u64 value;
1616+ u32 key;
1617+ char b;
1618+
1619+ zero_verdict_count(verd_mapfd);
1620+
1621+ p0 = socket_loopback(family, sotype | SOCK_NONBLOCK);
1622+ if (p0 < 0)
1623+ return;
1624+ len = sizeof(addr);
1625+ err = xgetsockname(p0, sockaddr(&addr), &len);
1626+ if (err)
1627+ goto close_peer0;
1628+
1629+ c0 = xsocket(family, sotype | SOCK_NONBLOCK, 0);
1630+ if (c0 < 0)
1631+ goto close_peer0;
1632+ err = xconnect(c0, sockaddr(&addr), len);
1633+ if (err)
1634+ goto close_cli0;
1635+ err = xgetsockname(c0, sockaddr(&addr), &len);
1636+ if (err)
1637+ goto close_cli0;
1638+ err = xconnect(p0, sockaddr(&addr), len);
1639+ if (err)
1640+ goto close_cli0;
1641+
1642+ p1 = socket_loopback(family, sotype | SOCK_NONBLOCK);
1643+ if (p1 < 0)
1644+ goto close_cli0;
1645+ err = xgetsockname(p1, sockaddr(&addr), &len);
1646+ if (err)
1647+ goto close_cli0;
1648+
1649+ c1 = xsocket(family, sotype | SOCK_NONBLOCK, 0);
1650+ if (c1 < 0)
1651+ goto close_peer1;
1652+ err = xconnect(c1, sockaddr(&addr), len);
1653+ if (err)
1654+ goto close_cli1;
1655+ err = xgetsockname(c1, sockaddr(&addr), &len);
1656+ if (err)
1657+ goto close_cli1;
1658+ err = xconnect(p1, sockaddr(&addr), len);
1659+ if (err)
1660+ goto close_cli1;
1661+
1662+ key = 0;
1663+ value = p0;
1664+ err = xbpf_map_update_elem(sock_mapfd, &key, &value, BPF_NOEXIST);
1665+ if (err)
1666+ goto close_cli1;
1667+
1668+ key = 1;
1669+ value = p1;
1670+ err = xbpf_map_update_elem(sock_mapfd, &key, &value, BPF_NOEXIST);
1671+ if (err)
1672+ goto close_cli1;
1673+
1674+ n = write(c1, "a", 1);
1675+ if (n < 0)
1676+ FAIL_ERRNO("%s: write", log_prefix);
1677+ if (n == 0)
1678+ FAIL("%s: incomplete write", log_prefix);
1679+ if (n < 1)
1680+ goto close_cli1;
1681+
1682+ key = SK_PASS;
1683+ err = xbpf_map_lookup_elem(verd_mapfd, &key, &pass);
1684+ if (err)
1685+ goto close_cli1;
1686+ if (pass != 1)
1687+ FAIL("%s: want pass count 1, have %d", log_prefix, pass);
1688+
1689+ n = read(mode == REDIR_INGRESS ? p0 : c0, &b, 1);
1690+ if (n < 0)
1691+ FAIL_ERRNO("%s: read", log_prefix);
1692+ if (n == 0)
1693+ FAIL("%s: incomplete read", log_prefix);
1694+
1695+close_cli1:
1696+ xclose(c1);
1697+close_peer1:
1698+ xclose(p1);
1699+close_cli0:
1700+ xclose(c0);
1701+close_peer0:
1702+ xclose(p0);
1703+}
1704+
1705+static void udp_skb_redir_to_connected(struct test_sockmap_listen *skel,
1706+ struct bpf_map *inner_map, int family)
1707+{
1708+ int verdict = bpf_program__fd(skel->progs.prog_skb_verdict);
1709+ int verdict_map = bpf_map__fd(skel->maps.verdict_map);
1710+ int sock_map = bpf_map__fd(inner_map);
1711+ int err;
1712+
1713+ err = xbpf_prog_attach(verdict, sock_map, BPF_SK_SKB_VERDICT, 0);
1714+ if (err)
1715+ return;
1716+
1717+ skel->bss->test_ingress = false;
1718+ udp_redir_to_connected(family, SOCK_DGRAM, sock_map, verdict_map,
1719+ REDIR_EGRESS);
1720+ skel->bss->test_ingress = true;
1721+ udp_redir_to_connected(family, SOCK_DGRAM, sock_map, verdict_map,
1722+ REDIR_INGRESS);
1723+
1724+ xbpf_prog_detach2(verdict, sock_map, BPF_SK_SKB_VERDICT);
1725+}
1726+
1727+static void test_udp_redir(struct test_sockmap_listen *skel, struct bpf_map *map,
1728+ int family)
1729+{
1730+ const char *family_name, *map_name;
1731+ char s[MAX_TEST_NAME];
1732+
1733+ family_name = family_str(family);
1734+ map_name = map_type_str(map);
1735+ snprintf(s, sizeof(s), "%s %s %s", map_name, family_name, __func__);
1736+ if (!test__start_subtest(s))
1737+ return;
1738+ udp_skb_redir_to_connected(skel, map, family);
1739+}
1740+
16061741 static void run_tests(struct test_sockmap_listen *skel, struct bpf_map *map,
16071742 int family)
16081743 {
@@ -1611,6 +1746,7 @@ static void run_tests(struct test_sockmap_listen *skel, struct bpf_map *map, · 2 unmodified lines
16111746 test_redir(skel, map, family, SOCK_STREAM);
16121747 test_reuseport(skel, map, family, SOCK_STREAM);
16131748 test_reuseport(skel, map, family, SOCK_DGRAM);
1749+ test_udp_redir(skel, map, family);
16141750 }
16151751
16161752 void test_sockmap_listen(void)
tools/testing/selftests/bpf/prog_tests/test_ima.c+4−2
@@ -68,7 +68,8 @@ void test_test_ima(void) · 67 unmodified lines
6868 goto close_prog;
6969
7070 snprintf(cmd, sizeof(cmd), "./ima_setup.sh setup %s", measured_dir);
71− if (CHECK_FAIL(system(cmd)))
71+ err = system(cmd);
72+ if (CHECK(err, "failed to run command", "%s, errno = %d\n", cmd, errno))
7273 goto close_clean;
7374
7475 err = run_measured_process(measured_dir, &skel->bss->monitored_pid);
@@ -81,7 +82,8 @@ void test_test_ima(void) · 6 unmodified lines
8182
8283 close_clean:
8384 snprintf(cmd, sizeof(cmd), "./ima_setup.sh cleanup %s", measured_dir);
84− CHECK_FAIL(system(cmd));
85+ err = system(cmd);
86+ CHECK(err, "failed to run command", "%s, errno = %d\n", cmd, errno);
8587 close_prog:
8688 ima__destroy(skel);
8789 }
tools/testing/selftests/bpf/progs/bpf_cubic.c+18−18
@@ -174,8 +174,8 @@ static __always_inline void bictcp_hystart_reset(struct sock *sk) · 173 unmodified lines
174174 * as long as it is used in one of the func ptr
175175 * under SEC(".struct_ops").
176176 */
177−SEC("struct_ops/bictcp_init")
178−void BPF_PROG(bictcp_init, struct sock *sk)
177+SEC("struct_ops/bpf_cubic_init")
178+void BPF_PROG(bpf_cubic_init, struct sock *sk)
179179 {
180180 struct bictcp *ca = inet_csk_ca(sk);
181181
@@ -192,7 +192,7 @@ void BPF_PROG(bictcp_init, struct sock *sk) · 10 unmodified lines
192192 * The remaining tcp-cubic functions have an easier way.
193193 */
194194 SEC("no-sec-prefix-bictcp_cwnd_event")
195−void BPF_PROG(bictcp_cwnd_event, struct sock *sk, enum tcp_ca_event event)
195+void BPF_PROG(bpf_cubic_cwnd_event, struct sock *sk, enum tcp_ca_event event)
196196 {
197197 if (event == CA_EVENT_TX_START) {
198198 struct bictcp *ca = inet_csk_ca(sk);
@@ -384,7 +384,7 @@ tcp_friendliness: · 185 unmodified lines
384384 }
385385
386386 /* Or simply use the BPF_STRUCT_OPS to avoid the SEC boiler plate. */
387−void BPF_STRUCT_OPS(bictcp_cong_avoid, struct sock *sk, __u32 ack, __u32 acked)
387+void BPF_STRUCT_OPS(bpf_cubic_cong_avoid, struct sock *sk, __u32 ack, __u32 acked)
388388 {
389389 struct tcp_sock *tp = tcp_sk(sk);
390390 struct bictcp *ca = inet_csk_ca(sk);
@@ -403,7 +403,7 @@ void BPF_STRUCT_OPS(bictcp_cong_avoid, struct sock *sk, __u32 ack, __u32 acked) · 12 unmodified lines
403403 tcp_cong_avoid_ai(tp, ca->cnt, acked);
404404 }
405405
406−__u32 BPF_STRUCT_OPS(bictcp_recalc_ssthresh, struct sock *sk)
406+__u32 BPF_STRUCT_OPS(bpf_cubic_recalc_ssthresh, struct sock *sk)
407407 {
408408 const struct tcp_sock *tp = tcp_sk(sk);
409409 struct bictcp *ca = inet_csk_ca(sk);
@@ -420,7 +420,7 @@ __u32 BPF_STRUCT_OPS(bictcp_recalc_ssthresh, struct sock *sk) · 10 unmodified lines
420420 return max((tp->snd_cwnd * beta) / BICTCP_BETA_SCALE, 2U);
421421 }
422422
423−void BPF_STRUCT_OPS(bictcp_state, struct sock *sk, __u8 new_state)
423+void BPF_STRUCT_OPS(bpf_cubic_state, struct sock *sk, __u8 new_state)
424424 {
425425 if (new_state == TCP_CA_Loss) {
426426 bictcp_reset(inet_csk_ca(sk));
@@ -496,7 +496,7 @@ static __always_inline void hystart_update(struct sock *sk, __u32 delay) · 69 unmodified lines
496496 }
497497 }
498498
499−void BPF_STRUCT_OPS(bictcp_acked, struct sock *sk,
499+void BPF_STRUCT_OPS(bpf_cubic_acked, struct sock *sk,
500500 const struct ack_sample *sample)
501501 {
502502 const struct tcp_sock *tp = tcp_sk(sk);
@@ -525,21 +525,21 @@ void BPF_STRUCT_OPS(bictcp_acked, struct sock *sk, · 22 unmodified lines
525525 hystart_update(sk, delay);
526526 }
527527
528−__u32 BPF_STRUCT_OPS(tcp_reno_undo_cwnd, struct sock *sk)
529−{
530− const struct tcp_sock *tp = tcp_sk(sk);
528+extern __u32 tcp_reno_undo_cwnd(struct sock *sk) __ksym;
531529
532− return max(tp->snd_cwnd, tp->prior_cwnd);
530+__u32 BPF_STRUCT_OPS(bpf_cubic_undo_cwnd, struct sock *sk)
531+{
532+ return tcp_reno_undo_cwnd(sk);
533533 }
534534
535535 SEC(".struct_ops")
536536 struct tcp_congestion_ops cubic = {
537− .init = (void *)bictcp_init,
538− .ssthresh = (void *)bictcp_recalc_ssthresh,
539− .cong_avoid = (void *)bictcp_cong_avoid,
540− .set_state = (void *)bictcp_state,
541− .undo_cwnd = (void *)tcp_reno_undo_cwnd,
542− .cwnd_event = (void *)bictcp_cwnd_event,
543− .pkts_acked = (void *)bictcp_acked,
537+ .init = (void *)bpf_cubic_init,
538+ .ssthresh = (void *)bpf_cubic_recalc_ssthresh,
539+ .cong_avoid = (void *)bpf_cubic_cong_avoid,
540+ .set_state = (void *)bpf_cubic_state,
541+ .undo_cwnd = (void *)bpf_cubic_undo_cwnd,
542+ .cwnd_event = (void *)bpf_cubic_cwnd_event,
543+ .pkts_acked = (void *)bpf_cubic_acked,
544544 .name = "bpf_cubic",
545545 };
tools/testing/selftests/bpf/progs/bpf_dctcp.c+6−16
@@ -194,22 +194,12 @@ __u32 BPF_PROG(dctcp_cwnd_undo, struct sock *sk) · 193 unmodified lines
194194 return max(tcp_sk(sk)->snd_cwnd, ca->loss_cwnd);
195195 }
196196
197−SEC("struct_ops/tcp_reno_cong_avoid")
198−void BPF_PROG(tcp_reno_cong_avoid, struct sock *sk, __u32 ack, __u32 acked)
199−{
200− struct tcp_sock *tp = tcp_sk(sk);
201−
202− if (!tcp_is_cwnd_limited(sk))
203− return;
197+extern void tcp_reno_cong_avoid(struct sock *sk, __u32 ack, __u32 acked) __ksym;
204198
205− /* In "safe" area, increase. */
206− if (tcp_in_slow_start(tp)) {
207− acked = tcp_slow_start(tp, acked);
208− if (!acked)
209− return;
210− }
211− /* In dangerous area, increase slowly. */
212− tcp_cong_avoid_ai(tp, tp->snd_cwnd, acked);
199+SEC("struct_ops/dctcp_reno_cong_avoid")
200+void BPF_PROG(dctcp_cong_avoid, struct sock *sk, __u32 ack, __u32 acked)
201+{
202+ tcp_reno_cong_avoid(sk, ack, acked);
213203 }
214204
215205 SEC(".struct_ops")
@@ -226,7 +216,7 @@ struct tcp_congestion_ops dctcp = { · 10 unmodified lines
226216 .in_ack_event = (void *)dctcp_update_alpha,
227217 .cwnd_event = (void *)dctcp_cwnd_event,
228218 .ssthresh = (void *)dctcp_ssthresh,
229− .cong_avoid = (void *)tcp_reno_cong_avoid,
219+ .cong_avoid = (void *)dctcp_cong_avoid,
230220 .undo_cwnd = (void *)dctcp_cwnd_undo,
231221 .set_state = (void *)dctcp_state,
232222 .flags = TCP_CONG_NEEDS_ECN,
tools/testing/selftests/bpf/progs/kfunc_call_test.c+47−0
@@ -0,0 +1,47 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2021 Facebook */
3+#include <linux/bpf.h>
4+#include <bpf/bpf_helpers.h>
5+#include "bpf_tcp_helpers.h"
6+
7+extern int bpf_kfunc_call_test2(struct sock *sk, __u32 a, __u32 b) __ksym;
8+extern __u64 bpf_kfunc_call_test1(struct sock *sk, __u32 a, __u64 b,
9+ __u32 c, __u64 d) __ksym;
10+
11+SEC("classifier")
12+int kfunc_call_test2(struct __sk_buff *skb)
13+{
14+ struct bpf_sock *sk = skb->sk;
15+
16+ if (!sk)
17+ return -1;
18+
19+ sk = bpf_sk_fullsock(sk);
20+ if (!sk)
21+ return -1;
22+
23+ return bpf_kfunc_call_test2((struct sock *)sk, 1, 2);
24+}
25+
26+SEC("classifier")
27+int kfunc_call_test1(struct __sk_buff *skb)
28+{
29+ struct bpf_sock *sk = skb->sk;
30+ __u64 a = 1ULL << 32;
31+ __u32 ret;
32+
33+ if (!sk)
34+ return -1;
35+
36+ sk = bpf_sk_fullsock(sk);
37+ if (!sk)
38+ return -1;
39+
40+ a = bpf_kfunc_call_test1((struct sock *)sk, 1, a | 2, 3, a | 4);
41+ ret = a >> 32; /* ret should be 2 */
42+ ret += (__u32)a; /* ret should be 12 */
43+
44+ return ret;
45+}
46+
47+char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/kfunc_call_test_subprog.c+42−0
@@ -0,0 +1,42 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2021 Facebook */
3+#include <linux/bpf.h>
4+#include <bpf/bpf_helpers.h>
5+#include "bpf_tcp_helpers.h"
6+
7+extern const int bpf_prog_active __ksym;
8+extern __u64 bpf_kfunc_call_test1(struct sock *sk, __u32 a, __u64 b,
9+ __u32 c, __u64 d) __ksym;
10+extern struct sock *bpf_kfunc_call_test3(struct sock *sk) __ksym;
11+int active_res = -1;
12+int sk_state = -1;
13+
14+int __noinline f1(struct __sk_buff *skb)
15+{
16+ struct bpf_sock *sk = skb->sk;
17+ int *active;
18+
19+ if (!sk)
20+ return -1;
21+
22+ sk = bpf_sk_fullsock(sk);
23+ if (!sk)
24+ return -1;
25+
26+ active = (int *)bpf_per_cpu_ptr(&bpf_prog_active,
27+ bpf_get_smp_processor_id());
28+ if (active)
29+ active_res = *active;
30+
31+ sk_state = bpf_kfunc_call_test3((struct sock *)sk)->__sk_common.skc_state;
32+
33+ return (__u32)bpf_kfunc_call_test1((struct sock *)sk, 1, 2, 3, 4);
34+}
35+
36+SEC("classifier")
37+int kfunc_call_test1(struct __sk_buff *skb)
38+{
39+ return f1(skb);
40+}
41+
42+char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/test_sockmap_listen.c+22−0
@@ -29,6 +29,7 @@ struct { · 28 unmodified lines
2929 } verdict_map SEC(".maps");
3030
3131 static volatile bool test_sockmap; /* toggled by user-space */
32+static volatile bool test_ingress; /* toggled by user-space */
3233
3334 SEC("sk_skb/stream_parser")
3435 int prog_stream_parser(struct __sk_buff *skb)
@@ -55,6 +56,27 @@ int prog_stream_verdict(struct __sk_buff *skb) · 20 unmodified lines
5556 return verdict;
5657 }
5758
59+SEC("sk_skb/skb_verdict")
60+int prog_skb_verdict(struct __sk_buff *skb)
61+{
62+ unsigned int *count;
63+ __u32 zero = 0;
64+ int verdict;
65+
66+ if (test_sockmap)
67+ verdict = bpf_sk_redirect_map(skb, &sock_map, zero,
68+ test_ingress ? BPF_F_INGRESS : 0);
69+ else
70+ verdict = bpf_sk_redirect_hash(skb, &sock_hash, &zero,
71+ test_ingress ? BPF_F_INGRESS : 0);
72+
73+ count = bpf_map_lookup_elem(&verdict_map, &verdict);
74+ if (count)
75+ (*count)++;
76+
77+ return verdict;
78+}
79+
5880 SEC("sk_msg")
5981 int prog_msg_verdict(struct sk_msg_md *msg)
6082 {
tools/testing/selftests/bpf/progs/test_sockmap_skb_verdict_attach.c+18−0
@@ -0,0 +1,18 @@
1+// SPDX-License-Identifier: GPL-2.0
2+#include "vmlinux.h"
3+#include <bpf/bpf_helpers.h>
4+
5+struct {
6+ __uint(type, BPF_MAP_TYPE_SOCKMAP);
7+ __uint(max_entries, 2);
8+ __type(key, __u32);
9+ __type(value, __u64);
10+} sock_map SEC(".maps");
11+
12+SEC("sk_skb/skb_verdict")
13+int prog_skb_verdict(struct __sk_buff *skb)
14+{
15+ return SK_DROP;
16+}
17+
18+char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/test_xsk.sh+2−1
@@ -107,7 +107,7 @@ setup_vethPairs() { · 106 unmodified lines
107107 echo "setting up ${VETH0}: namespace: ${NS0}"
108108 fi
109109 ip netns add ${NS1}
110− ip link add ${VETH0} type veth peer name ${VETH1}
110+ ip link add ${VETH0} numtxqueues 4 numrxqueues 4 type veth peer name ${VETH1} numtxqueues 4 numrxqueues 4
111111 if [ -f /proc/net/if_inet6 ]; then
112112 echo 1 > /proc/sys/net/ipv6/conf/${VETH0}/disable_ipv6
113113 fi
@@ -118,6 +118,7 @@ setup_vethPairs() { · 4 unmodified lines
118118 ip netns exec ${NS1} ip link set ${VETH1} mtu ${MTU}
119119 ip link set ${VETH0} mtu ${MTU}
120120 ip netns exec ${NS1} ip link set ${VETH1} up
121+ ip netns exec ${NS1} ip link set dev lo up
121122 ip link set ${VETH0} up
122123 }
123124
tools/testing/selftests/bpf/verifier/calls.c+6−6
@@ -19,7 +19,7 @@ · 18 unmodified lines
1919 BPF_MOV64_IMM(BPF_REG_0, 2),
2020 BPF_EXIT_INSN(),
2121 },
22− .errstr_unpriv = "function calls to other bpf functions are allowed for",
22+ .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
2323 .result_unpriv = REJECT,
2424 .result = ACCEPT,
2525 .retval = 1,
@@ -136,7 +136,7 @@ · 110 unmodified lines
136136 {
137137 "calls: wrong src reg",
138138 .insns = {
139− BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 2, 0, 0),
139+ BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 3, 0, 0),
140140 BPF_MOV64_IMM(BPF_REG_0, 1),
141141 BPF_EXIT_INSN(),
142142 },
@@ -397,7 +397,7 @@ · 254 unmodified lines
397397 BPF_MOV64_IMM(BPF_REG_0, 1),
398398 BPF_EXIT_INSN(),
399399 },
400− .errstr_unpriv = "function calls to other bpf functions are allowed for",
400+ .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
401401 .fixup_map_hash_48b = { 3 },
402402 .result_unpriv = REJECT,
403403 .result = ACCEPT,
@@ -1977,7 +1977,7 @@ · 1573 unmodified lines
19771977 BPF_EXIT_INSN(),
19781978 },
19791979 .prog_type = BPF_PROG_TYPE_SOCKET_FILTER,
1980− .errstr_unpriv = "function calls to other bpf functions are allowed for",
1980+ .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
19811981 .result_unpriv = REJECT,
19821982 .result = ACCEPT,
19831983 },
@@ -2003,7 +2003,7 @@ · 19 unmodified lines
20032003 BPF_EXIT_INSN(),
20042004 },
20052005 .prog_type = BPF_PROG_TYPE_SOCKET_FILTER,
2006− .errstr_unpriv = "function calls to other bpf functions are allowed for",
2006+ .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
20072007 .errstr = "!read_ok",
20082008 .result = REJECT,
20092009 },
@@ -2028,7 +2028,7 @@ · 18 unmodified lines
20282028 BPF_EXIT_INSN(),
20292029 },
20302030 .prog_type = BPF_PROG_TYPE_SOCKET_FILTER,
2031− .errstr_unpriv = "function calls to other bpf functions are allowed for",
2031+ .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
20322032 .errstr = "!read_ok",
20332033 .result = REJECT,
20342034 },
tools/testing/selftests/bpf/verifier/dead_code.c+5−5
@@ -85,7 +85,7 @@ · 84 unmodified lines
8585 BPF_MOV64_IMM(BPF_REG_0, 12),
8686 BPF_EXIT_INSN(),
8787 },
88− .errstr_unpriv = "function calls to other bpf functions are allowed for",
88+ .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
8989 .result_unpriv = REJECT,
9090 .result = ACCEPT,
9191 .retval = 7,
@@ -103,7 +103,7 @@ · 11 unmodified lines
103103 BPF_MOV64_IMM(BPF_REG_0, 12),
104104 BPF_EXIT_INSN(),
105105 },
106− .errstr_unpriv = "function calls to other bpf functions are allowed for",
106+ .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
107107 .result_unpriv = REJECT,
108108 .result = ACCEPT,
109109 .retval = 7,
@@ -121,7 +121,7 @@ · 11 unmodified lines
121121 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, -5),
122122 BPF_EXIT_INSN(),
123123 },
124− .errstr_unpriv = "function calls to other bpf functions are allowed for",
124+ .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
125125 .result_unpriv = REJECT,
126126 .result = ACCEPT,
127127 .retval = 7,
@@ -137,7 +137,7 @@ · 9 unmodified lines
137137 BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
138138 BPF_EXIT_INSN(),
139139 },
140− .errstr_unpriv = "function calls to other bpf functions are allowed for",
140+ .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
141141 .result_unpriv = REJECT,
142142 .result = ACCEPT,
143143 .retval = 2,
@@ -152,7 +152,7 @@ · 8 unmodified lines
152152 BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
153153 BPF_EXIT_INSN(),
154154 },
155− .errstr_unpriv = "function calls to other bpf functions are allowed for",
155+ .errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
156156 .result_unpriv = REJECT,
157157 .result = ACCEPT,
158158 .retval = 2,
tools/testing/selftests/bpf/vmtest.sh+28−11
@@ -24,15 +24,15 @@ EXIT_STATUS_FILE="${LOG_FILE_BASE}.exit_status" · 23 unmodified lines
2424 usage()
2525 {
2626 cat <<EOF
27−Usage: $0 [-i] [-d <output_dir>] -- [<command>]
27+Usage: $0 [-i] [-s] [-d <output_dir>] -- [<command>]
2828
2929 <command> is the command you would normally run when you are in
3030 tools/testing/selftests/bpf. e.g:
3131
3232 $0 -- ./test_progs -t test_lsm
3333
34−If no command is specified, "${DEFAULT_COMMAND}" will be run by
35−default.
34+If no command is specified and a debug shell (-s) is not requested,
35+"${DEFAULT_COMMAND}" will be run by default.
3636
3737 If you build your kernel using KBUILD_OUTPUT= or O= options, these
3838 can be passed as environment variables to the script:
@@ -49,6 +49,9 @@ Options: · 10 unmodified lines
4949 -d) Update the output directory (default: ${OUTPUT_DIR})
5050 -j) Number of jobs for compilation, similar to -j in make
5151 (default: ${NUM_COMPILE_JOBS})
52+ -s) Instead of powering off the VM, start an interactive
53+ shell. If <command> is specified, the shell runs after
54+ the command finishes executing
5255 EOF
5356 }
5457
@@ -149,6 +152,7 @@ update_init_script() · 94 unmodified lines
149152 local init_script_dir="${OUTPUT_DIR}/${MOUNT_DIR}/etc/rcS.d"
150153 local init_script="${init_script_dir}/S50-startup"
151154 local command="$1"
155+ local exit_command="$2"
152156
153157 mount_image
154158
@@ -162,9 +166,10 @@ EOF · 7 unmodified lines
162166
163167 fi
164168
165− sudo bash -c "cat >${init_script}" <<EOF
166−#!/bin/bash
169+ sudo bash -c "echo '#!/bin/bash' > ${init_script}"
167170
171+ if [[ "${command}" != "" ]]; then
172+ sudo bash -c "cat >>${init_script}" <<EOF
168173 # Have a default value in the exit status file
169174 # incase the VM is forcefully stopped.
170175 echo "130" > "/root/${EXIT_STATUS_FILE}"
@@ -175,9 +180,12 @@ echo "130" > "/root/${EXIT_STATUS_FILE}" · 4 unmodified lines
175180 stdbuf -oL -eL ${command}
176181 echo "\$?" > "/root/${EXIT_STATUS_FILE}"
177182 } 2>&1 | tee "/root/${LOG_FILE}"
178−poweroff -f
183+# Ensure that the logs are written to disk
184+sync
179185 EOF
186+ fi
180187
188+ sudo bash -c "echo ${exit_command} >> ${init_script}"
181189 sudo chmod a+x "${init_script}"
182190 unmount_image
183191 }
@@ -277,8 +285,10 @@ main() · 93 unmodified lines
277285 local kernel_bzimage="${kernel_checkout}/${X86_BZIMAGE}"
278286 local command="${DEFAULT_COMMAND}"
279287 local update_image="no"
288+ local exit_command="poweroff -f"
289+ local debug_shell="no"
280290
281− while getopts 'hkid:j:' opt; do
291+ while getopts 'hskid:j:' opt; do
282292 case ${opt} in
283293 i)
284294 update_image="yes"
@@ -289,6 +299,11 @@ main() · 4 unmodified lines
289299 j)
290300 NUM_COMPILE_JOBS="$OPTARG"
291301 ;;
302+ s)
303+ command=""
304+ debug_shell="yes"
305+ exit_command="bash"
306+ ;;
292307 h)
293308 usage
294309 exit 0
@@ -307,7 +322,7 @@ main() · 12 unmodified lines
307322 done
308323 shift $((OPTIND -1))
309324
310− if [[ $# -eq 0 ]]; then
325+ if [[ $# -eq 0 && "${debug_shell}" == "no" ]]; then
311326 echo "No command specified, will run ${DEFAULT_COMMAND} in the vm"
312327 else
313328 command="$@"
@@ -355,10 +370,12 @@ main() · 41 unmodified lines
355370 fi
356371
357372 update_selftests "${kernel_checkout}" "${make_command}"
358− update_init_script "${command}"
373+ update_init_script "${command}" "${exit_command}"
359374 run_vm "${kernel_bzimage}"
360− copy_logs
361− echo "Logs saved in ${OUTPUT_DIR}/${LOG_FILE}"
375+ if [[ "${command}" != "" ]]; then
376+ copy_logs
377+ echo "Logs saved in ${OUTPUT_DIR}/${LOG_FILE}"
378+ fi
362379 }
363380
364381 catch()
tools/testing/selftests/bpf/xdpxceiver.c+317−383
@@ -41,8 +41,12 @@ · 40 unmodified lines
4141 * Reduce the size of the RX ring to a fraction of the fill ring size.
4242 * iv. fill queue empty
4343 * Do not populate the fill queue and then try to receive pkts.
44+ * f. bpf_link resource persistence
45+ * Configure sockets at indexes 0 and 1, run a traffic on queue ids 0,
46+ * then remove xsk sockets from queue 0 on both veth interfaces and
47+ * finally run a traffic on queues ids 1
4448 *
45− * Total tests: 10
49+ * Total tests: 12
4650 *
4751 * Flow:
4852 * -----
@@ -93,6 +97,13 @@ typedef __u16 __sum16; · 44 unmodified lines
9397 #include "xdpxceiver.h"
9498 #include "../kselftest.h"
9599
100+static const char *MAC1 = "\x00\x0A\x56\x9E\xEE\x62";
101+static const char *MAC2 = "\x00\x0A\x56\x9E\xEE\x61";
102+static const char *IP1 = "192.168.100.162";
103+static const char *IP2 = "192.168.100.161";
104+static const u16 UDP_PORT1 = 2020;
105+static const u16 UDP_PORT2 = 2121;
106+
96107 static void __exit_with_error(int error, const char *file, const char *func, int line)
97108 {
98109 if (configured_mode == TEST_MODE_UNCONFIGURED) {
@@ -108,27 +119,12 @@ static void __exit_with_error(int error, const char *file, const char *func, int · 9 unmodified lines
108119 #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__)
109120
110121 #define print_ksft_result(void)\
111− (ksft_test_result_pass("PASS: %s %s %s%s%s\n", configured_mode ? "DRV" : "SKB",\
122+ (ksft_test_result_pass("PASS: %s %s %s%s%s%s\n", configured_mode ? "DRV" : "SKB",\
112123 test_type == TEST_TYPE_POLL ? "POLL" : "NOPOLL",\
113124 test_type == TEST_TYPE_TEARDOWN ? "Socket Teardown" : "",\
114125 test_type == TEST_TYPE_BIDI ? "Bi-directional Sockets" : "",\
115− test_type == TEST_TYPE_STATS ? "Stats" : ""))
116−
117−static void pthread_init_mutex(void)
118−{
119− pthread_mutex_init(&sync_mutex, NULL);
120− pthread_mutex_init(&sync_mutex_tx, NULL);
121− pthread_cond_init(&signal_rx_condition, NULL);
122− pthread_cond_init(&signal_tx_condition, NULL);
123−}
124−
125−static void pthread_destroy_mutex(void)
126−{
127− pthread_mutex_destroy(&sync_mutex);
128− pthread_mutex_destroy(&sync_mutex_tx);
129− pthread_cond_destroy(&signal_rx_condition);
130− pthread_cond_destroy(&signal_tx_condition);
131−}
126+ test_type == TEST_TYPE_STATS ? "Stats" : "",\
127+ test_type == TEST_TYPE_BPF_RES ? "BPF RES" : ""))
132128
133129 static void *memset32_htonl(void *dest, u32 val, u32 size)
134130 {
@@ -146,25 +142,12 @@ static void *memset32_htonl(void *dest, u32 val, u32 size) · 11 unmodified lines
146142 return dest;
147143 }
148144
149−/*
150− * This function code has been taken from
151− * Linux kernel lib/checksum.c
152− */
153−static inline unsigned short from32to16(unsigned int x)
154−{
155− /* add up 16-bit and 16-bit for 16+c bit */
156− x = (x & 0xffff) + (x >> 16);
157− /* add up carry.. */
158− x = (x & 0xffff) + (x >> 16);
159− return x;
160−}
161−
162145 /*
163146 * Fold a partial checksum
164147 * This function code has been taken from
165148 * Linux kernel include/asm-generic/checksum.h
166149 */
167−static inline __u16 csum_fold(__u32 csum)
150+static __u16 csum_fold(__u32 csum)
168151 {
169152 u32 sum = (__force u32)csum;
170153
@@ -177,7 +160,7 @@ static inline __u16 csum_fold(__u32 csum) · 6 unmodified lines
177160 * This function code has been taken from
178161 * Linux kernel lib/checksum.c
179162 */
180−static inline u32 from64to32(u64 x)
163+static u32 from64to32(u64 x)
181164 {
182165 /* add up 32-bit and 32-bit for 32+c bit */
183166 x = (x & 0xffffffff) + (x >> 32);
@@ -186,13 +169,11 @@ static inline u32 from64to32(u64 x) · 2 unmodified lines
186169 return (u32)x;
187170 }
188171
189−__u32 csum_tcpudp_nofold(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum);
190−
191172 /*
192173 * This function code has been taken from
193174 * Linux kernel lib/checksum.c
194175 */
195−__u32 csum_tcpudp_nofold(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum)
176+static __u32 csum_tcpudp_nofold(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum)
196177 {
197178 unsigned long long s = (__force u32)sum;
198179
@@ -210,13 +191,12 @@ __u32 csum_tcpudp_nofold(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u3 · 11 unmodified lines
210191 * This function has been taken from
211192 * Linux kernel include/asm-generic/checksum.h
212193 */
213−static inline __u16
214−csum_tcpudp_magic(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum)
194+static __u16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum)
215195 {
216196 return csum_fold(csum_tcpudp_nofold(saddr, daddr, len, proto, sum));
217197 }
218198
219−static inline u16 udp_csum(u32 saddr, u32 daddr, u32 len, u8 proto, u16 *udp_pkt)
199+static u16 udp_csum(u32 saddr, u32 daddr, u32 len, u8 proto, u16 *udp_pkt)
220200 {
221201 u32 csum = 0;
222202 u32 cnt = 0;
@@ -271,9 +251,8 @@ static void gen_eth_frame(struct xsk_umem_info *umem, u64 addr) · 48 unmodified lines
271251 memcpy(xsk_umem__get_data(umem->buffer, addr), pkt_data, PKT_SIZE);
272252 }
273253
274−static void xsk_configure_umem(struct ifobject *data, void *buffer, u64 size)
254+static void xsk_configure_umem(struct ifobject *data, void *buffer, int idx)
275255 {
276− int ret;
277256 struct xsk_umem_config cfg = {
278257 .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,
279258 .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,
@@ -281,17 +260,22 @@ static void xsk_configure_umem(struct ifobject *data, void *buffer, u64 size) · 1 unmodified line
281260 .frame_headroom = frame_headroom,
282261 .flags = XSK_UMEM__DEFAULT_FLAGS
283262 };
263+ int size = num_frames * XSK_UMEM__DEFAULT_FRAME_SIZE;
264+ struct xsk_umem_info *umem;
265+ int ret;
284266
285− data->umem = calloc(1, sizeof(struct xsk_umem_info));
286− if (!data->umem)
267+ umem = calloc(1, sizeof(struct xsk_umem_info));
268+ if (!umem)
287269 exit_with_error(errno);
288270
289− ret = xsk_umem__create(&data->umem->umem, buffer, size,
290− &data->umem->fq, &data->umem->cq, &cfg);
271+ ret = xsk_umem__create(&umem->umem, buffer, size,
272+ &umem->fq, &umem->cq, &cfg);
291273 if (ret)
292274 exit_with_error(ret);
293275
294− data->umem->buffer = buffer;
276+ umem->buffer = buffer;
277+
278+ data->umem_arr[idx] = umem;
295279 }
296280
297281 static void xsk_populate_fill_ring(struct xsk_umem_info *umem)
@@ -307,18 +291,19 @@ static void xsk_populate_fill_ring(struct xsk_umem_info *umem) · 9 unmodified lines
307291 xsk_ring_prod__submit(&umem->fq, XSK_RING_PROD__DEFAULT_NUM_DESCS);
308292 }
309293
310−static int xsk_configure_socket(struct ifobject *ifobject)
294+static int xsk_configure_socket(struct ifobject *ifobject, int idx)
311295 {
312296 struct xsk_socket_config cfg;
297+ struct xsk_socket_info *xsk;
313298 struct xsk_ring_cons *rxr;
314299 struct xsk_ring_prod *txr;
315300 int ret;
316301
317− ifobject->xsk = calloc(1, sizeof(struct xsk_socket_info));
318− if (!ifobject->xsk)
302+ xsk = calloc(1, sizeof(struct xsk_socket_info));
303+ if (!xsk)
319304 exit_with_error(errno);
320305
321− ifobject->xsk->umem = ifobject->umem;
306+ xsk->umem = ifobject->umem;
322307 cfg.rx_size = rxqsize;
323308 cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
324309 cfg.libbpf_flags = 0;
@@ -326,19 +311,20 @@ static int xsk_configure_socket(struct ifobject *ifobject) · 1 unmodified line
326311 cfg.bind_flags = xdp_bind_flags;
327312
328313 if (test_type != TEST_TYPE_BIDI) {
329− rxr = (ifobject->fv.vector == rx) ? &ifobject->xsk->rx : NULL;
330− txr = (ifobject->fv.vector == tx) ? &ifobject->xsk->tx : NULL;
314+ rxr = (ifobject->fv.vector == rx) ? &xsk->rx : NULL;
315+ txr = (ifobject->fv.vector == tx) ? &xsk->tx : NULL;
331316 } else {
332− rxr = &ifobject->xsk->rx;
333− txr = &ifobject->xsk->tx;
317+ rxr = &xsk->rx;
318+ txr = &xsk->tx;
334319 }
335320
336− ret = xsk_socket__create(&ifobject->xsk->xsk, ifobject->ifname,
337− opt_queue, ifobject->umem->umem, rxr, txr, &cfg);
338−
321+ ret = xsk_socket__create(&xsk->xsk, ifobject->ifname, idx,
322+ ifobject->umem->umem, rxr, txr, &cfg);
339323 if (ret)
340324 return 1;
341325
326+ ifobject->xsk_arr[idx] = xsk;
327+
342328 return 0;
343329 }
344330
@@ -364,12 +350,15 @@ static void usage(const char *prog) · 19 unmodified lines
364350 ksft_print_msg(str, prog);
365351 }
366352
367−static bool switch_namespace(int idx)
353+static int switch_namespace(const char *nsname)
368354 {
369355 char fqns[26] = "/var/run/netns/";
370356 int nsfd;
371357
372− strncat(fqns, ifdict[idx]->nsname, sizeof(fqns) - strlen(fqns) - 1);
358+ if (!nsname || strlen(nsname) == 0)
359+ return -1;
360+
361+ strncat(fqns, nsname, sizeof(fqns) - strlen(fqns) - 1);
373362 nsfd = open(fqns, O_RDONLY);
374363
375364 if (nsfd == -1)
@@ -378,26 +367,9 @@ static bool switch_namespace(int idx) · 2 unmodified lines
378367 if (setns(nsfd, 0) == -1)
379368 exit_with_error(errno);
380369
381− return true;
382−}
370+ print_verbose("NS switched: %s\n", nsname);
383371
384−static void *nsswitchthread(void *args)
385−{
386− struct targs *targs = args;
387−
388− targs->retptr = false;
389−
390− if (switch_namespace(targs->idx)) {
391− ifdict[targs->idx]->ifindex = if_nametoindex(ifdict[targs->idx]->ifname);
392− if (!ifdict[targs->idx]->ifindex) {
393− ksft_test_result_fail("ERROR: [%s] interface \"%s\" does not exist\n",
394− __func__, ifdict[targs->idx]->ifname);
395− } else {
396− print_verbose("Interface found: %s\n", ifdict[targs->idx]->ifname);
397− targs->retptr = true;
398− }
399− }
400− pthread_exit(NULL);
372+ return nsfd;
401373 }
402374
403375 static int validate_interfaces(void)
@@ -409,33 +381,6 @@ static int validate_interfaces(void) · 5 unmodified lines
409381 ret = false;
410382 ksft_test_result_fail("ERROR: interfaces: -i <int>,<ns> -i <int>,<ns>.");
411383 }
412− if (strcmp(ifdict[i]->nsname, "")) {
413− struct targs *targs;
414−
415− targs = malloc(sizeof(*targs));
416− if (!targs)
417− exit_with_error(errno);
418−
419− targs->idx = i;
420− if (pthread_create(&ns_thread, NULL, nsswitchthread, targs))
421− exit_with_error(errno);
422−
423− pthread_join(ns_thread, NULL);
424−
425− if (targs->retptr)
426− print_verbose("NS switched: %s\n", ifdict[i]->nsname);
427−
428− free(targs);
429− } else {
430− ifdict[i]->ifindex = if_nametoindex(ifdict[i]->ifname);
431− if (!ifdict[i]->ifindex) {
432− ksft_test_result_fail
433− ("ERROR: interface \"%s\" does not exist\n", ifdict[i]->ifname);
434− ret = false;
435− } else {
436− print_verbose("Interface found: %s\n", ifdict[i]->ifname);
437− }
438− }
439384 }
440385 return ret;
441386 }
@@ -447,7 +392,7 @@ static void parse_command_line(int argc, char **argv) · 5 unmodified lines
447392 opterr = 0;
448393
449394 for (;;) {
450− c = getopt_long(argc, argv, "i:q:DC:v", long_options, &option_index);
395+ c = getopt_long(argc, argv, "i:DC:v", long_options, &option_index);
451396
452397 if (c == -1)
453398 break;
@@ -467,9 +412,6 @@ static void parse_command_line(int argc, char **argv) · 13 unmodified lines
467412 MAX_INTERFACES_NAMESPACE_CHARS);
468413 interface_index++;
469414 break;
470− case 'q':
471− opt_queue = atoi(optarg);
472− break;
473415 case 'D':
474416 debug_pkt_dump = 1;
475417 break;
@@ -506,7 +448,7 @@ static void kick_tx(struct xsk_socket_info *xsk) · 30 unmodified lines
506448 exit_with_error(errno);
507449 }
508450
509−static inline void complete_tx_only(struct xsk_socket_info *xsk, int batch_size)
451+static void complete_tx_only(struct xsk_socket_info *xsk, int batch_size)
510452 {
511453 unsigned int rcvd;
512454 u32 idx;
@@ -514,7 +456,7 @@ static inline void complete_tx_only(struct xsk_socket_info *xsk, int batch_size) · 1 unmodified line
514456 if (!xsk->outstanding_tx)
515457 return;
516458
517− if (!NEED_WAKEUP || xsk_ring_prod__needs_wakeup(&xsk->tx))
459+ if (xsk_ring_prod__needs_wakeup(&xsk->tx))
518460 kick_tx(xsk);
519461
520462 rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx);
@@ -602,16 +544,15 @@ static void tx_only(struct xsk_socket_info *xsk, u32 *frameptr, int batch_size) · 81 unmodified lines
602544 xsk_ring_prod__submit(&xsk->tx, batch_size);
603545 if (!tx_invalid_test) {
604546 xsk->outstanding_tx += batch_size;
605− } else {
606− if (!NEED_WAKEUP || xsk_ring_prod__needs_wakeup(&xsk->tx))
607− kick_tx(xsk);
547+ } else if (xsk_ring_prod__needs_wakeup(&xsk->tx)) {
548+ kick_tx(xsk);
608549 }
609550 *frameptr += batch_size;
610551 *frameptr %= num_frames;
611552 complete_tx_only(xsk, batch_size);
612553 }
613554
614−static inline int get_batch_size(int pkt_cnt)
555+static int get_batch_size(int pkt_cnt)
615556 {
616557 if (!opt_pkt_count)
617558 return BATCH_SIZE;
@@ -667,45 +608,40 @@ static void tx_only_all(struct ifobject *ifobject) · 49 unmodified lines
667608
668609 static void worker_pkt_dump(void)
669610 {
670− struct in_addr ipaddr;
611+ struct ethhdr *ethhdr;
612+ struct iphdr *iphdr;
613+ struct udphdr *udphdr;
614+ char s[128];
615+ int payload;
616+ void *ptr;
671617
672618 fprintf(stdout, "---------------------------------------\n");
673619 for (int iter = 0; iter < num_frames - 1; iter++) {
620+ ptr = pkt_buf[iter]->payload;
621+ ethhdr = ptr;
622+ iphdr = ptr + sizeof(*ethhdr);
623+ udphdr = ptr + sizeof(*ethhdr) + sizeof(*iphdr);
624+
674625 /*extract L2 frame */
675626 fprintf(stdout, "DEBUG>> L2: dst mac: ");
676627 for (int i = 0; i < ETH_ALEN; i++)
677− fprintf(stdout, "%02X", ((struct ethhdr *)
678− pkt_buf[iter]->payload)->h_dest[i]);
628+ fprintf(stdout, "%02X", ethhdr->h_dest[i]);
679629
680630 fprintf(stdout, "\nDEBUG>> L2: src mac: ");
681631 for (int i = 0; i < ETH_ALEN; i++)
682− fprintf(stdout, "%02X", ((struct ethhdr *)
683− pkt_buf[iter]->payload)->h_source[i]);
632+ fprintf(stdout, "%02X", ethhdr->h_source[i]);
684633
685634 /*extract L3 frame */
686− fprintf(stdout, "\nDEBUG>> L3: ip_hdr->ihl: %02X\n",
687− ((struct iphdr *)(pkt_buf[iter]->payload + sizeof(struct ethhdr)))->ihl);
688−
689− ipaddr.s_addr =
690− ((struct iphdr *)(pkt_buf[iter]->payload + sizeof(struct ethhdr)))->saddr;
691− fprintf(stdout, "DEBUG>> L3: ip_hdr->saddr: %s\n", inet_ntoa(ipaddr));
692−
693− ipaddr.s_addr =
694− ((struct iphdr *)(pkt_buf[iter]->payload + sizeof(struct ethhdr)))->daddr;
695− fprintf(stdout, "DEBUG>> L3: ip_hdr->daddr: %s\n", inet_ntoa(ipaddr));
696−
635+ fprintf(stdout, "\nDEBUG>> L3: ip_hdr->ihl: %02X\n", iphdr->ihl);
636+ fprintf(stdout, "DEBUG>> L3: ip_hdr->saddr: %s\n",
637+ inet_ntop(AF_INET, &iphdr->saddr, s, sizeof(s)));
638+ fprintf(stdout, "DEBUG>> L3: ip_hdr->daddr: %s\n",
639+ inet_ntop(AF_INET, &iphdr->daddr, s, sizeof(s)));
697640 /*extract L4 frame */
698− fprintf(stdout, "DEBUG>> L4: udp_hdr->src: %d\n",
699− ntohs(((struct udphdr *)(pkt_buf[iter]->payload +
700− sizeof(struct ethhdr) +
701− sizeof(struct iphdr)))->source));
702−
703− fprintf(stdout, "DEBUG>> L4: udp_hdr->dst: %d\n",
704− ntohs(((struct udphdr *)(pkt_buf[iter]->payload +
705− sizeof(struct ethhdr) +
706− sizeof(struct iphdr)))->dest));
641+ fprintf(stdout, "DEBUG>> L4: udp_hdr->src: %d\n", ntohs(udphdr->source));
642+ fprintf(stdout, "DEBUG>> L4: udp_hdr->dst: %d\n", ntohs(udphdr->dest));
707643 /*extract L5 frame */
708− int payload = *((uint32_t *)(pkt_buf[iter]->payload + PKT_HDR_SIZE));
644+ payload = *((uint32_t *)(ptr + PKT_HDR_SIZE));
709645
710646 if (payload == EOT) {
711647 print_verbose("End-of-transmission frame received\n");
@@ -809,37 +745,69 @@ static void worker_pkt_validate(void) · 97 unmodified lines
809745 }
810746 }
811747
812−static void thread_common_ops(struct ifobject *ifobject, void *bufs, pthread_mutex_t *mutexptr,
813− atomic_int *spinningptr)
748+static void thread_common_ops(struct ifobject *ifobject, void *bufs)
814749 {
750+ int umem_sz = num_frames * XSK_UMEM__DEFAULT_FRAME_SIZE;
815751 int ctr = 0;
816752 int ret;
817753
818− xsk_configure_umem(ifobject, bufs, num_frames * XSK_UMEM__DEFAULT_FRAME_SIZE);
819− ret = xsk_configure_socket(ifobject);
754+ ifobject->ns_fd = switch_namespace(ifobject->nsname);
755+
756+ if (test_type == TEST_TYPE_BPF_RES)
757+ umem_sz *= 2;
758+
759+ bufs = mmap(NULL, umem_sz,
760+ PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
761+ if (bufs == MAP_FAILED)
762+ exit_with_error(errno);
763+
764+ xsk_configure_umem(ifobject, bufs, 0);
765+ ifobject->umem = ifobject->umem_arr[0];
766+ ret = xsk_configure_socket(ifobject, 0);
820767
821768 /* Retry Create Socket if it fails as xsk_socket__create()
822769 * is asynchronous
823− *
824− * Essential to lock Mutex here to prevent Tx thread from
825− * entering before Rx and causing a deadlock
826770 */
827− pthread_mutex_lock(mutexptr);
828771 while (ret && ctr < SOCK_RECONF_CTR) {
829− atomic_store(spinningptr, 1);
830− xsk_configure_umem(ifobject, bufs, num_frames * XSK_UMEM__DEFAULT_FRAME_SIZE);
831− ret = xsk_configure_socket(ifobject);
772+ xsk_configure_umem(ifobject, bufs, 0);
773+ ifobject->umem = ifobject->umem_arr[0];
774+ ret = xsk_configure_socket(ifobject, 0);
832775 usleep(USLEEP_MAX);
833776 ctr++;
834777 }
835− atomic_store(spinningptr, 0);
836− pthread_mutex_unlock(mutexptr);
837778
838779 if (ctr >= SOCK_RECONF_CTR)
839780 exit_with_error(ret);
781+
782+ ifobject->umem = ifobject->umem_arr[0];
783+ ifobject->xsk = ifobject->xsk_arr[0];
784+
785+ if (test_type == TEST_TYPE_BPF_RES) {
786+ xsk_configure_umem(ifobject, (u8 *)bufs + (umem_sz / 2), 1);
787+ ifobject->umem = ifobject->umem_arr[1];
788+ ret = xsk_configure_socket(ifobject, 1);
789+ }
790+
791+ ifobject->umem = ifobject->umem_arr[0];
792+ ifobject->xsk = ifobject->xsk_arr[0];
793+ print_verbose("Interface [%s] vector [%s]\n",
794+ ifobject->ifname, ifobject->fv.vector == tx ? "Tx" : "Rx");
795+}
796+
797+static bool testapp_is_test_two_stepped(void)
798+{
799+ return (test_type != TEST_TYPE_BIDI && test_type != TEST_TYPE_BPF_RES) || second_step;
800+}
801+
802+static void testapp_cleanup_xsk_res(struct ifobject *ifobj)
803+{
804+ if (testapp_is_test_two_stepped()) {
805+ xsk_socket__delete(ifobj->xsk->xsk);
806+ (void)xsk_umem__delete(ifobj->umem->umem);
807+ }
840808 }
841809
842−static void *worker_testapp_validate(void *arg)
810+static void *worker_testapp_validate_tx(void *arg)
843811 {
844812 struct udphdr *udp_hdr =
845813 (struct udphdr *)(pkt_data + sizeof(struct ethhdr) + sizeof(struct iphdr));
@@ -849,157 +817,97 @@ static void *worker_testapp_validate(void *arg) · 3 unmodified lines
849817 struct generic_data data;
850818 void *bufs = NULL;
851819
852− pthread_attr_setstacksize(&attr, THREAD_STACK);
853−
854− if (!bidi_pass) {
855− bufs = mmap(NULL, num_frames * XSK_UMEM__DEFAULT_FRAME_SIZE,
856− PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
857− if (bufs == MAP_FAILED)
858− exit_with_error(errno);
859−
860− if (strcmp(ifobject->nsname, ""))
861− switch_namespace(ifobject->ifdict_index);
820+ if (!second_step)
821+ thread_common_ops(ifobject, bufs);
822+
823+ for (int i = 0; i < num_frames; i++) {
824+ /*send EOT frame */
825+ if (i == (num_frames - 1))
826+ data.seqnum = -1;
827+ else
828+ data.seqnum = i;
829+ gen_udp_hdr(&data, ifobject, udp_hdr);
830+ gen_ip_hdr(ifobject, ip_hdr);
831+ gen_udp_csum(udp_hdr, ip_hdr);
832+ gen_eth_hdr(ifobject, eth_hdr);
833+ gen_eth_frame(ifobject->umem, i * XSK_UMEM__DEFAULT_FRAME_SIZE);
862834 }
863835
864− if (ifobject->fv.vector == tx) {
865− int spinningrxctr = 0;
836+ print_verbose("Sending %d packets on interface %s\n",
837+ (opt_pkt_count - 1), ifobject->ifname);
838+ tx_only_all(ifobject);
866839
867− if (!bidi_pass)
868− thread_common_ops(ifobject, bufs, &sync_mutex_tx, &spinning_tx);
869−
870− while (atomic_load(&spinning_rx) && spinningrxctr < SOCK_RECONF_CTR) {
871− spinningrxctr++;
872− usleep(USLEEP_MAX);
873− }
840+ testapp_cleanup_xsk_res(ifobject);
841+ pthread_exit(NULL);
842+}
874843
875− print_verbose("Interface [%s] vector [Tx]\n", ifobject->ifname);
876− for (int i = 0; i < num_frames; i++) {
877− /*send EOT frame */
878− if (i == (num_frames - 1))
879− data.seqnum = -1;
880− else
881− data.seqnum = i;
882− gen_udp_hdr(&data, ifobject, udp_hdr);
883− gen_ip_hdr(ifobject, ip_hdr);
884− gen_udp_csum(udp_hdr, ip_hdr);
885− gen_eth_hdr(ifobject, eth_hdr);
886− gen_eth_frame(ifobject->umem, i * XSK_UMEM__DEFAULT_FRAME_SIZE);
887− }
844+static void *worker_testapp_validate_rx(void *arg)
845+{
846+ struct ifobject *ifobject = (struct ifobject *)arg;
847+ struct pollfd fds[MAX_SOCKS] = { };
848+ void *bufs = NULL;
888849
889− print_verbose("Sending %d packets on interface %s\n",
890− (opt_pkt_count - 1), ifobject->ifname);
891− tx_only_all(ifobject);
892− } else if (ifobject->fv.vector == rx) {
893− struct pollfd fds[MAX_SOCKS] = { };
894− int ret;
895−
896− if (!bidi_pass)
897− thread_common_ops(ifobject, bufs, &sync_mutex_tx, &spinning_rx);
898−
899− print_verbose("Interface [%s] vector [Rx]\n", ifobject->ifname);
900− if (stat_test_type != STAT_TEST_RX_FILL_EMPTY)
901− xsk_populate_fill_ring(ifobject->umem);
902−
903− TAILQ_INIT(&head);
904− if (debug_pkt_dump) {
905− pkt_buf = calloc(num_frames, sizeof(*pkt_buf));
906− if (!pkt_buf)
907− exit_with_error(errno);
908− }
850+ if (!second_step)
851+ thread_common_ops(ifobject, bufs);
909852
910− fds[0].fd = xsk_socket__fd(ifobject->xsk->xsk);
911− fds[0].events = POLLIN;
853+ if (stat_test_type != STAT_TEST_RX_FILL_EMPTY)
854+ xsk_populate_fill_ring(ifobject->umem);
912855
913− pthread_mutex_lock(&sync_mutex);
914− pthread_cond_signal(&signal_rx_condition);
915− pthread_mutex_unlock(&sync_mutex);
856+ TAILQ_INIT(&head);
857+ if (debug_pkt_dump) {
858+ pkt_buf = calloc(num_frames, sizeof(*pkt_buf));
859+ if (!pkt_buf)
860+ exit_with_error(errno);
861+ }
916862
917− while (1) {
918− if (test_type == TEST_TYPE_POLL) {
919− ret = poll(fds, 1, POLL_TMOUT);
920− if (ret <= 0)
921− continue;
922− }
863+ fds[0].fd = xsk_socket__fd(ifobject->xsk->xsk);
864+ fds[0].events = POLLIN;
923865
924− if (test_type != TEST_TYPE_STATS) {
925− rx_pkt(ifobject->xsk, fds);
926− worker_pkt_validate();
927− } else {
928− worker_stats_validate(ifobject);
929− }
866+ pthread_barrier_wait(&barr);
930867
931− if (sigvar)
932− break;
868+ while (1) {
869+ if (test_type != TEST_TYPE_STATS) {
870+ rx_pkt(ifobject->xsk, fds);
871+ worker_pkt_validate();
872+ } else {
873+ worker_stats_validate(ifobject);
933874 }
875+ if (sigvar)
876+ break;
877+ }
934878
935− if (test_type != TEST_TYPE_STATS)
936− print_verbose("Received %d packets on interface %s\n",
937− pkt_counter, ifobject->ifname);
879+ print_verbose("Received %d packets on interface %s\n",
880+ pkt_counter, ifobject->ifname);
938881
939− if (test_type == TEST_TYPE_TEARDOWN)
940− print_verbose("Destroying socket\n");
941− }
882+ if (test_type == TEST_TYPE_TEARDOWN)
883+ print_verbose("Destroying socket\n");
942884
943− if ((test_type != TEST_TYPE_BIDI) || bidi_pass) {
944− xsk_socket__delete(ifobject->xsk->xsk);
945− (void)xsk_umem__delete(ifobject->umem->umem);
946− }
885+ testapp_cleanup_xsk_res(ifobject);
947886 pthread_exit(NULL);
948887 }
949888
950889 static void testapp_validate(void)
951890 {
952− struct timespec max_wait = { 0, 0 };
953891 bool bidi = test_type == TEST_TYPE_BIDI;
892+ bool bpf = test_type == TEST_TYPE_BPF_RES;
954893
955− pthread_attr_init(&attr);
956− pthread_attr_setstacksize(&attr, THREAD_STACK);
957−
958− if ((test_type == TEST_TYPE_BIDI) && bidi_pass) {
959− pthread_init_mutex();
960− if (!switching_notify) {
961− print_verbose("Switching Tx/Rx vectors\n");
962− switching_notify++;
963− }
964− }
965−
966− pthread_mutex_lock(&sync_mutex);
894+ if (pthread_barrier_init(&barr, NULL, 2))
895+ exit_with_error(errno);
967896
968897 /*Spawn RX thread */
969− if (!bidi || !bidi_pass) {
970− if (pthread_create(&t0, &attr, worker_testapp_validate, ifdict[1]))
971− exit_with_error(errno);
972− } else if (bidi && bidi_pass) {
973− /*switch Tx/Rx vectors */
974− ifdict[0]->fv.vector = rx;
975− if (pthread_create(&t0, &attr, worker_testapp_validate, ifdict[0]))
976− exit_with_error(errno);
977− }
978−
979− if (clock_gettime(CLOCK_REALTIME, &max_wait))
980− exit_with_error(errno);
981− max_wait.tv_sec += TMOUT_SEC;
898+ pthread_create(&t0, NULL, ifdict_rx->func_ptr, ifdict_rx);
982899
983− if (pthread_cond_timedwait(&signal_rx_condition, &sync_mutex, &max_wait) == ETIMEDOUT)
900+ pthread_barrier_wait(&barr);
901+ if (pthread_barrier_destroy(&barr))
984902 exit_with_error(errno);
985903
986− pthread_mutex_unlock(&sync_mutex);
987−
988904 /*Spawn TX thread */
989− if (!bidi || !bidi_pass) {
990− if (pthread_create(&t1, &attr, worker_testapp_validate, ifdict[0]))
991− exit_with_error(errno);
992− } else if (bidi && bidi_pass) {
993− /*switch Tx/Rx vectors */
994− ifdict[1]->fv.vector = tx;
995− if (pthread_create(&t1, &attr, worker_testapp_validate, ifdict[1]))
996− exit_with_error(errno);
997− }
905+ pthread_create(&t1, NULL, ifdict_tx->func_ptr, ifdict_tx);
998906
999907 pthread_join(t1, NULL);
1000908 pthread_join(t0, NULL);
1001909
1002− if (debug_pkt_dump) {
910+ if (debug_pkt_dump && test_type != TEST_TYPE_STATS) {
1003911 worker_pkt_dump();
1004912 for (int iter = 0; iter < num_frames - 1; iter++) {
1005913 free(pkt_buf[iter]->payload);
@@ -1008,20 +916,85 @@ static void testapp_validate(void) · 2 unmodified lines
1008916 free(pkt_buf);
1009917 }
1010918
1011− if (!(test_type == TEST_TYPE_TEARDOWN) && !bidi && !(test_type == TEST_TYPE_STATS))
919+ if (!(test_type == TEST_TYPE_TEARDOWN) && !bidi && !bpf && !(test_type == TEST_TYPE_STATS))
1012920 print_ksft_result();
1013921 }
1014922
1015−static void testapp_sockets(void)
923+static void testapp_teardown(void)
1016924 {
1017− for (int i = 0; i < ((test_type == TEST_TYPE_TEARDOWN) ? MAX_TEARDOWN_ITER : MAX_BIDI_ITER);
1018− i++) {
925+ int i;
926+
927+ for (i = 0; i < MAX_TEARDOWN_ITER; i++) {
1019928 pkt_counter = 0;
1020929 prev_pkt = -1;
1021930 sigvar = 0;
1022931 print_verbose("Creating socket\n");
1023932 testapp_validate();
1024− test_type == TEST_TYPE_BIDI ? bidi_pass++ : bidi_pass;
933+ }
934+
935+ print_ksft_result();
936+}
937+
938+static void swap_vectors(struct ifobject *ifobj1, struct ifobject *ifobj2)
939+{
940+ void *(*tmp_func_ptr)(void *) = ifobj1->func_ptr;
941+ enum fvector tmp_vector = ifobj1->fv.vector;
942+
943+ ifobj1->func_ptr = ifobj2->func_ptr;
944+ ifobj1->fv.vector = ifobj2->fv.vector;
945+
946+ ifobj2->func_ptr = tmp_func_ptr;
947+ ifobj2->fv.vector = tmp_vector;
948+
949+ ifdict_tx = ifobj1;
950+ ifdict_rx = ifobj2;
951+}
952+
953+static void testapp_bidi(void)
954+{
955+ for (int i = 0; i < MAX_BIDI_ITER; i++) {
956+ pkt_counter = 0;
957+ prev_pkt = -1;
958+ sigvar = 0;
959+ print_verbose("Creating socket\n");
960+ testapp_validate();
961+ if (!second_step) {
962+ print_verbose("Switching Tx/Rx vectors\n");
963+ swap_vectors(ifdict[1], ifdict[0]);
964+ }
965+ second_step = true;
966+ }
967+
968+ swap_vectors(ifdict[0], ifdict[1]);
969+
970+ print_ksft_result();
971+}
972+
973+static void swap_xsk_res(void)
974+{
975+ xsk_socket__delete(ifdict_tx->xsk->xsk);
976+ xsk_umem__delete(ifdict_tx->umem->umem);
977+ xsk_socket__delete(ifdict_rx->xsk->xsk);
978+ xsk_umem__delete(ifdict_rx->umem->umem);
979+ ifdict_tx->umem = ifdict_tx->umem_arr[1];
980+ ifdict_tx->xsk = ifdict_tx->xsk_arr[1];
981+ ifdict_rx->umem = ifdict_rx->umem_arr[1];
982+ ifdict_rx->xsk = ifdict_rx->xsk_arr[1];
983+}
984+
985+static void testapp_bpf_res(void)
986+{
987+ int i;
988+
989+ for (i = 0; i < MAX_BPF_ITER; i++) {
990+ pkt_counter = 0;
991+ prev_pkt = -1;
992+ sigvar = 0;
993+ print_verbose("Creating socket\n");
994+ testapp_validate();
995+ if (!second_step)
996+ swap_xsk_res();
997+ second_step = true;
1025998 }
1026999
10271000 print_ksft_result();
@@ -1053,78 +1026,33 @@ static void testapp_stats(void) · 25 unmodified lines
10531026 print_ksft_result();
10541027 }
10551028
1056−static void init_iface_config(struct ifaceconfigobj *ifaceconfig)
1057−{
1058− /*Init interface0 */
1059− ifdict[0]->fv.vector = tx;
1060− memcpy(ifdict[0]->dst_mac, ifaceconfig->dst_mac, ETH_ALEN);
1061− memcpy(ifdict[0]->src_mac, ifaceconfig->src_mac, ETH_ALEN);
1062− ifdict[0]->dst_ip = ifaceconfig->dst_ip.s_addr;
1063− ifdict[0]->src_ip = ifaceconfig->src_ip.s_addr;
1064− ifdict[0]->dst_port = ifaceconfig->dst_port;
1065− ifdict[0]->src_port = ifaceconfig->src_port;
1066−
1067− /*Init interface1 */
1068− ifdict[1]->fv.vector = rx;
1069− memcpy(ifdict[1]->dst_mac, ifaceconfig->src_mac, ETH_ALEN);
1070− memcpy(ifdict[1]->src_mac, ifaceconfig->dst_mac, ETH_ALEN);
1071− ifdict[1]->dst_ip = ifaceconfig->src_ip.s_addr;
1072− ifdict[1]->src_ip = ifaceconfig->dst_ip.s_addr;
1073− ifdict[1]->dst_port = ifaceconfig->src_port;
1074− ifdict[1]->src_port = ifaceconfig->dst_port;
1075−}
1076−
1077−static void *nsdisablemodethread(void *args)
1029+static void init_iface(struct ifobject *ifobj, const char *dst_mac,
1030+ const char *src_mac, const char *dst_ip,
1031+ const char *src_ip, const u16 dst_port,
1032+ const u16 src_port, enum fvector vector)
10781033 {
1079− struct targs *targs = args;
1034+ struct in_addr ip;
10801035
1081− targs->retptr = false;
1036+ memcpy(ifobj->dst_mac, dst_mac, ETH_ALEN);
1037+ memcpy(ifobj->src_mac, src_mac, ETH_ALEN);
10821038
1083− if (switch_namespace(targs->idx)) {
1084− targs->retptr = bpf_set_link_xdp_fd(ifdict[targs->idx]->ifindex, -1, targs->flags);
1085− } else {
1086− targs->retptr = errno;
1087− print_verbose("Failed to switch namespace to %s\n", ifdict[targs->idx]->nsname);
1088− }
1089−
1090− pthread_exit(NULL);
1091−}
1039+ inet_aton(dst_ip, &ip);
1040+ ifobj->dst_ip = ip.s_addr;
10921041
1093−static void disable_xdp_mode(int mode)
1094−{
1095− int err = 0;
1096− __u32 flags = XDP_FLAGS_UPDATE_IF_NOEXIST | mode;
1097− char *mode_str = mode & XDP_FLAGS_SKB_MODE ? "skb" : "drv";
1042+ inet_aton(src_ip, &ip);
1043+ ifobj->src_ip = ip.s_addr;
10981044
1099− for (int i = 0; i < MAX_INTERFACES; i++) {
1100− if (strcmp(ifdict[i]->nsname, "")) {
1101− struct targs *targs;
1102−
1103− targs = malloc(sizeof(*targs));
1104− memset(targs, 0, sizeof(*targs));
1105− if (!targs)
1106− exit_with_error(errno);
1107−
1108− targs->idx = i;
1109− targs->flags = flags;
1110− if (pthread_create(&ns_thread, NULL, nsdisablemodethread, targs))
1111− exit_with_error(errno);
1112−
1113− pthread_join(ns_thread, NULL);
1114− err = targs->retptr;
1115− free(targs);
1116− } else {
1117− err = bpf_set_link_xdp_fd(ifdict[i]->ifindex, -1, flags);
1118− }
1119−
1120− if (err) {
1121− print_verbose("Failed to disable %s mode on interface %s\n",
1122− mode_str, ifdict[i]->ifname);
1123− exit_with_error(err);
1124− }
1045+ ifobj->dst_port = dst_port;
1046+ ifobj->src_port = src_port;
11251047
1126− print_verbose("Disabled %s mode for interface: %s\n", mode_str, ifdict[i]->ifname);
1127− configured_mode = mode & XDP_FLAGS_SKB_MODE ? TEST_MODE_DRV : TEST_MODE_SKB;
1048+ if (vector == tx) {
1049+ ifobj->fv.vector = tx;
1050+ ifobj->func_ptr = worker_testapp_validate_tx;
1051+ ifdict_tx = ifobj;
1052+ } else {
1053+ ifobj->fv.vector = rx;
1054+ ifobj->func_ptr = worker_testapp_validate_rx;
1055+ ifdict_rx = ifobj;
11281056 }
11291057 }
11301058
@@ -1135,72 +1063,70 @@ static void run_pkt_test(int mode, int type) · 4 unmodified lines
11351063 /* reset defaults after potential previous test */
11361064 xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;
11371065 pkt_counter = 0;
1138− switching_notify = 0;
1139− bidi_pass = 0;
1066+ second_step = 0;
11401067 prev_pkt = -1;
1141− ifdict[0]->fv.vector = tx;
1142− ifdict[1]->fv.vector = rx;
11431068 sigvar = 0;
11441069 stat_test_type = -1;
11451070 rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
11461071 frame_headroom = XSK_UMEM__DEFAULT_FRAME_HEADROOM;
11471072
1073+ configured_mode = mode;
1074+
11481075 switch (mode) {
11491076 case (TEST_MODE_SKB):
1150− if (configured_mode == TEST_MODE_DRV)
1151− disable_xdp_mode(XDP_FLAGS_DRV_MODE);
11521077 xdp_flags |= XDP_FLAGS_SKB_MODE;
11531078 break;
11541079 case (TEST_MODE_DRV):
1155− if (configured_mode == TEST_MODE_SKB)
1156− disable_xdp_mode(XDP_FLAGS_SKB_MODE);
11571080 xdp_flags |= XDP_FLAGS_DRV_MODE;
11581081 break;
11591082 default:
11601083 break;
11611084 }
11621085
1163− pthread_init_mutex();
1164−
1165− if (test_type == TEST_TYPE_STATS)
1086+ switch (test_type) {
1087+ case TEST_TYPE_STATS:
11661088 testapp_stats();
1167− else if ((test_type != TEST_TYPE_TEARDOWN) && (test_type != TEST_TYPE_BIDI))
1089+ break;
1090+ case TEST_TYPE_TEARDOWN:
1091+ testapp_teardown();
1092+ break;
1093+ case TEST_TYPE_BIDI:
1094+ testapp_bidi();
1095+ break;
1096+ case TEST_TYPE_BPF_RES:
1097+ testapp_bpf_res();
1098+ break;
1099+ default:
11681100 testapp_validate();
1169− else
1170− testapp_sockets();
1171−
1172− pthread_destroy_mutex();
1101+ break;
1102+ }
11731103 }
11741104
11751105 int main(int argc, char **argv)
11761106 {
11771107 struct rlimit _rlim = { RLIM_INFINITY, RLIM_INFINITY };
1108+ bool failure = false;
1109+ int i, j;
11781110
11791111 if (setrlimit(RLIMIT_MEMLOCK, &_rlim))
11801112 exit_with_error(errno);
11811113
1182− const char *MAC1 = "\x00\x0A\x56\x9E\xEE\x62";
1183− const char *MAC2 = "\x00\x0A\x56\x9E\xEE\x61";
1184− const char *IP1 = "192.168.100.162";
1185− const char *IP2 = "192.168.100.161";
1186− u16 UDP_DST_PORT = 2020;
1187− u16 UDP_SRC_PORT = 2121;
1188− int i, j;
1189−
1190− ifaceconfig = malloc(sizeof(struct ifaceconfigobj));
1191− memcpy(ifaceconfig->dst_mac, MAC1, ETH_ALEN);
1192− memcpy(ifaceconfig->src_mac, MAC2, ETH_ALEN);
1193− inet_aton(IP1, &ifaceconfig->dst_ip);
1194− inet_aton(IP2, &ifaceconfig->src_ip);
1195− ifaceconfig->dst_port = UDP_DST_PORT;
1196− ifaceconfig->src_port = UDP_SRC_PORT;
1197−
11981114 for (int i = 0; i < MAX_INTERFACES; i++) {
11991115 ifdict[i] = malloc(sizeof(struct ifobject));
12001116 if (!ifdict[i])
12011117 exit_with_error(errno);
12021118
12031119 ifdict[i]->ifdict_index = i;
1120+ ifdict[i]->xsk_arr = calloc(2, sizeof(struct xsk_socket_info *));
1121+ if (!ifdict[i]->xsk_arr) {
1122+ failure = true;
1123+ goto cleanup;
1124+ }
1125+ ifdict[i]->umem_arr = calloc(2, sizeof(struct xsk_umem_info *));
1126+ if (!ifdict[i]->umem_arr) {
1127+ failure = true;
1128+ goto cleanup;
1129+ }
12041130 }
12051131
12061132 setlocale(LC_ALL, "");
@@ -1209,9 +1135,8 @@ int main(int argc, char **argv) · 2 unmodified lines
12091135
12101136 num_frames = ++opt_pkt_count;
12111137
1212− init_iface_config(ifaceconfig);
1213−
1214− disable_xdp_mode(XDP_FLAGS_DRV_MODE);
1138+ init_iface(ifdict[0], MAC1, MAC2, IP1, IP2, UDP_PORT1, UDP_PORT2, tx);
1139+ init_iface(ifdict[1], MAC2, MAC1, IP2, IP1, UDP_PORT2, UDP_PORT1, rx);
12151140
12161141 ksft_set_plan(TEST_MODE_MAX * TEST_TYPE_MAX);
12171142
@@ -1220,8 +1145,17 @@ int main(int argc, char **argv) · 2 unmodified lines
12201145 run_pkt_test(i, j);
12211146 }
12221147
1223− for (int i = 0; i < MAX_INTERFACES; i++)
1148+cleanup:
1149+ for (int i = 0; i < MAX_INTERFACES; i++) {
1150+ if (ifdict[i]->ns_fd != -1)
1151+ close(ifdict[i]->ns_fd);
1152+ free(ifdict[i]->xsk_arr);
1153+ free(ifdict[i]->umem_arr);
12241154 free(ifdict[i]);
1155+ }
1156+
1157+ if (failure)
1158+ exit_with_error(errno);
12251159
12261160 ksft_exit_pass();
12271161
tools/testing/selftests/bpf/xdpxceiver.h+15−34
@@ -23,6 +23,7 @@ · 22 unmodified lines
2323 #define MAX_SOCKS 1
2424 #define MAX_TEARDOWN_ITER 10
2525 #define MAX_BIDI_ITER 2
26+#define MAX_BPF_ITER 2
2627 #define PKT_HDR_SIZE (sizeof(struct ethhdr) + sizeof(struct iphdr) + \
2728 sizeof(struct udphdr))
2829 #define MIN_PKT_SIZE 64
@@ -33,14 +34,11 @@ · 4 unmodified lines
3334 #define IP_PKT_TOS 0x9
3435 #define UDP_PKT_SIZE (IP_PKT_SIZE - sizeof(struct iphdr))
3536 #define UDP_PKT_DATA_SIZE (UDP_PKT_SIZE - sizeof(struct udphdr))
36−#define TMOUT_SEC (3)
3737 #define EOT (-1)
3838 #define USLEEP_MAX 200000
39−#define THREAD_STACK 60000000
4039 #define SOCK_RECONF_CTR 10
4140 #define BATCH_SIZE 64
4241 #define POLL_TMOUT 1000
43−#define NEED_WAKEUP true
4442 #define DEFAULT_PKT_CNT 10000
4543 #define RX_FULL_RXQSIZE 32
4644
@@ -63,6 +61,7 @@ enum TEST_TYPES { · 16 unmodified lines
6361 TEST_TYPE_TEARDOWN,
6462 TEST_TYPE_BIDI,
6563 TEST_TYPE_STATS,
64+ TEST_TYPE_BPF_RES,
6665 TEST_TYPE_MAX
6766 };
6867
@@ -77,11 +76,9 @@ enum STAT_TEST_TYPES { · 8 unmodified lines
7776 static int configured_mode = TEST_MODE_UNCONFIGURED;
7877 static u8 debug_pkt_dump;
7978 static u32 num_frames;
80−static u8 switching_notify;
81−static u8 bidi_pass;
79+static bool second_step;
8280 static int test_type;
8381
84−static int opt_queue;
8582 static int opt_pkt_count;
8683 static u8 opt_verbose;
8784
@@ -125,48 +122,32 @@ struct generic_data { · 37 unmodified lines
125122 u32 seqnum;
126123 };
127124
128−struct ifaceconfigobj {
129− u8 dst_mac[ETH_ALEN];
130− u8 src_mac[ETH_ALEN];
131− struct in_addr dst_ip;
132− struct in_addr src_ip;
133− u16 src_port;
134− u16 dst_port;
135−} *ifaceconfig;
136−
137125 struct ifobject {
138− int ifindex;
139− int ifdict_index;
140126 char ifname[MAX_INTERFACE_NAME_CHARS];
141127 char nsname[MAX_INTERFACES_NAMESPACE_CHARS];
142− struct flow_vector fv;
143128 struct xsk_socket_info *xsk;
129+ struct xsk_socket_info **xsk_arr;
130+ struct xsk_umem_info **umem_arr;
144131 struct xsk_umem_info *umem;
145− u8 dst_mac[ETH_ALEN];
146− u8 src_mac[ETH_ALEN];
132+ void *(*func_ptr)(void *arg);
133+ struct flow_vector fv;
134+ int ns_fd;
135+ int ifdict_index;
147136 u32 dst_ip;
148137 u32 src_ip;
149138 u16 src_port;
150139 u16 dst_port;
140+ u8 dst_mac[ETH_ALEN];
141+ u8 src_mac[ETH_ALEN];
151142 };
152143
153144 static struct ifobject *ifdict[MAX_INTERFACES];
145+static struct ifobject *ifdict_rx;
146+static struct ifobject *ifdict_tx;
154147
155148 /*threads*/
156−atomic_int spinning_tx;
157−atomic_int spinning_rx;
158−pthread_mutex_t sync_mutex;
159−pthread_mutex_t sync_mutex_tx;
160−pthread_cond_t signal_rx_condition;
161−pthread_cond_t signal_tx_condition;
162−pthread_t t0, t1, ns_thread;
163−pthread_attr_t attr;
164−
165−struct targs {
166− u8 retptr;
167− int idx;
168− u32 flags;
169−};
149+pthread_barrier_t barr;
150+pthread_t t0, t1;
170151
171152 TAILQ_HEAD(head_s, pkt) head = TAILQ_HEAD_INITIALIZER(head);
172153 struct head_s *head_p;