lael-labs/linux

Documentation/bpf/bpf_design_QA.rst+15−0

@@ -258,3 +258,18 @@ Q: Can BPF functionality such as new program or map types, new · 257 unmodified lines
258	helpers, etc be added out of kernel module code?	258	helpers, etc be added out of kernel module code?
259		259
260	A: NO.	260	A: NO.
		261
		262	Q: Directly calling kernel function is an ABI?
		263	----------------------------------------------
		264	Q: Some kernel functions (e.g. tcp_slow_start) can be called
		265	by BPF programs. Do these kernel functions become an ABI?
		266
		267	A: NO.
		268
		269	The kernel function protos will change and the bpf programs will be
		270	rejected by the verifier. Also, for example, some of the bpf-callable
		271	kernel functions have already been used by other kernel tcp
		272	cc (congestion-control) implementations. If any of these kernel
		273	functions has changed, both the in-tree and out-of-tree kernel tcp cc
		274	implementations have to be changed. The same goes for the bpf
		275	programs and they have to be adjusted accordingly.

arch/x86/net/bpf_jit_comp.c+5−0

@@ -2346,3 +2346,8 @@ out: · 2345 unmodified lines
2346	tmp : orig_prog);	2346	tmp : orig_prog);
2347	return prog;	2347	return prog;
2348	}	2348	}
		2349
		2350	bool bpf_jit_supports_kfunc_call(void)
		2351	{
		2352	return true;
		2353	}

arch/x86/net/bpf_jit_comp32.c+198−0

@@ -1390,6 +1390,19 @@ static inline void emit_push_r64(const u8 src[], u8 **pprog) · 1389 unmodified lines
1390	*pprog = prog;	1390	*pprog = prog;
1391	}	1391	}
1392		1392
		1393	static void emit_push_r32(const u8 src[], u8 **pprog)
		1394	{
		1395	u8 prog = pprog;
		1396	int cnt = 0;
		1397
		1398	/* mov ecx,dword ptr [ebp+off] */
		1399	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src_lo));
		1400	/* push ecx */
		1401	EMIT1(0x51);
		1402
		1403	*pprog = prog;
		1404	}
		1405
1393	static u8 get_cond_jmp_opcode(const u8 op, bool is_cmp_lo)	1406	static u8 get_cond_jmp_opcode(const u8 op, bool is_cmp_lo)
1394	{	1407	{
1395	u8 jmp_cond;	1408	u8 jmp_cond;
@@ -1459,6 +1472,174 @@ static u8 get_cond_jmp_opcode(const u8 op, bool is_cmp_lo) · 63 unmodified lines
1459	return jmp_cond;	1472	return jmp_cond;
1460	}	1473	}
1461		1474
		1475	/* i386 kernel compiles with "-mregparm=3". From gcc document:
		1476	*
		1477	* ==== snippet ====
		1478	* regparm (number)
		1479	* On x86-32 targets, the regparm attribute causes the compiler
		1480	* to pass arguments number one to (number) if they are of integral
		1481	* type in registers EAX, EDX, and ECX instead of on the stack.
		1482	* Functions that take a variable number of arguments continue
		1483	* to be passed all of their arguments on the stack.
		1484	* ==== snippet ====
		1485	*
		1486	* The first three args of a function will be considered for
		1487	* putting into the 32bit register EAX, EDX, and ECX.
		1488	*
		1489	* Two 32bit registers are used to pass a 64bit arg.
		1490	*
		1491	* For example,
		1492	* void foo(u32 a, u32 b, u32 c, u32 d):
		1493	* u32 a: EAX
		1494	* u32 b: EDX
		1495	* u32 c: ECX
		1496	* u32 d: stack
		1497	*
		1498	* void foo(u64 a, u32 b, u32 c):
		1499	* u64 a: EAX (lo32) EDX (hi32)
		1500	* u32 b: ECX
		1501	* u32 c: stack
		1502	*
		1503	* void foo(u32 a, u64 b, u32 c):
		1504	* u32 a: EAX
		1505	* u64 b: EDX (lo32) ECX (hi32)
		1506	* u32 c: stack
		1507	*
		1508	* void foo(u32 a, u32 b, u64 c):
		1509	* u32 a: EAX
		1510	* u32 b: EDX
		1511	* u64 c: stack
		1512	*
		1513	* The return value will be stored in the EAX (and EDX for 64bit value).
		1514	*
		1515	* For example,
		1516	* u32 foo(u32 a, u32 b, u32 c):
		1517	* return value: EAX
		1518	*
		1519	* u64 foo(u32 a, u32 b, u32 c):
		1520	* return value: EAX (lo32) EDX (hi32)
		1521	*
		1522	* Notes:
		1523	* The verifier only accepts function having integer and pointers
		1524	* as its args and return value, so it does not have
		1525	* struct-by-value.
		1526	*
		1527	* emit_kfunc_call() finds out the btf_func_model by calling
		1528	* bpf_jit_find_kfunc_model(). A btf_func_model
		1529	* has the details about the number of args, size of each arg,
		1530	* and the size of the return value.
		1531	*
		1532	* It first decides how many args can be passed by EAX, EDX, and ECX.
		1533	* That will decide what args should be pushed to the stack:
		1534	* [first_stack_regno, last_stack_regno] are the bpf regnos
		1535	* that should be pushed to the stack.
		1536	*
		1537	* It will first push all args to the stack because the push
		1538	* will need to use ECX. Then, it moves
		1539	* [BPF_REG_1, first_stack_regno) to EAX, EDX, and ECX.
		1540	*
		1541	* When emitting a call (0xE8), it needs to figure out
		1542	* the jmp_offset relative to the jit-insn address immediately
		1543	* following the call (0xE8) instruction. At this point, it knows
		1544	* the end of the jit-insn address after completely translated the
		1545	* current (BPF_JMP \| BPF_CALL) bpf-insn. It is passed as "end_addr"
		1546	* to the emit_kfunc_call(). Thus, it can learn the "immediate-follow-call"
		1547	* address by figuring out how many jit-insn is generated between
		1548	* the call (0xE8) and the end_addr:
		1549	* - 0-1 jit-insn (3 bytes each) to restore the esp pointer if there
		1550	* is arg pushed to the stack.
		1551	* - 0-2 jit-insns (3 bytes each) to handle the return value.
		1552	*/
		1553	static int emit_kfunc_call(const struct bpf_prog bpf_prog, u8 end_addr,
		1554	const struct bpf_insn insn, u8 *pprog)
		1555	{
		1556	const u8 arg_regs[] = { IA32_EAX, IA32_EDX, IA32_ECX };
		1557	int i, cnt = 0, first_stack_regno, last_stack_regno;
		1558	int free_arg_regs = ARRAY_SIZE(arg_regs);
		1559	const struct btf_func_model *fm;
		1560	int bytes_in_stack = 0;
		1561	const u8 *cur_arg_reg;
		1562	u8 prog = pprog;
		1563	s64 jmp_offset;
		1564
		1565	fm = bpf_jit_find_kfunc_model(bpf_prog, insn);
		1566	if (!fm)
		1567	return -EINVAL;
		1568
		1569	first_stack_regno = BPF_REG_1;
		1570	for (i = 0; i < fm->nr_args; i++) {
		1571	int regs_needed = fm->arg_size[i] > sizeof(u32) ? 2 : 1;
		1572
		1573	if (regs_needed > free_arg_regs)
		1574	break;
		1575
		1576	free_arg_regs -= regs_needed;
		1577	first_stack_regno++;
		1578	}
		1579
		1580	/* Push the args to the stack */
		1581	last_stack_regno = BPF_REG_0 + fm->nr_args;
		1582	for (i = last_stack_regno; i >= first_stack_regno; i--) {
		1583	if (fm->arg_size[i - 1] > sizeof(u32)) {
		1584	emit_push_r64(bpf2ia32[i], &prog);
		1585	bytes_in_stack += 8;
		1586	} else {
		1587	emit_push_r32(bpf2ia32[i], &prog);
		1588	bytes_in_stack += 4;
		1589	}
		1590	}
		1591
		1592	cur_arg_reg = &arg_regs[0];
		1593	for (i = BPF_REG_1; i < first_stack_regno; i++) {
		1594	/* mov e[adc]x,dword ptr [ebp+off] */
		1595	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, *cur_arg_reg++),
		1596	STACK_VAR(bpf2ia32[i][0]));
		1597	if (fm->arg_size[i - 1] > sizeof(u32))
		1598	/* mov e[adc]x,dword ptr [ebp+off] */
		1599	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, *cur_arg_reg++),
		1600	STACK_VAR(bpf2ia32[i][1]));
		1601	}
		1602
		1603	if (bytes_in_stack)
		1604	/* add esp,"bytes_in_stack" */
		1605	end_addr -= 3;
		1606
		1607	/* mov dword ptr [ebp+off],edx */
		1608	if (fm->ret_size > sizeof(u32))
		1609	end_addr -= 3;
		1610
		1611	/* mov dword ptr [ebp+off],eax */
		1612	if (fm->ret_size)
		1613	end_addr -= 3;
		1614
		1615	jmp_offset = (u8 *)__bpf_call_base + insn->imm - end_addr;
		1616	if (!is_simm32(jmp_offset)) {
		1617	pr_err("unsupported BPF kernel function jmp_offset:%lld\n",
		1618	jmp_offset);
		1619	return -EINVAL;
		1620	}
		1621
		1622	EMIT1_off32(0xE8, jmp_offset);
		1623
		1624	if (fm->ret_size)
		1625	/* mov dword ptr [ebp+off],eax */
		1626	EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX),
		1627	STACK_VAR(bpf2ia32[BPF_REG_0][0]));
		1628
		1629	if (fm->ret_size > sizeof(u32))
		1630	/* mov dword ptr [ebp+off],edx */
		1631	EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX),
		1632	STACK_VAR(bpf2ia32[BPF_REG_0][1]));
		1633
		1634	if (bytes_in_stack)
		1635	/* add esp,"bytes_in_stack" */
		1636	EMIT3(0x83, add_1reg(0xC0, IA32_ESP), bytes_in_stack);
		1637
		1638	*pprog = prog;
		1639
		1640	return 0;
		1641	}
		1642
1462	static int do_jit(struct bpf_prog bpf_prog, int addrs, u8 *image,	1643	static int do_jit(struct bpf_prog bpf_prog, int addrs, u8 *image,
1463	int oldproglen, struct jit_context *ctx)	1644	int oldproglen, struct jit_context *ctx)
1464	{	1645	{
@@ -1888,6 +2069,18 @@ static int do_jit(struct bpf_prog bpf_prog, int addrs, u8 *image, · 423 unmodified lines
1888	if (insn->src_reg == BPF_PSEUDO_CALL)	2069	if (insn->src_reg == BPF_PSEUDO_CALL)
1889	goto notyet;	2070	goto notyet;
1890		2071
		2072	if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
		2073	int err;
		2074
		2075	err = emit_kfunc_call(bpf_prog,
		2076	image + addrs[i],
		2077	insn, &prog);
		2078
		2079	if (err)
		2080	return err;
		2081	break;
		2082	}
		2083
1891	func = (u8 *) __bpf_call_base + imm32;	2084	func = (u8 *) __bpf_call_base + imm32;
1892	jmp_offset = func - (image + addrs[i]);	2085	jmp_offset = func - (image + addrs[i]);
1893		2086
@@ -2393,3 +2586,8 @@ out: · 499 unmodified lines
2393	tmp : orig_prog);	2586	tmp : orig_prog);
2394	return prog;	2587	return prog;
2395	}	2588	}
		2589
		2590	bool bpf_jit_supports_kfunc_call(void)
		2591	{
		2592	return true;
		2593	}

drivers/net/veth.c+12−0

 	}
 }
+static void veth_get_channels(struct net_device *dev,
+			      struct ethtool_channels *channels)
+{
+	channels->tx_count = dev->real_num_tx_queues;
+	channels->rx_count = dev->real_num_rx_queues;
+	channels->max_tx = dev->real_num_tx_queues;
+	channels->max_rx = dev->real_num_rx_queues;
+	channels->combined_count = min(dev->real_num_rx_queues, dev->real_num_tx_queues);
+	channels->max_combined = min(dev->real_num_rx_queues, dev->real_num_tx_queues);
+}
 static const struct ethtool_ops veth_ethtool_ops = {
 	.get_drvinfo		= veth_get_drvinfo,
 	.get_link		= ethtool_op_get_link,
 	.get_ethtool_stats	= veth_get_ethtool_stats,
 	.get_link_ksettings	= veth_get_link_ksettings,
 	.get_ts_info		= ethtool_op_get_ts_info,
+	.get_channels		= veth_get_channels,
 };
 /* general routines */

include/linux/bpf-cgroup.h+49−8

@@ -20,14 +20,25 @@ struct bpf_sock_ops_kern; · 19 unmodified lines
20	struct bpf_cgroup_storage;	20	struct bpf_cgroup_storage;
21	struct ctl_table;	21	struct ctl_table;
22	struct ctl_table_header;	22	struct ctl_table_header;
		23	struct task_struct;
23		24
24	#ifdef CONFIG_CGROUP_BPF	25	#ifdef CONFIG_CGROUP_BPF
25		26
26	extern struct static_key_false cgroup_bpf_enabled_key[MAX_BPF_ATTACH_TYPE];	27	extern struct static_key_false cgroup_bpf_enabled_key[MAX_BPF_ATTACH_TYPE];
27	#define cgroup_bpf_enabled(type) static_branch_unlikely(&cgroup_bpf_enabled_key[type])	28	#define cgroup_bpf_enabled(type) static_branch_unlikely(&cgroup_bpf_enabled_key[type])
28		29
29	DECLARE_PER_CPU(struct bpf_cgroup_storage*,	30	#define BPF_CGROUP_STORAGE_NEST_MAX 8
30	bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);	31
		32	struct bpf_cgroup_storage_info {
		33	struct task_struct *task;
		34	struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE];
		35	};
		36
		37	/* For each cpu, permit maximum BPF_CGROUP_STORAGE_NEST_MAX number of tasks
		38	* to use bpf cgroup storage simultaneously.
		39	*/
		40	DECLARE_PER_CPU(struct bpf_cgroup_storage_info,
		41	bpf_cgroup_storage_info[BPF_CGROUP_STORAGE_NEST_MAX]);
31		42
32	#define for_each_cgroup_storage_type(stype) \	43	#define for_each_cgroup_storage_type(stype) \
33	for (stype = 0; stype < MAX_BPF_CGROUP_STORAGE_TYPE; stype++)	44	for (stype = 0; stype < MAX_BPF_CGROUP_STORAGE_TYPE; stype++)
@@ -161,13 +172,42 @@ static inline enum bpf_cgroup_storage_type cgroup_storage_type( · 127 unmodified lines
161	return BPF_CGROUP_STORAGE_SHARED;	172	return BPF_CGROUP_STORAGE_SHARED;
162	}	173	}
163		174
164	static inline void bpf_cgroup_storage_set(struct bpf_cgroup_storage	175	static inline int bpf_cgroup_storage_set(struct bpf_cgroup_storage
165	*storage[MAX_BPF_CGROUP_STORAGE_TYPE])	176	*storage[MAX_BPF_CGROUP_STORAGE_TYPE])
166	{	177	{
167	enum bpf_cgroup_storage_type stype;	178	enum bpf_cgroup_storage_type stype;
		179	int i, err = 0;
		180
		181	preempt_disable();
		182	for (i = 0; i < BPF_CGROUP_STORAGE_NEST_MAX; i++) {
		183	if (unlikely(this_cpu_read(bpf_cgroup_storage_info[i].task) != NULL))
		184	continue;
		185
		186	this_cpu_write(bpf_cgroup_storage_info[i].task, current);
		187	for_each_cgroup_storage_type(stype)
		188	this_cpu_write(bpf_cgroup_storage_info[i].storage[stype],
		189	storage[stype]);
		190	goto out;
		191	}
		192	err = -EBUSY;
		193	WARN_ON_ONCE(1);
		194
		195	out:
		196	preempt_enable();
		197	return err;
		198	}
		199
		200	static inline void bpf_cgroup_storage_unset(void)
		201	{
		202	int i;
		203
		204	for (i = 0; i < BPF_CGROUP_STORAGE_NEST_MAX; i++) {
		205	if (unlikely(this_cpu_read(bpf_cgroup_storage_info[i].task) != current))
		206	continue;
168		207
169	for_each_cgroup_storage_type(stype)	208	this_cpu_write(bpf_cgroup_storage_info[i].task, NULL);
170	this_cpu_write(bpf_cgroup_storage[stype], storage[stype]);	209	return;
		210	}
171	}	211	}
172		212
173	struct bpf_cgroup_storage *	213	struct bpf_cgroup_storage *
@@ -448,8 +488,9 @@ static inline int cgroup_bpf_prog_query(const union bpf_attr *attr, · 274 unmodified lines
448	return -EINVAL;	488	return -EINVAL;
449	}	489	}
450		490
451	static inline void bpf_cgroup_storage_set(	491	static inline int bpf_cgroup_storage_set(
452	struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE]) {}	492	struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE]) { return 0; }
		493	static inline void bpf_cgroup_storage_unset(void) {}
453	static inline int bpf_cgroup_storage_assign(struct bpf_prog_aux *aux,	494	static inline int bpf_cgroup_storage_assign(struct bpf_prog_aux *aux,
454	struct bpf_map *map) { return 0; }	495	struct bpf_map *map) { return 0; }
455	static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(	496	static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(

include/linux/bpf.h+51−7

 	u32 seq_priv_size;
 };
-/* map is generic key/value storage optionally accesible by eBPF programs */
+/* map is generic key/value storage optionally accessible by eBPF programs */
 struct bpf_map_ops {
 	/* funcs callable from userspace (via syscall) */
 	int (*map_alloc_check)(union bpf_attr *attr);
 	PTR_TO_PERCPU_BTF_ID,	 /* reg points to a percpu kernel variable */
 	PTR_TO_FUNC,		 /* reg points to a bpf program function */
 	PTR_TO_MAP_KEY,		 /* reg points to a map element key */
+	__BPF_REG_TYPE_MAX,
 };
 /* The information passed from prog-specific *_is_valid_access
 				 const struct btf_type *t, int off, int size,
 				 enum bpf_access_type atype,
 				 u32 *next_btf_id);
+	bool (*check_kfunc_call)(u32 kfunc_btf_id);
 };
 struct bpf_prog_offload_ops {
 	struct module *module;
 };
+struct bpf_kfunc_desc_tab;
 struct bpf_prog_aux {
 	atomic64_t refcnt;
 	u32 used_map_cnt;
 	struct bpf_prog **func;
 	void *jit_data; /* JIT specific data. arch dependent */
 	struct bpf_jit_poke_descriptor *poke_tab;
+	struct bpf_kfunc_desc_tab *kfunc_tab;
 	u32 size_poke_tab;
 	struct bpf_ksym ksym;
 	const struct bpf_prog_ops *ops;
 /* BPF program asks to set CN on the packet. */
 #define BPF_RET_SET_CN						(1 << 0)
+/* For BPF_PROG_RUN_ARRAY_FLAGS and __BPF_PROG_RUN_ARRAY,
+ * if bpf_cgroup_storage_set() failed, the rest of programs
+ * will not execute. This should be a really rare scenario
+ * as it requires BPF_CGROUP_STORAGE_NEST_MAX number of
+ * preemptions all between bpf_cgroup_storage_set() and
+ * bpf_cgroup_storage_unset() on the same cpu.
+ */
 #define BPF_PROG_RUN_ARRAY_FLAGS(array, ctx, func, ret_flags)		\
 	({								\
 		struct bpf_prog_array_item *_item;			\
 		_array = rcu_dereference(array);			\
 		_item = &_array->items[0];				\
 		while ((_prog = READ_ONCE(_item->prog))) {		\
-			bpf_cgroup_storage_set(_item->cgroup_storage);	\
+			if (unlikely(bpf_cgroup_storage_set(_item->cgroup_storage)))	\
+				break;					\
 			func_ret = func(_prog, ctx);			\
 			_ret &= (func_ret & 1);				\
 			*(ret_flags) |= (func_ret >> 1);			\
+			bpf_cgroup_storage_unset();			\
 			_item++;					\
 		}							\
 		rcu_read_unlock();					\
 			goto _out;			\
 		_item = &_array->items[0];		\
 		while ((_prog = READ_ONCE(_item->prog))) {		\
-			if (set_cg_storage)		\
+			if (!set_cg_storage) {			\
-				bpf_cgroup_storage_set(_item->cgroup_storage);	\
+				_ret &= func(_prog, ctx);	\
-			_ret &= func(_prog, ctx);	\
+			} else {				\
+				if (unlikely(bpf_cgroup_storage_set(_item->cgroup_storage)))	\
+					break;			\
+				_ret &= func(_prog, ctx);	\
+				bpf_cgroup_storage_unset();	\
+			}				\
 			_item++;			\
 		}					\
 _out:							\
 int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog,
 				const union bpf_attr *kattr,
 				union bpf_attr __user *uattr);
+bool bpf_prog_test_check_kfunc_call(u32 kfunc_id);
 bool btf_ctx_access(int off, int size, enum bpf_access_type type,
 		    const struct bpf_prog *prog,
 		    struct bpf_insn_access_aux *info);
 			   struct btf_func_model *m);
 struct bpf_reg_state;
-int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
+int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog,
-			     struct bpf_reg_state *regs);
+				struct bpf_reg_state *regs);
+int btf_check_kfunc_arg_match(struct bpf_verifier_env *env,
+			      const struct btf *btf, u32 func_id,
+			      struct bpf_reg_state *regs);
 int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
 			  struct bpf_reg_state *reg);
 int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog,
 const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id);
 void bpf_task_storage_free(struct task_struct *task);
+bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog);
+const struct btf_func_model *
+bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
+			 const struct bpf_insn *insn);
 #else /* !CONFIG_BPF_SYSCALL */
 static inline struct bpf_prog *bpf_prog_get(u32 ufd)
 {
 	return -ENOTSUPP;
 }
+static inline bool bpf_prog_test_check_kfunc_call(u32 kfunc_id)
+{
+	return false;
+}
 static inline void bpf_map_put(struct bpf_map *map)
 {
 }
 static inline void bpf_task_storage_free(struct task_struct *task)
 {
 }
+static inline bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
+{
+	return false;
+}
+static inline const struct btf_func_model *
+bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
+			 const struct bpf_insn *insn)
+{
+	return NULL;
+}
 #endif /* CONFIG_BPF_SYSCALL */
 void __bpf_free_used_btfs(struct bpf_prog_aux *aux,

include/linux/btf.h+6−0

 const struct btf_type *
 btf_resolve_size(const struct btf *btf, const struct btf_type *type,
 		 u32 *type_size);
+const char *btf_type_str(const struct btf_type *t);
 #define for_each_member(i, struct_type, member)			\
 	for (i = 0, member = btf_type_member(struct_type);	\
 	return BTF_INFO_KIND(t->info) == BTF_KIND_ENUM;
 }
+static inline bool btf_type_is_scalar(const struct btf_type *t)
+{
+	return btf_type_is_int(t) || btf_type_is_enum(t);
+}
 static inline bool btf_type_is_typedef(const struct btf_type *t)
 {
 	return BTF_INFO_KIND(t->info) == BTF_KIND_TYPEDEF;

include/linux/filter.h+2−11

 void bpf_prog_fill_jited_linfo(struct bpf_prog *prog,
 			       const u32 *insn_to_jit_off);
 int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog);
-void bpf_prog_free_jited_linfo(struct bpf_prog *prog);
+void bpf_prog_jit_attempt_done(struct bpf_prog *prog);
-void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog);
 struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags);
 struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flags);
 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog);
 void bpf_jit_compile(struct bpf_prog *prog);
 bool bpf_jit_needs_zext(void);
+bool bpf_jit_supports_kfunc_call(void);
 bool bpf_helper_changes_pkt_data(void *func);
 static inline bool bpf_dump_raw_ok(const struct cred *cred)
 void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb,
 					   int k, unsigned int size);
-static inline void *bpf_load_pointer(const struct sk_buff *skb, int k,
-				     unsigned int size, void *buffer)
-{
-	if (k >= 0)
-		return skb_header_pointer(skb, k, size, buffer);
-	return bpf_internal_load_pointer_neg_helper(skb, k, size);
-}
 static inline int bpf_tell_extensions(void)
 {
 	return SKF_AD_MAX;

include/linux/skbuff.h+1−0

@@ -3626,6 +3626,7 @@ int skb_splice_bits(struct sk_buff skb, struct sock sk, unsigned int offset, · 3625 unmodified lines
3626	unsigned int flags);	3626	unsigned int flags);
3627	int skb_send_sock_locked(struct sock sk, struct sk_buff skb, int offset,	3627	int skb_send_sock_locked(struct sock sk, struct sk_buff skb, int offset,
3628	int len);	3628	int len);
		3629	int skb_send_sock(struct sock sk, struct sk_buff skb, int offset, int len);
3629	void skb_copy_and_csum_dev(const struct sk_buff skb, u8 to);	3630	void skb_copy_and_csum_dev(const struct sk_buff skb, u8 to);
3630	unsigned int skb_zerocopy_headlen(const struct sk_buff *from);	3631	unsigned int skb_zerocopy_headlen(const struct sk_buff *from);
3631	int skb_zerocopy(struct sk_buff to, struct sk_buff from,	3632	int skb_zerocopy(struct sk_buff to, struct sk_buff from,

include/linux/skmsg.h+58−19

 	struct bpf_prog			*msg_parser;
 	struct bpf_prog			*stream_parser;
 	struct bpf_prog			*stream_verdict;
+	struct bpf_prog			*skb_verdict;
 };
 enum sk_psock_state_bits {
 #endif
 	struct sk_buff_head		ingress_skb;
 	struct list_head		ingress_msg;
+	spinlock_t			ingress_lock;
 	unsigned long			state;
 	struct list_head		link;
 	spinlock_t			link_lock;
 	void (*saved_close)(struct sock *sk, long timeout);
 	void (*saved_write_space)(struct sock *sk);
 	void (*saved_data_ready)(struct sock *sk);
+	int  (*psock_update_sk_prot)(struct sock *sk, bool restore);
 	struct proto			*sk_proto;
+	struct mutex			work_mutex;
 	struct sk_psock_work_state	work_state;
 	struct work_struct		work;
-	union {
+	struct rcu_work			rwork;
-		struct rcu_head		rcu;
-		struct work_struct	gc;
-	};
 };
 int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,
 			      struct sk_msg *msg, u32 bytes);
 int sk_msg_memcopy_from_iter(struct sock *sk, struct iov_iter *from,
 			     struct sk_msg *msg, u32 bytes);
+int sk_msg_wait_data(struct sock *sk, struct sk_psock *psock, int flags,
+		     long timeo, int *err);
+int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,
+		   int len, int flags);
 static inline void sk_msg_check_to_free(struct sk_msg *msg, u32 i, u32 bytes)
 {
 static inline void sk_psock_queue_msg(struct sk_psock *psock,
 				      struct sk_msg *msg)
 {
+	spin_lock_bh(&psock->ingress_lock);
 	list_add_tail(&msg->list, &psock->ingress_msg);
+	spin_unlock_bh(&psock->ingress_lock);
+}
+static inline struct sk_msg *sk_psock_dequeue_msg(struct sk_psock *psock)
+{
+	struct sk_msg *msg;
+	spin_lock_bh(&psock->ingress_lock);
+	msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list);
+	if (msg)
+		list_del(&msg->list);
+	spin_unlock_bh(&psock->ingress_lock);
+	return msg;
+}
+static inline struct sk_msg *sk_psock_peek_msg(struct sk_psock *psock)
+{
+	struct sk_msg *msg;
+	spin_lock_bh(&psock->ingress_lock);
+	msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list);
+	spin_unlock_bh(&psock->ingress_lock);
+	return msg;
+}
+static inline struct sk_msg *sk_psock_next_msg(struct sk_psock *psock,
+					       struct sk_msg *msg)
+{
+	struct sk_msg *ret;
+	spin_lock_bh(&psock->ingress_lock);
+	if (list_is_last(&msg->list, &psock->ingress_msg))
+		ret = NULL;
+	else
+		ret = list_next_entry(msg, list);
+	spin_unlock_bh(&psock->ingress_lock);
+	return ret;
 }
 static inline bool sk_psock_queue_empty(const struct sk_psock *psock)
 	return psock ? list_empty(&psock->ingress_msg) : true;
 }
+static inline void kfree_sk_msg(struct sk_msg *msg)
+{
+	if (msg->skb)
+		consume_skb(msg->skb);
+	kfree(msg);
+}
 static inline void sk_psock_report_error(struct sk_psock *psock, int err)
 {
 	struct sock *sk = psock->sk;
 }
 struct sk_psock *sk_psock_init(struct sock *sk, int node);
+void sk_psock_stop(struct sk_psock *psock, bool wait);
 #if IS_ENABLED(CONFIG_BPF_STREAM_PARSER)
 int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock);
 	}
 }
-static inline void sk_psock_update_proto(struct sock *sk,
-					 struct sk_psock *psock,
-					 struct proto *ops)
-{
-	/* Pairs with lockless read in sk_clone_lock() */
-	WRITE_ONCE(sk->sk_prot, ops);
-}
 static inline void sk_psock_restore_proto(struct sock *sk,
 					  struct sk_psock *psock)
 {
 	sk->sk_prot->unhash = psock->saved_unhash;
-	if (inet_csk_has_ulp(sk)) {
+	if (psock->psock_update_sk_prot)
-		tcp_update_ulp(sk, psock->sk_proto, psock->saved_write_space);
+		psock->psock_update_sk_prot(sk, true);
-	} else {
-		sk->sk_write_space = psock->saved_write_space;
-		/* Pairs with lockless read in sk_clone_lock() */
-		WRITE_ONCE(sk->sk_prot, psock->sk_proto);
-	}
 }
 static inline void sk_psock_set_state(struct sk_psock *psock,
 	psock_set_prog(&progs->msg_parser, NULL);
 	psock_set_prog(&progs->stream_parser, NULL);
 	psock_set_prog(&progs->stream_verdict, NULL);
+	psock_set_prog(&progs->skb_verdict, NULL);
 }
 int sk_psock_tls_strp_read(struct sk_psock *psock, struct sk_buff *skb);

include/net/bpf_sk_storage.h+0−1

@@ -27,7 +27,6 @@ struct bpf_local_storage_elem; · 26 unmodified lines
27	struct bpf_sk_storage_diag;	27	struct bpf_sk_storage_diag;
28	struct sk_buff;	28	struct sk_buff;
29	struct nlattr;	29	struct nlattr;
30	struct sock;
31		30
32	#ifdef CONFIG_BPF_SYSCALL	31	#ifdef CONFIG_BPF_SYSCALL
33	int bpf_sk_storage_clone(const struct sock sk, struct sock newsk);	32	int bpf_sk_storage_clone(const struct sock sk, struct sock newsk);

include/net/sock.h+3−0

 	void			(*unhash)(struct sock *sk);
 	void			(*rehash)(struct sock *sk);
 	int			(*get_port)(struct sock *sk, unsigned short snum);
+#ifdef CONFIG_BPF_SYSCALL
+	int			(*psock_update_sk_prot)(struct sock *sk, bool restore);
+#endif
 	/* Keeping track of sockets in use */
 #ifdef CONFIG_PROC_FS

include/net/tcp.h+1−2

 #ifdef CONFIG_BPF_SYSCALL
 struct proto *tcp_bpf_get_proto(struct sock *sk, struct sk_psock *psock);
+int tcp_bpf_update_proto(struct sock *sk, bool restore);
 void tcp_bpf_clone(const struct sock *sk, struct sock *newsk);
 #endif /* CONFIG_BPF_SYSCALL */
 int tcp_bpf_sendmsg_redir(struct sock *sk, struct sk_msg *msg, u32 bytes,
 			  int flags);
-int __tcp_bpf_recvmsg(struct sock *sk, struct sk_psock *psock,
-		      struct msghdr *msg, int len, int flags);
 #endif /* CONFIG_NET_SOCK_MSG */
 #if !defined(CONFIG_BPF_SYSCALL) || !defined(CONFIG_NET_SOCK_MSG)

include/net/udp.h+3−0

 			       struct sk_buff *skb);
 struct sock *udp6_lib_lookup_skb(const struct sk_buff *skb,
 				 __be16 sport, __be16 dport);
+int udp_read_sock(struct sock *sk, read_descriptor_t *desc,
+		  sk_read_actor_t recv_actor);
 /* UDP uses skb->dev_scratch to cache as much information as possible and avoid
  * possibly multiple cache miss on dequeue()
 #ifdef CONFIG_BPF_SYSCALL
 struct sk_psock;
 struct proto *udp_bpf_get_proto(struct sock *sk, struct sk_psock *psock);
+int udp_bpf_update_proto(struct sock *sk, bool restore);
 #endif
 #endif	/* _UDP_H */

include/uapi/linux/bpf.h+5−0

 	BPF_XDP_CPUMAP,
 	BPF_SK_LOOKUP,
 	BPF_XDP,
+	BPF_SK_SKB_VERDICT,
 	__MAX_BPF_ATTACH_TYPE
 };
  * offset to another bpf function
  */
 #define BPF_PSEUDO_CALL		1
+/* when bpf_call->src_reg == BPF_PSEUDO_KFUNC_CALL,
+ * bpf_call->imm == btf_id of a BTF_KIND_FUNC in the running kernel
+ */
+#define BPF_PSEUDO_KFUNC_CALL	2
 /* flags for BPF_MAP_UPDATE_ELEM command */
 enum {

kernel/bpf/btf.c+143−76

@@ -283,7 +283,7 @@ static const char * const btf_kind_str[NR_BTF_KINDS] = { · 282 unmodified lines
283	[BTF_KIND_FLOAT] = "FLOAT",	283	[BTF_KIND_FLOAT] = "FLOAT",
284	};	284	};
285		285
286	static const char btf_type_str(const struct btf_type t)	286	const char btf_type_str(const struct btf_type t)
287	{	287	{
288	return btf_kind_str[BTF_INFO_KIND(t->info)];	288	return btf_kind_str[BTF_INFO_KIND(t->info)];
289	}	289	}
@@ -789,7 +789,6 @@ static const struct btf_type btf_type_skip_qualifiers(const struct btf btf, · 499 unmodified lines
789		789
790	while (btf_type_is_modifier(t) &&	790	while (btf_type_is_modifier(t) &&
791	BTF_INFO_KIND(t->info) != BTF_KIND_TYPEDEF) {	791	BTF_INFO_KIND(t->info) != BTF_KIND_TYPEDEF) {
792	id = t->type;
793	t = btf_type_by_id(btf, t->type);	792	t = btf_type_by_id(btf, t->type);
794	}	793	}
795		794
@@ -4377,7 +4376,7 @@ static u8 bpf_ctx_convert_map[] = { · 3581 unmodified lines
4377	#undef BPF_LINK_TYPE	4376	#undef BPF_LINK_TYPE
4378		4377
4379	static const struct btf_member *	4378	static const struct btf_member *
4380	btf_get_prog_ctx_type(struct bpf_verifier_log log, struct btf btf,	4379	btf_get_prog_ctx_type(struct bpf_verifier_log log, const struct btf btf,
4381	const struct btf_type *t, enum bpf_prog_type prog_type,	4380	const struct btf_type *t, enum bpf_prog_type prog_type,
4382	int arg)	4381	int arg)
4383	{	4382	{
@@ -5362,122 +5361,190 @@ int btf_check_type_match(struct bpf_verifier_log log, const struct bpf_prog pr · 978 unmodified lines
5362	return btf_check_func_type_match(log, btf1, t1, btf2, t2);	5361	return btf_check_func_type_match(log, btf1, t1, btf2, t2);
5363	}	5362	}
5364		5363
5365	/* Compare BTF of a function with given bpf_reg_state.	5364	static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = {
5366	* Returns:	5365	#ifdef CONFIG_NET
5367	* EFAULT - there is a verifier bug. Abort verification.	5366	[PTR_TO_SOCKET] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK],
5368	* EINVAL - there is a type mismatch or BTF is not available.	5367	[PTR_TO_SOCK_COMMON] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
5369	* 0 - BTF matches with what bpf_reg_state expects.	5368	[PTR_TO_TCP_SOCK] = &btf_sock_ids[BTF_SOCK_TYPE_TCP],
5370	* Only PTR_TO_CTX and SCALAR_VALUE states are recognized.	5369	#endif
5371	*/	5370	};
5372	int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,	5371
5373	struct bpf_reg_state *regs)	5372	static int btf_check_func_arg_match(struct bpf_verifier_env *env,
		5373	const struct btf *btf, u32 func_id,
		5374	struct bpf_reg_state *regs,
		5375	bool ptr_to_mem_ok)
5374	{	5376	{
5375	struct bpf_verifier_log *log = &env->log;	5377	struct bpf_verifier_log *log = &env->log;
5376	struct bpf_prog *prog = env->prog;	5378	const char func_name, ref_tname;
5377	struct btf *btf = prog->aux->btf;
5378	const struct btf_param *args;
5379	const struct btf_type t, ref_t;	5379	const struct btf_type t, ref_t;
5380	u32 i, nargs, btf_id, type_size;	5380	const struct btf_param *args;
5381	const char *tname;	5381	u32 i, nargs, ref_id;
5382	bool is_global;
5383
5384	if (!prog->aux->func_info)
5385	return -EINVAL;
5386
5387	btf_id = prog->aux->func_info[subprog].type_id;
5388	if (!btf_id)
5389	return -EFAULT;
5390
5391	if (prog->aux->func_info_aux[subprog].unreliable)
5392	return -EINVAL;
5393		5382
5394	t = btf_type_by_id(btf, btf_id);	5383	t = btf_type_by_id(btf, func_id);
5395	if (!t \|\| !btf_type_is_func(t)) {	5384	if (!t \|\| !btf_type_is_func(t)) {
5396	/* These checks were already done by the verifier while loading	5385	/* These checks were already done by the verifier while loading
5397	* struct bpf_func_info	5386	* struct bpf_func_info or in add_kfunc_call().
5398	*/	5387	*/
5399	bpf_log(log, "BTF of func#%d doesn't point to KIND_FUNC\n",	5388	bpf_log(log, "BTF of func_id %u doesn't point to KIND_FUNC\n",
5400	subprog);	5389	func_id);
5401	return -EFAULT;	5390	return -EFAULT;
5402	}	5391	}
5403	tname = btf_name_by_offset(btf, t->name_off);	5392	func_name = btf_name_by_offset(btf, t->name_off);
5404		5393
5405	t = btf_type_by_id(btf, t->type);	5394	t = btf_type_by_id(btf, t->type);
5406	if (!t \|\| !btf_type_is_func_proto(t)) {	5395	if (!t \|\| !btf_type_is_func_proto(t)) {
5407	bpf_log(log, "Invalid BTF of func %s\n", tname);	5396	bpf_log(log, "Invalid BTF of func %s\n", func_name);
5408	return -EFAULT;	5397	return -EFAULT;
5409	}	5398	}
5410	args = (const struct btf_param *)(t + 1);	5399	args = (const struct btf_param *)(t + 1);
5411	nargs = btf_type_vlen(t);	5400	nargs = btf_type_vlen(t);
5412	if (nargs > MAX_BPF_FUNC_REG_ARGS) {	5401	if (nargs > MAX_BPF_FUNC_REG_ARGS) {
5413	bpf_log(log, "Function %s has %d > %d args\n", tname, nargs,	5402	bpf_log(log, "Function %s has %d > %d args\n", func_name, nargs,
5414	MAX_BPF_FUNC_REG_ARGS);	5403	MAX_BPF_FUNC_REG_ARGS);
5415	goto out;	5404	return -EINVAL;
5416	}	5405	}
5417		5406
5418	is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
5419	/* check that BTF function arguments match actual types that the	5407	/* check that BTF function arguments match actual types that the
5420	* verifier sees.	5408	* verifier sees.
5421	*/	5409	*/
5422	for (i = 0; i < nargs; i++) {	5410	for (i = 0; i < nargs; i++) {
5423	struct bpf_reg_state *reg = &regs[i + 1];	5411	u32 regno = i + 1;
		5412	struct bpf_reg_state *reg = &regs[regno];
5424		5413
5425	t = btf_type_by_id(btf, args[i].type);	5414	t = btf_type_skip_modifiers(btf, args[i].type, NULL);
5426	while (btf_type_is_modifier(t))	5415	if (btf_type_is_scalar(t)) {
5427	t = btf_type_by_id(btf, t->type);
5428	if (btf_type_is_int(t) \|\| btf_type_is_enum(t)) {
5429	if (reg->type == SCALAR_VALUE)	5416	if (reg->type == SCALAR_VALUE)
5430	continue;	5417	continue;
5431	bpf_log(log, "R%d is not a scalar\n", i + 1);	5418	bpf_log(log, "R%d is not a scalar\n", regno);
5432	goto out;	5419	return -EINVAL;
5433	}	5420	}
5434	if (btf_type_is_ptr(t)) {	5421
		5422	if (!btf_type_is_ptr(t)) {
		5423	bpf_log(log, "Unrecognized arg#%d type %s\n",
		5424	i, btf_type_str(t));
		5425	return -EINVAL;
		5426	}
		5427
		5428	ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id);
		5429	ref_tname = btf_name_by_offset(btf, ref_t->name_off);
		5430	if (btf_is_kernel(btf)) {
		5431	const struct btf_type *reg_ref_t;
		5432	const struct btf *reg_btf;
		5433	const char *reg_ref_tname;
		5434	u32 reg_ref_id;
		5435
		5436	if (!btf_type_is_struct(ref_t)) {
		5437	bpf_log(log, "kernel function %s args#%d pointer type %s %s is not supported\n",
		5438	func_name, i, btf_type_str(ref_t),
		5439	ref_tname);
		5440	return -EINVAL;
		5441	}
		5442
		5443	if (reg->type == PTR_TO_BTF_ID) {
		5444	reg_btf = reg->btf;
		5445	reg_ref_id = reg->btf_id;
		5446	} else if (reg2btf_ids[reg->type]) {
		5447	reg_btf = btf_vmlinux;
		5448	reg_ref_id = *reg2btf_ids[reg->type];
		5449	} else {
		5450	bpf_log(log, "kernel function %s args#%d expected pointer to %s %s but R%d is not a pointer to btf_id\n",
		5451	func_name, i,
		5452	btf_type_str(ref_t), ref_tname, regno);
		5453	return -EINVAL;
		5454	}
		5455
		5456	reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id,
		5457	&reg_ref_id);
		5458	reg_ref_tname = btf_name_by_offset(reg_btf,
		5459	reg_ref_t->name_off);
		5460	if (!btf_struct_ids_match(log, reg_btf, reg_ref_id,
		5461	reg->off, btf, ref_id)) {
		5462	bpf_log(log, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\n",
		5463	func_name, i,
		5464	btf_type_str(ref_t), ref_tname,
		5465	regno, btf_type_str(reg_ref_t),
		5466	reg_ref_tname);
		5467	return -EINVAL;
		5468	}
		5469	} else if (btf_get_prog_ctx_type(log, btf, t,
		5470	env->prog->type, i)) {
5435	/* If function expects ctx type in BTF check that caller	5471	/* If function expects ctx type in BTF check that caller
5436	* is passing PTR_TO_CTX.	5472	* is passing PTR_TO_CTX.
5437	*/	5473	*/
5438	if (btf_get_prog_ctx_type(log, btf, t, prog->type, i)) {	5474	if (reg->type != PTR_TO_CTX) {
5439	if (reg->type != PTR_TO_CTX) {	5475	bpf_log(log,
5440	bpf_log(log,	5476	"arg#%d expected pointer to ctx, but got %s\n",
5441	"arg#%d expected pointer to ctx, but got %s\n",	5477	i, btf_type_str(t));
5442	i, btf_kind_str[BTF_INFO_KIND(t->info)]);	5478	return -EINVAL;
5443	goto out;
5444	}
5445	if (check_ctx_reg(env, reg, i + 1))
5446	goto out;
5447	continue;
5448	}	5479	}
		5480	if (check_ctx_reg(env, reg, regno))
		5481	return -EINVAL;
		5482	} else if (ptr_to_mem_ok) {
		5483	const struct btf_type *resolve_ret;
		5484	u32 type_size;
5449		5485
5450	if (!is_global)	5486	resolve_ret = btf_resolve_size(btf, ref_t, &type_size);
5451	goto out;	5487	if (IS_ERR(resolve_ret)) {
5452
5453	t = btf_type_skip_modifiers(btf, t->type, NULL);
5454
5455	ref_t = btf_resolve_size(btf, t, &type_size);
5456	if (IS_ERR(ref_t)) {
5457	bpf_log(log,	5488	bpf_log(log,
5458	"arg#%d reference type('%s %s') size cannot be determined: %ld\n",	5489	"arg#%d reference type('%s %s') size cannot be determined: %ld\n",
5459	i, btf_type_str(t), btf_name_by_offset(btf, t->name_off),	5490	i, btf_type_str(ref_t), ref_tname,
5460	PTR_ERR(ref_t));	5491	PTR_ERR(resolve_ret));
5461	goto out;	5492	return -EINVAL;
5462	}	5493	}
5463		5494
5464	if (check_mem_reg(env, reg, i + 1, type_size))	5495	if (check_mem_reg(env, reg, regno, type_size))
5465	goto out;	5496	return -EINVAL;
5466		5497	} else {
5467	continue;	5498	return -EINVAL;
5468	}	5499	}
5469	bpf_log(log, "Unrecognized arg#%d type %s\n",
5470	i, btf_kind_str[BTF_INFO_KIND(t->info)]);
5471	goto out;
5472	}	5500	}
		5501
5473	return 0;	5502	return 0;
5474	out:	5503	}
		5504
		5505	/* Compare BTF of a function with given bpf_reg_state.
		5506	* Returns:
		5507	* EFAULT - there is a verifier bug. Abort verification.
		5508	* EINVAL - there is a type mismatch or BTF is not available.
		5509	* 0 - BTF matches with what bpf_reg_state expects.
		5510	* Only PTR_TO_CTX and SCALAR_VALUE states are recognized.
		5511	*/
		5512	int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog,
		5513	struct bpf_reg_state *regs)
		5514	{
		5515	struct bpf_prog *prog = env->prog;
		5516	struct btf *btf = prog->aux->btf;
		5517	bool is_global;
		5518	u32 btf_id;
		5519	int err;
		5520
		5521	if (!prog->aux->func_info)
		5522	return -EINVAL;
		5523
		5524	btf_id = prog->aux->func_info[subprog].type_id;
		5525	if (!btf_id)
		5526	return -EFAULT;
		5527
		5528	if (prog->aux->func_info_aux[subprog].unreliable)
		5529	return -EINVAL;
		5530
		5531	is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
		5532	err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global);
		5533
5475	/* Compiler optimizations can remove arguments from static functions	5534	/* Compiler optimizations can remove arguments from static functions
5476	* or mismatched type can be passed into a global function.	5535	* or mismatched type can be passed into a global function.
5477	* In such cases mark the function as unreliable from BTF point of view.	5536	* In such cases mark the function as unreliable from BTF point of view.
5478	*/	5537	*/
5479	prog->aux->func_info_aux[subprog].unreliable = true;	5538	if (err)
5480	return -EINVAL;	5539	prog->aux->func_info_aux[subprog].unreliable = true;
		5540	return err;
		5541	}
		5542
		5543	int btf_check_kfunc_arg_match(struct bpf_verifier_env *env,
		5544	const struct btf *btf, u32 func_id,
		5545	struct bpf_reg_state *regs)
		5546	{
		5547	return btf_check_func_arg_match(env, btf, func_id, regs, false);
5481	}	5548	}
5482		5549
5483	/* Convert BTF of a function into bpf_reg_state if possible	5550	/* Convert BTF of a function into bpf_reg_state if possible

kernel/bpf/core.c+24−23

 	if (!prog->aux->nr_linfo || !prog->jit_requested)
 		return 0;
-	prog->aux->jited_linfo = kcalloc(prog->aux->nr_linfo,
+	prog->aux->jited_linfo = kvcalloc(prog->aux->nr_linfo,
-					 sizeof(*prog->aux->jited_linfo),
+					  sizeof(*prog->aux->jited_linfo),
-					 GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
+					  GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
 	if (!prog->aux->jited_linfo)
 		return -ENOMEM;
 	return 0;
 }
-void bpf_prog_free_jited_linfo(struct bpf_prog *prog)
+void bpf_prog_jit_attempt_done(struct bpf_prog *prog)
 {
-	kfree(prog->aux->jited_linfo);
+	if (prog->aux->jited_linfo &&
-	prog->aux->jited_linfo = NULL;
+	    (!prog->jited || !prog->aux->jited_linfo[0])) {
-}
+		kvfree(prog->aux->jited_linfo);
+		prog->aux->jited_linfo = NULL;
+	}
-void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog)
+	kfree(prog->aux->kfunc_tab);
-{
+	prog->aux->kfunc_tab = NULL;
-	if (prog->aux->jited_linfo && !prog->aux->jited_linfo[0])
-		bpf_prog_free_jited_linfo(prog);
 }
 /* The jit engine is responsible to provide an array
 			insn_to_jit_off[linfo[i].insn_off - insn_start - 1];
 }
-void bpf_prog_free_linfo(struct bpf_prog *prog)
-{
-	bpf_prog_free_jited_linfo(prog);
-	kvfree(prog->aux->linfo);
-}
 struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,
 				  gfp_t gfp_extra_flags)
 {
 	/* In case of BPF to BPF calls, verifier did all the prep
 	 * work with regards to JITing, etc.
 	 */
+	bool jit_needed = false;
 	if (fp->bpf_func)
 		goto finalize;
+	if (IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON) ||
+	    bpf_prog_has_kfunc_call(fp))
+		jit_needed = true;
 	bpf_prog_select_func(fp);
 	/* eBPF JITs can rewrite the program in case constant
 			return fp;
 		fp = bpf_int_jit_compile(fp);
-		if (!fp->jited) {
+		bpf_prog_jit_attempt_done(fp);
-			bpf_prog_free_jited_linfo(fp);
+		if (!fp->jited && jit_needed) {
-#ifdef CONFIG_BPF_JIT_ALWAYS_ON
 			*err = -ENOTSUPP;
 			return fp;
-#endif
-		} else {
-			bpf_prog_free_unused_jited_linfo(fp);
 		}
 	} else {
 		*err = bpf_prog_offload_compile(fp);
 	return false;
 }
+bool __weak bpf_jit_supports_kfunc_call(void)
+{
+	return false;
+}
 /* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call
  * skb_copy_bits(), so provide a weak definition of it for NET-less config.
  */

kernel/bpf/disasm.c+10−3

 {
 	BUILD_BUG_ON(ARRAY_SIZE(func_id_str) != __BPF_FUNC_MAX_ID);
-	if (insn->src_reg != BPF_PSEUDO_CALL &&
+	if (!insn->src_reg &&
 	    insn->imm >= 0 && insn->imm < __BPF_FUNC_MAX_ID &&
 	    func_id_str[insn->imm])
 		return func_id_str[insn->imm];
-	if (cbs && cbs->cb_call)
+	if (cbs && cbs->cb_call) {
-		return cbs->cb_call(cbs->private_data, insn);
+		const char *res;
+		res = cbs->cb_call(cbs->private_data, insn);
+		if (res)
+			return res;
+	}
 	if (insn->src_reg == BPF_PSEUDO_CALL)
 		snprintf(buff, len, "%+d", insn->imm);
+	else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL)
+		snprintf(buff, len, "kernel-function");
 	return buff;
 }

kernel/bpf/helpers.c+11−4

 };
 #ifdef CONFIG_CGROUP_BPF
-DECLARE_PER_CPU(struct bpf_cgroup_storage*,
+DECLARE_PER_CPU(struct bpf_cgroup_storage_info,
-		bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);
+		bpf_cgroup_storage_info[BPF_CGROUP_STORAGE_NEST_MAX]);
 BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags)
 {
 	 * verifier checks that its value is correct.
 	 */
 	enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);
-	struct bpf_cgroup_storage *storage;
+	struct bpf_cgroup_storage *storage = NULL;
 	void *ptr;
+	int i;
-	storage = this_cpu_read(bpf_cgroup_storage[stype]);
+	for (i = 0; i < BPF_CGROUP_STORAGE_NEST_MAX; i++) {
+		if (unlikely(this_cpu_read(bpf_cgroup_storage_info[i].task) != current))
+			continue;
+		storage = this_cpu_read(bpf_cgroup_storage_info[i].storage[stype]);
+		break;
+	}
 	if (stype == BPF_CGROUP_STORAGE_SHARED)
 		ptr = &READ_ONCE(storage->buf)->data[0];

kernel/bpf/local_storage.c+3−2

 #include <linux/slab.h>
 #include <uapi/linux/btf.h>
-DEFINE_PER_CPU(struct bpf_cgroup_storage*, bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);
 #ifdef CONFIG_CGROUP_BPF
+DEFINE_PER_CPU(struct bpf_cgroup_storage_info,
+	       bpf_cgroup_storage_info[BPF_CGROUP_STORAGE_NEST_MAX]);
 #include "../cgroup/cgroup-internal.h"
 #define LOCAL_STORAGE_CREATE_FLAG_MASK					\

kernel/bpf/lpm_trie.c+3−0

 	.map_lookup_elem = trie_lookup_elem,
 	.map_update_elem = trie_update_elem,
 	.map_delete_elem = trie_delete_elem,
+	.map_lookup_batch = generic_map_lookup_batch,
+	.map_update_batch = generic_map_update_batch,
+	.map_delete_batch = generic_map_delete_batch,
 	.map_check_btf = trie_check_btf,
 	.map_btf_name = "lpm_trie",
 	.map_btf_id = &trie_map_btf_id,

kernel/bpf/syscall.c+4−1

 {
 	bpf_prog_kallsyms_del_all(prog);
 	btf_put(prog->aux->btf);
-	bpf_prog_free_linfo(prog);
+	kvfree(prog->aux->jited_linfo);
+	kvfree(prog->aux->linfo);
+	kfree(prog->aux->kfunc_tab);
 	if (prog->aux->attach_btf)
 		btf_put(prog->aux->attach_btf);
 		return BPF_PROG_TYPE_SK_MSG;
 	case BPF_SK_SKB_STREAM_PARSER:
 	case BPF_SK_SKB_STREAM_VERDICT:
+	case BPF_SK_SKB_VERDICT:
 		return BPF_PROG_TYPE_SK_SKB;
 	case BPF_LIRC_MODE2:
 		return BPF_PROG_TYPE_LIRC_MODE2;

kernel/bpf/verifier.c+348−42

@@ -234,6 +234,12 @@ static bool bpf_pseudo_call(const struct bpf_insn *insn) · 233 unmodified lines
234	insn->src_reg == BPF_PSEUDO_CALL;	234	insn->src_reg == BPF_PSEUDO_CALL;
235	}	235	}
236		236
		237	static bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn)
		238	{
		239	return insn->code == (BPF_JMP \| BPF_CALL) &&
		240	insn->src_reg == BPF_PSEUDO_KFUNC_CALL;
		241	}
		242
237	static bool bpf_pseudo_func(const struct bpf_insn *insn)	243	static bool bpf_pseudo_func(const struct bpf_insn *insn)
238	{	244	{
239	return insn->code == (BPF_LD \| BPF_IMM \| BPF_DW) &&	245	return insn->code == (BPF_LD \| BPF_IMM \| BPF_DW) &&
@@ -1554,47 +1560,205 @@ static int add_subprog(struct bpf_verifier_env *env, int off) · 1314 unmodified lines
1554	verbose(env, "too many subprograms\n");	1560	verbose(env, "too many subprograms\n");
1555	return -E2BIG;	1561	return -E2BIG;
1556	}	1562	}
		1563	/* determine subprog starts. The end is one before the next starts */
1557	env->subprog_info[env->subprog_cnt++].start = off;	1564	env->subprog_info[env->subprog_cnt++].start = off;
1558	sort(env->subprog_info, env->subprog_cnt,	1565	sort(env->subprog_info, env->subprog_cnt,
1559	sizeof(env->subprog_info[0]), cmp_subprogs, NULL);	1566	sizeof(env->subprog_info[0]), cmp_subprogs, NULL);
1560	return env->subprog_cnt - 1;	1567	return env->subprog_cnt - 1;
1561	}	1568	}
1562		1569
1563	static int check_subprogs(struct bpf_verifier_env *env)	1570	struct bpf_kfunc_desc {
		1571	struct btf_func_model func_model;
		1572	u32 func_id;
		1573	s32 imm;
		1574	};
		1575
		1576	#define MAX_KFUNC_DESCS 256
		1577	struct bpf_kfunc_desc_tab {
		1578	struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS];
		1579	u32 nr_descs;
		1580	};
		1581
		1582	static int kfunc_desc_cmp_by_id(const void a, const void b)
		1583	{
		1584	const struct bpf_kfunc_desc *d0 = a;
		1585	const struct bpf_kfunc_desc *d1 = b;
		1586
		1587	/* func_id is not greater than BTF_MAX_TYPE */
		1588	return d0->func_id - d1->func_id;
		1589	}
		1590
		1591	static const struct bpf_kfunc_desc *
		1592	find_kfunc_desc(const struct bpf_prog *prog, u32 func_id)
		1593	{
		1594	struct bpf_kfunc_desc desc = {
		1595	.func_id = func_id,
		1596	};
		1597	struct bpf_kfunc_desc_tab *tab;
		1598
		1599	tab = prog->aux->kfunc_tab;
		1600	return bsearch(&desc, tab->descs, tab->nr_descs,
		1601	sizeof(tab->descs[0]), kfunc_desc_cmp_by_id);
		1602	}
		1603
		1604	static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id)
		1605	{
		1606	const struct btf_type func, func_proto;
		1607	struct bpf_kfunc_desc_tab *tab;
		1608	struct bpf_prog_aux *prog_aux;
		1609	struct bpf_kfunc_desc *desc;
		1610	const char *func_name;
		1611	unsigned long addr;
		1612	int err;
		1613
		1614	prog_aux = env->prog->aux;
		1615	tab = prog_aux->kfunc_tab;
		1616	if (!tab) {
		1617	if (!btf_vmlinux) {
		1618	verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n");
		1619	return -ENOTSUPP;
		1620	}
		1621
		1622	if (!env->prog->jit_requested) {
		1623	verbose(env, "JIT is required for calling kernel function\n");
		1624	return -ENOTSUPP;
		1625	}
		1626
		1627	if (!bpf_jit_supports_kfunc_call()) {
		1628	verbose(env, "JIT does not support calling kernel function\n");
		1629	return -ENOTSUPP;
		1630	}
		1631
		1632	if (!env->prog->gpl_compatible) {
		1633	verbose(env, "cannot call kernel function from non-GPL compatible program\n");
		1634	return -EINVAL;
		1635	}
		1636
		1637	tab = kzalloc(sizeof(*tab), GFP_KERNEL);
		1638	if (!tab)
		1639	return -ENOMEM;
		1640	prog_aux->kfunc_tab = tab;
		1641	}
		1642
		1643	if (find_kfunc_desc(env->prog, func_id))
		1644	return 0;
		1645
		1646	if (tab->nr_descs == MAX_KFUNC_DESCS) {
		1647	verbose(env, "too many different kernel function calls\n");
		1648	return -E2BIG;
		1649	}
		1650
		1651	func = btf_type_by_id(btf_vmlinux, func_id);
		1652	if (!func \|\| !btf_type_is_func(func)) {
		1653	verbose(env, "kernel btf_id %u is not a function\n",
		1654	func_id);
		1655	return -EINVAL;
		1656	}
		1657	func_proto = btf_type_by_id(btf_vmlinux, func->type);
		1658	if (!func_proto \|\| !btf_type_is_func_proto(func_proto)) {
		1659	verbose(env, "kernel function btf_id %u does not have a valid func_proto\n",
		1660	func_id);
		1661	return -EINVAL;
		1662	}
		1663
		1664	func_name = btf_name_by_offset(btf_vmlinux, func->name_off);
		1665	addr = kallsyms_lookup_name(func_name);
		1666	if (!addr) {
		1667	verbose(env, "cannot find address for kernel function %s\n",
		1668	func_name);
		1669	return -EINVAL;
		1670	}
		1671
		1672	desc = &tab->descs[tab->nr_descs++];
		1673	desc->func_id = func_id;
		1674	desc->imm = BPF_CAST_CALL(addr) - __bpf_call_base;
		1675	err = btf_distill_func_proto(&env->log, btf_vmlinux,
		1676	func_proto, func_name,
		1677	&desc->func_model);
		1678	if (!err)
		1679	sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
		1680	kfunc_desc_cmp_by_id, NULL);
		1681	return err;
		1682	}
		1683
		1684	static int kfunc_desc_cmp_by_imm(const void a, const void b)
		1685	{
		1686	const struct bpf_kfunc_desc *d0 = a;
		1687	const struct bpf_kfunc_desc *d1 = b;
		1688
		1689	if (d0->imm > d1->imm)
		1690	return 1;
		1691	else if (d0->imm < d1->imm)
		1692	return -1;
		1693	return 0;
		1694	}
		1695
		1696	static void sort_kfunc_descs_by_imm(struct bpf_prog *prog)
		1697	{
		1698	struct bpf_kfunc_desc_tab *tab;
		1699
		1700	tab = prog->aux->kfunc_tab;
		1701	if (!tab)
		1702	return;
		1703
		1704	sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
		1705	kfunc_desc_cmp_by_imm, NULL);
		1706	}
		1707
		1708	bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
		1709	{
		1710	return !!prog->aux->kfunc_tab;
		1711	}
		1712
		1713	const struct btf_func_model *
		1714	bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
		1715	const struct bpf_insn *insn)
		1716	{
		1717	const struct bpf_kfunc_desc desc = {
		1718	.imm = insn->imm,
		1719	};
		1720	const struct bpf_kfunc_desc *res;
		1721	struct bpf_kfunc_desc_tab *tab;
		1722
		1723	tab = prog->aux->kfunc_tab;
		1724	res = bsearch(&desc, tab->descs, tab->nr_descs,
		1725	sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm);
		1726
		1727	return res ? &res->func_model : NULL;
		1728	}
		1729
		1730	static int add_subprog_and_kfunc(struct bpf_verifier_env *env)
1564	{	1731	{
1565	int i, ret, subprog_start, subprog_end, off, cur_subprog = 0;
1566	struct bpf_subprog_info *subprog = env->subprog_info;	1732	struct bpf_subprog_info *subprog = env->subprog_info;
1567	struct bpf_insn *insn = env->prog->insnsi;	1733	struct bpf_insn *insn = env->prog->insnsi;
1568	int insn_cnt = env->prog->len;	1734	int i, ret, insn_cnt = env->prog->len;
1569		1735
1570	/* Add entry function. */	1736	/* Add entry function. */
1571	ret = add_subprog(env, 0);	1737	ret = add_subprog(env, 0);
1572	if (ret < 0)	1738	if (ret)
1573	return ret;	1739	return ret;
1574		1740
1575	/* determine subprog starts. The end is one before the next starts */	1741	for (i = 0; i < insn_cnt; i++, insn++) {
1576	for (i = 0; i < insn_cnt; i++) {	1742	if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) &&
1577	if (bpf_pseudo_func(insn + i)) {	1743	!bpf_pseudo_kfunc_call(insn))
1578	if (!env->bpf_capable) {
1579	verbose(env,
1580	"function pointers are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
1581	return -EPERM;
1582	}
1583	ret = add_subprog(env, i + insn[i].imm + 1);
1584	if (ret < 0)
1585	return ret;
1586	/* remember subprog */
1587	insn[i + 1].imm = ret;
1588	continue;
1589	}
1590	if (!bpf_pseudo_call(insn + i))
1591	continue;	1744	continue;
		1745
1592	if (!env->bpf_capable) {	1746	if (!env->bpf_capable) {
1593	verbose(env,	1747	verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
1594	"function calls to other bpf functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
1595	return -EPERM;	1748	return -EPERM;
1596	}	1749	}
1597	ret = add_subprog(env, i + insn[i].imm + 1);	1750
		1751	if (bpf_pseudo_func(insn)) {
		1752	ret = add_subprog(env, i + insn->imm + 1);
		1753	if (ret >= 0)
		1754	/* remember subprog */
		1755	insn[1].imm = ret;
		1756	} else if (bpf_pseudo_call(insn)) {
		1757	ret = add_subprog(env, i + insn->imm + 1);
		1758	} else {
		1759	ret = add_kfunc_call(env, insn->imm);
		1760	}
		1761
1598	if (ret < 0)	1762	if (ret < 0)
1599	return ret;	1763	return ret;
1600	}	1764	}
@@ -1608,6 +1772,16 @@ static int check_subprogs(struct bpf_verifier_env *env) · 7 unmodified lines
1608	for (i = 0; i < env->subprog_cnt; i++)	1772	for (i = 0; i < env->subprog_cnt; i++)
1609	verbose(env, "func#%d @%d\n", i, subprog[i].start);	1773	verbose(env, "func#%d @%d\n", i, subprog[i].start);
1610		1774
		1775	return 0;
		1776	}
		1777
		1778	static int check_subprogs(struct bpf_verifier_env *env)
		1779	{
		1780	int i, subprog_start, subprog_end, off, cur_subprog = 0;
		1781	struct bpf_subprog_info *subprog = env->subprog_info;
		1782	struct bpf_insn *insn = env->prog->insnsi;
		1783	int insn_cnt = env->prog->len;
		1784
1611	/* now check that all jumps are within the same subprog */	1785	/* now check that all jumps are within the same subprog */
1612	subprog_start = subprog[cur_subprog].start;	1786	subprog_start = subprog[cur_subprog].start;
1613	subprog_end = subprog[cur_subprog + 1].start;	1787	subprog_end = subprog[cur_subprog + 1].start;
@@ -1916,6 +2090,17 @@ static int get_prev_insn_idx(struct bpf_verifier_state *st, int i, · 302 unmodified lines
1916	return i;	2090	return i;
1917	}	2091	}
1918		2092
		2093	static const char disasm_kfunc_name(void data, const struct bpf_insn *insn)
		2094	{
		2095	const struct btf_type *func;
		2096
		2097	if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL)
		2098	return NULL;
		2099
		2100	func = btf_type_by_id(btf_vmlinux, insn->imm);
		2101	return btf_name_by_offset(btf_vmlinux, func->name_off);
		2102	}
		2103
1919	/* For given verifier state backtrack_insn() is called from the last insn to	2104	/* For given verifier state backtrack_insn() is called from the last insn to
1920	* the first insn. Its purpose is to compute a bitmask of registers and	2105	* the first insn. Its purpose is to compute a bitmask of registers and
1921	* stack slots that needs precision in the parent verifier state.	2106	* stack slots that needs precision in the parent verifier state.
@@ -1924,6 +2109,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, · 2 unmodified lines
1924	u32 reg_mask, u64 stack_mask)	2109	u32 reg_mask, u64 stack_mask)
1925	{	2110	{
1926	const struct bpf_insn_cbs cbs = {	2111	const struct bpf_insn_cbs cbs = {
		2112	.cb_call = disasm_kfunc_name,
1927	.cb_print = verbose,	2113	.cb_print = verbose,
1928	.private_data = env,	2114	.private_data = env,
1929	};	2115	};
@@ -5365,7 +5551,7 @@ static int __check_func_call(struct bpf_verifier_env env, struct bpf_insn insn · 3435 unmodified lines
5365	func_info_aux = env->prog->aux->func_info_aux;	5551	func_info_aux = env->prog->aux->func_info_aux;
5366	if (func_info_aux)	5552	if (func_info_aux)
5367	is_global = func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;	5553	is_global = func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
5368	err = btf_check_func_arg_match(env, subprog, caller->regs);	5554	err = btf_check_subprog_arg_match(env, subprog, caller->regs);
5369	if (err == -EFAULT)	5555	if (err == -EFAULT)
5370	return err;	5556	return err;
5371	if (is_global) {	5557	if (is_global) {
@@ -5960,6 +6146,98 @@ static int check_helper_call(struct bpf_verifier_env env, struct bpf_insn insn · 588 unmodified lines
5960	return 0;	6146	return 0;
5961	}	6147	}
5962		6148
		6149	/* mark_btf_func_reg_size() is used when the reg size is determined by
		6150	* the BTF func_proto's return value size and argument.
		6151	*/
		6152	static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno,
		6153	size_t reg_size)
		6154	{
		6155	struct bpf_reg_state *reg = &cur_regs(env)[regno];
		6156
		6157	if (regno == BPF_REG_0) {
		6158	/* Function return value */
		6159	reg->live \|= REG_LIVE_WRITTEN;
		6160	reg->subreg_def = reg_size == sizeof(u64) ?
		6161	DEF_NOT_SUBREG : env->insn_idx + 1;
		6162	} else {
		6163	/* Function argument */
		6164	if (reg_size == sizeof(u64)) {
		6165	mark_insn_zext(env, reg);
		6166	mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
		6167	} else {
		6168	mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32);
		6169	}
		6170	}
		6171	}
		6172
		6173	static int check_kfunc_call(struct bpf_verifier_env env, struct bpf_insn insn)
		6174	{
		6175	const struct btf_type t, func, func_proto, ptr_type;
		6176	struct bpf_reg_state *regs = cur_regs(env);
		6177	const char func_name, ptr_type_name;
		6178	u32 i, nargs, func_id, ptr_type_id;
		6179	const struct btf_param *args;
		6180	int err;
		6181
		6182	func_id = insn->imm;
		6183	func = btf_type_by_id(btf_vmlinux, func_id);
		6184	func_name = btf_name_by_offset(btf_vmlinux, func->name_off);
		6185	func_proto = btf_type_by_id(btf_vmlinux, func->type);
		6186
		6187	if (!env->ops->check_kfunc_call \|\|
		6188	!env->ops->check_kfunc_call(func_id)) {
		6189	verbose(env, "calling kernel function %s is not allowed\n",
		6190	func_name);
		6191	return -EACCES;
		6192	}
		6193
		6194	/* Check the arguments */
		6195	err = btf_check_kfunc_arg_match(env, btf_vmlinux, func_id, regs);
		6196	if (err)
		6197	return err;
		6198
		6199	for (i = 0; i < CALLER_SAVED_REGS; i++)
		6200	mark_reg_not_init(env, regs, caller_saved[i]);
		6201
		6202	/* Check return type */
		6203	t = btf_type_skip_modifiers(btf_vmlinux, func_proto->type, NULL);
		6204	if (btf_type_is_scalar(t)) {
		6205	mark_reg_unknown(env, regs, BPF_REG_0);
		6206	mark_btf_func_reg_size(env, BPF_REG_0, t->size);
		6207	} else if (btf_type_is_ptr(t)) {
		6208	ptr_type = btf_type_skip_modifiers(btf_vmlinux, t->type,
		6209	&ptr_type_id);
		6210	if (!btf_type_is_struct(ptr_type)) {
		6211	ptr_type_name = btf_name_by_offset(btf_vmlinux,
		6212	ptr_type->name_off);
		6213	verbose(env, "kernel function %s returns pointer type %s %s is not supported\n",
		6214	func_name, btf_type_str(ptr_type),
		6215	ptr_type_name);
		6216	return -EINVAL;
		6217	}
		6218	mark_reg_known_zero(env, regs, BPF_REG_0);
		6219	regs[BPF_REG_0].btf = btf_vmlinux;
		6220	regs[BPF_REG_0].type = PTR_TO_BTF_ID;
		6221	regs[BPF_REG_0].btf_id = ptr_type_id;
		6222	mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *));
		6223	} /* else { add_kfunc_call() ensures it is btf_type_is_void(t) } */
		6224
		6225	nargs = btf_type_vlen(func_proto);
		6226	args = (const struct btf_param *)(func_proto + 1);
		6227	for (i = 0; i < nargs; i++) {
		6228	u32 regno = i + 1;
		6229
		6230	t = btf_type_skip_modifiers(btf_vmlinux, args[i].type, NULL);
		6231	if (btf_type_is_ptr(t))
		6232	mark_btf_func_reg_size(env, regno, sizeof(void *));
		6233	else
		6234	/* scalar. ensured by btf_check_kfunc_arg_match() */
		6235	mark_btf_func_reg_size(env, regno, t->size);
		6236	}
		6237
		6238	return 0;
		6239	}
		6240
5963	static bool signed_add_overflows(s64 a, s64 b)	6241	static bool signed_add_overflows(s64 a, s64 b)
5964	{	6242	{
5965	/* Do the add in u64, where overflow is well-defined */	6243	/* Do the add in u64, where overflow is well-defined */
@@ -6062,19 +6340,6 @@ static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, · 96 unmodified lines
6062	else	6340	else
6063	*ptr_limit = -off - 1;	6341	*ptr_limit = -off - 1;
6064	return *ptr_limit >= max ? -ERANGE : 0;	6342	return *ptr_limit >= max ? -ERANGE : 0;
6065	case PTR_TO_MAP_KEY:
6066	/* Currently, this code is not exercised as the only use
6067	* is bpf_for_each_map_elem() helper which requires
6068	* bpf_capble. The code has been tested manually for
6069	* future use.
6070	*/
6071	if (mask_to_left) {
6072	*ptr_limit = ptr_reg->umax_value + ptr_reg->off;
6073	} else {
6074	off = ptr_reg->smin_value + ptr_reg->off;
6075	*ptr_limit = ptr_reg->map_ptr->key_size - off;
6076	}
6077	return 0;
6078	case PTR_TO_MAP_VALUE:	6343	case PTR_TO_MAP_VALUE:
6079	max = ptr_reg->map_ptr->value_size;	6344	max = ptr_reg->map_ptr->value_size;
6080	if (mask_to_left) {	6345	if (mask_to_left) {
@@ -6281,7 +6546,6 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, · 200 unmodified lines
6281	verbose(env, "R%d pointer arithmetic on %s prohibited\n",	6546	verbose(env, "R%d pointer arithmetic on %s prohibited\n",
6282	dst, reg_type_str[ptr_reg->type]);	6547	dst, reg_type_str[ptr_reg->type]);
6283	return -EACCES;	6548	return -EACCES;
6284	case PTR_TO_MAP_KEY:
6285	case PTR_TO_MAP_VALUE:	6549	case PTR_TO_MAP_VALUE:
6286	if (!env->allow_ptr_leaks && !known && (smin_val < 0) != (smax_val < 0)) {	6550	if (!env->allow_ptr_leaks && !known && (smin_val < 0) != (smax_val < 0)) {
6287	verbose(env, "R%d has unknown scalar with mixed signed bounds, pointer arithmetic with it prohibited for !root\n",	6551	verbose(env, "R%d has unknown scalar with mixed signed bounds, pointer arithmetic with it prohibited for !root\n",
@@ -10176,6 +10440,7 @@ static int do_check(struct bpf_verifier_env *env) · 3888 unmodified lines
10176		10440
10177	if (env->log.level & BPF_LOG_LEVEL) {	10441	if (env->log.level & BPF_LOG_LEVEL) {
10178	const struct bpf_insn_cbs cbs = {	10442	const struct bpf_insn_cbs cbs = {
		10443	.cb_call = disasm_kfunc_name,
10179	.cb_print = verbose,	10444	.cb_print = verbose,
10180	.private_data = env,	10445	.private_data = env,
10181	};	10446	};
@@ -10323,7 +10588,8 @@ static int do_check(struct bpf_verifier_env *env) · 141 unmodified lines
10323	if (BPF_SRC(insn->code) != BPF_K \|\|	10588	if (BPF_SRC(insn->code) != BPF_K \|\|
10324	insn->off != 0 \|\|	10589	insn->off != 0 \|\|
10325	(insn->src_reg != BPF_REG_0 &&	10590	(insn->src_reg != BPF_REG_0 &&
10326	insn->src_reg != BPF_PSEUDO_CALL) \|\|	10591	insn->src_reg != BPF_PSEUDO_CALL &&
		10592	insn->src_reg != BPF_PSEUDO_KFUNC_CALL) \|\|
10327	insn->dst_reg != BPF_REG_0 \|\|	10593	insn->dst_reg != BPF_REG_0 \|\|
10328	class == BPF_JMP32) {	10594	class == BPF_JMP32) {
10329	verbose(env, "BPF_CALL uses reserved fields\n");	10595	verbose(env, "BPF_CALL uses reserved fields\n");
@@ -10338,6 +10604,8 @@ static int do_check(struct bpf_verifier_env *env) · 8 unmodified lines
10338	}	10604	}
10339	if (insn->src_reg == BPF_PSEUDO_CALL)	10605	if (insn->src_reg == BPF_PSEUDO_CALL)
10340	err = check_func_call(env, insn, &env->insn_idx);	10606	err = check_func_call(env, insn, &env->insn_idx);
		10607	else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL)
		10608	err = check_kfunc_call(env, insn);
10341	else	10609	else
10342	err = check_helper_call(env, insn, &env->insn_idx);	10610	err = check_helper_call(env, insn, &env->insn_idx);
10343	if (err)	10611	if (err)
@@ -11648,6 +11916,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 1304 unmodified lines
11648	func[i]->aux->name[0] = 'F';	11916	func[i]->aux->name[0] = 'F';
11649	func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;	11917	func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
11650	func[i]->jit_requested = 1;	11918	func[i]->jit_requested = 1;
		11919	func[i]->aux->kfunc_tab = prog->aux->kfunc_tab;
11651	func[i]->aux->linfo = prog->aux->linfo;	11920	func[i]->aux->linfo = prog->aux->linfo;
11652	func[i]->aux->nr_linfo = prog->aux->nr_linfo;	11921	func[i]->aux->nr_linfo = prog->aux->nr_linfo;
11653	func[i]->aux->jited_linfo = prog->aux->jited_linfo;	11922	func[i]->aux->jited_linfo = prog->aux->jited_linfo;
@@ -11755,7 +12024,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 101 unmodified lines
11755	prog->bpf_func = func[0]->bpf_func;	12024	prog->bpf_func = func[0]->bpf_func;
11756	prog->aux->func = func;	12025	prog->aux->func = func;
11757	prog->aux->func_cnt = env->subprog_cnt;	12026	prog->aux->func_cnt = env->subprog_cnt;
11758	bpf_prog_free_unused_jited_linfo(prog);	12027	bpf_prog_jit_attempt_done(prog);
11759	return 0;	12028	return 0;
11760	out_free:	12029	out_free:
11761	for (i = 0; i < env->subprog_cnt; i++) {	12030	for (i = 0; i < env->subprog_cnt; i++) {
@@ -11778,7 +12047,7 @@ out_undo_insn: · 16 unmodified lines
11778	insn->off = 0;	12047	insn->off = 0;
11779	insn->imm = env->insn_aux_data[i].call_imm;	12048	insn->imm = env->insn_aux_data[i].call_imm;
11780	}	12049	}
11781	bpf_prog_free_jited_linfo(prog);	12050	bpf_prog_jit_attempt_done(prog);
11782	return err;	12051	return err;
11783	}	12052	}
11784		12053
@@ -11787,6 +12056,7 @@ static int fixup_call_args(struct bpf_verifier_env *env) · 2 unmodified lines
11787	#ifndef CONFIG_BPF_JIT_ALWAYS_ON	12056	#ifndef CONFIG_BPF_JIT_ALWAYS_ON
11788	struct bpf_prog *prog = env->prog;	12057	struct bpf_prog *prog = env->prog;
11789	struct bpf_insn *insn = prog->insnsi;	12058	struct bpf_insn *insn = prog->insnsi;
		12059	bool has_kfunc_call = bpf_prog_has_kfunc_call(prog);
11790	int i, depth;	12060	int i, depth;
11791	#endif	12061	#endif
11792	int err = 0;	12062	int err = 0;
@@ -11800,6 +12070,10 @@ static int fixup_call_args(struct bpf_verifier_env *env) · 7 unmodified lines
11800	return err;	12070	return err;
11801	}	12071	}
11802	#ifndef CONFIG_BPF_JIT_ALWAYS_ON	12072	#ifndef CONFIG_BPF_JIT_ALWAYS_ON
		12073	if (has_kfunc_call) {
		12074	verbose(env, "calling kernel functions are not allowed in non-JITed programs\n");
		12075	return -EINVAL;
		12076	}
11803	if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {	12077	if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {
11804	/* When JIT fails the progs with bpf2bpf calls and tail_calls	12078	/* When JIT fails the progs with bpf2bpf calls and tail_calls
11805	* have to be rejected, since interpreter doesn't support them yet.	12079	* have to be rejected, since interpreter doesn't support them yet.
@@ -11828,6 +12102,26 @@ static int fixup_call_args(struct bpf_verifier_env *env) · 22 unmodified lines
11828	return err;	12102	return err;
11829	}	12103	}
11830		12104
		12105	static int fixup_kfunc_call(struct bpf_verifier_env *env,
		12106	struct bpf_insn *insn)
		12107	{
		12108	const struct bpf_kfunc_desc *desc;
		12109
		12110	/* insn->imm has the btf func_id. Replace it with
		12111	* an address (relative to __bpf_base_call).
		12112	*/
		12113	desc = find_kfunc_desc(env->prog, insn->imm);
		12114	if (!desc) {
		12115	verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n",
		12116	insn->imm);
		12117	return -EFAULT;
		12118	}
		12119
		12120	insn->imm = desc->imm;
		12121
		12122	return 0;
		12123	}
		12124
11831	/* Do various post-verification rewrites in a single program pass.	12125	/* Do various post-verification rewrites in a single program pass.
11832	* These rewrites simplify JIT and interpreter implementations.	12126	* These rewrites simplify JIT and interpreter implementations.
11833	*/	12127	*/
@@ -11963,6 +12257,12 @@ static int do_misc_fixups(struct bpf_verifier_env *env) · 129 unmodified lines
11963	continue;	12257	continue;
11964	if (insn->src_reg == BPF_PSEUDO_CALL)	12258	if (insn->src_reg == BPF_PSEUDO_CALL)
11965	continue;	12259	continue;
		12260	if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
		12261	ret = fixup_kfunc_call(env, insn);
		12262	if (ret)
		12263	return ret;
		12264	continue;
		12265	}
11966		12266
11967	if (insn->imm == BPF_FUNC_get_route_realm)	12267	if (insn->imm == BPF_FUNC_get_route_realm)
11968	prog->dst_needed = 1;	12268	prog->dst_needed = 1;
@@ -12192,6 +12492,8 @@ patch_call_imm: · 223 unmodified lines
12192	}	12492	}
12193	}	12493	}
12194		12494
		12495	sort_kfunc_descs_by_imm(env->prog);
		12496
12195	return 0;	12497	return 0;
12196	}	12498	}
12197		12499
@@ -12302,7 +12604,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) · 104 unmodified lines
12302	/* 1st arg to a function */	12604	/* 1st arg to a function */
12303	regs[BPF_REG_1].type = PTR_TO_CTX;	12605	regs[BPF_REG_1].type = PTR_TO_CTX;
12304	mark_reg_known_zero(env, regs, BPF_REG_1);	12606	mark_reg_known_zero(env, regs, BPF_REG_1);
12305	ret = btf_check_func_arg_match(env, subprog, regs);	12607	ret = btf_check_subprog_arg_match(env, subprog, regs);
12306	if (ret == -EFAULT)	12608	if (ret == -EFAULT)
12307	/* unlikely verifier bug. abort.	12609	/* unlikely verifier bug. abort.
12308	* ret == 0 and ret < 0 are sadly acceptable for	12610	* ret == 0 and ret < 0 are sadly acceptable for
@@ -12897,6 +13199,10 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr attr, · 588 unmodified lines
12897	if (!env->explored_states)	13199	if (!env->explored_states)
12898	goto skip_full_check;	13200	goto skip_full_check;
12899		13201
		13202	ret = add_subprog_and_kfunc(env);
		13203	if (ret < 0)
		13204	goto skip_full_check;
		13205
12900	ret = check_subprogs(env);	13206	ret = check_subprogs(env);
12901	if (ret < 0)	13207	if (ret < 0)
12902	goto skip_full_check;	13208	goto skip_full_check;

net/bpf/test_run.c+33−1

@@ -2,6 +2,7 @@ · 1 unmodified line
2	/* Copyright (c) 2017 Facebook	2	/* Copyright (c) 2017 Facebook
3	*/	3	*/
4	#include <linux/bpf.h>	4	#include <linux/bpf.h>
		5	#include <linux/btf_ids.h>
5	#include <linux/slab.h>	6	#include <linux/slab.h>
6	#include <linux/vmalloc.h>	7	#include <linux/vmalloc.h>
7	#include <linux/etherdevice.h>	8	#include <linux/etherdevice.h>
@@ -106,12 +107,16 @@ static int bpf_test_run(struct bpf_prog prog, void ctx, u32 repeat, · 98 unmodified lines
106		107
107	bpf_test_timer_enter(&t);	108	bpf_test_timer_enter(&t);
108	do {	109	do {
109	bpf_cgroup_storage_set(storage);	110	ret = bpf_cgroup_storage_set(storage);
		111	if (ret)
		112	break;
110		113
111	if (xdp)	114	if (xdp)
112	*retval = bpf_prog_run_xdp(prog, ctx);	115	*retval = bpf_prog_run_xdp(prog, ctx);
113	else	116	else
114	*retval = BPF_PROG_RUN(prog, ctx);	117	*retval = BPF_PROG_RUN(prog, ctx);
		118
		119	bpf_cgroup_storage_unset();
115	} while (bpf_test_timer_continue(&t, repeat, &ret, time));	120	} while (bpf_test_timer_continue(&t, repeat, &ret, time));
116	bpf_test_timer_leave(&t);	121	bpf_test_timer_leave(&t);
117		122
@@ -209,10 +214,37 @@ int noinline bpf_modify_return_test(int a, int *b) · 91 unmodified lines
209	*b += 1;	214	*b += 1;
210	return a + *b;	215	return a + *b;
211	}	216	}
		217
		218	u64 noinline bpf_kfunc_call_test1(struct sock *sk, u32 a, u64 b, u32 c, u64 d)
		219	{
		220	return a + b + c + d;
		221	}
		222
		223	int noinline bpf_kfunc_call_test2(struct sock *sk, u32 a, u32 b)
		224	{
		225	return a + b;
		226	}
		227
		228	struct sock * noinline bpf_kfunc_call_test3(struct sock *sk)
		229	{
		230	return sk;
		231	}
		232
212	__diag_pop();	233	__diag_pop();
213		234
214	ALLOW_ERROR_INJECTION(bpf_modify_return_test, ERRNO);	235	ALLOW_ERROR_INJECTION(bpf_modify_return_test, ERRNO);
215		236
		237	BTF_SET_START(test_sk_kfunc_ids)
		238	BTF_ID(func, bpf_kfunc_call_test1)
		239	BTF_ID(func, bpf_kfunc_call_test2)
		240	BTF_ID(func, bpf_kfunc_call_test3)
		241	BTF_SET_END(test_sk_kfunc_ids)
		242
		243	bool bpf_prog_test_check_kfunc_call(u32 kfunc_id)
		244	{
		245	return btf_id_set_contains(&test_sk_kfunc_ids, kfunc_id);
		246	}
		247
216	static void bpf_test_init(const union bpf_attr kattr, u32 size,	248	static void bpf_test_init(const union bpf_attr kattr, u32 size,
217	u32 headroom, u32 tailroom)	249	u32 headroom, u32 tailroom)
218	{	250	{

net/core/filter.c+1−0

@@ -9813,6 +9813,7 @@ const struct bpf_verifier_ops tc_cls_act_verifier_ops = { · 9812 unmodified lines
9813	.convert_ctx_access = tc_cls_act_convert_ctx_access,	9813	.convert_ctx_access = tc_cls_act_convert_ctx_access,
9814	.gen_prologue = tc_cls_act_prologue,	9814	.gen_prologue = tc_cls_act_prologue,
9815	.gen_ld_abs = bpf_gen_ld_abs,	9815	.gen_ld_abs = bpf_gen_ld_abs,
		9816	.check_kfunc_call = bpf_prog_test_check_kfunc_call,
9816	};	9817	};
9817		9818
9818	const struct bpf_prog_ops tc_cls_act_prog_ops = {	9819	const struct bpf_prog_ops tc_cls_act_prog_ops = {

net/core/skbuff.c+48−7

@@ -2500,9 +2500,32 @@ int skb_splice_bits(struct sk_buff skb, struct sock sk, unsigned int offset, · 2499 unmodified lines
2500	}	2500	}
2501	EXPORT_SYMBOL_GPL(skb_splice_bits);	2501	EXPORT_SYMBOL_GPL(skb_splice_bits);
2502		2502
2503	/* Send skb data on a socket. Socket must be locked. */	2503	static int sendmsg_unlocked(struct sock sk, struct msghdr msg,
2504	int skb_send_sock_locked(struct sock sk, struct sk_buff skb, int offset,	2504	struct kvec *vec, size_t num, size_t size)
2505	int len)	2505	{
		2506	struct socket *sock = sk->sk_socket;
		2507
		2508	if (!sock)
		2509	return -EINVAL;
		2510	return kernel_sendmsg(sock, msg, vec, num, size);
		2511	}
		2512
		2513	static int sendpage_unlocked(struct sock sk, struct page page, int offset,
		2514	size_t size, int flags)
		2515	{
		2516	struct socket *sock = sk->sk_socket;
		2517
		2518	if (!sock)
		2519	return -EINVAL;
		2520	return kernel_sendpage(sock, page, offset, size, flags);
		2521	}
		2522
		2523	typedef int (sendmsg_func)(struct sock sk, struct msghdr *msg,
		2524	struct kvec *vec, size_t num, size_t size);
		2525	typedef int (sendpage_func)(struct sock sk, struct page *page, int offset,
		2526	size_t size, int flags);
		2527	static int __skb_send_sock(struct sock sk, struct sk_buff skb, int offset,
		2528	int len, sendmsg_func sendmsg, sendpage_func sendpage)
2506	{	2529	{
2507	unsigned int orig_len = len;	2530	unsigned int orig_len = len;
2508	struct sk_buff *head = skb;	2531	struct sk_buff *head = skb;
@@ -2522,7 +2545,8 @@ do_frag_list: · 13 unmodified lines
2522	memset(&msg, 0, sizeof(msg));	2545	memset(&msg, 0, sizeof(msg));
2523	msg.msg_flags = MSG_DONTWAIT;	2546	msg.msg_flags = MSG_DONTWAIT;
2524		2547
2525	ret = kernel_sendmsg_locked(sk, &msg, &kv, 1, slen);	2548	ret = INDIRECT_CALL_2(sendmsg, kernel_sendmsg_locked,
		2549	sendmsg_unlocked, sk, &msg, &kv, 1, slen);
2526	if (ret <= 0)	2550	if (ret <= 0)
2527	goto error;	2551	goto error;
2528		2552
@@ -2553,9 +2577,11 @@ do_frag_list: · 24 unmodified lines
2553	slen = min_t(size_t, len, skb_frag_size(frag) - offset);	2577	slen = min_t(size_t, len, skb_frag_size(frag) - offset);
2554		2578
2555	while (slen) {	2579	while (slen) {
2556	ret = kernel_sendpage_locked(sk, skb_frag_page(frag),	2580	ret = INDIRECT_CALL_2(sendpage, kernel_sendpage_locked,
2557	skb_frag_off(frag) + offset,	2581	sendpage_unlocked, sk,
2558	slen, MSG_DONTWAIT);	2582	skb_frag_page(frag),
		2583	skb_frag_off(frag) + offset,
		2584	slen, MSG_DONTWAIT);
2559	if (ret <= 0)	2585	if (ret <= 0)
2560	goto error;	2586	goto error;
2561		2587
@@ -2587,8 +2613,23 @@ out: · 25 unmodified lines
2587	error:	2613	error:
2588	return orig_len == len ? ret : orig_len - len;	2614	return orig_len == len ? ret : orig_len - len;
2589	}	2615	}
		2616
		2617	/* Send skb data on a socket. Socket must be locked. */
		2618	int skb_send_sock_locked(struct sock sk, struct sk_buff skb, int offset,
		2619	int len)
		2620	{
		2621	return __skb_send_sock(sk, skb, offset, len, kernel_sendmsg_locked,
		2622	kernel_sendpage_locked);
		2623	}
2590	EXPORT_SYMBOL_GPL(skb_send_sock_locked);	2624	EXPORT_SYMBOL_GPL(skb_send_sock_locked);
2591		2625
		2626	/* Send skb data on a socket. Socket must be unlocked. */
		2627	int skb_send_sock(struct sock sk, struct sk_buff skb, int offset, int len)
		2628	{
		2629	return __skb_send_sock(sk, skb, offset, len, sendmsg_unlocked,
		2630	sendpage_unlocked);
		2631	}
		2632
2592	/**	2633	/**
2593	* skb_store_bits - store bits from kernel buffer to skb	2634	* skb_store_bits - store bits from kernel buffer to skb
2594	* @skb: destination buffer	2635	* @skb: destination buffer

net/core/skmsg.c+143−34

@@ -399,6 +399,104 @@ out: · 398 unmodified lines
399	}	399	}
400	EXPORT_SYMBOL_GPL(sk_msg_memcopy_from_iter);	400	EXPORT_SYMBOL_GPL(sk_msg_memcopy_from_iter);
401		401
		402	int sk_msg_wait_data(struct sock sk, struct sk_psock psock, int flags,
		403	long timeo, int *err)
		404	{
		405	DEFINE_WAIT_FUNC(wait, woken_wake_function);
		406	int ret = 0;
		407
		408	if (sk->sk_shutdown & RCV_SHUTDOWN)
		409	return 1;
		410
		411	if (!timeo)
		412	return ret;
		413
		414	add_wait_queue(sk_sleep(sk), &wait);
		415	sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
		416	ret = sk_wait_event(sk, &timeo,
		417	!list_empty(&psock->ingress_msg) \|\|
		418	!skb_queue_empty(&sk->sk_receive_queue), &wait);
		419	sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
		420	remove_wait_queue(sk_sleep(sk), &wait);
		421	return ret;
		422	}
		423	EXPORT_SYMBOL_GPL(sk_msg_wait_data);
		424
		425	/* Receive sk_msg from psock->ingress_msg to @msg. */
		426	int sk_msg_recvmsg(struct sock sk, struct sk_psock psock, struct msghdr *msg,
		427	int len, int flags)
		428	{
		429	struct iov_iter *iter = &msg->msg_iter;
		430	int peek = flags & MSG_PEEK;
		431	struct sk_msg *msg_rx;
		432	int i, copied = 0;
		433
		434	msg_rx = sk_psock_peek_msg(psock);
		435	while (copied != len) {
		436	struct scatterlist *sge;
		437
		438	if (unlikely(!msg_rx))
		439	break;
		440
		441	i = msg_rx->sg.start;
		442	do {
		443	struct page *page;
		444	int copy;
		445
		446	sge = sk_msg_elem(msg_rx, i);
		447	copy = sge->length;
		448	page = sg_page(sge);
		449	if (copied + copy > len)
		450	copy = len - copied;
		451	copy = copy_page_to_iter(page, sge->offset, copy, iter);
		452	if (!copy)
		453	return copied ? copied : -EFAULT;
		454
		455	copied += copy;
		456	if (likely(!peek)) {
		457	sge->offset += copy;
		458	sge->length -= copy;
		459	if (!msg_rx->skb)
		460	sk_mem_uncharge(sk, copy);
		461	msg_rx->sg.size -= copy;
		462
		463	if (!sge->length) {
		464	sk_msg_iter_var_next(i);
		465	if (!msg_rx->skb)
		466	put_page(page);
		467	}
		468	} else {
		469	/* Lets not optimize peek case if copy_page_to_iter
		470	* didn't copy the entire length lets just break.
		471	*/
		472	if (copy != sge->length)
		473	return copied;
		474	sk_msg_iter_var_next(i);
		475	}
		476
		477	if (copied == len)
		478	break;
		479	} while (i != msg_rx->sg.end);
		480
		481	if (unlikely(peek)) {
		482	msg_rx = sk_psock_next_msg(psock, msg_rx);
		483	if (!msg_rx)
		484	break;
		485	continue;
		486	}
		487
		488	msg_rx->sg.start = i;
		489	if (!sge->length && msg_rx->sg.start == msg_rx->sg.end) {
		490	msg_rx = sk_psock_dequeue_msg(psock);
		491	kfree_sk_msg(msg_rx);
		492	}
		493	msg_rx = sk_psock_peek_msg(psock);
		494	}
		495
		496	return copied;
		497	}
		498	EXPORT_SYMBOL_GPL(sk_msg_recvmsg);
		499
402	static struct sk_msg sk_psock_create_ingress_msg(struct sock sk,	500	static struct sk_msg sk_psock_create_ingress_msg(struct sock sk,
403	struct sk_buff *skb)	501	struct sk_buff *skb)
404	{	502	{
@@ -410,7 +508,7 @@ static struct sk_msg sk_psock_create_ingress_msg(struct sock sk, · 5 unmodified lines
410	if (!sk_rmem_schedule(sk, skb, skb->truesize))	508	if (!sk_rmem_schedule(sk, skb, skb->truesize))
411	return NULL;	509	return NULL;
412		510
413	msg = kzalloc(sizeof(*msg), __GFP_NOWARN \| GFP_ATOMIC);	511	msg = kzalloc(sizeof(*msg), __GFP_NOWARN \| GFP_KERNEL);
414	if (unlikely(!msg))	512	if (unlikely(!msg))
415	return NULL;	513	return NULL;
416		514
@@ -497,7 +595,7 @@ static int sk_psock_handle_skb(struct sk_psock psock, struct sk_buff skb, · 80 unmodified lines
497	if (!ingress) {	595	if (!ingress) {
498	if (!sock_writeable(psock->sk))	596	if (!sock_writeable(psock->sk))
499	return -EAGAIN;	597	return -EAGAIN;
500	return skb_send_sock_locked(psock->sk, skb, off, len);	598	return skb_send_sock(psock->sk, skb, off, len);
501	}	599	}
502	return sk_psock_skb_ingress(psock, skb);	600	return sk_psock_skb_ingress(psock, skb);
503	}	601	}
@@ -511,8 +609,7 @@ static void sk_psock_backlog(struct work_struct *work) · 7 unmodified lines
511	u32 len, off;	609	u32 len, off;
512	int ret;	610	int ret;
513		611
514	/* Lock sock to avoid losing sk_socket during loop. */	612	mutex_lock(&psock->work_mutex);
515	lock_sock(psock->sk);
516	if (state->skb) {	613	if (state->skb) {
517	skb = state->skb;	614	skb = state->skb;
518	len = state->len;	615	len = state->len;
@@ -529,7 +626,7 @@ start: · 10 unmodified lines
529	skb_bpf_redirect_clear(skb);	626	skb_bpf_redirect_clear(skb);
530	do {	627	do {
531	ret = -EIO;	628	ret = -EIO;
532	if (likely(psock->sk->sk_socket))	629	if (!sock_flag(psock->sk, SOCK_DEAD))
533	ret = sk_psock_handle_skb(psock, skb, off,	630	ret = sk_psock_handle_skb(psock, skb, off,
534	len, ingress);	631	len, ingress);
535	if (ret <= 0) {	632	if (ret <= 0) {
@@ -553,7 +650,7 @@ start: · 17 unmodified lines
553	kfree_skb(skb);	650	kfree_skb(skb);
554	}	651	}
555	end:	652	end:
556	release_sock(psock->sk);	653	mutex_unlock(&psock->work_mutex);
557	}	654	}
558		655
559	struct sk_psock sk_psock_init(struct sock sk, int node)	656	struct sk_psock sk_psock_init(struct sock sk, int node)
@@ -563,11 +660,6 @@ struct sk_psock sk_psock_init(struct sock sk, int node) · 3 unmodified lines
563		660
564	write_lock_bh(&sk->sk_callback_lock);	661	write_lock_bh(&sk->sk_callback_lock);
565		662
566	if (inet_csk_has_ulp(sk)) {
567	psock = ERR_PTR(-EINVAL);
568	goto out;
569	}
570
571	if (sk->sk_user_data) {	663	if (sk->sk_user_data) {
572	psock = ERR_PTR(-EBUSY);	664	psock = ERR_PTR(-EBUSY);
573	goto out;	665	goto out;
@@ -591,7 +683,9 @@ struct sk_psock sk_psock_init(struct sock sk, int node) · 17 unmodified lines
591	spin_lock_init(&psock->link_lock);	683	spin_lock_init(&psock->link_lock);
592		684
593	INIT_WORK(&psock->work, sk_psock_backlog);	685	INIT_WORK(&psock->work, sk_psock_backlog);
		686	mutex_init(&psock->work_mutex);
594	INIT_LIST_HEAD(&psock->ingress_msg);	687	INIT_LIST_HEAD(&psock->ingress_msg);
		688	spin_lock_init(&psock->ingress_lock);
595	skb_queue_head_init(&psock->ingress_skb);	689	skb_queue_head_init(&psock->ingress_skb);
596		690
597	sk_psock_set_state(psock, SK_PSOCK_TX_ENABLED);	691	sk_psock_set_state(psock, SK_PSOCK_TX_ENABLED);
@@ -630,11 +724,11 @@ static void __sk_psock_purge_ingress_msg(struct sk_psock *psock) · 32 unmodified lines
630	}	724	}
631	}	725	}
632		726
633	static void sk_psock_zap_ingress(struct sk_psock *psock)	727	static void __sk_psock_zap_ingress(struct sk_psock *psock)
634	{	728	{
635	struct sk_buff *skb;	729	struct sk_buff *skb;
636		730
637	while ((skb = __skb_dequeue(&psock->ingress_skb)) != NULL) {	731	while ((skb = skb_dequeue(&psock->ingress_skb)) != NULL) {
638	skb_bpf_redirect_clear(skb);	732	skb_bpf_redirect_clear(skb);
639	kfree_skb(skb);	733	kfree_skb(skb);
640	}	734	}
@@ -651,23 +745,35 @@ static void sk_psock_link_destroy(struct sk_psock *psock) · 10 unmodified lines
651	}	745	}
652	}	746	}
653		747
		748	void sk_psock_stop(struct sk_psock *psock, bool wait)
		749	{
		750	spin_lock_bh(&psock->ingress_lock);
		751	sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);
		752	sk_psock_cork_free(psock);
		753	__sk_psock_zap_ingress(psock);
		754	spin_unlock_bh(&psock->ingress_lock);
		755
		756	if (wait)
		757	cancel_work_sync(&psock->work);
		758	}
		759
654	static void sk_psock_done_strp(struct sk_psock *psock);	760	static void sk_psock_done_strp(struct sk_psock *psock);
655		761
656	static void sk_psock_destroy_deferred(struct work_struct *gc)	762	static void sk_psock_destroy(struct work_struct *work)
657	{	763	{
658	struct sk_psock *psock = container_of(gc, struct sk_psock, gc);	764	struct sk_psock *psock = container_of(to_rcu_work(work),
659		765	struct sk_psock, rwork);
660	/* No sk_callback_lock since already detached. */	766	/* No sk_callback_lock since already detached. */
661		767
662	sk_psock_done_strp(psock);	768	sk_psock_done_strp(psock);
663		769
664	cancel_work_sync(&psock->work);	770	cancel_work_sync(&psock->work);
		771	mutex_destroy(&psock->work_mutex);
665		772
666	psock_progs_drop(&psock->progs);	773	psock_progs_drop(&psock->progs);
667		774
668	sk_psock_link_destroy(psock);	775	sk_psock_link_destroy(psock);
669	sk_psock_cork_free(psock);	776	sk_psock_cork_free(psock);
670	sk_psock_zap_ingress(psock);
671		777
672	if (psock->sk_redir)	778	if (psock->sk_redir)
673	sock_put(psock->sk_redir);	779	sock_put(psock->sk_redir);
@@ -675,30 +781,21 @@ static void sk_psock_destroy_deferred(struct work_struct *gc) · 1 unmodified line
675	kfree(psock);	781	kfree(psock);
676	}	782	}
677		783
678	static void sk_psock_destroy(struct rcu_head *rcu)
679	{
680	struct sk_psock *psock = container_of(rcu, struct sk_psock, rcu);
681
682	INIT_WORK(&psock->gc, sk_psock_destroy_deferred);
683	schedule_work(&psock->gc);
684	}
685
686	void sk_psock_drop(struct sock sk, struct sk_psock psock)	784	void sk_psock_drop(struct sock sk, struct sk_psock psock)
687	{	785	{
688	sk_psock_cork_free(psock);	786	sk_psock_stop(psock, false);
689	sk_psock_zap_ingress(psock);
690		787
691	write_lock_bh(&sk->sk_callback_lock);	788	write_lock_bh(&sk->sk_callback_lock);
692	sk_psock_restore_proto(sk, psock);	789	sk_psock_restore_proto(sk, psock);
693	rcu_assign_sk_user_data(sk, NULL);	790	rcu_assign_sk_user_data(sk, NULL);
694	if (psock->progs.stream_parser)	791	if (psock->progs.stream_parser)
695	sk_psock_stop_strp(sk, psock);	792	sk_psock_stop_strp(sk, psock);
696	else if (psock->progs.stream_verdict)	793	else if (psock->progs.stream_verdict \|\| psock->progs.skb_verdict)
697	sk_psock_stop_verdict(sk, psock);	794	sk_psock_stop_verdict(sk, psock);
698	write_unlock_bh(&sk->sk_callback_lock);	795	write_unlock_bh(&sk->sk_callback_lock);
699	sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);
700		796
701	call_rcu(&psock->rcu, sk_psock_destroy);	797	INIT_RCU_WORK(&psock->rwork, sk_psock_destroy);
		798	queue_rcu_work(system_wq, &psock->rwork);
702	}	799	}
703	EXPORT_SYMBOL_GPL(sk_psock_drop);	800	EXPORT_SYMBOL_GPL(sk_psock_drop);
704		801
@@ -767,14 +864,20 @@ static void sk_psock_skb_redirect(struct sk_buff *skb) · 62 unmodified lines
767	* error that caused the pipe to break. We can't send a packet on	864	* error that caused the pipe to break. We can't send a packet on
768	* a socket that is in this state so we drop the skb.	865	* a socket that is in this state so we drop the skb.
769	*/	866	*/
770	if (!psock_other \|\| sock_flag(sk_other, SOCK_DEAD) \|\|	867	if (!psock_other \|\| sock_flag(sk_other, SOCK_DEAD)) {
771	!sk_psock_test_state(psock_other, SK_PSOCK_TX_ENABLED)) {	868	kfree_skb(skb);
		869	return;
		870	}
		871	spin_lock_bh(&psock_other->ingress_lock);
		872	if (!sk_psock_test_state(psock_other, SK_PSOCK_TX_ENABLED)) {
		873	spin_unlock_bh(&psock_other->ingress_lock);
772	kfree_skb(skb);	874	kfree_skb(skb);
773	return;	875	return;
774	}	876	}
775		877
776	skb_queue_tail(&psock_other->ingress_skb, skb);	878	skb_queue_tail(&psock_other->ingress_skb, skb);
777	schedule_work(&psock_other->work);	879	schedule_work(&psock_other->work);
		880	spin_unlock_bh(&psock_other->ingress_lock);
778	}	881	}
779		882
780	static void sk_psock_tls_verdict_apply(struct sk_buff skb, struct sock sk, int verdict)	883	static void sk_psock_tls_verdict_apply(struct sk_buff skb, struct sock sk, int verdict)
@@ -842,8 +945,12 @@ static void sk_psock_verdict_apply(struct sk_psock *psock, · 61 unmodified lines
842	err = sk_psock_skb_ingress_self(psock, skb);	945	err = sk_psock_skb_ingress_self(psock, skb);
843	}	946	}
844	if (err < 0) {	947	if (err < 0) {
845	skb_queue_tail(&psock->ingress_skb, skb);	948	spin_lock_bh(&psock->ingress_lock);
846	schedule_work(&psock->work);	949	if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) {
		950	skb_queue_tail(&psock->ingress_skb, skb);
		951	schedule_work(&psock->work);
		952	}
		953	spin_unlock_bh(&psock->ingress_lock);
847	}	954	}
848	break;	955	break;
849	case __SK_REDIRECT:	956	case __SK_REDIRECT:
@@ -1010,6 +1117,8 @@ static int sk_psock_verdict_recv(read_descriptor_t desc, struct sk_buff skb, · 160 unmodified lines
1010	}	1117	}
1011	skb_set_owner_r(skb, sk);	1118	skb_set_owner_r(skb, sk);
1012	prog = READ_ONCE(psock->progs.stream_verdict);	1119	prog = READ_ONCE(psock->progs.stream_verdict);
		1120	if (!prog)
		1121	prog = READ_ONCE(psock->progs.skb_verdict);
1013	if (likely(prog)) {	1122	if (likely(prog)) {
1014	skb_dst_drop(skb);	1123	skb_dst_drop(skb);
1015	skb_bpf_redirect_clear(skb);	1124	skb_bpf_redirect_clear(skb);

net/core/sock_map.c+55−63

@@ -26,6 +26,7 @@ struct bpf_stab { · 25 unmodified lines
26		26
27	static int sock_map_prog_update(struct bpf_map map, struct bpf_prog prog,	27	static int sock_map_prog_update(struct bpf_map map, struct bpf_prog prog,
28	struct bpf_prog *old, u32 which);	28	struct bpf_prog *old, u32 which);
		29	static struct sk_psock_progs sock_map_progs(struct bpf_map map);
29		30
30	static struct bpf_map sock_map_alloc(union bpf_attr attr)	31	static struct bpf_map sock_map_alloc(union bpf_attr attr)
31	{	32	{
@@ -155,6 +156,8 @@ static void sock_map_del_link(struct sock *sk, · 123 unmodified lines
155	strp_stop = true;	156	strp_stop = true;
156	if (psock->saved_data_ready && stab->progs.stream_verdict)	157	if (psock->saved_data_ready && stab->progs.stream_verdict)
157	verdict_stop = true;	158	verdict_stop = true;
		159	if (psock->saved_data_ready && stab->progs.skb_verdict)
		160	verdict_stop = true;
158	list_del(&link->list);	161	list_del(&link->list);
159	sk_psock_free_link(link);	162	sk_psock_free_link(link);
160	}	163	}
@@ -182,26 +185,10 @@ static void sock_map_unref(struct sock sk, void link_raw) · 21 unmodified lines
182		185
183	static int sock_map_init_proto(struct sock sk, struct sk_psock psock)	186	static int sock_map_init_proto(struct sock sk, struct sk_psock psock)
184	{	187	{
185	struct proto *prot;	188	if (!sk->sk_prot->psock_update_sk_prot)
186
187	switch (sk->sk_type) {
188	case SOCK_STREAM:
189	prot = tcp_bpf_get_proto(sk, psock);
190	break;
191
192	case SOCK_DGRAM:
193	prot = udp_bpf_get_proto(sk, psock);
194	break;
195
196	default:
197	return -EINVAL;	189	return -EINVAL;
198	}	190	psock->psock_update_sk_prot = sk->sk_prot->psock_update_sk_prot;
199		191	return sk->sk_prot->psock_update_sk_prot(sk, false);
200	if (IS_ERR(prot))
201	return PTR_ERR(prot);
202
203	sk_psock_update_proto(sk, psock, prot);
204	return 0;
205	}	192	}
206		193
207	static struct sk_psock sock_map_psock_get_checked(struct sock sk)	194	static struct sk_psock sock_map_psock_get_checked(struct sock sk)
@@ -224,13 +211,25 @@ out: · 16 unmodified lines
224	return psock;	211	return psock;
225	}	212	}
226		213
227	static int sock_map_link(struct bpf_map map, struct sk_psock_progs progs,	214	static bool sock_map_redirect_allowed(const struct sock *sk);
228	struct sock *sk)	215
		216	static int sock_map_link(struct bpf_map map, struct sock sk)
229	{	217	{
230	struct bpf_prog msg_parser, stream_parser, *stream_verdict;	218	struct sk_psock_progs *progs = sock_map_progs(map);
		219	struct bpf_prog *stream_verdict = NULL;
		220	struct bpf_prog *stream_parser = NULL;
		221	struct bpf_prog *skb_verdict = NULL;
		222	struct bpf_prog *msg_parser = NULL;
231	struct sk_psock *psock;	223	struct sk_psock *psock;
232	int ret;	224	int ret;
233		225
		226	/* Only sockets we can redirect into/from in BPF need to hold
		227	* refs to parser/verdict progs and have their sk_data_ready
		228	* and sk_write_space callbacks overridden.
		229	*/
		230	if (!sock_map_redirect_allowed(sk))
		231	goto no_progs;
		232
234	stream_verdict = READ_ONCE(progs->stream_verdict);	233	stream_verdict = READ_ONCE(progs->stream_verdict);
235	if (stream_verdict) {	234	if (stream_verdict) {
236	stream_verdict = bpf_prog_inc_not_zero(stream_verdict);	235	stream_verdict = bpf_prog_inc_not_zero(stream_verdict);
@@ -256,6 +255,16 @@ static int sock_map_link(struct bpf_map map, struct sk_psock_progs progs, · 19 unmodified lines
256	}	255	}
257	}	256	}
258		257
		258	skb_verdict = READ_ONCE(progs->skb_verdict);
		259	if (skb_verdict) {
		260	skb_verdict = bpf_prog_inc_not_zero(skb_verdict);
		261	if (IS_ERR(skb_verdict)) {
		262	ret = PTR_ERR(skb_verdict);
		263	goto out_put_msg_parser;
		264	}
		265	}
		266
		267	no_progs:
259	psock = sock_map_psock_get_checked(sk);	268	psock = sock_map_psock_get_checked(sk);
260	if (IS_ERR(psock)) {	269	if (IS_ERR(psock)) {
261	ret = PTR_ERR(psock);	270	ret = PTR_ERR(psock);
@@ -265,6 +274,9 @@ static int sock_map_link(struct bpf_map map, struct sk_psock_progs progs, · 3 unmodified lines
265	if (psock) {	274	if (psock) {
266	if ((msg_parser && READ_ONCE(psock->progs.msg_parser)) \|\|	275	if ((msg_parser && READ_ONCE(psock->progs.msg_parser)) \|\|
267	(stream_parser && READ_ONCE(psock->progs.stream_parser)) \|\|	276	(stream_parser && READ_ONCE(psock->progs.stream_parser)) \|\|
		277	(skb_verdict && READ_ONCE(psock->progs.skb_verdict)) \|\|
		278	(skb_verdict && READ_ONCE(psock->progs.stream_verdict)) \|\|
		279	(stream_verdict && READ_ONCE(psock->progs.skb_verdict)) \|\|
268	(stream_verdict && READ_ONCE(psock->progs.stream_verdict))) {	280	(stream_verdict && READ_ONCE(psock->progs.stream_verdict))) {
269	sk_psock_put(sk, psock);	281	sk_psock_put(sk, psock);
270	ret = -EBUSY;	282	ret = -EBUSY;
@@ -296,6 +308,9 @@ static int sock_map_link(struct bpf_map map, struct sk_psock_progs progs, · 25 unmodified lines
296	} else if (!stream_parser && stream_verdict && !psock->saved_data_ready) {	308	} else if (!stream_parser && stream_verdict && !psock->saved_data_ready) {
297	psock_set_prog(&psock->progs.stream_verdict, stream_verdict);	309	psock_set_prog(&psock->progs.stream_verdict, stream_verdict);
298	sk_psock_start_verdict(sk,psock);	310	sk_psock_start_verdict(sk,psock);
		311	} else if (!stream_verdict && skb_verdict && !psock->saved_data_ready) {
		312	psock_set_prog(&psock->progs.skb_verdict, skb_verdict);
		313	sk_psock_start_verdict(sk, psock);
299	}	314	}
300	write_unlock_bh(&sk->sk_callback_lock);	315	write_unlock_bh(&sk->sk_callback_lock);
301	return 0;	316	return 0;
@@ -304,6 +319,9 @@ out_unlock_drop: · 2 unmodified lines
304	out_drop:	319	out_drop:
305	sk_psock_put(sk, psock);	320	sk_psock_put(sk, psock);
306	out_progs:	321	out_progs:
		322	if (skb_verdict)
		323	bpf_prog_put(skb_verdict);
		324	out_put_msg_parser:
307	if (msg_parser)	325	if (msg_parser)
308	bpf_prog_put(msg_parser);	326	bpf_prog_put(msg_parser);
309	out_put_stream_parser:	327	out_put_stream_parser:
@@ -315,27 +333,6 @@ out_put_stream_verdict: · 5 unmodified lines
315	return ret;	333	return ret;
316	}	334	}
317		335
318	static int sock_map_link_no_progs(struct bpf_map map, struct sock sk)
319	{
320	struct sk_psock *psock;
321	int ret;
322
323	psock = sock_map_psock_get_checked(sk);
324	if (IS_ERR(psock))
325	return PTR_ERR(psock);
326
327	if (!psock) {
328	psock = sk_psock_init(sk, map->numa_node);
329	if (IS_ERR(psock))
330	return PTR_ERR(psock);
331	}
332
333	ret = sock_map_init_proto(sk, psock);
334	if (ret < 0)
335	sk_psock_put(sk, psock);
336	return ret;
337	}
338
339	static void sock_map_free(struct bpf_map *map)	336	static void sock_map_free(struct bpf_map *map)
340	{	337	{
341	struct bpf_stab *stab = container_of(map, struct bpf_stab, map);	338	struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
@@ -466,8 +463,6 @@ static int sock_map_get_next_key(struct bpf_map map, void key, void *next) · 124 unmodified lines
466	return 0;	463	return 0;
467	}	464	}
468		465
469	static bool sock_map_redirect_allowed(const struct sock *sk);
470
471	static int sock_map_update_common(struct bpf_map *map, u32 idx,	466	static int sock_map_update_common(struct bpf_map *map, u32 idx,
472	struct sock *sk, u64 flags)	467	struct sock *sk, u64 flags)
473	{	468	{
@@ -487,14 +482,7 @@ static int sock_map_update_common(struct bpf_map *map, u32 idx, · 13 unmodified lines
487	if (!link)	482	if (!link)
488	return -ENOMEM;	483	return -ENOMEM;
489		484
490	/* Only sockets we can redirect into/from in BPF need to hold	485	ret = sock_map_link(map, sk);
491	* refs to parser/verdict progs and have their sk_data_ready
492	* and sk_write_space callbacks overridden.
493	*/
494	if (sock_map_redirect_allowed(sk))
495	ret = sock_map_link(map, &stab->progs, sk);
496	else
497	ret = sock_map_link_no_progs(map, sk);
498	if (ret < 0)	486	if (ret < 0)
499	goto out_free;	487	goto out_free;
500		488
@@ -547,12 +535,15 @@ static bool sk_is_udp(const struct sock *sk) · 46 unmodified lines
547		535
548	static bool sock_map_redirect_allowed(const struct sock *sk)	536	static bool sock_map_redirect_allowed(const struct sock *sk)
549	{	537	{
550	return sk_is_tcp(sk) && sk->sk_state != TCP_LISTEN;	538	if (sk_is_tcp(sk))
		539	return sk->sk_state != TCP_LISTEN;
		540	else
		541	return sk->sk_state == TCP_ESTABLISHED;
551	}	542	}
552		543
553	static bool sock_map_sk_is_suitable(const struct sock *sk)	544	static bool sock_map_sk_is_suitable(const struct sock *sk)
554	{	545	{
555	return sk_is_tcp(sk) \|\| sk_is_udp(sk);	546	return !!sk->sk_prot->psock_update_sk_prot;
556	}	547	}
557		548
558	static bool sock_map_sk_state_allowed(const struct sock *sk)	549	static bool sock_map_sk_state_allowed(const struct sock *sk)
@@ -999,14 +990,7 @@ static int sock_hash_update_common(struct bpf_map map, void key, · 440 unmodified lines
999	if (!link)	990	if (!link)
1000	return -ENOMEM;	991	return -ENOMEM;
1001		992
1002	/* Only sockets we can redirect into/from in BPF need to hold	993	ret = sock_map_link(map, sk);
1003	* refs to parser/verdict progs and have their sk_data_ready
1004	* and sk_write_space callbacks overridden.
1005	*/
1006	if (sock_map_redirect_allowed(sk))
1007	ret = sock_map_link(map, &htab->progs, sk);
1008	else
1009	ret = sock_map_link_no_progs(map, sk);
1010	if (ret < 0)	994	if (ret < 0)
1011	goto out_free;	995	goto out_free;
1012		996
@@ -1466,8 +1450,15 @@ static int sock_map_prog_update(struct bpf_map map, struct bpf_prog prog, · 453 unmodified lines
1466	break;	1450	break;
1467	#endif	1451	#endif
1468	case BPF_SK_SKB_STREAM_VERDICT:	1452	case BPF_SK_SKB_STREAM_VERDICT:
		1453	if (progs->skb_verdict)
		1454	return -EBUSY;
1469	pprog = &progs->stream_verdict;	1455	pprog = &progs->stream_verdict;
1470	break;	1456	break;
		1457	case BPF_SK_SKB_VERDICT:
		1458	if (progs->stream_verdict)
		1459	return -EBUSY;
		1460	pprog = &progs->skb_verdict;
		1461	break;
1471	default:	1462	default:
1472	return -EOPNOTSUPP;	1463	return -EOPNOTSUPP;
1473	}	1464	}
@@ -1540,6 +1531,7 @@ void sock_map_close(struct sock *sk, long timeout) · 66 unmodified lines
1540	saved_close = psock->saved_close;	1531	saved_close = psock->saved_close;
1541	sock_map_remove_links(sk, psock);	1532	sock_map_remove_links(sk, psock);
1542	rcu_read_unlock();	1533	rcu_read_unlock();
		1534	sk_psock_stop(psock, true);
1543	release_sock(sk);	1535	release_sock(sk);
1544	saved_close(sk, timeout);	1536	saved_close(sk, timeout);
1545	}	1537	}

net/ipv4/af_inet.c+1−0

@@ -1070,6 +1070,7 @@ const struct proto_ops inet_dgram_ops = { · 1069 unmodified lines
1070	.setsockopt = sock_common_setsockopt,	1070	.setsockopt = sock_common_setsockopt,
1071	.getsockopt = sock_common_getsockopt,	1071	.getsockopt = sock_common_getsockopt,
1072	.sendmsg = inet_sendmsg,	1072	.sendmsg = inet_sendmsg,
		1073	.read_sock = udp_read_sock,
1073	.recvmsg = inet_recvmsg,	1074	.recvmsg = inet_recvmsg,
1074	.mmap = sock_no_mmap,	1075	.mmap = sock_no_mmap,
1075	.sendpage = inet_sendpage,	1076	.sendpage = inet_sendpage,

net/ipv4/bpf_tcp_ca.c+43−0

@@ -5,6 +5,7 @@ · 4 unmodified lines
5	#include <linux/bpf_verifier.h>	5	#include <linux/bpf_verifier.h>
6	#include <linux/bpf.h>	6	#include <linux/bpf.h>
7	#include <linux/btf.h>	7	#include <linux/btf.h>
		8	#include <linux/btf_ids.h>
8	#include <linux/filter.h>	9	#include <linux/filter.h>
9	#include <net/tcp.h>	10	#include <net/tcp.h>
10	#include <net/bpf_sk_storage.h>	11	#include <net/bpf_sk_storage.h>
@@ -178,10 +179,52 @@ bpf_tcp_ca_get_func_proto(enum bpf_func_id func_id, · 167 unmodified lines
178	}	179	}
179	}	180	}
180		181
		182	BTF_SET_START(bpf_tcp_ca_kfunc_ids)
		183	BTF_ID(func, tcp_reno_ssthresh)
		184	BTF_ID(func, tcp_reno_cong_avoid)
		185	BTF_ID(func, tcp_reno_undo_cwnd)
		186	BTF_ID(func, tcp_slow_start)
		187	BTF_ID(func, tcp_cong_avoid_ai)
		188	#ifdef CONFIG_DYNAMIC_FTRACE
		189	#if IS_BUILTIN(CONFIG_TCP_CONG_CUBIC)
		190	BTF_ID(func, cubictcp_init)
		191	BTF_ID(func, cubictcp_recalc_ssthresh)
		192	BTF_ID(func, cubictcp_cong_avoid)
		193	BTF_ID(func, cubictcp_state)
		194	BTF_ID(func, cubictcp_cwnd_event)
		195	BTF_ID(func, cubictcp_acked)
		196	#endif
		197	#if IS_BUILTIN(CONFIG_TCP_CONG_DCTCP)
		198	BTF_ID(func, dctcp_init)
		199	BTF_ID(func, dctcp_update_alpha)
		200	BTF_ID(func, dctcp_cwnd_event)
		201	BTF_ID(func, dctcp_ssthresh)
		202	BTF_ID(func, dctcp_cwnd_undo)
		203	BTF_ID(func, dctcp_state)
		204	#endif
		205	#if IS_BUILTIN(CONFIG_TCP_CONG_BBR)
		206	BTF_ID(func, bbr_init)
		207	BTF_ID(func, bbr_main)
		208	BTF_ID(func, bbr_sndbuf_expand)
		209	BTF_ID(func, bbr_undo_cwnd)
		210	BTF_ID(func, bbr_cwnd_event)
		211	BTF_ID(func, bbr_ssthresh)
		212	BTF_ID(func, bbr_min_tso_segs)
		213	BTF_ID(func, bbr_set_state)
		214	#endif
		215	#endif /* CONFIG_DYNAMIC_FTRACE */
		216	BTF_SET_END(bpf_tcp_ca_kfunc_ids)
		217
		218	static bool bpf_tcp_ca_check_kfunc_call(u32 kfunc_btf_id)
		219	{
		220	return btf_id_set_contains(&bpf_tcp_ca_kfunc_ids, kfunc_btf_id);
		221	}
		222
181	static const struct bpf_verifier_ops bpf_tcp_ca_verifier_ops = {	223	static const struct bpf_verifier_ops bpf_tcp_ca_verifier_ops = {
182	.get_func_proto = bpf_tcp_ca_get_func_proto,	224	.get_func_proto = bpf_tcp_ca_get_func_proto,
183	.is_valid_access = bpf_tcp_ca_is_valid_access,	225	.is_valid_access = bpf_tcp_ca_is_valid_access,
184	.btf_struct_access = bpf_tcp_ca_btf_struct_access,	226	.btf_struct_access = bpf_tcp_ca_btf_struct_access,
		227	.check_kfunc_call = bpf_tcp_ca_check_kfunc_call,
185	};	228	};
186		229
187	static int bpf_tcp_ca_init_member(const struct btf_type *t,	230	static int bpf_tcp_ca_init_member(const struct btf_type *t,

net/ipv4/tcp_bpf.c+23−107

@@ -10,86 +10,6 @@ · 9 unmodified lines
10	#include <net/inet_common.h>	10	#include <net/inet_common.h>
11	#include <net/tls.h>	11	#include <net/tls.h>
12		12
13	int __tcp_bpf_recvmsg(struct sock sk, struct sk_psock psock,
14	struct msghdr *msg, int len, int flags)
15	{
16	struct iov_iter *iter = &msg->msg_iter;
17	int peek = flags & MSG_PEEK;
18	struct sk_msg *msg_rx;
19	int i, copied = 0;
20
21	msg_rx = list_first_entry_or_null(&psock->ingress_msg,
22	struct sk_msg, list);
23
24	while (copied != len) {
25	struct scatterlist *sge;
26
27	if (unlikely(!msg_rx))
28	break;
29
30	i = msg_rx->sg.start;
31	do {
32	struct page *page;
33	int copy;
34
35	sge = sk_msg_elem(msg_rx, i);
36	copy = sge->length;
37	page = sg_page(sge);
38	if (copied + copy > len)
39	copy = len - copied;
40	copy = copy_page_to_iter(page, sge->offset, copy, iter);
41	if (!copy)
42	return copied ? copied : -EFAULT;
43
44	copied += copy;
45	if (likely(!peek)) {
46	sge->offset += copy;
47	sge->length -= copy;
48	if (!msg_rx->skb)
49	sk_mem_uncharge(sk, copy);
50	msg_rx->sg.size -= copy;
51
52	if (!sge->length) {
53	sk_msg_iter_var_next(i);
54	if (!msg_rx->skb)
55	put_page(page);
56	}
57	} else {
58	/* Lets not optimize peek case if copy_page_to_iter
59	* didn't copy the entire length lets just break.
60	*/
61	if (copy != sge->length)
62	return copied;
63	sk_msg_iter_var_next(i);
64	}
65
66	if (copied == len)
67	break;
68	} while (i != msg_rx->sg.end);
69
70	if (unlikely(peek)) {
71	if (msg_rx == list_last_entry(&psock->ingress_msg,
72	struct sk_msg, list))
73	break;
74	msg_rx = list_next_entry(msg_rx, list);
75	continue;
76	}
77
78	msg_rx->sg.start = i;
79	if (!sge->length && msg_rx->sg.start == msg_rx->sg.end) {
80	list_del(&msg_rx->list);
81	if (msg_rx->skb)
82	consume_skb(msg_rx->skb);
83	kfree(msg_rx);
84	}
85	msg_rx = list_first_entry_or_null(&psock->ingress_msg,
86	struct sk_msg, list);
87	}
88
89	return copied;
90	}
91	EXPORT_SYMBOL_GPL(__tcp_bpf_recvmsg);
92
93	static int bpf_tcp_ingress(struct sock sk, struct sk_psock psock,	13	static int bpf_tcp_ingress(struct sock sk, struct sk_psock psock,
94	struct sk_msg *msg, u32 apply_bytes, int flags)	14	struct sk_msg *msg, u32 apply_bytes, int flags)
95	{	15	{
@@ -243,28 +163,6 @@ static bool tcp_bpf_stream_read(const struct sock *sk) · 147 unmodified lines
243	return !empty;	163	return !empty;
244	}	164	}
245		165
246	static int tcp_bpf_wait_data(struct sock sk, struct sk_psock psock,
247	int flags, long timeo, int *err)
248	{
249	DEFINE_WAIT_FUNC(wait, woken_wake_function);
250	int ret = 0;
251
252	if (sk->sk_shutdown & RCV_SHUTDOWN)
253	return 1;
254
255	if (!timeo)
256	return ret;
257
258	add_wait_queue(sk_sleep(sk), &wait);
259	sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
260	ret = sk_wait_event(sk, &timeo,
261	!list_empty(&psock->ingress_msg) \|\|
262	!skb_queue_empty(&sk->sk_receive_queue), &wait);
263	sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
264	remove_wait_queue(sk_sleep(sk), &wait);
265	return ret;
266	}
267
268	static int tcp_bpf_recvmsg(struct sock sk, struct msghdr msg, size_t len,	166	static int tcp_bpf_recvmsg(struct sock sk, struct msghdr msg, size_t len,
269	int nonblock, int flags, int *addr_len)	167	int nonblock, int flags, int *addr_len)
270	{	168	{
@@ -284,13 +182,13 @@ static int tcp_bpf_recvmsg(struct sock sk, struct msghdr msg, size_t len, · 13 unmodified lines
284	}	182	}
285	lock_sock(sk);	183	lock_sock(sk);
286	msg_bytes_ready:	184	msg_bytes_ready:
287	copied = __tcp_bpf_recvmsg(sk, psock, msg, len, flags);	185	copied = sk_msg_recvmsg(sk, psock, msg, len, flags);
288	if (!copied) {	186	if (!copied) {
289	int data, err = 0;	187	int data, err = 0;
290	long timeo;	188	long timeo;
291		189
292	timeo = sock_rcvtimeo(sk, nonblock);	190	timeo = sock_rcvtimeo(sk, nonblock);
293	data = tcp_bpf_wait_data(sk, psock, flags, timeo, &err);	191	data = sk_msg_wait_data(sk, psock, flags, timeo, &err);
294	if (data) {	192	if (data) {
295	if (!sk_psock_queue_empty(psock))	193	if (!sk_psock_queue_empty(psock))
296	goto msg_bytes_ready;	194	goto msg_bytes_ready;
@@ -601,20 +499,38 @@ static int tcp_bpf_assert_proto_ops(struct proto *ops) · 304 unmodified lines
601	ops->sendpage == tcp_sendpage ? 0 : -ENOTSUPP;	499	ops->sendpage == tcp_sendpage ? 0 : -ENOTSUPP;
602	}	500	}
603		501
604	struct proto tcp_bpf_get_proto(struct sock sk, struct sk_psock *psock)	502	int tcp_bpf_update_proto(struct sock *sk, bool restore)
605	{	503	{
		504	struct sk_psock *psock = sk_psock(sk);
606	int family = sk->sk_family == AF_INET6 ? TCP_BPF_IPV6 : TCP_BPF_IPV4;	505	int family = sk->sk_family == AF_INET6 ? TCP_BPF_IPV6 : TCP_BPF_IPV4;
607	int config = psock->progs.msg_parser ? TCP_BPF_TX : TCP_BPF_BASE;	506	int config = psock->progs.msg_parser ? TCP_BPF_TX : TCP_BPF_BASE;
608		507
		508	if (restore) {
		509	if (inet_csk_has_ulp(sk)) {
		510	tcp_update_ulp(sk, psock->sk_proto, psock->saved_write_space);
		511	} else {
		512	sk->sk_write_space = psock->saved_write_space;
		513	/* Pairs with lockless read in sk_clone_lock() */
		514	WRITE_ONCE(sk->sk_prot, psock->sk_proto);
		515	}
		516	return 0;
		517	}
		518
		519	if (inet_csk_has_ulp(sk))
		520	return -EINVAL;
		521
609	if (sk->sk_family == AF_INET6) {	522	if (sk->sk_family == AF_INET6) {
610	if (tcp_bpf_assert_proto_ops(psock->sk_proto))	523	if (tcp_bpf_assert_proto_ops(psock->sk_proto))
611	return ERR_PTR(-EINVAL);	524	return -EINVAL;
612		525
613	tcp_bpf_check_v6_needs_rebuild(psock->sk_proto);	526	tcp_bpf_check_v6_needs_rebuild(psock->sk_proto);
614	}	527	}
615		528
616	return &tcp_bpf_prots[family][config];	529	/* Pairs with lockless read in sk_clone_lock() */
		530	WRITE_ONCE(sk->sk_prot, &tcp_bpf_prots[family][config]);
		531	return 0;
617	}	532	}
		533	EXPORT_SYMBOL_GPL(tcp_bpf_update_proto);
618		534
619	/* If a child got cloned from a listening socket that had tcp_bpf	535	/* If a child got cloned from a listening socket that had tcp_bpf
620	* protocol callbacks installed, we need to restore the callbacks to	536	* protocol callbacks installed, we need to restore the callbacks to

net/ipv4/tcp_cubic.c+12−12

@@ -124,7 +124,7 @@ static inline void bictcp_hystart_reset(struct sock *sk) · 123 unmodified lines
124	ca->sample_cnt = 0;	124	ca->sample_cnt = 0;
125	}	125	}
126		126
127	static void bictcp_init(struct sock *sk)	127	static void cubictcp_init(struct sock *sk)
128	{	128	{
129	struct bictcp *ca = inet_csk_ca(sk);	129	struct bictcp *ca = inet_csk_ca(sk);
130		130
@@ -137,7 +137,7 @@ static void bictcp_init(struct sock *sk) · 6 unmodified lines
137	tcp_sk(sk)->snd_ssthresh = initial_ssthresh;	137	tcp_sk(sk)->snd_ssthresh = initial_ssthresh;
138	}	138	}
139		139
140	static void bictcp_cwnd_event(struct sock *sk, enum tcp_ca_event event)	140	static void cubictcp_cwnd_event(struct sock *sk, enum tcp_ca_event event)
141	{	141	{
142	if (event == CA_EVENT_TX_START) {	142	if (event == CA_EVENT_TX_START) {
143	struct bictcp *ca = inet_csk_ca(sk);	143	struct bictcp *ca = inet_csk_ca(sk);
@@ -319,7 +319,7 @@ tcp_friendliness: · 175 unmodified lines
319	ca->cnt = max(ca->cnt, 2U);	319	ca->cnt = max(ca->cnt, 2U);
320	}	320	}
321		321
322	static void bictcp_cong_avoid(struct sock *sk, u32 ack, u32 acked)	322	static void cubictcp_cong_avoid(struct sock *sk, u32 ack, u32 acked)
323	{	323	{
324	struct tcp_sock *tp = tcp_sk(sk);	324	struct tcp_sock *tp = tcp_sk(sk);
325	struct bictcp *ca = inet_csk_ca(sk);	325	struct bictcp *ca = inet_csk_ca(sk);
@@ -338,7 +338,7 @@ static void bictcp_cong_avoid(struct sock *sk, u32 ack, u32 acked) · 12 unmodified lines
338	tcp_cong_avoid_ai(tp, ca->cnt, acked);	338	tcp_cong_avoid_ai(tp, ca->cnt, acked);
339	}	339	}
340		340
341	static u32 bictcp_recalc_ssthresh(struct sock *sk)	341	static u32 cubictcp_recalc_ssthresh(struct sock *sk)
342	{	342	{
343	const struct tcp_sock *tp = tcp_sk(sk);	343	const struct tcp_sock *tp = tcp_sk(sk);
344	struct bictcp *ca = inet_csk_ca(sk);	344	struct bictcp *ca = inet_csk_ca(sk);
@@ -355,7 +355,7 @@ static u32 bictcp_recalc_ssthresh(struct sock *sk) · 10 unmodified lines
355	return max((tp->snd_cwnd * beta) / BICTCP_BETA_SCALE, 2U);	355	return max((tp->snd_cwnd * beta) / BICTCP_BETA_SCALE, 2U);
356	}	356	}
357		357
358	static void bictcp_state(struct sock *sk, u8 new_state)	358	static void cubictcp_state(struct sock *sk, u8 new_state)
359	{	359	{
360	if (new_state == TCP_CA_Loss) {	360	if (new_state == TCP_CA_Loss) {
361	bictcp_reset(inet_csk_ca(sk));	361	bictcp_reset(inet_csk_ca(sk));
@@ -442,7 +442,7 @@ static void hystart_update(struct sock *sk, u32 delay) · 80 unmodified lines
442	}	442	}
443	}	443	}
444		444
445	static void bictcp_acked(struct sock sk, const struct ack_sample sample)	445	static void cubictcp_acked(struct sock sk, const struct ack_sample sample)
446	{	446	{
447	const struct tcp_sock *tp = tcp_sk(sk);	447	const struct tcp_sock *tp = tcp_sk(sk);
448	struct bictcp *ca = inet_csk_ca(sk);	448	struct bictcp *ca = inet_csk_ca(sk);
@@ -471,13 +471,13 @@ static void bictcp_acked(struct sock sk, const struct ack_sample sample) · 22 unmodified lines
471	}	471	}
472		472
473	static struct tcp_congestion_ops cubictcp __read_mostly = {	473	static struct tcp_congestion_ops cubictcp __read_mostly = {
474	.init = bictcp_init,	474	.init = cubictcp_init,
475	.ssthresh = bictcp_recalc_ssthresh,	475	.ssthresh = cubictcp_recalc_ssthresh,
476	.cong_avoid = bictcp_cong_avoid,	476	.cong_avoid = cubictcp_cong_avoid,
477	.set_state = bictcp_state,	477	.set_state = cubictcp_state,
478	.undo_cwnd = tcp_reno_undo_cwnd,	478	.undo_cwnd = tcp_reno_undo_cwnd,
479	.cwnd_event = bictcp_cwnd_event,	479	.cwnd_event = cubictcp_cwnd_event,
480	.pkts_acked = bictcp_acked,	480	.pkts_acked = cubictcp_acked,
481	.owner = THIS_MODULE,	481	.owner = THIS_MODULE,
482	.name = "cubic",	482	.name = "cubic",
483	};	483	};

net/ipv4/tcp_ipv4.c+3−0

 	.hash			= inet_hash,
 	.unhash			= inet_unhash,
 	.get_port		= inet_csk_get_port,
+#ifdef CONFIG_BPF_SYSCALL
+	.psock_update_sk_prot	= tcp_bpf_update_proto,
+#endif
 	.enter_memory_pressure	= tcp_enter_memory_pressure,
 	.leave_memory_pressure	= tcp_leave_memory_pressure,
 	.stream_memory_free	= tcp_stream_memory_free,

net/ipv4/udp.c+32−0

@@ -1782,6 +1782,35 @@ busy_check: · 1781 unmodified lines
1782	}	1782	}
1783	EXPORT_SYMBOL(__skb_recv_udp);	1783	EXPORT_SYMBOL(__skb_recv_udp);
1784		1784
		1785	int udp_read_sock(struct sock sk, read_descriptor_t desc,
		1786	sk_read_actor_t recv_actor)
		1787	{
		1788	int copied = 0;
		1789
		1790	while (1) {
		1791	struct sk_buff *skb;
		1792	int err, used;
		1793
		1794	skb = skb_recv_udp(sk, 0, 1, &err);
		1795	if (!skb)
		1796	return err;
		1797	used = recv_actor(desc, skb, 0, skb->len);
		1798	if (used <= 0) {
		1799	if (!copied)
		1800	copied = used;
		1801	break;
		1802	} else if (used <= skb->len) {
		1803	copied += used;
		1804	}
		1805
		1806	if (!desc->count)
		1807	break;
		1808	}
		1809
		1810	return copied;
		1811	}
		1812	EXPORT_SYMBOL(udp_read_sock);
		1813
1785	/*	1814	/*
1786	* This should be easy, if there is something there we	1815	* This should be easy, if there is something there we
1787	* return it, otherwise we block.	1816	* return it, otherwise we block.
@@ -2854,6 +2883,9 @@ struct proto udp_prot = { · 1066 unmodified lines
2854	.unhash = udp_lib_unhash,	2883	.unhash = udp_lib_unhash,
2855	.rehash = udp_v4_rehash,	2884	.rehash = udp_v4_rehash,
2856	.get_port = udp_v4_get_port,	2885	.get_port = udp_v4_get_port,
		2886	#ifdef CONFIG_BPF_SYSCALL
		2887	.psock_update_sk_prot = udp_bpf_update_proto,
		2888	#endif
2857	.memory_allocated = &udp_memory_allocated,	2889	.memory_allocated = &udp_memory_allocated,
2858	.sysctl_mem = sysctl_udp_mem,	2890	.sysctl_mem = sysctl_udp_mem,
2859	.sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_udp_wmem_min),	2891	.sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_udp_wmem_min),

net/ipv4/udp_bpf.c+76−3

@@ -4,6 +4,68 @@ · 3 unmodified lines
4	#include <linux/skmsg.h>	4	#include <linux/skmsg.h>
5	#include <net/sock.h>	5	#include <net/sock.h>
6	#include <net/udp.h>	6	#include <net/udp.h>
		7	#include <net/inet_common.h>
		8
		9	#include "udp_impl.h"
		10
		11	static struct proto *udpv6_prot_saved __read_mostly;
		12
		13	static int sk_udp_recvmsg(struct sock sk, struct msghdr msg, size_t len,
		14	int noblock, int flags, int *addr_len)
		15	{
		16	#if IS_ENABLED(CONFIG_IPV6)
		17	if (sk->sk_family == AF_INET6)
		18	return udpv6_prot_saved->recvmsg(sk, msg, len, noblock, flags,
		19	addr_len);
		20	#endif
		21	return udp_prot.recvmsg(sk, msg, len, noblock, flags, addr_len);
		22	}
		23
		24	static int udp_bpf_recvmsg(struct sock sk, struct msghdr msg, size_t len,
		25	int nonblock, int flags, int *addr_len)
		26	{
		27	struct sk_psock *psock;
		28	int copied, ret;
		29
		30	if (unlikely(flags & MSG_ERRQUEUE))
		31	return inet_recv_error(sk, msg, len, addr_len);
		32
		33	psock = sk_psock_get(sk);
		34	if (unlikely(!psock))
		35	return sk_udp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
		36
		37	lock_sock(sk);
		38	if (sk_psock_queue_empty(psock)) {
		39	ret = sk_udp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
		40	goto out;
		41	}
		42
		43	msg_bytes_ready:
		44	copied = sk_msg_recvmsg(sk, psock, msg, len, flags);
		45	if (!copied) {
		46	int data, err = 0;
		47	long timeo;
		48
		49	timeo = sock_rcvtimeo(sk, nonblock);
		50	data = sk_msg_wait_data(sk, psock, flags, timeo, &err);
		51	if (data) {
		52	if (!sk_psock_queue_empty(psock))
		53	goto msg_bytes_ready;
		54	ret = sk_udp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
		55	goto out;
		56	}
		57	if (err) {
		58	ret = err;
		59	goto out;
		60	}
		61	copied = -EAGAIN;
		62	}
		63	ret = copied;
		64	out:
		65	release_sock(sk);
		66	sk_psock_put(sk, psock);
		67	return ret;
		68	}
7		69
8	enum {	70	enum {
9	UDP_BPF_IPV4,	71	UDP_BPF_IPV4,
@@ -11,7 +73,6 @@ enum { · 1 unmodified line
11	UDP_BPF_NUM_PROTS,	73	UDP_BPF_NUM_PROTS,
12	};	74	};
13		75
14	static struct proto *udpv6_prot_saved __read_mostly;
15	static DEFINE_SPINLOCK(udpv6_prot_lock);	76	static DEFINE_SPINLOCK(udpv6_prot_lock);
16	static struct proto udp_bpf_prots[UDP_BPF_NUM_PROTS];	77	static struct proto udp_bpf_prots[UDP_BPF_NUM_PROTS];
17		78
@@ -20,6 +81,7 @@ static void udp_bpf_rebuild_protos(struct proto prot, const struct proto base) · 2 unmodified lines
20	prot = base;	81	prot = base;
21	prot->unhash = sock_map_unhash;	82	prot->unhash = sock_map_unhash;
22	prot->close = sock_map_close;	83	prot->close = sock_map_close;
		84	prot->recvmsg = udp_bpf_recvmsg;
23	}	85	}
24		86
25	static void udp_bpf_check_v6_needs_rebuild(struct proto *ops)	87	static void udp_bpf_check_v6_needs_rebuild(struct proto *ops)
@@ -41,12 +103,23 @@ static int __init udp_bpf_v4_build_proto(void) · 15 unmodified lines
41	}	103	}
42	core_initcall(udp_bpf_v4_build_proto);	104	core_initcall(udp_bpf_v4_build_proto);
43		105
44	struct proto udp_bpf_get_proto(struct sock sk, struct sk_psock *psock)	106	int udp_bpf_update_proto(struct sock *sk, bool restore)
45	{	107	{
46	int family = sk->sk_family == AF_INET ? UDP_BPF_IPV4 : UDP_BPF_IPV6;	108	int family = sk->sk_family == AF_INET ? UDP_BPF_IPV4 : UDP_BPF_IPV6;
		109	struct sk_psock *psock = sk_psock(sk);
		110
		111	if (restore) {
		112	sk->sk_write_space = psock->saved_write_space;
		113	/* Pairs with lockless read in sk_clone_lock() */
		114	WRITE_ONCE(sk->sk_prot, psock->sk_proto);
		115	return 0;
		116	}
47		117
48	if (sk->sk_family == AF_INET6)	118	if (sk->sk_family == AF_INET6)
49	udp_bpf_check_v6_needs_rebuild(psock->sk_proto);	119	udp_bpf_check_v6_needs_rebuild(psock->sk_proto);
50		120
51	return &udp_bpf_prots[family];	121	/* Pairs with lockless read in sk_clone_lock() */
		122	WRITE_ONCE(sk->sk_prot, &udp_bpf_prots[family]);
		123	return 0;
52	}	124	}
		125	EXPORT_SYMBOL_GPL(udp_bpf_update_proto);

net/ipv6/af_inet6.c+1−0

@@ -714,6 +714,7 @@ const struct proto_ops inet6_dgram_ops = { · 713 unmodified lines
714	.getsockopt = sock_common_getsockopt, /* ok */	714	.getsockopt = sock_common_getsockopt, /* ok */
715	.sendmsg = inet6_sendmsg, /* retpoline's sake */	715	.sendmsg = inet6_sendmsg, /* retpoline's sake */
716	.recvmsg = inet6_recvmsg, /* retpoline's sake */	716	.recvmsg = inet6_recvmsg, /* retpoline's sake */
		717	.read_sock = udp_read_sock,
717	.mmap = sock_no_mmap,	718	.mmap = sock_no_mmap,
718	.sendpage = sock_no_sendpage,	719	.sendpage = sock_no_sendpage,
719	.set_peek_off = sk_set_peek_off,	720	.set_peek_off = sk_set_peek_off,

net/ipv6/tcp_ipv6.c+3−0

 	.hash			= inet6_hash,
 	.unhash			= inet_unhash,
 	.get_port		= inet_csk_get_port,
+#ifdef CONFIG_BPF_SYSCALL
+	.psock_update_sk_prot	= tcp_bpf_update_proto,
+#endif
 	.enter_memory_pressure	= tcp_enter_memory_pressure,
 	.leave_memory_pressure	= tcp_leave_memory_pressure,
 	.stream_memory_free	= tcp_stream_memory_free,

net/ipv6/udp.c+3−0

 	.unhash			= udp_lib_unhash,
 	.rehash			= udp_v6_rehash,
 	.get_port		= udp_v6_get_port,
+#ifdef CONFIG_BPF_SYSCALL
+	.psock_update_sk_prot	= udp_bpf_update_proto,
+#endif
 	.memory_allocated	= &udp_memory_allocated,
 	.sysctl_mem		= sysctl_udp_mem,
 	.sysctl_wmem_offset     = offsetof(struct net, ipv4.sysctl_udp_wmem_min),

net/tls/tls_sw.c+2−2

@@ -1789,8 +1789,8 @@ int tls_sw_recvmsg(struct sock *sk, · 1788 unmodified lines
1789	skb = tls_wait_data(sk, psock, flags, timeo, &err);	1789	skb = tls_wait_data(sk, psock, flags, timeo, &err);
1790	if (!skb) {	1790	if (!skb) {
1791	if (psock) {	1791	if (psock) {
1792	int ret = __tcp_bpf_recvmsg(sk, psock,	1792	int ret = sk_msg_recvmsg(sk, psock, msg, len,
1793	msg, len, flags);	1793	flags);
1794		1794
1795	if (ret > 0) {	1795	if (ret > 0) {
1796	decrypted += ret;	1796	decrypted += ret;

samples/bpf/sampleip_kern.c+0−1

@@ -4,7 +4,6 @@ · 3 unmodified lines
4	* modify it under the terms of version 2 of the GNU General Public	4	* modify it under the terms of version 2 of the GNU General Public
5	* License as published by the Free Software Foundation.	5	* License as published by the Free Software Foundation.
6	*/	6	*/
7	#include <linux/version.h>
8	#include <linux/ptrace.h>	7	#include <linux/ptrace.h>
9	#include <uapi/linux/bpf.h>	8	#include <uapi/linux/bpf.h>
10	#include <uapi/linux/bpf_perf_event.h>	9	#include <uapi/linux/bpf_perf_event.h>

samples/bpf/trace_event_kern.c+0−1

@@ -5,7 +5,6 @@ · 4 unmodified lines
5	* License as published by the Free Software Foundation.	5	* License as published by the Free Software Foundation.
6	*/	6	*/
7	#include <linux/ptrace.h>	7	#include <linux/ptrace.h>
8	#include <linux/version.h>
9	#include <uapi/linux/bpf.h>	8	#include <uapi/linux/bpf.h>
10	#include <uapi/linux/bpf_perf_event.h>	9	#include <uapi/linux/bpf_perf_event.h>
11	#include <uapi/linux/perf_event.h>	10	#include <uapi/linux/perf_event.h>

samples/bpf/xdpsock_user.c+14−41

 static int opt_timeout = 1000;
 static bool opt_need_wakeup = true;
 static u32 opt_num_xsks = 1;
-static u32 prog_id;
 static bool opt_busy_poll;
 static bool opt_reduced_cap;
 	return NULL;
 }
-static void remove_xdp_program(void)
+static void int_exit(int sig)
 {
-	u32 curr_prog_id = 0;
+	benchmark_done = true;
-	int cmd = CLOSE_CONN;
-	if (bpf_get_link_xdp_id(opt_ifindex, &curr_prog_id, opt_xdp_flags)) {
-		printf("bpf_get_link_xdp_id failed\n");
-		exit(EXIT_FAILURE);
-	}
-	if (prog_id == curr_prog_id)
-		bpf_set_link_xdp_fd(opt_ifindex, -1, opt_xdp_flags);
-	else if (!curr_prog_id)
-		printf("couldn't find a prog id on a given interface\n");
-	else
-		printf("program on interface changed, not removing\n");
-	if (opt_reduced_cap) {
-		if (write(sock, &cmd, sizeof(int)) < 0) {
-			fprintf(stderr, "Error writing into stream socket: %s", strerror(errno));
-			exit(EXIT_FAILURE);
-		}
-	}
 }
-static void int_exit(int sig)
+static void __exit_with_error(int error, const char *file, const char *func,
+			      int line)
 {
-	benchmark_done = true;
+	fprintf(stderr, "%s:%s:%i: errno: %d/\"%s\"\n", file, func,
+		line, error, strerror(error));
+	exit(EXIT_FAILURE);
 }
+#define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__)
 static void xdpsock_cleanup(void)
 {
 	struct xsk_umem *umem = xsks[0]->umem->umem;
-	int i;
+	int i, cmd = CLOSE_CONN;
 	dump_stats();
 	for (i = 0; i < num_socks; i++)
 		xsk_socket__delete(xsks[i]->xsk);
 	(void)xsk_umem__delete(umem);
-	remove_xdp_program();
-}
-static void __exit_with_error(int error, const char *file, const char *func,
+	if (opt_reduced_cap) {
-			      int line)
+		if (write(sock, &cmd, sizeof(int)) < 0)
-{
+			exit_with_error(errno);
-	fprintf(stderr, "%s:%s:%i: errno: %d/\"%s\"\n", file, func,
+	}
-		line, error, strerror(error));
-	dump_stats();
-	remove_xdp_program();
-	exit(EXIT_FAILURE);
 }
-#define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, \
-						 __LINE__)
 static void swap_mac_addresses(void *data)
 {
 	struct ether_header *eth = (struct ether_header *)data;
 	if (ret)
 		exit_with_error(-ret);
-	ret = bpf_get_link_xdp_id(opt_ifindex, &prog_id, opt_xdp_flags);
-	if (ret)
-		exit_with_error(-ret);
 	xsk->app_stats.rx_empty_polls = 0;
 	xsk->app_stats.fill_fail_polls = 0;
 	xsk->app_stats.copy_tx_sendtos = 0;

tools/bpf/bpftool/common.c+1−0

@@ -57,6 +57,7 @@ const char * const attach_type_name[__MAX_BPF_ATTACH_TYPE] = { · 56 unmodified lines
57		57
58	[BPF_SK_SKB_STREAM_PARSER] = "sk_skb_stream_parser",	58	[BPF_SK_SKB_STREAM_PARSER] = "sk_skb_stream_parser",
59	[BPF_SK_SKB_STREAM_VERDICT] = "sk_skb_stream_verdict",	59	[BPF_SK_SKB_STREAM_VERDICT] = "sk_skb_stream_verdict",
		60	[BPF_SK_SKB_VERDICT] = "sk_skb_verdict",
60	[BPF_SK_MSG_VERDICT] = "sk_msg_verdict",	61	[BPF_SK_MSG_VERDICT] = "sk_msg_verdict",
61	[BPF_LIRC_MODE2] = "lirc_mode2",	62	[BPF_LIRC_MODE2] = "lirc_mode2",
62	[BPF_FLOW_DISSECTOR] = "flow_dissector",	63	[BPF_FLOW_DISSECTOR] = "flow_dissector",

tools/bpf/bpftool/prog.c+1−0

@@ -76,6 +76,7 @@ enum dump_mode { · 75 unmodified lines
76	static const char * const attach_type_strings[] = {	76	static const char * const attach_type_strings[] = {
77	[BPF_SK_SKB_STREAM_PARSER] = "stream_parser",	77	[BPF_SK_SKB_STREAM_PARSER] = "stream_parser",
78	[BPF_SK_SKB_STREAM_VERDICT] = "stream_verdict",	78	[BPF_SK_SKB_STREAM_VERDICT] = "stream_verdict",
		79	[BPF_SK_SKB_VERDICT] = "skb_verdict",
79	[BPF_SK_MSG_VERDICT] = "msg_verdict",	80	[BPF_SK_MSG_VERDICT] = "msg_verdict",
80	[BPF_FLOW_DISSECTOR] = "flow_dissector",	81	[BPF_FLOW_DISSECTOR] = "flow_dissector",
81	[__MAX_BPF_ATTACH_TYPE] = NULL,	82	[__MAX_BPF_ATTACH_TYPE] = NULL,

tools/bpf/resolve_btfids/main.c+5−6

 static int verbose;
-int eprintf(int level, int var, const char *fmt, ...)
+static int eprintf(int level, int var, const char *fmt, ...)
 {
 	va_list args;
-	int ret;
+	int ret = 0;
 	if (var >= level) {
 		va_start(args, fmt);
 static int symbols_collect(struct object *obj)
 {
 	Elf_Scn *scn = NULL;
-	int n, i, err = 0;
+	int n, i;
 	GElf_Shdr sh;
 	char *name;
 	 * Scan symbols and look for the ones starting with
 	 * __BTF_ID__* over .BTF_ids section.
 	 */
-	for (i = 0; !err && i < n; i++) {
+	for (i = 0; i < n; i++) {
-		char *tmp, *prefix;
+		char *prefix;
 		struct btf_id *id;
 		GElf_Sym sym;
-		int err = -1;
 		if (!gelf_getsym(obj->efile.symbols, i, &sym))
 			return -1;

tools/include/uapi/linux/bpf.h+5−0

 	BPF_XDP_CPUMAP,
 	BPF_SK_LOOKUP,
 	BPF_XDP,
+	BPF_SK_SKB_VERDICT,
 	__MAX_BPF_ATTACH_TYPE
 };
  * offset to another bpf function
  */
 #define BPF_PSEUDO_CALL		1
+/* when bpf_call->src_reg == BPF_PSEUDO_KFUNC_CALL,
+ * bpf_call->imm == btf_id of a BTF_KIND_FUNC in the running kernel
+ */
+#define BPF_PSEUDO_KFUNC_CALL	2
 /* flags for BPF_MAP_UPDATE_ELEM command */
 enum {

tools/lib/bpf/libbpf.c+300−103

@@ -185,7 +185,8 @@ enum reloc_type { · 184 unmodified lines
185	RELO_LD64,	185	RELO_LD64,
186	RELO_CALL,	186	RELO_CALL,
187	RELO_DATA,	187	RELO_DATA,
188	RELO_EXTERN,	188	RELO_EXTERN_VAR,
		189	RELO_EXTERN_FUNC,
189	RELO_SUBPROG_ADDR,	190	RELO_SUBPROG_ADDR,
190	};	191	};
191		192
@@ -573,14 +574,19 @@ static bool insn_is_subprog_call(const struct bpf_insn *insn) · 381 unmodified lines
573	insn->off == 0;	574	insn->off == 0;
574	}	575	}
575		576
576	static bool is_ldimm64(struct bpf_insn *insn)	577	static bool is_ldimm64_insn(struct bpf_insn *insn)
577	{	578	{
578	return insn->code == (BPF_LD \| BPF_IMM \| BPF_DW);	579	return insn->code == (BPF_LD \| BPF_IMM \| BPF_DW);
579	}	580	}
580		581
		582	static bool is_call_insn(const struct bpf_insn *insn)
		583	{
		584	return insn->code == (BPF_JMP \| BPF_CALL);
		585	}
		586
581	static bool insn_is_pseudo_func(struct bpf_insn *insn)	587	static bool insn_is_pseudo_func(struct bpf_insn *insn)
582	{	588	{
583	return is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC;	589	return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
584	}	590	}
585		591
586	static int	592	static int
@@ -1921,9 +1927,9 @@ resolve_func_ptr(const struct btf btf, __u32 id, __u32 res_id) · 1334 unmodified lines
1921	return btf_is_func_proto(t) ? t : NULL;	1927	return btf_is_func_proto(t) ? t : NULL;
1922	}	1928	}
1923		1929
1924	static const char btf_kind_str(const struct btf_type t)	1930	static const char *__btf_kind_str(__u16 kind)
1925	{	1931	{
1926	switch (btf_kind(t)) {	1932	switch (kind) {
1927	case BTF_KIND_UNKN: return "void";	1933	case BTF_KIND_UNKN: return "void";
1928	case BTF_KIND_INT: return "int";	1934	case BTF_KIND_INT: return "int";
1929	case BTF_KIND_PTR: return "ptr";	1935	case BTF_KIND_PTR: return "ptr";
@@ -1945,6 +1951,16 @@ static const char btf_kind_str(const struct btf_type t) · 15 unmodified lines
1945	}	1951	}
1946	}	1952	}
1947		1953
		1954	static const char btf_kind_str(const struct btf_type t)
		1955	{
		1956	return __btf_kind_str(btf_kind(t));
		1957	}
		1958
		1959	static enum btf_func_linkage btf_func_linkage(const struct btf_type *t)
		1960	{
		1961	return (enum btf_func_linkage)BTF_INFO_VLEN(t->info);
		1962	}
		1963
1948	/*	1964	/*
1949	* Fetch integer attribute of BTF map definition. Such attributes are	1965	* Fetch integer attribute of BTF map definition. Such attributes are
1950	* represented using a pointer to an array, in which dimensionality of array	1966	* represented using a pointer to an array, in which dimensionality of array
@@ -3009,7 +3025,7 @@ static bool sym_is_subprog(const GElf_Sym *sym, int text_shndx) · 1058 unmodified lines
3009	static int find_extern_btf_id(const struct btf btf, const char ext_name)	3025	static int find_extern_btf_id(const struct btf btf, const char ext_name)
3010	{	3026	{
3011	const struct btf_type *t;	3027	const struct btf_type *t;
3012	const char *var_name;	3028	const char *tname;
3013	int i, n;	3029	int i, n;
3014		3030
3015	if (!btf)	3031	if (!btf)
@@ -3019,14 +3035,18 @@ static int find_extern_btf_id(const struct btf btf, const char ext_name) · 3 unmodified lines
3019	for (i = 1; i <= n; i++) {	3035	for (i = 1; i <= n; i++) {
3020	t = btf__type_by_id(btf, i);	3036	t = btf__type_by_id(btf, i);
3021		3037
3022	if (!btf_is_var(t))	3038	if (!btf_is_var(t) && !btf_is_func(t))
3023	continue;	3039	continue;
3024		3040
3025	var_name = btf__name_by_offset(btf, t->name_off);	3041	tname = btf__name_by_offset(btf, t->name_off);
3026	if (strcmp(var_name, ext_name))	3042	if (strcmp(tname, ext_name))
3027	continue;	3043	continue;
3028		3044
3029	if (btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)	3045	if (btf_is_var(t) &&
		3046	btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
		3047	return -EINVAL;
		3048
		3049	if (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_EXTERN)
3030	return -EINVAL;	3050	return -EINVAL;
3031		3051
3032	return i;	3052	return i;
@@ -3139,12 +3159,48 @@ static int find_int_btf_id(const struct btf *btf) · 106 unmodified lines
3139	return 0;	3159	return 0;
3140	}	3160	}
3141		3161
		3162	static int add_dummy_ksym_var(struct btf *btf)
		3163	{
		3164	int i, int_btf_id, sec_btf_id, dummy_var_btf_id;
		3165	const struct btf_var_secinfo *vs;
		3166	const struct btf_type *sec;
		3167
		3168	sec_btf_id = btf__find_by_name_kind(btf, KSYMS_SEC,
		3169	BTF_KIND_DATASEC);
		3170	if (sec_btf_id < 0)
		3171	return 0;
		3172
		3173	sec = btf__type_by_id(btf, sec_btf_id);
		3174	vs = btf_var_secinfos(sec);
		3175	for (i = 0; i < btf_vlen(sec); i++, vs++) {
		3176	const struct btf_type *vt;
		3177
		3178	vt = btf__type_by_id(btf, vs->type);
		3179	if (btf_is_func(vt))
		3180	break;
		3181	}
		3182
		3183	/* No func in ksyms sec. No need to add dummy var. */
		3184	if (i == btf_vlen(sec))
		3185	return 0;
		3186
		3187	int_btf_id = find_int_btf_id(btf);
		3188	dummy_var_btf_id = btf__add_var(btf,
		3189	"dummy_ksym",
		3190	BTF_VAR_GLOBAL_ALLOCATED,
		3191	int_btf_id);
		3192	if (dummy_var_btf_id < 0)
		3193	pr_warn("cannot create a dummy_ksym var\n");
		3194
		3195	return dummy_var_btf_id;
		3196	}
		3197
3142	static int bpf_object__collect_externs(struct bpf_object *obj)	3198	static int bpf_object__collect_externs(struct bpf_object *obj)
3143	{	3199	{
3144	struct btf_type sec, kcfg_sec = NULL, *ksym_sec = NULL;	3200	struct btf_type sec, kcfg_sec = NULL, *ksym_sec = NULL;
3145	const struct btf_type *t;	3201	const struct btf_type *t;
3146	struct extern_desc *ext;	3202	struct extern_desc *ext;
3147	int i, n, off;	3203	int i, n, off, dummy_var_btf_id;
3148	const char ext_name, sec_name;	3204	const char ext_name, sec_name;
3149	Elf_Scn *scn;	3205	Elf_Scn *scn;
3150	GElf_Shdr sh;	3206	GElf_Shdr sh;
@@ -3156,6 +3212,10 @@ static int bpf_object__collect_externs(struct bpf_object *obj) · 5 unmodified lines
3156	if (elf_sec_hdr(obj, scn, &sh))	3212	if (elf_sec_hdr(obj, scn, &sh))
3157	return -LIBBPF_ERRNO__FORMAT;	3213	return -LIBBPF_ERRNO__FORMAT;
3158		3214
		3215	dummy_var_btf_id = add_dummy_ksym_var(obj->btf);
		3216	if (dummy_var_btf_id < 0)
		3217	return dummy_var_btf_id;
		3218
3159	n = sh.sh_size / sh.sh_entsize;	3219	n = sh.sh_size / sh.sh_entsize;
3160	pr_debug("looking for externs among %d symbols...\n", n);	3220	pr_debug("looking for externs among %d symbols...\n", n);
3161		3221
@@ -3200,6 +3260,11 @@ static int bpf_object__collect_externs(struct bpf_object *obj) · 38 unmodified lines
3200	sec_name = btf__name_by_offset(obj->btf, sec->name_off);	3260	sec_name = btf__name_by_offset(obj->btf, sec->name_off);
3201		3261
3202	if (strcmp(sec_name, KCONFIG_SEC) == 0) {	3262	if (strcmp(sec_name, KCONFIG_SEC) == 0) {
		3263	if (btf_is_func(t)) {
		3264	pr_warn("extern function %s is unsupported under %s section\n",
		3265	ext->name, KCONFIG_SEC);
		3266	return -ENOTSUP;
		3267	}
3203	kcfg_sec = sec;	3268	kcfg_sec = sec;
3204	ext->type = EXT_KCFG;	3269	ext->type = EXT_KCFG;
3205	ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);	3270	ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);
@@ -3221,6 +3286,11 @@ static int bpf_object__collect_externs(struct bpf_object *obj) · 15 unmodified lines
3221	return -ENOTSUP;	3286	return -ENOTSUP;
3222	}	3287	}
3223	} else if (strcmp(sec_name, KSYMS_SEC) == 0) {	3288	} else if (strcmp(sec_name, KSYMS_SEC) == 0) {
		3289	if (btf_is_func(t) && ext->is_weak) {
		3290	pr_warn("extern weak function %s is unsupported\n",
		3291	ext->name);
		3292	return -ENOTSUP;
		3293	}
3224	ksym_sec = sec;	3294	ksym_sec = sec;
3225	ext->type = EXT_KSYM;	3295	ext->type = EXT_KSYM;
3226	skip_mods_and_typedefs(obj->btf, t->type,	3296	skip_mods_and_typedefs(obj->btf, t->type,
@@ -3247,7 +3317,14 @@ static int bpf_object__collect_externs(struct bpf_object *obj) · 20 unmodified lines
3247	* extern variables in DATASEC	3317	* extern variables in DATASEC
3248	*/	3318	*/
3249	int int_btf_id = find_int_btf_id(obj->btf);	3319	int int_btf_id = find_int_btf_id(obj->btf);
		3320	/* For extern function, a dummy_var added earlier
		3321	* will be used to replace the vs->type and
		3322	* its name string will be used to refill
		3323	* the missing param's name.
		3324	*/
		3325	const struct btf_type *dummy_var;
3250		3326
		3327	dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id);
3251	for (i = 0; i < obj->nr_extern; i++) {	3328	for (i = 0; i < obj->nr_extern; i++) {
3252	ext = &obj->externs[i];	3329	ext = &obj->externs[i];
3253	if (ext->type != EXT_KSYM)	3330	if (ext->type != EXT_KSYM)
@@ -3266,12 +3343,32 @@ static int bpf_object__collect_externs(struct bpf_object *obj) · 12 unmodified lines
3266	ext_name = btf__name_by_offset(obj->btf, vt->name_off);	3343	ext_name = btf__name_by_offset(obj->btf, vt->name_off);
3267	ext = find_extern_by_name(obj, ext_name);	3344	ext = find_extern_by_name(obj, ext_name);
3268	if (!ext) {	3345	if (!ext) {
3269	pr_warn("failed to find extern definition for BTF var '%s'\n",	3346	pr_warn("failed to find extern definition for BTF %s '%s'\n",
3270	ext_name);	3347	btf_kind_str(vt), ext_name);
3271	return -ESRCH;	3348	return -ESRCH;
3272	}	3349	}
3273	btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;	3350	if (btf_is_func(vt)) {
3274	vt->type = int_btf_id;	3351	const struct btf_type *func_proto;
		3352	struct btf_param *param;
		3353	int j;
		3354
		3355	func_proto = btf__type_by_id(obj->btf,
		3356	vt->type);
		3357	param = btf_params(func_proto);
		3358	/* Reuse the dummy_var string if the
		3359	* func proto does not have param name.
		3360	*/
		3361	for (j = 0; j < btf_vlen(func_proto); j++)
		3362	if (param[j].type && !param[j].name_off)
		3363	param[j].name_off =
		3364	dummy_var->name_off;
		3365	vs->type = dummy_var_btf_id;
		3366	vt->info &= ~0xffff;
		3367	vt->info \|= BTF_FUNC_GLOBAL;
		3368	} else {
		3369	btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
		3370	vt->type = int_btf_id;
		3371	}
3275	vs->offset = off;	3372	vs->offset = off;
3276	vs->size = sizeof(int);	3373	vs->size = sizeof(int);
3277	}	3374	}
@@ -3403,31 +3500,7 @@ static int bpf_program__record_reloc(struct bpf_program *prog, · 125 unmodified lines
3403		3500
3404	reloc_desc->processed = false;	3501	reloc_desc->processed = false;
3405		3502
3406	/* sub-program call relocation */	3503	if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) {
3407	if (insn->code == (BPF_JMP \| BPF_CALL)) {
3408	if (insn->src_reg != BPF_PSEUDO_CALL) {
3409	pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
3410	return -LIBBPF_ERRNO__RELOC;
3411	}
3412	/* text_shndx can be 0, if no default "main" program exists */
3413	if (!shdr_idx \|\| shdr_idx != obj->efile.text_shndx) {
3414	sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
3415	pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n",
3416	prog->name, sym_name, sym_sec_name);
3417	return -LIBBPF_ERRNO__RELOC;
3418	}
3419	if (sym->st_value % BPF_INSN_SZ) {
3420	pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n",
3421	prog->name, sym_name, (size_t)sym->st_value);
3422	return -LIBBPF_ERRNO__RELOC;
3423	}
3424	reloc_desc->type = RELO_CALL;
3425	reloc_desc->insn_idx = insn_idx;
3426	reloc_desc->sym_off = sym->st_value;
3427	return 0;
3428	}
3429
3430	if (!is_ldimm64(insn)) {
3431	pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n",	3504	pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n",
3432	prog->name, sym_name, insn_idx, insn->code);	3505	prog->name, sym_name, insn_idx, insn->code);
3433	return -LIBBPF_ERRNO__RELOC;	3506	return -LIBBPF_ERRNO__RELOC;
@@ -3450,12 +3523,39 @@ static int bpf_program__record_reloc(struct bpf_program *prog, · 16 unmodified lines
3450	}	3523	}
3451	pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n",	3524	pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n",
3452	prog->name, i, ext->name, ext->sym_idx, insn_idx);	3525	prog->name, i, ext->name, ext->sym_idx, insn_idx);
3453	reloc_desc->type = RELO_EXTERN;	3526	if (insn->code == (BPF_JMP \| BPF_CALL))
		3527	reloc_desc->type = RELO_EXTERN_FUNC;
		3528	else
		3529	reloc_desc->type = RELO_EXTERN_VAR;
3454	reloc_desc->insn_idx = insn_idx;	3530	reloc_desc->insn_idx = insn_idx;
3455	reloc_desc->sym_off = i; /* sym_off stores extern index */	3531	reloc_desc->sym_off = i; /* sym_off stores extern index */
3456	return 0;	3532	return 0;
3457	}	3533	}
3458		3534
		3535	/* sub-program call relocation */
		3536	if (is_call_insn(insn)) {
		3537	if (insn->src_reg != BPF_PSEUDO_CALL) {
		3538	pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
		3539	return -LIBBPF_ERRNO__RELOC;
		3540	}
		3541	/* text_shndx can be 0, if no default "main" program exists */
		3542	if (!shdr_idx \|\| shdr_idx != obj->efile.text_shndx) {
		3543	sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
		3544	pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n",
		3545	prog->name, sym_name, sym_sec_name);
		3546	return -LIBBPF_ERRNO__RELOC;
		3547	}
		3548	if (sym->st_value % BPF_INSN_SZ) {
		3549	pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n",
		3550	prog->name, sym_name, (size_t)sym->st_value);
		3551	return -LIBBPF_ERRNO__RELOC;
		3552	}
		3553	reloc_desc->type = RELO_CALL;
		3554	reloc_desc->insn_idx = insn_idx;
		3555	reloc_desc->sym_off = sym->st_value;
		3556	return 0;
		3557	}
		3558
3459	if (!shdr_idx \|\| shdr_idx >= SHN_LORESERVE) {	3559	if (!shdr_idx \|\| shdr_idx >= SHN_LORESERVE) {
3460	pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n",	3560	pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n",
3461	prog->name, sym_name, shdr_idx);	3561	prog->name, sym_name, shdr_idx);
@@ -5695,7 +5795,7 @@ poison: · 2233 unmodified lines
5695	/* poison second part of ldimm64 to avoid confusing error from	5795	/* poison second part of ldimm64 to avoid confusing error from
5696	* verifier about "unknown opcode 00"	5796	* verifier about "unknown opcode 00"
5697	*/	5797	*/
5698	if (is_ldimm64(insn))	5798	if (is_ldimm64_insn(insn))
5699	bpf_core_poison_insn(prog, relo_idx, insn_idx + 1, insn + 1);	5799	bpf_core_poison_insn(prog, relo_idx, insn_idx + 1, insn + 1);
5700	bpf_core_poison_insn(prog, relo_idx, insn_idx, insn);	5800	bpf_core_poison_insn(prog, relo_idx, insn_idx, insn);
5701	return 0;	5801	return 0;
@@ -5771,7 +5871,7 @@ poison: · 69 unmodified lines
5771	case BPF_LD: {	5871	case BPF_LD: {
5772	__u64 imm;	5872	__u64 imm;
5773		5873
5774	if (!is_ldimm64(insn) \|\|	5874	if (!is_ldimm64_insn(insn) \|\|
5775	insn[0].src_reg != 0 \|\| insn[0].off != 0 \|\|	5875	insn[0].src_reg != 0 \|\| insn[0].off != 0 \|\|
5776	insn_idx + 1 >= prog->insns_cnt \|\|	5876	insn_idx + 1 >= prog->insns_cnt \|\|
5777	insn[1].code != 0 \|\| insn[1].dst_reg != 0 \|\|	5877	insn[1].code != 0 \|\| insn[1].dst_reg != 0 \|\|
@@ -6213,7 +6313,7 @@ bpf_object__relocate_data(struct bpf_object obj, struct bpf_program prog) · 435 unmodified lines
6213	insn[0].imm = obj->maps[relo->map_idx].fd;	6313	insn[0].imm = obj->maps[relo->map_idx].fd;
6214	relo->processed = true;	6314	relo->processed = true;
6215	break;	6315	break;
6216	case RELO_EXTERN:	6316	case RELO_EXTERN_VAR:
6217	ext = &obj->externs[relo->sym_off];	6317	ext = &obj->externs[relo->sym_off];
6218	if (ext->type == EXT_KCFG) {	6318	if (ext->type == EXT_KCFG) {
6219	insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;	6319	insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
@@ -6231,6 +6331,12 @@ bpf_object__relocate_data(struct bpf_object obj, struct bpf_program prog) · 11 unmodified lines
6231	}	6331	}
6232	relo->processed = true;	6332	relo->processed = true;
6233	break;	6333	break;
		6334	case RELO_EXTERN_FUNC:
		6335	ext = &obj->externs[relo->sym_off];
		6336	insn[0].src_reg = BPF_PSEUDO_KFUNC_CALL;
		6337	insn[0].imm = ext->ksym.kernel_btf_id;
		6338	relo->processed = true;
		6339	break;
6234	case RELO_SUBPROG_ADDR:	6340	case RELO_SUBPROG_ADDR:
6235	insn[0].src_reg = BPF_PSEUDO_FUNC;	6341	insn[0].src_reg = BPF_PSEUDO_FUNC;
6236	/* will be handled as a follow up pass */	6342	/* will be handled as a follow up pass */
@@ -7351,6 +7457,7 @@ static int bpf_object__read_kallsyms_file(struct bpf_object *obj) · 1114 unmodified lines
7351	{	7457	{
7352	char sym_type, sym_name[500];	7458	char sym_type, sym_name[500];
7353	unsigned long long sym_addr;	7459	unsigned long long sym_addr;
		7460	const struct btf_type *t;
7354	struct extern_desc *ext;	7461	struct extern_desc *ext;
7355	int ret, err = 0;	7462	int ret, err = 0;
7356	FILE *f;	7463	FILE *f;
@@ -7377,6 +7484,10 @@ static int bpf_object__read_kallsyms_file(struct bpf_object *obj) · 20 unmodified lines
7377	if (!ext \|\| ext->type != EXT_KSYM)	7484	if (!ext \|\| ext->type != EXT_KSYM)
7378	continue;	7485	continue;
7379		7486
		7487	t = btf__type_by_id(obj->btf, ext->btf_id);
		7488	if (!btf_is_var(t))
		7489	continue;
		7490
7380	if (ext->is_set && ext->ksym.addr != sym_addr) {	7491	if (ext->is_set && ext->ksym.addr != sym_addr) {
7381	pr_warn("extern (ksym) '%s' resolution is ambiguous: 0x%llx or 0x%llx\n",	7492	pr_warn("extern (ksym) '%s' resolution is ambiguous: 0x%llx or 0x%llx\n",
7382	sym_name, ext->ksym.addr, sym_addr);	7493	sym_name, ext->ksym.addr, sym_addr);
@@ -7395,75 +7506,151 @@ out: · 12 unmodified lines
7395	return err;	7506	return err;
7396	}	7507	}
7397		7508
7398	static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj)	7509	static int find_ksym_btf_id(struct bpf_object obj, const char ksym_name,
		7510	__u16 kind, struct btf **res_btf,
		7511	int *res_btf_fd)
7399	{	7512	{
7400	struct extern_desc *ext;	7513	int i, id, btf_fd, err;
7401	struct btf *btf;	7514	struct btf *btf;
7402	int i, j, id, btf_fd, err;
7403		7515
7404	for (i = 0; i < obj->nr_extern; i++) {	7516	btf = obj->btf_vmlinux;
7405	const struct btf_type targ_var, targ_type;	7517	btf_fd = 0;
7406	__u32 targ_type_id, local_type_id;	7518	id = btf__find_by_name_kind(btf, ksym_name, kind);
7407	const char *targ_var_name;
7408	int ret;
7409		7519
7410	ext = &obj->externs[i];	7520	if (id == -ENOENT) {
7411	if (ext->type != EXT_KSYM \|\| !ext->ksym.type_id)	7521	err = load_module_btfs(obj);
7412	continue;	7522	if (err)
7413		7523	return err;
7414	btf = obj->btf_vmlinux;
7415	btf_fd = 0;
7416	id = btf__find_by_name_kind(btf, ext->name, BTF_KIND_VAR);
7417	if (id == -ENOENT) {
7418	err = load_module_btfs(obj);
7419	if (err)
7420	return err;
7421		7524
7422	for (j = 0; j < obj->btf_module_cnt; j++) {	7525	for (i = 0; i < obj->btf_module_cnt; i++) {
7423	btf = obj->btf_modules[j].btf;	7526	btf = obj->btf_modules[i].btf;
7424	/* we assume module BTF FD is always >0 */	7527	/* we assume module BTF FD is always >0 */
7425	btf_fd = obj->btf_modules[j].fd;	7528	btf_fd = obj->btf_modules[i].fd;
7426	id = btf__find_by_name_kind(btf, ext->name, BTF_KIND_VAR);	7529	id = btf__find_by_name_kind(btf, ksym_name, kind);
7427	if (id != -ENOENT)	7530	if (id != -ENOENT)
7428	break;	7531	break;
7429	}
7430	}
7431	if (id <= 0) {
7432	pr_warn("extern (ksym) '%s': failed to find BTF ID in kernel BTF(s).\n",
7433	ext->name);
7434	return -ESRCH;
7435	}	7532	}
		7533	}
		7534	if (id <= 0) {
		7535	pr_warn("extern (%s ksym) '%s': failed to find BTF ID in kernel BTF(s).\n",
		7536	__btf_kind_str(kind), ksym_name);
		7537	return -ESRCH;
		7538	}
7436		7539
7437	/* find local type_id */	7540	*res_btf = btf;
7438	local_type_id = ext->ksym.type_id;	7541	*res_btf_fd = btf_fd;
		7542	return id;
		7543	}
7439		7544
7440	/* find target type_id */	7545	static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj,
7441	targ_var = btf__type_by_id(btf, id);	7546	struct extern_desc *ext)
7442	targ_var_name = btf__name_by_offset(btf, targ_var->name_off);	7547	{
7443	targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id);	7548	const struct btf_type targ_var, targ_type;
		7549	__u32 targ_type_id, local_type_id;
		7550	const char *targ_var_name;
		7551	int id, btf_fd = 0, err;
		7552	struct btf *btf = NULL;
7444		7553
7445	ret = bpf_core_types_are_compat(obj->btf, local_type_id,	7554	id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &btf_fd);
7446	btf, targ_type_id);	7555	if (id < 0)
7447	if (ret <= 0) {	7556	return id;
7448	const struct btf_type *local_type;
7449	const char targ_name, local_name;
7450		7557
7451	local_type = btf__type_by_id(obj->btf, local_type_id);	7558	/* find local type_id */
7452	local_name = btf__name_by_offset(obj->btf, local_type->name_off);	7559	local_type_id = ext->ksym.type_id;
7453	targ_name = btf__name_by_offset(btf, targ_type->name_off);
7454		7560
7455	pr_warn("extern (ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n",	7561	/* find target type_id */
7456	ext->name, local_type_id,	7562	targ_var = btf__type_by_id(btf, id);
7457	btf_kind_str(local_type), local_name, targ_type_id,	7563	targ_var_name = btf__name_by_offset(btf, targ_var->name_off);
7458	btf_kind_str(targ_type), targ_name);	7564	targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id);
7459	return -EINVAL;	7565
7460	}	7566	err = bpf_core_types_are_compat(obj->btf, local_type_id,
		7567	btf, targ_type_id);
		7568	if (err <= 0) {
		7569	const struct btf_type *local_type;
		7570	const char targ_name, local_name;
7461		7571
7462	ext->is_set = true;	7572	local_type = btf__type_by_id(obj->btf, local_type_id);
7463	ext->ksym.kernel_btf_obj_fd = btf_fd;	7573	local_name = btf__name_by_offset(obj->btf, local_type->name_off);
7464	ext->ksym.kernel_btf_id = id;	7574	targ_name = btf__name_by_offset(btf, targ_type->name_off);
7465	pr_debug("extern (ksym) '%s': resolved to [%d] %s %s\n",	7575
7466	ext->name, id, btf_kind_str(targ_var), targ_var_name);	7576	pr_warn("extern (var ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n",
		7577	ext->name, local_type_id,
		7578	btf_kind_str(local_type), local_name, targ_type_id,
		7579	btf_kind_str(targ_type), targ_name);
		7580	return -EINVAL;
		7581	}
		7582
		7583	ext->is_set = true;
		7584	ext->ksym.kernel_btf_obj_fd = btf_fd;
		7585	ext->ksym.kernel_btf_id = id;
		7586	pr_debug("extern (var ksym) '%s': resolved to [%d] %s %s\n",
		7587	ext->name, id, btf_kind_str(targ_var), targ_var_name);
		7588
		7589	return 0;
		7590	}
		7591
		7592	static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj,
		7593	struct extern_desc *ext)
		7594	{
		7595	int local_func_proto_id, kfunc_proto_id, kfunc_id;
		7596	const struct btf_type *kern_func;
		7597	struct btf *kern_btf = NULL;
		7598	int ret, kern_btf_fd = 0;
		7599
		7600	local_func_proto_id = ext->ksym.type_id;
		7601
		7602	kfunc_id = find_ksym_btf_id(obj, ext->name, BTF_KIND_FUNC,
		7603	&kern_btf, &kern_btf_fd);
		7604	if (kfunc_id < 0) {
		7605	pr_warn("extern (func ksym) '%s': not found in kernel BTF\n",
		7606	ext->name);
		7607	return kfunc_id;
		7608	}
		7609
		7610	if (kern_btf != obj->btf_vmlinux) {
		7611	pr_warn("extern (func ksym) '%s': function in kernel module is not supported\n",
		7612	ext->name);
		7613	return -ENOTSUP;
		7614	}
		7615
		7616	kern_func = btf__type_by_id(kern_btf, kfunc_id);
		7617	kfunc_proto_id = kern_func->type;
		7618
		7619	ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id,
		7620	kern_btf, kfunc_proto_id);
		7621	if (ret <= 0) {
		7622	pr_warn("extern (func ksym) '%s': func_proto [%d] incompatible with kernel [%d]\n",
		7623	ext->name, local_func_proto_id, kfunc_proto_id);
		7624	return -EINVAL;
		7625	}
		7626
		7627	ext->is_set = true;
		7628	ext->ksym.kernel_btf_obj_fd = kern_btf_fd;
		7629	ext->ksym.kernel_btf_id = kfunc_id;
		7630	pr_debug("extern (func ksym) '%s': resolved to kernel [%d]\n",
		7631	ext->name, kfunc_id);
		7632
		7633	return 0;
		7634	}
		7635
		7636	static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj)
		7637	{
		7638	const struct btf_type *t;
		7639	struct extern_desc *ext;
		7640	int i, err;
		7641
		7642	for (i = 0; i < obj->nr_extern; i++) {
		7643	ext = &obj->externs[i];
		7644	if (ext->type != EXT_KSYM \|\| !ext->ksym.type_id)
		7645	continue;
		7646
		7647	t = btf__type_by_id(obj->btf, ext->btf_id);
		7648	if (btf_is_var(t))
		7649	err = bpf_object__resolve_ksym_var_btf_id(obj, ext);
		7650	else
		7651	err = bpf_object__resolve_ksym_func_btf_id(obj, ext);
		7652	if (err)
		7653	return err;
7467	}	7654	}
7468	return 0;	7655	return 0;
7469	}	7656	}
@@ -8270,6 +8457,16 @@ int bpf_object__btf_fd(const struct bpf_object *obj) · 800 unmodified lines
8270	return obj->btf ? btf__fd(obj->btf) : -1;	8457	return obj->btf ? btf__fd(obj->btf) : -1;
8271	}	8458	}
8272		8459
		8460	int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version)
		8461	{
		8462	if (obj->loaded)
		8463	return -EINVAL;
		8464
		8465	obj->kern_version = kern_version;
		8466
		8467	return 0;
		8468	}
		8469
8273	int bpf_object__set_priv(struct bpf_object obj, void priv,	8470	int bpf_object__set_priv(struct bpf_object obj, void priv,
8274	bpf_object_clear_priv_t clear_priv)	8471	bpf_object_clear_priv_t clear_priv)
8275	{	8472	{
@@ -8458,7 +8655,7 @@ int bpf_program__nth_fd(const struct bpf_program *prog, int n) · 182 unmodified lines
8458	return fd;	8655	return fd;
8459	}	8656	}
8460		8657
8461	enum bpf_prog_type bpf_program__get_type(struct bpf_program *prog)	8658	enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog)
8462	{	8659	{
8463	return prog->type;	8660	return prog->type;
8464	}	8661	}
@@ -8503,7 +8700,7 @@ BPF_PROG_TYPE_FNS(extension, BPF_PROG_TYPE_EXT); · 38 unmodified lines
8503	BPF_PROG_TYPE_FNS(sk_lookup, BPF_PROG_TYPE_SK_LOOKUP);	8700	BPF_PROG_TYPE_FNS(sk_lookup, BPF_PROG_TYPE_SK_LOOKUP);
8504		8701
8505	enum bpf_attach_type	8702	enum bpf_attach_type
8506	bpf_program__get_expected_attach_type(struct bpf_program *prog)	8703	bpf_program__get_expected_attach_type(const struct bpf_program *prog)
8507	{	8704	{
8508	return prog->expected_attach_type;	8705	return prog->expected_attach_type;
8509	}	8706	}

tools/lib/bpf/libbpf.h+3−2

 LIBBPF_API const char *bpf_object__name(const struct bpf_object *obj);
 LIBBPF_API unsigned int bpf_object__kversion(const struct bpf_object *obj);
+LIBBPF_API int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version);
 struct btf;
 LIBBPF_API struct btf *bpf_object__btf(const struct bpf_object *obj);
 LIBBPF_API int bpf_program__set_extension(struct bpf_program *prog);
 LIBBPF_API int bpf_program__set_sk_lookup(struct bpf_program *prog);
-LIBBPF_API enum bpf_prog_type bpf_program__get_type(struct bpf_program *prog);
+LIBBPF_API enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog);
 LIBBPF_API void bpf_program__set_type(struct bpf_program *prog,
 				      enum bpf_prog_type type);
 LIBBPF_API enum bpf_attach_type
-bpf_program__get_expected_attach_type(struct bpf_program *prog);
+bpf_program__get_expected_attach_type(const struct bpf_program *prog);
 LIBBPF_API void
 bpf_program__set_expected_attach_type(struct bpf_program *prog,
 				      enum bpf_attach_type type);

tools/lib/bpf/libbpf.map+1−0

@@ -359,4 +359,5 @@ LIBBPF_0.4.0 { · 358 unmodified lines
359	bpf_linker__finalize;	359	bpf_linker__finalize;
360	bpf_linker__free;	360	bpf_linker__free;
361	bpf_linker__new;	361	bpf_linker__new;
		362	bpf_object__set_kversion;
362	} LIBBPF_0.3.0;	363	} LIBBPF_0.3.0;

tools/lib/bpf/linker.c+28−9

 	int sec_sym_idx;
 	/* section's DATASEC variable info, emitted on BTF finalization */
+	bool has_btf;
 	int sec_var_cnt;
 	struct btf_var_secinfo *sec_vars;
 			continue;
 		dst_sec = &linker->secs[src_sec->dst_id];
+		/* Mark section as having BTF regardless of the presence of
+		 * variables. In some cases compiler might generate empty BTF
+		 * with no variables information. E.g., when promoting local
+		 * array/structure variable initial values and BPF object
+		 * file otherwise has no read-only static variables in
+		 * .rodata. We need to preserve such empty BTF and just set
+		 * correct section size.
+		 */
+		dst_sec->has_btf = true;
 		t = btf__type_by_id(obj->btf, src_sec->sec_type_id);
 		src_var = btf_var_secinfos(t);
 		n = btf_vlen(t);
 	for (i = 1; i < linker->sec_cnt; i++) {
 		struct dst_sec *sec = &linker->secs[i];
-		if (!sec->sec_var_cnt)
+		if (!sec->has_btf)
 			continue;
 		id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz);
 			struct dst_sec *sec = &linker->secs[i];
 			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info);
-			if (sz < 0)
+			if (sz < 0) {
-				return sz;
+				err = sz;
+				goto out;
+			}
 			cur += sz;
 		}
 			struct dst_sec *sec = &linker->secs[i];
 			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info);
-			if (sz < 0)
+			if (sz < 0) {
-				return sz;
+				err = sz;
+				goto out;
+			}
 			cur += sz;
 		}
 			struct dst_sec *sec = &linker->secs[i];
 			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info);
-			if (sz < 0)
+			if (sz < 0) {
-				return sz;
+				err = sz;
+				goto out;
+			}
 			cur += sz;
 		}
 	if (err) {
 		linker->btf_ext = NULL;
 		pr_warn("failed to parse final .BTF.ext data: %d\n", err);
-		return err;
+		goto out;
 	}
-	return 0;
+out:
+	free(data);
+	return err;
 }

tools/lib/bpf/xsk.c+213−45

@@ -28,6 +28,7 @@ · 27 unmodified lines
28	#include <sys/mman.h>	28	#include <sys/mman.h>
29	#include <sys/socket.h>	29	#include <sys/socket.h>
30	#include <sys/types.h>	30	#include <sys/types.h>
		31	#include <linux/if_link.h>
31		32
32	#include "bpf.h"	33	#include "bpf.h"
33	#include "libbpf.h"	34	#include "libbpf.h"
@@ -70,8 +71,10 @@ struct xsk_ctx { · 36 unmodified lines
70	int ifindex;	71	int ifindex;
71	struct list_head list;	72	struct list_head list;
72	int prog_fd;	73	int prog_fd;
		74	int link_fd;
73	int xsks_map_fd;	75	int xsks_map_fd;
74	char ifname[IFNAMSIZ];	76	char ifname[IFNAMSIZ];
		77	bool has_bpf_link;
75	};	78	};
76		79
77	struct xsk_socket {	80	struct xsk_socket {
@@ -409,7 +412,7 @@ static int xsk_load_xdp_prog(struct xsk_socket *xsk) · 331 unmodified lines
409	static const int log_buf_size = 16 * 1024;	412	static const int log_buf_size = 16 * 1024;
410	struct xsk_ctx *ctx = xsk->ctx;	413	struct xsk_ctx *ctx = xsk->ctx;
411	char log_buf[log_buf_size];	414	char log_buf[log_buf_size];
412	int err, prog_fd;	415	int prog_fd;
413		416
414	/* This is the fallback C-program:	417	/* This is the fallback C-program:
415	* SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)	418	* SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)
@@ -499,14 +502,41 @@ static int xsk_load_xdp_prog(struct xsk_socket *xsk) · 83 unmodified lines
499	return prog_fd;	502	return prog_fd;
500	}	503	}
501		504
502	err = bpf_set_link_xdp_fd(xsk->ctx->ifindex, prog_fd,	505	ctx->prog_fd = prog_fd;
503	xsk->config.xdp_flags);	506	return 0;
		507	}
		508
		509	static int xsk_create_bpf_link(struct xsk_socket *xsk)
		510	{
		511	DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts);
		512	struct xsk_ctx *ctx = xsk->ctx;
		513	__u32 prog_id = 0;
		514	int link_fd;
		515	int err;
		516
		517	err = bpf_get_link_xdp_id(ctx->ifindex, &prog_id, xsk->config.xdp_flags);
504	if (err) {	518	if (err) {
505	close(prog_fd);	519	pr_warn("getting XDP prog id failed\n");
506	return err;	520	return err;
507	}	521	}
508		522
509	ctx->prog_fd = prog_fd;	523	/* if there's a netlink-based XDP prog loaded on interface, bail out
		524	* and ask user to do the removal by himself
		525	*/
		526	if (prog_id) {
		527	pr_warn("Netlink-based XDP prog detected, please unload it in order to launch AF_XDP prog\n");
		528	return -EINVAL;
		529	}
		530
		531	opts.flags = xsk->config.xdp_flags & ~(XDP_FLAGS_UPDATE_IF_NOEXIST \| XDP_FLAGS_REPLACE);
		532
		533	link_fd = bpf_link_create(ctx->prog_fd, ctx->ifindex, BPF_XDP, &opts);
		534	if (link_fd < 0) {
		535	pr_warn("bpf_link_create failed: %s\n", strerror(errno));
		536	return link_fd;
		537	}
		538
		539	ctx->link_fd = link_fd;
510	return 0;	540	return 0;
511	}	541	}
512		542
@@ -625,7 +655,6 @@ static int xsk_lookup_bpf_maps(struct xsk_socket *xsk) · 112 unmodified lines
625	close(fd);	655	close(fd);
626	}	656	}
627		657
628	err = 0;
629	if (ctx->xsks_map_fd == -1)	658	if (ctx->xsks_map_fd == -1)
630	err = -ENOENT;	659	err = -ENOENT;
631		660
@@ -642,6 +671,98 @@ static int xsk_set_bpf_maps(struct xsk_socket *xsk) · 10 unmodified lines
642	&xsk->fd, 0);	671	&xsk->fd, 0);
643	}	672	}
644		673
		674	static int xsk_link_lookup(int ifindex, __u32 prog_id, int link_fd)
		675	{
		676	struct bpf_link_info link_info;
		677	__u32 link_len;
		678	__u32 id = 0;
		679	int err;
		680	int fd;
		681
		682	while (true) {
		683	err = bpf_link_get_next_id(id, &id);
		684	if (err) {
		685	if (errno == ENOENT) {
		686	err = 0;
		687	break;
		688	}
		689	pr_warn("can't get next link: %s\n", strerror(errno));
		690	break;
		691	}
		692
		693	fd = bpf_link_get_fd_by_id(id);
		694	if (fd < 0) {
		695	if (errno == ENOENT)
		696	continue;
		697	pr_warn("can't get link by id (%u): %s\n", id, strerror(errno));
		698	err = -errno;
		699	break;
		700	}
		701
		702	link_len = sizeof(struct bpf_link_info);
		703	memset(&link_info, 0, link_len);
		704	err = bpf_obj_get_info_by_fd(fd, &link_info, &link_len);
		705	if (err) {
		706	pr_warn("can't get link info: %s\n", strerror(errno));
		707	close(fd);
		708	break;
		709	}
		710	if (link_info.type == BPF_LINK_TYPE_XDP) {
		711	if (link_info.xdp.ifindex == ifindex) {
		712	*link_fd = fd;
		713	if (prog_id)
		714	*prog_id = link_info.prog_id;
		715	break;
		716	}
		717	}
		718	close(fd);
		719	}
		720
		721	return err;
		722	}
		723
		724	static bool xsk_probe_bpf_link(void)
		725	{
		726	DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts,
		727	.flags = XDP_FLAGS_SKB_MODE);
		728	struct bpf_load_program_attr prog_attr;
		729	struct bpf_insn insns[2] = {
		730	BPF_MOV64_IMM(BPF_REG_0, XDP_PASS),
		731	BPF_EXIT_INSN()
		732	};
		733	int prog_fd, link_fd = -1;
		734	int ifindex_lo = 1;
		735	bool ret = false;
		736	int err;
		737
		738	err = xsk_link_lookup(ifindex_lo, NULL, &link_fd);
		739	if (err)
		740	return ret;
		741
		742	if (link_fd >= 0)
		743	return true;
		744
		745	memset(&prog_attr, 0, sizeof(prog_attr));
		746	prog_attr.prog_type = BPF_PROG_TYPE_XDP;
		747	prog_attr.insns = insns;
		748	prog_attr.insns_cnt = ARRAY_SIZE(insns);
		749	prog_attr.license = "GPL";
		750
		751	prog_fd = bpf_load_program_xattr(&prog_attr, NULL, 0);
		752	if (prog_fd < 0)
		753	return ret;
		754
		755	link_fd = bpf_link_create(prog_fd, ifindex_lo, BPF_XDP, &opts);
		756	close(prog_fd);
		757
		758	if (link_fd >= 0) {
		759	ret = true;
		760	close(link_fd);
		761	}
		762
		763	return ret;
		764	}
		765
645	static int xsk_create_xsk_struct(int ifindex, struct xsk_socket *xsk)	766	static int xsk_create_xsk_struct(int ifindex, struct xsk_socket *xsk)
646	{	767	{
647	char ifname[IFNAMSIZ];	768	char ifname[IFNAMSIZ];
@@ -663,64 +784,108 @@ static int xsk_create_xsk_struct(int ifindex, struct xsk_socket *xsk) · 15 unmodified lines
663	ctx->ifname[IFNAMSIZ - 1] = 0;	784	ctx->ifname[IFNAMSIZ - 1] = 0;
664		785
665	xsk->ctx = ctx;	786	xsk->ctx = ctx;
		787	xsk->ctx->has_bpf_link = xsk_probe_bpf_link();
666		788
667	return 0;	789	return 0;
668	}	790	}
669		791
670	static int __xsk_setup_xdp_prog(struct xsk_socket *_xdp,	792	static int xsk_init_xdp_res(struct xsk_socket *xsk,
671	int *xsks_map_fd)	793	int *xsks_map_fd)
672	{	794	{
673	struct xsk_socket *xsk = _xdp;
674	struct xsk_ctx *ctx = xsk->ctx;	795	struct xsk_ctx *ctx = xsk->ctx;
675	__u32 prog_id = 0;
676	int err;	796	int err;
677		797
678	err = bpf_get_link_xdp_id(ctx->ifindex, &prog_id,	798	err = xsk_create_bpf_maps(xsk);
679	xsk->config.xdp_flags);
680	if (err)	799	if (err)
681	return err;	800	return err;
682		801
683	if (!prog_id) {	802	err = xsk_load_xdp_prog(xsk);
684	err = xsk_create_bpf_maps(xsk);	803	if (err)
685	if (err)	804	goto err_load_xdp_prog;
686	return err;
687		805
688	err = xsk_load_xdp_prog(xsk);	806	if (ctx->has_bpf_link)
689	if (err) {	807	err = xsk_create_bpf_link(xsk);
690	goto err_load_xdp_prog;	808	else
691	}	809	err = bpf_set_link_xdp_fd(xsk->ctx->ifindex, ctx->prog_fd,
692	} else {	810	xsk->config.xdp_flags);
693	ctx->prog_fd = bpf_prog_get_fd_by_id(prog_id);
694	if (ctx->prog_fd < 0)
695	return -errno;
696	err = xsk_lookup_bpf_maps(xsk);
697	if (err) {
698	close(ctx->prog_fd);
699	return err;
700	}
701	}
702		811
703	if (xsk->rx) {	812	if (err)
704	err = xsk_set_bpf_maps(xsk);	813	goto err_attach_xdp_prog;
705	if (err) {
706	if (!prog_id) {
707	goto err_set_bpf_maps;
708	} else {
709	close(ctx->prog_fd);
710	return err;
711	}
712	}
713	}
714	if (xsks_map_fd)
715	*xsks_map_fd = ctx->xsks_map_fd;
716		814
717	return 0;	815	if (!xsk->rx)
		816	return err;
		817
		818	err = xsk_set_bpf_maps(xsk);
		819	if (err)
		820	goto err_set_bpf_maps;
		821
		822	return err;
718		823
719	err_set_bpf_maps:	824	err_set_bpf_maps:
		825	if (ctx->has_bpf_link)
		826	close(ctx->link_fd);
		827	else
		828	bpf_set_link_xdp_fd(ctx->ifindex, -1, 0);
		829	err_attach_xdp_prog:
720	close(ctx->prog_fd);	830	close(ctx->prog_fd);
721	bpf_set_link_xdp_fd(ctx->ifindex, -1, 0);
722	err_load_xdp_prog:	831	err_load_xdp_prog:
723	xsk_delete_bpf_maps(xsk);	832	xsk_delete_bpf_maps(xsk);
		833	return err;
		834	}
		835
		836	static int xsk_lookup_xdp_res(struct xsk_socket xsk, int xsks_map_fd, int prog_id)
		837	{
		838	struct xsk_ctx *ctx = xsk->ctx;
		839	int err;
		840
		841	ctx->prog_fd = bpf_prog_get_fd_by_id(prog_id);
		842	if (ctx->prog_fd < 0) {
		843	err = -errno;
		844	goto err_prog_fd;
		845	}
		846	err = xsk_lookup_bpf_maps(xsk);
		847	if (err)
		848	goto err_lookup_maps;
		849
		850	if (!xsk->rx)
		851	return err;
		852
		853	err = xsk_set_bpf_maps(xsk);
		854	if (err)
		855	goto err_set_maps;
		856
		857	return err;
		858
		859	err_set_maps:
		860	close(ctx->xsks_map_fd);
		861	err_lookup_maps:
		862	close(ctx->prog_fd);
		863	err_prog_fd:
		864	if (ctx->has_bpf_link)
		865	close(ctx->link_fd);
		866	return err;
		867	}
		868
		869	static int __xsk_setup_xdp_prog(struct xsk_socket _xdp, int xsks_map_fd)
		870	{
		871	struct xsk_socket *xsk = _xdp;
		872	struct xsk_ctx *ctx = xsk->ctx;
		873	__u32 prog_id = 0;
		874	int err;
		875
		876	if (ctx->has_bpf_link)
		877	err = xsk_link_lookup(ctx->ifindex, &prog_id, &ctx->link_fd);
		878	else
		879	err = bpf_get_link_xdp_id(ctx->ifindex, &prog_id, xsk->config.xdp_flags);
		880
		881	if (err)
		882	return err;
		883
		884	err = !prog_id ? xsk_init_xdp_res(xsk, xsks_map_fd) :
		885	xsk_lookup_xdp_res(xsk, xsks_map_fd, prog_id);
		886
		887	if (!err && xsks_map_fd)
		888	*xsks_map_fd = ctx->xsks_map_fd;
724		889
725	return err;	890	return err;
726	}	891	}
@@ -898,6 +1063,7 @@ int xsk_socket__create_shared(struct xsk_socket **xsk_ptr, · 171 unmodified lines
898	}	1063	}
899	}	1064	}
900	xsk->ctx = ctx;	1065	xsk->ctx = ctx;
		1066	xsk->ctx->has_bpf_link = xsk_probe_bpf_link();
901		1067
902	if (rx) {	1068	if (rx) {
903	err = setsockopt(xsk->fd, SOL_XDP, XDP_RX_RING,	1069	err = setsockopt(xsk->fd, SOL_XDP, XDP_RX_RING,
@@ -1054,6 +1220,8 @@ void xsk_socket__delete(struct xsk_socket *xsk) · 150 unmodified lines
1054	if (ctx->prog_fd != -1) {	1220	if (ctx->prog_fd != -1) {
1055	xsk_delete_bpf_maps(xsk);	1221	xsk_delete_bpf_maps(xsk);
1056	close(ctx->prog_fd);	1222	close(ctx->prog_fd);
		1223	if (ctx->has_bpf_link)
		1224	close(ctx->link_fd);
1057	}	1225	}
1058		1226
1059	err = xsk_get_mmap_offsets(xsk->fd, &off);	1227	err = xsk_get_mmap_offsets(xsk->fd, &off);

tools/testing/selftests/bpf/README.rst+14−0

@@ -179,3 +179,17 @@ types, which was introduced in `Clang 13`__. The older Clang versions will · 178 unmodified lines
179	either crash when compiling these tests, or generate an incorrect BTF.	179	either crash when compiling these tests, or generate an incorrect BTF.
180		180
181	__ https://reviews.llvm.org/D83289	181	__ https://reviews.llvm.org/D83289
		182
		183	Kernel function call test and Clang version
		184	===========================================
		185
		186	Some selftests (e.g. kfunc_call and bpf_tcp_ca) require a LLVM support
		187	to generate extern function in BTF. It was introduced in `Clang 13`__.
		188
		189	Without it, the error from compiling bpf selftests looks like:
		190
		191	.. code-block:: console
		192
		193	libbpf: failed to find BTF for extern 'tcp_slow_start' [25] section: -2
		194
		195	__ https://reviews.llvm.org/D93563

tools/testing/selftests/bpf/bpf_tcp_helpers.h+2−27

@@ -187,16 +187,6 @@ struct tcp_congestion_ops { · 186 unmodified lines
187	typeof(y) __y = (y); \	187	typeof(y) __y = (y); \
188	__x == 0 ? __y : ((__y == 0) ? __x : min(__x, __y)); })	188	__x == 0 ? __y : ((__y == 0) ? __x : min(__x, __y)); })
189		189
190	static __always_inline __u32 tcp_slow_start(struct tcp_sock *tp, __u32 acked)
191	{
192	__u32 cwnd = min(tp->snd_cwnd + acked, tp->snd_ssthresh);
193
194	acked -= cwnd - tp->snd_cwnd;
195	tp->snd_cwnd = min(cwnd, tp->snd_cwnd_clamp);
196
197	return acked;
198	}
199
200	static __always_inline bool tcp_in_slow_start(const struct tcp_sock *tp)	190	static __always_inline bool tcp_in_slow_start(const struct tcp_sock *tp)
201	{	191	{
202	return tp->snd_cwnd < tp->snd_ssthresh;	192	return tp->snd_cwnd < tp->snd_ssthresh;
@@ -213,22 +203,7 @@ static __always_inline bool tcp_is_cwnd_limited(const struct sock *sk) · 10 unmodified lines
213	return !!BPF_CORE_READ_BITFIELD(tp, is_cwnd_limited);	203	return !!BPF_CORE_READ_BITFIELD(tp, is_cwnd_limited);
214	}	204	}
215		205
216	static __always_inline void tcp_cong_avoid_ai(struct tcp_sock *tp, __u32 w, __u32 acked)	206	extern __u32 tcp_slow_start(struct tcp_sock *tp, __u32 acked) __ksym;
217	{	207	extern void tcp_cong_avoid_ai(struct tcp_sock *tp, __u32 w, __u32 acked) __ksym;
218	/* If credits accumulated at a higher w, apply them gently now. */
219	if (tp->snd_cwnd_cnt >= w) {
220	tp->snd_cwnd_cnt = 0;
221	tp->snd_cwnd++;
222	}
223
224	tp->snd_cwnd_cnt += acked;
225	if (tp->snd_cwnd_cnt >= w) {
226	__u32 delta = tp->snd_cwnd_cnt / w;
227
228	tp->snd_cwnd_cnt -= delta * w;
229	tp->snd_cwnd += delta;
230	}
231	tp->snd_cwnd = min(tp->snd_cwnd, tp->snd_cwnd_clamp);
232	}
233		208
234	#endif	209	#endif

tools/testing/selftests/bpf/map_tests/lpm_trie_map_batch_ops.c+158−0

@@ -0,0 +1,158 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	#include <arpa/inet.h>
	4	#include <linux/bpf.h>
	5	#include <netinet/in.h>
	6	#include <stdio.h>
	7	#include <errno.h>
	8	#include <string.h>
	9	#include <stdlib.h>
	10
	11	#include <bpf/bpf.h>
	12	#include <bpf/libbpf.h>
	13
	14	#include <test_maps.h>
	15
	16	struct test_lpm_key {
	17	__u32 prefix;
	18	struct in_addr ipv4;
	19	};
	20
	21	static void map_batch_update(int map_fd, __u32 max_entries,
	22	struct test_lpm_key keys, int values)
	23	{
	24	__u32 i;
	25	int err;
	26	char buff[16] = { 0 };
	27	DECLARE_LIBBPF_OPTS(bpf_map_batch_opts, opts,
	28	.elem_flags = 0,
	29	.flags = 0,
	30	);
	31
	32	for (i = 0; i < max_entries; i++) {
	33	keys[i].prefix = 32;
	34	snprintf(buff, 16, "192.168.1.%d", i + 1);
	35	inet_pton(AF_INET, buff, &keys[i].ipv4);
	36	values[i] = i + 1;
	37	}
	38
	39	err = bpf_map_update_batch(map_fd, keys, values, &max_entries, &opts);
	40	CHECK(err, "bpf_map_update_batch()", "error:%s\n", strerror(errno));
	41	}
	42
	43	static void map_batch_verify(int *visited, __u32 max_entries,
	44	struct test_lpm_key keys, int values)
	45	{
	46	char buff[16] = { 0 };
	47	int lower_byte = 0;
	48	__u32 i;
	49
	50	memset(visited, 0, max_entries * sizeof(*visited));
	51	for (i = 0; i < max_entries; i++) {
	52	inet_ntop(AF_INET, &keys[i].ipv4, buff, 32);
	53	CHECK(sscanf(buff, "192.168.1.%d", &lower_byte) == EOF,
	54	"sscanf()", "error: i %d\n", i);
	55	CHECK(lower_byte != values[i], "key/value checking",
	56	"error: i %d key %s value %d\n", i, buff, values[i]);
	57	visited[i] = 1;
	58	}
	59	for (i = 0; i < max_entries; i++) {
	60	CHECK(visited[i] != 1, "visited checking",
	61	"error: keys array at index %d missing\n", i);
	62	}
	63	}
	64
	65	void test_lpm_trie_map_batch_ops(void)
	66	{
	67	struct bpf_create_map_attr xattr = {
	68	.name = "lpm_trie_map",
	69	.map_type = BPF_MAP_TYPE_LPM_TRIE,
	70	.key_size = sizeof(struct test_lpm_key),
	71	.value_size = sizeof(int),
	72	.map_flags = BPF_F_NO_PREALLOC,
	73	};
	74	struct test_lpm_key *keys, key;
	75	int map_fd, values, visited;
	76	__u32 step, count, total, total_success;
	77	const __u32 max_entries = 10;
	78	__u64 batch = 0;
	79	int err;
	80	DECLARE_LIBBPF_OPTS(bpf_map_batch_opts, opts,
	81	.elem_flags = 0,
	82	.flags = 0,
	83	);
	84
	85	xattr.max_entries = max_entries;
	86	map_fd = bpf_create_map_xattr(&xattr);
	87	CHECK(map_fd == -1, "bpf_create_map_xattr()", "error:%s\n",
	88	strerror(errno));
	89
	90	keys = malloc(max_entries * sizeof(struct test_lpm_key));
	91	values = malloc(max_entries * sizeof(int));
	92	visited = malloc(max_entries * sizeof(int));
	93	CHECK(!keys \|\| !values \|\| !visited, "malloc()", "error:%s\n",
	94	strerror(errno));
	95
	96	total_success = 0;
	97	for (step = 1; step < max_entries; step++) {
	98	map_batch_update(map_fd, max_entries, keys, values);
	99	map_batch_verify(visited, max_entries, keys, values);
	100	memset(keys, 0, max_entries * sizeof(*keys));
	101	memset(values, 0, max_entries * sizeof(*values));
	102	batch = 0;
	103	total = 0;
	104	/* iteratively lookup/delete elements with 'step'
	105	* elements each.
	106	*/
	107	count = step;
	108	while (true) {
	109	err = bpf_map_lookup_batch(map_fd,
	110	total ? &batch : NULL, &batch,
	111	keys + total, values + total, &count, &opts);
	112
	113	CHECK((err && errno != ENOENT), "lookup with steps",
	114	"error: %s\n", strerror(errno));
	115
	116	total += count;
	117	if (err)
	118	break;
	119	}
	120
	121	CHECK(total != max_entries, "lookup with steps",
	122	"total = %u, max_entries = %u\n", total, max_entries);
	123
	124	map_batch_verify(visited, max_entries, keys, values);
	125
	126	total = 0;
	127	count = step;
	128	while (total < max_entries) {
	129	if (max_entries - total < step)
	130	count = max_entries - total;
	131	err = bpf_map_delete_batch(map_fd, keys + total, &count,
	132	&opts);
	133	CHECK((err && errno != ENOENT), "delete batch",
	134	"error: %s\n", strerror(errno));
	135	total += count;
	136	if (err)
	137	break;
	138	}
	139	CHECK(total != max_entries, "delete with steps",
	140	"total = %u, max_entries = %u\n", total, max_entries);
	141
	142	/* check map is empty, errono == ENOENT */
	143	err = bpf_map_get_next_key(map_fd, NULL, &key);
	144	CHECK(!err \|\| errno != ENOENT, "bpf_map_get_next_key()",
	145	"error: %s\n", strerror(errno));
	146
	147	total_success++;
	148	}
	149
	150	CHECK(total_success == 0, "check total_success",
	151	"unexpected failure\n");
	152
	153	printf("%s:PASS\n", __func__);
	154
	155	free(keys);
	156	free(values);
	157	free(visited);
	158	}

tools/testing/selftests/bpf/prog_tests/kfunc_call.c+59−0

@@ -0,0 +1,59 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2021 Facebook */
	3	#include <test_progs.h>
	4	#include <network_helpers.h>
	5	#include "kfunc_call_test.skel.h"
	6	#include "kfunc_call_test_subprog.skel.h"
	7
	8	static void test_main(void)
	9	{
	10	struct kfunc_call_test *skel;
	11	int prog_fd, retval, err;
	12
	13	skel = kfunc_call_test__open_and_load();
	14	if (!ASSERT_OK_PTR(skel, "skel"))
	15	return;
	16
	17	prog_fd = bpf_program__fd(skel->progs.kfunc_call_test1);
	18	err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
	19	NULL, NULL, (__u32 *)&retval, NULL);
	20	ASSERT_OK(err, "bpf_prog_test_run(test1)");
	21	ASSERT_EQ(retval, 12, "test1-retval");
	22
	23	prog_fd = bpf_program__fd(skel->progs.kfunc_call_test2);
	24	err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
	25	NULL, NULL, (__u32 *)&retval, NULL);
	26	ASSERT_OK(err, "bpf_prog_test_run(test2)");
	27	ASSERT_EQ(retval, 3, "test2-retval");
	28
	29	kfunc_call_test__destroy(skel);
	30	}
	31
	32	static void test_subprog(void)
	33	{
	34	struct kfunc_call_test_subprog *skel;
	35	int prog_fd, retval, err;
	36
	37	skel = kfunc_call_test_subprog__open_and_load();
	38	if (!ASSERT_OK_PTR(skel, "skel"))
	39	return;
	40
	41	prog_fd = bpf_program__fd(skel->progs.kfunc_call_test1);
	42	err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
	43	NULL, NULL, (__u32 *)&retval, NULL);
	44	ASSERT_OK(err, "bpf_prog_test_run(test1)");
	45	ASSERT_EQ(retval, 10, "test1-retval");
	46	ASSERT_NEQ(skel->data->active_res, -1, "active_res");
	47	ASSERT_EQ(skel->data->sk_state, BPF_TCP_CLOSE, "sk_state");
	48
	49	kfunc_call_test_subprog__destroy(skel);
	50	}
	51
	52	void test_kfunc_call(void)
	53	{
	54	if (test__start_subtest("main"))
	55	test_main();
	56
	57	if (test__start_subtest("subprog"))
	58	test_subprog();
	59	}

tools/testing/selftests/bpf/prog_tests/sockmap_basic.c+40−0

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	#include "test_skmsg_load_helpers.skel.h"	7	#include "test_skmsg_load_helpers.skel.h"
8	#include "test_sockmap_update.skel.h"	8	#include "test_sockmap_update.skel.h"
9	#include "test_sockmap_invalid_update.skel.h"	9	#include "test_sockmap_invalid_update.skel.h"
		10	#include "test_sockmap_skb_verdict_attach.skel.h"
10	#include "bpf_iter_sockmap.skel.h"	11	#include "bpf_iter_sockmap.skel.h"
11		12
12	#define TCP_REPAIR 19 /* TCP sock is under repair right now */	13	#define TCP_REPAIR 19 /* TCP sock is under repair right now */
@@ -281,6 +282,39 @@ out: · 268 unmodified lines
281	bpf_iter_sockmap__destroy(skel);	282	bpf_iter_sockmap__destroy(skel);
282	}	283	}
283		284
		285	static void test_sockmap_skb_verdict_attach(enum bpf_attach_type first,
		286	enum bpf_attach_type second)
		287	{
		288	struct test_sockmap_skb_verdict_attach *skel;
		289	int err, map, verdict;
		290
		291	skel = test_sockmap_skb_verdict_attach__open_and_load();
		292	if (CHECK_FAIL(!skel)) {
		293	perror("test_sockmap_skb_verdict_attach__open_and_load");
		294	return;
		295	}
		296
		297	verdict = bpf_program__fd(skel->progs.prog_skb_verdict);
		298	map = bpf_map__fd(skel->maps.sock_map);
		299
		300	err = bpf_prog_attach(verdict, map, first, 0);
		301	if (CHECK_FAIL(err)) {
		302	perror("bpf_prog_attach");
		303	goto out;
		304	}
		305
		306	err = bpf_prog_attach(verdict, map, second, 0);
		307	assert(err == -1 && errno == EBUSY);
		308
		309	err = bpf_prog_detach2(verdict, map, first);
		310	if (CHECK_FAIL(err)) {
		311	perror("bpf_prog_detach2");
		312	goto out;
		313	}
		314	out:
		315	test_sockmap_skb_verdict_attach__destroy(skel);
		316	}
		317
284	void test_sockmap_basic(void)	318	void test_sockmap_basic(void)
285	{	319	{
286	if (test__start_subtest("sockmap create_update_free"))	320	if (test__start_subtest("sockmap create_update_free"))
@@ -301,4 +335,10 @@ void test_sockmap_basic(void) · 14 unmodified lines
301	test_sockmap_copy(BPF_MAP_TYPE_SOCKMAP);	335	test_sockmap_copy(BPF_MAP_TYPE_SOCKMAP);
302	if (test__start_subtest("sockhash copy"))	336	if (test__start_subtest("sockhash copy"))
303	test_sockmap_copy(BPF_MAP_TYPE_SOCKHASH);	337	test_sockmap_copy(BPF_MAP_TYPE_SOCKHASH);
		338	if (test__start_subtest("sockmap skb_verdict attach")) {
		339	test_sockmap_skb_verdict_attach(BPF_SK_SKB_VERDICT,
		340	BPF_SK_SKB_STREAM_VERDICT);
		341	test_sockmap_skb_verdict_attach(BPF_SK_SKB_STREAM_VERDICT,
		342	BPF_SK_SKB_VERDICT);
		343	}
304	}	344	}

tools/testing/selftests/bpf/prog_tests/sockmap_listen.c+136−0

@@ -1603,6 +1603,141 @@ static void test_reuseport(struct test_sockmap_listen *skel, · 1602 unmodified lines
1603	}	1603	}
1604	}	1604	}
1605		1605
		1606	static void udp_redir_to_connected(int family, int sotype, int sock_mapfd,
		1607	int verd_mapfd, enum redir_mode mode)
		1608	{
		1609	const char *log_prefix = redir_mode_str(mode);
		1610	struct sockaddr_storage addr;
		1611	int c0, c1, p0, p1;
		1612	unsigned int pass;
		1613	socklen_t len;
		1614	int err, n;
		1615	u64 value;
		1616	u32 key;
		1617	char b;
		1618
		1619	zero_verdict_count(verd_mapfd);
		1620
		1621	p0 = socket_loopback(family, sotype \| SOCK_NONBLOCK);
		1622	if (p0 < 0)
		1623	return;
		1624	len = sizeof(addr);
		1625	err = xgetsockname(p0, sockaddr(&addr), &len);
		1626	if (err)
		1627	goto close_peer0;
		1628
		1629	c0 = xsocket(family, sotype \| SOCK_NONBLOCK, 0);
		1630	if (c0 < 0)
		1631	goto close_peer0;
		1632	err = xconnect(c0, sockaddr(&addr), len);
		1633	if (err)
		1634	goto close_cli0;
		1635	err = xgetsockname(c0, sockaddr(&addr), &len);
		1636	if (err)
		1637	goto close_cli0;
		1638	err = xconnect(p0, sockaddr(&addr), len);
		1639	if (err)
		1640	goto close_cli0;
		1641
		1642	p1 = socket_loopback(family, sotype \| SOCK_NONBLOCK);
		1643	if (p1 < 0)
		1644	goto close_cli0;
		1645	err = xgetsockname(p1, sockaddr(&addr), &len);
		1646	if (err)
		1647	goto close_cli0;
		1648
		1649	c1 = xsocket(family, sotype \| SOCK_NONBLOCK, 0);
		1650	if (c1 < 0)
		1651	goto close_peer1;
		1652	err = xconnect(c1, sockaddr(&addr), len);
		1653	if (err)
		1654	goto close_cli1;
		1655	err = xgetsockname(c1, sockaddr(&addr), &len);
		1656	if (err)
		1657	goto close_cli1;
		1658	err = xconnect(p1, sockaddr(&addr), len);
		1659	if (err)
		1660	goto close_cli1;
		1661
		1662	key = 0;
		1663	value = p0;
		1664	err = xbpf_map_update_elem(sock_mapfd, &key, &value, BPF_NOEXIST);
		1665	if (err)
		1666	goto close_cli1;
		1667
		1668	key = 1;
		1669	value = p1;
		1670	err = xbpf_map_update_elem(sock_mapfd, &key, &value, BPF_NOEXIST);
		1671	if (err)
		1672	goto close_cli1;
		1673
		1674	n = write(c1, "a", 1);
		1675	if (n < 0)
		1676	FAIL_ERRNO("%s: write", log_prefix);
		1677	if (n == 0)
		1678	FAIL("%s: incomplete write", log_prefix);
		1679	if (n < 1)
		1680	goto close_cli1;
		1681
		1682	key = SK_PASS;
		1683	err = xbpf_map_lookup_elem(verd_mapfd, &key, &pass);
		1684	if (err)
		1685	goto close_cli1;
		1686	if (pass != 1)
		1687	FAIL("%s: want pass count 1, have %d", log_prefix, pass);
		1688
		1689	n = read(mode == REDIR_INGRESS ? p0 : c0, &b, 1);
		1690	if (n < 0)
		1691	FAIL_ERRNO("%s: read", log_prefix);
		1692	if (n == 0)
		1693	FAIL("%s: incomplete read", log_prefix);
		1694
		1695	close_cli1:
		1696	xclose(c1);
		1697	close_peer1:
		1698	xclose(p1);
		1699	close_cli0:
		1700	xclose(c0);
		1701	close_peer0:
		1702	xclose(p0);
		1703	}
		1704
		1705	static void udp_skb_redir_to_connected(struct test_sockmap_listen *skel,
		1706	struct bpf_map *inner_map, int family)
		1707	{
		1708	int verdict = bpf_program__fd(skel->progs.prog_skb_verdict);
		1709	int verdict_map = bpf_map__fd(skel->maps.verdict_map);
		1710	int sock_map = bpf_map__fd(inner_map);
		1711	int err;
		1712
		1713	err = xbpf_prog_attach(verdict, sock_map, BPF_SK_SKB_VERDICT, 0);
		1714	if (err)
		1715	return;
		1716
		1717	skel->bss->test_ingress = false;
		1718	udp_redir_to_connected(family, SOCK_DGRAM, sock_map, verdict_map,
		1719	REDIR_EGRESS);
		1720	skel->bss->test_ingress = true;
		1721	udp_redir_to_connected(family, SOCK_DGRAM, sock_map, verdict_map,
		1722	REDIR_INGRESS);
		1723
		1724	xbpf_prog_detach2(verdict, sock_map, BPF_SK_SKB_VERDICT);
		1725	}
		1726
		1727	static void test_udp_redir(struct test_sockmap_listen skel, struct bpf_map map,
		1728	int family)
		1729	{
		1730	const char family_name, map_name;
		1731	char s[MAX_TEST_NAME];
		1732
		1733	family_name = family_str(family);
		1734	map_name = map_type_str(map);
		1735	snprintf(s, sizeof(s), "%s %s %s", map_name, family_name, __func__);
		1736	if (!test__start_subtest(s))
		1737	return;
		1738	udp_skb_redir_to_connected(skel, map, family);
		1739	}
		1740
1606	static void run_tests(struct test_sockmap_listen skel, struct bpf_map map,	1741	static void run_tests(struct test_sockmap_listen skel, struct bpf_map map,
1607	int family)	1742	int family)
1608	{	1743	{
@@ -1611,6 +1746,7 @@ static void run_tests(struct test_sockmap_listen skel, struct bpf_map map, · 2 unmodified lines
1611	test_redir(skel, map, family, SOCK_STREAM);	1746	test_redir(skel, map, family, SOCK_STREAM);
1612	test_reuseport(skel, map, family, SOCK_STREAM);	1747	test_reuseport(skel, map, family, SOCK_STREAM);
1613	test_reuseport(skel, map, family, SOCK_DGRAM);	1748	test_reuseport(skel, map, family, SOCK_DGRAM);
		1749	test_udp_redir(skel, map, family);
1614	}	1750	}
1615		1751
1616	void test_sockmap_listen(void)	1752	void test_sockmap_listen(void)

tools/testing/selftests/bpf/prog_tests/test_ima.c+4−2

 		goto close_prog;
 	snprintf(cmd, sizeof(cmd), "./ima_setup.sh setup %s", measured_dir);
-	if (CHECK_FAIL(system(cmd)))
+	err = system(cmd);
+	if (CHECK(err, "failed to run command", "%s, errno = %d\n", cmd, errno))
 		goto close_clean;
 	err = run_measured_process(measured_dir, &skel->bss->monitored_pid);
 close_clean:
 	snprintf(cmd, sizeof(cmd), "./ima_setup.sh cleanup %s", measured_dir);
-	CHECK_FAIL(system(cmd));
+	err = system(cmd);
+	CHECK(err, "failed to run command", "%s, errno = %d\n", cmd, errno);
 close_prog:
 	ima__destroy(skel);
 }

tools/testing/selftests/bpf/progs/bpf_cubic.c+18−18

  * as long as it is used in one of the func ptr
  * under SEC(".struct_ops").
  */
-SEC("struct_ops/bictcp_init")
+SEC("struct_ops/bpf_cubic_init")
-void BPF_PROG(bictcp_init, struct sock *sk)
+void BPF_PROG(bpf_cubic_init, struct sock *sk)
 {
 	struct bictcp *ca = inet_csk_ca(sk);
  * The remaining tcp-cubic functions have an easier way.
  */
 SEC("no-sec-prefix-bictcp_cwnd_event")
-void BPF_PROG(bictcp_cwnd_event, struct sock *sk, enum tcp_ca_event event)
+void BPF_PROG(bpf_cubic_cwnd_event, struct sock *sk, enum tcp_ca_event event)
 {
 	if (event == CA_EVENT_TX_START) {
 		struct bictcp *ca = inet_csk_ca(sk);
 }
 /* Or simply use the BPF_STRUCT_OPS to avoid the SEC boiler plate. */
-void BPF_STRUCT_OPS(bictcp_cong_avoid, struct sock *sk, __u32 ack, __u32 acked)
+void BPF_STRUCT_OPS(bpf_cubic_cong_avoid, struct sock *sk, __u32 ack, __u32 acked)
 {
 	struct tcp_sock *tp = tcp_sk(sk);
 	struct bictcp *ca = inet_csk_ca(sk);
 	tcp_cong_avoid_ai(tp, ca->cnt, acked);
 }
-__u32 BPF_STRUCT_OPS(bictcp_recalc_ssthresh, struct sock *sk)
+__u32 BPF_STRUCT_OPS(bpf_cubic_recalc_ssthresh, struct sock *sk)
 {
 	const struct tcp_sock *tp = tcp_sk(sk);
 	struct bictcp *ca = inet_csk_ca(sk);
 	return max((tp->snd_cwnd * beta) / BICTCP_BETA_SCALE, 2U);
 }
-void BPF_STRUCT_OPS(bictcp_state, struct sock *sk, __u8 new_state)
+void BPF_STRUCT_OPS(bpf_cubic_state, struct sock *sk, __u8 new_state)
 {
 	if (new_state == TCP_CA_Loss) {
 		bictcp_reset(inet_csk_ca(sk));
 	}
 }
-void BPF_STRUCT_OPS(bictcp_acked, struct sock *sk,
+void BPF_STRUCT_OPS(bpf_cubic_acked, struct sock *sk,
 		    const struct ack_sample *sample)
 {
 	const struct tcp_sock *tp = tcp_sk(sk);
 		hystart_update(sk, delay);
 }
-__u32 BPF_STRUCT_OPS(tcp_reno_undo_cwnd, struct sock *sk)
+extern __u32 tcp_reno_undo_cwnd(struct sock *sk) __ksym;
-{
-	const struct tcp_sock *tp = tcp_sk(sk);
-	return max(tp->snd_cwnd, tp->prior_cwnd);
+__u32 BPF_STRUCT_OPS(bpf_cubic_undo_cwnd, struct sock *sk)
+{
+	return tcp_reno_undo_cwnd(sk);
 }
 SEC(".struct_ops")
 struct tcp_congestion_ops cubic = {
-	.init		= (void *)bictcp_init,
+	.init		= (void *)bpf_cubic_init,
-	.ssthresh	= (void *)bictcp_recalc_ssthresh,
+	.ssthresh	= (void *)bpf_cubic_recalc_ssthresh,
-	.cong_avoid	= (void *)bictcp_cong_avoid,
+	.cong_avoid	= (void *)bpf_cubic_cong_avoid,
-	.set_state	= (void *)bictcp_state,
+	.set_state	= (void *)bpf_cubic_state,
-	.undo_cwnd	= (void *)tcp_reno_undo_cwnd,
+	.undo_cwnd	= (void *)bpf_cubic_undo_cwnd,
-	.cwnd_event	= (void *)bictcp_cwnd_event,
+	.cwnd_event	= (void *)bpf_cubic_cwnd_event,
-	.pkts_acked     = (void *)bictcp_acked,
+	.pkts_acked     = (void *)bpf_cubic_acked,
 	.name		= "bpf_cubic",
 };

tools/testing/selftests/bpf/progs/bpf_dctcp.c+6−16

 	return max(tcp_sk(sk)->snd_cwnd, ca->loss_cwnd);
 }
-SEC("struct_ops/tcp_reno_cong_avoid")
+extern void tcp_reno_cong_avoid(struct sock *sk, __u32 ack, __u32 acked) __ksym;
-void BPF_PROG(tcp_reno_cong_avoid, struct sock *sk, __u32 ack, __u32 acked)
-{
-	struct tcp_sock *tp = tcp_sk(sk);
-	if (!tcp_is_cwnd_limited(sk))
-		return;
-	/* In "safe" area, increase. */
+SEC("struct_ops/dctcp_reno_cong_avoid")
-	if (tcp_in_slow_start(tp)) {
+void BPF_PROG(dctcp_cong_avoid, struct sock *sk, __u32 ack, __u32 acked)
-		acked = tcp_slow_start(tp, acked);
+{
-		if (!acked)
+	tcp_reno_cong_avoid(sk, ack, acked);
-			return;
-	}
-	/* In dangerous area, increase slowly. */
-	tcp_cong_avoid_ai(tp, tp->snd_cwnd, acked);
 }
 SEC(".struct_ops")
 	.in_ack_event   = (void *)dctcp_update_alpha,
 	.cwnd_event	= (void *)dctcp_cwnd_event,
 	.ssthresh	= (void *)dctcp_ssthresh,
-	.cong_avoid	= (void *)tcp_reno_cong_avoid,
+	.cong_avoid	= (void *)dctcp_cong_avoid,
 	.undo_cwnd	= (void *)dctcp_cwnd_undo,
 	.set_state	= (void *)dctcp_state,
 	.flags		= TCP_CONG_NEEDS_ECN,

tools/testing/selftests/bpf/progs/kfunc_call_test.c+47−0

@@ -0,0 +1,47 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2021 Facebook */
	3	#include <linux/bpf.h>
	4	#include <bpf/bpf_helpers.h>
	5	#include "bpf_tcp_helpers.h"
	6
	7	extern int bpf_kfunc_call_test2(struct sock *sk, __u32 a, __u32 b) __ksym;
	8	extern __u64 bpf_kfunc_call_test1(struct sock *sk, __u32 a, __u64 b,
	9	__u32 c, __u64 d) __ksym;
	10
	11	SEC("classifier")
	12	int kfunc_call_test2(struct __sk_buff *skb)
	13	{
	14	struct bpf_sock *sk = skb->sk;
	15
	16	if (!sk)
	17	return -1;
	18
	19	sk = bpf_sk_fullsock(sk);
	20	if (!sk)
	21	return -1;
	22
	23	return bpf_kfunc_call_test2((struct sock *)sk, 1, 2);
	24	}
	25
	26	SEC("classifier")
	27	int kfunc_call_test1(struct __sk_buff *skb)
	28	{
	29	struct bpf_sock *sk = skb->sk;
	30	__u64 a = 1ULL << 32;
	31	__u32 ret;
	32
	33	if (!sk)
	34	return -1;
	35
	36	sk = bpf_sk_fullsock(sk);
	37	if (!sk)
	38	return -1;
	39
	40	a = bpf_kfunc_call_test1((struct sock *)sk, 1, a \| 2, 3, a \| 4);
	41	ret = a >> 32; /* ret should be 2 */
	42	ret += (__u32)a; /* ret should be 12 */
	43
	44	return ret;
	45	}
	46
	47	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/kfunc_call_test_subprog.c+42−0

@@ -0,0 +1,42 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2021 Facebook */
	3	#include <linux/bpf.h>
	4	#include <bpf/bpf_helpers.h>
	5	#include "bpf_tcp_helpers.h"
	6
	7	extern const int bpf_prog_active __ksym;
	8	extern __u64 bpf_kfunc_call_test1(struct sock *sk, __u32 a, __u64 b,
	9	__u32 c, __u64 d) __ksym;
	10	extern struct sock bpf_kfunc_call_test3(struct sock sk) __ksym;
	11	int active_res = -1;
	12	int sk_state = -1;
	13
	14	int __noinline f1(struct __sk_buff *skb)
	15	{
	16	struct bpf_sock *sk = skb->sk;
	17	int *active;
	18
	19	if (!sk)
	20	return -1;
	21
	22	sk = bpf_sk_fullsock(sk);
	23	if (!sk)
	24	return -1;
	25
	26	active = (int *)bpf_per_cpu_ptr(&bpf_prog_active,
	27	bpf_get_smp_processor_id());
	28	if (active)
	29	active_res = *active;
	30
	31	sk_state = bpf_kfunc_call_test3((struct sock *)sk)->__sk_common.skc_state;
	32
	33	return (__u32)bpf_kfunc_call_test1((struct sock *)sk, 1, 2, 3, 4);
	34	}
	35
	36	SEC("classifier")
	37	int kfunc_call_test1(struct __sk_buff *skb)
	38	{
	39	return f1(skb);
	40	}
	41
	42	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/test_sockmap_listen.c+22−0

@@ -29,6 +29,7 @@ struct { · 28 unmodified lines
29	} verdict_map SEC(".maps");	29	} verdict_map SEC(".maps");
30		30
31	static volatile bool test_sockmap; /* toggled by user-space */	31	static volatile bool test_sockmap; /* toggled by user-space */
		32	static volatile bool test_ingress; /* toggled by user-space */
32		33
33	SEC("sk_skb/stream_parser")	34	SEC("sk_skb/stream_parser")
34	int prog_stream_parser(struct __sk_buff *skb)	35	int prog_stream_parser(struct __sk_buff *skb)
@@ -55,6 +56,27 @@ int prog_stream_verdict(struct __sk_buff *skb) · 20 unmodified lines
55	return verdict;	56	return verdict;
56	}	57	}
57		58
		59	SEC("sk_skb/skb_verdict")
		60	int prog_skb_verdict(struct __sk_buff *skb)
		61	{
		62	unsigned int *count;
		63	__u32 zero = 0;
		64	int verdict;
		65
		66	if (test_sockmap)
		67	verdict = bpf_sk_redirect_map(skb, &sock_map, zero,
		68	test_ingress ? BPF_F_INGRESS : 0);
		69	else
		70	verdict = bpf_sk_redirect_hash(skb, &sock_hash, &zero,
		71	test_ingress ? BPF_F_INGRESS : 0);
		72
		73	count = bpf_map_lookup_elem(&verdict_map, &verdict);
		74	if (count)
		75	(*count)++;
		76
		77	return verdict;
		78	}
		79
58	SEC("sk_msg")	80	SEC("sk_msg")
59	int prog_msg_verdict(struct sk_msg_md *msg)	81	int prog_msg_verdict(struct sk_msg_md *msg)
60	{	82	{

tools/testing/selftests/bpf/progs/test_sockmap_skb_verdict_attach.c+18−0

@@ -0,0 +1,18 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	#include "vmlinux.h"
	3	#include <bpf/bpf_helpers.h>
	4
	5	struct {
	6	__uint(type, BPF_MAP_TYPE_SOCKMAP);
	7	__uint(max_entries, 2);
	8	__type(key, __u32);
	9	__type(value, __u64);
	10	} sock_map SEC(".maps");
	11
	12	SEC("sk_skb/skb_verdict")
	13	int prog_skb_verdict(struct __sk_buff *skb)
	14	{
	15	return SK_DROP;
	16	}
	17
	18	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/test_xsk.sh+2−1

 	        echo "setting up ${VETH0}: namespace: ${NS0}"
 	fi
 	ip netns add ${NS1}
-	ip link add ${VETH0} type veth peer name ${VETH1}
+	ip link add ${VETH0} numtxqueues 4 numrxqueues 4 type veth peer name ${VETH1} numtxqueues 4 numrxqueues 4
 	if [ -f /proc/net/if_inet6 ]; then
 		echo 1 > /proc/sys/net/ipv6/conf/${VETH0}/disable_ipv6
 	fi
 	ip netns exec ${NS1} ip link set ${VETH1} mtu ${MTU}
 	ip link set ${VETH0} mtu ${MTU}
 	ip netns exec ${NS1} ip link set ${VETH1} up
+	ip netns exec ${NS1} ip link set dev lo up
 	ip link set ${VETH0} up
 }

tools/testing/selftests/bpf/verifier/calls.c+6−6

@@ -19,7 +19,7 @@ · 18 unmodified lines
19	BPF_MOV64_IMM(BPF_REG_0, 2),	19	BPF_MOV64_IMM(BPF_REG_0, 2),
20	BPF_EXIT_INSN(),	20	BPF_EXIT_INSN(),
21	},	21	},
22	.errstr_unpriv = "function calls to other bpf functions are allowed for",	22	.errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
23	.result_unpriv = REJECT,	23	.result_unpriv = REJECT,
24	.result = ACCEPT,	24	.result = ACCEPT,
25	.retval = 1,	25	.retval = 1,
@@ -136,7 +136,7 @@ · 110 unmodified lines
136	{	136	{
137	"calls: wrong src reg",	137	"calls: wrong src reg",
138	.insns = {	138	.insns = {
139	BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 2, 0, 0),	139	BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 3, 0, 0),
140	BPF_MOV64_IMM(BPF_REG_0, 1),	140	BPF_MOV64_IMM(BPF_REG_0, 1),
141	BPF_EXIT_INSN(),	141	BPF_EXIT_INSN(),
142	},	142	},
@@ -397,7 +397,7 @@ · 254 unmodified lines
397	BPF_MOV64_IMM(BPF_REG_0, 1),	397	BPF_MOV64_IMM(BPF_REG_0, 1),
398	BPF_EXIT_INSN(),	398	BPF_EXIT_INSN(),
399	},	399	},
400	.errstr_unpriv = "function calls to other bpf functions are allowed for",	400	.errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
401	.fixup_map_hash_48b = { 3 },	401	.fixup_map_hash_48b = { 3 },
402	.result_unpriv = REJECT,	402	.result_unpriv = REJECT,
403	.result = ACCEPT,	403	.result = ACCEPT,
@@ -1977,7 +1977,7 @@ · 1573 unmodified lines
1977	BPF_EXIT_INSN(),	1977	BPF_EXIT_INSN(),
1978	},	1978	},
1979	.prog_type = BPF_PROG_TYPE_SOCKET_FILTER,	1979	.prog_type = BPF_PROG_TYPE_SOCKET_FILTER,
1980	.errstr_unpriv = "function calls to other bpf functions are allowed for",	1980	.errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
1981	.result_unpriv = REJECT,	1981	.result_unpriv = REJECT,
1982	.result = ACCEPT,	1982	.result = ACCEPT,
1983	},	1983	},
@@ -2003,7 +2003,7 @@ · 19 unmodified lines
2003	BPF_EXIT_INSN(),	2003	BPF_EXIT_INSN(),
2004	},	2004	},
2005	.prog_type = BPF_PROG_TYPE_SOCKET_FILTER,	2005	.prog_type = BPF_PROG_TYPE_SOCKET_FILTER,
2006	.errstr_unpriv = "function calls to other bpf functions are allowed for",	2006	.errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
2007	.errstr = "!read_ok",	2007	.errstr = "!read_ok",
2008	.result = REJECT,	2008	.result = REJECT,
2009	},	2009	},
@@ -2028,7 +2028,7 @@ · 18 unmodified lines
2028	BPF_EXIT_INSN(),	2028	BPF_EXIT_INSN(),
2029	},	2029	},
2030	.prog_type = BPF_PROG_TYPE_SOCKET_FILTER,	2030	.prog_type = BPF_PROG_TYPE_SOCKET_FILTER,
2031	.errstr_unpriv = "function calls to other bpf functions are allowed for",	2031	.errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
2032	.errstr = "!read_ok",	2032	.errstr = "!read_ok",
2033	.result = REJECT,	2033	.result = REJECT,
2034	},	2034	},

tools/testing/selftests/bpf/verifier/dead_code.c+5−5

@@ -85,7 +85,7 @@ · 84 unmodified lines
85	BPF_MOV64_IMM(BPF_REG_0, 12),	85	BPF_MOV64_IMM(BPF_REG_0, 12),
86	BPF_EXIT_INSN(),	86	BPF_EXIT_INSN(),
87	},	87	},
88	.errstr_unpriv = "function calls to other bpf functions are allowed for",	88	.errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
89	.result_unpriv = REJECT,	89	.result_unpriv = REJECT,
90	.result = ACCEPT,	90	.result = ACCEPT,
91	.retval = 7,	91	.retval = 7,
@@ -103,7 +103,7 @@ · 11 unmodified lines
103	BPF_MOV64_IMM(BPF_REG_0, 12),	103	BPF_MOV64_IMM(BPF_REG_0, 12),
104	BPF_EXIT_INSN(),	104	BPF_EXIT_INSN(),
105	},	105	},
106	.errstr_unpriv = "function calls to other bpf functions are allowed for",	106	.errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
107	.result_unpriv = REJECT,	107	.result_unpriv = REJECT,
108	.result = ACCEPT,	108	.result = ACCEPT,
109	.retval = 7,	109	.retval = 7,
@@ -121,7 +121,7 @@ · 11 unmodified lines
121	BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 1, 0, -5),	121	BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 1, 0, -5),
122	BPF_EXIT_INSN(),	122	BPF_EXIT_INSN(),
123	},	123	},
124	.errstr_unpriv = "function calls to other bpf functions are allowed for",	124	.errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
125	.result_unpriv = REJECT,	125	.result_unpriv = REJECT,
126	.result = ACCEPT,	126	.result = ACCEPT,
127	.retval = 7,	127	.retval = 7,
@@ -137,7 +137,7 @@ · 9 unmodified lines
137	BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),	137	BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
138	BPF_EXIT_INSN(),	138	BPF_EXIT_INSN(),
139	},	139	},
140	.errstr_unpriv = "function calls to other bpf functions are allowed for",	140	.errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
141	.result_unpriv = REJECT,	141	.result_unpriv = REJECT,
142	.result = ACCEPT,	142	.result = ACCEPT,
143	.retval = 2,	143	.retval = 2,
@@ -152,7 +152,7 @@ · 8 unmodified lines
152	BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),	152	BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
153	BPF_EXIT_INSN(),	153	BPF_EXIT_INSN(),
154	},	154	},
155	.errstr_unpriv = "function calls to other bpf functions are allowed for",	155	.errstr_unpriv = "loading/calling other bpf or kernel functions are allowed for",
156	.result_unpriv = REJECT,	156	.result_unpriv = REJECT,
157	.result = ACCEPT,	157	.result = ACCEPT,
158	.retval = 2,	158	.retval = 2,

tools/testing/selftests/bpf/vmtest.sh+28−11

 usage()
 {
 	cat <<EOF
-Usage: $0 [-i] [-d <output_dir>] -- [<command>]
+Usage: $0 [-i] [-s] [-d <output_dir>] -- [<command>]
 <command> is the command you would normally run when you are in
 tools/testing/selftests/bpf. e.g:
 	$0 -- ./test_progs -t test_lsm
-If no command is specified, "${DEFAULT_COMMAND}" will be run by
+If no command is specified and a debug shell (-s) is not requested,
-default.
+"${DEFAULT_COMMAND}" will be run by default.
 If you build your kernel using KBUILD_OUTPUT= or O= options, these
 can be passed as environment variables to the script:
 	-d)		Update the output directory (default: ${OUTPUT_DIR})
 	-j)		Number of jobs for compilation, similar to -j in make
 			(default: ${NUM_COMPILE_JOBS})
+	-s)		Instead of powering off the VM, start an interactive
+			shell. If <command> is specified, the shell runs after
+			the command finishes executing
 EOF
 }
 	local init_script_dir="${OUTPUT_DIR}/${MOUNT_DIR}/etc/rcS.d"
 	local init_script="${init_script_dir}/S50-startup"
 	local command="$1"
+	local exit_command="$2"
 	mount_image
 	fi
-	sudo bash -c "cat >${init_script}" <<EOF
+	sudo bash -c "echo '#!/bin/bash' > ${init_script}"
-#!/bin/bash
+	if [[ "${command}" != "" ]]; then
+		sudo bash -c "cat >>${init_script}" <<EOF
 # Have a default value in the exit status file
 # incase the VM is forcefully stopped.
 echo "130" > "/root/${EXIT_STATUS_FILE}"
 	stdbuf -oL -eL ${command}
 	echo "\$?" > "/root/${EXIT_STATUS_FILE}"
 } 2>&1 | tee "/root/${LOG_FILE}"
-poweroff -f
+# Ensure that the logs are written to disk
+sync
 EOF
+	fi
+	sudo bash -c "echo ${exit_command} >> ${init_script}"
 	sudo chmod a+x "${init_script}"
 	unmount_image
 }
 	local kernel_bzimage="${kernel_checkout}/${X86_BZIMAGE}"
 	local command="${DEFAULT_COMMAND}"
 	local update_image="no"
+	local exit_command="poweroff -f"
+	local debug_shell="no"
-	while getopts 'hkid:j:' opt; do
+	while getopts 'hskid:j:' opt; do
 		case ${opt} in
 		i)
 			update_image="yes"
 		j)
 			NUM_COMPILE_JOBS="$OPTARG"
 			;;
+		s)
+			command=""
+			debug_shell="yes"
+			exit_command="bash"
+			;;
 		h)
 			usage
 			exit 0
 	done
 	shift $((OPTIND -1))
-	if [[ $# -eq 0 ]]; then
+	if [[ $# -eq 0  && "${debug_shell}" == "no" ]]; then
 		echo "No command specified, will run ${DEFAULT_COMMAND} in the vm"
 	else
 		command="$@"
 	fi
 	update_selftests "${kernel_checkout}" "${make_command}"
-	update_init_script "${command}"
+	update_init_script "${command}" "${exit_command}"
 	run_vm "${kernel_bzimage}"
-	copy_logs
+	if [[ "${command}" != "" ]]; then
-	echo "Logs saved in ${OUTPUT_DIR}/${LOG_FILE}"
+		copy_logs
+		echo "Logs saved in ${OUTPUT_DIR}/${LOG_FILE}"
+	fi
 }
 catch()

tools/testing/selftests/bpf/xdpxceiver.c+317−383

@@ -41,8 +41,12 @@ · 40 unmodified lines
41	* Reduce the size of the RX ring to a fraction of the fill ring size.	41	* Reduce the size of the RX ring to a fraction of the fill ring size.
42	* iv. fill queue empty	42	* iv. fill queue empty
43	* Do not populate the fill queue and then try to receive pkts.	43	* Do not populate the fill queue and then try to receive pkts.
		44	* f. bpf_link resource persistence
		45	* Configure sockets at indexes 0 and 1, run a traffic on queue ids 0,
		46	* then remove xsk sockets from queue 0 on both veth interfaces and
		47	* finally run a traffic on queues ids 1
44	*	48	*
45	* Total tests: 10	49	* Total tests: 12
46	*	50	*
47	* Flow:	51	* Flow:
48	* -----	52	* -----
@@ -93,6 +97,13 @@ typedef __u16 __sum16; · 44 unmodified lines
93	#include "xdpxceiver.h"	97	#include "xdpxceiver.h"
94	#include "../kselftest.h"	98	#include "../kselftest.h"
95		99
		100	static const char *MAC1 = "\x00\x0A\x56\x9E\xEE\x62";
		101	static const char *MAC2 = "\x00\x0A\x56\x9E\xEE\x61";
		102	static const char *IP1 = "192.168.100.162";
		103	static const char *IP2 = "192.168.100.161";
		104	static const u16 UDP_PORT1 = 2020;
		105	static const u16 UDP_PORT2 = 2121;
		106
96	static void __exit_with_error(int error, const char file, const char func, int line)	107	static void __exit_with_error(int error, const char file, const char func, int line)
97	{	108	{
98	if (configured_mode == TEST_MODE_UNCONFIGURED) {	109	if (configured_mode == TEST_MODE_UNCONFIGURED) {
@@ -108,27 +119,12 @@ static void __exit_with_error(int error, const char file, const char func, int · 9 unmodified lines
108	#define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__)	119	#define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__)
109		120
110	#define print_ksft_result(void)\	121	#define print_ksft_result(void)\
111	(ksft_test_result_pass("PASS: %s %s %s%s%s\n", configured_mode ? "DRV" : "SKB",\	122	(ksft_test_result_pass("PASS: %s %s %s%s%s%s\n", configured_mode ? "DRV" : "SKB",\
112	test_type == TEST_TYPE_POLL ? "POLL" : "NOPOLL",\	123	test_type == TEST_TYPE_POLL ? "POLL" : "NOPOLL",\
113	test_type == TEST_TYPE_TEARDOWN ? "Socket Teardown" : "",\	124	test_type == TEST_TYPE_TEARDOWN ? "Socket Teardown" : "",\
114	test_type == TEST_TYPE_BIDI ? "Bi-directional Sockets" : "",\	125	test_type == TEST_TYPE_BIDI ? "Bi-directional Sockets" : "",\
115	test_type == TEST_TYPE_STATS ? "Stats" : ""))	126	test_type == TEST_TYPE_STATS ? "Stats" : "",\
116		127	test_type == TEST_TYPE_BPF_RES ? "BPF RES" : ""))
117	static void pthread_init_mutex(void)
118	{
119	pthread_mutex_init(&sync_mutex, NULL);
120	pthread_mutex_init(&sync_mutex_tx, NULL);
121	pthread_cond_init(&signal_rx_condition, NULL);
122	pthread_cond_init(&signal_tx_condition, NULL);
123	}
124
125	static void pthread_destroy_mutex(void)
126	{
127	pthread_mutex_destroy(&sync_mutex);
128	pthread_mutex_destroy(&sync_mutex_tx);
129	pthread_cond_destroy(&signal_rx_condition);
130	pthread_cond_destroy(&signal_tx_condition);
131	}
132		128
133	static void memset32_htonl(void dest, u32 val, u32 size)	129	static void memset32_htonl(void dest, u32 val, u32 size)
134	{	130	{
@@ -146,25 +142,12 @@ static void memset32_htonl(void dest, u32 val, u32 size) · 11 unmodified lines
146	return dest;	142	return dest;
147	}	143	}
148		144
149	/*
150	* This function code has been taken from
151	* Linux kernel lib/checksum.c
152	*/
153	static inline unsigned short from32to16(unsigned int x)
154	{
155	/* add up 16-bit and 16-bit for 16+c bit */
156	x = (x & 0xffff) + (x >> 16);
157	/* add up carry.. */
158	x = (x & 0xffff) + (x >> 16);
159	return x;
160	}
161
162	/*	145	/*
163	* Fold a partial checksum	146	* Fold a partial checksum
164	* This function code has been taken from	147	* This function code has been taken from
165	* Linux kernel include/asm-generic/checksum.h	148	* Linux kernel include/asm-generic/checksum.h
166	*/	149	*/
167	static inline __u16 csum_fold(__u32 csum)	150	static __u16 csum_fold(__u32 csum)
168	{	151	{
169	u32 sum = (__force u32)csum;	152	u32 sum = (__force u32)csum;
170		153
@@ -177,7 +160,7 @@ static inline __u16 csum_fold(__u32 csum) · 6 unmodified lines
177	* This function code has been taken from	160	* This function code has been taken from
178	* Linux kernel lib/checksum.c	161	* Linux kernel lib/checksum.c
179	*/	162	*/
180	static inline u32 from64to32(u64 x)	163	static u32 from64to32(u64 x)
181	{	164	{
182	/* add up 32-bit and 32-bit for 32+c bit */	165	/* add up 32-bit and 32-bit for 32+c bit */
183	x = (x & 0xffffffff) + (x >> 32);	166	x = (x & 0xffffffff) + (x >> 32);
@@ -186,13 +169,11 @@ static inline u32 from64to32(u64 x) · 2 unmodified lines
186	return (u32)x;	169	return (u32)x;
187	}	170	}
188		171
189	__u32 csum_tcpudp_nofold(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum);
190
191	/*	172	/*
192	* This function code has been taken from	173	* This function code has been taken from
193	* Linux kernel lib/checksum.c	174	* Linux kernel lib/checksum.c
194	*/	175	*/
195	__u32 csum_tcpudp_nofold(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum)	176	static __u32 csum_tcpudp_nofold(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum)
196	{	177	{
197	unsigned long long s = (__force u32)sum;	178	unsigned long long s = (__force u32)sum;
198		179
@@ -210,13 +191,12 @@ __u32 csum_tcpudp_nofold(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u3 · 11 unmodified lines
210	* This function has been taken from	191	* This function has been taken from
211	* Linux kernel include/asm-generic/checksum.h	192	* Linux kernel include/asm-generic/checksum.h
212	*/	193	*/
213	static inline __u16	194	static __u16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum)
214	csum_tcpudp_magic(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum)
215	{	195	{
216	return csum_fold(csum_tcpudp_nofold(saddr, daddr, len, proto, sum));	196	return csum_fold(csum_tcpudp_nofold(saddr, daddr, len, proto, sum));
217	}	197	}
218		198
219	static inline u16 udp_csum(u32 saddr, u32 daddr, u32 len, u8 proto, u16 *udp_pkt)	199	static u16 udp_csum(u32 saddr, u32 daddr, u32 len, u8 proto, u16 *udp_pkt)
220	{	200	{
221	u32 csum = 0;	201	u32 csum = 0;
222	u32 cnt = 0;	202	u32 cnt = 0;
@@ -271,9 +251,8 @@ static void gen_eth_frame(struct xsk_umem_info *umem, u64 addr) · 48 unmodified lines
271	memcpy(xsk_umem__get_data(umem->buffer, addr), pkt_data, PKT_SIZE);	251	memcpy(xsk_umem__get_data(umem->buffer, addr), pkt_data, PKT_SIZE);
272	}	252	}
273		253
274	static void xsk_configure_umem(struct ifobject data, void buffer, u64 size)	254	static void xsk_configure_umem(struct ifobject data, void buffer, int idx)
275	{	255	{
276	int ret;
277	struct xsk_umem_config cfg = {	256	struct xsk_umem_config cfg = {
278	.fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,	257	.fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,
279	.comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,	258	.comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,
@@ -281,17 +260,22 @@ static void xsk_configure_umem(struct ifobject data, void buffer, u64 size) · 1 unmodified line
281	.frame_headroom = frame_headroom,	260	.frame_headroom = frame_headroom,
282	.flags = XSK_UMEM__DEFAULT_FLAGS	261	.flags = XSK_UMEM__DEFAULT_FLAGS
283	};	262	};
		263	int size = num_frames * XSK_UMEM__DEFAULT_FRAME_SIZE;
		264	struct xsk_umem_info *umem;
		265	int ret;
284		266
285	data->umem = calloc(1, sizeof(struct xsk_umem_info));	267	umem = calloc(1, sizeof(struct xsk_umem_info));
286	if (!data->umem)	268	if (!umem)
287	exit_with_error(errno);	269	exit_with_error(errno);
288		270
289	ret = xsk_umem__create(&data->umem->umem, buffer, size,	271	ret = xsk_umem__create(&umem->umem, buffer, size,
290	&data->umem->fq, &data->umem->cq, &cfg);	272	&umem->fq, &umem->cq, &cfg);
291	if (ret)	273	if (ret)
292	exit_with_error(ret);	274	exit_with_error(ret);
293		275
294	data->umem->buffer = buffer;	276	umem->buffer = buffer;
		277
		278	data->umem_arr[idx] = umem;
295	}	279	}
296		280
297	static void xsk_populate_fill_ring(struct xsk_umem_info *umem)	281	static void xsk_populate_fill_ring(struct xsk_umem_info *umem)
@@ -307,18 +291,19 @@ static void xsk_populate_fill_ring(struct xsk_umem_info *umem) · 9 unmodified lines
307	xsk_ring_prod__submit(&umem->fq, XSK_RING_PROD__DEFAULT_NUM_DESCS);	291	xsk_ring_prod__submit(&umem->fq, XSK_RING_PROD__DEFAULT_NUM_DESCS);
308	}	292	}
309		293
310	static int xsk_configure_socket(struct ifobject *ifobject)	294	static int xsk_configure_socket(struct ifobject *ifobject, int idx)
311	{	295	{
312	struct xsk_socket_config cfg;	296	struct xsk_socket_config cfg;
		297	struct xsk_socket_info *xsk;
313	struct xsk_ring_cons *rxr;	298	struct xsk_ring_cons *rxr;
314	struct xsk_ring_prod *txr;	299	struct xsk_ring_prod *txr;
315	int ret;	300	int ret;
316		301
317	ifobject->xsk = calloc(1, sizeof(struct xsk_socket_info));	302	xsk = calloc(1, sizeof(struct xsk_socket_info));
318	if (!ifobject->xsk)	303	if (!xsk)
319	exit_with_error(errno);	304	exit_with_error(errno);
320		305
321	ifobject->xsk->umem = ifobject->umem;	306	xsk->umem = ifobject->umem;
322	cfg.rx_size = rxqsize;	307	cfg.rx_size = rxqsize;
323	cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;	308	cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
324	cfg.libbpf_flags = 0;	309	cfg.libbpf_flags = 0;
@@ -326,19 +311,20 @@ static int xsk_configure_socket(struct ifobject *ifobject) · 1 unmodified line
326	cfg.bind_flags = xdp_bind_flags;	311	cfg.bind_flags = xdp_bind_flags;
327		312
328	if (test_type != TEST_TYPE_BIDI) {	313	if (test_type != TEST_TYPE_BIDI) {
329	rxr = (ifobject->fv.vector == rx) ? &ifobject->xsk->rx : NULL;	314	rxr = (ifobject->fv.vector == rx) ? &xsk->rx : NULL;
330	txr = (ifobject->fv.vector == tx) ? &ifobject->xsk->tx : NULL;	315	txr = (ifobject->fv.vector == tx) ? &xsk->tx : NULL;
331	} else {	316	} else {
332	rxr = &ifobject->xsk->rx;	317	rxr = &xsk->rx;
333	txr = &ifobject->xsk->tx;	318	txr = &xsk->tx;
334	}	319	}
335		320
336	ret = xsk_socket__create(&ifobject->xsk->xsk, ifobject->ifname,	321	ret = xsk_socket__create(&xsk->xsk, ifobject->ifname, idx,
337	opt_queue, ifobject->umem->umem, rxr, txr, &cfg);	322	ifobject->umem->umem, rxr, txr, &cfg);
338
339	if (ret)	323	if (ret)
340	return 1;	324	return 1;
341		325
		326	ifobject->xsk_arr[idx] = xsk;
		327
342	return 0;	328	return 0;
343	}	329	}
344		330
@@ -364,12 +350,15 @@ static void usage(const char *prog) · 19 unmodified lines
364	ksft_print_msg(str, prog);	350	ksft_print_msg(str, prog);
365	}	351	}
366		352
367	static bool switch_namespace(int idx)	353	static int switch_namespace(const char *nsname)
368	{	354	{
369	char fqns[26] = "/var/run/netns/";	355	char fqns[26] = "/var/run/netns/";
370	int nsfd;	356	int nsfd;
371		357
372	strncat(fqns, ifdict[idx]->nsname, sizeof(fqns) - strlen(fqns) - 1);	358	if (!nsname \|\| strlen(nsname) == 0)
		359	return -1;
		360
		361	strncat(fqns, nsname, sizeof(fqns) - strlen(fqns) - 1);
373	nsfd = open(fqns, O_RDONLY);	362	nsfd = open(fqns, O_RDONLY);
374		363
375	if (nsfd == -1)	364	if (nsfd == -1)
@@ -378,26 +367,9 @@ static bool switch_namespace(int idx) · 2 unmodified lines
378	if (setns(nsfd, 0) == -1)	367	if (setns(nsfd, 0) == -1)
379	exit_with_error(errno);	368	exit_with_error(errno);
380		369
381	return true;	370	print_verbose("NS switched: %s\n", nsname);
382	}
383		371
384	static void nsswitchthread(void args)	372	return nsfd;
385	{
386	struct targs *targs = args;
387
388	targs->retptr = false;
389
390	if (switch_namespace(targs->idx)) {
391	ifdict[targs->idx]->ifindex = if_nametoindex(ifdict[targs->idx]->ifname);
392	if (!ifdict[targs->idx]->ifindex) {
393	ksft_test_result_fail("ERROR: [%s] interface \"%s\" does not exist\n",
394	__func__, ifdict[targs->idx]->ifname);
395	} else {
396	print_verbose("Interface found: %s\n", ifdict[targs->idx]->ifname);
397	targs->retptr = true;
398	}
399	}
400	pthread_exit(NULL);
401	}	373	}
402		374
403	static int validate_interfaces(void)	375	static int validate_interfaces(void)
@@ -409,33 +381,6 @@ static int validate_interfaces(void) · 5 unmodified lines
409	ret = false;	381	ret = false;
410	ksft_test_result_fail("ERROR: interfaces: -i <int>,<ns> -i <int>,<ns>.");	382	ksft_test_result_fail("ERROR: interfaces: -i <int>,<ns> -i <int>,<ns>.");
411	}	383	}
412	if (strcmp(ifdict[i]->nsname, "")) {
413	struct targs *targs;
414
415	targs = malloc(sizeof(*targs));
416	if (!targs)
417	exit_with_error(errno);
418
419	targs->idx = i;
420	if (pthread_create(&ns_thread, NULL, nsswitchthread, targs))
421	exit_with_error(errno);
422
423	pthread_join(ns_thread, NULL);
424
425	if (targs->retptr)
426	print_verbose("NS switched: %s\n", ifdict[i]->nsname);
427
428	free(targs);
429	} else {
430	ifdict[i]->ifindex = if_nametoindex(ifdict[i]->ifname);
431	if (!ifdict[i]->ifindex) {
432	ksft_test_result_fail
433	("ERROR: interface \"%s\" does not exist\n", ifdict[i]->ifname);
434	ret = false;
435	} else {
436	print_verbose("Interface found: %s\n", ifdict[i]->ifname);
437	}
438	}
439	}	384	}
440	return ret;	385	return ret;
441	}	386	}
@@ -447,7 +392,7 @@ static void parse_command_line(int argc, char **argv) · 5 unmodified lines
447	opterr = 0;	392	opterr = 0;
448		393
449	for (;;) {	394	for (;;) {
450	c = getopt_long(argc, argv, "i:q:DC:v", long_options, &option_index);	395	c = getopt_long(argc, argv, "i:DC:v", long_options, &option_index);
451		396
452	if (c == -1)	397	if (c == -1)
453	break;	398	break;
@@ -467,9 +412,6 @@ static void parse_command_line(int argc, char **argv) · 13 unmodified lines
467	MAX_INTERFACES_NAMESPACE_CHARS);	412	MAX_INTERFACES_NAMESPACE_CHARS);
468	interface_index++;	413	interface_index++;
469	break;	414	break;
470	case 'q':
471	opt_queue = atoi(optarg);
472	break;
473	case 'D':	415	case 'D':
474	debug_pkt_dump = 1;	416	debug_pkt_dump = 1;
475	break;	417	break;
@@ -506,7 +448,7 @@ static void kick_tx(struct xsk_socket_info *xsk) · 30 unmodified lines
506	exit_with_error(errno);	448	exit_with_error(errno);
507	}	449	}
508		450
509	static inline void complete_tx_only(struct xsk_socket_info *xsk, int batch_size)	451	static void complete_tx_only(struct xsk_socket_info *xsk, int batch_size)
510	{	452	{
511	unsigned int rcvd;	453	unsigned int rcvd;
512	u32 idx;	454	u32 idx;
@@ -514,7 +456,7 @@ static inline void complete_tx_only(struct xsk_socket_info *xsk, int batch_size) · 1 unmodified line
514	if (!xsk->outstanding_tx)	456	if (!xsk->outstanding_tx)
515	return;	457	return;
516		458
517	if (!NEED_WAKEUP \|\| xsk_ring_prod__needs_wakeup(&xsk->tx))	459	if (xsk_ring_prod__needs_wakeup(&xsk->tx))
518	kick_tx(xsk);	460	kick_tx(xsk);
519		461
520	rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx);	462	rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx);
@@ -602,16 +544,15 @@ static void tx_only(struct xsk_socket_info xsk, u32 frameptr, int batch_size) · 81 unmodified lines
602	xsk_ring_prod__submit(&xsk->tx, batch_size);	544	xsk_ring_prod__submit(&xsk->tx, batch_size);
603	if (!tx_invalid_test) {	545	if (!tx_invalid_test) {
604	xsk->outstanding_tx += batch_size;	546	xsk->outstanding_tx += batch_size;
605	} else {	547	} else if (xsk_ring_prod__needs_wakeup(&xsk->tx)) {
606	if (!NEED_WAKEUP \|\| xsk_ring_prod__needs_wakeup(&xsk->tx))	548	kick_tx(xsk);
607	kick_tx(xsk);
608	}	549	}
609	*frameptr += batch_size;	550	*frameptr += batch_size;
610	*frameptr %= num_frames;	551	*frameptr %= num_frames;
611	complete_tx_only(xsk, batch_size);	552	complete_tx_only(xsk, batch_size);
612	}	553	}
613		554
614	static inline int get_batch_size(int pkt_cnt)	555	static int get_batch_size(int pkt_cnt)
615	{	556	{
616	if (!opt_pkt_count)	557	if (!opt_pkt_count)
617	return BATCH_SIZE;	558	return BATCH_SIZE;
@@ -667,45 +608,40 @@ static void tx_only_all(struct ifobject *ifobject) · 49 unmodified lines
667		608
668	static void worker_pkt_dump(void)	609	static void worker_pkt_dump(void)
669	{	610	{
670	struct in_addr ipaddr;	611	struct ethhdr *ethhdr;
		612	struct iphdr *iphdr;
		613	struct udphdr *udphdr;
		614	char s[128];
		615	int payload;
		616	void *ptr;
671		617
672	fprintf(stdout, "---------------------------------------\n");	618	fprintf(stdout, "---------------------------------------\n");
673	for (int iter = 0; iter < num_frames - 1; iter++) {	619	for (int iter = 0; iter < num_frames - 1; iter++) {
		620	ptr = pkt_buf[iter]->payload;
		621	ethhdr = ptr;
		622	iphdr = ptr + sizeof(*ethhdr);
		623	udphdr = ptr + sizeof(ethhdr) + sizeof(iphdr);
		624
674	/extract L2 frame /	625	/extract L2 frame /
675	fprintf(stdout, "DEBUG>> L2: dst mac: ");	626	fprintf(stdout, "DEBUG>> L2: dst mac: ");
676	for (int i = 0; i < ETH_ALEN; i++)	627	for (int i = 0; i < ETH_ALEN; i++)
677	fprintf(stdout, "%02X", ((struct ethhdr *)	628	fprintf(stdout, "%02X", ethhdr->h_dest[i]);
678	pkt_buf[iter]->payload)->h_dest[i]);
679		629
680	fprintf(stdout, "\nDEBUG>> L2: src mac: ");	630	fprintf(stdout, "\nDEBUG>> L2: src mac: ");
681	for (int i = 0; i < ETH_ALEN; i++)	631	for (int i = 0; i < ETH_ALEN; i++)
682	fprintf(stdout, "%02X", ((struct ethhdr *)	632	fprintf(stdout, "%02X", ethhdr->h_source[i]);
683	pkt_buf[iter]->payload)->h_source[i]);
684		633
685	/extract L3 frame /	634	/extract L3 frame /
686	fprintf(stdout, "\nDEBUG>> L3: ip_hdr->ihl: %02X\n",	635	fprintf(stdout, "\nDEBUG>> L3: ip_hdr->ihl: %02X\n", iphdr->ihl);
687	((struct iphdr *)(pkt_buf[iter]->payload + sizeof(struct ethhdr)))->ihl);	636	fprintf(stdout, "DEBUG>> L3: ip_hdr->saddr: %s\n",
688		637	inet_ntop(AF_INET, &iphdr->saddr, s, sizeof(s)));
689	ipaddr.s_addr =	638	fprintf(stdout, "DEBUG>> L3: ip_hdr->daddr: %s\n",
690	((struct iphdr *)(pkt_buf[iter]->payload + sizeof(struct ethhdr)))->saddr;	639	inet_ntop(AF_INET, &iphdr->daddr, s, sizeof(s)));
691	fprintf(stdout, "DEBUG>> L3: ip_hdr->saddr: %s\n", inet_ntoa(ipaddr));
692
693	ipaddr.s_addr =
694	((struct iphdr *)(pkt_buf[iter]->payload + sizeof(struct ethhdr)))->daddr;
695	fprintf(stdout, "DEBUG>> L3: ip_hdr->daddr: %s\n", inet_ntoa(ipaddr));
696
697	/extract L4 frame /	640	/extract L4 frame /
698	fprintf(stdout, "DEBUG>> L4: udp_hdr->src: %d\n",	641	fprintf(stdout, "DEBUG>> L4: udp_hdr->src: %d\n", ntohs(udphdr->source));
699	ntohs(((struct udphdr *)(pkt_buf[iter]->payload +	642	fprintf(stdout, "DEBUG>> L4: udp_hdr->dst: %d\n", ntohs(udphdr->dest));
700	sizeof(struct ethhdr) +
701	sizeof(struct iphdr)))->source));
702
703	fprintf(stdout, "DEBUG>> L4: udp_hdr->dst: %d\n",
704	ntohs(((struct udphdr *)(pkt_buf[iter]->payload +
705	sizeof(struct ethhdr) +
706	sizeof(struct iphdr)))->dest));
707	/extract L5 frame /	643	/extract L5 frame /
708	int payload = ((uint32_t )(pkt_buf[iter]->payload + PKT_HDR_SIZE));	644	payload = ((uint32_t )(ptr + PKT_HDR_SIZE));
709		645
710	if (payload == EOT) {	646	if (payload == EOT) {
711	print_verbose("End-of-transmission frame received\n");	647	print_verbose("End-of-transmission frame received\n");
@@ -809,37 +745,69 @@ static void worker_pkt_validate(void) · 97 unmodified lines
809	}	745	}
810	}	746	}
811		747
812	static void thread_common_ops(struct ifobject ifobject, void bufs, pthread_mutex_t *mutexptr,	748	static void thread_common_ops(struct ifobject ifobject, void bufs)
813	atomic_int *spinningptr)
814	{	749	{
		750	int umem_sz = num_frames * XSK_UMEM__DEFAULT_FRAME_SIZE;
815	int ctr = 0;	751	int ctr = 0;
816	int ret;	752	int ret;
817		753
818	xsk_configure_umem(ifobject, bufs, num_frames * XSK_UMEM__DEFAULT_FRAME_SIZE);	754	ifobject->ns_fd = switch_namespace(ifobject->nsname);
819	ret = xsk_configure_socket(ifobject);	755
		756	if (test_type == TEST_TYPE_BPF_RES)
		757	umem_sz *= 2;
		758
		759	bufs = mmap(NULL, umem_sz,
		760	PROT_READ \| PROT_WRITE, MAP_PRIVATE \| MAP_ANONYMOUS, -1, 0);
		761	if (bufs == MAP_FAILED)
		762	exit_with_error(errno);
		763
		764	xsk_configure_umem(ifobject, bufs, 0);
		765	ifobject->umem = ifobject->umem_arr[0];
		766	ret = xsk_configure_socket(ifobject, 0);
820		767
821	/* Retry Create Socket if it fails as xsk_socket__create()	768	/* Retry Create Socket if it fails as xsk_socket__create()
822	* is asynchronous	769	* is asynchronous
823	*
824	* Essential to lock Mutex here to prevent Tx thread from
825	* entering before Rx and causing a deadlock
826	*/	770	*/
827	pthread_mutex_lock(mutexptr);
828	while (ret && ctr < SOCK_RECONF_CTR) {	771	while (ret && ctr < SOCK_RECONF_CTR) {
829	atomic_store(spinningptr, 1);	772	xsk_configure_umem(ifobject, bufs, 0);
830	xsk_configure_umem(ifobject, bufs, num_frames * XSK_UMEM__DEFAULT_FRAME_SIZE);	773	ifobject->umem = ifobject->umem_arr[0];
831	ret = xsk_configure_socket(ifobject);	774	ret = xsk_configure_socket(ifobject, 0);
832	usleep(USLEEP_MAX);	775	usleep(USLEEP_MAX);
833	ctr++;	776	ctr++;
834	}	777	}
835	atomic_store(spinningptr, 0);
836	pthread_mutex_unlock(mutexptr);
837		778
838	if (ctr >= SOCK_RECONF_CTR)	779	if (ctr >= SOCK_RECONF_CTR)
839	exit_with_error(ret);	780	exit_with_error(ret);
		781
		782	ifobject->umem = ifobject->umem_arr[0];
		783	ifobject->xsk = ifobject->xsk_arr[0];
		784
		785	if (test_type == TEST_TYPE_BPF_RES) {
		786	xsk_configure_umem(ifobject, (u8 *)bufs + (umem_sz / 2), 1);
		787	ifobject->umem = ifobject->umem_arr[1];
		788	ret = xsk_configure_socket(ifobject, 1);
		789	}
		790
		791	ifobject->umem = ifobject->umem_arr[0];
		792	ifobject->xsk = ifobject->xsk_arr[0];
		793	print_verbose("Interface [%s] vector [%s]\n",
		794	ifobject->ifname, ifobject->fv.vector == tx ? "Tx" : "Rx");
		795	}
		796
		797	static bool testapp_is_test_two_stepped(void)
		798	{
		799	return (test_type != TEST_TYPE_BIDI && test_type != TEST_TYPE_BPF_RES) \|\| second_step;
		800	}
		801
		802	static void testapp_cleanup_xsk_res(struct ifobject *ifobj)
		803	{
		804	if (testapp_is_test_two_stepped()) {
		805	xsk_socket__delete(ifobj->xsk->xsk);
		806	(void)xsk_umem__delete(ifobj->umem->umem);
		807	}
840	}	808	}
841		809
842	static void worker_testapp_validate(void arg)	810	static void worker_testapp_validate_tx(void arg)
843	{	811	{
844	struct udphdr *udp_hdr =	812	struct udphdr *udp_hdr =
845	(struct udphdr *)(pkt_data + sizeof(struct ethhdr) + sizeof(struct iphdr));	813	(struct udphdr *)(pkt_data + sizeof(struct ethhdr) + sizeof(struct iphdr));
@@ -849,157 +817,97 @@ static void worker_testapp_validate(void arg) · 3 unmodified lines
849	struct generic_data data;	817	struct generic_data data;
850	void *bufs = NULL;	818	void *bufs = NULL;
851		819
852	pthread_attr_setstacksize(&attr, THREAD_STACK);	820	if (!second_step)
853		821	thread_common_ops(ifobject, bufs);
854	if (!bidi_pass) {	822
855	bufs = mmap(NULL, num_frames * XSK_UMEM__DEFAULT_FRAME_SIZE,	823	for (int i = 0; i < num_frames; i++) {
856	PROT_READ \| PROT_WRITE, MAP_PRIVATE \| MAP_ANONYMOUS, -1, 0);	824	/send EOT frame /
857	if (bufs == MAP_FAILED)	825	if (i == (num_frames - 1))
858	exit_with_error(errno);	826	data.seqnum = -1;
859		827	else
860	if (strcmp(ifobject->nsname, ""))	828	data.seqnum = i;
861	switch_namespace(ifobject->ifdict_index);	829	gen_udp_hdr(&data, ifobject, udp_hdr);
		830	gen_ip_hdr(ifobject, ip_hdr);
		831	gen_udp_csum(udp_hdr, ip_hdr);
		832	gen_eth_hdr(ifobject, eth_hdr);
		833	gen_eth_frame(ifobject->umem, i * XSK_UMEM__DEFAULT_FRAME_SIZE);
862	}	834	}
863		835
864	if (ifobject->fv.vector == tx) {	836	print_verbose("Sending %d packets on interface %s\n",
865	int spinningrxctr = 0;	837	(opt_pkt_count - 1), ifobject->ifname);
		838	tx_only_all(ifobject);
866		839
867	if (!bidi_pass)	840	testapp_cleanup_xsk_res(ifobject);
868	thread_common_ops(ifobject, bufs, &sync_mutex_tx, &spinning_tx);	841	pthread_exit(NULL);
869		842	}
870	while (atomic_load(&spinning_rx) && spinningrxctr < SOCK_RECONF_CTR) {
871	spinningrxctr++;
872	usleep(USLEEP_MAX);
873	}
874		843
875	print_verbose("Interface [%s] vector [Tx]\n", ifobject->ifname);	844	static void worker_testapp_validate_rx(void arg)
876	for (int i = 0; i < num_frames; i++) {	845	{
877	/send EOT frame /	846	struct ifobject ifobject = (struct ifobject )arg;
878	if (i == (num_frames - 1))	847	struct pollfd fds[MAX_SOCKS] = { };
879	data.seqnum = -1;	848	void *bufs = NULL;
880	else
881	data.seqnum = i;
882	gen_udp_hdr(&data, ifobject, udp_hdr);
883	gen_ip_hdr(ifobject, ip_hdr);
884	gen_udp_csum(udp_hdr, ip_hdr);
885	gen_eth_hdr(ifobject, eth_hdr);
886	gen_eth_frame(ifobject->umem, i * XSK_UMEM__DEFAULT_FRAME_SIZE);
887	}
888		849
889	print_verbose("Sending %d packets on interface %s\n",	850	if (!second_step)
890	(opt_pkt_count - 1), ifobject->ifname);	851	thread_common_ops(ifobject, bufs);
891	tx_only_all(ifobject);
892	} else if (ifobject->fv.vector == rx) {
893	struct pollfd fds[MAX_SOCKS] = { };
894	int ret;
895
896	if (!bidi_pass)
897	thread_common_ops(ifobject, bufs, &sync_mutex_tx, &spinning_rx);
898
899	print_verbose("Interface [%s] vector [Rx]\n", ifobject->ifname);
900	if (stat_test_type != STAT_TEST_RX_FILL_EMPTY)
901	xsk_populate_fill_ring(ifobject->umem);
902
903	TAILQ_INIT(&head);
904	if (debug_pkt_dump) {
905	pkt_buf = calloc(num_frames, sizeof(*pkt_buf));
906	if (!pkt_buf)
907	exit_with_error(errno);
908	}
909		852
910	fds[0].fd = xsk_socket__fd(ifobject->xsk->xsk);	853	if (stat_test_type != STAT_TEST_RX_FILL_EMPTY)
911	fds[0].events = POLLIN;	854	xsk_populate_fill_ring(ifobject->umem);
912		855
913	pthread_mutex_lock(&sync_mutex);	856	TAILQ_INIT(&head);
914	pthread_cond_signal(&signal_rx_condition);	857	if (debug_pkt_dump) {
915	pthread_mutex_unlock(&sync_mutex);	858	pkt_buf = calloc(num_frames, sizeof(*pkt_buf));
		859	if (!pkt_buf)
		860	exit_with_error(errno);
		861	}
916		862
917	while (1) {	863	fds[0].fd = xsk_socket__fd(ifobject->xsk->xsk);
918	if (test_type == TEST_TYPE_POLL) {	864	fds[0].events = POLLIN;
919	ret = poll(fds, 1, POLL_TMOUT);
920	if (ret <= 0)
921	continue;
922	}
923		865
924	if (test_type != TEST_TYPE_STATS) {	866	pthread_barrier_wait(&barr);
925	rx_pkt(ifobject->xsk, fds);
926	worker_pkt_validate();
927	} else {
928	worker_stats_validate(ifobject);
929	}
930		867
931	if (sigvar)	868	while (1) {
932	break;	869	if (test_type != TEST_TYPE_STATS) {
		870	rx_pkt(ifobject->xsk, fds);
		871	worker_pkt_validate();
		872	} else {
		873	worker_stats_validate(ifobject);
933	}	874	}
		875	if (sigvar)
		876	break;
		877	}
934		878
935	if (test_type != TEST_TYPE_STATS)	879	print_verbose("Received %d packets on interface %s\n",
936	print_verbose("Received %d packets on interface %s\n",	880	pkt_counter, ifobject->ifname);
937	pkt_counter, ifobject->ifname);
938		881
939	if (test_type == TEST_TYPE_TEARDOWN)	882	if (test_type == TEST_TYPE_TEARDOWN)
940	print_verbose("Destroying socket\n");	883	print_verbose("Destroying socket\n");
941	}
942		884
943	if ((test_type != TEST_TYPE_BIDI) \|\| bidi_pass) {	885	testapp_cleanup_xsk_res(ifobject);
944	xsk_socket__delete(ifobject->xsk->xsk);
945	(void)xsk_umem__delete(ifobject->umem->umem);
946	}
947	pthread_exit(NULL);	886	pthread_exit(NULL);
948	}	887	}
949		888
950	static void testapp_validate(void)	889	static void testapp_validate(void)
951	{	890	{
952	struct timespec max_wait = { 0, 0 };
953	bool bidi = test_type == TEST_TYPE_BIDI;	891	bool bidi = test_type == TEST_TYPE_BIDI;
		892	bool bpf = test_type == TEST_TYPE_BPF_RES;
954		893
955	pthread_attr_init(&attr);	894	if (pthread_barrier_init(&barr, NULL, 2))
956	pthread_attr_setstacksize(&attr, THREAD_STACK);	895	exit_with_error(errno);
957
958	if ((test_type == TEST_TYPE_BIDI) && bidi_pass) {
959	pthread_init_mutex();
960	if (!switching_notify) {
961	print_verbose("Switching Tx/Rx vectors\n");
962	switching_notify++;
963	}
964	}
965
966	pthread_mutex_lock(&sync_mutex);
967		896
968	/Spawn RX thread /	897	/Spawn RX thread /
969	if (!bidi \|\| !bidi_pass) {	898	pthread_create(&t0, NULL, ifdict_rx->func_ptr, ifdict_rx);
970	if (pthread_create(&t0, &attr, worker_testapp_validate, ifdict[1]))
971	exit_with_error(errno);
972	} else if (bidi && bidi_pass) {
973	/switch Tx/Rx vectors /
974	ifdict[0]->fv.vector = rx;
975	if (pthread_create(&t0, &attr, worker_testapp_validate, ifdict[0]))
976	exit_with_error(errno);
977	}
978
979	if (clock_gettime(CLOCK_REALTIME, &max_wait))
980	exit_with_error(errno);
981	max_wait.tv_sec += TMOUT_SEC;
982		899
983	if (pthread_cond_timedwait(&signal_rx_condition, &sync_mutex, &max_wait) == ETIMEDOUT)	900	pthread_barrier_wait(&barr);
		901	if (pthread_barrier_destroy(&barr))
984	exit_with_error(errno);	902	exit_with_error(errno);
985		903
986	pthread_mutex_unlock(&sync_mutex);
987
988	/Spawn TX thread /	904	/Spawn TX thread /
989	if (!bidi \|\| !bidi_pass) {	905	pthread_create(&t1, NULL, ifdict_tx->func_ptr, ifdict_tx);
990	if (pthread_create(&t1, &attr, worker_testapp_validate, ifdict[0]))
991	exit_with_error(errno);
992	} else if (bidi && bidi_pass) {
993	/switch Tx/Rx vectors /
994	ifdict[1]->fv.vector = tx;
995	if (pthread_create(&t1, &attr, worker_testapp_validate, ifdict[1]))
996	exit_with_error(errno);
997	}
998		906
999	pthread_join(t1, NULL);	907	pthread_join(t1, NULL);
1000	pthread_join(t0, NULL);	908	pthread_join(t0, NULL);
1001		909
1002	if (debug_pkt_dump) {	910	if (debug_pkt_dump && test_type != TEST_TYPE_STATS) {
1003	worker_pkt_dump();	911	worker_pkt_dump();
1004	for (int iter = 0; iter < num_frames - 1; iter++) {	912	for (int iter = 0; iter < num_frames - 1; iter++) {
1005	free(pkt_buf[iter]->payload);	913	free(pkt_buf[iter]->payload);
@@ -1008,20 +916,85 @@ static void testapp_validate(void) · 2 unmodified lines
1008	free(pkt_buf);	916	free(pkt_buf);
1009	}	917	}
1010		918
1011	if (!(test_type == TEST_TYPE_TEARDOWN) && !bidi && !(test_type == TEST_TYPE_STATS))	919	if (!(test_type == TEST_TYPE_TEARDOWN) && !bidi && !bpf && !(test_type == TEST_TYPE_STATS))
1012	print_ksft_result();	920	print_ksft_result();
1013	}	921	}
1014		922
1015	static void testapp_sockets(void)	923	static void testapp_teardown(void)
1016	{	924	{
1017	for (int i = 0; i < ((test_type == TEST_TYPE_TEARDOWN) ? MAX_TEARDOWN_ITER : MAX_BIDI_ITER);	925	int i;
1018	i++) {	926
		927	for (i = 0; i < MAX_TEARDOWN_ITER; i++) {
1019	pkt_counter = 0;	928	pkt_counter = 0;
1020	prev_pkt = -1;	929	prev_pkt = -1;
1021	sigvar = 0;	930	sigvar = 0;
1022	print_verbose("Creating socket\n");	931	print_verbose("Creating socket\n");
1023	testapp_validate();	932	testapp_validate();
1024	test_type == TEST_TYPE_BIDI ? bidi_pass++ : bidi_pass;	933	}
		934
		935	print_ksft_result();
		936	}
		937
		938	static void swap_vectors(struct ifobject ifobj1, struct ifobject ifobj2)
		939	{
		940	void (tmp_func_ptr)(void *) = ifobj1->func_ptr;
		941	enum fvector tmp_vector = ifobj1->fv.vector;
		942
		943	ifobj1->func_ptr = ifobj2->func_ptr;
		944	ifobj1->fv.vector = ifobj2->fv.vector;
		945
		946	ifobj2->func_ptr = tmp_func_ptr;
		947	ifobj2->fv.vector = tmp_vector;
		948
		949	ifdict_tx = ifobj1;
		950	ifdict_rx = ifobj2;
		951	}
		952
		953	static void testapp_bidi(void)
		954	{
		955	for (int i = 0; i < MAX_BIDI_ITER; i++) {
		956	pkt_counter = 0;
		957	prev_pkt = -1;
		958	sigvar = 0;
		959	print_verbose("Creating socket\n");
		960	testapp_validate();
		961	if (!second_step) {
		962	print_verbose("Switching Tx/Rx vectors\n");
		963	swap_vectors(ifdict[1], ifdict[0]);
		964	}
		965	second_step = true;
		966	}
		967
		968	swap_vectors(ifdict[0], ifdict[1]);
		969
		970	print_ksft_result();
		971	}
		972
		973	static void swap_xsk_res(void)
		974	{
		975	xsk_socket__delete(ifdict_tx->xsk->xsk);
		976	xsk_umem__delete(ifdict_tx->umem->umem);
		977	xsk_socket__delete(ifdict_rx->xsk->xsk);
		978	xsk_umem__delete(ifdict_rx->umem->umem);
		979	ifdict_tx->umem = ifdict_tx->umem_arr[1];
		980	ifdict_tx->xsk = ifdict_tx->xsk_arr[1];
		981	ifdict_rx->umem = ifdict_rx->umem_arr[1];
		982	ifdict_rx->xsk = ifdict_rx->xsk_arr[1];
		983	}
		984
		985	static void testapp_bpf_res(void)
		986	{
		987	int i;
		988
		989	for (i = 0; i < MAX_BPF_ITER; i++) {
		990	pkt_counter = 0;
		991	prev_pkt = -1;
		992	sigvar = 0;
		993	print_verbose("Creating socket\n");
		994	testapp_validate();
		995	if (!second_step)
		996	swap_xsk_res();
		997	second_step = true;
1025	}	998	}
1026		999
1027	print_ksft_result();	1000	print_ksft_result();
@@ -1053,78 +1026,33 @@ static void testapp_stats(void) · 25 unmodified lines
1053	print_ksft_result();	1026	print_ksft_result();
1054	}	1027	}
1055		1028
1056	static void init_iface_config(struct ifaceconfigobj *ifaceconfig)	1029	static void init_iface(struct ifobject ifobj, const char dst_mac,
1057	{	1030	const char src_mac, const char dst_ip,
1058	/Init interface0 /	1031	const char *src_ip, const u16 dst_port,
1059	ifdict[0]->fv.vector = tx;	1032	const u16 src_port, enum fvector vector)
1060	memcpy(ifdict[0]->dst_mac, ifaceconfig->dst_mac, ETH_ALEN);
1061	memcpy(ifdict[0]->src_mac, ifaceconfig->src_mac, ETH_ALEN);
1062	ifdict[0]->dst_ip = ifaceconfig->dst_ip.s_addr;
1063	ifdict[0]->src_ip = ifaceconfig->src_ip.s_addr;
1064	ifdict[0]->dst_port = ifaceconfig->dst_port;
1065	ifdict[0]->src_port = ifaceconfig->src_port;
1066
1067	/Init interface1 /
1068	ifdict[1]->fv.vector = rx;
1069	memcpy(ifdict[1]->dst_mac, ifaceconfig->src_mac, ETH_ALEN);
1070	memcpy(ifdict[1]->src_mac, ifaceconfig->dst_mac, ETH_ALEN);
1071	ifdict[1]->dst_ip = ifaceconfig->src_ip.s_addr;
1072	ifdict[1]->src_ip = ifaceconfig->dst_ip.s_addr;
1073	ifdict[1]->dst_port = ifaceconfig->src_port;
1074	ifdict[1]->src_port = ifaceconfig->dst_port;
1075	}
1076
1077	static void nsdisablemodethread(void args)
1078	{	1033	{
1079	struct targs *targs = args;	1034	struct in_addr ip;
1080		1035
1081	targs->retptr = false;	1036	memcpy(ifobj->dst_mac, dst_mac, ETH_ALEN);
		1037	memcpy(ifobj->src_mac, src_mac, ETH_ALEN);
1082		1038
1083	if (switch_namespace(targs->idx)) {	1039	inet_aton(dst_ip, &ip);
1084	targs->retptr = bpf_set_link_xdp_fd(ifdict[targs->idx]->ifindex, -1, targs->flags);	1040	ifobj->dst_ip = ip.s_addr;
1085	} else {
1086	targs->retptr = errno;
1087	print_verbose("Failed to switch namespace to %s\n", ifdict[targs->idx]->nsname);
1088	}
1089
1090	pthread_exit(NULL);
1091	}
1092		1041
1093	static void disable_xdp_mode(int mode)	1042	inet_aton(src_ip, &ip);
1094	{	1043	ifobj->src_ip = ip.s_addr;
1095	int err = 0;
1096	__u32 flags = XDP_FLAGS_UPDATE_IF_NOEXIST \| mode;
1097	char *mode_str = mode & XDP_FLAGS_SKB_MODE ? "skb" : "drv";
1098		1044
1099	for (int i = 0; i < MAX_INTERFACES; i++) {	1045	ifobj->dst_port = dst_port;
1100	if (strcmp(ifdict[i]->nsname, "")) {	1046	ifobj->src_port = src_port;
1101	struct targs *targs;
1102
1103	targs = malloc(sizeof(*targs));
1104	memset(targs, 0, sizeof(*targs));
1105	if (!targs)
1106	exit_with_error(errno);
1107
1108	targs->idx = i;
1109	targs->flags = flags;
1110	if (pthread_create(&ns_thread, NULL, nsdisablemodethread, targs))
1111	exit_with_error(errno);
1112
1113	pthread_join(ns_thread, NULL);
1114	err = targs->retptr;
1115	free(targs);
1116	} else {
1117	err = bpf_set_link_xdp_fd(ifdict[i]->ifindex, -1, flags);
1118	}
1119
1120	if (err) {
1121	print_verbose("Failed to disable %s mode on interface %s\n",
1122	mode_str, ifdict[i]->ifname);
1123	exit_with_error(err);
1124	}
1125		1047
1126	print_verbose("Disabled %s mode for interface: %s\n", mode_str, ifdict[i]->ifname);	1048	if (vector == tx) {
1127	configured_mode = mode & XDP_FLAGS_SKB_MODE ? TEST_MODE_DRV : TEST_MODE_SKB;	1049	ifobj->fv.vector = tx;
		1050	ifobj->func_ptr = worker_testapp_validate_tx;
		1051	ifdict_tx = ifobj;
		1052	} else {
		1053	ifobj->fv.vector = rx;
		1054	ifobj->func_ptr = worker_testapp_validate_rx;
		1055	ifdict_rx = ifobj;
1128	}	1056	}
1129	}	1057	}
1130		1058
@@ -1135,72 +1063,70 @@ static void run_pkt_test(int mode, int type) · 4 unmodified lines
1135	/* reset defaults after potential previous test */	1063	/* reset defaults after potential previous test */
1136	xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;	1064	xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;
1137	pkt_counter = 0;	1065	pkt_counter = 0;
1138	switching_notify = 0;	1066	second_step = 0;
1139	bidi_pass = 0;
1140	prev_pkt = -1;	1067	prev_pkt = -1;
1141	ifdict[0]->fv.vector = tx;
1142	ifdict[1]->fv.vector = rx;
1143	sigvar = 0;	1068	sigvar = 0;
1144	stat_test_type = -1;	1069	stat_test_type = -1;
1145	rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;	1070	rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
1146	frame_headroom = XSK_UMEM__DEFAULT_FRAME_HEADROOM;	1071	frame_headroom = XSK_UMEM__DEFAULT_FRAME_HEADROOM;
1147		1072
		1073	configured_mode = mode;
		1074
1148	switch (mode) {	1075	switch (mode) {
1149	case (TEST_MODE_SKB):	1076	case (TEST_MODE_SKB):
1150	if (configured_mode == TEST_MODE_DRV)
1151	disable_xdp_mode(XDP_FLAGS_DRV_MODE);
1152	xdp_flags \|= XDP_FLAGS_SKB_MODE;	1077	xdp_flags \|= XDP_FLAGS_SKB_MODE;
1153	break;	1078	break;
1154	case (TEST_MODE_DRV):	1079	case (TEST_MODE_DRV):
1155	if (configured_mode == TEST_MODE_SKB)
1156	disable_xdp_mode(XDP_FLAGS_SKB_MODE);
1157	xdp_flags \|= XDP_FLAGS_DRV_MODE;	1080	xdp_flags \|= XDP_FLAGS_DRV_MODE;
1158	break;	1081	break;
1159	default:	1082	default:
1160	break;	1083	break;
1161	}	1084	}
1162		1085
1163	pthread_init_mutex();	1086	switch (test_type) {
1164		1087	case TEST_TYPE_STATS:
1165	if (test_type == TEST_TYPE_STATS)
1166	testapp_stats();	1088	testapp_stats();
1167	else if ((test_type != TEST_TYPE_TEARDOWN) && (test_type != TEST_TYPE_BIDI))	1089	break;
		1090	case TEST_TYPE_TEARDOWN:
		1091	testapp_teardown();
		1092	break;
		1093	case TEST_TYPE_BIDI:
		1094	testapp_bidi();
		1095	break;
		1096	case TEST_TYPE_BPF_RES:
		1097	testapp_bpf_res();
		1098	break;
		1099	default:
1168	testapp_validate();	1100	testapp_validate();
1169	else	1101	break;
1170	testapp_sockets();	1102	}
1171
1172	pthread_destroy_mutex();
1173	}	1103	}
1174		1104
1175	int main(int argc, char **argv)	1105	int main(int argc, char **argv)
1176	{	1106	{
1177	struct rlimit _rlim = { RLIM_INFINITY, RLIM_INFINITY };	1107	struct rlimit _rlim = { RLIM_INFINITY, RLIM_INFINITY };
		1108	bool failure = false;
		1109	int i, j;
1178		1110
1179	if (setrlimit(RLIMIT_MEMLOCK, &_rlim))	1111	if (setrlimit(RLIMIT_MEMLOCK, &_rlim))
1180	exit_with_error(errno);	1112	exit_with_error(errno);
1181		1113
1182	const char *MAC1 = "\x00\x0A\x56\x9E\xEE\x62";
1183	const char *MAC2 = "\x00\x0A\x56\x9E\xEE\x61";
1184	const char *IP1 = "192.168.100.162";
1185	const char *IP2 = "192.168.100.161";
1186	u16 UDP_DST_PORT = 2020;
1187	u16 UDP_SRC_PORT = 2121;
1188	int i, j;
1189
1190	ifaceconfig = malloc(sizeof(struct ifaceconfigobj));
1191	memcpy(ifaceconfig->dst_mac, MAC1, ETH_ALEN);
1192	memcpy(ifaceconfig->src_mac, MAC2, ETH_ALEN);
1193	inet_aton(IP1, &ifaceconfig->dst_ip);
1194	inet_aton(IP2, &ifaceconfig->src_ip);
1195	ifaceconfig->dst_port = UDP_DST_PORT;
1196	ifaceconfig->src_port = UDP_SRC_PORT;
1197
1198	for (int i = 0; i < MAX_INTERFACES; i++) {	1114	for (int i = 0; i < MAX_INTERFACES; i++) {
1199	ifdict[i] = malloc(sizeof(struct ifobject));	1115	ifdict[i] = malloc(sizeof(struct ifobject));
1200	if (!ifdict[i])	1116	if (!ifdict[i])
1201	exit_with_error(errno);	1117	exit_with_error(errno);
1202		1118
1203	ifdict[i]->ifdict_index = i;	1119	ifdict[i]->ifdict_index = i;
		1120	ifdict[i]->xsk_arr = calloc(2, sizeof(struct xsk_socket_info *));
		1121	if (!ifdict[i]->xsk_arr) {
		1122	failure = true;
		1123	goto cleanup;
		1124	}
		1125	ifdict[i]->umem_arr = calloc(2, sizeof(struct xsk_umem_info *));
		1126	if (!ifdict[i]->umem_arr) {
		1127	failure = true;
		1128	goto cleanup;
		1129	}
1204	}	1130	}
1205		1131
1206	setlocale(LC_ALL, "");	1132	setlocale(LC_ALL, "");
@@ -1209,9 +1135,8 @@ int main(int argc, char **argv) · 2 unmodified lines
1209		1135
1210	num_frames = ++opt_pkt_count;	1136	num_frames = ++opt_pkt_count;
1211		1137
1212	init_iface_config(ifaceconfig);	1138	init_iface(ifdict[0], MAC1, MAC2, IP1, IP2, UDP_PORT1, UDP_PORT2, tx);
1213		1139	init_iface(ifdict[1], MAC2, MAC1, IP2, IP1, UDP_PORT2, UDP_PORT1, rx);
1214	disable_xdp_mode(XDP_FLAGS_DRV_MODE);
1215		1140
1216	ksft_set_plan(TEST_MODE_MAX * TEST_TYPE_MAX);	1141	ksft_set_plan(TEST_MODE_MAX * TEST_TYPE_MAX);
1217		1142
@@ -1220,8 +1145,17 @@ int main(int argc, char **argv) · 2 unmodified lines
1220	run_pkt_test(i, j);	1145	run_pkt_test(i, j);
1221	}	1146	}
1222		1147
1223	for (int i = 0; i < MAX_INTERFACES; i++)	1148	cleanup:
		1149	for (int i = 0; i < MAX_INTERFACES; i++) {
		1150	if (ifdict[i]->ns_fd != -1)
		1151	close(ifdict[i]->ns_fd);
		1152	free(ifdict[i]->xsk_arr);
		1153	free(ifdict[i]->umem_arr);
1224	free(ifdict[i]);	1154	free(ifdict[i]);
		1155	}
		1156
		1157	if (failure)
		1158	exit_with_error(errno);
1225		1159
1226	ksft_exit_pass();	1160	ksft_exit_pass();
1227		1161

tools/testing/selftests/bpf/xdpxceiver.h+15−34

 #define MAX_SOCKS 1
 #define MAX_TEARDOWN_ITER 10
 #define MAX_BIDI_ITER 2
+#define MAX_BPF_ITER 2
 #define PKT_HDR_SIZE (sizeof(struct ethhdr) + sizeof(struct iphdr) + \
 			sizeof(struct udphdr))
 #define MIN_PKT_SIZE 64
 #define IP_PKT_TOS 0x9
 #define UDP_PKT_SIZE (IP_PKT_SIZE - sizeof(struct iphdr))
 #define UDP_PKT_DATA_SIZE (UDP_PKT_SIZE - sizeof(struct udphdr))
-#define TMOUT_SEC (3)
 #define EOT (-1)
 #define USLEEP_MAX 200000
-#define THREAD_STACK 60000000
 #define SOCK_RECONF_CTR 10
 #define BATCH_SIZE 64
 #define POLL_TMOUT 1000
-#define NEED_WAKEUP true
 #define DEFAULT_PKT_CNT 10000
 #define RX_FULL_RXQSIZE 32
 	TEST_TYPE_TEARDOWN,
 	TEST_TYPE_BIDI,
 	TEST_TYPE_STATS,
+	TEST_TYPE_BPF_RES,
 	TEST_TYPE_MAX
 };
 static int configured_mode = TEST_MODE_UNCONFIGURED;
 static u8 debug_pkt_dump;
 static u32 num_frames;
-static u8 switching_notify;
+static bool second_step;
-static u8 bidi_pass;
 static int test_type;
-static int opt_queue;
 static int opt_pkt_count;
 static u8 opt_verbose;
 	u32 seqnum;
 };
-struct ifaceconfigobj {
-	u8 dst_mac[ETH_ALEN];
-	u8 src_mac[ETH_ALEN];
-	struct in_addr dst_ip;
-	struct in_addr src_ip;
-	u16 src_port;
-	u16 dst_port;
-} *ifaceconfig;
 struct ifobject {
-	int ifindex;
-	int ifdict_index;
 	char ifname[MAX_INTERFACE_NAME_CHARS];
 	char nsname[MAX_INTERFACES_NAMESPACE_CHARS];
-	struct flow_vector fv;
 	struct xsk_socket_info *xsk;
+	struct xsk_socket_info **xsk_arr;
+	struct xsk_umem_info **umem_arr;
 	struct xsk_umem_info *umem;
-	u8 dst_mac[ETH_ALEN];
+	void *(*func_ptr)(void *arg);
-	u8 src_mac[ETH_ALEN];
+	struct flow_vector fv;
+	int ns_fd;
+	int ifdict_index;
 	u32 dst_ip;
 	u32 src_ip;
 	u16 src_port;
 	u16 dst_port;
+	u8 dst_mac[ETH_ALEN];
+	u8 src_mac[ETH_ALEN];
 };
 static struct ifobject *ifdict[MAX_INTERFACES];
+static struct ifobject *ifdict_rx;
+static struct ifobject *ifdict_tx;
 /*threads*/
-atomic_int spinning_tx;
+pthread_barrier_t barr;
-atomic_int spinning_rx;
+pthread_t t0, t1;
-pthread_mutex_t sync_mutex;
-pthread_mutex_t sync_mutex_tx;
-pthread_cond_t signal_rx_condition;
-pthread_cond_t signal_tx_condition;
-pthread_t t0, t1, ns_thread;
-pthread_attr_t attr;
-struct targs {
-	u8 retptr;
-	int idx;
-	u32 flags;
-};
 TAILQ_HEAD(head_s, pkt) head = TAILQ_HEAD_INITIALIZER(head);
 struct head_s *head_p;