Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Daniel Borkmann says: ==================== bpf-next 2018-11-30 The following pull-request contains BPF updates for your *net-next* tree. (Getting out bit earlier this time to pull in a dependency from bpf.) The main changes are: 1) Add libbpf ABI versioning and document API naming conventions as well as ABI versioning process, from Andrey. 2) Add a new sk_msg_pop_data() helper for sk_msg based BPF programs that is used in conjunction with sk_msg_push_data() for adding / removing meta data to the msg data, from John. 3) Optimize convert_bpf_ld_abs() for 0 offset and fix various lib and testsuite build failures on 32 bit, from David. 4) Make BPF prog dump for !JIT identical to how we dump subprogs when JIT is in use, from Yonghong. 5) Rename btf_get_from_id() to make it more conform with libbpf API naming conventions, from Martin. 6) Add a missing BPF kselftest config item, from Naresh. ====================

David S. Miller committed 7 years ago

Commit 93029d72 parents 3d58c9c b42699525 files · +760 −115Browse files

Unified · Split

25 files

include/linux/bpf.h +4 −2
include/linux/bpf_verifier.h +0 −1
include/uapi/linux/bpf.h +15 −1
kernel/bpf/core.c +2 −1
kernel/bpf/syscall.c +8 −25
kernel/bpf/verifier.c +38 −17
net/core/filter.c +173 −1
net/ipv4/tcp_bpf.c +14 −3
net/tls/tls_sw.c +9 −2
tools/bpf/bpftool/map.c +2 −2
tools/bpf/bpftool/prog.c +1 −1
tools/include/uapi/linux/bpf.h +15 −1
tools/lib/bpf/Makefile +21 −2
tools/lib/bpf/README.rst +139 −0
tools/lib/bpf/btf.c +2 −2
tools/lib/bpf/btf.h +1 −1
tools/lib/bpf/libbpf.c +2 −0
tools/lib/bpf/libbpf.map +121 −0
tools/lib/bpf/libbpf_errno.c +1 −0
tools/testing/selftests/bpf/bpf_helpers.h +2 −0
tools/testing/selftests/bpf/config +1 −0
tools/testing/selftests/bpf/test_btf.c +4 −31
tools/testing/selftests/bpf/test_progs.c +5 −5
tools/testing/selftests/bpf/test_sockmap.c +123 −4
tools/testing/selftests/bpf/test_sockmap_kern.h +57 −13

include/linux/bpf.h+4−2

@@ -299,7 +299,8 @@ struct bpf_prog_aux { · 298 unmodified lines
299	299	u32 max_pkt_offset;
300	300	u32 stack_depth;
301	301	u32 id;
302		− u32 func_cnt;
	302	+ u32 func_cnt; /* used by non-func prog as the number of func progs */
	303	+ u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */
303	304	bool offload_requested;
304	305	struct bpf_prog **func;
305	306	void jit_data; / JIT specific data. arch dependent */
@@ -317,7 +318,8 @@ struct bpf_prog_aux { · 11 unmodified lines
317	318	#endif
318	319	struct bpf_prog_offload *offload;
319	320	struct btf *btf;
320		− u32 type_id; /* type id for this prog/func */
	321	+ struct bpf_func_info *func_info;
	322	+ u32 func_info_cnt;
321	323	union {
322	324	struct work_struct work;
323	325	struct rcu_head rcu;

include/linux/bpf_verifier.h+0−1

@@ -204,7 +204,6 @@ static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log) · 203 unmodified lines
204	204	struct bpf_subprog_info {
205	205	u32 start; /* insn idx of function entry point */
206	206	u16 stack_depth; /* max. stack depth used by this function */
207		− u32 type_id; /* btf type_id for this subprog */
208	207	};
209	208
210	209	/* single container for all structs

include/uapi/linux/bpf.h+15−1

@@ -2268,6 +2268,19 @@ union bpf_attr { · 2267 unmodified lines
2268	2268	*
2269	2269	* Return
2270	2270	* 0 on success, or a negative error in case of failure.
	2271	+ *
	2272	+ * int bpf_msg_pop_data(struct sk_msg_buff *msg, u32 start, u32 pop, u64 flags)
	2273	+ * Description
	2274	+ * Will remove pop bytes from a msg starting at byte start.
	2275	+ * This may result in ENOMEM errors under certain situations if
	2276	+ * an allocation and copy are required due to a full ring buffer.
	2277	+ * However, the helper will try to avoid doing the allocation
	2278	+ * if possible. Other errors can occur if input parameters are
	2279	+ * invalid either due to start byte not being valid part of msg
	2280	+ * payload and/or pop value being to large.
	2281	+ *
	2282	+ * Return
	2283	+ * 0 on success, or a negative erro in case of failure.
2271	2284	*/
2272	2285	#define __BPF_FUNC_MAPPER(FN) \
2273	2286	FN(unspec), \
@@ -2360,7 +2373,8 @@ union bpf_attr { · 86 unmodified lines
2360	2373	FN(map_push_elem), \
2361	2374	FN(map_pop_elem), \
2362	2375	FN(map_peek_elem), \
2363		− FN(msg_push_data),
	2376	+ FN(msg_push_data), \
	2377	+ FN(msg_pop_data),
2364	2378
2365	2379	/* integer value in 'imm' field of BPF_CALL instruction selects which helper
2366	2380	* function eBPF program intends to call

kernel/bpf/core.c+2−1

@@ -411,7 +411,8 @@ static void bpf_get_prog_name(const struct bpf_prog prog, char sym) · 410 unmodified lines
411	411
412	412	/* prog->aux->name will be ignored if full btf name is available */
413	413	if (prog->aux->btf) {
414		− type = btf_type_by_id(prog->aux->btf, prog->aux->type_id);
	414	+ type = btf_type_by_id(prog->aux->btf,
	415	+ prog->aux->func_info[prog->aux->func_idx].type_id);
415	416	func_name = btf_name_by_offset(prog->aux->btf, type->name_off);
416	417	snprintf(sym, (size_t)(end - sym), "_%s", func_name);
417	418	return;

kernel/bpf/syscall.c+8−25

@@ -1214,6 +1214,7 @@ static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock) · 1213 unmodified lines
1214	1214	bpf_prog_free_id(prog, do_idr_lock);
1215	1215	bpf_prog_kallsyms_del_all(prog);
1216	1216	btf_put(prog->aux->btf);
	1217	+ kvfree(prog->aux->func_info);
1217	1218
1218	1219	call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
1219	1220	}
@@ -2219,46 +2220,28 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, · 999 unmodified lines
2219	2220	}
2220	2221
2221	2222	if (prog->aux->btf) {
	2223	+ u32 krec_size = sizeof(struct bpf_func_info);
2222	2224	u32 ucnt, urec_size;
2223	2225
2224	2226	info.btf_id = btf_id(prog->aux->btf);
2225	2227
2226	2228	ucnt = info.func_info_cnt;
2227		− info.func_info_cnt = prog->aux->func_cnt ? : 1;
	2229	+ info.func_info_cnt = prog->aux->func_info_cnt;
2228	2230	urec_size = info.func_info_rec_size;
2229		− info.func_info_rec_size = sizeof(struct bpf_func_info);
	2231	+ info.func_info_rec_size = krec_size;
2230	2232	if (ucnt) {
2231	2233	/* expect passed-in urec_size is what the kernel expects */
2232	2234	if (urec_size != info.func_info_rec_size)
2233	2235	return -EINVAL;
2234	2236
2235	2237	if (bpf_dump_raw_ok()) {
2236		− struct bpf_func_info kern_finfo;
2237	2238	char __user *user_finfo;
2238		− u32 i, insn_offset;
2239	2239
2240	2240	user_finfo = u64_to_user_ptr(info.func_info);
2241		− if (prog->aux->func_cnt) {
2242		− ucnt = min_t(u32, info.func_info_cnt, ucnt);
2243		− insn_offset = 0;
2244		− for (i = 0; i < ucnt; i++) {
2245		− kern_finfo.insn_offset = insn_offset;
2246		− kern_finfo.type_id = prog->aux->func[i]->aux->type_id;
2247		− if (copy_to_user(user_finfo, &kern_finfo,
2248		− sizeof(kern_finfo)))
2249		− return -EFAULT;
2250		−
2251		− /* func[i]->len holds the prog len */
2252		− insn_offset += prog->aux->func[i]->len;
2253		− user_finfo += urec_size;
2254		− }
2255		− } else {
2256		− kern_finfo.insn_offset = 0;
2257		− kern_finfo.type_id = prog->aux->type_id;
2258		− if (copy_to_user(user_finfo, &kern_finfo,
2259		− sizeof(kern_finfo)))
2260		− return -EFAULT;
2261		− }
	2241	+ ucnt = min_t(u32, info.func_info_cnt, ucnt);
	2242	+ if (copy_to_user(user_finfo, prog->aux->func_info,
	2243	+ krec_size * ucnt))
	2244	+ return -EFAULT;
2262	2245	} else {
2263	2246	info.func_info_cnt = 0;
2264	2247	}

kernel/bpf/verifier.c+38−17

@@ -4650,7 +4650,7 @@ static int check_btf_func(struct bpf_prog prog, struct bpf_verifier_env env, · 4649 unmodified lines
4650	4650	{
4651	4651	u32 i, nfuncs, urec_size, min_size, prev_offset;
4652	4652	u32 krec_size = sizeof(struct bpf_func_info);
4653		− struct bpf_func_info krecord = {};
	4653	+ struct bpf_func_info *krecord = NULL;
4654	4654	const struct btf_type *type;
4655	4655	void __user *urecord;
4656	4656	struct btf *btf;
@@ -4682,6 +4682,12 @@ static int check_btf_func(struct bpf_prog prog, struct bpf_verifier_env env, · 25 unmodified lines
4682	4682	urecord = u64_to_user_ptr(attr->func_info);
4683	4683	min_size = min_t(u32, krec_size, urec_size);
4684	4684
	4685	+ krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL \| __GFP_NOWARN);
	4686	+ if (!krecord) {
	4687	+ ret = -ENOMEM;
	4688	+ goto free_btf;
	4689	+ }
	4690	+
4685	4691	for (i = 0; i < nfuncs; i++) {
4686	4692	ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size);
4687	4693	if (ret) {
@@ -4696,59 +4702,69 @@ static int check_btf_func(struct bpf_prog prog, struct bpf_verifier_env env, · 8 unmodified lines
4696	4702	goto free_btf;
4697	4703	}
4698	4704
4699		− if (copy_from_user(&krecord, urecord, min_size)) {
	4705	+ if (copy_from_user(&krecord[i], urecord, min_size)) {
4700	4706	ret = -EFAULT;
4701	4707	goto free_btf;
4702	4708	}
4703	4709
4704	4710	/* check insn_offset */
4705	4711	if (i == 0) {
4706		− if (krecord.insn_offset) {
	4712	+ if (krecord[i].insn_offset) {
4707	4713	verbose(env,
4708	4714	"nonzero insn_offset %u for the first func info record",
4709		− krecord.insn_offset);
	4715	+ krecord[i].insn_offset);
4710	4716	ret = -EINVAL;
4711	4717	goto free_btf;
4712	4718	}
4713		− } else if (krecord.insn_offset <= prev_offset) {
	4719	+ } else if (krecord[i].insn_offset <= prev_offset) {
4714	4720	verbose(env,
4715	4721	"same or smaller insn offset (%u) than previous func info record (%u)",
4716		− krecord.insn_offset, prev_offset);
	4722	+ krecord[i].insn_offset, prev_offset);
4717	4723	ret = -EINVAL;
4718	4724	goto free_btf;
4719	4725	}
4720	4726
4721		− if (env->subprog_info[i].start != krecord.insn_offset) {
	4727	+ if (env->subprog_info[i].start != krecord[i].insn_offset) {
4722	4728	verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");
4723	4729	ret = -EINVAL;
4724	4730	goto free_btf;
4725	4731	}
4726	4732
4727	4733	/* check type_id */
4728		− type = btf_type_by_id(btf, krecord.type_id);
	4734	+ type = btf_type_by_id(btf, krecord[i].type_id);
4729	4735	if (!type \|\| BTF_INFO_KIND(type->info) != BTF_KIND_FUNC) {
4730	4736	verbose(env, "invalid type id %d in func info",
4731		− krecord.type_id);
	4737	+ krecord[i].type_id);
4732	4738	ret = -EINVAL;
4733	4739	goto free_btf;
4734	4740	}
4735	4741
4736		− if (i == 0)
4737		− prog->aux->type_id = krecord.type_id;
4738		− env->subprog_info[i].type_id = krecord.type_id;
4739		−
4740		− prev_offset = krecord.insn_offset;
	4742	+ prev_offset = krecord[i].insn_offset;
4741	4743	urecord += urec_size;
4742	4744	}
4743	4745
4744	4746	prog->aux->btf = btf;
	4747	+ prog->aux->func_info = krecord;
	4748	+ prog->aux->func_info_cnt = nfuncs;
4745	4749	return 0;
4746	4750
4747	4751	free_btf:
4748	4752	btf_put(btf);
	4753	+ kvfree(krecord);
4749	4754	return ret;
4750	4755	}
4751	4756
	4757	+static void adjust_btf_func(struct bpf_verifier_env *env)
	4758	+{
	4759	+ int i;
	4760	+
	4761	+ if (!env->prog->aux->func_info)
	4762	+ return;
	4763	+
	4764	+ for (i = 0; i < env->subprog_cnt; i++)
	4765	+ env->prog->aux->func_info[i].insn_offset = env->subprog_info[i].start;
	4766	+}
	4767	+
4752	4768	/* check %cur's range satisfies %old's */
4753	4769	static bool range_within(struct bpf_reg_state *old,
4754	4770	struct bpf_reg_state *cur)
@@ -6043,15 +6059,17 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 1288 unmodified lines
6043	6059	if (bpf_prog_calc_tag(func[i]))
6044	6060	goto out_free;
6045	6061	func[i]->is_func = 1;
	6062	+ func[i]->aux->func_idx = i;
	6063	+ /* the btf and func_info will be freed only at prog->aux */
	6064	+ func[i]->aux->btf = prog->aux->btf;
	6065	+ func[i]->aux->func_info = prog->aux->func_info;
	6066	+
6046	6067	/* Use bpf_prog_F_tag to indicate functions in stack traces.
6047	6068	* Long term would need debug info to populate names
6048	6069	*/
6049	6070	func[i]->aux->name[0] = 'F';
6050	6071	func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
6051	6072	func[i]->jit_requested = 1;
6052		− /* the btf will be freed only at prog->aux */
6053		− func[i]->aux->btf = prog->aux->btf;
6054		− func[i]->aux->type_id = env->subprog_info[i].type_id;
6055	6073	func[i] = bpf_int_jit_compile(func[i]);
6056	6074	if (!func[i]->jited) {
6057	6075	err = -ENOTSUPP;
@@ -6572,6 +6590,9 @@ skip_full_check: · 514 unmodified lines
6572	6590	convert_pseudo_ld_imm64(env);
6573	6591	}
6574	6592
	6593	+ if (ret == 0)
	6594	+ adjust_btf_func(env);
	6595	+
6575	6596	err_release_maps:
6576	6597	if (!env->prog->aux->used_maps)
6577	6598	/* if we didn't copy map pointers into bpf_prog_info, release

net/core/filter.c+173−1

@@ -463,7 +463,8 @@ static bool convert_bpf_ld_abs(struct sock_filter fp, struct bpf_insn *insnp) · 462 unmodified lines
463	463	bool ldx_off_ok = offset <= S16_MAX;
464	464
465	465	*insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_H);
466		− *insn++ = BPF_ALU64_IMM(BPF_SUB, BPF_REG_TMP, offset);
	466	+ if (offset)
	467	+ *insn++ = BPF_ALU64_IMM(BPF_SUB, BPF_REG_TMP, offset);
467	468	*insn++ = BPF_JMP_IMM(BPF_JSLT, BPF_REG_TMP,
468	469	size, 2 + endian + (!ldx_off_ok * 2));
469	470	if (ldx_off_ok) {
@@ -2424,6 +2425,174 @@ static const struct bpf_func_proto bpf_msg_push_data_proto = { · 1954 unmodified lines
2424	2425	.arg4_type = ARG_ANYTHING,
2425	2426	};
2426	2427
	2428	+static void sk_msg_shift_left(struct sk_msg *msg, int i)
	2429	+{
	2430	+ int prev;
	2431	+
	2432	+ do {
	2433	+ prev = i;
	2434	+ sk_msg_iter_var_next(i);
	2435	+ msg->sg.data[prev] = msg->sg.data[i];
	2436	+ } while (i != msg->sg.end);
	2437	+
	2438	+ sk_msg_iter_prev(msg, end);
	2439	+}
	2440	+
	2441	+static void sk_msg_shift_right(struct sk_msg *msg, int i)
	2442	+{
	2443	+ struct scatterlist tmp, sge;
	2444	+
	2445	+ sk_msg_iter_next(msg, end);
	2446	+ sge = sk_msg_elem_cpy(msg, i);
	2447	+ sk_msg_iter_var_next(i);
	2448	+ tmp = sk_msg_elem_cpy(msg, i);
	2449	+
	2450	+ while (i != msg->sg.end) {
	2451	+ msg->sg.data[i] = sge;
	2452	+ sk_msg_iter_var_next(i);
	2453	+ sge = tmp;
	2454	+ tmp = sk_msg_elem_cpy(msg, i);
	2455	+ }
	2456	+}
	2457	+
	2458	+BPF_CALL_4(bpf_msg_pop_data, struct sk_msg *, msg, u32, start,
	2459	+ u32, len, u64, flags)
	2460	+{
	2461	+ u32 i = 0, l, space, offset = 0;
	2462	+ u64 last = start + len;
	2463	+ int pop;
	2464	+
	2465	+ if (unlikely(flags))
	2466	+ return -EINVAL;
	2467	+
	2468	+ /* First find the starting scatterlist element */
	2469	+ i = msg->sg.start;
	2470	+ do {
	2471	+ l = sk_msg_elem(msg, i)->length;
	2472	+
	2473	+ if (start < offset + l)
	2474	+ break;
	2475	+ offset += l;
	2476	+ sk_msg_iter_var_next(i);
	2477	+ } while (i != msg->sg.end);
	2478	+
	2479	+ /* Bounds checks: start and pop must be inside message */
	2480	+ if (start >= offset + l \|\| last >= msg->sg.size)
	2481	+ return -EINVAL;
	2482	+
	2483	+ space = MAX_MSG_FRAGS - sk_msg_elem_used(msg);
	2484	+
	2485	+ pop = len;
	2486	+ /* --------------\| offset
	2487	+ * -\| start \|-------- len -------\|
	2488	+ *
	2489	+ * \|----- a ----\|-------- pop -------\|----- b ----\|
	2490	+ * \|______________________________________________\| length
	2491	+ *
	2492	+ *
	2493	+ * a: region at front of scatter element to save
	2494	+ * b: region at back of scatter element to save when length > A + pop
	2495	+ * pop: region to pop from element, same as input 'pop' here will be
	2496	+ * decremented below per iteration.
	2497	+ *
	2498	+ * Two top-level cases to handle when start != offset, first B is non
	2499	+ * zero and second B is zero corresponding to when a pop includes more
	2500	+ * than one element.
	2501	+ *
	2502	+ * Then if B is non-zero AND there is no space allocate space and
	2503	+ * compact A, B regions into page. If there is space shift ring to
	2504	+ * the rigth free'ing the next element in ring to place B, leaving
	2505	+ * A untouched except to reduce length.
	2506	+ */
	2507	+ if (start != offset) {
	2508	+ struct scatterlist nsge, sge = sk_msg_elem(msg, i);
	2509	+ int a = start;
	2510	+ int b = sge->length - pop - a;
	2511	+
	2512	+ sk_msg_iter_var_next(i);
	2513	+
	2514	+ if (pop < sge->length - a) {
	2515	+ if (space) {
	2516	+ sge->length = a;
	2517	+ sk_msg_shift_right(msg, i);
	2518	+ nsge = sk_msg_elem(msg, i);
	2519	+ get_page(sg_page(sge));
	2520	+ sg_set_page(nsge,
	2521	+ sg_page(sge),
	2522	+ b, sge->offset + pop + a);
	2523	+ } else {
	2524	+ struct page page, orig;
	2525	+ u8 to, from;
	2526	+
	2527	+ page = alloc_pages(__GFP_NOWARN \|
	2528	+ __GFP_COMP \| GFP_ATOMIC,
	2529	+ get_order(a + b));
	2530	+ if (unlikely(!page))
	2531	+ return -ENOMEM;
	2532	+
	2533	+ sge->length = a;
	2534	+ orig = sg_page(sge);
	2535	+ from = sg_virt(sge);
	2536	+ to = page_address(page);
	2537	+ memcpy(to, from, a);
	2538	+ memcpy(to + a, from + a + pop, b);
	2539	+ sg_set_page(sge, page, a + b, 0);
	2540	+ put_page(orig);
	2541	+ }
	2542	+ pop = 0;
	2543	+ } else if (pop >= sge->length - a) {
	2544	+ sge->length = a;
	2545	+ pop -= (sge->length - a);
	2546	+ }
	2547	+ }
	2548	+
	2549	+ /* From above the current layout _must_ be as follows,
	2550	+ *
	2551	+ * -\| offset
	2552	+ * -\| start
	2553	+ *
	2554	+ * \|---- pop ---\|---------------- b ------------\|
	2555	+ * \|____________________________________________\| length
	2556	+ *
	2557	+ * Offset and start of the current msg elem are equal because in the
	2558	+ * previous case we handled offset != start and either consumed the
	2559	+ * entire element and advanced to the next element OR pop == 0.
	2560	+ *
	2561	+ * Two cases to handle here are first pop is less than the length
	2562	+ * leaving some remainder b above. Simply adjust the element's layout
	2563	+ * in this case. Or pop >= length of the element so that b = 0. In this
	2564	+ * case advance to next element decrementing pop.
	2565	+ */
	2566	+ while (pop) {
	2567	+ struct scatterlist *sge = sk_msg_elem(msg, i);
	2568	+
	2569	+ if (pop < sge->length) {
	2570	+ sge->length -= pop;
	2571	+ sge->offset += pop;
	2572	+ pop = 0;
	2573	+ } else {
	2574	+ pop -= sge->length;
	2575	+ sk_msg_shift_left(msg, i);
	2576	+ }
	2577	+ sk_msg_iter_var_next(i);
	2578	+ }
	2579	+
	2580	+ sk_mem_uncharge(msg->sk, len - pop);
	2581	+ msg->sg.size -= (len - pop);
	2582	+ sk_msg_compute_data_pointers(msg);
	2583	+ return 0;
	2584	+}
	2585	+
	2586	+static const struct bpf_func_proto bpf_msg_pop_data_proto = {
	2587	+ .func = bpf_msg_pop_data,
	2588	+ .gpl_only = false,
	2589	+ .ret_type = RET_INTEGER,
	2590	+ .arg1_type = ARG_PTR_TO_CTX,
	2591	+ .arg2_type = ARG_ANYTHING,
	2592	+ .arg3_type = ARG_ANYTHING,
	2593	+ .arg4_type = ARG_ANYTHING,
	2594	+};
	2595	+
2427	2596	BPF_CALL_1(bpf_get_cgroup_classid, const struct sk_buff *, skb)
2428	2597	{
2429	2598	return task_get_classid(skb);
@@ -5097,6 +5266,7 @@ bool bpf_helper_changes_pkt_data(void *func) · 2667 unmodified lines
5097	5266	func == bpf_xdp_adjust_meta \|\|
5098	5267	func == bpf_msg_pull_data \|\|
5099	5268	func == bpf_msg_push_data \|\|
	5269	+ func == bpf_msg_pop_data \|\|
5100	5270	func == bpf_xdp_adjust_tail \|\|
5101	5271	#if IS_ENABLED(CONFIG_IPV6_SEG6_BPF)
5102	5272	func == bpf_lwt_seg6_store_bytes \|\|
@@ -5393,6 +5563,8 @@ sk_msg_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 290 unmodified lines
5393	5563	return &bpf_msg_pull_data_proto;
5394	5564	case BPF_FUNC_msg_push_data:
5395	5565	return &bpf_msg_push_data_proto;
	5566	+ case BPF_FUNC_msg_pop_data:
	5567	+ return &bpf_msg_pop_data_proto;
5396	5568	default:
5397	5569	return bpf_base_func_proto(func_id);
5398	5570	}

net/ipv4/tcp_bpf.c+14−3

@@ -289,12 +289,23 @@ static int tcp_bpf_send_verdict(struct sock sk, struct sk_psock psock, · 288 unmodified lines
289	289	{
290	290	bool cork = false, enospc = msg->sg.start == msg->sg.end;
291	291	struct sock *sk_redir;
292		− u32 tosend;
	292	+ u32 tosend, delta = 0;
293	293	int ret;
294	294
295	295	more_data:
296		− if (psock->eval == __SK_NONE)
	296	+ if (psock->eval == __SK_NONE) {
	297	+ /* Track delta in msg size to add/subtract it on SK_DROP from
	298	+ * returned to user copied size. This ensures user doesn't
	299	+ * get a positive return code with msg_cut_data and SK_DROP
	300	+ * verdict.
	301	+ */
	302	+ delta = msg->sg.size;
297	303	psock->eval = sk_psock_msg_verdict(sk, psock, msg);
	304	+ if (msg->sg.size < delta)
	305	+ delta -= msg->sg.size;
	306	+ else
	307	+ delta = 0;
	308	+ }
298	309
299	310	if (msg->cork_bytes &&
300	311	msg->cork_bytes > msg->sg.size && !enospc) {
@@ -350,7 +361,7 @@ more_data: · 49 unmodified lines
350	361	default:
351	362	sk_msg_free_partial(sk, msg, tosend);
352	363	sk_msg_apply_bytes(psock, tosend);
353		− *copied -= tosend;
	364	+ *copied -= (tosend + delta);
354	365	return -EACCES;
355	366	}
356	367

net/tls/tls_sw.c+9−2

@@ -687,6 +687,7 @@ static int bpf_exec_tx_verdict(struct sk_msg msg, struct sock sk, · 686 unmodified lines
687	687	struct sock *sk_redir;
688	688	struct tls_rec *rec;
689	689	int err = 0, send;
	690	+ u32 delta = 0;
690	691	bool enospc;
691	692
692	693	psock = sk_psock_get(sk);
@@ -694,8 +695,14 @@ static int bpf_exec_tx_verdict(struct sk_msg msg, struct sock sk, · 1 unmodified line
694	695	return tls_push_record(sk, flags, record_type);
695	696	more_data:
696	697	enospc = sk_msg_full(msg);
697		− if (psock->eval == __SK_NONE)
	698	+ if (psock->eval == __SK_NONE) {
	699	+ delta = msg->sg.size;
698	700	psock->eval = sk_psock_msg_verdict(sk, psock, msg);
	701	+ if (delta < msg->sg.size)
	702	+ delta -= msg->sg.size;
	703	+ else
	704	+ delta = 0;
	705	+ }
699	706	if (msg->cork_bytes && msg->cork_bytes > msg->sg.size &&
700	707	!enospc && !full_record) {
701	708	err = -ENOSPC;
@@ -743,7 +750,7 @@ more_data: · 41 unmodified lines
743	750	msg->apply_bytes -= send;
744	751	if (msg->sg.size == 0)
745	752	tls_free_open_rec(sk);
746		− *copied -= send;
	753	+ *copied -= (send + delta);
747	754	err = -EACCES;
748	755	}
749	756

tools/bpf/bpftool/map.c+2−2

@@ -713,7 +713,7 @@ static int do_dump(int argc, char **argv) · 712 unmodified lines
713	713
714	714	prev_key = NULL;
715	715
716		− err = btf_get_from_id(info.btf_id, &btf);
	716	+ err = btf__get_from_id(info.btf_id, &btf);
717	717	if (err) {
718	718	p_err("failed to get btf");
719	719	goto exit_free;
@@ -857,7 +857,7 @@ static int do_lookup(int argc, char **argv) · 137 unmodified lines
857	857	}
858	858
859	859	/* here means bpf_map_lookup_elem() succeeded */
860		− err = btf_get_from_id(info.btf_id, &btf);
	860	+ err = btf__get_from_id(info.btf_id, &btf);
861	861	if (err) {
862	862	p_err("failed to get btf");
863	863	goto exit_free;

tools/bpf/bpftool/prog.c+1−1

@@ -621,7 +621,7 @@ static int do_dump(int argc, char **argv) · 620 unmodified lines
621	621	goto err_free;
622	622	}
623	623
624		− if (info.btf_id && btf_get_from_id(info.btf_id, &btf)) {
	624	+ if (info.btf_id && btf__get_from_id(info.btf_id, &btf)) {
625	625	p_err("failed to get btf");
626	626	goto err_free;
627	627	}

tools/include/uapi/linux/bpf.h+15−1

@@ -2268,6 +2268,19 @@ union bpf_attr { · 2267 unmodified lines
2268	2268	*
2269	2269	* Return
2270	2270	* 0 on success, or a negative error in case of failure.
	2271	+ *
	2272	+ * int bpf_msg_pop_data(struct sk_msg_buff *msg, u32 start, u32 pop, u64 flags)
	2273	+ * Description
	2274	+ * Will remove pop bytes from a msg starting at byte start.
	2275	+ * This may result in ENOMEM errors under certain situations if
	2276	+ * an allocation and copy are required due to a full ring buffer.
	2277	+ * However, the helper will try to avoid doing the allocation
	2278	+ * if possible. Other errors can occur if input parameters are
	2279	+ * invalid either due to start byte not being valid part of msg
	2280	+ * payload and/or pop value being to large.
	2281	+ *
	2282	+ * Return
	2283	+ * 0 on success, or a negative erro in case of failure.
2271	2284	*/
2272	2285	#define __BPF_FUNC_MAPPER(FN) \
2273	2286	FN(unspec), \
@@ -2360,7 +2373,8 @@ union bpf_attr { · 86 unmodified lines
2360	2373	FN(map_push_elem), \
2361	2374	FN(map_pop_elem), \
2362	2375	FN(map_peek_elem), \
2363		− FN(msg_push_data),
	2376	+ FN(msg_push_data), \
	2377	+ FN(msg_pop_data),
2364	2378
2365	2379	/* integer value in 'imm' field of BPF_CALL instruction selects which helper
2366	2380	* function eBPF program intends to call

tools/lib/bpf/Makefile+21−2

@@ -145,6 +145,12 @@ include $(srctree)/tools/build/Makefile.include · 144 unmodified lines
145	145
146	146	BPF_IN := $(OUTPUT)libbpf-in.o
147	147	LIB_FILE := $(addprefix $(OUTPUT),$(LIB_FILE))
	148	+VERSION_SCRIPT := libbpf.map
	149	+
	150	+GLOBAL_SYM_COUNT = $(shell readelf -s $(BPF_IN) \| \
	151	+ awk '/GLOBAL/ && /DEFAULT/ && !/UND/ {s++} END{print s}')
	152	+VERSIONED_SYM_COUNT = $(shell readelf -s $(OUTPUT)libbpf.so \| \
	153	+ grep -Eo '[^ ]+@LIBBPF_' \| cut -d@ -f1 \| sort -u \| wc -l)
148	154
149	155	CMD_TARGETS = $(LIB_FILE)
150	156
@@ -158,7 +164,7 @@ TARGETS = $(CMD_TARGETS) · 7 unmodified lines
158	164
159	165	all: fixdep all_cmd
160	166
161		−all_cmd: $(CMD_TARGETS)
	167	+all_cmd: $(CMD_TARGETS) check
162	168
163	169	$(BPF_IN): force elfdep bpfdep
164	170	@(test -f ../../include/uapi/linux/bpf.h -a -f ../../../include/uapi/linux/bpf.h && ( \
@@ -176,7 +182,8 @@ $(BPF_IN): force elfdep bpfdep · 11 unmodified lines
176	182	$(Q)$(MAKE) $(build)=libbpf
177	183
178	184	$(OUTPUT)libbpf.so: $(BPF_IN)
179		− $(QUIET_LINK)$(CC) --shared $^ -o $@
	185	+ $(QUIET_LINK)$(CC) --shared -Wl,--version-script=$(VERSION_SCRIPT) \
	186	+ $^ -o $@
180	187
181	188	$(OUTPUT)libbpf.a: $(BPF_IN)
182	189	$(QUIET_LINK)$(RM) $@; $(AR) rcs $@ $^
@@ -184,6 +191,18 @@ $(OUTPUT)libbpf.a: $(BPF_IN) · 1 unmodified line
184	191	$(OUTPUT)test_libbpf: test_libbpf.cpp $(OUTPUT)libbpf.a
185	192	$(QUIET_LINK)$(CXX) $^ -lelf -o $@
186	193
	194	+check: check_abi
	195	+
	196	+check_abi: $(OUTPUT)libbpf.so
	197	+ @if [ "$(GLOBAL_SYM_COUNT)" != "$(VERSIONED_SYM_COUNT)" ]; then \
	198	+ echo "Warning: Num of global symbols in $(BPF_IN)" \
	199	+ "($(GLOBAL_SYM_COUNT)) does NOT match with num of" \
	200	+ "versioned symbols in $^ ($(VERSIONED_SYM_COUNT))." \
	201	+ "Please make sure all LIBBPF_API symbols are" \
	202	+ "versioned in $(VERSION_SCRIPT)." >&2; \
	203	+ exit 1; \
	204	+ fi
	205	+
187	206	define do_install
188	207	if [ ! -d '$(DESTDIR_SQ)$2' ]; then \
189	208	$(INSTALL) -d -m 755 '$(DESTDIR_SQ)$2'; \

tools/lib/bpf/README.rst+139−0

@@ -0,0 +1,139 @@
	1	+.. SPDX-License-Identifier: GPL-2.0
	2	+
	3	+libbpf API naming convention
	4	+============================
	5	+
	6	+libbpf API provides access to a few logically separated groups of
	7	+functions and types. Every group has its own naming convention
	8	+described here. It's recommended to follow these conventions whenever a
	9	+new function or type is added to keep libbpf API clean and consistent.
	10	+
	11	+All types and functions provided by libbpf API should have one of the
	12	+following prefixes: ``bpf_``, ``btf_``, ``libbpf_``.
	13	+
	14	+System call wrappers
	15	+--------------------
	16	+
	17	+System call wrappers are simple wrappers for commands supported by
	18	+sys_bpf system call. These wrappers should go to ``bpf.h`` header file
	19	+and map one-on-one to corresponding commands.
	20	+
	21	+For example ``bpf_map_lookup_elem`` wraps ``BPF_MAP_LOOKUP_ELEM``
	22	+command of sys_bpf, ``bpf_prog_attach`` wraps ``BPF_PROG_ATTACH``, etc.
	23	+
	24	+Objects
	25	+-------
	26	+
	27	+Another class of types and functions provided by libbpf API is "objects"
	28	+and functions to work with them. Objects are high-level abstractions
	29	+such as BPF program or BPF map. They're represented by corresponding
	30	+structures such as ``struct bpf_object``, ``struct bpf_program``,
	31	+``struct bpf_map``, etc.
	32	+
	33	+Structures are forward declared and access to their fields should be
	34	+provided via corresponding getters and setters rather than directly.
	35	+
	36	+These objects are associated with corresponding parts of ELF object that
	37	+contains compiled BPF programs.
	38	+
	39	+For example ``struct bpf_object`` represents ELF object itself created
	40	+from an ELF file or from a buffer, ``struct bpf_program`` represents a
	41	+program in ELF object and ``struct bpf_map`` is a map.
	42	+
	43	+Functions that work with an object have names built from object name,
	44	+double underscore and part that describes function purpose.
	45	+
	46	+For example ``bpf_object__open`` consists of the name of corresponding
	47	+object, ``bpf_object``, double underscore and ``open`` that defines the
	48	+purpose of the function to open ELF file and create ``bpf_object`` from
	49	+it.
	50	+
	51	+Another example: ``bpf_program__load`` is named for corresponding
	52	+object, ``bpf_program``, that is separated from other part of the name
	53	+by double underscore.
	54	+
	55	+All objects and corresponding functions other than BTF related should go
	56	+to ``libbpf.h``. BTF types and functions should go to ``btf.h``.
	57	+
	58	+Auxiliary functions
	59	+-------------------
	60	+
	61	+Auxiliary functions and types that don't fit well in any of categories
	62	+described above should have ``libbpf_`` prefix, e.g.
	63	+``libbpf_get_error`` or ``libbpf_prog_type_by_name``.
	64	+
	65	+libbpf ABI
	66	+==========
	67	+
	68	+libbpf can be both linked statically or used as DSO. To avoid possible
	69	+conflicts with other libraries an application is linked with, all
	70	+non-static libbpf symbols should have one of the prefixes mentioned in
	71	+API documentation above. See API naming convention to choose the right
	72	+name for a new symbol.
	73	+
	74	+Symbol visibility
	75	+-----------------
	76	+
	77	+libbpf follow the model when all global symbols have visibility "hidden"
	78	+by default and to make a symbol visible it has to be explicitly
	79	+attributed with ``LIBBPF_API`` macro. For example:
	80	+
	81	+.. code-block:: c
	82	+
	83	+ LIBBPF_API int bpf_prog_get_fd_by_id(__u32 id);
	84	+
	85	+This prevents from accidentally exporting a symbol, that is not supposed
	86	+to be a part of ABI what, in turn, improves both libbpf developer- and
	87	+user-experiences.
	88	+
	89	+ABI versionning
	90	+---------------
	91	+
	92	+To make future ABI extensions possible libbpf ABI is versioned.
	93	+Versioning is implemented by ``libbpf.map`` version script that is
	94	+passed to linker.
	95	+
	96	+Version name is ``LIBBPF_`` prefix + three-component numeric version,
	97	+starting from ``0.0.1``.
	98	+
	99	+Every time ABI is being changed, e.g. because a new symbol is added or
	100	+semantic of existing symbol is changed, ABI version should be bumped.
	101	+
	102	+For example, if current state of ``libbpf.map`` is:
	103	+
	104	+.. code-block::
	105	+ LIBBPF_0.0.1 {
	106	+ global:
	107	+ bpf_func_a;
	108	+ bpf_func_b;
	109	+ local:
	110	+ \*;
	111	+ };
	112	+
	113	+, and a new symbol ``bpf_func_c`` is being introduced, then
	114	+``libbpf.map`` should be changed like this:
	115	+
	116	+.. code-block::
	117	+ LIBBPF_0.0.1 {
	118	+ global:
	119	+ bpf_func_a;
	120	+ bpf_func_b;
	121	+ local:
	122	+ \*;
	123	+ };
	124	+ LIBBPF_0.0.2 {
	125	+ global:
	126	+ bpf_func_c;
	127	+ } LIBBPF_0.0.1;
	128	+
	129	+, where new version ``LIBBPF_0.0.2`` depends on the previous
	130	+``LIBBPF_0.0.1``.
	131	+
	132	+Format of version script and ways to handle ABI changes, including
	133	+incompatible ones, described in details in [1].
	134	+
	135	+Links
	136	+=====
	137	+
	138	+[1] https://www.akkadia.org/drepper/dsohowto.pdf
	139	+ (Chapter 3. Maintaining APIs and ABIs).

tools/lib/bpf/btf.c+2−2

@@ -415,7 +415,7 @@ const char btf__name_by_offset(const struct btf btf, __u32 offset) · 414 unmodified lines
415	415	return NULL;
416	416	}
417	417
418		−int btf_get_from_id(__u32 id, struct btf **btf)
	418	+int btf__get_from_id(__u32 id, struct btf **btf)
419	419	{
420	420	struct bpf_btf_info btf_info = { 0 };
421	421	__u32 len = sizeof(btf_info);
@@ -466,7 +466,7 @@ int btf_get_from_id(__u32 id, struct btf **btf) · 44 unmodified lines
466	466	goto exit_free;
467	467	}
468	468
469		− btf = btf__new((__u8 )btf_info.btf, btf_info.btf_size, NULL);
	469	+ btf = btf__new((__u8 )(long)btf_info.btf, btf_info.btf_size, NULL);
470	470	if (IS_ERR(*btf)) {
471	471	err = PTR_ERR(*btf);
472	472	*btf = NULL;

tools/lib/bpf/btf.h+1−1

@@ -73,7 +73,7 @@ LIBBPF_API __s64 btf__resolve_size(const struct btf *btf, __u32 type_id); · 72 unmodified lines
73	73	LIBBPF_API int btf__resolve_type(const struct btf *btf, __u32 type_id);
74	74	LIBBPF_API int btf__fd(const struct btf *btf);
75	75	LIBBPF_API const char btf__name_by_offset(const struct btf btf, __u32 offset);
76		−LIBBPF_API int btf_get_from_id(__u32 id, struct btf **btf);
	76	+LIBBPF_API int btf__get_from_id(__u32 id, struct btf **btf);
77	77
78	78	struct btf_ext btf_ext__new(__u8 data, __u32 size, btf_print_fn_t err_log);
79	79	void btf_ext__free(struct btf_ext *btf_ext);

tools/lib/bpf/libbpf.c+2−0

@@ -9,7 +9,9 @@ · 8 unmodified lines
9	9	* Copyright (C) 2017 Nicira, Inc.
10	10	*/
11	11
	12	+#ifndef _GNU_SOURCE
12	13	#define _GNU_SOURCE
	14	+#endif
13	15	#include <stdlib.h>
14	16	#include <stdio.h>
15	17	#include <stdarg.h>

tools/lib/bpf/libbpf.map+121−0

@@ -0,0 +1,121 @@
	1	+LIBBPF_0.0.1 {
	2	+ global:
	3	+ bpf_btf_get_fd_by_id;
	4	+ bpf_create_map;
	5	+ bpf_create_map_in_map;
	6	+ bpf_create_map_in_map_node;
	7	+ bpf_create_map_name;
	8	+ bpf_create_map_node;
	9	+ bpf_create_map_xattr;
	10	+ bpf_load_btf;
	11	+ bpf_load_program;
	12	+ bpf_load_program_xattr;
	13	+ bpf_map__btf_key_type_id;
	14	+ bpf_map__btf_value_type_id;
	15	+ bpf_map__def;
	16	+ bpf_map__fd;
	17	+ bpf_map__is_offload_neutral;
	18	+ bpf_map__name;
	19	+ bpf_map__next;
	20	+ bpf_map__pin;
	21	+ bpf_map__prev;
	22	+ bpf_map__priv;
	23	+ bpf_map__reuse_fd;
	24	+ bpf_map__set_ifindex;
	25	+ bpf_map__set_inner_map_fd;
	26	+ bpf_map__set_priv;
	27	+ bpf_map__unpin;
	28	+ bpf_map_delete_elem;
	29	+ bpf_map_get_fd_by_id;
	30	+ bpf_map_get_next_id;
	31	+ bpf_map_get_next_key;
	32	+ bpf_map_lookup_and_delete_elem;
	33	+ bpf_map_lookup_elem;
	34	+ bpf_map_update_elem;
	35	+ bpf_obj_get;
	36	+ bpf_obj_get_info_by_fd;
	37	+ bpf_obj_pin;
	38	+ bpf_object__btf_fd;
	39	+ bpf_object__close;
	40	+ bpf_object__find_map_by_name;
	41	+ bpf_object__find_map_by_offset;
	42	+ bpf_object__find_program_by_title;
	43	+ bpf_object__kversion;
	44	+ bpf_object__load;
	45	+ bpf_object__name;
	46	+ bpf_object__next;
	47	+ bpf_object__open;
	48	+ bpf_object__open_buffer;
	49	+ bpf_object__open_xattr;
	50	+ bpf_object__pin;
	51	+ bpf_object__pin_maps;
	52	+ bpf_object__pin_programs;
	53	+ bpf_object__priv;
	54	+ bpf_object__set_priv;
	55	+ bpf_object__unload;
	56	+ bpf_object__unpin_maps;
	57	+ bpf_object__unpin_programs;
	58	+ bpf_perf_event_read_simple;
	59	+ bpf_prog_attach;
	60	+ bpf_prog_detach;
	61	+ bpf_prog_detach2;
	62	+ bpf_prog_get_fd_by_id;
	63	+ bpf_prog_get_next_id;
	64	+ bpf_prog_load;
	65	+ bpf_prog_load_xattr;
	66	+ bpf_prog_query;
	67	+ bpf_prog_test_run;
	68	+ bpf_program__fd;
	69	+ bpf_program__is_kprobe;
	70	+ bpf_program__is_perf_event;
	71	+ bpf_program__is_raw_tracepoint;
	72	+ bpf_program__is_sched_act;
	73	+ bpf_program__is_sched_cls;
	74	+ bpf_program__is_socket_filter;
	75	+ bpf_program__is_tracepoint;
	76	+ bpf_program__is_xdp;
	77	+ bpf_program__load;
	78	+ bpf_program__next;
	79	+ bpf_program__nth_fd;
	80	+ bpf_program__pin;
	81	+ bpf_program__pin_instance;
	82	+ bpf_program__prev;
	83	+ bpf_program__priv;
	84	+ bpf_program__set_expected_attach_type;
	85	+ bpf_program__set_ifindex;
	86	+ bpf_program__set_kprobe;
	87	+ bpf_program__set_perf_event;
	88	+ bpf_program__set_prep;
	89	+ bpf_program__set_priv;
	90	+ bpf_program__set_raw_tracepoint;
	91	+ bpf_program__set_sched_act;
	92	+ bpf_program__set_sched_cls;
	93	+ bpf_program__set_socket_filter;
	94	+ bpf_program__set_tracepoint;
	95	+ bpf_program__set_type;
	96	+ bpf_program__set_xdp;
	97	+ bpf_program__title;
	98	+ bpf_program__unload;
	99	+ bpf_program__unpin;
	100	+ bpf_program__unpin_instance;
	101	+ bpf_raw_tracepoint_open;
	102	+ bpf_set_link_xdp_fd;
	103	+ bpf_task_fd_query;
	104	+ bpf_verify_program;
	105	+ btf__fd;
	106	+ btf__find_by_name;
	107	+ btf__free;
	108	+ btf__get_from_id;
	109	+ btf__name_by_offset;
	110	+ btf__new;
	111	+ btf__resolve_size;
	112	+ btf__resolve_type;
	113	+ btf__type_by_id;
	114	+ libbpf_attach_type_by_name;
	115	+ libbpf_get_error;
	116	+ libbpf_prog_type_by_name;
	117	+ libbpf_set_print;
	118	+ libbpf_strerror;
	119	+ local:
	120	+ *;
	121	+};

tools/lib/bpf/libbpf_errno.c+1−0

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	7	* Copyright (C) 2017 Nicira, Inc.
8	8	*/
9	9
	10	+#undef _GNU_SOURCE
10	11	#include <stdio.h>
11	12	#include <string.h>
12	13

tools/testing/selftests/bpf/bpf_helpers.h+2−0

@@ -113,6 +113,8 @@ static int (bpf_msg_pull_data)(void ctx, int start, int end, int flags) = · 112 unmodified lines
113	113	(void *) BPF_FUNC_msg_pull_data;
114	114	static int (bpf_msg_push_data)(void ctx, int start, int end, int flags) =
115	115	(void *) BPF_FUNC_msg_push_data;
	116	+static int (bpf_msg_pop_data)(void ctx, int start, int cut, int flags) =
	117	+ (void *) BPF_FUNC_msg_pop_data;
116	118	static int (bpf_bind)(void ctx, void *addr, int addr_len) =
117	119	(void *) BPF_FUNC_bind;
118	120	static int (bpf_xdp_adjust_tail)(void ctx, int offset) =

tools/testing/selftests/bpf/config+1−0

@@ -22,3 +22,4 @@ CONFIG_NET_CLS_FLOWER=m · 21 unmodified lines
22	22	CONFIG_LWTUNNEL=y
23	23	CONFIG_BPF_STREAM_PARSER=y
24	24	CONFIG_XDP_SOCKETS=y
	25	+CONFIG_FTRACE_SYSCALLS=y

tools/testing/selftests/bpf/test_btf.c+4−31

@@ -29,7 +29,6 @@ · 28 unmodified lines
29	29	static uint32_t pass_cnt;
30	30	static uint32_t error_cnt;
31	31	static uint32_t skip_cnt;
32		−static bool jit_enabled;
33	32
34	33	#define CHECK(condition, format...) ({ \
35	34	int __ret = !!(condition); \
@@ -65,24 +64,6 @@ static int __base_pr(const char *format, ...) · 29 unmodified lines
65	64	return err;
66	65	}
67	66
68		−static bool is_jit_enabled(void)
69		−{
70		− const char *jit_sysctl = "/proc/sys/net/core/bpf_jit_enable";
71		− bool enabled = false;
72		− int sysctl_fd;
73		−
74		− sysctl_fd = open(jit_sysctl, 0, O_RDONLY);
75		− if (sysctl_fd != -1) {
76		− char tmpc;
77		−
78		− if (read(sysctl_fd, &tmpc, sizeof(tmpc)) == 1)
79		− enabled = (tmpc != '0');
80		− close(sysctl_fd);
81		− }
82		−
83		− return enabled;
84		−}
85		−
86	67	#define BTF_INFO_ENC(kind, root, vlen) \
87	68	((!!(root) << 31) \| ((kind) << 24) \| ((vlen) & BTF_MAX_VLEN))
88	69
@@ -2547,8 +2528,8 @@ static int do_test_file(unsigned int test_num) · 2458 unmodified lines
2547	2528	test->btf_kv_notfound))
2548	2529	goto done;
2549	2530
2550		− if (!jit_enabled \|\| !has_btf_ext)
2551		− goto skip_jit;
	2531	+ if (!has_btf_ext)
	2532	+ goto skip;
2552	2533
2553	2534	/* get necessary program info */
2554	2535	info_len = sizeof(struct bpf_prog_info);
@@ -2604,7 +2585,7 @@ static int do_test_file(unsigned int test_num) · 49 unmodified lines
2604	2585	goto done;
2605	2586	}
2606	2587
2607		− err = btf_get_from_id(info.btf_id, &btf);
	2588	+ err = btf__get_from_id(info.btf_id, &btf);
2608	2589	if (CHECK(err, "cannot get btf from kernel, err: %d", err))
2609	2590	goto done;
2610	2591
@@ -2636,7 +2617,7 @@ static int do_test_file(unsigned int test_num) · 25 unmodified lines
2636	2617	finfo = (void *)finfo + rec_size;
2637	2618	}
2638	2619
2639		−skip_jit:
	2620	+skip:
2640	2621	fprintf(stderr, "OK");
2641	2622
2642	2623	done:
@@ -3270,12 +3251,6 @@ static int do_test_func_type(int test_num) · 627 unmodified lines
3270	3251	err = -1;
3271	3252	goto done;
3272	3253	}
3273		− if (!jit_enabled) {
3274		− skip_cnt++;
3275		− fprintf(stderr, "SKIPPED, please enable sysctl bpf_jit_enable\n");
3276		− err = 0;
3277		− goto done;
3278		− }
3279	3254
3280	3255	/* get necessary lens */
3281	3256	info_len = sizeof(struct bpf_prog_info);
@@ -3452,8 +3427,6 @@ int main(int argc, char **argv) · 170 unmodified lines
3452	3427	if (args.always_log)
3453	3428	libbpf_set_print(__base_pr, __base_pr, __base_pr);
3454	3429
3455		− jit_enabled = is_jit_enabled();
3456		−
3457	3430	if (args.raw_test)
3458	3431	err \|= test_raw();
3459	3432

tools/testing/selftests/bpf/test_progs.c+5−5

@@ -524,7 +524,7 @@ static void test_bpf_obj_id(void) · 523 unmodified lines
524	524	load_time < now - 60 \|\| load_time > now + 60 \|\|
525	525	prog_infos[i].created_by_uid != my_uid \|\|
526	526	prog_infos[i].nr_map_ids != 1 \|\|
527		− (int )prog_infos[i].map_ids != map_infos[i].id \|\|
	527	+ (int )(long)prog_infos[i].map_ids != map_infos[i].id \|\|
528	528	strcmp((char *)prog_infos[i].name, expected_prog_name),
529	529	"get-prog-info(fd)",
530	530	"err %d errno %d i %d type %d(%d) info_len %u(%Zu) jit_enabled %d jited_prog_len %u xlated_prog_len %u jited_prog %d xlated_prog %d load_time %lu(%lu) uid %u(%u) nr_map_ids %u(%u) map_id %u(%u) name %s(%s)\n",
@@ -539,7 +539,7 @@ static void test_bpf_obj_id(void) · 8 unmodified lines
539	539	load_time, now,
540	540	prog_infos[i].created_by_uid, my_uid,
541	541	prog_infos[i].nr_map_ids, 1,
542		− (int )prog_infos[i].map_ids, map_infos[i].id,
	542	+ (int )(long)prog_infos[i].map_ids, map_infos[i].id,
543	543	prog_infos[i].name, expected_prog_name))
544	544	goto done;
545	545	}
@@ -585,7 +585,7 @@ static void test_bpf_obj_id(void) · 39 unmodified lines
585	585	bzero(&prog_info, sizeof(prog_info));
586	586	info_len = sizeof(prog_info);
587	587
588		− saved_map_id = (int )(prog_infos[i].map_ids);
	588	+ saved_map_id = (int )((long)prog_infos[i].map_ids);
589	589	prog_info.map_ids = prog_infos[i].map_ids;
590	590	prog_info.nr_map_ids = 2;
591	591	err = bpf_obj_get_info_by_fd(prog_fd, &prog_info, &info_len);
@@ -593,12 +593,12 @@ static void test_bpf_obj_id(void) · 1 unmodified line
593	593	prog_infos[i].xlated_prog_insns = 0;
594	594	CHECK(err \|\| info_len != sizeof(struct bpf_prog_info) \|\|
595	595	memcmp(&prog_info, &prog_infos[i], info_len) \|\|
596		− (int )prog_info.map_ids != saved_map_id,
	596	+ (int )(long)prog_info.map_ids != saved_map_id,
597	597	"get-prog-info(next_id->fd)",
598	598	"err %d errno %d info_len %u(%Zu) memcmp %d map_id %u(%u)\n",
599	599	err, errno, info_len, sizeof(struct bpf_prog_info),
600	600	memcmp(&prog_info, &prog_infos[i], info_len),
601		− (int )prog_info.map_ids, saved_map_id);
	601	+ (int )(long)prog_info.map_ids, saved_map_id);
602	602	close(prog_fd);
603	603	}
604	604	CHECK(nr_id_found != nr_iters,

tools/testing/selftests/bpf/test_sockmap.c+123−4

@@ -79,6 +79,8 @@ int txmsg_start; · 78 unmodified lines
79	79	int txmsg_end;
80	80	int txmsg_start_push;
81	81	int txmsg_end_push;
	82	+int txmsg_start_pop;
	83	+int txmsg_pop;
82	84	int txmsg_ingress;
83	85	int txmsg_skb;
84	86	int ktls;
@@ -104,6 +106,8 @@ static const struct option long_options[] = { · 19 unmodified lines
104	106	{"txmsg_end", required_argument, NULL, 'e'},
105	107	{"txmsg_start_push", required_argument, NULL, 'p'},
106	108	{"txmsg_end_push", required_argument, NULL, 'q'},
	109	+ {"txmsg_start_pop", required_argument, NULL, 'w'},
	110	+ {"txmsg_pop", required_argument, NULL, 'x'},
107	111	{"txmsg_ingress", no_argument, &txmsg_ingress, 1 },
108	112	{"txmsg_skb", no_argument, &txmsg_skb, 1 },
109	113	{"ktls", no_argument, &ktls, 1 },
@@ -473,13 +477,27 @@ static int msg_loop(int fd, int iov_count, int iov_length, int cnt, · 363 unmodified lines
473	477	clock_gettime(CLOCK_MONOTONIC, &s->end);
474	478	} else {
475	479	int slct, recvp = 0, recv, max_fd = fd;
	480	+ float total_bytes, txmsg_pop_total;
476	481	int fd_flags = O_NONBLOCK;
477	482	struct timeval timeout;
478		− float total_bytes;
479	483	fd_set w;
480	484
481	485	fcntl(fd, fd_flags);
	486	+ /* Account for pop bytes noting each iteration of apply will
	487	+ * call msg_pop_data helper so we need to account for this
	488	+ * by calculating the number of apply iterations. Note user
	489	+ * of the tool can create cases where no data is sent by
	490	+ * manipulating pop/push/pull/etc. For example txmsg_apply 1
	491	+ * with txmsg_pop 1 will try to apply 1B at a time but each
	492	+ * iteration will then pop 1B so no data will ever be sent.
	493	+ * This is really only useful for testing edge cases in code
	494	+ * paths.
	495	+ */
482	496	total_bytes = (float)iov_count * (float)iov_length * (float)cnt;
	497	+ txmsg_pop_total = txmsg_pop;
	498	+ if (txmsg_apply)
	499	+ txmsg_pop_total *= (total_bytes / txmsg_apply);
	500	+ total_bytes -= txmsg_pop_total;
483	501	err = clock_gettime(CLOCK_MONOTONIC, &s->start);
484	502	if (err < 0)
485	503	perror("recv start time: ");
@@ -488,7 +506,7 @@ static int msg_loop(int fd, int iov_count, int iov_length, int cnt, · 2 unmodified lines
488	506	timeout.tv_sec = 0;
489	507	timeout.tv_usec = 300000;
490	508	} else {
491		− timeout.tv_sec = 1;
	509	+ timeout.tv_sec = 3;
492	510	timeout.tv_usec = 0;
493	511	}
494	512
@@ -503,7 +521,7 @@ static int msg_loop(int fd, int iov_count, int iov_length, int cnt, · 8 unmodified lines
503	521	goto out_errno;
504	522	} else if (!slct) {
505	523	if (opt->verbose)
506		− fprintf(stderr, "unexpected timeout\n");
	524	+ fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total);
507	525	errno = -EIO;
508	526	clock_gettime(CLOCK_MONOTONIC, &s->end);
509	527	goto out_errno;
@@ -619,7 +637,7 @@ static int sendmsg_test(struct sockmap_options *opt) · 109 unmodified lines
619	637	iov_count = 1;
620	638	err = msg_loop(rx_fd, iov_count, iov_buf,
621	639	cnt, &s, false, opt);
622		− if (err && opt->verbose)
	640	+ if (opt->verbose)
623	641	fprintf(stderr,
624	642	"msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
625	643	iov_count, iov_buf, cnt, err);
@@ -931,6 +949,39 @@ run: · 305 unmodified lines
931	949	}
932	950	}
933	951
	952	+ if (txmsg_start_pop) {
	953	+ i = 4;
	954	+ err = bpf_map_update_elem(map_fd[5],
	955	+ &i, &txmsg_start_pop, BPF_ANY);
	956	+ if (err) {
	957	+ fprintf(stderr,
	958	+ "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n",
	959	+ txmsg_start_pop, i, err, strerror(errno));
	960	+ goto out;
	961	+ }
	962	+ } else {
	963	+ i = 4;
	964	+ bpf_map_update_elem(map_fd[5],
	965	+ &i, &txmsg_start_pop, BPF_ANY);
	966	+ }
	967	+
	968	+ if (txmsg_pop) {
	969	+ i = 5;
	970	+ err = bpf_map_update_elem(map_fd[5],
	971	+ &i, &txmsg_pop, BPF_ANY);
	972	+ if (err) {
	973	+ fprintf(stderr,
	974	+ "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n",
	975	+ txmsg_pop, i, err, strerror(errno));
	976	+ goto out;
	977	+ }
	978	+ } else {
	979	+ i = 5;
	980	+ bpf_map_update_elem(map_fd[5],
	981	+ &i, &txmsg_pop, BPF_ANY);
	982	+
	983	+ }
	984	+
934	985	if (txmsg_ingress) {
935	986	int in = BPF_F_INGRESS;
936	987
@@ -1082,6 +1133,11 @@ static void test_options(char *options) · 145 unmodified lines
1082	1133	snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);
1083	1134	strncat(options, tstr, OPTSTRING);
1084	1135	}
	1136	+ if (txmsg_start_pop) {
	1137	+ snprintf(tstr, OPTSTRING, "pop (%d,%d),",
	1138	+ txmsg_start_pop, txmsg_start_pop + txmsg_pop);
	1139	+ strncat(options, tstr, OPTSTRING);
	1140	+ }
1085	1141	if (txmsg_ingress)
1086	1142	strncat(options, "ingress,", OPTSTRING);
1087	1143	if (txmsg_skb)
@@ -1264,6 +1320,7 @@ static int test_mixed(int cgrp) · 176 unmodified lines
1264	1320	txmsg_apply = txmsg_cork = 0;
1265	1321	txmsg_start = txmsg_end = 0;
1266	1322	txmsg_start_push = txmsg_end_push = 0;
	1323	+ txmsg_start_pop = txmsg_pop = 0;
1267	1324
1268	1325	/* Test small and large iov_count values with pass/redir/apply/cork */
1269	1326	txmsg_pass = 1;
@@ -1383,6 +1440,19 @@ static int test_start_end(int cgrp) · 113 unmodified lines
1383	1440	txmsg_end = 2;
1384	1441	txmsg_start_push = 1;
1385	1442	txmsg_end_push = 2;
	1443	+ txmsg_start_pop = 1;
	1444	+ txmsg_pop = 1;
	1445	+ err = test_txmsg(cgrp);
	1446	+ if (err)
	1447	+ goto out;
	1448	+
	1449	+ /* Cut a byte of pushed data but leave reamining in place */
	1450	+ txmsg_start = 1;
	1451	+ txmsg_end = 2;
	1452	+ txmsg_start_push = 1;
	1453	+ txmsg_end_push = 3;
	1454	+ txmsg_start_pop = 1;
	1455	+ txmsg_pop = 1;
1386	1456	err = test_txmsg(cgrp);
1387	1457	if (err)
1388	1458	goto out;
@@ -1393,6 +1463,9 @@ static int test_start_end(int cgrp) · 4 unmodified lines
1393	1463	opt.iov_length = 100;
1394	1464	txmsg_cork = 1600;
1395	1465
	1466	+ txmsg_start_pop = 0;
	1467	+ txmsg_pop = 0;
	1468	+
1396	1469	for (i = 99; i <= 1600; i += 500) {
1397	1470	txmsg_start = 0;
1398	1471	txmsg_end = i;
@@ -1403,6 +1476,17 @@ static int test_start_end(int cgrp) · 4 unmodified lines
1403	1476	goto out;
1404	1477	}
1405	1478
	1479	+ /* Test pop data in middle of cork */
	1480	+ for (i = 99; i <= 1600; i += 500) {
	1481	+ txmsg_start_pop = 10;
	1482	+ txmsg_pop = i;
	1483	+ err = test_exec(cgrp, &opt);
	1484	+ if (err)
	1485	+ goto out;
	1486	+ }
	1487	+ txmsg_start_pop = 0;
	1488	+ txmsg_pop = 0;
	1489	+
1406	1490	/* Test start/end with cork but pull data in middle */
1407	1491	for (i = 199; i <= 1600; i += 500) {
1408	1492	txmsg_start = 100;
@@ -1423,6 +1507,15 @@ static int test_start_end(int cgrp) · 14 unmodified lines
1423	1507	if (err)
1424	1508	goto out;
1425	1509
	1510	+ /* Test pop with cork pulling last sg entry */
	1511	+ txmsg_start_pop = 1500;
	1512	+ txmsg_pop = 1600;
	1513	+ err = test_exec(cgrp, &opt);
	1514	+ if (err)
	1515	+ goto out;
	1516	+ txmsg_start_pop = 0;
	1517	+ txmsg_pop = 0;
	1518	+
1426	1519	/* Test start/end pull of single byte in last page */
1427	1520	txmsg_start = 1111;
1428	1521	txmsg_end = 1112;
@@ -1432,6 +1525,13 @@ static int test_start_end(int cgrp) · 3 unmodified lines
1432	1525	if (err)
1433	1526	goto out;
1434	1527
	1528	+ /* Test pop of single byte in last page */
	1529	+ txmsg_start_pop = 1111;
	1530	+ txmsg_pop = 1112;
	1531	+ err = test_exec(cgrp, &opt);
	1532	+ if (err)
	1533	+ goto out;
	1534	+
1435	1535	/* Test start/end with end < start */
1436	1536	txmsg_start = 1111;
1437	1537	txmsg_end = 0;
@@ -1456,7 +1556,20 @@ static int test_start_end(int cgrp) · 18 unmodified lines
1456	1556	txmsg_start_push = 1601;
1457	1557	txmsg_end_push = 1600;
1458	1558	err = test_exec(cgrp, &opt);
	1559	+ if (err)
	1560	+ goto out;
	1561	+
	1562	+ /* Test pop with start > data */
	1563	+ txmsg_start_pop = 1601;
	1564	+ txmsg_pop = 1;
	1565	+ err = test_exec(cgrp, &opt);
	1566	+ if (err)
	1567	+ goto out;
1459	1568
	1569	+ /* Test pop with pop range > data */
	1570	+ txmsg_start_pop = 1599;
	1571	+ txmsg_pop = 10;
	1572	+ err = test_exec(cgrp, &opt);
1460	1573	out:
1461	1574	txmsg_start = 0;
1462	1575	txmsg_end = 0;
@@ -1641,6 +1754,12 @@ int main(int argc, char **argv) · 178 unmodified lines
1641	1754	case 'q':
1642	1755	txmsg_end_push = atoi(optarg);
1643	1756	break;
	1757	+ case 'w':
	1758	+ txmsg_start_pop = atoi(optarg);
	1759	+ break;
	1760	+ case 'x':
	1761	+ txmsg_pop = atoi(optarg);
	1762	+ break;
1644	1763	case 'a':
1645	1764	txmsg_apply = atoi(optarg);
1646	1765	break;

tools/testing/selftests/bpf/test_sockmap_kern.h+57−13

@@ -74,7 +74,7 @@ struct bpf_map_def SEC("maps") sock_bytes = { · 73 unmodified lines
74	74	.type = BPF_MAP_TYPE_ARRAY,
75	75	.key_size = sizeof(int),
76	76	.value_size = sizeof(int),
77		− .max_entries = 4
	77	+ .max_entries = 6
78	78	};
79	79
80	80	struct bpf_map_def SEC("maps") sock_redir_flags = {
@@ -181,8 +181,8 @@ int bpf_sockmap(struct bpf_sock_ops *skops) · 100 unmodified lines
181	181	SEC("sk_msg1")
182	182	int bpf_prog4(struct sk_msg_md *msg)
183	183	{
184		− int *bytes, zero = 0, one = 1, two = 2, three = 3;
185		− int start, end, start_push, end_push;
	184	+ int *bytes, zero = 0, one = 1, two = 2, three = 3, four = 4, five = 5;
	185	+ int start, end, start_push, end_push, start_pop, pop;
186	186
187	187	bytes = bpf_map_lookup_elem(&sock_apply_bytes, &zero);
188	188	if (bytes)
@@ -198,15 +198,19 @@ int bpf_prog4(struct sk_msg_md *msg) · 9 unmodified lines
198	198	end_push = bpf_map_lookup_elem(&sock_bytes, &three);
199	199	if (start_push && end_push)
200	200	bpf_msg_push_data(msg, start_push, end_push, 0);
	201	+ start_pop = bpf_map_lookup_elem(&sock_bytes, &four);
	202	+ pop = bpf_map_lookup_elem(&sock_bytes, &five);
	203	+ if (start_pop && pop)
	204	+ bpf_msg_pop_data(msg, start_pop, pop, 0);
201	205	return SK_PASS;
202	206	}
203	207
204	208	SEC("sk_msg2")
205	209	int bpf_prog5(struct sk_msg_md *msg)
206	210	{
207		− int zero = 0, one = 1, two = 2, three = 3;
208		− int start, end, start_push, end_push;
209		− int *bytes, len1, len2 = 0, len3;
	211	+ int zero = 0, one = 1, two = 2, three = 3, four = 4, five = 5;
	212	+ int start, end, start_push, end_push, start_pop, pop;
	213	+ int *bytes, len1, len2 = 0, len3, len4;
210	214	int err1 = -1, err2 = -1;
211	215
212	216	bytes = bpf_map_lookup_elem(&sock_apply_bytes, &zero);
@@ -247,6 +251,20 @@ int bpf_prog5(struct sk_msg_md *msg) · 34 unmodified lines
247	251	bpf_printk("sk_msg2: length push_update %i->%i\n",
248	252	len2 ? len2 : len1, len3);
249	253	}
	254	+ start_pop = bpf_map_lookup_elem(&sock_bytes, &four);
	255	+ pop = bpf_map_lookup_elem(&sock_bytes, &five);
	256	+ if (start_pop && pop) {
	257	+ int err;
	258	+
	259	+ bpf_printk("sk_msg2: pop(%i@%i)\n",
	260	+ start_pop, pop);
	261	+ err = bpf_msg_pop_data(msg, start_pop, pop, 0);
	262	+ if (err)
	263	+ bpf_printk("sk_msg2: pop_data err %i\n", err);
	264	+ len4 = (__u64)msg->data_end - (__u64)msg->data;
	265	+ bpf_printk("sk_msg2: length pop_data %i->%i\n",
	266	+ len1 ? len1 : 0, len4);
	267	+ }
250	268
251	269	bpf_printk("sk_msg2: data length %i err1 %i err2 %i\n",
252	270	len1, err1, err2);
@@ -256,8 +274,8 @@ int bpf_prog5(struct sk_msg_md *msg) · 3 unmodified lines
256	274	SEC("sk_msg3")
257	275	int bpf_prog6(struct sk_msg_md *msg)
258	276	{
259		− int bytes, start, end, start_push, end_push, f;
260		− int zero = 0, one = 1, two = 2, three = 3, key = 0;
	277	+ int zero = 0, one = 1, two = 2, three = 3, four = 4, five = 5, key = 0;
	278	+ int bytes, start, end, start_push, end_push, start_pop, pop, f;
261	279	__u64 flags = 0;
262	280
263	281	bytes = bpf_map_lookup_elem(&sock_apply_bytes, &zero);
@@ -277,6 +295,11 @@ int bpf_prog6(struct sk_msg_md *msg) · 13 unmodified lines
277	295	if (start_push && end_push)
278	296	bpf_msg_push_data(msg, start_push, end_push, 0);
279	297
	298	+ start_pop = bpf_map_lookup_elem(&sock_bytes, &four);
	299	+ pop = bpf_map_lookup_elem(&sock_bytes, &five);
	300	+ if (start_pop && pop)
	301	+ bpf_msg_pop_data(msg, start_pop, pop, 0);
	302	+
280	303	f = bpf_map_lookup_elem(&sock_redir_flags, &zero);
281	304	if (f && *f) {
282	305	key = 2;
@@ -292,8 +315,9 @@ int bpf_prog6(struct sk_msg_md *msg) · 9 unmodified lines
292	315	SEC("sk_msg4")
293	316	int bpf_prog7(struct sk_msg_md *msg)
294	317	{
295		− int zero = 0, one = 1, two = 2, three = 3, len1, len2 = 0, len3;
296		− int bytes, start, end, start_push, end_push, f;
	318	+ int bytes, start, end, start_push, end_push, start_pop, pop, f;
	319	+ int zero = 0, one = 1, two = 2, three = 3, four = 4, five = 5;
	320	+ int len1, len2 = 0, len3, len4;
297	321	int err1 = 0, err2 = 0, key = 0;
298	322	__u64 flags = 0;
299	323
@@ -335,6 +359,22 @@ int bpf_prog7(struct sk_msg_md *msg) · 35 unmodified lines
335	359	len2 ? len2 : len1, len3);
336	360	}
337	361
	362	+ start_pop = bpf_map_lookup_elem(&sock_bytes, &four);
	363	+ pop = bpf_map_lookup_elem(&sock_bytes, &five);
	364	+ if (start_pop && pop) {
	365	+ int err;
	366	+
	367	+ bpf_printk("sk_msg4: pop(%i@%i)\n",
	368	+ start_pop, pop);
	369	+ err = bpf_msg_pop_data(msg, start_pop, pop, 0);
	370	+ if (err)
	371	+ bpf_printk("sk_msg4: pop_data err %i\n", err);
	372	+ len4 = (__u64)msg->data_end - (__u64)msg->data;
	373	+ bpf_printk("sk_msg4: length pop_data %i->%i\n",
	374	+ len1 ? len1 : 0, len4);
	375	+ }
	376	+
	377	+
338	378	f = bpf_map_lookup_elem(&sock_redir_flags, &zero);
339	379	if (f && *f) {
340	380	key = 2;
@@ -389,8 +429,8 @@ int bpf_prog9(struct sk_msg_md *msg) · 48 unmodified lines
389	429	SEC("sk_msg7")
390	430	int bpf_prog10(struct sk_msg_md *msg)
391	431	{
392		− int bytes, start, end, start_push, *end_push;
393		− int zero = 0, one = 1, two = 2, three = 3;
	432	+ int bytes, start, end, start_push, end_push, start_pop, *pop;
	433	+ int zero = 0, one = 1, two = 2, three = 3, four = 4, five = 5;
394	434
395	435	bytes = bpf_map_lookup_elem(&sock_apply_bytes, &zero);
396	436	if (bytes)
@@ -406,7 +446,11 @@ int bpf_prog10(struct sk_msg_md *msg) · 9 unmodified lines
406	446	end_push = bpf_map_lookup_elem(&sock_bytes, &three);
407	447	if (start_push && end_push)
408	448	bpf_msg_push_data(msg, start_push, end_push, 0);
409		−
	449	+ start_pop = bpf_map_lookup_elem(&sock_bytes, &four);
	450	+ pop = bpf_map_lookup_elem(&sock_bytes, &five);
	451	+ if (start_pop && pop)
	452	+ bpf_msg_pop_data(msg, start_pop, pop, 0);
	453	+ bpf_printk("return sk drop\n");
410	454	return SK_DROP;
411	455	}
412	456