lael-labs/linux

include/linux/bpf.h+4−2

 	u32 max_pkt_offset;
 	u32 stack_depth;
 	u32 id;
-	u32 func_cnt;
+	u32 func_cnt; /* used by non-func prog as the number of func progs */
+	u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */
 	bool offload_requested;
 	struct bpf_prog **func;
 	void *jit_data; /* JIT specific data. arch dependent */
 #endif
 	struct bpf_prog_offload *offload;
 	struct btf *btf;
-	u32 type_id; /* type id for this prog/func */
+	struct bpf_func_info *func_info;
+	u32 func_info_cnt;
 	union {
 		struct work_struct work;
 		struct rcu_head	rcu;

include/linux/bpf_verifier.h+0−1

@@ -204,7 +204,6 @@ static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log) · 203 unmodified lines
204	struct bpf_subprog_info {	204	struct bpf_subprog_info {
205	u32 start; /* insn idx of function entry point */	205	u32 start; /* insn idx of function entry point */
206	u16 stack_depth; /* max. stack depth used by this function */	206	u16 stack_depth; /* max. stack depth used by this function */
207	u32 type_id; /* btf type_id for this subprog */
208	};	207	};
209		208
210	/* single container for all structs	209	/* single container for all structs

include/uapi/linux/bpf.h+15−1

@@ -2268,6 +2268,19 @@ union bpf_attr { · 2267 unmodified lines
2268	*	2268	*
2269	* Return	2269	* Return
2270	* 0 on success, or a negative error in case of failure.	2270	* 0 on success, or a negative error in case of failure.
		2271	*
		2272	* int bpf_msg_pop_data(struct sk_msg_buff *msg, u32 start, u32 pop, u64 flags)
		2273	* Description
		2274	* Will remove pop bytes from a msg starting at byte start.
		2275	* This may result in ENOMEM errors under certain situations if
		2276	* an allocation and copy are required due to a full ring buffer.
		2277	* However, the helper will try to avoid doing the allocation
		2278	* if possible. Other errors can occur if input parameters are
		2279	* invalid either due to start byte not being valid part of msg
		2280	* payload and/or pop value being to large.
		2281	*
		2282	* Return
		2283	* 0 on success, or a negative erro in case of failure.
2271	*/	2284	*/
2272	#define __BPF_FUNC_MAPPER(FN) \	2285	#define __BPF_FUNC_MAPPER(FN) \
2273	FN(unspec), \	2286	FN(unspec), \
@@ -2360,7 +2373,8 @@ union bpf_attr { · 86 unmodified lines
2360	FN(map_push_elem), \	2373	FN(map_push_elem), \
2361	FN(map_pop_elem), \	2374	FN(map_pop_elem), \
2362	FN(map_peek_elem), \	2375	FN(map_peek_elem), \
2363	FN(msg_push_data),	2376	FN(msg_push_data), \
		2377	FN(msg_pop_data),
2364		2378
2365	/* integer value in 'imm' field of BPF_CALL instruction selects which helper	2379	/* integer value in 'imm' field of BPF_CALL instruction selects which helper
2366	* function eBPF program intends to call	2380	* function eBPF program intends to call

kernel/bpf/core.c+2−1

@@ -411,7 +411,8 @@ static void bpf_get_prog_name(const struct bpf_prog prog, char sym) · 410 unmodified lines
411		411
412	/* prog->aux->name will be ignored if full btf name is available */	412	/* prog->aux->name will be ignored if full btf name is available */
413	if (prog->aux->btf) {	413	if (prog->aux->btf) {
414	type = btf_type_by_id(prog->aux->btf, prog->aux->type_id);	414	type = btf_type_by_id(prog->aux->btf,
		415	prog->aux->func_info[prog->aux->func_idx].type_id);
415	func_name = btf_name_by_offset(prog->aux->btf, type->name_off);	416	func_name = btf_name_by_offset(prog->aux->btf, type->name_off);
416	snprintf(sym, (size_t)(end - sym), "_%s", func_name);	417	snprintf(sym, (size_t)(end - sym), "_%s", func_name);
417	return;	418	return;

kernel/bpf/syscall.c+8−25

 		bpf_prog_free_id(prog, do_idr_lock);
 		bpf_prog_kallsyms_del_all(prog);
 		btf_put(prog->aux->btf);
+		kvfree(prog->aux->func_info);
 		call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
 	}
 	}
 	if (prog->aux->btf) {
+		u32 krec_size = sizeof(struct bpf_func_info);
 		u32 ucnt, urec_size;
 		info.btf_id = btf_id(prog->aux->btf);
 		ucnt = info.func_info_cnt;
-		info.func_info_cnt = prog->aux->func_cnt ? : 1;
+		info.func_info_cnt = prog->aux->func_info_cnt;
 		urec_size = info.func_info_rec_size;
-		info.func_info_rec_size = sizeof(struct bpf_func_info);
+		info.func_info_rec_size = krec_size;
 		if (ucnt) {
 			/* expect passed-in urec_size is what the kernel expects */
 			if (urec_size != info.func_info_rec_size)
 				return -EINVAL;
 			if (bpf_dump_raw_ok()) {
-				struct bpf_func_info kern_finfo;
 				char __user *user_finfo;
-				u32 i, insn_offset;
 				user_finfo = u64_to_user_ptr(info.func_info);
-				if (prog->aux->func_cnt) {
+				ucnt = min_t(u32, info.func_info_cnt, ucnt);
-					ucnt = min_t(u32, info.func_info_cnt, ucnt);
+				if (copy_to_user(user_finfo, prog->aux->func_info,
-					insn_offset = 0;
+						 krec_size * ucnt))
-					for (i = 0; i < ucnt; i++) {
+					return -EFAULT;
-						kern_finfo.insn_offset = insn_offset;
-						kern_finfo.type_id = prog->aux->func[i]->aux->type_id;
-						if (copy_to_user(user_finfo, &kern_finfo,
-								 sizeof(kern_finfo)))
-							return -EFAULT;
-						/* func[i]->len holds the prog len */
-						insn_offset += prog->aux->func[i]->len;
-						user_finfo += urec_size;
-					}
-				} else {
-					kern_finfo.insn_offset = 0;
-					kern_finfo.type_id = prog->aux->type_id;
-					if (copy_to_user(user_finfo, &kern_finfo,
-							 sizeof(kern_finfo)))
-						return -EFAULT;
-				}
 			} else {
 				info.func_info_cnt = 0;
 			}

kernel/bpf/verifier.c+38−17

 {
 	u32 i, nfuncs, urec_size, min_size, prev_offset;
 	u32 krec_size = sizeof(struct bpf_func_info);
-	struct bpf_func_info krecord = {};
+	struct bpf_func_info *krecord = NULL;
 	const struct btf_type *type;
 	void __user *urecord;
 	struct btf *btf;
 	urecord = u64_to_user_ptr(attr->func_info);
 	min_size = min_t(u32, krec_size, urec_size);
+	krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN);
+	if (!krecord) {
+		ret = -ENOMEM;
+		goto free_btf;
+	}
 	for (i = 0; i < nfuncs; i++) {
 		ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size);
 		if (ret) {
 			goto free_btf;
 		}
-		if (copy_from_user(&krecord, urecord, min_size)) {
+		if (copy_from_user(&krecord[i], urecord, min_size)) {
 			ret = -EFAULT;
 			goto free_btf;
 		}
 		/* check insn_offset */
 		if (i == 0) {
-			if (krecord.insn_offset) {
+			if (krecord[i].insn_offset) {
 				verbose(env,
 					"nonzero insn_offset %u for the first func info record",
-					krecord.insn_offset);
+					krecord[i].insn_offset);
 				ret = -EINVAL;
 				goto free_btf;
 			}
-		} else if (krecord.insn_offset <= prev_offset) {
+		} else if (krecord[i].insn_offset <= prev_offset) {
 			verbose(env,
 				"same or smaller insn offset (%u) than previous func info record (%u)",
-				krecord.insn_offset, prev_offset);
+				krecord[i].insn_offset, prev_offset);
 			ret = -EINVAL;
 			goto free_btf;
 		}
-		if (env->subprog_info[i].start != krecord.insn_offset) {
+		if (env->subprog_info[i].start != krecord[i].insn_offset) {
 			verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");
 			ret = -EINVAL;
 			goto free_btf;
 		}
 		/* check type_id */
-		type = btf_type_by_id(btf, krecord.type_id);
+		type = btf_type_by_id(btf, krecord[i].type_id);
 		if (!type || BTF_INFO_KIND(type->info) != BTF_KIND_FUNC) {
 			verbose(env, "invalid type id %d in func info",
-				krecord.type_id);
+				krecord[i].type_id);
 			ret = -EINVAL;
 			goto free_btf;
 		}
-		if (i == 0)
+		prev_offset = krecord[i].insn_offset;
-			prog->aux->type_id = krecord.type_id;
-		env->subprog_info[i].type_id = krecord.type_id;
-		prev_offset = krecord.insn_offset;
 		urecord += urec_size;
 	}
 	prog->aux->btf = btf;
+	prog->aux->func_info = krecord;
+	prog->aux->func_info_cnt = nfuncs;
 	return 0;
 free_btf:
 	btf_put(btf);
+	kvfree(krecord);
 	return ret;
 }
+static void adjust_btf_func(struct bpf_verifier_env *env)
+{
+	int i;
+	if (!env->prog->aux->func_info)
+		return;
+	for (i = 0; i < env->subprog_cnt; i++)
+		env->prog->aux->func_info[i].insn_offset = env->subprog_info[i].start;
+}
 /* check %cur's range satisfies %old's */
 static bool range_within(struct bpf_reg_state *old,
 			 struct bpf_reg_state *cur)
 		if (bpf_prog_calc_tag(func[i]))
 			goto out_free;
 		func[i]->is_func = 1;
+		func[i]->aux->func_idx = i;
+		/* the btf and func_info will be freed only at prog->aux */
+		func[i]->aux->btf = prog->aux->btf;
+		func[i]->aux->func_info = prog->aux->func_info;
 		/* Use bpf_prog_F_tag to indicate functions in stack traces.
 		 * Long term would need debug info to populate names
 		 */
 		func[i]->aux->name[0] = 'F';
 		func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
 		func[i]->jit_requested = 1;
-		/* the btf will be freed only at prog->aux */
-		func[i]->aux->btf = prog->aux->btf;
-		func[i]->aux->type_id = env->subprog_info[i].type_id;
 		func[i] = bpf_int_jit_compile(func[i]);
 		if (!func[i]->jited) {
 			err = -ENOTSUPP;
 		convert_pseudo_ld_imm64(env);
 	}
+	if (ret == 0)
+		adjust_btf_func(env);
 err_release_maps:
 	if (!env->prog->aux->used_maps)
 		/* if we didn't copy map pointers into bpf_prog_info, release

net/core/filter.c+173−1

@@ -463,7 +463,8 @@ static bool convert_bpf_ld_abs(struct sock_filter fp, struct bpf_insn *insnp) · 462 unmodified lines
463	bool ldx_off_ok = offset <= S16_MAX;	463	bool ldx_off_ok = offset <= S16_MAX;
464		464
465	*insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_H);	465	*insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_H);
466	*insn++ = BPF_ALU64_IMM(BPF_SUB, BPF_REG_TMP, offset);	466	if (offset)
		467	*insn++ = BPF_ALU64_IMM(BPF_SUB, BPF_REG_TMP, offset);
467	*insn++ = BPF_JMP_IMM(BPF_JSLT, BPF_REG_TMP,	468	*insn++ = BPF_JMP_IMM(BPF_JSLT, BPF_REG_TMP,
468	size, 2 + endian + (!ldx_off_ok * 2));	469	size, 2 + endian + (!ldx_off_ok * 2));
469	if (ldx_off_ok) {	470	if (ldx_off_ok) {
@@ -2424,6 +2425,174 @@ static const struct bpf_func_proto bpf_msg_push_data_proto = { · 1954 unmodified lines
2424	.arg4_type = ARG_ANYTHING,	2425	.arg4_type = ARG_ANYTHING,
2425	};	2426	};
2426		2427
		2428	static void sk_msg_shift_left(struct sk_msg *msg, int i)
		2429	{
		2430	int prev;
		2431
		2432	do {
		2433	prev = i;
		2434	sk_msg_iter_var_next(i);
		2435	msg->sg.data[prev] = msg->sg.data[i];
		2436	} while (i != msg->sg.end);
		2437
		2438	sk_msg_iter_prev(msg, end);
		2439	}
		2440
		2441	static void sk_msg_shift_right(struct sk_msg *msg, int i)
		2442	{
		2443	struct scatterlist tmp, sge;
		2444
		2445	sk_msg_iter_next(msg, end);
		2446	sge = sk_msg_elem_cpy(msg, i);
		2447	sk_msg_iter_var_next(i);
		2448	tmp = sk_msg_elem_cpy(msg, i);
		2449
		2450	while (i != msg->sg.end) {
		2451	msg->sg.data[i] = sge;
		2452	sk_msg_iter_var_next(i);
		2453	sge = tmp;
		2454	tmp = sk_msg_elem_cpy(msg, i);
		2455	}
		2456	}
		2457
		2458	BPF_CALL_4(bpf_msg_pop_data, struct sk_msg *, msg, u32, start,
		2459	u32, len, u64, flags)
		2460	{
		2461	u32 i = 0, l, space, offset = 0;
		2462	u64 last = start + len;
		2463	int pop;
		2464
		2465	if (unlikely(flags))
		2466	return -EINVAL;
		2467
		2468	/* First find the starting scatterlist element */
		2469	i = msg->sg.start;
		2470	do {
		2471	l = sk_msg_elem(msg, i)->length;
		2472
		2473	if (start < offset + l)
		2474	break;
		2475	offset += l;
		2476	sk_msg_iter_var_next(i);
		2477	} while (i != msg->sg.end);
		2478
		2479	/* Bounds checks: start and pop must be inside message */
		2480	if (start >= offset + l \|\| last >= msg->sg.size)
		2481	return -EINVAL;
		2482
		2483	space = MAX_MSG_FRAGS - sk_msg_elem_used(msg);
		2484
		2485	pop = len;
		2486	/* --------------\| offset
		2487	* -\| start \|-------- len -------\|
		2488	*
		2489	* \|----- a ----\|-------- pop -------\|----- b ----\|
		2490	* \|______________________________________________\| length
		2491	*
		2492	*
		2493	* a: region at front of scatter element to save
		2494	* b: region at back of scatter element to save when length > A + pop
		2495	* pop: region to pop from element, same as input 'pop' here will be
		2496	* decremented below per iteration.
		2497	*
		2498	* Two top-level cases to handle when start != offset, first B is non
		2499	* zero and second B is zero corresponding to when a pop includes more
		2500	* than one element.
		2501	*
		2502	* Then if B is non-zero AND there is no space allocate space and
		2503	* compact A, B regions into page. If there is space shift ring to
		2504	* the rigth free'ing the next element in ring to place B, leaving
		2505	* A untouched except to reduce length.
		2506	*/
		2507	if (start != offset) {
		2508	struct scatterlist nsge, sge = sk_msg_elem(msg, i);
		2509	int a = start;
		2510	int b = sge->length - pop - a;
		2511
		2512	sk_msg_iter_var_next(i);
		2513
		2514	if (pop < sge->length - a) {
		2515	if (space) {
		2516	sge->length = a;
		2517	sk_msg_shift_right(msg, i);
		2518	nsge = sk_msg_elem(msg, i);
		2519	get_page(sg_page(sge));
		2520	sg_set_page(nsge,
		2521	sg_page(sge),
		2522	b, sge->offset + pop + a);
		2523	} else {
		2524	struct page page, orig;
		2525	u8 to, from;
		2526
		2527	page = alloc_pages(__GFP_NOWARN \|
		2528	__GFP_COMP \| GFP_ATOMIC,
		2529	get_order(a + b));
		2530	if (unlikely(!page))
		2531	return -ENOMEM;
		2532
		2533	sge->length = a;
		2534	orig = sg_page(sge);
		2535	from = sg_virt(sge);
		2536	to = page_address(page);
		2537	memcpy(to, from, a);
		2538	memcpy(to + a, from + a + pop, b);
		2539	sg_set_page(sge, page, a + b, 0);
		2540	put_page(orig);
		2541	}
		2542	pop = 0;
		2543	} else if (pop >= sge->length - a) {
		2544	sge->length = a;
		2545	pop -= (sge->length - a);
		2546	}
		2547	}
		2548
		2549	/* From above the current layout _must_ be as follows,
		2550	*
		2551	* -\| offset
		2552	* -\| start
		2553	*
		2554	* \|---- pop ---\|---------------- b ------------\|
		2555	* \|____________________________________________\| length
		2556	*
		2557	* Offset and start of the current msg elem are equal because in the
		2558	* previous case we handled offset != start and either consumed the
		2559	* entire element and advanced to the next element OR pop == 0.
		2560	*
		2561	* Two cases to handle here are first pop is less than the length
		2562	* leaving some remainder b above. Simply adjust the element's layout
		2563	* in this case. Or pop >= length of the element so that b = 0. In this
		2564	* case advance to next element decrementing pop.
		2565	*/
		2566	while (pop) {
		2567	struct scatterlist *sge = sk_msg_elem(msg, i);
		2568
		2569	if (pop < sge->length) {
		2570	sge->length -= pop;
		2571	sge->offset += pop;
		2572	pop = 0;
		2573	} else {
		2574	pop -= sge->length;
		2575	sk_msg_shift_left(msg, i);
		2576	}
		2577	sk_msg_iter_var_next(i);
		2578	}
		2579
		2580	sk_mem_uncharge(msg->sk, len - pop);
		2581	msg->sg.size -= (len - pop);
		2582	sk_msg_compute_data_pointers(msg);
		2583	return 0;
		2584	}
		2585
		2586	static const struct bpf_func_proto bpf_msg_pop_data_proto = {
		2587	.func = bpf_msg_pop_data,
		2588	.gpl_only = false,
		2589	.ret_type = RET_INTEGER,
		2590	.arg1_type = ARG_PTR_TO_CTX,
		2591	.arg2_type = ARG_ANYTHING,
		2592	.arg3_type = ARG_ANYTHING,
		2593	.arg4_type = ARG_ANYTHING,
		2594	};
		2595
2427	BPF_CALL_1(bpf_get_cgroup_classid, const struct sk_buff *, skb)	2596	BPF_CALL_1(bpf_get_cgroup_classid, const struct sk_buff *, skb)
2428	{	2597	{
2429	return task_get_classid(skb);	2598	return task_get_classid(skb);
@@ -5097,6 +5266,7 @@ bool bpf_helper_changes_pkt_data(void *func) · 2667 unmodified lines
5097	func == bpf_xdp_adjust_meta \|\|	5266	func == bpf_xdp_adjust_meta \|\|
5098	func == bpf_msg_pull_data \|\|	5267	func == bpf_msg_pull_data \|\|
5099	func == bpf_msg_push_data \|\|	5268	func == bpf_msg_push_data \|\|
		5269	func == bpf_msg_pop_data \|\|
5100	func == bpf_xdp_adjust_tail \|\|	5270	func == bpf_xdp_adjust_tail \|\|
5101	#if IS_ENABLED(CONFIG_IPV6_SEG6_BPF)	5271	#if IS_ENABLED(CONFIG_IPV6_SEG6_BPF)
5102	func == bpf_lwt_seg6_store_bytes \|\|	5272	func == bpf_lwt_seg6_store_bytes \|\|
@@ -5393,6 +5563,8 @@ sk_msg_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 290 unmodified lines
5393	return &bpf_msg_pull_data_proto;	5563	return &bpf_msg_pull_data_proto;
5394	case BPF_FUNC_msg_push_data:	5564	case BPF_FUNC_msg_push_data:
5395	return &bpf_msg_push_data_proto;	5565	return &bpf_msg_push_data_proto;
		5566	case BPF_FUNC_msg_pop_data:
		5567	return &bpf_msg_pop_data_proto;
5396	default:	5568	default:
5397	return bpf_base_func_proto(func_id);	5569	return bpf_base_func_proto(func_id);
5398	}	5570	}

net/ipv4/tcp_bpf.c+14−3

 {
 	bool cork = false, enospc = msg->sg.start == msg->sg.end;
 	struct sock *sk_redir;
-	u32 tosend;
+	u32 tosend, delta = 0;
 	int ret;
 more_data:
-	if (psock->eval == __SK_NONE)
+	if (psock->eval == __SK_NONE) {
+		/* Track delta in msg size to add/subtract it on SK_DROP from
+		 * returned to user copied size. This ensures user doesn't
+		 * get a positive return code with msg_cut_data and SK_DROP
+		 * verdict.
+		 */
+		delta = msg->sg.size;
 		psock->eval = sk_psock_msg_verdict(sk, psock, msg);
+		if (msg->sg.size < delta)
+			delta -= msg->sg.size;
+		else
+			delta = 0;
+	}
 	if (msg->cork_bytes &&
 	    msg->cork_bytes > msg->sg.size && !enospc) {
 	default:
 		sk_msg_free_partial(sk, msg, tosend);
 		sk_msg_apply_bytes(psock, tosend);
-		*copied -= tosend;
+		*copied -= (tosend + delta);
 		return -EACCES;
 	}

net/tls/tls_sw.c+9−2

 	struct sock *sk_redir;
 	struct tls_rec *rec;
 	int err = 0, send;
+	u32 delta = 0;
 	bool enospc;
 	psock = sk_psock_get(sk);
 		return tls_push_record(sk, flags, record_type);
 more_data:
 	enospc = sk_msg_full(msg);
-	if (psock->eval == __SK_NONE)
+	if (psock->eval == __SK_NONE) {
+		delta = msg->sg.size;
 		psock->eval = sk_psock_msg_verdict(sk, psock, msg);
+		if (delta < msg->sg.size)
+			delta -= msg->sg.size;
+		else
+			delta = 0;
+	}
 	if (msg->cork_bytes && msg->cork_bytes > msg->sg.size &&
 	    !enospc && !full_record) {
 		err = -ENOSPC;
 			msg->apply_bytes -= send;
 		if (msg->sg.size == 0)
 			tls_free_open_rec(sk);
-		*copied -= send;
+		*copied -= (send + delta);
 		err = -EACCES;
 	}

tools/bpf/bpftool/map.c+2−2

@@ -713,7 +713,7 @@ static int do_dump(int argc, char **argv) · 712 unmodified lines
713		713
714	prev_key = NULL;	714	prev_key = NULL;
715		715
716	err = btf_get_from_id(info.btf_id, &btf);	716	err = btf__get_from_id(info.btf_id, &btf);
717	if (err) {	717	if (err) {
718	p_err("failed to get btf");	718	p_err("failed to get btf");
719	goto exit_free;	719	goto exit_free;
@@ -857,7 +857,7 @@ static int do_lookup(int argc, char **argv) · 137 unmodified lines
857	}	857	}
858		858
859	/* here means bpf_map_lookup_elem() succeeded */	859	/* here means bpf_map_lookup_elem() succeeded */
860	err = btf_get_from_id(info.btf_id, &btf);	860	err = btf__get_from_id(info.btf_id, &btf);
861	if (err) {	861	if (err) {
862	p_err("failed to get btf");	862	p_err("failed to get btf");
863	goto exit_free;	863	goto exit_free;

tools/bpf/bpftool/prog.c+1−1

@@ -621,7 +621,7 @@ static int do_dump(int argc, char **argv) · 620 unmodified lines
621	goto err_free;	621	goto err_free;
622	}	622	}
623		623
624	if (info.btf_id && btf_get_from_id(info.btf_id, &btf)) {	624	if (info.btf_id && btf__get_from_id(info.btf_id, &btf)) {
625	p_err("failed to get btf");	625	p_err("failed to get btf");
626	goto err_free;	626	goto err_free;
627	}	627	}

tools/include/uapi/linux/bpf.h+15−1

@@ -2268,6 +2268,19 @@ union bpf_attr { · 2267 unmodified lines
2268	*	2268	*
2269	* Return	2269	* Return
2270	* 0 on success, or a negative error in case of failure.	2270	* 0 on success, or a negative error in case of failure.
		2271	*
		2272	* int bpf_msg_pop_data(struct sk_msg_buff *msg, u32 start, u32 pop, u64 flags)
		2273	* Description
		2274	* Will remove pop bytes from a msg starting at byte start.
		2275	* This may result in ENOMEM errors under certain situations if
		2276	* an allocation and copy are required due to a full ring buffer.
		2277	* However, the helper will try to avoid doing the allocation
		2278	* if possible. Other errors can occur if input parameters are
		2279	* invalid either due to start byte not being valid part of msg
		2280	* payload and/or pop value being to large.
		2281	*
		2282	* Return
		2283	* 0 on success, or a negative erro in case of failure.
2271	*/	2284	*/
2272	#define __BPF_FUNC_MAPPER(FN) \	2285	#define __BPF_FUNC_MAPPER(FN) \
2273	FN(unspec), \	2286	FN(unspec), \
@@ -2360,7 +2373,8 @@ union bpf_attr { · 86 unmodified lines
2360	FN(map_push_elem), \	2373	FN(map_push_elem), \
2361	FN(map_pop_elem), \	2374	FN(map_pop_elem), \
2362	FN(map_peek_elem), \	2375	FN(map_peek_elem), \
2363	FN(msg_push_data),	2376	FN(msg_push_data), \
		2377	FN(msg_pop_data),
2364		2378
2365	/* integer value in 'imm' field of BPF_CALL instruction selects which helper	2379	/* integer value in 'imm' field of BPF_CALL instruction selects which helper
2366	* function eBPF program intends to call	2380	* function eBPF program intends to call

tools/lib/bpf/Makefile+21−2

 BPF_IN    := $(OUTPUT)libbpf-in.o
 LIB_FILE := $(addprefix $(OUTPUT),$(LIB_FILE))
+VERSION_SCRIPT := libbpf.map
+GLOBAL_SYM_COUNT = $(shell readelf -s $(BPF_IN) | \
+			   awk '/GLOBAL/ && /DEFAULT/ && !/UND/ {s++} END{print s}')
+VERSIONED_SYM_COUNT = $(shell readelf -s $(OUTPUT)libbpf.so | \
+			      grep -Eo '[^ ]+@LIBBPF_' | cut -d@ -f1 | sort -u | wc -l)
 CMD_TARGETS = $(LIB_FILE)
 all: fixdep all_cmd
-all_cmd: $(CMD_TARGETS)
+all_cmd: $(CMD_TARGETS) check
 $(BPF_IN): force elfdep bpfdep
 	@(test -f ../../include/uapi/linux/bpf.h -a -f ../../../include/uapi/linux/bpf.h && ( \
 	$(Q)$(MAKE) $(build)=libbpf
 $(OUTPUT)libbpf.so: $(BPF_IN)
-	$(QUIET_LINK)$(CC) --shared $^ -o $@
+	$(QUIET_LINK)$(CC) --shared -Wl,--version-script=$(VERSION_SCRIPT) \
+		$^ -o $@
 $(OUTPUT)libbpf.a: $(BPF_IN)
 	$(QUIET_LINK)$(RM) $@; $(AR) rcs $@ $^
 $(OUTPUT)test_libbpf: test_libbpf.cpp $(OUTPUT)libbpf.a
 	$(QUIET_LINK)$(CXX) $^ -lelf -o $@
+check: check_abi
+check_abi: $(OUTPUT)libbpf.so
+	@if [ "$(GLOBAL_SYM_COUNT)" != "$(VERSIONED_SYM_COUNT)" ]; then	 \
+		echo "Warning: Num of global symbols in $(BPF_IN)"	 \
+		     "($(GLOBAL_SYM_COUNT)) does NOT match with num of"	 \
+		     "versioned symbols in $^ ($(VERSIONED_SYM_COUNT))." \
+		     "Please make sure all LIBBPF_API symbols are"	 \
+		     "versioned in $(VERSION_SCRIPT)." >&2;		 \
+		exit 1;							 \
+	fi
 define do_install
 	if [ ! -d '$(DESTDIR_SQ)$2' ]; then		\
 		$(INSTALL) -d -m 755 '$(DESTDIR_SQ)$2';	\

tools/lib/bpf/README.rst+139−0

@@ -0,0 +1,139 @@
	1	.. SPDX-License-Identifier: GPL-2.0
	2
	3	libbpf API naming convention
	4	============================
	5
	6	libbpf API provides access to a few logically separated groups of
	7	functions and types. Every group has its own naming convention
	8	described here. It's recommended to follow these conventions whenever a
	9	new function or type is added to keep libbpf API clean and consistent.
	10
	11	All types and functions provided by libbpf API should have one of the
	12	following prefixes: ``bpf_``, ``btf_``, ``libbpf_``.
	13
	14	System call wrappers
	15	--------------------
	16
	17	System call wrappers are simple wrappers for commands supported by
	18	sys_bpf system call. These wrappers should go to ``bpf.h`` header file
	19	and map one-on-one to corresponding commands.
	20
	21	For example ``bpf_map_lookup_elem`` wraps ``BPF_MAP_LOOKUP_ELEM``
	22	command of sys_bpf, ``bpf_prog_attach`` wraps ``BPF_PROG_ATTACH``, etc.
	23
	24	Objects
	25	-------
	26
	27	Another class of types and functions provided by libbpf API is "objects"
	28	and functions to work with them. Objects are high-level abstractions
	29	such as BPF program or BPF map. They're represented by corresponding
	30	structures such as ``struct bpf_object``, ``struct bpf_program``,
	31	``struct bpf_map``, etc.
	32
	33	Structures are forward declared and access to their fields should be
	34	provided via corresponding getters and setters rather than directly.
	35
	36	These objects are associated with corresponding parts of ELF object that
	37	contains compiled BPF programs.
	38
	39	For example ``struct bpf_object`` represents ELF object itself created
	40	from an ELF file or from a buffer, ``struct bpf_program`` represents a
	41	program in ELF object and ``struct bpf_map`` is a map.
	42
	43	Functions that work with an object have names built from object name,
	44	double underscore and part that describes function purpose.
	45
	46	For example ``bpf_object__open`` consists of the name of corresponding
	47	object, ``bpf_object``, double underscore and ``open`` that defines the
	48	purpose of the function to open ELF file and create ``bpf_object`` from
	49	it.
	50
	51	Another example: ``bpf_program__load`` is named for corresponding
	52	object, ``bpf_program``, that is separated from other part of the name
	53	by double underscore.
	54
	55	All objects and corresponding functions other than BTF related should go
	56	to ``libbpf.h``. BTF types and functions should go to ``btf.h``.
	57
	58	Auxiliary functions
	59	-------------------
	60
	61	Auxiliary functions and types that don't fit well in any of categories
	62	described above should have ``libbpf_`` prefix, e.g.
	63	``libbpf_get_error`` or ``libbpf_prog_type_by_name``.
	64
	65	libbpf ABI
	66	==========
	67
	68	libbpf can be both linked statically or used as DSO. To avoid possible
	69	conflicts with other libraries an application is linked with, all
	70	non-static libbpf symbols should have one of the prefixes mentioned in
	71	API documentation above. See API naming convention to choose the right
	72	name for a new symbol.
	73
	74	Symbol visibility
	75	-----------------
	76
	77	libbpf follow the model when all global symbols have visibility "hidden"
	78	by default and to make a symbol visible it has to be explicitly
	79	attributed with ``LIBBPF_API`` macro. For example:
	80
	81	.. code-block:: c
	82
	83	LIBBPF_API int bpf_prog_get_fd_by_id(__u32 id);
	84
	85	This prevents from accidentally exporting a symbol, that is not supposed
	86	to be a part of ABI what, in turn, improves both libbpf developer- and
	87	user-experiences.
	88
	89	ABI versionning
	90	---------------
	91
	92	To make future ABI extensions possible libbpf ABI is versioned.
	93	Versioning is implemented by ``libbpf.map`` version script that is
	94	passed to linker.
	95
	96	Version name is ``LIBBPF_`` prefix + three-component numeric version,
	97	starting from ``0.0.1``.
	98
	99	Every time ABI is being changed, e.g. because a new symbol is added or
	100	semantic of existing symbol is changed, ABI version should be bumped.
	101
	102	For example, if current state of ``libbpf.map`` is:
	103
	104	.. code-block::
	105	LIBBPF_0.0.1 {
	106	global:
	107	bpf_func_a;
	108	bpf_func_b;
	109	local:
	110	\*;
	111	};
	112
	113	, and a new symbol ``bpf_func_c`` is being introduced, then
	114	``libbpf.map`` should be changed like this:
	115
	116	.. code-block::
	117	LIBBPF_0.0.1 {
	118	global:
	119	bpf_func_a;
	120	bpf_func_b;
	121	local:
	122	\*;
	123	};
	124	LIBBPF_0.0.2 {
	125	global:
	126	bpf_func_c;
	127	} LIBBPF_0.0.1;
	128
	129	, where new version ``LIBBPF_0.0.2`` depends on the previous
	130	``LIBBPF_0.0.1``.
	131
	132	Format of version script and ways to handle ABI changes, including
	133	incompatible ones, described in details in [1].
	134
	135	Links
	136	=====
	137
	138	[1] https://www.akkadia.org/drepper/dsohowto.pdf
	139	(Chapter 3. Maintaining APIs and ABIs).

tools/lib/bpf/btf.c+2−2

@@ -415,7 +415,7 @@ const char btf__name_by_offset(const struct btf btf, __u32 offset) · 414 unmodified lines
415	return NULL;	415	return NULL;
416	}	416	}
417		417
418	int btf_get_from_id(__u32 id, struct btf **btf)	418	int btf__get_from_id(__u32 id, struct btf **btf)
419	{	419	{
420	struct bpf_btf_info btf_info = { 0 };	420	struct bpf_btf_info btf_info = { 0 };
421	__u32 len = sizeof(btf_info);	421	__u32 len = sizeof(btf_info);
@@ -466,7 +466,7 @@ int btf_get_from_id(__u32 id, struct btf **btf) · 44 unmodified lines
466	goto exit_free;	466	goto exit_free;
467	}	467	}
468		468
469	btf = btf__new((__u8 )btf_info.btf, btf_info.btf_size, NULL);	469	btf = btf__new((__u8 )(long)btf_info.btf, btf_info.btf_size, NULL);
470	if (IS_ERR(*btf)) {	470	if (IS_ERR(*btf)) {
471	err = PTR_ERR(*btf);	471	err = PTR_ERR(*btf);
472	*btf = NULL;	472	*btf = NULL;

tools/lib/bpf/btf.h+1−1

@@ -73,7 +73,7 @@ LIBBPF_API __s64 btf__resolve_size(const struct btf *btf, __u32 type_id); · 72 unmodified lines
73	LIBBPF_API int btf__resolve_type(const struct btf *btf, __u32 type_id);	73	LIBBPF_API int btf__resolve_type(const struct btf *btf, __u32 type_id);
74	LIBBPF_API int btf__fd(const struct btf *btf);	74	LIBBPF_API int btf__fd(const struct btf *btf);
75	LIBBPF_API const char btf__name_by_offset(const struct btf btf, __u32 offset);	75	LIBBPF_API const char btf__name_by_offset(const struct btf btf, __u32 offset);
76	LIBBPF_API int btf_get_from_id(__u32 id, struct btf **btf);	76	LIBBPF_API int btf__get_from_id(__u32 id, struct btf **btf);
77		77
78	struct btf_ext btf_ext__new(__u8 data, __u32 size, btf_print_fn_t err_log);	78	struct btf_ext btf_ext__new(__u8 data, __u32 size, btf_print_fn_t err_log);
79	void btf_ext__free(struct btf_ext *btf_ext);	79	void btf_ext__free(struct btf_ext *btf_ext);

tools/lib/bpf/libbpf.c+2−0

  * Copyright (C) 2017 Nicira, Inc.
  */
+#ifndef _GNU_SOURCE
 #define _GNU_SOURCE
+#endif
 #include <stdlib.h>
 #include <stdio.h>
 #include <stdarg.h>

tools/lib/bpf/libbpf.map+121−0

@@ -0,0 +1,121 @@
	1	LIBBPF_0.0.1 {
	2	global:
	3	bpf_btf_get_fd_by_id;
	4	bpf_create_map;
	5	bpf_create_map_in_map;
	6	bpf_create_map_in_map_node;
	7	bpf_create_map_name;
	8	bpf_create_map_node;
	9	bpf_create_map_xattr;
	10	bpf_load_btf;
	11	bpf_load_program;
	12	bpf_load_program_xattr;
	13	bpf_map__btf_key_type_id;
	14	bpf_map__btf_value_type_id;
	15	bpf_map__def;
	16	bpf_map__fd;
	17	bpf_map__is_offload_neutral;
	18	bpf_map__name;
	19	bpf_map__next;
	20	bpf_map__pin;
	21	bpf_map__prev;
	22	bpf_map__priv;
	23	bpf_map__reuse_fd;
	24	bpf_map__set_ifindex;
	25	bpf_map__set_inner_map_fd;
	26	bpf_map__set_priv;
	27	bpf_map__unpin;
	28	bpf_map_delete_elem;
	29	bpf_map_get_fd_by_id;
	30	bpf_map_get_next_id;
	31	bpf_map_get_next_key;
	32	bpf_map_lookup_and_delete_elem;
	33	bpf_map_lookup_elem;
	34	bpf_map_update_elem;
	35	bpf_obj_get;
	36	bpf_obj_get_info_by_fd;
	37	bpf_obj_pin;
	38	bpf_object__btf_fd;
	39	bpf_object__close;
	40	bpf_object__find_map_by_name;
	41	bpf_object__find_map_by_offset;
	42	bpf_object__find_program_by_title;
	43	bpf_object__kversion;
	44	bpf_object__load;
	45	bpf_object__name;
	46	bpf_object__next;
	47	bpf_object__open;
	48	bpf_object__open_buffer;
	49	bpf_object__open_xattr;
	50	bpf_object__pin;
	51	bpf_object__pin_maps;
	52	bpf_object__pin_programs;
	53	bpf_object__priv;
	54	bpf_object__set_priv;
	55	bpf_object__unload;
	56	bpf_object__unpin_maps;
	57	bpf_object__unpin_programs;
	58	bpf_perf_event_read_simple;
	59	bpf_prog_attach;
	60	bpf_prog_detach;
	61	bpf_prog_detach2;
	62	bpf_prog_get_fd_by_id;
	63	bpf_prog_get_next_id;
	64	bpf_prog_load;
	65	bpf_prog_load_xattr;
	66	bpf_prog_query;
	67	bpf_prog_test_run;
	68	bpf_program__fd;
	69	bpf_program__is_kprobe;
	70	bpf_program__is_perf_event;
	71	bpf_program__is_raw_tracepoint;
	72	bpf_program__is_sched_act;
	73	bpf_program__is_sched_cls;
	74	bpf_program__is_socket_filter;
	75	bpf_program__is_tracepoint;
	76	bpf_program__is_xdp;
	77	bpf_program__load;
	78	bpf_program__next;
	79	bpf_program__nth_fd;
	80	bpf_program__pin;
	81	bpf_program__pin_instance;
	82	bpf_program__prev;
	83	bpf_program__priv;
	84	bpf_program__set_expected_attach_type;
	85	bpf_program__set_ifindex;
	86	bpf_program__set_kprobe;
	87	bpf_program__set_perf_event;
	88	bpf_program__set_prep;
	89	bpf_program__set_priv;
	90	bpf_program__set_raw_tracepoint;
	91	bpf_program__set_sched_act;
	92	bpf_program__set_sched_cls;
	93	bpf_program__set_socket_filter;
	94	bpf_program__set_tracepoint;
	95	bpf_program__set_type;
	96	bpf_program__set_xdp;
	97	bpf_program__title;
	98	bpf_program__unload;
	99	bpf_program__unpin;
	100	bpf_program__unpin_instance;
	101	bpf_raw_tracepoint_open;
	102	bpf_set_link_xdp_fd;
	103	bpf_task_fd_query;
	104	bpf_verify_program;
	105	btf__fd;
	106	btf__find_by_name;
	107	btf__free;
	108	btf__get_from_id;
	109	btf__name_by_offset;
	110	btf__new;
	111	btf__resolve_size;
	112	btf__resolve_type;
	113	btf__type_by_id;
	114	libbpf_attach_type_by_name;
	115	libbpf_get_error;
	116	libbpf_prog_type_by_name;
	117	libbpf_set_print;
	118	libbpf_strerror;
	119	local:
	120	*;
	121	};

tools/lib/bpf/libbpf_errno.c+1−0

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	* Copyright (C) 2017 Nicira, Inc.	7	* Copyright (C) 2017 Nicira, Inc.
8	*/	8	*/
9		9
		10	#undef _GNU_SOURCE
10	#include <stdio.h>	11	#include <stdio.h>
11	#include <string.h>	12	#include <string.h>
12		13

tools/testing/selftests/bpf/bpf_helpers.h+2−0

 	(void *) BPF_FUNC_msg_pull_data;
 static int (*bpf_msg_push_data)(void *ctx, int start, int end, int flags) =
 	(void *) BPF_FUNC_msg_push_data;
+static int (*bpf_msg_pop_data)(void *ctx, int start, int cut, int flags) =
+	(void *) BPF_FUNC_msg_pop_data;
 static int (*bpf_bind)(void *ctx, void *addr, int addr_len) =
 	(void *) BPF_FUNC_bind;
 static int (*bpf_xdp_adjust_tail)(void *ctx, int offset) =

tools/testing/selftests/bpf/config+1−0

@@ -22,3 +22,4 @@ CONFIG_NET_CLS_FLOWER=m · 21 unmodified lines
22	CONFIG_LWTUNNEL=y	22	CONFIG_LWTUNNEL=y
23	CONFIG_BPF_STREAM_PARSER=y	23	CONFIG_BPF_STREAM_PARSER=y
24	CONFIG_XDP_SOCKETS=y	24	CONFIG_XDP_SOCKETS=y
		25	CONFIG_FTRACE_SYSCALLS=y

tools/testing/selftests/bpf/test_btf.c+4−31

 static uint32_t pass_cnt;
 static uint32_t error_cnt;
 static uint32_t skip_cnt;
-static bool jit_enabled;
 #define CHECK(condition, format...) ({					\
 	int __ret = !!(condition);					\
 	return err;
 }
-static bool is_jit_enabled(void)
-{
-	const char *jit_sysctl = "/proc/sys/net/core/bpf_jit_enable";
-	bool enabled = false;
-	int sysctl_fd;
-	sysctl_fd = open(jit_sysctl, 0, O_RDONLY);
-	if (sysctl_fd != -1) {
-		char tmpc;
-		if (read(sysctl_fd, &tmpc, sizeof(tmpc)) == 1)
-			enabled = (tmpc != '0');
-		close(sysctl_fd);
-	}
-	return enabled;
-}
 #define BTF_INFO_ENC(kind, root, vlen)			\
 	((!!(root) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN))
 		  test->btf_kv_notfound))
 		goto done;
-	if (!jit_enabled || !has_btf_ext)
+	if (!has_btf_ext)
-		goto skip_jit;
+		goto skip;
 	/* get necessary program info */
 	info_len = sizeof(struct bpf_prog_info);
 		goto done;
 	}
-	err = btf_get_from_id(info.btf_id, &btf);
+	err = btf__get_from_id(info.btf_id, &btf);
 	if (CHECK(err, "cannot get btf from kernel, err: %d", err))
 		goto done;
 		finfo = (void *)finfo + rec_size;
 	}
-skip_jit:
+skip:
 	fprintf(stderr, "OK");
 done:
 		err = -1;
 		goto done;
 	}
-	if (!jit_enabled) {
-		skip_cnt++;
-		fprintf(stderr, "SKIPPED, please enable sysctl bpf_jit_enable\n");
-		err = 0;
-		goto done;
-	}
 	/* get necessary lens */
 	info_len = sizeof(struct bpf_prog_info);
 	if (args.always_log)
 		libbpf_set_print(__base_pr, __base_pr, __base_pr);
-	jit_enabled = is_jit_enabled();
 	if (args.raw_test)
 		err |= test_raw();

tools/testing/selftests/bpf/test_progs.c+5−5

@@ -524,7 +524,7 @@ static void test_bpf_obj_id(void) · 523 unmodified lines
524	load_time < now - 60 \|\| load_time > now + 60 \|\|	524	load_time < now - 60 \|\| load_time > now + 60 \|\|
525	prog_infos[i].created_by_uid != my_uid \|\|	525	prog_infos[i].created_by_uid != my_uid \|\|
526	prog_infos[i].nr_map_ids != 1 \|\|	526	prog_infos[i].nr_map_ids != 1 \|\|
527	(int )prog_infos[i].map_ids != map_infos[i].id \|\|	527	(int )(long)prog_infos[i].map_ids != map_infos[i].id \|\|
528	strcmp((char *)prog_infos[i].name, expected_prog_name),	528	strcmp((char *)prog_infos[i].name, expected_prog_name),
529	"get-prog-info(fd)",	529	"get-prog-info(fd)",
530	"err %d errno %d i %d type %d(%d) info_len %u(%Zu) jit_enabled %d jited_prog_len %u xlated_prog_len %u jited_prog %d xlated_prog %d load_time %lu(%lu) uid %u(%u) nr_map_ids %u(%u) map_id %u(%u) name %s(%s)\n",	530	"err %d errno %d i %d type %d(%d) info_len %u(%Zu) jit_enabled %d jited_prog_len %u xlated_prog_len %u jited_prog %d xlated_prog %d load_time %lu(%lu) uid %u(%u) nr_map_ids %u(%u) map_id %u(%u) name %s(%s)\n",
@@ -539,7 +539,7 @@ static void test_bpf_obj_id(void) · 8 unmodified lines
539	load_time, now,	539	load_time, now,
540	prog_infos[i].created_by_uid, my_uid,	540	prog_infos[i].created_by_uid, my_uid,
541	prog_infos[i].nr_map_ids, 1,	541	prog_infos[i].nr_map_ids, 1,
542	(int )prog_infos[i].map_ids, map_infos[i].id,	542	(int )(long)prog_infos[i].map_ids, map_infos[i].id,
543	prog_infos[i].name, expected_prog_name))	543	prog_infos[i].name, expected_prog_name))
544	goto done;	544	goto done;
545	}	545	}
@@ -585,7 +585,7 @@ static void test_bpf_obj_id(void) · 39 unmodified lines
585	bzero(&prog_info, sizeof(prog_info));	585	bzero(&prog_info, sizeof(prog_info));
586	info_len = sizeof(prog_info);	586	info_len = sizeof(prog_info);
587		587
588	saved_map_id = (int )(prog_infos[i].map_ids);	588	saved_map_id = (int )((long)prog_infos[i].map_ids);
589	prog_info.map_ids = prog_infos[i].map_ids;	589	prog_info.map_ids = prog_infos[i].map_ids;
590	prog_info.nr_map_ids = 2;	590	prog_info.nr_map_ids = 2;
591	err = bpf_obj_get_info_by_fd(prog_fd, &prog_info, &info_len);	591	err = bpf_obj_get_info_by_fd(prog_fd, &prog_info, &info_len);
@@ -593,12 +593,12 @@ static void test_bpf_obj_id(void) · 1 unmodified line
593	prog_infos[i].xlated_prog_insns = 0;	593	prog_infos[i].xlated_prog_insns = 0;
594	CHECK(err \|\| info_len != sizeof(struct bpf_prog_info) \|\|	594	CHECK(err \|\| info_len != sizeof(struct bpf_prog_info) \|\|
595	memcmp(&prog_info, &prog_infos[i], info_len) \|\|	595	memcmp(&prog_info, &prog_infos[i], info_len) \|\|
596	(int )prog_info.map_ids != saved_map_id,	596	(int )(long)prog_info.map_ids != saved_map_id,
597	"get-prog-info(next_id->fd)",	597	"get-prog-info(next_id->fd)",
598	"err %d errno %d info_len %u(%Zu) memcmp %d map_id %u(%u)\n",	598	"err %d errno %d info_len %u(%Zu) memcmp %d map_id %u(%u)\n",
599	err, errno, info_len, sizeof(struct bpf_prog_info),	599	err, errno, info_len, sizeof(struct bpf_prog_info),
600	memcmp(&prog_info, &prog_infos[i], info_len),	600	memcmp(&prog_info, &prog_infos[i], info_len),
601	(int )prog_info.map_ids, saved_map_id);	601	(int )(long)prog_info.map_ids, saved_map_id);
602	close(prog_fd);	602	close(prog_fd);
603	}	603	}
604	CHECK(nr_id_found != nr_iters,	604	CHECK(nr_id_found != nr_iters,

tools/testing/selftests/bpf/test_sockmap.c+123−4

@@ -79,6 +79,8 @@ int txmsg_start; · 78 unmodified lines
79	int txmsg_end;	79	int txmsg_end;
80	int txmsg_start_push;	80	int txmsg_start_push;
81	int txmsg_end_push;	81	int txmsg_end_push;
		82	int txmsg_start_pop;
		83	int txmsg_pop;
82	int txmsg_ingress;	84	int txmsg_ingress;
83	int txmsg_skb;	85	int txmsg_skb;
84	int ktls;	86	int ktls;
@@ -104,6 +106,8 @@ static const struct option long_options[] = { · 19 unmodified lines
104	{"txmsg_end", required_argument, NULL, 'e'},	106	{"txmsg_end", required_argument, NULL, 'e'},
105	{"txmsg_start_push", required_argument, NULL, 'p'},	107	{"txmsg_start_push", required_argument, NULL, 'p'},
106	{"txmsg_end_push", required_argument, NULL, 'q'},	108	{"txmsg_end_push", required_argument, NULL, 'q'},
		109	{"txmsg_start_pop", required_argument, NULL, 'w'},
		110	{"txmsg_pop", required_argument, NULL, 'x'},
107	{"txmsg_ingress", no_argument, &txmsg_ingress, 1 },	111	{"txmsg_ingress", no_argument, &txmsg_ingress, 1 },
108	{"txmsg_skb", no_argument, &txmsg_skb, 1 },	112	{"txmsg_skb", no_argument, &txmsg_skb, 1 },
109	{"ktls", no_argument, &ktls, 1 },	113	{"ktls", no_argument, &ktls, 1 },
@@ -473,13 +477,27 @@ static int msg_loop(int fd, int iov_count, int iov_length, int cnt, · 363 unmodified lines
473	clock_gettime(CLOCK_MONOTONIC, &s->end);	477	clock_gettime(CLOCK_MONOTONIC, &s->end);
474	} else {	478	} else {
475	int slct, recvp = 0, recv, max_fd = fd;	479	int slct, recvp = 0, recv, max_fd = fd;
		480	float total_bytes, txmsg_pop_total;
476	int fd_flags = O_NONBLOCK;	481	int fd_flags = O_NONBLOCK;
477	struct timeval timeout;	482	struct timeval timeout;
478	float total_bytes;
479	fd_set w;	483	fd_set w;
480		484
481	fcntl(fd, fd_flags);	485	fcntl(fd, fd_flags);
		486	/* Account for pop bytes noting each iteration of apply will
		487	* call msg_pop_data helper so we need to account for this
		488	* by calculating the number of apply iterations. Note user
		489	* of the tool can create cases where no data is sent by
		490	* manipulating pop/push/pull/etc. For example txmsg_apply 1
		491	* with txmsg_pop 1 will try to apply 1B at a time but each
		492	* iteration will then pop 1B so no data will ever be sent.
		493	* This is really only useful for testing edge cases in code
		494	* paths.
		495	*/
482	total_bytes = (float)iov_count * (float)iov_length * (float)cnt;	496	total_bytes = (float)iov_count * (float)iov_length * (float)cnt;
		497	txmsg_pop_total = txmsg_pop;
		498	if (txmsg_apply)
		499	txmsg_pop_total *= (total_bytes / txmsg_apply);
		500	total_bytes -= txmsg_pop_total;
483	err = clock_gettime(CLOCK_MONOTONIC, &s->start);	501	err = clock_gettime(CLOCK_MONOTONIC, &s->start);
484	if (err < 0)	502	if (err < 0)
485	perror("recv start time: ");	503	perror("recv start time: ");
@@ -488,7 +506,7 @@ static int msg_loop(int fd, int iov_count, int iov_length, int cnt, · 2 unmodified lines
488	timeout.tv_sec = 0;	506	timeout.tv_sec = 0;
489	timeout.tv_usec = 300000;	507	timeout.tv_usec = 300000;
490	} else {	508	} else {
491	timeout.tv_sec = 1;	509	timeout.tv_sec = 3;
492	timeout.tv_usec = 0;	510	timeout.tv_usec = 0;
493	}	511	}
494		512
@@ -503,7 +521,7 @@ static int msg_loop(int fd, int iov_count, int iov_length, int cnt, · 8 unmodified lines
503	goto out_errno;	521	goto out_errno;
504	} else if (!slct) {	522	} else if (!slct) {
505	if (opt->verbose)	523	if (opt->verbose)
506	fprintf(stderr, "unexpected timeout\n");	524	fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total);
507	errno = -EIO;	525	errno = -EIO;
508	clock_gettime(CLOCK_MONOTONIC, &s->end);	526	clock_gettime(CLOCK_MONOTONIC, &s->end);
509	goto out_errno;	527	goto out_errno;
@@ -619,7 +637,7 @@ static int sendmsg_test(struct sockmap_options *opt) · 109 unmodified lines
619	iov_count = 1;	637	iov_count = 1;
620	err = msg_loop(rx_fd, iov_count, iov_buf,	638	err = msg_loop(rx_fd, iov_count, iov_buf,
621	cnt, &s, false, opt);	639	cnt, &s, false, opt);
622	if (err && opt->verbose)	640	if (opt->verbose)
623	fprintf(stderr,	641	fprintf(stderr,
624	"msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",	642	"msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
625	iov_count, iov_buf, cnt, err);	643	iov_count, iov_buf, cnt, err);
@@ -931,6 +949,39 @@ run: · 305 unmodified lines
931	}	949	}
932	}	950	}
933		951
		952	if (txmsg_start_pop) {
		953	i = 4;
		954	err = bpf_map_update_elem(map_fd[5],
		955	&i, &txmsg_start_pop, BPF_ANY);
		956	if (err) {
		957	fprintf(stderr,
		958	"ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n",
		959	txmsg_start_pop, i, err, strerror(errno));
		960	goto out;
		961	}
		962	} else {
		963	i = 4;
		964	bpf_map_update_elem(map_fd[5],
		965	&i, &txmsg_start_pop, BPF_ANY);
		966	}
		967
		968	if (txmsg_pop) {
		969	i = 5;
		970	err = bpf_map_update_elem(map_fd[5],
		971	&i, &txmsg_pop, BPF_ANY);
		972	if (err) {
		973	fprintf(stderr,
		974	"ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n",
		975	txmsg_pop, i, err, strerror(errno));
		976	goto out;
		977	}
		978	} else {
		979	i = 5;
		980	bpf_map_update_elem(map_fd[5],
		981	&i, &txmsg_pop, BPF_ANY);
		982
		983	}
		984
934	if (txmsg_ingress) {	985	if (txmsg_ingress) {
935	int in = BPF_F_INGRESS;	986	int in = BPF_F_INGRESS;
936		987
@@ -1082,6 +1133,11 @@ static void test_options(char *options) · 145 unmodified lines
1082	snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);	1133	snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);
1083	strncat(options, tstr, OPTSTRING);	1134	strncat(options, tstr, OPTSTRING);
1084	}	1135	}
		1136	if (txmsg_start_pop) {
		1137	snprintf(tstr, OPTSTRING, "pop (%d,%d),",
		1138	txmsg_start_pop, txmsg_start_pop + txmsg_pop);
		1139	strncat(options, tstr, OPTSTRING);
		1140	}
1085	if (txmsg_ingress)	1141	if (txmsg_ingress)
1086	strncat(options, "ingress,", OPTSTRING);	1142	strncat(options, "ingress,", OPTSTRING);
1087	if (txmsg_skb)	1143	if (txmsg_skb)
@@ -1264,6 +1320,7 @@ static int test_mixed(int cgrp) · 176 unmodified lines
1264	txmsg_apply = txmsg_cork = 0;	1320	txmsg_apply = txmsg_cork = 0;
1265	txmsg_start = txmsg_end = 0;	1321	txmsg_start = txmsg_end = 0;
1266	txmsg_start_push = txmsg_end_push = 0;	1322	txmsg_start_push = txmsg_end_push = 0;
		1323	txmsg_start_pop = txmsg_pop = 0;
1267		1324
1268	/* Test small and large iov_count values with pass/redir/apply/cork */	1325	/* Test small and large iov_count values with pass/redir/apply/cork */
1269	txmsg_pass = 1;	1326	txmsg_pass = 1;
@@ -1383,6 +1440,19 @@ static int test_start_end(int cgrp) · 113 unmodified lines
1383	txmsg_end = 2;	1440	txmsg_end = 2;
1384	txmsg_start_push = 1;	1441	txmsg_start_push = 1;
1385	txmsg_end_push = 2;	1442	txmsg_end_push = 2;
		1443	txmsg_start_pop = 1;
		1444	txmsg_pop = 1;
		1445	err = test_txmsg(cgrp);
		1446	if (err)
		1447	goto out;
		1448
		1449	/* Cut a byte of pushed data but leave reamining in place */
		1450	txmsg_start = 1;
		1451	txmsg_end = 2;
		1452	txmsg_start_push = 1;
		1453	txmsg_end_push = 3;
		1454	txmsg_start_pop = 1;
		1455	txmsg_pop = 1;
1386	err = test_txmsg(cgrp);	1456	err = test_txmsg(cgrp);
1387	if (err)	1457	if (err)
1388	goto out;	1458	goto out;
@@ -1393,6 +1463,9 @@ static int test_start_end(int cgrp) · 4 unmodified lines
1393	opt.iov_length = 100;	1463	opt.iov_length = 100;
1394	txmsg_cork = 1600;	1464	txmsg_cork = 1600;
1395		1465
		1466	txmsg_start_pop = 0;
		1467	txmsg_pop = 0;
		1468
1396	for (i = 99; i <= 1600; i += 500) {	1469	for (i = 99; i <= 1600; i += 500) {
1397	txmsg_start = 0;	1470	txmsg_start = 0;
1398	txmsg_end = i;	1471	txmsg_end = i;
@@ -1403,6 +1476,17 @@ static int test_start_end(int cgrp) · 4 unmodified lines
1403	goto out;	1476	goto out;
1404	}	1477	}
1405		1478
		1479	/* Test pop data in middle of cork */
		1480	for (i = 99; i <= 1600; i += 500) {
		1481	txmsg_start_pop = 10;
		1482	txmsg_pop = i;
		1483	err = test_exec(cgrp, &opt);
		1484	if (err)
		1485	goto out;
		1486	}
		1487	txmsg_start_pop = 0;
		1488	txmsg_pop = 0;
		1489
1406	/* Test start/end with cork but pull data in middle */	1490	/* Test start/end with cork but pull data in middle */
1407	for (i = 199; i <= 1600; i += 500) {	1491	for (i = 199; i <= 1600; i += 500) {
1408	txmsg_start = 100;	1492	txmsg_start = 100;
@@ -1423,6 +1507,15 @@ static int test_start_end(int cgrp) · 14 unmodified lines
1423	if (err)	1507	if (err)
1424	goto out;	1508	goto out;
1425		1509
		1510	/* Test pop with cork pulling last sg entry */
		1511	txmsg_start_pop = 1500;
		1512	txmsg_pop = 1600;
		1513	err = test_exec(cgrp, &opt);
		1514	if (err)
		1515	goto out;
		1516	txmsg_start_pop = 0;
		1517	txmsg_pop = 0;
		1518
1426	/* Test start/end pull of single byte in last page */	1519	/* Test start/end pull of single byte in last page */
1427	txmsg_start = 1111;	1520	txmsg_start = 1111;
1428	txmsg_end = 1112;	1521	txmsg_end = 1112;
@@ -1432,6 +1525,13 @@ static int test_start_end(int cgrp) · 3 unmodified lines
1432	if (err)	1525	if (err)
1433	goto out;	1526	goto out;
1434		1527
		1528	/* Test pop of single byte in last page */
		1529	txmsg_start_pop = 1111;
		1530	txmsg_pop = 1112;
		1531	err = test_exec(cgrp, &opt);
		1532	if (err)
		1533	goto out;
		1534
1435	/* Test start/end with end < start */	1535	/* Test start/end with end < start */
1436	txmsg_start = 1111;	1536	txmsg_start = 1111;
1437	txmsg_end = 0;	1537	txmsg_end = 0;
@@ -1456,7 +1556,20 @@ static int test_start_end(int cgrp) · 18 unmodified lines
1456	txmsg_start_push = 1601;	1556	txmsg_start_push = 1601;
1457	txmsg_end_push = 1600;	1557	txmsg_end_push = 1600;
1458	err = test_exec(cgrp, &opt);	1558	err = test_exec(cgrp, &opt);
		1559	if (err)
		1560	goto out;
		1561
		1562	/* Test pop with start > data */
		1563	txmsg_start_pop = 1601;
		1564	txmsg_pop = 1;
		1565	err = test_exec(cgrp, &opt);
		1566	if (err)
		1567	goto out;
1459		1568
		1569	/* Test pop with pop range > data */
		1570	txmsg_start_pop = 1599;
		1571	txmsg_pop = 10;
		1572	err = test_exec(cgrp, &opt);
1460	out:	1573	out:
1461	txmsg_start = 0;	1574	txmsg_start = 0;
1462	txmsg_end = 0;	1575	txmsg_end = 0;
@@ -1641,6 +1754,12 @@ int main(int argc, char **argv) · 178 unmodified lines
1641	case 'q':	1754	case 'q':
1642	txmsg_end_push = atoi(optarg);	1755	txmsg_end_push = atoi(optarg);
1643	break;	1756	break;
		1757	case 'w':
		1758	txmsg_start_pop = atoi(optarg);
		1759	break;
		1760	case 'x':
		1761	txmsg_pop = atoi(optarg);
		1762	break;
1644	case 'a':	1763	case 'a':
1645	txmsg_apply = atoi(optarg);	1764	txmsg_apply = atoi(optarg);
1646	break;	1765	break;

tools/testing/selftests/bpf/test_sockmap_kern.h+57−13

@@ -74,7 +74,7 @@ struct bpf_map_def SEC("maps") sock_bytes = { · 73 unmodified lines
74	.type = BPF_MAP_TYPE_ARRAY,	74	.type = BPF_MAP_TYPE_ARRAY,
75	.key_size = sizeof(int),	75	.key_size = sizeof(int),
76	.value_size = sizeof(int),	76	.value_size = sizeof(int),
77	.max_entries = 4	77	.max_entries = 6
78	};	78	};
79		79
80	struct bpf_map_def SEC("maps") sock_redir_flags = {	80	struct bpf_map_def SEC("maps") sock_redir_flags = {
@@ -181,8 +181,8 @@ int bpf_sockmap(struct bpf_sock_ops *skops) · 100 unmodified lines
181	SEC("sk_msg1")	181	SEC("sk_msg1")
182	int bpf_prog4(struct sk_msg_md *msg)	182	int bpf_prog4(struct sk_msg_md *msg)
183	{	183	{
184	int *bytes, zero = 0, one = 1, two = 2, three = 3;	184	int *bytes, zero = 0, one = 1, two = 2, three = 3, four = 4, five = 5;
185	int start, end, start_push, end_push;	185	int start, end, start_push, end_push, start_pop, pop;
186		186
187	bytes = bpf_map_lookup_elem(&sock_apply_bytes, &zero);	187	bytes = bpf_map_lookup_elem(&sock_apply_bytes, &zero);
188	if (bytes)	188	if (bytes)
@@ -198,15 +198,19 @@ int bpf_prog4(struct sk_msg_md *msg) · 9 unmodified lines
198	end_push = bpf_map_lookup_elem(&sock_bytes, &three);	198	end_push = bpf_map_lookup_elem(&sock_bytes, &three);
199	if (start_push && end_push)	199	if (start_push && end_push)
200	bpf_msg_push_data(msg, start_push, end_push, 0);	200	bpf_msg_push_data(msg, start_push, end_push, 0);
		201	start_pop = bpf_map_lookup_elem(&sock_bytes, &four);
		202	pop = bpf_map_lookup_elem(&sock_bytes, &five);
		203	if (start_pop && pop)
		204	bpf_msg_pop_data(msg, start_pop, pop, 0);
201	return SK_PASS;	205	return SK_PASS;
202	}	206	}
203		207
204	SEC("sk_msg2")	208	SEC("sk_msg2")
205	int bpf_prog5(struct sk_msg_md *msg)	209	int bpf_prog5(struct sk_msg_md *msg)
206	{	210	{
207	int zero = 0, one = 1, two = 2, three = 3;	211	int zero = 0, one = 1, two = 2, three = 3, four = 4, five = 5;
208	int start, end, start_push, end_push;	212	int start, end, start_push, end_push, start_pop, pop;
209	int *bytes, len1, len2 = 0, len3;	213	int *bytes, len1, len2 = 0, len3, len4;
210	int err1 = -1, err2 = -1;	214	int err1 = -1, err2 = -1;
211		215
212	bytes = bpf_map_lookup_elem(&sock_apply_bytes, &zero);	216	bytes = bpf_map_lookup_elem(&sock_apply_bytes, &zero);
@@ -247,6 +251,20 @@ int bpf_prog5(struct sk_msg_md *msg) · 34 unmodified lines
247	bpf_printk("sk_msg2: length push_update %i->%i\n",	251	bpf_printk("sk_msg2: length push_update %i->%i\n",
248	len2 ? len2 : len1, len3);	252	len2 ? len2 : len1, len3);
249	}	253	}
		254	start_pop = bpf_map_lookup_elem(&sock_bytes, &four);
		255	pop = bpf_map_lookup_elem(&sock_bytes, &five);
		256	if (start_pop && pop) {
		257	int err;
		258
		259	bpf_printk("sk_msg2: pop(%i@%i)\n",
		260	start_pop, pop);
		261	err = bpf_msg_pop_data(msg, start_pop, pop, 0);
		262	if (err)
		263	bpf_printk("sk_msg2: pop_data err %i\n", err);
		264	len4 = (__u64)msg->data_end - (__u64)msg->data;
		265	bpf_printk("sk_msg2: length pop_data %i->%i\n",
		266	len1 ? len1 : 0, len4);
		267	}
250		268
251	bpf_printk("sk_msg2: data length %i err1 %i err2 %i\n",	269	bpf_printk("sk_msg2: data length %i err1 %i err2 %i\n",
252	len1, err1, err2);	270	len1, err1, err2);
@@ -256,8 +274,8 @@ int bpf_prog5(struct sk_msg_md *msg) · 3 unmodified lines
256	SEC("sk_msg3")	274	SEC("sk_msg3")
257	int bpf_prog6(struct sk_msg_md *msg)	275	int bpf_prog6(struct sk_msg_md *msg)
258	{	276	{
259	int bytes, start, end, start_push, end_push, f;	277	int zero = 0, one = 1, two = 2, three = 3, four = 4, five = 5, key = 0;
260	int zero = 0, one = 1, two = 2, three = 3, key = 0;	278	int bytes, start, end, start_push, end_push, start_pop, pop, f;
261	__u64 flags = 0;	279	__u64 flags = 0;
262		280
263	bytes = bpf_map_lookup_elem(&sock_apply_bytes, &zero);	281	bytes = bpf_map_lookup_elem(&sock_apply_bytes, &zero);
@@ -277,6 +295,11 @@ int bpf_prog6(struct sk_msg_md *msg) · 13 unmodified lines
277	if (start_push && end_push)	295	if (start_push && end_push)
278	bpf_msg_push_data(msg, start_push, end_push, 0);	296	bpf_msg_push_data(msg, start_push, end_push, 0);
279		297
		298	start_pop = bpf_map_lookup_elem(&sock_bytes, &four);
		299	pop = bpf_map_lookup_elem(&sock_bytes, &five);
		300	if (start_pop && pop)
		301	bpf_msg_pop_data(msg, start_pop, pop, 0);
		302
280	f = bpf_map_lookup_elem(&sock_redir_flags, &zero);	303	f = bpf_map_lookup_elem(&sock_redir_flags, &zero);
281	if (f && *f) {	304	if (f && *f) {
282	key = 2;	305	key = 2;
@@ -292,8 +315,9 @@ int bpf_prog6(struct sk_msg_md *msg) · 9 unmodified lines
292	SEC("sk_msg4")	315	SEC("sk_msg4")
293	int bpf_prog7(struct sk_msg_md *msg)	316	int bpf_prog7(struct sk_msg_md *msg)
294	{	317	{
295	int zero = 0, one = 1, two = 2, three = 3, len1, len2 = 0, len3;	318	int bytes, start, end, start_push, end_push, start_pop, pop, f;
296	int bytes, start, end, start_push, end_push, f;	319	int zero = 0, one = 1, two = 2, three = 3, four = 4, five = 5;
		320	int len1, len2 = 0, len3, len4;
297	int err1 = 0, err2 = 0, key = 0;	321	int err1 = 0, err2 = 0, key = 0;
298	__u64 flags = 0;	322	__u64 flags = 0;
299		323
@@ -335,6 +359,22 @@ int bpf_prog7(struct sk_msg_md *msg) · 35 unmodified lines
335	len2 ? len2 : len1, len3);	359	len2 ? len2 : len1, len3);
336	}	360	}
337		361
		362	start_pop = bpf_map_lookup_elem(&sock_bytes, &four);
		363	pop = bpf_map_lookup_elem(&sock_bytes, &five);
		364	if (start_pop && pop) {
		365	int err;
		366
		367	bpf_printk("sk_msg4: pop(%i@%i)\n",
		368	start_pop, pop);
		369	err = bpf_msg_pop_data(msg, start_pop, pop, 0);
		370	if (err)
		371	bpf_printk("sk_msg4: pop_data err %i\n", err);
		372	len4 = (__u64)msg->data_end - (__u64)msg->data;
		373	bpf_printk("sk_msg4: length pop_data %i->%i\n",
		374	len1 ? len1 : 0, len4);
		375	}
		376
		377
338	f = bpf_map_lookup_elem(&sock_redir_flags, &zero);	378	f = bpf_map_lookup_elem(&sock_redir_flags, &zero);
339	if (f && *f) {	379	if (f && *f) {
340	key = 2;	380	key = 2;
@@ -389,8 +429,8 @@ int bpf_prog9(struct sk_msg_md *msg) · 48 unmodified lines
389	SEC("sk_msg7")	429	SEC("sk_msg7")
390	int bpf_prog10(struct sk_msg_md *msg)	430	int bpf_prog10(struct sk_msg_md *msg)
391	{	431	{
392	int bytes, start, end, start_push, *end_push;	432	int bytes, start, end, start_push, end_push, start_pop, *pop;
393	int zero = 0, one = 1, two = 2, three = 3;	433	int zero = 0, one = 1, two = 2, three = 3, four = 4, five = 5;
394		434
395	bytes = bpf_map_lookup_elem(&sock_apply_bytes, &zero);	435	bytes = bpf_map_lookup_elem(&sock_apply_bytes, &zero);
396	if (bytes)	436	if (bytes)
@@ -406,7 +446,11 @@ int bpf_prog10(struct sk_msg_md *msg) · 9 unmodified lines
406	end_push = bpf_map_lookup_elem(&sock_bytes, &three);	446	end_push = bpf_map_lookup_elem(&sock_bytes, &three);
407	if (start_push && end_push)	447	if (start_push && end_push)
408	bpf_msg_push_data(msg, start_push, end_push, 0);	448	bpf_msg_push_data(msg, start_push, end_push, 0);
409		449	start_pop = bpf_map_lookup_elem(&sock_bytes, &four);
		450	pop = bpf_map_lookup_elem(&sock_bytes, &five);
		451	if (start_pop && pop)
		452	bpf_msg_pop_data(msg, start_pop, pop, 0);
		453	bpf_printk("return sk drop\n");
410	return SK_DROP;	454	return SK_DROP;
411	}	455	}
412		456