Merge branch 'introduce-bpf_dynptr_copy-kfunc'

Mykyta Yatsenko says: ==================== introduce bpf_dynptr_copy kfunc From: Mykyta Yatsenko <yatsenko@meta.com> Introduce a new kfunc, bpf_dynptr_copy, which enables copying of data from one dynptr to another. This functionality may be useful in scenarios such as capturing XDP data to a ring buffer. The patch set is split into 3 patches: 1. Refactor bpf_dynptr_read and bpf_dynptr_write by extracting code into static functions, that allows calling them with no compiler warnings 2. Introduce bpf_dynptr_copy 3. Add tests for bpf_dynptr_copy v2->v3: * Implemented bpf_memcmp in dynptr_success.c test, as __builtin_memcmp was not inlined on GCC-BPF. ====================

Andrii Nakryiko committed 1 year ago

Commit ee2fdd02 parents b0f2ee6 064e9aa3 files · +211 −9Browse files

Unified · Split

3 files

kernel/bpf/helpers.c +72 −4
tools/testing/selftests/bpf/prog_tests/dynptr.c +21 −0
tools/testing/selftests/bpf/progs/dynptr_success.c +118 −5

kernel/bpf/helpers.c+72−4

@@ -1759,8 +1759,8 @@ static const struct bpf_func_proto bpf_dynptr_from_mem_proto = { · 1758 unmodified lines
1759	1759	.arg4_type = ARG_PTR_TO_DYNPTR \| DYNPTR_TYPE_LOCAL \| MEM_UNINIT \| MEM_WRITE,
1760	1760	};
1761	1761
1762		−BPF_CALL_5(bpf_dynptr_read, void , dst, u32, len, const struct bpf_dynptr_kern , src,
1763		− u32, offset, u64, flags)
	1762	+static int __bpf_dynptr_read(void dst, u32 len, const struct bpf_dynptr_kern src,
	1763	+ u32 offset, u64 flags)
1764	1764	{
1765	1765	enum bpf_dynptr_type type;
1766	1766	int err;
@@ -1793,6 +1793,12 @@ BPF_CALL_5(bpf_dynptr_read, void *, dst, u32, len, const struct bpf_dynptr_kern · 26 unmodified lines
1793	1793	}
1794	1794	}
1795	1795
	1796	+BPF_CALL_5(bpf_dynptr_read, void , dst, u32, len, const struct bpf_dynptr_kern , src,
	1797	+ u32, offset, u64, flags)
	1798	+{
	1799	+ return __bpf_dynptr_read(dst, len, src, offset, flags);
	1800	+}
	1801	+
1796	1802	static const struct bpf_func_proto bpf_dynptr_read_proto = {
1797	1803	.func = bpf_dynptr_read,
1798	1804	.gpl_only = false,
@@ -1804,8 +1810,8 @@ static const struct bpf_func_proto bpf_dynptr_read_proto = { · 5 unmodified lines
1804	1810	.arg5_type = ARG_ANYTHING,
1805	1811	};
1806	1812
1807		−BPF_CALL_5(bpf_dynptr_write, const struct bpf_dynptr_kern , dst, u32, offset, void , src,
1808		− u32, len, u64, flags)
	1813	+static int __bpf_dynptr_write(const struct bpf_dynptr_kern dst, u32 offset, void src,
	1814	+ u32 len, u64 flags)
1809	1815	{
1810	1816	enum bpf_dynptr_type type;
1811	1817	int err;
@@ -1843,6 +1849,12 @@ BPF_CALL_5(bpf_dynptr_write, const struct bpf_dynptr_kern *, dst, u32, offset, v · 31 unmodified lines
1843	1849	}
1844	1850	}
1845	1851
	1852	+BPF_CALL_5(bpf_dynptr_write, const struct bpf_dynptr_kern , dst, u32, offset, void , src,
	1853	+ u32, len, u64, flags)
	1854	+{
	1855	+ return __bpf_dynptr_write(dst, offset, src, len, flags);
	1856	+}
	1857	+
1846	1858	static const struct bpf_func_proto bpf_dynptr_write_proto = {
1847	1859	.func = bpf_dynptr_write,
1848	1860	.gpl_only = false,
@@ -2758,6 +2770,61 @@ __bpf_kfunc int bpf_dynptr_clone(const struct bpf_dynptr *p, · 909 unmodified lines
2758	2770	return 0;
2759	2771	}
2760	2772
	2773	+/**
	2774	+ * bpf_dynptr_copy() - Copy data from one dynptr to another.
	2775	+ * @dst_ptr: Destination dynptr - where data should be copied to
	2776	+ * @dst_off: Offset into the destination dynptr
	2777	+ * @src_ptr: Source dynptr - where data should be copied from
	2778	+ * @src_off: Offset into the source dynptr
	2779	+ * @size: Length of the data to copy from source to destination
	2780	+ *
	2781	+ * Copies data from source dynptr to destination dynptr.
	2782	+ * Returns 0 on success; negative error, otherwise.
	2783	+ */
	2784	+__bpf_kfunc int bpf_dynptr_copy(struct bpf_dynptr *dst_ptr, u32 dst_off,
	2785	+ struct bpf_dynptr *src_ptr, u32 src_off, u32 size)
	2786	+{
	2787	+ struct bpf_dynptr_kern dst = (struct bpf_dynptr_kern )dst_ptr;
	2788	+ struct bpf_dynptr_kern src = (struct bpf_dynptr_kern )src_ptr;
	2789	+ void src_slice, dst_slice;
	2790	+ char buf[256];
	2791	+ u32 off;
	2792	+
	2793	+ src_slice = bpf_dynptr_slice(src_ptr, src_off, NULL, size);
	2794	+ dst_slice = bpf_dynptr_slice_rdwr(dst_ptr, dst_off, NULL, size);
	2795	+
	2796	+ if (src_slice && dst_slice) {
	2797	+ memmove(dst_slice, src_slice, size);
	2798	+ return 0;
	2799	+ }
	2800	+
	2801	+ if (src_slice)
	2802	+ return __bpf_dynptr_write(dst, dst_off, src_slice, size, 0);
	2803	+
	2804	+ if (dst_slice)
	2805	+ return __bpf_dynptr_read(dst_slice, size, src, src_off, 0);
	2806	+
	2807	+ if (bpf_dynptr_check_off_len(dst, dst_off, size) \|\|
	2808	+ bpf_dynptr_check_off_len(src, src_off, size))
	2809	+ return -E2BIG;
	2810	+
	2811	+ off = 0;
	2812	+ while (off < size) {
	2813	+ u32 chunk_sz = min_t(u32, sizeof(buf), size - off);
	2814	+ int err;
	2815	+
	2816	+ err = __bpf_dynptr_read(buf, chunk_sz, src, src_off + off, 0);
	2817	+ if (err)
	2818	+ return err;
	2819	+ err = __bpf_dynptr_write(dst, dst_off + off, buf, chunk_sz, 0);
	2820	+ if (err)
	2821	+ return err;
	2822	+
	2823	+ off += chunk_sz;
	2824	+ }
	2825	+ return 0;
	2826	+}
	2827	+
2761	2828	__bpf_kfunc void bpf_cast_to_kern_ctx(void obj)
2762	2829	{
2763	2830	return obj;
@@ -3206,6 +3273,7 @@ BTF_ID_FLAGS(func, bpf_dynptr_is_null) · 442 unmodified lines
3206	3273	BTF_ID_FLAGS(func, bpf_dynptr_is_rdonly)
3207	3274	BTF_ID_FLAGS(func, bpf_dynptr_size)
3208	3275	BTF_ID_FLAGS(func, bpf_dynptr_clone)
	3276	+BTF_ID_FLAGS(func, bpf_dynptr_copy)
3209	3277	#ifdef CONFIG_NET
3210	3278	BTF_ID_FLAGS(func, bpf_modify_return_test_tp)
3211	3279	#endif

tools/testing/selftests/bpf/prog_tests/dynptr.c+21−0

@@ -10,6 +10,7 @@ enum test_setup_type { · 9 unmodified lines
10	10	SETUP_SYSCALL_SLEEP,
11	11	SETUP_SKB_PROG,
12	12	SETUP_SKB_PROG_TP,
	13	+ SETUP_XDP_PROG,
13	14	};
14	15
15	16	static struct {
@@ -18,6 +19,8 @@ static struct { · 2 unmodified lines
18	19	} success_tests[] = {
19	20	{"test_read_write", SETUP_SYSCALL_SLEEP},
20	21	{"test_dynptr_data", SETUP_SYSCALL_SLEEP},
	22	+ {"test_dynptr_copy", SETUP_SYSCALL_SLEEP},
	23	+ {"test_dynptr_copy_xdp", SETUP_XDP_PROG},
21	24	{"test_ringbuf", SETUP_SYSCALL_SLEEP},
22	25	{"test_skb_readonly", SETUP_SKB_PROG},
23	26	{"test_dynptr_skb_data", SETUP_SKB_PROG},
@@ -120,6 +123,24 @@ static void verify_success(const char *prog_name, enum test_setup_type setup_typ · 96 unmodified lines
120	123
121	124	break;
122	125	}
	126	+ case SETUP_XDP_PROG:
	127	+ {
	128	+ char data[5000];
	129	+ int err, prog_fd;
	130	+ LIBBPF_OPTS(bpf_test_run_opts, opts,
	131	+ .data_in = &data,
	132	+ .data_size_in = sizeof(data),
	133	+ .repeat = 1,
	134	+ );
	135	+
	136	+ prog_fd = bpf_program__fd(prog);
	137	+ err = bpf_prog_test_run_opts(prog_fd, &opts);
	138	+
	139	+ if (!ASSERT_OK(err, "test_run"))
	140	+ goto cleanup;
	141	+
	142	+ break;
	143	+ }
123	144	}
124	145
125	146	ASSERT_EQ(skel->bss->err, 0, "err");

tools/testing/selftests/bpf/progs/dynptr_success.c+118−5

@@ -1,20 +1,19 @@
1	1	// SPDX-License-Identifier: GPL-2.0
2	2	/* Copyright (c) 2022 Facebook */
3	3
	4	+#include <vmlinux.h>
4	5	#include <string.h>
5	6	#include <stdbool.h>
6		−#include <linux/bpf.h>
7	7	#include <bpf/bpf_helpers.h>
8	8	#include <bpf/bpf_tracing.h>
9	9	#include "bpf_misc.h"
10		−#include "bpf_kfuncs.h"
11	10	#include "errno.h"
12	11
13	12	char _license[] SEC("license") = "GPL";
14	13
15	14	int pid, err, val;
16	15
17		−struct sample {
	16	+struct ringbuf_sample {
18	17	int pid;
19	18	int seq;
20	19	long value;
@@ -121,7 +120,7 @@ int test_dynptr_data(void *ctx) · 100 unmodified lines
121	120
122	121	static int ringbuf_callback(__u32 index, void *data)
123	122	{
124		− struct sample *sample;
	123	+ struct ringbuf_sample *sample;
125	124
126	125	struct bpf_dynptr ptr = (struct bpf_dynptr )data;
127	126
@@ -138,7 +137,7 @@ SEC("?tp/syscalls/sys_enter_nanosleep") · 10 unmodified lines
138	137	int test_ringbuf(void *ctx)
139	138	{
140	139	struct bpf_dynptr ptr;
141		− struct sample *sample;
	140	+ struct ringbuf_sample *sample;
142	141
143	142	if (bpf_get_current_pid_tgid() >> 32 != pid)
144	143	return 0;
@@ -567,3 +566,117 @@ int BPF_PROG(test_dynptr_skb_tp_btf, void skb, void location) · 422 unmodified lines
567	566
568	567	return 1;
569	568	}
	569	+
	570	+static inline int bpf_memcmp(const char a, const char b, u32 size)
	571	+{
	572	+ int i;
	573	+
	574	+ bpf_for(i, 0, size) {
	575	+ if (a[i] != b[i])
	576	+ return a[i] < b[i] ? -1 : 1;
	577	+ }
	578	+ return 0;
	579	+}
	580	+
	581	+SEC("?tp/syscalls/sys_enter_nanosleep")
	582	+int test_dynptr_copy(void *ctx)
	583	+{
	584	+ char data[] = "hello there, world!!";
	585	+ char buf[32] = {'\0'};
	586	+ __u32 sz = sizeof(data);
	587	+ struct bpf_dynptr src, dst;
	588	+
	589	+ bpf_ringbuf_reserve_dynptr(&ringbuf, sz, 0, &src);
	590	+ bpf_ringbuf_reserve_dynptr(&ringbuf, sz, 0, &dst);
	591	+
	592	+ /* Test basic case of copying contiguous memory backed dynptrs */
	593	+ err = bpf_dynptr_write(&src, 0, data, sz, 0);
	594	+ err = err ?: bpf_dynptr_copy(&dst, 0, &src, 0, sz);
	595	+ err = err ?: bpf_dynptr_read(buf, sz, &dst, 0, 0);
	596	+ err = err ?: bpf_memcmp(data, buf, sz);
	597	+
	598	+ /* Test that offsets are handled correctly */
	599	+ err = err ?: bpf_dynptr_copy(&dst, 3, &src, 5, sz - 5);
	600	+ err = err ?: bpf_dynptr_read(buf, sz - 5, &dst, 3, 0);
	601	+ err = err ?: bpf_memcmp(data + 5, buf, sz - 5);
	602	+
	603	+ bpf_ringbuf_discard_dynptr(&src, 0);
	604	+ bpf_ringbuf_discard_dynptr(&dst, 0);
	605	+ return 0;
	606	+}
	607	+
	608	+SEC("xdp")
	609	+int test_dynptr_copy_xdp(struct xdp_md *xdp)
	610	+{
	611	+ struct bpf_dynptr ptr_buf, ptr_xdp;
	612	+ char data[] = "qwertyuiopasdfghjkl";
	613	+ char buf[32] = {'\0'};
	614	+ __u32 len = sizeof(data);
	615	+ int i, chunks = 200;
	616	+
	617	+ /* ptr_xdp is backed by non-contiguous memory */
	618	+ bpf_dynptr_from_xdp(xdp, 0, &ptr_xdp);
	619	+ bpf_ringbuf_reserve_dynptr(&ringbuf, len * chunks, 0, &ptr_buf);
	620	+
	621	+ /* Destination dynptr is backed by non-contiguous memory */
	622	+ bpf_for(i, 0, chunks) {
	623	+ err = bpf_dynptr_write(&ptr_buf, i * len, data, len, 0);
	624	+ if (err)
	625	+ goto out;
	626	+ }
	627	+
	628	+ err = bpf_dynptr_copy(&ptr_xdp, 0, &ptr_buf, 0, len * chunks);
	629	+ if (err)
	630	+ goto out;
	631	+
	632	+ bpf_for(i, 0, chunks) {
	633	+ __builtin_memset(buf, 0, sizeof(buf));
	634	+ err = bpf_dynptr_read(&buf, len, &ptr_xdp, i * len, 0);
	635	+ if (err)
	636	+ goto out;
	637	+ if (bpf_memcmp(data, buf, len) != 0)
	638	+ goto out;
	639	+ }
	640	+
	641	+ /* Source dynptr is backed by non-contiguous memory */
	642	+ __builtin_memset(buf, 0, sizeof(buf));
	643	+ bpf_for(i, 0, chunks) {
	644	+ err = bpf_dynptr_write(&ptr_buf, i * len, buf, len, 0);
	645	+ if (err)
	646	+ goto out;
	647	+ }
	648	+
	649	+ err = bpf_dynptr_copy(&ptr_buf, 0, &ptr_xdp, 0, len * chunks);
	650	+ if (err)
	651	+ goto out;
	652	+
	653	+ bpf_for(i, 0, chunks) {
	654	+ __builtin_memset(buf, 0, sizeof(buf));
	655	+ err = bpf_dynptr_read(&buf, len, &ptr_buf, i * len, 0);
	656	+ if (err)
	657	+ goto out;
	658	+ if (bpf_memcmp(data, buf, len) != 0)
	659	+ goto out;
	660	+ }
	661	+
	662	+ /* Both source and destination dynptrs are backed by non-contiguous memory */
	663	+ err = bpf_dynptr_copy(&ptr_xdp, 2, &ptr_xdp, len, len * (chunks - 1));
	664	+ if (err)
	665	+ goto out;
	666	+
	667	+ bpf_for(i, 0, chunks - 1) {
	668	+ __builtin_memset(buf, 0, sizeof(buf));
	669	+ err = bpf_dynptr_read(&buf, len, &ptr_xdp, 2 + i * len, 0);
	670	+ if (err)
	671	+ goto out;
	672	+ if (bpf_memcmp(data, buf, len) != 0)
	673	+ goto out;
	674	+ }
	675	+
	676	+ if (bpf_dynptr_copy(&ptr_xdp, 2000, &ptr_xdp, 0, len * chunks) != -E2BIG)
	677	+ err = 1;
	678	+
	679	+out:
	680	+ bpf_ringbuf_discard_dynptr(&ptr_buf, 0);
	681	+ return XDP_DROP;
	682	+}

Merge branch 'introduce-bpf_dynptr_copy-kfunc'

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code