Mykyta Yatsenko says: ==================== introduce bpf_dynptr_copy kfunc From: Mykyta Yatsenko <yatsenko@meta.com> Introduce a new kfunc, bpf_dynptr_copy, which enables copying of data from one dynptr to another. This functionality may be useful in scenarios such as capturing XDP data to a ring buffer. The patch set is split into 3 patches: 1. Refactor bpf_dynptr_read and bpf_dynptr_write by extracting code into static functions, that allows calling them with no compiler warnings 2. Introduce bpf_dynptr_copy 3. Add tests for bpf_dynptr_copy v2->v3: * Implemented bpf_memcmp in dynptr_success.c test, as __builtin_memcmp was not inlined on GCC-BPF. ====================
Unified · Split
| @@ -1759,8 +1759,8 @@ static const struct bpf_func_proto bpf_dynptr_from_mem_proto = { · 1758 unmodified lines | |||
| 1759 | .arg4_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_LOCAL | MEM_UNINIT | MEM_WRITE, | 1759 | .arg4_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_LOCAL | MEM_UNINIT | MEM_WRITE, |
| 1760 | }; | 1760 | }; |
| 1761 | 1761 | ||
| 1762 | BPF_CALL_5(bpf_dynptr_read, void *, dst, u32, len, const struct bpf_dynptr_kern *, src, | 1762 | static int __bpf_dynptr_read(void *dst, u32 len, const struct bpf_dynptr_kern *src, |
| 1763 | u32, offset, u64, flags) | 1763 | u32 offset, u64 flags) |
| 1764 | { | 1764 | { |
| 1765 | enum bpf_dynptr_type type; | 1765 | enum bpf_dynptr_type type; |
| 1766 | int err; | 1766 | int err; |
| @@ -1793,6 +1793,12 @@ BPF_CALL_5(bpf_dynptr_read, void *, dst, u32, len, const struct bpf_dynptr_kern · 26 unmodified lines | |||
| 1793 | } | 1793 | } |
| 1794 | } | 1794 | } |
| 1795 | 1795 | ||
| 1796 | BPF_CALL_5(bpf_dynptr_read, void *, dst, u32, len, const struct bpf_dynptr_kern *, src, | ||
| 1797 | u32, offset, u64, flags) | ||
| 1798 | { | ||
| 1799 | return __bpf_dynptr_read(dst, len, src, offset, flags); | ||
| 1800 | } | ||
| 1801 | |||
| 1796 | static const struct bpf_func_proto bpf_dynptr_read_proto = { | 1802 | static const struct bpf_func_proto bpf_dynptr_read_proto = { |
| 1797 | .func = bpf_dynptr_read, | 1803 | .func = bpf_dynptr_read, |
| 1798 | .gpl_only = false, | 1804 | .gpl_only = false, |
| @@ -1804,8 +1810,8 @@ static const struct bpf_func_proto bpf_dynptr_read_proto = { · 5 unmodified lines | |||
| 1804 | .arg5_type = ARG_ANYTHING, | 1810 | .arg5_type = ARG_ANYTHING, |
| 1805 | }; | 1811 | }; |
| 1806 | 1812 | ||
| 1807 | BPF_CALL_5(bpf_dynptr_write, const struct bpf_dynptr_kern *, dst, u32, offset, void *, src, | 1813 | static int __bpf_dynptr_write(const struct bpf_dynptr_kern *dst, u32 offset, void *src, |
| 1808 | u32, len, u64, flags) | 1814 | u32 len, u64 flags) |
| 1809 | { | 1815 | { |
| 1810 | enum bpf_dynptr_type type; | 1816 | enum bpf_dynptr_type type; |
| 1811 | int err; | 1817 | int err; |
| @@ -1843,6 +1849,12 @@ BPF_CALL_5(bpf_dynptr_write, const struct bpf_dynptr_kern *, dst, u32, offset, v · 31 unmodified lines | |||
| 1843 | } | 1849 | } |
| 1844 | } | 1850 | } |
| 1845 | 1851 | ||
| 1852 | BPF_CALL_5(bpf_dynptr_write, const struct bpf_dynptr_kern *, dst, u32, offset, void *, src, | ||
| 1853 | u32, len, u64, flags) | ||
| 1854 | { | ||
| 1855 | return __bpf_dynptr_write(dst, offset, src, len, flags); | ||
| 1856 | } | ||
| 1857 | |||
| 1846 | static const struct bpf_func_proto bpf_dynptr_write_proto = { | 1858 | static const struct bpf_func_proto bpf_dynptr_write_proto = { |
| 1847 | .func = bpf_dynptr_write, | 1859 | .func = bpf_dynptr_write, |
| 1848 | .gpl_only = false, | 1860 | .gpl_only = false, |
| @@ -2758,6 +2770,61 @@ __bpf_kfunc int bpf_dynptr_clone(const struct bpf_dynptr *p, · 909 unmodified lines | |||
| 2758 | return 0; | 2770 | return 0; |
| 2759 | } | 2771 | } |
| 2760 | 2772 | ||
| 2773 | /** | ||
| 2774 | * bpf_dynptr_copy() - Copy data from one dynptr to another. | ||
| 2775 | * @dst_ptr: Destination dynptr - where data should be copied to | ||
| 2776 | * @dst_off: Offset into the destination dynptr | ||
| 2777 | * @src_ptr: Source dynptr - where data should be copied from | ||
| 2778 | * @src_off: Offset into the source dynptr | ||
| 2779 | * @size: Length of the data to copy from source to destination | ||
| 2780 | * | ||
| 2781 | * Copies data from source dynptr to destination dynptr. | ||
| 2782 | * Returns 0 on success; negative error, otherwise. | ||
| 2783 | */ | ||
| 2784 | __bpf_kfunc int bpf_dynptr_copy(struct bpf_dynptr *dst_ptr, u32 dst_off, | ||
| 2785 | struct bpf_dynptr *src_ptr, u32 src_off, u32 size) | ||
| 2786 | { | ||
| 2787 | struct bpf_dynptr_kern *dst = (struct bpf_dynptr_kern *)dst_ptr; | ||
| 2788 | struct bpf_dynptr_kern *src = (struct bpf_dynptr_kern *)src_ptr; | ||
| 2789 | void *src_slice, *dst_slice; | ||
| 2790 | char buf[256]; | ||
| 2791 | u32 off; | ||
| 2792 | |||
| 2793 | src_slice = bpf_dynptr_slice(src_ptr, src_off, NULL, size); | ||
| 2794 | dst_slice = bpf_dynptr_slice_rdwr(dst_ptr, dst_off, NULL, size); | ||
| 2795 | |||
| 2796 | if (src_slice && dst_slice) { | ||
| 2797 | memmove(dst_slice, src_slice, size); | ||
| 2798 | return 0; | ||
| 2799 | } | ||
| 2800 | |||
| 2801 | if (src_slice) | ||
| 2802 | return __bpf_dynptr_write(dst, dst_off, src_slice, size, 0); | ||
| 2803 | |||
| 2804 | if (dst_slice) | ||
| 2805 | return __bpf_dynptr_read(dst_slice, size, src, src_off, 0); | ||
| 2806 | |||
| 2807 | if (bpf_dynptr_check_off_len(dst, dst_off, size) || | ||
| 2808 | bpf_dynptr_check_off_len(src, src_off, size)) | ||
| 2809 | return -E2BIG; | ||
| 2810 | |||
| 2811 | off = 0; | ||
| 2812 | while (off < size) { | ||
| 2813 | u32 chunk_sz = min_t(u32, sizeof(buf), size - off); | ||
| 2814 | int err; | ||
| 2815 | |||
| 2816 | err = __bpf_dynptr_read(buf, chunk_sz, src, src_off + off, 0); | ||
| 2817 | if (err) | ||
| 2818 | return err; | ||
| 2819 | err = __bpf_dynptr_write(dst, dst_off + off, buf, chunk_sz, 0); | ||
| 2820 | if (err) | ||
| 2821 | return err; | ||
| 2822 | |||
| 2823 | off += chunk_sz; | ||
| 2824 | } | ||
| 2825 | return 0; | ||
| 2826 | } | ||
| 2827 | |||
| 2761 | __bpf_kfunc void *bpf_cast_to_kern_ctx(void *obj) | 2828 | __bpf_kfunc void *bpf_cast_to_kern_ctx(void *obj) |
| 2762 | { | 2829 | { |
| 2763 | return obj; | 2830 | return obj; |
| @@ -3206,6 +3273,7 @@ BTF_ID_FLAGS(func, bpf_dynptr_is_null) · 442 unmodified lines | |||
| 3206 | BTF_ID_FLAGS(func, bpf_dynptr_is_rdonly) | 3273 | BTF_ID_FLAGS(func, bpf_dynptr_is_rdonly) |
| 3207 | BTF_ID_FLAGS(func, bpf_dynptr_size) | 3274 | BTF_ID_FLAGS(func, bpf_dynptr_size) |
| 3208 | BTF_ID_FLAGS(func, bpf_dynptr_clone) | 3275 | BTF_ID_FLAGS(func, bpf_dynptr_clone) |
| 3276 | BTF_ID_FLAGS(func, bpf_dynptr_copy) | ||
| 3209 | #ifdef CONFIG_NET | 3277 | #ifdef CONFIG_NET |
| 3210 | BTF_ID_FLAGS(func, bpf_modify_return_test_tp) | 3278 | BTF_ID_FLAGS(func, bpf_modify_return_test_tp) |
| 3211 | #endif | 3279 | #endif |
| @@ -10,6 +10,7 @@ enum test_setup_type { · 9 unmodified lines | |||
| 10 | SETUP_SYSCALL_SLEEP, | 10 | SETUP_SYSCALL_SLEEP, |
| 11 | SETUP_SKB_PROG, | 11 | SETUP_SKB_PROG, |
| 12 | SETUP_SKB_PROG_TP, | 12 | SETUP_SKB_PROG_TP, |
| 13 | SETUP_XDP_PROG, | ||
| 13 | }; | 14 | }; |
| 14 | 15 | ||
| 15 | static struct { | 16 | static struct { |
| @@ -18,6 +19,8 @@ static struct { · 2 unmodified lines | |||
| 18 | } success_tests[] = { | 19 | } success_tests[] = { |
| 19 | {"test_read_write", SETUP_SYSCALL_SLEEP}, | 20 | {"test_read_write", SETUP_SYSCALL_SLEEP}, |
| 20 | {"test_dynptr_data", SETUP_SYSCALL_SLEEP}, | 21 | {"test_dynptr_data", SETUP_SYSCALL_SLEEP}, |
| 22 | {"test_dynptr_copy", SETUP_SYSCALL_SLEEP}, | ||
| 23 | {"test_dynptr_copy_xdp", SETUP_XDP_PROG}, | ||
| 21 | {"test_ringbuf", SETUP_SYSCALL_SLEEP}, | 24 | {"test_ringbuf", SETUP_SYSCALL_SLEEP}, |
| 22 | {"test_skb_readonly", SETUP_SKB_PROG}, | 25 | {"test_skb_readonly", SETUP_SKB_PROG}, |
| 23 | {"test_dynptr_skb_data", SETUP_SKB_PROG}, | 26 | {"test_dynptr_skb_data", SETUP_SKB_PROG}, |
| @@ -120,6 +123,24 @@ static void verify_success(const char *prog_name, enum test_setup_type setup_typ · 96 unmodified lines | |||
| 120 | 123 | ||
| 121 | break; | 124 | break; |
| 122 | } | 125 | } |
| 126 | case SETUP_XDP_PROG: | ||
| 127 | { | ||
| 128 | char data[5000]; | ||
| 129 | int err, prog_fd; | ||
| 130 | LIBBPF_OPTS(bpf_test_run_opts, opts, | ||
| 131 | .data_in = &data, | ||
| 132 | .data_size_in = sizeof(data), | ||
| 133 | .repeat = 1, | ||
| 134 | ); | ||
| 135 | |||
| 136 | prog_fd = bpf_program__fd(prog); | ||
| 137 | err = bpf_prog_test_run_opts(prog_fd, &opts); | ||
| 138 | |||
| 139 | if (!ASSERT_OK(err, "test_run")) | ||
| 140 | goto cleanup; | ||
| 141 | |||
| 142 | break; | ||
| 143 | } | ||
| 123 | } | 144 | } |
| 124 | 145 | ||
| 125 | ASSERT_EQ(skel->bss->err, 0, "err"); | 146 | ASSERT_EQ(skel->bss->err, 0, "err"); |
| @@ -1,20 +1,19 @@ | |||
| 1 | // SPDX-License-Identifier: GPL-2.0 | 1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | /* Copyright (c) 2022 Facebook */ | 2 | /* Copyright (c) 2022 Facebook */ |
| 3 | 3 | ||
| 4 | #include <vmlinux.h> | ||
| 4 | #include <string.h> | 5 | #include <string.h> |
| 5 | #include <stdbool.h> | 6 | #include <stdbool.h> |
| 6 | #include <linux/bpf.h> | ||
| 7 | #include <bpf/bpf_helpers.h> | 7 | #include <bpf/bpf_helpers.h> |
| 8 | #include <bpf/bpf_tracing.h> | 8 | #include <bpf/bpf_tracing.h> |
| 9 | #include "bpf_misc.h" | 9 | #include "bpf_misc.h" |
| 10 | #include "bpf_kfuncs.h" | ||
| 11 | #include "errno.h" | 10 | #include "errno.h" |
| 12 | 11 | ||
| 13 | char _license[] SEC("license") = "GPL"; | 12 | char _license[] SEC("license") = "GPL"; |
| 14 | 13 | ||
| 15 | int pid, err, val; | 14 | int pid, err, val; |
| 16 | 15 | ||
| 17 | struct sample { | 16 | struct ringbuf_sample { |
| 18 | int pid; | 17 | int pid; |
| 19 | int seq; | 18 | int seq; |
| 20 | long value; | 19 | long value; |
| @@ -121,7 +120,7 @@ int test_dynptr_data(void *ctx) · 100 unmodified lines | |||
| 121 | 120 | ||
| 122 | static int ringbuf_callback(__u32 index, void *data) | 121 | static int ringbuf_callback(__u32 index, void *data) |
| 123 | { | 122 | { |
| 124 | struct sample *sample; | 123 | struct ringbuf_sample *sample; |
| 125 | 124 | ||
| 126 | struct bpf_dynptr *ptr = (struct bpf_dynptr *)data; | 125 | struct bpf_dynptr *ptr = (struct bpf_dynptr *)data; |
| 127 | 126 | ||
| @@ -138,7 +137,7 @@ SEC("?tp/syscalls/sys_enter_nanosleep") · 10 unmodified lines | |||
| 138 | int test_ringbuf(void *ctx) | 137 | int test_ringbuf(void *ctx) |
| 139 | { | 138 | { |
| 140 | struct bpf_dynptr ptr; | 139 | struct bpf_dynptr ptr; |
| 141 | struct sample *sample; | 140 | struct ringbuf_sample *sample; |
| 142 | 141 | ||
| 143 | if (bpf_get_current_pid_tgid() >> 32 != pid) | 142 | if (bpf_get_current_pid_tgid() >> 32 != pid) |
| 144 | return 0; | 143 | return 0; |
| @@ -567,3 +566,117 @@ int BPF_PROG(test_dynptr_skb_tp_btf, void *skb, void *location) · 422 unmodified lines | |||
| 567 | 566 | ||
| 568 | return 1; | 567 | return 1; |
| 569 | } | 568 | } |
| 569 | |||
| 570 | static inline int bpf_memcmp(const char *a, const char *b, u32 size) | ||
| 571 | { | ||
| 572 | int i; | ||
| 573 | |||
| 574 | bpf_for(i, 0, size) { | ||
| 575 | if (a[i] != b[i]) | ||
| 576 | return a[i] < b[i] ? -1 : 1; | ||
| 577 | } | ||
| 578 | return 0; | ||
| 579 | } | ||
| 580 | |||
| 581 | SEC("?tp/syscalls/sys_enter_nanosleep") | ||
| 582 | int test_dynptr_copy(void *ctx) | ||
| 583 | { | ||
| 584 | char data[] = "hello there, world!!"; | ||
| 585 | char buf[32] = {'\0'}; | ||
| 586 | __u32 sz = sizeof(data); | ||
| 587 | struct bpf_dynptr src, dst; | ||
| 588 | |||
| 589 | bpf_ringbuf_reserve_dynptr(&ringbuf, sz, 0, &src); | ||
| 590 | bpf_ringbuf_reserve_dynptr(&ringbuf, sz, 0, &dst); | ||
| 591 | |||
| 592 | /* Test basic case of copying contiguous memory backed dynptrs */ | ||
| 593 | err = bpf_dynptr_write(&src, 0, data, sz, 0); | ||
| 594 | err = err ?: bpf_dynptr_copy(&dst, 0, &src, 0, sz); | ||
| 595 | err = err ?: bpf_dynptr_read(buf, sz, &dst, 0, 0); | ||
| 596 | err = err ?: bpf_memcmp(data, buf, sz); | ||
| 597 | |||
| 598 | /* Test that offsets are handled correctly */ | ||
| 599 | err = err ?: bpf_dynptr_copy(&dst, 3, &src, 5, sz - 5); | ||
| 600 | err = err ?: bpf_dynptr_read(buf, sz - 5, &dst, 3, 0); | ||
| 601 | err = err ?: bpf_memcmp(data + 5, buf, sz - 5); | ||
| 602 | |||
| 603 | bpf_ringbuf_discard_dynptr(&src, 0); | ||
| 604 | bpf_ringbuf_discard_dynptr(&dst, 0); | ||
| 605 | return 0; | ||
| 606 | } | ||
| 607 | |||
| 608 | SEC("xdp") | ||
| 609 | int test_dynptr_copy_xdp(struct xdp_md *xdp) | ||
| 610 | { | ||
| 611 | struct bpf_dynptr ptr_buf, ptr_xdp; | ||
| 612 | char data[] = "qwertyuiopasdfghjkl"; | ||
| 613 | char buf[32] = {'\0'}; | ||
| 614 | __u32 len = sizeof(data); | ||
| 615 | int i, chunks = 200; | ||
| 616 | |||
| 617 | /* ptr_xdp is backed by non-contiguous memory */ | ||
| 618 | bpf_dynptr_from_xdp(xdp, 0, &ptr_xdp); | ||
| 619 | bpf_ringbuf_reserve_dynptr(&ringbuf, len * chunks, 0, &ptr_buf); | ||
| 620 | |||
| 621 | /* Destination dynptr is backed by non-contiguous memory */ | ||
| 622 | bpf_for(i, 0, chunks) { | ||
| 623 | err = bpf_dynptr_write(&ptr_buf, i * len, data, len, 0); | ||
| 624 | if (err) | ||
| 625 | goto out; | ||
| 626 | } | ||
| 627 | |||
| 628 | err = bpf_dynptr_copy(&ptr_xdp, 0, &ptr_buf, 0, len * chunks); | ||
| 629 | if (err) | ||
| 630 | goto out; | ||
| 631 | |||
| 632 | bpf_for(i, 0, chunks) { | ||
| 633 | __builtin_memset(buf, 0, sizeof(buf)); | ||
| 634 | err = bpf_dynptr_read(&buf, len, &ptr_xdp, i * len, 0); | ||
| 635 | if (err) | ||
| 636 | goto out; | ||
| 637 | if (bpf_memcmp(data, buf, len) != 0) | ||
| 638 | goto out; | ||
| 639 | } | ||
| 640 | |||
| 641 | /* Source dynptr is backed by non-contiguous memory */ | ||
| 642 | __builtin_memset(buf, 0, sizeof(buf)); | ||
| 643 | bpf_for(i, 0, chunks) { | ||
| 644 | err = bpf_dynptr_write(&ptr_buf, i * len, buf, len, 0); | ||
| 645 | if (err) | ||
| 646 | goto out; | ||
| 647 | } | ||
| 648 | |||
| 649 | err = bpf_dynptr_copy(&ptr_buf, 0, &ptr_xdp, 0, len * chunks); | ||
| 650 | if (err) | ||
| 651 | goto out; | ||
| 652 | |||
| 653 | bpf_for(i, 0, chunks) { | ||
| 654 | __builtin_memset(buf, 0, sizeof(buf)); | ||
| 655 | err = bpf_dynptr_read(&buf, len, &ptr_buf, i * len, 0); | ||
| 656 | if (err) | ||
| 657 | goto out; | ||
| 658 | if (bpf_memcmp(data, buf, len) != 0) | ||
| 659 | goto out; | ||
| 660 | } | ||
| 661 | |||
| 662 | /* Both source and destination dynptrs are backed by non-contiguous memory */ | ||
| 663 | err = bpf_dynptr_copy(&ptr_xdp, 2, &ptr_xdp, len, len * (chunks - 1)); | ||
| 664 | if (err) | ||
| 665 | goto out; | ||
| 666 | |||
| 667 | bpf_for(i, 0, chunks - 1) { | ||
| 668 | __builtin_memset(buf, 0, sizeof(buf)); | ||
| 669 | err = bpf_dynptr_read(&buf, len, &ptr_xdp, 2 + i * len, 0); | ||
| 670 | if (err) | ||
| 671 | goto out; | ||
| 672 | if (bpf_memcmp(data, buf, len) != 0) | ||
| 673 | goto out; | ||
| 674 | } | ||
| 675 | |||
| 676 | if (bpf_dynptr_copy(&ptr_xdp, 2000, &ptr_xdp, 0, len * chunks) != -E2BIG) | ||
| 677 | err = 1; | ||
| 678 | |||
| 679 | out: | ||
| 680 | bpf_ringbuf_discard_dynptr(&ptr_buf, 0); | ||
| 681 | return XDP_DROP; | ||
| 682 | } | ||