Daniel Borkmann says: ==================== pull-request: bpf-next 2020-09-01 The following pull-request contains BPF updates for your *net-next* tree. There are two small conflicts when pulling, resolve as follows: 1) Merge conflict in tools/lib/bpf/libbpf.c between 88a82120282b ("libbpf: Factor out common ELF operations and improve logging") in bpf-next and 1e891e513e16 ("libbpf: Fix map index used in error message") in net-next. Resolve by taking the hunk in bpf-next: [...] scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx); data = elf_sec_data(obj, scn); if (!scn || !data) { pr_warn("elf: failed to get %s map definitions for %s\n", MAPS_ELF_SEC, obj->path); return -EINVAL; } [...] 2) Merge conflict in drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c between 9647c57b11e5 ("xsk: i40e: ice: ixgbe: mlx5: Test for dma_need_sync earlier for better performance") in bpf-next and e20f0dbf204f ("net/mlx5e: RX, Add a prefetch command for small L1_CACHE_BYTES") in net-next. Resolve the two locations by retaining net_prefetch() and taking xsk_buff_dma_sync_for_cpu() from bpf-next. Should look like: [...] xdp_set_data_meta_invalid(xdp); xsk_buff_dma_sync_for_cpu(xdp, rq->xsk_pool); net_prefetch(xdp->data); [...] We've added 133 non-merge commits during the last 14 day(s) which contain a total of 246 files changed, 13832 insertions(+), 3105 deletions(-). The main changes are: 1) Initial support for sleepable BPF programs along with bpf_copy_from_user() helper for tracing to reliably access user memory, from Alexei Starovoitov. 2) Add BPF infra for writing and parsing TCP header options, from Martin KaFai Lau. 3) bpf_d_path() helper for returning full path for given 'struct path', from Jiri Olsa. 4) AF_XDP support for shared umems between devices and queues, from Magnus Karlsson. 5) Initial prep work for full BPF-to-BPF call support in libbpf, from Andrii Nakryiko. 6) Generalize bpf_sk_storage map & add local storage for inodes, from KP Singh. 7) Implement sockmap/hash updates from BPF context, from Lorenz Bauer. 8) BPF xor verification for scalar types & add BPF link iterator, from Yonghong Song. 9) Use target's prog type for BPF_PROG_TYPE_EXT prog verification, from Udip Pant. 10) Rework BPF tracing samples to use libbpf loader, from Daniel T. Lee. 11) Fix xdpsock sample to really cycle through all buffers, from Weqaar Janjua. 12) Improve type safety for tun/veth XDP frame handling, from Maciej Żenczykowski. 13) Various smaller cleanups and improvements all over the place. ====================
Showing the changes to include/uapi (1 of 246 files). Show all
Unified · Split
| @@ -155,6 +155,7 @@ enum bpf_map_type { · 154 unmodified lines | |||
| 155 | BPF_MAP_TYPE_DEVMAP_HASH, | 155 | BPF_MAP_TYPE_DEVMAP_HASH, |
| 156 | BPF_MAP_TYPE_STRUCT_OPS, | 156 | BPF_MAP_TYPE_STRUCT_OPS, |
| 157 | BPF_MAP_TYPE_RINGBUF, | 157 | BPF_MAP_TYPE_RINGBUF, |
| 158 | BPF_MAP_TYPE_INODE_STORAGE, | ||
| 158 | }; | 159 | }; |
| 159 | 160 | ||
| 160 | /* Note that tracing related programs such as | 161 | /* Note that tracing related programs such as |
| @@ -345,6 +346,14 @@ enum bpf_link_type { · 184 unmodified lines | |||
| 345 | /* The verifier internal test flag. Behavior is undefined */ | 346 | /* The verifier internal test flag. Behavior is undefined */ |
| 346 | #define BPF_F_TEST_STATE_FREQ (1U << 3) | 347 | #define BPF_F_TEST_STATE_FREQ (1U << 3) |
| 347 | 348 | ||
| 349 | /* If BPF_F_SLEEPABLE is used in BPF_PROG_LOAD command, the verifier will | ||
| 350 | * restrict map and helper usage for such programs. Sleepable BPF programs can | ||
| 351 | * only be attached to hooks where kernel execution context allows sleeping. | ||
| 352 | * Such programs are allowed to use helpers that may sleep like | ||
| 353 | * bpf_copy_from_user(). | ||
| 354 | */ | ||
| 355 | #define BPF_F_SLEEPABLE (1U << 4) | ||
| 356 | |||
| 348 | /* When BPF ldimm64's insn[0].src_reg != 0 then this can have | 357 | /* When BPF ldimm64's insn[0].src_reg != 0 then this can have |
| 349 | * two extensions: | 358 | * two extensions: |
| 350 | * | 359 | * |
| @@ -2807,7 +2816,7 @@ union bpf_attr { · 2456 unmodified lines | |||
| 2807 | * | 2816 | * |
| 2808 | * **-ERANGE** if resulting value was out of range. | 2817 | * **-ERANGE** if resulting value was out of range. |
| 2809 | * | 2818 | * |
| 2810 | * void *bpf_sk_storage_get(struct bpf_map *map, struct bpf_sock *sk, void *value, u64 flags) | 2819 | * void *bpf_sk_storage_get(struct bpf_map *map, void *sk, void *value, u64 flags) |
| 2811 | * Description | 2820 | * Description |
| 2812 | * Get a bpf-local-storage from a *sk*. | 2821 | * Get a bpf-local-storage from a *sk*. |
| 2813 | * | 2822 | * |
| @@ -2823,6 +2832,9 @@ union bpf_attr { · 9 unmodified lines | |||
| 2823 | * "type". The bpf-local-storage "type" (i.e. the *map*) is | 2832 | * "type". The bpf-local-storage "type" (i.e. the *map*) is |
| 2824 | * searched against all bpf-local-storages residing at *sk*. | 2833 | * searched against all bpf-local-storages residing at *sk*. |
| 2825 | * | 2834 | * |
| 2835 | * *sk* is a kernel **struct sock** pointer for LSM program. | ||
| 2836 | * *sk* is a **struct bpf_sock** pointer for other program types. | ||
| 2837 | * | ||
| 2826 | * An optional *flags* (**BPF_SK_STORAGE_GET_F_CREATE**) can be | 2838 | * An optional *flags* (**BPF_SK_STORAGE_GET_F_CREATE**) can be |
| 2827 | * used such that a new bpf-local-storage will be | 2839 | * used such that a new bpf-local-storage will be |
| 2828 | * created if one does not exist. *value* can be used | 2840 | * created if one does not exist. *value* can be used |
| @@ -2835,7 +2847,7 @@ union bpf_attr { · 6 unmodified lines | |||
| 2835 | * **NULL** if not found or there was an error in adding | 2847 | * **NULL** if not found or there was an error in adding |
| 2836 | * a new bpf-local-storage. | 2848 | * a new bpf-local-storage. |
| 2837 | * | 2849 | * |
| 2838 | * long bpf_sk_storage_delete(struct bpf_map *map, struct bpf_sock *sk) | 2850 | * long bpf_sk_storage_delete(struct bpf_map *map, void *sk) |
| 2839 | * Description | 2851 | * Description |
| 2840 | * Delete a bpf-local-storage from a *sk*. | 2852 | * Delete a bpf-local-storage from a *sk*. |
| 2841 | * Return | 2853 | * Return |
| @@ -3395,6 +3407,175 @@ union bpf_attr { · 553 unmodified lines | |||
| 3395 | * A non-negative value equal to or less than *size* on success, | 3407 | * A non-negative value equal to or less than *size* on success, |
| 3396 | * or a negative error in case of failure. | 3408 | * or a negative error in case of failure. |
| 3397 | * | 3409 | * |
| 3410 | * long bpf_load_hdr_opt(struct bpf_sock_ops *skops, void *searchby_res, u32 len, u64 flags) | ||
| 3411 | * Description | ||
| 3412 | * Load header option. Support reading a particular TCP header | ||
| 3413 | * option for bpf program (BPF_PROG_TYPE_SOCK_OPS). | ||
| 3414 | * | ||
| 3415 | * If *flags* is 0, it will search the option from the | ||
| 3416 | * sock_ops->skb_data. The comment in "struct bpf_sock_ops" | ||
| 3417 | * has details on what skb_data contains under different | ||
| 3418 | * sock_ops->op. | ||
| 3419 | * | ||
| 3420 | * The first byte of the *searchby_res* specifies the | ||
| 3421 | * kind that it wants to search. | ||
| 3422 | * | ||
| 3423 | * If the searching kind is an experimental kind | ||
| 3424 | * (i.e. 253 or 254 according to RFC6994). It also | ||
| 3425 | * needs to specify the "magic" which is either | ||
| 3426 | * 2 bytes or 4 bytes. It then also needs to | ||
| 3427 | * specify the size of the magic by using | ||
| 3428 | * the 2nd byte which is "kind-length" of a TCP | ||
| 3429 | * header option and the "kind-length" also | ||
| 3430 | * includes the first 2 bytes "kind" and "kind-length" | ||
| 3431 | * itself as a normal TCP header option also does. | ||
| 3432 | * | ||
| 3433 | * For example, to search experimental kind 254 with | ||
| 3434 | * 2 byte magic 0xeB9F, the searchby_res should be | ||
| 3435 | * [ 254, 4, 0xeB, 0x9F, 0, 0, .... 0 ]. | ||
| 3436 | * | ||
| 3437 | * To search for the standard window scale option (3), | ||
| 3438 | * the searchby_res should be [ 3, 0, 0, .... 0 ]. | ||
| 3439 | * Note, kind-length must be 0 for regular option. | ||
| 3440 | * | ||
| 3441 | * Searching for No-Op (0) and End-of-Option-List (1) are | ||
| 3442 | * not supported. | ||
| 3443 | * | ||
| 3444 | * *len* must be at least 2 bytes which is the minimal size | ||
| 3445 | * of a header option. | ||
| 3446 | * | ||
| 3447 | * Supported flags: | ||
| 3448 | * * **BPF_LOAD_HDR_OPT_TCP_SYN** to search from the | ||
| 3449 | * saved_syn packet or the just-received syn packet. | ||
| 3450 | * | ||
| 3451 | * Return | ||
| 3452 | * >0 when found, the header option is copied to *searchby_res*. | ||
| 3453 | * The return value is the total length copied. | ||
| 3454 | * | ||
| 3455 | * **-EINVAL** If param is invalid | ||
| 3456 | * | ||
| 3457 | * **-ENOMSG** The option is not found | ||
| 3458 | * | ||
| 3459 | * **-ENOENT** No syn packet available when | ||
| 3460 | * **BPF_LOAD_HDR_OPT_TCP_SYN** is used | ||
| 3461 | * | ||
| 3462 | * **-ENOSPC** Not enough space. Only *len* number of | ||
| 3463 | * bytes are copied. | ||
| 3464 | * | ||
| 3465 | * **-EFAULT** Cannot parse the header options in the packet | ||
| 3466 | * | ||
| 3467 | * **-EPERM** This helper cannot be used under the | ||
| 3468 | * current sock_ops->op. | ||
| 3469 | * | ||
| 3470 | * long bpf_store_hdr_opt(struct bpf_sock_ops *skops, const void *from, u32 len, u64 flags) | ||
| 3471 | * Description | ||
| 3472 | * Store header option. The data will be copied | ||
| 3473 | * from buffer *from* with length *len* to the TCP header. | ||
| 3474 | * | ||
| 3475 | * The buffer *from* should have the whole option that | ||
| 3476 | * includes the kind, kind-length, and the actual | ||
| 3477 | * option data. The *len* must be at least kind-length | ||
| 3478 | * long. The kind-length does not have to be 4 byte | ||
| 3479 | * aligned. The kernel will take care of the padding | ||
| 3480 | * and setting the 4 bytes aligned value to th->doff. | ||
| 3481 | * | ||
| 3482 | * This helper will check for duplicated option | ||
| 3483 | * by searching the same option in the outgoing skb. | ||
| 3484 | * | ||
| 3485 | * This helper can only be called during | ||
| 3486 | * BPF_SOCK_OPS_WRITE_HDR_OPT_CB. | ||
| 3487 | * | ||
| 3488 | * Return | ||
| 3489 | * 0 on success, or negative error in case of failure: | ||
| 3490 | * | ||
| 3491 | * **-EINVAL** If param is invalid | ||
| 3492 | * | ||
| 3493 | * **-ENOSPC** Not enough space in the header. | ||
| 3494 | * Nothing has been written | ||
| 3495 | * | ||
| 3496 | * **-EEXIST** The option has already existed | ||
| 3497 | * | ||
| 3498 | * **-EFAULT** Cannot parse the existing header options | ||
| 3499 | * | ||
| 3500 | * **-EPERM** This helper cannot be used under the | ||
| 3501 | * current sock_ops->op. | ||
| 3502 | * | ||
| 3503 | * long bpf_reserve_hdr_opt(struct bpf_sock_ops *skops, u32 len, u64 flags) | ||
| 3504 | * Description | ||
| 3505 | * Reserve *len* bytes for the bpf header option. The | ||
| 3506 | * space will be used by bpf_store_hdr_opt() later in | ||
| 3507 | * BPF_SOCK_OPS_WRITE_HDR_OPT_CB. | ||
| 3508 | * | ||
| 3509 | * If bpf_reserve_hdr_opt() is called multiple times, | ||
| 3510 | * the total number of bytes will be reserved. | ||
| 3511 | * | ||
| 3512 | * This helper can only be called during | ||
| 3513 | * BPF_SOCK_OPS_HDR_OPT_LEN_CB. | ||
| 3514 | * | ||
| 3515 | * Return | ||
| 3516 | * 0 on success, or negative error in case of failure: | ||
| 3517 | * | ||
| 3518 | * **-EINVAL** if param is invalid | ||
| 3519 | * | ||
| 3520 | * **-ENOSPC** Not enough space in the header. | ||
| 3521 | * | ||
| 3522 | * **-EPERM** This helper cannot be used under the | ||
| 3523 | * current sock_ops->op. | ||
| 3524 | * | ||
| 3525 | * void *bpf_inode_storage_get(struct bpf_map *map, void *inode, void *value, u64 flags) | ||
| 3526 | * Description | ||
| 3527 | * Get a bpf_local_storage from an *inode*. | ||
| 3528 | * | ||
| 3529 | * Logically, it could be thought of as getting the value from | ||
| 3530 | * a *map* with *inode* as the **key**. From this | ||
| 3531 | * perspective, the usage is not much different from | ||
| 3532 | * **bpf_map_lookup_elem**\ (*map*, **&**\ *inode*) except this | ||
| 3533 | * helper enforces the key must be an inode and the map must also | ||
| 3534 | * be a **BPF_MAP_TYPE_INODE_STORAGE**. | ||
| 3535 | * | ||
| 3536 | * Underneath, the value is stored locally at *inode* instead of | ||
| 3537 | * the *map*. The *map* is used as the bpf-local-storage | ||
| 3538 | * "type". The bpf-local-storage "type" (i.e. the *map*) is | ||
| 3539 | * searched against all bpf_local_storage residing at *inode*. | ||
| 3540 | * | ||
| 3541 | * An optional *flags* (**BPF_LOCAL_STORAGE_GET_F_CREATE**) can be | ||
| 3542 | * used such that a new bpf_local_storage will be | ||
| 3543 | * created if one does not exist. *value* can be used | ||
| 3544 | * together with **BPF_LOCAL_STORAGE_GET_F_CREATE** to specify | ||
| 3545 | * the initial value of a bpf_local_storage. If *value* is | ||
| 3546 | * **NULL**, the new bpf_local_storage will be zero initialized. | ||
| 3547 | * Return | ||
| 3548 | * A bpf_local_storage pointer is returned on success. | ||
| 3549 | * | ||
| 3550 | * **NULL** if not found or there was an error in adding | ||
| 3551 | * a new bpf_local_storage. | ||
| 3552 | * | ||
| 3553 | * int bpf_inode_storage_delete(struct bpf_map *map, void *inode) | ||
| 3554 | * Description | ||
| 3555 | * Delete a bpf_local_storage from an *inode*. | ||
| 3556 | * Return | ||
| 3557 | * 0 on success. | ||
| 3558 | * | ||
| 3559 | * **-ENOENT** if the bpf_local_storage cannot be found. | ||
| 3560 | * | ||
| 3561 | * long bpf_d_path(struct path *path, char *buf, u32 sz) | ||
| 3562 | * Description | ||
| 3563 | * Return full path for given 'struct path' object, which | ||
| 3564 | * needs to be the kernel BTF 'path' object. The path is | ||
| 3565 | * returned in the provided buffer 'buf' of size 'sz' and | ||
| 3566 | * is zero terminated. | ||
| 3567 | * | ||
| 3568 | * Return | ||
| 3569 | * On success, the strictly positive length of the string, | ||
| 3570 | * including the trailing NUL character. On error, a negative | ||
| 3571 | * value. | ||
| 3572 | * | ||
| 3573 | * long bpf_copy_from_user(void *dst, u32 size, const void *user_ptr) | ||
| 3574 | * Description | ||
| 3575 | * Read *size* bytes from user space address *user_ptr* and store | ||
| 3576 | * the data in *dst*. This is a wrapper of copy_from_user(). | ||
| 3577 | * Return | ||
| 3578 | * 0 on success, or a negative error in case of failure. | ||
| 3398 | */ | 3579 | */ |
| 3399 | #define __BPF_FUNC_MAPPER(FN) \ | 3580 | #define __BPF_FUNC_MAPPER(FN) \ |
| 3400 | FN(unspec), \ | 3581 | FN(unspec), \ |
| @@ -3539,6 +3720,13 @@ union bpf_attr { · 138 unmodified lines | |||
| 3539 | FN(skc_to_tcp_request_sock), \ | 3720 | FN(skc_to_tcp_request_sock), \ |
| 3540 | FN(skc_to_udp6_sock), \ | 3721 | FN(skc_to_udp6_sock), \ |
| 3541 | FN(get_task_stack), \ | 3722 | FN(get_task_stack), \ |
| 3723 | FN(load_hdr_opt), \ | ||
| 3724 | FN(store_hdr_opt), \ | ||
| 3725 | FN(reserve_hdr_opt), \ | ||
| 3726 | FN(inode_storage_get), \ | ||
| 3727 | FN(inode_storage_delete), \ | ||
| 3728 | FN(d_path), \ | ||
| 3729 | FN(copy_from_user), \ | ||
| 3542 | /* */ | 3730 | /* */ |
| 3543 | 3731 | ||
| 3544 | /* integer value in 'imm' field of BPF_CALL instruction selects which helper | 3732 | /* integer value in 'imm' field of BPF_CALL instruction selects which helper |
| @@ -3648,9 +3836,13 @@ enum { · 103 unmodified lines | |||
| 3648 | BPF_F_SYSCTL_BASE_NAME = (1ULL << 0), | 3836 | BPF_F_SYSCTL_BASE_NAME = (1ULL << 0), |
| 3649 | }; | 3837 | }; |
| 3650 | 3838 | ||
| 3651 | /* BPF_FUNC_sk_storage_get flags */ | 3839 | /* BPF_FUNC_<kernel_obj>_storage_get flags */ |
| 3652 | enum { | 3840 | enum { |
| 3653 | BPF_SK_STORAGE_GET_F_CREATE = (1ULL << 0), | 3841 | BPF_LOCAL_STORAGE_GET_F_CREATE = (1ULL << 0), |
| 3842 | /* BPF_SK_STORAGE_GET_F_CREATE is only kept for backward compatibility | ||
| 3843 | * and BPF_LOCAL_STORAGE_GET_F_CREATE must be used instead. | ||
| 3844 | */ | ||
| 3845 | BPF_SK_STORAGE_GET_F_CREATE = BPF_LOCAL_STORAGE_GET_F_CREATE, | ||
| 3654 | }; | 3846 | }; |
| 3655 | 3847 | ||
| 3656 | /* BPF_FUNC_read_branch_records flags. */ | 3848 | /* BPF_FUNC_read_branch_records flags. */ |
| @@ -4071,6 +4263,15 @@ struct bpf_link_info { · 414 unmodified lines | |||
| 4071 | __u64 cgroup_id; | 4263 | __u64 cgroup_id; |
| 4072 | __u32 attach_type; | 4264 | __u32 attach_type; |
| 4073 | } cgroup; | 4265 | } cgroup; |
| 4266 | struct { | ||
| 4267 | __aligned_u64 target_name; /* in/out: target_name buffer ptr */ | ||
| 4268 | __u32 target_name_len; /* in/out: target_name buffer len */ | ||
| 4269 | union { | ||
| 4270 | struct { | ||
| 4271 | __u32 map_id; | ||
| 4272 | } map; | ||
| 4273 | }; | ||
| 4274 | } iter; | ||
| 4074 | struct { | 4275 | struct { |
| 4075 | __u32 netns_ino; | 4276 | __u32 netns_ino; |
| 4076 | __u32 attach_type; | 4277 | __u32 attach_type; |
| @@ -4158,6 +4359,36 @@ struct bpf_sock_ops { · 81 unmodified lines | |||
| 4158 | __u64 bytes_received; | 4359 | __u64 bytes_received; |
| 4159 | __u64 bytes_acked; | 4360 | __u64 bytes_acked; |
| 4160 | __bpf_md_ptr(struct bpf_sock *, sk); | 4361 | __bpf_md_ptr(struct bpf_sock *, sk); |
| 4362 | /* [skb_data, skb_data_end) covers the whole TCP header. | ||
| 4363 | * | ||
| 4364 | * BPF_SOCK_OPS_PARSE_HDR_OPT_CB: The packet received | ||
| 4365 | * BPF_SOCK_OPS_HDR_OPT_LEN_CB: Not useful because the | ||
| 4366 | * header has not been written. | ||
| 4367 | * BPF_SOCK_OPS_WRITE_HDR_OPT_CB: The header and options have | ||
| 4368 | * been written so far. | ||
| 4369 | * BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB: The SYNACK that concludes | ||
| 4370 | * the 3WHS. | ||
| 4371 | * BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB: The ACK that concludes | ||
| 4372 | * the 3WHS. | ||
| 4373 | * | ||
| 4374 | * bpf_load_hdr_opt() can also be used to read a particular option. | ||
| 4375 | */ | ||
| 4376 | __bpf_md_ptr(void *, skb_data); | ||
| 4377 | __bpf_md_ptr(void *, skb_data_end); | ||
| 4378 | __u32 skb_len; /* The total length of a packet. | ||
| 4379 | * It includes the header, options, | ||
| 4380 | * and payload. | ||
| 4381 | */ | ||
| 4382 | __u32 skb_tcp_flags; /* tcp_flags of the header. It provides | ||
| 4383 | * an easy way to check for tcp_flags | ||
| 4384 | * without parsing skb_data. | ||
| 4385 | * | ||
| 4386 | * In particular, the skb_tcp_flags | ||
| 4387 | * will still be available in | ||
| 4388 | * BPF_SOCK_OPS_HDR_OPT_LEN even though | ||
| 4389 | * the outgoing header has not | ||
| 4390 | * been written yet. | ||
| 4391 | */ | ||
| 4161 | }; | 4392 | }; |
| 4162 | 4393 | ||
| 4163 | /* Definitions for bpf_sock_ops_cb_flags */ | 4394 | /* Definitions for bpf_sock_ops_cb_flags */ |
| @@ -4166,8 +4397,51 @@ enum { · 2 unmodified lines | |||
| 4166 | BPF_SOCK_OPS_RETRANS_CB_FLAG = (1<<1), | 4397 | BPF_SOCK_OPS_RETRANS_CB_FLAG = (1<<1), |
| 4167 | BPF_SOCK_OPS_STATE_CB_FLAG = (1<<2), | 4398 | BPF_SOCK_OPS_STATE_CB_FLAG = (1<<2), |
| 4168 | BPF_SOCK_OPS_RTT_CB_FLAG = (1<<3), | 4399 | BPF_SOCK_OPS_RTT_CB_FLAG = (1<<3), |
| 4400 | /* Call bpf for all received TCP headers. The bpf prog will be | ||
| 4401 | * called under sock_ops->op == BPF_SOCK_OPS_PARSE_HDR_OPT_CB | ||
| 4402 | * | ||
| 4403 | * Please refer to the comment in BPF_SOCK_OPS_PARSE_HDR_OPT_CB | ||
| 4404 | * for the header option related helpers that will be useful | ||
| 4405 | * to the bpf programs. | ||
| 4406 | * | ||
| 4407 | * It could be used at the client/active side (i.e. connect() side) | ||
| 4408 | * when the server told it that the server was in syncookie | ||
| 4409 | * mode and required the active side to resend the bpf-written | ||
| 4410 | * options. The active side can keep writing the bpf-options until | ||
| 4411 | * it received a valid packet from the server side to confirm | ||
| 4412 | * the earlier packet (and options) has been received. The later | ||
| 4413 | * example patch is using it like this at the active side when the | ||
| 4414 | * server is in syncookie mode. | ||
| 4415 | * | ||
| 4416 | * The bpf prog will usually turn this off in the common cases. | ||
| 4417 | */ | ||
| 4418 | BPF_SOCK_OPS_PARSE_ALL_HDR_OPT_CB_FLAG = (1<<4), | ||
| 4419 | /* Call bpf when kernel has received a header option that | ||
| 4420 | * the kernel cannot handle. The bpf prog will be called under | ||
| 4421 | * sock_ops->op == BPF_SOCK_OPS_PARSE_HDR_OPT_CB. | ||
| 4422 | * | ||
| 4423 | * Please refer to the comment in BPF_SOCK_OPS_PARSE_HDR_OPT_CB | ||
| 4424 | * for the header option related helpers that will be useful | ||
| 4425 | * to the bpf programs. | ||
| 4426 | */ | ||
| 4427 | BPF_SOCK_OPS_PARSE_UNKNOWN_HDR_OPT_CB_FLAG = (1<<5), | ||
| 4428 | /* Call bpf when the kernel is writing header options for the | ||
| 4429 | * outgoing packet. The bpf prog will first be called | ||
| 4430 | * to reserve space in a skb under | ||
| 4431 | * sock_ops->op == BPF_SOCK_OPS_HDR_OPT_LEN_CB. Then | ||
| 4432 | * the bpf prog will be called to write the header option(s) | ||
| 4433 | * under sock_ops->op == BPF_SOCK_OPS_WRITE_HDR_OPT_CB. | ||
| 4434 | * | ||
| 4435 | * Please refer to the comment in BPF_SOCK_OPS_HDR_OPT_LEN_CB | ||
| 4436 | * and BPF_SOCK_OPS_WRITE_HDR_OPT_CB for the header option | ||
| 4437 | * related helpers that will be useful to the bpf programs. | ||
| 4438 | * | ||
| 4439 | * The kernel gets its chance to reserve space and write | ||
| 4440 | * options first before the BPF program does. | ||
| 4441 | */ | ||
| 4442 | BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG = (1<<6), | ||
| 4169 | /* Mask of all currently supported cb flags */ | 4443 | /* Mask of all currently supported cb flags */ |
| 4170 | BPF_SOCK_OPS_ALL_CB_FLAGS = 0xF, | 4444 | BPF_SOCK_OPS_ALL_CB_FLAGS = 0x7F, |
| 4171 | }; | 4445 | }; |
| 4172 | 4446 | ||
| 4173 | /* List of known BPF sock_ops operators. | 4447 | /* List of known BPF sock_ops operators. |
| @@ -4223,6 +4497,63 @@ enum { · 49 unmodified lines | |||
| 4223 | */ | 4497 | */ |
| 4224 | BPF_SOCK_OPS_RTT_CB, /* Called on every RTT. | 4498 | BPF_SOCK_OPS_RTT_CB, /* Called on every RTT. |
| 4225 | */ | 4499 | */ |
| 4500 | BPF_SOCK_OPS_PARSE_HDR_OPT_CB, /* Parse the header option. | ||
| 4501 | * It will be called to handle | ||
| 4502 | * the packets received at | ||
| 4503 | * an already established | ||
| 4504 | * connection. | ||
| 4505 | * | ||
| 4506 | * sock_ops->skb_data: | ||
| 4507 | * Referring to the received skb. | ||
| 4508 | * It covers the TCP header only. | ||
| 4509 | * | ||
| 4510 | * bpf_load_hdr_opt() can also | ||
| 4511 | * be used to search for a | ||
| 4512 | * particular option. | ||
| 4513 | */ | ||
| 4514 | BPF_SOCK_OPS_HDR_OPT_LEN_CB, /* Reserve space for writing the | ||
| 4515 | * header option later in | ||
| 4516 | * BPF_SOCK_OPS_WRITE_HDR_OPT_CB. | ||
| 4517 | * Arg1: bool want_cookie. (in | ||
| 4518 | * writing SYNACK only) | ||
| 4519 | * | ||
| 4520 | * sock_ops->skb_data: | ||
| 4521 | * Not available because no header has | ||
| 4522 | * been written yet. | ||
| 4523 | * | ||
| 4524 | * sock_ops->skb_tcp_flags: | ||
| 4525 | * The tcp_flags of the | ||
| 4526 | * outgoing skb. (e.g. SYN, ACK, FIN). | ||
| 4527 | * | ||
| 4528 | * bpf_reserve_hdr_opt() should | ||
| 4529 | * be used to reserve space. | ||
| 4530 | */ | ||
| 4531 | BPF_SOCK_OPS_WRITE_HDR_OPT_CB, /* Write the header options | ||
| 4532 | * Arg1: bool want_cookie. (in | ||
| 4533 | * writing SYNACK only) | ||
| 4534 | * | ||
| 4535 | * sock_ops->skb_data: | ||
| 4536 | * Referring to the outgoing skb. | ||
| 4537 | * It covers the TCP header | ||
| 4538 | * that has already been written | ||
| 4539 | * by the kernel and the | ||
| 4540 | * earlier bpf-progs. | ||
| 4541 | * | ||
| 4542 | * sock_ops->skb_tcp_flags: | ||
| 4543 | * The tcp_flags of the outgoing | ||
| 4544 | * skb. (e.g. SYN, ACK, FIN). | ||
| 4545 | * | ||
| 4546 | * bpf_store_hdr_opt() should | ||
| 4547 | * be used to write the | ||
| 4548 | * option. | ||
| 4549 | * | ||
| 4550 | * bpf_load_hdr_opt() can also | ||
| 4551 | * be used to search for a | ||
| 4552 | * particular option that | ||
| 4553 | * has already been written | ||
| 4554 | * by the kernel or the | ||
| 4555 | * earlier bpf-progs. | ||
| 4556 | */ | ||
| 4226 | }; | 4557 | }; |
| 4227 | 4558 | ||
| 4228 | /* List of TCP states. There is a build check in net/ipv4/tcp.c to detect | 4559 | /* List of TCP states. There is a build check in net/ipv4/tcp.c to detect |
| @@ -4250,6 +4581,63 @@ enum { · 21 unmodified lines | |||
| 4250 | enum { | 4581 | enum { |
| 4251 | TCP_BPF_IW = 1001, /* Set TCP initial congestion window */ | 4582 | TCP_BPF_IW = 1001, /* Set TCP initial congestion window */ |
| 4252 | TCP_BPF_SNDCWND_CLAMP = 1002, /* Set sndcwnd_clamp */ | 4583 | TCP_BPF_SNDCWND_CLAMP = 1002, /* Set sndcwnd_clamp */ |
| 4584 | TCP_BPF_DELACK_MAX = 1003, /* Max delay ack in usecs */ | ||
| 4585 | TCP_BPF_RTO_MIN = 1004, /* Min delay ack in usecs */ | ||
| 4586 | /* Copy the SYN pkt to optval | ||
| 4587 | * | ||
| 4588 | * BPF_PROG_TYPE_SOCK_OPS only. It is similar to the | ||
| 4589 | * bpf_getsockopt(TCP_SAVED_SYN) but it does not limit | ||
| 4590 | * to only getting from the saved_syn. It can either get the | ||
| 4591 | * syn packet from: | ||
| 4592 | * | ||
| 4593 | * 1. the just-received SYN packet (only available when writing the | ||
| 4594 | * SYNACK). It will be useful when it is not necessary to | ||
| 4595 | * save the SYN packet for latter use. It is also the only way | ||
| 4596 | * to get the SYN during syncookie mode because the syn | ||
| 4597 | * packet cannot be saved during syncookie. | ||
| 4598 | * | ||
| 4599 | * OR | ||
| 4600 | * | ||
| 4601 | * 2. the earlier saved syn which was done by | ||
| 4602 | * bpf_setsockopt(TCP_SAVE_SYN). | ||
| 4603 | * | ||
| 4604 | * The bpf_getsockopt(TCP_BPF_SYN*) option will hide where the | ||
| 4605 | * SYN packet is obtained. | ||
| 4606 | * | ||
| 4607 | * If the bpf-prog does not need the IP[46] header, the | ||
| 4608 | * bpf-prog can avoid parsing the IP header by using | ||
| 4609 | * TCP_BPF_SYN. Otherwise, the bpf-prog can get both | ||
| 4610 | * IP[46] and TCP header by using TCP_BPF_SYN_IP. | ||
| 4611 | * | ||
| 4612 | * >0: Total number of bytes copied | ||
| 4613 | * -ENOSPC: Not enough space in optval. Only optlen number of | ||
| 4614 | * bytes is copied. | ||
| 4615 | * -ENOENT: The SYN skb is not available now and the earlier SYN pkt | ||
| 4616 | * is not saved by setsockopt(TCP_SAVE_SYN). | ||
| 4617 | */ | ||
| 4618 | TCP_BPF_SYN = 1005, /* Copy the TCP header */ | ||
| 4619 | TCP_BPF_SYN_IP = 1006, /* Copy the IP[46] and TCP header */ | ||
| 4620 | TCP_BPF_SYN_MAC = 1007, /* Copy the MAC, IP[46], and TCP header */ | ||
| 4621 | }; | ||
| 4622 | |||
| 4623 | enum { | ||
| 4624 | BPF_LOAD_HDR_OPT_TCP_SYN = (1ULL << 0), | ||
| 4625 | }; | ||
| 4626 | |||
| 4627 | /* args[0] value during BPF_SOCK_OPS_HDR_OPT_LEN_CB and | ||
| 4628 | * BPF_SOCK_OPS_WRITE_HDR_OPT_CB. | ||
| 4629 | */ | ||
| 4630 | enum { | ||
| 4631 | BPF_WRITE_HDR_TCP_CURRENT_MSS = 1, /* Kernel is finding the | ||
| 4632 | * total option spaces | ||
| 4633 | * required for an established | ||
| 4634 | * sk in order to calculate the | ||
| 4635 | * MSS. No skb is actually | ||
| 4636 | * sent. | ||
| 4637 | */ | ||
| 4638 | BPF_WRITE_HDR_TCP_SYNACK_COOKIE = 2, /* Kernel is in syncookie mode | ||
| 4639 | * when sending a SYN. | ||
| 4640 | */ | ||
| 4253 | }; | 4641 | }; |
| 4254 | 4642 | ||
| 4255 | struct bpf_perf_event_value { | 4643 | struct bpf_perf_event_value { |