History

Merge branch 'sockmap: introduce BPF_SK_SKB_VERDICT and support UDP'

Cong Wang says: ==================== From: Cong Wang <cong.wang@bytedance.com> We have thousands of services connected to a daemon on every host via AF_UNIX dgram sockets, after they are moved into VM, we have to add a proxy to forward these communications from VM to host, because rewriting thousands of them is not practical. This proxy uses an AF_UNIX socket connected to services and a UDP socket to connect to the host. It is inefficient because data is copied between kernel space and user space twice, and we can not use splice() which only supports TCP. Therefore, we want to use sockmap to do the splicing without going to user-space at all (after the initial setup). Currently sockmap only fully supports TCP, UDP is partially supported as it is only allowed to add into sockmap. This patchset, as the second part of the original large patchset, extends sockmap with: 1) cross-protocol support with BPF_SK_SKB_VERDICT; 2) full UDP support. On the high level, ->read_sock() is required for each protocol to support sockmap redirection, and in order to do sock proto update, a new ops ->psock_update_sk_prot() is introduced, which is also required. And the BPF ->recvmsg() is also needed to replace the original ->recvmsg() to retrieve skmsg. To make life easier, we have to get rid of lock_sock() in sk_psock_handle_skb(), otherwise we would have to implement ->sendmsg_locked() on top of ->sendmsg(), which is ugly. Please see each patch for more details. To see the big picture, the original patchset is available here: https://github.com/congwang/linux/tree/sockmap this patchset is also available: https://github.com/congwang/linux/tree/sockmap2 --- v8: get rid of 'offset' in udp_read_sock() add checks for skb_verdict/stream_verdict conflict add two cleanup patches for sock_map_link() add a new test case v7: use work_mutex to protect psock->work return err in udp_read_sock() add patch 6/13 clean up test case v6: get rid of sk_psock_zap_ingress() add rcu work patch v5: use INDIRECT_CALL_2() for function pointers use ingress_lock to fix a race condition found by Jacub rename two helper functions v4: get rid of lock_sock() in sk_psock_handle_skb() get rid of udp_sendmsg_locked() remove an empty line update cover letter v3: export tcp/udp_update_proto() rename sk->sk_prot->psock_update_sk_prot() improve changelogs v2: separate from the original large patchset rebase to the latest bpf-next split UDP test case move inet_csk_has_ulp() check to tcp_bpf.c clean up udp_read_sock() ====================

Alexei Starovoitov committed 5 years ago

Commit 89d69c52 parents e27bfef 8d7cb7426 files · +677 −237Browse files

Unified · Split

26 files
  1. include/linux/skbuff.h +1 −0
  2. include/linux/skmsg.h +58 −19
  3. include/net/sock.h +3 −0
  4. include/net/tcp.h +1 −2
  5. include/net/udp.h +3 −0
  6. include/uapi/linux/bpf.h +1 −0
  7. kernel/bpf/syscall.c +1 −0
  8. net/core/skbuff.c +48 −7
  9. net/core/skmsg.c +143 −34
  10. net/core/sock_map.c +55 −63
  11. net/ipv4/af_inet.c +1 −0
  12. net/ipv4/tcp_bpf.c +23 −107
  13. net/ipv4/tcp_ipv4.c +3 −0
  14. net/ipv4/udp.c +32 −0
  15. net/ipv4/udp_bpf.c +76 −3
  16. net/ipv6/af_inet6.c +1 −0
  17. net/ipv6/tcp_ipv6.c +3 −0
  18. net/ipv6/udp.c +3 −0
  19. net/tls/tls_sw.c +2 −2
  20. tools/bpf/bpftool/common.c +1 −0
  21. tools/bpf/bpftool/prog.c +1 −0
  22. tools/include/uapi/linux/bpf.h +1 −0
  23. tools/testing/selftests/bpf/prog_tests/sockmap_basic.c +40 −0
  24. tools/testing/selftests/bpf/prog_tests/sockmap_listen.c +136 −0
  25. tools/testing/selftests/bpf/progs/test_sockmap_listen.c +22 −0
  26. tools/testing/selftests/bpf/progs/test_sockmap_skb_verdict_attach.c +18 −0
include/linux/skbuff.h+1−0
@@ -3626,6 +3626,7 @@ int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset, · 3625 unmodified lines
3626 unsigned int flags);3626 unsigned int flags);
3627int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,3627int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
3628 int len);3628 int len);
3629int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len);
3629void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to);3630void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to);
3630unsigned int skb_zerocopy_headlen(const struct sk_buff *from);3631unsigned int skb_zerocopy_headlen(const struct sk_buff *from);
3631int skb_zerocopy(struct sk_buff *to, struct sk_buff *from,3632int skb_zerocopy(struct sk_buff *to, struct sk_buff *from,
include/linux/skmsg.h+58−19
@@ -58,6 +58,7 @@ struct sk_psock_progs { · 57 unmodified lines
58 struct bpf_prog *msg_parser;58 struct bpf_prog *msg_parser;
59 struct bpf_prog *stream_parser;59 struct bpf_prog *stream_parser;
60 struct bpf_prog *stream_verdict;60 struct bpf_prog *stream_verdict;
61 struct bpf_prog *skb_verdict;
61};62};
62 63
63enum sk_psock_state_bits {64enum sk_psock_state_bits {
@@ -89,6 +90,7 @@ struct sk_psock { · 25 unmodified lines
89#endif90#endif
90 struct sk_buff_head ingress_skb;91 struct sk_buff_head ingress_skb;
91 struct list_head ingress_msg;92 struct list_head ingress_msg;
93 spinlock_t ingress_lock;
92 unsigned long state;94 unsigned long state;
93 struct list_head link;95 struct list_head link;
94 spinlock_t link_lock;96 spinlock_t link_lock;
@@ -97,13 +99,12 @@ struct sk_psock { · 2 unmodified lines
97 void (*saved_close)(struct sock *sk, long timeout);99 void (*saved_close)(struct sock *sk, long timeout);
98 void (*saved_write_space)(struct sock *sk);100 void (*saved_write_space)(struct sock *sk);
99 void (*saved_data_ready)(struct sock *sk);101 void (*saved_data_ready)(struct sock *sk);
102 int (*psock_update_sk_prot)(struct sock *sk, bool restore);
100 struct proto *sk_proto;103 struct proto *sk_proto;
104 struct mutex work_mutex;
101 struct sk_psock_work_state work_state;105 struct sk_psock_work_state work_state;
102 struct work_struct work;106 struct work_struct work;
103 union {107 struct rcu_work rwork;
104 struct rcu_head rcu;
105 struct work_struct gc;
106 };
107};108};
108 109
109int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,110int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,
@@ -124,6 +125,10 @@ int sk_msg_zerocopy_from_iter(struct sock *sk, struct iov_iter *from, · 14 unmodified lines
124 struct sk_msg *msg, u32 bytes);125 struct sk_msg *msg, u32 bytes);
125int sk_msg_memcopy_from_iter(struct sock *sk, struct iov_iter *from,126int sk_msg_memcopy_from_iter(struct sock *sk, struct iov_iter *from,
126 struct sk_msg *msg, u32 bytes);127 struct sk_msg *msg, u32 bytes);
128int sk_msg_wait_data(struct sock *sk, struct sk_psock *psock, int flags,
129 long timeo, int *err);
130int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,
131 int len, int flags);
127 132
128static inline void sk_msg_check_to_free(struct sk_msg *msg, u32 i, u32 bytes)133static inline void sk_msg_check_to_free(struct sk_msg *msg, u32 i, u32 bytes)
129{134{
@@ -284,7 +289,45 @@ static inline struct sk_psock *sk_psock(const struct sock *sk) · 154 unmodified lines
284static inline void sk_psock_queue_msg(struct sk_psock *psock,289static inline void sk_psock_queue_msg(struct sk_psock *psock,
285 struct sk_msg *msg)290 struct sk_msg *msg)
286{291{
292 spin_lock_bh(&psock->ingress_lock);
287 list_add_tail(&msg->list, &psock->ingress_msg);293 list_add_tail(&msg->list, &psock->ingress_msg);
294 spin_unlock_bh(&psock->ingress_lock);
295}
296
297static inline struct sk_msg *sk_psock_dequeue_msg(struct sk_psock *psock)
298{
299 struct sk_msg *msg;
300
301 spin_lock_bh(&psock->ingress_lock);
302 msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list);
303 if (msg)
304 list_del(&msg->list);
305 spin_unlock_bh(&psock->ingress_lock);
306 return msg;
307}
308
309static inline struct sk_msg *sk_psock_peek_msg(struct sk_psock *psock)
310{
311 struct sk_msg *msg;
312
313 spin_lock_bh(&psock->ingress_lock);
314 msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list);
315 spin_unlock_bh(&psock->ingress_lock);
316 return msg;
317}
318
319static inline struct sk_msg *sk_psock_next_msg(struct sk_psock *psock,
320 struct sk_msg *msg)
321{
322 struct sk_msg *ret;
323
324 spin_lock_bh(&psock->ingress_lock);
325 if (list_is_last(&msg->list, &psock->ingress_msg))
326 ret = NULL;
327 else
328 ret = list_next_entry(msg, list);
329 spin_unlock_bh(&psock->ingress_lock);
330 return ret;
288}331}
289 332
290static inline bool sk_psock_queue_empty(const struct sk_psock *psock)333static inline bool sk_psock_queue_empty(const struct sk_psock *psock)
@@ -292,6 +335,13 @@ static inline bool sk_psock_queue_empty(const struct sk_psock *psock) · 1 unmodified line
292 return psock ? list_empty(&psock->ingress_msg) : true;335 return psock ? list_empty(&psock->ingress_msg) : true;
293}336}
294 337
338static inline void kfree_sk_msg(struct sk_msg *msg)
339{
340 if (msg->skb)
341 consume_skb(msg->skb);
342 kfree(msg);
343}
344
295static inline void sk_psock_report_error(struct sk_psock *psock, int err)345static inline void sk_psock_report_error(struct sk_psock *psock, int err)
296{346{
297 struct sock *sk = psock->sk;347 struct sock *sk = psock->sk;
@@ -301,6 +351,7 @@ static inline void sk_psock_report_error(struct sk_psock *psock, int err) · 3 unmodified lines
301}351}
302 352
303struct sk_psock *sk_psock_init(struct sock *sk, int node);353struct sk_psock *sk_psock_init(struct sock *sk, int node);
354void sk_psock_stop(struct sk_psock *psock, bool wait);
304 355
305#if IS_ENABLED(CONFIG_BPF_STREAM_PARSER)356#if IS_ENABLED(CONFIG_BPF_STREAM_PARSER)
306int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock);357int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock);
@@ -349,25 +400,12 @@ static inline void sk_psock_cork_free(struct sk_psock *psock) · 42 unmodified lines
349 }400 }
350}401}
351 402
352static inline void sk_psock_update_proto(struct sock *sk,
353 struct sk_psock *psock,
354 struct proto *ops)
355{
356 /* Pairs with lockless read in sk_clone_lock() */
357 WRITE_ONCE(sk->sk_prot, ops);
358}
359
360static inline void sk_psock_restore_proto(struct sock *sk,403static inline void sk_psock_restore_proto(struct sock *sk,
361 struct sk_psock *psock)404 struct sk_psock *psock)
362{405{
363 sk->sk_prot->unhash = psock->saved_unhash;406 sk->sk_prot->unhash = psock->saved_unhash;
364 if (inet_csk_has_ulp(sk)) {407 if (psock->psock_update_sk_prot)
365 tcp_update_ulp(sk, psock->sk_proto, psock->saved_write_space);408 psock->psock_update_sk_prot(sk, true);
366 } else {
367 sk->sk_write_space = psock->saved_write_space;
368 /* Pairs with lockless read in sk_clone_lock() */
369 WRITE_ONCE(sk->sk_prot, psock->sk_proto);
370 }
371}409}
372 410
373static inline void sk_psock_set_state(struct sk_psock *psock,411static inline void sk_psock_set_state(struct sk_psock *psock,
@@ -442,6 +480,7 @@ static inline void psock_progs_drop(struct sk_psock_progs *progs) · 68 unmodified lines
442 psock_set_prog(&progs->msg_parser, NULL);480 psock_set_prog(&progs->msg_parser, NULL);
443 psock_set_prog(&progs->stream_parser, NULL);481 psock_set_prog(&progs->stream_parser, NULL);
444 psock_set_prog(&progs->stream_verdict, NULL);482 psock_set_prog(&progs->stream_verdict, NULL);
483 psock_set_prog(&progs->skb_verdict, NULL);
445}484}
446 485
447int sk_psock_tls_strp_read(struct sk_psock *psock, struct sk_buff *skb);486int sk_psock_tls_strp_read(struct sk_psock *psock, struct sk_buff *skb);
include/net/sock.h+3−0
@@ -1184,6 +1184,9 @@ struct proto { · 1183 unmodified lines
1184 void (*unhash)(struct sock *sk);1184 void (*unhash)(struct sock *sk);
1185 void (*rehash)(struct sock *sk);1185 void (*rehash)(struct sock *sk);
1186 int (*get_port)(struct sock *sk, unsigned short snum);1186 int (*get_port)(struct sock *sk, unsigned short snum);
1187#ifdef CONFIG_BPF_SYSCALL
1188 int (*psock_update_sk_prot)(struct sock *sk, bool restore);
1189#endif
1187 1190
1188 /* Keeping track of sockets in use */1191 /* Keeping track of sockets in use */
1189#ifdef CONFIG_PROC_FS1192#ifdef CONFIG_PROC_FS
include/net/tcp.h+1−2
@@ -2203,13 +2203,12 @@ struct sk_psock; · 2202 unmodified lines
2203 2203
2204#ifdef CONFIG_BPF_SYSCALL2204#ifdef CONFIG_BPF_SYSCALL
2205struct proto *tcp_bpf_get_proto(struct sock *sk, struct sk_psock *psock);2205struct proto *tcp_bpf_get_proto(struct sock *sk, struct sk_psock *psock);
2206int tcp_bpf_update_proto(struct sock *sk, bool restore);
2206void tcp_bpf_clone(const struct sock *sk, struct sock *newsk);2207void tcp_bpf_clone(const struct sock *sk, struct sock *newsk);
2207#endif /* CONFIG_BPF_SYSCALL */2208#endif /* CONFIG_BPF_SYSCALL */
2208 2209
2209int tcp_bpf_sendmsg_redir(struct sock *sk, struct sk_msg *msg, u32 bytes,2210int tcp_bpf_sendmsg_redir(struct sock *sk, struct sk_msg *msg, u32 bytes,
2210 int flags);2211 int flags);
2211int __tcp_bpf_recvmsg(struct sock *sk, struct sk_psock *psock,
2212 struct msghdr *msg, int len, int flags);
2213#endif /* CONFIG_NET_SOCK_MSG */2212#endif /* CONFIG_NET_SOCK_MSG */
2214 2213
2215#if !defined(CONFIG_BPF_SYSCALL) || !defined(CONFIG_NET_SOCK_MSG)2214#if !defined(CONFIG_BPF_SYSCALL) || !defined(CONFIG_NET_SOCK_MSG)
include/net/udp.h+3−0
@@ -329,6 +329,8 @@ struct sock *__udp6_lib_lookup(struct net *net, · 328 unmodified lines
329 struct sk_buff *skb);329 struct sk_buff *skb);
330struct sock *udp6_lib_lookup_skb(const struct sk_buff *skb,330struct sock *udp6_lib_lookup_skb(const struct sk_buff *skb,
331 __be16 sport, __be16 dport);331 __be16 sport, __be16 dport);
332int udp_read_sock(struct sock *sk, read_descriptor_t *desc,
333 sk_read_actor_t recv_actor);
332 334
333/* UDP uses skb->dev_scratch to cache as much information as possible and avoid335/* UDP uses skb->dev_scratch to cache as much information as possible and avoid
334 * possibly multiple cache miss on dequeue()336 * possibly multiple cache miss on dequeue()
@@ -518,6 +520,7 @@ static inline struct sk_buff *udp_rcv_segment(struct sock *sk, · 183 unmodified lines
518#ifdef CONFIG_BPF_SYSCALL520#ifdef CONFIG_BPF_SYSCALL
519struct sk_psock;521struct sk_psock;
520struct proto *udp_bpf_get_proto(struct sock *sk, struct sk_psock *psock);522struct proto *udp_bpf_get_proto(struct sock *sk, struct sk_psock *psock);
523int udp_bpf_update_proto(struct sock *sk, bool restore);
521#endif524#endif
522 525
523#endif /* _UDP_H */526#endif /* _UDP_H */
include/uapi/linux/bpf.h+1−0
@@ -957,6 +957,7 @@ enum bpf_attach_type { · 956 unmodified lines
957 BPF_XDP_CPUMAP,957 BPF_XDP_CPUMAP,
958 BPF_SK_LOOKUP,958 BPF_SK_LOOKUP,
959 BPF_XDP,959 BPF_XDP,
960 BPF_SK_SKB_VERDICT,
960 __MAX_BPF_ATTACH_TYPE961 __MAX_BPF_ATTACH_TYPE
961};962};
962 963
kernel/bpf/syscall.c+1−0
@@ -2948,6 +2948,7 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type) · 2947 unmodified lines
2948 return BPF_PROG_TYPE_SK_MSG;2948 return BPF_PROG_TYPE_SK_MSG;
2949 case BPF_SK_SKB_STREAM_PARSER:2949 case BPF_SK_SKB_STREAM_PARSER:
2950 case BPF_SK_SKB_STREAM_VERDICT:2950 case BPF_SK_SKB_STREAM_VERDICT:
2951 case BPF_SK_SKB_VERDICT:
2951 return BPF_PROG_TYPE_SK_SKB;2952 return BPF_PROG_TYPE_SK_SKB;
2952 case BPF_LIRC_MODE2:2953 case BPF_LIRC_MODE2:
2953 return BPF_PROG_TYPE_LIRC_MODE2;2954 return BPF_PROG_TYPE_LIRC_MODE2;
net/core/skbuff.c+48−7
@@ -2500,9 +2500,32 @@ int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset, · 2499 unmodified lines
2500}2500}
2501EXPORT_SYMBOL_GPL(skb_splice_bits);2501EXPORT_SYMBOL_GPL(skb_splice_bits);
2502 2502
2503/* Send skb data on a socket. Socket must be locked. */2503static int sendmsg_unlocked(struct sock *sk, struct msghdr *msg,
2504int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,2504 struct kvec *vec, size_t num, size_t size)
2505 int len)2505{
2506 struct socket *sock = sk->sk_socket;
2507
2508 if (!sock)
2509 return -EINVAL;
2510 return kernel_sendmsg(sock, msg, vec, num, size);
2511}
2512
2513static int sendpage_unlocked(struct sock *sk, struct page *page, int offset,
2514 size_t size, int flags)
2515{
2516 struct socket *sock = sk->sk_socket;
2517
2518 if (!sock)
2519 return -EINVAL;
2520 return kernel_sendpage(sock, page, offset, size, flags);
2521}
2522
2523typedef int (*sendmsg_func)(struct sock *sk, struct msghdr *msg,
2524 struct kvec *vec, size_t num, size_t size);
2525typedef int (*sendpage_func)(struct sock *sk, struct page *page, int offset,
2526 size_t size, int flags);
2527static int __skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset,
2528 int len, sendmsg_func sendmsg, sendpage_func sendpage)
2506{2529{
2507 unsigned int orig_len = len;2530 unsigned int orig_len = len;
2508 struct sk_buff *head = skb;2531 struct sk_buff *head = skb;
@@ -2522,7 +2545,8 @@ do_frag_list: · 13 unmodified lines
2522 memset(&msg, 0, sizeof(msg));2545 memset(&msg, 0, sizeof(msg));
2523 msg.msg_flags = MSG_DONTWAIT;2546 msg.msg_flags = MSG_DONTWAIT;
2524 2547
2525 ret = kernel_sendmsg_locked(sk, &msg, &kv, 1, slen);2548 ret = INDIRECT_CALL_2(sendmsg, kernel_sendmsg_locked,
2549 sendmsg_unlocked, sk, &msg, &kv, 1, slen);
2526 if (ret <= 0)2550 if (ret <= 0)
2527 goto error;2551 goto error;
2528 2552
@@ -2553,9 +2577,11 @@ do_frag_list: · 24 unmodified lines
2553 slen = min_t(size_t, len, skb_frag_size(frag) - offset);2577 slen = min_t(size_t, len, skb_frag_size(frag) - offset);
2554 2578
2555 while (slen) {2579 while (slen) {
2556 ret = kernel_sendpage_locked(sk, skb_frag_page(frag),2580 ret = INDIRECT_CALL_2(sendpage, kernel_sendpage_locked,
2557 skb_frag_off(frag) + offset,2581 sendpage_unlocked, sk,
2558 slen, MSG_DONTWAIT);2582 skb_frag_page(frag),
2583 skb_frag_off(frag) + offset,
2584 slen, MSG_DONTWAIT);
2559 if (ret <= 0)2585 if (ret <= 0)
2560 goto error;2586 goto error;
2561 2587
@@ -2587,8 +2613,23 @@ out: · 25 unmodified lines
2587error:2613error:
2588 return orig_len == len ? ret : orig_len - len;2614 return orig_len == len ? ret : orig_len - len;
2589}2615}
2616
2617/* Send skb data on a socket. Socket must be locked. */
2618int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
2619 int len)
2620{
2621 return __skb_send_sock(sk, skb, offset, len, kernel_sendmsg_locked,
2622 kernel_sendpage_locked);
2623}
2590EXPORT_SYMBOL_GPL(skb_send_sock_locked);2624EXPORT_SYMBOL_GPL(skb_send_sock_locked);
2591 2625
2626/* Send skb data on a socket. Socket must be unlocked. */
2627int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len)
2628{
2629 return __skb_send_sock(sk, skb, offset, len, sendmsg_unlocked,
2630 sendpage_unlocked);
2631}
2632
2592/**2633/**
2593 * skb_store_bits - store bits from kernel buffer to skb2634 * skb_store_bits - store bits from kernel buffer to skb
2594 * @skb: destination buffer2635 * @skb: destination buffer
net/core/skmsg.c+143−34
@@ -399,6 +399,104 @@ out: · 398 unmodified lines
399}399}
400EXPORT_SYMBOL_GPL(sk_msg_memcopy_from_iter);400EXPORT_SYMBOL_GPL(sk_msg_memcopy_from_iter);
401 401
402int sk_msg_wait_data(struct sock *sk, struct sk_psock *psock, int flags,
403 long timeo, int *err)
404{
405 DEFINE_WAIT_FUNC(wait, woken_wake_function);
406 int ret = 0;
407
408 if (sk->sk_shutdown & RCV_SHUTDOWN)
409 return 1;
410
411 if (!timeo)
412 return ret;
413
414 add_wait_queue(sk_sleep(sk), &wait);
415 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
416 ret = sk_wait_event(sk, &timeo,
417 !list_empty(&psock->ingress_msg) ||
418 !skb_queue_empty(&sk->sk_receive_queue), &wait);
419 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
420 remove_wait_queue(sk_sleep(sk), &wait);
421 return ret;
422}
423EXPORT_SYMBOL_GPL(sk_msg_wait_data);
424
425/* Receive sk_msg from psock->ingress_msg to @msg. */
426int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,
427 int len, int flags)
428{
429 struct iov_iter *iter = &msg->msg_iter;
430 int peek = flags & MSG_PEEK;
431 struct sk_msg *msg_rx;
432 int i, copied = 0;
433
434 msg_rx = sk_psock_peek_msg(psock);
435 while (copied != len) {
436 struct scatterlist *sge;
437
438 if (unlikely(!msg_rx))
439 break;
440
441 i = msg_rx->sg.start;
442 do {
443 struct page *page;
444 int copy;
445
446 sge = sk_msg_elem(msg_rx, i);
447 copy = sge->length;
448 page = sg_page(sge);
449 if (copied + copy > len)
450 copy = len - copied;
451 copy = copy_page_to_iter(page, sge->offset, copy, iter);
452 if (!copy)
453 return copied ? copied : -EFAULT;
454
455 copied += copy;
456 if (likely(!peek)) {
457 sge->offset += copy;
458 sge->length -= copy;
459 if (!msg_rx->skb)
460 sk_mem_uncharge(sk, copy);
461 msg_rx->sg.size -= copy;
462
463 if (!sge->length) {
464 sk_msg_iter_var_next(i);
465 if (!msg_rx->skb)
466 put_page(page);
467 }
468 } else {
469 /* Lets not optimize peek case if copy_page_to_iter
470 * didn't copy the entire length lets just break.
471 */
472 if (copy != sge->length)
473 return copied;
474 sk_msg_iter_var_next(i);
475 }
476
477 if (copied == len)
478 break;
479 } while (i != msg_rx->sg.end);
480
481 if (unlikely(peek)) {
482 msg_rx = sk_psock_next_msg(psock, msg_rx);
483 if (!msg_rx)
484 break;
485 continue;
486 }
487
488 msg_rx->sg.start = i;
489 if (!sge->length && msg_rx->sg.start == msg_rx->sg.end) {
490 msg_rx = sk_psock_dequeue_msg(psock);
491 kfree_sk_msg(msg_rx);
492 }
493 msg_rx = sk_psock_peek_msg(psock);
494 }
495
496 return copied;
497}
498EXPORT_SYMBOL_GPL(sk_msg_recvmsg);
499
402static struct sk_msg *sk_psock_create_ingress_msg(struct sock *sk,500static struct sk_msg *sk_psock_create_ingress_msg(struct sock *sk,
403 struct sk_buff *skb)501 struct sk_buff *skb)
404{502{
@@ -410,7 +508,7 @@ static struct sk_msg *sk_psock_create_ingress_msg(struct sock *sk, · 5 unmodified lines
410 if (!sk_rmem_schedule(sk, skb, skb->truesize))508 if (!sk_rmem_schedule(sk, skb, skb->truesize))
411 return NULL;509 return NULL;
412 510
413 msg = kzalloc(sizeof(*msg), __GFP_NOWARN | GFP_ATOMIC);511 msg = kzalloc(sizeof(*msg), __GFP_NOWARN | GFP_KERNEL);
414 if (unlikely(!msg))512 if (unlikely(!msg))
415 return NULL;513 return NULL;
416 514
@@ -497,7 +595,7 @@ static int sk_psock_handle_skb(struct sk_psock *psock, struct sk_buff *skb, · 80 unmodified lines
497 if (!ingress) {595 if (!ingress) {
498 if (!sock_writeable(psock->sk))596 if (!sock_writeable(psock->sk))
499 return -EAGAIN;597 return -EAGAIN;
500 return skb_send_sock_locked(psock->sk, skb, off, len);598 return skb_send_sock(psock->sk, skb, off, len);
501 }599 }
502 return sk_psock_skb_ingress(psock, skb);600 return sk_psock_skb_ingress(psock, skb);
503}601}
@@ -511,8 +609,7 @@ static void sk_psock_backlog(struct work_struct *work) · 7 unmodified lines
511 u32 len, off;609 u32 len, off;
512 int ret;610 int ret;
513 611
514 /* Lock sock to avoid losing sk_socket during loop. */612 mutex_lock(&psock->work_mutex);
515 lock_sock(psock->sk);
516 if (state->skb) {613 if (state->skb) {
517 skb = state->skb;614 skb = state->skb;
518 len = state->len;615 len = state->len;
@@ -529,7 +626,7 @@ start: · 10 unmodified lines
529 skb_bpf_redirect_clear(skb);626 skb_bpf_redirect_clear(skb);
530 do {627 do {
531 ret = -EIO;628 ret = -EIO;
532 if (likely(psock->sk->sk_socket))629 if (!sock_flag(psock->sk, SOCK_DEAD))
533 ret = sk_psock_handle_skb(psock, skb, off,630 ret = sk_psock_handle_skb(psock, skb, off,
534 len, ingress);631 len, ingress);
535 if (ret <= 0) {632 if (ret <= 0) {
@@ -553,7 +650,7 @@ start: · 17 unmodified lines
553 kfree_skb(skb);650 kfree_skb(skb);
554 }651 }
555end:652end:
556 release_sock(psock->sk);653 mutex_unlock(&psock->work_mutex);
557}654}
558 655
559struct sk_psock *sk_psock_init(struct sock *sk, int node)656struct sk_psock *sk_psock_init(struct sock *sk, int node)
@@ -563,11 +660,6 @@ struct sk_psock *sk_psock_init(struct sock *sk, int node) · 3 unmodified lines
563 660
564 write_lock_bh(&sk->sk_callback_lock);661 write_lock_bh(&sk->sk_callback_lock);
565 662
566 if (inet_csk_has_ulp(sk)) {
567 psock = ERR_PTR(-EINVAL);
568 goto out;
569 }
570
571 if (sk->sk_user_data) {663 if (sk->sk_user_data) {
572 psock = ERR_PTR(-EBUSY);664 psock = ERR_PTR(-EBUSY);
573 goto out;665 goto out;
@@ -591,7 +683,9 @@ struct sk_psock *sk_psock_init(struct sock *sk, int node) · 17 unmodified lines
591 spin_lock_init(&psock->link_lock);683 spin_lock_init(&psock->link_lock);
592 684
593 INIT_WORK(&psock->work, sk_psock_backlog);685 INIT_WORK(&psock->work, sk_psock_backlog);
686 mutex_init(&psock->work_mutex);
594 INIT_LIST_HEAD(&psock->ingress_msg);687 INIT_LIST_HEAD(&psock->ingress_msg);
688 spin_lock_init(&psock->ingress_lock);
595 skb_queue_head_init(&psock->ingress_skb);689 skb_queue_head_init(&psock->ingress_skb);
596 690
597 sk_psock_set_state(psock, SK_PSOCK_TX_ENABLED);691 sk_psock_set_state(psock, SK_PSOCK_TX_ENABLED);
@@ -630,11 +724,11 @@ static void __sk_psock_purge_ingress_msg(struct sk_psock *psock) · 32 unmodified lines
630 }724 }
631}725}
632 726
633static void sk_psock_zap_ingress(struct sk_psock *psock)727static void __sk_psock_zap_ingress(struct sk_psock *psock)
634{728{
635 struct sk_buff *skb;729 struct sk_buff *skb;
636 730
637 while ((skb = __skb_dequeue(&psock->ingress_skb)) != NULL) {731 while ((skb = skb_dequeue(&psock->ingress_skb)) != NULL) {
638 skb_bpf_redirect_clear(skb);732 skb_bpf_redirect_clear(skb);
639 kfree_skb(skb);733 kfree_skb(skb);
640 }734 }
@@ -651,23 +745,35 @@ static void sk_psock_link_destroy(struct sk_psock *psock) · 10 unmodified lines
651 }745 }
652}746}
653 747
748void sk_psock_stop(struct sk_psock *psock, bool wait)
749{
750 spin_lock_bh(&psock->ingress_lock);
751 sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);
752 sk_psock_cork_free(psock);
753 __sk_psock_zap_ingress(psock);
754 spin_unlock_bh(&psock->ingress_lock);
755
756 if (wait)
757 cancel_work_sync(&psock->work);
758}
759
654static void sk_psock_done_strp(struct sk_psock *psock);760static void sk_psock_done_strp(struct sk_psock *psock);
655 761
656static void sk_psock_destroy_deferred(struct work_struct *gc)762static void sk_psock_destroy(struct work_struct *work)
657{763{
658 struct sk_psock *psock = container_of(gc, struct sk_psock, gc);764 struct sk_psock *psock = container_of(to_rcu_work(work),
659 765 struct sk_psock, rwork);
660 /* No sk_callback_lock since already detached. */766 /* No sk_callback_lock since already detached. */
661 767
662 sk_psock_done_strp(psock);768 sk_psock_done_strp(psock);
663 769
664 cancel_work_sync(&psock->work);770 cancel_work_sync(&psock->work);
771 mutex_destroy(&psock->work_mutex);
665 772
666 psock_progs_drop(&psock->progs);773 psock_progs_drop(&psock->progs);
667 774
668 sk_psock_link_destroy(psock);775 sk_psock_link_destroy(psock);
669 sk_psock_cork_free(psock);776 sk_psock_cork_free(psock);
670 sk_psock_zap_ingress(psock);
671 777
672 if (psock->sk_redir)778 if (psock->sk_redir)
673 sock_put(psock->sk_redir);779 sock_put(psock->sk_redir);
@@ -675,30 +781,21 @@ static void sk_psock_destroy_deferred(struct work_struct *gc) · 1 unmodified line
675 kfree(psock);781 kfree(psock);
676}782}
677 783
678static void sk_psock_destroy(struct rcu_head *rcu)
679{
680 struct sk_psock *psock = container_of(rcu, struct sk_psock, rcu);
681
682 INIT_WORK(&psock->gc, sk_psock_destroy_deferred);
683 schedule_work(&psock->gc);
684}
685
686void sk_psock_drop(struct sock *sk, struct sk_psock *psock)784void sk_psock_drop(struct sock *sk, struct sk_psock *psock)
687{785{
688 sk_psock_cork_free(psock);786 sk_psock_stop(psock, false);
689 sk_psock_zap_ingress(psock);
690 787
691 write_lock_bh(&sk->sk_callback_lock);788 write_lock_bh(&sk->sk_callback_lock);
692 sk_psock_restore_proto(sk, psock);789 sk_psock_restore_proto(sk, psock);
693 rcu_assign_sk_user_data(sk, NULL);790 rcu_assign_sk_user_data(sk, NULL);
694 if (psock->progs.stream_parser)791 if (psock->progs.stream_parser)
695 sk_psock_stop_strp(sk, psock);792 sk_psock_stop_strp(sk, psock);
696 else if (psock->progs.stream_verdict)793 else if (psock->progs.stream_verdict || psock->progs.skb_verdict)
697 sk_psock_stop_verdict(sk, psock);794 sk_psock_stop_verdict(sk, psock);
698 write_unlock_bh(&sk->sk_callback_lock);795 write_unlock_bh(&sk->sk_callback_lock);
699 sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);
700 796
701 call_rcu(&psock->rcu, sk_psock_destroy);797 INIT_RCU_WORK(&psock->rwork, sk_psock_destroy);
798 queue_rcu_work(system_wq, &psock->rwork);
702}799}
703EXPORT_SYMBOL_GPL(sk_psock_drop);800EXPORT_SYMBOL_GPL(sk_psock_drop);
704 801
@@ -767,14 +864,20 @@ static void sk_psock_skb_redirect(struct sk_buff *skb) · 62 unmodified lines
767 * error that caused the pipe to break. We can't send a packet on864 * error that caused the pipe to break. We can't send a packet on
768 * a socket that is in this state so we drop the skb.865 * a socket that is in this state so we drop the skb.
769 */866 */
770 if (!psock_other || sock_flag(sk_other, SOCK_DEAD) ||867 if (!psock_other || sock_flag(sk_other, SOCK_DEAD)) {
771 !sk_psock_test_state(psock_other, SK_PSOCK_TX_ENABLED)) {868 kfree_skb(skb);
869 return;
870 }
871 spin_lock_bh(&psock_other->ingress_lock);
872 if (!sk_psock_test_state(psock_other, SK_PSOCK_TX_ENABLED)) {
873 spin_unlock_bh(&psock_other->ingress_lock);
772 kfree_skb(skb);874 kfree_skb(skb);
773 return;875 return;
774 }876 }
775 877
776 skb_queue_tail(&psock_other->ingress_skb, skb);878 skb_queue_tail(&psock_other->ingress_skb, skb);
777 schedule_work(&psock_other->work);879 schedule_work(&psock_other->work);
880 spin_unlock_bh(&psock_other->ingress_lock);
778}881}
779 882
780static void sk_psock_tls_verdict_apply(struct sk_buff *skb, struct sock *sk, int verdict)883static void sk_psock_tls_verdict_apply(struct sk_buff *skb, struct sock *sk, int verdict)
@@ -842,8 +945,12 @@ static void sk_psock_verdict_apply(struct sk_psock *psock, · 61 unmodified lines
842 err = sk_psock_skb_ingress_self(psock, skb);945 err = sk_psock_skb_ingress_self(psock, skb);
843 }946 }
844 if (err < 0) {947 if (err < 0) {
845 skb_queue_tail(&psock->ingress_skb, skb);948 spin_lock_bh(&psock->ingress_lock);
846 schedule_work(&psock->work);949 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) {
950 skb_queue_tail(&psock->ingress_skb, skb);
951 schedule_work(&psock->work);
952 }
953 spin_unlock_bh(&psock->ingress_lock);
847 }954 }
848 break;955 break;
849 case __SK_REDIRECT:956 case __SK_REDIRECT:
@@ -1010,6 +1117,8 @@ static int sk_psock_verdict_recv(read_descriptor_t *desc, struct sk_buff *skb, · 160 unmodified lines
1010 }1117 }
1011 skb_set_owner_r(skb, sk);1118 skb_set_owner_r(skb, sk);
1012 prog = READ_ONCE(psock->progs.stream_verdict);1119 prog = READ_ONCE(psock->progs.stream_verdict);
1120 if (!prog)
1121 prog = READ_ONCE(psock->progs.skb_verdict);
1013 if (likely(prog)) {1122 if (likely(prog)) {
1014 skb_dst_drop(skb);1123 skb_dst_drop(skb);
1015 skb_bpf_redirect_clear(skb);1124 skb_bpf_redirect_clear(skb);
net/core/sock_map.c+55−63
@@ -26,6 +26,7 @@ struct bpf_stab { · 25 unmodified lines
26 26
27static int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog,27static int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog,
28 struct bpf_prog *old, u32 which);28 struct bpf_prog *old, u32 which);
29static struct sk_psock_progs *sock_map_progs(struct bpf_map *map);
29 30
30static struct bpf_map *sock_map_alloc(union bpf_attr *attr)31static struct bpf_map *sock_map_alloc(union bpf_attr *attr)
31{32{
@@ -155,6 +156,8 @@ static void sock_map_del_link(struct sock *sk, · 123 unmodified lines
155 strp_stop = true;156 strp_stop = true;
156 if (psock->saved_data_ready && stab->progs.stream_verdict)157 if (psock->saved_data_ready && stab->progs.stream_verdict)
157 verdict_stop = true;158 verdict_stop = true;
159 if (psock->saved_data_ready && stab->progs.skb_verdict)
160 verdict_stop = true;
158 list_del(&link->list);161 list_del(&link->list);
159 sk_psock_free_link(link);162 sk_psock_free_link(link);
160 }163 }
@@ -182,26 +185,10 @@ static void sock_map_unref(struct sock *sk, void *link_raw) · 21 unmodified lines
182 185
183static int sock_map_init_proto(struct sock *sk, struct sk_psock *psock)186static int sock_map_init_proto(struct sock *sk, struct sk_psock *psock)
184{187{
185 struct proto *prot;188 if (!sk->sk_prot->psock_update_sk_prot)
186
187 switch (sk->sk_type) {
188 case SOCK_STREAM:
189 prot = tcp_bpf_get_proto(sk, psock);
190 break;
191
192 case SOCK_DGRAM:
193 prot = udp_bpf_get_proto(sk, psock);
194 break;
195
196 default:
197 return -EINVAL;189 return -EINVAL;
198 }190 psock->psock_update_sk_prot = sk->sk_prot->psock_update_sk_prot;
199 191 return sk->sk_prot->psock_update_sk_prot(sk, false);
200 if (IS_ERR(prot))
201 return PTR_ERR(prot);
202
203 sk_psock_update_proto(sk, psock, prot);
204 return 0;
205}192}
206 193
207static struct sk_psock *sock_map_psock_get_checked(struct sock *sk)194static struct sk_psock *sock_map_psock_get_checked(struct sock *sk)
@@ -224,13 +211,25 @@ out: · 16 unmodified lines
224 return psock;211 return psock;
225}212}
226 213
227static int sock_map_link(struct bpf_map *map, struct sk_psock_progs *progs,214static bool sock_map_redirect_allowed(const struct sock *sk);
228 struct sock *sk)215
216static int sock_map_link(struct bpf_map *map, struct sock *sk)
229{217{
230 struct bpf_prog *msg_parser, *stream_parser, *stream_verdict;218 struct sk_psock_progs *progs = sock_map_progs(map);
219 struct bpf_prog *stream_verdict = NULL;
220 struct bpf_prog *stream_parser = NULL;
221 struct bpf_prog *skb_verdict = NULL;
222 struct bpf_prog *msg_parser = NULL;
231 struct sk_psock *psock;223 struct sk_psock *psock;
232 int ret;224 int ret;
233 225
226 /* Only sockets we can redirect into/from in BPF need to hold
227 * refs to parser/verdict progs and have their sk_data_ready
228 * and sk_write_space callbacks overridden.
229 */
230 if (!sock_map_redirect_allowed(sk))
231 goto no_progs;
232
234 stream_verdict = READ_ONCE(progs->stream_verdict);233 stream_verdict = READ_ONCE(progs->stream_verdict);
235 if (stream_verdict) {234 if (stream_verdict) {
236 stream_verdict = bpf_prog_inc_not_zero(stream_verdict);235 stream_verdict = bpf_prog_inc_not_zero(stream_verdict);
@@ -256,6 +255,16 @@ static int sock_map_link(struct bpf_map *map, struct sk_psock_progs *progs, · 19 unmodified lines
256 }255 }
257 }256 }
258 257
258 skb_verdict = READ_ONCE(progs->skb_verdict);
259 if (skb_verdict) {
260 skb_verdict = bpf_prog_inc_not_zero(skb_verdict);
261 if (IS_ERR(skb_verdict)) {
262 ret = PTR_ERR(skb_verdict);
263 goto out_put_msg_parser;
264 }
265 }
266
267no_progs:
259 psock = sock_map_psock_get_checked(sk);268 psock = sock_map_psock_get_checked(sk);
260 if (IS_ERR(psock)) {269 if (IS_ERR(psock)) {
261 ret = PTR_ERR(psock);270 ret = PTR_ERR(psock);
@@ -265,6 +274,9 @@ static int sock_map_link(struct bpf_map *map, struct sk_psock_progs *progs, · 3 unmodified lines
265 if (psock) {274 if (psock) {
266 if ((msg_parser && READ_ONCE(psock->progs.msg_parser)) ||275 if ((msg_parser && READ_ONCE(psock->progs.msg_parser)) ||
267 (stream_parser && READ_ONCE(psock->progs.stream_parser)) ||276 (stream_parser && READ_ONCE(psock->progs.stream_parser)) ||
277 (skb_verdict && READ_ONCE(psock->progs.skb_verdict)) ||
278 (skb_verdict && READ_ONCE(psock->progs.stream_verdict)) ||
279 (stream_verdict && READ_ONCE(psock->progs.skb_verdict)) ||
268 (stream_verdict && READ_ONCE(psock->progs.stream_verdict))) {280 (stream_verdict && READ_ONCE(psock->progs.stream_verdict))) {
269 sk_psock_put(sk, psock);281 sk_psock_put(sk, psock);
270 ret = -EBUSY;282 ret = -EBUSY;
@@ -296,6 +308,9 @@ static int sock_map_link(struct bpf_map *map, struct sk_psock_progs *progs, · 25 unmodified lines
296 } else if (!stream_parser && stream_verdict && !psock->saved_data_ready) {308 } else if (!stream_parser && stream_verdict && !psock->saved_data_ready) {
297 psock_set_prog(&psock->progs.stream_verdict, stream_verdict);309 psock_set_prog(&psock->progs.stream_verdict, stream_verdict);
298 sk_psock_start_verdict(sk,psock);310 sk_psock_start_verdict(sk,psock);
311 } else if (!stream_verdict && skb_verdict && !psock->saved_data_ready) {
312 psock_set_prog(&psock->progs.skb_verdict, skb_verdict);
313 sk_psock_start_verdict(sk, psock);
299 }314 }
300 write_unlock_bh(&sk->sk_callback_lock);315 write_unlock_bh(&sk->sk_callback_lock);
301 return 0;316 return 0;
@@ -304,6 +319,9 @@ out_unlock_drop: · 2 unmodified lines
304out_drop:319out_drop:
305 sk_psock_put(sk, psock);320 sk_psock_put(sk, psock);
306out_progs:321out_progs:
322 if (skb_verdict)
323 bpf_prog_put(skb_verdict);
324out_put_msg_parser:
307 if (msg_parser)325 if (msg_parser)
308 bpf_prog_put(msg_parser);326 bpf_prog_put(msg_parser);
309out_put_stream_parser:327out_put_stream_parser:
@@ -315,27 +333,6 @@ out_put_stream_verdict: · 5 unmodified lines
315 return ret;333 return ret;
316}334}
317 335
318static int sock_map_link_no_progs(struct bpf_map *map, struct sock *sk)
319{
320 struct sk_psock *psock;
321 int ret;
322
323 psock = sock_map_psock_get_checked(sk);
324 if (IS_ERR(psock))
325 return PTR_ERR(psock);
326
327 if (!psock) {
328 psock = sk_psock_init(sk, map->numa_node);
329 if (IS_ERR(psock))
330 return PTR_ERR(psock);
331 }
332
333 ret = sock_map_init_proto(sk, psock);
334 if (ret < 0)
335 sk_psock_put(sk, psock);
336 return ret;
337}
338
339static void sock_map_free(struct bpf_map *map)336static void sock_map_free(struct bpf_map *map)
340{337{
341 struct bpf_stab *stab = container_of(map, struct bpf_stab, map);338 struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
@@ -466,8 +463,6 @@ static int sock_map_get_next_key(struct bpf_map *map, void *key, void *next) · 124 unmodified lines
466 return 0;463 return 0;
467}464}
468 465
469static bool sock_map_redirect_allowed(const struct sock *sk);
470
471static int sock_map_update_common(struct bpf_map *map, u32 idx,466static int sock_map_update_common(struct bpf_map *map, u32 idx,
472 struct sock *sk, u64 flags)467 struct sock *sk, u64 flags)
473{468{
@@ -487,14 +482,7 @@ static int sock_map_update_common(struct bpf_map *map, u32 idx, · 13 unmodified lines
487 if (!link)482 if (!link)
488 return -ENOMEM;483 return -ENOMEM;
489 484
490 /* Only sockets we can redirect into/from in BPF need to hold485 ret = sock_map_link(map, sk);
491 * refs to parser/verdict progs and have their sk_data_ready
492 * and sk_write_space callbacks overridden.
493 */
494 if (sock_map_redirect_allowed(sk))
495 ret = sock_map_link(map, &stab->progs, sk);
496 else
497 ret = sock_map_link_no_progs(map, sk);
498 if (ret < 0)486 if (ret < 0)
499 goto out_free;487 goto out_free;
500 488
@@ -547,12 +535,15 @@ static bool sk_is_udp(const struct sock *sk) · 46 unmodified lines
547 535
548static bool sock_map_redirect_allowed(const struct sock *sk)536static bool sock_map_redirect_allowed(const struct sock *sk)
549{537{
550 return sk_is_tcp(sk) && sk->sk_state != TCP_LISTEN;538 if (sk_is_tcp(sk))
539 return sk->sk_state != TCP_LISTEN;
540 else
541 return sk->sk_state == TCP_ESTABLISHED;
551}542}
552 543
553static bool sock_map_sk_is_suitable(const struct sock *sk)544static bool sock_map_sk_is_suitable(const struct sock *sk)
554{545{
555 return sk_is_tcp(sk) || sk_is_udp(sk);546 return !!sk->sk_prot->psock_update_sk_prot;
556}547}
557 548
558static bool sock_map_sk_state_allowed(const struct sock *sk)549static bool sock_map_sk_state_allowed(const struct sock *sk)
@@ -999,14 +990,7 @@ static int sock_hash_update_common(struct bpf_map *map, void *key, · 440 unmodified lines
999 if (!link)990 if (!link)
1000 return -ENOMEM;991 return -ENOMEM;
1001 992
1002 /* Only sockets we can redirect into/from in BPF need to hold993 ret = sock_map_link(map, sk);
1003 * refs to parser/verdict progs and have their sk_data_ready
1004 * and sk_write_space callbacks overridden.
1005 */
1006 if (sock_map_redirect_allowed(sk))
1007 ret = sock_map_link(map, &htab->progs, sk);
1008 else
1009 ret = sock_map_link_no_progs(map, sk);
1010 if (ret < 0)994 if (ret < 0)
1011 goto out_free;995 goto out_free;
1012 996
@@ -1466,8 +1450,15 @@ static int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog, · 453 unmodified lines
1466 break;1450 break;
1467#endif1451#endif
1468 case BPF_SK_SKB_STREAM_VERDICT:1452 case BPF_SK_SKB_STREAM_VERDICT:
1453 if (progs->skb_verdict)
1454 return -EBUSY;
1469 pprog = &progs->stream_verdict;1455 pprog = &progs->stream_verdict;
1470 break;1456 break;
1457 case BPF_SK_SKB_VERDICT:
1458 if (progs->stream_verdict)
1459 return -EBUSY;
1460 pprog = &progs->skb_verdict;
1461 break;
1471 default:1462 default:
1472 return -EOPNOTSUPP;1463 return -EOPNOTSUPP;
1473 }1464 }
@@ -1540,6 +1531,7 @@ void sock_map_close(struct sock *sk, long timeout) · 66 unmodified lines
1540 saved_close = psock->saved_close;1531 saved_close = psock->saved_close;
1541 sock_map_remove_links(sk, psock);1532 sock_map_remove_links(sk, psock);
1542 rcu_read_unlock();1533 rcu_read_unlock();
1534 sk_psock_stop(psock, true);
1543 release_sock(sk);1535 release_sock(sk);
1544 saved_close(sk, timeout);1536 saved_close(sk, timeout);
1545}1537}
net/ipv4/af_inet.c+1−0
@@ -1070,6 +1070,7 @@ const struct proto_ops inet_dgram_ops = { · 1069 unmodified lines
1070 .setsockopt = sock_common_setsockopt,1070 .setsockopt = sock_common_setsockopt,
1071 .getsockopt = sock_common_getsockopt,1071 .getsockopt = sock_common_getsockopt,
1072 .sendmsg = inet_sendmsg,1072 .sendmsg = inet_sendmsg,
1073 .read_sock = udp_read_sock,
1073 .recvmsg = inet_recvmsg,1074 .recvmsg = inet_recvmsg,
1074 .mmap = sock_no_mmap,1075 .mmap = sock_no_mmap,
1075 .sendpage = inet_sendpage,1076 .sendpage = inet_sendpage,
net/ipv4/tcp_bpf.c+23−107
@@ -10,86 +10,6 @@ · 9 unmodified lines
10#include <net/inet_common.h>10#include <net/inet_common.h>
11#include <net/tls.h>11#include <net/tls.h>
12 12
13int __tcp_bpf_recvmsg(struct sock *sk, struct sk_psock *psock,
14 struct msghdr *msg, int len, int flags)
15{
16 struct iov_iter *iter = &msg->msg_iter;
17 int peek = flags & MSG_PEEK;
18 struct sk_msg *msg_rx;
19 int i, copied = 0;
20
21 msg_rx = list_first_entry_or_null(&psock->ingress_msg,
22 struct sk_msg, list);
23
24 while (copied != len) {
25 struct scatterlist *sge;
26
27 if (unlikely(!msg_rx))
28 break;
29
30 i = msg_rx->sg.start;
31 do {
32 struct page *page;
33 int copy;
34
35 sge = sk_msg_elem(msg_rx, i);
36 copy = sge->length;
37 page = sg_page(sge);
38 if (copied + copy > len)
39 copy = len - copied;
40 copy = copy_page_to_iter(page, sge->offset, copy, iter);
41 if (!copy)
42 return copied ? copied : -EFAULT;
43
44 copied += copy;
45 if (likely(!peek)) {
46 sge->offset += copy;
47 sge->length -= copy;
48 if (!msg_rx->skb)
49 sk_mem_uncharge(sk, copy);
50 msg_rx->sg.size -= copy;
51
52 if (!sge->length) {
53 sk_msg_iter_var_next(i);
54 if (!msg_rx->skb)
55 put_page(page);
56 }
57 } else {
58 /* Lets not optimize peek case if copy_page_to_iter
59 * didn't copy the entire length lets just break.
60 */
61 if (copy != sge->length)
62 return copied;
63 sk_msg_iter_var_next(i);
64 }
65
66 if (copied == len)
67 break;
68 } while (i != msg_rx->sg.end);
69
70 if (unlikely(peek)) {
71 if (msg_rx == list_last_entry(&psock->ingress_msg,
72 struct sk_msg, list))
73 break;
74 msg_rx = list_next_entry(msg_rx, list);
75 continue;
76 }
77
78 msg_rx->sg.start = i;
79 if (!sge->length && msg_rx->sg.start == msg_rx->sg.end) {
80 list_del(&msg_rx->list);
81 if (msg_rx->skb)
82 consume_skb(msg_rx->skb);
83 kfree(msg_rx);
84 }
85 msg_rx = list_first_entry_or_null(&psock->ingress_msg,
86 struct sk_msg, list);
87 }
88
89 return copied;
90}
91EXPORT_SYMBOL_GPL(__tcp_bpf_recvmsg);
92
93static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock,13static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock,
94 struct sk_msg *msg, u32 apply_bytes, int flags)14 struct sk_msg *msg, u32 apply_bytes, int flags)
95{15{
@@ -243,28 +163,6 @@ static bool tcp_bpf_stream_read(const struct sock *sk) · 147 unmodified lines
243 return !empty;163 return !empty;
244}164}
245 165
246static int tcp_bpf_wait_data(struct sock *sk, struct sk_psock *psock,
247 int flags, long timeo, int *err)
248{
249 DEFINE_WAIT_FUNC(wait, woken_wake_function);
250 int ret = 0;
251
252 if (sk->sk_shutdown & RCV_SHUTDOWN)
253 return 1;
254
255 if (!timeo)
256 return ret;
257
258 add_wait_queue(sk_sleep(sk), &wait);
259 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
260 ret = sk_wait_event(sk, &timeo,
261 !list_empty(&psock->ingress_msg) ||
262 !skb_queue_empty(&sk->sk_receive_queue), &wait);
263 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
264 remove_wait_queue(sk_sleep(sk), &wait);
265 return ret;
266}
267
268static int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,166static int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
269 int nonblock, int flags, int *addr_len)167 int nonblock, int flags, int *addr_len)
270{168{
@@ -284,13 +182,13 @@ static int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, · 13 unmodified lines
284 }182 }
285 lock_sock(sk);183 lock_sock(sk);
286msg_bytes_ready:184msg_bytes_ready:
287 copied = __tcp_bpf_recvmsg(sk, psock, msg, len, flags);185 copied = sk_msg_recvmsg(sk, psock, msg, len, flags);
288 if (!copied) {186 if (!copied) {
289 int data, err = 0;187 int data, err = 0;
290 long timeo;188 long timeo;
291 189
292 timeo = sock_rcvtimeo(sk, nonblock);190 timeo = sock_rcvtimeo(sk, nonblock);
293 data = tcp_bpf_wait_data(sk, psock, flags, timeo, &err);191 data = sk_msg_wait_data(sk, psock, flags, timeo, &err);
294 if (data) {192 if (data) {
295 if (!sk_psock_queue_empty(psock))193 if (!sk_psock_queue_empty(psock))
296 goto msg_bytes_ready;194 goto msg_bytes_ready;
@@ -601,20 +499,38 @@ static int tcp_bpf_assert_proto_ops(struct proto *ops) · 304 unmodified lines
601 ops->sendpage == tcp_sendpage ? 0 : -ENOTSUPP;499 ops->sendpage == tcp_sendpage ? 0 : -ENOTSUPP;
602}500}
603 501
604struct proto *tcp_bpf_get_proto(struct sock *sk, struct sk_psock *psock)502int tcp_bpf_update_proto(struct sock *sk, bool restore)
605{503{
504 struct sk_psock *psock = sk_psock(sk);
606 int family = sk->sk_family == AF_INET6 ? TCP_BPF_IPV6 : TCP_BPF_IPV4;505 int family = sk->sk_family == AF_INET6 ? TCP_BPF_IPV6 : TCP_BPF_IPV4;
607 int config = psock->progs.msg_parser ? TCP_BPF_TX : TCP_BPF_BASE;506 int config = psock->progs.msg_parser ? TCP_BPF_TX : TCP_BPF_BASE;
608 507
508 if (restore) {
509 if (inet_csk_has_ulp(sk)) {
510 tcp_update_ulp(sk, psock->sk_proto, psock->saved_write_space);
511 } else {
512 sk->sk_write_space = psock->saved_write_space;
513 /* Pairs with lockless read in sk_clone_lock() */
514 WRITE_ONCE(sk->sk_prot, psock->sk_proto);
515 }
516 return 0;
517 }
518
519 if (inet_csk_has_ulp(sk))
520 return -EINVAL;
521
609 if (sk->sk_family == AF_INET6) {522 if (sk->sk_family == AF_INET6) {
610 if (tcp_bpf_assert_proto_ops(psock->sk_proto))523 if (tcp_bpf_assert_proto_ops(psock->sk_proto))
611 return ERR_PTR(-EINVAL);524 return -EINVAL;
612 525
613 tcp_bpf_check_v6_needs_rebuild(psock->sk_proto);526 tcp_bpf_check_v6_needs_rebuild(psock->sk_proto);
614 }527 }
615 528
616 return &tcp_bpf_prots[family][config];529 /* Pairs with lockless read in sk_clone_lock() */
530 WRITE_ONCE(sk->sk_prot, &tcp_bpf_prots[family][config]);
531 return 0;
617}532}
533EXPORT_SYMBOL_GPL(tcp_bpf_update_proto);
618 534
619/* If a child got cloned from a listening socket that had tcp_bpf535/* If a child got cloned from a listening socket that had tcp_bpf
620 * protocol callbacks installed, we need to restore the callbacks to536 * protocol callbacks installed, we need to restore the callbacks to
net/ipv4/tcp_ipv4.c+3−0
@@ -2806,6 +2806,9 @@ struct proto tcp_prot = { · 2805 unmodified lines
2806 .hash = inet_hash,2806 .hash = inet_hash,
2807 .unhash = inet_unhash,2807 .unhash = inet_unhash,
2808 .get_port = inet_csk_get_port,2808 .get_port = inet_csk_get_port,
2809#ifdef CONFIG_BPF_SYSCALL
2810 .psock_update_sk_prot = tcp_bpf_update_proto,
2811#endif
2809 .enter_memory_pressure = tcp_enter_memory_pressure,2812 .enter_memory_pressure = tcp_enter_memory_pressure,
2810 .leave_memory_pressure = tcp_leave_memory_pressure,2813 .leave_memory_pressure = tcp_leave_memory_pressure,
2811 .stream_memory_free = tcp_stream_memory_free,2814 .stream_memory_free = tcp_stream_memory_free,
net/ipv4/udp.c+32−0
@@ -1782,6 +1782,35 @@ busy_check: · 1781 unmodified lines
1782}1782}
1783EXPORT_SYMBOL(__skb_recv_udp);1783EXPORT_SYMBOL(__skb_recv_udp);
1784 1784
1785int udp_read_sock(struct sock *sk, read_descriptor_t *desc,
1786 sk_read_actor_t recv_actor)
1787{
1788 int copied = 0;
1789
1790 while (1) {
1791 struct sk_buff *skb;
1792 int err, used;
1793
1794 skb = skb_recv_udp(sk, 0, 1, &err);
1795 if (!skb)
1796 return err;
1797 used = recv_actor(desc, skb, 0, skb->len);
1798 if (used <= 0) {
1799 if (!copied)
1800 copied = used;
1801 break;
1802 } else if (used <= skb->len) {
1803 copied += used;
1804 }
1805
1806 if (!desc->count)
1807 break;
1808 }
1809
1810 return copied;
1811}
1812EXPORT_SYMBOL(udp_read_sock);
1813
1785/*1814/*
1786 * This should be easy, if there is something there we1815 * This should be easy, if there is something there we
1787 * return it, otherwise we block.1816 * return it, otherwise we block.
@@ -2849,6 +2878,9 @@ struct proto udp_prot = { · 1061 unmodified lines
2849 .unhash = udp_lib_unhash,2878 .unhash = udp_lib_unhash,
2850 .rehash = udp_v4_rehash,2879 .rehash = udp_v4_rehash,
2851 .get_port = udp_v4_get_port,2880 .get_port = udp_v4_get_port,
2881#ifdef CONFIG_BPF_SYSCALL
2882 .psock_update_sk_prot = udp_bpf_update_proto,
2883#endif
2852 .memory_allocated = &udp_memory_allocated,2884 .memory_allocated = &udp_memory_allocated,
2853 .sysctl_mem = sysctl_udp_mem,2885 .sysctl_mem = sysctl_udp_mem,
2854 .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_udp_wmem_min),2886 .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_udp_wmem_min),
net/ipv4/udp_bpf.c+76−3
@@ -4,6 +4,68 @@ · 3 unmodified lines
4#include <linux/skmsg.h>4#include <linux/skmsg.h>
5#include <net/sock.h>5#include <net/sock.h>
6#include <net/udp.h>6#include <net/udp.h>
7#include <net/inet_common.h>
8
9#include "udp_impl.h"
10
11static struct proto *udpv6_prot_saved __read_mostly;
12
13static int sk_udp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
14 int noblock, int flags, int *addr_len)
15{
16#if IS_ENABLED(CONFIG_IPV6)
17 if (sk->sk_family == AF_INET6)
18 return udpv6_prot_saved->recvmsg(sk, msg, len, noblock, flags,
19 addr_len);
20#endif
21 return udp_prot.recvmsg(sk, msg, len, noblock, flags, addr_len);
22}
23
24static int udp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
25 int nonblock, int flags, int *addr_len)
26{
27 struct sk_psock *psock;
28 int copied, ret;
29
30 if (unlikely(flags & MSG_ERRQUEUE))
31 return inet_recv_error(sk, msg, len, addr_len);
32
33 psock = sk_psock_get(sk);
34 if (unlikely(!psock))
35 return sk_udp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
36
37 lock_sock(sk);
38 if (sk_psock_queue_empty(psock)) {
39 ret = sk_udp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
40 goto out;
41 }
42
43msg_bytes_ready:
44 copied = sk_msg_recvmsg(sk, psock, msg, len, flags);
45 if (!copied) {
46 int data, err = 0;
47 long timeo;
48
49 timeo = sock_rcvtimeo(sk, nonblock);
50 data = sk_msg_wait_data(sk, psock, flags, timeo, &err);
51 if (data) {
52 if (!sk_psock_queue_empty(psock))
53 goto msg_bytes_ready;
54 ret = sk_udp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
55 goto out;
56 }
57 if (err) {
58 ret = err;
59 goto out;
60 }
61 copied = -EAGAIN;
62 }
63 ret = copied;
64out:
65 release_sock(sk);
66 sk_psock_put(sk, psock);
67 return ret;
68}
7 69
8enum {70enum {
9 UDP_BPF_IPV4,71 UDP_BPF_IPV4,
@@ -11,7 +73,6 @@ enum { · 1 unmodified line
11 UDP_BPF_NUM_PROTS,73 UDP_BPF_NUM_PROTS,
12};74};
13 75
14static struct proto *udpv6_prot_saved __read_mostly;
15static DEFINE_SPINLOCK(udpv6_prot_lock);76static DEFINE_SPINLOCK(udpv6_prot_lock);
16static struct proto udp_bpf_prots[UDP_BPF_NUM_PROTS];77static struct proto udp_bpf_prots[UDP_BPF_NUM_PROTS];
17 78
@@ -20,6 +81,7 @@ static void udp_bpf_rebuild_protos(struct proto *prot, const struct proto *base) · 2 unmodified lines
20 *prot = *base;81 *prot = *base;
21 prot->unhash = sock_map_unhash;82 prot->unhash = sock_map_unhash;
22 prot->close = sock_map_close;83 prot->close = sock_map_close;
84 prot->recvmsg = udp_bpf_recvmsg;
23}85}
24 86
25static void udp_bpf_check_v6_needs_rebuild(struct proto *ops)87static void udp_bpf_check_v6_needs_rebuild(struct proto *ops)
@@ -41,12 +103,23 @@ static int __init udp_bpf_v4_build_proto(void) · 15 unmodified lines
41}103}
42core_initcall(udp_bpf_v4_build_proto);104core_initcall(udp_bpf_v4_build_proto);
43 105
44struct proto *udp_bpf_get_proto(struct sock *sk, struct sk_psock *psock)106int udp_bpf_update_proto(struct sock *sk, bool restore)
45{107{
46 int family = sk->sk_family == AF_INET ? UDP_BPF_IPV4 : UDP_BPF_IPV6;108 int family = sk->sk_family == AF_INET ? UDP_BPF_IPV4 : UDP_BPF_IPV6;
109 struct sk_psock *psock = sk_psock(sk);
110
111 if (restore) {
112 sk->sk_write_space = psock->saved_write_space;
113 /* Pairs with lockless read in sk_clone_lock() */
114 WRITE_ONCE(sk->sk_prot, psock->sk_proto);
115 return 0;
116 }
47 117
48 if (sk->sk_family == AF_INET6)118 if (sk->sk_family == AF_INET6)
49 udp_bpf_check_v6_needs_rebuild(psock->sk_proto);119 udp_bpf_check_v6_needs_rebuild(psock->sk_proto);
50 120
51 return &udp_bpf_prots[family];121 /* Pairs with lockless read in sk_clone_lock() */
122 WRITE_ONCE(sk->sk_prot, &udp_bpf_prots[family]);
123 return 0;
52}124}
125EXPORT_SYMBOL_GPL(udp_bpf_update_proto);
net/ipv6/af_inet6.c+1−0
@@ -714,6 +714,7 @@ const struct proto_ops inet6_dgram_ops = { · 713 unmodified lines
714 .getsockopt = sock_common_getsockopt, /* ok */714 .getsockopt = sock_common_getsockopt, /* ok */
715 .sendmsg = inet6_sendmsg, /* retpoline's sake */715 .sendmsg = inet6_sendmsg, /* retpoline's sake */
716 .recvmsg = inet6_recvmsg, /* retpoline's sake */716 .recvmsg = inet6_recvmsg, /* retpoline's sake */
717 .read_sock = udp_read_sock,
717 .mmap = sock_no_mmap,718 .mmap = sock_no_mmap,
718 .sendpage = sock_no_sendpage,719 .sendpage = sock_no_sendpage,
719 .set_peek_off = sk_set_peek_off,720 .set_peek_off = sk_set_peek_off,
net/ipv6/tcp_ipv6.c+3−0
@@ -2139,6 +2139,9 @@ struct proto tcpv6_prot = { · 2138 unmodified lines
2139 .hash = inet6_hash,2139 .hash = inet6_hash,
2140 .unhash = inet_unhash,2140 .unhash = inet_unhash,
2141 .get_port = inet_csk_get_port,2141 .get_port = inet_csk_get_port,
2142#ifdef CONFIG_BPF_SYSCALL
2143 .psock_update_sk_prot = tcp_bpf_update_proto,
2144#endif
2142 .enter_memory_pressure = tcp_enter_memory_pressure,2145 .enter_memory_pressure = tcp_enter_memory_pressure,
2143 .leave_memory_pressure = tcp_leave_memory_pressure,2146 .leave_memory_pressure = tcp_leave_memory_pressure,
2144 .stream_memory_free = tcp_stream_memory_free,2147 .stream_memory_free = tcp_stream_memory_free,
net/ipv6/udp.c+3−0
@@ -1713,6 +1713,9 @@ struct proto udpv6_prot = { · 1712 unmodified lines
1713 .unhash = udp_lib_unhash,1713 .unhash = udp_lib_unhash,
1714 .rehash = udp_v6_rehash,1714 .rehash = udp_v6_rehash,
1715 .get_port = udp_v6_get_port,1715 .get_port = udp_v6_get_port,
1716#ifdef CONFIG_BPF_SYSCALL
1717 .psock_update_sk_prot = udp_bpf_update_proto,
1718#endif
1716 .memory_allocated = &udp_memory_allocated,1719 .memory_allocated = &udp_memory_allocated,
1717 .sysctl_mem = sysctl_udp_mem,1720 .sysctl_mem = sysctl_udp_mem,
1718 .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_udp_wmem_min),1721 .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_udp_wmem_min),
net/tls/tls_sw.c+2−2
@@ -1789,8 +1789,8 @@ int tls_sw_recvmsg(struct sock *sk, · 1788 unmodified lines
1789 skb = tls_wait_data(sk, psock, flags, timeo, &err);1789 skb = tls_wait_data(sk, psock, flags, timeo, &err);
1790 if (!skb) {1790 if (!skb) {
1791 if (psock) {1791 if (psock) {
1792 int ret = __tcp_bpf_recvmsg(sk, psock,1792 int ret = sk_msg_recvmsg(sk, psock, msg, len,
1793 msg, len, flags);1793 flags);
1794 1794
1795 if (ret > 0) {1795 if (ret > 0) {
1796 decrypted += ret;1796 decrypted += ret;
tools/bpf/bpftool/common.c+1−0
@@ -57,6 +57,7 @@ const char * const attach_type_name[__MAX_BPF_ATTACH_TYPE] = { · 56 unmodified lines
57 57
58 [BPF_SK_SKB_STREAM_PARSER] = "sk_skb_stream_parser",58 [BPF_SK_SKB_STREAM_PARSER] = "sk_skb_stream_parser",
59 [BPF_SK_SKB_STREAM_VERDICT] = "sk_skb_stream_verdict",59 [BPF_SK_SKB_STREAM_VERDICT] = "sk_skb_stream_verdict",
60 [BPF_SK_SKB_VERDICT] = "sk_skb_verdict",
60 [BPF_SK_MSG_VERDICT] = "sk_msg_verdict",61 [BPF_SK_MSG_VERDICT] = "sk_msg_verdict",
61 [BPF_LIRC_MODE2] = "lirc_mode2",62 [BPF_LIRC_MODE2] = "lirc_mode2",
62 [BPF_FLOW_DISSECTOR] = "flow_dissector",63 [BPF_FLOW_DISSECTOR] = "flow_dissector",
tools/bpf/bpftool/prog.c+1−0
@@ -76,6 +76,7 @@ enum dump_mode { · 75 unmodified lines
76static const char * const attach_type_strings[] = {76static const char * const attach_type_strings[] = {
77 [BPF_SK_SKB_STREAM_PARSER] = "stream_parser",77 [BPF_SK_SKB_STREAM_PARSER] = "stream_parser",
78 [BPF_SK_SKB_STREAM_VERDICT] = "stream_verdict",78 [BPF_SK_SKB_STREAM_VERDICT] = "stream_verdict",
79 [BPF_SK_SKB_VERDICT] = "skb_verdict",
79 [BPF_SK_MSG_VERDICT] = "msg_verdict",80 [BPF_SK_MSG_VERDICT] = "msg_verdict",
80 [BPF_FLOW_DISSECTOR] = "flow_dissector",81 [BPF_FLOW_DISSECTOR] = "flow_dissector",
81 [__MAX_BPF_ATTACH_TYPE] = NULL,82 [__MAX_BPF_ATTACH_TYPE] = NULL,
tools/include/uapi/linux/bpf.h+1−0
@@ -957,6 +957,7 @@ enum bpf_attach_type { · 956 unmodified lines
957 BPF_XDP_CPUMAP,957 BPF_XDP_CPUMAP,
958 BPF_SK_LOOKUP,958 BPF_SK_LOOKUP,
959 BPF_XDP,959 BPF_XDP,
960 BPF_SK_SKB_VERDICT,
960 __MAX_BPF_ATTACH_TYPE961 __MAX_BPF_ATTACH_TYPE
961};962};
962 963
tools/testing/selftests/bpf/prog_tests/sockmap_basic.c+40−0
@@ -7,6 +7,7 @@ · 6 unmodified lines
7#include "test_skmsg_load_helpers.skel.h"7#include "test_skmsg_load_helpers.skel.h"
8#include "test_sockmap_update.skel.h"8#include "test_sockmap_update.skel.h"
9#include "test_sockmap_invalid_update.skel.h"9#include "test_sockmap_invalid_update.skel.h"
10#include "test_sockmap_skb_verdict_attach.skel.h"
10#include "bpf_iter_sockmap.skel.h"11#include "bpf_iter_sockmap.skel.h"
11 12
12#define TCP_REPAIR 19 /* TCP sock is under repair right now */13#define TCP_REPAIR 19 /* TCP sock is under repair right now */
@@ -281,6 +282,39 @@ out: · 268 unmodified lines
281 bpf_iter_sockmap__destroy(skel);282 bpf_iter_sockmap__destroy(skel);
282}283}
283 284
285static void test_sockmap_skb_verdict_attach(enum bpf_attach_type first,
286 enum bpf_attach_type second)
287{
288 struct test_sockmap_skb_verdict_attach *skel;
289 int err, map, verdict;
290
291 skel = test_sockmap_skb_verdict_attach__open_and_load();
292 if (CHECK_FAIL(!skel)) {
293 perror("test_sockmap_skb_verdict_attach__open_and_load");
294 return;
295 }
296
297 verdict = bpf_program__fd(skel->progs.prog_skb_verdict);
298 map = bpf_map__fd(skel->maps.sock_map);
299
300 err = bpf_prog_attach(verdict, map, first, 0);
301 if (CHECK_FAIL(err)) {
302 perror("bpf_prog_attach");
303 goto out;
304 }
305
306 err = bpf_prog_attach(verdict, map, second, 0);
307 assert(err == -1 && errno == EBUSY);
308
309 err = bpf_prog_detach2(verdict, map, first);
310 if (CHECK_FAIL(err)) {
311 perror("bpf_prog_detach2");
312 goto out;
313 }
314out:
315 test_sockmap_skb_verdict_attach__destroy(skel);
316}
317
284void test_sockmap_basic(void)318void test_sockmap_basic(void)
285{319{
286 if (test__start_subtest("sockmap create_update_free"))320 if (test__start_subtest("sockmap create_update_free"))
@@ -301,4 +335,10 @@ void test_sockmap_basic(void) · 14 unmodified lines
301 test_sockmap_copy(BPF_MAP_TYPE_SOCKMAP);335 test_sockmap_copy(BPF_MAP_TYPE_SOCKMAP);
302 if (test__start_subtest("sockhash copy"))336 if (test__start_subtest("sockhash copy"))
303 test_sockmap_copy(BPF_MAP_TYPE_SOCKHASH);337 test_sockmap_copy(BPF_MAP_TYPE_SOCKHASH);
338 if (test__start_subtest("sockmap skb_verdict attach")) {
339 test_sockmap_skb_verdict_attach(BPF_SK_SKB_VERDICT,
340 BPF_SK_SKB_STREAM_VERDICT);
341 test_sockmap_skb_verdict_attach(BPF_SK_SKB_STREAM_VERDICT,
342 BPF_SK_SKB_VERDICT);
343 }
304}344}
tools/testing/selftests/bpf/prog_tests/sockmap_listen.c+136−0
@@ -1603,6 +1603,141 @@ static void test_reuseport(struct test_sockmap_listen *skel, · 1602 unmodified lines
1603 }1603 }
1604}1604}
1605 1605
1606static void udp_redir_to_connected(int family, int sotype, int sock_mapfd,
1607 int verd_mapfd, enum redir_mode mode)
1608{
1609 const char *log_prefix = redir_mode_str(mode);
1610 struct sockaddr_storage addr;
1611 int c0, c1, p0, p1;
1612 unsigned int pass;
1613 socklen_t len;
1614 int err, n;
1615 u64 value;
1616 u32 key;
1617 char b;
1618
1619 zero_verdict_count(verd_mapfd);
1620
1621 p0 = socket_loopback(family, sotype | SOCK_NONBLOCK);
1622 if (p0 < 0)
1623 return;
1624 len = sizeof(addr);
1625 err = xgetsockname(p0, sockaddr(&addr), &len);
1626 if (err)
1627 goto close_peer0;
1628
1629 c0 = xsocket(family, sotype | SOCK_NONBLOCK, 0);
1630 if (c0 < 0)
1631 goto close_peer0;
1632 err = xconnect(c0, sockaddr(&addr), len);
1633 if (err)
1634 goto close_cli0;
1635 err = xgetsockname(c0, sockaddr(&addr), &len);
1636 if (err)
1637 goto close_cli0;
1638 err = xconnect(p0, sockaddr(&addr), len);
1639 if (err)
1640 goto close_cli0;
1641
1642 p1 = socket_loopback(family, sotype | SOCK_NONBLOCK);
1643 if (p1 < 0)
1644 goto close_cli0;
1645 err = xgetsockname(p1, sockaddr(&addr), &len);
1646 if (err)
1647 goto close_cli0;
1648
1649 c1 = xsocket(family, sotype | SOCK_NONBLOCK, 0);
1650 if (c1 < 0)
1651 goto close_peer1;
1652 err = xconnect(c1, sockaddr(&addr), len);
1653 if (err)
1654 goto close_cli1;
1655 err = xgetsockname(c1, sockaddr(&addr), &len);
1656 if (err)
1657 goto close_cli1;
1658 err = xconnect(p1, sockaddr(&addr), len);
1659 if (err)
1660 goto close_cli1;
1661
1662 key = 0;
1663 value = p0;
1664 err = xbpf_map_update_elem(sock_mapfd, &key, &value, BPF_NOEXIST);
1665 if (err)
1666 goto close_cli1;
1667
1668 key = 1;
1669 value = p1;
1670 err = xbpf_map_update_elem(sock_mapfd, &key, &value, BPF_NOEXIST);
1671 if (err)
1672 goto close_cli1;
1673
1674 n = write(c1, "a", 1);
1675 if (n < 0)
1676 FAIL_ERRNO("%s: write", log_prefix);
1677 if (n == 0)
1678 FAIL("%s: incomplete write", log_prefix);
1679 if (n < 1)
1680 goto close_cli1;
1681
1682 key = SK_PASS;
1683 err = xbpf_map_lookup_elem(verd_mapfd, &key, &pass);
1684 if (err)
1685 goto close_cli1;
1686 if (pass != 1)
1687 FAIL("%s: want pass count 1, have %d", log_prefix, pass);
1688
1689 n = read(mode == REDIR_INGRESS ? p0 : c0, &b, 1);
1690 if (n < 0)
1691 FAIL_ERRNO("%s: read", log_prefix);
1692 if (n == 0)
1693 FAIL("%s: incomplete read", log_prefix);
1694
1695close_cli1:
1696 xclose(c1);
1697close_peer1:
1698 xclose(p1);
1699close_cli0:
1700 xclose(c0);
1701close_peer0:
1702 xclose(p0);
1703}
1704
1705static void udp_skb_redir_to_connected(struct test_sockmap_listen *skel,
1706 struct bpf_map *inner_map, int family)
1707{
1708 int verdict = bpf_program__fd(skel->progs.prog_skb_verdict);
1709 int verdict_map = bpf_map__fd(skel->maps.verdict_map);
1710 int sock_map = bpf_map__fd(inner_map);
1711 int err;
1712
1713 err = xbpf_prog_attach(verdict, sock_map, BPF_SK_SKB_VERDICT, 0);
1714 if (err)
1715 return;
1716
1717 skel->bss->test_ingress = false;
1718 udp_redir_to_connected(family, SOCK_DGRAM, sock_map, verdict_map,
1719 REDIR_EGRESS);
1720 skel->bss->test_ingress = true;
1721 udp_redir_to_connected(family, SOCK_DGRAM, sock_map, verdict_map,
1722 REDIR_INGRESS);
1723
1724 xbpf_prog_detach2(verdict, sock_map, BPF_SK_SKB_VERDICT);
1725}
1726
1727static void test_udp_redir(struct test_sockmap_listen *skel, struct bpf_map *map,
1728 int family)
1729{
1730 const char *family_name, *map_name;
1731 char s[MAX_TEST_NAME];
1732
1733 family_name = family_str(family);
1734 map_name = map_type_str(map);
1735 snprintf(s, sizeof(s), "%s %s %s", map_name, family_name, __func__);
1736 if (!test__start_subtest(s))
1737 return;
1738 udp_skb_redir_to_connected(skel, map, family);
1739}
1740
1606static void run_tests(struct test_sockmap_listen *skel, struct bpf_map *map,1741static void run_tests(struct test_sockmap_listen *skel, struct bpf_map *map,
1607 int family)1742 int family)
1608{1743{
@@ -1611,6 +1746,7 @@ static void run_tests(struct test_sockmap_listen *skel, struct bpf_map *map, · 2 unmodified lines
1611 test_redir(skel, map, family, SOCK_STREAM);1746 test_redir(skel, map, family, SOCK_STREAM);
1612 test_reuseport(skel, map, family, SOCK_STREAM);1747 test_reuseport(skel, map, family, SOCK_STREAM);
1613 test_reuseport(skel, map, family, SOCK_DGRAM);1748 test_reuseport(skel, map, family, SOCK_DGRAM);
1749 test_udp_redir(skel, map, family);
1614}1750}
1615 1751
1616void test_sockmap_listen(void)1752void test_sockmap_listen(void)
tools/testing/selftests/bpf/progs/test_sockmap_listen.c+22−0
@@ -29,6 +29,7 @@ struct { · 28 unmodified lines
29} verdict_map SEC(".maps");29} verdict_map SEC(".maps");
30 30
31static volatile bool test_sockmap; /* toggled by user-space */31static volatile bool test_sockmap; /* toggled by user-space */
32static volatile bool test_ingress; /* toggled by user-space */
32 33
33SEC("sk_skb/stream_parser")34SEC("sk_skb/stream_parser")
34int prog_stream_parser(struct __sk_buff *skb)35int prog_stream_parser(struct __sk_buff *skb)
@@ -55,6 +56,27 @@ int prog_stream_verdict(struct __sk_buff *skb) · 20 unmodified lines
55 return verdict;56 return verdict;
56}57}
57 58
59SEC("sk_skb/skb_verdict")
60int prog_skb_verdict(struct __sk_buff *skb)
61{
62 unsigned int *count;
63 __u32 zero = 0;
64 int verdict;
65
66 if (test_sockmap)
67 verdict = bpf_sk_redirect_map(skb, &sock_map, zero,
68 test_ingress ? BPF_F_INGRESS : 0);
69 else
70 verdict = bpf_sk_redirect_hash(skb, &sock_hash, &zero,
71 test_ingress ? BPF_F_INGRESS : 0);
72
73 count = bpf_map_lookup_elem(&verdict_map, &verdict);
74 if (count)
75 (*count)++;
76
77 return verdict;
78}
79
58SEC("sk_msg")80SEC("sk_msg")
59int prog_msg_verdict(struct sk_msg_md *msg)81int prog_msg_verdict(struct sk_msg_md *msg)
60{82{
tools/testing/selftests/bpf/progs/test_sockmap_skb_verdict_attach.c+18−0
@@ -0,0 +1,18 @@
1// SPDX-License-Identifier: GPL-2.0
2#include "vmlinux.h"
3#include <bpf/bpf_helpers.h>
4
5struct {
6 __uint(type, BPF_MAP_TYPE_SOCKMAP);
7 __uint(max_entries, 2);
8 __type(key, __u32);
9 __type(value, __u64);
10} sock_map SEC(".maps");
11
12SEC("sk_skb/skb_verdict")
13int prog_skb_verdict(struct __sk_buff *skb)
14{
15 return SK_DROP;
16}
17
18char _license[] SEC("license") = "GPL";

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t