Merge branch 'sockmap: introduce BPF_SK_SKB_VERDICT and support UDP'

Cong Wang says: ==================== From: Cong Wang <cong.wang@bytedance.com> We have thousands of services connected to a daemon on every host via AF_UNIX dgram sockets, after they are moved into VM, we have to add a proxy to forward these communications from VM to host, because rewriting thousands of them is not practical. This proxy uses an AF_UNIX socket connected to services and a UDP socket to connect to the host. It is inefficient because data is copied between kernel space and user space twice, and we can not use splice() which only supports TCP. Therefore, we want to use sockmap to do the splicing without going to user-space at all (after the initial setup). Currently sockmap only fully supports TCP, UDP is partially supported as it is only allowed to add into sockmap. This patchset, as the second part of the original large patchset, extends sockmap with: 1) cross-protocol support with BPF_SK_SKB_VERDICT; 2) full UDP support. On the high level, ->read_sock() is required for each protocol to support sockmap redirection, and in order to do sock proto update, a new ops ->psock_update_sk_prot() is introduced, which is also required. And the BPF ->recvmsg() is also needed to replace the original ->recvmsg() to retrieve skmsg. To make life easier, we have to get rid of lock_sock() in sk_psock_handle_skb(), otherwise we would have to implement ->sendmsg_locked() on top of ->sendmsg(), which is ugly. Please see each patch for more details. To see the big picture, the original patchset is available here: https://github.com/congwang/linux/tree/sockmap this patchset is also available: https://github.com/congwang/linux/tree/sockmap2 --- v8: get rid of 'offset' in udp_read_sock() add checks for skb_verdict/stream_verdict conflict add two cleanup patches for sock_map_link() add a new test case v7: use work_mutex to protect psock->work return err in udp_read_sock() add patch 6/13 clean up test case v6: get rid of sk_psock_zap_ingress() add rcu work patch v5: use INDIRECT_CALL_2() for function pointers use ingress_lock to fix a race condition found by Jacub rename two helper functions v4: get rid of lock_sock() in sk_psock_handle_skb() get rid of udp_sendmsg_locked() remove an empty line update cover letter v3: export tcp/udp_update_proto() rename sk->sk_prot->psock_update_sk_prot() improve changelogs v2: separate from the original large patchset rebase to the latest bpf-next split UDP test case move inet_csk_has_ulp() check to tcp_bpf.c clean up udp_read_sock() ====================

Alexei Starovoitov committed 5 years ago

Commit 89d69c52 parents e27bfef 8d7cb7426 files · +677 −237Browse files

Unified · Split

26 files

include/linux/skbuff.h +1 −0
include/linux/skmsg.h +58 −19
include/net/sock.h +3 −0
include/net/tcp.h +1 −2
include/net/udp.h +3 −0
include/uapi/linux/bpf.h +1 −0
kernel/bpf/syscall.c +1 −0
net/core/skbuff.c +48 −7
net/core/skmsg.c +143 −34
net/core/sock_map.c +55 −63
net/ipv4/af_inet.c +1 −0
net/ipv4/tcp_bpf.c +23 −107
net/ipv4/tcp_ipv4.c +3 −0
net/ipv4/udp.c +32 −0
net/ipv4/udp_bpf.c +76 −3
net/ipv6/af_inet6.c +1 −0
net/ipv6/tcp_ipv6.c +3 −0
net/ipv6/udp.c +3 −0
net/tls/tls_sw.c +2 −2
tools/bpf/bpftool/common.c +1 −0
tools/bpf/bpftool/prog.c +1 −0
tools/include/uapi/linux/bpf.h +1 −0
tools/testing/selftests/bpf/prog_tests/sockmap_basic.c +40 −0
tools/testing/selftests/bpf/prog_tests/sockmap_listen.c +136 −0
tools/testing/selftests/bpf/progs/test_sockmap_listen.c +22 −0
tools/testing/selftests/bpf/progs/test_sockmap_skb_verdict_attach.c +18 −0

include/linux/skbuff.h+1−0

@@ -3626,6 +3626,7 @@ int skb_splice_bits(struct sk_buff skb, struct sock sk, unsigned int offset, · 3625 unmodified lines
3626	3626	unsigned int flags);
3627	3627	int skb_send_sock_locked(struct sock sk, struct sk_buff skb, int offset,
3628	3628	int len);
	3629	+int skb_send_sock(struct sock sk, struct sk_buff skb, int offset, int len);
3629	3630	void skb_copy_and_csum_dev(const struct sk_buff skb, u8 to);
3630	3631	unsigned int skb_zerocopy_headlen(const struct sk_buff *from);
3631	3632	int skb_zerocopy(struct sk_buff to, struct sk_buff from,

include/linux/skmsg.h+58−19

@@ -58,6 +58,7 @@ struct sk_psock_progs { · 57 unmodified lines
58	58	struct bpf_prog *msg_parser;
59	59	struct bpf_prog *stream_parser;
60	60	struct bpf_prog *stream_verdict;
	61	+ struct bpf_prog *skb_verdict;
61	62	};
62	63
63	64	enum sk_psock_state_bits {
@@ -89,6 +90,7 @@ struct sk_psock { · 25 unmodified lines
89	90	#endif
90	91	struct sk_buff_head ingress_skb;
91	92	struct list_head ingress_msg;
	93	+ spinlock_t ingress_lock;
92	94	unsigned long state;
93	95	struct list_head link;
94	96	spinlock_t link_lock;
@@ -97,13 +99,12 @@ struct sk_psock { · 2 unmodified lines
97	99	void (saved_close)(struct sock sk, long timeout);
98	100	void (saved_write_space)(struct sock sk);
99	101	void (saved_data_ready)(struct sock sk);
	102	+ int (psock_update_sk_prot)(struct sock sk, bool restore);
100	103	struct proto *sk_proto;
	104	+ struct mutex work_mutex;
101	105	struct sk_psock_work_state work_state;
102	106	struct work_struct work;
103		− union {
104		− struct rcu_head rcu;
105		− struct work_struct gc;
106		− };
	107	+ struct rcu_work rwork;
107	108	};
108	109
109	110	int sk_msg_alloc(struct sock sk, struct sk_msg msg, int len,
@@ -124,6 +125,10 @@ int sk_msg_zerocopy_from_iter(struct sock sk, struct iov_iter from, · 14 unmodified lines
124	125	struct sk_msg *msg, u32 bytes);
125	126	int sk_msg_memcopy_from_iter(struct sock sk, struct iov_iter from,
126	127	struct sk_msg *msg, u32 bytes);
	128	+int sk_msg_wait_data(struct sock sk, struct sk_psock psock, int flags,
	129	+ long timeo, int *err);
	130	+int sk_msg_recvmsg(struct sock sk, struct sk_psock psock, struct msghdr *msg,
	131	+ int len, int flags);
127	132
128	133	static inline void sk_msg_check_to_free(struct sk_msg *msg, u32 i, u32 bytes)
129	134	{
@@ -284,7 +289,45 @@ static inline struct sk_psock sk_psock(const struct sock sk) · 154 unmodified lines
284	289	static inline void sk_psock_queue_msg(struct sk_psock *psock,
285	290	struct sk_msg *msg)
286	291	{
	292	+ spin_lock_bh(&psock->ingress_lock);
287	293	list_add_tail(&msg->list, &psock->ingress_msg);
	294	+ spin_unlock_bh(&psock->ingress_lock);
	295	+}
	296	+
	297	+static inline struct sk_msg sk_psock_dequeue_msg(struct sk_psock psock)
	298	+{
	299	+ struct sk_msg *msg;
	300	+
	301	+ spin_lock_bh(&psock->ingress_lock);
	302	+ msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list);
	303	+ if (msg)
	304	+ list_del(&msg->list);
	305	+ spin_unlock_bh(&psock->ingress_lock);
	306	+ return msg;
	307	+}
	308	+
	309	+static inline struct sk_msg sk_psock_peek_msg(struct sk_psock psock)
	310	+{
	311	+ struct sk_msg *msg;
	312	+
	313	+ spin_lock_bh(&psock->ingress_lock);
	314	+ msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list);
	315	+ spin_unlock_bh(&psock->ingress_lock);
	316	+ return msg;
	317	+}
	318	+
	319	+static inline struct sk_msg sk_psock_next_msg(struct sk_psock psock,
	320	+ struct sk_msg *msg)
	321	+{
	322	+ struct sk_msg *ret;
	323	+
	324	+ spin_lock_bh(&psock->ingress_lock);
	325	+ if (list_is_last(&msg->list, &psock->ingress_msg))
	326	+ ret = NULL;
	327	+ else
	328	+ ret = list_next_entry(msg, list);
	329	+ spin_unlock_bh(&psock->ingress_lock);
	330	+ return ret;
288	331	}
289	332
290	333	static inline bool sk_psock_queue_empty(const struct sk_psock *psock)
@@ -292,6 +335,13 @@ static inline bool sk_psock_queue_empty(const struct sk_psock *psock) · 1 unmodified line
292	335	return psock ? list_empty(&psock->ingress_msg) : true;
293	336	}
294	337
	338	+static inline void kfree_sk_msg(struct sk_msg *msg)
	339	+{
	340	+ if (msg->skb)
	341	+ consume_skb(msg->skb);
	342	+ kfree(msg);
	343	+}
	344	+
295	345	static inline void sk_psock_report_error(struct sk_psock *psock, int err)
296	346	{
297	347	struct sock *sk = psock->sk;
@@ -301,6 +351,7 @@ static inline void sk_psock_report_error(struct sk_psock *psock, int err) · 3 unmodified lines
301	351	}
302	352
303	353	struct sk_psock sk_psock_init(struct sock sk, int node);
	354	+void sk_psock_stop(struct sk_psock *psock, bool wait);
304	355
305	356	#if IS_ENABLED(CONFIG_BPF_STREAM_PARSER)
306	357	int sk_psock_init_strp(struct sock sk, struct sk_psock psock);
@@ -349,25 +400,12 @@ static inline void sk_psock_cork_free(struct sk_psock *psock) · 42 unmodified lines
349	400	}
350	401	}
351	402
352		−static inline void sk_psock_update_proto(struct sock *sk,
353		− struct sk_psock *psock,
354		− struct proto *ops)
355		−{
356		− /* Pairs with lockless read in sk_clone_lock() */
357		− WRITE_ONCE(sk->sk_prot, ops);
358		−}
359		−
360	403	static inline void sk_psock_restore_proto(struct sock *sk,
361	404	struct sk_psock *psock)
362	405	{
363	406	sk->sk_prot->unhash = psock->saved_unhash;
364		− if (inet_csk_has_ulp(sk)) {
365		− tcp_update_ulp(sk, psock->sk_proto, psock->saved_write_space);
366		− } else {
367		− sk->sk_write_space = psock->saved_write_space;
368		− /* Pairs with lockless read in sk_clone_lock() */
369		− WRITE_ONCE(sk->sk_prot, psock->sk_proto);
370		− }
	407	+ if (psock->psock_update_sk_prot)
	408	+ psock->psock_update_sk_prot(sk, true);
371	409	}
372	410
373	411	static inline void sk_psock_set_state(struct sk_psock *psock,
@@ -442,6 +480,7 @@ static inline void psock_progs_drop(struct sk_psock_progs *progs) · 68 unmodified lines
442	480	psock_set_prog(&progs->msg_parser, NULL);
443	481	psock_set_prog(&progs->stream_parser, NULL);
444	482	psock_set_prog(&progs->stream_verdict, NULL);
	483	+ psock_set_prog(&progs->skb_verdict, NULL);
445	484	}
446	485
447	486	int sk_psock_tls_strp_read(struct sk_psock psock, struct sk_buff skb);

include/net/sock.h+3−0

@@ -1184,6 +1184,9 @@ struct proto { · 1183 unmodified lines
1184	1184	void (unhash)(struct sock sk);
1185	1185	void (rehash)(struct sock sk);
1186	1186	int (get_port)(struct sock sk, unsigned short snum);
	1187	+#ifdef CONFIG_BPF_SYSCALL
	1188	+ int (psock_update_sk_prot)(struct sock sk, bool restore);
	1189	+#endif
1187	1190
1188	1191	/* Keeping track of sockets in use */
1189	1192	#ifdef CONFIG_PROC_FS

include/net/tcp.h+1−2

@@ -2203,13 +2203,12 @@ struct sk_psock; · 2202 unmodified lines
2203	2203
2204	2204	#ifdef CONFIG_BPF_SYSCALL
2205	2205	struct proto tcp_bpf_get_proto(struct sock sk, struct sk_psock *psock);
	2206	+int tcp_bpf_update_proto(struct sock *sk, bool restore);
2206	2207	void tcp_bpf_clone(const struct sock sk, struct sock newsk);
2207	2208	#endif /* CONFIG_BPF_SYSCALL */
2208	2209
2209	2210	int tcp_bpf_sendmsg_redir(struct sock sk, struct sk_msg msg, u32 bytes,
2210	2211	int flags);
2211		−int __tcp_bpf_recvmsg(struct sock sk, struct sk_psock psock,
2212		− struct msghdr *msg, int len, int flags);
2213	2212	#endif /* CONFIG_NET_SOCK_MSG */
2214	2213
2215	2214	#if !defined(CONFIG_BPF_SYSCALL) \|\| !defined(CONFIG_NET_SOCK_MSG)

include/net/udp.h+3−0

@@ -329,6 +329,8 @@ struct sock __udp6_lib_lookup(struct net net, · 328 unmodified lines
329	329	struct sk_buff *skb);
330	330	struct sock udp6_lib_lookup_skb(const struct sk_buff skb,
331	331	__be16 sport, __be16 dport);
	332	+int udp_read_sock(struct sock sk, read_descriptor_t desc,
	333	+ sk_read_actor_t recv_actor);
332	334
333	335	/* UDP uses skb->dev_scratch to cache as much information as possible and avoid
334	336	* possibly multiple cache miss on dequeue()
@@ -518,6 +520,7 @@ static inline struct sk_buff udp_rcv_segment(struct sock sk, · 183 unmodified lines
518	520	#ifdef CONFIG_BPF_SYSCALL
519	521	struct sk_psock;
520	522	struct proto udp_bpf_get_proto(struct sock sk, struct sk_psock *psock);
	523	+int udp_bpf_update_proto(struct sock *sk, bool restore);
521	524	#endif
522	525
523	526	#endif /* _UDP_H */

include/uapi/linux/bpf.h+1−0

@@ -957,6 +957,7 @@ enum bpf_attach_type { · 956 unmodified lines
957	957	BPF_XDP_CPUMAP,
958	958	BPF_SK_LOOKUP,
959	959	BPF_XDP,
	960	+ BPF_SK_SKB_VERDICT,
960	961	__MAX_BPF_ATTACH_TYPE
961	962	};
962	963

kernel/bpf/syscall.c+1−0

@@ -2948,6 +2948,7 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type) · 2947 unmodified lines
2948	2948	return BPF_PROG_TYPE_SK_MSG;
2949	2949	case BPF_SK_SKB_STREAM_PARSER:
2950	2950	case BPF_SK_SKB_STREAM_VERDICT:
	2951	+ case BPF_SK_SKB_VERDICT:
2951	2952	return BPF_PROG_TYPE_SK_SKB;
2952	2953	case BPF_LIRC_MODE2:
2953	2954	return BPF_PROG_TYPE_LIRC_MODE2;

net/core/skbuff.c+48−7

@@ -2500,9 +2500,32 @@ int skb_splice_bits(struct sk_buff skb, struct sock sk, unsigned int offset, · 2499 unmodified lines
2500	2500	}
2501	2501	EXPORT_SYMBOL_GPL(skb_splice_bits);
2502	2502
2503		−/* Send skb data on a socket. Socket must be locked. */
2504		−int skb_send_sock_locked(struct sock sk, struct sk_buff skb, int offset,
2505		− int len)
	2503	+static int sendmsg_unlocked(struct sock sk, struct msghdr msg,
	2504	+ struct kvec *vec, size_t num, size_t size)
	2505	+{
	2506	+ struct socket *sock = sk->sk_socket;
	2507	+
	2508	+ if (!sock)
	2509	+ return -EINVAL;
	2510	+ return kernel_sendmsg(sock, msg, vec, num, size);
	2511	+}
	2512	+
	2513	+static int sendpage_unlocked(struct sock sk, struct page page, int offset,
	2514	+ size_t size, int flags)
	2515	+{
	2516	+ struct socket *sock = sk->sk_socket;
	2517	+
	2518	+ if (!sock)
	2519	+ return -EINVAL;
	2520	+ return kernel_sendpage(sock, page, offset, size, flags);
	2521	+}
	2522	+
	2523	+typedef int (sendmsg_func)(struct sock sk, struct msghdr *msg,
	2524	+ struct kvec *vec, size_t num, size_t size);
	2525	+typedef int (sendpage_func)(struct sock sk, struct page *page, int offset,
	2526	+ size_t size, int flags);
	2527	+static int __skb_send_sock(struct sock sk, struct sk_buff skb, int offset,
	2528	+ int len, sendmsg_func sendmsg, sendpage_func sendpage)
2506	2529	{
2507	2530	unsigned int orig_len = len;
2508	2531	struct sk_buff *head = skb;
@@ -2522,7 +2545,8 @@ do_frag_list: · 13 unmodified lines
2522	2545	memset(&msg, 0, sizeof(msg));
2523	2546	msg.msg_flags = MSG_DONTWAIT;
2524	2547
2525		− ret = kernel_sendmsg_locked(sk, &msg, &kv, 1, slen);
	2548	+ ret = INDIRECT_CALL_2(sendmsg, kernel_sendmsg_locked,
	2549	+ sendmsg_unlocked, sk, &msg, &kv, 1, slen);
2526	2550	if (ret <= 0)
2527	2551	goto error;
2528	2552
@@ -2553,9 +2577,11 @@ do_frag_list: · 24 unmodified lines
2553	2577	slen = min_t(size_t, len, skb_frag_size(frag) - offset);
2554	2578
2555	2579	while (slen) {
2556		− ret = kernel_sendpage_locked(sk, skb_frag_page(frag),
2557		− skb_frag_off(frag) + offset,
2558		− slen, MSG_DONTWAIT);
	2580	+ ret = INDIRECT_CALL_2(sendpage, kernel_sendpage_locked,
	2581	+ sendpage_unlocked, sk,
	2582	+ skb_frag_page(frag),
	2583	+ skb_frag_off(frag) + offset,
	2584	+ slen, MSG_DONTWAIT);
2559	2585	if (ret <= 0)
2560	2586	goto error;
2561	2587
@@ -2587,8 +2613,23 @@ out: · 25 unmodified lines
2587	2613	error:
2588	2614	return orig_len == len ? ret : orig_len - len;
2589	2615	}
	2616	+
	2617	+/* Send skb data on a socket. Socket must be locked. */
	2618	+int skb_send_sock_locked(struct sock sk, struct sk_buff skb, int offset,
	2619	+ int len)
	2620	+{
	2621	+ return __skb_send_sock(sk, skb, offset, len, kernel_sendmsg_locked,
	2622	+ kernel_sendpage_locked);
	2623	+}
2590	2624	EXPORT_SYMBOL_GPL(skb_send_sock_locked);
2591	2625
	2626	+/* Send skb data on a socket. Socket must be unlocked. */
	2627	+int skb_send_sock(struct sock sk, struct sk_buff skb, int offset, int len)
	2628	+{
	2629	+ return __skb_send_sock(sk, skb, offset, len, sendmsg_unlocked,
	2630	+ sendpage_unlocked);
	2631	+}
	2632	+
2592	2633	/**
2593	2634	* skb_store_bits - store bits from kernel buffer to skb
2594	2635	* @skb: destination buffer

net/core/skmsg.c+143−34

@@ -399,6 +399,104 @@ out: · 398 unmodified lines
399	399	}
400	400	EXPORT_SYMBOL_GPL(sk_msg_memcopy_from_iter);
401	401
	402	+int sk_msg_wait_data(struct sock sk, struct sk_psock psock, int flags,
	403	+ long timeo, int *err)
	404	+{
	405	+ DEFINE_WAIT_FUNC(wait, woken_wake_function);
	406	+ int ret = 0;
	407	+
	408	+ if (sk->sk_shutdown & RCV_SHUTDOWN)
	409	+ return 1;
	410	+
	411	+ if (!timeo)
	412	+ return ret;
	413	+
	414	+ add_wait_queue(sk_sleep(sk), &wait);
	415	+ sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
	416	+ ret = sk_wait_event(sk, &timeo,
	417	+ !list_empty(&psock->ingress_msg) \|\|
	418	+ !skb_queue_empty(&sk->sk_receive_queue), &wait);
	419	+ sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
	420	+ remove_wait_queue(sk_sleep(sk), &wait);
	421	+ return ret;
	422	+}
	423	+EXPORT_SYMBOL_GPL(sk_msg_wait_data);
	424	+
	425	+/* Receive sk_msg from psock->ingress_msg to @msg. */
	426	+int sk_msg_recvmsg(struct sock sk, struct sk_psock psock, struct msghdr *msg,
	427	+ int len, int flags)
	428	+{
	429	+ struct iov_iter *iter = &msg->msg_iter;
	430	+ int peek = flags & MSG_PEEK;
	431	+ struct sk_msg *msg_rx;
	432	+ int i, copied = 0;
	433	+
	434	+ msg_rx = sk_psock_peek_msg(psock);
	435	+ while (copied != len) {
	436	+ struct scatterlist *sge;
	437	+
	438	+ if (unlikely(!msg_rx))
	439	+ break;
	440	+
	441	+ i = msg_rx->sg.start;
	442	+ do {
	443	+ struct page *page;
	444	+ int copy;
	445	+
	446	+ sge = sk_msg_elem(msg_rx, i);
	447	+ copy = sge->length;
	448	+ page = sg_page(sge);
	449	+ if (copied + copy > len)
	450	+ copy = len - copied;
	451	+ copy = copy_page_to_iter(page, sge->offset, copy, iter);
	452	+ if (!copy)
	453	+ return copied ? copied : -EFAULT;
	454	+
	455	+ copied += copy;
	456	+ if (likely(!peek)) {
	457	+ sge->offset += copy;
	458	+ sge->length -= copy;
	459	+ if (!msg_rx->skb)
	460	+ sk_mem_uncharge(sk, copy);
	461	+ msg_rx->sg.size -= copy;
	462	+
	463	+ if (!sge->length) {
	464	+ sk_msg_iter_var_next(i);
	465	+ if (!msg_rx->skb)
	466	+ put_page(page);
	467	+ }
	468	+ } else {
	469	+ /* Lets not optimize peek case if copy_page_to_iter
	470	+ * didn't copy the entire length lets just break.
	471	+ */
	472	+ if (copy != sge->length)
	473	+ return copied;
	474	+ sk_msg_iter_var_next(i);
	475	+ }
	476	+
	477	+ if (copied == len)
	478	+ break;
	479	+ } while (i != msg_rx->sg.end);
	480	+
	481	+ if (unlikely(peek)) {
	482	+ msg_rx = sk_psock_next_msg(psock, msg_rx);
	483	+ if (!msg_rx)
	484	+ break;
	485	+ continue;
	486	+ }
	487	+
	488	+ msg_rx->sg.start = i;
	489	+ if (!sge->length && msg_rx->sg.start == msg_rx->sg.end) {
	490	+ msg_rx = sk_psock_dequeue_msg(psock);
	491	+ kfree_sk_msg(msg_rx);
	492	+ }
	493	+ msg_rx = sk_psock_peek_msg(psock);
	494	+ }
	495	+
	496	+ return copied;
	497	+}
	498	+EXPORT_SYMBOL_GPL(sk_msg_recvmsg);
	499	+
402	500	static struct sk_msg sk_psock_create_ingress_msg(struct sock sk,
403	501	struct sk_buff *skb)
404	502	{
@@ -410,7 +508,7 @@ static struct sk_msg sk_psock_create_ingress_msg(struct sock sk, · 5 unmodified lines
410	508	if (!sk_rmem_schedule(sk, skb, skb->truesize))
411	509	return NULL;
412	510
413		− msg = kzalloc(sizeof(*msg), __GFP_NOWARN \| GFP_ATOMIC);
	511	+ msg = kzalloc(sizeof(*msg), __GFP_NOWARN \| GFP_KERNEL);
414	512	if (unlikely(!msg))
415	513	return NULL;
416	514
@@ -497,7 +595,7 @@ static int sk_psock_handle_skb(struct sk_psock psock, struct sk_buff skb, · 80 unmodified lines
497	595	if (!ingress) {
498	596	if (!sock_writeable(psock->sk))
499	597	return -EAGAIN;
500		− return skb_send_sock_locked(psock->sk, skb, off, len);
	598	+ return skb_send_sock(psock->sk, skb, off, len);
501	599	}
502	600	return sk_psock_skb_ingress(psock, skb);
503	601	}
@@ -511,8 +609,7 @@ static void sk_psock_backlog(struct work_struct *work) · 7 unmodified lines
511	609	u32 len, off;
512	610	int ret;
513	611
514		− /* Lock sock to avoid losing sk_socket during loop. */
515		− lock_sock(psock->sk);
	612	+ mutex_lock(&psock->work_mutex);
516	613	if (state->skb) {
517	614	skb = state->skb;
518	615	len = state->len;
@@ -529,7 +626,7 @@ start: · 10 unmodified lines
529	626	skb_bpf_redirect_clear(skb);
530	627	do {
531	628	ret = -EIO;
532		− if (likely(psock->sk->sk_socket))
	629	+ if (!sock_flag(psock->sk, SOCK_DEAD))
533	630	ret = sk_psock_handle_skb(psock, skb, off,
534	631	len, ingress);
535	632	if (ret <= 0) {
@@ -553,7 +650,7 @@ start: · 17 unmodified lines
553	650	kfree_skb(skb);
554	651	}
555	652	end:
556		− release_sock(psock->sk);
	653	+ mutex_unlock(&psock->work_mutex);
557	654	}
558	655
559	656	struct sk_psock sk_psock_init(struct sock sk, int node)
@@ -563,11 +660,6 @@ struct sk_psock sk_psock_init(struct sock sk, int node) · 3 unmodified lines
563	660
564	661	write_lock_bh(&sk->sk_callback_lock);
565	662
566		− if (inet_csk_has_ulp(sk)) {
567		− psock = ERR_PTR(-EINVAL);
568		− goto out;
569		− }
570		−
571	663	if (sk->sk_user_data) {
572	664	psock = ERR_PTR(-EBUSY);
573	665	goto out;
@@ -591,7 +683,9 @@ struct sk_psock sk_psock_init(struct sock sk, int node) · 17 unmodified lines
591	683	spin_lock_init(&psock->link_lock);
592	684
593	685	INIT_WORK(&psock->work, sk_psock_backlog);
	686	+ mutex_init(&psock->work_mutex);
594	687	INIT_LIST_HEAD(&psock->ingress_msg);
	688	+ spin_lock_init(&psock->ingress_lock);
595	689	skb_queue_head_init(&psock->ingress_skb);
596	690
597	691	sk_psock_set_state(psock, SK_PSOCK_TX_ENABLED);
@@ -630,11 +724,11 @@ static void __sk_psock_purge_ingress_msg(struct sk_psock *psock) · 32 unmodified lines
630	724	}
631	725	}
632	726
633		−static void sk_psock_zap_ingress(struct sk_psock *psock)
	727	+static void __sk_psock_zap_ingress(struct sk_psock *psock)
634	728	{
635	729	struct sk_buff *skb;
636	730
637		− while ((skb = __skb_dequeue(&psock->ingress_skb)) != NULL) {
	731	+ while ((skb = skb_dequeue(&psock->ingress_skb)) != NULL) {
638	732	skb_bpf_redirect_clear(skb);
639	733	kfree_skb(skb);
640	734	}
@@ -651,23 +745,35 @@ static void sk_psock_link_destroy(struct sk_psock *psock) · 10 unmodified lines
651	745	}
652	746	}
653	747
	748	+void sk_psock_stop(struct sk_psock *psock, bool wait)
	749	+{
	750	+ spin_lock_bh(&psock->ingress_lock);
	751	+ sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);
	752	+ sk_psock_cork_free(psock);
	753	+ __sk_psock_zap_ingress(psock);
	754	+ spin_unlock_bh(&psock->ingress_lock);
	755	+
	756	+ if (wait)
	757	+ cancel_work_sync(&psock->work);
	758	+}
	759	+
654	760	static void sk_psock_done_strp(struct sk_psock *psock);
655	761
656		−static void sk_psock_destroy_deferred(struct work_struct *gc)
	762	+static void sk_psock_destroy(struct work_struct *work)
657	763	{
658		− struct sk_psock *psock = container_of(gc, struct sk_psock, gc);
659		−
	764	+ struct sk_psock *psock = container_of(to_rcu_work(work),
	765	+ struct sk_psock, rwork);
660	766	/* No sk_callback_lock since already detached. */
661	767
662	768	sk_psock_done_strp(psock);
663	769
664	770	cancel_work_sync(&psock->work);
	771	+ mutex_destroy(&psock->work_mutex);
665	772
666	773	psock_progs_drop(&psock->progs);
667	774
668	775	sk_psock_link_destroy(psock);
669	776	sk_psock_cork_free(psock);
670		− sk_psock_zap_ingress(psock);
671	777
672	778	if (psock->sk_redir)
673	779	sock_put(psock->sk_redir);
@@ -675,30 +781,21 @@ static void sk_psock_destroy_deferred(struct work_struct *gc) · 1 unmodified line
675	781	kfree(psock);
676	782	}
677	783
678		−static void sk_psock_destroy(struct rcu_head *rcu)
679		−{
680		− struct sk_psock *psock = container_of(rcu, struct sk_psock, rcu);
681		−
682		− INIT_WORK(&psock->gc, sk_psock_destroy_deferred);
683		− schedule_work(&psock->gc);
684		−}
685		−
686	784	void sk_psock_drop(struct sock sk, struct sk_psock psock)
687	785	{
688		− sk_psock_cork_free(psock);
689		− sk_psock_zap_ingress(psock);
	786	+ sk_psock_stop(psock, false);
690	787
691	788	write_lock_bh(&sk->sk_callback_lock);
692	789	sk_psock_restore_proto(sk, psock);
693	790	rcu_assign_sk_user_data(sk, NULL);
694	791	if (psock->progs.stream_parser)
695	792	sk_psock_stop_strp(sk, psock);
696		− else if (psock->progs.stream_verdict)
	793	+ else if (psock->progs.stream_verdict \|\| psock->progs.skb_verdict)
697	794	sk_psock_stop_verdict(sk, psock);
698	795	write_unlock_bh(&sk->sk_callback_lock);
699		− sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);
700	796
701		− call_rcu(&psock->rcu, sk_psock_destroy);
	797	+ INIT_RCU_WORK(&psock->rwork, sk_psock_destroy);
	798	+ queue_rcu_work(system_wq, &psock->rwork);
702	799	}
703	800	EXPORT_SYMBOL_GPL(sk_psock_drop);
704	801
@@ -767,14 +864,20 @@ static void sk_psock_skb_redirect(struct sk_buff *skb) · 62 unmodified lines
767	864	* error that caused the pipe to break. We can't send a packet on
768	865	* a socket that is in this state so we drop the skb.
769	866	*/
770		− if (!psock_other \|\| sock_flag(sk_other, SOCK_DEAD) \|\|
771		− !sk_psock_test_state(psock_other, SK_PSOCK_TX_ENABLED)) {
	867	+ if (!psock_other \|\| sock_flag(sk_other, SOCK_DEAD)) {
	868	+ kfree_skb(skb);
	869	+ return;
	870	+ }
	871	+ spin_lock_bh(&psock_other->ingress_lock);
	872	+ if (!sk_psock_test_state(psock_other, SK_PSOCK_TX_ENABLED)) {
	873	+ spin_unlock_bh(&psock_other->ingress_lock);
772	874	kfree_skb(skb);
773	875	return;
774	876	}
775	877
776	878	skb_queue_tail(&psock_other->ingress_skb, skb);
777	879	schedule_work(&psock_other->work);
	880	+ spin_unlock_bh(&psock_other->ingress_lock);
778	881	}
779	882
780	883	static void sk_psock_tls_verdict_apply(struct sk_buff skb, struct sock sk, int verdict)
@@ -842,8 +945,12 @@ static void sk_psock_verdict_apply(struct sk_psock *psock, · 61 unmodified lines
842	945	err = sk_psock_skb_ingress_self(psock, skb);
843	946	}
844	947	if (err < 0) {
845		− skb_queue_tail(&psock->ingress_skb, skb);
846		− schedule_work(&psock->work);
	948	+ spin_lock_bh(&psock->ingress_lock);
	949	+ if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) {
	950	+ skb_queue_tail(&psock->ingress_skb, skb);
	951	+ schedule_work(&psock->work);
	952	+ }
	953	+ spin_unlock_bh(&psock->ingress_lock);
847	954	}
848	955	break;
849	956	case __SK_REDIRECT:
@@ -1010,6 +1117,8 @@ static int sk_psock_verdict_recv(read_descriptor_t desc, struct sk_buff skb, · 160 unmodified lines
1010	1117	}
1011	1118	skb_set_owner_r(skb, sk);
1012	1119	prog = READ_ONCE(psock->progs.stream_verdict);
	1120	+ if (!prog)
	1121	+ prog = READ_ONCE(psock->progs.skb_verdict);
1013	1122	if (likely(prog)) {
1014	1123	skb_dst_drop(skb);
1015	1124	skb_bpf_redirect_clear(skb);

net/core/sock_map.c+55−63

@@ -26,6 +26,7 @@ struct bpf_stab { · 25 unmodified lines
26	26
27	27	static int sock_map_prog_update(struct bpf_map map, struct bpf_prog prog,
28	28	struct bpf_prog *old, u32 which);
	29	+static struct sk_psock_progs sock_map_progs(struct bpf_map map);
29	30
30	31	static struct bpf_map sock_map_alloc(union bpf_attr attr)
31	32	{
@@ -155,6 +156,8 @@ static void sock_map_del_link(struct sock *sk, · 123 unmodified lines
155	156	strp_stop = true;
156	157	if (psock->saved_data_ready && stab->progs.stream_verdict)
157	158	verdict_stop = true;
	159	+ if (psock->saved_data_ready && stab->progs.skb_verdict)
	160	+ verdict_stop = true;
158	161	list_del(&link->list);
159	162	sk_psock_free_link(link);
160	163	}
@@ -182,26 +185,10 @@ static void sock_map_unref(struct sock sk, void link_raw) · 21 unmodified lines
182	185
183	186	static int sock_map_init_proto(struct sock sk, struct sk_psock psock)
184	187	{
185		− struct proto *prot;
186		−
187		− switch (sk->sk_type) {
188		− case SOCK_STREAM:
189		− prot = tcp_bpf_get_proto(sk, psock);
190		− break;
191		−
192		− case SOCK_DGRAM:
193		− prot = udp_bpf_get_proto(sk, psock);
194		− break;
195		−
196		− default:
	188	+ if (!sk->sk_prot->psock_update_sk_prot)
197	189	return -EINVAL;
198		− }
199		−
200		− if (IS_ERR(prot))
201		− return PTR_ERR(prot);
202		−
203		− sk_psock_update_proto(sk, psock, prot);
204		− return 0;
	190	+ psock->psock_update_sk_prot = sk->sk_prot->psock_update_sk_prot;
	191	+ return sk->sk_prot->psock_update_sk_prot(sk, false);
205	192	}
206	193
207	194	static struct sk_psock sock_map_psock_get_checked(struct sock sk)
@@ -224,13 +211,25 @@ out: · 16 unmodified lines
224	211	return psock;
225	212	}
226	213
227		−static int sock_map_link(struct bpf_map map, struct sk_psock_progs progs,
228		− struct sock *sk)
	214	+static bool sock_map_redirect_allowed(const struct sock *sk);
	215	+
	216	+static int sock_map_link(struct bpf_map map, struct sock sk)
229	217	{
230		− struct bpf_prog msg_parser, stream_parser, *stream_verdict;
	218	+ struct sk_psock_progs *progs = sock_map_progs(map);
	219	+ struct bpf_prog *stream_verdict = NULL;
	220	+ struct bpf_prog *stream_parser = NULL;
	221	+ struct bpf_prog *skb_verdict = NULL;
	222	+ struct bpf_prog *msg_parser = NULL;
231	223	struct sk_psock *psock;
232	224	int ret;
233	225
	226	+ /* Only sockets we can redirect into/from in BPF need to hold
	227	+ * refs to parser/verdict progs and have their sk_data_ready
	228	+ * and sk_write_space callbacks overridden.
	229	+ */
	230	+ if (!sock_map_redirect_allowed(sk))
	231	+ goto no_progs;
	232	+
234	233	stream_verdict = READ_ONCE(progs->stream_verdict);
235	234	if (stream_verdict) {
236	235	stream_verdict = bpf_prog_inc_not_zero(stream_verdict);
@@ -256,6 +255,16 @@ static int sock_map_link(struct bpf_map map, struct sk_psock_progs progs, · 19 unmodified lines
256	255	}
257	256	}
258	257
	258	+ skb_verdict = READ_ONCE(progs->skb_verdict);
	259	+ if (skb_verdict) {
	260	+ skb_verdict = bpf_prog_inc_not_zero(skb_verdict);
	261	+ if (IS_ERR(skb_verdict)) {
	262	+ ret = PTR_ERR(skb_verdict);
	263	+ goto out_put_msg_parser;
	264	+ }
	265	+ }
	266	+
	267	+no_progs:
259	268	psock = sock_map_psock_get_checked(sk);
260	269	if (IS_ERR(psock)) {
261	270	ret = PTR_ERR(psock);
@@ -265,6 +274,9 @@ static int sock_map_link(struct bpf_map map, struct sk_psock_progs progs, · 3 unmodified lines
265	274	if (psock) {
266	275	if ((msg_parser && READ_ONCE(psock->progs.msg_parser)) \|\|
267	276	(stream_parser && READ_ONCE(psock->progs.stream_parser)) \|\|
	277	+ (skb_verdict && READ_ONCE(psock->progs.skb_verdict)) \|\|
	278	+ (skb_verdict && READ_ONCE(psock->progs.stream_verdict)) \|\|
	279	+ (stream_verdict && READ_ONCE(psock->progs.skb_verdict)) \|\|
268	280	(stream_verdict && READ_ONCE(psock->progs.stream_verdict))) {
269	281	sk_psock_put(sk, psock);
270	282	ret = -EBUSY;
@@ -296,6 +308,9 @@ static int sock_map_link(struct bpf_map map, struct sk_psock_progs progs, · 25 unmodified lines
296	308	} else if (!stream_parser && stream_verdict && !psock->saved_data_ready) {
297	309	psock_set_prog(&psock->progs.stream_verdict, stream_verdict);
298	310	sk_psock_start_verdict(sk,psock);
	311	+ } else if (!stream_verdict && skb_verdict && !psock->saved_data_ready) {
	312	+ psock_set_prog(&psock->progs.skb_verdict, skb_verdict);
	313	+ sk_psock_start_verdict(sk, psock);
299	314	}
300	315	write_unlock_bh(&sk->sk_callback_lock);
301	316	return 0;
@@ -304,6 +319,9 @@ out_unlock_drop: · 2 unmodified lines
304	319	out_drop:
305	320	sk_psock_put(sk, psock);
306	321	out_progs:
	322	+ if (skb_verdict)
	323	+ bpf_prog_put(skb_verdict);
	324	+out_put_msg_parser:
307	325	if (msg_parser)
308	326	bpf_prog_put(msg_parser);
309	327	out_put_stream_parser:
@@ -315,27 +333,6 @@ out_put_stream_verdict: · 5 unmodified lines
315	333	return ret;
316	334	}
317	335
318		−static int sock_map_link_no_progs(struct bpf_map map, struct sock sk)
319		−{
320		− struct sk_psock *psock;
321		− int ret;
322		−
323		− psock = sock_map_psock_get_checked(sk);
324		− if (IS_ERR(psock))
325		− return PTR_ERR(psock);
326		−
327		− if (!psock) {
328		− psock = sk_psock_init(sk, map->numa_node);
329		− if (IS_ERR(psock))
330		− return PTR_ERR(psock);
331		− }
332		−
333		− ret = sock_map_init_proto(sk, psock);
334		− if (ret < 0)
335		− sk_psock_put(sk, psock);
336		− return ret;
337		−}
338		−
339	336	static void sock_map_free(struct bpf_map *map)
340	337	{
341	338	struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
@@ -466,8 +463,6 @@ static int sock_map_get_next_key(struct bpf_map map, void key, void *next) · 124 unmodified lines
466	463	return 0;
467	464	}
468	465
469		−static bool sock_map_redirect_allowed(const struct sock *sk);
470		−
471	466	static int sock_map_update_common(struct bpf_map *map, u32 idx,
472	467	struct sock *sk, u64 flags)
473	468	{
@@ -487,14 +482,7 @@ static int sock_map_update_common(struct bpf_map *map, u32 idx, · 13 unmodified lines
487	482	if (!link)
488	483	return -ENOMEM;
489	484
490		− /* Only sockets we can redirect into/from in BPF need to hold
491		− * refs to parser/verdict progs and have their sk_data_ready
492		− * and sk_write_space callbacks overridden.
493		− */
494		− if (sock_map_redirect_allowed(sk))
495		− ret = sock_map_link(map, &stab->progs, sk);
496		− else
497		− ret = sock_map_link_no_progs(map, sk);
	485	+ ret = sock_map_link(map, sk);
498	486	if (ret < 0)
499	487	goto out_free;
500	488
@@ -547,12 +535,15 @@ static bool sk_is_udp(const struct sock *sk) · 46 unmodified lines
547	535
548	536	static bool sock_map_redirect_allowed(const struct sock *sk)
549	537	{
550		− return sk_is_tcp(sk) && sk->sk_state != TCP_LISTEN;
	538	+ if (sk_is_tcp(sk))
	539	+ return sk->sk_state != TCP_LISTEN;
	540	+ else
	541	+ return sk->sk_state == TCP_ESTABLISHED;
551	542	}
552	543
553	544	static bool sock_map_sk_is_suitable(const struct sock *sk)
554	545	{
555		− return sk_is_tcp(sk) \|\| sk_is_udp(sk);
	546	+ return !!sk->sk_prot->psock_update_sk_prot;
556	547	}
557	548
558	549	static bool sock_map_sk_state_allowed(const struct sock *sk)
@@ -999,14 +990,7 @@ static int sock_hash_update_common(struct bpf_map map, void key, · 440 unmodified lines
999	990	if (!link)
1000	991	return -ENOMEM;
1001	992
1002		− /* Only sockets we can redirect into/from in BPF need to hold
1003		− * refs to parser/verdict progs and have their sk_data_ready
1004		− * and sk_write_space callbacks overridden.
1005		− */
1006		− if (sock_map_redirect_allowed(sk))
1007		− ret = sock_map_link(map, &htab->progs, sk);
1008		− else
1009		− ret = sock_map_link_no_progs(map, sk);
	993	+ ret = sock_map_link(map, sk);
1010	994	if (ret < 0)
1011	995	goto out_free;
1012	996
@@ -1466,8 +1450,15 @@ static int sock_map_prog_update(struct bpf_map map, struct bpf_prog prog, · 453 unmodified lines
1466	1450	break;
1467	1451	#endif
1468	1452	case BPF_SK_SKB_STREAM_VERDICT:
	1453	+ if (progs->skb_verdict)
	1454	+ return -EBUSY;
1469	1455	pprog = &progs->stream_verdict;
1470	1456	break;
	1457	+ case BPF_SK_SKB_VERDICT:
	1458	+ if (progs->stream_verdict)
	1459	+ return -EBUSY;
	1460	+ pprog = &progs->skb_verdict;
	1461	+ break;
1471	1462	default:
1472	1463	return -EOPNOTSUPP;
1473	1464	}
@@ -1540,6 +1531,7 @@ void sock_map_close(struct sock *sk, long timeout) · 66 unmodified lines
1540	1531	saved_close = psock->saved_close;
1541	1532	sock_map_remove_links(sk, psock);
1542	1533	rcu_read_unlock();
	1534	+ sk_psock_stop(psock, true);
1543	1535	release_sock(sk);
1544	1536	saved_close(sk, timeout);
1545	1537	}

net/ipv4/af_inet.c+1−0

@@ -1070,6 +1070,7 @@ const struct proto_ops inet_dgram_ops = { · 1069 unmodified lines
1070	1070	.setsockopt = sock_common_setsockopt,
1071	1071	.getsockopt = sock_common_getsockopt,
1072	1072	.sendmsg = inet_sendmsg,
	1073	+ .read_sock = udp_read_sock,
1073	1074	.recvmsg = inet_recvmsg,
1074	1075	.mmap = sock_no_mmap,
1075	1076	.sendpage = inet_sendpage,

net/ipv4/tcp_bpf.c+23−107

@@ -10,86 +10,6 @@ · 9 unmodified lines
10	10	#include <net/inet_common.h>
11	11	#include <net/tls.h>
12	12
13		−int __tcp_bpf_recvmsg(struct sock sk, struct sk_psock psock,
14		− struct msghdr *msg, int len, int flags)
15		−{
16		− struct iov_iter *iter = &msg->msg_iter;
17		− int peek = flags & MSG_PEEK;
18		− struct sk_msg *msg_rx;
19		− int i, copied = 0;
20		−
21		− msg_rx = list_first_entry_or_null(&psock->ingress_msg,
22		− struct sk_msg, list);
23		−
24		− while (copied != len) {
25		− struct scatterlist *sge;
26		−
27		− if (unlikely(!msg_rx))
28		− break;
29		−
30		− i = msg_rx->sg.start;
31		− do {
32		− struct page *page;
33		− int copy;
34		−
35		− sge = sk_msg_elem(msg_rx, i);
36		− copy = sge->length;
37		− page = sg_page(sge);
38		− if (copied + copy > len)
39		− copy = len - copied;
40		− copy = copy_page_to_iter(page, sge->offset, copy, iter);
41		− if (!copy)
42		− return copied ? copied : -EFAULT;
43		−
44		− copied += copy;
45		− if (likely(!peek)) {
46		− sge->offset += copy;
47		− sge->length -= copy;
48		− if (!msg_rx->skb)
49		− sk_mem_uncharge(sk, copy);
50		− msg_rx->sg.size -= copy;
51		−
52		− if (!sge->length) {
53		− sk_msg_iter_var_next(i);
54		− if (!msg_rx->skb)
55		− put_page(page);
56		− }
57		− } else {
58		− /* Lets not optimize peek case if copy_page_to_iter
59		− * didn't copy the entire length lets just break.
60		− */
61		− if (copy != sge->length)
62		− return copied;
63		− sk_msg_iter_var_next(i);
64		− }
65		−
66		− if (copied == len)
67		− break;
68		− } while (i != msg_rx->sg.end);
69		−
70		− if (unlikely(peek)) {
71		− if (msg_rx == list_last_entry(&psock->ingress_msg,
72		− struct sk_msg, list))
73		− break;
74		− msg_rx = list_next_entry(msg_rx, list);
75		− continue;
76		− }
77		−
78		− msg_rx->sg.start = i;
79		− if (!sge->length && msg_rx->sg.start == msg_rx->sg.end) {
80		− list_del(&msg_rx->list);
81		− if (msg_rx->skb)
82		− consume_skb(msg_rx->skb);
83		− kfree(msg_rx);
84		− }
85		− msg_rx = list_first_entry_or_null(&psock->ingress_msg,
86		− struct sk_msg, list);
87		− }
88		−
89		− return copied;
90		−}
91		−EXPORT_SYMBOL_GPL(__tcp_bpf_recvmsg);
92		−
93	13	static int bpf_tcp_ingress(struct sock sk, struct sk_psock psock,
94	14	struct sk_msg *msg, u32 apply_bytes, int flags)
95	15	{
@@ -243,28 +163,6 @@ static bool tcp_bpf_stream_read(const struct sock *sk) · 147 unmodified lines
243	163	return !empty;
244	164	}
245	165
246		−static int tcp_bpf_wait_data(struct sock sk, struct sk_psock psock,
247		− int flags, long timeo, int *err)
248		−{
249		− DEFINE_WAIT_FUNC(wait, woken_wake_function);
250		− int ret = 0;
251		−
252		− if (sk->sk_shutdown & RCV_SHUTDOWN)
253		− return 1;
254		−
255		− if (!timeo)
256		− return ret;
257		−
258		− add_wait_queue(sk_sleep(sk), &wait);
259		− sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
260		− ret = sk_wait_event(sk, &timeo,
261		− !list_empty(&psock->ingress_msg) \|\|
262		− !skb_queue_empty(&sk->sk_receive_queue), &wait);
263		− sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
264		− remove_wait_queue(sk_sleep(sk), &wait);
265		− return ret;
266		−}
267		−
268	166	static int tcp_bpf_recvmsg(struct sock sk, struct msghdr msg, size_t len,
269	167	int nonblock, int flags, int *addr_len)
270	168	{
@@ -284,13 +182,13 @@ static int tcp_bpf_recvmsg(struct sock sk, struct msghdr msg, size_t len, · 13 unmodified lines
284	182	}
285	183	lock_sock(sk);
286	184	msg_bytes_ready:
287		− copied = __tcp_bpf_recvmsg(sk, psock, msg, len, flags);
	185	+ copied = sk_msg_recvmsg(sk, psock, msg, len, flags);
288	186	if (!copied) {
289	187	int data, err = 0;
290	188	long timeo;
291	189
292	190	timeo = sock_rcvtimeo(sk, nonblock);
293		− data = tcp_bpf_wait_data(sk, psock, flags, timeo, &err);
	191	+ data = sk_msg_wait_data(sk, psock, flags, timeo, &err);
294	192	if (data) {
295	193	if (!sk_psock_queue_empty(psock))
296	194	goto msg_bytes_ready;
@@ -601,20 +499,38 @@ static int tcp_bpf_assert_proto_ops(struct proto *ops) · 304 unmodified lines
601	499	ops->sendpage == tcp_sendpage ? 0 : -ENOTSUPP;
602	500	}
603	501
604		−struct proto tcp_bpf_get_proto(struct sock sk, struct sk_psock *psock)
	502	+int tcp_bpf_update_proto(struct sock *sk, bool restore)
605	503	{
	504	+ struct sk_psock *psock = sk_psock(sk);
606	505	int family = sk->sk_family == AF_INET6 ? TCP_BPF_IPV6 : TCP_BPF_IPV4;
607	506	int config = psock->progs.msg_parser ? TCP_BPF_TX : TCP_BPF_BASE;
608	507
	508	+ if (restore) {
	509	+ if (inet_csk_has_ulp(sk)) {
	510	+ tcp_update_ulp(sk, psock->sk_proto, psock->saved_write_space);
	511	+ } else {
	512	+ sk->sk_write_space = psock->saved_write_space;
	513	+ /* Pairs with lockless read in sk_clone_lock() */
	514	+ WRITE_ONCE(sk->sk_prot, psock->sk_proto);
	515	+ }
	516	+ return 0;
	517	+ }
	518	+
	519	+ if (inet_csk_has_ulp(sk))
	520	+ return -EINVAL;
	521	+
609	522	if (sk->sk_family == AF_INET6) {
610	523	if (tcp_bpf_assert_proto_ops(psock->sk_proto))
611		− return ERR_PTR(-EINVAL);
	524	+ return -EINVAL;
612	525
613	526	tcp_bpf_check_v6_needs_rebuild(psock->sk_proto);
614	527	}
615	528
616		− return &tcp_bpf_prots[family][config];
	529	+ /* Pairs with lockless read in sk_clone_lock() */
	530	+ WRITE_ONCE(sk->sk_prot, &tcp_bpf_prots[family][config]);
	531	+ return 0;
617	532	}
	533	+EXPORT_SYMBOL_GPL(tcp_bpf_update_proto);
618	534
619	535	/* If a child got cloned from a listening socket that had tcp_bpf
620	536	* protocol callbacks installed, we need to restore the callbacks to

net/ipv4/tcp_ipv4.c+3−0

@@ -2806,6 +2806,9 @@ struct proto tcp_prot = { · 2805 unmodified lines
2806	2806	.hash = inet_hash,
2807	2807	.unhash = inet_unhash,
2808	2808	.get_port = inet_csk_get_port,
	2809	+#ifdef CONFIG_BPF_SYSCALL
	2810	+ .psock_update_sk_prot = tcp_bpf_update_proto,
	2811	+#endif
2809	2812	.enter_memory_pressure = tcp_enter_memory_pressure,
2810	2813	.leave_memory_pressure = tcp_leave_memory_pressure,
2811	2814	.stream_memory_free = tcp_stream_memory_free,

net/ipv4/udp.c+32−0

@@ -1782,6 +1782,35 @@ busy_check: · 1781 unmodified lines
1782	1782	}
1783	1783	EXPORT_SYMBOL(__skb_recv_udp);
1784	1784
	1785	+int udp_read_sock(struct sock sk, read_descriptor_t desc,
	1786	+ sk_read_actor_t recv_actor)
	1787	+{
	1788	+ int copied = 0;
	1789	+
	1790	+ while (1) {
	1791	+ struct sk_buff *skb;
	1792	+ int err, used;
	1793	+
	1794	+ skb = skb_recv_udp(sk, 0, 1, &err);
	1795	+ if (!skb)
	1796	+ return err;
	1797	+ used = recv_actor(desc, skb, 0, skb->len);
	1798	+ if (used <= 0) {
	1799	+ if (!copied)
	1800	+ copied = used;
	1801	+ break;
	1802	+ } else if (used <= skb->len) {
	1803	+ copied += used;
	1804	+ }
	1805	+
	1806	+ if (!desc->count)
	1807	+ break;
	1808	+ }
	1809	+
	1810	+ return copied;
	1811	+}
	1812	+EXPORT_SYMBOL(udp_read_sock);
	1813	+
1785	1814	/*
1786	1815	* This should be easy, if there is something there we
1787	1816	* return it, otherwise we block.
@@ -2849,6 +2878,9 @@ struct proto udp_prot = { · 1061 unmodified lines
2849	2878	.unhash = udp_lib_unhash,
2850	2879	.rehash = udp_v4_rehash,
2851	2880	.get_port = udp_v4_get_port,
	2881	+#ifdef CONFIG_BPF_SYSCALL
	2882	+ .psock_update_sk_prot = udp_bpf_update_proto,
	2883	+#endif
2852	2884	.memory_allocated = &udp_memory_allocated,
2853	2885	.sysctl_mem = sysctl_udp_mem,
2854	2886	.sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_udp_wmem_min),

net/ipv4/udp_bpf.c+76−3

@@ -4,6 +4,68 @@ · 3 unmodified lines
4	4	#include <linux/skmsg.h>
5	5	#include <net/sock.h>
6	6	#include <net/udp.h>
	7	+#include <net/inet_common.h>
	8	+
	9	+#include "udp_impl.h"
	10	+
	11	+static struct proto *udpv6_prot_saved __read_mostly;
	12	+
	13	+static int sk_udp_recvmsg(struct sock sk, struct msghdr msg, size_t len,
	14	+ int noblock, int flags, int *addr_len)
	15	+{
	16	+#if IS_ENABLED(CONFIG_IPV6)
	17	+ if (sk->sk_family == AF_INET6)
	18	+ return udpv6_prot_saved->recvmsg(sk, msg, len, noblock, flags,
	19	+ addr_len);
	20	+#endif
	21	+ return udp_prot.recvmsg(sk, msg, len, noblock, flags, addr_len);
	22	+}
	23	+
	24	+static int udp_bpf_recvmsg(struct sock sk, struct msghdr msg, size_t len,
	25	+ int nonblock, int flags, int *addr_len)
	26	+{
	27	+ struct sk_psock *psock;
	28	+ int copied, ret;
	29	+
	30	+ if (unlikely(flags & MSG_ERRQUEUE))
	31	+ return inet_recv_error(sk, msg, len, addr_len);
	32	+
	33	+ psock = sk_psock_get(sk);
	34	+ if (unlikely(!psock))
	35	+ return sk_udp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
	36	+
	37	+ lock_sock(sk);
	38	+ if (sk_psock_queue_empty(psock)) {
	39	+ ret = sk_udp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
	40	+ goto out;
	41	+ }
	42	+
	43	+msg_bytes_ready:
	44	+ copied = sk_msg_recvmsg(sk, psock, msg, len, flags);
	45	+ if (!copied) {
	46	+ int data, err = 0;
	47	+ long timeo;
	48	+
	49	+ timeo = sock_rcvtimeo(sk, nonblock);
	50	+ data = sk_msg_wait_data(sk, psock, flags, timeo, &err);
	51	+ if (data) {
	52	+ if (!sk_psock_queue_empty(psock))
	53	+ goto msg_bytes_ready;
	54	+ ret = sk_udp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
	55	+ goto out;
	56	+ }
	57	+ if (err) {
	58	+ ret = err;
	59	+ goto out;
	60	+ }
	61	+ copied = -EAGAIN;
	62	+ }
	63	+ ret = copied;
	64	+out:
	65	+ release_sock(sk);
	66	+ sk_psock_put(sk, psock);
	67	+ return ret;
	68	+}
7	69
8	70	enum {
9	71	UDP_BPF_IPV4,
@@ -11,7 +73,6 @@ enum { · 1 unmodified line
11	73	UDP_BPF_NUM_PROTS,
12	74	};
13	75
14		−static struct proto *udpv6_prot_saved __read_mostly;
15	76	static DEFINE_SPINLOCK(udpv6_prot_lock);
16	77	static struct proto udp_bpf_prots[UDP_BPF_NUM_PROTS];
17	78
@@ -20,6 +81,7 @@ static void udp_bpf_rebuild_protos(struct proto prot, const struct proto base) · 2 unmodified lines
20	81	prot = base;
21	82	prot->unhash = sock_map_unhash;
22	83	prot->close = sock_map_close;
	84	+ prot->recvmsg = udp_bpf_recvmsg;
23	85	}
24	86
25	87	static void udp_bpf_check_v6_needs_rebuild(struct proto *ops)
@@ -41,12 +103,23 @@ static int __init udp_bpf_v4_build_proto(void) · 15 unmodified lines
41	103	}
42	104	core_initcall(udp_bpf_v4_build_proto);
43	105
44		−struct proto udp_bpf_get_proto(struct sock sk, struct sk_psock *psock)
	106	+int udp_bpf_update_proto(struct sock *sk, bool restore)
45	107	{
46	108	int family = sk->sk_family == AF_INET ? UDP_BPF_IPV4 : UDP_BPF_IPV6;
	109	+ struct sk_psock *psock = sk_psock(sk);
	110	+
	111	+ if (restore) {
	112	+ sk->sk_write_space = psock->saved_write_space;
	113	+ /* Pairs with lockless read in sk_clone_lock() */
	114	+ WRITE_ONCE(sk->sk_prot, psock->sk_proto);
	115	+ return 0;
	116	+ }
47	117
48	118	if (sk->sk_family == AF_INET6)
49	119	udp_bpf_check_v6_needs_rebuild(psock->sk_proto);
50	120
51		− return &udp_bpf_prots[family];
	121	+ /* Pairs with lockless read in sk_clone_lock() */
	122	+ WRITE_ONCE(sk->sk_prot, &udp_bpf_prots[family]);
	123	+ return 0;
52	124	}
	125	+EXPORT_SYMBOL_GPL(udp_bpf_update_proto);

net/ipv6/af_inet6.c+1−0

@@ -714,6 +714,7 @@ const struct proto_ops inet6_dgram_ops = { · 713 unmodified lines
714	714	.getsockopt = sock_common_getsockopt, /* ok */
715	715	.sendmsg = inet6_sendmsg, /* retpoline's sake */
716	716	.recvmsg = inet6_recvmsg, /* retpoline's sake */
	717	+ .read_sock = udp_read_sock,
717	718	.mmap = sock_no_mmap,
718	719	.sendpage = sock_no_sendpage,
719	720	.set_peek_off = sk_set_peek_off,

net/ipv6/tcp_ipv6.c+3−0

@@ -2139,6 +2139,9 @@ struct proto tcpv6_prot = { · 2138 unmodified lines
2139	2139	.hash = inet6_hash,
2140	2140	.unhash = inet_unhash,
2141	2141	.get_port = inet_csk_get_port,
	2142	+#ifdef CONFIG_BPF_SYSCALL
	2143	+ .psock_update_sk_prot = tcp_bpf_update_proto,
	2144	+#endif
2142	2145	.enter_memory_pressure = tcp_enter_memory_pressure,
2143	2146	.leave_memory_pressure = tcp_leave_memory_pressure,
2144	2147	.stream_memory_free = tcp_stream_memory_free,

net/ipv6/udp.c+3−0

@@ -1713,6 +1713,9 @@ struct proto udpv6_prot = { · 1712 unmodified lines
1713	1713	.unhash = udp_lib_unhash,
1714	1714	.rehash = udp_v6_rehash,
1715	1715	.get_port = udp_v6_get_port,
	1716	+#ifdef CONFIG_BPF_SYSCALL
	1717	+ .psock_update_sk_prot = udp_bpf_update_proto,
	1718	+#endif
1716	1719	.memory_allocated = &udp_memory_allocated,
1717	1720	.sysctl_mem = sysctl_udp_mem,
1718	1721	.sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_udp_wmem_min),

net/tls/tls_sw.c+2−2

@@ -1789,8 +1789,8 @@ int tls_sw_recvmsg(struct sock *sk, · 1788 unmodified lines
1789	1789	skb = tls_wait_data(sk, psock, flags, timeo, &err);
1790	1790	if (!skb) {
1791	1791	if (psock) {
1792		− int ret = __tcp_bpf_recvmsg(sk, psock,
1793		− msg, len, flags);
	1792	+ int ret = sk_msg_recvmsg(sk, psock, msg, len,
	1793	+ flags);
1794	1794
1795	1795	if (ret > 0) {
1796	1796	decrypted += ret;

tools/bpf/bpftool/common.c+1−0

@@ -57,6 +57,7 @@ const char * const attach_type_name[__MAX_BPF_ATTACH_TYPE] = { · 56 unmodified lines
57	57
58	58	[BPF_SK_SKB_STREAM_PARSER] = "sk_skb_stream_parser",
59	59	[BPF_SK_SKB_STREAM_VERDICT] = "sk_skb_stream_verdict",
	60	+ [BPF_SK_SKB_VERDICT] = "sk_skb_verdict",
60	61	[BPF_SK_MSG_VERDICT] = "sk_msg_verdict",
61	62	[BPF_LIRC_MODE2] = "lirc_mode2",
62	63	[BPF_FLOW_DISSECTOR] = "flow_dissector",

tools/bpf/bpftool/prog.c+1−0

@@ -76,6 +76,7 @@ enum dump_mode { · 75 unmodified lines
76	76	static const char * const attach_type_strings[] = {
77	77	[BPF_SK_SKB_STREAM_PARSER] = "stream_parser",
78	78	[BPF_SK_SKB_STREAM_VERDICT] = "stream_verdict",
	79	+ [BPF_SK_SKB_VERDICT] = "skb_verdict",
79	80	[BPF_SK_MSG_VERDICT] = "msg_verdict",
80	81	[BPF_FLOW_DISSECTOR] = "flow_dissector",
81	82	[__MAX_BPF_ATTACH_TYPE] = NULL,

tools/include/uapi/linux/bpf.h+1−0

@@ -957,6 +957,7 @@ enum bpf_attach_type { · 956 unmodified lines
957	957	BPF_XDP_CPUMAP,
958	958	BPF_SK_LOOKUP,
959	959	BPF_XDP,
	960	+ BPF_SK_SKB_VERDICT,
960	961	__MAX_BPF_ATTACH_TYPE
961	962	};
962	963

tools/testing/selftests/bpf/prog_tests/sockmap_basic.c+40−0

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	7	#include "test_skmsg_load_helpers.skel.h"
8	8	#include "test_sockmap_update.skel.h"
9	9	#include "test_sockmap_invalid_update.skel.h"
	10	+#include "test_sockmap_skb_verdict_attach.skel.h"
10	11	#include "bpf_iter_sockmap.skel.h"
11	12
12	13	#define TCP_REPAIR 19 /* TCP sock is under repair right now */
@@ -281,6 +282,39 @@ out: · 268 unmodified lines
281	282	bpf_iter_sockmap__destroy(skel);
282	283	}
283	284
	285	+static void test_sockmap_skb_verdict_attach(enum bpf_attach_type first,
	286	+ enum bpf_attach_type second)
	287	+{
	288	+ struct test_sockmap_skb_verdict_attach *skel;
	289	+ int err, map, verdict;
	290	+
	291	+ skel = test_sockmap_skb_verdict_attach__open_and_load();
	292	+ if (CHECK_FAIL(!skel)) {
	293	+ perror("test_sockmap_skb_verdict_attach__open_and_load");
	294	+ return;
	295	+ }
	296	+
	297	+ verdict = bpf_program__fd(skel->progs.prog_skb_verdict);
	298	+ map = bpf_map__fd(skel->maps.sock_map);
	299	+
	300	+ err = bpf_prog_attach(verdict, map, first, 0);
	301	+ if (CHECK_FAIL(err)) {
	302	+ perror("bpf_prog_attach");
	303	+ goto out;
	304	+ }
	305	+
	306	+ err = bpf_prog_attach(verdict, map, second, 0);
	307	+ assert(err == -1 && errno == EBUSY);
	308	+
	309	+ err = bpf_prog_detach2(verdict, map, first);
	310	+ if (CHECK_FAIL(err)) {
	311	+ perror("bpf_prog_detach2");
	312	+ goto out;
	313	+ }
	314	+out:
	315	+ test_sockmap_skb_verdict_attach__destroy(skel);
	316	+}
	317	+
284	318	void test_sockmap_basic(void)
285	319	{
286	320	if (test__start_subtest("sockmap create_update_free"))
@@ -301,4 +335,10 @@ void test_sockmap_basic(void) · 14 unmodified lines
301	335	test_sockmap_copy(BPF_MAP_TYPE_SOCKMAP);
302	336	if (test__start_subtest("sockhash copy"))
303	337	test_sockmap_copy(BPF_MAP_TYPE_SOCKHASH);
	338	+ if (test__start_subtest("sockmap skb_verdict attach")) {
	339	+ test_sockmap_skb_verdict_attach(BPF_SK_SKB_VERDICT,
	340	+ BPF_SK_SKB_STREAM_VERDICT);
	341	+ test_sockmap_skb_verdict_attach(BPF_SK_SKB_STREAM_VERDICT,
	342	+ BPF_SK_SKB_VERDICT);
	343	+ }
304	344	}

tools/testing/selftests/bpf/prog_tests/sockmap_listen.c+136−0

@@ -1603,6 +1603,141 @@ static void test_reuseport(struct test_sockmap_listen *skel, · 1602 unmodified lines
1603	1603	}
1604	1604	}
1605	1605
	1606	+static void udp_redir_to_connected(int family, int sotype, int sock_mapfd,
	1607	+ int verd_mapfd, enum redir_mode mode)
	1608	+{
	1609	+ const char *log_prefix = redir_mode_str(mode);
	1610	+ struct sockaddr_storage addr;
	1611	+ int c0, c1, p0, p1;
	1612	+ unsigned int pass;
	1613	+ socklen_t len;
	1614	+ int err, n;
	1615	+ u64 value;
	1616	+ u32 key;
	1617	+ char b;
	1618	+
	1619	+ zero_verdict_count(verd_mapfd);
	1620	+
	1621	+ p0 = socket_loopback(family, sotype \| SOCK_NONBLOCK);
	1622	+ if (p0 < 0)
	1623	+ return;
	1624	+ len = sizeof(addr);
	1625	+ err = xgetsockname(p0, sockaddr(&addr), &len);
	1626	+ if (err)
	1627	+ goto close_peer0;
	1628	+
	1629	+ c0 = xsocket(family, sotype \| SOCK_NONBLOCK, 0);
	1630	+ if (c0 < 0)
	1631	+ goto close_peer0;
	1632	+ err = xconnect(c0, sockaddr(&addr), len);
	1633	+ if (err)
	1634	+ goto close_cli0;
	1635	+ err = xgetsockname(c0, sockaddr(&addr), &len);
	1636	+ if (err)
	1637	+ goto close_cli0;
	1638	+ err = xconnect(p0, sockaddr(&addr), len);
	1639	+ if (err)
	1640	+ goto close_cli0;
	1641	+
	1642	+ p1 = socket_loopback(family, sotype \| SOCK_NONBLOCK);
	1643	+ if (p1 < 0)
	1644	+ goto close_cli0;
	1645	+ err = xgetsockname(p1, sockaddr(&addr), &len);
	1646	+ if (err)
	1647	+ goto close_cli0;
	1648	+
	1649	+ c1 = xsocket(family, sotype \| SOCK_NONBLOCK, 0);
	1650	+ if (c1 < 0)
	1651	+ goto close_peer1;
	1652	+ err = xconnect(c1, sockaddr(&addr), len);
	1653	+ if (err)
	1654	+ goto close_cli1;
	1655	+ err = xgetsockname(c1, sockaddr(&addr), &len);
	1656	+ if (err)
	1657	+ goto close_cli1;
	1658	+ err = xconnect(p1, sockaddr(&addr), len);
	1659	+ if (err)
	1660	+ goto close_cli1;
	1661	+
	1662	+ key = 0;
	1663	+ value = p0;
	1664	+ err = xbpf_map_update_elem(sock_mapfd, &key, &value, BPF_NOEXIST);
	1665	+ if (err)
	1666	+ goto close_cli1;
	1667	+
	1668	+ key = 1;
	1669	+ value = p1;
	1670	+ err = xbpf_map_update_elem(sock_mapfd, &key, &value, BPF_NOEXIST);
	1671	+ if (err)
	1672	+ goto close_cli1;
	1673	+
	1674	+ n = write(c1, "a", 1);
	1675	+ if (n < 0)
	1676	+ FAIL_ERRNO("%s: write", log_prefix);
	1677	+ if (n == 0)
	1678	+ FAIL("%s: incomplete write", log_prefix);
	1679	+ if (n < 1)
	1680	+ goto close_cli1;
	1681	+
	1682	+ key = SK_PASS;
	1683	+ err = xbpf_map_lookup_elem(verd_mapfd, &key, &pass);
	1684	+ if (err)
	1685	+ goto close_cli1;
	1686	+ if (pass != 1)
	1687	+ FAIL("%s: want pass count 1, have %d", log_prefix, pass);
	1688	+
	1689	+ n = read(mode == REDIR_INGRESS ? p0 : c0, &b, 1);
	1690	+ if (n < 0)
	1691	+ FAIL_ERRNO("%s: read", log_prefix);
	1692	+ if (n == 0)
	1693	+ FAIL("%s: incomplete read", log_prefix);
	1694	+
	1695	+close_cli1:
	1696	+ xclose(c1);
	1697	+close_peer1:
	1698	+ xclose(p1);
	1699	+close_cli0:
	1700	+ xclose(c0);
	1701	+close_peer0:
	1702	+ xclose(p0);
	1703	+}
	1704	+
	1705	+static void udp_skb_redir_to_connected(struct test_sockmap_listen *skel,
	1706	+ struct bpf_map *inner_map, int family)
	1707	+{
	1708	+ int verdict = bpf_program__fd(skel->progs.prog_skb_verdict);
	1709	+ int verdict_map = bpf_map__fd(skel->maps.verdict_map);
	1710	+ int sock_map = bpf_map__fd(inner_map);
	1711	+ int err;
	1712	+
	1713	+ err = xbpf_prog_attach(verdict, sock_map, BPF_SK_SKB_VERDICT, 0);
	1714	+ if (err)
	1715	+ return;
	1716	+
	1717	+ skel->bss->test_ingress = false;
	1718	+ udp_redir_to_connected(family, SOCK_DGRAM, sock_map, verdict_map,
	1719	+ REDIR_EGRESS);
	1720	+ skel->bss->test_ingress = true;
	1721	+ udp_redir_to_connected(family, SOCK_DGRAM, sock_map, verdict_map,
	1722	+ REDIR_INGRESS);
	1723	+
	1724	+ xbpf_prog_detach2(verdict, sock_map, BPF_SK_SKB_VERDICT);
	1725	+}
	1726	+
	1727	+static void test_udp_redir(struct test_sockmap_listen skel, struct bpf_map map,
	1728	+ int family)
	1729	+{
	1730	+ const char family_name, map_name;
	1731	+ char s[MAX_TEST_NAME];
	1732	+
	1733	+ family_name = family_str(family);
	1734	+ map_name = map_type_str(map);
	1735	+ snprintf(s, sizeof(s), "%s %s %s", map_name, family_name, __func__);
	1736	+ if (!test__start_subtest(s))
	1737	+ return;
	1738	+ udp_skb_redir_to_connected(skel, map, family);
	1739	+}
	1740	+
1606	1741	static void run_tests(struct test_sockmap_listen skel, struct bpf_map map,
1607	1742	int family)
1608	1743	{
@@ -1611,6 +1746,7 @@ static void run_tests(struct test_sockmap_listen skel, struct bpf_map map, · 2 unmodified lines
1611	1746	test_redir(skel, map, family, SOCK_STREAM);
1612	1747	test_reuseport(skel, map, family, SOCK_STREAM);
1613	1748	test_reuseport(skel, map, family, SOCK_DGRAM);
	1749	+ test_udp_redir(skel, map, family);
1614	1750	}
1615	1751
1616	1752	void test_sockmap_listen(void)

tools/testing/selftests/bpf/progs/test_sockmap_listen.c+22−0

@@ -29,6 +29,7 @@ struct { · 28 unmodified lines
29	29	} verdict_map SEC(".maps");
30	30
31	31	static volatile bool test_sockmap; /* toggled by user-space */
	32	+static volatile bool test_ingress; /* toggled by user-space */
32	33
33	34	SEC("sk_skb/stream_parser")
34	35	int prog_stream_parser(struct __sk_buff *skb)
@@ -55,6 +56,27 @@ int prog_stream_verdict(struct __sk_buff *skb) · 20 unmodified lines
55	56	return verdict;
56	57	}
57	58
	59	+SEC("sk_skb/skb_verdict")
	60	+int prog_skb_verdict(struct __sk_buff *skb)
	61	+{
	62	+ unsigned int *count;
	63	+ __u32 zero = 0;
	64	+ int verdict;
	65	+
	66	+ if (test_sockmap)
	67	+ verdict = bpf_sk_redirect_map(skb, &sock_map, zero,
	68	+ test_ingress ? BPF_F_INGRESS : 0);
	69	+ else
	70	+ verdict = bpf_sk_redirect_hash(skb, &sock_hash, &zero,
	71	+ test_ingress ? BPF_F_INGRESS : 0);
	72	+
	73	+ count = bpf_map_lookup_elem(&verdict_map, &verdict);
	74	+ if (count)
	75	+ (*count)++;
	76	+
	77	+ return verdict;
	78	+}
	79	+
58	80	SEC("sk_msg")
59	81	int prog_msg_verdict(struct sk_msg_md *msg)
60	82	{

tools/testing/selftests/bpf/progs/test_sockmap_skb_verdict_attach.c+18−0

@@ -0,0 +1,18 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+#include "vmlinux.h"
	3	+#include <bpf/bpf_helpers.h>
	4	+
	5	+struct {
	6	+ __uint(type, BPF_MAP_TYPE_SOCKMAP);
	7	+ __uint(max_entries, 2);
	8	+ __type(key, __u32);
	9	+ __type(value, __u64);
	10	+} sock_map SEC(".maps");
	11	+
	12	+SEC("sk_skb/skb_verdict")
	13	+int prog_skb_verdict(struct __sk_buff *skb)
	14	+{
	15	+ return SK_DROP;
	16	+}
	17	+
	18	+char _license[] SEC("license") = "GPL";

Merge branch 'sockmap: introduce BPF_SK_SKB_VERDICT and support UDP'

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code