2473 lines · 59.2 KB · C
RawDownload
// SPDX-License-Identifier: GPL-2.0/* Multipath TCP * * Copyright (c) 2017 - 2019, Intel Corporation. */ #define pr_fmt(fmt) "MPTCP: " fmt #include <linux/kernel.h>#include <linux/module.h>#include <linux/netdevice.h>#include <linux/sched/signal.h>#include <linux/atomic.h>#include <net/sock.h>#include <net/inet_common.h>#include <net/inet_hashtables.h>#include <net/protocol.h>#include <net/tcp.h>#include <net/tcp_states.h>#if IS_ENABLED(CONFIG_MPTCP_IPV6)#include <net/transp_v6.h>#endif#include <net/mptcp.h>#include "protocol.h"#include "mib.h" #define MPTCP_SAME_STATE TCP_MAX_STATES #if IS_ENABLED(CONFIG_MPTCP_IPV6)struct mptcp6_sock {	struct mptcp_sock msk;	struct ipv6_pinfo np;};#endif struct mptcp_skb_cb {	u32 offset;}; #define MPTCP_SKB_CB(__skb)	((struct mptcp_skb_cb *)&((__skb)->cb[0])) static struct percpu_counter mptcp_sockets_allocated; /* If msk has an initial subflow socket, and the MP_CAPABLE handshake has not * completed yet or has failed, return the subflow socket. * Otherwise return NULL. */static struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk){	if (!msk->subflow || READ_ONCE(msk->can_ack))		return NULL; 	return msk->subflow;} static bool mptcp_is_tcpsk(struct sock *sk){	struct socket *sock = sk->sk_socket; 	if (unlikely(sk->sk_prot == &tcp_prot)) {		/* we are being invoked after mptcp_accept() has		 * accepted a non-mp-capable flow: sk is a tcp_sk,		 * not an mptcp one.		 *		 * Hand the socket over to tcp so all further socket ops		 * bypass mptcp.		 */		sock->ops = &inet_stream_ops;		return true;#if IS_ENABLED(CONFIG_MPTCP_IPV6)	} else if (unlikely(sk->sk_prot == &tcpv6_prot)) {		sock->ops = &inet6_stream_ops;		return true;#endif	} 	return false;} static struct sock *__mptcp_tcp_fallback(struct mptcp_sock *msk){	sock_owned_by_me((const struct sock *)msk); 	if (likely(!__mptcp_check_fallback(msk)))		return NULL; 	return msk->first;} static int __mptcp_socket_create(struct mptcp_sock *msk){	struct mptcp_subflow_context *subflow;	struct sock *sk = (struct sock *)msk;	struct socket *ssock;	int err; 	err = mptcp_subflow_create_socket(sk, &ssock);	if (err)		return err; 	msk->first = ssock->sk;	msk->subflow = ssock;	subflow = mptcp_subflow_ctx(ssock->sk);	list_add(&subflow->node, &msk->conn_list);	subflow->request_mptcp = 1; 	/* accept() will wait on first subflow sk_wq, and we always wakes up	 * via msk->sk_socket	 */	RCU_INIT_POINTER(msk->first->sk_wq, &sk->sk_socket->wq); 	return 0;} static void __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk,			     struct sk_buff *skb,			     unsigned int offset, size_t copy_len){	struct sock *sk = (struct sock *)msk;	struct sk_buff *tail; 	__skb_unlink(skb, &ssk->sk_receive_queue); 	skb_ext_reset(skb);	skb_orphan(skb);	msk->ack_seq += copy_len; 	tail = skb_peek_tail(&sk->sk_receive_queue);	if (offset == 0 && tail) {		bool fragstolen;		int delta; 		if (skb_try_coalesce(tail, skb, &fragstolen, &delta)) {			kfree_skb_partial(skb, fragstolen);			atomic_add(delta, &sk->sk_rmem_alloc);			sk_mem_charge(sk, delta);			return;		}	} 	skb_set_owner_r(skb, sk);	__skb_queue_tail(&sk->sk_receive_queue, skb);	MPTCP_SKB_CB(skb)->offset = offset;} static void mptcp_stop_timer(struct sock *sk){	struct inet_connection_sock *icsk = inet_csk(sk); 	sk_stop_timer(sk, &icsk->icsk_retransmit_timer);	mptcp_sk(sk)->timer_ival = 0;} /* both sockets must be locked */static bool mptcp_subflow_dsn_valid(const struct mptcp_sock *msk,				    struct sock *ssk){	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);	u64 dsn = mptcp_subflow_get_mapped_dsn(subflow); 	/* revalidate data sequence number.	 *	 * mptcp_subflow_data_available() is usually called	 * without msk lock.  Its unlikely (but possible)	 * that msk->ack_seq has been advanced since the last	 * call found in-sequence data.	 */	if (likely(dsn == msk->ack_seq))		return true; 	subflow->data_avail = 0;	return mptcp_subflow_data_available(ssk);} static void mptcp_check_data_fin_ack(struct sock *sk){	struct mptcp_sock *msk = mptcp_sk(sk); 	if (__mptcp_check_fallback(msk))		return; 	/* Look for an acknowledged DATA_FIN */	if (((1 << sk->sk_state) &	     (TCPF_FIN_WAIT1 | TCPF_CLOSING | TCPF_LAST_ACK)) &&	    msk->write_seq == atomic64_read(&msk->snd_una)) {		mptcp_stop_timer(sk); 		WRITE_ONCE(msk->snd_data_fin_enable, 0); 		switch (sk->sk_state) {		case TCP_FIN_WAIT1:			inet_sk_state_store(sk, TCP_FIN_WAIT2);			sk->sk_state_change(sk);			break;		case TCP_CLOSING:			fallthrough;		case TCP_LAST_ACK:			inet_sk_state_store(sk, TCP_CLOSE);			sk->sk_state_change(sk);			break;		} 		if (sk->sk_shutdown == SHUTDOWN_MASK ||		    sk->sk_state == TCP_CLOSE)			sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);		else			sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);	}} static bool mptcp_pending_data_fin(struct sock *sk, u64 *seq){	struct mptcp_sock *msk = mptcp_sk(sk); 	if (READ_ONCE(msk->rcv_data_fin) &&	    ((1 << sk->sk_state) &	     (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2))) {		u64 rcv_data_fin_seq = READ_ONCE(msk->rcv_data_fin_seq); 		if (msk->ack_seq == rcv_data_fin_seq) {			if (seq)				*seq = rcv_data_fin_seq; 			return true;		}	} 	return false;} static void mptcp_set_timeout(const struct sock *sk, const struct sock *ssk){	long tout = ssk && inet_csk(ssk)->icsk_pending ?				      inet_csk(ssk)->icsk_timeout - jiffies : 0; 	if (tout <= 0)		tout = mptcp_sk(sk)->timer_ival;	mptcp_sk(sk)->timer_ival = tout > 0 ? tout : TCP_RTO_MIN;} static void mptcp_check_data_fin(struct sock *sk){	struct mptcp_sock *msk = mptcp_sk(sk);	u64 rcv_data_fin_seq; 	if (__mptcp_check_fallback(msk) || !msk->first)		return; 	/* Need to ack a DATA_FIN received from a peer while this side	 * of the connection is in ESTABLISHED, FIN_WAIT1, or FIN_WAIT2.	 * msk->rcv_data_fin was set when parsing the incoming options	 * at the subflow level and the msk lock was not held, so this	 * is the first opportunity to act on the DATA_FIN and change	 * the msk state.	 *	 * If we are caught up to the sequence number of the incoming	 * DATA_FIN, send the DATA_ACK now and do state transition.  If	 * not caught up, do nothing and let the recv code send DATA_ACK	 * when catching up.	 */ 	if (mptcp_pending_data_fin(sk, &rcv_data_fin_seq)) {		struct mptcp_subflow_context *subflow; 		msk->ack_seq++;		WRITE_ONCE(msk->rcv_data_fin, 0); 		sk->sk_shutdown |= RCV_SHUTDOWN;		smp_mb__before_atomic(); /* SHUTDOWN must be visible first */		set_bit(MPTCP_DATA_READY, &msk->flags); 		switch (sk->sk_state) {		case TCP_ESTABLISHED:			inet_sk_state_store(sk, TCP_CLOSE_WAIT);			break;		case TCP_FIN_WAIT1:			inet_sk_state_store(sk, TCP_CLOSING);			break;		case TCP_FIN_WAIT2:			inet_sk_state_store(sk, TCP_CLOSE);			// @@ Close subflows now?			break;		default:			/* Other states not expected */			WARN_ON_ONCE(1);			break;		} 		mptcp_set_timeout(sk, NULL);		mptcp_for_each_subflow(msk, subflow) {			struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 			lock_sock(ssk);			tcp_send_ack(ssk);			release_sock(ssk);		} 		sk->sk_state_change(sk); 		if (sk->sk_shutdown == SHUTDOWN_MASK ||		    sk->sk_state == TCP_CLOSE)			sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);		else			sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);	}} static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk,					   struct sock *ssk,					   unsigned int *bytes){	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);	struct sock *sk = (struct sock *)msk;	unsigned int moved = 0;	bool more_data_avail;	struct tcp_sock *tp;	bool done = false; 	if (!mptcp_subflow_dsn_valid(msk, ssk)) {		*bytes = 0;		return false;	} 	tp = tcp_sk(ssk);	do {		u32 map_remaining, offset;		u32 seq = tp->copied_seq;		struct sk_buff *skb;		bool fin; 		/* try to move as much data as available */		map_remaining = subflow->map_data_len -				mptcp_subflow_get_map_offset(subflow); 		skb = skb_peek(&ssk->sk_receive_queue);		if (!skb)			break; 		if (__mptcp_check_fallback(msk)) {			/* if we are running under the workqueue, TCP could have			 * collapsed skbs between dummy map creation and now			 * be sure to adjust the size			 */			map_remaining = skb->len;			subflow->map_data_len = skb->len;		} 		offset = seq - TCP_SKB_CB(skb)->seq;		fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN;		if (fin) {			done = true;			seq++;		} 		if (offset < skb->len) {			size_t len = skb->len - offset; 			if (tp->urg_data)				done = true; 			__mptcp_move_skb(msk, ssk, skb, offset, len);			seq += len;			moved += len; 			if (WARN_ON_ONCE(map_remaining < len))				break;		} else {			WARN_ON_ONCE(!fin);			sk_eat_skb(ssk, skb);			done = true;		} 		WRITE_ONCE(tp->copied_seq, seq);		more_data_avail = mptcp_subflow_data_available(ssk); 		if (atomic_read(&sk->sk_rmem_alloc) > READ_ONCE(sk->sk_rcvbuf)) {			done = true;			break;		}	} while (more_data_avail); 	*bytes = moved; 	/* If the moves have caught up with the DATA_FIN sequence number	 * it's time to ack the DATA_FIN and change socket state, but	 * this is not a good place to change state. Let the workqueue	 * do it.	 */	if (mptcp_pending_data_fin(sk, NULL) &&	    schedule_work(&msk->work))		sock_hold(sk); 	return done;} /* In most cases we will be able to lock the mptcp socket.  If its already * owned, we need to defer to the work queue to avoid ABBA deadlock. */static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk){	struct sock *sk = (struct sock *)msk;	unsigned int moved = 0; 	if (READ_ONCE(sk->sk_lock.owned))		return false; 	if (unlikely(!spin_trylock_bh(&sk->sk_lock.slock)))		return false; 	/* must re-check after taking the lock */	if (!READ_ONCE(sk->sk_lock.owned))		__mptcp_move_skbs_from_subflow(msk, ssk, &moved); 	spin_unlock_bh(&sk->sk_lock.slock); 	return moved > 0;} void mptcp_data_ready(struct sock *sk, struct sock *ssk){	struct mptcp_sock *msk = mptcp_sk(sk); 	set_bit(MPTCP_DATA_READY, &msk->flags); 	if (atomic_read(&sk->sk_rmem_alloc) < READ_ONCE(sk->sk_rcvbuf) &&	    move_skbs_to_msk(msk, ssk))		goto wake; 	/* don't schedule if mptcp sk is (still) over limit */	if (atomic_read(&sk->sk_rmem_alloc) > READ_ONCE(sk->sk_rcvbuf))		goto wake; 	/* mptcp socket is owned, release_cb should retry */	if (!test_and_set_bit(TCP_DELACK_TIMER_DEFERRED,			      &sk->sk_tsq_flags)) {		sock_hold(sk); 		/* need to try again, its possible release_cb() has already		 * been called after the test_and_set_bit() above.		 */		move_skbs_to_msk(msk, ssk);	}wake:	sk->sk_data_ready(sk);} static void __mptcp_flush_join_list(struct mptcp_sock *msk){	if (likely(list_empty(&msk->join_list)))		return; 	spin_lock_bh(&msk->join_list_lock);	list_splice_tail_init(&msk->join_list, &msk->conn_list);	spin_unlock_bh(&msk->join_list_lock);} static bool mptcp_timer_pending(struct sock *sk){	return timer_pending(&inet_csk(sk)->icsk_retransmit_timer);} static void mptcp_reset_timer(struct sock *sk){	struct inet_connection_sock *icsk = inet_csk(sk);	unsigned long tout; 	/* should never be called with mptcp level timer cleared */	tout = READ_ONCE(mptcp_sk(sk)->timer_ival);	if (WARN_ON_ONCE(!tout))		tout = TCP_RTO_MIN;	sk_reset_timer(sk, &icsk->icsk_retransmit_timer, jiffies + tout);} void mptcp_data_acked(struct sock *sk){	mptcp_reset_timer(sk); 	if ((!sk_stream_is_writeable(sk) ||	     (inet_sk_state_load(sk) != TCP_ESTABLISHED)) &&	    schedule_work(&mptcp_sk(sk)->work))		sock_hold(sk);} void mptcp_subflow_eof(struct sock *sk){	struct mptcp_sock *msk = mptcp_sk(sk); 	if (!test_and_set_bit(MPTCP_WORK_EOF, &msk->flags) &&	    schedule_work(&msk->work))		sock_hold(sk);} static void mptcp_check_for_eof(struct mptcp_sock *msk){	struct mptcp_subflow_context *subflow;	struct sock *sk = (struct sock *)msk;	int receivers = 0; 	mptcp_for_each_subflow(msk, subflow)		receivers += !subflow->rx_eof;

Lines 1–500 of 2473.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t