// SPDX-License-Identifier: GPL-2.0/* Multipath TCP * * Copyright (c) 2017 - 2019, Intel Corporation. */ #define pr_fmt(fmt) "MPTCP: " fmt #include <linux/kernel.h>#include <linux/module.h>#include <linux/netdevice.h>#include <linux/sched/signal.h>#include <linux/atomic.h>#include <net/sock.h>#include <net/inet_common.h>#include <net/inet_hashtables.h>#include <net/protocol.h>#include <net/tcp.h>#include <net/tcp_states.h>#if IS_ENABLED(CONFIG_MPTCP_IPV6)#include <net/transp_v6.h>#endif#include <net/mptcp.h>#include "protocol.h"#include "mib.h" #define MPTCP_SAME_STATE TCP_MAX_STATES #if IS_ENABLED(CONFIG_MPTCP_IPV6)struct mptcp6_sock { struct mptcp_sock msk; struct ipv6_pinfo np;};#endif struct mptcp_skb_cb { u32 offset;}; #define MPTCP_SKB_CB(__skb) ((struct mptcp_skb_cb *)&((__skb)->cb[0])) static struct percpu_counter mptcp_sockets_allocated; /* If msk has an initial subflow socket, and the MP_CAPABLE handshake has not * completed yet or has failed, return the subflow socket. * Otherwise return NULL. */static struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk){ if (!msk->subflow || READ_ONCE(msk->can_ack)) return NULL; return msk->subflow;} static bool mptcp_is_tcpsk(struct sock *sk){ struct socket *sock = sk->sk_socket; if (unlikely(sk->sk_prot == &tcp_prot)) { /* we are being invoked after mptcp_accept() has * accepted a non-mp-capable flow: sk is a tcp_sk, * not an mptcp one. * * Hand the socket over to tcp so all further socket ops * bypass mptcp. */ sock->ops = &inet_stream_ops; return true;#if IS_ENABLED(CONFIG_MPTCP_IPV6) } else if (unlikely(sk->sk_prot == &tcpv6_prot)) { sock->ops = &inet6_stream_ops; return true;#endif } return false;} static struct sock *__mptcp_tcp_fallback(struct mptcp_sock *msk){ sock_owned_by_me((const struct sock *)msk); if (likely(!__mptcp_check_fallback(msk))) return NULL; return msk->first;} static int __mptcp_socket_create(struct mptcp_sock *msk){ struct mptcp_subflow_context *subflow; struct sock *sk = (struct sock *)msk; struct socket *ssock; int err; err = mptcp_subflow_create_socket(sk, &ssock); if (err) return err; msk->first = ssock->sk; msk->subflow = ssock; subflow = mptcp_subflow_ctx(ssock->sk); list_add(&subflow->node, &msk->conn_list); subflow->request_mptcp = 1; /* accept() will wait on first subflow sk_wq, and we always wakes up * via msk->sk_socket */ RCU_INIT_POINTER(msk->first->sk_wq, &sk->sk_socket->wq); return 0;} static void __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk, struct sk_buff *skb, unsigned int offset, size_t copy_len){ struct sock *sk = (struct sock *)msk; struct sk_buff *tail; __skb_unlink(skb, &ssk->sk_receive_queue); skb_ext_reset(skb); skb_orphan(skb); msk->ack_seq += copy_len; tail = skb_peek_tail(&sk->sk_receive_queue); if (offset == 0 && tail) { bool fragstolen; int delta; if (skb_try_coalesce(tail, skb, &fragstolen, &delta)) { kfree_skb_partial(skb, fragstolen); atomic_add(delta, &sk->sk_rmem_alloc); sk_mem_charge(sk, delta); return; } } skb_set_owner_r(skb, sk); __skb_queue_tail(&sk->sk_receive_queue, skb); MPTCP_SKB_CB(skb)->offset = offset;} static void mptcp_stop_timer(struct sock *sk){ struct inet_connection_sock *icsk = inet_csk(sk); sk_stop_timer(sk, &icsk->icsk_retransmit_timer); mptcp_sk(sk)->timer_ival = 0;} /* both sockets must be locked */static bool mptcp_subflow_dsn_valid(const struct mptcp_sock *msk, struct sock *ssk){ struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); u64 dsn = mptcp_subflow_get_mapped_dsn(subflow); /* revalidate data sequence number. * * mptcp_subflow_data_available() is usually called * without msk lock. Its unlikely (but possible) * that msk->ack_seq has been advanced since the last * call found in-sequence data. */ if (likely(dsn == msk->ack_seq)) return true; subflow->data_avail = 0; return mptcp_subflow_data_available(ssk);} static void mptcp_check_data_fin_ack(struct sock *sk){ struct mptcp_sock *msk = mptcp_sk(sk); if (__mptcp_check_fallback(msk)) return; /* Look for an acknowledged DATA_FIN */ if (((1 << sk->sk_state) & (TCPF_FIN_WAIT1 | TCPF_CLOSING | TCPF_LAST_ACK)) && msk->write_seq == atomic64_read(&msk->snd_una)) { mptcp_stop_timer(sk); WRITE_ONCE(msk->snd_data_fin_enable, 0); switch (sk->sk_state) { case TCP_FIN_WAIT1: inet_sk_state_store(sk, TCP_FIN_WAIT2); sk->sk_state_change(sk); break; case TCP_CLOSING: fallthrough; case TCP_LAST_ACK: inet_sk_state_store(sk, TCP_CLOSE); sk->sk_state_change(sk); break; } if (sk->sk_shutdown == SHUTDOWN_MASK || sk->sk_state == TCP_CLOSE) sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP); else sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); }} static bool mptcp_pending_data_fin(struct sock *sk, u64 *seq){ struct mptcp_sock *msk = mptcp_sk(sk); if (READ_ONCE(msk->rcv_data_fin) && ((1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2))) { u64 rcv_data_fin_seq = READ_ONCE(msk->rcv_data_fin_seq); if (msk->ack_seq == rcv_data_fin_seq) { if (seq) *seq = rcv_data_fin_seq; return true; } } return false;} static void mptcp_set_timeout(const struct sock *sk, const struct sock *ssk){ long tout = ssk && inet_csk(ssk)->icsk_pending ? inet_csk(ssk)->icsk_timeout - jiffies : 0; if (tout <= 0) tout = mptcp_sk(sk)->timer_ival; mptcp_sk(sk)->timer_ival = tout > 0 ? tout : TCP_RTO_MIN;} static void mptcp_check_data_fin(struct sock *sk){ struct mptcp_sock *msk = mptcp_sk(sk); u64 rcv_data_fin_seq; if (__mptcp_check_fallback(msk) || !msk->first) return; /* Need to ack a DATA_FIN received from a peer while this side * of the connection is in ESTABLISHED, FIN_WAIT1, or FIN_WAIT2. * msk->rcv_data_fin was set when parsing the incoming options * at the subflow level and the msk lock was not held, so this * is the first opportunity to act on the DATA_FIN and change * the msk state. * * If we are caught up to the sequence number of the incoming * DATA_FIN, send the DATA_ACK now and do state transition. If * not caught up, do nothing and let the recv code send DATA_ACK * when catching up. */ if (mptcp_pending_data_fin(sk, &rcv_data_fin_seq)) { struct mptcp_subflow_context *subflow; msk->ack_seq++; WRITE_ONCE(msk->rcv_data_fin, 0); sk->sk_shutdown |= RCV_SHUTDOWN; smp_mb__before_atomic(); /* SHUTDOWN must be visible first */ set_bit(MPTCP_DATA_READY, &msk->flags); switch (sk->sk_state) { case TCP_ESTABLISHED: inet_sk_state_store(sk, TCP_CLOSE_WAIT); break; case TCP_FIN_WAIT1: inet_sk_state_store(sk, TCP_CLOSING); break; case TCP_FIN_WAIT2: inet_sk_state_store(sk, TCP_CLOSE); // @@ Close subflows now? break; default: /* Other states not expected */ WARN_ON_ONCE(1); break; } mptcp_set_timeout(sk, NULL); mptcp_for_each_subflow(msk, subflow) { struct sock *ssk = mptcp_subflow_tcp_sock(subflow); lock_sock(ssk); tcp_send_ack(ssk); release_sock(ssk); } sk->sk_state_change(sk); if (sk->sk_shutdown == SHUTDOWN_MASK || sk->sk_state == TCP_CLOSE) sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP); else sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); }} static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk, struct sock *ssk, unsigned int *bytes){ struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); struct sock *sk = (struct sock *)msk; unsigned int moved = 0; bool more_data_avail; struct tcp_sock *tp; bool done = false; if (!mptcp_subflow_dsn_valid(msk, ssk)) { *bytes = 0; return false; } tp = tcp_sk(ssk); do { u32 map_remaining, offset; u32 seq = tp->copied_seq; struct sk_buff *skb; bool fin; /* try to move as much data as available */ map_remaining = subflow->map_data_len - mptcp_subflow_get_map_offset(subflow); skb = skb_peek(&ssk->sk_receive_queue); if (!skb) break; if (__mptcp_check_fallback(msk)) { /* if we are running under the workqueue, TCP could have * collapsed skbs between dummy map creation and now * be sure to adjust the size */ map_remaining = skb->len; subflow->map_data_len = skb->len; } offset = seq - TCP_SKB_CB(skb)->seq; fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN; if (fin) { done = true; seq++; } if (offset < skb->len) { size_t len = skb->len - offset; if (tp->urg_data) done = true; __mptcp_move_skb(msk, ssk, skb, offset, len); seq += len; moved += len; if (WARN_ON_ONCE(map_remaining < len)) break; } else { WARN_ON_ONCE(!fin); sk_eat_skb(ssk, skb); done = true; } WRITE_ONCE(tp->copied_seq, seq); more_data_avail = mptcp_subflow_data_available(ssk); if (atomic_read(&sk->sk_rmem_alloc) > READ_ONCE(sk->sk_rcvbuf)) { done = true; break; } } while (more_data_avail); *bytes = moved; /* If the moves have caught up with the DATA_FIN sequence number * it's time to ack the DATA_FIN and change socket state, but * this is not a good place to change state. Let the workqueue * do it. */ if (mptcp_pending_data_fin(sk, NULL) && schedule_work(&msk->work)) sock_hold(sk); return done;} /* In most cases we will be able to lock the mptcp socket. If its already * owned, we need to defer to the work queue to avoid ABBA deadlock. */static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk){ struct sock *sk = (struct sock *)msk; unsigned int moved = 0; if (READ_ONCE(sk->sk_lock.owned)) return false; if (unlikely(!spin_trylock_bh(&sk->sk_lock.slock))) return false; /* must re-check after taking the lock */ if (!READ_ONCE(sk->sk_lock.owned)) __mptcp_move_skbs_from_subflow(msk, ssk, &moved); spin_unlock_bh(&sk->sk_lock.slock); return moved > 0;} void mptcp_data_ready(struct sock *sk, struct sock *ssk){ struct mptcp_sock *msk = mptcp_sk(sk); set_bit(MPTCP_DATA_READY, &msk->flags); if (atomic_read(&sk->sk_rmem_alloc) < READ_ONCE(sk->sk_rcvbuf) && move_skbs_to_msk(msk, ssk)) goto wake; /* don't schedule if mptcp sk is (still) over limit */ if (atomic_read(&sk->sk_rmem_alloc) > READ_ONCE(sk->sk_rcvbuf)) goto wake; /* mptcp socket is owned, release_cb should retry */ if (!test_and_set_bit(TCP_DELACK_TIMER_DEFERRED, &sk->sk_tsq_flags)) { sock_hold(sk); /* need to try again, its possible release_cb() has already * been called after the test_and_set_bit() above. */ move_skbs_to_msk(msk, ssk); }wake: sk->sk_data_ready(sk);} static void __mptcp_flush_join_list(struct mptcp_sock *msk){ if (likely(list_empty(&msk->join_list))) return; spin_lock_bh(&msk->join_list_lock); list_splice_tail_init(&msk->join_list, &msk->conn_list); spin_unlock_bh(&msk->join_list_lock);} static bool mptcp_timer_pending(struct sock *sk){ return timer_pending(&inet_csk(sk)->icsk_retransmit_timer);} static void mptcp_reset_timer(struct sock *sk){ struct inet_connection_sock *icsk = inet_csk(sk); unsigned long tout; /* should never be called with mptcp level timer cleared */ tout = READ_ONCE(mptcp_sk(sk)->timer_ival); if (WARN_ON_ONCE(!tout)) tout = TCP_RTO_MIN; sk_reset_timer(sk, &icsk->icsk_retransmit_timer, jiffies + tout);} void mptcp_data_acked(struct sock *sk){ mptcp_reset_timer(sk); if ((!sk_stream_is_writeable(sk) || (inet_sk_state_load(sk) != TCP_ESTABLISHED)) && schedule_work(&mptcp_sk(sk)->work)) sock_hold(sk);} void mptcp_subflow_eof(struct sock *sk){ struct mptcp_sock *msk = mptcp_sk(sk); if (!test_and_set_bit(MPTCP_WORK_EOF, &msk->flags) && schedule_work(&msk->work)) sock_hold(sk);} static void mptcp_check_for_eof(struct mptcp_sock *msk){ struct mptcp_subflow_context *subflow; struct sock *sk = (struct sock *)msk; int receivers = 0; mptcp_for_each_subflow(msk, subflow) receivers += !subflow->rx_eof;Lines 1–500 of 2473. Load more
// SPDX-License-Identifier: GPL-2.0 /* Multipath TCP * * Copyright (c) 2017 - 2019, Intel Corporation. */ #define pr_fmt(fmt) "MPTCP: " fmt #include <linux/kernel.h> #include <linux/module.h> #include <linux/netdevice.h> #include <linux/sched/signal.h> #include <linux/atomic.h> #include <net/sock.h> #include <net/inet_common.h> #include <net/inet_hashtables.h> #include <net/protocol.h> #include <net/tcp.h> #include <net/tcp_states.h> #if IS_ENABLED(CONFIG_MPTCP_IPV6) #include <net/transp_v6.h> #endif #include <net/mptcp.h> #include "protocol.h" #include "mib.h" #define MPTCP_SAME_STATE TCP_MAX_STATES #if IS_ENABLED(CONFIG_MPTCP_IPV6) struct mptcp6_sock { struct mptcp_sock msk; struct ipv6_pinfo np; }; #endif struct mptcp_skb_cb { u32 offset; }; #define MPTCP_SKB_CB(__skb) ((struct mptcp_skb_cb *)&((__skb)->cb[0])) static struct percpu_counter mptcp_sockets_allocated; /* If msk has an initial subflow socket, and the MP_CAPABLE handshake has not * completed yet or has failed, return the subflow socket. * Otherwise return NULL. */ static struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk) { if (!msk->subflow || READ_ONCE(msk->can_ack)) return NULL; return msk->subflow; } static bool mptcp_is_tcpsk(struct sock *sk) { struct socket *sock = sk->sk_socket; if (unlikely(sk->sk_prot == &tcp_prot)) { /* we are being invoked after mptcp_accept() has * accepted a non-mp-capable flow: sk is a tcp_sk, * not an mptcp one. * * Hand the socket over to tcp so all further socket ops * bypass mptcp. */ sock->ops = &inet_stream_ops; return true; #if IS_ENABLED(CONFIG_MPTCP_IPV6) } else if (unlikely(sk->sk_prot == &tcpv6_prot)) { sock->ops = &inet6_stream_ops; return true; #endif } return false; } static struct sock *__mptcp_tcp_fallback(struct mptcp_sock *msk) { sock_owned_by_me((const struct sock *)msk); if (likely(!__mptcp_check_fallback(msk))) return NULL; return msk->first; } static int __mptcp_socket_create(struct mptcp_sock *msk) { struct mptcp_subflow_context *subflow; struct sock *sk = (struct sock *)msk; struct socket *ssock; int err; err = mptcp_subflow_create_socket(sk, &ssock); if (err) return err; msk->first = ssock->sk; msk->subflow = ssock; subflow = mptcp_subflow_ctx(ssock->sk); list_add(&subflow->node, &msk->conn_list); subflow->request_mptcp = 1; /* accept() will wait on first subflow sk_wq, and we always wakes up * via msk->sk_socket */ RCU_INIT_POINTER(msk->first->sk_wq, &sk->sk_socket->wq); return 0; } static void __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk, struct sk_buff *skb, unsigned int offset, size_t copy_len) { struct sock *sk = (struct sock *)msk; struct sk_buff *tail; __skb_unlink(skb, &ssk->sk_receive_queue); skb_ext_reset(skb); skb_orphan(skb); msk->ack_seq += copy_len; tail = skb_peek_tail(&sk->sk_receive_queue); if (offset == 0 && tail) { bool fragstolen; int delta; if (skb_try_coalesce(tail, skb, &fragstolen, &delta)) { kfree_skb_partial(skb, fragstolen); atomic_add(delta, &sk->sk_rmem_alloc); sk_mem_charge(sk, delta); return; } } skb_set_owner_r(skb, sk); __skb_queue_tail(&sk->sk_receive_queue, skb); MPTCP_SKB_CB(skb)->offset = offset; } static void mptcp_stop_timer(struct sock *sk) { struct inet_connection_sock *icsk = inet_csk(sk); sk_stop_timer(sk, &icsk->icsk_retransmit_timer); mptcp_sk(sk)->timer_ival = 0; } /* both sockets must be locked */ static bool mptcp_subflow_dsn_valid(const struct mptcp_sock *msk, struct sock *ssk) { struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); u64 dsn = mptcp_subflow_get_mapped_dsn(subflow); /* revalidate data sequence number. * * mptcp_subflow_data_available() is usually called * without msk lock. Its unlikely (but possible) * that msk->ack_seq has been advanced since the last * call found in-sequence data. */ if (likely(dsn == msk->ack_seq)) return true; subflow->data_avail = 0; return mptcp_subflow_data_available(ssk); } static void mptcp_check_data_fin_ack(struct sock *sk) { struct mptcp_sock *msk = mptcp_sk(sk); if (__mptcp_check_fallback(msk)) return; /* Look for an acknowledged DATA_FIN */ if (((1 << sk->sk_state) & (TCPF_FIN_WAIT1 | TCPF_CLOSING | TCPF_LAST_ACK)) && msk->write_seq == atomic64_read(&msk->snd_una)) { mptcp_stop_timer(sk); WRITE_ONCE(msk->snd_data_fin_enable, 0); switch (sk->sk_state) { case TCP_FIN_WAIT1: inet_sk_state_store(sk, TCP_FIN_WAIT2); sk->sk_state_change(sk); break; case TCP_CLOSING: fallthrough; case TCP_LAST_ACK: inet_sk_state_store(sk, TCP_CLOSE); sk->sk_state_change(sk); break; } if (sk->sk_shutdown == SHUTDOWN_MASK || sk->sk_state == TCP_CLOSE) sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP); else sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); } } static bool mptcp_pending_data_fin(struct sock *sk, u64 *seq) { struct mptcp_sock *msk = mptcp_sk(sk); if (READ_ONCE(msk->rcv_data_fin) && ((1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2))) { u64 rcv_data_fin_seq = READ_ONCE(msk->rcv_data_fin_seq); if (msk->ack_seq == rcv_data_fin_seq) { if (seq) *seq = rcv_data_fin_seq; return true; } } return false; } static void mptcp_set_timeout(const struct sock *sk, const struct sock *ssk) { long tout = ssk && inet_csk(ssk)->icsk_pending ? inet_csk(ssk)->icsk_timeout - jiffies : 0; if (tout <= 0) tout = mptcp_sk(sk)->timer_ival; mptcp_sk(sk)->timer_ival = tout > 0 ? tout : TCP_RTO_MIN; } static void mptcp_check_data_fin(struct sock *sk) { struct mptcp_sock *msk = mptcp_sk(sk); u64 rcv_data_fin_seq; if (__mptcp_check_fallback(msk) || !msk->first) return; /* Need to ack a DATA_FIN received from a peer while this side * of the connection is in ESTABLISHED, FIN_WAIT1, or FIN_WAIT2. * msk->rcv_data_fin was set when parsing the incoming options * at the subflow level and the msk lock was not held, so this * is the first opportunity to act on the DATA_FIN and change * the msk state. * * If we are caught up to the sequence number of the incoming * DATA_FIN, send the DATA_ACK now and do state transition. If * not caught up, do nothing and let the recv code send DATA_ACK * when catching up. */ if (mptcp_pending_data_fin(sk, &rcv_data_fin_seq)) { struct mptcp_subflow_context *subflow; msk->ack_seq++; WRITE_ONCE(msk->rcv_data_fin, 0); sk->sk_shutdown |= RCV_SHUTDOWN; smp_mb__before_atomic(); /* SHUTDOWN must be visible first */ set_bit(MPTCP_DATA_READY, &msk->flags); switch (sk->sk_state) { case TCP_ESTABLISHED: inet_sk_state_store(sk, TCP_CLOSE_WAIT); break; case TCP_FIN_WAIT1: inet_sk_state_store(sk, TCP_CLOSING); break; case TCP_FIN_WAIT2: inet_sk_state_store(sk, TCP_CLOSE); // @@ Close subflows now? break; default: /* Other states not expected */ WARN_ON_ONCE(1); break; } mptcp_set_timeout(sk, NULL); mptcp_for_each_subflow(msk, subflow) { struct sock *ssk = mptcp_subflow_tcp_sock(subflow); lock_sock(ssk); tcp_send_ack(ssk); release_sock(ssk); } sk->sk_state_change(sk); if (sk->sk_shutdown == SHUTDOWN_MASK || sk->sk_state == TCP_CLOSE) sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP); else sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); } } static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk, struct sock *ssk, unsigned int *bytes) { struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); struct sock *sk = (struct sock *)msk; unsigned int moved = 0; bool more_data_avail; struct tcp_sock *tp; bool done = false; if (!mptcp_subflow_dsn_valid(msk, ssk)) { *bytes = 0; return false; } tp = tcp_sk(ssk); do { u32 map_remaining, offset; u32 seq = tp->copied_seq; struct sk_buff *skb; bool fin; /* try to move as much data as available */ map_remaining = subflow->map_data_len - mptcp_subflow_get_map_offset(subflow); skb = skb_peek(&ssk->sk_receive_queue); if (!skb) break; if (__mptcp_check_fallback(msk)) { /* if we are running under the workqueue, TCP could have * collapsed skbs between dummy map creation and now * be sure to adjust the size */ map_remaining = skb->len; subflow->map_data_len = skb->len; } offset = seq - TCP_SKB_CB(skb)->seq; fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN; if (fin) { done = true; seq++; } if (offset < skb->len) { size_t len = skb->len - offset; if (tp->urg_data) done = true; __mptcp_move_skb(msk, ssk, skb, offset, len); seq += len; moved += len; if (WARN_ON_ONCE(map_remaining < len)) break; } else { WARN_ON_ONCE(!fin); sk_eat_skb(ssk, skb); done = true; } WRITE_ONCE(tp->copied_seq, seq); more_data_avail = mptcp_subflow_data_available(ssk); if (atomic_read(&sk->sk_rmem_alloc) > READ_ONCE(sk->sk_rcvbuf)) { done = true; break; } } while (more_data_avail); *bytes = moved; /* If the moves have caught up with the DATA_FIN sequence number * it's time to ack the DATA_FIN and change socket state, but * this is not a good place to change state. Let the workqueue * do it. */ if (mptcp_pending_data_fin(sk, NULL) && schedule_work(&msk->work)) sock_hold(sk); return done; } /* In most cases we will be able to lock the mptcp socket. If its already * owned, we need to defer to the work queue to avoid ABBA deadlock. */ static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk) { struct sock *sk = (struct sock *)msk; unsigned int moved = 0; if (READ_ONCE(sk->sk_lock.owned)) return false; if (unlikely(!spin_trylock_bh(&sk->sk_lock.slock))) return false; /* must re-check after taking the lock */ if (!READ_ONCE(sk->sk_lock.owned)) __mptcp_move_skbs_from_subflow(msk, ssk, &moved); spin_unlock_bh(&sk->sk_lock.slock); return moved > 0; } void mptcp_data_ready(struct sock *sk, struct sock *ssk) { struct mptcp_sock *msk = mptcp_sk(sk); set_bit(MPTCP_DATA_READY, &msk->flags); if (atomic_read(&sk->sk_rmem_alloc) < READ_ONCE(sk->sk_rcvbuf) && move_skbs_to_msk(msk, ssk)) goto wake; /* don't schedule if mptcp sk is (still) over limit */ if (atomic_read(&sk->sk_rmem_alloc) > READ_ONCE(sk->sk_rcvbuf)) goto wake; /* mptcp socket is owned, release_cb should retry */ if (!test_and_set_bit(TCP_DELACK_TIMER_DEFERRED, &sk->sk_tsq_flags)) { sock_hold(sk); /* need to try again, its possible release_cb() has already * been called after the test_and_set_bit() above. */ move_skbs_to_msk(msk, ssk); } wake: sk->sk_data_ready(sk); } static void __mptcp_flush_join_list(struct mptcp_sock *msk) { if (likely(list_empty(&msk->join_list))) return; spin_lock_bh(&msk->join_list_lock); list_splice_tail_init(&msk->join_list, &msk->conn_list); spin_unlock_bh(&msk->join_list_lock); } static bool mptcp_timer_pending(struct sock *sk) { return timer_pending(&inet_csk(sk)->icsk_retransmit_timer); } static void mptcp_reset_timer(struct sock *sk) { struct inet_connection_sock *icsk = inet_csk(sk); unsigned long tout; /* should never be called with mptcp level timer cleared */ tout = READ_ONCE(mptcp_sk(sk)->timer_ival); if (WARN_ON_ONCE(!tout)) tout = TCP_RTO_MIN; sk_reset_timer(sk, &icsk->icsk_retransmit_timer, jiffies + tout); } void mptcp_data_acked(struct sock *sk) { mptcp_reset_timer(sk); if ((!sk_stream_is_writeable(sk) || (inet_sk_state_load(sk) != TCP_ESTABLISHED)) && schedule_work(&mptcp_sk(sk)->work)) sock_hold(sk); } void mptcp_subflow_eof(struct sock *sk) { struct mptcp_sock *msk = mptcp_sk(sk); if (!test_and_set_bit(MPTCP_WORK_EOF, &msk->flags) && schedule_work(&msk->work)) sock_hold(sk); } static void mptcp_check_for_eof(struct mptcp_sock *msk) { struct mptcp_subflow_context *subflow; struct sock *sk = (struct sock *)msk; int receivers = 0; mptcp_for_each_subflow(msk, subflow) receivers += !subflow->rx_eof;