/* SPDX-License-Identifier: GPL-2.0 *//* Multipath TCP * * Copyright (c) 2017 - 2019, Intel Corporation. */ #ifndef __MPTCP_PROTOCOL_H#define __MPTCP_PROTOCOL_H #include <linux/random.h>#include <net/tcp.h>#include <net/inet_connection_sock.h>#include <uapi/linux/mptcp.h>#include <net/genetlink.h>#include <net/rstreason.h> #define MPTCP_SUPPORTED_VERSION 1 /* MPTCP option bits */#define OPTION_MPTCP_MPC_SYN BIT(0)#define OPTION_MPTCP_MPC_SYNACK BIT(1)#define OPTION_MPTCP_MPC_ACK BIT(2)#define OPTION_MPTCP_MPJ_SYN BIT(3)#define OPTION_MPTCP_MPJ_SYNACK BIT(4)#define OPTION_MPTCP_MPJ_ACK BIT(5)#define OPTION_MPTCP_ADD_ADDR BIT(6)#define OPTION_MPTCP_RM_ADDR BIT(7)#define OPTION_MPTCP_FASTCLOSE BIT(8)#define OPTION_MPTCP_PRIO BIT(9)#define OPTION_MPTCP_RST BIT(10)#define OPTION_MPTCP_DSS BIT(11)#define OPTION_MPTCP_FAIL BIT(12) #define OPTION_MPTCP_CSUMREQD BIT(13) #define OPTIONS_MPTCP_MPC (OPTION_MPTCP_MPC_SYN | OPTION_MPTCP_MPC_SYNACK | \ OPTION_MPTCP_MPC_ACK)#define OPTIONS_MPTCP_MPJ (OPTION_MPTCP_MPJ_SYN | OPTION_MPTCP_MPJ_SYNACK | \ OPTION_MPTCP_MPJ_ACK)#define OPTIONS_MPTCP_DSS (OPTION_MPTCP_DSS | OPTION_MPTCP_CSUMREQD) /* MPTCP option subtypes */#define MPTCPOPT_MP_CAPABLE 0#define MPTCPOPT_MP_JOIN 1#define MPTCPOPT_DSS 2#define MPTCPOPT_ADD_ADDR 3#define MPTCPOPT_RM_ADDR 4#define MPTCPOPT_MP_PRIO 5#define MPTCPOPT_MP_FAIL 6#define MPTCPOPT_MP_FASTCLOSE 7#define MPTCPOPT_RST 8 /* MPTCP suboption lengths */#define TCPOLEN_MPTCP_MPC_SYN 4#define TCPOLEN_MPTCP_MPC_SYNACK 12#define TCPOLEN_MPTCP_MPC_ACK 20#define TCPOLEN_MPTCP_MPC_ACK_DATA 22#define TCPOLEN_MPTCP_MPJ_SYN 12#define TCPOLEN_MPTCP_MPJ_SYNACK 16#define TCPOLEN_MPTCP_MPJ_ACK 24#define TCPOLEN_MPTCP_DSS_BASE 4#define TCPOLEN_MPTCP_DSS_ACK32 4#define TCPOLEN_MPTCP_DSS_ACK64 8#define TCPOLEN_MPTCP_DSS_MAP32 10#define TCPOLEN_MPTCP_DSS_MAP64 14#define TCPOLEN_MPTCP_DSS_CHECKSUM 2#define TCPOLEN_MPTCP_ADD_ADDR 16#define TCPOLEN_MPTCP_ADD_ADDR_PORT 18#define TCPOLEN_MPTCP_ADD_ADDR_BASE 8#define TCPOLEN_MPTCP_ADD_ADDR_BASE_PORT 10#define TCPOLEN_MPTCP_ADD_ADDR6 28#define TCPOLEN_MPTCP_ADD_ADDR6_PORT 30#define TCPOLEN_MPTCP_ADD_ADDR6_BASE 20#define TCPOLEN_MPTCP_ADD_ADDR6_BASE_PORT 22#define TCPOLEN_MPTCP_PORT_LEN 2#define TCPOLEN_MPTCP_PORT_ALIGN 2#define TCPOLEN_MPTCP_RM_ADDR_BASE 3#define TCPOLEN_MPTCP_PRIO 3#define TCPOLEN_MPTCP_PRIO_ALIGN 4#define TCPOLEN_MPTCP_FASTCLOSE 12#define TCPOLEN_MPTCP_RST 4#define TCPOLEN_MPTCP_FAIL 12 #define TCPOLEN_MPTCP_MPC_ACK_DATA_CSUM (TCPOLEN_MPTCP_DSS_CHECKSUM + TCPOLEN_MPTCP_MPC_ACK_DATA) /* MPTCP MP_JOIN flags */#define MPTCPOPT_BACKUP BIT(0)#define MPTCPOPT_THMAC_LEN 8 /* MPTCP MP_CAPABLE flags */#define MPTCP_VERSION_MASK (0x0F)#define MPTCP_CAP_CHECKSUM_REQD BIT(7)#define MPTCP_CAP_EXTENSIBILITY BIT(6)#define MPTCP_CAP_DENY_JOIN_ID0 BIT(5)#define MPTCP_CAP_HMAC_SHA256 BIT(0)#define MPTCP_CAP_FLAG_MASK (0x1F) /* MPTCP DSS flags */#define MPTCP_DSS_DATA_FIN BIT(4)#define MPTCP_DSS_DSN64 BIT(3)#define MPTCP_DSS_HAS_MAP BIT(2)#define MPTCP_DSS_ACK64 BIT(1)#define MPTCP_DSS_HAS_ACK BIT(0)#define MPTCP_DSS_FLAG_MASK (0x1F) /* MPTCP ADD_ADDR flags */#define MPTCP_ADDR_ECHO BIT(0) /* MPTCP MP_PRIO flags */#define MPTCP_PRIO_BKUP BIT(0) /* MPTCP TCPRST flags */#define MPTCP_RST_TRANSIENT BIT(0) /* MPTCP socket atomic flags */#define MPTCP_WORK_RTX 1#define MPTCP_FALLBACK_DONE 2#define MPTCP_WORK_CLOSE_SUBFLOW 3#define MPTCP_RTX_ENABLED 4 /* MPTCP socket release cb flags */#define MPTCP_PUSH_PENDING 1#define MPTCP_CLEAN_UNA 2#define MPTCP_ERROR_REPORT 3#define MPTCP_RETRANSMIT 4#define MPTCP_FLUSH_JOIN_LIST 5#define MPTCP_SYNC_STATE 6#define MPTCP_SYNC_SNDBUF 7 struct mptcp_skb_cb { u64 map_seq; u64 end_seq; u32 offset; u8 has_rxtstamp; u8 cant_coalesce;}; #define MPTCP_SKB_CB(__skb) ((struct mptcp_skb_cb *)&((__skb)->cb[0])) static inline bool before64(__u64 seq1, __u64 seq2){ return (__s64)(seq1 - seq2) < 0;} #define after64(seq2, seq1) before64(seq1, seq2) struct mptcp_options_received { u64 sndr_key; u64 rcvr_key; u64 data_ack; u64 data_seq; u32 subflow_seq; u16 data_len; __sum16 csum; struct_group(status, u16 suboptions; u16 use_map:1, dsn64:1, data_fin:1, use_ack:1, ack64:1, mpc_map:1, reset_reason:4, reset_transient:1, echo:1, backup:1, deny_join_id0:1, __unused:2; ); u8 join_id; u32 token; u32 nonce; u64 thmac; u8 hmac[MPTCPOPT_HMAC_LEN]; struct mptcp_addr_info addr; struct mptcp_rm_list rm_list; u64 ahmac; u64 fail_seq;}; static inline __be32 mptcp_option(u8 subopt, u8 len, u8 nib, u8 field){ return htonl((TCPOPT_MPTCP << 24) | (len << 16) | (subopt << 12) | ((nib & 0xF) << 8) | field);} enum mptcp_pm_status { MPTCP_PM_ADD_ADDR_RECEIVED, MPTCP_PM_ADD_ADDR_SEND_ACK, MPTCP_PM_RM_ADDR_RECEIVED, MPTCP_PM_ESTABLISHED, MPTCP_PM_SUBFLOW_ESTABLISHED, MPTCP_PM_ALREADY_ESTABLISHED, /* persistent status, set after ESTABLISHED event */ MPTCP_PM_MPC_ENDPOINT_ACCOUNTED, /* persistent status, set after MPC local address is * accounted int id_avail_bitmap */ MPTCP_PM_DESTROYING, /* To fence out PM list allocs */}; enum mptcp_pm_type { MPTCP_PM_TYPE_KERNEL = 0, MPTCP_PM_TYPE_USERSPACE, __MPTCP_PM_TYPE_NR, __MPTCP_PM_TYPE_MAX = __MPTCP_PM_TYPE_NR - 1,}; /* Status bits below MPTCP_PM_ALREADY_ESTABLISHED need pm worker actions */#define MPTCP_PM_WORK_MASK ((1 << MPTCP_PM_ALREADY_ESTABLISHED) - 1) enum mptcp_addr_signal_status { MPTCP_ADD_ADDR_SIGNAL, MPTCP_ADD_ADDR_ECHO, MPTCP_RM_ADDR_SIGNAL,}; /* max value of mptcp_addr_info.id */#define MPTCP_PM_MAX_ADDR_ID U8_MAX struct mptcp_pm_data { struct mptcp_addr_info local; struct mptcp_addr_info remote; struct list_head anno_list; struct list_head userspace_pm_local_addr_list; spinlock_t lock; /*protects the whole PM data */ struct_group(reset, u8 addr_signal; bool server_side; bool work_pending; bool accept_addr; bool accept_subflow; bool remote_deny_join_id0; u8 add_addr_signaled; u8 add_addr_accepted; u8 local_addr_used; u8 pm_type; u8 extra_subflows; u8 status; ); DECLARE_BITMAP(id_avail_bitmap, MPTCP_PM_MAX_ADDR_ID + 1); struct mptcp_rm_list rm_list_tx; struct mptcp_rm_list rm_list_rx;}; struct mptcp_pm_local { struct mptcp_addr_info addr; u32 flags; int ifindex;}; struct mptcp_pm_addr_entry { struct list_head list; struct mptcp_addr_info addr; u32 flags; int ifindex; struct socket *lsk;}; struct mptcp_data_frag { struct list_head list; u64 data_seq; u16 data_len; u16 offset; u8 overhead; u8 eor; /* currently using 1 bit */ u16 already_sent; struct page *page;}; /* Arbitrary compromise between as low as possible to react timely to subflow * close event and as big as possible to avoid being fouled by biased large * samples due to peer sending data on a different subflow WRT to the incoming * ack. */#define MPTCP_RTT_SAMPLES 5 /* MPTCP connection sock */struct mptcp_sock { /* inet_connection_sock must be the first member */ struct inet_connection_sock sk; u64 local_key; /* protected by the first subflow socket lock * lockless access read */ u64 remote_key; /* same as above */ u64 write_seq; u64 bytes_sent; u64 snd_nxt; u64 bytes_received; u64 ack_seq; atomic64_t rcv_wnd_sent; u64 rcv_data_fin_seq; u64 bytes_retrans; u64 bytes_consumed; int snd_burst; int old_wspace; u64 recovery_snd_nxt; /* in recovery mode accept up to this seq; * recovery related fields are under data_lock * protection */ u64 bytes_acked; u64 snd_una; u64 wnd_end; u32 last_data_sent; u32 last_data_recv; u32 last_ack_recv; unsigned long timer_ival; u32 token; unsigned long flags; unsigned long cb_flags; bool recovery; /* closing subflow write queue reinjected */ bool can_ack; bool fully_established; bool rcv_data_fin; bool snd_data_fin_enable; bool rcv_fastclose; bool use_64bit_ack; /* Set when we received a 64-bit DSN */ bool csum_enabled; bool allow_infinite_fallback; u8 pending_state; /* A subflow asked to set this sk_state, * protected by the msk data lock */ u8 mpc_endpoint_id; u8 recvmsg_inq:1, cork:1, nodelay:1, fastopening:1, in_accept_queue:1, free_first:1, rcvspace_init:1, fastclosing:1; u32 notsent_lowat; int keepalive_cnt; int keepalive_idle; int keepalive_intvl; int maxseg; struct work_struct work; struct sk_buff *ooo_last_skb; struct rb_root out_of_order_queue; struct list_head conn_list; struct list_head rtx_queue; struct mptcp_data_frag *first_pending; struct list_head join_list; struct sock *first; /* The mptcp ops can safely dereference, using suitable * ONCE annotation, the subflow outside the socket * lock as such sock is freed after close(). */ struct mptcp_pm_data pm; struct mptcp_sched_ops *sched; /* Most recent rtt_us observed by in use incoming subflows. */ struct { u32 samples[MPTCP_RTT_SAMPLES]; u32 next_sample; } rcv_rtt_est; struct { int space; /* bytes copied in last measurement window */ int copied; /* bytes copied in this measurement window */ u64 time; /* start time of measurement window */ } rcvq_space; u8 scaling_ratio; bool allow_subflows; u32 subflow_id; u32 setsockopt_seq; char ca_name[TCP_CA_NAME_MAX]; spinlock_t fallback_lock; /* protects fallback, * allow_infinite_fallback and * allow_join */ struct list_head backlog_list; /* protected by the data lock */ u32 backlog_len; u32 backlog_unaccounted;}; #define mptcp_data_lock(sk) spin_lock_bh(&(sk)->sk_lock.slock)#define mptcp_data_unlock(sk) spin_unlock_bh(&(sk)->sk_lock.slock) #define mptcp_for_each_subflow(__msk, __subflow) \ list_for_each_entry(__subflow, &((__msk)->conn_list), node)#define mptcp_for_each_subflow_safe(__msk, __subflow, __tmp) \ list_for_each_entry_safe(__subflow, __tmp, &((__msk)->conn_list), node)#define mptcp_next_subflow(__msk, __subflow) \ list_next_entry_circular(__subflow, &((__msk)->conn_list), node) extern struct genl_family mptcp_genl_family; static inline void msk_owned_by_me(const struct mptcp_sock *msk){ sock_owned_by_me((const struct sock *)msk);} #ifdef CONFIG_DEBUG_NET/* MPTCP-specific: we might (indirectly) call this helper with the wrong sk */#undef tcp_sk#define tcp_sk(ptr) ({ \ typeof(ptr) _ptr = (ptr); \ WARN_ON(_ptr->sk_protocol != IPPROTO_TCP); \ container_of_const(_ptr, struct tcp_sock, inet_conn.icsk_inet.sk); \})#define mptcp_sk(ptr) ({ \ typeof(ptr) _ptr = (ptr); \ WARN_ON(_ptr->sk_protocol != IPPROTO_MPTCP); \ container_of_const(_ptr, struct mptcp_sock, sk.icsk_inet.sk); \}) #else /* !CONFIG_DEBUG_NET */#define mptcp_sk(ptr) container_of_const(ptr, struct mptcp_sock, sk.icsk_inet.sk)#endif static inline int mptcp_win_from_space(const struct sock *sk, int space){ return __tcp_win_from_space(mptcp_sk(sk)->scaling_ratio, space);} static inline int mptcp_space_from_win(const struct sock *sk, int win){ return __tcp_space_from_win(mptcp_sk(sk)->scaling_ratio, win);} static inline int __mptcp_space(const struct sock *sk){ return mptcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) - READ_ONCE(mptcp_sk(sk)->backlog_len) - sk_rmem_alloc_get(sk));} static inline struct mptcp_data_frag *mptcp_send_head(const struct sock *sk){ const struct mptcp_sock *msk = mptcp_sk(sk); return msk->first_pending;} static inline void mptcp_init_rtt_est(struct mptcp_sock *msk){ int i; for (i = 0; i < MPTCP_RTT_SAMPLES; ++i) msk->rcv_rtt_est.samples[i] = U32_MAX; msk->rcv_rtt_est.next_sample = 0; msk->scaling_ratio = TCP_DEFAULT_SCALING_RATIO;} static inline u32 mptcp_rtt_us_est(const struct mptcp_sock *msk){ u32 rtt_us = READ_ONCE(msk->rcv_rtt_est.samples[0]); int i; /* Lockless access of collected samples. */ for (i = 1; i < MPTCP_RTT_SAMPLES; ++i) rtt_us = min(rtt_us, READ_ONCE(msk->rcv_rtt_est.samples[i])); return rtt_us;} static inline struct mptcp_data_frag *mptcp_send_next(struct sock *sk){ struct mptcp_sock *msk = mptcp_sk(sk); struct mptcp_data_frag *cur; cur = msk->first_pending; return list_is_last(&cur->list, &msk->rtx_queue) ? NULL : list_next_entry(cur, list);} static inline struct mptcp_data_frag *mptcp_pending_tail(const struct sock *sk){ const struct mptcp_sock *msk = mptcp_sk(sk); if (!msk->first_pending) return NULL; if (WARN_ON_ONCE(list_empty(&msk->rtx_queue))) return NULL; return list_last_entry(&msk->rtx_queue, struct mptcp_data_frag, list);} static inline struct mptcp_data_frag *mptcp_rtx_head(struct sock *sk){ struct mptcp_sock *msk = mptcp_sk(sk); if (msk->snd_una == msk->snd_nxt) return NULL; return list_first_entry_or_null(&msk->rtx_queue, struct mptcp_data_frag, list);} struct csum_pseudo_header { __be64 data_seq; __be32 subflow_seq; __be16 data_len; __sum16 csum;Lines 1–500 of 1384. Load more
/* SPDX-License-Identifier: GPL-2.0 */ /* Multipath TCP * * Copyright (c) 2017 - 2019, Intel Corporation. */ #ifndef __MPTCP_PROTOCOL_H #define __MPTCP_PROTOCOL_H #include <linux/random.h> #include <net/tcp.h> #include <net/inet_connection_sock.h> #include <uapi/linux/mptcp.h> #include <net/genetlink.h> #include <net/rstreason.h> #define MPTCP_SUPPORTED_VERSION 1 /* MPTCP option bits */ #define OPTION_MPTCP_MPC_SYN BIT(0) #define OPTION_MPTCP_MPC_SYNACK BIT(1) #define OPTION_MPTCP_MPC_ACK BIT(2) #define OPTION_MPTCP_MPJ_SYN BIT(3) #define OPTION_MPTCP_MPJ_SYNACK BIT(4) #define OPTION_MPTCP_MPJ_ACK BIT(5) #define OPTION_MPTCP_ADD_ADDR BIT(6) #define OPTION_MPTCP_RM_ADDR BIT(7) #define OPTION_MPTCP_FASTCLOSE BIT(8) #define OPTION_MPTCP_PRIO BIT(9) #define OPTION_MPTCP_RST BIT(10) #define OPTION_MPTCP_DSS BIT(11) #define OPTION_MPTCP_FAIL BIT(12) #define OPTION_MPTCP_CSUMREQD BIT(13) #define OPTIONS_MPTCP_MPC (OPTION_MPTCP_MPC_SYN | OPTION_MPTCP_MPC_SYNACK | \ OPTION_MPTCP_MPC_ACK) #define OPTIONS_MPTCP_MPJ (OPTION_MPTCP_MPJ_SYN | OPTION_MPTCP_MPJ_SYNACK | \ OPTION_MPTCP_MPJ_ACK) #define OPTIONS_MPTCP_DSS (OPTION_MPTCP_DSS | OPTION_MPTCP_CSUMREQD) /* MPTCP option subtypes */ #define MPTCPOPT_MP_CAPABLE 0 #define MPTCPOPT_MP_JOIN 1 #define MPTCPOPT_DSS 2 #define MPTCPOPT_ADD_ADDR 3 #define MPTCPOPT_RM_ADDR 4 #define MPTCPOPT_MP_PRIO 5 #define MPTCPOPT_MP_FAIL 6 #define MPTCPOPT_MP_FASTCLOSE 7 #define MPTCPOPT_RST 8 /* MPTCP suboption lengths */ #define TCPOLEN_MPTCP_MPC_SYN 4 #define TCPOLEN_MPTCP_MPC_SYNACK 12 #define TCPOLEN_MPTCP_MPC_ACK 20 #define TCPOLEN_MPTCP_MPC_ACK_DATA 22 #define TCPOLEN_MPTCP_MPJ_SYN 12 #define TCPOLEN_MPTCP_MPJ_SYNACK 16 #define TCPOLEN_MPTCP_MPJ_ACK 24 #define TCPOLEN_MPTCP_DSS_BASE 4 #define TCPOLEN_MPTCP_DSS_ACK32 4 #define TCPOLEN_MPTCP_DSS_ACK64 8 #define TCPOLEN_MPTCP_DSS_MAP32 10 #define TCPOLEN_MPTCP_DSS_MAP64 14 #define TCPOLEN_MPTCP_DSS_CHECKSUM 2 #define TCPOLEN_MPTCP_ADD_ADDR 16 #define TCPOLEN_MPTCP_ADD_ADDR_PORT 18 #define TCPOLEN_MPTCP_ADD_ADDR_BASE 8 #define TCPOLEN_MPTCP_ADD_ADDR_BASE_PORT 10 #define TCPOLEN_MPTCP_ADD_ADDR6 28 #define TCPOLEN_MPTCP_ADD_ADDR6_PORT 30 #define TCPOLEN_MPTCP_ADD_ADDR6_BASE 20 #define TCPOLEN_MPTCP_ADD_ADDR6_BASE_PORT 22 #define TCPOLEN_MPTCP_PORT_LEN 2 #define TCPOLEN_MPTCP_PORT_ALIGN 2 #define TCPOLEN_MPTCP_RM_ADDR_BASE 3 #define TCPOLEN_MPTCP_PRIO 3 #define TCPOLEN_MPTCP_PRIO_ALIGN 4 #define TCPOLEN_MPTCP_FASTCLOSE 12 #define TCPOLEN_MPTCP_RST 4 #define TCPOLEN_MPTCP_FAIL 12 #define TCPOLEN_MPTCP_MPC_ACK_DATA_CSUM (TCPOLEN_MPTCP_DSS_CHECKSUM + TCPOLEN_MPTCP_MPC_ACK_DATA) /* MPTCP MP_JOIN flags */ #define MPTCPOPT_BACKUP BIT(0) #define MPTCPOPT_THMAC_LEN 8 /* MPTCP MP_CAPABLE flags */ #define MPTCP_VERSION_MASK (0x0F) #define MPTCP_CAP_CHECKSUM_REQD BIT(7) #define MPTCP_CAP_EXTENSIBILITY BIT(6) #define MPTCP_CAP_DENY_JOIN_ID0 BIT(5) #define MPTCP_CAP_HMAC_SHA256 BIT(0) #define MPTCP_CAP_FLAG_MASK (0x1F) /* MPTCP DSS flags */ #define MPTCP_DSS_DATA_FIN BIT(4) #define MPTCP_DSS_DSN64 BIT(3) #define MPTCP_DSS_HAS_MAP BIT(2) #define MPTCP_DSS_ACK64 BIT(1) #define MPTCP_DSS_HAS_ACK BIT(0) #define MPTCP_DSS_FLAG_MASK (0x1F) /* MPTCP ADD_ADDR flags */ #define MPTCP_ADDR_ECHO BIT(0) /* MPTCP MP_PRIO flags */ #define MPTCP_PRIO_BKUP BIT(0) /* MPTCP TCPRST flags */ #define MPTCP_RST_TRANSIENT BIT(0) /* MPTCP socket atomic flags */ #define MPTCP_WORK_RTX 1 #define MPTCP_FALLBACK_DONE 2 #define MPTCP_WORK_CLOSE_SUBFLOW 3 #define MPTCP_RTX_ENABLED 4 /* MPTCP socket release cb flags */ #define MPTCP_PUSH_PENDING 1 #define MPTCP_CLEAN_UNA 2 #define MPTCP_ERROR_REPORT 3 #define MPTCP_RETRANSMIT 4 #define MPTCP_FLUSH_JOIN_LIST 5 #define MPTCP_SYNC_STATE 6 #define MPTCP_SYNC_SNDBUF 7 struct mptcp_skb_cb { u64 map_seq; u64 end_seq; u32 offset; u8 has_rxtstamp; u8 cant_coalesce; }; #define MPTCP_SKB_CB(__skb) ((struct mptcp_skb_cb *)&((__skb)->cb[0])) static inline bool before64(__u64 seq1, __u64 seq2) { return (__s64)(seq1 - seq2) < 0; } #define after64(seq2, seq1) before64(seq1, seq2) struct mptcp_options_received { u64 sndr_key; u64 rcvr_key; u64 data_ack; u64 data_seq; u32 subflow_seq; u16 data_len; __sum16 csum; struct_group(status, u16 suboptions; u16 use_map:1, dsn64:1, data_fin:1, use_ack:1, ack64:1, mpc_map:1, reset_reason:4, reset_transient:1, echo:1, backup:1, deny_join_id0:1, __unused:2; ); u8 join_id; u32 token; u32 nonce; u64 thmac; u8 hmac[MPTCPOPT_HMAC_LEN]; struct mptcp_addr_info addr; struct mptcp_rm_list rm_list; u64 ahmac; u64 fail_seq; }; static inline __be32 mptcp_option(u8 subopt, u8 len, u8 nib, u8 field) { return htonl((TCPOPT_MPTCP << 24) | (len << 16) | (subopt << 12) | ((nib & 0xF) << 8) | field); } enum mptcp_pm_status { MPTCP_PM_ADD_ADDR_RECEIVED, MPTCP_PM_ADD_ADDR_SEND_ACK, MPTCP_PM_RM_ADDR_RECEIVED, MPTCP_PM_ESTABLISHED, MPTCP_PM_SUBFLOW_ESTABLISHED, MPTCP_PM_ALREADY_ESTABLISHED, /* persistent status, set after ESTABLISHED event */ MPTCP_PM_MPC_ENDPOINT_ACCOUNTED, /* persistent status, set after MPC local address is * accounted int id_avail_bitmap */ MPTCP_PM_DESTROYING, /* To fence out PM list allocs */ }; enum mptcp_pm_type { MPTCP_PM_TYPE_KERNEL = 0, MPTCP_PM_TYPE_USERSPACE, __MPTCP_PM_TYPE_NR, __MPTCP_PM_TYPE_MAX = __MPTCP_PM_TYPE_NR - 1, }; /* Status bits below MPTCP_PM_ALREADY_ESTABLISHED need pm worker actions */ #define MPTCP_PM_WORK_MASK ((1 << MPTCP_PM_ALREADY_ESTABLISHED) - 1) enum mptcp_addr_signal_status { MPTCP_ADD_ADDR_SIGNAL, MPTCP_ADD_ADDR_ECHO, MPTCP_RM_ADDR_SIGNAL, }; /* max value of mptcp_addr_info.id */ #define MPTCP_PM_MAX_ADDR_ID U8_MAX struct mptcp_pm_data { struct mptcp_addr_info local; struct mptcp_addr_info remote; struct list_head anno_list; struct list_head userspace_pm_local_addr_list; spinlock_t lock; /*protects the whole PM data */ struct_group(reset, u8 addr_signal; bool server_side; bool work_pending; bool accept_addr; bool accept_subflow; bool remote_deny_join_id0; u8 add_addr_signaled; u8 add_addr_accepted; u8 local_addr_used; u8 pm_type; u8 extra_subflows; u8 status; ); DECLARE_BITMAP(id_avail_bitmap, MPTCP_PM_MAX_ADDR_ID + 1); struct mptcp_rm_list rm_list_tx; struct mptcp_rm_list rm_list_rx; }; struct mptcp_pm_local { struct mptcp_addr_info addr; u32 flags; int ifindex; }; struct mptcp_pm_addr_entry { struct list_head list; struct mptcp_addr_info addr; u32 flags; int ifindex; struct socket *lsk; }; struct mptcp_data_frag { struct list_head list; u64 data_seq; u16 data_len; u16 offset; u8 overhead; u8 eor; /* currently using 1 bit */ u16 already_sent; struct page *page; }; /* Arbitrary compromise between as low as possible to react timely to subflow * close event and as big as possible to avoid being fouled by biased large * samples due to peer sending data on a different subflow WRT to the incoming * ack. */ #define MPTCP_RTT_SAMPLES 5 /* MPTCP connection sock */ struct mptcp_sock { /* inet_connection_sock must be the first member */ struct inet_connection_sock sk; u64 local_key; /* protected by the first subflow socket lock * lockless access read */ u64 remote_key; /* same as above */ u64 write_seq; u64 bytes_sent; u64 snd_nxt; u64 bytes_received; u64 ack_seq; atomic64_t rcv_wnd_sent; u64 rcv_data_fin_seq; u64 bytes_retrans; u64 bytes_consumed; int snd_burst; int old_wspace; u64 recovery_snd_nxt; /* in recovery mode accept up to this seq; * recovery related fields are under data_lock * protection */ u64 bytes_acked; u64 snd_una; u64 wnd_end; u32 last_data_sent; u32 last_data_recv; u32 last_ack_recv; unsigned long timer_ival; u32 token; unsigned long flags; unsigned long cb_flags; bool recovery; /* closing subflow write queue reinjected */ bool can_ack; bool fully_established; bool rcv_data_fin; bool snd_data_fin_enable; bool rcv_fastclose; bool use_64bit_ack; /* Set when we received a 64-bit DSN */ bool csum_enabled; bool allow_infinite_fallback; u8 pending_state; /* A subflow asked to set this sk_state, * protected by the msk data lock */ u8 mpc_endpoint_id; u8 recvmsg_inq:1, cork:1, nodelay:1, fastopening:1, in_accept_queue:1, free_first:1, rcvspace_init:1, fastclosing:1; u32 notsent_lowat; int keepalive_cnt; int keepalive_idle; int keepalive_intvl; int maxseg; struct work_struct work; struct sk_buff *ooo_last_skb; struct rb_root out_of_order_queue; struct list_head conn_list; struct list_head rtx_queue; struct mptcp_data_frag *first_pending; struct list_head join_list; struct sock *first; /* The mptcp ops can safely dereference, using suitable * ONCE annotation, the subflow outside the socket * lock as such sock is freed after close(). */ struct mptcp_pm_data pm; struct mptcp_sched_ops *sched; /* Most recent rtt_us observed by in use incoming subflows. */ struct { u32 samples[MPTCP_RTT_SAMPLES]; u32 next_sample; } rcv_rtt_est; struct { int space; /* bytes copied in last measurement window */ int copied; /* bytes copied in this measurement window */ u64 time; /* start time of measurement window */ } rcvq_space; u8 scaling_ratio; bool allow_subflows; u32 subflow_id; u32 setsockopt_seq; char ca_name[TCP_CA_NAME_MAX]; spinlock_t fallback_lock; /* protects fallback, * allow_infinite_fallback and * allow_join */ struct list_head backlog_list; /* protected by the data lock */ u32 backlog_len; u32 backlog_unaccounted; }; #define mptcp_data_lock(sk) spin_lock_bh(&(sk)->sk_lock.slock) #define mptcp_data_unlock(sk) spin_unlock_bh(&(sk)->sk_lock.slock) #define mptcp_for_each_subflow(__msk, __subflow) \ list_for_each_entry(__subflow, &((__msk)->conn_list), node) #define mptcp_for_each_subflow_safe(__msk, __subflow, __tmp) \ list_for_each_entry_safe(__subflow, __tmp, &((__msk)->conn_list), node) #define mptcp_next_subflow(__msk, __subflow) \ list_next_entry_circular(__subflow, &((__msk)->conn_list), node) extern struct genl_family mptcp_genl_family; static inline void msk_owned_by_me(const struct mptcp_sock *msk) { sock_owned_by_me((const struct sock *)msk); } #ifdef CONFIG_DEBUG_NET /* MPTCP-specific: we might (indirectly) call this helper with the wrong sk */ #undef tcp_sk #define tcp_sk(ptr) ({ \ typeof(ptr) _ptr = (ptr); \ WARN_ON(_ptr->sk_protocol != IPPROTO_TCP); \ container_of_const(_ptr, struct tcp_sock, inet_conn.icsk_inet.sk); \ }) #define mptcp_sk(ptr) ({ \ typeof(ptr) _ptr = (ptr); \ WARN_ON(_ptr->sk_protocol != IPPROTO_MPTCP); \ container_of_const(_ptr, struct mptcp_sock, sk.icsk_inet.sk); \ }) #else /* !CONFIG_DEBUG_NET */ #define mptcp_sk(ptr) container_of_const(ptr, struct mptcp_sock, sk.icsk_inet.sk) #endif static inline int mptcp_win_from_space(const struct sock *sk, int space) { return __tcp_win_from_space(mptcp_sk(sk)->scaling_ratio, space); } static inline int mptcp_space_from_win(const struct sock *sk, int win) { return __tcp_space_from_win(mptcp_sk(sk)->scaling_ratio, win); } static inline int __mptcp_space(const struct sock *sk) { return mptcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) - READ_ONCE(mptcp_sk(sk)->backlog_len) - sk_rmem_alloc_get(sk)); } static inline struct mptcp_data_frag *mptcp_send_head(const struct sock *sk) { const struct mptcp_sock *msk = mptcp_sk(sk); return msk->first_pending; } static inline void mptcp_init_rtt_est(struct mptcp_sock *msk) { int i; for (i = 0; i < MPTCP_RTT_SAMPLES; ++i) msk->rcv_rtt_est.samples[i] = U32_MAX; msk->rcv_rtt_est.next_sample = 0; msk->scaling_ratio = TCP_DEFAULT_SCALING_RATIO; } static inline u32 mptcp_rtt_us_est(const struct mptcp_sock *msk) { u32 rtt_us = READ_ONCE(msk->rcv_rtt_est.samples[0]); int i; /* Lockless access of collected samples. */ for (i = 1; i < MPTCP_RTT_SAMPLES; ++i) rtt_us = min(rtt_us, READ_ONCE(msk->rcv_rtt_est.samples[i])); return rtt_us; } static inline struct mptcp_data_frag *mptcp_send_next(struct sock *sk) { struct mptcp_sock *msk = mptcp_sk(sk); struct mptcp_data_frag *cur; cur = msk->first_pending; return list_is_last(&cur->list, &msk->rtx_queue) ? NULL : list_next_entry(cur, list); } static inline struct mptcp_data_frag *mptcp_pending_tail(const struct sock *sk) { const struct mptcp_sock *msk = mptcp_sk(sk); if (!msk->first_pending) return NULL; if (WARN_ON_ONCE(list_empty(&msk->rtx_queue))) return NULL; return list_last_entry(&msk->rtx_queue, struct mptcp_data_frag, list); } static inline struct mptcp_data_frag *mptcp_rtx_head(struct sock *sk) { struct mptcp_sock *msk = mptcp_sk(sk); if (msk->snd_una == msk->snd_nxt) return NULL; return list_first_entry_or_null(&msk->rtx_queue, struct mptcp_data_frag, list); } struct csum_pseudo_header { __be64 data_seq; __be32 subflow_seq; __be16 data_len; __sum16 csum;