Michal Kubecek and Firo Yang did a very nice analysis of crashes happening in __inet_lookup_established(). Since a TCP socket can go from TCP_ESTABLISH to TCP_LISTEN (via a close()/socket()/listen() cycle) without a RCU grace period, I should not have changed listeners linkage in their hash table. They must use the nulls protocol (Documentation/RCU/rculist_nulls.txt), so that a lookup can detect a socket in a hash list was moved in another one. Since we added code in commit d296ba60d8e2 ("soreuseport: Resolve merge conflict for v4/v6 ordering fix"), we have to add hlist_nulls_add_tail_rcu() helper.
Showing the changes to include/net (2 of 6 files). Show all
Unified · Split
| @@ -103,13 +103,19 @@ struct inet_bind_hashbucket { · 102 unmodified lines | ||
| 103 | 103 | struct hlist_head chain; |
| 104 | 104 | }; |
| 105 | 105 | |
| 106 | −/* | |
| 107 | − * Sockets can be hashed in established or listening table | |
| 106 | +/* Sockets can be hashed in established or listening table. | |
| 107 | + * We must use different 'nulls' end-of-chain value for all hash buckets : | |
| 108 | + * A socket might transition from ESTABLISH to LISTEN state without | |
| 109 | + * RCU grace period. A lookup in ehash table needs to handle this case. | |
| 108 | 110 | */ |
| 111 | +#define LISTENING_NULLS_BASE (1U << 29) | |
| 109 | 112 | struct inet_listen_hashbucket { |
| 110 | 113 | spinlock_t lock; |
| 111 | 114 | unsigned int count; |
| 112 | − struct hlist_head head; | |
| 115 | + union { | |
| 116 | + struct hlist_head head; | |
| 117 | + struct hlist_nulls_head nulls_head; | |
| 118 | + }; | |
| 113 | 119 | }; |
| 114 | 120 | |
| 115 | 121 | /* This is for listening sockets, thus all sockets which possess wildcards. */ |
| @@ -722,6 +722,11 @@ static inline void __sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_h · 721 unmodified lines | ||
| 722 | 722 | hlist_nulls_add_head_rcu(&sk->sk_nulls_node, list); |
| 723 | 723 | } |
| 724 | 724 | |
| 725 | +static inline void __sk_nulls_add_node_tail_rcu(struct sock *sk, struct hlist_nulls_head *list) | |
| 726 | +{ | |
| 727 | + hlist_nulls_add_tail_rcu(&sk->sk_nulls_node, list); | |
| 728 | +} | |
| 729 | + | |
| 725 | 730 | static inline void sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list) |
| 726 | 731 | { |
| 727 | 732 | sock_hold(sk); |