History

Revert "net: Add a second bind table hashed by port and address"

This reverts: commit d5a42de8bdbe ("net: Add a second bind table hashed by port and address") commit 538aaf9b2383 ("selftests: Add test for timing a bind request to a port with a populated bhash entry") Link: https://lore.kernel.org/netdev/20220520001834.2247810-1-kuba@kernel.org/ There are a few things that need to be fixed here: * Updating bhash2 in cases where the socket's rcv saddr changes * Adding bhash2 hashbucket locks Links to syzbot reports: https://lore.kernel.org/netdev/00000000000022208805e0df247a@google.com/ https://lore.kernel.org/netdev/0000000000003f33bc05dfaf44fe@google.com/

Joanne Koong committed 4 years ago

Commit 593d1eb1 parent 219b51a10 files · +83 −611Browse files

Unified · Split

10 files
  1. include/net/inet_connection_sock.h +0 −3
  2. include/net/inet_hashtables.h +1 −67
  3. include/net/sock.h +0 −14
  4. net/dccp/proto.c +5 −28
  5. net/ipv4/inet_connection_sock.c +64 −183
  6. net/ipv4/inet_hashtables.c +11 −182
  7. net/ipv4/tcp.c +2 −12
  8. tools/testing/selftests/net/.gitignore +0 −1
  9. tools/testing/selftests/net/Makefile +0 −2
  10. tools/testing/selftests/net/bind_bhash_test.c +0 −119
include/net/inet_connection_sock.h+0−3
@@ -25,7 +25,6 @@ · 24 unmodified lines
25#undef INET_CSK_CLEAR_TIMERS25#undef INET_CSK_CLEAR_TIMERS
26 26
27struct inet_bind_bucket;27struct inet_bind_bucket;
28struct inet_bind2_bucket;
29struct tcp_congestion_ops;28struct tcp_congestion_ops;
30 29
31/*30/*
@@ -58,7 +57,6 @@ struct inet_connection_sock_af_ops { · 26 unmodified lines
58 *57 *
59 * @icsk_accept_queue: FIFO of established children58 * @icsk_accept_queue: FIFO of established children
60 * @icsk_bind_hash: Bind node59 * @icsk_bind_hash: Bind node
61 * @icsk_bind2_hash: Bind node in the bhash2 table
62 * @icsk_timeout: Timeout60 * @icsk_timeout: Timeout
63 * @icsk_retransmit_timer: Resend (no ack)61 * @icsk_retransmit_timer: Resend (no ack)
64 * @icsk_rto: Retransmit timeout62 * @icsk_rto: Retransmit timeout
@@ -85,7 +83,6 @@ struct inet_connection_sock { · 20 unmodified lines
85 struct inet_sock icsk_inet;83 struct inet_sock icsk_inet;
86 struct request_sock_queue icsk_accept_queue;84 struct request_sock_queue icsk_accept_queue;
87 struct inet_bind_bucket *icsk_bind_hash;85 struct inet_bind_bucket *icsk_bind_hash;
88 struct inet_bind2_bucket *icsk_bind2_hash;
89 unsigned long icsk_timeout;86 unsigned long icsk_timeout;
90 struct timer_list icsk_retransmit_timer;87 struct timer_list icsk_retransmit_timer;
91 struct timer_list icsk_delack_timer;88 struct timer_list icsk_delack_timer;
include/net/inet_hashtables.h+1−67
@@ -90,32 +90,11 @@ struct inet_bind_bucket { · 89 unmodified lines
90 struct hlist_head owners;90 struct hlist_head owners;
91};91};
92 92
93struct inet_bind2_bucket {
94 possible_net_t ib_net;
95 int l3mdev;
96 unsigned short port;
97 union {
98#if IS_ENABLED(CONFIG_IPV6)
99 struct in6_addr v6_rcv_saddr;
100#endif
101 __be32 rcv_saddr;
102 };
103 /* Node in the inet2_bind_hashbucket chain */
104 struct hlist_node node;
105 /* List of sockets hashed to this bucket */
106 struct hlist_head owners;
107};
108
109static inline struct net *ib_net(struct inet_bind_bucket *ib)93static inline struct net *ib_net(struct inet_bind_bucket *ib)
110{94{
111 return read_pnet(&ib->ib_net);95 return read_pnet(&ib->ib_net);
112}96}
113 97
114static inline struct net *ib2_net(struct inet_bind2_bucket *ib)
115{
116 return read_pnet(&ib->ib_net);
117}
118
119#define inet_bind_bucket_for_each(tb, head) \98#define inet_bind_bucket_for_each(tb, head) \
120 hlist_for_each_entry(tb, head, node)99 hlist_for_each_entry(tb, head, node)
121 100
@@ -124,15 +103,6 @@ struct inet_bind_hashbucket { · 2 unmodified lines
124 struct hlist_head chain;103 struct hlist_head chain;
125};104};
126 105
127/* This is synchronized using the inet_bind_hashbucket's spinlock.
128 * Instead of having separate spinlocks, the inet_bind2_hashbucket can share
129 * the inet_bind_hashbucket's given that in every case where the bhash2 table
130 * is useful, a lookup in the bhash table also occurs.
131 */
132struct inet_bind2_hashbucket {
133 struct hlist_head chain;
134};
135
136/* Sockets can be hashed in established or listening table.106/* Sockets can be hashed in established or listening table.
137 * We must use different 'nulls' end-of-chain value for all hash buckets :107 * We must use different 'nulls' end-of-chain value for all hash buckets :
138 * A socket might transition from ESTABLISH to LISTEN state without108 * A socket might transition from ESTABLISH to LISTEN state without
@@ -164,12 +134,6 @@ struct inet_hashinfo { · 25 unmodified lines
164 */134 */
165 struct kmem_cache *bind_bucket_cachep;135 struct kmem_cache *bind_bucket_cachep;
166 struct inet_bind_hashbucket *bhash;136 struct inet_bind_hashbucket *bhash;
167 /* The 2nd binding table hashed by port and address.
168 * This is used primarily for expediting the resolution of bind
169 * conflicts.
170 */
171 struct kmem_cache *bind2_bucket_cachep;
172 struct inet_bind2_hashbucket *bhash2;
173 unsigned int bhash_size;137 unsigned int bhash_size;
174 138
175 /* The 2nd listener table hashed by local port and address */139 /* The 2nd listener table hashed by local port and address */
@@ -229,36 +193,6 @@ inet_bind_bucket_create(struct kmem_cache *cachep, struct net *net, · 53 unmodified lines
229void inet_bind_bucket_destroy(struct kmem_cache *cachep,193void inet_bind_bucket_destroy(struct kmem_cache *cachep,
230 struct inet_bind_bucket *tb);194 struct inet_bind_bucket *tb);
231 195
232static inline bool check_bind_bucket_match(struct inet_bind_bucket *tb,
233 struct net *net,
234 const unsigned short port,
235 int l3mdev)
236{
237 return net_eq(ib_net(tb), net) && tb->port == port &&
238 tb->l3mdev == l3mdev;
239}
240
241struct inet_bind2_bucket *
242inet_bind2_bucket_create(struct kmem_cache *cachep, struct net *net,
243 struct inet_bind2_hashbucket *head,
244 const unsigned short port, int l3mdev,
245 const struct sock *sk);
246
247void inet_bind2_bucket_destroy(struct kmem_cache *cachep,
248 struct inet_bind2_bucket *tb);
249
250struct inet_bind2_bucket *
251inet_bind2_bucket_find(struct inet_hashinfo *hinfo, struct net *net,
252 const unsigned short port, int l3mdev,
253 struct sock *sk,
254 struct inet_bind2_hashbucket **head);
255
256bool check_bind2_bucket_match_nulladdr(struct inet_bind2_bucket *tb,
257 struct net *net,
258 const unsigned short port,
259 int l3mdev,
260 const struct sock *sk);
261
262static inline u32 inet_bhashfn(const struct net *net, const __u16 lport,196static inline u32 inet_bhashfn(const struct net *net, const __u16 lport,
263 const u32 bhash_size)197 const u32 bhash_size)
264{198{
@@ -266,7 +200,7 @@ static inline u32 inet_bhashfn(const struct net *net, const __u16 lport, · 1 unmodified line
266}200}
267 201
268void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb,202void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb,
269 struct inet_bind2_bucket *tb2, const unsigned short snum);203 const unsigned short snum);
270 204
271/* Caller must disable local BH processing. */205/* Caller must disable local BH processing. */
272int __inet_inherit_port(const struct sock *sk, struct sock *child);206int __inet_inherit_port(const struct sock *sk, struct sock *child);
include/net/sock.h+0−14
@@ -348,7 +348,6 @@ struct sk_filter; · 347 unmodified lines
348 * @sk_txtime_report_errors: set report errors mode for SO_TXTIME348 * @sk_txtime_report_errors: set report errors mode for SO_TXTIME
349 * @sk_txtime_unused: unused txtime flags349 * @sk_txtime_unused: unused txtime flags
350 * @ns_tracker: tracker for netns reference350 * @ns_tracker: tracker for netns reference
351 * @sk_bind2_node: bind node in the bhash2 table
352 */351 */
353struct sock {352struct sock {
354 /*353 /*
@@ -538,7 +537,6 @@ struct sock { · 183 unmodified lines
538#endif537#endif
539 struct rcu_head sk_rcu;538 struct rcu_head sk_rcu;
540 netns_tracker ns_tracker;539 netns_tracker ns_tracker;
541 struct hlist_node sk_bind2_node;
542};540};
543 541
544enum sk_pacing {542enum sk_pacing {
@@ -819,16 +817,6 @@ static inline void sk_add_bind_node(struct sock *sk, · 274 unmodified lines
819 hlist_add_head(&sk->sk_bind_node, list);817 hlist_add_head(&sk->sk_bind_node, list);
820}818}
821 819
822static inline void __sk_del_bind2_node(struct sock *sk)
823{
824 __hlist_del(&sk->sk_bind2_node);
825}
826
827static inline void sk_add_bind2_node(struct sock *sk, struct hlist_head *list)
828{
829 hlist_add_head(&sk->sk_bind2_node, list);
830}
831
832#define sk_for_each(__sk, list) \820#define sk_for_each(__sk, list) \
833 hlist_for_each_entry(__sk, list, sk_node)821 hlist_for_each_entry(__sk, list, sk_node)
834#define sk_for_each_rcu(__sk, list) \822#define sk_for_each_rcu(__sk, list) \
@@ -846,8 +834,6 @@ static inline void sk_add_bind2_node(struct sock *sk, struct hlist_head *list) · 11 unmodified lines
846 hlist_for_each_entry_safe(__sk, tmp, list, sk_node)834 hlist_for_each_entry_safe(__sk, tmp, list, sk_node)
847#define sk_for_each_bound(__sk, list) \835#define sk_for_each_bound(__sk, list) \
848 hlist_for_each_entry(__sk, list, sk_bind_node)836 hlist_for_each_entry(__sk, list, sk_bind_node)
849#define sk_for_each_bound_bhash2(__sk, list) \
850 hlist_for_each_entry(__sk, list, sk_bind2_node)
851 837
852/**838/**
853 * sk_for_each_entry_offset_rcu - iterate over a list at a given struct offset839 * sk_for_each_entry_offset_rcu - iterate over a list at a given struct offset
net/dccp/proto.c+5−28
@@ -1120,12 +1120,6 @@ static int __init dccp_init(void) · 1119 unmodified lines
1120 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT, NULL);1120 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT, NULL);
1121 if (!dccp_hashinfo.bind_bucket_cachep)1121 if (!dccp_hashinfo.bind_bucket_cachep)
1122 goto out_free_hashinfo2;1122 goto out_free_hashinfo2;
1123 dccp_hashinfo.bind2_bucket_cachep =
1124 kmem_cache_create("dccp_bind2_bucket",
1125 sizeof(struct inet_bind2_bucket), 0,
1126 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT, NULL);
1127 if (!dccp_hashinfo.bind2_bucket_cachep)
1128 goto out_free_bind_bucket_cachep;
1129 1123
1130 /*1124 /*
1131 * Size and allocate the main established and bind bucket1125 * Size and allocate the main established and bind bucket
@@ -1156,7 +1150,7 @@ static int __init dccp_init(void) · 24 unmodified lines
1156 1150
1157 if (!dccp_hashinfo.ehash) {1151 if (!dccp_hashinfo.ehash) {
1158 DCCP_CRIT("Failed to allocate DCCP established hash table");1152 DCCP_CRIT("Failed to allocate DCCP established hash table");
1159 goto out_free_bind2_bucket_cachep;1153 goto out_free_bind_bucket_cachep;
1160 }1154 }
1161 1155
1162 for (i = 0; i <= dccp_hashinfo.ehash_mask; i++)1156 for (i = 0; i <= dccp_hashinfo.ehash_mask; i++)
@@ -1182,23 +1176,14 @@ static int __init dccp_init(void) · 19 unmodified lines
1182 goto out_free_dccp_locks;1176 goto out_free_dccp_locks;
1183 }1177 }
1184 1178
1185 dccp_hashinfo.bhash2 = (struct inet_bind2_hashbucket *)
1186 __get_free_pages(GFP_ATOMIC | __GFP_NOWARN, bhash_order);
1187
1188 if (!dccp_hashinfo.bhash2) {
1189 DCCP_CRIT("Failed to allocate DCCP bind2 hash table");
1190 goto out_free_dccp_bhash;
1191 }
1192
1193 for (i = 0; i < dccp_hashinfo.bhash_size; i++) {1179 for (i = 0; i < dccp_hashinfo.bhash_size; i++) {
1194 spin_lock_init(&dccp_hashinfo.bhash[i].lock);1180 spin_lock_init(&dccp_hashinfo.bhash[i].lock);
1195 INIT_HLIST_HEAD(&dccp_hashinfo.bhash[i].chain);1181 INIT_HLIST_HEAD(&dccp_hashinfo.bhash[i].chain);
1196 INIT_HLIST_HEAD(&dccp_hashinfo.bhash2[i].chain);
1197 }1182 }
1198 1183
1199 rc = dccp_mib_init();1184 rc = dccp_mib_init();
1200 if (rc)1185 if (rc)
1201 goto out_free_dccp_bhash2;1186 goto out_free_dccp_bhash;
1202 1187
1203 rc = dccp_ackvec_init();1188 rc = dccp_ackvec_init();
1204 if (rc)1189 if (rc)
@@ -1222,38 +1207,30 @@ out_ackvec_exit: · 17 unmodified lines
1222 dccp_ackvec_exit();1207 dccp_ackvec_exit();
1223out_free_dccp_mib:1208out_free_dccp_mib:
1224 dccp_mib_exit();1209 dccp_mib_exit();
1225out_free_dccp_bhash2:
1226 free_pages((unsigned long)dccp_hashinfo.bhash2, bhash_order);
1227out_free_dccp_bhash:1210out_free_dccp_bhash:
1228 free_pages((unsigned long)dccp_hashinfo.bhash, bhash_order);1211 free_pages((unsigned long)dccp_hashinfo.bhash, bhash_order);
1229out_free_dccp_locks:1212out_free_dccp_locks:
1230 inet_ehash_locks_free(&dccp_hashinfo);1213 inet_ehash_locks_free(&dccp_hashinfo);
1231out_free_dccp_ehash:1214out_free_dccp_ehash:
1232 free_pages((unsigned long)dccp_hashinfo.ehash, ehash_order);1215 free_pages((unsigned long)dccp_hashinfo.ehash, ehash_order);
1233out_free_bind2_bucket_cachep:
1234 kmem_cache_destroy(dccp_hashinfo.bind2_bucket_cachep);
1235out_free_bind_bucket_cachep:1216out_free_bind_bucket_cachep:
1236 kmem_cache_destroy(dccp_hashinfo.bind_bucket_cachep);1217 kmem_cache_destroy(dccp_hashinfo.bind_bucket_cachep);
1237out_free_hashinfo2:1218out_free_hashinfo2:
1238 inet_hashinfo2_free_mod(&dccp_hashinfo);1219 inet_hashinfo2_free_mod(&dccp_hashinfo);
1239out_fail:1220out_fail:
1240 dccp_hashinfo.bhash = NULL;1221 dccp_hashinfo.bhash = NULL;
1241 dccp_hashinfo.bhash2 = NULL;
1242 dccp_hashinfo.ehash = NULL;1222 dccp_hashinfo.ehash = NULL;
1243 dccp_hashinfo.bind_bucket_cachep = NULL;1223 dccp_hashinfo.bind_bucket_cachep = NULL;
1244 dccp_hashinfo.bind2_bucket_cachep = NULL;
1245 return rc;1224 return rc;
1246}1225}
1247 1226
1248static void __exit dccp_fini(void)1227static void __exit dccp_fini(void)
1249{1228{
1250 int bhash_order = get_order(dccp_hashinfo.bhash_size *
1251 sizeof(struct inet_bind_hashbucket));
1252
1253 ccid_cleanup_builtins();1229 ccid_cleanup_builtins();
1254 dccp_mib_exit();1230 dccp_mib_exit();
1255 free_pages((unsigned long)dccp_hashinfo.bhash, bhash_order);1231 free_pages((unsigned long)dccp_hashinfo.bhash,
1256 free_pages((unsigned long)dccp_hashinfo.bhash2, bhash_order);1232 get_order(dccp_hashinfo.bhash_size *
1233 sizeof(struct inet_bind_hashbucket)));
1257 free_pages((unsigned long)dccp_hashinfo.ehash,1234 free_pages((unsigned long)dccp_hashinfo.ehash,
1258 get_order((dccp_hashinfo.ehash_mask + 1) *1235 get_order((dccp_hashinfo.ehash_mask + 1) *
1259 sizeof(struct inet_ehash_bucket)));1236 sizeof(struct inet_ehash_bucket)));
net/ipv4/inet_connection_sock.c+64−183
@@ -117,32 +117,6 @@ bool inet_rcv_saddr_any(const struct sock *sk) · 116 unmodified lines
117 return !sk->sk_rcv_saddr;117 return !sk->sk_rcv_saddr;
118}118}
119 119
120static bool use_bhash2_on_bind(const struct sock *sk)
121{
122#if IS_ENABLED(CONFIG_IPV6)
123 int addr_type;
124
125 if (sk->sk_family == AF_INET6) {
126 addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr);
127 return addr_type != IPV6_ADDR_ANY &&
128 addr_type != IPV6_ADDR_MAPPED;
129 }
130#endif
131 return sk->sk_rcv_saddr != htonl(INADDR_ANY);
132}
133
134static u32 get_bhash2_nulladdr_hash(const struct sock *sk, struct net *net,
135 int port)
136{
137#if IS_ENABLED(CONFIG_IPV6)
138 struct in6_addr nulladdr = {};
139
140 if (sk->sk_family == AF_INET6)
141 return ipv6_portaddr_hash(net, &nulladdr, port);
142#endif
143 return ipv4_portaddr_hash(net, 0, port);
144}
145
146void inet_get_local_port_range(struct net *net, int *low, int *high)120void inet_get_local_port_range(struct net *net, int *low, int *high)
147{121{
148 unsigned int seq;122 unsigned int seq;
@@ -156,71 +130,16 @@ void inet_get_local_port_range(struct net *net, int *low, int *high) · 7 unmodified lines
156}130}
157EXPORT_SYMBOL(inet_get_local_port_range);131EXPORT_SYMBOL(inet_get_local_port_range);
158 132
159static bool bind_conflict_exist(const struct sock *sk, struct sock *sk2,133static int inet_csk_bind_conflict(const struct sock *sk,
160 kuid_t sk_uid, bool relax,134 const struct inet_bind_bucket *tb,
161 bool reuseport_cb_ok, bool reuseport_ok)
162{
163 int bound_dev_if2;
164
165 if (sk == sk2)
166 return false;
167
168 bound_dev_if2 = READ_ONCE(sk2->sk_bound_dev_if);
169
170 if (!sk->sk_bound_dev_if || !bound_dev_if2 ||
171 sk->sk_bound_dev_if == bound_dev_if2) {
172 if (sk->sk_reuse && sk2->sk_reuse &&
173 sk2->sk_state != TCP_LISTEN) {
174 if (!relax || (!reuseport_ok && sk->sk_reuseport &&
175 sk2->sk_reuseport && reuseport_cb_ok &&
176 (sk2->sk_state == TCP_TIME_WAIT ||
177 uid_eq(sk_uid, sock_i_uid(sk2)))))
178 return true;
179 } else if (!reuseport_ok || !sk->sk_reuseport ||
180 !sk2->sk_reuseport || !reuseport_cb_ok ||
181 (sk2->sk_state != TCP_TIME_WAIT &&
182 !uid_eq(sk_uid, sock_i_uid(sk2)))) {
183 return true;
184 }
185 }
186 return false;
187}
188
189static bool check_bhash2_conflict(const struct sock *sk,
190 struct inet_bind2_bucket *tb2, kuid_t sk_uid,
191 bool relax, bool reuseport_cb_ok,
192 bool reuseport_ok)
193{
194 struct sock *sk2;
195
196 sk_for_each_bound_bhash2(sk2, &tb2->owners) {
197 if (sk->sk_family == AF_INET && ipv6_only_sock(sk2))
198 continue;
199
200 if (bind_conflict_exist(sk, sk2, sk_uid, relax,
201 reuseport_cb_ok, reuseport_ok))
202 return true;
203 }
204 return false;
205}
206
207/* This should be called only when the corresponding inet_bind_bucket spinlock
208 * is held
209 */
210static int inet_csk_bind_conflict(const struct sock *sk, int port,
211 struct inet_bind_bucket *tb,
212 struct inet_bind2_bucket *tb2, /* may be null */
213 bool relax, bool reuseport_ok)135 bool relax, bool reuseport_ok)
214{136{
215 struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;
216 kuid_t uid = sock_i_uid((struct sock *)sk);
217 struct sock_reuseport *reuseport_cb;
218 struct inet_bind2_hashbucket *head2;
219 bool reuseport_cb_ok;
220 struct sock *sk2;137 struct sock *sk2;
221 struct net *net;138 bool reuseport_cb_ok;
222 int l3mdev;139 bool reuse = sk->sk_reuse;
223 u32 hash;140 bool reuseport = !!sk->sk_reuseport;
141 struct sock_reuseport *reuseport_cb;
142 kuid_t uid = sock_i_uid((struct sock *)sk);
224 143
225 rcu_read_lock();144 rcu_read_lock();
226 reuseport_cb = rcu_dereference(sk->sk_reuseport_cb);145 reuseport_cb = rcu_dereference(sk->sk_reuseport_cb);
@@ -231,42 +150,40 @@ static int inet_csk_bind_conflict(const struct sock *sk, int port, · 4 unmodified lines
231 /*150 /*
232 * Unlike other sk lookup places we do not check151 * Unlike other sk lookup places we do not check
233 * for sk_net here, since _all_ the socks listed152 * for sk_net here, since _all_ the socks listed
234 * in tb->owners and tb2->owners list belong153 * in tb->owners list belong to the same net - the
235 * to the same net154 * one this bucket belongs to.
236 */155 */
237 156
238 if (!use_bhash2_on_bind(sk)) {157 sk_for_each_bound(sk2, &tb->owners) {
239 sk_for_each_bound(sk2, &tb->owners)158 int bound_dev_if2;
240 if (bind_conflict_exist(sk, sk2, uid, relax,
241 reuseport_cb_ok, reuseport_ok) &&
242 inet_rcv_saddr_equal(sk, sk2, true))
243 return true;
244 159
245 return false;160 if (sk == sk2)
161 continue;
162 bound_dev_if2 = READ_ONCE(sk2->sk_bound_dev_if);
163 if ((!sk->sk_bound_dev_if ||
164 !bound_dev_if2 ||
165 sk->sk_bound_dev_if == bound_dev_if2)) {
166 if (reuse && sk2->sk_reuse &&
167 sk2->sk_state != TCP_LISTEN) {
168 if ((!relax ||
169 (!reuseport_ok &&
170 reuseport && sk2->sk_reuseport &&
171 reuseport_cb_ok &&
172 (sk2->sk_state == TCP_TIME_WAIT ||
173 uid_eq(uid, sock_i_uid(sk2))))) &&
174 inet_rcv_saddr_equal(sk, sk2, true))
175 break;
176 } else if (!reuseport_ok ||
177 !reuseport || !sk2->sk_reuseport ||
178 !reuseport_cb_ok ||
179 (sk2->sk_state != TCP_TIME_WAIT &&
180 !uid_eq(uid, sock_i_uid(sk2)))) {
181 if (inet_rcv_saddr_equal(sk, sk2, true))
182 break;
183 }
184 }
246 }185 }
247 186 return sk2 != NULL;
248 if (tb2 && check_bhash2_conflict(sk, tb2, uid, relax, reuseport_cb_ok,
249 reuseport_ok))
250 return true;
251
252 net = sock_net(sk);
253
254 /* check there's no conflict with an existing IPV6_ADDR_ANY (if ipv6) or
255 * INADDR_ANY (if ipv4) socket.
256 */
257 hash = get_bhash2_nulladdr_hash(sk, net, port);
258 head2 = &hinfo->bhash2[hash & (hinfo->bhash_size - 1)];
259
260 l3mdev = inet_sk_bound_l3mdev(sk);
261 inet_bind_bucket_for_each(tb2, &head2->chain)
262 if (check_bind2_bucket_match_nulladdr(tb2, net, port, l3mdev, sk))
263 break;
264
265 if (tb2 && check_bhash2_conflict(sk, tb2, uid, relax, reuseport_cb_ok,
266 reuseport_ok))
267 return true;
268
269 return false;
270}187}
271 188
272/*189/*
@@ -274,20 +191,16 @@ static int inet_csk_bind_conflict(const struct sock *sk, int port, · 1 unmodified line
274 * inet_bind_hashbucket lock held.191 * inet_bind_hashbucket lock held.
275 */192 */
276static struct inet_bind_hashbucket *193static struct inet_bind_hashbucket *
277inet_csk_find_open_port(struct sock *sk, struct inet_bind_bucket **tb_ret,194inet_csk_find_open_port(struct sock *sk, struct inet_bind_bucket **tb_ret, int *port_ret)
278 struct inet_bind2_bucket **tb2_ret,
279 struct inet_bind2_hashbucket **head2_ret, int *port_ret)
280{195{
281 struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;196 struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;
282 struct inet_bind2_hashbucket *head2;197 int port = 0;
283 struct inet_bind_hashbucket *head;198 struct inet_bind_hashbucket *head;
284 struct net *net = sock_net(sk);199 struct net *net = sock_net(sk);
200 bool relax = false;
285 int i, low, high, attempt_half;201 int i, low, high, attempt_half;
286 struct inet_bind2_bucket *tb2;
287 struct inet_bind_bucket *tb;202 struct inet_bind_bucket *tb;
288 u32 remaining, offset;203 u32 remaining, offset;
289 bool relax = false;
290 int port = 0;
291 int l3mdev;204 int l3mdev;
292 205
293 l3mdev = inet_sk_bound_l3mdev(sk);206 l3mdev = inet_sk_bound_l3mdev(sk);
@@ -326,12 +239,10 @@ other_parity_scan: · 32 unmodified lines
326 head = &hinfo->bhash[inet_bhashfn(net, port,239 head = &hinfo->bhash[inet_bhashfn(net, port,
327 hinfo->bhash_size)];240 hinfo->bhash_size)];
328 spin_lock_bh(&head->lock);241 spin_lock_bh(&head->lock);
329 tb2 = inet_bind2_bucket_find(hinfo, net, port, l3mdev, sk,
330 &head2);
331 inet_bind_bucket_for_each(tb, &head->chain)242 inet_bind_bucket_for_each(tb, &head->chain)
332 if (check_bind_bucket_match(tb, net, port, l3mdev)) {243 if (net_eq(ib_net(tb), net) && tb->l3mdev == l3mdev &&
333 if (!inet_csk_bind_conflict(sk, port, tb, tb2,244 tb->port == port) {
334 relax, false))245 if (!inet_csk_bind_conflict(sk, tb, relax, false))
335 goto success;246 goto success;
336 goto next_port;247 goto next_port;
337 }248 }
@@ -361,8 +272,6 @@ next_port: · 23 unmodified lines
361success:272success:
362 *port_ret = port;273 *port_ret = port;
363 *tb_ret = tb;274 *tb_ret = tb;
364 *tb2_ret = tb2;
365 *head2_ret = head2;
366 return head;275 return head;
367}276}
368 277
@@ -458,81 +367,54 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum) · 89 unmodified lines
458{367{
459 bool reuse = sk->sk_reuse && sk->sk_state != TCP_LISTEN;368 bool reuse = sk->sk_reuse && sk->sk_state != TCP_LISTEN;
460 struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;369 struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;
461 bool bhash_created = false, bhash2_created = false;370 int ret = 1, port = snum;
462 struct inet_bind2_bucket *tb2 = NULL;
463 struct inet_bind2_hashbucket *head2;
464 struct inet_bind_bucket *tb = NULL;
465 struct inet_bind_hashbucket *head;371 struct inet_bind_hashbucket *head;
466 struct net *net = sock_net(sk);372 struct net *net = sock_net(sk);
467 int ret = 1, port = snum;373 struct inet_bind_bucket *tb = NULL;
468 bool found_port = false;
469 int l3mdev;374 int l3mdev;
470 375
471 l3mdev = inet_sk_bound_l3mdev(sk);376 l3mdev = inet_sk_bound_l3mdev(sk);
472 377
473 if (!port) {378 if (!port) {
474 head = inet_csk_find_open_port(sk, &tb, &tb2, &head2, &port);379 head = inet_csk_find_open_port(sk, &tb, &port);
475 if (!head)380 if (!head)
476 return ret;381 return ret;
477 if (tb && tb2)
478 goto success;
479 found_port = true;
480 } else {
481 head = &hinfo->bhash[inet_bhashfn(net, port,
482 hinfo->bhash_size)];
483 spin_lock_bh(&head->lock);
484 inet_bind_bucket_for_each(tb, &head->chain)
485 if (check_bind_bucket_match(tb, net, port, l3mdev))
486 break;
487
488 tb2 = inet_bind2_bucket_find(hinfo, net, port, l3mdev, sk,
489 &head2);
490 }
491
492 if (!tb) {
493 tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep, net,
494 head, port, l3mdev);
495 if (!tb)382 if (!tb)
496 goto fail_unlock;383 goto tb_not_found;
497 bhash_created = true;384 goto success;
498 }
499
500 if (!tb2) {
501 tb2 = inet_bind2_bucket_create(hinfo->bind2_bucket_cachep,
502 net, head2, port, l3mdev, sk);
503 if (!tb2)
504 goto fail_unlock;
505 bhash2_created = true;
506 }385 }
507 386 head = &hinfo->bhash[inet_bhashfn(net, port,
508 /* If we had to find an open port, we already checked for conflicts */387 hinfo->bhash_size)];
509 if (!found_port && !hlist_empty(&tb->owners)) {388 spin_lock_bh(&head->lock);
389 inet_bind_bucket_for_each(tb, &head->chain)
390 if (net_eq(ib_net(tb), net) && tb->l3mdev == l3mdev &&
391 tb->port == port)
392 goto tb_found;
393tb_not_found:
394 tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep,
395 net, head, port, l3mdev);
396 if (!tb)
397 goto fail_unlock;
398tb_found:
399 if (!hlist_empty(&tb->owners)) {
510 if (sk->sk_reuse == SK_FORCE_REUSE)400 if (sk->sk_reuse == SK_FORCE_REUSE)
511 goto success;401 goto success;
512 402
513 if ((tb->fastreuse > 0 && reuse) ||403 if ((tb->fastreuse > 0 && reuse) ||
514 sk_reuseport_match(tb, sk))404 sk_reuseport_match(tb, sk))
515 goto success;405 goto success;
516 if (inet_csk_bind_conflict(sk, port, tb, tb2, true, true))406 if (inet_csk_bind_conflict(sk, tb, true, true))
517 goto fail_unlock;407 goto fail_unlock;
518 }408 }
519success:409success:
520 inet_csk_update_fastreuse(tb, sk);410 inet_csk_update_fastreuse(tb, sk);
521 411
522 if (!inet_csk(sk)->icsk_bind_hash)412 if (!inet_csk(sk)->icsk_bind_hash)
523 inet_bind_hash(sk, tb, tb2, port);413 inet_bind_hash(sk, tb, port);
524 WARN_ON(inet_csk(sk)->icsk_bind_hash != tb);414 WARN_ON(inet_csk(sk)->icsk_bind_hash != tb);
525 WARN_ON(inet_csk(sk)->icsk_bind2_hash != tb2);
526 ret = 0;415 ret = 0;
527 416
528fail_unlock:417fail_unlock:
529 if (ret) {
530 if (bhash_created)
531 inet_bind_bucket_destroy(hinfo->bind_bucket_cachep, tb);
532 if (bhash2_created)
533 inet_bind2_bucket_destroy(hinfo->bind2_bucket_cachep,
534 tb2);
535 }
536 spin_unlock_bh(&head->lock);418 spin_unlock_bh(&head->lock);
537 return ret;419 return ret;
538}420}
@@ -1079,7 +961,6 @@ struct sock *inet_csk_clone_lock(const struct sock *sk, · 540 unmodified lines
1079 961
1080 inet_sk_set_state(newsk, TCP_SYN_RECV);962 inet_sk_set_state(newsk, TCP_SYN_RECV);
1081 newicsk->icsk_bind_hash = NULL;963 newicsk->icsk_bind_hash = NULL;
1082 newicsk->icsk_bind2_hash = NULL;
1083 964
1084 inet_sk(newsk)->inet_dport = inet_rsk(req)->ir_rmt_port;965 inet_sk(newsk)->inet_dport = inet_rsk(req)->ir_rmt_port;
1085 inet_sk(newsk)->inet_num = inet_rsk(req)->ir_num;966 inet_sk(newsk)->inet_num = inet_rsk(req)->ir_num;
net/ipv4/inet_hashtables.c+11−182
@@ -81,41 +81,6 @@ struct inet_bind_bucket *inet_bind_bucket_create(struct kmem_cache *cachep, · 80 unmodified lines
81 return tb;81 return tb;
82}82}
83 83
84struct inet_bind2_bucket *inet_bind2_bucket_create(struct kmem_cache *cachep,
85 struct net *net,
86 struct inet_bind2_hashbucket *head,
87 const unsigned short port,
88 int l3mdev,
89 const struct sock *sk)
90{
91 struct inet_bind2_bucket *tb = kmem_cache_alloc(cachep, GFP_ATOMIC);
92
93 if (tb) {
94 write_pnet(&tb->ib_net, net);
95 tb->l3mdev = l3mdev;
96 tb->port = port;
97#if IS_ENABLED(CONFIG_IPV6)
98 if (sk->sk_family == AF_INET6)
99 tb->v6_rcv_saddr = sk->sk_v6_rcv_saddr;
100 else
101#endif
102 tb->rcv_saddr = sk->sk_rcv_saddr;
103 INIT_HLIST_HEAD(&tb->owners);
104 hlist_add_head(&tb->node, &head->chain);
105 }
106 return tb;
107}
108
109static bool bind2_bucket_addr_match(struct inet_bind2_bucket *tb2, struct sock *sk)
110{
111#if IS_ENABLED(CONFIG_IPV6)
112 if (sk->sk_family == AF_INET6)
113 return ipv6_addr_equal(&tb2->v6_rcv_saddr,
114 &sk->sk_v6_rcv_saddr);
115#endif
116 return tb2->rcv_saddr == sk->sk_rcv_saddr;
117}
118
119/*84/*
120 * Caller must hold hashbucket lock for this tb with local BH disabled85 * Caller must hold hashbucket lock for this tb with local BH disabled
121 */86 */
@@ -127,25 +92,12 @@ void inet_bind_bucket_destroy(struct kmem_cache *cachep, struct inet_bind_bucket · 5 unmodified lines
127 }92 }
128}93}
129 94
130/* Caller must hold the lock for the corresponding hashbucket in the bhash table
131 * with local BH disabled
132 */
133void inet_bind2_bucket_destroy(struct kmem_cache *cachep, struct inet_bind2_bucket *tb)
134{
135 if (hlist_empty(&tb->owners)) {
136 __hlist_del(&tb->node);
137 kmem_cache_free(cachep, tb);
138 }
139}
140
141void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb,95void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb,
142 struct inet_bind2_bucket *tb2, const unsigned short snum)96 const unsigned short snum)
143{97{
144 inet_sk(sk)->inet_num = snum;98 inet_sk(sk)->inet_num = snum;
145 sk_add_bind_node(sk, &tb->owners);99 sk_add_bind_node(sk, &tb->owners);
146 inet_csk(sk)->icsk_bind_hash = tb;100 inet_csk(sk)->icsk_bind_hash = tb;
147 sk_add_bind2_node(sk, &tb2->owners);
148 inet_csk(sk)->icsk_bind2_hash = tb2;
149}101}
150 102
151/*103/*
@@ -157,7 +109,6 @@ static void __inet_put_port(struct sock *sk) · 5 unmodified lines
157 const int bhash = inet_bhashfn(sock_net(sk), inet_sk(sk)->inet_num,109 const int bhash = inet_bhashfn(sock_net(sk), inet_sk(sk)->inet_num,
158 hashinfo->bhash_size);110 hashinfo->bhash_size);
159 struct inet_bind_hashbucket *head = &hashinfo->bhash[bhash];111 struct inet_bind_hashbucket *head = &hashinfo->bhash[bhash];
160 struct inet_bind2_bucket *tb2;
161 struct inet_bind_bucket *tb;112 struct inet_bind_bucket *tb;
162 113
163 spin_lock(&head->lock);114 spin_lock(&head->lock);
@@ -166,13 +117,6 @@ static void __inet_put_port(struct sock *sk) · 2 unmodified lines
166 inet_csk(sk)->icsk_bind_hash = NULL;117 inet_csk(sk)->icsk_bind_hash = NULL;
167 inet_sk(sk)->inet_num = 0;118 inet_sk(sk)->inet_num = 0;
168 inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);119 inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
169
170 if (inet_csk(sk)->icsk_bind2_hash) {
171 tb2 = inet_csk(sk)->icsk_bind2_hash;
172 __sk_del_bind2_node(sk);
173 inet_csk(sk)->icsk_bind2_hash = NULL;
174 inet_bind2_bucket_destroy(hashinfo->bind2_bucket_cachep, tb2);
175 }
176 spin_unlock(&head->lock);120 spin_unlock(&head->lock);
177}121}
178 122
@@ -189,19 +133,14 @@ int __inet_inherit_port(const struct sock *sk, struct sock *child) · 10 unmodified lines
189 struct inet_hashinfo *table = sk->sk_prot->h.hashinfo;133 struct inet_hashinfo *table = sk->sk_prot->h.hashinfo;
190 unsigned short port = inet_sk(child)->inet_num;134 unsigned short port = inet_sk(child)->inet_num;
191 const int bhash = inet_bhashfn(sock_net(sk), port,135 const int bhash = inet_bhashfn(sock_net(sk), port,
192 table->bhash_size);136 table->bhash_size);
193 struct inet_bind_hashbucket *head = &table->bhash[bhash];137 struct inet_bind_hashbucket *head = &table->bhash[bhash];
194 struct inet_bind2_hashbucket *head_bhash2;
195 bool created_inet_bind_bucket = false;
196 struct net *net = sock_net(sk);
197 struct inet_bind2_bucket *tb2;
198 struct inet_bind_bucket *tb;138 struct inet_bind_bucket *tb;
199 int l3mdev;139 int l3mdev;
200 140
201 spin_lock(&head->lock);141 spin_lock(&head->lock);
202 tb = inet_csk(sk)->icsk_bind_hash;142 tb = inet_csk(sk)->icsk_bind_hash;
203 tb2 = inet_csk(sk)->icsk_bind2_hash;143 if (unlikely(!tb)) {
204 if (unlikely(!tb || !tb2)) {
205 spin_unlock(&head->lock);144 spin_unlock(&head->lock);
206 return -ENOENT;145 return -ENOENT;
207 }146 }
@@ -214,45 +153,25 @@ int __inet_inherit_port(const struct sock *sk, struct sock *child) · 6 unmodified lines
214 * as that of the child socket. We have to look up or153 * as that of the child socket. We have to look up or
215 * create a new bind bucket for the child here. */154 * create a new bind bucket for the child here. */
216 inet_bind_bucket_for_each(tb, &head->chain) {155 inet_bind_bucket_for_each(tb, &head->chain) {
217 if (check_bind_bucket_match(tb, net, port, l3mdev))156 if (net_eq(ib_net(tb), sock_net(sk)) &&
157 tb->l3mdev == l3mdev && tb->port == port)
218 break;158 break;
219 }159 }
220 if (!tb) {160 if (!tb) {
221 tb = inet_bind_bucket_create(table->bind_bucket_cachep,161 tb = inet_bind_bucket_create(table->bind_bucket_cachep,
222 net, head, port, l3mdev);162 sock_net(sk), head, port,
163 l3mdev);
223 if (!tb) {164 if (!tb) {
224 spin_unlock(&head->lock);165 spin_unlock(&head->lock);
225 return -ENOMEM;166 return -ENOMEM;
226 }167 }
227 created_inet_bind_bucket = true;
228 }168 }
229 inet_csk_update_fastreuse(tb, child);169 inet_csk_update_fastreuse(tb, child);
230
231 goto bhash2_find;
232 } else if (!bind2_bucket_addr_match(tb2, child)) {
233 l3mdev = inet_sk_bound_l3mdev(sk);
234
235bhash2_find:
236 tb2 = inet_bind2_bucket_find(table, net, port, l3mdev, child,
237 &head_bhash2);
238 if (!tb2) {
239 tb2 = inet_bind2_bucket_create(table->bind2_bucket_cachep,
240 net, head_bhash2, port,
241 l3mdev, child);
242 if (!tb2)
243 goto error;
244 }
245 }170 }
246 inet_bind_hash(child, tb, tb2, port);171 inet_bind_hash(child, tb, port);
247 spin_unlock(&head->lock);172 spin_unlock(&head->lock);
248 173
249 return 0;174 return 0;
250
251error:
252 if (created_inet_bind_bucket)
253 inet_bind_bucket_destroy(table->bind_bucket_cachep, tb);
254 spin_unlock(&head->lock);
255 return -ENOMEM;
256}175}
257EXPORT_SYMBOL_GPL(__inet_inherit_port);176EXPORT_SYMBOL_GPL(__inet_inherit_port);
258 177
@@ -756,76 +675,6 @@ void inet_unhash(struct sock *sk) · 497 unmodified lines
756}675}
757EXPORT_SYMBOL_GPL(inet_unhash);676EXPORT_SYMBOL_GPL(inet_unhash);
758 677
759static bool check_bind2_bucket_match(struct inet_bind2_bucket *tb,
760 struct net *net, unsigned short port,
761 int l3mdev, struct sock *sk)
762{
763#if IS_ENABLED(CONFIG_IPV6)
764 if (sk->sk_family == AF_INET6)
765 return net_eq(ib2_net(tb), net) && tb->port == port &&
766 tb->l3mdev == l3mdev &&
767 ipv6_addr_equal(&tb->v6_rcv_saddr, &sk->sk_v6_rcv_saddr);
768 else
769#endif
770 return net_eq(ib2_net(tb), net) && tb->port == port &&
771 tb->l3mdev == l3mdev && tb->rcv_saddr == sk->sk_rcv_saddr;
772}
773
774bool check_bind2_bucket_match_nulladdr(struct inet_bind2_bucket *tb,
775 struct net *net, const unsigned short port,
776 int l3mdev, const struct sock *sk)
777{
778#if IS_ENABLED(CONFIG_IPV6)
779 struct in6_addr nulladdr = {};
780
781 if (sk->sk_family == AF_INET6)
782 return net_eq(ib2_net(tb), net) && tb->port == port &&
783 tb->l3mdev == l3mdev &&
784 ipv6_addr_equal(&tb->v6_rcv_saddr, &nulladdr);
785 else
786#endif
787 return net_eq(ib2_net(tb), net) && tb->port == port &&
788 tb->l3mdev == l3mdev && tb->rcv_saddr == 0;
789}
790
791static struct inet_bind2_hashbucket *
792inet_bhashfn_portaddr(struct inet_hashinfo *hinfo, const struct sock *sk,
793 const struct net *net, unsigned short port)
794{
795 u32 hash;
796
797#if IS_ENABLED(CONFIG_IPV6)
798 if (sk->sk_family == AF_INET6)
799 hash = ipv6_portaddr_hash(net, &sk->sk_v6_rcv_saddr, port);
800 else
801#endif
802 hash = ipv4_portaddr_hash(net, sk->sk_rcv_saddr, port);
803 return &hinfo->bhash2[hash & (hinfo->bhash_size - 1)];
804}
805
806/* This should only be called when the spinlock for the socket's corresponding
807 * bind_hashbucket is held
808 */
809struct inet_bind2_bucket *
810inet_bind2_bucket_find(struct inet_hashinfo *hinfo, struct net *net,
811 const unsigned short port, int l3mdev, struct sock *sk,
812 struct inet_bind2_hashbucket **head)
813{
814 struct inet_bind2_bucket *bhash2 = NULL;
815 struct inet_bind2_hashbucket *h;
816
817 h = inet_bhashfn_portaddr(hinfo, sk, net, port);
818 inet_bind_bucket_for_each(bhash2, &h->chain) {
819 if (check_bind2_bucket_match(bhash2, net, port, l3mdev, sk))
820 break;
821 }
822
823 if (head)
824 *head = h;
825
826 return bhash2;
827}
828
829/* RFC 6056 3.3.4. Algorithm 4: Double-Hash Port Selection Algorithm678/* RFC 6056 3.3.4. Algorithm 4: Double-Hash Port Selection Algorithm
830 * Note that we use 32bit integers (vs RFC 'short integers')679 * Note that we use 32bit integers (vs RFC 'short integers')
831 * because 2^16 is not a multiple of num_ephemeral and this680 * because 2^16 is not a multiple of num_ephemeral and this
@@ -846,13 +695,10 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row, · 14 unmodified lines
846{695{
847 struct inet_hashinfo *hinfo = death_row->hashinfo;696 struct inet_hashinfo *hinfo = death_row->hashinfo;
848 struct inet_timewait_sock *tw = NULL;697 struct inet_timewait_sock *tw = NULL;
849 struct inet_bind2_hashbucket *head2;
850 struct inet_bind_hashbucket *head;698 struct inet_bind_hashbucket *head;
851 int port = inet_sk(sk)->inet_num;699 int port = inet_sk(sk)->inet_num;
852 struct net *net = sock_net(sk);700 struct net *net = sock_net(sk);
853 struct inet_bind2_bucket *tb2;
854 struct inet_bind_bucket *tb;701 struct inet_bind_bucket *tb;
855 bool tb_created = false;
856 u32 remaining, offset;702 u32 remaining, offset;
857 int ret, i, low, high;703 int ret, i, low, high;
858 int l3mdev;704 int l3mdev;
@@ -909,7 +755,8 @@ other_parity_scan: · 50 unmodified lines
909 * the established check is already unique enough.755 * the established check is already unique enough.
910 */756 */
911 inet_bind_bucket_for_each(tb, &head->chain) {757 inet_bind_bucket_for_each(tb, &head->chain) {
912 if (check_bind_bucket_match(tb, net, port, l3mdev)) {758 if (net_eq(ib_net(tb), net) && tb->l3mdev == l3mdev &&
759 tb->port == port) {
913 if (tb->fastreuse >= 0 ||760 if (tb->fastreuse >= 0 ||
914 tb->fastreuseport >= 0)761 tb->fastreuseport >= 0)
915 goto next_port;762 goto next_port;
@@ -927,7 +774,6 @@ other_parity_scan: · 11 unmodified lines
927 spin_unlock_bh(&head->lock);774 spin_unlock_bh(&head->lock);
928 return -ENOMEM;775 return -ENOMEM;
929 }776 }
930 tb_created = true;
931 tb->fastreuse = -1;777 tb->fastreuse = -1;
932 tb->fastreuseport = -1;778 tb->fastreuseport = -1;
933 goto ok;779 goto ok;
@@ -943,17 +789,6 @@ next_port: · 9 unmodified lines
943 return -EADDRNOTAVAIL;789 return -EADDRNOTAVAIL;
944 790
945ok:791ok:
946 /* Find the corresponding tb2 bucket since we need to
947 * add the socket to the bhash2 table as well
948 */
949 tb2 = inet_bind2_bucket_find(hinfo, net, port, l3mdev, sk, &head2);
950 if (!tb2) {
951 tb2 = inet_bind2_bucket_create(hinfo->bind2_bucket_cachep, net,
952 head2, port, l3mdev, sk);
953 if (!tb2)
954 goto error;
955 }
956
957 /* Here we want to add a little bit of randomness to the next source792 /* Here we want to add a little bit of randomness to the next source
958 * port that will be chosen. We use a max() with a random here so that793 * port that will be chosen. We use a max() with a random here so that
959 * on low contention the randomness is maximal and on high contention794 * on low contention the randomness is maximal and on high contention
@@ -963,7 +798,7 @@ ok: · 3 unmodified lines
963 WRITE_ONCE(table_perturb[index], READ_ONCE(table_perturb[index]) + i + 2);798 WRITE_ONCE(table_perturb[index], READ_ONCE(table_perturb[index]) + i + 2);
964 799
965 /* Head lock still held and bh's disabled */800 /* Head lock still held and bh's disabled */
966 inet_bind_hash(sk, tb, tb2, port);801 inet_bind_hash(sk, tb, port);
967 if (sk_unhashed(sk)) {802 if (sk_unhashed(sk)) {
968 inet_sk(sk)->inet_sport = htons(port);803 inet_sk(sk)->inet_sport = htons(port);
969 inet_ehash_nolisten(sk, (struct sock *)tw, NULL);804 inet_ehash_nolisten(sk, (struct sock *)tw, NULL);
@@ -975,12 +810,6 @@ ok: · 5 unmodified lines
975 inet_twsk_deschedule_put(tw);810 inet_twsk_deschedule_put(tw);
976 local_bh_enable();811 local_bh_enable();
977 return 0;812 return 0;
978
979error:
980 if (tb_created)
981 inet_bind_bucket_destroy(hinfo->bind_bucket_cachep, tb);
982 spin_unlock_bh(&head->lock);
983 return -ENOMEM;
984}813}
985 814
986/*815/*
net/ipv4/tcp.c+2−12
@@ -4604,12 +4604,6 @@ void __init tcp_init(void) · 4603 unmodified lines
4604 SLAB_HWCACHE_ALIGN | SLAB_PANIC |4604 SLAB_HWCACHE_ALIGN | SLAB_PANIC |
4605 SLAB_ACCOUNT,4605 SLAB_ACCOUNT,
4606 NULL);4606 NULL);
4607 tcp_hashinfo.bind2_bucket_cachep =
4608 kmem_cache_create("tcp_bind2_bucket",
4609 sizeof(struct inet_bind2_bucket), 0,
4610 SLAB_HWCACHE_ALIGN | SLAB_PANIC |
4611 SLAB_ACCOUNT,
4612 NULL);
4613 4607
4614 /* Size and allocate the main established and bind bucket4608 /* Size and allocate the main established and bind bucket
4615 * hash tables.4609 * hash tables.
@@ -4632,9 +4626,8 @@ void __init tcp_init(void) · 16 unmodified lines
4632 if (inet_ehash_locks_alloc(&tcp_hashinfo))4626 if (inet_ehash_locks_alloc(&tcp_hashinfo))
4633 panic("TCP: failed to alloc ehash_locks");4627 panic("TCP: failed to alloc ehash_locks");
4634 tcp_hashinfo.bhash =4628 tcp_hashinfo.bhash =
4635 alloc_large_system_hash("TCP bind bhash tables",4629 alloc_large_system_hash("TCP bind",
4636 sizeof(struct inet_bind_hashbucket) +4630 sizeof(struct inet_bind_hashbucket),
4637 sizeof(struct inet_bind2_hashbucket),
4638 tcp_hashinfo.ehash_mask + 1,4631 tcp_hashinfo.ehash_mask + 1,
4639 17, /* one slot per 128 KB of memory */4632 17, /* one slot per 128 KB of memory */
4640 0,4633 0,
@@ -4643,12 +4636,9 @@ void __init tcp_init(void) · 2 unmodified lines
4643 0,4636 0,
4644 64 * 1024);4637 64 * 1024);
4645 tcp_hashinfo.bhash_size = 1U << tcp_hashinfo.bhash_size;4638 tcp_hashinfo.bhash_size = 1U << tcp_hashinfo.bhash_size;
4646 tcp_hashinfo.bhash2 =
4647 (struct inet_bind2_hashbucket *)(tcp_hashinfo.bhash + tcp_hashinfo.bhash_size);
4648 for (i = 0; i < tcp_hashinfo.bhash_size; i++) {4639 for (i = 0; i < tcp_hashinfo.bhash_size; i++) {
4649 spin_lock_init(&tcp_hashinfo.bhash[i].lock);4640 spin_lock_init(&tcp_hashinfo.bhash[i].lock);
4650 INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain);4641 INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain);
4651 INIT_HLIST_HEAD(&tcp_hashinfo.bhash2[i].chain);
4652 }4642 }
4653 4643
4654 4644
tools/testing/selftests/net/.gitignore+0−1
@@ -37,4 +37,3 @@ gro · 36 unmodified lines
37ioam6_parser37ioam6_parser
38toeplitz38toeplitz
39cmsg_sender39cmsg_sender
40bind_bhash_test
tools/testing/selftests/net/Makefile+0−2
@@ -59,7 +59,6 @@ TEST_GEN_FILES += toeplitz · 58 unmodified lines
59TEST_GEN_FILES += cmsg_sender59TEST_GEN_FILES += cmsg_sender
60TEST_GEN_FILES += stress_reuseport_listen60TEST_GEN_FILES += stress_reuseport_listen
61TEST_PROGS += test_vxlan_vnifiltering.sh61TEST_PROGS += test_vxlan_vnifiltering.sh
62TEST_GEN_FILES += bind_bhash_test
63 62
64TEST_FILES := settings63TEST_FILES := settings
65 64
@@ -70,5 +69,4 @@ include bpf/Makefile · 4 unmodified lines
70 69
71$(OUTPUT)/reuseport_bpf_numa: LDLIBS += -lnuma70$(OUTPUT)/reuseport_bpf_numa: LDLIBS += -lnuma
72$(OUTPUT)/tcp_mmap: LDLIBS += -lpthread71$(OUTPUT)/tcp_mmap: LDLIBS += -lpthread
73$(OUTPUT)/bind_bhash_test: LDLIBS += -lpthread
74$(OUTPUT)/tcp_inq: LDLIBS += -lpthread72$(OUTPUT)/tcp_inq: LDLIBS += -lpthread
tools/testing/selftests/net/bind_bhash_test.c+0−119
@@ -1,119 +0,0 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * This times how long it takes to bind to a port when the port already
4 * has multiple sockets in its bhash table.
5 *
6 * In the setup(), we populate the port's bhash table with
7 * MAX_THREADS * MAX_CONNECTIONS number of entries.
8 */
9
10#include <unistd.h>
11#include <stdio.h>
12#include <netdb.h>
13#include <pthread.h>
14
15#define MAX_THREADS 600
16#define MAX_CONNECTIONS 40
17
18static const char *bind_addr = "::1";
19static const char *port;
20
21static int fd_array[MAX_THREADS][MAX_CONNECTIONS];
22
23static int bind_socket(int opt, const char *addr)
24{
25 struct addrinfo *res, hint = {};
26 int sock_fd, reuse = 1, err;
27
28 sock_fd = socket(AF_INET6, SOCK_STREAM, 0);
29 if (sock_fd < 0) {
30 perror("socket fd err");
31 return -1;
32 }
33
34 hint.ai_family = AF_INET6;
35 hint.ai_socktype = SOCK_STREAM;
36
37 err = getaddrinfo(addr, port, &hint, &res);
38 if (err) {
39 perror("getaddrinfo failed");
40 return -1;
41 }
42
43 if (opt) {
44 err = setsockopt(sock_fd, SOL_SOCKET, opt, &reuse, sizeof(reuse));
45 if (err) {
46 perror("setsockopt failed");
47 return -1;
48 }
49 }
50
51 err = bind(sock_fd, res->ai_addr, res->ai_addrlen);
52 if (err) {
53 perror("failed to bind to port");
54 return -1;
55 }
56
57 return sock_fd;
58}
59
60static void *setup(void *arg)
61{
62 int sock_fd, i;
63 int *array = (int *)arg;
64
65 for (i = 0; i < MAX_CONNECTIONS; i++) {
66 sock_fd = bind_socket(SO_REUSEADDR | SO_REUSEPORT, bind_addr);
67 if (sock_fd < 0)
68 return NULL;
69 array[i] = sock_fd;
70 }
71
72 return NULL;
73}
74
75int main(int argc, const char *argv[])
76{
77 int listener_fd, sock_fd, i, j;
78 pthread_t tid[MAX_THREADS];
79 clock_t begin, end;
80
81 if (argc != 2) {
82 printf("Usage: listener <port>\n");
83 return -1;
84 }
85
86 port = argv[1];
87
88 listener_fd = bind_socket(SO_REUSEADDR | SO_REUSEPORT, bind_addr);
89 if (listen(listener_fd, 100) < 0) {
90 perror("listen failed");
91 return -1;
92 }
93
94 /* Set up threads to populate the bhash table entry for the port */
95 for (i = 0; i < MAX_THREADS; i++)
96 pthread_create(&tid[i], NULL, setup, fd_array[i]);
97
98 for (i = 0; i < MAX_THREADS; i++)
99 pthread_join(tid[i], NULL);
100
101 begin = clock();
102
103 /* Bind to the same port on a different address */
104 sock_fd = bind_socket(0, "2001:0db8:0:f101::1");
105
106 end = clock();
107
108 printf("time spent = %f\n", (double)(end - begin) / CLOCKS_PER_SEC);
109
110 /* clean up */
111 close(sock_fd);
112 close(listener_fd);
113 for (i = 0; i < MAX_THREADS; i++) {
114 for (j = 0; i < MAX_THREADS; i++)
115 close(fd_array[i][j]);
116 }
117
118 return 0;
119}