History

Revert "net: Add a second bind table hashed by port and address"

This reverts: commit d5a42de8bdbe ("net: Add a second bind table hashed by port and address") commit 538aaf9b2383 ("selftests: Add test for timing a bind request to a port with a populated bhash entry") Link: https://lore.kernel.org/netdev/20220520001834.2247810-1-kuba@kernel.org/ There are a few things that need to be fixed here: * Updating bhash2 in cases where the socket's rcv saddr changes * Adding bhash2 hashbucket locks Links to syzbot reports: https://lore.kernel.org/netdev/00000000000022208805e0df247a@google.com/ https://lore.kernel.org/netdev/0000000000003f33bc05dfaf44fe@google.com/

Joanne Koong committed 4 years ago

Commit 593d1eb1 parent 219b51a10 files · +83 −611Browse files

Unified · Split

10 files
  1. include/net/inet_connection_sock.h +0 −3
  2. include/net/inet_hashtables.h +1 −67
  3. include/net/sock.h +0 −14
  4. net/dccp/proto.c +5 −28
  5. net/ipv4/inet_connection_sock.c +64 −183
  6. net/ipv4/inet_hashtables.c +11 −182
  7. net/ipv4/tcp.c +2 −12
  8. tools/testing/selftests/net/.gitignore +0 −1
  9. tools/testing/selftests/net/Makefile +0 −2
  10. tools/testing/selftests/net/bind_bhash_test.c +0 −119
include/net/inet_connection_sock.h+0−3
@@ -25,7 +25,6 @@ · 24 unmodified lines
2525 #undef INET_CSK_CLEAR_TIMERS
2626
2727 struct inet_bind_bucket;
28−struct inet_bind2_bucket;
2928 struct tcp_congestion_ops;
3029
3130 /*
@@ -58,7 +57,6 @@ struct inet_connection_sock_af_ops { · 26 unmodified lines
5857 *
5958 * @icsk_accept_queue: FIFO of established children
6059 * @icsk_bind_hash: Bind node
61− * @icsk_bind2_hash: Bind node in the bhash2 table
6260 * @icsk_timeout: Timeout
6361 * @icsk_retransmit_timer: Resend (no ack)
6462 * @icsk_rto: Retransmit timeout
@@ -85,7 +83,6 @@ struct inet_connection_sock { · 20 unmodified lines
8583 struct inet_sock icsk_inet;
8684 struct request_sock_queue icsk_accept_queue;
8785 struct inet_bind_bucket *icsk_bind_hash;
88− struct inet_bind2_bucket *icsk_bind2_hash;
8986 unsigned long icsk_timeout;
9087 struct timer_list icsk_retransmit_timer;
9188 struct timer_list icsk_delack_timer;
include/net/inet_hashtables.h+1−67
@@ -90,32 +90,11 @@ struct inet_bind_bucket { · 89 unmodified lines
9090 struct hlist_head owners;
9191 };
9292
93−struct inet_bind2_bucket {
94− possible_net_t ib_net;
95− int l3mdev;
96− unsigned short port;
97− union {
98−#if IS_ENABLED(CONFIG_IPV6)
99− struct in6_addr v6_rcv_saddr;
100−#endif
101− __be32 rcv_saddr;
102− };
103− /* Node in the inet2_bind_hashbucket chain */
104− struct hlist_node node;
105− /* List of sockets hashed to this bucket */
106− struct hlist_head owners;
107−};
108−
10993 static inline struct net *ib_net(struct inet_bind_bucket *ib)
11094 {
11195 return read_pnet(&ib->ib_net);
11296 }
11397
114−static inline struct net *ib2_net(struct inet_bind2_bucket *ib)
115−{
116− return read_pnet(&ib->ib_net);
117−}
118−
11998 #define inet_bind_bucket_for_each(tb, head) \
12099 hlist_for_each_entry(tb, head, node)
121100
@@ -124,15 +103,6 @@ struct inet_bind_hashbucket { · 2 unmodified lines
124103 struct hlist_head chain;
125104 };
126105
127−/* This is synchronized using the inet_bind_hashbucket's spinlock.
128− * Instead of having separate spinlocks, the inet_bind2_hashbucket can share
129− * the inet_bind_hashbucket's given that in every case where the bhash2 table
130− * is useful, a lookup in the bhash table also occurs.
131− */
132−struct inet_bind2_hashbucket {
133− struct hlist_head chain;
134−};
135−
136106 /* Sockets can be hashed in established or listening table.
137107 * We must use different 'nulls' end-of-chain value for all hash buckets :
138108 * A socket might transition from ESTABLISH to LISTEN state without
@@ -164,12 +134,6 @@ struct inet_hashinfo { · 25 unmodified lines
164134 */
165135 struct kmem_cache *bind_bucket_cachep;
166136 struct inet_bind_hashbucket *bhash;
167− /* The 2nd binding table hashed by port and address.
168− * This is used primarily for expediting the resolution of bind
169− * conflicts.
170− */
171− struct kmem_cache *bind2_bucket_cachep;
172− struct inet_bind2_hashbucket *bhash2;
173137 unsigned int bhash_size;
174138
175139 /* The 2nd listener table hashed by local port and address */
@@ -229,36 +193,6 @@ inet_bind_bucket_create(struct kmem_cache *cachep, struct net *net, · 53 unmodified lines
229193 void inet_bind_bucket_destroy(struct kmem_cache *cachep,
230194 struct inet_bind_bucket *tb);
231195
232−static inline bool check_bind_bucket_match(struct inet_bind_bucket *tb,
233− struct net *net,
234− const unsigned short port,
235− int l3mdev)
236−{
237− return net_eq(ib_net(tb), net) && tb->port == port &&
238− tb->l3mdev == l3mdev;
239−}
240−
241−struct inet_bind2_bucket *
242−inet_bind2_bucket_create(struct kmem_cache *cachep, struct net *net,
243− struct inet_bind2_hashbucket *head,
244− const unsigned short port, int l3mdev,
245− const struct sock *sk);
246−
247−void inet_bind2_bucket_destroy(struct kmem_cache *cachep,
248− struct inet_bind2_bucket *tb);
249−
250−struct inet_bind2_bucket *
251−inet_bind2_bucket_find(struct inet_hashinfo *hinfo, struct net *net,
252− const unsigned short port, int l3mdev,
253− struct sock *sk,
254− struct inet_bind2_hashbucket **head);
255−
256−bool check_bind2_bucket_match_nulladdr(struct inet_bind2_bucket *tb,
257− struct net *net,
258− const unsigned short port,
259− int l3mdev,
260− const struct sock *sk);
261−
262196 static inline u32 inet_bhashfn(const struct net *net, const __u16 lport,
263197 const u32 bhash_size)
264198 {
@@ -266,7 +200,7 @@ static inline u32 inet_bhashfn(const struct net *net, const __u16 lport, · 1 unmodified line
266200 }
267201
268202 void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb,
269− struct inet_bind2_bucket *tb2, const unsigned short snum);
203+ const unsigned short snum);
270204
271205 /* Caller must disable local BH processing. */
272206 int __inet_inherit_port(const struct sock *sk, struct sock *child);
include/net/sock.h+0−14
@@ -348,7 +348,6 @@ struct sk_filter; · 347 unmodified lines
348348 * @sk_txtime_report_errors: set report errors mode for SO_TXTIME
349349 * @sk_txtime_unused: unused txtime flags
350350 * @ns_tracker: tracker for netns reference
351− * @sk_bind2_node: bind node in the bhash2 table
352351 */
353352 struct sock {
354353 /*
@@ -538,7 +537,6 @@ struct sock { · 183 unmodified lines
538537 #endif
539538 struct rcu_head sk_rcu;
540539 netns_tracker ns_tracker;
541− struct hlist_node sk_bind2_node;
542540 };
543541
544542 enum sk_pacing {
@@ -819,16 +817,6 @@ static inline void sk_add_bind_node(struct sock *sk, · 274 unmodified lines
819817 hlist_add_head(&sk->sk_bind_node, list);
820818 }
821819
822−static inline void __sk_del_bind2_node(struct sock *sk)
823−{
824− __hlist_del(&sk->sk_bind2_node);
825−}
826−
827−static inline void sk_add_bind2_node(struct sock *sk, struct hlist_head *list)
828−{
829− hlist_add_head(&sk->sk_bind2_node, list);
830−}
831−
832820 #define sk_for_each(__sk, list) \
833821 hlist_for_each_entry(__sk, list, sk_node)
834822 #define sk_for_each_rcu(__sk, list) \
@@ -846,8 +834,6 @@ static inline void sk_add_bind2_node(struct sock *sk, struct hlist_head *list) · 11 unmodified lines
846834 hlist_for_each_entry_safe(__sk, tmp, list, sk_node)
847835 #define sk_for_each_bound(__sk, list) \
848836 hlist_for_each_entry(__sk, list, sk_bind_node)
849−#define sk_for_each_bound_bhash2(__sk, list) \
850− hlist_for_each_entry(__sk, list, sk_bind2_node)
851837
852838 /**
853839 * sk_for_each_entry_offset_rcu - iterate over a list at a given struct offset
net/dccp/proto.c+5−28
@@ -1120,12 +1120,6 @@ static int __init dccp_init(void) · 1119 unmodified lines
11201120 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT, NULL);
11211121 if (!dccp_hashinfo.bind_bucket_cachep)
11221122 goto out_free_hashinfo2;
1123− dccp_hashinfo.bind2_bucket_cachep =
1124− kmem_cache_create("dccp_bind2_bucket",
1125− sizeof(struct inet_bind2_bucket), 0,
1126− SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT, NULL);
1127− if (!dccp_hashinfo.bind2_bucket_cachep)
1128− goto out_free_bind_bucket_cachep;
11291123
11301124 /*
11311125 * Size and allocate the main established and bind bucket
@@ -1156,7 +1150,7 @@ static int __init dccp_init(void) · 24 unmodified lines
11561150
11571151 if (!dccp_hashinfo.ehash) {
11581152 DCCP_CRIT("Failed to allocate DCCP established hash table");
1159− goto out_free_bind2_bucket_cachep;
1153+ goto out_free_bind_bucket_cachep;
11601154 }
11611155
11621156 for (i = 0; i <= dccp_hashinfo.ehash_mask; i++)
@@ -1182,23 +1176,14 @@ static int __init dccp_init(void) · 19 unmodified lines
11821176 goto out_free_dccp_locks;
11831177 }
11841178
1185− dccp_hashinfo.bhash2 = (struct inet_bind2_hashbucket *)
1186− __get_free_pages(GFP_ATOMIC | __GFP_NOWARN, bhash_order);
1187−
1188− if (!dccp_hashinfo.bhash2) {
1189− DCCP_CRIT("Failed to allocate DCCP bind2 hash table");
1190− goto out_free_dccp_bhash;
1191− }
1192−
11931179 for (i = 0; i < dccp_hashinfo.bhash_size; i++) {
11941180 spin_lock_init(&dccp_hashinfo.bhash[i].lock);
11951181 INIT_HLIST_HEAD(&dccp_hashinfo.bhash[i].chain);
1196− INIT_HLIST_HEAD(&dccp_hashinfo.bhash2[i].chain);
11971182 }
11981183
11991184 rc = dccp_mib_init();
12001185 if (rc)
1201− goto out_free_dccp_bhash2;
1186+ goto out_free_dccp_bhash;
12021187
12031188 rc = dccp_ackvec_init();
12041189 if (rc)
@@ -1222,38 +1207,30 @@ out_ackvec_exit: · 17 unmodified lines
12221207 dccp_ackvec_exit();
12231208 out_free_dccp_mib:
12241209 dccp_mib_exit();
1225−out_free_dccp_bhash2:
1226− free_pages((unsigned long)dccp_hashinfo.bhash2, bhash_order);
12271210 out_free_dccp_bhash:
12281211 free_pages((unsigned long)dccp_hashinfo.bhash, bhash_order);
12291212 out_free_dccp_locks:
12301213 inet_ehash_locks_free(&dccp_hashinfo);
12311214 out_free_dccp_ehash:
12321215 free_pages((unsigned long)dccp_hashinfo.ehash, ehash_order);
1233−out_free_bind2_bucket_cachep:
1234− kmem_cache_destroy(dccp_hashinfo.bind2_bucket_cachep);
12351216 out_free_bind_bucket_cachep:
12361217 kmem_cache_destroy(dccp_hashinfo.bind_bucket_cachep);
12371218 out_free_hashinfo2:
12381219 inet_hashinfo2_free_mod(&dccp_hashinfo);
12391220 out_fail:
12401221 dccp_hashinfo.bhash = NULL;
1241− dccp_hashinfo.bhash2 = NULL;
12421222 dccp_hashinfo.ehash = NULL;
12431223 dccp_hashinfo.bind_bucket_cachep = NULL;
1244− dccp_hashinfo.bind2_bucket_cachep = NULL;
12451224 return rc;
12461225 }
12471226
12481227 static void __exit dccp_fini(void)
12491228 {
1250− int bhash_order = get_order(dccp_hashinfo.bhash_size *
1251− sizeof(struct inet_bind_hashbucket));
1252−
12531229 ccid_cleanup_builtins();
12541230 dccp_mib_exit();
1255− free_pages((unsigned long)dccp_hashinfo.bhash, bhash_order);
1256− free_pages((unsigned long)dccp_hashinfo.bhash2, bhash_order);
1231+ free_pages((unsigned long)dccp_hashinfo.bhash,
1232+ get_order(dccp_hashinfo.bhash_size *
1233+ sizeof(struct inet_bind_hashbucket)));
12571234 free_pages((unsigned long)dccp_hashinfo.ehash,
12581235 get_order((dccp_hashinfo.ehash_mask + 1) *
12591236 sizeof(struct inet_ehash_bucket)));
net/ipv4/inet_connection_sock.c+64−183
@@ -117,32 +117,6 @@ bool inet_rcv_saddr_any(const struct sock *sk) · 116 unmodified lines
117117 return !sk->sk_rcv_saddr;
118118 }
119119
120−static bool use_bhash2_on_bind(const struct sock *sk)
121−{
122−#if IS_ENABLED(CONFIG_IPV6)
123− int addr_type;
124−
125− if (sk->sk_family == AF_INET6) {
126− addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr);
127− return addr_type != IPV6_ADDR_ANY &&
128− addr_type != IPV6_ADDR_MAPPED;
129− }
130−#endif
131− return sk->sk_rcv_saddr != htonl(INADDR_ANY);
132−}
133−
134−static u32 get_bhash2_nulladdr_hash(const struct sock *sk, struct net *net,
135− int port)
136−{
137−#if IS_ENABLED(CONFIG_IPV6)
138− struct in6_addr nulladdr = {};
139−
140− if (sk->sk_family == AF_INET6)
141− return ipv6_portaddr_hash(net, &nulladdr, port);
142−#endif
143− return ipv4_portaddr_hash(net, 0, port);
144−}
145−
146120 void inet_get_local_port_range(struct net *net, int *low, int *high)
147121 {
148122 unsigned int seq;
@@ -156,71 +130,16 @@ void inet_get_local_port_range(struct net *net, int *low, int *high) · 7 unmodified lines
156130 }
157131 EXPORT_SYMBOL(inet_get_local_port_range);
158132
159−static bool bind_conflict_exist(const struct sock *sk, struct sock *sk2,
160− kuid_t sk_uid, bool relax,
161− bool reuseport_cb_ok, bool reuseport_ok)
162−{
163− int bound_dev_if2;
164−
165− if (sk == sk2)
166− return false;
167−
168− bound_dev_if2 = READ_ONCE(sk2->sk_bound_dev_if);
169−
170− if (!sk->sk_bound_dev_if || !bound_dev_if2 ||
171− sk->sk_bound_dev_if == bound_dev_if2) {
172− if (sk->sk_reuse && sk2->sk_reuse &&
173− sk2->sk_state != TCP_LISTEN) {
174− if (!relax || (!reuseport_ok && sk->sk_reuseport &&
175− sk2->sk_reuseport && reuseport_cb_ok &&
176− (sk2->sk_state == TCP_TIME_WAIT ||
177− uid_eq(sk_uid, sock_i_uid(sk2)))))
178− return true;
179− } else if (!reuseport_ok || !sk->sk_reuseport ||
180− !sk2->sk_reuseport || !reuseport_cb_ok ||
181− (sk2->sk_state != TCP_TIME_WAIT &&
182− !uid_eq(sk_uid, sock_i_uid(sk2)))) {
183− return true;
184− }
185− }
186− return false;
187−}
188−
189−static bool check_bhash2_conflict(const struct sock *sk,
190− struct inet_bind2_bucket *tb2, kuid_t sk_uid,
191− bool relax, bool reuseport_cb_ok,
192− bool reuseport_ok)
193−{
194− struct sock *sk2;
195−
196− sk_for_each_bound_bhash2(sk2, &tb2->owners) {
197− if (sk->sk_family == AF_INET && ipv6_only_sock(sk2))
198− continue;
199−
200− if (bind_conflict_exist(sk, sk2, sk_uid, relax,
201− reuseport_cb_ok, reuseport_ok))
202− return true;
203− }
204− return false;
205−}
206−
207−/* This should be called only when the corresponding inet_bind_bucket spinlock
208− * is held
209− */
210−static int inet_csk_bind_conflict(const struct sock *sk, int port,
211− struct inet_bind_bucket *tb,
212− struct inet_bind2_bucket *tb2, /* may be null */
133+static int inet_csk_bind_conflict(const struct sock *sk,
134+ const struct inet_bind_bucket *tb,
213135 bool relax, bool reuseport_ok)
214136 {
215− struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;
216− kuid_t uid = sock_i_uid((struct sock *)sk);
217− struct sock_reuseport *reuseport_cb;
218− struct inet_bind2_hashbucket *head2;
219− bool reuseport_cb_ok;
220137 struct sock *sk2;
221− struct net *net;
222− int l3mdev;
223− u32 hash;
138+ bool reuseport_cb_ok;
139+ bool reuse = sk->sk_reuse;
140+ bool reuseport = !!sk->sk_reuseport;
141+ struct sock_reuseport *reuseport_cb;
142+ kuid_t uid = sock_i_uid((struct sock *)sk);
224143
225144 rcu_read_lock();
226145 reuseport_cb = rcu_dereference(sk->sk_reuseport_cb);
@@ -231,42 +150,40 @@ static int inet_csk_bind_conflict(const struct sock *sk, int port, · 4 unmodified lines
231150 /*
232151 * Unlike other sk lookup places we do not check
233152 * for sk_net here, since _all_ the socks listed
234− * in tb->owners and tb2->owners list belong
235− * to the same net
153+ * in tb->owners list belong to the same net - the
154+ * one this bucket belongs to.
236155 */
237156
238− if (!use_bhash2_on_bind(sk)) {
239− sk_for_each_bound(sk2, &tb->owners)
240− if (bind_conflict_exist(sk, sk2, uid, relax,
241− reuseport_cb_ok, reuseport_ok) &&
242− inet_rcv_saddr_equal(sk, sk2, true))
243− return true;
157+ sk_for_each_bound(sk2, &tb->owners) {
158+ int bound_dev_if2;
244159
245− return false;
160+ if (sk == sk2)
161+ continue;
162+ bound_dev_if2 = READ_ONCE(sk2->sk_bound_dev_if);
163+ if ((!sk->sk_bound_dev_if ||
164+ !bound_dev_if2 ||
165+ sk->sk_bound_dev_if == bound_dev_if2)) {
166+ if (reuse && sk2->sk_reuse &&
167+ sk2->sk_state != TCP_LISTEN) {
168+ if ((!relax ||
169+ (!reuseport_ok &&
170+ reuseport && sk2->sk_reuseport &&
171+ reuseport_cb_ok &&
172+ (sk2->sk_state == TCP_TIME_WAIT ||
173+ uid_eq(uid, sock_i_uid(sk2))))) &&
174+ inet_rcv_saddr_equal(sk, sk2, true))
175+ break;
176+ } else if (!reuseport_ok ||
177+ !reuseport || !sk2->sk_reuseport ||
178+ !reuseport_cb_ok ||
179+ (sk2->sk_state != TCP_TIME_WAIT &&
180+ !uid_eq(uid, sock_i_uid(sk2)))) {
181+ if (inet_rcv_saddr_equal(sk, sk2, true))
182+ break;
183+ }
184+ }
246185 }
247−
248− if (tb2 && check_bhash2_conflict(sk, tb2, uid, relax, reuseport_cb_ok,
249− reuseport_ok))
250− return true;
251−
252− net = sock_net(sk);
253−
254− /* check there's no conflict with an existing IPV6_ADDR_ANY (if ipv6) or
255− * INADDR_ANY (if ipv4) socket.
256− */
257− hash = get_bhash2_nulladdr_hash(sk, net, port);
258− head2 = &hinfo->bhash2[hash & (hinfo->bhash_size - 1)];
259−
260− l3mdev = inet_sk_bound_l3mdev(sk);
261− inet_bind_bucket_for_each(tb2, &head2->chain)
262− if (check_bind2_bucket_match_nulladdr(tb2, net, port, l3mdev, sk))
263− break;
264−
265− if (tb2 && check_bhash2_conflict(sk, tb2, uid, relax, reuseport_cb_ok,
266− reuseport_ok))
267− return true;
268−
269− return false;
186+ return sk2 != NULL;
270187 }
271188
272189 /*
@@ -274,20 +191,16 @@ static int inet_csk_bind_conflict(const struct sock *sk, int port, · 1 unmodified line
274191 * inet_bind_hashbucket lock held.
275192 */
276193 static struct inet_bind_hashbucket *
277−inet_csk_find_open_port(struct sock *sk, struct inet_bind_bucket **tb_ret,
278− struct inet_bind2_bucket **tb2_ret,
279− struct inet_bind2_hashbucket **head2_ret, int *port_ret)
194+inet_csk_find_open_port(struct sock *sk, struct inet_bind_bucket **tb_ret, int *port_ret)
280195 {
281196 struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;
282− struct inet_bind2_hashbucket *head2;
197+ int port = 0;
283198 struct inet_bind_hashbucket *head;
284199 struct net *net = sock_net(sk);
200+ bool relax = false;
285201 int i, low, high, attempt_half;
286− struct inet_bind2_bucket *tb2;
287202 struct inet_bind_bucket *tb;
288203 u32 remaining, offset;
289− bool relax = false;
290− int port = 0;
291204 int l3mdev;
292205
293206 l3mdev = inet_sk_bound_l3mdev(sk);
@@ -326,12 +239,10 @@ other_parity_scan: · 32 unmodified lines
326239 head = &hinfo->bhash[inet_bhashfn(net, port,
327240 hinfo->bhash_size)];
328241 spin_lock_bh(&head->lock);
329− tb2 = inet_bind2_bucket_find(hinfo, net, port, l3mdev, sk,
330− &head2);
331242 inet_bind_bucket_for_each(tb, &head->chain)
332− if (check_bind_bucket_match(tb, net, port, l3mdev)) {
333− if (!inet_csk_bind_conflict(sk, port, tb, tb2,
334− relax, false))
243+ if (net_eq(ib_net(tb), net) && tb->l3mdev == l3mdev &&
244+ tb->port == port) {
245+ if (!inet_csk_bind_conflict(sk, tb, relax, false))
335246 goto success;
336247 goto next_port;
337248 }
@@ -361,8 +272,6 @@ next_port: · 23 unmodified lines
361272 success:
362273 *port_ret = port;
363274 *tb_ret = tb;
364− *tb2_ret = tb2;
365− *head2_ret = head2;
366275 return head;
367276 }
368277
@@ -458,81 +367,54 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum) · 89 unmodified lines
458367 {
459368 bool reuse = sk->sk_reuse && sk->sk_state != TCP_LISTEN;
460369 struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;
461− bool bhash_created = false, bhash2_created = false;
462− struct inet_bind2_bucket *tb2 = NULL;
463− struct inet_bind2_hashbucket *head2;
464− struct inet_bind_bucket *tb = NULL;
370+ int ret = 1, port = snum;
465371 struct inet_bind_hashbucket *head;
466372 struct net *net = sock_net(sk);
467− int ret = 1, port = snum;
468− bool found_port = false;
373+ struct inet_bind_bucket *tb = NULL;
469374 int l3mdev;
470375
471376 l3mdev = inet_sk_bound_l3mdev(sk);
472377
473378 if (!port) {
474− head = inet_csk_find_open_port(sk, &tb, &tb2, &head2, &port);
379+ head = inet_csk_find_open_port(sk, &tb, &port);
475380 if (!head)
476381 return ret;
477− if (tb && tb2)
478− goto success;
479− found_port = true;
480− } else {
481− head = &hinfo->bhash[inet_bhashfn(net, port,
482− hinfo->bhash_size)];
483− spin_lock_bh(&head->lock);
484− inet_bind_bucket_for_each(tb, &head->chain)
485− if (check_bind_bucket_match(tb, net, port, l3mdev))
486− break;
487−
488− tb2 = inet_bind2_bucket_find(hinfo, net, port, l3mdev, sk,
489− &head2);
490− }
491−
492− if (!tb) {
493− tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep, net,
494− head, port, l3mdev);
495382 if (!tb)
496− goto fail_unlock;
497− bhash_created = true;
498− }
499−
500− if (!tb2) {
501− tb2 = inet_bind2_bucket_create(hinfo->bind2_bucket_cachep,
502− net, head2, port, l3mdev, sk);
503− if (!tb2)
504− goto fail_unlock;
505− bhash2_created = true;
383+ goto tb_not_found;
384+ goto success;
506385 }
507−
508− /* If we had to find an open port, we already checked for conflicts */
509− if (!found_port && !hlist_empty(&tb->owners)) {
386+ head = &hinfo->bhash[inet_bhashfn(net, port,
387+ hinfo->bhash_size)];
388+ spin_lock_bh(&head->lock);
389+ inet_bind_bucket_for_each(tb, &head->chain)
390+ if (net_eq(ib_net(tb), net) && tb->l3mdev == l3mdev &&
391+ tb->port == port)
392+ goto tb_found;
393+tb_not_found:
394+ tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep,
395+ net, head, port, l3mdev);
396+ if (!tb)
397+ goto fail_unlock;
398+tb_found:
399+ if (!hlist_empty(&tb->owners)) {
510400 if (sk->sk_reuse == SK_FORCE_REUSE)
511401 goto success;
512402
513403 if ((tb->fastreuse > 0 && reuse) ||
514404 sk_reuseport_match(tb, sk))
515405 goto success;
516− if (inet_csk_bind_conflict(sk, port, tb, tb2, true, true))
406+ if (inet_csk_bind_conflict(sk, tb, true, true))
517407 goto fail_unlock;
518408 }
519409 success:
520410 inet_csk_update_fastreuse(tb, sk);
521411
522412 if (!inet_csk(sk)->icsk_bind_hash)
523− inet_bind_hash(sk, tb, tb2, port);
413+ inet_bind_hash(sk, tb, port);
524414 WARN_ON(inet_csk(sk)->icsk_bind_hash != tb);
525− WARN_ON(inet_csk(sk)->icsk_bind2_hash != tb2);
526415 ret = 0;
527416
528417 fail_unlock:
529− if (ret) {
530− if (bhash_created)
531− inet_bind_bucket_destroy(hinfo->bind_bucket_cachep, tb);
532− if (bhash2_created)
533− inet_bind2_bucket_destroy(hinfo->bind2_bucket_cachep,
534− tb2);
535− }
536418 spin_unlock_bh(&head->lock);
537419 return ret;
538420 }
@@ -1079,7 +961,6 @@ struct sock *inet_csk_clone_lock(const struct sock *sk, · 540 unmodified lines
1079961
1080962 inet_sk_set_state(newsk, TCP_SYN_RECV);
1081963 newicsk->icsk_bind_hash = NULL;
1082− newicsk->icsk_bind2_hash = NULL;
1083964
1084965 inet_sk(newsk)->inet_dport = inet_rsk(req)->ir_rmt_port;
1085966 inet_sk(newsk)->inet_num = inet_rsk(req)->ir_num;
net/ipv4/inet_hashtables.c+11−182
@@ -81,41 +81,6 @@ struct inet_bind_bucket *inet_bind_bucket_create(struct kmem_cache *cachep, · 80 unmodified lines
8181 return tb;
8282 }
8383
84−struct inet_bind2_bucket *inet_bind2_bucket_create(struct kmem_cache *cachep,
85− struct net *net,
86− struct inet_bind2_hashbucket *head,
87− const unsigned short port,
88− int l3mdev,
89− const struct sock *sk)
90−{
91− struct inet_bind2_bucket *tb = kmem_cache_alloc(cachep, GFP_ATOMIC);
92−
93− if (tb) {
94− write_pnet(&tb->ib_net, net);
95− tb->l3mdev = l3mdev;
96− tb->port = port;
97−#if IS_ENABLED(CONFIG_IPV6)
98− if (sk->sk_family == AF_INET6)
99− tb->v6_rcv_saddr = sk->sk_v6_rcv_saddr;
100− else
101−#endif
102− tb->rcv_saddr = sk->sk_rcv_saddr;
103− INIT_HLIST_HEAD(&tb->owners);
104− hlist_add_head(&tb->node, &head->chain);
105− }
106− return tb;
107−}
108−
109−static bool bind2_bucket_addr_match(struct inet_bind2_bucket *tb2, struct sock *sk)
110−{
111−#if IS_ENABLED(CONFIG_IPV6)
112− if (sk->sk_family == AF_INET6)
113− return ipv6_addr_equal(&tb2->v6_rcv_saddr,
114− &sk->sk_v6_rcv_saddr);
115−#endif
116− return tb2->rcv_saddr == sk->sk_rcv_saddr;
117−}
118−
11984 /*
12085 * Caller must hold hashbucket lock for this tb with local BH disabled
12186 */
@@ -127,25 +92,12 @@ void inet_bind_bucket_destroy(struct kmem_cache *cachep, struct inet_bind_bucket · 5 unmodified lines
12792 }
12893 }
12994
130−/* Caller must hold the lock for the corresponding hashbucket in the bhash table
131− * with local BH disabled
132− */
133−void inet_bind2_bucket_destroy(struct kmem_cache *cachep, struct inet_bind2_bucket *tb)
134−{
135− if (hlist_empty(&tb->owners)) {
136− __hlist_del(&tb->node);
137− kmem_cache_free(cachep, tb);
138− }
139−}
140−
14195 void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb,
142− struct inet_bind2_bucket *tb2, const unsigned short snum)
96+ const unsigned short snum)
14397 {
14498 inet_sk(sk)->inet_num = snum;
14599 sk_add_bind_node(sk, &tb->owners);
146100 inet_csk(sk)->icsk_bind_hash = tb;
147− sk_add_bind2_node(sk, &tb2->owners);
148− inet_csk(sk)->icsk_bind2_hash = tb2;
149101 }
150102
151103 /*
@@ -157,7 +109,6 @@ static void __inet_put_port(struct sock *sk) · 5 unmodified lines
157109 const int bhash = inet_bhashfn(sock_net(sk), inet_sk(sk)->inet_num,
158110 hashinfo->bhash_size);
159111 struct inet_bind_hashbucket *head = &hashinfo->bhash[bhash];
160− struct inet_bind2_bucket *tb2;
161112 struct inet_bind_bucket *tb;
162113
163114 spin_lock(&head->lock);
@@ -166,13 +117,6 @@ static void __inet_put_port(struct sock *sk) · 2 unmodified lines
166117 inet_csk(sk)->icsk_bind_hash = NULL;
167118 inet_sk(sk)->inet_num = 0;
168119 inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
169−
170− if (inet_csk(sk)->icsk_bind2_hash) {
171− tb2 = inet_csk(sk)->icsk_bind2_hash;
172− __sk_del_bind2_node(sk);
173− inet_csk(sk)->icsk_bind2_hash = NULL;
174− inet_bind2_bucket_destroy(hashinfo->bind2_bucket_cachep, tb2);
175− }
176120 spin_unlock(&head->lock);
177121 }
178122
@@ -189,19 +133,14 @@ int __inet_inherit_port(const struct sock *sk, struct sock *child) · 10 unmodified lines
189133 struct inet_hashinfo *table = sk->sk_prot->h.hashinfo;
190134 unsigned short port = inet_sk(child)->inet_num;
191135 const int bhash = inet_bhashfn(sock_net(sk), port,
192− table->bhash_size);
136+ table->bhash_size);
193137 struct inet_bind_hashbucket *head = &table->bhash[bhash];
194− struct inet_bind2_hashbucket *head_bhash2;
195− bool created_inet_bind_bucket = false;
196− struct net *net = sock_net(sk);
197− struct inet_bind2_bucket *tb2;
198138 struct inet_bind_bucket *tb;
199139 int l3mdev;
200140
201141 spin_lock(&head->lock);
202142 tb = inet_csk(sk)->icsk_bind_hash;
203− tb2 = inet_csk(sk)->icsk_bind2_hash;
204− if (unlikely(!tb || !tb2)) {
143+ if (unlikely(!tb)) {
205144 spin_unlock(&head->lock);
206145 return -ENOENT;
207146 }
@@ -214,45 +153,25 @@ int __inet_inherit_port(const struct sock *sk, struct sock *child) · 6 unmodified lines
214153 * as that of the child socket. We have to look up or
215154 * create a new bind bucket for the child here. */
216155 inet_bind_bucket_for_each(tb, &head->chain) {
217− if (check_bind_bucket_match(tb, net, port, l3mdev))
156+ if (net_eq(ib_net(tb), sock_net(sk)) &&
157+ tb->l3mdev == l3mdev && tb->port == port)
218158 break;
219159 }
220160 if (!tb) {
221161 tb = inet_bind_bucket_create(table->bind_bucket_cachep,
222− net, head, port, l3mdev);
162+ sock_net(sk), head, port,
163+ l3mdev);
223164 if (!tb) {
224165 spin_unlock(&head->lock);
225166 return -ENOMEM;
226167 }
227− created_inet_bind_bucket = true;
228168 }
229169 inet_csk_update_fastreuse(tb, child);
230−
231− goto bhash2_find;
232− } else if (!bind2_bucket_addr_match(tb2, child)) {
233− l3mdev = inet_sk_bound_l3mdev(sk);
234−
235−bhash2_find:
236− tb2 = inet_bind2_bucket_find(table, net, port, l3mdev, child,
237− &head_bhash2);
238− if (!tb2) {
239− tb2 = inet_bind2_bucket_create(table->bind2_bucket_cachep,
240− net, head_bhash2, port,
241− l3mdev, child);
242− if (!tb2)
243− goto error;
244− }
245170 }
246− inet_bind_hash(child, tb, tb2, port);
171+ inet_bind_hash(child, tb, port);
247172 spin_unlock(&head->lock);
248173
249174 return 0;
250−
251−error:
252− if (created_inet_bind_bucket)
253− inet_bind_bucket_destroy(table->bind_bucket_cachep, tb);
254− spin_unlock(&head->lock);
255− return -ENOMEM;
256175 }
257176 EXPORT_SYMBOL_GPL(__inet_inherit_port);
258177
@@ -756,76 +675,6 @@ void inet_unhash(struct sock *sk) · 497 unmodified lines
756675 }
757676 EXPORT_SYMBOL_GPL(inet_unhash);
758677
759−static bool check_bind2_bucket_match(struct inet_bind2_bucket *tb,
760− struct net *net, unsigned short port,
761− int l3mdev, struct sock *sk)
762−{
763−#if IS_ENABLED(CONFIG_IPV6)
764− if (sk->sk_family == AF_INET6)
765− return net_eq(ib2_net(tb), net) && tb->port == port &&
766− tb->l3mdev == l3mdev &&
767− ipv6_addr_equal(&tb->v6_rcv_saddr, &sk->sk_v6_rcv_saddr);
768− else
769−#endif
770− return net_eq(ib2_net(tb), net) && tb->port == port &&
771− tb->l3mdev == l3mdev && tb->rcv_saddr == sk->sk_rcv_saddr;
772−}
773−
774−bool check_bind2_bucket_match_nulladdr(struct inet_bind2_bucket *tb,
775− struct net *net, const unsigned short port,
776− int l3mdev, const struct sock *sk)
777−{
778−#if IS_ENABLED(CONFIG_IPV6)
779− struct in6_addr nulladdr = {};
780−
781− if (sk->sk_family == AF_INET6)
782− return net_eq(ib2_net(tb), net) && tb->port == port &&
783− tb->l3mdev == l3mdev &&
784− ipv6_addr_equal(&tb->v6_rcv_saddr, &nulladdr);
785− else
786−#endif
787− return net_eq(ib2_net(tb), net) && tb->port == port &&
788− tb->l3mdev == l3mdev && tb->rcv_saddr == 0;
789−}
790−
791−static struct inet_bind2_hashbucket *
792−inet_bhashfn_portaddr(struct inet_hashinfo *hinfo, const struct sock *sk,
793− const struct net *net, unsigned short port)
794−{
795− u32 hash;
796−
797−#if IS_ENABLED(CONFIG_IPV6)
798− if (sk->sk_family == AF_INET6)
799− hash = ipv6_portaddr_hash(net, &sk->sk_v6_rcv_saddr, port);
800− else
801−#endif
802− hash = ipv4_portaddr_hash(net, sk->sk_rcv_saddr, port);
803− return &hinfo->bhash2[hash & (hinfo->bhash_size - 1)];
804−}
805−
806−/* This should only be called when the spinlock for the socket's corresponding
807− * bind_hashbucket is held
808− */
809−struct inet_bind2_bucket *
810−inet_bind2_bucket_find(struct inet_hashinfo *hinfo, struct net *net,
811− const unsigned short port, int l3mdev, struct sock *sk,
812− struct inet_bind2_hashbucket **head)
813−{
814− struct inet_bind2_bucket *bhash2 = NULL;
815− struct inet_bind2_hashbucket *h;
816−
817− h = inet_bhashfn_portaddr(hinfo, sk, net, port);
818− inet_bind_bucket_for_each(bhash2, &h->chain) {
819− if (check_bind2_bucket_match(bhash2, net, port, l3mdev, sk))
820− break;
821− }
822−
823− if (head)
824− *head = h;
825−
826− return bhash2;
827−}
828−
829678 /* RFC 6056 3.3.4. Algorithm 4: Double-Hash Port Selection Algorithm
830679 * Note that we use 32bit integers (vs RFC 'short integers')
831680 * because 2^16 is not a multiple of num_ephemeral and this
@@ -846,13 +695,10 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row, · 14 unmodified lines
846695 {
847696 struct inet_hashinfo *hinfo = death_row->hashinfo;
848697 struct inet_timewait_sock *tw = NULL;
849− struct inet_bind2_hashbucket *head2;
850698 struct inet_bind_hashbucket *head;
851699 int port = inet_sk(sk)->inet_num;
852700 struct net *net = sock_net(sk);
853− struct inet_bind2_bucket *tb2;
854701 struct inet_bind_bucket *tb;
855− bool tb_created = false;
856702 u32 remaining, offset;
857703 int ret, i, low, high;
858704 int l3mdev;
@@ -909,7 +755,8 @@ other_parity_scan: · 50 unmodified lines
909755 * the established check is already unique enough.
910756 */
911757 inet_bind_bucket_for_each(tb, &head->chain) {
912− if (check_bind_bucket_match(tb, net, port, l3mdev)) {
758+ if (net_eq(ib_net(tb), net) && tb->l3mdev == l3mdev &&
759+ tb->port == port) {
913760 if (tb->fastreuse >= 0 ||
914761 tb->fastreuseport >= 0)
915762 goto next_port;
@@ -927,7 +774,6 @@ other_parity_scan: · 11 unmodified lines
927774 spin_unlock_bh(&head->lock);
928775 return -ENOMEM;
929776 }
930− tb_created = true;
931777 tb->fastreuse = -1;
932778 tb->fastreuseport = -1;
933779 goto ok;
@@ -943,17 +789,6 @@ next_port: · 9 unmodified lines
943789 return -EADDRNOTAVAIL;
944790
945791 ok:
946− /* Find the corresponding tb2 bucket since we need to
947− * add the socket to the bhash2 table as well
948− */
949− tb2 = inet_bind2_bucket_find(hinfo, net, port, l3mdev, sk, &head2);
950− if (!tb2) {
951− tb2 = inet_bind2_bucket_create(hinfo->bind2_bucket_cachep, net,
952− head2, port, l3mdev, sk);
953− if (!tb2)
954− goto error;
955− }
956−
957792 /* Here we want to add a little bit of randomness to the next source
958793 * port that will be chosen. We use a max() with a random here so that
959794 * on low contention the randomness is maximal and on high contention
@@ -963,7 +798,7 @@ ok: · 3 unmodified lines
963798 WRITE_ONCE(table_perturb[index], READ_ONCE(table_perturb[index]) + i + 2);
964799
965800 /* Head lock still held and bh's disabled */
966− inet_bind_hash(sk, tb, tb2, port);
801+ inet_bind_hash(sk, tb, port);
967802 if (sk_unhashed(sk)) {
968803 inet_sk(sk)->inet_sport = htons(port);
969804 inet_ehash_nolisten(sk, (struct sock *)tw, NULL);
@@ -975,12 +810,6 @@ ok: · 5 unmodified lines
975810 inet_twsk_deschedule_put(tw);
976811 local_bh_enable();
977812 return 0;
978−
979−error:
980− if (tb_created)
981− inet_bind_bucket_destroy(hinfo->bind_bucket_cachep, tb);
982− spin_unlock_bh(&head->lock);
983− return -ENOMEM;
984813 }
985814
986815 /*
net/ipv4/tcp.c+2−12
@@ -4604,12 +4604,6 @@ void __init tcp_init(void) · 4603 unmodified lines
46044604 SLAB_HWCACHE_ALIGN | SLAB_PANIC |
46054605 SLAB_ACCOUNT,
46064606 NULL);
4607− tcp_hashinfo.bind2_bucket_cachep =
4608− kmem_cache_create("tcp_bind2_bucket",
4609− sizeof(struct inet_bind2_bucket), 0,
4610− SLAB_HWCACHE_ALIGN | SLAB_PANIC |
4611− SLAB_ACCOUNT,
4612− NULL);
46134607
46144608 /* Size and allocate the main established and bind bucket
46154609 * hash tables.
@@ -4632,9 +4626,8 @@ void __init tcp_init(void) · 16 unmodified lines
46324626 if (inet_ehash_locks_alloc(&tcp_hashinfo))
46334627 panic("TCP: failed to alloc ehash_locks");
46344628 tcp_hashinfo.bhash =
4635− alloc_large_system_hash("TCP bind bhash tables",
4636− sizeof(struct inet_bind_hashbucket) +
4637− sizeof(struct inet_bind2_hashbucket),
4629+ alloc_large_system_hash("TCP bind",
4630+ sizeof(struct inet_bind_hashbucket),
46384631 tcp_hashinfo.ehash_mask + 1,
46394632 17, /* one slot per 128 KB of memory */
46404633 0,
@@ -4643,12 +4636,9 @@ void __init tcp_init(void) · 2 unmodified lines
46434636 0,
46444637 64 * 1024);
46454638 tcp_hashinfo.bhash_size = 1U << tcp_hashinfo.bhash_size;
4646− tcp_hashinfo.bhash2 =
4647− (struct inet_bind2_hashbucket *)(tcp_hashinfo.bhash + tcp_hashinfo.bhash_size);
46484639 for (i = 0; i < tcp_hashinfo.bhash_size; i++) {
46494640 spin_lock_init(&tcp_hashinfo.bhash[i].lock);
46504641 INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain);
4651− INIT_HLIST_HEAD(&tcp_hashinfo.bhash2[i].chain);
46524642 }
46534643
46544644
tools/testing/selftests/net/.gitignore+0−1
@@ -37,4 +37,3 @@ gro · 36 unmodified lines
3737 ioam6_parser
3838 toeplitz
3939 cmsg_sender
40−bind_bhash_test
tools/testing/selftests/net/Makefile+0−2
@@ -59,7 +59,6 @@ TEST_GEN_FILES += toeplitz · 58 unmodified lines
5959 TEST_GEN_FILES += cmsg_sender
6060 TEST_GEN_FILES += stress_reuseport_listen
6161 TEST_PROGS += test_vxlan_vnifiltering.sh
62−TEST_GEN_FILES += bind_bhash_test
6362
6463 TEST_FILES := settings
6564
@@ -70,5 +69,4 @@ include bpf/Makefile · 4 unmodified lines
7069
7170 $(OUTPUT)/reuseport_bpf_numa: LDLIBS += -lnuma
7271 $(OUTPUT)/tcp_mmap: LDLIBS += -lpthread
73−$(OUTPUT)/bind_bhash_test: LDLIBS += -lpthread
7472 $(OUTPUT)/tcp_inq: LDLIBS += -lpthread
tools/testing/selftests/net/bind_bhash_test.c+0−119
@@ -1,119 +0,0 @@
1−// SPDX-License-Identifier: GPL-2.0
2−/*
3− * This times how long it takes to bind to a port when the port already
4− * has multiple sockets in its bhash table.
5− *
6− * In the setup(), we populate the port's bhash table with
7− * MAX_THREADS * MAX_CONNECTIONS number of entries.
8− */
9−
10−#include <unistd.h>
11−#include <stdio.h>
12−#include <netdb.h>
13−#include <pthread.h>
14−
15−#define MAX_THREADS 600
16−#define MAX_CONNECTIONS 40
17−
18−static const char *bind_addr = "::1";
19−static const char *port;
20−
21−static int fd_array[MAX_THREADS][MAX_CONNECTIONS];
22−
23−static int bind_socket(int opt, const char *addr)
24−{
25− struct addrinfo *res, hint = {};
26− int sock_fd, reuse = 1, err;
27−
28− sock_fd = socket(AF_INET6, SOCK_STREAM, 0);
29− if (sock_fd < 0) {
30− perror("socket fd err");
31− return -1;
32− }
33−
34− hint.ai_family = AF_INET6;
35− hint.ai_socktype = SOCK_STREAM;
36−
37− err = getaddrinfo(addr, port, &hint, &res);
38− if (err) {
39− perror("getaddrinfo failed");
40− return -1;
41− }
42−
43− if (opt) {
44− err = setsockopt(sock_fd, SOL_SOCKET, opt, &reuse, sizeof(reuse));
45− if (err) {
46− perror("setsockopt failed");
47− return -1;
48− }
49− }
50−
51− err = bind(sock_fd, res->ai_addr, res->ai_addrlen);
52− if (err) {
53− perror("failed to bind to port");
54− return -1;
55− }
56−
57− return sock_fd;
58−}
59−
60−static void *setup(void *arg)
61−{
62− int sock_fd, i;
63− int *array = (int *)arg;
64−
65− for (i = 0; i < MAX_CONNECTIONS; i++) {
66− sock_fd = bind_socket(SO_REUSEADDR | SO_REUSEPORT, bind_addr);
67− if (sock_fd < 0)
68− return NULL;
69− array[i] = sock_fd;
70− }
71−
72− return NULL;
73−}
74−
75−int main(int argc, const char *argv[])
76−{
77− int listener_fd, sock_fd, i, j;
78− pthread_t tid[MAX_THREADS];
79− clock_t begin, end;
80−
81− if (argc != 2) {
82− printf("Usage: listener <port>\n");
83− return -1;
84− }
85−
86− port = argv[1];
87−
88− listener_fd = bind_socket(SO_REUSEADDR | SO_REUSEPORT, bind_addr);
89− if (listen(listener_fd, 100) < 0) {
90− perror("listen failed");
91− return -1;
92− }
93−
94− /* Set up threads to populate the bhash table entry for the port */
95− for (i = 0; i < MAX_THREADS; i++)
96− pthread_create(&tid[i], NULL, setup, fd_array[i]);
97−
98− for (i = 0; i < MAX_THREADS; i++)
99− pthread_join(tid[i], NULL);
100−
101− begin = clock();
102−
103− /* Bind to the same port on a different address */
104− sock_fd = bind_socket(0, "2001:0db8:0:f101::1");
105−
106− end = clock();
107−
108− printf("time spent = %f\n", (double)(end - begin) / CLOCKS_PER_SEC);
109−
110− /* clean up */
111− close(sock_fd);
112− close(listener_fd);
113− for (i = 0; i < MAX_THREADS; i++) {
114− for (j = 0; i < MAX_THREADS; i++)
115− close(fd_array[i][j]);
116− }
117−
118− return 0;
119−}