INET6_MATCH() runs without holding a lock on the socket. We probably need to annotate most reads. This patch makes INET6_MATCH() an inline function to ease our changes. v2: inline function only defined if IS_ENABLED(CONFIG_IPV6) Change the name to inet6_match(), this is no longer a macro.
Unified · Split
| @@ -103,15 +103,25 @@ struct sock *inet6_lookup(struct net *net, struct inet_hashinfo *hashinfo, · 102 unmodified lines | |||
| 103 | const int dif); | 103 | const int dif); |
| 104 | 104 | ||
| 105 | int inet6_hash(struct sock *sk); | 105 | int inet6_hash(struct sock *sk); |
| 106 | #endif /* IS_ENABLED(CONFIG_IPV6) */ | ||
| 107 | 106 | ||
| 108 | #define INET6_MATCH(__sk, __net, __saddr, __daddr, __ports, __dif, __sdif) \ | 107 | static inline bool inet6_match(struct net *net, const struct sock *sk, |
| 109 | (((__sk)->sk_portpair == (__ports)) && \ | 108 | const struct in6_addr *saddr, |
| 110 | ((__sk)->sk_family == AF_INET6) && \ | 109 | const struct in6_addr *daddr, |
| 111 | ipv6_addr_equal(&(__sk)->sk_v6_daddr, (__saddr)) && \ | 110 | const __portpair ports, |
| 112 | ipv6_addr_equal(&(__sk)->sk_v6_rcv_saddr, (__daddr)) && \ | 111 | const int dif, const int sdif) |
| 113 | (((__sk)->sk_bound_dev_if == (__dif)) || \ | 112 | { |
| 114 | ((__sk)->sk_bound_dev_if == (__sdif))) && \ | 113 | int bound_dev_if; |
| 115 | net_eq(sock_net(__sk), (__net))) | 114 | |
| 115 | if (!net_eq(sock_net(sk), net) || | ||
| 116 | sk->sk_family != AF_INET6 || | ||
| 117 | sk->sk_portpair != ports || | ||
| 118 | !ipv6_addr_equal(&sk->sk_v6_daddr, saddr) || | ||
| 119 | !ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr)) | ||
| 120 | return false; | ||
| 121 | |||
| 122 | bound_dev_if = READ_ONCE(sk->sk_bound_dev_if); | ||
| 123 | return bound_dev_if == dif || bound_dev_if == sdif; | ||
| 124 | } | ||
| 125 | #endif /* IS_ENABLED(CONFIG_IPV6) */ | ||
| 116 | 126 | ||
| 117 | #endif /* _INET6_HASHTABLES_H */ | 127 | #endif /* _INET6_HASHTABLES_H */ |
| @@ -499,7 +499,7 @@ static bool inet_ehash_lookup_by_sk(struct sock *sk, · 498 unmodified lines | |||
| 499 | } | 499 | } |
| 500 | #if IS_ENABLED(CONFIG_IPV6) | 500 | #if IS_ENABLED(CONFIG_IPV6) |
| 501 | else if (sk->sk_family == AF_INET6) { | 501 | else if (sk->sk_family == AF_INET6) { |
| 502 | if (unlikely(INET6_MATCH(esk, net, | 502 | if (unlikely(inet6_match(net, esk, |
| 503 | &sk->sk_v6_daddr, | 503 | &sk->sk_v6_daddr, |
| 504 | &sk->sk_v6_rcv_saddr, | 504 | &sk->sk_v6_rcv_saddr, |
| 505 | ports, dif, sdif))) { | 505 | ports, dif, sdif))) { |
| @@ -71,12 +71,12 @@ begin: · 70 unmodified lines | |||
| 71 | sk_nulls_for_each_rcu(sk, node, &head->chain) { | 71 | sk_nulls_for_each_rcu(sk, node, &head->chain) { |
| 72 | if (sk->sk_hash != hash) | 72 | if (sk->sk_hash != hash) |
| 73 | continue; | 73 | continue; |
| 74 | if (!INET6_MATCH(sk, net, saddr, daddr, ports, dif, sdif)) | 74 | if (!inet6_match(net, sk, saddr, daddr, ports, dif, sdif)) |
| 75 | continue; | 75 | continue; |
| 76 | if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt))) | 76 | if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt))) |
| 77 | goto out; | 77 | goto out; |
| 78 | 78 | ||
| 79 | if (unlikely(!INET6_MATCH(sk, net, saddr, daddr, ports, dif, sdif))) { | 79 | if (unlikely(!inet6_match(net, sk, saddr, daddr, ports, dif, sdif))) { |
| 80 | sock_gen_put(sk); | 80 | sock_gen_put(sk); |
| 81 | goto begin; | 81 | goto begin; |
| 82 | } | 82 | } |
| @@ -268,7 +268,7 @@ static int __inet6_check_established(struct inet_timewait_death_row *death_row, · 185 unmodified lines | |||
| 268 | if (sk2->sk_hash != hash) | 268 | if (sk2->sk_hash != hash) |
| 269 | continue; | 269 | continue; |
| 270 | 270 | ||
| 271 | if (likely(INET6_MATCH(sk2, net, saddr, daddr, ports, | 271 | if (likely(inet6_match(net, sk2, saddr, daddr, ports, |
| 272 | dif, sdif))) { | 272 | dif, sdif))) { |
| 273 | if (sk2->sk_state == TCP_TIME_WAIT) { | 273 | if (sk2->sk_state == TCP_TIME_WAIT) { |
| 274 | tw = inet_twsk(sk2); | 274 | tw = inet_twsk(sk2); |
| @@ -1044,7 +1044,7 @@ static struct sock *__udp6_lib_demux_lookup(struct net *net, · 1043 unmodified lines | |||
| 1044 | 1044 | ||
| 1045 | udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) { | 1045 | udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) { |
| 1046 | if (sk->sk_state == TCP_ESTABLISHED && | 1046 | if (sk->sk_state == TCP_ESTABLISHED && |
| 1047 | INET6_MATCH(sk, net, rmt_addr, loc_addr, ports, dif, sdif)) | 1047 | inet6_match(net, sk, rmt_addr, loc_addr, ports, dif, sdif)) |
| 1048 | return sk; | 1048 | return sk; |
| 1049 | /* Only check first socket in chain */ | 1049 | /* Only check first socket in chain */ |
| 1050 | break; | 1050 | break; |