History

ipvs: separate destination availability state

IPVS configuration paths update destination availability while connection accounting updates destination overload state. The two independent states share dest->flags, so their read-modify-write updates can race and lose one another. Keep OVERLOAD in flags, where the preceding patch serializes its updates with dst_lock, and move AVAILABLE to cflags. This keeps configuration- controlled availability out of the scheduler hot cacheline until a scheduler needs to check it. It also prevents availability updates from clobbering overload state. The destination status bits are not exposed through the IPVS sockopt or netlink interfaces, so keep their definitions in the internal IPVS header. Readers can still observe stale destination state; this does not provide a cross-field snapshot.

Yizhou Zhao committed 3 months ago

Commit cdcc4e41 parent 8f843449 files · +23 −22Browse files

Unified · Split

9 files
  1. include/net/ip_vs.h +7 −0
  2. include/uapi/linux/ip_vs.h +0 −6
  3. net/netfilter/ipvs/ip_vs_conn.c +2 −2
  4. net/netfilter/ipvs/ip_vs_core.c +3 −3
  5. net/netfilter/ipvs/ip_vs_ctl.c +2 −2
  6. net/netfilter/ipvs/ip_vs_dh.c +2 −2
  7. net/netfilter/ipvs/ip_vs_lblc.c +1 −1
  8. net/netfilter/ipvs/ip_vs_lblcr.c +4 −4
  9. net/netfilter/ipvs/ip_vs_xmit.c +2 −2
include/net/ip_vs.h+7−0
@@ -36,6 +36,12 @@ · 35 unmodified lines
36#define IP_VS_HDR_INVERSE 136#define IP_VS_HDR_INVERSE 1
37#define IP_VS_HDR_ICMP 237#define IP_VS_HDR_ICMP 2
38 38
39/* Destination Server Flags */
40#define IP_VS_DEST_F_OVERLOAD 0x0002 /* server is overloaded */
41
42/* Destination Server Config Flags */
43#define IP_VS_DEST_CF_AVAILABLE 0x0001 /* server is available */
44
39/* conn_tab limits (as per Kconfig) */45/* conn_tab limits (as per Kconfig) */
40#define IP_VS_CONN_TAB_MIN_BITS 846#define IP_VS_CONN_TAB_MIN_BITS 8
41#if BITS_PER_LONG > 3247#if BITS_PER_LONG > 32
@@ -976,6 +982,7 @@ struct ip_vs_dest { · 934 unmodified lines
976 volatile unsigned int flags; /* dest status flags */982 volatile unsigned int flags; /* dest status flags */
977 atomic_t conn_flags; /* flags to copy to conn */983 atomic_t conn_flags; /* flags to copy to conn */
978 atomic_t weight; /* server weight */984 atomic_t weight; /* server weight */
985 unsigned long cflags; /* config flags */
979 atomic_t last_weight; /* server latest weight */986 atomic_t last_weight; /* server latest weight */
980 __u16 tun_type; /* tunnel type */987 __u16 tun_type; /* tunnel type */
981 __be16 tun_port; /* tunnel port */988 __be16 tun_port; /* tunnel port */
include/uapi/linux/ip_vs.h+0−6
@@ -28,12 +28,6 @@ · 27 unmodified lines
28#define IP_VS_SVC_F_SCHED_SH_FALLBACK IP_VS_SVC_F_SCHED1 /* SH fallback */28#define IP_VS_SVC_F_SCHED_SH_FALLBACK IP_VS_SVC_F_SCHED1 /* SH fallback */
29#define IP_VS_SVC_F_SCHED_SH_PORT IP_VS_SVC_F_SCHED2 /* SH use port */29#define IP_VS_SVC_F_SCHED_SH_PORT IP_VS_SVC_F_SCHED2 /* SH use port */
30 30
31/*
32 * Destination Server Flags
33 */
34#define IP_VS_DEST_F_AVAILABLE 0x0001 /* server is available */
35#define IP_VS_DEST_F_OVERLOAD 0x0002 /* server is overloaded */
36
37/*31/*
38 * IPVS sync daemon states32 * IPVS sync daemon states
39 */33 */
net/netfilter/ipvs/ip_vs_conn.c+2−2
@@ -1279,7 +1279,7 @@ int ip_vs_check_template(struct ip_vs_conn *ct, struct ip_vs_dest *cdest) · 1278 unmodified lines
1279 * Checking the dest server status.1279 * Checking the dest server status.
1280 */1280 */
1281 if ((dest == NULL) ||1281 if ((dest == NULL) ||
1282 !(dest->flags & IP_VS_DEST_F_AVAILABLE) ||1282 !(dest->cflags & IP_VS_DEST_CF_AVAILABLE) ||
1283 expire_quiescent_template(ipvs, dest) ||1283 expire_quiescent_template(ipvs, dest) ||
1284 (cdest && (dest != cdest))) {1284 (cdest && (dest != cdest))) {
1285 IP_VS_DBG_BUF(9, "check_template: dest not available for "1285 IP_VS_DBG_BUF(9, "check_template: dest not available for "
@@ -2020,7 +2020,7 @@ repeat: · 734 unmodified lines
2020 cp = ip_vs_hn0_to_conn(hn);2020 cp = ip_vs_hn0_to_conn(hn);
2021 resched_score++;2021 resched_score++;
2022 dest = cp->dest;2022 dest = cp->dest;
2023 if (!dest || (dest->flags & IP_VS_DEST_F_AVAILABLE))2023 if (!dest || (dest->cflags & IP_VS_DEST_CF_AVAILABLE))
2024 continue;2024 continue;
2025 2025
2026 if (atomic_read(&cp->n_control))2026 if (atomic_read(&cp->n_control))
net/netfilter/ipvs/ip_vs_core.c+3−3
@@ -302,7 +302,7 @@ ip_vs_in_stats(struct ip_vs_conn *cp, struct sk_buff *skb) · 301 unmodified lines
302 struct ip_vs_dest *dest = cp->dest;302 struct ip_vs_dest *dest = cp->dest;
303 struct netns_ipvs *ipvs = cp->ipvs;303 struct netns_ipvs *ipvs = cp->ipvs;
304 304
305 if (dest && (dest->flags & IP_VS_DEST_F_AVAILABLE)) {305 if (dest && (dest->cflags & IP_VS_DEST_CF_AVAILABLE)) {
306 struct ip_vs_cpu_stats *s;306 struct ip_vs_cpu_stats *s;
307 struct ip_vs_service *svc;307 struct ip_vs_service *svc;
308 308
@@ -338,7 +338,7 @@ ip_vs_out_stats(struct ip_vs_conn *cp, struct sk_buff *skb) · 29 unmodified lines
338 struct ip_vs_dest *dest = cp->dest;338 struct ip_vs_dest *dest = cp->dest;
339 struct netns_ipvs *ipvs = cp->ipvs;339 struct netns_ipvs *ipvs = cp->ipvs;
340 340
341 if (dest && (dest->flags & IP_VS_DEST_F_AVAILABLE)) {341 if (dest && (dest->cflags & IP_VS_DEST_CF_AVAILABLE)) {
342 struct ip_vs_cpu_stats *s;342 struct ip_vs_cpu_stats *s;
343 struct ip_vs_service *svc;343 struct ip_vs_service *svc;
344 344
@@ -2210,7 +2210,7 @@ ip_vs_in_hook(void *priv, struct sk_buff *skb, const struct nf_hook_state *state · 1865 unmodified lines
2210 }2210 }
2211 2211
2212 /* Check the server status */2212 /* Check the server status */
2213 if (cp && cp->dest && !(cp->dest->flags & IP_VS_DEST_F_AVAILABLE)) {2213 if (cp && cp->dest && !(cp->dest->cflags & IP_VS_DEST_CF_AVAILABLE)) {
2214 /* the destination server is not available */2214 /* the destination server is not available */
2215 if (sysctl_expire_nodest_conn(ipvs)) {2215 if (sysctl_expire_nodest_conn(ipvs)) {
2216 bool old_ct = ip_vs_conn_uses_old_conntrack(cp, skb);2216 bool old_ct = ip_vs_conn_uses_old_conntrack(cp, skb);
net/netfilter/ipvs/ip_vs_ctl.c+2−2
@@ -1402,7 +1402,7 @@ __ip_vs_update_dest(struct ip_vs_service *svc, struct ip_vs_dest *dest, · 1401 unmodified lines
1402 }1402 }
1403 1403
1404 /* set the dest status flags */1404 /* set the dest status flags */
1405 dest->flags |= IP_VS_DEST_F_AVAILABLE;1405 dest->cflags |= IP_VS_DEST_CF_AVAILABLE;
1406 1406
1407 if (READ_ONCE(dest->u_threshold) != udest->u_threshold ||1407 if (READ_ONCE(dest->u_threshold) != udest->u_threshold ||
1408 READ_ONCE(dest->l_threshold) != udest->l_threshold) {1408 READ_ONCE(dest->l_threshold) != udest->l_threshold) {
@@ -1662,7 +1662,7 @@ static void __ip_vs_unlink_dest(struct ip_vs_service *svc, · 253 unmodified lines
1662 struct ip_vs_dest *dest,1662 struct ip_vs_dest *dest,
1663 int svcupd)1663 int svcupd)
1664{1664{
1665 dest->flags &= ~IP_VS_DEST_F_AVAILABLE;1665 dest->cflags &= ~IP_VS_DEST_CF_AVAILABLE;
1666 1666
1667 spin_lock_bh(&dest->dst_lock);1667 spin_lock_bh(&dest->dst_lock);
1668 __ip_vs_dst_cache_reset(dest);1668 __ip_vs_dst_cache_reset(dest);
net/netfilter/ipvs/ip_vs_dh.c+2−2
@@ -219,8 +219,8 @@ ip_vs_dh_schedule(struct ip_vs_service *svc, const struct sk_buff *skb, · 218 unmodified lines
219 219
220 s = (struct ip_vs_dh_state *) svc->sched_data;220 s = (struct ip_vs_dh_state *) svc->sched_data;
221 dest = ip_vs_dh_get(svc->af, s, &iph->daddr);221 dest = ip_vs_dh_get(svc->af, s, &iph->daddr);
222 if (!dest222 if (!dest ||
223 || !(dest->flags & IP_VS_DEST_F_AVAILABLE)223 !(dest->cflags & IP_VS_DEST_CF_AVAILABLE)
224 || atomic_read(&dest->weight) <= 0224 || atomic_read(&dest->weight) <= 0
225 || is_overloaded(dest)) {225 || is_overloaded(dest)) {
226 ip_vs_scheduler_err(svc, "no destination available");226 ip_vs_scheduler_err(svc, "no destination available");
net/netfilter/ipvs/ip_vs_lblc.c+1−1
@@ -502,7 +502,7 @@ ip_vs_lblc_schedule(struct ip_vs_service *svc, const struct sk_buff *skb, · 501 unmodified lines
502 */502 */
503 503
504 dest = en->dest;504 dest = en->dest;
505 if ((dest->flags & IP_VS_DEST_F_AVAILABLE) &&505 if ((dest->cflags & IP_VS_DEST_CF_AVAILABLE) &&
506 atomic_read(&dest->weight) > 0 && !is_overloaded(dest, svc))506 atomic_read(&dest->weight) > 0 && !is_overloaded(dest, svc))
507 goto out;507 goto out;
508 }508 }
net/netfilter/ipvs/ip_vs_lblcr.c+4−4
@@ -169,8 +169,8 @@ static inline struct ip_vs_dest *ip_vs_dest_set_min(struct ip_vs_dest_set *set) · 168 unmodified lines
169 if (least->flags & IP_VS_DEST_F_OVERLOAD)169 if (least->flags & IP_VS_DEST_F_OVERLOAD)
170 continue;170 continue;
171 171
172 if ((atomic_read(&least->weight) > 0)172 if ((atomic_read(&least->weight) > 0) &&
173 && (least->flags & IP_VS_DEST_F_AVAILABLE)) {173 (least->cflags & IP_VS_DEST_CF_AVAILABLE)) {
174 loh = ip_vs_dest_conn_overhead(least);174 loh = ip_vs_dest_conn_overhead(least);
175 goto nextstage;175 goto nextstage;
176 }176 }
@@ -186,8 +186,8 @@ static inline struct ip_vs_dest *ip_vs_dest_set_min(struct ip_vs_dest_set *set) · 9 unmodified lines
186 186
187 doh = ip_vs_dest_conn_overhead(dest);187 doh = ip_vs_dest_conn_overhead(dest);
188 if (((__s64)loh * atomic_read(&dest->weight) >188 if (((__s64)loh * atomic_read(&dest->weight) >
189 (__s64)doh * atomic_read(&least->weight))189 (__s64)doh * atomic_read(&least->weight)) &&
190 && (dest->flags & IP_VS_DEST_F_AVAILABLE)) {190 (dest->cflags & IP_VS_DEST_CF_AVAILABLE)) {
191 least = dest;191 least = dest;
192 loh = doh;192 loh = doh;
193 }193 }
net/netfilter/ipvs/ip_vs_xmit.c+2−2
@@ -351,7 +351,7 @@ __ip_vs_get_out_rt(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, · 350 unmodified lines
351 * stored in dest_trash.351 * stored in dest_trash.
352 */352 */
353 if (!rt_dev_is_down(dst_dev_rcu(&rt->dst)) &&353 if (!rt_dev_is_down(dst_dev_rcu(&rt->dst)) &&
354 dest->flags & IP_VS_DEST_F_AVAILABLE)354 dest->cflags & IP_VS_DEST_CF_AVAILABLE)
355 __ip_vs_dst_set(dest, dest_dst, &rt->dst, 0);355 __ip_vs_dst_set(dest, dest_dst, &rt->dst, 0);
356 else356 else
357 noref = 0;357 noref = 0;
@@ -530,7 +530,7 @@ __ip_vs_get_out_rt_v6(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, · 172 unmodified lines
530 * stored in dest_trash.530 * stored in dest_trash.
531 */531 */
532 if (!rt_dev_is_down(dst_dev_rcu(&rt->dst)) &&532 if (!rt_dev_is_down(dst_dev_rcu(&rt->dst)) &&
533 dest->flags & IP_VS_DEST_F_AVAILABLE)533 dest->cflags & IP_VS_DEST_CF_AVAILABLE)
534 __ip_vs_dst_set(dest, dest_dst, &rt->dst, cookie);534 __ip_vs_dst_set(dest, dest_dst, &rt->dst, cookie);
535 else535 else
536 noref = 0;536 noref = 0;