ipvs: separate destination availability state

IPVS configuration paths update destination availability while connection accounting updates destination overload state. The two independent states share dest->flags, so their read-modify-write updates can race and lose one another. Keep OVERLOAD in flags, where the preceding patch serializes its updates with dst_lock, and move AVAILABLE to cflags. This keeps configuration- controlled availability out of the scheduler hot cacheline until a scheduler needs to check it. It also prevents availability updates from clobbering overload state. The destination status bits are not exposed through the IPVS sockopt or netlink interfaces, so keep their definitions in the internal IPVS header. Readers can still observe stale destination state; this does not provide a cross-field snapshot.

Yizhou Zhao committed 3 months ago

Commit cdcc4e41 parent 8f843449 files · +23 −22Browse files

Unified · Split

9 files

include/net/ip_vs.h +7 −0
include/uapi/linux/ip_vs.h +0 −6
net/netfilter/ipvs/ip_vs_conn.c +2 −2
net/netfilter/ipvs/ip_vs_core.c +3 −3
net/netfilter/ipvs/ip_vs_ctl.c +2 −2
net/netfilter/ipvs/ip_vs_dh.c +2 −2
net/netfilter/ipvs/ip_vs_lblc.c +1 −1
net/netfilter/ipvs/ip_vs_lblcr.c +4 −4
net/netfilter/ipvs/ip_vs_xmit.c +2 −2

include/net/ip_vs.h+7−0

@@ -36,6 +36,12 @@ · 35 unmodified lines
36	36	#define IP_VS_HDR_INVERSE 1
37	37	#define IP_VS_HDR_ICMP 2
38	38
	39	+/* Destination Server Flags */
	40	+#define IP_VS_DEST_F_OVERLOAD 0x0002 /* server is overloaded */
	41	+
	42	+/* Destination Server Config Flags */
	43	+#define IP_VS_DEST_CF_AVAILABLE 0x0001 /* server is available */
	44	+
39	45	/* conn_tab limits (as per Kconfig) */
40	46	#define IP_VS_CONN_TAB_MIN_BITS 8
41	47	#if BITS_PER_LONG > 32
@@ -976,6 +982,7 @@ struct ip_vs_dest { · 934 unmodified lines
976	982	volatile unsigned int flags; /* dest status flags */
977	983	atomic_t conn_flags; /* flags to copy to conn */
978	984	atomic_t weight; /* server weight */
	985	+ unsigned long cflags; /* config flags */
979	986	atomic_t last_weight; /* server latest weight */
980	987	__u16 tun_type; /* tunnel type */
981	988	__be16 tun_port; /* tunnel port */

include/uapi/linux/ip_vs.h+0−6

@@ -28,12 +28,6 @@ · 27 unmodified lines
28	28	#define IP_VS_SVC_F_SCHED_SH_FALLBACK IP_VS_SVC_F_SCHED1 /* SH fallback */
29	29	#define IP_VS_SVC_F_SCHED_SH_PORT IP_VS_SVC_F_SCHED2 /* SH use port */
30	30
31		−/*
32		− * Destination Server Flags
33		− */
34		−#define IP_VS_DEST_F_AVAILABLE 0x0001 /* server is available */
35		−#define IP_VS_DEST_F_OVERLOAD 0x0002 /* server is overloaded */
36		−
37	31	/*
38	32	* IPVS sync daemon states
39	33	*/

net/netfilter/ipvs/ip_vs_conn.c+2−2

@@ -1279,7 +1279,7 @@ int ip_vs_check_template(struct ip_vs_conn ct, struct ip_vs_dest cdest) · 1278 unmodified lines
1279	1279	* Checking the dest server status.
1280	1280	*/
1281	1281	if ((dest == NULL) \|\|
1282		− !(dest->flags & IP_VS_DEST_F_AVAILABLE) \|\|
	1282	+ !(dest->cflags & IP_VS_DEST_CF_AVAILABLE) \|\|
1283	1283	expire_quiescent_template(ipvs, dest) \|\|
1284	1284	(cdest && (dest != cdest))) {
1285	1285	IP_VS_DBG_BUF(9, "check_template: dest not available for "
@@ -2020,7 +2020,7 @@ repeat: · 734 unmodified lines
2020	2020	cp = ip_vs_hn0_to_conn(hn);
2021	2021	resched_score++;
2022	2022	dest = cp->dest;
2023		− if (!dest \|\| (dest->flags & IP_VS_DEST_F_AVAILABLE))
	2023	+ if (!dest \|\| (dest->cflags & IP_VS_DEST_CF_AVAILABLE))
2024	2024	continue;
2025	2025
2026	2026	if (atomic_read(&cp->n_control))

net/netfilter/ipvs/ip_vs_core.c+3−3

@@ -302,7 +302,7 @@ ip_vs_in_stats(struct ip_vs_conn cp, struct sk_buff skb) · 301 unmodified lines
302	302	struct ip_vs_dest *dest = cp->dest;
303	303	struct netns_ipvs *ipvs = cp->ipvs;
304	304
305		− if (dest && (dest->flags & IP_VS_DEST_F_AVAILABLE)) {
	305	+ if (dest && (dest->cflags & IP_VS_DEST_CF_AVAILABLE)) {
306	306	struct ip_vs_cpu_stats *s;
307	307	struct ip_vs_service *svc;
308	308
@@ -338,7 +338,7 @@ ip_vs_out_stats(struct ip_vs_conn cp, struct sk_buff skb) · 29 unmodified lines
338	338	struct ip_vs_dest *dest = cp->dest;
339	339	struct netns_ipvs *ipvs = cp->ipvs;
340	340
341		− if (dest && (dest->flags & IP_VS_DEST_F_AVAILABLE)) {
	341	+ if (dest && (dest->cflags & IP_VS_DEST_CF_AVAILABLE)) {
342	342	struct ip_vs_cpu_stats *s;
343	343	struct ip_vs_service *svc;
344	344
@@ -2210,7 +2210,7 @@ ip_vs_in_hook(void priv, struct sk_buff skb, const struct nf_hook_state *state · 1865 unmodified lines
2210	2210	}
2211	2211
2212	2212	/* Check the server status */
2213		− if (cp && cp->dest && !(cp->dest->flags & IP_VS_DEST_F_AVAILABLE)) {
	2213	+ if (cp && cp->dest && !(cp->dest->cflags & IP_VS_DEST_CF_AVAILABLE)) {
2214	2214	/* the destination server is not available */
2215	2215	if (sysctl_expire_nodest_conn(ipvs)) {
2216	2216	bool old_ct = ip_vs_conn_uses_old_conntrack(cp, skb);

net/netfilter/ipvs/ip_vs_ctl.c+2−2

@@ -1402,7 +1402,7 @@ __ip_vs_update_dest(struct ip_vs_service svc, struct ip_vs_dest dest, · 1401 unmodified lines
1402	1402	}
1403	1403
1404	1404	/* set the dest status flags */
1405		− dest->flags \|= IP_VS_DEST_F_AVAILABLE;
	1405	+ dest->cflags \|= IP_VS_DEST_CF_AVAILABLE;
1406	1406
1407	1407	if (READ_ONCE(dest->u_threshold) != udest->u_threshold \|\|
1408	1408	READ_ONCE(dest->l_threshold) != udest->l_threshold) {
@@ -1662,7 +1662,7 @@ static void __ip_vs_unlink_dest(struct ip_vs_service *svc, · 253 unmodified lines
1662	1662	struct ip_vs_dest *dest,
1663	1663	int svcupd)
1664	1664	{
1665		− dest->flags &= ~IP_VS_DEST_F_AVAILABLE;
	1665	+ dest->cflags &= ~IP_VS_DEST_CF_AVAILABLE;
1666	1666
1667	1667	spin_lock_bh(&dest->dst_lock);
1668	1668	__ip_vs_dst_cache_reset(dest);

net/netfilter/ipvs/ip_vs_dh.c+2−2

@@ -219,8 +219,8 @@ ip_vs_dh_schedule(struct ip_vs_service svc, const struct sk_buff skb, · 218 unmodified lines
219	219
220	220	s = (struct ip_vs_dh_state *) svc->sched_data;
221	221	dest = ip_vs_dh_get(svc->af, s, &iph->daddr);
222		− if (!dest
223		− \|\| !(dest->flags & IP_VS_DEST_F_AVAILABLE)
	222	+ if (!dest \|\|
	223	+ !(dest->cflags & IP_VS_DEST_CF_AVAILABLE)
224	224	\|\| atomic_read(&dest->weight) <= 0
225	225	\|\| is_overloaded(dest)) {
226	226	ip_vs_scheduler_err(svc, "no destination available");

net/netfilter/ipvs/ip_vs_lblc.c+1−1

@@ -502,7 +502,7 @@ ip_vs_lblc_schedule(struct ip_vs_service svc, const struct sk_buff skb, · 501 unmodified lines
502	502	*/
503	503
504	504	dest = en->dest;
505		− if ((dest->flags & IP_VS_DEST_F_AVAILABLE) &&
	505	+ if ((dest->cflags & IP_VS_DEST_CF_AVAILABLE) &&
506	506	atomic_read(&dest->weight) > 0 && !is_overloaded(dest, svc))
507	507	goto out;
508	508	}

net/netfilter/ipvs/ip_vs_lblcr.c+4−4

@@ -169,8 +169,8 @@ static inline struct ip_vs_dest ip_vs_dest_set_min(struct ip_vs_dest_set set) · 168 unmodified lines
169	169	if (least->flags & IP_VS_DEST_F_OVERLOAD)
170	170	continue;
171	171
172		− if ((atomic_read(&least->weight) > 0)
173		− && (least->flags & IP_VS_DEST_F_AVAILABLE)) {
	172	+ if ((atomic_read(&least->weight) > 0) &&
	173	+ (least->cflags & IP_VS_DEST_CF_AVAILABLE)) {
174	174	loh = ip_vs_dest_conn_overhead(least);
175	175	goto nextstage;
176	176	}
@@ -186,8 +186,8 @@ static inline struct ip_vs_dest ip_vs_dest_set_min(struct ip_vs_dest_set set) · 9 unmodified lines
186	186
187	187	doh = ip_vs_dest_conn_overhead(dest);
188	188	if (((__s64)loh * atomic_read(&dest->weight) >
189		− (__s64)doh * atomic_read(&least->weight))
190		− && (dest->flags & IP_VS_DEST_F_AVAILABLE)) {
	189	+ (__s64)doh * atomic_read(&least->weight)) &&
	190	+ (dest->cflags & IP_VS_DEST_CF_AVAILABLE)) {
191	191	least = dest;
192	192	loh = doh;
193	193	}

net/netfilter/ipvs/ip_vs_xmit.c+2−2

@@ -351,7 +351,7 @@ __ip_vs_get_out_rt(struct netns_ipvs ipvs, int skb_af, struct sk_buff skb, · 350 unmodified lines
351	351	* stored in dest_trash.
352	352	*/
353	353	if (!rt_dev_is_down(dst_dev_rcu(&rt->dst)) &&
354		− dest->flags & IP_VS_DEST_F_AVAILABLE)
	354	+ dest->cflags & IP_VS_DEST_CF_AVAILABLE)
355	355	__ip_vs_dst_set(dest, dest_dst, &rt->dst, 0);
356	356	else
357	357	noref = 0;
@@ -530,7 +530,7 @@ __ip_vs_get_out_rt_v6(struct netns_ipvs ipvs, int skb_af, struct sk_buff skb, · 172 unmodified lines
530	530	* stored in dest_trash.
531	531	*/
532	532	if (!rt_dev_is_down(dst_dev_rcu(&rt->dst)) &&
533		− dest->flags & IP_VS_DEST_F_AVAILABLE)
	533	+ dest->cflags & IP_VS_DEST_CF_AVAILABLE)
534	534	__ip_vs_dst_set(dest, dest_dst, &rt->dst, cookie);
535	535	else
536	536	noref = 0;