net/sched: act_police: add support for packet-per-second policing

Allow a policer action to enforce a rate-limit based on packets-per-second, configurable using a packet-per-second rate and burst parameters. e.g. tc filter add dev tap1 parent ffff: u32 match \ u32 0 0 police pkts_rate 3000 pkts_burst 1000 Testing was unable to uncover a performance impact of this change on existing features.

Baowen Zheng committed 5 years ago

Commit 2ffe0391 parent 6a56e195 files · +162 −36Browse files

Unified · Split

5 files

include/net/sch_generic.h +14 −0
include/net/tc_act/tc_police.h +44 −4
include/uapi/linux/pkt_cls.h +2 −0
net/sched/act_police.c +52 −7
net/sched/sch_generic.c +50 −25

include/net/sch_generic.h+14−0

@@ -1242,6 +1242,20 @@ static inline void psched_ratecfg_getrate(struct tc_ratespec *res, · 1241 unmodified lines
1242	1242	res->linklayer = (r->linklayer & TC_LINKLAYER_MASK);
1243	1243	}
1244	1244
	1245	+struct psched_pktrate {
	1246	+ u64 rate_pkts_ps; /* packets per second */
	1247	+ u32 mult;
	1248	+ u8 shift;
	1249	+};
	1250	+
	1251	+static inline u64 psched_pkt2t_ns(const struct psched_pktrate *r,
	1252	+ unsigned int pkt_num)
	1253	+{
	1254	+ return ((u64)pkt_num * r->mult) >> r->shift;
	1255	+}
	1256	+
	1257	+void psched_ppscfg_precompute(struct psched_pktrate *r, u64 pktrate64);
	1258	+
1245	1259	/* Mini Qdisc serves for specific needs of ingress/clsact Qdisc.
1246	1260	* The fast path only needs to access filter list and to update stats
1247	1261	*/

include/net/tc_act/tc_police.h+44−4

@@ -10,10 +10,13 @@ struct tcf_police_params { · 9 unmodified lines
10	10	s64 tcfp_burst;
11	11	u32 tcfp_mtu;
12	12	s64 tcfp_mtu_ptoks;
	13	+ s64 tcfp_pkt_burst;
13	14	struct psched_ratecfg rate;
14	15	bool rate_present;
15	16	struct psched_ratecfg peak;
16	17	bool peak_present;
	18	+ struct psched_pktrate ppsrate;
	19	+ bool pps_present;
17	20	struct rcu_head rcu;
18	21	};
19	22
@@ -24,6 +27,7 @@ struct tcf_police { · 4 unmodified lines
24	27	spinlock_t tcfp_lock ____cacheline_aligned_in_smp;
25	28	s64 tcfp_toks;
26	29	s64 tcfp_ptoks;
	30	+ s64 tcfp_pkttoks;
27	31	s64 tcfp_t_c;
28	32	};
29	33
@@ -99,14 +103,50 @@ static inline u32 tcf_police_burst(const struct tc_action *act) · 69 unmodified lines
99	103
100	104	static inline u64 tcf_police_rate_pkt_ps(const struct tc_action *act)
101	105	{
102		− /* Not implemented */
103		− return 0;
	106	+ struct tcf_police *police = to_police(act);
	107	+ struct tcf_police_params *params;
	108	+
	109	+ params = rcu_dereference_protected(police->params,
	110	+ lockdep_is_held(&police->tcf_lock));
	111	+ return params->ppsrate.rate_pkts_ps;
104	112	}
105	113
106	114	static inline u32 tcf_police_burst_pkt(const struct tc_action *act)
107	115	{
108		− /* Not implemented */
109		− return 0;
	116	+ struct tcf_police *police = to_police(act);
	117	+ struct tcf_police_params *params;
	118	+ u32 burst;
	119	+
	120	+ params = rcu_dereference_protected(police->params,
	121	+ lockdep_is_held(&police->tcf_lock));
	122	+
	123	+ /*
	124	+ * "rate" pkts "burst" nanoseconds
	125	+ * ------------ * -------------------
	126	+ * 1 second 2^6 ticks
	127	+ *
	128	+ * ------------------------------------
	129	+ * NSEC_PER_SEC nanoseconds
	130	+ * ------------------------
	131	+ * 2^6 ticks
	132	+ *
	133	+ * "rate" pkts "burst" nanoseconds 2^6 ticks
	134	+ * = ------------ * ------------------- * ------------------------
	135	+ * 1 second 2^6 ticks NSEC_PER_SEC nanoseconds
	136	+ *
	137	+ * "rate" * "burst"
	138	+ * = ---------------- pkts/nanosecond
	139	+ * NSEC_PER_SEC^2
	140	+ *
	141	+ *
	142	+ * "rate" * "burst"
	143	+ * = ---------------- pkts/second
	144	+ * NSEC_PER_SEC
	145	+ */
	146	+ burst = div_u64(params->tcfp_pkt_burst * params->ppsrate.rate_pkts_ps,
	147	+ NSEC_PER_SEC);
	148	+
	149	+ return burst;
110	150	}
111	151
112	152	static inline u32 tcf_police_tcfp_mtu(const struct tc_action *act)

include/uapi/linux/pkt_cls.h+2−0

@@ -190,6 +190,8 @@ enum { · 189 unmodified lines
190	190	TCA_POLICE_PAD,
191	191	TCA_POLICE_RATE64,
192	192	TCA_POLICE_PEAKRATE64,
	193	+ TCA_POLICE_PKTRATE64,
	194	+ TCA_POLICE_PKTBURST64,
193	195	__TCA_POLICE_MAX
194	196	#define TCA_POLICE_RESULT TCA_POLICE_RESULT
195	197	};

net/sched/act_police.c+52−7

@@ -42,6 +42,8 @@ static const struct nla_policy police_policy[TCA_POLICE_MAX + 1] = { · 41 unmodified lines
42	42	[TCA_POLICE_RESULT] = { .type = NLA_U32 },
43	43	[TCA_POLICE_RATE64] = { .type = NLA_U64 },
44	44	[TCA_POLICE_PEAKRATE64] = { .type = NLA_U64 },
	45	+ [TCA_POLICE_PKTRATE64] = { .type = NLA_U64, .min = 1 },
	46	+ [TCA_POLICE_PKTBURST64] = { .type = NLA_U64, .min = 1 },
45	47	};
46	48
47	49	static int tcf_police_init(struct net net, struct nlattr nla,
@@ -61,6 +63,7 @@ static int tcf_police_init(struct net net, struct nlattr nla, · 13 unmodified lines
61	63	bool exists = false;
62	64	u32 index;
63	65	u64 rate64, prate64;
	66	+ u64 pps, ppsburst;
64	67
65	68	if (nla == NULL)
66	69	return -EINVAL;
@@ -142,6 +145,21 @@ static int tcf_police_init(struct net net, struct nlattr nla, · 75 unmodified lines
142	145	}
143	146	}
144	147
	148	+ if ((tb[TCA_POLICE_PKTRATE64] && !tb[TCA_POLICE_PKTBURST64]) \|\|
	149	+ (!tb[TCA_POLICE_PKTRATE64] && tb[TCA_POLICE_PKTBURST64])) {
	150	+ NL_SET_ERR_MSG(extack,
	151	+ "Both or neither packet-per-second burst and rate must be provided");
	152	+ err = -EINVAL;
	153	+ goto failure;
	154	+ }
	155	+
	156	+ if (tb[TCA_POLICE_PKTRATE64] && R_tab) {
	157	+ NL_SET_ERR_MSG(extack,
	158	+ "packet-per-second and byte-per-second rate limits not allowed in same action");
	159	+ err = -EINVAL;
	160	+ goto failure;
	161	+ }
	162	+
145	163	new = kzalloc(sizeof(*new), GFP_KERNEL);
146	164	if (unlikely(!new)) {
147	165	err = -ENOMEM;
@@ -183,6 +201,14 @@ static int tcf_police_init(struct net net, struct nlattr nla, · 35 unmodified lines
183	201	if (tb[TCA_POLICE_AVRATE])
184	202	new->tcfp_ewma_rate = nla_get_u32(tb[TCA_POLICE_AVRATE]);
185	203
	204	+ if (tb[TCA_POLICE_PKTRATE64]) {
	205	+ pps = nla_get_u64(tb[TCA_POLICE_PKTRATE64]);
	206	+ ppsburst = nla_get_u64(tb[TCA_POLICE_PKTBURST64]);
	207	+ new->pps_present = true;
	208	+ new->tcfp_pkt_burst = PSCHED_TICKS2NS(ppsburst);
	209	+ psched_ppscfg_precompute(&new->ppsrate, pps);
	210	+ }
	211	+
186	212	spin_lock_bh(&police->tcf_lock);
187	213	spin_lock_bh(&police->tcfp_lock);
188	214	police->tcfp_t_c = ktime_get_ns();
@@ -217,8 +243,8 @@ static int tcf_police_act(struct sk_buff skb, const struct tc_action a, · 28 unmodified lines
217	243	struct tcf_result *res)
218	244	{
219	245	struct tcf_police *police = to_police(a);
	246	+ s64 now, toks, ppstoks = 0, ptoks = 0;
220	247	struct tcf_police_params *p;
221		− s64 now, toks, ptoks = 0;
222	248	int ret;
223	249
224	250	tcf_lastuse_update(&police->tcf_tm);
@@ -236,7 +262,7 @@ static int tcf_police_act(struct sk_buff skb, const struct tc_action a, · 11 unmodified lines
236	262	}
237	263
238	264	if (qdisc_pkt_len(skb) <= p->tcfp_mtu) {
239		− if (!p->rate_present) {
	265	+ if (!p->rate_present && !p->pps_present) {
240	266	ret = p->tcfp_result;
241	267	goto end;
242	268	}
@@ -251,14 +277,23 @@ static int tcf_police_act(struct sk_buff skb, const struct tc_action a, · 8 unmodified lines
251	277	ptoks -= (s64)psched_l2t_ns(&p->peak,
252	278	qdisc_pkt_len(skb));
253	279	}
254		− toks += police->tcfp_toks;
255		− if (toks > p->tcfp_burst)
256		− toks = p->tcfp_burst;
257		− toks -= (s64)psched_l2t_ns(&p->rate, qdisc_pkt_len(skb));
258		− if ((toks\|ptoks) >= 0) {
	280	+ if (p->rate_present) {
	281	+ toks += police->tcfp_toks;
	282	+ if (toks > p->tcfp_burst)
	283	+ toks = p->tcfp_burst;
	284	+ toks -= (s64)psched_l2t_ns(&p->rate, qdisc_pkt_len(skb));
	285	+ } else if (p->pps_present) {
	286	+ ppstoks = min_t(s64, now - police->tcfp_t_c, p->tcfp_pkt_burst);
	287	+ ppstoks += police->tcfp_pkttoks;
	288	+ if (ppstoks > p->tcfp_pkt_burst)
	289	+ ppstoks = p->tcfp_pkt_burst;
	290	+ ppstoks -= (s64)psched_pkt2t_ns(&p->ppsrate, 1);
	291	+ }
	292	+ if ((toks \| ptoks \| ppstoks) >= 0) {
259	293	police->tcfp_t_c = now;
260	294	police->tcfp_toks = toks;
261	295	police->tcfp_ptoks = ptoks;
	296	+ police->tcfp_pkttoks = ppstoks;
262	297	spin_unlock_bh(&police->tcfp_lock);
263	298	ret = p->tcfp_result;
264	299	goto inc_drops;
@@ -331,6 +366,16 @@ static int tcf_police_dump(struct sk_buff skb, struct tc_action a, · 66 unmodified lines
331	366	TCA_POLICE_PAD))
332	367	goto nla_put_failure;
333	368	}
	369	+ if (p->pps_present) {
	370	+ if (nla_put_u64_64bit(skb, TCA_POLICE_PKTRATE64,
	371	+ police->params->ppsrate.rate_pkts_ps,
	372	+ TCA_POLICE_PAD))
	373	+ goto nla_put_failure;
	374	+ if (nla_put_u64_64bit(skb, TCA_POLICE_PKTBURST64,
	375	+ PSCHED_NS2TICKS(p->tcfp_pkt_burst),
	376	+ TCA_POLICE_PAD))
	377	+ goto nla_put_failure;
	378	+ }
334	379	if (nla_put(skb, TCA_POLICE_TBF, sizeof(opt), &opt))
335	380	goto nla_put_failure;
336	381	if (p->tcfp_result &&

net/sched/sch_generic.c+50−25

@@ -1325,6 +1325,48 @@ void dev_shutdown(struct net_device *dev) · 1324 unmodified lines
1325	1325	WARN_ON(timer_pending(&dev->watchdog_timer));
1326	1326	}
1327	1327
	1328	+/**
	1329	+ * psched_ratecfg_precompute__() - Pre-compute values for reciprocal division
	1330	+ * @rate: Rate to compute reciprocal division values of
	1331	+ * @mult: Multiplier for reciprocal division
	1332	+ * @shift: Shift for reciprocal division
	1333	+ *
	1334	+ * The multiplier and shift for reciprocal division by rate are stored
	1335	+ * in mult and shift.
	1336	+ *
	1337	+ * The deal here is to replace a divide by a reciprocal one
	1338	+ * in fast path (a reciprocal divide is a multiply and a shift)
	1339	+ *
	1340	+ * Normal formula would be :
	1341	+ * time_in_ns = (NSEC_PER_SEC * len) / rate_bps
	1342	+ *
	1343	+ * We compute mult/shift to use instead :
	1344	+ * time_in_ns = (len * mult) >> shift;
	1345	+ *
	1346	+ * We try to get the highest possible mult value for accuracy,
	1347	+ * but have to make sure no overflows will ever happen.
	1348	+ *
	1349	+ * reciprocal_value() is not used here it doesn't handle 64-bit values.
	1350	+ */
	1351	+static void psched_ratecfg_precompute__(u64 rate, u32 mult, u8 shift)
	1352	+{
	1353	+ u64 factor = NSEC_PER_SEC;
	1354	+
	1355	+ *mult = 1;
	1356	+ *shift = 0;
	1357	+
	1358	+ if (rate <= 0)
	1359	+ return;
	1360	+
	1361	+ for (;;) {
	1362	+ *mult = div64_u64(factor, rate);
	1363	+ if (*mult & (1U << 31) \|\| factor & (1ULL << 63))
	1364	+ break;
	1365	+ factor <<= 1;
	1366	+ (*shift)++;
	1367	+ }
	1368	+}
	1369	+
1328	1370	void psched_ratecfg_precompute(struct psched_ratecfg *r,
1329	1371	const struct tc_ratespec *conf,
1330	1372	u64 rate64)
@@ -1333,34 +1375,17 @@ void psched_ratecfg_precompute(struct psched_ratecfg *r, · 2 unmodified lines
1333	1375	r->overhead = conf->overhead;
1334	1376	r->rate_bytes_ps = max_t(u64, conf->rate, rate64);
1335	1377	r->linklayer = (conf->linklayer & TC_LINKLAYER_MASK);
1336		− r->mult = 1;
1337		− /*
1338		− * The deal here is to replace a divide by a reciprocal one
1339		− * in fast path (a reciprocal divide is a multiply and a shift)
1340		− *
1341		− * Normal formula would be :
1342		− * time_in_ns = (NSEC_PER_SEC * len) / rate_bps
1343		− *
1344		− * We compute mult/shift to use instead :
1345		− * time_in_ns = (len * mult) >> shift;
1346		− *
1347		− * We try to get the highest possible mult value for accuracy,
1348		− * but have to make sure no overflows will ever happen.
1349		− */
1350		− if (r->rate_bytes_ps > 0) {
1351		− u64 factor = NSEC_PER_SEC;
1352		−
1353		− for (;;) {
1354		− r->mult = div64_u64(factor, r->rate_bytes_ps);
1355		− if (r->mult & (1U << 31) \|\| factor & (1ULL << 63))
1356		− break;
1357		− factor <<= 1;
1358		− r->shift++;
1359		− }
1360		− }
	1378	+ psched_ratecfg_precompute__(r->rate_bytes_ps, &r->mult, &r->shift);
1361	1379	}
1362	1380	EXPORT_SYMBOL(psched_ratecfg_precompute);
1363	1381
	1382	+void psched_ppscfg_precompute(struct psched_pktrate *r, u64 pktrate64)
	1383	+{
	1384	+ r->rate_pkts_ps = pktrate64;
	1385	+ psched_ratecfg_precompute__(r->rate_pkts_ps, &r->mult, &r->shift);
	1386	+}
	1387	+EXPORT_SYMBOL(psched_ppscfg_precompute);
	1388	+
1364	1389	static void mini_qdisc_rcu_func(struct rcu_head *head)
1365	1390	{
1366	1391	}