History

net/sched: act_police: add support for packet-per-second policing

Allow a policer action to enforce a rate-limit based on packets-per-second, configurable using a packet-per-second rate and burst parameters. e.g. tc filter add dev tap1 parent ffff: u32 match \ u32 0 0 police pkts_rate 3000 pkts_burst 1000 Testing was unable to uncover a performance impact of this change on existing features.

Baowen Zheng committed 5 years ago

Commit 2ffe0391 parent 6a56e195 files · +162 −36Browse files

Unified · Split

5 files
  1. include/net/sch_generic.h +14 −0
  2. include/net/tc_act/tc_police.h +44 −4
  3. include/uapi/linux/pkt_cls.h +2 −0
  4. net/sched/act_police.c +52 −7
  5. net/sched/sch_generic.c +50 −25
include/net/sch_generic.h+14−0
@@ -1242,6 +1242,20 @@ static inline void psched_ratecfg_getrate(struct tc_ratespec *res, · 1241 unmodified lines
1242 res->linklayer = (r->linklayer & TC_LINKLAYER_MASK);1242 res->linklayer = (r->linklayer & TC_LINKLAYER_MASK);
1243}1243}
1244 1244
1245struct psched_pktrate {
1246 u64 rate_pkts_ps; /* packets per second */
1247 u32 mult;
1248 u8 shift;
1249};
1250
1251static inline u64 psched_pkt2t_ns(const struct psched_pktrate *r,
1252 unsigned int pkt_num)
1253{
1254 return ((u64)pkt_num * r->mult) >> r->shift;
1255}
1256
1257void psched_ppscfg_precompute(struct psched_pktrate *r, u64 pktrate64);
1258
1245/* Mini Qdisc serves for specific needs of ingress/clsact Qdisc.1259/* Mini Qdisc serves for specific needs of ingress/clsact Qdisc.
1246 * The fast path only needs to access filter list and to update stats1260 * The fast path only needs to access filter list and to update stats
1247 */1261 */
include/net/tc_act/tc_police.h+44−4
@@ -10,10 +10,13 @@ struct tcf_police_params { · 9 unmodified lines
10 s64 tcfp_burst;10 s64 tcfp_burst;
11 u32 tcfp_mtu;11 u32 tcfp_mtu;
12 s64 tcfp_mtu_ptoks;12 s64 tcfp_mtu_ptoks;
13 s64 tcfp_pkt_burst;
13 struct psched_ratecfg rate;14 struct psched_ratecfg rate;
14 bool rate_present;15 bool rate_present;
15 struct psched_ratecfg peak;16 struct psched_ratecfg peak;
16 bool peak_present;17 bool peak_present;
18 struct psched_pktrate ppsrate;
19 bool pps_present;
17 struct rcu_head rcu;20 struct rcu_head rcu;
18};21};
19 22
@@ -24,6 +27,7 @@ struct tcf_police { · 4 unmodified lines
24 spinlock_t tcfp_lock ____cacheline_aligned_in_smp;27 spinlock_t tcfp_lock ____cacheline_aligned_in_smp;
25 s64 tcfp_toks;28 s64 tcfp_toks;
26 s64 tcfp_ptoks;29 s64 tcfp_ptoks;
30 s64 tcfp_pkttoks;
27 s64 tcfp_t_c;31 s64 tcfp_t_c;
28};32};
29 33
@@ -99,14 +103,50 @@ static inline u32 tcf_police_burst(const struct tc_action *act) · 69 unmodified lines
99 103
100static inline u64 tcf_police_rate_pkt_ps(const struct tc_action *act)104static inline u64 tcf_police_rate_pkt_ps(const struct tc_action *act)
101{105{
102 /* Not implemented */106 struct tcf_police *police = to_police(act);
103 return 0;107 struct tcf_police_params *params;
108
109 params = rcu_dereference_protected(police->params,
110 lockdep_is_held(&police->tcf_lock));
111 return params->ppsrate.rate_pkts_ps;
104}112}
105 113
106static inline u32 tcf_police_burst_pkt(const struct tc_action *act)114static inline u32 tcf_police_burst_pkt(const struct tc_action *act)
107{115{
108 /* Not implemented */116 struct tcf_police *police = to_police(act);
109 return 0;117 struct tcf_police_params *params;
118 u32 burst;
119
120 params = rcu_dereference_protected(police->params,
121 lockdep_is_held(&police->tcf_lock));
122
123 /*
124 * "rate" pkts "burst" nanoseconds
125 * ------------ * -------------------
126 * 1 second 2^6 ticks
127 *
128 * ------------------------------------
129 * NSEC_PER_SEC nanoseconds
130 * ------------------------
131 * 2^6 ticks
132 *
133 * "rate" pkts "burst" nanoseconds 2^6 ticks
134 * = ------------ * ------------------- * ------------------------
135 * 1 second 2^6 ticks NSEC_PER_SEC nanoseconds
136 *
137 * "rate" * "burst"
138 * = ---------------- pkts/nanosecond
139 * NSEC_PER_SEC^2
140 *
141 *
142 * "rate" * "burst"
143 * = ---------------- pkts/second
144 * NSEC_PER_SEC
145 */
146 burst = div_u64(params->tcfp_pkt_burst * params->ppsrate.rate_pkts_ps,
147 NSEC_PER_SEC);
148
149 return burst;
110}150}
111 151
112static inline u32 tcf_police_tcfp_mtu(const struct tc_action *act)152static inline u32 tcf_police_tcfp_mtu(const struct tc_action *act)
include/uapi/linux/pkt_cls.h+2−0
@@ -190,6 +190,8 @@ enum { · 189 unmodified lines
190 TCA_POLICE_PAD,190 TCA_POLICE_PAD,
191 TCA_POLICE_RATE64,191 TCA_POLICE_RATE64,
192 TCA_POLICE_PEAKRATE64,192 TCA_POLICE_PEAKRATE64,
193 TCA_POLICE_PKTRATE64,
194 TCA_POLICE_PKTBURST64,
193 __TCA_POLICE_MAX195 __TCA_POLICE_MAX
194#define TCA_POLICE_RESULT TCA_POLICE_RESULT196#define TCA_POLICE_RESULT TCA_POLICE_RESULT
195};197};
net/sched/act_police.c+52−7
@@ -42,6 +42,8 @@ static const struct nla_policy police_policy[TCA_POLICE_MAX + 1] = { · 41 unmodified lines
42 [TCA_POLICE_RESULT] = { .type = NLA_U32 },42 [TCA_POLICE_RESULT] = { .type = NLA_U32 },
43 [TCA_POLICE_RATE64] = { .type = NLA_U64 },43 [TCA_POLICE_RATE64] = { .type = NLA_U64 },
44 [TCA_POLICE_PEAKRATE64] = { .type = NLA_U64 },44 [TCA_POLICE_PEAKRATE64] = { .type = NLA_U64 },
45 [TCA_POLICE_PKTRATE64] = { .type = NLA_U64, .min = 1 },
46 [TCA_POLICE_PKTBURST64] = { .type = NLA_U64, .min = 1 },
45};47};
46 48
47static int tcf_police_init(struct net *net, struct nlattr *nla,49static int tcf_police_init(struct net *net, struct nlattr *nla,
@@ -61,6 +63,7 @@ static int tcf_police_init(struct net *net, struct nlattr *nla, · 13 unmodified lines
61 bool exists = false;63 bool exists = false;
62 u32 index;64 u32 index;
63 u64 rate64, prate64;65 u64 rate64, prate64;
66 u64 pps, ppsburst;
64 67
65 if (nla == NULL)68 if (nla == NULL)
66 return -EINVAL;69 return -EINVAL;
@@ -142,6 +145,21 @@ static int tcf_police_init(struct net *net, struct nlattr *nla, · 75 unmodified lines
142 }145 }
143 }146 }
144 147
148 if ((tb[TCA_POLICE_PKTRATE64] && !tb[TCA_POLICE_PKTBURST64]) ||
149 (!tb[TCA_POLICE_PKTRATE64] && tb[TCA_POLICE_PKTBURST64])) {
150 NL_SET_ERR_MSG(extack,
151 "Both or neither packet-per-second burst and rate must be provided");
152 err = -EINVAL;
153 goto failure;
154 }
155
156 if (tb[TCA_POLICE_PKTRATE64] && R_tab) {
157 NL_SET_ERR_MSG(extack,
158 "packet-per-second and byte-per-second rate limits not allowed in same action");
159 err = -EINVAL;
160 goto failure;
161 }
162
145 new = kzalloc(sizeof(*new), GFP_KERNEL);163 new = kzalloc(sizeof(*new), GFP_KERNEL);
146 if (unlikely(!new)) {164 if (unlikely(!new)) {
147 err = -ENOMEM;165 err = -ENOMEM;
@@ -183,6 +201,14 @@ static int tcf_police_init(struct net *net, struct nlattr *nla, · 35 unmodified lines
183 if (tb[TCA_POLICE_AVRATE])201 if (tb[TCA_POLICE_AVRATE])
184 new->tcfp_ewma_rate = nla_get_u32(tb[TCA_POLICE_AVRATE]);202 new->tcfp_ewma_rate = nla_get_u32(tb[TCA_POLICE_AVRATE]);
185 203
204 if (tb[TCA_POLICE_PKTRATE64]) {
205 pps = nla_get_u64(tb[TCA_POLICE_PKTRATE64]);
206 ppsburst = nla_get_u64(tb[TCA_POLICE_PKTBURST64]);
207 new->pps_present = true;
208 new->tcfp_pkt_burst = PSCHED_TICKS2NS(ppsburst);
209 psched_ppscfg_precompute(&new->ppsrate, pps);
210 }
211
186 spin_lock_bh(&police->tcf_lock);212 spin_lock_bh(&police->tcf_lock);
187 spin_lock_bh(&police->tcfp_lock);213 spin_lock_bh(&police->tcfp_lock);
188 police->tcfp_t_c = ktime_get_ns();214 police->tcfp_t_c = ktime_get_ns();
@@ -217,8 +243,8 @@ static int tcf_police_act(struct sk_buff *skb, const struct tc_action *a, · 28 unmodified lines
217 struct tcf_result *res)243 struct tcf_result *res)
218{244{
219 struct tcf_police *police = to_police(a);245 struct tcf_police *police = to_police(a);
246 s64 now, toks, ppstoks = 0, ptoks = 0;
220 struct tcf_police_params *p;247 struct tcf_police_params *p;
221 s64 now, toks, ptoks = 0;
222 int ret;248 int ret;
223 249
224 tcf_lastuse_update(&police->tcf_tm);250 tcf_lastuse_update(&police->tcf_tm);
@@ -236,7 +262,7 @@ static int tcf_police_act(struct sk_buff *skb, const struct tc_action *a, · 11 unmodified lines
236 }262 }
237 263
238 if (qdisc_pkt_len(skb) <= p->tcfp_mtu) {264 if (qdisc_pkt_len(skb) <= p->tcfp_mtu) {
239 if (!p->rate_present) {265 if (!p->rate_present && !p->pps_present) {
240 ret = p->tcfp_result;266 ret = p->tcfp_result;
241 goto end;267 goto end;
242 }268 }
@@ -251,14 +277,23 @@ static int tcf_police_act(struct sk_buff *skb, const struct tc_action *a, · 8 unmodified lines
251 ptoks -= (s64)psched_l2t_ns(&p->peak,277 ptoks -= (s64)psched_l2t_ns(&p->peak,
252 qdisc_pkt_len(skb));278 qdisc_pkt_len(skb));
253 }279 }
254 toks += police->tcfp_toks;280 if (p->rate_present) {
255 if (toks > p->tcfp_burst)281 toks += police->tcfp_toks;
256 toks = p->tcfp_burst;282 if (toks > p->tcfp_burst)
257 toks -= (s64)psched_l2t_ns(&p->rate, qdisc_pkt_len(skb));283 toks = p->tcfp_burst;
258 if ((toks|ptoks) >= 0) {284 toks -= (s64)psched_l2t_ns(&p->rate, qdisc_pkt_len(skb));
285 } else if (p->pps_present) {
286 ppstoks = min_t(s64, now - police->tcfp_t_c, p->tcfp_pkt_burst);
287 ppstoks += police->tcfp_pkttoks;
288 if (ppstoks > p->tcfp_pkt_burst)
289 ppstoks = p->tcfp_pkt_burst;
290 ppstoks -= (s64)psched_pkt2t_ns(&p->ppsrate, 1);
291 }
292 if ((toks | ptoks | ppstoks) >= 0) {
259 police->tcfp_t_c = now;293 police->tcfp_t_c = now;
260 police->tcfp_toks = toks;294 police->tcfp_toks = toks;
261 police->tcfp_ptoks = ptoks;295 police->tcfp_ptoks = ptoks;
296 police->tcfp_pkttoks = ppstoks;
262 spin_unlock_bh(&police->tcfp_lock);297 spin_unlock_bh(&police->tcfp_lock);
263 ret = p->tcfp_result;298 ret = p->tcfp_result;
264 goto inc_drops;299 goto inc_drops;
@@ -331,6 +366,16 @@ static int tcf_police_dump(struct sk_buff *skb, struct tc_action *a, · 66 unmodified lines
331 TCA_POLICE_PAD))366 TCA_POLICE_PAD))
332 goto nla_put_failure;367 goto nla_put_failure;
333 }368 }
369 if (p->pps_present) {
370 if (nla_put_u64_64bit(skb, TCA_POLICE_PKTRATE64,
371 police->params->ppsrate.rate_pkts_ps,
372 TCA_POLICE_PAD))
373 goto nla_put_failure;
374 if (nla_put_u64_64bit(skb, TCA_POLICE_PKTBURST64,
375 PSCHED_NS2TICKS(p->tcfp_pkt_burst),
376 TCA_POLICE_PAD))
377 goto nla_put_failure;
378 }
334 if (nla_put(skb, TCA_POLICE_TBF, sizeof(opt), &opt))379 if (nla_put(skb, TCA_POLICE_TBF, sizeof(opt), &opt))
335 goto nla_put_failure;380 goto nla_put_failure;
336 if (p->tcfp_result &&381 if (p->tcfp_result &&
net/sched/sch_generic.c+50−25
@@ -1325,6 +1325,48 @@ void dev_shutdown(struct net_device *dev) · 1324 unmodified lines
1325 WARN_ON(timer_pending(&dev->watchdog_timer));1325 WARN_ON(timer_pending(&dev->watchdog_timer));
1326}1326}
1327 1327
1328/**
1329 * psched_ratecfg_precompute__() - Pre-compute values for reciprocal division
1330 * @rate: Rate to compute reciprocal division values of
1331 * @mult: Multiplier for reciprocal division
1332 * @shift: Shift for reciprocal division
1333 *
1334 * The multiplier and shift for reciprocal division by rate are stored
1335 * in mult and shift.
1336 *
1337 * The deal here is to replace a divide by a reciprocal one
1338 * in fast path (a reciprocal divide is a multiply and a shift)
1339 *
1340 * Normal formula would be :
1341 * time_in_ns = (NSEC_PER_SEC * len) / rate_bps
1342 *
1343 * We compute mult/shift to use instead :
1344 * time_in_ns = (len * mult) >> shift;
1345 *
1346 * We try to get the highest possible mult value for accuracy,
1347 * but have to make sure no overflows will ever happen.
1348 *
1349 * reciprocal_value() is not used here it doesn't handle 64-bit values.
1350 */
1351static void psched_ratecfg_precompute__(u64 rate, u32 *mult, u8 *shift)
1352{
1353 u64 factor = NSEC_PER_SEC;
1354
1355 *mult = 1;
1356 *shift = 0;
1357
1358 if (rate <= 0)
1359 return;
1360
1361 for (;;) {
1362 *mult = div64_u64(factor, rate);
1363 if (*mult & (1U << 31) || factor & (1ULL << 63))
1364 break;
1365 factor <<= 1;
1366 (*shift)++;
1367 }
1368}
1369
1328void psched_ratecfg_precompute(struct psched_ratecfg *r,1370void psched_ratecfg_precompute(struct psched_ratecfg *r,
1329 const struct tc_ratespec *conf,1371 const struct tc_ratespec *conf,
1330 u64 rate64)1372 u64 rate64)
@@ -1333,34 +1375,17 @@ void psched_ratecfg_precompute(struct psched_ratecfg *r, · 2 unmodified lines
1333 r->overhead = conf->overhead;1375 r->overhead = conf->overhead;
1334 r->rate_bytes_ps = max_t(u64, conf->rate, rate64);1376 r->rate_bytes_ps = max_t(u64, conf->rate, rate64);
1335 r->linklayer = (conf->linklayer & TC_LINKLAYER_MASK);1377 r->linklayer = (conf->linklayer & TC_LINKLAYER_MASK);
1336 r->mult = 1;1378 psched_ratecfg_precompute__(r->rate_bytes_ps, &r->mult, &r->shift);
1337 /*
1338 * The deal here is to replace a divide by a reciprocal one
1339 * in fast path (a reciprocal divide is a multiply and a shift)
1340 *
1341 * Normal formula would be :
1342 * time_in_ns = (NSEC_PER_SEC * len) / rate_bps
1343 *
1344 * We compute mult/shift to use instead :
1345 * time_in_ns = (len * mult) >> shift;
1346 *
1347 * We try to get the highest possible mult value for accuracy,
1348 * but have to make sure no overflows will ever happen.
1349 */
1350 if (r->rate_bytes_ps > 0) {
1351 u64 factor = NSEC_PER_SEC;
1352
1353 for (;;) {
1354 r->mult = div64_u64(factor, r->rate_bytes_ps);
1355 if (r->mult & (1U << 31) || factor & (1ULL << 63))
1356 break;
1357 factor <<= 1;
1358 r->shift++;
1359 }
1360 }
1361}1379}
1362EXPORT_SYMBOL(psched_ratecfg_precompute);1380EXPORT_SYMBOL(psched_ratecfg_precompute);
1363 1381
1382void psched_ppscfg_precompute(struct psched_pktrate *r, u64 pktrate64)
1383{
1384 r->rate_pkts_ps = pktrate64;
1385 psched_ratecfg_precompute__(r->rate_pkts_ps, &r->mult, &r->shift);
1386}
1387EXPORT_SYMBOL(psched_ppscfg_precompute);
1388
1364static void mini_qdisc_rcu_func(struct rcu_head *head)1389static void mini_qdisc_rcu_func(struct rcu_head *head)
1365{1390{
1366}1391}