History

net/mlx5: E-Switch, Increase number of chains and priorities

Increase the number of chains and priorities to support the whole range available in tc. We use unmanaged tables and ignore flow level to create more tables than what we declared to fs_core steering, and we manage the connections between the tables themselves. To support that we need FW with ignore_flow_level capability. Otherwise the old behaviour will be used, where we are limited by the number of levels we declared (4 chains, 16 prios).

Paul Blakey committed 6 years ago

Commit 278d51f1 parent 39ac2373 files · +232 −12Browse files

Unified · Split

3 files
  1. drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +2 −1
  2. drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_chains.c +227 −11
  3. drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_chains.h +3 −0
drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c+2−1
@@ -151,7 +151,7 @@ mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, · 150 unmodified lines
151151 if (attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH) {
152152 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
153153 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
154− dest[i].ft = esw->fdb_table.offloads.slow_fdb;
154+ dest[i].ft = mlx5_esw_chains_get_tc_end_ft(esw);
155155 i++;
156156 } else if (attr->dest_chain) {
157157 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
@@ -275,6 +275,7 @@ mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw, · 117 unmodified lines
275275 if (attr->outer_match_level != MLX5_MATCH_NONE)
276276 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
277277
278+ flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
278279 rule = mlx5_add_flow_rules(fast_fdb, spec, &flow_act, dest, i);
279280
280281 if (IS_ERR(rule))
drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_chains.c+227−11
@@ -16,6 +16,10 @@ · 15 unmodified lines
1616 #define esw_chains_ht(esw) (esw_chains_priv(esw)->chains_ht)
1717 #define esw_prios_ht(esw) (esw_chains_priv(esw)->prios_ht)
1818 #define fdb_pool_left(esw) (esw_chains_priv(esw)->fdb_left)
19+#define tc_slow_fdb(esw) ((esw)->fdb_table.offloads.slow_fdb)
20+#define tc_end_fdb(esw) (esw_chains_priv(esw)->tc_end_fdb)
21+#define fdb_ignore_flow_level_supported(esw) \
22+ (MLX5_CAP_ESW_FLOWTABLE_FDB((esw)->dev, ignore_flow_level))
1923
2024 #define ESW_OFFLOADS_NUM_GROUPS 4
2125
@@ -39,6 +43,8 @@ struct mlx5_esw_chains_priv { · 17 unmodified lines
3943 /* Protects above chains_ht and prios_ht */
4044 struct mutex lock;
4145
46+ struct mlx5_flow_table *tc_end_fdb;
47+
4248 int fdb_left[ARRAY_SIZE(ESW_POOLS)];
4349 };
4450
@@ -50,6 +56,7 @@ struct fdb_chain { · 5 unmodified lines
5056 int ref;
5157
5258 struct mlx5_eswitch *esw;
59+ struct list_head prios_list;
5360 };
5461
5562 struct fdb_prio_key {
@@ -60,6 +67,7 @@ struct fdb_prio_key { · 4 unmodified lines
6067
6168 struct fdb_prio {
6269 struct rhash_head node;
70+ struct list_head list;
6371
6472 struct fdb_prio_key key;
6573
@@ -67,6 +75,9 @@ struct fdb_prio { · 1 unmodified line
6775
6876 struct fdb_chain *fdb_chain;
6977 struct mlx5_flow_table *fdb;
78+ struct mlx5_flow_table *next_fdb;
79+ struct mlx5_flow_group *miss_group;
80+ struct mlx5_flow_handle *miss_rule;
7081 };
7182
7283 static const struct rhashtable_params chain_params = {
@@ -93,6 +104,9 @@ u32 mlx5_esw_chains_get_chain_range(struct mlx5_eswitch *esw) · 20 unmodified lines
93104 if (!mlx5_esw_chains_prios_supported(esw))
94105 return 1;
95106
107+ if (fdb_ignore_flow_level_supported(esw))
108+ return UINT_MAX - 1;
109+
96110 return FDB_TC_MAX_CHAIN;
97111 }
98112
@@ -106,11 +120,17 @@ u32 mlx5_esw_chains_get_prio_range(struct mlx5_eswitch *esw) · 7 unmodified lines
106120 if (!mlx5_esw_chains_prios_supported(esw))
107121 return 1;
108122
123+ if (fdb_ignore_flow_level_supported(esw))
124+ return UINT_MAX;
125+
109126 return FDB_TC_MAX_PRIO;
110127 }
111128
112129 static unsigned int mlx5_esw_chains_get_level_range(struct mlx5_eswitch *esw)
113130 {
131+ if (fdb_ignore_flow_level_supported(esw))
132+ return UINT_MAX;
133+
114134 return FDB_TC_LEVELS_PER_PRIO;
115135 }
116136
@@ -181,13 +201,40 @@ mlx5_esw_chains_create_fdb_table(struct mlx5_eswitch *esw, · 64 unmodified lines
181201 sz = mlx5_esw_chains_get_avail_sz_from_pool(esw, POOL_NEXT_SIZE);
182202 if (!sz)
183203 return ERR_PTR(-ENOSPC);
184−
185204 ft_attr.max_fte = sz;
186− ft_attr.level = level;
187− ft_attr.prio = prio - 1;
188− ft_attr.autogroup.max_num_groups = ESW_OFFLOADS_NUM_GROUPS;
189− ns = mlx5_get_fdb_sub_ns(esw->dev, chain);
190205
206+ /* We use tc_slow_fdb(esw) as the table's next_ft till
207+ * ignore_flow_level is allowed on FT creation and not just for FTEs.
208+ * Instead caller should add an explicit miss rule if needed.
209+ */
210+ ft_attr.next_ft = tc_slow_fdb(esw);
211+
212+ /* The root table(chain 0, prio 1, level 0) is required to be
213+ * connected to the previous prio (FDB_BYPASS_PATH if exists).
214+ * We always create it, as a managed table, in order to align with
215+ * fs_core logic.
216+ */
217+ if (!fdb_ignore_flow_level_supported(esw) ||
218+ (chain == 0 && prio == 1 && level == 0)) {
219+ ft_attr.level = level;
220+ ft_attr.prio = prio - 1;
221+ ns = mlx5_get_fdb_sub_ns(esw->dev, chain);
222+ } else {
223+ ft_attr.flags |= MLX5_FLOW_TABLE_UNMANAGED;
224+ ft_attr.prio = FDB_TC_OFFLOAD;
225+ /* Firmware doesn't allow us to create another level 0 table,
226+ * so we create all unmanaged tables as level 1.
227+ *
228+ * To connect them, we use explicit miss rules with
229+ * ignore_flow_level. Caller is responsible to create
230+ * these rules (if needed).
231+ */
232+ ft_attr.level = 1;
233+ ns = mlx5_get_flow_namespace(esw->dev, MLX5_FLOW_NAMESPACE_FDB);
234+ }
235+
236+ ft_attr.autogroup.num_reserved_entries = 2;
237+ ft_attr.autogroup.max_num_groups = ESW_OFFLOADS_NUM_GROUPS;
191238 fdb = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
192239 if (IS_ERR(fdb)) {
193240 esw_warn(esw->dev,
@@ -220,6 +267,7 @@ mlx5_esw_chains_create_fdb_chain(struct mlx5_eswitch *esw, u32 chain) · 26 unmodified lines
220267
221268 fdb_chain->esw = esw;
222269 fdb_chain->chain = chain;
270+ INIT_LIST_HEAD(&fdb_chain->prios_list);
223271
224272 err = rhashtable_insert_fast(&esw_chains_ht(esw), &fdb_chain->node,
225273 chain_params);
@@ -261,6 +309,79 @@ mlx5_esw_chains_get_fdb_chain(struct mlx5_eswitch *esw, u32 chain) · 35 unmodified lines
261309 return fdb_chain;
262310 }
263311
312+static struct mlx5_flow_handle *
313+mlx5_esw_chains_add_miss_rule(struct mlx5_flow_table *fdb,
314+ struct mlx5_flow_table *next_fdb)
315+{
316+ static const struct mlx5_flow_spec spec = {};
317+ struct mlx5_flow_destination dest = {};
318+ struct mlx5_flow_act act = {};
319+
320+ act.flags = FLOW_ACT_IGNORE_FLOW_LEVEL | FLOW_ACT_NO_APPEND;
321+ act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
322+ dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
323+ dest.ft = next_fdb;
324+
325+ return mlx5_add_flow_rules(fdb, &spec, &act, &dest, 1);
326+}
327+
328+static int
329+mlx5_esw_chains_update_prio_prevs(struct fdb_prio *fdb_prio,
330+ struct mlx5_flow_table *next_fdb)
331+{
332+ struct mlx5_flow_handle *miss_rules[FDB_TC_LEVELS_PER_PRIO + 1] = {};
333+ struct fdb_chain *fdb_chain = fdb_prio->fdb_chain;
334+ struct fdb_prio *pos;
335+ int n = 0, err;
336+
337+ if (fdb_prio->key.level)
338+ return 0;
339+
340+ /* Iterate in reverse order until reaching the level 0 rule of
341+ * the previous priority, adding all the miss rules first, so we can
342+ * revert them if any of them fails.
343+ */
344+ pos = fdb_prio;
345+ list_for_each_entry_continue_reverse(pos,
346+ &fdb_chain->prios_list,
347+ list) {
348+ miss_rules[n] = mlx5_esw_chains_add_miss_rule(pos->fdb,
349+ next_fdb);
350+ if (IS_ERR(miss_rules[n])) {
351+ err = PTR_ERR(miss_rules[n]);
352+ goto err_prev_rule;
353+ }
354+
355+ n++;
356+ if (!pos->key.level)
357+ break;
358+ }
359+
360+ /* Success, delete old miss rules, and update the pointers. */
361+ n = 0;
362+ pos = fdb_prio;
363+ list_for_each_entry_continue_reverse(pos,
364+ &fdb_chain->prios_list,
365+ list) {
366+ mlx5_del_flow_rules(pos->miss_rule);
367+
368+ pos->miss_rule = miss_rules[n];
369+ pos->next_fdb = next_fdb;
370+
371+ n++;
372+ if (!pos->key.level)
373+ break;
374+ }
375+
376+ return 0;
377+
378+err_prev_rule:
379+ while (--n >= 0)
380+ mlx5_del_flow_rules(miss_rules[n]);
381+
382+ return err;
383+}
384+
264385 static void
265386 mlx5_esw_chains_put_fdb_chain(struct fdb_chain *fdb_chain)
266387 {
@@ -272,9 +393,15 @@ static struct fdb_prio * · 5 unmodified lines
272393 mlx5_esw_chains_create_fdb_prio(struct mlx5_eswitch *esw,
273394 u32 chain, u32 prio, u32 level)
274395 {
396+ int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
397+ struct mlx5_flow_handle *miss_rule = NULL;
398+ struct mlx5_flow_group *miss_group;
275399 struct fdb_prio *fdb_prio = NULL;
400+ struct mlx5_flow_table *next_fdb;
276401 struct fdb_chain *fdb_chain;
277402 struct mlx5_flow_table *fdb;
403+ struct list_head *pos;
404+ u32 *flow_group_in;
278405 int err;
279406
280407 fdb_chain = mlx5_esw_chains_get_fdb_chain(esw, chain);
@@ -282,18 +409,65 @@ mlx5_esw_chains_create_fdb_prio(struct mlx5_eswitch *esw, · 1 unmodified line
282409 return ERR_CAST(fdb_chain);
283410
284411 fdb_prio = kvzalloc(sizeof(*fdb_prio), GFP_KERNEL);
285− if (!fdb_prio) {
412+ flow_group_in = kvzalloc(inlen, GFP_KERNEL);
413+ if (!fdb_prio || !flow_group_in) {
286414 err = -ENOMEM;
287415 goto err_alloc;
288416 }
289417
290− fdb = mlx5_esw_chains_create_fdb_table(esw, fdb_chain->chain, prio,
291− level);
418+ /* Chain's prio list is sorted by prio and level.
419+ * And all levels of some prio point to the next prio's level 0.
420+ * Example list (prio, level):
421+ * (3,0)->(3,1)->(5,0)->(5,1)->(6,1)->(7,0)
422+ * In hardware, we will we have the following pointers:
423+ * (3,0) -> (5,0) -> (7,0) -> Slow path
424+ * (3,1) -> (5,0)
425+ * (5,1) -> (7,0)
426+ * (6,1) -> (7,0)
427+ */
428+
429+ /* Default miss for each chain: */
430+ next_fdb = (chain == mlx5_esw_chains_get_ft_chain(esw)) ?
431+ tc_slow_fdb(esw) :
432+ tc_end_fdb(esw);
433+ list_for_each(pos, &fdb_chain->prios_list) {
434+ struct fdb_prio *p = list_entry(pos, struct fdb_prio, list);
435+
436+ /* exit on first pos that is larger */
437+ if (prio < p->key.prio || (prio == p->key.prio &&
438+ level < p->key.level)) {
439+ /* Get next level 0 table */
440+ next_fdb = p->key.level == 0 ? p->fdb : p->next_fdb;
441+ break;
442+ }
443+ }
444+
445+ fdb = mlx5_esw_chains_create_fdb_table(esw, chain, prio, level);
292446 if (IS_ERR(fdb)) {
293447 err = PTR_ERR(fdb);
294448 goto err_create;
295449 }
296450
451+ MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index,
452+ fdb->max_fte - 2);
453+ MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
454+ fdb->max_fte - 1);
455+ miss_group = mlx5_create_flow_group(fdb, flow_group_in);
456+ if (IS_ERR(miss_group)) {
457+ err = PTR_ERR(miss_group);
458+ goto err_group;
459+ }
460+
461+ /* Add miss rule to next_fdb */
462+ miss_rule = mlx5_esw_chains_add_miss_rule(fdb, next_fdb);
463+ if (IS_ERR(miss_rule)) {
464+ err = PTR_ERR(miss_rule);
465+ goto err_miss_rule;
466+ }
467+
468+ fdb_prio->miss_group = miss_group;
469+ fdb_prio->miss_rule = miss_rule;
470+ fdb_prio->next_fdb = next_fdb;
297471 fdb_prio->fdb_chain = fdb_chain;
298472 fdb_prio->key.chain = chain;
299473 fdb_prio->key.prio = prio;
@@ -305,13 +479,30 @@ mlx5_esw_chains_create_fdb_prio(struct mlx5_eswitch *esw, · 5 unmodified lines
305479 if (err)
306480 goto err_insert;
307481
482+ list_add(&fdb_prio->list, pos->prev);
483+
484+ /* Table is ready, connect it */
485+ err = mlx5_esw_chains_update_prio_prevs(fdb_prio, fdb);
486+ if (err)
487+ goto err_update;
488+
489+ kvfree(flow_group_in);
308490 return fdb_prio;
309491
492+err_update:
493+ list_del(&fdb_prio->list);
494+ rhashtable_remove_fast(&esw_prios_ht(esw), &fdb_prio->node,
495+ prio_params);
310496 err_insert:
497+ mlx5_del_flow_rules(miss_rule);
498+err_miss_rule:
499+ mlx5_destroy_flow_group(miss_group);
500+err_group:
311501 mlx5_esw_chains_destroy_fdb_table(esw, fdb);
312502 err_create:
313− kvfree(fdb_prio);
314503 err_alloc:
504+ kvfree(fdb_prio);
505+ kvfree(flow_group_in);
315506 mlx5_esw_chains_put_fdb_chain(fdb_chain);
316507 return ERR_PTR(err);
317508 }
@@ -322,8 +513,14 @@ mlx5_esw_chains_destroy_fdb_prio(struct mlx5_eswitch *esw, · 4 unmodified lines
322513 {
323514 struct fdb_chain *fdb_chain = fdb_prio->fdb_chain;
324515
516+ WARN_ON(mlx5_esw_chains_update_prio_prevs(fdb_prio,
517+ fdb_prio->next_fdb));
518+
519+ list_del(&fdb_prio->list);
325520 rhashtable_remove_fast(&esw_prios_ht(esw), &fdb_prio->node,
326521 prio_params);
522+ mlx5_del_flow_rules(fdb_prio->miss_rule);
523+ mlx5_destroy_flow_group(fdb_prio->miss_group);
327524 mlx5_esw_chains_destroy_fdb_table(esw, fdb_prio->fdb);
328525 mlx5_esw_chains_put_fdb_chain(fdb_chain);
329526 kvfree(fdb_prio);
@@ -415,6 +612,12 @@ err_get_prio: · 85 unmodified lines
415612 chain, prio, level);
416613 }
417614
615+struct mlx5_flow_table *
616+mlx5_esw_chains_get_tc_end_ft(struct mlx5_eswitch *esw)
617+{
618+ return tc_end_fdb(esw);
619+}
620+
418621 static int
419622 mlx5_esw_chains_init(struct mlx5_eswitch *esw)
420623 {
@@ -484,11 +687,21 @@ mlx5_esw_chains_open(struct mlx5_eswitch *esw) · 63 unmodified lines
484687 struct mlx5_flow_table *ft;
485688 int err;
486689
487− /* Always open the root for fast path */
488− ft = mlx5_esw_chains_get_table(esw, 0, 1, 0);
690+ /* Create tc_end_fdb(esw) which is the always created ft chain */
691+ ft = mlx5_esw_chains_get_table(esw, mlx5_esw_chains_get_ft_chain(esw),
692+ 1, 0);
489693 if (IS_ERR(ft))
490694 return PTR_ERR(ft);
491695
696+ tc_end_fdb(esw) = ft;
697+
698+ /* Always open the root for fast path */
699+ ft = mlx5_esw_chains_get_table(esw, 0, 1, 0);
700+ if (IS_ERR(ft)) {
701+ err = PTR_ERR(ft);
702+ goto level_0_err;
703+ }
704+
492705 /* Open level 1 for split rules now if prios isn't supported */
493706 if (!mlx5_esw_chains_prios_supported(esw)) {
494707 ft = mlx5_esw_chains_get_table(esw, 0, 1, 1);
@@ -503,6 +716,8 @@ mlx5_esw_chains_open(struct mlx5_eswitch *esw) · 8 unmodified lines
503716
504717 level_1_err:
505718 mlx5_esw_chains_put_table(esw, 0, 1, 0);
719+level_0_err:
720+ mlx5_esw_chains_put_table(esw, mlx5_esw_chains_get_ft_chain(esw), 1, 0);
506721 return err;
507722 }
508723
@@ -512,6 +727,7 @@ mlx5_esw_chains_close(struct mlx5_eswitch *esw) · 3 unmodified lines
512727 if (!mlx5_esw_chains_prios_supported(esw))
513728 mlx5_esw_chains_put_table(esw, 0, 1, 1);
514729 mlx5_esw_chains_put_table(esw, 0, 1, 0);
730+ mlx5_esw_chains_put_table(esw, mlx5_esw_chains_get_ft_chain(esw), 1, 0);
515731 }
516732
517733 int
drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_chains.h+3−0
@@ -20,6 +20,9 @@ void · 19 unmodified lines
2020 mlx5_esw_chains_put_table(struct mlx5_eswitch *esw, u32 chain, u32 prio,
2121 u32 level);
2222
23+struct mlx5_flow_table *
24+mlx5_esw_chains_get_tc_end_ft(struct mlx5_eswitch *esw);
25+
2326 int mlx5_esw_chains_create(struct mlx5_eswitch *esw);
2427 void mlx5_esw_chains_destroy(struct mlx5_eswitch *esw);
2528

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t