net/mlx5: E-Switch, Refactor chains and priorities

To support the entire chain and prio range (32bit + 16bit), instead of a using a static array of chains/prios of limited size, create them dynamically, and use a rhashtable to search for existing chains/prio combinations. This will be used in next patch to actually increase the number using unamanged tables support and ignore flow level capability.

Paul Blakey committed 6 years ago

Commit 39ac2371 parent 82270e17 files · +637 −292Browse files

Unified · Split

7 files

drivers/net/ethernet/mellanox/mlx5/core/Makefile +1 −1
drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +6 −5
drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +8 −6
drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +8 −22
drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +45 −258
drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_chains.c +542 −0
drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_chains.h +27 −0

drivers/net/ethernet/mellanox/mlx5/core/Makefile+1−1

@@ -42,7 +42,7 @@ mlx5_core-$(CONFIG_PCI_HYPERV_INTERFACE) += en/hv_vhca_stats.o · 41 unmodified lines
42	42	# Core extra
43	43	#
44	44	mlx5_core-$(CONFIG_MLX5_ESWITCH) += eswitch.o eswitch_offloads.o eswitch_offloads_termtbl.o \
45		− ecpf.o rdma.o
	45	+ ecpf.o rdma.o eswitch_offloads_chains.o
46	46	mlx5_core-$(CONFIG_MLX5_MPFS) += lib/mpfs.o
47	47	mlx5_core-$(CONFIG_VXLAN) += lib/vxlan.o
48	48	mlx5_core-$(CONFIG_PTP_1588_CLOCK) += lib/clock.o

drivers/net/ethernet/mellanox/mlx5/core/en_rep.c+6−5

@@ -41,6 +41,7 @@ · 40 unmodified lines
41	41	#include <net/ipv6_stubs.h>
42	42
43	43	#include "eswitch.h"
	44	+#include "eswitch_offloads_chains.h"
44	45	#include "en.h"
45	46	#include "en_rep.h"
46	47	#include "en_tc.h"
@@ -1262,25 +1263,25 @@ static int mlx5e_rep_setup_ft_cb(enum tc_setup_type type, void *type_data, · 1215 unmodified lines
1262	1263	case TC_SETUP_CLSFLOWER:
1263	1264	memcpy(&tmp, f, sizeof(*f));
1264	1265
1265		− if (!mlx5_eswitch_prios_supported(esw) \|\|
	1266	+ if (!mlx5_esw_chains_prios_supported(esw) \|\|
1266	1267	tmp.common.chain_index)
1267	1268	return -EOPNOTSUPP;
1268	1269
1269	1270	/* Re-use tc offload path by moving the ft flow to the
1270	1271	* reserved ft chain.
1271	1272	*
1272		− * FT offload can use prio range [0, INT_MAX], so we
1273		− * normalize it to range [1, mlx5_eswitch_get_prio_range(esw)]
	1273	+ * FT offload can use prio range [0, INT_MAX], so we normalize
	1274	+ * it to range [1, mlx5_esw_chains_get_prio_range(esw)]
1274	1275	* as with tc, where prio 0 isn't supported.
1275	1276	*
1276	1277	* We only support chain 0 of FT offload.
1277	1278	*/
1278		− if (tmp.common.prio >= mlx5_eswitch_get_prio_range(esw))
	1279	+ if (tmp.common.prio >= mlx5_esw_chains_get_prio_range(esw))
1279	1280	return -EOPNOTSUPP;
1280	1281	if (tmp.common.chain_index != 0)
1281	1282	return -EOPNOTSUPP;
1282	1283
1283		− tmp.common.chain_index = mlx5_eswitch_get_ft_chain(esw);
	1284	+ tmp.common.chain_index = mlx5_esw_chains_get_ft_chain(esw);
1284	1285	tmp.common.prio++;
1285	1286	err = mlx5e_rep_setup_tc_cls_flower(priv, &tmp, flags);
1286	1287	memcpy(&f->stats, &tmp.stats, sizeof(f->stats));

drivers/net/ethernet/mellanox/mlx5/core/en_tc.c+8−6

@@ -51,6 +51,7 @@ · 50 unmodified lines
51	51	#include "en_rep.h"
52	52	#include "en_tc.h"
53	53	#include "eswitch.h"
	54	+#include "eswitch_offloads_chains.h"
54	55	#include "fs_core.h"
55	56	#include "en/port.h"
56	57	#include "en/tc_tun.h"
@@ -1083,7 +1084,7 @@ mlx5e_tc_offload_to_slow_path(struct mlx5_eswitch *esw, · 1026 unmodified lines
1083	1084	memcpy(slow_attr, flow->esw_attr, sizeof(*slow_attr));
1084	1085	slow_attr->action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1085	1086	slow_attr->split_count = 0;
1086		− slow_attr->dest_chain = FDB_TC_SLOW_PATH_CHAIN;
	1087	+ slow_attr->flags \|= MLX5_ESW_ATTR_FLAG_SLOW_PATH;
1087	1088
1088	1089	rule = mlx5e_tc_offload_fdb_rules(esw, flow, spec, slow_attr);
1089	1090	if (!IS_ERR(rule))
@@ -1100,7 +1101,7 @@ mlx5e_tc_unoffload_from_slow_path(struct mlx5_eswitch *esw, · 10 unmodified lines
1100	1101	memcpy(slow_attr, flow->esw_attr, sizeof(*slow_attr));
1101	1102	slow_attr->action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1102	1103	slow_attr->split_count = 0;
1103		− slow_attr->dest_chain = FDB_TC_SLOW_PATH_CHAIN;
	1104	+ slow_attr->flags \|= MLX5_ESW_ATTR_FLAG_SLOW_PATH;
1104	1105	mlx5e_tc_unoffload_fdb_rules(esw, flow, slow_attr);
1105	1106	flow_flag_clear(flow, SLOW);
1106	1107	}
@@ -1160,19 +1161,18 @@ mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv, · 53 unmodified lines
1160	1161	struct netlink_ext_ack *extack)
1161	1162	{
1162	1163	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
1163		− u32 max_chain = mlx5_eswitch_get_chain_range(esw);
1164	1164	struct mlx5_esw_flow_attr *attr = flow->esw_attr;
1165	1165	struct mlx5e_tc_flow_parse_attr *parse_attr = attr->parse_attr;
1166		− u16 max_prio = mlx5_eswitch_get_prio_range(esw);
1167	1166	struct net_device out_dev, encap_dev = NULL;
1168	1167	struct mlx5_fc *counter = NULL;
1169	1168	struct mlx5e_rep_priv *rpriv;
1170	1169	struct mlx5e_priv *out_priv;
1171	1170	bool encap_valid = true;
	1171	+ u32 max_prio, max_chain;
1172	1172	int err = 0;
1173	1173	int out_index;
1174	1174
1175		− if (!mlx5_eswitch_prios_supported(esw) && attr->prio != 1) {
	1175	+ if (!mlx5_esw_chains_prios_supported(esw) && attr->prio != 1) {
1176	1176	NL_SET_ERR_MSG(extack, "E-switch priorities unsupported, upgrade FW");
1177	1177	return -EOPNOTSUPP;
1178	1178	}
@@ -1182,11 +1182,13 @@ mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv, · 3 unmodified lines
1182	1182	* FDB_FT_CHAIN which is outside tc range.
1183	1183	* See mlx5e_rep_setup_ft_cb().
1184	1184	*/
	1185	+ max_chain = mlx5_esw_chains_get_chain_range(esw);
1185	1186	if (!mlx5e_is_ft_flow(flow) && attr->chain > max_chain) {
1186	1187	NL_SET_ERR_MSG(extack, "Requested chain is out of supported range");
1187	1188	return -EOPNOTSUPP;
1188	1189	}
1189	1190
	1191	+ max_prio = mlx5_esw_chains_get_prio_range(esw);
1190	1192	if (attr->prio > max_prio) {
1191	1193	NL_SET_ERR_MSG(extack, "Requested priority is out of supported range");
1192	1194	return -EOPNOTSUPP;
@@ -3469,7 +3471,7 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, · 2276 unmodified lines
3469	3471	break;
3470	3472	case FLOW_ACTION_GOTO: {
3471	3473	u32 dest_chain = act->chain_index;
3472		− u32 max_chain = mlx5_eswitch_get_chain_range(esw);
	3474	+ u32 max_chain = mlx5_esw_chains_get_chain_range(esw);
3473	3475
3474	3476	if (ft_flow) {
3475	3477	NL_SET_ERR_MSG_MOD(extack, "Goto action is not supported");

drivers/net/ethernet/mellanox/mlx5/core/eswitch.h+8−22

@@ -157,7 +157,7 @@ enum offloads_fdb_flags { · 156 unmodified lines
157	157	ESW_FDB_CHAINS_AND_PRIOS_SUPPORTED = BIT(0),
158	158	};
159	159
160		−extern const unsigned int ESW_POOLS[4];
	160	+struct mlx5_esw_chains_priv;
161	161
162	162	struct mlx5_eswitch_fdb {
163	163	union {
@@ -182,14 +182,7 @@ struct mlx5_eswitch_fdb { · 18 unmodified lines
182	182	struct mlx5_flow_handle *miss_rule_multi;
183	183	int vlan_push_pop_refcount;
184	184
185		− struct {
186		− struct mlx5_flow_table *fdb;
187		− u32 num_rules;
188		− } fdb_prio[FDB_NUM_CHAINS][FDB_TC_MAX_PRIO + 1][FDB_TC_LEVELS_PER_PRIO];
189		− /* Protects fdb_prio table */
190		− struct mutex fdb_prio_lock;
191		−
192		− int fdb_left[ARRAY_SIZE(ESW_POOLS)];
	185	+ struct mlx5_esw_chains_priv *esw_chains_priv;
193	186	} offloads;
194	187	};
195	188	u32 flags;
@@ -355,18 +348,6 @@ mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch *esw, · 159 unmodified lines
355	348	struct mlx5_flow_handle *rule,
356	349	struct mlx5_esw_flow_attr *attr);
357	350
358		−bool
359		−mlx5_eswitch_prios_supported(struct mlx5_eswitch *esw);
360		−
361		−u16
362		−mlx5_eswitch_get_prio_range(struct mlx5_eswitch *esw);
363		−
364		−u32
365		−mlx5_eswitch_get_chain_range(struct mlx5_eswitch *esw);
366		−
367		−unsigned int
368		−mlx5_eswitch_get_ft_chain(struct mlx5_eswitch *esw);
369		−
370	351	struct mlx5_flow_handle *
371	352	mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport,
372	353	struct mlx5_flow_destination *dest);
@@ -391,6 +372,11 @@ enum { · 18 unmodified lines
391	372	MLX5_ESW_DEST_ENCAP_VALID = BIT(1),
392	373	};
393	374
	375	+enum {
	376	+ MLX5_ESW_ATTR_FLAG_VLAN_HANDLED = BIT(0),
	377	+ MLX5_ESW_ATTR_FLAG_SLOW_PATH = BIT(1),
	378	+};
	379	+
394	380	struct mlx5_esw_flow_attr {
395	381	struct mlx5_eswitch_rep *in_rep;
396	382	struct mlx5_core_dev *in_mdev;
@@ -404,7 +390,6 @@ struct mlx5_esw_flow_attr { · 7 unmodified lines
404	390	u16 vlan_vid[MLX5_FS_VLAN_DEPTH];
405	391	u8 vlan_prio[MLX5_FS_VLAN_DEPTH];
406	392	u8 total_vlan;
407		− bool vlan_handled;
408	393	struct {
409	394	u32 flags;
410	395	struct mlx5_eswitch_rep *rep;
@@ -419,6 +404,7 @@ struct mlx5_esw_flow_attr { · 8 unmodified lines
419	404	u32 chain;
420	405	u16 prio;
421	406	u32 dest_chain;
	407	+ u32 flags;
422	408	struct mlx5e_tc_flow_parse_attr *parse_attr;
423	409	};
424	410

drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c+45−258

@@ -37,6 +37,7 @@ · 36 unmodified lines
37	37	#include <linux/mlx5/fs.h>
38	38	#include "mlx5_core.h"
39	39	#include "eswitch.h"
	40	+#include "eswitch_offloads_chains.h"
40	41	#include "rdma.h"
41	42	#include "en.h"
42	43	#include "fs_core.h"
@@ -47,10 +48,6 @@ · 4 unmodified lines
47	48	* one for multicast.
48	49	*/
49	50	#define MLX5_ESW_MISS_FLOWS (2)
50		−
51		−#define fdb_prio_table(esw, chain, prio, level) \
52		− (esw)->fdb_table.offloads.fdb_prio[(chain)][(prio)][(level)]
53		−
54	51	#define UPLINK_REP_INDEX 0
55	52
56	53	static struct mlx5_eswitch_rep mlx5_eswitch_get_rep(struct mlx5_eswitch esw,
@@ -62,37 +59,6 @@ static struct mlx5_eswitch_rep mlx5_eswitch_get_rep(struct mlx5_eswitch esw, · 5 unmodified lines
62	59	return &esw->offloads.vport_reps[idx];
63	60	}
64	61
65		−static struct mlx5_flow_table *
66		−esw_get_prio_table(struct mlx5_eswitch *esw, u32 chain, u16 prio, int level);
67		−static void
68		−esw_put_prio_table(struct mlx5_eswitch *esw, u32 chain, u16 prio, int level);
69		−
70		−bool mlx5_eswitch_prios_supported(struct mlx5_eswitch *esw)
71		−{
72		− return (!!(esw->fdb_table.flags & ESW_FDB_CHAINS_AND_PRIOS_SUPPORTED));
73		−}
74		−
75		−u32 mlx5_eswitch_get_chain_range(struct mlx5_eswitch *esw)
76		−{
77		− if (esw->fdb_table.flags & ESW_FDB_CHAINS_AND_PRIOS_SUPPORTED)
78		− return FDB_TC_MAX_CHAIN;
79		−
80		− return 0;
81		−}
82		−
83		−u32 mlx5_eswitch_get_ft_chain(struct mlx5_eswitch *esw)
84		−{
85		− return mlx5_eswitch_get_chain_range(esw) + 1;
86		−}
87		−
88		−u16 mlx5_eswitch_get_prio_range(struct mlx5_eswitch *esw)
89		−{
90		− if (esw->fdb_table.flags & ESW_FDB_CHAINS_AND_PRIOS_SUPPORTED)
91		− return FDB_TC_MAX_PRIO;
92		−
93		− return 1;
94		−}
95		−
96	62	static bool
97	63	esw_check_ingress_prio_tag_enabled(const struct mlx5_eswitch *esw,
98	64	const struct mlx5_vport *vport)
@@ -180,10 +146,17 @@ mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, · 81 unmodified lines
180	146	}
181	147
182	148	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
183		− if (attr->dest_chain) {
184		− struct mlx5_flow_table *ft;
	149	+ struct mlx5_flow_table *ft;
185	150
186		− ft = esw_get_prio_table(esw, attr->dest_chain, 1, 0);
	151	+ if (attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH) {
	152	+ flow_act.flags \|= FLOW_ACT_IGNORE_FLOW_LEVEL;
	153	+ dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
	154	+ dest[i].ft = esw->fdb_table.offloads.slow_fdb;
	155	+ i++;
	156	+ } else if (attr->dest_chain) {
	157	+ flow_act.flags \|= FLOW_ACT_IGNORE_FLOW_LEVEL;
	158	+ ft = mlx5_esw_chains_get_table(esw, attr->dest_chain,
	159	+ 1, 0);
187	160	if (IS_ERR(ft)) {
188	161	rule = ERR_CAST(ft);
189	162	goto err_create_goto_table;
@@ -228,7 +201,8 @@ mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, · 38 unmodified lines
228	201	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
229	202	flow_act.modify_hdr = attr->modify_hdr;
230	203
231		− fdb = esw_get_prio_table(esw, attr->chain, attr->prio, !!split);
	204	+ fdb = mlx5_esw_chains_get_table(esw, attr->chain, attr->prio,
	205	+ !!split);
232	206	if (IS_ERR(fdb)) {
233	207	rule = ERR_CAST(fdb);
234	208	goto err_esw_get;
@@ -247,10 +221,10 @@ mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, · 12 unmodified lines
247	221	return rule;
248	222
249	223	err_add_rule:
250		− esw_put_prio_table(esw, attr->chain, attr->prio, !!split);
	224	+ mlx5_esw_chains_put_table(esw, attr->chain, attr->prio, !!split);
251	225	err_esw_get:
252		− if (attr->dest_chain)
253		− esw_put_prio_table(esw, attr->dest_chain, 1, 0);
	226	+ if (!(attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH) && attr->dest_chain)
	227	+ mlx5_esw_chains_put_table(esw, attr->dest_chain, 1, 0);
254	228	err_create_goto_table:
255	229	return rule;
256	230	}
@@ -267,13 +241,13 @@ mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw, · 10 unmodified lines
267	241	struct mlx5_flow_handle *rule;
268	242	int i;
269	243
270		− fast_fdb = esw_get_prio_table(esw, attr->chain, attr->prio, 0);
	244	+ fast_fdb = mlx5_esw_chains_get_table(esw, attr->chain, attr->prio, 0);
271	245	if (IS_ERR(fast_fdb)) {
272	246	rule = ERR_CAST(fast_fdb);
273	247	goto err_get_fast;
274	248	}
275	249
276		− fwd_fdb = esw_get_prio_table(esw, attr->chain, attr->prio, 1);
	250	+ fwd_fdb = mlx5_esw_chains_get_table(esw, attr->chain, attr->prio, 1);
277	251	if (IS_ERR(fwd_fdb)) {
278	252	rule = ERR_CAST(fwd_fdb);
279	253	goto err_get_fwd;
@@ -310,9 +284,9 @@ mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw, · 30 unmodified lines
310	284
311	285	return rule;
312	286	add_err:
313		− esw_put_prio_table(esw, attr->chain, attr->prio, 1);
	287	+ mlx5_esw_chains_put_table(esw, attr->chain, attr->prio, 1);
314	288	err_get_fwd:
315		− esw_put_prio_table(esw, attr->chain, attr->prio, 0);
	289	+ mlx5_esw_chains_put_table(esw, attr->chain, attr->prio, 0);
316	290	err_get_fast:
317	291	return rule;
318	292	}
@@ -337,12 +311,13 @@ __mlx5_eswitch_del_rule(struct mlx5_eswitch *esw, · 18 unmodified lines
337	311	atomic64_dec(&esw->offloads.num_flows);
338	312
339	313	if (fwd_rule) {
340		− esw_put_prio_table(esw, attr->chain, attr->prio, 1);
341		− esw_put_prio_table(esw, attr->chain, attr->prio, 0);
	314	+ mlx5_esw_chains_put_table(esw, attr->chain, attr->prio, 1);
	315	+ mlx5_esw_chains_put_table(esw, attr->chain, attr->prio, 0);
342	316	} else {
343		− esw_put_prio_table(esw, attr->chain, attr->prio, !!split);
	317	+ mlx5_esw_chains_put_table(esw, attr->chain, attr->prio,
	318	+ !!split);
344	319	if (attr->dest_chain)
345		− esw_put_prio_table(esw, attr->dest_chain, 1, 0);
	320	+ mlx5_esw_chains_put_table(esw, attr->dest_chain, 1, 0);
346	321	}
347	322	}
348	323
@@ -456,7 +431,7 @@ int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch *esw, · 107 unmodified lines
456	431	if (err)
457	432	goto unlock;
458	433
459		− attr->vlan_handled = false;
	434	+ attr->flags &= ~MLX5_ESW_ATTR_FLAG_VLAN_HANDLED;
460	435
461	436	vport = esw_vlan_action_get_vport(attr, push, pop);
462	437
@@ -464,7 +439,7 @@ int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch *esw, · 1 unmodified line
464	439	/* tracks VF --> wire rules without vlan push action */
465	440	if (attr->dests[0].rep->vport == MLX5_VPORT_UPLINK) {
466	441	vport->vlan_refcount++;
467		− attr->vlan_handled = true;
	442	+ attr->flags \|= MLX5_ESW_ATTR_FLAG_VLAN_HANDLED;
468	443	}
469	444
470	445	goto unlock;
@@ -495,7 +470,7 @@ skip_set_push: · 24 unmodified lines
495	470	}
496	471	out:
497	472	if (!err)
498		− attr->vlan_handled = true;
	473	+ attr->flags \|= MLX5_ESW_ATTR_FLAG_VLAN_HANDLED;
499	474	unlock:
500	475	mutex_unlock(&esw->state_lock);
501	476	return err;
@@ -513,7 +488,7 @@ int mlx5_eswitch_del_vlan_action(struct mlx5_eswitch *esw, · 11 unmodified lines
513	488	if (mlx5_eswitch_vlan_actions_supported(esw->dev, 1))
514	489	return 0;
515	490
516		− if (!attr->vlan_handled)
	491	+ if (!(attr->flags & MLX5_ESW_ATTR_FLAG_VLAN_HANDLED))
517	492	return 0;
518	493
519	494	push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH);
@@ -587,8 +562,8 @@ mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *esw, u16 vport, · 67 unmodified lines
587	562	dest.vport.num = vport;
588	563	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
589	564
590		− flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, spec,
591		− &flow_act, &dest, 1);
	565	+ flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
	566	+ spec, &flow_act, &dest, 1);
592	567	if (IS_ERR(flow_rule))
593	568	esw_warn(esw->dev, "FDB: Failed to add send to vport rule err %ld\n", PTR_ERR(flow_rule));
594	569	out:
@@ -829,8 +804,8 @@ static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw) · 234 unmodified lines
829	804	dest.vport.num = esw->manager_vport;
830	805	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
831	806
832		− flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, spec,
833		− &flow_act, &dest, 1);
	807	+ flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
	808	+ spec, &flow_act, &dest, 1);
834	809	if (IS_ERR(flow_rule)) {
835	810	err = PTR_ERR(flow_rule);
836	811	esw_warn(esw->dev, "FDB: Failed to add unicast miss flow rule err %d\n", err);
@@ -844,8 +819,8 @@ static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw) · 7 unmodified lines
844	819	dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v,
845	820	outer_headers.dmac_47_16);
846	821	dmac_v[0] = 0x01;
847		− flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, spec,
848		− &flow_act, &dest, 1);
	822	+ flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
	823	+ spec, &flow_act, &dest, 1);
849	824	if (IS_ERR(flow_rule)) {
850	825	err = PTR_ERR(flow_rule);
851	826	esw_warn(esw->dev, "FDB: Failed to add multicast miss flow rule err %d\n", err);
@@ -860,175 +835,6 @@ out: · 8 unmodified lines
860	835	return err;
861	836	}
862	837
863		−#define ESW_OFFLOADS_NUM_GROUPS 4
864		−
865		−/* Firmware currently has 4 pool of 4 sizes that it supports (ESW_POOLS),
866		− * and a virtual memory region of 16M (ESW_SIZE), this region is duplicated
867		− * for each flow table pool. We can allocate up to 16M of each pool,
868		− * and we keep track of how much we used via put/get_sz_to_pool.
869		− * Firmware doesn't report any of this for now.
870		− * ESW_POOL is expected to be sorted from large to small
871		− */
872		−#define ESW_SIZE (16 * 1024 * 1024)
873		−const unsigned int ESW_POOLS[4] = { 4 * 1024 * 1024, 1 * 1024 * 1024,
874		− 64 * 1024, 4 * 1024 };
875		−
876		−static int
877		−get_sz_from_pool(struct mlx5_eswitch *esw)
878		−{
879		− int sz = 0, i;
880		−
881		− for (i = 0; i < ARRAY_SIZE(ESW_POOLS); i++) {
882		− if (esw->fdb_table.offloads.fdb_left[i]) {
883		− --esw->fdb_table.offloads.fdb_left[i];
884		− sz = ESW_POOLS[i];
885		− break;
886		− }
887		− }
888		−
889		− return sz;
890		−}
891		−
892		−static void
893		−put_sz_to_pool(struct mlx5_eswitch *esw, int sz)
894		−{
895		− int i;
896		−
897		− for (i = 0; i < ARRAY_SIZE(ESW_POOLS); i++) {
898		− if (sz >= ESW_POOLS[i]) {
899		− ++esw->fdb_table.offloads.fdb_left[i];
900		− break;
901		− }
902		− }
903		−}
904		−
905		−static struct mlx5_flow_table *
906		−create_next_size_table(struct mlx5_eswitch *esw,
907		− struct mlx5_flow_namespace *ns,
908		− u16 table_prio,
909		− int level,
910		− u32 flags)
911		−{
912		− struct mlx5_flow_table_attr ft_attr = {};
913		− struct mlx5_flow_table *fdb;
914		− int sz;
915		−
916		− sz = get_sz_from_pool(esw);
917		− if (!sz)
918		− return ERR_PTR(-ENOSPC);
919		−
920		− ft_attr.max_fte = sz;
921		− ft_attr.prio = table_prio;
922		− ft_attr.level = level;
923		− ft_attr.flags = flags;
924		− ft_attr.autogroup.max_num_groups = ESW_OFFLOADS_NUM_GROUPS;
925		− fdb = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
926		− if (IS_ERR(fdb)) {
927		− esw_warn(esw->dev, "Failed to create FDB Table err %d (table prio: %d, level: %d, size: %d)\n",
928		− (int)PTR_ERR(fdb), table_prio, level, sz);
929		− put_sz_to_pool(esw, sz);
930		− }
931		−
932		− return fdb;
933		−}
934		−
935		−static struct mlx5_flow_table *
936		−esw_get_prio_table(struct mlx5_eswitch *esw, u32 chain, u16 prio, int level)
937		−{
938		− struct mlx5_core_dev *dev = esw->dev;
939		− struct mlx5_flow_table *fdb = NULL;
940		− struct mlx5_flow_namespace *ns;
941		− int table_prio, l = 0;
942		− u32 flags = 0;
943		−
944		− if (chain == FDB_TC_SLOW_PATH_CHAIN)
945		− return esw->fdb_table.offloads.slow_fdb;
946		−
947		− mutex_lock(&esw->fdb_table.offloads.fdb_prio_lock);
948		−
949		− fdb = fdb_prio_table(esw, chain, prio, level).fdb;
950		− if (fdb) {
951		− /* take ref on earlier levels as well */
952		− while (level >= 0)
953		− fdb_prio_table(esw, chain, prio, level--).num_rules++;
954		− mutex_unlock(&esw->fdb_table.offloads.fdb_prio_lock);
955		− return fdb;
956		− }
957		−
958		− ns = mlx5_get_fdb_sub_ns(dev, chain);
959		− if (!ns) {
960		− esw_warn(dev, "Failed to get FDB sub namespace\n");
961		− mutex_unlock(&esw->fdb_table.offloads.fdb_prio_lock);
962		− return ERR_PTR(-EOPNOTSUPP);
963		− }
964		−
965		− if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
966		− flags \|= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT \|
967		− MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
968		−
969		− table_prio = prio - 1;
970		−
971		− /* create earlier levels for correct fs_core lookup when
972		− * connecting tables
973		− */
974		− for (l = 0; l <= level; l++) {
975		− if (fdb_prio_table(esw, chain, prio, l).fdb) {
976		− fdb_prio_table(esw, chain, prio, l).num_rules++;
977		− continue;
978		− }
979		−
980		− fdb = create_next_size_table(esw, ns, table_prio, l, flags);
981		− if (IS_ERR(fdb)) {
982		− l--;
983		− goto err_create_fdb;
984		− }
985		−
986		− fdb_prio_table(esw, chain, prio, l).fdb = fdb;
987		− fdb_prio_table(esw, chain, prio, l).num_rules = 1;
988		− }
989		−
990		− mutex_unlock(&esw->fdb_table.offloads.fdb_prio_lock);
991		− return fdb;
992		−
993		−err_create_fdb:
994		− mutex_unlock(&esw->fdb_table.offloads.fdb_prio_lock);
995		− if (l >= 0)
996		− esw_put_prio_table(esw, chain, prio, l);
997		−
998		− return fdb;
999		−}
1000		−
1001		−static void
1002		−esw_put_prio_table(struct mlx5_eswitch *esw, u32 chain, u16 prio, int level)
1003		−{
1004		− int l;
1005		−
1006		− if (chain == FDB_TC_SLOW_PATH_CHAIN)
1007		− return;
1008		−
1009		− mutex_lock(&esw->fdb_table.offloads.fdb_prio_lock);
1010		−
1011		− for (l = level; l >= 0; l--) {
1012		− if (--(fdb_prio_table(esw, chain, prio, l).num_rules) > 0)
1013		− continue;
1014		−
1015		− put_sz_to_pool(esw, fdb_prio_table(esw, chain, prio, l).fdb->max_fte);
1016		− mlx5_destroy_flow_table(fdb_prio_table(esw, chain, prio, l).fdb);
1017		− fdb_prio_table(esw, chain, prio, l).fdb = NULL;
1018		− }
1019		−
1020		− mutex_unlock(&esw->fdb_table.offloads.fdb_prio_lock);
1021		−}
1022		−
1023		−static void esw_destroy_offloads_fast_fdb_tables(struct mlx5_eswitch *esw)
1024		−{
1025		− /* If lazy creation isn't supported, deref the fast path tables */
1026		− if (!(esw->fdb_table.flags & ESW_FDB_CHAINS_AND_PRIOS_SUPPORTED)) {
1027		− esw_put_prio_table(esw, 0, 1, 1);
1028		− esw_put_prio_table(esw, 0, 1, 0);
1029		− }
1030		−}
1031		−
1032	838	#define MAX_PF_SQ 256
1033	839	#define MAX_SQ_NVPORTS 32
1034	840
@@ -1061,16 +867,16 @@ static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw, int nvports) · 26 unmodified lines
1061	867	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1062	868	struct mlx5_flow_table_attr ft_attr = {};
1063	869	struct mlx5_core_dev *dev = esw->dev;
1064		− u32 *flow_group_in, max_flow_counter;
1065	870	struct mlx5_flow_namespace *root_ns;
1066	871	struct mlx5_flow_table *fdb = NULL;
1067		− int table_size, ix, err = 0, i;
	872	+ u32 flags = 0, *flow_group_in;
	873	+ int table_size, ix, err = 0;
1068	874	struct mlx5_flow_group *g;
1069		− u32 flags = 0, fdb_max;
1070	875	void *match_criteria;
1071	876	u8 *dmac;
1072	877
1073	878	esw_debug(esw->dev, "Create offloads FDB Tables\n");
	879	+
1074	880	flow_group_in = kvzalloc(inlen, GFP_KERNEL);
1075	881	if (!flow_group_in)
1076	882	return -ENOMEM;
@@ -1089,19 +895,6 @@ static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw, int nvports) · 12 unmodified lines
1089	895	goto ns_err;
1090	896	}
1091	897
1092		− max_flow_counter = (MLX5_CAP_GEN(dev, max_flow_counter_31_16) << 16) \|
1093		− MLX5_CAP_GEN(dev, max_flow_counter_15_0);
1094		− fdb_max = 1 << MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size);
1095		−
1096		− esw_debug(dev, "Create offloads FDB table, min (max esw size(2^%d), max counters(%d), groups(%d), max flow table size(%d))\n",
1097		− MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size),
1098		− max_flow_counter, ESW_OFFLOADS_NUM_GROUPS,
1099		− fdb_max);
1100		−
1101		− for (i = 0; i < ARRAY_SIZE(ESW_POOLS); i++)
1102		− esw->fdb_table.offloads.fdb_left[i] =
1103		− ESW_POOLS[i] <= fdb_max ? ESW_SIZE / ESW_POOLS[i] : 0;
1104		−
1105	898	table_size = nvports * MAX_SQ_NVPORTS + MAX_PF_SQ +
1106	899	MLX5_ESW_MISS_FLOWS + esw->total_vports;
1107	900
@@ -1124,16 +917,10 @@ static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw, int nvports) · 16 unmodified lines
1124	917	}
1125	918	esw->fdb_table.offloads.slow_fdb = fdb;
1126	919
1127		− /* If lazy creation isn't supported, open the fast path tables now */
1128		− if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, multi_fdb_encap) &&
1129		− esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) {
1130		− esw->fdb_table.flags &= ~ESW_FDB_CHAINS_AND_PRIOS_SUPPORTED;
1131		− esw_warn(dev, "Lazy creation of flow tables isn't supported, ignoring priorities\n");
1132		− esw_get_prio_table(esw, 0, 1, 0);
1133		− esw_get_prio_table(esw, 0, 1, 1);
1134		− } else {
1135		− esw_debug(dev, "Lazy creation of flow tables supported, deferring table opening\n");
1136		− esw->fdb_table.flags \|= ESW_FDB_CHAINS_AND_PRIOS_SUPPORTED;
	920	+ err = mlx5_esw_chains_create(esw);
	921	+ if (err) {
	922	+ esw_warn(dev, "Failed to create fdb chains err(%d)\n", err);
	923	+ goto fdb_chains_err;
1137	924	}
1138	925
1139	926	/* create send-to-vport group */
@@ -1224,7 +1011,8 @@ miss_err: · 84 unmodified lines
1224	1011	peer_miss_err:
1225	1012	mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);
1226	1013	send_vport_err:
1227		− esw_destroy_offloads_fast_fdb_tables(esw);
	1014	+ mlx5_esw_chains_destroy(esw);
	1015	+fdb_chains_err:
1228	1016	mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb);
1229	1017	slow_fdb_err:
1230	1018	/* Holds true only as long as DMFS is the default */
@@ -1246,8 +1034,8 @@ static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw) · 15 unmodified lines
1246	1034	mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp);
1247	1035	mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);
1248	1036
	1037	+ mlx5_esw_chains_destroy(esw);
1249	1038	mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb);
1250		− esw_destroy_offloads_fast_fdb_tables(esw);
1251	1039	/* Holds true only as long as DMFS is the default */
1252	1040	mlx5_flow_namespace_set_mode(esw->fdb_table.offloads.ns,
1253	1041	MLX5_FLOW_STEERING_MODE_DMFS);
@@ -2117,7 +1905,6 @@ static int esw_offloads_steering_init(struct mlx5_eswitch *esw) · 863 unmodified lines
2117	1905	total_vports = num_vfs + MLX5_SPECIAL_VPORTS(esw->dev);
2118	1906
2119	1907	memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb));
2120		− mutex_init(&esw->fdb_table.offloads.fdb_prio_lock);
2121	1908
2122	1909	err = esw_create_uplink_offloads_acl_tables(esw);
2123	1910	if (err)

drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_chains.c+542−0

@@ -0,0 +1,542 @@
	1	+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
	2	+// Copyright (c) 2020 Mellanox Technologies.
	3	+
	4	+#include <linux/mlx5/driver.h>
	5	+#include <linux/mlx5/mlx5_ifc.h>
	6	+#include <linux/mlx5/fs.h>
	7	+
	8	+#include "eswitch_offloads_chains.h"
	9	+#include "mlx5_core.h"
	10	+#include "fs_core.h"
	11	+#include "eswitch.h"
	12	+#include "en.h"
	13	+
	14	+#define esw_chains_priv(esw) ((esw)->fdb_table.offloads.esw_chains_priv)
	15	+#define esw_chains_lock(esw) (esw_chains_priv(esw)->lock)
	16	+#define esw_chains_ht(esw) (esw_chains_priv(esw)->chains_ht)
	17	+#define esw_prios_ht(esw) (esw_chains_priv(esw)->prios_ht)
	18	+#define fdb_pool_left(esw) (esw_chains_priv(esw)->fdb_left)
	19	+
	20	+#define ESW_OFFLOADS_NUM_GROUPS 4
	21	+
	22	+/* Firmware currently has 4 pool of 4 sizes that it supports (ESW_POOLS),
	23	+ * and a virtual memory region of 16M (ESW_SIZE), this region is duplicated
	24	+ * for each flow table pool. We can allocate up to 16M of each pool,
	25	+ * and we keep track of how much we used via get_next_avail_sz_from_pool.
	26	+ * Firmware doesn't report any of this for now.
	27	+ * ESW_POOL is expected to be sorted from large to small and match firmware
	28	+ * pools.
	29	+ */
	30	+#define ESW_SIZE (16 * 1024 * 1024)
	31	+const unsigned int ESW_POOLS[] = { 4 * 1024 * 1024,
	32	+ 1 * 1024 * 1024,
	33	+ 64 * 1024,
	34	+ 4 * 1024, };
	35	+
	36	+struct mlx5_esw_chains_priv {
	37	+ struct rhashtable chains_ht;
	38	+ struct rhashtable prios_ht;
	39	+ /* Protects above chains_ht and prios_ht */
	40	+ struct mutex lock;
	41	+
	42	+ int fdb_left[ARRAY_SIZE(ESW_POOLS)];
	43	+};
	44	+
	45	+struct fdb_chain {
	46	+ struct rhash_head node;
	47	+
	48	+ u32 chain;
	49	+
	50	+ int ref;
	51	+
	52	+ struct mlx5_eswitch *esw;
	53	+};
	54	+
	55	+struct fdb_prio_key {
	56	+ u32 chain;
	57	+ u32 prio;
	58	+ u32 level;
	59	+};
	60	+
	61	+struct fdb_prio {
	62	+ struct rhash_head node;
	63	+
	64	+ struct fdb_prio_key key;
	65	+
	66	+ int ref;
	67	+
	68	+ struct fdb_chain *fdb_chain;
	69	+ struct mlx5_flow_table *fdb;
	70	+};
	71	+
	72	+static const struct rhashtable_params chain_params = {
	73	+ .head_offset = offsetof(struct fdb_chain, node),
	74	+ .key_offset = offsetof(struct fdb_chain, chain),
	75	+ .key_len = sizeof_field(struct fdb_chain, chain),
	76	+ .automatic_shrinking = true,
	77	+};
	78	+
	79	+static const struct rhashtable_params prio_params = {
	80	+ .head_offset = offsetof(struct fdb_prio, node),
	81	+ .key_offset = offsetof(struct fdb_prio, key),
	82	+ .key_len = sizeof_field(struct fdb_prio, key),
	83	+ .automatic_shrinking = true,
	84	+};
	85	+
	86	+bool mlx5_esw_chains_prios_supported(struct mlx5_eswitch *esw)
	87	+{
	88	+ return esw->fdb_table.flags & ESW_FDB_CHAINS_AND_PRIOS_SUPPORTED;
	89	+}
	90	+
	91	+u32 mlx5_esw_chains_get_chain_range(struct mlx5_eswitch *esw)
	92	+{
	93	+ if (!mlx5_esw_chains_prios_supported(esw))
	94	+ return 1;
	95	+
	96	+ return FDB_TC_MAX_CHAIN;
	97	+}
	98	+
	99	+u32 mlx5_esw_chains_get_ft_chain(struct mlx5_eswitch *esw)
	100	+{
	101	+ return mlx5_esw_chains_get_chain_range(esw) + 1;
	102	+}
	103	+
	104	+u32 mlx5_esw_chains_get_prio_range(struct mlx5_eswitch *esw)
	105	+{
	106	+ if (!mlx5_esw_chains_prios_supported(esw))
	107	+ return 1;
	108	+
	109	+ return FDB_TC_MAX_PRIO;
	110	+}
	111	+
	112	+static unsigned int mlx5_esw_chains_get_level_range(struct mlx5_eswitch *esw)
	113	+{
	114	+ return FDB_TC_LEVELS_PER_PRIO;
	115	+}
	116	+
	117	+#define POOL_NEXT_SIZE 0
	118	+static int
	119	+mlx5_esw_chains_get_avail_sz_from_pool(struct mlx5_eswitch *esw,
	120	+ int desired_size)
	121	+{
	122	+ int i, found_i = -1;
	123	+
	124	+ for (i = ARRAY_SIZE(ESW_POOLS) - 1; i >= 0; i--) {
	125	+ if (fdb_pool_left(esw)[i] && ESW_POOLS[i] > desired_size) {
	126	+ found_i = i;
	127	+ if (desired_size != POOL_NEXT_SIZE)
	128	+ break;
	129	+ }
	130	+ }
	131	+
	132	+ if (found_i != -1) {
	133	+ --fdb_pool_left(esw)[found_i];
	134	+ return ESW_POOLS[found_i];
	135	+ }
	136	+
	137	+ return 0;
	138	+}
	139	+
	140	+static void
	141	+mlx5_esw_chains_put_sz_to_pool(struct mlx5_eswitch *esw, int sz)
	142	+{
	143	+ int i;
	144	+
	145	+ for (i = ARRAY_SIZE(ESW_POOLS) - 1; i >= 0; i--) {
	146	+ if (sz == ESW_POOLS[i]) {
	147	+ ++fdb_pool_left(esw)[i];
	148	+ return;
	149	+ }
	150	+ }
	151	+
	152	+ WARN_ONCE(1, "Couldn't find size %d in fdb size pool", sz);
	153	+}
	154	+
	155	+static void
	156	+mlx5_esw_chains_init_sz_pool(struct mlx5_eswitch *esw)
	157	+{
	158	+ u32 fdb_max;
	159	+ int i;
	160	+
	161	+ fdb_max = 1 << MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, log_max_ft_size);
	162	+
	163	+ for (i = ARRAY_SIZE(ESW_POOLS) - 1; i >= 0; i--)
	164	+ fdb_pool_left(esw)[i] =
	165	+ ESW_POOLS[i] <= fdb_max ? ESW_SIZE / ESW_POOLS[i] : 0;
	166	+}
	167	+
	168	+static struct mlx5_flow_table *
	169	+mlx5_esw_chains_create_fdb_table(struct mlx5_eswitch *esw,
	170	+ u32 chain, u32 prio, u32 level)
	171	+{
	172	+ struct mlx5_flow_table_attr ft_attr = {};
	173	+ struct mlx5_flow_namespace *ns;
	174	+ struct mlx5_flow_table *fdb;
	175	+ int sz;
	176	+
	177	+ if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
	178	+ ft_attr.flags \|= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT \|
	179	+ MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
	180	+
	181	+ sz = mlx5_esw_chains_get_avail_sz_from_pool(esw, POOL_NEXT_SIZE);
	182	+ if (!sz)
	183	+ return ERR_PTR(-ENOSPC);
	184	+
	185	+ ft_attr.max_fte = sz;
	186	+ ft_attr.level = level;
	187	+ ft_attr.prio = prio - 1;
	188	+ ft_attr.autogroup.max_num_groups = ESW_OFFLOADS_NUM_GROUPS;
	189	+ ns = mlx5_get_fdb_sub_ns(esw->dev, chain);
	190	+
	191	+ fdb = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
	192	+ if (IS_ERR(fdb)) {
	193	+ esw_warn(esw->dev,
	194	+ "Failed to create FDB table err %d (chain: %d, prio: %d, level: %d, size: %d)\n",
	195	+ (int)PTR_ERR(fdb), chain, prio, level, sz);
	196	+ mlx5_esw_chains_put_sz_to_pool(esw, sz);
	197	+ return fdb;
	198	+ }
	199	+
	200	+ return fdb;
	201	+}
	202	+
	203	+static void
	204	+mlx5_esw_chains_destroy_fdb_table(struct mlx5_eswitch *esw,
	205	+ struct mlx5_flow_table *fdb)
	206	+{
	207	+ mlx5_esw_chains_put_sz_to_pool(esw, fdb->max_fte);
	208	+ mlx5_destroy_flow_table(fdb);
	209	+}
	210	+
	211	+static struct fdb_chain *
	212	+mlx5_esw_chains_create_fdb_chain(struct mlx5_eswitch *esw, u32 chain)
	213	+{
	214	+ struct fdb_chain *fdb_chain = NULL;
	215	+ int err;
	216	+
	217	+ fdb_chain = kvzalloc(sizeof(*fdb_chain), GFP_KERNEL);
	218	+ if (!fdb_chain)
	219	+ return ERR_PTR(-ENOMEM);
	220	+
	221	+ fdb_chain->esw = esw;
	222	+ fdb_chain->chain = chain;
	223	+
	224	+ err = rhashtable_insert_fast(&esw_chains_ht(esw), &fdb_chain->node,
	225	+ chain_params);
	226	+ if (err)
	227	+ goto err_insert;
	228	+
	229	+ return fdb_chain;
	230	+
	231	+err_insert:
	232	+ kvfree(fdb_chain);
	233	+ return ERR_PTR(err);
	234	+}
	235	+
	236	+static void
	237	+mlx5_esw_chains_destroy_fdb_chain(struct fdb_chain *fdb_chain)
	238	+{
	239	+ struct mlx5_eswitch *esw = fdb_chain->esw;
	240	+
	241	+ rhashtable_remove_fast(&esw_chains_ht(esw), &fdb_chain->node,
	242	+ chain_params);
	243	+ kvfree(fdb_chain);
	244	+}
	245	+
	246	+static struct fdb_chain *
	247	+mlx5_esw_chains_get_fdb_chain(struct mlx5_eswitch *esw, u32 chain)
	248	+{
	249	+ struct fdb_chain *fdb_chain;
	250	+
	251	+ fdb_chain = rhashtable_lookup_fast(&esw_chains_ht(esw), &chain,
	252	+ chain_params);
	253	+ if (!fdb_chain) {
	254	+ fdb_chain = mlx5_esw_chains_create_fdb_chain(esw, chain);
	255	+ if (IS_ERR(fdb_chain))
	256	+ return fdb_chain;
	257	+ }
	258	+
	259	+ fdb_chain->ref++;
	260	+
	261	+ return fdb_chain;
	262	+}
	263	+
	264	+static void
	265	+mlx5_esw_chains_put_fdb_chain(struct fdb_chain *fdb_chain)
	266	+{
	267	+ if (--fdb_chain->ref == 0)
	268	+ mlx5_esw_chains_destroy_fdb_chain(fdb_chain);
	269	+}
	270	+
	271	+static struct fdb_prio *
	272	+mlx5_esw_chains_create_fdb_prio(struct mlx5_eswitch *esw,
	273	+ u32 chain, u32 prio, u32 level)
	274	+{
	275	+ struct fdb_prio *fdb_prio = NULL;
	276	+ struct fdb_chain *fdb_chain;
	277	+ struct mlx5_flow_table *fdb;
	278	+ int err;
	279	+
	280	+ fdb_chain = mlx5_esw_chains_get_fdb_chain(esw, chain);
	281	+ if (IS_ERR(fdb_chain))
	282	+ return ERR_CAST(fdb_chain);
	283	+
	284	+ fdb_prio = kvzalloc(sizeof(*fdb_prio), GFP_KERNEL);
	285	+ if (!fdb_prio) {
	286	+ err = -ENOMEM;
	287	+ goto err_alloc;
	288	+ }
	289	+
	290	+ fdb = mlx5_esw_chains_create_fdb_table(esw, fdb_chain->chain, prio,
	291	+ level);
	292	+ if (IS_ERR(fdb)) {
	293	+ err = PTR_ERR(fdb);
	294	+ goto err_create;
	295	+ }
	296	+
	297	+ fdb_prio->fdb_chain = fdb_chain;
	298	+ fdb_prio->key.chain = chain;
	299	+ fdb_prio->key.prio = prio;
	300	+ fdb_prio->key.level = level;
	301	+ fdb_prio->fdb = fdb;
	302	+
	303	+ err = rhashtable_insert_fast(&esw_prios_ht(esw), &fdb_prio->node,
	304	+ prio_params);
	305	+ if (err)
	306	+ goto err_insert;
	307	+
	308	+ return fdb_prio;
	309	+
	310	+err_insert:
	311	+ mlx5_esw_chains_destroy_fdb_table(esw, fdb);
	312	+err_create:
	313	+ kvfree(fdb_prio);
	314	+err_alloc:
	315	+ mlx5_esw_chains_put_fdb_chain(fdb_chain);
	316	+ return ERR_PTR(err);
	317	+}
	318	+
	319	+static void
	320	+mlx5_esw_chains_destroy_fdb_prio(struct mlx5_eswitch *esw,
	321	+ struct fdb_prio *fdb_prio)
	322	+{
	323	+ struct fdb_chain *fdb_chain = fdb_prio->fdb_chain;
	324	+
	325	+ rhashtable_remove_fast(&esw_prios_ht(esw), &fdb_prio->node,
	326	+ prio_params);
	327	+ mlx5_esw_chains_destroy_fdb_table(esw, fdb_prio->fdb);
	328	+ mlx5_esw_chains_put_fdb_chain(fdb_chain);
	329	+ kvfree(fdb_prio);
	330	+}
	331	+
	332	+struct mlx5_flow_table *
	333	+mlx5_esw_chains_get_table(struct mlx5_eswitch *esw, u32 chain, u32 prio,
	334	+ u32 level)
	335	+{
	336	+ struct mlx5_flow_table *prev_fts;
	337	+ struct fdb_prio *fdb_prio;
	338	+ struct fdb_prio_key key;
	339	+ int l = 0;
	340	+
	341	+ if ((chain > mlx5_esw_chains_get_chain_range(esw) &&
	342	+ chain != mlx5_esw_chains_get_ft_chain(esw)) \|\|
	343	+ prio > mlx5_esw_chains_get_prio_range(esw) \|\|
	344	+ level > mlx5_esw_chains_get_level_range(esw))
	345	+ return ERR_PTR(-EOPNOTSUPP);
	346	+
	347	+ /* create earlier levels for correct fs_core lookup when
	348	+ * connecting tables.
	349	+ */
	350	+ for (l = 0; l < level; l++) {
	351	+ prev_fts = mlx5_esw_chains_get_table(esw, chain, prio, l);
	352	+ if (IS_ERR(prev_fts)) {
	353	+ fdb_prio = ERR_CAST(prev_fts);
	354	+ goto err_get_prevs;
	355	+ }
	356	+ }
	357	+
	358	+ key.chain = chain;
	359	+ key.prio = prio;
	360	+ key.level = level;
	361	+
	362	+ mutex_lock(&esw_chains_lock(esw));
	363	+ fdb_prio = rhashtable_lookup_fast(&esw_prios_ht(esw), &key,
	364	+ prio_params);
	365	+ if (!fdb_prio) {
	366	+ fdb_prio = mlx5_esw_chains_create_fdb_prio(esw, chain,
	367	+ prio, level);
	368	+ if (IS_ERR(fdb_prio))
	369	+ goto err_create_prio;
	370	+ }
	371	+
	372	+ ++fdb_prio->ref;
	373	+ mutex_unlock(&esw_chains_lock(esw));
	374	+
	375	+ return fdb_prio->fdb;
	376	+
	377	+err_create_prio:
	378	+ mutex_unlock(&esw_chains_lock(esw));
	379	+err_get_prevs:
	380	+ while (--l >= 0)
	381	+ mlx5_esw_chains_put_table(esw, chain, prio, l);
	382	+ return ERR_CAST(fdb_prio);
	383	+}
	384	+
	385	+void
	386	+mlx5_esw_chains_put_table(struct mlx5_eswitch *esw, u32 chain, u32 prio,
	387	+ u32 level)
	388	+{
	389	+ struct fdb_prio *fdb_prio;
	390	+ struct fdb_prio_key key;
	391	+
	392	+ key.chain = chain;
	393	+ key.prio = prio;
	394	+ key.level = level;
	395	+
	396	+ mutex_lock(&esw_chains_lock(esw));
	397	+ fdb_prio = rhashtable_lookup_fast(&esw_prios_ht(esw), &key,
	398	+ prio_params);
	399	+ if (!fdb_prio)
	400	+ goto err_get_prio;
	401	+
	402	+ if (--fdb_prio->ref == 0)
	403	+ mlx5_esw_chains_destroy_fdb_prio(esw, fdb_prio);
	404	+ mutex_unlock(&esw_chains_lock(esw));
	405	+
	406	+ while (level-- > 0)
	407	+ mlx5_esw_chains_put_table(esw, chain, prio, level);
	408	+
	409	+ return;
	410	+
	411	+err_get_prio:
	412	+ mutex_unlock(&esw_chains_lock(esw));
	413	+ WARN_ONCE(1,
	414	+ "Couldn't find table: (chain: %d prio: %d level: %d)",
	415	+ chain, prio, level);
	416	+}
	417	+
	418	+static int
	419	+mlx5_esw_chains_init(struct mlx5_eswitch *esw)
	420	+{
	421	+ struct mlx5_esw_chains_priv *chains_priv;
	422	+ struct mlx5_core_dev *dev = esw->dev;
	423	+ u32 max_flow_counter, fdb_max;
	424	+ int err;
	425	+
	426	+ chains_priv = kzalloc(sizeof(*chains_priv), GFP_KERNEL);
	427	+ if (!chains_priv)
	428	+ return -ENOMEM;
	429	+ esw_chains_priv(esw) = chains_priv;
	430	+
	431	+ max_flow_counter = (MLX5_CAP_GEN(dev, max_flow_counter_31_16) << 16) \|
	432	+ MLX5_CAP_GEN(dev, max_flow_counter_15_0);
	433	+ fdb_max = 1 << MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size);
	434	+
	435	+ esw_debug(dev,
	436	+ "Init esw offloads chains, max counters(%d), groups(%d), max flow table size(%d)\n",
	437	+ max_flow_counter, ESW_OFFLOADS_NUM_GROUPS, fdb_max);
	438	+
	439	+ mlx5_esw_chains_init_sz_pool(esw);
	440	+
	441	+ if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, multi_fdb_encap) &&
	442	+ esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) {
	443	+ esw->fdb_table.flags &= ~ESW_FDB_CHAINS_AND_PRIOS_SUPPORTED;
	444	+ esw_warn(dev, "Tc chains and priorities offload aren't supported, update firmware if needed\n");
	445	+ } else {
	446	+ esw->fdb_table.flags \|= ESW_FDB_CHAINS_AND_PRIOS_SUPPORTED;
	447	+ esw_info(dev, "Supported tc offload range - chains: %u, prios: %u\n",
	448	+ mlx5_esw_chains_get_chain_range(esw),
	449	+ mlx5_esw_chains_get_prio_range(esw));
	450	+ }
	451	+
	452	+ err = rhashtable_init(&esw_chains_ht(esw), &chain_params);
	453	+ if (err)
	454	+ goto init_chains_ht_err;
	455	+
	456	+ err = rhashtable_init(&esw_prios_ht(esw), &prio_params);
	457	+ if (err)
	458	+ goto init_prios_ht_err;
	459	+
	460	+ mutex_init(&esw_chains_lock(esw));
	461	+
	462	+ return 0;
	463	+
	464	+init_prios_ht_err:
	465	+ rhashtable_destroy(&esw_chains_ht(esw));
	466	+init_chains_ht_err:
	467	+ kfree(chains_priv);
	468	+ return err;
	469	+}
	470	+
	471	+static void
	472	+mlx5_esw_chains_cleanup(struct mlx5_eswitch *esw)
	473	+{
	474	+ mutex_destroy(&esw_chains_lock(esw));
	475	+ rhashtable_destroy(&esw_prios_ht(esw));
	476	+ rhashtable_destroy(&esw_chains_ht(esw));
	477	+
	478	+ kfree(esw_chains_priv(esw));
	479	+}
	480	+
	481	+static int
	482	+mlx5_esw_chains_open(struct mlx5_eswitch *esw)
	483	+{
	484	+ struct mlx5_flow_table *ft;
	485	+ int err;
	486	+
	487	+ /* Always open the root for fast path */
	488	+ ft = mlx5_esw_chains_get_table(esw, 0, 1, 0);
	489	+ if (IS_ERR(ft))
	490	+ return PTR_ERR(ft);
	491	+
	492	+ /* Open level 1 for split rules now if prios isn't supported */
	493	+ if (!mlx5_esw_chains_prios_supported(esw)) {
	494	+ ft = mlx5_esw_chains_get_table(esw, 0, 1, 1);
	495	+
	496	+ if (IS_ERR(ft)) {
	497	+ err = PTR_ERR(ft);
	498	+ goto level_1_err;
	499	+ }
	500	+ }
	501	+
	502	+ return 0;
	503	+
	504	+level_1_err:
	505	+ mlx5_esw_chains_put_table(esw, 0, 1, 0);
	506	+ return err;
	507	+}
	508	+
	509	+static void
	510	+mlx5_esw_chains_close(struct mlx5_eswitch *esw)
	511	+{
	512	+ if (!mlx5_esw_chains_prios_supported(esw))
	513	+ mlx5_esw_chains_put_table(esw, 0, 1, 1);
	514	+ mlx5_esw_chains_put_table(esw, 0, 1, 0);
	515	+}
	516	+
	517	+int
	518	+mlx5_esw_chains_create(struct mlx5_eswitch *esw)
	519	+{
	520	+ int err;
	521	+
	522	+ err = mlx5_esw_chains_init(esw);
	523	+ if (err)
	524	+ return err;
	525	+
	526	+ err = mlx5_esw_chains_open(esw);
	527	+ if (err)
	528	+ goto err_open;
	529	+
	530	+ return 0;
	531	+
	532	+err_open:
	533	+ mlx5_esw_chains_cleanup(esw);
	534	+ return err;
	535	+}
	536	+
	537	+void
	538	+mlx5_esw_chains_destroy(struct mlx5_eswitch *esw)
	539	+{
	540	+ mlx5_esw_chains_close(esw);
	541	+ mlx5_esw_chains_cleanup(esw);
	542	+}

drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_chains.h+27−0

@@ -0,0 +1,27 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
	2	+/* Copyright (c) 2020 Mellanox Technologies. */
	3	+
	4	+#ifndef __ML5_ESW_CHAINS_H__
	5	+#define __ML5_ESW_CHAINS_H__
	6	+
	7	+bool
	8	+mlx5_esw_chains_prios_supported(struct mlx5_eswitch *esw);
	9	+u32
	10	+mlx5_esw_chains_get_prio_range(struct mlx5_eswitch *esw);
	11	+u32
	12	+mlx5_esw_chains_get_chain_range(struct mlx5_eswitch *esw);
	13	+u32
	14	+mlx5_esw_chains_get_ft_chain(struct mlx5_eswitch *esw);
	15	+
	16	+struct mlx5_flow_table *
	17	+mlx5_esw_chains_get_table(struct mlx5_eswitch *esw, u32 chain, u32 prio,
	18	+ u32 level);
	19	+void
	20	+mlx5_esw_chains_put_table(struct mlx5_eswitch *esw, u32 chain, u32 prio,
	21	+ u32 level);
	22	+
	23	+int mlx5_esw_chains_create(struct mlx5_eswitch *esw);
	24	+void mlx5_esw_chains_destroy(struct mlx5_eswitch *esw);
	25	+
	26	+#endif /* __ML5_ESW_CHAINS_H__ */
	27	+

net/mlx5: E-Switch, Refactor chains and priorities

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code