History

net/mlx5: E-Switch, Refactor chains and priorities

To support the entire chain and prio range (32bit + 16bit), instead of a using a static array of chains/prios of limited size, create them dynamically, and use a rhashtable to search for existing chains/prio combinations. This will be used in next patch to actually increase the number using unamanged tables support and ignore flow level capability.

Paul Blakey committed 6 years ago

Commit 39ac2371 parent 82270e17 files · +637 −292Browse files

Unified · Split

7 files
  1. drivers/net/ethernet/mellanox/mlx5/core/Makefile +1 −1
  2. drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +6 −5
  3. drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +8 −6
  4. drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +8 −22
  5. drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +45 −258
  6. drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_chains.c +542 −0
  7. drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_chains.h +27 −0
drivers/net/ethernet/mellanox/mlx5/core/Makefile+1−1
@@ -42,7 +42,7 @@ mlx5_core-$(CONFIG_PCI_HYPERV_INTERFACE) += en/hv_vhca_stats.o · 41 unmodified lines
42# Core extra42# Core extra
43#43#
44mlx5_core-$(CONFIG_MLX5_ESWITCH) += eswitch.o eswitch_offloads.o eswitch_offloads_termtbl.o \44mlx5_core-$(CONFIG_MLX5_ESWITCH) += eswitch.o eswitch_offloads.o eswitch_offloads_termtbl.o \
45 ecpf.o rdma.o45 ecpf.o rdma.o eswitch_offloads_chains.o
46mlx5_core-$(CONFIG_MLX5_MPFS) += lib/mpfs.o46mlx5_core-$(CONFIG_MLX5_MPFS) += lib/mpfs.o
47mlx5_core-$(CONFIG_VXLAN) += lib/vxlan.o47mlx5_core-$(CONFIG_VXLAN) += lib/vxlan.o
48mlx5_core-$(CONFIG_PTP_1588_CLOCK) += lib/clock.o48mlx5_core-$(CONFIG_PTP_1588_CLOCK) += lib/clock.o
drivers/net/ethernet/mellanox/mlx5/core/en_rep.c+6−5
@@ -41,6 +41,7 @@ · 40 unmodified lines
41#include <net/ipv6_stubs.h>41#include <net/ipv6_stubs.h>
42 42
43#include "eswitch.h"43#include "eswitch.h"
44#include "eswitch_offloads_chains.h"
44#include "en.h"45#include "en.h"
45#include "en_rep.h"46#include "en_rep.h"
46#include "en_tc.h"47#include "en_tc.h"
@@ -1262,25 +1263,25 @@ static int mlx5e_rep_setup_ft_cb(enum tc_setup_type type, void *type_data, · 1215 unmodified lines
1262 case TC_SETUP_CLSFLOWER:1263 case TC_SETUP_CLSFLOWER:
1263 memcpy(&tmp, f, sizeof(*f));1264 memcpy(&tmp, f, sizeof(*f));
1264 1265
1265 if (!mlx5_eswitch_prios_supported(esw) ||1266 if (!mlx5_esw_chains_prios_supported(esw) ||
1266 tmp.common.chain_index)1267 tmp.common.chain_index)
1267 return -EOPNOTSUPP;1268 return -EOPNOTSUPP;
1268 1269
1269 /* Re-use tc offload path by moving the ft flow to the1270 /* Re-use tc offload path by moving the ft flow to the
1270 * reserved ft chain.1271 * reserved ft chain.
1271 *1272 *
1272 * FT offload can use prio range [0, INT_MAX], so we1273 * FT offload can use prio range [0, INT_MAX], so we normalize
1273 * normalize it to range [1, mlx5_eswitch_get_prio_range(esw)]1274 * it to range [1, mlx5_esw_chains_get_prio_range(esw)]
1274 * as with tc, where prio 0 isn't supported.1275 * as with tc, where prio 0 isn't supported.
1275 *1276 *
1276 * We only support chain 0 of FT offload.1277 * We only support chain 0 of FT offload.
1277 */1278 */
1278 if (tmp.common.prio >= mlx5_eswitch_get_prio_range(esw))1279 if (tmp.common.prio >= mlx5_esw_chains_get_prio_range(esw))
1279 return -EOPNOTSUPP;1280 return -EOPNOTSUPP;
1280 if (tmp.common.chain_index != 0)1281 if (tmp.common.chain_index != 0)
1281 return -EOPNOTSUPP;1282 return -EOPNOTSUPP;
1282 1283
1283 tmp.common.chain_index = mlx5_eswitch_get_ft_chain(esw);1284 tmp.common.chain_index = mlx5_esw_chains_get_ft_chain(esw);
1284 tmp.common.prio++;1285 tmp.common.prio++;
1285 err = mlx5e_rep_setup_tc_cls_flower(priv, &tmp, flags);1286 err = mlx5e_rep_setup_tc_cls_flower(priv, &tmp, flags);
1286 memcpy(&f->stats, &tmp.stats, sizeof(f->stats));1287 memcpy(&f->stats, &tmp.stats, sizeof(f->stats));
drivers/net/ethernet/mellanox/mlx5/core/en_tc.c+8−6
@@ -51,6 +51,7 @@ · 50 unmodified lines
51#include "en_rep.h"51#include "en_rep.h"
52#include "en_tc.h"52#include "en_tc.h"
53#include "eswitch.h"53#include "eswitch.h"
54#include "eswitch_offloads_chains.h"
54#include "fs_core.h"55#include "fs_core.h"
55#include "en/port.h"56#include "en/port.h"
56#include "en/tc_tun.h"57#include "en/tc_tun.h"
@@ -1083,7 +1084,7 @@ mlx5e_tc_offload_to_slow_path(struct mlx5_eswitch *esw, · 1026 unmodified lines
1083 memcpy(slow_attr, flow->esw_attr, sizeof(*slow_attr));1084 memcpy(slow_attr, flow->esw_attr, sizeof(*slow_attr));
1084 slow_attr->action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;1085 slow_attr->action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1085 slow_attr->split_count = 0;1086 slow_attr->split_count = 0;
1086 slow_attr->dest_chain = FDB_TC_SLOW_PATH_CHAIN;1087 slow_attr->flags |= MLX5_ESW_ATTR_FLAG_SLOW_PATH;
1087 1088
1088 rule = mlx5e_tc_offload_fdb_rules(esw, flow, spec, slow_attr);1089 rule = mlx5e_tc_offload_fdb_rules(esw, flow, spec, slow_attr);
1089 if (!IS_ERR(rule))1090 if (!IS_ERR(rule))
@@ -1100,7 +1101,7 @@ mlx5e_tc_unoffload_from_slow_path(struct mlx5_eswitch *esw, · 10 unmodified lines
1100 memcpy(slow_attr, flow->esw_attr, sizeof(*slow_attr));1101 memcpy(slow_attr, flow->esw_attr, sizeof(*slow_attr));
1101 slow_attr->action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;1102 slow_attr->action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1102 slow_attr->split_count = 0;1103 slow_attr->split_count = 0;
1103 slow_attr->dest_chain = FDB_TC_SLOW_PATH_CHAIN;1104 slow_attr->flags |= MLX5_ESW_ATTR_FLAG_SLOW_PATH;
1104 mlx5e_tc_unoffload_fdb_rules(esw, flow, slow_attr);1105 mlx5e_tc_unoffload_fdb_rules(esw, flow, slow_attr);
1105 flow_flag_clear(flow, SLOW);1106 flow_flag_clear(flow, SLOW);
1106}1107}
@@ -1160,19 +1161,18 @@ mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv, · 53 unmodified lines
1160 struct netlink_ext_ack *extack)1161 struct netlink_ext_ack *extack)
1161{1162{
1162 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;1163 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
1163 u32 max_chain = mlx5_eswitch_get_chain_range(esw);
1164 struct mlx5_esw_flow_attr *attr = flow->esw_attr;1164 struct mlx5_esw_flow_attr *attr = flow->esw_attr;
1165 struct mlx5e_tc_flow_parse_attr *parse_attr = attr->parse_attr;1165 struct mlx5e_tc_flow_parse_attr *parse_attr = attr->parse_attr;
1166 u16 max_prio = mlx5_eswitch_get_prio_range(esw);
1167 struct net_device *out_dev, *encap_dev = NULL;1166 struct net_device *out_dev, *encap_dev = NULL;
1168 struct mlx5_fc *counter = NULL;1167 struct mlx5_fc *counter = NULL;
1169 struct mlx5e_rep_priv *rpriv;1168 struct mlx5e_rep_priv *rpriv;
1170 struct mlx5e_priv *out_priv;1169 struct mlx5e_priv *out_priv;
1171 bool encap_valid = true;1170 bool encap_valid = true;
1171 u32 max_prio, max_chain;
1172 int err = 0;1172 int err = 0;
1173 int out_index;1173 int out_index;
1174 1174
1175 if (!mlx5_eswitch_prios_supported(esw) && attr->prio != 1) {1175 if (!mlx5_esw_chains_prios_supported(esw) && attr->prio != 1) {
1176 NL_SET_ERR_MSG(extack, "E-switch priorities unsupported, upgrade FW");1176 NL_SET_ERR_MSG(extack, "E-switch priorities unsupported, upgrade FW");
1177 return -EOPNOTSUPP;1177 return -EOPNOTSUPP;
1178 }1178 }
@@ -1182,11 +1182,13 @@ mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv, · 3 unmodified lines
1182 * FDB_FT_CHAIN which is outside tc range.1182 * FDB_FT_CHAIN which is outside tc range.
1183 * See mlx5e_rep_setup_ft_cb().1183 * See mlx5e_rep_setup_ft_cb().
1184 */1184 */
1185 max_chain = mlx5_esw_chains_get_chain_range(esw);
1185 if (!mlx5e_is_ft_flow(flow) && attr->chain > max_chain) {1186 if (!mlx5e_is_ft_flow(flow) && attr->chain > max_chain) {
1186 NL_SET_ERR_MSG(extack, "Requested chain is out of supported range");1187 NL_SET_ERR_MSG(extack, "Requested chain is out of supported range");
1187 return -EOPNOTSUPP;1188 return -EOPNOTSUPP;
1188 }1189 }
1189 1190
1191 max_prio = mlx5_esw_chains_get_prio_range(esw);
1190 if (attr->prio > max_prio) {1192 if (attr->prio > max_prio) {
1191 NL_SET_ERR_MSG(extack, "Requested priority is out of supported range");1193 NL_SET_ERR_MSG(extack, "Requested priority is out of supported range");
1192 return -EOPNOTSUPP;1194 return -EOPNOTSUPP;
@@ -3469,7 +3471,7 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, · 2276 unmodified lines
3469 break;3471 break;
3470 case FLOW_ACTION_GOTO: {3472 case FLOW_ACTION_GOTO: {
3471 u32 dest_chain = act->chain_index;3473 u32 dest_chain = act->chain_index;
3472 u32 max_chain = mlx5_eswitch_get_chain_range(esw);3474 u32 max_chain = mlx5_esw_chains_get_chain_range(esw);
3473 3475
3474 if (ft_flow) {3476 if (ft_flow) {
3475 NL_SET_ERR_MSG_MOD(extack, "Goto action is not supported");3477 NL_SET_ERR_MSG_MOD(extack, "Goto action is not supported");
drivers/net/ethernet/mellanox/mlx5/core/eswitch.h+8−22
@@ -157,7 +157,7 @@ enum offloads_fdb_flags { · 156 unmodified lines
157 ESW_FDB_CHAINS_AND_PRIOS_SUPPORTED = BIT(0),157 ESW_FDB_CHAINS_AND_PRIOS_SUPPORTED = BIT(0),
158};158};
159 159
160extern const unsigned int ESW_POOLS[4];160struct mlx5_esw_chains_priv;
161 161
162struct mlx5_eswitch_fdb {162struct mlx5_eswitch_fdb {
163 union {163 union {
@@ -182,14 +182,7 @@ struct mlx5_eswitch_fdb { · 18 unmodified lines
182 struct mlx5_flow_handle *miss_rule_multi;182 struct mlx5_flow_handle *miss_rule_multi;
183 int vlan_push_pop_refcount;183 int vlan_push_pop_refcount;
184 184
185 struct {185 struct mlx5_esw_chains_priv *esw_chains_priv;
186 struct mlx5_flow_table *fdb;
187 u32 num_rules;
188 } fdb_prio[FDB_NUM_CHAINS][FDB_TC_MAX_PRIO + 1][FDB_TC_LEVELS_PER_PRIO];
189 /* Protects fdb_prio table */
190 struct mutex fdb_prio_lock;
191
192 int fdb_left[ARRAY_SIZE(ESW_POOLS)];
193 } offloads;186 } offloads;
194 };187 };
195 u32 flags;188 u32 flags;
@@ -355,18 +348,6 @@ mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch *esw, · 159 unmodified lines
355 struct mlx5_flow_handle *rule,348 struct mlx5_flow_handle *rule,
356 struct mlx5_esw_flow_attr *attr);349 struct mlx5_esw_flow_attr *attr);
357 350
358bool
359mlx5_eswitch_prios_supported(struct mlx5_eswitch *esw);
360
361u16
362mlx5_eswitch_get_prio_range(struct mlx5_eswitch *esw);
363
364u32
365mlx5_eswitch_get_chain_range(struct mlx5_eswitch *esw);
366
367unsigned int
368mlx5_eswitch_get_ft_chain(struct mlx5_eswitch *esw);
369
370struct mlx5_flow_handle *351struct mlx5_flow_handle *
371mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport,352mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport,
372 struct mlx5_flow_destination *dest);353 struct mlx5_flow_destination *dest);
@@ -391,6 +372,11 @@ enum { · 18 unmodified lines
391 MLX5_ESW_DEST_ENCAP_VALID = BIT(1),372 MLX5_ESW_DEST_ENCAP_VALID = BIT(1),
392};373};
393 374
375enum {
376 MLX5_ESW_ATTR_FLAG_VLAN_HANDLED = BIT(0),
377 MLX5_ESW_ATTR_FLAG_SLOW_PATH = BIT(1),
378};
379
394struct mlx5_esw_flow_attr {380struct mlx5_esw_flow_attr {
395 struct mlx5_eswitch_rep *in_rep;381 struct mlx5_eswitch_rep *in_rep;
396 struct mlx5_core_dev *in_mdev;382 struct mlx5_core_dev *in_mdev;
@@ -404,7 +390,6 @@ struct mlx5_esw_flow_attr { · 7 unmodified lines
404 u16 vlan_vid[MLX5_FS_VLAN_DEPTH];390 u16 vlan_vid[MLX5_FS_VLAN_DEPTH];
405 u8 vlan_prio[MLX5_FS_VLAN_DEPTH];391 u8 vlan_prio[MLX5_FS_VLAN_DEPTH];
406 u8 total_vlan;392 u8 total_vlan;
407 bool vlan_handled;
408 struct {393 struct {
409 u32 flags;394 u32 flags;
410 struct mlx5_eswitch_rep *rep;395 struct mlx5_eswitch_rep *rep;
@@ -419,6 +404,7 @@ struct mlx5_esw_flow_attr { · 8 unmodified lines
419 u32 chain;404 u32 chain;
420 u16 prio;405 u16 prio;
421 u32 dest_chain;406 u32 dest_chain;
407 u32 flags;
422 struct mlx5e_tc_flow_parse_attr *parse_attr;408 struct mlx5e_tc_flow_parse_attr *parse_attr;
423};409};
424 410
drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c+45−258
@@ -37,6 +37,7 @@ · 36 unmodified lines
37#include <linux/mlx5/fs.h>37#include <linux/mlx5/fs.h>
38#include "mlx5_core.h"38#include "mlx5_core.h"
39#include "eswitch.h"39#include "eswitch.h"
40#include "eswitch_offloads_chains.h"
40#include "rdma.h"41#include "rdma.h"
41#include "en.h"42#include "en.h"
42#include "fs_core.h"43#include "fs_core.h"
@@ -47,10 +48,6 @@ · 4 unmodified lines
47 * one for multicast.48 * one for multicast.
48 */49 */
49#define MLX5_ESW_MISS_FLOWS (2)50#define MLX5_ESW_MISS_FLOWS (2)
50
51#define fdb_prio_table(esw, chain, prio, level) \
52 (esw)->fdb_table.offloads.fdb_prio[(chain)][(prio)][(level)]
53
54#define UPLINK_REP_INDEX 051#define UPLINK_REP_INDEX 0
55 52
56static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw,53static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw,
@@ -62,37 +59,6 @@ static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw, · 5 unmodified lines
62 return &esw->offloads.vport_reps[idx];59 return &esw->offloads.vport_reps[idx];
63}60}
64 61
65static struct mlx5_flow_table *
66esw_get_prio_table(struct mlx5_eswitch *esw, u32 chain, u16 prio, int level);
67static void
68esw_put_prio_table(struct mlx5_eswitch *esw, u32 chain, u16 prio, int level);
69
70bool mlx5_eswitch_prios_supported(struct mlx5_eswitch *esw)
71{
72 return (!!(esw->fdb_table.flags & ESW_FDB_CHAINS_AND_PRIOS_SUPPORTED));
73}
74
75u32 mlx5_eswitch_get_chain_range(struct mlx5_eswitch *esw)
76{
77 if (esw->fdb_table.flags & ESW_FDB_CHAINS_AND_PRIOS_SUPPORTED)
78 return FDB_TC_MAX_CHAIN;
79
80 return 0;
81}
82
83u32 mlx5_eswitch_get_ft_chain(struct mlx5_eswitch *esw)
84{
85 return mlx5_eswitch_get_chain_range(esw) + 1;
86}
87
88u16 mlx5_eswitch_get_prio_range(struct mlx5_eswitch *esw)
89{
90 if (esw->fdb_table.flags & ESW_FDB_CHAINS_AND_PRIOS_SUPPORTED)
91 return FDB_TC_MAX_PRIO;
92
93 return 1;
94}
95
96static bool62static bool
97esw_check_ingress_prio_tag_enabled(const struct mlx5_eswitch *esw,63esw_check_ingress_prio_tag_enabled(const struct mlx5_eswitch *esw,
98 const struct mlx5_vport *vport)64 const struct mlx5_vport *vport)
@@ -180,10 +146,17 @@ mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, · 81 unmodified lines
180 }146 }
181 147
182 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {148 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
183 if (attr->dest_chain) {149 struct mlx5_flow_table *ft;
184 struct mlx5_flow_table *ft;
185 150
186 ft = esw_get_prio_table(esw, attr->dest_chain, 1, 0);151 if (attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH) {
152 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
153 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
154 dest[i].ft = esw->fdb_table.offloads.slow_fdb;
155 i++;
156 } else if (attr->dest_chain) {
157 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
158 ft = mlx5_esw_chains_get_table(esw, attr->dest_chain,
159 1, 0);
187 if (IS_ERR(ft)) {160 if (IS_ERR(ft)) {
188 rule = ERR_CAST(ft);161 rule = ERR_CAST(ft);
189 goto err_create_goto_table;162 goto err_create_goto_table;
@@ -228,7 +201,8 @@ mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, · 38 unmodified lines
228 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)201 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
229 flow_act.modify_hdr = attr->modify_hdr;202 flow_act.modify_hdr = attr->modify_hdr;
230 203
231 fdb = esw_get_prio_table(esw, attr->chain, attr->prio, !!split);204 fdb = mlx5_esw_chains_get_table(esw, attr->chain, attr->prio,
205 !!split);
232 if (IS_ERR(fdb)) {206 if (IS_ERR(fdb)) {
233 rule = ERR_CAST(fdb);207 rule = ERR_CAST(fdb);
234 goto err_esw_get;208 goto err_esw_get;
@@ -247,10 +221,10 @@ mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, · 12 unmodified lines
247 return rule;221 return rule;
248 222
249err_add_rule:223err_add_rule:
250 esw_put_prio_table(esw, attr->chain, attr->prio, !!split);224 mlx5_esw_chains_put_table(esw, attr->chain, attr->prio, !!split);
251err_esw_get:225err_esw_get:
252 if (attr->dest_chain)226 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH) && attr->dest_chain)
253 esw_put_prio_table(esw, attr->dest_chain, 1, 0);227 mlx5_esw_chains_put_table(esw, attr->dest_chain, 1, 0);
254err_create_goto_table:228err_create_goto_table:
255 return rule;229 return rule;
256}230}
@@ -267,13 +241,13 @@ mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw, · 10 unmodified lines
267 struct mlx5_flow_handle *rule;241 struct mlx5_flow_handle *rule;
268 int i;242 int i;
269 243
270 fast_fdb = esw_get_prio_table(esw, attr->chain, attr->prio, 0);244 fast_fdb = mlx5_esw_chains_get_table(esw, attr->chain, attr->prio, 0);
271 if (IS_ERR(fast_fdb)) {245 if (IS_ERR(fast_fdb)) {
272 rule = ERR_CAST(fast_fdb);246 rule = ERR_CAST(fast_fdb);
273 goto err_get_fast;247 goto err_get_fast;
274 }248 }
275 249
276 fwd_fdb = esw_get_prio_table(esw, attr->chain, attr->prio, 1);250 fwd_fdb = mlx5_esw_chains_get_table(esw, attr->chain, attr->prio, 1);
277 if (IS_ERR(fwd_fdb)) {251 if (IS_ERR(fwd_fdb)) {
278 rule = ERR_CAST(fwd_fdb);252 rule = ERR_CAST(fwd_fdb);
279 goto err_get_fwd;253 goto err_get_fwd;
@@ -310,9 +284,9 @@ mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw, · 30 unmodified lines
310 284
311 return rule;285 return rule;
312add_err:286add_err:
313 esw_put_prio_table(esw, attr->chain, attr->prio, 1);287 mlx5_esw_chains_put_table(esw, attr->chain, attr->prio, 1);
314err_get_fwd:288err_get_fwd:
315 esw_put_prio_table(esw, attr->chain, attr->prio, 0);289 mlx5_esw_chains_put_table(esw, attr->chain, attr->prio, 0);
316err_get_fast:290err_get_fast:
317 return rule;291 return rule;
318}292}
@@ -337,12 +311,13 @@ __mlx5_eswitch_del_rule(struct mlx5_eswitch *esw, · 18 unmodified lines
337 atomic64_dec(&esw->offloads.num_flows);311 atomic64_dec(&esw->offloads.num_flows);
338 312
339 if (fwd_rule) {313 if (fwd_rule) {
340 esw_put_prio_table(esw, attr->chain, attr->prio, 1);314 mlx5_esw_chains_put_table(esw, attr->chain, attr->prio, 1);
341 esw_put_prio_table(esw, attr->chain, attr->prio, 0);315 mlx5_esw_chains_put_table(esw, attr->chain, attr->prio, 0);
342 } else {316 } else {
343 esw_put_prio_table(esw, attr->chain, attr->prio, !!split);317 mlx5_esw_chains_put_table(esw, attr->chain, attr->prio,
318 !!split);
344 if (attr->dest_chain)319 if (attr->dest_chain)
345 esw_put_prio_table(esw, attr->dest_chain, 1, 0);320 mlx5_esw_chains_put_table(esw, attr->dest_chain, 1, 0);
346 }321 }
347}322}
348 323
@@ -456,7 +431,7 @@ int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch *esw, · 107 unmodified lines
456 if (err)431 if (err)
457 goto unlock;432 goto unlock;
458 433
459 attr->vlan_handled = false;434 attr->flags &= ~MLX5_ESW_ATTR_FLAG_VLAN_HANDLED;
460 435
461 vport = esw_vlan_action_get_vport(attr, push, pop);436 vport = esw_vlan_action_get_vport(attr, push, pop);
462 437
@@ -464,7 +439,7 @@ int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch *esw, · 1 unmodified line
464 /* tracks VF --> wire rules without vlan push action */439 /* tracks VF --> wire rules without vlan push action */
465 if (attr->dests[0].rep->vport == MLX5_VPORT_UPLINK) {440 if (attr->dests[0].rep->vport == MLX5_VPORT_UPLINK) {
466 vport->vlan_refcount++;441 vport->vlan_refcount++;
467 attr->vlan_handled = true;442 attr->flags |= MLX5_ESW_ATTR_FLAG_VLAN_HANDLED;
468 }443 }
469 444
470 goto unlock;445 goto unlock;
@@ -495,7 +470,7 @@ skip_set_push: · 24 unmodified lines
495 }470 }
496out:471out:
497 if (!err)472 if (!err)
498 attr->vlan_handled = true;473 attr->flags |= MLX5_ESW_ATTR_FLAG_VLAN_HANDLED;
499unlock:474unlock:
500 mutex_unlock(&esw->state_lock);475 mutex_unlock(&esw->state_lock);
501 return err;476 return err;
@@ -513,7 +488,7 @@ int mlx5_eswitch_del_vlan_action(struct mlx5_eswitch *esw, · 11 unmodified lines
513 if (mlx5_eswitch_vlan_actions_supported(esw->dev, 1))488 if (mlx5_eswitch_vlan_actions_supported(esw->dev, 1))
514 return 0;489 return 0;
515 490
516 if (!attr->vlan_handled)491 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_VLAN_HANDLED))
517 return 0;492 return 0;
518 493
519 push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH);494 push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH);
@@ -587,8 +562,8 @@ mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *esw, u16 vport, · 67 unmodified lines
587 dest.vport.num = vport;562 dest.vport.num = vport;
588 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;563 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
589 564
590 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, spec,565 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
591 &flow_act, &dest, 1);566 spec, &flow_act, &dest, 1);
592 if (IS_ERR(flow_rule))567 if (IS_ERR(flow_rule))
593 esw_warn(esw->dev, "FDB: Failed to add send to vport rule err %ld\n", PTR_ERR(flow_rule));568 esw_warn(esw->dev, "FDB: Failed to add send to vport rule err %ld\n", PTR_ERR(flow_rule));
594out:569out:
@@ -829,8 +804,8 @@ static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw) · 234 unmodified lines
829 dest.vport.num = esw->manager_vport;804 dest.vport.num = esw->manager_vport;
830 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;805 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
831 806
832 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, spec,807 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
833 &flow_act, &dest, 1);808 spec, &flow_act, &dest, 1);
834 if (IS_ERR(flow_rule)) {809 if (IS_ERR(flow_rule)) {
835 err = PTR_ERR(flow_rule);810 err = PTR_ERR(flow_rule);
836 esw_warn(esw->dev, "FDB: Failed to add unicast miss flow rule err %d\n", err);811 esw_warn(esw->dev, "FDB: Failed to add unicast miss flow rule err %d\n", err);
@@ -844,8 +819,8 @@ static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw) · 7 unmodified lines
844 dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v,819 dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v,
845 outer_headers.dmac_47_16);820 outer_headers.dmac_47_16);
846 dmac_v[0] = 0x01;821 dmac_v[0] = 0x01;
847 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, spec,822 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
848 &flow_act, &dest, 1);823 spec, &flow_act, &dest, 1);
849 if (IS_ERR(flow_rule)) {824 if (IS_ERR(flow_rule)) {
850 err = PTR_ERR(flow_rule);825 err = PTR_ERR(flow_rule);
851 esw_warn(esw->dev, "FDB: Failed to add multicast miss flow rule err %d\n", err);826 esw_warn(esw->dev, "FDB: Failed to add multicast miss flow rule err %d\n", err);
@@ -860,175 +835,6 @@ out: · 8 unmodified lines
860 return err;835 return err;
861}836}
862 837
863#define ESW_OFFLOADS_NUM_GROUPS 4
864
865/* Firmware currently has 4 pool of 4 sizes that it supports (ESW_POOLS),
866 * and a virtual memory region of 16M (ESW_SIZE), this region is duplicated
867 * for each flow table pool. We can allocate up to 16M of each pool,
868 * and we keep track of how much we used via put/get_sz_to_pool.
869 * Firmware doesn't report any of this for now.
870 * ESW_POOL is expected to be sorted from large to small
871 */
872#define ESW_SIZE (16 * 1024 * 1024)
873const unsigned int ESW_POOLS[4] = { 4 * 1024 * 1024, 1 * 1024 * 1024,
874 64 * 1024, 4 * 1024 };
875
876static int
877get_sz_from_pool(struct mlx5_eswitch *esw)
878{
879 int sz = 0, i;
880
881 for (i = 0; i < ARRAY_SIZE(ESW_POOLS); i++) {
882 if (esw->fdb_table.offloads.fdb_left[i]) {
883 --esw->fdb_table.offloads.fdb_left[i];
884 sz = ESW_POOLS[i];
885 break;
886 }
887 }
888
889 return sz;
890}
891
892static void
893put_sz_to_pool(struct mlx5_eswitch *esw, int sz)
894{
895 int i;
896
897 for (i = 0; i < ARRAY_SIZE(ESW_POOLS); i++) {
898 if (sz >= ESW_POOLS[i]) {
899 ++esw->fdb_table.offloads.fdb_left[i];
900 break;
901 }
902 }
903}
904
905static struct mlx5_flow_table *
906create_next_size_table(struct mlx5_eswitch *esw,
907 struct mlx5_flow_namespace *ns,
908 u16 table_prio,
909 int level,
910 u32 flags)
911{
912 struct mlx5_flow_table_attr ft_attr = {};
913 struct mlx5_flow_table *fdb;
914 int sz;
915
916 sz = get_sz_from_pool(esw);
917 if (!sz)
918 return ERR_PTR(-ENOSPC);
919
920 ft_attr.max_fte = sz;
921 ft_attr.prio = table_prio;
922 ft_attr.level = level;
923 ft_attr.flags = flags;
924 ft_attr.autogroup.max_num_groups = ESW_OFFLOADS_NUM_GROUPS;
925 fdb = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
926 if (IS_ERR(fdb)) {
927 esw_warn(esw->dev, "Failed to create FDB Table err %d (table prio: %d, level: %d, size: %d)\n",
928 (int)PTR_ERR(fdb), table_prio, level, sz);
929 put_sz_to_pool(esw, sz);
930 }
931
932 return fdb;
933}
934
935static struct mlx5_flow_table *
936esw_get_prio_table(struct mlx5_eswitch *esw, u32 chain, u16 prio, int level)
937{
938 struct mlx5_core_dev *dev = esw->dev;
939 struct mlx5_flow_table *fdb = NULL;
940 struct mlx5_flow_namespace *ns;
941 int table_prio, l = 0;
942 u32 flags = 0;
943
944 if (chain == FDB_TC_SLOW_PATH_CHAIN)
945 return esw->fdb_table.offloads.slow_fdb;
946
947 mutex_lock(&esw->fdb_table.offloads.fdb_prio_lock);
948
949 fdb = fdb_prio_table(esw, chain, prio, level).fdb;
950 if (fdb) {
951 /* take ref on earlier levels as well */
952 while (level >= 0)
953 fdb_prio_table(esw, chain, prio, level--).num_rules++;
954 mutex_unlock(&esw->fdb_table.offloads.fdb_prio_lock);
955 return fdb;
956 }
957
958 ns = mlx5_get_fdb_sub_ns(dev, chain);
959 if (!ns) {
960 esw_warn(dev, "Failed to get FDB sub namespace\n");
961 mutex_unlock(&esw->fdb_table.offloads.fdb_prio_lock);
962 return ERR_PTR(-EOPNOTSUPP);
963 }
964
965 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
966 flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT |
967 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
968
969 table_prio = prio - 1;
970
971 /* create earlier levels for correct fs_core lookup when
972 * connecting tables
973 */
974 for (l = 0; l <= level; l++) {
975 if (fdb_prio_table(esw, chain, prio, l).fdb) {
976 fdb_prio_table(esw, chain, prio, l).num_rules++;
977 continue;
978 }
979
980 fdb = create_next_size_table(esw, ns, table_prio, l, flags);
981 if (IS_ERR(fdb)) {
982 l--;
983 goto err_create_fdb;
984 }
985
986 fdb_prio_table(esw, chain, prio, l).fdb = fdb;
987 fdb_prio_table(esw, chain, prio, l).num_rules = 1;
988 }
989
990 mutex_unlock(&esw->fdb_table.offloads.fdb_prio_lock);
991 return fdb;
992
993err_create_fdb:
994 mutex_unlock(&esw->fdb_table.offloads.fdb_prio_lock);
995 if (l >= 0)
996 esw_put_prio_table(esw, chain, prio, l);
997
998 return fdb;
999}
1000
1001static void
1002esw_put_prio_table(struct mlx5_eswitch *esw, u32 chain, u16 prio, int level)
1003{
1004 int l;
1005
1006 if (chain == FDB_TC_SLOW_PATH_CHAIN)
1007 return;
1008
1009 mutex_lock(&esw->fdb_table.offloads.fdb_prio_lock);
1010
1011 for (l = level; l >= 0; l--) {
1012 if (--(fdb_prio_table(esw, chain, prio, l).num_rules) > 0)
1013 continue;
1014
1015 put_sz_to_pool(esw, fdb_prio_table(esw, chain, prio, l).fdb->max_fte);
1016 mlx5_destroy_flow_table(fdb_prio_table(esw, chain, prio, l).fdb);
1017 fdb_prio_table(esw, chain, prio, l).fdb = NULL;
1018 }
1019
1020 mutex_unlock(&esw->fdb_table.offloads.fdb_prio_lock);
1021}
1022
1023static void esw_destroy_offloads_fast_fdb_tables(struct mlx5_eswitch *esw)
1024{
1025 /* If lazy creation isn't supported, deref the fast path tables */
1026 if (!(esw->fdb_table.flags & ESW_FDB_CHAINS_AND_PRIOS_SUPPORTED)) {
1027 esw_put_prio_table(esw, 0, 1, 1);
1028 esw_put_prio_table(esw, 0, 1, 0);
1029 }
1030}
1031
1032#define MAX_PF_SQ 256838#define MAX_PF_SQ 256
1033#define MAX_SQ_NVPORTS 32839#define MAX_SQ_NVPORTS 32
1034 840
@@ -1061,16 +867,16 @@ static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw, int nvports) · 26 unmodified lines
1061 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);867 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1062 struct mlx5_flow_table_attr ft_attr = {};868 struct mlx5_flow_table_attr ft_attr = {};
1063 struct mlx5_core_dev *dev = esw->dev;869 struct mlx5_core_dev *dev = esw->dev;
1064 u32 *flow_group_in, max_flow_counter;
1065 struct mlx5_flow_namespace *root_ns;870 struct mlx5_flow_namespace *root_ns;
1066 struct mlx5_flow_table *fdb = NULL;871 struct mlx5_flow_table *fdb = NULL;
1067 int table_size, ix, err = 0, i;872 u32 flags = 0, *flow_group_in;
873 int table_size, ix, err = 0;
1068 struct mlx5_flow_group *g;874 struct mlx5_flow_group *g;
1069 u32 flags = 0, fdb_max;
1070 void *match_criteria;875 void *match_criteria;
1071 u8 *dmac;876 u8 *dmac;
1072 877
1073 esw_debug(esw->dev, "Create offloads FDB Tables\n");878 esw_debug(esw->dev, "Create offloads FDB Tables\n");
879
1074 flow_group_in = kvzalloc(inlen, GFP_KERNEL);880 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
1075 if (!flow_group_in)881 if (!flow_group_in)
1076 return -ENOMEM;882 return -ENOMEM;
@@ -1089,19 +895,6 @@ static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw, int nvports) · 12 unmodified lines
1089 goto ns_err;895 goto ns_err;
1090 }896 }
1091 897
1092 max_flow_counter = (MLX5_CAP_GEN(dev, max_flow_counter_31_16) << 16) |
1093 MLX5_CAP_GEN(dev, max_flow_counter_15_0);
1094 fdb_max = 1 << MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size);
1095
1096 esw_debug(dev, "Create offloads FDB table, min (max esw size(2^%d), max counters(%d), groups(%d), max flow table size(%d))\n",
1097 MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size),
1098 max_flow_counter, ESW_OFFLOADS_NUM_GROUPS,
1099 fdb_max);
1100
1101 for (i = 0; i < ARRAY_SIZE(ESW_POOLS); i++)
1102 esw->fdb_table.offloads.fdb_left[i] =
1103 ESW_POOLS[i] <= fdb_max ? ESW_SIZE / ESW_POOLS[i] : 0;
1104
1105 table_size = nvports * MAX_SQ_NVPORTS + MAX_PF_SQ +898 table_size = nvports * MAX_SQ_NVPORTS + MAX_PF_SQ +
1106 MLX5_ESW_MISS_FLOWS + esw->total_vports;899 MLX5_ESW_MISS_FLOWS + esw->total_vports;
1107 900
@@ -1124,16 +917,10 @@ static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw, int nvports) · 16 unmodified lines
1124 }917 }
1125 esw->fdb_table.offloads.slow_fdb = fdb;918 esw->fdb_table.offloads.slow_fdb = fdb;
1126 919
1127 /* If lazy creation isn't supported, open the fast path tables now */920 err = mlx5_esw_chains_create(esw);
1128 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, multi_fdb_encap) &&921 if (err) {
1129 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) {922 esw_warn(dev, "Failed to create fdb chains err(%d)\n", err);
1130 esw->fdb_table.flags &= ~ESW_FDB_CHAINS_AND_PRIOS_SUPPORTED;923 goto fdb_chains_err;
1131 esw_warn(dev, "Lazy creation of flow tables isn't supported, ignoring priorities\n");
1132 esw_get_prio_table(esw, 0, 1, 0);
1133 esw_get_prio_table(esw, 0, 1, 1);
1134 } else {
1135 esw_debug(dev, "Lazy creation of flow tables supported, deferring table opening\n");
1136 esw->fdb_table.flags |= ESW_FDB_CHAINS_AND_PRIOS_SUPPORTED;
1137 }924 }
1138 925
1139 /* create send-to-vport group */926 /* create send-to-vport group */
@@ -1224,7 +1011,8 @@ miss_err: · 84 unmodified lines
1224peer_miss_err:1011peer_miss_err:
1225 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);1012 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);
1226send_vport_err:1013send_vport_err:
1227 esw_destroy_offloads_fast_fdb_tables(esw);1014 mlx5_esw_chains_destroy(esw);
1015fdb_chains_err:
1228 mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb);1016 mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb);
1229slow_fdb_err:1017slow_fdb_err:
1230 /* Holds true only as long as DMFS is the default */1018 /* Holds true only as long as DMFS is the default */
@@ -1246,8 +1034,8 @@ static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw) · 15 unmodified lines
1246 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp);1034 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp);
1247 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);1035 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);
1248 1036
1037 mlx5_esw_chains_destroy(esw);
1249 mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb);1038 mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb);
1250 esw_destroy_offloads_fast_fdb_tables(esw);
1251 /* Holds true only as long as DMFS is the default */1039 /* Holds true only as long as DMFS is the default */
1252 mlx5_flow_namespace_set_mode(esw->fdb_table.offloads.ns,1040 mlx5_flow_namespace_set_mode(esw->fdb_table.offloads.ns,
1253 MLX5_FLOW_STEERING_MODE_DMFS);1041 MLX5_FLOW_STEERING_MODE_DMFS);
@@ -2117,7 +1905,6 @@ static int esw_offloads_steering_init(struct mlx5_eswitch *esw) · 863 unmodified lines
2117 total_vports = num_vfs + MLX5_SPECIAL_VPORTS(esw->dev);1905 total_vports = num_vfs + MLX5_SPECIAL_VPORTS(esw->dev);
2118 1906
2119 memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb));1907 memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb));
2120 mutex_init(&esw->fdb_table.offloads.fdb_prio_lock);
2121 1908
2122 err = esw_create_uplink_offloads_acl_tables(esw);1909 err = esw_create_uplink_offloads_acl_tables(esw);
2123 if (err)1910 if (err)
drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_chains.c+542−0
@@ -0,0 +1,542 @@
1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2// Copyright (c) 2020 Mellanox Technologies.
3
4#include <linux/mlx5/driver.h>
5#include <linux/mlx5/mlx5_ifc.h>
6#include <linux/mlx5/fs.h>
7
8#include "eswitch_offloads_chains.h"
9#include "mlx5_core.h"
10#include "fs_core.h"
11#include "eswitch.h"
12#include "en.h"
13
14#define esw_chains_priv(esw) ((esw)->fdb_table.offloads.esw_chains_priv)
15#define esw_chains_lock(esw) (esw_chains_priv(esw)->lock)
16#define esw_chains_ht(esw) (esw_chains_priv(esw)->chains_ht)
17#define esw_prios_ht(esw) (esw_chains_priv(esw)->prios_ht)
18#define fdb_pool_left(esw) (esw_chains_priv(esw)->fdb_left)
19
20#define ESW_OFFLOADS_NUM_GROUPS 4
21
22/* Firmware currently has 4 pool of 4 sizes that it supports (ESW_POOLS),
23 * and a virtual memory region of 16M (ESW_SIZE), this region is duplicated
24 * for each flow table pool. We can allocate up to 16M of each pool,
25 * and we keep track of how much we used via get_next_avail_sz_from_pool.
26 * Firmware doesn't report any of this for now.
27 * ESW_POOL is expected to be sorted from large to small and match firmware
28 * pools.
29 */
30#define ESW_SIZE (16 * 1024 * 1024)
31const unsigned int ESW_POOLS[] = { 4 * 1024 * 1024,
32 1 * 1024 * 1024,
33 64 * 1024,
34 4 * 1024, };
35
36struct mlx5_esw_chains_priv {
37 struct rhashtable chains_ht;
38 struct rhashtable prios_ht;
39 /* Protects above chains_ht and prios_ht */
40 struct mutex lock;
41
42 int fdb_left[ARRAY_SIZE(ESW_POOLS)];
43};
44
45struct fdb_chain {
46 struct rhash_head node;
47
48 u32 chain;
49
50 int ref;
51
52 struct mlx5_eswitch *esw;
53};
54
55struct fdb_prio_key {
56 u32 chain;
57 u32 prio;
58 u32 level;
59};
60
61struct fdb_prio {
62 struct rhash_head node;
63
64 struct fdb_prio_key key;
65
66 int ref;
67
68 struct fdb_chain *fdb_chain;
69 struct mlx5_flow_table *fdb;
70};
71
72static const struct rhashtable_params chain_params = {
73 .head_offset = offsetof(struct fdb_chain, node),
74 .key_offset = offsetof(struct fdb_chain, chain),
75 .key_len = sizeof_field(struct fdb_chain, chain),
76 .automatic_shrinking = true,
77};
78
79static const struct rhashtable_params prio_params = {
80 .head_offset = offsetof(struct fdb_prio, node),
81 .key_offset = offsetof(struct fdb_prio, key),
82 .key_len = sizeof_field(struct fdb_prio, key),
83 .automatic_shrinking = true,
84};
85
86bool mlx5_esw_chains_prios_supported(struct mlx5_eswitch *esw)
87{
88 return esw->fdb_table.flags & ESW_FDB_CHAINS_AND_PRIOS_SUPPORTED;
89}
90
91u32 mlx5_esw_chains_get_chain_range(struct mlx5_eswitch *esw)
92{
93 if (!mlx5_esw_chains_prios_supported(esw))
94 return 1;
95
96 return FDB_TC_MAX_CHAIN;
97}
98
99u32 mlx5_esw_chains_get_ft_chain(struct mlx5_eswitch *esw)
100{
101 return mlx5_esw_chains_get_chain_range(esw) + 1;
102}
103
104u32 mlx5_esw_chains_get_prio_range(struct mlx5_eswitch *esw)
105{
106 if (!mlx5_esw_chains_prios_supported(esw))
107 return 1;
108
109 return FDB_TC_MAX_PRIO;
110}
111
112static unsigned int mlx5_esw_chains_get_level_range(struct mlx5_eswitch *esw)
113{
114 return FDB_TC_LEVELS_PER_PRIO;
115}
116
117#define POOL_NEXT_SIZE 0
118static int
119mlx5_esw_chains_get_avail_sz_from_pool(struct mlx5_eswitch *esw,
120 int desired_size)
121{
122 int i, found_i = -1;
123
124 for (i = ARRAY_SIZE(ESW_POOLS) - 1; i >= 0; i--) {
125 if (fdb_pool_left(esw)[i] && ESW_POOLS[i] > desired_size) {
126 found_i = i;
127 if (desired_size != POOL_NEXT_SIZE)
128 break;
129 }
130 }
131
132 if (found_i != -1) {
133 --fdb_pool_left(esw)[found_i];
134 return ESW_POOLS[found_i];
135 }
136
137 return 0;
138}
139
140static void
141mlx5_esw_chains_put_sz_to_pool(struct mlx5_eswitch *esw, int sz)
142{
143 int i;
144
145 for (i = ARRAY_SIZE(ESW_POOLS) - 1; i >= 0; i--) {
146 if (sz == ESW_POOLS[i]) {
147 ++fdb_pool_left(esw)[i];
148 return;
149 }
150 }
151
152 WARN_ONCE(1, "Couldn't find size %d in fdb size pool", sz);
153}
154
155static void
156mlx5_esw_chains_init_sz_pool(struct mlx5_eswitch *esw)
157{
158 u32 fdb_max;
159 int i;
160
161 fdb_max = 1 << MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, log_max_ft_size);
162
163 for (i = ARRAY_SIZE(ESW_POOLS) - 1; i >= 0; i--)
164 fdb_pool_left(esw)[i] =
165 ESW_POOLS[i] <= fdb_max ? ESW_SIZE / ESW_POOLS[i] : 0;
166}
167
168static struct mlx5_flow_table *
169mlx5_esw_chains_create_fdb_table(struct mlx5_eswitch *esw,
170 u32 chain, u32 prio, u32 level)
171{
172 struct mlx5_flow_table_attr ft_attr = {};
173 struct mlx5_flow_namespace *ns;
174 struct mlx5_flow_table *fdb;
175 int sz;
176
177 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
178 ft_attr.flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT |
179 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
180
181 sz = mlx5_esw_chains_get_avail_sz_from_pool(esw, POOL_NEXT_SIZE);
182 if (!sz)
183 return ERR_PTR(-ENOSPC);
184
185 ft_attr.max_fte = sz;
186 ft_attr.level = level;
187 ft_attr.prio = prio - 1;
188 ft_attr.autogroup.max_num_groups = ESW_OFFLOADS_NUM_GROUPS;
189 ns = mlx5_get_fdb_sub_ns(esw->dev, chain);
190
191 fdb = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
192 if (IS_ERR(fdb)) {
193 esw_warn(esw->dev,
194 "Failed to create FDB table err %d (chain: %d, prio: %d, level: %d, size: %d)\n",
195 (int)PTR_ERR(fdb), chain, prio, level, sz);
196 mlx5_esw_chains_put_sz_to_pool(esw, sz);
197 return fdb;
198 }
199
200 return fdb;
201}
202
203static void
204mlx5_esw_chains_destroy_fdb_table(struct mlx5_eswitch *esw,
205 struct mlx5_flow_table *fdb)
206{
207 mlx5_esw_chains_put_sz_to_pool(esw, fdb->max_fte);
208 mlx5_destroy_flow_table(fdb);
209}
210
211static struct fdb_chain *
212mlx5_esw_chains_create_fdb_chain(struct mlx5_eswitch *esw, u32 chain)
213{
214 struct fdb_chain *fdb_chain = NULL;
215 int err;
216
217 fdb_chain = kvzalloc(sizeof(*fdb_chain), GFP_KERNEL);
218 if (!fdb_chain)
219 return ERR_PTR(-ENOMEM);
220
221 fdb_chain->esw = esw;
222 fdb_chain->chain = chain;
223
224 err = rhashtable_insert_fast(&esw_chains_ht(esw), &fdb_chain->node,
225 chain_params);
226 if (err)
227 goto err_insert;
228
229 return fdb_chain;
230
231err_insert:
232 kvfree(fdb_chain);
233 return ERR_PTR(err);
234}
235
236static void
237mlx5_esw_chains_destroy_fdb_chain(struct fdb_chain *fdb_chain)
238{
239 struct mlx5_eswitch *esw = fdb_chain->esw;
240
241 rhashtable_remove_fast(&esw_chains_ht(esw), &fdb_chain->node,
242 chain_params);
243 kvfree(fdb_chain);
244}
245
246static struct fdb_chain *
247mlx5_esw_chains_get_fdb_chain(struct mlx5_eswitch *esw, u32 chain)
248{
249 struct fdb_chain *fdb_chain;
250
251 fdb_chain = rhashtable_lookup_fast(&esw_chains_ht(esw), &chain,
252 chain_params);
253 if (!fdb_chain) {
254 fdb_chain = mlx5_esw_chains_create_fdb_chain(esw, chain);
255 if (IS_ERR(fdb_chain))
256 return fdb_chain;
257 }
258
259 fdb_chain->ref++;
260
261 return fdb_chain;
262}
263
264static void
265mlx5_esw_chains_put_fdb_chain(struct fdb_chain *fdb_chain)
266{
267 if (--fdb_chain->ref == 0)
268 mlx5_esw_chains_destroy_fdb_chain(fdb_chain);
269}
270
271static struct fdb_prio *
272mlx5_esw_chains_create_fdb_prio(struct mlx5_eswitch *esw,
273 u32 chain, u32 prio, u32 level)
274{
275 struct fdb_prio *fdb_prio = NULL;
276 struct fdb_chain *fdb_chain;
277 struct mlx5_flow_table *fdb;
278 int err;
279
280 fdb_chain = mlx5_esw_chains_get_fdb_chain(esw, chain);
281 if (IS_ERR(fdb_chain))
282 return ERR_CAST(fdb_chain);
283
284 fdb_prio = kvzalloc(sizeof(*fdb_prio), GFP_KERNEL);
285 if (!fdb_prio) {
286 err = -ENOMEM;
287 goto err_alloc;
288 }
289
290 fdb = mlx5_esw_chains_create_fdb_table(esw, fdb_chain->chain, prio,
291 level);
292 if (IS_ERR(fdb)) {
293 err = PTR_ERR(fdb);
294 goto err_create;
295 }
296
297 fdb_prio->fdb_chain = fdb_chain;
298 fdb_prio->key.chain = chain;
299 fdb_prio->key.prio = prio;
300 fdb_prio->key.level = level;
301 fdb_prio->fdb = fdb;
302
303 err = rhashtable_insert_fast(&esw_prios_ht(esw), &fdb_prio->node,
304 prio_params);
305 if (err)
306 goto err_insert;
307
308 return fdb_prio;
309
310err_insert:
311 mlx5_esw_chains_destroy_fdb_table(esw, fdb);
312err_create:
313 kvfree(fdb_prio);
314err_alloc:
315 mlx5_esw_chains_put_fdb_chain(fdb_chain);
316 return ERR_PTR(err);
317}
318
319static void
320mlx5_esw_chains_destroy_fdb_prio(struct mlx5_eswitch *esw,
321 struct fdb_prio *fdb_prio)
322{
323 struct fdb_chain *fdb_chain = fdb_prio->fdb_chain;
324
325 rhashtable_remove_fast(&esw_prios_ht(esw), &fdb_prio->node,
326 prio_params);
327 mlx5_esw_chains_destroy_fdb_table(esw, fdb_prio->fdb);
328 mlx5_esw_chains_put_fdb_chain(fdb_chain);
329 kvfree(fdb_prio);
330}
331
332struct mlx5_flow_table *
333mlx5_esw_chains_get_table(struct mlx5_eswitch *esw, u32 chain, u32 prio,
334 u32 level)
335{
336 struct mlx5_flow_table *prev_fts;
337 struct fdb_prio *fdb_prio;
338 struct fdb_prio_key key;
339 int l = 0;
340
341 if ((chain > mlx5_esw_chains_get_chain_range(esw) &&
342 chain != mlx5_esw_chains_get_ft_chain(esw)) ||
343 prio > mlx5_esw_chains_get_prio_range(esw) ||
344 level > mlx5_esw_chains_get_level_range(esw))
345 return ERR_PTR(-EOPNOTSUPP);
346
347 /* create earlier levels for correct fs_core lookup when
348 * connecting tables.
349 */
350 for (l = 0; l < level; l++) {
351 prev_fts = mlx5_esw_chains_get_table(esw, chain, prio, l);
352 if (IS_ERR(prev_fts)) {
353 fdb_prio = ERR_CAST(prev_fts);
354 goto err_get_prevs;
355 }
356 }
357
358 key.chain = chain;
359 key.prio = prio;
360 key.level = level;
361
362 mutex_lock(&esw_chains_lock(esw));
363 fdb_prio = rhashtable_lookup_fast(&esw_prios_ht(esw), &key,
364 prio_params);
365 if (!fdb_prio) {
366 fdb_prio = mlx5_esw_chains_create_fdb_prio(esw, chain,
367 prio, level);
368 if (IS_ERR(fdb_prio))
369 goto err_create_prio;
370 }
371
372 ++fdb_prio->ref;
373 mutex_unlock(&esw_chains_lock(esw));
374
375 return fdb_prio->fdb;
376
377err_create_prio:
378 mutex_unlock(&esw_chains_lock(esw));
379err_get_prevs:
380 while (--l >= 0)
381 mlx5_esw_chains_put_table(esw, chain, prio, l);
382 return ERR_CAST(fdb_prio);
383}
384
385void
386mlx5_esw_chains_put_table(struct mlx5_eswitch *esw, u32 chain, u32 prio,
387 u32 level)
388{
389 struct fdb_prio *fdb_prio;
390 struct fdb_prio_key key;
391
392 key.chain = chain;
393 key.prio = prio;
394 key.level = level;
395
396 mutex_lock(&esw_chains_lock(esw));
397 fdb_prio = rhashtable_lookup_fast(&esw_prios_ht(esw), &key,
398 prio_params);
399 if (!fdb_prio)
400 goto err_get_prio;
401
402 if (--fdb_prio->ref == 0)
403 mlx5_esw_chains_destroy_fdb_prio(esw, fdb_prio);
404 mutex_unlock(&esw_chains_lock(esw));
405
406 while (level-- > 0)
407 mlx5_esw_chains_put_table(esw, chain, prio, level);
408
409 return;
410
411err_get_prio:
412 mutex_unlock(&esw_chains_lock(esw));
413 WARN_ONCE(1,
414 "Couldn't find table: (chain: %d prio: %d level: %d)",
415 chain, prio, level);
416}
417
418static int
419mlx5_esw_chains_init(struct mlx5_eswitch *esw)
420{
421 struct mlx5_esw_chains_priv *chains_priv;
422 struct mlx5_core_dev *dev = esw->dev;
423 u32 max_flow_counter, fdb_max;
424 int err;
425
426 chains_priv = kzalloc(sizeof(*chains_priv), GFP_KERNEL);
427 if (!chains_priv)
428 return -ENOMEM;
429 esw_chains_priv(esw) = chains_priv;
430
431 max_flow_counter = (MLX5_CAP_GEN(dev, max_flow_counter_31_16) << 16) |
432 MLX5_CAP_GEN(dev, max_flow_counter_15_0);
433 fdb_max = 1 << MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size);
434
435 esw_debug(dev,
436 "Init esw offloads chains, max counters(%d), groups(%d), max flow table size(%d)\n",
437 max_flow_counter, ESW_OFFLOADS_NUM_GROUPS, fdb_max);
438
439 mlx5_esw_chains_init_sz_pool(esw);
440
441 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, multi_fdb_encap) &&
442 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) {
443 esw->fdb_table.flags &= ~ESW_FDB_CHAINS_AND_PRIOS_SUPPORTED;
444 esw_warn(dev, "Tc chains and priorities offload aren't supported, update firmware if needed\n");
445 } else {
446 esw->fdb_table.flags |= ESW_FDB_CHAINS_AND_PRIOS_SUPPORTED;
447 esw_info(dev, "Supported tc offload range - chains: %u, prios: %u\n",
448 mlx5_esw_chains_get_chain_range(esw),
449 mlx5_esw_chains_get_prio_range(esw));
450 }
451
452 err = rhashtable_init(&esw_chains_ht(esw), &chain_params);
453 if (err)
454 goto init_chains_ht_err;
455
456 err = rhashtable_init(&esw_prios_ht(esw), &prio_params);
457 if (err)
458 goto init_prios_ht_err;
459
460 mutex_init(&esw_chains_lock(esw));
461
462 return 0;
463
464init_prios_ht_err:
465 rhashtable_destroy(&esw_chains_ht(esw));
466init_chains_ht_err:
467 kfree(chains_priv);
468 return err;
469}
470
471static void
472mlx5_esw_chains_cleanup(struct mlx5_eswitch *esw)
473{
474 mutex_destroy(&esw_chains_lock(esw));
475 rhashtable_destroy(&esw_prios_ht(esw));
476 rhashtable_destroy(&esw_chains_ht(esw));
477
478 kfree(esw_chains_priv(esw));
479}
480
481static int
482mlx5_esw_chains_open(struct mlx5_eswitch *esw)
483{
484 struct mlx5_flow_table *ft;
485 int err;
486
487 /* Always open the root for fast path */
488 ft = mlx5_esw_chains_get_table(esw, 0, 1, 0);
489 if (IS_ERR(ft))
490 return PTR_ERR(ft);
491
492 /* Open level 1 for split rules now if prios isn't supported */
493 if (!mlx5_esw_chains_prios_supported(esw)) {
494 ft = mlx5_esw_chains_get_table(esw, 0, 1, 1);
495
496 if (IS_ERR(ft)) {
497 err = PTR_ERR(ft);
498 goto level_1_err;
499 }
500 }
501
502 return 0;
503
504level_1_err:
505 mlx5_esw_chains_put_table(esw, 0, 1, 0);
506 return err;
507}
508
509static void
510mlx5_esw_chains_close(struct mlx5_eswitch *esw)
511{
512 if (!mlx5_esw_chains_prios_supported(esw))
513 mlx5_esw_chains_put_table(esw, 0, 1, 1);
514 mlx5_esw_chains_put_table(esw, 0, 1, 0);
515}
516
517int
518mlx5_esw_chains_create(struct mlx5_eswitch *esw)
519{
520 int err;
521
522 err = mlx5_esw_chains_init(esw);
523 if (err)
524 return err;
525
526 err = mlx5_esw_chains_open(esw);
527 if (err)
528 goto err_open;
529
530 return 0;
531
532err_open:
533 mlx5_esw_chains_cleanup(esw);
534 return err;
535}
536
537void
538mlx5_esw_chains_destroy(struct mlx5_eswitch *esw)
539{
540 mlx5_esw_chains_close(esw);
541 mlx5_esw_chains_cleanup(esw);
542}
drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_chains.h+27−0
@@ -0,0 +1,27 @@
1/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2/* Copyright (c) 2020 Mellanox Technologies. */
3
4#ifndef __ML5_ESW_CHAINS_H__
5#define __ML5_ESW_CHAINS_H__
6
7bool
8mlx5_esw_chains_prios_supported(struct mlx5_eswitch *esw);
9u32
10mlx5_esw_chains_get_prio_range(struct mlx5_eswitch *esw);
11u32
12mlx5_esw_chains_get_chain_range(struct mlx5_eswitch *esw);
13u32
14mlx5_esw_chains_get_ft_chain(struct mlx5_eswitch *esw);
15
16struct mlx5_flow_table *
17mlx5_esw_chains_get_table(struct mlx5_eswitch *esw, u32 chain, u32 prio,
18 u32 level);
19void
20mlx5_esw_chains_put_table(struct mlx5_eswitch *esw, u32 chain, u32 prio,
21 u32 level);
22
23int mlx5_esw_chains_create(struct mlx5_eswitch *esw);
24void mlx5_esw_chains_destroy(struct mlx5_eswitch *esw);
25
26#endif /* __ML5_ESW_CHAINS_H__ */
27

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t