Move the device memory allocation and deallocation commands SW ICM memory to mlx5_core to expose this API for all mlx5_core users. This comes as preparation for supporting SW steering in kernel where it will be required to allocate and register device memory for direct rule insertion. In addition, an API to register this device memory for future remote access operations is introduced using the create_mkey commands.
Unified · Split
| @@ -186,136 +186,6 @@ int mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, u64 length) · 185 unmodified lines | ||
| 186 | 186 | return err; |
| 187 | 187 | } |
| 188 | 188 | |
| 189 | −int mlx5_cmd_alloc_sw_icm(struct mlx5_dm *dm, int type, u64 length, | |
| 190 | − u16 uid, phys_addr_t *addr, u32 *obj_id) | |
| 191 | −{ | |
| 192 | − struct mlx5_core_dev *dev = dm->dev; | |
| 193 | − u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {}; | |
| 194 | − u32 in[MLX5_ST_SZ_DW(create_sw_icm_in)] = {}; | |
| 195 | − unsigned long *block_map; | |
| 196 | − u64 icm_start_addr; | |
| 197 | − u32 log_icm_size; | |
| 198 | − u32 num_blocks; | |
| 199 | − u32 max_blocks; | |
| 200 | − u64 block_idx; | |
| 201 | − void *sw_icm; | |
| 202 | − int ret; | |
| 203 | − | |
| 204 | − MLX5_SET(general_obj_in_cmd_hdr, in, opcode, | |
| 205 | − MLX5_CMD_OP_CREATE_GENERAL_OBJECT); | |
| 206 | − MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_OBJ_TYPE_SW_ICM); | |
| 207 | − MLX5_SET(general_obj_in_cmd_hdr, in, uid, uid); | |
| 208 | − | |
| 209 | − switch (type) { | |
| 210 | − case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM: | |
| 211 | − icm_start_addr = MLX5_CAP64_DEV_MEM(dev, | |
| 212 | − steering_sw_icm_start_address); | |
| 213 | − log_icm_size = MLX5_CAP_DEV_MEM(dev, log_steering_sw_icm_size); | |
| 214 | − block_map = dm->steering_sw_icm_alloc_blocks; | |
| 215 | − break; | |
| 216 | − case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM: | |
| 217 | − icm_start_addr = MLX5_CAP64_DEV_MEM(dev, | |
| 218 | − header_modify_sw_icm_start_address); | |
| 219 | − log_icm_size = MLX5_CAP_DEV_MEM(dev, | |
| 220 | − log_header_modify_sw_icm_size); | |
| 221 | − block_map = dm->header_modify_sw_icm_alloc_blocks; | |
| 222 | − break; | |
| 223 | − default: | |
| 224 | − return -EINVAL; | |
| 225 | − } | |
| 226 | − | |
| 227 | − num_blocks = (length + MLX5_SW_ICM_BLOCK_SIZE(dev) - 1) >> | |
| 228 | − MLX5_LOG_SW_ICM_BLOCK_SIZE(dev); | |
| 229 | − max_blocks = BIT(log_icm_size - MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)); | |
| 230 | − spin_lock(&dm->lock); | |
| 231 | − block_idx = bitmap_find_next_zero_area(block_map, | |
| 232 | − max_blocks, | |
| 233 | − 0, | |
| 234 | − num_blocks, 0); | |
| 235 | − | |
| 236 | − if (block_idx < max_blocks) | |
| 237 | − bitmap_set(block_map, | |
| 238 | − block_idx, num_blocks); | |
| 239 | − | |
| 240 | − spin_unlock(&dm->lock); | |
| 241 | − | |
| 242 | − if (block_idx >= max_blocks) | |
| 243 | − return -ENOMEM; | |
| 244 | − | |
| 245 | − sw_icm = MLX5_ADDR_OF(create_sw_icm_in, in, sw_icm); | |
| 246 | − icm_start_addr += block_idx << MLX5_LOG_SW_ICM_BLOCK_SIZE(dev); | |
| 247 | − MLX5_SET64(sw_icm, sw_icm, sw_icm_start_addr, | |
| 248 | − icm_start_addr); | |
| 249 | − MLX5_SET(sw_icm, sw_icm, log_sw_icm_size, ilog2(length)); | |
| 250 | − | |
| 251 | − ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| 252 | − if (ret) { | |
| 253 | − spin_lock(&dm->lock); | |
| 254 | − bitmap_clear(block_map, | |
| 255 | − block_idx, num_blocks); | |
| 256 | − spin_unlock(&dm->lock); | |
| 257 | − | |
| 258 | − return ret; | |
| 259 | − } | |
| 260 | − | |
| 261 | − *addr = icm_start_addr; | |
| 262 | − *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); | |
| 263 | − | |
| 264 | − return 0; | |
| 265 | −} | |
| 266 | − | |
| 267 | −int mlx5_cmd_dealloc_sw_icm(struct mlx5_dm *dm, int type, u64 length, | |
| 268 | − u16 uid, phys_addr_t addr, u32 obj_id) | |
| 269 | −{ | |
| 270 | − struct mlx5_core_dev *dev = dm->dev; | |
| 271 | − u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {}; | |
| 272 | − u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {}; | |
| 273 | − unsigned long *block_map; | |
| 274 | − u32 num_blocks; | |
| 275 | − u64 start_idx; | |
| 276 | − int err; | |
| 277 | − | |
| 278 | − num_blocks = (length + MLX5_SW_ICM_BLOCK_SIZE(dev) - 1) >> | |
| 279 | − MLX5_LOG_SW_ICM_BLOCK_SIZE(dev); | |
| 280 | − | |
| 281 | − switch (type) { | |
| 282 | − case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM: | |
| 283 | − start_idx = | |
| 284 | − (addr - MLX5_CAP64_DEV_MEM( | |
| 285 | − dev, steering_sw_icm_start_address)) >> | |
| 286 | − MLX5_LOG_SW_ICM_BLOCK_SIZE(dev); | |
| 287 | − block_map = dm->steering_sw_icm_alloc_blocks; | |
| 288 | − break; | |
| 289 | − case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM: | |
| 290 | − start_idx = | |
| 291 | − (addr - | |
| 292 | − MLX5_CAP64_DEV_MEM( | |
| 293 | − dev, header_modify_sw_icm_start_address)) >> | |
| 294 | − MLX5_LOG_SW_ICM_BLOCK_SIZE(dev); | |
| 295 | − block_map = dm->header_modify_sw_icm_alloc_blocks; | |
| 296 | − break; | |
| 297 | − default: | |
| 298 | − return -EINVAL; | |
| 299 | − } | |
| 300 | − | |
| 301 | − MLX5_SET(general_obj_in_cmd_hdr, in, opcode, | |
| 302 | − MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); | |
| 303 | − MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_OBJ_TYPE_SW_ICM); | |
| 304 | − MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id); | |
| 305 | − MLX5_SET(general_obj_in_cmd_hdr, in, uid, uid); | |
| 306 | − | |
| 307 | − err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| 308 | − if (err) | |
| 309 | − return err; | |
| 310 | − | |
| 311 | − spin_lock(&dm->lock); | |
| 312 | − bitmap_clear(block_map, | |
| 313 | − start_idx, num_blocks); | |
| 314 | − spin_unlock(&dm->lock); | |
| 315 | − | |
| 316 | − return 0; | |
| 317 | −} | |
| 318 | − | |
| 319 | 189 | int mlx5_cmd_query_ext_ppcnt_counters(struct mlx5_core_dev *dev, void *out) |
| 320 | 190 | { |
| 321 | 191 | u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {}; |
| @@ -65,8 +65,4 @@ int mlx5_cmd_alloc_q_counter(struct mlx5_core_dev *dev, u16 *counter_id, · 64 unmodified lines | ||
| 65 | 65 | u16 uid); |
| 66 | 66 | int mlx5_cmd_mad_ifc(struct mlx5_core_dev *dev, const void *inb, void *outb, |
| 67 | 67 | u16 opmod, u8 port); |
| 68 | −int mlx5_cmd_alloc_sw_icm(struct mlx5_dm *dm, int type, u64 length, | |
| 69 | − u16 uid, phys_addr_t *addr, u32 *obj_id); | |
| 70 | −int mlx5_cmd_dealloc_sw_icm(struct mlx5_dm *dm, int type, u64 length, | |
| 71 | − u16 uid, phys_addr_t addr, u32 obj_id); | |
| 72 | 68 | #endif /* MLX5_IB_CMD_H */ |
| @@ -2280,6 +2280,7 @@ static inline int check_dm_type_support(struct mlx5_ib_dev *dev, · 2279 unmodified lines | ||
| 2280 | 2280 | return -EOPNOTSUPP; |
| 2281 | 2281 | break; |
| 2282 | 2282 | case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM: |
| 2283 | + case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM: | |
| 2283 | 2284 | if (!capable(CAP_SYS_RAWIO) || |
| 2284 | 2285 | !capable(CAP_NET_RAW)) |
| 2285 | 2286 | return -EPERM; |
| @@ -2344,20 +2345,20 @@ static int handle_alloc_dm_sw_icm(struct ib_ucontext *ctx, · 58 unmodified lines | ||
| 2344 | 2345 | struct uverbs_attr_bundle *attrs, |
| 2345 | 2346 | int type) |
| 2346 | 2347 | { |
| 2347 | − struct mlx5_dm *dm_db = &to_mdev(ctx->device)->dm; | |
| 2348 | + struct mlx5_core_dev *dev = to_mdev(ctx->device)->mdev; | |
| 2348 | 2349 | u64 act_size; |
| 2349 | 2350 | int err; |
| 2350 | 2351 | |
| 2351 | 2352 | /* Allocation size must a multiple of the basic block size |
| 2352 | 2353 | * and a power of 2. |
| 2353 | 2354 | */ |
| 2354 | − act_size = round_up(attr->length, MLX5_SW_ICM_BLOCK_SIZE(dm_db->dev)); | |
| 2355 | + act_size = round_up(attr->length, MLX5_SW_ICM_BLOCK_SIZE(dev)); | |
| 2355 | 2356 | act_size = roundup_pow_of_two(act_size); |
| 2356 | 2357 | |
| 2357 | 2358 | dm->size = act_size; |
| 2358 | − err = mlx5_cmd_alloc_sw_icm(dm_db, type, act_size, | |
| 2359 | − to_mucontext(ctx)->devx_uid, &dm->dev_addr, | |
| 2360 | − &dm->icm_dm.obj_id); | |
| 2359 | + err = mlx5_dm_sw_icm_alloc(dev, type, act_size, | |
| 2360 | + to_mucontext(ctx)->devx_uid, &dm->dev_addr, | |
| 2361 | + &dm->icm_dm.obj_id); | |
| 2361 | 2362 | if (err) |
| 2362 | 2363 | return err; |
| 2363 | 2364 | |
| @@ -2365,9 +2366,9 @@ static int handle_alloc_dm_sw_icm(struct ib_ucontext *ctx, · 1 unmodified line | ||
| 2365 | 2366 | MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET, |
| 2366 | 2367 | &dm->dev_addr, sizeof(dm->dev_addr)); |
| 2367 | 2368 | if (err) |
| 2368 | − mlx5_cmd_dealloc_sw_icm(dm_db, type, dm->size, | |
| 2369 | − to_mucontext(ctx)->devx_uid, | |
| 2370 | − dm->dev_addr, dm->icm_dm.obj_id); | |
| 2369 | + mlx5_dm_sw_icm_dealloc(dev, type, dm->size, | |
| 2370 | + to_mucontext(ctx)->devx_uid, dm->dev_addr, | |
| 2371 | + dm->icm_dm.obj_id); | |
| 2371 | 2372 | |
| 2372 | 2373 | return err; |
| 2373 | 2374 | } |
| @@ -2407,8 +2408,14 @@ struct ib_dm *mlx5_ib_alloc_dm(struct ib_device *ibdev, · 33 unmodified lines | ||
| 2407 | 2408 | attrs); |
| 2408 | 2409 | break; |
| 2409 | 2410 | case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM: |
| 2411 | + err = handle_alloc_dm_sw_icm(context, dm, | |
| 2412 | + attr, attrs, | |
| 2413 | + MLX5_SW_ICM_TYPE_STEERING); | |
| 2414 | + break; | |
| 2410 | 2415 | case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM: |
| 2411 | − err = handle_alloc_dm_sw_icm(context, dm, attr, attrs, type); | |
| 2416 | + err = handle_alloc_dm_sw_icm(context, dm, | |
| 2417 | + attr, attrs, | |
| 2418 | + MLX5_SW_ICM_TYPE_HEADER_MODIFY); | |
| 2412 | 2419 | break; |
| 2413 | 2420 | default: |
| 2414 | 2421 | err = -EOPNOTSUPP; |
| @@ -2428,6 +2435,7 @@ int mlx5_ib_dealloc_dm(struct ib_dm *ibdm, struct uverbs_attr_bundle *attrs) · 13 unmodified lines | ||
| 2428 | 2435 | { |
| 2429 | 2436 | struct mlx5_ib_ucontext *ctx = rdma_udata_to_drv_context( |
| 2430 | 2437 | &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext); |
| 2438 | + struct mlx5_core_dev *dev = to_mdev(ibdm->device)->mdev; | |
| 2431 | 2439 | struct mlx5_dm *dm_db = &to_mdev(ibdm->device)->dm; |
| 2432 | 2440 | struct mlx5_ib_dm *dm = to_mdm(ibdm); |
| 2433 | 2441 | u32 page_idx; |
| @@ -2439,19 +2447,23 @@ int mlx5_ib_dealloc_dm(struct ib_dm *ibdm, struct uverbs_attr_bundle *attrs) · 5 unmodified lines | ||
| 2439 | 2447 | if (ret) |
| 2440 | 2448 | return ret; |
| 2441 | 2449 | |
| 2442 | − page_idx = (dm->dev_addr - | |
| 2443 | − pci_resource_start(dm_db->dev->pdev, 0) - | |
| 2444 | − MLX5_CAP64_DEV_MEM(dm_db->dev, | |
| 2445 | − memic_bar_start_addr)) >> | |
| 2446 | − PAGE_SHIFT; | |
| 2450 | + page_idx = (dm->dev_addr - pci_resource_start(dev->pdev, 0) - | |
| 2451 | + MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr)) >> | |
| 2452 | + PAGE_SHIFT; | |
| 2447 | 2453 | bitmap_clear(ctx->dm_pages, page_idx, |
| 2448 | 2454 | DIV_ROUND_UP(dm->size, PAGE_SIZE)); |
| 2449 | 2455 | break; |
| 2450 | 2456 | case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM: |
| 2457 | + ret = mlx5_dm_sw_icm_dealloc(dev, MLX5_SW_ICM_TYPE_STEERING, | |
| 2458 | + dm->size, ctx->devx_uid, dm->dev_addr, | |
| 2459 | + dm->icm_dm.obj_id); | |
| 2460 | + if (ret) | |
| 2461 | + return ret; | |
| 2462 | + break; | |
| 2451 | 2463 | case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM: |
| 2452 | − ret = mlx5_cmd_dealloc_sw_icm(dm_db, dm->type, dm->size, | |
| 2453 | − ctx->devx_uid, dm->dev_addr, | |
| 2454 | − dm->icm_dm.obj_id); | |
| 2464 | + ret = mlx5_dm_sw_icm_dealloc(dev, MLX5_SW_ICM_TYPE_HEADER_MODIFY, | |
| 2465 | + dm->size, ctx->devx_uid, dm->dev_addr, | |
| 2466 | + dm->icm_dm.obj_id); | |
| 2455 | 2467 | if (ret) |
| 2456 | 2468 | return ret; |
| 2457 | 2469 | break; |
| @@ -6097,8 +6109,6 @@ static struct ib_counters *mlx5_ib_create_counters(struct ib_device *device, · 3639 unmodified lines | ||
| 6097 | 6109 | |
| 6098 | 6110 | static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev) |
| 6099 | 6111 | { |
| 6100 | − struct mlx5_core_dev *mdev = dev->mdev; | |
| 6101 | − | |
| 6102 | 6112 | mlx5_ib_cleanup_multiport_master(dev); |
| 6103 | 6113 | if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) { |
| 6104 | 6114 | srcu_barrier(&dev->mr_srcu); |
| @@ -6106,29 +6116,11 @@ static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev) · 1 unmodified line | ||
| 6106 | 6116 | } |
| 6107 | 6117 | |
| 6108 | 6118 | WARN_ON(!bitmap_empty(dev->dm.memic_alloc_pages, MLX5_MAX_MEMIC_PAGES)); |
| 6109 | − | |
| 6110 | − WARN_ON(dev->dm.steering_sw_icm_alloc_blocks && | |
| 6111 | − !bitmap_empty( | |
| 6112 | − dev->dm.steering_sw_icm_alloc_blocks, | |
| 6113 | − BIT(MLX5_CAP_DEV_MEM(mdev, log_steering_sw_icm_size) - | |
| 6114 | − MLX5_LOG_SW_ICM_BLOCK_SIZE(mdev)))); | |
| 6115 | − | |
| 6116 | − kfree(dev->dm.steering_sw_icm_alloc_blocks); | |
| 6117 | − | |
| 6118 | − WARN_ON(dev->dm.header_modify_sw_icm_alloc_blocks && | |
| 6119 | − !bitmap_empty(dev->dm.header_modify_sw_icm_alloc_blocks, | |
| 6120 | − BIT(MLX5_CAP_DEV_MEM( | |
| 6121 | − mdev, log_header_modify_sw_icm_size) - | |
| 6122 | − MLX5_LOG_SW_ICM_BLOCK_SIZE(mdev)))); | |
| 6123 | − | |
| 6124 | − kfree(dev->dm.header_modify_sw_icm_alloc_blocks); | |
| 6125 | 6119 | } |
| 6126 | 6120 | |
| 6127 | 6121 | static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev) |
| 6128 | 6122 | { |
| 6129 | 6123 | struct mlx5_core_dev *mdev = dev->mdev; |
| 6130 | − u64 header_modify_icm_blocks = 0; | |
| 6131 | − u64 steering_icm_blocks = 0; | |
| 6132 | 6124 | int err; |
| 6133 | 6125 | int i; |
| 6134 | 6126 | |
| @@ -6173,51 +6165,17 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev) · 38 unmodified lines | ||
| 6173 | 6165 | INIT_LIST_HEAD(&dev->qp_list); |
| 6174 | 6166 | spin_lock_init(&dev->reset_flow_resource_lock); |
| 6175 | 6167 | |
| 6176 | − if (MLX5_CAP_GEN_64(mdev, general_obj_types) & | |
| 6177 | − MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM) { | |
| 6178 | − if (MLX5_CAP64_DEV_MEM(mdev, steering_sw_icm_start_address)) { | |
| 6179 | − steering_icm_blocks = | |
| 6180 | − BIT(MLX5_CAP_DEV_MEM(mdev, | |
| 6181 | − log_steering_sw_icm_size) - | |
| 6182 | − MLX5_LOG_SW_ICM_BLOCK_SIZE(mdev)); | |
| 6183 | − | |
| 6184 | − dev->dm.steering_sw_icm_alloc_blocks = | |
| 6185 | − kcalloc(BITS_TO_LONGS(steering_icm_blocks), | |
| 6186 | − sizeof(unsigned long), GFP_KERNEL); | |
| 6187 | − if (!dev->dm.steering_sw_icm_alloc_blocks) | |
| 6188 | − goto err_mp; | |
| 6189 | − } | |
| 6190 | − | |
| 6191 | − if (MLX5_CAP64_DEV_MEM(mdev, | |
| 6192 | − header_modify_sw_icm_start_address)) { | |
| 6193 | − header_modify_icm_blocks = BIT( | |
| 6194 | − MLX5_CAP_DEV_MEM( | |
| 6195 | − mdev, log_header_modify_sw_icm_size) - | |
| 6196 | − MLX5_LOG_SW_ICM_BLOCK_SIZE(mdev)); | |
| 6197 | − | |
| 6198 | − dev->dm.header_modify_sw_icm_alloc_blocks = | |
| 6199 | − kcalloc(BITS_TO_LONGS(header_modify_icm_blocks), | |
| 6200 | − sizeof(unsigned long), GFP_KERNEL); | |
| 6201 | − if (!dev->dm.header_modify_sw_icm_alloc_blocks) | |
| 6202 | − goto err_dm; | |
| 6203 | − } | |
| 6204 | − } | |
| 6205 | − | |
| 6206 | 6168 | spin_lock_init(&dev->dm.lock); |
| 6207 | 6169 | dev->dm.dev = mdev; |
| 6208 | 6170 | |
| 6209 | 6171 | if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) { |
| 6210 | 6172 | err = init_srcu_struct(&dev->mr_srcu); |
| 6211 | 6173 | if (err) |
| 6212 | − goto err_dm; | |
| 6174 | + goto err_mp; | |
| 6213 | 6175 | } |
| 6214 | 6176 | |
| 6215 | 6177 | return 0; |
| 6216 | 6178 | |
| 6217 | −err_dm: | |
| 6218 | − kfree(dev->dm.steering_sw_icm_alloc_blocks); | |
| 6219 | − kfree(dev->dm.header_modify_sw_icm_alloc_blocks); | |
| 6220 | − | |
| 6221 | 6179 | err_mp: |
| 6222 | 6180 | mlx5_ib_cleanup_multiport_master(dev); |
| 6223 | 6181 | |
| @@ -880,8 +880,6 @@ struct mlx5_dm { · 879 unmodified lines | ||
| 880 | 880 | */ |
| 881 | 881 | spinlock_t lock; |
| 882 | 882 | DECLARE_BITMAP(memic_alloc_pages, MLX5_MAX_MEMIC_PAGES); |
| 883 | − unsigned long *steering_sw_icm_alloc_blocks; | |
| 884 | − unsigned long *header_modify_sw_icm_alloc_blocks; | |
| 885 | 883 | }; |
| 886 | 884 | |
| 887 | 885 | struct mlx5_read_counters_attr { |
| @@ -15,7 +15,7 @@ mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \ · 14 unmodified lines | ||
| 15 | 15 | health.o mcg.o cq.o alloc.o qp.o port.o mr.o pd.o \ |
| 16 | 16 | transobj.o vport.o sriov.o fs_cmd.o fs_core.o pci_irq.o \ |
| 17 | 17 | fs_counters.o rl.o lag.o dev.o events.o wq.o lib/gid.o \ |
| 18 | − lib/devcom.o lib/pci_vsc.o diag/fs_tracepoint.o \ | |
| 18 | + lib/devcom.o lib/pci_vsc.o lib/dm.o diag/fs_tracepoint.o \ | |
| 19 | 19 | diag/fw_tracer.o diag/crdump.o devlink.o |
| 20 | 20 | |
| 21 | 21 | # |
| @@ -0,0 +1,223 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB | |
| 2 | +// Copyright (c) 2019 Mellanox Technologies | |
| 3 | + | |
| 4 | +#include <linux/mlx5/driver.h> | |
| 5 | +#include <linux/mlx5/device.h> | |
| 6 | + | |
| 7 | +#include "mlx5_core.h" | |
| 8 | +#include "lib/mlx5.h" | |
| 9 | + | |
| 10 | +struct mlx5_dm { | |
| 11 | + /* protect access to icm bitmask */ | |
| 12 | + spinlock_t lock; | |
| 13 | + unsigned long *steering_sw_icm_alloc_blocks; | |
| 14 | + unsigned long *header_modify_sw_icm_alloc_blocks; | |
| 15 | +}; | |
| 16 | + | |
| 17 | +struct mlx5_dm *mlx5_dm_create(struct mlx5_core_dev *dev) | |
| 18 | +{ | |
| 19 | + u64 header_modify_icm_blocks = 0; | |
| 20 | + u64 steering_icm_blocks = 0; | |
| 21 | + struct mlx5_dm *dm; | |
| 22 | + | |
| 23 | + if (!(MLX5_CAP_GEN_64(dev, general_obj_types) & MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM)) | |
| 24 | + return 0; | |
| 25 | + | |
| 26 | + dm = kzalloc(sizeof(*dm), GFP_KERNEL); | |
| 27 | + if (!dm) | |
| 28 | + return ERR_PTR(-ENOMEM); | |
| 29 | + | |
| 30 | + spin_lock_init(&dm->lock); | |
| 31 | + | |
| 32 | + if (MLX5_CAP64_DEV_MEM(dev, steering_sw_icm_start_address)) { | |
| 33 | + steering_icm_blocks = | |
| 34 | + BIT(MLX5_CAP_DEV_MEM(dev, log_steering_sw_icm_size) - | |
| 35 | + MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)); | |
| 36 | + | |
| 37 | + dm->steering_sw_icm_alloc_blocks = | |
| 38 | + kcalloc(BITS_TO_LONGS(steering_icm_blocks), | |
| 39 | + sizeof(unsigned long), GFP_KERNEL); | |
| 40 | + if (!dm->steering_sw_icm_alloc_blocks) | |
| 41 | + goto err_steering; | |
| 42 | + } | |
| 43 | + | |
| 44 | + if (MLX5_CAP64_DEV_MEM(dev, header_modify_sw_icm_start_address)) { | |
| 45 | + header_modify_icm_blocks = | |
| 46 | + BIT(MLX5_CAP_DEV_MEM(dev, log_header_modify_sw_icm_size) - | |
| 47 | + MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)); | |
| 48 | + | |
| 49 | + dm->header_modify_sw_icm_alloc_blocks = | |
| 50 | + kcalloc(BITS_TO_LONGS(header_modify_icm_blocks), | |
| 51 | + sizeof(unsigned long), GFP_KERNEL); | |
| 52 | + if (!dm->header_modify_sw_icm_alloc_blocks) | |
| 53 | + goto err_modify_hdr; | |
| 54 | + } | |
| 55 | + | |
| 56 | + return dm; | |
| 57 | + | |
| 58 | +err_modify_hdr: | |
| 59 | + kfree(dm->steering_sw_icm_alloc_blocks); | |
| 60 | + | |
| 61 | +err_steering: | |
| 62 | + kfree(dm); | |
| 63 | + | |
| 64 | + return ERR_PTR(-ENOMEM); | |
| 65 | +} | |
| 66 | + | |
| 67 | +void mlx5_dm_cleanup(struct mlx5_core_dev *dev) | |
| 68 | +{ | |
| 69 | + struct mlx5_dm *dm = dev->dm; | |
| 70 | + | |
| 71 | + if (!dev->dm) | |
| 72 | + return; | |
| 73 | + | |
| 74 | + if (dm->steering_sw_icm_alloc_blocks) { | |
| 75 | + WARN_ON(!bitmap_empty(dm->steering_sw_icm_alloc_blocks, | |
| 76 | + BIT(MLX5_CAP_DEV_MEM(dev, log_steering_sw_icm_size) - | |
| 77 | + MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)))); | |
| 78 | + kfree(dm->steering_sw_icm_alloc_blocks); | |
| 79 | + } | |
| 80 | + | |
| 81 | + if (dm->header_modify_sw_icm_alloc_blocks) { | |
| 82 | + WARN_ON(!bitmap_empty(dm->header_modify_sw_icm_alloc_blocks, | |
| 83 | + BIT(MLX5_CAP_DEV_MEM(dev, | |
| 84 | + log_header_modify_sw_icm_size) - | |
| 85 | + MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)))); | |
| 86 | + kfree(dm->header_modify_sw_icm_alloc_blocks); | |
| 87 | + } | |
| 88 | + | |
| 89 | + kfree(dm); | |
| 90 | +} | |
| 91 | + | |
| 92 | +int mlx5_dm_sw_icm_alloc(struct mlx5_core_dev *dev, enum mlx5_sw_icm_type type, | |
| 93 | + u64 length, u16 uid, phys_addr_t *addr, u32 *obj_id) | |
| 94 | +{ | |
| 95 | + u32 num_blocks = DIV_ROUND_UP_ULL(length, MLX5_SW_ICM_BLOCK_SIZE(dev)); | |
| 96 | + u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {}; | |
| 97 | + u32 in[MLX5_ST_SZ_DW(create_sw_icm_in)] = {}; | |
| 98 | + struct mlx5_dm *dm = dev->dm; | |
| 99 | + unsigned long *block_map; | |
| 100 | + u64 icm_start_addr; | |
| 101 | + u32 log_icm_size; | |
| 102 | + u32 max_blocks; | |
| 103 | + u64 block_idx; | |
| 104 | + void *sw_icm; | |
| 105 | + int ret; | |
| 106 | + | |
| 107 | + if (!dev->dm) | |
| 108 | + return -EOPNOTSUPP; | |
| 109 | + | |
| 110 | + if (!length || (length & (length - 1)) || | |
| 111 | + length & (MLX5_SW_ICM_BLOCK_SIZE(dev) - 1)) | |
| 112 | + return -EINVAL; | |
| 113 | + | |
| 114 | + MLX5_SET(general_obj_in_cmd_hdr, in, opcode, | |
| 115 | + MLX5_CMD_OP_CREATE_GENERAL_OBJECT); | |
| 116 | + MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_OBJ_TYPE_SW_ICM); | |
| 117 | + MLX5_SET(general_obj_in_cmd_hdr, in, uid, uid); | |
| 118 | + | |
| 119 | + switch (type) { | |
| 120 | + case MLX5_SW_ICM_TYPE_STEERING: | |
| 121 | + icm_start_addr = MLX5_CAP64_DEV_MEM(dev, steering_sw_icm_start_address); | |
| 122 | + log_icm_size = MLX5_CAP_DEV_MEM(dev, log_steering_sw_icm_size); | |
| 123 | + block_map = dm->steering_sw_icm_alloc_blocks; | |
| 124 | + break; | |
| 125 | + case MLX5_SW_ICM_TYPE_HEADER_MODIFY: | |
| 126 | + icm_start_addr = MLX5_CAP64_DEV_MEM(dev, header_modify_sw_icm_start_address); | |
| 127 | + log_icm_size = MLX5_CAP_DEV_MEM(dev, | |
| 128 | + log_header_modify_sw_icm_size); | |
| 129 | + block_map = dm->header_modify_sw_icm_alloc_blocks; | |
| 130 | + break; | |
| 131 | + default: | |
| 132 | + return -EINVAL; | |
| 133 | + } | |
| 134 | + | |
| 135 | + if (!block_map) | |
| 136 | + return -EOPNOTSUPP; | |
| 137 | + | |
| 138 | + max_blocks = BIT(log_icm_size - MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)); | |
| 139 | + spin_lock(&dm->lock); | |
| 140 | + block_idx = bitmap_find_next_zero_area(block_map, | |
| 141 | + max_blocks, | |
| 142 | + 0, | |
| 143 | + num_blocks, 0); | |
| 144 | + | |
| 145 | + if (block_idx < max_blocks) | |
| 146 | + bitmap_set(block_map, | |
| 147 | + block_idx, num_blocks); | |
| 148 | + | |
| 149 | + spin_unlock(&dm->lock); | |
| 150 | + | |
| 151 | + if (block_idx >= max_blocks) | |
| 152 | + return -ENOMEM; | |
| 153 | + | |
| 154 | + sw_icm = MLX5_ADDR_OF(create_sw_icm_in, in, sw_icm); | |
| 155 | + icm_start_addr += block_idx << MLX5_LOG_SW_ICM_BLOCK_SIZE(dev); | |
| 156 | + MLX5_SET64(sw_icm, sw_icm, sw_icm_start_addr, | |
| 157 | + icm_start_addr); | |
| 158 | + MLX5_SET(sw_icm, sw_icm, log_sw_icm_size, ilog2(length)); | |
| 159 | + | |
| 160 | + ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| 161 | + if (ret) { | |
| 162 | + spin_lock(&dm->lock); | |
| 163 | + bitmap_clear(block_map, | |
| 164 | + block_idx, num_blocks); | |
| 165 | + spin_unlock(&dm->lock); | |
| 166 | + | |
| 167 | + return ret; | |
| 168 | + } | |
| 169 | + | |
| 170 | + *addr = icm_start_addr; | |
| 171 | + *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); | |
| 172 | + | |
| 173 | + return 0; | |
| 174 | +} | |
| 175 | +EXPORT_SYMBOL_GPL(mlx5_dm_sw_icm_alloc); | |
| 176 | + | |
| 177 | +int mlx5_dm_sw_icm_dealloc(struct mlx5_core_dev *dev, enum mlx5_sw_icm_type type, | |
| 178 | + u64 length, u16 uid, phys_addr_t addr, u32 obj_id) | |
| 179 | +{ | |
| 180 | + u32 num_blocks = DIV_ROUND_UP_ULL(length, MLX5_SW_ICM_BLOCK_SIZE(dev)); | |
| 181 | + u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {}; | |
| 182 | + u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {}; | |
| 183 | + struct mlx5_dm *dm = dev->dm; | |
| 184 | + unsigned long *block_map; | |
| 185 | + u64 icm_start_addr; | |
| 186 | + u64 start_idx; | |
| 187 | + int err; | |
| 188 | + | |
| 189 | + if (!dev->dm) | |
| 190 | + return -EOPNOTSUPP; | |
| 191 | + | |
| 192 | + switch (type) { | |
| 193 | + case MLX5_SW_ICM_TYPE_STEERING: | |
| 194 | + icm_start_addr = MLX5_CAP64_DEV_MEM(dev, steering_sw_icm_start_address); | |
| 195 | + block_map = dm->steering_sw_icm_alloc_blocks; | |
| 196 | + break; | |
| 197 | + case MLX5_SW_ICM_TYPE_HEADER_MODIFY: | |
| 198 | + icm_start_addr = MLX5_CAP64_DEV_MEM(dev, header_modify_sw_icm_start_address); | |
| 199 | + block_map = dm->header_modify_sw_icm_alloc_blocks; | |
| 200 | + break; | |
| 201 | + default: | |
| 202 | + return -EINVAL; | |
| 203 | + } | |
| 204 | + | |
| 205 | + MLX5_SET(general_obj_in_cmd_hdr, in, opcode, | |
| 206 | + MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); | |
| 207 | + MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_OBJ_TYPE_SW_ICM); | |
| 208 | + MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id); | |
| 209 | + MLX5_SET(general_obj_in_cmd_hdr, in, uid, uid); | |
| 210 | + | |
| 211 | + err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| 212 | + if (err) | |
| 213 | + return err; | |
| 214 | + | |
| 215 | + start_idx = (addr - icm_start_addr) >> MLX5_LOG_SW_ICM_BLOCK_SIZE(dev); | |
| 216 | + spin_lock(&dm->lock); | |
| 217 | + bitmap_clear(block_map, | |
| 218 | + start_idx, num_blocks); | |
| 219 | + spin_unlock(&dm->lock); | |
| 220 | + | |
| 221 | + return 0; | |
| 222 | +} | |
| 223 | +EXPORT_SYMBOL_GPL(mlx5_dm_sw_icm_dealloc); | |
| @@ -879,6 +879,10 @@ static int mlx5_init_once(struct mlx5_core_dev *dev) · 878 unmodified lines | ||
| 879 | 879 | goto err_eswitch_cleanup; |
| 880 | 880 | } |
| 881 | 881 | |
| 882 | + dev->dm = mlx5_dm_create(dev); | |
| 883 | + if (IS_ERR(dev->dm)) | |
| 884 | + mlx5_core_warn(dev, "Failed to init device memory%d\n", err); | |
| 885 | + | |
| 882 | 886 | dev->tracer = mlx5_fw_tracer_create(dev); |
| 883 | 887 | |
| 884 | 888 | return 0; |
| @@ -912,6 +916,7 @@ err_devcom: · 27 unmodified lines | ||
| 912 | 916 | static void mlx5_cleanup_once(struct mlx5_core_dev *dev) |
| 913 | 917 | { |
| 914 | 918 | mlx5_fw_tracer_destroy(dev->tracer); |
| 919 | + mlx5_dm_cleanup(dev); | |
| 915 | 920 | mlx5_fpga_cleanup(dev); |
| 916 | 921 | mlx5_eswitch_cleanup(dev->priv.eswitch); |
| 917 | 922 | mlx5_sriov_cleanup(dev); |
| @@ -198,6 +198,9 @@ int mlx5_set_mtpps(struct mlx5_core_dev *mdev, u32 *mtpps, u32 mtpps_size); · 197 unmodified lines | ||
| 198 | 198 | int mlx5_query_mtppse(struct mlx5_core_dev *mdev, u8 pin, u8 *arm, u8 *mode); |
| 199 | 199 | int mlx5_set_mtppse(struct mlx5_core_dev *mdev, u8 pin, u8 arm, u8 mode); |
| 200 | 200 | |
| 201 | +struct mlx5_dm *mlx5_dm_create(struct mlx5_core_dev *dev); | |
| 202 | +void mlx5_dm_cleanup(struct mlx5_core_dev *dev); | |
| 203 | + | |
| 201 | 204 | #define MLX5_PPS_CAP(mdev) (MLX5_CAP_GEN((mdev), pps) && \ |
| 202 | 205 | MLX5_CAP_GEN((mdev), pps_modify) && \ |
| 203 | 206 | MLX5_CAP_MCAM_FEATURE((mdev), mtpps_fs) && \ |
| @@ -622,6 +622,11 @@ struct mlx5e_resources { · 621 unmodified lines | ||
| 622 | 622 | struct mlx5_sq_bfreg bfreg; |
| 623 | 623 | }; |
| 624 | 624 | |
| 625 | +enum mlx5_sw_icm_type { | |
| 626 | + MLX5_SW_ICM_TYPE_STEERING, | |
| 627 | + MLX5_SW_ICM_TYPE_HEADER_MODIFY, | |
| 628 | +}; | |
| 629 | + | |
| 625 | 630 | #define MLX5_MAX_RESERVED_GIDS 8 |
| 626 | 631 | |
| 627 | 632 | struct mlx5_rsvd_gids { |
| @@ -653,10 +658,14 @@ struct mlx5_clock { · 25 unmodified lines | ||
| 653 | 658 | struct mlx5_pps pps_info; |
| 654 | 659 | }; |
| 655 | 660 | |
| 661 | +struct mlx5_dm; | |
| 656 | 662 | struct mlx5_fw_tracer; |
| 657 | 663 | struct mlx5_vxlan; |
| 658 | 664 | struct mlx5_geneve; |
| 659 | 665 | |
| 666 | +#define MLX5_LOG_SW_ICM_BLOCK_SIZE(dev) (MLX5_CAP_DEV_MEM(dev, log_sw_icm_alloc_granularity)) | |
| 667 | +#define MLX5_SW_ICM_BLOCK_SIZE(dev) (1 << MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)) | |
| 668 | + | |
| 660 | 669 | struct mlx5_core_dev { |
| 661 | 670 | struct device *device; |
| 662 | 671 | enum mlx5_coredev_type coredev_type; |
| @@ -690,6 +699,7 @@ struct mlx5_core_dev { · 27 unmodified lines | ||
| 690 | 699 | atomic_t num_qps; |
| 691 | 700 | u32 issi; |
| 692 | 701 | struct mlx5e_resources mlx5e_res; |
| 702 | + struct mlx5_dm *dm; | |
| 693 | 703 | struct mlx5_vxlan *vxlan; |
| 694 | 704 | struct mlx5_geneve *geneve; |
| 695 | 705 | struct { |
| @@ -1072,6 +1082,10 @@ int mlx5_lag_query_cong_counters(struct mlx5_core_dev *dev, · 376 unmodified lines | ||
| 1072 | 1082 | size_t *offsets); |
| 1073 | 1083 | struct mlx5_uars_page *mlx5_get_uars_page(struct mlx5_core_dev *mdev); |
| 1074 | 1084 | void mlx5_put_uars_page(struct mlx5_core_dev *mdev, struct mlx5_uars_page *up); |
| 1085 | +int mlx5_dm_sw_icm_alloc(struct mlx5_core_dev *dev, enum mlx5_sw_icm_type type, | |
| 1086 | + u64 length, u16 uid, phys_addr_t *addr, u32 *obj_id); | |
| 1087 | +int mlx5_dm_sw_icm_dealloc(struct mlx5_core_dev *dev, enum mlx5_sw_icm_type type, | |
| 1088 | + u64 length, u16 uid, phys_addr_t addr, u32 obj_id); | |
| 1075 | 1089 | |
| 1076 | 1090 | #ifdef CONFIG_MLX5_CORE_IPOIB |
| 1077 | 1091 | struct net_device *mlx5_rdma_netdev_alloc(struct mlx5_core_dev *mdev, |