mlx5 devices (Connect-IB, ConnectX-4, ConnectX-4-LX) has a limitation where rdma read work queue entries cannot exceed 512 bytes. A rdma_read wqe needs to fit in 512 bytes: - wqe control segment (16 bytes) - rdma segment (16 bytes) - scatter elements (16 bytes each) So max_sge_rd should be: (512 - 16 - 16) / 16 = 30.
Unified · Split
| @@ -530,7 +530,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev, · 529 unmodified lines | ||
| 530 | 530 | sizeof(struct mlx5_wqe_ctrl_seg)) / |
| 531 | 531 | sizeof(struct mlx5_wqe_data_seg); |
| 532 | 532 | props->max_sge = min(max_rq_sg, max_sq_sg); |
| 533 | − props->max_sge_rd = props->max_sge; | |
| 533 | + props->max_sge_rd = MLX5_MAX_SGE_RD; | |
| 534 | 534 | props->max_cq = 1 << MLX5_CAP_GEN(mdev, log_max_cq); |
| 535 | 535 | props->max_cqe = (1 << MLX5_CAP_GEN(mdev, log_max_cq_sz)) - 1; |
| 536 | 536 | props->max_mr = 1 << MLX5_CAP_GEN(mdev, log_max_mkey); |
| @@ -392,6 +392,17 @@ enum { · 391 unmodified lines | ||
| 392 | 392 | MLX5_CAP_OFF_CMDIF_CSUM = 46, |
| 393 | 393 | }; |
| 394 | 394 | |
| 395 | +enum { | |
| 396 | + /* | |
| 397 | + * Max wqe size for rdma read is 512 bytes, so this | |
| 398 | + * limits our max_sge_rd as the wqe needs to fit: | |
| 399 | + * - ctrl segment (16 bytes) | |
| 400 | + * - rdma segment (16 bytes) | |
| 401 | + * - scatter elements (16 bytes each) | |
| 402 | + */ | |
| 403 | + MLX5_MAX_SGE_RD = (512 - 16 - 16) / 16 | |
| 404 | +}; | |
| 405 | + | |
| 395 | 406 | struct mlx5_inbox_hdr { |
| 396 | 407 | __be16 opcode; |
| 397 | 408 | u8 rsvd[4]; |