IB/mlx5: Expose correct max_sge_rd limit

mlx5 devices (Connect-IB, ConnectX-4, ConnectX-4-LX) has a limitation where rdma read work queue entries cannot exceed 512 bytes. A rdma_read wqe needs to fit in 512 bytes: - wqe control segment (16 bytes) - rdma segment (16 bytes) - scatter elements (16 bytes each) So max_sge_rd should be: (512 - 16 - 16) / 16 = 30.

Sagi Grimberg committed 10 years ago

Commit 986ef951 parent 32cc92c2 files · +12 −1Browse files

Unified · Split

2 files

drivers/infiniband/hw/mlx5/main.c +1 −1
include/linux/mlx5/device.h +11 −0

drivers/infiniband/hw/mlx5/main.c+1−1

@@ -530,7 +530,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev, · 529 unmodified lines
530	sizeof(struct mlx5_wqe_ctrl_seg)) /	530	sizeof(struct mlx5_wqe_ctrl_seg)) /
531	sizeof(struct mlx5_wqe_data_seg);	531	sizeof(struct mlx5_wqe_data_seg);
532	props->max_sge = min(max_rq_sg, max_sq_sg);	532	props->max_sge = min(max_rq_sg, max_sq_sg);
533	props->max_sge_rd = props->max_sge;	533	props->max_sge_rd = MLX5_MAX_SGE_RD;
534	props->max_cq = 1 << MLX5_CAP_GEN(mdev, log_max_cq);	534	props->max_cq = 1 << MLX5_CAP_GEN(mdev, log_max_cq);
535	props->max_cqe = (1 << MLX5_CAP_GEN(mdev, log_max_cq_sz)) - 1;	535	props->max_cqe = (1 << MLX5_CAP_GEN(mdev, log_max_cq_sz)) - 1;
536	props->max_mr = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);	536	props->max_mr = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);

include/linux/mlx5/device.h+11−0

@@ -392,6 +392,17 @@ enum { · 391 unmodified lines
392	MLX5_CAP_OFF_CMDIF_CSUM = 46,	392	MLX5_CAP_OFF_CMDIF_CSUM = 46,
393	};	393	};
394		394
		395	enum {
		396	/*
		397	* Max wqe size for rdma read is 512 bytes, so this
		398	* limits our max_sge_rd as the wqe needs to fit:
		399	* - ctrl segment (16 bytes)
		400	* - rdma segment (16 bytes)
		401	* - scatter elements (16 bytes each)
		402	*/
		403	MLX5_MAX_SGE_RD = (512 - 16 - 16) / 16
		404	};
		405
395	struct mlx5_inbox_hdr {	406	struct mlx5_inbox_hdr {
396	__be16 opcode;	407	__be16 opcode;
397	u8 rsvd[4];	408	u8 rsvd[4];