lael-labs/linux

drivers/infiniband/core/cma.c+159−141

@@ -479,13 +479,20 @@ static int compare_netdev_and_ip(int ifindex_a, struct sockaddr *sa, · 478 unmodified lines
479	if (sa->sa_family != sb->sa_family)	479	if (sa->sa_family != sb->sa_family)
480	return sa->sa_family - sb->sa_family;	480	return sa->sa_family - sb->sa_family;
481		481
482	if (sa->sa_family == AF_INET)	482	if (sa->sa_family == AF_INET &&
483	return memcmp((char )&((struct sockaddr_in )sa)->sin_addr,	483	__builtin_object_size(sa, 0) >= sizeof(struct sockaddr_in)) {
484	(char )&((struct sockaddr_in )sb)->sin_addr,	484	return memcmp(&((struct sockaddr_in *)sa)->sin_addr,
		485	&((struct sockaddr_in *)sb)->sin_addr,
485	sizeof(((struct sockaddr_in *)sa)->sin_addr));	486	sizeof(((struct sockaddr_in *)sa)->sin_addr));
		487	}
		488
		489	if (sa->sa_family == AF_INET6 &&
		490	__builtin_object_size(sa, 0) >= sizeof(struct sockaddr_in6)) {
		491	return ipv6_addr_cmp(&((struct sockaddr_in6 *)sa)->sin6_addr,
		492	&((struct sockaddr_in6 *)sb)->sin6_addr);
		493	}
486		494
487	return ipv6_addr_cmp(&((struct sockaddr_in6 *)sa)->sin6_addr,	495	return -1;
488	&((struct sockaddr_in6 *)sb)->sin6_addr);
489	}	496	}
490		497
491	static int cma_add_id_to_tree(struct rdma_id_private *node_id_priv)	498	static int cma_add_id_to_tree(struct rdma_id_private *node_id_priv)
@@ -2819,8 +2826,8 @@ int rdma_set_min_rnr_timer(struct rdma_cm_id *id, u8 min_rnr_timer) · 2327 unmodified lines
2819	}	2826	}
2820	EXPORT_SYMBOL(rdma_set_min_rnr_timer);	2827	EXPORT_SYMBOL(rdma_set_min_rnr_timer);
2821		2828
2822	static void route_set_path_rec_inbound(struct cma_work *work,	2829	static int route_set_path_rec_inbound(struct cma_work *work,
2823	struct sa_path_rec *path_rec)	2830	struct sa_path_rec *path_rec)
2824	{	2831	{
2825	struct rdma_route *route = &work->id->id.route;	2832	struct rdma_route *route = &work->id->id.route;
2826		2833
@@ -2828,14 +2835,15 @@ static void route_set_path_rec_inbound(struct cma_work *work, · 1 unmodified line
2828	route->path_rec_inbound =	2835	route->path_rec_inbound =
2829	kzalloc(sizeof(*route->path_rec_inbound), GFP_KERNEL);	2836	kzalloc(sizeof(*route->path_rec_inbound), GFP_KERNEL);
2830	if (!route->path_rec_inbound)	2837	if (!route->path_rec_inbound)
2831	return;	2838	return -ENOMEM;
2832	}	2839	}
2833		2840
2834	route->path_rec_inbound = path_rec;	2841	route->path_rec_inbound = path_rec;
		2842	return 0;
2835	}	2843	}
2836		2844
2837	static void route_set_path_rec_outbound(struct cma_work *work,	2845	static int route_set_path_rec_outbound(struct cma_work *work,
2838	struct sa_path_rec *path_rec)	2846	struct sa_path_rec *path_rec)
2839	{	2847	{
2840	struct rdma_route *route = &work->id->id.route;	2848	struct rdma_route *route = &work->id->id.route;
2841		2849
@@ -2843,14 +2851,15 @@ static void route_set_path_rec_outbound(struct cma_work *work, · 1 unmodified line
2843	route->path_rec_outbound =	2851	route->path_rec_outbound =
2844	kzalloc(sizeof(*route->path_rec_outbound), GFP_KERNEL);	2852	kzalloc(sizeof(*route->path_rec_outbound), GFP_KERNEL);
2845	if (!route->path_rec_outbound)	2853	if (!route->path_rec_outbound)
2846	return;	2854	return -ENOMEM;
2847	}	2855	}
2848		2856
2849	route->path_rec_outbound = path_rec;	2857	route->path_rec_outbound = path_rec;
		2858	return 0;
2850	}	2859	}
2851		2860
2852	static void cma_query_handler(int status, struct sa_path_rec *path_rec,	2861	static void cma_query_handler(int status, struct sa_path_rec *path_rec,
2853	int num_prs, void *context)	2862	unsigned int num_prs, void *context)
2854	{	2863	{
2855	struct cma_work *work = context;	2864	struct cma_work *work = context;
2856	struct rdma_route *route;	2865	struct rdma_route *route;
@@ -2865,13 +2874,15 @@ static void cma_query_handler(int status, struct sa_path_rec *path_rec, · 8 unmodified lines
2865	if (!path_rec[i].flags \|\| (path_rec[i].flags & IB_PATH_GMP))	2874	if (!path_rec[i].flags \|\| (path_rec[i].flags & IB_PATH_GMP))
2866	*route->path_rec = path_rec[i];	2875	*route->path_rec = path_rec[i];
2867	else if (path_rec[i].flags & IB_PATH_INBOUND)	2876	else if (path_rec[i].flags & IB_PATH_INBOUND)
2868	route_set_path_rec_inbound(work, &path_rec[i]);	2877	status = route_set_path_rec_inbound(work, &path_rec[i]);
2869	else if (path_rec[i].flags & IB_PATH_OUTBOUND)	2878	else if (path_rec[i].flags & IB_PATH_OUTBOUND)
2870	route_set_path_rec_outbound(work, &path_rec[i]);	2879	status = route_set_path_rec_outbound(work,
2871	}	2880	&path_rec[i]);
2872	if (!route->path_rec) {	2881	else
2873	status = -EINVAL;	2882	status = -EINVAL;
2874	goto fail;	2883
		2884	if (status)
		2885	goto fail;
2875	}	2886	}
2876		2887
2877	route->num_pri_alt_paths = 1;	2888	route->num_pri_alt_paths = 1;
@@ -3541,121 +3552,6 @@ err: · 663 unmodified lines
3541	return ret;	3552	return ret;
3542	}	3553	}
3543		3554
3544	static int cma_bind_addr(struct rdma_cm_id id, struct sockaddr src_addr,
3545	const struct sockaddr *dst_addr)
3546	{
3547	struct sockaddr_storage zero_sock = {};
3548
3549	if (src_addr && src_addr->sa_family)
3550	return rdma_bind_addr(id, src_addr);
3551
3552	/*
3553	* When the src_addr is not specified, automatically supply an any addr
3554	*/
3555	zero_sock.ss_family = dst_addr->sa_family;
3556	if (IS_ENABLED(CONFIG_IPV6) && dst_addr->sa_family == AF_INET6) {
3557	struct sockaddr_in6 *src_addr6 =
3558	(struct sockaddr_in6 *)&zero_sock;
3559	struct sockaddr_in6 *dst_addr6 =
3560	(struct sockaddr_in6 *)dst_addr;
3561
3562	src_addr6->sin6_scope_id = dst_addr6->sin6_scope_id;
3563	if (ipv6_addr_type(&dst_addr6->sin6_addr) & IPV6_ADDR_LINKLOCAL)
3564	id->route.addr.dev_addr.bound_dev_if =
3565	dst_addr6->sin6_scope_id;
3566	} else if (dst_addr->sa_family == AF_IB) {
3567	((struct sockaddr_ib *)&zero_sock)->sib_pkey =
3568	((struct sockaddr_ib *)dst_addr)->sib_pkey;
3569	}
3570	return rdma_bind_addr(id, (struct sockaddr *)&zero_sock);
3571	}
3572
3573	/*
3574	* If required, resolve the source address for bind and leave the id_priv in
3575	* state RDMA_CM_ADDR_BOUND. This oddly uses the state to determine the prior
3576	* calls made by ULP, a previously bound ID will not be re-bound and src_addr is
3577	* ignored.
3578	*/
3579	static int resolve_prepare_src(struct rdma_id_private *id_priv,
3580	struct sockaddr *src_addr,
3581	const struct sockaddr *dst_addr)
3582	{
3583	int ret;
3584
3585	memcpy(cma_dst_addr(id_priv), dst_addr, rdma_addr_size(dst_addr));
3586	if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY)) {
3587	/* For a well behaved ULP state will be RDMA_CM_IDLE */
3588	ret = cma_bind_addr(&id_priv->id, src_addr, dst_addr);
3589	if (ret)
3590	goto err_dst;
3591	if (WARN_ON(!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND,
3592	RDMA_CM_ADDR_QUERY))) {
3593	ret = -EINVAL;
3594	goto err_dst;
3595	}
3596	}
3597
3598	if (cma_family(id_priv) != dst_addr->sa_family) {
3599	ret = -EINVAL;
3600	goto err_state;
3601	}
3602	return 0;
3603
3604	err_state:
3605	cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
3606	err_dst:
3607	memset(cma_dst_addr(id_priv), 0, rdma_addr_size(dst_addr));
3608	return ret;
3609	}
3610
3611	int rdma_resolve_addr(struct rdma_cm_id id, struct sockaddr src_addr,
3612	const struct sockaddr *dst_addr, unsigned long timeout_ms)
3613	{
3614	struct rdma_id_private *id_priv =
3615	container_of(id, struct rdma_id_private, id);
3616	int ret;
3617
3618	ret = resolve_prepare_src(id_priv, src_addr, dst_addr);
3619	if (ret)
3620	return ret;
3621
3622	if (cma_any_addr(dst_addr)) {
3623	ret = cma_resolve_loopback(id_priv);
3624	} else {
3625	if (dst_addr->sa_family == AF_IB) {
3626	ret = cma_resolve_ib_addr(id_priv);
3627	} else {
3628	/*
3629	* The FSM can return back to RDMA_CM_ADDR_BOUND after
3630	* rdma_resolve_ip() is called, eg through the error
3631	* path in addr_handler(). If this happens the existing
3632	* request must be canceled before issuing a new one.
3633	* Since canceling a request is a bit slow and this
3634	* oddball path is rare, keep track once a request has
3635	* been issued. The track turns out to be a permanent
3636	* state since this is the only cancel as it is
3637	* immediately before rdma_resolve_ip().
3638	*/
3639	if (id_priv->used_resolve_ip)
3640	rdma_addr_cancel(&id->route.addr.dev_addr);
3641	else
3642	id_priv->used_resolve_ip = 1;
3643	ret = rdma_resolve_ip(cma_src_addr(id_priv), dst_addr,
3644	&id->route.addr.dev_addr,
3645	timeout_ms, addr_handler,
3646	false, id_priv);
3647	}
3648	}
3649	if (ret)
3650	goto err;
3651
3652	return 0;
3653	err:
3654	cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
3655	return ret;
3656	}
3657	EXPORT_SYMBOL(rdma_resolve_addr);
3658
3659	int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse)	3555	int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse)
3660	{	3556	{
3661	struct rdma_id_private *id_priv;	3557	struct rdma_id_private *id_priv;
@@ -4058,27 +3954,26 @@ err: · 396 unmodified lines
4058	}	3954	}
4059	EXPORT_SYMBOL(rdma_listen);	3955	EXPORT_SYMBOL(rdma_listen);
4060		3956
4061	int rdma_bind_addr(struct rdma_cm_id id, struct sockaddr addr)	3957	static int rdma_bind_addr_dst(struct rdma_id_private *id_priv,
		3958	struct sockaddr addr, const struct sockaddr daddr)
4062	{	3959	{
4063	struct rdma_id_private *id_priv;	3960	struct sockaddr *id_daddr;
4064	int ret;	3961	int ret;
4065	struct sockaddr *daddr;
4066		3962
4067	if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6 &&	3963	if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6 &&
4068	addr->sa_family != AF_IB)	3964	addr->sa_family != AF_IB)
4069	return -EAFNOSUPPORT;	3965	return -EAFNOSUPPORT;
4070		3966
4071	id_priv = container_of(id, struct rdma_id_private, id);
4072	if (!cma_comp_exch(id_priv, RDMA_CM_IDLE, RDMA_CM_ADDR_BOUND))	3967	if (!cma_comp_exch(id_priv, RDMA_CM_IDLE, RDMA_CM_ADDR_BOUND))
4073	return -EINVAL;	3968	return -EINVAL;
4074		3969
4075	ret = cma_check_linklocal(&id->route.addr.dev_addr, addr);	3970	ret = cma_check_linklocal(&id_priv->id.route.addr.dev_addr, addr);
4076	if (ret)	3971	if (ret)
4077	goto err1;	3972	goto err1;
4078		3973
4079	memcpy(cma_src_addr(id_priv), addr, rdma_addr_size(addr));	3974	memcpy(cma_src_addr(id_priv), addr, rdma_addr_size(addr));
4080	if (!cma_any_addr(addr)) {	3975	if (!cma_any_addr(addr)) {
4081	ret = cma_translate_addr(addr, &id->route.addr.dev_addr);	3976	ret = cma_translate_addr(addr, &id_priv->id.route.addr.dev_addr);
4082	if (ret)	3977	if (ret)
4083	goto err1;	3978	goto err1;
4084		3979
@@ -4098,8 +3993,10 @@ int rdma_bind_addr(struct rdma_cm_id id, struct sockaddr addr) · 13 unmodified lines
4098	}	3993	}
4099	#endif	3994	#endif
4100	}	3995	}
4101	daddr = cma_dst_addr(id_priv);	3996	id_daddr = cma_dst_addr(id_priv);
4102	daddr->sa_family = addr->sa_family;	3997	if (daddr != id_daddr)
		3998	memcpy(id_daddr, daddr, rdma_addr_size(addr));
		3999	id_daddr->sa_family = addr->sa_family;
4103		4000
4104	ret = cma_get_port(id_priv);	4001	ret = cma_get_port(id_priv);
4105	if (ret)	4002	if (ret)
@@ -4115,6 +4012,127 @@ err1: · 9 unmodified lines
4115	cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE);	4012	cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE);
4116	return ret;	4013	return ret;
4117	}	4014	}
		4015
		4016	static int cma_bind_addr(struct rdma_cm_id id, struct sockaddr src_addr,
		4017	const struct sockaddr *dst_addr)
		4018	{
		4019	struct rdma_id_private *id_priv =
		4020	container_of(id, struct rdma_id_private, id);
		4021	struct sockaddr_storage zero_sock = {};
		4022
		4023	if (src_addr && src_addr->sa_family)
		4024	return rdma_bind_addr_dst(id_priv, src_addr, dst_addr);
		4025
		4026	/*
		4027	* When the src_addr is not specified, automatically supply an any addr
		4028	*/
		4029	zero_sock.ss_family = dst_addr->sa_family;
		4030	if (IS_ENABLED(CONFIG_IPV6) && dst_addr->sa_family == AF_INET6) {
		4031	struct sockaddr_in6 *src_addr6 =
		4032	(struct sockaddr_in6 *)&zero_sock;
		4033	struct sockaddr_in6 *dst_addr6 =
		4034	(struct sockaddr_in6 *)dst_addr;
		4035
		4036	src_addr6->sin6_scope_id = dst_addr6->sin6_scope_id;
		4037	if (ipv6_addr_type(&dst_addr6->sin6_addr) & IPV6_ADDR_LINKLOCAL)
		4038	id->route.addr.dev_addr.bound_dev_if =
		4039	dst_addr6->sin6_scope_id;
		4040	} else if (dst_addr->sa_family == AF_IB) {
		4041	((struct sockaddr_ib *)&zero_sock)->sib_pkey =
		4042	((struct sockaddr_ib *)dst_addr)->sib_pkey;
		4043	}
		4044	return rdma_bind_addr_dst(id_priv, (struct sockaddr *)&zero_sock, dst_addr);
		4045	}
		4046
		4047	/*
		4048	* If required, resolve the source address for bind and leave the id_priv in
		4049	* state RDMA_CM_ADDR_BOUND. This oddly uses the state to determine the prior
		4050	* calls made by ULP, a previously bound ID will not be re-bound and src_addr is
		4051	* ignored.
		4052	*/
		4053	static int resolve_prepare_src(struct rdma_id_private *id_priv,
		4054	struct sockaddr *src_addr,
		4055	const struct sockaddr *dst_addr)
		4056	{
		4057	int ret;
		4058
		4059	if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY)) {
		4060	/* For a well behaved ULP state will be RDMA_CM_IDLE */
		4061	ret = cma_bind_addr(&id_priv->id, src_addr, dst_addr);
		4062	if (ret)
		4063	return ret;
		4064	if (WARN_ON(!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND,
		4065	RDMA_CM_ADDR_QUERY)))
		4066	return -EINVAL;
		4067
		4068	}
		4069
		4070	if (cma_family(id_priv) != dst_addr->sa_family) {
		4071	ret = -EINVAL;
		4072	goto err_state;
		4073	}
		4074	return 0;
		4075
		4076	err_state:
		4077	cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
		4078	return ret;
		4079	}
		4080
		4081	int rdma_resolve_addr(struct rdma_cm_id id, struct sockaddr src_addr,
		4082	const struct sockaddr *dst_addr, unsigned long timeout_ms)
		4083	{
		4084	struct rdma_id_private *id_priv =
		4085	container_of(id, struct rdma_id_private, id);
		4086	int ret;
		4087
		4088	ret = resolve_prepare_src(id_priv, src_addr, dst_addr);
		4089	if (ret)
		4090	return ret;
		4091
		4092	if (cma_any_addr(dst_addr)) {
		4093	ret = cma_resolve_loopback(id_priv);
		4094	} else {
		4095	if (dst_addr->sa_family == AF_IB) {
		4096	ret = cma_resolve_ib_addr(id_priv);
		4097	} else {
		4098	/*
		4099	* The FSM can return back to RDMA_CM_ADDR_BOUND after
		4100	* rdma_resolve_ip() is called, eg through the error
		4101	* path in addr_handler(). If this happens the existing
		4102	* request must be canceled before issuing a new one.
		4103	* Since canceling a request is a bit slow and this
		4104	* oddball path is rare, keep track once a request has
		4105	* been issued. The track turns out to be a permanent
		4106	* state since this is the only cancel as it is
		4107	* immediately before rdma_resolve_ip().
		4108	*/
		4109	if (id_priv->used_resolve_ip)
		4110	rdma_addr_cancel(&id->route.addr.dev_addr);
		4111	else
		4112	id_priv->used_resolve_ip = 1;
		4113	ret = rdma_resolve_ip(cma_src_addr(id_priv), dst_addr,
		4114	&id->route.addr.dev_addr,
		4115	timeout_ms, addr_handler,
		4116	false, id_priv);
		4117	}
		4118	}
		4119	if (ret)
		4120	goto err;
		4121
		4122	return 0;
		4123	err:
		4124	cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
		4125	return ret;
		4126	}
		4127	EXPORT_SYMBOL(rdma_resolve_addr);
		4128
		4129	int rdma_bind_addr(struct rdma_cm_id id, struct sockaddr addr)
		4130	{
		4131	struct rdma_id_private *id_priv =
		4132	container_of(id, struct rdma_id_private, id);
		4133
		4134	return rdma_bind_addr_dst(id_priv, addr, cma_dst_addr(id_priv));
		4135	}
4118	EXPORT_SYMBOL(rdma_bind_addr);	4136	EXPORT_SYMBOL(rdma_bind_addr);
4119		4137
4120	static int cma_format_hdr(void hdr, struct rdma_id_private id_priv)	4138	static int cma_format_hdr(void hdr, struct rdma_id_private id_priv)

drivers/infiniband/core/sa_query.c+55−116

@@ -106,7 +106,7 @@ struct ib_sa_device { · 105 unmodified lines
106		106
107	struct ib_sa_query {	107	struct ib_sa_query {
108	void (callback)(struct ib_sa_query sa_query, int status,	108	void (callback)(struct ib_sa_query sa_query, int status,
109	int num_prs, struct ib_sa_mad *mad);	109	struct ib_sa_mad *mad);
110	void (release)(struct ib_sa_query );	110	void (release)(struct ib_sa_query );
111	struct ib_sa_client *client;	111	struct ib_sa_client *client;
112	struct ib_sa_port *port;	112	struct ib_sa_port *port;
@@ -118,12 +118,6 @@ struct ib_sa_query { · 5 unmodified lines
118	u32 seq; /* Local svc request sequence number */	118	u32 seq; /* Local svc request sequence number */
119	unsigned long timeout; /* Local svc timeout */	119	unsigned long timeout; /* Local svc timeout */
120	u8 path_use; /* How will the pathrecord be used */	120	u8 path_use; /* How will the pathrecord be used */
121
122	/* A separate buffer to save pathrecords of a response, as in cases
123	* like IB/netlink, mulptiple pathrecords are supported, so that
124	* mad->data is not large enough to hold them
125	*/
126	void *resp_pr_data;
127	};	121	};
128		122
129	#define IB_SA_ENABLE_LOCAL_SERVICE 0x00000001	123	#define IB_SA_ENABLE_LOCAL_SERVICE 0x00000001
@@ -132,7 +126,7 @@ struct ib_sa_query { · 2 unmodified lines
132		126
133	struct ib_sa_path_query {	127	struct ib_sa_path_query {
134	void (callback)(int status, struct sa_path_rec rec,	128	void (callback)(int status, struct sa_path_rec rec,
135	int num_paths, void *context);	129	unsigned int num_paths, void *context);
136	void *context;	130	void *context;
137	struct ib_sa_query sa_query;	131	struct ib_sa_query sa_query;
138	struct sa_path_rec *conv_pr;	132	struct sa_path_rec *conv_pr;
@@ -690,6 +684,8 @@ static const struct ib_field guidinfo_rec_table[] = { · 551 unmodified lines
690	.size_bits = 512 },	684	.size_bits = 512 },
691	};	685	};
692		686
		687	#define RDMA_PRIMARY_PATH_MAX_REC_NUM 3
		688
693	static inline void ib_sa_disable_local_svc(struct ib_sa_query *query)	689	static inline void ib_sa_disable_local_svc(struct ib_sa_query *query)
694	{	690	{
695	query->flags &= ~IB_SA_ENABLE_LOCAL_SERVICE;	691	query->flags &= ~IB_SA_ENABLE_LOCAL_SERVICE;
@@ -874,30 +870,21 @@ static void send_handler(struct ib_mad_agent *agent, · 178 unmodified lines
874	static void ib_nl_process_good_resolve_rsp(struct ib_sa_query *query,	870	static void ib_nl_process_good_resolve_rsp(struct ib_sa_query *query,
875	const struct nlmsghdr *nlh)	871	const struct nlmsghdr *nlh)
876	{	872	{
877	struct ib_path_rec_data srec, drec;	873	struct sa_path_rec recs[RDMA_PRIMARY_PATH_MAX_REC_NUM];
878	struct ib_sa_path_query *path_query;	874	struct ib_sa_path_query *path_query;
		875	struct ib_path_rec_data *rec_data;
879	struct ib_mad_send_wc mad_send_wc;	876	struct ib_mad_send_wc mad_send_wc;
880	const struct nlattr head, curr;	877	const struct nlattr head, curr;
881	struct ib_sa_mad *mad = NULL;	878	struct ib_sa_mad *mad = NULL;
882	int len, rem, num_prs = 0;	879	int len, rem, status = -EIO;
		880	unsigned int num_prs = 0;
883	u32 mask = 0;	881	u32 mask = 0;
884	int status = -EIO;
885		882
886	if (!query->callback)	883	if (!query->callback)
887	goto out;	884	goto out;
888		885
889	path_query = container_of(query, struct ib_sa_path_query, sa_query);	886	path_query = container_of(query, struct ib_sa_path_query, sa_query);
890	mad = query->mad_buf->mad;	887	mad = query->mad_buf->mad;
891	if (!path_query->conv_pr &&
892	(be16_to_cpu(mad->mad_hdr.attr_id) == IB_SA_ATTR_PATH_REC)) {
893	/* Need a larger buffer for possible multiple PRs */
894	query->resp_pr_data = kvcalloc(RDMA_PRIMARY_PATH_MAX_REC_NUM,
895	sizeof(*drec), GFP_KERNEL);
896	if (!query->resp_pr_data) {
897	query->callback(query, -ENOMEM, 0, NULL);
898	return;
899	}
900	}
901		888
902	head = (const struct nlattr *) nlmsg_data(nlh);	889	head = (const struct nlattr *) nlmsg_data(nlh);
903	len = nlmsg_len(nlh);	890	len = nlmsg_len(nlh);
@@ -917,36 +904,41 @@ static void ib_nl_process_good_resolve_rsp(struct ib_sa_query *query, · 13 unmodified lines
917	break;	904	break;
918	}	905	}
919		906
920	drec = (struct ib_path_rec_data *)query->resp_pr_data;
921	nla_for_each_attr(curr, head, len, rem) {	907	nla_for_each_attr(curr, head, len, rem) {
922	if (curr->nla_type != LS_NLA_TYPE_PATH_RECORD)	908	if (curr->nla_type != LS_NLA_TYPE_PATH_RECORD)
923	continue;	909	continue;
924		910
925	srec = nla_data(curr);	911	rec_data = nla_data(curr);
926	if ((srec->flags & mask) != mask)	912	if ((rec_data->flags & mask) != mask)
927	continue;	913	continue;
928		914
929	status = 0;	915	if ((query->flags & IB_SA_QUERY_OPA) \|\|
930	if (!drec) {	916	path_query->conv_pr) {
931	memcpy(mad->data, srec->path_rec,	917	mad->mad_hdr.method \|= IB_MGMT_METHOD_RESP;
932	sizeof(srec->path_rec));	918	memcpy(mad->data, rec_data->path_rec,
933	num_prs = 1;	919	sizeof(rec_data->path_rec));
934	break;	920	query->callback(query, 0, mad);
		921	goto out;
935	}	922	}
936		923
937	memcpy(drec, srec, sizeof(*drec));	924	status = 0;
938	drec++;	925	ib_unpack(path_rec_table, ARRAY_SIZE(path_rec_table),
		926	rec_data->path_rec, &recs[num_prs]);
		927	recs[num_prs].flags = rec_data->flags;
		928	recs[num_prs].rec_type = SA_PATH_REC_TYPE_IB;
		929	sa_path_set_dmac_zero(&recs[num_prs]);
		930
939	num_prs++;	931	num_prs++;
940	if (num_prs >= RDMA_PRIMARY_PATH_MAX_REC_NUM)	932	if (num_prs >= RDMA_PRIMARY_PATH_MAX_REC_NUM)
941	break;	933	break;
942	}	934	}
943		935
944	if (!status)	936	if (!status) {
945	mad->mad_hdr.method \|= IB_MGMT_METHOD_RESP;	937	mad->mad_hdr.method \|= IB_MGMT_METHOD_RESP;
946		938	path_query->callback(status, recs, num_prs,
947	query->callback(query, status, num_prs, mad);	939	path_query->context);
948	kvfree(query->resp_pr_data);	940	} else
949	query->resp_pr_data = NULL;	941	query->callback(query, status, mad);
950		942
951	out:	943	out:
952	mad_send_wc.send_buf = query->mad_buf;	944	mad_send_wc.send_buf = query->mad_buf;
@@ -1451,11 +1443,26 @@ static int opa_pr_query_possible(struct ib_sa_client *client, · 498 unmodified lines
1451	return PR_IB_SUPPORTED;	1443	return PR_IB_SUPPORTED;
1452	}	1444	}
1453		1445
1454	static void ib_sa_pr_callback_single(struct ib_sa_path_query *query,	1446	static void ib_sa_path_rec_callback(struct ib_sa_query *sa_query,
1455	int status, struct ib_sa_mad *mad)	1447	int status, struct ib_sa_mad *mad)
1456	{	1448	{
		1449	struct ib_sa_path_query *query =
		1450	container_of(sa_query, struct ib_sa_path_query, sa_query);
1457	struct sa_path_rec rec = {};	1451	struct sa_path_rec rec = {};
1458		1452
		1453	if (!mad) {
		1454	query->callback(status, NULL, 0, query->context);
		1455	return;
		1456	}
		1457
		1458	if (sa_query->flags & IB_SA_QUERY_OPA) {
		1459	ib_unpack(opa_path_rec_table, ARRAY_SIZE(opa_path_rec_table),
		1460	mad->data, &rec);
		1461	rec.rec_type = SA_PATH_REC_TYPE_OPA;
		1462	query->callback(status, &rec, 1, query->context);
		1463	return;
		1464	}
		1465
1459	ib_unpack(path_rec_table, ARRAY_SIZE(path_rec_table),	1466	ib_unpack(path_rec_table, ARRAY_SIZE(path_rec_table),
1460	mad->data, &rec);	1467	mad->data, &rec);
1461	rec.rec_type = SA_PATH_REC_TYPE_IB;	1468	rec.rec_type = SA_PATH_REC_TYPE_IB;
@@ -1472,71 +1479,6 @@ static void ib_sa_pr_callback_single(struct ib_sa_path_query *query, · 10 unmodified lines
1472	}	1479	}
1473	}	1480	}
1474		1481
1475	/**
1476	* ib_sa_pr_callback_multiple() - Parse path records then do callback.
1477	*
1478	* In a multiple-PR case the PRs are saved in "query->resp_pr_data"
1479	* (instead of"mad->data") and with "ib_path_rec_data" structure format,
1480	* so that rec->flags can be set to indicate the type of PR.
1481	* This is valid only in IB fabric.
1482	*/
1483	static void ib_sa_pr_callback_multiple(struct ib_sa_path_query *query,
1484	int status, int num_prs,
1485	struct ib_path_rec_data *rec_data)
1486	{
1487	struct sa_path_rec *rec;
1488	int i;
1489
1490	rec = kvcalloc(num_prs, sizeof(*rec), GFP_KERNEL);
1491	if (!rec) {
1492	query->callback(-ENOMEM, NULL, 0, query->context);
1493	return;
1494	}
1495
1496	for (i = 0; i < num_prs; i++) {
1497	ib_unpack(path_rec_table, ARRAY_SIZE(path_rec_table),
1498	rec_data[i].path_rec, rec + i);
1499	rec[i].rec_type = SA_PATH_REC_TYPE_IB;
1500	sa_path_set_dmac_zero(rec + i);
1501	rec[i].flags = rec_data[i].flags;
1502	}
1503
1504	query->callback(status, rec, num_prs, query->context);
1505	kvfree(rec);
1506	}
1507
1508	static void ib_sa_path_rec_callback(struct ib_sa_query *sa_query,
1509	int status, int num_prs,
1510	struct ib_sa_mad *mad)
1511	{
1512	struct ib_sa_path_query *query =
1513	container_of(sa_query, struct ib_sa_path_query, sa_query);
1514	struct sa_path_rec rec;
1515
1516	if (!mad \|\| !num_prs) {
1517	query->callback(status, NULL, 0, query->context);
1518	return;
1519	}
1520
1521	if (sa_query->flags & IB_SA_QUERY_OPA) {
1522	if (num_prs != 1) {
1523	query->callback(-EINVAL, NULL, 0, query->context);
1524	return;
1525	}
1526
1527	ib_unpack(opa_path_rec_table, ARRAY_SIZE(opa_path_rec_table),
1528	mad->data, &rec);
1529	rec.rec_type = SA_PATH_REC_TYPE_OPA;
1530	query->callback(status, &rec, num_prs, query->context);
1531	} else {
1532	if (!sa_query->resp_pr_data)
1533	ib_sa_pr_callback_single(query, status, mad);
1534	else
1535	ib_sa_pr_callback_multiple(query, status, num_prs,
1536	sa_query->resp_pr_data);
1537	}
1538	}
1539
1540	static void ib_sa_path_rec_release(struct ib_sa_query *sa_query)	1482	static void ib_sa_path_rec_release(struct ib_sa_query *sa_query)
1541	{	1483	{
1542	struct ib_sa_path_query *query =	1484	struct ib_sa_path_query *query =
@@ -1578,7 +1520,7 @@ int ib_sa_path_rec_get(struct ib_sa_client *client, · 35 unmodified lines
1578	unsigned long timeout_ms, gfp_t gfp_mask,	1520	unsigned long timeout_ms, gfp_t gfp_mask,
1579	void (*callback)(int status,	1521	void (*callback)(int status,
1580	struct sa_path_rec *resp,	1522	struct sa_path_rec *resp,
1581	int num_paths, void *context),	1523	unsigned int num_paths, void *context),
1582	void *context,	1524	void *context,
1583	struct ib_sa_query **sa_query)	1525	struct ib_sa_query **sa_query)
1584	{	1526	{
@@ -1677,8 +1619,7 @@ err1: · 92 unmodified lines
1677	EXPORT_SYMBOL(ib_sa_path_rec_get);	1619	EXPORT_SYMBOL(ib_sa_path_rec_get);
1678		1620
1679	static void ib_sa_mcmember_rec_callback(struct ib_sa_query *sa_query,	1621	static void ib_sa_mcmember_rec_callback(struct ib_sa_query *sa_query,
1680	int status, int num_prs,	1622	int status, struct ib_sa_mad *mad)
1681	struct ib_sa_mad *mad)
1682	{	1623	{
1683	struct ib_sa_mcmember_query *query =	1624	struct ib_sa_mcmember_query *query =
1684	container_of(sa_query, struct ib_sa_mcmember_query, sa_query);	1625	container_of(sa_query, struct ib_sa_mcmember_query, sa_query);
@@ -1769,8 +1710,7 @@ err1: · 84 unmodified lines
1769		1710
1770	/* Support GuidInfoRecord */	1711	/* Support GuidInfoRecord */
1771	static void ib_sa_guidinfo_rec_callback(struct ib_sa_query *sa_query,	1712	static void ib_sa_guidinfo_rec_callback(struct ib_sa_query *sa_query,
1772	int status, int num_paths,	1713	int status, struct ib_sa_mad *mad)
1773	struct ib_sa_mad *mad)
1774	{	1714	{
1775	struct ib_sa_guidinfo_query *query =	1715	struct ib_sa_guidinfo_query *query =
1776	container_of(sa_query, struct ib_sa_guidinfo_query, sa_query);	1716	container_of(sa_query, struct ib_sa_guidinfo_query, sa_query);
@@ -1879,8 +1819,7 @@ static void ib_classportinfo_cb(void *context) · 102 unmodified lines
1879	}	1819	}
1880		1820
1881	static void ib_sa_classport_info_rec_callback(struct ib_sa_query *sa_query,	1821	static void ib_sa_classport_info_rec_callback(struct ib_sa_query *sa_query,
1882	int status, int num_prs,	1822	int status, struct ib_sa_mad *mad)
1883	struct ib_sa_mad *mad)
1884	{	1823	{
1885	unsigned long flags;	1824	unsigned long flags;
1886	struct ib_sa_classport_info_query *query =	1825	struct ib_sa_classport_info_query *query =
@@ -2055,13 +1994,13 @@ static void send_handler(struct ib_mad_agent *agent, · 168 unmodified lines
2055	/* No callback -- already got recv */	1994	/* No callback -- already got recv */
2056	break;	1995	break;
2057	case IB_WC_RESP_TIMEOUT_ERR:	1996	case IB_WC_RESP_TIMEOUT_ERR:
2058	query->callback(query, -ETIMEDOUT, 0, NULL);	1997	query->callback(query, -ETIMEDOUT, NULL);
2059	break;	1998	break;
2060	case IB_WC_WR_FLUSH_ERR:	1999	case IB_WC_WR_FLUSH_ERR:
2061	query->callback(query, -EINTR, 0, NULL);	2000	query->callback(query, -EINTR, NULL);
2062	break;	2001	break;
2063	default:	2002	default:
2064	query->callback(query, -EIO, 0, NULL);	2003	query->callback(query, -EIO, NULL);
2065	break;	2004	break;
2066	}	2005	}
2067		2006
@@ -2089,10 +2028,10 @@ static void recv_handler(struct ib_mad_agent *mad_agent, · 21 unmodified lines
2089	if (mad_recv_wc->wc->status == IB_WC_SUCCESS)	2028	if (mad_recv_wc->wc->status == IB_WC_SUCCESS)
2090	query->callback(query,	2029	query->callback(query,
2091	mad_recv_wc->recv_buf.mad->mad_hdr.status ?	2030	mad_recv_wc->recv_buf.mad->mad_hdr.status ?
2092	-EINVAL : 0, 1,	2031	-EINVAL : 0,
2093	(struct ib_sa_mad *) mad_recv_wc->recv_buf.mad);	2032	(struct ib_sa_mad *) mad_recv_wc->recv_buf.mad);
2094	else	2033	else
2095	query->callback(query, -EIO, 0, NULL);	2034	query->callback(query, -EIO, NULL);
2096	}	2035	}
2097		2036
2098	ib_free_recv_mad(mad_recv_wc);	2037	ib_free_recv_mad(mad_recv_wc);

drivers/infiniband/hw/cxgb4/cm.c+7−0

 	u16 tcp_opt = ntohs(req->tcp_opt);
 	ep = get_ep_from_tid(dev, tid);
+	if (!ep)
+		return 0;
 	pr_debug("ep %p tid %u\n", ep, ep->hwtid);
 	ep->snd_seq = be32_to_cpu(req->snd_isn);
 	ep->rcv_seq = be32_to_cpu(req->rcv_isn);
 	if (neigh->dev->flags & IFF_LOOPBACK) {
 		pdev = ip_dev_find(&init_net, iph->daddr);
+		if (!pdev) {
+			pr_err("%s - failed to find device!\n", __func__);
+			goto free_dst;
+		}
 		e = cxgb4_l2t_get(dev->rdev.lldi.l2t, neigh,
 				    pdev, 0);
 		pi = (struct port_info *)netdev_priv(pdev);

drivers/infiniband/hw/cxgb4/cq.c+1−1

@@ -767,7 +767,7 @@ static int __c4iw_poll_cq_one(struct c4iw_cq chp, struct c4iw_qp qhp, · 766 unmodified lines
767	goto out;	767	goto out;
768		768
769	wc->wr_id = cookie;	769	wc->wr_id = cookie;
770	wc->qp = qhp ? &qhp->ibqp : NULL;	770	wc->qp = &qhp->ibqp;
771	wc->vendor_err = CQE_STATUS(&cqe);	771	wc->vendor_err = CQE_STATUS(&cqe);
772	wc->wc_flags = 0;	772	wc->wc_flags = 0;
773		773

drivers/infiniband/hw/cxgb4/restrack.c+1−1

@@ -238,7 +238,7 @@ int c4iw_fill_res_cm_id_entry(struct sk_buff *msg, · 237 unmodified lines
238	if (rdma_nl_put_driver_u64_hex(msg, "history", epcp->history))	238	if (rdma_nl_put_driver_u64_hex(msg, "history", epcp->history))
239	goto err_cancel_table;	239	goto err_cancel_table;
240		240
241	if (epcp->state == LISTEN) {	241	if (listen_ep) {
242	if (rdma_nl_put_driver_u32(msg, "stid", listen_ep->stid))	242	if (rdma_nl_put_driver_u32(msg, "stid", listen_ep->stid))
243	goto err_cancel_table;	243	goto err_cancel_table;
244	if (rdma_nl_put_driver_u32(msg, "backlog", listen_ep->backlog))	244	if (rdma_nl_put_driver_u32(msg, "backlog", listen_ep->backlog))

drivers/infiniband/hw/cxgb4/t4fw_ri_api.h+6−20

 	__be16	nsge;
 	__be32	len0;
 	__be64	addr0;
-#ifndef C99_NOT_SUPPORTED
 	struct fw_ri_dsge_pair sge[];
-#endif
 };
 struct fw_ri_sge {
 	__u8	r1;
 	__be16	nsge;
 	__be32	r2;
-#ifndef C99_NOT_SUPPORTED
 	struct fw_ri_sge sge[];
-#endif
 };
 struct fw_ri_immd {
 	__u8	r1;
 	__be16	r2;
 	__be32	immdlen;
-#ifndef C99_NOT_SUPPORTED
 	__u8	data[];
-#endif
 };
 struct fw_ri_tpte {
 	__be32 op_nres;
 	__be32 len16_pkd;
 	__u64  cookie;
-#ifndef C99_NOT_SUPPORTED
 	struct fw_ri_res res[];
-#endif
 };
 #define FW_RI_RES_WR_NRES_S	0
 	__be32 plen;
 	__be32 stag_sink;
 	__be64 to_sink;
-#ifndef C99_NOT_SUPPORTED
 	union {
-		struct fw_ri_immd immd_src[0];
+		DECLARE_FLEX_ARRAY(struct fw_ri_immd, immd_src);
-		struct fw_ri_isgl isgl_src[0];
+		DECLARE_FLEX_ARRAY(struct fw_ri_isgl, isgl_src);
 	} u;
-#endif
 };
 struct fw_ri_send_wr {
 	__be32 plen;
 	__be32 r3;
 	__be64 r4;
-#ifndef C99_NOT_SUPPORTED
 	union {
-		struct fw_ri_immd immd_src[0];
+		DECLARE_FLEX_ARRAY(struct fw_ri_immd, immd_src);
-		struct fw_ri_isgl isgl_src[0];
+		DECLARE_FLEX_ARRAY(struct fw_ri_isgl, isgl_src);
 	} u;
-#endif
 };
 #define FW_RI_SEND_WR_SENDOP_S		0
 		struct fw_ri_isgl isgl_src;
 	} u_cmpl;
 	__be64 r3;
-#ifndef C99_NOT_SUPPORTED
 	union fw_ri_write {
-		struct fw_ri_immd immd_src[0];
+		DECLARE_FLEX_ARRAY(struct fw_ri_immd, immd_src);
-		struct fw_ri_isgl isgl_src[0];
+		DECLARE_FLEX_ARRAY(struct fw_ri_isgl, isgl_src);
 	} u;
-#endif
 };
 struct fw_ri_rdma_read_wr {

drivers/infiniband/hw/erdma/erdma_hw.h+2−2

@@ -397,7 +397,7 @@ struct erdma_write_sqe { · 396 unmodified lines
397		397
398	__le32 rsvd;	398	__le32 rsvd;
399		399
400	struct erdma_sge sgl[0];	400	struct erdma_sge sgl[];
401	};	401	};
402		402
403	struct erdma_send_sqe {	403	struct erdma_send_sqe {
@@ -408,7 +408,7 @@ struct erdma_send_sqe { · 4 unmodified lines
408	};	408	};
409		409
410	__le32 length;	410	__le32 length;
411	struct erdma_sge sgl[0];	411	struct erdma_sge sgl[];
412	};	412	};
413		413
414	struct erdma_readreq_sqe {	414	struct erdma_readreq_sqe {

drivers/infiniband/hw/erdma/erdma_verbs.c+3−1

 		prot = pgprot_device(vma->vm_page_prot);
 		break;
 	default:
-		return -EINVAL;
+		err = -EINVAL;
+		goto put_entry;
 	}
 	err = rdma_user_mmap_io(ctx, vma, PFN_DOWN(entry->address), PAGE_SIZE,
 				prot, rdma_entry);
+put_entry:
 	rdma_user_mmap_entry_put(rdma_entry);
 	return err;
 }

drivers/infiniband/hw/hfi1/chip.c+32−27

 static void handle_temp_err(struct hfi1_devdata *dd);
 static void dc_shutdown(struct hfi1_devdata *dd);
 static void dc_start(struct hfi1_devdata *dd);
-static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp,
+static int qos_rmt_entries(unsigned int n_krcv_queues, unsigned int *mp,
 			   unsigned int *np);
 static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd);
 static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms);
 	int ret;
 	unsigned ngroups;
 	int rmt_count;
-	int user_rmt_reduced;
 	u32 n_usr_ctxts;
 	u32 send_contexts = chip_send_contexts(dd);
 	u32 rcv_contexts = chip_rcv_contexts(dd);
 					 (num_kernel_contexts + n_usr_ctxts),
 					 &node_affinity.real_cpu_mask);
 	/*
-	 * The RMT entries are currently allocated as shown below:
+	 * RMT entries are allocated as follows:
-	 * 1. QOS (0 to 128 entries);
+	 * 1. QOS (0 to 128 entries)
-	 * 2. FECN (num_kernel_context - 1 + num_user_contexts +
+	 * 2. FECN (num_kernel_context - 1 [a] + num_user_contexts +
-	 *    num_netdev_contexts);
+	 *          num_netdev_contexts [b])
-	 * 3. netdev (num_netdev_contexts).
+	 * 3. netdev (NUM_NETDEV_MAP_ENTRIES)
-	 * It should be noted that FECN oversubscribe num_netdev_contexts
+	 *
-	 * entries of RMT because both netdev and PSM could allocate any receive
+	 * Notes:
-	 * context between dd->first_dyn_alloc_text and dd->num_rcv_contexts,
+	 * [a] Kernel contexts (except control) are included in FECN if kernel
-	 * and PSM FECN must reserve an RMT entry for each possible PSM receive
+	 *     TID_RDMA is active.
-	 * context.
+	 * [b] Netdev and user contexts are randomly allocated from the same
+	 *     context pool, so FECN must cover all contexts in the pool.
 	 */
-	rmt_count = qos_rmt_entries(dd, NULL, NULL) + (num_netdev_contexts * 2);
+	rmt_count = qos_rmt_entries(num_kernel_contexts - 1, NULL, NULL)
-	if (HFI1_CAP_IS_KSET(TID_RDMA))
+		    + (HFI1_CAP_IS_KSET(TID_RDMA) ? num_kernel_contexts - 1
-		rmt_count += num_kernel_contexts - 1;
+						  : 0)
-	if (rmt_count + n_usr_ctxts > NUM_MAP_ENTRIES) {
+		    + n_usr_ctxts
-		user_rmt_reduced = NUM_MAP_ENTRIES - rmt_count;
+		    + num_netdev_contexts
-		dd_dev_err(dd,
+		    + NUM_NETDEV_MAP_ENTRIES;
-			   "RMT size is reducing the number of user receive contexts from %u to %d\n",
+	if (rmt_count > NUM_MAP_ENTRIES) {
-			   n_usr_ctxts,
+		int over = rmt_count - NUM_MAP_ENTRIES;
-			   user_rmt_reduced);
+		/* try to squish user contexts, minimum of 1 */
-		/* recalculate */
+		if (over >= n_usr_ctxts) {
-		n_usr_ctxts = user_rmt_reduced;
+			dd_dev_err(dd, "RMT overflow: reduce the requested number of contexts\n");
+			return -EINVAL;
+		}
+		dd_dev_err(dd, "RMT overflow: reducing # user contexts from %u to %u\n",
+			   n_usr_ctxts, n_usr_ctxts - over);
+		n_usr_ctxts -= over;
 	}
 	/* the first N are kernel contexts, the rest are user/netdev contexts */
 }
 /* return the number of RSM map table entries that will be used for QOS */
-static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp,
+static int qos_rmt_entries(unsigned int n_krcv_queues, unsigned int *mp,
 			   unsigned int *np)
 {
 	int i;
 	unsigned int m, n;
-	u8 max_by_vl = 0;
+	uint max_by_vl = 0;
 	/* is QOS active at all? */
-	if (dd->n_krcv_queues <= MIN_KERNEL_KCTXTS ||
+	if (n_krcv_queues < MIN_KERNEL_KCTXTS ||
 	    num_vls == 1 ||
 	    krcvqsset <= 1)
 		goto no_qos;
 	if (!rmt)
 		goto bail;
-	rmt_entries = qos_rmt_entries(dd, &m, &n);
+	rmt_entries = qos_rmt_entries(dd->n_krcv_queues - 1, &m, &n);
 	if (rmt_entries == 0)
 		goto bail;
 	qpns_per_vl = 1 << m;

drivers/infiniband/hw/hfi1/exp_rcv.h+3−2

 	return grp;
 }
-static inline u32 rcventry2tidinfo(u32 rcventry)
+static inline u32 create_tid(u32 rcventry, u32 npages)
 {
 	u32 pair = rcventry & ~0x1;
 	return EXP_TID_SET(IDX, pair >> 1) |
-		EXP_TID_SET(CTRL, 1 << (rcventry - pair));
+		EXP_TID_SET(CTRL, 1 << (rcventry - pair)) |
+		EXP_TID_SET(LEN, npages);
 }
 /**

drivers/infiniband/hw/hfi1/file_ops.c+57−24

 	return reqs;
 }
+static inline void mmap_cdbg(u16 ctxt, u8 subctxt, u8 type, u8 mapio, u8 vmf,
+			     u64 memaddr, void *memvirt, dma_addr_t memdma,
+			     ssize_t memlen, struct vm_area_struct *vma)
+{
+	hfi1_cdbg(PROC,
+		  "%u:%u type:%u io/vf/dma:%d/%d/%d, addr:0x%llx, len:%lu(%lu), flags:0x%lx",
+		  ctxt, subctxt, type, mapio, vmf, !!memdma,
+		  memaddr ?: (u64)memvirt, memlen,
+		  vma->vm_end - vma->vm_start, vma->vm_flags);
+}
 static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
 {
 	struct hfi1_filedata *fd = fp->private_data;
 	u64 token = vma->vm_pgoff << PAGE_SHIFT,
 		memaddr = 0;
 	void *memvirt = NULL;
+	dma_addr_t memdma = 0;
 	u8 subctxt, mapio = 0, vmf = 0, type;
 	ssize_t memlen = 0;
 	int ret = 0;
 		goto done;
 	}
+	/*
+	 * vm_pgoff is used as a buffer selector cookie.  Always mmap from
+	 * the beginning.
+	 */
+	vma->vm_pgoff = 0;
 	flags = vma->vm_flags;
 	switch (type) {
 		vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
 		mapio = 1;
 		break;
-	case PIO_CRED:
+	case PIO_CRED: {
+		u64 cr_page_offset;
 		if (flags & VM_WRITE) {
 			ret = -EPERM;
 			goto done;
 		 * second or third page allocated for credit returns (if number
 		 * of enabled contexts > 64 and 128 respectively).
 		 */
-		memvirt = dd->cr_base[uctxt->numa_id].va;
+		cr_page_offset = ((u64)uctxt->sc->hw_free -
-		memaddr = virt_to_phys(memvirt) +
+			  	     (u64)dd->cr_base[uctxt->numa_id].va) &
-			(((u64)uctxt->sc->hw_free -
+				   PAGE_MASK;
-			  (u64)dd->cr_base[uctxt->numa_id].va) & PAGE_MASK);
+		memvirt = dd->cr_base[uctxt->numa_id].va + cr_page_offset;
+		memdma = dd->cr_base[uctxt->numa_id].dma + cr_page_offset;
 		memlen = PAGE_SIZE;
 		flags &= ~VM_MAYWRITE;
 		flags |= VM_DONTCOPY | VM_DONTEXPAND;
 		 * memory been flagged as non-cached?
 		 */
 		/* vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); */
-		mapio = 1;
 		break;
+	}
 	case RCV_HDRQ:
 		memlen = rcvhdrq_size(uctxt);
 		memvirt = uctxt->rcvhdrq;
+		memdma = uctxt->rcvhdrq_dma;
 		break;
 	case RCV_EGRBUF: {
-		unsigned long addr;
+		unsigned long vm_start_save;
+		unsigned long vm_end_save;
 		int i;
 		/*
 		 * The RcvEgr buffer need to be handled differently
 			goto done;
 		}
 		vm_flags_clear(vma, VM_MAYWRITE);
-		addr = vma->vm_start;
+		/*
+		 * Mmap multiple separate allocations into a single vma.  From
+		 * here, dma_mmap_coherent() calls dma_direct_mmap(), which
+		 * requires the mmap to exactly fill the vma starting at
+		 * vma_start.  Adjust the vma start and end for each eager
+		 * buffer segment mapped.  Restore the originals when done.
+		 */
+		vm_start_save = vma->vm_start;
+		vm_end_save = vma->vm_end;
+		vma->vm_end = vma->vm_start;
 		for (i = 0 ; i < uctxt->egrbufs.numbufs; i++) {
 			memlen = uctxt->egrbufs.buffers[i].len;
 			memvirt = uctxt->egrbufs.buffers[i].addr;
-			ret = remap_pfn_range(
+			memdma = uctxt->egrbufs.buffers[i].dma;
-				vma, addr,
+			vma->vm_end += memlen;
-				/*
+			mmap_cdbg(ctxt, subctxt, type, mapio, vmf, memaddr,
-				 * virt_to_pfn() does the same, but
+				  memvirt, memdma, memlen, vma);
-				 * it's not available on x86_64
+			ret = dma_mmap_coherent(&dd->pcidev->dev, vma,
-				 * when CONFIG_MMU is enabled.
+						memvirt, memdma, memlen);
-				 */
+			if (ret < 0) {
-				PFN_DOWN(__pa(memvirt)),
+				vma->vm_start = vm_start_save;
-				memlen,
+				vma->vm_end = vm_end_save;
-				vma->vm_page_prot);
-			if (ret < 0)
 				goto done;
-			addr += memlen;
+			}
+			vma->vm_start += memlen;
 		}
+		vma->vm_start = vm_start_save;
+		vma->vm_end = vm_end_save;
 		ret = 0;
 		goto done;
 	}
 		}
 		memlen = PAGE_SIZE;
 		memvirt = (void *)hfi1_rcvhdrtail_kvaddr(uctxt);
+		memdma = uctxt->rcvhdrqtailaddr_dma;
 		flags &= ~VM_MAYWRITE;
 		break;
 	case SUBCTXT_UREGS:
 	}
 	vm_flags_reset(vma, flags);
-	hfi1_cdbg(PROC,
+	mmap_cdbg(ctxt, subctxt, type, mapio, vmf, memaddr, memvirt, memdma,
-		  "%u:%u type:%u io/vf:%d/%d, addr:0x%llx, len:%lu(%lu), flags:0x%lx\n",
+		  memlen, vma);
-		    ctxt, subctxt, type, mapio, vmf, memaddr, memlen,
-		    vma->vm_end - vma->vm_start, vma->vm_flags);
 	if (vmf) {
 		vma->vm_pgoff = PFN_DOWN(memaddr);
 		vma->vm_ops = &vm_ops;
 		ret = 0;
+	} else if (memdma) {
+		ret = dma_mmap_coherent(&dd->pcidev->dev, vma,
+					memvirt, memdma, memlen);
 	} else if (mapio) {
 		ret = io_remap_pfn_range(vma, vma->vm_start,
 					 PFN_DOWN(memaddr),

drivers/infiniband/hw/hfi1/init.c+1−1

@@ -464,7 +464,7 @@ bail: · 463 unmodified lines
464	*	464	*
465	* This wrapper is the free function that matches hfi1_create_ctxtdata().	465	* This wrapper is the free function that matches hfi1_create_ctxtdata().
466	* When a context is done being used (kernel or user), this function is called	466	* When a context is done being used (kernel or user), this function is called
467	* for the "final" put to match the kref init from hf1i_create_ctxtdata().	467	* for the "final" put to match the kref init from hfi1_create_ctxtdata().
468	* Other users of the context do a get/put sequence to make sure that the	468	* Other users of the context do a get/put sequence to make sure that the
469	* structure isn't removed while in use.	469	* structure isn't removed while in use.
470	*/	470	*/

drivers/infiniband/hw/hfi1/sdma.c+2−2

 {
 	int rval = 0;
-	tx->num_desc++;
+	if ((unlikely(tx->num_desc + 1 == tx->desc_limit))) {
-	if ((unlikely(tx->num_desc == tx->desc_limit))) {
 		rval = _extend_sdma_tx_descs(dd, tx);
 		if (rval) {
 			__sdma_txclean(dd, tx);
 		SDMA_MAP_NONE,
 		dd->sdma_pad_phys,
 		sizeof(u32) - (tx->packet_len & (sizeof(u32) - 1)));
+	tx->num_desc++;
 	_sdma_close_tx(dd, tx);
 	return rval;
 }

drivers/infiniband/hw/hfi1/sdma.h+7−8

@@ -631,14 +631,13 @@ static inline void sdma_txclean(struct hfi1_devdata dd, struct sdma_txreq tx) · 630 unmodified lines
631	static inline void _sdma_close_tx(struct hfi1_devdata *dd,	631	static inline void _sdma_close_tx(struct hfi1_devdata *dd,
632	struct sdma_txreq *tx)	632	struct sdma_txreq *tx)
633	{	633	{
634	tx->descp[tx->num_desc].qw[0] \|=	634	u16 last_desc = tx->num_desc - 1;
635	SDMA_DESC0_LAST_DESC_FLAG;	635
636	tx->descp[tx->num_desc].qw[1] \|=	636	tx->descp[last_desc].qw[0] \|= SDMA_DESC0_LAST_DESC_FLAG;
637	dd->default_desc1;	637	tx->descp[last_desc].qw[1] \|= dd->default_desc1;
638	if (tx->flags & SDMA_TXREQ_F_URGENT)	638	if (tx->flags & SDMA_TXREQ_F_URGENT)
639	tx->descp[tx->num_desc].qw[1] \|=	639	tx->descp[last_desc].qw[1] \|= (SDMA_DESC1_HEAD_TO_HOST_FLAG \|
640	(SDMA_DESC1_HEAD_TO_HOST_FLAG \|	640	SDMA_DESC1_INT_REQ_FLAG);
641	SDMA_DESC1_INT_REQ_FLAG);
642	}	641	}
643		642
644	static inline int _sdma_txadd_daddr(	643	static inline int _sdma_txadd_daddr(
@@ -655,6 +654,7 @@ static inline int _sdma_txadd_daddr( · 10 unmodified lines
655	type,	654	type,
656	addr, len);	655	addr, len);
657	WARN_ON(len > tx->tlen);	656	WARN_ON(len > tx->tlen);
		657	tx->num_desc++;
658	tx->tlen -= len;	658	tx->tlen -= len;
659	/* special cases for last */	659	/* special cases for last */
660	if (!tx->tlen) {	660	if (!tx->tlen) {
@@ -666,7 +666,6 @@ static inline int _sdma_txadd_daddr( · 5 unmodified lines
666	_sdma_close_tx(dd, tx);	666	_sdma_close_tx(dd, tx);
667	}	667	}
668	}	668	}
669	tx->num_desc++;
670	return rval;	669	return rval;
671	}	670	}
672		671

drivers/infiniband/hw/hfi1/user_exp_rcv.c+20−26

 			         const struct mmu_notifier_range *range,
 			         unsigned long cur_seq);
 static int program_rcvarray(struct hfi1_filedata *fd, struct tid_user_buf *,
-			    struct tid_group *grp,
+			    struct tid_group *grp, u16 count,
-			    unsigned int start, u16 count,
 			    u32 *tidlist, unsigned int *tididx,
 			    unsigned int *pmapped);
 static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo);
 	int ret = 0, need_group = 0, pinned;
 	struct hfi1_ctxtdata *uctxt = fd->uctxt;
 	struct hfi1_devdata *dd = uctxt->dd;
-	unsigned int ngroups, pageidx = 0, pageset_count,
+	unsigned int ngroups, pageset_count,
 		tididx = 0, mapped, mapped_pages = 0;
 	u32 *tidlist = NULL;
 	struct tid_user_buf *tidbuf;
 			tid_group_pop(&uctxt->tid_group_list);
 		ret = program_rcvarray(fd, tidbuf, grp,
-				       pageidx, dd->rcv_entries.group_size,
+				       dd->rcv_entries.group_size,
 				       tidlist, &tididx, &mapped);
 		/*
 		 * If there was a failure to program the RcvArray
 		tid_group_add_tail(grp, &uctxt->tid_full_list);
 		ngroups--;
-		pageidx += ret;
 		mapped_pages += mapped;
 	}
-	while (pageidx < pageset_count) {
+	while (tididx < pageset_count) {
 		struct tid_group *grp, *ptr;
 		/*
 		 * If we don't have any partially used tid groups, check
 		 */
 		list_for_each_entry_safe(grp, ptr, &uctxt->tid_used_list.list,
 					 list) {
-			unsigned use = min_t(unsigned, pageset_count - pageidx,
+			unsigned use = min_t(unsigned, pageset_count - tididx,
 					     grp->size - grp->used);
 			ret = program_rcvarray(fd, tidbuf, grp,
-					       pageidx, use, tidlist,
+					       use, tidlist,
 					       &tididx, &mapped);
 			if (ret < 0) {
 				hfi1_cdbg(TID,
 					tid_group_move(grp,
 						       &uctxt->tid_used_list,
 						       &uctxt->tid_full_list);
-				pageidx += ret;
 				mapped_pages += mapped;
 				need_group = 0;
 				/* Check if we are done so we break out early */
-				if (pageidx >= pageset_count)
+				if (tididx >= pageset_count)
 					break;
 			} else if (WARN_ON(ret == 0)) {
 				/*
  *	  struct tid_pageset holding information on physically contiguous
  *	  chunks from the user buffer), and other fields.
  * @grp: RcvArray group
- * @start: starting index into sets array
  * @count: number of struct tid_pageset's to program
  * @tidlist: the array of u32 elements when the information about the
  *           programmed RcvArray entries is to be encoded.
  * number of RcvArray entries programmed.
  */
 static int program_rcvarray(struct hfi1_filedata *fd, struct tid_user_buf *tbuf,
-			    struct tid_group *grp,
+			    struct tid_group *grp, u16 count,
-			    unsigned int start, u16 count,
 			    u32 *tidlist, unsigned int *tididx,
 			    unsigned int *pmapped)
 {
 	struct hfi1_ctxtdata *uctxt = fd->uctxt;
 	struct hfi1_devdata *dd = uctxt->dd;
 	u16 idx;
+	unsigned int start = *tididx;
 	u32 tidinfo = 0, rcventry, useidx = 0;
 	int mapped = 0;
 			return ret;
 		mapped += npages;
-		tidinfo = rcventry2tidinfo(rcventry - uctxt->expected_base) |
+		tidinfo = create_tid(rcventry - uctxt->expected_base, npages);
-			EXP_TID_SET(LEN, npages);
 		tidlist[(*tididx)++] = tidinfo;
 		grp->used++;
 		grp->map |= 1 << useidx++;
 	struct hfi1_ctxtdata *uctxt = fd->uctxt;
 	struct hfi1_devdata *dd = uctxt->dd;
 	struct tid_rb_node *node;
-	u8 tidctrl = EXP_TID_GET(tidinfo, CTRL);
+	u32 tidctrl = EXP_TID_GET(tidinfo, CTRL);
 	u32 tididx = EXP_TID_GET(tidinfo, IDX) << 1, rcventry;
-	if (tididx >= uctxt->expected_count) {
+	if (tidctrl == 0x3 || tidctrl == 0x0)
-		dd_dev_err(dd, "Invalid RcvArray entry (%u) index for ctxt %u\n",
-			   tididx, uctxt->ctxt);
-		return -EINVAL;
-	}
-	if (tidctrl == 0x3)
 		return -EINVAL;
 	rcventry = tididx + (tidctrl - 1);
+	if (rcventry >= uctxt->expected_count) {
+		dd_dev_err(dd, "Invalid RcvArray entry (%u) index for ctxt %u\n",
+			   rcventry, uctxt->ctxt);
+		return -EINVAL;
+	}
 	node = fd->entry_to_rb[rcventry];
 	if (!node || node->rcventry != (uctxt->expected_base + rcventry))
 		return -EBADF;
 	spin_lock(&fdata->invalid_lock);
 	if (fdata->invalid_tid_idx < uctxt->expected_count) {
 		fdata->invalid_tids[fdata->invalid_tid_idx] =
-			rcventry2tidinfo(node->rcventry - uctxt->expected_base);
+			create_tid(node->rcventry - uctxt->expected_base,
-		fdata->invalid_tids[fdata->invalid_tid_idx] |=
+				   node->npages);
-			EXP_TID_SET(LEN, node->npages);
 		if (!fdata->invalid_tid_idx) {
 			unsigned long *ev;

drivers/infiniband/hw/hfi1/user_pages.c+40−21

@@ -29,33 +29,52 @@ MODULE_PARM_DESC(cache_size, "Send and receive side cache size limit (in MB)"); · 28 unmodified lines
29	bool hfi1_can_pin_pages(struct hfi1_devdata dd, struct mm_struct mm,	29	bool hfi1_can_pin_pages(struct hfi1_devdata dd, struct mm_struct mm,
30	u32 nlocked, u32 npages)	30	u32 nlocked, u32 npages)
31	{	31	{
32	unsigned long ulimit = rlimit(RLIMIT_MEMLOCK), pinned, cache_limit,	32	unsigned long ulimit_pages;
33	size = (cache_size * (1UL << 20)); /* convert to bytes */	33	unsigned long cache_limit_pages;
34	unsigned int usr_ctxts =	34	unsigned int usr_ctxts;
35	dd->num_rcv_contexts - dd->first_dyn_alloc_ctxt;
36	bool can_lock = capable(CAP_IPC_LOCK);
37		35
38	/*	36	/*
39	* Calculate per-cache size. The calculation below uses only a quarter	37	* Perform RLIMIT_MEMLOCK based checks unless CAP_IPC_LOCK is present.
40	* of the available per-context limit. This leaves space for other
41	* pinning. Should we worry about shared ctxts?
42	*/	38	*/
43	cache_limit = (ulimit / usr_ctxts) / 4;	39	if (!capable(CAP_IPC_LOCK)) {
44		40	ulimit_pages =
45	/* If ulimit isn't set to "unlimited" and is smaller than cache_size. */	41	DIV_ROUND_DOWN_ULL(rlimit(RLIMIT_MEMLOCK), PAGE_SIZE);
46	if (ulimit != (-1UL) && size > cache_limit)	42
47	size = cache_limit;	43	/*
48		44	* Pinning these pages would exceed this process's locked memory
49	/* Convert to number of pages */	45	* limit.
50	size = DIV_ROUND_UP(size, PAGE_SIZE);	46	*/
51		47	if (atomic64_read(&mm->pinned_vm) + npages > ulimit_pages)
52	pinned = atomic64_read(&mm->pinned_vm);	48	return false;
		49
		50	/*
		51	* Only allow 1/4 of the user's RLIMIT_MEMLOCK to be used for HFI
		52	* caches. This fraction is then equally distributed among all
		53	* existing user contexts. Note that if RLIMIT_MEMLOCK is
		54	* 'unlimited' (-1), the value of this limit will be > 2^42 pages
		55	* (2^64 / 2^12 / 2^8 / 2^2).
		56	*
		57	* The effectiveness of this check may be reduced if I/O occurs on
		58	* some user contexts before all user contexts are created. This
		59	* check assumes that this process is the only one using this
		60	* context (e.g., the corresponding fd was not passed to another
		61	* process for concurrent access) as there is no per-context,
		62	* per-process tracking of pinned pages. It also assumes that each
		63	* user context has only one cache to limit.
		64	*/
		65	usr_ctxts = dd->num_rcv_contexts - dd->first_dyn_alloc_ctxt;
		66	if (nlocked + npages > (ulimit_pages / usr_ctxts / 4))
		67	return false;
		68	}
53		69
54	/* First, check the absolute limit against all pinned pages. */	70	/*
55	if (pinned + npages >= ulimit && !can_lock)	71	* Pinning these pages would exceed the size limit for this cache.
		72	*/
		73	cache_limit_pages = cache_size * (1024 * 1024) / PAGE_SIZE;
		74	if (nlocked + npages > cache_limit_pages)
56	return false;	75	return false;
57		76
58	return ((nlocked + npages) <= size) \|\| can_lock;	77	return true;
59	}	78	}
60		79
61	int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr, size_t npages,	80	int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr, size_t npages,

drivers/infiniband/hw/hfi1/verbs.c+32−49

 	"DRIVER_EgrHdrFull"
 };
-static DEFINE_MUTEX(cntr_names_lock); /* protects the *_cntr_names bufers */
 static struct rdma_stat_desc *dev_cntr_descs;
 static struct rdma_stat_desc *port_cntr_descs;
 int num_driver_cntrs = ARRAY_SIZE(driver_cntr_names);
 static int num_dev_cntrs;
 static int num_port_cntrs;
-static int cntr_names_initialized;
 /*
  * Convert a list of names separated by '\n' into an array of NULL terminated
 			   int num_extra_names, int *num_cntrs,
 			   struct rdma_stat_desc **cntr_descs)
 {
-	struct rdma_stat_desc *q;
+	struct rdma_stat_desc *names_out;
-	char *names_out, *p;
+	char *p;
 	int i, n;
 	n = 0;
 		if (names_in[i] == '\n')
 			n++;
-	names_out =
+	names_out = kzalloc((n + num_extra_names) * sizeof(*names_out)
-		kzalloc((n + num_extra_names) * sizeof(*q) + names_len,
+				+ names_len,
-			GFP_KERNEL);
+			    GFP_KERNEL);
 	if (!names_out) {
 		*num_cntrs = 0;
 		*cntr_descs = NULL;
 		return -ENOMEM;
 	}
-	p = names_out + (n + num_extra_names) * sizeof(*q);
+	p = (char *)&names_out[n + num_extra_names];
 	memcpy(p, names_in, names_len);
-	q = (struct rdma_stat_desc *)names_out;
 	for (i = 0; i < n; i++) {
-		q[i].name = p;
+		names_out[i].name = p;
 		p = strchr(p, '\n');
 		*p++ = '\0';
 	}
 	*num_cntrs = n;
-	*cntr_descs = (struct rdma_stat_desc *)names_out;
+	*cntr_descs = names_out;
 	return 0;
 }
-static int init_counters(struct ib_device *ibdev)
-{
-	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);
-	int i, err = 0;
-	mutex_lock(&cntr_names_lock);
-	if (cntr_names_initialized)
-		goto out_unlock;
-	err = init_cntr_names(dd->cntrnames, dd->cntrnameslen, num_driver_cntrs,
-			      &num_dev_cntrs, &dev_cntr_descs);
-	if (err)
-		goto out_unlock;
-	for (i = 0; i < num_driver_cntrs; i++)
-		dev_cntr_descs[num_dev_cntrs + i].name = driver_cntr_names[i];
-	err = init_cntr_names(dd->portcntrnames, dd->portcntrnameslen, 0,
-			      &num_port_cntrs, &port_cntr_descs);
-	if (err) {
-		kfree(dev_cntr_descs);
-		dev_cntr_descs = NULL;
-		goto out_unlock;
-	}
-	cntr_names_initialized = 1;
-out_unlock:
-	mutex_unlock(&cntr_names_lock);
-	return err;
-}
 static struct rdma_hw_stats *hfi1_alloc_hw_device_stats(struct ib_device *ibdev)
 {
-	if (init_counters(ibdev))
+	if (!dev_cntr_descs) {
-		return NULL;
+		struct hfi1_devdata *dd = dd_from_ibdev(ibdev);
+		int i, err;
+		err = init_cntr_names(dd->cntrnames, dd->cntrnameslen,
+				      num_driver_cntrs,
+				      &num_dev_cntrs, &dev_cntr_descs);
+		if (err)
+			return NULL;
+		for (i = 0; i < num_driver_cntrs; i++)
+			dev_cntr_descs[num_dev_cntrs + i].name =
+							driver_cntr_names[i];
+	}
 	return rdma_alloc_hw_stats_struct(dev_cntr_descs,
 					  num_dev_cntrs + num_driver_cntrs,
 					  RDMA_HW_STATS_DEFAULT_LIFESPAN);
 static struct rdma_hw_stats *hfi_alloc_hw_port_stats(struct ib_device *ibdev,
 						     u32 port_num)
 {
-	if (init_counters(ibdev))
+	if (!port_cntr_descs) {
-		return NULL;
+		struct hfi1_devdata *dd = dd_from_ibdev(ibdev);
+		int err;
+		err = init_cntr_names(dd->portcntrnames, dd->portcntrnameslen,
+				      0,
+				      &num_port_cntrs, &port_cntr_descs);
+		if (err)
+			return NULL;
+	}
 	return rdma_alloc_hw_stats_struct(port_cntr_descs, num_port_cntrs,
 					  RDMA_HW_STATS_DEFAULT_LIFESPAN);
 }
 	del_timer_sync(&dev->mem_timer);
 	verbs_txreq_exit(dev);
-	mutex_lock(&cntr_names_lock);
 	kfree(dev_cntr_descs);
 	kfree(port_cntr_descs);
 	dev_cntr_descs = NULL;
 	port_cntr_descs = NULL;
-	cntr_names_initialized = 0;
-	mutex_unlock(&cntr_names_lock);
 }
 void hfi1_cnp_rcv(struct hfi1_packet *packet)

drivers/infiniband/hw/hns/hns_roce_device.h+2−17

 	HNS_ROCE_CAP_FLAG_DIRECT_WQE		= BIT(12),
 	HNS_ROCE_CAP_FLAG_SDI_MODE		= BIT(14),
 	HNS_ROCE_CAP_FLAG_STASH			= BIT(17),
+	HNS_ROCE_CAP_FLAG_CQE_INLINE		= BIT(19),
 };
 #define HNS_ROCE_DB_TYPE_COUNT			2
 struct hns_roce_dev;
-struct hns_roce_rinl_sge {
-	void			*addr;
-	u32			len;
-};
-struct hns_roce_rinl_wqe {
-	struct hns_roce_rinl_sge *sg_list;
-	u32			 sge_cnt;
-};
-struct hns_roce_rinl_buf {
-	struct hns_roce_rinl_wqe *wqe_list;
-	u32			 wqe_cnt;
-};
 enum {
 	HNS_ROCE_FLUSH_FLAG = 0,
 };
 	/* 0: flush needed, 1: unneeded */
 	unsigned long		flush_flag;
 	struct hns_roce_work	flush_work;
-	struct hns_roce_rinl_buf rq_inl_buf;
 	struct list_head	node; /* all qps are on a list */
 	struct list_head	rq_node; /* all recv qps are on a list */
 	struct list_head	sq_node; /* all send qps are on a list */
 			 u32 step_idx);
 	int (*modify_qp)(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
 			 int attr_mask, enum ib_qp_state cur_state,
-			 enum ib_qp_state new_state);
+			 enum ib_qp_state new_state, struct ib_udata *udata);
 	int (*qp_flow_control_init)(struct hns_roce_dev *hr_dev,
 			 struct hns_roce_qp *hr_qp);
 	void (*dereg_mr)(struct hns_roce_dev *hr_dev);

drivers/infiniband/hw/hns/hns_roce_hw_v2.c+33−76

 static void fill_rq_wqe(struct hns_roce_qp *hr_qp, const struct ib_recv_wr *wr,
 			u32 wqe_idx, u32 max_sge)
 {
-	struct hns_roce_rinl_sge *sge_list;
 	void *wqe = NULL;
-	u32 i;
 	wqe = hns_roce_get_recv_wqe(hr_qp, wqe_idx);
 	fill_recv_sge_to_wqe(wr, wqe, max_sge, hr_qp->rq.rsv_sge);
-	/* rq support inline data */
-	if (hr_qp->rq_inl_buf.wqe_cnt) {
-		sge_list = hr_qp->rq_inl_buf.wqe_list[wqe_idx].sg_list;
-		hr_qp->rq_inl_buf.wqe_list[wqe_idx].sge_cnt = (u32)wr->num_sge;
-		for (i = 0; i < wr->num_sge; i++) {
-			sge_list[i].addr = (void *)(u64)wr->sg_list[i].addr;
-			sge_list[i].len = wr->sg_list[i].length;
-		}
-	}
 }
 static int hns_roce_v2_post_recv(struct ib_qp *ibqp,
 	attr->port_num = 1;
 	attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;
 	ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_INIT,
-				    IB_QPS_INIT);
+				    IB_QPS_INIT, NULL);
 	if (ret) {
 		ibdev_err(ibdev, "failed to modify qp to init, ret = %d.\n",
 			  ret);
 	rdma_ah_set_sl(&attr->ah_attr, (u8)sl_num);
 	ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_INIT,
-				    IB_QPS_RTR);
+				    IB_QPS_RTR, NULL);
 	hr_dev->loop_idc = loopback;
 	if (ret) {
 		ibdev_err(ibdev, "failed to modify qp to rtr, ret = %d.\n",
 	attr->retry_cnt = HNS_ROCE_FREE_MR_USED_QP_RETRY_CNT;
 	attr->timeout = HNS_ROCE_FREE_MR_USED_QP_TIMEOUT;
 	ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_RTR,
-				    IB_QPS_RTS);
+				    IB_QPS_RTS, NULL);
 	if (ret)
 		ibdev_err(ibdev, "failed to modify qp to rts, ret = %d.\n",
 			  ret);
 	return 0;
 }
-static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe,
-					struct hns_roce_qp *qp,
-					struct ib_wc *wc)
-{
-	struct hns_roce_rinl_sge *sge_list;
-	u32 wr_num, wr_cnt, sge_num;
-	u32 sge_cnt, data_len, size;
-	void *wqe_buf;
-	wr_num = hr_reg_read(cqe, CQE_WQE_IDX);
-	wr_cnt = wr_num & (qp->rq.wqe_cnt - 1);
-	sge_list = qp->rq_inl_buf.wqe_list[wr_cnt].sg_list;
-	sge_num = qp->rq_inl_buf.wqe_list[wr_cnt].sge_cnt;
-	wqe_buf = hns_roce_get_recv_wqe(qp, wr_cnt);
-	data_len = wc->byte_len;
-	for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); sge_cnt++) {
-		size = min(sge_list[sge_cnt].len, data_len);
-		memcpy((void *)sge_list[sge_cnt].addr, wqe_buf, size);
-		data_len -= size;
-		wqe_buf += size;
-	}
-	if (unlikely(data_len)) {
-		wc->status = IB_WC_LOC_LEN_ERR;
-		return -EAGAIN;
-	}
-	return 0;
-}
 static int sw_comp(struct hns_roce_qp *hr_qp, struct hns_roce_wq *wq,
 		   int num_entries, struct ib_wc *wc)
 {
 		wc->opcode = ib_opcode;
 }
-static inline bool is_rq_inl_enabled(struct ib_wc *wc, u32 hr_opcode,
-				     struct hns_roce_v2_cqe *cqe)
-{
-	return wc->qp->qp_type != IB_QPT_UD && wc->qp->qp_type != IB_QPT_GSI &&
-	       (hr_opcode == HNS_ROCE_V2_OPCODE_SEND ||
-		hr_opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_IMM ||
-		hr_opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_INV) &&
-	       hr_reg_read(cqe, CQE_RQ_INLINE);
-}
 static int fill_recv_wc(struct ib_wc *wc, struct hns_roce_v2_cqe *cqe)
 {
-	struct hns_roce_qp *qp = to_hr_qp(wc->qp);
 	u32 hr_opcode;
 	int ib_opcode;
-	int ret;
 	wc->byte_len = le32_to_cpu(cqe->byte_cnt);
 	else
 		wc->opcode = ib_opcode;
-	if (is_rq_inl_enabled(wc, hr_opcode, cqe)) {
-		ret = hns_roce_handle_recv_inl_wqe(cqe, qp, wc);
-		if (unlikely(ret))
-			return ret;
-	}
 	wc->sl = hr_reg_read(cqe, CQE_SL);
 	wc->src_qp = hr_reg_read(cqe, CQE_RMT_QPN);
 	wc->slid = 0;
 	hr_reg_write(context, QPC_RQ_DB_RECORD_ADDR_H,
 		     upper_32_bits(hr_qp->rdb.dma));
-	if (ibqp->qp_type != IB_QPT_UD && ibqp->qp_type != IB_QPT_GSI)
-		hr_reg_write_bool(context, QPC_RQIE,
-			     hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE);
 	hr_reg_write(context, QPC_RX_CQN, get_cqn(ibqp->recv_cq));
 	if (ibqp->srq) {
 static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
 				 const struct ib_qp_attr *attr, int attr_mask,
 				 struct hns_roce_v2_qp_context *context,
-				 struct hns_roce_v2_qp_context *qpc_mask)
+				 struct hns_roce_v2_qp_context *qpc_mask,
+				 struct ib_udata *udata)
 {
+	struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(udata,
+					  struct hns_roce_ucontext, ibucontext);
 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
 	struct ib_device *ibdev = &hr_dev->ib_dev;
 	hr_reg_write(context, QPC_LP_SGEN_INI, 3);
 	hr_reg_clear(qpc_mask, QPC_LP_SGEN_INI);
+	if (udata && ibqp->qp_type == IB_QPT_RC &&
+	    (uctx->config & HNS_ROCE_RQ_INLINE_FLAGS)) {
+		hr_reg_write_bool(context, QPC_RQIE,
+				  hr_dev->caps.flags &
+				  HNS_ROCE_CAP_FLAG_RQ_INLINE);
+		hr_reg_clear(qpc_mask, QPC_RQIE);
+	}
+	if (udata &&
+	    (ibqp->qp_type == IB_QPT_RC || ibqp->qp_type == IB_QPT_XRC_TGT) &&
+	    (uctx->config & HNS_ROCE_CQE_INLINE_FLAGS)) {
+		hr_reg_write_bool(context, QPC_CQEIE,
+				  hr_dev->caps.flags &
+				  HNS_ROCE_CAP_FLAG_CQE_INLINE);
+		hr_reg_clear(qpc_mask, QPC_CQEIE);
+		hr_reg_write(context, QPC_CQEIS, 0);
+		hr_reg_clear(qpc_mask, QPC_CQEIS);
+	}
 	return 0;
 }
 				      enum ib_qp_state cur_state,
 				      enum ib_qp_state new_state,
 				      struct hns_roce_v2_qp_context *context,
-				      struct hns_roce_v2_qp_context *qpc_mask)
+				      struct hns_roce_v2_qp_context *qpc_mask,
+				      struct ib_udata *udata)
 {
 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 	int ret = 0;
 		modify_qp_init_to_init(ibqp, attr, context, qpc_mask);
 	} else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
 		ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context,
-					    qpc_mask);
+					    qpc_mask, udata);
 	} else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
 		ret = modify_qp_rtr_to_rts(ibqp, attr, attr_mask, context,
 					   qpc_mask);
 static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
 				 const struct ib_qp_attr *attr,
 				 int attr_mask, enum ib_qp_state cur_state,
-				 enum ib_qp_state new_state)
+				 enum ib_qp_state new_state, struct ib_udata *udata)
 {
 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
 	memset(qpc_mask, 0xff, hr_dev->caps.qpc_sz);
 	ret = hns_roce_v2_set_abs_fields(ibqp, attr, attr_mask, cur_state,
-					 new_state, context, qpc_mask);
+					 new_state, context, qpc_mask, udata);
 	if (ret)
 		goto out;
 	if (modify_qp_is_ok(hr_qp)) {
 		/* Modify qp to reset before destroying qp */
 		ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0,
-					    hr_qp->state, IB_QPS_RESET);
+					    hr_qp->state, IB_QPS_RESET, udata);
 		if (ret)
 			ibdev_err(ibdev,
 				  "failed to modify QP to RST, ret = %d.\n",

drivers/infiniband/hw/hns/hns_roce_hw_v2.h+2−1

 #define QPC_RQ_RTY_TX_ERR QPC_FIELD_LOC(607, 607)
 #define QPC_RX_CQN QPC_FIELD_LOC(631, 608)
 #define QPC_XRC_QP_TYPE QPC_FIELD_LOC(632, 632)
-#define QPC_RSV3 QPC_FIELD_LOC(634, 633)
+#define QPC_CQEIE QPC_FIELD_LOC(633, 633)
+#define QPC_CQEIS QPC_FIELD_LOC(634, 634)
 #define QPC_MIN_RNR_TIME QPC_FIELD_LOC(639, 635)
 #define QPC_RQ_PRODUCER_IDX QPC_FIELD_LOC(655, 640)
 #define QPC_RQ_CONSUMER_IDX QPC_FIELD_LOC(671, 656)

drivers/infiniband/hw/hns/hns_roce_main.c+15−2

 		resp.max_inline_data = hr_dev->caps.max_sq_inline;
 	}
+	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) {
+		context->config |= ucmd.config & HNS_ROCE_RQ_INLINE_FLAGS;
+		if (context->config & HNS_ROCE_RQ_INLINE_FLAGS)
+			resp.config |= HNS_ROCE_RSP_RQ_INLINE_FLAGS;
+	}
+	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQE_INLINE) {
+		context->config |= ucmd.config & HNS_ROCE_CQE_INLINE_FLAGS;
+		if (context->config & HNS_ROCE_CQE_INLINE_FLAGS)
+			resp.config |= HNS_ROCE_RSP_CQE_INLINE_FLAGS;
+	}
 	ret = hns_roce_uar_alloc(hr_dev, &context->uar);
 	if (ret)
 		goto error_fail_uar_alloc;
 		prot = pgprot_device(vma->vm_page_prot);
 		break;
 	default:
-		return -EINVAL;
+		ret = -EINVAL;
+		goto out;
 	}
 	ret = rdma_user_mmap_io(uctx, vma, pfn, rdma_entry->npages * PAGE_SIZE,
 				prot, rdma_entry);
+out:
 	rdma_user_mmap_entry_put(rdma_entry);
 	return ret;
 }

drivers/infiniband/hw/hns/hns_roce_qp.c+1−65

@@ -433,7 +433,6 @@ static int set_rq_size(struct hns_roce_dev hr_dev, struct ib_qp_cap cap, · 432 unmodified lines
433	if (!has_rq) {	433	if (!has_rq) {
434	hr_qp->rq.wqe_cnt = 0;	434	hr_qp->rq.wqe_cnt = 0;
435	hr_qp->rq.max_gs = 0;	435	hr_qp->rq.max_gs = 0;
436	hr_qp->rq_inl_buf.wqe_cnt = 0;
437	cap->max_recv_wr = 0;	436	cap->max_recv_wr = 0;
438	cap->max_recv_sge = 0;	437	cap->max_recv_sge = 0;
439		438
@@ -463,12 +462,6 @@ static int set_rq_size(struct hns_roce_dev hr_dev, struct ib_qp_cap cap, · 23 unmodified lines
463	hr_qp->rq.max_gs);	462	hr_qp->rq.max_gs);
464		463
465	hr_qp->rq.wqe_cnt = cnt;	464	hr_qp->rq.wqe_cnt = cnt;
466	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE &&
467	hr_qp->ibqp.qp_type != IB_QPT_UD &&
468	hr_qp->ibqp.qp_type != IB_QPT_GSI)
469	hr_qp->rq_inl_buf.wqe_cnt = cnt;
470	else
471	hr_qp->rq_inl_buf.wqe_cnt = 0;
472		465
473	cap->max_recv_wr = cnt;	466	cap->max_recv_wr = cnt;
474	cap->max_recv_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge;	467	cap->max_recv_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge;
@@ -732,49 +725,6 @@ static int hns_roce_qp_has_rq(struct ib_qp_init_attr *attr) · 257 unmodified lines
732	return 1;	725	return 1;
733	}	726	}
734		727
735	static int alloc_rq_inline_buf(struct hns_roce_qp *hr_qp,
736	struct ib_qp_init_attr *init_attr)
737	{
738	u32 max_recv_sge = init_attr->cap.max_recv_sge;
739	u32 wqe_cnt = hr_qp->rq_inl_buf.wqe_cnt;
740	struct hns_roce_rinl_wqe *wqe_list;
741	int i;
742
743	/* allocate recv inline buf */
744	wqe_list = kcalloc(wqe_cnt, sizeof(struct hns_roce_rinl_wqe),
745	GFP_KERNEL);
746	if (!wqe_list)
747	goto err;
748
749	/* Allocate a continuous buffer for all inline sge we need */
750	wqe_list[0].sg_list = kcalloc(wqe_cnt, (max_recv_sge *
751	sizeof(struct hns_roce_rinl_sge)),
752	GFP_KERNEL);
753	if (!wqe_list[0].sg_list)
754	goto err_wqe_list;
755
756	/* Assign buffers of sg_list to each inline wqe */
757	for (i = 1; i < wqe_cnt; i++)
758	wqe_list[i].sg_list = &wqe_list[0].sg_list[i * max_recv_sge];
759
760	hr_qp->rq_inl_buf.wqe_list = wqe_list;
761
762	return 0;
763
764	err_wqe_list:
765	kfree(wqe_list);
766
767	err:
768	return -ENOMEM;
769	}
770
771	static void free_rq_inline_buf(struct hns_roce_qp *hr_qp)
772	{
773	if (hr_qp->rq_inl_buf.wqe_list)
774	kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list);
775	kfree(hr_qp->rq_inl_buf.wqe_list);
776	}
777
778	static int alloc_qp_buf(struct hns_roce_dev hr_dev, struct hns_roce_qp hr_qp,	728	static int alloc_qp_buf(struct hns_roce_dev hr_dev, struct hns_roce_qp hr_qp,
779	struct ib_qp_init_attr *init_attr,	729	struct ib_qp_init_attr *init_attr,
780	struct ib_udata *udata, unsigned long addr)	730	struct ib_udata *udata, unsigned long addr)
@@ -783,18 +733,6 @@ static int alloc_qp_buf(struct hns_roce_dev hr_dev, struct hns_roce_qp hr_qp, · 2 unmodified lines
783	struct hns_roce_buf_attr buf_attr = {};	733	struct hns_roce_buf_attr buf_attr = {};
784	int ret;	734	int ret;
785		735
786	if (!udata && hr_qp->rq_inl_buf.wqe_cnt) {
787	ret = alloc_rq_inline_buf(hr_qp, init_attr);
788	if (ret) {
789	ibdev_err(ibdev,
790	"failed to alloc inline buf, ret = %d.\n",
791	ret);
792	return ret;
793	}
794	} else {
795	hr_qp->rq_inl_buf.wqe_list = NULL;
796	}
797
798	ret = set_wqe_buf_attr(hr_dev, hr_qp, &buf_attr);	736	ret = set_wqe_buf_attr(hr_dev, hr_qp, &buf_attr);
799	if (ret) {	737	if (ret) {
800	ibdev_err(ibdev, "failed to split WQE buf, ret = %d.\n", ret);	738	ibdev_err(ibdev, "failed to split WQE buf, ret = %d.\n", ret);
@@ -814,7 +752,6 @@ static int alloc_qp_buf(struct hns_roce_dev hr_dev, struct hns_roce_qp hr_qp, · 13 unmodified lines
814	return 0;	752	return 0;
815		753
816	err_inline:	754	err_inline:
817	free_rq_inline_buf(hr_qp);
818		755
819	return ret;	756	return ret;
820	}	757	}
@@ -822,7 +759,6 @@ err_inline: · 1 unmodified line
822	static void free_qp_buf(struct hns_roce_dev hr_dev, struct hns_roce_qp hr_qp)	759	static void free_qp_buf(struct hns_roce_dev hr_dev, struct hns_roce_qp hr_qp)
823	{	760	{
824	hns_roce_mtr_destroy(hr_dev, &hr_qp->mtr);	761	hns_roce_mtr_destroy(hr_dev, &hr_qp->mtr);
825	free_rq_inline_buf(hr_qp);
826	}	762	}
827		763
828	static inline bool user_qp_has_sdb(struct hns_roce_dev *hr_dev,	764	static inline bool user_qp_has_sdb(struct hns_roce_dev *hr_dev,
@@ -1410,7 +1346,7 @@ int hns_roce_modify_qp(struct ib_qp ibqp, struct ib_qp_attr attr, · 581 unmodified lines
1410	goto out;	1346	goto out;
1411		1347
1412	ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state,	1348	ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state,
1413	new_state);	1349	new_state, udata);
1414		1350
1415	out:	1351	out:
1416	mutex_unlock(&hr_qp->mutex);	1352	mutex_unlock(&hr_qp->mutex);

drivers/infiniband/hw/irdma/hw.c+2−0

@@ -483,6 +483,8 @@ static int irdma_save_msix_info(struct irdma_pci_f *rf) · 482 unmodified lines
483	iw_qvlist->num_vectors = rf->msix_count;	483	iw_qvlist->num_vectors = rf->msix_count;
484	if (rf->msix_count <= num_online_cpus())	484	if (rf->msix_count <= num_online_cpus())
485	rf->msix_shared = true;	485	rf->msix_shared = true;
		486	else if (rf->msix_count > num_online_cpus() + 1)
		487	rf->msix_count = num_online_cpus() + 1;
486		488
487	pmsix = rf->msix_entries;	489	pmsix = rf->msix_entries;
488	for (i = 0, ceq_idx = 0; i < rf->msix_count; i++, iw_qvinfo++) {	490	for (i = 0, ceq_idx = 0; i < rf->msix_count; i++, iw_qvinfo++) {

drivers/infiniband/hw/irdma/verbs.c+210−103

@@ -2745,6 +2745,162 @@ static int irdma_hwreg_mr(struct irdma_device iwdev, struct irdma_mr iwmr, · 2744 unmodified lines
2745	return ret;	2745	return ret;
2746	}	2746	}
2747		2747
		2748	static int irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access)
		2749	{
		2750	struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);
		2751	struct irdma_pbl *iwpbl = &iwmr->iwpbl;
		2752	bool use_pbles;
		2753	u32 stag;
		2754	int err;
		2755
		2756	use_pbles = iwmr->page_cnt != 1;
		2757
		2758	err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles, false);
		2759	if (err)
		2760	return err;
		2761
		2762	if (use_pbles) {
		2763	err = irdma_check_mr_contiguous(&iwpbl->pble_alloc,
		2764	iwmr->page_size);
		2765	if (err) {
		2766	irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc);
		2767	iwpbl->pbl_allocated = false;
		2768	}
		2769	}
		2770
		2771	stag = irdma_create_stag(iwdev);
		2772	if (!stag) {
		2773	err = -ENOMEM;
		2774	goto free_pble;
		2775	}
		2776
		2777	iwmr->stag = stag;
		2778	iwmr->ibmr.rkey = stag;
		2779	iwmr->ibmr.lkey = stag;
		2780	err = irdma_hwreg_mr(iwdev, iwmr, access);
		2781	if (err)
		2782	goto err_hwreg;
		2783
		2784	return 0;
		2785
		2786	err_hwreg:
		2787	irdma_free_stag(iwdev, stag);
		2788
		2789	free_pble:
		2790	if (iwpbl->pble_alloc.level != PBLE_LEVEL_0 && iwpbl->pbl_allocated)
		2791	irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc);
		2792
		2793	return err;
		2794	}
		2795
		2796	static struct irdma_mr irdma_alloc_iwmr(struct ib_umem region,
		2797	struct ib_pd *pd, u64 virt,
		2798	enum irdma_memreg_type reg_type)
		2799	{
		2800	struct irdma_device *iwdev = to_iwdev(pd->device);
		2801	struct irdma_pbl *iwpbl = NULL;
		2802	struct irdma_mr *iwmr = NULL;
		2803	unsigned long pgsz_bitmap;
		2804
		2805	iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
		2806	if (!iwmr)
		2807	return ERR_PTR(-ENOMEM);
		2808
		2809	iwpbl = &iwmr->iwpbl;
		2810	iwpbl->iwmr = iwmr;
		2811	iwmr->region = region;
		2812	iwmr->ibmr.pd = pd;
		2813	iwmr->ibmr.device = pd->device;
		2814	iwmr->ibmr.iova = virt;
		2815	iwmr->type = reg_type;
		2816
		2817	pgsz_bitmap = (reg_type == IRDMA_MEMREG_TYPE_MEM) ?
		2818	iwdev->rf->sc_dev.hw_attrs.page_size_cap : PAGE_SIZE;
		2819
		2820	iwmr->page_size = ib_umem_find_best_pgsz(region, pgsz_bitmap, virt);
		2821	if (unlikely(!iwmr->page_size)) {
		2822	kfree(iwmr);
		2823	return ERR_PTR(-EOPNOTSUPP);
		2824	}
		2825
		2826	iwmr->len = region->length;
		2827	iwpbl->user_base = virt;
		2828	iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size);
		2829
		2830	return iwmr;
		2831	}
		2832
		2833	static void irdma_free_iwmr(struct irdma_mr *iwmr)
		2834	{
		2835	kfree(iwmr);
		2836	}
		2837
		2838	static int irdma_reg_user_mr_type_qp(struct irdma_mem_reg_req req,
		2839	struct ib_udata *udata,
		2840	struct irdma_mr *iwmr)
		2841	{
		2842	struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);
		2843	struct irdma_pbl *iwpbl = &iwmr->iwpbl;
		2844	struct irdma_ucontext *ucontext = NULL;
		2845	unsigned long flags;
		2846	bool use_pbles;
		2847	u32 total;
		2848	int err;
		2849
		2850	total = req.sq_pages + req.rq_pages + 1;
		2851	if (total > iwmr->page_cnt)
		2852	return -EINVAL;
		2853
		2854	total = req.sq_pages + req.rq_pages;
		2855	use_pbles = total > 2;
		2856	err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
		2857	if (err)
		2858	return err;
		2859
		2860	ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext,
		2861	ibucontext);
		2862	spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
		2863	list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list);
		2864	iwpbl->on_list = true;
		2865	spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
		2866
		2867	return 0;
		2868	}
		2869
		2870	static int irdma_reg_user_mr_type_cq(struct irdma_mem_reg_req req,
		2871	struct ib_udata *udata,
		2872	struct irdma_mr *iwmr)
		2873	{
		2874	struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);
		2875	struct irdma_pbl *iwpbl = &iwmr->iwpbl;
		2876	struct irdma_ucontext *ucontext = NULL;
		2877	u8 shadow_pgcnt = 1;
		2878	unsigned long flags;
		2879	bool use_pbles;
		2880	u32 total;
		2881	int err;
		2882
		2883	if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE)
		2884	shadow_pgcnt = 0;
		2885	total = req.cq_pages + shadow_pgcnt;
		2886	if (total > iwmr->page_cnt)
		2887	return -EINVAL;
		2888
		2889	use_pbles = req.cq_pages > 1;
		2890	err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
		2891	if (err)
		2892	return err;
		2893
		2894	ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext,
		2895	ibucontext);
		2896	spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
		2897	list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list);
		2898	iwpbl->on_list = true;
		2899	spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
		2900
		2901	return 0;
		2902	}
		2903
2748	/**	2904	/**
2749	* irdma_reg_user_mr - Register a user memory region	2905	* irdma_reg_user_mr - Register a user memory region
2750	* @pd: ptr of pd	2906	* @pd: ptr of pd
@@ -2760,18 +2916,10 @@ static struct ib_mr irdma_reg_user_mr(struct ib_pd pd, u64 start, u64 len, · 9 unmodified lines
2760	{	2916	{
2761	#define IRDMA_MEM_REG_MIN_REQ_LEN offsetofend(struct irdma_mem_reg_req, sq_pages)	2917	#define IRDMA_MEM_REG_MIN_REQ_LEN offsetofend(struct irdma_mem_reg_req, sq_pages)
2762	struct irdma_device *iwdev = to_iwdev(pd->device);	2918	struct irdma_device *iwdev = to_iwdev(pd->device);
2763	struct irdma_ucontext *ucontext;	2919	struct irdma_mem_reg_req req = {};
2764	struct irdma_pble_alloc *palloc;	2920	struct ib_umem *region = NULL;
2765	struct irdma_pbl *iwpbl;	2921	struct irdma_mr *iwmr = NULL;
2766	struct irdma_mr *iwmr;	2922	int err;
2767	struct ib_umem *region;
2768	struct irdma_mem_reg_req req;
2769	u32 total, stag = 0;
2770	u8 shadow_pgcnt = 1;
2771	bool use_pbles = false;
2772	unsigned long flags;
2773	int err = -EINVAL;
2774	int ret;
2775		2923
2776	if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)	2924	if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)
2777	return ERR_PTR(-EINVAL);	2925	return ERR_PTR(-EINVAL);
@@ -2792,122 +2940,80 @@ static struct ib_mr irdma_reg_user_mr(struct ib_pd pd, u64 start, u64 len, · 14 unmodified lines
2792	return ERR_PTR(-EFAULT);	2940	return ERR_PTR(-EFAULT);
2793	}	2941	}
2794		2942
2795	iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);	2943	iwmr = irdma_alloc_iwmr(region, pd, virt, req.reg_type);
2796	if (!iwmr) {	2944	if (IS_ERR(iwmr)) {
2797	ib_umem_release(region);	2945	ib_umem_release(region);
2798	return ERR_PTR(-ENOMEM);	2946	return (struct ib_mr *)iwmr;
2799	}
2800
2801	iwpbl = &iwmr->iwpbl;
2802	iwpbl->iwmr = iwmr;
2803	iwmr->region = region;
2804	iwmr->ibmr.pd = pd;
2805	iwmr->ibmr.device = pd->device;
2806	iwmr->ibmr.iova = virt;
2807	iwmr->page_size = PAGE_SIZE;
2808
2809	if (req.reg_type == IRDMA_MEMREG_TYPE_MEM) {
2810	iwmr->page_size = ib_umem_find_best_pgsz(region,
2811	iwdev->rf->sc_dev.hw_attrs.page_size_cap,
2812	virt);
2813	if (unlikely(!iwmr->page_size)) {
2814	kfree(iwmr);
2815	ib_umem_release(region);
2816	return ERR_PTR(-EOPNOTSUPP);
2817	}
2818	}	2947	}
2819	iwmr->len = region->length;
2820	iwpbl->user_base = virt;
2821	palloc = &iwpbl->pble_alloc;
2822	iwmr->type = req.reg_type;
2823	iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size);
2824		2948
2825	switch (req.reg_type) {	2949	switch (req.reg_type) {
2826	case IRDMA_MEMREG_TYPE_QP:	2950	case IRDMA_MEMREG_TYPE_QP:
2827	total = req.sq_pages + req.rq_pages + shadow_pgcnt;	2951	err = irdma_reg_user_mr_type_qp(req, udata, iwmr);
2828	if (total > iwmr->page_cnt) {
2829	err = -EINVAL;
2830	goto error;
2831	}
2832	total = req.sq_pages + req.rq_pages;
2833	use_pbles = (total > 2);
2834	err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
2835	if (err)	2952	if (err)
2836	goto error;	2953	goto error;
2837		2954
2838	ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext,
2839	ibucontext);
2840	spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
2841	list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list);
2842	iwpbl->on_list = true;
2843	spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
2844	break;	2955	break;
2845	case IRDMA_MEMREG_TYPE_CQ:	2956	case IRDMA_MEMREG_TYPE_CQ:
2846	if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE)	2957	err = irdma_reg_user_mr_type_cq(req, udata, iwmr);
2847	shadow_pgcnt = 0;
2848	total = req.cq_pages + shadow_pgcnt;
2849	if (total > iwmr->page_cnt) {
2850	err = -EINVAL;
2851	goto error;
2852	}
2853
2854	use_pbles = (req.cq_pages > 1);
2855	err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
2856	if (err)	2958	if (err)
2857	goto error;	2959	goto error;
2858
2859	ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext,
2860	ibucontext);
2861	spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
2862	list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list);
2863	iwpbl->on_list = true;
2864	spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
2865	break;	2960	break;
2866	case IRDMA_MEMREG_TYPE_MEM:	2961	case IRDMA_MEMREG_TYPE_MEM:
2867	use_pbles = (iwmr->page_cnt != 1);	2962	err = irdma_reg_user_mr_type_mem(iwmr, access);
2868
2869	err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles, false);
2870	if (err)	2963	if (err)
2871	goto error;	2964	goto error;
2872		2965
2873	if (use_pbles) {
2874	ret = irdma_check_mr_contiguous(palloc,
2875	iwmr->page_size);
2876	if (ret) {
2877	irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
2878	iwpbl->pbl_allocated = false;
2879	}
2880	}
2881
2882	stag = irdma_create_stag(iwdev);
2883	if (!stag) {
2884	err = -ENOMEM;
2885	goto error;
2886	}
2887
2888	iwmr->stag = stag;
2889	iwmr->ibmr.rkey = stag;
2890	iwmr->ibmr.lkey = stag;
2891	err = irdma_hwreg_mr(iwdev, iwmr, access);
2892	if (err) {
2893	irdma_free_stag(iwdev, stag);
2894	goto error;
2895	}
2896
2897	break;	2966	break;
2898	default:	2967	default:
		2968	err = -EINVAL;
2899	goto error;	2969	goto error;
2900	}	2970	}
2901		2971
2902	iwmr->type = req.reg_type;
2903
2904	return &iwmr->ibmr;	2972	return &iwmr->ibmr;
2905
2906	error:	2973	error:
2907	if (palloc->level != PBLE_LEVEL_0 && iwpbl->pbl_allocated)
2908	irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
2909	ib_umem_release(region);	2974	ib_umem_release(region);
2910	kfree(iwmr);	2975	irdma_free_iwmr(iwmr);
		2976
		2977	return ERR_PTR(err);
		2978	}
		2979
		2980	static struct ib_mr irdma_reg_user_mr_dmabuf(struct ib_pd pd, u64 start,
		2981	u64 len, u64 virt,
		2982	int fd, int access,
		2983	struct ib_udata *udata)
		2984	{
		2985	struct irdma_device *iwdev = to_iwdev(pd->device);
		2986	struct ib_umem_dmabuf *umem_dmabuf;
		2987	struct irdma_mr *iwmr;
		2988	int err;
		2989
		2990	if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)
		2991	return ERR_PTR(-EINVAL);
		2992
		2993	umem_dmabuf = ib_umem_dmabuf_get_pinned(pd->device, start, len, fd, access);
		2994	if (IS_ERR(umem_dmabuf)) {
		2995	err = PTR_ERR(umem_dmabuf);
		2996	ibdev_dbg(&iwdev->ibdev, "Failed to get dmabuf umem[%d]\n", err);
		2997	return ERR_PTR(err);
		2998	}
		2999
		3000	iwmr = irdma_alloc_iwmr(&umem_dmabuf->umem, pd, virt, IRDMA_MEMREG_TYPE_MEM);
		3001	if (IS_ERR(iwmr)) {
		3002	err = PTR_ERR(iwmr);
		3003	goto err_release;
		3004	}
		3005
		3006	err = irdma_reg_user_mr_type_mem(iwmr, access);
		3007	if (err)
		3008	goto err_iwmr;
		3009
		3010	return &iwmr->ibmr;
		3011
		3012	err_iwmr:
		3013	irdma_free_iwmr(iwmr);
		3014
		3015	err_release:
		3016	ib_umem_release(&umem_dmabuf->umem);
2911		3017
2912	return ERR_PTR(err);	3018	return ERR_PTR(err);
2913	}	3019	}
@@ -4418,6 +4524,7 @@ static const struct ib_device_ops irdma_dev_ops = { · 1504 unmodified lines
4418	.query_port = irdma_query_port,	4524	.query_port = irdma_query_port,
4419	.query_qp = irdma_query_qp,	4525	.query_qp = irdma_query_qp,
4420	.reg_user_mr = irdma_reg_user_mr,	4526	.reg_user_mr = irdma_reg_user_mr,
		4527	.reg_user_mr_dmabuf = irdma_reg_user_mr_dmabuf,
4421	.req_notify_cq = irdma_req_notify_cq,	4528	.req_notify_cq = irdma_req_notify_cq,
4422	.resize_cq = irdma_resize_cq,	4529	.resize_cq = irdma_resize_cq,
4423	INIT_RDMA_OBJ_SIZE(ib_pd, irdma_pd, ibpd),	4530	INIT_RDMA_OBJ_SIZE(ib_pd, irdma_pd, ibpd),

drivers/infiniband/hw/mana/main.c+12−10

 mana_ib_gd_first_dma_region(struct mana_ib_dev *dev,
 			    struct gdma_context *gc,
 			    struct gdma_create_dma_region_req *create_req,
-			    size_t num_pages, mana_handle_t *gdma_region)
+			    size_t num_pages, mana_handle_t *gdma_region,
+			    u32 expected_status)
 {
 	struct gdma_create_dma_region_resp create_resp = {};
 	unsigned int create_req_msg_size;
 	err = mana_gd_send_request(gc, create_req_msg_size, create_req,
 				   sizeof(create_resp), &create_resp);
-	if (err || create_resp.hdr.status) {
+	if (err || create_resp.hdr.status != expected_status) {
 		ibdev_dbg(&dev->ib_dev,
 			  "Failed to create DMA region: %d, 0x%x\n",
 			  err, create_resp.hdr.status);
 	page_addr_list = create_req->page_addr_list;
 	rdma_umem_for_each_dma_block(umem, &biter, page_sz) {
+		u32 expected_status = 0;
 		page_addr_list[tail++] = rdma_block_iter_dma_address(&biter);
 		if (tail < num_pages_to_handle)
 			continue;
+		if (num_pages_processed + num_pages_to_handle <
+		    num_pages_total)
+			expected_status = GDMA_STATUS_MORE_ENTRIES;
 		if (!num_pages_processed) {
 			/* First create message */
 			err = mana_ib_gd_first_dma_region(dev, gc, create_req,
-							  tail, gdma_region);
+							  tail, gdma_region,
+							  expected_status);
 			if (err)
 				goto out;
 			page_addr_list = add_req->page_addr_list;
 		} else {
 			/* Subsequent create messages */
-			u32 expected_s = 0;
-			if (num_pages_processed + num_pages_to_handle <
-			    num_pages_total)
-				expected_s = GDMA_STATUS_MORE_ENTRIES;
 			err = mana_ib_gd_add_dma_region(dev, gc, add_req, tail,
-							expected_s);
+							expected_status);
 			if (err)
 				break;
 		}

drivers/infiniband/hw/mlx4/main.c+8−0

 	if (!wq)
 		return -ENOMEM;
+	err = mlx4_ib_qp_event_init();
+	if (err)
+		goto clean_qp_event;
 	err = mlx4_ib_cm_init();
 	if (err)
 		goto clean_wq;
 	mlx4_ib_cm_destroy();
 clean_wq:
+	mlx4_ib_qp_event_cleanup();
+clean_qp_event:
 	destroy_workqueue(wq);
 	return err;
 }
 	mlx4_unregister_interface(&mlx4_ib_interface);
 	mlx4_ib_mcg_destroy();
 	mlx4_ib_cm_destroy();
+	mlx4_ib_qp_event_cleanup();
 	destroy_workqueue(wq);
 }

drivers/infiniband/hw/mlx4/mlx4_ib.h+3−0

@@ -940,4 +940,7 @@ int mlx4_ib_umem_calc_optimal_mtt_size(struct ib_umem *umem, u64 start_va, · 939 unmodified lines
940	int mlx4_ib_cm_init(void);	940	int mlx4_ib_cm_init(void);
941	void mlx4_ib_cm_destroy(void);	941	void mlx4_ib_cm_destroy(void);
942		942
		943	int mlx4_ib_qp_event_init(void);
		944	void mlx4_ib_qp_event_cleanup(void);
		945
943	#endif /* MLX4_IB_H */	946	#endif /* MLX4_IB_H */

drivers/infiniband/hw/mlx4/qp.c+85−36

@@ -102,6 +102,14 @@ enum mlx4_ib_source_type { · 101 unmodified lines
102	MLX4_IB_RWQ_SRC = 1,	102	MLX4_IB_RWQ_SRC = 1,
103	};	103	};
104		104
		105	struct mlx4_ib_qp_event_work {
		106	struct work_struct work;
		107	struct mlx4_qp *qp;
		108	enum mlx4_event type;
		109	};
		110
		111	static struct workqueue_struct *mlx4_ib_qp_event_wq;
		112
105	static int is_tunnel_qp(struct mlx4_ib_dev dev, struct mlx4_ib_qp qp)	113	static int is_tunnel_qp(struct mlx4_ib_dev dev, struct mlx4_ib_qp qp)
106	{	114	{
107	if (!mlx4_is_master(dev->dev))	115	if (!mlx4_is_master(dev->dev))
@@ -200,50 +208,77 @@ static void stamp_send_wqe(struct mlx4_ib_qp *qp, int n) · 92 unmodified lines
200	}	208	}
201	}	209	}
202		210
		211	static void mlx4_ib_handle_qp_event(struct work_struct *_work)
		212	{
		213	struct mlx4_ib_qp_event_work *qpe_work =
		214	container_of(_work, struct mlx4_ib_qp_event_work, work);
		215	struct ib_qp *ibqp = &to_mibqp(qpe_work->qp)->ibqp;
		216	struct ib_event event = {};
		217
		218	event.device = ibqp->device;
		219	event.element.qp = ibqp;
		220
		221	switch (qpe_work->type) {
		222	case MLX4_EVENT_TYPE_PATH_MIG:
		223	event.event = IB_EVENT_PATH_MIG;
		224	break;
		225	case MLX4_EVENT_TYPE_COMM_EST:
		226	event.event = IB_EVENT_COMM_EST;
		227	break;
		228	case MLX4_EVENT_TYPE_SQ_DRAINED:
		229	event.event = IB_EVENT_SQ_DRAINED;
		230	break;
		231	case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE:
		232	event.event = IB_EVENT_QP_LAST_WQE_REACHED;
		233	break;
		234	case MLX4_EVENT_TYPE_WQ_CATAS_ERROR:
		235	event.event = IB_EVENT_QP_FATAL;
		236	break;
		237	case MLX4_EVENT_TYPE_PATH_MIG_FAILED:
		238	event.event = IB_EVENT_PATH_MIG_ERR;
		239	break;
		240	case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
		241	event.event = IB_EVENT_QP_REQ_ERR;
		242	break;
		243	case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR:
		244	event.event = IB_EVENT_QP_ACCESS_ERR;
		245	break;
		246	default:
		247	pr_warn("Unexpected event type %d on QP %06x\n",
		248	qpe_work->type, qpe_work->qp->qpn);
		249	goto out;
		250	}
		251
		252	ibqp->event_handler(&event, ibqp->qp_context);
		253
		254	out:
		255	mlx4_put_qp(qpe_work->qp);
		256	kfree(qpe_work);
		257	}
		258
203	static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type)	259	static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type)
204	{	260	{
205	struct ib_event event;
206	struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;	261	struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;
		262	struct mlx4_ib_qp_event_work *qpe_work;
207		263
208	if (type == MLX4_EVENT_TYPE_PATH_MIG)	264	if (type == MLX4_EVENT_TYPE_PATH_MIG)
209	to_mibqp(qp)->port = to_mibqp(qp)->alt_port;	265	to_mibqp(qp)->port = to_mibqp(qp)->alt_port;
210		266
211	if (ibqp->event_handler) {	267	if (!ibqp->event_handler)
212	event.device = ibqp->device;	268	goto out_no_handler;
213	event.element.qp = ibqp;
214	switch (type) {
215	case MLX4_EVENT_TYPE_PATH_MIG:
216	event.event = IB_EVENT_PATH_MIG;
217	break;
218	case MLX4_EVENT_TYPE_COMM_EST:
219	event.event = IB_EVENT_COMM_EST;
220	break;
221	case MLX4_EVENT_TYPE_SQ_DRAINED:
222	event.event = IB_EVENT_SQ_DRAINED;
223	break;
224	case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE:
225	event.event = IB_EVENT_QP_LAST_WQE_REACHED;
226	break;
227	case MLX4_EVENT_TYPE_WQ_CATAS_ERROR:
228	event.event = IB_EVENT_QP_FATAL;
229	break;
230	case MLX4_EVENT_TYPE_PATH_MIG_FAILED:
231	event.event = IB_EVENT_PATH_MIG_ERR;
232	break;
233	case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
234	event.event = IB_EVENT_QP_REQ_ERR;
235	break;
236	case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR:
237	event.event = IB_EVENT_QP_ACCESS_ERR;
238	break;
239	default:
240	pr_warn("Unexpected event type %d "
241	"on QP %06x\n", type, qp->qpn);
242	return;
243	}
244		269
245	ibqp->event_handler(&event, ibqp->qp_context);	270	qpe_work = kzalloc(sizeof(*qpe_work), GFP_ATOMIC);
246	}	271	if (!qpe_work)
		272	goto out_no_handler;
		273
		274	qpe_work->qp = qp;
		275	qpe_work->type = type;
		276	INIT_WORK(&qpe_work->work, mlx4_ib_handle_qp_event);
		277	queue_work(mlx4_ib_qp_event_wq, &qpe_work->work);
		278	return;
		279
		280	out_no_handler:
		281	mlx4_put_qp(qp);
247	}	282	}
248		283
249	static void mlx4_ib_wq_event(struct mlx4_qp *qp, enum mlx4_event type)	284	static void mlx4_ib_wq_event(struct mlx4_qp *qp, enum mlx4_event type)
@@ -4468,3 +4503,17 @@ void mlx4_ib_drain_rq(struct ib_qp *qp) · 4218 unmodified lines
4468		4503
4469	handle_drain_completion(cq, &rdrain, dev);	4504	handle_drain_completion(cq, &rdrain, dev);
4470	}	4505	}
		4506
		4507	int mlx4_ib_qp_event_init(void)
		4508	{
		4509	mlx4_ib_qp_event_wq = alloc_ordered_workqueue("mlx4_ib_qp_event_wq", 0);
		4510	if (!mlx4_ib_qp_event_wq)
		4511	return -ENOMEM;
		4512
		4513	return 0;
		4514	}
		4515
		4516	void mlx4_ib_qp_event_cleanup(void)
		4517	{
		4518	destroy_workqueue(mlx4_ib_qp_event_wq);
		4519	}

drivers/infiniband/hw/mlx5/cmd.c+26−19

@@ -5,34 +5,41 @@ · 4 unmodified lines
5		5
6	#include "cmd.h"	6	#include "cmd.h"
7		7
8	int mlx5_cmd_dump_fill_mkey(struct mlx5_core_dev dev, u32 mkey)	8	int mlx5r_cmd_query_special_mkeys(struct mlx5_ib_dev *dev)
9	{	9	{
10	u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {};	10	u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {};
11	u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)] = {};	11	u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)] = {};
		12	bool is_terminate, is_dump, is_null;
12	int err;	13	int err;
13		14
14	MLX5_SET(query_special_contexts_in, in, opcode,	15	is_terminate = MLX5_CAP_GEN(dev->mdev, terminate_scatter_list_mkey);
15	MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS);	16	is_dump = MLX5_CAP_GEN(dev->mdev, dump_fill_mkey);
16	err = mlx5_cmd_exec_inout(dev, query_special_contexts, in, out);	17	is_null = MLX5_CAP_GEN(dev->mdev, null_mkey);
17	if (!err)
18	*mkey = MLX5_GET(query_special_contexts_out, out,
19	dump_fill_mkey);
20	return err;
21	}
22		18
23	int mlx5_cmd_null_mkey(struct mlx5_core_dev dev, u32 null_mkey)	19	dev->mkeys.terminate_scatter_list_mkey = MLX5_TERMINATE_SCATTER_LIST_LKEY;
24	{	20	if (!is_terminate && !is_dump && !is_null)
25	u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {};	21	return 0;
26	u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)] = {};
27	int err;
28		22
29	MLX5_SET(query_special_contexts_in, in, opcode,	23	MLX5_SET(query_special_contexts_in, in, opcode,
30	MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS);	24	MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS);
31	err = mlx5_cmd_exec_inout(dev, query_special_contexts, in, out);	25	err = mlx5_cmd_exec_inout(dev->mdev, query_special_contexts, in, out);
32	if (!err)	26	if (err)
33	*null_mkey = MLX5_GET(query_special_contexts_out, out,	27	return err;
34	null_mkey);	28
35	return err;	29	if (is_dump)
		30	dev->mkeys.dump_fill_mkey = MLX5_GET(query_special_contexts_out,
		31	out, dump_fill_mkey);
		32
		33	if (is_null)
		34	dev->mkeys.null_mkey = cpu_to_be32(
		35	MLX5_GET(query_special_contexts_out, out, null_mkey));
		36
		37	if (is_terminate)
		38	dev->mkeys.terminate_scatter_list_mkey =
		39	cpu_to_be32(MLX5_GET(query_special_contexts_out, out,
		40	terminate_scatter_list_mkey));
		41
		42	return 0;
36	}	43	}
37		44
38	int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point,	45	int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point,

drivers/infiniband/hw/mlx5/cmd.h+1−2

@@ -37,8 +37,7 @@ · 36 unmodified lines
37	#include <linux/kernel.h>	37	#include <linux/kernel.h>
38	#include <linux/mlx5/driver.h>	38	#include <linux/mlx5/driver.h>
39		39
40	int mlx5_cmd_dump_fill_mkey(struct mlx5_core_dev dev, u32 mkey);	40	int mlx5r_cmd_query_special_mkeys(struct mlx5_ib_dev *dev);
41	int mlx5_cmd_null_mkey(struct mlx5_core_dev dev, u32 null_mkey);
42	int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point,	41	int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point,
43	void *out);	42	void *out);
44	int mlx5_cmd_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn, u16 uid);	43	int mlx5_cmd_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn, u16 uid);

drivers/infiniband/hw/mlx5/cong.c+25−3

@@ -38,6 +38,7 @@ · 37 unmodified lines
38	enum mlx5_ib_cong_node_type {	38	enum mlx5_ib_cong_node_type {
39	MLX5_IB_RROCE_ECN_RP = 1,	39	MLX5_IB_RROCE_ECN_RP = 1,
40	MLX5_IB_RROCE_ECN_NP = 2,	40	MLX5_IB_RROCE_ECN_NP = 2,
		41	MLX5_IB_RROCE_GENERAL = 3,
41	};	42	};
42		43
43	static const char * const mlx5_ib_dbg_cc_name[] = {	44	static const char * const mlx5_ib_dbg_cc_name[] = {
@@ -61,6 +62,8 @@ static const char * const mlx5_ib_dbg_cc_name[] = { · 17 unmodified lines
61	"np_cnp_dscp",	62	"np_cnp_dscp",
62	"np_cnp_prio_mode",	63	"np_cnp_prio_mode",
63	"np_cnp_prio",	64	"np_cnp_prio",
		65	"rtt_resp_dscp_valid",
		66	"rtt_resp_dscp",
64	};	67	};
65		68
66	#define MLX5_IB_RP_CLAMP_TGT_RATE_ATTR BIT(1)	69	#define MLX5_IB_RP_CLAMP_TGT_RATE_ATTR BIT(1)
@@ -84,14 +87,18 @@ static const char * const mlx5_ib_dbg_cc_name[] = { · 17 unmodified lines
84	#define MLX5_IB_NP_CNP_DSCP_ATTR BIT(3)	87	#define MLX5_IB_NP_CNP_DSCP_ATTR BIT(3)
85	#define MLX5_IB_NP_CNP_PRIO_MODE_ATTR BIT(4)	88	#define MLX5_IB_NP_CNP_PRIO_MODE_ATTR BIT(4)
86		89
		90	#define MLX5_IB_GENERAL_RTT_RESP_DSCP_ATTR BIT(0)
		91
87	static enum mlx5_ib_cong_node_type	92	static enum mlx5_ib_cong_node_type
88	mlx5_ib_param_to_node(enum mlx5_ib_dbg_cc_types param_offset)	93	mlx5_ib_param_to_node(enum mlx5_ib_dbg_cc_types param_offset)
89	{	94	{
90	if (param_offset >= MLX5_IB_DBG_CC_RP_CLAMP_TGT_RATE &&	95	if (param_offset <= MLX5_IB_DBG_CC_RP_GD)
91	param_offset <= MLX5_IB_DBG_CC_RP_GD)
92	return MLX5_IB_RROCE_ECN_RP;	96	return MLX5_IB_RROCE_ECN_RP;
93	else	97
		98	if (param_offset <= MLX5_IB_DBG_CC_NP_CNP_PRIO)
94	return MLX5_IB_RROCE_ECN_NP;	99	return MLX5_IB_RROCE_ECN_NP;
		100
		101	return MLX5_IB_RROCE_GENERAL;
95	}	102	}
96		103
97	static u32 mlx5_get_cc_param_val(void *field, int offset)	104	static u32 mlx5_get_cc_param_val(void *field, int offset)
@@ -157,6 +164,12 @@ static u32 mlx5_get_cc_param_val(void *field, int offset) · 59 unmodified lines
157	case MLX5_IB_DBG_CC_NP_CNP_PRIO:	164	case MLX5_IB_DBG_CC_NP_CNP_PRIO:
158	return MLX5_GET(cong_control_r_roce_ecn_np, field,	165	return MLX5_GET(cong_control_r_roce_ecn_np, field,
159	cnp_802p_prio);	166	cnp_802p_prio);
		167	case MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP_VALID:
		168	return MLX5_GET(cong_control_r_roce_general, field,
		169	rtt_resp_dscp_valid);
		170	case MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP:
		171	return MLX5_GET(cong_control_r_roce_general, field,
		172	rtt_resp_dscp);
160	default:	173	default:
161	return 0;	174	return 0;
162	}	175	}
@@ -264,6 +277,15 @@ static void mlx5_ib_set_cc_param_mask_val(void *field, int offset, · 101 unmodified lines
264	MLX5_SET(cong_control_r_roce_ecn_np, field, cnp_prio_mode, 0);	277	MLX5_SET(cong_control_r_roce_ecn_np, field, cnp_prio_mode, 0);
265	MLX5_SET(cong_control_r_roce_ecn_np, field, cnp_802p_prio, var);	278	MLX5_SET(cong_control_r_roce_ecn_np, field, cnp_802p_prio, var);
266	break;	279	break;
		280	case MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP_VALID:
		281	*attr_mask \|= MLX5_IB_GENERAL_RTT_RESP_DSCP_ATTR;
		282	MLX5_SET(cong_control_r_roce_general, field, rtt_resp_dscp_valid, var);
		283	break;
		284	case MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP:
		285	*attr_mask \|= MLX5_IB_GENERAL_RTT_RESP_DSCP_ATTR;
		286	MLX5_SET(cong_control_r_roce_general, field, rtt_resp_dscp_valid, 1);
		287	MLX5_SET(cong_control_r_roce_general, field, rtt_resp_dscp, var);
		288	break;
267	}	289	}
268	}	290	}
269		291

drivers/infiniband/hw/mlx5/main.c+13−11

 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
 	struct mlx5_ib_ucontext *context = to_mucontext(uctx);
 	struct mlx5_bfreg_info *bfregi = &context->bfregi;
-	int err;
 	if (MLX5_CAP_GEN(dev->mdev, dump_fill_mkey)) {
-		err = mlx5_cmd_dump_fill_mkey(dev->mdev,
+		resp->dump_fill_mkey = dev->mkeys.dump_fill_mkey;
-					      &resp->dump_fill_mkey);
-		if (err)
-			return err;
 		resp->comp_mask |=
 			MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_DUMP_FILL_MKEY;
 	}
 		dev->port[i].roce.last_port_state = IB_PORT_DOWN;
 	}
+	err = mlx5r_cmd_query_special_mkeys(dev);
+	if (err)
+		return err;
 	err = mlx5_ib_init_multiport_master(dev);
 	if (err)
 		return err;
 static void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev)
 {
-	int err;
+	mlx5_mkey_cache_cleanup(dev);
-	err = mlx5_mkey_cache_cleanup(dev);
-	if (err)
-		mlx5_ib_warn(dev, "mr cache cleanup failed\n");
 	mlx5r_umr_resource_cleanup(dev);
 }
 		return -ENOMEM;
 	}
+	ret = mlx5_ib_qp_event_init();
+	if (ret)
+		goto qp_event_err;
 	mlx5_ib_odp_init();
 	ret = mlx5r_rep_init();
 	if (ret)
 mp_err:
 	mlx5r_rep_cleanup();
 rep_err:
+	mlx5_ib_qp_event_cleanup();
+qp_event_err:
 	destroy_workqueue(mlx5_ib_event_wq);
 	free_page((unsigned long)xlt_emergency_page);
 	return ret;
 	auxiliary_driver_unregister(&mlx5r_mp_driver);
 	mlx5r_rep_cleanup();
+	mlx5_ib_qp_event_cleanup();
 	destroy_workqueue(mlx5_ib_event_wq);
 	free_page((unsigned long)xlt_emergency_page);
 }

drivers/infiniband/hw/mlx5/mlx5_ib.h+39−12

 	MLX5_MKEY_INDIRECT_DEVX,
 };
+struct mlx5r_cache_rb_key {
+	u8 ats:1;
+	unsigned int access_mode;
+	unsigned int access_flags;
+	unsigned int ndescs;
+};
 struct mlx5_ib_mkey {
 	u32 key;
 	enum mlx5_mkey_type type;
 	unsigned int ndescs;
 	struct wait_queue_head wait;
 	refcount_t usecount;
+	/* User Mkey must hold either a rb_key or a cache_ent. */
+	struct mlx5r_cache_rb_key rb_key;
 	struct mlx5_cache_ent *cache_ent;
 };
 	unsigned long		reserved;
 	char                    name[4];
-	u32                     order;
-	u32			access_mode;
-	u32			page;
-	unsigned int		ndescs;
+	struct rb_node		node;
+	struct mlx5r_cache_rb_key rb_key;
+	u8 is_tmp:1;
 	u8 disabled:1;
 	u8 fill_to_high_water:1;
 struct mlx5_mkey_cache {
 	struct workqueue_struct *wq;
-	struct mlx5_cache_ent	ent[MAX_MKEY_CACHE_ENTRIES];
+	struct rb_root		rb_root;
-	struct dentry		*root;
+	struct mutex		rb_lock;
+	struct dentry		*fs_root;
 	unsigned long		last_add;
+	struct delayed_work	remove_ent_dwork;
 };
 struct mlx5_ib_port_resources {
 	MLX5_IB_DBG_CC_NP_CNP_DSCP,
 	MLX5_IB_DBG_CC_NP_CNP_PRIO_MODE,
 	MLX5_IB_DBG_CC_NP_CNP_PRIO,
+	MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP_VALID,
+	MLX5_IB_DBG_CC_GENERAL_RTT_RESP_DSCP,
 	MLX5_IB_DBG_CC_MAX,
 };
 	u8 ext_port_cap;
 };
+struct mlx5_special_mkeys {
+	u32 dump_fill_mkey;
+	__be32 null_mkey;
+	__be32 terminate_scatter_list_mkey;
+};
 struct mlx5_ib_dev {
 	struct ib_device		ib_dev;
 	struct mlx5_core_dev		*mdev;
 	struct xarray		odp_mkeys;
-	u32			null_mkey;
 	struct mlx5_ib_flow_db	*flow_db;
 	/* protect resources needed as part of reset flow */
 	spinlock_t		reset_flow_resource_lock;
 	struct mlx5_port_caps port_caps[MLX5_MAX_PORTS];
 	u16 pkey_table_len;
 	u8 lag_ports;
+	struct mlx5_special_mkeys mkeys;
 };
 static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)
 void mlx5_ib_copy_pas(u64 *old, u64 *new, int step, int num);
 int mlx5_ib_get_cqe_size(struct ib_cq *ibcq);
 int mlx5_mkey_cache_init(struct mlx5_ib_dev *dev);
-int mlx5_mkey_cache_cleanup(struct mlx5_ib_dev *dev);
+void mlx5_mkey_cache_cleanup(struct mlx5_ib_dev *dev);
+struct mlx5_cache_ent *
+mlx5r_cache_create_ent_locked(struct mlx5_ib_dev *dev,
+			      struct mlx5r_cache_rb_key rb_key,
+			      bool persistent_entry);
 struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,
-				       struct mlx5_cache_ent *ent,
+				       int access_flags, int access_mode,
-				       int access_flags);
+				       int ndescs);
 int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
 			    struct ib_mr_status *mr_status);
 void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *ibdev);
 int __init mlx5_ib_odp_init(void);
 void mlx5_ib_odp_cleanup(void);
-void mlx5_odp_init_mkey_cache_entry(struct mlx5_cache_ent *ent);
+int mlx5_odp_init_mkey_cache(struct mlx5_ib_dev *dev);
 void mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries,
 			   struct mlx5_ib_mr *mr, int flags);
 static inline void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *ibdev) {}
 static inline int mlx5_ib_odp_init(void) { return 0; }
 static inline void mlx5_ib_odp_cleanup(void)				    {}
-static inline void mlx5_odp_init_mkey_cache_entry(struct mlx5_cache_ent *ent) {}
+static inline int mlx5_odp_init_mkey_cache(struct mlx5_ib_dev *dev)
+{
+	return 0;
+}
 static inline void mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries,
 					 struct mlx5_ib_mr *mr, int flags) {}

drivers/infiniband/hw/mlx5/mr.c+379−111

@@ -140,19 +140,16 @@ static void create_mkey_warn(struct mlx5_ib_dev dev, int status, void out) · 139 unmodified lines
140	mlx5_cmd_out_err(dev->mdev, MLX5_CMD_OP_CREATE_MKEY, 0, out);	140	mlx5_cmd_out_err(dev->mdev, MLX5_CMD_OP_CREATE_MKEY, 0, out);
141	}	141	}
142		142
143		143	static int push_mkey_locked(struct mlx5_cache_ent *ent, bool limit_pendings,
144	static int push_mkey(struct mlx5_cache_ent *ent, bool limit_pendings,	144	void *to_store)
145	void *to_store)
146	{	145	{
147	XA_STATE(xas, &ent->mkeys, 0);	146	XA_STATE(xas, &ent->mkeys, 0);
148	void *curr;	147	void *curr;
149		148
150	xa_lock_irq(&ent->mkeys);
151	if (limit_pendings &&	149	if (limit_pendings &&
152	(ent->reserved - ent->stored) > MAX_PENDING_REG_MR) {	150	(ent->reserved - ent->stored) > MAX_PENDING_REG_MR)
153	xa_unlock_irq(&ent->mkeys);
154	return -EAGAIN;	151	return -EAGAIN;
155	}	152
156	while (1) {	153	while (1) {
157	/*	154	/*
158	* This is cmpxchg (NULL, XA_ZERO_ENTRY) however this version	155	* This is cmpxchg (NULL, XA_ZERO_ENTRY) however this version
@@ -191,6 +188,7 @@ static int push_mkey(struct mlx5_cache_ent *ent, bool limit_pendings, · 32 unmodified lines
191	break;	188	break;
192	xa_lock_irq(&ent->mkeys);	189	xa_lock_irq(&ent->mkeys);
193	}	190	}
		191	xa_lock_irq(&ent->mkeys);
194	if (xas_error(&xas))	192	if (xas_error(&xas))
195	return xas_error(&xas);	193	return xas_error(&xas);
196	if (WARN_ON(curr))	194	if (WARN_ON(curr))
@@ -198,6 +196,17 @@ static int push_mkey(struct mlx5_cache_ent *ent, bool limit_pendings, · 1 unmodified line
198	return 0;	196	return 0;
199	}	197	}
200		198
		199	static int push_mkey(struct mlx5_cache_ent *ent, bool limit_pendings,
		200	void *to_store)
		201	{
		202	int ret;
		203
		204	xa_lock_irq(&ent->mkeys);
		205	ret = push_mkey_locked(ent, limit_pendings, to_store);
		206	xa_unlock_irq(&ent->mkeys);
		207	return ret;
		208	}
		209
201	static void undo_push_reserve_mkey(struct mlx5_cache_ent *ent)	210	static void undo_push_reserve_mkey(struct mlx5_cache_ent *ent)
202	{	211	{
203	void *old;	212	void *old;
@@ -292,12 +301,14 @@ static void set_cache_mkc(struct mlx5_cache_ent ent, void mkc) · 88 unmodified lines
292	set_mkc_access_pd_addr_fields(mkc, 0, 0, ent->dev->umrc.pd);	301	set_mkc_access_pd_addr_fields(mkc, 0, 0, ent->dev->umrc.pd);
293	MLX5_SET(mkc, mkc, free, 1);	302	MLX5_SET(mkc, mkc, free, 1);
294	MLX5_SET(mkc, mkc, umr_en, 1);	303	MLX5_SET(mkc, mkc, umr_en, 1);
295	MLX5_SET(mkc, mkc, access_mode_1_0, ent->access_mode & 0x3);	304	MLX5_SET(mkc, mkc, access_mode_1_0, ent->rb_key.access_mode & 0x3);
296	MLX5_SET(mkc, mkc, access_mode_4_2, (ent->access_mode >> 2) & 0x7);	305	MLX5_SET(mkc, mkc, access_mode_4_2,
		306	(ent->rb_key.access_mode >> 2) & 0x7);
297		307
298	MLX5_SET(mkc, mkc, translations_octword_size,	308	MLX5_SET(mkc, mkc, translations_octword_size,
299	get_mkc_octo_size(ent->access_mode, ent->ndescs));	309	get_mkc_octo_size(ent->rb_key.access_mode,
300	MLX5_SET(mkc, mkc, log_page_size, ent->page);	310	ent->rb_key.ndescs));
		311	MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
301	}	312	}
302		313
303	/* Asynchronously schedule new MRs to be populated in the cache. */	314	/* Asynchronously schedule new MRs to be populated in the cache. */
@@ -515,18 +526,22 @@ static const struct file_operations limit_fops = { · 211 unmodified lines
515		526
516	static bool someone_adding(struct mlx5_mkey_cache *cache)	527	static bool someone_adding(struct mlx5_mkey_cache *cache)
517	{	528	{
518	unsigned int i;	529	struct mlx5_cache_ent *ent;
519		530	struct rb_node *node;
520	for (i = 0; i < MAX_MKEY_CACHE_ENTRIES; i++) {	531	bool ret;
521	struct mlx5_cache_ent *ent = &cache->ent[i];
522	bool ret;
523		532
		533	mutex_lock(&cache->rb_lock);
		534	for (node = rb_first(&cache->rb_root); node; node = rb_next(node)) {
		535	ent = rb_entry(node, struct mlx5_cache_ent, node);
524	xa_lock_irq(&ent->mkeys);	536	xa_lock_irq(&ent->mkeys);
525	ret = ent->stored < ent->limit;	537	ret = ent->stored < ent->limit;
526	xa_unlock_irq(&ent->mkeys);	538	xa_unlock_irq(&ent->mkeys);
527	if (ret)	539	if (ret) {
		540	mutex_unlock(&cache->rb_lock);
528	return true;	541	return true;
		542	}
529	}	543	}
		544	mutex_unlock(&cache->rb_lock);
530	return false;	545	return false;
531	}	546	}
532		547
@@ -539,7 +554,7 @@ static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent) · 6 unmodified lines
539	{	554	{
540	lockdep_assert_held(&ent->mkeys.xa_lock);	555	lockdep_assert_held(&ent->mkeys.xa_lock);
541		556
542	if (ent->disabled \|\| READ_ONCE(ent->dev->fill_delay))	557	if (ent->disabled \|\| READ_ONCE(ent->dev->fill_delay) \|\| ent->is_tmp)
543	return;	558	return;
544	if (ent->stored < ent->limit) {	559	if (ent->stored < ent->limit) {
545	ent->fill_to_high_water = true;	560	ent->fill_to_high_water = true;
@@ -590,8 +605,8 @@ static void __cache_work_func(struct mlx5_cache_ent *ent) · 44 unmodified lines
590	if (err != -EAGAIN) {	605	if (err != -EAGAIN) {
591	mlx5_ib_warn(	606	mlx5_ib_warn(
592	dev,	607	dev,
593	"command failed order %d, err %d\n",	608	"add keys command failed, err %d\n",
594	ent->order, err);	609	err);
595	queue_delayed_work(cache->wq, &ent->dwork,	610	queue_delayed_work(cache->wq, &ent->dwork,
596	msecs_to_jiffies(1000));	611	msecs_to_jiffies(1000));
597	}	612	}
@@ -637,16 +652,99 @@ static void delayed_cache_work_func(struct work_struct *work) · 39 unmodified lines
637	__cache_work_func(ent);	652	__cache_work_func(ent);
638	}	653	}
639		654
640	struct mlx5_ib_mr mlx5_mr_cache_alloc(struct mlx5_ib_dev dev,	655	static int cache_ent_key_cmp(struct mlx5r_cache_rb_key key1,
641	struct mlx5_cache_ent *ent,	656	struct mlx5r_cache_rb_key key2)
642	int access_flags)	657	{
		658	int res;
		659
		660	res = key1.ats - key2.ats;
		661	if (res)
		662	return res;
		663
		664	res = key1.access_mode - key2.access_mode;
		665	if (res)
		666	return res;
		667
		668	res = key1.access_flags - key2.access_flags;
		669	if (res)
		670	return res;
		671
		672	/*
		673	* keep ndescs the last in the compare table since the find function
		674	* searches for an exact match on all properties and only closest
		675	* match in size.
		676	*/
		677	return key1.ndescs - key2.ndescs;
		678	}
		679
		680	static int mlx5_cache_ent_insert(struct mlx5_mkey_cache *cache,
		681	struct mlx5_cache_ent *ent)
		682	{
		683	struct rb_node *new = &cache->rb_root.rb_node, parent = NULL;
		684	struct mlx5_cache_ent *cur;
		685	int cmp;
		686
		687	/* Figure out where to put new node */
		688	while (*new) {
		689	cur = rb_entry(*new, struct mlx5_cache_ent, node);
		690	parent = *new;
		691	cmp = cache_ent_key_cmp(cur->rb_key, ent->rb_key);
		692	if (cmp > 0)
		693	new = &((*new)->rb_left);
		694	if (cmp < 0)
		695	new = &((*new)->rb_right);
		696	if (cmp == 0) {
		697	mutex_unlock(&cache->rb_lock);
		698	return -EEXIST;
		699	}
		700	}
		701
		702	/* Add new node and rebalance tree. */
		703	rb_link_node(&ent->node, parent, new);
		704	rb_insert_color(&ent->node, &cache->rb_root);
		705
		706	return 0;
		707	}
		708
		709	static struct mlx5_cache_ent *
		710	mkey_cache_ent_from_rb_key(struct mlx5_ib_dev *dev,
		711	struct mlx5r_cache_rb_key rb_key)
		712	{
		713	struct rb_node *node = dev->cache.rb_root.rb_node;
		714	struct mlx5_cache_ent cur, smallest = NULL;
		715	int cmp;
		716
		717	/*
		718	* Find the smallest ent with order >= requested_order.
		719	*/
		720	while (node) {
		721	cur = rb_entry(node, struct mlx5_cache_ent, node);
		722	cmp = cache_ent_key_cmp(cur->rb_key, rb_key);
		723	if (cmp > 0) {
		724	smallest = cur;
		725	node = node->rb_left;
		726	}
		727	if (cmp < 0)
		728	node = node->rb_right;
		729	if (cmp == 0)
		730	return cur;
		731	}
		732
		733	return (smallest &&
		734	smallest->rb_key.access_mode == rb_key.access_mode &&
		735	smallest->rb_key.access_flags == rb_key.access_flags &&
		736	smallest->rb_key.ats == rb_key.ats) ?
		737	smallest :
		738	NULL;
		739	}
		740
		741	static struct mlx5_ib_mr _mlx5_mr_cache_alloc(struct mlx5_ib_dev dev,
		742	struct mlx5_cache_ent *ent,
		743	int access_flags)
643	{	744	{
644	struct mlx5_ib_mr *mr;	745	struct mlx5_ib_mr *mr;
645	int err;	746	int err;
646		747
647	if (!mlx5r_umr_can_reconfig(dev, 0, access_flags))
648	return ERR_PTR(-EOPNOTSUPP);
649
650	mr = kzalloc(sizeof(*mr), GFP_KERNEL);	748	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
651	if (!mr)	749	if (!mr)
652	return ERR_PTR(-ENOMEM);	750	return ERR_PTR(-ENOMEM);
@@ -677,10 +775,48 @@ struct mlx5_ib_mr mlx5_mr_cache_alloc(struct mlx5_ib_dev dev, · 24 unmodified lines
677	return mr;	775	return mr;
678	}	776	}
679		777
680	static void clean_keys(struct mlx5_ib_dev *dev, int c)	778	static int get_unchangeable_access_flags(struct mlx5_ib_dev *dev,
		779	int access_flags)
		780	{
		781	int ret = 0;
		782
		783	if ((access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
		784	MLX5_CAP_GEN(dev->mdev, atomic) &&
		785	MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled))
		786	ret \|= IB_ACCESS_REMOTE_ATOMIC;
		787
		788	if ((access_flags & IB_ACCESS_RELAXED_ORDERING) &&
		789	MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write) &&
		790	!MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr))
		791	ret \|= IB_ACCESS_RELAXED_ORDERING;
		792
		793	if ((access_flags & IB_ACCESS_RELAXED_ORDERING) &&
		794	MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read) &&
		795	!MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr))
		796	ret \|= IB_ACCESS_RELAXED_ORDERING;
		797
		798	return ret;
		799	}
		800
		801	struct mlx5_ib_mr mlx5_mr_cache_alloc(struct mlx5_ib_dev dev,
		802	int access_flags, int access_mode,
		803	int ndescs)
		804	{
		805	struct mlx5r_cache_rb_key rb_key = {
		806	.ndescs = ndescs,
		807	.access_mode = access_mode,
		808	.access_flags = get_unchangeable_access_flags(dev, access_flags)
		809	};
		810	struct mlx5_cache_ent *ent = mkey_cache_ent_from_rb_key(dev, rb_key);
		811
		812	if (!ent)
		813	return ERR_PTR(-EOPNOTSUPP);
		814
		815	return _mlx5_mr_cache_alloc(dev, ent, access_flags);
		816	}
		817
		818	static void clean_keys(struct mlx5_ib_dev dev, struct mlx5_cache_ent ent)
681	{	819	{
682	struct mlx5_mkey_cache *cache = &dev->cache;
683	struct mlx5_cache_ent *ent = &cache->ent[c];
684	u32 mkey;	820	u32 mkey;
685		821
686	cancel_delayed_work(&ent->dwork);	822	cancel_delayed_work(&ent->dwork);
@@ -699,31 +835,39 @@ static void mlx5_mkey_cache_debugfs_cleanup(struct mlx5_ib_dev *dev) · 12 unmodified lines
699	if (!mlx5_debugfs_root \|\| dev->is_rep)	835	if (!mlx5_debugfs_root \|\| dev->is_rep)
700	return;	836	return;
701		837
702	debugfs_remove_recursive(dev->cache.root);	838	debugfs_remove_recursive(dev->cache.fs_root);
703	dev->cache.root = NULL;	839	dev->cache.fs_root = NULL;
704	}	840	}
705		841
706	static void mlx5_mkey_cache_debugfs_init(struct mlx5_ib_dev *dev)	842	static void mlx5_mkey_cache_debugfs_add_ent(struct mlx5_ib_dev *dev,
		843	struct mlx5_cache_ent *ent)
707	{	844	{
708	struct mlx5_mkey_cache *cache = &dev->cache;	845	int order = order_base_2(ent->rb_key.ndescs);
709	struct mlx5_cache_ent *ent;
710	struct dentry *dir;	846	struct dentry *dir;
711	int i;
712		847
713	if (!mlx5_debugfs_root \|\| dev->is_rep)	848	if (!mlx5_debugfs_root \|\| dev->is_rep)
714	return;	849	return;
715		850
716	cache->root = debugfs_create_dir("mr_cache", mlx5_debugfs_get_dev_root(dev->mdev));	851	if (ent->rb_key.access_mode == MLX5_MKC_ACCESS_MODE_KSM)
		852	order = MLX5_IMR_KSM_CACHE_ENTRY + 2;
717		853
718	for (i = 0; i < MAX_MKEY_CACHE_ENTRIES; i++) {	854	sprintf(ent->name, "%d", order);
719	ent = &cache->ent[i];	855	dir = debugfs_create_dir(ent->name, dev->cache.fs_root);
720	sprintf(ent->name, "%d", ent->order);	856	debugfs_create_file("size", 0600, dir, ent, &size_fops);
721	dir = debugfs_create_dir(ent->name, cache->root);	857	debugfs_create_file("limit", 0600, dir, ent, &limit_fops);
722	debugfs_create_file("size", 0600, dir, ent, &size_fops);	858	debugfs_create_ulong("cur", 0400, dir, &ent->stored);
723	debugfs_create_file("limit", 0600, dir, ent, &limit_fops);	859	debugfs_create_u32("miss", 0600, dir, &ent->miss);
724	debugfs_create_ulong("cur", 0400, dir, &ent->stored);	860	}
725	debugfs_create_u32("miss", 0600, dir, &ent->miss);	861
726	}	862	static void mlx5_mkey_cache_debugfs_init(struct mlx5_ib_dev *dev)
		863	{
		864	struct dentry *dbg_root = mlx5_debugfs_get_dev_root(dev->mdev);
		865	struct mlx5_mkey_cache *cache = &dev->cache;
		866
		867	if (!mlx5_debugfs_root \|\| dev->is_rep)
		868	return;
		869
		870	cache->fs_root = debugfs_create_dir("mr_cache", dbg_root);
727	}	871	}
728		872
729	static void delay_time_func(struct timer_list *t)	873	static void delay_time_func(struct timer_list *t)
@@ -733,13 +877,100 @@ static void delay_time_func(struct timer_list *t) · 3 unmodified lines
733	WRITE_ONCE(dev->fill_delay, 0);	877	WRITE_ONCE(dev->fill_delay, 0);
734	}	878	}
735		879
		880	struct mlx5_cache_ent *
		881	mlx5r_cache_create_ent_locked(struct mlx5_ib_dev *dev,
		882	struct mlx5r_cache_rb_key rb_key,
		883	bool persistent_entry)
		884	{
		885	struct mlx5_cache_ent *ent;
		886	int order;
		887	int ret;
		888
		889	ent = kzalloc(sizeof(*ent), GFP_KERNEL);
		890	if (!ent)
		891	return ERR_PTR(-ENOMEM);
		892
		893	xa_init_flags(&ent->mkeys, XA_FLAGS_LOCK_IRQ);
		894	ent->rb_key = rb_key;
		895	ent->dev = dev;
		896	ent->is_tmp = !persistent_entry;
		897
		898	INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func);
		899
		900	ret = mlx5_cache_ent_insert(&dev->cache, ent);
		901	if (ret) {
		902	kfree(ent);
		903	return ERR_PTR(ret);
		904	}
		905
		906	if (persistent_entry) {
		907	if (rb_key.access_mode == MLX5_MKC_ACCESS_MODE_KSM)
		908	order = MLX5_IMR_KSM_CACHE_ENTRY;
		909	else
		910	order = order_base_2(rb_key.ndescs) - 2;
		911
		912	if ((dev->mdev->profile.mask & MLX5_PROF_MASK_MR_CACHE) &&
		913	!dev->is_rep && mlx5_core_is_pf(dev->mdev) &&
		914	mlx5r_umr_can_load_pas(dev, 0))
		915	ent->limit = dev->mdev->profile.mr_cache[order].limit;
		916	else
		917	ent->limit = 0;
		918
		919	mlx5_mkey_cache_debugfs_add_ent(dev, ent);
		920	} else {
		921	mod_delayed_work(ent->dev->cache.wq,
		922	&ent->dev->cache.remove_ent_dwork,
		923	msecs_to_jiffies(30 * 1000));
		924	}
		925
		926	return ent;
		927	}
		928
		929	static void remove_ent_work_func(struct work_struct *work)
		930	{
		931	struct mlx5_mkey_cache *cache;
		932	struct mlx5_cache_ent *ent;
		933	struct rb_node *cur;
		934
		935	cache = container_of(work, struct mlx5_mkey_cache,
		936	remove_ent_dwork.work);
		937	mutex_lock(&cache->rb_lock);
		938	cur = rb_last(&cache->rb_root);
		939	while (cur) {
		940	ent = rb_entry(cur, struct mlx5_cache_ent, node);
		941	cur = rb_prev(cur);
		942	mutex_unlock(&cache->rb_lock);
		943
		944	xa_lock_irq(&ent->mkeys);
		945	if (!ent->is_tmp) {
		946	xa_unlock_irq(&ent->mkeys);
		947	mutex_lock(&cache->rb_lock);
		948	continue;
		949	}
		950	xa_unlock_irq(&ent->mkeys);
		951
		952	clean_keys(ent->dev, ent);
		953	mutex_lock(&cache->rb_lock);
		954	}
		955	mutex_unlock(&cache->rb_lock);
		956	}
		957
736	int mlx5_mkey_cache_init(struct mlx5_ib_dev *dev)	958	int mlx5_mkey_cache_init(struct mlx5_ib_dev *dev)
737	{	959	{
738	struct mlx5_mkey_cache *cache = &dev->cache;	960	struct mlx5_mkey_cache *cache = &dev->cache;
		961	struct rb_root *root = &dev->cache.rb_root;
		962	struct mlx5r_cache_rb_key rb_key = {
		963	.access_mode = MLX5_MKC_ACCESS_MODE_MTT,
		964	};
739	struct mlx5_cache_ent *ent;	965	struct mlx5_cache_ent *ent;
		966	struct rb_node *node;
		967	int ret;
740	int i;	968	int i;
741		969
742	mutex_init(&dev->slow_path_mutex);	970	mutex_init(&dev->slow_path_mutex);
		971	mutex_init(&dev->cache.rb_lock);
		972	dev->cache.rb_root = RB_ROOT;
		973	INIT_DELAYED_WORK(&dev->cache.remove_ent_dwork, remove_ent_work_func);
743	cache->wq = alloc_ordered_workqueue("mkey_cache", WQ_MEM_RECLAIM);	974	cache->wq = alloc_ordered_workqueue("mkey_cache", WQ_MEM_RECLAIM);
744	if (!cache->wq) {	975	if (!cache->wq) {
745	mlx5_ib_warn(dev, "failed to create work queue\n");	976	mlx5_ib_warn(dev, "failed to create work queue\n");
@@ -748,52 +979,51 @@ int mlx5_mkey_cache_init(struct mlx5_ib_dev *dev) · 2 unmodified lines
748		979
749	mlx5_cmd_init_async_ctx(dev->mdev, &dev->async_ctx);	980	mlx5_cmd_init_async_ctx(dev->mdev, &dev->async_ctx);
750	timer_setup(&dev->delay_timer, delay_time_func, 0);	981	timer_setup(&dev->delay_timer, delay_time_func, 0);
751	for (i = 0; i < MAX_MKEY_CACHE_ENTRIES; i++) {	982	mlx5_mkey_cache_debugfs_init(dev);
752	ent = &cache->ent[i];	983	mutex_lock(&cache->rb_lock);
753	xa_init_flags(&ent->mkeys, XA_FLAGS_LOCK_IRQ);	984	for (i = 0; i <= mkey_cache_max_order(dev); i++) {
754	ent->order = i + 2;	985	rb_key.ndescs = 1 << (i + 2);
755	ent->dev = dev;	986	ent = mlx5r_cache_create_ent_locked(dev, rb_key, true);
756	ent->limit = 0;	987	if (IS_ERR(ent)) {
757		988	ret = PTR_ERR(ent);
758	INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func);	989	goto err;
759
760	if (i > MKEY_CACHE_LAST_STD_ENTRY) {
761	mlx5_odp_init_mkey_cache_entry(ent);
762	continue;
763	}	990	}
		991	}
764		992
765	if (ent->order > mkey_cache_max_order(dev))	993	ret = mlx5_odp_init_mkey_cache(dev);
766	continue;	994	if (ret)
		995	goto err;
767		996
768	ent->page = PAGE_SHIFT;	997	mutex_unlock(&cache->rb_lock);
769	ent->ndescs = 1 << ent->order;	998	for (node = rb_first(root); node; node = rb_next(node)) {
770	ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT;	999	ent = rb_entry(node, struct mlx5_cache_ent, node);
771	if ((dev->mdev->profile.mask & MLX5_PROF_MASK_MR_CACHE) &&
772	!dev->is_rep && mlx5_core_is_pf(dev->mdev) &&
773	mlx5r_umr_can_load_pas(dev, 0))
774	ent->limit = dev->mdev->profile.mr_cache[i].limit;
775	else
776	ent->limit = 0;
777	xa_lock_irq(&ent->mkeys);	1000	xa_lock_irq(&ent->mkeys);
778	queue_adjust_cache_locked(ent);	1001	queue_adjust_cache_locked(ent);
779	xa_unlock_irq(&ent->mkeys);	1002	xa_unlock_irq(&ent->mkeys);
780	}	1003	}
781		1004
782	mlx5_mkey_cache_debugfs_init(dev);
783
784	return 0;	1005	return 0;
		1006
		1007	err:
		1008	mutex_unlock(&cache->rb_lock);
		1009	mlx5_mkey_cache_debugfs_cleanup(dev);
		1010	mlx5_ib_warn(dev, "failed to create mkey cache entry\n");
		1011	return ret;
785	}	1012	}
786		1013
787	int mlx5_mkey_cache_cleanup(struct mlx5_ib_dev *dev)	1014	void mlx5_mkey_cache_cleanup(struct mlx5_ib_dev *dev)
788	{	1015	{
789	unsigned int i;	1016	struct rb_root *root = &dev->cache.rb_root;
		1017	struct mlx5_cache_ent *ent;
		1018	struct rb_node *node;
790		1019
791	if (!dev->cache.wq)	1020	if (!dev->cache.wq)
792	return 0;	1021	return;
793
794	for (i = 0; i < MAX_MKEY_CACHE_ENTRIES; i++) {
795	struct mlx5_cache_ent *ent = &dev->cache.ent[i];
796		1022
		1023	cancel_delayed_work_sync(&dev->cache.remove_ent_dwork);
		1024	mutex_lock(&dev->cache.rb_lock);
		1025	for (node = rb_first(root); node; node = rb_next(node)) {
		1026	ent = rb_entry(node, struct mlx5_cache_ent, node);
797	xa_lock_irq(&ent->mkeys);	1027	xa_lock_irq(&ent->mkeys);
798	ent->disabled = true;	1028	ent->disabled = true;
799	xa_unlock_irq(&ent->mkeys);	1029	xa_unlock_irq(&ent->mkeys);
@@ -803,13 +1033,18 @@ int mlx5_mkey_cache_cleanup(struct mlx5_ib_dev *dev) · 3 unmodified lines
803	mlx5_mkey_cache_debugfs_cleanup(dev);	1033	mlx5_mkey_cache_debugfs_cleanup(dev);
804	mlx5_cmd_cleanup_async_ctx(&dev->async_ctx);	1034	mlx5_cmd_cleanup_async_ctx(&dev->async_ctx);
805		1035
806	for (i = 0; i < MAX_MKEY_CACHE_ENTRIES; i++)	1036	node = rb_first(root);
807	clean_keys(dev, i);	1037	while (node) {
		1038	ent = rb_entry(node, struct mlx5_cache_ent, node);
		1039	node = rb_next(node);
		1040	clean_keys(dev, ent);
		1041	rb_erase(&ent->node, root);
		1042	kfree(ent);
		1043	}
		1044	mutex_unlock(&dev->cache.rb_lock);
808		1045
809	destroy_workqueue(dev->cache.wq);	1046	destroy_workqueue(dev->cache.wq);
810	del_timer_sync(&dev->delay_timer);	1047	del_timer_sync(&dev->delay_timer);
811
812	return 0;
813	}	1048	}
814		1049
815	struct ib_mr mlx5_ib_get_dma_mr(struct ib_pd pd, int acc)	1050	struct ib_mr mlx5_ib_get_dma_mr(struct ib_pd pd, int acc)
@@ -873,23 +1108,10 @@ static int get_octo_len(u64 addr, u64 len, int page_shift) · 57 unmodified lines
873	static int mkey_cache_max_order(struct mlx5_ib_dev *dev)	1108	static int mkey_cache_max_order(struct mlx5_ib_dev *dev)
874	{	1109	{
875	if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset))	1110	if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset))
876	return MKEY_CACHE_LAST_STD_ENTRY + 2;	1111	return MKEY_CACHE_LAST_STD_ENTRY;
877	return MLX5_MAX_UMR_SHIFT;	1112	return MLX5_MAX_UMR_SHIFT;
878	}	1113	}
879		1114
880	static struct mlx5_cache_ent mkey_cache_ent_from_order(struct mlx5_ib_dev dev,
881	unsigned int order)
882	{
883	struct mlx5_mkey_cache *cache = &dev->cache;
884
885	if (order < cache->ent[0].order)
886	return &cache->ent[0];
887	order = order - cache->ent[0].order;
888	if (order > MKEY_CACHE_LAST_STD_ENTRY)
889	return NULL;
890	return &cache->ent[order];
891	}
892
893	static void set_mr_fields(struct mlx5_ib_dev dev, struct mlx5_ib_mr mr,	1115	static void set_mr_fields(struct mlx5_ib_dev dev, struct mlx5_ib_mr mr,
894	u64 length, int access_flags, u64 iova)	1116	u64 length, int access_flags, u64 iova)
895	{	1117	{
@@ -916,6 +1138,9 @@ static struct mlx5_ib_mr alloc_cacheable_mr(struct ib_pd pd, · 20 unmodified lines
916	struct ib_umem *umem, u64 iova,	1138	struct ib_umem *umem, u64 iova,
917	int access_flags)	1139	int access_flags)
918	{	1140	{
		1141	struct mlx5r_cache_rb_key rb_key = {
		1142	.access_mode = MLX5_MKC_ACCESS_MODE_MTT,
		1143	};
919	struct mlx5_ib_dev *dev = to_mdev(pd->device);	1144	struct mlx5_ib_dev *dev = to_mdev(pd->device);
920	struct mlx5_cache_ent *ent;	1145	struct mlx5_cache_ent *ent;
921	struct mlx5_ib_mr *mr;	1146	struct mlx5_ib_mr *mr;
@@ -928,22 +1153,26 @@ static struct mlx5_ib_mr alloc_cacheable_mr(struct ib_pd pd, · 6 unmodified lines
928	0, iova);	1153	0, iova);
929	if (WARN_ON(!page_size))	1154	if (WARN_ON(!page_size))
930	return ERR_PTR(-EINVAL);	1155	return ERR_PTR(-EINVAL);
931	ent = mkey_cache_ent_from_order(	1156
932	dev, order_base_2(ib_umem_num_dma_blocks(umem, page_size)));	1157	rb_key.ndescs = ib_umem_num_dma_blocks(umem, page_size);
		1158	rb_key.ats = mlx5_umem_needs_ats(dev, umem, access_flags);
		1159	rb_key.access_flags = get_unchangeable_access_flags(dev, access_flags);
		1160	ent = mkey_cache_ent_from_rb_key(dev, rb_key);
933	/*	1161	/*
934	* Matches access in alloc_cache_mr(). If the MR can't come from the	1162	* If the MR can't come from the cache then synchronously create an uncached
935	* cache then synchronously create an uncached one.	1163	* one.
936	*/	1164	*/
937	if (!ent \|\| ent->limit == 0 \|\|	1165	if (!ent) {
938	!mlx5r_umr_can_reconfig(dev, 0, access_flags) \|\|
939	mlx5_umem_needs_ats(dev, umem, access_flags)) {
940	mutex_lock(&dev->slow_path_mutex);	1166	mutex_lock(&dev->slow_path_mutex);
941	mr = reg_create(pd, umem, iova, access_flags, page_size, false);	1167	mr = reg_create(pd, umem, iova, access_flags, page_size, false);
942	mutex_unlock(&dev->slow_path_mutex);	1168	mutex_unlock(&dev->slow_path_mutex);
		1169	if (IS_ERR(mr))
		1170	return mr;
		1171	mr->mmkey.rb_key = rb_key;
943	return mr;	1172	return mr;
944	}	1173	}
945		1174
946	mr = mlx5_mr_cache_alloc(dev, ent, access_flags);	1175	mr = _mlx5_mr_cache_alloc(dev, ent, access_flags);
947	if (IS_ERR(mr))	1176	if (IS_ERR(mr))
948	return mr;	1177	return mr;
949		1178
@@ -1030,6 +1259,7 @@ static struct mlx5_ib_mr reg_create(struct ib_pd pd, struct ib_umem *umem, · 80 unmodified lines
1030	goto err_2;	1259	goto err_2;
1031	}	1260	}
1032	mr->mmkey.type = MLX5_MKEY_MR;	1261	mr->mmkey.type = MLX5_MKEY_MR;
		1262	mr->mmkey.ndescs = get_octo_len(iova, umem->length, mr->page_shift);
1033	mr->umem = umem;	1263	mr->umem = umem;
1034	set_mr_fields(dev, mr, umem->length, access_flags, iova);	1264	set_mr_fields(dev, mr, umem->length, access_flags, iova);
1035	kvfree(in);	1265	kvfree(in);
@@ -1372,7 +1602,7 @@ static bool can_use_umr_rereg_pas(struct mlx5_ib_mr *mr, · 336 unmodified lines
1372	mlx5_umem_find_best_pgsz(new_umem, mkc, log_page_size, 0, iova);	1602	mlx5_umem_find_best_pgsz(new_umem, mkc, log_page_size, 0, iova);
1373	if (WARN_ON(!*page_size))	1603	if (WARN_ON(!*page_size))
1374	return false;	1604	return false;
1375	return (1ULL << mr->mmkey.cache_ent->order) >=	1605	return (mr->mmkey.cache_ent->rb_key.ndescs) >=
1376	ib_umem_num_dma_blocks(new_umem, *page_size);	1606	ib_umem_num_dma_blocks(new_umem, *page_size);
1377	}	1607	}
1378		1608
@@ -1567,6 +1797,49 @@ mlx5_free_priv_descs(struct mlx5_ib_mr *mr) · 188 unmodified lines
1567	}	1797	}
1568	}	1798	}
1569		1799
		1800	static int cache_ent_find_and_store(struct mlx5_ib_dev *dev,
		1801	struct mlx5_ib_mr *mr)
		1802	{
		1803	struct mlx5_mkey_cache *cache = &dev->cache;
		1804	struct mlx5_cache_ent *ent;
		1805	int ret;
		1806
		1807	if (mr->mmkey.cache_ent) {
		1808	xa_lock_irq(&mr->mmkey.cache_ent->mkeys);
		1809	mr->mmkey.cache_ent->in_use--;
		1810	goto end;
		1811	}
		1812
		1813	mutex_lock(&cache->rb_lock);
		1814	ent = mkey_cache_ent_from_rb_key(dev, mr->mmkey.rb_key);
		1815	if (ent) {
		1816	if (ent->rb_key.ndescs == mr->mmkey.rb_key.ndescs) {
		1817	if (ent->disabled) {
		1818	mutex_unlock(&cache->rb_lock);
		1819	return -EOPNOTSUPP;
		1820	}
		1821	mr->mmkey.cache_ent = ent;
		1822	xa_lock_irq(&mr->mmkey.cache_ent->mkeys);
		1823	mutex_unlock(&cache->rb_lock);
		1824	goto end;
		1825	}
		1826	}
		1827
		1828	ent = mlx5r_cache_create_ent_locked(dev, mr->mmkey.rb_key, false);
		1829	mutex_unlock(&cache->rb_lock);
		1830	if (IS_ERR(ent))
		1831	return PTR_ERR(ent);
		1832
		1833	mr->mmkey.cache_ent = ent;
		1834	xa_lock_irq(&mr->mmkey.cache_ent->mkeys);
		1835
		1836	end:
		1837	ret = push_mkey_locked(mr->mmkey.cache_ent, false,
		1838	xa_mk_value(mr->mmkey.key));
		1839	xa_unlock_irq(&mr->mmkey.cache_ent->mkeys);
		1840	return ret;
		1841	}
		1842
1570	int mlx5_ib_dereg_mr(struct ib_mr ibmr, struct ib_udata udata)	1843	int mlx5_ib_dereg_mr(struct ib_mr ibmr, struct ib_udata udata)
1571	{	1844	{
1572	struct mlx5_ib_mr *mr = to_mmr(ibmr);	1845	struct mlx5_ib_mr *mr = to_mmr(ibmr);
@@ -1612,16 +1885,11 @@ int mlx5_ib_dereg_mr(struct ib_mr ibmr, struct ib_udata udata) · 39 unmodified lines
1612	}	1885	}
1613		1886
1614	/* Stop DMA */	1887	/* Stop DMA */
1615	if (mr->mmkey.cache_ent) {	1888	if (mr->umem && mlx5r_umr_can_load_pas(dev, mr->umem->length))
1616	xa_lock_irq(&mr->mmkey.cache_ent->mkeys);
1617	mr->mmkey.cache_ent->in_use--;
1618	xa_unlock_irq(&mr->mmkey.cache_ent->mkeys);
1619
1620	if (mlx5r_umr_revoke_mr(mr) \|\|	1889	if (mlx5r_umr_revoke_mr(mr) \|\|
1621	push_mkey(mr->mmkey.cache_ent, false,	1890	cache_ent_find_and_store(dev, mr))
1622	xa_mk_value(mr->mmkey.key)))
1623	mr->mmkey.cache_ent = NULL;	1891	mr->mmkey.cache_ent = NULL;
1624	}	1892
1625	if (!mr->mmkey.cache_ent) {	1893	if (!mr->mmkey.cache_ent) {
1626	rc = destroy_mkey(to_mdev(mr->ibmr.device), mr);	1894	rc = destroy_mkey(to_mdev(mr->ibmr.device), mr);
1627	if (rc)	1895	if (rc)

drivers/infiniband/hw/mlx5/odp.c+27−40

@@ -104,7 +104,7 @@ static void populate_klm(struct mlx5_klm *pklm, size_t idx, size_t nentries, · 103 unmodified lines
104	if (flags & MLX5_IB_UPD_XLT_ZAP) {	104	if (flags & MLX5_IB_UPD_XLT_ZAP) {
105	for (; pklm != end; pklm++, idx++) {	105	for (; pklm != end; pklm++, idx++) {
106	pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE);	106	pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE);
107	pklm->key = cpu_to_be32(mr_to_mdev(imr)->null_mkey);	107	pklm->key = mr_to_mdev(imr)->mkeys.null_mkey;
108	pklm->va = 0;	108	pklm->va = 0;
109	}	109	}
110	return;	110	return;
@@ -137,7 +137,7 @@ static void populate_klm(struct mlx5_klm *pklm, size_t idx, size_t nentries, · 26 unmodified lines
137	pklm->key = cpu_to_be32(mtt->ibmr.lkey);	137	pklm->key = cpu_to_be32(mtt->ibmr.lkey);
138	pklm->va = cpu_to_be64(idx * MLX5_IMR_MTT_SIZE);	138	pklm->va = cpu_to_be64(idx * MLX5_IMR_MTT_SIZE);
139	} else {	139	} else {
140	pklm->key = cpu_to_be32(mr_to_mdev(imr)->null_mkey);	140	pklm->key = mr_to_mdev(imr)->mkeys.null_mkey;
141	pklm->va = 0;	141	pklm->va = 0;
142	}	142	}
143	}	143	}
@@ -417,8 +417,9 @@ static struct mlx5_ib_mr implicit_get_child_mr(struct mlx5_ib_mr imr, · 273 unmodified lines
417	if (IS_ERR(odp))	417	if (IS_ERR(odp))
418	return ERR_CAST(odp);	418	return ERR_CAST(odp);
419		419
420	mr = mlx5_mr_cache_alloc(dev, &dev->cache.ent[MLX5_IMR_MTT_CACHE_ENTRY],	420	mr = mlx5_mr_cache_alloc(dev, imr->access_flags,
421	imr->access_flags);	421	MLX5_MKC_ACCESS_MODE_MTT,
		422	MLX5_IMR_MTT_ENTRIES);
422	if (IS_ERR(mr)) {	423	if (IS_ERR(mr)) {
423	ib_umem_odp_release(odp);	424	ib_umem_odp_release(odp);
424	return mr;	425	return mr;
@@ -492,9 +493,8 @@ struct mlx5_ib_mr mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd pd, · 67 unmodified lines
492	if (IS_ERR(umem_odp))	493	if (IS_ERR(umem_odp))
493	return ERR_CAST(umem_odp);	494	return ERR_CAST(umem_odp);
494		495
495	imr = mlx5_mr_cache_alloc(dev,	496	imr = mlx5_mr_cache_alloc(dev, access_flags, MLX5_MKC_ACCESS_MODE_KSM,
496	&dev->cache.ent[MLX5_IMR_KSM_CACHE_ENTRY],	497	mlx5_imr_ksm_entries);
497	access_flags);
498	if (IS_ERR(imr)) {	498	if (IS_ERR(imr)) {
499	ib_umem_odp_release(umem_odp);	499	ib_umem_odp_release(umem_odp);
500	return imr;	500	return imr;
@@ -986,7 +986,7 @@ static int pagefault_data_segments(struct mlx5_ib_dev *dev, · 485 unmodified lines
986	{	986	{
987	int ret = 0, npages = 0;	987	int ret = 0, npages = 0;
988	u64 io_virt;	988	u64 io_virt;
989	u32 key;	989	__be32 key;
990	u32 byte_count;	990	u32 byte_count;
991	size_t bcnt;	991	size_t bcnt;
992	int inline_segment;	992	int inline_segment;
@@ -1000,7 +1000,7 @@ static int pagefault_data_segments(struct mlx5_ib_dev *dev, · 7 unmodified lines
1000	struct mlx5_wqe_data_seg *dseg = wqe;	1000	struct mlx5_wqe_data_seg *dseg = wqe;
1001		1001
1002	io_virt = be64_to_cpu(dseg->addr);	1002	io_virt = be64_to_cpu(dseg->addr);
1003	key = be32_to_cpu(dseg->lkey);	1003	key = dseg->lkey;
1004	byte_count = be32_to_cpu(dseg->byte_count);	1004	byte_count = be32_to_cpu(dseg->byte_count);
1005	inline_segment = !!(byte_count & MLX5_INLINE_SEG);	1005	inline_segment = !!(byte_count & MLX5_INLINE_SEG);
1006	bcnt = byte_count & ~MLX5_INLINE_SEG;	1006	bcnt = byte_count & ~MLX5_INLINE_SEG;
@@ -1014,7 +1014,8 @@ static int pagefault_data_segments(struct mlx5_ib_dev *dev, · 7 unmodified lines
1014	}	1014	}
1015		1015
1016	/* receive WQE end of sg list. */	1016	/* receive WQE end of sg list. */
1017	if (receive_queue && bcnt == 0 && key == MLX5_INVALID_LKEY &&	1017	if (receive_queue && bcnt == 0 &&
		1018	key == dev->mkeys.terminate_scatter_list_mkey &&
1018	io_virt == 0)	1019	io_virt == 0)
1019	break;	1020	break;
1020		1021
@@ -1034,7 +1035,7 @@ static int pagefault_data_segments(struct mlx5_ib_dev *dev, · 13 unmodified lines
1034	continue;	1035	continue;
1035	}	1036	}
1036		1037
1037	ret = pagefault_single_data_segment(dev, NULL, key,	1038	ret = pagefault_single_data_segment(dev, NULL, be32_to_cpu(key),
1038	io_virt, bcnt,	1039	io_virt, bcnt,
1039	&pfault->bytes_committed,	1040	&pfault->bytes_committed,
1040	bytes_mapped);	1041	bytes_mapped);
@@ -1587,26 +1588,22 @@ mlx5_ib_odp_destroy_eq(struct mlx5_ib_dev dev, struct mlx5_ib_pf_eq eq) · 546 unmodified lines
1587	return err;	1588	return err;
1588	}	1589	}
1589		1590
1590	void mlx5_odp_init_mkey_cache_entry(struct mlx5_cache_ent *ent)	1591	int mlx5_odp_init_mkey_cache(struct mlx5_ib_dev *dev)
1591	{	1592	{
1592	if (!(ent->dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT))	1593	struct mlx5r_cache_rb_key rb_key = {
1593	return;	1594	.access_mode = MLX5_MKC_ACCESS_MODE_KSM,
		1595	.ndescs = mlx5_imr_ksm_entries,
		1596	};
		1597	struct mlx5_cache_ent *ent;
1594		1598
1595	switch (ent->order - 2) {	1599	if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT))
1596	case MLX5_IMR_MTT_CACHE_ENTRY:	1600	return 0;
1597	ent->page = PAGE_SHIFT;
1598	ent->ndescs = MLX5_IMR_MTT_ENTRIES;
1599	ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
1600	ent->limit = 0;
1601	break;
1602		1601
1603	case MLX5_IMR_KSM_CACHE_ENTRY:	1602	ent = mlx5r_cache_create_ent_locked(dev, rb_key, true);
1604	ent->page = MLX5_KSM_PAGE_SHIFT;	1603	if (IS_ERR(ent))
1605	ent->ndescs = mlx5_imr_ksm_entries;	1604	return PTR_ERR(ent);
1606	ent->access_mode = MLX5_MKC_ACCESS_MODE_KSM;	1605
1607	ent->limit = 0;	1606	return 0;
1608	break;
1609	}
1610	}	1607	}
1611		1608
1612	static const struct ib_device_ops mlx5_ib_dev_odp_ops = {	1609	static const struct ib_device_ops mlx5_ib_dev_odp_ops = {
@@ -1615,25 +1612,15 @@ static const struct ib_device_ops mlx5_ib_dev_odp_ops = { · 2 unmodified lines
1615		1612
1616	int mlx5_ib_odp_init_one(struct mlx5_ib_dev *dev)	1613	int mlx5_ib_odp_init_one(struct mlx5_ib_dev *dev)
1617	{	1614	{
1618	int ret = 0;
1619
1620	internal_fill_odp_caps(dev);	1615	internal_fill_odp_caps(dev);
1621		1616
1622	if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT))	1617	if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT))
1623	return ret;	1618	return 0;
1624		1619
1625	ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_odp_ops);	1620	ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_odp_ops);
1626		1621
1627	if (dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT) {
1628	ret = mlx5_cmd_null_mkey(dev->mdev, &dev->null_mkey);
1629	if (ret) {
1630	mlx5_ib_err(dev, "Error getting null_mkey %d\n", ret);
1631	return ret;
1632	}
1633	}
1634
1635	mutex_init(&dev->odp_eq_mutex);	1622	mutex_init(&dev->odp_eq_mutex);
1636	return ret;	1623	return 0;
1637	}	1624	}
1638		1625
1639	void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *dev)	1626	void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *dev)

drivers/infiniband/hw/mlx5/qp.c+128−36

@@ -71,6 +71,14 @@ struct mlx5_modify_raw_qp_param { · 70 unmodified lines
71	u32 port;	71	u32 port;
72	};	72	};
73		73
		74	struct mlx5_ib_qp_event_work {
		75	struct work_struct work;
		76	struct mlx5_core_qp *qp;
		77	int type;
		78	};
		79
		80	static struct workqueue_struct *mlx5_ib_qp_event_wq;
		81
74	static void get_cqs(enum ib_qp_type qp_type,	82	static void get_cqs(enum ib_qp_type qp_type,
75	struct ib_cq ib_send_cq, struct ib_cq ib_recv_cq,	83	struct ib_cq ib_send_cq, struct ib_cq ib_recv_cq,
76	struct mlx5_ib_cq send_cq, struct mlx5_ib_cq recv_cq);	84	struct mlx5_ib_cq send_cq, struct mlx5_ib_cq recv_cq);
@@ -302,51 +310,120 @@ int mlx5_ib_read_wqe_srq(struct mlx5_ib_srq srq, int wqe_index, void buffer, · 225 unmodified lines
302	return mlx5_ib_read_user_wqe_srq(srq, wqe_index, buffer, buflen, bc);	310	return mlx5_ib_read_user_wqe_srq(srq, wqe_index, buffer, buflen, bc);
303	}	311	}
304		312
		313	static void mlx5_ib_qp_err_syndrome(struct ib_qp *ibqp)
		314	{
		315	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
		316	int outlen = MLX5_ST_SZ_BYTES(query_qp_out);
		317	struct mlx5_ib_qp *qp = to_mqp(ibqp);
		318	void pas_ext_union, err_syn;
		319	u32 *outb;
		320	int err;
		321
		322	if (!MLX5_CAP_GEN(dev->mdev, qpc_extension) \|\|
		323	!MLX5_CAP_GEN(dev->mdev, qp_error_syndrome))
		324	return;
		325
		326	outb = kzalloc(outlen, GFP_KERNEL);
		327	if (!outb)
		328	return;
		329
		330	err = mlx5_core_qp_query(dev, &qp->trans_qp.base.mqp, outb, outlen,
		331	true);
		332	if (err)
		333	goto out;
		334
		335	pas_ext_union =
		336	MLX5_ADDR_OF(query_qp_out, outb, qp_pas_or_qpc_ext_and_pas);
		337	err_syn = MLX5_ADDR_OF(qpc_extension_and_pas_list_in, pas_ext_union,
		338	qpc_data_extension.error_syndrome);
		339
		340	pr_err("%s/%d: QP %d error: %s (0x%x 0x%x 0x%x)\n",
		341	ibqp->device->name, ibqp->port, ibqp->qp_num,
		342	ib_wc_status_msg(
		343	MLX5_GET(cqe_error_syndrome, err_syn, syndrome)),
		344	MLX5_GET(cqe_error_syndrome, err_syn, vendor_error_syndrome),
		345	MLX5_GET(cqe_error_syndrome, err_syn, hw_syndrome_type),
		346	MLX5_GET(cqe_error_syndrome, err_syn, hw_error_syndrome));
		347	out:
		348	kfree(outb);
		349	}
		350
		351	static void mlx5_ib_handle_qp_event(struct work_struct *_work)
		352	{
		353	struct mlx5_ib_qp_event_work *qpe_work =
		354	container_of(_work, struct mlx5_ib_qp_event_work, work);
		355	struct ib_qp *ibqp = &to_mibqp(qpe_work->qp)->ibqp;
		356	struct ib_event event = {};
		357
		358	event.device = ibqp->device;
		359	event.element.qp = ibqp;
		360	switch (qpe_work->type) {
		361	case MLX5_EVENT_TYPE_PATH_MIG:
		362	event.event = IB_EVENT_PATH_MIG;
		363	break;
		364	case MLX5_EVENT_TYPE_COMM_EST:
		365	event.event = IB_EVENT_COMM_EST;
		366	break;
		367	case MLX5_EVENT_TYPE_SQ_DRAINED:
		368	event.event = IB_EVENT_SQ_DRAINED;
		369	break;
		370	case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
		371	event.event = IB_EVENT_QP_LAST_WQE_REACHED;
		372	break;
		373	case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
		374	event.event = IB_EVENT_QP_FATAL;
		375	break;
		376	case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
		377	event.event = IB_EVENT_PATH_MIG_ERR;
		378	break;
		379	case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
		380	event.event = IB_EVENT_QP_REQ_ERR;
		381	break;
		382	case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
		383	event.event = IB_EVENT_QP_ACCESS_ERR;
		384	break;
		385	default:
		386	pr_warn("mlx5_ib: Unexpected event type %d on QP %06x\n",
		387	qpe_work->type, qpe_work->qp->qpn);
		388	goto out;
		389	}
		390
		391	if ((event.event == IB_EVENT_QP_FATAL) \|\|
		392	(event.event == IB_EVENT_QP_ACCESS_ERR))
		393	mlx5_ib_qp_err_syndrome(ibqp);
		394
		395	ibqp->event_handler(&event, ibqp->qp_context);
		396
		397	out:
		398	mlx5_core_res_put(&qpe_work->qp->common);
		399	kfree(qpe_work);
		400	}
		401
305	static void mlx5_ib_qp_event(struct mlx5_core_qp *qp, int type)	402	static void mlx5_ib_qp_event(struct mlx5_core_qp *qp, int type)
306	{	403	{
307	struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;	404	struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;
308	struct ib_event event;	405	struct mlx5_ib_qp_event_work *qpe_work;
309		406
310	if (type == MLX5_EVENT_TYPE_PATH_MIG) {	407	if (type == MLX5_EVENT_TYPE_PATH_MIG) {
311	/* This event is only valid for trans_qps */	408	/* This event is only valid for trans_qps */
312	to_mibqp(qp)->port = to_mibqp(qp)->trans_qp.alt_port;	409	to_mibqp(qp)->port = to_mibqp(qp)->trans_qp.alt_port;
313	}	410	}
314		411
315	if (ibqp->event_handler) {	412	if (!ibqp->event_handler)
316	event.device = ibqp->device;	413	goto out_no_handler;
317	event.element.qp = ibqp;
318	switch (type) {
319	case MLX5_EVENT_TYPE_PATH_MIG:
320	event.event = IB_EVENT_PATH_MIG;
321	break;
322	case MLX5_EVENT_TYPE_COMM_EST:
323	event.event = IB_EVENT_COMM_EST;
324	break;
325	case MLX5_EVENT_TYPE_SQ_DRAINED:
326	event.event = IB_EVENT_SQ_DRAINED;
327	break;
328	case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
329	event.event = IB_EVENT_QP_LAST_WQE_REACHED;
330	break;
331	case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
332	event.event = IB_EVENT_QP_FATAL;
333	break;
334	case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
335	event.event = IB_EVENT_PATH_MIG_ERR;
336	break;
337	case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
338	event.event = IB_EVENT_QP_REQ_ERR;
339	break;
340	case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
341	event.event = IB_EVENT_QP_ACCESS_ERR;
342	break;
343	default:
344	pr_warn("mlx5_ib: Unexpected event type %d on QP %06x\n", type, qp->qpn);
345	return;
346	}
347		414
348	ibqp->event_handler(&event, ibqp->qp_context);	415	qpe_work = kzalloc(sizeof(*qpe_work), GFP_ATOMIC);
349	}	416	if (!qpe_work)
		417	goto out_no_handler;
		418
		419	qpe_work->qp = qp;
		420	qpe_work->type = type;
		421	INIT_WORK(&qpe_work->work, mlx5_ib_handle_qp_event);
		422	queue_work(mlx5_ib_qp_event_wq, &qpe_work->work);
		423	return;
		424
		425	out_no_handler:
		426	mlx5_core_res_put(&qp->common);
350	}	427	}
351		428
352	static int set_rq_size(struct mlx5_ib_dev dev, struct ib_qp_cap cap,	429	static int set_rq_size(struct mlx5_ib_dev dev, struct ib_qp_cap cap,
@@ -4827,7 +4904,8 @@ static int query_qp_attr(struct mlx5_ib_dev dev, struct mlx5_ib_qp qp, · 4474 unmodified lines
4827	if (!outb)	4904	if (!outb)
4828	return -ENOMEM;	4905	return -ENOMEM;
4829		4906
4830	err = mlx5_core_qp_query(dev, &qp->trans_qp.base.mqp, outb, outlen);	4907	err = mlx5_core_qp_query(dev, &qp->trans_qp.base.mqp, outb, outlen,
		4908	false);
4831	if (err)	4909	if (err)
4832	goto out;	4910	goto out;
4833		4911
@@ -5720,3 +5798,17 @@ out: · 886 unmodified lines
5720	mutex_unlock(&mqp->mutex);	5798	mutex_unlock(&mqp->mutex);
5721	return err;	5799	return err;
5722	}	5800	}
		5801
		5802	int mlx5_ib_qp_event_init(void)
		5803	{
		5804	mlx5_ib_qp_event_wq = alloc_ordered_workqueue("mlx5_ib_qp_event_wq", 0);
		5805	if (!mlx5_ib_qp_event_wq)
		5806	return -ENOMEM;
		5807
		5808	return 0;
		5809	}
		5810
		5811	void mlx5_ib_qp_event_cleanup(void)
		5812	{
		5813	destroy_workqueue(mlx5_ib_qp_event_wq);
		5814	}

drivers/infiniband/hw/mlx5/qp.h+3−1

 int mlx5_core_destroy_qp(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp);
 int mlx5_core_destroy_dct(struct mlx5_ib_dev *dev, struct mlx5_core_dct *dct);
 int mlx5_core_qp_query(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp,
-		       u32 *out, int outlen);
+		       u32 *out, int outlen, bool qpc_ext);
 int mlx5_core_dct_query(struct mlx5_ib_dev *dev, struct mlx5_core_dct *dct,
 			u32 *out, int outlen);
 int mlx5_core_xrcd_alloc(struct mlx5_ib_dev *dev, u32 *xrcdn);
 int mlx5_core_xrcd_dealloc(struct mlx5_ib_dev *dev, u32 xrcdn);
 int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter *counter);
+int mlx5_ib_qp_event_init(void);
+void mlx5_ib_qp_event_cleanup(void);
 #endif /* _MLX5_IB_QP_H */

drivers/infiniband/hw/mlx5/qpc.c+5−2

 	case MLX5_RES_SQ:
 		qp = (struct mlx5_core_qp *)common;
 		qp->event(qp, event_type);
-		break;
+		/* Need to put resource in event handler */
+		return NOTIFY_OK;
 	case MLX5_RES_DCT:
 		dct = (struct mlx5_core_dct *)common;
 		if (event_type == MLX5_EVENT_TYPE_DCT_DRAINED)
 }
 int mlx5_core_qp_query(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp,
-		       u32 *out, int outlen)
+		       u32 *out, int outlen, bool qpc_ext)
 {
 	u32 in[MLX5_ST_SZ_DW(query_qp_in)] = {};
 	MLX5_SET(query_qp_in, in, opcode, MLX5_CMD_OP_QUERY_QP);
 	MLX5_SET(query_qp_in, in, qpn, qp->qpn);
+	MLX5_SET(query_qp_in, in, qpc_ext, qpc_ext);
 	return mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, outlen);
 }

drivers/infiniband/hw/mlx5/srq.c+1−1

@@ -447,7 +447,7 @@ int mlx5_ib_post_srq_recv(struct ib_srq ibsrq, const struct ib_recv_wr wr, · 446 unmodified lines
447		447
448	if (i < srq->msrq.max_avail_gather) {	448	if (i < srq->msrq.max_avail_gather) {
449	scat[i].byte_count = 0;	449	scat[i].byte_count = 0;
450	scat[i].lkey = cpu_to_be32(MLX5_INVALID_LKEY);	450	scat[i].lkey = dev->mkeys.terminate_scatter_list_mkey;
451	scat[i].addr = 0;	451	scat[i].addr = 0;
452	}	452	}
453	}	453	}

drivers/infiniband/hw/mlx5/umr.c+1−3

 	mlx5r_umr_set_update_xlt_data_seg(&wqe.data_seg, &sg);
 	cur_mtt = mtt;
-	rdma_for_each_block(mr->umem->sgt_append.sgt.sgl, &biter,
+	rdma_umem_for_each_dma_block(mr->umem, &biter, BIT(mr->page_shift)) {
-			    mr->umem->sgt_append.sgt.nents,
-			    BIT(mr->page_shift)) {
 		if (cur_mtt == (void *)mtt + sg.length) {
 			dma_sync_single_for_device(ddev, sg.addr, sg.length,
 						   DMA_TO_DEVICE);

drivers/infiniband/hw/mlx5/wr.c+1−1

@@ -1252,7 +1252,7 @@ int mlx5_ib_post_recv(struct ib_qp ibqp, const struct ib_recv_wr wr, · 1251 unmodified lines
1252		1252
1253	if (i < qp->rq.max_gs) {	1253	if (i < qp->rq.max_gs) {
1254	scat[i].byte_count = 0;	1254	scat[i].byte_count = 0;
1255	scat[i].lkey = cpu_to_be32(MLX5_INVALID_LKEY);	1255	scat[i].lkey = dev->mkeys.terminate_scatter_list_mkey;
1256	scat[i].addr = 0;	1256	scat[i].addr = 0;
1257	}	1257	}
1258		1258

drivers/infiniband/sw/rxe/rxe.h+38−0

@@ -57,6 +57,44 @@ · 56 unmodified lines
57	#define rxe_dbg_mw(mw, fmt, ...) ibdev_dbg((mw)->ibmw.device, \	57	#define rxe_dbg_mw(mw, fmt, ...) ibdev_dbg((mw)->ibmw.device, \
58	"mw#%d %s: " fmt, (mw)->elem.index, __func__, ##__VA_ARGS__)	58	"mw#%d %s: " fmt, (mw)->elem.index, __func__, ##__VA_ARGS__)
59		59
		60	/* responder states */
		61	enum resp_states {
		62	RESPST_NONE,
		63	RESPST_GET_REQ,
		64	RESPST_CHK_PSN,
		65	RESPST_CHK_OP_SEQ,
		66	RESPST_CHK_OP_VALID,
		67	RESPST_CHK_RESOURCE,
		68	RESPST_CHK_LENGTH,
		69	RESPST_CHK_RKEY,
		70	RESPST_EXECUTE,
		71	RESPST_READ_REPLY,
		72	RESPST_ATOMIC_REPLY,
		73	RESPST_ATOMIC_WRITE_REPLY,
		74	RESPST_PROCESS_FLUSH,
		75	RESPST_COMPLETE,
		76	RESPST_ACKNOWLEDGE,
		77	RESPST_CLEANUP,
		78	RESPST_DUPLICATE_REQUEST,
		79	RESPST_ERR_MALFORMED_WQE,
		80	RESPST_ERR_UNSUPPORTED_OPCODE,
		81	RESPST_ERR_MISALIGNED_ATOMIC,
		82	RESPST_ERR_PSN_OUT_OF_SEQ,
		83	RESPST_ERR_MISSING_OPCODE_FIRST,
		84	RESPST_ERR_MISSING_OPCODE_LAST_C,
		85	RESPST_ERR_MISSING_OPCODE_LAST_D1E,
		86	RESPST_ERR_TOO_MANY_RDMA_ATM_REQ,
		87	RESPST_ERR_RNR,
		88	RESPST_ERR_RKEY_VIOLATION,
		89	RESPST_ERR_INVALIDATE_RKEY,
		90	RESPST_ERR_LENGTH,
		91	RESPST_ERR_CQ_OVERFLOW,
		92	RESPST_ERROR,
		93	RESPST_RESET,
		94	RESPST_DONE,
		95	RESPST_EXIT,
		96	};
		97
60	void rxe_set_mtu(struct rxe_dev *rxe, unsigned int dev_mtu);	98	void rxe_set_mtu(struct rxe_dev *rxe, unsigned int dev_mtu);
61		99
62	int rxe_add(struct rxe_dev rxe, unsigned int mtu, const char ibdev_name);	100	int rxe_add(struct rxe_dev rxe, unsigned int mtu, const char ibdev_name);

drivers/infiniband/sw/rxe/rxe_loc.h+8−4

 int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
 		     int access, struct rxe_mr *mr);
 int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr);
-int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, int length);
+int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length);
-int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
+int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr,
-		enum rxe_mr_copy_dir dir);
+		unsigned int length, enum rxe_mr_copy_dir dir);
 int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma,
 	      void *addr, int length, enum rxe_mr_copy_dir dir);
-void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length);
+int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
+		  int sg_nents, unsigned int *sg_offset);
+int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
+			u64 compare, u64 swap_add, u64 *orig_val);
+int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value);
 struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
 			 enum rxe_mr_lookup_type type);
 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);

drivers/infiniband/sw/rxe/rxe_mr.c+338−268

@@ -26,22 +26,22 @@ u8 rxe_get_next_key(u32 last_key) · 25 unmodified lines
26		26
27	int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)	27	int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)
28	{	28	{
29
30
31	switch (mr->ibmr.type) {	29	switch (mr->ibmr.type) {
32	case IB_MR_TYPE_DMA:	30	case IB_MR_TYPE_DMA:
33	return 0;	31	return 0;
34		32
35	case IB_MR_TYPE_USER:	33	case IB_MR_TYPE_USER:
36	case IB_MR_TYPE_MEM_REG:	34	case IB_MR_TYPE_MEM_REG:
37	if (iova < mr->ibmr.iova \|\| length > mr->ibmr.length \|\|	35	if (iova < mr->ibmr.iova \|\|
38	iova > mr->ibmr.iova + mr->ibmr.length - length)	36	iova + length > mr->ibmr.iova + mr->ibmr.length) {
39	return -EFAULT;	37	rxe_dbg_mr(mr, "iova/length out of range");
		38	return -EINVAL;
		39	}
40	return 0;	40	return 0;
41		41
42	default:	42	default:
43	rxe_dbg_mr(mr, "type (%d) not supported\n", mr->ibmr.type);	43	rxe_dbg_mr(mr, "mr type not supported\n");
44	return -EFAULT;	44	return -EINVAL;
45	}	45	}
46	}	46	}
47		47
@@ -62,57 +62,31 @@ static void rxe_mr_init(int access, struct rxe_mr *mr) · 14 unmodified lines
62	mr->lkey = mr->ibmr.lkey = lkey;	62	mr->lkey = mr->ibmr.lkey = lkey;
63	mr->rkey = mr->ibmr.rkey = rkey;	63	mr->rkey = mr->ibmr.rkey = rkey;
64		64
		65	mr->access = access;
		66	mr->ibmr.page_size = PAGE_SIZE;
		67	mr->page_mask = PAGE_MASK;
		68	mr->page_shift = PAGE_SHIFT;
65	mr->state = RXE_MR_STATE_INVALID;	69	mr->state = RXE_MR_STATE_INVALID;
66	}	70	}
67		71
68	static int rxe_mr_alloc(struct rxe_mr *mr, int num_buf)
69	{
70	int i;
71	int num_map;
72	struct rxe_map **map = mr->map;
73
74	num_map = (num_buf + RXE_BUF_PER_MAP - 1) / RXE_BUF_PER_MAP;
75
76	mr->map = kmalloc_array(num_map, sizeof(*map), GFP_KERNEL);
77	if (!mr->map)
78	goto err1;
79
80	for (i = 0; i < num_map; i++) {
81	mr->map[i] = kmalloc(sizeof(**map), GFP_KERNEL);
82	if (!mr->map[i])
83	goto err2;
84	}
85
86	BUILD_BUG_ON(!is_power_of_2(RXE_BUF_PER_MAP));
87
88	mr->map_shift = ilog2(RXE_BUF_PER_MAP);
89	mr->map_mask = RXE_BUF_PER_MAP - 1;
90
91	mr->num_buf = num_buf;
92	mr->num_map = num_map;
93	mr->max_buf = num_map * RXE_BUF_PER_MAP;
94
95	return 0;
96
97	err2:
98	for (i--; i >= 0; i--)
99	kfree(mr->map[i]);
100
101	kfree(mr->map);
102	mr->map = NULL;
103	err1:
104	return -ENOMEM;
105	}
106
107	void rxe_mr_init_dma(int access, struct rxe_mr *mr)	72	void rxe_mr_init_dma(int access, struct rxe_mr *mr)
108	{	73	{
109	rxe_mr_init(access, mr);	74	rxe_mr_init(access, mr);
110		75
111	mr->access = access;
112	mr->state = RXE_MR_STATE_VALID;	76	mr->state = RXE_MR_STATE_VALID;
113	mr->ibmr.type = IB_MR_TYPE_DMA;	77	mr->ibmr.type = IB_MR_TYPE_DMA;
114	}	78	}
115		79
		80	static unsigned long rxe_mr_iova_to_index(struct rxe_mr *mr, u64 iova)
		81	{
		82	return (iova >> mr->page_shift) - (mr->ibmr.iova >> mr->page_shift);
		83	}
		84
		85	static unsigned long rxe_mr_iova_to_page_offset(struct rxe_mr *mr, u64 iova)
		86	{
		87	return iova & (mr_page_size(mr) - 1);
		88	}
		89
116	static bool is_pmem_page(struct page *pg)	90	static bool is_pmem_page(struct page *pg)
117	{	91	{
118	unsigned long paddr = page_to_phys(pg);	92	unsigned long paddr = page_to_phys(pg);
@@ -122,86 +96,98 @@ static bool is_pmem_page(struct page *pg) · 3 unmodified lines
122	IORES_DESC_PERSISTENT_MEMORY);	96	IORES_DESC_PERSISTENT_MEMORY);
123	}	97	}
124		98
		99	static int rxe_mr_fill_pages_from_sgt(struct rxe_mr mr, struct sg_table sgt)
		100	{
		101	XA_STATE(xas, &mr->page_list, 0);
		102	struct sg_page_iter sg_iter;
		103	struct page *page;
		104	bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT);
		105
		106	__sg_page_iter_start(&sg_iter, sgt->sgl, sgt->orig_nents, 0);
		107	if (!__sg_page_iter_next(&sg_iter))
		108	return 0;
		109
		110	do {
		111	xas_lock(&xas);
		112	while (true) {
		113	page = sg_page_iter_page(&sg_iter);
		114
		115	if (persistent && !is_pmem_page(page)) {
		116	rxe_dbg_mr(mr, "Page can't be persistent\n");
		117	xas_set_err(&xas, -EINVAL);
		118	break;
		119	}
		120
		121	xas_store(&xas, page);
		122	if (xas_error(&xas))
		123	break;
		124	xas_next(&xas);
		125	if (!__sg_page_iter_next(&sg_iter))
		126	break;
		127	}
		128	xas_unlock(&xas);
		129	} while (xas_nomem(&xas, GFP_KERNEL));
		130
		131	return xas_error(&xas);
		132	}
		133
125	int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,	134	int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
126	int access, struct rxe_mr *mr)	135	int access, struct rxe_mr *mr)
127	{	136	{
128	struct rxe_map **map;	137	struct ib_umem *umem;
129	struct rxe_phys_buf *buf = NULL;
130	struct ib_umem *umem;
131	struct sg_page_iter sg_iter;
132	int num_buf;
133	void *vaddr;
134	int err;	138	int err;
135		139
		140	rxe_mr_init(access, mr);
		141
		142	xa_init(&mr->page_list);
		143
136	umem = ib_umem_get(&rxe->ib_dev, start, length, access);	144	umem = ib_umem_get(&rxe->ib_dev, start, length, access);
137	if (IS_ERR(umem)) {	145	if (IS_ERR(umem)) {
138	rxe_dbg_mr(mr, "Unable to pin memory region err = %d\n",	146	rxe_dbg_mr(mr, "Unable to pin memory region err = %d\n",
139	(int)PTR_ERR(umem));	147	(int)PTR_ERR(umem));
140	err = PTR_ERR(umem);	148	return PTR_ERR(umem);
141	goto err_out;
142	}	149	}
143		150
144	num_buf = ib_umem_num_pages(umem);	151	err = rxe_mr_fill_pages_from_sgt(mr, &umem->sgt_append.sgt);
145
146	rxe_mr_init(access, mr);
147
148	err = rxe_mr_alloc(mr, num_buf);
149	if (err) {	152	if (err) {
150	rxe_dbg_mr(mr, "Unable to allocate memory for map\n");	153	ib_umem_release(umem);
151	goto err_release_umem;	154	return err;
152	}	155	}
153		156
154	mr->page_shift = PAGE_SHIFT;	157	mr->umem = umem;
155	mr->page_mask = PAGE_SIZE - 1;	158	mr->ibmr.type = IB_MR_TYPE_USER;
156		159	mr->state = RXE_MR_STATE_VALID;
157	num_buf = 0;
158	map = mr->map;
159	if (length > 0) {
160	bool persistent_access = access & IB_ACCESS_FLUSH_PERSISTENT;
161
162	buf = map[0]->buf;
163	for_each_sgtable_page (&umem->sgt_append.sgt, &sg_iter, 0) {
164	struct page *pg = sg_page_iter_page(&sg_iter);
165		160
166	if (persistent_access && !is_pmem_page(pg)) {	161	return 0;
167	rxe_dbg_mr(mr, "Unable to register persistent access to non-pmem device\n");	162	}
168	err = -EINVAL;
169	goto err_release_umem;
170	}
171		163
172	if (num_buf >= RXE_BUF_PER_MAP) {	164	static int rxe_mr_alloc(struct rxe_mr *mr, int num_buf)
173	map++;	165	{
174	buf = map[0]->buf;	166	XA_STATE(xas, &mr->page_list, 0);
175	num_buf = 0;	167	int i = 0;
176	}	168	int err;
177		169
178	vaddr = page_address(pg);	170	xa_init(&mr->page_list);
179	if (!vaddr) {
180	rxe_dbg_mr(mr, "Unable to get virtual address\n");
181	err = -ENOMEM;
182	goto err_release_umem;
183	}
184	buf->addr = (uintptr_t)vaddr;
185	buf->size = PAGE_SIZE;
186	num_buf++;
187	buf++;
188		171
		172	do {
		173	xas_lock(&xas);
		174	while (i != num_buf) {
		175	xas_store(&xas, XA_ZERO_ENTRY);
		176	if (xas_error(&xas))
		177	break;
		178	xas_next(&xas);
		179	i++;
189	}	180	}
190	}	181	xas_unlock(&xas);
		182	} while (xas_nomem(&xas, GFP_KERNEL));
191		183
192	mr->umem = umem;	184	err = xas_error(&xas);
193	mr->access = access;	185	if (err)
194	mr->offset = ib_umem_offset(umem);	186	return err;
195	mr->state = RXE_MR_STATE_VALID;
196	mr->ibmr.type = IB_MR_TYPE_USER;
197	mr->ibmr.page_size = PAGE_SIZE;
198		187
199	return 0;	188	mr->num_buf = num_buf;
200		189
201	err_release_umem:	190	return 0;
202	ib_umem_release(umem);
203	err_out:
204	return err;
205	}	191	}
206		192
207	int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr)	193	int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr)
@@ -215,7 +201,6 @@ int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr) · 7 unmodified lines
215	if (err)	201	if (err)
216	goto err1;	202	goto err1;
217		203
218	mr->max_buf = max_pages;
219	mr->state = RXE_MR_STATE_FREE;	204	mr->state = RXE_MR_STATE_FREE;
220	mr->ibmr.type = IB_MR_TYPE_MEM_REG;	205	mr->ibmr.type = IB_MR_TYPE_MEM_REG;
221		206
@@ -225,187 +210,125 @@ err1: · 3 unmodified lines
225	return err;	210	return err;
226	}	211	}
227		212
228	static void lookup_iova(struct rxe_mr mr, u64 iova, int m_out, int *n_out,	213	static int rxe_set_page(struct ib_mr *ibmr, u64 iova)
229	size_t *offset_out)
230	{	214	{
231	size_t offset = iova - mr->ibmr.iova + mr->offset;	215	struct rxe_mr *mr = to_rmr(ibmr);
232	int map_index;	216	struct page *page = virt_to_page(iova & mr->page_mask);
233	int buf_index;	217	bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT);
234	u64 length;	218	int err;
235
236	if (likely(mr->page_shift)) {
237	*offset_out = offset & mr->page_mask;
238	offset >>= mr->page_shift;
239	*n_out = offset & mr->map_mask;
240	*m_out = offset >> mr->map_shift;
241	} else {
242	map_index = 0;
243	buf_index = 0;
244		219
245	length = mr->map[map_index]->buf[buf_index].size;	220	if (persistent && !is_pmem_page(page)) {
		221	rxe_dbg_mr(mr, "Page cannot be persistent\n");
		222	return -EINVAL;
		223	}
246		224
247	while (offset >= length) {	225	if (unlikely(mr->nbuf == mr->num_buf))
248	offset -= length;	226	return -ENOMEM;
249	buf_index++;
250		227
251	if (buf_index == RXE_BUF_PER_MAP) {	228	err = xa_err(xa_store(&mr->page_list, mr->nbuf, page, GFP_KERNEL));
252	map_index++;	229	if (err)
253	buf_index = 0;	230	return err;
254	}
255	length = mr->map[map_index]->buf[buf_index].size;
256	}
257		231
258	*m_out = map_index;	232	mr->nbuf++;
259	*n_out = buf_index;	233	return 0;
260	*offset_out = offset;
261	}
262	}	234	}
263		235
264	void iova_to_vaddr(struct rxe_mr mr, u64 iova, int length)	236	int rxe_map_mr_sg(struct ib_mr ibmr, struct scatterlist sgl,
		237	int sg_nents, unsigned int *sg_offset)
265	{	238	{
266	size_t offset;	239	struct rxe_mr *mr = to_rmr(ibmr);
267	int m, n;	240	unsigned int page_size = mr_page_size(mr);
268	void *addr;
269
270	if (mr->state != RXE_MR_STATE_VALID) {
271	rxe_dbg_mr(mr, "Not in valid state\n");
272	addr = NULL;
273	goto out;
274	}
275
276	if (!mr->map) {
277	addr = (void *)(uintptr_t)iova;
278	goto out;
279	}
280
281	if (mr_check_range(mr, iova, length)) {
282	rxe_dbg_mr(mr, "Range violation\n");
283	addr = NULL;
284	goto out;
285	}
286
287	lookup_iova(mr, iova, &m, &n, &offset);
288
289	if (offset + length > mr->map[m]->buf[n].size) {
290	rxe_dbg_mr(mr, "Crosses page boundary\n");
291	addr = NULL;
292	goto out;
293	}
294		241
295	addr = (void *)(uintptr_t)mr->map[m]->buf[n].addr + offset;	242	mr->nbuf = 0;
		243	mr->page_shift = ilog2(page_size);
		244	mr->page_mask = ~((u64)page_size - 1);
		245	mr->page_offset = mr->ibmr.iova & (page_size - 1);
296		246
297	out:	247	return ib_sg_to_pages(ibmr, sgl, sg_nents, sg_offset, rxe_set_page);
298	return addr;
299	}	248	}
300		249
301	int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, int length)	250	static int rxe_mr_copy_xarray(struct rxe_mr mr, u64 iova, void addr,
		251	unsigned int length, enum rxe_mr_copy_dir dir)
302	{	252	{
303	size_t offset;	253	unsigned int page_offset = rxe_mr_iova_to_page_offset(mr, iova);
304		254	unsigned long index = rxe_mr_iova_to_index(mr, iova);
305	if (length == 0)	255	unsigned int bytes;
306	return 0;	256	struct page *page;
307		257	void *va;
308	if (mr->ibmr.type == IB_MR_TYPE_DMA)
309	return -EFAULT;
310		258
311	offset = (iova - mr->ibmr.iova + mr->offset) & mr->page_mask;	259	while (length) {
312	while (length > 0) {	260	page = xa_load(&mr->page_list, index);
313	u8 *va;	261	if (!page)
314	int bytes;
315
316	bytes = mr->ibmr.page_size - offset;
317	if (bytes > length)
318	bytes = length;
319
320	va = iova_to_vaddr(mr, iova, length);
321	if (!va)
322	return -EFAULT;	262	return -EFAULT;
323		263
324	arch_wb_cache_pmem(va, bytes);	264	bytes = min_t(unsigned int, length,
325		265	mr_page_size(mr) - page_offset);
		266	va = kmap_local_page(page);
		267	if (dir == RXE_FROM_MR_OBJ)
		268	memcpy(addr, va + page_offset, bytes);
		269	else
		270	memcpy(va + page_offset, addr, bytes);
		271	kunmap_local(va);
		272
		273	page_offset = 0;
		274	addr += bytes;
326	length -= bytes;	275	length -= bytes;
327	iova += bytes;	276	index++;
328	offset = 0;
329	}	277	}
330		278
331	return 0;	279	return 0;
332	}	280	}
333		281
334	/* copy data from a range (vaddr, vaddr+length-1) to or from	282	static void rxe_mr_copy_dma(struct rxe_mr mr, u64 iova, void addr,
335	* a mr object starting at iova.	283	unsigned int length, enum rxe_mr_copy_dir dir)
336	*/
337	int rxe_mr_copy(struct rxe_mr mr, u64 iova, void addr, int length,
338	enum rxe_mr_copy_dir dir)
339	{	284	{
340	int err;	285	unsigned int page_offset = iova & (PAGE_SIZE - 1);
341	int bytes;	286	unsigned int bytes;
342	u8 *va;	287	struct page *page;
343	struct rxe_map **map;	288	u8 *va;
344	struct rxe_phys_buf *buf;
345	int m;
346	int i;
347	size_t offset;
348		289
349	if (length == 0)	290	while (length) {
350	return 0;	291	page = virt_to_page(iova & mr->page_mask);
351		292	bytes = min_t(unsigned int, length,
352	if (mr->ibmr.type == IB_MR_TYPE_DMA) {	293	PAGE_SIZE - page_offset);
353	u8 src, dest;	294	va = kmap_local_page(page);
		295
		296	if (dir == RXE_TO_MR_OBJ)
		297	memcpy(va + page_offset, addr, bytes);
		298	else
		299	memcpy(addr, va + page_offset, bytes);
		300
		301	kunmap_local(va);
		302	page_offset = 0;
		303	iova += bytes;
		304	addr += bytes;
		305	length -= bytes;
		306	}
		307	}
354		308
355	src = (dir == RXE_TO_MR_OBJ) ? addr : ((void *)(uintptr_t)iova);	309	int rxe_mr_copy(struct rxe_mr mr, u64 iova, void addr,
		310	unsigned int length, enum rxe_mr_copy_dir dir)
		311	{
		312	int err;
356		313
357	dest = (dir == RXE_TO_MR_OBJ) ? ((void *)(uintptr_t)iova) : addr;	314	if (length == 0)
		315	return 0;
358		316
359	memcpy(dest, src, length);	317	if (WARN_ON(!mr))
		318	return -EINVAL;
360		319
		320	if (mr->ibmr.type == IB_MR_TYPE_DMA) {
		321	rxe_mr_copy_dma(mr, iova, addr, length, dir);
361	return 0;	322	return 0;
362	}	323	}
363		324
364	WARN_ON_ONCE(!mr->map);
365
366	err = mr_check_range(mr, iova, length);	325	err = mr_check_range(mr, iova, length);
367	if (err) {	326	if (unlikely(err)) {
368	err = -EFAULT;	327	rxe_dbg_mr(mr, "iova out of range");
369	goto err1;	328	return err;
370	}
371
372	lookup_iova(mr, iova, &m, &i, &offset);
373
374	map = mr->map + m;
375	buf = map[0]->buf + i;
376
377	while (length > 0) {
378	u8 src, dest;
379
380	va = (u8 *)(uintptr_t)buf->addr + offset;
381	src = (dir == RXE_TO_MR_OBJ) ? addr : va;
382	dest = (dir == RXE_TO_MR_OBJ) ? va : addr;
383
384	bytes = buf->size - offset;
385
386	if (bytes > length)
387	bytes = length;
388
389	memcpy(dest, src, bytes);
390
391	length -= bytes;
392	addr += bytes;
393
394	offset = 0;
395	buf++;
396	i++;
397
398	if (i == RXE_BUF_PER_MAP) {
399	i = 0;
400	map++;
401	buf = map[0]->buf;
402	}
403	}	329	}
404		330
405	return 0;	331	return rxe_mr_copy_xarray(mr, iova, addr, length, dir);
406
407	err1:
408	return err;
409	}	332	}
410		333
411	/* copy data in or out of a wqe, i.e. sg list	334	/* copy data in or out of a wqe, i.e. sg list
@@ -477,7 +400,6 @@ int copy_data( · 65 unmodified lines
477		400
478	if (bytes > 0) {	401	if (bytes > 0) {
479	iova = sge->addr + offset;	402	iova = sge->addr + offset;
480
481	err = rxe_mr_copy(mr, iova, addr, bytes, dir);	403	err = rxe_mr_copy(mr, iova, addr, bytes, dir);
482	if (err)	404	if (err)
483	goto err2;	405	goto err2;
@@ -504,6 +426,165 @@ err1: · 20 unmodified lines
504	return err;	426	return err;
505	}	427	}
506		428
		429	int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length)
		430	{
		431	unsigned int page_offset;
		432	unsigned long index;
		433	struct page *page;
		434	unsigned int bytes;
		435	int err;
		436	u8 *va;
		437
		438	/* mr must be valid even if length is zero */
		439	if (WARN_ON(!mr))
		440	return -EINVAL;
		441
		442	if (length == 0)
		443	return 0;
		444
		445	if (mr->ibmr.type == IB_MR_TYPE_DMA)
		446	return -EFAULT;
		447
		448	err = mr_check_range(mr, iova, length);
		449	if (err)
		450	return err;
		451
		452	while (length > 0) {
		453	index = rxe_mr_iova_to_index(mr, iova);
		454	page = xa_load(&mr->page_list, index);
		455	page_offset = rxe_mr_iova_to_page_offset(mr, iova);
		456	if (!page)
		457	return -EFAULT;
		458	bytes = min_t(unsigned int, length,
		459	mr_page_size(mr) - page_offset);
		460
		461	va = kmap_local_page(page);
		462	arch_wb_cache_pmem(va + page_offset, bytes);
		463	kunmap_local(va);
		464
		465	length -= bytes;
		466	iova += bytes;
		467	page_offset = 0;
		468	}
		469
		470	return 0;
		471	}
		472
		473	/* Guarantee atomicity of atomic operations at the machine level. */
		474	static DEFINE_SPINLOCK(atomic_ops_lock);
		475
		476	int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
		477	u64 compare, u64 swap_add, u64 *orig_val)
		478	{
		479	unsigned int page_offset;
		480	struct page *page;
		481	u64 value;
		482	u64 *va;
		483
		484	if (unlikely(mr->state != RXE_MR_STATE_VALID)) {
		485	rxe_dbg_mr(mr, "mr not in valid state");
		486	return RESPST_ERR_RKEY_VIOLATION;
		487	}
		488
		489	if (mr->ibmr.type == IB_MR_TYPE_DMA) {
		490	page_offset = iova & (PAGE_SIZE - 1);
		491	page = virt_to_page(iova & PAGE_MASK);
		492	} else {
		493	unsigned long index;
		494	int err;
		495
		496	err = mr_check_range(mr, iova, sizeof(value));
		497	if (err) {
		498	rxe_dbg_mr(mr, "iova out of range");
		499	return RESPST_ERR_RKEY_VIOLATION;
		500	}
		501	page_offset = rxe_mr_iova_to_page_offset(mr, iova);
		502	index = rxe_mr_iova_to_index(mr, iova);
		503	page = xa_load(&mr->page_list, index);
		504	if (!page)
		505	return RESPST_ERR_RKEY_VIOLATION;
		506	}
		507
		508	if (unlikely(page_offset & 0x7)) {
		509	rxe_dbg_mr(mr, "iova not aligned");
		510	return RESPST_ERR_MISALIGNED_ATOMIC;
		511	}
		512
		513	va = kmap_local_page(page);
		514
		515	spin_lock_bh(&atomic_ops_lock);
		516	value = *orig_val = va[page_offset >> 3];
		517
		518	if (opcode == IB_OPCODE_RC_COMPARE_SWAP) {
		519	if (value == compare)
		520	va[page_offset >> 3] = swap_add;
		521	} else {
		522	value += swap_add;
		523	va[page_offset >> 3] = value;
		524	}
		525	spin_unlock_bh(&atomic_ops_lock);
		526
		527	kunmap_local(va);
		528
		529	return 0;
		530	}
		531
		532	#if defined CONFIG_64BIT
		533	/* only implemented or called for 64 bit architectures */
		534	int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)
		535	{
		536	unsigned int page_offset;
		537	struct page *page;
		538	u64 *va;
		539
		540	/* See IBA oA19-28 */
		541	if (unlikely(mr->state != RXE_MR_STATE_VALID)) {
		542	rxe_dbg_mr(mr, "mr not in valid state");
		543	return RESPST_ERR_RKEY_VIOLATION;
		544	}
		545
		546	if (mr->ibmr.type == IB_MR_TYPE_DMA) {
		547	page_offset = iova & (PAGE_SIZE - 1);
		548	page = virt_to_page(iova & PAGE_MASK);
		549	} else {
		550	unsigned long index;
		551	int err;
		552
		553	/* See IBA oA19-28 */
		554	err = mr_check_range(mr, iova, sizeof(value));
		555	if (unlikely(err)) {
		556	rxe_dbg_mr(mr, "iova out of range");
		557	return RESPST_ERR_RKEY_VIOLATION;
		558	}
		559	page_offset = rxe_mr_iova_to_page_offset(mr, iova);
		560	index = rxe_mr_iova_to_index(mr, iova);
		561	page = xa_load(&mr->page_list, index);
		562	if (!page)
		563	return RESPST_ERR_RKEY_VIOLATION;
		564	}
		565
		566	/* See IBA A19.4.2 */
		567	if (unlikely(page_offset & 0x7)) {
		568	rxe_dbg_mr(mr, "misaligned address");
		569	return RESPST_ERR_MISALIGNED_ATOMIC;
		570	}
		571
		572	va = kmap_local_page(page);
		573
		574	/* Do atomic write after all prior operations have completed */
		575	smp_store_release(&va[page_offset >> 3], value);
		576
		577	kunmap_local(va);
		578
		579	return 0;
		580	}
		581	#else
		582	int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)
		583	{
		584	return RESPST_ERR_UNSUPPORTED_OPCODE;
		585	}
		586	#endif
		587
507	int advance_dma_data(struct rxe_dma_info *dma, unsigned int length)	588	int advance_dma_data(struct rxe_dma_info *dma, unsigned int length)
508	{	589	{
509	struct rxe_sge *sge = &dma->sge[dma->cur_sge];	590	struct rxe_sge *sge = &dma->sge[dma->cur_sge];
@@ -537,12 +618,6 @@ int advance_dma_data(struct rxe_dma_info *dma, unsigned int length) · 27 unmodified lines
537	return 0;	618	return 0;
538	}	619	}
539		620
540	/* (1) find the mr corresponding to lkey/rkey
541	* depending on lookup_type
542	* (2) verify that the (qp) pd matches the mr pd
543	* (3) verify that the mr can support the requested access
544	* (4) verify that mr state is valid
545	*/
546	struct rxe_mr lookup_mr(struct rxe_pd pd, int access, u32 key,	621	struct rxe_mr lookup_mr(struct rxe_pd pd, int access, u32 key,
547	enum rxe_mr_lookup_type type)	622	enum rxe_mr_lookup_type type)
548	{	623	{
@@ -656,22 +731,17 @@ int rxe_dereg_mr(struct ib_mr ibmr, struct ib_udata udata) · 107 unmodified lines
656	return -EINVAL;	731	return -EINVAL;
657		732
658	rxe_cleanup(mr);	733	rxe_cleanup(mr);
659		734	kfree_rcu(mr);
660	return 0;	735	return 0;
661	}	736	}
662		737
663	void rxe_mr_cleanup(struct rxe_pool_elem *elem)	738	void rxe_mr_cleanup(struct rxe_pool_elem *elem)
664	{	739	{
665	struct rxe_mr mr = container_of(elem, typeof(mr), elem);	740	struct rxe_mr mr = container_of(elem, typeof(mr), elem);
666	int i;
667		741
668	rxe_put(mr_pd(mr));	742	rxe_put(mr_pd(mr));
669	ib_umem_release(mr->umem);	743	ib_umem_release(mr->umem);
670		744
671	if (mr->map) {	745	if (mr->ibmr.type != IB_MR_TYPE_DMA)
672	for (i = 0; i < mr->num_map; i++)	746	xa_destroy(&mr->page_list);
673	kfree(mr->map[i]);
674
675	kfree(mr->map);
676	}
677	}	747	}

drivers/infiniband/sw/rxe/rxe_pool.c+0−46

@@ -116,55 +116,12 @@ void rxe_pool_cleanup(struct rxe_pool *pool) · 115 unmodified lines
116	WARN_ON(!xa_empty(&pool->xa));	116	WARN_ON(!xa_empty(&pool->xa));
117	}	117	}
118		118
119	void rxe_alloc(struct rxe_pool pool)
120	{
121	struct rxe_pool_elem *elem;
122	void *obj;
123	int err;
124
125	if (WARN_ON(!(pool->type == RXE_TYPE_MR)))
126	return NULL;
127
128	if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
129	goto err_cnt;
130
131	obj = kzalloc(pool->elem_size, GFP_KERNEL);
132	if (!obj)
133	goto err_cnt;
134
135	elem = (struct rxe_pool_elem )((u8 )obj + pool->elem_offset);
136
137	elem->pool = pool;
138	elem->obj = obj;
139	kref_init(&elem->ref_cnt);
140	init_completion(&elem->complete);
141
142	/* allocate index in array but leave pointer as NULL so it
143	* can't be looked up until rxe_finalize() is called
144	*/
145	err = xa_alloc_cyclic(&pool->xa, &elem->index, NULL, pool->limit,
146	&pool->next, GFP_KERNEL);
147	if (err < 0)
148	goto err_free;
149
150	return obj;
151
152	err_free:
153	kfree(obj);
154	err_cnt:
155	atomic_dec(&pool->num_elem);
156	return NULL;
157	}
158
159	int __rxe_add_to_pool(struct rxe_pool pool, struct rxe_pool_elem elem,	119	int __rxe_add_to_pool(struct rxe_pool pool, struct rxe_pool_elem elem,
160	bool sleepable)	120	bool sleepable)
161	{	121	{
162	int err;	122	int err;
163	gfp_t gfp_flags;	123	gfp_t gfp_flags;
164		124
165	if (WARN_ON(pool->type == RXE_TYPE_MR))
166	return -EINVAL;
167
168	if (atomic_inc_return(&pool->num_elem) > pool->max_elem)	125	if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
169	goto err_cnt;	126	goto err_cnt;
170		127
@@ -275,9 +232,6 @@ int __rxe_cleanup(struct rxe_pool_elem *elem, bool sleepable) · 104 unmodified lines
275	if (pool->cleanup)	232	if (pool->cleanup)
276	pool->cleanup(elem);	233	pool->cleanup(elem);
277		234
278	if (pool->type == RXE_TYPE_MR)
279	kfree_rcu(elem->obj);
280
281	atomic_dec(&pool->num_elem);	235	atomic_dec(&pool->num_elem);
282		236
283	return err;	237	return err;

drivers/infiniband/sw/rxe/rxe_pool.h+0−3

@@ -54,9 +54,6 @@ void rxe_pool_init(struct rxe_dev rxe, struct rxe_pool pool, · 53 unmodified lines
54	/* free resources from object pool */	54	/* free resources from object pool */
55	void rxe_pool_cleanup(struct rxe_pool *pool);	55	void rxe_pool_cleanup(struct rxe_pool *pool);
56		56
57	/* allocate an object from pool */
58	void rxe_alloc(struct rxe_pool pool);
59
60	/* connect already allocated object to pool */	57	/* connect already allocated object to pool */
61	int __rxe_add_to_pool(struct rxe_pool pool, struct rxe_pool_elem elem,	58	int __rxe_add_to_pool(struct rxe_pool pool, struct rxe_pool_elem elem,
62	bool sleepable);	59	bool sleepable);

drivers/infiniband/sw/rxe/rxe_queue.h+66−42

 /**
  * enum queue_type - type of queue
  * @QUEUE_TYPE_TO_CLIENT:	Queue is written by rxe driver and
- *				read by client. Used by rxe driver only.
+ *				read by client which may be a user space
+ *				application or a kernel ulp.
+ *				Used by rxe internals only.
  * @QUEUE_TYPE_FROM_CLIENT:	Queue is written by client and
- *				read by rxe driver. Used by rxe driver only.
+ *				read by rxe driver.
- * @QUEUE_TYPE_TO_DRIVER:	Queue is written by client and
+ *				Used by rxe internals only.
- *				read by rxe driver. Used by kernel client only.
+ * @QUEUE_TYPE_FROM_ULP:	Queue is written by kernel ulp and
- * @QUEUE_TYPE_FROM_DRIVER:	Queue is written by rxe driver and
+ *				read by rxe driver.
- *				read by client. Used by kernel client only.
+ *				Used by kernel verbs APIs only on
+ *				behalf of ulps.
+ * @QUEUE_TYPE_TO_ULP:		Queue is written by rxe driver and
+ *				read by kernel ulp.
+ *				Used by kernel verbs APIs only on
+ *				behalf of ulps.
  */
 enum queue_type {
 	QUEUE_TYPE_TO_CLIENT,
 	QUEUE_TYPE_FROM_CLIENT,
-	QUEUE_TYPE_TO_DRIVER,
+	QUEUE_TYPE_FROM_ULP,
-	QUEUE_TYPE_FROM_DRIVER,
+	QUEUE_TYPE_TO_ULP,
 };
 struct rxe_queue_buf;
 	u32			index_mask;
 	enum queue_type		type;
 	/* private copy of index for shared queues between
-	 * kernel space and user space. Kernel reads and writes
+	 * driver and clients. Driver reads and writes
 	 * this copy and then replicates to rxe_queue_buf
-	 * for read access by user space.
+	 * for read access by clients.
 	 */
 	u32			index;
 };
 	switch (type) {
 	case QUEUE_TYPE_FROM_CLIENT:
-		/* protect user index */
+		/* used by rxe, client owns the index */
 		prod = smp_load_acquire(&q->buf->producer_index);
 		break;
 	case QUEUE_TYPE_TO_CLIENT:
+		/* used by rxe which owns the index */
 		prod = q->index;
 		break;
-	case QUEUE_TYPE_FROM_DRIVER:
+	case QUEUE_TYPE_FROM_ULP:
-		/* protect driver index */
+		/* used by ulp which owns the index */
-		prod = smp_load_acquire(&q->buf->producer_index);
-		break;
-	case QUEUE_TYPE_TO_DRIVER:
 		prod = q->buf->producer_index;
 		break;
+	case QUEUE_TYPE_TO_ULP:
+		/* used by ulp, rxe owns the index */
+		prod = smp_load_acquire(&q->buf->producer_index);
+		break;
 	}
 	return prod;
 	switch (type) {
 	case QUEUE_TYPE_FROM_CLIENT:
+		/* used by rxe which owns the index */
 		cons = q->index;
 		break;
 	case QUEUE_TYPE_TO_CLIENT:
-		/* protect user index */
+		/* used by rxe, client owns the index */
 		cons = smp_load_acquire(&q->buf->consumer_index);
 		break;
-	case QUEUE_TYPE_FROM_DRIVER:
+	case QUEUE_TYPE_FROM_ULP:
-		cons = q->buf->consumer_index;
+		/* used by ulp, rxe owns the index */
-		break;
-	case QUEUE_TYPE_TO_DRIVER:
-		/* protect driver index */
 		cons = smp_load_acquire(&q->buf->consumer_index);
 		break;
+	case QUEUE_TYPE_TO_ULP:
+		/* used by ulp which owns the index */
+		cons = q->buf->consumer_index;
+		break;
 	}
 	return cons;
 	switch (type) {
 	case QUEUE_TYPE_FROM_CLIENT:
-		pr_warn("%s: attempt to advance client index\n",
+		/* used by rxe, client owns the index */
-			__func__);
+		if (WARN_ON(1))
+			pr_warn("%s: attempt to advance client index\n",
+				__func__);
 		break;
 	case QUEUE_TYPE_TO_CLIENT:
+		/* used by rxe which owns the index */
 		prod = q->index;
 		prod = (prod + 1) & q->index_mask;
 		q->index = prod;
-		/* protect user index */
+		/* release so client can read it safely */
 		smp_store_release(&q->buf->producer_index, prod);
 		break;
-	case QUEUE_TYPE_FROM_DRIVER:
+	case QUEUE_TYPE_FROM_ULP:
-		pr_warn("%s: attempt to advance driver index\n",
+		/* used by ulp which owns the index */
-			__func__);
-		break;
-	case QUEUE_TYPE_TO_DRIVER:
 		prod = q->buf->producer_index;
 		prod = (prod + 1) & q->index_mask;
-		q->buf->producer_index = prod;
+		/* release so rxe can read it safely */
+		smp_store_release(&q->buf->producer_index, prod);
+		break;
+	case QUEUE_TYPE_TO_ULP:
+		/* used by ulp, rxe owns the index */
+		if (WARN_ON(1))
+			pr_warn("%s: attempt to advance driver index\n",
+				__func__);
 		break;
 	}
 }
 	switch (type) {
 	case QUEUE_TYPE_FROM_CLIENT:
-		cons = q->index;
+		/* used by rxe which owns the index */
-		cons = (cons + 1) & q->index_mask;
+		cons = (q->index + 1) & q->index_mask;
 		q->index = cons;
-		/* protect user index */
+		/* release so client can read it safely */
 		smp_store_release(&q->buf->consumer_index, cons);
 		break;
 	case QUEUE_TYPE_TO_CLIENT:
-		pr_warn("%s: attempt to advance client index\n",
+		/* used by rxe, client owns the index */
-			__func__);
+		if (WARN_ON(1))
+			pr_warn("%s: attempt to advance client index\n",
+				__func__);
+		break;
+	case QUEUE_TYPE_FROM_ULP:
+		/* used by ulp, rxe owns the index */
+		if (WARN_ON(1))
+			pr_warn("%s: attempt to advance driver index\n",
+				__func__);
 		break;
-	case QUEUE_TYPE_FROM_DRIVER:
+	case QUEUE_TYPE_TO_ULP:
+		/* used by ulp which owns the index */
 		cons = q->buf->consumer_index;
 		cons = (cons + 1) & q->index_mask;
-		q->buf->consumer_index = cons;
+		/* release so rxe can read it safely */
-		break;
+		smp_store_release(&q->buf->consumer_index, cons);
-	case QUEUE_TYPE_TO_DRIVER:
-		pr_warn("%s: attempt to advance driver index\n",
-			__func__);
 		break;
 	}
 }

drivers/infiniband/sw/rxe/rxe_resp.c+75−127

@@ -10,43 +10,6 @@ · 9 unmodified lines
10	#include "rxe_loc.h"	10	#include "rxe_loc.h"
11	#include "rxe_queue.h"	11	#include "rxe_queue.h"
12		12
13	enum resp_states {
14	RESPST_NONE,
15	RESPST_GET_REQ,
16	RESPST_CHK_PSN,
17	RESPST_CHK_OP_SEQ,
18	RESPST_CHK_OP_VALID,
19	RESPST_CHK_RESOURCE,
20	RESPST_CHK_LENGTH,
21	RESPST_CHK_RKEY,
22	RESPST_EXECUTE,
23	RESPST_READ_REPLY,
24	RESPST_ATOMIC_REPLY,
25	RESPST_ATOMIC_WRITE_REPLY,
26	RESPST_PROCESS_FLUSH,
27	RESPST_COMPLETE,
28	RESPST_ACKNOWLEDGE,
29	RESPST_CLEANUP,
30	RESPST_DUPLICATE_REQUEST,
31	RESPST_ERR_MALFORMED_WQE,
32	RESPST_ERR_UNSUPPORTED_OPCODE,
33	RESPST_ERR_MISALIGNED_ATOMIC,
34	RESPST_ERR_PSN_OUT_OF_SEQ,
35	RESPST_ERR_MISSING_OPCODE_FIRST,
36	RESPST_ERR_MISSING_OPCODE_LAST_C,
37	RESPST_ERR_MISSING_OPCODE_LAST_D1E,
38	RESPST_ERR_TOO_MANY_RDMA_ATM_REQ,
39	RESPST_ERR_RNR,
40	RESPST_ERR_RKEY_VIOLATION,
41	RESPST_ERR_INVALIDATE_RKEY,
42	RESPST_ERR_LENGTH,
43	RESPST_ERR_CQ_OVERFLOW,
44	RESPST_ERROR,
45	RESPST_RESET,
46	RESPST_DONE,
47	RESPST_EXIT,
48	};
49
50	static char *resp_state_name[] = {	13	static char *resp_state_name[] = {
51	[RESPST_NONE] = "NONE",	14	[RESPST_NONE] = "NONE",
52	[RESPST_GET_REQ] = "GET_REQ",	15	[RESPST_GET_REQ] = "GET_REQ",
@@ -457,13 +420,23 @@ static enum resp_states rxe_resp_check_length(struct rxe_qp *qp, · 404 unmodified lines
457	return RESPST_CHK_RKEY;	420	return RESPST_CHK_RKEY;
458	}	421	}
459		422
		423	/* if the reth length field is zero we can assume nothing
		424	* about the rkey value and should not validate or use it.
		425	* Instead set qp->resp.rkey to 0 which is an invalid rkey
		426	* value since the minimum index part is 1.
		427	*/
460	static void qp_resp_from_reth(struct rxe_qp qp, struct rxe_pkt_info pkt)	428	static void qp_resp_from_reth(struct rxe_qp qp, struct rxe_pkt_info pkt)
461	{	429	{
		430	unsigned int length = reth_len(pkt);
		431
462	qp->resp.va = reth_va(pkt);	432	qp->resp.va = reth_va(pkt);
463	qp->resp.offset = 0;	433	qp->resp.offset = 0;
464	qp->resp.rkey = reth_rkey(pkt);	434	qp->resp.resid = length;
465	qp->resp.resid = reth_len(pkt);	435	qp->resp.length = length;
466	qp->resp.length = reth_len(pkt);	436	if (pkt->mask & RXE_READ_OR_WRITE_MASK && length == 0)
		437	qp->resp.rkey = 0;
		438	else
		439	qp->resp.rkey = reth_rkey(pkt);
467	}	440	}
468		441
469	static void qp_resp_from_atmeth(struct rxe_qp qp, struct rxe_pkt_info pkt)	442	static void qp_resp_from_atmeth(struct rxe_qp qp, struct rxe_pkt_info pkt)
@@ -474,6 +447,10 @@ static void qp_resp_from_atmeth(struct rxe_qp qp, struct rxe_pkt_info pkt) · 4 unmodified lines
474	qp->resp.resid = sizeof(u64);	447	qp->resp.resid = sizeof(u64);
475	}	448	}
476		449
		450	/* resolve the packet rkey to qp->resp.mr or set qp->resp.mr to NULL
		451	* if an invalid rkey is received or the rdma length is zero. For middle
		452	* or last packets use the stored value of mr.
		453	*/
477	static enum resp_states check_rkey(struct rxe_qp *qp,	454	static enum resp_states check_rkey(struct rxe_qp *qp,
478	struct rxe_pkt_info *pkt)	455	struct rxe_pkt_info *pkt)
479	{	456	{
@@ -510,10 +487,12 @@ static enum resp_states check_rkey(struct rxe_qp *qp, · 30 unmodified lines
510	return RESPST_EXECUTE;	487	return RESPST_EXECUTE;
511	}	488	}
512		489
513	/* A zero-byte op is not required to set an addr or rkey. See C9-88 */	490	/* A zero-byte read or write op is not required to
		491	* set an addr or rkey. See C9-88
		492	*/
514	if ((pkt->mask & RXE_READ_OR_WRITE_MASK) &&	493	if ((pkt->mask & RXE_READ_OR_WRITE_MASK) &&
515	(pkt->mask & RXE_RETH_MASK) &&	494	(pkt->mask & RXE_RETH_MASK) && reth_len(pkt) == 0) {
516	reth_len(pkt) == 0) {	495	qp->resp.mr = NULL;
517	return RESPST_EXECUTE;	496	return RESPST_EXECUTE;
518	}	497	}
519		498
@@ -592,6 +571,7 @@ skip_check_range: · 72 unmodified lines
592	return RESPST_EXECUTE;	571	return RESPST_EXECUTE;
593		572
594	err:	573	err:
		574	qp->resp.mr = NULL;
595	if (mr)	575	if (mr)
596	rxe_put(mr);	576	rxe_put(mr);
597	if (mw)	577	if (mw)
@@ -725,17 +705,12 @@ static enum resp_states process_flush(struct rxe_qp *qp, · 127 unmodified lines
725	return RESPST_ACKNOWLEDGE;	705	return RESPST_ACKNOWLEDGE;
726	}	706	}
727		707
728	/* Guarantee atomicity of atomic operations at the machine level. */
729	static DEFINE_SPINLOCK(atomic_ops_lock);
730
731	static enum resp_states atomic_reply(struct rxe_qp *qp,	708	static enum resp_states atomic_reply(struct rxe_qp *qp,
732	struct rxe_pkt_info *pkt)	709	struct rxe_pkt_info *pkt)
733	{	710	{
734	u64 *vaddr;
735	enum resp_states ret;
736	struct rxe_mr *mr = qp->resp.mr;	711	struct rxe_mr *mr = qp->resp.mr;
737	struct resp_res *res = qp->resp.res;	712	struct resp_res *res = qp->resp.res;
738	u64 value;	713	int err;
739		714
740	if (!res) {	715	if (!res) {
741	res = rxe_prepare_res(qp, pkt, RXE_ATOMIC_MASK);	716	res = rxe_prepare_res(qp, pkt, RXE_ATOMIC_MASK);
@@ -743,32 +718,14 @@ static enum resp_states atomic_reply(struct rxe_qp *qp, · 1 unmodified line
743	}	718	}
744		719
745	if (!res->replay) {	720	if (!res->replay) {
746	if (mr->state != RXE_MR_STATE_VALID) {	721	u64 iova = qp->resp.va + qp->resp.offset;
747	ret = RESPST_ERR_RKEY_VIOLATION;
748	goto out;
749	}
750
751	vaddr = iova_to_vaddr(mr, qp->resp.va + qp->resp.offset,
752	sizeof(u64));
753
754	/* check vaddr is 8 bytes aligned. */
755	if (!vaddr \|\| (uintptr_t)vaddr & 7) {
756	ret = RESPST_ERR_MISALIGNED_ATOMIC;
757	goto out;
758	}
759		722
760	spin_lock_bh(&atomic_ops_lock);	723	err = rxe_mr_do_atomic_op(mr, iova, pkt->opcode,
761	res->atomic.orig_val = value = *vaddr;	724	atmeth_comp(pkt),
762		725	atmeth_swap_add(pkt),
763	if (pkt->opcode == IB_OPCODE_RC_COMPARE_SWAP) {	726	&res->atomic.orig_val);
764	if (value == atmeth_comp(pkt))	727	if (err)
765	value = atmeth_swap_add(pkt);	728	return err;
766	} else {
767	value += atmeth_swap_add(pkt);
768	}
769
770	*vaddr = value;
771	spin_unlock_bh(&atomic_ops_lock);
772		729
773	qp->resp.msn++;	730	qp->resp.msn++;
774		731
@@ -780,35 +737,35 @@ static enum resp_states atomic_reply(struct rxe_qp *qp, · 5 unmodified lines
780	qp->resp.status = IB_WC_SUCCESS;	737	qp->resp.status = IB_WC_SUCCESS;
781	}	738	}
782		739
783	ret = RESPST_ACKNOWLEDGE;	740	return RESPST_ACKNOWLEDGE;
784	out:
785	return ret;
786	}	741	}
787		742
788	#ifdef CONFIG_64BIT	743	static enum resp_states atomic_write_reply(struct rxe_qp *qp,
789	static enum resp_states do_atomic_write(struct rxe_qp *qp,	744	struct rxe_pkt_info *pkt)
790	struct rxe_pkt_info *pkt)
791	{	745	{
792	struct rxe_mr *mr = qp->resp.mr;	746	struct resp_res *res = qp->resp.res;
793	int payload = payload_size(pkt);	747	struct rxe_mr *mr;
794	u64 src, *dst;	748	u64 value;
795		749	u64 iova;
796	if (mr->state != RXE_MR_STATE_VALID)	750	int err;
797	return RESPST_ERR_RKEY_VIOLATION;
798		751
799	memcpy(&src, payload_addr(pkt), payload);	752	if (!res) {
		753	res = rxe_prepare_res(qp, pkt, RXE_ATOMIC_WRITE_MASK);
		754	qp->resp.res = res;
		755	}
800		756
801	dst = iova_to_vaddr(mr, qp->resp.va + qp->resp.offset, payload);	757	if (res->replay)
802	/* check vaddr is 8 bytes aligned. */	758	return RESPST_ACKNOWLEDGE;
803	if (!dst \|\| (uintptr_t)dst & 7)
804	return RESPST_ERR_MISALIGNED_ATOMIC;
805		759
806	/* Do atomic write after all prior operations have completed */	760	mr = qp->resp.mr;
807	smp_store_release(dst, src);	761	value = (u64 )payload_addr(pkt);
		762	iova = qp->resp.va + qp->resp.offset;
808		763
809	/* decrease resp.resid to zero */	764	err = rxe_mr_do_atomic_write(mr, iova, value);
810	qp->resp.resid -= sizeof(payload);	765	if (err)
		766	return err;
811		767
		768	qp->resp.resid = 0;
812	qp->resp.msn++;	769	qp->resp.msn++;
813		770
814	/* next expected psn, read handles this separately */	771	/* next expected psn, read handles this separately */
@@ -817,29 +774,8 @@ static enum resp_states do_atomic_write(struct rxe_qp *qp, · 2 unmodified lines
817		774
818	qp->resp.opcode = pkt->opcode;	775	qp->resp.opcode = pkt->opcode;
819	qp->resp.status = IB_WC_SUCCESS;	776	qp->resp.status = IB_WC_SUCCESS;
820	return RESPST_ACKNOWLEDGE;
821	}
822	#else
823	static enum resp_states do_atomic_write(struct rxe_qp *qp,
824	struct rxe_pkt_info *pkt)
825	{
826	return RESPST_ERR_UNSUPPORTED_OPCODE;
827	}
828	#endif /* CONFIG_64BIT */
829		777
830	static enum resp_states atomic_write_reply(struct rxe_qp *qp,	778	return RESPST_ACKNOWLEDGE;
831	struct rxe_pkt_info *pkt)
832	{
833	struct resp_res *res = qp->resp.res;
834
835	if (!res) {
836	res = rxe_prepare_res(qp, pkt, RXE_ATOMIC_WRITE_MASK);
837	qp->resp.res = res;
838	}
839
840	if (res->replay)
841	return RESPST_ACKNOWLEDGE;
842	return do_atomic_write(qp, pkt);
843	}	779	}
844		780
845	static struct sk_buff prepare_ack_packet(struct rxe_qp qp,	781	static struct sk_buff prepare_ack_packet(struct rxe_qp qp,
@@ -966,7 +902,11 @@ static enum resp_states read_reply(struct rxe_qp *qp, · 120 unmodified lines
966	}	902	}
967		903
968	if (res->state == rdatm_res_state_new) {	904	if (res->state == rdatm_res_state_new) {
969	if (!res->replay) {	905	if (!res->replay \|\| qp->resp.length == 0) {
		906	/* if length == 0 mr will be NULL (is ok)
		907	* otherwise qp->resp.mr holds a ref on mr
		908	* which we transfer to mr and drop below.
		909	*/
970	mr = qp->resp.mr;	910	mr = qp->resp.mr;
971	qp->resp.mr = NULL;	911	qp->resp.mr = NULL;
972	} else {	912	} else {
@@ -980,6 +920,10 @@ static enum resp_states read_reply(struct rxe_qp *qp, · 7 unmodified lines
980	else	920	else
981	opcode = IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST;	921	opcode = IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST;
982	} else {	922	} else {
		923	/* re-lookup mr from rkey on all later packets.
		924	* length will be non-zero. This can fail if someone
		925	* modifies or destroys the mr since the first packet.
		926	*/
983	mr = rxe_recheck_mr(qp, res->read.rkey);	927	mr = rxe_recheck_mr(qp, res->read.rkey);
984	if (!mr)	928	if (!mr)
985	return RESPST_ERR_RKEY_VIOLATION;	929	return RESPST_ERR_RKEY_VIOLATION;
@@ -997,18 +941,16 @@ static enum resp_states read_reply(struct rxe_qp *qp, · 11 unmodified lines
997	skb = prepare_ack_packet(qp, &ack_pkt, opcode, payload,	941	skb = prepare_ack_packet(qp, &ack_pkt, opcode, payload,
998	res->cur_psn, AETH_ACK_UNLIMITED);	942	res->cur_psn, AETH_ACK_UNLIMITED);
999	if (!skb) {	943	if (!skb) {
1000	if (mr)	944	state = RESPST_ERR_RNR;
1001	rxe_put(mr);	945	goto err_out;
1002	return RESPST_ERR_RNR;
1003	}	946	}
1004		947
1005	err = rxe_mr_copy(mr, res->read.va, payload_addr(&ack_pkt),	948	err = rxe_mr_copy(mr, res->read.va, payload_addr(&ack_pkt),
1006	payload, RXE_FROM_MR_OBJ);	949	payload, RXE_FROM_MR_OBJ);
1007	if (mr)
1008	rxe_put(mr);
1009	if (err) {	950	if (err) {
1010	kfree_skb(skb);	951	kfree_skb(skb);
1011	return RESPST_ERR_RKEY_VIOLATION;	952	state = RESPST_ERR_RKEY_VIOLATION;
		953	goto err_out;
1012	}	954	}
1013		955
1014	if (bth_pad(&ack_pkt)) {	956	if (bth_pad(&ack_pkt)) {
@@ -1017,9 +959,12 @@ static enum resp_states read_reply(struct rxe_qp *qp, · 2 unmodified lines
1017	memset(pad, 0, bth_pad(&ack_pkt));	959	memset(pad, 0, bth_pad(&ack_pkt));
1018	}	960	}
1019		961
		962	/* rxe_xmit_packet always consumes the skb */
1020	err = rxe_xmit_packet(qp, &ack_pkt, skb);	963	err = rxe_xmit_packet(qp, &ack_pkt, skb);
1021	if (err)	964	if (err) {
1022	return RESPST_ERR_RNR;	965	state = RESPST_ERR_RNR;
		966	goto err_out;
		967	}
1023		968
1024	res->read.va += payload;	969	res->read.va += payload;
1025	res->read.resid -= payload;	970	res->read.resid -= payload;
@@ -1036,6 +981,9 @@ static enum resp_states read_reply(struct rxe_qp *qp, · 10 unmodified lines
1036	state = RESPST_CLEANUP;	981	state = RESPST_CLEANUP;
1037	}	982	}
1038		983
		984	err_out:
		985	if (mr)
		986	rxe_put(mr);
1039	return state;	987	return state;
1040	}	988	}
1041		989

drivers/infiniband/sw/rxe/rxe_verbs.c+53−62

 	int num_sge = ibwr->num_sge;
 	int full;
-	full = queue_full(rq->queue, QUEUE_TYPE_TO_DRIVER);
+	full = queue_full(rq->queue, QUEUE_TYPE_FROM_ULP);
 	if (unlikely(full))
 		return -ENOMEM;
 	for (i = 0; i < num_sge; i++)
 		length += ibwr->sg_list[i].length;
-	recv_wqe = queue_producer_addr(rq->queue, QUEUE_TYPE_TO_DRIVER);
+	recv_wqe = queue_producer_addr(rq->queue, QUEUE_TYPE_FROM_ULP);
 	recv_wqe->wr_id = ibwr->wr_id;
 	memcpy(recv_wqe->dma.sge, ibwr->sg_list,
 	recv_wqe->dma.cur_sge		= 0;
 	recv_wqe->dma.sge_offset	= 0;
-	queue_advance_producer(rq->queue, QUEUE_TYPE_TO_DRIVER);
+	queue_advance_producer(rq->queue, QUEUE_TYPE_FROM_ULP);
 	return 0;
 }
 	spin_lock_irqsave(&qp->sq.sq_lock, flags);
-	full = queue_full(sq->queue, QUEUE_TYPE_TO_DRIVER);
+	full = queue_full(sq->queue, QUEUE_TYPE_FROM_ULP);
 	if (unlikely(full)) {
 		spin_unlock_irqrestore(&qp->sq.sq_lock, flags);
 		return -ENOMEM;
 	}
-	send_wqe = queue_producer_addr(sq->queue, QUEUE_TYPE_TO_DRIVER);
+	send_wqe = queue_producer_addr(sq->queue, QUEUE_TYPE_FROM_ULP);
 	init_send_wqe(qp, ibwr, mask, length, send_wqe);
-	queue_advance_producer(sq->queue, QUEUE_TYPE_TO_DRIVER);
+	queue_advance_producer(sq->queue, QUEUE_TYPE_FROM_ULP);
 	spin_unlock_irqrestore(&qp->sq.sq_lock, flags);
 	spin_lock_irqsave(&cq->cq_lock, flags);
 	for (i = 0; i < num_entries; i++) {
-		cqe = queue_head(cq->queue, QUEUE_TYPE_FROM_DRIVER);
+		cqe = queue_head(cq->queue, QUEUE_TYPE_TO_ULP);
 		if (!cqe)
 			break;
 		memcpy(wc++, &cqe->ibwc, sizeof(*wc));
-		queue_advance_consumer(cq->queue, QUEUE_TYPE_FROM_DRIVER);
+		queue_advance_consumer(cq->queue, QUEUE_TYPE_TO_ULP);
 	}
 	spin_unlock_irqrestore(&cq->cq_lock, flags);
 	struct rxe_cq *cq = to_rcq(ibcq);
 	int count;
-	count = queue_count(cq->queue, QUEUE_TYPE_FROM_DRIVER);
+	count = queue_count(cq->queue, QUEUE_TYPE_TO_ULP);
 	return (count > wc_cnt) ? wc_cnt : count;
 }
 	if (cq->notify != IB_CQ_NEXT_COMP)
 		cq->notify = flags & IB_CQ_SOLICITED_MASK;
-	empty = queue_empty(cq->queue, QUEUE_TYPE_FROM_DRIVER);
+	empty = queue_empty(cq->queue, QUEUE_TYPE_TO_ULP);
 	if ((flags & IB_CQ_REPORT_MISSED_EVENTS) && !empty)
 		ret = 1;
 	struct rxe_dev *rxe = to_rdev(ibpd->device);
 	struct rxe_pd *pd = to_rpd(ibpd);
 	struct rxe_mr *mr;
+	int err;
-	mr = rxe_alloc(&rxe->mr_pool);
+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
-	if (!mr)
+	if (!mr) {
-		return ERR_PTR(-ENOMEM);
+		err = -ENOMEM;
+		goto err_out;
+	}
+	err = rxe_add_to_pool(&rxe->mr_pool, mr);
+	if (err)
+		goto err_free;
 	rxe_get(pd);
 	mr->ibmr.pd = ibpd;
 	rxe_mr_init_dma(access, mr);
 	rxe_finalize(mr);
 	return &mr->ibmr;
+err_free:
+	kfree(mr);
+err_out:
+	return ERR_PTR(err);
 }
 static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd,
 	struct rxe_pd *pd = to_rpd(ibpd);
 	struct rxe_mr *mr;
-	mr = rxe_alloc(&rxe->mr_pool);
+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
-	if (!mr)
+	if (!mr) {
-		return ERR_PTR(-ENOMEM);
+		err = -ENOMEM;
+		goto err_out;
+	}
+	err = rxe_add_to_pool(&rxe->mr_pool, mr);
+	if (err)
+		goto err_free;
 	rxe_get(pd);
 	mr->ibmr.pd = ibpd;
 	err = rxe_mr_init_user(rxe, start, length, iova, access, mr);
 	if (err)
-		goto err1;
+		goto err_cleanup;
 	rxe_finalize(mr);
 	return &mr->ibmr;
-err1:
+err_cleanup:
 	rxe_cleanup(mr);
+err_free:
+	kfree(mr);
+err_out:
 	return ERR_PTR(err);
 }
 	if (mr_type != IB_MR_TYPE_MEM_REG)
 		return ERR_PTR(-EINVAL);
-	mr = rxe_alloc(&rxe->mr_pool);
+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
-	if (!mr)
+	if (!mr) {
-		return ERR_PTR(-ENOMEM);
+		err = -ENOMEM;
+		goto err_out;
+	}
+	err = rxe_add_to_pool(&rxe->mr_pool, mr);
+	if (err)
+		goto err_free;
 	rxe_get(pd);
 	mr->ibmr.pd = ibpd;
 	err = rxe_mr_init_fast(max_num_sg, mr);
 	if (err)
-		goto err1;
+		goto err_cleanup;
 	rxe_finalize(mr);
 	return &mr->ibmr;
-err1:
+err_cleanup:
 	rxe_cleanup(mr);
+err_free:
+	kfree(mr);
+err_out:
 	return ERR_PTR(err);
 }
-static int rxe_set_page(struct ib_mr *ibmr, u64 addr)
-{
-	struct rxe_mr *mr = to_rmr(ibmr);
-	struct rxe_map *map;
-	struct rxe_phys_buf *buf;
-	if (unlikely(mr->nbuf == mr->num_buf))
-		return -ENOMEM;
-	map = mr->map[mr->nbuf / RXE_BUF_PER_MAP];
-	buf = &map->buf[mr->nbuf % RXE_BUF_PER_MAP];
-	buf->addr = addr;
-	buf->size = ibmr->page_size;
-	mr->nbuf++;
-	return 0;
-}
-static int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
-			 int sg_nents, unsigned int *sg_offset)
-{
-	struct rxe_mr *mr = to_rmr(ibmr);
-	int n;
-	mr->nbuf = 0;
-	n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, rxe_set_page);
-	mr->page_shift = ilog2(ibmr->page_size);
-	mr->page_mask = ibmr->page_size - 1;
-	mr->offset = ibmr->iova & mr->page_mask;
-	return n;
-}
 static ssize_t parent_show(struct device *device,
 			   struct device_attribute *attr, char *buf)
 {

drivers/infiniband/sw/rxe/rxe_verbs.h+10−22

 	RXE_LOOKUP_REMOTE,
 };
-#define RXE_BUF_PER_MAP		(PAGE_SIZE / sizeof(struct rxe_phys_buf))
-struct rxe_phys_buf {
-	u64      addr;
-	u64      size;
-};
-struct rxe_map {
-	struct rxe_phys_buf	buf[RXE_BUF_PER_MAP];
-};
 static inline int rkey_is_mw(u32 rkey)
 {
 	u32 index = rkey >> 8;
 	u32			lkey;
 	u32			rkey;
 	enum rxe_mr_state	state;
-	u32			offset;
 	int			access;
+	atomic_t		num_mw;
-	int			page_shift;
+	unsigned int		page_offset;
-	int			page_mask;
+	unsigned int		page_shift;
-	int			map_shift;
+	u64			page_mask;
-	int			map_mask;
 	u32			num_buf;
 	u32			nbuf;
-	u32			max_buf;
+	struct xarray		page_list;
-	u32			num_map;
-	atomic_t		num_mw;
-	struct rxe_map		**map;
 };
+static inline unsigned int mr_page_size(struct rxe_mr *mr)
+{
+	return mr ? mr->ibmr.page_size : PAGE_SIZE;
+}
 enum rxe_mw_state {
 	RXE_MW_STATE_INVALID	= RXE_MR_STATE_INVALID,
 	RXE_MW_STATE_FREE	= RXE_MR_STATE_FREE,

drivers/infiniband/sw/siw/siw_mem.c+12−11

@@ -398,7 +398,7 @@ struct siw_umem *siw_umem_get(u64 start, u64 len, bool writable) · 397 unmodified lines
398		398
399	mlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;	399	mlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
400		400
401	if (num_pages + atomic64_read(&mm_s->pinned_vm) > mlock_limit) {	401	if (atomic64_add_return(num_pages, &mm_s->pinned_vm) > mlock_limit) {
402	rv = -ENOMEM;	402	rv = -ENOMEM;
403	goto out_sem_up;	403	goto out_sem_up;
404	}	404	}
@@ -411,30 +411,27 @@ struct siw_umem *siw_umem_get(u64 start, u64 len, bool writable) · 6 unmodified lines
411	goto out_sem_up;	411	goto out_sem_up;
412	}	412	}
413	for (i = 0; num_pages; i++) {	413	for (i = 0; num_pages; i++) {
414	int got, nents = min_t(int, num_pages, PAGES_PER_CHUNK);	414	int nents = min_t(int, num_pages, PAGES_PER_CHUNK);
415		415	struct page **plist =
416	umem->page_chunk[i].plist =
417	kcalloc(nents, sizeof(struct page *), GFP_KERNEL);	416	kcalloc(nents, sizeof(struct page *), GFP_KERNEL);
418	if (!umem->page_chunk[i].plist) {	417
		418	if (!plist) {
419	rv = -ENOMEM;	419	rv = -ENOMEM;
420	goto out_sem_up;	420	goto out_sem_up;
421	}	421	}
422	got = 0;	422	umem->page_chunk[i].plist = plist;
423	while (nents) {	423	while (nents) {
424	struct page **plist = &umem->page_chunk[i].plist[got];
425
426	rv = pin_user_pages(first_page_va, nents, foll_flags,	424	rv = pin_user_pages(first_page_va, nents, foll_flags,
427	plist, NULL);	425	plist, NULL);
428	if (rv < 0)	426	if (rv < 0)
429	goto out_sem_up;	427	goto out_sem_up;
430		428
431	umem->num_pages += rv;	429	umem->num_pages += rv;
432	atomic64_add(rv, &mm_s->pinned_vm);
433	first_page_va += rv * PAGE_SIZE;	430	first_page_va += rv * PAGE_SIZE;
		431	plist += rv;
434	nents -= rv;	432	nents -= rv;
435	got += rv;	433	num_pages -= rv;
436	}	434	}
437	num_pages -= got;
438	}	435	}
439	out_sem_up:	436	out_sem_up:
440	mmap_read_unlock(mm_s);	437	mmap_read_unlock(mm_s);
@@ -442,6 +439,10 @@ out_sem_up: · 1 unmodified line
442	if (rv > 0)	439	if (rv > 0)
443	return umem;	440	return umem;
444		441
		442	/* Adjust accounting for pages not pinned */
		443	if (num_pages)
		444	atomic64_sub(num_pages, &mm_s->pinned_vm);
		445
445	siw_umem_release(umem, false);	446	siw_umem_release(umem, false);
446		447
447	return ERR_PTR(rv);	448	return ERR_PTR(rv);

drivers/infiniband/ulp/ipoib/ipoib_main.c+1−1

@@ -742,7 +742,7 @@ void ipoib_flush_paths(struct net_device *dev) · 741 unmodified lines
742		742
743	static void path_rec_completion(int status,	743	static void path_rec_completion(int status,
744	struct sa_path_rec *pathrec,	744	struct sa_path_rec *pathrec,
745	int num_prs, void *path_ptr)	745	unsigned int num_prs, void *path_ptr)
746	{	746	{
747	struct ipoib_path *path = path_ptr;	747	struct ipoib_path *path = path_ptr;
748	struct net_device *dev = path->dev;	748	struct net_device *dev = path->dev;

drivers/infiniband/ulp/srp/ib_srp.c+1−1

@@ -699,7 +699,7 @@ static void srp_free_ch_ib(struct srp_target_port *target, · 698 unmodified lines
699		699
700	static void srp_path_rec_completion(int status,	700	static void srp_path_rec_completion(int status,
701	struct sa_path_rec *pathrec,	701	struct sa_path_rec *pathrec,
702	int num_paths, void *ch_ptr)	702	unsigned int num_paths, void *ch_ptr)
703	{	703	{
704	struct srp_rdma_ch *ch = ch_ptr;	704	struct srp_rdma_ch *ch = ch_ptr;
705	struct srp_target_port *target = ch->target;	705	struct srp_target_port *target = ch->target;

drivers/net/ethernet/mellanox/mlx4/qp.c+9−5

 #define MLX4_BF_QP_SKIP_MASK	0xc0
 #define MLX4_MAX_BF_QP_RANGE	0x40
+void mlx4_put_qp(struct mlx4_qp *qp)
+{
+	if (refcount_dec_and_test(&qp->refcount))
+		complete(&qp->free);
+}
+EXPORT_SYMBOL_GPL(mlx4_put_qp);
 void mlx4_qp_event(struct mlx4_dev *dev, u32 qpn, int event_type)
 {
 	struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table;
 		return;
 	}
+	/* Need to call mlx4_put_qp() in event handler */
 	qp->event(qp, event_type);
-	if (refcount_dec_and_test(&qp->refcount))
-		complete(&qp->free);
 }
 /* used for INIT/CLOSE port logic */
 void mlx4_qp_free(struct mlx4_dev *dev, struct mlx4_qp *qp)
 {
-	if (refcount_dec_and_test(&qp->refcount))
+	mlx4_put_qp(qp);
-		complete(&qp->free);
 	wait_for_completion(&qp->free);
 	mlx4_qp_free_icm(dev, qp->qpn);

drivers/net/ethernet/mellanox/mlx5/core/en_main.c+21−1

@@ -668,6 +668,26 @@ static void mlx5e_rq_free_shampo(struct mlx5e_rq *rq) · 667 unmodified lines
668	mlx5e_rq_shampo_hd_free(rq);	668	mlx5e_rq_shampo_hd_free(rq);
669	}	669	}
670		670
		671	static __be32 mlx5e_get_terminate_scatter_list_mkey(struct mlx5_core_dev *dev)
		672	{
		673	u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {};
		674	u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)] = {};
		675	int res;
		676
		677	if (!MLX5_CAP_GEN(dev, terminate_scatter_list_mkey))
		678	return MLX5_TERMINATE_SCATTER_LIST_LKEY;
		679
		680	MLX5_SET(query_special_contexts_in, in, opcode,
		681	MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS);
		682	res = mlx5_cmd_exec_inout(dev, query_special_contexts, in, out);
		683	if (res)
		684	return MLX5_TERMINATE_SCATTER_LIST_LKEY;
		685
		686	res = MLX5_GET(query_special_contexts_out, out,
		687	terminate_scatter_list_mkey);
		688	return cpu_to_be32(res);
		689	}
		690
671	static int mlx5e_alloc_rq(struct mlx5e_params *params,	691	static int mlx5e_alloc_rq(struct mlx5e_params *params,
672	struct mlx5e_xsk_param *xsk,	692	struct mlx5e_xsk_param *xsk,
673	struct mlx5e_rq_param *rqp,	693	struct mlx5e_rq_param *rqp,
@@ -832,7 +852,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, · 158 unmodified lines
832	/* check if num_frags is not a pow of two */	852	/* check if num_frags is not a pow of two */
833	if (rq->wqe.info.num_frags < (1 << rq->wqe.info.log_num_frags)) {	853	if (rq->wqe.info.num_frags < (1 << rq->wqe.info.log_num_frags)) {
834	wqe->data[f].byte_count = 0;	854	wqe->data[f].byte_count = 0;
835	wqe->data[f].lkey = cpu_to_be32(MLX5_INVALID_LKEY);	855	wqe->data[f].lkey = mlx5e_get_terminate_scatter_list_mkey(mdev);
836	wqe->data[f].addr = 0;	856	wqe->data[f].addr = 0;
837	}	857	}
838	}	858	}

include/linux/mlx4/qp.h+1−0

@@ -504,4 +504,5 @@ static inline u16 folded_qp(u32 q) · 503 unmodified lines
504		504
505	u16 mlx4_qp_roce_entropy(struct mlx4_dev *dev, u32 qpn);	505	u16 mlx4_qp_roce_entropy(struct mlx4_dev *dev, u32 qpn);
506		506
		507	void mlx4_put_qp(struct mlx4_qp *qp);
507	#endif /* MLX4_QP_H */	508	#endif /* MLX4_QP_H */

include/linux/mlx5/driver.h+0−1

@@ -742,7 +742,6 @@ enum { · 741 unmodified lines
742		742
743	enum {	743	enum {
744	MKEY_CACHE_LAST_STD_ENTRY = 20,	744	MKEY_CACHE_LAST_STD_ENTRY = 20,
745	MLX5_IMR_MTT_CACHE_ENTRY,
746	MLX5_IMR_KSM_CACHE_ENTRY,	745	MLX5_IMR_KSM_CACHE_ENTRY,
747	MAX_MKEY_CACHE_ENTRIES	746	MAX_MKEY_CACHE_ENTRIES
748	};	747	};

include/linux/mlx5/mlx5_ifc.h+20−2

 	u8         relaxed_ordering_write[0x1];
 	u8         relaxed_ordering_read[0x1];
 	u8         log_max_mkey[0x6];
-	u8         reserved_at_f0[0x8];
+	u8         reserved_at_f0[0x6];
+	u8	   terminate_scatter_list_mkey[0x1];
+	u8	   repeated_mkey[0x1];
 	u8         dump_fill_mkey[0x1];
 	u8         reserved_at_f9[0x2];
 	u8         fast_teardown[0x1];
 	u8         reserved_at_360[0x4a0];
 };
+struct mlx5_ifc_cong_control_r_roce_general_bits {
+	u8         reserved_at_0[0x80];
+	u8         reserved_at_80[0x10];
+	u8         rtt_resp_dscp_valid[0x1];
+	u8         reserved_at_91[0x9];
+	u8         rtt_resp_dscp[0x6];
+	u8         reserved_at_a0[0x760];
+};
 struct mlx5_ifc_cong_control_802_1qau_rp_bits {
 	u8         reserved_at_0[0x80];
 	struct mlx5_ifc_cong_control_802_1qau_rp_bits cong_control_802_1qau_rp;
 	struct mlx5_ifc_cong_control_r_roce_ecn_rp_bits cong_control_r_roce_ecn_rp;
 	struct mlx5_ifc_cong_control_r_roce_ecn_np_bits cong_control_r_roce_ecn_np;
+	struct mlx5_ifc_cong_control_r_roce_general_bits cong_control_r_roce_general;
 	u8         reserved_at_0[0x800];
 };
 	u8         null_mkey[0x20];
-	u8         reserved_at_a0[0x60];
+	u8	   terminate_scatter_list_mkey[0x20];
+	u8	   repeated_mkey[0x20];
+	u8         reserved_at_a0[0x20];
 };
 struct mlx5_ifc_query_special_contexts_in_bits {

include/linux/mlx5/qp.h+1−1

@@ -36,7 +36,7 @@ · 35 unmodified lines
36	#include <linux/mlx5/device.h>	36	#include <linux/mlx5/device.h>
37	#include <linux/mlx5/driver.h>	37	#include <linux/mlx5/driver.h>
38		38
39	#define MLX5_INVALID_LKEY 0x100	39	#define MLX5_TERMINATE_SCATTER_LIST_LKEY cpu_to_be32(0x100)
40	/* UMR (3 WQE_BB's) + SIG (3 WQE_BB's) + PSV (mem) + PSV (wire) */	40	/* UMR (3 WQE_BB's) + SIG (3 WQE_BB's) + PSV (mem) + PSV (wire) */
41	#define MLX5_SIG_WQE_SIZE (MLX5_SEND_WQE_BB * 8)	41	#define MLX5_SIG_WQE_SIZE (MLX5_SEND_WQE_BB * 8)
42	#define MLX5_DIF_SIZE 8	42	#define MLX5_DIF_SIZE 8

include/rdma/ib_sa.h+1−1

@@ -414,7 +414,7 @@ int ib_sa_path_rec_get(struct ib_sa_client client, struct ib_device device, · 413 unmodified lines
414	ib_sa_comp_mask comp_mask, unsigned long timeout_ms,	414	ib_sa_comp_mask comp_mask, unsigned long timeout_ms,
415	gfp_t gfp_mask,	415	gfp_t gfp_mask,
416	void (callback)(int status, struct sa_path_rec resp,	416	void (callback)(int status, struct sa_path_rec resp,
417	int num_prs, void *context),	417	unsigned int num_prs, void *context),
418	void context, struct ib_sa_query *query);	418	void context, struct ib_sa_query *query);
419		419
420	struct ib_sa_multicast {	420	struct ib_sa_multicast {

include/rdma/ib_umem.h+0−1

@@ -25,7 +25,6 @@ struct ib_umem { · 24 unmodified lines
25	u32 writable : 1;	25	u32 writable : 1;
26	u32 is_odp : 1;	26	u32 is_odp : 1;
27	u32 is_dmabuf : 1;	27	u32 is_dmabuf : 1;
28	struct work_struct work;
29	struct sg_append_table sgt_append;	28	struct sg_append_table sgt_append;
30	};	29	};
31		30

include/rdma/ib_verbs.h+1−1

@@ -1168,7 +1168,7 @@ enum ib_qp_create_flags { · 1167 unmodified lines
1168	*/	1168	*/
1169		1169
1170	struct ib_qp_init_attr {	1170	struct ib_qp_init_attr {
1171	/* Consumer's event_handler callback must not block */	1171	/* This callback occurs in workqueue context */
1172	void (event_handler)(struct ib_event , void *);	1172	void (event_handler)(struct ib_event , void *);
1173		1173
1174	void *qp_context;	1174	void *qp_context;

include/rdma/rdma_cm.h+0−1

@@ -49,7 +49,6 @@ struct rdma_addr { · 48 unmodified lines
49	struct rdma_dev_addr dev_addr;	49	struct rdma_dev_addr dev_addr;
50	};	50	};
51		51
52	#define RDMA_PRIMARY_PATH_MAX_REC_NUM 3
53	struct rdma_route {	52	struct rdma_route {
54	struct rdma_addr addr;	53	struct rdma_addr addr;
55	struct sa_path_rec *path_rec;	54	struct sa_path_rec *path_rec;

include/rdma/restrack.h+2−2

@@ -162,8 +162,8 @@ struct rdma_restrack_entry rdma_restrack_get_byid(struct ib_device dev, · 161 unmodified lines
162	* rdma_restrack_no_track() - don't add resource to the DB	162	* rdma_restrack_no_track() - don't add resource to the DB
163	* @res: resource entry	163	* @res: resource entry
164	*	164	*
165	* Every user of thie API should be cross examined.	165	* Every user of this API should be cross examined.
166	* Probaby you don't need to use this function.	166	* Probably you don't need to use this function.
167	*/	167	*/
168	static inline void rdma_restrack_no_track(struct rdma_restrack_entry *res)	168	static inline void rdma_restrack_no_track(struct rdma_restrack_entry *res)
169	{	169	{

include/uapi/rdma/hns-abi.h+4−0

 enum {
 	HNS_ROCE_EXSGE_FLAGS = 1 << 0,
+	HNS_ROCE_RQ_INLINE_FLAGS = 1 << 1,
+	HNS_ROCE_CQE_INLINE_FLAGS = 1 << 2,
 };
 enum {
 	HNS_ROCE_RSP_EXSGE_FLAGS = 1 << 0,
+	HNS_ROCE_RSP_RQ_INLINE_FLAGS = 1 << 1,
+	HNS_ROCE_RSP_CQE_INLINE_FLAGS = 1 << 2,
 };
 struct hns_roce_ib_alloc_ucontext_resp {