lael-labs/linux

MAINTAINERS+1−0

@@ -3621,6 +3621,7 @@ M: Selvin Xavier <selvin.xavier@broadcom.com> · 3620 unmodified lines
3621	M: Devesh Sharma <devesh.sharma@broadcom.com>	3621	M: Devesh Sharma <devesh.sharma@broadcom.com>
3622	M: Somnath Kotur <somnath.kotur@broadcom.com>	3622	M: Somnath Kotur <somnath.kotur@broadcom.com>
3623	M: Sriharsha Basavapatna <sriharsha.basavapatna@broadcom.com>	3623	M: Sriharsha Basavapatna <sriharsha.basavapatna@broadcom.com>
		3624	M: Naresh Kumar PBS <nareshkumar.pbs@broadcom.com>
3624	L: linux-rdma@vger.kernel.org	3625	L: linux-rdma@vger.kernel.org
3625	S: Supported	3626	S: Supported
3626	W: http://www.broadcom.com	3627	W: http://www.broadcom.com

drivers/infiniband/Kconfig+0−8

@@ -37,14 +37,6 @@ config INFINIBAND_USER_ACCESS · 36 unmodified lines
37	libibverbs, libibcm and a hardware driver library from	37	libibverbs, libibcm and a hardware driver library from
38	rdma-core <https://github.com/linux-rdma/rdma-core>.	38	rdma-core <https://github.com/linux-rdma/rdma-core>.
39		39
40	config INFINIBAND_EXP_LEGACY_VERBS_NEW_UAPI
41	bool "Allow experimental legacy verbs in new ioctl uAPI (EXPERIMENTAL)"
42	depends on INFINIBAND_USER_ACCESS
43	help
44	IOCTL based uAPI support for Infiniband is enabled by default for
45	new verbs only. This allows userspace to invoke the IOCTL based uAPI
46	for current legacy verbs too.
47
48	config INFINIBAND_USER_MEM	40	config INFINIBAND_USER_MEM
49	bool	41	bool
50	depends on INFINIBAND_USER_ACCESS != n	42	depends on INFINIBAND_USER_ACCESS != n

drivers/infiniband/core/cache.c+29−16

 	cache = device->port_data[port_num].cache.pkey;
-	if (index < 0 || index >= cache->table_len)
+	if (!cache || index < 0 || index >= cache->table_len)
 		ret = -EINVAL;
 	else
 		*pkey = cache->table[index];
 	read_lock_irqsave(&device->cache_lock, flags);
 	cache = device->port_data[port_num].cache.pkey;
+	if (!cache) {
+		ret = -EINVAL;
+		goto err;
+	}
 	*index = -1;
 		ret = 0;
 	}
+err:
 	read_unlock_irqrestore(&device->cache_lock, flags);
 	return ret;
 	read_lock_irqsave(&device->cache_lock, flags);
 	cache = device->port_data[port_num].cache.pkey;
+	if (!cache) {
+		ret = -EINVAL;
+		goto err;
+	}
 	*index = -1;
 			break;
 		}
+err:
 	read_unlock_irqrestore(&device->cache_lock, flags);
 	return ret;
 			goto err;
 	}
-	pkey_cache = kmalloc(struct_size(pkey_cache, table,
+	if (tprops->pkey_tbl_len) {
-					 tprops->pkey_tbl_len),
+		pkey_cache = kmalloc(struct_size(pkey_cache, table,
-			     GFP_KERNEL);
+						 tprops->pkey_tbl_len),
-	if (!pkey_cache) {
+				     GFP_KERNEL);
-		ret = -ENOMEM;
+		if (!pkey_cache) {
-		goto err;
+			ret = -ENOMEM;
-	}
+			goto err;
+		}
-	pkey_cache->table_len = tprops->pkey_tbl_len;
+		pkey_cache->table_len = tprops->pkey_tbl_len;
-	for (i = 0; i < pkey_cache->table_len; ++i) {
+		for (i = 0; i < pkey_cache->table_len; ++i) {
-		ret = ib_query_pkey(device, port, i, pkey_cache->table + i);
+			ret = ib_query_pkey(device, port, i,
-		if (ret) {
+					    pkey_cache->table + i);
-			dev_warn(&device->dev,
+			if (ret) {
-				 "ib_query_pkey failed (%d) for index %d\n",
+				dev_warn(&device->dev,
-				 ret, i);
+					 "ib_query_pkey failed (%d) for index %d\n",
-			goto err;
+					 ret, i);
+				goto err;
+			}
 		}
 	}

drivers/infiniband/core/cma.c+123−134

@@ -428,19 +428,6 @@ static int cma_comp_exch(struct rdma_id_private *id_priv, · 427 unmodified lines
428	return ret;	428	return ret;
429	}	429	}
430		430
431	static enum rdma_cm_state cma_exch(struct rdma_id_private *id_priv,
432	enum rdma_cm_state exch)
433	{
434	unsigned long flags;
435	enum rdma_cm_state old;
436
437	spin_lock_irqsave(&id_priv->lock, flags);
438	old = id_priv->state;
439	id_priv->state = exch;
440	spin_unlock_irqrestore(&id_priv->lock, flags);
441	return old;
442	}
443
444	static inline u8 cma_get_ip_ver(const struct cma_hdr *hdr)	431	static inline u8 cma_get_ip_ver(const struct cma_hdr *hdr)
445	{	432	{
446	return hdr->ip_version >> 4;	433	return hdr->ip_version >> 4;
@@ -1829,23 +1816,11 @@ static void cma_leave_mc_groups(struct rdma_id_private *id_priv) · 1382 unmodified lines
1829	}	1816	}
1830	}	1817	}
1831		1818
1832	void rdma_destroy_id(struct rdma_cm_id *id)	1819	static void _destroy_id(struct rdma_id_private *id_priv,
		1820	enum rdma_cm_state state)
1833	{	1821	{
1834	struct rdma_id_private *id_priv;
1835	enum rdma_cm_state state;
1836
1837	id_priv = container_of(id, struct rdma_id_private, id);
1838	trace_cm_id_destroy(id_priv);
1839	state = cma_exch(id_priv, RDMA_CM_DESTROYING);
1840	cma_cancel_operation(id_priv, state);	1822	cma_cancel_operation(id_priv, state);
1841		1823
1842	/*
1843	* Wait for any active callback to finish. New callbacks will find
1844	* the id_priv state set to destroying and abort.
1845	*/
1846	mutex_lock(&id_priv->handler_mutex);
1847	mutex_unlock(&id_priv->handler_mutex);
1848
1849	rdma_restrack_del(&id_priv->res);	1824	rdma_restrack_del(&id_priv->res);
1850	if (id_priv->cma_dev) {	1825	if (id_priv->cma_dev) {
1851	if (rdma_cap_ib_cm(id_priv->id.device, 1)) {	1826	if (rdma_cap_ib_cm(id_priv->id.device, 1)) {
@@ -1874,6 +1849,42 @@ void rdma_destroy_id(struct rdma_cm_id *id) · 22 unmodified lines
1874	put_net(id_priv->id.route.addr.dev_addr.net);	1849	put_net(id_priv->id.route.addr.dev_addr.net);
1875	kfree(id_priv);	1850	kfree(id_priv);
1876	}	1851	}
		1852
		1853	/*
		1854	* destroy an ID from within the handler_mutex. This ensures that no other
		1855	* handlers can start running concurrently.
		1856	*/
		1857	static void destroy_id_handler_unlock(struct rdma_id_private *id_priv)
		1858	__releases(&idprv->handler_mutex)
		1859	{
		1860	enum rdma_cm_state state;
		1861	unsigned long flags;
		1862
		1863	trace_cm_id_destroy(id_priv);
		1864
		1865	/*
		1866	* Setting the state to destroyed under the handler mutex provides a
		1867	* fence against calling handler callbacks. If this is invoked due to
		1868	* the failure of a handler callback then it guarentees that no future
		1869	* handlers will be called.
		1870	*/
		1871	lockdep_assert_held(&id_priv->handler_mutex);
		1872	spin_lock_irqsave(&id_priv->lock, flags);
		1873	state = id_priv->state;
		1874	id_priv->state = RDMA_CM_DESTROYING;
		1875	spin_unlock_irqrestore(&id_priv->lock, flags);
		1876	mutex_unlock(&id_priv->handler_mutex);
		1877	_destroy_id(id_priv, state);
		1878	}
		1879
		1880	void rdma_destroy_id(struct rdma_cm_id *id)
		1881	{
		1882	struct rdma_id_private *id_priv =
		1883	container_of(id, struct rdma_id_private, id);
		1884
		1885	mutex_lock(&id_priv->handler_mutex);
		1886	destroy_id_handler_unlock(id_priv);
		1887	}
1877	EXPORT_SYMBOL(rdma_destroy_id);	1888	EXPORT_SYMBOL(rdma_destroy_id);
1878		1889
1879	static int cma_rep_recv(struct rdma_id_private *id_priv)	1890	static int cma_rep_recv(struct rdma_id_private *id_priv)
@@ -1925,6 +1936,8 @@ static int cma_cm_event_handler(struct rdma_id_private *id_priv, · 45 unmodified lines
1925	{	1936	{
1926	int ret;	1937	int ret;
1927		1938
		1939	lockdep_assert_held(&id_priv->handler_mutex);
		1940
1928	trace_cm_event_handler(id_priv, event);	1941	trace_cm_event_handler(id_priv, event);
1929	ret = id_priv->id.event_handler(&id_priv->id, event);	1942	ret = id_priv->id.event_handler(&id_priv->id, event);
1930	trace_cm_event_done(id_priv, event, ret);	1943	trace_cm_event_done(id_priv, event, ret);
@@ -1936,7 +1949,7 @@ static int cma_ib_handler(struct ib_cm_id *cm_id, · 5 unmodified lines
1936	{	1949	{
1937	struct rdma_id_private *id_priv = cm_id->context;	1950	struct rdma_id_private *id_priv = cm_id->context;
1938	struct rdma_cm_event event = {};	1951	struct rdma_cm_event event = {};
1939	int ret = 0;	1952	int ret;
1940		1953
1941	mutex_lock(&id_priv->handler_mutex);	1954	mutex_lock(&id_priv->handler_mutex);
1942	if ((ib_event->event != IB_CM_TIMEWAIT_EXIT &&	1955	if ((ib_event->event != IB_CM_TIMEWAIT_EXIT &&
@@ -2005,14 +2018,12 @@ static int cma_ib_handler(struct ib_cm_id *cm_id, · 62 unmodified lines
2005	if (ret) {	2018	if (ret) {
2006	/* Destroy the CM ID by returning a non-zero value. */	2019	/* Destroy the CM ID by returning a non-zero value. */
2007	id_priv->cm_id.ib = NULL;	2020	id_priv->cm_id.ib = NULL;
2008	cma_exch(id_priv, RDMA_CM_DESTROYING);	2021	destroy_id_handler_unlock(id_priv);
2009	mutex_unlock(&id_priv->handler_mutex);
2010	rdma_destroy_id(&id_priv->id);
2011	return ret;	2022	return ret;
2012	}	2023	}
2013	out:	2024	out:
2014	mutex_unlock(&id_priv->handler_mutex);	2025	mutex_unlock(&id_priv->handler_mutex);
2015	return ret;	2026	return 0;
2016	}	2027	}
2017		2028
2018	static struct rdma_id_private *	2029	static struct rdma_id_private *
@@ -2174,7 +2185,7 @@ static int cma_ib_req_handler(struct ib_cm_id *cm_id, · 155 unmodified lines
2174	mutex_lock(&listen_id->handler_mutex);	2185	mutex_lock(&listen_id->handler_mutex);
2175	if (listen_id->state != RDMA_CM_LISTEN) {	2186	if (listen_id->state != RDMA_CM_LISTEN) {
2176	ret = -ECONNABORTED;	2187	ret = -ECONNABORTED;
2177	goto err1;	2188	goto err_unlock;
2178	}	2189	}
2179		2190
2180	offset = cma_user_data_offset(listen_id);	2191	offset = cma_user_data_offset(listen_id);
@@ -2191,55 +2202,38 @@ static int cma_ib_req_handler(struct ib_cm_id *cm_id, · 10 unmodified lines
2191	}	2202	}
2192	if (!conn_id) {	2203	if (!conn_id) {
2193	ret = -ENOMEM;	2204	ret = -ENOMEM;
2194	goto err1;	2205	goto err_unlock;
2195	}	2206	}
2196		2207
2197	mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);	2208	mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
2198	ret = cma_ib_acquire_dev(conn_id, listen_id, &req);	2209	ret = cma_ib_acquire_dev(conn_id, listen_id, &req);
2199	if (ret)	2210	if (ret) {
2200	goto err2;	2211	destroy_id_handler_unlock(conn_id);
		2212	goto err_unlock;
		2213	}
2201		2214
2202	conn_id->cm_id.ib = cm_id;	2215	conn_id->cm_id.ib = cm_id;
2203	cm_id->context = conn_id;	2216	cm_id->context = conn_id;
2204	cm_id->cm_handler = cma_ib_handler;	2217	cm_id->cm_handler = cma_ib_handler;
2205		2218
2206	/*
2207	* Protect against the user destroying conn_id from another thread
2208	* until we're done accessing it.
2209	*/
2210	cma_id_get(conn_id);
2211	ret = cma_cm_event_handler(conn_id, &event);	2219	ret = cma_cm_event_handler(conn_id, &event);
2212	if (ret)	2220	if (ret) {
2213	goto err3;	2221	/* Destroy the CM ID by returning a non-zero value. */
2214	/*	2222	conn_id->cm_id.ib = NULL;
2215	* Acquire mutex to prevent user executing rdma_destroy_id()	2223	mutex_unlock(&listen_id->handler_mutex);
2216	* while we're accessing the cm_id.	2224	destroy_id_handler_unlock(conn_id);
2217	*/	2225	goto net_dev_put;
2218	mutex_lock(&lock);	2226	}
		2227
2219	if (cma_comp(conn_id, RDMA_CM_CONNECT) &&	2228	if (cma_comp(conn_id, RDMA_CM_CONNECT) &&
2220	(conn_id->id.qp_type != IB_QPT_UD)) {	2229	(conn_id->id.qp_type != IB_QPT_UD)) {
2221	trace_cm_send_mra(cm_id->context);	2230	trace_cm_send_mra(cm_id->context);
2222	ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);	2231	ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
2223	}	2232	}
2224	mutex_unlock(&lock);
2225	mutex_unlock(&conn_id->handler_mutex);	2233	mutex_unlock(&conn_id->handler_mutex);
2226	mutex_unlock(&listen_id->handler_mutex);
2227	cma_id_put(conn_id);
2228	if (net_dev)
2229	dev_put(net_dev);
2230	return 0;
2231		2234
2232	err3:	2235	err_unlock:
2233	cma_id_put(conn_id);
2234	/* Destroy the CM ID by returning a non-zero value. */
2235	conn_id->cm_id.ib = NULL;
2236	err2:
2237	cma_exch(conn_id, RDMA_CM_DESTROYING);
2238	mutex_unlock(&conn_id->handler_mutex);
2239	err1:
2240	mutex_unlock(&listen_id->handler_mutex);	2236	mutex_unlock(&listen_id->handler_mutex);
2241	if (conn_id)
2242	rdma_destroy_id(&conn_id->id);
2243		2237
2244	net_dev_put:	2238	net_dev_put:
2245	if (net_dev)	2239	if (net_dev)
@@ -2339,9 +2333,7 @@ static int cma_iw_handler(struct iw_cm_id iw_id, struct iw_cm_event iw_event) · 93 unmodified lines
2339	if (ret) {	2333	if (ret) {
2340	/* Destroy the CM ID by returning a non-zero value. */	2334	/* Destroy the CM ID by returning a non-zero value. */
2341	id_priv->cm_id.iw = NULL;	2335	id_priv->cm_id.iw = NULL;
2342	cma_exch(id_priv, RDMA_CM_DESTROYING);	2336	destroy_id_handler_unlock(id_priv);
2343	mutex_unlock(&id_priv->handler_mutex);
2344	rdma_destroy_id(&id_priv->id);
2345	return ret;	2337	return ret;
2346	}	2338	}
2347		2339
@@ -2388,16 +2380,16 @@ static int iw_conn_req_handler(struct iw_cm_id *cm_id, · 40 unmodified lines
2388		2380
2389	ret = rdma_translate_ip(laddr, &conn_id->id.route.addr.dev_addr);	2381	ret = rdma_translate_ip(laddr, &conn_id->id.route.addr.dev_addr);
2390	if (ret) {	2382	if (ret) {
2391	mutex_unlock(&conn_id->handler_mutex);	2383	mutex_unlock(&listen_id->handler_mutex);
2392	rdma_destroy_id(new_cm_id);	2384	destroy_id_handler_unlock(conn_id);
2393	goto out;	2385	return ret;
2394	}	2386	}
2395		2387
2396	ret = cma_iw_acquire_dev(conn_id, listen_id);	2388	ret = cma_iw_acquire_dev(conn_id, listen_id);
2397	if (ret) {	2389	if (ret) {
2398	mutex_unlock(&conn_id->handler_mutex);	2390	mutex_unlock(&listen_id->handler_mutex);
2399	rdma_destroy_id(new_cm_id);	2391	destroy_id_handler_unlock(conn_id);
2400	goto out;	2392	return ret;
2401	}	2393	}
2402		2394
2403	conn_id->cm_id.iw = cm_id;	2395	conn_id->cm_id.iw = cm_id;
@@ -2407,25 +2399,16 @@ static int iw_conn_req_handler(struct iw_cm_id *cm_id, · 3 unmodified lines
2407	memcpy(cma_src_addr(conn_id), laddr, rdma_addr_size(laddr));	2399	memcpy(cma_src_addr(conn_id), laddr, rdma_addr_size(laddr));
2408	memcpy(cma_dst_addr(conn_id), raddr, rdma_addr_size(raddr));	2400	memcpy(cma_dst_addr(conn_id), raddr, rdma_addr_size(raddr));
2409		2401
2410	/*
2411	* Protect against the user destroying conn_id from another thread
2412	* until we're done accessing it.
2413	*/
2414	cma_id_get(conn_id);
2415	ret = cma_cm_event_handler(conn_id, &event);	2402	ret = cma_cm_event_handler(conn_id, &event);
2416	if (ret) {	2403	if (ret) {
2417	/* User wants to destroy the CM ID */	2404	/* User wants to destroy the CM ID */
2418	conn_id->cm_id.iw = NULL;	2405	conn_id->cm_id.iw = NULL;
2419	cma_exch(conn_id, RDMA_CM_DESTROYING);
2420	mutex_unlock(&conn_id->handler_mutex);
2421	mutex_unlock(&listen_id->handler_mutex);	2406	mutex_unlock(&listen_id->handler_mutex);
2422	cma_id_put(conn_id);	2407	destroy_id_handler_unlock(conn_id);
2423	rdma_destroy_id(&conn_id->id);
2424	return ret;	2408	return ret;
2425	}	2409	}
2426		2410
2427	mutex_unlock(&conn_id->handler_mutex);	2411	mutex_unlock(&conn_id->handler_mutex);
2428	cma_id_put(conn_id);
2429		2412
2430	out:	2413	out:
2431	mutex_unlock(&listen_id->handler_mutex);	2414	mutex_unlock(&listen_id->handler_mutex);
@@ -2482,6 +2465,10 @@ static int cma_listen_handler(struct rdma_cm_id *id, · 50 unmodified lines
2482	{	2465	{
2483	struct rdma_id_private *id_priv = id->context;	2466	struct rdma_id_private *id_priv = id->context;
2484		2467
		2468	/* Listening IDs are always destroyed on removal */
		2469	if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL)
		2470	return -1;
		2471
2485	id->context = id_priv->id.context;	2472	id->context = id_priv->id.context;
2486	id->event_handler = id_priv->id.event_handler;	2473	id->event_handler = id_priv->id.event_handler;
2487	trace_cm_event_handler(id_priv, event);	2474	trace_cm_event_handler(id_priv, event);
@@ -2657,21 +2644,21 @@ static void cma_work_handler(struct work_struct *_work) · 169 unmodified lines
2657	{	2644	{
2658	struct cma_work *work = container_of(_work, struct cma_work, work);	2645	struct cma_work *work = container_of(_work, struct cma_work, work);
2659	struct rdma_id_private *id_priv = work->id;	2646	struct rdma_id_private *id_priv = work->id;
2660	int destroy = 0;
2661		2647
2662	mutex_lock(&id_priv->handler_mutex);	2648	mutex_lock(&id_priv->handler_mutex);
2663	if (!cma_comp_exch(id_priv, work->old_state, work->new_state))	2649	if (!cma_comp_exch(id_priv, work->old_state, work->new_state))
2664	goto out;	2650	goto out_unlock;
2665		2651
2666	if (cma_cm_event_handler(id_priv, &work->event)) {	2652	if (cma_cm_event_handler(id_priv, &work->event)) {
2667	cma_exch(id_priv, RDMA_CM_DESTROYING);	2653	cma_id_put(id_priv);
2668	destroy = 1;	2654	destroy_id_handler_unlock(id_priv);
		2655	goto out_free;
2669	}	2656	}
2670	out:	2657
		2658	out_unlock:
2671	mutex_unlock(&id_priv->handler_mutex);	2659	mutex_unlock(&id_priv->handler_mutex);
2672	cma_id_put(id_priv);	2660	cma_id_put(id_priv);
2673	if (destroy)	2661	out_free:
2674	rdma_destroy_id(&id_priv->id);
2675	kfree(work);	2662	kfree(work);
2676	}	2663	}
2677		2664
@@ -2679,23 +2666,22 @@ static void cma_ndev_work_handler(struct work_struct *_work) · 1 unmodified line
2679	{	2666	{
2680	struct cma_ndev_work *work = container_of(_work, struct cma_ndev_work, work);	2667	struct cma_ndev_work *work = container_of(_work, struct cma_ndev_work, work);
2681	struct rdma_id_private *id_priv = work->id;	2668	struct rdma_id_private *id_priv = work->id;
2682	int destroy = 0;
2683		2669
2684	mutex_lock(&id_priv->handler_mutex);	2670	mutex_lock(&id_priv->handler_mutex);
2685	if (id_priv->state == RDMA_CM_DESTROYING \|\|	2671	if (id_priv->state == RDMA_CM_DESTROYING \|\|
2686	id_priv->state == RDMA_CM_DEVICE_REMOVAL)	2672	id_priv->state == RDMA_CM_DEVICE_REMOVAL)
2687	goto out;	2673	goto out_unlock;
2688		2674
2689	if (cma_cm_event_handler(id_priv, &work->event)) {	2675	if (cma_cm_event_handler(id_priv, &work->event)) {
2690	cma_exch(id_priv, RDMA_CM_DESTROYING);	2676	cma_id_put(id_priv);
2691	destroy = 1;	2677	destroy_id_handler_unlock(id_priv);
		2678	goto out_free;
2692	}	2679	}
2693		2680
2694	out:	2681	out_unlock:
2695	mutex_unlock(&id_priv->handler_mutex);	2682	mutex_unlock(&id_priv->handler_mutex);
2696	cma_id_put(id_priv);	2683	cma_id_put(id_priv);
2697	if (destroy)	2684	out_free:
2698	rdma_destroy_id(&id_priv->id);
2699	kfree(work);	2685	kfree(work);
2700	}	2686	}
2701		2687
@@ -3171,9 +3157,7 @@ static void addr_handler(int status, struct sockaddr *src_addr, · 469 unmodified lines
3171	event.event = RDMA_CM_EVENT_ADDR_RESOLVED;	3157	event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
3172		3158
3173	if (cma_cm_event_handler(id_priv, &event)) {	3159	if (cma_cm_event_handler(id_priv, &event)) {
3174	cma_exch(id_priv, RDMA_CM_DESTROYING);	3160	destroy_id_handler_unlock(id_priv);
3175	mutex_unlock(&id_priv->handler_mutex);
3176	rdma_destroy_id(&id_priv->id);
3177	return;	3161	return;
3178	}	3162	}
3179	out:	3163	out:
@@ -3790,7 +3774,7 @@ static int cma_sidr_rep_handler(struct ib_cm_id *cm_id, · 610 unmodified lines
3790	struct rdma_cm_event event = {};	3774	struct rdma_cm_event event = {};
3791	const struct ib_cm_sidr_rep_event_param *rep =	3775	const struct ib_cm_sidr_rep_event_param *rep =
3792	&ib_event->param.sidr_rep_rcvd;	3776	&ib_event->param.sidr_rep_rcvd;
3793	int ret = 0;	3777	int ret;
3794		3778
3795	mutex_lock(&id_priv->handler_mutex);	3779	mutex_lock(&id_priv->handler_mutex);
3796	if (id_priv->state != RDMA_CM_CONNECT)	3780	if (id_priv->state != RDMA_CM_CONNECT)
@@ -3840,14 +3824,12 @@ static int cma_sidr_rep_handler(struct ib_cm_id *cm_id, · 43 unmodified lines
3840	if (ret) {	3824	if (ret) {
3841	/* Destroy the CM ID by returning a non-zero value. */	3825	/* Destroy the CM ID by returning a non-zero value. */
3842	id_priv->cm_id.ib = NULL;	3826	id_priv->cm_id.ib = NULL;
3843	cma_exch(id_priv, RDMA_CM_DESTROYING);	3827	destroy_id_handler_unlock(id_priv);
3844	mutex_unlock(&id_priv->handler_mutex);
3845	rdma_destroy_id(&id_priv->id);
3846	return ret;	3828	return ret;
3847	}	3829	}
3848	out:	3830	out:
3849	mutex_unlock(&id_priv->handler_mutex);	3831	mutex_unlock(&id_priv->handler_mutex);
3850	return ret;	3832	return 0;
3851	}	3833	}
3852		3834
3853	static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,	3835	static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,
@@ -4372,9 +4354,7 @@ static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast) · 518 unmodified lines
4372		4354
4373	rdma_destroy_ah_attr(&event.param.ud.ah_attr);	4355	rdma_destroy_ah_attr(&event.param.ud.ah_attr);
4374	if (ret) {	4356	if (ret) {
4375	cma_exch(id_priv, RDMA_CM_DESTROYING);	4357	destroy_id_handler_unlock(id_priv);
4376	mutex_unlock(&id_priv->handler_mutex);
4377	rdma_destroy_id(&id_priv->id);
4378	return 0;	4358	return 0;
4379	}	4359	}
4380		4360
@@ -4789,50 +4769,59 @@ free_cma_dev: · 408 unmodified lines
4789	return ret;	4769	return ret;
4790	}	4770	}
4791		4771
4792	static int cma_remove_id_dev(struct rdma_id_private *id_priv)	4772	static void cma_send_device_removal_put(struct rdma_id_private *id_priv)
4793	{	4773	{
4794	struct rdma_cm_event event = {};	4774	struct rdma_cm_event event = { .event = RDMA_CM_EVENT_DEVICE_REMOVAL };
4795	enum rdma_cm_state state;	4775	enum rdma_cm_state state;
4796	int ret = 0;	4776	unsigned long flags;
4797
4798	/* Record that we want to remove the device */
4799	state = cma_exch(id_priv, RDMA_CM_DEVICE_REMOVAL);
4800	if (state == RDMA_CM_DESTROYING)
4801	return 0;
4802		4777
4803	cma_cancel_operation(id_priv, state);
4804	mutex_lock(&id_priv->handler_mutex);	4778	mutex_lock(&id_priv->handler_mutex);
		4779	/* Record that we want to remove the device */
		4780	spin_lock_irqsave(&id_priv->lock, flags);
		4781	state = id_priv->state;
		4782	if (state == RDMA_CM_DESTROYING \|\| state == RDMA_CM_DEVICE_REMOVAL) {
		4783	spin_unlock_irqrestore(&id_priv->lock, flags);
		4784	mutex_unlock(&id_priv->handler_mutex);
		4785	cma_id_put(id_priv);
		4786	return;
		4787	}
		4788	id_priv->state = RDMA_CM_DEVICE_REMOVAL;
		4789	spin_unlock_irqrestore(&id_priv->lock, flags);
4805		4790
4806	/* Check for destruction from another callback. */	4791	if (cma_cm_event_handler(id_priv, &event)) {
4807	if (!cma_comp(id_priv, RDMA_CM_DEVICE_REMOVAL))	4792	/*
4808	goto out;	4793	* At this point the ULP promises it won't call
4809		4794	* rdma_destroy_id() concurrently
4810	event.event = RDMA_CM_EVENT_DEVICE_REMOVAL;	4795	*/
4811	ret = cma_cm_event_handler(id_priv, &event);	4796	cma_id_put(id_priv);
4812	out:	4797	mutex_unlock(&id_priv->handler_mutex);
		4798	trace_cm_id_destroy(id_priv);
		4799	_destroy_id(id_priv, state);
		4800	return;
		4801	}
4813	mutex_unlock(&id_priv->handler_mutex);	4802	mutex_unlock(&id_priv->handler_mutex);
4814	return ret;	4803
		4804	/*
		4805	* If this races with destroy then the thread that first assigns state
		4806	* to a destroying does the cancel.
		4807	*/
		4808	cma_cancel_operation(id_priv, state);
		4809	cma_id_put(id_priv);
4815	}	4810	}
4816		4811
4817	static void cma_process_remove(struct cma_device *cma_dev)	4812	static void cma_process_remove(struct cma_device *cma_dev)
4818	{	4813	{
4819	struct rdma_id_private *id_priv;
4820	int ret;
4821
4822	mutex_lock(&lock);	4814	mutex_lock(&lock);
4823	while (!list_empty(&cma_dev->id_list)) {	4815	while (!list_empty(&cma_dev->id_list)) {
4824	id_priv = list_entry(cma_dev->id_list.next,	4816	struct rdma_id_private *id_priv = list_first_entry(
4825	struct rdma_id_private, list);	4817	&cma_dev->id_list, struct rdma_id_private, list);
4826		4818
4827	list_del(&id_priv->listen_list);	4819	list_del(&id_priv->listen_list);
4828	list_del_init(&id_priv->list);	4820	list_del_init(&id_priv->list);
4829	cma_id_get(id_priv);	4821	cma_id_get(id_priv);
4830	mutex_unlock(&lock);	4822	mutex_unlock(&lock);
4831		4823
4832	ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv);	4824	cma_send_device_removal_put(id_priv);
4833	cma_id_put(id_priv);
4834	if (ret)
4835	rdma_destroy_id(&id_priv->id);
4836		4825
4837	mutex_lock(&lock);	4826	mutex_lock(&lock);
4838	}	4827	}

drivers/infiniband/core/counters.c+7−17

 #include "core_priv.h"
 #include "restrack.h"
-#define ALL_AUTO_MODE_MASKS (RDMA_COUNTER_MASK_QP_TYPE)
+#define ALL_AUTO_MODE_MASKS (RDMA_COUNTER_MASK_QP_TYPE | RDMA_COUNTER_MASK_PID)
 static int __counter_set_mode(struct rdma_counter_mode *curr,
 			      enum rdma_nl_counter_mode new_mode,
 	struct auto_mode_param *param = &counter->mode.param;
 	bool match = true;
-	/*
-	 * Ensure that counter belongs to the right PID.  This operation can
-	 * race with user space which kills the process and leaves QP and
-	 * counters orphans.
-	 *
-	 * It is not a big deal because exitted task will leave both QP and
-	 * counter in the same bucket of zombie process. Just ensure that
-	 * process is still alive before procedding.
-	 *
-	 */
-	if (task_pid_nr(counter->res.task) != task_pid_nr(qp->res.task) ||
-	    !task_pid_nr(qp->res.task))
-		return false;
 	if (auto_mask & RDMA_COUNTER_MASK_QP_TYPE)
 		match &= (param->qp_type == qp->qp_type);
+	if (auto_mask & RDMA_COUNTER_MASK_PID)
+		match &= (task_pid_nr(counter->res.task) ==
+			  task_pid_nr(qp->res.task));
 	return match;
 }
 	struct rdma_counter *counter;
 	int ret;
-	if (!qp->res.valid)
+	if (!qp->res.valid || rdma_is_kernel_res(&qp->res))
 		return 0;
 	if (!rdma_is_port_valid(dev, port))
 		goto err;
 	}
-	if (counter->res.task != qp->res.task) {
+	if (rdma_is_kernel_res(&counter->res) != rdma_is_kernel_res(&qp->res)) {
 		ret = -EINVAL;
 		goto err_task;
 	}

drivers/infiniband/core/device.c+22−6

 	} mandatory_table[] = {
 		IB_MANDATORY_FUNC(query_device),
 		IB_MANDATORY_FUNC(query_port),
-		IB_MANDATORY_FUNC(query_pkey),
 		IB_MANDATORY_FUNC(alloc_pd),
 		IB_MANDATORY_FUNC(dealloc_pd),
 		IB_MANDATORY_FUNC(create_qp),
 	return ret;
 }
+static void prevent_dealloc_device(struct ib_device *ib_dev)
+{
+}
 /**
  * ib_register_device - Register an IB device with IB core
  * @device: Device to register
 		 * possibility for a parallel unregistration along with this
 		 * error flow. Since we have a refcount here we know any
 		 * parallel flow is stopped in disable_device and will see the
-		 * NULL pointers, causing the responsibility to
+		 * special dealloc_driver pointer, causing the responsibility to
 		 * ib_dealloc_device() to revert back to this thread.
 		 */
 		dealloc_fn = device->ops.dealloc_driver;
-		device->ops.dealloc_driver = NULL;
+		device->ops.dealloc_driver = prevent_dealloc_device;
 		ib_device_put(device);
 		__ib_unregister_device(device);
 		device->ops.dealloc_driver = dealloc_fn;
 	 * Drivers using the new flow may not call ib_dealloc_device except
 	 * in error unwind prior to registration success.
 	 */
-	if (ib_dev->ops.dealloc_driver) {
+	if (ib_dev->ops.dealloc_driver &&
+	    ib_dev->ops.dealloc_driver != prevent_dealloc_device) {
 		WARN_ON(kref_read(&ib_dev->dev.kobj.kref) <= 1);
 		ib_dealloc_device(ib_dev);
 	}
 	if (!rdma_is_port_valid(device, port_num))
 		return -EINVAL;
+	if (!device->ops.query_pkey)
+		return -EOPNOTSUPP;
 	return device->ops.query_pkey(device, port_num, index, pkey);
 }
 EXPORT_SYMBOL(ib_query_pkey);
 	SET_DEVICE_OP(dev_ops, drain_rq);
 	SET_DEVICE_OP(dev_ops, drain_sq);
 	SET_DEVICE_OP(dev_ops, enable_driver);
-	SET_DEVICE_OP(dev_ops, fill_res_entry);
+	SET_DEVICE_OP(dev_ops, fill_res_cm_id_entry);
-	SET_DEVICE_OP(dev_ops, fill_stat_entry);
+	SET_DEVICE_OP(dev_ops, fill_res_cq_entry);
+	SET_DEVICE_OP(dev_ops, fill_res_cq_entry_raw);
+	SET_DEVICE_OP(dev_ops, fill_res_mr_entry);
+	SET_DEVICE_OP(dev_ops, fill_res_mr_entry_raw);
+	SET_DEVICE_OP(dev_ops, fill_res_qp_entry);
+	SET_DEVICE_OP(dev_ops, fill_res_qp_entry_raw);
+	SET_DEVICE_OP(dev_ops, fill_stat_mr_entry);
 	SET_DEVICE_OP(dev_ops, get_dev_fw_str);
 	SET_DEVICE_OP(dev_ops, get_dma_mr);
 	SET_DEVICE_OP(dev_ops, get_hw_stats);
 	SET_DEVICE_OP(dev_ops, query_port);
 	SET_DEVICE_OP(dev_ops, query_qp);
 	SET_DEVICE_OP(dev_ops, query_srq);
+	SET_DEVICE_OP(dev_ops, query_ucontext);
 	SET_DEVICE_OP(dev_ops, rdma_netdev_get_params);
 	SET_DEVICE_OP(dev_ops, read_counters);
 	SET_DEVICE_OP(dev_ops, reg_dm_mr);
 	SET_DEVICE_OP(dev_ops, set_vf_link_state);
 	SET_OBJ_SIZE(dev_ops, ib_ah);
+	SET_OBJ_SIZE(dev_ops, ib_counters);
 	SET_OBJ_SIZE(dev_ops, ib_cq);
 	SET_OBJ_SIZE(dev_ops, ib_pd);
 	SET_OBJ_SIZE(dev_ops, ib_srq);
 	SET_OBJ_SIZE(dev_ops, ib_ucontext);
+	SET_OBJ_SIZE(dev_ops, ib_xrcd);
 }
 EXPORT_SYMBOL(ib_set_device_ops);

drivers/infiniband/core/mad.c+15−15

@@ -402,7 +402,7 @@ struct ib_mad_agent ib_register_mad_agent(struct ib_device device, · 401 unmodified lines
402	INIT_DELAYED_WORK(&mad_agent_priv->timed_work, timeout_sends);	402	INIT_DELAYED_WORK(&mad_agent_priv->timed_work, timeout_sends);
403	INIT_LIST_HEAD(&mad_agent_priv->local_list);	403	INIT_LIST_HEAD(&mad_agent_priv->local_list);
404	INIT_WORK(&mad_agent_priv->local_work, local_completions);	404	INIT_WORK(&mad_agent_priv->local_work, local_completions);
405	atomic_set(&mad_agent_priv->refcount, 1);	405	refcount_set(&mad_agent_priv->refcount, 1);
406	init_completion(&mad_agent_priv->comp);	406	init_completion(&mad_agent_priv->comp);
407		407
408	ret2 = ib_mad_agent_security_setup(&mad_agent_priv->agent, qp_type);	408	ret2 = ib_mad_agent_security_setup(&mad_agent_priv->agent, qp_type);
@@ -484,7 +484,7 @@ EXPORT_SYMBOL(ib_register_mad_agent); · 75 unmodified lines
484		484
485	static inline void deref_mad_agent(struct ib_mad_agent_private *mad_agent_priv)	485	static inline void deref_mad_agent(struct ib_mad_agent_private *mad_agent_priv)
486	{	486	{
487	if (atomic_dec_and_test(&mad_agent_priv->refcount))	487	if (refcount_dec_and_test(&mad_agent_priv->refcount))
488	complete(&mad_agent_priv->comp);	488	complete(&mad_agent_priv->comp);
489	}	489	}
490		490
@@ -718,7 +718,7 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv, · 227 unmodified lines
718	* Reference MAD agent until receive	718	* Reference MAD agent until receive
719	* side of local completion handled	719	* side of local completion handled
720	*/	720	*/
721	atomic_inc(&mad_agent_priv->refcount);	721	refcount_inc(&mad_agent_priv->refcount);
722	} else	722	} else
723	kfree(mad_priv);	723	kfree(mad_priv);
724	break;	724	break;
@@ -758,7 +758,7 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv, · 33 unmodified lines
758	local->return_wc_byte_len = mad_size;	758	local->return_wc_byte_len = mad_size;
759	}	759	}
760	/* Reference MAD agent until send side of local completion handled */	760	/* Reference MAD agent until send side of local completion handled */
761	atomic_inc(&mad_agent_priv->refcount);	761	refcount_inc(&mad_agent_priv->refcount);
762	/* Queue local completion to local list */	762	/* Queue local completion to local list */
763	spin_lock_irqsave(&mad_agent_priv->lock, flags);	763	spin_lock_irqsave(&mad_agent_priv->lock, flags);
764	list_add_tail(&local->completion_list, &mad_agent_priv->local_list);	764	list_add_tail(&local->completion_list, &mad_agent_priv->local_list);
@@ -916,7 +916,7 @@ struct ib_mad_send_buf * ib_create_send_mad(struct ib_mad_agent *mad_agent, · 151 unmodified lines
916	}	916	}
917		917
918	mad_send_wr->send_buf.mad_agent = mad_agent;	918	mad_send_wr->send_buf.mad_agent = mad_agent;
919	atomic_inc(&mad_agent_priv->refcount);	919	refcount_inc(&mad_agent_priv->refcount);
920	return &mad_send_wr->send_buf;	920	return &mad_send_wr->send_buf;
921	}	921	}
922	EXPORT_SYMBOL(ib_create_send_mad);	922	EXPORT_SYMBOL(ib_create_send_mad);
@@ -1131,7 +1131,7 @@ int ib_post_send_mad(struct ib_mad_send_buf *send_buf, · 208 unmodified lines
1131	mad_send_wr->status = IB_WC_SUCCESS;	1131	mad_send_wr->status = IB_WC_SUCCESS;
1132		1132
1133	/* Reference MAD agent until send completes */	1133	/* Reference MAD agent until send completes */
1134	atomic_inc(&mad_agent_priv->refcount);	1134	refcount_inc(&mad_agent_priv->refcount);
1135	spin_lock_irqsave(&mad_agent_priv->lock, flags);	1135	spin_lock_irqsave(&mad_agent_priv->lock, flags);
1136	list_add_tail(&mad_send_wr->agent_list,	1136	list_add_tail(&mad_send_wr->agent_list,
1137	&mad_agent_priv->send_list);	1137	&mad_agent_priv->send_list);
@@ -1148,7 +1148,7 @@ int ib_post_send_mad(struct ib_mad_send_buf *send_buf, · 10 unmodified lines
1148	spin_lock_irqsave(&mad_agent_priv->lock, flags);	1148	spin_lock_irqsave(&mad_agent_priv->lock, flags);
1149	list_del(&mad_send_wr->agent_list);	1149	list_del(&mad_send_wr->agent_list);
1150	spin_unlock_irqrestore(&mad_agent_priv->lock, flags);	1150	spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
1151	atomic_dec(&mad_agent_priv->refcount);	1151	deref_mad_agent(mad_agent_priv);
1152	goto error;	1152	goto error;
1153	}	1153	}
1154	}	1154	}
@@ -1554,7 +1554,7 @@ find_mad_agent(struct ib_mad_port_private *port_priv, · 399 unmodified lines
1554	hi_tid = be64_to_cpu(mad_hdr->tid) >> 32;	1554	hi_tid = be64_to_cpu(mad_hdr->tid) >> 32;
1555	rcu_read_lock();	1555	rcu_read_lock();
1556	mad_agent = xa_load(&ib_mad_clients, hi_tid);	1556	mad_agent = xa_load(&ib_mad_clients, hi_tid);
1557	if (mad_agent && !atomic_inc_not_zero(&mad_agent->refcount))	1557	if (mad_agent && !refcount_inc_not_zero(&mad_agent->refcount))
1558	mad_agent = NULL;	1558	mad_agent = NULL;
1559	rcu_read_unlock();	1559	rcu_read_unlock();
1560	} else {	1560	} else {
@@ -1606,7 +1606,7 @@ find_mad_agent(struct ib_mad_port_private *port_priv, · 45 unmodified lines
1606	}	1606	}
1607	}	1607	}
1608	if (mad_agent)	1608	if (mad_agent)
1609	atomic_inc(&mad_agent->refcount);	1609	refcount_inc(&mad_agent->refcount);
1610	out:	1610	out:
1611	spin_unlock_irqrestore(&port_priv->reg_lock, flags);	1611	spin_unlock_irqrestore(&port_priv->reg_lock, flags);
1612	}	1612	}
@@ -1831,7 +1831,7 @@ static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv, · 218 unmodified lines
1831	mad_agent_priv->agent.recv_handler(	1831	mad_agent_priv->agent.recv_handler(
1832	&mad_agent_priv->agent, NULL,	1832	&mad_agent_priv->agent, NULL,
1833	mad_recv_wc);	1833	mad_recv_wc);
1834	atomic_dec(&mad_agent_priv->refcount);	1834	deref_mad_agent(mad_agent_priv);
1835	} else {	1835	} else {
1836	/* not user rmpp, revert to normal behavior and	1836	/* not user rmpp, revert to normal behavior and
1837	* drop the mad */	1837	* drop the mad */
@@ -1848,7 +1848,7 @@ static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv, · 10 unmodified lines
1848	&mad_agent_priv->agent,	1848	&mad_agent_priv->agent,
1849	&mad_send_wr->send_buf,	1849	&mad_send_wr->send_buf,
1850	mad_recv_wc);	1850	mad_recv_wc);
1851	atomic_dec(&mad_agent_priv->refcount);	1851	deref_mad_agent(mad_agent_priv);
1852		1852
1853	mad_send_wc.status = IB_WC_SUCCESS;	1853	mad_send_wc.status = IB_WC_SUCCESS;
1854	mad_send_wc.vendor_err = 0;	1854	mad_send_wc.vendor_err = 0;
@@ -2438,7 +2438,7 @@ static void cancel_mads(struct ib_mad_agent_private *mad_agent_priv) · 583 unmodified lines
2438	list_del(&mad_send_wr->agent_list);	2438	list_del(&mad_send_wr->agent_list);
2439	mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,	2439	mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
2440	&mad_send_wc);	2440	&mad_send_wc);
2441	atomic_dec(&mad_agent_priv->refcount);	2441	deref_mad_agent(mad_agent_priv);
2442	}	2442	}
2443	}	2443	}
2444		2444
@@ -2572,7 +2572,7 @@ static void local_completions(struct work_struct *work) · 127 unmodified lines
2572	&local->mad_send_wr->send_buf,	2572	&local->mad_send_wr->send_buf,
2573	&local->mad_priv->header.recv_wc);	2573	&local->mad_priv->header.recv_wc);
2574	spin_lock_irqsave(&recv_mad_agent->lock, flags);	2574	spin_lock_irqsave(&recv_mad_agent->lock, flags);
2575	atomic_dec(&recv_mad_agent->refcount);	2575	deref_mad_agent(recv_mad_agent);
2576	spin_unlock_irqrestore(&recv_mad_agent->lock, flags);	2576	spin_unlock_irqrestore(&recv_mad_agent->lock, flags);
2577	}	2577	}
2578		2578
@@ -2585,7 +2585,7 @@ local_send_completion: · 6 unmodified lines
2585	&mad_send_wc);	2585	&mad_send_wc);
2586		2586
2587	spin_lock_irqsave(&mad_agent_priv->lock, flags);	2587	spin_lock_irqsave(&mad_agent_priv->lock, flags);
2588	atomic_dec(&mad_agent_priv->refcount);	2588	deref_mad_agent(mad_agent_priv);
2589	if (free_mad)	2589	if (free_mad)
2590	kfree(local->mad_priv);	2590	kfree(local->mad_priv);
2591	kfree(local);	2591	kfree(local);
@@ -2671,7 +2671,7 @@ static void timeout_sends(struct work_struct *work) · 79 unmodified lines
2671	mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,	2671	mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
2672	&mad_send_wc);	2672	&mad_send_wc);
2673		2673
2674	atomic_dec(&mad_agent_priv->refcount);	2674	deref_mad_agent(mad_agent_priv);
2675	spin_lock_irqsave(&mad_agent_priv->lock, flags);	2675	spin_lock_irqsave(&mad_agent_priv->lock, flags);
2676	}	2676	}
2677	spin_unlock_irqrestore(&mad_agent_priv->lock, flags);	2677	spin_unlock_irqrestore(&mad_agent_priv->lock, flags);

drivers/infiniband/core/mad_priv.h+1−1

@@ -103,7 +103,7 @@ struct ib_mad_agent_private { · 102 unmodified lines
103	struct work_struct local_work;	103	struct work_struct local_work;
104	struct list_head rmpp_list;	104	struct list_head rmpp_list;
105		105
106	atomic_t refcount;	106	refcount_t refcount;
107	union {	107	union {
108	struct completion comp;	108	struct completion comp;
109	struct rcu_head rcu;	109	struct rcu_head rcu;

drivers/infiniband/core/mad_rmpp.c+9−18

 enum rmpp_state {
 	RMPP_STATE_ACTIVE,
 	RMPP_STATE_TIMEOUT,
-	RMPP_STATE_COMPLETE,
+	RMPP_STATE_COMPLETE
-	RMPP_STATE_CANCELING
 };
 struct mad_rmpp_recv {
 	struct completion comp;
 	enum rmpp_state state;
 	spinlock_t lock;
-	atomic_t refcount;
+	refcount_t refcount;
 	struct ib_ah *ah;
 	struct ib_mad_recv_wc *rmpp_wc;
 static inline void deref_rmpp_recv(struct mad_rmpp_recv *rmpp_recv)
 {
-	if (atomic_dec_and_test(&rmpp_recv->refcount))
+	if (refcount_dec_and_test(&rmpp_recv->refcount))
 		complete(&rmpp_recv->comp);
 }
 	unsigned long flags;
 	spin_lock_irqsave(&agent->lock, flags);
-	list_for_each_entry(rmpp_recv, &agent->rmpp_list, list) {
-		if (rmpp_recv->state != RMPP_STATE_COMPLETE)
-			ib_free_recv_mad(rmpp_recv->rmpp_wc);
-		rmpp_recv->state = RMPP_STATE_CANCELING;
-	}
-	spin_unlock_irqrestore(&agent->lock, flags);
 	list_for_each_entry(rmpp_recv, &agent->rmpp_list, list) {
 		cancel_delayed_work(&rmpp_recv->timeout_work);
 		cancel_delayed_work(&rmpp_recv->cleanup_work);
 	}
+	spin_unlock_irqrestore(&agent->lock, flags);
 	flush_workqueue(agent->qp_info->port_priv->wq);
 	list_for_each_entry_safe(rmpp_recv, temp_rmpp_recv,
 				 &agent->rmpp_list, list) {
 		list_del(&rmpp_recv->list);
+		if (rmpp_recv->state != RMPP_STATE_COMPLETE)
+			ib_free_recv_mad(rmpp_recv->rmpp_wc);
 		destroy_rmpp_recv(rmpp_recv);
 	}
 }
 	unsigned long flags;
 	spin_lock_irqsave(&rmpp_recv->agent->lock, flags);
-	if (rmpp_recv->state == RMPP_STATE_CANCELING) {
-		spin_unlock_irqrestore(&rmpp_recv->agent->lock, flags);
-		return;
-	}
 	list_del(&rmpp_recv->list);
 	spin_unlock_irqrestore(&rmpp_recv->agent->lock, flags);
 	destroy_rmpp_recv(rmpp_recv);
 	INIT_DELAYED_WORK(&rmpp_recv->cleanup_work, recv_cleanup_handler);
 	spin_lock_init(&rmpp_recv->lock);
 	rmpp_recv->state = RMPP_STATE_ACTIVE;
-	atomic_set(&rmpp_recv->refcount, 1);
+	refcount_set(&rmpp_recv->refcount, 1);
 	rmpp_recv->rmpp_wc = mad_recv_wc;
 	rmpp_recv->cur_seg_buf = &mad_recv_wc->recv_buf;
 	spin_lock_irqsave(&agent->lock, flags);
 	rmpp_recv = find_rmpp_recv(agent, mad_recv_wc);
 	if (rmpp_recv)
-		atomic_inc(&rmpp_recv->refcount);
+		refcount_inc(&rmpp_recv->refcount);
 	spin_unlock_irqrestore(&agent->lock, flags);
 	return rmpp_recv;
 }
 		destroy_rmpp_recv(rmpp_recv);
 		return continue_rmpp(agent, mad_recv_wc);
 	}
-	atomic_inc(&rmpp_recv->refcount);
+	refcount_inc(&rmpp_recv->refcount);
 	if (get_last_flag(&mad_recv_wc->recv_buf)) {
 		rmpp_recv->state = RMPP_STATE_COMPLETE;

drivers/infiniband/core/nldev.c+126−97

 	[RDMA_NLDEV_ATTR_RES_PS]		= { .type = NLA_U32 },
 	[RDMA_NLDEV_ATTR_RES_QP]		= { .type = NLA_NESTED },
 	[RDMA_NLDEV_ATTR_RES_QP_ENTRY]		= { .type = NLA_NESTED },
+	[RDMA_NLDEV_ATTR_RES_RAW]		= { .type = NLA_BINARY },
 	[RDMA_NLDEV_ATTR_RES_RKEY]		= { .type = NLA_U32 },
 	[RDMA_NLDEV_ATTR_RES_RQPN]		= { .type = NLA_U32 },
 	[RDMA_NLDEV_ATTR_RES_RQ_PSN]		= { .type = NLA_U32 },
 	return err ? -EMSGSIZE : 0;
 }
-static bool fill_res_entry(struct ib_device *dev, struct sk_buff *msg,
+static int fill_res_qp_entry_query(struct sk_buff *msg,
-			   struct rdma_restrack_entry *res)
+				   struct rdma_restrack_entry *res,
+				   struct ib_device *dev,
+				   struct ib_qp *qp)
 {
-	if (!dev->ops.fill_res_entry)
-		return false;
-	return dev->ops.fill_res_entry(msg, res);
-}
-static bool fill_stat_entry(struct ib_device *dev, struct sk_buff *msg,
-			    struct rdma_restrack_entry *res)
-{
-	if (!dev->ops.fill_stat_entry)
-		return false;
-	return dev->ops.fill_stat_entry(msg, res);
-}
-static int fill_res_qp_entry(struct sk_buff *msg, bool has_cap_net_admin,
-			     struct rdma_restrack_entry *res, uint32_t port)
-{
-	struct ib_qp *qp = container_of(res, struct ib_qp, res);
-	struct ib_device *dev = qp->device;
 	struct ib_qp_init_attr qp_init_attr;
 	struct ib_qp_attr qp_attr;
 	int ret;
 	if (ret)
 		return ret;
-	if (port && port != qp_attr.port_num)
-		return -EAGAIN;
-	/* In create_qp() port is not set yet */
-	if (qp_attr.port_num &&
-	    nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, qp_attr.port_num))
-		goto err;
-	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qp->qp_num))
-		goto err;
 	if (qp->qp_type == IB_QPT_RC || qp->qp_type == IB_QPT_UC) {
 		if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RQPN,
 				qp_attr.dest_qp_num))
 	if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_STATE, qp_attr.qp_state))
 		goto err;
+	if (dev->ops.fill_res_qp_entry)
+		return dev->ops.fill_res_qp_entry(msg, qp);
+	return 0;
+err:	return -EMSGSIZE;
+}
+static int fill_res_qp_entry(struct sk_buff *msg, bool has_cap_net_admin,
+			     struct rdma_restrack_entry *res, uint32_t port)
+{
+	struct ib_qp *qp = container_of(res, struct ib_qp, res);
+	struct ib_device *dev = qp->device;
+	int ret;
+	if (port && port != qp->port)
+		return -EAGAIN;
+	/* In create_qp() port is not set yet */
+	if (qp->port && nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, qp->port))
+		return -EINVAL;
+	ret = nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qp->qp_num);
+	if (ret)
+		return -EMSGSIZE;
 	if (!rdma_is_kernel_res(res) &&
 	    nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, qp->pd->res.id))
-		goto err;
+		return -EMSGSIZE;
-	if (fill_res_name_pid(msg, res))
+	ret = fill_res_name_pid(msg, res);
-		goto err;
+	if (ret)
+		return -EMSGSIZE;
-	if (fill_res_entry(dev, msg, res))
+	return fill_res_qp_entry_query(msg, res, dev, qp);
-		goto err;
+}
-	return 0;
+static int fill_res_qp_raw_entry(struct sk_buff *msg, bool has_cap_net_admin,
+				 struct rdma_restrack_entry *res, uint32_t port)
+{
+	struct ib_qp *qp = container_of(res, struct ib_qp, res);
+	struct ib_device *dev = qp->device;
-err:	return -EMSGSIZE;
+	if (port && port != qp->port)
+		return -EAGAIN;
+	if (!dev->ops.fill_res_qp_entry_raw)
+		return -EINVAL;
+	return dev->ops.fill_res_qp_entry_raw(msg, qp);
 }
 static int fill_res_cm_id_entry(struct sk_buff *msg, bool has_cap_net_admin,
 	if (fill_res_name_pid(msg, res))
 		goto err;
-	if (fill_res_entry(dev, msg, res))
+	if (dev->ops.fill_res_cm_id_entry)
-		goto err;
+		return dev->ops.fill_res_cm_id_entry(msg, cm_id);
 	return 0;
 err: return -EMSGSIZE;
 	struct ib_device *dev = cq->device;
 	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQE, cq->cqe))
-		goto err;
+		return -EMSGSIZE;
 	if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_USECNT,
 			      atomic_read(&cq->usecnt), RDMA_NLDEV_ATTR_PAD))
-		goto err;
+		return -EMSGSIZE;
 	/* Poll context is only valid for kernel CQs */
 	if (rdma_is_kernel_res(res) &&
 	    nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_POLL_CTX, cq->poll_ctx))
-		goto err;
+		return -EMSGSIZE;
 	if (nla_put_u8(msg, RDMA_NLDEV_ATTR_DEV_DIM, (cq->dim != NULL)))
-		goto err;
+		return -EMSGSIZE;
 	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQN, res->id))
-		goto err;
+		return -EMSGSIZE;
 	if (!rdma_is_kernel_res(res) &&
 	    nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN,
 			cq->uobject->uevent.uobject.context->res.id))
-		goto err;
+		return -EMSGSIZE;
 	if (fill_res_name_pid(msg, res))
-		goto err;
+		return -EMSGSIZE;
-	if (fill_res_entry(dev, msg, res))
+	return (dev->ops.fill_res_cq_entry) ?
-		goto err;
+		dev->ops.fill_res_cq_entry(msg, cq) : 0;
+}
-	return 0;
+static int fill_res_cq_raw_entry(struct sk_buff *msg, bool has_cap_net_admin,
+				 struct rdma_restrack_entry *res, uint32_t port)
+{
+	struct ib_cq *cq = container_of(res, struct ib_cq, res);
+	struct ib_device *dev = cq->device;
-err:	return -EMSGSIZE;
+	if (!dev->ops.fill_res_cq_entry_raw)
+		return -EINVAL;
+	return dev->ops.fill_res_cq_entry_raw(msg, cq);
 }
 static int fill_res_mr_entry(struct sk_buff *msg, bool has_cap_net_admin,
 	if (has_cap_net_admin) {
 		if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RKEY, mr->rkey))
-			goto err;
+			return -EMSGSIZE;
 		if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LKEY, mr->lkey))
-			goto err;
+			return -EMSGSIZE;
 	}
 	if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_MRLEN, mr->length,
 			      RDMA_NLDEV_ATTR_PAD))
-		goto err;
+		return -EMSGSIZE;
 	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res->id))
-		goto err;
+		return -EMSGSIZE;
 	if (!rdma_is_kernel_res(res) &&
 	    nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, mr->pd->res.id))
-		goto err;
+		return -EMSGSIZE;
 	if (fill_res_name_pid(msg, res))
-		goto err;
+		return -EMSGSIZE;
-	if (fill_res_entry(dev, msg, res))
+	return (dev->ops.fill_res_mr_entry) ?
-		goto err;
+		       dev->ops.fill_res_mr_entry(msg, mr) :
+		       0;
+}
-	return 0;
+static int fill_res_mr_raw_entry(struct sk_buff *msg, bool has_cap_net_admin,
+				 struct rdma_restrack_entry *res, uint32_t port)
+{
+	struct ib_mr *mr = container_of(res, struct ib_mr, res);
+	struct ib_device *dev = mr->pd->device;
-err:	return -EMSGSIZE;
+	if (!dev->ops.fill_res_mr_entry_raw)
+		return -EINVAL;
+	return dev->ops.fill_res_mr_entry_raw(msg, mr);
 }
 static int fill_res_pd_entry(struct sk_buff *msg, bool has_cap_net_admin,
 			     struct rdma_restrack_entry *res, uint32_t port)
 {
 	struct ib_pd *pd = container_of(res, struct ib_pd, res);
-	struct ib_device *dev = pd->device;
 	if (has_cap_net_admin) {
 		if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY,
 			pd->uobject->context->res.id))
 		goto err;
-	if (fill_res_name_pid(msg, res))
+	return fill_res_name_pid(msg, res);
-		goto err;
-	if (fill_res_entry(dev, msg, res))
-		goto err;
-	return 0;
 err:	return -EMSGSIZE;
 }
 	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_MODE, m->mode))
 		return -EMSGSIZE;
-	if (m->mode == RDMA_COUNTER_MODE_AUTO)
+	if (m->mode == RDMA_COUNTER_MODE_AUTO) {
 		if ((m->mask & RDMA_COUNTER_MASK_QP_TYPE) &&
 		    nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, m->param.qp_type))
 			return -EMSGSIZE;
+		if ((m->mask & RDMA_COUNTER_MASK_PID) &&
+		    fill_res_name_pid(msg, &counter->res))
+			return -EMSGSIZE;
+	}
 	return 0;
 }
 	xa_lock(&rt->xa);
 	xa_for_each(&rt->xa, id, res) {
 		qp = container_of(res, struct ib_qp, res);
-		if (qp->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW))
-			continue;
 		if (!qp->counter || (qp->counter->id != counter->id))
 			continue;
 	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res->id))
 		goto err;
-	if (fill_stat_entry(dev, msg, res))
+	if (dev->ops.fill_stat_mr_entry)
-		goto err;
+		return dev->ops.fill_stat_mr_entry(msg, mr);
 	return 0;
 err:
 	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, counter->port) ||
 	    nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_COUNTER_ID, counter->id) ||
-	    fill_res_name_pid(msg, &counter->res) ||
 	    fill_stat_counter_mode(msg, counter) ||
 	    fill_stat_counter_qps(msg, counter) ||
 	    fill_stat_counter_hwcounters(msg, counter))
 struct nldev_fill_res_entry {
 	enum rdma_nldev_attr nldev_attr;
-	enum rdma_nldev_command nldev_cmd;
 	u8 flags;
 	u32 entry;
 	u32 id;
 static const struct nldev_fill_res_entry fill_entries[RDMA_RESTRACK_MAX] = {
 	[RDMA_RESTRACK_QP] = {
-		.nldev_cmd = RDMA_NLDEV_CMD_RES_QP_GET,
 		.nldev_attr = RDMA_NLDEV_ATTR_RES_QP,
 		.entry = RDMA_NLDEV_ATTR_RES_QP_ENTRY,
 		.id = RDMA_NLDEV_ATTR_RES_LQPN,
 	},
 	[RDMA_RESTRACK_CM_ID] = {
-		.nldev_cmd = RDMA_NLDEV_CMD_RES_CM_ID_GET,
 		.nldev_attr = RDMA_NLDEV_ATTR_RES_CM_ID,
 		.entry = RDMA_NLDEV_ATTR_RES_CM_ID_ENTRY,
 		.id = RDMA_NLDEV_ATTR_RES_CM_IDN,
 	},
 	[RDMA_RESTRACK_CQ] = {
-		.nldev_cmd = RDMA_NLDEV_CMD_RES_CQ_GET,
 		.nldev_attr = RDMA_NLDEV_ATTR_RES_CQ,
 		.flags = NLDEV_PER_DEV,
 		.entry = RDMA_NLDEV_ATTR_RES_CQ_ENTRY,
 		.id = RDMA_NLDEV_ATTR_RES_CQN,
 	},
 	[RDMA_RESTRACK_MR] = {
-		.nldev_cmd = RDMA_NLDEV_CMD_RES_MR_GET,
 		.nldev_attr = RDMA_NLDEV_ATTR_RES_MR,
 		.flags = NLDEV_PER_DEV,
 		.entry = RDMA_NLDEV_ATTR_RES_MR_ENTRY,
 		.id = RDMA_NLDEV_ATTR_RES_MRN,
 	},
 	[RDMA_RESTRACK_PD] = {
-		.nldev_cmd = RDMA_NLDEV_CMD_RES_PD_GET,
 		.nldev_attr = RDMA_NLDEV_ATTR_RES_PD,
 		.flags = NLDEV_PER_DEV,
 		.entry = RDMA_NLDEV_ATTR_RES_PD_ENTRY,
 		.id = RDMA_NLDEV_ATTR_RES_PDN,
 	},
 	[RDMA_RESTRACK_COUNTER] = {
-		.nldev_cmd = RDMA_NLDEV_CMD_STAT_GET,
 		.nldev_attr = RDMA_NLDEV_ATTR_STAT_COUNTER,
 		.entry = RDMA_NLDEV_ATTR_STAT_COUNTER_ENTRY,
 		.id = RDMA_NLDEV_ATTR_STAT_COUNTER_ID,
 	}
 	nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
-			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, fe->nldev_cmd),
+			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
+					 RDMA_NL_GET_OP(nlh->nlmsg_type)),
 			0, 0);
 	if (fill_nldev_handle(msg, device)) {
 	}
 	nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
-			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, fe->nldev_cmd),
+			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
+					 RDMA_NL_GET_OP(cb->nlh->nlmsg_type)),
 			0, NLM_F_MULTI);
 	if (fill_nldev_handle(skb, device)) {
 	return ret;
 }
-#define RES_GET_FUNCS(name, type)					       \
+#define RES_GET_FUNCS(name, type)                                              \
-	static int nldev_res_get_##name##_dumpit(struct sk_buff *skb,	       \
+	static int nldev_res_get_##name##_dumpit(struct sk_buff *skb,          \
 						 struct netlink_callback *cb)  \
-	{								       \
+	{                                                                      \
-		return res_get_common_dumpit(skb, cb, type,		       \
+		return res_get_common_dumpit(skb, cb, type,                    \
-					     fill_res_##name##_entry);	       \
+					     fill_res_##name##_entry);         \
-	}								       \
+	}                                                                      \
-	static int nldev_res_get_##name##_doit(struct sk_buff *skb,	       \
+	static int nldev_res_get_##name##_doit(struct sk_buff *skb,            \
-					       struct nlmsghdr *nlh,	       \
+					       struct nlmsghdr *nlh,           \
 					       struct netlink_ext_ack *extack) \
-	{								       \
+	{                                                                      \
-		return res_get_common_doit(skb, nlh, extack, type,	       \
+		return res_get_common_doit(skb, nlh, extack, type,             \
-					   fill_res_##name##_entry);	       \
+					   fill_res_##name##_entry);           \
 	}
 RES_GET_FUNCS(qp, RDMA_RESTRACK_QP);
+RES_GET_FUNCS(qp_raw, RDMA_RESTRACK_QP);
 RES_GET_FUNCS(cm_id, RDMA_RESTRACK_CM_ID);
 RES_GET_FUNCS(cq, RDMA_RESTRACK_CQ);
+RES_GET_FUNCS(cq_raw, RDMA_RESTRACK_CQ);
 RES_GET_FUNCS(pd, RDMA_RESTRACK_PD);
 RES_GET_FUNCS(mr, RDMA_RESTRACK_MR);
+RES_GET_FUNCS(mr_raw, RDMA_RESTRACK_MR);
 RES_GET_FUNCS(counter, RDMA_RESTRACK_COUNTER);
 static LIST_HEAD(link_ops);
 		.doit = nldev_stat_del_doit,
 		.flags = RDMA_NL_ADMIN_PERM,
 	},
+	[RDMA_NLDEV_CMD_RES_QP_GET_RAW] = {
+		.doit = nldev_res_get_qp_raw_doit,
+		.dump = nldev_res_get_qp_raw_dumpit,
+		.flags = RDMA_NL_ADMIN_PERM,
+	},
+	[RDMA_NLDEV_CMD_RES_CQ_GET_RAW] = {
+		.doit = nldev_res_get_cq_raw_doit,
+		.dump = nldev_res_get_cq_raw_dumpit,
+		.flags = RDMA_NL_ADMIN_PERM,
+	},
+	[RDMA_NLDEV_CMD_RES_MR_GET_RAW] = {
+		.doit = nldev_res_get_mr_raw_doit,
+		.dump = nldev_res_get_mr_raw_dumpit,
+		.flags = RDMA_NL_ADMIN_PERM,
+	},
 };
 void __init nldev_init(void)

drivers/infiniband/core/sysfs.c+41−20

 	struct ib_device      *ibdev;
 	struct gid_attr_group *gid_attr_group;
 	struct attribute_group gid_group;
-	struct attribute_group pkey_group;
+	struct attribute_group *pkey_group;
 	struct attribute_group *pma_table;
 	struct attribute_group *hw_stats_ag;
 	struct rdma_hw_stats   *hw_stats;
 		kfree(p->gid_group.attrs);
 	}
-	if (p->pkey_group.attrs) {
+	if (p->pkey_group) {
-		for (i = 0; (a = p->pkey_group.attrs[i]); ++i)
+		if (p->pkey_group->attrs) {
-			kfree(a);
+			for (i = 0; (a = p->pkey_group->attrs[i]); ++i)
+				kfree(a);
+			kfree(p->pkey_group->attrs);
+		}
-		kfree(p->pkey_group.attrs);
+		kfree(p->pkey_group);
+		p->pkey_group = NULL;
 	}
 	kfree(p);
 	if (ret)
 		goto err_free_gid_type;
-	p->pkey_group.name  = "pkeys";
+	if (attr.pkey_tbl_len) {
-	p->pkey_group.attrs = alloc_group_attrs(show_port_pkey,
+		p->pkey_group = kzalloc(sizeof(*p->pkey_group), GFP_KERNEL);
-						attr.pkey_tbl_len);
+		if (!p->pkey_group) {
-	if (!p->pkey_group.attrs) {
+			ret = -ENOMEM;
-		ret = -ENOMEM;
+			goto err_remove_gid_type;
-		goto err_remove_gid_type;
+		}
+		p->pkey_group->name  = "pkeys";
+		p->pkey_group->attrs = alloc_group_attrs(show_port_pkey,
+							 attr.pkey_tbl_len);
+		if (!p->pkey_group->attrs) {
+			ret = -ENOMEM;
+			goto err_free_pkey_group;
+		}
+		ret = sysfs_create_group(&p->kobj, p->pkey_group);
+		if (ret)
+			goto err_free_pkey;
 	}
-	ret = sysfs_create_group(&p->kobj, &p->pkey_group);
-	if (ret)
-		goto err_free_pkey;
 	if (device->ops.init_port && is_full_dev) {
 		ret = device->ops.init_port(device, port_num, &p->kobj);
 	return 0;
 err_remove_pkey:
-	sysfs_remove_group(&p->kobj, &p->pkey_group);
+	if (p->pkey_group)
+		sysfs_remove_group(&p->kobj, p->pkey_group);
 err_free_pkey:
-	for (i = 0; i < attr.pkey_tbl_len; ++i)
+	if (p->pkey_group) {
-		kfree(p->pkey_group.attrs[i]);
+		for (i = 0; i < attr.pkey_tbl_len; ++i)
+			kfree(p->pkey_group->attrs[i]);
+		kfree(p->pkey_group->attrs);
+		p->pkey_group->attrs = NULL;
+	}
-	kfree(p->pkey_group.attrs);
+err_free_pkey_group:
-	p->pkey_group.attrs = NULL;
+	kfree(p->pkey_group);
 err_remove_gid_type:
 	sysfs_remove_group(&p->gid_attr_group->kobj,
 		if (port->pma_table)
 			sysfs_remove_group(p, port->pma_table);
-		sysfs_remove_group(p, &port->pkey_group);
+		if (port->pkey_group)
+			sysfs_remove_group(p, port->pkey_group);
 		sysfs_remove_group(p, &port->gid_group);
 		sysfs_remove_group(&port->gid_attr_group->kobj,
 				   &port->gid_attr_group->ndev);

drivers/infiniband/core/trace.c+0−2

@@ -9,6 +9,4 @@ · 8 unmodified lines
9		9
10	#define CREATE_TRACE_POINTS	10	#define CREATE_TRACE_POINTS
11		11
12	#include <rdma/ib_verbs.h>
13
14	#include <trace/events/rdma_core.h>	12	#include <trace/events/rdma_core.h>

drivers/infiniband/core/umem.c+1−0

@@ -261,6 +261,7 @@ struct ib_umem ib_umem_get(struct ib_device device, unsigned long addr, · 260 unmodified lines
261	sg = umem->sg_head.sgl;	261	sg = umem->sg_head.sgl;
262		262
263	while (npages) {	263	while (npages) {
		264	cond_resched();
264	ret = pin_user_pages_fast(cur_base,	265	ret = pin_user_pages_fast(cur_base,
265	min_t(unsigned long, npages,	266	min_t(unsigned long, npages,
266	PAGE_SIZE /	267	PAGE_SIZE /

drivers/infiniband/core/umem_odp.c+2−0

  *        ib_alloc_implicit_odp_umem()
  * @addr: The starting userspace VA
  * @size: The length of the userspace VA
+ * @ops: MMU interval ops, currently only @invalidate
  */
 struct ib_umem_odp *
 ib_umem_odp_alloc_child(struct ib_umem_odp *root, unsigned long addr,
  * @addr: userspace virtual address to start at
  * @size: length of region to pin
  * @access: IB_ACCESS_xxx flags for memory being pinned
+ * @ops: MMU interval ops, currently only @invalidate
  *
  * The driver should use when the access flags indicate ODP memory. It avoids
  * pinning, instead, stores the mm for future page fault handling in

drivers/infiniband/core/uverbs_cmd.c+99−222

@@ -415,8 +415,8 @@ static int ib_uverbs_query_port(struct uverbs_attr_bundle *attrs) · 414 unmodified lines
415		415
416	static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs)	416	static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs)
417	{	417	{
		418	struct ib_uverbs_alloc_pd_resp resp = {};
418	struct ib_uverbs_alloc_pd cmd;	419	struct ib_uverbs_alloc_pd cmd;
419	struct ib_uverbs_alloc_pd_resp resp;
420	struct ib_uobject *uobj;	420	struct ib_uobject *uobj;
421	struct ib_pd *pd;	421	struct ib_pd *pd;
422	int ret;	422	int ret;
@@ -438,29 +438,20 @@ static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs) · 15 unmodified lines
438		438
439	pd->device = ib_dev;	439	pd->device = ib_dev;
440	pd->uobject = uobj;	440	pd->uobject = uobj;
441	pd->__internal_mr = NULL;
442	atomic_set(&pd->usecnt, 0);	441	atomic_set(&pd->usecnt, 0);
443	pd->res.type = RDMA_RESTRACK_PD;	442	pd->res.type = RDMA_RESTRACK_PD;
444		443
445	ret = ib_dev->ops.alloc_pd(pd, &attrs->driver_udata);	444	ret = ib_dev->ops.alloc_pd(pd, &attrs->driver_udata);
446	if (ret)	445	if (ret)
447	goto err_alloc;	446	goto err_alloc;
448
449	uobj->object = pd;
450	memset(&resp, 0, sizeof resp);
451	resp.pd_handle = uobj->id;
452	rdma_restrack_uadd(&pd->res);	447	rdma_restrack_uadd(&pd->res);
453		448
454	ret = uverbs_response(attrs, &resp, sizeof(resp));	449	uobj->object = pd;
455	if (ret)	450	uobj_finalize_uobj_create(uobj, attrs);
456	goto err_copy;
457		451
458	rdma_alloc_commit_uobject(uobj, attrs);	452	resp.pd_handle = uobj->id;
459	return 0;	453	return uverbs_response(attrs, &resp, sizeof(resp));
460		454
461	err_copy:
462	ib_dealloc_pd_user(pd, uverbs_get_cleared_udata(attrs));
463	pd = NULL;
464	err_alloc:	455	err_alloc:
465	kfree(pd);	456	kfree(pd);
466	err:	457	err:
@@ -568,15 +559,15 @@ static void xrcd_table_delete(struct ib_uverbs_device *dev, · 101 unmodified lines
568	static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs)	559	static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs)
569	{	560	{
570	struct ib_uverbs_device *ibudev = attrs->ufile->device;	561	struct ib_uverbs_device *ibudev = attrs->ufile->device;
		562	struct ib_uverbs_open_xrcd_resp resp = {};
571	struct ib_uverbs_open_xrcd cmd;	563	struct ib_uverbs_open_xrcd cmd;
572	struct ib_uverbs_open_xrcd_resp resp;
573	struct ib_uxrcd_object *obj;	564	struct ib_uxrcd_object *obj;
574	struct ib_xrcd *xrcd = NULL;	565	struct ib_xrcd *xrcd = NULL;
575	struct fd f = {NULL, 0};
576	struct inode *inode = NULL;	566	struct inode *inode = NULL;
577	int ret = 0;
578	int new_xrcd = 0;	567	int new_xrcd = 0;
579	struct ib_device *ib_dev;	568	struct ib_device *ib_dev;
		569	struct fd f = {};
		570	int ret;
580		571
581	ret = uverbs_request(attrs, &cmd, sizeof(cmd));	572	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
582	if (ret)	573	if (ret)
@@ -614,24 +605,16 @@ static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs) · 31 unmodified lines
614	}	605	}
615		606
616	if (!xrcd) {	607	if (!xrcd) {
617	xrcd = ib_dev->ops.alloc_xrcd(ib_dev, &attrs->driver_udata);	608	xrcd = ib_alloc_xrcd_user(ib_dev, inode, &attrs->driver_udata);
618	if (IS_ERR(xrcd)) {	609	if (IS_ERR(xrcd)) {
619	ret = PTR_ERR(xrcd);	610	ret = PTR_ERR(xrcd);
620	goto err;	611	goto err;
621	}	612	}
622
623	xrcd->inode = inode;
624	xrcd->device = ib_dev;
625	atomic_set(&xrcd->usecnt, 0);
626	mutex_init(&xrcd->tgt_qp_mutex);
627	INIT_LIST_HEAD(&xrcd->tgt_qp_list);
628	new_xrcd = 1;	613	new_xrcd = 1;
629	}	614	}
630		615
631	atomic_set(&obj->refcnt, 0);	616	atomic_set(&obj->refcnt, 0);
632	obj->uobject.object = xrcd;	617	obj->uobject.object = xrcd;
633	memset(&resp, 0, sizeof resp);
634	resp.xrcd_handle = obj->uobject.id;
635		618
636	if (inode) {	619	if (inode) {
637	if (new_xrcd) {	620	if (new_xrcd) {
@@ -643,27 +626,17 @@ static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs) · 5 unmodified lines
643	atomic_inc(&xrcd->usecnt);	626	atomic_inc(&xrcd->usecnt);
644	}	627	}
645		628
646	ret = uverbs_response(attrs, &resp, sizeof(resp));
647	if (ret)
648	goto err_copy;
649
650	if (f.file)	629	if (f.file)
651	fdput(f);	630	fdput(f);
652		631
653	mutex_unlock(&ibudev->xrcd_tree_mutex);	632	mutex_unlock(&ibudev->xrcd_tree_mutex);
		633	uobj_finalize_uobj_create(&obj->uobject, attrs);
654		634
655	rdma_alloc_commit_uobject(&obj->uobject, attrs);	635	resp.xrcd_handle = obj->uobject.id;
656	return 0;	636	return uverbs_response(attrs, &resp, sizeof(resp));
657
658	err_copy:
659	if (inode) {
660	if (new_xrcd)
661	xrcd_table_delete(ibudev, inode);
662	atomic_dec(&xrcd->usecnt);
663	}
664		637
665	err_dealloc_xrcd:	638	err_dealloc_xrcd:
666	ib_dealloc_xrcd(xrcd, uverbs_get_cleared_udata(attrs));	639	ib_dealloc_xrcd_user(xrcd, uverbs_get_cleared_udata(attrs));
667		640
668	err:	641	err:
669	uobj_alloc_abort(&obj->uobject, attrs);	642	uobj_alloc_abort(&obj->uobject, attrs);
@@ -701,7 +674,7 @@ int ib_uverbs_dealloc_xrcd(struct ib_uobject uobject, struct ib_xrcd xrcd, · 31 unmodified lines
701	if (inode && !atomic_dec_and_test(&xrcd->usecnt))	674	if (inode && !atomic_dec_and_test(&xrcd->usecnt))
702	return 0;	675	return 0;
703		676
704	ret = ib_dealloc_xrcd(xrcd, &attrs->driver_udata);	677	ret = ib_dealloc_xrcd_user(xrcd, &attrs->driver_udata);
705		678
706	if (ib_is_destroy_retryable(ret, why, uobject)) {	679	if (ib_is_destroy_retryable(ret, why, uobject)) {
707	atomic_inc(&xrcd->usecnt);	680	atomic_inc(&xrcd->usecnt);
@@ -716,8 +689,8 @@ int ib_uverbs_dealloc_xrcd(struct ib_uobject uobject, struct ib_xrcd xrcd, · 8 unmodified lines
716		689
717	static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs)	690	static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs)
718	{	691	{
		692	struct ib_uverbs_reg_mr_resp resp = {};
719	struct ib_uverbs_reg_mr cmd;	693	struct ib_uverbs_reg_mr cmd;
720	struct ib_uverbs_reg_mr_resp resp;
721	struct ib_uobject *uobj;	694	struct ib_uobject *uobj;
722	struct ib_pd *pd;	695	struct ib_pd *pd;
723	struct ib_mr *mr;	696	struct ib_mr *mr;
@@ -770,30 +743,20 @@ static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs) · 46 unmodified lines
770	mr->uobject = uobj;	743	mr->uobject = uobj;
771	atomic_inc(&pd->usecnt);	744	atomic_inc(&pd->usecnt);
772	mr->res.type = RDMA_RESTRACK_MR;	745	mr->res.type = RDMA_RESTRACK_MR;
		746	mr->iova = cmd.hca_va;
773	rdma_restrack_uadd(&mr->res);	747	rdma_restrack_uadd(&mr->res);
774		748
775	uobj->object = mr;	749	uobj->object = mr;
776
777	memset(&resp, 0, sizeof resp);
778	resp.lkey = mr->lkey;
779	resp.rkey = mr->rkey;
780	resp.mr_handle = uobj->id;
781
782	ret = uverbs_response(attrs, &resp, sizeof(resp));
783	if (ret)
784	goto err_copy;
785
786	uobj_put_obj_read(pd);	750	uobj_put_obj_read(pd);
		751	uobj_finalize_uobj_create(uobj, attrs);
787		752
788	rdma_alloc_commit_uobject(uobj, attrs);	753	resp.lkey = mr->lkey;
789	return 0;	754	resp.rkey = mr->rkey;
790		755	resp.mr_handle = uobj->id;
791	err_copy:	756	return uverbs_response(attrs, &resp, sizeof(resp));
792	ib_dereg_mr_user(mr, uverbs_get_cleared_udata(attrs));
793		757
794	err_put:	758	err_put:
795	uobj_put_obj_read(pd);	759	uobj_put_obj_read(pd);
796
797	err_free:	760	err_free:
798	uobj_alloc_abort(uobj, attrs);	761	uobj_alloc_abort(uobj, attrs);
799	return ret;	762	return ret;
@@ -861,6 +824,9 @@ static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs) · 61 unmodified lines
861	atomic_dec(&old_pd->usecnt);	824	atomic_dec(&old_pd->usecnt);
862	}	825	}
863		826
		827	if (cmd.flags & IB_MR_REREG_TRANS)
		828	mr->iova = cmd.hca_va;
		829
864	memset(&resp, 0, sizeof(resp));	830	memset(&resp, 0, sizeof(resp));
865	resp.lkey = mr->lkey;	831	resp.lkey = mr->lkey;
866	resp.rkey = mr->rkey;	832	resp.rkey = mr->rkey;
@@ -930,21 +896,13 @@ static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs) · 63 unmodified lines
930	atomic_inc(&pd->usecnt);	896	atomic_inc(&pd->usecnt);
931		897
932	uobj->object = mw;	898	uobj->object = mw;
		899	uobj_put_obj_read(pd);
		900	uobj_finalize_uobj_create(uobj, attrs);
933		901
934	memset(&resp, 0, sizeof(resp));	902	resp.rkey = mw->rkey;
935	resp.rkey = mw->rkey;
936	resp.mw_handle = uobj->id;	903	resp.mw_handle = uobj->id;
		904	return uverbs_response(attrs, &resp, sizeof(resp));
937		905
938	ret = uverbs_response(attrs, &resp, sizeof(resp));
939	if (ret)
940	goto err_copy;
941
942	uobj_put_obj_read(pd);
943	rdma_alloc_commit_uobject(uobj, attrs);
944	return 0;
945
946	err_copy:
947	uverbs_dealloc_mw(mw);
948	err_put:	906	err_put:
949	uobj_put_obj_read(pd);	907	uobj_put_obj_read(pd);
950	err_free:	908	err_free:
@@ -981,40 +939,33 @@ static int ib_uverbs_create_comp_channel(struct uverbs_attr_bundle *attrs) · 30 unmodified lines
981	if (IS_ERR(uobj))	939	if (IS_ERR(uobj))
982	return PTR_ERR(uobj);	940	return PTR_ERR(uobj);
983		941
984	resp.fd = uobj->id;
985
986	ev_file = container_of(uobj, struct ib_uverbs_completion_event_file,	942	ev_file = container_of(uobj, struct ib_uverbs_completion_event_file,
987	uobj);	943	uobj);
988	ib_uverbs_init_event_queue(&ev_file->ev_queue);	944	ib_uverbs_init_event_queue(&ev_file->ev_queue);
		945	uobj_finalize_uobj_create(uobj, attrs);
989		946
990	ret = uverbs_response(attrs, &resp, sizeof(resp));	947	resp.fd = uobj->id;
991	if (ret) {	948	return uverbs_response(attrs, &resp, sizeof(resp));
992	uobj_alloc_abort(uobj, attrs);
993	return ret;
994	}
995
996	rdma_alloc_commit_uobject(uobj, attrs);
997	return 0;
998	}	949	}
999		950
1000	static struct ib_ucq_object create_cq(struct uverbs_attr_bundle attrs,	951	static int create_cq(struct uverbs_attr_bundle *attrs,
1001	struct ib_uverbs_ex_create_cq *cmd)	952	struct ib_uverbs_ex_create_cq *cmd)
1002	{	953	{
1003	struct ib_ucq_object *obj;	954	struct ib_ucq_object *obj;
1004	struct ib_uverbs_completion_event_file *ev_file = NULL;	955	struct ib_uverbs_completion_event_file *ev_file = NULL;
1005	struct ib_cq *cq;	956	struct ib_cq *cq;
1006	int ret;	957	int ret;
1007	struct ib_uverbs_ex_create_cq_resp resp;	958	struct ib_uverbs_ex_create_cq_resp resp = {};
1008	struct ib_cq_init_attr attr = {};	959	struct ib_cq_init_attr attr = {};
1009	struct ib_device *ib_dev;	960	struct ib_device *ib_dev;
1010		961
1011	if (cmd->comp_vector >= attrs->ufile->device->num_comp_vectors)	962	if (cmd->comp_vector >= attrs->ufile->device->num_comp_vectors)
1012	return ERR_PTR(-EINVAL);	963	return -EINVAL;
1013		964
1014	obj = (struct ib_ucq_object *)uobj_alloc(UVERBS_OBJECT_CQ, attrs,	965	obj = (struct ib_ucq_object *)uobj_alloc(UVERBS_OBJECT_CQ, attrs,
1015	&ib_dev);	966	&ib_dev);
1016	if (IS_ERR(obj))	967	if (IS_ERR(obj))
1017	return obj;	968	return PTR_ERR(obj);
1018		969
1019	if (cmd->comp_channel >= 0) {	970	if (cmd->comp_channel >= 0) {
1020	ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel, attrs);	971	ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel, attrs);
@@ -1043,53 +994,38 @@ static struct ib_ucq_object create_cq(struct uverbs_attr_bundle attrs, · 22 unmodified lines
1043	cq->event_handler = ib_uverbs_cq_event_handler;	994	cq->event_handler = ib_uverbs_cq_event_handler;
1044	cq->cq_context = ev_file ? &ev_file->ev_queue : NULL;	995	cq->cq_context = ev_file ? &ev_file->ev_queue : NULL;
1045	atomic_set(&cq->usecnt, 0);	996	atomic_set(&cq->usecnt, 0);
		997	cq->res.type = RDMA_RESTRACK_CQ;
1046		998
1047	ret = ib_dev->ops.create_cq(cq, &attr, &attrs->driver_udata);	999	ret = ib_dev->ops.create_cq(cq, &attr, &attrs->driver_udata);
1048	if (ret)	1000	if (ret)
1049	goto err_free;	1001	goto err_free;
		1002	rdma_restrack_uadd(&cq->res);
1050		1003
1051	obj->uevent.uobject.object = cq;	1004	obj->uevent.uobject.object = cq;
1052	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);	1005	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);
1053	if (obj->uevent.event_file)	1006	if (obj->uevent.event_file)
1054	uverbs_uobject_get(&obj->uevent.event_file->uobj);	1007	uverbs_uobject_get(&obj->uevent.event_file->uobj);
		1008	uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);
1055		1009
1056	memset(&resp, 0, sizeof resp);
1057	resp.base.cq_handle = obj->uevent.uobject.id;	1010	resp.base.cq_handle = obj->uevent.uobject.id;
1058	resp.base.cqe = cq->cqe;	1011	resp.base.cqe = cq->cqe;
1059	resp.response_length = uverbs_response_length(attrs, sizeof(resp));	1012	resp.response_length = uverbs_response_length(attrs, sizeof(resp));
		1013	return uverbs_response(attrs, &resp, sizeof(resp));
1060		1014
1061	cq->res.type = RDMA_RESTRACK_CQ;
1062	rdma_restrack_uadd(&cq->res);
1063
1064	ret = uverbs_response(attrs, &resp, sizeof(resp));
1065	if (ret)
1066	goto err_cb;
1067
1068	rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs);
1069	return obj;
1070
1071	err_cb:
1072	if (obj->uevent.event_file)
1073	uverbs_uobject_put(&obj->uevent.event_file->uobj);
1074	ib_destroy_cq_user(cq, uverbs_get_cleared_udata(attrs));
1075	cq = NULL;
1076	err_free:	1015	err_free:
1077	kfree(cq);	1016	kfree(cq);
1078	err_file:	1017	err_file:
1079	if (ev_file)	1018	if (ev_file)
1080	ib_uverbs_release_ucq(ev_file, obj);	1019	ib_uverbs_release_ucq(ev_file, obj);
1081
1082	err:	1020	err:
1083	uobj_alloc_abort(&obj->uevent.uobject, attrs);	1021	uobj_alloc_abort(&obj->uevent.uobject, attrs);
1084		1022	return ret;
1085	return ERR_PTR(ret);
1086	}	1023	}
1087		1024
1088	static int ib_uverbs_create_cq(struct uverbs_attr_bundle *attrs)	1025	static int ib_uverbs_create_cq(struct uverbs_attr_bundle *attrs)
1089	{	1026	{
1090	struct ib_uverbs_create_cq cmd;	1027	struct ib_uverbs_create_cq cmd;
1091	struct ib_uverbs_ex_create_cq cmd_ex;	1028	struct ib_uverbs_ex_create_cq cmd_ex;
1092	struct ib_ucq_object *obj;
1093	int ret;	1029	int ret;
1094		1030
1095	ret = uverbs_request(attrs, &cmd, sizeof(cmd));	1031	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
@@ -1102,14 +1038,12 @@ static int ib_uverbs_create_cq(struct uverbs_attr_bundle *attrs) · 6 unmodified lines
1102	cmd_ex.comp_vector = cmd.comp_vector;	1038	cmd_ex.comp_vector = cmd.comp_vector;
1103	cmd_ex.comp_channel = cmd.comp_channel;	1039	cmd_ex.comp_channel = cmd.comp_channel;
1104		1040
1105	obj = create_cq(attrs, &cmd_ex);	1041	return create_cq(attrs, &cmd_ex);
1106	return PTR_ERR_OR_ZERO(obj);
1107	}	1042	}
1108		1043
1109	static int ib_uverbs_ex_create_cq(struct uverbs_attr_bundle *attrs)	1044	static int ib_uverbs_ex_create_cq(struct uverbs_attr_bundle *attrs)
1110	{	1045	{
1111	struct ib_uverbs_ex_create_cq cmd;	1046	struct ib_uverbs_ex_create_cq cmd;
1112	struct ib_ucq_object *obj;
1113	int ret;	1047	int ret;
1114		1048
1115	ret = uverbs_request(attrs, &cmd, sizeof(cmd));	1049	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
@@ -1122,8 +1056,7 @@ static int ib_uverbs_ex_create_cq(struct uverbs_attr_bundle *attrs) · 6 unmodified lines
1122	if (cmd.reserved)	1056	if (cmd.reserved)
1123	return -EINVAL;	1057	return -EINVAL;
1124		1058
1125	obj = create_cq(attrs, &cmd);	1059	return create_cq(attrs, &cmd);
1126	return PTR_ERR_OR_ZERO(obj);
1127	}	1060	}
1128		1061
1129	static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs)	1062	static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs)
@@ -1131,7 +1064,7 @@ static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs) · 1 unmodified line
1131	struct ib_uverbs_resize_cq cmd;	1064	struct ib_uverbs_resize_cq cmd;
1132	struct ib_uverbs_resize_cq_resp resp = {};	1065	struct ib_uverbs_resize_cq_resp resp = {};
1133	struct ib_cq *cq;	1066	struct ib_cq *cq;
1134	int ret = -EINVAL;	1067	int ret;
1135		1068
1136	ret = uverbs_request(attrs, &cmd, sizeof(cmd));	1069	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1137	if (ret)	1070	if (ret)
@@ -1298,7 +1231,7 @@ static int create_qp(struct uverbs_attr_bundle *attrs, · 160 unmodified lines
1298	struct ib_srq *srq = NULL;	1231	struct ib_srq *srq = NULL;
1299	struct ib_qp *qp;	1232	struct ib_qp *qp;
1300	struct ib_qp_init_attr attr = {};	1233	struct ib_qp_init_attr attr = {};
1301	struct ib_uverbs_ex_create_qp_resp resp;	1234	struct ib_uverbs_ex_create_qp_resp resp = {};
1302	int ret;	1235	int ret;
1303	struct ib_rwq_ind_table *ind_tbl = NULL;	1236	struct ib_rwq_ind_table *ind_tbl = NULL;
1304	bool has_sq = true;	1237	bool has_sq = true;
@@ -1468,20 +1401,6 @@ static int create_qp(struct uverbs_attr_bundle *attrs, · 163 unmodified lines
1468	if (obj->uevent.event_file)	1401	if (obj->uevent.event_file)
1469	uverbs_uobject_get(&obj->uevent.event_file->uobj);	1402	uverbs_uobject_get(&obj->uevent.event_file->uobj);
1470		1403
1471	memset(&resp, 0, sizeof resp);
1472	resp.base.qpn = qp->qp_num;
1473	resp.base.qp_handle = obj->uevent.uobject.id;
1474	resp.base.max_recv_sge = attr.cap.max_recv_sge;
1475	resp.base.max_send_sge = attr.cap.max_send_sge;
1476	resp.base.max_recv_wr = attr.cap.max_recv_wr;
1477	resp.base.max_send_wr = attr.cap.max_send_wr;
1478	resp.base.max_inline_data = attr.cap.max_inline_data;
1479	resp.response_length = uverbs_response_length(attrs, sizeof(resp));
1480
1481	ret = uverbs_response(attrs, &resp, sizeof(resp));
1482	if (ret)
1483	goto err_uevent;
1484
1485	if (xrcd) {	1404	if (xrcd) {
1486	obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object,	1405	obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object,
1487	uobject);	1406	uobject);
@@ -1502,12 +1421,18 @@ static int create_qp(struct uverbs_attr_bundle *attrs, · 14 unmodified lines
1502	UVERBS_LOOKUP_READ);	1421	UVERBS_LOOKUP_READ);
1503	if (ind_tbl)	1422	if (ind_tbl)
1504	uobj_put_obj_read(ind_tbl);	1423	uobj_put_obj_read(ind_tbl);
		1424	uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);
		1425
		1426	resp.base.qpn = qp->qp_num;
		1427	resp.base.qp_handle = obj->uevent.uobject.id;
		1428	resp.base.max_recv_sge = attr.cap.max_recv_sge;
		1429	resp.base.max_send_sge = attr.cap.max_send_sge;
		1430	resp.base.max_recv_wr = attr.cap.max_recv_wr;
		1431	resp.base.max_send_wr = attr.cap.max_send_wr;
		1432	resp.base.max_inline_data = attr.cap.max_inline_data;
		1433	resp.response_length = uverbs_response_length(attrs, sizeof(resp));
		1434	return uverbs_response(attrs, &resp, sizeof(resp));
1505		1435
1506	rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs);
1507	return 0;
1508	err_uevent:
1509	if (obj->uevent.event_file)
1510	uverbs_uobject_put(&obj->uevent.event_file->uobj);
1511	err_cb:	1436	err_cb:
1512	ib_destroy_qp_user(qp, uverbs_get_cleared_udata(attrs));	1437	ib_destroy_qp_user(qp, uverbs_get_cleared_udata(attrs));
1513		1438
@@ -1580,14 +1505,14 @@ static int ib_uverbs_ex_create_qp(struct uverbs_attr_bundle *attrs) · 66 unmodified lines
1580		1505
1581	static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs)	1506	static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs)
1582	{	1507	{
		1508	struct ib_uverbs_create_qp_resp resp = {};
1583	struct ib_uverbs_open_qp cmd;	1509	struct ib_uverbs_open_qp cmd;
1584	struct ib_uverbs_create_qp_resp resp;
1585	struct ib_uqp_object *obj;	1510	struct ib_uqp_object *obj;
1586	struct ib_xrcd *xrcd;	1511	struct ib_xrcd *xrcd;
1587	struct ib_uobject *xrcd_uobj;
1588	struct ib_qp *qp;	1512	struct ib_qp *qp;
1589	struct ib_qp_open_attr attr = {};	1513	struct ib_qp_open_attr attr = {};
1590	int ret;	1514	int ret;
		1515	struct ib_uobject *xrcd_uobj;
1591	struct ib_device *ib_dev;	1516	struct ib_device *ib_dev;
1592		1517
1593	ret = uverbs_request(attrs, &cmd, sizeof(cmd));	1518	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
@@ -1627,24 +1552,16 @@ static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs) · 33 unmodified lines
1627	obj->uevent.uobject.object = qp;	1552	obj->uevent.uobject.object = qp;
1628	obj->uevent.uobject.user_handle = cmd.user_handle;	1553	obj->uevent.uobject.user_handle = cmd.user_handle;
1629		1554
1630	memset(&resp, 0, sizeof resp);
1631	resp.qpn = qp->qp_num;
1632	resp.qp_handle = obj->uevent.uobject.id;
1633
1634	ret = uverbs_response(attrs, &resp, sizeof(resp));
1635	if (ret)
1636	goto err_destroy;
1637
1638	obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);	1555	obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
1639	atomic_inc(&obj->uxrcd->refcnt);	1556	atomic_inc(&obj->uxrcd->refcnt);
1640	qp->uobject = obj;	1557	qp->uobject = obj;
1641	uobj_put_read(xrcd_uobj);	1558	uobj_put_read(xrcd_uobj);
		1559	uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);
1642		1560
1643	rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs);	1561	resp.qpn = qp->qp_num;
1644	return 0;	1562	resp.qp_handle = obj->uevent.uobject.id;
		1563	return uverbs_response(attrs, &resp, sizeof(resp));
1645		1564
1646	err_destroy:
1647	ib_destroy_qp_user(qp, uverbs_get_cleared_udata(attrs));
1648	err_xrcd:	1565	err_xrcd:
1649	uobj_put_read(xrcd_uobj);	1566	uobj_put_read(xrcd_uobj);
1650	err_put:	1567	err_put:
@@ -1980,7 +1897,7 @@ static int ib_uverbs_ex_modify_qp(struct uverbs_attr_bundle *attrs) · 329 unmodified lines
1980	* Last bit is reserved for extending the attr_mask by	1897	* Last bit is reserved for extending the attr_mask by
1981	* using another field.	1898	* using another field.
1982	*/	1899	*/
1983	BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1 << 31));	1900	BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1ULL << 31));
1984		1901
1985	if (cmd.base.attr_mask &	1902	if (cmd.base.attr_mask &
1986	~((IB_USER_LAST_QP_ATTR_MASK << 1) - 1))	1903	~((IB_USER_LAST_QP_ATTR_MASK << 1) - 1))
@@ -2480,24 +2397,14 @@ static int ib_uverbs_create_ah(struct uverbs_attr_bundle *attrs) · 493 unmodified lines
2480	ah->uobject = uobj;	2397	ah->uobject = uobj;
2481	uobj->user_handle = cmd.user_handle;	2398	uobj->user_handle = cmd.user_handle;
2482	uobj->object = ah;	2399	uobj->object = ah;
2483
2484	resp.ah_handle = uobj->id;
2485
2486	ret = uverbs_response(attrs, &resp, sizeof(resp));
2487	if (ret)
2488	goto err_copy;
2489
2490	uobj_put_obj_read(pd);	2400	uobj_put_obj_read(pd);
2491	rdma_alloc_commit_uobject(uobj, attrs);	2401	uobj_finalize_uobj_create(uobj, attrs);
2492	return 0;
2493		2402
2494	err_copy:	2403	resp.ah_handle = uobj->id;
2495	rdma_destroy_ah_user(ah, RDMA_DESTROY_AH_SLEEPABLE,	2404	return uverbs_response(attrs, &resp, sizeof(resp));
2496	uverbs_get_cleared_udata(attrs));
2497		2405
2498	err_put:	2406	err_put:
2499	uobj_put_obj_read(pd);	2407	uobj_put_obj_read(pd);
2500
2501	err:	2408	err:
2502	uobj_alloc_abort(uobj, attrs);	2409	uobj_alloc_abort(uobj, attrs);
2503	return ret;	2410	return ret;
@@ -2989,26 +2896,18 @@ static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs) · 485 unmodified lines
2989	if (obj->uevent.event_file)	2896	if (obj->uevent.event_file)
2990	uverbs_uobject_get(&obj->uevent.event_file->uobj);	2897	uverbs_uobject_get(&obj->uevent.event_file->uobj);
2991		2898
2992	memset(&resp, 0, sizeof(resp));	2899	uobj_put_obj_read(pd);
		2900	rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,
		2901	UVERBS_LOOKUP_READ);
		2902	uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);
		2903
2993	resp.wq_handle = obj->uevent.uobject.id;	2904	resp.wq_handle = obj->uevent.uobject.id;
2994	resp.max_sge = wq_init_attr.max_sge;	2905	resp.max_sge = wq_init_attr.max_sge;
2995	resp.max_wr = wq_init_attr.max_wr;	2906	resp.max_wr = wq_init_attr.max_wr;
2996	resp.wqn = wq->wq_num;	2907	resp.wqn = wq->wq_num;
2997	resp.response_length = uverbs_response_length(attrs, sizeof(resp));	2908	resp.response_length = uverbs_response_length(attrs, sizeof(resp));
2998	err = uverbs_response(attrs, &resp, sizeof(resp));	2909	return uverbs_response(attrs, &resp, sizeof(resp));
2999	if (err)
3000	goto err_copy;
3001
3002	uobj_put_obj_read(pd);
3003	rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,
3004	UVERBS_LOOKUP_READ);
3005	rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs);
3006	return 0;
3007		2910
3008	err_copy:
3009	if (obj->uevent.event_file)
3010	uverbs_uobject_put(&obj->uevent.event_file->uobj);
3011	ib_destroy_wq(wq, uverbs_get_cleared_udata(attrs));
3012	err_put_cq:	2911	err_put_cq:
3013	rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,	2912	rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,
3014	UVERBS_LOOKUP_READ);	2913	UVERBS_LOOKUP_READ);
@@ -3093,7 +2992,7 @@ static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs) · 78 unmodified lines
3093	struct ib_wq **wqs = NULL;	2992	struct ib_wq **wqs = NULL;
3094	u32 *wqs_handles = NULL;	2993	u32 *wqs_handles = NULL;
3095	struct ib_wq *wq = NULL;	2994	struct ib_wq *wq = NULL;
3096	int i, j, num_read_wqs;	2995	int i, num_read_wqs;
3097	u32 num_wq_handles;	2996	u32 num_wq_handles;
3098	struct uverbs_req_iter iter;	2997	struct uverbs_req_iter iter;
3099	struct ib_device *ib_dev;	2998	struct ib_device *ib_dev;
@@ -3139,6 +3038,7 @@ static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs) · 39 unmodified lines
3139	}	3038	}
3140		3039
3141	wqs[num_read_wqs] = wq;	3040	wqs[num_read_wqs] = wq;
		3041	atomic_inc(&wqs[num_read_wqs]->usecnt);
3142	}	3042	}
3143		3043
3144	uobj = uobj_alloc(UVERBS_OBJECT_RWQ_IND_TBL, attrs, &ib_dev);	3044	uobj = uobj_alloc(UVERBS_OBJECT_RWQ_IND_TBL, attrs, &ib_dev);
@@ -3166,33 +3066,24 @@ static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs) · 21 unmodified lines
3166	atomic_set(&rwq_ind_tbl->usecnt, 0);	3066	atomic_set(&rwq_ind_tbl->usecnt, 0);
3167		3067
3168	for (i = 0; i < num_wq_handles; i++)	3068	for (i = 0; i < num_wq_handles; i++)
3169	atomic_inc(&wqs[i]->usecnt);	3069	rdma_lookup_put_uobject(&wqs[i]->uobject->uevent.uobject,
		3070	UVERBS_LOOKUP_READ);
		3071	kfree(wqs_handles);
		3072	uobj_finalize_uobj_create(uobj, attrs);
3170		3073
3171	resp.ind_tbl_handle = uobj->id;	3074	resp.ind_tbl_handle = uobj->id;
3172	resp.ind_tbl_num = rwq_ind_tbl->ind_tbl_num;	3075	resp.ind_tbl_num = rwq_ind_tbl->ind_tbl_num;
3173	resp.response_length = uverbs_response_length(attrs, sizeof(resp));	3076	resp.response_length = uverbs_response_length(attrs, sizeof(resp));
		3077	return uverbs_response(attrs, &resp, sizeof(resp));
3174		3078
3175	err = uverbs_response(attrs, &resp, sizeof(resp));
3176	if (err)
3177	goto err_copy;
3178
3179	kfree(wqs_handles);
3180
3181	for (j = 0; j < num_read_wqs; j++)
3182	rdma_lookup_put_uobject(&wqs[j]->uobject->uevent.uobject,
3183	UVERBS_LOOKUP_READ);
3184
3185	rdma_alloc_commit_uobject(uobj, attrs);
3186	return 0;
3187
3188	err_copy:
3189	ib_destroy_rwq_ind_table(rwq_ind_tbl);
3190	err_uobj:	3079	err_uobj:
3191	uobj_alloc_abort(uobj, attrs);	3080	uobj_alloc_abort(uobj, attrs);
3192	put_wqs:	3081	put_wqs:
3193	for (j = 0; j < num_read_wqs; j++)	3082	for (i = 0; i < num_read_wqs; i++) {
3194	rdma_lookup_put_uobject(&wqs[j]->uobject->uevent.uobject,	3083	rdma_lookup_put_uobject(&wqs[i]->uobject->uevent.uobject,
3195	UVERBS_LOOKUP_READ);	3084	UVERBS_LOOKUP_READ);
		3085	atomic_dec(&wqs[i]->usecnt);
		3086	}
3196	err_free:	3087	err_free:
3197	kfree(wqs_handles);	3088	kfree(wqs_handles);
3198	kfree(wqs);	3089	kfree(wqs);
@@ -3218,7 +3109,7 @@ static int ib_uverbs_ex_destroy_rwq_ind_table(struct uverbs_attr_bundle *attrs) · 19 unmodified lines
3218	static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs)	3109	static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs)
3219	{	3110	{
3220	struct ib_uverbs_create_flow cmd;	3111	struct ib_uverbs_create_flow cmd;
3221	struct ib_uverbs_create_flow_resp resp;	3112	struct ib_uverbs_create_flow_resp resp = {};
3222	struct ib_uobject *uobj;	3113	struct ib_uobject *uobj;
3223	struct ib_flow *flow_id;	3114	struct ib_flow *flow_id;
3224	struct ib_uverbs_flow_attr *kern_flow_attr;	3115	struct ib_uverbs_flow_attr *kern_flow_attr;
@@ -3351,23 +3242,17 @@ static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs) · 126 unmodified lines
3351		3242
3352	ib_set_flow(uobj, flow_id, qp, qp->device, uflow_res);	3243	ib_set_flow(uobj, flow_id, qp, qp->device, uflow_res);
3353		3244
3354	memset(&resp, 0, sizeof(resp));
3355	resp.flow_handle = uobj->id;
3356
3357	err = uverbs_response(attrs, &resp, sizeof(resp));
3358	if (err)
3359	goto err_copy;
3360
3361	rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,	3245	rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
3362	UVERBS_LOOKUP_READ);	3246	UVERBS_LOOKUP_READ);
3363	kfree(flow_attr);	3247	kfree(flow_attr);
		3248
3364	if (cmd.flow_attr.num_of_specs)	3249	if (cmd.flow_attr.num_of_specs)
3365	kfree(kern_flow_attr);	3250	kfree(kern_flow_attr);
3366	rdma_alloc_commit_uobject(uobj, attrs);	3251	uobj_finalize_uobj_create(uobj, attrs);
3367	return 0;	3252
3368	err_copy:	3253	resp.flow_handle = uobj->id;
3369	if (!qp->device->ops.destroy_flow(flow_id))	3254	return uverbs_response(attrs, &resp, sizeof(resp));
3370	atomic_dec(&qp->usecnt);	3255
3371	err_free:	3256	err_free:
3372	ib_uverbs_flow_resources_free(uflow_res);	3257	ib_uverbs_flow_resources_free(uflow_res);
3373	err_free_flow_attr:	3258	err_free_flow_attr:
@@ -3402,13 +3287,13 @@ static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs, · 28 unmodified lines
3402	struct ib_uverbs_create_xsrq *cmd,	3287	struct ib_uverbs_create_xsrq *cmd,
3403	struct ib_udata *udata)	3288	struct ib_udata *udata)
3404	{	3289	{
3405	struct ib_uverbs_create_srq_resp resp;	3290	struct ib_uverbs_create_srq_resp resp = {};
3406	struct ib_usrq_object *obj;	3291	struct ib_usrq_object *obj;
3407	struct ib_pd *pd;	3292	struct ib_pd *pd;
3408	struct ib_srq *srq;	3293	struct ib_srq *srq;
3409	struct ib_uobject *xrcd_uobj;
3410	struct ib_srq_init_attr attr;	3294	struct ib_srq_init_attr attr;
3411	int ret;	3295	int ret;
		3296	struct ib_uobject *xrcd_uobj;
3412	struct ib_device *ib_dev;	3297	struct ib_device *ib_dev;
3413		3298
3414	obj = (struct ib_usrq_object *)uobj_alloc(UVERBS_OBJECT_SRQ, attrs,	3299	obj = (struct ib_usrq_object *)uobj_alloc(UVERBS_OBJECT_SRQ, attrs,
@@ -3473,17 +3358,9 @@ static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs, · 58 unmodified lines
3473	if (obj->uevent.event_file)	3358	if (obj->uevent.event_file)
3474	uverbs_uobject_get(&obj->uevent.event_file->uobj);	3359	uverbs_uobject_get(&obj->uevent.event_file->uobj);
3475		3360
3476	memset(&resp, 0, sizeof resp);
3477	resp.srq_handle = obj->uevent.uobject.id;
3478	resp.max_wr = attr.attr.max_wr;
3479	resp.max_sge = attr.attr.max_sge;
3480	if (cmd->srq_type == IB_SRQT_XRC)	3361	if (cmd->srq_type == IB_SRQT_XRC)
3481	resp.srqn = srq->ext.xrc.srq_num;	3362	resp.srqn = srq->ext.xrc.srq_num;
3482		3363
3483	ret = uverbs_response(attrs, &resp, sizeof(resp));
3484	if (ret)
3485	goto err_copy;
3486
3487	if (cmd->srq_type == IB_SRQT_XRC)	3364	if (cmd->srq_type == IB_SRQT_XRC)
3488	uobj_put_read(xrcd_uobj);	3365	uobj_put_read(xrcd_uobj);
3489		3366
@@ -3492,13 +3369,13 @@ static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs, · 2 unmodified lines
3492	UVERBS_LOOKUP_READ);	3369	UVERBS_LOOKUP_READ);
3493		3370
3494	uobj_put_obj_read(pd);	3371	uobj_put_obj_read(pd);
3495	rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs);	3372	uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);
3496	return 0;	3373
		3374	resp.srq_handle = obj->uevent.uobject.id;
		3375	resp.max_wr = attr.attr.max_wr;
		3376	resp.max_sge = attr.attr.max_sge;
		3377	return uverbs_response(attrs, &resp, sizeof(resp));
3497		3378
3498	err_copy:
3499	if (obj->uevent.event_file)
3500	uverbs_uobject_put(&obj->uevent.event_file->uobj);
3501	ib_destroy_srq_user(srq, uverbs_get_cleared_udata(attrs));
3502	err_put_pd:	3379	err_put_pd:
3503	uobj_put_obj_read(pd);	3380	uobj_put_obj_read(pd);
3504	err_put_cq:	3381	err_put_cq:

drivers/infiniband/core/uverbs_ioctl.c+1−0

@@ -790,6 +790,7 @@ int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle, · 789 unmodified lines
790	}	790	}
791	return uverbs_copy_to(bundle, idx, from, size);	791	return uverbs_copy_to(bundle, idx, from, size);
792	}	792	}
		793	EXPORT_SYMBOL(uverbs_copy_to_struct_or_zero);
793		794
794	/* Once called an abort will call through to the type's destroy_hw() */	795	/* Once called an abort will call through to the type's destroy_hw() */
795	void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle,	796	void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle,

drivers/infiniband/core/uverbs_main.c+4−0

 	memset(bundle.attr_present, 0, sizeof(bundle.attr_present));
 	bundle.ufile = file;
 	bundle.context = NULL; /* only valid if bundle has uobject */
+	bundle.uobject = NULL;
 	if (!method_elm->is_ex) {
 		size_t in_len = hdr.in_words * 4 - sizeof(hdr);
 		size_t out_len = hdr.out_words * 4;
 	}
 	ret = method_elm->handler(&bundle);
+	if (bundle.uobject)
+		uverbs_finalize_object(bundle.uobject, UVERBS_ACCESS_NEW, true,
+				       !ret, &bundle);
 out_unlock:
 	srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
 	return (ret) ? : count;

drivers/infiniband/core/uverbs_std_types_counters.c+9−8

 	if (ret)
 		return ret;
-	return counters->device->ops.destroy_counters(counters);
+	counters->device->ops.destroy_counters(counters);
+	kfree(counters);
+	return 0;
 }
 static int UVERBS_HANDLER(UVERBS_METHOD_COUNTERS_CREATE)(
 	if (!ib_dev->ops.create_counters)
 		return -EOPNOTSUPP;
-	counters = ib_dev->ops.create_counters(ib_dev, attrs);
+	counters = rdma_zalloc_drv_obj(ib_dev, ib_counters);
-	if (IS_ERR(counters)) {
+	if (!counters)
-		ret = PTR_ERR(counters);
+		return -ENOMEM;
-		goto err_create_counters;
-	}
 	counters->device = ib_dev;
 	counters->uobject = uobj;
 	uobj->object = counters;
 	atomic_set(&counters->usecnt, 0);
-	return 0;
+	ret = ib_dev->ops.create_counters(counters, attrs);
+	if (ret)
+		kfree(counters);
-err_create_counters:
 	return ret;
 }

drivers/infiniband/core/uverbs_std_types_cq.c+0−3

 DECLARE_UVERBS_NAMED_OBJECT(
 	UVERBS_OBJECT_CQ,
 	UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_ucq_object), uverbs_free_cq),
-#if IS_ENABLED(CONFIG_INFINIBAND_EXP_LEGACY_VERBS_NEW_UAPI)
 	&UVERBS_METHOD(UVERBS_METHOD_CQ_CREATE),
 	&UVERBS_METHOD(UVERBS_METHOD_CQ_DESTROY)
-#endif
 );
 const struct uapi_definition uverbs_def_obj_cq[] = {

drivers/infiniband/core/uverbs_std_types_device.c+46−2

@@ -38,7 +38,12 @@ static int UVERBS_HANDLER(UVERBS_METHOD_INVOKE_WRITE)( · 37 unmodified lines
38	attrs->ucore.outlen < method_elm->resp_size)	38	attrs->ucore.outlen < method_elm->resp_size)
39	return -ENOSPC;	39	return -ENOSPC;
40		40
41	return method_elm->handler(attrs);	41	attrs->uobject = NULL;
		42	rc = method_elm->handler(attrs);
		43	if (attrs->uobject)
		44	uverbs_finalize_object(attrs->uobject, UVERBS_ACCESS_NEW, true,
		45	!rc, attrs);
		46	return rc;
42	}	47	}
43		48
44	DECLARE_UVERBS_NAMED_METHOD(UVERBS_METHOD_INVOKE_WRITE,	49	DECLARE_UVERBS_NAMED_METHOD(UVERBS_METHOD_INVOKE_WRITE,
@@ -229,6 +234,37 @@ static int UVERBS_HANDLER(UVERBS_METHOD_GET_CONTEXT)( · 184 unmodified lines
229	return 0;	234	return 0;
230	}	235	}
231		236
		237	static int UVERBS_HANDLER(UVERBS_METHOD_QUERY_CONTEXT)(
		238	struct uverbs_attr_bundle *attrs)
		239	{
		240	u64 core_support = IB_UVERBS_CORE_SUPPORT_OPTIONAL_MR_ACCESS;
		241	struct ib_ucontext *ucontext;
		242	struct ib_device *ib_dev;
		243	u32 num_comp;
		244	int ret;
		245
		246	ucontext = ib_uverbs_get_ucontext(attrs);
		247	if (IS_ERR(ucontext))
		248	return PTR_ERR(ucontext);
		249	ib_dev = ucontext->device;
		250
		251	if (!ib_dev->ops.query_ucontext)
		252	return -EOPNOTSUPP;
		253
		254	num_comp = attrs->ufile->device->num_comp_vectors;
		255	ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_CONTEXT_NUM_COMP_VECTORS,
		256	&num_comp, sizeof(num_comp));
		257	if (IS_UVERBS_COPY_ERR(ret))
		258	return ret;
		259
		260	ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_CONTEXT_CORE_SUPPORT,
		261	&core_support, sizeof(core_support));
		262	if (IS_UVERBS_COPY_ERR(ret))
		263	return ret;
		264
		265	return ucontext->device->ops.query_ucontext(ucontext, attrs);
		266	}
		267
232	DECLARE_UVERBS_NAMED_METHOD(	268	DECLARE_UVERBS_NAMED_METHOD(
233	UVERBS_METHOD_GET_CONTEXT,	269	UVERBS_METHOD_GET_CONTEXT,
234	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS,	270	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS,
@@ -237,6 +273,13 @@ DECLARE_UVERBS_NAMED_METHOD( · 2 unmodified lines
237	UVERBS_ATTR_TYPE(u64), UA_OPTIONAL),	273	UVERBS_ATTR_TYPE(u64), UA_OPTIONAL),
238	UVERBS_ATTR_UHW());	274	UVERBS_ATTR_UHW());
239		275
		276	DECLARE_UVERBS_NAMED_METHOD(
		277	UVERBS_METHOD_QUERY_CONTEXT,
		278	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_CONTEXT_NUM_COMP_VECTORS,
		279	UVERBS_ATTR_TYPE(u32), UA_OPTIONAL),
		280	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_CONTEXT_CORE_SUPPORT,
		281	UVERBS_ATTR_TYPE(u64), UA_OPTIONAL));
		282
240	DECLARE_UVERBS_NAMED_METHOD(	283	DECLARE_UVERBS_NAMED_METHOD(
241	UVERBS_METHOD_INFO_HANDLES,	284	UVERBS_METHOD_INFO_HANDLES,
242	/* Also includes any device specific object ids */	285	/* Also includes any device specific object ids */
@@ -260,7 +303,8 @@ DECLARE_UVERBS_GLOBAL_METHODS(UVERBS_OBJECT_DEVICE, · 17 unmodified lines
260	&UVERBS_METHOD(UVERBS_METHOD_GET_CONTEXT),	303	&UVERBS_METHOD(UVERBS_METHOD_GET_CONTEXT),
261	&UVERBS_METHOD(UVERBS_METHOD_INVOKE_WRITE),	304	&UVERBS_METHOD(UVERBS_METHOD_INVOKE_WRITE),
262	&UVERBS_METHOD(UVERBS_METHOD_INFO_HANDLES),	305	&UVERBS_METHOD(UVERBS_METHOD_INFO_HANDLES),
263	&UVERBS_METHOD(UVERBS_METHOD_QUERY_PORT));	306	&UVERBS_METHOD(UVERBS_METHOD_QUERY_PORT),
		307	&UVERBS_METHOD(UVERBS_METHOD_QUERY_CONTEXT));
264		308
265	const struct uapi_definition uverbs_def_obj_device[] = {	309	const struct uapi_definition uverbs_def_obj_device[] = {
266	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_DEVICE),	310	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_DEVICE),

drivers/infiniband/core/uverbs_std_types_mr.c+52−2

@@ -69,7 +69,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_ADVISE_MR)( · 68 unmodified lines
69		69
70	num_sge = uverbs_attr_ptr_get_array_size(	70	num_sge = uverbs_attr_ptr_get_array_size(
71	attrs, UVERBS_ATTR_ADVISE_MR_SGE_LIST, sizeof(struct ib_sge));	71	attrs, UVERBS_ATTR_ADVISE_MR_SGE_LIST, sizeof(struct ib_sge));
72	if (num_sge < 0)	72	if (num_sge <= 0)
73	return num_sge;	73	return num_sge;
74		74
75	sg_list = uverbs_attr_get_alloced_ptr(attrs,	75	sg_list = uverbs_attr_get_alloced_ptr(attrs,
@@ -148,6 +148,36 @@ static int UVERBS_HANDLER(UVERBS_METHOD_DM_MR_REG)( · 72 unmodified lines
148	return ret;	148	return ret;
149	}	149	}
150		150
		151	static int UVERBS_HANDLER(UVERBS_METHOD_QUERY_MR)(
		152	struct uverbs_attr_bundle *attrs)
		153	{
		154	struct ib_mr *mr =
		155	uverbs_attr_get_obj(attrs, UVERBS_ATTR_QUERY_MR_HANDLE);
		156	int ret;
		157
		158	ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_MR_RESP_LKEY, &mr->lkey,
		159	sizeof(mr->lkey));
		160	if (ret)
		161	return ret;
		162
		163	ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_MR_RESP_RKEY,
		164	&mr->rkey, sizeof(mr->rkey));
		165
		166	if (ret)
		167	return ret;
		168
		169	ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_MR_RESP_LENGTH,
		170	&mr->length, sizeof(mr->length));
		171
		172	if (ret)
		173	return ret;
		174
		175	ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_MR_RESP_IOVA,
		176	&mr->iova, sizeof(mr->iova));
		177
		178	return IS_UVERBS_COPY_ERR(ret) ? ret : 0;
		179	}
		180
151	DECLARE_UVERBS_NAMED_METHOD(	181	DECLARE_UVERBS_NAMED_METHOD(
152	UVERBS_METHOD_ADVISE_MR,	182	UVERBS_METHOD_ADVISE_MR,
153	UVERBS_ATTR_IDR(UVERBS_ATTR_ADVISE_MR_PD_HANDLE,	183	UVERBS_ATTR_IDR(UVERBS_ATTR_ADVISE_MR_PD_HANDLE,
@@ -165,6 +195,25 @@ DECLARE_UVERBS_NAMED_METHOD( · 11 unmodified lines
165	UA_MANDATORY,	195	UA_MANDATORY,
166	UA_ALLOC_AND_COPY));	196	UA_ALLOC_AND_COPY));
167		197
		198	DECLARE_UVERBS_NAMED_METHOD(
		199	UVERBS_METHOD_QUERY_MR,
		200	UVERBS_ATTR_IDR(UVERBS_ATTR_QUERY_MR_HANDLE,
		201	UVERBS_OBJECT_MR,
		202	UVERBS_ACCESS_READ,
		203	UA_MANDATORY),
		204	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_MR_RESP_RKEY,
		205	UVERBS_ATTR_TYPE(u32),
		206	UA_MANDATORY),
		207	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_MR_RESP_LKEY,
		208	UVERBS_ATTR_TYPE(u32),
		209	UA_MANDATORY),
		210	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_MR_RESP_LENGTH,
		211	UVERBS_ATTR_TYPE(u64),
		212	UA_MANDATORY),
		213	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_MR_RESP_IOVA,
		214	UVERBS_ATTR_TYPE(u64),
		215	UA_OPTIONAL));
		216
168	DECLARE_UVERBS_NAMED_METHOD(	217	DECLARE_UVERBS_NAMED_METHOD(
169	UVERBS_METHOD_DM_MR_REG,	218	UVERBS_METHOD_DM_MR_REG,
170	UVERBS_ATTR_IDR(UVERBS_ATTR_REG_DM_MR_HANDLE,	219	UVERBS_ATTR_IDR(UVERBS_ATTR_REG_DM_MR_HANDLE,
@@ -206,7 +255,8 @@ DECLARE_UVERBS_NAMED_OBJECT( · 35 unmodified lines
206	UVERBS_TYPE_ALLOC_IDR(uverbs_free_mr),	255	UVERBS_TYPE_ALLOC_IDR(uverbs_free_mr),
207	&UVERBS_METHOD(UVERBS_METHOD_DM_MR_REG),	256	&UVERBS_METHOD(UVERBS_METHOD_DM_MR_REG),
208	&UVERBS_METHOD(UVERBS_METHOD_MR_DESTROY),	257	&UVERBS_METHOD(UVERBS_METHOD_MR_DESTROY),
209	&UVERBS_METHOD(UVERBS_METHOD_ADVISE_MR));	258	&UVERBS_METHOD(UVERBS_METHOD_ADVISE_MR),
		259	&UVERBS_METHOD(UVERBS_METHOD_QUERY_MR));
210		260
211	const struct uapi_definition uverbs_def_obj_mr[] = {	261	const struct uapi_definition uverbs_def_obj_mr[] = {
212	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_MR,	262	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_MR,

drivers/infiniband/core/verbs.c+78−107

@@ -988,8 +988,8 @@ EXPORT_SYMBOL(rdma_destroy_ah_user); · 987 unmodified lines
988	* @srq_init_attr: A list of initial attributes required to create the	988	* @srq_init_attr: A list of initial attributes required to create the
989	* SRQ. If SRQ creation succeeds, then the attributes are updated to	989	* SRQ. If SRQ creation succeeds, then the attributes are updated to
990	* the actual capabilities of the created SRQ.	990	* the actual capabilities of the created SRQ.
991	* @uobject - uobject pointer if this is not a kernel SRQ	991	* @uobject: uobject pointer if this is not a kernel SRQ
992	* @udata - udata pointer if this is not a kernel SRQ	992	* @udata: udata pointer if this is not a kernel SRQ
993	*	993	*
994	* srq_attr->max_wr and srq_attr->max_sge are read the determine the	994	* srq_attr->max_wr and srq_attr->max_sge are read the determine the
995	* requested size of the SRQ, and set to the actual values allocated	995	* requested size of the SRQ, and set to the actual values allocated
@@ -1090,13 +1090,6 @@ static void __ib_shared_qp_event_handler(struct ib_event event, void context) · 94 unmodified lines
1090	spin_unlock_irqrestore(&qp->device->qp_open_list_lock, flags);	1090	spin_unlock_irqrestore(&qp->device->qp_open_list_lock, flags);
1091	}	1091	}
1092		1092
1093	static void __ib_insert_xrcd_qp(struct ib_xrcd xrcd, struct ib_qp qp)
1094	{
1095	mutex_lock(&xrcd->tgt_qp_mutex);
1096	list_add(&qp->xrcd_list, &xrcd->tgt_qp_list);
1097	mutex_unlock(&xrcd->tgt_qp_mutex);
1098	}
1099
1100	static struct ib_qp __ib_open_qp(struct ib_qp real_qp,	1093	static struct ib_qp __ib_open_qp(struct ib_qp real_qp,
1101	void (event_handler)(struct ib_event , void *),	1094	void (event_handler)(struct ib_event , void *),
1102	void *qp_context)	1095	void *qp_context)
@@ -1139,16 +1132,15 @@ struct ib_qp ib_open_qp(struct ib_xrcd xrcd, · 36 unmodified lines
1139	if (qp_open_attr->qp_type != IB_QPT_XRC_TGT)	1132	if (qp_open_attr->qp_type != IB_QPT_XRC_TGT)
1140	return ERR_PTR(-EINVAL);	1133	return ERR_PTR(-EINVAL);
1141		1134
1142	qp = ERR_PTR(-EINVAL);	1135	down_read(&xrcd->tgt_qps_rwsem);
1143	mutex_lock(&xrcd->tgt_qp_mutex);	1136	real_qp = xa_load(&xrcd->tgt_qps, qp_open_attr->qp_num);
1144	list_for_each_entry(real_qp, &xrcd->tgt_qp_list, xrcd_list) {	1137	if (!real_qp) {
1145	if (real_qp->qp_num == qp_open_attr->qp_num) {	1138	up_read(&xrcd->tgt_qps_rwsem);
1146	qp = __ib_open_qp(real_qp, qp_open_attr->event_handler,	1139	return ERR_PTR(-EINVAL);
1147	qp_open_attr->qp_context);
1148	break;
1149	}
1150	}	1140	}
1151	mutex_unlock(&xrcd->tgt_qp_mutex);	1141	qp = __ib_open_qp(real_qp, qp_open_attr->event_handler,
		1142	qp_open_attr->qp_context);
		1143	up_read(&xrcd->tgt_qps_rwsem);
1152	return qp;	1144	return qp;
1153	}	1145	}
1154	EXPORT_SYMBOL(ib_open_qp);	1146	EXPORT_SYMBOL(ib_open_qp);
@@ -1157,6 +1149,7 @@ static struct ib_qp create_xrc_qp_user(struct ib_qp qp, · 2 unmodified lines
1157	struct ib_qp_init_attr *qp_init_attr)	1149	struct ib_qp_init_attr *qp_init_attr)
1158	{	1150	{
1159	struct ib_qp *real_qp = qp;	1151	struct ib_qp *real_qp = qp;
		1152	int err;
1160		1153
1161	qp->event_handler = __ib_shared_qp_event_handler;	1154	qp->event_handler = __ib_shared_qp_event_handler;
1162	qp->qp_context = qp;	1155	qp->qp_context = qp;
@@ -1172,7 +1165,12 @@ static struct ib_qp create_xrc_qp_user(struct ib_qp qp, · 9 unmodified lines
1172	if (IS_ERR(qp))	1165	if (IS_ERR(qp))
1173	return qp;	1166	return qp;
1174		1167
1175	__ib_insert_xrcd_qp(qp_init_attr->xrcd, real_qp);	1168	err = xa_err(xa_store(&qp_init_attr->xrcd->tgt_qps, real_qp->qp_num,
		1169	real_qp, GFP_KERNEL));
		1170	if (err) {
		1171	ib_close_qp(qp);
		1172	return ERR_PTR(err);
		1173	}
1176	return qp;	1174	return qp;
1177	}	1175	}
1178		1176
@@ -1712,7 +1710,7 @@ static int _ib_modify_qp(struct ib_qp qp, struct ib_qp_attr attr, · 533 unmodified lines
1712	if (!(rdma_protocol_ib(qp->device,	1710	if (!(rdma_protocol_ib(qp->device,
1713	attr->alt_ah_attr.port_num) &&	1711	attr->alt_ah_attr.port_num) &&
1714	rdma_protocol_ib(qp->device, port))) {	1712	rdma_protocol_ib(qp->device, port))) {
1715	ret = EINVAL;	1713	ret = -EINVAL;
1716	goto out;	1714	goto out;
1717	}	1715	}
1718	}	1716	}
@@ -1887,21 +1885,18 @@ static int __ib_destroy_shared_qp(struct ib_qp *qp) · 168 unmodified lines
1887		1885
1888	real_qp = qp->real_qp;	1886	real_qp = qp->real_qp;
1889	xrcd = real_qp->xrcd;	1887	xrcd = real_qp->xrcd;
1890		1888	down_write(&xrcd->tgt_qps_rwsem);
1891	mutex_lock(&xrcd->tgt_qp_mutex);
1892	ib_close_qp(qp);	1889	ib_close_qp(qp);
1893	if (atomic_read(&real_qp->usecnt) == 0)	1890	if (atomic_read(&real_qp->usecnt) == 0)
1894	list_del(&real_qp->xrcd_list);	1891	xa_erase(&xrcd->tgt_qps, real_qp->qp_num);
1895	else	1892	else
1896	real_qp = NULL;	1893	real_qp = NULL;
1897	mutex_unlock(&xrcd->tgt_qp_mutex);	1894	up_write(&xrcd->tgt_qps_rwsem);
1898		1895
1899	if (real_qp) {	1896	if (real_qp) {
1900	ret = ib_destroy_qp(real_qp);	1897	ret = ib_destroy_qp(real_qp);
1901	if (!ret)	1898	if (!ret)
1902	atomic_dec(&xrcd->usecnt);	1899	atomic_dec(&xrcd->usecnt);
1903	else
1904	__ib_insert_xrcd_qp(xrcd, real_qp);
1905	}	1900	}
1906		1901
1907	return 0;	1902	return 0;
@@ -2077,6 +2072,9 @@ int ib_advise_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice, · 169 unmodified lines
2077	if (!pd->device->ops.advise_mr)	2072	if (!pd->device->ops.advise_mr)
2078	return -EOPNOTSUPP;	2073	return -EOPNOTSUPP;
2079		2074
		2075	if (!num_sge)
		2076	return 0;
		2077
2080	return pd->device->ops.advise_mr(pd, advice, flags, sg_list, num_sge,	2078	return pd->device->ops.advise_mr(pd, advice, flags, sg_list, num_sge,
2081	NULL);	2079	NULL);
2082	}	2080	}
@@ -2104,11 +2102,10 @@ int ib_dereg_mr_user(struct ib_mr mr, struct ib_udata udata) · 21 unmodified lines
2104	EXPORT_SYMBOL(ib_dereg_mr_user);	2102	EXPORT_SYMBOL(ib_dereg_mr_user);
2105		2103
2106	/**	2104	/**
2107	* ib_alloc_mr_user() - Allocates a memory region	2105	* ib_alloc_mr() - Allocates a memory region
2108	* @pd: protection domain associated with the region	2106	* @pd: protection domain associated with the region
2109	* @mr_type: memory region type	2107	* @mr_type: memory region type
2110	* @max_num_sg: maximum sg entries available for registration.	2108	* @max_num_sg: maximum sg entries available for registration.
2111	* @udata: user data or null for kernel objects
2112	*	2109	*
2113	* Notes:	2110	* Notes:
2114	* Memory registeration page/sg lists must not exceed max_num_sg.	2111	* Memory registeration page/sg lists must not exceed max_num_sg.
@@ -2116,8 +2113,8 @@ EXPORT_SYMBOL(ib_dereg_mr_user); · 1 unmodified line
2116	* max_num_sg * used_page_size.	2113	* max_num_sg * used_page_size.
2117	*	2114	*
2118	*/	2115	*/
2119	struct ib_mr ib_alloc_mr_user(struct ib_pd pd, enum ib_mr_type mr_type,	2116	struct ib_mr ib_alloc_mr(struct ib_pd pd, enum ib_mr_type mr_type,
2120	u32 max_num_sg, struct ib_udata *udata)	2117	u32 max_num_sg)
2121	{	2118	{
2122	struct ib_mr *mr;	2119	struct ib_mr *mr;
2123		2120
@@ -2132,25 +2129,26 @@ struct ib_mr ib_alloc_mr_user(struct ib_pd pd, enum ib_mr_type mr_type, · 8 unmodified lines
2132	goto out;	2129	goto out;
2133	}	2130	}
2134		2131
2135	mr = pd->device->ops.alloc_mr(pd, mr_type, max_num_sg, udata);	2132	mr = pd->device->ops.alloc_mr(pd, mr_type, max_num_sg);
2136	if (!IS_ERR(mr)) {	2133	if (IS_ERR(mr))
2137	mr->device = pd->device;	2134	goto out;
2138	mr->pd = pd;	2135
2139	mr->dm = NULL;	2136	mr->device = pd->device;
2140	mr->uobject = NULL;	2137	mr->pd = pd;
2141	atomic_inc(&pd->usecnt);	2138	mr->dm = NULL;
2142	mr->need_inval = false;	2139	mr->uobject = NULL;
2143	mr->res.type = RDMA_RESTRACK_MR;	2140	atomic_inc(&pd->usecnt);
2144	rdma_restrack_kadd(&mr->res);	2141	mr->need_inval = false;
2145	mr->type = mr_type;	2142	mr->res.type = RDMA_RESTRACK_MR;
2146	mr->sig_attrs = NULL;	2143	rdma_restrack_kadd(&mr->res);
2147	}	2144	mr->type = mr_type;
		2145	mr->sig_attrs = NULL;
2148		2146
2149	out:	2147	out:
2150	trace_mr_alloc(pd, mr_type, max_num_sg, mr);	2148	trace_mr_alloc(pd, mr_type, max_num_sg, mr);
2151	return mr;	2149	return mr;
2152	}	2150	}
2153	EXPORT_SYMBOL(ib_alloc_mr_user);	2151	EXPORT_SYMBOL(ib_alloc_mr);
2154		2152
2155	/**	2153	/**
2156	* ib_alloc_mr_integrity() - Allocates an integrity memory region	2154	* ib_alloc_mr_integrity() - Allocates an integrity memory region
@@ -2288,45 +2286,57 @@ int ib_detach_mcast(struct ib_qp qp, union ib_gid gid, u16 lid) · 131 unmodified lines
2288	}	2286	}
2289	EXPORT_SYMBOL(ib_detach_mcast);	2287	EXPORT_SYMBOL(ib_detach_mcast);
2290		2288
2291	struct ib_xrcd __ib_alloc_xrcd(struct ib_device device, const char *caller)	2289	/**
		2290	* ib_alloc_xrcd_user - Allocates an XRC domain.
		2291	* @device: The device on which to allocate the XRC domain.
		2292	* @inode: inode to connect XRCD
		2293	* @udata: Valid user data or NULL for kernel object
		2294	*/
		2295	struct ib_xrcd ib_alloc_xrcd_user(struct ib_device device,
		2296	struct inode inode, struct ib_udata udata)
2292	{	2297	{
2293	struct ib_xrcd *xrcd;	2298	struct ib_xrcd *xrcd;
		2299	int ret;
2294		2300
2295	if (!device->ops.alloc_xrcd)	2301	if (!device->ops.alloc_xrcd)
2296	return ERR_PTR(-EOPNOTSUPP);	2302	return ERR_PTR(-EOPNOTSUPP);
2297		2303
2298	xrcd = device->ops.alloc_xrcd(device, NULL);	2304	xrcd = rdma_zalloc_drv_obj(device, ib_xrcd);
2299	if (!IS_ERR(xrcd)) {	2305	if (!xrcd)
2300	xrcd->device = device;	2306	return ERR_PTR(-ENOMEM);
2301	xrcd->inode = NULL;
2302	atomic_set(&xrcd->usecnt, 0);
2303	mutex_init(&xrcd->tgt_qp_mutex);
2304	INIT_LIST_HEAD(&xrcd->tgt_qp_list);
2305	}
2306		2307
		2308	xrcd->device = device;
		2309	xrcd->inode = inode;
		2310	atomic_set(&xrcd->usecnt, 0);
		2311	init_rwsem(&xrcd->tgt_qps_rwsem);
		2312	xa_init(&xrcd->tgt_qps);
		2313
		2314	ret = device->ops.alloc_xrcd(xrcd, udata);
		2315	if (ret)
		2316	goto err;
2307	return xrcd;	2317	return xrcd;
		2318	err:
		2319	kfree(xrcd);
		2320	return ERR_PTR(ret);
2308	}	2321	}
2309	EXPORT_SYMBOL(__ib_alloc_xrcd);	2322	EXPORT_SYMBOL(ib_alloc_xrcd_user);
2310		2323
2311	int ib_dealloc_xrcd(struct ib_xrcd xrcd, struct ib_udata udata)	2324	/**
		2325	* ib_dealloc_xrcd_user - Deallocates an XRC domain.
		2326	* @xrcd: The XRC domain to deallocate.
		2327	* @udata: Valid user data or NULL for kernel object
		2328	*/
		2329	int ib_dealloc_xrcd_user(struct ib_xrcd xrcd, struct ib_udata udata)
2312	{	2330	{
2313	struct ib_qp *qp;
2314	int ret;
2315
2316	if (atomic_read(&xrcd->usecnt))	2331	if (atomic_read(&xrcd->usecnt))
2317	return -EBUSY;	2332	return -EBUSY;
2318		2333
2319	while (!list_empty(&xrcd->tgt_qp_list)) {	2334	WARN_ON(!xa_empty(&xrcd->tgt_qps));
2320	qp = list_entry(xrcd->tgt_qp_list.next, struct ib_qp, xrcd_list);	2335	xrcd->device->ops.dealloc_xrcd(xrcd, udata);
2321	ret = ib_destroy_qp(qp);	2336	kfree(xrcd);
2322	if (ret)	2337	return 0;
2323	return ret;
2324	}
2325	mutex_destroy(&xrcd->tgt_qp_mutex);
2326
2327	return xrcd->device->ops.dealloc_xrcd(xrcd, udata);
2328	}	2338	}
2329	EXPORT_SYMBOL(ib_dealloc_xrcd);	2339	EXPORT_SYMBOL(ib_dealloc_xrcd_user);
2330		2340
2331	/**	2341	/**
2332	* ib_create_wq - Creates a WQ associated with the specified protection	2342	* ib_create_wq - Creates a WQ associated with the specified protection
@@ -2409,45 +2419,6 @@ int ib_modify_wq(struct ib_wq wq, struct ib_wq_attr wq_attr, · 76 unmodified lines
2409	}	2419	}
2410	EXPORT_SYMBOL(ib_modify_wq);	2420	EXPORT_SYMBOL(ib_modify_wq);
2411		2421
2412	/*
2413	* ib_create_rwq_ind_table - Creates a RQ Indirection Table.
2414	* @device: The device on which to create the rwq indirection table.
2415	* @ib_rwq_ind_table_init_attr: A list of initial attributes required to
2416	* create the Indirection Table.
2417	*
2418	* Note: The life time of ib_rwq_ind_table_init_attr->ind_tbl is not less
2419	* than the created ib_rwq_ind_table object and the caller is responsible
2420	* for its memory allocation/free.
2421	*/
2422	struct ib_rwq_ind_table ib_create_rwq_ind_table(struct ib_device device,
2423	struct ib_rwq_ind_table_init_attr *init_attr)
2424	{
2425	struct ib_rwq_ind_table *rwq_ind_table;
2426	int i;
2427	u32 table_size;
2428
2429	if (!device->ops.create_rwq_ind_table)
2430	return ERR_PTR(-EOPNOTSUPP);
2431
2432	table_size = (1 << init_attr->log_ind_tbl_size);
2433	rwq_ind_table = device->ops.create_rwq_ind_table(device,
2434	init_attr, NULL);
2435	if (IS_ERR(rwq_ind_table))
2436	return rwq_ind_table;
2437
2438	rwq_ind_table->ind_tbl = init_attr->ind_tbl;
2439	rwq_ind_table->log_ind_tbl_size = init_attr->log_ind_tbl_size;
2440	rwq_ind_table->device = device;
2441	rwq_ind_table->uobject = NULL;
2442	atomic_set(&rwq_ind_table->usecnt, 0);
2443
2444	for (i = 0; i < table_size; i++)
2445	atomic_inc(&rwq_ind_table->ind_tbl[i]->usecnt);
2446
2447	return rwq_ind_table;
2448	}
2449	EXPORT_SYMBOL(ib_create_rwq_ind_table);
2450
2451	/*	2422	/*
2452	* ib_destroy_rwq_ind_table - Destroys the specified Indirection Table.	2423	* ib_destroy_rwq_ind_table - Destroys the specified Indirection Table.
2453	* @wq_ind_table: The Indirection Table to destroy.	2424	* @wq_ind_table: The Indirection Table to destroy.

drivers/infiniband/hw/bnxt_re/ib_verbs.c+128−42

@@ -842,16 +842,79 @@ static u8 __from_ib_qp_type(enum ib_qp_type type) · 841 unmodified lines
842	}	842	}
843	}	843	}
844		844
		845	static u16 bnxt_re_setup_rwqe_size(struct bnxt_qplib_qp *qplqp,
		846	int rsge, int max)
		847	{
		848	if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
		849	rsge = max;
		850	return bnxt_re_get_rwqe_size(rsge);
		851	}
		852
		853	static u16 bnxt_re_get_wqe_size(int ilsize, int nsge)
		854	{
		855	u16 wqe_size, calc_ils;
		856
		857	wqe_size = bnxt_re_get_swqe_size(nsge);
		858	if (ilsize) {
		859	calc_ils = sizeof(struct sq_send_hdr) + ilsize;
		860	wqe_size = max_t(u16, calc_ils, wqe_size);
		861	wqe_size = ALIGN(wqe_size, sizeof(struct sq_send_hdr));
		862	}
		863	return wqe_size;
		864	}
		865
		866	static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp,
		867	struct ib_qp_init_attr *init_attr)
		868	{
		869	struct bnxt_qplib_dev_attr *dev_attr;
		870	struct bnxt_qplib_qp *qplqp;
		871	struct bnxt_re_dev *rdev;
		872	struct bnxt_qplib_q *sq;
		873	int align, ilsize;
		874
		875	rdev = qp->rdev;
		876	qplqp = &qp->qplib_qp;
		877	sq = &qplqp->sq;
		878	dev_attr = &rdev->dev_attr;
		879
		880	align = sizeof(struct sq_send_hdr);
		881	ilsize = ALIGN(init_attr->cap.max_inline_data, align);
		882
		883	sq->wqe_size = bnxt_re_get_wqe_size(ilsize, sq->max_sge);
		884	if (sq->wqe_size > bnxt_re_get_swqe_size(dev_attr->max_qp_sges))
		885	return -EINVAL;
		886	/* For gen p4 and gen p5 backward compatibility mode
		887	* wqe size is fixed to 128 bytes
		888	*/
		889	if (sq->wqe_size < bnxt_re_get_swqe_size(dev_attr->max_qp_sges) &&
		890	qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
		891	sq->wqe_size = bnxt_re_get_swqe_size(dev_attr->max_qp_sges);
		892
		893	if (init_attr->cap.max_inline_data) {
		894	qplqp->max_inline_data = sq->wqe_size -
		895	sizeof(struct sq_send_hdr);
		896	init_attr->cap.max_inline_data = qplqp->max_inline_data;
		897	if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
		898	sq->max_sge = qplqp->max_inline_data /
		899	sizeof(struct sq_sge);
		900	}
		901
		902	return 0;
		903	}
		904
845	static int bnxt_re_init_user_qp(struct bnxt_re_dev rdev, struct bnxt_re_pd pd,	905	static int bnxt_re_init_user_qp(struct bnxt_re_dev rdev, struct bnxt_re_pd pd,
846	struct bnxt_re_qp qp, struct ib_udata udata)	906	struct bnxt_re_qp qp, struct ib_udata udata)
847	{	907	{
		908	struct bnxt_qplib_qp *qplib_qp;
		909	struct bnxt_re_ucontext *cntx;
848	struct bnxt_re_qp_req ureq;	910	struct bnxt_re_qp_req ureq;
849	struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp;
850	struct ib_umem *umem;
851	int bytes = 0, psn_sz;	911	int bytes = 0, psn_sz;
852	struct bnxt_re_ucontext *cntx = rdma_udata_to_drv_context(	912	struct ib_umem *umem;
853	udata, struct bnxt_re_ucontext, ib_uctx);	913	int psn_nume;
854		914
		915	qplib_qp = &qp->qplib_qp;
		916	cntx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext,
		917	ib_uctx);
855	if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))	918	if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
856	return -EFAULT;	919	return -EFAULT;
857		920
@@ -859,10 +922,15 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev rdev, struct bnxt_re_pd pd, · 1 unmodified line
859	/* Consider mapping PSN search memory only for RC QPs. */	922	/* Consider mapping PSN search memory only for RC QPs. */
860	if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) {	923	if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) {
861	psn_sz = bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx) ?	924	psn_sz = bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx) ?
862	sizeof(struct sq_psn_search_ext) :	925	sizeof(struct sq_psn_search_ext) :
863	sizeof(struct sq_psn_search);	926	sizeof(struct sq_psn_search);
864	bytes += (qplib_qp->sq.max_wqe * psn_sz);	927	psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
		928	qplib_qp->sq.max_wqe :
		929	((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) /
		930	sizeof(struct bnxt_qplib_sge));
		931	bytes += (psn_nume * psn_sz);
865	}	932	}
		933
866	bytes = PAGE_ALIGN(bytes);	934	bytes = PAGE_ALIGN(bytes);
867	umem = ib_umem_get(&rdev->ibdev, ureq.qpsva, bytes,	935	umem = ib_umem_get(&rdev->ibdev, ureq.qpsva, bytes,
868	IB_ACCESS_LOCAL_WRITE);	936	IB_ACCESS_LOCAL_WRITE);
@@ -975,7 +1043,7 @@ static struct bnxt_re_qp *bnxt_re_create_shadow_qp · 106 unmodified lines
975	qp->qplib_qp.sig_type = true;	1043	qp->qplib_qp.sig_type = true;
976		1044
977	/* Shadow QP SQ depth should be same as QP1 RQ depth */	1045	/* Shadow QP SQ depth should be same as QP1 RQ depth */
978	qp->qplib_qp.sq.wqe_size = bnxt_re_get_swqe_size();	1046	qp->qplib_qp.sq.wqe_size = bnxt_re_get_wqe_size(0, 6);
979	qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe;	1047	qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe;
980	qp->qplib_qp.sq.max_sge = 2;	1048	qp->qplib_qp.sq.max_sge = 2;
981	/* Q full delta can be 1 since it is internal QP */	1049	/* Q full delta can be 1 since it is internal QP */
@@ -986,7 +1054,7 @@ static struct bnxt_re_qp *bnxt_re_create_shadow_qp · 4 unmodified lines
986	qp->qplib_qp.scq = qp1_qp->scq;	1054	qp->qplib_qp.scq = qp1_qp->scq;
987	qp->qplib_qp.rcq = qp1_qp->rcq;	1055	qp->qplib_qp.rcq = qp1_qp->rcq;
988		1056
989	qp->qplib_qp.rq.wqe_size = bnxt_re_get_rwqe_size();	1057	qp->qplib_qp.rq.wqe_size = bnxt_re_get_rwqe_size(6);
990	qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe;	1058	qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe;
991	qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge;	1059	qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge;
992	/* Q full delta can be 1 since it is internal QP */	1060	/* Q full delta can be 1 since it is internal QP */
@@ -1041,19 +1109,21 @@ static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp, · 48 unmodified lines
1041	qplqp->srq = &srq->qplib_srq;	1109	qplqp->srq = &srq->qplib_srq;
1042	rq->max_wqe = 0;	1110	rq->max_wqe = 0;
1043	} else {	1111	} else {
1044	rq->wqe_size = bnxt_re_get_rwqe_size();	1112	rq->max_sge = init_attr->cap.max_recv_sge;
		1113	if (rq->max_sge > dev_attr->max_qp_sges)
		1114	rq->max_sge = dev_attr->max_qp_sges;
		1115	init_attr->cap.max_recv_sge = rq->max_sge;
		1116	rq->wqe_size = bnxt_re_setup_rwqe_size(qplqp, rq->max_sge,
		1117	dev_attr->max_qp_sges);
1045	/* Allocate 1 more than what's provided so posting max doesn't	1118	/* Allocate 1 more than what's provided so posting max doesn't
1046	* mean empty.	1119	* mean empty.
1047	*/	1120	*/
1048	entries = roundup_pow_of_two(init_attr->cap.max_recv_wr + 1);	1121	entries = roundup_pow_of_two(init_attr->cap.max_recv_wr + 1);
1049	rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1);	1122	rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1);
1050	rq->q_full_delta = rq->max_wqe - init_attr->cap.max_recv_wr;	1123	rq->q_full_delta = 0;
1051	rq->max_sge = init_attr->cap.max_recv_sge;	1124	rq->sg_info.pgsize = PAGE_SIZE;
1052	if (rq->max_sge > dev_attr->max_qp_sges)	1125	rq->sg_info.pgshft = PAGE_SHIFT;
1053	rq->max_sge = dev_attr->max_qp_sges;
1054	}	1126	}
1055	rq->sg_info.pgsize = PAGE_SIZE;
1056	rq->sg_info.pgshft = PAGE_SHIFT;
1057		1127
1058	return 0;	1128	return 0;
1059	}	1129	}
@@ -1068,41 +1138,48 @@ static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp) · 8 unmodified lines
1068	qplqp = &qp->qplib_qp;	1138	qplqp = &qp->qplib_qp;
1069	dev_attr = &rdev->dev_attr;	1139	dev_attr = &rdev->dev_attr;
1070		1140
1071	qplqp->rq.max_sge = dev_attr->max_qp_sges;	1141	if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) {
1072	if (qplqp->rq.max_sge > dev_attr->max_qp_sges)
1073	qplqp->rq.max_sge = dev_attr->max_qp_sges;	1142	qplqp->rq.max_sge = dev_attr->max_qp_sges;
1074	qplqp->rq.max_sge = 6;	1143	if (qplqp->rq.max_sge > dev_attr->max_qp_sges)
		1144	qplqp->rq.max_sge = dev_attr->max_qp_sges;
		1145	qplqp->rq.max_sge = 6;
		1146	}
1075	}	1147	}
1076		1148
1077	static void bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,	1149	static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,
1078	struct ib_qp_init_attr *init_attr,	1150	struct ib_qp_init_attr *init_attr,
1079	struct ib_udata *udata)	1151	struct ib_udata *udata)
1080	{	1152	{
1081	struct bnxt_qplib_dev_attr *dev_attr;	1153	struct bnxt_qplib_dev_attr *dev_attr;
1082	struct bnxt_qplib_qp *qplqp;	1154	struct bnxt_qplib_qp *qplqp;
1083	struct bnxt_re_dev *rdev;	1155	struct bnxt_re_dev *rdev;
1084	struct bnxt_qplib_q *sq;	1156	struct bnxt_qplib_q *sq;
1085	int entries;	1157	int entries;
		1158	int diff;
		1159	int rc;
1086		1160
1087	rdev = qp->rdev;	1161	rdev = qp->rdev;
1088	qplqp = &qp->qplib_qp;	1162	qplqp = &qp->qplib_qp;
1089	sq = &qplqp->sq;	1163	sq = &qplqp->sq;
1090	dev_attr = &rdev->dev_attr;	1164	dev_attr = &rdev->dev_attr;
1091		1165
1092	sq->wqe_size = bnxt_re_get_swqe_size();
1093	sq->max_sge = init_attr->cap.max_send_sge;	1166	sq->max_sge = init_attr->cap.max_send_sge;
1094	if (sq->max_sge > dev_attr->max_qp_sges)	1167	if (sq->max_sge > dev_attr->max_qp_sges) {
1095	sq->max_sge = dev_attr->max_qp_sges;	1168	sq->max_sge = dev_attr->max_qp_sges;
1096	/*	1169	init_attr->cap.max_send_sge = sq->max_sge;
1097	* Change the SQ depth if user has requested minimum using	1170	}
1098	* configfs. Only supported for kernel consumers	1171
1099	*/	1172	rc = bnxt_re_setup_swqe_size(qp, init_attr);
		1173	if (rc)
		1174	return rc;
		1175
1100	entries = init_attr->cap.max_send_wr;	1176	entries = init_attr->cap.max_send_wr;
1101	/* Allocate 128 + 1 more than what's provided */	1177	/* Allocate 128 + 1 more than what's provided */
1102	entries = roundup_pow_of_two(entries + BNXT_QPLIB_RESERVED_QP_WRS + 1);	1178	diff = (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) ?
1103	sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes +	1179	0 : BNXT_QPLIB_RESERVED_QP_WRS;
1104	BNXT_QPLIB_RESERVED_QP_WRS + 1);	1180	entries = roundup_pow_of_two(entries + diff + 1);
1105	sq->q_full_delta = BNXT_QPLIB_RESERVED_QP_WRS + 1;	1181	sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1);
		1182	sq->q_full_delta = diff + 1;
1106	/*	1183	/*
1107	* Reserving one slot for Phantom WQE. Application can	1184	* Reserving one slot for Phantom WQE. Application can
1108	* post one extra entry in this case. But allowing this to avoid	1185	* post one extra entry in this case. But allowing this to avoid
@@ -1111,6 +1188,8 @@ static void bnxt_re_init_sq_attr(struct bnxt_re_qp *qp, · 2 unmodified lines
1111	qplqp->sq.q_full_delta -= 1;	1188	qplqp->sq.q_full_delta -= 1;
1112	qplqp->sq.sg_info.pgsize = PAGE_SIZE;	1189	qplqp->sq.sg_info.pgsize = PAGE_SIZE;
1113	qplqp->sq.sg_info.pgshft = PAGE_SHIFT;	1190	qplqp->sq.sg_info.pgshft = PAGE_SHIFT;
		1191
		1192	return 0;
1114	}	1193	}
1115		1194
1116	static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp,	1195	static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp,
@@ -1125,13 +1204,16 @@ static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp, · 8 unmodified lines
1125	qplqp = &qp->qplib_qp;	1204	qplqp = &qp->qplib_qp;
1126	dev_attr = &rdev->dev_attr;	1205	dev_attr = &rdev->dev_attr;
1127		1206
1128	entries = roundup_pow_of_two(init_attr->cap.max_send_wr + 1);	1207	if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) {
1129	qplqp->sq.max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1);	1208	entries = roundup_pow_of_two(init_attr->cap.max_send_wr + 1);
1130	qplqp->sq.q_full_delta = qplqp->sq.max_wqe -	1209	qplqp->sq.max_wqe = min_t(u32, entries,
1131	init_attr->cap.max_send_wr;	1210	dev_attr->max_qp_wqes + 1);
1132	qplqp->sq.max_sge++; /* Need one extra sge to put UD header */	1211	qplqp->sq.q_full_delta = qplqp->sq.max_wqe -
1133	if (qplqp->sq.max_sge > dev_attr->max_qp_sges)	1212	init_attr->cap.max_send_wr;
1134	qplqp->sq.max_sge = dev_attr->max_qp_sges;	1213	qplqp->sq.max_sge++; /* Need one extra sge to put UD header */
		1214	if (qplqp->sq.max_sge > dev_attr->max_qp_sges)
		1215	qplqp->sq.max_sge = dev_attr->max_qp_sges;
		1216	}
1135	}	1217	}
1136		1218
1137	static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev,	1219	static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev,
@@ -1183,6 +1265,7 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp qp, struct bnxt_re_pd pd, · 45 unmodified lines
1183	goto out;	1265	goto out;
1184	}	1266	}
1185	qplqp->type = (u8)qptype;	1267	qplqp->type = (u8)qptype;
		1268	qplqp->wqe_mode = rdev->chip_ctx->modes.wqe_mode;
1186		1269
1187	if (init_attr->qp_type == IB_QPT_RC) {	1270	if (init_attr->qp_type == IB_QPT_RC) {
1188	qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom;	1271	qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom;
@@ -1226,7 +1309,9 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp qp, struct bnxt_re_pd pd, · 37 unmodified lines
1226	bnxt_re_adjust_gsi_rq_attr(qp);	1309	bnxt_re_adjust_gsi_rq_attr(qp);
1227		1310
1228	/* Setup SQ */	1311	/* Setup SQ */
1229	bnxt_re_init_sq_attr(qp, init_attr, udata);	1312	rc = bnxt_re_init_sq_attr(qp, init_attr, udata);
		1313	if (rc)
		1314	goto out;
1230	if (init_attr->qp_type == IB_QPT_GSI)	1315	if (init_attr->qp_type == IB_QPT_GSI)
1231	bnxt_re_adjust_gsi_sq_attr(qp, init_attr);	1316	bnxt_re_adjust_gsi_sq_attr(qp, init_attr);
1232		1317
@@ -1574,8 +1659,9 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq, · 341 unmodified lines
1574	entries = dev_attr->max_srq_wqes + 1;	1659	entries = dev_attr->max_srq_wqes + 1;
1575	srq->qplib_srq.max_wqe = entries;	1660	srq->qplib_srq.max_wqe = entries;
1576		1661
1577	srq->qplib_srq.wqe_size = bnxt_re_get_rwqe_size();
1578	srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge;	1662	srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge;
		1663	srq->qplib_srq.wqe_size =
		1664	bnxt_re_get_rwqe_size(srq->qplib_srq.max_sge);
1579	srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit;	1665	srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit;
1580	srq->srq_limit = srq_init_attr->attr.srq_limit;	1666	srq->srq_limit = srq_init_attr->attr.srq_limit;
1581	srq->qplib_srq.eventq_hw_ring_id = rdev->nq[0].ring_id;	1667	srq->qplib_srq.eventq_hw_ring_id = rdev->nq[0].ring_id;
@@ -3569,7 +3655,7 @@ int bnxt_re_map_mr_sg(struct ib_mr ib_mr, struct scatterlist sg, int sg_nents, · 1987 unmodified lines
3569	}	3655	}
3570		3656
3571	struct ib_mr bnxt_re_alloc_mr(struct ib_pd ib_pd, enum ib_mr_type type,	3657	struct ib_mr bnxt_re_alloc_mr(struct ib_pd ib_pd, enum ib_mr_type type,
3572	u32 max_num_sg, struct ib_udata *udata)	3658	u32 max_num_sg)
3573	{	3659	{
3574	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);	3660	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
3575	struct bnxt_re_dev *rdev = pd->rdev;	3661	struct bnxt_re_dev *rdev = pd->rdev;

drivers/infiniband/hw/bnxt_re/ib_verbs.h+5−5

@@ -136,14 +136,14 @@ struct bnxt_re_ucontext { · 135 unmodified lines
136	spinlock_t sh_lock; /* protect shpg */	136	spinlock_t sh_lock; /* protect shpg */
137	};	137	};
138		138
139	static inline u16 bnxt_re_get_swqe_size(void)	139	static inline u16 bnxt_re_get_swqe_size(int nsge)
140	{	140	{
141	return sizeof(struct sq_send);	141	return sizeof(struct sq_send_hdr) + nsge * sizeof(struct sq_sge);
142	}	142	}
143		143
144	static inline u16 bnxt_re_get_rwqe_size(void)	144	static inline u16 bnxt_re_get_rwqe_size(int nsge)
145	{	145	{
146	return sizeof(struct rq_wqe);	146	return sizeof(struct rq_wqe_hdr) + (nsge * sizeof(struct sq_sge));
147	}	147	}
148		148
149	int bnxt_re_query_device(struct ib_device *ibdev,	149	int bnxt_re_query_device(struct ib_device *ibdev,
@@ -201,7 +201,7 @@ struct ib_mr bnxt_re_get_dma_mr(struct ib_pd pd, int mr_access_flags); · 51 unmodified lines
201	int bnxt_re_map_mr_sg(struct ib_mr ib_mr, struct scatterlist sg, int sg_nents,	201	int bnxt_re_map_mr_sg(struct ib_mr ib_mr, struct scatterlist sg, int sg_nents,
202	unsigned int *sg_offset);	202	unsigned int *sg_offset);
203	struct ib_mr bnxt_re_alloc_mr(struct ib_pd ib_pd, enum ib_mr_type mr_type,	203	struct ib_mr bnxt_re_alloc_mr(struct ib_pd ib_pd, enum ib_mr_type mr_type,
204	u32 max_num_sg, struct ib_udata *udata);	204	u32 max_num_sg);
205	int bnxt_re_dereg_mr(struct ib_mr mr, struct ib_udata udata);	205	int bnxt_re_dereg_mr(struct ib_mr mr, struct ib_udata udata);
206	struct ib_mw bnxt_re_alloc_mw(struct ib_pd ib_pd, enum ib_mw_type type,	206	struct ib_mw bnxt_re_alloc_mw(struct ib_pd ib_pd, enum ib_mw_type type,
207	struct ib_udata *udata);	207	struct ib_udata *udata);

drivers/infiniband/hw/bnxt_re/main.c+17−6

 static void bnxt_re_dealloc_driver(struct ib_device *ib_dev);
 static void bnxt_re_stop_irq(void *handle);
+static void bnxt_re_set_drv_mode(struct bnxt_re_dev *rdev, u8 mode)
+{
+	struct bnxt_qplib_chip_ctx *cctx;
+	cctx = rdev->chip_ctx;
+	cctx->modes.wqe_mode = bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx) ?
+			       mode : BNXT_QPLIB_WQE_MODE_STATIC;
+}
 static void bnxt_re_destroy_chip_ctx(struct bnxt_re_dev *rdev)
 {
 	struct bnxt_qplib_chip_ctx *chip_ctx;
 	kfree(chip_ctx);
 }
-static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev)
+static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev, u8 wqe_mode)
 {
 	struct bnxt_qplib_chip_ctx *chip_ctx;
 	struct bnxt_en_dev *en_dev;
 	rdev->qplib_res.cctx = rdev->chip_ctx;
 	rdev->rcfw.res = &rdev->qplib_res;
+	bnxt_re_set_drv_mode(rdev, wqe_mode);
 	return 0;
 }
 	schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000));
 }
-static int bnxt_re_dev_init(struct bnxt_re_dev *rdev)
+static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 wqe_mode)
 {
 	struct bnxt_qplib_creq_ctx *creq;
 	struct bnxt_re_ring_attr rattr;
 	}
 	set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags);
-	rc = bnxt_re_setup_chip_ctx(rdev);
+	rc = bnxt_re_setup_chip_ctx(rdev, wqe_mode);
 	if (rc) {
 		ibdev_err(&rdev->ibdev, "Failed to get chip context\n");
 		return -EINVAL;
 }
 static int bnxt_re_add_device(struct bnxt_re_dev **rdev,
-			      struct net_device *netdev)
+			      struct net_device *netdev, u8 wqe_mode)
 {
 	int rc;
 	}
 	pci_dev_get((*rdev)->en_dev->pdev);
-	rc = bnxt_re_dev_init(*rdev);
+	rc = bnxt_re_dev_init(*rdev, wqe_mode);
 	if (rc) {
 		pci_dev_put((*rdev)->en_dev->pdev);
 		bnxt_re_dev_unreg(*rdev);
 	case NETDEV_REGISTER:
 		if (rdev)
 			break;
-		rc = bnxt_re_add_device(&rdev, real_dev);
+		rc = bnxt_re_add_device(&rdev, real_dev,
+					BNXT_QPLIB_WQE_MODE_STATIC);
 		if (!rc)
 			sch_work = true;
 		release = false;

drivers/infiniband/hw/bnxt_re/qplib_fp.c+425−326

@@ -178,11 +178,11 @@ static void bnxt_qplib_free_qp_hdr_buf(struct bnxt_qplib_res *res, · 177 unmodified lines
178		178
179	if (qp->rq_hdr_buf)	179	if (qp->rq_hdr_buf)
180	dma_free_coherent(&res->pdev->dev,	180	dma_free_coherent(&res->pdev->dev,
181	rq->hwq.max_elements * qp->rq_hdr_buf_size,	181	rq->max_wqe * qp->rq_hdr_buf_size,
182	qp->rq_hdr_buf, qp->rq_hdr_buf_map);	182	qp->rq_hdr_buf, qp->rq_hdr_buf_map);
183	if (qp->sq_hdr_buf)	183	if (qp->sq_hdr_buf)
184	dma_free_coherent(&res->pdev->dev,	184	dma_free_coherent(&res->pdev->dev,
185	sq->hwq.max_elements * qp->sq_hdr_buf_size,	185	sq->max_wqe * qp->sq_hdr_buf_size,
186	qp->sq_hdr_buf, qp->sq_hdr_buf_map);	186	qp->sq_hdr_buf, qp->sq_hdr_buf_map);
187	qp->rq_hdr_buf = NULL;	187	qp->rq_hdr_buf = NULL;
188	qp->sq_hdr_buf = NULL;	188	qp->sq_hdr_buf = NULL;
@@ -199,10 +199,9 @@ static int bnxt_qplib_alloc_qp_hdr_buf(struct bnxt_qplib_res *res, · 10 unmodified lines
199	struct bnxt_qplib_q *sq = &qp->sq;	199	struct bnxt_qplib_q *sq = &qp->sq;
200	int rc = 0;	200	int rc = 0;
201		201
202	if (qp->sq_hdr_buf_size && sq->hwq.max_elements) {	202	if (qp->sq_hdr_buf_size && sq->max_wqe) {
203	qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,	203	qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
204	sq->hwq.max_elements *	204	sq->max_wqe * qp->sq_hdr_buf_size,
205	qp->sq_hdr_buf_size,
206	&qp->sq_hdr_buf_map, GFP_KERNEL);	205	&qp->sq_hdr_buf_map, GFP_KERNEL);
207	if (!qp->sq_hdr_buf) {	206	if (!qp->sq_hdr_buf) {
208	rc = -ENOMEM;	207	rc = -ENOMEM;
@@ -212,9 +211,9 @@ static int bnxt_qplib_alloc_qp_hdr_buf(struct bnxt_qplib_res *res, · 3 unmodified lines
212	}	211	}
213	}	212	}
214		213
215	if (qp->rq_hdr_buf_size && rq->hwq.max_elements) {	214	if (qp->rq_hdr_buf_size && rq->max_wqe) {
216	qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,	215	qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
217	rq->hwq.max_elements *	216	rq->max_wqe *
218	qp->rq_hdr_buf_size,	217	qp->rq_hdr_buf_size,
219	&qp->rq_hdr_buf_map,	218	&qp->rq_hdr_buf_map,
220	GFP_KERNEL);	219	GFP_KERNEL);
@@ -661,6 +660,7 @@ int bnxt_qplib_create_srq(struct bnxt_qplib_res *res, · 440 unmodified lines
661	srq->dbinfo.hwq = &srq->hwq;	660	srq->dbinfo.hwq = &srq->hwq;
662	srq->dbinfo.xid = srq->id;	661	srq->dbinfo.xid = srq->id;
663	srq->dbinfo.db = srq->dpi->dbr;	662	srq->dbinfo.db = srq->dpi->dbr;
		663	srq->dbinfo.max_slot = 1;
664	srq->dbinfo.priv_db = res->dpi_tbl.dbr_bar_reg_iomem;	664	srq->dbinfo.priv_db = res->dpi_tbl.dbr_bar_reg_iomem;
665	if (srq->threshold)	665	if (srq->threshold)
666	bnxt_qplib_armen_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ_ARMENA);	666	bnxt_qplib_armen_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ_ARMENA);
@@ -784,6 +784,28 @@ done: · 117 unmodified lines
784	}	784	}
785		785
786	/* QP */	786	/* QP */
		787
		788	static int bnxt_qplib_alloc_init_swq(struct bnxt_qplib_q *que)
		789	{
		790	int rc = 0;
		791	int indx;
		792
		793	que->swq = kcalloc(que->max_wqe, sizeof(*que->swq), GFP_KERNEL);
		794	if (!que->swq) {
		795	rc = -ENOMEM;
		796	goto out;
		797	}
		798
		799	que->swq_start = 0;
		800	que->swq_last = que->max_wqe - 1;
		801	for (indx = 0; indx < que->max_wqe; indx++)
		802	que->swq[indx].next_idx = indx + 1;
		803	que->swq[que->swq_last].next_idx = 0; /* Make it circular */
		804	que->swq_last = 0;
		805	out:
		806	return rc;
		807	}
		808
787	int bnxt_qplib_create_qp1(struct bnxt_qplib_res res, struct bnxt_qplib_qp qp)	809	int bnxt_qplib_create_qp1(struct bnxt_qplib_res res, struct bnxt_qplib_qp qp)
788	{	810	{
789	struct bnxt_qplib_hwq_attr hwq_attr = {};	811	struct bnxt_qplib_hwq_attr hwq_attr = {};
@@ -808,71 +830,63 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res res, struct bnxt_qplib_qp qp) · 18 unmodified lines
808	/* SQ */	830	/* SQ */
809	hwq_attr.res = res;	831	hwq_attr.res = res;
810	hwq_attr.sginfo = &sq->sg_info;	832	hwq_attr.sginfo = &sq->sg_info;
811	hwq_attr.depth = sq->max_wqe;	833	hwq_attr.stride = sizeof(struct sq_sge);
812	hwq_attr.stride = sq->wqe_size;	834	hwq_attr.depth = bnxt_qplib_get_depth(sq);
813	hwq_attr.type = HWQ_TYPE_QUEUE;	835	hwq_attr.type = HWQ_TYPE_QUEUE;
814	rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);	836	rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);
815	if (rc)	837	if (rc)
816	goto exit;	838	goto exit;
817		839
818	sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);	840	rc = bnxt_qplib_alloc_init_swq(sq);
819	if (!sq->swq) {	841	if (rc)
820	rc = -ENOMEM;
821	goto fail_sq;	842	goto fail_sq;
822	}	843
		844	req.sq_size = cpu_to_le32(bnxt_qplib_set_sq_size(sq, qp->wqe_mode));
823	pbl = &sq->hwq.pbl[PBL_LVL_0];	845	pbl = &sq->hwq.pbl[PBL_LVL_0];
824	req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);	846	req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
825	pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) <<	847	pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) <<
826	CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT);	848	CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT);
827	pg_sz_lvl \|= (sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK);	849	pg_sz_lvl \|= (sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK);
828	req.sq_pg_size_sq_lvl = pg_sz_lvl;	850	req.sq_pg_size_sq_lvl = pg_sz_lvl;
		851	req.sq_fwo_sq_sge =
		852	cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) <<
		853	CMDQ_CREATE_QP1_SQ_SGE_SFT);
		854	req.scq_cid = cpu_to_le32(qp->scq->id);
829		855
830	if (qp->scq)
831	req.scq_cid = cpu_to_le32(qp->scq->id);
832	/* RQ */	856	/* RQ */
833	if (rq->max_wqe) {	857	if (rq->max_wqe) {
834	hwq_attr.res = res;	858	hwq_attr.res = res;
835	hwq_attr.sginfo = &rq->sg_info;	859	hwq_attr.sginfo = &rq->sg_info;
836	hwq_attr.stride = rq->wqe_size;	860	hwq_attr.stride = sizeof(struct sq_sge);
837	hwq_attr.depth = qp->rq.max_wqe;	861	hwq_attr.depth = bnxt_qplib_get_depth(rq);
838	hwq_attr.type = HWQ_TYPE_QUEUE;	862	hwq_attr.type = HWQ_TYPE_QUEUE;
839	rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);	863	rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);
840	if (rc)	864	if (rc)
841	goto fail_sq;	865	goto sq_swq;
842		866	rc = bnxt_qplib_alloc_init_swq(rq);
843	rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),	867	if (rc)
844	GFP_KERNEL);
845	if (!rq->swq) {
846	rc = -ENOMEM;
847	goto fail_rq;	868	goto fail_rq;
848	}	869	req.rq_size = cpu_to_le32(rq->max_wqe);
849	pbl = &rq->hwq.pbl[PBL_LVL_0];	870	pbl = &rq->hwq.pbl[PBL_LVL_0];
850	req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);	871	req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
851	pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) <<	872	pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) <<
852	CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT);	873	CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT);
853	pg_sz_lvl \|= (rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK);	874	pg_sz_lvl \|= (rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK);
854	req.rq_pg_size_rq_lvl = pg_sz_lvl;	875	req.rq_pg_size_rq_lvl = pg_sz_lvl;
855	if (qp->rcq)	876	req.rq_fwo_rq_sge =
856	req.rcq_cid = cpu_to_le32(qp->rcq->id);	877	cpu_to_le16((rq->max_sge &
		878	CMDQ_CREATE_QP1_RQ_SGE_MASK) <<
		879	CMDQ_CREATE_QP1_RQ_SGE_SFT);
857	}	880	}
		881	req.rcq_cid = cpu_to_le32(qp->rcq->id);
858	/* Header buffer - allow hdr_buf pass in */	882	/* Header buffer - allow hdr_buf pass in */
859	rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);	883	rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
860	if (rc) {	884	if (rc) {
861	rc = -ENOMEM;	885	rc = -ENOMEM;
862	goto fail;	886	goto rq_rwq;
863	}	887	}
864	qp_flags \|= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE;	888	qp_flags \|= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE;
865	req.qp_flags = cpu_to_le32(qp_flags);	889	req.qp_flags = cpu_to_le32(qp_flags);
866	req.sq_size = cpu_to_le32(sq->hwq.max_elements);
867	req.rq_size = cpu_to_le32(rq->hwq.max_elements);
868
869	req.sq_fwo_sq_sge =
870	cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) <<
871	CMDQ_CREATE_QP1_SQ_SGE_SFT);
872	req.rq_fwo_rq_sge =
873	cpu_to_le16((rq->max_sge & CMDQ_CREATE_QP1_RQ_SGE_MASK) <<
874	CMDQ_CREATE_QP1_RQ_SGE_SFT);
875
876	req.pd_id = cpu_to_le32(qp->pd->id);	890	req.pd_id = cpu_to_le32(qp->pd->id);
877		891
878	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,	892	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
@@ -886,10 +900,12 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res res, struct bnxt_qplib_qp qp) · 7 unmodified lines
886	sq->dbinfo.hwq = &sq->hwq;	900	sq->dbinfo.hwq = &sq->hwq;
887	sq->dbinfo.xid = qp->id;	901	sq->dbinfo.xid = qp->id;
888	sq->dbinfo.db = qp->dpi->dbr;	902	sq->dbinfo.db = qp->dpi->dbr;
		903	sq->dbinfo.max_slot = bnxt_qplib_set_sq_max_slot(qp->wqe_mode);
889	if (rq->max_wqe) {	904	if (rq->max_wqe) {
890	rq->dbinfo.hwq = &rq->hwq;	905	rq->dbinfo.hwq = &rq->hwq;
891	rq->dbinfo.xid = qp->id;	906	rq->dbinfo.xid = qp->id;
892	rq->dbinfo.db = qp->dpi->dbr;	907	rq->dbinfo.db = qp->dpi->dbr;
		908	rq->dbinfo.max_slot = bnxt_qplib_set_rq_max_slot(rq->wqe_size);
893	}	909	}
894	rcfw->qp_tbl[qp->id].qp_id = qp->id;	910	rcfw->qp_tbl[qp->id].qp_id = qp->id;
895	rcfw->qp_tbl[qp->id].qp_handle = (void *)qp;	911	rcfw->qp_tbl[qp->id].qp_handle = (void *)qp;
@@ -898,12 +914,14 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res res, struct bnxt_qplib_qp qp) · 2 unmodified lines
898		914
899	fail:	915	fail:
900	bnxt_qplib_free_qp_hdr_buf(res, qp);	916	bnxt_qplib_free_qp_hdr_buf(res, qp);
		917	rq_rwq:
		918	kfree(rq->swq);
901	fail_rq:	919	fail_rq:
902	bnxt_qplib_free_hwq(res, &rq->hwq);	920	bnxt_qplib_free_hwq(res, &rq->hwq);
903	kfree(rq->swq);	921	sq_swq:
		922	kfree(sq->swq);
904	fail_sq:	923	fail_sq:
905	bnxt_qplib_free_hwq(res, &sq->hwq);	924	bnxt_qplib_free_hwq(res, &sq->hwq);
906	kfree(sq->swq);
907	exit:	925	exit:
908	return rc;	926	return rc;
909	}	927	}
@@ -912,26 +930,18 @@ static void bnxt_qplib_init_psn_ptr(struct bnxt_qplib_qp *qp, int size) · 2 unmodified lines
912	{	930	{
913	struct bnxt_qplib_hwq *hwq;	931	struct bnxt_qplib_hwq *hwq;
914	struct bnxt_qplib_q *sq;	932	struct bnxt_qplib_q *sq;
915	u64 fpsne, psne, psn_pg;	933	u64 fpsne, psn_pg;
916	u16 indx_pad = 0, indx;	934	u16 indx_pad = 0;
917	u16 pg_num, pg_indx;
918	u64 *page;
919		935
920	sq = &qp->sq;	936	sq = &qp->sq;
921	hwq = &sq->hwq;	937	hwq = &sq->hwq;
922		938	fpsne = (u64)bnxt_qplib_get_qe(hwq, hwq->depth, &psn_pg);
923	fpsne = (u64)bnxt_qplib_get_qe(hwq, hwq->max_elements, &psn_pg);
924	if (!IS_ALIGNED(fpsne, PAGE_SIZE))	939	if (!IS_ALIGNED(fpsne, PAGE_SIZE))
925	indx_pad = ALIGN(fpsne, PAGE_SIZE) / size;	940	indx_pad = ALIGN(fpsne, PAGE_SIZE) / size;
926		941
927	page = (u64 *)psn_pg;	942	hwq->pad_pgofft = indx_pad;
928	for (indx = 0; indx < hwq->max_elements; indx++) {	943	hwq->pad_pg = (u64 *)psn_pg;
929	pg_num = (indx + indx_pad) / (PAGE_SIZE / size);	944	hwq->pad_stride = size;
930	pg_indx = (indx + indx_pad) % (PAGE_SIZE / size);
931	psne = page[pg_num] + pg_indx * size;
932	sq->swq[indx].psn_ext = (struct sq_psn_search_ext *)psne;
933	sq->swq[indx].psn_search = (struct sq_psn_search *)psne;
934	}
935	}	945	}
936		946
937	int bnxt_qplib_create_qp(struct bnxt_qplib_res res, struct bnxt_qplib_qp qp)	947	int bnxt_qplib_create_qp(struct bnxt_qplib_res res, struct bnxt_qplib_qp qp)
@@ -944,12 +954,12 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res res, struct bnxt_qplib_qp qp) · 6 unmodified lines
944	struct creq_create_qp_resp resp;	954	struct creq_create_qp_resp resp;
945	int rc, req_size, psn_sz = 0;	955	int rc, req_size, psn_sz = 0;
946	struct bnxt_qplib_hwq *xrrq;	956	struct bnxt_qplib_hwq *xrrq;
947	u16 cmd_flags = 0, max_ssge;
948	struct bnxt_qplib_pbl *pbl;	957	struct bnxt_qplib_pbl *pbl;
949	struct cmdq_create_qp req;	958	struct cmdq_create_qp req;
		959	u16 cmd_flags = 0;
950	u32 qp_flags = 0;	960	u32 qp_flags = 0;
951	u8 pg_sz_lvl;	961	u8 pg_sz_lvl;
952	u16 max_rsge;	962	u16 nsge;
953		963
954	RCFW_CMD_PREP(req, CREATE_QP, cmd_flags);	964	RCFW_CMD_PREP(req, CREATE_QP, cmd_flags);
955		965
@@ -967,97 +977,78 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res res, struct bnxt_qplib_qp qp) · 11 unmodified lines
967		977
968	hwq_attr.res = res;	978	hwq_attr.res = res;
969	hwq_attr.sginfo = &sq->sg_info;	979	hwq_attr.sginfo = &sq->sg_info;
970	hwq_attr.stride = sq->wqe_size;	980	hwq_attr.stride = sizeof(struct sq_sge);
971	hwq_attr.depth = sq->max_wqe;	981	hwq_attr.depth = bnxt_qplib_get_depth(sq);
972	hwq_attr.aux_stride = psn_sz;	982	hwq_attr.aux_stride = psn_sz;
973	hwq_attr.aux_depth = hwq_attr.depth;	983	hwq_attr.aux_depth = bnxt_qplib_set_sq_size(sq, qp->wqe_mode);
974	hwq_attr.type = HWQ_TYPE_QUEUE;	984	hwq_attr.type = HWQ_TYPE_QUEUE;
975	rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);	985	rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);
976	if (rc)	986	if (rc)
977	goto exit;	987	goto exit;
978		988
979	sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);	989	rc = bnxt_qplib_alloc_init_swq(sq);
980	if (!sq->swq) {	990	if (rc)
981	rc = -ENOMEM;
982	goto fail_sq;	991	goto fail_sq;
983	}
984		992
985	if (psn_sz)	993	if (psn_sz)
986	bnxt_qplib_init_psn_ptr(qp, psn_sz);	994	bnxt_qplib_init_psn_ptr(qp, psn_sz);
987		995
		996	req.sq_size = cpu_to_le32(bnxt_qplib_set_sq_size(sq, qp->wqe_mode));
988	pbl = &sq->hwq.pbl[PBL_LVL_0];	997	pbl = &sq->hwq.pbl[PBL_LVL_0];
989	req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);	998	req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
990	pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) <<	999	pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) <<
991	CMDQ_CREATE_QP_SQ_PG_SIZE_SFT);	1000	CMDQ_CREATE_QP_SQ_PG_SIZE_SFT);
992	pg_sz_lvl \|= (sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK);	1001	pg_sz_lvl \|= (sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK);
993	req.sq_pg_size_sq_lvl = pg_sz_lvl;	1002	req.sq_pg_size_sq_lvl = pg_sz_lvl;
994		1003	req.sq_fwo_sq_sge =
995	if (qp->scq)	1004	cpu_to_le16(((sq->max_sge & CMDQ_CREATE_QP_SQ_SGE_MASK) <<
996	req.scq_cid = cpu_to_le32(qp->scq->id);	1005	CMDQ_CREATE_QP_SQ_SGE_SFT) \| 0);
		1006	req.scq_cid = cpu_to_le32(qp->scq->id);
997		1007
998	/* RQ */	1008	/* RQ */
999	if (rq->max_wqe) {	1009	if (!qp->srq) {
1000	hwq_attr.res = res;	1010	hwq_attr.res = res;
1001	hwq_attr.sginfo = &rq->sg_info;	1011	hwq_attr.sginfo = &rq->sg_info;
1002	hwq_attr.stride = rq->wqe_size;	1012	hwq_attr.stride = sizeof(struct sq_sge);
1003	hwq_attr.depth = rq->max_wqe;	1013	hwq_attr.depth = bnxt_qplib_get_depth(rq);
1004	hwq_attr.aux_stride = 0;	1014	hwq_attr.aux_stride = 0;
1005	hwq_attr.aux_depth = 0;	1015	hwq_attr.aux_depth = 0;
1006	hwq_attr.type = HWQ_TYPE_QUEUE;	1016	hwq_attr.type = HWQ_TYPE_QUEUE;
1007	rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);	1017	rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);
1008	if (rc)	1018	if (rc)
1009	goto fail_sq;	1019	goto sq_swq;
1010		1020	rc = bnxt_qplib_alloc_init_swq(rq);
1011	rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),	1021	if (rc)
1012	GFP_KERNEL);
1013	if (!rq->swq) {
1014	rc = -ENOMEM;
1015	goto fail_rq;	1022	goto fail_rq;
1016	}	1023
		1024	req.rq_size = cpu_to_le32(rq->max_wqe);
1017	pbl = &rq->hwq.pbl[PBL_LVL_0];	1025	pbl = &rq->hwq.pbl[PBL_LVL_0];
1018	req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);	1026	req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
1019	pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) <<	1027	pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) <<
1020	CMDQ_CREATE_QP_RQ_PG_SIZE_SFT);	1028	CMDQ_CREATE_QP_RQ_PG_SIZE_SFT);
1021	pg_sz_lvl \|= (rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK);	1029	pg_sz_lvl \|= (rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK);
1022	req.rq_pg_size_rq_lvl = pg_sz_lvl;	1030	req.rq_pg_size_rq_lvl = pg_sz_lvl;
		1031	nsge = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
		1032	6 : rq->max_sge;
		1033	req.rq_fwo_rq_sge =
		1034	cpu_to_le16(((nsge &
		1035	CMDQ_CREATE_QP_RQ_SGE_MASK) <<
		1036	CMDQ_CREATE_QP_RQ_SGE_SFT) \| 0);
1023	} else {	1037	} else {
1024	/* SRQ */	1038	/* SRQ */
1025	if (qp->srq) {	1039	qp_flags \|= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED;
1026	qp_flags \|= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED;	1040	req.srq_cid = cpu_to_le32(qp->srq->id);
1027	req.srq_cid = cpu_to_le32(qp->srq->id);
1028	}
1029	}	1041	}
1030		1042	req.rcq_cid = cpu_to_le32(qp->rcq->id);
1031	if (qp->rcq)
1032	req.rcq_cid = cpu_to_le32(qp->rcq->id);
1033		1043
1034	qp_flags \|= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE;	1044	qp_flags \|= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE;
1035	qp_flags \|= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED;	1045	qp_flags \|= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED;
1036	if (qp->sig_type)	1046	if (qp->sig_type)
1037	qp_flags \|= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION;	1047	qp_flags \|= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION;
		1048	if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE)
		1049	qp_flags \|= CMDQ_CREATE_QP_QP_FLAGS_VARIABLE_SIZED_WQE_ENABLED;
1038	req.qp_flags = cpu_to_le32(qp_flags);	1050	req.qp_flags = cpu_to_le32(qp_flags);
1039		1051
1040	req.sq_size = cpu_to_le32(sq->hwq.max_elements);
1041	req.rq_size = cpu_to_le32(rq->hwq.max_elements);
1042	qp->sq_hdr_buf = NULL;
1043	qp->rq_hdr_buf = NULL;
1044
1045	rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
1046	if (rc)
1047	goto fail_rq;
1048
1049	/* CTRL-22434: Irrespective of the requested SGE count on the SQ
1050	* always create the QP with max send sges possible if the requested
1051	* inline size is greater than 0.
1052	*/
1053	max_ssge = qp->max_inline_data ? 6 : sq->max_sge;
1054	req.sq_fwo_sq_sge = cpu_to_le16(
1055	((max_ssge & CMDQ_CREATE_QP_SQ_SGE_MASK)
1056	<< CMDQ_CREATE_QP_SQ_SGE_SFT) \| 0);
1057	max_rsge = bnxt_qplib_is_chip_gen_p5(res->cctx) ? 6 : rq->max_sge;
1058	req.rq_fwo_rq_sge = cpu_to_le16(
1059	((max_rsge & CMDQ_CREATE_QP_RQ_SGE_MASK)
1060	<< CMDQ_CREATE_QP_RQ_SGE_SFT) \| 0);
1061	/* ORRQ and IRRQ */	1052	/* ORRQ and IRRQ */
1062	if (psn_sz) {	1053	if (psn_sz) {
1063	xrrq = &qp->orrq;	1054	xrrq = &qp->orrq;
@@ -1078,7 +1069,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res res, struct bnxt_qplib_qp qp) · 14 unmodified lines
1078	hwq_attr.type = HWQ_TYPE_CTX;	1069	hwq_attr.type = HWQ_TYPE_CTX;
1079	rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr);	1070	rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr);
1080	if (rc)	1071	if (rc)
1081	goto fail_buf_free;	1072	goto rq_swq;
1082	pbl = &xrrq->pbl[PBL_LVL_0];	1073	pbl = &xrrq->pbl[PBL_LVL_0];
1083	req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);	1074	req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
1084		1075
@@ -1113,30 +1104,29 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res res, struct bnxt_qplib_qp qp) · 28 unmodified lines
1113	sq->dbinfo.hwq = &sq->hwq;	1104	sq->dbinfo.hwq = &sq->hwq;
1114	sq->dbinfo.xid = qp->id;	1105	sq->dbinfo.xid = qp->id;
1115	sq->dbinfo.db = qp->dpi->dbr;	1106	sq->dbinfo.db = qp->dpi->dbr;
		1107	sq->dbinfo.max_slot = bnxt_qplib_set_sq_max_slot(qp->wqe_mode);
1116	if (rq->max_wqe) {	1108	if (rq->max_wqe) {
1117	rq->dbinfo.hwq = &rq->hwq;	1109	rq->dbinfo.hwq = &rq->hwq;
1118	rq->dbinfo.xid = qp->id;	1110	rq->dbinfo.xid = qp->id;
1119	rq->dbinfo.db = qp->dpi->dbr;	1111	rq->dbinfo.db = qp->dpi->dbr;
		1112	rq->dbinfo.max_slot = bnxt_qplib_set_rq_max_slot(rq->wqe_size);
1120	}	1113	}
1121	rcfw->qp_tbl[qp->id].qp_id = qp->id;	1114	rcfw->qp_tbl[qp->id].qp_id = qp->id;
1122	rcfw->qp_tbl[qp->id].qp_handle = (void *)qp;	1115	rcfw->qp_tbl[qp->id].qp_handle = (void *)qp;
1123		1116
1124	return 0;	1117	return 0;
1125
1126	fail:	1118	fail:
1127	if (qp->irrq.max_elements)	1119	bnxt_qplib_free_hwq(res, &qp->irrq);
1128	bnxt_qplib_free_hwq(res, &qp->irrq);
1129	fail_orrq:	1120	fail_orrq:
1130	if (qp->orrq.max_elements)	1121	bnxt_qplib_free_hwq(res, &qp->orrq);
1131	bnxt_qplib_free_hwq(res, &qp->orrq);	1122	rq_swq:
1132	fail_buf_free:	1123	kfree(rq->swq);
1133	bnxt_qplib_free_qp_hdr_buf(res, qp);
1134	fail_rq:	1124	fail_rq:
1135	bnxt_qplib_free_hwq(res, &rq->hwq);	1125	bnxt_qplib_free_hwq(res, &rq->hwq);
1136	kfree(rq->swq);	1126	sq_swq:
		1127	kfree(sq->swq);
1137	fail_sq:	1128	fail_sq:
1138	bnxt_qplib_free_hwq(res, &sq->hwq);	1129	bnxt_qplib_free_hwq(res, &sq->hwq);
1139	kfree(sq->swq);
1140	exit:	1130	exit:
1141	return rc;	1131	return rc;
1142	}	1132	}
@@ -1512,7 +1502,7 @@ void bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp qp, · 369 unmodified lines
1512	memset(sge, 0, sizeof(*sge));	1502	memset(sge, 0, sizeof(*sge));
1513		1503
1514	if (qp->sq_hdr_buf) {	1504	if (qp->sq_hdr_buf) {
1515	sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);	1505	sw_prod = sq->swq_start;
1516	sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map +	1506	sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map +
1517	sw_prod * qp->sq_hdr_buf_size);	1507	sw_prod * qp->sq_hdr_buf_size);
1518	sge->lkey = 0xFFFFFFFF;	1508	sge->lkey = 0xFFFFFFFF;
@@ -1526,7 +1516,7 @@ u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp) · 7 unmodified lines
1526	{	1516	{
1527	struct bnxt_qplib_q *rq = &qp->rq;	1517	struct bnxt_qplib_q *rq = &qp->rq;
1528		1518
1529	return HWQ_CMP(rq->hwq.prod, &rq->hwq);	1519	return rq->swq_start;
1530	}	1520	}
1531		1521
1532	dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index)	1522	dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index)
@@ -1543,7 +1533,7 @@ void bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp qp, · 10 unmodified lines
1543	memset(sge, 0, sizeof(*sge));	1533	memset(sge, 0, sizeof(*sge));
1544		1534
1545	if (qp->rq_hdr_buf) {	1535	if (qp->rq_hdr_buf) {
1546	sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);	1536	sw_prod = rq->swq_start;
1547	sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map +	1537	sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map +
1548	sw_prod * qp->rq_hdr_buf_size);	1538	sw_prod * qp->rq_hdr_buf_size);
1549	sge->lkey = 0xFFFFFFFF;	1539	sge->lkey = 0xFFFFFFFF;
@@ -1562,6 +1552,8 @@ static void bnxt_qplib_fill_psn_search(struct bnxt_qplib_qp *qp, · 12 unmodified lines
1562	u32 flg_npsn;	1552	u32 flg_npsn;
1563	u32 op_spsn;	1553	u32 op_spsn;
1564		1554
		1555	if (!swq->psn_search)
		1556	return;
1565	psns = swq->psn_search;	1557	psns = swq->psn_search;
1566	psns_ext = swq->psn_ext;	1558	psns_ext = swq->psn_ext;
1567		1559
@@ -1575,12 +1567,122 @@ static void bnxt_qplib_fill_psn_search(struct bnxt_qplib_qp *qp, · 7 unmodified lines
1575	if (bnxt_qplib_is_chip_gen_p5(qp->cctx)) {	1567	if (bnxt_qplib_is_chip_gen_p5(qp->cctx)) {
1576	psns_ext->opcode_start_psn = cpu_to_le32(op_spsn);	1568	psns_ext->opcode_start_psn = cpu_to_le32(op_spsn);
1577	psns_ext->flags_next_psn = cpu_to_le32(flg_npsn);	1569	psns_ext->flags_next_psn = cpu_to_le32(flg_npsn);
		1570	psns_ext->start_slot_idx = cpu_to_le16(swq->slot_idx);
1578	} else {	1571	} else {
1579	psns->opcode_start_psn = cpu_to_le32(op_spsn);	1572	psns->opcode_start_psn = cpu_to_le32(op_spsn);
1580	psns->flags_next_psn = cpu_to_le32(flg_npsn);	1573	psns->flags_next_psn = cpu_to_le32(flg_npsn);
1581	}	1574	}
1582	}	1575	}
1583		1576
		1577	static int bnxt_qplib_put_inline(struct bnxt_qplib_qp *qp,
		1578	struct bnxt_qplib_swqe *wqe,
		1579	u16 *idx)
		1580	{
		1581	struct bnxt_qplib_hwq *hwq;
		1582	int len, t_len, offt;
		1583	bool pull_dst = true;
		1584	void *il_dst = NULL;
		1585	void *il_src = NULL;
		1586	int t_cplen, cplen;
		1587	int indx;
		1588
		1589	hwq = &qp->sq.hwq;
		1590	t_len = 0;
		1591	for (indx = 0; indx < wqe->num_sge; indx++) {
		1592	len = wqe->sg_list[indx].size;
		1593	il_src = (void *)wqe->sg_list[indx].addr;
		1594	t_len += len;
		1595	if (t_len > qp->max_inline_data)
		1596	goto bad;
		1597	while (len) {
		1598	if (pull_dst) {
		1599	pull_dst = false;
		1600	il_dst = bnxt_qplib_get_prod_qe(hwq, *idx);
		1601	(*idx)++;
		1602	t_cplen = 0;
		1603	offt = 0;
		1604	}
		1605	cplen = min_t(int, len, sizeof(struct sq_sge));
		1606	cplen = min_t(int, cplen,
		1607	(sizeof(struct sq_sge) - offt));
		1608	memcpy(il_dst, il_src, cplen);
		1609	t_cplen += cplen;
		1610	il_src += cplen;
		1611	il_dst += cplen;
		1612	offt += cplen;
		1613	len -= cplen;
		1614	if (t_cplen == sizeof(struct sq_sge))
		1615	pull_dst = true;
		1616	}
		1617	}
		1618
		1619	return t_len;
		1620	bad:
		1621	return -ENOMEM;
		1622	}
		1623
		1624	static u32 bnxt_qplib_put_sges(struct bnxt_qplib_hwq *hwq,
		1625	struct bnxt_qplib_sge *ssge,
		1626	u16 nsge, u16 *idx)
		1627	{
		1628	struct sq_sge *dsge;
		1629	int indx, len = 0;
		1630
		1631	for (indx = 0; indx < nsge; indx++, (*idx)++) {
		1632	dsge = bnxt_qplib_get_prod_qe(hwq, *idx);
		1633	dsge->va_or_pa = cpu_to_le64(ssge[indx].addr);
		1634	dsge->l_key = cpu_to_le32(ssge[indx].lkey);
		1635	dsge->size = cpu_to_le32(ssge[indx].size);
		1636	len += ssge[indx].size;
		1637	}
		1638
		1639	return len;
		1640	}
		1641
		1642	static u16 bnxt_qplib_required_slots(struct bnxt_qplib_qp *qp,
		1643	struct bnxt_qplib_swqe *wqe,
		1644	u16 wqe_sz, u16 qdf, u8 mode)
		1645	{
		1646	u32 ilsize, bytes;
		1647	u16 nsge;
		1648	u16 slot;
		1649
		1650	nsge = wqe->num_sge;
		1651	/* Adding sq_send_hdr is a misnomer, for rq also hdr size is same. */
		1652	bytes = sizeof(struct sq_send_hdr) + nsge * sizeof(struct sq_sge);
		1653	if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) {
		1654	ilsize = bnxt_qplib_calc_ilsize(wqe, qp->max_inline_data);
		1655	bytes = ALIGN(ilsize, sizeof(struct sq_sge));
		1656	bytes += sizeof(struct sq_send_hdr);
		1657	}
		1658
		1659	*qdf = __xlate_qfd(qp->sq.q_full_delta, bytes);
		1660	slot = bytes >> 4;
		1661	*wqe_sz = slot;
		1662	if (mode == BNXT_QPLIB_WQE_MODE_STATIC)
		1663	slot = 8;
		1664	return slot;
		1665	}
		1666
		1667	static void bnxt_qplib_pull_psn_buff(struct bnxt_qplib_q *sq,
		1668	struct bnxt_qplib_swq *swq)
		1669	{
		1670	struct bnxt_qplib_hwq *hwq;
		1671	u32 pg_num, pg_indx;
		1672	void *buff;
		1673	u32 tail;
		1674
		1675	hwq = &sq->hwq;
		1676	if (!hwq->pad_pg)
		1677	return;
		1678	tail = swq->slot_idx / sq->dbinfo.max_slot;
		1679	pg_num = (tail + hwq->pad_pgofft) / (PAGE_SIZE / hwq->pad_stride);
		1680	pg_indx = (tail + hwq->pad_pgofft) % (PAGE_SIZE / hwq->pad_stride);
		1681	buff = (void )(hwq->pad_pg[pg_num] + pg_indx hwq->pad_stride);
		1682	swq->psn_ext = buff;
		1683	swq->psn_search = buff;
		1684	}
		1685
1584	void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp)	1686	void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp)
1585	{	1687	{
1586	struct bnxt_qplib_q *sq = &qp->sq;	1688	struct bnxt_qplib_q *sq = &qp->sq;
@@ -1594,88 +1696,84 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, · 7 unmodified lines
1594	struct bnxt_qplib_nq_work *nq_work = NULL;	1696	struct bnxt_qplib_nq_work *nq_work = NULL;
1595	int i, rc = 0, data_len = 0, pkt_num = 0;	1697	int i, rc = 0, data_len = 0, pkt_num = 0;
1596	struct bnxt_qplib_q *sq = &qp->sq;	1698	struct bnxt_qplib_q *sq = &qp->sq;
1597	struct sq_send *hw_sq_send_hdr;	1699	struct bnxt_qplib_hwq *hwq;
1598	struct bnxt_qplib_swq *swq;	1700	struct bnxt_qplib_swq *swq;
1599	bool sch_handler = false;	1701	bool sch_handler = false;
1600	struct sq_sge *hw_sge;	1702	u16 wqe_sz, qdf = 0;
1601	u8 wqe_size16;	1703	void *base_hdr;
		1704	void *ext_hdr;
1602	__le32 temp32;	1705	__le32 temp32;
1603	u32 sw_prod;	1706	u32 wqe_idx;
		1707	u32 slots;
		1708	u16 idx;
1604		1709
1605	if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS) {	1710	hwq = &sq->hwq;
1606	if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {	1711	if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS &&
1607	sch_handler = true;	1712	qp->state != CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1608	dev_dbg(&sq->hwq.pdev->dev,	1713	dev_err(&hwq->pdev->dev,
1609	"%s Error QP. Scheduling for poll_cq\n",	1714	"QPLIB: FP: QP (0x%x) is in the 0x%x state",
1610	__func__);	1715	qp->id, qp->state);
1611	goto queue_err;	1716	rc = -EINVAL;
1612	}	1717	goto done;
1613	}	1718	}
1614		1719
1615	if (bnxt_qplib_queue_full(sq)) {	1720	slots = bnxt_qplib_required_slots(qp, wqe, &wqe_sz, &qdf, qp->wqe_mode);
1616	dev_err(&sq->hwq.pdev->dev,	1721	if (bnxt_qplib_queue_full(sq, slots + qdf)) {
		1722	dev_err(&hwq->pdev->dev,
1617	"prod = %#x cons = %#x qdepth = %#x delta = %#x\n",	1723	"prod = %#x cons = %#x qdepth = %#x delta = %#x\n",
1618	sq->hwq.prod, sq->hwq.cons, sq->hwq.max_elements,	1724	hwq->prod, hwq->cons, hwq->depth, sq->q_full_delta);
1619	sq->q_full_delta);
1620	rc = -ENOMEM;	1725	rc = -ENOMEM;
1621	goto done;	1726	goto done;
1622	}	1727	}
1623	sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);	1728
1624	swq = &sq->swq[sw_prod];	1729	swq = bnxt_qplib_get_swqe(sq, &wqe_idx);
		1730	bnxt_qplib_pull_psn_buff(sq, swq);
		1731
		1732	idx = 0;
		1733	swq->slot_idx = hwq->prod;
		1734	swq->slots = slots;
1625	swq->wr_id = wqe->wr_id;	1735	swq->wr_id = wqe->wr_id;
1626	swq->type = wqe->type;	1736	swq->type = wqe->type;
1627	swq->flags = wqe->flags;	1737	swq->flags = wqe->flags;
		1738	swq->start_psn = sq->psn & BTH_PSN_MASK;
1628	if (qp->sig_type)	1739	if (qp->sig_type)
1629	swq->flags \|= SQ_SEND_FLAGS_SIGNAL_COMP;	1740	swq->flags \|= SQ_SEND_FLAGS_SIGNAL_COMP;
1630	swq->start_psn = sq->psn & BTH_PSN_MASK;
1631
1632	hw_sq_send_hdr = bnxt_qplib_get_qe(&sq->hwq, sw_prod, NULL);
1633	memset(hw_sq_send_hdr, 0, sq->wqe_size);
1634		1741
1635	if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) {	1742	if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1636	/* Copy the inline data */	1743	sch_handler = true;
1637	if (wqe->inline_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {	1744	dev_dbg(&hwq->pdev->dev,
1638	dev_warn(&sq->hwq.pdev->dev,	1745	"%s Error QP. Scheduling for poll_cq\n", __func__);
1639	"Inline data length > 96 detected\n");	1746	goto queue_err;
1640	data_len = BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH;
1641	} else {
1642	data_len = wqe->inline_len;
1643	}
1644	memcpy(hw_sq_send_hdr->data, wqe->inline_data, data_len);
1645	wqe_size16 = (data_len + 15) >> 4;
1646	} else {
1647	for (i = 0, hw_sge = (struct sq_sge *)hw_sq_send_hdr->data;
1648	i < wqe->num_sge; i++, hw_sge++) {
1649	hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
1650	hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
1651	hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
1652	data_len += wqe->sg_list[i].size;
1653	}
1654	/* Each SGE entry = 1 WQE size16 */
1655	wqe_size16 = wqe->num_sge;
1656	/* HW requires wqe size has room for atleast one SGE even if
1657	* none was supplied by ULP
1658	*/
1659	if (!wqe->num_sge)
1660	wqe_size16++;
1661	}	1747	}
1662		1748
		1749	base_hdr = bnxt_qplib_get_prod_qe(hwq, idx++);
		1750	ext_hdr = bnxt_qplib_get_prod_qe(hwq, idx++);
		1751	memset(base_hdr, 0, sizeof(struct sq_sge));
		1752	memset(ext_hdr, 0, sizeof(struct sq_sge));
		1753
		1754	if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE)
		1755	/* Copy the inline data */
		1756	data_len = bnxt_qplib_put_inline(qp, wqe, &idx);
		1757	else
		1758	data_len = bnxt_qplib_put_sges(hwq, wqe->sg_list, wqe->num_sge,
		1759	&idx);
		1760	if (data_len < 0)
		1761	goto queue_err;
1663	/* Specifics */	1762	/* Specifics */
1664	switch (wqe->type) {	1763	switch (wqe->type) {
1665	case BNXT_QPLIB_SWQE_TYPE_SEND:	1764	case BNXT_QPLIB_SWQE_TYPE_SEND:
1666	if (qp->type == CMDQ_CREATE_QP1_TYPE_GSI) {	1765	if (qp->type == CMDQ_CREATE_QP1_TYPE_GSI) {
		1766	struct sq_send_raweth_qp1_hdr *sqe = base_hdr;
		1767	struct sq_raw_ext_hdr *ext_sqe = ext_hdr;
1667	/* Assemble info for Raw Ethertype QPs */	1768	/* Assemble info for Raw Ethertype QPs */
1668	struct sq_send_raweth_qp1 *sqe =
1669	(struct sq_send_raweth_qp1 *)hw_sq_send_hdr;
1670		1769
1671	sqe->wqe_type = wqe->type;	1770	sqe->wqe_type = wqe->type;
1672	sqe->flags = wqe->flags;	1771	sqe->flags = wqe->flags;
1673	sqe->wqe_size = wqe_size16 +	1772	sqe->wqe_size = wqe_sz;
1674	((offsetof(typeof(*sqe), data) + 15) >> 4);
1675	sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action);	1773	sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action);
1676	sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags);	1774	sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags);
1677	sqe->length = cpu_to_le32(data_len);	1775	sqe->length = cpu_to_le32(data_len);
1678	sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta &	1776	ext_sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta &
1679	SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) <<	1777	SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) <<
1680	SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT);	1778	SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT);
1681		1779
@@ -1685,27 +1783,24 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, · 3 unmodified lines
1685	case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:	1783	case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
1686	case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:	1784	case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
1687	{	1785	{
1688	struct sq_send sqe = (struct sq_send )hw_sq_send_hdr;	1786	struct sq_ud_ext_hdr *ext_sqe = ext_hdr;
		1787	struct sq_send_hdr *sqe = base_hdr;
1689		1788
1690	sqe->wqe_type = wqe->type;	1789	sqe->wqe_type = wqe->type;
1691	sqe->flags = wqe->flags;	1790	sqe->flags = wqe->flags;
1692	sqe->wqe_size = wqe_size16 +	1791	sqe->wqe_size = wqe_sz;
1693	((offsetof(typeof(*sqe), data) + 15) >> 4);	1792	sqe->inv_key_or_imm_data = cpu_to_le32(wqe->send.inv_key);
1694	sqe->inv_key_or_imm_data = cpu_to_le32(
1695	wqe->send.inv_key);
1696	if (qp->type == CMDQ_CREATE_QP_TYPE_UD \|\|	1793	if (qp->type == CMDQ_CREATE_QP_TYPE_UD \|\|
1697	qp->type == CMDQ_CREATE_QP_TYPE_GSI) {	1794	qp->type == CMDQ_CREATE_QP_TYPE_GSI) {
1698	sqe->q_key = cpu_to_le32(wqe->send.q_key);	1795	sqe->q_key = cpu_to_le32(wqe->send.q_key);
1699	sqe->dst_qp = cpu_to_le32(
1700	wqe->send.dst_qp & SQ_SEND_DST_QP_MASK);
1701	sqe->length = cpu_to_le32(data_len);	1796	sqe->length = cpu_to_le32(data_len);
1702	sqe->avid = cpu_to_le32(wqe->send.avid &
1703	SQ_SEND_AVID_MASK);
1704	sq->psn = (sq->psn + 1) & BTH_PSN_MASK;	1797	sq->psn = (sq->psn + 1) & BTH_PSN_MASK;
		1798	ext_sqe->dst_qp = cpu_to_le32(wqe->send.dst_qp &
		1799	SQ_SEND_DST_QP_MASK);
		1800	ext_sqe->avid = cpu_to_le32(wqe->send.avid &
		1801	SQ_SEND_AVID_MASK);
1705	} else {	1802	} else {
1706	sqe->length = cpu_to_le32(data_len);	1803	sqe->length = cpu_to_le32(data_len);
1707	sqe->dst_qp = 0;
1708	sqe->avid = 0;
1709	if (qp->mtu)	1804	if (qp->mtu)
1710	pkt_num = (data_len + qp->mtu - 1) / qp->mtu;	1805	pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1711	if (!pkt_num)	1806	if (!pkt_num)
@@ -1718,16 +1813,16 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, · 6 unmodified lines
1718	case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:	1813	case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
1719	case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:	1814	case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
1720	{	1815	{
1721	struct sq_rdma sqe = (struct sq_rdma )hw_sq_send_hdr;	1816	struct sq_rdma_ext_hdr *ext_sqe = ext_hdr;
		1817	struct sq_rdma_hdr *sqe = base_hdr;
1722		1818
1723	sqe->wqe_type = wqe->type;	1819	sqe->wqe_type = wqe->type;
1724	sqe->flags = wqe->flags;	1820	sqe->flags = wqe->flags;
1725	sqe->wqe_size = wqe_size16 +	1821	sqe->wqe_size = wqe_sz;
1726	((offsetof(typeof(*sqe), data) + 15) >> 4);
1727	sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key);	1822	sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key);
1728	sqe->length = cpu_to_le32((u32)data_len);	1823	sqe->length = cpu_to_le32((u32)data_len);
1729	sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va);	1824	ext_sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va);
1730	sqe->remote_key = cpu_to_le32(wqe->rdma.r_key);	1825	ext_sqe->remote_key = cpu_to_le32(wqe->rdma.r_key);
1731	if (qp->mtu)	1826	if (qp->mtu)
1732	pkt_num = (data_len + qp->mtu - 1) / qp->mtu;	1827	pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1733	if (!pkt_num)	1828	if (!pkt_num)
@@ -1738,14 +1833,15 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, · 4 unmodified lines
1738	case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:	1833	case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
1739	case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:	1834	case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
1740	{	1835	{
1741	struct sq_atomic sqe = (struct sq_atomic )hw_sq_send_hdr;	1836	struct sq_atomic_ext_hdr *ext_sqe = ext_hdr;
		1837	struct sq_atomic_hdr *sqe = base_hdr;
1742		1838
1743	sqe->wqe_type = wqe->type;	1839	sqe->wqe_type = wqe->type;
1744	sqe->flags = wqe->flags;	1840	sqe->flags = wqe->flags;
1745	sqe->remote_key = cpu_to_le32(wqe->atomic.r_key);	1841	sqe->remote_key = cpu_to_le32(wqe->atomic.r_key);
1746	sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va);	1842	sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va);
1747	sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data);	1843	ext_sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data);
1748	sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data);	1844	ext_sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data);
1749	if (qp->mtu)	1845	if (qp->mtu)
1750	pkt_num = (data_len + qp->mtu - 1) / qp->mtu;	1846	pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1751	if (!pkt_num)	1847	if (!pkt_num)
@@ -1755,8 +1851,7 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, · 3 unmodified lines
1755	}	1851	}
1756	case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:	1852	case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
1757	{	1853	{
1758	struct sq_localinvalidate *sqe =	1854	struct sq_localinvalidate *sqe = base_hdr;
1759	(struct sq_localinvalidate *)hw_sq_send_hdr;
1760		1855
1761	sqe->wqe_type = wqe->type;	1856	sqe->wqe_type = wqe->type;
1762	sqe->flags = wqe->flags;	1857	sqe->flags = wqe->flags;
@@ -1766,7 +1861,8 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, · 3 unmodified lines
1766	}	1861	}
1767	case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR:	1862	case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR:
1768	{	1863	{
1769	struct sq_fr_pmr sqe = (struct sq_fr_pmr )hw_sq_send_hdr;	1864	struct sq_fr_pmr_ext_hdr *ext_sqe = ext_hdr;
		1865	struct sq_fr_pmr_hdr *sqe = base_hdr;
1770		1866
1771	sqe->wqe_type = wqe->type;	1867	sqe->wqe_type = wqe->type;
1772	sqe->flags = wqe->flags;	1868	sqe->flags = wqe->flags;
@@ -1790,14 +1886,15 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, · 17 unmodified lines
1790	wqe->frmr.pbl_ptr[i] = cpu_to_le64(	1886	wqe->frmr.pbl_ptr[i] = cpu_to_le64(
1791	wqe->frmr.page_list[i] \|	1887	wqe->frmr.page_list[i] \|
1792	PTU_PTE_VALID);	1888	PTU_PTE_VALID);
1793	sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr);	1889	ext_sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr);
1794	sqe->va = cpu_to_le64(wqe->frmr.va);	1890	ext_sqe->va = cpu_to_le64(wqe->frmr.va);
1795		1891
1796	break;	1892	break;
1797	}	1893	}
1798	case BNXT_QPLIB_SWQE_TYPE_BIND_MW:	1894	case BNXT_QPLIB_SWQE_TYPE_BIND_MW:
1799	{	1895	{
1800	struct sq_bind sqe = (struct sq_bind )hw_sq_send_hdr;	1896	struct sq_bind_ext_hdr *ext_sqe = ext_hdr;
		1897	struct sq_bind_hdr *sqe = base_hdr;
1801		1898
1802	sqe->wqe_type = wqe->type;	1899	sqe->wqe_type = wqe->type;
1803	sqe->flags = wqe->flags;	1900	sqe->flags = wqe->flags;
@@ -1806,9 +1903,8 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, · 2 unmodified lines
1806	(wqe->bind.zero_based ? SQ_BIND_ZERO_BASED : 0);	1903	(wqe->bind.zero_based ? SQ_BIND_ZERO_BASED : 0);
1807	sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key);	1904	sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key);
1808	sqe->l_key = cpu_to_le32(wqe->bind.r_key);	1905	sqe->l_key = cpu_to_le32(wqe->bind.r_key);
1809	sqe->va = cpu_to_le64(wqe->bind.va);	1906	ext_sqe->va = cpu_to_le64(wqe->bind.va);
1810	temp32 = cpu_to_le32(wqe->bind.length);	1907	ext_sqe->length_lo = cpu_to_le32(wqe->bind.length);
1811	memcpy(&sqe->length, &temp32, sizeof(wqe->bind.length));
1812	break;	1908	break;
1813	}	1909	}
1814	default:	1910	default:
@@ -1817,23 +1913,11 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, · 2 unmodified lines
1817	goto done;	1913	goto done;
1818	}	1914	}
1819	swq->next_psn = sq->psn & BTH_PSN_MASK;	1915	swq->next_psn = sq->psn & BTH_PSN_MASK;
1820	if (qp->type == CMDQ_CREATE_QP_TYPE_RC)	1916	bnxt_qplib_fill_psn_search(qp, wqe, swq);
1821	bnxt_qplib_fill_psn_search(qp, wqe, swq);
1822	queue_err:	1917	queue_err:
1823	if (sch_handler) {	1918	bnxt_qplib_swq_mod_start(sq, wqe_idx);
1824	/* Store the ULP info in the software structures */	1919	bnxt_qplib_hwq_incr_prod(hwq, swq->slots);
1825	sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1826	swq = &sq->swq[sw_prod];
1827	swq->wr_id = wqe->wr_id;
1828	swq->type = wqe->type;
1829	swq->flags = wqe->flags;
1830	if (qp->sig_type)
1831	swq->flags \|= SQ_SEND_FLAGS_SIGNAL_COMP;
1832	swq->start_psn = sq->psn & BTH_PSN_MASK;
1833	}
1834	sq->hwq.prod++;
1835	qp->wqe_cnt++;	1920	qp->wqe_cnt++;
1836
1837	done:	1921	done:
1838	if (sch_handler) {	1922	if (sch_handler) {
1839	nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);	1923	nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);
@@ -1843,7 +1927,7 @@ done: · 3 unmodified lines
1843	INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);	1927	INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);
1844	queue_work(qp->scq->nq->cqn_wq, &nq_work->work);	1928	queue_work(qp->scq->nq->cqn_wq, &nq_work->work);
1845	} else {	1929	} else {
1846	dev_err(&sq->hwq.pdev->dev,	1930	dev_err(&hwq->pdev->dev,
1847	"FP: Failed to allocate SQ nq_work!\n");	1931	"FP: Failed to allocate SQ nq_work!\n");
1848	rc = -ENOMEM;	1932	rc = -ENOMEM;
1849	}	1933	}
@@ -1863,58 +1947,65 @@ int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp, · 13 unmodified lines
1863	{	1947	{
1864	struct bnxt_qplib_nq_work *nq_work = NULL;	1948	struct bnxt_qplib_nq_work *nq_work = NULL;
1865	struct bnxt_qplib_q *rq = &qp->rq;	1949	struct bnxt_qplib_q *rq = &qp->rq;
		1950	struct rq_wqe_hdr *base_hdr;
		1951	struct rq_ext_hdr *ext_hdr;
		1952	struct bnxt_qplib_hwq *hwq;
		1953	struct bnxt_qplib_swq *swq;
1866	bool sch_handler = false;	1954	bool sch_handler = false;
1867	struct sq_sge *hw_sge;	1955	u16 wqe_sz, idx;
1868	struct rq_wqe *rqe;	1956	u32 wqe_idx;
1869	int i, rc = 0;	1957	int rc = 0;
1870	u32 sw_prod;
1871		1958
1872	if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {	1959	hwq = &rq->hwq;
1873	sch_handler = true;	1960	if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {
1874	dev_dbg(&rq->hwq.pdev->dev,	1961	dev_err(&hwq->pdev->dev,
1875	"%s: Error QP. Scheduling for poll_cq\n", __func__);	1962	"QPLIB: FP: QP (0x%x) is in the 0x%x state",
1876	goto queue_err;	1963	qp->id, qp->state);
		1964	rc = -EINVAL;
		1965	goto done;
1877	}	1966	}
1878	if (bnxt_qplib_queue_full(rq)) {	1967
1879	dev_err(&rq->hwq.pdev->dev,	1968	if (bnxt_qplib_queue_full(rq, rq->dbinfo.max_slot)) {
		1969	dev_err(&hwq->pdev->dev,
1880	"FP: QP (0x%x) RQ is full!\n", qp->id);	1970	"FP: QP (0x%x) RQ is full!\n", qp->id);
1881	rc = -EINVAL;	1971	rc = -EINVAL;
1882	goto done;	1972	goto done;
1883	}	1973	}
1884	sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1885	rq->swq[sw_prod].wr_id = wqe->wr_id;
1886		1974
1887	rqe = bnxt_qplib_get_qe(&rq->hwq, sw_prod, NULL);	1975	swq = bnxt_qplib_get_swqe(rq, &wqe_idx);
1888	memset(rqe, 0, rq->wqe_size);	1976	swq->wr_id = wqe->wr_id;
		1977	swq->slots = rq->dbinfo.max_slot;
1889		1978
1890	/* Calculate wqe_size16 and data_len */	1979	if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1891	for (i = 0, hw_sge = (struct sq_sge *)rqe->data;	1980	sch_handler = true;
1892	i < wqe->num_sge; i++, hw_sge++) {	1981	dev_dbg(&hwq->pdev->dev,
1893	hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);	1982	"%s: Error QP. Scheduling for poll_cq\n", __func__);
1894	hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);	1983	goto queue_err;
1895	hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
1896	}	1984	}
1897	rqe->wqe_type = wqe->type;
1898	rqe->flags = wqe->flags;
1899	rqe->wqe_size = wqe->num_sge +
1900	((offsetof(typeof(*rqe), data) + 15) >> 4);
1901	/* HW requires wqe size has room for atleast one SGE even if none
1902	* was supplied by ULP
1903	*/
1904	if (!wqe->num_sge)
1905	rqe->wqe_size++;
1906
1907	/* Supply the rqe->wr_id index to the wr_id_tbl for now */
1908	rqe->wr_id[0] = cpu_to_le32(sw_prod);
1909		1985
		1986	idx = 0;
		1987	base_hdr = bnxt_qplib_get_prod_qe(hwq, idx++);
		1988	ext_hdr = bnxt_qplib_get_prod_qe(hwq, idx++);
		1989	memset(base_hdr, 0, sizeof(struct sq_sge));
		1990	memset(ext_hdr, 0, sizeof(struct sq_sge));
		1991	wqe_sz = (sizeof(struct rq_wqe_hdr) +
		1992	wqe->num_sge * sizeof(struct sq_sge)) >> 4;
		1993	bnxt_qplib_put_sges(hwq, wqe->sg_list, wqe->num_sge, &idx);
		1994	if (!wqe->num_sge) {
		1995	struct sq_sge *sge;
		1996
		1997	sge = bnxt_qplib_get_prod_qe(hwq, idx++);
		1998	sge->size = 0;
		1999	wqe_sz++;
		2000	}
		2001	base_hdr->wqe_type = wqe->type;
		2002	base_hdr->flags = wqe->flags;
		2003	base_hdr->wqe_size = wqe_sz;
		2004	base_hdr->wr_id[0] = cpu_to_le32(wqe_idx);
1910	queue_err:	2005	queue_err:
1911	if (sch_handler) {	2006	bnxt_qplib_swq_mod_start(rq, wqe_idx);
1912	/* Store the ULP info in the software structures */	2007	bnxt_qplib_hwq_incr_prod(hwq, swq->slots);
1913	sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);	2008	done:
1914	rq->swq[sw_prod].wr_id = wqe->wr_id;
1915	}
1916
1917	rq->hwq.prod++;
1918	if (sch_handler) {	2009	if (sch_handler) {
1919	nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);	2010	nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);
1920	if (nq_work) {	2011	if (nq_work) {
@@ -1923,12 +2014,12 @@ queue_err: · 2 unmodified lines
1923	INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);	2014	INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);
1924	queue_work(qp->rcq->nq->cqn_wq, &nq_work->work);	2015	queue_work(qp->rcq->nq->cqn_wq, &nq_work->work);
1925	} else {	2016	} else {
1926	dev_err(&rq->hwq.pdev->dev,	2017	dev_err(&hwq->pdev->dev,
1927	"FP: Failed to allocate RQ nq_work!\n");	2018	"FP: Failed to allocate RQ nq_work!\n");
1928	rc = -ENOMEM;	2019	rc = -ENOMEM;
1929	}	2020	}
1930	}	2021	}
1931	done:	2022
1932	return rc;	2023	return rc;
1933	}	2024	}
1934		2025
@@ -2026,20 +2117,19 @@ int bnxt_qplib_destroy_cq(struct bnxt_qplib_res res, struct bnxt_qplib_cq cq) · 91 unmodified lines
2026	static int __flush_sq(struct bnxt_qplib_q sq, struct bnxt_qplib_qp qp,	2117	static int __flush_sq(struct bnxt_qplib_q sq, struct bnxt_qplib_qp qp,
2027	struct bnxt_qplib_cqe *pcqe, int budget)	2118	struct bnxt_qplib_cqe *pcqe, int budget)
2028	{	2119	{
2029	u32 sw_prod, sw_cons;
2030	struct bnxt_qplib_cqe *cqe;	2120	struct bnxt_qplib_cqe *cqe;
		2121	u32 start, last;
2031	int rc = 0;	2122	int rc = 0;
2032		2123
2033	/* Now complete all outstanding SQEs with FLUSHED_ERR */	2124	/* Now complete all outstanding SQEs with FLUSHED_ERR */
2034	sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);	2125	start = sq->swq_start;
2035	cqe = *pcqe;	2126	cqe = *pcqe;
2036	while (*budget) {	2127	while (*budget) {
2037	sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);	2128	last = sq->swq_last;
2038	if (sw_cons == sw_prod) {	2129	if (start == last)
2039	break;	2130	break;
2040	}
2041	/* Skip the FENCE WQE completions */	2131	/* Skip the FENCE WQE completions */
2042	if (sq->swq[sw_cons].wr_id == BNXT_QPLIB_FENCE_WRID) {	2132	if (sq->swq[last].wr_id == BNXT_QPLIB_FENCE_WRID) {
2043	bnxt_qplib_cancel_phantom_processing(qp);	2133	bnxt_qplib_cancel_phantom_processing(qp);
2044	goto skip_compl;	2134	goto skip_compl;
2045	}	2135	}
@@ -2047,16 +2137,17 @@ static int __flush_sq(struct bnxt_qplib_q sq, struct bnxt_qplib_qp qp, · 1 unmodified line
2047	cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR;	2137	cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR;
2048	cqe->opcode = CQ_BASE_CQE_TYPE_REQ;	2138	cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2049	cqe->qp_handle = (u64)(unsigned long)qp;	2139	cqe->qp_handle = (u64)(unsigned long)qp;
2050	cqe->wr_id = sq->swq[sw_cons].wr_id;	2140	cqe->wr_id = sq->swq[last].wr_id;
2051	cqe->src_qp = qp->id;	2141	cqe->src_qp = qp->id;
2052	cqe->type = sq->swq[sw_cons].type;	2142	cqe->type = sq->swq[last].type;
2053	cqe++;	2143	cqe++;
2054	(*budget)--;	2144	(*budget)--;
2055	skip_compl:	2145	skip_compl:
2056	sq->hwq.cons++;	2146	bnxt_qplib_hwq_incr_cons(&sq->hwq, sq->swq[last].slots);
		2147	sq->swq_last = sq->swq[last].next_idx;
2057	}	2148	}
2058	*pcqe = cqe;	2149	*pcqe = cqe;
2059	if (!(*budget) && HWQ_CMP(sq->hwq.cons, &sq->hwq) != sw_prod)	2150	if (!(*budget) && sq->swq_last != start)
2060	/* Out of budget */	2151	/* Out of budget */
2061	rc = -EAGAIN;	2152	rc = -EAGAIN;
2062		2153
@@ -2067,9 +2158,9 @@ static int __flush_rq(struct bnxt_qplib_q rq, struct bnxt_qplib_qp qp, · 4 unmodified lines
2067	struct bnxt_qplib_cqe *pcqe, int budget)	2158	struct bnxt_qplib_cqe *pcqe, int budget)
2068	{	2159	{
2069	struct bnxt_qplib_cqe *cqe;	2160	struct bnxt_qplib_cqe *cqe;
2070	u32 sw_prod, sw_cons;	2161	u32 start, last;
2071	int rc = 0;
2072	int opcode = 0;	2162	int opcode = 0;
		2163	int rc = 0;
2073		2164
2074	switch (qp->type) {	2165	switch (qp->type) {
2075	case CMDQ_CREATE_QP1_TYPE_GSI:	2166	case CMDQ_CREATE_QP1_TYPE_GSI:
@@ -2085,24 +2176,25 @@ static int __flush_rq(struct bnxt_qplib_q rq, struct bnxt_qplib_qp qp, · 9 unmodified lines
2085	}	2176	}
2086		2177
2087	/* Flush the rest of the RQ */	2178	/* Flush the rest of the RQ */
2088	sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);	2179	start = rq->swq_start;
2089	cqe = *pcqe;	2180	cqe = *pcqe;
2090	while (*budget) {	2181	while (*budget) {
2091	sw_cons = HWQ_CMP(rq->hwq.cons, &rq->hwq);	2182	last = rq->swq_last;
2092	if (sw_cons == sw_prod)	2183	if (last == start)
2093	break;	2184	break;
2094	memset(cqe, 0, sizeof(*cqe));	2185	memset(cqe, 0, sizeof(*cqe));
2095	cqe->status =	2186	cqe->status =
2096	CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR;	2187	CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR;
2097	cqe->opcode = opcode;	2188	cqe->opcode = opcode;
2098	cqe->qp_handle = (unsigned long)qp;	2189	cqe->qp_handle = (unsigned long)qp;
2099	cqe->wr_id = rq->swq[sw_cons].wr_id;	2190	cqe->wr_id = rq->swq[last].wr_id;
2100	cqe++;	2191	cqe++;
2101	(*budget)--;	2192	(*budget)--;
2102	rq->hwq.cons++;	2193	bnxt_qplib_hwq_incr_cons(&rq->hwq, rq->swq[last].slots);
		2194	rq->swq_last = rq->swq[last].next_idx;
2103	}	2195	}
2104	*pcqe = cqe;	2196	*pcqe = cqe;
2105	if (!*budget && HWQ_CMP(rq->hwq.cons, &rq->hwq) != sw_prod)	2197	if (!*budget && rq->swq_last != start)
2106	/* Out of budget */	2198	/* Out of budget */
2107	rc = -EAGAIN;	2199	rc = -EAGAIN;
2108		2200
@@ -2125,7 +2217,7 @@ void bnxt_qplib_mark_qp_error(void *qp_handle) · 16 unmodified lines
2125	* CQE is track from sw_cq_cons to max_element but valid only if VALID=1	2217	* CQE is track from sw_cq_cons to max_element but valid only if VALID=1
2126	*/	2218	*/
2127	static int do_wa9060(struct bnxt_qplib_qp qp, struct bnxt_qplib_cq cq,	2219	static int do_wa9060(struct bnxt_qplib_qp qp, struct bnxt_qplib_cq cq,
2128	u32 cq_cons, u32 sw_sq_cons, u32 cqe_sq_cons)	2220	u32 cq_cons, u32 swq_last, u32 cqe_sq_cons)
2129	{	2221	{
2130	u32 peek_sw_cq_cons, peek_raw_cq_cons, peek_sq_cons_idx;	2222	u32 peek_sw_cq_cons, peek_raw_cq_cons, peek_sq_cons_idx;
2131	struct bnxt_qplib_q *sq = &qp->sq;	2223	struct bnxt_qplib_q *sq = &qp->sq;
@@ -2138,7 +2230,7 @@ static int do_wa9060(struct bnxt_qplib_qp qp, struct bnxt_qplib_cq cq, · 6 unmodified lines
2138		2230
2139	/* Normal mode */	2231	/* Normal mode */
2140	/* Check for the psn_search marking before completing */	2232	/* Check for the psn_search marking before completing */
2141	swq = &sq->swq[sw_sq_cons];	2233	swq = &sq->swq[swq_last];
2142	if (swq->psn_search &&	2234	if (swq->psn_search &&
2143	le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) {	2235	le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) {
2144	/* Unmark */	2236	/* Unmark */
@@ -2147,7 +2239,7 @@ static int do_wa9060(struct bnxt_qplib_qp qp, struct bnxt_qplib_cq cq, · 2 unmodified lines
2147	& ~0x80000000);	2239	& ~0x80000000);
2148	dev_dbg(&cq->hwq.pdev->dev,	2240	dev_dbg(&cq->hwq.pdev->dev,
2149	"FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n",	2241	"FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n",
2150	cq_cons, qp->id, sw_sq_cons, cqe_sq_cons);	2242	cq_cons, qp->id, swq_last, cqe_sq_cons);
2151	sq->condition = true;	2243	sq->condition = true;
2152	sq->send_phantom = true;	2244	sq->send_phantom = true;
2153		2245
@@ -2184,9 +2276,10 @@ static int do_wa9060(struct bnxt_qplib_qp qp, struct bnxt_qplib_cq cq, · 30 unmodified lines
2184	le64_to_cpu	2276	le64_to_cpu
2185	(peek_req_hwcqe->qp_handle));	2277	(peek_req_hwcqe->qp_handle));
2186	peek_sq = &peek_qp->sq;	2278	peek_sq = &peek_qp->sq;
2187	peek_sq_cons_idx = HWQ_CMP(le16_to_cpu(	2279	peek_sq_cons_idx =
2188	peek_req_hwcqe->sq_cons_idx) - 1	2280	((le16_to_cpu(
2189	, &sq->hwq);	2281	peek_req_hwcqe->sq_cons_idx)
		2282	- 1) % sq->max_wqe);
2190	/* If the hwcqe's sq's wr_id matches */	2283	/* If the hwcqe's sq's wr_id matches */
2191	if (peek_sq == sq &&	2284	if (peek_sq == sq &&
2192	sq->swq[peek_sq_cons_idx].wr_id ==	2285	sq->swq[peek_sq_cons_idx].wr_id ==
@@ -2214,7 +2307,7 @@ static int do_wa9060(struct bnxt_qplib_qp qp, struct bnxt_qplib_cq cq, · 21 unmodified lines
2214	}	2307	}
2215	dev_err(&cq->hwq.pdev->dev,	2308	dev_err(&cq->hwq.pdev->dev,
2216	"Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x\n",	2309	"Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x\n",
2217	cq_cons, qp->id, sw_sq_cons, cqe_sq_cons);	2310	cq_cons, qp->id, swq_last, cqe_sq_cons);
2218	rc = -EINVAL;	2311	rc = -EINVAL;
2219	}	2312	}
2220	out:	2313	out:
@@ -2226,11 +2319,11 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, · 5 unmodified lines
2226	struct bnxt_qplib_cqe *pcqe, int budget,	2319	struct bnxt_qplib_cqe *pcqe, int budget,
2227	u32 cq_cons, struct bnxt_qplib_qp **lib_qp)	2320	u32 cq_cons, struct bnxt_qplib_qp **lib_qp)
2228	{	2321	{
2229	u32 sw_sq_cons, cqe_sq_cons;
2230	struct bnxt_qplib_swq *swq;	2322	struct bnxt_qplib_swq *swq;
2231	struct bnxt_qplib_cqe *cqe;	2323	struct bnxt_qplib_cqe *cqe;
2232	struct bnxt_qplib_qp *qp;	2324	struct bnxt_qplib_qp *qp;
2233	struct bnxt_qplib_q *sq;	2325	struct bnxt_qplib_q *sq;
		2326	u32 cqe_sq_cons;
2234	int rc = 0;	2327	int rc = 0;
2235		2328
2236	qp = (struct bnxt_qplib_qp *)((unsigned long)	2329	qp = (struct bnxt_qplib_qp *)((unsigned long)
@@ -2242,14 +2335,7 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, · 5 unmodified lines
2242	}	2335	}
2243	sq = &qp->sq;	2336	sq = &qp->sq;
2244		2337
2245	cqe_sq_cons = HWQ_CMP(le16_to_cpu(hwcqe->sq_cons_idx), &sq->hwq);	2338	cqe_sq_cons = le16_to_cpu(hwcqe->sq_cons_idx) % sq->max_wqe;
2246	if (cqe_sq_cons > sq->hwq.max_elements) {
2247	dev_err(&cq->hwq.pdev->dev,
2248	"FP: CQ Process req reported sq_cons_idx 0x%x which exceeded max 0x%x\n",
2249	cqe_sq_cons, sq->hwq.max_elements);
2250	return -EINVAL;
2251	}
2252
2253	if (qp->sq.flushed) {	2339	if (qp->sq.flushed) {
2254	dev_dbg(&cq->hwq.pdev->dev,	2340	dev_dbg(&cq->hwq.pdev->dev,
2255	"%s: QP in Flush QP = %p\n", __func__, qp);	2341	"%s: QP in Flush QP = %p\n", __func__, qp);
@@ -2261,12 +2347,11 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, · 5 unmodified lines
2261	*/	2347	*/
2262	cqe = *pcqe;	2348	cqe = *pcqe;
2263	while (*budget) {	2349	while (*budget) {
2264	sw_sq_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);	2350	if (sq->swq_last == cqe_sq_cons)
2265	if (sw_sq_cons == cqe_sq_cons)
2266	/* Done */	2351	/* Done */
2267	break;	2352	break;
2268		2353
2269	swq = &sq->swq[sw_sq_cons];	2354	swq = &sq->swq[sq->swq_last];
2270	memset(cqe, 0, sizeof(*cqe));	2355	memset(cqe, 0, sizeof(*cqe));
2271	cqe->opcode = CQ_BASE_CQE_TYPE_REQ;	2356	cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2272	cqe->qp_handle = (u64)(unsigned long)qp;	2357	cqe->qp_handle = (u64)(unsigned long)qp;
@@ -2280,12 +2365,12 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, · 7 unmodified lines
2280	* of the request being signaled or not, it must complete with	2365	* of the request being signaled or not, it must complete with
2281	* the hwcqe error status	2366	* the hwcqe error status
2282	*/	2367	*/
2283	if (HWQ_CMP((sw_sq_cons + 1), &sq->hwq) == cqe_sq_cons &&	2368	if (swq->next_idx == cqe_sq_cons &&
2284	hwcqe->status != CQ_REQ_STATUS_OK) {	2369	hwcqe->status != CQ_REQ_STATUS_OK) {
2285	cqe->status = hwcqe->status;	2370	cqe->status = hwcqe->status;
2286	dev_err(&cq->hwq.pdev->dev,	2371	dev_err(&cq->hwq.pdev->dev,
2287	"FP: CQ Processed Req wr_id[%d] = 0x%llx with status 0x%x\n",	2372	"FP: CQ Processed Req wr_id[%d] = 0x%llx with status 0x%x\n",
2288	sw_sq_cons, cqe->wr_id, cqe->status);	2373	sq->swq_last, cqe->wr_id, cqe->status);
2289	cqe++;	2374	cqe++;
2290	(*budget)--;	2375	(*budget)--;
2291	bnxt_qplib_mark_qp_error(qp);	2376	bnxt_qplib_mark_qp_error(qp);
@@ -2293,7 +2378,7 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, · 1 unmodified line
2293	bnxt_qplib_add_flush_qp(qp);	2378	bnxt_qplib_add_flush_qp(qp);
2294	} else {	2379	} else {
2295	/* Before we complete, do WA 9060 */	2380	/* Before we complete, do WA 9060 */
2296	if (do_wa9060(qp, cq, cq_cons, sw_sq_cons,	2381	if (do_wa9060(qp, cq, cq_cons, sq->swq_last,
2297	cqe_sq_cons)) {	2382	cqe_sq_cons)) {
2298	*lib_qp = qp;	2383	*lib_qp = qp;
2299	goto out;	2384	goto out;
@@ -2305,13 +2390,14 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, · 5 unmodified lines
2305	}	2390	}
2306	}	2391	}
2307	skip:	2392	skip:
2308	sq->hwq.cons++;	2393	bnxt_qplib_hwq_incr_cons(&sq->hwq, swq->slots);
		2394	sq->swq_last = swq->next_idx;
2309	if (sq->single)	2395	if (sq->single)
2310	break;	2396	break;
2311	}	2397	}
2312	out:	2398	out:
2313	*pcqe = cqe;	2399	*pcqe = cqe;
2314	if (HWQ_CMP(sq->hwq.cons, &sq->hwq) != cqe_sq_cons) {	2400	if (sq->swq_last != cqe_sq_cons) {
2315	/* Out of budget */	2401	/* Out of budget */
2316	rc = -EAGAIN;	2402	rc = -EAGAIN;
2317	goto done;	2403	goto done;
@@ -2386,17 +2472,23 @@ static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq, · 68 unmodified lines
2386	(*budget)--;	2472	(*budget)--;
2387	*pcqe = cqe;	2473	*pcqe = cqe;
2388	} else {	2474	} else {
		2475	struct bnxt_qplib_swq *swq;
		2476
2389	rq = &qp->rq;	2477	rq = &qp->rq;
2390	if (wr_id_idx >= rq->hwq.max_elements) {	2478	if (wr_id_idx > (rq->max_wqe - 1)) {
2391	dev_err(&cq->hwq.pdev->dev,	2479	dev_err(&cq->hwq.pdev->dev,
2392	"FP: CQ Process RC wr_id idx 0x%x exceeded RQ max 0x%x\n",	2480	"FP: CQ Process RC wr_id idx 0x%x exceeded RQ max 0x%x\n",
2393	wr_id_idx, rq->hwq.max_elements);	2481	wr_id_idx, rq->max_wqe);
2394	return -EINVAL;	2482	return -EINVAL;
2395	}	2483	}
2396	cqe->wr_id = rq->swq[wr_id_idx].wr_id;	2484	if (wr_id_idx != rq->swq_last)
		2485	return -EINVAL;
		2486	swq = &rq->swq[rq->swq_last];
		2487	cqe->wr_id = swq->wr_id;
2397	cqe++;	2488	cqe++;
2398	(*budget)--;	2489	(*budget)--;
2399	rq->hwq.cons++;	2490	bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots);
		2491	rq->swq_last = swq->next_idx;
2400	*pcqe = cqe;	2492	*pcqe = cqe;
2401		2493
2402	if (hwcqe->status != CQ_RES_RC_STATUS_OK) {	2494	if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
@@ -2467,18 +2559,24 @@ static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq, · 64 unmodified lines
2467	(*budget)--;	2559	(*budget)--;
2468	*pcqe = cqe;	2560	*pcqe = cqe;
2469	} else {	2561	} else {
		2562	struct bnxt_qplib_swq *swq;
		2563
2470	rq = &qp->rq;	2564	rq = &qp->rq;
2471	if (wr_id_idx >= rq->hwq.max_elements) {	2565	if (wr_id_idx > (rq->max_wqe - 1)) {
2472	dev_err(&cq->hwq.pdev->dev,	2566	dev_err(&cq->hwq.pdev->dev,
2473	"FP: CQ Process UD wr_id idx 0x%x exceeded RQ max 0x%x\n",	2567	"FP: CQ Process UD wr_id idx 0x%x exceeded RQ max 0x%x\n",
2474	wr_id_idx, rq->hwq.max_elements);	2568	wr_id_idx, rq->max_wqe);
2475	return -EINVAL;	2569	return -EINVAL;
2476	}	2570	}
2477		2571
2478	cqe->wr_id = rq->swq[wr_id_idx].wr_id;	2572	if (rq->swq_last != wr_id_idx)
		2573	return -EINVAL;
		2574	swq = &rq->swq[rq->swq_last];
		2575	cqe->wr_id = swq->wr_id;
2479	cqe++;	2576	cqe++;
2480	(*budget)--;	2577	(*budget)--;
2481	rq->hwq.cons++;	2578	bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots);
		2579	rq->swq_last = swq->next_idx;
2482	*pcqe = cqe;	2580	*pcqe = cqe;
2483		2581
2484	if (hwcqe->status != CQ_RES_RC_STATUS_OK) {	2582	if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
@@ -2569,17 +2667,23 @@ static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq, · 84 unmodified lines
2569	(*budget)--;	2667	(*budget)--;
2570	*pcqe = cqe;	2668	*pcqe = cqe;
2571	} else {	2669	} else {
		2670	struct bnxt_qplib_swq *swq;
		2671
2572	rq = &qp->rq;	2672	rq = &qp->rq;
2573	if (wr_id_idx >= rq->hwq.max_elements) {	2673	if (wr_id_idx > (rq->max_wqe - 1)) {
2574	dev_err(&cq->hwq.pdev->dev,	2674	dev_err(&cq->hwq.pdev->dev,
2575	"FP: CQ Process Raw/QP1 RQ wr_id idx 0x%x exceeded RQ max 0x%x\n",	2675	"FP: CQ Process Raw/QP1 RQ wr_id idx 0x%x exceeded RQ max 0x%x\n",
2576	wr_id_idx, rq->hwq.max_elements);	2676	wr_id_idx, rq->max_wqe);
2577	return -EINVAL;	2677	return -EINVAL;
2578	}	2678	}
2579	cqe->wr_id = rq->swq[wr_id_idx].wr_id;	2679	if (rq->swq_last != wr_id_idx)
		2680	return -EINVAL;
		2681	swq = &rq->swq[rq->swq_last];
		2682	cqe->wr_id = swq->wr_id;
2580	cqe++;	2683	cqe++;
2581	(*budget)--;	2684	(*budget)--;
2582	rq->hwq.cons++;	2685	bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots);
		2686	rq->swq_last = swq->next_idx;
2583	*pcqe = cqe;	2687	*pcqe = cqe;
2584		2688
2585	if (hwcqe->status != CQ_RES_RC_STATUS_OK) {	2689	if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
@@ -2601,7 +2705,7 @@ static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq, · 15 unmodified lines
2601	struct bnxt_qplib_qp *qp;	2705	struct bnxt_qplib_qp *qp;
2602	struct bnxt_qplib_q sq, rq;	2706	struct bnxt_qplib_q sq, rq;
2603	struct bnxt_qplib_cqe *cqe;	2707	struct bnxt_qplib_cqe *cqe;
2604	u32 sw_cons = 0, cqe_cons;	2708	u32 swq_last = 0, cqe_cons;
2605	int rc = 0;	2709	int rc = 0;
2606		2710
2607	/* Check the Status */	2711	/* Check the Status */
@@ -2627,13 +2731,7 @@ static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq, · 19 unmodified lines
2627	cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx);	2731	cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx);
2628	if (cqe_cons == 0xFFFF)	2732	if (cqe_cons == 0xFFFF)
2629	goto do_rq;	2733	goto do_rq;
2630		2734	cqe_cons %= sq->max_wqe;
2631	if (cqe_cons > sq->hwq.max_elements) {
2632	dev_err(&cq->hwq.pdev->dev,
2633	"FP: CQ Process terminal reported sq_cons_idx 0x%x which exceeded max 0x%x\n",
2634	cqe_cons, sq->hwq.max_elements);
2635	goto do_rq;
2636	}
2637		2735
2638	if (qp->sq.flushed) {	2736	if (qp->sq.flushed) {
2639	dev_dbg(&cq->hwq.pdev->dev,	2737	dev_dbg(&cq->hwq.pdev->dev,
@@ -2647,24 +2745,25 @@ static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq, · 7 unmodified lines
2647	*/	2745	*/
2648	cqe = *pcqe;	2746	cqe = *pcqe;
2649	while (*budget) {	2747	while (*budget) {
2650	sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);	2748	swq_last = sq->swq_last;
2651	if (sw_cons == cqe_cons)	2749	if (swq_last == cqe_cons)
2652	break;	2750	break;
2653	if (sq->swq[sw_cons].flags & SQ_SEND_FLAGS_SIGNAL_COMP) {	2751	if (sq->swq[swq_last].flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
2654	memset(cqe, 0, sizeof(*cqe));	2752	memset(cqe, 0, sizeof(*cqe));
2655	cqe->status = CQ_REQ_STATUS_OK;	2753	cqe->status = CQ_REQ_STATUS_OK;
2656	cqe->opcode = CQ_BASE_CQE_TYPE_REQ;	2754	cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2657	cqe->qp_handle = (u64)(unsigned long)qp;	2755	cqe->qp_handle = (u64)(unsigned long)qp;
2658	cqe->src_qp = qp->id;	2756	cqe->src_qp = qp->id;
2659	cqe->wr_id = sq->swq[sw_cons].wr_id;	2757	cqe->wr_id = sq->swq[swq_last].wr_id;
2660	cqe->type = sq->swq[sw_cons].type;	2758	cqe->type = sq->swq[swq_last].type;
2661	cqe++;	2759	cqe++;
2662	(*budget)--;	2760	(*budget)--;
2663	}	2761	}
2664	sq->hwq.cons++;	2762	bnxt_qplib_hwq_incr_cons(&sq->hwq, sq->swq[swq_last].slots);
		2763	sq->swq_last = sq->swq[swq_last].next_idx;
2665	}	2764	}
2666	*pcqe = cqe;	2765	*pcqe = cqe;
2667	if (!(*budget) && sw_cons != cqe_cons) {	2766	if (!(*budget) && swq_last != cqe_cons) {
2668	/* Out of budget */	2767	/* Out of budget */
2669	rc = -EAGAIN;	2768	rc = -EAGAIN;
2670	goto sq_done;	2769	goto sq_done;
@@ -2676,10 +2775,10 @@ do_rq: · 5 unmodified lines
2676	cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx);	2775	cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx);
2677	if (cqe_cons == 0xFFFF) {	2776	if (cqe_cons == 0xFFFF) {
2678	goto done;	2777	goto done;
2679	} else if (cqe_cons > rq->hwq.max_elements) {	2778	} else if (cqe_cons > rq->max_wqe - 1) {
2680	dev_err(&cq->hwq.pdev->dev,	2779	dev_err(&cq->hwq.pdev->dev,
2681	"FP: CQ Processed terminal reported rq_cons_idx 0x%x exceeds max 0x%x\n",	2780	"FP: CQ Processed terminal reported rq_cons_idx 0x%x exceeds max 0x%x\n",
2682	cqe_cons, rq->hwq.max_elements);	2781	cqe_cons, rq->max_wqe);
2683	goto done;	2782	goto done;
2684	}	2783	}
2685		2784

drivers/infiniband/hw/bnxt_re/qplib_fp.h+122−5

@@ -39,6 +39,51 @@ · 38 unmodified lines
39	#ifndef __BNXT_QPLIB_FP_H__	39	#ifndef __BNXT_QPLIB_FP_H__
40	#define __BNXT_QPLIB_FP_H__	40	#define __BNXT_QPLIB_FP_H__
41		41
		42	/* Few helper structures temporarily defined here
		43	* should get rid of these when roce_hsi.h is updated
		44	* in original code base
		45	*/
		46	struct sq_ud_ext_hdr {
		47	__le32 dst_qp;
		48	__le32 avid;
		49	__le64 rsvd;
		50	};
		51
		52	struct sq_raw_ext_hdr {
		53	__le32 cfa_meta;
		54	__le32 rsvd0;
		55	__le64 rsvd1;
		56	};
		57
		58	struct sq_rdma_ext_hdr {
		59	__le64 remote_va;
		60	__le32 remote_key;
		61	__le32 rsvd;
		62	};
		63
		64	struct sq_atomic_ext_hdr {
		65	__le64 swap_data;
		66	__le64 cmp_data;
		67	};
		68
		69	struct sq_fr_pmr_ext_hdr {
		70	__le64 pblptr;
		71	__le64 va;
		72	};
		73
		74	struct sq_bind_ext_hdr {
		75	__le64 va;
		76	__le32 length_lo;
		77	__le32 length_hi;
		78	};
		79
		80	struct rq_ext_hdr {
		81	__le64 rsvd1;
		82	__le64 rsvd2;
		83	};
		84
		85	/* Helper structures end */
		86
42	struct bnxt_qplib_srq {	87	struct bnxt_qplib_srq {
43	struct bnxt_qplib_pd *pd;	88	struct bnxt_qplib_pd *pd;
44	struct bnxt_qplib_dpi *dpi;	89	struct bnxt_qplib_dpi *dpi;
@@ -74,6 +119,8 @@ struct bnxt_qplib_swq { · 29 unmodified lines
74	u8 flags;	119	u8 flags;
75	u32 start_psn;	120	u32 start_psn;
76	u32 next_psn;	121	u32 next_psn;
		122	u32 slot_idx;
		123	u8 slots;
77	struct sq_psn_search *psn_search;	124	struct sq_psn_search *psn_search;
78	struct sq_psn_search_ext *psn_ext;	125	struct sq_psn_search_ext *psn_ext;
79	};	126	};
@@ -213,6 +260,8 @@ struct bnxt_qplib_q { · 133 unmodified lines
213	u32 phantom_cqe_cnt;	260	u32 phantom_cqe_cnt;
214	u32 next_cq_cons;	261	u32 next_cq_cons;
215	bool flushed;	262	bool flushed;
		263	u32 swq_start;
		264	u32 swq_last;
216	};	265	};
217		266
218	struct bnxt_qplib_qp {	267	struct bnxt_qplib_qp {
@@ -224,9 +273,10 @@ struct bnxt_qplib_qp { · 5 unmodified lines
224	u32 id;	273	u32 id;
225	u8 type;	274	u8 type;
226	u8 sig_type;	275	u8 sig_type;
227	u32 modify_flags;	276	u8 wqe_mode;
228	u8 state;	277	u8 state;
229	u8 cur_qp_state;	278	u8 cur_qp_state;
		279	u64 modify_flags;
230	u32 max_inline_data;	280	u32 max_inline_data;
231	u32 mtu;	281	u32 mtu;
232	u8 path_mtu;	282	u8 path_mtu;
@@ -300,11 +350,18 @@ struct bnxt_qplib_qp { · 67 unmodified lines
300	(!!((hdr)->cqe_type_toggle & CQ_BASE_TOGGLE) == \	350	(!!((hdr)->cqe_type_toggle & CQ_BASE_TOGGLE) == \
301	!((raw_cons) & (cp_bit)))	351	!((raw_cons) & (cp_bit)))
302		352
303	static inline bool bnxt_qplib_queue_full(struct bnxt_qplib_q *qplib_q)	353	static inline bool bnxt_qplib_queue_full(struct bnxt_qplib_q *que,
		354	u8 slots)
304	{	355	{
305	return HWQ_CMP((qplib_q->hwq.prod + qplib_q->q_full_delta),	356	struct bnxt_qplib_hwq *hwq;
306	&qplib_q->hwq) == HWQ_CMP(qplib_q->hwq.cons,	357	int avail;
307	&qplib_q->hwq);	358
		359	hwq = &que->hwq;
		360	/* False full is possible, retrying post-send makes sense */
		361	avail = hwq->cons - hwq->prod;
		362	if (hwq->cons <= hwq->prod)
		363	avail += hwq->depth;
		364	return avail <= slots;
308	}	365	}
309		366
310	struct bnxt_qplib_cqe {	367	struct bnxt_qplib_cqe {
@@ -489,4 +546,64 @@ int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq, · 178 unmodified lines
489	struct bnxt_qplib_cqe *cqe,	546	struct bnxt_qplib_cqe *cqe,
490	int num_cqes);	547	int num_cqes);
491	void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp);	548	void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp);
		549
		550	static inline void bnxt_qplib_get_swqe(struct bnxt_qplib_q que, u32 *swq_idx)
		551	{
		552	u32 idx;
		553
		554	idx = que->swq_start;
		555	if (swq_idx)
		556	*swq_idx = idx;
		557	return &que->swq[idx];
		558	}
		559
		560	static inline void bnxt_qplib_swq_mod_start(struct bnxt_qplib_q *que, u32 idx)
		561	{
		562	que->swq_start = que->swq[idx].next_idx;
		563	}
		564
		565	static inline u32 bnxt_qplib_get_depth(struct bnxt_qplib_q *que)
		566	{
		567	return (que->wqe_size * que->max_wqe) / sizeof(struct sq_sge);
		568	}
		569
		570	static inline u32 bnxt_qplib_set_sq_size(struct bnxt_qplib_q *que, u8 wqe_mode)
		571	{
		572	return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
		573	que->max_wqe : bnxt_qplib_get_depth(que);
		574	}
		575
		576	static inline u32 bnxt_qplib_set_sq_max_slot(u8 wqe_mode)
		577	{
		578	return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
		579	sizeof(struct sq_send) / sizeof(struct sq_sge) : 1;
		580	}
		581
		582	static inline u32 bnxt_qplib_set_rq_max_slot(u32 wqe_size)
		583	{
		584	return (wqe_size / sizeof(struct sq_sge));
		585	}
		586
		587	static inline u16 __xlate_qfd(u16 delta, u16 wqe_bytes)
		588	{
		589	/* For Cu/Wh delta = 128, stride = 16, wqe_bytes = 128
		590	* For Gen-p5 B/C mode delta = 0, stride = 16, wqe_bytes = 128.
		591	* For Gen-p5 delta = 0, stride = 16, 32 <= wqe_bytes <= 512.
		592	* when 8916 is disabled.
		593	*/
		594	return (delta * wqe_bytes) / sizeof(struct sq_sge);
		595	}
		596
		597	static inline u16 bnxt_qplib_calc_ilsize(struct bnxt_qplib_swqe *wqe, u16 max)
		598	{
		599	u16 size = 0;
		600	int indx;
		601
		602	for (indx = 0; indx < wqe->num_sge; indx++)
		603	size += wqe->sg_list[indx].size;
		604	if (size > max)
		605	size = max;
		606
		607	return size;
		608	}
492	#endif /* __BNXT_QPLIB_FP_H__ */	609	#endif /* __BNXT_QPLIB_FP_H__ */

drivers/infiniband/hw/bnxt_re/qplib_res.h+46−12

 extern const struct bnxt_qplib_gid bnxt_qplib_gid_zero;
+#define CHIP_NUM_57508		0x1750
+#define CHIP_NUM_57504		0x1751
+#define CHIP_NUM_57502		0x1752
+enum bnxt_qplib_wqe_mode {
+	BNXT_QPLIB_WQE_MODE_STATIC	= 0x00,
+	BNXT_QPLIB_WQE_MODE_VARIABLE	= 0x01,
+	BNXT_QPLIB_WQE_MODE_INVALID	= 0x02
+};
+struct bnxt_qplib_drv_modes {
+	u8	wqe_mode;
+	/* Other modes to follow here */
+};
+struct bnxt_qplib_chip_ctx {
+	u16	chip_num;
+	u8	chip_rev;
+	u8	chip_metal;
+	struct bnxt_qplib_drv_modes modes;
+};
 #define PTR_CNT_PER_PG		(PAGE_SIZE / sizeof(void *))
 #define PTR_MAX_IDX_PER_PG	(PTR_CNT_PER_PG - 1)
 #define PTR_PG(x)		(((x) & ~PTR_MAX_IDX_PER_PG) / PTR_CNT_PER_PG)
 	u32				cons;		/* raw */
 	u8				cp_bit;
 	u8				is_user;
+	u64				*pad_pg;
+	u32				pad_stride;
+	u32				pad_pgofft;
 };
 struct bnxt_qplib_db_info {
 	void __iomem		*priv_db;
 	struct bnxt_qplib_hwq	*hwq;
 	u32			xid;
+	u32			max_slot;
 };
 /* Tables */
 	u64				hwrm_intf_ver;
 };
-struct bnxt_qplib_chip_ctx {
-	u16	chip_num;
-	u8	chip_rev;
-	u8	chip_metal;
-};
-#define CHIP_NUM_57508		0x1750
-#define CHIP_NUM_57504		0x1751
-#define CHIP_NUM_57502		0x1752
 struct bnxt_qplib_res {
 	struct pci_dev			*pdev;
 	struct bnxt_qplib_chip_ctx	*cctx;
 	return (void *)(hwq->pbl_ptr[pg_num] + hwq->element_size * pg_idx);
 }
+static inline void *bnxt_qplib_get_prod_qe(struct bnxt_qplib_hwq *hwq, u32 idx)
+{
+	idx += hwq->prod;
+	if (idx >= hwq->depth)
+		idx -= hwq->depth;
+	return bnxt_qplib_get_qe(hwq, idx, NULL);
+}
 #define to_bnxt_qplib(ptr, type, member)	\
 	container_of(ptr, type, member)
 			 struct bnxt_qplib_ctx *ctx,
 			 bool virt_fn, bool is_p5);
+static inline void bnxt_qplib_hwq_incr_prod(struct bnxt_qplib_hwq *hwq, u32 cnt)
+{
+	hwq->prod = (hwq->prod + cnt) % hwq->depth;
+}
+static inline void bnxt_qplib_hwq_incr_cons(struct bnxt_qplib_hwq *hwq,
+					    u32 cnt)
+{
+	hwq->cons = (hwq->cons + cnt) % hwq->depth;
+}
 static inline void bnxt_qplib_ring_db32(struct bnxt_qplib_db_info *info,
 					bool arm)
 {
 	key = (info->xid & DBC_DBC_XID_MASK) | DBC_DBC_PATH_ROCE | type;
 	key <<= 32;
-	key |= (info->hwq->prod & (info->hwq->max_elements - 1)) &
+	key |= ((info->hwq->prod / info->max_slot)) & DBC_DBC_INDEX_MASK;
-		DBC_DBC_INDEX_MASK;
 	writeq(key, info->db);
 }

drivers/infiniband/hw/bnxt_re/roce_hsi.h+1−0

@@ -1126,6 +1126,7 @@ struct cmdq_create_qp { · 1125 unmodified lines
1126	#define CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION 0x2UL	1126	#define CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION 0x2UL
1127	#define CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE 0x4UL	1127	#define CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE 0x4UL
1128	#define CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED 0x8UL	1128	#define CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED 0x8UL
		1129	#define CMDQ_CREATE_QP_QP_FLAGS_VARIABLE_SIZED_WQE_ENABLED 0x10UL
1129	u8 type;	1130	u8 type;
1130	#define CMDQ_CREATE_QP_TYPE_RC 0x2UL	1131	#define CMDQ_CREATE_QP_TYPE_RC 0x2UL
1131	#define CMDQ_CREATE_QP_TYPE_UD 0x4UL	1132	#define CMDQ_CREATE_QP_TYPE_UD 0x4UL

drivers/infiniband/hw/cxgb4/iw_cxgb4.h+5−4

 void c4iw_qp_add_ref(struct ib_qp *qp);
 void c4iw_qp_rem_ref(struct ib_qp *qp);
 struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
-			    u32 max_num_sg, struct ib_udata *udata);
+			    u32 max_num_sg);
 int c4iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
 		   unsigned int *sg_offset);
 int c4iw_dealloc_mw(struct ib_mw *mw);
 		       const struct ib_recv_wr **bad_wr);
 struct c4iw_wr_wait *c4iw_alloc_wr_wait(gfp_t gfp);
-typedef int c4iw_restrack_func(struct sk_buff *msg,
+int c4iw_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ibmr);
-			       struct rdma_restrack_entry *res);
+int c4iw_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ibcq);
-extern c4iw_restrack_func *c4iw_restrack_funcs[RDMA_RESTRACK_MAX];
+int c4iw_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ibqp);
+int c4iw_fill_res_cm_id_entry(struct sk_buff *msg, struct rdma_cm_id *cm_id);
 #endif

drivers/infiniband/hw/cxgb4/mem.c+1−2

 	mmid = stag >> 8;
 	mhp->ibmr.rkey = mhp->ibmr.lkey = stag;
 	mhp->ibmr.length = mhp->attr.len;
-	mhp->ibmr.iova = mhp->attr.va_fbo;
 	mhp->ibmr.page_size = 1U << (mhp->attr.page_size + 12);
 	pr_debug("mmid 0x%x mhp %p\n", mmid, mhp);
 	return xa_insert_irq(&mhp->rhp->mrs, mmid, mhp, GFP_KERNEL);
 }
 struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
-			    u32 max_num_sg, struct ib_udata *udata)
+			    u32 max_num_sg)
 {
 	struct c4iw_dev *rhp;
 	struct c4iw_pd *php;

drivers/infiniband/hw/cxgb4/provider.c+3−19

 	return 0;
 }
-static int c4iw_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
-			   u16 *pkey)
-{
-	pr_debug("ibdev %p\n", ibdev);
-	*pkey = 0;
-	return 0;
-}
 static int c4iw_query_gid(struct ib_device *ibdev, u8 port, int index,
 			  union ib_gid *gid)
 {
 	    IB_PORT_DEVICE_MGMT_SUP |
 	    IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;
 	props->gid_tbl_len = 1;
-	props->pkey_tbl_len = 1;
 	props->max_msg_sz = -1;
 	return ret;
 	if (err)
 		return err;
-	immutable->pkey_tbl_len = attr.pkey_tbl_len;
 	immutable->gid_tbl_len = attr.gid_tbl_len;
 	return 0;
 		 FW_HDR_FW_VER_BUILD_G(c4iw_dev->rdev.lldi.fw_vers));
 }
-static int fill_res_entry(struct sk_buff *msg, struct rdma_restrack_entry *res)
-{
-	return (res->type < ARRAY_SIZE(c4iw_restrack_funcs) &&
-		c4iw_restrack_funcs[res->type]) ?
-		c4iw_restrack_funcs[res->type](msg, res) : 0;
-}
 static const struct ib_device_ops c4iw_dev_ops = {
 	.owner = THIS_MODULE,
 	.driver_id = RDMA_DRIVER_CXGB4,
 	.destroy_cq = c4iw_destroy_cq,
 	.destroy_qp = c4iw_destroy_qp,
 	.destroy_srq = c4iw_destroy_srq,
-	.fill_res_entry = fill_res_entry,
+	.fill_res_cq_entry = c4iw_fill_res_cq_entry,
+	.fill_res_cm_id_entry = c4iw_fill_res_cm_id_entry,
+	.fill_res_mr_entry = c4iw_fill_res_mr_entry,
 	.get_dev_fw_str = get_dev_fw_str,
 	.get_dma_mr = c4iw_get_dma_mr,
 	.get_hw_stats = c4iw_get_mib,
 	.post_srq_recv = c4iw_post_srq_recv,
 	.query_device = c4iw_query_device,
 	.query_gid = c4iw_query_gid,
-	.query_pkey = c4iw_query_pkey,
 	.query_port = c4iw_query_port,
 	.query_qp = c4iw_ib_query_qp,
 	.reg_user_mr = c4iw_reg_user_mr,

drivers/infiniband/hw/cxgb4/restrack.c+5−19

 	return -EMSGSIZE;
 }
-static int fill_res_qp_entry(struct sk_buff *msg,
+int c4iw_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ibqp)
-			     struct rdma_restrack_entry *res)
 {
-	struct ib_qp *ibqp = container_of(res, struct ib_qp, res);
 	struct t4_swsqe *fsp = NULL, *lsp = NULL;
 	struct c4iw_qp *qhp = to_c4iw_qp(ibqp);
 	u16 first_sq_idx = 0, last_sq_idx = 0;
 	struct c4iw_ep ep;
 };
-static int fill_res_ep_entry(struct sk_buff *msg,
+int c4iw_fill_res_cm_id_entry(struct sk_buff *msg,
-			     struct rdma_restrack_entry *res)
+			      struct rdma_cm_id *cm_id)
 {
-	struct rdma_cm_id *cm_id = rdma_res_to_id(res);
 	struct nlattr *table_attr;
 	struct c4iw_ep_common *epcp;
 	struct c4iw_listen_ep *listen_ep = NULL;
 	return -EMSGSIZE;
 }
-static int fill_res_cq_entry(struct sk_buff *msg,
+int c4iw_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ibcq)
-			     struct rdma_restrack_entry *res)
 {
-	struct ib_cq *ibcq = container_of(res, struct ib_cq, res);
 	struct c4iw_cq *chp = to_c4iw_cq(ibcq);
 	struct nlattr *table_attr;
 	struct t4_cqe hwcqes[2];
 	return -EMSGSIZE;
 }
-static int fill_res_mr_entry(struct sk_buff *msg,
+int c4iw_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ibmr)
-			     struct rdma_restrack_entry *res)
 {
-	struct ib_mr *ibmr = container_of(res, struct ib_mr, res);
 	struct c4iw_mr *mhp = to_c4iw_mr(ibmr);
 	struct c4iw_dev *dev = mhp->rhp;
 	u32 stag = mhp->attr.stag;
 err:
 	return -EMSGSIZE;
 }
-c4iw_restrack_func *c4iw_restrack_funcs[RDMA_RESTRACK_MAX] = {
-	[RDMA_RESTRACK_QP]	= fill_res_qp_entry,
-	[RDMA_RESTRACK_CM_ID]	= fill_res_ep_entry,
-	[RDMA_RESTRACK_CQ]	= fill_res_cq_entry,
-	[RDMA_RESTRACK_MR]	= fill_res_mr_entry,
-};

drivers/infiniband/hw/efa/efa_admin_cmds_defs.h+13−2

 	/* Number of sub-CQs to be created for each CQ */
 	u16 sub_cqs_per_cq;
-	/* MBZ */
+	/* Minimum number of WQEs per SQ */
-	u16 reserved;
+	u16 min_sq_depth;
 	/* Maximum number of SGEs (buffers) allowed for a single send WQE */
 	u16 max_wr_send_sges;
 	/* Maximum number of SGEs for a single RDMA read WQE */
 	u16 max_wr_rdma_sges;
+	/*
+	 * Maximum number of bytes that can be written to SQ between two
+	 * consecutive doorbells (in units of 64B). Driver must ensure that only
+	 * complete WQEs are written to queue before issuing a doorbell.
+	 * Examples: max_tx_batch=16 and WQE size = 64B, means up to 16 WQEs can
+	 * be written to SQ between two consecutive doorbells. max_tx_batch=11
+	 * and WQE size = 128B, means up to 5 WQEs can be written to SQ between
+	 * two consecutive doorbells. Zero means unlimited.
+	 */
+	u16 max_tx_batch;
 };
 struct efa_admin_feature_aenq_desc {

drivers/infiniband/hw/efa/efa_com_cmd.c+2−0

@@ -480,6 +480,8 @@ int efa_com_get_device_attr(struct efa_com_dev *edev, · 479 unmodified lines
480	result->max_llq_size = resp.u.queue_attr.max_llq_size;	480	result->max_llq_size = resp.u.queue_attr.max_llq_size;
481	result->sub_cqs_per_cq = resp.u.queue_attr.sub_cqs_per_cq;	481	result->sub_cqs_per_cq = resp.u.queue_attr.sub_cqs_per_cq;
482	result->max_wr_rdma_sge = resp.u.queue_attr.max_wr_rdma_sges;	482	result->max_wr_rdma_sge = resp.u.queue_attr.max_wr_rdma_sges;
		483	result->max_tx_batch = resp.u.queue_attr.max_tx_batch;
		484	result->min_sq_depth = resp.u.queue_attr.min_sq_depth;
483		485
484	err = efa_com_get_feature(edev, &resp, EFA_ADMIN_NETWORK_ATTR);	486	err = efa_com_get_feature(edev, &resp, EFA_ADMIN_NETWORK_ATTR);
485	if (err) {	487	if (err) {

drivers/infiniband/hw/efa/efa_com_cmd.h+2−0

@@ -127,6 +127,8 @@ struct efa_com_get_device_attr_result { · 126 unmodified lines
127	u16 max_sq_sge;	127	u16 max_sq_sge;
128	u16 max_rq_sge;	128	u16 max_rq_sge;
129	u16 max_wr_rdma_sge;	129	u16 max_wr_rdma_sge;
		130	u16 max_tx_batch;
		131	u16 min_sq_depth;
130	u8 db_bar;	132	u8 db_bar;
131	};	133	};
132		134

drivers/infiniband/hw/efa/efa_main.c+4−2

 #include "efa.h"
-#define PCI_DEV_ID_EFA_VF 0xefa0
+#define PCI_DEV_ID_EFA0_VF 0xefa0
+#define PCI_DEV_ID_EFA1_VF 0xefa1
 static const struct pci_device_id efa_pci_tbl[] = {
-	{ PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA_VF) },
+	{ PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA0_VF) },
+	{ PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA1_VF) },
 	{ }
 };

drivers/infiniband/hw/efa/efa_verbs.c+42−0

@@ -1502,11 +1502,39 @@ static int efa_dealloc_uar(struct efa_dev *dev, u16 uarn) · 1501 unmodified lines
1502	return efa_com_dealloc_uar(&dev->edev, &params);	1502	return efa_com_dealloc_uar(&dev->edev, &params);
1503	}	1503	}
1504		1504
		1505	#define EFA_CHECK_USER_COMP(_dev, _comp_mask, _attr, _mask, _attr_str) \
		1506	(_attr_str = (!(_dev)->dev_attr._attr \|\| ((_comp_mask) & (_mask))) ? \
		1507	NULL : #_attr)
		1508
		1509	static int efa_user_comp_handshake(const struct ib_ucontext *ibucontext,
		1510	const struct efa_ibv_alloc_ucontext_cmd *cmd)
		1511	{
		1512	struct efa_dev *dev = to_edev(ibucontext->device);
		1513	char *attr_str;
		1514
		1515	if (EFA_CHECK_USER_COMP(dev, cmd->comp_mask, max_tx_batch,
		1516	EFA_ALLOC_UCONTEXT_CMD_COMP_TX_BATCH, attr_str))
		1517	goto err;
		1518
		1519	if (EFA_CHECK_USER_COMP(dev, cmd->comp_mask, min_sq_depth,
		1520	EFA_ALLOC_UCONTEXT_CMD_COMP_MIN_SQ_WR,
		1521	attr_str))
		1522	goto err;
		1523
		1524	return 0;
		1525
		1526	err:
		1527	ibdev_dbg(&dev->ibdev, "Userspace handshake failed for %s attribute\n",
		1528	attr_str);
		1529	return -EOPNOTSUPP;
		1530	}
		1531
1505	int efa_alloc_ucontext(struct ib_ucontext ibucontext, struct ib_udata udata)	1532	int efa_alloc_ucontext(struct ib_ucontext ibucontext, struct ib_udata udata)
1506	{	1533	{
1507	struct efa_ucontext *ucontext = to_eucontext(ibucontext);	1534	struct efa_ucontext *ucontext = to_eucontext(ibucontext);
1508	struct efa_dev *dev = to_edev(ibucontext->device);	1535	struct efa_dev *dev = to_edev(ibucontext->device);
1509	struct efa_ibv_alloc_ucontext_resp resp = {};	1536	struct efa_ibv_alloc_ucontext_resp resp = {};
		1537	struct efa_ibv_alloc_ucontext_cmd cmd = {};
1510	struct efa_com_alloc_uar_result result;	1538	struct efa_com_alloc_uar_result result;
1511	int err;	1539	int err;
1512		1540
@@ -1515,6 +1543,18 @@ int efa_alloc_ucontext(struct ib_ucontext ibucontext, struct ib_udata udata) · 2 unmodified lines
1515	* we will ack input fields in our response.	1543	* we will ack input fields in our response.
1516	*/	1544	*/
1517		1545
		1546	err = ib_copy_from_udata(&cmd, udata,
		1547	min(sizeof(cmd), udata->inlen));
		1548	if (err) {
		1549	ibdev_dbg(&dev->ibdev,
		1550	"Cannot copy udata for alloc_ucontext\n");
		1551	goto err_out;
		1552	}
		1553
		1554	err = efa_user_comp_handshake(ibucontext, &cmd);
		1555	if (err)
		1556	goto err_out;
		1557
1518	err = efa_com_alloc_uar(&dev->edev, &result);	1558	err = efa_com_alloc_uar(&dev->edev, &result);
1519	if (err)	1559	if (err)
1520	goto err_out;	1560	goto err_out;
@@ -1526,6 +1566,8 @@ int efa_alloc_ucontext(struct ib_ucontext ibucontext, struct ib_udata udata) · 5 unmodified lines
1526	resp.sub_cqs_per_cq = dev->dev_attr.sub_cqs_per_cq;	1566	resp.sub_cqs_per_cq = dev->dev_attr.sub_cqs_per_cq;
1527	resp.inline_buf_size = dev->dev_attr.inline_buf_size;	1567	resp.inline_buf_size = dev->dev_attr.inline_buf_size;
1528	resp.max_llq_size = dev->dev_attr.max_llq_size;	1568	resp.max_llq_size = dev->dev_attr.max_llq_size;
		1569	resp.max_tx_batch = dev->dev_attr.max_tx_batch;
		1570	resp.min_sq_wr = dev->dev_attr.min_sq_depth;
1529		1571
1530	if (udata && udata->outlen) {	1572	if (udata && udata->outlen) {
1531	err = ib_copy_to_udata(udata, &resp,	1573	err = ib_copy_to_udata(udata, &resp,

drivers/infiniband/hw/hfi1/chip.c+14−13

 	case 1: return OPA_LINK_WIDTH_1X;
 	case 2: return OPA_LINK_WIDTH_2X;
 	case 3: return OPA_LINK_WIDTH_3X;
+	case 4: return OPA_LINK_WIDTH_4X;
 	default:
 		dd_dev_info(dd, "%s: invalid width %d, using 4\n",
 			    __func__, width);
-		/* fall through */
+		return OPA_LINK_WIDTH_4X;
-	case 4: return OPA_LINK_WIDTH_4X;
 	}
 }
 		case 0:
 			dd->pport[0].link_speed_active = OPA_LINK_SPEED_12_5G;
 			break;
+		case 1:
+			dd->pport[0].link_speed_active = OPA_LINK_SPEED_25G;
+			break;
 		default:
 			dd_dev_err(dd,
 				   "%s: unexpected max rate %d, using 25Gb\n",
 				   __func__, (int)max_rate);
-			/* fall through */
-		case 1:
 			dd->pport[0].link_speed_active = OPA_LINK_SPEED_25G;
 			break;
 		}
 static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate)
 {
 	switch (chip_lstate) {
-	default:
-		dd_dev_err(dd,
-			   "Unknown logical state 0x%x, reporting IB_PORT_DOWN\n",
-			   chip_lstate);
-		/* fall through */
 	case LSTATE_DOWN:
 		return IB_PORT_DOWN;
 	case LSTATE_INIT:
 		return IB_PORT_ARMED;
 	case LSTATE_ACTIVE:
 		return IB_PORT_ACTIVE;
+	default:
+		dd_dev_err(dd,
+			   "Unknown logical state 0x%x, reporting IB_PORT_DOWN\n",
+			   chip_lstate);
+		return IB_PORT_DOWN;
 	}
 }
 {
 	/* look at the HFI meta-states only */
 	switch (chip_pstate & 0xf0) {
-	default:
-		dd_dev_err(dd, "Unexpected chip physical state of 0x%x\n",
-			   chip_pstate);
-		/* fall through */
 	case PLS_DISABLED:
 		return IB_PORTPHYSSTATE_DISABLED;
 	case PLS_OFFLINE:
 		return IB_PORTPHYSSTATE_LINKUP;
 	case PLS_PHYTEST:
 		return IB_PORTPHYSSTATE_PHY_TEST;
+	default:
+		dd_dev_err(dd, "Unexpected chip physical state of 0x%x\n",
+			   chip_pstate);
+		return IB_PORTPHYSSTATE_DISABLED;
 	}
 }

drivers/infiniband/hw/hfi1/firmware.c+0−16

 									2;
 				break;
 			case PLATFORM_CONFIG_RX_PRESET_TABLE:
-				/* fall through */
 			case PLATFORM_CONFIG_TX_PRESET_TABLE:
-				/* fall through */
 			case PLATFORM_CONFIG_QSFP_ATTEN_TABLE:
-				/* fall through */
 			case PLATFORM_CONFIG_VARIABLE_SETTINGS_TABLE:
 				pcfgcache->config_tables[table_type].num_table =
 							table_length_dwords;
 			/* metadata table */
 			switch (table_type) {
 			case PLATFORM_CONFIG_SYSTEM_TABLE:
-				/* fall through */
 			case PLATFORM_CONFIG_PORT_TABLE:
-				/* fall through */
 			case PLATFORM_CONFIG_RX_PRESET_TABLE:
-				/* fall through */
 			case PLATFORM_CONFIG_TX_PRESET_TABLE:
-				/* fall through */
 			case PLATFORM_CONFIG_QSFP_ATTEN_TABLE:
-				/* fall through */
 			case PLATFORM_CONFIG_VARIABLE_SETTINGS_TABLE:
 				break;
 			default:
 	switch (table) {
 	case PLATFORM_CONFIG_SYSTEM_TABLE:
-		/* fall through */
 	case PLATFORM_CONFIG_PORT_TABLE:
-		/* fall through */
 	case PLATFORM_CONFIG_RX_PRESET_TABLE:
-		/* fall through */
 	case PLATFORM_CONFIG_TX_PRESET_TABLE:
-		/* fall through */
 	case PLATFORM_CONFIG_QSFP_ATTEN_TABLE:
-		/* fall through */
 	case PLATFORM_CONFIG_VARIABLE_SETTINGS_TABLE:
 		if (field && field < platform_config_table_limits[table])
 			src_ptr =
 			pcfgcache->config_tables[table_type].table;
 		break;
 	case PLATFORM_CONFIG_RX_PRESET_TABLE:
-		/* fall through */
 	case PLATFORM_CONFIG_TX_PRESET_TABLE:
-		/* fall through */
 	case PLATFORM_CONFIG_QSFP_ATTEN_TABLE:
-		/* fall through */
 	case PLATFORM_CONFIG_VARIABLE_SETTINGS_TABLE:
 		src_ptr = pcfgcache->config_tables[table_type].table;

drivers/infiniband/hw/hfi1/mad.c+4−5

 			/* Bad mkey not a violation below level 2 */
 			if (ibp->rvp.mkeyprot < 2)
 				break;
-			/* fall through */
+			fallthrough;
 		case IB_MGMT_METHOD_SET:
 		case IB_MGMT_METHOD_TRAP_REPRESS:
 			if (ibp->rvp.mkey_violations != 0xFFFF)
 	case IB_PORT_NOP:
 		if (phys_state == IB_PORTPHYSSTATE_NOP)
 			break;
-		/* FALLTHROUGH */
+		fallthrough;
 	case IB_PORT_DOWN:
 		if (phys_state == IB_PORTPHYSSTATE_NOP) {
 			link_state = HLS_DN_DOWNDEF;
 	 * can be changed from the default values
 	 */
 	case OPA_VLARB_PREEMPT_ELEMENTS:
-		/* FALLTHROUGH */
 	case OPA_VLARB_PREEMPT_MATRIX:
 		smp->status |= IB_SMP_UNSUP_METH_ATTR;
 		break;
 			return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED;
 		if (ibp->rvp.port_cap_flags & IB_PORT_SM)
 			return IB_MAD_RESULT_SUCCESS;
-		/* FALLTHROUGH */
+		fallthrough;
 	default:
 		smp->status |= IB_SMP_UNSUP_METH_ATTR;
 		ret = reply((struct ib_mad_hdr *)smp);
 			return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED;
 		if (ibp->rvp.port_cap_flags & IB_PORT_SM)
 			return IB_MAD_RESULT_SUCCESS;
-		/* FALLTHROUGH */
+		fallthrough;
 	default:
 		smp->status |= IB_SMP_UNSUP_METH_ATTR;
 		ret = reply((struct ib_mad_hdr *)smp);

drivers/infiniband/hw/hfi1/pcie.c+15−7

 	ret = pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &linkcap);
 	if (ret) {
 		dd_dev_err(dd, "Unable to read from PCI config\n");
-		return ret;
+		return pcibios_err_to_errno(ret);
 	}
 	if ((linkcap & PCI_EXP_LNKCAP_SLS) != PCI_EXP_LNKCAP_SLS_8_0GB) {
 	return 0;
 }
-/* restore command and BARs after a reset has wiped them out */
+/**
+ * Restore command and BARs after a reset has wiped them out
+ *
+ * Returns 0 on success, otherwise a negative error value
+ */
 int restore_pci_variables(struct hfi1_devdata *dd)
 {
-	int ret = 0;
+	int ret;
 	ret = pci_write_config_word(dd->pcidev, PCI_COMMAND, dd->pci_command);
 	if (ret)
 error:
 	dd_dev_err(dd, "Unable to write to PCI config\n");
-	return ret;
+	return pcibios_err_to_errno(ret);
 }
-/* Save BARs and command to rewrite after device reset */
+/**
+ * Save BARs and command to rewrite after device reset
+ *
+ * Returns 0 on success, otherwise a negative error value
+ */
 int save_pci_variables(struct hfi1_devdata *dd)
 {
-	int ret = 0;
+	int ret;
 	ret = pci_read_config_dword(dd->pcidev, PCI_BASE_ADDRESS_0,
 				    &dd->pcibar0);
 error:
 	dd_dev_err(dd, "Unable to read from PCI config\n");
-	return ret;
+	return pcibios_err_to_errno(ret);
 }
 /*

drivers/infiniband/hw/hfi1/pio.c+1−1

@@ -86,7 +86,7 @@ void pio_send_control(struct hfi1_devdata *dd, int op) · 85 unmodified lines
86	switch (op) {	86	switch (op) {
87	case PSC_GLOBAL_ENABLE:	87	case PSC_GLOBAL_ENABLE:
88	reg \|= SEND_CTRL_SEND_ENABLE_SMASK;	88	reg \|= SEND_CTRL_SEND_ENABLE_SMASK;
89	/* Fall through */	89	fallthrough;
90	case PSC_DATA_VL_ENABLE:	90	case PSC_DATA_VL_ENABLE:
91	mask = 0;	91	mask = 0;
92	for (i = 0; i < ARRAY_SIZE(dd->vld); i++)	92	for (i = 0; i < ARRAY_SIZE(dd->vld); i++)

drivers/infiniband/hw/hfi1/pio_copy.c+6−6

@@ -191,22 +191,22 @@ static inline void jcopy(u8 dest, const u8 src, u32 n) · 190 unmodified lines
191	switch (n) {	191	switch (n) {
192	case 7:	192	case 7:
193	dest++ = src++;	193	dest++ = src++;
194	/* fall through */	194	fallthrough;
195	case 6:	195	case 6:
196	dest++ = src++;	196	dest++ = src++;
197	/* fall through */	197	fallthrough;
198	case 5:	198	case 5:
199	dest++ = src++;	199	dest++ = src++;
200	/* fall through */	200	fallthrough;
201	case 4:	201	case 4:
202	dest++ = src++;	202	dest++ = src++;
203	/* fall through */	203	fallthrough;
204	case 3:	204	case 3:
205	dest++ = src++;	205	dest++ = src++;
206	/* fall through */	206	fallthrough;
207	case 2:	207	case 2:
208	dest++ = src++;	208	dest++ = src++;
209	/* fall through */	209	fallthrough;
210	case 1:	210	case 1:
211	dest++ = src++;	211	dest++ = src++;
212	/* fall through */	212	/* fall through */

drivers/infiniband/hw/hfi1/platform.c+5−5

@@ -668,8 +668,8 @@ static u8 aoc_low_power_setting(struct hfi1_pportdata *ppd) · 667 unmodified lines
668		668
669	/* active optical cables only */	669	/* active optical cables only */
670	switch ((cache[QSFP_MOD_TECH_OFFS] & 0xF0) >> 4) {	670	switch ((cache[QSFP_MOD_TECH_OFFS] & 0xF0) >> 4) {
671	case 0x0 ... 0x9: /* fallthrough */	671	case 0x0 ... 0x9: fallthrough;
672	case 0xC: /* fallthrough */	672	case 0xC: fallthrough;
673	case 0xE:	673	case 0xE:
674	/* active AOC */	674	/* active AOC */
675	power_class = get_qsfp_power_class(cache[QSFP_MOD_PWR_OFFS]);	675	power_class = get_qsfp_power_class(cache[QSFP_MOD_PWR_OFFS]);
@@ -899,8 +899,8 @@ static int tune_qsfp(struct hfi1_pportdata *ppd, · 223 unmodified lines
899		899
900	*ptr_tuning_method = OPA_PASSIVE_TUNING;	900	*ptr_tuning_method = OPA_PASSIVE_TUNING;
901	break;	901	break;
902	case 0x0 ... 0x9: /* fallthrough */	902	case 0x0 ... 0x9: fallthrough;
903	case 0xC: /* fallthrough */	903	case 0xC: fallthrough;
904	case 0xE:	904	case 0xE:
905	ret = tune_active_qsfp(ppd, ptr_tx_preset, ptr_rx_preset,	905	ret = tune_active_qsfp(ppd, ptr_tx_preset, ptr_rx_preset,
906	ptr_total_atten);	906	ptr_total_atten);
@@ -909,7 +909,7 @@ static int tune_qsfp(struct hfi1_pportdata *ppd, · 2 unmodified lines
909		909
910	*ptr_tuning_method = OPA_ACTIVE_TUNING;	910	*ptr_tuning_method = OPA_ACTIVE_TUNING;
911	break;	911	break;
912	case 0xD: /* fallthrough */	912	case 0xD: fallthrough;
913	case 0xF:	913	case 0xF:
914	default:	914	default:
915	dd_dev_warn(ppd->dd, "%s: Unknown/unsupported cable\n",	915	dd_dev_warn(ppd->dd, "%s: Unknown/unsupported cable\n",

drivers/infiniband/hw/hfi1/qp.c+1−1

@@ -312,7 +312,7 @@ int hfi1_setup_wqe(struct rvt_qp qp, struct rvt_swqe wqe, bool *call_send) · 311 unmodified lines
312	switch (qp->ibqp.qp_type) {	312	switch (qp->ibqp.qp_type) {
313	case IB_QPT_RC:	313	case IB_QPT_RC:
314	hfi1_setup_tid_rdma_wqe(qp, wqe);	314	hfi1_setup_tid_rdma_wqe(qp, wqe);
315	/* fall through */	315	fallthrough;
316	case IB_QPT_UC:	316	case IB_QPT_UC:
317	if (wqe->length > 0x80000000U)	317	if (wqe->length > 0x80000000U)
318	return -EINVAL;	318	return -EINVAL;

drivers/infiniband/hw/hfi1/qp.h+0−14

@@ -112,20 +112,6 @@ static inline void clear_ahg(struct rvt_qp *qp) · 111 unmodified lines
112	qp->s_ahgidx = -1;	112	qp->s_ahgidx = -1;
113	}	113	}
114		114
115	/**
116	* hfi1_create_qp - create a queue pair for a device
117	* @ibpd: the protection domain who's device we create the queue pair for
118	* @init_attr: the attributes of the queue pair
119	* @udata: user data for libibverbs.so
120	*
121	* Returns the queue pair on success, otherwise returns an errno.
122	*
123	* Called by the ib_create_qp() core verbs function.
124	*/
125	struct ib_qp hfi1_create_qp(struct ib_pd ibpd,
126	struct ib_qp_init_attr *init_attr,
127	struct ib_udata *udata);
128
129	/**	115	/**
130	* hfi1_qp_wakeup - wake up on the indicated event	116	* hfi1_qp_wakeup - wake up on the indicated event
131	* @qp: the QP	117	* @qp: the QP

drivers/infiniband/hw/hfi1/qsfp.c+2−2

@@ -231,7 +231,7 @@ static int i2c_bus_write(struct hfi1_devdata dd, struct hfi1_i2c_bus i2c, · 230 unmodified lines
231	break;	231	break;
232	case 2:	232	case 2:
233	offset_bytes[1] = (offset >> 8) & 0xff;	233	offset_bytes[1] = (offset >> 8) & 0xff;
234	/* fall through */	234	fallthrough;
235	case 1:	235	case 1:
236	num_msgs = 2;	236	num_msgs = 2;
237	offset_bytes[0] = offset & 0xff;	237	offset_bytes[0] = offset & 0xff;
@@ -279,7 +279,7 @@ static int i2c_bus_read(struct hfi1_devdata dd, struct hfi1_i2c_bus bus, · 41 unmodified lines
279	break;	279	break;
280	case 2:	280	case 2:
281	offset_bytes[1] = (offset >> 8) & 0xff;	281	offset_bytes[1] = (offset >> 8) & 0xff;
282	/* fall through */	282	fallthrough;
283	case 1:	283	case 1:
284	num_msgs = 2;	284	num_msgs = 2;
285	offset_bytes[0] = offset & 0xff;	285	offset_bytes[0] = offset & 0xff;

drivers/infiniband/hw/hfi1/rc.c+12−13

@@ -141,7 +141,7 @@ static int make_rc_ack(struct hfi1_ibdev dev, struct rvt_qp qp, · 140 unmodified lines
141	case OP(RDMA_READ_RESPONSE_ONLY):	141	case OP(RDMA_READ_RESPONSE_ONLY):
142	e = &qp->s_ack_queue[qp->s_tail_ack_queue];	142	e = &qp->s_ack_queue[qp->s_tail_ack_queue];
143	release_rdma_sge_mr(e);	143	release_rdma_sge_mr(e);
144	/* FALLTHROUGH */	144	fallthrough;
145	case OP(ATOMIC_ACKNOWLEDGE):	145	case OP(ATOMIC_ACKNOWLEDGE):
146	/*	146	/*
147	* We can increment the tail pointer now that the last	147	* We can increment the tail pointer now that the last
@@ -160,7 +160,7 @@ static int make_rc_ack(struct hfi1_ibdev dev, struct rvt_qp qp, · 12 unmodified lines
160	qp->s_acked_ack_queue = next;	160	qp->s_acked_ack_queue = next;
161	qp->s_tail_ack_queue = next;	161	qp->s_tail_ack_queue = next;
162	trace_hfi1_rsp_make_rc_ack(qp, e->psn);	162	trace_hfi1_rsp_make_rc_ack(qp, e->psn);
163	/* FALLTHROUGH */	163	fallthrough;
164	case OP(SEND_ONLY):	164	case OP(SEND_ONLY):
165	case OP(ACKNOWLEDGE):	165	case OP(ACKNOWLEDGE):
166	/* Check for no next entry in the queue. */	166	/* Check for no next entry in the queue. */
@@ -267,7 +267,7 @@ static int make_rc_ack(struct hfi1_ibdev dev, struct rvt_qp qp, · 100 unmodified lines
267		267
268	case OP(RDMA_READ_RESPONSE_FIRST):	268	case OP(RDMA_READ_RESPONSE_FIRST):
269	qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE);	269	qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE);
270	/* FALLTHROUGH */	270	fallthrough;
271	case OP(RDMA_READ_RESPONSE_MIDDLE):	271	case OP(RDMA_READ_RESPONSE_MIDDLE):
272	ps->s_txreq->ss = &qp->s_ack_rdma_sge;	272	ps->s_txreq->ss = &qp->s_ack_rdma_sge;
273	ps->s_txreq->mr = qp->s_ack_rdma_sge.sge.mr;	273	ps->s_txreq->mr = qp->s_ack_rdma_sge.sge.mr;
@@ -881,8 +881,7 @@ no_flow_control: · 607 unmodified lines
881	goto bail;	881	goto bail;
882	}	882	}
883	qp->s_num_rd_atomic++;	883	qp->s_num_rd_atomic++;
884		884	fallthrough;
885	/* FALLTHROUGH */
886	case IB_WR_OPFN:	885	case IB_WR_OPFN:
887	if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT))	886	if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
888	qp->s_lsn++;	887	qp->s_lsn++;
@@ -946,10 +945,10 @@ no_flow_control: · 57 unmodified lines
946	* See restart_rc().	945	* See restart_rc().
947	*/	946	*/
948	qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);	947	qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);
949	/* FALLTHROUGH */	948	fallthrough;
950	case OP(SEND_FIRST):	949	case OP(SEND_FIRST):
951	qp->s_state = OP(SEND_MIDDLE);	950	qp->s_state = OP(SEND_MIDDLE);
952	/* FALLTHROUGH */	951	fallthrough;
953	case OP(SEND_MIDDLE):	952	case OP(SEND_MIDDLE):
954	bth2 = mask_psn(qp->s_psn++);	953	bth2 = mask_psn(qp->s_psn++);
955	ss = &qp->s_sge;	954	ss = &qp->s_sge;
@@ -991,10 +990,10 @@ no_flow_control: · 35 unmodified lines
991	* See restart_rc().	990	* See restart_rc().
992	*/	991	*/
993	qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);	992	qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);
994	/* FALLTHROUGH */	993	fallthrough;
995	case OP(RDMA_WRITE_FIRST):	994	case OP(RDMA_WRITE_FIRST):
996	qp->s_state = OP(RDMA_WRITE_MIDDLE);	995	qp->s_state = OP(RDMA_WRITE_MIDDLE);
997	/* FALLTHROUGH */	996	fallthrough;
998	case OP(RDMA_WRITE_MIDDLE):	997	case OP(RDMA_WRITE_MIDDLE):
999	bth2 = mask_psn(qp->s_psn++);	998	bth2 = mask_psn(qp->s_psn++);
1000	ss = &qp->s_sge;	999	ss = &qp->s_sge;
@@ -2901,7 +2900,7 @@ void hfi1_rc_rcv(struct hfi1_packet *packet) · 1900 unmodified lines
2901	if (!ret)	2900	if (!ret)
2902	goto rnr_nak;	2901	goto rnr_nak;
2903	qp->r_rcv_len = 0;	2902	qp->r_rcv_len = 0;
2904	/* FALLTHROUGH */	2903	fallthrough;
2905	case OP(SEND_MIDDLE):	2904	case OP(SEND_MIDDLE):
2906	case OP(RDMA_WRITE_MIDDLE):	2905	case OP(RDMA_WRITE_MIDDLE):
2907	send_middle:	2906	send_middle:
@@ -2941,7 +2940,7 @@ send_middle: · 33 unmodified lines
2941	goto no_immediate_data;	2940	goto no_immediate_data;
2942	if (opcode == OP(SEND_ONLY_WITH_INVALIDATE))	2941	if (opcode == OP(SEND_ONLY_WITH_INVALIDATE))
2943	goto send_last_inv;	2942	goto send_last_inv;
2944	/* FALLTHROUGH -- for SEND_ONLY_WITH_IMMEDIATE */	2943	fallthrough; /* for SEND_ONLY_WITH_IMMEDIATE */
2945	case OP(SEND_LAST_WITH_IMMEDIATE):	2944	case OP(SEND_LAST_WITH_IMMEDIATE):
2946	send_last_imm:	2945	send_last_imm:
2947	wc.ex.imm_data = ohdr->u.imm_data;	2946	wc.ex.imm_data = ohdr->u.imm_data;
@@ -2957,7 +2956,7 @@ send_last_inv: · 9 unmodified lines
2957	goto send_last;	2956	goto send_last;
2958	case OP(RDMA_WRITE_LAST):	2957	case OP(RDMA_WRITE_LAST):
2959	copy_last = rvt_is_user_qp(qp);	2958	copy_last = rvt_is_user_qp(qp);
2960	/* fall through */	2959	fallthrough;
2961	case OP(SEND_LAST):	2960	case OP(SEND_LAST):
2962	no_immediate_data:	2961	no_immediate_data:
2963	wc.wc_flags = 0;	2962	wc.wc_flags = 0;
@@ -3010,7 +3009,7 @@ send_last: · 46 unmodified lines
3010		3009
3011	case OP(RDMA_WRITE_ONLY):	3010	case OP(RDMA_WRITE_ONLY):
3012	copy_last = rvt_is_user_qp(qp);	3011	copy_last = rvt_is_user_qp(qp);
3013	/* fall through */	3012	fallthrough;
3014	case OP(RDMA_WRITE_FIRST):	3013	case OP(RDMA_WRITE_FIRST):
3015	case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE):	3014	case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE):
3016	if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))	3015	if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))

drivers/infiniband/hw/hfi1/sdma.c+4−5

 			 * 7220, e.g.
 			 */
 			ss->go_s99_running = 1;
-			/* fall through -- and start dma engine */
+			fallthrough;	/* and start dma engine */
 		case sdma_event_e10_go_hw_start:
 			/* This reference means the state machine is started */
 			sdma_get(&sde->state);
 		case sdma_event_e70_go_idle:
 			break;
 		case sdma_event_e85_link_down:
-			/* fall through */
 		case sdma_event_e80_hw_freeze:
 			sdma_set_state(sde, sdma_state_s80_hw_freeze);
 			atomic_dec(&sde->dd->sdma_unfreeze_count);
 		case sdma_event_e60_hw_halted:
 			need_progress = 1;
 			sdma_err_progress_check_schedule(sde);
-			/* fall through */
+			fallthrough;
 		case sdma_event_e90_sw_halted:
 			/*
 			* SW initiated halt does not perform engines
 			break;
 		case sdma_event_e85_link_down:
 			ss->go_s99_running = 0;
-			/* fall through */
+			fallthrough;
 		case sdma_event_e80_hw_freeze:
 			sdma_set_state(sde, sdma_state_s80_hw_freeze);
 			atomic_dec(&sde->dd->sdma_unfreeze_count);
 		tx->num_desc++;
 		tx->descs[2].qw[0] = 0;
 		tx->descs[2].qw[1] = 0;
-		/* FALLTHROUGH */
+		fallthrough;
 	case SDMA_AHG_APPLY_UPDATE2:
 		tx->num_desc++;
 		tx->descs[1].qw[0] = 0;

drivers/infiniband/hw/hfi1/tid_rdma.c+2−2

@@ -3227,7 +3227,7 @@ bool hfi1_tid_rdma_wqe_interlock(struct rvt_qp qp, struct rvt_swqe wqe) · 3226 unmodified lines
3227	case IB_WR_RDMA_READ:	3227	case IB_WR_RDMA_READ:
3228	if (prev->wr.opcode != IB_WR_TID_RDMA_WRITE)	3228	if (prev->wr.opcode != IB_WR_TID_RDMA_WRITE)
3229	break;	3229	break;
3230	/* fall through */	3230	fallthrough;
3231	case IB_WR_TID_RDMA_READ:	3231	case IB_WR_TID_RDMA_READ:
3232	switch (prev->wr.opcode) {	3232	switch (prev->wr.opcode) {
3233	case IB_WR_RDMA_READ:	3233	case IB_WR_RDMA_READ:
@@ -5067,7 +5067,7 @@ int hfi1_make_tid_rdma_pkt(struct rvt_qp qp, struct hfi1_pkt_state ps) · 1833 unmodified lines
5067	if (priv->s_state == TID_OP(WRITE_REQ))	5067	if (priv->s_state == TID_OP(WRITE_REQ))
5068	hfi1_tid_rdma_restart_req(qp, wqe, &bth2);	5068	hfi1_tid_rdma_restart_req(qp, wqe, &bth2);
5069	priv->s_state = TID_OP(WRITE_DATA);	5069	priv->s_state = TID_OP(WRITE_DATA);
5070	/* fall through */	5070	fallthrough;
5071		5071
5072	case TID_OP(WRITE_DATA):	5072	case TID_OP(WRITE_DATA):
5073	/*	5073	/*

drivers/infiniband/hw/hfi1/uc.c+4−4

@@ -216,7 +216,7 @@ int hfi1_make_uc_req(struct rvt_qp qp, struct hfi1_pkt_state ps) · 215 unmodified lines
216		216
217	case OP(SEND_FIRST):	217	case OP(SEND_FIRST):
218	qp->s_state = OP(SEND_MIDDLE);	218	qp->s_state = OP(SEND_MIDDLE);
219	/* FALLTHROUGH */	219	fallthrough;
220	case OP(SEND_MIDDLE):	220	case OP(SEND_MIDDLE):
221	len = qp->s_len;	221	len = qp->s_len;
222	if (len > pmtu) {	222	if (len > pmtu) {
@@ -241,7 +241,7 @@ int hfi1_make_uc_req(struct rvt_qp qp, struct hfi1_pkt_state ps) · 18 unmodified lines
241		241
242	case OP(RDMA_WRITE_FIRST):	242	case OP(RDMA_WRITE_FIRST):
243	qp->s_state = OP(RDMA_WRITE_MIDDLE);	243	qp->s_state = OP(RDMA_WRITE_MIDDLE);
244	/* FALLTHROUGH */	244	fallthrough;
245	case OP(RDMA_WRITE_MIDDLE):	245	case OP(RDMA_WRITE_MIDDLE):
246	len = qp->s_len;	246	len = qp->s_len;
247	if (len > pmtu) {	247	if (len > pmtu) {
@@ -414,7 +414,7 @@ send_first: · 166 unmodified lines
414	goto no_immediate_data;	414	goto no_immediate_data;
415	else if (opcode == OP(SEND_ONLY_WITH_IMMEDIATE))	415	else if (opcode == OP(SEND_ONLY_WITH_IMMEDIATE))
416	goto send_last_imm;	416	goto send_last_imm;
417	/* FALLTHROUGH */	417	fallthrough;
418	case OP(SEND_MIDDLE):	418	case OP(SEND_MIDDLE):
419	/* Check for invalid length PMTU or posted rwqe len. */	419	/* Check for invalid length PMTU or posted rwqe len. */
420	/*	420	/*
@@ -515,7 +515,7 @@ rdma_first: · 94 unmodified lines
515	wc.ex.imm_data = ohdr->u.rc.imm_data;	515	wc.ex.imm_data = ohdr->u.rc.imm_data;
516	goto rdma_last_imm;	516	goto rdma_last_imm;
517	}	517	}
518	/* FALLTHROUGH */	518	fallthrough;
519	case OP(RDMA_WRITE_MIDDLE):	519	case OP(RDMA_WRITE_MIDDLE):
520	/* Check for invalid length PMTU or posted rwqe len. */	520	/* Check for invalid length PMTU or posted rwqe len. */
521	if (unlikely(tlen != (hdrsize + pmtu + 4)))	521	if (unlikely(tlen != (hdrsize + pmtu + 4)))

drivers/infiniband/hw/hns/hns_roce_device.h+16−15

 #define DRV_NAME "hns_roce"
-/* hip08 is a pci device, it includes two version according pci version id */
+/* hip08 is a pci device */
-#define PCI_REVISION_ID_HIP08_A			0x20
+#define PCI_REVISION_ID_HIP08			0x21
-#define PCI_REVISION_ID_HIP08_B			0x21
 #define HNS_ROCE_HW_VER1	('h' << 24 | 'i' << 16 | '0' << 8 | '6')
 	bool mtt_only; /* only alloc buffer-required MTT memory */
 };
+struct hns_roce_hem_cfg {
+	dma_addr_t	root_ba; /* root BA table's address */
+	bool		is_direct; /* addressing without BA table */
+	unsigned int	ba_pg_shift; /* BA table page shift */
+	unsigned int	buf_pg_shift; /* buffer page shift */
+	unsigned int	buf_pg_count;  /* buffer page count */
+	struct hns_roce_buf_region region[HNS_ROCE_MAX_BT_REGION];
+	int		region_count;
+};
 /* memory translate region */
 struct hns_roce_mtr {
 	struct hns_roce_hem_list hem_list; /* multi-hop addressing resource */
 	struct ib_umem		*umem; /* user space buffer */
 	struct hns_roce_buf	*kmem; /* kernel space buffer */
-	struct {
+	struct hns_roce_hem_cfg  hem_cfg; /* config for hardware addressing */
-		dma_addr_t	root_ba; /* root BA table's address */
-		bool		is_direct; /* addressing without BA table */
-		unsigned int	ba_pg_shift; /* BA table page shift */
-		unsigned int	buf_pg_shift; /* buffer page shift */
-		int		buf_pg_count;  /* buffer page count */
-		struct hns_roce_buf_region region[HNS_ROCE_MAX_BT_REGION];
-		unsigned int	region_count;
-	} hem_cfg; /* config for hardware addressing */
 };
 struct hns_roce_mw {
 			   u64 virt_addr, int mr_access_flags, struct ib_pd *pd,
 			   struct ib_udata *udata);
 struct ib_mr *hns_roce_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
-				u32 max_num_sg, struct ib_udata *udata);
+				u32 max_num_sg);
 int hns_roce_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
 		       unsigned int *sg_offset);
 int hns_roce_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
 int hns_roce_init(struct hns_roce_dev *hr_dev);
 void hns_roce_exit(struct hns_roce_dev *hr_dev);
-int hns_roce_fill_res_entry(struct sk_buff *msg,
+int hns_roce_fill_res_cq_entry(struct sk_buff *msg,
-			    struct rdma_restrack_entry *res);
+			       struct ib_cq *ib_cq);
 #endif /* _HNS_ROCE_DEVICE_H */

drivers/infiniband/hw/hns/hns_roce_hw_v1.c+3−4

@@ -2483,7 +2483,6 @@ static int find_wqe_mtt(struct hns_roce_dev hr_dev, struct hns_roce_qp hr_qp, · 2482 unmodified lines
2483	u64 sq_ba, u64 rq_ba, dma_addr_t *bt_ba)	2483	u64 sq_ba, u64 rq_ba, dma_addr_t *bt_ba)
2484	{	2484	{
2485	struct ib_device *ibdev = &hr_dev->ib_dev;	2485	struct ib_device *ibdev = &hr_dev->ib_dev;
2486	int rq_pa_start;
2487	int count;	2486	int count;
2488		2487
2489	count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, sq_ba, 1, bt_ba);	2488	count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, sq_ba, 1, bt_ba);
@@ -2491,9 +2490,9 @@ static int find_wqe_mtt(struct hns_roce_dev hr_dev, struct hns_roce_qp hr_qp, · 1 unmodified line
2491	ibdev_err(ibdev, "Failed to find SQ ba\n");	2490	ibdev_err(ibdev, "Failed to find SQ ba\n");
2492	return -ENOBUFS;	2491	return -ENOBUFS;
2493	}	2492	}
2494	rq_pa_start = hr_qp->rq.offset >> hr_qp->mtr.hem_cfg.buf_pg_shift;	2493
2495	count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, rq_pa_start, rq_ba, 1,	2494	count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->rq.offset, rq_ba,
2496	NULL);	2495	1, NULL);
2497	if (!count) {	2496	if (!count) {
2498	ibdev_err(ibdev, "Failed to find RQ ba\n");	2497	ibdev_err(ibdev, "Failed to find RQ ba\n");
2499	return -ENOBUFS;	2498	return -ENOBUFS;

drivers/infiniband/hw/hns/hns_roce_hw_v2.c+118−135

@@ -91,10 +91,11 @@ static u32 to_hr_opcode(u32 ib_opcode) · 90 unmodified lines
91	}	91	}
92		92
93	static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,	93	static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
94	void wqe, const struct ib_reg_wr wr)	94	const struct ib_reg_wr *wr)
95	{	95	{
		96	struct hns_roce_wqe_frmr_seg *fseg =
		97	(void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
96	struct hns_roce_mr *mr = to_hr_mr(wr->mr);	98	struct hns_roce_mr *mr = to_hr_mr(wr->mr);
97	struct hns_roce_wqe_frmr_seg *fseg = wqe;
98	u64 pbl_ba;	99	u64 pbl_ba;
99		100
100	/* use ib_access_flags */	101	/* use ib_access_flags */
@@ -128,14 +129,16 @@ static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, · 27 unmodified lines
128	V2_RC_FRMR_WQE_BYTE_40_BLK_MODE_S, 0);	129	V2_RC_FRMR_WQE_BYTE_40_BLK_MODE_S, 0);
129	}	130	}
130		131
131	static void set_atomic_seg(const struct ib_send_wr wr, void wqe,	132	static void set_atomic_seg(const struct ib_send_wr *wr,
132	struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,	133	struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
133	unsigned int valid_num_sge)	134	unsigned int valid_num_sge)
134	{	135	{
135	struct hns_roce_wqe_atomic_seg *aseg;	136	struct hns_roce_v2_wqe_data_seg *dseg =
		137	(void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
		138	struct hns_roce_wqe_atomic_seg *aseg =
		139	(void *)dseg + sizeof(struct hns_roce_v2_wqe_data_seg);
136		140
137	set_data_seg_v2(wqe, wr->sg_list);	141	set_data_seg_v2(dseg, wr->sg_list);
138	aseg = wqe + sizeof(struct hns_roce_v2_wqe_data_seg);
139		142
140	if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) {	143	if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
141	aseg->fetchadd_swap_data = cpu_to_le64(atomic_wr(wr)->swap);	144	aseg->fetchadd_swap_data = cpu_to_le64(atomic_wr(wr)->swap);
@@ -143,7 +146,7 @@ static void set_atomic_seg(const struct ib_send_wr wr, void wqe, · 1 unmodified line
143	} else {	146	} else {
144	aseg->fetchadd_swap_data =	147	aseg->fetchadd_swap_data =
145	cpu_to_le64(atomic_wr(wr)->compare_add);	148	cpu_to_le64(atomic_wr(wr)->compare_add);
146	aseg->cmp_data = 0;	149	aseg->cmp_data = 0;
147	}	150	}
148		151
149	roce_set_field(rc_sq_wqe->byte_16, V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,	152	roce_set_field(rc_sq_wqe->byte_16, V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
@@ -176,13 +179,15 @@ static void set_extend_sge(struct hns_roce_qp qp, const struct ib_send_wr wr, · 26 unmodified lines
176		179
177	static int set_rwqe_data_seg(struct ib_qp ibqp, const struct ib_send_wr wr,	180	static int set_rwqe_data_seg(struct ib_qp ibqp, const struct ib_send_wr wr,
178	struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,	181	struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
179	void wqe, unsigned int sge_ind,	182	unsigned int *sge_ind,
180	unsigned int valid_num_sge)	183	unsigned int valid_num_sge)
181	{	184	{
182	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);	185	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
183	struct hns_roce_v2_wqe_data_seg *dseg = wqe;	186	struct hns_roce_v2_wqe_data_seg *dseg =
		187	(void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
184	struct ib_device *ibdev = &hr_dev->ib_dev;	188	struct ib_device *ibdev = &hr_dev->ib_dev;
185	struct hns_roce_qp *qp = to_hr_qp(ibqp);	189	struct hns_roce_qp *qp = to_hr_qp(ibqp);
		190	void *wqe = dseg;
186	int j = 0;	191	int j = 0;
187	int i;	192	int i;
188		193
@@ -438,7 +443,6 @@ static inline int set_rc_wqe(struct hns_roce_qp *qp, · 249 unmodified lines
438	roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_OWNER_S,	443	roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_OWNER_S,
439	owner_bit);	444	owner_bit);
440		445
441	wqe += sizeof(struct hns_roce_v2_rc_send_wqe);
442	switch (wr->opcode) {	446	switch (wr->opcode) {
443	case IB_WR_RDMA_READ:	447	case IB_WR_RDMA_READ:
444	case IB_WR_RDMA_WRITE:	448	case IB_WR_RDMA_WRITE:
@@ -451,7 +455,7 @@ static inline int set_rc_wqe(struct hns_roce_qp *qp, · 6 unmodified lines
451	rc_sq_wqe->inv_key = cpu_to_le32(wr->ex.invalidate_rkey);	455	rc_sq_wqe->inv_key = cpu_to_le32(wr->ex.invalidate_rkey);
452	break;	456	break;
453	case IB_WR_REG_MR:	457	case IB_WR_REG_MR:
454	set_frmr_seg(rc_sq_wqe, wqe, reg_wr(wr));	458	set_frmr_seg(rc_sq_wqe, reg_wr(wr));
455	break;	459	break;
456	case IB_WR_ATOMIC_CMP_AND_SWP:	460	case IB_WR_ATOMIC_CMP_AND_SWP:
457	case IB_WR_ATOMIC_FETCH_AND_ADD:	461	case IB_WR_ATOMIC_FETCH_AND_ADD:
@@ -468,10 +472,10 @@ static inline int set_rc_wqe(struct hns_roce_qp *qp, · 10 unmodified lines
468		472
469	if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP \|\|	473	if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP \|\|
470	wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD)	474	wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD)
471	set_atomic_seg(wr, wqe, rc_sq_wqe, valid_num_sge);	475	set_atomic_seg(wr, rc_sq_wqe, valid_num_sge);
472	else if (wr->opcode != IB_WR_REG_MR)	476	else if (wr->opcode != IB_WR_REG_MR)
473	ret = set_rwqe_data_seg(&qp->ibqp, wr, rc_sq_wqe,	477	ret = set_rwqe_data_seg(&qp->ibqp, wr, rc_sq_wqe,
474	wqe, &curr_idx, valid_num_sge);	478	&curr_idx, valid_num_sge);
475		479
476	*sge_idx = curr_idx;	480	*sge_idx = curr_idx;
477		481
@@ -1510,8 +1514,6 @@ static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev) · 1032 unmodified lines
1510		1514
1511	req_a = (struct hns_roce_vf_res_a *)desc[0].data;	1515	req_a = (struct hns_roce_vf_res_a *)desc[0].data;
1512	req_b = (struct hns_roce_vf_res_b *)desc[1].data;	1516	req_b = (struct hns_roce_vf_res_b *)desc[1].data;
1513	memset(req_a, 0, sizeof(*req_a));
1514	memset(req_b, 0, sizeof(*req_b));
1515	for (i = 0; i < 2; i++) {	1517	for (i = 0; i < 2; i++) {
1516	hns_roce_cmq_setup_basic_desc(&desc[i],	1518	hns_roce_cmq_setup_basic_desc(&desc[i],
1517	HNS_ROCE_OPC_ALLOC_VF_RES, false);	1519	HNS_ROCE_OPC_ALLOC_VF_RES, false);
@@ -1744,27 +1746,25 @@ static void set_default_caps(struct hns_roce_dev *hr_dev) · 226 unmodified lines
1744	caps->max_srq_wrs = HNS_ROCE_V2_MAX_SRQ_WR;	1746	caps->max_srq_wrs = HNS_ROCE_V2_MAX_SRQ_WR;
1745	caps->max_srq_sges = HNS_ROCE_V2_MAX_SRQ_SGE;	1747	caps->max_srq_sges = HNS_ROCE_V2_MAX_SRQ_SGE;
1746		1748
1747	if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP08_B) {	1749	caps->flags \|= HNS_ROCE_CAP_FLAG_ATOMIC \| HNS_ROCE_CAP_FLAG_MW \|
1748	caps->flags \|= HNS_ROCE_CAP_FLAG_ATOMIC \| HNS_ROCE_CAP_FLAG_MW \|	1750	HNS_ROCE_CAP_FLAG_SRQ \| HNS_ROCE_CAP_FLAG_FRMR \|
1749	HNS_ROCE_CAP_FLAG_SRQ \| HNS_ROCE_CAP_FLAG_FRMR \|	1751	HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL;
1750	HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL;
1751		1752
1752	caps->num_qpc_timer = HNS_ROCE_V2_MAX_QPC_TIMER_NUM;	1753	caps->num_qpc_timer = HNS_ROCE_V2_MAX_QPC_TIMER_NUM;
1753	caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ;	1754	caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ;
1754	caps->qpc_timer_ba_pg_sz = 0;	1755	caps->qpc_timer_ba_pg_sz = 0;
1755	caps->qpc_timer_buf_pg_sz = 0;	1756	caps->qpc_timer_buf_pg_sz = 0;
1756	caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0;	1757	caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
1757	caps->num_cqc_timer = HNS_ROCE_V2_MAX_CQC_TIMER_NUM;	1758	caps->num_cqc_timer = HNS_ROCE_V2_MAX_CQC_TIMER_NUM;
1758	caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;	1759	caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;
1759	caps->cqc_timer_ba_pg_sz = 0;	1760	caps->cqc_timer_ba_pg_sz = 0;
1760	caps->cqc_timer_buf_pg_sz = 0;	1761	caps->cqc_timer_buf_pg_sz = 0;
1761	caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0;	1762	caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
1762		1763
1763	caps->sccc_entry_sz = HNS_ROCE_V2_SCCC_ENTRY_SZ;	1764	caps->sccc_entry_sz = HNS_ROCE_V2_SCCC_ENTRY_SZ;
1764	caps->sccc_ba_pg_sz = 0;	1765	caps->sccc_ba_pg_sz = 0;
1765	caps->sccc_buf_pg_sz = 0;	1766	caps->sccc_buf_pg_sz = 0;
1766	caps->sccc_hop_num = HNS_ROCE_SCCC_HOP_NUM;	1767	caps->sccc_hop_num = HNS_ROCE_SCCC_HOP_NUM;
1767	}
1768	}	1768	}
1769		1769
1770	static void calc_pg_sz(int obj_num, int obj_size, int hop_num, int ctx_bt_num,	1770	static void calc_pg_sz(int obj_num, int obj_size, int hop_num, int ctx_bt_num,
@@ -1995,20 +1995,18 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) · 224 unmodified lines
1995	caps->srqc_bt_num, &caps->srqc_buf_pg_sz,	1995	caps->srqc_bt_num, &caps->srqc_buf_pg_sz,
1996	&caps->srqc_ba_pg_sz, HEM_TYPE_SRQC);	1996	&caps->srqc_ba_pg_sz, HEM_TYPE_SRQC);
1997		1997
1998	if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP08_B) {	1998	caps->sccc_hop_num = ctx_hop_num;
1999	caps->sccc_hop_num = ctx_hop_num;	1999	caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
2000	caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0;	2000	caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
2001	caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
2002		2001
2003	calc_pg_sz(caps->num_qps, caps->sccc_entry_sz,	2002	calc_pg_sz(caps->num_qps, caps->sccc_entry_sz,
2004	caps->sccc_hop_num, caps->sccc_bt_num,	2003	caps->sccc_hop_num, caps->sccc_bt_num,
2005	&caps->sccc_buf_pg_sz, &caps->sccc_ba_pg_sz,	2004	&caps->sccc_buf_pg_sz, &caps->sccc_ba_pg_sz,
2006	HEM_TYPE_SCCC);	2005	HEM_TYPE_SCCC);
2007	calc_pg_sz(caps->num_cqc_timer, caps->cqc_timer_entry_sz,	2006	calc_pg_sz(caps->num_cqc_timer, caps->cqc_timer_entry_sz,
2008	caps->cqc_timer_hop_num, caps->cqc_timer_bt_num,	2007	caps->cqc_timer_hop_num, caps->cqc_timer_bt_num,
2009	&caps->cqc_timer_buf_pg_sz,	2008	&caps->cqc_timer_buf_pg_sz,
2010	&caps->cqc_timer_ba_pg_sz, HEM_TYPE_CQC_TIMER);	2009	&caps->cqc_timer_ba_pg_sz, HEM_TYPE_CQC_TIMER);
2011	}
2012		2010
2013	calc_pg_sz(caps->num_cqe_segs, caps->mtt_entry_sz, caps->cqe_hop_num,	2011	calc_pg_sz(caps->num_cqe_segs, caps->mtt_entry_sz, caps->cqe_hop_num,
2014	1, &caps->cqe_buf_pg_sz, &caps->cqe_ba_pg_sz, HEM_TYPE_CQE);	2012	1, &caps->cqe_buf_pg_sz, &caps->cqe_ba_pg_sz, HEM_TYPE_CQE);
@@ -2055,22 +2053,19 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) · 40 unmodified lines
2055	return ret;	2053	return ret;
2056	}	2054	}
2057		2055
2058	if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP08_B) {	2056	ret = hns_roce_query_pf_timer_resource(hr_dev);
2059	ret = hns_roce_query_pf_timer_resource(hr_dev);	2057	if (ret) {
2060	if (ret) {	2058	dev_err(hr_dev->dev,
2061	dev_err(hr_dev->dev,	2059	"failed to query pf timer resource, ret = %d.\n", ret);
2062	"Query pf timer resource fail, ret = %d.\n",	2060	return ret;
2063	ret);	2061	}
2064	return ret;
2065	}
2066		2062
2067	ret = hns_roce_set_vf_switch_param(hr_dev, 0);	2063	ret = hns_roce_set_vf_switch_param(hr_dev, 0);
2068	if (ret) {	2064	if (ret) {
2069	dev_err(hr_dev->dev,	2065	dev_err(hr_dev->dev,
2070	"Set function switch param fail, ret = %d.\n",	2066	"failed to set function switch param, ret = %d.\n",
2071	ret);	2067	ret);
2072	return ret;	2068	return ret;
2073	}
2074	}	2069	}
2075		2070
2076	hr_dev->vendor_part_id = hr_dev->pci_dev->device;	2071	hr_dev->vendor_part_id = hr_dev->pci_dev->device;
@@ -2336,8 +2331,7 @@ static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev) · 259 unmodified lines
2336	{	2331	{
2337	struct hns_roce_v2_priv *priv = hr_dev->priv;	2332	struct hns_roce_v2_priv *priv = hr_dev->priv;
2338		2333
2339	if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP08_B)	2334	hns_roce_function_clear(hr_dev);
2340	hns_roce_function_clear(hr_dev);
2341		2335
2342	hns_roce_free_link_table(hr_dev, &priv->tpq);	2336	hns_roce_free_link_table(hr_dev, &priv->tpq);
2343	hns_roce_free_link_table(hr_dev, &priv->tsq);	2337	hns_roce_free_link_table(hr_dev, &priv->tsq);
@@ -3053,6 +3047,7 @@ static void get_cqe_status(struct hns_roce_dev hr_dev, struct hns_roce_qp qp, · 709 unmodified lines
3053	IB_WC_RETRY_EXC_ERR },	3047	IB_WC_RETRY_EXC_ERR },
3054	{ HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR, IB_WC_RNR_RETRY_EXC_ERR },	3048	{ HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR, IB_WC_RNR_RETRY_EXC_ERR },
3055	{ HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR, IB_WC_REM_ABORT_ERR },	3049	{ HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR, IB_WC_REM_ABORT_ERR },
		3050	{ HNS_ROCE_CQE_V2_GENERAL_ERR, IB_WC_GENERAL_ERR}
3056	};	3051	};
3057		3052
3058	u32 cqe_status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M,	3053	u32 cqe_status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M,
@@ -3074,6 +3069,14 @@ static void get_cqe_status(struct hns_roce_dev hr_dev, struct hns_roce_qp qp, · 15 unmodified lines
3074	print_hex_dump(KERN_ERR, "", DUMP_PREFIX_NONE, 16, 4, cqe,	3069	print_hex_dump(KERN_ERR, "", DUMP_PREFIX_NONE, 16, 4, cqe,
3075	sizeof(*cqe), false);	3070	sizeof(*cqe), false);
3076		3071
		3072	/*
		3073	* For hns ROCEE, GENERAL_ERR is an error type that is not defined in
		3074	* the standard protocol, the driver must ignore it and needn't to set
		3075	* the QP to an error state.
		3076	*/
		3077	if (cqe_status == HNS_ROCE_CQE_V2_GENERAL_ERR)
		3078	return;
		3079
3077	/*	3080	/*
3078	* Hip08 hardware cannot flush the WQEs in SQ/RQ if the QP state gets	3081	* Hip08 hardware cannot flush the WQEs in SQ/RQ if the QP state gets
3079	* into errored mode. Hence, as a workaround to this hardware	3082	* into errored mode. Hence, as a workaround to this hardware
@@ -3170,51 +3173,51 @@ static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq, · 90 unmodified lines
3170	/* SQ corresponding to CQE */	3173	/* SQ corresponding to CQE */
3171	switch (roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M,	3174	switch (roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M,
3172	V2_CQE_BYTE_4_OPCODE_S) & 0x1f) {	3175	V2_CQE_BYTE_4_OPCODE_S) & 0x1f) {
3173	case HNS_ROCE_SQ_OPCODE_SEND:	3176	case HNS_ROCE_V2_WQE_OP_SEND:
3174	wc->opcode = IB_WC_SEND;	3177	wc->opcode = IB_WC_SEND;
3175	break;	3178	break;
3176	case HNS_ROCE_SQ_OPCODE_SEND_WITH_INV:	3179	case HNS_ROCE_V2_WQE_OP_SEND_WITH_INV:
3177	wc->opcode = IB_WC_SEND;	3180	wc->opcode = IB_WC_SEND;
3178	break;	3181	break;
3179	case HNS_ROCE_SQ_OPCODE_SEND_WITH_IMM:	3182	case HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM:
3180	wc->opcode = IB_WC_SEND;	3183	wc->opcode = IB_WC_SEND;
3181	wc->wc_flags \|= IB_WC_WITH_IMM;	3184	wc->wc_flags \|= IB_WC_WITH_IMM;
3182	break;	3185	break;
3183	case HNS_ROCE_SQ_OPCODE_RDMA_READ:	3186	case HNS_ROCE_V2_WQE_OP_RDMA_READ:
3184	wc->opcode = IB_WC_RDMA_READ;	3187	wc->opcode = IB_WC_RDMA_READ;
3185	wc->byte_len = le32_to_cpu(cqe->byte_cnt);	3188	wc->byte_len = le32_to_cpu(cqe->byte_cnt);
3186	break;	3189	break;
3187	case HNS_ROCE_SQ_OPCODE_RDMA_WRITE:	3190	case HNS_ROCE_V2_WQE_OP_RDMA_WRITE:
3188	wc->opcode = IB_WC_RDMA_WRITE;	3191	wc->opcode = IB_WC_RDMA_WRITE;
3189	break;	3192	break;
3190	case HNS_ROCE_SQ_OPCODE_RDMA_WRITE_WITH_IMM:	3193	case HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM:
3191	wc->opcode = IB_WC_RDMA_WRITE;	3194	wc->opcode = IB_WC_RDMA_WRITE;
3192	wc->wc_flags \|= IB_WC_WITH_IMM;	3195	wc->wc_flags \|= IB_WC_WITH_IMM;
3193	break;	3196	break;
3194	case HNS_ROCE_SQ_OPCODE_LOCAL_INV:	3197	case HNS_ROCE_V2_WQE_OP_LOCAL_INV:
3195	wc->opcode = IB_WC_LOCAL_INV;	3198	wc->opcode = IB_WC_LOCAL_INV;
3196	wc->wc_flags \|= IB_WC_WITH_INVALIDATE;	3199	wc->wc_flags \|= IB_WC_WITH_INVALIDATE;
3197	break;	3200	break;
3198	case HNS_ROCE_SQ_OPCODE_ATOMIC_COMP_AND_SWAP:	3201	case HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP:
3199	wc->opcode = IB_WC_COMP_SWAP;	3202	wc->opcode = IB_WC_COMP_SWAP;
3200	wc->byte_len = 8;	3203	wc->byte_len = 8;
3201	break;	3204	break;
3202	case HNS_ROCE_SQ_OPCODE_ATOMIC_FETCH_AND_ADD:	3205	case HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD:
3203	wc->opcode = IB_WC_FETCH_ADD;	3206	wc->opcode = IB_WC_FETCH_ADD;
3204	wc->byte_len = 8;	3207	wc->byte_len = 8;
3205	break;	3208	break;
3206	case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_COMP_AND_SWAP:	3209	case HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP:
3207	wc->opcode = IB_WC_MASKED_COMP_SWAP;	3210	wc->opcode = IB_WC_MASKED_COMP_SWAP;
3208	wc->byte_len = 8;	3211	wc->byte_len = 8;
3209	break;	3212	break;
3210	case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_FETCH_AND_ADD:	3213	case HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD:
3211	wc->opcode = IB_WC_MASKED_FETCH_ADD;	3214	wc->opcode = IB_WC_MASKED_FETCH_ADD;
3212	wc->byte_len = 8;	3215	wc->byte_len = 8;
3213	break;	3216	break;
3214	case HNS_ROCE_SQ_OPCODE_FAST_REG_WR:	3217	case HNS_ROCE_V2_WQE_OP_FAST_REG_PMR:
3215	wc->opcode = IB_WC_REG_MR;	3218	wc->opcode = IB_WC_REG_MR;
3216	break;	3219	break;
3217	case HNS_ROCE_SQ_OPCODE_BIND_MW:	3220	case HNS_ROCE_V2_WQE_OP_BIND_MW:
3218	wc->opcode = IB_WC_REG_MR;	3221	wc->opcode = IB_WC_REG_MR;
3219	break;	3222	break;
3220	default:	3223	default:
@@ -3374,11 +3377,33 @@ static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type, · 153 unmodified lines
3374	return op + step_idx;	3377	return op + step_idx;
3375	}	3378	}
3376		3379
		3380	static int set_hem_to_hw(struct hns_roce_dev *hr_dev, int obj, u64 bt_ba,
		3381	u32 hem_type, int step_idx)
		3382	{
		3383	struct hns_roce_cmd_mailbox *mailbox;
		3384	int ret;
		3385	int op;
		3386
		3387	op = get_op_for_set_hem(hr_dev, hem_type, step_idx);
		3388	if (op < 0)
		3389	return 0;
		3390
		3391	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
		3392	if (IS_ERR(mailbox))
		3393	return PTR_ERR(mailbox);
		3394
		3395	ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, obj,
		3396	0, op, HNS_ROCE_CMD_TIMEOUT_MSECS);
		3397
		3398	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
		3399
		3400	return ret;
		3401	}
		3402
3377	static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,	3403	static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,
3378	struct hns_roce_hem_table *table, int obj,	3404	struct hns_roce_hem_table *table, int obj,
3379	int step_idx)	3405	int step_idx)
3380	{	3406	{
3381	struct hns_roce_cmd_mailbox *mailbox;
3382	struct hns_roce_hem_iter iter;	3407	struct hns_roce_hem_iter iter;
3383	struct hns_roce_hem_mhop mhop;	3408	struct hns_roce_hem_mhop mhop;
3384	struct hns_roce_hem *hem;	3409	struct hns_roce_hem *hem;
@@ -3390,7 +3415,6 @@ static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev, · 5 unmodified lines
3390	u64 bt_ba = 0;	3415	u64 bt_ba = 0;
3391	u32 chunk_ba_num;	3416	u32 chunk_ba_num;
3392	u32 hop_num;	3417	u32 hop_num;
3393	int op;
3394		3418
3395	if (!hns_roce_check_whether_mhop(hr_dev, table->type))	3419	if (!hns_roce_check_whether_mhop(hr_dev, table->type))
3396	return 0;	3420	return 0;
@@ -3412,14 +3436,6 @@ static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev, · 15 unmodified lines
3412	hem_idx = i;	3436	hem_idx = i;
3413	}	3437	}
3414		3438
3415	op = get_op_for_set_hem(hr_dev, table->type, step_idx);
3416	if (op == -EINVAL)
3417	return 0;
3418
3419	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
3420	if (IS_ERR(mailbox))
3421	return PTR_ERR(mailbox);
3422
3423	if (table->type == HEM_TYPE_SCCC)	3439	if (table->type == HEM_TYPE_SCCC)
3424	obj = mhop.l0_idx;	3440	obj = mhop.l0_idx;
3425		3441
@@ -3428,11 +3444,8 @@ static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev, · 2 unmodified lines
3428	for (hns_roce_hem_first(hem, &iter);	3444	for (hns_roce_hem_first(hem, &iter);
3429	!hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {	3445	!hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
3430	bt_ba = hns_roce_hem_addr(&iter);	3446	bt_ba = hns_roce_hem_addr(&iter);
3431		3447	ret = set_hem_to_hw(hr_dev, obj, bt_ba, table->type,
3432	/* configure the ba, tag, and op */	3448	step_idx);
3433	ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma,
3434	obj, 0, op,
3435	HNS_ROCE_CMD_TIMEOUT_MSECS);
3436	}	3449	}
3437	} else {	3450	} else {
3438	if (step_idx == 0)	3451	if (step_idx == 0)
@@ -3440,12 +3453,9 @@ static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev, · 1 unmodified line
3440	else if (step_idx == 1 && hop_num == 2)	3453	else if (step_idx == 1 && hop_num == 2)
3441	bt_ba = table->bt_l1_dma_addr[l1_idx];	3454	bt_ba = table->bt_l1_dma_addr[l1_idx];
3442		3455
3443	/* configure the ba, tag, and op */	3456	ret = set_hem_to_hw(hr_dev, obj, bt_ba, table->type, step_idx);
3444	ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, obj,
3445	0, op, HNS_ROCE_CMD_TIMEOUT_MSECS);
3446	}	3457	}
3447		3458
3448	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
3449	return ret;	3459	return ret;
3450	}	3460	}
3451		3461
@@ -3745,51 +3755,23 @@ static void modify_qp_init_to_init(struct ib_qp *ibqp, · 293 unmodified lines
3745	}	3755	}
3746	}	3756	}
3747		3757
3748	static bool check_wqe_rq_mtt_count(struct hns_roce_dev *hr_dev,
3749	struct hns_roce_qp *hr_qp, int mtt_cnt,
3750	u32 page_size)
3751	{
3752	struct ib_device *ibdev = &hr_dev->ib_dev;
3753
3754	if (hr_qp->rq.wqe_cnt < 1)
3755	return true;
3756
3757	if (mtt_cnt < 1) {
3758	ibdev_err(ibdev, "failed to find RQWQE buf ba of QP(0x%lx)\n",
3759	hr_qp->qpn);
3760	return false;
3761	}
3762
3763	if (mtt_cnt < MTT_MIN_COUNT &&
3764	(hr_qp->rq.offset + page_size) < hr_qp->buff_size) {
3765	ibdev_err(ibdev,
3766	"failed to find next RQWQE buf ba of QP(0x%lx)\n",
3767	hr_qp->qpn);
3768	return false;
3769	}
3770
3771	return true;
3772	}
3773
3774	static int config_qp_rq_buf(struct hns_roce_dev *hr_dev,	3758	static int config_qp_rq_buf(struct hns_roce_dev *hr_dev,
3775	struct hns_roce_qp *hr_qp,	3759	struct hns_roce_qp *hr_qp,
3776	struct hns_roce_v2_qp_context *context,	3760	struct hns_roce_v2_qp_context *context,
3777	struct hns_roce_v2_qp_context *qpc_mask)	3761	struct hns_roce_v2_qp_context *qpc_mask)
3778	{	3762	{
3779	struct ib_qp *ibqp = &hr_qp->ibqp;
3780	u64 mtts[MTT_MIN_COUNT] = { 0 };	3763	u64 mtts[MTT_MIN_COUNT] = { 0 };
3781	u64 wqe_sge_ba;	3764	u64 wqe_sge_ba;
3782	u32 page_size;
3783	int count;	3765	int count;
3784		3766
3785	/* Search qp buf's mtts */	3767	/* Search qp buf's mtts */
3786	page_size = 1 << hr_qp->mtr.hem_cfg.buf_pg_shift;	3768	count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->rq.offset, mtts,
3787	count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
3788	hr_qp->rq.offset / page_size, mtts,
3789	MTT_MIN_COUNT, &wqe_sge_ba);	3769	MTT_MIN_COUNT, &wqe_sge_ba);
3790	if (!ibqp->srq)	3770	if (hr_qp->rq.wqe_cnt && count < 1) {
3791	if (!check_wqe_rq_mtt_count(hr_dev, hr_qp, count, page_size))	3771	ibdev_err(&hr_dev->ib_dev,
3792	return -EINVAL;	3772	"failed to find RQ WQE, QPN = 0x%lx.\n", hr_qp->qpn);
		3773	return -EINVAL;
		3774	}
3793		3775
3794	context->wqe_sge_ba = cpu_to_le32(wqe_sge_ba >> 3);	3776	context->wqe_sge_ba = cpu_to_le32(wqe_sge_ba >> 3);
3795	qpc_mask->wqe_sge_ba = 0;	3777	qpc_mask->wqe_sge_ba = 0;
@@ -3891,7 +3873,6 @@ static int config_qp_sq_buf(struct hns_roce_dev *hr_dev, · 95 unmodified lines
3891	struct ib_device *ibdev = &hr_dev->ib_dev;	3873	struct ib_device *ibdev = &hr_dev->ib_dev;
3892	u64 sge_cur_blk = 0;	3874	u64 sge_cur_blk = 0;
3893	u64 sq_cur_blk = 0;	3875	u64 sq_cur_blk = 0;
3894	u32 page_size;
3895	int count;	3876	int count;
3896		3877
3897	/* search qp buf's mtts */	3878	/* search qp buf's mtts */
@@ -3902,9 +3883,8 @@ static int config_qp_sq_buf(struct hns_roce_dev *hr_dev, · 4 unmodified lines
3902	return -EINVAL;	3883	return -EINVAL;
3903	}	3884	}
3904	if (hr_qp->sge.sge_cnt > 0) {	3885	if (hr_qp->sge.sge_cnt > 0) {
3905	page_size = 1 << hr_qp->mtr.hem_cfg.buf_pg_shift;
3906	count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,	3886	count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
3907	hr_qp->sge.offset / page_size,	3887	hr_qp->sge.offset,
3908	&sge_cur_blk, 1, NULL);	3888	&sge_cur_blk, 1, NULL);
3909	if (count < 1) {	3889	if (count < 1) {
3910	ibdev_err(ibdev, "failed to find QP(0x%lx) SGE buf.\n",	3890	ibdev_err(ibdev, "failed to find QP(0x%lx) SGE buf.\n",
@@ -4265,12 +4245,13 @@ static int hns_roce_v2_set_path(struct ib_qp *ibqp, · 354 unmodified lines
4265	roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M,	4245	roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M,
4266	V2_QPC_BYTE_24_HOP_LIMIT_S, 0);	4246	V2_QPC_BYTE_24_HOP_LIMIT_S, 0);
4267		4247
4268	if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP08_B && is_udp)	4248	if (is_udp)
4269	roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,	4249	roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
4270	V2_QPC_BYTE_24_TC_S, grh->traffic_class >> 2);	4250	V2_QPC_BYTE_24_TC_S, grh->traffic_class >> 2);
4271	else	4251	else
4272	roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,	4252	roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
4273	V2_QPC_BYTE_24_TC_S, grh->traffic_class);	4253	V2_QPC_BYTE_24_TC_S, grh->traffic_class);
		4254
4274	roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,	4255	roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
4275	V2_QPC_BYTE_24_TC_S, 0);	4256	V2_QPC_BYTE_24_TC_S, 0);
4276	roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,	4257	roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
@@ -4301,7 +4282,9 @@ static bool check_qp_state(enum ib_qp_state cur_state, · 24 unmodified lines
4301	[IB_QPS_RTR] = { [IB_QPS_RESET] = true,	4282	[IB_QPS_RTR] = { [IB_QPS_RESET] = true,
4302	[IB_QPS_RTS] = true,	4283	[IB_QPS_RTS] = true,
4303	[IB_QPS_ERR] = true },	4284	[IB_QPS_ERR] = true },
4304	[IB_QPS_RTS] = { [IB_QPS_RESET] = true, [IB_QPS_ERR] = true },	4285	[IB_QPS_RTS] = { [IB_QPS_RESET] = true,
		4286	[IB_QPS_RTS] = true,
		4287	[IB_QPS_ERR] = true },
4305	[IB_QPS_SQD] = {},	4288	[IB_QPS_SQD] = {},
4306	[IB_QPS_SQE] = {},	4289	[IB_QPS_SQE] = {},
4307	[IB_QPS_ERR] = { [IB_QPS_RESET] = true, [IB_QPS_ERR] = true }	4290	[IB_QPS_ERR] = { [IB_QPS_RESET] = true, [IB_QPS_ERR] = true }

drivers/infiniband/hw/hns/hns_roce_hw_v2.h+2−17

 	HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD	= 0x9,
 	HNS_ROCE_V2_WQE_OP_FAST_REG_PMR			= 0xa,
 	HNS_ROCE_V2_WQE_OP_LOCAL_INV			= 0xb,
-	HNS_ROCE_V2_WQE_OP_BIND_MW_TYPE			= 0xc,
+	HNS_ROCE_V2_WQE_OP_BIND_MW			= 0xc,
 	HNS_ROCE_V2_WQE_OP_MASK				= 0x1f,
 };
-enum {
-	HNS_ROCE_SQ_OPCODE_SEND = 0x0,
-	HNS_ROCE_SQ_OPCODE_SEND_WITH_INV = 0x1,
-	HNS_ROCE_SQ_OPCODE_SEND_WITH_IMM = 0x2,
-	HNS_ROCE_SQ_OPCODE_RDMA_WRITE = 0x3,
-	HNS_ROCE_SQ_OPCODE_RDMA_WRITE_WITH_IMM = 0x4,
-	HNS_ROCE_SQ_OPCODE_RDMA_READ = 0x5,
-	HNS_ROCE_SQ_OPCODE_ATOMIC_COMP_AND_SWAP = 0x6,
-	HNS_ROCE_SQ_OPCODE_ATOMIC_FETCH_AND_ADD = 0x7,
-	HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_COMP_AND_SWAP = 0x8,
-	HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_FETCH_AND_ADD = 0x9,
-	HNS_ROCE_SQ_OPCODE_FAST_REG_WR = 0xa,
-	HNS_ROCE_SQ_OPCODE_LOCAL_INV = 0xb,
-	HNS_ROCE_SQ_OPCODE_BIND_MW = 0xc,
-};
 enum {
 	/* rq operations */
 	HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM = 0x0,
 	HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR		= 0x15,
 	HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR		= 0x16,
 	HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR		= 0x22,
+	HNS_ROCE_CQE_V2_GENERAL_ERR			= 0x23,
 	HNS_ROCE_V2_CQE_STATUS_MASK			= 0xff,
 };

drivers/infiniband/hw/hns/hns_roce_main.c+1−1

@@ -428,7 +428,7 @@ static const struct ib_device_ops hns_roce_dev_ops = { · 427 unmodified lines
428	.destroy_ah = hns_roce_destroy_ah,	428	.destroy_ah = hns_roce_destroy_ah,
429	.destroy_cq = hns_roce_destroy_cq,	429	.destroy_cq = hns_roce_destroy_cq,
430	.disassociate_ucontext = hns_roce_disassociate_ucontext,	430	.disassociate_ucontext = hns_roce_disassociate_ucontext,
431	.fill_res_entry = hns_roce_fill_res_entry,	431	.fill_res_cq_entry = hns_roce_fill_res_cq_entry,
432	.get_dma_mr = hns_roce_get_dma_mr,	432	.get_dma_mr = hns_roce_get_dma_mr,
433	.get_link_layer = hns_roce_get_link_layer,	433	.get_link_layer = hns_roce_get_link_layer,
434	.get_port_immutable = hns_roce_port_immutable,	434	.get_port_immutable = hns_roce_port_immutable,

drivers/infiniband/hw/hns/hns_roce_mr.c+127−83

@@ -415,7 +415,7 @@ int hns_roce_dereg_mr(struct ib_mr ibmr, struct ib_udata udata) · 414 unmodified lines
415	}	415	}
416		416
417	struct ib_mr hns_roce_alloc_mr(struct ib_pd pd, enum ib_mr_type mr_type,	417	struct ib_mr hns_roce_alloc_mr(struct ib_pd pd, enum ib_mr_type mr_type,
418	u32 max_num_sg, struct ib_udata *udata)	418	u32 max_num_sg)
419	{	419	{
420	struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);	420	struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
421	struct device *dev = hr_dev->dev;	421	struct device *dev = hr_dev->dev;
@@ -871,6 +871,15 @@ int hns_roce_mtr_map(struct hns_roce_dev hr_dev, struct hns_roce_mtr mtr, · 449 unmodified lines
871	int err;	871	int err;
872	int i;	872	int i;
873		873
		874	/*
		875	* Only use the first page address as root ba when hopnum is 0, this
		876	* is because the addresses of all pages are consecutive in this case.
		877	*/
		878	if (mtr->hem_cfg.is_direct) {
		879	mtr->hem_cfg.root_ba = pages[0];
		880	return 0;
		881	}
		882
874	for (i = 0; i < mtr->hem_cfg.region_count; i++) {	883	for (i = 0; i < mtr->hem_cfg.region_count; i++) {
875	r = &mtr->hem_cfg.region[i];	884	r = &mtr->hem_cfg.region[i];
876	if (r->offset + r->count > page_cnt) {	885	if (r->offset + r->count > page_cnt) {
@@ -896,6 +905,8 @@ int hns_roce_mtr_map(struct hns_roce_dev hr_dev, struct hns_roce_mtr mtr, · 19 unmodified lines
896	int hns_roce_mtr_find(struct hns_roce_dev hr_dev, struct hns_roce_mtr mtr,	905	int hns_roce_mtr_find(struct hns_roce_dev hr_dev, struct hns_roce_mtr mtr,
897	int offset, u64 mtt_buf, int mtt_max, u64 base_addr)	906	int offset, u64 mtt_buf, int mtt_max, u64 base_addr)
898	{	907	{
		908	struct hns_roce_hem_cfg *cfg = &mtr->hem_cfg;
		909	int start_index;
899	int mtt_count;	910	int mtt_count;
900	int total = 0;	911	int total = 0;
901	__le64 *mtts;	912	__le64 *mtts;
@@ -907,26 +918,32 @@ int hns_roce_mtr_find(struct hns_roce_dev hr_dev, struct hns_roce_mtr mtr, · 5 unmodified lines
907	goto done;	918	goto done;
908		919
909	/* no mtt memory in direct mode, so just return the buffer address */	920	/* no mtt memory in direct mode, so just return the buffer address */
910	if (mtr->hem_cfg.is_direct) {	921	if (cfg->is_direct) {
911	npage = offset;	922	start_index = offset >> HNS_HW_PAGE_SHIFT;
912	for (total = 0; total < mtt_max; total++, npage++) {	923	for (mtt_count = 0; mtt_count < cfg->region_count &&
913	addr = mtr->hem_cfg.root_ba +	924	total < mtt_max; mtt_count++) {
914	(npage << mtr->hem_cfg.buf_pg_shift);	925	npage = cfg->region[mtt_count].offset;
915		926	if (npage < start_index)
		927	continue;
		928
		929	addr = cfg->root_ba + (npage << HNS_HW_PAGE_SHIFT);
916	if (hr_dev->hw_rev == HNS_ROCE_HW_VER1)	930	if (hr_dev->hw_rev == HNS_ROCE_HW_VER1)
917	mtt_buf[total] = to_hr_hw_page_addr(addr);	931	mtt_buf[total] = to_hr_hw_page_addr(addr);
918	else	932	else
919	mtt_buf[total] = addr;	933	mtt_buf[total] = addr;
		934
		935	total++;
920	}	936	}
921		937
922	goto done;	938	goto done;
923	}	939	}
924		940
		941	start_index = offset >> cfg->buf_pg_shift;
925	left = mtt_max;	942	left = mtt_max;
926	while (left > 0) {	943	while (left > 0) {
927	mtt_count = 0;	944	mtt_count = 0;
928	mtts = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list,	945	mtts = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list,
929	offset + total,	946	start_index + total,
930	&mtt_count, NULL);	947	&mtt_count, NULL);
931	if (!mtts \|\| !mtt_count)	948	if (!mtts \|\| !mtt_count)
932	goto done;	949	goto done;
@@ -939,104 +956,136 @@ int hns_roce_mtr_find(struct hns_roce_dev hr_dev, struct hns_roce_mtr mtr, · 6 unmodified lines
939		956
940	done:	957	done:
941	if (base_addr)	958	if (base_addr)
942	*base_addr = mtr->hem_cfg.root_ba;	959	*base_addr = cfg->root_ba;
943		960
944	return total;	961	return total;
945	}	962	}
946		963
947	/* convert buffer size to page index and page count */	964	static int mtr_init_buf_cfg(struct hns_roce_dev *hr_dev,
948	static unsigned int mtr_init_region(struct hns_roce_buf_attr *attr,	965	struct hns_roce_buf_attr *attr,
949	int page_cnt,	966	struct hns_roce_hem_cfg *cfg,
950	struct hns_roce_buf_region *regions,	967	unsigned int *buf_page_shift)
951	int region_cnt, unsigned int page_shift)
952	{	968	{
953	unsigned int page_size = 1 << page_shift;
954	int max_region = attr->region_count;
955	struct hns_roce_buf_region *r;	969	struct hns_roce_buf_region *r;
956	unsigned int i = 0;	970	unsigned int page_shift = 0;
957	int page_idx = 0;	971	int page_cnt = 0;
958		972	size_t buf_size;
959	for (; i < region_cnt && i < max_region && page_idx < page_cnt; i++) {	973	int region_cnt;
960	r = &regions[i];	974
961	r->hopnum = attr->region[i].hopnum == HNS_ROCE_HOP_NUM_0 ?	975	if (cfg->is_direct) {
962	0 : attr->region[i].hopnum;	976	buf_size = cfg->buf_pg_count << cfg->buf_pg_shift;
963	r->offset = page_idx;	977	page_cnt = DIV_ROUND_UP(buf_size, HNS_HW_PAGE_SIZE);
964	r->count = DIV_ROUND_UP(attr->region[i].size, page_size);	978	/*
965	page_idx += r->count;	979	* When HEM buffer use level-0 addressing, the page size equals
		980	* the buffer size, and the the page size = 4K * 2^N.
		981	*/
		982	cfg->buf_pg_shift = HNS_HW_PAGE_SHIFT + order_base_2(page_cnt);
		983	if (attr->region_count > 1) {
		984	cfg->buf_pg_count = page_cnt;
		985	page_shift = HNS_HW_PAGE_SHIFT;
		986	} else {
		987	cfg->buf_pg_count = 1;
		988	page_shift = cfg->buf_pg_shift;
		989	if (buf_size != 1 << page_shift) {
		990	ibdev_err(&hr_dev->ib_dev,
		991	"failed to check direct size %zu shift %d.\n",
		992	buf_size, page_shift);
		993	return -EINVAL;
		994	}
		995	}
		996	} else {
		997	page_shift = cfg->buf_pg_shift;
		998	}
		999
		1000	/* convert buffer size to page index and page count */
		1001	for (page_cnt = 0, region_cnt = 0; page_cnt < cfg->buf_pg_count &&
		1002	region_cnt < attr->region_count &&
		1003	region_cnt < ARRAY_SIZE(cfg->region); region_cnt++) {
		1004	r = &cfg->region[region_cnt];
		1005	r->offset = page_cnt;
		1006	buf_size = hr_hw_page_align(attr->region[region_cnt].size);
		1007	r->count = DIV_ROUND_UP(buf_size, 1 << page_shift);
		1008	page_cnt += r->count;
		1009	r->hopnum = to_hr_hem_hopnum(attr->region[region_cnt].hopnum,
		1010	r->count);
		1011	}
		1012
		1013	if (region_cnt < 1) {
		1014	ibdev_err(&hr_dev->ib_dev,
		1015	"failed to check mtr region count, pages = %d.\n",
		1016	cfg->buf_pg_count);
		1017	return -ENOBUFS;
966	}	1018	}
967		1019
968	return i;	1020	cfg->region_count = region_cnt;
		1021	*buf_page_shift = page_shift;
		1022
		1023	return page_cnt;
969	}	1024	}
970		1025
971	/**	1026	/**
972	* hns_roce_mtr_create - Create hns memory translate region.	1027	* hns_roce_mtr_create - Create hns memory translate region.
973	*	1028	*
974	* @mtr: memory translate region	1029	* @mtr: memory translate region
975	* @init_attr: init attribute for creating mtr	1030	* @buf_attr: buffer attribute for creating mtr
976	* @page_shift: page shift for multi-hop base address table	1031	* @ba_page_shift: page shift for multi-hop base address table
977	* @udata: user space context, if it's NULL, means kernel space	1032	* @udata: user space context, if it's NULL, means kernel space
978	* @user_addr: userspace virtual address to start at	1033	* @user_addr: userspace virtual address to start at
979	* @buf_alloced: mtr has private buffer, true means need to alloc
980	*/	1034	*/
981	int hns_roce_mtr_create(struct hns_roce_dev hr_dev, struct hns_roce_mtr mtr,	1035	int hns_roce_mtr_create(struct hns_roce_dev hr_dev, struct hns_roce_mtr mtr,
982	struct hns_roce_buf_attr *buf_attr,	1036	struct hns_roce_buf_attr *buf_attr,
983	unsigned int page_shift, struct ib_udata *udata,	1037	unsigned int ba_page_shift, struct ib_udata *udata,
984	unsigned long user_addr)	1038	unsigned long user_addr)
985	{	1039	{
		1040	struct hns_roce_hem_cfg *cfg = &mtr->hem_cfg;
986	struct ib_device *ibdev = &hr_dev->ib_dev;	1041	struct ib_device *ibdev = &hr_dev->ib_dev;
		1042	unsigned int buf_page_shift = 0;
987	dma_addr_t *pages = NULL;	1043	dma_addr_t *pages = NULL;
988	int region_cnt = 0;
989	int all_pg_cnt;	1044	int all_pg_cnt;
990	int get_pg_cnt;	1045	int get_pg_cnt;
991	bool has_mtt;	1046	int ret = 0;
992	int err = 0;	1047
		1048	/* if disable mtt, all pages must in a continuous address range */
		1049	cfg->is_direct = !mtr_has_mtt(buf_attr);
993		1050
994	has_mtt = mtr_has_mtt(buf_attr);
995	/* if buffer only need mtt, just init the hem cfg */	1051	/* if buffer only need mtt, just init the hem cfg */
996	if (buf_attr->mtt_only) {	1052	if (buf_attr->mtt_only) {
997	mtr->hem_cfg.buf_pg_shift = buf_attr->page_shift;	1053	cfg->buf_pg_shift = buf_attr->page_shift;
998	mtr->hem_cfg.buf_pg_count = mtr_bufs_size(buf_attr) >>	1054	cfg->buf_pg_count = mtr_bufs_size(buf_attr) >>
999	buf_attr->page_shift;	1055	buf_attr->page_shift;
1000	mtr->umem = NULL;	1056	mtr->umem = NULL;
1001	mtr->kmem = NULL;	1057	mtr->kmem = NULL;
1002	} else {	1058	} else {
1003	err = mtr_alloc_bufs(hr_dev, mtr, buf_attr, !has_mtt, udata,	1059	ret = mtr_alloc_bufs(hr_dev, mtr, buf_attr, cfg->is_direct,
1004	user_addr);	1060	udata, user_addr);
1005	if (err) {	1061	if (ret) {
1006	ibdev_err(ibdev, "Failed to alloc mtr bufs, err %d\n",	1062	ibdev_err(ibdev,
1007	err);	1063	"failed to alloc mtr bufs, ret = %d.\n", ret);
1008	return err;	1064	return ret;
1009	}	1065	}
1010	}	1066	}
1011		1067
1012	/* alloc mtt memory */	1068	all_pg_cnt = mtr_init_buf_cfg(hr_dev, buf_attr, cfg, &buf_page_shift);
1013	all_pg_cnt = mtr->hem_cfg.buf_pg_count;	1069	if (all_pg_cnt < 1) {
1014	hns_roce_hem_list_init(&mtr->hem_list);	1070	ret = -ENOBUFS;
1015	mtr->hem_cfg.is_direct = !has_mtt;	1071	ibdev_err(ibdev, "failed to init mtr buf cfg.\n");
1016	mtr->hem_cfg.ba_pg_shift = page_shift;
1017	mtr->hem_cfg.region_count = 0;
1018	region_cnt = mtr_init_region(buf_attr, all_pg_cnt,
1019	mtr->hem_cfg.region,
1020	ARRAY_SIZE(mtr->hem_cfg.region),
1021	mtr->hem_cfg.buf_pg_shift);
1022	if (region_cnt < 1) {
1023	err = -ENOBUFS;
1024	ibdev_err(ibdev, "failed to init mtr region %d\n", region_cnt);
1025	goto err_alloc_bufs;	1072	goto err_alloc_bufs;
1026	}	1073	}
1027		1074
1028	mtr->hem_cfg.region_count = region_cnt;	1075	hns_roce_hem_list_init(&mtr->hem_list);
1029		1076	if (!cfg->is_direct) {
1030	if (has_mtt) {	1077	ret = hns_roce_hem_list_request(hr_dev, &mtr->hem_list,
1031	err = hns_roce_hem_list_request(hr_dev, &mtr->hem_list,	1078	cfg->region, cfg->region_count,
1032	mtr->hem_cfg.region, region_cnt,	1079	ba_page_shift);
1033	page_shift);	1080	if (ret) {
1034	if (err) {	1081	ibdev_err(ibdev, "failed to request mtr hem, ret = %d.\n",
1035	ibdev_err(ibdev, "Failed to request mtr hem, err %d\n",	1082	ret);
1036	err);
1037	goto err_alloc_bufs;	1083	goto err_alloc_bufs;
1038	}	1084	}
1039	mtr->hem_cfg.root_ba = mtr->hem_list.root_ba;	1085	cfg->root_ba = mtr->hem_list.root_ba;
		1086	cfg->ba_pg_shift = ba_page_shift;
		1087	} else {
		1088	cfg->ba_pg_shift = cfg->buf_pg_shift;
1040	}	1089	}
1041		1090
1042	/* no buffer to map */	1091	/* no buffer to map */
@@ -1046,31 +1095,26 @@ int hns_roce_mtr_create(struct hns_roce_dev hr_dev, struct hns_roce_mtr mtr, · 3 unmodified lines
1046	/* alloc a tmp array to store buffer's dma address */	1095	/* alloc a tmp array to store buffer's dma address */
1047	pages = kvcalloc(all_pg_cnt, sizeof(dma_addr_t), GFP_KERNEL);	1096	pages = kvcalloc(all_pg_cnt, sizeof(dma_addr_t), GFP_KERNEL);
1048	if (!pages) {	1097	if (!pages) {
1049	err = -ENOMEM;	1098	ret = -ENOMEM;
1050	ibdev_err(ibdev, "Failed to alloc mtr page list %d\n",	1099	ibdev_err(ibdev, "failed to alloc mtr page list %d.\n",
1051	all_pg_cnt);	1100	all_pg_cnt);
1052	goto err_alloc_hem_list;	1101	goto err_alloc_hem_list;
1053	}	1102	}
1054		1103
1055	get_pg_cnt = mtr_get_pages(hr_dev, mtr, pages, all_pg_cnt,	1104	get_pg_cnt = mtr_get_pages(hr_dev, mtr, pages, all_pg_cnt,
1056	mtr->hem_cfg.buf_pg_shift);	1105	buf_page_shift);
1057	if (get_pg_cnt != all_pg_cnt) {	1106	if (get_pg_cnt != all_pg_cnt) {
1058	ibdev_err(ibdev, "Failed to get mtr page %d != %d\n",	1107	ibdev_err(ibdev, "failed to get mtr page %d != %d.\n",
1059	get_pg_cnt, all_pg_cnt);	1108	get_pg_cnt, all_pg_cnt);
1060	err = -ENOBUFS;	1109	ret = -ENOBUFS;
1061	goto err_alloc_page_list;	1110	goto err_alloc_page_list;
1062	}	1111	}
1063		1112
1064	if (!has_mtt) {	1113	/* write buffer's dma address to BA table */
1065	mtr->hem_cfg.root_ba = pages[0];	1114	ret = hns_roce_mtr_map(hr_dev, mtr, pages, all_pg_cnt);
1066	} else {	1115	if (ret) {
1067	/* write buffer's dma address to BA table */	1116	ibdev_err(ibdev, "failed to map mtr pages, ret = %d.\n", ret);
1068	err = hns_roce_mtr_map(hr_dev, mtr, pages, all_pg_cnt);	1117	goto err_alloc_page_list;
1069	if (err) {
1070	ibdev_err(ibdev, "Failed to map mtr pages, err %d\n",
1071	err);
1072	goto err_alloc_page_list;
1073	}
1074	}	1118	}
1075		1119
1076	/* drop tmp array */	1120	/* drop tmp array */
@@ -1082,7 +1126,7 @@ err_alloc_hem_list: · 5 unmodified lines
1082	hns_roce_hem_list_release(hr_dev, &mtr->hem_list);	1126	hns_roce_hem_list_release(hr_dev, &mtr->hem_list);
1083	err_alloc_bufs:	1127	err_alloc_bufs:
1084	mtr_free_bufs(hr_dev, mtr);	1128	mtr_free_bufs(hr_dev, mtr);
1085	return err;	1129	return ret;
1086	}	1130	}
1087		1131
1088	void hns_roce_mtr_destroy(struct hns_roce_dev hr_dev, struct hns_roce_mtr mtr)	1132	void hns_roce_mtr_destroy(struct hns_roce_dev hr_dev, struct hns_roce_mtr mtr)

drivers/infiniband/hw/hns/hns_roce_qp.c+0−10

 				struct hns_roce_qp *hr_qp,
 				struct ib_qp_cap *cap)
 {
-	struct ib_device *ibdev = &hr_dev->ib_dev;
 	u32 cnt;
 	cnt = max(1U, cap->max_send_sge);
 	} else if (hr_qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE) {
 		cnt = roundup_pow_of_two(sq_wqe_cnt *
 				     (hr_qp->sq.max_gs - HNS_ROCE_SGE_IN_WQE));
-		if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08_A) {
-			if (cnt > hr_dev->caps.max_extend_sg) {
-				ibdev_err(ibdev,
-					  "failed to check exSGE num, exSGE num = %d.\n",
-					  cnt);
-				return -EINVAL;
-			}
-		}
 	} else {
 		cnt = 0;
 	}

drivers/infiniband/hw/hns/hns_roce_restrack.c+2−12

 	return -EMSGSIZE;
 }
-static int hns_roce_fill_res_cq_entry(struct sk_buff *msg,
+int hns_roce_fill_res_cq_entry(struct sk_buff *msg,
-				      struct rdma_restrack_entry *res)
+			       struct ib_cq *ib_cq)
 {
-	struct ib_cq *ib_cq = container_of(res, struct ib_cq, res);
 	struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device);
 	struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);
 	struct hns_roce_v2_cq_context *context;
 	kfree(context);
 	return ret;
 }
-int hns_roce_fill_res_entry(struct sk_buff *msg,
-			    struct rdma_restrack_entry *res)
-{
-	if (res->type == RDMA_RESTRACK_CQ)
-		return hns_roce_fill_res_cq_entry(msg, res);
-	return 0;
-}

drivers/infiniband/hw/i40iw/i40iw_verbs.c+1−21

 	props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
 		IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;
 	props->gid_tbl_len = 1;
-	props->pkey_tbl_len = 1;
 	props->active_width = IB_WIDTH_4X;
 	props->active_speed = 1;
 	props->max_msg_sz = I40IW_MAX_OUTBOUND_MESSAGE_SIZE;
  * @pd: ibpd pointer
  * @mr_type: memory for stag registrion
  * @max_num_sg: man number of pages
- * @udata: user data or NULL for kernel objects
  */
 static struct ib_mr *i40iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
-				    u32 max_num_sg, struct ib_udata *udata)
+				    u32 max_num_sg)
 {
 	struct i40iw_pd *iwpd = to_iwpd(pd);
 	struct i40iw_device *iwdev = to_iwdev(pd->device);
 	if (err)
 		return err;
-	immutable->pkey_tbl_len = attr.pkey_tbl_len;
 	immutable->gid_tbl_len = attr.gid_tbl_len;
 	return 0;
 	return 0;
 }
-/**
- * i40iw_query_pkey - Query partition key
- * @ibdev: device pointer from stack
- * @port: port number
- * @index: index of pkey
- * @pkey: pointer to store the pkey
- */
-static int i40iw_query_pkey(struct ib_device *ibdev,
-			    u8 port,
-			    u16 index,
-			    u16 *pkey)
-{
-	*pkey = 0;
-	return 0;
-}
 static const struct ib_device_ops i40iw_dev_ops = {
 	.owner = THIS_MODULE,
 	.driver_id = RDMA_DRIVER_I40IW,
 	.post_send = i40iw_post_send,
 	.query_device = i40iw_query_device,
 	.query_gid = i40iw_query_gid,
-	.query_pkey = i40iw_query_pkey,
 	.query_port = i40iw_query_port,
 	.query_qp = i40iw_query_qp,
 	.reg_user_mr = i40iw_reg_user_mr,

drivers/infiniband/hw/mlx4/main.c+15−22

 	mlx4_pd_free(to_mdev(pd->device)->dev, to_mpd(pd)->pdn);
 }
-static struct ib_xrcd *mlx4_ib_alloc_xrcd(struct ib_device *ibdev,
+static int mlx4_ib_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
-					  struct ib_udata *udata)
 {
-	struct mlx4_ib_xrcd *xrcd;
+	struct mlx4_ib_dev *dev = to_mdev(ibxrcd->device);
+	struct mlx4_ib_xrcd *xrcd = to_mxrcd(ibxrcd);
 	struct ib_cq_init_attr cq_attr = {};
 	int err;
-	if (!(to_mdev(ibdev)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC))
+	if (!(dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC))
-		return ERR_PTR(-ENOSYS);
+		return -EOPNOTSUPP;
-	xrcd = kmalloc(sizeof *xrcd, GFP_KERNEL);
-	if (!xrcd)
-		return ERR_PTR(-ENOMEM);
-	err = mlx4_xrcd_alloc(to_mdev(ibdev)->dev, &xrcd->xrcdn);
+	err = mlx4_xrcd_alloc(dev->dev, &xrcd->xrcdn);
 	if (err)
-		goto err1;
+		return err;
-	xrcd->pd = ib_alloc_pd(ibdev, 0);
+	xrcd->pd = ib_alloc_pd(ibxrcd->device, 0);
 	if (IS_ERR(xrcd->pd)) {
 		err = PTR_ERR(xrcd->pd);
 		goto err2;
 	}
 	cq_attr.cqe = 1;
-	xrcd->cq = ib_create_cq(ibdev, NULL, NULL, xrcd, &cq_attr);
+	xrcd->cq = ib_create_cq(ibxrcd->device, NULL, NULL, xrcd, &cq_attr);
 	if (IS_ERR(xrcd->cq)) {
 		err = PTR_ERR(xrcd->cq);
 		goto err3;
 	}
-	return &xrcd->ibxrcd;
+	return 0;
 err3:
 	ib_dealloc_pd(xrcd->pd);
 err2:
-	mlx4_xrcd_free(to_mdev(ibdev)->dev, xrcd->xrcdn);
+	mlx4_xrcd_free(dev->dev, xrcd->xrcdn);
-err1:
+	return err;
-	kfree(xrcd);
-	return ERR_PTR(err);
 }
-static int mlx4_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)
+static void mlx4_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)
 {
 	ib_destroy_cq(to_mxrcd(xrcd)->cq);
 	ib_dealloc_pd(to_mxrcd(xrcd)->pd);
 	mlx4_xrcd_free(to_mdev(xrcd->device)->dev, to_mxrcd(xrcd)->xrcdn);
-	kfree(xrcd);
-	return 0;
 }
 static int add_gid_entry(struct ib_qp *ibqp, union ib_gid *gid)
 static const struct ib_device_ops mlx4_ib_dev_xrc_ops = {
 	.alloc_xrcd = mlx4_ib_alloc_xrcd,
 	.dealloc_xrcd = mlx4_ib_dealloc_xrcd,
+	INIT_RDMA_OBJ_SIZE(ib_xrcd, mlx4_ib_xrcd, ibxrcd),
 };
 static const struct ib_device_ops mlx4_ib_dev_fs_ops = {

drivers/infiniband/hw/mlx4/mlx4_ib.h+1−1

@@ -729,7 +729,7 @@ struct ib_mw mlx4_ib_alloc_mw(struct ib_pd pd, enum ib_mw_type type, · 728 unmodified lines
729	struct ib_udata *udata);	729	struct ib_udata *udata);
730	int mlx4_ib_dealloc_mw(struct ib_mw *mw);	730	int mlx4_ib_dealloc_mw(struct ib_mw *mw);
731	struct ib_mr mlx4_ib_alloc_mr(struct ib_pd pd, enum ib_mr_type mr_type,	731	struct ib_mr mlx4_ib_alloc_mr(struct ib_pd pd, enum ib_mr_type mr_type,
732	u32 max_num_sg, struct ib_udata *udata);	732	u32 max_num_sg);
733	int mlx4_ib_map_mr_sg(struct ib_mr ibmr, struct scatterlist sg, int sg_nents,	733	int mlx4_ib_map_mr_sg(struct ib_mr ibmr, struct scatterlist sg, int sg_nents,
734	unsigned int *sg_offset);	734	unsigned int *sg_offset);
735	int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);	735	int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);

drivers/infiniband/hw/mlx4/mr.c+1−2

 	mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key;
 	mr->ibmr.length = length;
-	mr->ibmr.iova = virt_addr;
 	mr->ibmr.page_size = 1U << shift;
 	return &mr->ibmr;
 }
 struct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
-			       u32 max_num_sg, struct ib_udata *udata)
+			       u32 max_num_sg)
 {
 	struct mlx4_ib_dev *dev = to_mdev(pd->device);
 	struct mlx4_ib_mr *mr;

drivers/infiniband/hw/mlx5/Makefile+4−2

 mlx5_ib-y := ah.o \
 	     cmd.o \
 	     cong.o \
+	     counters.o \
 	     cq.o \
 	     doorbell.o \
 	     gsi.o \
 mlx5_ib-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += odp.o
 mlx5_ib-$(CONFIG_MLX5_ESWITCH) += ib_rep.o
 mlx5_ib-$(CONFIG_INFINIBAND_USER_ACCESS) += devx.o \
-					    flow.o \
+					    fs.o \
-					    qos.o
+					    qos.o \
+					    std_types.o

drivers/infiniband/hw/mlx5/cmd.c+0−12

@@ -148,18 +148,6 @@ void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, u64 length) · 147 unmodified lines
148	spin_unlock(&dm->lock);	148	spin_unlock(&dm->lock);
149	}	149	}
150		150
151	int mlx5_cmd_query_ext_ppcnt_counters(struct mlx5_core_dev dev, void out)
152	{
153	u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {};
154	int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
155
156	MLX5_SET(ppcnt_reg, in, local_port, 1);
157
158	MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP);
159	return mlx5_core_access_reg(dev, in, sz, out, sz, MLX5_REG_PPCNT,
160	0, 0);
161	}
162
163	void mlx5_cmd_destroy_tir(struct mlx5_core_dev *dev, u32 tirn, u16 uid)	151	void mlx5_cmd_destroy_tir(struct mlx5_core_dev *dev, u32 tirn, u16 uid)
164	{	152	{
165	u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {};	153	u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {};

drivers/infiniband/hw/mlx5/cmd.h+0−1

@@ -41,7 +41,6 @@ int mlx5_cmd_dump_fill_mkey(struct mlx5_core_dev dev, u32 mkey); · 40 unmodified lines
41	int mlx5_cmd_null_mkey(struct mlx5_core_dev dev, u32 null_mkey);	41	int mlx5_cmd_null_mkey(struct mlx5_core_dev dev, u32 null_mkey);
42	int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point,	42	int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point,
43	void *out);	43	void *out);
44	int mlx5_cmd_query_ext_ppcnt_counters(struct mlx5_core_dev dev, void out);
45	int mlx5_cmd_alloc_memic(struct mlx5_dm dm, phys_addr_t addr,	44	int mlx5_cmd_alloc_memic(struct mlx5_dm dm, phys_addr_t addr,
46	u64 length, u32 alignment);	45	u64 length, u32 alignment);
47	void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, u64 length);	46	void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, u64 length);

drivers/infiniband/hw/mlx5/counters.c+709−0

@@ -0,0 +1,709 @@
	1	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
	2	/*
	3	* Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
	4	*/
	5
	6	#include "mlx5_ib.h"
	7	#include <linux/mlx5/eswitch.h>
	8	#include "counters.h"
	9	#include "ib_rep.h"
	10	#include "qp.h"
	11
	12	struct mlx5_ib_counter {
	13	const char *name;
	14	size_t offset;
	15	};
	16
	17	#define INIT_Q_COUNTER(_name) \
	18	{ .name = #_name, .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)}
	19
	20	static const struct mlx5_ib_counter basic_q_cnts[] = {
	21	INIT_Q_COUNTER(rx_write_requests),
	22	INIT_Q_COUNTER(rx_read_requests),
	23	INIT_Q_COUNTER(rx_atomic_requests),
	24	INIT_Q_COUNTER(out_of_buffer),
	25	};
	26
	27	static const struct mlx5_ib_counter out_of_seq_q_cnts[] = {
	28	INIT_Q_COUNTER(out_of_sequence),
	29	};
	30
	31	static const struct mlx5_ib_counter retrans_q_cnts[] = {
	32	INIT_Q_COUNTER(duplicate_request),
	33	INIT_Q_COUNTER(rnr_nak_retry_err),
	34	INIT_Q_COUNTER(packet_seq_err),
	35	INIT_Q_COUNTER(implied_nak_seq_err),
	36	INIT_Q_COUNTER(local_ack_timeout_err),
	37	};
	38
	39	#define INIT_CONG_COUNTER(_name) \
	40	{ .name = #_name, .offset = \
	41	MLX5_BYTE_OFF(query_cong_statistics_out, _name ## _high)}
	42
	43	static const struct mlx5_ib_counter cong_cnts[] = {
	44	INIT_CONG_COUNTER(rp_cnp_ignored),
	45	INIT_CONG_COUNTER(rp_cnp_handled),
	46	INIT_CONG_COUNTER(np_ecn_marked_roce_packets),
	47	INIT_CONG_COUNTER(np_cnp_sent),
	48	};
	49
	50	static const struct mlx5_ib_counter extended_err_cnts[] = {
	51	INIT_Q_COUNTER(resp_local_length_error),
	52	INIT_Q_COUNTER(resp_cqe_error),
	53	INIT_Q_COUNTER(req_cqe_error),
	54	INIT_Q_COUNTER(req_remote_invalid_request),
	55	INIT_Q_COUNTER(req_remote_access_errors),
	56	INIT_Q_COUNTER(resp_remote_access_errors),
	57	INIT_Q_COUNTER(resp_cqe_flush_error),
	58	INIT_Q_COUNTER(req_cqe_flush_error),
	59	};
	60
	61	static const struct mlx5_ib_counter roce_accl_cnts[] = {
	62	INIT_Q_COUNTER(roce_adp_retrans),
	63	INIT_Q_COUNTER(roce_adp_retrans_to),
	64	INIT_Q_COUNTER(roce_slow_restart),
	65	INIT_Q_COUNTER(roce_slow_restart_cnps),
	66	INIT_Q_COUNTER(roce_slow_restart_trans),
	67	};
	68
	69	#define INIT_EXT_PPCNT_COUNTER(_name) \
	70	{ .name = #_name, .offset = \
	71	MLX5_BYTE_OFF(ppcnt_reg, \
	72	counter_set.eth_extended_cntrs_grp_data_layout._name##_high)}
	73
	74	static const struct mlx5_ib_counter ext_ppcnt_cnts[] = {
	75	INIT_EXT_PPCNT_COUNTER(rx_icrc_encapsulated),
	76	};
	77
	78	static int mlx5_ib_read_counters(struct ib_counters *counters,
	79	struct ib_counters_read_attr *read_attr,
	80	struct uverbs_attr_bundle *attrs)
	81	{
	82	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
	83	struct mlx5_read_counters_attr mread_attr = {};
	84	struct mlx5_ib_flow_counters_desc *desc;
	85	int ret, i;
	86
	87	mutex_lock(&mcounters->mcntrs_mutex);
	88	if (mcounters->cntrs_max_index > read_attr->ncounters) {
	89	ret = -EINVAL;
	90	goto err_bound;
	91	}
	92
	93	mread_attr.out = kcalloc(mcounters->counters_num, sizeof(u64),
	94	GFP_KERNEL);
	95	if (!mread_attr.out) {
	96	ret = -ENOMEM;
	97	goto err_bound;
	98	}
	99
	100	mread_attr.hw_cntrs_hndl = mcounters->hw_cntrs_hndl;
	101	mread_attr.flags = read_attr->flags;
	102	ret = mcounters->read_counters(counters->device, &mread_attr);
	103	if (ret)
	104	goto err_read;
	105
	106	/* do the pass over the counters data array to assign according to the
	107	* descriptions and indexing pairs
	108	*/
	109	desc = mcounters->counters_data;
	110	for (i = 0; i < mcounters->ncounters; i++)
	111	read_attr->counters_buff[desc[i].index] += mread_attr.out[desc[i].description];
	112
	113	err_read:
	114	kfree(mread_attr.out);
	115	err_bound:
	116	mutex_unlock(&mcounters->mcntrs_mutex);
	117	return ret;
	118	}
	119
	120	static void mlx5_ib_destroy_counters(struct ib_counters *counters)
	121	{
	122	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
	123
	124	mlx5_ib_counters_clear_description(counters);
	125	if (mcounters->hw_cntrs_hndl)
	126	mlx5_fc_destroy(to_mdev(counters->device)->mdev,
	127	mcounters->hw_cntrs_hndl);
	128	}
	129
	130	static int mlx5_ib_create_counters(struct ib_counters *counters,
	131	struct uverbs_attr_bundle *attrs)
	132	{
	133	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
	134
	135	mutex_init(&mcounters->mcntrs_mutex);
	136	return 0;
	137	}
	138
	139
	140	static bool is_mdev_switchdev_mode(const struct mlx5_core_dev *mdev)
	141	{
	142	return MLX5_ESWITCH_MANAGER(mdev) &&
	143	mlx5_ib_eswitch_mode(mdev->priv.eswitch) ==
	144	MLX5_ESWITCH_OFFLOADS;
	145	}
	146
	147	static const struct mlx5_ib_counters get_counters(struct mlx5_ib_dev dev,
	148	u8 port_num)
	149	{
	150	return is_mdev_switchdev_mode(dev->mdev) ? &dev->port[0].cnts :
	151	&dev->port[port_num].cnts;
	152	}
	153
	154	/**
	155	* mlx5_ib_get_counters_id - Returns counters id to use for device+port
	156	* @dev: Pointer to mlx5 IB device
	157	* @port_num: Zero based port number
	158	*
	159	* mlx5_ib_get_counters_id() Returns counters set id to use for given
	160	* device port combination in switchdev and non switchdev mode of the
	161	* parent device.
	162	*/
	163	u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u8 port_num)
	164	{
	165	const struct mlx5_ib_counters *cnts = get_counters(dev, port_num);
	166
	167	return cnts->set_id;
	168	}
	169
	170	static struct rdma_hw_stats mlx5_ib_alloc_hw_stats(struct ib_device ibdev,
	171	u8 port_num)
	172	{
	173	struct mlx5_ib_dev *dev = to_mdev(ibdev);
	174	const struct mlx5_ib_counters *cnts;
	175	bool is_switchdev = is_mdev_switchdev_mode(dev->mdev);
	176
	177	if ((is_switchdev && port_num) \|\| (!is_switchdev && !port_num))
	178	return NULL;
	179
	180	cnts = get_counters(dev, port_num - 1);
	181
	182	return rdma_alloc_hw_stats_struct(cnts->names,
	183	cnts->num_q_counters +
	184	cnts->num_cong_counters +
	185	cnts->num_ext_ppcnt_counters,
	186	RDMA_HW_STATS_DEFAULT_LIFESPAN);
	187	}
	188
	189	static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev,
	190	const struct mlx5_ib_counters *cnts,
	191	struct rdma_hw_stats *stats,
	192	u16 set_id)
	193	{
	194	u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {};
	195	u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {};
	196	__be32 val;
	197	int ret, i;
	198
	199	MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER);
	200	MLX5_SET(query_q_counter_in, in, counter_set_id, set_id);
	201	ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out);
	202	if (ret)
	203	return ret;
	204
	205	for (i = 0; i < cnts->num_q_counters; i++) {
	206	val = (__be32 )((void *)out + cnts->offsets[i]);
	207	stats->value[i] = (u64)be32_to_cpu(val);
	208	}
	209
	210	return 0;
	211	}
	212
	213	static int mlx5_ib_query_ext_ppcnt_counters(struct mlx5_ib_dev *dev,
	214	const struct mlx5_ib_counters *cnts,
	215	struct rdma_hw_stats *stats)
	216	{
	217	int offset = cnts->num_q_counters + cnts->num_cong_counters;
	218	u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {};
	219	int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
	220	int ret, i;
	221	void *out;
	222
	223	out = kvzalloc(sz, GFP_KERNEL);
	224	if (!out)
	225	return -ENOMEM;
	226
	227	MLX5_SET(ppcnt_reg, in, local_port, 1);
	228	MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP);
	229	ret = mlx5_core_access_reg(dev->mdev, in, sz, out, sz, MLX5_REG_PPCNT,
	230	0, 0);
	231	if (ret)
	232	goto free;
	233
	234	for (i = 0; i < cnts->num_ext_ppcnt_counters; i++)
	235	stats->value[i + offset] =
	236	be64_to_cpup((__be64 *)(out +
	237	cnts->offsets[i + offset]));
	238	free:
	239	kvfree(out);
	240	return ret;
	241	}
	242
	243	static int mlx5_ib_get_hw_stats(struct ib_device *ibdev,
	244	struct rdma_hw_stats *stats,
	245	u8 port_num, int index)
	246	{
	247	struct mlx5_ib_dev *dev = to_mdev(ibdev);
	248	const struct mlx5_ib_counters *cnts = get_counters(dev, port_num - 1);
	249	struct mlx5_core_dev *mdev;
	250	int ret, num_counters;
	251	u8 mdev_port_num;
	252
	253	if (!stats)
	254	return -EINVAL;
	255
	256	num_counters = cnts->num_q_counters +
	257	cnts->num_cong_counters +
	258	cnts->num_ext_ppcnt_counters;
	259
	260	/* q_counters are per IB device, query the master mdev */
	261	ret = mlx5_ib_query_q_counters(dev->mdev, cnts, stats, cnts->set_id);
	262	if (ret)
	263	return ret;
	264
	265	if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
	266	ret = mlx5_ib_query_ext_ppcnt_counters(dev, cnts, stats);
	267	if (ret)
	268	return ret;
	269	}
	270
	271	if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
	272	mdev = mlx5_ib_get_native_port_mdev(dev, port_num,
	273	&mdev_port_num);
	274	if (!mdev) {
	275	/* If port is not affiliated yet, its in down state
	276	* which doesn't have any counters yet, so it would be
	277	* zero. So no need to read from the HCA.
	278	*/
	279	goto done;
	280	}
	281	ret = mlx5_lag_query_cong_counters(dev->mdev,
	282	stats->value +
	283	cnts->num_q_counters,
	284	cnts->num_cong_counters,
	285	cnts->offsets +
	286	cnts->num_q_counters);
	287
	288	mlx5_ib_put_native_port_mdev(dev, port_num);
	289	if (ret)
	290	return ret;
	291	}
	292
	293	done:
	294	return num_counters;
	295	}
	296
	297	static struct rdma_hw_stats *
	298	mlx5_ib_counter_alloc_stats(struct rdma_counter *counter)
	299	{
	300	struct mlx5_ib_dev *dev = to_mdev(counter->device);
	301	const struct mlx5_ib_counters *cnts =
	302	get_counters(dev, counter->port - 1);
	303
	304	return rdma_alloc_hw_stats_struct(cnts->names,
	305	cnts->num_q_counters +
	306	cnts->num_cong_counters +
	307	cnts->num_ext_ppcnt_counters,
	308	RDMA_HW_STATS_DEFAULT_LIFESPAN);
	309	}
	310
	311	static int mlx5_ib_counter_update_stats(struct rdma_counter *counter)
	312	{
	313	struct mlx5_ib_dev *dev = to_mdev(counter->device);
	314	const struct mlx5_ib_counters *cnts =
	315	get_counters(dev, counter->port - 1);
	316
	317	return mlx5_ib_query_q_counters(dev->mdev, cnts,
	318	counter->stats, counter->id);
	319	}
	320
	321	static int mlx5_ib_counter_dealloc(struct rdma_counter *counter)
	322	{
	323	struct mlx5_ib_dev *dev = to_mdev(counter->device);
	324	u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
	325
	326	if (!counter->id)
	327	return 0;
	328
	329	MLX5_SET(dealloc_q_counter_in, in, opcode,
	330	MLX5_CMD_OP_DEALLOC_Q_COUNTER);
	331	MLX5_SET(dealloc_q_counter_in, in, counter_set_id, counter->id);
	332	return mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in);
	333	}
	334
	335	static int mlx5_ib_counter_bind_qp(struct rdma_counter *counter,
	336	struct ib_qp *qp)
	337	{
	338	struct mlx5_ib_dev *dev = to_mdev(qp->device);
	339	int err;
	340
	341	if (!counter->id) {
	342	u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
	343	u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
	344
	345	MLX5_SET(alloc_q_counter_in, in, opcode,
	346	MLX5_CMD_OP_ALLOC_Q_COUNTER);
	347	MLX5_SET(alloc_q_counter_in, in, uid, MLX5_SHARED_RESOURCE_UID);
	348	err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out);
	349	if (err)
	350	return err;
	351	counter->id =
	352	MLX5_GET(alloc_q_counter_out, out, counter_set_id);
	353	}
	354
	355	err = mlx5_ib_qp_set_counter(qp, counter);
	356	if (err)
	357	goto fail_set_counter;
	358
	359	return 0;
	360
	361	fail_set_counter:
	362	mlx5_ib_counter_dealloc(counter);
	363	counter->id = 0;
	364
	365	return err;
	366	}
	367
	368	static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp)
	369	{
	370	return mlx5_ib_qp_set_counter(qp, NULL);
	371	}
	372
	373
	374	static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev,
	375	const char **names,
	376	size_t *offsets)
	377	{
	378	int i;
	379	int j = 0;
	380
	381	for (i = 0; i < ARRAY_SIZE(basic_q_cnts); i++, j++) {
	382	names[j] = basic_q_cnts[i].name;
	383	offsets[j] = basic_q_cnts[i].offset;
	384	}
	385
	386	if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) {
	387	for (i = 0; i < ARRAY_SIZE(out_of_seq_q_cnts); i++, j++) {
	388	names[j] = out_of_seq_q_cnts[i].name;
	389	offsets[j] = out_of_seq_q_cnts[i].offset;
	390	}
	391	}
	392
	393	if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) {
	394	for (i = 0; i < ARRAY_SIZE(retrans_q_cnts); i++, j++) {
	395	names[j] = retrans_q_cnts[i].name;
	396	offsets[j] = retrans_q_cnts[i].offset;
	397	}
	398	}
	399
	400	if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) {
	401	for (i = 0; i < ARRAY_SIZE(extended_err_cnts); i++, j++) {
	402	names[j] = extended_err_cnts[i].name;
	403	offsets[j] = extended_err_cnts[i].offset;
	404	}
	405	}
	406
	407	if (MLX5_CAP_GEN(dev->mdev, roce_accl)) {
	408	for (i = 0; i < ARRAY_SIZE(roce_accl_cnts); i++, j++) {
	409	names[j] = roce_accl_cnts[i].name;
	410	offsets[j] = roce_accl_cnts[i].offset;
	411	}
	412	}
	413
	414	if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
	415	for (i = 0; i < ARRAY_SIZE(cong_cnts); i++, j++) {
	416	names[j] = cong_cnts[i].name;
	417	offsets[j] = cong_cnts[i].offset;
	418	}
	419	}
	420
	421	if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
	422	for (i = 0; i < ARRAY_SIZE(ext_ppcnt_cnts); i++, j++) {
	423	names[j] = ext_ppcnt_cnts[i].name;
	424	offsets[j] = ext_ppcnt_cnts[i].offset;
	425	}
	426	}
	427	}
	428
	429
	430	static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev,
	431	struct mlx5_ib_counters *cnts)
	432	{
	433	u32 num_counters;
	434
	435	num_counters = ARRAY_SIZE(basic_q_cnts);
	436
	437	if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt))
	438	num_counters += ARRAY_SIZE(out_of_seq_q_cnts);
	439
	440	if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters))
	441	num_counters += ARRAY_SIZE(retrans_q_cnts);
	442
	443	if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters))
	444	num_counters += ARRAY_SIZE(extended_err_cnts);
	445
	446	if (MLX5_CAP_GEN(dev->mdev, roce_accl))
	447	num_counters += ARRAY_SIZE(roce_accl_cnts);
	448
	449	cnts->num_q_counters = num_counters;
	450
	451	if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
	452	cnts->num_cong_counters = ARRAY_SIZE(cong_cnts);
	453	num_counters += ARRAY_SIZE(cong_cnts);
	454	}
	455	if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
	456	cnts->num_ext_ppcnt_counters = ARRAY_SIZE(ext_ppcnt_cnts);
	457	num_counters += ARRAY_SIZE(ext_ppcnt_cnts);
	458	}
	459	cnts->names = kcalloc(num_counters, sizeof(cnts->names), GFP_KERNEL);
	460	if (!cnts->names)
	461	return -ENOMEM;
	462
	463	cnts->offsets = kcalloc(num_counters,
	464	sizeof(cnts->offsets), GFP_KERNEL);
	465	if (!cnts->offsets)
	466	goto err_names;
	467
	468	return 0;
	469
	470	err_names:
	471	kfree(cnts->names);
	472	cnts->names = NULL;
	473	return -ENOMEM;
	474	}
	475
	476	static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev)
	477	{
	478	u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
	479	int num_cnt_ports;
	480	int i;
	481
	482	num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports;
	483
	484	MLX5_SET(dealloc_q_counter_in, in, opcode,
	485	MLX5_CMD_OP_DEALLOC_Q_COUNTER);
	486
	487	for (i = 0; i < num_cnt_ports; i++) {
	488	if (dev->port[i].cnts.set_id) {
	489	MLX5_SET(dealloc_q_counter_in, in, counter_set_id,
	490	dev->port[i].cnts.set_id);
	491	mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in);
	492	}
	493	kfree(dev->port[i].cnts.names);
	494	kfree(dev->port[i].cnts.offsets);
	495	}
	496	}
	497
	498	static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev)
	499	{
	500	u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
	501	u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
	502	int num_cnt_ports;
	503	int err = 0;
	504	int i;
	505	bool is_shared;
	506
	507	MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER);
	508	is_shared = MLX5_CAP_GEN(dev->mdev, log_max_uctx) != 0;
	509	num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports;
	510
	511	for (i = 0; i < num_cnt_ports; i++) {
	512	err = __mlx5_ib_alloc_counters(dev, &dev->port[i].cnts);
	513	if (err)
	514	goto err_alloc;
	515
	516	mlx5_ib_fill_counters(dev, dev->port[i].cnts.names,
	517	dev->port[i].cnts.offsets);
	518
	519	MLX5_SET(alloc_q_counter_in, in, uid,
	520	is_shared ? MLX5_SHARED_RESOURCE_UID : 0);
	521
	522	err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out);
	523	if (err) {
	524	mlx5_ib_warn(dev,
	525	"couldn't allocate queue counter for port %d, err %d\n",
	526	i + 1, err);
	527	goto err_alloc;
	528	}
	529
	530	dev->port[i].cnts.set_id =
	531	MLX5_GET(alloc_q_counter_out, out, counter_set_id);
	532	}
	533	return 0;
	534
	535	err_alloc:
	536	mlx5_ib_dealloc_counters(dev);
	537	return err;
	538	}
	539
	540	static int read_flow_counters(struct ib_device *ibdev,
	541	struct mlx5_read_counters_attr *read_attr)
	542	{
	543	struct mlx5_fc *fc = read_attr->hw_cntrs_hndl;
	544	struct mlx5_ib_dev *dev = to_mdev(ibdev);
	545
	546	return mlx5_fc_query(dev->mdev, fc,
	547	&read_attr->out[IB_COUNTER_PACKETS],
	548	&read_attr->out[IB_COUNTER_BYTES]);
	549	}
	550
	551	/* flow counters currently expose two counters packets and bytes */
	552	#define FLOW_COUNTERS_NUM 2
	553	static int counters_set_description(
	554	struct ib_counters *counters, enum mlx5_ib_counters_type counters_type,
	555	struct mlx5_ib_flow_counters_desc *desc_data, u32 ncounters)
	556	{
	557	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
	558	u32 cntrs_max_index = 0;
	559	int i;
	560
	561	if (counters_type != MLX5_IB_COUNTERS_FLOW)
	562	return -EINVAL;
	563
	564	/* init the fields for the object */
	565	mcounters->type = counters_type;
	566	mcounters->read_counters = read_flow_counters;
	567	mcounters->counters_num = FLOW_COUNTERS_NUM;
	568	mcounters->ncounters = ncounters;
	569	/* each counter entry have both description and index pair */
	570	for (i = 0; i < ncounters; i++) {
	571	if (desc_data[i].description > IB_COUNTER_BYTES)
	572	return -EINVAL;
	573
	574	if (cntrs_max_index <= desc_data[i].index)
	575	cntrs_max_index = desc_data[i].index + 1;
	576	}
	577
	578	mutex_lock(&mcounters->mcntrs_mutex);
	579	mcounters->counters_data = desc_data;
	580	mcounters->cntrs_max_index = cntrs_max_index;
	581	mutex_unlock(&mcounters->mcntrs_mutex);
	582
	583	return 0;
	584	}
	585
	586	#define MAX_COUNTERS_NUM (USHRT_MAX / (sizeof(u32) * 2))
	587	int mlx5_ib_flow_counters_set_data(struct ib_counters *ibcounters,
	588	struct mlx5_ib_create_flow *ucmd)
	589	{
	590	struct mlx5_ib_mcounters *mcounters = to_mcounters(ibcounters);
	591	struct mlx5_ib_flow_counters_data *cntrs_data = NULL;
	592	struct mlx5_ib_flow_counters_desc *desc_data = NULL;
	593	bool hw_hndl = false;
	594	int ret = 0;
	595
	596	if (ucmd && ucmd->ncounters_data != 0) {
	597	cntrs_data = ucmd->data;
	598	if (cntrs_data->ncounters > MAX_COUNTERS_NUM)
	599	return -EINVAL;
	600
	601	desc_data = kcalloc(cntrs_data->ncounters,
	602	sizeof(*desc_data),
	603	GFP_KERNEL);
	604	if (!desc_data)
	605	return -ENOMEM;
	606
	607	if (copy_from_user(desc_data,
	608	u64_to_user_ptr(cntrs_data->counters_data),
	609	sizeof(desc_data) cntrs_data->ncounters)) {
	610	ret = -EFAULT;
	611	goto free;
	612	}
	613	}
	614
	615	if (!mcounters->hw_cntrs_hndl) {
	616	mcounters->hw_cntrs_hndl = mlx5_fc_create(
	617	to_mdev(ibcounters->device)->mdev, false);
	618	if (IS_ERR(mcounters->hw_cntrs_hndl)) {
	619	ret = PTR_ERR(mcounters->hw_cntrs_hndl);
	620	goto free;
	621	}
	622	hw_hndl = true;
	623	}
	624
	625	if (desc_data) {
	626	/* counters already bound to at least one flow */
	627	if (mcounters->cntrs_max_index) {
	628	ret = -EINVAL;
	629	goto free_hndl;
	630	}
	631
	632	ret = counters_set_description(ibcounters,
	633	MLX5_IB_COUNTERS_FLOW,
	634	desc_data,
	635	cntrs_data->ncounters);
	636	if (ret)
	637	goto free_hndl;
	638
	639	} else if (!mcounters->cntrs_max_index) {
	640	/* counters not bound yet, must have udata passed */
	641	ret = -EINVAL;
	642	goto free_hndl;
	643	}
	644
	645	return 0;
	646
	647	free_hndl:
	648	if (hw_hndl) {
	649	mlx5_fc_destroy(to_mdev(ibcounters->device)->mdev,
	650	mcounters->hw_cntrs_hndl);
	651	mcounters->hw_cntrs_hndl = NULL;
	652	}
	653	free:
	654	kfree(desc_data);
	655	return ret;
	656	}
	657
	658	void mlx5_ib_counters_clear_description(struct ib_counters *counters)
	659	{
	660	struct mlx5_ib_mcounters *mcounters;
	661
	662	if (!counters \|\| atomic_read(&counters->usecnt) != 1)
	663	return;
	664
	665	mcounters = to_mcounters(counters);
	666
	667	mutex_lock(&mcounters->mcntrs_mutex);
	668	kfree(mcounters->counters_data);
	669	mcounters->counters_data = NULL;
	670	mcounters->cntrs_max_index = 0;
	671	mutex_unlock(&mcounters->mcntrs_mutex);
	672	}
	673
	674	static const struct ib_device_ops hw_stats_ops = {
	675	.alloc_hw_stats = mlx5_ib_alloc_hw_stats,
	676	.get_hw_stats = mlx5_ib_get_hw_stats,
	677	.counter_bind_qp = mlx5_ib_counter_bind_qp,
	678	.counter_unbind_qp = mlx5_ib_counter_unbind_qp,
	679	.counter_dealloc = mlx5_ib_counter_dealloc,
	680	.counter_alloc_stats = mlx5_ib_counter_alloc_stats,
	681	.counter_update_stats = mlx5_ib_counter_update_stats,
	682	};
	683
	684	static const struct ib_device_ops counters_ops = {
	685	.create_counters = mlx5_ib_create_counters,
	686	.destroy_counters = mlx5_ib_destroy_counters,
	687	.read_counters = mlx5_ib_read_counters,
	688
	689	INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs),
	690	};
	691
	692	int mlx5_ib_counters_init(struct mlx5_ib_dev *dev)
	693	{
	694	ib_set_device_ops(&dev->ib_dev, &counters_ops);
	695
	696	if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
	697	return 0;
	698
	699	ib_set_device_ops(&dev->ib_dev, &hw_stats_ops);
	700	return mlx5_ib_alloc_counters(dev);
	701	}
	702
	703	void mlx5_ib_counters_cleanup(struct mlx5_ib_dev *dev)
	704	{
	705	if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
	706	return;
	707
	708	mlx5_ib_dealloc_counters(dev);
	709	}

drivers/infiniband/hw/mlx5/counters.h+17−0

@@ -0,0 +1,17 @@
	1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
	2	/*
	3	* Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
	4	*/
	5
	6	#ifndef _MLX5_IB_COUNTERS_H
	7	#define _MLX5_IB_COUNTERS_H
	8
	9	#include "mlx5_ib.h"
	10
	11	int mlx5_ib_counters_init(struct mlx5_ib_dev *dev);
	12	void mlx5_ib_counters_cleanup(struct mlx5_ib_dev *dev);
	13	void mlx5_ib_counters_clear_description(struct ib_counters *counters);
	14	int mlx5_ib_flow_counters_set_data(struct ib_counters *ibcounters,
	15	struct mlx5_ib_create_flow *ucmd);
	16	u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u8 port_num);
	17	#endif /* _MLX5_IB_COUNTERS_H */

drivers/infiniband/hw/mlx5/devx.c+28−74

 #include <linux/mlx5/driver.h>
 #include <linux/mlx5/fs.h>
 #include "mlx5_ib.h"
+#include "devx.h"
 #include "qp.h"
 #include <linux/xarray.h>
 	u8 is_destroyed:1;
 };
-#define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in)
-struct devx_obj {
-	struct mlx5_ib_dev	*ib_dev;
-	u64			obj_id;
-	u32			dinlen; /* destroy inbox length */
-	u32			dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW];
-	u32			flags;
-	union {
-		struct mlx5_ib_devx_mr	devx_mr;
-		struct mlx5_core_dct	core_dct;
-		struct mlx5_core_cq	core_cq;
-		u32			flow_counter_bulk_size;
-	};
-	struct list_head event_sub; /* holds devx_event_subscription entries */
-};
 struct devx_umem {
 	struct mlx5_core_dev		*mdev;
 	struct ib_umem			*umem;
 	mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
 }
-bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type)
-{
-	struct devx_obj *devx_obj = obj;
-	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
-	switch (opcode) {
-	case MLX5_CMD_OP_DESTROY_TIR:
-		*dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
-		*dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox,
-				    obj_id);
-		return true;
-	case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
-		*dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
-		*dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox,
-				    table_id);
-		return true;
-	default:
-		return false;
-	}
-}
-bool mlx5_ib_devx_is_flow_counter(void *obj, u32 offset, u32 *counter_id)
-{
-	struct devx_obj *devx_obj = obj;
-	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
-	if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) {
-		if (offset && offset >= devx_obj->flow_counter_bulk_size)
-			return false;
-		*counter_id = MLX5_GET(dealloc_flow_counter_in,
-				       devx_obj->dinbox,
-				       flow_counter_id);
-		*counter_id += offset;
-		return true;
-	}
-	return false;
-}
 static bool is_legacy_unaffiliated_event_num(u16 event_num)
 {
 	switch (event_num) {
 	return NOTIFY_OK;
 }
-void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev)
+int mlx5_ib_devx_init(struct mlx5_ib_dev *dev)
 {
 	struct mlx5_devx_event_table *table = &dev->devx_event_table;
+	int uid;
-	xa_init(&table->event_xa);
+	uid = mlx5_ib_devx_create(dev, false);
-	mutex_init(&table->event_xa_lock);
+	if (uid > 0) {
-	MLX5_NB_INIT(&table->devx_nb, devx_event_notifier, NOTIFY_ANY);
+		dev->devx_whitelist_uid = uid;
-	mlx5_eq_notifier_register(dev->mdev, &table->devx_nb);
+		xa_init(&table->event_xa);
+		mutex_init(&table->event_xa_lock);
+		MLX5_NB_INIT(&table->devx_nb, devx_event_notifier, NOTIFY_ANY);
+		mlx5_eq_notifier_register(dev->mdev, &table->devx_nb);
+	}
+	return 0;
 }
-void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev)
+void mlx5_ib_devx_cleanup(struct mlx5_ib_dev *dev)
 {
 	struct mlx5_devx_event_table *table = &dev->devx_event_table;
 	struct devx_event_subscription *sub, *tmp;
 	void *entry;
 	unsigned long id;
-	mlx5_eq_notifier_unregister(dev->mdev, &table->devx_nb);
+	if (dev->devx_whitelist_uid) {
-	mutex_lock(&dev->devx_event_table.event_xa_lock);
+		mlx5_eq_notifier_unregister(dev->mdev, &table->devx_nb);
-	xa_for_each(&table->event_xa, id, entry) {
+		mutex_lock(&dev->devx_event_table.event_xa_lock);
-		event = entry;
+		xa_for_each(&table->event_xa, id, entry) {
-		list_for_each_entry_safe(sub, tmp, &event->unaffiliated_list,
+			event = entry;
-					 xa_list)
+			list_for_each_entry_safe(
-			devx_cleanup_subscription(dev, sub);
+				sub, tmp, &event->unaffiliated_list, xa_list)
-		kfree(entry);
+				devx_cleanup_subscription(dev, sub);
+			kfree(entry);
+		}
+		mutex_unlock(&dev->devx_event_table.event_xa_lock);
+		xa_destroy(&table->event_xa);
+		mlx5_ib_devx_destroy(dev, dev->devx_whitelist_uid);
 	}
-	mutex_unlock(&dev->devx_event_table.event_xa_lock);
-	xa_destroy(&table->event_xa);
 }
 static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf,

drivers/infiniband/hw/mlx5/devx.h+45−0

@@ -0,0 +1,45 @@
	1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
	2	/*
	3	* Copyright (c) 2019-2020, Mellanox Technologies inc. All rights reserved.
	4	*/
	5
	6	#ifndef _MLX5_IB_DEVX_H
	7	#define _MLX5_IB_DEVX_H
	8
	9	#include "mlx5_ib.h"
	10
	11	#define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in)
	12	struct devx_obj {
	13	struct mlx5_ib_dev *ib_dev;
	14	u64 obj_id;
	15	u32 dinlen; /* destroy inbox length */
	16	u32 dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW];
	17	u32 flags;
	18	union {
	19	struct mlx5_ib_devx_mr devx_mr;
	20	struct mlx5_core_dct core_dct;
	21	struct mlx5_core_cq core_cq;
	22	u32 flow_counter_bulk_size;
	23	};
	24	struct list_head event_sub; /* holds devx_event_subscription entries */
	25	};
	26	#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)
	27	int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user);
	28	void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid);
	29	int mlx5_ib_devx_init(struct mlx5_ib_dev *dev);
	30	void mlx5_ib_devx_cleanup(struct mlx5_ib_dev *dev);
	31	#else
	32	static inline int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user)
	33	{
	34	return -EOPNOTSUPP;
	35	}
	36	static inline void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid) {}
	37	static inline int mlx5_ib_devx_init(struct mlx5_ib_dev *dev)
	38	{
	39	return 0;
	40	}
	41	static inline void mlx5_ib_devx_cleanup(struct mlx5_ib_dev *dev)
	42	{
	43	}
	44	#endif
	45	#endif /* _MLX5_IB_DEVX_H */

drivers/infiniband/hw/mlx5/flow.c+0−765

@@ -1,765 +0,0 @@
1	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2	/*
3	* Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
4	*/
5
6	#include <rdma/ib_user_verbs.h>
7	#include <rdma/ib_verbs.h>
8	#include <rdma/uverbs_types.h>
9	#include <rdma/uverbs_ioctl.h>
10	#include <rdma/uverbs_std_types.h>
11	#include <rdma/mlx5_user_ioctl_cmds.h>
12	#include <rdma/mlx5_user_ioctl_verbs.h>
13	#include <rdma/ib_umem.h>
14	#include <linux/mlx5/driver.h>
15	#include <linux/mlx5/fs.h>
16	#include "mlx5_ib.h"
17
18	#define UVERBS_MODULE_NAME mlx5_ib
19	#include <rdma/uverbs_named_ioctl.h>
20
21	static int
22	mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type,
23	enum mlx5_flow_namespace_type *namespace)
24	{
25	switch (table_type) {
26	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX:
27	*namespace = MLX5_FLOW_NAMESPACE_BYPASS;
28	break;
29	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX:
30	*namespace = MLX5_FLOW_NAMESPACE_EGRESS;
31	break;
32	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB:
33	*namespace = MLX5_FLOW_NAMESPACE_FDB;
34	break;
35	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX:
36	*namespace = MLX5_FLOW_NAMESPACE_RDMA_RX;
37	break;
38	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX:
39	*namespace = MLX5_FLOW_NAMESPACE_RDMA_TX;
40	break;
41	default:
42	return -EINVAL;
43	}
44
45	return 0;
46	}
47
48	static const struct uverbs_attr_spec mlx5_ib_flow_type[] = {
49	[MLX5_IB_FLOW_TYPE_NORMAL] = {
50	.type = UVERBS_ATTR_TYPE_PTR_IN,
51	.u.ptr = {
52	.len = sizeof(u16), /* data is priority */
53	.min_len = sizeof(u16),
54	}
55	},
56	[MLX5_IB_FLOW_TYPE_SNIFFER] = {
57	.type = UVERBS_ATTR_TYPE_PTR_IN,
58	UVERBS_ATTR_NO_DATA(),
59	},
60	[MLX5_IB_FLOW_TYPE_ALL_DEFAULT] = {
61	.type = UVERBS_ATTR_TYPE_PTR_IN,
62	UVERBS_ATTR_NO_DATA(),
63	},
64	[MLX5_IB_FLOW_TYPE_MC_DEFAULT] = {
65	.type = UVERBS_ATTR_TYPE_PTR_IN,
66	UVERBS_ATTR_NO_DATA(),
67	},
68	};
69
70	static int get_dests(struct uverbs_attr_bundle *attrs,
71	struct mlx5_ib_flow_matcher fs_matcher, int dest_id,
72	int dest_type, struct ib_qp qp, u32 flags)
73	{
74	bool dest_devx, dest_qp;
75	void *devx_obj;
76	int err;
77
78	dest_devx = uverbs_attr_is_valid(attrs,
79	MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);
80	dest_qp = uverbs_attr_is_valid(attrs,
81	MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);
82
83	*flags = 0;
84	err = uverbs_get_flags32(flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_FLAGS,
85	MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS \|
86	MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP);
87	if (err)
88	return err;
89
90	/* Both flags are not allowed */
91	if (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS &&
92	*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)
93	return -EINVAL;
94
95	if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) {
96	if (dest_devx && (dest_qp \|\| *flags))
97	return -EINVAL;
98	else if (dest_qp && *flags)
99	return -EINVAL;
100	}
101
102	/* Allow only DEVX object, drop as dest for FDB */
103	if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB && !(dest_devx \|\|
104	(*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)))
105	return -EINVAL;
106
107	/* Allow only DEVX object or QP as dest when inserting to RDMA_RX */
108	if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) &&
109	((!dest_devx && !dest_qp) \|\| (dest_devx && dest_qp)))
110	return -EINVAL;
111
112	*qp = NULL;
113	if (dest_devx) {
114	devx_obj =
115	uverbs_attr_get_obj(attrs,
116	MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);
117
118	/* Verify that the given DEVX object is a flow
119	* steering destination.
120	*/
121	if (!mlx5_ib_devx_is_flow_dest(devx_obj, dest_id, dest_type))
122	return -EINVAL;
123	/* Allow only flow table as dest when inserting to FDB or RDMA_RX */
124	if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB \|\|
125	fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) &&
126	*dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE)
127	return -EINVAL;
128	} else if (dest_qp) {
129	struct mlx5_ib_qp *mqp;
130
131	*qp = uverbs_attr_get_obj(attrs,
132	MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);
133	if (IS_ERR(*qp))
134	return PTR_ERR(*qp);
135
136	if ((*qp)->qp_type != IB_QPT_RAW_PACKET)
137	return -EINVAL;
138
139	mqp = to_mqp(*qp);
140	if (mqp->is_rss)
141	*dest_id = mqp->rss_qp.tirn;
142	else
143	*dest_id = mqp->raw_packet_qp.rq.tirn;
144	*dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
145	} else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) {
146	*dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT;
147	}
148
149	if (*dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR &&
150	fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS)
151	return -EINVAL;
152
153	return 0;
154	}
155
156	#define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2
157	static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)(
158	struct uverbs_attr_bundle *attrs)
159	{
160	struct mlx5_flow_context flow_context = {.flow_tag =
161	MLX5_FS_DEFAULT_FLOW_TAG};
162	u32 *offset_attr, offset = 0, counter_id = 0;
163	int dest_id, dest_type, inlen, len, ret, i;
164	struct mlx5_ib_flow_handler *flow_handler;
165	struct mlx5_ib_flow_matcher *fs_matcher;
166	struct ib_uobject **arr_flow_actions;
167	struct ib_uflow_resources *uflow_res;
168	struct mlx5_flow_act flow_act = {};
169	struct ib_qp *qp = NULL;
170	void devx_obj, cmd_in;
171	struct ib_uobject *uobj;
172	struct mlx5_ib_dev *dev;
173	u32 flags;
174
175	if (!capable(CAP_NET_RAW))
176	return -EPERM;
177
178	fs_matcher = uverbs_attr_get_obj(attrs,
179	MLX5_IB_ATTR_CREATE_FLOW_MATCHER);
180	uobj = uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE);
181	dev = mlx5_udata_to_mdev(&attrs->driver_udata);
182
183	if (get_dests(attrs, fs_matcher, &dest_id, &dest_type, &qp, &flags))
184	return -EINVAL;
185
186	if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS)
187	flow_act.action \|= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS;
188
189	if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)
190	flow_act.action \|= MLX5_FLOW_CONTEXT_ACTION_DROP;
191
192	len = uverbs_attr_get_uobjs_arr(attrs,
193	MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, &arr_flow_actions);
194	if (len) {
195	devx_obj = arr_flow_actions[0]->object;
196
197	if (uverbs_attr_is_valid(attrs,
198	MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET)) {
199
200	int num_offsets = uverbs_attr_ptr_get_array_size(
201	attrs,
202	MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET,
203	sizeof(u32));
204
205	if (num_offsets != 1)
206	return -EINVAL;
207
208	offset_attr = uverbs_attr_get_alloced_ptr(
209	attrs,
210	MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET);
211	offset = *offset_attr;
212	}
213
214	if (!mlx5_ib_devx_is_flow_counter(devx_obj, offset,
215	&counter_id))
216	return -EINVAL;
217
218	flow_act.action \|= MLX5_FLOW_CONTEXT_ACTION_COUNT;
219	}
220
221	cmd_in = uverbs_attr_get_alloced_ptr(
222	attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE);
223	inlen = uverbs_attr_get_len(attrs,
224	MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE);
225
226	uflow_res = flow_resources_alloc(MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS);
227	if (!uflow_res)
228	return -ENOMEM;
229
230	len = uverbs_attr_get_uobjs_arr(attrs,
231	MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, &arr_flow_actions);
232	for (i = 0; i < len; i++) {
233	struct mlx5_ib_flow_action *maction =
234	to_mflow_act(arr_flow_actions[i]->object);
235
236	ret = parse_flow_flow_action(maction, false, &flow_act);
237	if (ret)
238	goto err_out;
239	flow_resources_add(uflow_res, IB_FLOW_SPEC_ACTION_HANDLE,
240	arr_flow_actions[i]->object);
241	}
242
243	ret = uverbs_copy_from(&flow_context.flow_tag, attrs,
244	MLX5_IB_ATTR_CREATE_FLOW_TAG);
245	if (!ret) {
246	if (flow_context.flow_tag >= BIT(24)) {
247	ret = -EINVAL;
248	goto err_out;
249	}
250	flow_context.flags \|= FLOW_CONTEXT_HAS_TAG;
251	}
252
253	flow_handler = mlx5_ib_raw_fs_rule_add(dev, fs_matcher,
254	&flow_context,
255	&flow_act,
256	counter_id,
257	cmd_in, inlen,
258	dest_id, dest_type);
259	if (IS_ERR(flow_handler)) {
260	ret = PTR_ERR(flow_handler);
261	goto err_out;
262	}
263
264	ib_set_flow(uobj, &flow_handler->ibflow, qp, &dev->ib_dev, uflow_res);
265
266	return 0;
267	err_out:
268	ib_uverbs_flow_resources_free(uflow_res);
269	return ret;
270	}
271
272	static int flow_matcher_cleanup(struct ib_uobject *uobject,
273	enum rdma_remove_reason why,
274	struct uverbs_attr_bundle *attrs)
275	{
276	struct mlx5_ib_flow_matcher *obj = uobject->object;
277	int ret;
278
279	ret = ib_destroy_usecnt(&obj->usecnt, why, uobject);
280	if (ret)
281	return ret;
282
283	kfree(obj);
284	return 0;
285	}
286
287	static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs,
288	struct mlx5_ib_flow_matcher *obj)
289	{
290	enum mlx5_ib_uapi_flow_table_type ft_type =
291	MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX;
292	u32 flags;
293	int err;
294
295	/* New users should use MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE and older
296	* users should switch to it. We leave this to not break userspace
297	*/
298	if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE) &&
299	uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS))
300	return -EINVAL;
301
302	if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE)) {
303	err = uverbs_get_const(&ft_type, attrs,
304	MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE);
305	if (err)
306	return err;
307
308	err = mlx5_ib_ft_type_to_namespace(ft_type, &obj->ns_type);
309	if (err)
310	return err;
311
312	return 0;
313	}
314
315	if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) {
316	err = uverbs_get_flags32(&flags, attrs,
317	MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,
318	IB_FLOW_ATTR_FLAGS_EGRESS);
319	if (err)
320	return err;
321
322	if (flags) {
323	mlx5_ib_ft_type_to_namespace(
324	MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX,
325	&obj->ns_type);
326	return 0;
327	}
328	}
329
330	obj->ns_type = MLX5_FLOW_NAMESPACE_BYPASS;
331
332	return 0;
333	}
334
335	static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)(
336	struct uverbs_attr_bundle *attrs)
337	{
338	struct ib_uobject *uobj = uverbs_attr_get_uobject(
339	attrs, MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE);
340	struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
341	struct mlx5_ib_flow_matcher *obj;
342	int err;
343
344	obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL);
345	if (!obj)
346	return -ENOMEM;
347
348	obj->mask_len = uverbs_attr_get_len(
349	attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK);
350	err = uverbs_copy_from(&obj->matcher_mask,
351	attrs,
352	MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK);
353	if (err)
354	goto end;
355
356	obj->flow_type = uverbs_attr_get_enum_id(
357	attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE);
358
359	if (obj->flow_type == MLX5_IB_FLOW_TYPE_NORMAL) {
360	err = uverbs_copy_from(&obj->priority,
361	attrs,
362	MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE);
363	if (err)
364	goto end;
365	}
366
367	err = uverbs_copy_from(&obj->match_criteria_enable,
368	attrs,
369	MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA);
370	if (err)
371	goto end;
372
373	err = mlx5_ib_matcher_ns(attrs, obj);
374	if (err)
375	goto end;
376
377	uobj->object = obj;
378	obj->mdev = dev->mdev;
379	atomic_set(&obj->usecnt, 0);
380	return 0;
381
382	end:
383	kfree(obj);
384	return err;
385	}
386
387	void mlx5_ib_destroy_flow_action_raw(struct mlx5_ib_flow_action *maction)
388	{
389	switch (maction->flow_action_raw.sub_type) {
390	case MLX5_IB_FLOW_ACTION_MODIFY_HEADER:
391	mlx5_modify_header_dealloc(maction->flow_action_raw.dev->mdev,
392	maction->flow_action_raw.modify_hdr);
393	break;
394	case MLX5_IB_FLOW_ACTION_PACKET_REFORMAT:
395	mlx5_packet_reformat_dealloc(maction->flow_action_raw.dev->mdev,
396	maction->flow_action_raw.pkt_reformat);
397	break;
398	case MLX5_IB_FLOW_ACTION_DECAP:
399	break;
400	default:
401	break;
402	}
403	}
404
405	static struct ib_flow_action *
406	mlx5_ib_create_modify_header(struct mlx5_ib_dev *dev,
407	enum mlx5_ib_uapi_flow_table_type ft_type,
408	u8 num_actions, void *in)
409	{
410	enum mlx5_flow_namespace_type namespace;
411	struct mlx5_ib_flow_action *maction;
412	int ret;
413
414	ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace);
415	if (ret)
416	return ERR_PTR(-EINVAL);
417
418	maction = kzalloc(sizeof(*maction), GFP_KERNEL);
419	if (!maction)
420	return ERR_PTR(-ENOMEM);
421
422	maction->flow_action_raw.modify_hdr =
423	mlx5_modify_header_alloc(dev->mdev, namespace, num_actions, in);
424
425	if (IS_ERR(maction->flow_action_raw.modify_hdr)) {
426	ret = PTR_ERR(maction->flow_action_raw.modify_hdr);
427	kfree(maction);
428	return ERR_PTR(ret);
429	}
430	maction->flow_action_raw.sub_type =
431	MLX5_IB_FLOW_ACTION_MODIFY_HEADER;
432	maction->flow_action_raw.dev = dev;
433
434	return &maction->ib_action;
435	}
436
437	static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev *dev)
438	{
439	return MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
440	max_modify_header_actions) \|\|
441	MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev,
442	max_modify_header_actions) \|\|
443	MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev,
444	max_modify_header_actions);
445	}
446
447	static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)(
448	struct uverbs_attr_bundle *attrs)
449	{
450	struct ib_uobject *uobj = uverbs_attr_get_uobject(
451	attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE);
452	struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
453	enum mlx5_ib_uapi_flow_table_type ft_type;
454	struct ib_flow_action *action;
455	int num_actions;
456	void *in;
457	int ret;
458
459	if (!mlx5_ib_modify_header_supported(mdev))
460	return -EOPNOTSUPP;
461
462	in = uverbs_attr_get_alloced_ptr(attrs,
463	MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM);
464
465	num_actions = uverbs_attr_ptr_get_array_size(
466	attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM,
467	MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto));
468	if (num_actions < 0)
469	return num_actions;
470
471	ret = uverbs_get_const(&ft_type, attrs,
472	MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE);
473	if (ret)
474	return ret;
475	action = mlx5_ib_create_modify_header(mdev, ft_type, num_actions, in);
476	if (IS_ERR(action))
477	return PTR_ERR(action);
478
479	uverbs_flow_action_fill_action(action, uobj, &mdev->ib_dev,
480	IB_FLOW_ACTION_UNSPECIFIED);
481
482	return 0;
483	}
484
485	static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev *ibdev,
486	u8 packet_reformat_type,
487	u8 ft_type)
488	{
489	switch (packet_reformat_type) {
490	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL:
491	if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX)
492	return MLX5_CAP_FLOWTABLE(ibdev->mdev,
493	encap_general_header);
494	break;
495	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL:
496	if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX)
497	return MLX5_CAP_FLOWTABLE_NIC_TX(ibdev->mdev,
498	reformat_l2_to_l3_tunnel);
499	break;
500	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2:
501	if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX)
502	return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev,
503	reformat_l3_tunnel_to_l2);
504	break;
505	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2:
506	if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX)
507	return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, decap);
508	break;
509	default:
510	break;
511	}
512
513	return false;
514	}
515
516	static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt, u8 *prm_prt)
517	{
518	switch (dv_prt) {
519	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL:
520	*prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL;
521	break;
522	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2:
523	*prm_prt = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2;
524	break;
525	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL:
526	*prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
527	break;
528	default:
529	return -EINVAL;
530	}
531
532	return 0;
533	}
534
535	static int mlx5_ib_flow_action_create_packet_reformat_ctx(
536	struct mlx5_ib_dev *dev,
537	struct mlx5_ib_flow_action *maction,
538	u8 ft_type, u8 dv_prt,
539	void *in, size_t len)
540	{
541	enum mlx5_flow_namespace_type namespace;
542	u8 prm_prt;
543	int ret;
544
545	ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace);
546	if (ret)
547	return ret;
548
549	ret = mlx5_ib_dv_to_prm_packet_reforamt_type(dv_prt, &prm_prt);
550	if (ret)
551	return ret;
552
553	maction->flow_action_raw.pkt_reformat =
554	mlx5_packet_reformat_alloc(dev->mdev, prm_prt, len,
555	in, namespace);
556	if (IS_ERR(maction->flow_action_raw.pkt_reformat)) {
557	ret = PTR_ERR(maction->flow_action_raw.pkt_reformat);
558	return ret;
559	}
560
561	maction->flow_action_raw.sub_type =
562	MLX5_IB_FLOW_ACTION_PACKET_REFORMAT;
563	maction->flow_action_raw.dev = dev;
564
565	return 0;
566	}
567
568	static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)(
569	struct uverbs_attr_bundle *attrs)
570	{
571	struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
572	MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE);
573	struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
574	enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt;
575	enum mlx5_ib_uapi_flow_table_type ft_type;
576	struct mlx5_ib_flow_action *maction;
577	int ret;
578
579	ret = uverbs_get_const(&ft_type, attrs,
580	MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE);
581	if (ret)
582	return ret;
583
584	ret = uverbs_get_const(&dv_prt, attrs,
585	MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE);
586	if (ret)
587	return ret;
588
589	if (!mlx5_ib_flow_action_packet_reformat_valid(mdev, dv_prt, ft_type))
590	return -EOPNOTSUPP;
591
592	maction = kzalloc(sizeof(*maction), GFP_KERNEL);
593	if (!maction)
594	return -ENOMEM;
595
596	if (dv_prt ==
597	MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2) {
598	maction->flow_action_raw.sub_type =
599	MLX5_IB_FLOW_ACTION_DECAP;
600	maction->flow_action_raw.dev = mdev;
601	} else {
602	void *in;
603	int len;
604
605	in = uverbs_attr_get_alloced_ptr(attrs,
606	MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF);
607	if (IS_ERR(in)) {
608	ret = PTR_ERR(in);
609	goto free_maction;
610	}
611
612	len = uverbs_attr_get_len(attrs,
613	MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF);
614
615	ret = mlx5_ib_flow_action_create_packet_reformat_ctx(mdev,
616	maction, ft_type, dv_prt, in, len);
617	if (ret)
618	goto free_maction;
619	}
620
621	uverbs_flow_action_fill_action(&maction->ib_action, uobj, &mdev->ib_dev,
622	IB_FLOW_ACTION_UNSPECIFIED);
623	return 0;
624
625	free_maction:
626	kfree(maction);
627	return ret;
628	}
629
630	DECLARE_UVERBS_NAMED_METHOD(
631	MLX5_IB_METHOD_CREATE_FLOW,
632	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE,
633	UVERBS_OBJECT_FLOW,
634	UVERBS_ACCESS_NEW,
635	UA_MANDATORY),
636	UVERBS_ATTR_PTR_IN(
637	MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE,
638	UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)),
639	UA_MANDATORY,
640	UA_ALLOC_AND_COPY),
641	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_MATCHER,
642	MLX5_IB_OBJECT_FLOW_MATCHER,
643	UVERBS_ACCESS_READ,
644	UA_MANDATORY),
645	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_QP,
646	UVERBS_OBJECT_QP,
647	UVERBS_ACCESS_READ),
648	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX,
649	MLX5_IB_OBJECT_DEVX_OBJ,
650	UVERBS_ACCESS_READ),
651	UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS,
652	UVERBS_OBJECT_FLOW_ACTION,
653	UVERBS_ACCESS_READ, 1,
654	MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS,
655	UA_OPTIONAL),
656	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_TAG,
657	UVERBS_ATTR_TYPE(u32),
658	UA_OPTIONAL),
659	UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX,
660	MLX5_IB_OBJECT_DEVX_OBJ,
661	UVERBS_ACCESS_READ, 1, 1,
662	UA_OPTIONAL),
663	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET,
664	UVERBS_ATTR_MIN_SIZE(sizeof(u32)),
665	UA_OPTIONAL,
666	UA_ALLOC_AND_COPY),
667	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_FLAGS,
668	enum mlx5_ib_create_flow_flags,
669	UA_OPTIONAL));
670
671	DECLARE_UVERBS_NAMED_METHOD_DESTROY(
672	MLX5_IB_METHOD_DESTROY_FLOW,
673	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE,
674	UVERBS_OBJECT_FLOW,
675	UVERBS_ACCESS_DESTROY,
676	UA_MANDATORY));
677
678	ADD_UVERBS_METHODS(mlx5_ib_fs,
679	UVERBS_OBJECT_FLOW,
680	&UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW),
681	&UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW));
682
683	DECLARE_UVERBS_NAMED_METHOD(
684	MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER,
685	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE,
686	UVERBS_OBJECT_FLOW_ACTION,
687	UVERBS_ACCESS_NEW,
688	UA_MANDATORY),
689	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM,
690	UVERBS_ATTR_MIN_SIZE(MLX5_UN_SZ_BYTES(
691	set_add_copy_action_in_auto)),
692	UA_MANDATORY,
693	UA_ALLOC_AND_COPY),
694	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE,
695	enum mlx5_ib_uapi_flow_table_type,
696	UA_MANDATORY));
697
698	DECLARE_UVERBS_NAMED_METHOD(
699	MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT,
700	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE,
701	UVERBS_OBJECT_FLOW_ACTION,
702	UVERBS_ACCESS_NEW,
703	UA_MANDATORY),
704	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF,
705	UVERBS_ATTR_MIN_SIZE(1),
706	UA_ALLOC_AND_COPY,
707	UA_OPTIONAL),
708	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE,
709	enum mlx5_ib_uapi_flow_action_packet_reformat_type,
710	UA_MANDATORY),
711	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE,
712	enum mlx5_ib_uapi_flow_table_type,
713	UA_MANDATORY));
714
715	ADD_UVERBS_METHODS(
716	mlx5_ib_flow_actions,
717	UVERBS_OBJECT_FLOW_ACTION,
718	&UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER),
719	&UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT));
720
721	DECLARE_UVERBS_NAMED_METHOD(
722	MLX5_IB_METHOD_FLOW_MATCHER_CREATE,
723	UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE,
724	MLX5_IB_OBJECT_FLOW_MATCHER,
725	UVERBS_ACCESS_NEW,
726	UA_MANDATORY),
727	UVERBS_ATTR_PTR_IN(
728	MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK,
729	UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)),
730	UA_MANDATORY),
731	UVERBS_ATTR_ENUM_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE,
732	mlx5_ib_flow_type,
733	UA_MANDATORY),
734	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA,
735	UVERBS_ATTR_TYPE(u8),
736	UA_MANDATORY),
737	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,
738	enum ib_flow_flags,
739	UA_OPTIONAL),
740	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE,
741	enum mlx5_ib_uapi_flow_table_type,
742	UA_OPTIONAL));
743
744	DECLARE_UVERBS_NAMED_METHOD_DESTROY(
745	MLX5_IB_METHOD_FLOW_MATCHER_DESTROY,
746	UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE,
747	MLX5_IB_OBJECT_FLOW_MATCHER,
748	UVERBS_ACCESS_DESTROY,
749	UA_MANDATORY));
750
751	DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER,
752	UVERBS_TYPE_ALLOC_IDR(flow_matcher_cleanup),
753	&UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE),
754	&UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY));
755
756	const struct uapi_definition mlx5_ib_flow_defs[] = {
757	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
758	MLX5_IB_OBJECT_FLOW_MATCHER),
759	UAPI_DEF_CHAIN_OBJ_TREE(
760	UVERBS_OBJECT_FLOW,
761	&mlx5_ib_fs),
762	UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION,
763	&mlx5_ib_flow_actions),
764	{},
765	};

drivers/infiniband/hw/mlx5/fs.c+2516−0

@@ -0,0 +1,2516 @@
	1	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
	2	/*
	3	* Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
	4	*/
	5
	6	#include <rdma/ib_user_verbs.h>
	7	#include <rdma/ib_verbs.h>
	8	#include <rdma/uverbs_types.h>
	9	#include <rdma/uverbs_ioctl.h>
	10	#include <rdma/uverbs_std_types.h>
	11	#include <rdma/mlx5_user_ioctl_cmds.h>
	12	#include <rdma/mlx5_user_ioctl_verbs.h>
	13	#include <rdma/ib_umem.h>
	14	#include <linux/mlx5/driver.h>
	15	#include <linux/mlx5/fs.h>
	16	#include <linux/mlx5/fs_helpers.h>
	17	#include <linux/mlx5/accel.h>
	18	#include <linux/mlx5/eswitch.h>
	19	#include "mlx5_ib.h"
	20	#include "counters.h"
	21	#include "devx.h"
	22	#include "fs.h"
	23
	24	#define UVERBS_MODULE_NAME mlx5_ib
	25	#include <rdma/uverbs_named_ioctl.h>
	26
	27	enum {
	28	MATCH_CRITERIA_ENABLE_OUTER_BIT,
	29	MATCH_CRITERIA_ENABLE_MISC_BIT,
	30	MATCH_CRITERIA_ENABLE_INNER_BIT,
	31	MATCH_CRITERIA_ENABLE_MISC2_BIT
	32	};
	33
	34	#define HEADER_IS_ZERO(match_criteria, headers) \
	35	!(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \
	36	0, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
	37
	38	static u8 get_match_criteria_enable(u32 *match_criteria)
	39	{
	40	u8 match_criteria_enable;
	41
	42	match_criteria_enable =
	43	(!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
	44	MATCH_CRITERIA_ENABLE_OUTER_BIT;
	45	match_criteria_enable \|=
	46	(!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
	47	MATCH_CRITERIA_ENABLE_MISC_BIT;
	48	match_criteria_enable \|=
	49	(!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
	50	MATCH_CRITERIA_ENABLE_INNER_BIT;
	51	match_criteria_enable \|=
	52	(!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
	53	MATCH_CRITERIA_ENABLE_MISC2_BIT;
	54
	55	return match_criteria_enable;
	56	}
	57
	58	static int set_proto(void outer_c, void outer_v, u8 mask, u8 val)
	59	{
	60	u8 entry_mask;
	61	u8 entry_val;
	62	int err = 0;
	63
	64	if (!mask)
	65	goto out;
	66
	67	entry_mask = MLX5_GET(fte_match_set_lyr_2_4, outer_c,
	68	ip_protocol);
	69	entry_val = MLX5_GET(fte_match_set_lyr_2_4, outer_v,
	70	ip_protocol);
	71	if (!entry_mask) {
	72	MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask);
	73	MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val);
	74	goto out;
	75	}
	76	/* Don't override existing ip protocol */
	77	if (mask != entry_mask \|\| val != entry_val)
	78	err = -EINVAL;
	79	out:
	80	return err;
	81	}
	82
	83	static void set_flow_label(void misc_c, void misc_v, u32 mask, u32 val,
	84	bool inner)
	85	{
	86	if (inner) {
	87	MLX5_SET(fte_match_set_misc,
	88	misc_c, inner_ipv6_flow_label, mask);
	89	MLX5_SET(fte_match_set_misc,
	90	misc_v, inner_ipv6_flow_label, val);
	91	} else {
	92	MLX5_SET(fte_match_set_misc,
	93	misc_c, outer_ipv6_flow_label, mask);
	94	MLX5_SET(fte_match_set_misc,
	95	misc_v, outer_ipv6_flow_label, val);
	96	}
	97	}
	98
	99	static void set_tos(void outer_c, void outer_v, u8 mask, u8 val)
	100	{
	101	MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask);
	102	MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val);
	103	MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2);
	104	MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2);
	105	}
	106
	107	static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask)
	108	{
	109	if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) &&
	110	!(field_support & MLX5_FIELD_SUPPORT_MPLS_LABEL))
	111	return -EOPNOTSUPP;
	112
	113	if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) &&
	114	!(field_support & MLX5_FIELD_SUPPORT_MPLS_EXP))
	115	return -EOPNOTSUPP;
	116
	117	if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) &&
	118	!(field_support & MLX5_FIELD_SUPPORT_MPLS_S_BOS))
	119	return -EOPNOTSUPP;
	120
	121	if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) &&
	122	!(field_support & MLX5_FIELD_SUPPORT_MPLS_TTL))
	123	return -EOPNOTSUPP;
	124
	125	return 0;
	126	}
	127
	128	#define LAST_ETH_FIELD vlan_tag
	129	#define LAST_IB_FIELD sl
	130	#define LAST_IPV4_FIELD tos
	131	#define LAST_IPV6_FIELD traffic_class
	132	#define LAST_TCP_UDP_FIELD src_port
	133	#define LAST_TUNNEL_FIELD tunnel_id
	134	#define LAST_FLOW_TAG_FIELD tag_id
	135	#define LAST_DROP_FIELD size
	136	#define LAST_COUNTERS_FIELD counters
	137
	138	/* Field is the last supported field */
	139	#define FIELDS_NOT_SUPPORTED(filter, field)\
	140	memchr_inv((void *)&filter.field +\
	141	sizeof(filter.field), 0,\
	142	sizeof(filter) -\
	143	offsetof(typeof(filter), field) -\
	144	sizeof(filter.field))
	145
	146	int parse_flow_flow_action(struct mlx5_ib_flow_action *maction,
	147	bool is_egress,
	148	struct mlx5_flow_act *action)
	149	{
	150
	151	switch (maction->ib_action.type) {
	152	case IB_FLOW_ACTION_ESP:
	153	if (action->action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT \|
	154	MLX5_FLOW_CONTEXT_ACTION_DECRYPT))
	155	return -EINVAL;
	156	/* Currently only AES_GCM keymat is supported by the driver */
	157	action->esp_id = (uintptr_t)maction->esp_aes_gcm.ctx;
	158	action->action \|= is_egress ?
	159	MLX5_FLOW_CONTEXT_ACTION_ENCRYPT :
	160	MLX5_FLOW_CONTEXT_ACTION_DECRYPT;
	161	return 0;
	162	case IB_FLOW_ACTION_UNSPECIFIED:
	163	if (maction->flow_action_raw.sub_type ==
	164	MLX5_IB_FLOW_ACTION_MODIFY_HEADER) {
	165	if (action->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
	166	return -EINVAL;
	167	action->action \|= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
	168	action->modify_hdr =
	169	maction->flow_action_raw.modify_hdr;
	170	return 0;
	171	}
	172	if (maction->flow_action_raw.sub_type ==
	173	MLX5_IB_FLOW_ACTION_DECAP) {
	174	if (action->action & MLX5_FLOW_CONTEXT_ACTION_DECAP)
	175	return -EINVAL;
	176	action->action \|= MLX5_FLOW_CONTEXT_ACTION_DECAP;
	177	return 0;
	178	}
	179	if (maction->flow_action_raw.sub_type ==
	180	MLX5_IB_FLOW_ACTION_PACKET_REFORMAT) {
	181	if (action->action &
	182	MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT)
	183	return -EINVAL;
	184	action->action \|=
	185	MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
	186	action->pkt_reformat =
	187	maction->flow_action_raw.pkt_reformat;
	188	return 0;
	189	}
	190	fallthrough;
	191	default:
	192	return -EOPNOTSUPP;
	193	}
	194	}
	195
	196	static int parse_flow_attr(struct mlx5_core_dev *mdev,
	197	struct mlx5_flow_spec *spec,
	198	const union ib_flow_spec *ib_spec,
	199	const struct ib_flow_attr *flow_attr,
	200	struct mlx5_flow_act *action, u32 prev_type)
	201	{
	202	struct mlx5_flow_context *flow_context = &spec->flow_context;
	203	u32 *match_c = spec->match_criteria;
	204	u32 *match_v = spec->match_value;
	205	void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c,
	206	misc_parameters);
	207	void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v,
	208	misc_parameters);
	209	void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c,
	210	misc_parameters_2);
	211	void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v,
	212	misc_parameters_2);
	213	void *headers_c;
	214	void *headers_v;
	215	int match_ipv;
	216	int ret;
	217
	218	if (ib_spec->type & IB_FLOW_SPEC_INNER) {
	219	headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
	220	inner_headers);
	221	headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
	222	inner_headers);
	223	match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
	224	ft_field_support.inner_ip_version);
	225	} else {
	226	headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
	227	outer_headers);
	228	headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
	229	outer_headers);
	230	match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
	231	ft_field_support.outer_ip_version);
	232	}
	233
	234	switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) {
	235	case IB_FLOW_SPEC_ETH:
	236	if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD))
	237	return -EOPNOTSUPP;
	238
	239	ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
	240	dmac_47_16),
	241	ib_spec->eth.mask.dst_mac);
	242	ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
	243	dmac_47_16),
	244	ib_spec->eth.val.dst_mac);
	245
	246	ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
	247	smac_47_16),
	248	ib_spec->eth.mask.src_mac);
	249	ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
	250	smac_47_16),
	251	ib_spec->eth.val.src_mac);
	252
	253	if (ib_spec->eth.mask.vlan_tag) {
	254	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
	255	cvlan_tag, 1);
	256	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
	257	cvlan_tag, 1);
	258
	259	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
	260	first_vid, ntohs(ib_spec->eth.mask.vlan_tag));
	261	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
	262	first_vid, ntohs(ib_spec->eth.val.vlan_tag));
	263
	264	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
	265	first_cfi,
	266	ntohs(ib_spec->eth.mask.vlan_tag) >> 12);
	267	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
	268	first_cfi,
	269	ntohs(ib_spec->eth.val.vlan_tag) >> 12);
	270
	271	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
	272	first_prio,
	273	ntohs(ib_spec->eth.mask.vlan_tag) >> 13);
	274	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
	275	first_prio,
	276	ntohs(ib_spec->eth.val.vlan_tag) >> 13);
	277	}
	278	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
	279	ethertype, ntohs(ib_spec->eth.mask.ether_type));
	280	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
	281	ethertype, ntohs(ib_spec->eth.val.ether_type));
	282	break;
	283	case IB_FLOW_SPEC_IPV4:
	284	if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD))
	285	return -EOPNOTSUPP;
	286
	287	if (match_ipv) {
	288	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
	289	ip_version, 0xf);
	290	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
	291	ip_version, MLX5_FS_IPV4_VERSION);
	292	} else {
	293	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
	294	ethertype, 0xffff);
	295	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
	296	ethertype, ETH_P_IP);
	297	}
	298
	299	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
	300	src_ipv4_src_ipv6.ipv4_layout.ipv4),
	301	&ib_spec->ipv4.mask.src_ip,
	302	sizeof(ib_spec->ipv4.mask.src_ip));
	303	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
	304	src_ipv4_src_ipv6.ipv4_layout.ipv4),
	305	&ib_spec->ipv4.val.src_ip,
	306	sizeof(ib_spec->ipv4.val.src_ip));
	307	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
	308	dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
	309	&ib_spec->ipv4.mask.dst_ip,
	310	sizeof(ib_spec->ipv4.mask.dst_ip));
	311	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
	312	dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
	313	&ib_spec->ipv4.val.dst_ip,
	314	sizeof(ib_spec->ipv4.val.dst_ip));
	315
	316	set_tos(headers_c, headers_v,
	317	ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos);
	318
	319	if (set_proto(headers_c, headers_v,
	320	ib_spec->ipv4.mask.proto,
	321	ib_spec->ipv4.val.proto))
	322	return -EINVAL;
	323	break;
	324	case IB_FLOW_SPEC_IPV6:
	325	if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD))
	326	return -EOPNOTSUPP;
	327
	328	if (match_ipv) {
	329	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
	330	ip_version, 0xf);
	331	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
	332	ip_version, MLX5_FS_IPV6_VERSION);
	333	} else {
	334	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
	335	ethertype, 0xffff);
	336	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
	337	ethertype, ETH_P_IPV6);
	338	}
	339
	340	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
	341	src_ipv4_src_ipv6.ipv6_layout.ipv6),
	342	&ib_spec->ipv6.mask.src_ip,
	343	sizeof(ib_spec->ipv6.mask.src_ip));
	344	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
	345	src_ipv4_src_ipv6.ipv6_layout.ipv6),
	346	&ib_spec->ipv6.val.src_ip,
	347	sizeof(ib_spec->ipv6.val.src_ip));
	348	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
	349	dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
	350	&ib_spec->ipv6.mask.dst_ip,
	351	sizeof(ib_spec->ipv6.mask.dst_ip));
	352	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
	353	dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
	354	&ib_spec->ipv6.val.dst_ip,
	355	sizeof(ib_spec->ipv6.val.dst_ip));
	356
	357	set_tos(headers_c, headers_v,
	358	ib_spec->ipv6.mask.traffic_class,
	359	ib_spec->ipv6.val.traffic_class);
	360
	361	if (set_proto(headers_c, headers_v,
	362	ib_spec->ipv6.mask.next_hdr,
	363	ib_spec->ipv6.val.next_hdr))
	364	return -EINVAL;
	365
	366	set_flow_label(misc_params_c, misc_params_v,
	367	ntohl(ib_spec->ipv6.mask.flow_label),
	368	ntohl(ib_spec->ipv6.val.flow_label),
	369	ib_spec->type & IB_FLOW_SPEC_INNER);
	370	break;
	371	case IB_FLOW_SPEC_ESP:
	372	if (ib_spec->esp.mask.seq)
	373	return -EOPNOTSUPP;
	374
	375	MLX5_SET(fte_match_set_misc, misc_params_c, outer_esp_spi,
	376	ntohl(ib_spec->esp.mask.spi));
	377	MLX5_SET(fte_match_set_misc, misc_params_v, outer_esp_spi,
	378	ntohl(ib_spec->esp.val.spi));
	379	break;
	380	case IB_FLOW_SPEC_TCP:
	381	if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
	382	LAST_TCP_UDP_FIELD))
	383	return -EOPNOTSUPP;
	384
	385	if (set_proto(headers_c, headers_v, 0xff, IPPROTO_TCP))
	386	return -EINVAL;
	387
	388	MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport,
	389	ntohs(ib_spec->tcp_udp.mask.src_port));
	390	MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
	391	ntohs(ib_spec->tcp_udp.val.src_port));
	392
	393	MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport,
	394	ntohs(ib_spec->tcp_udp.mask.dst_port));
	395	MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
	396	ntohs(ib_spec->tcp_udp.val.dst_port));
	397	break;
	398	case IB_FLOW_SPEC_UDP:
	399	if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
	400	LAST_TCP_UDP_FIELD))
	401	return -EOPNOTSUPP;
	402
	403	if (set_proto(headers_c, headers_v, 0xff, IPPROTO_UDP))
	404	return -EINVAL;
	405
	406	MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport,
	407	ntohs(ib_spec->tcp_udp.mask.src_port));
	408	MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
	409	ntohs(ib_spec->tcp_udp.val.src_port));
	410
	411	MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport,
	412	ntohs(ib_spec->tcp_udp.mask.dst_port));
	413	MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
	414	ntohs(ib_spec->tcp_udp.val.dst_port));
	415	break;
	416	case IB_FLOW_SPEC_GRE:
	417	if (ib_spec->gre.mask.c_ks_res0_ver)
	418	return -EOPNOTSUPP;
	419
	420	if (set_proto(headers_c, headers_v, 0xff, IPPROTO_GRE))
	421	return -EINVAL;
	422
	423	MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
	424	0xff);
	425	MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
	426	IPPROTO_GRE);
	427
	428	MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol,
	429	ntohs(ib_spec->gre.mask.protocol));
	430	MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol,
	431	ntohs(ib_spec->gre.val.protocol));
	432
	433	memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c,
	434	gre_key.nvgre.hi),
	435	&ib_spec->gre.mask.key,
	436	sizeof(ib_spec->gre.mask.key));
	437	memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v,
	438	gre_key.nvgre.hi),
	439	&ib_spec->gre.val.key,
	440	sizeof(ib_spec->gre.val.key));
	441	break;
	442	case IB_FLOW_SPEC_MPLS:
	443	switch (prev_type) {
	444	case IB_FLOW_SPEC_UDP:
	445	if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
	446	ft_field_support.outer_first_mpls_over_udp),
	447	&ib_spec->mpls.mask.tag))
	448	return -EOPNOTSUPP;
	449
	450	memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
	451	outer_first_mpls_over_udp),
	452	&ib_spec->mpls.val.tag,
	453	sizeof(ib_spec->mpls.val.tag));
	454	memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
	455	outer_first_mpls_over_udp),
	456	&ib_spec->mpls.mask.tag,
	457	sizeof(ib_spec->mpls.mask.tag));
	458	break;
	459	case IB_FLOW_SPEC_GRE:
	460	if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
	461	ft_field_support.outer_first_mpls_over_gre),
	462	&ib_spec->mpls.mask.tag))
	463	return -EOPNOTSUPP;
	464
	465	memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
	466	outer_first_mpls_over_gre),
	467	&ib_spec->mpls.val.tag,
	468	sizeof(ib_spec->mpls.val.tag));
	469	memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
	470	outer_first_mpls_over_gre),
	471	&ib_spec->mpls.mask.tag,
	472	sizeof(ib_spec->mpls.mask.tag));
	473	break;
	474	default:
	475	if (ib_spec->type & IB_FLOW_SPEC_INNER) {
	476	if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
	477	ft_field_support.inner_first_mpls),
	478	&ib_spec->mpls.mask.tag))
	479	return -EOPNOTSUPP;
	480
	481	memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
	482	inner_first_mpls),
	483	&ib_spec->mpls.val.tag,
	484	sizeof(ib_spec->mpls.val.tag));
	485	memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
	486	inner_first_mpls),
	487	&ib_spec->mpls.mask.tag,
	488	sizeof(ib_spec->mpls.mask.tag));
	489	} else {
	490	if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
	491	ft_field_support.outer_first_mpls),
	492	&ib_spec->mpls.mask.tag))
	493	return -EOPNOTSUPP;
	494
	495	memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
	496	outer_first_mpls),
	497	&ib_spec->mpls.val.tag,
	498	sizeof(ib_spec->mpls.val.tag));
	499	memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
	500	outer_first_mpls),
	501	&ib_spec->mpls.mask.tag,
	502	sizeof(ib_spec->mpls.mask.tag));
	503	}
	504	}
	505	break;
	506	case IB_FLOW_SPEC_VXLAN_TUNNEL:
	507	if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask,
	508	LAST_TUNNEL_FIELD))
	509	return -EOPNOTSUPP;
	510
	511	MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni,
	512	ntohl(ib_spec->tunnel.mask.tunnel_id));
	513	MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni,
	514	ntohl(ib_spec->tunnel.val.tunnel_id));
	515	break;
	516	case IB_FLOW_SPEC_ACTION_TAG:
	517	if (FIELDS_NOT_SUPPORTED(ib_spec->flow_tag,
	518	LAST_FLOW_TAG_FIELD))
	519	return -EOPNOTSUPP;
	520	if (ib_spec->flow_tag.tag_id >= BIT(24))
	521	return -EINVAL;
	522
	523	flow_context->flow_tag = ib_spec->flow_tag.tag_id;
	524	flow_context->flags \|= FLOW_CONTEXT_HAS_TAG;
	525	break;
	526	case IB_FLOW_SPEC_ACTION_DROP:
	527	if (FIELDS_NOT_SUPPORTED(ib_spec->drop,
	528	LAST_DROP_FIELD))
	529	return -EOPNOTSUPP;
	530	action->action \|= MLX5_FLOW_CONTEXT_ACTION_DROP;
	531	break;
	532	case IB_FLOW_SPEC_ACTION_HANDLE:
	533	ret = parse_flow_flow_action(to_mflow_act(ib_spec->action.act),
	534	flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS, action);
	535	if (ret)
	536	return ret;
	537	break;
	538	case IB_FLOW_SPEC_ACTION_COUNT:
	539	if (FIELDS_NOT_SUPPORTED(ib_spec->flow_count,
	540	LAST_COUNTERS_FIELD))
	541	return -EOPNOTSUPP;
	542
	543	/* for now support only one counters spec per flow */
	544	if (action->action & MLX5_FLOW_CONTEXT_ACTION_COUNT)
	545	return -EINVAL;
	546
	547	action->counters = ib_spec->flow_count.counters;
	548	action->action \|= MLX5_FLOW_CONTEXT_ACTION_COUNT;
	549	break;
	550	default:
	551	return -EINVAL;
	552	}
	553
	554	return 0;
	555	}
	556
	557	/* If a flow could catch both multicast and unicast packets,
	558	* it won't fall into the multicast flow steering table and this rule
	559	* could steal other multicast packets.
	560	*/
	561	static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr)
	562	{
	563	union ib_flow_spec *flow_spec;
	564
	565	if (ib_attr->type != IB_FLOW_ATTR_NORMAL \|\|
	566	ib_attr->num_of_specs < 1)
	567	return false;
	568
	569	flow_spec = (union ib_flow_spec *)(ib_attr + 1);
	570	if (flow_spec->type == IB_FLOW_SPEC_IPV4) {
	571	struct ib_flow_spec_ipv4 *ipv4_spec;
	572
	573	ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec;
	574	if (ipv4_is_multicast(ipv4_spec->val.dst_ip))
	575	return true;
	576
	577	return false;
	578	}
	579
	580	if (flow_spec->type == IB_FLOW_SPEC_ETH) {
	581	struct ib_flow_spec_eth *eth_spec;
	582
	583	eth_spec = (struct ib_flow_spec_eth *)flow_spec;
	584	return is_multicast_ether_addr(eth_spec->mask.dst_mac) &&
	585	is_multicast_ether_addr(eth_spec->val.dst_mac);
	586	}
	587
	588	return false;
	589	}
	590
	591	enum valid_spec {
	592	VALID_SPEC_INVALID,
	593	VALID_SPEC_VALID,
	594	VALID_SPEC_NA,
	595	};
	596
	597	static enum valid_spec
	598	is_valid_esp_aes_gcm(struct mlx5_core_dev *mdev,
	599	const struct mlx5_flow_spec *spec,
	600	const struct mlx5_flow_act *flow_act,
	601	bool egress)
	602	{
	603	const u32 *match_c = spec->match_criteria;
	604	bool is_crypto =
	605	(flow_act->action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT \|
	606	MLX5_FLOW_CONTEXT_ACTION_DECRYPT));
	607	bool is_ipsec = mlx5_fs_is_ipsec_flow(match_c);
	608	bool is_drop = flow_act->action & MLX5_FLOW_CONTEXT_ACTION_DROP;
	609
	610	/*
	611	* Currently only crypto is supported in egress, when regular egress
	612	* rules would be supported, always return VALID_SPEC_NA.
	613	*/
	614	if (!is_crypto)
	615	return VALID_SPEC_NA;
	616
	617	return is_crypto && is_ipsec &&
	618	(!egress \|\| (!is_drop &&
	619	!(spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG))) ?
	620	VALID_SPEC_VALID : VALID_SPEC_INVALID;
	621	}
	622
	623	static bool is_valid_spec(struct mlx5_core_dev *mdev,
	624	const struct mlx5_flow_spec *spec,
	625	const struct mlx5_flow_act *flow_act,
	626	bool egress)
	627	{
	628	/* We curretly only support ipsec egress flow */
	629	return is_valid_esp_aes_gcm(mdev, spec, flow_act, egress) != VALID_SPEC_INVALID;
	630	}
	631
	632	static bool is_valid_ethertype(struct mlx5_core_dev *mdev,
	633	const struct ib_flow_attr *flow_attr,
	634	bool check_inner)
	635	{
	636	union ib_flow_spec ib_spec = (union ib_flow_spec )(flow_attr + 1);
	637	int match_ipv = check_inner ?
	638	MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
	639	ft_field_support.inner_ip_version) :
	640	MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
	641	ft_field_support.outer_ip_version);
	642	int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0;
	643	bool ipv4_spec_valid, ipv6_spec_valid;
	644	unsigned int ip_spec_type = 0;
	645	bool has_ethertype = false;
	646	unsigned int spec_index;
	647	bool mask_valid = true;
	648	u16 eth_type = 0;
	649	bool type_valid;
	650
	651	/* Validate that ethertype is correct */
	652	for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
	653	if ((ib_spec->type == (IB_FLOW_SPEC_ETH \| inner_bit)) &&
	654	ib_spec->eth.mask.ether_type) {
	655	mask_valid = (ib_spec->eth.mask.ether_type ==
	656	htons(0xffff));
	657	has_ethertype = true;
	658	eth_type = ntohs(ib_spec->eth.val.ether_type);
	659	} else if ((ib_spec->type == (IB_FLOW_SPEC_IPV4 \| inner_bit)) \|\|
	660	(ib_spec->type == (IB_FLOW_SPEC_IPV6 \| inner_bit))) {
	661	ip_spec_type = ib_spec->type;
	662	}
	663	ib_spec = (void *)ib_spec + ib_spec->size;
	664	}
	665
	666	type_valid = (!has_ethertype) \|\| (!ip_spec_type);
	667	if (!type_valid && mask_valid) {
	668	ipv4_spec_valid = (eth_type == ETH_P_IP) &&
	669	(ip_spec_type == (IB_FLOW_SPEC_IPV4 \| inner_bit));
	670	ipv6_spec_valid = (eth_type == ETH_P_IPV6) &&
	671	(ip_spec_type == (IB_FLOW_SPEC_IPV6 \| inner_bit));
	672
	673	type_valid = (ipv4_spec_valid) \|\| (ipv6_spec_valid) \|\|
	674	(((eth_type == ETH_P_MPLS_UC) \|\|
	675	(eth_type == ETH_P_MPLS_MC)) && match_ipv);
	676	}
	677
	678	return type_valid;
	679	}
	680
	681	static bool is_valid_attr(struct mlx5_core_dev *mdev,
	682	const struct ib_flow_attr *flow_attr)
	683	{
	684	return is_valid_ethertype(mdev, flow_attr, false) &&
	685	is_valid_ethertype(mdev, flow_attr, true);
	686	}
	687
	688	static void put_flow_table(struct mlx5_ib_dev *dev,
	689	struct mlx5_ib_flow_prio *prio, bool ft_added)
	690	{
	691	prio->refcount -= !!ft_added;
	692	if (!prio->refcount) {
	693	mlx5_destroy_flow_table(prio->flow_table);
	694	prio->flow_table = NULL;
	695	}
	696	}
	697
	698	static int mlx5_ib_destroy_flow(struct ib_flow *flow_id)
	699	{
	700	struct mlx5_ib_flow_handler *handler = container_of(flow_id,
	701	struct mlx5_ib_flow_handler,
	702	ibflow);
	703	struct mlx5_ib_flow_handler iter, tmp;
	704	struct mlx5_ib_dev *dev = handler->dev;
	705
	706	mutex_lock(&dev->flow_db->lock);
	707
	708	list_for_each_entry_safe(iter, tmp, &handler->list, list) {
	709	mlx5_del_flow_rules(iter->rule);
	710	put_flow_table(dev, iter->prio, true);
	711	list_del(&iter->list);
	712	kfree(iter);
	713	}
	714
	715	mlx5_del_flow_rules(handler->rule);
	716	put_flow_table(dev, handler->prio, true);
	717	mlx5_ib_counters_clear_description(handler->ibcounters);
	718	mutex_unlock(&dev->flow_db->lock);
	719	if (handler->flow_matcher)
	720	atomic_dec(&handler->flow_matcher->usecnt);
	721	kfree(handler);
	722
	723	return 0;
	724	}
	725
	726	static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap)
	727	{
	728	priority *= 2;
	729	if (!dont_trap)
	730	priority++;
	731	return priority;
	732	}
	733
	734	enum flow_table_type {
	735	MLX5_IB_FT_RX,
	736	MLX5_IB_FT_TX
	737	};
	738
	739	#define MLX5_FS_MAX_TYPES 6
	740	#define MLX5_FS_MAX_ENTRIES BIT(16)
	741
	742	static struct mlx5_ib_flow_prio _get_prio(struct mlx5_flow_namespace ns,
	743	struct mlx5_ib_flow_prio *prio,
	744	int priority,
	745	int num_entries, int num_groups,
	746	u32 flags)
	747	{
	748	struct mlx5_flow_table_attr ft_attr = {};
	749	struct mlx5_flow_table *ft;
	750
	751	ft_attr.prio = priority;
	752	ft_attr.max_fte = num_entries;
	753	ft_attr.flags = flags;
	754	ft_attr.autogroup.max_num_groups = num_groups;
	755	ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
	756	if (IS_ERR(ft))
	757	return ERR_CAST(ft);
	758
	759	prio->flow_table = ft;
	760	prio->refcount = 0;
	761	return prio;
	762	}
	763
	764	static struct mlx5_ib_flow_prio get_flow_table(struct mlx5_ib_dev dev,
	765	struct ib_flow_attr *flow_attr,
	766	enum flow_table_type ft_type)
	767	{
	768	bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP;
	769	struct mlx5_flow_namespace *ns = NULL;
	770	struct mlx5_ib_flow_prio *prio;
	771	struct mlx5_flow_table *ft;
	772	int max_table_size;
	773	int num_entries;
	774	int num_groups;
	775	bool esw_encap;
	776	u32 flags = 0;
	777	int priority;
	778
	779	max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
	780	log_max_ft_size));
	781	esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) !=
	782	DEVLINK_ESWITCH_ENCAP_MODE_NONE;
	783	if (flow_attr->type == IB_FLOW_ATTR_NORMAL) {
	784	enum mlx5_flow_namespace_type fn_type;
	785
	786	if (flow_is_multicast_only(flow_attr) &&
	787	!dont_trap)
	788	priority = MLX5_IB_FLOW_MCAST_PRIO;
	789	else
	790	priority = ib_prio_to_core_prio(flow_attr->priority,
	791	dont_trap);
	792	if (ft_type == MLX5_IB_FT_RX) {
	793	fn_type = MLX5_FLOW_NAMESPACE_BYPASS;
	794	prio = &dev->flow_db->prios[priority];
	795	if (!dev->is_rep && !esw_encap &&
	796	MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap))
	797	flags \|= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
	798	if (!dev->is_rep && !esw_encap &&
	799	MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
	800	reformat_l3_tunnel_to_l2))
	801	flags \|= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
	802	} else {
	803	max_table_size =
	804	BIT(MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev,
	805	log_max_ft_size));
	806	fn_type = MLX5_FLOW_NAMESPACE_EGRESS;
	807	prio = &dev->flow_db->egress_prios[priority];
	808	if (!dev->is_rep && !esw_encap &&
	809	MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat))
	810	flags \|= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
	811	}
	812	ns = mlx5_get_flow_namespace(dev->mdev, fn_type);
	813	num_entries = MLX5_FS_MAX_ENTRIES;
	814	num_groups = MLX5_FS_MAX_TYPES;
	815	} else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT \|\|
	816	flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) {
	817	ns = mlx5_get_flow_namespace(dev->mdev,
	818	MLX5_FLOW_NAMESPACE_LEFTOVERS);
	819	build_leftovers_ft_param(&priority,
	820	&num_entries,
	821	&num_groups);
	822	prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO];
	823	} else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
	824	if (!MLX5_CAP_FLOWTABLE(dev->mdev,
	825	allow_sniffer_and_nic_rx_shared_tir))
	826	return ERR_PTR(-EOPNOTSUPP);
	827
	828	ns = mlx5_get_flow_namespace(dev->mdev, ft_type == MLX5_IB_FT_RX ?
	829	MLX5_FLOW_NAMESPACE_SNIFFER_RX :
	830	MLX5_FLOW_NAMESPACE_SNIFFER_TX);
	831
	832	prio = &dev->flow_db->sniffer[ft_type];
	833	priority = 0;
	834	num_entries = 1;
	835	num_groups = 1;
	836	}
	837
	838	if (!ns)
	839	return ERR_PTR(-EOPNOTSUPP);
	840
	841	max_table_size = min_t(int, num_entries, max_table_size);
	842
	843	ft = prio->flow_table;
	844	if (!ft)
	845	return _get_prio(ns, prio, priority, max_table_size, num_groups,
	846	flags);
	847
	848	return prio;
	849	}
	850
	851	static void set_underlay_qp(struct mlx5_ib_dev *dev,
	852	struct mlx5_flow_spec *spec,
	853	u32 underlay_qpn)
	854	{
	855	void *misc_params_c = MLX5_ADDR_OF(fte_match_param,
	856	spec->match_criteria,
	857	misc_parameters);
	858	void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
	859	misc_parameters);
	860
	861	if (underlay_qpn &&
	862	MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
	863	ft_field_support.bth_dst_qp)) {
	864	MLX5_SET(fte_match_set_misc,
	865	misc_params_v, bth_dst_qp, underlay_qpn);
	866	MLX5_SET(fte_match_set_misc,
	867	misc_params_c, bth_dst_qp, 0xffffff);
	868	}
	869	}
	870
	871	static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev *dev,
	872	struct mlx5_flow_spec *spec,
	873	struct mlx5_eswitch_rep *rep)
	874	{
	875	struct mlx5_eswitch *esw = dev->mdev->priv.eswitch;
	876	void *misc;
	877
	878	if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
	879	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
	880	misc_parameters_2);
	881
	882	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
	883	mlx5_eswitch_get_vport_metadata_for_match(esw,
	884	rep->vport));
	885	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
	886	misc_parameters_2);
	887
	888	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
	889	mlx5_eswitch_get_vport_metadata_mask());
	890	} else {
	891	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
	892	misc_parameters);
	893
	894	MLX5_SET(fte_match_set_misc, misc, source_port, rep->vport);
	895
	896	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
	897	misc_parameters);
	898
	899	MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
	900	}
	901	}
	902
	903	static struct mlx5_ib_flow_handler _create_flow_rule(struct mlx5_ib_dev dev,
	904	struct mlx5_ib_flow_prio *ft_prio,
	905	const struct ib_flow_attr *flow_attr,
	906	struct mlx5_flow_destination *dst,
	907	u32 underlay_qpn,
	908	struct mlx5_ib_create_flow *ucmd)
	909	{
	910	struct mlx5_flow_table *ft = ft_prio->flow_table;
	911	struct mlx5_ib_flow_handler *handler;
	912	struct mlx5_flow_act flow_act = {};
	913	struct mlx5_flow_spec *spec;
	914	struct mlx5_flow_destination dest_arr[2] = {};
	915	struct mlx5_flow_destination *rule_dst = dest_arr;
	916	const void ib_flow = (const void )flow_attr + sizeof(*flow_attr);
	917	unsigned int spec_index;
	918	u32 prev_type = 0;
	919	int err = 0;
	920	int dest_num = 0;
	921	bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS;
	922
	923	if (!is_valid_attr(dev->mdev, flow_attr))
	924	return ERR_PTR(-EINVAL);
	925
	926	if (dev->is_rep && is_egress)
	927	return ERR_PTR(-EINVAL);
	928
	929	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
	930	handler = kzalloc(sizeof(*handler), GFP_KERNEL);
	931	if (!handler \|\| !spec) {
	932	err = -ENOMEM;
	933	goto free;
	934	}
	935
	936	INIT_LIST_HEAD(&handler->list);
	937
	938	for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
	939	err = parse_flow_attr(dev->mdev, spec,
	940	ib_flow, flow_attr, &flow_act,
	941	prev_type);
	942	if (err < 0)
	943	goto free;
	944
	945	prev_type = ((union ib_flow_spec *)ib_flow)->type;
	946	ib_flow += ((union ib_flow_spec *)ib_flow)->size;
	947	}
	948
	949	if (dst && !(flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP)) {
	950	memcpy(&dest_arr[0], dst, sizeof(*dst));
	951	dest_num++;
	952	}
	953
	954	if (!flow_is_multicast_only(flow_attr))
	955	set_underlay_qp(dev, spec, underlay_qpn);
	956
	957	if (dev->is_rep) {
	958	struct mlx5_eswitch_rep *rep;
	959
	960	rep = dev->port[flow_attr->port - 1].rep;
	961	if (!rep) {
	962	err = -EINVAL;
	963	goto free;
	964	}
	965
	966	mlx5_ib_set_rule_source_port(dev, spec, rep);
	967	}
	968
	969	spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria);
	970
	971	if (is_egress &&
	972	!is_valid_spec(dev->mdev, spec, &flow_act, is_egress)) {
	973	err = -EINVAL;
	974	goto free;
	975	}
	976
	977	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
	978	struct mlx5_ib_mcounters *mcounters;
	979
	980	err = mlx5_ib_flow_counters_set_data(flow_act.counters, ucmd);
	981	if (err)
	982	goto free;
	983
	984	mcounters = to_mcounters(flow_act.counters);
	985	handler->ibcounters = flow_act.counters;
	986	dest_arr[dest_num].type =
	987	MLX5_FLOW_DESTINATION_TYPE_COUNTER;
	988	dest_arr[dest_num].counter_id =
	989	mlx5_fc_id(mcounters->hw_cntrs_hndl);
	990	dest_num++;
	991	}
	992
	993	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP) {
	994	if (!dest_num)
	995	rule_dst = NULL;
	996	} else {
	997	if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP)
	998	flow_act.action \|=
	999	MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO;
	1000	if (is_egress)
	1001	flow_act.action \|= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
	1002	else if (dest_num)
	1003	flow_act.action \|= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
	1004	}
	1005
	1006	if ((spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG) &&
	1007	(flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT \|\|
	1008	flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) {
	1009	mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n",
	1010	spec->flow_context.flow_tag, flow_attr->type);
	1011	err = -EINVAL;
	1012	goto free;
	1013	}
	1014	handler->rule = mlx5_add_flow_rules(ft, spec,
	1015	&flow_act,
	1016	rule_dst, dest_num);
	1017
	1018	if (IS_ERR(handler->rule)) {
	1019	err = PTR_ERR(handler->rule);
	1020	goto free;
	1021	}
	1022
	1023	ft_prio->refcount++;
	1024	handler->prio = ft_prio;
	1025	handler->dev = dev;
	1026
	1027	ft_prio->flow_table = ft;
	1028	free:
	1029	if (err && handler) {
	1030	mlx5_ib_counters_clear_description(handler->ibcounters);
	1031	kfree(handler);
	1032	}
	1033	kvfree(spec);
	1034	return err ? ERR_PTR(err) : handler;
	1035	}
	1036
	1037	static struct mlx5_ib_flow_handler create_flow_rule(struct mlx5_ib_dev dev,
	1038	struct mlx5_ib_flow_prio *ft_prio,
	1039	const struct ib_flow_attr *flow_attr,
	1040	struct mlx5_flow_destination *dst)
	1041	{
	1042	return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL);
	1043	}
	1044
	1045	enum {
	1046	LEFTOVERS_MC,
	1047	LEFTOVERS_UC,
	1048	};
	1049
	1050	static struct mlx5_ib_flow_handler create_leftovers_rule(struct mlx5_ib_dev dev,
	1051	struct mlx5_ib_flow_prio *ft_prio,
	1052	struct ib_flow_attr *flow_attr,
	1053	struct mlx5_flow_destination *dst)
	1054	{
	1055	struct mlx5_ib_flow_handler *handler_ucast = NULL;
	1056	struct mlx5_ib_flow_handler *handler = NULL;
	1057
	1058	static struct {
	1059	struct ib_flow_attr flow_attr;
	1060	struct ib_flow_spec_eth eth_flow;
	1061	} leftovers_specs[] = {
	1062	[LEFTOVERS_MC] = {
	1063	.flow_attr = {
	1064	.num_of_specs = 1,
	1065	.size = sizeof(leftovers_specs[0])
	1066	},
	1067	.eth_flow = {
	1068	.type = IB_FLOW_SPEC_ETH,
	1069	.size = sizeof(struct ib_flow_spec_eth),
	1070	.mask = {.dst_mac = {0x1} },
	1071	.val = {.dst_mac = {0x1} }
	1072	}
	1073	},
	1074	[LEFTOVERS_UC] = {
	1075	.flow_attr = {
	1076	.num_of_specs = 1,
	1077	.size = sizeof(leftovers_specs[0])
	1078	},
	1079	.eth_flow = {
	1080	.type = IB_FLOW_SPEC_ETH,
	1081	.size = sizeof(struct ib_flow_spec_eth),
	1082	.mask = {.dst_mac = {0x1} },
	1083	.val = {.dst_mac = {} }
	1084	}
	1085	}
	1086	};
	1087
	1088	handler = create_flow_rule(dev, ft_prio,
	1089	&leftovers_specs[LEFTOVERS_MC].flow_attr,
	1090	dst);
	1091	if (!IS_ERR(handler) &&
	1092	flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) {
	1093	handler_ucast = create_flow_rule(dev, ft_prio,
	1094	&leftovers_specs[LEFTOVERS_UC].flow_attr,
	1095	dst);
	1096	if (IS_ERR(handler_ucast)) {
	1097	mlx5_del_flow_rules(handler->rule);
	1098	ft_prio->refcount--;
	1099	kfree(handler);
	1100	handler = handler_ucast;
	1101	} else {
	1102	list_add(&handler_ucast->list, &handler->list);
	1103	}
	1104	}
	1105
	1106	return handler;
	1107	}
	1108
	1109	static struct mlx5_ib_flow_handler create_sniffer_rule(struct mlx5_ib_dev dev,
	1110	struct mlx5_ib_flow_prio *ft_rx,
	1111	struct mlx5_ib_flow_prio *ft_tx,
	1112	struct mlx5_flow_destination *dst)
	1113	{
	1114	struct mlx5_ib_flow_handler *handler_rx;
	1115	struct mlx5_ib_flow_handler *handler_tx;
	1116	int err;
	1117	static const struct ib_flow_attr flow_attr = {
	1118	.num_of_specs = 0,
	1119	.size = sizeof(flow_attr)
	1120	};
	1121
	1122	handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst);
	1123	if (IS_ERR(handler_rx)) {
	1124	err = PTR_ERR(handler_rx);
	1125	goto err;
	1126	}
	1127
	1128	handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst);
	1129	if (IS_ERR(handler_tx)) {
	1130	err = PTR_ERR(handler_tx);
	1131	goto err_tx;
	1132	}
	1133
	1134	list_add(&handler_tx->list, &handler_rx->list);
	1135
	1136	return handler_rx;
	1137
	1138	err_tx:
	1139	mlx5_del_flow_rules(handler_rx->rule);
	1140	ft_rx->refcount--;
	1141	kfree(handler_rx);
	1142	err:
	1143	return ERR_PTR(err);
	1144	}
	1145
	1146
	1147	static struct ib_flow mlx5_ib_create_flow(struct ib_qp qp,
	1148	struct ib_flow_attr *flow_attr,
	1149	int domain,
	1150	struct ib_udata *udata)
	1151	{
	1152	struct mlx5_ib_dev *dev = to_mdev(qp->device);
	1153	struct mlx5_ib_qp *mqp = to_mqp(qp);
	1154	struct mlx5_ib_flow_handler *handler = NULL;
	1155	struct mlx5_flow_destination *dst = NULL;
	1156	struct mlx5_ib_flow_prio *ft_prio_tx = NULL;
	1157	struct mlx5_ib_flow_prio *ft_prio;
	1158	bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS;
	1159	struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr;
	1160	size_t min_ucmd_sz, required_ucmd_sz;
	1161	int err;
	1162	int underlay_qpn;
	1163
	1164	if (udata && udata->inlen) {
	1165	min_ucmd_sz = offsetof(typeof(ucmd_hdr), reserved) +
	1166	sizeof(ucmd_hdr.reserved);
	1167	if (udata->inlen < min_ucmd_sz)
	1168	return ERR_PTR(-EOPNOTSUPP);
	1169
	1170	err = ib_copy_from_udata(&ucmd_hdr, udata, min_ucmd_sz);
	1171	if (err)
	1172	return ERR_PTR(err);
	1173
	1174	/* currently supports only one counters data */
	1175	if (ucmd_hdr.ncounters_data > 1)
	1176	return ERR_PTR(-EINVAL);
	1177
	1178	required_ucmd_sz = min_ucmd_sz +
	1179	sizeof(struct mlx5_ib_flow_counters_data) *
	1180	ucmd_hdr.ncounters_data;
	1181	if (udata->inlen > required_ucmd_sz &&
	1182	!ib_is_udata_cleared(udata, required_ucmd_sz,
	1183	udata->inlen - required_ucmd_sz))
	1184	return ERR_PTR(-EOPNOTSUPP);
	1185
	1186	ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL);
	1187	if (!ucmd)
	1188	return ERR_PTR(-ENOMEM);
	1189
	1190	err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz);
	1191	if (err)
	1192	goto free_ucmd;
	1193	}
	1194
	1195	if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO) {
	1196	err = -ENOMEM;
	1197	goto free_ucmd;
	1198	}
	1199
	1200	if (domain != IB_FLOW_DOMAIN_USER \|\|
	1201	flow_attr->port > dev->num_ports \|\|
	1202	(flow_attr->flags & ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP \|
	1203	IB_FLOW_ATTR_FLAGS_EGRESS))) {
	1204	err = -EINVAL;
	1205	goto free_ucmd;
	1206	}
	1207
	1208	if (is_egress &&
	1209	(flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT \|\|
	1210	flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) {
	1211	err = -EINVAL;
	1212	goto free_ucmd;
	1213	}
	1214
	1215	dst = kzalloc(sizeof(*dst), GFP_KERNEL);
	1216	if (!dst) {
	1217	err = -ENOMEM;
	1218	goto free_ucmd;
	1219	}
	1220
	1221	mutex_lock(&dev->flow_db->lock);
	1222
	1223	ft_prio = get_flow_table(dev, flow_attr,
	1224	is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX);
	1225	if (IS_ERR(ft_prio)) {
	1226	err = PTR_ERR(ft_prio);
	1227	goto unlock;
	1228	}
	1229	if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
	1230	ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX);
	1231	if (IS_ERR(ft_prio_tx)) {
	1232	err = PTR_ERR(ft_prio_tx);
	1233	ft_prio_tx = NULL;
	1234	goto destroy_ft;
	1235	}
	1236	}
	1237
	1238	if (is_egress) {
	1239	dst->type = MLX5_FLOW_DESTINATION_TYPE_PORT;
	1240	} else {
	1241	dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR;
	1242	if (mqp->is_rss)
	1243	dst->tir_num = mqp->rss_qp.tirn;
	1244	else
	1245	dst->tir_num = mqp->raw_packet_qp.rq.tirn;
	1246	}
	1247
	1248	if (flow_attr->type == IB_FLOW_ATTR_NORMAL) {
	1249	underlay_qpn = (mqp->flags & IB_QP_CREATE_SOURCE_QPN) ?
	1250	mqp->underlay_qpn :
	1251	0;
	1252	handler = _create_flow_rule(dev, ft_prio, flow_attr, dst,
	1253	underlay_qpn, ucmd);
	1254	} else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT \|\|
	1255	flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) {
	1256	handler = create_leftovers_rule(dev, ft_prio, flow_attr,
	1257	dst);
	1258	} else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
	1259	handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst);
	1260	} else {
	1261	err = -EINVAL;
	1262	goto destroy_ft;
	1263	}
	1264
	1265	if (IS_ERR(handler)) {
	1266	err = PTR_ERR(handler);
	1267	handler = NULL;
	1268	goto destroy_ft;
	1269	}
	1270
	1271	mutex_unlock(&dev->flow_db->lock);
	1272	kfree(dst);
	1273	kfree(ucmd);
	1274
	1275	return &handler->ibflow;
	1276
	1277	destroy_ft:
	1278	put_flow_table(dev, ft_prio, false);
	1279	if (ft_prio_tx)
	1280	put_flow_table(dev, ft_prio_tx, false);
	1281	unlock:
	1282	mutex_unlock(&dev->flow_db->lock);
	1283	kfree(dst);
	1284	free_ucmd:
	1285	kfree(ucmd);
	1286	return ERR_PTR(err);
	1287	}
	1288
	1289	static struct mlx5_ib_flow_prio *
	1290	_get_flow_table(struct mlx5_ib_dev *dev,
	1291	struct mlx5_ib_flow_matcher *fs_matcher,
	1292	bool mcast)
	1293	{
	1294	struct mlx5_flow_namespace *ns = NULL;
	1295	struct mlx5_ib_flow_prio *prio = NULL;
	1296	int max_table_size = 0;
	1297	bool esw_encap;
	1298	u32 flags = 0;
	1299	int priority;
	1300
	1301	if (mcast)
	1302	priority = MLX5_IB_FLOW_MCAST_PRIO;
	1303	else
	1304	priority = ib_prio_to_core_prio(fs_matcher->priority, false);
	1305
	1306	esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) !=
	1307	DEVLINK_ESWITCH_ENCAP_MODE_NONE;
	1308	if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) {
	1309	max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
	1310	log_max_ft_size));
	1311	if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap)
	1312	flags \|= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
	1313	if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
	1314	reformat_l3_tunnel_to_l2) &&
	1315	!esw_encap)
	1316	flags \|= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
	1317	} else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) {
	1318	max_table_size = BIT(
	1319	MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size));
	1320	if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) && !esw_encap)
	1321	flags \|= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
	1322	} else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB) {
	1323	max_table_size = BIT(
	1324	MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size));
	1325	if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap)
	1326	flags \|= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
	1327	if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, reformat_l3_tunnel_to_l2) &&
	1328	esw_encap)
	1329	flags \|= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
	1330	priority = FDB_BYPASS_PATH;
	1331	} else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) {
	1332	max_table_size =
	1333	BIT(MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev,
	1334	log_max_ft_size));
	1335	priority = fs_matcher->priority;
	1336	} else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) {
	1337	max_table_size =
	1338	BIT(MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev,
	1339	log_max_ft_size));
	1340	priority = fs_matcher->priority;
	1341	}
	1342
	1343	max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES);
	1344
	1345	ns = mlx5_get_flow_namespace(dev->mdev, fs_matcher->ns_type);
	1346	if (!ns)
	1347	return ERR_PTR(-EOPNOTSUPP);
	1348
	1349	if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS)
	1350	prio = &dev->flow_db->prios[priority];
	1351	else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS)
	1352	prio = &dev->flow_db->egress_prios[priority];
	1353	else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB)
	1354	prio = &dev->flow_db->fdb;
	1355	else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX)
	1356	prio = &dev->flow_db->rdma_rx[priority];
	1357	else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX)
	1358	prio = &dev->flow_db->rdma_tx[priority];
	1359
	1360	if (!prio)
	1361	return ERR_PTR(-EINVAL);
	1362
	1363	if (prio->flow_table)
	1364	return prio;
	1365
	1366	return _get_prio(ns, prio, priority, max_table_size,
	1367	MLX5_FS_MAX_TYPES, flags);
	1368	}
	1369
	1370	static struct mlx5_ib_flow_handler *
	1371	_create_raw_flow_rule(struct mlx5_ib_dev *dev,
	1372	struct mlx5_ib_flow_prio *ft_prio,
	1373	struct mlx5_flow_destination *dst,
	1374	struct mlx5_ib_flow_matcher *fs_matcher,
	1375	struct mlx5_flow_context *flow_context,
	1376	struct mlx5_flow_act *flow_act,
	1377	void *cmd_in, int inlen,
	1378	int dst_num)
	1379	{
	1380	struct mlx5_ib_flow_handler *handler;
	1381	struct mlx5_flow_spec *spec;
	1382	struct mlx5_flow_table *ft = ft_prio->flow_table;
	1383	int err = 0;
	1384
	1385	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
	1386	handler = kzalloc(sizeof(*handler), GFP_KERNEL);
	1387	if (!handler \|\| !spec) {
	1388	err = -ENOMEM;
	1389	goto free;
	1390	}
	1391
	1392	INIT_LIST_HEAD(&handler->list);
	1393
	1394	memcpy(spec->match_value, cmd_in, inlen);
	1395	memcpy(spec->match_criteria, fs_matcher->matcher_mask.match_params,
	1396	fs_matcher->mask_len);
	1397	spec->match_criteria_enable = fs_matcher->match_criteria_enable;
	1398	spec->flow_context = *flow_context;
	1399
	1400	handler->rule = mlx5_add_flow_rules(ft, spec,
	1401	flow_act, dst, dst_num);
	1402
	1403	if (IS_ERR(handler->rule)) {
	1404	err = PTR_ERR(handler->rule);
	1405	goto free;
	1406	}
	1407
	1408	ft_prio->refcount++;
	1409	handler->prio = ft_prio;
	1410	handler->dev = dev;
	1411	ft_prio->flow_table = ft;
	1412
	1413	free:
	1414	if (err)
	1415	kfree(handler);
	1416	kvfree(spec);
	1417	return err ? ERR_PTR(err) : handler;
	1418	}
	1419
	1420	static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher,
	1421	void *match_v)
	1422	{
	1423	void *match_c;
	1424	void match_v_set_lyr_2_4, match_c_set_lyr_2_4;
	1425	void dmac, dmac_mask;
	1426	void ipv4, ipv4_mask;
	1427
	1428	if (!(fs_matcher->match_criteria_enable &
	1429	(1 << MATCH_CRITERIA_ENABLE_OUTER_BIT)))
	1430	return false;
	1431
	1432	match_c = fs_matcher->matcher_mask.match_params;
	1433	match_v_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_v,
	1434	outer_headers);
	1435	match_c_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_c,
	1436	outer_headers);
	1437
	1438	dmac = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,
	1439	dmac_47_16);
	1440	dmac_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,
	1441	dmac_47_16);
	1442
	1443	if (is_multicast_ether_addr(dmac) &&
	1444	is_multicast_ether_addr(dmac_mask))
	1445	return true;
	1446
	1447	ipv4 = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,
	1448	dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
	1449
	1450	ipv4_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,
	1451	dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
	1452
	1453	if (ipv4_is_multicast((__be32 )(ipv4)) &&
	1454	ipv4_is_multicast((__be32 )(ipv4_mask)))
	1455	return true;
	1456
	1457	return false;
	1458	}
	1459
	1460	static struct mlx5_ib_flow_handler *raw_fs_rule_add(
	1461	struct mlx5_ib_dev dev, struct mlx5_ib_flow_matcher fs_matcher,
	1462	struct mlx5_flow_context flow_context, struct mlx5_flow_act flow_act,
	1463	u32 counter_id, void *cmd_in, int inlen, int dest_id, int dest_type)
	1464	{
	1465	struct mlx5_flow_destination *dst;
	1466	struct mlx5_ib_flow_prio *ft_prio;
	1467	struct mlx5_ib_flow_handler *handler;
	1468	int dst_num = 0;
	1469	bool mcast;
	1470	int err;
	1471
	1472	if (fs_matcher->flow_type != MLX5_IB_FLOW_TYPE_NORMAL)
	1473	return ERR_PTR(-EOPNOTSUPP);
	1474
	1475	if (fs_matcher->priority > MLX5_IB_FLOW_LAST_PRIO)
	1476	return ERR_PTR(-ENOMEM);
	1477
	1478	dst = kcalloc(2, sizeof(*dst), GFP_KERNEL);
	1479	if (!dst)
	1480	return ERR_PTR(-ENOMEM);
	1481
	1482	mcast = raw_fs_is_multicast(fs_matcher, cmd_in);
	1483	mutex_lock(&dev->flow_db->lock);
	1484
	1485	ft_prio = _get_flow_table(dev, fs_matcher, mcast);
	1486	if (IS_ERR(ft_prio)) {
	1487	err = PTR_ERR(ft_prio);
	1488	goto unlock;
	1489	}
	1490
	1491	if (dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR) {
	1492	dst[dst_num].type = dest_type;
	1493	dst[dst_num++].tir_num = dest_id;
	1494	flow_act->action \|= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
	1495	} else if (dest_type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) {
	1496	dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM;
	1497	dst[dst_num++].ft_num = dest_id;
	1498	flow_act->action \|= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
	1499	} else if (dest_type == MLX5_FLOW_DESTINATION_TYPE_PORT) {
	1500	dst[dst_num++].type = MLX5_FLOW_DESTINATION_TYPE_PORT;
	1501	flow_act->action \|= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
	1502	}
	1503
	1504
	1505	if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
	1506	dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
	1507	dst[dst_num].counter_id = counter_id;
	1508	dst_num++;
	1509	}
	1510
	1511	handler = _create_raw_flow_rule(dev, ft_prio, dst, fs_matcher,
	1512	flow_context, flow_act,
	1513	cmd_in, inlen, dst_num);
	1514
	1515	if (IS_ERR(handler)) {
	1516	err = PTR_ERR(handler);
	1517	goto destroy_ft;
	1518	}
	1519
	1520	mutex_unlock(&dev->flow_db->lock);
	1521	atomic_inc(&fs_matcher->usecnt);
	1522	handler->flow_matcher = fs_matcher;
	1523
	1524	kfree(dst);
	1525
	1526	return handler;
	1527
	1528	destroy_ft:
	1529	put_flow_table(dev, ft_prio, false);
	1530	unlock:
	1531	mutex_unlock(&dev->flow_db->lock);
	1532	kfree(dst);
	1533
	1534	return ERR_PTR(err);
	1535	}
	1536
	1537	static u32 mlx5_ib_flow_action_flags_to_accel_xfrm_flags(u32 mlx5_flags)
	1538	{
	1539	u32 flags = 0;
	1540
	1541	if (mlx5_flags & MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA)
	1542	flags \|= MLX5_ACCEL_XFRM_FLAG_REQUIRE_METADATA;
	1543
	1544	return flags;
	1545	}
	1546
	1547	#define MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED \
	1548	MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA
	1549	static struct ib_flow_action *
	1550	mlx5_ib_create_flow_action_esp(struct ib_device *device,
	1551	const struct ib_flow_action_attrs_esp *attr,
	1552	struct uverbs_attr_bundle *attrs)
	1553	{
	1554	struct mlx5_ib_dev *mdev = to_mdev(device);
	1555	struct ib_uverbs_flow_action_esp_keymat_aes_gcm *aes_gcm;
	1556	struct mlx5_accel_esp_xfrm_attrs accel_attrs = {};
	1557	struct mlx5_ib_flow_action *action;
	1558	u64 action_flags;
	1559	u64 flags;
	1560	int err = 0;
	1561
	1562	err = uverbs_get_flags64(
	1563	&action_flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_ACTION_FLAGS,
	1564	((MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED << 1) - 1));
	1565	if (err)
	1566	return ERR_PTR(err);
	1567
	1568	flags = mlx5_ib_flow_action_flags_to_accel_xfrm_flags(action_flags);
	1569
	1570	/* We current only support a subset of the standard features. Only a
	1571	* keymat of type AES_GCM, with icv_len == 16, iv_algo == SEQ and esn
	1572	* (with overlap). Full offload mode isn't supported.
	1573	*/
	1574	if (!attr->keymat \|\| attr->replay \|\| attr->encap \|\|
	1575	attr->spi \|\| attr->seq \|\| attr->tfc_pad \|\|
	1576	attr->hard_limit_pkts \|\|
	1577	(attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED \|
	1578	IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT)))
	1579	return ERR_PTR(-EOPNOTSUPP);
	1580
	1581	if (attr->keymat->protocol !=
	1582	IB_UVERBS_FLOW_ACTION_ESP_KEYMAT_AES_GCM)
	1583	return ERR_PTR(-EOPNOTSUPP);
	1584
	1585	aes_gcm = &attr->keymat->keymat.aes_gcm;
	1586
	1587	if (aes_gcm->icv_len != 16 \|\|
	1588	aes_gcm->iv_algo != IB_UVERBS_FLOW_ACTION_IV_ALGO_SEQ)
	1589	return ERR_PTR(-EOPNOTSUPP);
	1590
	1591	action = kmalloc(sizeof(*action), GFP_KERNEL);
	1592	if (!action)
	1593	return ERR_PTR(-ENOMEM);
	1594
	1595	action->esp_aes_gcm.ib_flags = attr->flags;
	1596	memcpy(&accel_attrs.keymat.aes_gcm.aes_key, &aes_gcm->aes_key,
	1597	sizeof(accel_attrs.keymat.aes_gcm.aes_key));
	1598	accel_attrs.keymat.aes_gcm.key_len = aes_gcm->key_len * 8;
	1599	memcpy(&accel_attrs.keymat.aes_gcm.salt, &aes_gcm->salt,
	1600	sizeof(accel_attrs.keymat.aes_gcm.salt));
	1601	memcpy(&accel_attrs.keymat.aes_gcm.seq_iv, &aes_gcm->iv,
	1602	sizeof(accel_attrs.keymat.aes_gcm.seq_iv));
	1603	accel_attrs.keymat.aes_gcm.icv_len = aes_gcm->icv_len * 8;
	1604	accel_attrs.keymat.aes_gcm.iv_algo = MLX5_ACCEL_ESP_AES_GCM_IV_ALGO_SEQ;
	1605	accel_attrs.keymat_type = MLX5_ACCEL_ESP_KEYMAT_AES_GCM;
	1606
	1607	accel_attrs.esn = attr->esn;
	1608	if (attr->flags & IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED)
	1609	accel_attrs.flags \|= MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED;
	1610	if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)
	1611	accel_attrs.flags \|= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;
	1612
	1613	if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT)
	1614	accel_attrs.action \|= MLX5_ACCEL_ESP_ACTION_ENCRYPT;
	1615
	1616	action->esp_aes_gcm.ctx =
	1617	mlx5_accel_esp_create_xfrm(mdev->mdev, &accel_attrs, flags);
	1618	if (IS_ERR(action->esp_aes_gcm.ctx)) {
	1619	err = PTR_ERR(action->esp_aes_gcm.ctx);
	1620	goto err_parse;
	1621	}
	1622
	1623	action->esp_aes_gcm.ib_flags = attr->flags;
	1624
	1625	return &action->ib_action;
	1626
	1627	err_parse:
	1628	kfree(action);
	1629	return ERR_PTR(err);
	1630	}
	1631
	1632	static int
	1633	mlx5_ib_modify_flow_action_esp(struct ib_flow_action *action,
	1634	const struct ib_flow_action_attrs_esp *attr,
	1635	struct uverbs_attr_bundle *attrs)
	1636	{
	1637	struct mlx5_ib_flow_action *maction = to_mflow_act(action);
	1638	struct mlx5_accel_esp_xfrm_attrs accel_attrs;
	1639	int err = 0;
	1640
	1641	if (attr->keymat \|\| attr->replay \|\| attr->encap \|\|
	1642	attr->spi \|\| attr->seq \|\| attr->tfc_pad \|\|
	1643	attr->hard_limit_pkts \|\|
	1644	(attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED \|
	1645	IB_FLOW_ACTION_ESP_FLAGS_MOD_ESP_ATTRS \|
	1646	IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)))
	1647	return -EOPNOTSUPP;
	1648
	1649	/* Only the ESN value or the MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP can
	1650	* be modified.
	1651	*/
	1652	if (!(maction->esp_aes_gcm.ib_flags &
	1653	IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED) &&
	1654	attr->flags & (IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED \|
	1655	IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW))
	1656	return -EINVAL;
	1657
	1658	memcpy(&accel_attrs, &maction->esp_aes_gcm.ctx->attrs,
	1659	sizeof(accel_attrs));
	1660
	1661	accel_attrs.esn = attr->esn;
	1662	if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)
	1663	accel_attrs.flags \|= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;
	1664	else
	1665	accel_attrs.flags &= ~MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;
	1666
	1667	err = mlx5_accel_esp_modify_xfrm(maction->esp_aes_gcm.ctx,
	1668	&accel_attrs);
	1669	if (err)
	1670	return err;
	1671
	1672	maction->esp_aes_gcm.ib_flags &=
	1673	~IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW;
	1674	maction->esp_aes_gcm.ib_flags \|=
	1675	attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW;
	1676
	1677	return 0;
	1678	}
	1679
	1680	static void destroy_flow_action_raw(struct mlx5_ib_flow_action *maction)
	1681	{
	1682	switch (maction->flow_action_raw.sub_type) {
	1683	case MLX5_IB_FLOW_ACTION_MODIFY_HEADER:
	1684	mlx5_modify_header_dealloc(maction->flow_action_raw.dev->mdev,
	1685	maction->flow_action_raw.modify_hdr);
	1686	break;
	1687	case MLX5_IB_FLOW_ACTION_PACKET_REFORMAT:
	1688	mlx5_packet_reformat_dealloc(maction->flow_action_raw.dev->mdev,
	1689	maction->flow_action_raw.pkt_reformat);
	1690	break;
	1691	case MLX5_IB_FLOW_ACTION_DECAP:
	1692	break;
	1693	default:
	1694	break;
	1695	}
	1696	}
	1697
	1698	static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action)
	1699	{
	1700	struct mlx5_ib_flow_action *maction = to_mflow_act(action);
	1701
	1702	switch (action->type) {
	1703	case IB_FLOW_ACTION_ESP:
	1704	/*
	1705	* We only support aes_gcm by now, so we implicitly know this is
	1706	* the underline crypto.
	1707	*/
	1708	mlx5_accel_esp_destroy_xfrm(maction->esp_aes_gcm.ctx);
	1709	break;
	1710	case IB_FLOW_ACTION_UNSPECIFIED:
	1711	destroy_flow_action_raw(maction);
	1712	break;
	1713	default:
	1714	WARN_ON(true);
	1715	break;
	1716	}
	1717
	1718	kfree(maction);
	1719	return 0;
	1720	}
	1721
	1722	static int
	1723	mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type,
	1724	enum mlx5_flow_namespace_type *namespace)
	1725	{
	1726	switch (table_type) {
	1727	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX:
	1728	*namespace = MLX5_FLOW_NAMESPACE_BYPASS;
	1729	break;
	1730	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX:
	1731	*namespace = MLX5_FLOW_NAMESPACE_EGRESS;
	1732	break;
	1733	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB:
	1734	*namespace = MLX5_FLOW_NAMESPACE_FDB;
	1735	break;
	1736	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX:
	1737	*namespace = MLX5_FLOW_NAMESPACE_RDMA_RX;
	1738	break;
	1739	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX:
	1740	*namespace = MLX5_FLOW_NAMESPACE_RDMA_TX;
	1741	break;
	1742	default:
	1743	return -EINVAL;
	1744	}
	1745
	1746	return 0;
	1747	}
	1748
	1749	static const struct uverbs_attr_spec mlx5_ib_flow_type[] = {
	1750	[MLX5_IB_FLOW_TYPE_NORMAL] = {
	1751	.type = UVERBS_ATTR_TYPE_PTR_IN,
	1752	.u.ptr = {
	1753	.len = sizeof(u16), /* data is priority */
	1754	.min_len = sizeof(u16),
	1755	}
	1756	},
	1757	[MLX5_IB_FLOW_TYPE_SNIFFER] = {
	1758	.type = UVERBS_ATTR_TYPE_PTR_IN,
	1759	UVERBS_ATTR_NO_DATA(),
	1760	},
	1761	[MLX5_IB_FLOW_TYPE_ALL_DEFAULT] = {
	1762	.type = UVERBS_ATTR_TYPE_PTR_IN,
	1763	UVERBS_ATTR_NO_DATA(),
	1764	},
	1765	[MLX5_IB_FLOW_TYPE_MC_DEFAULT] = {
	1766	.type = UVERBS_ATTR_TYPE_PTR_IN,
	1767	UVERBS_ATTR_NO_DATA(),
	1768	},
	1769	};
	1770
	1771	static bool is_flow_dest(void obj, int dest_id, int *dest_type)
	1772	{
	1773	struct devx_obj *devx_obj = obj;
	1774	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
	1775
	1776	switch (opcode) {
	1777	case MLX5_CMD_OP_DESTROY_TIR:
	1778	*dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
	1779	*dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox,
	1780	obj_id);
	1781	return true;
	1782
	1783	case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
	1784	*dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
	1785	*dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox,
	1786	table_id);
	1787	return true;
	1788	default:
	1789	return false;
	1790	}
	1791	}
	1792
	1793	static int get_dests(struct uverbs_attr_bundle *attrs,
	1794	struct mlx5_ib_flow_matcher fs_matcher, int dest_id,
	1795	int dest_type, struct ib_qp qp, u32 flags)
	1796	{
	1797	bool dest_devx, dest_qp;
	1798	void *devx_obj;
	1799	int err;
	1800
	1801	dest_devx = uverbs_attr_is_valid(attrs,
	1802	MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);
	1803	dest_qp = uverbs_attr_is_valid(attrs,
	1804	MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);
	1805
	1806	*flags = 0;
	1807	err = uverbs_get_flags32(flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_FLAGS,
	1808	MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS \|
	1809	MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP);
	1810	if (err)
	1811	return err;
	1812
	1813	/* Both flags are not allowed */
	1814	if (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS &&
	1815	*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)
	1816	return -EINVAL;
	1817
	1818	if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) {
	1819	if (dest_devx && (dest_qp \|\| *flags))
	1820	return -EINVAL;
	1821	else if (dest_qp && *flags)
	1822	return -EINVAL;
	1823	}
	1824
	1825	/* Allow only DEVX object, drop as dest for FDB */
	1826	if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB && !(dest_devx \|\|
	1827	(*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)))
	1828	return -EINVAL;
	1829
	1830	/* Allow only DEVX object or QP as dest when inserting to RDMA_RX */
	1831	if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) &&
	1832	((!dest_devx && !dest_qp) \|\| (dest_devx && dest_qp)))
	1833	return -EINVAL;
	1834
	1835	*qp = NULL;
	1836	if (dest_devx) {
	1837	devx_obj =
	1838	uverbs_attr_get_obj(attrs,
	1839	MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);
	1840
	1841	/* Verify that the given DEVX object is a flow
	1842	* steering destination.
	1843	*/
	1844	if (!is_flow_dest(devx_obj, dest_id, dest_type))
	1845	return -EINVAL;
	1846	/* Allow only flow table as dest when inserting to FDB or RDMA_RX */
	1847	if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB \|\|
	1848	fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) &&
	1849	*dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE)
	1850	return -EINVAL;
	1851	} else if (dest_qp) {
	1852	struct mlx5_ib_qp *mqp;
	1853
	1854	*qp = uverbs_attr_get_obj(attrs,
	1855	MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);
	1856	if (IS_ERR(*qp))
	1857	return PTR_ERR(*qp);
	1858
	1859	if ((*qp)->qp_type != IB_QPT_RAW_PACKET)
	1860	return -EINVAL;
	1861
	1862	mqp = to_mqp(*qp);
	1863	if (mqp->is_rss)
	1864	*dest_id = mqp->rss_qp.tirn;
	1865	else
	1866	*dest_id = mqp->raw_packet_qp.rq.tirn;
	1867	*dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
	1868	} else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS \|\|
	1869	fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) {
	1870	*dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT;
	1871	}
	1872
	1873	if (*dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR &&
	1874	(fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS \|\|
	1875	fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX))
	1876	return -EINVAL;
	1877
	1878	return 0;
	1879	}
	1880
	1881	static bool is_flow_counter(void obj, u32 offset, u32 counter_id)
	1882	{
	1883	struct devx_obj *devx_obj = obj;
	1884	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
	1885
	1886	if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) {
	1887
	1888	if (offset && offset >= devx_obj->flow_counter_bulk_size)
	1889	return false;
	1890
	1891	*counter_id = MLX5_GET(dealloc_flow_counter_in,
	1892	devx_obj->dinbox,
	1893	flow_counter_id);
	1894	*counter_id += offset;
	1895	return true;
	1896	}
	1897
	1898	return false;
	1899	}
	1900
	1901	#define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2
	1902	static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)(
	1903	struct uverbs_attr_bundle *attrs)
	1904	{
	1905	struct mlx5_flow_context flow_context = {.flow_tag =
	1906	MLX5_FS_DEFAULT_FLOW_TAG};
	1907	u32 *offset_attr, offset = 0, counter_id = 0;
	1908	int dest_id, dest_type = -1, inlen, len, ret, i;
	1909	struct mlx5_ib_flow_handler *flow_handler;
	1910	struct mlx5_ib_flow_matcher *fs_matcher;
	1911	struct ib_uobject **arr_flow_actions;
	1912	struct ib_uflow_resources *uflow_res;
	1913	struct mlx5_flow_act flow_act = {};
	1914	struct ib_qp *qp = NULL;
	1915	void devx_obj, cmd_in;
	1916	struct ib_uobject *uobj;
	1917	struct mlx5_ib_dev *dev;
	1918	u32 flags;
	1919
	1920	if (!capable(CAP_NET_RAW))
	1921	return -EPERM;
	1922
	1923	fs_matcher = uverbs_attr_get_obj(attrs,
	1924	MLX5_IB_ATTR_CREATE_FLOW_MATCHER);
	1925	uobj = uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE);
	1926	dev = mlx5_udata_to_mdev(&attrs->driver_udata);
	1927
	1928	if (get_dests(attrs, fs_matcher, &dest_id, &dest_type, &qp, &flags))
	1929	return -EINVAL;
	1930
	1931	if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS)
	1932	flow_act.action \|= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS;
	1933
	1934	if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)
	1935	flow_act.action \|= MLX5_FLOW_CONTEXT_ACTION_DROP;
	1936
	1937	len = uverbs_attr_get_uobjs_arr(attrs,
	1938	MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, &arr_flow_actions);
	1939	if (len) {
	1940	devx_obj = arr_flow_actions[0]->object;
	1941
	1942	if (uverbs_attr_is_valid(attrs,
	1943	MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET)) {
	1944
	1945	int num_offsets = uverbs_attr_ptr_get_array_size(
	1946	attrs,
	1947	MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET,
	1948	sizeof(u32));
	1949
	1950	if (num_offsets != 1)
	1951	return -EINVAL;
	1952
	1953	offset_attr = uverbs_attr_get_alloced_ptr(
	1954	attrs,
	1955	MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET);
	1956	offset = *offset_attr;
	1957	}
	1958
	1959	if (!is_flow_counter(devx_obj, offset, &counter_id))
	1960	return -EINVAL;
	1961
	1962	flow_act.action \|= MLX5_FLOW_CONTEXT_ACTION_COUNT;
	1963	}
	1964
	1965	cmd_in = uverbs_attr_get_alloced_ptr(
	1966	attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE);
	1967	inlen = uverbs_attr_get_len(attrs,
	1968	MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE);
	1969
	1970	uflow_res = flow_resources_alloc(MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS);
	1971	if (!uflow_res)
	1972	return -ENOMEM;
	1973
	1974	len = uverbs_attr_get_uobjs_arr(attrs,
	1975	MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, &arr_flow_actions);
	1976	for (i = 0; i < len; i++) {
	1977	struct mlx5_ib_flow_action *maction =
	1978	to_mflow_act(arr_flow_actions[i]->object);
	1979
	1980	ret = parse_flow_flow_action(maction, false, &flow_act);
	1981	if (ret)
	1982	goto err_out;
	1983	flow_resources_add(uflow_res, IB_FLOW_SPEC_ACTION_HANDLE,
	1984	arr_flow_actions[i]->object);
	1985	}
	1986
	1987	ret = uverbs_copy_from(&flow_context.flow_tag, attrs,
	1988	MLX5_IB_ATTR_CREATE_FLOW_TAG);
	1989	if (!ret) {
	1990	if (flow_context.flow_tag >= BIT(24)) {
	1991	ret = -EINVAL;
	1992	goto err_out;
	1993	}
	1994	flow_context.flags \|= FLOW_CONTEXT_HAS_TAG;
	1995	}
	1996
	1997	flow_handler =
	1998	raw_fs_rule_add(dev, fs_matcher, &flow_context, &flow_act,
	1999	counter_id, cmd_in, inlen, dest_id, dest_type);
	2000	if (IS_ERR(flow_handler)) {
	2001	ret = PTR_ERR(flow_handler);
	2002	goto err_out;
	2003	}
	2004
	2005	ib_set_flow(uobj, &flow_handler->ibflow, qp, &dev->ib_dev, uflow_res);
	2006
	2007	return 0;
	2008	err_out:
	2009	ib_uverbs_flow_resources_free(uflow_res);
	2010	return ret;
	2011	}
	2012
	2013	static int flow_matcher_cleanup(struct ib_uobject *uobject,
	2014	enum rdma_remove_reason why,
	2015	struct uverbs_attr_bundle *attrs)
	2016	{
	2017	struct mlx5_ib_flow_matcher *obj = uobject->object;
	2018	int ret;
	2019
	2020	ret = ib_destroy_usecnt(&obj->usecnt, why, uobject);
	2021	if (ret)
	2022	return ret;
	2023
	2024	kfree(obj);
	2025	return 0;
	2026	}
	2027
	2028	static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs,
	2029	struct mlx5_ib_flow_matcher *obj)
	2030	{
	2031	enum mlx5_ib_uapi_flow_table_type ft_type =
	2032	MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX;
	2033	u32 flags;
	2034	int err;
	2035
	2036	/* New users should use MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE and older
	2037	* users should switch to it. We leave this to not break userspace
	2038	*/
	2039	if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE) &&
	2040	uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS))
	2041	return -EINVAL;
	2042
	2043	if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE)) {
	2044	err = uverbs_get_const(&ft_type, attrs,
	2045	MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE);
	2046	if (err)
	2047	return err;
	2048
	2049	err = mlx5_ib_ft_type_to_namespace(ft_type, &obj->ns_type);
	2050	if (err)
	2051	return err;
	2052
	2053	return 0;
	2054	}
	2055
	2056	if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) {
	2057	err = uverbs_get_flags32(&flags, attrs,
	2058	MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,
	2059	IB_FLOW_ATTR_FLAGS_EGRESS);
	2060	if (err)
	2061	return err;
	2062
	2063	if (flags) {
	2064	mlx5_ib_ft_type_to_namespace(
	2065	MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX,
	2066	&obj->ns_type);
	2067	return 0;
	2068	}
	2069	}
	2070
	2071	obj->ns_type = MLX5_FLOW_NAMESPACE_BYPASS;
	2072
	2073	return 0;
	2074	}
	2075
	2076	static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)(
	2077	struct uverbs_attr_bundle *attrs)
	2078	{
	2079	struct ib_uobject *uobj = uverbs_attr_get_uobject(
	2080	attrs, MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE);
	2081	struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
	2082	struct mlx5_ib_flow_matcher *obj;
	2083	int err;
	2084
	2085	obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL);
	2086	if (!obj)
	2087	return -ENOMEM;
	2088
	2089	obj->mask_len = uverbs_attr_get_len(
	2090	attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK);
	2091	err = uverbs_copy_from(&obj->matcher_mask,
	2092	attrs,
	2093	MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK);
	2094	if (err)
	2095	goto end;
	2096
	2097	obj->flow_type = uverbs_attr_get_enum_id(
	2098	attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE);
	2099
	2100	if (obj->flow_type == MLX5_IB_FLOW_TYPE_NORMAL) {
	2101	err = uverbs_copy_from(&obj->priority,
	2102	attrs,
	2103	MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE);
	2104	if (err)
	2105	goto end;
	2106	}
	2107
	2108	err = uverbs_copy_from(&obj->match_criteria_enable,
	2109	attrs,
	2110	MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA);
	2111	if (err)
	2112	goto end;
	2113
	2114	err = mlx5_ib_matcher_ns(attrs, obj);
	2115	if (err)
	2116	goto end;
	2117
	2118	uobj->object = obj;
	2119	obj->mdev = dev->mdev;
	2120	atomic_set(&obj->usecnt, 0);
	2121	return 0;
	2122
	2123	end:
	2124	kfree(obj);
	2125	return err;
	2126	}
	2127
	2128	static struct ib_flow_action *
	2129	mlx5_ib_create_modify_header(struct mlx5_ib_dev *dev,
	2130	enum mlx5_ib_uapi_flow_table_type ft_type,
	2131	u8 num_actions, void *in)
	2132	{
	2133	enum mlx5_flow_namespace_type namespace;
	2134	struct mlx5_ib_flow_action *maction;
	2135	int ret;
	2136
	2137	ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace);
	2138	if (ret)
	2139	return ERR_PTR(-EINVAL);
	2140
	2141	maction = kzalloc(sizeof(*maction), GFP_KERNEL);
	2142	if (!maction)
	2143	return ERR_PTR(-ENOMEM);
	2144
	2145	maction->flow_action_raw.modify_hdr =
	2146	mlx5_modify_header_alloc(dev->mdev, namespace, num_actions, in);
	2147
	2148	if (IS_ERR(maction->flow_action_raw.modify_hdr)) {
	2149	ret = PTR_ERR(maction->flow_action_raw.modify_hdr);
	2150	kfree(maction);
	2151	return ERR_PTR(ret);
	2152	}
	2153	maction->flow_action_raw.sub_type =
	2154	MLX5_IB_FLOW_ACTION_MODIFY_HEADER;
	2155	maction->flow_action_raw.dev = dev;
	2156
	2157	return &maction->ib_action;
	2158	}
	2159
	2160	static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev *dev)
	2161	{
	2162	return MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
	2163	max_modify_header_actions) \|\|
	2164	MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev,
	2165	max_modify_header_actions) \|\|
	2166	MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev,
	2167	max_modify_header_actions);
	2168	}
	2169
	2170	static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)(
	2171	struct uverbs_attr_bundle *attrs)
	2172	{
	2173	struct ib_uobject *uobj = uverbs_attr_get_uobject(
	2174	attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE);
	2175	struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
	2176	enum mlx5_ib_uapi_flow_table_type ft_type;
	2177	struct ib_flow_action *action;
	2178	int num_actions;
	2179	void *in;
	2180	int ret;
	2181
	2182	if (!mlx5_ib_modify_header_supported(mdev))
	2183	return -EOPNOTSUPP;
	2184
	2185	in = uverbs_attr_get_alloced_ptr(attrs,
	2186	MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM);
	2187
	2188	num_actions = uverbs_attr_ptr_get_array_size(
	2189	attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM,
	2190	MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto));
	2191	if (num_actions < 0)
	2192	return num_actions;
	2193
	2194	ret = uverbs_get_const(&ft_type, attrs,
	2195	MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE);
	2196	if (ret)
	2197	return ret;
	2198	action = mlx5_ib_create_modify_header(mdev, ft_type, num_actions, in);
	2199	if (IS_ERR(action))
	2200	return PTR_ERR(action);
	2201
	2202	uverbs_flow_action_fill_action(action, uobj, &mdev->ib_dev,
	2203	IB_FLOW_ACTION_UNSPECIFIED);
	2204
	2205	return 0;
	2206	}
	2207
	2208	static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev *ibdev,
	2209	u8 packet_reformat_type,
	2210	u8 ft_type)
	2211	{
	2212	switch (packet_reformat_type) {
	2213	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL:
	2214	if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX)
	2215	return MLX5_CAP_FLOWTABLE(ibdev->mdev,
	2216	encap_general_header);
	2217	break;
	2218	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL:
	2219	if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX)
	2220	return MLX5_CAP_FLOWTABLE_NIC_TX(ibdev->mdev,
	2221	reformat_l2_to_l3_tunnel);
	2222	break;
	2223	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2:
	2224	if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX)
	2225	return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev,
	2226	reformat_l3_tunnel_to_l2);
	2227	break;
	2228	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2:
	2229	if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX)
	2230	return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, decap);
	2231	break;
	2232	default:
	2233	break;
	2234	}
	2235
	2236	return false;
	2237	}
	2238
	2239	static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt, u8 *prm_prt)
	2240	{
	2241	switch (dv_prt) {
	2242	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL:
	2243	*prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL;
	2244	break;
	2245	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2:
	2246	*prm_prt = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2;
	2247	break;
	2248	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL:
	2249	*prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
	2250	break;
	2251	default:
	2252	return -EINVAL;
	2253	}
	2254
	2255	return 0;
	2256	}
	2257
	2258	static int mlx5_ib_flow_action_create_packet_reformat_ctx(
	2259	struct mlx5_ib_dev *dev,
	2260	struct mlx5_ib_flow_action *maction,
	2261	u8 ft_type, u8 dv_prt,
	2262	void *in, size_t len)
	2263	{
	2264	enum mlx5_flow_namespace_type namespace;
	2265	u8 prm_prt;
	2266	int ret;
	2267
	2268	ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace);
	2269	if (ret)
	2270	return ret;
	2271
	2272	ret = mlx5_ib_dv_to_prm_packet_reforamt_type(dv_prt, &prm_prt);
	2273	if (ret)
	2274	return ret;
	2275
	2276	maction->flow_action_raw.pkt_reformat =
	2277	mlx5_packet_reformat_alloc(dev->mdev, prm_prt, len,
	2278	in, namespace);
	2279	if (IS_ERR(maction->flow_action_raw.pkt_reformat)) {
	2280	ret = PTR_ERR(maction->flow_action_raw.pkt_reformat);
	2281	return ret;
	2282	}
	2283
	2284	maction->flow_action_raw.sub_type =
	2285	MLX5_IB_FLOW_ACTION_PACKET_REFORMAT;
	2286	maction->flow_action_raw.dev = dev;
	2287
	2288	return 0;
	2289	}
	2290
	2291	static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)(
	2292	struct uverbs_attr_bundle *attrs)
	2293	{
	2294	struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
	2295	MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE);
	2296	struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
	2297	enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt;
	2298	enum mlx5_ib_uapi_flow_table_type ft_type;
	2299	struct mlx5_ib_flow_action *maction;
	2300	int ret;
	2301
	2302	ret = uverbs_get_const(&ft_type, attrs,
	2303	MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE);
	2304	if (ret)
	2305	return ret;
	2306
	2307	ret = uverbs_get_const(&dv_prt, attrs,
	2308	MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE);
	2309	if (ret)
	2310	return ret;
	2311
	2312	if (!mlx5_ib_flow_action_packet_reformat_valid(mdev, dv_prt, ft_type))
	2313	return -EOPNOTSUPP;
	2314
	2315	maction = kzalloc(sizeof(*maction), GFP_KERNEL);
	2316	if (!maction)
	2317	return -ENOMEM;
	2318
	2319	if (dv_prt ==
	2320	MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2) {
	2321	maction->flow_action_raw.sub_type =
	2322	MLX5_IB_FLOW_ACTION_DECAP;
	2323	maction->flow_action_raw.dev = mdev;
	2324	} else {
	2325	void *in;
	2326	int len;
	2327
	2328	in = uverbs_attr_get_alloced_ptr(attrs,
	2329	MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF);
	2330	if (IS_ERR(in)) {
	2331	ret = PTR_ERR(in);
	2332	goto free_maction;
	2333	}
	2334
	2335	len = uverbs_attr_get_len(attrs,
	2336	MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF);
	2337
	2338	ret = mlx5_ib_flow_action_create_packet_reformat_ctx(mdev,
	2339	maction, ft_type, dv_prt, in, len);
	2340	if (ret)
	2341	goto free_maction;
	2342	}
	2343
	2344	uverbs_flow_action_fill_action(&maction->ib_action, uobj, &mdev->ib_dev,
	2345	IB_FLOW_ACTION_UNSPECIFIED);
	2346	return 0;
	2347
	2348	free_maction:
	2349	kfree(maction);
	2350	return ret;
	2351	}
	2352
	2353	DECLARE_UVERBS_NAMED_METHOD(
	2354	MLX5_IB_METHOD_CREATE_FLOW,
	2355	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE,
	2356	UVERBS_OBJECT_FLOW,
	2357	UVERBS_ACCESS_NEW,
	2358	UA_MANDATORY),
	2359	UVERBS_ATTR_PTR_IN(
	2360	MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE,
	2361	UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)),
	2362	UA_MANDATORY,
	2363	UA_ALLOC_AND_COPY),
	2364	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_MATCHER,
	2365	MLX5_IB_OBJECT_FLOW_MATCHER,
	2366	UVERBS_ACCESS_READ,
	2367	UA_MANDATORY),
	2368	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_QP,
	2369	UVERBS_OBJECT_QP,
	2370	UVERBS_ACCESS_READ),
	2371	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX,
	2372	MLX5_IB_OBJECT_DEVX_OBJ,
	2373	UVERBS_ACCESS_READ),
	2374	UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS,
	2375	UVERBS_OBJECT_FLOW_ACTION,
	2376	UVERBS_ACCESS_READ, 1,
	2377	MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS,
	2378	UA_OPTIONAL),
	2379	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_TAG,
	2380	UVERBS_ATTR_TYPE(u32),
	2381	UA_OPTIONAL),
	2382	UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX,
	2383	MLX5_IB_OBJECT_DEVX_OBJ,
	2384	UVERBS_ACCESS_READ, 1, 1,
	2385	UA_OPTIONAL),
	2386	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET,
	2387	UVERBS_ATTR_MIN_SIZE(sizeof(u32)),
	2388	UA_OPTIONAL,
	2389	UA_ALLOC_AND_COPY),
	2390	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_FLAGS,
	2391	enum mlx5_ib_create_flow_flags,
	2392	UA_OPTIONAL));
	2393
	2394	DECLARE_UVERBS_NAMED_METHOD_DESTROY(
	2395	MLX5_IB_METHOD_DESTROY_FLOW,
	2396	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE,
	2397	UVERBS_OBJECT_FLOW,
	2398	UVERBS_ACCESS_DESTROY,
	2399	UA_MANDATORY));
	2400
	2401	ADD_UVERBS_METHODS(mlx5_ib_fs,
	2402	UVERBS_OBJECT_FLOW,
	2403	&UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW),
	2404	&UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW));
	2405
	2406	DECLARE_UVERBS_NAMED_METHOD(
	2407	MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER,
	2408	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE,
	2409	UVERBS_OBJECT_FLOW_ACTION,
	2410	UVERBS_ACCESS_NEW,
	2411	UA_MANDATORY),
	2412	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM,
	2413	UVERBS_ATTR_MIN_SIZE(MLX5_UN_SZ_BYTES(
	2414	set_add_copy_action_in_auto)),
	2415	UA_MANDATORY,
	2416	UA_ALLOC_AND_COPY),
	2417	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE,
	2418	enum mlx5_ib_uapi_flow_table_type,
	2419	UA_MANDATORY));
	2420
	2421	DECLARE_UVERBS_NAMED_METHOD(
	2422	MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT,
	2423	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE,
	2424	UVERBS_OBJECT_FLOW_ACTION,
	2425	UVERBS_ACCESS_NEW,
	2426	UA_MANDATORY),
	2427	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF,
	2428	UVERBS_ATTR_MIN_SIZE(1),
	2429	UA_ALLOC_AND_COPY,
	2430	UA_OPTIONAL),
	2431	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE,
	2432	enum mlx5_ib_uapi_flow_action_packet_reformat_type,
	2433	UA_MANDATORY),
	2434	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE,
	2435	enum mlx5_ib_uapi_flow_table_type,
	2436	UA_MANDATORY));
	2437
	2438	ADD_UVERBS_METHODS(
	2439	mlx5_ib_flow_actions,
	2440	UVERBS_OBJECT_FLOW_ACTION,
	2441	&UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER),
	2442	&UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT));
	2443
	2444	DECLARE_UVERBS_NAMED_METHOD(
	2445	MLX5_IB_METHOD_FLOW_MATCHER_CREATE,
	2446	UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE,
	2447	MLX5_IB_OBJECT_FLOW_MATCHER,
	2448	UVERBS_ACCESS_NEW,
	2449	UA_MANDATORY),
	2450	UVERBS_ATTR_PTR_IN(
	2451	MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK,
	2452	UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)),
	2453	UA_MANDATORY),
	2454	UVERBS_ATTR_ENUM_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE,
	2455	mlx5_ib_flow_type,
	2456	UA_MANDATORY),
	2457	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA,
	2458	UVERBS_ATTR_TYPE(u8),
	2459	UA_MANDATORY),
	2460	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,
	2461	enum ib_flow_flags,
	2462	UA_OPTIONAL),
	2463	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE,
	2464	enum mlx5_ib_uapi_flow_table_type,
	2465	UA_OPTIONAL));
	2466
	2467	DECLARE_UVERBS_NAMED_METHOD_DESTROY(
	2468	MLX5_IB_METHOD_FLOW_MATCHER_DESTROY,
	2469	UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE,
	2470	MLX5_IB_OBJECT_FLOW_MATCHER,
	2471	UVERBS_ACCESS_DESTROY,
	2472	UA_MANDATORY));
	2473
	2474	DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER,
	2475	UVERBS_TYPE_ALLOC_IDR(flow_matcher_cleanup),
	2476	&UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE),
	2477	&UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY));
	2478
	2479	const struct uapi_definition mlx5_ib_flow_defs[] = {
	2480	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
	2481	MLX5_IB_OBJECT_FLOW_MATCHER),
	2482	UAPI_DEF_CHAIN_OBJ_TREE(
	2483	UVERBS_OBJECT_FLOW,
	2484	&mlx5_ib_fs),
	2485	UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION,
	2486	&mlx5_ib_flow_actions),
	2487	{},
	2488	};
	2489
	2490	static const struct ib_device_ops flow_ops = {
	2491	.create_flow = mlx5_ib_create_flow,
	2492	.destroy_flow = mlx5_ib_destroy_flow,
	2493	.destroy_flow_action = mlx5_ib_destroy_flow_action,
	2494	};
	2495
	2496	static const struct ib_device_ops flow_ipsec_ops = {
	2497	.create_flow_action_esp = mlx5_ib_create_flow_action_esp,
	2498	.modify_flow_action_esp = mlx5_ib_modify_flow_action_esp,
	2499	};
	2500
	2501	int mlx5_ib_fs_init(struct mlx5_ib_dev *dev)
	2502	{
	2503	dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL);
	2504
	2505	if (!dev->flow_db)
	2506	return -ENOMEM;
	2507
	2508	mutex_init(&dev->flow_db->lock);
	2509
	2510	ib_set_device_ops(&dev->ib_dev, &flow_ops);
	2511	if (mlx5_accel_ipsec_device_caps(dev->mdev) &
	2512	MLX5_ACCEL_IPSEC_CAP_DEVICE)
	2513	ib_set_device_ops(&dev->ib_dev, &flow_ipsec_ops);
	2514
	2515	return 0;
	2516	}

drivers/infiniband/hw/mlx5/fs.h+29−0

@@ -0,0 +1,29 @@
	1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
	2	/*
	3	* Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
	4	*/
	5
	6	#ifndef _MLX5_IB_FS_H
	7	#define _MLX5_IB_FS_H
	8
	9	#include "mlx5_ib.h"
	10
	11	#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)
	12	int mlx5_ib_fs_init(struct mlx5_ib_dev *dev);
	13	#else
	14	static inline int mlx5_ib_fs_init(struct mlx5_ib_dev *dev)
	15	{
	16	dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL);
	17
	18	if (!dev->flow_db)
	19	return -ENOMEM;
	20
	21	mutex_init(&dev->flow_db->lock);
	22	return 0;
	23	}
	24	#endif
	25	static inline void mlx5_ib_fs_cleanup(struct mlx5_ib_dev *dev)
	26	{
	27	kfree(dev->flow_db);
	28	}
	29	#endif /* _MLX5_IB_FS_H */

drivers/infiniband/hw/mlx5/main.c+937−3443

@@ -1,33 +1,6 @@
		1	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
1	/*	2	/*
2	* Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.	3	* Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
3	*
4	* This software is available to you under a choice of one of two
5	* licenses. You may choose to be licensed under the terms of the GNU
6	* General Public License (GPL) Version 2, available from the file
7	* COPYING in the main directory of this source tree, or the
8	* OpenIB.org BSD license below:
9	*
10	* Redistribution and use in source and binary forms, with or
11	* without modification, are permitted provided that the following
12	* conditions are met:
13	*
14	* - Redistributions of source code must retain the above
15	* copyright notice, this list of conditions and the following
16	* disclaimer.
17	*
18	* - Redistributions in binary form must reproduce the above
19	* copyright notice, this list of conditions and the following
20	* disclaimer in the documentation and/or other materials
21	* provided with the distribution.
22	*
23	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30	* SOFTWARE.
31	*/	4	*/
32		5
33	#include <linux/debugfs.h>	6	#include <linux/debugfs.h>
@@ -59,10 +32,13 @@ · 25 unmodified lines
59	#include "mlx5_ib.h"	32	#include "mlx5_ib.h"
60	#include "ib_rep.h"	33	#include "ib_rep.h"
61	#include "cmd.h"	34	#include "cmd.h"
		35	#include "devx.h"
		36	#include "fs.h"
62	#include "srq.h"	37	#include "srq.h"
63	#include "qp.h"	38	#include "qp.h"
64	#include "wr.h"	39	#include "wr.h"
65	#include <linux/mlx5/fs_helpers.h>	40	#include "restrack.h"
		41	#include "counters.h"
66	#include <linux/mlx5/accel.h>	42	#include <linux/mlx5/accel.h>
67	#include <rdma/uverbs_std_types.h>	43	#include <rdma/uverbs_std_types.h>
68	#include <rdma/mlx5_user_ioctl_verbs.h>	44	#include <rdma/mlx5_user_ioctl_verbs.h>
@@ -311,9 +287,6 @@ struct mlx5_core_dev mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev ibdev, · 242 unmodified lines
311	*native_port_num = 1;	287	*native_port_num = 1;
312		288
313	port = &ibdev->port[ib_port_num - 1];	289	port = &ibdev->port[ib_port_num - 1];
314	if (!port)
315	return NULL;
316
317	spin_lock(&port->mp.mpi_lock);	290	spin_lock(&port->mp.mpi_lock);
318	mpi = ibdev->port[ib_port_num - 1].mp.mpi;	291	mpi = ibdev->port[ib_port_num - 1].mp.mpi;
319	if (mpi && !mpi->unaffiliate) {	292	if (mpi && !mpi->unaffiliate) {
@@ -1765,6 +1738,92 @@ static void mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev *dev, u32 tdn, · 1445 unmodified lines
1765	mlx5_ib_disable_lb(dev, true, false);	1738	mlx5_ib_disable_lb(dev, true, false);
1766	}	1739	}
1767		1740
		1741	static int set_ucontext_resp(struct ib_ucontext *uctx,
		1742	struct mlx5_ib_alloc_ucontext_resp *resp)
		1743	{
		1744	struct ib_device *ibdev = uctx->device;
		1745	struct mlx5_ib_dev *dev = to_mdev(ibdev);
		1746	struct mlx5_ib_ucontext *context = to_mucontext(uctx);
		1747	struct mlx5_bfreg_info *bfregi = &context->bfregi;
		1748	int err;
		1749
		1750	if (MLX5_CAP_GEN(dev->mdev, dump_fill_mkey)) {
		1751	err = mlx5_cmd_dump_fill_mkey(dev->mdev,
		1752	&resp->dump_fill_mkey);
		1753	if (err)
		1754	return err;
		1755	resp->comp_mask \|=
		1756	MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_DUMP_FILL_MKEY;
		1757	}
		1758
		1759	resp->qp_tab_size = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp);
		1760	if (dev->wc_support)
		1761	resp->bf_reg_size = 1 << MLX5_CAP_GEN(dev->mdev,
		1762	log_bf_reg_size);
		1763	resp->cache_line_size = cache_line_size();
		1764	resp->max_sq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq);
		1765	resp->max_rq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq);
		1766	resp->max_send_wqebb = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
		1767	resp->max_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
		1768	resp->max_srq_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
		1769	resp->cqe_version = context->cqe_version;
		1770	resp->log_uar_size = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
		1771	MLX5_ADAPTER_PAGE_SHIFT : PAGE_SHIFT;
		1772	resp->num_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
		1773	MLX5_CAP_GEN(dev->mdev,
		1774	num_of_uars_per_page) : 1;
		1775
		1776	if (mlx5_accel_ipsec_device_caps(dev->mdev) &
		1777	MLX5_ACCEL_IPSEC_CAP_DEVICE) {
		1778	if (mlx5_get_flow_namespace(dev->mdev,
		1779	MLX5_FLOW_NAMESPACE_EGRESS))
		1780	resp->flow_action_flags \|= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM;
		1781	if (mlx5_accel_ipsec_device_caps(dev->mdev) &
		1782	MLX5_ACCEL_IPSEC_CAP_REQUIRED_METADATA)
		1783	resp->flow_action_flags \|= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_REQ_METADATA;
		1784	if (MLX5_CAP_FLOWTABLE(dev->mdev, flow_table_properties_nic_receive.ft_field_support.outer_esp_spi))
		1785	resp->flow_action_flags \|= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_SPI_STEERING;
		1786	if (mlx5_accel_ipsec_device_caps(dev->mdev) &
		1787	MLX5_ACCEL_IPSEC_CAP_TX_IV_IS_ESN)
		1788	resp->flow_action_flags \|= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_TX_IV_IS_ESN;
		1789	/* MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_FULL_OFFLOAD is currently always 0 */
		1790	}
		1791
		1792	resp->tot_bfregs = bfregi->lib_uar_dyn ? 0 :
		1793	bfregi->total_num_bfregs - bfregi->num_dyn_bfregs;
		1794	resp->num_ports = dev->num_ports;
		1795	resp->cmds_supp_uhw \|= MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE \|
		1796	MLX5_USER_CMDS_SUPP_UHW_CREATE_AH;
		1797
		1798	if (mlx5_ib_port_link_layer(ibdev, 1) == IB_LINK_LAYER_ETHERNET) {
		1799	mlx5_query_min_inline(dev->mdev, &resp->eth_min_inline);
		1800	resp->eth_min_inline++;
		1801	}
		1802
		1803	if (dev->mdev->clock_info)
		1804	resp->clock_info_versions = BIT(MLX5_IB_CLOCK_INFO_V1);
		1805
		1806	/*
		1807	* We don't want to expose information from the PCI bar that is located
		1808	* after 4096 bytes, so if the arch only supports larger pages, let's
		1809	* pretend we don't support reading the HCA's core clock. This is also
		1810	* forced by mmap function.
		1811	*/
		1812	if (PAGE_SIZE <= 4096) {
		1813	resp->comp_mask \|=
		1814	MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET;
		1815	resp->hca_core_clock_offset =
		1816	offsetof(struct mlx5_init_seg,
		1817	internal_timer_h) % PAGE_SIZE;
		1818	}
		1819
		1820	if (MLX5_CAP_GEN(dev->mdev, ece_support))
		1821	resp->comp_mask \|= MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_ECE;
		1822
		1823	resp->num_dyn_bfregs = bfregi->num_dyn_bfregs;
		1824	return 0;
		1825	}
		1826
1768	static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,	1827	static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
1769	struct ib_udata *udata)	1828	struct ib_udata *udata)
1770	{	1829	{
@@ -1772,14 +1831,12 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx, · 1 unmodified line
1772	struct mlx5_ib_dev *dev = to_mdev(ibdev);	1831	struct mlx5_ib_dev *dev = to_mdev(ibdev);
1773	struct mlx5_ib_alloc_ucontext_req_v2 req = {};	1832	struct mlx5_ib_alloc_ucontext_req_v2 req = {};
1774	struct mlx5_ib_alloc_ucontext_resp resp = {};	1833	struct mlx5_ib_alloc_ucontext_resp resp = {};
1775	struct mlx5_core_dev *mdev = dev->mdev;
1776	struct mlx5_ib_ucontext *context = to_mucontext(uctx);	1834	struct mlx5_ib_ucontext *context = to_mucontext(uctx);
1777	struct mlx5_bfreg_info *bfregi;	1835	struct mlx5_bfreg_info *bfregi;
1778	int ver;	1836	int ver;
1779	int err;	1837	int err;
1780	size_t min_req_v2 = offsetof(struct mlx5_ib_alloc_ucontext_req_v2,	1838	size_t min_req_v2 = offsetof(struct mlx5_ib_alloc_ucontext_req_v2,
1781	max_cqe_version);	1839	max_cqe_version);
1782	u32 dump_fill_mkey;
1783	bool lib_uar_4k;	1840	bool lib_uar_4k;
1784	bool lib_uar_dyn;	1841	bool lib_uar_dyn;
1785		1842
@@ -1808,37 +1865,6 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx, · 22 unmodified lines
1808	if (req.num_low_latency_bfregs > req.total_num_bfregs - 1)	1865	if (req.num_low_latency_bfregs > req.total_num_bfregs - 1)
1809	return -EINVAL;	1866	return -EINVAL;
1810		1867
1811	resp.qp_tab_size = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp);
1812	if (dev->wc_support)
1813	resp.bf_reg_size = 1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size);
1814	resp.cache_line_size = cache_line_size();
1815	resp.max_sq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq);
1816	resp.max_rq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq);
1817	resp.max_send_wqebb = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
1818	resp.max_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
1819	resp.max_srq_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
1820	resp.cqe_version = min_t(__u8,
1821	(__u8)MLX5_CAP_GEN(dev->mdev, cqe_version),
1822	req.max_cqe_version);
1823	resp.log_uar_size = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1824	MLX5_ADAPTER_PAGE_SHIFT : PAGE_SHIFT;
1825	resp.num_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1826	MLX5_CAP_GEN(dev->mdev, num_of_uars_per_page) : 1;
1827	resp.response_length = min(offsetof(typeof(resp), response_length) +
1828	sizeof(resp.response_length), udata->outlen);
1829
1830	if (mlx5_accel_ipsec_device_caps(dev->mdev) & MLX5_ACCEL_IPSEC_CAP_DEVICE) {
1831	if (mlx5_get_flow_namespace(dev->mdev, MLX5_FLOW_NAMESPACE_EGRESS))
1832	resp.flow_action_flags \|= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM;
1833	if (mlx5_accel_ipsec_device_caps(dev->mdev) & MLX5_ACCEL_IPSEC_CAP_REQUIRED_METADATA)
1834	resp.flow_action_flags \|= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_REQ_METADATA;
1835	if (MLX5_CAP_FLOWTABLE(dev->mdev, flow_table_properties_nic_receive.ft_field_support.outer_esp_spi))
1836	resp.flow_action_flags \|= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_SPI_STEERING;
1837	if (mlx5_accel_ipsec_device_caps(dev->mdev) & MLX5_ACCEL_IPSEC_CAP_TX_IV_IS_ESN)
1838	resp.flow_action_flags \|= MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_TX_IV_IS_ESN;
1839	/* MLX5_USER_ALLOC_UCONTEXT_FLOW_ACTION_FLAGS_ESP_AES_GCM_FULL_OFFLOAD is currently always 0 */
1840	}
1841
1842	lib_uar_4k = req.lib_caps & MLX5_LIB_CAP_4K_UAR;	1868	lib_uar_4k = req.lib_caps & MLX5_LIB_CAP_4K_UAR;
1843	lib_uar_dyn = req.lib_caps & MLX5_LIB_CAP_DYN_UAR;	1869	lib_uar_dyn = req.lib_caps & MLX5_LIB_CAP_DYN_UAR;
1844	bfregi = &context->bfregi;	1870	bfregi = &context->bfregi;
@@ -1887,87 +1913,24 @@ uar_done: · 42 unmodified lines
1887	if (err)	1913	if (err)
1888	goto out_devx;	1914	goto out_devx;
1889		1915
1890	if (MLX5_CAP_GEN(dev->mdev, dump_fill_mkey)) {
1891	err = mlx5_cmd_dump_fill_mkey(dev->mdev, &dump_fill_mkey);
1892	if (err)
1893	goto out_mdev;
1894	}
1895
1896	INIT_LIST_HEAD(&context->db_page_list);	1916	INIT_LIST_HEAD(&context->db_page_list);
1897	mutex_init(&context->db_page_mutex);	1917	mutex_init(&context->db_page_mutex);
1898		1918
1899	resp.tot_bfregs = lib_uar_dyn ? 0 : req.total_num_bfregs;	1919	context->cqe_version = min_t(__u8,
1900	resp.num_ports = dev->num_ports;	1920	(__u8)MLX5_CAP_GEN(dev->mdev, cqe_version),
1901		1921	req.max_cqe_version);
1902	if (offsetofend(typeof(resp), cqe_version) <= udata->outlen)
1903	resp.response_length += sizeof(resp.cqe_version);
1904
1905	if (offsetofend(typeof(resp), cmds_supp_uhw) <= udata->outlen) {
1906	resp.cmds_supp_uhw \|= MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE \|
1907	MLX5_USER_CMDS_SUPP_UHW_CREATE_AH;
1908	resp.response_length += sizeof(resp.cmds_supp_uhw);
1909	}
1910
1911	if (offsetofend(typeof(resp), eth_min_inline) <= udata->outlen) {
1912	if (mlx5_ib_port_link_layer(ibdev, 1) == IB_LINK_LAYER_ETHERNET) {
1913	mlx5_query_min_inline(dev->mdev, &resp.eth_min_inline);
1914	resp.eth_min_inline++;
1915	}
1916	resp.response_length += sizeof(resp.eth_min_inline);
1917	}
1918
1919	if (offsetofend(typeof(resp), clock_info_versions) <= udata->outlen) {
1920	if (mdev->clock_info)
1921	resp.clock_info_versions = BIT(MLX5_IB_CLOCK_INFO_V1);
1922	resp.response_length += sizeof(resp.clock_info_versions);
1923	}
1924
1925	/*
1926	* We don't want to expose information from the PCI bar that is located
1927	* after 4096 bytes, so if the arch only supports larger pages, let's
1928	* pretend we don't support reading the HCA's core clock. This is also
1929	* forced by mmap function.
1930	*/
1931	if (offsetofend(typeof(resp), hca_core_clock_offset) <= udata->outlen) {
1932	if (PAGE_SIZE <= 4096) {
1933	resp.comp_mask \|=
1934	MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET;
1935	resp.hca_core_clock_offset =
1936	offsetof(struct mlx5_init_seg, internal_timer_h) % PAGE_SIZE;
1937	}
1938	resp.response_length += sizeof(resp.hca_core_clock_offset);
1939	}
1940
1941	if (offsetofend(typeof(resp), log_uar_size) <= udata->outlen)
1942	resp.response_length += sizeof(resp.log_uar_size);
1943
1944	if (offsetofend(typeof(resp), num_uars_per_page) <= udata->outlen)
1945	resp.response_length += sizeof(resp.num_uars_per_page);
1946
1947	if (offsetofend(typeof(resp), num_dyn_bfregs) <= udata->outlen) {
1948	resp.num_dyn_bfregs = bfregi->num_dyn_bfregs;
1949	resp.response_length += sizeof(resp.num_dyn_bfregs);
1950	}
1951
1952	if (offsetofend(typeof(resp), dump_fill_mkey) <= udata->outlen) {
1953	if (MLX5_CAP_GEN(dev->mdev, dump_fill_mkey)) {
1954	resp.dump_fill_mkey = dump_fill_mkey;
1955	resp.comp_mask \|=
1956	MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_DUMP_FILL_MKEY;
1957	}
1958	resp.response_length += sizeof(resp.dump_fill_mkey);
1959	}
1960		1922
1961	if (MLX5_CAP_GEN(dev->mdev, ece_support))	1923	err = set_ucontext_resp(uctx, &resp);
1962	resp.comp_mask \|= MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_ECE;	1924	if (err)
		1925	goto out_mdev;
1963		1926
		1927	resp.response_length = min(udata->outlen, sizeof(resp));
1964	err = ib_copy_to_udata(udata, &resp, resp.response_length);	1928	err = ib_copy_to_udata(udata, &resp, resp.response_length);
1965	if (err)	1929	if (err)
1966	goto out_mdev;	1930	goto out_mdev;
1967		1931
1968	bfregi->ver = ver;	1932	bfregi->ver = ver;
1969	bfregi->num_low_latency_bfregs = req.num_low_latency_bfregs;	1933	bfregi->num_low_latency_bfregs = req.num_low_latency_bfregs;
1970	context->cqe_version = resp.cqe_version;
1971	context->lib_caps = req.lib_caps;	1934	context->lib_caps = req.lib_caps;
1972	print_lib_caps(dev, context->lib_caps);	1935	print_lib_caps(dev, context->lib_caps);
1973		1936
@@ -2000,6 +1963,29 @@ out_ctx: · 26 unmodified lines
2000	return err;	1963	return err;
2001	}	1964	}
2002		1965
		1966	static int mlx5_ib_query_ucontext(struct ib_ucontext *ibcontext,
		1967	struct uverbs_attr_bundle *attrs)
		1968	{
		1969	struct mlx5_ib_alloc_ucontext_resp uctx_resp = {};
		1970	int ret;
		1971
		1972	ret = set_ucontext_resp(ibcontext, &uctx_resp);
		1973	if (ret)
		1974	return ret;
		1975
		1976	uctx_resp.response_length =
		1977	min_t(size_t,
		1978	uverbs_attr_get_len(attrs,
		1979	MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX),
		1980	sizeof(uctx_resp));
		1981
		1982	ret = uverbs_copy_to_struct_or_zero(attrs,
		1983	MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX,
		1984	&uctx_resp,
		1985	sizeof(uctx_resp));
		1986	return ret;
		1987	}
		1988
2003	static void mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)	1989	static void mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
2004	{	1990	{
2005	struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);	1991	struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
@@ -2591,3205 +2577,786 @@ static void mlx5_ib_dealloc_pd(struct ib_pd pd, struct ib_udata udata) · 585 unmodified lines
2591	mlx5_cmd_dealloc_pd(mdev->mdev, mpd->pdn, mpd->uid);	2577	mlx5_cmd_dealloc_pd(mdev->mdev, mpd->pdn, mpd->uid);
2592	}	2578	}
2593		2579
2594	enum {	2580	static int mlx5_ib_mcg_attach(struct ib_qp ibqp, union ib_gid gid, u16 lid)
2595	MATCH_CRITERIA_ENABLE_OUTER_BIT,
2596	MATCH_CRITERIA_ENABLE_MISC_BIT,
2597	MATCH_CRITERIA_ENABLE_INNER_BIT,
2598	MATCH_CRITERIA_ENABLE_MISC2_BIT
2599	};
2600
2601	#define HEADER_IS_ZERO(match_criteria, headers) \
2602	!(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \
2603	0, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
2604
2605	static u8 get_match_criteria_enable(u32 *match_criteria)
2606	{	2581	{
2607	u8 match_criteria_enable;	2582	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
2608		2583	struct mlx5_ib_qp *mqp = to_mqp(ibqp);
2609	match_criteria_enable =	2584	int err;
2610	(!HEADER_IS_ZERO(match_criteria, outer_headers)) <<	2585	u16 uid;
2611	MATCH_CRITERIA_ENABLE_OUTER_BIT;
2612	match_criteria_enable \|=
2613	(!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
2614	MATCH_CRITERIA_ENABLE_MISC_BIT;
2615	match_criteria_enable \|=
2616	(!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
2617	MATCH_CRITERIA_ENABLE_INNER_BIT;
2618	match_criteria_enable \|=
2619	(!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
2620	MATCH_CRITERIA_ENABLE_MISC2_BIT;
2621		2586
2622	return match_criteria_enable;	2587	uid = ibqp->pd ?
2623	}	2588	to_mpd(ibqp->pd)->uid : 0;
2624		2589
2625	static int set_proto(void outer_c, void outer_v, u8 mask, u8 val)	2590	if (mqp->flags & IB_QP_CREATE_SOURCE_QPN) {
2626	{	2591	mlx5_ib_dbg(dev, "Attaching a multi cast group to underlay QP is not supported\n");
2627	u8 entry_mask;	2592	return -EOPNOTSUPP;
2628	u8 entry_val;	2593	}
2629	int err = 0;
2630		2594
2631	if (!mask)	2595	err = mlx5_cmd_attach_mcg(dev->mdev, gid, ibqp->qp_num, uid);
2632	goto out;	2596	if (err)
		2597	mlx5_ib_warn(dev, "failed attaching QPN 0x%x, MGID %pI6\n",
		2598	ibqp->qp_num, gid->raw);
2633		2599
2634	entry_mask = MLX5_GET(fte_match_set_lyr_2_4, outer_c,
2635	ip_protocol);
2636	entry_val = MLX5_GET(fte_match_set_lyr_2_4, outer_v,
2637	ip_protocol);
2638	if (!entry_mask) {
2639	MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask);
2640	MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val);
2641	goto out;
2642	}
2643	/* Don't override existing ip protocol */
2644	if (mask != entry_mask \|\| val != entry_val)
2645	err = -EINVAL;
2646	out:
2647	return err;	2600	return err;
2648	}	2601	}
2649		2602
2650	static void set_flow_label(void misc_c, void misc_v, u32 mask, u32 val,	2603	static int mlx5_ib_mcg_detach(struct ib_qp ibqp, union ib_gid gid, u16 lid)
2651	bool inner)
2652	{	2604	{
2653	if (inner) {	2605	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
2654	MLX5_SET(fte_match_set_misc,	2606	int err;
2655	misc_c, inner_ipv6_flow_label, mask);	2607	u16 uid;
2656	MLX5_SET(fte_match_set_misc,
2657	misc_v, inner_ipv6_flow_label, val);
2658	} else {
2659	MLX5_SET(fte_match_set_misc,
2660	misc_c, outer_ipv6_flow_label, mask);
2661	MLX5_SET(fte_match_set_misc,
2662	misc_v, outer_ipv6_flow_label, val);
2663	}
2664	}
2665		2608
2666	static void set_tos(void outer_c, void outer_v, u8 mask, u8 val)	2609	uid = ibqp->pd ?
2667	{	2610	to_mpd(ibqp->pd)->uid : 0;
2668	MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask);	2611	err = mlx5_cmd_detach_mcg(dev->mdev, gid, ibqp->qp_num, uid);
2669	MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val);	2612	if (err)
2670	MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2);	2613	mlx5_ib_warn(dev, "failed detaching QPN 0x%x, MGID %pI6\n",
2671	MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2);	2614	ibqp->qp_num, gid->raw);
		2615
		2616	return err;
2672	}	2617	}
2673		2618
2674	static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask)	2619	static int init_node_data(struct mlx5_ib_dev *dev)
2675	{	2620	{
2676	if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) &&	2621	int err;
2677	!(field_support & MLX5_FIELD_SUPPORT_MPLS_LABEL))
2678	return -EOPNOTSUPP;
2679
2680	if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) &&
2681	!(field_support & MLX5_FIELD_SUPPORT_MPLS_EXP))
2682	return -EOPNOTSUPP;
2683
2684	if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) &&
2685	!(field_support & MLX5_FIELD_SUPPORT_MPLS_S_BOS))
2686	return -EOPNOTSUPP;
2687		2622
2688	if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) &&	2623	err = mlx5_query_node_desc(dev, dev->ib_dev.node_desc);
2689	!(field_support & MLX5_FIELD_SUPPORT_MPLS_TTL))	2624	if (err)
2690	return -EOPNOTSUPP;	2625	return err;
2691		2626
2692	return 0;	2627	dev->mdev->rev_id = dev->mdev->pdev->revision;
2693	}
2694		2628
2695	#define LAST_ETH_FIELD vlan_tag	2629	return mlx5_query_node_guid(dev, &dev->ib_dev.node_guid);
2696	#define LAST_IB_FIELD sl
2697	#define LAST_IPV4_FIELD tos
2698	#define LAST_IPV6_FIELD traffic_class
2699	#define LAST_TCP_UDP_FIELD src_port
2700	#define LAST_TUNNEL_FIELD tunnel_id
2701	#define LAST_FLOW_TAG_FIELD tag_id
2702	#define LAST_DROP_FIELD size
2703	#define LAST_COUNTERS_FIELD counters
2704
2705	/* Field is the last supported field */
2706	#define FIELDS_NOT_SUPPORTED(filter, field)\
2707	memchr_inv((void *)&filter.field +\
2708	sizeof(filter.field), 0,\
2709	sizeof(filter) -\
2710	offsetof(typeof(filter), field) -\
2711	sizeof(filter.field))
2712
2713	int parse_flow_flow_action(struct mlx5_ib_flow_action *maction,
2714	bool is_egress,
2715	struct mlx5_flow_act *action)
2716	{
2717
2718	switch (maction->ib_action.type) {
2719	case IB_FLOW_ACTION_ESP:
2720	if (action->action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT \|
2721	MLX5_FLOW_CONTEXT_ACTION_DECRYPT))
2722	return -EINVAL;
2723	/* Currently only AES_GCM keymat is supported by the driver */
2724	action->esp_id = (uintptr_t)maction->esp_aes_gcm.ctx;
2725	action->action \|= is_egress ?
2726	MLX5_FLOW_CONTEXT_ACTION_ENCRYPT :
2727	MLX5_FLOW_CONTEXT_ACTION_DECRYPT;
2728	return 0;
2729	case IB_FLOW_ACTION_UNSPECIFIED:
2730	if (maction->flow_action_raw.sub_type ==
2731	MLX5_IB_FLOW_ACTION_MODIFY_HEADER) {
2732	if (action->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
2733	return -EINVAL;
2734	action->action \|= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
2735	action->modify_hdr =
2736	maction->flow_action_raw.modify_hdr;
2737	return 0;
2738	}
2739	if (maction->flow_action_raw.sub_type ==
2740	MLX5_IB_FLOW_ACTION_DECAP) {
2741	if (action->action & MLX5_FLOW_CONTEXT_ACTION_DECAP)
2742	return -EINVAL;
2743	action->action \|= MLX5_FLOW_CONTEXT_ACTION_DECAP;
2744	return 0;
2745	}
2746	if (maction->flow_action_raw.sub_type ==
2747	MLX5_IB_FLOW_ACTION_PACKET_REFORMAT) {
2748	if (action->action &
2749	MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT)
2750	return -EINVAL;
2751	action->action \|=
2752	MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
2753	action->pkt_reformat =
2754	maction->flow_action_raw.pkt_reformat;
2755	return 0;
2756	}
2757	/* fall through */
2758	default:
2759	return -EOPNOTSUPP;
2760	}
2761	}	2630	}
2762		2631
2763	static int parse_flow_attr(struct mlx5_core_dev *mdev,	2632	static ssize_t fw_pages_show(struct device *device,
2764	struct mlx5_flow_spec *spec,	2633	struct device_attribute attr, char buf)
2765	const union ib_flow_spec *ib_spec,	2634	{
2766	const struct ib_flow_attr *flow_attr,	2635	struct mlx5_ib_dev *dev =
2767	struct mlx5_flow_act *action, u32 prev_type)	2636	rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2768	{
2769	struct mlx5_flow_context *flow_context = &spec->flow_context;
2770	u32 *match_c = spec->match_criteria;
2771	u32 *match_v = spec->match_value;
2772	void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c,
2773	misc_parameters);
2774	void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v,
2775	misc_parameters);
2776	void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c,
2777	misc_parameters_2);
2778	void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v,
2779	misc_parameters_2);
2780	void *headers_c;
2781	void *headers_v;
2782	int match_ipv;
2783	int ret;
2784		2637
2785	if (ib_spec->type & IB_FLOW_SPEC_INNER) {	2638	return sprintf(buf, "%d\n", dev->mdev->priv.fw_pages);
2786	headers_c = MLX5_ADDR_OF(fte_match_param, match_c,	2639	}
2787	inner_headers);	2640	static DEVICE_ATTR_RO(fw_pages);
2788	headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
2789	inner_headers);
2790	match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
2791	ft_field_support.inner_ip_version);
2792	} else {
2793	headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
2794	outer_headers);
2795	headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
2796	outer_headers);
2797	match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
2798	ft_field_support.outer_ip_version);
2799	}
2800		2641
2801	switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) {	2642	static ssize_t reg_pages_show(struct device *device,
2802	case IB_FLOW_SPEC_ETH:	2643	struct device_attribute attr, char buf)
2803	if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD))	2644	{
2804	return -EOPNOTSUPP;	2645	struct mlx5_ib_dev *dev =
		2646	rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2805		2647
2806	ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,	2648	return sprintf(buf, "%d\n", atomic_read(&dev->mdev->priv.reg_pages));
2807	dmac_47_16),	2649	}
2808	ib_spec->eth.mask.dst_mac);	2650	static DEVICE_ATTR_RO(reg_pages);
2809	ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2810	dmac_47_16),
2811	ib_spec->eth.val.dst_mac);
2812
2813	ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
2814	smac_47_16),
2815	ib_spec->eth.mask.src_mac);
2816	ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2817	smac_47_16),
2818	ib_spec->eth.val.src_mac);
2819
2820	if (ib_spec->eth.mask.vlan_tag) {
2821	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2822	cvlan_tag, 1);
2823	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2824	cvlan_tag, 1);
2825
2826	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2827	first_vid, ntohs(ib_spec->eth.mask.vlan_tag));
2828	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2829	first_vid, ntohs(ib_spec->eth.val.vlan_tag));
2830
2831	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2832	first_cfi,
2833	ntohs(ib_spec->eth.mask.vlan_tag) >> 12);
2834	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2835	first_cfi,
2836	ntohs(ib_spec->eth.val.vlan_tag) >> 12);
2837
2838	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2839	first_prio,
2840	ntohs(ib_spec->eth.mask.vlan_tag) >> 13);
2841	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2842	first_prio,
2843	ntohs(ib_spec->eth.val.vlan_tag) >> 13);
2844	}
2845	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2846	ethertype, ntohs(ib_spec->eth.mask.ether_type));
2847	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2848	ethertype, ntohs(ib_spec->eth.val.ether_type));
2849	break;
2850	case IB_FLOW_SPEC_IPV4:
2851	if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD))
2852	return -EOPNOTSUPP;
2853		2651
2854	if (match_ipv) {	2652	static ssize_t hca_type_show(struct device *device,
2855	MLX5_SET(fte_match_set_lyr_2_4, headers_c,	2653	struct device_attribute attr, char buf)
2856	ip_version, 0xf);	2654	{
2857	MLX5_SET(fte_match_set_lyr_2_4, headers_v,	2655	struct mlx5_ib_dev *dev =
2858	ip_version, MLX5_FS_IPV4_VERSION);	2656	rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2859	} else {
2860	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2861	ethertype, 0xffff);
2862	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2863	ethertype, ETH_P_IP);
2864	}
2865		2657
2866	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,	2658	return sprintf(buf, "MT%d\n", dev->mdev->pdev->device);
2867	src_ipv4_src_ipv6.ipv4_layout.ipv4),	2659	}
2868	&ib_spec->ipv4.mask.src_ip,	2660	static DEVICE_ATTR_RO(hca_type);
2869	sizeof(ib_spec->ipv4.mask.src_ip));
2870	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2871	src_ipv4_src_ipv6.ipv4_layout.ipv4),
2872	&ib_spec->ipv4.val.src_ip,
2873	sizeof(ib_spec->ipv4.val.src_ip));
2874	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
2875	dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
2876	&ib_spec->ipv4.mask.dst_ip,
2877	sizeof(ib_spec->ipv4.mask.dst_ip));
2878	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2879	dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
2880	&ib_spec->ipv4.val.dst_ip,
2881	sizeof(ib_spec->ipv4.val.dst_ip));
2882
2883	set_tos(headers_c, headers_v,
2884	ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos);
2885
2886	if (set_proto(headers_c, headers_v,
2887	ib_spec->ipv4.mask.proto,
2888	ib_spec->ipv4.val.proto))
2889	return -EINVAL;
2890	break;
2891	case IB_FLOW_SPEC_IPV6:
2892	if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD))
2893	return -EOPNOTSUPP;
2894		2661
2895	if (match_ipv) {	2662	static ssize_t hw_rev_show(struct device *device,
2896	MLX5_SET(fte_match_set_lyr_2_4, headers_c,	2663	struct device_attribute attr, char buf)
2897	ip_version, 0xf);	2664	{
2898	MLX5_SET(fte_match_set_lyr_2_4, headers_v,	2665	struct mlx5_ib_dev *dev =
2899	ip_version, MLX5_FS_IPV6_VERSION);	2666	rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2900	} else {
2901	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2902	ethertype, 0xffff);
2903	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2904	ethertype, ETH_P_IPV6);
2905	}
2906		2667
2907	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,	2668	return sprintf(buf, "%x\n", dev->mdev->rev_id);
2908	src_ipv4_src_ipv6.ipv6_layout.ipv6),	2669	}
2909	&ib_spec->ipv6.mask.src_ip,	2670	static DEVICE_ATTR_RO(hw_rev);
2910	sizeof(ib_spec->ipv6.mask.src_ip));
2911	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2912	src_ipv4_src_ipv6.ipv6_layout.ipv6),
2913	&ib_spec->ipv6.val.src_ip,
2914	sizeof(ib_spec->ipv6.val.src_ip));
2915	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
2916	dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
2917	&ib_spec->ipv6.mask.dst_ip,
2918	sizeof(ib_spec->ipv6.mask.dst_ip));
2919	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2920	dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
2921	&ib_spec->ipv6.val.dst_ip,
2922	sizeof(ib_spec->ipv6.val.dst_ip));
2923
2924	set_tos(headers_c, headers_v,
2925	ib_spec->ipv6.mask.traffic_class,
2926	ib_spec->ipv6.val.traffic_class);
2927
2928	if (set_proto(headers_c, headers_v,
2929	ib_spec->ipv6.mask.next_hdr,
2930	ib_spec->ipv6.val.next_hdr))
2931	return -EINVAL;
2932		2671
2933	set_flow_label(misc_params_c, misc_params_v,	2672	static ssize_t board_id_show(struct device *device,
2934	ntohl(ib_spec->ipv6.mask.flow_label),	2673	struct device_attribute attr, char buf)
2935	ntohl(ib_spec->ipv6.val.flow_label),	2674	{
2936	ib_spec->type & IB_FLOW_SPEC_INNER);	2675	struct mlx5_ib_dev *dev =
2937	break;	2676	rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2938	case IB_FLOW_SPEC_ESP:
2939	if (ib_spec->esp.mask.seq)
2940	return -EOPNOTSUPP;
2941		2677
2942	MLX5_SET(fte_match_set_misc, misc_params_c, outer_esp_spi,	2678	return sprintf(buf, "%.*s\n", MLX5_BOARD_ID_LEN,
2943	ntohl(ib_spec->esp.mask.spi));	2679	dev->mdev->board_id);
2944	MLX5_SET(fte_match_set_misc, misc_params_v, outer_esp_spi,	2680	}
2945	ntohl(ib_spec->esp.val.spi));	2681	static DEVICE_ATTR_RO(board_id);
2946	break;
2947	case IB_FLOW_SPEC_TCP:
2948	if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
2949	LAST_TCP_UDP_FIELD))
2950	return -EOPNOTSUPP;
2951		2682
2952	if (set_proto(headers_c, headers_v, 0xff, IPPROTO_TCP))	2683	static struct attribute *mlx5_class_attributes[] = {
2953	return -EINVAL;	2684	&dev_attr_hw_rev.attr,
		2685	&dev_attr_hca_type.attr,
		2686	&dev_attr_board_id.attr,
		2687	&dev_attr_fw_pages.attr,
		2688	&dev_attr_reg_pages.attr,
		2689	NULL,
		2690	};
2954		2691
2955	MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport,	2692	static const struct attribute_group mlx5_attr_group = {
2956	ntohs(ib_spec->tcp_udp.mask.src_port));	2693	.attrs = mlx5_class_attributes,
2957	MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,	2694	};
2958	ntohs(ib_spec->tcp_udp.val.src_port));
2959		2695
2960	MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport,	2696	static void pkey_change_handler(struct work_struct *work)
2961	ntohs(ib_spec->tcp_udp.mask.dst_port));	2697	{
2962	MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,	2698	struct mlx5_ib_port_resources *ports =
2963	ntohs(ib_spec->tcp_udp.val.dst_port));	2699	container_of(work, struct mlx5_ib_port_resources,
2964	break;	2700	pkey_change_work);
2965	case IB_FLOW_SPEC_UDP:
2966	if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
2967	LAST_TCP_UDP_FIELD))
2968	return -EOPNOTSUPP;
2969		2701
2970	if (set_proto(headers_c, headers_v, 0xff, IPPROTO_UDP))	2702	mutex_lock(&ports->devr->mutex);
2971	return -EINVAL;	2703	mlx5_ib_gsi_pkey_change(ports->gsi);
		2704	mutex_unlock(&ports->devr->mutex);
		2705	}
2972		2706
2973	MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport,	2707	static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev)
2974	ntohs(ib_spec->tcp_udp.mask.src_port));	2708	{
2975	MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,	2709	struct mlx5_ib_qp *mqp;
2976	ntohs(ib_spec->tcp_udp.val.src_port));	2710	struct mlx5_ib_cq send_mcq, recv_mcq;
		2711	struct mlx5_core_cq *mcq;
		2712	struct list_head cq_armed_list;
		2713	unsigned long flags_qp;
		2714	unsigned long flags_cq;
		2715	unsigned long flags;
2977		2716
2978	MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport,	2717	INIT_LIST_HEAD(&cq_armed_list);
2979	ntohs(ib_spec->tcp_udp.mask.dst_port));
2980	MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
2981	ntohs(ib_spec->tcp_udp.val.dst_port));
2982	break;
2983	case IB_FLOW_SPEC_GRE:
2984	if (ib_spec->gre.mask.c_ks_res0_ver)
2985	return -EOPNOTSUPP;
2986		2718
2987	if (set_proto(headers_c, headers_v, 0xff, IPPROTO_GRE))	2719	/* Go over qp list reside on that ibdev, sync with create/destroy qp.*/
2988	return -EINVAL;	2720	spin_lock_irqsave(&ibdev->reset_flow_resource_lock, flags);
2989		2721	list_for_each_entry(mqp, &ibdev->qp_list, qps_list) {
2990	MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,	2722	spin_lock_irqsave(&mqp->sq.lock, flags_qp);
2991	0xff);	2723	if (mqp->sq.tail != mqp->sq.head) {
2992	MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,	2724	send_mcq = to_mcq(mqp->ibqp.send_cq);
2993	IPPROTO_GRE);	2725	spin_lock_irqsave(&send_mcq->lock, flags_cq);
2994		2726	if (send_mcq->mcq.comp &&
2995	MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol,	2727	mqp->ibqp.send_cq->comp_handler) {
2996	ntohs(ib_spec->gre.mask.protocol));	2728	if (!send_mcq->mcq.reset_notify_added) {
2997	MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol,	2729	send_mcq->mcq.reset_notify_added = 1;
2998	ntohs(ib_spec->gre.val.protocol));	2730	list_add_tail(&send_mcq->mcq.reset_notify,
2999		2731	&cq_armed_list);
3000	memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c,	2732	}
3001	gre_key.nvgre.hi),
3002	&ib_spec->gre.mask.key,
3003	sizeof(ib_spec->gre.mask.key));
3004	memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v,
3005	gre_key.nvgre.hi),
3006	&ib_spec->gre.val.key,
3007	sizeof(ib_spec->gre.val.key));
3008	break;
3009	case IB_FLOW_SPEC_MPLS:
3010	switch (prev_type) {
3011	case IB_FLOW_SPEC_UDP:
3012	if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
3013	ft_field_support.outer_first_mpls_over_udp),
3014	&ib_spec->mpls.mask.tag))
3015	return -EOPNOTSUPP;
3016
3017	memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
3018	outer_first_mpls_over_udp),
3019	&ib_spec->mpls.val.tag,
3020	sizeof(ib_spec->mpls.val.tag));
3021	memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
3022	outer_first_mpls_over_udp),
3023	&ib_spec->mpls.mask.tag,
3024	sizeof(ib_spec->mpls.mask.tag));
3025	break;
3026	case IB_FLOW_SPEC_GRE:
3027	if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
3028	ft_field_support.outer_first_mpls_over_gre),
3029	&ib_spec->mpls.mask.tag))
3030	return -EOPNOTSUPP;
3031
3032	memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
3033	outer_first_mpls_over_gre),
3034	&ib_spec->mpls.val.tag,
3035	sizeof(ib_spec->mpls.val.tag));
3036	memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
3037	outer_first_mpls_over_gre),
3038	&ib_spec->mpls.mask.tag,
3039	sizeof(ib_spec->mpls.mask.tag));
3040	break;
3041	default:
3042	if (ib_spec->type & IB_FLOW_SPEC_INNER) {
3043	if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
3044	ft_field_support.inner_first_mpls),
3045	&ib_spec->mpls.mask.tag))
3046	return -EOPNOTSUPP;
3047
3048	memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
3049	inner_first_mpls),
3050	&ib_spec->mpls.val.tag,
3051	sizeof(ib_spec->mpls.val.tag));
3052	memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
3053	inner_first_mpls),
3054	&ib_spec->mpls.mask.tag,
3055	sizeof(ib_spec->mpls.mask.tag));
3056	} else {
3057	if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
3058	ft_field_support.outer_first_mpls),
3059	&ib_spec->mpls.mask.tag))
3060	return -EOPNOTSUPP;
3061
3062	memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
3063	outer_first_mpls),
3064	&ib_spec->mpls.val.tag,
3065	sizeof(ib_spec->mpls.val.tag));
3066	memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
3067	outer_first_mpls),
3068	&ib_spec->mpls.mask.tag,
3069	sizeof(ib_spec->mpls.mask.tag));
3070	}	2733	}
		2734	spin_unlock_irqrestore(&send_mcq->lock, flags_cq);
3071	}	2735	}
3072	break;	2736	spin_unlock_irqrestore(&mqp->sq.lock, flags_qp);
3073	case IB_FLOW_SPEC_VXLAN_TUNNEL:	2737	spin_lock_irqsave(&mqp->rq.lock, flags_qp);
3074	if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask,	2738	/* no handling is needed for SRQ */
3075	LAST_TUNNEL_FIELD))	2739	if (!mqp->ibqp.srq) {
3076	return -EOPNOTSUPP;	2740	if (mqp->rq.tail != mqp->rq.head) {
3077		2741	recv_mcq = to_mcq(mqp->ibqp.recv_cq);
3078	MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni,	2742	spin_lock_irqsave(&recv_mcq->lock, flags_cq);
3079	ntohl(ib_spec->tunnel.mask.tunnel_id));	2743	if (recv_mcq->mcq.comp &&
3080	MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni,	2744	mqp->ibqp.recv_cq->comp_handler) {
3081	ntohl(ib_spec->tunnel.val.tunnel_id));	2745	if (!recv_mcq->mcq.reset_notify_added) {
3082	break;	2746	recv_mcq->mcq.reset_notify_added = 1;
3083	case IB_FLOW_SPEC_ACTION_TAG:	2747	list_add_tail(&recv_mcq->mcq.reset_notify,
3084	if (FIELDS_NOT_SUPPORTED(ib_spec->flow_tag,	2748	&cq_armed_list);
3085	LAST_FLOW_TAG_FIELD))	2749	}
3086	return -EOPNOTSUPP;	2750	}
3087	if (ib_spec->flow_tag.tag_id >= BIT(24))	2751	spin_unlock_irqrestore(&recv_mcq->lock,
3088	return -EINVAL;	2752	flags_cq);
3089		2753	}
3090	flow_context->flow_tag = ib_spec->flow_tag.tag_id;	2754	}
3091	flow_context->flags \|= FLOW_CONTEXT_HAS_TAG;	2755	spin_unlock_irqrestore(&mqp->rq.lock, flags_qp);
3092	break;
3093	case IB_FLOW_SPEC_ACTION_DROP:
3094	if (FIELDS_NOT_SUPPORTED(ib_spec->drop,
3095	LAST_DROP_FIELD))
3096	return -EOPNOTSUPP;
3097	action->action \|= MLX5_FLOW_CONTEXT_ACTION_DROP;
3098	break;
3099	case IB_FLOW_SPEC_ACTION_HANDLE:
3100	ret = parse_flow_flow_action(to_mflow_act(ib_spec->action.act),
3101	flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS, action);
3102	if (ret)
3103	return ret;
3104	break;
3105	case IB_FLOW_SPEC_ACTION_COUNT:
3106	if (FIELDS_NOT_SUPPORTED(ib_spec->flow_count,
3107	LAST_COUNTERS_FIELD))
3108	return -EOPNOTSUPP;
3109
3110	/* for now support only one counters spec per flow */
3111	if (action->action & MLX5_FLOW_CONTEXT_ACTION_COUNT)
3112	return -EINVAL;
3113
3114	action->counters = ib_spec->flow_count.counters;
3115	action->action \|= MLX5_FLOW_CONTEXT_ACTION_COUNT;
3116	break;
3117	default:
3118	return -EINVAL;
3119	}
3120
3121	return 0;
3122	}
3123
3124	/* If a flow could catch both multicast and unicast packets,
3125	* it won't fall into the multicast flow steering table and this rule
3126	* could steal other multicast packets.
3127	*/
3128	static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr)
3129	{
3130	union ib_flow_spec *flow_spec;
3131
3132	if (ib_attr->type != IB_FLOW_ATTR_NORMAL \|\|
3133	ib_attr->num_of_specs < 1)
3134	return false;
3135
3136	flow_spec = (union ib_flow_spec *)(ib_attr + 1);
3137	if (flow_spec->type == IB_FLOW_SPEC_IPV4) {
3138	struct ib_flow_spec_ipv4 *ipv4_spec;
3139
3140	ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec;
3141	if (ipv4_is_multicast(ipv4_spec->val.dst_ip))
3142	return true;
3143
3144	return false;
3145	}	2756	}
3146		2757	/*At that point all inflight post send were put to be executed as of we
3147	if (flow_spec->type == IB_FLOW_SPEC_ETH) {	2758	* lock/unlock above locks Now need to arm all involved CQs.
3148	struct ib_flow_spec_eth *eth_spec;	2759	*/
3149		2760	list_for_each_entry(mcq, &cq_armed_list, reset_notify) {
3150	eth_spec = (struct ib_flow_spec_eth *)flow_spec;	2761	mcq->comp(mcq, NULL);
3151	return is_multicast_ether_addr(eth_spec->mask.dst_mac) &&
3152	is_multicast_ether_addr(eth_spec->val.dst_mac);
3153	}	2762	}
3154		2763	spin_unlock_irqrestore(&ibdev->reset_flow_resource_lock, flags);
3155	return false;
3156	}	2764	}
3157		2765
3158	enum valid_spec {	2766	static void delay_drop_handler(struct work_struct *work)
3159	VALID_SPEC_INVALID,
3160	VALID_SPEC_VALID,
3161	VALID_SPEC_NA,
3162	};
3163
3164	static enum valid_spec
3165	is_valid_esp_aes_gcm(struct mlx5_core_dev *mdev,
3166	const struct mlx5_flow_spec *spec,
3167	const struct mlx5_flow_act *flow_act,
3168	bool egress)
3169	{	2767	{
3170	const u32 *match_c = spec->match_criteria;	2768	int err;
3171	bool is_crypto =	2769	struct mlx5_ib_delay_drop *delay_drop =
3172	(flow_act->action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT \|	2770	container_of(work, struct mlx5_ib_delay_drop,
3173	MLX5_FLOW_CONTEXT_ACTION_DECRYPT));	2771	delay_drop_work);
3174	bool is_ipsec = mlx5_fs_is_ipsec_flow(match_c);
3175	bool is_drop = flow_act->action & MLX5_FLOW_CONTEXT_ACTION_DROP;
3176
3177	/*
3178	* Currently only crypto is supported in egress, when regular egress
3179	* rules would be supported, always return VALID_SPEC_NA.
3180	*/
3181	if (!is_crypto)
3182	return VALID_SPEC_NA;
3183
3184	return is_crypto && is_ipsec &&
3185	(!egress \|\| (!is_drop &&
3186	!(spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG))) ?
3187	VALID_SPEC_VALID : VALID_SPEC_INVALID;
3188	}
3189
3190	static bool is_valid_spec(struct mlx5_core_dev *mdev,
3191	const struct mlx5_flow_spec *spec,
3192	const struct mlx5_flow_act *flow_act,
3193	bool egress)
3194	{
3195	/* We curretly only support ipsec egress flow */
3196	return is_valid_esp_aes_gcm(mdev, spec, flow_act, egress) != VALID_SPEC_INVALID;
3197	}
3198
3199	static bool is_valid_ethertype(struct mlx5_core_dev *mdev,
3200	const struct ib_flow_attr *flow_attr,
3201	bool check_inner)
3202	{
3203	union ib_flow_spec ib_spec = (union ib_flow_spec )(flow_attr + 1);
3204	int match_ipv = check_inner ?
3205	MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
3206	ft_field_support.inner_ip_version) :
3207	MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
3208	ft_field_support.outer_ip_version);
3209	int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0;
3210	bool ipv4_spec_valid, ipv6_spec_valid;
3211	unsigned int ip_spec_type = 0;
3212	bool has_ethertype = false;
3213	unsigned int spec_index;
3214	bool mask_valid = true;
3215	u16 eth_type = 0;
3216	bool type_valid;
3217
3218	/* Validate that ethertype is correct */
3219	for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
3220	if ((ib_spec->type == (IB_FLOW_SPEC_ETH \| inner_bit)) &&
3221	ib_spec->eth.mask.ether_type) {
3222	mask_valid = (ib_spec->eth.mask.ether_type ==
3223	htons(0xffff));
3224	has_ethertype = true;
3225	eth_type = ntohs(ib_spec->eth.val.ether_type);
3226	} else if ((ib_spec->type == (IB_FLOW_SPEC_IPV4 \| inner_bit)) \|\|
3227	(ib_spec->type == (IB_FLOW_SPEC_IPV6 \| inner_bit))) {
3228	ip_spec_type = ib_spec->type;
3229	}
3230	ib_spec = (void *)ib_spec + ib_spec->size;
3231	}
3232		2772
3233	type_valid = (!has_ethertype) \|\| (!ip_spec_type);	2773	atomic_inc(&delay_drop->events_cnt);
3234	if (!type_valid && mask_valid) {
3235	ipv4_spec_valid = (eth_type == ETH_P_IP) &&
3236	(ip_spec_type == (IB_FLOW_SPEC_IPV4 \| inner_bit));
3237	ipv6_spec_valid = (eth_type == ETH_P_IPV6) &&
3238	(ip_spec_type == (IB_FLOW_SPEC_IPV6 \| inner_bit));
3239		2774
3240	type_valid = (ipv4_spec_valid) \|\| (ipv6_spec_valid) \|\|	2775	mutex_lock(&delay_drop->lock);
3241	(((eth_type == ETH_P_MPLS_UC) \|\|	2776	err = mlx5_core_set_delay_drop(delay_drop->dev, delay_drop->timeout);
3242	(eth_type == ETH_P_MPLS_MC)) && match_ipv);	2777	if (err) {
		2778	mlx5_ib_warn(delay_drop->dev, "Failed to set delay drop, timeout=%u\n",
		2779	delay_drop->timeout);
		2780	delay_drop->activate = false;
3243	}	2781	}
3244		2782	mutex_unlock(&delay_drop->lock);
3245	return type_valid;
3246	}	2783	}
3247		2784
3248	static bool is_valid_attr(struct mlx5_core_dev *mdev,	2785	static void handle_general_event(struct mlx5_ib_dev ibdev, struct mlx5_eqe eqe,
3249	const struct ib_flow_attr *flow_attr)	2786	struct ib_event *ibev)
3250	{	2787	{
3251	return is_valid_ethertype(mdev, flow_attr, false) &&	2788	u8 port = (eqe->data.port.port >> 4) & 0xf;
3252	is_valid_ethertype(mdev, flow_attr, true);
3253	}
3254		2789
3255	static void put_flow_table(struct mlx5_ib_dev *dev,	2790	switch (eqe->sub_type) {
3256	struct mlx5_ib_flow_prio *prio, bool ft_added)	2791	case MLX5_GENERAL_SUBTYPE_DELAY_DROP_TIMEOUT:
3257	{	2792	if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) ==
3258	prio->refcount -= !!ft_added;	2793	IB_LINK_LAYER_ETHERNET)
3259	if (!prio->refcount) {	2794	schedule_work(&ibdev->delay_drop.delay_drop_work);
3260	mlx5_destroy_flow_table(prio->flow_table);	2795	break;
3261	prio->flow_table = NULL;	2796	default: /* do nothing */
		2797	return;
3262	}	2798	}
3263	}	2799	}
3264		2800
3265	static void counters_clear_description(struct ib_counters *counters)	2801	static int handle_port_change(struct mlx5_ib_dev ibdev, struct mlx5_eqe eqe,
		2802	struct ib_event *ibev)
3266	{	2803	{
3267	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);	2804	u8 port = (eqe->data.port.port >> 4) & 0xf;
3268		2805
3269	mutex_lock(&mcounters->mcntrs_mutex);	2806	ibev->element.port_num = port;
3270	kfree(mcounters->counters_data);
3271	mcounters->counters_data = NULL;
3272	mcounters->cntrs_max_index = 0;
3273	mutex_unlock(&mcounters->mcntrs_mutex);
3274	}
3275		2807
3276	static int mlx5_ib_destroy_flow(struct ib_flow *flow_id)	2808	switch (eqe->sub_type) {
3277	{	2809	case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
3278	struct mlx5_ib_flow_handler *handler = container_of(flow_id,	2810	case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
3279	struct mlx5_ib_flow_handler,	2811	case MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED:
3280	ibflow);	2812	/* In RoCE, port up/down events are handled in
3281	struct mlx5_ib_flow_handler iter, tmp;	2813	* mlx5_netdev_event().
3282	struct mlx5_ib_dev *dev = handler->dev;	2814	*/
		2815	if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) ==
		2816	IB_LINK_LAYER_ETHERNET)
		2817	return -EINVAL;
		2818
		2819	ibev->event = (eqe->sub_type == MLX5_PORT_CHANGE_SUBTYPE_ACTIVE) ?
		2820	IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
		2821	break;
3283		2822
3284	mutex_lock(&dev->flow_db->lock);	2823	case MLX5_PORT_CHANGE_SUBTYPE_LID:
		2824	ibev->event = IB_EVENT_LID_CHANGE;
		2825	break;
3285		2826
3286	list_for_each_entry_safe(iter, tmp, &handler->list, list) {	2827	case MLX5_PORT_CHANGE_SUBTYPE_PKEY:
3287	mlx5_del_flow_rules(iter->rule);	2828	ibev->event = IB_EVENT_PKEY_CHANGE;
3288	put_flow_table(dev, iter->prio, true);	2829	schedule_work(&ibdev->devr.ports[port - 1].pkey_change_work);
3289	list_del(&iter->list);	2830	break;
3290	kfree(iter);
3291	}
3292		2831
3293	mlx5_del_flow_rules(handler->rule);	2832	case MLX5_PORT_CHANGE_SUBTYPE_GUID:
3294	put_flow_table(dev, handler->prio, true);	2833	ibev->event = IB_EVENT_GID_CHANGE;
3295	if (handler->ibcounters &&	2834	break;
3296	atomic_read(&handler->ibcounters->usecnt) == 1)
3297	counters_clear_description(handler->ibcounters);
3298		2835
3299	mutex_unlock(&dev->flow_db->lock);	2836	case MLX5_PORT_CHANGE_SUBTYPE_CLIENT_REREG:
3300	if (handler->flow_matcher)	2837	ibev->event = IB_EVENT_CLIENT_REREGISTER;
3301	atomic_dec(&handler->flow_matcher->usecnt);	2838	break;
3302	kfree(handler);	2839	default:
		2840	return -EINVAL;
		2841	}
3303		2842
3304	return 0;	2843	return 0;
3305	}	2844	}
3306		2845
3307	static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap)	2846	static void mlx5_ib_handle_event(struct work_struct *_work)
3308	{
3309	priority *= 2;
3310	if (!dont_trap)
3311	priority++;
3312	return priority;
3313	}
3314
3315	enum flow_table_type {
3316	MLX5_IB_FT_RX,
3317	MLX5_IB_FT_TX
3318	};
3319
3320	#define MLX5_FS_MAX_TYPES 6
3321	#define MLX5_FS_MAX_ENTRIES BIT(16)
3322
3323	static struct mlx5_ib_flow_prio _get_prio(struct mlx5_flow_namespace ns,
3324	struct mlx5_ib_flow_prio *prio,
3325	int priority,
3326	int num_entries, int num_groups,
3327	u32 flags)
3328	{	2847	{
3329	struct mlx5_flow_table_attr ft_attr = {};	2848	struct mlx5_ib_event_work *work =
3330	struct mlx5_flow_table *ft;	2849	container_of(_work, struct mlx5_ib_event_work, work);
3331		2850	struct mlx5_ib_dev *ibdev;
3332	ft_attr.prio = priority;	2851	struct ib_event ibev;
3333	ft_attr.max_fte = num_entries;	2852	bool fatal = false;
3334	ft_attr.flags = flags;
3335	ft_attr.autogroup.max_num_groups = num_groups;
3336	ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
3337	if (IS_ERR(ft))
3338	return ERR_CAST(ft);
3339
3340	prio->flow_table = ft;
3341	prio->refcount = 0;
3342	return prio;
3343	}
3344		2853
3345	static struct mlx5_ib_flow_prio get_flow_table(struct mlx5_ib_dev dev,	2854	if (work->is_slave) {
3346	struct ib_flow_attr *flow_attr,	2855	ibdev = mlx5_ib_get_ibdev_from_mpi(work->mpi);
3347	enum flow_table_type ft_type)	2856	if (!ibdev)
3348	{	2857	goto out;
3349	bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP;	2858	} else {
3350	struct mlx5_flow_namespace *ns = NULL;	2859	ibdev = work->dev;
3351	struct mlx5_ib_flow_prio *prio;
3352	struct mlx5_flow_table *ft;
3353	int max_table_size;
3354	int num_entries;
3355	int num_groups;
3356	bool esw_encap;
3357	u32 flags = 0;
3358	int priority;
3359
3360	max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
3361	log_max_ft_size));
3362	esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) !=
3363	DEVLINK_ESWITCH_ENCAP_MODE_NONE;
3364	if (flow_attr->type == IB_FLOW_ATTR_NORMAL) {
3365	enum mlx5_flow_namespace_type fn_type;
3366
3367	if (flow_is_multicast_only(flow_attr) &&
3368	!dont_trap)
3369	priority = MLX5_IB_FLOW_MCAST_PRIO;
3370	else
3371	priority = ib_prio_to_core_prio(flow_attr->priority,
3372	dont_trap);
3373	if (ft_type == MLX5_IB_FT_RX) {
3374	fn_type = MLX5_FLOW_NAMESPACE_BYPASS;
3375	prio = &dev->flow_db->prios[priority];
3376	if (!dev->is_rep && !esw_encap &&
3377	MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap))
3378	flags \|= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
3379	if (!dev->is_rep && !esw_encap &&
3380	MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
3381	reformat_l3_tunnel_to_l2))
3382	flags \|= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
3383	} else {
3384	max_table_size =
3385	BIT(MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev,
3386	log_max_ft_size));
3387	fn_type = MLX5_FLOW_NAMESPACE_EGRESS;
3388	prio = &dev->flow_db->egress_prios[priority];
3389	if (!dev->is_rep && !esw_encap &&
3390	MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat))
3391	flags \|= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
3392	}
3393	ns = mlx5_get_flow_namespace(dev->mdev, fn_type);
3394	num_entries = MLX5_FS_MAX_ENTRIES;
3395	num_groups = MLX5_FS_MAX_TYPES;
3396	} else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT \|\|
3397	flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) {
3398	ns = mlx5_get_flow_namespace(dev->mdev,
3399	MLX5_FLOW_NAMESPACE_LEFTOVERS);
3400	build_leftovers_ft_param(&priority,
3401	&num_entries,
3402	&num_groups);
3403	prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO];
3404	} else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
3405	if (!MLX5_CAP_FLOWTABLE(dev->mdev,
3406	allow_sniffer_and_nic_rx_shared_tir))
3407	return ERR_PTR(-ENOTSUPP);
3408
3409	ns = mlx5_get_flow_namespace(dev->mdev, ft_type == MLX5_IB_FT_RX ?
3410	MLX5_FLOW_NAMESPACE_SNIFFER_RX :
3411	MLX5_FLOW_NAMESPACE_SNIFFER_TX);
3412
3413	prio = &dev->flow_db->sniffer[ft_type];
3414	priority = 0;
3415	num_entries = 1;
3416	num_groups = 1;
3417	}	2860	}
3418		2861
3419	if (!ns)	2862	switch (work->event) {
3420	return ERR_PTR(-ENOTSUPP);	2863	case MLX5_DEV_EVENT_SYS_ERROR:
3421		2864	ibev.event = IB_EVENT_DEVICE_FATAL;
3422	max_table_size = min_t(int, num_entries, max_table_size);	2865	mlx5_ib_handle_internal_error(ibdev);
3423		2866	ibev.element.port_num = (u8)(unsigned long)work->param;
3424	ft = prio->flow_table;	2867	fatal = true;
3425	if (!ft)	2868	break;
3426	return _get_prio(ns, prio, priority, max_table_size, num_groups,	2869	case MLX5_EVENT_TYPE_PORT_CHANGE:
3427	flags);	2870	if (handle_port_change(ibdev, work->param, &ibev))
3428		2871	goto out;
3429	return prio;	2872	break;
3430	}	2873	case MLX5_EVENT_TYPE_GENERAL_EVENT:
		2874	handle_general_event(ibdev, work->param, &ibev);
		2875	/* fall through */
		2876	default:
		2877	goto out;
		2878	}
3431		2879
3432	static void set_underlay_qp(struct mlx5_ib_dev *dev,	2880	ibev.device = &ibdev->ib_dev;
3433	struct mlx5_flow_spec *spec,
3434	u32 underlay_qpn)
3435	{
3436	void *misc_params_c = MLX5_ADDR_OF(fte_match_param,
3437	spec->match_criteria,
3438	misc_parameters);
3439	void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
3440	misc_parameters);
3441		2881
3442	if (underlay_qpn &&	2882	if (!rdma_is_port_valid(&ibdev->ib_dev, ibev.element.port_num)) {
3443	MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,	2883	mlx5_ib_warn(ibdev, "warning: event on port %d\n", ibev.element.port_num);
3444	ft_field_support.bth_dst_qp)) {	2884	goto out;
3445	MLX5_SET(fte_match_set_misc,
3446	misc_params_v, bth_dst_qp, underlay_qpn);
3447	MLX5_SET(fte_match_set_misc,
3448	misc_params_c, bth_dst_qp, 0xffffff);
3449	}	2885	}
3450	}
3451		2886
3452	static int read_flow_counters(struct ib_device *ibdev,	2887	if (ibdev->ib_active)
3453	struct mlx5_read_counters_attr *read_attr)	2888	ib_dispatch_event(&ibev);
3454	{
3455	struct mlx5_fc *fc = read_attr->hw_cntrs_hndl;
3456	struct mlx5_ib_dev *dev = to_mdev(ibdev);
3457		2889
3458	return mlx5_fc_query(dev->mdev, fc,	2890	if (fatal)
3459	&read_attr->out[IB_COUNTER_PACKETS],	2891	ibdev->ib_active = false;
3460	&read_attr->out[IB_COUNTER_BYTES]);	2892	out:
		2893	kfree(work);
3461	}	2894	}
3462		2895
3463	/* flow counters currently expose two counters packets and bytes */	2896	static int mlx5_ib_event(struct notifier_block *nb,
3464	#define FLOW_COUNTERS_NUM 2	2897	unsigned long event, void *param)
3465	static int counters_set_description(struct ib_counters *counters,
3466	enum mlx5_ib_counters_type counters_type,
3467	struct mlx5_ib_flow_counters_desc *desc_data,
3468	u32 ncounters)
3469	{	2898	{
3470	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);	2899	struct mlx5_ib_event_work *work;
3471	u32 cntrs_max_index = 0;
3472	int i;
3473
3474	if (counters_type != MLX5_IB_COUNTERS_FLOW)
3475	return -EINVAL;
3476		2900
3477	/* init the fields for the object */	2901	work = kmalloc(sizeof(*work), GFP_ATOMIC);
3478	mcounters->type = counters_type;	2902	if (!work)
3479	mcounters->read_counters = read_flow_counters;	2903	return NOTIFY_DONE;
3480	mcounters->counters_num = FLOW_COUNTERS_NUM;
3481	mcounters->ncounters = ncounters;
3482	/* each counter entry have both description and index pair */
3483	for (i = 0; i < ncounters; i++) {
3484	if (desc_data[i].description > IB_COUNTER_BYTES)
3485	return -EINVAL;
3486		2904
3487	if (cntrs_max_index <= desc_data[i].index)	2905	INIT_WORK(&work->work, mlx5_ib_handle_event);
3488	cntrs_max_index = desc_data[i].index + 1;	2906	work->dev = container_of(nb, struct mlx5_ib_dev, mdev_events);
3489	}	2907	work->is_slave = false;
		2908	work->param = param;
		2909	work->event = event;
3490		2910
3491	mutex_lock(&mcounters->mcntrs_mutex);	2911	queue_work(mlx5_ib_event_wq, &work->work);
3492	mcounters->counters_data = desc_data;
3493	mcounters->cntrs_max_index = cntrs_max_index;
3494	mutex_unlock(&mcounters->mcntrs_mutex);
3495		2912
3496	return 0;	2913	return NOTIFY_OK;
3497	}	2914	}
3498		2915
3499	#define MAX_COUNTERS_NUM (USHRT_MAX / (sizeof(u32) * 2))	2916	static int mlx5_ib_event_slave_port(struct notifier_block *nb,
3500	static int flow_counters_set_data(struct ib_counters *ibcounters,	2917	unsigned long event, void *param)
3501	struct mlx5_ib_create_flow *ucmd)
3502	{	2918	{
3503	struct mlx5_ib_mcounters *mcounters = to_mcounters(ibcounters);	2919	struct mlx5_ib_event_work *work;
3504	struct mlx5_ib_flow_counters_data *cntrs_data = NULL;
3505	struct mlx5_ib_flow_counters_desc *desc_data = NULL;
3506	bool hw_hndl = false;
3507	int ret = 0;
3508
3509	if (ucmd && ucmd->ncounters_data != 0) {
3510	cntrs_data = ucmd->data;
3511	if (cntrs_data->ncounters > MAX_COUNTERS_NUM)
3512	return -EINVAL;
3513
3514	desc_data = kcalloc(cntrs_data->ncounters,
3515	sizeof(*desc_data),
3516	GFP_KERNEL);
3517	if (!desc_data)
3518	return -ENOMEM;
3519
3520	if (copy_from_user(desc_data,
3521	u64_to_user_ptr(cntrs_data->counters_data),
3522	sizeof(desc_data) cntrs_data->ncounters)) {
3523	ret = -EFAULT;
3524	goto free;
3525	}
3526	}
3527
3528	if (!mcounters->hw_cntrs_hndl) {
3529	mcounters->hw_cntrs_hndl = mlx5_fc_create(
3530	to_mdev(ibcounters->device)->mdev, false);
3531	if (IS_ERR(mcounters->hw_cntrs_hndl)) {
3532	ret = PTR_ERR(mcounters->hw_cntrs_hndl);
3533	goto free;
3534	}
3535	hw_hndl = true;
3536	}
3537
3538	if (desc_data) {
3539	/* counters already bound to at least one flow */
3540	if (mcounters->cntrs_max_index) {
3541	ret = -EINVAL;
3542	goto free_hndl;
3543	}
3544
3545	ret = counters_set_description(ibcounters,
3546	MLX5_IB_COUNTERS_FLOW,
3547	desc_data,
3548	cntrs_data->ncounters);
3549	if (ret)
3550	goto free_hndl;
3551
3552	} else if (!mcounters->cntrs_max_index) {
3553	/* counters not bound yet, must have udata passed */
3554	ret = -EINVAL;
3555	goto free_hndl;
3556	}
3557
3558	return 0;
3559
3560	free_hndl:
3561	if (hw_hndl) {
3562	mlx5_fc_destroy(to_mdev(ibcounters->device)->mdev,
3563	mcounters->hw_cntrs_hndl);
3564	mcounters->hw_cntrs_hndl = NULL;
3565	}
3566	free:
3567	kfree(desc_data);
3568	return ret;
3569	}
3570
3571	static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev *dev,
3572	struct mlx5_flow_spec *spec,
3573	struct mlx5_eswitch_rep *rep)
3574	{
3575	struct mlx5_eswitch *esw = dev->mdev->priv.eswitch;
3576	void *misc;
3577
3578	if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
3579	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
3580	misc_parameters_2);
3581
3582	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
3583	mlx5_eswitch_get_vport_metadata_for_match(esw,
3584	rep->vport));
3585	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
3586	misc_parameters_2);
3587
3588	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
3589	mlx5_eswitch_get_vport_metadata_mask());
3590	} else {
3591	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
3592	misc_parameters);
3593
3594	MLX5_SET(fte_match_set_misc, misc, source_port, rep->vport);
3595
3596	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
3597	misc_parameters);
3598
3599	MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
3600	}
3601	}
3602
3603	static struct mlx5_ib_flow_handler _create_flow_rule(struct mlx5_ib_dev dev,
3604	struct mlx5_ib_flow_prio *ft_prio,
3605	const struct ib_flow_attr *flow_attr,
3606	struct mlx5_flow_destination *dst,
3607	u32 underlay_qpn,
3608	struct mlx5_ib_create_flow *ucmd)
3609	{
3610	struct mlx5_flow_table *ft = ft_prio->flow_table;
3611	struct mlx5_ib_flow_handler *handler;
3612	struct mlx5_flow_act flow_act = {};
3613	struct mlx5_flow_spec *spec;
3614	struct mlx5_flow_destination dest_arr[2] = {};
3615	struct mlx5_flow_destination *rule_dst = dest_arr;
3616	const void ib_flow = (const void )flow_attr + sizeof(*flow_attr);
3617	unsigned int spec_index;
3618	u32 prev_type = 0;
3619	int err = 0;
3620	int dest_num = 0;
3621	bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS;
3622
3623	if (!is_valid_attr(dev->mdev, flow_attr))
3624	return ERR_PTR(-EINVAL);
3625
3626	if (dev->is_rep && is_egress)
3627	return ERR_PTR(-EINVAL);
3628
3629	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
3630	handler = kzalloc(sizeof(*handler), GFP_KERNEL);
3631	if (!handler \|\| !spec) {
3632	err = -ENOMEM;
3633	goto free;
3634	}
3635
3636	INIT_LIST_HEAD(&handler->list);
3637
3638	for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
3639	err = parse_flow_attr(dev->mdev, spec,
3640	ib_flow, flow_attr, &flow_act,
3641	prev_type);
3642	if (err < 0)
3643	goto free;
3644
3645	prev_type = ((union ib_flow_spec *)ib_flow)->type;
3646	ib_flow += ((union ib_flow_spec *)ib_flow)->size;
3647	}
3648
3649	if (dst && !(flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP)) {
3650	memcpy(&dest_arr[0], dst, sizeof(*dst));
3651	dest_num++;
3652	}
3653
3654	if (!flow_is_multicast_only(flow_attr))
3655	set_underlay_qp(dev, spec, underlay_qpn);
3656
3657	if (dev->is_rep) {
3658	struct mlx5_eswitch_rep *rep;
3659
3660	rep = dev->port[flow_attr->port - 1].rep;
3661	if (!rep) {
3662	err = -EINVAL;
3663	goto free;
3664	}
3665
3666	mlx5_ib_set_rule_source_port(dev, spec, rep);
3667	}
3668
3669	spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria);
3670
3671	if (is_egress &&
3672	!is_valid_spec(dev->mdev, spec, &flow_act, is_egress)) {
3673	err = -EINVAL;
3674	goto free;
3675	}
3676
3677	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
3678	struct mlx5_ib_mcounters *mcounters;
3679
3680	err = flow_counters_set_data(flow_act.counters, ucmd);
3681	if (err)
3682	goto free;
3683
3684	mcounters = to_mcounters(flow_act.counters);
3685	handler->ibcounters = flow_act.counters;
3686	dest_arr[dest_num].type =
3687	MLX5_FLOW_DESTINATION_TYPE_COUNTER;
3688	dest_arr[dest_num].counter_id =
3689	mlx5_fc_id(mcounters->hw_cntrs_hndl);
3690	dest_num++;
3691	}
3692
3693	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP) {
3694	if (!dest_num)
3695	rule_dst = NULL;
3696	} else {
3697	if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP)
3698	flow_act.action \|=
3699	MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO;
3700	if (is_egress)
3701	flow_act.action \|= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
3702	else if (dest_num)
3703	flow_act.action \|= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
3704	}
3705
3706	if ((spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG) &&
3707	(flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT \|\|
3708	flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) {
3709	mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n",
3710	spec->flow_context.flow_tag, flow_attr->type);
3711	err = -EINVAL;
3712	goto free;
3713	}
3714	handler->rule = mlx5_add_flow_rules(ft, spec,
3715	&flow_act,
3716	rule_dst, dest_num);
3717
3718	if (IS_ERR(handler->rule)) {
3719	err = PTR_ERR(handler->rule);
3720	goto free;
3721	}
3722
3723	ft_prio->refcount++;
3724	handler->prio = ft_prio;
3725	handler->dev = dev;
3726
3727	ft_prio->flow_table = ft;
3728	free:
3729	if (err && handler) {
3730	if (handler->ibcounters &&
3731	atomic_read(&handler->ibcounters->usecnt) == 1)
3732	counters_clear_description(handler->ibcounters);
3733	kfree(handler);
3734	}
3735	kvfree(spec);
3736	return err ? ERR_PTR(err) : handler;
3737	}
3738
3739	static struct mlx5_ib_flow_handler create_flow_rule(struct mlx5_ib_dev dev,
3740	struct mlx5_ib_flow_prio *ft_prio,
3741	const struct ib_flow_attr *flow_attr,
3742	struct mlx5_flow_destination *dst)
3743	{
3744	return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL);
3745	}
3746
3747	enum {
3748	LEFTOVERS_MC,
3749	LEFTOVERS_UC,
3750	};
3751
3752	static struct mlx5_ib_flow_handler create_leftovers_rule(struct mlx5_ib_dev dev,
3753	struct mlx5_ib_flow_prio *ft_prio,
3754	struct ib_flow_attr *flow_attr,
3755	struct mlx5_flow_destination *dst)
3756	{
3757	struct mlx5_ib_flow_handler *handler_ucast = NULL;
3758	struct mlx5_ib_flow_handler *handler = NULL;
3759
3760	static struct {
3761	struct ib_flow_attr flow_attr;
3762	struct ib_flow_spec_eth eth_flow;
3763	} leftovers_specs[] = {
3764	[LEFTOVERS_MC] = {
3765	.flow_attr = {
3766	.num_of_specs = 1,
3767	.size = sizeof(leftovers_specs[0])
3768	},
3769	.eth_flow = {
3770	.type = IB_FLOW_SPEC_ETH,
3771	.size = sizeof(struct ib_flow_spec_eth),
3772	.mask = {.dst_mac = {0x1} },
3773	.val = {.dst_mac = {0x1} }
3774	}
3775	},
3776	[LEFTOVERS_UC] = {
3777	.flow_attr = {
3778	.num_of_specs = 1,
3779	.size = sizeof(leftovers_specs[0])
3780	},
3781	.eth_flow = {
3782	.type = IB_FLOW_SPEC_ETH,
3783	.size = sizeof(struct ib_flow_spec_eth),
3784	.mask = {.dst_mac = {0x1} },
3785	.val = {.dst_mac = {} }
3786	}
3787	}
3788	};
3789
3790	handler = create_flow_rule(dev, ft_prio,
3791	&leftovers_specs[LEFTOVERS_MC].flow_attr,
3792	dst);
3793	if (!IS_ERR(handler) &&
3794	flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) {
3795	handler_ucast = create_flow_rule(dev, ft_prio,
3796	&leftovers_specs[LEFTOVERS_UC].flow_attr,
3797	dst);
3798	if (IS_ERR(handler_ucast)) {
3799	mlx5_del_flow_rules(handler->rule);
3800	ft_prio->refcount--;
3801	kfree(handler);
3802	handler = handler_ucast;
3803	} else {
3804	list_add(&handler_ucast->list, &handler->list);
3805	}
3806	}
3807
3808	return handler;
3809	}
3810
3811	static struct mlx5_ib_flow_handler create_sniffer_rule(struct mlx5_ib_dev dev,
3812	struct mlx5_ib_flow_prio *ft_rx,
3813	struct mlx5_ib_flow_prio *ft_tx,
3814	struct mlx5_flow_destination *dst)
3815	{
3816	struct mlx5_ib_flow_handler *handler_rx;
3817	struct mlx5_ib_flow_handler *handler_tx;
3818	int err;
3819	static const struct ib_flow_attr flow_attr = {
3820	.num_of_specs = 0,
3821	.size = sizeof(flow_attr)
3822	};
3823
3824	handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst);
3825	if (IS_ERR(handler_rx)) {
3826	err = PTR_ERR(handler_rx);
3827	goto err;
3828	}
3829
3830	handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst);
3831	if (IS_ERR(handler_tx)) {
3832	err = PTR_ERR(handler_tx);
3833	goto err_tx;
3834	}
3835
3836	list_add(&handler_tx->list, &handler_rx->list);
3837
3838	return handler_rx;
3839
3840	err_tx:
3841	mlx5_del_flow_rules(handler_rx->rule);
3842	ft_rx->refcount--;
3843	kfree(handler_rx);
3844	err:
3845	return ERR_PTR(err);
3846	}
3847
3848	static struct ib_flow mlx5_ib_create_flow(struct ib_qp qp,
3849	struct ib_flow_attr *flow_attr,
3850	int domain,
3851	struct ib_udata *udata)
3852	{
3853	struct mlx5_ib_dev *dev = to_mdev(qp->device);
3854	struct mlx5_ib_qp *mqp = to_mqp(qp);
3855	struct mlx5_ib_flow_handler *handler = NULL;
3856	struct mlx5_flow_destination *dst = NULL;
3857	struct mlx5_ib_flow_prio *ft_prio_tx = NULL;
3858	struct mlx5_ib_flow_prio *ft_prio;
3859	bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS;
3860	struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr;
3861	size_t min_ucmd_sz, required_ucmd_sz;
3862	int err;
3863	int underlay_qpn;
3864
3865	if (udata && udata->inlen) {
3866	min_ucmd_sz = offsetof(typeof(ucmd_hdr), reserved) +
3867	sizeof(ucmd_hdr.reserved);
3868	if (udata->inlen < min_ucmd_sz)
3869	return ERR_PTR(-EOPNOTSUPP);
3870
3871	err = ib_copy_from_udata(&ucmd_hdr, udata, min_ucmd_sz);
3872	if (err)
3873	return ERR_PTR(err);
3874
3875	/* currently supports only one counters data */
3876	if (ucmd_hdr.ncounters_data > 1)
3877	return ERR_PTR(-EINVAL);
3878
3879	required_ucmd_sz = min_ucmd_sz +
3880	sizeof(struct mlx5_ib_flow_counters_data) *
3881	ucmd_hdr.ncounters_data;
3882	if (udata->inlen > required_ucmd_sz &&
3883	!ib_is_udata_cleared(udata, required_ucmd_sz,
3884	udata->inlen - required_ucmd_sz))
3885	return ERR_PTR(-EOPNOTSUPP);
3886
3887	ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL);
3888	if (!ucmd)
3889	return ERR_PTR(-ENOMEM);
3890
3891	err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz);
3892	if (err)
3893	goto free_ucmd;
3894	}
3895
3896	if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO) {
3897	err = -ENOMEM;
3898	goto free_ucmd;
3899	}
3900
3901	if (domain != IB_FLOW_DOMAIN_USER \|\|
3902	flow_attr->port > dev->num_ports \|\|
3903	(flow_attr->flags & ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP \|
3904	IB_FLOW_ATTR_FLAGS_EGRESS))) {
3905	err = -EINVAL;
3906	goto free_ucmd;
3907	}
3908
3909	if (is_egress &&
3910	(flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT \|\|
3911	flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) {
3912	err = -EINVAL;
3913	goto free_ucmd;
3914	}
3915
3916	dst = kzalloc(sizeof(*dst), GFP_KERNEL);
3917	if (!dst) {
3918	err = -ENOMEM;
3919	goto free_ucmd;
3920	}
3921
3922	mutex_lock(&dev->flow_db->lock);
3923
3924	ft_prio = get_flow_table(dev, flow_attr,
3925	is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX);
3926	if (IS_ERR(ft_prio)) {
3927	err = PTR_ERR(ft_prio);
3928	goto unlock;
3929	}
3930	if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
3931	ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX);
3932	if (IS_ERR(ft_prio_tx)) {
3933	err = PTR_ERR(ft_prio_tx);
3934	ft_prio_tx = NULL;
3935	goto destroy_ft;
3936	}
3937	}
3938
3939	if (is_egress) {
3940	dst->type = MLX5_FLOW_DESTINATION_TYPE_PORT;
3941	} else {
3942	dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR;
3943	if (mqp->is_rss)
3944	dst->tir_num = mqp->rss_qp.tirn;
3945	else
3946	dst->tir_num = mqp->raw_packet_qp.rq.tirn;
3947	}
3948
3949	if (flow_attr->type == IB_FLOW_ATTR_NORMAL) {
3950	underlay_qpn = (mqp->flags & IB_QP_CREATE_SOURCE_QPN) ?
3951	mqp->underlay_qpn :
3952	0;
3953	handler = _create_flow_rule(dev, ft_prio, flow_attr, dst,
3954	underlay_qpn, ucmd);
3955	} else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT \|\|
3956	flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) {
3957	handler = create_leftovers_rule(dev, ft_prio, flow_attr,
3958	dst);
3959	} else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
3960	handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst);
3961	} else {
3962	err = -EINVAL;
3963	goto destroy_ft;
3964	}
3965
3966	if (IS_ERR(handler)) {
3967	err = PTR_ERR(handler);
3968	handler = NULL;
3969	goto destroy_ft;
3970	}
3971
3972	mutex_unlock(&dev->flow_db->lock);
3973	kfree(dst);
3974	kfree(ucmd);
3975
3976	return &handler->ibflow;
3977
3978	destroy_ft:
3979	put_flow_table(dev, ft_prio, false);
3980	if (ft_prio_tx)
3981	put_flow_table(dev, ft_prio_tx, false);
3982	unlock:
3983	mutex_unlock(&dev->flow_db->lock);
3984	kfree(dst);
3985	free_ucmd:
3986	kfree(ucmd);
3987	return ERR_PTR(err);
3988	}
3989
3990	static struct mlx5_ib_flow_prio *
3991	_get_flow_table(struct mlx5_ib_dev *dev,
3992	struct mlx5_ib_flow_matcher *fs_matcher,
3993	bool mcast)
3994	{
3995	struct mlx5_flow_namespace *ns = NULL;
3996	struct mlx5_ib_flow_prio *prio = NULL;
3997	int max_table_size = 0;
3998	bool esw_encap;
3999	u32 flags = 0;
4000	int priority;
4001
4002	if (mcast)
4003	priority = MLX5_IB_FLOW_MCAST_PRIO;
4004	else
4005	priority = ib_prio_to_core_prio(fs_matcher->priority, false);
4006
4007	esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) !=
4008	DEVLINK_ESWITCH_ENCAP_MODE_NONE;
4009	if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) {
4010	max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
4011	log_max_ft_size));
4012	if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap)
4013	flags \|= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
4014	if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
4015	reformat_l3_tunnel_to_l2) &&
4016	!esw_encap)
4017	flags \|= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
4018	} else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) {
4019	max_table_size = BIT(
4020	MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size));
4021	if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) && !esw_encap)
4022	flags \|= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
4023	} else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB) {
4024	max_table_size = BIT(
4025	MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size));
4026	if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap)
4027	flags \|= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
4028	if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, reformat_l3_tunnel_to_l2) &&
4029	esw_encap)
4030	flags \|= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
4031	priority = FDB_BYPASS_PATH;
4032	} else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) {
4033	max_table_size =
4034	BIT(MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev,
4035	log_max_ft_size));
4036	priority = fs_matcher->priority;
4037	} else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) {
4038	max_table_size =
4039	BIT(MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev,
4040	log_max_ft_size));
4041	priority = fs_matcher->priority;
4042	}
4043
4044	max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES);
4045
4046	ns = mlx5_get_flow_namespace(dev->mdev, fs_matcher->ns_type);
4047	if (!ns)
4048	return ERR_PTR(-ENOTSUPP);
4049
4050	if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS)
4051	prio = &dev->flow_db->prios[priority];
4052	else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS)
4053	prio = &dev->flow_db->egress_prios[priority];
4054	else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB)
4055	prio = &dev->flow_db->fdb;
4056	else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX)
4057	prio = &dev->flow_db->rdma_rx[priority];
4058	else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX)
4059	prio = &dev->flow_db->rdma_tx[priority];
4060
4061	if (!prio)
4062	return ERR_PTR(-EINVAL);
4063
4064	if (prio->flow_table)
4065	return prio;
4066
4067	return _get_prio(ns, prio, priority, max_table_size,
4068	MLX5_FS_MAX_TYPES, flags);
4069	}
4070
4071	static struct mlx5_ib_flow_handler *
4072	_create_raw_flow_rule(struct mlx5_ib_dev *dev,
4073	struct mlx5_ib_flow_prio *ft_prio,
4074	struct mlx5_flow_destination *dst,
4075	struct mlx5_ib_flow_matcher *fs_matcher,
4076	struct mlx5_flow_context *flow_context,
4077	struct mlx5_flow_act *flow_act,
4078	void *cmd_in, int inlen,
4079	int dst_num)
4080	{
4081	struct mlx5_ib_flow_handler *handler;
4082	struct mlx5_flow_spec *spec;
4083	struct mlx5_flow_table *ft = ft_prio->flow_table;
4084	int err = 0;
4085
4086	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
4087	handler = kzalloc(sizeof(*handler), GFP_KERNEL);
4088	if (!handler \|\| !spec) {
4089	err = -ENOMEM;
4090	goto free;
4091	}
4092
4093	INIT_LIST_HEAD(&handler->list);
4094
4095	memcpy(spec->match_value, cmd_in, inlen);
4096	memcpy(spec->match_criteria, fs_matcher->matcher_mask.match_params,
4097	fs_matcher->mask_len);
4098	spec->match_criteria_enable = fs_matcher->match_criteria_enable;
4099	spec->flow_context = *flow_context;
4100
4101	handler->rule = mlx5_add_flow_rules(ft, spec,
4102	flow_act, dst, dst_num);
4103
4104	if (IS_ERR(handler->rule)) {
4105	err = PTR_ERR(handler->rule);
4106	goto free;
4107	}
4108
4109	ft_prio->refcount++;
4110	handler->prio = ft_prio;
4111	handler->dev = dev;
4112	ft_prio->flow_table = ft;
4113
4114	free:
4115	if (err)
4116	kfree(handler);
4117	kvfree(spec);
4118	return err ? ERR_PTR(err) : handler;
4119	}
4120
4121	static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher,
4122	void *match_v)
4123	{
4124	void *match_c;
4125	void match_v_set_lyr_2_4, match_c_set_lyr_2_4;
4126	void dmac, dmac_mask;
4127	void ipv4, ipv4_mask;
4128
4129	if (!(fs_matcher->match_criteria_enable &
4130	(1 << MATCH_CRITERIA_ENABLE_OUTER_BIT)))
4131	return false;
4132
4133	match_c = fs_matcher->matcher_mask.match_params;
4134	match_v_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_v,
4135	outer_headers);
4136	match_c_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_c,
4137	outer_headers);
4138
4139	dmac = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,
4140	dmac_47_16);
4141	dmac_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,
4142	dmac_47_16);
4143
4144	if (is_multicast_ether_addr(dmac) &&
4145	is_multicast_ether_addr(dmac_mask))
4146	return true;
4147
4148	ipv4 = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,
4149	dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
4150
4151	ipv4_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,
4152	dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
4153
4154	if (ipv4_is_multicast((__be32 )(ipv4)) &&
4155	ipv4_is_multicast((__be32 )(ipv4_mask)))
4156	return true;
4157
4158	return false;
4159	}
4160
4161	struct mlx5_ib_flow_handler *
4162	mlx5_ib_raw_fs_rule_add(struct mlx5_ib_dev *dev,
4163	struct mlx5_ib_flow_matcher *fs_matcher,
4164	struct mlx5_flow_context *flow_context,
4165	struct mlx5_flow_act *flow_act,
4166	u32 counter_id,
4167	void *cmd_in, int inlen, int dest_id,
4168	int dest_type)
4169	{
4170	struct mlx5_flow_destination *dst;
4171	struct mlx5_ib_flow_prio *ft_prio;
4172	struct mlx5_ib_flow_handler *handler;
4173	int dst_num = 0;
4174	bool mcast;
4175	int err;
4176
4177	if (fs_matcher->flow_type != MLX5_IB_FLOW_TYPE_NORMAL)
4178	return ERR_PTR(-EOPNOTSUPP);
4179
4180	if (fs_matcher->priority > MLX5_IB_FLOW_LAST_PRIO)
4181	return ERR_PTR(-ENOMEM);
4182
4183	dst = kcalloc(2, sizeof(*dst), GFP_KERNEL);
4184	if (!dst)
4185	return ERR_PTR(-ENOMEM);
4186
4187	mcast = raw_fs_is_multicast(fs_matcher, cmd_in);
4188	mutex_lock(&dev->flow_db->lock);
4189
4190	ft_prio = _get_flow_table(dev, fs_matcher, mcast);
4191	if (IS_ERR(ft_prio)) {
4192	err = PTR_ERR(ft_prio);
4193	goto unlock;
4194	}
4195
4196	if (dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR) {
4197	dst[dst_num].type = dest_type;
4198	dst[dst_num++].tir_num = dest_id;
4199	flow_act->action \|= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
4200	} else if (dest_type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) {
4201	dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM;
4202	dst[dst_num++].ft_num = dest_id;
4203	flow_act->action \|= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
4204	} else if (dest_type == MLX5_FLOW_DESTINATION_TYPE_PORT) {
4205	dst[dst_num++].type = MLX5_FLOW_DESTINATION_TYPE_PORT;
4206	flow_act->action \|= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
4207	}
4208
4209
4210	if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
4211	dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
4212	dst[dst_num].counter_id = counter_id;
4213	dst_num++;
4214	}
4215
4216	handler = _create_raw_flow_rule(dev, ft_prio, dst, fs_matcher,
4217	flow_context, flow_act,
4218	cmd_in, inlen, dst_num);
4219
4220	if (IS_ERR(handler)) {
4221	err = PTR_ERR(handler);
4222	goto destroy_ft;
4223	}
4224
4225	mutex_unlock(&dev->flow_db->lock);
4226	atomic_inc(&fs_matcher->usecnt);
4227	handler->flow_matcher = fs_matcher;
4228
4229	kfree(dst);
4230
4231	return handler;
4232
4233	destroy_ft:
4234	put_flow_table(dev, ft_prio, false);
4235	unlock:
4236	mutex_unlock(&dev->flow_db->lock);
4237	kfree(dst);
4238
4239	return ERR_PTR(err);
4240	}
4241
4242	static u32 mlx5_ib_flow_action_flags_to_accel_xfrm_flags(u32 mlx5_flags)
4243	{
4244	u32 flags = 0;
4245
4246	if (mlx5_flags & MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA)
4247	flags \|= MLX5_ACCEL_XFRM_FLAG_REQUIRE_METADATA;
4248
4249	return flags;
4250	}
4251
4252	#define MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA
4253	static struct ib_flow_action *
4254	mlx5_ib_create_flow_action_esp(struct ib_device *device,
4255	const struct ib_flow_action_attrs_esp *attr,
4256	struct uverbs_attr_bundle *attrs)
4257	{
4258	struct mlx5_ib_dev *mdev = to_mdev(device);
4259	struct ib_uverbs_flow_action_esp_keymat_aes_gcm *aes_gcm;
4260	struct mlx5_accel_esp_xfrm_attrs accel_attrs = {};
4261	struct mlx5_ib_flow_action *action;
4262	u64 action_flags;
4263	u64 flags;
4264	int err = 0;
4265
4266	err = uverbs_get_flags64(
4267	&action_flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_ACTION_FLAGS,
4268	((MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED << 1) - 1));
4269	if (err)
4270	return ERR_PTR(err);
4271
4272	flags = mlx5_ib_flow_action_flags_to_accel_xfrm_flags(action_flags);
4273
4274	/* We current only support a subset of the standard features. Only a
4275	* keymat of type AES_GCM, with icv_len == 16, iv_algo == SEQ and esn
4276	* (with overlap). Full offload mode isn't supported.
4277	*/
4278	if (!attr->keymat \|\| attr->replay \|\| attr->encap \|\|
4279	attr->spi \|\| attr->seq \|\| attr->tfc_pad \|\|
4280	attr->hard_limit_pkts \|\|
4281	(attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED \|
4282	IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT)))
4283	return ERR_PTR(-EOPNOTSUPP);
4284
4285	if (attr->keymat->protocol !=
4286	IB_UVERBS_FLOW_ACTION_ESP_KEYMAT_AES_GCM)
4287	return ERR_PTR(-EOPNOTSUPP);
4288
4289	aes_gcm = &attr->keymat->keymat.aes_gcm;
4290
4291	if (aes_gcm->icv_len != 16 \|\|
4292	aes_gcm->iv_algo != IB_UVERBS_FLOW_ACTION_IV_ALGO_SEQ)
4293	return ERR_PTR(-EOPNOTSUPP);
4294
4295	action = kmalloc(sizeof(*action), GFP_KERNEL);
4296	if (!action)
4297	return ERR_PTR(-ENOMEM);
4298
4299	action->esp_aes_gcm.ib_flags = attr->flags;
4300	memcpy(&accel_attrs.keymat.aes_gcm.aes_key, &aes_gcm->aes_key,
4301	sizeof(accel_attrs.keymat.aes_gcm.aes_key));
4302	accel_attrs.keymat.aes_gcm.key_len = aes_gcm->key_len * 8;
4303	memcpy(&accel_attrs.keymat.aes_gcm.salt, &aes_gcm->salt,
4304	sizeof(accel_attrs.keymat.aes_gcm.salt));
4305	memcpy(&accel_attrs.keymat.aes_gcm.seq_iv, &aes_gcm->iv,
4306	sizeof(accel_attrs.keymat.aes_gcm.seq_iv));
4307	accel_attrs.keymat.aes_gcm.icv_len = aes_gcm->icv_len * 8;
4308	accel_attrs.keymat.aes_gcm.iv_algo = MLX5_ACCEL_ESP_AES_GCM_IV_ALGO_SEQ;
4309	accel_attrs.keymat_type = MLX5_ACCEL_ESP_KEYMAT_AES_GCM;
4310
4311	accel_attrs.esn = attr->esn;
4312	if (attr->flags & IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED)
4313	accel_attrs.flags \|= MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED;
4314	if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)
4315	accel_attrs.flags \|= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;
4316
4317	if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT)
4318	accel_attrs.action \|= MLX5_ACCEL_ESP_ACTION_ENCRYPT;
4319
4320	action->esp_aes_gcm.ctx =
4321	mlx5_accel_esp_create_xfrm(mdev->mdev, &accel_attrs, flags);
4322	if (IS_ERR(action->esp_aes_gcm.ctx)) {
4323	err = PTR_ERR(action->esp_aes_gcm.ctx);
4324	goto err_parse;
4325	}
4326
4327	action->esp_aes_gcm.ib_flags = attr->flags;
4328
4329	return &action->ib_action;
4330
4331	err_parse:
4332	kfree(action);
4333	return ERR_PTR(err);
4334	}
4335
4336	static int
4337	mlx5_ib_modify_flow_action_esp(struct ib_flow_action *action,
4338	const struct ib_flow_action_attrs_esp *attr,
4339	struct uverbs_attr_bundle *attrs)
4340	{
4341	struct mlx5_ib_flow_action *maction = to_mflow_act(action);
4342	struct mlx5_accel_esp_xfrm_attrs accel_attrs;
4343	int err = 0;
4344
4345	if (attr->keymat \|\| attr->replay \|\| attr->encap \|\|
4346	attr->spi \|\| attr->seq \|\| attr->tfc_pad \|\|
4347	attr->hard_limit_pkts \|\|
4348	(attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED \|
4349	IB_FLOW_ACTION_ESP_FLAGS_MOD_ESP_ATTRS \|
4350	IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)))
4351	return -EOPNOTSUPP;
4352
4353	/* Only the ESN value or the MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP can
4354	* be modified.
4355	*/
4356	if (!(maction->esp_aes_gcm.ib_flags &
4357	IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED) &&
4358	attr->flags & (IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED \|
4359	IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW))
4360	return -EINVAL;
4361
4362	memcpy(&accel_attrs, &maction->esp_aes_gcm.ctx->attrs,
4363	sizeof(accel_attrs));
4364
4365	accel_attrs.esn = attr->esn;
4366	if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)
4367	accel_attrs.flags \|= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;
4368	else
4369	accel_attrs.flags &= ~MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;
4370
4371	err = mlx5_accel_esp_modify_xfrm(maction->esp_aes_gcm.ctx,
4372	&accel_attrs);
4373	if (err)
4374	return err;
4375
4376	maction->esp_aes_gcm.ib_flags &=
4377	~IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW;
4378	maction->esp_aes_gcm.ib_flags \|=
4379	attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW;
4380
4381	return 0;
4382	}
4383
4384	static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action)
4385	{
4386	struct mlx5_ib_flow_action *maction = to_mflow_act(action);
4387
4388	switch (action->type) {
4389	case IB_FLOW_ACTION_ESP:
4390	/*
4391	* We only support aes_gcm by now, so we implicitly know this is
4392	* the underline crypto.
4393	*/
4394	mlx5_accel_esp_destroy_xfrm(maction->esp_aes_gcm.ctx);
4395	break;
4396	case IB_FLOW_ACTION_UNSPECIFIED:
4397	mlx5_ib_destroy_flow_action_raw(maction);
4398	break;
4399	default:
4400	WARN_ON(true);
4401	break;
4402	}
4403
4404	kfree(maction);
4405	return 0;
4406	}
4407
4408	static int mlx5_ib_mcg_attach(struct ib_qp ibqp, union ib_gid gid, u16 lid)
4409	{
4410	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
4411	struct mlx5_ib_qp *mqp = to_mqp(ibqp);
4412	int err;
4413	u16 uid;
4414
4415	uid = ibqp->pd ?
4416	to_mpd(ibqp->pd)->uid : 0;
4417
4418	if (mqp->flags & IB_QP_CREATE_SOURCE_QPN) {
4419	mlx5_ib_dbg(dev, "Attaching a multi cast group to underlay QP is not supported\n");
4420	return -EOPNOTSUPP;
4421	}
4422
4423	err = mlx5_cmd_attach_mcg(dev->mdev, gid, ibqp->qp_num, uid);
4424	if (err)
4425	mlx5_ib_warn(dev, "failed attaching QPN 0x%x, MGID %pI6\n",
4426	ibqp->qp_num, gid->raw);
4427
4428	return err;
4429	}
4430
4431	static int mlx5_ib_mcg_detach(struct ib_qp ibqp, union ib_gid gid, u16 lid)
4432	{
4433	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
4434	int err;
4435	u16 uid;
4436
4437	uid = ibqp->pd ?
4438	to_mpd(ibqp->pd)->uid : 0;
4439	err = mlx5_cmd_detach_mcg(dev->mdev, gid, ibqp->qp_num, uid);
4440	if (err)
4441	mlx5_ib_warn(dev, "failed detaching QPN 0x%x, MGID %pI6\n",
4442	ibqp->qp_num, gid->raw);
4443
4444	return err;
4445	}
4446
4447	static int init_node_data(struct mlx5_ib_dev *dev)
4448	{
4449	int err;
4450
4451	err = mlx5_query_node_desc(dev, dev->ib_dev.node_desc);
4452	if (err)
4453	return err;
4454
4455	dev->mdev->rev_id = dev->mdev->pdev->revision;
4456
4457	return mlx5_query_node_guid(dev, &dev->ib_dev.node_guid);
4458	}
4459
4460	static ssize_t fw_pages_show(struct device *device,
4461	struct device_attribute attr, char buf)
4462	{
4463	struct mlx5_ib_dev *dev =
4464	rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
4465
4466	return sprintf(buf, "%d\n", dev->mdev->priv.fw_pages);
4467	}
4468	static DEVICE_ATTR_RO(fw_pages);
4469
4470	static ssize_t reg_pages_show(struct device *device,
4471	struct device_attribute attr, char buf)
4472	{
4473	struct mlx5_ib_dev *dev =
4474	rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
4475
4476	return sprintf(buf, "%d\n", atomic_read(&dev->mdev->priv.reg_pages));
4477	}
4478	static DEVICE_ATTR_RO(reg_pages);
4479
4480	static ssize_t hca_type_show(struct device *device,
4481	struct device_attribute attr, char buf)
4482	{
4483	struct mlx5_ib_dev *dev =
4484	rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
4485
4486	return sprintf(buf, "MT%d\n", dev->mdev->pdev->device);
4487	}
4488	static DEVICE_ATTR_RO(hca_type);
4489
4490	static ssize_t hw_rev_show(struct device *device,
4491	struct device_attribute attr, char buf)
4492	{
4493	struct mlx5_ib_dev *dev =
4494	rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
4495
4496	return sprintf(buf, "%x\n", dev->mdev->rev_id);
4497	}
4498	static DEVICE_ATTR_RO(hw_rev);
4499
4500	static ssize_t board_id_show(struct device *device,
4501	struct device_attribute attr, char buf)
4502	{
4503	struct mlx5_ib_dev *dev =
4504	rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
4505
4506	return sprintf(buf, "%.*s\n", MLX5_BOARD_ID_LEN,
4507	dev->mdev->board_id);
4508	}
4509	static DEVICE_ATTR_RO(board_id);
4510
4511	static struct attribute *mlx5_class_attributes[] = {
4512	&dev_attr_hw_rev.attr,
4513	&dev_attr_hca_type.attr,
4514	&dev_attr_board_id.attr,
4515	&dev_attr_fw_pages.attr,
4516	&dev_attr_reg_pages.attr,
4517	NULL,
4518	};
4519
4520	static const struct attribute_group mlx5_attr_group = {
4521	.attrs = mlx5_class_attributes,
4522	};
4523
4524	static void pkey_change_handler(struct work_struct *work)
4525	{
4526	struct mlx5_ib_port_resources *ports =
4527	container_of(work, struct mlx5_ib_port_resources,
4528	pkey_change_work);
4529
4530	mutex_lock(&ports->devr->mutex);
4531	mlx5_ib_gsi_pkey_change(ports->gsi);
4532	mutex_unlock(&ports->devr->mutex);
4533	}
4534
4535	static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev)
4536	{
4537	struct mlx5_ib_qp *mqp;
4538	struct mlx5_ib_cq send_mcq, recv_mcq;
4539	struct mlx5_core_cq *mcq;
4540	struct list_head cq_armed_list;
4541	unsigned long flags_qp;
4542	unsigned long flags_cq;
4543	unsigned long flags;
4544
4545	INIT_LIST_HEAD(&cq_armed_list);
4546
4547	/* Go over qp list reside on that ibdev, sync with create/destroy qp.*/
4548	spin_lock_irqsave(&ibdev->reset_flow_resource_lock, flags);
4549	list_for_each_entry(mqp, &ibdev->qp_list, qps_list) {
4550	spin_lock_irqsave(&mqp->sq.lock, flags_qp);
4551	if (mqp->sq.tail != mqp->sq.head) {
4552	send_mcq = to_mcq(mqp->ibqp.send_cq);
4553	spin_lock_irqsave(&send_mcq->lock, flags_cq);
4554	if (send_mcq->mcq.comp &&
4555	mqp->ibqp.send_cq->comp_handler) {
4556	if (!send_mcq->mcq.reset_notify_added) {
4557	send_mcq->mcq.reset_notify_added = 1;
4558	list_add_tail(&send_mcq->mcq.reset_notify,
4559	&cq_armed_list);
4560	}
4561	}
4562	spin_unlock_irqrestore(&send_mcq->lock, flags_cq);
4563	}
4564	spin_unlock_irqrestore(&mqp->sq.lock, flags_qp);
4565	spin_lock_irqsave(&mqp->rq.lock, flags_qp);
4566	/* no handling is needed for SRQ */
4567	if (!mqp->ibqp.srq) {
4568	if (mqp->rq.tail != mqp->rq.head) {
4569	recv_mcq = to_mcq(mqp->ibqp.recv_cq);
4570	spin_lock_irqsave(&recv_mcq->lock, flags_cq);
4571	if (recv_mcq->mcq.comp &&
4572	mqp->ibqp.recv_cq->comp_handler) {
4573	if (!recv_mcq->mcq.reset_notify_added) {
4574	recv_mcq->mcq.reset_notify_added = 1;
4575	list_add_tail(&recv_mcq->mcq.reset_notify,
4576	&cq_armed_list);
4577	}
4578	}
4579	spin_unlock_irqrestore(&recv_mcq->lock,
4580	flags_cq);
4581	}
4582	}
4583	spin_unlock_irqrestore(&mqp->rq.lock, flags_qp);
4584	}
4585	/*At that point all inflight post send were put to be executed as of we
4586	* lock/unlock above locks Now need to arm all involved CQs.
4587	*/
4588	list_for_each_entry(mcq, &cq_armed_list, reset_notify) {
4589	mcq->comp(mcq, NULL);
4590	}
4591	spin_unlock_irqrestore(&ibdev->reset_flow_resource_lock, flags);
4592	}
4593
4594	static void delay_drop_handler(struct work_struct *work)
4595	{
4596	int err;
4597	struct mlx5_ib_delay_drop *delay_drop =
4598	container_of(work, struct mlx5_ib_delay_drop,
4599	delay_drop_work);
4600
4601	atomic_inc(&delay_drop->events_cnt);
4602
4603	mutex_lock(&delay_drop->lock);
4604	err = mlx5_core_set_delay_drop(delay_drop->dev, delay_drop->timeout);
4605	if (err) {
4606	mlx5_ib_warn(delay_drop->dev, "Failed to set delay drop, timeout=%u\n",
4607	delay_drop->timeout);
4608	delay_drop->activate = false;
4609	}
4610	mutex_unlock(&delay_drop->lock);
4611	}
4612
4613	static void handle_general_event(struct mlx5_ib_dev ibdev, struct mlx5_eqe eqe,
4614	struct ib_event *ibev)
4615	{
4616	u8 port = (eqe->data.port.port >> 4) & 0xf;
4617
4618	switch (eqe->sub_type) {
4619	case MLX5_GENERAL_SUBTYPE_DELAY_DROP_TIMEOUT:
4620	if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) ==
4621	IB_LINK_LAYER_ETHERNET)
4622	schedule_work(&ibdev->delay_drop.delay_drop_work);
4623	break;
4624	default: /* do nothing */
4625	return;
4626	}
4627	}
4628
4629	static int handle_port_change(struct mlx5_ib_dev ibdev, struct mlx5_eqe eqe,
4630	struct ib_event *ibev)
4631	{
4632	u8 port = (eqe->data.port.port >> 4) & 0xf;
4633
4634	ibev->element.port_num = port;
4635
4636	switch (eqe->sub_type) {
4637	case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
4638	case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
4639	case MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED:
4640	/* In RoCE, port up/down events are handled in
4641	* mlx5_netdev_event().
4642	*/
4643	if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) ==
4644	IB_LINK_LAYER_ETHERNET)
4645	return -EINVAL;
4646
4647	ibev->event = (eqe->sub_type == MLX5_PORT_CHANGE_SUBTYPE_ACTIVE) ?
4648	IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
4649	break;
4650
4651	case MLX5_PORT_CHANGE_SUBTYPE_LID:
4652	ibev->event = IB_EVENT_LID_CHANGE;
4653	break;
4654
4655	case MLX5_PORT_CHANGE_SUBTYPE_PKEY:
4656	ibev->event = IB_EVENT_PKEY_CHANGE;
4657	schedule_work(&ibdev->devr.ports[port - 1].pkey_change_work);
4658	break;
4659
4660	case MLX5_PORT_CHANGE_SUBTYPE_GUID:
4661	ibev->event = IB_EVENT_GID_CHANGE;
4662	break;
4663
4664	case MLX5_PORT_CHANGE_SUBTYPE_CLIENT_REREG:
4665	ibev->event = IB_EVENT_CLIENT_REREGISTER;
4666	break;
4667	default:
4668	return -EINVAL;
4669	}
4670
4671	return 0;
4672	}
4673
4674	static void mlx5_ib_handle_event(struct work_struct *_work)
4675	{
4676	struct mlx5_ib_event_work *work =
4677	container_of(_work, struct mlx5_ib_event_work, work);
4678	struct mlx5_ib_dev *ibdev;
4679	struct ib_event ibev;
4680	bool fatal = false;
4681
4682	if (work->is_slave) {
4683	ibdev = mlx5_ib_get_ibdev_from_mpi(work->mpi);
4684	if (!ibdev)
4685	goto out;
4686	} else {
4687	ibdev = work->dev;
4688	}
4689
4690	switch (work->event) {
4691	case MLX5_DEV_EVENT_SYS_ERROR:
4692	ibev.event = IB_EVENT_DEVICE_FATAL;
4693	mlx5_ib_handle_internal_error(ibdev);
4694	ibev.element.port_num = (u8)(unsigned long)work->param;
4695	fatal = true;
4696	break;
4697	case MLX5_EVENT_TYPE_PORT_CHANGE:
4698	if (handle_port_change(ibdev, work->param, &ibev))
4699	goto out;
4700	break;
4701	case MLX5_EVENT_TYPE_GENERAL_EVENT:
4702	handle_general_event(ibdev, work->param, &ibev);
4703	/* fall through */
4704	default:
4705	goto out;
4706	}
4707
4708	ibev.device = &ibdev->ib_dev;
4709
4710	if (!rdma_is_port_valid(&ibdev->ib_dev, ibev.element.port_num)) {
4711	mlx5_ib_warn(ibdev, "warning: event on port %d\n", ibev.element.port_num);
4712	goto out;
4713	}
4714
4715	if (ibdev->ib_active)
4716	ib_dispatch_event(&ibev);
4717
4718	if (fatal)
4719	ibdev->ib_active = false;
4720	out:
4721	kfree(work);
4722	}
4723
4724	static int mlx5_ib_event(struct notifier_block *nb,
4725	unsigned long event, void *param)
4726	{
4727	struct mlx5_ib_event_work *work;
4728
4729	work = kmalloc(sizeof(*work), GFP_ATOMIC);
4730	if (!work)
4731	return NOTIFY_DONE;
4732
4733	INIT_WORK(&work->work, mlx5_ib_handle_event);
4734	work->dev = container_of(nb, struct mlx5_ib_dev, mdev_events);
4735	work->is_slave = false;
4736	work->param = param;
4737	work->event = event;
4738
4739	queue_work(mlx5_ib_event_wq, &work->work);
4740
4741	return NOTIFY_OK;
4742	}
4743
4744	static int mlx5_ib_event_slave_port(struct notifier_block *nb,
4745	unsigned long event, void *param)
4746	{
4747	struct mlx5_ib_event_work *work;
4748
4749	work = kmalloc(sizeof(*work), GFP_ATOMIC);
4750	if (!work)
4751	return NOTIFY_DONE;
4752
4753	INIT_WORK(&work->work, mlx5_ib_handle_event);
4754	work->mpi = container_of(nb, struct mlx5_ib_multiport_info, mdev_events);
4755	work->is_slave = true;
4756	work->param = param;
4757	work->event = event;
4758	queue_work(mlx5_ib_event_wq, &work->work);
4759
4760	return NOTIFY_OK;
4761	}
4762
4763	static int set_has_smi_cap(struct mlx5_ib_dev *dev)
4764	{
4765	struct mlx5_hca_vport_context vport_ctx;
4766	int err;
4767	int port;
4768
4769	for (port = 1; port <= ARRAY_SIZE(dev->mdev->port_caps); port++) {
4770	dev->mdev->port_caps[port - 1].has_smi = false;
4771	if (MLX5_CAP_GEN(dev->mdev, port_type) ==
4772	MLX5_CAP_PORT_TYPE_IB) {
4773	if (MLX5_CAP_GEN(dev->mdev, ib_virt)) {
4774	err = mlx5_query_hca_vport_context(dev->mdev, 0,
4775	port, 0,
4776	&vport_ctx);
4777	if (err) {
4778	mlx5_ib_err(dev, "query_hca_vport_context for port=%d failed %d\n",
4779	port, err);
4780	return err;
4781	}
4782	dev->mdev->port_caps[port - 1].has_smi =
4783	vport_ctx.has_smi;
4784	} else {
4785	dev->mdev->port_caps[port - 1].has_smi = true;
4786	}
4787	}
4788	}
4789	return 0;
4790	}
4791
4792	static void get_ext_port_caps(struct mlx5_ib_dev *dev)
4793	{
4794	int port;
4795
4796	for (port = 1; port <= dev->num_ports; port++)
4797	mlx5_query_ext_port_caps(dev, port);
4798	}
4799
4800	static int __get_port_caps(struct mlx5_ib_dev *dev, u8 port)
4801	{
4802	struct ib_device_attr *dprops = NULL;
4803	struct ib_port_attr *pprops = NULL;
4804	int err = -ENOMEM;
4805
4806	pprops = kzalloc(sizeof(*pprops), GFP_KERNEL);
4807	if (!pprops)
4808	goto out;
4809
4810	dprops = kmalloc(sizeof(*dprops), GFP_KERNEL);
4811	if (!dprops)
4812	goto out;
4813
4814	err = mlx5_ib_query_device(&dev->ib_dev, dprops, NULL);
4815	if (err) {
4816	mlx5_ib_warn(dev, "query_device failed %d\n", err);
4817	goto out;
4818	}
4819
4820	err = mlx5_ib_query_port(&dev->ib_dev, port, pprops);
4821	if (err) {
4822	mlx5_ib_warn(dev, "query_port %d failed %d\n",
4823	port, err);
4824	goto out;
4825	}
4826
4827	dev->mdev->port_caps[port - 1].pkey_table_len =
4828	dprops->max_pkeys;
4829	dev->mdev->port_caps[port - 1].gid_table_len =
4830	pprops->gid_tbl_len;
4831	mlx5_ib_dbg(dev, "port %d: pkey_table_len %d, gid_table_len %d\n",
4832	port, dprops->max_pkeys, pprops->gid_tbl_len);
4833
4834	out:
4835	kfree(pprops);
4836	kfree(dprops);
4837
4838	return err;
4839	}
4840
4841	static int get_port_caps(struct mlx5_ib_dev *dev, u8 port)
4842	{
4843	/* For representors use port 1, is this is the only native
4844	* port
4845	*/
4846	if (dev->is_rep)
4847	return __get_port_caps(dev, 1);
4848	return __get_port_caps(dev, port);
4849	}
4850
4851	static void destroy_umrc_res(struct mlx5_ib_dev *dev)
4852	{
4853	int err;
4854
4855	err = mlx5_mr_cache_cleanup(dev);
4856	if (err)
4857	mlx5_ib_warn(dev, "mr cache cleanup failed\n");
4858
4859	if (dev->umrc.qp)
4860	mlx5_ib_destroy_qp(dev->umrc.qp, NULL);
4861	if (dev->umrc.cq)
4862	ib_free_cq(dev->umrc.cq);
4863	if (dev->umrc.pd)
4864	ib_dealloc_pd(dev->umrc.pd);
4865	}
4866
4867	enum {
4868	MAX_UMR_WR = 128,
4869	};
4870
4871	static int create_umr_res(struct mlx5_ib_dev *dev)
4872	{
4873	struct ib_qp_init_attr *init_attr = NULL;
4874	struct ib_qp_attr *attr = NULL;
4875	struct ib_pd *pd;
4876	struct ib_cq *cq;
4877	struct ib_qp *qp;
4878	int ret;
4879
4880	attr = kzalloc(sizeof(*attr), GFP_KERNEL);
4881	init_attr = kzalloc(sizeof(*init_attr), GFP_KERNEL);
4882	if (!attr \|\| !init_attr) {
4883	ret = -ENOMEM;
4884	goto error_0;
4885	}
4886
4887	pd = ib_alloc_pd(&dev->ib_dev, 0);
4888	if (IS_ERR(pd)) {
4889	mlx5_ib_dbg(dev, "Couldn't create PD for sync UMR QP\n");
4890	ret = PTR_ERR(pd);
4891	goto error_0;
4892	}
4893
4894	cq = ib_alloc_cq(&dev->ib_dev, NULL, 128, 0, IB_POLL_SOFTIRQ);
4895	if (IS_ERR(cq)) {
4896	mlx5_ib_dbg(dev, "Couldn't create CQ for sync UMR QP\n");
4897	ret = PTR_ERR(cq);
4898	goto error_2;
4899	}
4900
4901	init_attr->send_cq = cq;
4902	init_attr->recv_cq = cq;
4903	init_attr->sq_sig_type = IB_SIGNAL_ALL_WR;
4904	init_attr->cap.max_send_wr = MAX_UMR_WR;
4905	init_attr->cap.max_send_sge = 1;
4906	init_attr->qp_type = MLX5_IB_QPT_REG_UMR;
4907	init_attr->port_num = 1;
4908	qp = mlx5_ib_create_qp(pd, init_attr, NULL);
4909	if (IS_ERR(qp)) {
4910	mlx5_ib_dbg(dev, "Couldn't create sync UMR QP\n");
4911	ret = PTR_ERR(qp);
4912	goto error_3;
4913	}
4914	qp->device = &dev->ib_dev;
4915	qp->real_qp = qp;
4916	qp->uobject = NULL;
4917	qp->qp_type = MLX5_IB_QPT_REG_UMR;
4918	qp->send_cq = init_attr->send_cq;
4919	qp->recv_cq = init_attr->recv_cq;
4920
4921	attr->qp_state = IB_QPS_INIT;
4922	attr->port_num = 1;
4923	ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE \| IB_QP_PKEY_INDEX \|
4924	IB_QP_PORT, NULL);
4925	if (ret) {
4926	mlx5_ib_dbg(dev, "Couldn't modify UMR QP\n");
4927	goto error_4;
4928	}
4929
4930	memset(attr, 0, sizeof(*attr));
4931	attr->qp_state = IB_QPS_RTR;
4932	attr->path_mtu = IB_MTU_256;
4933
4934	ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL);
4935	if (ret) {
4936	mlx5_ib_dbg(dev, "Couldn't modify umr QP to rtr\n");
4937	goto error_4;
4938	}
4939
4940	memset(attr, 0, sizeof(*attr));
4941	attr->qp_state = IB_QPS_RTS;
4942	ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL);
4943	if (ret) {
4944	mlx5_ib_dbg(dev, "Couldn't modify umr QP to rts\n");
4945	goto error_4;
4946	}
4947
4948	dev->umrc.qp = qp;
4949	dev->umrc.cq = cq;
4950	dev->umrc.pd = pd;
4951
4952	sema_init(&dev->umrc.sem, MAX_UMR_WR);
4953	ret = mlx5_mr_cache_init(dev);
4954	if (ret) {
4955	mlx5_ib_warn(dev, "mr cache init failed %d\n", ret);
4956	goto error_4;
4957	}
4958
4959	kfree(attr);
4960	kfree(init_attr);
4961
4962	return 0;
4963
4964	error_4:
4965	mlx5_ib_destroy_qp(qp, NULL);
4966	dev->umrc.qp = NULL;
4967
4968	error_3:
4969	ib_free_cq(cq);
4970	dev->umrc.cq = NULL;
4971
4972	error_2:
4973	ib_dealloc_pd(pd);
4974	dev->umrc.pd = NULL;
4975
4976	error_0:
4977	kfree(attr);
4978	kfree(init_attr);
4979	return ret;
4980	}
4981
4982	static u8 mlx5_get_umr_fence(u8 umr_fence_cap)
4983	{
4984	switch (umr_fence_cap) {
4985	case MLX5_CAP_UMR_FENCE_NONE:
4986	return MLX5_FENCE_MODE_NONE;
4987	case MLX5_CAP_UMR_FENCE_SMALL:
4988	return MLX5_FENCE_MODE_INITIATOR_SMALL;
4989	default:
4990	return MLX5_FENCE_MODE_STRONG_ORDERING;
4991	}
4992	}
4993
4994	static int create_dev_resources(struct mlx5_ib_resources *devr)
4995	{
4996	struct ib_srq_init_attr attr;
4997	struct mlx5_ib_dev *dev;
4998	struct ib_device *ibdev;
4999	struct ib_cq_init_attr cq_attr = {.cqe = 1};
5000	int port;
5001	int ret = 0;
5002
5003	dev = container_of(devr, struct mlx5_ib_dev, devr);
5004	ibdev = &dev->ib_dev;
5005
5006	mutex_init(&devr->mutex);
5007
5008	devr->p0 = rdma_zalloc_drv_obj(ibdev, ib_pd);
5009	if (!devr->p0)
5010	return -ENOMEM;
5011
5012	devr->p0->device = ibdev;
5013	devr->p0->uobject = NULL;
5014	atomic_set(&devr->p0->usecnt, 0);
5015
5016	ret = mlx5_ib_alloc_pd(devr->p0, NULL);
5017	if (ret)
5018	goto error0;
5019
5020	devr->c0 = rdma_zalloc_drv_obj(ibdev, ib_cq);
5021	if (!devr->c0) {
5022	ret = -ENOMEM;
5023	goto error1;
5024	}
5025
5026	devr->c0->device = &dev->ib_dev;
5027	atomic_set(&devr->c0->usecnt, 0);
5028
5029	ret = mlx5_ib_create_cq(devr->c0, &cq_attr, NULL);
5030	if (ret)
5031	goto err_create_cq;
5032
5033	devr->x0 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL);
5034	if (IS_ERR(devr->x0)) {
5035	ret = PTR_ERR(devr->x0);
5036	goto error2;
5037	}
5038	devr->x0->device = &dev->ib_dev;
5039	devr->x0->inode = NULL;
5040	atomic_set(&devr->x0->usecnt, 0);
5041	mutex_init(&devr->x0->tgt_qp_mutex);
5042	INIT_LIST_HEAD(&devr->x0->tgt_qp_list);
5043
5044	devr->x1 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL);
5045	if (IS_ERR(devr->x1)) {
5046	ret = PTR_ERR(devr->x1);
5047	goto error3;
5048	}
5049	devr->x1->device = &dev->ib_dev;
5050	devr->x1->inode = NULL;
5051	atomic_set(&devr->x1->usecnt, 0);
5052	mutex_init(&devr->x1->tgt_qp_mutex);
5053	INIT_LIST_HEAD(&devr->x1->tgt_qp_list);
5054
5055	memset(&attr, 0, sizeof(attr));
5056	attr.attr.max_sge = 1;
5057	attr.attr.max_wr = 1;
5058	attr.srq_type = IB_SRQT_XRC;
5059	attr.ext.cq = devr->c0;
5060	attr.ext.xrc.xrcd = devr->x0;
5061
5062	devr->s0 = rdma_zalloc_drv_obj(ibdev, ib_srq);
5063	if (!devr->s0) {
5064	ret = -ENOMEM;
5065	goto error4;
5066	}
5067
5068	devr->s0->device = &dev->ib_dev;
5069	devr->s0->pd = devr->p0;
5070	devr->s0->srq_type = IB_SRQT_XRC;
5071	devr->s0->ext.xrc.xrcd = devr->x0;
5072	devr->s0->ext.cq = devr->c0;
5073	ret = mlx5_ib_create_srq(devr->s0, &attr, NULL);
5074	if (ret)
5075	goto err_create;
5076
5077	atomic_inc(&devr->s0->ext.xrc.xrcd->usecnt);
5078	atomic_inc(&devr->s0->ext.cq->usecnt);
5079	atomic_inc(&devr->p0->usecnt);
5080	atomic_set(&devr->s0->usecnt, 0);
5081
5082	memset(&attr, 0, sizeof(attr));
5083	attr.attr.max_sge = 1;
5084	attr.attr.max_wr = 1;
5085	attr.srq_type = IB_SRQT_BASIC;
5086	devr->s1 = rdma_zalloc_drv_obj(ibdev, ib_srq);
5087	if (!devr->s1) {
5088	ret = -ENOMEM;
5089	goto error5;
5090	}
5091
5092	devr->s1->device = &dev->ib_dev;
5093	devr->s1->pd = devr->p0;
5094	devr->s1->srq_type = IB_SRQT_BASIC;
5095	devr->s1->ext.cq = devr->c0;
5096
5097	ret = mlx5_ib_create_srq(devr->s1, &attr, NULL);
5098	if (ret)
5099	goto error6;
5100
5101	atomic_inc(&devr->p0->usecnt);
5102	atomic_set(&devr->s1->usecnt, 0);
5103
5104	for (port = 0; port < ARRAY_SIZE(devr->ports); ++port) {
5105	INIT_WORK(&devr->ports[port].pkey_change_work,
5106	pkey_change_handler);
5107	devr->ports[port].devr = devr;
5108	}
5109
5110	return 0;
5111
5112	error6:
5113	kfree(devr->s1);
5114	error5:
5115	mlx5_ib_destroy_srq(devr->s0, NULL);
5116	err_create:
5117	kfree(devr->s0);
5118	error4:
5119	mlx5_ib_dealloc_xrcd(devr->x1, NULL);
5120	error3:
5121	mlx5_ib_dealloc_xrcd(devr->x0, NULL);
5122	error2:
5123	mlx5_ib_destroy_cq(devr->c0, NULL);
5124	err_create_cq:
5125	kfree(devr->c0);
5126	error1:
5127	mlx5_ib_dealloc_pd(devr->p0, NULL);
5128	error0:
5129	kfree(devr->p0);
5130	return ret;
5131	}
5132
5133	static void destroy_dev_resources(struct mlx5_ib_resources *devr)
5134	{
5135	int port;
5136
5137	mlx5_ib_destroy_srq(devr->s1, NULL);
5138	kfree(devr->s1);
5139	mlx5_ib_destroy_srq(devr->s0, NULL);
5140	kfree(devr->s0);
5141	mlx5_ib_dealloc_xrcd(devr->x0, NULL);
5142	mlx5_ib_dealloc_xrcd(devr->x1, NULL);
5143	mlx5_ib_destroy_cq(devr->c0, NULL);
5144	kfree(devr->c0);
5145	mlx5_ib_dealloc_pd(devr->p0, NULL);
5146	kfree(devr->p0);
5147
5148	/* Make sure no change P_Key work items are still executing */
5149	for (port = 0; port < ARRAY_SIZE(devr->ports); ++port)
5150	cancel_work_sync(&devr->ports[port].pkey_change_work);
5151	}
5152
5153	static u32 get_core_cap_flags(struct ib_device *ibdev,
5154	struct mlx5_hca_vport_context *rep)
5155	{
5156	struct mlx5_ib_dev *dev = to_mdev(ibdev);
5157	enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, 1);
5158	u8 l3_type_cap = MLX5_CAP_ROCE(dev->mdev, l3_type);
5159	u8 roce_version_cap = MLX5_CAP_ROCE(dev->mdev, roce_version);
5160	bool raw_support = !mlx5_core_mp_enabled(dev->mdev);
5161	u32 ret = 0;
5162
5163	if (rep->grh_required)
5164	ret \|= RDMA_CORE_CAP_IB_GRH_REQUIRED;
5165
5166	if (ll == IB_LINK_LAYER_INFINIBAND)
5167	return ret \| RDMA_CORE_PORT_IBA_IB;
5168
5169	if (raw_support)
5170	ret \|= RDMA_CORE_PORT_RAW_PACKET;
5171
5172	if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV4_CAP))
5173	return ret;
5174
5175	if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV6_CAP))
5176	return ret;
5177
5178	if (roce_version_cap & MLX5_ROCE_VERSION_1_CAP)
5179	ret \|= RDMA_CORE_PORT_IBA_ROCE;
5180
5181	if (roce_version_cap & MLX5_ROCE_VERSION_2_CAP)
5182	ret \|= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
5183
5184	return ret;
5185	}
5186
5187	static int mlx5_port_immutable(struct ib_device *ibdev, u8 port_num,
5188	struct ib_port_immutable *immutable)
5189	{
5190	struct ib_port_attr attr;
5191	struct mlx5_ib_dev *dev = to_mdev(ibdev);
5192	enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, port_num);
5193	struct mlx5_hca_vport_context rep = {0};
5194	int err;
5195
5196	err = ib_query_port(ibdev, port_num, &attr);
5197	if (err)
5198	return err;
5199
5200	if (ll == IB_LINK_LAYER_INFINIBAND) {
5201	err = mlx5_query_hca_vport_context(dev->mdev, 0, port_num, 0,
5202	&rep);
5203	if (err)
5204	return err;
5205	}
5206
5207	immutable->pkey_tbl_len = attr.pkey_tbl_len;
5208	immutable->gid_tbl_len = attr.gid_tbl_len;
5209	immutable->core_cap_flags = get_core_cap_flags(ibdev, &rep);
5210	immutable->max_mad_size = IB_MGMT_MAD_SIZE;
5211
5212	return 0;
5213	}
5214
5215	static int mlx5_port_rep_immutable(struct ib_device *ibdev, u8 port_num,
5216	struct ib_port_immutable *immutable)
5217	{
5218	struct ib_port_attr attr;
5219	int err;
5220
5221	immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
5222
5223	err = ib_query_port(ibdev, port_num, &attr);
5224	if (err)
5225	return err;
5226
5227	immutable->pkey_tbl_len = attr.pkey_tbl_len;
5228	immutable->gid_tbl_len = attr.gid_tbl_len;
5229	immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
5230
5231	return 0;
5232	}
5233
5234	static void get_dev_fw_str(struct ib_device ibdev, char str)
5235	{
5236	struct mlx5_ib_dev *dev =
5237	container_of(ibdev, struct mlx5_ib_dev, ib_dev);
5238	snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%04d",
5239	fw_rev_maj(dev->mdev), fw_rev_min(dev->mdev),
5240	fw_rev_sub(dev->mdev));
5241	}
5242
5243	static int mlx5_eth_lag_init(struct mlx5_ib_dev *dev)
5244	{
5245	struct mlx5_core_dev *mdev = dev->mdev;
5246	struct mlx5_flow_namespace *ns = mlx5_get_flow_namespace(mdev,
5247	MLX5_FLOW_NAMESPACE_LAG);
5248	struct mlx5_flow_table *ft;
5249	int err;
5250
5251	if (!ns \|\| !mlx5_lag_is_roce(mdev))
5252	return 0;
5253
5254	err = mlx5_cmd_create_vport_lag(mdev);
5255	if (err)
5256	return err;
5257
5258	ft = mlx5_create_lag_demux_flow_table(ns, 0, 0);
5259	if (IS_ERR(ft)) {
5260	err = PTR_ERR(ft);
5261	goto err_destroy_vport_lag;
5262	}
5263
5264	dev->flow_db->lag_demux_ft = ft;
5265	dev->lag_active = true;
5266	return 0;
5267
5268	err_destroy_vport_lag:
5269	mlx5_cmd_destroy_vport_lag(mdev);
5270	return err;
5271	}
5272
5273	static void mlx5_eth_lag_cleanup(struct mlx5_ib_dev *dev)
5274	{
5275	struct mlx5_core_dev *mdev = dev->mdev;
5276
5277	if (dev->lag_active) {
5278	dev->lag_active = false;
5279		2920
5280	mlx5_destroy_flow_table(dev->flow_db->lag_demux_ft);	2921	work = kmalloc(sizeof(*work), GFP_ATOMIC);
5281	dev->flow_db->lag_demux_ft = NULL;	2922	if (!work)
		2923	return NOTIFY_DONE;
5282		2924
5283	mlx5_cmd_destroy_vport_lag(mdev);	2925	INIT_WORK(&work->work, mlx5_ib_handle_event);
5284	}	2926	work->mpi = container_of(nb, struct mlx5_ib_multiport_info, mdev_events);
		2927	work->is_slave = true;
		2928	work->param = param;
		2929	work->event = event;
		2930	queue_work(mlx5_ib_event_wq, &work->work);
		2931
		2932	return NOTIFY_OK;
5285	}	2933	}
5286		2934
5287	static int mlx5_add_netdev_notifier(struct mlx5_ib_dev *dev, u8 port_num)	2935	static int set_has_smi_cap(struct mlx5_ib_dev *dev)
5288	{	2936	{
		2937	struct mlx5_hca_vport_context vport_ctx;
5289	int err;	2938	int err;
		2939	int port;
5290		2940
5291	dev->port[port_num].roce.nb.notifier_call = mlx5_netdev_event;	2941	for (port = 1; port <= ARRAY_SIZE(dev->mdev->port_caps); port++) {
5292	err = register_netdevice_notifier(&dev->port[port_num].roce.nb);	2942	dev->mdev->port_caps[port - 1].has_smi = false;
5293	if (err) {	2943	if (MLX5_CAP_GEN(dev->mdev, port_type) ==
5294	dev->port[port_num].roce.nb.notifier_call = NULL;	2944	MLX5_CAP_PORT_TYPE_IB) {
5295	return err;	2945	if (MLX5_CAP_GEN(dev->mdev, ib_virt)) {
		2946	err = mlx5_query_hca_vport_context(dev->mdev, 0,
		2947	port, 0,
		2948	&vport_ctx);
		2949	if (err) {
		2950	mlx5_ib_err(dev, "query_hca_vport_context for port=%d failed %d\n",
		2951	port, err);
		2952	return err;
		2953	}
		2954	dev->mdev->port_caps[port - 1].has_smi =
		2955	vport_ctx.has_smi;
		2956	} else {
		2957	dev->mdev->port_caps[port - 1].has_smi = true;
		2958	}
		2959	}
5296	}	2960	}
5297
5298	return 0;	2961	return 0;
5299	}	2962	}
5300		2963
5301	static void mlx5_remove_netdev_notifier(struct mlx5_ib_dev *dev, u8 port_num)	2964	static void get_ext_port_caps(struct mlx5_ib_dev *dev)
5302	{	2965	{
5303	if (dev->port[port_num].roce.nb.notifier_call) {	2966	int port;
5304	unregister_netdevice_notifier(&dev->port[port_num].roce.nb);	2967
5305	dev->port[port_num].roce.nb.notifier_call = NULL;	2968	for (port = 1; port <= dev->num_ports; port++)
5306	}	2969	mlx5_query_ext_port_caps(dev, port);
5307	}	2970	}
5308		2971
5309	static int mlx5_enable_eth(struct mlx5_ib_dev *dev)	2972	static int __get_port_caps(struct mlx5_ib_dev *dev, u8 port)
5310	{	2973	{
5311	int err;	2974	struct ib_device_attr *dprops = NULL;
		2975	struct ib_port_attr *pprops = NULL;
		2976	int err = -ENOMEM;
5312		2977
5313	err = mlx5_nic_vport_enable_roce(dev->mdev);	2978	pprops = kzalloc(sizeof(*pprops), GFP_KERNEL);
5314	if (err)	2979	if (!pprops)
5315	return err;	2980	goto out;
5316		2981
5317	err = mlx5_eth_lag_init(dev);	2982	dprops = kmalloc(sizeof(*dprops), GFP_KERNEL);
5318	if (err)	2983	if (!dprops)
5319	goto err_disable_roce;	2984	goto out;
5320		2985
5321	return 0;	2986	err = mlx5_ib_query_device(&dev->ib_dev, dprops, NULL);
		2987	if (err) {
		2988	mlx5_ib_warn(dev, "query_device failed %d\n", err);
		2989	goto out;
		2990	}
5322		2991
5323	err_disable_roce:	2992	err = mlx5_ib_query_port(&dev->ib_dev, port, pprops);
5324	mlx5_nic_vport_disable_roce(dev->mdev);	2993	if (err) {
		2994	mlx5_ib_warn(dev, "query_port %d failed %d\n",
		2995	port, err);
		2996	goto out;
		2997	}
		2998
		2999	dev->mdev->port_caps[port - 1].pkey_table_len =
		3000	dprops->max_pkeys;
		3001	dev->mdev->port_caps[port - 1].gid_table_len =
		3002	pprops->gid_tbl_len;
		3003	mlx5_ib_dbg(dev, "port %d: pkey_table_len %d, gid_table_len %d\n",
		3004	port, dprops->max_pkeys, pprops->gid_tbl_len);
		3005
		3006	out:
		3007	kfree(pprops);
		3008	kfree(dprops);
5325		3009
5326	return err;	3010	return err;
5327	}	3011	}
5328		3012
5329	static void mlx5_disable_eth(struct mlx5_ib_dev *dev)	3013	static int get_port_caps(struct mlx5_ib_dev *dev, u8 port)
5330	{	3014	{
5331	mlx5_eth_lag_cleanup(dev);	3015	/* For representors use port 1, is this is the only native
5332	mlx5_nic_vport_disable_roce(dev->mdev);	3016	* port
		3017	*/
		3018	if (dev->is_rep)
		3019	return __get_port_caps(dev, 1);
		3020	return __get_port_caps(dev, port);
5333	}	3021	}
5334		3022
5335	struct mlx5_ib_counter {	3023	static u8 mlx5_get_umr_fence(u8 umr_fence_cap)
5336	const char *name;	3024	{
5337	size_t offset;	3025	switch (umr_fence_cap) {
5338	};	3026	case MLX5_CAP_UMR_FENCE_NONE:
5339		3027	return MLX5_FENCE_MODE_NONE;
5340	#define INIT_Q_COUNTER(_name) \	3028	case MLX5_CAP_UMR_FENCE_SMALL:
5341	{ .name = #_name, .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)}	3029	return MLX5_FENCE_MODE_INITIATOR_SMALL;
		3030	default:
		3031	return MLX5_FENCE_MODE_STRONG_ORDERING;
		3032	}
		3033	}
5342		3034
5343	static const struct mlx5_ib_counter basic_q_cnts[] = {	3035	static int mlx5_ib_dev_res_init(struct mlx5_ib_dev *dev)
5344	INIT_Q_COUNTER(rx_write_requests),	3036	{
5345	INIT_Q_COUNTER(rx_read_requests),	3037	struct mlx5_ib_resources *devr = &dev->devr;
5346	INIT_Q_COUNTER(rx_atomic_requests),	3038	struct ib_srq_init_attr attr;
5347	INIT_Q_COUNTER(out_of_buffer),	3039	struct ib_device *ibdev;
5348	};	3040	struct ib_cq_init_attr cq_attr = {.cqe = 1};
		3041	int port;
		3042	int ret = 0;
5349		3043
5350	static const struct mlx5_ib_counter out_of_seq_q_cnts[] = {	3044	ibdev = &dev->ib_dev;
5351	INIT_Q_COUNTER(out_of_sequence),
5352	};
5353		3045
5354	static const struct mlx5_ib_counter retrans_q_cnts[] = {	3046	if (!MLX5_CAP_GEN(dev->mdev, xrc))
5355	INIT_Q_COUNTER(duplicate_request),	3047	return -EOPNOTSUPP;
5356	INIT_Q_COUNTER(rnr_nak_retry_err),
5357	INIT_Q_COUNTER(packet_seq_err),
5358	INIT_Q_COUNTER(implied_nak_seq_err),
5359	INIT_Q_COUNTER(local_ack_timeout_err),
5360	};
5361		3048
5362	#define INIT_CONG_COUNTER(_name) \	3049	mutex_init(&devr->mutex);
5363	{ .name = #_name, .offset = \
5364	MLX5_BYTE_OFF(query_cong_statistics_out, _name ## _high)}
5365		3050
5366	static const struct mlx5_ib_counter cong_cnts[] = {	3051	devr->p0 = rdma_zalloc_drv_obj(ibdev, ib_pd);
5367	INIT_CONG_COUNTER(rp_cnp_ignored),	3052	if (!devr->p0)
5368	INIT_CONG_COUNTER(rp_cnp_handled),	3053	return -ENOMEM;
5369	INIT_CONG_COUNTER(np_ecn_marked_roce_packets),
5370	INIT_CONG_COUNTER(np_cnp_sent),
5371	};
5372		3054
5373	static const struct mlx5_ib_counter extended_err_cnts[] = {	3055	devr->p0->device = ibdev;
5374	INIT_Q_COUNTER(resp_local_length_error),	3056	devr->p0->uobject = NULL;
5375	INIT_Q_COUNTER(resp_cqe_error),	3057	atomic_set(&devr->p0->usecnt, 0);
5376	INIT_Q_COUNTER(req_cqe_error),
5377	INIT_Q_COUNTER(req_remote_invalid_request),
5378	INIT_Q_COUNTER(req_remote_access_errors),
5379	INIT_Q_COUNTER(resp_remote_access_errors),
5380	INIT_Q_COUNTER(resp_cqe_flush_error),
5381	INIT_Q_COUNTER(req_cqe_flush_error),
5382	};
5383		3058
5384	static const struct mlx5_ib_counter roce_accl_cnts[] = {	3059	ret = mlx5_ib_alloc_pd(devr->p0, NULL);
5385	INIT_Q_COUNTER(roce_adp_retrans),	3060	if (ret)
5386	INIT_Q_COUNTER(roce_adp_retrans_to),	3061	goto error0;
5387	INIT_Q_COUNTER(roce_slow_restart),
5388	INIT_Q_COUNTER(roce_slow_restart_cnps),
5389	INIT_Q_COUNTER(roce_slow_restart_trans),
5390	};
5391		3062
5392	#define INIT_EXT_PPCNT_COUNTER(_name) \	3063	devr->c0 = rdma_zalloc_drv_obj(ibdev, ib_cq);
5393	{ .name = #_name, .offset = \	3064	if (!devr->c0) {
5394	MLX5_BYTE_OFF(ppcnt_reg, \	3065	ret = -ENOMEM;
5395	counter_set.eth_extended_cntrs_grp_data_layout._name##_high)}	3066	goto error1;
		3067	}
5396		3068
5397	static const struct mlx5_ib_counter ext_ppcnt_cnts[] = {	3069	devr->c0->device = &dev->ib_dev;
5398	INIT_EXT_PPCNT_COUNTER(rx_icrc_encapsulated),	3070	atomic_set(&devr->c0->usecnt, 0);
5399	};
5400		3071
5401	static bool is_mdev_switchdev_mode(const struct mlx5_core_dev *mdev)	3072	ret = mlx5_ib_create_cq(devr->c0, &cq_attr, NULL);
5402	{	3073	if (ret)
5403	return MLX5_ESWITCH_MANAGER(mdev) &&	3074	goto err_create_cq;
5404	mlx5_ib_eswitch_mode(mdev->priv.eswitch) ==
5405	MLX5_ESWITCH_OFFLOADS;
5406	}
5407		3075
5408	static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev)	3076	ret = mlx5_cmd_xrcd_alloc(dev->mdev, &devr->xrcdn0, 0);
5409	{	3077	if (ret)
5410	u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};	3078	goto error2;
5411	int num_cnt_ports;
5412	int i;
5413		3079
5414	num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports;	3080	ret = mlx5_cmd_xrcd_alloc(dev->mdev, &devr->xrcdn1, 0);
		3081	if (ret)
		3082	goto error3;
5415		3083
5416	MLX5_SET(dealloc_q_counter_in, in, opcode,	3084	memset(&attr, 0, sizeof(attr));
5417	MLX5_CMD_OP_DEALLOC_Q_COUNTER);	3085	attr.attr.max_sge = 1;
		3086	attr.attr.max_wr = 1;
		3087	attr.srq_type = IB_SRQT_XRC;
		3088	attr.ext.cq = devr->c0;
5418		3089
5419	for (i = 0; i < num_cnt_ports; i++) {	3090	devr->s0 = rdma_zalloc_drv_obj(ibdev, ib_srq);
5420	if (dev->port[i].cnts.set_id) {	3091	if (!devr->s0) {
5421	MLX5_SET(dealloc_q_counter_in, in, counter_set_id,	3092	ret = -ENOMEM;
5422	dev->port[i].cnts.set_id);	3093	goto error4;
5423	mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in);
5424	}
5425	kfree(dev->port[i].cnts.names);
5426	kfree(dev->port[i].cnts.offsets);
5427	}	3094	}
5428	}
5429
5430	static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev,
5431	struct mlx5_ib_counters *cnts)
5432	{
5433	u32 num_counters;
5434		3095
5435	num_counters = ARRAY_SIZE(basic_q_cnts);	3096	devr->s0->device = &dev->ib_dev;
		3097	devr->s0->pd = devr->p0;
		3098	devr->s0->srq_type = IB_SRQT_XRC;
		3099	devr->s0->ext.cq = devr->c0;
		3100	ret = mlx5_ib_create_srq(devr->s0, &attr, NULL);
		3101	if (ret)
		3102	goto err_create;
5436		3103
5437	if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt))	3104	atomic_inc(&devr->s0->ext.cq->usecnt);
5438	num_counters += ARRAY_SIZE(out_of_seq_q_cnts);	3105	atomic_inc(&devr->p0->usecnt);
		3106	atomic_set(&devr->s0->usecnt, 0);
5439		3107
5440	if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters))	3108	memset(&attr, 0, sizeof(attr));
5441	num_counters += ARRAY_SIZE(retrans_q_cnts);	3109	attr.attr.max_sge = 1;
		3110	attr.attr.max_wr = 1;
		3111	attr.srq_type = IB_SRQT_BASIC;
		3112	devr->s1 = rdma_zalloc_drv_obj(ibdev, ib_srq);
		3113	if (!devr->s1) {
		3114	ret = -ENOMEM;
		3115	goto error5;
		3116	}
5442		3117
5443	if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters))	3118	devr->s1->device = &dev->ib_dev;
5444	num_counters += ARRAY_SIZE(extended_err_cnts);	3119	devr->s1->pd = devr->p0;
		3120	devr->s1->srq_type = IB_SRQT_BASIC;
		3121	devr->s1->ext.cq = devr->c0;
5445		3122
5446	if (MLX5_CAP_GEN(dev->mdev, roce_accl))	3123	ret = mlx5_ib_create_srq(devr->s1, &attr, NULL);
5447	num_counters += ARRAY_SIZE(roce_accl_cnts);	3124	if (ret)
		3125	goto error6;
5448		3126
5449	cnts->num_q_counters = num_counters;	3127	atomic_inc(&devr->p0->usecnt);
		3128	atomic_set(&devr->s1->usecnt, 0);
5450		3129
5451	if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {	3130	for (port = 0; port < ARRAY_SIZE(devr->ports); ++port) {
5452	cnts->num_cong_counters = ARRAY_SIZE(cong_cnts);	3131	INIT_WORK(&devr->ports[port].pkey_change_work,
5453	num_counters += ARRAY_SIZE(cong_cnts);	3132	pkey_change_handler);
5454	}	3133	devr->ports[port].devr = devr;
5455	if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
5456	cnts->num_ext_ppcnt_counters = ARRAY_SIZE(ext_ppcnt_cnts);
5457	num_counters += ARRAY_SIZE(ext_ppcnt_cnts);
5458	}	3134	}
5459	cnts->names = kcalloc(num_counters, sizeof(cnts->names), GFP_KERNEL);
5460	if (!cnts->names)
5461	return -ENOMEM;
5462
5463	cnts->offsets = kcalloc(num_counters,
5464	sizeof(cnts->offsets), GFP_KERNEL);
5465	if (!cnts->offsets)
5466	goto err_names;
5467		3135
5468	return 0;	3136	return 0;
5469		3137
5470	err_names:	3138	error6:
5471	kfree(cnts->names);	3139	kfree(devr->s1);
5472	cnts->names = NULL;	3140	error5:
5473	return -ENOMEM;	3141	mlx5_ib_destroy_srq(devr->s0, NULL);
		3142	err_create:
		3143	kfree(devr->s0);
		3144	error4:
		3145	mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0);
		3146	error3:
		3147	mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0);
		3148	error2:
		3149	mlx5_ib_destroy_cq(devr->c0, NULL);
		3150	err_create_cq:
		3151	kfree(devr->c0);
		3152	error1:
		3153	mlx5_ib_dealloc_pd(devr->p0, NULL);
		3154	error0:
		3155	kfree(devr->p0);
		3156	return ret;
5474	}	3157	}
5475		3158
5476	static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev,	3159	static void mlx5_ib_dev_res_cleanup(struct mlx5_ib_dev *dev)
5477	const char **names,
5478	size_t *offsets)
5479	{	3160	{
5480	int i;	3161	struct mlx5_ib_resources *devr = &dev->devr;
5481	int j = 0;	3162	int port;
5482
5483	for (i = 0; i < ARRAY_SIZE(basic_q_cnts); i++, j++) {
5484	names[j] = basic_q_cnts[i].name;
5485	offsets[j] = basic_q_cnts[i].offset;
5486	}
5487
5488	if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) {
5489	for (i = 0; i < ARRAY_SIZE(out_of_seq_q_cnts); i++, j++) {
5490	names[j] = out_of_seq_q_cnts[i].name;
5491	offsets[j] = out_of_seq_q_cnts[i].offset;
5492	}
5493	}
5494
5495	if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) {
5496	for (i = 0; i < ARRAY_SIZE(retrans_q_cnts); i++, j++) {
5497	names[j] = retrans_q_cnts[i].name;
5498	offsets[j] = retrans_q_cnts[i].offset;
5499	}
5500	}
5501
5502	if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) {
5503	for (i = 0; i < ARRAY_SIZE(extended_err_cnts); i++, j++) {
5504	names[j] = extended_err_cnts[i].name;
5505	offsets[j] = extended_err_cnts[i].offset;
5506	}
5507	}
5508
5509	if (MLX5_CAP_GEN(dev->mdev, roce_accl)) {
5510	for (i = 0; i < ARRAY_SIZE(roce_accl_cnts); i++, j++) {
5511	names[j] = roce_accl_cnts[i].name;
5512	offsets[j] = roce_accl_cnts[i].offset;
5513	}
5514	}
5515		3163
5516	if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {	3164	mlx5_ib_destroy_srq(devr->s1, NULL);
5517	for (i = 0; i < ARRAY_SIZE(cong_cnts); i++, j++) {	3165	kfree(devr->s1);
5518	names[j] = cong_cnts[i].name;	3166	mlx5_ib_destroy_srq(devr->s0, NULL);
5519	offsets[j] = cong_cnts[i].offset;	3167	kfree(devr->s0);
5520	}	3168	mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0);
5521	}	3169	mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0);
		3170	mlx5_ib_destroy_cq(devr->c0, NULL);
		3171	kfree(devr->c0);
		3172	mlx5_ib_dealloc_pd(devr->p0, NULL);
		3173	kfree(devr->p0);
5522		3174
5523	if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {	3175	/* Make sure no change P_Key work items are still executing */
5524	for (i = 0; i < ARRAY_SIZE(ext_ppcnt_cnts); i++, j++) {	3176	for (port = 0; port < ARRAY_SIZE(devr->ports); ++port)
5525	names[j] = ext_ppcnt_cnts[i].name;	3177	cancel_work_sync(&devr->ports[port].pkey_change_work);
5526	offsets[j] = ext_ppcnt_cnts[i].offset;
5527	}
5528	}
5529	}	3178	}
5530		3179
5531	static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev)	3180	static u32 get_core_cap_flags(struct ib_device *ibdev,
		3181	struct mlx5_hca_vport_context *rep)
5532	{	3182	{
5533	u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};	3183	struct mlx5_ib_dev *dev = to_mdev(ibdev);
5534	u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};	3184	enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, 1);
5535	int num_cnt_ports;	3185	u8 l3_type_cap = MLX5_CAP_ROCE(dev->mdev, l3_type);
5536	int err = 0;	3186	u8 roce_version_cap = MLX5_CAP_ROCE(dev->mdev, roce_version);
5537	int i;	3187	bool raw_support = !mlx5_core_mp_enabled(dev->mdev);
5538	bool is_shared;	3188	u32 ret = 0;
5539
5540	MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER);
5541	is_shared = MLX5_CAP_GEN(dev->mdev, log_max_uctx) != 0;
5542	num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports;
5543
5544	for (i = 0; i < num_cnt_ports; i++) {
5545	err = __mlx5_ib_alloc_counters(dev, &dev->port[i].cnts);
5546	if (err)
5547	goto err_alloc;
5548		3189
5549	mlx5_ib_fill_counters(dev, dev->port[i].cnts.names,	3190	if (rep->grh_required)
5550	dev->port[i].cnts.offsets);	3191	ret \|= RDMA_CORE_CAP_IB_GRH_REQUIRED;
5551		3192
5552	MLX5_SET(alloc_q_counter_in, in, uid,	3193	if (ll == IB_LINK_LAYER_INFINIBAND)
5553	is_shared ? MLX5_SHARED_RESOURCE_UID : 0);	3194	return ret \| RDMA_CORE_PORT_IBA_IB;
5554		3195
5555	err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out);	3196	if (raw_support)
5556	if (err) {	3197	ret \|= RDMA_CORE_PORT_RAW_PACKET;
5557	mlx5_ib_warn(dev,
5558	"couldn't allocate queue counter for port %d, err %d\n",
5559	i + 1, err);
5560	goto err_alloc;
5561	}
5562		3198
5563	dev->port[i].cnts.set_id =	3199	if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV4_CAP))
5564	MLX5_GET(alloc_q_counter_out, out, counter_set_id);	3200	return ret;
5565	}
5566	return 0;
5567		3201
5568	err_alloc:	3202	if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV6_CAP))
5569	mlx5_ib_dealloc_counters(dev);	3203	return ret;
5570	return err;
5571	}
5572		3204
5573	static const struct mlx5_ib_counters get_counters(struct mlx5_ib_dev dev,	3205	if (roce_version_cap & MLX5_ROCE_VERSION_1_CAP)
5574	u8 port_num)	3206	ret \|= RDMA_CORE_PORT_IBA_ROCE;
5575	{
5576	return is_mdev_switchdev_mode(dev->mdev) ? &dev->port[0].cnts :
5577	&dev->port[port_num].cnts;
5578	}
5579		3207
5580	/**	3208	if (roce_version_cap & MLX5_ROCE_VERSION_2_CAP)
5581	* mlx5_ib_get_counters_id - Returns counters id to use for device+port	3209	ret \|= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
5582	* @dev: Pointer to mlx5 IB device
5583	* @port_num: Zero based port number
5584	*
5585	* mlx5_ib_get_counters_id() Returns counters set id to use for given
5586	* device port combination in switchdev and non switchdev mode of the
5587	* parent device.
5588	*/
5589	u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u8 port_num)
5590	{
5591	const struct mlx5_ib_counters *cnts = get_counters(dev, port_num);
5592		3210
5593	return cnts->set_id;	3211	return ret;
5594	}	3212	}
5595		3213
5596	static struct rdma_hw_stats mlx5_ib_alloc_hw_stats(struct ib_device ibdev,	3214	static int mlx5_port_immutable(struct ib_device *ibdev, u8 port_num,
5597	u8 port_num)	3215	struct ib_port_immutable *immutable)
5598	{	3216	{
		3217	struct ib_port_attr attr;
5599	struct mlx5_ib_dev *dev = to_mdev(ibdev);	3218	struct mlx5_ib_dev *dev = to_mdev(ibdev);
5600	const struct mlx5_ib_counters *cnts;	3219	enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, port_num);
5601	bool is_switchdev = is_mdev_switchdev_mode(dev->mdev);	3220	struct mlx5_hca_vport_context rep = {0};
		3221	int err;
5602		3222
5603	if ((is_switchdev && port_num) \|\| (!is_switchdev && !port_num))	3223	err = ib_query_port(ibdev, port_num, &attr);
5604	return NULL;	3224	if (err)
		3225	return err;
		3226
		3227	if (ll == IB_LINK_LAYER_INFINIBAND) {
		3228	err = mlx5_query_hca_vport_context(dev->mdev, 0, port_num, 0,
		3229	&rep);
		3230	if (err)
		3231	return err;
		3232	}
5605		3233
5606	cnts = get_counters(dev, port_num - 1);	3234	immutable->pkey_tbl_len = attr.pkey_tbl_len;
		3235	immutable->gid_tbl_len = attr.gid_tbl_len;
		3236	immutable->core_cap_flags = get_core_cap_flags(ibdev, &rep);
		3237	immutable->max_mad_size = IB_MGMT_MAD_SIZE;
5607		3238
5608	return rdma_alloc_hw_stats_struct(cnts->names,	3239	return 0;
5609	cnts->num_q_counters +
5610	cnts->num_cong_counters +
5611	cnts->num_ext_ppcnt_counters,
5612	RDMA_HW_STATS_DEFAULT_LIFESPAN);
5613	}	3240	}
5614		3241
5615	static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev,	3242	static int mlx5_port_rep_immutable(struct ib_device *ibdev, u8 port_num,
5616	const struct mlx5_ib_counters *cnts,	3243	struct ib_port_immutable *immutable)
5617	struct rdma_hw_stats *stats,
5618	u16 set_id)
5619	{	3244	{
5620	u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {};	3245	struct ib_port_attr attr;
5621	u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {};	3246	int err;
5622	__be32 val;
5623	int ret, i;
5624		3247
5625	MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER);	3248	immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
5626	MLX5_SET(query_q_counter_in, in, counter_set_id, set_id);
5627	ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out);
5628	if (ret)
5629	return ret;
5630		3249
5631	for (i = 0; i < cnts->num_q_counters; i++) {	3250	err = ib_query_port(ibdev, port_num, &attr);
5632	val = (__be32 )((void *)out + cnts->offsets[i]);	3251	if (err)
5633	stats->value[i] = (u64)be32_to_cpu(val);	3252	return err;
5634	}	3253
		3254	immutable->pkey_tbl_len = attr.pkey_tbl_len;
		3255	immutable->gid_tbl_len = attr.gid_tbl_len;
		3256	immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
5635		3257
5636	return 0;	3258	return 0;
5637	}	3259	}
5638		3260
5639	static int mlx5_ib_query_ext_ppcnt_counters(struct mlx5_ib_dev *dev,	3261	static void get_dev_fw_str(struct ib_device ibdev, char str)
5640	const struct mlx5_ib_counters *cnts,
5641	struct rdma_hw_stats *stats)
5642	{	3262	{
5643	int offset = cnts->num_q_counters + cnts->num_cong_counters;	3263	struct mlx5_ib_dev *dev =
5644	int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);	3264	container_of(ibdev, struct mlx5_ib_dev, ib_dev);
5645	int ret, i;	3265	snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%04d",
5646	void *out;	3266	fw_rev_maj(dev->mdev), fw_rev_min(dev->mdev),
5647		3267	fw_rev_sub(dev->mdev));
5648	out = kvzalloc(sz, GFP_KERNEL);
5649	if (!out)
5650	return -ENOMEM;
5651
5652	ret = mlx5_cmd_query_ext_ppcnt_counters(dev->mdev, out);
5653	if (ret)
5654	goto free;
5655
5656	for (i = 0; i < cnts->num_ext_ppcnt_counters; i++)
5657	stats->value[i + offset] =
5658	be64_to_cpup((__be64 *)(out +
5659	cnts->offsets[i + offset]));
5660	free:
5661	kvfree(out);
5662	return ret;
5663	}	3268	}
5664		3269
5665	static int mlx5_ib_get_hw_stats(struct ib_device *ibdev,	3270	static int mlx5_eth_lag_init(struct mlx5_ib_dev *dev)
5666	struct rdma_hw_stats *stats,
5667	u8 port_num, int index)
5668	{	3271	{
5669	struct mlx5_ib_dev *dev = to_mdev(ibdev);	3272	struct mlx5_core_dev *mdev = dev->mdev;
5670	const struct mlx5_ib_counters *cnts = get_counters(dev, port_num - 1);	3273	struct mlx5_flow_namespace *ns = mlx5_get_flow_namespace(mdev,
5671	struct mlx5_core_dev *mdev;	3274	MLX5_FLOW_NAMESPACE_LAG);
5672	int ret, num_counters;	3275	struct mlx5_flow_table *ft;
5673	u8 mdev_port_num;	3276	int err;
5674
5675	if (!stats)
5676	return -EINVAL;
5677		3277
5678	num_counters = cnts->num_q_counters +	3278	if (!ns \|\| !mlx5_lag_is_roce(mdev))
5679	cnts->num_cong_counters +	3279	return 0;
5680	cnts->num_ext_ppcnt_counters;
5681		3280
5682	/* q_counters are per IB device, query the master mdev */	3281	err = mlx5_cmd_create_vport_lag(mdev);
5683	ret = mlx5_ib_query_q_counters(dev->mdev, cnts, stats, cnts->set_id);	3282	if (err)
5684	if (ret)	3283	return err;
5685	return ret;
5686		3284
5687	if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {	3285	ft = mlx5_create_lag_demux_flow_table(ns, 0, 0);
5688	ret = mlx5_ib_query_ext_ppcnt_counters(dev, cnts, stats);	3286	if (IS_ERR(ft)) {
5689	if (ret)	3287	err = PTR_ERR(ft);
5690	return ret;	3288	goto err_destroy_vport_lag;
5691	}	3289	}
5692		3290
5693	if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {	3291	dev->flow_db->lag_demux_ft = ft;
5694	mdev = mlx5_ib_get_native_port_mdev(dev, port_num,	3292	dev->lag_active = true;
5695	&mdev_port_num);	3293	return 0;
5696	if (!mdev) {
5697	/* If port is not affiliated yet, its in down state
5698	* which doesn't have any counters yet, so it would be
5699	* zero. So no need to read from the HCA.
5700	*/
5701	goto done;
5702	}
5703	ret = mlx5_lag_query_cong_counters(dev->mdev,
5704	stats->value +
5705	cnts->num_q_counters,
5706	cnts->num_cong_counters,
5707	cnts->offsets +
5708	cnts->num_q_counters);
5709
5710	mlx5_ib_put_native_port_mdev(dev, port_num);
5711	if (ret)
5712	return ret;
5713	}
5714		3294
5715	done:	3295	err_destroy_vport_lag:
5716	return num_counters;	3296	mlx5_cmd_destroy_vport_lag(mdev);
		3297	return err;
5717	}	3298	}
5718		3299
5719	static struct rdma_hw_stats *	3300	static void mlx5_eth_lag_cleanup(struct mlx5_ib_dev *dev)
5720	mlx5_ib_counter_alloc_stats(struct rdma_counter *counter)
5721	{	3301	{
5722	struct mlx5_ib_dev *dev = to_mdev(counter->device);	3302	struct mlx5_core_dev *mdev = dev->mdev;
5723	const struct mlx5_ib_counters *cnts =
5724	get_counters(dev, counter->port - 1);
5725		3303
5726	return rdma_alloc_hw_stats_struct(cnts->names,	3304	if (dev->lag_active) {
5727	cnts->num_q_counters +	3305	dev->lag_active = false;
5728	cnts->num_cong_counters +
5729	cnts->num_ext_ppcnt_counters,
5730	RDMA_HW_STATS_DEFAULT_LIFESPAN);
5731	}
5732		3306
5733	static int mlx5_ib_counter_update_stats(struct rdma_counter *counter)	3307	mlx5_destroy_flow_table(dev->flow_db->lag_demux_ft);
5734	{	3308	dev->flow_db->lag_demux_ft = NULL;
5735	struct mlx5_ib_dev *dev = to_mdev(counter->device);
5736	const struct mlx5_ib_counters *cnts =
5737	get_counters(dev, counter->port - 1);
5738		3309
5739	return mlx5_ib_query_q_counters(dev->mdev, cnts,	3310	mlx5_cmd_destroy_vport_lag(mdev);
5740	counter->stats, counter->id);	3311	}
5741	}	3312	}
5742		3313
5743	static int mlx5_ib_counter_dealloc(struct rdma_counter *counter)	3314	static int mlx5_add_netdev_notifier(struct mlx5_ib_dev *dev, u8 port_num)
5744	{	3315	{
5745	struct mlx5_ib_dev *dev = to_mdev(counter->device);	3316	int err;
5746	u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
5747		3317
5748	if (!counter->id)	3318	dev->port[port_num].roce.nb.notifier_call = mlx5_netdev_event;
5749	return 0;	3319	err = register_netdevice_notifier(&dev->port[port_num].roce.nb);
		3320	if (err) {
		3321	dev->port[port_num].roce.nb.notifier_call = NULL;
		3322	return err;
		3323	}
5750		3324
5751	MLX5_SET(dealloc_q_counter_in, in, opcode,	3325	return 0;
5752	MLX5_CMD_OP_DEALLOC_Q_COUNTER);
5753	MLX5_SET(dealloc_q_counter_in, in, counter_set_id, counter->id);
5754	return mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in);
5755	}	3326	}
5756		3327
5757	static int mlx5_ib_counter_bind_qp(struct rdma_counter *counter,	3328	static void mlx5_remove_netdev_notifier(struct mlx5_ib_dev *dev, u8 port_num)
5758	struct ib_qp *qp)
5759	{	3329	{
5760	struct mlx5_ib_dev *dev = to_mdev(qp->device);	3330	if (dev->port[port_num].roce.nb.notifier_call) {
5761	int err;	3331	unregister_netdevice_notifier(&dev->port[port_num].roce.nb);
		3332	dev->port[port_num].roce.nb.notifier_call = NULL;
		3333	}
		3334	}
5762		3335
5763	if (!counter->id) {	3336	static int mlx5_enable_eth(struct mlx5_ib_dev *dev)
5764	u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};	3337	{
5765	u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};	3338	int err;
5766		3339
5767	MLX5_SET(alloc_q_counter_in, in, opcode,	3340	err = mlx5_nic_vport_enable_roce(dev->mdev);
5768	MLX5_CMD_OP_ALLOC_Q_COUNTER);	3341	if (err)
5769	MLX5_SET(alloc_q_counter_in, in, uid, MLX5_SHARED_RESOURCE_UID);	3342	return err;
5770	err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out);
5771	if (err)
5772	return err;
5773	counter->id =
5774	MLX5_GET(alloc_q_counter_out, out, counter_set_id);
5775	}
5776		3343
5777	err = mlx5_ib_qp_set_counter(qp, counter);	3344	err = mlx5_eth_lag_init(dev);
5778	if (err)	3345	if (err)
5779	goto fail_set_counter;	3346	goto err_disable_roce;
5780		3347
5781	return 0;	3348	return 0;
5782		3349
5783	fail_set_counter:	3350	err_disable_roce:
5784	mlx5_ib_counter_dealloc(counter);	3351	mlx5_nic_vport_disable_roce(dev->mdev);
5785	counter->id = 0;
5786		3352
5787	return err;	3353	return err;
5788	}	3354	}
5789		3355
5790	static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp)	3356	static void mlx5_disable_eth(struct mlx5_ib_dev *dev)
5791	{	3357	{
5792	return mlx5_ib_qp_set_counter(qp, NULL);	3358	mlx5_eth_lag_cleanup(dev);
		3359	mlx5_nic_vport_disable_roce(dev->mdev);
5793	}	3360	}
5794		3361
5795	static int mlx5_ib_rn_get_params(struct ib_device *device, u8 port_num,	3362	static int mlx5_ib_rn_get_params(struct ib_device *device, u8 port_num,
@@ -5802,23 +3369,6 @@ static int mlx5_ib_rn_get_params(struct ib_device *device, u8 port_num, · 6 unmodified lines
5802	return mlx5_rdma_rn_get_params(to_mdev(device)->mdev, device, params);	3369	return mlx5_rdma_rn_get_params(to_mdev(device)->mdev, device, params);
5803	}	3370	}
5804		3371
5805	static void delay_drop_debugfs_cleanup(struct mlx5_ib_dev *dev)
5806	{
5807	if (!dev->delay_drop.dir_debugfs)
5808	return;
5809	debugfs_remove_recursive(dev->delay_drop.dir_debugfs);
5810	dev->delay_drop.dir_debugfs = NULL;
5811	}
5812
5813	static void cancel_delay_drop(struct mlx5_ib_dev *dev)
5814	{
5815	if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
5816	return;
5817
5818	cancel_work_sync(&dev->delay_drop.delay_drop_work);
5819	delay_drop_debugfs_cleanup(dev);
5820	}
5821
5822	static ssize_t delay_drop_timeout_read(struct file filp, char __user buf,	3372	static ssize_t delay_drop_timeout_read(struct file filp, char __user buf,
5823	size_t count, loff_t *pos)	3373	size_t count, loff_t *pos)
5824	{	3374	{
@@ -5858,40 +3408,6 @@ static const struct file_operations fops_delay_drop_timeout = { · 33 unmodified lines
5858	.read = delay_drop_timeout_read,	3408	.read = delay_drop_timeout_read,
5859	};	3409	};
5860		3410
5861	static void delay_drop_debugfs_init(struct mlx5_ib_dev *dev)
5862	{
5863	struct dentry *root;
5864
5865	if (!mlx5_debugfs_root)
5866	return;
5867
5868	root = debugfs_create_dir("delay_drop", dev->mdev->priv.dbg_root);
5869	dev->delay_drop.dir_debugfs = root;
5870
5871	debugfs_create_atomic_t("num_timeout_events", 0400, root,
5872	&dev->delay_drop.events_cnt);
5873	debugfs_create_atomic_t("num_rqs", 0400, root,
5874	&dev->delay_drop.rqs_cnt);
5875	debugfs_create_file("timeout", 0600, root, &dev->delay_drop,
5876	&fops_delay_drop_timeout);
5877	}
5878
5879	static void init_delay_drop(struct mlx5_ib_dev *dev)
5880	{
5881	if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
5882	return;
5883
5884	mutex_init(&dev->delay_drop.lock);
5885	dev->delay_drop.dev = dev;
5886	dev->delay_drop.activate = false;
5887	dev->delay_drop.timeout = MLX5_MAX_DELAY_DROP_TIMEOUT_MS * 1000;
5888	INIT_WORK(&dev->delay_drop.delay_drop_work, delay_drop_handler);
5889	atomic_set(&dev->delay_drop.rqs_cnt, 0);
5890	atomic_set(&dev->delay_drop.events_cnt, 0);
5891
5892	delay_drop_debugfs_init(dev);
5893	}
5894
5895	static void mlx5_ib_unbind_slave_port(struct mlx5_ib_dev *ibdev,	3411	static void mlx5_ib_unbind_slave_port(struct mlx5_ib_dev *ibdev,
5896	struct mlx5_ib_multiport_info *mpi)	3412	struct mlx5_ib_multiport_info *mpi)
5897	{	3413	{
@@ -6385,90 +3901,32 @@ ADD_UVERBS_ATTRIBUTES_SIMPLE( · 487 unmodified lines
6385	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_ACTION_FLAGS,	3901	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_ACTION_FLAGS,
6386	enum mlx5_ib_uapi_flow_action_flags));	3902	enum mlx5_ib_uapi_flow_action_flags));
6387		3903
		3904	ADD_UVERBS_ATTRIBUTES_SIMPLE(
		3905	mlx5_ib_query_context,
		3906	UVERBS_OBJECT_DEVICE,
		3907	UVERBS_METHOD_QUERY_CONTEXT,
		3908	UVERBS_ATTR_PTR_OUT(
		3909	MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX,
		3910	UVERBS_ATTR_STRUCT(struct mlx5_ib_alloc_ucontext_resp,
		3911	dump_fill_mkey),
		3912	UA_MANDATORY));
		3913
6388	static const struct uapi_definition mlx5_ib_defs[] = {	3914	static const struct uapi_definition mlx5_ib_defs[] = {
6389	UAPI_DEF_CHAIN(mlx5_ib_devx_defs),	3915	UAPI_DEF_CHAIN(mlx5_ib_devx_defs),
6390	UAPI_DEF_CHAIN(mlx5_ib_flow_defs),	3916	UAPI_DEF_CHAIN(mlx5_ib_flow_defs),
6391	UAPI_DEF_CHAIN(mlx5_ib_qos_defs),	3917	UAPI_DEF_CHAIN(mlx5_ib_qos_defs),
		3918	UAPI_DEF_CHAIN(mlx5_ib_std_types_defs),
6392		3919
6393	UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION,	3920	UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION,
6394	&mlx5_ib_flow_action),	3921	&mlx5_ib_flow_action),
6395	UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DM, &mlx5_ib_dm),	3922	UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DM, &mlx5_ib_dm),
		3923	UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DEVICE, &mlx5_ib_query_context),
6396	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_VAR,	3924	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_VAR,
6397	UAPI_DEF_IS_OBJ_SUPPORTED(var_is_supported)),	3925	UAPI_DEF_IS_OBJ_SUPPORTED(var_is_supported)),
6398	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_UAR),	3926	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_UAR),
6399	{}	3927	{}
6400	};	3928	};
6401		3929
6402	static int mlx5_ib_read_counters(struct ib_counters *counters,
6403	struct ib_counters_read_attr *read_attr,
6404	struct uverbs_attr_bundle *attrs)
6405	{
6406	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
6407	struct mlx5_read_counters_attr mread_attr = {};
6408	struct mlx5_ib_flow_counters_desc *desc;
6409	int ret, i;
6410
6411	mutex_lock(&mcounters->mcntrs_mutex);
6412	if (mcounters->cntrs_max_index > read_attr->ncounters) {
6413	ret = -EINVAL;
6414	goto err_bound;
6415	}
6416
6417	mread_attr.out = kcalloc(mcounters->counters_num, sizeof(u64),
6418	GFP_KERNEL);
6419	if (!mread_attr.out) {
6420	ret = -ENOMEM;
6421	goto err_bound;
6422	}
6423
6424	mread_attr.hw_cntrs_hndl = mcounters->hw_cntrs_hndl;
6425	mread_attr.flags = read_attr->flags;
6426	ret = mcounters->read_counters(counters->device, &mread_attr);
6427	if (ret)
6428	goto err_read;
6429
6430	/* do the pass over the counters data array to assign according to the
6431	* descriptions and indexing pairs
6432	*/
6433	desc = mcounters->counters_data;
6434	for (i = 0; i < mcounters->ncounters; i++)
6435	read_attr->counters_buff[desc[i].index] += mread_attr.out[desc[i].description];
6436
6437	err_read:
6438	kfree(mread_attr.out);
6439	err_bound:
6440	mutex_unlock(&mcounters->mcntrs_mutex);
6441	return ret;
6442	}
6443
6444	static int mlx5_ib_destroy_counters(struct ib_counters *counters)
6445	{
6446	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
6447
6448	counters_clear_description(counters);
6449	if (mcounters->hw_cntrs_hndl)
6450	mlx5_fc_destroy(to_mdev(counters->device)->mdev,
6451	mcounters->hw_cntrs_hndl);
6452
6453	kfree(mcounters);
6454
6455	return 0;
6456	}
6457
6458	static struct ib_counters mlx5_ib_create_counters(struct ib_device device,
6459	struct uverbs_attr_bundle *attrs)
6460	{
6461	struct mlx5_ib_mcounters *mcounters;
6462
6463	mcounters = kzalloc(sizeof(*mcounters), GFP_KERNEL);
6464	if (!mcounters)
6465	return ERR_PTR(-ENOMEM);
6466
6467	mutex_init(&mcounters->mcntrs_mutex);
6468
6469	return &mcounters->ibcntrs;
6470	}
6471
6472	static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev)	3930	static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev)
6473	{	3931	{
6474	mlx5_ib_cleanup_multiport_master(dev);	3932	mlx5_ib_cleanup_multiport_master(dev);
@@ -6547,21 +4005,16 @@ err_mp: · 72 unmodified lines
6547	return -ENOMEM;	4005	return -ENOMEM;
6548	}	4006	}
6549		4007
6550	static int mlx5_ib_stage_flow_db_init(struct mlx5_ib_dev *dev)	4008	static int mlx5_ib_enable_driver(struct ib_device *dev)
6551	{	4009	{
6552	dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL);	4010	struct mlx5_ib_dev *mdev = to_mdev(dev);
6553		4011	int ret;
6554	if (!dev->flow_db)
6555	return -ENOMEM;
6556
6557	mutex_init(&dev->flow_db->lock);
6558		4012
6559	return 0;	4013	ret = mlx5_ib_test_wc(mdev);
6560	}	4014	mlx5_ib_dbg(mdev, "Write-Combining %s",
		4015	mdev->wc_support ? "supported" : "not supported");
6561		4016
6562	static void mlx5_ib_stage_flow_db_cleanup(struct mlx5_ib_dev *dev)	4017	return ret;
6563	{
6564	kfree(dev->flow_db);
6565	}	4018	}
6566		4019
6567	static const struct ib_device_ops mlx5_ib_dev_ops = {	4020	static const struct ib_device_ops mlx5_ib_dev_ops = {
@@ -6577,9 +4030,7 @@ static const struct ib_device_ops mlx5_ib_dev_ops = { · 9 unmodified lines
6577	.attach_mcast = mlx5_ib_mcg_attach,	4030	.attach_mcast = mlx5_ib_mcg_attach,
6578	.check_mr_status = mlx5_ib_check_mr_status,	4031	.check_mr_status = mlx5_ib_check_mr_status,
6579	.create_ah = mlx5_ib_create_ah,	4032	.create_ah = mlx5_ib_create_ah,
6580	.create_counters = mlx5_ib_create_counters,
6581	.create_cq = mlx5_ib_create_cq,	4033	.create_cq = mlx5_ib_create_cq,
6582	.create_flow = mlx5_ib_create_flow,
6583	.create_qp = mlx5_ib_create_qp,	4034	.create_qp = mlx5_ib_create_qp,
6584	.create_srq = mlx5_ib_create_srq,	4035	.create_srq = mlx5_ib_create_srq,
6585	.dealloc_pd = mlx5_ib_dealloc_pd,	4036	.dealloc_pd = mlx5_ib_dealloc_pd,
@@ -6587,10 +4038,7 @@ static const struct ib_device_ops mlx5_ib_dev_ops = { · 1 unmodified line
6587	.del_gid = mlx5_ib_del_gid,	4038	.del_gid = mlx5_ib_del_gid,
6588	.dereg_mr = mlx5_ib_dereg_mr,	4039	.dereg_mr = mlx5_ib_dereg_mr,
6589	.destroy_ah = mlx5_ib_destroy_ah,	4040	.destroy_ah = mlx5_ib_destroy_ah,
6590	.destroy_counters = mlx5_ib_destroy_counters,
6591	.destroy_cq = mlx5_ib_destroy_cq,	4041	.destroy_cq = mlx5_ib_destroy_cq,
6592	.destroy_flow = mlx5_ib_destroy_flow,
6593	.destroy_flow_action = mlx5_ib_destroy_flow_action,
6594	.destroy_qp = mlx5_ib_destroy_qp,	4042	.destroy_qp = mlx5_ib_destroy_qp,
6595	.destroy_srq = mlx5_ib_destroy_srq,	4043	.destroy_srq = mlx5_ib_destroy_srq,
6596	.detach_mcast = mlx5_ib_mcg_detach,	4044	.detach_mcast = mlx5_ib_mcg_detach,
@@ -6598,8 +4046,6 @@ static const struct ib_device_ops mlx5_ib_dev_ops = { · 1 unmodified line
6598	.drain_rq = mlx5_ib_drain_rq,	4046	.drain_rq = mlx5_ib_drain_rq,
6599	.drain_sq = mlx5_ib_drain_sq,	4047	.drain_sq = mlx5_ib_drain_sq,
6600	.enable_driver = mlx5_ib_enable_driver,	4048	.enable_driver = mlx5_ib_enable_driver,
6601	.fill_res_entry = mlx5_ib_fill_res_entry,
6602	.fill_stat_entry = mlx5_ib_fill_stat_entry,
6603	.get_dev_fw_str = get_dev_fw_str,	4049	.get_dev_fw_str = get_dev_fw_str,
6604	.get_dma_mr = mlx5_ib_get_dma_mr,	4050	.get_dma_mr = mlx5_ib_get_dma_mr,
6605	.get_link_layer = mlx5_ib_port_link_layer,	4051	.get_link_layer = mlx5_ib_port_link_layer,
@@ -6623,24 +4069,20 @@ static const struct ib_device_ops mlx5_ib_dev_ops = { · 17 unmodified lines
6623	.query_pkey = mlx5_ib_query_pkey,	4069	.query_pkey = mlx5_ib_query_pkey,
6624	.query_qp = mlx5_ib_query_qp,	4070	.query_qp = mlx5_ib_query_qp,
6625	.query_srq = mlx5_ib_query_srq,	4071	.query_srq = mlx5_ib_query_srq,
6626	.read_counters = mlx5_ib_read_counters,	4072	.query_ucontext = mlx5_ib_query_ucontext,
6627	.reg_user_mr = mlx5_ib_reg_user_mr,	4073	.reg_user_mr = mlx5_ib_reg_user_mr,
6628	.req_notify_cq = mlx5_ib_arm_cq,	4074	.req_notify_cq = mlx5_ib_arm_cq,
6629	.rereg_user_mr = mlx5_ib_rereg_user_mr,	4075	.rereg_user_mr = mlx5_ib_rereg_user_mr,
6630	.resize_cq = mlx5_ib_resize_cq,	4076	.resize_cq = mlx5_ib_resize_cq,
6631		4077
6632	INIT_RDMA_OBJ_SIZE(ib_ah, mlx5_ib_ah, ibah),	4078	INIT_RDMA_OBJ_SIZE(ib_ah, mlx5_ib_ah, ibah),
		4079	INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs),
6633	INIT_RDMA_OBJ_SIZE(ib_cq, mlx5_ib_cq, ibcq),	4080	INIT_RDMA_OBJ_SIZE(ib_cq, mlx5_ib_cq, ibcq),
6634	INIT_RDMA_OBJ_SIZE(ib_pd, mlx5_ib_pd, ibpd),	4081	INIT_RDMA_OBJ_SIZE(ib_pd, mlx5_ib_pd, ibpd),
6635	INIT_RDMA_OBJ_SIZE(ib_srq, mlx5_ib_srq, ibsrq),	4082	INIT_RDMA_OBJ_SIZE(ib_srq, mlx5_ib_srq, ibsrq),
6636	INIT_RDMA_OBJ_SIZE(ib_ucontext, mlx5_ib_ucontext, ibucontext),	4083	INIT_RDMA_OBJ_SIZE(ib_ucontext, mlx5_ib_ucontext, ibucontext),
6637	};	4084	};
6638		4085
6639	static const struct ib_device_ops mlx5_ib_dev_flow_ipsec_ops = {
6640	.create_flow_action_esp = mlx5_ib_create_flow_action_esp,
6641	.modify_flow_action_esp = mlx5_ib_modify_flow_action_esp,
6642	};
6643
6644	static const struct ib_device_ops mlx5_ib_dev_ipoib_enhanced_ops = {	4086	static const struct ib_device_ops mlx5_ib_dev_ipoib_enhanced_ops = {
6645	.rdma_netdev_get_params = mlx5_ib_rn_get_params,	4087	.rdma_netdev_get_params = mlx5_ib_rn_get_params,
6646	};	4088	};
@@ -6661,6 +4103,8 @@ static const struct ib_device_ops mlx5_ib_dev_mw_ops = { · 14 unmodified lines
6661	static const struct ib_device_ops mlx5_ib_dev_xrc_ops = {	4103	static const struct ib_device_ops mlx5_ib_dev_xrc_ops = {
6662	.alloc_xrcd = mlx5_ib_alloc_xrcd,	4104	.alloc_xrcd = mlx5_ib_alloc_xrcd,
6663	.dealloc_xrcd = mlx5_ib_dealloc_xrcd,	4105	.dealloc_xrcd = mlx5_ib_dealloc_xrcd,
		4106
		4107	INIT_RDMA_OBJ_SIZE(ib_xrcd, mlx5_ib_xrcd, ibxrcd),
6664	};	4108	};
6665		4109
6666	static const struct ib_device_ops mlx5_ib_dev_dm_ops = {	4110	static const struct ib_device_ops mlx5_ib_dev_dm_ops = {
@@ -6769,9 +4213,6 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) · 102 unmodified lines
6769	MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM)	4213	MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM)
6770	ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_dm_ops);	4214	ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_dm_ops);
6771		4215
6772	if (mlx5_accel_ipsec_device_caps(dev->mdev) &
6773	MLX5_ACCEL_IPSEC_CAP_DEVICE)
6774	ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_flow_ipsec_ops);
6775	ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_ops);	4216	ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_ops);
6776		4217
6777	if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS))	4218	if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS))
@@ -6829,65 +4270,36 @@ static const struct ib_device_ops mlx5_ib_dev_common_roce_ops = { · 51 unmodified lines
6829	.modify_wq = mlx5_ib_modify_wq,	4270	.modify_wq = mlx5_ib_modify_wq,
6830	};	4271	};
6831		4272
6832	static int mlx5_ib_stage_common_roce_init(struct mlx5_ib_dev *dev)	4273	static int mlx5_ib_roce_init(struct mlx5_ib_dev *dev)
6833	{
6834	u8 port_num;
6835
6836	dev->ib_dev.uverbs_ex_cmd_mask \|=
6837	(1ull << IB_USER_VERBS_EX_CMD_CREATE_WQ) \|
6838	(1ull << IB_USER_VERBS_EX_CMD_MODIFY_WQ) \|
6839	(1ull << IB_USER_VERBS_EX_CMD_DESTROY_WQ) \|
6840	(1ull << IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL) \|
6841	(1ull << IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL);
6842	ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_common_roce_ops);
6843
6844	port_num = mlx5_core_native_port_num(dev->mdev) - 1;
6845
6846	/* Register only for native ports */
6847	return mlx5_add_netdev_notifier(dev, port_num);
6848	}
6849
6850	static void mlx5_ib_stage_common_roce_cleanup(struct mlx5_ib_dev *dev)
6851	{
6852	u8 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
6853
6854	mlx5_remove_netdev_notifier(dev, port_num);
6855	}
6856
6857	static int mlx5_ib_stage_raw_eth_roce_init(struct mlx5_ib_dev *dev)
6858	{
6859	struct mlx5_core_dev *mdev = dev->mdev;
6860	enum rdma_link_layer ll;
6861	int port_type_cap;
6862	int err = 0;
6863
6864	port_type_cap = MLX5_CAP_GEN(mdev, port_type);
6865	ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
6866
6867	if (ll == IB_LINK_LAYER_ETHERNET)
6868	err = mlx5_ib_stage_common_roce_init(dev);
6869
6870	return err;
6871	}
6872
6873	static void mlx5_ib_stage_raw_eth_roce_cleanup(struct mlx5_ib_dev *dev)
6874	{
6875	mlx5_ib_stage_common_roce_cleanup(dev);
6876	}
6877
6878	static int mlx5_ib_stage_roce_init(struct mlx5_ib_dev *dev)
6879	{	4274	{
6880	struct mlx5_core_dev *mdev = dev->mdev;	4275	struct mlx5_core_dev *mdev = dev->mdev;
6881	enum rdma_link_layer ll;	4276	enum rdma_link_layer ll;
6882	int port_type_cap;	4277	int port_type_cap;
		4278	u8 port_num = 0;
6883	int err;	4279	int err;
6884		4280
6885	port_type_cap = MLX5_CAP_GEN(mdev, port_type);	4281	port_type_cap = MLX5_CAP_GEN(mdev, port_type);
6886	ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);	4282	ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
6887		4283
6888	if (ll == IB_LINK_LAYER_ETHERNET) {	4284	if (ll == IB_LINK_LAYER_ETHERNET) {
6889	err = mlx5_ib_stage_common_roce_init(dev);	4285	dev->ib_dev.uverbs_ex_cmd_mask \|=
6890	if (err)	4286	(1ull << IB_USER_VERBS_EX_CMD_CREATE_WQ) \|
		4287	(1ull << IB_USER_VERBS_EX_CMD_MODIFY_WQ) \|
		4288	(1ull << IB_USER_VERBS_EX_CMD_DESTROY_WQ) \|
		4289	(1ull << IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL) \|
		4290	(1ull << IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL);
		4291	ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_common_roce_ops);
		4292
		4293	port_num = mlx5_core_native_port_num(dev->mdev) - 1;
		4294
		4295	/* Register only for native ports */
		4296	err = mlx5_add_netdev_notifier(dev, port_num);
		4297	if (err \|\| dev->is_rep \|\| !mlx5_is_roce_enabled(mdev))
		4298	/*
		4299	* We don't enable ETH interface for
		4300	* 1. IB representors
		4301	* 2. User disabled ROCE through devlink interface
		4302	*/
6891	return err;	4303	return err;
6892		4304
6893	err = mlx5_enable_eth(dev);	4305	err = mlx5_enable_eth(dev);
@@ -6897,71 +4309,27 @@ static int mlx5_ib_stage_roce_init(struct mlx5_ib_dev *dev) · 3 unmodified lines
6897		4309
6898	return 0;	4310	return 0;
6899	cleanup:	4311	cleanup:
6900	mlx5_ib_stage_common_roce_cleanup(dev);	4312	mlx5_remove_netdev_notifier(dev, port_num);
6901
6902	return err;	4313	return err;
6903	}	4314	}
6904		4315
6905	static void mlx5_ib_stage_roce_cleanup(struct mlx5_ib_dev *dev)	4316	static void mlx5_ib_roce_cleanup(struct mlx5_ib_dev *dev)
6906	{	4317	{
6907	struct mlx5_core_dev *mdev = dev->mdev;	4318	struct mlx5_core_dev *mdev = dev->mdev;
6908	enum rdma_link_layer ll;	4319	enum rdma_link_layer ll;
6909	int port_type_cap;	4320	int port_type_cap;
		4321	u8 port_num;
6910		4322
6911	port_type_cap = MLX5_CAP_GEN(mdev, port_type);	4323	port_type_cap = MLX5_CAP_GEN(mdev, port_type);
6912	ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);	4324	ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
6913		4325
6914	if (ll == IB_LINK_LAYER_ETHERNET) {	4326	if (ll == IB_LINK_LAYER_ETHERNET) {
6915	mlx5_disable_eth(dev);	4327	if (!dev->is_rep)
6916	mlx5_ib_stage_common_roce_cleanup(dev);	4328	mlx5_disable_eth(dev);
6917	}
6918	}
6919
6920	static int mlx5_ib_stage_dev_res_init(struct mlx5_ib_dev *dev)
6921	{
6922	return create_dev_resources(&dev->devr);
6923	}
6924
6925	static void mlx5_ib_stage_dev_res_cleanup(struct mlx5_ib_dev *dev)
6926	{
6927	destroy_dev_resources(&dev->devr);
6928	}
6929
6930	static int mlx5_ib_stage_odp_init(struct mlx5_ib_dev *dev)
6931	{
6932	return mlx5_ib_odp_init_one(dev);
6933	}
6934
6935	static void mlx5_ib_stage_odp_cleanup(struct mlx5_ib_dev *dev)
6936	{
6937	mlx5_ib_odp_cleanup_one(dev);
6938	}
6939
6940	static const struct ib_device_ops mlx5_ib_dev_hw_stats_ops = {
6941	.alloc_hw_stats = mlx5_ib_alloc_hw_stats,
6942	.get_hw_stats = mlx5_ib_get_hw_stats,
6943	.counter_bind_qp = mlx5_ib_counter_bind_qp,
6944	.counter_unbind_qp = mlx5_ib_counter_unbind_qp,
6945	.counter_dealloc = mlx5_ib_counter_dealloc,
6946	.counter_alloc_stats = mlx5_ib_counter_alloc_stats,
6947	.counter_update_stats = mlx5_ib_counter_update_stats,
6948	};
6949
6950	static int mlx5_ib_stage_counters_init(struct mlx5_ib_dev *dev)
6951	{
6952	if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) {
6953	ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_hw_stats_ops);
6954		4329
6955	return mlx5_ib_alloc_counters(dev);	4330	port_num = mlx5_core_native_port_num(dev->mdev) - 1;
		4331	mlx5_remove_netdev_notifier(dev, port_num);
6956	}	4332	}
6957
6958	return 0;
6959	}
6960
6961	static void mlx5_ib_stage_counters_cleanup(struct mlx5_ib_dev *dev)
6962	{
6963	if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
6964	mlx5_ib_dealloc_counters(dev);
6965	}	4333	}
6966		4334
6967	static int mlx5_ib_stage_cong_debugfs_init(struct mlx5_ib_dev *dev)	4335	static int mlx5_ib_stage_cong_debugfs_init(struct mlx5_ib_dev *dev)
@@ -7023,7 +4391,18 @@ static int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev) · 55 unmodified lines
7023		4391
7024	static void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev)	4392	static void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev)
7025	{	4393	{
7026	destroy_umrc_res(dev);	4394	int err;
		4395
		4396	err = mlx5_mr_cache_cleanup(dev);
		4397	if (err)
		4398	mlx5_ib_warn(dev, "mr cache cleanup failed\n");
		4399
		4400	if (dev->umrc.qp)
		4401	mlx5_ib_destroy_qp(dev->umrc.qp, NULL);
		4402	if (dev->umrc.cq)
		4403	ib_free_cq(dev->umrc.cq);
		4404	if (dev->umrc.pd)
		4405	ib_dealloc_pd(dev->umrc.pd);
7027	}	4406	}
7028		4407
7029	static void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev)	4408	static void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev)
@@ -7031,21 +4410,162 @@ static void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev) · 1 unmodified line
7031	ib_unregister_device(&dev->ib_dev);	4410	ib_unregister_device(&dev->ib_dev);
7032	}	4411	}
7033		4412
		4413	enum {
		4414	MAX_UMR_WR = 128,
		4415	};
		4416
7034	static int mlx5_ib_stage_post_ib_reg_umr_init(struct mlx5_ib_dev *dev)	4417	static int mlx5_ib_stage_post_ib_reg_umr_init(struct mlx5_ib_dev *dev)
7035	{	4418	{
7036	return create_umr_res(dev);	4419	struct ib_qp_init_attr *init_attr = NULL;
		4420	struct ib_qp_attr *attr = NULL;
		4421	struct ib_pd *pd;
		4422	struct ib_cq *cq;
		4423	struct ib_qp *qp;
		4424	int ret;
		4425
		4426	attr = kzalloc(sizeof(*attr), GFP_KERNEL);
		4427	init_attr = kzalloc(sizeof(*init_attr), GFP_KERNEL);
		4428	if (!attr \|\| !init_attr) {
		4429	ret = -ENOMEM;
		4430	goto error_0;
		4431	}
		4432
		4433	pd = ib_alloc_pd(&dev->ib_dev, 0);
		4434	if (IS_ERR(pd)) {
		4435	mlx5_ib_dbg(dev, "Couldn't create PD for sync UMR QP\n");
		4436	ret = PTR_ERR(pd);
		4437	goto error_0;
		4438	}
		4439
		4440	cq = ib_alloc_cq(&dev->ib_dev, NULL, 128, 0, IB_POLL_SOFTIRQ);
		4441	if (IS_ERR(cq)) {
		4442	mlx5_ib_dbg(dev, "Couldn't create CQ for sync UMR QP\n");
		4443	ret = PTR_ERR(cq);
		4444	goto error_2;
		4445	}
		4446
		4447	init_attr->send_cq = cq;
		4448	init_attr->recv_cq = cq;
		4449	init_attr->sq_sig_type = IB_SIGNAL_ALL_WR;
		4450	init_attr->cap.max_send_wr = MAX_UMR_WR;
		4451	init_attr->cap.max_send_sge = 1;
		4452	init_attr->qp_type = MLX5_IB_QPT_REG_UMR;
		4453	init_attr->port_num = 1;
		4454	qp = mlx5_ib_create_qp(pd, init_attr, NULL);
		4455	if (IS_ERR(qp)) {
		4456	mlx5_ib_dbg(dev, "Couldn't create sync UMR QP\n");
		4457	ret = PTR_ERR(qp);
		4458	goto error_3;
		4459	}
		4460	qp->device = &dev->ib_dev;
		4461	qp->real_qp = qp;
		4462	qp->uobject = NULL;
		4463	qp->qp_type = MLX5_IB_QPT_REG_UMR;
		4464	qp->send_cq = init_attr->send_cq;
		4465	qp->recv_cq = init_attr->recv_cq;
		4466
		4467	attr->qp_state = IB_QPS_INIT;
		4468	attr->port_num = 1;
		4469	ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE \| IB_QP_PKEY_INDEX \|
		4470	IB_QP_PORT, NULL);
		4471	if (ret) {
		4472	mlx5_ib_dbg(dev, "Couldn't modify UMR QP\n");
		4473	goto error_4;
		4474	}
		4475
		4476	memset(attr, 0, sizeof(*attr));
		4477	attr->qp_state = IB_QPS_RTR;
		4478	attr->path_mtu = IB_MTU_256;
		4479
		4480	ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL);
		4481	if (ret) {
		4482	mlx5_ib_dbg(dev, "Couldn't modify umr QP to rtr\n");
		4483	goto error_4;
		4484	}
		4485
		4486	memset(attr, 0, sizeof(*attr));
		4487	attr->qp_state = IB_QPS_RTS;
		4488	ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL);
		4489	if (ret) {
		4490	mlx5_ib_dbg(dev, "Couldn't modify umr QP to rts\n");
		4491	goto error_4;
		4492	}
		4493
		4494	dev->umrc.qp = qp;
		4495	dev->umrc.cq = cq;
		4496	dev->umrc.pd = pd;
		4497
		4498	sema_init(&dev->umrc.sem, MAX_UMR_WR);
		4499	ret = mlx5_mr_cache_init(dev);
		4500	if (ret) {
		4501	mlx5_ib_warn(dev, "mr cache init failed %d\n", ret);
		4502	goto error_4;
		4503	}
		4504
		4505	kfree(attr);
		4506	kfree(init_attr);
		4507
		4508	return 0;
		4509
		4510	error_4:
		4511	mlx5_ib_destroy_qp(qp, NULL);
		4512	dev->umrc.qp = NULL;
		4513
		4514	error_3:
		4515	ib_free_cq(cq);
		4516	dev->umrc.cq = NULL;
		4517
		4518	error_2:
		4519	ib_dealloc_pd(pd);
		4520	dev->umrc.pd = NULL;
		4521
		4522	error_0:
		4523	kfree(attr);
		4524	kfree(init_attr);
		4525	return ret;
7037	}	4526	}
7038		4527
7039	static int mlx5_ib_stage_delay_drop_init(struct mlx5_ib_dev *dev)	4528	static int mlx5_ib_stage_delay_drop_init(struct mlx5_ib_dev *dev)
7040	{	4529	{
7041	init_delay_drop(dev);	4530	struct dentry *root;
		4531
		4532	if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
		4533	return 0;
		4534
		4535	mutex_init(&dev->delay_drop.lock);
		4536	dev->delay_drop.dev = dev;
		4537	dev->delay_drop.activate = false;
		4538	dev->delay_drop.timeout = MLX5_MAX_DELAY_DROP_TIMEOUT_MS * 1000;
		4539	INIT_WORK(&dev->delay_drop.delay_drop_work, delay_drop_handler);
		4540	atomic_set(&dev->delay_drop.rqs_cnt, 0);
		4541	atomic_set(&dev->delay_drop.events_cnt, 0);
		4542
		4543	if (!mlx5_debugfs_root)
		4544	return 0;
		4545
		4546	root = debugfs_create_dir("delay_drop", dev->mdev->priv.dbg_root);
		4547	dev->delay_drop.dir_debugfs = root;
7042		4548
		4549	debugfs_create_atomic_t("num_timeout_events", 0400, root,
		4550	&dev->delay_drop.events_cnt);
		4551	debugfs_create_atomic_t("num_rqs", 0400, root,
		4552	&dev->delay_drop.rqs_cnt);
		4553	debugfs_create_file("timeout", 0600, root, &dev->delay_drop,
		4554	&fops_delay_drop_timeout);
7043	return 0;	4555	return 0;
7044	}	4556	}
7045		4557
7046	static void mlx5_ib_stage_delay_drop_cleanup(struct mlx5_ib_dev *dev)	4558	static void mlx5_ib_stage_delay_drop_cleanup(struct mlx5_ib_dev *dev)
7047	{	4559	{
7048	cancel_delay_drop(dev);	4560	if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
		4561	return;
		4562
		4563	cancel_work_sync(&dev->delay_drop.delay_drop_work);
		4564	if (!dev->delay_drop.dir_debugfs)
		4565	return;
		4566
		4567	debugfs_remove_recursive(dev->delay_drop.dir_debugfs);
		4568	dev->delay_drop.dir_debugfs = NULL;
7049	}	4569	}
7050		4570
7051	static int mlx5_ib_stage_dev_notifier_init(struct mlx5_ib_dev *dev)	4571	static int mlx5_ib_stage_dev_notifier_init(struct mlx5_ib_dev *dev)
@@ -7060,38 +4580,6 @@ static void mlx5_ib_stage_dev_notifier_cleanup(struct mlx5_ib_dev *dev) · 8 unmodified lines
7060	mlx5_notifier_unregister(dev->mdev, &dev->mdev_events);	4580	mlx5_notifier_unregister(dev->mdev, &dev->mdev_events);
7061	}	4581	}
7062		4582
7063	static int mlx5_ib_stage_devx_init(struct mlx5_ib_dev *dev)
7064	{
7065	int uid;
7066
7067	uid = mlx5_ib_devx_create(dev, false);
7068	if (uid > 0) {
7069	dev->devx_whitelist_uid = uid;
7070	mlx5_ib_devx_init_event_table(dev);
7071	}
7072
7073	return 0;
7074	}
7075	static void mlx5_ib_stage_devx_cleanup(struct mlx5_ib_dev *dev)
7076	{
7077	if (dev->devx_whitelist_uid) {
7078	mlx5_ib_devx_cleanup_event_table(dev);
7079	mlx5_ib_devx_destroy(dev, dev->devx_whitelist_uid);
7080	}
7081	}
7082
7083	int mlx5_ib_enable_driver(struct ib_device *dev)
7084	{
7085	struct mlx5_ib_dev *mdev = to_mdev(dev);
7086	int ret;
7087
7088	ret = mlx5_ib_test_wc(mdev);
7089	mlx5_ib_dbg(mdev, "Write-Combining %s",
7090	mdev->wc_support ? "supported" : "not supported");
7091
7092	return ret;
7093	}
7094
7095	void __mlx5_ib_remove(struct mlx5_ib_dev *dev,	4583	void __mlx5_ib_remove(struct mlx5_ib_dev *dev,
7096	const struct mlx5_ib_profile *profile,	4584	const struct mlx5_ib_profile *profile,
7097	int stage)	4585	int stage)
@@ -7139,9 +4627,9 @@ static const struct mlx5_ib_profile pf_profile = { · 41 unmodified lines
7139	STAGE_CREATE(MLX5_IB_STAGE_INIT,	4627	STAGE_CREATE(MLX5_IB_STAGE_INIT,
7140	mlx5_ib_stage_init_init,	4628	mlx5_ib_stage_init_init,
7141	mlx5_ib_stage_init_cleanup),	4629	mlx5_ib_stage_init_cleanup),
7142	STAGE_CREATE(MLX5_IB_STAGE_FLOW_DB,	4630	STAGE_CREATE(MLX5_IB_STAGE_FS,
7143	mlx5_ib_stage_flow_db_init,	4631	mlx5_ib_fs_init,
7144	mlx5_ib_stage_flow_db_cleanup),	4632	mlx5_ib_fs_cleanup),
7145	STAGE_CREATE(MLX5_IB_STAGE_CAPS,	4633	STAGE_CREATE(MLX5_IB_STAGE_CAPS,
7146	mlx5_ib_stage_caps_init,	4634	mlx5_ib_stage_caps_init,
7147	mlx5_ib_stage_caps_cleanup),	4635	mlx5_ib_stage_caps_cleanup),
@@ -7149,8 +4637,8 @@ static const struct mlx5_ib_profile pf_profile = { · 1 unmodified line
7149	mlx5_ib_stage_non_default_cb,	4637	mlx5_ib_stage_non_default_cb,
7150	NULL),	4638	NULL),
7151	STAGE_CREATE(MLX5_IB_STAGE_ROCE,	4639	STAGE_CREATE(MLX5_IB_STAGE_ROCE,
7152	mlx5_ib_stage_roce_init,	4640	mlx5_ib_roce_init,
7153	mlx5_ib_stage_roce_cleanup),	4641	mlx5_ib_roce_cleanup),
7154	STAGE_CREATE(MLX5_IB_STAGE_QP,	4642	STAGE_CREATE(MLX5_IB_STAGE_QP,
7155	mlx5_init_qp_table,	4643	mlx5_init_qp_table,
7156	mlx5_cleanup_qp_table),	4644	mlx5_cleanup_qp_table),
@@ -7158,17 +4646,17 @@ static const struct mlx5_ib_profile pf_profile = { · 1 unmodified line
7158	mlx5_init_srq_table,	4646	mlx5_init_srq_table,
7159	mlx5_cleanup_srq_table),	4647	mlx5_cleanup_srq_table),
7160	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,	4648	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
7161	mlx5_ib_stage_dev_res_init,	4649	mlx5_ib_dev_res_init,
7162	mlx5_ib_stage_dev_res_cleanup),	4650	mlx5_ib_dev_res_cleanup),
7163	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,	4651	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,
7164	mlx5_ib_stage_dev_notifier_init,	4652	mlx5_ib_stage_dev_notifier_init,
7165	mlx5_ib_stage_dev_notifier_cleanup),	4653	mlx5_ib_stage_dev_notifier_cleanup),
7166	STAGE_CREATE(MLX5_IB_STAGE_ODP,	4654	STAGE_CREATE(MLX5_IB_STAGE_ODP,
7167	mlx5_ib_stage_odp_init,	4655	mlx5_ib_odp_init_one,
7168	mlx5_ib_stage_odp_cleanup),	4656	mlx5_ib_odp_cleanup_one),
7169	STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,	4657	STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
7170	mlx5_ib_stage_counters_init,	4658	mlx5_ib_counters_init,
7171	mlx5_ib_stage_counters_cleanup),	4659	mlx5_ib_counters_cleanup),
7172	STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS,	4660	STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS,
7173	mlx5_ib_stage_cong_debugfs_init,	4661	mlx5_ib_stage_cong_debugfs_init,
7174	mlx5_ib_stage_cong_debugfs_cleanup),	4662	mlx5_ib_stage_cong_debugfs_cleanup),
@@ -7182,8 +4670,8 @@ static const struct mlx5_ib_profile pf_profile = { · 7 unmodified lines
7182	NULL,	4670	NULL,
7183	mlx5_ib_stage_pre_ib_reg_umr_cleanup),	4671	mlx5_ib_stage_pre_ib_reg_umr_cleanup),
7184	STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID,	4672	STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID,
7185	mlx5_ib_stage_devx_init,	4673	mlx5_ib_devx_init,
7186	mlx5_ib_stage_devx_cleanup),	4674	mlx5_ib_devx_cleanup),
7187	STAGE_CREATE(MLX5_IB_STAGE_IB_REG,	4675	STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
7188	mlx5_ib_stage_ib_reg_init,	4676	mlx5_ib_stage_ib_reg_init,
7189	mlx5_ib_stage_ib_reg_cleanup),	4677	mlx5_ib_stage_ib_reg_cleanup),
@@ -7193,15 +4681,18 @@ static const struct mlx5_ib_profile pf_profile = { · 3 unmodified lines
7193	STAGE_CREATE(MLX5_IB_STAGE_DELAY_DROP,	4681	STAGE_CREATE(MLX5_IB_STAGE_DELAY_DROP,
7194	mlx5_ib_stage_delay_drop_init,	4682	mlx5_ib_stage_delay_drop_init,
7195	mlx5_ib_stage_delay_drop_cleanup),	4683	mlx5_ib_stage_delay_drop_cleanup),
		4684	STAGE_CREATE(MLX5_IB_STAGE_RESTRACK,
		4685	mlx5_ib_restrack_init,
		4686	NULL),
7196	};	4687	};
7197		4688
7198	const struct mlx5_ib_profile raw_eth_profile = {	4689	const struct mlx5_ib_profile raw_eth_profile = {
7199	STAGE_CREATE(MLX5_IB_STAGE_INIT,	4690	STAGE_CREATE(MLX5_IB_STAGE_INIT,
7200	mlx5_ib_stage_init_init,	4691	mlx5_ib_stage_init_init,
7201	mlx5_ib_stage_init_cleanup),	4692	mlx5_ib_stage_init_cleanup),
7202	STAGE_CREATE(MLX5_IB_STAGE_FLOW_DB,	4693	STAGE_CREATE(MLX5_IB_STAGE_FS,
7203	mlx5_ib_stage_flow_db_init,	4694	mlx5_ib_fs_init,
7204	mlx5_ib_stage_flow_db_cleanup),	4695	mlx5_ib_fs_cleanup),
7205	STAGE_CREATE(MLX5_IB_STAGE_CAPS,	4696	STAGE_CREATE(MLX5_IB_STAGE_CAPS,
7206	mlx5_ib_stage_caps_init,	4697	mlx5_ib_stage_caps_init,
7207	mlx5_ib_stage_caps_cleanup),	4698	mlx5_ib_stage_caps_cleanup),
@@ -7209,8 +4700,8 @@ const struct mlx5_ib_profile raw_eth_profile = { · 1 unmodified line
7209	mlx5_ib_stage_raw_eth_non_default_cb,	4700	mlx5_ib_stage_raw_eth_non_default_cb,
7210	NULL),	4701	NULL),
7211	STAGE_CREATE(MLX5_IB_STAGE_ROCE,	4702	STAGE_CREATE(MLX5_IB_STAGE_ROCE,
7212	mlx5_ib_stage_raw_eth_roce_init,	4703	mlx5_ib_roce_init,
7213	mlx5_ib_stage_raw_eth_roce_cleanup),	4704	mlx5_ib_roce_cleanup),
7214	STAGE_CREATE(MLX5_IB_STAGE_QP,	4705	STAGE_CREATE(MLX5_IB_STAGE_QP,
7215	mlx5_init_qp_table,	4706	mlx5_init_qp_table,
7216	mlx5_cleanup_qp_table),	4707	mlx5_cleanup_qp_table),
@@ -7218,14 +4709,14 @@ const struct mlx5_ib_profile raw_eth_profile = { · 1 unmodified line
7218	mlx5_init_srq_table,	4709	mlx5_init_srq_table,
7219	mlx5_cleanup_srq_table),	4710	mlx5_cleanup_srq_table),
7220	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,	4711	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
7221	mlx5_ib_stage_dev_res_init,	4712	mlx5_ib_dev_res_init,
7222	mlx5_ib_stage_dev_res_cleanup),	4713	mlx5_ib_dev_res_cleanup),
7223	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,	4714	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,
7224	mlx5_ib_stage_dev_notifier_init,	4715	mlx5_ib_stage_dev_notifier_init,
7225	mlx5_ib_stage_dev_notifier_cleanup),	4716	mlx5_ib_stage_dev_notifier_cleanup),
7226	STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,	4717	STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
7227	mlx5_ib_stage_counters_init,	4718	mlx5_ib_counters_init,
7228	mlx5_ib_stage_counters_cleanup),	4719	mlx5_ib_counters_cleanup),
7229	STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS,	4720	STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS,
7230	mlx5_ib_stage_cong_debugfs_init,	4721	mlx5_ib_stage_cong_debugfs_init,
7231	mlx5_ib_stage_cong_debugfs_cleanup),	4722	mlx5_ib_stage_cong_debugfs_cleanup),
@@ -7239,14 +4730,17 @@ const struct mlx5_ib_profile raw_eth_profile = { · 7 unmodified lines
7239	NULL,	4730	NULL,
7240	mlx5_ib_stage_pre_ib_reg_umr_cleanup),	4731	mlx5_ib_stage_pre_ib_reg_umr_cleanup),
7241	STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID,	4732	STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID,
7242	mlx5_ib_stage_devx_init,	4733	mlx5_ib_devx_init,
7243	mlx5_ib_stage_devx_cleanup),	4734	mlx5_ib_devx_cleanup),
7244	STAGE_CREATE(MLX5_IB_STAGE_IB_REG,	4735	STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
7245	mlx5_ib_stage_ib_reg_init,	4736	mlx5_ib_stage_ib_reg_init,
7246	mlx5_ib_stage_ib_reg_cleanup),	4737	mlx5_ib_stage_ib_reg_cleanup),
7247	STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR,	4738	STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR,
7248	mlx5_ib_stage_post_ib_reg_umr_init,	4739	mlx5_ib_stage_post_ib_reg_umr_init,
7249	NULL),	4740	NULL),
		4741	STAGE_CREATE(MLX5_IB_STAGE_RESTRACK,
		4742	mlx5_ib_restrack_init,
		4743	NULL),
7250	};	4744	};
7251		4745
7252	static void mlx5_ib_add_slave_port(struct mlx5_core_dev mdev)	4746	static void mlx5_ib_add_slave_port(struct mlx5_core_dev mdev)

drivers/infiniband/hw/mlx5/mlx5_ib.h+17−92

+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
 /*
- * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
  */
 #ifndef MLX5_IB_H
 struct mlx5_ib_resources {
 	struct ib_cq	*c0;
-	struct ib_xrcd	*x0;
+	u32 xrcdn0;
-	struct ib_xrcd	*x1;
+	u32 xrcdn1;
 	struct ib_pd	*p0;
 	struct ib_srq	*s0;
 	struct ib_srq	*s1;
 enum mlx5_ib_stages {
 	MLX5_IB_STAGE_INIT,
-	MLX5_IB_STAGE_FLOW_DB,
+	MLX5_IB_STAGE_FS,
 	MLX5_IB_STAGE_CAPS,
 	MLX5_IB_STAGE_NON_DEFAULT_CB,
 	MLX5_IB_STAGE_ROCE,
 	MLX5_IB_STAGE_IB_REG,
 	MLX5_IB_STAGE_POST_IB_REG_UMR,
 	MLX5_IB_STAGE_DELAY_DROP,
-	MLX5_IB_STAGE_CLASS_ATTR,
+	MLX5_IB_STAGE_RESTRACK,
 	MLX5_IB_STAGE_MAX,
 };
 	return container_of(core_qp, struct mlx5_ib_rwq, core_qp);
 }
-static inline struct mlx5_ib_mr *to_mibmr(struct mlx5_core_mkey *mmkey)
-{
-	return container_of(mmkey, struct mlx5_ib_mr, mmkey);
-}
 static inline struct mlx5_ib_pd *to_mpd(struct ib_pd *ibpd)
 {
 	return container_of(ibpd, struct mlx5_ib_pd, ibpd);
 			  struct ib_pd *pd, struct ib_udata *udata);
 int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
 struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
-			       u32 max_num_sg, struct ib_udata *udata);
+			       u32 max_num_sg);
 struct ib_mr *mlx5_ib_alloc_mr_integrity(struct ib_pd *pd,
 					 u32 max_num_sg,
 					 u32 max_num_meta_sg);
 			const struct ib_wc *in_wc, const struct ib_grh *in_grh,
 			const struct ib_mad *in, struct ib_mad *out,
 			size_t *out_mad_size, u16 *out_mad_pkey_index);
-struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev,
+int mlx5_ib_alloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata);
-				   struct ib_udata *udata);
+void mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata);
-int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata);
 int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset);
 int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port);
 int mlx5_query_mad_ifc_smp_attr_node_info(struct ib_device *ibdev,
 						   u8 *native_port_num);
 void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *dev,
 				  u8 port_num);
-int mlx5_ib_fill_res_entry(struct sk_buff *msg,
-			   struct rdma_restrack_entry *res);
-int mlx5_ib_fill_stat_entry(struct sk_buff *msg,
-			    struct rdma_restrack_entry *res);
 extern const struct uapi_definition mlx5_ib_devx_defs[];
 extern const struct uapi_definition mlx5_ib_flow_defs[];
 extern const struct uapi_definition mlx5_ib_qos_defs[];
+extern const struct uapi_definition mlx5_ib_std_types_defs[];
-#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)
-int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user);
-void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid);
-void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev);
-void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev);
-struct mlx5_ib_flow_handler *mlx5_ib_raw_fs_rule_add(
-	struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher,
-	struct mlx5_flow_context *flow_context,
-	struct mlx5_flow_act *flow_act, u32 counter_id,
-	void *cmd_in, int inlen, int dest_id, int dest_type);
-bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type);
-bool mlx5_ib_devx_is_flow_counter(void *obj, u32 offset, u32 *counter_id);
-void mlx5_ib_destroy_flow_action_raw(struct mlx5_ib_flow_action *maction);
-#else
-static inline int
-mlx5_ib_devx_create(struct mlx5_ib_dev *dev,
-			   bool is_user) { return -EOPNOTSUPP; }
-static inline void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid) {}
-static inline void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev) {}
-static inline void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev) {}
-static inline bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id,
-					     int *dest_type)
-{
-	return false;
-}
-static inline void
-mlx5_ib_destroy_flow_action_raw(struct mlx5_ib_flow_action *maction)
-{
-	return;
-};
-#endif
 static inline void init_query_mad(struct ib_smp *mad)
 {
 	mad->base_version  = 1;
 	mad->method	   = IB_MGMT_METHOD_GET;
 }
-static inline u8 convert_access(int acc)
-{
-	return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC       : 0) |
-	       (acc & IB_ACCESS_REMOTE_WRITE  ? MLX5_PERM_REMOTE_WRITE : 0) |
-	       (acc & IB_ACCESS_REMOTE_READ   ? MLX5_PERM_REMOTE_READ  : 0) |
-	       (acc & IB_ACCESS_LOCAL_WRITE   ? MLX5_PERM_LOCAL_WRITE  : 0) |
-	       MLX5_PERM_LOCAL_READ;
-}
 static inline int is_qp1(enum ib_qp_type qp_type)
 {
 	return qp_type == MLX5_IB_QPT_HW_GSI;
 			struct mlx5_bfreg_info *bfregi, u32 bfregn,
 			bool dyn_bfreg);
-int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter *counter);
-u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u8 port_num);
 static inline bool mlx5_ib_can_use_umr(struct mlx5_ib_dev *dev,
 				       bool do_modify_atomic, int access_flags)
 {
 		return false;
 	if (access_flags & IB_ACCESS_RELAXED_ORDERING &&
-	    (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write) ||
+	    MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write) &&
-	     MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read)))
+	    !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr))
+		return false;
+	if (access_flags & IB_ACCESS_RELAXED_ORDERING &&
+	     MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read) &&
+	     !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr))
 		return false;
 	return true;
 }
-int mlx5_ib_enable_driver(struct ib_device *dev);
 int mlx5_ib_test_wc(struct mlx5_ib_dev *dev);
 static inline bool mlx5_ib_lag_should_assign_affinity(struct mlx5_ib_dev *dev)

drivers/infiniband/hw/mlx5/mr.c+1−1

@@ -1961,7 +1961,7 @@ err_free: · 1960 unmodified lines
1961	}	1961	}
1962		1962
1963	struct ib_mr mlx5_ib_alloc_mr(struct ib_pd pd, enum ib_mr_type mr_type,	1963	struct ib_mr mlx5_ib_alloc_mr(struct ib_pd pd, enum ib_mr_type mr_type,
1964	u32 max_num_sg, struct ib_udata *udata)	1964	u32 max_num_sg)
1965	{	1965	{
1966	return __mlx5_ib_alloc_mr(pd, mr_type, max_num_sg, 0);	1966	return __mlx5_ib_alloc_mr(pd, mr_type, max_num_sg, 0);
1967	}	1967	}

drivers/infiniband/hw/mlx5/odp.c+19−9

 {
 	struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem);
+	lockdep_assert_held(&mr->dev->odp_srcu);
 	if (unlikely(io_virt < mr->mmkey.iova))
 		return -EFAULT;
 		if (ret < 0)
 			goto srcu_unlock;
-		/*
-		 * When prefetching a page, page fault is generated
-		 * in order to bring the page to the main memory.
-		 * In the current flow, page faults are being counted.
-		 */
 		mlx5_update_odp_stats(mr, faults, ret);
 		npages += ret;
 {
 	struct prefetch_mr_work *work =
 		container_of(w, struct prefetch_mr_work, work);
+	struct mlx5_ib_dev *dev;
 	u32 bytes_mapped = 0;
+	int srcu_key;
+	int ret;
 	u32 i;
-	for (i = 0; i < work->num_sge; ++i)
+	/* We rely on IB/core that work is executed if we have num_sge != 0 only. */
-		pagefault_mr(work->frags[i].mr, work->frags[i].io_virt,
+	WARN_ON(!work->num_sge);
-			     work->frags[i].length, &bytes_mapped,
+	dev = work->frags[0].mr->dev;
-			     work->pf_flags);
+	/* SRCU should be held when calling to mlx5_odp_populate_xlt() */
+	srcu_key = srcu_read_lock(&dev->odp_srcu);
+	for (i = 0; i < work->num_sge; ++i) {
+		ret = pagefault_mr(work->frags[i].mr, work->frags[i].io_virt,
+				   work->frags[i].length, &bytes_mapped,
+				   work->pf_flags);
+		if (ret <= 0)
+			continue;
+		mlx5_update_odp_stats(work->frags[i].mr, prefetch, ret);
+	}
+	srcu_read_unlock(&dev->odp_srcu, srcu_key);
 	destroy_prefetch_work(work);
 }
 				   &bytes_mapped, pf_flags);
 		if (ret < 0)
 			goto out;
+		mlx5_update_odp_stats(mr, prefetch, ret);
 	}
 	ret = 0;

drivers/infiniband/hw/mlx5/qp.c+25−46

@@ -38,6 +38,7 @@ · 37 unmodified lines
38	#include <linux/mlx5/fs.h>	38	#include <linux/mlx5/fs.h>
39	#include "mlx5_ib.h"	39	#include "mlx5_ib.h"
40	#include "ib_rep.h"	40	#include "ib_rep.h"
		41	#include "counters.h"
41	#include "cmd.h"	42	#include "cmd.h"
42	#include "qp.h"	43	#include "qp.h"
43	#include "wr.h"	44	#include "wr.h"
@@ -2031,15 +2032,15 @@ static int create_user_qp(struct mlx5_ib_dev dev, struct ib_pd pd, · 1987 unmodified lines
2031	switch (init_attr->qp_type) {	2032	switch (init_attr->qp_type) {
2032	case IB_QPT_XRC_INI:	2033	case IB_QPT_XRC_INI:
2033	MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);	2034	MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
2034	MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn);	2035	MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);
2035	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);	2036	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);
2036	break;	2037	break;
2037	default:	2038	default:
2038	if (init_attr->srq) {	2039	if (init_attr->srq) {
2039	MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x0)->xrcdn);	2040	MLX5_SET(qpc, qpc, xrcd, devr->xrcdn0);
2040	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(init_attr->srq)->msrq.srqn);	2041	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(init_attr->srq)->msrq.srqn);
2041	} else {	2042	} else {
2042	MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn);	2043	MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);
2043	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s1)->msrq.srqn);	2044	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s1)->msrq.srqn);
2044	}	2045	}
2045	}	2046	}
@@ -2178,11 +2179,11 @@ static int create_kernel_qp(struct mlx5_ib_dev dev, struct ib_pd pd, · 132 unmodified lines
2178	MLX5_SET(qpc, qpc, no_sq, 1);	2179	MLX5_SET(qpc, qpc, no_sq, 1);
2179		2180
2180	if (attr->srq) {	2181	if (attr->srq) {
2181	MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x0)->xrcdn);	2182	MLX5_SET(qpc, qpc, xrcd, devr->xrcdn0);
2182	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,	2183	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
2183	to_msrq(attr->srq)->msrq.srqn);	2184	to_msrq(attr->srq)->msrq.srqn);
2184	} else {	2185	} else {
2185	MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn);	2186	MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);
2186	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,	2187	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
2187	to_msrq(devr->s1)->msrq.srqn);	2188	to_msrq(devr->s1)->msrq.srqn);
2188	}	2189	}
@@ -3554,7 +3555,7 @@ static int modify_raw_packet_qp(struct mlx5_ib_dev dev, struct mlx5_ib_qp qp, · 1365 unmodified lines
3554	switch (raw_qp_param->operation) {	3555	switch (raw_qp_param->operation) {
3555	case MLX5_CMD_OP_RST2INIT_QP:	3556	case MLX5_CMD_OP_RST2INIT_QP:
3556	rq_state = MLX5_RQC_STATE_RDY;	3557	rq_state = MLX5_RQC_STATE_RDY;
3557	sq_state = MLX5_SQC_STATE_RDY;	3558	sq_state = MLX5_SQC_STATE_RST;
3558	break;	3559	break;
3559	case MLX5_CMD_OP_2ERR_QP:	3560	case MLX5_CMD_OP_2ERR_QP:
3560	rq_state = MLX5_RQC_STATE_ERR;	3561	rq_state = MLX5_RQC_STATE_ERR;
@@ -3566,13 +3567,11 @@ static int modify_raw_packet_qp(struct mlx5_ib_dev dev, struct mlx5_ib_qp qp, · 5 unmodified lines
3566	break;	3567	break;
3567	case MLX5_CMD_OP_RTR2RTS_QP:	3568	case MLX5_CMD_OP_RTR2RTS_QP:
3568	case MLX5_CMD_OP_RTS2RTS_QP:	3569	case MLX5_CMD_OP_RTS2RTS_QP:
3569	if (raw_qp_param->set_mask ==	3570	if (raw_qp_param->set_mask & ~MLX5_RAW_QP_RATE_LIMIT)
3570	MLX5_RAW_QP_RATE_LIMIT) {	3571	return -EINVAL;
3571	modify_rq = 0;	3572
3572	sq_state = sq->state;	3573	modify_rq = 0;
3573	} else {	3574	sq_state = MLX5_SQC_STATE_RDY;
3574	return raw_qp_param->set_mask ? -EINVAL : 0;
3575	}
3576	break;	3575	break;
3577	case MLX5_CMD_OP_INIT2INIT_QP:	3576	case MLX5_CMD_OP_INIT2INIT_QP:
3578	case MLX5_CMD_OP_INIT2RTR_QP:	3577	case MLX5_CMD_OP_INIT2RTR_QP:
@@ -4114,9 +4113,9 @@ static int mlx5_ib_modify_dct(struct ib_qp ibqp, struct ib_qp_attr attr, · 535 unmodified lines
4114	struct mlx5_ib_qp *qp = to_mqp(ibqp);	4113	struct mlx5_ib_qp *qp = to_mqp(ibqp);
4115	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);	4114	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
4116	enum ib_qp_state cur_state, new_state;	4115	enum ib_qp_state cur_state, new_state;
4117	int err = 0;
4118	int required = IB_QP_STATE;	4116	int required = IB_QP_STATE;
4119	void *dctc;	4117	void *dctc;
		4118	int err;
4120		4119
4121	if (!(attr_mask & IB_QP_STATE))	4120	if (!(attr_mask & IB_QP_STATE))
4122	return -EINVAL;	4121	return -EINVAL;
@@ -4208,11 +4207,9 @@ static int mlx5_ib_modify_dct(struct ib_qp ibqp, struct ib_qp_attr attr, · 85 unmodified lines
4208	mlx5_ib_warn(dev, "Modify DCT: Invalid transition from %d to %d\n", cur_state, new_state);	4207	mlx5_ib_warn(dev, "Modify DCT: Invalid transition from %d to %d\n", cur_state, new_state);
4209	return -EINVAL;	4208	return -EINVAL;
4210	}	4209	}
4211	if (err)	4210
4212	qp->state = IB_QPS_ERR;	4211	qp->state = new_state;
4213	else	4212	return 0;
4214	qp->state = new_state;
4215	return err;
4216	}	4213	}
4217		4214
4218	int mlx5_ib_modify_qp(struct ib_qp ibqp, struct ib_qp_attr attr,	4215	int mlx5_ib_modify_qp(struct ib_qp ibqp, struct ib_qp_attr attr,
@@ -4450,7 +4447,7 @@ static int sqrq_state_to_qp_state(u8 sq_state, u8 rq_state, · 231 unmodified lines
4450	[MLX5_SQ_STATE_NA] = IB_QPS_RESET,	4447	[MLX5_SQ_STATE_NA] = IB_QPS_RESET,
4451	},	4448	},
4452	[MLX5_RQC_STATE_RDY] = {	4449	[MLX5_RQC_STATE_RDY] = {
4453	[MLX5_SQC_STATE_RST] = MLX5_QP_STATE_BAD,	4450	[MLX5_SQC_STATE_RST] = MLX5_QP_STATE,
4454	[MLX5_SQC_STATE_RDY] = MLX5_QP_STATE,	4451	[MLX5_SQC_STATE_RDY] = MLX5_QP_STATE,
4455	[MLX5_SQC_STATE_ERR] = IB_QPS_SQE,	4452	[MLX5_SQC_STATE_ERR] = IB_QPS_SQE,
4456	[MLX5_SQ_STATE_NA] = MLX5_QP_STATE,	4453	[MLX5_SQ_STATE_NA] = MLX5_QP_STATE,
@@ -4462,7 +4459,7 @@ static int sqrq_state_to_qp_state(u8 sq_state, u8 rq_state, · 5 unmodified lines
4462	[MLX5_SQ_STATE_NA] = IB_QPS_ERR,	4459	[MLX5_SQ_STATE_NA] = IB_QPS_ERR,
4463	},	4460	},
4464	[MLX5_RQ_STATE_NA] = {	4461	[MLX5_RQ_STATE_NA] = {
4465	[MLX5_SQC_STATE_RST] = IB_QPS_RESET,	4462	[MLX5_SQC_STATE_RST] = MLX5_QP_STATE,
4466	[MLX5_SQC_STATE_RDY] = MLX5_QP_STATE,	4463	[MLX5_SQC_STATE_RDY] = MLX5_QP_STATE,
4467	[MLX5_SQC_STATE_ERR] = MLX5_QP_STATE,	4464	[MLX5_SQC_STATE_ERR] = MLX5_QP_STATE,
4468	[MLX5_SQ_STATE_NA] = MLX5_QP_STATE_BAD,	4465	[MLX5_SQ_STATE_NA] = MLX5_QP_STATE_BAD,
@@ -4708,41 +4705,23 @@ out: · 239 unmodified lines
4708	return err;	4705	return err;
4709	}	4706	}
4710		4707
4711	struct ib_xrcd mlx5_ib_alloc_xrcd(struct ib_device ibdev,	4708	int mlx5_ib_alloc_xrcd(struct ib_xrcd ibxrcd, struct ib_udata udata)
4712	struct ib_udata *udata)
4713	{	4709	{
4714	struct mlx5_ib_dev *dev = to_mdev(ibdev);	4710	struct mlx5_ib_dev *dev = to_mdev(ibxrcd->device);
4715	struct mlx5_ib_xrcd *xrcd;	4711	struct mlx5_ib_xrcd *xrcd = to_mxrcd(ibxrcd);
4716	int err;
4717		4712
4718	if (!MLX5_CAP_GEN(dev->mdev, xrc))	4713	if (!MLX5_CAP_GEN(dev->mdev, xrc))
4719	return ERR_PTR(-ENOSYS);	4714	return -EOPNOTSUPP;
4720
4721	xrcd = kmalloc(sizeof(*xrcd), GFP_KERNEL);
4722	if (!xrcd)
4723	return ERR_PTR(-ENOMEM);
4724
4725	err = mlx5_cmd_xrcd_alloc(dev->mdev, &xrcd->xrcdn, 0);
4726	if (err) {
4727	kfree(xrcd);
4728	return ERR_PTR(-ENOMEM);
4729	}
4730		4715
4731	return &xrcd->ibxrcd;	4716	return mlx5_cmd_xrcd_alloc(dev->mdev, &xrcd->xrcdn, 0);
4732	}	4717	}
4733		4718
4734	int mlx5_ib_dealloc_xrcd(struct ib_xrcd xrcd, struct ib_udata udata)	4719	void mlx5_ib_dealloc_xrcd(struct ib_xrcd xrcd, struct ib_udata udata)
4735	{	4720	{
4736	struct mlx5_ib_dev *dev = to_mdev(xrcd->device);	4721	struct mlx5_ib_dev *dev = to_mdev(xrcd->device);
4737	u32 xrcdn = to_mxrcd(xrcd)->xrcdn;	4722	u32 xrcdn = to_mxrcd(xrcd)->xrcdn;
4738	int err;
4739		4723
4740	err = mlx5_cmd_xrcd_dealloc(dev->mdev, xrcdn, 0);	4724	mlx5_cmd_xrcd_dealloc(dev->mdev, xrcdn, 0);
4741	if (err)
4742	mlx5_ib_warn(dev, "failed to dealloc xrcdn 0x%x\n", xrcdn);
4743
4744	kfree(xrcd);
4745	return 0;
4746	}	4725	}
4747		4726
4748	static void mlx5_ib_wq_event(struct mlx5_core_qp *core_qp, int type)	4727	static void mlx5_ib_wq_event(struct mlx5_core_qp *core_qp, int type)

drivers/infiniband/hw/mlx5/qp.h+1−0

@@ -43,4 +43,5 @@ void mlx5_core_res_put(struct mlx5_core_rsc_common *res); · 42 unmodified lines
43		43
44	int mlx5_core_xrcd_alloc(struct mlx5_ib_dev dev, u32 xrcdn);	44	int mlx5_core_xrcd_alloc(struct mlx5_ib_dev dev, u32 xrcdn);
45	int mlx5_core_xrcd_dealloc(struct mlx5_ib_dev *dev, u32 xrcdn);	45	int mlx5_core_xrcd_dealloc(struct mlx5_ib_dev *dev, u32 xrcdn);
		46	int mlx5_ib_qp_set_counter(struct ib_qp qp, struct rdma_counter counter);
46	#endif /* _MLX5_IB_QP_H */	47	#endif /* _MLX5_IB_QP_H */

drivers/infiniband/hw/mlx5/restrack.c+105−16

@@ -1,17 +1,85 @@
1	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB	1	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2	/*	2	/*
3	* Copyright (c) 2019, Mellanox Technologies inc. All rights reserved.	3	* Copyright (c) 2019-2020, Mellanox Technologies Ltd. All rights reserved.
4	*/	4	*/
5		5
6	#include <uapi/rdma/rdma_netlink.h>	6	#include <uapi/rdma/rdma_netlink.h>
		7	#include <linux/mlx5/rsc_dump.h>
7	#include <rdma/ib_umem_odp.h>	8	#include <rdma/ib_umem_odp.h>
8	#include <rdma/restrack.h>	9	#include <rdma/restrack.h>
9	#include "mlx5_ib.h"	10	#include "mlx5_ib.h"
		11	#include "restrack.h"
10		12
11	static int fill_stat_mr_entry(struct sk_buff *msg,	13	#define MAX_DUMP_SIZE 1024
12	struct rdma_restrack_entry *res)	14
		15	static int dump_rsc(struct mlx5_core_dev *dev, enum mlx5_sgmt_type type,
		16	int index, void data, int data_len)
		17	{
		18	struct mlx5_core_dev *mdev = dev;
		19	struct mlx5_rsc_dump_cmd *cmd;
		20	struct mlx5_rsc_key key = {};
		21	struct page *page;
		22	int offset = 0;
		23	int err = 0;
		24	int cmd_err;
		25	int size;
		26
		27	page = alloc_page(GFP_KERNEL);
		28	if (!page)
		29	return -ENOMEM;
		30
		31	key.size = PAGE_SIZE;
		32	key.rsc = type;
		33	key.index1 = index;
		34	key.num_of_obj1 = 1;
		35
		36	cmd = mlx5_rsc_dump_cmd_create(mdev, &key);
		37	if (IS_ERR(cmd)) {
		38	err = PTR_ERR(cmd);
		39	goto free_page;
		40	}
		41
		42	do {
		43	cmd_err = mlx5_rsc_dump_next(mdev, cmd, page, &size);
		44	if (cmd_err < 0 \|\| size + offset > MAX_DUMP_SIZE) {
		45	err = cmd_err;
		46	goto destroy_cmd;
		47	}
		48	memcpy(data + offset, page_address(page), size);
		49	offset += size;
		50	} while (cmd_err > 0);
		51	*data_len = offset;
		52
		53	destroy_cmd:
		54	mlx5_rsc_dump_cmd_destroy(cmd);
		55	free_page:
		56	__free_page(page);
		57	return err;
		58	}
		59
		60	static int fill_res_raw(struct sk_buff msg, struct mlx5_ib_dev dev,
		61	enum mlx5_sgmt_type type, u32 key)
		62	{
		63	int len = 0;
		64	void *data;
		65	int err;
		66
		67	data = kzalloc(MAX_DUMP_SIZE, GFP_KERNEL);
		68	if (!data)
		69	return -ENOMEM;
		70
		71	err = dump_rsc(dev->mdev, type, key, data, &len);
		72	if (err)
		73	goto out;
		74
		75	err = nla_put(msg, RDMA_NLDEV_ATTR_RES_RAW, len, data);
		76	out:
		77	kfree(data);
		78	return err;
		79	}
		80
		81	static int fill_stat_mr_entry(struct sk_buff msg, struct ib_mr ibmr)
13	{	82	{
14	struct ib_mr *ibmr = container_of(res, struct ib_mr, res);
15	struct mlx5_ib_mr *mr = to_mmr(ibmr);	83	struct mlx5_ib_mr *mr = to_mmr(ibmr);
16	struct nlattr *table_attr;	84	struct nlattr *table_attr;
17		85
@@ -31,6 +99,9 @@ static int fill_stat_mr_entry(struct sk_buff *msg, · 13 unmodified lines
31	msg, "page_invalidations",	99	msg, "page_invalidations",
32	atomic64_read(&mr->odp_stats.invalidations)))	100	atomic64_read(&mr->odp_stats.invalidations)))
33	goto err_table;	101	goto err_table;
		102	if (rdma_nl_stat_hwcounter_entry(msg, "page_prefetch",
		103	atomic64_read(&mr->odp_stats.prefetch)))
		104	goto err_table;
34		105
35	nla_nest_end(msg, table_attr);	106	nla_nest_end(msg, table_attr);
36	return 0;	107	return 0;
@@ -41,10 +112,16 @@ err: · 4 unmodified lines
41	return -EMSGSIZE;	112	return -EMSGSIZE;
42	}	113	}
43		114
44	static int fill_res_mr_entry(struct sk_buff *msg,	115	static int fill_res_mr_entry_raw(struct sk_buff msg, struct ib_mr ibmr)
45	struct rdma_restrack_entry *res)	116	{
		117	struct mlx5_ib_mr *mr = to_mmr(ibmr);
		118
		119	return fill_res_raw(msg, mr->dev, MLX5_SGMT_TYPE_PRM_QUERY_MKEY,
		120	mlx5_mkey_to_idx(mr->mmkey.key));
		121	}
		122
		123	static int fill_res_mr_entry(struct sk_buff msg, struct ib_mr ibmr)
46	{	124	{
47	struct ib_mr *ibmr = container_of(res, struct ib_mr, res);
48	struct mlx5_ib_mr *mr = to_mmr(ibmr);	125	struct mlx5_ib_mr *mr = to_mmr(ibmr);
49	struct nlattr *table_attr;	126	struct nlattr *table_attr;
50		127
@@ -71,20 +148,32 @@ err: · 20 unmodified lines
71	return -EMSGSIZE;	148	return -EMSGSIZE;
72	}	149	}
73		150
74	int mlx5_ib_fill_res_entry(struct sk_buff *msg,	151	static int fill_res_cq_entry_raw(struct sk_buff msg, struct ib_cq ibcq)
75	struct rdma_restrack_entry *res)
76	{	152	{
77	if (res->type == RDMA_RESTRACK_MR)	153	struct mlx5_ib_dev *dev = to_mdev(ibcq->device);
78	return fill_res_mr_entry(msg, res);	154	struct mlx5_ib_cq *cq = to_mcq(ibcq);
79		155
80	return 0;	156	return fill_res_raw(msg, dev, MLX5_SGMT_TYPE_PRM_QUERY_CQ, cq->mcq.cqn);
81	}	157	}
82		158
83	int mlx5_ib_fill_stat_entry(struct sk_buff *msg,	159	static int fill_res_qp_entry_raw(struct sk_buff msg, struct ib_qp ibqp)
84	struct rdma_restrack_entry *res)
85	{	160	{
86	if (res->type == RDMA_RESTRACK_MR)	161	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
87	return fill_stat_mr_entry(msg, res);	162
		163	return fill_res_raw(msg, dev, MLX5_SGMT_TYPE_PRM_QUERY_QP,
		164	ibqp->qp_num);
		165	}
88		166
		167	static const struct ib_device_ops restrack_ops = {
		168	.fill_res_cq_entry_raw = fill_res_cq_entry_raw,
		169	.fill_res_mr_entry = fill_res_mr_entry,
		170	.fill_res_mr_entry_raw = fill_res_mr_entry_raw,
		171	.fill_res_qp_entry_raw = fill_res_qp_entry_raw,
		172	.fill_stat_mr_entry = fill_stat_mr_entry,
		173	};
		174
		175	int mlx5_ib_restrack_init(struct mlx5_ib_dev *dev)
		176	{
		177	ib_set_device_ops(&dev->ib_dev, &restrack_ops);
89	return 0;	178	return 0;
90	}	179	}

drivers/infiniband/hw/mlx5/restrack.h+13−0

@@ -0,0 +1,13 @@
	1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
	2	/*
	3	* Copyright (c) 2013-2020, Mellanox Technologies Ltd. All rights reserved.
	4	*/
	5
	6	#ifndef _MLX5_IB_RESTRACK_H
	7	#define _MLX5_IB_RESTRACK_H
	8
	9	#include "mlx5_ib.h"
	10
	11	int mlx5_ib_restrack_init(struct mlx5_ib_dev *dev);
	12
	13	#endif /* _MLX5_IB_RESTRACK_H */

drivers/infiniband/hw/mlx5/srq.c+2−2

@@ -274,10 +274,10 @@ int mlx5_ib_create_srq(struct ib_srq *ib_srq, · 273 unmodified lines
274	if (srq->wq_sig)	274	if (srq->wq_sig)
275	in.flags \|= MLX5_SRQ_FLAG_WQ_SIG;	275	in.flags \|= MLX5_SRQ_FLAG_WQ_SIG;
276		276
277	if (init_attr->srq_type == IB_SRQT_XRC)	277	if (init_attr->srq_type == IB_SRQT_XRC && init_attr->ext.xrc.xrcd)
278	in.xrcd = to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn;	278	in.xrcd = to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn;
279	else	279	else
280	in.xrcd = to_mxrcd(dev->devr.x0)->xrcdn;	280	in.xrcd = dev->devr.xrcdn0;
281		281
282	if (init_attr->srq_type == IB_SRQT_TM) {	282	if (init_attr->srq_type == IB_SRQT_TM) {
283	in.tm_log_list_size =	283	in.tm_log_list_size =

drivers/infiniband/hw/mlx5/std_types.c+45−0

@@ -0,0 +1,45 @@
	1	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
	2	/*
	3	* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved.
	4	*/
	5
	6	#include <rdma/uverbs_ioctl.h>
	7	#include <rdma/mlx5_user_ioctl_cmds.h>
	8	#include <rdma/mlx5_user_ioctl_verbs.h>
	9	#include <linux/mlx5/driver.h>
	10	#include "mlx5_ib.h"
	11
	12	#define UVERBS_MODULE_NAME mlx5_ib
	13	#include <rdma/uverbs_named_ioctl.h>
	14
	15	static int UVERBS_HANDLER(MLX5_IB_METHOD_PD_QUERY)(
	16	struct uverbs_attr_bundle *attrs)
	17	{
	18	struct ib_pd *pd =
	19	uverbs_attr_get_obj(attrs, MLX5_IB_ATTR_QUERY_PD_HANDLE);
	20	struct mlx5_ib_pd *mpd = to_mpd(pd);
	21
	22	return uverbs_copy_to(attrs, MLX5_IB_ATTR_QUERY_PD_RESP_PDN,
	23	&mpd->pdn, sizeof(mpd->pdn));
	24	}
	25
	26	DECLARE_UVERBS_NAMED_METHOD(
	27	MLX5_IB_METHOD_PD_QUERY,
	28	UVERBS_ATTR_IDR(MLX5_IB_ATTR_QUERY_PD_HANDLE,
	29	UVERBS_OBJECT_PD,
	30	UVERBS_ACCESS_READ,
	31	UA_MANDATORY),
	32	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_QUERY_PD_RESP_PDN,
	33	UVERBS_ATTR_TYPE(u32),
	34	UA_MANDATORY));
	35
	36	ADD_UVERBS_METHODS(mlx5_ib_pd,
	37	UVERBS_OBJECT_PD,
	38	&UVERBS_METHOD(MLX5_IB_METHOD_PD_QUERY));
	39
	40	const struct uapi_definition mlx5_ib_std_types_defs[] = {
	41	UAPI_DEF_CHAIN_OBJ_TREE(
	42	UVERBS_OBJECT_PD,
	43	&mlx5_ib_pd),
	44	{},
	45	};

drivers/infiniband/hw/mlx5/wr.c+50−18

 	return cpu_to_be64(result);
 }
-static __be64 get_umr_update_access_mask(int atomic)
+static __be64 get_umr_update_access_mask(int atomic,
+					 int relaxed_ordering_write,
+					 int relaxed_ordering_read)
 {
 	u64 result;
 	if (atomic)
 		result |= MLX5_MKEY_MASK_A;
+	if (relaxed_ordering_write)
+		result |= MLX5_MKEY_MASK_RELAXED_ORDERING_WRITE;
+	if (relaxed_ordering_read)
+		result |= MLX5_MKEY_MASK_RELAXED_ORDERING_READ;
 	return cpu_to_be64(result);
 }
 static int umr_check_mkey_mask(struct mlx5_ib_dev *dev, u64 mask)
 {
-	if ((mask & MLX5_MKEY_MASK_PAGE_SIZE &&
+	if (mask & MLX5_MKEY_MASK_PAGE_SIZE &&
-	     MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled)) ||
+	    MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled))
-	    (mask & MLX5_MKEY_MASK_A &&
-	     MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled)))
 		return -EPERM;
+	if (mask & MLX5_MKEY_MASK_A &&
+	    MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled))
+		return -EPERM;
+	if (mask & MLX5_MKEY_MASK_RELAXED_ORDERING_WRITE &&
+	    !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr))
+		return -EPERM;
+	if (mask & MLX5_MKEY_MASK_RELAXED_ORDERING_READ &&
+	    !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr))
+		return -EPERM;
 	return 0;
 }
 static int set_reg_umr_segment(struct mlx5_ib_dev *dev,
 			       struct mlx5_wqe_umr_ctrl_seg *umr,
-			       const struct ib_send_wr *wr, int atomic)
+			       const struct ib_send_wr *wr)
 {
 	const struct mlx5_umr_wr *umrwr = umr_wr(wr);
 	if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION)
 		umr->mkey_mask |= get_umr_update_translation_mask();
 	if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS) {
-		umr->mkey_mask |= get_umr_update_access_mask(atomic);
+		umr->mkey_mask |= get_umr_update_access_mask(
+			!!(MLX5_CAP_GEN(dev->mdev, atomic)),
+			!!(MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr)),
+			!!(MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr)));
 		umr->mkey_mask |= get_umr_update_pd_mask();
 	}
 	if (wr->send_flags & MLX5_IB_SEND_UMR_ENABLE_MR)
 	memset(seg, 0, sizeof(*seg));
 	if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR)
-		seg->status = MLX5_MKEY_STATUS_FREE;
+		MLX5_SET(mkc, seg, free, 1);
+	MLX5_SET(mkc, seg, a,
+		 !!(umrwr->access_flags & IB_ACCESS_REMOTE_ATOMIC));
+	MLX5_SET(mkc, seg, rw,
+		 !!(umrwr->access_flags & IB_ACCESS_REMOTE_WRITE));
+	MLX5_SET(mkc, seg, rr, !!(umrwr->access_flags & IB_ACCESS_REMOTE_READ));
+	MLX5_SET(mkc, seg, lw, !!(umrwr->access_flags & IB_ACCESS_LOCAL_WRITE));
+	MLX5_SET(mkc, seg, lr, 1);
+	MLX5_SET(mkc, seg, relaxed_ordering_write,
+		 !!(umrwr->access_flags & IB_ACCESS_RELAXED_ORDERING));
+	MLX5_SET(mkc, seg, relaxed_ordering_read,
+		 !!(umrwr->access_flags & IB_ACCESS_RELAXED_ORDERING));
-	seg->flags = convert_access(umrwr->access_flags);
 	if (umrwr->pd)
-		seg->flags_pd = cpu_to_be32(to_mpd(umrwr->pd)->pdn);
+		MLX5_SET(mkc, seg, pd, to_mpd(umrwr->pd)->pdn);
 	if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION &&
 	    !umrwr->length)
-		seg->flags_pd |= cpu_to_be32(MLX5_MKEY_LEN64);
+		MLX5_SET(mkc, seg, length64, 1);
-	seg->start_addr = cpu_to_be64(umrwr->virt_addr);
+	MLX5_SET64(mkc, seg, start_addr, umrwr->virt_addr);
-	seg->len = cpu_to_be64(umrwr->length);
+	MLX5_SET64(mkc, seg, len, umrwr->length);
-	seg->log2_page_size = umrwr->page_shift;
+	MLX5_SET(mkc, seg, log_page_size, umrwr->page_shift);
-	seg->qpn_mkey7_0 = cpu_to_be32(0xffffff00 |
+	MLX5_SET(mkc, seg, qpn, 0xffffff);
-				       mlx5_mkey_variant(umrwr->mkey));
+	MLX5_SET(mkc, seg, mkey_7_0, mlx5_mkey_variant(umrwr->mkey));
 }
 static void set_reg_data_seg(struct mlx5_wqe_data_seg *dseg,
 	qp->sq.wr_data[idx] = MLX5_IB_WR_UMR;
 	(*ctrl)->imm = cpu_to_be32(umr_wr(wr)->mkey);
-	err = set_reg_umr_segment(dev, *seg, wr,
+	err = set_reg_umr_segment(dev, *seg, wr);
-				  !!(MLX5_CAP_GEN(dev->mdev, atomic)));
 	if (unlikely(err))
 		goto out;
 	*seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);

drivers/infiniband/hw/ocrdma/ocrdma_verbs.c+1−1

@@ -2901,7 +2901,7 @@ int ocrdma_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags cq_flags) · 2900 unmodified lines
2901	}	2901	}
2902		2902
2903	struct ib_mr ocrdma_alloc_mr(struct ib_pd ibpd, enum ib_mr_type mr_type,	2903	struct ib_mr ocrdma_alloc_mr(struct ib_pd ibpd, enum ib_mr_type mr_type,
2904	u32 max_num_sg, struct ib_udata *udata)	2904	u32 max_num_sg)
2905	{	2905	{
2906	int status;	2906	int status;
2907	struct ocrdma_mr *mr;	2907	struct ocrdma_mr *mr;

drivers/infiniband/hw/ocrdma/ocrdma_verbs.h+1−1

@@ -101,7 +101,7 @@ struct ib_mr ocrdma_get_dma_mr(struct ib_pd , int acc); · 100 unmodified lines
101	struct ib_mr ocrdma_reg_user_mr(struct ib_pd , u64 start, u64 length,	101	struct ib_mr ocrdma_reg_user_mr(struct ib_pd , u64 start, u64 length,
102	u64 virt, int acc, struct ib_udata *);	102	u64 virt, int acc, struct ib_udata *);
103	struct ib_mr ocrdma_alloc_mr(struct ib_pd pd, enum ib_mr_type mr_type,	103	struct ib_mr ocrdma_alloc_mr(struct ib_pd pd, enum ib_mr_type mr_type,
104	u32 max_num_sg, struct ib_udata *udata);	104	u32 max_num_sg);
105	int ocrdma_map_mr_sg(struct ib_mr ibmr, struct scatterlist sg, int sg_nents,	105	int ocrdma_map_mr_sg(struct ib_mr ibmr, struct scatterlist sg, int sg_nents,
106	unsigned int *sg_offset);	106	unsigned int *sg_offset);
107		107

drivers/infiniband/hw/qedr/main.c+1−2

 	if (err)
 		return err;
-	immutable->pkey_tbl_len = 1;
 	immutable->gid_tbl_len = 1;
 	immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
 	immutable->max_mad_size = 0;
 static const struct ib_device_ops qedr_roce_dev_ops = {
 	.get_port_immutable = qedr_roce_port_immutable,
+	.query_pkey = qedr_query_pkey,
 };
 static void qedr_roce_register_device(struct qedr_dev *dev)
 	.post_srq_recv = qedr_post_srq_recv,
 	.process_mad = qedr_process_mad,
 	.query_device = qedr_query_device,
-	.query_pkey = qedr_query_pkey,
 	.query_port = qedr_query_port,
 	.query_qp = qedr_query_qp,
 	.query_srq = qedr_query_srq,

drivers/infiniband/hw/qedr/qedr.h+3−2

 	u32 dpi_size;
 	u16 dpi;
 	bool db_rec;
+	u8 edpm_mode;
 };
 union db_prod32 {
 	u32 wqe_prod;
 	u32 sge_prod;
 	u32 wr_prod_cnt;
-	u32 wr_cons_cnt;
+	atomic_t wr_cons_cnt;
 	u32 num_elems;
-	u32 *virt_prod_pair_addr;
+	struct rdma_srq_producers *virt_prod_pair_addr;
 	dma_addr_t phy_prod_pair_addr;
 };

drivers/infiniband/hw/qedr/verbs.c+25−20

@@ -239,7 +239,6 @@ int qedr_query_port(struct ib_device ibdev, u8 port, struct ib_port_attr attr) · 238 unmodified lines
239	attr->ip_gids = true;	239	attr->ip_gids = true;
240	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {	240	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
241	attr->gid_tbl_len = 1;	241	attr->gid_tbl_len = 1;
242	attr->pkey_tbl_len = 1;
243	} else {	242	} else {
244	attr->gid_tbl_len = QEDR_MAX_SGID;	243	attr->gid_tbl_len = QEDR_MAX_SGID;
245	attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;	244	attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
@@ -275,7 +274,8 @@ int qedr_alloc_ucontext(struct ib_ucontext uctx, struct ib_udata udata) · 29 unmodified lines
275	DP_ERR(dev, "Problem copying data from user space\n");	274	DP_ERR(dev, "Problem copying data from user space\n");
276	return -EFAULT;	275	return -EFAULT;
277	}	276	}
278		277	ctx->edpm_mode = !!(ureq.context_flags &
		278	QEDR_ALLOC_UCTX_EDPM_MODE);
279	ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC);	279	ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC);
280	}	280	}
281		281
@@ -316,11 +316,15 @@ int qedr_alloc_ucontext(struct ib_ucontext uctx, struct ib_udata udata) · 34 unmodified lines
316	uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY;	316	uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY;
317	else	317	else
318	uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED \|	318	uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED \|
319	QEDR_DPM_TYPE_ROCE_LEGACY;	319	QEDR_DPM_TYPE_ROCE_LEGACY \|
		320	QEDR_DPM_TYPE_ROCE_EDPM_MODE;
320		321
321	uresp.dpm_flags \|= QEDR_DPM_SIZES_SET;	322	if (ureq.context_flags & QEDR_SUPPORT_DPM_SIZES) {
322	uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE;	323	uresp.dpm_flags \|= QEDR_DPM_SIZES_SET;
323	uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE;	324	uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE;
		325	uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE;
		326	uresp.edpm_limit_size = QEDR_EDPM_MAX_SIZE;
		327	}
324		328
325	uresp.wids_enabled = 1;	329	uresp.wids_enabled = 1;
326	uresp.wid_count = oparams.wid_count;	330	uresp.wid_count = oparams.wid_count;
@@ -1754,7 +1758,7 @@ static int qedr_create_user_qp(struct qedr_dev *dev, · 1427 unmodified lines
1754	struct qed_rdma_create_qp_out_params out_params;	1758	struct qed_rdma_create_qp_out_params out_params;
1755	struct qedr_pd *pd = get_qedr_pd(ibpd);	1759	struct qedr_pd *pd = get_qedr_pd(ibpd);
1756	struct qedr_create_qp_uresp uresp;	1760	struct qedr_create_qp_uresp uresp;
1757	struct qedr_ucontext *ctx = NULL;	1761	struct qedr_ucontext *ctx = pd ? pd->uctx : NULL;
1758	struct qedr_create_qp_ureq ureq;	1762	struct qedr_create_qp_ureq ureq;
1759	int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);	1763	int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
1760	int rc = -EINVAL;	1764	int rc = -EINVAL;
@@ -1792,6 +1796,9 @@ static int qedr_create_user_qp(struct qedr_dev *dev, · 31 unmodified lines
1792	in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;	1796	in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1793	}	1797	}
1794		1798
		1799	if (ctx)
		1800	SET_FIELD(in_params.flags, QED_ROCE_EDPM_MODE, ctx->edpm_mode);
		1801
1795	qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,	1802	qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1796	&in_params, &out_params);	1803	&in_params, &out_params);
1797		1804
@@ -3004,7 +3011,7 @@ err0: · 1206 unmodified lines
3004	}	3011	}
3005		3012
3006	struct ib_mr qedr_alloc_mr(struct ib_pd ibpd, enum ib_mr_type mr_type,	3013	struct ib_mr qedr_alloc_mr(struct ib_pd ibpd, enum ib_mr_type mr_type,
3007	u32 max_num_sg, struct ib_udata *udata)	3014	u32 max_num_sg)
3008	{	3015	{
3009	struct qedr_mr *mr;	3016	struct qedr_mr *mr;
3010		3017
@@ -3687,7 +3694,7 @@ static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq) · 676 unmodified lines
3687	* count and consumer count and subtract it from max	3694	* count and consumer count and subtract it from max
3688	* work request supported so that we get elements left.	3695	* work request supported so that we get elements left.
3689	*/	3696	*/
3690	used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt;	3697	used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt);
3691		3698
3692	return hw_srq->max_wr - used;	3699	return hw_srq->max_wr - used;
3693	}	3700	}
@@ -3702,7 +3709,6 @@ int qedr_post_srq_recv(struct ib_srq ibsrq, const struct ib_recv_wr wr, · 8 unmodified lines
3702	unsigned long flags;	3709	unsigned long flags;
3703	int status = 0;	3710	int status = 0;
3704	u32 num_sge;	3711	u32 num_sge;
3705	u32 offset;
3706		3712
3707	spin_lock_irqsave(&srq->lock, flags);	3713	spin_lock_irqsave(&srq->lock, flags);
3708		3714
@@ -3715,7 +3721,8 @@ int qedr_post_srq_recv(struct ib_srq ibsrq, const struct ib_recv_wr wr, · 6 unmodified lines
3715	if (!qedr_srq_elem_left(hw_srq) \|\|	3721	if (!qedr_srq_elem_left(hw_srq) \|\|
3716	wr->num_sge > srq->hw_srq.max_sges) {	3722	wr->num_sge > srq->hw_srq.max_sges) {
3717	DP_ERR(dev, "Can't post WR (%d,%d) \|\| (%d > %d)\n",	3723	DP_ERR(dev, "Can't post WR (%d,%d) \|\| (%d > %d)\n",
3718	hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt,	3724	hw_srq->wr_prod_cnt,
		3725	atomic_read(&hw_srq->wr_cons_cnt),
3719	wr->num_sge, srq->hw_srq.max_sges);	3726	wr->num_sge, srq->hw_srq.max_sges);
3720	status = -ENOMEM;	3727	status = -ENOMEM;
3721	*bad_wr = wr;	3728	*bad_wr = wr;
@@ -3749,22 +3756,20 @@ int qedr_post_srq_recv(struct ib_srq ibsrq, const struct ib_recv_wr wr, · 27 unmodified lines
3749	hw_srq->sge_prod++;	3756	hw_srq->sge_prod++;
3750	}	3757	}
3751		3758
3752	/* Flush WQE and SGE information before	3759	/* Update WQE and SGE information before
3753	* updating producer.	3760	* updating producer.
3754	*/	3761	*/
3755	wmb();	3762	dma_wmb();
3756		3763
3757	/* SRQ producer is 8 bytes. Need to update SGE producer index	3764	/* SRQ producer is 8 bytes. Need to update SGE producer index
3758	* in first 4 bytes and need to update WQE producer in	3765	* in first 4 bytes and need to update WQE producer in
3759	* next 4 bytes.	3766	* next 4 bytes.
3760	*/	3767	*/
3761	*srq->hw_srq.virt_prod_pair_addr = hw_srq->sge_prod;	3768	srq->hw_srq.virt_prod_pair_addr->sge_prod = hw_srq->sge_prod;
3762	offset = offsetof(struct rdma_srq_producers, wqe_prod);	3769	/* Make sure sge producer is updated first */
3763	((u8 )srq->hw_srq.virt_prod_pair_addr + offset) =	3770	dma_wmb();
3764	hw_srq->wqe_prod;	3771	srq->hw_srq.virt_prod_pair_addr->wqe_prod = hw_srq->wqe_prod;
3765		3772
3766	/* Flush producer after updating it. */
3767	wmb();
3768	wr = wr->next;	3773	wr = wr->next;
3769	}	3774	}
3770		3775
@@ -4183,7 +4188,7 @@ static int process_resp_one_srq(struct qedr_dev dev, struct qedr_qp qp, · 412 unmodified lines
4183	} else {	4188	} else {
4184	__process_resp_one(dev, qp, cq, wc, resp, wr_id);	4189	__process_resp_one(dev, qp, cq, wc, resp, wr_id);
4185	}	4190	}
4186	srq->hw_srq.wr_cons_cnt++;	4191	atomic_inc(&srq->hw_srq.wr_cons_cnt);
4187		4192
4188	return 1;	4193	return 1;
4189	}	4194	}

drivers/infiniband/hw/qedr/verbs.h+1−1

@@ -84,7 +84,7 @@ int qedr_map_mr_sg(struct ib_mr ibmr, struct scatterlist sg, · 83 unmodified lines
84	int sg_nents, unsigned int *sg_offset);	84	int sg_nents, unsigned int *sg_offset);
85		85
86	struct ib_mr qedr_alloc_mr(struct ib_pd pd, enum ib_mr_type mr_type,	86	struct ib_mr qedr_alloc_mr(struct ib_pd pd, enum ib_mr_type mr_type,
87	u32 max_num_sg, struct ib_udata *udata);	87	u32 max_num_sg);
88	int qedr_poll_cq(struct ib_cq , int num_entries, struct ib_wc wc);	88	int qedr_poll_cq(struct ib_cq , int num_entries, struct ib_wc wc);
89	int qedr_post_send(struct ib_qp , const struct ib_send_wr ,	89	int qedr_post_send(struct ib_qp , const struct ib_send_wr ,
90	const struct ib_send_wr **bad_wr);	90	const struct ib_send_wr **bad_wr);

drivers/infiniband/hw/usnic/usnic_fwd.c+2−2

@@ -214,7 +214,7 @@ usnic_fwd_alloc_flow(struct usnic_fwd_dev ufdev, struct filter filter, · 213 unmodified lines
214	if (!flow)	214	if (!flow)
215	return ERR_PTR(-ENOMEM);	215	return ERR_PTR(-ENOMEM);
216		216
217	tlv = pci_alloc_consistent(pdev, tlv_size, &tlv_pa);	217	tlv = dma_alloc_coherent(&pdev->dev, tlv_size, &tlv_pa, GFP_ATOMIC);
218	if (!tlv) {	218	if (!tlv) {
219	usnic_err("Failed to allocate memory\n");	219	usnic_err("Failed to allocate memory\n");
220	status = -ENOMEM;	220	status = -ENOMEM;
@@ -258,7 +258,7 @@ usnic_fwd_alloc_flow(struct usnic_fwd_dev ufdev, struct filter filter, · 37 unmodified lines
258		258
259	out_free_tlv:	259	out_free_tlv:
260	spin_unlock(&ufdev->lock);	260	spin_unlock(&ufdev->lock);
261	pci_free_consistent(pdev, tlv_size, tlv, tlv_pa);	261	dma_free_coherent(&pdev->dev, tlv_size, tlv, tlv_pa);
262	if (!status)	262	if (!status)
263	return flow;	263	return flow;
264	out_free_flow:	264	out_free_flow:

drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c+1−1

@@ -202,7 +202,7 @@ err_umem: · 201 unmodified lines
202	* @return: ib_mr pointer on success, otherwise returns an errno.	202	* @return: ib_mr pointer on success, otherwise returns an errno.
203	*/	203	*/
204	struct ib_mr pvrdma_alloc_mr(struct ib_pd pd, enum ib_mr_type mr_type,	204	struct ib_mr pvrdma_alloc_mr(struct ib_pd pd, enum ib_mr_type mr_type,
205	u32 max_num_sg, struct ib_udata *udata)	205	u32 max_num_sg)
206	{	206	{
207	struct pvrdma_dev *dev = to_vdev(pd->device);	207	struct pvrdma_dev *dev = to_vdev(pd->device);
208	struct pvrdma_user_mr *mr;	208	struct pvrdma_user_mr *mr;

drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h+1−1

@@ -406,7 +406,7 @@ struct ib_mr pvrdma_reg_user_mr(struct ib_pd pd, u64 start, u64 length, · 405 unmodified lines
406	struct ib_udata *udata);	406	struct ib_udata *udata);
407	int pvrdma_dereg_mr(struct ib_mr mr, struct ib_udata udata);	407	int pvrdma_dereg_mr(struct ib_mr mr, struct ib_udata udata);
408	struct ib_mr pvrdma_alloc_mr(struct ib_pd pd, enum ib_mr_type mr_type,	408	struct ib_mr pvrdma_alloc_mr(struct ib_pd pd, enum ib_mr_type mr_type,
409	u32 max_num_sg, struct ib_udata *udata);	409	u32 max_num_sg);
410	int pvrdma_map_mr_sg(struct ib_mr ibmr, struct scatterlist sg,	410	int pvrdma_map_mr_sg(struct ib_mr ibmr, struct scatterlist sg,
411	int sg_nents, unsigned int *sg_offset);	411	int sg_nents, unsigned int *sg_offset);
412	int pvrdma_create_cq(struct ib_cq ibcq, const struct ib_cq_init_attr attr,	412	int pvrdma_create_cq(struct ib_cq ibcq, const struct ib_cq_init_attr attr,

drivers/infiniband/sw/rdmavt/ah.c+1−2

 /**
  * rvt_create_ah - create an address handle
  * @ibah: the IB address handle
- * @ah_attr: the attributes of the AH
+ * @init_attr: the attributes of the AH
- * @create_flags: create address handle flags (see enum rdma_create_ah_flags)
  * @udata: pointer to user's input output buffer information.
  *
  * This may be called from interrupt context.

drivers/infiniband/sw/rdmavt/mr.c+1−1

@@ -576,7 +576,7 @@ out: · 575 unmodified lines
576	* Return: the memory region on success, otherwise return an errno.	576	* Return: the memory region on success, otherwise return an errno.
577	*/	577	*/
578	struct ib_mr rvt_alloc_mr(struct ib_pd pd, enum ib_mr_type mr_type,	578	struct ib_mr rvt_alloc_mr(struct ib_pd pd, enum ib_mr_type mr_type,
579	u32 max_num_sg, struct ib_udata *udata)	579	u32 max_num_sg)
580	{	580	{
581	struct rvt_mr *mr;	581	struct rvt_mr *mr;
582		582

drivers/infiniband/sw/rdmavt/mr.h+1−1

@@ -71,7 +71,7 @@ struct ib_mr rvt_reg_user_mr(struct ib_pd pd, u64 start, u64 length, · 70 unmodified lines
71	struct ib_udata *udata);	71	struct ib_udata *udata);
72	int rvt_dereg_mr(struct ib_mr ibmr, struct ib_udata udata);	72	int rvt_dereg_mr(struct ib_mr ibmr, struct ib_udata udata);
73	struct ib_mr rvt_alloc_mr(struct ib_pd pd, enum ib_mr_type mr_type,	73	struct ib_mr rvt_alloc_mr(struct ib_pd pd, enum ib_mr_type mr_type,
74	u32 max_num_sg, struct ib_udata *udata);	74	u32 max_num_sg);
75	int rvt_map_mr_sg(struct ib_mr ibmr, struct scatterlist sg,	75	int rvt_map_mr_sg(struct ib_mr ibmr, struct scatterlist sg,
76	int sg_nents, unsigned int *sg_offset);	76	int sg_nents, unsigned int *sg_offset);
77		77

drivers/infiniband/sw/rxe/rxe.c+4−37

 MODULE_DESCRIPTION("Soft RDMA transport");
 MODULE_LICENSE("Dual BSD/GPL");
-/* free resources for all ports on a device */
-static void rxe_cleanup_ports(struct rxe_dev *rxe)
-{
-	kfree(rxe->port.pkey_tbl);
-	rxe->port.pkey_tbl = NULL;
-}
 /* free resources for a rxe device all objects created for this device must
  * have been destroyed
  */
 	rxe_pool_cleanup(&rxe->mc_grp_pool);
 	rxe_pool_cleanup(&rxe->mc_elem_pool);
-	rxe_cleanup_ports(rxe);
 	if (rxe->tfm)
 		crypto_free_shash(rxe->tfm);
 }
 }
 /* initialize port attributes */
-static int rxe_init_port_param(struct rxe_port *port)
+static void rxe_init_port_param(struct rxe_port *port)
 {
 	port->attr.state		= IB_PORT_DOWN;
 	port->attr.max_mtu		= IB_MTU_4096;
 	port->attr.phys_state		= RXE_PORT_PHYS_STATE;
 	port->mtu_cap			= ib_mtu_enum_to_int(IB_MTU_256);
 	port->subnet_prefix		= cpu_to_be64(RXE_PORT_SUBNET_PREFIX);
-	return 0;
 }
 /* initialize port state, note IB convention that HCA ports are always
  * numbered from 1
  */
-static int rxe_init_ports(struct rxe_dev *rxe)
+static void rxe_init_ports(struct rxe_dev *rxe)
 {
 	struct rxe_port *port = &rxe->port;
 	rxe_init_port_param(port);
-	if (!port->attr.pkey_tbl_len || !port->attr.gid_tbl_len)
-		return -EINVAL;
-	port->pkey_tbl = kcalloc(port->attr.pkey_tbl_len,
-			sizeof(*port->pkey_tbl), GFP_KERNEL);
-	if (!port->pkey_tbl)
-		return -ENOMEM;
-	port->pkey_tbl[0] = 0xffff;
 	addrconf_addr_eui48((unsigned char *)&port->port_guid,
 			    rxe->ndev->dev_addr);
 	spin_lock_init(&port->port_lock);
-	return 0;
 }
 /* init pools of managed objects */
 	/* init default device parameters */
 	rxe_init_device_param(rxe);
-	err = rxe_init_ports(rxe);
+	rxe_init_ports(rxe);
-	if (err)
-		goto err1;
 	err = rxe_init_pools(rxe);
 	if (err)
-		goto err2;
+		return err;
 	/* init pending mmap list */
 	spin_lock_init(&rxe->mmap_offset_lock);
 	mutex_init(&rxe->usdev_lock);
 	return 0;
-err2:
-	rxe_cleanup_ports(rxe);
-err1:
-	return err;
 }
 void rxe_set_mtu(struct rxe_dev *rxe, unsigned int ndev_mtu)

drivers/infiniband/sw/rxe/rxe_loc.h+2−6

 	from_mem_obj,
 };
-int rxe_mem_init_dma(struct rxe_pd *pd,
+void rxe_mem_init_dma(struct rxe_pd *pd,
-		     int access, struct rxe_mem *mem);
+		      int access, struct rxe_mem *mem);
 int rxe_mem_init_user(struct rxe_pd *pd, u64 start,
 		      u64 length, u64 iova, int access, struct ib_udata *udata,
 int mem_check_range(struct rxe_mem *mem, u64 iova, size_t length);
-int rxe_mem_map_pages(struct rxe_dev *rxe, struct rxe_mem *mem,
-		      u64 *page, int num_pages, u64 iova);
 void rxe_mem_cleanup(struct rxe_pool_entry *arg);
 int advance_dma_data(struct rxe_dma_info *dma, unsigned int length);
 struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av,
 				int paylen, struct rxe_pkt_info *pkt);
 int rxe_prepare(struct rxe_pkt_info *pkt, struct sk_buff *skb, u32 *crc);
-enum rdma_link_layer rxe_link_layer(struct rxe_dev *rxe, unsigned int port_num);
 const char *rxe_parent_name(struct rxe_dev *rxe, unsigned int port_num);
 struct device *rxe_dma_device(struct rxe_dev *rxe);
 int rxe_mcast_add(struct rxe_dev *rxe, union ib_gid *mgid);

drivers/infiniband/sw/rxe/rxe_mr.c+2−48

@@ -144,8 +144,8 @@ err1: · 143 unmodified lines
144	return -ENOMEM;	144	return -ENOMEM;
145	}	145	}
146		146
147	int rxe_mem_init_dma(struct rxe_pd *pd,	147	void rxe_mem_init_dma(struct rxe_pd *pd,
148	int access, struct rxe_mem *mem)	148	int access, struct rxe_mem *mem)
149	{	149	{
150	rxe_mem_init(access, mem);	150	rxe_mem_init(access, mem);
151		151
@@ -153,8 +153,6 @@ int rxe_mem_init_dma(struct rxe_pd *pd, · 1 unmodified line
153	mem->access = access;	153	mem->access = access;
154	mem->state = RXE_MEM_STATE_VALID;	154	mem->state = RXE_MEM_STATE_VALID;
155	mem->type = RXE_MEM_TYPE_DMA;	155	mem->type = RXE_MEM_TYPE_DMA;
156
157	return 0;
158	}	156	}
159		157
160	int rxe_mem_init_user(struct rxe_pd *pd, u64 start,	158	int rxe_mem_init_user(struct rxe_pd *pd, u64 start,
@@ -587,47 +585,3 @@ struct rxe_mem lookup_mem(struct rxe_pd pd, int access, u32 key, · 426 unmodified lines
587		585
588	return mem;	586	return mem;
589	}	587	}
590
591	int rxe_mem_map_pages(struct rxe_dev rxe, struct rxe_mem mem,
592	u64 *page, int num_pages, u64 iova)
593	{
594	int i;
595	int num_buf;
596	int err;
597	struct rxe_map **map;
598	struct rxe_phys_buf *buf;
599	int page_size;
600
601	if (num_pages > mem->max_buf) {
602	err = -EINVAL;
603	goto err1;
604	}
605
606	num_buf = 0;
607	page_size = 1 << mem->page_shift;
608	map = mem->map;
609	buf = map[0]->buf;
610
611	for (i = 0; i < num_pages; i++) {
612	buf->addr = *page++;
613	buf->size = page_size;
614	buf++;
615	num_buf++;
616
617	if (num_buf == RXE_BUF_PER_MAP) {
618	map++;
619	buf = map[0]->buf;
620	num_buf = 0;
621	}
622	}
623
624	mem->iova = iova;
625	mem->va = iova;
626	mem->length = num_pages << mem->page_shift;
627	mem->state = RXE_MEM_STATE_VALID;
628
629	return 0;
630
631	err1:
632	return err;
633	}

drivers/infiniband/sw/rxe/rxe_net.c+0−5

 	return rxe->ndev->name;
 }
-enum rdma_link_layer rxe_link_layer(struct rxe_dev *rxe, unsigned int port_num)
-{
-	return IB_LINK_LAYER_ETHERNET;
-}
 int rxe_net_add(const char *ibdev_name, struct net_device *ndev)
 {
 	int err;

drivers/infiniband/sw/rxe/rxe_param.h+2−2

@@ -100,7 +100,7 @@ enum rxe_device_param { · 99 unmodified lines
100	RXE_MAX_SRQ_SGE = 27,	100	RXE_MAX_SRQ_SGE = 27,
101	RXE_MIN_SRQ_SGE = 1,	101	RXE_MIN_SRQ_SGE = 1,
102	RXE_MAX_FMR_PAGE_LIST_LEN = 512,	102	RXE_MAX_FMR_PAGE_LIST_LEN = 512,
103	RXE_MAX_PKEYS = 64,	103	RXE_MAX_PKEYS = 1,
104	RXE_LOCAL_CA_ACK_DELAY = 15,	104	RXE_LOCAL_CA_ACK_DELAY = 15,
105		105
106	RXE_MAX_UCONTEXT = 512,	106	RXE_MAX_UCONTEXT = 512,
@@ -148,7 +148,7 @@ enum rxe_port_param { · 41 unmodified lines
148	RXE_PORT_INIT_TYPE_REPLY = 0,	148	RXE_PORT_INIT_TYPE_REPLY = 0,
149	RXE_PORT_ACTIVE_WIDTH = IB_WIDTH_1X,	149	RXE_PORT_ACTIVE_WIDTH = IB_WIDTH_1X,
150	RXE_PORT_ACTIVE_SPEED = 1,	150	RXE_PORT_ACTIVE_SPEED = 1,
151	RXE_PORT_PKEY_TBL_LEN = 64,	151	RXE_PORT_PKEY_TBL_LEN = 1,
152	RXE_PORT_PHYS_STATE = IB_PORT_PHYS_STATE_POLLING,	152	RXE_PORT_PHYS_STATE = IB_PORT_PHYS_STATE_POLLING,
153	RXE_PORT_SUBNET_PREFIX = 0xfe80000000000000ULL,	153	RXE_PORT_SUBNET_PREFIX = 0xfe80000000000000ULL,
154	};	154	};

drivers/infiniband/sw/rxe/rxe_recv.c+9−26

 static int check_keys(struct rxe_dev *rxe, struct rxe_pkt_info *pkt,
 		      u32 qpn, struct rxe_qp *qp)
 {
-	int i;
-	int found_pkey = 0;
 	struct rxe_port *port = &rxe->port;
 	u16 pkey = bth_pkey(pkt);
 	pkt->pkey_index = 0;
-	if (qpn == 1) {
+	if (!pkey_match(pkey, IB_DEFAULT_PKEY_FULL)) {
-		for (i = 0; i < port->attr.pkey_tbl_len; i++) {
+		pr_warn_ratelimited("bad pkey = 0x%x\n", pkey);
-			if (pkey_match(pkey, port->pkey_tbl[i])) {
+		set_bad_pkey_cntr(port);
-				pkt->pkey_index = i;
+		goto err1;
-				found_pkey = 1;
-				break;
-			}
-		}
-		if (!found_pkey) {
-			pr_warn_ratelimited("bad pkey = 0x%x\n", pkey);
-			set_bad_pkey_cntr(port);
-			goto err1;
-		}
-	} else {
-		if (unlikely(!pkey_match(pkey,
-					 port->pkey_tbl[qp->attr.pkey_index]
-					))) {
-			pr_warn_ratelimited("bad pkey = 0x%0x\n", pkey);
-			set_bad_pkey_cntr(port);
-			goto err1;
-		}
-		pkt->pkey_index = qp->attr.pkey_index;
 	}
 	if ((qp_type(qp) == IB_QPT_UD || qp_type(qp) == IB_QPT_GSI) &&
 static int rxe_match_dgid(struct rxe_dev *rxe, struct sk_buff *skb)
 {
+	struct rxe_pkt_info *pkt = SKB_TO_PKT(skb);
 	const struct ib_gid_attr *gid_attr;
 	union ib_gid dgid;
 	union ib_gid *pdgid;
+	if (pkt->mask & RXE_LOOPBACK_MASK)
+		return 0;
 	if (skb->protocol == htons(ETH_P_IP)) {
 		ipv6_addr_set_v4mapped(ip_hdr(skb)->daddr,
 				       (struct in6_addr *)&dgid);
 	if (unlikely(skb->len < pkt->offset + RXE_BTH_BYTES))
 		goto drop;
-	if (unlikely(rxe_match_dgid(rxe, skb) < 0)) {
+	if (rxe_match_dgid(rxe, skb) < 0) {
 		pr_warn_ratelimited("failed matching dgid\n");
 		goto drop;
 	}

drivers/infiniband/sw/rxe/rxe_req.c+1−4

 				       struct rxe_pkt_info *pkt)
 {
 	struct rxe_dev		*rxe = to_rdev(qp->ibqp.device);
-	struct rxe_port		*port = &rxe->port;
 	struct sk_buff		*skb;
 	struct rxe_send_wr	*ibwr = &wqe->wr;
 	struct rxe_av		*av;
 			(pkt->mask & (RXE_WRITE_MASK | RXE_IMMDT_MASK)) ==
 			(RXE_WRITE_MASK | RXE_IMMDT_MASK));
-	pkey = (qp_type(qp) == IB_QPT_GSI) ?
+	pkey = IB_DEFAULT_PKEY_FULL;
-		 port->pkey_tbl[ibwr->wr.ud.pkey_index] :
-		 port->pkey_tbl[qp->attr.pkey_index];
 	qp_num = (pkt->mask & RXE_DETH_MASK) ? ibwr->wr.ud.remote_qpn :
 					 qp->attr.dest_qp_num;

drivers/infiniband/sw/rxe/rxe_verbs.c+12−36

@@ -83,22 +83,11 @@ static int rxe_query_port(struct ib_device *dev, · 82 unmodified lines
83	static int rxe_query_pkey(struct ib_device *device,	83	static int rxe_query_pkey(struct ib_device *device,
84	u8 port_num, u16 index, u16 *pkey)	84	u8 port_num, u16 index, u16 *pkey)
85	{	85	{
86	struct rxe_dev *rxe = to_rdev(device);	86	if (index > 0)
87	struct rxe_port *port;	87	return -EINVAL;
88
89	port = &rxe->port;
90
91	if (unlikely(index >= port->attr.pkey_tbl_len)) {
92	dev_warn(device->dev.parent, "invalid index = %d\n",
93	index);
94	goto err1;
95	}
96		88
97	*pkey = port->pkey_tbl[index];	89	*pkey = IB_DEFAULT_PKEY_FULL;
98	return 0;	90	return 0;
99
100	err1:
101	return -EINVAL;
102	}	91	}
103		92
104	static int rxe_modify_device(struct ib_device *dev,	93	static int rxe_modify_device(struct ib_device *dev,
@@ -141,9 +130,7 @@ static int rxe_modify_port(struct ib_device *dev, · 36 unmodified lines
141	static enum rdma_link_layer rxe_get_link_layer(struct ib_device *dev,	130	static enum rdma_link_layer rxe_get_link_layer(struct ib_device *dev,
142	u8 port_num)	131	u8 port_num)
143	{	132	{
144	struct rxe_dev *rxe = to_rdev(dev);	133	return IB_LINK_LAYER_ETHERNET;
145
146	return rxe_link_layer(rxe, port_num);
147	}	134	}
148		135
149	static int rxe_alloc_ucontext(struct ib_ucontext uctx, struct ib_udata udata)	136	static int rxe_alloc_ucontext(struct ib_ucontext uctx, struct ib_udata udata)
@@ -684,6 +671,7 @@ static int rxe_post_send_kernel(struct rxe_qp qp, const struct ib_send_wr wr, · 534 unmodified lines
684	unsigned int mask;	671	unsigned int mask;
685	unsigned int length = 0;	672	unsigned int length = 0;
686	int i;	673	int i;
		674	struct ib_send_wr *next;
687		675
688	while (wr) {	676	while (wr) {
689	mask = wr_opcode_mask(wr->opcode, qp);	677	mask = wr_opcode_mask(wr->opcode, qp);
@@ -700,6 +688,8 @@ static int rxe_post_send_kernel(struct rxe_qp qp, const struct ib_send_wr wr, · 10 unmodified lines
700	break;	688	break;
701	}	689	}
702		690
		691	next = wr->next;
		692
703	length = 0;	693	length = 0;
704	for (i = 0; i < wr->num_sge; i++)	694	for (i = 0; i < wr->num_sge; i++)
705	length += wr->sg_list[i].length;	695	length += wr->sg_list[i].length;
@@ -710,7 +700,7 @@ static int rxe_post_send_kernel(struct rxe_qp qp, const struct ib_send_wr wr, · 4 unmodified lines
710	*bad_wr = wr;	700	*bad_wr = wr;
711	break;	701	break;
712	}	702	}
713	wr = wr->next;	703	wr = next;
714	}	704	}
715		705
716	rxe_run_task(&qp->req.task, 1);	706	rxe_run_task(&qp->req.task, 1);
@@ -901,30 +891,16 @@ static struct ib_mr rxe_get_dma_mr(struct ib_pd ibpd, int access) · 184 unmodified lines
901	struct rxe_dev *rxe = to_rdev(ibpd->device);	891	struct rxe_dev *rxe = to_rdev(ibpd->device);
902	struct rxe_pd *pd = to_rpd(ibpd);	892	struct rxe_pd *pd = to_rpd(ibpd);
903	struct rxe_mem *mr;	893	struct rxe_mem *mr;
904	int err;
905		894
906	mr = rxe_alloc(&rxe->mr_pool);	895	mr = rxe_alloc(&rxe->mr_pool);
907	if (!mr) {	896	if (!mr)
908	err = -ENOMEM;	897	return ERR_PTR(-ENOMEM);
909	goto err1;
910	}
911		898
912	rxe_add_index(mr);	899	rxe_add_index(mr);
913
914	rxe_add_ref(pd);	900	rxe_add_ref(pd);
915		901	rxe_mem_init_dma(pd, access, mr);
916	err = rxe_mem_init_dma(pd, access, mr);
917	if (err)
918	goto err2;
919		902
920	return &mr->ibmr;	903	return &mr->ibmr;
921
922	err2:
923	rxe_drop_ref(pd);
924	rxe_drop_index(mr);
925	rxe_drop_ref(mr);
926	err1:
927	return ERR_PTR(err);
928	}	904	}
929		905
930	static struct ib_mr rxe_reg_user_mr(struct ib_pd ibpd,	906	static struct ib_mr rxe_reg_user_mr(struct ib_pd ibpd,
@@ -975,7 +951,7 @@ static int rxe_dereg_mr(struct ib_mr ibmr, struct ib_udata udata) · 44 unmodified lines
975	}	951	}
976		952
977	static struct ib_mr rxe_alloc_mr(struct ib_pd ibpd, enum ib_mr_type mr_type,	953	static struct ib_mr rxe_alloc_mr(struct ib_pd ibpd, enum ib_mr_type mr_type,
978	u32 max_num_sg, struct ib_udata *udata)	954	u32 max_num_sg)
979	{	955	{
980	struct rxe_dev *rxe = to_rdev(ibpd->device);	956	struct rxe_dev *rxe = to_rdev(ibpd->device);
981	struct rxe_pd *pd = to_rpd(ibpd);	957	struct rxe_pd *pd = to_rpd(ibpd);

drivers/infiniband/sw/rxe/rxe_verbs.h+0−1

@@ -371,7 +371,6 @@ struct rxe_mc_elem { · 370 unmodified lines
371		371
372	struct rxe_port {	372	struct rxe_port {
373	struct ib_port_attr attr;	373	struct ib_port_attr attr;
374	u16 *pkey_tbl;
375	__be64 port_guid;	374	__be64 port_guid;
376	__be64 subnet_prefix;	375	__be64 subnet_prefix;
377	spinlock_t port_lock; /* guard port */	376	spinlock_t port_lock; /* guard port */

drivers/infiniband/sw/siw/siw_main.c+0−1

@@ -289,7 +289,6 @@ static const struct ib_device_ops siw_device_ops = { · 288 unmodified lines
289	.post_srq_recv = siw_post_srq_recv,	289	.post_srq_recv = siw_post_srq_recv,
290	.query_device = siw_query_device,	290	.query_device = siw_query_device,
291	.query_gid = siw_query_gid,	291	.query_gid = siw_query_gid,
292	.query_pkey = siw_query_pkey,
293	.query_port = siw_query_port,	292	.query_port = siw_query_port,
294	.query_qp = siw_query_qp,	293	.query_qp = siw_query_qp,
295	.query_srq = siw_query_srq,	294	.query_srq = siw_query_srq,

drivers/infiniband/sw/siw/siw_verbs.c+1−10

 	attr->active_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
 	attr->phys_state = sdev->state == IB_PORT_ACTIVE ?
 		IB_PORT_PHYS_STATE_LINK_UP : IB_PORT_PHYS_STATE_DISABLED;
-	attr->pkey_tbl_len = 1;
 	attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP;
 	attr->state = sdev->state;
 	/*
 	if (rv)
 		return rv;
-	port_immutable->pkey_tbl_len = attr.pkey_tbl_len;
 	port_immutable->gid_tbl_len = attr.gid_tbl_len;
 	port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
 	return 0;
 }
-int siw_query_pkey(struct ib_device *base_dev, u8 port, u16 idx, u16 *pkey)
-{
-	/* Report the default pkey */
-	*pkey = 0xffff;
-	return 0;
-}
 int siw_query_gid(struct ib_device *base_dev, u8 port, int idx,
 		  union ib_gid *gid)
 {
 }
 struct ib_mr *siw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
-			   u32 max_sge, struct ib_udata *udata)
+			   u32 max_sge)
 {
 	struct siw_device *sdev = to_siw_dev(pd->device);
 	struct siw_mr *mr = NULL;

drivers/infiniband/sw/siw/siw_verbs.h+1−2

 		  struct ib_udata *udata);
 int siw_query_port(struct ib_device *base_dev, u8 port,
 		   struct ib_port_attr *attr);
-int siw_query_pkey(struct ib_device *base_dev, u8 port, u16 idx, u16 *pkey);
 int siw_query_gid(struct ib_device *base_dev, u8 port, int idx,
 		  union ib_gid *gid);
 int siw_alloc_pd(struct ib_pd *base_pd, struct ib_udata *udata);
 struct ib_mr *siw_reg_user_mr(struct ib_pd *base_pd, u64 start, u64 len,
 			      u64 rnic_va, int rights, struct ib_udata *udata);
 struct ib_mr *siw_alloc_mr(struct ib_pd *base_pd, enum ib_mr_type mr_type,
-			   u32 max_sge, struct ib_udata *udata);
+			   u32 max_sge);
 struct ib_mr *siw_get_dma_mr(struct ib_pd *base_pd, int rights);
 int siw_map_mr_sg(struct ib_mr *base_mr, struct scatterlist *sl, int num_sle,
 		  unsigned int *sg_off);

drivers/infiniband/ulp/ipoib/ipoib.h+2−2

@@ -515,7 +515,7 @@ void ipoib_ib_dev_cleanup(struct net_device *dev); · 514 unmodified lines
515		515
516	int ipoib_ib_dev_open_default(struct net_device *dev);	516	int ipoib_ib_dev_open_default(struct net_device *dev);
517	int ipoib_ib_dev_open(struct net_device *dev);	517	int ipoib_ib_dev_open(struct net_device *dev);
518	int ipoib_ib_dev_stop(struct net_device *dev);	518	void ipoib_ib_dev_stop(struct net_device *dev);
519	void ipoib_ib_dev_up(struct net_device *dev);	519	void ipoib_ib_dev_up(struct net_device *dev);
520	void ipoib_ib_dev_down(struct net_device *dev);	520	void ipoib_ib_dev_down(struct net_device *dev);
521	int ipoib_ib_dev_stop_default(struct net_device *dev);	521	int ipoib_ib_dev_stop_default(struct net_device *dev);
@@ -527,7 +527,7 @@ void ipoib_mcast_send(struct net_device dev, u8 daddr, struct sk_buff *skb); · 5 unmodified lines
527		527
528	void ipoib_mcast_restart_task(struct work_struct *work);	528	void ipoib_mcast_restart_task(struct work_struct *work);
529	void ipoib_mcast_start_thread(struct net_device *dev);	529	void ipoib_mcast_start_thread(struct net_device *dev);
530	int ipoib_mcast_stop_thread(struct net_device *dev);	530	void ipoib_mcast_stop_thread(struct net_device *dev);
531		531
532	void ipoib_mcast_dev_down(struct net_device *dev);	532	void ipoib_mcast_dev_down(struct net_device *dev);
533	void ipoib_mcast_dev_flush(struct net_device *dev);	533	void ipoib_mcast_dev_flush(struct net_device *dev);

drivers/infiniband/ulp/ipoib/ipoib_ib.c+29−38

@@ -670,13 +670,12 @@ int ipoib_send(struct net_device dev, struct sk_buff skb, · 669 unmodified lines
670	return rc;	670	return rc;
671	}	671	}
672		672
673	static void __ipoib_reap_ah(struct net_device *dev)	673	static void ipoib_reap_dead_ahs(struct ipoib_dev_priv *priv)
674	{	674	{
675	struct ipoib_dev_priv *priv = ipoib_priv(dev);
676	struct ipoib_ah ah, tah;	675	struct ipoib_ah ah, tah;
677	unsigned long flags;	676	unsigned long flags;
678		677
679	netif_tx_lock_bh(dev);	678	netif_tx_lock_bh(priv->dev);
680	spin_lock_irqsave(&priv->lock, flags);	679	spin_lock_irqsave(&priv->lock, flags);
681		680
682	list_for_each_entry_safe(ah, tah, &priv->dead_ahs, list)	681	list_for_each_entry_safe(ah, tah, &priv->dead_ahs, list)
@@ -687,37 +686,37 @@ static void __ipoib_reap_ah(struct net_device *dev) · 4 unmodified lines
687	}	686	}
688		687
689	spin_unlock_irqrestore(&priv->lock, flags);	688	spin_unlock_irqrestore(&priv->lock, flags);
690	netif_tx_unlock_bh(dev);	689	netif_tx_unlock_bh(priv->dev);
691	}	690	}
692		691
693	void ipoib_reap_ah(struct work_struct *work)	692	void ipoib_reap_ah(struct work_struct *work)
694	{	693	{
695	struct ipoib_dev_priv *priv =	694	struct ipoib_dev_priv *priv =
696	container_of(work, struct ipoib_dev_priv, ah_reap_task.work);	695	container_of(work, struct ipoib_dev_priv, ah_reap_task.work);
697	struct net_device *dev = priv->dev;
698		696
699	__ipoib_reap_ah(dev);	697	ipoib_reap_dead_ahs(priv);
700		698
701	if (!test_bit(IPOIB_STOP_REAPER, &priv->flags))	699	if (!test_bit(IPOIB_STOP_REAPER, &priv->flags))
702	queue_delayed_work(priv->wq, &priv->ah_reap_task,	700	queue_delayed_work(priv->wq, &priv->ah_reap_task,
703	round_jiffies_relative(HZ));	701	round_jiffies_relative(HZ));
704	}	702	}
705		703
706	static void ipoib_flush_ah(struct net_device *dev)	704	static void ipoib_start_ah_reaper(struct ipoib_dev_priv *priv)
707	{	705	{
708	struct ipoib_dev_priv *priv = ipoib_priv(dev);	706	clear_bit(IPOIB_STOP_REAPER, &priv->flags);
709		707	queue_delayed_work(priv->wq, &priv->ah_reap_task,
710	cancel_delayed_work(&priv->ah_reap_task);	708	round_jiffies_relative(HZ));
711	flush_workqueue(priv->wq);
712	ipoib_reap_ah(&priv->ah_reap_task.work);
713	}	709	}
714		710
715	static void ipoib_stop_ah(struct net_device *dev)	711	static void ipoib_stop_ah_reaper(struct ipoib_dev_priv *priv)
716	{	712	{
717	struct ipoib_dev_priv *priv = ipoib_priv(dev);
718
719	set_bit(IPOIB_STOP_REAPER, &priv->flags);	713	set_bit(IPOIB_STOP_REAPER, &priv->flags);
720	ipoib_flush_ah(dev);	714	cancel_delayed_work(&priv->ah_reap_task);
		715	/*
		716	* After ipoib_stop_ah_reaper() we always go through
		717	* ipoib_reap_dead_ahs() which ensures the work is really stopped and
		718	* does a final flush out of the dead_ah's list
		719	*/
721	}	720	}
722		721
723	static int recvs_pending(struct net_device *dev)	722	static int recvs_pending(struct net_device *dev)
@@ -846,18 +845,6 @@ timeout: · 122 unmodified lines
846	return 0;	845	return 0;
847	}	846	}
848		847
849	int ipoib_ib_dev_stop(struct net_device *dev)
850	{
851	struct ipoib_dev_priv *priv = ipoib_priv(dev);
852
853	priv->rn_ops->ndo_stop(dev);
854
855	clear_bit(IPOIB_FLAG_INITIALIZED, &priv->flags);
856	ipoib_flush_ah(dev);
857
858	return 0;
859	}
860
861	int ipoib_ib_dev_open_default(struct net_device *dev)	848	int ipoib_ib_dev_open_default(struct net_device *dev)
862	{	849	{
863	struct ipoib_dev_priv *priv = ipoib_priv(dev);	850	struct ipoib_dev_priv *priv = ipoib_priv(dev);
@@ -901,10 +888,7 @@ int ipoib_ib_dev_open(struct net_device *dev) · 37 unmodified lines
901	return -1;	888	return -1;
902	}	889	}
903		890
904	clear_bit(IPOIB_STOP_REAPER, &priv->flags);	891	ipoib_start_ah_reaper(priv);
905	queue_delayed_work(priv->wq, &priv->ah_reap_task,
906	round_jiffies_relative(HZ));
907
908	if (priv->rn_ops->ndo_open(dev)) {	892	if (priv->rn_ops->ndo_open(dev)) {
909	pr_warn("%s: Failed to open dev\n", dev->name);	893	pr_warn("%s: Failed to open dev\n", dev->name);
910	goto dev_stop;	894	goto dev_stop;
@@ -915,13 +899,20 @@ int ipoib_ib_dev_open(struct net_device *dev) · 4 unmodified lines
915	return 0;	899	return 0;
916		900
917	dev_stop:	901	dev_stop:
918	set_bit(IPOIB_STOP_REAPER, &priv->flags);	902	ipoib_stop_ah_reaper(priv);
919	cancel_delayed_work(&priv->ah_reap_task);
920	set_bit(IPOIB_FLAG_INITIALIZED, &priv->flags);
921	ipoib_ib_dev_stop(dev);
922	return -1;	903	return -1;
923	}	904	}
924		905
		906	void ipoib_ib_dev_stop(struct net_device *dev)
		907	{
		908	struct ipoib_dev_priv *priv = ipoib_priv(dev);
		909
		910	priv->rn_ops->ndo_stop(dev);
		911
		912	clear_bit(IPOIB_FLAG_INITIALIZED, &priv->flags);
		913	ipoib_stop_ah_reaper(priv);
		914	}
		915
925	void ipoib_pkey_dev_check_presence(struct net_device *dev)	916	void ipoib_pkey_dev_check_presence(struct net_device *dev)
926	{	917	{
927	struct ipoib_dev_priv *priv = ipoib_priv(dev);	918	struct ipoib_dev_priv *priv = ipoib_priv(dev);
@@ -1232,7 +1223,7 @@ static void __ipoib_ib_dev_flush(struct ipoib_dev_priv *priv, · 304 unmodified lines
1232	ipoib_mcast_dev_flush(dev);	1223	ipoib_mcast_dev_flush(dev);
1233	if (oper_up)	1224	if (oper_up)
1234	set_bit(IPOIB_FLAG_OPER_UP, &priv->flags);	1225	set_bit(IPOIB_FLAG_OPER_UP, &priv->flags);
1235	ipoib_flush_ah(dev);	1226	ipoib_reap_dead_ahs(priv);
1236	}	1227	}
1237		1228
1238	if (level >= IPOIB_FLUSH_NORMAL)	1229	if (level >= IPOIB_FLUSH_NORMAL)
@@ -1307,7 +1298,7 @@ void ipoib_ib_dev_cleanup(struct net_device *dev) · 68 unmodified lines
1307	* the neighbor garbage collection is stopped and reaped.	1298	* the neighbor garbage collection is stopped and reaped.
1308	* That should all be done now, so make a final ah flush.	1299	* That should all be done now, so make a final ah flush.
1309	*/	1300	*/
1310	ipoib_stop_ah(dev);	1301	ipoib_reap_dead_ahs(priv);
1311		1302
1312	clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);	1303	clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
1313		1304

drivers/infiniband/ulp/ipoib/ipoib_main.c+11−2

 	priv->max_ib_mtu = ppriv->max_ib_mtu;
 	set_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags);
-	memcpy(priv->dev->dev_addr, ppriv->dev->dev_addr, INFINIBAND_ALEN);
+	if (memchr_inv(priv->dev->dev_addr, 0, INFINIBAND_ALEN))
-	memcpy(&priv->local_gid, &ppriv->local_gid, sizeof(priv->local_gid));
+		memcpy(&priv->local_gid, priv->dev->dev_addr + 4,
+		       sizeof(priv->local_gid));
+	else {
+		memcpy(priv->dev->dev_addr, ppriv->dev->dev_addr,
+		       INFINIBAND_ALEN);
+		memcpy(&priv->local_gid, &ppriv->local_gid,
+		       sizeof(priv->local_gid));
+	}
 }
 static int ipoib_ndo_init(struct net_device *ndev)
 	/* no more works over the priv->wq */
 	if (priv->wq) {
+		/* See ipoib_mcast_carrier_on_task() */
+		WARN_ON(test_bit(IPOIB_FLAG_OPER_UP, &priv->flags));
 		flush_workqueue(priv->wq);
 		destroy_workqueue(priv->wq);
 		priv->wq = NULL;

drivers/infiniband/ulp/ipoib/ipoib_multicast.c+1−3

 	spin_unlock_irqrestore(&priv->lock, flags);
 }
-int ipoib_mcast_stop_thread(struct net_device *dev)
+void ipoib_mcast_stop_thread(struct net_device *dev)
 {
 	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 	ipoib_dbg_mcast(priv, "stopping multicast thread\n");
 	cancel_delayed_work_sync(&priv->mcast_task);
-	return 0;
 }
 static int ipoib_mcast_leave(struct net_device *dev, struct ipoib_mcast *mcast)

drivers/infiniband/ulp/iser/iscsi_iser.h+6−19

 struct ib_conn;
 struct iscsi_iser_task;
-/**
- * struct iser_comp - iSER completion context
- *
- * @cq:         completion queue
- * @active_qps: Number of active QPs attached
- *              to completion context
- */
-struct iser_comp {
-	struct ib_cq		*cq;
-	int                      active_qps;
-};
 /**
  * struct iser_device - iSER device handle
  *
  * @event_handler: IB events handle routine
  * @ig_list:	   entry in devices list
  * @refcount:      Reference counter, dominated by open iser connections
- * @comps_used:    Number of completion contexts used, Min between online
- *                 cpus and device max completion vectors
- * @comps:         Dinamically allocated array of completion handlers
  */
 struct iser_device {
 	struct ib_device             *ib_device;
 	struct ib_event_handler      event_handler;
 	struct list_head             ig_list;
 	int                          refcount;
-	int			     comps_used;
-	struct iser_comp	     *comps;
 };
 /**
  * @list:           entry in connection fastreg pool
  * @rsc:            data buffer registration resources
  * @sig_protected:  is region protected indicator
+ * @all_list:       first and last list members
  */
 struct iser_fr_desc {
 	struct list_head		  list;
  * @list:                list of fastreg descriptors
  * @lock:                protects fastreg pool
  * @size:                size of the pool
+ * @all_list:            first and last list members
  */
 struct iser_fr_pool {
 	struct list_head        list;
  *
  * @cma_id:              rdma_cm connection maneger handle
  * @qp:                  Connection Queue-pair
+ * @cq:                  Connection completion queue
+ * @cq_size:             The number of max outstanding completions
  * @post_recv_buf_count: post receive counter
  * @sig_count:           send work request signal count
  * @rx_wr:               receive work request for batch posts
  * @device:              reference to iser device
- * @comp:                iser completion context
  * @fr_pool:             connection fast registration poool
  * @pi_support:          Indicate device T10-PI support
  * @reg_cqe:             completion handler
 struct ib_conn {
 	struct rdma_cm_id           *cma_id;
 	struct ib_qp	            *qp;
+	struct ib_cq		    *cq;
+	u32			    cq_size;
 	int                          post_recv_buf_count;
 	u8                           sig_count;
 	struct ib_recv_wr	     rx_wr[ISER_MIN_POSTED_RX];
 	struct iser_device          *device;
-	struct iser_comp	    *comp;
 	struct iser_fr_pool          fr_pool;
 	bool			     pi_support;
 	struct ib_cqe		     reg_cqe;

drivers/infiniband/ulp/iser/iser_verbs.c+25−87

@@ -68,59 +68,23 @@ static void iser_event_handler(struct ib_event_handler *handler, · 67 unmodified lines
68	static int iser_create_device_ib_res(struct iser_device *device)	68	static int iser_create_device_ib_res(struct iser_device *device)
69	{	69	{
70	struct ib_device *ib_dev = device->ib_device;	70	struct ib_device *ib_dev = device->ib_device;
71	int i, max_cqe;
72		71
73	if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) {	72	if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) {
74	iser_err("IB device does not support memory registrations\n");	73	iser_err("IB device does not support memory registrations\n");
75	return -1;	74	return -1;
76	}	75	}
77		76
78	device->comps_used = min_t(int, num_online_cpus(),
79	ib_dev->num_comp_vectors);
80
81	device->comps = kcalloc(device->comps_used, sizeof(*device->comps),
82	GFP_KERNEL);
83	if (!device->comps)
84	goto comps_err;
85
86	max_cqe = min(ISER_MAX_CQ_LEN, ib_dev->attrs.max_cqe);
87
88	iser_info("using %d CQs, device %s supports %d vectors max_cqe %d\n",
89	device->comps_used, dev_name(&ib_dev->dev),
90	ib_dev->num_comp_vectors, max_cqe);
91
92	device->pd = ib_alloc_pd(ib_dev,	77	device->pd = ib_alloc_pd(ib_dev,
93	iser_always_reg ? 0 : IB_PD_UNSAFE_GLOBAL_RKEY);	78	iser_always_reg ? 0 : IB_PD_UNSAFE_GLOBAL_RKEY);
94	if (IS_ERR(device->pd))	79	if (IS_ERR(device->pd))
95	goto pd_err;	80	goto pd_err;
96		81
97	for (i = 0; i < device->comps_used; i++) {
98	struct iser_comp *comp = &device->comps[i];
99
100	comp->cq = ib_alloc_cq(ib_dev, comp, max_cqe, i,
101	IB_POLL_SOFTIRQ);
102	if (IS_ERR(comp->cq)) {
103	comp->cq = NULL;
104	goto cq_err;
105	}
106	}
107
108	INIT_IB_EVENT_HANDLER(&device->event_handler, ib_dev,	82	INIT_IB_EVENT_HANDLER(&device->event_handler, ib_dev,
109	iser_event_handler);	83	iser_event_handler);
110	ib_register_event_handler(&device->event_handler);	84	ib_register_event_handler(&device->event_handler);
111	return 0;	85	return 0;
112		86
113	cq_err:
114	for (i = 0; i < device->comps_used; i++) {
115	struct iser_comp *comp = &device->comps[i];
116
117	if (comp->cq)
118	ib_free_cq(comp->cq);
119	}
120	ib_dealloc_pd(device->pd);
121	pd_err:	87	pd_err:
122	kfree(device->comps);
123	comps_err:
124	iser_err("failed to allocate an IB resource\n");	88	iser_err("failed to allocate an IB resource\n");
125	return -1;	89	return -1;
126	}	90	}
@@ -131,20 +95,9 @@ comps_err: · 4 unmodified lines
131	*/	95	*/
132	static void iser_free_device_ib_res(struct iser_device *device)	96	static void iser_free_device_ib_res(struct iser_device *device)
133	{	97	{
134	int i;
135
136	for (i = 0; i < device->comps_used; i++) {
137	struct iser_comp *comp = &device->comps[i];
138
139	ib_free_cq(comp->cq);
140	comp->cq = NULL;
141	}
142
143	ib_unregister_event_handler(&device->event_handler);	98	ib_unregister_event_handler(&device->event_handler);
144	ib_dealloc_pd(device->pd);	99	ib_dealloc_pd(device->pd);
145		100
146	kfree(device->comps);
147	device->comps = NULL;
148	device->pd = NULL;	101	device->pd = NULL;
149	}	102	}
150		103
@@ -287,70 +240,57 @@ static int iser_create_ib_conn_res(struct ib_conn *ib_conn) · 136 unmodified lines
287	struct ib_device *ib_dev;	240	struct ib_device *ib_dev;
288	struct ib_qp_init_attr init_attr;	241	struct ib_qp_init_attr init_attr;
289	int ret = -ENOMEM;	242	int ret = -ENOMEM;
290	int index, min_index = 0;	243	unsigned int max_send_wr, cq_size;
291		244
292	BUG_ON(ib_conn->device == NULL);	245	BUG_ON(ib_conn->device == NULL);
293		246
294	device = ib_conn->device;	247	device = ib_conn->device;
295	ib_dev = device->ib_device;	248	ib_dev = device->ib_device;
296		249
297	memset(&init_attr, 0, sizeof init_attr);	250	if (ib_conn->pi_support)
		251	max_send_wr = ISER_QP_SIG_MAX_REQ_DTOS + 1;
		252	else
		253	max_send_wr = ISER_QP_MAX_REQ_DTOS + 1;
		254	max_send_wr = min_t(unsigned int, max_send_wr,
		255	(unsigned int)ib_dev->attrs.max_qp_wr);
298		256
299	mutex_lock(&ig.connlist_mutex);	257	cq_size = max_send_wr + ISER_QP_MAX_RECV_DTOS;
300	/* select the CQ with the minimal number of usages */	258	ib_conn->cq = ib_cq_pool_get(ib_dev, cq_size, -1, IB_POLL_SOFTIRQ);
301	for (index = 0; index < device->comps_used; index++) {	259	if (IS_ERR(ib_conn->cq)) {
302	if (device->comps[index].active_qps <	260	ret = PTR_ERR(ib_conn->cq);
303	device->comps[min_index].active_qps)	261	goto cq_err;
304	min_index = index;
305	}	262	}
306	ib_conn->comp = &device->comps[min_index];	263	ib_conn->cq_size = cq_size;
307	ib_conn->comp->active_qps++;	264
308	mutex_unlock(&ig.connlist_mutex);	265	memset(&init_attr, 0, sizeof(init_attr));
309	iser_info("cq index %d used for ib_conn %p\n", min_index, ib_conn);
310		266
311	init_attr.event_handler = iser_qp_event_callback;	267	init_attr.event_handler = iser_qp_event_callback;
312	init_attr.qp_context = (void *)ib_conn;	268	init_attr.qp_context = (void *)ib_conn;
313	init_attr.send_cq = ib_conn->comp->cq;	269	init_attr.send_cq = ib_conn->cq;
314	init_attr.recv_cq = ib_conn->comp->cq;	270	init_attr.recv_cq = ib_conn->cq;
315	init_attr.cap.max_recv_wr = ISER_QP_MAX_RECV_DTOS;	271	init_attr.cap.max_recv_wr = ISER_QP_MAX_RECV_DTOS;
316	init_attr.cap.max_send_sge = 2;	272	init_attr.cap.max_send_sge = 2;
317	init_attr.cap.max_recv_sge = 1;	273	init_attr.cap.max_recv_sge = 1;
318	init_attr.sq_sig_type = IB_SIGNAL_REQ_WR;	274	init_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
319	init_attr.qp_type = IB_QPT_RC;	275	init_attr.qp_type = IB_QPT_RC;
320	if (ib_conn->pi_support) {	276	init_attr.cap.max_send_wr = max_send_wr;
321	init_attr.cap.max_send_wr = ISER_QP_SIG_MAX_REQ_DTOS + 1;	277	if (ib_conn->pi_support)
322	init_attr.create_flags \|= IB_QP_CREATE_INTEGRITY_EN;	278	init_attr.create_flags \|= IB_QP_CREATE_INTEGRITY_EN;
323	iser_conn->max_cmds =	279	iser_conn->max_cmds = ISER_GET_MAX_XMIT_CMDS(max_send_wr - 1);
324	ISER_GET_MAX_XMIT_CMDS(ISER_QP_SIG_MAX_REQ_DTOS);
325	} else {
326	if (ib_dev->attrs.max_qp_wr > ISER_QP_MAX_REQ_DTOS) {
327	init_attr.cap.max_send_wr = ISER_QP_MAX_REQ_DTOS + 1;
328	iser_conn->max_cmds =
329	ISER_GET_MAX_XMIT_CMDS(ISER_QP_MAX_REQ_DTOS);
330	} else {
331	init_attr.cap.max_send_wr = ib_dev->attrs.max_qp_wr;
332	iser_conn->max_cmds =
333	ISER_GET_MAX_XMIT_CMDS(ib_dev->attrs.max_qp_wr);
334	iser_dbg("device %s supports max_send_wr %d\n",
335	dev_name(&device->ib_device->dev),
336	ib_dev->attrs.max_qp_wr);
337	}
338	}
339		280
340	ret = rdma_create_qp(ib_conn->cma_id, device->pd, &init_attr);	281	ret = rdma_create_qp(ib_conn->cma_id, device->pd, &init_attr);
341	if (ret)	282	if (ret)
342	goto out_err;	283	goto out_err;
343		284
344	ib_conn->qp = ib_conn->cma_id->qp;	285	ib_conn->qp = ib_conn->cma_id->qp;
345	iser_info("setting conn %p cma_id %p qp %p\n",	286	iser_info("setting conn %p cma_id %p qp %p max_send_wr %d\n",
346	ib_conn, ib_conn->cma_id,	287	ib_conn, ib_conn->cma_id,
347	ib_conn->cma_id->qp);	288	ib_conn->cma_id->qp, max_send_wr);
348	return ret;	289	return ret;
349		290
350	out_err:	291	out_err:
351	mutex_lock(&ig.connlist_mutex);	292	ib_cq_pool_put(ib_conn->cq, ib_conn->cq_size);
352	ib_conn->comp->active_qps--;	293	cq_err:
353	mutex_unlock(&ig.connlist_mutex);
354	iser_err("unable to alloc mem or create resource, err %d\n", ret);	294	iser_err("unable to alloc mem or create resource, err %d\n", ret);
355		295
356	return ret;	296	return ret;
@@ -462,10 +402,8 @@ static void iser_free_ib_conn_res(struct iser_conn *iser_conn, · 105 unmodified lines
462	iser_conn, ib_conn->cma_id, ib_conn->qp);	402	iser_conn, ib_conn->cma_id, ib_conn->qp);
463		403
464	if (ib_conn->qp != NULL) {	404	if (ib_conn->qp != NULL) {
465	mutex_lock(&ig.connlist_mutex);
466	ib_conn->comp->active_qps--;
467	mutex_unlock(&ig.connlist_mutex);
468	rdma_destroy_qp(ib_conn->cma_id);	405	rdma_destroy_qp(ib_conn->cma_id);
		406	ib_cq_pool_put(ib_conn->cq, ib_conn->cq_size);
469	ib_conn->qp = NULL;	407	ib_conn->qp = NULL;
470	}	408	}
471		409

drivers/infiniband/ulp/isert/ib_isert.c+36−139

 #include "ib_isert.h"
-#define	ISERT_MAX_CONN		8
-#define ISER_MAX_RX_CQ_LEN	(ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN)
-#define ISER_MAX_TX_CQ_LEN \
-	((ISERT_QP_MAX_REQ_DTOS + ISCSI_DEF_XMIT_CMDS_MAX) * ISERT_MAX_CONN)
-#define ISER_MAX_CQ_LEN		(ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \
-				 ISERT_MAX_CONN)
 static int isert_debug_level;
 module_param_named(debug_level, isert_debug_level, int, 0644);
 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)");
 	}
 }
-static struct isert_comp *
-isert_comp_get(struct isert_conn *isert_conn)
-{
-	struct isert_device *device = isert_conn->device;
-	struct isert_comp *comp;
-	int i, min = 0;
-	mutex_lock(&device_list_mutex);
-	for (i = 0; i < device->comps_used; i++)
-		if (device->comps[i].active_qps <
-		    device->comps[min].active_qps)
-			min = i;
-	comp = &device->comps[min];
-	comp->active_qps++;
-	mutex_unlock(&device_list_mutex);
-	isert_info("conn %p, using comp %p min_index: %d\n",
-		   isert_conn, comp, min);
-	return comp;
-}
-static void
-isert_comp_put(struct isert_comp *comp)
-{
-	mutex_lock(&device_list_mutex);
-	comp->active_qps--;
-	mutex_unlock(&device_list_mutex);
-}
 static struct ib_qp *
 isert_create_qp(struct isert_conn *isert_conn,
-		struct isert_comp *comp,
 		struct rdma_cm_id *cma_id)
 {
+	u32 cq_size = ISERT_QP_MAX_REQ_DTOS + ISERT_QP_MAX_RECV_DTOS + 2;
 	struct isert_device *device = isert_conn->device;
+	struct ib_device *ib_dev = device->ib_device;
 	struct ib_qp_init_attr attr;
-	int ret;
+	int ret, factor;
+	isert_conn->cq = ib_cq_pool_get(ib_dev, cq_size, -1, IB_POLL_WORKQUEUE);
+	if (IS_ERR(isert_conn->cq)) {
+		isert_err("Unable to allocate cq\n");
+		ret = PTR_ERR(isert_conn->cq);
+		return ERR_PTR(ret);
+	}
+	isert_conn->cq_size = cq_size;
 	memset(&attr, 0, sizeof(struct ib_qp_init_attr));
 	attr.event_handler = isert_qp_event_callback;
 	attr.qp_context = isert_conn;
-	attr.send_cq = comp->cq;
+	attr.send_cq = isert_conn->cq;
-	attr.recv_cq = comp->cq;
+	attr.recv_cq = isert_conn->cq;
 	attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS + 1;
 	attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1;
-	attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX;
+	factor = rdma_rw_mr_factor(device->ib_device, cma_id->port_num,
+				   ISCSI_ISER_MAX_SG_TABLESIZE);
+	attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX * factor;
 	attr.cap.max_send_sge = device->ib_device->attrs.max_send_sge;
 	attr.cap.max_recv_sge = 1;
 	attr.sq_sig_type = IB_SIGNAL_REQ_WR;
 	ret = rdma_create_qp(cma_id, device->pd, &attr);
 	if (ret) {
 		isert_err("rdma_create_qp failed for cma_id %d\n", ret);
+		ib_cq_pool_put(isert_conn->cq, isert_conn->cq_size);
 		return ERR_PTR(ret);
 	}
 	return cma_id->qp;
 }
-static int
-isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id)
-{
-	struct isert_comp *comp;
-	int ret;
-	comp = isert_comp_get(isert_conn);
-	isert_conn->qp = isert_create_qp(isert_conn, comp, cma_id);
-	if (IS_ERR(isert_conn->qp)) {
-		ret = PTR_ERR(isert_conn->qp);
-		goto err;
-	}
-	return 0;
-err:
-	isert_comp_put(comp);
-	return ret;
-}
 static int
 isert_alloc_rx_descriptors(struct isert_conn *isert_conn)
 {
 	isert_conn->rx_descs = NULL;
 }
-static void
-isert_free_comps(struct isert_device *device)
-{
-	int i;
-	for (i = 0; i < device->comps_used; i++) {
-		struct isert_comp *comp = &device->comps[i];
-		if (comp->cq)
-			ib_free_cq(comp->cq);
-	}
-	kfree(device->comps);
-}
-static int
-isert_alloc_comps(struct isert_device *device)
-{
-	int i, max_cqe, ret = 0;
-	device->comps_used = min(ISERT_MAX_CQ, min_t(int, num_online_cpus(),
-				 device->ib_device->num_comp_vectors));
-	isert_info("Using %d CQs, %s supports %d vectors support "
-		   "pi_capable %d\n",
-		   device->comps_used, dev_name(&device->ib_device->dev),
-		   device->ib_device->num_comp_vectors,
-		   device->pi_capable);
-	device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp),
-				GFP_KERNEL);
-	if (!device->comps)
-		return -ENOMEM;
-	max_cqe = min(ISER_MAX_CQ_LEN, device->ib_device->attrs.max_cqe);
-	for (i = 0; i < device->comps_used; i++) {
-		struct isert_comp *comp = &device->comps[i];
-		comp->device = device;
-		comp->cq = ib_alloc_cq(device->ib_device, comp, max_cqe, i,
-				IB_POLL_WORKQUEUE);
-		if (IS_ERR(comp->cq)) {
-			isert_err("Unable to allocate cq\n");
-			ret = PTR_ERR(comp->cq);
-			comp->cq = NULL;
-			goto out_cq;
-		}
-	}
-	return 0;
-out_cq:
-	isert_free_comps(device);
-	return ret;
-}
 static int
 isert_create_device_ib_res(struct isert_device *device)
 {
 		  ib_dev->attrs.max_send_sge, ib_dev->attrs.max_recv_sge);
 	isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd);
-	ret = isert_alloc_comps(device);
-	if (ret)
-		goto out;
 	device->pd = ib_alloc_pd(ib_dev, 0);
 	if (IS_ERR(device->pd)) {
 		ret = PTR_ERR(device->pd);
 		isert_err("failed to allocate pd, device %p, ret=%d\n",
 			  device, ret);
-		goto out_cq;
+		return ret;
 	}
 	/* Check signature cap */
 			     IB_DEVICE_INTEGRITY_HANDOVER ? true : false;
 	return 0;
-out_cq:
-	isert_free_comps(device);
-out:
-	if (ret > 0)
-		ret = -EINVAL;
-	return ret;
 }
 static void
 	isert_info("device %p\n", device);
 	ib_dealloc_pd(device->pd);
-	isert_free_comps(device);
 }
 static void
 	}
 }
+static void
+isert_destroy_qp(struct isert_conn *isert_conn)
+{
+	ib_destroy_qp(isert_conn->qp);
+	ib_cq_pool_put(isert_conn->cq, isert_conn->cq_size);
+}
 static int
 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
 {
 	isert_set_nego_params(isert_conn, &event->param.conn);
-	ret = isert_conn_setup_qp(isert_conn, cma_id);
+	isert_conn->qp = isert_create_qp(isert_conn, cma_id);
-	if (ret)
+	if (IS_ERR(isert_conn->qp)) {
+		ret = PTR_ERR(isert_conn->qp);
 		goto out_conn_dev;
+	}
 	ret = isert_login_post_recv(isert_conn);
 	if (ret)
-		goto out_conn_dev;
+		goto out_destroy_qp;
 	ret = isert_rdma_accept(isert_conn);
 	if (ret)
-		goto out_conn_dev;
+		goto out_destroy_qp;
 	mutex_lock(&isert_np->mutex);
 	list_add_tail(&isert_conn->node, &isert_np->accepted);
 	return 0;
+out_destroy_qp:
+	isert_destroy_qp(isert_conn);
 out_conn_dev:
 	isert_device_put(device);
 out_rsp_dma_map:
 	    !isert_conn->dev_removed)
 		rdma_destroy_id(isert_conn->cm_id);
-	if (isert_conn->qp) {
+	if (isert_conn->qp)
-		struct isert_comp *comp = isert_conn->qp->recv_cq->cq_context;
+		isert_destroy_qp(isert_conn);
-		isert_comp_put(comp);
-		ib_destroy_qp(isert_conn->qp);
-	}
 	if (isert_conn->login_req_buf)
 		isert_free_login_buf(isert_conn);

drivers/infiniband/ulp/isert/ib_isert.h+4−17

 		(ISER_RX_PAYLOAD_SIZE + sizeof(u64) + sizeof(struct ib_sge) + \
 		 sizeof(struct ib_cqe) + sizeof(bool)))
-#define ISCSI_ISER_SG_TABLESIZE		256
+/* Maximum support is 16MB I/O size */
+#define ISCSI_ISER_MAX_SG_TABLESIZE	4096
 enum isert_desc_type {
 	ISCSI_TX_CONTROL,
 	struct iser_tx_desc	login_tx_desc;
 	struct rdma_cm_id	*cm_id;
 	struct ib_qp		*qp;
+	struct ib_cq		*cq;
+	u32			cq_size;
 	struct isert_device	*device;
 	struct mutex		mutex;
 	struct kref		kref;
 	bool			dev_removed;
 };
-#define ISERT_MAX_CQ 64
-/**
- * struct isert_comp - iSER completion context
- *
- * @device:     pointer to device handle
- * @cq:         completion queue
- * @active_qps: Number of active QPs attached
- *              to completion context
- */
-struct isert_comp {
-	struct isert_device     *device;
-	struct ib_cq		*cq;
-	int                      active_qps;
-};
 struct isert_device {
 	bool			pi_capable;
 	int			refcount;

drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.h+23−0

  * struct opa_vesw_info - OPA vnic switch information
  * @fabric_id: 10-bit fabric id
  * @vesw_id: 12-bit virtual ethernet switch id
+ * @rsvd0: reserved bytes
  * @def_port_mask: bitmask of default ports
+ * @rsvd1: reserved bytes
  * @pkey: partition key
+ * @rsvd2: reserved bytes
  * @u_mcast_dlid: unknown multicast dlid
  * @u_ucast_dlid: array of unknown unicast dlids
+ * @rsvd3: reserved bytes
  * @rc: routing control
  * @eth_mtu: Ethernet MTU
+ * @rsvd4: reserved bytes
  */
 struct opa_vesw_info {
 	__be16  fabric_id;
  * struct opa_per_veswport_info - OPA vnic per port information
  * @port_num: port number
  * @eth_link_status: current ethernet link state
+ * @rsvd0: reserved bytes
  * @base_mac_addr: base mac address
  * @config_state: configured port state
  * @oper_state: operational port state
  * @max_mac_tbl_ent: max number of mac table entries
  * @max_smac_ent: max smac entries in mac table
  * @mac_tbl_digest: mac table digest
+ * @rsvd1: reserved bytes
  * @encap_slid: base slid for the port
  * @pcp_to_sc_uc: sc by pcp index for unicast ethernet packets
  * @pcp_to_vl_uc: vl by pcp index for unicast ethernet packets
  * @non_vlan_vl_uc: vl for non-vlan unicast ethernet packets
  * @non_vlan_sc_mc: sc for non-vlan multicast ethernet packets
  * @non_vlan_vl_mc: vl for non-vlan multicast ethernet packets
+ * @rsvd2: reserved bytes
  * @uc_macs_gen_count: generation count for unicast macs list
  * @mc_macs_gen_count: generation count for multicast macs list
+ * @rsvd3: reserved bytes
  */
 struct opa_per_veswport_info {
 	__be32  port_num;
  * @rx_512_1023: received packet length is >=512 and < 1023 bytes
  * @rx_1024_1518: received packet length is >=1024 and < 1518 bytes
  * @rx_1519_max: received packet length >= 1519 bytes
+ * @reserved: reserved bytes
  *
  * All the above are counters of corresponding conditions.
  */
  * @veswport_num: virtual ethernet switch port number
  * @tx_errors: transmit errors
  * @rx_errors: receive errors
+ * @rsvd0: reserved bytes
  * @tx_smac_filt: smac filter errors
+ * @rsvd1: reserved bytes
+ * @rsvd2: reserved bytes
+ * @rsvd3: reserved bytes
  * @tx_dlid_zero: transmit packets with invalid dlid
+ * @rsvd4: reserved bytes
  * @tx_logic: other transmit errors
+ * @rsvd5: reserved bytes
  * @tx_drop_state: packet tansmission in non-forward port state
  * @rx_bad_veswid: received packet with invalid vesw id
+ * @rsvd6: reserved bytes
  * @rx_runt: received ethernet packet with length < 64 bytes
  * @rx_oversize: received ethernet packet with length > MTU size
+ * @rsvd7: reserved bytes
  * @rx_eth_down: received packets when interface is down
  * @rx_drop_state: received packets in non-forwarding port state
  * @rx_logic: other receive errors
+ * @rsvd8: reserved bytes
+ * @rsvd9: reserved bytes
  *
  * All the above are counters of corresponding error conditions.
  */
  * struct opa_vnic_vema_mad - Generic VEMA MAD
  * @mad_hdr: Generic MAD header
  * @rmpp_hdr: RMPP header for vendor specific MADs
+ * @reserved: reserved bytes
  * @oui: Unique org identifier
  * @data: MAD data
  */
  * @trap_num: Trap number
  * @toggle_count: Notice toggle bit and count value
  * @issuer_lid: Trap issuer's lid
+ * @reserved: reserved bytes
  * @issuer_gid: Issuer GID (only if Report method)
  * @raw_data: Trap message body
  */
  * struct opa_vnic_vema_mad_trap - Generic VEMA MAD Trap
  * @mad_hdr: Generic MAD header
  * @rmpp_hdr: RMPP header for vendor specific MADs
+ * @reserved: reserved bytes
  * @oui: Unique org identifier
  * @notice: Notice structure
  */

drivers/infiniband/ulp/rtrs/rtrs-clt.c+13−3

 #include <linux/module.h>
 #include <linux/rculist.h>
+#include <linux/random.h>
 #include "rtrs-clt.h"
 #include "rtrs-log.h"
  * leads to "false positives" failed reconnect attempts
  */
 #define RTRS_RECONNECT_BACKOFF 1000
+/*
+ * Wait for additional random time between 0 and 8 seconds
+ * before starting to reconnect to avoid clients reconnecting
+ * all at once in case of a major network outage
+ */
+#define RTRS_RECONNECT_SEED 8
 MODULE_DESCRIPTION("RDMA Transport Client");
 MODULE_LICENSE("GPL");
 		 */
 		delay_ms = clt->reconnect_delay_sec * 1000;
 		queue_delayed_work(rtrs_wq, &sess->reconnect_dwork,
-				   msecs_to_jiffies(delay_ms));
+				   msecs_to_jiffies(delay_ms +
+						    prandom_u32() % RTRS_RECONNECT_SEED));
 	} else {
 		/*
 		 * Error can happen just on establishing new connection,
 		sess->stats->reconnects.fail_cnt++;
 		delay_ms = clt->reconnect_delay_sec * 1000;
 		queue_delayed_work(rtrs_wq, &sess->reconnect_dwork,
-				   msecs_to_jiffies(delay_ms));
+				   msecs_to_jiffies(delay_ms +
+						    prandom_u32() %
+						    RTRS_RECONNECT_SEED));
 	}
 }
 		pr_err("Failed to create rtrs-client dev class\n");
 		return PTR_ERR(rtrs_clt_dev_class);
 	}
-	rtrs_wq = alloc_workqueue("rtrs_client_wq", WQ_MEM_RECLAIM, 0);
+	rtrs_wq = alloc_workqueue("rtrs_client_wq", 0, 0);
 	if (!rtrs_wq) {
 		class_destroy(rtrs_clt_dev_class);
 		return -ENOMEM;

drivers/infiniband/ulp/rtrs/rtrs-srv.c+1−1

@@ -2150,7 +2150,7 @@ static int __init rtrs_server_init(void) · 2149 unmodified lines
2150	err = PTR_ERR(rtrs_dev_class);	2150	err = PTR_ERR(rtrs_dev_class);
2151	goto out_chunk_pool;	2151	goto out_chunk_pool;
2152	}	2152	}
2153	rtrs_wq = alloc_workqueue("rtrs_server_wq", WQ_MEM_RECLAIM, 0);	2153	rtrs_wq = alloc_workqueue("rtrs_server_wq", 0, 0);
2154	if (!rtrs_wq) {	2154	if (!rtrs_wq) {
2155	err = -ENOMEM;	2155	err = -ENOMEM;
2156	goto out_dev_class;	2156	goto out_dev_class;

drivers/infiniband/ulp/srpt/ib_srpt.c+9−11

 static void srpt_zerolength_write_done(struct ib_cq *cq, struct ib_wc *wc)
 {
-	struct srpt_rdma_ch *ch = cq->cq_context;
+	struct srpt_rdma_ch *ch = wc->qp->qp_context;
 	pr_debug("%s-%d wc->status %d\n", ch->sess_name, ch->qp->qp_num,
 		 wc->status);
  */
 static void srpt_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc)
 {
-	struct srpt_rdma_ch *ch = cq->cq_context;
+	struct srpt_rdma_ch *ch = wc->qp->qp_context;
 	struct srpt_send_ioctx *ioctx =
 		container_of(wc->wr_cqe, struct srpt_send_ioctx, rdma_cqe);
 static void srpt_recv_done(struct ib_cq *cq, struct ib_wc *wc)
 {
-	struct srpt_rdma_ch *ch = cq->cq_context;
+	struct srpt_rdma_ch *ch = wc->qp->qp_context;
 	struct srpt_recv_ioctx *ioctx =
 		container_of(wc->wr_cqe, struct srpt_recv_ioctx, ioctx.cqe);
  */
 static void srpt_send_done(struct ib_cq *cq, struct ib_wc *wc)
 {
-	struct srpt_rdma_ch *ch = cq->cq_context;
+	struct srpt_rdma_ch *ch = wc->qp->qp_context;
 	struct srpt_send_ioctx *ioctx =
 		container_of(wc->wr_cqe, struct srpt_send_ioctx, ioctx.cqe);
 	enum srpt_command_state state;
 		goto out;
 retry:
-	ch->cq = ib_alloc_cq_any(sdev->device, ch, ch->rq_size + sq_size,
+	ch->cq = ib_cq_pool_get(sdev->device, ch->rq_size + sq_size, -1,
 				 IB_POLL_WORKQUEUE);
 	if (IS_ERR(ch->cq)) {
 		ret = PTR_ERR(ch->cq);
 		       ch->rq_size + sq_size, ret);
 		goto out;
 	}
+	ch->cq_size = ch->rq_size + sq_size;
 	qp_init->qp_context = (void *)ch;
 	qp_init->event_handler
 		if (retry) {
 			pr_debug("failed to create queue pair with sq_size = %d (%d) - retrying\n",
 				 sq_size, ret);
-			ib_free_cq(ch->cq);
+			ib_cq_pool_put(ch->cq, ch->cq_size);
 			sq_size = max(sq_size / 2, MIN_SRPT_SQ_SIZE);
 			goto retry;
 		} else {
 err_destroy_cq:
 	ch->qp = NULL;
-	ib_free_cq(ch->cq);
+	ib_cq_pool_put(ch->cq, ch->cq_size);
 	goto out;
 }
 static void srpt_destroy_ch_ib(struct srpt_rdma_ch *ch)
 {
 	ib_destroy_qp(ch->qp);
-	ib_free_cq(ch->cq);
+	ib_cq_pool_put(ch->cq, ch->cq_size);
 }
 /**
 	WARN_ON_ONCE(irqs_disabled());
-	if (WARN_ON(!sdev || !req))
-		return -EINVAL;
 	it_iu_len = be32_to_cpu(req->req_it_iu_len);
 	pr_info("Received SRP_LOGIN_REQ with i_port_id %pI6, t_port_id %pI6 and it_iu_len %d on port %d (guid=%pI6); pkey %#04x\n",

drivers/infiniband/ulp/srpt/ib_srpt.h+1−0

@@ -300,6 +300,7 @@ struct srpt_rdma_ch { · 299 unmodified lines
300	} rdma_cm;	300	} rdma_cm;
301	};	301	};
302	struct ib_cq *cq;	302	struct ib_cq *cq;
		303	u32 cq_size;
303	struct ib_cqe zw_cqe;	304	struct ib_cqe zw_cqe;
304	struct rcu_head rcu;	305	struct rcu_head rcu;
305	struct kref kref;	306	struct kref kref;

drivers/net/ethernet/mellanox/mlx5/core/alloc.c+9−2

 }
 EXPORT_SYMBOL_GPL(mlx5_fill_page_array);
-void mlx5_fill_page_frag_array(struct mlx5_frag_buf *buf, __be64 *pas)
+void mlx5_fill_page_frag_array_perm(struct mlx5_frag_buf *buf, __be64 *pas, u8 perm)
 {
 	int i;
+	WARN_ON(perm & 0xfc);
 	for (i = 0; i < buf->npages; i++)
-		pas[i] = cpu_to_be64(buf->frags[i].map);
+		pas[i] = cpu_to_be64(buf->frags[i].map | perm);
+}
+EXPORT_SYMBOL_GPL(mlx5_fill_page_frag_array_perm);
+void mlx5_fill_page_frag_array(struct mlx5_frag_buf *buf, __be64 *pas)
+{
+	mlx5_fill_page_frag_array_perm(buf, pas, 0);
 }
 EXPORT_SYMBOL_GPL(mlx5_fill_page_frag_array);

drivers/net/ethernet/mellanox/mlx5/core/fs_core.c+1−0

@@ -1577,6 +1577,7 @@ static struct mlx5_flow_handle add_rule_fg(struct mlx5_flow_group fg, · 1576 unmodified lines
1577	static bool counter_is_valid(u32 action)	1577	static bool counter_is_valid(u32 action)
1578	{	1578	{
1579	return (action & (MLX5_FLOW_CONTEXT_ACTION_DROP \|	1579	return (action & (MLX5_FLOW_CONTEXT_ACTION_DROP \|
		1580	MLX5_FLOW_CONTEXT_ACTION_ALLOW \|
1580	MLX5_FLOW_CONTEXT_ACTION_FWD_DEST));	1581	MLX5_FLOW_CONTEXT_ACTION_FWD_DEST));
1581	}	1582	}
1582		1583

drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c+7−0

 int mlx5i_create_underlay_qp(struct mlx5e_priv *priv)
 {
+	unsigned char *dev_addr = priv->netdev->dev_addr;
 	u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
 	u32 in[MLX5_ST_SZ_DW(create_qp_in)] = {};
 	struct mlx5i_priv *ipriv = priv->ppriv;
 	void *addr_path;
+	int qpn = 0;
 	int ret = 0;
 	void *qpc;
+	if (MLX5_CAP_GEN(priv->mdev, mkey_by_name)) {
+		qpn = (dev_addr[1] << 16) + (dev_addr[2] << 8) + dev_addr[3];
+		MLX5_SET(create_qp_in, in, input_qpn, qpn);
+	}
 	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
 	MLX5_SET(qpc, qpc, st, MLX5_QP_ST_UD);
 	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);

drivers/net/ethernet/mellanox/mlx5/core/main.c+3−0

@@ -557,6 +557,9 @@ static int handle_hca_cap(struct mlx5_core_dev dev, void set_ctx) · 556 unmodified lines
557	if (MLX5_CAP_GEN_MAX(dev, release_all_pages))	557	if (MLX5_CAP_GEN_MAX(dev, release_all_pages))
558	MLX5_SET(cmd_hca_cap, set_hca_cap, release_all_pages, 1);	558	MLX5_SET(cmd_hca_cap, set_hca_cap, release_all_pages, 1);
559		559
		560	if (MLX5_CAP_GEN_MAX(dev, mkey_by_name))
		561	MLX5_SET(cmd_hca_cap, set_hca_cap, mkey_by_name, 1);
		562
560	return set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE);	563	return set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE);
561	}	564	}
562		565

include/linux/mlx5/device.h+5−4

 	MLX5_MKEY_MASK_RW		= 1ull << 20,
 	MLX5_MKEY_MASK_A		= 1ull << 21,
 	MLX5_MKEY_MASK_SMALL_FENCE	= 1ull << 23,
-	MLX5_MKEY_MASK_FREE		= 1ull << 29,
+	MLX5_MKEY_MASK_RELAXED_ORDERING_WRITE	= 1ull << 25,
+	MLX5_MKEY_MASK_FREE			= 1ull << 29,
+	MLX5_MKEY_MASK_RELAXED_ORDERING_READ	= 1ull << 47,
 };
 enum {
 	MLX5_MKEY_REMOTE_INVAL	= 1 << 24,
 	MLX5_MKEY_FLAG_SYNC_UMR = 1 << 29,
 	MLX5_MKEY_BSF_EN	= 1 << 30,
-	MLX5_MKEY_LEN64		= 1 << 31,
 };
 struct mlx5_mkey_seg {
 	MLX5_ADDR_OF(device_event_cap, (mdev)->caps.hca_cur[MLX5_CAP_DEV_EVENT], cap)
 #define MLX5_CAP_DEV_VDPA_EMULATION(mdev, cap)\
-	MLX5_GET(device_virtio_emulation_cap, \
+	MLX5_GET(virtio_emulation_cap, \
 		(mdev)->caps.hca_cur[MLX5_CAP_VDPA_EMULATION], cap)
 #define MLX5_CAP64_DEV_VDPA_EMULATION(mdev, cap)\
-	MLX5_GET64(device_virtio_emulation_cap, \
+	MLX5_GET64(virtio_emulation_cap, \
 		(mdev)->caps.hca_cur[MLX5_CAP_VDPA_EMULATION], cap)
 #define MLX5_CAP_IPSEC(mdev, cap)\

include/linux/mlx5/driver.h+2−0

 void mlx5_unregister_debugfs(void);
 void mlx5_fill_page_array(struct mlx5_frag_buf *buf, __be64 *pas);
+void mlx5_fill_page_frag_array_perm(struct mlx5_frag_buf *buf, __be64 *pas, u8 perm);
 void mlx5_fill_page_frag_array(struct mlx5_frag_buf *frag_buf, __be64 *pas);
 int mlx5_vector2eqn(struct mlx5_core_dev *dev, int vector, int *eqn,
 		    unsigned int *irqn);
 enum {
 	MLX5_INTERFACE_PROTOCOL_IB  = 0,
 	MLX5_INTERFACE_PROTOCOL_ETH = 1,
+	MLX5_INTERFACE_PROTOCOL_VDPA = 2,
 };
 struct mlx5_interface {

include/linux/mlx5/mlx5_ifc.h+113−16

 enum {
 	MLX5_OBJ_TYPE_GENEVE_TLV_OPT = 0x000b,
+	MLX5_OBJ_TYPE_VIRTIO_NET_Q = 0x000d,
 	MLX5_OBJ_TYPE_MKEY = 0xff01,
 	MLX5_OBJ_TYPE_QP = 0xff02,
 	MLX5_OBJ_TYPE_PSV = 0xff03,
 	u8         user_unaffiliated_events[4][0x40];
 };
-struct mlx5_ifc_device_virtio_emulation_cap_bits {
+struct mlx5_ifc_virtio_emulation_cap_bits {
-	u8         reserved_at_0[0x20];
+	u8         desc_tunnel_offload_type[0x1];
+	u8         eth_frame_offload_type[0x1];
+	u8         virtio_version_1_0[0x1];
+	u8         device_features_bits_mask[0xd];
+	u8         event_mode[0x8];
+	u8         virtio_queue_type[0x8];
-	u8         reserved_at_20[0x13];
+	u8         max_tunnel_desc[0x10];
+	u8         reserved_at_30[0x3];
 	u8         log_doorbell_stride[0x5];
 	u8         reserved_at_38[0x3];
 	u8         log_doorbell_bar_size[0x5];
 	u8         doorbell_bar_offset[0x40];
-	u8         reserved_at_80[0x780];
+	u8         max_emulated_devices[0x8];
+	u8         max_num_virtio_queues[0x18];
+	u8         reserved_at_a0[0x60];
+	u8         umem_1_buffer_param_a[0x20];
+	u8         umem_1_buffer_param_b[0x20];
+	u8         umem_2_buffer_param_a[0x20];
+	u8         umem_2_buffer_param_b[0x20];
+	u8         umem_3_buffer_param_a[0x20];
+	u8         umem_3_buffer_param_b[0x20];
+	u8         reserved_at_1c0[0x640];
 };
 enum {
 	u8         max_sgl_for_optimized_performance[0x8];
 	u8         log_max_cq_sz[0x8];
-	u8         reserved_at_d0[0xb];
+	u8         relaxed_ordering_write_umr[0x1];
+	u8         relaxed_ordering_read_umr[0x1];
+	u8         reserved_at_d2[0x7];
+	u8         virtio_net_device_emualtion_manager[0x1];
+	u8         virtio_blk_device_emualtion_manager[0x1];
 	u8         log_max_cq[0x5];
 	u8         log_max_eq_sz[0x8];
 	u8         bf[0x1];
 	u8         driver_version[0x1];
 	u8         pad_tx_eth_packet[0x1];
-	u8         reserved_at_263[0x8];
+	u8         reserved_at_263[0x3];
+	u8         mkey_by_name[0x1];
+	u8         reserved_at_267[0x4];
 	u8         log_bf_reg_size[0x5];
 	u8         reserved_at_270[0x8];
 	struct mlx5_ifc_fpga_cap_bits fpga_cap;
 	struct mlx5_ifc_tls_cap_bits tls_cap;
 	struct mlx5_ifc_device_mem_cap_bits device_mem_cap;
-	struct mlx5_ifc_device_virtio_emulation_cap_bits virtio_emulation_cap;
+	struct mlx5_ifc_virtio_emulation_cap_bits virtio_emulation_cap;
 	u8         reserved_at_0[0x8000];
 };
 };
 struct mlx5_ifc_rqtc_bits {
-	u8         reserved_at_0[0xa0];
+	u8    reserved_at_0[0xa0];
-	u8         reserved_at_a0[0x10];
+	u8    reserved_at_a0[0x5];
-	u8         rqt_max_size[0x10];
+	u8    list_q_type[0x3];
+	u8    reserved_at_a8[0x8];
+	u8    rqt_max_size[0x10];
-	u8         reserved_at_c0[0x10];
+	u8    rq_vhca_id_format[0x1];
-	u8         rqt_actual_size[0x10];
+	u8    reserved_at_c1[0xf];
+	u8    rqt_actual_size[0x10];
-	u8         reserved_at_e0[0x6a0];
+	u8    reserved_at_e0[0x6a0];
 	struct mlx5_ifc_rq_num_bits rq_num[];
 };
 struct mlx5_ifc_destroy_mkey_in_bits {
 	u8         opcode[0x10];
-	u8         reserved_at_10[0x10];
+	u8         uid[0x10];
 	u8         reserved_at_20[0x10];
 	u8         op_mod[0x10];
 	u8         reserved_at_20[0x10];
 	u8         op_mod[0x10];
-	u8         reserved_at_40[0x40];
+	u8         reserved_at_40[0x8];
+	u8         input_qpn[0x18];
+	u8         reserved_at_60[0x20];
 	u8         opt_param_mask[0x20];
 	u8         ece[0x20];
 struct mlx5_ifc_create_mkey_in_bits {
 	u8         opcode[0x10];
-	u8         reserved_at_10[0x10];
+	u8         uid[0x10];
 	u8         reserved_at_20[0x10];
 	u8         op_mod[0x10];
 	struct mlx5_ifc_umem_bits  umem;
 };
+struct mlx5_ifc_create_umem_out_bits {
+	u8         status[0x8];
+	u8         reserved_at_8[0x18];
+	u8         syndrome[0x20];
+	u8         reserved_at_40[0x8];
+	u8         umem_id[0x18];
+	u8         reserved_at_60[0x20];
+};
+struct mlx5_ifc_destroy_umem_in_bits {
+	u8        opcode[0x10];
+	u8        uid[0x10];
+	u8        reserved_at_20[0x10];
+	u8        op_mod[0x10];
+	u8        reserved_at_40[0x8];
+	u8        umem_id[0x18];
+	u8        reserved_at_60[0x20];
+};
+struct mlx5_ifc_destroy_umem_out_bits {
+	u8        status[0x8];
+	u8        reserved_at_8[0x18];
+	u8        syndrome[0x20];
+	u8        reserved_at_40[0x40];
+};
 struct mlx5_ifc_create_uctx_in_bits {
 	u8         opcode[0x10];
 	u8         reserved_at_10[0x10];
 	struct mlx5_ifc_uctx_bits  uctx;
 };
+struct mlx5_ifc_create_uctx_out_bits {
+	u8         status[0x8];
+	u8         reserved_at_8[0x18];
+	u8         syndrome[0x20];
+	u8         reserved_at_40[0x10];
+	u8         uid[0x10];
+	u8         reserved_at_60[0x20];
+};
 struct mlx5_ifc_destroy_uctx_in_bits {
 	u8         opcode[0x10];
 	u8         reserved_at_10[0x10];
 	u8         reserved_at_60[0x20];
 };
+struct mlx5_ifc_destroy_uctx_out_bits {
+	u8         status[0x8];
+	u8         reserved_at_8[0x18];
+	u8         syndrome[0x20];
+	u8          reserved_at_40[0x40];
+};
 struct mlx5_ifc_create_sw_icm_in_bits {
 	struct mlx5_ifc_general_obj_in_cmd_hdr_bits   hdr;
 	struct mlx5_ifc_sw_icm_bits		      sw_icm;
 	u8         hw_offset_record_number[0x18];
 };
+enum {
+	MLX5_MTT_PERM_READ	= 1 << 0,
+	MLX5_MTT_PERM_WRITE	= 1 << 1,
+	MLX5_MTT_PERM_RW	= MLX5_MTT_PERM_READ | MLX5_MTT_PERM_WRITE,
+};
 #endif /* MLX5_IFC_H */

include/rdma/ib.h+2−29

@@ -1,36 +1,9 @@
		1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
1	/*	2	/*
2	* Copyright (c) 2010 Intel Corporation. All rights reserved.	3	* Copyright (c) 2010 Intel Corporation. All rights reserved.
3	*
4	* This software is available to you under a choice of one of two
5	* licenses. You may choose to be licensed under the terms of the GNU
6	* General Public License (GPL) Version 2, available from the file
7	* COPYING in the main directory of this source tree, or the
8	* OpenIB.org BSD license below:
9	*
10	* Redistribution and use in source and binary forms, with or
11	* without modification, are permitted provided that the following
12	* conditions are met:
13	*
14	* - Redistributions of source code must retain the above
15	* copyright notice, this list of conditions and the following
16	* disclaimer.
17	*
18	* - Redistributions in binary form must reproduce the above
19	* copyright notice, this list of conditions and the following
20	* disclaimer in the documentation and/or other materials
21	* provided with the distribution.
22	*
23	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30	* SOFTWARE.
31	*/	4	*/
32		5
33	#if !defined(_RDMA_IB_H)	6	#ifndef _RDMA_IB_H
34	#define _RDMA_IB_H	7	#define _RDMA_IB_H
35		8
36	#include <linux/types.h>	9	#include <linux/types.h>

include/rdma/ib_addr.h+2−29

@@ -1,37 +1,10 @@
		1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
1	/*	2	/*
2	* Copyright (c) 2005 Voltaire Inc. All rights reserved.	3	* Copyright (c) 2005 Voltaire Inc. All rights reserved.
3	* Copyright (c) 2005 Intel Corporation. All rights reserved.	4	* Copyright (c) 2005 Intel Corporation. All rights reserved.
4	*
5	* This software is available to you under a choice of one of two
6	* licenses. You may choose to be licensed under the terms of the GNU
7	* General Public License (GPL) Version 2, available from the file
8	* COPYING in the main directory of this source tree, or the
9	* OpenIB.org BSD license below:
10	*
11	* Redistribution and use in source and binary forms, with or
12	* without modification, are permitted provided that the following
13	* conditions are met:
14	*
15	* - Redistributions of source code must retain the above
16	* copyright notice, this list of conditions and the following
17	* disclaimer.
18	*
19	* - Redistributions in binary form must reproduce the above
20	* copyright notice, this list of conditions and the following
21	* disclaimer in the documentation and/or other materials
22	* provided with the distribution.
23	*
24	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31	* SOFTWARE.
32	*/	5	*/
33		6
34	#if !defined(IB_ADDR_H)	7	#ifndef IB_ADDR_H
35	#define IB_ADDR_H	8	#define IB_ADDR_H
36		9
37	#include <linux/in.h>	10	#include <linux/in.h>

include/rdma/ib_cache.h+1−28

@@ -1,35 +1,8 @@
		1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
1	/*	2	/*
2	* Copyright (c) 2004 Topspin Communications. All rights reserved.	3	* Copyright (c) 2004 Topspin Communications. All rights reserved.
3	* Copyright (c) 2005 Intel Corporation. All rights reserved.	4	* Copyright (c) 2005 Intel Corporation. All rights reserved.
4	* Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.	5	* Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
5	*
6	* This software is available to you under a choice of one of two
7	* licenses. You may choose to be licensed under the terms of the GNU
8	* General Public License (GPL) Version 2, available from the file
9	* COPYING in the main directory of this source tree, or the
10	* OpenIB.org BSD license below:
11	*
12	* Redistribution and use in source and binary forms, with or
13	* without modification, are permitted provided that the following
14	* conditions are met:
15	*
16	* - Redistributions of source code must retain the above
17	* copyright notice, this list of conditions and the following
18	* disclaimer.
19	*
20	* - Redistributions in binary form must reproduce the above
21	* copyright notice, this list of conditions and the following
22	* disclaimer in the documentation and/or other materials
23	* provided with the distribution.
24	*
25	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32	* SOFTWARE.
33	*/	6	*/
34		7
35	#ifndef _IB_CACHE_H	8	#ifndef _IB_CACHE_H

include/rdma/ib_cm.h+1−0

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	* Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.	6	* Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
7	* Copyright (c) 2019, Mellanox Technologies inc. All rights reserved.	7	* Copyright (c) 2019, Mellanox Technologies inc. All rights reserved.
8	*/	8	*/
		9
9	#ifndef IB_CM_H	10	#ifndef IB_CM_H
10	#define IB_CM_H	11	#define IB_CM_H
11		12

include/rdma/ib_hdrs.h+1−43

@@ -1,48 +1,6 @@
		1	/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
1	/*	2	/*
2	* Copyright(c) 2016 - 2018 Intel Corporation.	3	* Copyright(c) 2016 - 2018 Intel Corporation.
3	*
4	* This file is provided under a dual BSD/GPLv2 license. When using or
5	* redistributing this file, you may do so under either license.
6	*
7	* GPL LICENSE SUMMARY
8	*
9	* This program is free software; you can redistribute it and/or modify
10	* it under the terms of version 2 of the GNU General Public License as
11	* published by the Free Software Foundation.
12	*
13	* This program is distributed in the hope that it will be useful, but
14	* WITHOUT ANY WARRANTY; without even the implied warranty of
15	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16	* General Public License for more details.
17	*
18	* BSD LICENSE
19	*
20	* Redistribution and use in source and binary forms, with or without
21	* modification, are permitted provided that the following conditions
22	* are met:
23	*
24	* - Redistributions of source code must retain the above copyright
25	* notice, this list of conditions and the following disclaimer.
26	* - Redistributions in binary form must reproduce the above copyright
27	* notice, this list of conditions and the following disclaimer in
28	* the documentation and/or other materials provided with the
29	* distribution.
30	* - Neither the name of Intel Corporation nor the names of its
31	* contributors may be used to endorse or promote products derived
32	* from this software without specific prior written permission.
33	*
34	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45	*
46	*/	4	*/
47		5
48	#ifndef IB_HDRS_H	6	#ifndef IB_HDRS_H

include/rdma/ib_mad.h+2−29

@@ -1,40 +1,13 @@
		1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
1	/*	2	/*
2	* Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved.	3	* Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved.
3	* Copyright (c) 2004 Infinicon Corporation. All rights reserved.	4	* Copyright (c) 2004 Infinicon Corporation. All rights reserved.
4	* Copyright (c) 2004 Intel Corporation. All rights reserved.	5	* Copyright (c) 2004 Intel Corporation. All rights reserved.
5	* Copyright (c) 2004 Topspin Corporation. All rights reserved.	6	* Copyright (c) 2004 Topspin Corporation. All rights reserved.
6	* Copyright (c) 2004-2006 Voltaire Corporation. All rights reserved.	7	* Copyright (c) 2004-2006 Voltaire Corporation. All rights reserved.
7	*
8	* This software is available to you under a choice of one of two
9	* licenses. You may choose to be licensed under the terms of the GNU
10	* General Public License (GPL) Version 2, available from the file
11	* COPYING in the main directory of this source tree, or the
12	* OpenIB.org BSD license below:
13	*
14	* Redistribution and use in source and binary forms, with or
15	* without modification, are permitted provided that the following
16	* conditions are met:
17	*
18	* - Redistributions of source code must retain the above
19	* copyright notice, this list of conditions and the following
20	* disclaimer.
21	*
22	* - Redistributions in binary form must reproduce the above
23	* copyright notice, this list of conditions and the following
24	* disclaimer in the documentation and/or other materials
25	* provided with the distribution.
26	*
27	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
31	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
32	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
33	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34	* SOFTWARE.
35	*/	8	*/
36		9
37	#if !defined(IB_MAD_H)	10	#ifndef IB_MAD_H
38	#define IB_MAD_H	11	#define IB_MAD_H
39		12
40	#include <linux/list.h>	13	#include <linux/list.h>

include/rdma/ib_marshall.h+2−29

@@ -1,36 +1,9 @@
		1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
1	/*	2	/*
2	* Copyright (c) 2005-2006 Intel Corporation. All rights reserved.	3	* Copyright (c) 2005-2006 Intel Corporation. All rights reserved.
3	*
4	* This software is available to you under a choice of one of two
5	* licenses. You may choose to be licensed under the terms of the GNU
6	* General Public License (GPL) Version 2, available from the file
7	* COPYING in the main directory of this source tree, or the
8	* OpenIB.org BSD license below:
9	*
10	* Redistribution and use in source and binary forms, with or
11	* without modification, are permitted provided that the following
12	* conditions are met:
13	*
14	* - Redistributions of source code must retain the above
15	* copyright notice, this list of conditions and the following
16	* disclaimer.
17	*
18	* - Redistributions in binary form must reproduce the above
19	* copyright notice, this list of conditions and the following
20	* disclaimer in the documentation and/or other materials
21	* provided with the distribution.
22	*
23	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30	* SOFTWARE.
31	*/	4	*/
32		5
33	#if !defined(IB_USER_MARSHALL_H)	6	#ifndef IB_USER_MARSHALL_H
34	#define IB_USER_MARSHALL_H	7	#define IB_USER_MARSHALL_H
35		8
36	#include <rdma/ib_verbs.h>	9	#include <rdma/ib_verbs.h>

include/rdma/ib_pack.h+1−28

@@ -1,33 +1,6 @@
		1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
1	/*	2	/*
2	* Copyright (c) 2004 Topspin Corporation. All rights reserved.	3	* Copyright (c) 2004 Topspin Corporation. All rights reserved.
3	*
4	* This software is available to you under a choice of one of two
5	* licenses. You may choose to be licensed under the terms of the GNU
6	* General Public License (GPL) Version 2, available from the file
7	* COPYING in the main directory of this source tree, or the
8	* OpenIB.org BSD license below:
9	*
10	* Redistribution and use in source and binary forms, with or
11	* without modification, are permitted provided that the following
12	* conditions are met:
13	*
14	* - Redistributions of source code must retain the above
15	* copyright notice, this list of conditions and the following
16	* disclaimer.
17	*
18	* - Redistributions in binary form must reproduce the above
19	* copyright notice, this list of conditions and the following
20	* disclaimer in the documentation and/or other materials
21	* provided with the distribution.
22	*
23	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30	* SOFTWARE.
31	*/	4	*/
32		5
33	#ifndef IB_PACK_H	6	#ifndef IB_PACK_H

include/rdma/ib_pma.h+2−29

@@ -1,38 +1,11 @@
		1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
1	/*	2	/*
2	* Copyright (c) 2006, 2007, 2008, 2009, 2010 QLogic Corporation.	3	* Copyright (c) 2006, 2007, 2008, 2009, 2010 QLogic Corporation.
3	* All rights reserved.	4	* All rights reserved.
4	* Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.	5	* Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
5	*
6	* This software is available to you under a choice of one of two
7	* licenses. You may choose to be licensed under the terms of the GNU
8	* General Public License (GPL) Version 2, available from the file
9	* COPYING in the main directory of this source tree, or the
10	* OpenIB.org BSD license below:
11	*
12	* Redistribution and use in source and binary forms, with or
13	* without modification, are permitted provided that the following
14	* conditions are met:
15	*
16	* - Redistributions of source code must retain the above
17	* copyright notice, this list of conditions and the following
18	* disclaimer.
19	*
20	* - Redistributions in binary form must reproduce the above
21	* copyright notice, this list of conditions and the following
22	* disclaimer in the documentation and/or other materials
23	* provided with the distribution.
24	*
25	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32	* SOFTWARE.
33	*/	6	*/
34		7
35	#if !defined(IB_PMA_H)	8	#ifndef IB_PMA_H
36	#define IB_PMA_H	9	#define IB_PMA_H
37		10
38	#include <rdma/ib_mad.h>	11	#include <rdma/ib_mad.h>

include/rdma/ib_sa.h+1−28

@@ -1,35 +1,8 @@
		1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
1	/*	2	/*
2	* Copyright (c) 2004 Topspin Communications. All rights reserved.	3	* Copyright (c) 2004 Topspin Communications. All rights reserved.
3	* Copyright (c) 2005 Voltaire, Inc. All rights reserved.	4	* Copyright (c) 2005 Voltaire, Inc. All rights reserved.
4	* Copyright (c) 2006 Intel Corporation. All rights reserved.	5	* Copyright (c) 2006 Intel Corporation. All rights reserved.
5	*
6	* This software is available to you under a choice of one of two
7	* licenses. You may choose to be licensed under the terms of the GNU
8	* General Public License (GPL) Version 2, available from the file
9	* COPYING in the main directory of this source tree, or the
10	* OpenIB.org BSD license below:
11	*
12	* Redistribution and use in source and binary forms, with or
13	* without modification, are permitted provided that the following
14	* conditions are met:
15	*
16	* - Redistributions of source code must retain the above
17	* copyright notice, this list of conditions and the following
18	* disclaimer.
19	*
20	* - Redistributions in binary form must reproduce the above
21	* copyright notice, this list of conditions and the following
22	* disclaimer in the documentation and/or other materials
23	* provided with the distribution.
24	*
25	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32	* SOFTWARE.
33	*/	6	*/
34		7
35	#ifndef IB_SA_H	8	#ifndef IB_SA_H

include/rdma/ib_smi.h+2−29

@@ -1,40 +1,13 @@
		1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
1	/*	2	/*
2	* Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved.	3	* Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved.
3	* Copyright (c) 2004 Infinicon Corporation. All rights reserved.	4	* Copyright (c) 2004 Infinicon Corporation. All rights reserved.
4	* Copyright (c) 2004 Intel Corporation. All rights reserved.	5	* Copyright (c) 2004 Intel Corporation. All rights reserved.
5	* Copyright (c) 2004 Topspin Corporation. All rights reserved.	6	* Copyright (c) 2004 Topspin Corporation. All rights reserved.
6	* Copyright (c) 2004 Voltaire Corporation. All rights reserved.	7	* Copyright (c) 2004 Voltaire Corporation. All rights reserved.
7	*
8	* This software is available to you under a choice of one of two
9	* licenses. You may choose to be licensed under the terms of the GNU
10	* General Public License (GPL) Version 2, available from the file
11	* COPYING in the main directory of this source tree, or the
12	* OpenIB.org BSD license below:
13	*
14	* Redistribution and use in source and binary forms, with or
15	* without modification, are permitted provided that the following
16	* conditions are met:
17	*
18	* - Redistributions of source code must retain the above
19	* copyright notice, this list of conditions and the following
20	* disclaimer.
21	*
22	* - Redistributions in binary form must reproduce the above
23	* copyright notice, this list of conditions and the following
24	* disclaimer in the documentation and/or other materials
25	* provided with the distribution.
26	*
27	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
31	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
32	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
33	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34	* SOFTWARE.
35	*/	8	*/
36		9
37	#if !defined(IB_SMI_H)	10	#ifndef IB_SMI_H
38	#define IB_SMI_H	11	#define IB_SMI_H
39		12
40	#include <rdma/ib_mad.h>	13	#include <rdma/ib_mad.h>

include/rdma/ib_umem.h+1−28

@@ -1,33 +1,6 @@
		1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
1	/*	2	/*
2	* Copyright (c) 2007 Cisco Systems. All rights reserved.	3	* Copyright (c) 2007 Cisco Systems. All rights reserved.
3	*
4	* This software is available to you under a choice of one of two
5	* licenses. You may choose to be licensed under the terms of the GNU
6	* General Public License (GPL) Version 2, available from the file
7	* COPYING in the main directory of this source tree, or the
8	* OpenIB.org BSD license below:
9	*
10	* Redistribution and use in source and binary forms, with or
11	* without modification, are permitted provided that the following
12	* conditions are met:
13	*
14	* - Redistributions of source code must retain the above
15	* copyright notice, this list of conditions and the following
16	* disclaimer.
17	*
18	* - Redistributions in binary form must reproduce the above
19	* copyright notice, this list of conditions and the following
20	* disclaimer in the documentation and/or other materials
21	* provided with the distribution.
22	*
23	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30	* SOFTWARE.
31	*/	4	*/
32		5
33	#ifndef IB_UMEM_H	6	#ifndef IB_UMEM_H

include/rdma/ib_umem_odp.h+1−28

@@ -1,33 +1,6 @@
		1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
1	/*	2	/*
2	* Copyright (c) 2014 Mellanox Technologies. All rights reserved.	3	* Copyright (c) 2014 Mellanox Technologies. All rights reserved.
3	*
4	* This software is available to you under a choice of one of two
5	* licenses. You may choose to be licensed under the terms of the GNU
6	* General Public License (GPL) Version 2, available from the file
7	* COPYING in the main directory of this source tree, or the
8	* OpenIB.org BSD license below:
9	*
10	* Redistribution and use in source and binary forms, with or
11	* without modification, are permitted provided that the following
12	* conditions are met:
13	*
14	* - Redistributions of source code must retain the above
15	* copyright notice, this list of conditions and the following
16	* disclaimer.
17	*
18	* - Redistributions in binary form must reproduce the above
19	* copyright notice, this list of conditions and the following
20	* disclaimer in the documentation and/or other materials
21	* provided with the distribution.
22	*
23	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30	* SOFTWARE.
31	*/	4	*/
32		5
33	#ifndef IB_UMEM_ODP_H	6	#ifndef IB_UMEM_ODP_H

include/rdma/ib_verbs.h+31−69

@@ -1,3 +1,4 @@
		1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
1	/*	2	/*
2	* Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved.	3	* Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved.
3	* Copyright (c) 2004 Infinicon Corporation. All rights reserved.	4	* Copyright (c) 2004 Infinicon Corporation. All rights reserved.
@@ -6,37 +7,9 @@ · 2 unmodified lines
6	* Copyright (c) 2004 Voltaire Corporation. All rights reserved.	7	* Copyright (c) 2004 Voltaire Corporation. All rights reserved.
7	* Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.	8	* Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
8	* Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved.	9	* Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved.
9	*
10	* This software is available to you under a choice of one of two
11	* licenses. You may choose to be licensed under the terms of the GNU
12	* General Public License (GPL) Version 2, available from the file
13	* COPYING in the main directory of this source tree, or the
14	* OpenIB.org BSD license below:
15	*
16	* Redistribution and use in source and binary forms, with or
17	* without modification, are permitted provided that the following
18	* conditions are met:
19	*
20	* - Redistributions of source code must retain the above
21	* copyright notice, this list of conditions and the following
22	* disclaimer.
23	*
24	* - Redistributions in binary form must reproduce the above
25	* copyright notice, this list of conditions and the following
26	* disclaimer in the documentation and/or other materials
27	* provided with the distribution.
28	*
29	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
33	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
34	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
35	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36	* SOFTWARE.
37	*/	10	*/
38		11
39	#if !defined(IB_VERBS_H)	12	#ifndef IB_VERBS_H
40	#define IB_VERBS_H	13	#define IB_VERBS_H
41		14
42	#include <linux/types.h>	15	#include <linux/types.h>
@@ -75,6 +48,7 @@ struct ib_umem_odp; · 32 unmodified lines
75	struct ib_uqp_object;	48	struct ib_uqp_object;
76	struct ib_usrq_object;	49	struct ib_usrq_object;
77	struct ib_uwq_object;	50	struct ib_uwq_object;
		51	struct rdma_cm_id;
78		52
79	extern struct workqueue_struct *ib_wq;	53	extern struct workqueue_struct *ib_wq;
80	extern struct workqueue_struct *ib_comp_wq;	54	extern struct workqueue_struct *ib_comp_wq;
@@ -1566,9 +1540,8 @@ struct ib_xrcd { · 1485 unmodified lines
1566	struct ib_device *device;	1540	struct ib_device *device;
1567	atomic_t usecnt; /* count all exposed resources */	1541	atomic_t usecnt; /* count all exposed resources */
1568	struct inode *inode;	1542	struct inode *inode;
1569		1543	struct rw_semaphore tgt_qps_rwsem;
1570	struct mutex tgt_qp_mutex;	1544	struct xarray tgt_qps;
1571	struct list_head tgt_qp_list;
1572	};	1545	};
1573		1546
1574	struct ib_ah {	1547	struct ib_ah {
@@ -2270,6 +2243,7 @@ struct rdma_netdev_alloc_params { · 695 unmodified lines
2270	struct ib_odp_counters {	2243	struct ib_odp_counters {
2271	atomic64_t faults;	2244	atomic64_t faults;
2272	atomic64_t invalidations;	2245	atomic64_t invalidations;
		2246	atomic64_t prefetch;
2273	};	2247	};
2274		2248
2275	struct ib_counters {	2249	struct ib_counters {
@@ -2476,7 +2450,7 @@ struct ib_device_ops { · 200 unmodified lines
2476	struct ib_pd pd, struct ib_udata udata);	2450	struct ib_pd pd, struct ib_udata udata);
2477	int (dereg_mr)(struct ib_mr mr, struct ib_udata *udata);	2451	int (dereg_mr)(struct ib_mr mr, struct ib_udata *udata);
2478	struct ib_mr (alloc_mr)(struct ib_pd *pd, enum ib_mr_type mr_type,	2452	struct ib_mr (alloc_mr)(struct ib_pd *pd, enum ib_mr_type mr_type,
2479	u32 max_num_sg, struct ib_udata *udata);	2453	u32 max_num_sg);
2480	struct ib_mr (alloc_mr_integrity)(struct ib_pd *pd,	2454	struct ib_mr (alloc_mr_integrity)(struct ib_pd *pd,
2481	u32 max_num_data_sg,	2455	u32 max_num_data_sg,
2482	u32 max_num_meta_sg);	2456	u32 max_num_meta_sg);
@@ -2493,9 +2467,8 @@ struct ib_device_ops { · 10 unmodified lines
2493	int (dealloc_mw)(struct ib_mw mw);	2467	int (dealloc_mw)(struct ib_mw mw);
2494	int (attach_mcast)(struct ib_qp qp, union ib_gid *gid, u16 lid);	2468	int (attach_mcast)(struct ib_qp qp, union ib_gid *gid, u16 lid);
2495	int (detach_mcast)(struct ib_qp qp, union ib_gid *gid, u16 lid);	2469	int (detach_mcast)(struct ib_qp qp, union ib_gid *gid, u16 lid);
2496	struct ib_xrcd (alloc_xrcd)(struct ib_device *device,	2470	int (alloc_xrcd)(struct ib_xrcd xrcd, struct ib_udata *udata);
2497	struct ib_udata *udata);	2471	void (dealloc_xrcd)(struct ib_xrcd xrcd, struct ib_udata *udata);
2498	int (dealloc_xrcd)(struct ib_xrcd xrcd, struct ib_udata *udata);
2499	struct ib_flow (create_flow)(struct ib_qp *qp,	2472	struct ib_flow (create_flow)(struct ib_qp *qp,
2500	struct ib_flow_attr *flow_attr,	2473	struct ib_flow_attr *flow_attr,
2501	int domain, struct ib_udata *udata);	2474	int domain, struct ib_udata *udata);
@@ -2539,9 +2512,9 @@ struct ib_device_ops { · 37 unmodified lines
2539	struct ib_mr (reg_dm_mr)(struct ib_pd pd, struct ib_dm dm,	2512	struct ib_mr (reg_dm_mr)(struct ib_pd pd, struct ib_dm dm,
2540	struct ib_dm_mr_attr *attr,	2513	struct ib_dm_mr_attr *attr,
2541	struct uverbs_attr_bundle *attrs);	2514	struct uverbs_attr_bundle *attrs);
2542	struct ib_counters (create_counters)(	2515	int (create_counters)(struct ib_counters counters,
2543	struct ib_device device, struct uverbs_attr_bundle attrs);	2516	struct uverbs_attr_bundle *attrs);
2544	int (destroy_counters)(struct ib_counters counters);	2517	void (destroy_counters)(struct ib_counters counters);
2545	int (read_counters)(struct ib_counters counters,	2518	int (read_counters)(struct ib_counters counters,
2546	struct ib_counters_read_attr *counters_read_attr,	2519	struct ib_counters_read_attr *counters_read_attr,
2547	struct uverbs_attr_bundle *attrs);	2520	struct uverbs_attr_bundle *attrs);
@@ -2581,8 +2554,13 @@ struct ib_device_ops { · 33 unmodified lines
2581	/**	2554	/**
2582	* Allows rdma drivers to add their own restrack attributes.	2555	* Allows rdma drivers to add their own restrack attributes.
2583	*/	2556	*/
2584	int (fill_res_entry)(struct sk_buff msg,	2557	int (fill_res_mr_entry)(struct sk_buff msg, struct ib_mr *ibmr);
2585	struct rdma_restrack_entry *entry);	2558	int (fill_res_mr_entry_raw)(struct sk_buff msg, struct ib_mr *ibmr);
		2559	int (fill_res_cq_entry)(struct sk_buff msg, struct ib_cq *ibcq);
		2560	int (fill_res_cq_entry_raw)(struct sk_buff msg, struct ib_cq *ibcq);
		2561	int (fill_res_qp_entry)(struct sk_buff msg, struct ib_qp *ibqp);
		2562	int (fill_res_qp_entry_raw)(struct sk_buff msg, struct ib_qp *ibqp);
		2563	int (fill_res_cm_id_entry)(struct sk_buff msg, struct rdma_cm_id *id);
2586		2564
2587	/* Device lifecycle callbacks */	2565	/* Device lifecycle callbacks */
2588	/*	2566	/*
@@ -2637,14 +2615,19 @@ struct ib_device_ops { · 48 unmodified lines
2637	* Allows rdma drivers to add their own restrack attributes	2615	* Allows rdma drivers to add their own restrack attributes
2638	* dumped via 'rdma stat' iproute2 command.	2616	* dumped via 'rdma stat' iproute2 command.
2639	*/	2617	*/
2640	int (fill_stat_entry)(struct sk_buff msg,	2618	int (fill_stat_mr_entry)(struct sk_buff msg, struct ib_mr *ibmr);
2641	struct rdma_restrack_entry *entry);	2619
		2620	/* query driver for its ucontext properties */
		2621	int (query_ucontext)(struct ib_ucontext context,
		2622	struct uverbs_attr_bundle *attrs);
2642		2623
2643	DECLARE_RDMA_OBJ_SIZE(ib_ah);	2624	DECLARE_RDMA_OBJ_SIZE(ib_ah);
		2625	DECLARE_RDMA_OBJ_SIZE(ib_counters);
2644	DECLARE_RDMA_OBJ_SIZE(ib_cq);	2626	DECLARE_RDMA_OBJ_SIZE(ib_cq);
2645	DECLARE_RDMA_OBJ_SIZE(ib_pd);	2627	DECLARE_RDMA_OBJ_SIZE(ib_pd);
2646	DECLARE_RDMA_OBJ_SIZE(ib_srq);	2628	DECLARE_RDMA_OBJ_SIZE(ib_srq);
2647	DECLARE_RDMA_OBJ_SIZE(ib_ucontext);	2629	DECLARE_RDMA_OBJ_SIZE(ib_ucontext);
		2630	DECLARE_RDMA_OBJ_SIZE(ib_xrcd);
2648	};	2631	};
2649		2632
2650	struct ib_core_device {	2633	struct ib_core_device {
@@ -4263,14 +4246,8 @@ static inline int ib_dereg_mr(struct ib_mr *mr) · 1612 unmodified lines
4263	return ib_dereg_mr_user(mr, NULL);	4246	return ib_dereg_mr_user(mr, NULL);
4264	}	4247	}
4265		4248
4266	struct ib_mr ib_alloc_mr_user(struct ib_pd pd, enum ib_mr_type mr_type,	4249	struct ib_mr ib_alloc_mr(struct ib_pd pd, enum ib_mr_type mr_type,
4267	u32 max_num_sg, struct ib_udata *udata);	4250	u32 max_num_sg);
4268
4269	static inline struct ib_mr ib_alloc_mr(struct ib_pd pd,
4270	enum ib_mr_type mr_type, u32 max_num_sg)
4271	{
4272	return ib_alloc_mr_user(pd, mr_type, max_num_sg, NULL);
4273	}
4274		4251
4275	struct ib_mr ib_alloc_mr_integrity(struct ib_pd pd,	4252	struct ib_mr ib_alloc_mr_integrity(struct ib_pd pd,
4276	u32 max_num_data_sg,	4253	u32 max_num_data_sg,
@@ -4321,21 +4298,9 @@ int ib_attach_mcast(struct ib_qp qp, union ib_gid gid, u16 lid); · 44 unmodified lines
4321	*/	4298	*/
4322	int ib_detach_mcast(struct ib_qp qp, union ib_gid gid, u16 lid);	4299	int ib_detach_mcast(struct ib_qp qp, union ib_gid gid, u16 lid);
4323		4300
4324	/**	4301	struct ib_xrcd ib_alloc_xrcd_user(struct ib_device device,
4325	* ib_alloc_xrcd - Allocates an XRC domain.	4302	struct inode inode, struct ib_udata udata);
4326	* @device: The device on which to allocate the XRC domain.	4303	int ib_dealloc_xrcd_user(struct ib_xrcd xrcd, struct ib_udata udata);
4327	* @caller: Module name for kernel consumers
4328	*/
4329	struct ib_xrcd __ib_alloc_xrcd(struct ib_device device, const char *caller);
4330	#define ib_alloc_xrcd(device) \
4331	__ib_alloc_xrcd((device), KBUILD_MODNAME)
4332
4333	/**
4334	* ib_dealloc_xrcd - Deallocates an XRC domain.
4335	* @xrcd: The XRC domain to deallocate.
4336	* @udata: Valid user data or NULL for kernel object
4337	*/
4338	int ib_dealloc_xrcd(struct ib_xrcd xrcd, struct ib_udata udata);
4339		4304
4340	static inline int ib_check_mr_access(int flags)	4305	static inline int ib_check_mr_access(int flags)
4341	{	4306	{
@@ -4417,9 +4382,6 @@ struct ib_wq ib_create_wq(struct ib_pd pd, · 75 unmodified lines
4417	int ib_destroy_wq(struct ib_wq wq, struct ib_udata udata);	4382	int ib_destroy_wq(struct ib_wq wq, struct ib_udata udata);
4418	int ib_modify_wq(struct ib_wq wq, struct ib_wq_attr attr,	4383	int ib_modify_wq(struct ib_wq wq, struct ib_wq_attr attr,
4419	u32 wq_attr_mask);	4384	u32 wq_attr_mask);
4420	struct ib_rwq_ind_table ib_create_rwq_ind_table(struct ib_device device,
4421	struct ib_rwq_ind_table_init_attr*
4422	wq_ind_table_init_attr);
4423	int ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table);	4385	int ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table);
4424		4386
4425	int ib_map_mr_sg(struct ib_mr mr, struct scatterlist sg, int sg_nents,	4387	int ib_map_mr_sg(struct ib_mr mr, struct scatterlist sg, int sg_nents,

include/rdma/iw_cm.h+2−28

@@ -1,35 +1,9 @@
		1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
1	/*	2	/*
2	* Copyright (c) 2005 Network Appliance, Inc. All rights reserved.	3	* Copyright (c) 2005 Network Appliance, Inc. All rights reserved.
3	* Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.	4	* Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
4	*
5	* This software is available to you under a choice of one of two
6	* licenses. You may choose to be licensed under the terms of the GNU
7	* General Public License (GPL) Version 2, available from the file
8	* COPYING in the main directory of this source tree, or the
9	* OpenIB.org BSD license below:
10	*
11	* Redistribution and use in source and binary forms, with or
12	* without modification, are permitted provided that the following
13	* conditions are met:
14	*
15	* - Redistributions of source code must retain the above
16	* copyright notice, this list of conditions and the following
17	* disclaimer.
18	*
19	* - Redistributions in binary form must reproduce the above
20	* copyright notice, this list of conditions and the following
21	* disclaimer in the documentation and/or other materials
22	* provided with the distribution.
23	*
24	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31	* SOFTWARE.
32	*/	5	*/
		6
33	#ifndef IW_CM_H	7	#ifndef IW_CM_H
34	#define IW_CM_H	8	#define IW_CM_H
35		9

include/rdma/iw_portmap.h+2−28

@@ -1,35 +1,9 @@
		1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
1	/*	2	/*
2	* Copyright (c) 2014 Intel Corporation. All rights reserved.	3	* Copyright (c) 2014 Intel Corporation. All rights reserved.
3	* Copyright (c) 2014 Chelsio, Inc. All rights reserved.	4	* Copyright (c) 2014 Chelsio, Inc. All rights reserved.
4	*
5	* This software is available to you under a choice of one of two
6	* licenses. You may choose to be licensed under the terms of the GNU
7	* General Public License (GPL) Version 2, available from the file
8	* COPYING in the main directory of this source tree, or the
9	* OpenIB.org BSD license below:
10	*
11	* Redistribution and use in source and binary forms, with or
12	* without modification, are permitted provided that the following
13	* conditions are met:
14	*
15	* - Redistributions of source code must retain the above
16	* copyright notice, this list of conditions and the following
17	* disclaimer.
18	*
19	* - Redistributions in binary form must reproduce the above
20	* copyright notice, this list of conditions and the following
21	* disclaimer in the documentation and/or other materials
22	* provided with the distribution.
23	*
24	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31	* SOFTWARE.
32	*/	5	*/
		6
33	#ifndef _IW_PORTMAP_H	7	#ifndef _IW_PORTMAP_H
34	#define _IW_PORTMAP_H	8	#define _IW_PORTMAP_H
35		9

include/rdma/opa_addr.h+1−43

@@ -1,48 +1,6 @@
		1	/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
1	/*	2	/*
2	* Copyright(c) 2017 Intel Corporation.	3	* Copyright(c) 2017 Intel Corporation.
3	*
4	* This file is provided under a dual BSD/GPLv2 license. When using or
5	* redistributing this file, you may do so under either license.
6	*
7	* GPL LICENSE SUMMARY
8	*
9	* This program is free software; you can redistribute it and/or modify
10	* it under the terms of version 2 of the GNU General Public License as
11	* published by the Free Software Foundation.
12	*
13	* This program is distributed in the hope that it will be useful, but
14	* WITHOUT ANY WARRANTY; without even the implied warranty of
15	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16	* General Public License for more details.
17	*
18	* BSD LICENSE
19	*
20	* Redistribution and use in source and binary forms, with or without
21	* modification, are permitted provided that the following conditions
22	* are met:
23	*
24	* - Redistributions of source code must retain the above copyright
25	* notice, this list of conditions and the following disclaimer.
26	* - Redistributions in binary form must reproduce the above copyright
27	* notice, this list of conditions and the following disclaimer in
28	* the documentation and/or other materials provided with the
29	* distribution.
30	* - Neither the name of Intel Corporation nor the names of its
31	* contributors may be used to endorse or promote products derived
32	* from this software without specific prior written permission.
33	*
34	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45	*
46	*/	4	*/
47		5
48	#ifndef OPA_ADDR_H	6	#ifndef OPA_ADDR_H

include/rdma/opa_port_info.h+2−29

@@ -1,36 +1,9 @@
		1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
1	/*	2	/*
2	* Copyright (c) 2014-2020 Intel Corporation. All rights reserved.	3	* Copyright (c) 2014-2020 Intel Corporation. All rights reserved.
3	*
4	* This software is available to you under a choice of one of two
5	* licenses. You may choose to be licensed under the terms of the GNU
6	* General Public License (GPL) Version 2, available from the file
7	* COPYING in the main directory of this source tree, or the
8	* OpenIB.org BSD license below:
9	*
10	* Redistribution and use in source and binary forms, with or
11	* without modification, are permitted provided that the following
12	* conditions are met:
13	*
14	* - Redistributions of source code must retain the above
15	* copyright notice, this list of conditions and the following
16	* disclaimer.
17	*
18	* - Redistributions in binary form must reproduce the above
19	* copyright notice, this list of conditions and the following
20	* disclaimer in the documentation and/or other materials
21	* provided with the distribution.
22	*
23	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30	* SOFTWARE.
31	*/	4	*/
32		5
33	#if !defined(OPA_PORT_INFO_H)	6	#ifndef OPA_PORT_INFO_H
34	#define OPA_PORT_INFO_H	7	#define OPA_PORT_INFO_H
35		8
36	#include <rdma/opa_smi.h>	9	#include <rdma/opa_smi.h>

include/rdma/opa_smi.h+2−29

@@ -1,36 +1,9 @@
		1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
1	/*	2	/*
2	* Copyright (c) 2014 Intel Corporation. All rights reserved.	3	* Copyright (c) 2014 Intel Corporation. All rights reserved.
3	*
4	* This software is available to you under a choice of one of two
5	* licenses. You may choose to be licensed under the terms of the GNU
6	* General Public License (GPL) Version 2, available from the file
7	* COPYING in the main directory of this source tree, or the
8	* OpenIB.org BSD license below:
9	*
10	* Redistribution and use in source and binary forms, with or
11	* without modification, are permitted provided that the following
12	* conditions are met:
13	*
14	* - Redistributions of source code must retain the above
15	* copyright notice, this list of conditions and the following
16	* disclaimer.
17	*
18	* - Redistributions in binary form must reproduce the above
19	* copyright notice, this list of conditions and the following
20	* disclaimer in the documentation and/or other materials
21	* provided with the distribution.
22	*
23	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30	* SOFTWARE.
31	*/	4	*/
32		5
33	#if !defined(OPA_SMI_H)	6	#ifndef OPA_SMI_H
34	#define OPA_SMI_H	7	#define OPA_SMI_H
35		8
36	#include <rdma/ib_mad.h>	9	#include <rdma/ib_mad.h>

include/rdma/opa_vnic.h+4−45

@@ -1,52 +1,11 @@
1	#ifndef _OPA_VNIC_H	1	/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2	#define _OPA_VNIC_H
3	/*	2	/*
4	* Copyright(c) 2017 - 2020 Intel Corporation.	3	* Copyright(c) 2017 - 2020 Intel Corporation.
5	*
6	* This file is provided under a dual BSD/GPLv2 license. When using or
7	* redistributing this file, you may do so under either license.
8	*
9	* GPL LICENSE SUMMARY
10	*
11	* This program is free software; you can redistribute it and/or modify
12	* it under the terms of version 2 of the GNU General Public License as
13	* published by the Free Software Foundation.
14	*
15	* This program is distributed in the hope that it will be useful, but
16	* WITHOUT ANY WARRANTY; without even the implied warranty of
17	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18	* General Public License for more details.
19	*
20	* BSD LICENSE
21	*
22	* Redistribution and use in source and binary forms, with or without
23	* modification, are permitted provided that the following conditions
24	* are met:
25	*
26	* - Redistributions of source code must retain the above copyright
27	* notice, this list of conditions and the following disclaimer.
28	* - Redistributions in binary form must reproduce the above copyright
29	* notice, this list of conditions and the following disclaimer in
30	* the documentation and/or other materials provided with the
31	* distribution.
32	* - Neither the name of Intel Corporation nor the names of its
33	* contributors may be used to endorse or promote products derived
34	* from this software without specific prior written permission.
35	*
36	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
37	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
38	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
39	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
40	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
41	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
42	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
43	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
44	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
45	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
46	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
47	*
48	*/	4	*/
49		5
		6	#ifndef _OPA_VNIC_H
		7	#define _OPA_VNIC_H
		8
50	/*	9	/*
51	* This file contains Intel Omni-Path (OPA) Virtual Network Interface	10	* This file contains Intel Omni-Path (OPA) Virtual Network Interface
52	* Controller (VNIC) specific declarations.	11	* Controller (VNIC) specific declarations.

include/rdma/rdma_cm.h+2−29

@@ -1,37 +1,10 @@
		1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
1	/*	2	/*
2	* Copyright (c) 2005 Voltaire Inc. All rights reserved.	3	* Copyright (c) 2005 Voltaire Inc. All rights reserved.
3	* Copyright (c) 2005 Intel Corporation. All rights reserved.	4	* Copyright (c) 2005 Intel Corporation. All rights reserved.
4	*
5	* This software is available to you under a choice of one of two
6	* licenses. You may choose to be licensed under the terms of the GNU
7	* General Public License (GPL) Version 2, available from the file
8	* COPYING in the main directory of this source tree, or the
9	* OpenIB.org BSD license below:
10	*
11	* Redistribution and use in source and binary forms, with or
12	* without modification, are permitted provided that the following
13	* conditions are met:
14	*
15	* - Redistributions of source code must retain the above
16	* copyright notice, this list of conditions and the following
17	* disclaimer.
18	*
19	* - Redistributions in binary form must reproduce the above
20	* copyright notice, this list of conditions and the following
21	* disclaimer in the documentation and/or other materials
22	* provided with the distribution.
23	*
24	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31	* SOFTWARE.
32	*/	5	*/
33		6
34	#if !defined(RDMA_CM_H)	7	#ifndef RDMA_CM_H
35	#define RDMA_CM_H	8	#define RDMA_CM_H
36		9
37	#include <linux/socket.h>	10	#include <linux/socket.h>

include/rdma/rdma_cm_ib.h+2−29

@@ -1,36 +1,9 @@
		1	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
1	/*	2	/*
2	* Copyright (c) 2006 Intel Corporation. All rights reserved.	3	* Copyright (c) 2006 Intel Corporation. All rights reserved.
3	*
4	* This software is available to you under a choice of one of two
5	* licenses. You may choose to be licensed under the terms of the GNU
6	* General Public License (GPL) Version 2, available from the file
7	* COPYING in the main directory of this source tree, or the
8	* OpenIB.org BSD license below:
9	*
10	* Redistribution and use in source and binary forms, with or
11	* without modification, are permitted provided that the following
12	* conditions are met:
13	*
14	* - Redistributions of source code must retain the above
15	* copyright notice, this list of conditions and the following
16	* disclaimer.
17	*
18	* - Redistributions in binary form must reproduce the above
19	* copyright notice, this list of conditions and the following
20	* disclaimer in the documentation and/or other materials
21	* provided with the distribution.
22	*
23	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30	* SOFTWARE.
31	*/	4	*/
32		5
33	#if !defined(RDMA_CM_IB_H)	6	#ifndef RDMA_CM_IB_H
34	#define RDMA_CM_IB_H	7	#define RDMA_CM_IB_H
35		8
36	#include <rdma/rdma_cm.h>	9	#include <rdma/rdma_cm.h>

include/rdma/rdma_netlink.h+1−1

@@ -1,8 +1,8 @@
1	/* SPDX-License-Identifier: GPL-2.0 */	1	/* SPDX-License-Identifier: GPL-2.0 */
		2
2	#ifndef _RDMA_NETLINK_H	3	#ifndef _RDMA_NETLINK_H
3	#define _RDMA_NETLINK_H	4	#define _RDMA_NETLINK_H
4		5
5
6	#include <linux/netlink.h>	6	#include <linux/netlink.h>
7	#include <uapi/rdma/rdma_netlink.h>	7	#include <uapi/rdma/rdma_netlink.h>
8		8

include/rdma/rdma_vt.h+4−46

@@ -1,53 +1,11 @@
1	#ifndef DEF_RDMA_VT_H	1	/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2	#define DEF_RDMA_VT_H
3
4	/*	2	/*
5	* Copyright(c) 2016 - 2019 Intel Corporation.	3	* Copyright(c) 2016 - 2019 Intel Corporation.
6	*
7	* This file is provided under a dual BSD/GPLv2 license. When using or
8	* redistributing this file, you may do so under either license.
9	*
10	* GPL LICENSE SUMMARY
11	*
12	* This program is free software; you can redistribute it and/or modify
13	* it under the terms of version 2 of the GNU General Public License as
14	* published by the Free Software Foundation.
15	*
16	* This program is distributed in the hope that it will be useful, but
17	* WITHOUT ANY WARRANTY; without even the implied warranty of
18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19	* General Public License for more details.
20	*
21	* BSD LICENSE
22	*
23	* Redistribution and use in source and binary forms, with or without
24	* modification, are permitted provided that the following conditions
25	* are met:
26	*
27	* - Redistributions of source code must retain the above copyright
28	* notice, this list of conditions and the following disclaimer.