lael-labs/linux

fs/lockd/host.c+2−1

@@ -464,7 +464,8 @@ nlm_bind_host(struct nlm_host *host) · 463 unmodified lines
464	.version = host->h_version,	464	.version = host->h_version,
465	.authflavor = RPC_AUTH_UNIX,	465	.authflavor = RPC_AUTH_UNIX,
466	.flags = (RPC_CLNT_CREATE_NOPING \|	466	.flags = (RPC_CLNT_CREATE_NOPING \|
467	RPC_CLNT_CREATE_AUTOBIND),	467	RPC_CLNT_CREATE_AUTOBIND \|
		468	RPC_CLNT_CREATE_REUSEPORT),
468	.cred = host->h_cred,	469	.cred = host->h_cred,
469	};	470	};
470		471

fs/nfs/callback.h+2−2

@@ -72,8 +72,8 @@ struct cb_getattrres { · 71 unmodified lines
72	uint32_t bitmap[2];	72	uint32_t bitmap[2];
73	uint64_t size;	73	uint64_t size;
74	uint64_t change_attr;	74	uint64_t change_attr;
75	struct timespec ctime;	75	struct timespec64 ctime;
76	struct timespec mtime;	76	struct timespec64 mtime;
77	};	77	};
78		78
79	struct cb_recallargs {	79	struct cb_recallargs {

fs/nfs/callback_proc.c+3−5

 	struct cb_getattrargs *args = argp;
 	struct cb_getattrres *res = resp;
 	struct nfs_delegation *delegation;
-	struct nfs_inode *nfsi;
 	struct inode *inode;
 	res->status = htonl(NFS4ERR_OP_NOT_IN_SESSION);
 				-ntohl(res->status));
 		goto out;
 	}
-	nfsi = NFS_I(inode);
 	rcu_read_lock();
-	delegation = rcu_dereference(nfsi->delegation);
+	delegation = nfs4_get_valid_delegation(inode);
 	if (delegation == NULL || (delegation->type & FMODE_WRITE) == 0)
 		goto out_iput;
 	res->size = i_size_read(inode);
 	res->change_attr = delegation->change_attr;
 	if (nfs_have_writebacks(inode))
 		res->change_attr++;
-	res->ctime = timespec64_to_timespec(inode->i_ctime);
+	res->ctime = inode->i_ctime;
-	res->mtime = timespec64_to_timespec(inode->i_mtime);
+	res->mtime = inode->i_mtime;
 	res->bitmap[0] = (FATTR4_WORD0_CHANGE|FATTR4_WORD0_SIZE) &
 		args->bitmap[0];
 	res->bitmap[1] = (FATTR4_WORD1_TIME_METADATA|FATTR4_WORD1_TIME_MODIFY) &

fs/nfs/callback_xdr.c+3−3

@@ -627,7 +627,7 @@ static __be32 encode_attr_size(struct xdr_stream xdr, const uint32_t bitmap, u · 626 unmodified lines
627	return 0;	627	return 0;
628	}	628	}
629		629
630	static __be32 encode_attr_time(struct xdr_stream xdr, const struct timespec time)	630	static __be32 encode_attr_time(struct xdr_stream xdr, const struct timespec64 time)
631	{	631	{
632	__be32 *p;	632	__be32 *p;
633		633
@@ -639,14 +639,14 @@ static __be32 encode_attr_time(struct xdr_stream xdr, const struct timespec ti · 5 unmodified lines
639	return 0;	639	return 0;
640	}	640	}
641		641
642	static __be32 encode_attr_ctime(struct xdr_stream xdr, const uint32_t bitmap, const struct timespec *time)	642	static __be32 encode_attr_ctime(struct xdr_stream xdr, const uint32_t bitmap, const struct timespec64 *time)
643	{	643	{
644	if (!(bitmap[1] & FATTR4_WORD1_TIME_METADATA))	644	if (!(bitmap[1] & FATTR4_WORD1_TIME_METADATA))
645	return 0;	645	return 0;
646	return encode_attr_time(xdr,time);	646	return encode_attr_time(xdr,time);
647	}	647	}
648		648
649	static __be32 encode_attr_mtime(struct xdr_stream xdr, const uint32_t bitmap, const struct timespec *time)	649	static __be32 encode_attr_mtime(struct xdr_stream xdr, const uint32_t bitmap, const struct timespec64 *time)
650	{	650	{
651	if (!(bitmap[1] & FATTR4_WORD1_TIME_MODIFY))	651	if (!(bitmap[1] & FATTR4_WORD1_TIME_MODIFY))
652	return 0;	652	return 0;

fs/nfs/client.c+11−0

 		/* Match nfsv4 minorversion */
 		if (clp->cl_minorversion != data->minorversion)
 			continue;
+		/* Match request for a dedicated DS */
+		if (test_bit(NFS_CS_DS, &data->init_flags) !=
+		    test_bit(NFS_CS_DS, &clp->cl_flags))
+			continue;
 		/* Match the full socket address */
 		if (!rpc_cmp_addr_port(sap, clap))
 			/* Match all xprt_switch full socket addresses */
 		args.flags |= RPC_CLNT_CREATE_NONPRIVPORT;
 	if (test_bit(NFS_CS_INFINITE_SLOTS, &clp->cl_flags))
 		args.flags |= RPC_CLNT_CREATE_INFINITE_SLOTS;
+	if (test_bit(NFS_CS_NOPING, &clp->cl_flags))
+		args.flags |= RPC_CLNT_CREATE_NOPING;
+	if (test_bit(NFS_CS_REUSEPORT, &clp->cl_flags))
+		args.flags |= RPC_CLNT_CREATE_REUSEPORT;
 	if (!IS_ERR(clp->cl_rpcclient))
 		return 0;
 		.timeparms = &timeparms,
 		.cred = server->cred,
 		.nconnect = data->nfs_server.nconnect,
+		.init_flags = (1UL << NFS_CS_REUSEPORT),
 	};
 	struct nfs_client *clp;
 	int error;

fs/nfs/delegation.c+111−47

@@ -199,7 +199,7 @@ void nfs_inode_reclaim_delegation(struct inode inode, const struct cred cred, · 198 unmodified lines
199	delegation = rcu_dereference(NFS_I(inode)->delegation);	199	delegation = rcu_dereference(NFS_I(inode)->delegation);
200	if (delegation != NULL) {	200	if (delegation != NULL) {
201	spin_lock(&delegation->lock);	201	spin_lock(&delegation->lock);
202	if (delegation->inode != NULL) {	202	if (nfs4_is_valid_delegation(delegation, 0)) {
203	nfs4_stateid_copy(&delegation->stateid, stateid);	203	nfs4_stateid_copy(&delegation->stateid, stateid);
204	delegation->type = type;	204	delegation->type = type;
205	delegation->pagemod_limit = pagemod_limit;	205	delegation->pagemod_limit = pagemod_limit;
@@ -229,7 +229,6 @@ static int nfs_do_return_delegation(struct inode inode, struct nfs_delegation · 23 unmodified lines
229	delegation->cred,	229	delegation->cred,
230	&delegation->stateid,	230	&delegation->stateid,
231	issync);	231	issync);
232	nfs_free_delegation(delegation);
233	return res;	232	return res;
234	}	233	}
235		234
@@ -298,7 +297,10 @@ nfs_detach_delegation_locked(struct nfs_inode *nfsi, · 62 unmodified lines
298	return NULL;	297	return NULL;
299		298
300	spin_lock(&delegation->lock);	299	spin_lock(&delegation->lock);
301	set_bit(NFS_DELEGATION_RETURNING, &delegation->flags);	300	if (!delegation->inode) {
		301	spin_unlock(&delegation->lock);
		302	return NULL;
		303	}
302	list_del_rcu(&delegation->super_list);	304	list_del_rcu(&delegation->super_list);
303	delegation->inode = NULL;	305	delegation->inode = NULL;
304	rcu_assign_pointer(nfsi->delegation, NULL);	306	rcu_assign_pointer(nfsi->delegation, NULL);
@@ -325,10 +327,12 @@ nfs_inode_detach_delegation(struct inode *inode) · 20 unmodified lines
325	struct nfs_server *server = NFS_SERVER(inode);	327	struct nfs_server *server = NFS_SERVER(inode);
326	struct nfs_delegation *delegation;	328	struct nfs_delegation *delegation;
327		329
328	delegation = nfs_start_delegation_return(nfsi);	330	rcu_read_lock();
329	if (delegation == NULL)	331	delegation = rcu_dereference(nfsi->delegation);
330	return NULL;	332	if (delegation != NULL)
331	return nfs_detach_delegation(nfsi, delegation, server);	333	delegation = nfs_detach_delegation(nfsi, delegation, server);
		334	rcu_read_unlock();
		335	return delegation;
332	}	336	}
333		337
334	static void	338	static void
@@ -339,6 +343,7 @@ nfs_update_inplace_delegation(struct nfs_delegation *delegation, · 4 unmodified lines
339	delegation->stateid.seqid = update->stateid.seqid;	343	delegation->stateid.seqid = update->stateid.seqid;
340	smp_wmb();	344	smp_wmb();
341	delegation->type = update->type;	345	delegation->type = update->type;
		346	clear_bit(NFS_DELEGATION_REVOKED, &delegation->flags);
342	}	347	}
343	}	348	}
344		349
@@ -379,14 +384,18 @@ int nfs_inode_set_delegation(struct inode inode, const struct cred cred, · 34 unmodified lines
379	spin_lock(&clp->cl_lock);	384	spin_lock(&clp->cl_lock);
380	old_delegation = rcu_dereference_protected(nfsi->delegation,	385	old_delegation = rcu_dereference_protected(nfsi->delegation,
381	lockdep_is_held(&clp->cl_lock));	386	lockdep_is_held(&clp->cl_lock));
382	if (old_delegation != NULL) {	387	if (old_delegation == NULL)
383	/* Is this an update of the existing delegation? */	388	goto add_new;
384	if (nfs4_stateid_match_other(&old_delegation->stateid,	389	/* Is this an update of the existing delegation? */
385	&delegation->stateid)) {	390	if (nfs4_stateid_match_other(&old_delegation->stateid,
386	nfs_update_inplace_delegation(old_delegation,	391	&delegation->stateid)) {
387	delegation);	392	spin_lock(&old_delegation->lock);
388	goto out;	393	nfs_update_inplace_delegation(old_delegation,
389	}	394	delegation);
		395	spin_unlock(&old_delegation->lock);
		396	goto out;
		397	}
		398	if (!test_bit(NFS_DELEGATION_REVOKED, &old_delegation->flags)) {
390	/*	399	/*
391	* Deal with broken servers that hand out two	400	* Deal with broken servers that hand out two
392	* delegations for the same file.	401	* delegations for the same file.
@@ -405,11 +414,11 @@ int nfs_inode_set_delegation(struct inode inode, const struct cred cred, · 12 unmodified lines
405	if (test_and_set_bit(NFS_DELEGATION_RETURNING,	414	if (test_and_set_bit(NFS_DELEGATION_RETURNING,
406	&old_delegation->flags))	415	&old_delegation->flags))
407	goto out;	416	goto out;
408	freeme = nfs_detach_delegation_locked(nfsi,
409	old_delegation, clp);
410	if (freeme == NULL)
411	goto out;
412	}	417	}
		418	freeme = nfs_detach_delegation_locked(nfsi, old_delegation, clp);
		419	if (freeme == NULL)
		420	goto out;
		421	add_new:
413	list_add_tail_rcu(&delegation->super_list, &server->delegations);	422	list_add_tail_rcu(&delegation->super_list, &server->delegations);
414	rcu_assign_pointer(nfsi->delegation, delegation);	423	rcu_assign_pointer(nfsi->delegation, delegation);
415	delegation = NULL;	424	delegation = NULL;
@@ -424,8 +433,10 @@ out: · 8 unmodified lines
424	spin_unlock(&clp->cl_lock);	433	spin_unlock(&clp->cl_lock);
425	if (delegation != NULL)	434	if (delegation != NULL)
426	nfs_free_delegation(delegation);	435	nfs_free_delegation(delegation);
427	if (freeme != NULL)	436	if (freeme != NULL) {
428	nfs_do_return_delegation(inode, freeme, 0);	437	nfs_do_return_delegation(inode, freeme, 0);
		438	nfs_free_delegation(freeme);
		439	}
429	return status;	440	return status;
430	}	441	}
431		442
@@ -435,7 +446,6 @@ out: · 3 unmodified lines
435	static int nfs_end_delegation_return(struct inode inode, struct nfs_delegation delegation, int issync)	446	static int nfs_end_delegation_return(struct inode inode, struct nfs_delegation delegation, int issync)
436	{	447	{
437	struct nfs_client *clp = NFS_SERVER(inode)->nfs_client;	448	struct nfs_client *clp = NFS_SERVER(inode)->nfs_client;
438	struct nfs_inode *nfsi = NFS_I(inode);
439	int err = 0;	449	int err = 0;
440		450
441	if (delegation == NULL)	451	if (delegation == NULL)
@@ -457,8 +467,6 @@ static int nfs_end_delegation_return(struct inode *inode, struct nfs_delegation · 15 unmodified lines
457	nfs_abort_delegation_return(delegation, clp);	467	nfs_abort_delegation_return(delegation, clp);
458	goto out;	468	goto out;
459	}	469	}
460	if (!nfs_detach_delegation(nfsi, delegation, NFS_SERVER(inode)))
461	goto out;
462		470
463	err = nfs_do_return_delegation(inode, delegation, issync);	471	err = nfs_do_return_delegation(inode, delegation, issync);
464	out:	472	out:
@@ -469,8 +477,6 @@ static bool nfs_delegation_need_return(struct nfs_delegation *delegation) · 4 unmodified lines
469	{	477	{
470	bool ret = false;	478	bool ret = false;
471		479
472	if (test_bit(NFS_DELEGATION_RETURNING, &delegation->flags))
473	goto out;
474	if (test_and_clear_bit(NFS_DELEGATION_RETURN, &delegation->flags))	480	if (test_and_clear_bit(NFS_DELEGATION_RETURN, &delegation->flags))
475	ret = true;	481	ret = true;
476	if (test_and_clear_bit(NFS_DELEGATION_RETURN_IF_CLOSED, &delegation->flags) && !ret) {	482	if (test_and_clear_bit(NFS_DELEGATION_RETURN_IF_CLOSED, &delegation->flags) && !ret) {
@@ -482,7 +488,10 @@ static bool nfs_delegation_need_return(struct nfs_delegation *delegation) · 5 unmodified lines
482	ret = true;	488	ret = true;
483	spin_unlock(&delegation->lock);	489	spin_unlock(&delegation->lock);
484	}	490	}
485	out:	491	if (test_bit(NFS_DELEGATION_RETURNING, &delegation->flags) \|\|
		492	test_bit(NFS_DELEGATION_REVOKED, &delegation->flags))
		493	ret = false;
		494
486	return ret;	495	return ret;
487	}	496	}
488		497
@@ -585,19 +594,23 @@ restart: · 96 unmodified lines
585	}	594	}
586		595
587	/**	596	/**
588	* nfs_inode_return_delegation_noreclaim - return delegation, don't reclaim opens	597	* nfs_inode_evict_delegation - return delegation, don't reclaim opens
589	* @inode: inode to process	598	* @inode: inode to process
590	*	599	*
591	* Does not protect against delegation reclaims, therefore really only safe	600	* Does not protect against delegation reclaims, therefore really only safe
592	* to be called from nfs4_clear_inode().	601	* to be called from nfs4_clear_inode(). Guaranteed to always free
		602	* the delegation structure.
593	*/	603	*/
594	void nfs_inode_return_delegation_noreclaim(struct inode *inode)	604	void nfs_inode_evict_delegation(struct inode *inode)
595	{	605	{
596	struct nfs_delegation *delegation;	606	struct nfs_delegation *delegation;
597		607
598	delegation = nfs_inode_detach_delegation(inode);	608	delegation = nfs_inode_detach_delegation(inode);
599	if (delegation != NULL)	609	if (delegation != NULL) {
		610	set_bit(NFS_DELEGATION_INODE_FREEING, &delegation->flags);
600	nfs_do_return_delegation(inode, delegation, 1);	611	nfs_do_return_delegation(inode, delegation, 1);
		612	nfs_free_delegation(delegation);
		613	}
601	}	614	}
602		615
603	/**	616	/**
@@ -633,10 +646,18 @@ int nfs4_inode_return_delegation(struct inode *inode) · 29 unmodified lines
633	*/	646	*/
634	int nfs4_inode_make_writeable(struct inode *inode)	647	int nfs4_inode_make_writeable(struct inode *inode)
635	{	648	{
636	if (!nfs4_has_session(NFS_SERVER(inode)->nfs_client) \|\|	649	struct nfs_delegation *delegation;
637	!nfs4_check_delegation(inode, FMODE_WRITE))	650
638	return nfs4_inode_return_delegation(inode);	651	rcu_read_lock();
639	return 0;	652	delegation = nfs4_get_valid_delegation(inode);
		653	if (delegation == NULL \|\|
		654	(nfs4_has_session(NFS_SERVER(inode)->nfs_client) &&
		655	(delegation->type & FMODE_WRITE))) {
		656	rcu_read_unlock();
		657	return 0;
		658	}
		659	rcu_read_unlock();
		660	return nfs4_inode_return_delegation(inode);
640	}	661	}
641		662
642	static void nfs_mark_return_if_closed_delegation(struct nfs_server *server,	663	static void nfs_mark_return_if_closed_delegation(struct nfs_server *server,
@@ -744,10 +765,9 @@ static void nfs_mark_delegation_revoked(struct nfs_server *server, · 101 unmodified lines
744	{	765	{
745	set_bit(NFS_DELEGATION_REVOKED, &delegation->flags);	766	set_bit(NFS_DELEGATION_REVOKED, &delegation->flags);
746	delegation->stateid.type = NFS4_INVALID_STATEID_TYPE;	767	delegation->stateid.type = NFS4_INVALID_STATEID_TYPE;
747	nfs_mark_return_delegation(server, delegation);
748	}	768	}
749		769
750	static bool nfs_revoke_delegation(struct inode *inode,	770	static void nfs_revoke_delegation(struct inode *inode,
751	const nfs4_stateid *stateid)	771	const nfs4_stateid *stateid)
752	{	772	{
753	struct nfs_delegation *delegation;	773	struct nfs_delegation *delegation;
@@ -761,29 +781,69 @@ static bool nfs_revoke_delegation(struct inode *inode, · 7 unmodified lines
761	if (stateid == NULL) {	781	if (stateid == NULL) {
762	nfs4_stateid_copy(&tmp, &delegation->stateid);	782	nfs4_stateid_copy(&tmp, &delegation->stateid);
763	stateid = &tmp;	783	stateid = &tmp;
764	} else if (!nfs4_stateid_match(stateid, &delegation->stateid))	784	} else {
765	goto out;	785	if (!nfs4_stateid_match_other(stateid, &delegation->stateid))
		786	goto out;
		787	spin_lock(&delegation->lock);
		788	if (stateid->seqid) {
		789	if (nfs4_stateid_is_newer(&delegation->stateid, stateid)) {
		790	spin_unlock(&delegation->lock);
		791	goto out;
		792	}
		793	delegation->stateid.seqid = stateid->seqid;
		794	}
		795	spin_unlock(&delegation->lock);
		796	}
766	nfs_mark_delegation_revoked(NFS_SERVER(inode), delegation);	797	nfs_mark_delegation_revoked(NFS_SERVER(inode), delegation);
767	ret = true;	798	ret = true;
768	out:	799	out:
769	rcu_read_unlock();	800	rcu_read_unlock();
770	if (ret)	801	if (ret)
771	nfs_inode_find_state_and_recover(inode, stateid);	802	nfs_inode_find_state_and_recover(inode, stateid);
772	return ret;
773	}	803	}
774		804
775	void nfs_remove_bad_delegation(struct inode *inode,	805	void nfs_remove_bad_delegation(struct inode *inode,
776	const nfs4_stateid *stateid)	806	const nfs4_stateid *stateid)
		807	{
		808	nfs_revoke_delegation(inode, stateid);
		809	}
		810	EXPORT_SYMBOL_GPL(nfs_remove_bad_delegation);
		811
		812	void nfs_delegation_mark_returned(struct inode *inode,
		813	const nfs4_stateid *stateid)
777	{	814	{
778	struct nfs_delegation *delegation;	815	struct nfs_delegation *delegation;
779		816
780	if (!nfs_revoke_delegation(inode, stateid))	817	if (!inode)
781	return;	818	return;
782	delegation = nfs_inode_detach_delegation(inode);	819
783	if (delegation)	820	rcu_read_lock();
784	nfs_free_delegation(delegation);	821	delegation = rcu_dereference(NFS_I(inode)->delegation);
		822	if (!delegation)
		823	goto out_rcu_unlock;
		824
		825	spin_lock(&delegation->lock);
		826	if (!nfs4_stateid_match_other(stateid, &delegation->stateid))
		827	goto out_spin_unlock;
		828	if (stateid->seqid) {
		829	/* If delegation->stateid is newer, dont mark as returned */
		830	if (nfs4_stateid_is_newer(&delegation->stateid, stateid))
		831	goto out_clear_returning;
		832	if (delegation->stateid.seqid != stateid->seqid)
		833	delegation->stateid.seqid = stateid->seqid;
		834	}
		835
		836	nfs_mark_delegation_revoked(NFS_SERVER(inode), delegation);
		837
		838	out_clear_returning:
		839	clear_bit(NFS_DELEGATION_RETURNING, &delegation->flags);
		840	out_spin_unlock:
		841	spin_unlock(&delegation->lock);
		842	out_rcu_unlock:
		843	rcu_read_unlock();
		844
		845	nfs_inode_find_state_and_recover(inode, stateid);
785	}	846	}
786	EXPORT_SYMBOL_GPL(nfs_remove_bad_delegation);
787		847
788	/**	848	/**
789	* nfs_expire_unused_delegation_types	849	* nfs_expire_unused_delegation_types
@@ -840,7 +900,7 @@ int nfs_async_inode_return_delegation(struct inode *inode, · 50 unmodified lines
840	struct nfs_delegation *delegation;	900	struct nfs_delegation *delegation;
841		901
842	rcu_read_lock();	902	rcu_read_lock();
843	delegation = rcu_dereference(NFS_I(inode)->delegation);	903	delegation = nfs4_get_valid_delegation(inode);
844	if (delegation == NULL)	904	if (delegation == NULL)
845	goto out_enoent;	905	goto out_enoent;
846	if (stateid != NULL &&	906	if (stateid != NULL &&
@@ -866,6 +926,7 @@ nfs_delegation_find_inode_server(struct nfs_server *server, · 19 unmodified lines
866	list_for_each_entry_rcu(delegation, &server->delegations, super_list) {	926	list_for_each_entry_rcu(delegation, &server->delegations, super_list) {
867	spin_lock(&delegation->lock);	927	spin_lock(&delegation->lock);
868	if (delegation->inode != NULL &&	928	if (delegation->inode != NULL &&
		929	!test_bit(NFS_DELEGATION_REVOKED, &delegation->flags) &&
869	nfs_compare_fh(fhandle, &NFS_I(delegation->inode)->fh) == 0) {	930	nfs_compare_fh(fhandle, &NFS_I(delegation->inode)->fh) == 0) {
870	freeme = igrab(delegation->inode);	931	freeme = igrab(delegation->inode);
871	if (freeme && nfs_sb_active(freeme->i_sb))	932	if (freeme && nfs_sb_active(freeme->i_sb))
@@ -1140,7 +1201,8 @@ void nfs_inode_find_delegation_state_and_recover(struct inode *inode, · 268 unmodified lines
1140	rcu_read_lock();	1201	rcu_read_lock();
1141	delegation = rcu_dereference(NFS_I(inode)->delegation);	1202	delegation = rcu_dereference(NFS_I(inode)->delegation);
1142	if (delegation &&	1203	if (delegation &&
1143	nfs4_stateid_match_other(&delegation->stateid, stateid)) {	1204	nfs4_stateid_match_or_older(&delegation->stateid, stateid) &&
		1205	!test_bit(NFS_DELEGATION_REVOKED, &delegation->flags)) {
1144	nfs_mark_test_expired_delegation(NFS_SERVER(inode), delegation);	1206	nfs_mark_test_expired_delegation(NFS_SERVER(inode), delegation);
1145	found = true;	1207	found = true;
1146	}	1208	}
@@ -1189,7 +1251,9 @@ bool nfs4_refresh_delegation_stateid(nfs4_stateid dst, struct inode inode) · 42 unmodified lines
1189	rcu_read_lock();	1251	rcu_read_lock();
1190	delegation = rcu_dereference(NFS_I(inode)->delegation);	1252	delegation = rcu_dereference(NFS_I(inode)->delegation);
1191	if (delegation != NULL &&	1253	if (delegation != NULL &&
1192	nfs4_stateid_match_other(dst, &delegation->stateid)) {	1254	nfs4_stateid_match_other(dst, &delegation->stateid) &&
		1255	nfs4_stateid_is_newer(&delegation->stateid, dst) &&
		1256	!test_bit(NFS_DELEGATION_REVOKED, &delegation->flags)) {
1193	dst->seqid = delegation->stateid.seqid;	1257	dst->seqid = delegation->stateid.seqid;
1194	ret = true;	1258	ret = true;
1195	}	1259	}

fs/nfs/delegation.h+2−1

 		fmode_t type, const nfs4_stateid *stateid, unsigned long pagemod_limit);
 int nfs4_inode_return_delegation(struct inode *inode);
 int nfs_async_inode_return_delegation(struct inode *inode, const nfs4_stateid *stateid);
-void nfs_inode_return_delegation_noreclaim(struct inode *inode);
+void nfs_inode_evict_delegation(struct inode *inode);
 struct inode *nfs_delegation_find_inode(struct nfs_client *clp, const struct nfs_fh *fhandle);
 void nfs_server_return_all_delegations(struct nfs_server *);
 int nfs_client_return_marked_delegations(struct nfs_client *clp);
 int nfs_delegations_present(struct nfs_client *clp);
 void nfs_remove_bad_delegation(struct inode *inode, const nfs4_stateid *stateid);
+void nfs_delegation_mark_returned(struct inode *inode, const nfs4_stateid *stateid);
 void nfs_delegation_mark_reclaim(struct nfs_client *clp);
 void nfs_delegation_reap_unclaimed(struct nfs_client *clp);

fs/nfs/export.c+1−0

@@ -105,6 +105,7 @@ nfs_fh_to_dentry(struct super_block sb, struct fid fid, · 104 unmodified lines
105	ret = rpc_ops->getattr(NFS_SB(sb), server_fh, fattr, label, NULL);	105	ret = rpc_ops->getattr(NFS_SB(sb), server_fh, fattr, label, NULL);
106	if (ret) {	106	if (ret) {
107	dprintk("%s: getattr failed %d\n", __func__, ret);	107	dprintk("%s: getattr failed %d\n", __func__, ret);
		108	trace_nfs_fh_to_dentry(sb, server_fh, fattr->fileid, ret);
108	dentry = ERR_PTR(ret);	109	dentry = ERR_PTR(ret);
109	goto out_free_label;	110	goto out_free_label;
110	}	111	}

fs/nfs/file.c+1−1

@@ -649,7 +649,7 @@ out: · 648 unmodified lines
649		649
650	out_swapfile:	650	out_swapfile:
651	printk(KERN_INFO "NFS: attempt to write to active swap file!\n");	651	printk(KERN_INFO "NFS: attempt to write to active swap file!\n");
652	return -EBUSY;	652	return -ETXTBSY;
653	}	653	}
654	EXPORT_SYMBOL_GPL(nfs_file_write);	654	EXPORT_SYMBOL_GPL(nfs_file_write);
655		655

fs/nfs/inode.c+27−27

@@ -504,15 +504,15 @@ nfs_fhget(struct super_block sb, struct nfs_fh fh, struct nfs_fattr *fattr, st · 503 unmodified lines
504	nfsi->read_cache_jiffies = fattr->time_start;	504	nfsi->read_cache_jiffies = fattr->time_start;
505	nfsi->attr_gencount = fattr->gencount;	505	nfsi->attr_gencount = fattr->gencount;
506	if (fattr->valid & NFS_ATTR_FATTR_ATIME)	506	if (fattr->valid & NFS_ATTR_FATTR_ATIME)
507	inode->i_atime = timespec_to_timespec64(fattr->atime);	507	inode->i_atime = fattr->atime;
508	else if (nfs_server_capable(inode, NFS_CAP_ATIME))	508	else if (nfs_server_capable(inode, NFS_CAP_ATIME))
509	nfs_set_cache_invalid(inode, NFS_INO_INVALID_ATIME);	509	nfs_set_cache_invalid(inode, NFS_INO_INVALID_ATIME);
510	if (fattr->valid & NFS_ATTR_FATTR_MTIME)	510	if (fattr->valid & NFS_ATTR_FATTR_MTIME)
511	inode->i_mtime = timespec_to_timespec64(fattr->mtime);	511	inode->i_mtime = fattr->mtime;
512	else if (nfs_server_capable(inode, NFS_CAP_MTIME))	512	else if (nfs_server_capable(inode, NFS_CAP_MTIME))
513	nfs_set_cache_invalid(inode, NFS_INO_INVALID_MTIME);	513	nfs_set_cache_invalid(inode, NFS_INO_INVALID_MTIME);
514	if (fattr->valid & NFS_ATTR_FATTR_CTIME)	514	if (fattr->valid & NFS_ATTR_FATTR_CTIME)
515	inode->i_ctime = timespec_to_timespec64(fattr->ctime);	515	inode->i_ctime = fattr->ctime;
516	else if (nfs_server_capable(inode, NFS_CAP_CTIME))	516	else if (nfs_server_capable(inode, NFS_CAP_CTIME))
517	nfs_set_cache_invalid(inode, NFS_INO_INVALID_CTIME);	517	nfs_set_cache_invalid(inode, NFS_INO_INVALID_CTIME);
518	if (fattr->valid & NFS_ATTR_FATTR_CHANGE)	518	if (fattr->valid & NFS_ATTR_FATTR_CHANGE)
@@ -698,7 +698,7 @@ void nfs_setattr_update_inode(struct inode inode, struct iattr attr, · 179 unmodified lines
698	if ((attr->ia_valid & ATTR_GID) != 0)	698	if ((attr->ia_valid & ATTR_GID) != 0)
699	inode->i_gid = attr->ia_gid;	699	inode->i_gid = attr->ia_gid;
700	if (fattr->valid & NFS_ATTR_FATTR_CTIME)	700	if (fattr->valid & NFS_ATTR_FATTR_CTIME)
701	inode->i_ctime = timespec_to_timespec64(fattr->ctime);	701	inode->i_ctime = fattr->ctime;
702	else	702	else
703	nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE	703	nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE
704	\| NFS_INO_INVALID_CTIME);	704	\| NFS_INO_INVALID_CTIME);
@@ -709,14 +709,14 @@ void nfs_setattr_update_inode(struct inode inode, struct iattr attr, · 4 unmodified lines
709	NFS_I(inode)->cache_validity &= ~(NFS_INO_INVALID_ATIME	709	NFS_I(inode)->cache_validity &= ~(NFS_INO_INVALID_ATIME
710	\| NFS_INO_INVALID_CTIME);	710	\| NFS_INO_INVALID_CTIME);
711	if (fattr->valid & NFS_ATTR_FATTR_ATIME)	711	if (fattr->valid & NFS_ATTR_FATTR_ATIME)
712	inode->i_atime = timespec_to_timespec64(fattr->atime);	712	inode->i_atime = fattr->atime;
713	else if (attr->ia_valid & ATTR_ATIME_SET)	713	else if (attr->ia_valid & ATTR_ATIME_SET)
714	inode->i_atime = attr->ia_atime;	714	inode->i_atime = attr->ia_atime;
715	else	715	else
716	nfs_set_cache_invalid(inode, NFS_INO_INVALID_ATIME);	716	nfs_set_cache_invalid(inode, NFS_INO_INVALID_ATIME);
717		717
718	if (fattr->valid & NFS_ATTR_FATTR_CTIME)	718	if (fattr->valid & NFS_ATTR_FATTR_CTIME)
719	inode->i_ctime = timespec_to_timespec64(fattr->ctime);	719	inode->i_ctime = fattr->ctime;
720	else	720	else
721	nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE	721	nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE
722	\| NFS_INO_INVALID_CTIME);	722	\| NFS_INO_INVALID_CTIME);
@@ -725,14 +725,14 @@ void nfs_setattr_update_inode(struct inode inode, struct iattr attr, · 2 unmodified lines
725	NFS_I(inode)->cache_validity &= ~(NFS_INO_INVALID_MTIME	725	NFS_I(inode)->cache_validity &= ~(NFS_INO_INVALID_MTIME
726	\| NFS_INO_INVALID_CTIME);	726	\| NFS_INO_INVALID_CTIME);
727	if (fattr->valid & NFS_ATTR_FATTR_MTIME)	727	if (fattr->valid & NFS_ATTR_FATTR_MTIME)
728	inode->i_mtime = timespec_to_timespec64(fattr->mtime);	728	inode->i_mtime = fattr->mtime;
729	else if (attr->ia_valid & ATTR_MTIME_SET)	729	else if (attr->ia_valid & ATTR_MTIME_SET)
730	inode->i_mtime = attr->ia_mtime;	730	inode->i_mtime = attr->ia_mtime;
731	else	731	else
732	nfs_set_cache_invalid(inode, NFS_INO_INVALID_MTIME);	732	nfs_set_cache_invalid(inode, NFS_INO_INVALID_MTIME);
733		733
734	if (fattr->valid & NFS_ATTR_FATTR_CTIME)	734	if (fattr->valid & NFS_ATTR_FATTR_CTIME)
735	inode->i_ctime = timespec_to_timespec64(fattr->ctime);	735	inode->i_ctime = fattr->ctime;
736	else	736	else
737	nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE	737	nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE
738	\| NFS_INO_INVALID_CTIME);	738	\| NFS_INO_INVALID_CTIME);
@@ -1351,7 +1351,7 @@ static bool nfs_file_has_buffered_writers(struct nfs_inode *nfsi) · 612 unmodified lines
1351		1351
1352	static void nfs_wcc_update_inode(struct inode inode, struct nfs_fattr fattr)	1352	static void nfs_wcc_update_inode(struct inode inode, struct nfs_fattr fattr)
1353	{	1353	{
1354	struct timespec ts;	1354	struct timespec64 ts;
1355		1355
1356	if ((fattr->valid & NFS_ATTR_FATTR_PRECHANGE)	1356	if ((fattr->valid & NFS_ATTR_FATTR_PRECHANGE)
1357	&& (fattr->valid & NFS_ATTR_FATTR_CHANGE)	1357	&& (fattr->valid & NFS_ATTR_FATTR_CHANGE)
@@ -1361,18 +1361,18 @@ static void nfs_wcc_update_inode(struct inode inode, struct nfs_fattr fattr) · 3 unmodified lines
1361	nfs_set_cache_invalid(inode, NFS_INO_INVALID_DATA);	1361	nfs_set_cache_invalid(inode, NFS_INO_INVALID_DATA);
1362	}	1362	}
1363	/* If we have atomic WCC data, we may update some attributes */	1363	/* If we have atomic WCC data, we may update some attributes */
1364	ts = timespec64_to_timespec(inode->i_ctime);	1364	ts = inode->i_ctime;
1365	if ((fattr->valid & NFS_ATTR_FATTR_PRECTIME)	1365	if ((fattr->valid & NFS_ATTR_FATTR_PRECTIME)
1366	&& (fattr->valid & NFS_ATTR_FATTR_CTIME)	1366	&& (fattr->valid & NFS_ATTR_FATTR_CTIME)
1367	&& timespec_equal(&ts, &fattr->pre_ctime)) {	1367	&& timespec64_equal(&ts, &fattr->pre_ctime)) {
1368	inode->i_ctime = timespec_to_timespec64(fattr->ctime);	1368	inode->i_ctime = fattr->ctime;
1369	}	1369	}
1370		1370
1371	ts = timespec64_to_timespec(inode->i_mtime);	1371	ts = inode->i_mtime;
1372	if ((fattr->valid & NFS_ATTR_FATTR_PREMTIME)	1372	if ((fattr->valid & NFS_ATTR_FATTR_PREMTIME)
1373	&& (fattr->valid & NFS_ATTR_FATTR_MTIME)	1373	&& (fattr->valid & NFS_ATTR_FATTR_MTIME)
1374	&& timespec_equal(&ts, &fattr->pre_mtime)) {	1374	&& timespec64_equal(&ts, &fattr->pre_mtime)) {
1375	inode->i_mtime = timespec_to_timespec64(fattr->mtime);	1375	inode->i_mtime = fattr->mtime;
1376	if (S_ISDIR(inode->i_mode))	1376	if (S_ISDIR(inode->i_mode))
1377	nfs_set_cache_invalid(inode, NFS_INO_INVALID_DATA);	1377	nfs_set_cache_invalid(inode, NFS_INO_INVALID_DATA);
1378	}	1378	}
@@ -1398,7 +1398,7 @@ static int nfs_check_inode_attributes(struct inode inode, struct nfs_fattr fat · 19 unmodified lines
1398	struct nfs_inode *nfsi = NFS_I(inode);	1398	struct nfs_inode *nfsi = NFS_I(inode);
1399	loff_t cur_size, new_isize;	1399	loff_t cur_size, new_isize;
1400	unsigned long invalid = 0;	1400	unsigned long invalid = 0;
1401	struct timespec ts;	1401	struct timespec64 ts;
1402		1402
1403	if (NFS_PROTO(inode)->have_delegation(inode, FMODE_READ))	1403	if (NFS_PROTO(inode)->have_delegation(inode, FMODE_READ))
1404	return 0;	1404	return 0;
@@ -1425,12 +1425,12 @@ static int nfs_check_inode_attributes(struct inode inode, struct nfs_fattr fat · 20 unmodified lines
1425	invalid \|= NFS_INO_INVALID_CHANGE	1425	invalid \|= NFS_INO_INVALID_CHANGE
1426	\| NFS_INO_REVAL_PAGECACHE;	1426	\| NFS_INO_REVAL_PAGECACHE;
1427		1427
1428	ts = timespec64_to_timespec(inode->i_mtime);	1428	ts = inode->i_mtime;
1429	if ((fattr->valid & NFS_ATTR_FATTR_MTIME) && !timespec_equal(&ts, &fattr->mtime))	1429	if ((fattr->valid & NFS_ATTR_FATTR_MTIME) && !timespec64_equal(&ts, &fattr->mtime))
1430	invalid \|= NFS_INO_INVALID_MTIME;	1430	invalid \|= NFS_INO_INVALID_MTIME;
1431		1431
1432	ts = timespec64_to_timespec(inode->i_ctime);	1432	ts = inode->i_ctime;
1433	if ((fattr->valid & NFS_ATTR_FATTR_CTIME) && !timespec_equal(&ts, &fattr->ctime))	1433	if ((fattr->valid & NFS_ATTR_FATTR_CTIME) && !timespec64_equal(&ts, &fattr->ctime))
1434	invalid \|= NFS_INO_INVALID_CTIME;	1434	invalid \|= NFS_INO_INVALID_CTIME;
1435		1435
1436	if (fattr->valid & NFS_ATTR_FATTR_SIZE) {	1436	if (fattr->valid & NFS_ATTR_FATTR_SIZE) {
@@ -1460,8 +1460,8 @@ static int nfs_check_inode_attributes(struct inode inode, struct nfs_fattr fat · 23 unmodified lines
1460	if ((fattr->valid & NFS_ATTR_FATTR_NLINK) && inode->i_nlink != fattr->nlink)	1460	if ((fattr->valid & NFS_ATTR_FATTR_NLINK) && inode->i_nlink != fattr->nlink)
1461	invalid \|= NFS_INO_INVALID_OTHER;	1461	invalid \|= NFS_INO_INVALID_OTHER;
1462		1462
1463	ts = timespec64_to_timespec(inode->i_atime);	1463	ts = inode->i_atime;
1464	if ((fattr->valid & NFS_ATTR_FATTR_ATIME) && !timespec_equal(&ts, &fattr->atime))	1464	if ((fattr->valid & NFS_ATTR_FATTR_ATIME) && !timespec64_equal(&ts, &fattr->atime))
1465	invalid \|= NFS_INO_INVALID_ATIME;	1465	invalid \|= NFS_INO_INVALID_ATIME;
1466		1466
1467	if (invalid != 0)	1467	if (invalid != 0)
@@ -1733,12 +1733,12 @@ int nfs_post_op_update_inode_force_wcc_locked(struct inode *inode, struct nfs_fa · 265 unmodified lines
1733	}	1733	}
1734	if ((fattr->valid & NFS_ATTR_FATTR_CTIME) != 0 &&	1734	if ((fattr->valid & NFS_ATTR_FATTR_CTIME) != 0 &&
1735	(fattr->valid & NFS_ATTR_FATTR_PRECTIME) == 0) {	1735	(fattr->valid & NFS_ATTR_FATTR_PRECTIME) == 0) {
1736	fattr->pre_ctime = timespec64_to_timespec(inode->i_ctime);	1736	fattr->pre_ctime = inode->i_ctime;
1737	fattr->valid \|= NFS_ATTR_FATTR_PRECTIME;	1737	fattr->valid \|= NFS_ATTR_FATTR_PRECTIME;
1738	}	1738	}
1739	if ((fattr->valid & NFS_ATTR_FATTR_MTIME) != 0 &&	1739	if ((fattr->valid & NFS_ATTR_FATTR_MTIME) != 0 &&
1740	(fattr->valid & NFS_ATTR_FATTR_PREMTIME) == 0) {	1740	(fattr->valid & NFS_ATTR_FATTR_PREMTIME) == 0) {
1741	fattr->pre_mtime = timespec64_to_timespec(inode->i_mtime);	1741	fattr->pre_mtime = inode->i_mtime;
1742	fattr->valid \|= NFS_ATTR_FATTR_PREMTIME;	1742	fattr->valid \|= NFS_ATTR_FATTR_PREMTIME;
1743	}	1743	}
1744	if ((fattr->valid & NFS_ATTR_FATTR_SIZE) != 0 &&	1744	if ((fattr->valid & NFS_ATTR_FATTR_SIZE) != 0 &&
@@ -1899,7 +1899,7 @@ static int nfs_update_inode(struct inode inode, struct nfs_fattr fattr) · 154 unmodified lines
1899	}	1899	}
1900		1900
1901	if (fattr->valid & NFS_ATTR_FATTR_MTIME) {	1901	if (fattr->valid & NFS_ATTR_FATTR_MTIME) {
1902	inode->i_mtime = timespec_to_timespec64(fattr->mtime);	1902	inode->i_mtime = fattr->mtime;
1903	} else if (server->caps & NFS_CAP_MTIME) {	1903	} else if (server->caps & NFS_CAP_MTIME) {
1904	nfsi->cache_validity \|= save_cache_validity &	1904	nfsi->cache_validity \|= save_cache_validity &
1905	(NFS_INO_INVALID_MTIME	1905	(NFS_INO_INVALID_MTIME
@@ -1908,7 +1908,7 @@ static int nfs_update_inode(struct inode inode, struct nfs_fattr fattr) · 2 unmodified lines
1908	}	1908	}
1909		1909
1910	if (fattr->valid & NFS_ATTR_FATTR_CTIME) {	1910	if (fattr->valid & NFS_ATTR_FATTR_CTIME) {
1911	inode->i_ctime = timespec_to_timespec64(fattr->ctime);	1911	inode->i_ctime = fattr->ctime;
1912	} else if (server->caps & NFS_CAP_CTIME) {	1912	} else if (server->caps & NFS_CAP_CTIME) {
1913	nfsi->cache_validity \|= save_cache_validity &	1913	nfsi->cache_validity \|= save_cache_validity &
1914	(NFS_INO_INVALID_CTIME	1914	(NFS_INO_INVALID_CTIME
@@ -1946,7 +1946,7 @@ static int nfs_update_inode(struct inode inode, struct nfs_fattr fattr) · 31 unmodified lines
1946		1946
1947		1947
1948	if (fattr->valid & NFS_ATTR_FATTR_ATIME)	1948	if (fattr->valid & NFS_ATTR_FATTR_ATIME)
1949	inode->i_atime = timespec_to_timespec64(fattr->atime);	1949	inode->i_atime = fattr->atime;
1950	else if (server->caps & NFS_CAP_ATIME) {	1950	else if (server->caps & NFS_CAP_ATIME) {
1951	nfsi->cache_validity \|= save_cache_validity &	1951	nfsi->cache_validity \|= save_cache_validity &
1952	(NFS_INO_INVALID_ATIME	1952	(NFS_INO_INVALID_ATIME

fs/nfs/internal.h+1−1

@@ -713,7 +713,7 @@ unsigned int nfs_page_array_len(unsigned int base, size_t len) · 712 unmodified lines
713	* 102410241024.	713	* 102410241024.
714	*/	714	*/
715	static inline	715	static inline
716	u64 nfs_timespec_to_change_attr(const struct timespec *ts)	716	u64 nfs_timespec_to_change_attr(const struct timespec64 *ts)
717	{	717	{
718	return ((u64)ts->tv_sec << 30) + ts->tv_nsec;	718	return ((u64)ts->tv_sec << 30) + ts->tv_nsec;
719	}	719	}

fs/nfs/namespace.c+3−0

@@ -157,6 +157,9 @@ struct vfsmount nfs_d_automount(struct path path) · 156 unmodified lines
157	if (IS_ERR(mnt))	157	if (IS_ERR(mnt))
158	goto out;	158	goto out;
159		159
		160	if (nfs_mountpoint_expiry_timeout < 0)
		161	goto out;
		162
160	mntget(mnt); /* prevent immediate expiration */	163	mntget(mnt); /* prevent immediate expiration */
161	mnt_set_expiry(mnt, &nfs_automount_list);	164	mnt_set_expiry(mnt, &nfs_automount_list);
162	schedule_delayed_work(&nfs_automount_task, nfs_mountpoint_expiry_timeout);	165	schedule_delayed_work(&nfs_automount_task, nfs_mountpoint_expiry_timeout);

fs/nfs/nfs2xdr.c+8−13

  *		unsigned int useconds;
  *	};
  */
-static __be32 *xdr_encode_time(__be32 *p, const struct timespec *timep)
+static __be32 *xdr_encode_time(__be32 *p, const struct timespec64 *timep)
 {
-	*p++ = cpu_to_be32(timep->tv_sec);
+	*p++ = cpu_to_be32((u32)timep->tv_sec);
 	if (timep->tv_nsec != 0)
 		*p++ = cpu_to_be32(timep->tv_nsec / NSEC_PER_USEC);
 	else
  * Illustrated" by Brent Callaghan, Addison-Wesley, ISBN 0-201-32750-5.
  */
 static __be32 *xdr_encode_current_server_time(__be32 *p,
-					      const struct timespec *timep)
+					      const struct timespec64 *timep)
 {
 	*p++ = cpu_to_be32(timep->tv_sec);
 	*p++ = cpu_to_be32(1000000);
 	return p;
 }
-static __be32 *xdr_decode_time(__be32 *p, struct timespec *timep)
+static __be32 *xdr_decode_time(__be32 *p, struct timespec64 *timep)
 {
 	timep->tv_sec = be32_to_cpup(p++);
 	timep->tv_nsec = be32_to_cpup(p++) * NSEC_PER_USEC;
 static void encode_sattr(struct xdr_stream *xdr, const struct iattr *attr,
 		struct user_namespace *userns)
 {
-	struct timespec ts;
 	__be32 *p;
 	p = xdr_reserve_space(xdr, NFS_sattr_sz << 2);
 		*p++ = cpu_to_be32(NFS2_SATTR_NOT_SET);
 	if (attr->ia_valid & ATTR_ATIME_SET) {
-		ts = timespec64_to_timespec(attr->ia_atime);
+		p = xdr_encode_time(p, &attr->ia_atime);
-		p = xdr_encode_time(p, &ts);
 	} else if (attr->ia_valid & ATTR_ATIME) {
-		ts = timespec64_to_timespec(attr->ia_atime);
+		p = xdr_encode_current_server_time(p, &attr->ia_atime);
-		p = xdr_encode_current_server_time(p, &ts);
 	} else
 		p = xdr_time_not_set(p);
 	if (attr->ia_valid & ATTR_MTIME_SET) {
-		ts = timespec64_to_timespec(attr->ia_atime);
+		xdr_encode_time(p, &attr->ia_mtime);
-		xdr_encode_time(p, &ts);
 	} else if (attr->ia_valid & ATTR_MTIME) {
-		ts = timespec64_to_timespec(attr->ia_mtime);
+		xdr_encode_current_server_time(p, &attr->ia_mtime);
-		xdr_encode_current_server_time(p, &ts);
 	} else
 		xdr_time_not_set(p);
 }

fs/nfs/nfs3client.c+4−1

@@ -106,7 +106,10 @@ struct nfs_client nfs3_set_ds_client(struct nfs_server mds_srv, · 105 unmodified lines
106	cl_init.nconnect = mds_clp->cl_nconnect;	106	cl_init.nconnect = mds_clp->cl_nconnect;
107		107
108	if (mds_srv->flags & NFS_MOUNT_NORESVPORT)	108	if (mds_srv->flags & NFS_MOUNT_NORESVPORT)
109	set_bit(NFS_CS_NORESVPORT, &cl_init.init_flags);	109	__set_bit(NFS_CS_NORESVPORT, &cl_init.init_flags);
		110
		111	__set_bit(NFS_CS_NOPING, &cl_init.init_flags);
		112	__set_bit(NFS_CS_DS, &cl_init.init_flags);
110		113
111	/* Use the MDS nfs_client cl_ipaddr. */	114	/* Use the MDS nfs_client cl_ipaddr. */
112	nfs_init_timeout_values(&ds_timeout, ds_proto, ds_timeo, ds_retrans);	115	nfs_init_timeout_values(&ds_timeout, ds_proto, ds_timeo, ds_retrans);

fs/nfs/nfs3xdr.c+5−9

  *		uint32	nseconds;
  *	};
  */
-static __be32 *xdr_encode_nfstime3(__be32 *p, const struct timespec *timep)
+static __be32 *xdr_encode_nfstime3(__be32 *p, const struct timespec64 *timep)
 {
-	*p++ = cpu_to_be32(timep->tv_sec);
+	*p++ = cpu_to_be32((u32)timep->tv_sec);
 	*p++ = cpu_to_be32(timep->tv_nsec);
 	return p;
 }
-static __be32 *xdr_decode_nfstime3(__be32 *p, struct timespec *timep)
+static __be32 *xdr_decode_nfstime3(__be32 *p, struct timespec64 *timep)
 {
 	timep->tv_sec = be32_to_cpup(p++);
 	timep->tv_nsec = be32_to_cpup(p++);
 static void encode_sattr3(struct xdr_stream *xdr, const struct iattr *attr,
 		struct user_namespace *userns)
 {
-	struct timespec ts;
 	u32 nbytes;
 	__be32 *p;
 		*p++ = xdr_zero;
 	if (attr->ia_valid & ATTR_ATIME_SET) {
-		struct timespec ts;
 		*p++ = xdr_two;
-		ts = timespec64_to_timespec(attr->ia_atime);
+		p = xdr_encode_nfstime3(p, &attr->ia_atime);
-		p = xdr_encode_nfstime3(p, &ts);
 	} else if (attr->ia_valid & ATTR_ATIME) {
 		*p++ = xdr_one;
 	} else
 	if (attr->ia_valid & ATTR_MTIME_SET) {
 		*p++ = xdr_two;
-		ts = timespec64_to_timespec(attr->ia_mtime);
+		xdr_encode_nfstime3(p, &attr->ia_mtime);
-		xdr_encode_nfstime3(p, &ts);
 	} else if (attr->ia_valid & ATTR_MTIME) {
 		*p = xdr_one;
 	} else

fs/nfs/nfs42.h+14−1

 #define PNFS_LAYOUTSTATS_MAXDEV (4)
 /* nfs4.2proc.c */
+#ifdef CONFIG_NFS_V4_2
 int nfs42_proc_allocate(struct file *, loff_t, loff_t);
-ssize_t nfs42_proc_copy(struct file *, loff_t, struct file *, loff_t, size_t);
+ssize_t nfs42_proc_copy(struct file *, loff_t, struct file *, loff_t, size_t,
+			struct nl4_server *, nfs4_stateid *, bool);
 int nfs42_proc_deallocate(struct file *, loff_t, loff_t);
 loff_t nfs42_proc_llseek(struct file *, loff_t, int);
 int nfs42_proc_layoutstats_generic(struct nfs_server *,
 int nfs42_proc_layouterror(struct pnfs_layout_segment *lseg,
 			   const struct nfs42_layout_error *errors,
 			   size_t n);
+int nfs42_proc_copy_notify(struct file *, struct file *,
+			   struct nfs42_copy_notify_res *);
+static inline bool nfs42_files_from_same_server(struct file *in,
+						struct file *out)
+{
+	struct nfs_client *c_in = (NFS_SERVER(file_inode(in)))->nfs_client;
+	struct nfs_client *c_out = (NFS_SERVER(file_inode(out)))->nfs_client;
+	return nfs4_check_serverowner_major_id(c_in->cl_serverowner,
+					       c_out->cl_serverowner);
+}
+#endif /* CONFIG_NFS_V4_2 */
 #endif /* __LINUX_FS_NFS_NFS4_2_H */

fs/nfs/nfs42proc.c+168−33

@@ -3,6 +3,7 @@ · 2 unmodified lines
3	* Copyright (c) 2014 Anna Schumaker <Anna.Schumaker@Netapp.com>	3	* Copyright (c) 2014 Anna Schumaker <Anna.Schumaker@Netapp.com>
4	*/	4	*/
5	#include <linux/fs.h>	5	#include <linux/fs.h>
		6	#include <linux/sunrpc/addr.h>
6	#include <linux/sunrpc/sched.h>	7	#include <linux/sunrpc/sched.h>
7	#include <linux/nfs.h>	8	#include <linux/nfs.h>
8	#include <linux/nfs3.h>	9	#include <linux/nfs3.h>
@@ -15,10 +16,30 @@ · 6 unmodified lines
15	#include "pnfs.h"	16	#include "pnfs.h"
16	#include "nfs4session.h"	17	#include "nfs4session.h"
17	#include "internal.h"	18	#include "internal.h"
		19	#include "delegation.h"
18		20
19	#define NFSDBG_FACILITY NFSDBG_PROC	21	#define NFSDBG_FACILITY NFSDBG_PROC
20	static int nfs42_do_offload_cancel_async(struct file dst, nfs4_stateid std);	22	static int nfs42_do_offload_cancel_async(struct file dst, nfs4_stateid std);
21		23
		24	static void nfs42_set_netaddr(struct file filep, struct nfs42_netaddr naddr)
		25	{
		26	struct nfs_client *clp = (NFS_SERVER(file_inode(filep)))->nfs_client;
		27	unsigned short port = 2049;
		28
		29	rcu_read_lock();
		30	naddr->netid_len = scnprintf(naddr->netid,
		31	sizeof(naddr->netid), "%s",
		32	rpc_peeraddr2str(clp->cl_rpcclient,
		33	RPC_DISPLAY_NETID));
		34	naddr->addr_len = scnprintf(naddr->addr,
		35	sizeof(naddr->addr),
		36	"%s.%u.%u",
		37	rpc_peeraddr2str(clp->cl_rpcclient,
		38	RPC_DISPLAY_ADDR),
		39	port >> 8, port & 255);
		40	rcu_read_unlock();
		41	}
		42
22	static int _nfs42_proc_fallocate(struct rpc_message msg, struct file filep,	43	static int _nfs42_proc_fallocate(struct rpc_message msg, struct file filep,
23	struct nfs_lock_context *lock, loff_t offset, loff_t len)	44	struct nfs_lock_context *lock, loff_t offset, loff_t len)
24	{	45	{
@@ -28,7 +49,7 @@ static int _nfs42_proc_fallocate(struct rpc_message msg, struct file filep, · 3 unmodified lines
28	.falloc_fh = NFS_FH(inode),	49	.falloc_fh = NFS_FH(inode),
29	.falloc_offset = offset,	50	.falloc_offset = offset,
30	.falloc_length = len,	51	.falloc_length = len,
31	.falloc_bitmask = server->cache_consistency_bitmask,	52	.falloc_bitmask = nfs4_fattr_bitmap,
32	};	53	};
33	struct nfs42_falloc_res res = {	54	struct nfs42_falloc_res res = {
34	.falloc_server = server,	55	.falloc_server = server,
@@ -132,22 +153,26 @@ out_unlock: · 97 unmodified lines
132	}	153	}
133		154
134	static int handle_async_copy(struct nfs42_copy_res *res,	155	static int handle_async_copy(struct nfs42_copy_res *res,
135	struct nfs_server *server,	156	struct nfs_server *dst_server,
		157	struct nfs_server *src_server,
136	struct file *src,	158	struct file *src,
137	struct file *dst,	159	struct file *dst,
138	nfs4_stateid *src_stateid)	160	nfs4_stateid *src_stateid,
		161	bool *restart)
139	{	162	{
140	struct nfs4_copy_state copy, tmp_copy;	163	struct nfs4_copy_state copy, tmp_copy;
141	int status = NFS4_OK;	164	int status = NFS4_OK;
142	bool found_pending = false;	165	bool found_pending = false;
143	struct nfs_open_context *ctx = nfs_file_open_context(dst);	166	struct nfs_open_context *dst_ctx = nfs_file_open_context(dst);
		167	struct nfs_open_context *src_ctx = nfs_file_open_context(src);
144		168
145	copy = kzalloc(sizeof(struct nfs4_copy_state), GFP_NOFS);	169	copy = kzalloc(sizeof(struct nfs4_copy_state), GFP_NOFS);
146	if (!copy)	170	if (!copy)
147	return -ENOMEM;	171	return -ENOMEM;
148		172
149	spin_lock(&server->nfs_client->cl_lock);	173	spin_lock(&dst_server->nfs_client->cl_lock);
150	list_for_each_entry(tmp_copy, &server->nfs_client->pending_cb_stateids,	174	list_for_each_entry(tmp_copy,
		175	&dst_server->nfs_client->pending_cb_stateids,
151	copies) {	176	copies) {
152	if (memcmp(&res->write_res.stateid, &tmp_copy->stateid,	177	if (memcmp(&res->write_res.stateid, &tmp_copy->stateid,
153	NFS4_STATEID_SIZE))	178	NFS4_STATEID_SIZE))
@@ -157,7 +182,7 @@ static int handle_async_copy(struct nfs42_copy_res *res, · 3 unmodified lines
157	break;	182	break;
158	}	183	}
159	if (found_pending) {	184	if (found_pending) {
160	spin_unlock(&server->nfs_client->cl_lock);	185	spin_unlock(&dst_server->nfs_client->cl_lock);
161	kfree(copy);	186	kfree(copy);
162	copy = tmp_copy;	187	copy = tmp_copy;
163	goto out;	188	goto out;
@@ -165,19 +190,32 @@ static int handle_async_copy(struct nfs42_copy_res *res, · 1 unmodified line
165		190
166	memcpy(&copy->stateid, &res->write_res.stateid, NFS4_STATEID_SIZE);	191	memcpy(&copy->stateid, &res->write_res.stateid, NFS4_STATEID_SIZE);
167	init_completion(&copy->completion);	192	init_completion(&copy->completion);
168	copy->parent_state = ctx->state;	193	copy->parent_dst_state = dst_ctx->state;
		194	copy->parent_src_state = src_ctx->state;
169		195
170	list_add_tail(&copy->copies, &server->ss_copies);	196	list_add_tail(&copy->copies, &dst_server->ss_copies);
171	spin_unlock(&server->nfs_client->cl_lock);	197	spin_unlock(&dst_server->nfs_client->cl_lock);
		198
		199	if (dst_server != src_server) {
		200	spin_lock(&src_server->nfs_client->cl_lock);
		201	list_add_tail(&copy->src_copies, &src_server->ss_copies);
		202	spin_unlock(&src_server->nfs_client->cl_lock);
		203	}
172		204
173	status = wait_for_completion_interruptible(&copy->completion);	205	status = wait_for_completion_interruptible(&copy->completion);
174	spin_lock(&server->nfs_client->cl_lock);	206	spin_lock(&dst_server->nfs_client->cl_lock);
175	list_del_init(&copy->copies);	207	list_del_init(&copy->copies);
176	spin_unlock(&server->nfs_client->cl_lock);	208	spin_unlock(&dst_server->nfs_client->cl_lock);
		209	if (dst_server != src_server) {
		210	spin_lock(&src_server->nfs_client->cl_lock);
		211	list_del_init(&copy->src_copies);
		212	spin_unlock(&src_server->nfs_client->cl_lock);
		213	}
177	if (status == -ERESTARTSYS) {	214	if (status == -ERESTARTSYS) {
178	goto out_cancel;	215	goto out_cancel;
179	} else if (copy->flags) {	216	} else if (copy->flags \|\| copy->error == NFS4ERR_PARTNER_NO_AUTH) {
180	status = -EAGAIN;	217	status = -EAGAIN;
		218	*restart = true;
181	goto out_cancel;	219	goto out_cancel;
182	}	220	}
183	out:	221	out:
@@ -185,12 +223,14 @@ out: · 1 unmodified line
185	memcpy(&res->write_res.verifier, &copy->verf, sizeof(copy->verf));	223	memcpy(&res->write_res.verifier, &copy->verf, sizeof(copy->verf));
186	status = -copy->error;	224	status = -copy->error;
187		225
		226	out_free:
188	kfree(copy);	227	kfree(copy);
189	return status;	228	return status;
190	out_cancel:	229	out_cancel:
191	nfs42_do_offload_cancel_async(dst, &copy->stateid);	230	nfs42_do_offload_cancel_async(dst, &copy->stateid);
192	kfree(copy);	231	if (!nfs42_files_from_same_server(src, dst))
193	return status;	232	nfs42_do_offload_cancel_async(src, src_stateid);
		233	goto out_free;
194	}	234	}
195		235
196	static int process_copy_commit(struct file *dst, loff_t pos_dst,	236	static int process_copy_commit(struct file *dst, loff_t pos_dst,
@@ -222,7 +262,10 @@ static ssize_t _nfs42_proc_copy(struct file *src, · 25 unmodified lines
222	struct file *dst,	262	struct file *dst,
223	struct nfs_lock_context *dst_lock,	263	struct nfs_lock_context *dst_lock,
224	struct nfs42_copy_args *args,	264	struct nfs42_copy_args *args,
225	struct nfs42_copy_res *res)	265	struct nfs42_copy_res *res,
		266	struct nl4_server *nss,
		267	nfs4_stateid *cnr_stateid,
		268	bool *restart)
226	{	269	{
227	struct rpc_message msg = {	270	struct rpc_message msg = {
228	.rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_COPY],	271	.rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_COPY],
@@ -230,17 +273,23 @@ static ssize_t _nfs42_proc_copy(struct file *src, · 1 unmodified line
230	.rpc_resp = res,	273	.rpc_resp = res,
231	};	274	};
232	struct inode *dst_inode = file_inode(dst);	275	struct inode *dst_inode = file_inode(dst);
233	struct nfs_server *server = NFS_SERVER(dst_inode);	276	struct inode *src_inode = file_inode(src);
		277	struct nfs_server *dst_server = NFS_SERVER(dst_inode);
		278	struct nfs_server *src_server = NFS_SERVER(src_inode);
234	loff_t pos_src = args->src_pos;	279	loff_t pos_src = args->src_pos;
235	loff_t pos_dst = args->dst_pos;	280	loff_t pos_dst = args->dst_pos;
236	size_t count = args->count;	281	size_t count = args->count;
237	ssize_t status;	282	ssize_t status;
238		283
239	status = nfs4_set_rw_stateid(&args->src_stateid, src_lock->open_context,	284	if (nss) {
240	src_lock, FMODE_READ);	285	args->cp_src = nss;
241	if (status)	286	nfs4_stateid_copy(&args->src_stateid, cnr_stateid);
242	return status;	287	} else {
243		288	status = nfs4_set_rw_stateid(&args->src_stateid,
		289	src_lock->open_context, src_lock, FMODE_READ);
		290	if (status)
		291	return status;
		292	}
244	status = nfs_filemap_write_and_wait_range(file_inode(src)->i_mapping,	293	status = nfs_filemap_write_and_wait_range(file_inode(src)->i_mapping,
245	pos_src, pos_src + (loff_t)count - 1);	294	pos_src, pos_src + (loff_t)count - 1);
246	if (status)	295	if (status)
@@ -262,13 +311,15 @@ static ssize_t _nfs42_proc_copy(struct file *src, · 15 unmodified lines
262	if (!res->commit_res.verf)	311	if (!res->commit_res.verf)
263	return -ENOMEM;	312	return -ENOMEM;
264	}	313	}
		314	set_bit(NFS_CLNT_SRC_SSC_COPY_STATE,
		315	&src_lock->open_context->state->flags);
265	set_bit(NFS_CLNT_DST_SSC_COPY_STATE,	316	set_bit(NFS_CLNT_DST_SSC_COPY_STATE,
266	&dst_lock->open_context->state->flags);	317	&dst_lock->open_context->state->flags);
267		318
268	status = nfs4_call_sync(server->client, server, &msg,	319	status = nfs4_call_sync(dst_server->client, dst_server, &msg,
269	&args->seq_args, &res->seq_res, 0);	320	&args->seq_args, &res->seq_res, 0);
270	if (status == -ENOTSUPP)	321	if (status == -ENOTSUPP)
271	server->caps &= ~NFS_CAP_COPY;	322	dst_server->caps &= ~NFS_CAP_COPY;
272	if (status)	323	if (status)
273	goto out;	324	goto out;
274		325
@@ -280,8 +331,8 @@ static ssize_t _nfs42_proc_copy(struct file *src, · 5 unmodified lines
280	}	331	}
281		332
282	if (!res->synchronous) {	333	if (!res->synchronous) {
283	status = handle_async_copy(res, server, src, dst,	334	status = handle_async_copy(res, dst_server, src_server, src,
284	&args->src_stateid);	335	dst, &args->src_stateid, restart);
285	if (status)	336	if (status)
286	return status;	337	return status;
287	}	338	}
@@ -304,8 +355,9 @@ out: · 16 unmodified lines
304	}	355	}
305		356
306	ssize_t nfs42_proc_copy(struct file *src, loff_t pos_src,	357	ssize_t nfs42_proc_copy(struct file *src, loff_t pos_src,
307	struct file *dst, loff_t pos_dst,	358	struct file *dst, loff_t pos_dst, size_t count,
308	size_t count)	359	struct nl4_server *nss,
		360	nfs4_stateid *cnr_stateid, bool sync)
309	{	361	{
310	struct nfs_server *server = NFS_SERVER(file_inode(dst));	362	struct nfs_server *server = NFS_SERVER(file_inode(dst));
311	struct nfs_lock_context *src_lock;	363	struct nfs_lock_context *src_lock;
@@ -316,7 +368,7 @@ ssize_t nfs42_proc_copy(struct file *src, loff_t pos_src, · 4 unmodified lines
316	.dst_fh = NFS_FH(file_inode(dst)),	368	.dst_fh = NFS_FH(file_inode(dst)),
317	.dst_pos = pos_dst,	369	.dst_pos = pos_dst,
318	.count = count,	370	.count = count,
319	.sync = false,	371	.sync = sync,
320	};	372	};
321	struct nfs42_copy_res res;	373	struct nfs42_copy_res res;
322	struct nfs4_exception src_exception = {	374	struct nfs4_exception src_exception = {
@@ -328,6 +380,7 @@ ssize_t nfs42_proc_copy(struct file *src, loff_t pos_src, · 5 unmodified lines
328	.stateid = &args.dst_stateid,	380	.stateid = &args.dst_stateid,
329	};	381	};
330	ssize_t err, err2;	382	ssize_t err, err2;
		383	bool restart = false;
331		384
332	src_lock = nfs_get_lock_context(nfs_file_open_context(src));	385	src_lock = nfs_get_lock_context(nfs_file_open_context(src));
333	if (IS_ERR(src_lock))	386	if (IS_ERR(src_lock))
@@ -347,21 +400,33 @@ ssize_t nfs42_proc_copy(struct file *src, loff_t pos_src, · 13 unmodified lines
347	inode_lock(file_inode(dst));	400	inode_lock(file_inode(dst));
348	err = _nfs42_proc_copy(src, src_lock,	401	err = _nfs42_proc_copy(src, src_lock,
349	dst, dst_lock,	402	dst, dst_lock,
350	&args, &res);	403	&args, &res,
		404	nss, cnr_stateid, &restart);
351	inode_unlock(file_inode(dst));	405	inode_unlock(file_inode(dst));
352		406
353	if (err >= 0)	407	if (err >= 0)
354	break;	408	break;
355	if (err == -ENOTSUPP) {	409	if (err == -ENOTSUPP &&
		410	nfs42_files_from_same_server(src, dst)) {
356	err = -EOPNOTSUPP;	411	err = -EOPNOTSUPP;
357	break;	412	break;
358	} else if (err == -EAGAIN) {	413	} else if (err == -EAGAIN) {
359	dst_exception.retry = 1;	414	if (!restart) {
360	continue;	415	dst_exception.retry = 1;
		416	continue;
		417	}
		418	break;
361	} else if (err == -NFS4ERR_OFFLOAD_NO_REQS && !args.sync) {	419	} else if (err == -NFS4ERR_OFFLOAD_NO_REQS && !args.sync) {
362	args.sync = true;	420	args.sync = true;
363	dst_exception.retry = 1;	421	dst_exception.retry = 1;
364	continue;	422	continue;
		423	} else if ((err == -ESTALE \|\|
		424	err == -NFS4ERR_OFFLOAD_DENIED \|\|
		425	err == -ENOTSUPP) &&
		426	!nfs42_files_from_same_server(src, dst)) {
		427	nfs42_do_offload_cancel_async(src, &args.src_stateid);
		428	err = -EOPNOTSUPP;
		429	break;
365	}	430	}
366		431
367	err2 = nfs4_handle_exception(server, err, &src_exception);	432	err2 = nfs4_handle_exception(server, err, &src_exception);
@@ -459,6 +524,76 @@ static int nfs42_do_offload_cancel_async(struct file *dst, · 91 unmodified lines
459	return status;	524	return status;
460	}	525	}
461		526
		527	static int _nfs42_proc_copy_notify(struct file src, struct file dst,
		528	struct nfs42_copy_notify_args *args,
		529	struct nfs42_copy_notify_res *res)
		530	{
		531	struct nfs_server *src_server = NFS_SERVER(file_inode(src));
		532	struct rpc_message msg = {
		533	.rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_COPY_NOTIFY],
		534	.rpc_argp = args,
		535	.rpc_resp = res,
		536	};
		537	int status;
		538	struct nfs_open_context *ctx;
		539	struct nfs_lock_context *l_ctx;
		540
		541	ctx = get_nfs_open_context(nfs_file_open_context(src));
		542	l_ctx = nfs_get_lock_context(ctx);
		543	if (IS_ERR(l_ctx))
		544	return PTR_ERR(l_ctx);
		545
		546	status = nfs4_set_rw_stateid(&args->cna_src_stateid, ctx, l_ctx,
		547	FMODE_READ);
		548	nfs_put_lock_context(l_ctx);
		549	if (status)
		550	return status;
		551
		552	status = nfs4_call_sync(src_server->client, src_server, &msg,
		553	&args->cna_seq_args, &res->cnr_seq_res, 0);
		554	if (status == -ENOTSUPP)
		555	src_server->caps &= ~NFS_CAP_COPY_NOTIFY;
		556
		557	put_nfs_open_context(nfs_file_open_context(src));
		558	return status;
		559	}
		560
		561	int nfs42_proc_copy_notify(struct file src, struct file dst,
		562	struct nfs42_copy_notify_res *res)
		563	{
		564	struct nfs_server *src_server = NFS_SERVER(file_inode(src));
		565	struct nfs42_copy_notify_args *args;
		566	struct nfs4_exception exception = {
		567	.inode = file_inode(src),
		568	};
		569	int status;
		570
		571	if (!(src_server->caps & NFS_CAP_COPY_NOTIFY))
		572	return -EOPNOTSUPP;
		573
		574	args = kzalloc(sizeof(struct nfs42_copy_notify_args), GFP_NOFS);
		575	if (args == NULL)
		576	return -ENOMEM;
		577
		578	args->cna_src_fh = NFS_FH(file_inode(src)),
		579	args->cna_dst.nl4_type = NL4_NETADDR;
		580	nfs42_set_netaddr(dst, &args->cna_dst.u.nl4_addr);
		581	exception.stateid = &args->cna_src_stateid;
		582
		583	do {
		584	status = _nfs42_proc_copy_notify(src, dst, args, res);
		585	if (status == -ENOTSUPP) {
		586	status = -EOPNOTSUPP;
		587	goto out;
		588	}
		589	status = nfs4_handle_exception(src_server, status, &exception);
		590	} while (exception.retry);
		591
		592	out:
		593	kfree(args);
		594	return status;
		595	}
		596
462	static loff_t _nfs42_proc_llseek(struct file *filep,	597	static loff_t _nfs42_proc_llseek(struct file *filep,
463	struct nfs_lock_context *lock, loff_t offset, int whence)	598	struct nfs_lock_context *lock, loff_t offset, int whence)
464	{	599	{

fs/nfs/nfs42xdr.c+188−2

@@ -21,7 +21,10 @@ · 20 unmodified lines
21	#define encode_copy_maxsz (op_encode_hdr_maxsz + \	21	#define encode_copy_maxsz (op_encode_hdr_maxsz + \
22	XDR_QUADLEN(NFS4_STATEID_SIZE) + \	22	XDR_QUADLEN(NFS4_STATEID_SIZE) + \
23	XDR_QUADLEN(NFS4_STATEID_SIZE) + \	23	XDR_QUADLEN(NFS4_STATEID_SIZE) + \
24	2 + 2 + 2 + 1 + 1 + 1)	24	2 + 2 + 2 + 1 + 1 + 1 +\
		25	1 + /* One cnr_source_server */\
		26	1 + /* nl4_type */ \
		27	1 + XDR_QUADLEN(NFS4_OPAQUE_LIMIT))
25	#define decode_copy_maxsz (op_decode_hdr_maxsz + \	28	#define decode_copy_maxsz (op_decode_hdr_maxsz + \
26	NFS42_WRITE_RES_SIZE + \	29	NFS42_WRITE_RES_SIZE + \
27	1 /* cr_consecutive */ + \	30	1 /* cr_consecutive */ + \
@@ -29,6 +32,16 @@ · 1 unmodified line
29	#define encode_offload_cancel_maxsz (op_encode_hdr_maxsz + \	32	#define encode_offload_cancel_maxsz (op_encode_hdr_maxsz + \
30	XDR_QUADLEN(NFS4_STATEID_SIZE))	33	XDR_QUADLEN(NFS4_STATEID_SIZE))
31	#define decode_offload_cancel_maxsz (op_decode_hdr_maxsz)	34	#define decode_offload_cancel_maxsz (op_decode_hdr_maxsz)
		35	#define encode_copy_notify_maxsz (op_encode_hdr_maxsz + \
		36	XDR_QUADLEN(NFS4_STATEID_SIZE) + \
		37	1 + /* nl4_type */ \
		38	1 + XDR_QUADLEN(NFS4_OPAQUE_LIMIT))
		39	#define decode_copy_notify_maxsz (op_decode_hdr_maxsz + \
		40	3 + /* cnr_lease_time */\
		41	XDR_QUADLEN(NFS4_STATEID_SIZE) + \
		42	1 + /* Support 1 cnr_source_server */\
		43	1 + /* nl4_type */ \
		44	1 + XDR_QUADLEN(NFS4_OPAQUE_LIMIT))
32	#define encode_deallocate_maxsz (op_encode_hdr_maxsz + \	45	#define encode_deallocate_maxsz (op_encode_hdr_maxsz + \
33	encode_fallocate_maxsz)	46	encode_fallocate_maxsz)
34	#define decode_deallocate_maxsz (op_decode_hdr_maxsz)	47	#define decode_deallocate_maxsz (op_decode_hdr_maxsz)
@@ -99,6 +112,12 @@ · 64 unmodified lines
99	decode_sequence_maxsz + \	112	decode_sequence_maxsz + \
100	decode_putfh_maxsz + \	113	decode_putfh_maxsz + \
101	decode_offload_cancel_maxsz)	114	decode_offload_cancel_maxsz)
		115	#define NFS4_enc_copy_notify_sz (compound_encode_hdr_maxsz + \
		116	encode_putfh_maxsz + \
		117	encode_copy_notify_maxsz)
		118	#define NFS4_dec_copy_notify_sz (compound_decode_hdr_maxsz + \
		119	decode_putfh_maxsz + \
		120	decode_copy_notify_maxsz)
102	#define NFS4_enc_deallocate_sz (compound_encode_hdr_maxsz + \	121	#define NFS4_enc_deallocate_sz (compound_encode_hdr_maxsz + \
103	encode_sequence_maxsz + \	122	encode_sequence_maxsz + \
104	encode_putfh_maxsz + \	123	encode_putfh_maxsz + \
@@ -166,6 +185,26 @@ static void encode_allocate(struct xdr_stream *xdr, · 61 unmodified lines
166	encode_fallocate(xdr, args);	185	encode_fallocate(xdr, args);
167	}	186	}
168		187
		188	static void encode_nl4_server(struct xdr_stream *xdr,
		189	const struct nl4_server *ns)
		190	{
		191	encode_uint32(xdr, ns->nl4_type);
		192	switch (ns->nl4_type) {
		193	case NL4_NAME:
		194	case NL4_URL:
		195	encode_string(xdr, ns->u.nl4_str_sz, ns->u.nl4_str);
		196	break;
		197	case NL4_NETADDR:
		198	encode_string(xdr, ns->u.nl4_addr.netid_len,
		199	ns->u.nl4_addr.netid);
		200	encode_string(xdr, ns->u.nl4_addr.addr_len,
		201	ns->u.nl4_addr.addr);
		202	break;
		203	default:
		204	WARN_ON_ONCE(1);
		205	}
		206	}
		207
169	static void encode_copy(struct xdr_stream *xdr,	208	static void encode_copy(struct xdr_stream *xdr,
170	const struct nfs42_copy_args *args,	209	const struct nfs42_copy_args *args,
171	struct compound_hdr *hdr)	210	struct compound_hdr *hdr)
@@ -180,7 +219,12 @@ static void encode_copy(struct xdr_stream *xdr, · 8 unmodified lines
180		219
181	encode_uint32(xdr, 1); /* consecutive = true */	220	encode_uint32(xdr, 1); /* consecutive = true */
182	encode_uint32(xdr, args->sync);	221	encode_uint32(xdr, args->sync);
183	encode_uint32(xdr, 0); /* src server list */	222	if (args->cp_src == NULL) { /* intra-ssc */
		223	encode_uint32(xdr, 0); /* no src server list */
		224	return;
		225	}
		226	encode_uint32(xdr, 1); /* supporting 1 server */
		227	encode_nl4_server(xdr, args->cp_src);
184	}	228	}
185		229
186	static void encode_offload_cancel(struct xdr_stream *xdr,	230	static void encode_offload_cancel(struct xdr_stream *xdr,
@@ -191,6 +235,15 @@ static void encode_offload_cancel(struct xdr_stream *xdr, · 4 unmodified lines
191	encode_nfs4_stateid(xdr, &args->osa_stateid);	235	encode_nfs4_stateid(xdr, &args->osa_stateid);
192	}	236	}
193		237
		238	static void encode_copy_notify(struct xdr_stream *xdr,
		239	const struct nfs42_copy_notify_args *args,
		240	struct compound_hdr *hdr)
		241	{
		242	encode_op_hdr(xdr, OP_COPY_NOTIFY, decode_copy_notify_maxsz, hdr);
		243	encode_nfs4_stateid(xdr, &args->cna_src_stateid);
		244	encode_nl4_server(xdr, &args->cna_dst);
		245	}
		246
194	static void encode_deallocate(struct xdr_stream *xdr,	247	static void encode_deallocate(struct xdr_stream *xdr,
195	const struct nfs42_falloc_args *args,	248	const struct nfs42_falloc_args *args,
196	struct compound_hdr *hdr)	249	struct compound_hdr *hdr)
@@ -354,6 +407,25 @@ static void nfs4_xdr_enc_offload_cancel(struct rpc_rqst *req, · 157 unmodified lines
354	encode_nops(&hdr);	407	encode_nops(&hdr);
355	}	408	}
356		409
		410	/*
		411	* Encode COPY_NOTIFY request
		412	*/
		413	static void nfs4_xdr_enc_copy_notify(struct rpc_rqst *req,
		414	struct xdr_stream *xdr,
		415	const void *data)
		416	{
		417	const struct nfs42_copy_notify_args *args = data;
		418	struct compound_hdr hdr = {
		419	.minorversion = nfs4_xdr_minorversion(&args->cna_seq_args),
		420	};
		421
		422	encode_compound_hdr(xdr, req, &hdr);
		423	encode_sequence(xdr, &args->cna_seq_args, &hdr);
		424	encode_putfh(xdr, args->cna_src_fh, &hdr);
		425	encode_copy_notify(xdr, args, &hdr);
		426	encode_nops(&hdr);
		427	}
		428
357	/*	429	/*
358	* Encode DEALLOCATE request	430	* Encode DEALLOCATE request
359	*/	431	*/
@@ -490,6 +562,58 @@ static int decode_write_response(struct xdr_stream *xdr, · 130 unmodified lines
490	return decode_verifier(xdr, &res->verifier.verifier);	562	return decode_verifier(xdr, &res->verifier.verifier);
491	}	563	}
492		564
		565	static int decode_nl4_server(struct xdr_stream xdr, struct nl4_server ns)
		566	{
		567	struct nfs42_netaddr *naddr;
		568	uint32_t dummy;
		569	char *dummy_str;
		570	__be32 *p;
		571	int status;
		572
		573	/* nl_type */
		574	p = xdr_inline_decode(xdr, 4);
		575	if (unlikely(!p))
		576	return -EIO;
		577	ns->nl4_type = be32_to_cpup(p);
		578	switch (ns->nl4_type) {
		579	case NL4_NAME:
		580	case NL4_URL:
		581	status = decode_opaque_inline(xdr, &dummy, &dummy_str);
		582	if (unlikely(status))
		583	return status;
		584	if (unlikely(dummy > NFS4_OPAQUE_LIMIT))
		585	return -EIO;
		586	memcpy(&ns->u.nl4_str, dummy_str, dummy);
		587	ns->u.nl4_str_sz = dummy;
		588	break;
		589	case NL4_NETADDR:
		590	naddr = &ns->u.nl4_addr;
		591
		592	/* netid string */
		593	status = decode_opaque_inline(xdr, &dummy, &dummy_str);
		594	if (unlikely(status))
		595	return status;
		596	if (unlikely(dummy > RPCBIND_MAXNETIDLEN))
		597	return -EIO;
		598	naddr->netid_len = dummy;
		599	memcpy(naddr->netid, dummy_str, naddr->netid_len);
		600
		601	/* uaddr string */
		602	status = decode_opaque_inline(xdr, &dummy, &dummy_str);
		603	if (unlikely(status))
		604	return status;
		605	if (unlikely(dummy > RPCBIND_MAXUADDRLEN))
		606	return -EIO;
		607	naddr->addr_len = dummy;
		608	memcpy(naddr->addr, dummy_str, naddr->addr_len);
		609	break;
		610	default:
		611	WARN_ON_ONCE(1);
		612	return -EIO;
		613	}
		614	return 0;
		615	}
		616
493	static int decode_copy_requirements(struct xdr_stream *xdr,	617	static int decode_copy_requirements(struct xdr_stream *xdr,
494	struct nfs42_copy_res *res) {	618	struct nfs42_copy_res *res) {
495	__be32 *p;	619	__be32 *p;
@@ -529,6 +653,42 @@ static int decode_offload_cancel(struct xdr_stream *xdr, · 33 unmodified lines
529	return decode_op_hdr(xdr, OP_OFFLOAD_CANCEL);	653	return decode_op_hdr(xdr, OP_OFFLOAD_CANCEL);
530	}	654	}
531		655
		656	static int decode_copy_notify(struct xdr_stream *xdr,
		657	struct nfs42_copy_notify_res *res)
		658	{
		659	__be32 *p;
		660	int status, count;
		661
		662	status = decode_op_hdr(xdr, OP_COPY_NOTIFY);
		663	if (status)
		664	return status;
		665	/* cnr_lease_time */
		666	p = xdr_inline_decode(xdr, 12);
		667	if (unlikely(!p))
		668	return -EIO;
		669	p = xdr_decode_hyper(p, &res->cnr_lease_time.seconds);
		670	res->cnr_lease_time.nseconds = be32_to_cpup(p);
		671
		672	status = decode_opaque_fixed(xdr, &res->cnr_stateid, NFS4_STATEID_SIZE);
		673	if (unlikely(status))
		674	return -EIO;
		675
		676	/* number of source addresses */
		677	p = xdr_inline_decode(xdr, 4);
		678	if (unlikely(!p))
		679	return -EIO;
		680
		681	count = be32_to_cpup(p);
		682	if (count > 1)
		683	pr_warn("NFS: %s: nsvr %d > Supported. Use first servers\n",
		684	__func__, count);
		685
		686	status = decode_nl4_server(xdr, &res->cnr_src);
		687	if (unlikely(status))
		688	return -EIO;
		689	return 0;
		690	}
		691
532	static int decode_deallocate(struct xdr_stream xdr, struct nfs42_falloc_res res)	692	static int decode_deallocate(struct xdr_stream xdr, struct nfs42_falloc_res res)
533	{	693	{
534	return decode_op_hdr(xdr, OP_DEALLOCATE);	694	return decode_op_hdr(xdr, OP_DEALLOCATE);
@@ -656,6 +816,32 @@ out: · 121 unmodified lines
656	return status;	816	return status;
657	}	817	}
658		818
		819	/*
		820	* Decode COPY_NOTIFY response
		821	*/
		822	static int nfs4_xdr_dec_copy_notify(struct rpc_rqst *rqstp,
		823	struct xdr_stream *xdr,
		824	void *data)
		825	{
		826	struct nfs42_copy_notify_res *res = data;
		827	struct compound_hdr hdr;
		828	int status;
		829
		830	status = decode_compound_hdr(xdr, &hdr);
		831	if (status)
		832	goto out;
		833	status = decode_sequence(xdr, &res->cnr_seq_res, rqstp);
		834	if (status)
		835	goto out;
		836	status = decode_putfh(xdr);
		837	if (status)
		838	goto out;
		839	status = decode_copy_notify(xdr, res);
		840
		841	out:
		842	return status;
		843	}
		844
659	/*	845	/*
660	* Decode DEALLOCATE request	846	* Decode DEALLOCATE request
661	*/	847	*/

fs/nfs/nfs4_fs.h+19−2

 	NFS_STATE_RECOVERY_FAILED,	/* OPEN stateid state recovery failed */
 	NFS_STATE_MAY_NOTIFY_LOCK,	/* server may CB_NOTIFY_LOCK */
 	NFS_STATE_CHANGE_WAIT,		/* A state changing operation is outstanding */
-#ifdef CONFIG_NFS_V4_2
 	NFS_CLNT_DST_SSC_COPY_STATE,    /* dst server open state on client*/
-#endif /* CONFIG_NFS_V4_2 */
+	NFS_CLNT_SRC_SSC_COPY_STATE,    /* src server open state on client*/
+	NFS_SRV_SSC_COPY_STATE,		/* ssc state on the dst server */
 };
 struct nfs4_state {
 		const struct nfs_open_context *ctx,
 		const struct nfs_lock_context *l_ctx,
 		fmode_t fmode);
+extern int nfs4_proc_getattr(struct nfs_server *server, struct nfs_fh *fhandle,
+			     struct nfs_fattr *fattr, struct nfs4_label *label,
+			     struct inode *inode);
+extern int update_open_stateid(struct nfs4_state *state,
+				const nfs4_stateid *open_stateid,
+				const nfs4_stateid *deleg_stateid,
+				fmode_t fmode);
 extern int nfs4_proc_get_lease_time(struct nfs_client *clp,
 		struct nfs_fsinfo *fsinfo);
 /* nfs4state.c */
+extern const nfs4_stateid current_stateid;
 const struct cred *nfs4_get_clid_cred(struct nfs_client *clp);
 const struct cred *nfs4_get_machine_cred(struct nfs_client *clp);
 const struct cred *nfs4_get_renew_cred(struct nfs_client *clp);
 			struct nfs_client **, const struct cred *);
 extern void nfs4_schedule_session_recovery(struct nfs4_session *, int);
 extern void nfs41_notify_server(struct nfs_client *);
+bool nfs4_check_serverowner_major_id(struct nfs41_server_owner *o1,
+			struct nfs41_server_owner *o2);
 #else
 static inline void nfs4_schedule_session_recovery(struct nfs4_session *session, int err)
 {
 	return (s32)(be32_to_cpu(s1->seqid) - be32_to_cpu(s2->seqid)) > 0;
 }
+static inline bool nfs4_stateid_match_or_older(const nfs4_stateid *dst, const nfs4_stateid *src)
+{
+	return nfs4_stateid_match_other(dst, src) &&
+		!(src->seqid && nfs4_stateid_is_newer(dst, src));
+}
 static inline void nfs4_stateid_seqid_inc(nfs4_stateid *s1)
 {
 	u32 seqid = be32_to_cpu(s1->seqid);

fs/nfs/nfs4client.c+8−5

 /*
  * Returns true if the server major ids match
  */
-static bool
+bool
 nfs4_check_serverowner_major_id(struct nfs41_server_owner *o1,
 				struct nfs41_server_owner *o2)
 {
 	};
 	struct nfs_client *clp;
-	if (minorversion > 0 && proto == XPRT_TRANSPORT_TCP)
+	if (minorversion == 0)
+		__set_bit(NFS_CS_REUSEPORT, &cl_init.init_flags);
+	else if (proto == XPRT_TRANSPORT_TCP)
 		cl_init.nconnect = nconnect;
 	if (server->flags & NFS_MOUNT_NORESVPORT)
-		set_bit(NFS_CS_NORESVPORT, &cl_init.init_flags);
+		__set_bit(NFS_CS_NORESVPORT, &cl_init.init_flags);
 	if (server->options & NFS_OPTION_MIGRATION)
-		set_bit(NFS_CS_MIGRATION, &cl_init.init_flags);
+		__set_bit(NFS_CS_MIGRATION, &cl_init.init_flags);
 	if (test_bit(NFS_MIG_TSM_POSSIBLE, &server->mig_status))
-		set_bit(NFS_CS_TSM_POSSIBLE, &cl_init.init_flags);
+		__set_bit(NFS_CS_TSM_POSSIBLE, &cl_init.init_flags);
 	server->port = rpc_get_port(addr);
 	/* Allocate or find a client reference we can use */

fs/nfs/nfs4file.c+139−2

@@ -133,14 +133,55 @@ static ssize_t __nfs4_copy_file_range(struct file *file_in, loff_t pos_in, · 132 unmodified lines
133	struct file *file_out, loff_t pos_out,	133	struct file *file_out, loff_t pos_out,
134	size_t count, unsigned int flags)	134	size_t count, unsigned int flags)
135	{	135	{
		136	struct nfs42_copy_notify_res *cn_resp = NULL;
		137	struct nl4_server *nss = NULL;
		138	nfs4_stateid *cnrs = NULL;
		139	ssize_t ret;
		140	bool sync = false;
		141
136	/* Only offload copy if superblock is the same */	142	/* Only offload copy if superblock is the same */
137	if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb)	143	if (file_in->f_op != &nfs4_file_operations)
138	return -EXDEV;	144	return -EXDEV;
139	if (!nfs_server_capable(file_inode(file_out), NFS_CAP_COPY))	145	if (!nfs_server_capable(file_inode(file_out), NFS_CAP_COPY))
140	return -EOPNOTSUPP;	146	return -EOPNOTSUPP;
141	if (file_inode(file_in) == file_inode(file_out))	147	if (file_inode(file_in) == file_inode(file_out))
142	return -EOPNOTSUPP;	148	return -EOPNOTSUPP;
143	return nfs42_proc_copy(file_in, pos_in, file_out, pos_out, count);	149	/* if the copy size if smaller than 2 RPC payloads, make it
		150	* synchronous
		151	*/
		152	if (count <= 2 * NFS_SERVER(file_inode(file_in))->rsize)
		153	sync = true;
		154	retry:
		155	if (!nfs42_files_from_same_server(file_in, file_out)) {
		156	/* for inter copy, if copy size if smaller than 12 RPC
		157	* payloads, fallback to traditional copy. There are
		158	* 14 RPCs during an NFSv4.x mount between source/dest
		159	* servers.
		160	*/
		161	if (sync \|\|
		162	count <= 14 * NFS_SERVER(file_inode(file_in))->rsize)
		163	return -EOPNOTSUPP;
		164	cn_resp = kzalloc(sizeof(struct nfs42_copy_notify_res),
		165	GFP_NOFS);
		166	if (unlikely(cn_resp == NULL))
		167	return -ENOMEM;
		168
		169	ret = nfs42_proc_copy_notify(file_in, file_out, cn_resp);
		170	if (ret) {
		171	ret = -EOPNOTSUPP;
		172	goto out;
		173	}
		174	nss = &cn_resp->cnr_src;
		175	cnrs = &cn_resp->cnr_stateid;
		176	}
		177	ret = nfs42_proc_copy(file_in, pos_in, file_out, pos_out, count,
		178	nss, cnrs, sync);
		179	out:
		180	if (!nfs42_files_from_same_server(file_in, file_out))
		181	kfree(cn_resp);
		182	if (ret == -EAGAIN)
		183	goto retry;
		184	return ret;
144	}	185	}
145		186
146	static ssize_t nfs4_copy_file_range(struct file *file_in, loff_t pos_in,	187	static ssize_t nfs4_copy_file_range(struct file *file_in, loff_t pos_in,
@@ -263,6 +304,102 @@ out_unlock: · 116 unmodified lines
263	out:	304	out:
264	return ret < 0 ? ret : count;	305	return ret < 0 ? ret : count;
265	}	306	}
		307
		308	static int read_name_gen = 1;
		309	#define SSC_READ_NAME_BODY "ssc_read_%d"
		310
		311	struct file *
		312	nfs42_ssc_open(struct vfsmount ss_mnt, struct nfs_fh src_fh,
		313	nfs4_stateid *stateid)
		314	{
		315	struct nfs_fattr fattr;
		316	struct file filep, res;
		317	struct nfs_server *server;
		318	struct inode *r_ino = NULL;
		319	struct nfs_open_context *ctx;
		320	struct nfs4_state_owner *sp;
		321	char *read_name = NULL;
		322	int len, status = 0;
		323
		324	server = NFS_SERVER(ss_mnt->mnt_root->d_inode);
		325
		326	nfs_fattr_init(&fattr);
		327
		328	status = nfs4_proc_getattr(server, src_fh, &fattr, NULL, NULL);
		329	if (status < 0) {
		330	res = ERR_PTR(status);
		331	goto out;
		332	}
		333
		334	res = ERR_PTR(-ENOMEM);
		335	len = strlen(SSC_READ_NAME_BODY) + 16;
		336	read_name = kzalloc(len, GFP_NOFS);
		337	if (read_name == NULL)
		338	goto out;
		339	snprintf(read_name, len, SSC_READ_NAME_BODY, read_name_gen++);
		340
		341	r_ino = nfs_fhget(ss_mnt->mnt_root->d_inode->i_sb, src_fh, &fattr,
		342	NULL);
		343	if (IS_ERR(r_ino)) {
		344	res = ERR_CAST(r_ino);
		345	goto out_free_name;
		346	}
		347
		348	filep = alloc_file_pseudo(r_ino, ss_mnt, read_name, FMODE_READ,
		349	r_ino->i_fop);
		350	if (IS_ERR(filep)) {
		351	res = ERR_CAST(filep);
		352	goto out_free_name;
		353	}
		354	filep->f_mode \|= FMODE_READ;
		355
		356	ctx = alloc_nfs_open_context(filep->f_path.dentry, filep->f_mode,
		357	filep);
		358	if (IS_ERR(ctx)) {
		359	res = ERR_CAST(ctx);
		360	goto out_filep;
		361	}
		362
		363	res = ERR_PTR(-EINVAL);
		364	sp = nfs4_get_state_owner(server, ctx->cred, GFP_KERNEL);
		365	if (sp == NULL)
		366	goto out_ctx;
		367
		368	ctx->state = nfs4_get_open_state(r_ino, sp);
		369	if (ctx->state == NULL)
		370	goto out_stateowner;
		371
		372	set_bit(NFS_SRV_SSC_COPY_STATE, &ctx->state->flags);
		373	set_bit(NFS_OPEN_STATE, &ctx->state->flags);
		374	memcpy(&ctx->state->open_stateid.other, &stateid->other,
		375	NFS4_STATEID_OTHER_SIZE);
		376	update_open_stateid(ctx->state, stateid, NULL, filep->f_mode);
		377
		378	nfs_file_set_open_context(filep, ctx);
		379	put_nfs_open_context(ctx);
		380
		381	file_ra_state_init(&filep->f_ra, filep->f_mapping->host->i_mapping);
		382	res = filep;
		383	out_free_name:
		384	kfree(read_name);
		385	out:
		386	return res;
		387	out_stateowner:
		388	nfs4_put_state_owner(sp);
		389	out_ctx:
		390	put_nfs_open_context(ctx);
		391	out_filep:
		392	fput(filep);
		393	goto out_free_name;
		394	}
		395	EXPORT_SYMBOL_GPL(nfs42_ssc_open);
		396	void nfs42_ssc_close(struct file *filep)
		397	{
		398	struct nfs_open_context *ctx = nfs_file_open_context(filep);
		399
		400	ctx->state->flags = 0;
		401	}
		402	EXPORT_SYMBOL_GPL(nfs42_ssc_close);
266	#endif /* CONFIG_NFS_V4_2 */	403	#endif /* CONFIG_NFS_V4_2 */
267		404
268	const struct file_operations nfs4_file_operations = {	405	const struct file_operations nfs4_file_operations = {

fs/nfs/nfs4proc.c+46−23

 static int _nfs4_recover_proc_open(struct nfs4_opendata *data);
 static int nfs4_do_fsinfo(struct nfs_server *, struct nfs_fh *, struct nfs_fsinfo *);
 static void nfs_fixup_referral_attributes(struct nfs_fattr *fattr);
-static int nfs4_proc_getattr(struct nfs_server *, struct nfs_fh *, struct nfs_fattr *, struct nfs4_label *label, struct inode *inode);
 static int _nfs4_proc_getattr(struct nfs_server *server, struct nfs_fh *fhandle, struct nfs_fattr *fattr, struct nfs4_label *label, struct inode *inode);
 static int nfs4_do_setattr(struct inode *inode, const struct cred *cred,
 			    struct nfs_fattr *fattr, struct iattr *sattr,
 		case -NFS4ERR_ADMIN_REVOKED:
 		case -NFS4ERR_EXPIRED:
 		case -NFS4ERR_BAD_STATEID:
+		case -NFS4ERR_PARTNER_NO_AUTH:
 			if (inode != NULL && stateid != NULL) {
 				nfs_inode_find_state_and_recover(inode,
 						stateid);
 		case -NFS4ERR_DEADSESSION:
 		case -NFS4ERR_SEQ_FALSE_RETRY:
 		case -NFS4ERR_SEQ_MISORDERED:
-			dprintk("%s ERROR: %d Reset session\n", __func__,
+			/* Handled in nfs41_sequence_process() */
-				errorcode);
-			nfs4_schedule_session_recovery(clp->cl_session, errorcode);
 			goto wait_on_recovery;
 #endif /* defined(CONFIG_NFS_V4_1) */
 		case -NFS4ERR_FILE_OPEN:
 	struct nfs4_session *session;
 	struct nfs4_slot *slot = res->sr_slot;
 	struct nfs_client *clp;
+	int status;
 	int ret = 1;
 	if (slot == NULL)
 	session = slot->table->session;
 	trace_nfs4_sequence_done(session, res);
+	status = res->sr_status;
+	if (task->tk_status == -NFS4ERR_DEADSESSION)
+		status = -NFS4ERR_DEADSESSION;
 	/* Check the SEQUENCE operation status */
-	switch (res->sr_status) {
+	switch (status) {
 	case 0:
 		/* Mark this sequence number as having been acked */
 		nfs4_slot_sequence_acked(slot, slot->seq_nr);
 		 */
 		slot->seq_nr = slot->seq_nr_highest_sent;
 		goto out_retry;
+	case -NFS4ERR_BADSESSION:
+	case -NFS4ERR_DEADSESSION:
+	case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION:
+		goto session_recover;
 	default:
 		/* Just update the slot sequence no. */
 		slot->seq_done = 1;
 out_noaction:
 	return ret;
 session_recover:
-	nfs4_schedule_session_recovery(session, res->sr_status);
+	nfs4_schedule_session_recovery(session, status);
-	goto retry_nowait;
+	dprintk("%s ERROR: %d Reset session\n", __func__, status);
+	nfs41_sequence_free_slot(res);
+	goto out;
 retry_new_seq:
 	++slot->seq_nr;
 retry_nowait:
 	write_sequnlock(&state->seqlock);
 }
-static int update_open_stateid(struct nfs4_state *state,
+int update_open_stateid(struct nfs4_state *state,
 		const nfs4_stateid *open_stateid,
 		const nfs4_stateid *delegation,
 		fmode_t fmode)
 		ret = 1;
 	}
-	deleg_cur = rcu_dereference(nfsi->delegation);
+	deleg_cur = nfs4_get_valid_delegation(state->inode);
 	if (deleg_cur == NULL)
 		goto no_delegation;
 	if (delegation == NULL)
 		delegation = &deleg_cur->stateid;
-	else if (!nfs4_stateid_match(&deleg_cur->stateid, delegation))
+	else if (!nfs4_stateid_match_other(&deleg_cur->stateid, delegation))
 		goto no_delegation_unlock;
 	nfs_mark_delegation_referenced(deleg_cur);
 	fmode &= FMODE_READ|FMODE_WRITE;
 	rcu_read_lock();
-	delegation = rcu_dereference(NFS_I(inode)->delegation);
+	delegation = nfs4_get_valid_delegation(inode);
 	if (delegation == NULL || (delegation->type & fmode) == fmode) {
 		rcu_read_unlock();
 		return;
 		case -NFS4ERR_BAD_HIGH_SLOT:
 		case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION:
 		case -NFS4ERR_DEADSESSION:
-			nfs4_schedule_session_recovery(server->nfs_client->cl_session, err);
 			return -EAGAIN;
 		case -NFS4ERR_STALE_CLIENTID:
 		case -NFS4ERR_STALE_STATEID:
 	return nfs4_call_sync(server->client, server, &msg, &args.seq_args, &res.seq_res, 0);
 }
-static int nfs4_proc_getattr(struct nfs_server *server, struct nfs_fh *fhandle,
+int nfs4_proc_getattr(struct nfs_server *server, struct nfs_fh *fhandle,
 				struct nfs_fattr *fattr, struct nfs4_label *label,
 				struct inode *inode)
 {
 		const struct nfs_lock_context *l_ctx,
 		fmode_t fmode)
 {
-	nfs4_stateid current_stateid;
+	nfs4_stateid _current_stateid;
 	/* If the current stateid represents a lost lock, then exit */
-	if (nfs4_set_rw_stateid(&current_stateid, ctx, l_ctx, fmode) == -EIO)
+	if (nfs4_set_rw_stateid(&_current_stateid, ctx, l_ctx, fmode) == -EIO)
 		return true;
-	return nfs4_stateid_match(stateid, &current_stateid);
+	return nfs4_stateid_match(stateid, &_current_stateid);
 }
 static bool nfs4_error_stateid_expired(int err)
 		task->tk_status = 0;
 		break;
 	case -NFS4ERR_OLD_STATEID:
-		if (nfs4_refresh_delegation_stateid(&data->stateid, data->inode))
+		if (!nfs4_refresh_delegation_stateid(&data->stateid, data->inode))
-			goto out_restart;
+			nfs4_stateid_seqid_inc(&data->stateid);
-		task->tk_status = 0;
+		if (data->args.bitmask) {
-		break;
+			data->args.bitmask = NULL;
+			data->res.fattr = NULL;
+		}
+		goto out_restart;
 	case -NFS4ERR_ACCESS:
 		if (data->args.bitmask) {
 			data->args.bitmask = NULL;
 		if (exception.retry)
 			goto out_restart;
 	}
+	nfs_delegation_mark_returned(data->inode, data->args.stateid);
 	data->rpc_status = task->tk_status;
 	return;
 out_restart:
 	d_data = (struct nfs4_delegreturndata *)data;
-	if (!d_data->lr.roc && nfs4_wait_on_layoutreturn(d_data->inode, task))
+	if (!d_data->lr.roc && nfs4_wait_on_layoutreturn(d_data->inode, task)) {
+		nfs4_sequence_done(task, &d_data->res.seq_res);
 		return;
+	}
 	lo = d_data->args.lr_args ? d_data->args.lr_args->layout : NULL;
 	if (lo && !pnfs_layout_is_valid(lo)) {
 static void
 nfs4_bind_one_conn_to_session_done(struct rpc_task *task, void *calldata)
 {
+	struct nfs41_bind_conn_to_session_args *args = task->tk_msg.rpc_argp;
+	struct nfs_client *clp = args->client;
+	switch (task->tk_status) {
+	case -NFS4ERR_BADSESSION:
+	case -NFS4ERR_DEADSESSION:
+		nfs4_schedule_session_recovery(clp->cl_session,
+				task->tk_status);
+	}
 }
 static const struct rpc_call_ops nfs4_bind_one_conn_to_session_ops = {
 	case -NFS4ERR_BADSESSION:
 	case -NFS4ERR_DEADSESSION:
 	case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION:
-		nfs4_schedule_session_recovery(clp->cl_session,
-				task->tk_status);
 		break;
 	default:
 		nfs4_schedule_lease_recovery(clp);
 		| NFS_CAP_ALLOCATE
 		| NFS_CAP_COPY
 		| NFS_CAP_OFFLOAD_CANCEL
+		| NFS_CAP_COPY_NOTIFY
 		| NFS_CAP_DEALLOCATE
 		| NFS_CAP_SEEK
 		| NFS_CAP_LAYOUTSTATS

fs/nfs/nfs4state.c+43−8

 #include "nfs4session.h"
 #include "pnfs.h"
 #include "netns.h"
+#include "nfs4trace.h"
 #define NFSDBG_FACILITY		NFSDBG_STATE
 	list_for_each_entry(pos, &state->lock_states, ls_locks) {
 		if (!test_bit(NFS_LOCK_INITIALIZED, &pos->ls_flags))
 			continue;
-		if (nfs4_stateid_match_other(&pos->ls_stateid, stateid))
+		if (nfs4_stateid_match_or_older(&pos->ls_stateid, stateid))
 			return pos;
 	}
 	return NULL;
 		state = ctx->state;
 		if (state == NULL)
 			continue;
-		if (nfs4_stateid_match_other(&state->stateid, stateid) &&
+		if (nfs4_stateid_match_or_older(&state->stateid, stateid) &&
 		    nfs4_state_mark_reclaim_nograce(clp, state)) {
 			found = true;
 			continue;
 		}
-		if (nfs4_stateid_match_other(&state->open_stateid, stateid) &&
+		if (test_bit(NFS_OPEN_STATE, &state->flags) &&
+		    nfs4_stateid_match_or_older(&state->open_stateid, stateid) &&
 		    nfs4_state_mark_reclaim_nograce(clp, state)) {
 			found = true;
 			continue;
 {
 	struct nfs4_copy_state *copy;
-	if (!test_bit(NFS_CLNT_DST_SSC_COPY_STATE, &state->flags))
+	if (!test_bit(NFS_CLNT_DST_SSC_COPY_STATE, &state->flags) &&
+		!test_bit(NFS_CLNT_SRC_SSC_COPY_STATE, &state->flags))
 		return;
 	spin_lock(&sp->so_server->nfs_client->cl_lock);
 	list_for_each_entry(copy, &sp->so_server->ss_copies, copies) {
-		if (!nfs4_stateid_match_other(&state->stateid, &copy->parent_state->stateid))
+		if ((test_bit(NFS_CLNT_DST_SSC_COPY_STATE, &state->flags) &&
-			continue;
+				!nfs4_stateid_match_other(&state->stateid,
+				&copy->parent_dst_state->stateid)))
+				continue;
 		copy->flags = 1;
-		complete(&copy->completion);
+		if (test_and_clear_bit(NFS_CLNT_DST_SSC_COPY_STATE,
-		break;
+				&state->flags)) {
+			clear_bit(NFS_CLNT_SRC_SSC_COPY_STATE, &state->flags);
+			complete(&copy->completion);
+		}
+	}
+	list_for_each_entry(copy, &sp->so_server->ss_copies, src_copies) {
+		if ((test_bit(NFS_CLNT_SRC_SSC_COPY_STATE, &state->flags) &&
+				!nfs4_stateid_match_other(&state->stateid,
+				&copy->parent_src_state->stateid)))
+				continue;
+		copy->flags = 1;
+		if (test_and_clear_bit(NFS_CLNT_DST_SSC_COPY_STATE,
+				&state->flags))
+			complete(&copy->completion);
 	}
 	spin_unlock(&sp->so_server->nfs_client->cl_lock);
 }
 	if (!test_bit(NFS_DELEGATED_STATE, &state->flags)) {
 		spin_lock(&state->state_lock);
 		list_for_each_entry(lock, &state->lock_states, ls_locks) {
+			trace_nfs4_state_lock_reclaim(state, lock);
 			if (!test_bit(NFS_LOCK_INITIALIZED, &lock->ls_flags))
 				pr_warn_ratelimited("NFS: %s: Lock reclaim failed!\n", __func__);
 		}
 	struct nfs4_state *state;
 	unsigned int loop = 0;
 	int status = 0;
+#ifdef CONFIG_NFS_V4_2
+	bool found_ssc_copy_state = false;
+#endif /* CONFIG_NFS_V4_2 */
 	/* Note: we rely on the sp->so_states list being ordered
 	 * so that we always reclaim open(O_RDWR) and/or open(O_WRITE)
 			continue;
 		if (state->state == 0)
 			continue;
+#ifdef CONFIG_NFS_V4_2
+		if (test_bit(NFS_SRV_SSC_COPY_STATE, &state->flags)) {
+			nfs4_state_mark_recovery_failed(state, -EIO);
+			found_ssc_copy_state = true;
+			continue;
+		}
+#endif /* CONFIG_NFS_V4_2 */
 		refcount_inc(&state->count);
 		spin_unlock(&sp->so_lock);
 		status = __nfs4_reclaim_open_state(sp, state, ops);
 	}
 	raw_write_seqcount_end(&sp->so_reclaim_seqcount);
 	spin_unlock(&sp->so_lock);
+#ifdef CONFIG_NFS_V4_2
+	if (found_ssc_copy_state)
+		return -EIO;
+#endif /* CONFIG_NFS_V4_2 */
 	return 0;
 out_err:
 	nfs4_put_open_state(state);
 	/* Ensure exclusive access to NFSv4 state */
 	do {
+		trace_nfs4_state_mgr(clp);
 		clear_bit(NFS4CLNT_RUN_MANAGER, &clp->cl_state);
 		if (test_bit(NFS4CLNT_PURGE_STATE, &clp->cl_state)) {
 			section = "purge state";
 out_error:
 	if (strlen(section))
 		section_sep = ": ";
+	trace_nfs4_state_mgr_failed(clp, section, status);
 	pr_warn_ratelimited("NFS: state manager%s%s failed on NFSv4 server %s"
 			" with error %d\n", section_sep, section,
 			clp->cl_hostname, -status);

fs/nfs/nfs4super.c+2−2

@@ -92,8 +92,8 @@ static void nfs4_evict_inode(struct inode *inode) · 91 unmodified lines
92	{	92	{
93	truncate_inode_pages_final(&inode->i_data);	93	truncate_inode_pages_final(&inode->i_data);
94	clear_inode(inode);	94	clear_inode(inode);
95	/* If we are holding a delegation, return it! */	95	/* If we are holding a delegation, return and free it */
96	nfs_inode_return_delegation_noreclaim(inode);	96	nfs_inode_evict_delegation(inode);
97	/* Note that above delegreturn would trigger pnfs return-on-close */	97	/* Note that above delegreturn would trigger pnfs return-on-close */
98	pnfs_return_layout(inode);	98	pnfs_return_layout(inode);
99	pnfs_destroy_layout(NFS_I(inode));	99	pnfs_destroy_layout(NFS_I(inode));

fs/nfs/nfs4trace.h+175−0

@@ -562,6 +562,99 @@ TRACE_EVENT(nfs4_setup_sequence, · 561 unmodified lines
562	)	562	)
563	);	563	);
564		564
		565	TRACE_DEFINE_ENUM(NFS4CLNT_MANAGER_RUNNING);
		566	TRACE_DEFINE_ENUM(NFS4CLNT_CHECK_LEASE);
		567	TRACE_DEFINE_ENUM(NFS4CLNT_LEASE_EXPIRED);
		568	TRACE_DEFINE_ENUM(NFS4CLNT_RECLAIM_REBOOT);
		569	TRACE_DEFINE_ENUM(NFS4CLNT_RECLAIM_NOGRACE);
		570	TRACE_DEFINE_ENUM(NFS4CLNT_DELEGRETURN);
		571	TRACE_DEFINE_ENUM(NFS4CLNT_SESSION_RESET);
		572	TRACE_DEFINE_ENUM(NFS4CLNT_LEASE_CONFIRM);
		573	TRACE_DEFINE_ENUM(NFS4CLNT_SERVER_SCOPE_MISMATCH);
		574	TRACE_DEFINE_ENUM(NFS4CLNT_PURGE_STATE);
		575	TRACE_DEFINE_ENUM(NFS4CLNT_BIND_CONN_TO_SESSION);
		576	TRACE_DEFINE_ENUM(NFS4CLNT_MOVED);
		577	TRACE_DEFINE_ENUM(NFS4CLNT_LEASE_MOVED);
		578	TRACE_DEFINE_ENUM(NFS4CLNT_DELEGATION_EXPIRED);
		579	TRACE_DEFINE_ENUM(NFS4CLNT_RUN_MANAGER);
		580	TRACE_DEFINE_ENUM(NFS4CLNT_DELEGRETURN_RUNNING);
		581
		582	#define show_nfs4_clp_state(state) \
		583	__print_flags(state, "\|", \
		584	{ NFS4CLNT_MANAGER_RUNNING, "MANAGER_RUNNING" }, \
		585	{ NFS4CLNT_CHECK_LEASE, "CHECK_LEASE" }, \
		586	{ NFS4CLNT_LEASE_EXPIRED, "LEASE_EXPIRED" }, \
		587	{ NFS4CLNT_RECLAIM_REBOOT, "RECLAIM_REBOOT" }, \
		588	{ NFS4CLNT_RECLAIM_NOGRACE, "RECLAIM_NOGRACE" }, \
		589	{ NFS4CLNT_DELEGRETURN, "DELEGRETURN" }, \
		590	{ NFS4CLNT_SESSION_RESET, "SESSION_RESET" }, \
		591	{ NFS4CLNT_LEASE_CONFIRM, "LEASE_CONFIRM" }, \
		592	{ NFS4CLNT_SERVER_SCOPE_MISMATCH, \
		593	"SERVER_SCOPE_MISMATCH" }, \
		594	{ NFS4CLNT_PURGE_STATE, "PURGE_STATE" }, \
		595	{ NFS4CLNT_BIND_CONN_TO_SESSION, \
		596	"BIND_CONN_TO_SESSION" }, \
		597	{ NFS4CLNT_MOVED, "MOVED" }, \
		598	{ NFS4CLNT_LEASE_MOVED, "LEASE_MOVED" }, \
		599	{ NFS4CLNT_DELEGATION_EXPIRED, "DELEGATION_EXPIRED" }, \
		600	{ NFS4CLNT_RUN_MANAGER, "RUN_MANAGER" }, \
		601	{ NFS4CLNT_DELEGRETURN_RUNNING, "DELEGRETURN_RUNNING" })
		602
		603	TRACE_EVENT(nfs4_state_mgr,
		604	TP_PROTO(
		605	const struct nfs_client *clp
		606	),
		607
		608	TP_ARGS(clp),
		609
		610	TP_STRUCT__entry(
		611	__field(unsigned long, state)
		612	__string(hostname, clp->cl_hostname)
		613	),
		614
		615	TP_fast_assign(
		616	__entry->state = clp->cl_state;
		617	__assign_str(hostname, clp->cl_hostname)
		618	),
		619
		620	TP_printk(
		621	"hostname=%s clp state=%s", __get_str(hostname),
		622	show_nfs4_clp_state(__entry->state)
		623	)
		624	)
		625
		626	TRACE_EVENT(nfs4_state_mgr_failed,
		627	TP_PROTO(
		628	const struct nfs_client *clp,
		629	const char *section,
		630	int status
		631	),
		632
		633	TP_ARGS(clp, section, status),
		634
		635	TP_STRUCT__entry(
		636	__field(unsigned long, error)
		637	__field(unsigned long, state)
		638	__string(hostname, clp->cl_hostname)
		639	__string(section, section)
		640	),
		641
		642	TP_fast_assign(
		643	__entry->error = status;
		644	__entry->state = clp->cl_state;
		645	__assign_str(hostname, clp->cl_hostname);
		646	__assign_str(section, section);
		647	),
		648
		649	TP_printk(
		650	"hostname=%s clp state=%s error=%ld (%s) section=%s",
		651	__get_str(hostname),
		652	show_nfs4_clp_state(__entry->state), -__entry->error,
		653	show_nfsv4_errors(__entry->error), __get_str(section)
		654
		655	)
		656	)
		657
565	TRACE_EVENT(nfs4_xdr_status,	658	TRACE_EVENT(nfs4_xdr_status,
566	TP_PROTO(	659	TP_PROTO(
567	const struct xdr_stream *xdr,	660	const struct xdr_stream *xdr,
@@ -929,6 +1022,88 @@ TRACE_EVENT(nfs4_set_lock, · 361 unmodified lines
929	)	1022	)
930	);	1023	);
931		1024
		1025	TRACE_DEFINE_ENUM(LK_STATE_IN_USE);
		1026	TRACE_DEFINE_ENUM(NFS_DELEGATED_STATE);
		1027	TRACE_DEFINE_ENUM(NFS_OPEN_STATE);
		1028	TRACE_DEFINE_ENUM(NFS_O_RDONLY_STATE);
		1029	TRACE_DEFINE_ENUM(NFS_O_WRONLY_STATE);
		1030	TRACE_DEFINE_ENUM(NFS_O_RDWR_STATE);
		1031	TRACE_DEFINE_ENUM(NFS_STATE_RECLAIM_REBOOT);
		1032	TRACE_DEFINE_ENUM(NFS_STATE_RECLAIM_NOGRACE);
		1033	TRACE_DEFINE_ENUM(NFS_STATE_POSIX_LOCKS);
		1034	TRACE_DEFINE_ENUM(NFS_STATE_RECOVERY_FAILED);
		1035	TRACE_DEFINE_ENUM(NFS_STATE_MAY_NOTIFY_LOCK);
		1036	TRACE_DEFINE_ENUM(NFS_STATE_CHANGE_WAIT);
		1037	TRACE_DEFINE_ENUM(NFS_CLNT_DST_SSC_COPY_STATE);
		1038	TRACE_DEFINE_ENUM(NFS_CLNT_SRC_SSC_COPY_STATE);
		1039	TRACE_DEFINE_ENUM(NFS_SRV_SSC_COPY_STATE);
		1040
		1041	#define show_nfs4_state_flags(flags) \
		1042	__print_flags(flags, "\|", \
		1043	{ LK_STATE_IN_USE, "IN_USE" }, \
		1044	{ NFS_DELEGATED_STATE, "DELEGATED" }, \
		1045	{ NFS_OPEN_STATE, "OPEN" }, \
		1046	{ NFS_O_RDONLY_STATE, "O_RDONLY" }, \
		1047	{ NFS_O_WRONLY_STATE, "O_WRONLY" }, \
		1048	{ NFS_O_RDWR_STATE, "O_RDWR" }, \
		1049	{ NFS_STATE_RECLAIM_REBOOT, "RECLAIM_REBOOT" }, \
		1050	{ NFS_STATE_RECLAIM_NOGRACE, "RECLAIM_NOGRACE" }, \
		1051	{ NFS_STATE_POSIX_LOCKS, "POSIX_LOCKS" }, \
		1052	{ NFS_STATE_RECOVERY_FAILED, "RECOVERY_FAILED" }, \
		1053	{ NFS_STATE_MAY_NOTIFY_LOCK, "MAY_NOTIFY_LOCK" }, \
		1054	{ NFS_STATE_CHANGE_WAIT, "CHANGE_WAIT" }, \
		1055	{ NFS_CLNT_DST_SSC_COPY_STATE, "CLNT_DST_SSC_COPY" }, \
		1056	{ NFS_CLNT_SRC_SSC_COPY_STATE, "CLNT_SRC_SSC_COPY" }, \
		1057	{ NFS_SRV_SSC_COPY_STATE, "SRV_SSC_COPY" })
		1058
		1059	#define show_nfs4_lock_flags(flags) \
		1060	__print_flags(flags, "\|", \
		1061	{ BIT(NFS_LOCK_INITIALIZED), "INITIALIZED" }, \
		1062	{ BIT(NFS_LOCK_LOST), "LOST" })
		1063
		1064	TRACE_EVENT(nfs4_state_lock_reclaim,
		1065	TP_PROTO(
		1066	const struct nfs4_state *state,
		1067	const struct nfs4_lock_state *lock
		1068	),
		1069
		1070	TP_ARGS(state, lock),
		1071
		1072	TP_STRUCT__entry(
		1073	__field(dev_t, dev)
		1074	__field(u32, fhandle)
		1075	__field(u64, fileid)
		1076	__field(unsigned long, state_flags)
		1077	__field(unsigned long, lock_flags)
		1078	__field(int, stateid_seq)
		1079	__field(u32, stateid_hash)
		1080	),
		1081
		1082	TP_fast_assign(
		1083	const struct inode *inode = state->inode;
		1084
		1085	__entry->dev = inode->i_sb->s_dev;
		1086	__entry->fileid = NFS_FILEID(inode);
		1087	__entry->fhandle = nfs_fhandle_hash(NFS_FH(inode));
		1088	__entry->state_flags = state->flags;
		1089	__entry->lock_flags = lock->ls_flags;
		1090	__entry->stateid_seq =
		1091	be32_to_cpu(state->stateid.seqid);
		1092	__entry->stateid_hash =
		1093	nfs_stateid_hash(&state->stateid);
		1094	),
		1095
		1096	TP_printk(
		1097	"fileid=%02x:%02x:%llu fhandle=0x%08x "
		1098	"stateid=%d:0x%08x state_flags=%s lock_flags=%s",
		1099	MAJOR(__entry->dev), MINOR(__entry->dev),
		1100	(unsigned long long)__entry->fileid, __entry->fhandle,
		1101	__entry->stateid_seq, __entry->stateid_hash,
		1102	show_nfs4_state_flags(__entry->state_flags),
		1103	show_nfs4_lock_flags(__entry->lock_flags)
		1104	)
		1105	)
		1106
932	DECLARE_EVENT_CLASS(nfs4_set_delegation_event,	1107	DECLARE_EVENT_CLASS(nfs4_set_delegation_event,
933	TP_PROTO(	1108	TP_PROTO(
934	const struct inode *inode,	1109	const struct inode *inode,

fs/nfs/nfs4xdr.c+11−13

 }
 static __be32 *
-xdr_encode_nfstime4(__be32 *p, const struct timespec *t)
+xdr_encode_nfstime4(__be32 *p, const struct timespec64 *t)
 {
 	p = xdr_encode_hyper(p, (__s64)t->tv_sec);
 	*p++ = cpu_to_be32(t->tv_nsec);
 				const struct nfs_server *server,
 				const uint32_t attrmask[])
 {
-	struct timespec ts;
 	char owner_name[IDMAP_NAMESZ];
 	char owner_group[IDMAP_NAMESZ];
 	int owner_namelen = 0;
 	if (bmval[1] & FATTR4_WORD1_TIME_ACCESS_SET) {
 		if (iap->ia_valid & ATTR_ATIME_SET) {
 			*p++ = cpu_to_be32(NFS4_SET_TO_CLIENT_TIME);
-			ts = timespec64_to_timespec(iap->ia_atime);
+			p = xdr_encode_nfstime4(p, &iap->ia_atime);
-			p = xdr_encode_nfstime4(p, &ts);
 		} else
 			*p++ = cpu_to_be32(NFS4_SET_TO_SERVER_TIME);
 	}
 	if (bmval[1] & FATTR4_WORD1_TIME_MODIFY_SET) {
 		if (iap->ia_valid & ATTR_MTIME_SET) {
 			*p++ = cpu_to_be32(NFS4_SET_TO_CLIENT_TIME);
-			ts = timespec64_to_timespec(iap->ia_mtime);
+			p = xdr_encode_nfstime4(p, &iap->ia_mtime);
-			p = xdr_encode_nfstime4(p, &ts);
 		} else
 			*p++ = cpu_to_be32(NFS4_SET_TO_SERVER_TIME);
 	}
 }
 static __be32 *
-xdr_decode_nfstime4(__be32 *p, struct timespec *t)
+xdr_decode_nfstime4(__be32 *p, struct timespec64 *t)
 {
 	__u64 sec;
 	p = xdr_decode_hyper(p, &sec);
-	t-> tv_sec = (time_t)sec;
+	t-> tv_sec = sec;
 	t->tv_nsec = be32_to_cpup(p++);
 	return p;
 }
-static int decode_attr_time(struct xdr_stream *xdr, struct timespec *time)
+static int decode_attr_time(struct xdr_stream *xdr, struct timespec64 *time)
 {
 	__be32 *p;
 	return 0;
 }
-static int decode_attr_time_access(struct xdr_stream *xdr, uint32_t *bitmap, struct timespec *time)
+static int decode_attr_time_access(struct xdr_stream *xdr, uint32_t *bitmap, struct timespec64 *time)
 {
 	int status = 0;
 	return status;
 }
-static int decode_attr_time_metadata(struct xdr_stream *xdr, uint32_t *bitmap, struct timespec *time)
+static int decode_attr_time_metadata(struct xdr_stream *xdr, uint32_t *bitmap, struct timespec64 *time)
 {
 	int status = 0;
 }
 static int decode_attr_time_delta(struct xdr_stream *xdr, uint32_t *bitmap,
-				  struct timespec *time)
+				  struct timespec64 *time)
 {
 	int status = 0;
 	return status;
 }
-static int decode_attr_time_modify(struct xdr_stream *xdr, uint32_t *bitmap, struct timespec *time)
+static int decode_attr_time_modify(struct xdr_stream *xdr, uint32_t *bitmap, struct timespec64 *time)
 {
 	int status = 0;
 	PROC42(CLONE,		enc_clone,		dec_clone),
 	PROC42(COPY,		enc_copy,		dec_copy),
 	PROC42(OFFLOAD_CANCEL,	enc_offload_cancel,	dec_offload_cancel),
+	PROC42(COPY_NOTIFY,	enc_copy_notify,	dec_copy_notify),
 	PROC(LOOKUPP,		enc_lookupp,		dec_lookupp),
 	PROC42(LAYOUTERROR,	enc_layouterror,	dec_layouterror),
 };

fs/nfs/nfstrace.h+33−0

@@ -1065,6 +1065,39 @@ TRACE_EVENT(nfs_commit_done, · 1064 unmodified lines
1065	)	1065	)
1066	);	1066	);
1067		1067
		1068	TRACE_EVENT(nfs_fh_to_dentry,
		1069	TP_PROTO(
		1070	const struct super_block *sb,
		1071	const struct nfs_fh *fh,
		1072	u64 fileid,
		1073	int error
		1074	),
		1075
		1076	TP_ARGS(sb, fh, fileid, error),
		1077
		1078	TP_STRUCT__entry(
		1079	__field(int, error)
		1080	__field(dev_t, dev)
		1081	__field(u32, fhandle)
		1082	__field(u64, fileid)
		1083	),
		1084
		1085	TP_fast_assign(
		1086	__entry->error = error;
		1087	__entry->dev = sb->s_dev;
		1088	__entry->fileid = fileid;
		1089	__entry->fhandle = nfs_fhandle_hash(fh);
		1090	),
		1091
		1092	TP_printk(
		1093	"error=%d fileid=%02x:%02x:%llu fhandle=0x%08x ",
		1094	__entry->error,
		1095	MAJOR(__entry->dev), MINOR(__entry->dev),
		1096	(unsigned long long)__entry->fileid,
		1097	__entry->fhandle
		1098	)
		1099	);
		1100
1068	TRACE_DEFINE_ENUM(NFS_OK);	1101	TRACE_DEFINE_ENUM(NFS_OK);
1069	TRACE_DEFINE_ENUM(NFSERR_PERM);	1102	TRACE_DEFINE_ENUM(NFSERR_PERM);
1070	TRACE_DEFINE_ENUM(NFSERR_NOENT);	1103	TRACE_DEFINE_ENUM(NFSERR_NOENT);

fs/nfs/pnfs.c+0−2

@@ -2160,8 +2160,6 @@ out_unlock: · 2159 unmodified lines
2160	return NULL;	2160	return NULL;
2161	}	2161	}
2162		2162
2163	extern const nfs4_stateid current_stateid;
2164
2165	static void _lgopen_prepare_attached(struct nfs4_opendata *data,	2163	static void _lgopen_prepare_attached(struct nfs4_opendata *data,
2166	struct nfs_open_context *ctx)	2164	struct nfs_open_context *ctx)
2167	{	2165	{

fs/nfs/super.c+3−3

@@ -1592,7 +1592,7 @@ static int nfs_parse_mount_options(char *raw, · 1591 unmodified lines
1592	dfprintk(MOUNT, "NFS: invalid "	1592	dfprintk(MOUNT, "NFS: invalid "
1593	"lookupcache argument\n");	1593	"lookupcache argument\n");
1594	return 0;	1594	return 0;
1595	};	1595	}
1596	break;	1596	break;
1597	case Opt_fscache_uniq:	1597	case Opt_fscache_uniq:
1598	if (nfs_get_option_str(args, &mnt->fscache_uniq))	1598	if (nfs_get_option_str(args, &mnt->fscache_uniq))
@@ -1625,7 +1625,7 @@ static int nfs_parse_mount_options(char *raw, · 26 unmodified lines
1625	dfprintk(MOUNT, "NFS: invalid "	1625	dfprintk(MOUNT, "NFS: invalid "
1626	"local_lock argument\n");	1626	"local_lock argument\n");
1627	return 0;	1627	return 0;
1628	};	1628	}
1629	break;	1629	break;
1630		1630
1631	/*	1631	/*
@@ -2585,7 +2585,7 @@ static void nfs_get_cache_cookie(struct super_block *sb, · 953 unmodified lines
2585	if (mnt_s->fscache_key) {	2585	if (mnt_s->fscache_key) {
2586	uniq = mnt_s->fscache_key->key.uniquifier;	2586	uniq = mnt_s->fscache_key->key.uniquifier;
2587	ulen = mnt_s->fscache_key->key.uniq_len;	2587	ulen = mnt_s->fscache_key->key.uniq_len;
2588	};	2588	}
2589	} else	2589	} else
2590	return;	2590	return;
2591		2591

fs/nfs/sysfs.c+1−2

@@ -121,8 +121,7 @@ static void nfs_netns_client_release(struct kobject *kobj) · 120 unmodified lines
121	struct nfs_netns_client,	121	struct nfs_netns_client,
122	kobject);	122	kobject);
123		123
124	if (c->identifier)	124	kfree(c->identifier);
125	kfree(c->identifier);
126	kfree(c);	125	kfree(c);
127	}	126	}
128		127

include/linux/nfs4.h+26−0

@@ -16,6 +16,7 @@ · 15 unmodified lines
16	#include <linux/list.h>	16	#include <linux/list.h>
17	#include <linux/uidgid.h>	17	#include <linux/uidgid.h>
18	#include <uapi/linux/nfs4.h>	18	#include <uapi/linux/nfs4.h>
		19	#include <linux/sunrpc/msg_prot.h>
19		20
20	enum nfs4_acl_whotype {	21	enum nfs4_acl_whotype {
21	NFS4_ACL_WHO_NAMED = 0,	22	NFS4_ACL_WHO_NAMED = 0,
@@ -539,6 +540,8 @@ enum { · 517 unmodified lines
539		540
540	NFSPROC4_CLNT_LOOKUPP,	541	NFSPROC4_CLNT_LOOKUPP,
541	NFSPROC4_CLNT_LAYOUTERROR,	542	NFSPROC4_CLNT_LAYOUTERROR,
		543
		544	NFSPROC4_CLNT_COPY_NOTIFY,
542	};	545	};
543		546
544	/* nfs41 types */	547	/* nfs41 types */
@@ -674,4 +677,27 @@ struct nfs4_op_map { · 129 unmodified lines
674	} u;	677	} u;
675	};	678	};
676		679
		680	struct nfs42_netaddr {
		681	char netid[RPCBIND_MAXNETIDLEN];
		682	char addr[RPCBIND_MAXUADDRLEN + 1];
		683	u32 netid_len;
		684	u32 addr_len;
		685	};
		686
		687	enum netloc_type4 {
		688	NL4_NAME = 1,
		689	NL4_URL = 2,
		690	NL4_NETADDR = 3,
		691	};
		692
		693	struct nl4_server {
		694	enum netloc_type4 nl4_type;
		695	union {
		696	struct { /* NL4_NAME, NL4_URL */
		697	int nl4_str_sz;
		698	char nl4_str[NFS4_OPAQUE_LIMIT + 1];
		699	};
		700	struct nfs42_netaddr nl4_addr; /* NL4_NETADDR */
		701	} u;
		702	};
677	#endif	703	#endif

include/linux/nfs_fs.h+3−1

 struct nfs4_copy_state {
 	struct list_head	copies;
+	struct list_head	src_copies;
 	nfs4_stateid		stateid;
 	struct completion	completion;
 	uint64_t		count;
 	struct nfs_writeverf	verf;
 	int			error;
 	int			flags;
-	struct nfs4_state	*parent_state;
+	struct nfs4_state	*parent_src_state;
+	struct nfs4_state	*parent_dst_state;
 };
 /*

include/linux/nfs_fs_sb.h+5−1

 #define NFS_CS_INFINITE_SLOTS	3		/* - don't limit TCP slots */
 #define NFS_CS_NO_RETRANS_TIMEOUT	4	/* - Disable retransmit timeouts */
 #define NFS_CS_TSM_POSSIBLE	5		/* - Maybe state migration */
+#define NFS_CS_NOPING		6		/* - don't ping on connect */
+#define NFS_CS_DS		7		/* - Server is a DS */
+#define NFS_CS_REUSEPORT	8		/* - reuse src port on reconnect */
 	struct sockaddr_storage	cl_addr;	/* server identifier */
 	size_t			cl_addrlen;
 	char *			cl_hostname;	/* hostname of server */
 	struct nfs_fsid		fsid;
 	__u64			maxfilesize;	/* maximum file size */
-	struct timespec		time_delta;	/* smallest time granularity */
+	struct timespec64	time_delta;	/* smallest time granularity */
 	unsigned long		mount_time;	/* when this fs was mounted */
 	struct super_block	*super;		/* VFS super block */
 	dev_t			s_dev;		/* superblock dev numbers */
 #define NFS_CAP_COPY		(1U << 24)
 #define NFS_CAP_OFFLOAD_CANCEL	(1U << 25)
 #define NFS_CAP_LAYOUTERROR	(1U << 26)
+#define NFS_CAP_COPY_NOTIFY	(1U << 27)
 #endif

include/linux/nfs_xdr.h+24−7

 	struct nfs_fsid		fsid;
 	__u64			fileid;
 	__u64			mounted_on_fileid;
-	struct timespec		atime;
+	struct timespec64	atime;
-	struct timespec		mtime;
+	struct timespec64	mtime;
-	struct timespec		ctime;
+	struct timespec64	ctime;
 	__u64			change_attr;	/* NFSv4 change attribute */
 	__u64			pre_change_attr;/* pre-op NFSv4 change attribute */
 	__u64			pre_size;	/* pre_op_attr.size	  */
-	struct timespec		pre_mtime;	/* pre_op_attr.mtime	  */
+	struct timespec64	pre_mtime;	/* pre_op_attr.mtime	  */
-	struct timespec		pre_ctime;	/* pre_op_attr.ctime	  */
+	struct timespec64	pre_ctime;	/* pre_op_attr.ctime	  */
 	unsigned long		time_start;
 	unsigned long		gencount;
 	struct nfs4_string	*owner_name;
 	__u32			wtmult;	/* writes should be multiple of this */
 	__u32			dtpref;	/* pref. readdir transfer size */
 	__u64			maxfilesize;
-	struct timespec		time_delta; /* server time granularity */
+	struct timespec64	time_delta; /* server time granularity */
 	__u32			lease_time; /* in seconds */
 	__u32			nlayouttypes; /* number of layouttypes */
 	__u32			layouttype[NFS_MAX_LAYOUT_TYPES]; /* supported pnfs layout driver */
 	struct nfs_fh *		fh;
 	struct iattr *		sattr;
 	unsigned int		guard;
-	struct timespec		guardtime;
+	struct timespec64	guardtime;
 };
 struct nfs3_diropargs {
 	u64				count;
 	bool				sync;
+	struct nl4_server		*cp_src;
 };
 struct nfs42_write_res {
 	int				osr_status;
 };
+struct nfs42_copy_notify_args {
+	struct nfs4_sequence_args	cna_seq_args;
+	struct nfs_fh		*cna_src_fh;
+	nfs4_stateid		cna_src_stateid;
+	struct nl4_server	cna_dst;
+};
+struct nfs42_copy_notify_res {
+	struct nfs4_sequence_res	cnr_seq_res;
+	struct nfstime4		cnr_lease_time;
+	nfs4_stateid		cnr_stateid;
+	struct nl4_server	cnr_src;
+};
 struct nfs42_seek_args {
 	struct nfs4_sequence_args	seq_args;

include/linux/sunrpc/clnt.h+1−0

@@ -149,6 +149,7 @@ struct rpc_add_xprt_test { · 148 unmodified lines
149	#define RPC_CLNT_CREATE_NO_IDLE_TIMEOUT (1UL << 8)	149	#define RPC_CLNT_CREATE_NO_IDLE_TIMEOUT (1UL << 8)
150	#define RPC_CLNT_CREATE_NO_RETRANS_TIMEOUT (1UL << 9)	150	#define RPC_CLNT_CREATE_NO_RETRANS_TIMEOUT (1UL << 9)
151	#define RPC_CLNT_CREATE_SOFTERR (1UL << 10)	151	#define RPC_CLNT_CREATE_SOFTERR (1UL << 10)
		152	#define RPC_CLNT_CREATE_REUSEPORT (1UL << 11)
152		153
153	struct rpc_clnt rpc_create(struct rpc_create_args args);	154	struct rpc_clnt rpc_create(struct rpc_create_args args);
154	struct rpc_clnt rpc_bind_new_program(struct rpc_clnt ,	155	struct rpc_clnt rpc_bind_new_program(struct rpc_clnt ,

include/linux/sunrpc/xprt.h+2−1

 	unsigned int		min_reqs;	/* min number of slots */
 	unsigned int		num_reqs;	/* total slots */
 	unsigned long		state;		/* transport state */
-	unsigned char		resvport   : 1; /* use a reserved port */
+	unsigned char		resvport   : 1,	/* use a reserved port */
+				reuseport  : 1; /* reuse port on reconnect */
 	atomic_t		swapper;	/* we're swapping over this
 						   transport */
 	unsigned int		bind_index;	/* bind function index */

include/trace/events/rpcrdma.h+132−66

 				),					\
 				TP_ARGS(r_xprt))
+DECLARE_EVENT_CLASS(xprtrdma_connect_class,
+	TP_PROTO(
+		const struct rpcrdma_xprt *r_xprt,
+		int rc
+	),
+	TP_ARGS(r_xprt, rc),
+	TP_STRUCT__entry(
+		__field(const void *, r_xprt)
+		__field(int, rc)
+		__field(int, connect_status)
+		__string(addr, rpcrdma_addrstr(r_xprt))
+		__string(port, rpcrdma_portstr(r_xprt))
+	),
+	TP_fast_assign(
+		__entry->r_xprt = r_xprt;
+		__entry->rc = rc;
+		__entry->connect_status = r_xprt->rx_ep.rep_connected;
+		__assign_str(addr, rpcrdma_addrstr(r_xprt));
+		__assign_str(port, rpcrdma_portstr(r_xprt));
+	),
+	TP_printk("peer=[%s]:%s r_xprt=%p: rc=%d connect status=%d",
+		__get_str(addr), __get_str(port), __entry->r_xprt,
+		__entry->rc, __entry->connect_status
+	)
+);
+#define DEFINE_CONN_EVENT(name)						\
+		DEFINE_EVENT(xprtrdma_connect_class, xprtrdma_##name,	\
+				TP_PROTO(				\
+					const struct rpcrdma_xprt *r_xprt, \
+					int rc				\
+				),					\
+				TP_ARGS(r_xprt, rc))
 DECLARE_EVENT_CLASS(xprtrdma_rdch_event,
 	TP_PROTO(
 		const struct rpc_task *task,
 	)
 );
-TRACE_EVENT(xprtrdma_disconnect,
+TRACE_EVENT(xprtrdma_inline_thresh,
 	TP_PROTO(
-		const struct rpcrdma_xprt *r_xprt,
+		const struct rpcrdma_xprt *r_xprt
-		int status
 	),
-	TP_ARGS(r_xprt, status),
+	TP_ARGS(r_xprt),
 	TP_STRUCT__entry(
 		__field(const void *, r_xprt)
-		__field(int, status)
+		__field(unsigned int, inline_send)
-		__field(int, connected)
+		__field(unsigned int, inline_recv)
+		__field(unsigned int, max_send)
+		__field(unsigned int, max_recv)
 		__string(addr, rpcrdma_addrstr(r_xprt))
 		__string(port, rpcrdma_portstr(r_xprt))
 	),
 	TP_fast_assign(
+		const struct rpcrdma_ep *ep = &r_xprt->rx_ep;
 		__entry->r_xprt = r_xprt;
-		__entry->status = status;
+		__entry->inline_send = ep->rep_inline_send;
-		__entry->connected = r_xprt->rx_ep.rep_connected;
+		__entry->inline_recv = ep->rep_inline_recv;
+		__entry->max_send = ep->rep_max_inline_send;
+		__entry->max_recv = ep->rep_max_inline_recv;
 		__assign_str(addr, rpcrdma_addrstr(r_xprt));
 		__assign_str(port, rpcrdma_portstr(r_xprt));
 	),
-	TP_printk("peer=[%s]:%s r_xprt=%p: status=%d %sconnected",
+	TP_printk("peer=[%s]:%s r_xprt=%p neg send/recv=%u/%u, calc send/recv=%u/%u",
-		__get_str(addr), __get_str(port),
+		__get_str(addr), __get_str(port), __entry->r_xprt,
-		__entry->r_xprt, __entry->status,
+		__entry->inline_send, __entry->inline_recv,
-		__entry->connected == 1 ? "still " : "dis"
+		__entry->max_send, __entry->max_recv
 	)
 );
-DEFINE_RXPRT_EVENT(xprtrdma_conn_start);
+DEFINE_CONN_EVENT(connect);
-DEFINE_RXPRT_EVENT(xprtrdma_conn_tout);
+DEFINE_CONN_EVENT(disconnect);
 DEFINE_RXPRT_EVENT(xprtrdma_create);
 DEFINE_RXPRT_EVENT(xprtrdma_op_destroy);
 DEFINE_RXPRT_EVENT(xprtrdma_remove);
 DEFINE_RXPRT_EVENT(xprtrdma_reinsert);
-DEFINE_RXPRT_EVENT(xprtrdma_reconnect);
 DEFINE_RXPRT_EVENT(xprtrdma_op_inject_dsc);
 DEFINE_RXPRT_EVENT(xprtrdma_op_close);
-DEFINE_RXPRT_EVENT(xprtrdma_op_connect);
+DEFINE_RXPRT_EVENT(xprtrdma_op_setport);
+TRACE_EVENT(xprtrdma_op_connect,
+	TP_PROTO(
+		const struct rpcrdma_xprt *r_xprt,
+		unsigned long delay
+	),
+	TP_ARGS(r_xprt, delay),
+	TP_STRUCT__entry(
+		__field(const void *, r_xprt)
+		__field(unsigned long, delay)
+		__string(addr, rpcrdma_addrstr(r_xprt))
+		__string(port, rpcrdma_portstr(r_xprt))
+	),
+	TP_fast_assign(
+		__entry->r_xprt = r_xprt;
+		__entry->delay = delay;
+		__assign_str(addr, rpcrdma_addrstr(r_xprt));
+		__assign_str(port, rpcrdma_portstr(r_xprt));
+	),
+	TP_printk("peer=[%s]:%s r_xprt=%p delay=%lu",
+		__get_str(addr), __get_str(port), __entry->r_xprt,
+		__entry->delay
+	)
+);
 TRACE_EVENT(xprtrdma_op_set_cto,
 	TP_PROTO(
 DEFINE_WRCH_EVENT(reply);
 TRACE_DEFINE_ENUM(rpcrdma_noch);
+TRACE_DEFINE_ENUM(rpcrdma_noch_pullup);
+TRACE_DEFINE_ENUM(rpcrdma_noch_mapped);
 TRACE_DEFINE_ENUM(rpcrdma_readch);
 TRACE_DEFINE_ENUM(rpcrdma_areadch);
 TRACE_DEFINE_ENUM(rpcrdma_writech);
 #define xprtrdma_show_chunktype(x)					\
 		__print_symbolic(x,					\
 				{ rpcrdma_noch, "inline" },		\
+				{ rpcrdma_noch_pullup, "pullup" },	\
+				{ rpcrdma_noch_mapped, "mapped" },	\
 				{ rpcrdma_readch, "read list" },	\
 				{ rpcrdma_areadch, "*read list" },	\
 				{ rpcrdma_writech, "write list" },	\
 		__entry->client_id = rqst->rq_task->tk_client ?
 				     rqst->rq_task->tk_client->cl_clid : -1;
 		__entry->req = req;
-		__entry->num_sge = req->rl_sendctx->sc_wr.num_sge;
+		__entry->num_sge = req->rl_wr.num_sge;
-		__entry->signaled = req->rl_sendctx->sc_wr.send_flags &
+		__entry->signaled = req->rl_wr.send_flags & IB_SEND_SIGNALED;
-				    IB_SEND_SIGNALED;
 		__entry->status = status;
 	),
 	)
 );
+TRACE_EVENT(xprtrdma_post_linv,
+	TP_PROTO(
+		const struct rpcrdma_req *req,
+		int status
+	),
+	TP_ARGS(req, status),
+	TP_STRUCT__entry(
+		__field(const void *, req)
+		__field(int, status)
+		__field(u32, xid)
+	),
+	TP_fast_assign(
+		__entry->req = req;
+		__entry->status = status;
+		__entry->xid = be32_to_cpu(req->rl_slot.rq_xid);
+	),
+	TP_printk("req=%p xid=0x%08x status=%d",
+		__entry->req, __entry->xid, __entry->status
+	)
+);
 /**
  ** Completion events
  **/
 TRACE_EVENT(xprtrdma_fixup,
 	TP_PROTO(
 		const struct rpc_rqst *rqst,
-		int len,
+		unsigned long fixup
-		int hdrlen
 	),
-	TP_ARGS(rqst, len, hdrlen),
+	TP_ARGS(rqst, fixup),
 	TP_STRUCT__entry(
 		__field(unsigned int, task_id)
 		__field(unsigned int, client_id)
-		__field(const void *, base)
+		__field(unsigned long, fixup)
-		__field(int, len)
+		__field(size_t, headlen)
-		__field(int, hdrlen)
+		__field(unsigned int, pagelen)
-	),
+		__field(size_t, taillen)
-	TP_fast_assign(
-		__entry->task_id = rqst->rq_task->tk_pid;
-		__entry->client_id = rqst->rq_task->tk_client->cl_clid;
-		__entry->base = rqst->rq_rcv_buf.head[0].iov_base;
-		__entry->len = len;
-		__entry->hdrlen = hdrlen;
-	),
-	TP_printk("task:%u@%u base=%p len=%d hdrlen=%d",
-		__entry->task_id, __entry->client_id,
-		__entry->base, __entry->len, __entry->hdrlen
-	)
-);
-TRACE_EVENT(xprtrdma_fixup_pg,
-	TP_PROTO(
-		const struct rpc_rqst *rqst,
-		int pageno,
-		const void *pos,
-		int len,
-		int curlen
-	),
-	TP_ARGS(rqst, pageno, pos, len, curlen),
-	TP_STRUCT__entry(
-		__field(unsigned int, task_id)
-		__field(unsigned int, client_id)
-		__field(const void *, pos)
-		__field(int, pageno)
-		__field(int, len)
-		__field(int, curlen)
 	),
 	TP_fast_assign(
 		__entry->task_id = rqst->rq_task->tk_pid;
 		__entry->client_id = rqst->rq_task->tk_client->cl_clid;
-		__entry->pos = pos;
+		__entry->fixup = fixup;
-		__entry->pageno = pageno;
+		__entry->headlen = rqst->rq_rcv_buf.head[0].iov_len;
-		__entry->len = len;
+		__entry->pagelen = rqst->rq_rcv_buf.page_len;
-		__entry->curlen = curlen;
+		__entry->taillen = rqst->rq_rcv_buf.tail[0].iov_len;
 	),
-	TP_printk("task:%u@%u pageno=%d pos=%p len=%d curlen=%d",
+	TP_printk("task:%u@%u fixup=%lu xdr=%zu/%u/%zu",
-		__entry->task_id, __entry->client_id,
+		__entry->task_id, __entry->client_id, __entry->fixup,
-		__entry->pageno, __entry->pos, __entry->len, __entry->curlen
+		__entry->headlen, __entry->pagelen, __entry->taillen
 	)
 );

include/trace/events/sunrpc.h+94−0

@@ -165,6 +165,7 @@ DECLARE_EVENT_CLASS(rpc_task_running, · 164 unmodified lines
165	DEFINE_RPC_RUNNING_EVENT(begin);	165	DEFINE_RPC_RUNNING_EVENT(begin);
166	DEFINE_RPC_RUNNING_EVENT(run_action);	166	DEFINE_RPC_RUNNING_EVENT(run_action);
167	DEFINE_RPC_RUNNING_EVENT(complete);	167	DEFINE_RPC_RUNNING_EVENT(complete);
		168	DEFINE_RPC_RUNNING_EVENT(end);
168		169
169	DECLARE_EVENT_CLASS(rpc_task_queued,	170	DECLARE_EVENT_CLASS(rpc_task_queued,
170		171
@@ -777,6 +778,99 @@ TRACE_EVENT(xprt_ping, · 606 unmodified lines
777	__get_str(addr), __get_str(port), __entry->status)	778	__get_str(addr), __get_str(port), __entry->status)
778	);	779	);
779		780
		781	DECLARE_EVENT_CLASS(xprt_writelock_event,
		782	TP_PROTO(
		783	const struct rpc_xprt xprt, const struct rpc_task task
		784	),
		785
		786	TP_ARGS(xprt, task),
		787
		788	TP_STRUCT__entry(
		789	__field(unsigned int, task_id)
		790	__field(unsigned int, client_id)
		791	__field(unsigned int, snd_task_id)
		792	),
		793
		794	TP_fast_assign(
		795	if (task) {
		796	__entry->task_id = task->tk_pid;
		797	__entry->client_id = task->tk_client ?
		798	task->tk_client->cl_clid : -1;
		799	} else {
		800	__entry->task_id = -1;
		801	__entry->client_id = -1;
		802	}
		803	__entry->snd_task_id = xprt->snd_task ?
		804	xprt->snd_task->tk_pid : -1;
		805	),
		806
		807	TP_printk("task:%u@%u snd_task:%u",
		808	__entry->task_id, __entry->client_id,
		809	__entry->snd_task_id)
		810	);
		811
		812	#define DEFINE_WRITELOCK_EVENT(name) \
		813	DEFINE_EVENT(xprt_writelock_event, xprt_##name, \
		814	TP_PROTO( \
		815	const struct rpc_xprt *xprt, \
		816	const struct rpc_task *task \
		817	), \
		818	TP_ARGS(xprt, task))
		819
		820	DEFINE_WRITELOCK_EVENT(reserve_xprt);
		821	DEFINE_WRITELOCK_EVENT(release_xprt);
		822
		823	DECLARE_EVENT_CLASS(xprt_cong_event,
		824	TP_PROTO(
		825	const struct rpc_xprt xprt, const struct rpc_task task
		826	),
		827
		828	TP_ARGS(xprt, task),
		829
		830	TP_STRUCT__entry(
		831	__field(unsigned int, task_id)
		832	__field(unsigned int, client_id)
		833	__field(unsigned int, snd_task_id)
		834	__field(unsigned long, cong)
		835	__field(unsigned long, cwnd)
		836	__field(bool, wait)
		837	),
		838
		839	TP_fast_assign(
		840	if (task) {
		841	__entry->task_id = task->tk_pid;
		842	__entry->client_id = task->tk_client ?
		843	task->tk_client->cl_clid : -1;
		844	} else {
		845	__entry->task_id = -1;
		846	__entry->client_id = -1;
		847	}
		848	__entry->snd_task_id = xprt->snd_task ?
		849	xprt->snd_task->tk_pid : -1;
		850	__entry->cong = xprt->cong;
		851	__entry->cwnd = xprt->cwnd;
		852	__entry->wait = test_bit(XPRT_CWND_WAIT, &xprt->state);
		853	),
		854
		855	TP_printk("task:%u@%u snd_task:%u cong=%lu cwnd=%lu%s",
		856	__entry->task_id, __entry->client_id,
		857	__entry->snd_task_id, __entry->cong, __entry->cwnd,
		858	__entry->wait ? " (wait)" : "")
		859	);
		860
		861	#define DEFINE_CONG_EVENT(name) \
		862	DEFINE_EVENT(xprt_cong_event, xprt_##name, \
		863	TP_PROTO( \
		864	const struct rpc_xprt *xprt, \
		865	const struct rpc_task *task \
		866	), \
		867	TP_ARGS(xprt, task))
		868
		869	DEFINE_CONG_EVENT(reserve_cong);
		870	DEFINE_CONG_EVENT(release_cong);
		871	DEFINE_CONG_EVENT(get_cong);
		872	DEFINE_CONG_EVENT(put_cong);
		873
780	TRACE_EVENT(xs_stream_read_data,	874	TRACE_EVENT(xs_stream_read_data,
781	TP_PROTO(struct rpc_xprt *xprt, ssize_t err, size_t total),	875	TP_PROTO(struct rpc_xprt *xprt, ssize_t err, size_t total),
782		876

net/sunrpc/clnt.c+8−13

 	xprt->resvport = 1;
 	if (args->flags & RPC_CLNT_CREATE_NONPRIVPORT)
 		xprt->resvport = 0;
+	xprt->reuseport = 0;
+	if (args->flags & RPC_CLNT_CREATE_REUSEPORT)
+		xprt->reuseport = 1;
 	clnt = rpc_create_xprt(args, xprt);
 	if (IS_ERR(clnt) || args->nconnect <= 1)
 			return;
 		}
-		printk(KERN_ERR "%s: status=%d, but no request slot, exiting\n",
-				__func__, status);
 		rpc_call_rpcerror(task, -EIO);
 		return;
 	}
 	 * Even though there was an error, we may have acquired
 	 * a request slot somehow.  Make sure not to leak it.
 	 */
-	if (task->tk_rqstp) {
+	if (task->tk_rqstp)
-		printk(KERN_ERR "%s: status=%d, request allocated anyway\n",
-				__func__, status);
 		xprt_release(task);
-	}
 	switch (status) {
 	case -ENOMEM:
 	case -EAGAIN:	/* woken up; retry */
 		task->tk_action = call_retry_reserve;
 		return;
-	case -EIO:	/* probably a shutdown */
-		break;
 	default:
-		printk(KERN_ERR "%s: unrecognized error %d, exiting\n",
+		rpc_call_rpcerror(task, status);
-				__func__, status);
-		break;
 	}
-	rpc_call_rpcerror(task, status);
 }
 /*
 	struct rpc_xprt *xprt;
 	unsigned long connect_timeout;
 	unsigned long reconnect_timeout;
-	unsigned char resvport;
+	unsigned char resvport, reuseport;
 	int ret = 0;
 	rcu_read_lock();
 		return -EAGAIN;
 	}
 	resvport = xprt->resvport;
+	reuseport = xprt->reuseport;
 	connect_timeout = xprt->connect_timeout;
 	reconnect_timeout = xprt->max_reconnect_timeout;
 	rcu_read_unlock();
 		goto out_put_switch;
 	}
 	xprt->resvport = resvport;
+	xprt->reuseport = reuseport;
 	if (xprt->ops->set_connect_timeout != NULL)
 		xprt->ops->set_connect_timeout(xprt,
 				connect_timeout,

net/sunrpc/sched.c+2−1

 	rpc_reset_waitqueue_priority(queue);
 	queue->qlen = 0;
 	queue->timer_list.expires = 0;
-	INIT_DEFERRABLE_WORK(&queue->timer_list.dwork, __rpc_queue_timer_fn);
+	INIT_DELAYED_WORK(&queue->timer_list.dwork, __rpc_queue_timer_fn);
 	INIT_LIST_HEAD(&queue->timer_list.list);
 	rpc_assign_waitqueue_name(queue, qname);
 }
  */
 void rpc_exit_task(struct rpc_task *task)
 {
+	trace_rpc_task_end(task, task->tk_action);
 	task->tk_action = NULL;
 	if (task->tk_ops->rpc_count_stats)
 		task->tk_ops->rpc_count_stats(task, task->tk_calldata);

net/sunrpc/xdr.c+5−6

@@ -436,13 +436,12 @@ xdr_shrink_bufhead(struct xdr_buf *buf, size_t len) · 435 unmodified lines
436	}	436	}
437		437
438	/**	438	/**
439	* xdr_shrink_pagelen	439	* xdr_shrink_pagelen - shrinks buf->pages by up to @len bytes
440	* @buf: xdr_buf	440	* @buf: xdr_buf
441	* @len: bytes to remove from buf->pages	441	* @len: bytes to remove from buf->pages
442	*	442	*
443	* Shrinks XDR buffer's page array buf->pages by	443	* The extra data is not lost, but is instead moved into buf->tail.
444	* 'len' bytes. The extra data is not lost, but is instead	444	* Returns the actual number of bytes moved.
445	* moved into the tail.
446	*/	445	*/
447	static unsigned int	446	static unsigned int
448	xdr_shrink_pagelen(struct xdr_buf *buf, size_t len)	447	xdr_shrink_pagelen(struct xdr_buf *buf, size_t len)
@@ -455,8 +454,8 @@ xdr_shrink_pagelen(struct xdr_buf *buf, size_t len) · 6 unmodified lines
455		454
456	result = 0;	455	result = 0;
457	tail = buf->tail;	456	tail = buf->tail;
458	BUG_ON (len > pglen);	457	if (len > buf->page_len)
459		458	len = buf-> page_len;
460	tailbuf_len = buf->buflen - buf->head->iov_len - buf->page_len;	459	tailbuf_len = buf->buflen - buf->head->iov_len - buf->page_len;
461		460
462	/* Shift the tail first */	461	/* Shift the tail first */

net/sunrpc/xprt.c+13−9

 	if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) {
 		if (task == xprt->snd_task)
-			return 1;
+			goto out_locked;
 		goto out_sleep;
 	}
 	if (test_bit(XPRT_WRITE_SPACE, &xprt->state))
 		goto out_unlock;
 	xprt->snd_task = task;
+out_locked:
+	trace_xprt_reserve_xprt(xprt, task);
 	return 1;
 out_unlock:
 	xprt_clear_locked(xprt);
 out_sleep:
-	dprintk("RPC: %5u failed to lock transport %p\n",
-			task->tk_pid, xprt);
 	task->tk_status = -EAGAIN;
 	if  (RPC_IS_SOFT(task))
 		rpc_sleep_on_timeout(&xprt->sending, task, NULL,
 	if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) {
 		if (task == xprt->snd_task)
-			return 1;
+			goto out_locked;
 		goto out_sleep;
 	}
 	if (req == NULL) {
 		xprt->snd_task = task;
-		return 1;
+		goto out_locked;
 	}
 	if (test_bit(XPRT_WRITE_SPACE, &xprt->state))
 		goto out_unlock;
 	if (!xprt_need_congestion_window_wait(xprt)) {
 		xprt->snd_task = task;
-		return 1;
+		goto out_locked;
 	}
 out_unlock:
 	xprt_clear_locked(xprt);
 out_sleep:
-	dprintk("RPC: %5u failed to lock transport %p\n", task->tk_pid, xprt);
 	task->tk_status = -EAGAIN;
 	if (RPC_IS_SOFT(task))
 		rpc_sleep_on_timeout(&xprt->sending, task, NULL,
 	else
 		rpc_sleep_on(&xprt->sending, task, NULL);
 	return 0;
+out_locked:
+	trace_xprt_reserve_cong(xprt, task);
+	return 1;
 }
 EXPORT_SYMBOL_GPL(xprt_reserve_xprt_cong);
 		xprt_clear_locked(xprt);
 		__xprt_lock_write_next(xprt);
 	}
+	trace_xprt_release_xprt(xprt, task);
 }
 EXPORT_SYMBOL_GPL(xprt_release_xprt);
 		xprt_clear_locked(xprt);
 		__xprt_lock_write_next_cong(xprt);
 	}
+	trace_xprt_release_cong(xprt, task);
 }
 EXPORT_SYMBOL_GPL(xprt_release_xprt_cong);
 {
 	if (req->rq_cong)
 		return 1;
-	dprintk("RPC: %5u xprt_cwnd_limited cong = %lu cwnd = %lu\n",
+	trace_xprt_get_cong(xprt, req->rq_task);
-			req->rq_task->tk_pid, xprt->cong, xprt->cwnd);
 	if (RPCXPRT_CONGESTED(xprt)) {
 		xprt_set_congestion_window_wait(xprt);
 		return 0;
 	req->rq_cong = 0;
 	xprt->cong -= RPC_CWNDSCALE;
 	xprt_test_and_clear_congestion_window_wait(xprt);
+	trace_xprt_put_cong(xprt, req->rq_task);
 	__xprt_lock_write_next_cong(xprt);
 }

net/sunrpc/xprtrdma/backchannel.c+1−1

@@ -79,7 +79,7 @@ static int rpcrdma_bc_marshal_reply(struct rpc_rqst *rqst) · 78 unmodified lines
79	*p = xdr_zero;	79	*p = xdr_zero;
80		80
81	if (rpcrdma_prepare_send_sges(r_xprt, req, RPCRDMA_HDRLEN_MIN,	81	if (rpcrdma_prepare_send_sges(r_xprt, req, RPCRDMA_HDRLEN_MIN,
82	&rqst->rq_snd_buf, rpcrdma_noch))	82	&rqst->rq_snd_buf, rpcrdma_noch_pullup))
83	return -EIO;	83	return -EIO;
84		84
85	trace_xprtrdma_cb_reply(rqst);	85	trace_xprtrdma_cb_reply(rqst);

net/sunrpc/xprtrdma/frwr_ops.c+11−42

  * connect worker from running concurrently.
  *
  * When the underlying transport disconnects, MRs that are in flight
- * are flushed and are likely unusable. Thus all flushed MRs are
+ * are flushed and are likely unusable. Thus all MRs are destroyed.
- * destroyed. New MRs are created on demand.
+ * New MRs are created on demand.
  */
 #include <linux/sunrpc/rpc_rdma.h>
 	kfree(mr);
 }
-static void frwr_mr_recycle(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr *mr)
+static void frwr_mr_recycle(struct rpcrdma_mr *mr)
 {
+	struct rpcrdma_xprt *r_xprt = mr->mr_xprt;
 	trace_xprtrdma_mr_recycle(mr);
 	if (mr->mr_dir != DMA_NONE) {
 	frwr_release_mr(mr);
 }
-/* MRs are dynamically allocated, so simply clean up and release the MR.
- * A replacement MR will subsequently be allocated on demand.
- */
-static void
-frwr_mr_recycle_worker(struct work_struct *work)
-{
-	struct rpcrdma_mr *mr = container_of(work, struct rpcrdma_mr,
-					     mr_recycle);
-	frwr_mr_recycle(mr->mr_xprt, mr);
-}
-/* frwr_recycle - Discard MRs
- * @req: request to reset
- *
- * Used after a reconnect. These MRs could be in flight, we can't
- * tell. Safe thing to do is release them.
- */
-void frwr_recycle(struct rpcrdma_req *req)
-{
-	struct rpcrdma_mr *mr;
-	while ((mr = rpcrdma_mr_pop(&req->rl_registered)))
-		frwr_mr_recycle(mr->mr_xprt, mr);
-}
 /* frwr_reset - Place MRs back on the free list
  * @req: request to reset
  *
 	struct ib_mr *frmr;
 	int rc;
-	/* NB: ib_alloc_mr and device drivers typically allocate
-	 *     memory with GFP_KERNEL.
-	 */
 	frmr = ib_alloc_mr(ia->ri_pd, ia->ri_mrtype, depth);
 	if (IS_ERR(frmr))
 		goto out_mr_err;
 	mr->frwr.fr_mr = frmr;
 	mr->mr_dir = DMA_NONE;
 	INIT_LIST_HEAD(&mr->mr_list);
-	INIT_WORK(&mr->mr_recycle, frwr_mr_recycle_worker);
 	init_completion(&mr->frwr.fr_linv_done);
 	sg_init_table(sg, depth);
 	struct ib_send_wr *post_wr;
 	struct rpcrdma_mr *mr;
-	post_wr = &req->rl_sendctx->sc_wr;
+	post_wr = &req->rl_wr;
 	list_for_each_entry(mr, &req->rl_registered, mr_list) {
 		struct rpcrdma_frwr *frwr;
 		post_wr = &frwr->fr_regwr.wr;
 	}
-	/* If ib_post_send fails, the next ->send_request for
-	 * @req will queue these MRs for recovery.
-	 */
 	return ib_post_send(ia->ri_id->qp, post_wr, NULL);
 }
 static void __frwr_release_mr(struct ib_wc *wc, struct rpcrdma_mr *mr)
 {
 	if (wc->status != IB_WC_SUCCESS)
-		rpcrdma_mr_recycle(mr);
+		frwr_mr_recycle(mr);
 	else
 		rpcrdma_mr_put(mr);
 }
 	 */
 	bad_wr = NULL;
 	rc = ib_post_send(r_xprt->rx_ia.ri_id->qp, first, &bad_wr);
-	trace_xprtrdma_post_send(req, rc);
 	/* The final LOCAL_INV WR in the chain is supposed to
 	 * do the wake. If it was never posted, the wake will
 	/* Recycle MRs in the LOCAL_INV chain that did not get posted.
 	 */
+	trace_xprtrdma_post_linv(req, rc);
 	while (bad_wr) {
 		frwr = container_of(bad_wr, struct rpcrdma_frwr,
 				    fr_invwr);
 		bad_wr = bad_wr->next;
 		list_del_init(&mr->mr_list);
-		rpcrdma_mr_recycle(mr);
+		frwr_mr_recycle(mr);
 	}
 }
 	 */
 	bad_wr = NULL;
 	rc = ib_post_send(r_xprt->rx_ia.ri_id->qp, first, &bad_wr);
-	trace_xprtrdma_post_send(req, rc);
 	if (!rc)
 		return;
 	/* Recycle MRs in the LOCAL_INV chain that did not get posted.
 	 */
+	trace_xprtrdma_post_linv(req, rc);
 	while (bad_wr) {
 		frwr = container_of(bad_wr, struct rpcrdma_frwr, fr_invwr);
 		mr = container_of(frwr, struct rpcrdma_mr, frwr);
 		bad_wr = bad_wr->next;
-		rpcrdma_mr_recycle(mr);
+		frwr_mr_recycle(mr);
 	}
 	/* The final LOCAL_INV WR in the chain is supposed to

net/sunrpc/xprtrdma/rpc_rdma.c+268−145

@@ -78,8 +78,6 @@ static unsigned int rpcrdma_max_call_header_size(unsigned int maxsegs) · 77 unmodified lines
78	size += rpcrdma_segment_maxsz * sizeof(__be32);	78	size += rpcrdma_segment_maxsz * sizeof(__be32);
79	size += sizeof(__be32); /* list discriminator */	79	size += sizeof(__be32); /* list discriminator */
80		80
81	dprintk("RPC: %s: max call header size = %u\n",
82	__func__, size);
83	return size;	81	return size;
84	}	82	}
85		83
@@ -100,8 +98,6 @@ static unsigned int rpcrdma_max_reply_header_size(unsigned int maxsegs) · 14 unmodified lines
100	size += maxsegs * rpcrdma_segment_maxsz * sizeof(__be32);	98	size += maxsegs * rpcrdma_segment_maxsz * sizeof(__be32);
101	size += sizeof(__be32); /* list discriminator */	99	size += sizeof(__be32); /* list discriminator */
102		100
103	dprintk("RPC: %s: max reply header size = %u\n",
104	__func__, size);
105	return size;	101	return size;
106	}	102	}
107		103
@@ -363,8 +359,7 @@ static struct rpcrdma_mr_seg rpcrdma_mr_prepare(struct rpcrdma_xprt r_xprt, · 255 unmodified lines
363	out_getmr_err:	359	out_getmr_err:
364	trace_xprtrdma_nomrs(req);	360	trace_xprtrdma_nomrs(req);
365	xprt_wait_for_buffer_space(&r_xprt->rx_xprt);	361	xprt_wait_for_buffer_space(&r_xprt->rx_xprt);
366	if (r_xprt->rx_ep.rep_connected != -ENODEV)	362	rpcrdma_mrs_refresh(r_xprt);
367	schedule_work(&r_xprt->rx_buf.rb_refresh_worker);
368	return ERR_PTR(-EAGAIN);	363	return ERR_PTR(-EAGAIN);
369	}	364	}
370		365
@@ -393,7 +388,7 @@ static int rpcrdma_encode_read_list(struct rpcrdma_xprt *r_xprt, · 22 unmodified lines
393	unsigned int pos;	388	unsigned int pos;
394	int nsegs;	389	int nsegs;
395		390
396	if (rtype == rpcrdma_noch)	391	if (rtype == rpcrdma_noch_pullup \|\| rtype == rpcrdma_noch_mapped)
397	goto done;	392	goto done;
398		393
399	pos = rqst->rq_snd_buf.head[0].iov_len;	394	pos = rqst->rq_snd_buf.head[0].iov_len;
@@ -565,6 +560,7 @@ static void rpcrdma_sendctx_done(struct kref *kref) · 165 unmodified lines
565	*/	560	*/
566	void rpcrdma_sendctx_unmap(struct rpcrdma_sendctx *sc)	561	void rpcrdma_sendctx_unmap(struct rpcrdma_sendctx *sc)
567	{	562	{
		563	struct rpcrdma_regbuf *rb = sc->sc_req->rl_sendbuf;
568	struct ib_sge *sge;	564	struct ib_sge *sge;
569		565
570	if (!sc->sc_unmap_count)	566	if (!sc->sc_unmap_count)
@@ -576,7 +572,7 @@ void rpcrdma_sendctx_unmap(struct rpcrdma_sendctx *sc) · 5 unmodified lines
576	*/	572	*/
577	for (sge = &sc->sc_sges[2]; sc->sc_unmap_count;	573	for (sge = &sc->sc_sges[2]; sc->sc_unmap_count;
578	++sge, --sc->sc_unmap_count)	574	++sge, --sc->sc_unmap_count)
579	ib_dma_unmap_page(sc->sc_device, sge->addr, sge->length,	575	ib_dma_unmap_page(rdmab_device(rb), sge->addr, sge->length,
580	DMA_TO_DEVICE);	576	DMA_TO_DEVICE);
581		577
582	kref_put(&sc->sc_req->rl_kref, rpcrdma_sendctx_done);	578	kref_put(&sc->sc_req->rl_kref, rpcrdma_sendctx_done);
@@ -589,149 +585,228 @@ static bool rpcrdma_prepare_hdr_sge(struct rpcrdma_xprt *r_xprt, · 6 unmodified lines
589	{	585	{
590	struct rpcrdma_sendctx *sc = req->rl_sendctx;	586	struct rpcrdma_sendctx *sc = req->rl_sendctx;
591	struct rpcrdma_regbuf *rb = req->rl_rdmabuf;	587	struct rpcrdma_regbuf *rb = req->rl_rdmabuf;
592	struct ib_sge *sge = sc->sc_sges;	588	struct ib_sge *sge = &sc->sc_sges[req->rl_wr.num_sge++];
593		589
594	if (!rpcrdma_regbuf_dma_map(r_xprt, rb))	590	if (!rpcrdma_regbuf_dma_map(r_xprt, rb))
595	goto out_regbuf;	591	return false;
596	sge->addr = rdmab_addr(rb);	592	sge->addr = rdmab_addr(rb);
597	sge->length = len;	593	sge->length = len;
598	sge->lkey = rdmab_lkey(rb);	594	sge->lkey = rdmab_lkey(rb);
599		595
600	ib_dma_sync_single_for_device(rdmab_device(rb), sge->addr, sge->length,	596	ib_dma_sync_single_for_device(rdmab_device(rb), sge->addr, sge->length,
601	DMA_TO_DEVICE);	597	DMA_TO_DEVICE);
602	sc->sc_wr.num_sge++;
603	return true;	598	return true;
604
605	out_regbuf:
606	pr_err("rpcrdma: failed to DMA map a Send buffer\n");
607	return false;
608	}	599	}
609		600
610	/* Prepare the Send SGEs. The head and tail iovec, and each entry	601	/* The head iovec is straightforward, as it is usually already
611	* in the page list, gets its own SGE.	602	* DMA-mapped. Sync the content that has changed.
612	*/	603	*/
613	static bool rpcrdma_prepare_msg_sges(struct rpcrdma_xprt *r_xprt,	604	static bool rpcrdma_prepare_head_iov(struct rpcrdma_xprt *r_xprt,
614	struct rpcrdma_req *req,	605	struct rpcrdma_req *req, unsigned int len)
615	struct xdr_buf *xdr,
616	enum rpcrdma_chunktype rtype)
617	{	606	{
618	struct rpcrdma_sendctx *sc = req->rl_sendctx;	607	struct rpcrdma_sendctx *sc = req->rl_sendctx;
619	unsigned int sge_no, page_base, len, remaining;	608	struct ib_sge *sge = &sc->sc_sges[req->rl_wr.num_sge++];
620	struct rpcrdma_regbuf *rb = req->rl_sendbuf;	609	struct rpcrdma_regbuf *rb = req->rl_sendbuf;
621	struct ib_sge *sge = sc->sc_sges;
622	struct page page, *ppages;
623		610
624	/* The head iovec is straightforward, as it is already
625	* DMA-mapped. Sync the content that has changed.
626	*/
627	if (!rpcrdma_regbuf_dma_map(r_xprt, rb))	611	if (!rpcrdma_regbuf_dma_map(r_xprt, rb))
628	goto out_regbuf;	612	return false;
629	sc->sc_device = rdmab_device(rb);
630	sge_no = 1;
631	sge[sge_no].addr = rdmab_addr(rb);
632	sge[sge_no].length = xdr->head[0].iov_len;
633	sge[sge_no].lkey = rdmab_lkey(rb);
634	ib_dma_sync_single_for_device(rdmab_device(rb), sge[sge_no].addr,
635	sge[sge_no].length, DMA_TO_DEVICE);
636
637	/* If there is a Read chunk, the page list is being handled
638	* via explicit RDMA, and thus is skipped here. However, the
639	* tail iovec may include an XDR pad for the page list, as
640	* well as additional content, and may not reside in the
641	* same page as the head iovec.
642	*/
643	if (rtype == rpcrdma_readch) {
644	len = xdr->tail[0].iov_len;
645		613
646	/* Do not include the tail if it is only an XDR pad */	614	sge->addr = rdmab_addr(rb);
647	if (len < 4)	615	sge->length = len;
648	goto out;	616	sge->lkey = rdmab_lkey(rb);
649		617
650	page = virt_to_page(xdr->tail[0].iov_base);	618	ib_dma_sync_single_for_device(rdmab_device(rb), sge->addr, sge->length,
651	page_base = offset_in_page(xdr->tail[0].iov_base);	619	DMA_TO_DEVICE);
		620	return true;
		621	}
652		622
653	/* If the content in the page list is an odd length,	623	/* If there is a page list present, DMA map and prepare an
654	* xdr_write_pages() has added a pad at the beginning	624	* SGE for each page to be sent.
655	* of the tail iovec. Force the tail's non-pad content	625	*/
656	* to land at the next XDR position in the Send message.	626	static bool rpcrdma_prepare_pagelist(struct rpcrdma_req *req,
657	*/	627	struct xdr_buf *xdr)
658	page_base += len & 3;	628	{
659	len -= len & 3;	629	struct rpcrdma_sendctx *sc = req->rl_sendctx;
660	goto map_tail;	630	struct rpcrdma_regbuf *rb = req->rl_sendbuf;
661	}	631	unsigned int page_base, len, remaining;
		632	struct page **ppages;
		633	struct ib_sge *sge;
662		634
663	/* If there is a page list present, temporarily DMA map	635	ppages = xdr->pages + (xdr->page_base >> PAGE_SHIFT);
664	* and prepare an SGE for each page to be sent.	636	page_base = offset_in_page(xdr->page_base);
665	*/	637	remaining = xdr->page_len;
666	if (xdr->page_len) {	638	while (remaining) {
667	ppages = xdr->pages + (xdr->page_base >> PAGE_SHIFT);	639	sge = &sc->sc_sges[req->rl_wr.num_sge++];
668	page_base = offset_in_page(xdr->page_base);	640	len = min_t(unsigned int, PAGE_SIZE - page_base, remaining);
669	remaining = xdr->page_len;	641	sge->addr = ib_dma_map_page(rdmab_device(rb), *ppages,
670	while (remaining) {	642	page_base, len, DMA_TO_DEVICE);
671	sge_no++;	643	if (ib_dma_mapping_error(rdmab_device(rb), sge->addr))
672	if (sge_no > RPCRDMA_MAX_SEND_SGES - 2)	644	goto out_mapping_err;
673	goto out_mapping_overflow;
674
675	len = min_t(u32, PAGE_SIZE - page_base, remaining);
676	sge[sge_no].addr =
677	ib_dma_map_page(rdmab_device(rb), *ppages,
678	page_base, len, DMA_TO_DEVICE);
679	if (ib_dma_mapping_error(rdmab_device(rb),
680	sge[sge_no].addr))
681	goto out_mapping_err;
682	sge[sge_no].length = len;
683	sge[sge_no].lkey = rdmab_lkey(rb);
684
685	sc->sc_unmap_count++;
686	ppages++;
687	remaining -= len;
688	page_base = 0;
689	}
690	}
691		645
692	/* The tail iovec is not always constructed in the same	646	sge->length = len;
693	* page where the head iovec resides (see, for example,	647	sge->lkey = rdmab_lkey(rb);
694	* gss_wrap_req_priv). To neatly accommodate that case,
695	* DMA map it separately.
696	*/
697	if (xdr->tail[0].iov_len) {
698	page = virt_to_page(xdr->tail[0].iov_base);
699	page_base = offset_in_page(xdr->tail[0].iov_base);
700	len = xdr->tail[0].iov_len;
701		648
702	map_tail:
703	sge_no++;
704	sge[sge_no].addr =
705	ib_dma_map_page(rdmab_device(rb), page, page_base, len,
706	DMA_TO_DEVICE);
707	if (ib_dma_mapping_error(rdmab_device(rb), sge[sge_no].addr))
708	goto out_mapping_err;
709	sge[sge_no].length = len;
710	sge[sge_no].lkey = rdmab_lkey(rb);
711	sc->sc_unmap_count++;	649	sc->sc_unmap_count++;
		650	ppages++;
		651	remaining -= len;
		652	page_base = 0;
712	}	653	}
713		654
714	out:
715	sc->sc_wr.num_sge += sge_no;
716	if (sc->sc_unmap_count)
717	kref_get(&req->rl_kref);
718	return true;	655	return true;
719		656
720	out_regbuf:	657	out_mapping_err:
721	pr_err("rpcrdma: failed to DMA map a Send buffer\n");	658	trace_xprtrdma_dma_maperr(sge->addr);
722	return false;	659	return false;
		660	}
723		661
724	out_mapping_overflow:	662	/* The tail iovec may include an XDR pad for the page list,
725	rpcrdma_sendctx_unmap(sc);	663	* as well as additional content, and may not reside in the
726	pr_err("rpcrdma: too many Send SGEs (%u)\n", sge_no);	664	* same page as the head iovec.
727	return false;	665	*/
		666	static bool rpcrdma_prepare_tail_iov(struct rpcrdma_req *req,
		667	struct xdr_buf *xdr,
		668	unsigned int page_base, unsigned int len)
		669	{
		670	struct rpcrdma_sendctx *sc = req->rl_sendctx;
		671	struct ib_sge *sge = &sc->sc_sges[req->rl_wr.num_sge++];
		672	struct rpcrdma_regbuf *rb = req->rl_sendbuf;
		673	struct page *page = virt_to_page(xdr->tail[0].iov_base);
		674
		675	sge->addr = ib_dma_map_page(rdmab_device(rb), page, page_base, len,
		676	DMA_TO_DEVICE);
		677	if (ib_dma_mapping_error(rdmab_device(rb), sge->addr))
		678	goto out_mapping_err;
		679
		680	sge->length = len;
		681	sge->lkey = rdmab_lkey(rb);
		682	++sc->sc_unmap_count;
		683	return true;
728		684
729	out_mapping_err:	685	out_mapping_err:
730	rpcrdma_sendctx_unmap(sc);	686	trace_xprtrdma_dma_maperr(sge->addr);
731	trace_xprtrdma_dma_maperr(sge[sge_no].addr);
732	return false;	687	return false;
733	}	688	}
734		689
		690	/* Copy the tail to the end of the head buffer.
		691	*/
		692	static void rpcrdma_pullup_tail_iov(struct rpcrdma_xprt *r_xprt,
		693	struct rpcrdma_req *req,
		694	struct xdr_buf *xdr)
		695	{
		696	unsigned char *dst;
		697
		698	dst = (unsigned char *)xdr->head[0].iov_base;
		699	dst += xdr->head[0].iov_len + xdr->page_len;
		700	memmove(dst, xdr->tail[0].iov_base, xdr->tail[0].iov_len);
		701	r_xprt->rx_stats.pullup_copy_count += xdr->tail[0].iov_len;
		702	}
		703
		704	/* Copy pagelist content into the head buffer.
		705	*/
		706	static void rpcrdma_pullup_pagelist(struct rpcrdma_xprt *r_xprt,
		707	struct rpcrdma_req *req,
		708	struct xdr_buf *xdr)
		709	{
		710	unsigned int len, page_base, remaining;
		711	struct page **ppages;
		712	unsigned char src, dst;
		713
		714	dst = (unsigned char *)xdr->head[0].iov_base;
		715	dst += xdr->head[0].iov_len;
		716	ppages = xdr->pages + (xdr->page_base >> PAGE_SHIFT);
		717	page_base = offset_in_page(xdr->page_base);
		718	remaining = xdr->page_len;
		719	while (remaining) {
		720	src = page_address(*ppages);
		721	src += page_base;
		722	len = min_t(unsigned int, PAGE_SIZE - page_base, remaining);
		723	memcpy(dst, src, len);
		724	r_xprt->rx_stats.pullup_copy_count += len;
		725
		726	ppages++;
		727	dst += len;
		728	remaining -= len;
		729	page_base = 0;
		730	}
		731	}
		732
		733	/* Copy the contents of @xdr into @rl_sendbuf and DMA sync it.
		734	* When the head, pagelist, and tail are small, a pull-up copy
		735	* is considerably less costly than DMA mapping the components
		736	* of @xdr.
		737	*
		738	* Assumptions:
		739	* - the caller has already verified that the total length
		740	* of the RPC Call body will fit into @rl_sendbuf.
		741	*/
		742	static bool rpcrdma_prepare_noch_pullup(struct rpcrdma_xprt *r_xprt,
		743	struct rpcrdma_req *req,
		744	struct xdr_buf *xdr)
		745	{
		746	if (unlikely(xdr->tail[0].iov_len))
		747	rpcrdma_pullup_tail_iov(r_xprt, req, xdr);
		748
		749	if (unlikely(xdr->page_len))
		750	rpcrdma_pullup_pagelist(r_xprt, req, xdr);
		751
		752	/* The whole RPC message resides in the head iovec now */
		753	return rpcrdma_prepare_head_iov(r_xprt, req, xdr->len);
		754	}
		755
		756	static bool rpcrdma_prepare_noch_mapped(struct rpcrdma_xprt *r_xprt,
		757	struct rpcrdma_req *req,
		758	struct xdr_buf *xdr)
		759	{
		760	struct kvec *tail = &xdr->tail[0];
		761
		762	if (!rpcrdma_prepare_head_iov(r_xprt, req, xdr->head[0].iov_len))
		763	return false;
		764	if (xdr->page_len)
		765	if (!rpcrdma_prepare_pagelist(req, xdr))
		766	return false;
		767	if (tail->iov_len)
		768	if (!rpcrdma_prepare_tail_iov(req, xdr,
		769	offset_in_page(tail->iov_base),
		770	tail->iov_len))
		771	return false;
		772
		773	if (req->rl_sendctx->sc_unmap_count)
		774	kref_get(&req->rl_kref);
		775	return true;
		776	}
		777
		778	static bool rpcrdma_prepare_readch(struct rpcrdma_xprt *r_xprt,
		779	struct rpcrdma_req *req,
		780	struct xdr_buf *xdr)
		781	{
		782	if (!rpcrdma_prepare_head_iov(r_xprt, req, xdr->head[0].iov_len))
		783	return false;
		784
		785	/* If there is a Read chunk, the page list is being handled
		786	* via explicit RDMA, and thus is skipped here.
		787	*/
		788
		789	/* Do not include the tail if it is only an XDR pad */
		790	if (xdr->tail[0].iov_len > 3) {
		791	unsigned int page_base, len;
		792
		793	/* If the content in the page list is an odd length,
		794	* xdr_write_pages() adds a pad at the beginning of
		795	* the tail iovec. Force the tail's non-pad content to
		796	* land at the next XDR position in the Send message.
		797	*/
		798	page_base = offset_in_page(xdr->tail[0].iov_base);
		799	len = xdr->tail[0].iov_len;
		800	page_base += len & 3;
		801	len -= len & 3;
		802	if (!rpcrdma_prepare_tail_iov(req, xdr, page_base, len))
		803	return false;
		804	kref_get(&req->rl_kref);
		805	}
		806
		807	return true;
		808	}
		809
735	/**	810	/**
736	* rpcrdma_prepare_send_sges - Construct SGEs for a Send WR	811	* rpcrdma_prepare_send_sges - Construct SGEs for a Send WR
737	* @r_xprt: controlling transport	812	* @r_xprt: controlling transport
@@ -742,31 +817,53 @@ out_mapping_err: · 4 unmodified lines
742	*	817	*
743	* Returns 0 on success; otherwise a negative errno is returned.	818	* Returns 0 on success; otherwise a negative errno is returned.
744	*/	819	*/
745	int	820	inline int rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt,
746	rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt,	821	struct rpcrdma_req *req, u32 hdrlen,
747	struct rpcrdma_req *req, u32 hdrlen,	822	struct xdr_buf *xdr,
748	struct xdr_buf *xdr, enum rpcrdma_chunktype rtype)	823	enum rpcrdma_chunktype rtype)
749	{	824	{
750	int ret;	825	int ret;
751		826
752	ret = -EAGAIN;	827	ret = -EAGAIN;
753	req->rl_sendctx = rpcrdma_sendctx_get_locked(r_xprt);	828	req->rl_sendctx = rpcrdma_sendctx_get_locked(r_xprt);
754	if (!req->rl_sendctx)	829	if (!req->rl_sendctx)
755	goto err;	830	goto out_nosc;
756	req->rl_sendctx->sc_wr.num_sge = 0;
757	req->rl_sendctx->sc_unmap_count = 0;	831	req->rl_sendctx->sc_unmap_count = 0;
758	req->rl_sendctx->sc_req = req;	832	req->rl_sendctx->sc_req = req;
759	kref_init(&req->rl_kref);	833	kref_init(&req->rl_kref);
		834	req->rl_wr.wr_cqe = &req->rl_sendctx->sc_cqe;
		835	req->rl_wr.sg_list = req->rl_sendctx->sc_sges;
		836	req->rl_wr.num_sge = 0;
		837	req->rl_wr.opcode = IB_WR_SEND;
760		838
761	ret = -EIO;	839	ret = -EIO;
762	if (!rpcrdma_prepare_hdr_sge(r_xprt, req, hdrlen))	840	if (!rpcrdma_prepare_hdr_sge(r_xprt, req, hdrlen))
763	goto err;	841	goto out_unmap;
764	if (rtype != rpcrdma_areadch)	842
765	if (!rpcrdma_prepare_msg_sges(r_xprt, req, xdr, rtype))	843	switch (rtype) {
766	goto err;	844	case rpcrdma_noch_pullup:
		845	if (!rpcrdma_prepare_noch_pullup(r_xprt, req, xdr))
		846	goto out_unmap;
		847	break;
		848	case rpcrdma_noch_mapped:
		849	if (!rpcrdma_prepare_noch_mapped(r_xprt, req, xdr))
		850	goto out_unmap;
		851	break;
		852	case rpcrdma_readch:
		853	if (!rpcrdma_prepare_readch(r_xprt, req, xdr))
		854	goto out_unmap;
		855	break;
		856	case rpcrdma_areadch:
		857	break;
		858	default:
		859	goto out_unmap;
		860	}
		861
767	return 0;	862	return 0;
768		863
769	err:	864	out_unmap:
		865	rpcrdma_sendctx_unmap(req->rl_sendctx);
		866	out_nosc:
770	trace_xprtrdma_prepsend_failed(&req->rl_slot, ret);	867	trace_xprtrdma_prepsend_failed(&req->rl_slot, ret);
771	return ret;	868	return ret;
772	}	869	}
@@ -796,6 +893,7 @@ rpcrdma_marshal_req(struct rpcrdma_xprt r_xprt, struct rpc_rqst rqst) · 23 unmodified lines
796	struct rpcrdma_req *req = rpcr_to_rdmar(rqst);	893	struct rpcrdma_req *req = rpcr_to_rdmar(rqst);
797	struct xdr_stream *xdr = &req->rl_stream;	894	struct xdr_stream *xdr = &req->rl_stream;
798	enum rpcrdma_chunktype rtype, wtype;	895	enum rpcrdma_chunktype rtype, wtype;
		896	struct xdr_buf *buf = &rqst->rq_snd_buf;
799	bool ddp_allowed;	897	bool ddp_allowed;
800	__be32 *p;	898	__be32 *p;
801	int ret;	899	int ret;
@@ -853,8 +951,9 @@ rpcrdma_marshal_req(struct rpcrdma_xprt r_xprt, struct rpc_rqst rqst) · 51 unmodified lines
853	*/	951	*/
854	if (rpcrdma_args_inline(r_xprt, rqst)) {	952	if (rpcrdma_args_inline(r_xprt, rqst)) {
855	*p++ = rdma_msg;	953	*p++ = rdma_msg;
856	rtype = rpcrdma_noch;	954	rtype = buf->len < rdmab_length(req->rl_sendbuf) ?
857	} else if (ddp_allowed && rqst->rq_snd_buf.flags & XDRBUF_WRITE) {	955	rpcrdma_noch_pullup : rpcrdma_noch_mapped;
		956	} else if (ddp_allowed && buf->flags & XDRBUF_WRITE) {
858	*p++ = rdma_msg;	957	*p++ = rdma_msg;
859	rtype = rpcrdma_readch;	958	rtype = rpcrdma_readch;
860	} else {	959	} else {
@@ -863,12 +962,6 @@ rpcrdma_marshal_req(struct rpcrdma_xprt r_xprt, struct rpc_rqst rqst) · 2 unmodified lines
863	rtype = rpcrdma_areadch;	962	rtype = rpcrdma_areadch;
864	}	963	}
865		964
866	/* If this is a retransmit, discard previously registered
867	* chunks. Very likely the connection has been replaced,
868	* so these registrations are invalid and unusable.
869	*/
870	frwr_recycle(req);
871
872	/* This implementation supports the following combinations	965	/* This implementation supports the following combinations
873	* of chunk lists in one RPC-over-RDMA Call message:	966	* of chunk lists in one RPC-over-RDMA Call message:
874	*	967	*
@@ -902,7 +995,7 @@ rpcrdma_marshal_req(struct rpcrdma_xprt r_xprt, struct rpc_rqst rqst) · 27 unmodified lines
902	goto out_err;	995	goto out_err;
903		996
904	ret = rpcrdma_prepare_send_sges(r_xprt, req, req->rl_hdrbuf.len,	997	ret = rpcrdma_prepare_send_sges(r_xprt, req, req->rl_hdrbuf.len,
905	&rqst->rq_snd_buf, rtype);	998	buf, rtype);
906	if (ret)	999	if (ret)
907	goto out_err;	1000	goto out_err;
908		1001
@@ -916,6 +1009,40 @@ out_err: · 7 unmodified lines
916	return ret;	1009	return ret;
917	}	1010	}
918		1011
		1012	static void __rpcrdma_update_cwnd_locked(struct rpc_xprt *xprt,
		1013	struct rpcrdma_buffer *buf,
		1014	u32 grant)
		1015	{
		1016	buf->rb_credits = grant;
		1017	xprt->cwnd = grant << RPC_CWNDSHIFT;
		1018	}
		1019
		1020	static void rpcrdma_update_cwnd(struct rpcrdma_xprt *r_xprt, u32 grant)
		1021	{
		1022	struct rpc_xprt *xprt = &r_xprt->rx_xprt;
		1023
		1024	spin_lock(&xprt->transport_lock);
		1025	__rpcrdma_update_cwnd_locked(xprt, &r_xprt->rx_buf, grant);
		1026	spin_unlock(&xprt->transport_lock);
		1027	}
		1028
		1029	/**
		1030	* rpcrdma_reset_cwnd - Reset the xprt's congestion window
		1031	* @r_xprt: controlling transport instance
		1032	*
		1033	* Prepare @r_xprt for the next connection by reinitializing
		1034	* its credit grant to one (see RFC 8166, Section 3.3.3).
		1035	*/
		1036	void rpcrdma_reset_cwnd(struct rpcrdma_xprt *r_xprt)
		1037	{
		1038	struct rpc_xprt *xprt = &r_xprt->rx_xprt;
		1039
		1040	spin_lock(&xprt->transport_lock);
		1041	xprt->cong = 0;
		1042	__rpcrdma_update_cwnd_locked(xprt, &r_xprt->rx_buf, 1);
		1043	spin_unlock(&xprt->transport_lock);
		1044	}
		1045
919	/**	1046	/**
920	* rpcrdma_inline_fixup - Scatter inline received data into rqst's iovecs	1047	* rpcrdma_inline_fixup - Scatter inline received data into rqst's iovecs
921	* @rqst: controlling RPC request	1048	* @rqst: controlling RPC request
@@ -955,7 +1082,6 @@ rpcrdma_inline_fixup(struct rpc_rqst rqst, char srcp, int copy_len, int pad) · 33 unmodified lines
955	curlen = rqst->rq_rcv_buf.head[0].iov_len;	1082	curlen = rqst->rq_rcv_buf.head[0].iov_len;
956	if (curlen > copy_len)	1083	if (curlen > copy_len)
957	curlen = copy_len;	1084	curlen = copy_len;
958	trace_xprtrdma_fixup(rqst, copy_len, curlen);
959	srcp += curlen;	1085	srcp += curlen;
960	copy_len -= curlen;	1086	copy_len -= curlen;
961		1087
@@ -975,8 +1101,6 @@ rpcrdma_inline_fixup(struct rpc_rqst rqst, char srcp, int copy_len, int pad) · 13 unmodified lines
975	if (curlen > pagelist_len)	1101	if (curlen > pagelist_len)
976	curlen = pagelist_len;	1102	curlen = pagelist_len;
977		1103
978	trace_xprtrdma_fixup_pg(rqst, i, srcp,
979	copy_len, curlen);
980	destp = kmap_atomic(ppages[i]);	1104	destp = kmap_atomic(ppages[i]);
981	memcpy(destp + page_base, srcp, curlen);	1105	memcpy(destp + page_base, srcp, curlen);
982	flush_dcache_page(ppages[i]);	1106	flush_dcache_page(ppages[i]);
@@ -1008,6 +1132,8 @@ rpcrdma_inline_fixup(struct rpc_rqst rqst, char srcp, int copy_len, int pad) · 25 unmodified lines
1008	rqst->rq_private_buf.tail[0].iov_base = srcp;	1132	rqst->rq_private_buf.tail[0].iov_base = srcp;
1009	}	1133	}
1010		1134
		1135	if (fixup_copy_count)
		1136	trace_xprtrdma_fixup(rqst, fixup_copy_count);
1011	return fixup_copy_count;	1137	return fixup_copy_count;
1012	}	1138	}
1013		1139
@@ -1356,12 +1482,9 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *rep) · 342 unmodified lines
1356	credits = 1; /* don't deadlock */	1482	credits = 1; /* don't deadlock */
1357	else if (credits > buf->rb_max_requests)	1483	else if (credits > buf->rb_max_requests)
1358	credits = buf->rb_max_requests;	1484	credits = buf->rb_max_requests;
1359	if (buf->rb_credits != credits) {	1485	if (buf->rb_credits != credits)
1360	spin_lock(&xprt->transport_lock);	1486	rpcrdma_update_cwnd(r_xprt, credits);
1361	buf->rb_credits = credits;	1487	rpcrdma_post_recvs(r_xprt, false);
1362	xprt->cwnd = credits << RPC_CWNDSHIFT;
1363	spin_unlock(&xprt->transport_lock);
1364	}
1365		1488
1366	req = rpcr_to_rdmar(rqst);	1489	req = rpcr_to_rdmar(rqst);
1367	if (req->rl_reply) {	1490	if (req->rl_reply) {

net/sunrpc/xprtrdma/transport.c+10−23

 	rc = rpcrdma_ep_connect(&r_xprt->rx_ep, &r_xprt->rx_ia);
 	xprt_clear_connecting(xprt);
 	if (r_xprt->rx_ep.rep_connected > 0) {
-		if (!xprt_test_and_set_connected(xprt)) {
+		xprt->stat.connect_count++;
-			xprt->stat.connect_count++;
+		xprt->stat.connect_time += (long)jiffies -
-			xprt->stat.connect_time += (long)jiffies -
+					   xprt->stat.connect_start;
-						   xprt->stat.connect_start;
+		xprt_set_connected(xprt);
-			xprt_wake_pending_tasks(xprt, -EAGAIN);
+		rc = -EAGAIN;
-		}
-	} else {
-		if (xprt_test_and_clear_connected(xprt))
-			xprt_wake_pending_tasks(xprt, rc);
 	}
+	xprt_wake_pending_tasks(xprt, rc);
 }
 /**
 		return;
 	rpcrdma_ep_disconnect(ep, ia);
-	/* Prepare @xprt for the next connection by reinitializing
-	 * its credit grant to one (see RFC 8166, Section 3.3.3).
-	 */
-	r_xprt->rx_buf.rb_credits = 1;
-	xprt->cwnd = RPC_CWNDSHIFT;
 out:
 	xprt->reestablish_timeout = 0;
 	++xprt->connect_cookie;
 	struct sockaddr *sap = (struct sockaddr *)&xprt->addr;
 	char buf[8];
-	dprintk("RPC:       %s: setting port for xprt %p (%s:%s) to %u\n",
-		__func__, xprt,
-		xprt->address_strings[RPC_DISPLAY_ADDR],
-		xprt->address_strings[RPC_DISPLAY_PORT],
-		port);
 	rpc_set_port(sap, port);
 	kfree(xprt->address_strings[RPC_DISPLAY_PORT]);
 	kfree(xprt->address_strings[RPC_DISPLAY_HEX_PORT]);
 	snprintf(buf, sizeof(buf), "%4hx", port);
 	xprt->address_strings[RPC_DISPLAY_HEX_PORT] = kstrdup(buf, GFP_KERNEL);
+	trace_xprtrdma_op_setport(container_of(xprt, struct rpcrdma_xprt,
+					       rx_xprt));
 }
 /**
 	struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);
 	unsigned long delay;
-	trace_xprtrdma_op_connect(r_xprt);
 	delay = 0;
 	if (r_xprt->rx_ep.rep_connected != 0) {
 		delay = xprt_reconnect_delay(xprt);
 		xprt_reconnect_backoff(xprt, RPCRDMA_INIT_REEST_TO);
 	}
+	trace_xprtrdma_op_connect(r_xprt, delay);
 	queue_delayed_work(xprtiod_workqueue, &r_xprt->rx_connect_worker,
 			   delay);
 }

net/sunrpc/xprtrdma/verbs.c+110−84

@@ -74,17 +74,17 @@ · 73 unmodified lines
74	/*	74	/*
75	* internal functions	75	* internal functions
76	*/	76	*/
77	static void rpcrdma_sendctx_put_locked(struct rpcrdma_sendctx *sc);	77	static void rpcrdma_sendctx_put_locked(struct rpcrdma_xprt *r_xprt,
		78	struct rpcrdma_sendctx *sc);
		79	static void rpcrdma_reqs_reset(struct rpcrdma_xprt *r_xprt);
78	static void rpcrdma_reps_destroy(struct rpcrdma_buffer *buf);	80	static void rpcrdma_reps_destroy(struct rpcrdma_buffer *buf);
79	static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt);	81	static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt);
80	static void rpcrdma_mrs_destroy(struct rpcrdma_buffer *buf);	82	static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt);
81	static void rpcrdma_mr_free(struct rpcrdma_mr *mr);
82	static struct rpcrdma_regbuf *	83	static struct rpcrdma_regbuf *
83	rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction,	84	rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction,
84	gfp_t flags);	85	gfp_t flags);
85	static void rpcrdma_regbuf_dma_unmap(struct rpcrdma_regbuf *rb);	86	static void rpcrdma_regbuf_dma_unmap(struct rpcrdma_regbuf *rb);
86	static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb);	87	static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb);
87	static void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp);
88		88
89	/* Wait for outstanding transport work to finish. ib_drain_qp	89	/* Wait for outstanding transport work to finish. ib_drain_qp
90	* handles the drains in the wrong order for us, so open code	90	* handles the drains in the wrong order for us, so open code
@@ -125,7 +125,7 @@ rpcrdma_qp_event_handler(struct ib_event event, void context) · 34 unmodified lines
125		125
126	/**	126	/**
127	* rpcrdma_wc_send - Invoked by RDMA provider for each polled Send WC	127	* rpcrdma_wc_send - Invoked by RDMA provider for each polled Send WC
128	* @cq: completion queue (ignored)	128	* @cq: completion queue
129	* @wc: completed WR	129	* @wc: completed WR
130	*	130	*
131	*/	131	*/
@@ -138,7 +138,7 @@ rpcrdma_wc_send(struct ib_cq cq, struct ib_wc wc) · 6 unmodified lines
138		138
139	/* WARNING: Only wr_cqe and status are reliable at this point */	139	/* WARNING: Only wr_cqe and status are reliable at this point */
140	trace_xprtrdma_wc_send(sc, wc);	140	trace_xprtrdma_wc_send(sc, wc);
141	rpcrdma_sendctx_put_locked(sc);	141	rpcrdma_sendctx_put_locked((struct rpcrdma_xprt *)cq->cq_context, sc);
142	}	142	}
143		143
144	/**	144	/**
@@ -170,7 +170,6 @@ rpcrdma_wc_receive(struct ib_cq cq, struct ib_wc wc) · 25 unmodified lines
170	rdmab_addr(rep->rr_rdmabuf),	170	rdmab_addr(rep->rr_rdmabuf),
171	wc->byte_len, DMA_FROM_DEVICE);	171	wc->byte_len, DMA_FROM_DEVICE);
172		172
173	rpcrdma_post_recvs(r_xprt, false);
174	rpcrdma_reply_handler(rep);	173	rpcrdma_reply_handler(rep);
175	return;	174	return;
176		175
@@ -178,11 +177,11 @@ out_flushed: · 1 unmodified line
178	rpcrdma_recv_buffer_put(rep);	177	rpcrdma_recv_buffer_put(rep);
179	}	178	}
180		179
181	static void	180	static void rpcrdma_update_cm_private(struct rpcrdma_xprt *r_xprt,
182	rpcrdma_update_connect_private(struct rpcrdma_xprt *r_xprt,	181	struct rdma_conn_param *param)
183	struct rdma_conn_param *param)
184	{	182	{
185	const struct rpcrdma_connect_private *pmsg = param->private_data;	183	const struct rpcrdma_connect_private *pmsg = param->private_data;
		184	struct rpcrdma_ep *ep = &r_xprt->rx_ep;
186	unsigned int rsize, wsize;	185	unsigned int rsize, wsize;
187		186
188	/* Default settings for RPC-over-RDMA Version One */	187	/* Default settings for RPC-over-RDMA Version One */
@@ -198,13 +197,11 @@ rpcrdma_update_connect_private(struct rpcrdma_xprt *r_xprt, · 9 unmodified lines
198	wsize = rpcrdma_decode_buffer_size(pmsg->cp_recv_size);	197	wsize = rpcrdma_decode_buffer_size(pmsg->cp_recv_size);
199	}	198	}
200		199
201	if (rsize < r_xprt->rx_ep.rep_inline_recv)	200	if (rsize < ep->rep_inline_recv)
202	r_xprt->rx_ep.rep_inline_recv = rsize;	201	ep->rep_inline_recv = rsize;
203	if (wsize < r_xprt->rx_ep.rep_inline_send)	202	if (wsize < ep->rep_inline_send)
204	r_xprt->rx_ep.rep_inline_send = wsize;	203	ep->rep_inline_send = wsize;
205	dprintk("RPC: %s: max send %u, max recv %u\n", __func__,	204
206	r_xprt->rx_ep.rep_inline_send,
207	r_xprt->rx_ep.rep_inline_recv);
208	rpcrdma_set_max_header_sizes(r_xprt);	205	rpcrdma_set_max_header_sizes(r_xprt);
209	}	206	}
210		207
@@ -258,7 +255,8 @@ rpcrdma_cm_event_handler(struct rdma_cm_id id, struct rdma_cm_event event) · 47 unmodified lines
258	case RDMA_CM_EVENT_ESTABLISHED:	255	case RDMA_CM_EVENT_ESTABLISHED:
259	++xprt->connect_cookie;	256	++xprt->connect_cookie;
260	ep->rep_connected = 1;	257	ep->rep_connected = 1;
261	rpcrdma_update_connect_private(r_xprt, &event->param.conn);	258	rpcrdma_update_cm_private(r_xprt, &event->param.conn);
		259	trace_xprtrdma_inline_thresh(r_xprt);
262	wake_up_all(&ep->rep_connect_wait);	260	wake_up_all(&ep->rep_connect_wait);
263	break;	261	break;
264	case RDMA_CM_EVENT_CONNECT_ERROR:	262	case RDMA_CM_EVENT_CONNECT_ERROR:
@@ -298,8 +296,6 @@ rpcrdma_create_id(struct rpcrdma_xprt xprt, struct rpcrdma_ia ia) · 33 unmodified lines
298	struct rdma_cm_id *id;	296	struct rdma_cm_id *id;
299	int rc;	297	int rc;
300		298
301	trace_xprtrdma_conn_start(xprt);
302
303	init_completion(&ia->ri_done);	299	init_completion(&ia->ri_done);
304	init_completion(&ia->ri_remove_done);	300	init_completion(&ia->ri_remove_done);
305		301
@@ -315,10 +311,8 @@ rpcrdma_create_id(struct rpcrdma_xprt xprt, struct rpcrdma_ia ia) · 9 unmodified lines
315	if (rc)	311	if (rc)
316	goto out;	312	goto out;
317	rc = wait_for_completion_interruptible_timeout(&ia->ri_done, wtimeout);	313	rc = wait_for_completion_interruptible_timeout(&ia->ri_done, wtimeout);
318	if (rc < 0) {	314	if (rc < 0)
319	trace_xprtrdma_conn_tout(xprt);
320	goto out;	315	goto out;
321	}
322		316
323	rc = ia->ri_async_rc;	317	rc = ia->ri_async_rc;
324	if (rc)	318	if (rc)
@@ -329,10 +323,8 @@ rpcrdma_create_id(struct rpcrdma_xprt xprt, struct rpcrdma_ia ia) · 4 unmodified lines
329	if (rc)	323	if (rc)
330	goto out;	324	goto out;
331	rc = wait_for_completion_interruptible_timeout(&ia->ri_done, wtimeout);	325	rc = wait_for_completion_interruptible_timeout(&ia->ri_done, wtimeout);
332	if (rc < 0) {	326	if (rc < 0)
333	trace_xprtrdma_conn_tout(xprt);
334	goto out;	327	goto out;
335	}
336	rc = ia->ri_async_rc;	328	rc = ia->ri_async_rc;
337	if (rc)	329	if (rc)
338	goto out;	330	goto out;
@@ -409,8 +401,6 @@ rpcrdma_ia_remove(struct rpcrdma_ia *ia) · 70 unmodified lines
409	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;	401	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
410	struct rpcrdma_req *req;	402	struct rpcrdma_req *req;
411		403
412	cancel_work_sync(&buf->rb_refresh_worker);
413
414	/* This is similar to rpcrdma_ep_destroy, but:	404	/* This is similar to rpcrdma_ep_destroy, but:
415	* - Don't cancel the connect worker.	405	* - Don't cancel the connect worker.
416	* - Don't call rpcrdma_ep_disconnect, which waits	406	* - Don't call rpcrdma_ep_disconnect, which waits
@@ -437,7 +427,7 @@ rpcrdma_ia_remove(struct rpcrdma_ia *ia) · 20 unmodified lines
437	rpcrdma_regbuf_dma_unmap(req->rl_sendbuf);	427	rpcrdma_regbuf_dma_unmap(req->rl_sendbuf);
438	rpcrdma_regbuf_dma_unmap(req->rl_recvbuf);	428	rpcrdma_regbuf_dma_unmap(req->rl_recvbuf);
439	}	429	}
440	rpcrdma_mrs_destroy(buf);	430	rpcrdma_mrs_destroy(r_xprt);
441	ib_dealloc_pd(ia->ri_pd);	431	ib_dealloc_pd(ia->ri_pd);
442	ia->ri_pd = NULL;	432	ia->ri_pd = NULL;
443		433
@@ -522,7 +512,7 @@ int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt) · 78 unmodified lines
522	init_waitqueue_head(&ep->rep_connect_wait);	512	init_waitqueue_head(&ep->rep_connect_wait);
523	ep->rep_receive_count = 0;	513	ep->rep_receive_count = 0;
524		514
525	sendcq = ib_alloc_cq_any(ia->ri_id->device, NULL,	515	sendcq = ib_alloc_cq_any(ia->ri_id->device, r_xprt,
526	ep->rep_attr.cap.max_send_wr + 1,	516	ep->rep_attr.cap.max_send_wr + 1,
527	IB_POLL_WORKQUEUE);	517	IB_POLL_WORKQUEUE);
528	if (IS_ERR(sendcq)) {	518	if (IS_ERR(sendcq)) {
@@ -630,8 +620,6 @@ static int rpcrdma_ep_recreate_xprt(struct rpcrdma_xprt *r_xprt, · 101 unmodified lines
630	pr_err("rpcrdma: rdma_create_qp returned %d\n", err);	620	pr_err("rpcrdma: rdma_create_qp returned %d\n", err);
631	goto out3;	621	goto out3;
632	}	622	}
633
634	rpcrdma_mrs_create(r_xprt);
635	return 0;	623	return 0;
636		624
637	out3:	625	out3:
@@ -649,8 +637,6 @@ static int rpcrdma_ep_reconnect(struct rpcrdma_xprt *r_xprt, · 11 unmodified lines
649	struct rdma_cm_id id, old;	637	struct rdma_cm_id id, old;
650	int err, rc;	638	int err, rc;
651		639
652	trace_xprtrdma_reconnect(r_xprt);
653
654	rpcrdma_ep_disconnect(&r_xprt->rx_ep, ia);	640	rpcrdma_ep_disconnect(&r_xprt->rx_ep, ia);
655		641
656	rc = -EHOSTUNREACH;	642	rc = -EHOSTUNREACH;
@@ -705,7 +691,6 @@ retry: · 48 unmodified lines
705	memcpy(&qp_init_attr, &ep->rep_attr, sizeof(qp_init_attr));	691	memcpy(&qp_init_attr, &ep->rep_attr, sizeof(qp_init_attr));
706	switch (ep->rep_connected) {	692	switch (ep->rep_connected) {
707	case 0:	693	case 0:
708	dprintk("RPC: %s: connecting...\n", __func__);
709	rc = rdma_create_qp(ia->ri_id, ia->ri_pd, &qp_init_attr);	694	rc = rdma_create_qp(ia->ri_id, ia->ri_pd, &qp_init_attr);
710	if (rc) {	695	if (rc) {
711	rc = -ENETUNREACH;	696	rc = -ENETUNREACH;
@@ -726,6 +711,7 @@ retry: · 14 unmodified lines
726	ep->rep_connected = 0;	711	ep->rep_connected = 0;
727	xprt_clear_connected(xprt);	712	xprt_clear_connected(xprt);
728		713
		714	rpcrdma_reset_cwnd(r_xprt);
729	rpcrdma_post_recvs(r_xprt, true);	715	rpcrdma_post_recvs(r_xprt, true);
730		716
731	rc = rdma_connect(ia->ri_id, &ep->rep_remote_cma);	717	rc = rdma_connect(ia->ri_id, &ep->rep_remote_cma);
@@ -742,13 +728,14 @@ retry: · 10 unmodified lines
742	goto out;	728	goto out;
743	}	729	}
744		730
745	dprintk("RPC: %s: connected\n", __func__);	731	rpcrdma_mrs_create(r_xprt);
746		732
747	out:	733	out:
748	if (rc)	734	if (rc)
749	ep->rep_connected = rc;	735	ep->rep_connected = rc;
750		736
751	out_noupdate:	737	out_noupdate:
		738	trace_xprtrdma_connect(r_xprt, rc);
752	return rc;	739	return rc;
753	}	740	}
754		741
@@ -757,11 +744,8 @@ out_noupdate: · 2 unmodified lines
757	* @ep: endpoint to disconnect	744	* @ep: endpoint to disconnect
758	* @ia: associated interface adapter	745	* @ia: associated interface adapter
759	*	746	*
760	* This is separate from destroy to facilitate the ability	747	* Caller serializes. Either the transport send lock is held,
761	* to reconnect without recreating the endpoint.	748	* or we're being called to destroy the transport.
762	*
763	* This call is not reentrant, and must not be made in parallel
764	* on the same endpoint.
765	*/	749	*/
766	void	750	void
767	rpcrdma_ep_disconnect(struct rpcrdma_ep ep, struct rpcrdma_ia ia)	751	rpcrdma_ep_disconnect(struct rpcrdma_ep ep, struct rpcrdma_ia ia)
@@ -780,6 +764,8 @@ rpcrdma_ep_disconnect(struct rpcrdma_ep ep, struct rpcrdma_ia ia) · 12 unmodified lines
780	trace_xprtrdma_disconnect(r_xprt, rc);	764	trace_xprtrdma_disconnect(r_xprt, rc);
781		765
782	rpcrdma_xprt_drain(r_xprt);	766	rpcrdma_xprt_drain(r_xprt);
		767	rpcrdma_reqs_reset(r_xprt);
		768	rpcrdma_mrs_destroy(r_xprt);
783	}	769	}
784		770
785	/* Fixed-size circular FIFO queue. This implementation is wait-free and	771	/* Fixed-size circular FIFO queue. This implementation is wait-free and
@@ -817,9 +803,6 @@ static struct rpcrdma_sendctx rpcrdma_sendctx_create(struct rpcrdma_ia ia) · 31 unmodified lines
817	if (!sc)	803	if (!sc)
818	return NULL;	804	return NULL;
819		805
820	sc->sc_wr.wr_cqe = &sc->sc_cqe;
821	sc->sc_wr.sg_list = sc->sc_sges;
822	sc->sc_wr.opcode = IB_WR_SEND;
823	sc->sc_cqe.done = rpcrdma_wc_send;	806	sc->sc_cqe.done = rpcrdma_wc_send;
824	return sc;	807	return sc;
825	}	808	}
@@ -847,7 +830,6 @@ static int rpcrdma_sendctxs_create(struct rpcrdma_xprt *r_xprt) · 21 unmodified lines
847	if (!sc)	830	if (!sc)
848	return -ENOMEM;	831	return -ENOMEM;
849		832
850	sc->sc_xprt = r_xprt;
851	buf->rb_sc_ctxs[i] = sc;	833	buf->rb_sc_ctxs[i] = sc;
852	}	834	}
853		835
@@ -910,6 +892,7 @@ out_emptyq: · 56 unmodified lines
910		892
911	/**	893	/**
912	* rpcrdma_sendctx_put_locked - Release a send context	894	* rpcrdma_sendctx_put_locked - Release a send context
		895	* @r_xprt: controlling transport instance
913	* @sc: send context to release	896	* @sc: send context to release
914	*	897	*
915	* Usage: Called from Send completion to return a sendctxt	898	* Usage: Called from Send completion to return a sendctxt
@@ -917,10 +900,10 @@ out_emptyq: · 1 unmodified line
917	*	900	*
918	* The caller serializes calls to this function (per transport).	901	* The caller serializes calls to this function (per transport).
919	*/	902	*/
920	static void	903	static void rpcrdma_sendctx_put_locked(struct rpcrdma_xprt *r_xprt,
921	rpcrdma_sendctx_put_locked(struct rpcrdma_sendctx *sc)	904	struct rpcrdma_sendctx *sc)
922	{	905	{
923	struct rpcrdma_buffer *buf = &sc->sc_xprt->rx_buf;	906	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
924	unsigned long next_tail;	907	unsigned long next_tail;
925		908
926	/* Unmap SGEs of previously completed but unsignaled	909	/* Unmap SGEs of previously completed but unsignaled
@@ -938,7 +921,7 @@ rpcrdma_sendctx_put_locked(struct rpcrdma_sendctx *sc) · 11 unmodified lines
938	/* Paired with READ_ONCE */	921	/* Paired with READ_ONCE */
939	smp_store_release(&buf->rb_sc_tail, next_tail);	922	smp_store_release(&buf->rb_sc_tail, next_tail);
940		923
941	xprt_write_space(&sc->sc_xprt->rx_xprt);	924	xprt_write_space(&r_xprt->rx_xprt);
942	}	925	}
943		926
944	static void	927	static void
@@ -965,7 +948,7 @@ rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt) · 20 unmodified lines
965	mr->mr_xprt = r_xprt;	948	mr->mr_xprt = r_xprt;
966		949
967	spin_lock(&buf->rb_lock);	950	spin_lock(&buf->rb_lock);
968	list_add(&mr->mr_list, &buf->rb_mrs);	951	rpcrdma_mr_push(mr, &buf->rb_mrs);
969	list_add(&mr->mr_all, &buf->rb_all_mrs);	952	list_add(&mr->mr_all, &buf->rb_all_mrs);
970	spin_unlock(&buf->rb_lock);	953	spin_unlock(&buf->rb_lock);
971	}	954	}
@@ -986,6 +969,28 @@ rpcrdma_mr_refresh_worker(struct work_struct *work) · 14 unmodified lines
986	xprt_write_space(&r_xprt->rx_xprt);	969	xprt_write_space(&r_xprt->rx_xprt);
987	}	970	}
988		971
		972	/**
		973	* rpcrdma_mrs_refresh - Wake the MR refresh worker
		974	* @r_xprt: controlling transport instance
		975	*
		976	*/
		977	void rpcrdma_mrs_refresh(struct rpcrdma_xprt *r_xprt)
		978	{
		979	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
		980	struct rpcrdma_ep *ep = &r_xprt->rx_ep;
		981
		982	/* If there is no underlying device, it's no use to
		983	* wake the refresh worker.
		984	*/
		985	if (ep->rep_connected != -ENODEV) {
		986	/* The work is scheduled on a WQ_MEM_RECLAIM
		987	* workqueue in order to prevent MR allocation
		988	* from recursing into NFS during direct reclaim.
		989	*/
		990	queue_work(xprtiod_workqueue, &buf->rb_refresh_worker);
		991	}
		992	}
		993
989	/**	994	/**
990	* rpcrdma_req_create - Allocate an rpcrdma_req object	995	* rpcrdma_req_create - Allocate an rpcrdma_req object
991	* @r_xprt: controlling r_xprt	996	* @r_xprt: controlling r_xprt
@@ -1042,6 +1047,26 @@ out1: · 50 unmodified lines
1042	return NULL;	1047	return NULL;
1043	}	1048	}
1044		1049
		1050	/**
		1051	* rpcrdma_reqs_reset - Reset all reqs owned by a transport
		1052	* @r_xprt: controlling transport instance
		1053	*
		1054	* ASSUMPTION: the rb_allreqs list is stable for the duration,
		1055	* and thus can be walked without holding rb_lock. Eg. the
		1056	* caller is holding the transport send lock to exclude
		1057	* device removal or disconnection.
		1058	*/
		1059	static void rpcrdma_reqs_reset(struct rpcrdma_xprt *r_xprt)
		1060	{
		1061	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
		1062	struct rpcrdma_req *req;
		1063
		1064	list_for_each_entry(req, &buf->rb_allreqs, rl_all) {
		1065	/* Credits are valid only for one connection */
		1066	req->rl_slot.rq_cong = 0;
		1067	}
		1068	}
		1069
1045	static struct rpcrdma_rep rpcrdma_rep_create(struct rpcrdma_xprt r_xprt,	1070	static struct rpcrdma_rep rpcrdma_rep_create(struct rpcrdma_xprt r_xprt,
1046	bool temp)	1071	bool temp)
1047	{	1072	{
@@ -1125,8 +1150,6 @@ int rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt) · 77 unmodified lines
1125	INIT_LIST_HEAD(&buf->rb_all_mrs);	1150	INIT_LIST_HEAD(&buf->rb_all_mrs);
1126	INIT_WORK(&buf->rb_refresh_worker, rpcrdma_mr_refresh_worker);	1151	INIT_WORK(&buf->rb_refresh_worker, rpcrdma_mr_refresh_worker);
1127		1152
1128	rpcrdma_mrs_create(r_xprt);
1129
1130	INIT_LIST_HEAD(&buf->rb_send_bufs);	1153	INIT_LIST_HEAD(&buf->rb_send_bufs);
1131	INIT_LIST_HEAD(&buf->rb_allreqs);	1154	INIT_LIST_HEAD(&buf->rb_allreqs);
1132		1155
@@ -1134,14 +1157,13 @@ int rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt) · 1 unmodified line
1134	for (i = 0; i < buf->rb_max_requests; i++) {	1157	for (i = 0; i < buf->rb_max_requests; i++) {
1135	struct rpcrdma_req *req;	1158	struct rpcrdma_req *req;
1136		1159
1137	req = rpcrdma_req_create(r_xprt, RPCRDMA_V1_DEF_INLINE_SIZE,	1160	req = rpcrdma_req_create(r_xprt, RPCRDMA_V1_DEF_INLINE_SIZE * 2,
1138	GFP_KERNEL);	1161	GFP_KERNEL);
1139	if (!req)	1162	if (!req)
1140	goto out;	1163	goto out;
1141	list_add(&req->rl_list, &buf->rb_send_bufs);	1164	list_add(&req->rl_list, &buf->rb_send_bufs);
1142	}	1165	}
1143		1166
1144	buf->rb_credits = 1;
1145	init_llist_head(&buf->rb_free_reps);	1167	init_llist_head(&buf->rb_free_reps);
1146		1168
1147	rc = rpcrdma_sendctxs_create(r_xprt);	1169	rc = rpcrdma_sendctxs_create(r_xprt);
@@ -1158,15 +1180,24 @@ out: · 10 unmodified lines
1158	* rpcrdma_req_destroy - Destroy an rpcrdma_req object	1180	* rpcrdma_req_destroy - Destroy an rpcrdma_req object
1159	* @req: unused object to be destroyed	1181	* @req: unused object to be destroyed
1160	*	1182	*
1161	* This function assumes that the caller prevents concurrent device	1183	* Relies on caller holding the transport send lock to protect
1162	* unload and transport tear-down.	1184	* removing req->rl_all from buf->rb_all_reqs safely.
1163	*/	1185	*/
1164	void rpcrdma_req_destroy(struct rpcrdma_req *req)	1186	void rpcrdma_req_destroy(struct rpcrdma_req *req)
1165	{	1187	{
		1188	struct rpcrdma_mr *mr;
		1189
1166	list_del(&req->rl_all);	1190	list_del(&req->rl_all);
1167		1191
1168	while (!list_empty(&req->rl_free_mrs))	1192	while ((mr = rpcrdma_mr_pop(&req->rl_free_mrs))) {
1169	rpcrdma_mr_free(rpcrdma_mr_pop(&req->rl_free_mrs));	1193	struct rpcrdma_buffer *buf = &mr->mr_xprt->rx_buf;
		1194
		1195	spin_lock(&buf->rb_lock);
		1196	list_del(&mr->mr_all);
		1197	spin_unlock(&buf->rb_lock);
		1198
		1199	frwr_release_mr(mr);
		1200	}
1170		1201
1171	rpcrdma_regbuf_free(req->rl_recvbuf);	1202	rpcrdma_regbuf_free(req->rl_recvbuf);
1172	rpcrdma_regbuf_free(req->rl_sendbuf);	1203	rpcrdma_regbuf_free(req->rl_sendbuf);
@@ -1174,28 +1205,33 @@ void rpcrdma_req_destroy(struct rpcrdma_req *req) · 1 unmodified line
1174	kfree(req);	1205	kfree(req);
1175	}	1206	}
1176		1207
1177	static void	1208	/**
1178	rpcrdma_mrs_destroy(struct rpcrdma_buffer *buf)	1209	* rpcrdma_mrs_destroy - Release all of a transport's MRs
		1210	* @r_xprt: controlling transport instance
		1211	*
		1212	* Relies on caller holding the transport send lock to protect
		1213	* removing mr->mr_list from req->rl_free_mrs safely.
		1214	*/
		1215	static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt)
1179	{	1216	{
1180	struct rpcrdma_xprt *r_xprt = container_of(buf, struct rpcrdma_xprt,	1217	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
1181	rx_buf);
1182	struct rpcrdma_mr *mr;	1218	struct rpcrdma_mr *mr;
1183	unsigned int count;
1184		1219
1185	count = 0;	1220	cancel_work_sync(&buf->rb_refresh_worker);
		1221
1186	spin_lock(&buf->rb_lock);	1222	spin_lock(&buf->rb_lock);
1187	while ((mr = list_first_entry_or_null(&buf->rb_all_mrs,	1223	while ((mr = list_first_entry_or_null(&buf->rb_all_mrs,
1188	struct rpcrdma_mr,	1224	struct rpcrdma_mr,
1189	mr_all)) != NULL) {	1225	mr_all)) != NULL) {
		1226	list_del(&mr->mr_list);
1190	list_del(&mr->mr_all);	1227	list_del(&mr->mr_all);
1191	spin_unlock(&buf->rb_lock);	1228	spin_unlock(&buf->rb_lock);
1192		1229
1193	frwr_release_mr(mr);	1230	frwr_release_mr(mr);
1194	count++;	1231
1195	spin_lock(&buf->rb_lock);	1232	spin_lock(&buf->rb_lock);
1196	}	1233	}
1197	spin_unlock(&buf->rb_lock);	1234	spin_unlock(&buf->rb_lock);
1198	r_xprt->rx_stats.mrs_allocated = 0;
1199	}	1235	}
1200		1236
1201	/**	1237	/**
@@ -1209,8 +1245,6 @@ rpcrdma_mrs_destroy(struct rpcrdma_buffer *buf) · 7 unmodified lines
1209	void	1245	void
1210	rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf)	1246	rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf)
1211	{	1247	{
1212	cancel_work_sync(&buf->rb_refresh_worker);
1213
1214	rpcrdma_sendctxs_destroy(buf);	1248	rpcrdma_sendctxs_destroy(buf);
1215	rpcrdma_reps_destroy(buf);	1249	rpcrdma_reps_destroy(buf);
1216		1250
@@ -1222,8 +1256,6 @@ rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf) · 5 unmodified lines
1222	list_del(&req->rl_list);	1256	list_del(&req->rl_list);
1223	rpcrdma_req_destroy(req);	1257	rpcrdma_req_destroy(req);
1224	}	1258	}
1225
1226	rpcrdma_mrs_destroy(buf);
1227	}	1259	}
1228		1260
1229	/**	1261	/**
@@ -1264,17 +1296,6 @@ void rpcrdma_mr_put(struct rpcrdma_mr *mr) · 34 unmodified lines
1264	rpcrdma_mr_push(mr, &mr->mr_req->rl_free_mrs);	1296	rpcrdma_mr_push(mr, &mr->mr_req->rl_free_mrs);
1265	}	1297	}
1266		1298
1267	static void rpcrdma_mr_free(struct rpcrdma_mr *mr)
1268	{
1269	struct rpcrdma_xprt *r_xprt = mr->mr_xprt;
1270	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
1271
1272	mr->mr_req = NULL;
1273	spin_lock(&buf->rb_lock);
1274	rpcrdma_mr_push(mr, &buf->rb_mrs);
1275	spin_unlock(&buf->rb_lock);
1276	}
1277
1278	/**	1299	/**
1279	* rpcrdma_buffer_get - Get a request buffer	1300	* rpcrdma_buffer_get - Get a request buffer
1280	* @buffers: Buffer pool from which to obtain a buffer	1301	* @buffers: Buffer pool from which to obtain a buffer
@@ -1437,7 +1458,7 @@ rpcrdma_ep_post(struct rpcrdma_ia *ia, · 156 unmodified lines
1437	struct rpcrdma_ep *ep,	1458	struct rpcrdma_ep *ep,
1438	struct rpcrdma_req *req)	1459	struct rpcrdma_req *req)
1439	{	1460	{
1440	struct ib_send_wr *send_wr = &req->rl_sendctx->sc_wr;	1461	struct ib_send_wr *send_wr = &req->rl_wr;
1441	int rc;	1462	int rc;
1442		1463
1443	if (!ep->rep_send_count \|\| kref_read(&req->rl_kref) > 1) {	1464	if (!ep->rep_send_count \|\| kref_read(&req->rl_kref) > 1) {
@@ -1455,8 +1476,13 @@ rpcrdma_ep_post(struct rpcrdma_ia *ia, · 11 unmodified lines
1455	return 0;	1476	return 0;
1456	}	1477	}
1457		1478
1458	static void	1479	/**
1459	rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp)	1480	* rpcrdma_post_recvs - Refill the Receive Queue
		1481	* @r_xprt: controlling transport instance
		1482	* @temp: mark Receive buffers to be deleted after use
		1483	*
		1484	*/
		1485	void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp)
1460	{	1486	{
1461	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;	1487	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
1462	struct rpcrdma_ep *ep = &r_xprt->rx_ep;	1488	struct rpcrdma_ep *ep = &r_xprt->rx_ep;

net/sunrpc/xprtrdma/xprt_rdma.h+6−12

@@ -218,12 +218,8 @@ enum { · 217 unmodified lines
218	/* struct rpcrdma_sendctx - DMA mapped SGEs to unmap after Send completes	218	/* struct rpcrdma_sendctx - DMA mapped SGEs to unmap after Send completes
219	*/	219	*/
220	struct rpcrdma_req;	220	struct rpcrdma_req;
221	struct rpcrdma_xprt;
222	struct rpcrdma_sendctx {	221	struct rpcrdma_sendctx {
223	struct ib_send_wr sc_wr;
224	struct ib_cqe sc_cqe;	222	struct ib_cqe sc_cqe;
225	struct ib_device *sc_device;
226	struct rpcrdma_xprt *sc_xprt;
227	struct rpcrdma_req *sc_req;	223	struct rpcrdma_req *sc_req;
228	unsigned int sc_unmap_count;	224	unsigned int sc_unmap_count;
229	struct ib_sge sc_sges[];	225	struct ib_sge sc_sges[];
@@ -257,7 +253,6 @@ struct rpcrdma_mr { · 27 unmodified lines
257	u32 mr_handle;	253	u32 mr_handle;
258	u32 mr_length;	254	u32 mr_length;
259	u64 mr_offset;	255	u64 mr_offset;
260	struct work_struct mr_recycle;
261	struct list_head mr_all;	256	struct list_head mr_all;
262	};	257	};
263		258
@@ -318,6 +313,7 @@ struct rpcrdma_req { · 54 unmodified lines
318	struct rpcrdma_rep *rl_reply;	313	struct rpcrdma_rep *rl_reply;
319	struct xdr_stream rl_stream;	314	struct xdr_stream rl_stream;
320	struct xdr_buf rl_hdrbuf;	315	struct xdr_buf rl_hdrbuf;
		316	struct ib_send_wr rl_wr;
321	struct rpcrdma_sendctx *rl_sendctx;	317	struct rpcrdma_sendctx *rl_sendctx;
322	struct rpcrdma_regbuf rl_rdmabuf; / xprt header */	318	struct rpcrdma_regbuf rl_rdmabuf; / xprt header */
323	struct rpcrdma_regbuf rl_sendbuf; / rq_snd_buf */	319	struct rpcrdma_regbuf rl_sendbuf; / rq_snd_buf */
@@ -474,6 +470,7 @@ void rpcrdma_ep_disconnect(struct rpcrdma_ep , struct rpcrdma_ia ); · 150 unmodified lines
474		470
475	int rpcrdma_ep_post(struct rpcrdma_ia , struct rpcrdma_ep ,	471	int rpcrdma_ep_post(struct rpcrdma_ia , struct rpcrdma_ep ,
476	struct rpcrdma_req *);	472	struct rpcrdma_req *);
		473	void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp);
477		474
478	/*	475	/*
479	* Buffer calls - xprtrdma/verbs.c	476	* Buffer calls - xprtrdma/verbs.c
@@ -487,12 +484,7 @@ struct rpcrdma_sendctx rpcrdma_sendctx_get_locked(struct rpcrdma_xprt r_xprt); · 7 unmodified lines
487		484
488	struct rpcrdma_mr rpcrdma_mr_get(struct rpcrdma_xprt r_xprt);	485	struct rpcrdma_mr rpcrdma_mr_get(struct rpcrdma_xprt r_xprt);
489	void rpcrdma_mr_put(struct rpcrdma_mr *mr);	486	void rpcrdma_mr_put(struct rpcrdma_mr *mr);
490		487	void rpcrdma_mrs_refresh(struct rpcrdma_xprt *r_xprt);
491	static inline void
492	rpcrdma_mr_recycle(struct rpcrdma_mr *mr)
493	{
494	schedule_work(&mr->mr_recycle);
495	}
496		488
497	struct rpcrdma_req rpcrdma_buffer_get(struct rpcrdma_buffer );	489	struct rpcrdma_req rpcrdma_buffer_get(struct rpcrdma_buffer );
498	void rpcrdma_buffer_put(struct rpcrdma_buffer *buffers,	490	void rpcrdma_buffer_put(struct rpcrdma_buffer *buffers,
@@ -542,7 +534,6 @@ rpcrdma_data_dir(bool writing) · 43 unmodified lines
542	/* Memory registration calls xprtrdma/frwr_ops.c	534	/* Memory registration calls xprtrdma/frwr_ops.c
543	*/	535	*/
544	bool frwr_is_supported(struct ib_device *device);	536	bool frwr_is_supported(struct ib_device *device);
545	void frwr_recycle(struct rpcrdma_req *req);
546	void frwr_reset(struct rpcrdma_req *req);	537	void frwr_reset(struct rpcrdma_req *req);
547	int frwr_open(struct rpcrdma_ia ia, struct rpcrdma_ep ep);	538	int frwr_open(struct rpcrdma_ia ia, struct rpcrdma_ep ep);
548	int frwr_init_mr(struct rpcrdma_ia ia, struct rpcrdma_mr mr);	539	int frwr_init_mr(struct rpcrdma_ia ia, struct rpcrdma_mr mr);
@@ -563,6 +554,8 @@ void frwr_unmap_async(struct rpcrdma_xprt r_xprt, struct rpcrdma_req req); · 14 unmodified lines
563		554
564	enum rpcrdma_chunktype {	555	enum rpcrdma_chunktype {
565	rpcrdma_noch = 0,	556	rpcrdma_noch = 0,
		557	rpcrdma_noch_pullup,
		558	rpcrdma_noch_mapped,
566	rpcrdma_readch,	559	rpcrdma_readch,
567	rpcrdma_areadch,	560	rpcrdma_areadch,
568	rpcrdma_writech,	561	rpcrdma_writech,
@@ -576,6 +569,7 @@ int rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt, · 7 unmodified lines
576	void rpcrdma_sendctx_unmap(struct rpcrdma_sendctx *sc);	569	void rpcrdma_sendctx_unmap(struct rpcrdma_sendctx *sc);
577	int rpcrdma_marshal_req(struct rpcrdma_xprt r_xprt, struct rpc_rqst rqst);	570	int rpcrdma_marshal_req(struct rpcrdma_xprt r_xprt, struct rpc_rqst rqst);
578	void rpcrdma_set_max_header_sizes(struct rpcrdma_xprt *);	571	void rpcrdma_set_max_header_sizes(struct rpcrdma_xprt *);
		572	void rpcrdma_reset_cwnd(struct rpcrdma_xprt *r_xprt);
579	void rpcrdma_complete_rqst(struct rpcrdma_rep *rep);	573	void rpcrdma_complete_rqst(struct rpcrdma_rep *rep);
580	void rpcrdma_reply_handler(struct rpcrdma_rep *rep);	574	void rpcrdma_reply_handler(struct rpcrdma_rep *rep);
581		575

net/sunrpc/xprtsock.c+1−1

@@ -1752,7 +1752,7 @@ static void xs_set_port(struct rpc_xprt *xprt, unsigned short port) · 1751 unmodified lines
1752		1752
1753	static void xs_set_srcport(struct sock_xprt transport, struct socket sock)	1753	static void xs_set_srcport(struct sock_xprt transport, struct socket sock)
1754	{	1754	{
1755	if (transport->srcport == 0)	1755	if (transport->srcport == 0 && transport->xprt.reuseport)
1756	transport->srcport = xs_sock_getport(sock);	1756	transport->srcport = xs_sock_getport(sock);
1757	}	1757	}
1758		1758

Merge tag 'nfs-for-5.5-1' of git://git.linux-nfs.org/projects/trondmy/linux-nfs

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code