Merge tag 'nfs-for-5.5-1' of git://git.linux-nfs.org/projects/trondmy/linux-nfs

Pull NFS client updates from Trond Myklebust: "Highlights include: Features: - NFSv4.2 now supports cross device offloaded copy (i.e. offloaded copy of a file from one source server to a different target server). - New RDMA tracepoints for debugging congestion control and Local Invalidate WRs. Bugfixes and cleanups - Drop the NFSv4.1 session slot if nfs4_delegreturn_prepare waits for layoutreturn - Handle bad/dead sessions correctly in nfs41_sequence_process() - Various bugfixes to the delegation return operation. - Various bugfixes pertaining to delegations that have been revoked. - Cleanups to the NFS timespec code to avoid unnecessary conversions between timespec and timespec64. - Fix unstable RDMA connections after a reconnect - Close race between waking an RDMA sender and posting a receive - Wake pending RDMA tasks if connection fails - Fix MR list corruption, and clean up MR usage - Fix another RPCSEC_GSS issue with MIC buffer space" * tag 'nfs-for-5.5-1' of git://git.linux-nfs.org/projects/trondmy/linux-nfs: (79 commits) SUNRPC: Capture completion of all RPC tasks SUNRPC: Fix another issue with MIC buffer space NFS4: Trace lock reclaims NFS4: Trace state recovery operation NFSv4.2 fix memory leak in nfs42_ssc_open NFSv4.2 fix kfree in __nfs42_copy_file_range NFS: remove duplicated include from nfs4file.c NFSv4: Make _nfs42_proc_copy_notify() static NFS: Fallocate should use the nfs4_fattr_bitmap NFS: Return -ETXTBSY when attempting to write to a swapfile fs: nfs: sysfs: Remove NULL check before kfree NFS: remove unneeded semicolon NFSv4: add declaration of current_stateid NFSv4.x: Drop the slot if nfs4_delegreturn_prepare waits for layoutreturn NFSv4.x: Handle bad/dead sessions correctly in nfs41_sequence_process() nfsv4: Move NFSPROC4_CLNT_COPY_NOTIFY to end of list SUNRPC: Avoid RPC delays when exiting suspend NFS: Add a tracepoint in nfs_fh_to_dentry() NFSv4: Don't retry the GETATTR on old stateid in nfs4_delegreturn_done() NFSv4: Handle NFS4ERR_OLD_STATEID in delegreturn ...

Linus Torvalds committed 6 years ago

Commit fb9bf402 parents 94e89b4 a264aba49 files · +1756 −622Browse files

Unified · Split

49 files

fs/lockd/host.c +2 −1
fs/nfs/callback.h +2 −2
fs/nfs/callback_proc.c +3 −5
fs/nfs/callback_xdr.c +3 −3
fs/nfs/client.c +11 −0
fs/nfs/delegation.c +111 −47
fs/nfs/delegation.h +2 −1
fs/nfs/export.c +1 −0
fs/nfs/file.c +1 −1
fs/nfs/inode.c +27 −27
fs/nfs/internal.h +1 −1
fs/nfs/namespace.c +3 −0
fs/nfs/nfs2xdr.c +8 −13
fs/nfs/nfs3client.c +4 −1
fs/nfs/nfs3xdr.c +5 −9
fs/nfs/nfs42.h +14 −1
fs/nfs/nfs42proc.c +168 −33
fs/nfs/nfs42xdr.c +188 −2
fs/nfs/nfs4_fs.h +19 −2
fs/nfs/nfs4client.c +8 −5
fs/nfs/nfs4file.c +139 −2
fs/nfs/nfs4proc.c +46 −23
fs/nfs/nfs4state.c +43 −8
fs/nfs/nfs4super.c +2 −2
fs/nfs/nfs4trace.h +175 −0
fs/nfs/nfs4xdr.c +11 −13
fs/nfs/nfstrace.h +33 −0
fs/nfs/pnfs.c +0 −2
fs/nfs/super.c +3 −3
fs/nfs/sysfs.c +1 −2
include/linux/nfs4.h +26 −0
include/linux/nfs_fs.h +3 −1
include/linux/nfs_fs_sb.h +5 −1
include/linux/nfs_xdr.h +24 −7
include/linux/sunrpc/clnt.h +1 −0
include/linux/sunrpc/xprt.h +2 −1
include/trace/events/rpcrdma.h +132 −66
include/trace/events/sunrpc.h +94 −0
net/sunrpc/clnt.c +8 −13
net/sunrpc/sched.c +2 −1
net/sunrpc/xdr.c +5 −6
net/sunrpc/xprt.c +13 −9
net/sunrpc/xprtrdma/backchannel.c +1 −1
net/sunrpc/xprtrdma/frwr_ops.c +11 −42
net/sunrpc/xprtrdma/rpc_rdma.c +268 −145
net/sunrpc/xprtrdma/transport.c +10 −23
net/sunrpc/xprtrdma/verbs.c +110 −84
net/sunrpc/xprtrdma/xprt_rdma.h +6 −12
net/sunrpc/xprtsock.c +1 −1

fs/lockd/host.c+2−1

@@ -464,7 +464,8 @@ nlm_bind_host(struct nlm_host *host) · 463 unmodified lines
464	464	.version = host->h_version,
465	465	.authflavor = RPC_AUTH_UNIX,
466	466	.flags = (RPC_CLNT_CREATE_NOPING \|
467		− RPC_CLNT_CREATE_AUTOBIND),
	467	+ RPC_CLNT_CREATE_AUTOBIND \|
	468	+ RPC_CLNT_CREATE_REUSEPORT),
468	469	.cred = host->h_cred,
469	470	};
470	471

fs/nfs/callback.h+2−2

@@ -72,8 +72,8 @@ struct cb_getattrres { · 71 unmodified lines
72	72	uint32_t bitmap[2];
73	73	uint64_t size;
74	74	uint64_t change_attr;
75		− struct timespec ctime;
76		− struct timespec mtime;
	75	+ struct timespec64 ctime;
	76	+ struct timespec64 mtime;
77	77	};
78	78
79	79	struct cb_recallargs {

fs/nfs/callback_proc.c+3−5

@@ -26,7 +26,6 @@ __be32 nfs4_callback_getattr(void argp, void resp, · 25 unmodified lines
26	26	struct cb_getattrargs *args = argp;
27	27	struct cb_getattrres *res = resp;
28	28	struct nfs_delegation *delegation;
29		− struct nfs_inode *nfsi;
30	29	struct inode *inode;
31	30
32	31	res->status = htonl(NFS4ERR_OP_NOT_IN_SESSION);
@@ -47,17 +46,16 @@ __be32 nfs4_callback_getattr(void argp, void resp, · 14 unmodified lines
47	46	-ntohl(res->status));
48	47	goto out;
49	48	}
50		− nfsi = NFS_I(inode);
51	49	rcu_read_lock();
52		− delegation = rcu_dereference(nfsi->delegation);
	50	+ delegation = nfs4_get_valid_delegation(inode);
53	51	if (delegation == NULL \|\| (delegation->type & FMODE_WRITE) == 0)
54	52	goto out_iput;
55	53	res->size = i_size_read(inode);
56	54	res->change_attr = delegation->change_attr;
57	55	if (nfs_have_writebacks(inode))
58	56	res->change_attr++;
59		− res->ctime = timespec64_to_timespec(inode->i_ctime);
60		− res->mtime = timespec64_to_timespec(inode->i_mtime);
	57	+ res->ctime = inode->i_ctime;
	58	+ res->mtime = inode->i_mtime;
61	59	res->bitmap[0] = (FATTR4_WORD0_CHANGE\|FATTR4_WORD0_SIZE) &
62	60	args->bitmap[0];
63	61	res->bitmap[1] = (FATTR4_WORD1_TIME_METADATA\|FATTR4_WORD1_TIME_MODIFY) &

fs/nfs/callback_xdr.c+3−3

@@ -627,7 +627,7 @@ static __be32 encode_attr_size(struct xdr_stream xdr, const uint32_t bitmap, u · 626 unmodified lines
627	627	return 0;
628	628	}
629	629
630		−static __be32 encode_attr_time(struct xdr_stream xdr, const struct timespec time)
	630	+static __be32 encode_attr_time(struct xdr_stream xdr, const struct timespec64 time)
631	631	{
632	632	__be32 *p;
633	633
@@ -639,14 +639,14 @@ static __be32 encode_attr_time(struct xdr_stream xdr, const struct timespec ti · 5 unmodified lines
639	639	return 0;
640	640	}
641	641
642		−static __be32 encode_attr_ctime(struct xdr_stream xdr, const uint32_t bitmap, const struct timespec *time)
	642	+static __be32 encode_attr_ctime(struct xdr_stream xdr, const uint32_t bitmap, const struct timespec64 *time)
643	643	{
644	644	if (!(bitmap[1] & FATTR4_WORD1_TIME_METADATA))
645	645	return 0;
646	646	return encode_attr_time(xdr,time);
647	647	}
648	648
649		−static __be32 encode_attr_mtime(struct xdr_stream xdr, const uint32_t bitmap, const struct timespec *time)
	649	+static __be32 encode_attr_mtime(struct xdr_stream xdr, const uint32_t bitmap, const struct timespec64 *time)
650	650	{
651	651	if (!(bitmap[1] & FATTR4_WORD1_TIME_MODIFY))
652	652	return 0;

fs/nfs/client.c+11−0

@@ -312,6 +312,12 @@ again: · 311 unmodified lines
312	312	/* Match nfsv4 minorversion */
313	313	if (clp->cl_minorversion != data->minorversion)
314	314	continue;
	315	+
	316	+ /* Match request for a dedicated DS */
	317	+ if (test_bit(NFS_CS_DS, &data->init_flags) !=
	318	+ test_bit(NFS_CS_DS, &clp->cl_flags))
	319	+ continue;
	320	+
315	321	/* Match the full socket address */
316	322	if (!rpc_cmp_addr_port(sap, clap))
317	323	/* Match all xprt_switch full socket addresses */
@@ -515,6 +521,10 @@ int nfs_create_rpc_client(struct nfs_client *clp, · 197 unmodified lines
515	521	args.flags \|= RPC_CLNT_CREATE_NONPRIVPORT;
516	522	if (test_bit(NFS_CS_INFINITE_SLOTS, &clp->cl_flags))
517	523	args.flags \|= RPC_CLNT_CREATE_INFINITE_SLOTS;
	524	+ if (test_bit(NFS_CS_NOPING, &clp->cl_flags))
	525	+ args.flags \|= RPC_CLNT_CREATE_NOPING;
	526	+ if (test_bit(NFS_CS_REUSEPORT, &clp->cl_flags))
	527	+ args.flags \|= RPC_CLNT_CREATE_REUSEPORT;
518	528
519	529	if (!IS_ERR(clp->cl_rpcclient))
520	530	return 0;
@@ -662,6 +672,7 @@ static int nfs_init_server(struct nfs_server *server, · 141 unmodified lines
662	672	.timeparms = &timeparms,
663	673	.cred = server->cred,
664	674	.nconnect = data->nfs_server.nconnect,
	675	+ .init_flags = (1UL << NFS_CS_REUSEPORT),
665	676	};
666	677	struct nfs_client *clp;
667	678	int error;

fs/nfs/delegation.c+111−47

@@ -199,7 +199,7 @@ void nfs_inode_reclaim_delegation(struct inode inode, const struct cred cred, · 198 unmodified lines
199	199	delegation = rcu_dereference(NFS_I(inode)->delegation);
200	200	if (delegation != NULL) {
201	201	spin_lock(&delegation->lock);
202		− if (delegation->inode != NULL) {
	202	+ if (nfs4_is_valid_delegation(delegation, 0)) {
203	203	nfs4_stateid_copy(&delegation->stateid, stateid);
204	204	delegation->type = type;
205	205	delegation->pagemod_limit = pagemod_limit;
@@ -229,7 +229,6 @@ static int nfs_do_return_delegation(struct inode inode, struct nfs_delegation · 23 unmodified lines
229	229	delegation->cred,
230	230	&delegation->stateid,
231	231	issync);
232		− nfs_free_delegation(delegation);
233	232	return res;
234	233	}
235	234
@@ -298,7 +297,10 @@ nfs_detach_delegation_locked(struct nfs_inode *nfsi, · 62 unmodified lines
298	297	return NULL;
299	298
300	299	spin_lock(&delegation->lock);
301		− set_bit(NFS_DELEGATION_RETURNING, &delegation->flags);
	300	+ if (!delegation->inode) {
	301	+ spin_unlock(&delegation->lock);
	302	+ return NULL;
	303	+ }
302	304	list_del_rcu(&delegation->super_list);
303	305	delegation->inode = NULL;
304	306	rcu_assign_pointer(nfsi->delegation, NULL);
@@ -325,10 +327,12 @@ nfs_inode_detach_delegation(struct inode *inode) · 20 unmodified lines
325	327	struct nfs_server *server = NFS_SERVER(inode);
326	328	struct nfs_delegation *delegation;
327	329
328		− delegation = nfs_start_delegation_return(nfsi);
329		− if (delegation == NULL)
330		− return NULL;
331		− return nfs_detach_delegation(nfsi, delegation, server);
	330	+ rcu_read_lock();
	331	+ delegation = rcu_dereference(nfsi->delegation);
	332	+ if (delegation != NULL)
	333	+ delegation = nfs_detach_delegation(nfsi, delegation, server);
	334	+ rcu_read_unlock();
	335	+ return delegation;
332	336	}
333	337
334	338	static void
@@ -339,6 +343,7 @@ nfs_update_inplace_delegation(struct nfs_delegation *delegation, · 4 unmodified lines
339	343	delegation->stateid.seqid = update->stateid.seqid;
340	344	smp_wmb();
341	345	delegation->type = update->type;
	346	+ clear_bit(NFS_DELEGATION_REVOKED, &delegation->flags);
342	347	}
343	348	}
344	349
@@ -379,14 +384,18 @@ int nfs_inode_set_delegation(struct inode inode, const struct cred cred, · 34 unmodified lines
379	384	spin_lock(&clp->cl_lock);
380	385	old_delegation = rcu_dereference_protected(nfsi->delegation,
381	386	lockdep_is_held(&clp->cl_lock));
382		− if (old_delegation != NULL) {
383		− /* Is this an update of the existing delegation? */
384		− if (nfs4_stateid_match_other(&old_delegation->stateid,
385		− &delegation->stateid)) {
386		− nfs_update_inplace_delegation(old_delegation,
387		− delegation);
388		− goto out;
389		− }
	387	+ if (old_delegation == NULL)
	388	+ goto add_new;
	389	+ /* Is this an update of the existing delegation? */
	390	+ if (nfs4_stateid_match_other(&old_delegation->stateid,
	391	+ &delegation->stateid)) {
	392	+ spin_lock(&old_delegation->lock);
	393	+ nfs_update_inplace_delegation(old_delegation,
	394	+ delegation);
	395	+ spin_unlock(&old_delegation->lock);
	396	+ goto out;
	397	+ }
	398	+ if (!test_bit(NFS_DELEGATION_REVOKED, &old_delegation->flags)) {
390	399	/*
391	400	* Deal with broken servers that hand out two
392	401	* delegations for the same file.
@@ -405,11 +414,11 @@ int nfs_inode_set_delegation(struct inode inode, const struct cred cred, · 12 unmodified lines
405	414	if (test_and_set_bit(NFS_DELEGATION_RETURNING,
406	415	&old_delegation->flags))
407	416	goto out;
408		− freeme = nfs_detach_delegation_locked(nfsi,
409		− old_delegation, clp);
410		− if (freeme == NULL)
411		− goto out;
412	417	}
	418	+ freeme = nfs_detach_delegation_locked(nfsi, old_delegation, clp);
	419	+ if (freeme == NULL)
	420	+ goto out;
	421	+add_new:
413	422	list_add_tail_rcu(&delegation->super_list, &server->delegations);
414	423	rcu_assign_pointer(nfsi->delegation, delegation);
415	424	delegation = NULL;
@@ -424,8 +433,10 @@ out: · 8 unmodified lines
424	433	spin_unlock(&clp->cl_lock);
425	434	if (delegation != NULL)
426	435	nfs_free_delegation(delegation);
427		− if (freeme != NULL)
	436	+ if (freeme != NULL) {
428	437	nfs_do_return_delegation(inode, freeme, 0);
	438	+ nfs_free_delegation(freeme);
	439	+ }
429	440	return status;
430	441	}
431	442
@@ -435,7 +446,6 @@ out: · 3 unmodified lines
435	446	static int nfs_end_delegation_return(struct inode inode, struct nfs_delegation delegation, int issync)
436	447	{
437	448	struct nfs_client *clp = NFS_SERVER(inode)->nfs_client;
438		− struct nfs_inode *nfsi = NFS_I(inode);
439	449	int err = 0;
440	450
441	451	if (delegation == NULL)
@@ -457,8 +467,6 @@ static int nfs_end_delegation_return(struct inode *inode, struct nfs_delegation · 15 unmodified lines
457	467	nfs_abort_delegation_return(delegation, clp);
458	468	goto out;
459	469	}
460		− if (!nfs_detach_delegation(nfsi, delegation, NFS_SERVER(inode)))
461		− goto out;
462	470
463	471	err = nfs_do_return_delegation(inode, delegation, issync);
464	472	out:
@@ -469,8 +477,6 @@ static bool nfs_delegation_need_return(struct nfs_delegation *delegation) · 4 unmodified lines
469	477	{
470	478	bool ret = false;
471	479
472		− if (test_bit(NFS_DELEGATION_RETURNING, &delegation->flags))
473		− goto out;
474	480	if (test_and_clear_bit(NFS_DELEGATION_RETURN, &delegation->flags))
475	481	ret = true;
476	482	if (test_and_clear_bit(NFS_DELEGATION_RETURN_IF_CLOSED, &delegation->flags) && !ret) {
@@ -482,7 +488,10 @@ static bool nfs_delegation_need_return(struct nfs_delegation *delegation) · 5 unmodified lines
482	488	ret = true;
483	489	spin_unlock(&delegation->lock);
484	490	}
485		−out:
	491	+ if (test_bit(NFS_DELEGATION_RETURNING, &delegation->flags) \|\|
	492	+ test_bit(NFS_DELEGATION_REVOKED, &delegation->flags))
	493	+ ret = false;
	494	+
486	495	return ret;
487	496	}
488	497
@@ -585,19 +594,23 @@ restart: · 96 unmodified lines
585	594	}
586	595
587	596	/**
588		− * nfs_inode_return_delegation_noreclaim - return delegation, don't reclaim opens
	597	+ * nfs_inode_evict_delegation - return delegation, don't reclaim opens
589	598	* @inode: inode to process
590	599	*
591	600	* Does not protect against delegation reclaims, therefore really only safe
592		− * to be called from nfs4_clear_inode().
	601	+ * to be called from nfs4_clear_inode(). Guaranteed to always free
	602	+ * the delegation structure.
593	603	*/
594		−void nfs_inode_return_delegation_noreclaim(struct inode *inode)
	604	+void nfs_inode_evict_delegation(struct inode *inode)
595	605	{
596	606	struct nfs_delegation *delegation;
597	607
598	608	delegation = nfs_inode_detach_delegation(inode);
599		− if (delegation != NULL)
	609	+ if (delegation != NULL) {
	610	+ set_bit(NFS_DELEGATION_INODE_FREEING, &delegation->flags);
600	611	nfs_do_return_delegation(inode, delegation, 1);
	612	+ nfs_free_delegation(delegation);
	613	+ }
601	614	}
602	615
603	616	/**
@@ -633,10 +646,18 @@ int nfs4_inode_return_delegation(struct inode *inode) · 29 unmodified lines
633	646	*/
634	647	int nfs4_inode_make_writeable(struct inode *inode)
635	648	{
636		− if (!nfs4_has_session(NFS_SERVER(inode)->nfs_client) \|\|
637		− !nfs4_check_delegation(inode, FMODE_WRITE))
638		− return nfs4_inode_return_delegation(inode);
639		− return 0;
	649	+ struct nfs_delegation *delegation;
	650	+
	651	+ rcu_read_lock();
	652	+ delegation = nfs4_get_valid_delegation(inode);
	653	+ if (delegation == NULL \|\|
	654	+ (nfs4_has_session(NFS_SERVER(inode)->nfs_client) &&
	655	+ (delegation->type & FMODE_WRITE))) {
	656	+ rcu_read_unlock();
	657	+ return 0;
	658	+ }
	659	+ rcu_read_unlock();
	660	+ return nfs4_inode_return_delegation(inode);
640	661	}
641	662
642	663	static void nfs_mark_return_if_closed_delegation(struct nfs_server *server,
@@ -744,10 +765,9 @@ static void nfs_mark_delegation_revoked(struct nfs_server *server, · 101 unmodified lines
744	765	{
745	766	set_bit(NFS_DELEGATION_REVOKED, &delegation->flags);
746	767	delegation->stateid.type = NFS4_INVALID_STATEID_TYPE;
747		− nfs_mark_return_delegation(server, delegation);
748	768	}
749	769
750		−static bool nfs_revoke_delegation(struct inode *inode,
	770	+static void nfs_revoke_delegation(struct inode *inode,
751	771	const nfs4_stateid *stateid)
752	772	{
753	773	struct nfs_delegation *delegation;
@@ -761,29 +781,69 @@ static bool nfs_revoke_delegation(struct inode *inode, · 7 unmodified lines
761	781	if (stateid == NULL) {
762	782	nfs4_stateid_copy(&tmp, &delegation->stateid);
763	783	stateid = &tmp;
764		− } else if (!nfs4_stateid_match(stateid, &delegation->stateid))
765		− goto out;
	784	+ } else {
	785	+ if (!nfs4_stateid_match_other(stateid, &delegation->stateid))
	786	+ goto out;
	787	+ spin_lock(&delegation->lock);
	788	+ if (stateid->seqid) {
	789	+ if (nfs4_stateid_is_newer(&delegation->stateid, stateid)) {
	790	+ spin_unlock(&delegation->lock);
	791	+ goto out;
	792	+ }
	793	+ delegation->stateid.seqid = stateid->seqid;
	794	+ }
	795	+ spin_unlock(&delegation->lock);
	796	+ }
766	797	nfs_mark_delegation_revoked(NFS_SERVER(inode), delegation);
767	798	ret = true;
768	799	out:
769	800	rcu_read_unlock();
770	801	if (ret)
771	802	nfs_inode_find_state_and_recover(inode, stateid);
772		− return ret;
773	803	}
774	804
775	805	void nfs_remove_bad_delegation(struct inode *inode,
776	806	const nfs4_stateid *stateid)
	807	+{
	808	+ nfs_revoke_delegation(inode, stateid);
	809	+}
	810	+EXPORT_SYMBOL_GPL(nfs_remove_bad_delegation);
	811	+
	812	+void nfs_delegation_mark_returned(struct inode *inode,
	813	+ const nfs4_stateid *stateid)
777	814	{
778	815	struct nfs_delegation *delegation;
779	816
780		− if (!nfs_revoke_delegation(inode, stateid))
	817	+ if (!inode)
781	818	return;
782		− delegation = nfs_inode_detach_delegation(inode);
783		− if (delegation)
784		− nfs_free_delegation(delegation);
	819	+
	820	+ rcu_read_lock();
	821	+ delegation = rcu_dereference(NFS_I(inode)->delegation);
	822	+ if (!delegation)
	823	+ goto out_rcu_unlock;
	824	+
	825	+ spin_lock(&delegation->lock);
	826	+ if (!nfs4_stateid_match_other(stateid, &delegation->stateid))
	827	+ goto out_spin_unlock;
	828	+ if (stateid->seqid) {
	829	+ /* If delegation->stateid is newer, dont mark as returned */
	830	+ if (nfs4_stateid_is_newer(&delegation->stateid, stateid))
	831	+ goto out_clear_returning;
	832	+ if (delegation->stateid.seqid != stateid->seqid)
	833	+ delegation->stateid.seqid = stateid->seqid;
	834	+ }
	835	+
	836	+ nfs_mark_delegation_revoked(NFS_SERVER(inode), delegation);
	837	+
	838	+out_clear_returning:
	839	+ clear_bit(NFS_DELEGATION_RETURNING, &delegation->flags);
	840	+out_spin_unlock:
	841	+ spin_unlock(&delegation->lock);
	842	+out_rcu_unlock:
	843	+ rcu_read_unlock();
	844	+
	845	+ nfs_inode_find_state_and_recover(inode, stateid);
785	846	}
786		−EXPORT_SYMBOL_GPL(nfs_remove_bad_delegation);
787	847
788	848	/**
789	849	* nfs_expire_unused_delegation_types
@@ -840,7 +900,7 @@ int nfs_async_inode_return_delegation(struct inode *inode, · 50 unmodified lines
840	900	struct nfs_delegation *delegation;
841	901
842	902	rcu_read_lock();
843		− delegation = rcu_dereference(NFS_I(inode)->delegation);
	903	+ delegation = nfs4_get_valid_delegation(inode);
844	904	if (delegation == NULL)
845	905	goto out_enoent;
846	906	if (stateid != NULL &&
@@ -866,6 +926,7 @@ nfs_delegation_find_inode_server(struct nfs_server *server, · 19 unmodified lines
866	926	list_for_each_entry_rcu(delegation, &server->delegations, super_list) {
867	927	spin_lock(&delegation->lock);
868	928	if (delegation->inode != NULL &&
	929	+ !test_bit(NFS_DELEGATION_REVOKED, &delegation->flags) &&
869	930	nfs_compare_fh(fhandle, &NFS_I(delegation->inode)->fh) == 0) {
870	931	freeme = igrab(delegation->inode);
871	932	if (freeme && nfs_sb_active(freeme->i_sb))
@@ -1140,7 +1201,8 @@ void nfs_inode_find_delegation_state_and_recover(struct inode *inode, · 268 unmodified lines
1140	1201	rcu_read_lock();
1141	1202	delegation = rcu_dereference(NFS_I(inode)->delegation);
1142	1203	if (delegation &&
1143		− nfs4_stateid_match_other(&delegation->stateid, stateid)) {
	1204	+ nfs4_stateid_match_or_older(&delegation->stateid, stateid) &&
	1205	+ !test_bit(NFS_DELEGATION_REVOKED, &delegation->flags)) {
1144	1206	nfs_mark_test_expired_delegation(NFS_SERVER(inode), delegation);
1145	1207	found = true;
1146	1208	}
@@ -1189,7 +1251,9 @@ bool nfs4_refresh_delegation_stateid(nfs4_stateid dst, struct inode inode) · 42 unmodified lines
1189	1251	rcu_read_lock();
1190	1252	delegation = rcu_dereference(NFS_I(inode)->delegation);
1191	1253	if (delegation != NULL &&
1192		− nfs4_stateid_match_other(dst, &delegation->stateid)) {
	1254	+ nfs4_stateid_match_other(dst, &delegation->stateid) &&
	1255	+ nfs4_stateid_is_newer(&delegation->stateid, dst) &&
	1256	+ !test_bit(NFS_DELEGATION_REVOKED, &delegation->flags)) {
1193	1257	dst->seqid = delegation->stateid.seqid;
1194	1258	ret = true;
1195	1259	}

fs/nfs/delegation.h+2−1

@@ -43,7 +43,7 @@ void nfs_inode_reclaim_delegation(struct inode inode, const struct cred cred, · 42 unmodified lines
43	43	fmode_t type, const nfs4_stateid *stateid, unsigned long pagemod_limit);
44	44	int nfs4_inode_return_delegation(struct inode *inode);
45	45	int nfs_async_inode_return_delegation(struct inode inode, const nfs4_stateid stateid);
46		−void nfs_inode_return_delegation_noreclaim(struct inode *inode);
	46	+void nfs_inode_evict_delegation(struct inode *inode);
47	47
48	48	struct inode nfs_delegation_find_inode(struct nfs_client clp, const struct nfs_fh *fhandle);
49	49	void nfs_server_return_all_delegations(struct nfs_server *);
@@ -53,6 +53,7 @@ void nfs_expire_unreferenced_delegations(struct nfs_client *clp); · 3 unmodified lines
53	53	int nfs_client_return_marked_delegations(struct nfs_client *clp);
54	54	int nfs_delegations_present(struct nfs_client *clp);
55	55	void nfs_remove_bad_delegation(struct inode inode, const nfs4_stateid stateid);
	56	+void nfs_delegation_mark_returned(struct inode inode, const nfs4_stateid stateid);
56	57
57	58	void nfs_delegation_mark_reclaim(struct nfs_client *clp);
58	59	void nfs_delegation_reap_unclaimed(struct nfs_client *clp);

fs/nfs/export.c+1−0

@@ -105,6 +105,7 @@ nfs_fh_to_dentry(struct super_block sb, struct fid fid, · 104 unmodified lines
105	105	ret = rpc_ops->getattr(NFS_SB(sb), server_fh, fattr, label, NULL);
106	106	if (ret) {
107	107	dprintk("%s: getattr failed %d\n", __func__, ret);
	108	+ trace_nfs_fh_to_dentry(sb, server_fh, fattr->fileid, ret);
108	109	dentry = ERR_PTR(ret);
109	110	goto out_free_label;
110	111	}

fs/nfs/file.c+1−1

@@ -649,7 +649,7 @@ out: · 648 unmodified lines
649	649
650	650	out_swapfile:
651	651	printk(KERN_INFO "NFS: attempt to write to active swap file!\n");
652		− return -EBUSY;
	652	+ return -ETXTBSY;
653	653	}
654	654	EXPORT_SYMBOL_GPL(nfs_file_write);
655	655

fs/nfs/inode.c+27−27

@@ -504,15 +504,15 @@ nfs_fhget(struct super_block sb, struct nfs_fh fh, struct nfs_fattr *fattr, st · 503 unmodified lines
504	504	nfsi->read_cache_jiffies = fattr->time_start;
505	505	nfsi->attr_gencount = fattr->gencount;
506	506	if (fattr->valid & NFS_ATTR_FATTR_ATIME)
507		− inode->i_atime = timespec_to_timespec64(fattr->atime);
	507	+ inode->i_atime = fattr->atime;
508	508	else if (nfs_server_capable(inode, NFS_CAP_ATIME))
509	509	nfs_set_cache_invalid(inode, NFS_INO_INVALID_ATIME);
510	510	if (fattr->valid & NFS_ATTR_FATTR_MTIME)
511		− inode->i_mtime = timespec_to_timespec64(fattr->mtime);
	511	+ inode->i_mtime = fattr->mtime;
512	512	else if (nfs_server_capable(inode, NFS_CAP_MTIME))
513	513	nfs_set_cache_invalid(inode, NFS_INO_INVALID_MTIME);
514	514	if (fattr->valid & NFS_ATTR_FATTR_CTIME)
515		− inode->i_ctime = timespec_to_timespec64(fattr->ctime);
	515	+ inode->i_ctime = fattr->ctime;
516	516	else if (nfs_server_capable(inode, NFS_CAP_CTIME))
517	517	nfs_set_cache_invalid(inode, NFS_INO_INVALID_CTIME);
518	518	if (fattr->valid & NFS_ATTR_FATTR_CHANGE)
@@ -698,7 +698,7 @@ void nfs_setattr_update_inode(struct inode inode, struct iattr attr, · 179 unmodified lines
698	698	if ((attr->ia_valid & ATTR_GID) != 0)
699	699	inode->i_gid = attr->ia_gid;
700	700	if (fattr->valid & NFS_ATTR_FATTR_CTIME)
701		− inode->i_ctime = timespec_to_timespec64(fattr->ctime);
	701	+ inode->i_ctime = fattr->ctime;
702	702	else
703	703	nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE
704	704	\| NFS_INO_INVALID_CTIME);
@@ -709,14 +709,14 @@ void nfs_setattr_update_inode(struct inode inode, struct iattr attr, · 4 unmodified lines
709	709	NFS_I(inode)->cache_validity &= ~(NFS_INO_INVALID_ATIME
710	710	\| NFS_INO_INVALID_CTIME);
711	711	if (fattr->valid & NFS_ATTR_FATTR_ATIME)
712		− inode->i_atime = timespec_to_timespec64(fattr->atime);
	712	+ inode->i_atime = fattr->atime;
713	713	else if (attr->ia_valid & ATTR_ATIME_SET)
714	714	inode->i_atime = attr->ia_atime;
715	715	else
716	716	nfs_set_cache_invalid(inode, NFS_INO_INVALID_ATIME);
717	717
718	718	if (fattr->valid & NFS_ATTR_FATTR_CTIME)
719		− inode->i_ctime = timespec_to_timespec64(fattr->ctime);
	719	+ inode->i_ctime = fattr->ctime;
720	720	else
721	721	nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE
722	722	\| NFS_INO_INVALID_CTIME);
@@ -725,14 +725,14 @@ void nfs_setattr_update_inode(struct inode inode, struct iattr attr, · 2 unmodified lines
725	725	NFS_I(inode)->cache_validity &= ~(NFS_INO_INVALID_MTIME
726	726	\| NFS_INO_INVALID_CTIME);
727	727	if (fattr->valid & NFS_ATTR_FATTR_MTIME)
728		− inode->i_mtime = timespec_to_timespec64(fattr->mtime);
	728	+ inode->i_mtime = fattr->mtime;
729	729	else if (attr->ia_valid & ATTR_MTIME_SET)
730	730	inode->i_mtime = attr->ia_mtime;
731	731	else
732	732	nfs_set_cache_invalid(inode, NFS_INO_INVALID_MTIME);
733	733
734	734	if (fattr->valid & NFS_ATTR_FATTR_CTIME)
735		− inode->i_ctime = timespec_to_timespec64(fattr->ctime);
	735	+ inode->i_ctime = fattr->ctime;
736	736	else
737	737	nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE
738	738	\| NFS_INO_INVALID_CTIME);
@@ -1351,7 +1351,7 @@ static bool nfs_file_has_buffered_writers(struct nfs_inode *nfsi) · 612 unmodified lines
1351	1351
1352	1352	static void nfs_wcc_update_inode(struct inode inode, struct nfs_fattr fattr)
1353	1353	{
1354		− struct timespec ts;
	1354	+ struct timespec64 ts;
1355	1355
1356	1356	if ((fattr->valid & NFS_ATTR_FATTR_PRECHANGE)
1357	1357	&& (fattr->valid & NFS_ATTR_FATTR_CHANGE)
@@ -1361,18 +1361,18 @@ static void nfs_wcc_update_inode(struct inode inode, struct nfs_fattr fattr) · 3 unmodified lines
1361	1361	nfs_set_cache_invalid(inode, NFS_INO_INVALID_DATA);
1362	1362	}
1363	1363	/* If we have atomic WCC data, we may update some attributes */
1364		− ts = timespec64_to_timespec(inode->i_ctime);
	1364	+ ts = inode->i_ctime;
1365	1365	if ((fattr->valid & NFS_ATTR_FATTR_PRECTIME)
1366	1366	&& (fattr->valid & NFS_ATTR_FATTR_CTIME)
1367		− && timespec_equal(&ts, &fattr->pre_ctime)) {
1368		− inode->i_ctime = timespec_to_timespec64(fattr->ctime);
	1367	+ && timespec64_equal(&ts, &fattr->pre_ctime)) {
	1368	+ inode->i_ctime = fattr->ctime;
1369	1369	}
1370	1370
1371		− ts = timespec64_to_timespec(inode->i_mtime);
	1371	+ ts = inode->i_mtime;
1372	1372	if ((fattr->valid & NFS_ATTR_FATTR_PREMTIME)
1373	1373	&& (fattr->valid & NFS_ATTR_FATTR_MTIME)
1374		− && timespec_equal(&ts, &fattr->pre_mtime)) {
1375		− inode->i_mtime = timespec_to_timespec64(fattr->mtime);
	1374	+ && timespec64_equal(&ts, &fattr->pre_mtime)) {
	1375	+ inode->i_mtime = fattr->mtime;
1376	1376	if (S_ISDIR(inode->i_mode))
1377	1377	nfs_set_cache_invalid(inode, NFS_INO_INVALID_DATA);
1378	1378	}
@@ -1398,7 +1398,7 @@ static int nfs_check_inode_attributes(struct inode inode, struct nfs_fattr fat · 19 unmodified lines
1398	1398	struct nfs_inode *nfsi = NFS_I(inode);
1399	1399	loff_t cur_size, new_isize;
1400	1400	unsigned long invalid = 0;
1401		− struct timespec ts;
	1401	+ struct timespec64 ts;
1402	1402
1403	1403	if (NFS_PROTO(inode)->have_delegation(inode, FMODE_READ))
1404	1404	return 0;
@@ -1425,12 +1425,12 @@ static int nfs_check_inode_attributes(struct inode inode, struct nfs_fattr fat · 20 unmodified lines
1425	1425	invalid \|= NFS_INO_INVALID_CHANGE
1426	1426	\| NFS_INO_REVAL_PAGECACHE;
1427	1427
1428		− ts = timespec64_to_timespec(inode->i_mtime);
1429		− if ((fattr->valid & NFS_ATTR_FATTR_MTIME) && !timespec_equal(&ts, &fattr->mtime))
	1428	+ ts = inode->i_mtime;
	1429	+ if ((fattr->valid & NFS_ATTR_FATTR_MTIME) && !timespec64_equal(&ts, &fattr->mtime))
1430	1430	invalid \|= NFS_INO_INVALID_MTIME;
1431	1431
1432		− ts = timespec64_to_timespec(inode->i_ctime);
1433		− if ((fattr->valid & NFS_ATTR_FATTR_CTIME) && !timespec_equal(&ts, &fattr->ctime))
	1432	+ ts = inode->i_ctime;
	1433	+ if ((fattr->valid & NFS_ATTR_FATTR_CTIME) && !timespec64_equal(&ts, &fattr->ctime))
1434	1434	invalid \|= NFS_INO_INVALID_CTIME;
1435	1435
1436	1436	if (fattr->valid & NFS_ATTR_FATTR_SIZE) {
@@ -1460,8 +1460,8 @@ static int nfs_check_inode_attributes(struct inode inode, struct nfs_fattr fat · 23 unmodified lines
1460	1460	if ((fattr->valid & NFS_ATTR_FATTR_NLINK) && inode->i_nlink != fattr->nlink)
1461	1461	invalid \|= NFS_INO_INVALID_OTHER;
1462	1462
1463		− ts = timespec64_to_timespec(inode->i_atime);
1464		− if ((fattr->valid & NFS_ATTR_FATTR_ATIME) && !timespec_equal(&ts, &fattr->atime))
	1463	+ ts = inode->i_atime;
	1464	+ if ((fattr->valid & NFS_ATTR_FATTR_ATIME) && !timespec64_equal(&ts, &fattr->atime))
1465	1465	invalid \|= NFS_INO_INVALID_ATIME;
1466	1466
1467	1467	if (invalid != 0)
@@ -1733,12 +1733,12 @@ int nfs_post_op_update_inode_force_wcc_locked(struct inode *inode, struct nfs_fa · 265 unmodified lines
1733	1733	}
1734	1734	if ((fattr->valid & NFS_ATTR_FATTR_CTIME) != 0 &&
1735	1735	(fattr->valid & NFS_ATTR_FATTR_PRECTIME) == 0) {
1736		− fattr->pre_ctime = timespec64_to_timespec(inode->i_ctime);
	1736	+ fattr->pre_ctime = inode->i_ctime;
1737	1737	fattr->valid \|= NFS_ATTR_FATTR_PRECTIME;
1738	1738	}
1739	1739	if ((fattr->valid & NFS_ATTR_FATTR_MTIME) != 0 &&
1740	1740	(fattr->valid & NFS_ATTR_FATTR_PREMTIME) == 0) {
1741		− fattr->pre_mtime = timespec64_to_timespec(inode->i_mtime);
	1741	+ fattr->pre_mtime = inode->i_mtime;
1742	1742	fattr->valid \|= NFS_ATTR_FATTR_PREMTIME;
1743	1743	}
1744	1744	if ((fattr->valid & NFS_ATTR_FATTR_SIZE) != 0 &&
@@ -1899,7 +1899,7 @@ static int nfs_update_inode(struct inode inode, struct nfs_fattr fattr) · 154 unmodified lines
1899	1899	}
1900	1900
1901	1901	if (fattr->valid & NFS_ATTR_FATTR_MTIME) {
1902		− inode->i_mtime = timespec_to_timespec64(fattr->mtime);
	1902	+ inode->i_mtime = fattr->mtime;
1903	1903	} else if (server->caps & NFS_CAP_MTIME) {
1904	1904	nfsi->cache_validity \|= save_cache_validity &
1905	1905	(NFS_INO_INVALID_MTIME
@@ -1908,7 +1908,7 @@ static int nfs_update_inode(struct inode inode, struct nfs_fattr fattr) · 2 unmodified lines
1908	1908	}
1909	1909
1910	1910	if (fattr->valid & NFS_ATTR_FATTR_CTIME) {
1911		− inode->i_ctime = timespec_to_timespec64(fattr->ctime);
	1911	+ inode->i_ctime = fattr->ctime;
1912	1912	} else if (server->caps & NFS_CAP_CTIME) {
1913	1913	nfsi->cache_validity \|= save_cache_validity &
1914	1914	(NFS_INO_INVALID_CTIME
@@ -1946,7 +1946,7 @@ static int nfs_update_inode(struct inode inode, struct nfs_fattr fattr) · 31 unmodified lines
1946	1946
1947	1947
1948	1948	if (fattr->valid & NFS_ATTR_FATTR_ATIME)
1949		− inode->i_atime = timespec_to_timespec64(fattr->atime);
	1949	+ inode->i_atime = fattr->atime;
1950	1950	else if (server->caps & NFS_CAP_ATIME) {
1951	1951	nfsi->cache_validity \|= save_cache_validity &
1952	1952	(NFS_INO_INVALID_ATIME

fs/nfs/internal.h+1−1

@@ -713,7 +713,7 @@ unsigned int nfs_page_array_len(unsigned int base, size_t len) · 712 unmodified lines
713	713	* 102410241024.
714	714	*/
715	715	static inline
716		−u64 nfs_timespec_to_change_attr(const struct timespec *ts)
	716	+u64 nfs_timespec_to_change_attr(const struct timespec64 *ts)
717	717	{
718	718	return ((u64)ts->tv_sec << 30) + ts->tv_nsec;
719	719	}

fs/nfs/namespace.c+3−0

@@ -157,6 +157,9 @@ struct vfsmount nfs_d_automount(struct path path) · 156 unmodified lines
157	157	if (IS_ERR(mnt))
158	158	goto out;
159	159
	160	+ if (nfs_mountpoint_expiry_timeout < 0)
	161	+ goto out;
	162	+
160	163	mntget(mnt); /* prevent immediate expiration */
161	164	mnt_set_expiry(mnt, &nfs_automount_list);
162	165	schedule_delayed_work(&nfs_automount_task, nfs_mountpoint_expiry_timeout);

fs/nfs/nfs2xdr.c+8−13

@@ -209,9 +209,9 @@ static int decode_fhandle(struct xdr_stream xdr, struct nfs_fh fh) · 208 unmodified lines
209	209	* unsigned int useconds;
210	210	* };
211	211	*/
212		−static __be32 xdr_encode_time(__be32 p, const struct timespec *timep)
	212	+static __be32 xdr_encode_time(__be32 p, const struct timespec64 *timep)
213	213	{
214		− *p++ = cpu_to_be32(timep->tv_sec);
	214	+ *p++ = cpu_to_be32((u32)timep->tv_sec);
215	215	if (timep->tv_nsec != 0)
216	216	*p++ = cpu_to_be32(timep->tv_nsec / NSEC_PER_USEC);
217	217	else
@@ -227,14 +227,14 @@ static __be32 xdr_encode_time(__be32 p, const struct timespec *timep) · 9 unmodified lines
227	227	* Illustrated" by Brent Callaghan, Addison-Wesley, ISBN 0-201-32750-5.
228	228	*/
229	229	static __be32 xdr_encode_current_server_time(__be32 p,
230		− const struct timespec *timep)
	230	+ const struct timespec64 *timep)
231	231	{
232	232	*p++ = cpu_to_be32(timep->tv_sec);
233	233	*p++ = cpu_to_be32(1000000);
234	234	return p;
235	235	}
236	236
237		−static __be32 xdr_decode_time(__be32 p, struct timespec *timep)
	237	+static __be32 xdr_decode_time(__be32 p, struct timespec64 *timep)
238	238	{
239	239	timep->tv_sec = be32_to_cpup(p++);
240	240	timep->tv_nsec = be32_to_cpup(p++) * NSEC_PER_USEC;
@@ -339,7 +339,6 @@ static __be32 xdr_time_not_set(__be32 p) · 98 unmodified lines
339	339	static void encode_sattr(struct xdr_stream xdr, const struct iattr attr,
340	340	struct user_namespace *userns)
341	341	{
342		− struct timespec ts;
343	342	__be32 *p;
344	343
345	344	p = xdr_reserve_space(xdr, NFS_sattr_sz << 2);
@@ -362,19 +361,15 @@ static void encode_sattr(struct xdr_stream xdr, const struct iattr attr, · 16 unmodified lines
362	361	*p++ = cpu_to_be32(NFS2_SATTR_NOT_SET);
363	362
364	363	if (attr->ia_valid & ATTR_ATIME_SET) {
365		− ts = timespec64_to_timespec(attr->ia_atime);
366		− p = xdr_encode_time(p, &ts);
	364	+ p = xdr_encode_time(p, &attr->ia_atime);
367	365	} else if (attr->ia_valid & ATTR_ATIME) {
368		− ts = timespec64_to_timespec(attr->ia_atime);
369		− p = xdr_encode_current_server_time(p, &ts);
	366	+ p = xdr_encode_current_server_time(p, &attr->ia_atime);
370	367	} else
371	368	p = xdr_time_not_set(p);
372	369	if (attr->ia_valid & ATTR_MTIME_SET) {
373		− ts = timespec64_to_timespec(attr->ia_atime);
374		− xdr_encode_time(p, &ts);
	370	+ xdr_encode_time(p, &attr->ia_mtime);
375	371	} else if (attr->ia_valid & ATTR_MTIME) {
376		− ts = timespec64_to_timespec(attr->ia_mtime);
377		− xdr_encode_current_server_time(p, &ts);
	372	+ xdr_encode_current_server_time(p, &attr->ia_mtime);
378	373	} else
379	374	xdr_time_not_set(p);
380	375	}

fs/nfs/nfs3client.c+4−1

@@ -106,7 +106,10 @@ struct nfs_client nfs3_set_ds_client(struct nfs_server mds_srv, · 105 unmodified lines
106	106	cl_init.nconnect = mds_clp->cl_nconnect;
107	107
108	108	if (mds_srv->flags & NFS_MOUNT_NORESVPORT)
109		− set_bit(NFS_CS_NORESVPORT, &cl_init.init_flags);
	109	+ __set_bit(NFS_CS_NORESVPORT, &cl_init.init_flags);
	110	+
	111	+ __set_bit(NFS_CS_NOPING, &cl_init.init_flags);
	112	+ __set_bit(NFS_CS_DS, &cl_init.init_flags);
110	113
111	114	/* Use the MDS nfs_client cl_ipaddr. */
112	115	nfs_init_timeout_values(&ds_timeout, ds_proto, ds_timeo, ds_retrans);

fs/nfs/nfs3xdr.c+5−9

@@ -456,14 +456,14 @@ static void zero_nfs_fh3(struct nfs_fh *fh) · 455 unmodified lines
456	456	* uint32 nseconds;
457	457	* };
458	458	*/
459		−static __be32 xdr_encode_nfstime3(__be32 p, const struct timespec *timep)
	459	+static __be32 xdr_encode_nfstime3(__be32 p, const struct timespec64 *timep)
460	460	{
461		− *p++ = cpu_to_be32(timep->tv_sec);
	461	+ *p++ = cpu_to_be32((u32)timep->tv_sec);
462	462	*p++ = cpu_to_be32(timep->tv_nsec);
463	463	return p;
464	464	}
465	465
466		−static __be32 xdr_decode_nfstime3(__be32 p, struct timespec *timep)
	466	+static __be32 xdr_decode_nfstime3(__be32 p, struct timespec64 *timep)
467	467	{
468	468	timep->tv_sec = be32_to_cpup(p++);
469	469	timep->tv_nsec = be32_to_cpup(p++);
@@ -533,7 +533,6 @@ static __be32 xdr_decode_nfstime3(__be32 p, struct timespec *timep) · 63 unmodified lines
533	533	static void encode_sattr3(struct xdr_stream xdr, const struct iattr attr,
534	534	struct user_namespace *userns)
535	535	{
536		− struct timespec ts;
537	536	u32 nbytes;
538	537	__be32 *p;
539	538
@@ -583,10 +582,8 @@ static void encode_sattr3(struct xdr_stream xdr, const struct iattr attr, · 43 unmodified lines
583	582	*p++ = xdr_zero;
584	583
585	584	if (attr->ia_valid & ATTR_ATIME_SET) {
586		− struct timespec ts;
587	585	*p++ = xdr_two;
588		− ts = timespec64_to_timespec(attr->ia_atime);
589		− p = xdr_encode_nfstime3(p, &ts);
	586	+ p = xdr_encode_nfstime3(p, &attr->ia_atime);
590	587	} else if (attr->ia_valid & ATTR_ATIME) {
591	588	*p++ = xdr_one;
592	589	} else
@@ -594,8 +591,7 @@ static void encode_sattr3(struct xdr_stream xdr, const struct iattr attr, · 1 unmodified line
594	591
595	592	if (attr->ia_valid & ATTR_MTIME_SET) {
596	593	*p++ = xdr_two;
597		− ts = timespec64_to_timespec(attr->ia_mtime);
598		− xdr_encode_nfstime3(p, &ts);
	594	+ xdr_encode_nfstime3(p, &attr->ia_mtime);
599	595	} else if (attr->ia_valid & ATTR_MTIME) {
600	596	*p = xdr_one;
601	597	} else

fs/nfs/nfs42.h+14−1

@@ -13,8 +13,10 @@ · 12 unmodified lines
13	13	#define PNFS_LAYOUTSTATS_MAXDEV (4)
14	14
15	15	/* nfs4.2proc.c */
	16	+#ifdef CONFIG_NFS_V4_2
16	17	int nfs42_proc_allocate(struct file *, loff_t, loff_t);
17		−ssize_t nfs42_proc_copy(struct file , loff_t, struct file , loff_t, size_t);
	18	+ssize_t nfs42_proc_copy(struct file , loff_t, struct file , loff_t, size_t,
	19	+ struct nl4_server , nfs4_stateid , bool);
18	20	int nfs42_proc_deallocate(struct file *, loff_t, loff_t);
19	21	loff_t nfs42_proc_llseek(struct file *, loff_t, int);
20	22	int nfs42_proc_layoutstats_generic(struct nfs_server *,
@@ -23,5 +25,16 @@ int nfs42_proc_clone(struct file , struct file , loff_t, loff_t, loff_t); · 2 unmodified lines
23	25	int nfs42_proc_layouterror(struct pnfs_layout_segment *lseg,
24	26	const struct nfs42_layout_error *errors,
25	27	size_t n);
	28	+int nfs42_proc_copy_notify(struct file , struct file ,
	29	+ struct nfs42_copy_notify_res *);
	30	+static inline bool nfs42_files_from_same_server(struct file *in,
	31	+ struct file *out)
	32	+{
	33	+ struct nfs_client *c_in = (NFS_SERVER(file_inode(in)))->nfs_client;
	34	+ struct nfs_client *c_out = (NFS_SERVER(file_inode(out)))->nfs_client;
26	35
	36	+ return nfs4_check_serverowner_major_id(c_in->cl_serverowner,
	37	+ c_out->cl_serverowner);
	38	+}
	39	+#endif /* CONFIG_NFS_V4_2 */
27	40	#endif /* __LINUX_FS_NFS_NFS4_2_H */

fs/nfs/nfs42proc.c+168−33

@@ -3,6 +3,7 @@ · 2 unmodified lines
3	3	* Copyright (c) 2014 Anna Schumaker <Anna.Schumaker@Netapp.com>
4	4	*/
5	5	#include <linux/fs.h>
	6	+#include <linux/sunrpc/addr.h>
6	7	#include <linux/sunrpc/sched.h>
7	8	#include <linux/nfs.h>
8	9	#include <linux/nfs3.h>
@@ -15,10 +16,30 @@ · 6 unmodified lines
15	16	#include "pnfs.h"
16	17	#include "nfs4session.h"
17	18	#include "internal.h"
	19	+#include "delegation.h"
18	20
19	21	#define NFSDBG_FACILITY NFSDBG_PROC
20	22	static int nfs42_do_offload_cancel_async(struct file dst, nfs4_stateid std);
21	23
	24	+static void nfs42_set_netaddr(struct file filep, struct nfs42_netaddr naddr)
	25	+{
	26	+ struct nfs_client *clp = (NFS_SERVER(file_inode(filep)))->nfs_client;
	27	+ unsigned short port = 2049;
	28	+
	29	+ rcu_read_lock();
	30	+ naddr->netid_len = scnprintf(naddr->netid,
	31	+ sizeof(naddr->netid), "%s",
	32	+ rpc_peeraddr2str(clp->cl_rpcclient,
	33	+ RPC_DISPLAY_NETID));
	34	+ naddr->addr_len = scnprintf(naddr->addr,
	35	+ sizeof(naddr->addr),
	36	+ "%s.%u.%u",
	37	+ rpc_peeraddr2str(clp->cl_rpcclient,
	38	+ RPC_DISPLAY_ADDR),
	39	+ port >> 8, port & 255);
	40	+ rcu_read_unlock();
	41	+}
	42	+
22	43	static int _nfs42_proc_fallocate(struct rpc_message msg, struct file filep,
23	44	struct nfs_lock_context *lock, loff_t offset, loff_t len)
24	45	{
@@ -28,7 +49,7 @@ static int _nfs42_proc_fallocate(struct rpc_message msg, struct file filep, · 3 unmodified lines
28	49	.falloc_fh = NFS_FH(inode),
29	50	.falloc_offset = offset,
30	51	.falloc_length = len,
31		− .falloc_bitmask = server->cache_consistency_bitmask,
	52	+ .falloc_bitmask = nfs4_fattr_bitmap,
32	53	};
33	54	struct nfs42_falloc_res res = {
34	55	.falloc_server = server,
@@ -132,22 +153,26 @@ out_unlock: · 97 unmodified lines
132	153	}
133	154
134	155	static int handle_async_copy(struct nfs42_copy_res *res,
135		− struct nfs_server *server,
	156	+ struct nfs_server *dst_server,
	157	+ struct nfs_server *src_server,
136	158	struct file *src,
137	159	struct file *dst,
138		− nfs4_stateid *src_stateid)
	160	+ nfs4_stateid *src_stateid,
	161	+ bool *restart)
139	162	{
140	163	struct nfs4_copy_state copy, tmp_copy;
141	164	int status = NFS4_OK;
142	165	bool found_pending = false;
143		− struct nfs_open_context *ctx = nfs_file_open_context(dst);
	166	+ struct nfs_open_context *dst_ctx = nfs_file_open_context(dst);
	167	+ struct nfs_open_context *src_ctx = nfs_file_open_context(src);
144	168
145	169	copy = kzalloc(sizeof(struct nfs4_copy_state), GFP_NOFS);
146	170	if (!copy)
147	171	return -ENOMEM;
148	172
149		− spin_lock(&server->nfs_client->cl_lock);
150		− list_for_each_entry(tmp_copy, &server->nfs_client->pending_cb_stateids,
	173	+ spin_lock(&dst_server->nfs_client->cl_lock);
	174	+ list_for_each_entry(tmp_copy,
	175	+ &dst_server->nfs_client->pending_cb_stateids,
151	176	copies) {
152	177	if (memcmp(&res->write_res.stateid, &tmp_copy->stateid,
153	178	NFS4_STATEID_SIZE))
@@ -157,7 +182,7 @@ static int handle_async_copy(struct nfs42_copy_res *res, · 3 unmodified lines
157	182	break;
158	183	}
159	184	if (found_pending) {
160		− spin_unlock(&server->nfs_client->cl_lock);
	185	+ spin_unlock(&dst_server->nfs_client->cl_lock);
161	186	kfree(copy);
162	187	copy = tmp_copy;
163	188	goto out;
@@ -165,19 +190,32 @@ static int handle_async_copy(struct nfs42_copy_res *res, · 1 unmodified line
165	190
166	191	memcpy(&copy->stateid, &res->write_res.stateid, NFS4_STATEID_SIZE);
167	192	init_completion(&copy->completion);
168		− copy->parent_state = ctx->state;
	193	+ copy->parent_dst_state = dst_ctx->state;
	194	+ copy->parent_src_state = src_ctx->state;
169	195
170		− list_add_tail(&copy->copies, &server->ss_copies);
171		− spin_unlock(&server->nfs_client->cl_lock);
	196	+ list_add_tail(&copy->copies, &dst_server->ss_copies);
	197	+ spin_unlock(&dst_server->nfs_client->cl_lock);
	198	+
	199	+ if (dst_server != src_server) {
	200	+ spin_lock(&src_server->nfs_client->cl_lock);
	201	+ list_add_tail(&copy->src_copies, &src_server->ss_copies);
	202	+ spin_unlock(&src_server->nfs_client->cl_lock);
	203	+ }
172	204
173	205	status = wait_for_completion_interruptible(&copy->completion);
174		− spin_lock(&server->nfs_client->cl_lock);
	206	+ spin_lock(&dst_server->nfs_client->cl_lock);
175	207	list_del_init(&copy->copies);
176		− spin_unlock(&server->nfs_client->cl_lock);
	208	+ spin_unlock(&dst_server->nfs_client->cl_lock);
	209	+ if (dst_server != src_server) {
	210	+ spin_lock(&src_server->nfs_client->cl_lock);
	211	+ list_del_init(&copy->src_copies);
	212	+ spin_unlock(&src_server->nfs_client->cl_lock);
	213	+ }
177	214	if (status == -ERESTARTSYS) {
178	215	goto out_cancel;
179		− } else if (copy->flags) {
	216	+ } else if (copy->flags \|\| copy->error == NFS4ERR_PARTNER_NO_AUTH) {
180	217	status = -EAGAIN;
	218	+ *restart = true;
181	219	goto out_cancel;
182	220	}
183	221	out:
@@ -185,12 +223,14 @@ out: · 1 unmodified line
185	223	memcpy(&res->write_res.verifier, &copy->verf, sizeof(copy->verf));
186	224	status = -copy->error;
187	225
	226	+out_free:
188	227	kfree(copy);
189	228	return status;
190	229	out_cancel:
191	230	nfs42_do_offload_cancel_async(dst, &copy->stateid);
192		− kfree(copy);
193		− return status;
	231	+ if (!nfs42_files_from_same_server(src, dst))
	232	+ nfs42_do_offload_cancel_async(src, src_stateid);
	233	+ goto out_free;
194	234	}
195	235
196	236	static int process_copy_commit(struct file *dst, loff_t pos_dst,
@@ -222,7 +262,10 @@ static ssize_t _nfs42_proc_copy(struct file *src, · 25 unmodified lines
222	262	struct file *dst,
223	263	struct nfs_lock_context *dst_lock,
224	264	struct nfs42_copy_args *args,
225		− struct nfs42_copy_res *res)
	265	+ struct nfs42_copy_res *res,
	266	+ struct nl4_server *nss,
	267	+ nfs4_stateid *cnr_stateid,
	268	+ bool *restart)
226	269	{
227	270	struct rpc_message msg = {
228	271	.rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_COPY],
@@ -230,17 +273,23 @@ static ssize_t _nfs42_proc_copy(struct file *src, · 1 unmodified line
230	273	.rpc_resp = res,
231	274	};
232	275	struct inode *dst_inode = file_inode(dst);
233		− struct nfs_server *server = NFS_SERVER(dst_inode);
	276	+ struct inode *src_inode = file_inode(src);
	277	+ struct nfs_server *dst_server = NFS_SERVER(dst_inode);
	278	+ struct nfs_server *src_server = NFS_SERVER(src_inode);
234	279	loff_t pos_src = args->src_pos;
235	280	loff_t pos_dst = args->dst_pos;
236	281	size_t count = args->count;
237	282	ssize_t status;
238	283
239		− status = nfs4_set_rw_stateid(&args->src_stateid, src_lock->open_context,
240		− src_lock, FMODE_READ);
241		− if (status)
242		− return status;
243		−
	284	+ if (nss) {
	285	+ args->cp_src = nss;
	286	+ nfs4_stateid_copy(&args->src_stateid, cnr_stateid);
	287	+ } else {
	288	+ status = nfs4_set_rw_stateid(&args->src_stateid,
	289	+ src_lock->open_context, src_lock, FMODE_READ);
	290	+ if (status)
	291	+ return status;
	292	+ }
244	293	status = nfs_filemap_write_and_wait_range(file_inode(src)->i_mapping,
245	294	pos_src, pos_src + (loff_t)count - 1);
246	295	if (status)
@@ -262,13 +311,15 @@ static ssize_t _nfs42_proc_copy(struct file *src, · 15 unmodified lines
262	311	if (!res->commit_res.verf)
263	312	return -ENOMEM;
264	313	}
	314	+ set_bit(NFS_CLNT_SRC_SSC_COPY_STATE,
	315	+ &src_lock->open_context->state->flags);
265	316	set_bit(NFS_CLNT_DST_SSC_COPY_STATE,
266	317	&dst_lock->open_context->state->flags);
267	318
268		− status = nfs4_call_sync(server->client, server, &msg,
	319	+ status = nfs4_call_sync(dst_server->client, dst_server, &msg,
269	320	&args->seq_args, &res->seq_res, 0);
270	321	if (status == -ENOTSUPP)
271		− server->caps &= ~NFS_CAP_COPY;
	322	+ dst_server->caps &= ~NFS_CAP_COPY;
272	323	if (status)
273	324	goto out;
274	325
@@ -280,8 +331,8 @@ static ssize_t _nfs42_proc_copy(struct file *src, · 5 unmodified lines
280	331	}
281	332
282	333	if (!res->synchronous) {
283		− status = handle_async_copy(res, server, src, dst,
284		− &args->src_stateid);
	334	+ status = handle_async_copy(res, dst_server, src_server, src,
	335	+ dst, &args->src_stateid, restart);
285	336	if (status)
286	337	return status;
287	338	}
@@ -304,8 +355,9 @@ out: · 16 unmodified lines
304	355	}
305	356
306	357	ssize_t nfs42_proc_copy(struct file *src, loff_t pos_src,
307		− struct file *dst, loff_t pos_dst,
308		− size_t count)
	358	+ struct file *dst, loff_t pos_dst, size_t count,
	359	+ struct nl4_server *nss,
	360	+ nfs4_stateid *cnr_stateid, bool sync)
309	361	{
310	362	struct nfs_server *server = NFS_SERVER(file_inode(dst));
311	363	struct nfs_lock_context *src_lock;
@@ -316,7 +368,7 @@ ssize_t nfs42_proc_copy(struct file *src, loff_t pos_src, · 4 unmodified lines
316	368	.dst_fh = NFS_FH(file_inode(dst)),
317	369	.dst_pos = pos_dst,
318	370	.count = count,
319		− .sync = false,
	371	+ .sync = sync,
320	372	};
321	373	struct nfs42_copy_res res;
322	374	struct nfs4_exception src_exception = {
@@ -328,6 +380,7 @@ ssize_t nfs42_proc_copy(struct file *src, loff_t pos_src, · 5 unmodified lines
328	380	.stateid = &args.dst_stateid,
329	381	};
330	382	ssize_t err, err2;
	383	+ bool restart = false;
331	384
332	385	src_lock = nfs_get_lock_context(nfs_file_open_context(src));
333	386	if (IS_ERR(src_lock))
@@ -347,21 +400,33 @@ ssize_t nfs42_proc_copy(struct file *src, loff_t pos_src, · 13 unmodified lines
347	400	inode_lock(file_inode(dst));
348	401	err = _nfs42_proc_copy(src, src_lock,
349	402	dst, dst_lock,
350		− &args, &res);
	403	+ &args, &res,
	404	+ nss, cnr_stateid, &restart);
351	405	inode_unlock(file_inode(dst));
352	406
353	407	if (err >= 0)
354	408	break;
355		− if (err == -ENOTSUPP) {
	409	+ if (err == -ENOTSUPP &&
	410	+ nfs42_files_from_same_server(src, dst)) {
356	411	err = -EOPNOTSUPP;
357	412	break;
358	413	} else if (err == -EAGAIN) {
359		− dst_exception.retry = 1;
360		− continue;
	414	+ if (!restart) {
	415	+ dst_exception.retry = 1;
	416	+ continue;
	417	+ }
	418	+ break;
361	419	} else if (err == -NFS4ERR_OFFLOAD_NO_REQS && !args.sync) {
362	420	args.sync = true;
363	421	dst_exception.retry = 1;
364	422	continue;
	423	+ } else if ((err == -ESTALE \|\|
	424	+ err == -NFS4ERR_OFFLOAD_DENIED \|\|
	425	+ err == -ENOTSUPP) &&
	426	+ !nfs42_files_from_same_server(src, dst)) {
	427	+ nfs42_do_offload_cancel_async(src, &args.src_stateid);
	428	+ err = -EOPNOTSUPP;
	429	+ break;
365	430	}
366	431
367	432	err2 = nfs4_handle_exception(server, err, &src_exception);
@@ -459,6 +524,76 @@ static int nfs42_do_offload_cancel_async(struct file *dst, · 91 unmodified lines
459	524	return status;
460	525	}
461	526
	527	+static int _nfs42_proc_copy_notify(struct file src, struct file dst,
	528	+ struct nfs42_copy_notify_args *args,
	529	+ struct nfs42_copy_notify_res *res)
	530	+{
	531	+ struct nfs_server *src_server = NFS_SERVER(file_inode(src));
	532	+ struct rpc_message msg = {
	533	+ .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_COPY_NOTIFY],
	534	+ .rpc_argp = args,
	535	+ .rpc_resp = res,
	536	+ };
	537	+ int status;
	538	+ struct nfs_open_context *ctx;
	539	+ struct nfs_lock_context *l_ctx;
	540	+
	541	+ ctx = get_nfs_open_context(nfs_file_open_context(src));
	542	+ l_ctx = nfs_get_lock_context(ctx);
	543	+ if (IS_ERR(l_ctx))
	544	+ return PTR_ERR(l_ctx);
	545	+
	546	+ status = nfs4_set_rw_stateid(&args->cna_src_stateid, ctx, l_ctx,
	547	+ FMODE_READ);
	548	+ nfs_put_lock_context(l_ctx);
	549	+ if (status)
	550	+ return status;
	551	+
	552	+ status = nfs4_call_sync(src_server->client, src_server, &msg,
	553	+ &args->cna_seq_args, &res->cnr_seq_res, 0);
	554	+ if (status == -ENOTSUPP)
	555	+ src_server->caps &= ~NFS_CAP_COPY_NOTIFY;
	556	+
	557	+ put_nfs_open_context(nfs_file_open_context(src));
	558	+ return status;
	559	+}
	560	+
	561	+int nfs42_proc_copy_notify(struct file src, struct file dst,
	562	+ struct nfs42_copy_notify_res *res)
	563	+{
	564	+ struct nfs_server *src_server = NFS_SERVER(file_inode(src));
	565	+ struct nfs42_copy_notify_args *args;
	566	+ struct nfs4_exception exception = {
	567	+ .inode = file_inode(src),
	568	+ };
	569	+ int status;
	570	+
	571	+ if (!(src_server->caps & NFS_CAP_COPY_NOTIFY))
	572	+ return -EOPNOTSUPP;
	573	+
	574	+ args = kzalloc(sizeof(struct nfs42_copy_notify_args), GFP_NOFS);
	575	+ if (args == NULL)
	576	+ return -ENOMEM;
	577	+
	578	+ args->cna_src_fh = NFS_FH(file_inode(src)),
	579	+ args->cna_dst.nl4_type = NL4_NETADDR;
	580	+ nfs42_set_netaddr(dst, &args->cna_dst.u.nl4_addr);
	581	+ exception.stateid = &args->cna_src_stateid;
	582	+
	583	+ do {
	584	+ status = _nfs42_proc_copy_notify(src, dst, args, res);
	585	+ if (status == -ENOTSUPP) {
	586	+ status = -EOPNOTSUPP;
	587	+ goto out;
	588	+ }
	589	+ status = nfs4_handle_exception(src_server, status, &exception);
	590	+ } while (exception.retry);
	591	+
	592	+out:
	593	+ kfree(args);
	594	+ return status;
	595	+}
	596	+
462	597	static loff_t _nfs42_proc_llseek(struct file *filep,
463	598	struct nfs_lock_context *lock, loff_t offset, int whence)
464	599	{

fs/nfs/nfs42xdr.c+188−2

@@ -21,7 +21,10 @@ · 20 unmodified lines
21	21	#define encode_copy_maxsz (op_encode_hdr_maxsz + \
22	22	XDR_QUADLEN(NFS4_STATEID_SIZE) + \
23	23	XDR_QUADLEN(NFS4_STATEID_SIZE) + \
24		− 2 + 2 + 2 + 1 + 1 + 1)
	24	+ 2 + 2 + 2 + 1 + 1 + 1 +\
	25	+ 1 + /* One cnr_source_server */\
	26	+ 1 + /* nl4_type */ \
	27	+ 1 + XDR_QUADLEN(NFS4_OPAQUE_LIMIT))
25	28	#define decode_copy_maxsz (op_decode_hdr_maxsz + \
26	29	NFS42_WRITE_RES_SIZE + \
27	30	1 /* cr_consecutive */ + \
@@ -29,6 +32,16 @@ · 1 unmodified line
29	32	#define encode_offload_cancel_maxsz (op_encode_hdr_maxsz + \
30	33	XDR_QUADLEN(NFS4_STATEID_SIZE))
31	34	#define decode_offload_cancel_maxsz (op_decode_hdr_maxsz)
	35	+#define encode_copy_notify_maxsz (op_encode_hdr_maxsz + \
	36	+ XDR_QUADLEN(NFS4_STATEID_SIZE) + \
	37	+ 1 + /* nl4_type */ \
	38	+ 1 + XDR_QUADLEN(NFS4_OPAQUE_LIMIT))
	39	+#define decode_copy_notify_maxsz (op_decode_hdr_maxsz + \
	40	+ 3 + /* cnr_lease_time */\
	41	+ XDR_QUADLEN(NFS4_STATEID_SIZE) + \
	42	+ 1 + /* Support 1 cnr_source_server */\
	43	+ 1 + /* nl4_type */ \
	44	+ 1 + XDR_QUADLEN(NFS4_OPAQUE_LIMIT))
32	45	#define encode_deallocate_maxsz (op_encode_hdr_maxsz + \
33	46	encode_fallocate_maxsz)
34	47	#define decode_deallocate_maxsz (op_decode_hdr_maxsz)
@@ -99,6 +112,12 @@ · 64 unmodified lines
99	112	decode_sequence_maxsz + \
100	113	decode_putfh_maxsz + \
101	114	decode_offload_cancel_maxsz)
	115	+#define NFS4_enc_copy_notify_sz (compound_encode_hdr_maxsz + \
	116	+ encode_putfh_maxsz + \
	117	+ encode_copy_notify_maxsz)
	118	+#define NFS4_dec_copy_notify_sz (compound_decode_hdr_maxsz + \
	119	+ decode_putfh_maxsz + \
	120	+ decode_copy_notify_maxsz)
102	121	#define NFS4_enc_deallocate_sz (compound_encode_hdr_maxsz + \
103	122	encode_sequence_maxsz + \
104	123	encode_putfh_maxsz + \
@@ -166,6 +185,26 @@ static void encode_allocate(struct xdr_stream *xdr, · 61 unmodified lines
166	185	encode_fallocate(xdr, args);
167	186	}
168	187
	188	+static void encode_nl4_server(struct xdr_stream *xdr,
	189	+ const struct nl4_server *ns)
	190	+{
	191	+ encode_uint32(xdr, ns->nl4_type);
	192	+ switch (ns->nl4_type) {
	193	+ case NL4_NAME:
	194	+ case NL4_URL:
	195	+ encode_string(xdr, ns->u.nl4_str_sz, ns->u.nl4_str);
	196	+ break;
	197	+ case NL4_NETADDR:
	198	+ encode_string(xdr, ns->u.nl4_addr.netid_len,
	199	+ ns->u.nl4_addr.netid);
	200	+ encode_string(xdr, ns->u.nl4_addr.addr_len,
	201	+ ns->u.nl4_addr.addr);
	202	+ break;
	203	+ default:
	204	+ WARN_ON_ONCE(1);
	205	+ }
	206	+}
	207	+
169	208	static void encode_copy(struct xdr_stream *xdr,
170	209	const struct nfs42_copy_args *args,
171	210	struct compound_hdr *hdr)
@@ -180,7 +219,12 @@ static void encode_copy(struct xdr_stream *xdr, · 8 unmodified lines
180	219
181	220	encode_uint32(xdr, 1); /* consecutive = true */
182	221	encode_uint32(xdr, args->sync);
183		− encode_uint32(xdr, 0); /* src server list */
	222	+ if (args->cp_src == NULL) { /* intra-ssc */
	223	+ encode_uint32(xdr, 0); /* no src server list */
	224	+ return;
	225	+ }
	226	+ encode_uint32(xdr, 1); /* supporting 1 server */
	227	+ encode_nl4_server(xdr, args->cp_src);
184	228	}
185	229
186	230	static void encode_offload_cancel(struct xdr_stream *xdr,
@@ -191,6 +235,15 @@ static void encode_offload_cancel(struct xdr_stream *xdr, · 4 unmodified lines
191	235	encode_nfs4_stateid(xdr, &args->osa_stateid);
192	236	}
193	237
	238	+static void encode_copy_notify(struct xdr_stream *xdr,
	239	+ const struct nfs42_copy_notify_args *args,
	240	+ struct compound_hdr *hdr)
	241	+{
	242	+ encode_op_hdr(xdr, OP_COPY_NOTIFY, decode_copy_notify_maxsz, hdr);
	243	+ encode_nfs4_stateid(xdr, &args->cna_src_stateid);
	244	+ encode_nl4_server(xdr, &args->cna_dst);
	245	+}
	246	+
194	247	static void encode_deallocate(struct xdr_stream *xdr,
195	248	const struct nfs42_falloc_args *args,
196	249	struct compound_hdr *hdr)
@@ -354,6 +407,25 @@ static void nfs4_xdr_enc_offload_cancel(struct rpc_rqst *req, · 157 unmodified lines
354	407	encode_nops(&hdr);
355	408	}
356	409
	410	+/*
	411	+ * Encode COPY_NOTIFY request
	412	+ */
	413	+static void nfs4_xdr_enc_copy_notify(struct rpc_rqst *req,
	414	+ struct xdr_stream *xdr,
	415	+ const void *data)
	416	+{
	417	+ const struct nfs42_copy_notify_args *args = data;
	418	+ struct compound_hdr hdr = {
	419	+ .minorversion = nfs4_xdr_minorversion(&args->cna_seq_args),
	420	+ };
	421	+
	422	+ encode_compound_hdr(xdr, req, &hdr);
	423	+ encode_sequence(xdr, &args->cna_seq_args, &hdr);
	424	+ encode_putfh(xdr, args->cna_src_fh, &hdr);
	425	+ encode_copy_notify(xdr, args, &hdr);
	426	+ encode_nops(&hdr);
	427	+}
	428	+
357	429	/*
358	430	* Encode DEALLOCATE request
359	431	*/
@@ -490,6 +562,58 @@ static int decode_write_response(struct xdr_stream *xdr, · 130 unmodified lines
490	562	return decode_verifier(xdr, &res->verifier.verifier);
491	563	}
492	564
	565	+static int decode_nl4_server(struct xdr_stream xdr, struct nl4_server ns)
	566	+{
	567	+ struct nfs42_netaddr *naddr;
	568	+ uint32_t dummy;
	569	+ char *dummy_str;
	570	+ __be32 *p;
	571	+ int status;
	572	+
	573	+ /* nl_type */
	574	+ p = xdr_inline_decode(xdr, 4);
	575	+ if (unlikely(!p))
	576	+ return -EIO;
	577	+ ns->nl4_type = be32_to_cpup(p);
	578	+ switch (ns->nl4_type) {
	579	+ case NL4_NAME:
	580	+ case NL4_URL:
	581	+ status = decode_opaque_inline(xdr, &dummy, &dummy_str);
	582	+ if (unlikely(status))
	583	+ return status;
	584	+ if (unlikely(dummy > NFS4_OPAQUE_LIMIT))
	585	+ return -EIO;
	586	+ memcpy(&ns->u.nl4_str, dummy_str, dummy);
	587	+ ns->u.nl4_str_sz = dummy;
	588	+ break;
	589	+ case NL4_NETADDR:
	590	+ naddr = &ns->u.nl4_addr;
	591	+
	592	+ /* netid string */
	593	+ status = decode_opaque_inline(xdr, &dummy, &dummy_str);
	594	+ if (unlikely(status))
	595	+ return status;
	596	+ if (unlikely(dummy > RPCBIND_MAXNETIDLEN))
	597	+ return -EIO;
	598	+ naddr->netid_len = dummy;
	599	+ memcpy(naddr->netid, dummy_str, naddr->netid_len);
	600	+
	601	+ /* uaddr string */
	602	+ status = decode_opaque_inline(xdr, &dummy, &dummy_str);
	603	+ if (unlikely(status))
	604	+ return status;
	605	+ if (unlikely(dummy > RPCBIND_MAXUADDRLEN))
	606	+ return -EIO;
	607	+ naddr->addr_len = dummy;
	608	+ memcpy(naddr->addr, dummy_str, naddr->addr_len);
	609	+ break;
	610	+ default:
	611	+ WARN_ON_ONCE(1);
	612	+ return -EIO;
	613	+ }
	614	+ return 0;
	615	+}
	616	+
493	617	static int decode_copy_requirements(struct xdr_stream *xdr,
494	618	struct nfs42_copy_res *res) {
495	619	__be32 *p;
@@ -529,6 +653,42 @@ static int decode_offload_cancel(struct xdr_stream *xdr, · 33 unmodified lines
529	653	return decode_op_hdr(xdr, OP_OFFLOAD_CANCEL);
530	654	}
531	655
	656	+static int decode_copy_notify(struct xdr_stream *xdr,
	657	+ struct nfs42_copy_notify_res *res)
	658	+{
	659	+ __be32 *p;
	660	+ int status, count;
	661	+
	662	+ status = decode_op_hdr(xdr, OP_COPY_NOTIFY);
	663	+ if (status)
	664	+ return status;
	665	+ /* cnr_lease_time */
	666	+ p = xdr_inline_decode(xdr, 12);
	667	+ if (unlikely(!p))
	668	+ return -EIO;
	669	+ p = xdr_decode_hyper(p, &res->cnr_lease_time.seconds);
	670	+ res->cnr_lease_time.nseconds = be32_to_cpup(p);
	671	+
	672	+ status = decode_opaque_fixed(xdr, &res->cnr_stateid, NFS4_STATEID_SIZE);
	673	+ if (unlikely(status))
	674	+ return -EIO;
	675	+
	676	+ /* number of source addresses */
	677	+ p = xdr_inline_decode(xdr, 4);
	678	+ if (unlikely(!p))
	679	+ return -EIO;
	680	+
	681	+ count = be32_to_cpup(p);
	682	+ if (count > 1)
	683	+ pr_warn("NFS: %s: nsvr %d > Supported. Use first servers\n",
	684	+ __func__, count);
	685	+
	686	+ status = decode_nl4_server(xdr, &res->cnr_src);
	687	+ if (unlikely(status))
	688	+ return -EIO;
	689	+ return 0;
	690	+}
	691	+
532	692	static int decode_deallocate(struct xdr_stream xdr, struct nfs42_falloc_res res)
533	693	{
534	694	return decode_op_hdr(xdr, OP_DEALLOCATE);
@@ -656,6 +816,32 @@ out: · 121 unmodified lines
656	816	return status;
657	817	}
658	818
	819	+/*
	820	+ * Decode COPY_NOTIFY response
	821	+ */
	822	+static int nfs4_xdr_dec_copy_notify(struct rpc_rqst *rqstp,
	823	+ struct xdr_stream *xdr,
	824	+ void *data)
	825	+{
	826	+ struct nfs42_copy_notify_res *res = data;
	827	+ struct compound_hdr hdr;
	828	+ int status;
	829	+
	830	+ status = decode_compound_hdr(xdr, &hdr);
	831	+ if (status)
	832	+ goto out;
	833	+ status = decode_sequence(xdr, &res->cnr_seq_res, rqstp);
	834	+ if (status)
	835	+ goto out;
	836	+ status = decode_putfh(xdr);
	837	+ if (status)
	838	+ goto out;
	839	+ status = decode_copy_notify(xdr, res);
	840	+
	841	+out:
	842	+ return status;
	843	+}
	844	+
659	845	/*
660	846	* Decode DEALLOCATE request
661	847	*/

fs/nfs/nfs4_fs.h+19−2

@@ -166,9 +166,9 @@ enum { · 165 unmodified lines
166	166	NFS_STATE_RECOVERY_FAILED, /* OPEN stateid state recovery failed */
167	167	NFS_STATE_MAY_NOTIFY_LOCK, /* server may CB_NOTIFY_LOCK */
168	168	NFS_STATE_CHANGE_WAIT, /* A state changing operation is outstanding */
169		−#ifdef CONFIG_NFS_V4_2
170	169	NFS_CLNT_DST_SSC_COPY_STATE, /* dst server open state on client*/
171		−#endif /* CONFIG_NFS_V4_2 */
	170	+ NFS_CLNT_SRC_SSC_COPY_STATE, /* src server open state on client*/
	171	+ NFS_SRV_SSC_COPY_STATE, /* ssc state on the dst server */
172	172	};
173	173
174	174	struct nfs4_state {
@@ -311,6 +311,13 @@ extern int nfs4_set_rw_stateid(nfs4_stateid *stateid, · 136 unmodified lines
311	311	const struct nfs_open_context *ctx,
312	312	const struct nfs_lock_context *l_ctx,
313	313	fmode_t fmode);
	314	+extern int nfs4_proc_getattr(struct nfs_server server, struct nfs_fh fhandle,
	315	+ struct nfs_fattr fattr, struct nfs4_label label,
	316	+ struct inode *inode);
	317	+extern int update_open_stateid(struct nfs4_state *state,
	318	+ const nfs4_stateid *open_stateid,
	319	+ const nfs4_stateid *deleg_stateid,
	320	+ fmode_t fmode);
314	321
315	322	extern int nfs4_proc_get_lease_time(struct nfs_client *clp,
316	323	struct nfs_fsinfo *fsinfo);
@@ -445,6 +452,8 @@ extern void nfs4_set_lease_period(struct nfs_client *clp, · 128 unmodified lines
445	452
446	453
447	454	/* nfs4state.c */
	455	+extern const nfs4_stateid current_stateid;
	456	+
448	457	const struct cred nfs4_get_clid_cred(struct nfs_client clp);
449	458	const struct cred nfs4_get_machine_cred(struct nfs_client clp);
450	459	const struct cred nfs4_get_renew_cred(struct nfs_client clp);
@@ -457,6 +466,8 @@ int nfs41_discover_server_trunking(struct nfs_client *clp, · 6 unmodified lines
457	466	struct nfs_client *, const struct cred );
458	467	extern void nfs4_schedule_session_recovery(struct nfs4_session *, int);
459	468	extern void nfs41_notify_server(struct nfs_client *);
	469	+bool nfs4_check_serverowner_major_id(struct nfs41_server_owner *o1,
	470	+ struct nfs41_server_owner *o2);
460	471	#else
461	472	static inline void nfs4_schedule_session_recovery(struct nfs4_session *session, int err)
462	473	{
@@ -572,6 +583,12 @@ static inline bool nfs4_stateid_is_newer(const nfs4_stateid *s1, const nfs4_stat · 109 unmodified lines
572	583	return (s32)(be32_to_cpu(s1->seqid) - be32_to_cpu(s2->seqid)) > 0;
573	584	}
574	585
	586	+static inline bool nfs4_stateid_match_or_older(const nfs4_stateid dst, const nfs4_stateid src)
	587	+{
	588	+ return nfs4_stateid_match_other(dst, src) &&
	589	+ !(src->seqid && nfs4_stateid_is_newer(dst, src));
	590	+}
	591	+
575	592	static inline void nfs4_stateid_seqid_inc(nfs4_stateid *s1)
576	593	{
577	594	u32 seqid = be32_to_cpu(s1->seqid);

fs/nfs/nfs4client.c+8−5

@@ -629,7 +629,7 @@ out: · 628 unmodified lines
629	629	/*
630	630	* Returns true if the server major ids match
631	631	*/
632		−static bool
	632	+bool
633	633	nfs4_check_serverowner_major_id(struct nfs41_server_owner *o1,
634	634	struct nfs41_server_owner *o2)
635	635	{
@@ -879,14 +879,17 @@ static int nfs4_set_client(struct nfs_server *server, · 243 unmodified lines
879	879	};
880	880	struct nfs_client *clp;
881	881
882		− if (minorversion > 0 && proto == XPRT_TRANSPORT_TCP)
	882	+ if (minorversion == 0)
	883	+ __set_bit(NFS_CS_REUSEPORT, &cl_init.init_flags);
	884	+ else if (proto == XPRT_TRANSPORT_TCP)
883	885	cl_init.nconnect = nconnect;
	886	+
884	887	if (server->flags & NFS_MOUNT_NORESVPORT)
885		− set_bit(NFS_CS_NORESVPORT, &cl_init.init_flags);
	888	+ __set_bit(NFS_CS_NORESVPORT, &cl_init.init_flags);
886	889	if (server->options & NFS_OPTION_MIGRATION)
887		− set_bit(NFS_CS_MIGRATION, &cl_init.init_flags);
	890	+ __set_bit(NFS_CS_MIGRATION, &cl_init.init_flags);
888	891	if (test_bit(NFS_MIG_TSM_POSSIBLE, &server->mig_status))
889		− set_bit(NFS_CS_TSM_POSSIBLE, &cl_init.init_flags);
	892	+ __set_bit(NFS_CS_TSM_POSSIBLE, &cl_init.init_flags);
890	893	server->port = rpc_get_port(addr);
891	894
892	895	/* Allocate or find a client reference we can use */

fs/nfs/nfs4file.c+139−2

@@ -133,14 +133,55 @@ static ssize_t __nfs4_copy_file_range(struct file *file_in, loff_t pos_in, · 132 unmodified lines
133	133	struct file *file_out, loff_t pos_out,
134	134	size_t count, unsigned int flags)
135	135	{
	136	+ struct nfs42_copy_notify_res *cn_resp = NULL;
	137	+ struct nl4_server *nss = NULL;
	138	+ nfs4_stateid *cnrs = NULL;
	139	+ ssize_t ret;
	140	+ bool sync = false;
	141	+
136	142	/* Only offload copy if superblock is the same */
137		− if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb)
	143	+ if (file_in->f_op != &nfs4_file_operations)
138	144	return -EXDEV;
139	145	if (!nfs_server_capable(file_inode(file_out), NFS_CAP_COPY))
140	146	return -EOPNOTSUPP;
141	147	if (file_inode(file_in) == file_inode(file_out))
142	148	return -EOPNOTSUPP;
143		− return nfs42_proc_copy(file_in, pos_in, file_out, pos_out, count);
	149	+ /* if the copy size if smaller than 2 RPC payloads, make it
	150	+ * synchronous
	151	+ */
	152	+ if (count <= 2 * NFS_SERVER(file_inode(file_in))->rsize)
	153	+ sync = true;
	154	+retry:
	155	+ if (!nfs42_files_from_same_server(file_in, file_out)) {
	156	+ /* for inter copy, if copy size if smaller than 12 RPC
	157	+ * payloads, fallback to traditional copy. There are
	158	+ * 14 RPCs during an NFSv4.x mount between source/dest
	159	+ * servers.
	160	+ */
	161	+ if (sync \|\|
	162	+ count <= 14 * NFS_SERVER(file_inode(file_in))->rsize)
	163	+ return -EOPNOTSUPP;
	164	+ cn_resp = kzalloc(sizeof(struct nfs42_copy_notify_res),
	165	+ GFP_NOFS);
	166	+ if (unlikely(cn_resp == NULL))
	167	+ return -ENOMEM;
	168	+
	169	+ ret = nfs42_proc_copy_notify(file_in, file_out, cn_resp);
	170	+ if (ret) {
	171	+ ret = -EOPNOTSUPP;
	172	+ goto out;
	173	+ }
	174	+ nss = &cn_resp->cnr_src;
	175	+ cnrs = &cn_resp->cnr_stateid;
	176	+ }
	177	+ ret = nfs42_proc_copy(file_in, pos_in, file_out, pos_out, count,
	178	+ nss, cnrs, sync);
	179	+out:
	180	+ if (!nfs42_files_from_same_server(file_in, file_out))
	181	+ kfree(cn_resp);
	182	+ if (ret == -EAGAIN)
	183	+ goto retry;
	184	+ return ret;
144	185	}
145	186
146	187	static ssize_t nfs4_copy_file_range(struct file *file_in, loff_t pos_in,
@@ -263,6 +304,102 @@ out_unlock: · 116 unmodified lines
263	304	out:
264	305	return ret < 0 ? ret : count;
265	306	}
	307	+
	308	+static int read_name_gen = 1;
	309	+#define SSC_READ_NAME_BODY "ssc_read_%d"
	310	+
	311	+struct file *
	312	+nfs42_ssc_open(struct vfsmount ss_mnt, struct nfs_fh src_fh,
	313	+ nfs4_stateid *stateid)
	314	+{
	315	+ struct nfs_fattr fattr;
	316	+ struct file filep, res;
	317	+ struct nfs_server *server;
	318	+ struct inode *r_ino = NULL;
	319	+ struct nfs_open_context *ctx;
	320	+ struct nfs4_state_owner *sp;
	321	+ char *read_name = NULL;
	322	+ int len, status = 0;
	323	+
	324	+ server = NFS_SERVER(ss_mnt->mnt_root->d_inode);
	325	+
	326	+ nfs_fattr_init(&fattr);
	327	+
	328	+ status = nfs4_proc_getattr(server, src_fh, &fattr, NULL, NULL);
	329	+ if (status < 0) {
	330	+ res = ERR_PTR(status);
	331	+ goto out;
	332	+ }
	333	+
	334	+ res = ERR_PTR(-ENOMEM);
	335	+ len = strlen(SSC_READ_NAME_BODY) + 16;
	336	+ read_name = kzalloc(len, GFP_NOFS);
	337	+ if (read_name == NULL)
	338	+ goto out;
	339	+ snprintf(read_name, len, SSC_READ_NAME_BODY, read_name_gen++);
	340	+
	341	+ r_ino = nfs_fhget(ss_mnt->mnt_root->d_inode->i_sb, src_fh, &fattr,
	342	+ NULL);
	343	+ if (IS_ERR(r_ino)) {
	344	+ res = ERR_CAST(r_ino);
	345	+ goto out_free_name;
	346	+ }
	347	+
	348	+ filep = alloc_file_pseudo(r_ino, ss_mnt, read_name, FMODE_READ,
	349	+ r_ino->i_fop);
	350	+ if (IS_ERR(filep)) {
	351	+ res = ERR_CAST(filep);
	352	+ goto out_free_name;
	353	+ }
	354	+ filep->f_mode \|= FMODE_READ;
	355	+
	356	+ ctx = alloc_nfs_open_context(filep->f_path.dentry, filep->f_mode,
	357	+ filep);
	358	+ if (IS_ERR(ctx)) {
	359	+ res = ERR_CAST(ctx);
	360	+ goto out_filep;
	361	+ }
	362	+
	363	+ res = ERR_PTR(-EINVAL);
	364	+ sp = nfs4_get_state_owner(server, ctx->cred, GFP_KERNEL);
	365	+ if (sp == NULL)
	366	+ goto out_ctx;
	367	+
	368	+ ctx->state = nfs4_get_open_state(r_ino, sp);
	369	+ if (ctx->state == NULL)
	370	+ goto out_stateowner;
	371	+
	372	+ set_bit(NFS_SRV_SSC_COPY_STATE, &ctx->state->flags);
	373	+ set_bit(NFS_OPEN_STATE, &ctx->state->flags);
	374	+ memcpy(&ctx->state->open_stateid.other, &stateid->other,
	375	+ NFS4_STATEID_OTHER_SIZE);
	376	+ update_open_stateid(ctx->state, stateid, NULL, filep->f_mode);
	377	+
	378	+ nfs_file_set_open_context(filep, ctx);
	379	+ put_nfs_open_context(ctx);
	380	+
	381	+ file_ra_state_init(&filep->f_ra, filep->f_mapping->host->i_mapping);
	382	+ res = filep;
	383	+out_free_name:
	384	+ kfree(read_name);
	385	+out:
	386	+ return res;
	387	+out_stateowner:
	388	+ nfs4_put_state_owner(sp);
	389	+out_ctx:
	390	+ put_nfs_open_context(ctx);
	391	+out_filep:
	392	+ fput(filep);
	393	+ goto out_free_name;
	394	+}
	395	+EXPORT_SYMBOL_GPL(nfs42_ssc_open);
	396	+void nfs42_ssc_close(struct file *filep)
	397	+{
	398	+ struct nfs_open_context *ctx = nfs_file_open_context(filep);
	399	+
	400	+ ctx->state->flags = 0;
	401	+}
	402	+EXPORT_SYMBOL_GPL(nfs42_ssc_close);
266	403	#endif /* CONFIG_NFS_V4_2 */
267	404
268	405	const struct file_operations nfs4_file_operations = {

fs/nfs/nfs4proc.c+46−23

@@ -91,7 +91,6 @@ struct nfs4_opendata; · 90 unmodified lines
91	91	static int _nfs4_recover_proc_open(struct nfs4_opendata *data);
92	92	static int nfs4_do_fsinfo(struct nfs_server , struct nfs_fh , struct nfs_fsinfo *);
93	93	static void nfs_fixup_referral_attributes(struct nfs_fattr *fattr);
94		−static int nfs4_proc_getattr(struct nfs_server , struct nfs_fh , struct nfs_fattr , struct nfs4_label label, struct inode *inode);
95	94	static int _nfs4_proc_getattr(struct nfs_server server, struct nfs_fh fhandle, struct nfs_fattr fattr, struct nfs4_label label, struct inode *inode);
96	95	static int nfs4_do_setattr(struct inode inode, const struct cred cred,
97	96	struct nfs_fattr fattr, struct iattr sattr,
@@ -476,6 +475,7 @@ static int nfs4_do_handle_exception(struct nfs_server *server, · 378 unmodified lines
476	475	case -NFS4ERR_ADMIN_REVOKED:
477	476	case -NFS4ERR_EXPIRED:
478	477	case -NFS4ERR_BAD_STATEID:
	478	+ case -NFS4ERR_PARTNER_NO_AUTH:
479	479	if (inode != NULL && stateid != NULL) {
480	480	nfs_inode_find_state_and_recover(inode,
481	481	stateid);
@@ -521,9 +521,7 @@ static int nfs4_do_handle_exception(struct nfs_server *server, · 39 unmodified lines
521	521	case -NFS4ERR_DEADSESSION:
522	522	case -NFS4ERR_SEQ_FALSE_RETRY:
523	523	case -NFS4ERR_SEQ_MISORDERED:
524		− dprintk("%s ERROR: %d Reset session\n", __func__,
525		− errorcode);
526		− nfs4_schedule_session_recovery(clp->cl_session, errorcode);
	524	+ /* Handled in nfs41_sequence_process() */
527	525	goto wait_on_recovery;
528	526	#endif /* defined(CONFIG_NFS_V4_1) */
529	527	case -NFS4ERR_FILE_OPEN:
@@ -782,6 +780,7 @@ static int nfs41_sequence_process(struct rpc_task *task, · 252 unmodified lines
782	780	struct nfs4_session *session;
783	781	struct nfs4_slot *slot = res->sr_slot;
784	782	struct nfs_client *clp;
	783	+ int status;
785	784	int ret = 1;
786	785
787	786	if (slot == NULL)
@@ -793,8 +792,13 @@ static int nfs41_sequence_process(struct rpc_task *task, · 5 unmodified lines
793	792	session = slot->table->session;
794	793
795	794	trace_nfs4_sequence_done(session, res);
	795	+
	796	+ status = res->sr_status;
	797	+ if (task->tk_status == -NFS4ERR_DEADSESSION)
	798	+ status = -NFS4ERR_DEADSESSION;
	799	+
796	800	/* Check the SEQUENCE operation status */
797		− switch (res->sr_status) {
	801	+ switch (status) {
798	802	case 0:
799	803	/* Mark this sequence number as having been acked */
800	804	nfs4_slot_sequence_acked(slot, slot->seq_nr);
@@ -866,6 +870,10 @@ static int nfs41_sequence_process(struct rpc_task *task, · 65 unmodified lines
866	870	*/
867	871	slot->seq_nr = slot->seq_nr_highest_sent;
868	872	goto out_retry;
	873	+ case -NFS4ERR_BADSESSION:
	874	+ case -NFS4ERR_DEADSESSION:
	875	+ case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION:
	876	+ goto session_recover;
869	877	default:
870	878	/* Just update the slot sequence no. */
871	879	slot->seq_done = 1;
@@ -876,8 +884,10 @@ out: · 4 unmodified lines
876	884	out_noaction:
877	885	return ret;
878	886	session_recover:
879		− nfs4_schedule_session_recovery(session, res->sr_status);
880		− goto retry_nowait;
	887	+ nfs4_schedule_session_recovery(session, status);
	888	+ dprintk("%s ERROR: %d Reset session\n", __func__, status);
	889	+ nfs41_sequence_free_slot(res);
	890	+ goto out;
881	891	retry_new_seq:
882	892	++slot->seq_nr;
883	893	retry_nowait:
@@ -1716,7 +1726,7 @@ static void nfs_state_clear_delegation(struct nfs4_state *state) · 832 unmodified lines
1716	1726	write_sequnlock(&state->seqlock);
1717	1727	}
1718	1728
1719		−static int update_open_stateid(struct nfs4_state *state,
	1729	+int update_open_stateid(struct nfs4_state *state,
1720	1730	const nfs4_stateid *open_stateid,
1721	1731	const nfs4_stateid *delegation,
1722	1732	fmode_t fmode)
@@ -1737,7 +1747,7 @@ static int update_open_stateid(struct nfs4_state *state, · 14 unmodified lines
1737	1747	ret = 1;
1738	1748	}
1739	1749
1740		− deleg_cur = rcu_dereference(nfsi->delegation);
	1750	+ deleg_cur = nfs4_get_valid_delegation(state->inode);
1741	1751	if (deleg_cur == NULL)
1742	1752	goto no_delegation;
1743	1753
@@ -1749,7 +1759,7 @@ static int update_open_stateid(struct nfs4_state *state, · 5 unmodified lines
1749	1759
1750	1760	if (delegation == NULL)
1751	1761	delegation = &deleg_cur->stateid;
1752		− else if (!nfs4_stateid_match(&deleg_cur->stateid, delegation))
	1762	+ else if (!nfs4_stateid_match_other(&deleg_cur->stateid, delegation))
1753	1763	goto no_delegation_unlock;
1754	1764
1755	1765	nfs_mark_delegation_referenced(deleg_cur);
@@ -1796,7 +1806,7 @@ static void nfs4_return_incompatible_delegation(struct inode *inode, fmode_t fmo · 40 unmodified lines
1796	1806
1797	1807	fmode &= FMODE_READ\|FMODE_WRITE;
1798	1808	rcu_read_lock();
1799		− delegation = rcu_dereference(NFS_I(inode)->delegation);
	1809	+ delegation = nfs4_get_valid_delegation(inode);
1800	1810	if (delegation == NULL \|\| (delegation->type & fmode) == fmode) {
1801	1811	rcu_read_unlock();
1802	1812	return;
@@ -2188,7 +2198,6 @@ static int nfs4_handle_delegation_recall_error(struct nfs_server *server, struct · 385 unmodified lines
2188	2198	case -NFS4ERR_BAD_HIGH_SLOT:
2189	2199	case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION:
2190	2200	case -NFS4ERR_DEADSESSION:
2191		− nfs4_schedule_session_recovery(server->nfs_client->cl_session, err);
2192	2201	return -EAGAIN;
2193	2202	case -NFS4ERR_STALE_CLIENTID:
2194	2203	case -NFS4ERR_STALE_STATEID:
@@ -4062,7 +4071,7 @@ static int _nfs4_proc_getattr(struct nfs_server server, struct nfs_fh fhandle, · 1867 unmodified lines
4062	4071	return nfs4_call_sync(server->client, server, &msg, &args.seq_args, &res.seq_res, 0);
4063	4072	}
4064	4073
4065		−static int nfs4_proc_getattr(struct nfs_server server, struct nfs_fh fhandle,
	4074	+int nfs4_proc_getattr(struct nfs_server server, struct nfs_fh fhandle,
4066	4075	struct nfs_fattr fattr, struct nfs4_label label,
4067	4076	struct inode *inode)
4068	4077	{
@@ -5098,12 +5107,12 @@ static bool nfs4_stateid_is_current(nfs4_stateid *stateid, · 1029 unmodified lines
5098	5107	const struct nfs_lock_context *l_ctx,
5099	5108	fmode_t fmode)
5100	5109	{
5101		− nfs4_stateid current_stateid;
	5110	+ nfs4_stateid _current_stateid;
5102	5111
5103	5112	/* If the current stateid represents a lost lock, then exit */
5104		− if (nfs4_set_rw_stateid(&current_stateid, ctx, l_ctx, fmode) == -EIO)
	5113	+ if (nfs4_set_rw_stateid(&_current_stateid, ctx, l_ctx, fmode) == -EIO)
5105	5114	return true;
5106		− return nfs4_stateid_match(stateid, &current_stateid);
	5115	+ return nfs4_stateid_match(stateid, &_current_stateid);
5107	5116	}
5108	5117
5109	5118	static bool nfs4_error_stateid_expired(int err)
@@ -6196,10 +6205,13 @@ static void nfs4_delegreturn_done(struct rpc_task task, void calldata) · 1086 unmodified lines
6196	6205	task->tk_status = 0;
6197	6206	break;
6198	6207	case -NFS4ERR_OLD_STATEID:
6199		− if (nfs4_refresh_delegation_stateid(&data->stateid, data->inode))
6200		− goto out_restart;
6201		− task->tk_status = 0;
6202		− break;
	6208	+ if (!nfs4_refresh_delegation_stateid(&data->stateid, data->inode))
	6209	+ nfs4_stateid_seqid_inc(&data->stateid);
	6210	+ if (data->args.bitmask) {
	6211	+ data->args.bitmask = NULL;
	6212	+ data->res.fattr = NULL;
	6213	+ }
	6214	+ goto out_restart;
6203	6215	case -NFS4ERR_ACCESS:
6204	6216	if (data->args.bitmask) {
6205	6217	data->args.bitmask = NULL;
@@ -6214,6 +6226,7 @@ static void nfs4_delegreturn_done(struct rpc_task task, void calldata) · 8 unmodified lines
6214	6226	if (exception.retry)
6215	6227	goto out_restart;
6216	6228	}
	6229	+ nfs_delegation_mark_returned(data->inode, data->args.stateid);
6217	6230	data->rpc_status = task->tk_status;
6218	6231	return;
6219	6232	out_restart:
@@ -6243,8 +6256,10 @@ static void nfs4_delegreturn_prepare(struct rpc_task task, void data) · 23 unmodified lines
6243	6256
6244	6257	d_data = (struct nfs4_delegreturndata *)data;
6245	6258
6246		− if (!d_data->lr.roc && nfs4_wait_on_layoutreturn(d_data->inode, task))
	6259	+ if (!d_data->lr.roc && nfs4_wait_on_layoutreturn(d_data->inode, task)) {
	6260	+ nfs4_sequence_done(task, &d_data->res.seq_res);
6247	6261	return;
	6262	+ }
6248	6263
6249	6264	lo = d_data->args.lr_args ? d_data->args.lr_args->layout : NULL;
6250	6265	if (lo && !pnfs_layout_is_valid(lo)) {
@@ -7820,6 +7835,15 @@ nfs41_same_server_scope(struct nfs41_server_scope *a, · 1569 unmodified lines
7820	7835	static void
7821	7836	nfs4_bind_one_conn_to_session_done(struct rpc_task task, void calldata)
7822	7837	{
	7838	+ struct nfs41_bind_conn_to_session_args *args = task->tk_msg.rpc_argp;
	7839	+ struct nfs_client *clp = args->client;
	7840	+
	7841	+ switch (task->tk_status) {
	7842	+ case -NFS4ERR_BADSESSION:
	7843	+ case -NFS4ERR_DEADSESSION:
	7844	+ nfs4_schedule_session_recovery(clp->cl_session,
	7845	+ task->tk_status);
	7846	+ }
7823	7847	}
7824	7848
7825	7849	static const struct rpc_call_ops nfs4_bind_one_conn_to_session_ops = {
@@ -8867,8 +8891,6 @@ static int nfs41_reclaim_complete_handle_errors(struct rpc_task *task, struct nf · 1041 unmodified lines
8867	8891	case -NFS4ERR_BADSESSION:
8868	8892	case -NFS4ERR_DEADSESSION:
8869	8893	case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION:
8870		− nfs4_schedule_session_recovery(clp->cl_session,
8871		− task->tk_status);
8872	8894	break;
8873	8895	default:
8874	8896	nfs4_schedule_lease_recovery(clp);
@@ -9897,6 +9919,7 @@ static const struct nfs4_minor_version_ops nfs_v4_2_minor_ops = { · 1022 unmodified lines
9897	9919	\| NFS_CAP_ALLOCATE
9898	9920	\| NFS_CAP_COPY
9899	9921	\| NFS_CAP_OFFLOAD_CANCEL
	9922	+ \| NFS_CAP_COPY_NOTIFY
9900	9923	\| NFS_CAP_DEALLOCATE
9901	9924	\| NFS_CAP_SEEK
9902	9925	\| NFS_CAP_LAYOUTSTATS

fs/nfs/nfs4state.c+43−8

@@ -60,6 +60,7 @@ · 59 unmodified lines
60	60	#include "nfs4session.h"
61	61	#include "pnfs.h"
62	62	#include "netns.h"
	63	+#include "nfs4trace.h"
63	64
64	65	#define NFSDBG_FACILITY NFSDBG_STATE
65	66
@@ -1407,7 +1408,7 @@ nfs_state_find_lock_state_by_stateid(struct nfs4_state *state, · 1341 unmodified lines
1407	1408	list_for_each_entry(pos, &state->lock_states, ls_locks) {
1408	1409	if (!test_bit(NFS_LOCK_INITIALIZED, &pos->ls_flags))
1409	1410	continue;
1410		− if (nfs4_stateid_match_other(&pos->ls_stateid, stateid))
	1411	+ if (nfs4_stateid_match_or_older(&pos->ls_stateid, stateid))
1411	1412	return pos;
1412	1413	}
1413	1414	return NULL;
@@ -1441,12 +1442,13 @@ void nfs_inode_find_state_and_recover(struct inode *inode, · 27 unmodified lines
1441	1442	state = ctx->state;
1442	1443	if (state == NULL)
1443	1444	continue;
1444		− if (nfs4_stateid_match_other(&state->stateid, stateid) &&
	1445	+ if (nfs4_stateid_match_or_older(&state->stateid, stateid) &&
1445	1446	nfs4_state_mark_reclaim_nograce(clp, state)) {
1446	1447	found = true;
1447	1448	continue;
1448	1449	}
1449		− if (nfs4_stateid_match_other(&state->open_stateid, stateid) &&
	1450	+ if (test_bit(NFS_OPEN_STATE, &state->flags) &&
	1451	+ nfs4_stateid_match_or_older(&state->open_stateid, stateid) &&
1450	1452	nfs4_state_mark_reclaim_nograce(clp, state)) {
1451	1453	found = true;
1452	1454	continue;
@@ -1556,16 +1558,32 @@ static void nfs42_complete_copies(struct nfs4_state_owner *sp, struct nfs4_state · 103 unmodified lines
1556	1558	{
1557	1559	struct nfs4_copy_state *copy;
1558	1560
1559		− if (!test_bit(NFS_CLNT_DST_SSC_COPY_STATE, &state->flags))
	1561	+ if (!test_bit(NFS_CLNT_DST_SSC_COPY_STATE, &state->flags) &&
	1562	+ !test_bit(NFS_CLNT_SRC_SSC_COPY_STATE, &state->flags))
1560	1563	return;
1561	1564
1562	1565	spin_lock(&sp->so_server->nfs_client->cl_lock);
1563	1566	list_for_each_entry(copy, &sp->so_server->ss_copies, copies) {
1564		− if (!nfs4_stateid_match_other(&state->stateid, &copy->parent_state->stateid))
1565		− continue;
	1567	+ if ((test_bit(NFS_CLNT_DST_SSC_COPY_STATE, &state->flags) &&
	1568	+ !nfs4_stateid_match_other(&state->stateid,
	1569	+ &copy->parent_dst_state->stateid)))
	1570	+ continue;
1566	1571	copy->flags = 1;
1567		− complete(&copy->completion);
1568		− break;
	1572	+ if (test_and_clear_bit(NFS_CLNT_DST_SSC_COPY_STATE,
	1573	+ &state->flags)) {
	1574	+ clear_bit(NFS_CLNT_SRC_SSC_COPY_STATE, &state->flags);
	1575	+ complete(&copy->completion);
	1576	+ }
	1577	+ }
	1578	+ list_for_each_entry(copy, &sp->so_server->ss_copies, src_copies) {
	1579	+ if ((test_bit(NFS_CLNT_SRC_SSC_COPY_STATE, &state->flags) &&
	1580	+ !nfs4_stateid_match_other(&state->stateid,
	1581	+ &copy->parent_src_state->stateid)))
	1582	+ continue;
	1583	+ copy->flags = 1;
	1584	+ if (test_and_clear_bit(NFS_CLNT_DST_SSC_COPY_STATE,
	1585	+ &state->flags))
	1586	+ complete(&copy->completion);
1569	1587	}
1570	1588	spin_unlock(&sp->so_server->nfs_client->cl_lock);
1571	1589	}
@@ -1593,6 +1611,7 @@ static int __nfs4_reclaim_open_state(struct nfs4_state_owner *sp, struct nfs4_st · 21 unmodified lines
1593	1611	if (!test_bit(NFS_DELEGATED_STATE, &state->flags)) {
1594	1612	spin_lock(&state->state_lock);
1595	1613	list_for_each_entry(lock, &state->lock_states, ls_locks) {
	1614	+ trace_nfs4_state_lock_reclaim(state, lock);
1596	1615	if (!test_bit(NFS_LOCK_INITIALIZED, &lock->ls_flags))
1597	1616	pr_warn_ratelimited("NFS: %s: Lock reclaim failed!\n", __func__);
1598	1617	}
@@ -1609,6 +1628,9 @@ static int nfs4_reclaim_open_state(struct nfs4_state_owner *sp, const struct nfs · 10 unmodified lines
1609	1628	struct nfs4_state *state;
1610	1629	unsigned int loop = 0;
1611	1630	int status = 0;
	1631	+#ifdef CONFIG_NFS_V4_2
	1632	+ bool found_ssc_copy_state = false;
	1633	+#endif /* CONFIG_NFS_V4_2 */
1612	1634
1613	1635	/* Note: we rely on the sp->so_states list being ordered
1614	1636	* so that we always reclaim open(O_RDWR) and/or open(O_WRITE)
@@ -1628,6 +1650,13 @@ restart: · 13 unmodified lines
1628	1650	continue;
1629	1651	if (state->state == 0)
1630	1652	continue;
	1653	+#ifdef CONFIG_NFS_V4_2
	1654	+ if (test_bit(NFS_SRV_SSC_COPY_STATE, &state->flags)) {
	1655	+ nfs4_state_mark_recovery_failed(state, -EIO);
	1656	+ found_ssc_copy_state = true;
	1657	+ continue;
	1658	+ }
	1659	+#endif /* CONFIG_NFS_V4_2 */
1631	1660	refcount_inc(&state->count);
1632	1661	spin_unlock(&sp->so_lock);
1633	1662	status = __nfs4_reclaim_open_state(sp, state, ops);
@@ -1682,6 +1711,10 @@ restart: · 48 unmodified lines
1682	1711	}
1683	1712	raw_write_seqcount_end(&sp->so_reclaim_seqcount);
1684	1713	spin_unlock(&sp->so_lock);
	1714	+#ifdef CONFIG_NFS_V4_2
	1715	+ if (found_ssc_copy_state)
	1716	+ return -EIO;
	1717	+#endif /* CONFIG_NFS_V4_2 */
1685	1718	return 0;
1686	1719	out_err:
1687	1720	nfs4_put_open_state(state);
@@ -2508,6 +2541,7 @@ static void nfs4_state_manager(struct nfs_client *clp) · 820 unmodified lines
2508	2541
2509	2542	/* Ensure exclusive access to NFSv4 state */
2510	2543	do {
	2544	+ trace_nfs4_state_mgr(clp);
2511	2545	clear_bit(NFS4CLNT_RUN_MANAGER, &clp->cl_state);
2512	2546	if (test_bit(NFS4CLNT_PURGE_STATE, &clp->cl_state)) {
2513	2547	section = "purge state";
@@ -2621,6 +2655,7 @@ static void nfs4_state_manager(struct nfs_client *clp) · 107 unmodified lines
2621	2655	out_error:
2622	2656	if (strlen(section))
2623	2657	section_sep = ": ";
	2658	+ trace_nfs4_state_mgr_failed(clp, section, status);
2624	2659	pr_warn_ratelimited("NFS: state manager%s%s failed on NFSv4 server %s"
2625	2660	" with error %d\n", section_sep, section,
2626	2661	clp->cl_hostname, -status);

fs/nfs/nfs4super.c+2−2

@@ -92,8 +92,8 @@ static void nfs4_evict_inode(struct inode *inode) · 91 unmodified lines
92	92	{
93	93	truncate_inode_pages_final(&inode->i_data);
94	94	clear_inode(inode);
95		− /* If we are holding a delegation, return it! */
96		− nfs_inode_return_delegation_noreclaim(inode);
	95	+ /* If we are holding a delegation, return and free it */
	96	+ nfs_inode_evict_delegation(inode);
97	97	/* Note that above delegreturn would trigger pnfs return-on-close */
98	98	pnfs_return_layout(inode);
99	99	pnfs_destroy_layout(NFS_I(inode));

fs/nfs/nfs4trace.h+175−0

@@ -562,6 +562,99 @@ TRACE_EVENT(nfs4_setup_sequence, · 561 unmodified lines
562	562	)
563	563	);
564	564
	565	+TRACE_DEFINE_ENUM(NFS4CLNT_MANAGER_RUNNING);
	566	+TRACE_DEFINE_ENUM(NFS4CLNT_CHECK_LEASE);
	567	+TRACE_DEFINE_ENUM(NFS4CLNT_LEASE_EXPIRED);
	568	+TRACE_DEFINE_ENUM(NFS4CLNT_RECLAIM_REBOOT);
	569	+TRACE_DEFINE_ENUM(NFS4CLNT_RECLAIM_NOGRACE);
	570	+TRACE_DEFINE_ENUM(NFS4CLNT_DELEGRETURN);
	571	+TRACE_DEFINE_ENUM(NFS4CLNT_SESSION_RESET);
	572	+TRACE_DEFINE_ENUM(NFS4CLNT_LEASE_CONFIRM);
	573	+TRACE_DEFINE_ENUM(NFS4CLNT_SERVER_SCOPE_MISMATCH);
	574	+TRACE_DEFINE_ENUM(NFS4CLNT_PURGE_STATE);
	575	+TRACE_DEFINE_ENUM(NFS4CLNT_BIND_CONN_TO_SESSION);
	576	+TRACE_DEFINE_ENUM(NFS4CLNT_MOVED);
	577	+TRACE_DEFINE_ENUM(NFS4CLNT_LEASE_MOVED);
	578	+TRACE_DEFINE_ENUM(NFS4CLNT_DELEGATION_EXPIRED);
	579	+TRACE_DEFINE_ENUM(NFS4CLNT_RUN_MANAGER);
	580	+TRACE_DEFINE_ENUM(NFS4CLNT_DELEGRETURN_RUNNING);
	581	+
	582	+#define show_nfs4_clp_state(state) \
	583	+ __print_flags(state, "\|", \
	584	+ { NFS4CLNT_MANAGER_RUNNING, "MANAGER_RUNNING" }, \
	585	+ { NFS4CLNT_CHECK_LEASE, "CHECK_LEASE" }, \
	586	+ { NFS4CLNT_LEASE_EXPIRED, "LEASE_EXPIRED" }, \
	587	+ { NFS4CLNT_RECLAIM_REBOOT, "RECLAIM_REBOOT" }, \
	588	+ { NFS4CLNT_RECLAIM_NOGRACE, "RECLAIM_NOGRACE" }, \
	589	+ { NFS4CLNT_DELEGRETURN, "DELEGRETURN" }, \
	590	+ { NFS4CLNT_SESSION_RESET, "SESSION_RESET" }, \
	591	+ { NFS4CLNT_LEASE_CONFIRM, "LEASE_CONFIRM" }, \
	592	+ { NFS4CLNT_SERVER_SCOPE_MISMATCH, \
	593	+ "SERVER_SCOPE_MISMATCH" }, \
	594	+ { NFS4CLNT_PURGE_STATE, "PURGE_STATE" }, \
	595	+ { NFS4CLNT_BIND_CONN_TO_SESSION, \
	596	+ "BIND_CONN_TO_SESSION" }, \
	597	+ { NFS4CLNT_MOVED, "MOVED" }, \
	598	+ { NFS4CLNT_LEASE_MOVED, "LEASE_MOVED" }, \
	599	+ { NFS4CLNT_DELEGATION_EXPIRED, "DELEGATION_EXPIRED" }, \
	600	+ { NFS4CLNT_RUN_MANAGER, "RUN_MANAGER" }, \
	601	+ { NFS4CLNT_DELEGRETURN_RUNNING, "DELEGRETURN_RUNNING" })
	602	+
	603	+TRACE_EVENT(nfs4_state_mgr,
	604	+ TP_PROTO(
	605	+ const struct nfs_client *clp
	606	+ ),
	607	+
	608	+ TP_ARGS(clp),
	609	+
	610	+ TP_STRUCT__entry(
	611	+ __field(unsigned long, state)
	612	+ __string(hostname, clp->cl_hostname)
	613	+ ),
	614	+
	615	+ TP_fast_assign(
	616	+ __entry->state = clp->cl_state;
	617	+ __assign_str(hostname, clp->cl_hostname)
	618	+ ),
	619	+
	620	+ TP_printk(
	621	+ "hostname=%s clp state=%s", __get_str(hostname),
	622	+ show_nfs4_clp_state(__entry->state)
	623	+ )
	624	+)
	625	+
	626	+TRACE_EVENT(nfs4_state_mgr_failed,
	627	+ TP_PROTO(
	628	+ const struct nfs_client *clp,
	629	+ const char *section,
	630	+ int status
	631	+ ),
	632	+
	633	+ TP_ARGS(clp, section, status),
	634	+
	635	+ TP_STRUCT__entry(
	636	+ __field(unsigned long, error)
	637	+ __field(unsigned long, state)
	638	+ __string(hostname, clp->cl_hostname)
	639	+ __string(section, section)
	640	+ ),
	641	+
	642	+ TP_fast_assign(
	643	+ __entry->error = status;
	644	+ __entry->state = clp->cl_state;
	645	+ __assign_str(hostname, clp->cl_hostname);
	646	+ __assign_str(section, section);
	647	+ ),
	648	+
	649	+ TP_printk(
	650	+ "hostname=%s clp state=%s error=%ld (%s) section=%s",
	651	+ __get_str(hostname),
	652	+ show_nfs4_clp_state(__entry->state), -__entry->error,
	653	+ show_nfsv4_errors(__entry->error), __get_str(section)
	654	+
	655	+ )
	656	+)
	657	+
565	658	TRACE_EVENT(nfs4_xdr_status,
566	659	TP_PROTO(
567	660	const struct xdr_stream *xdr,
@@ -929,6 +1022,88 @@ TRACE_EVENT(nfs4_set_lock, · 361 unmodified lines
929	1022	)
930	1023	);
931	1024
	1025	+TRACE_DEFINE_ENUM(LK_STATE_IN_USE);
	1026	+TRACE_DEFINE_ENUM(NFS_DELEGATED_STATE);
	1027	+TRACE_DEFINE_ENUM(NFS_OPEN_STATE);
	1028	+TRACE_DEFINE_ENUM(NFS_O_RDONLY_STATE);
	1029	+TRACE_DEFINE_ENUM(NFS_O_WRONLY_STATE);
	1030	+TRACE_DEFINE_ENUM(NFS_O_RDWR_STATE);
	1031	+TRACE_DEFINE_ENUM(NFS_STATE_RECLAIM_REBOOT);
	1032	+TRACE_DEFINE_ENUM(NFS_STATE_RECLAIM_NOGRACE);
	1033	+TRACE_DEFINE_ENUM(NFS_STATE_POSIX_LOCKS);
	1034	+TRACE_DEFINE_ENUM(NFS_STATE_RECOVERY_FAILED);
	1035	+TRACE_DEFINE_ENUM(NFS_STATE_MAY_NOTIFY_LOCK);
	1036	+TRACE_DEFINE_ENUM(NFS_STATE_CHANGE_WAIT);
	1037	+TRACE_DEFINE_ENUM(NFS_CLNT_DST_SSC_COPY_STATE);
	1038	+TRACE_DEFINE_ENUM(NFS_CLNT_SRC_SSC_COPY_STATE);
	1039	+TRACE_DEFINE_ENUM(NFS_SRV_SSC_COPY_STATE);
	1040	+
	1041	+#define show_nfs4_state_flags(flags) \
	1042	+ __print_flags(flags, "\|", \
	1043	+ { LK_STATE_IN_USE, "IN_USE" }, \
	1044	+ { NFS_DELEGATED_STATE, "DELEGATED" }, \
	1045	+ { NFS_OPEN_STATE, "OPEN" }, \
	1046	+ { NFS_O_RDONLY_STATE, "O_RDONLY" }, \
	1047	+ { NFS_O_WRONLY_STATE, "O_WRONLY" }, \
	1048	+ { NFS_O_RDWR_STATE, "O_RDWR" }, \
	1049	+ { NFS_STATE_RECLAIM_REBOOT, "RECLAIM_REBOOT" }, \
	1050	+ { NFS_STATE_RECLAIM_NOGRACE, "RECLAIM_NOGRACE" }, \
	1051	+ { NFS_STATE_POSIX_LOCKS, "POSIX_LOCKS" }, \
	1052	+ { NFS_STATE_RECOVERY_FAILED, "RECOVERY_FAILED" }, \
	1053	+ { NFS_STATE_MAY_NOTIFY_LOCK, "MAY_NOTIFY_LOCK" }, \
	1054	+ { NFS_STATE_CHANGE_WAIT, "CHANGE_WAIT" }, \
	1055	+ { NFS_CLNT_DST_SSC_COPY_STATE, "CLNT_DST_SSC_COPY" }, \
	1056	+ { NFS_CLNT_SRC_SSC_COPY_STATE, "CLNT_SRC_SSC_COPY" }, \
	1057	+ { NFS_SRV_SSC_COPY_STATE, "SRV_SSC_COPY" })
	1058	+
	1059	+#define show_nfs4_lock_flags(flags) \
	1060	+ __print_flags(flags, "\|", \
	1061	+ { BIT(NFS_LOCK_INITIALIZED), "INITIALIZED" }, \
	1062	+ { BIT(NFS_LOCK_LOST), "LOST" })
	1063	+
	1064	+TRACE_EVENT(nfs4_state_lock_reclaim,
	1065	+ TP_PROTO(
	1066	+ const struct nfs4_state *state,
	1067	+ const struct nfs4_lock_state *lock
	1068	+ ),
	1069	+
	1070	+ TP_ARGS(state, lock),
	1071	+
	1072	+ TP_STRUCT__entry(
	1073	+ __field(dev_t, dev)
	1074	+ __field(u32, fhandle)
	1075	+ __field(u64, fileid)
	1076	+ __field(unsigned long, state_flags)
	1077	+ __field(unsigned long, lock_flags)
	1078	+ __field(int, stateid_seq)
	1079	+ __field(u32, stateid_hash)
	1080	+ ),
	1081	+
	1082	+ TP_fast_assign(
	1083	+ const struct inode *inode = state->inode;
	1084	+
	1085	+ __entry->dev = inode->i_sb->s_dev;
	1086	+ __entry->fileid = NFS_FILEID(inode);
	1087	+ __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode));
	1088	+ __entry->state_flags = state->flags;
	1089	+ __entry->lock_flags = lock->ls_flags;
	1090	+ __entry->stateid_seq =
	1091	+ be32_to_cpu(state->stateid.seqid);
	1092	+ __entry->stateid_hash =
	1093	+ nfs_stateid_hash(&state->stateid);
	1094	+ ),
	1095	+
	1096	+ TP_printk(
	1097	+ "fileid=%02x:%02x:%llu fhandle=0x%08x "
	1098	+ "stateid=%d:0x%08x state_flags=%s lock_flags=%s",
	1099	+ MAJOR(__entry->dev), MINOR(__entry->dev),
	1100	+ (unsigned long long)__entry->fileid, __entry->fhandle,
	1101	+ __entry->stateid_seq, __entry->stateid_hash,
	1102	+ show_nfs4_state_flags(__entry->state_flags),
	1103	+ show_nfs4_lock_flags(__entry->lock_flags)
	1104	+ )
	1105	+)
	1106	+
932	1107	DECLARE_EVENT_CLASS(nfs4_set_delegation_event,
933	1108	TP_PROTO(
934	1109	const struct inode *inode,

fs/nfs/nfs4xdr.c+11−13

@@ -1059,7 +1059,7 @@ static void encode_nfs4_verifier(struct xdr_stream xdr, const nfs4_verifier ve · 1058 unmodified lines
1059	1059	}
1060	1060
1061	1061	static __be32 *
1062		−xdr_encode_nfstime4(__be32 p, const struct timespec t)
	1062	+xdr_encode_nfstime4(__be32 p, const struct timespec64 t)
1063	1063	{
1064	1064	p = xdr_encode_hyper(p, (__s64)t->tv_sec);
1065	1065	*p++ = cpu_to_be32(t->tv_nsec);
@@ -1072,7 +1072,6 @@ static void encode_attrs(struct xdr_stream xdr, const struct iattr iap, · 6 unmodified lines
1072	1072	const struct nfs_server *server,
1073	1073	const uint32_t attrmask[])
1074	1074	{
1075		− struct timespec ts;
1076	1075	char owner_name[IDMAP_NAMESZ];
1077	1076	char owner_group[IDMAP_NAMESZ];
1078	1077	int owner_namelen = 0;
@@ -1161,16 +1160,14 @@ static void encode_attrs(struct xdr_stream xdr, const struct iattr iap, · 82 unmodified lines
1161	1160	if (bmval[1] & FATTR4_WORD1_TIME_ACCESS_SET) {
1162	1161	if (iap->ia_valid & ATTR_ATIME_SET) {
1163	1162	*p++ = cpu_to_be32(NFS4_SET_TO_CLIENT_TIME);
1164		− ts = timespec64_to_timespec(iap->ia_atime);
1165		− p = xdr_encode_nfstime4(p, &ts);
	1163	+ p = xdr_encode_nfstime4(p, &iap->ia_atime);
1166	1164	} else
1167	1165	*p++ = cpu_to_be32(NFS4_SET_TO_SERVER_TIME);
1168	1166	}
1169	1167	if (bmval[1] & FATTR4_WORD1_TIME_MODIFY_SET) {
1170	1168	if (iap->ia_valid & ATTR_MTIME_SET) {
1171	1169	*p++ = cpu_to_be32(NFS4_SET_TO_CLIENT_TIME);
1172		− ts = timespec64_to_timespec(iap->ia_mtime);
1173		− p = xdr_encode_nfstime4(p, &ts);
	1170	+ p = xdr_encode_nfstime4(p, &iap->ia_mtime);
1174	1171	} else
1175	1172	*p++ = cpu_to_be32(NFS4_SET_TO_SERVER_TIME);
1176	1173	}
@@ -4065,17 +4062,17 @@ static int decode_attr_space_used(struct xdr_stream xdr, uint32_t bitmap, uint · 2888 unmodified lines
4065	4062	}
4066	4063
4067	4064	static __be32 *
4068		−xdr_decode_nfstime4(__be32 p, struct timespec t)
	4065	+xdr_decode_nfstime4(__be32 p, struct timespec64 t)
4069	4066	{
4070	4067	__u64 sec;
4071	4068
4072	4069	p = xdr_decode_hyper(p, &sec);
4073		− t-> tv_sec = (time_t)sec;
	4070	+ t-> tv_sec = sec;
4074	4071	t->tv_nsec = be32_to_cpup(p++);
4075	4072	return p;
4076	4073	}
4077	4074
4078		−static int decode_attr_time(struct xdr_stream xdr, struct timespec time)
	4075	+static int decode_attr_time(struct xdr_stream xdr, struct timespec64 time)
4079	4076	{
4080	4077	__be32 *p;
4081	4078
@@ -4086,7 +4083,7 @@ static int decode_attr_time(struct xdr_stream xdr, struct timespec time) · 4 unmodified lines
4086	4083	return 0;
4087	4084	}
4088	4085
4089		−static int decode_attr_time_access(struct xdr_stream xdr, uint32_t bitmap, struct timespec *time)
	4086	+static int decode_attr_time_access(struct xdr_stream xdr, uint32_t bitmap, struct timespec64 *time)
4090	4087	{
4091	4088	int status = 0;
4092	4089
@@ -4104,7 +4101,7 @@ static int decode_attr_time_access(struct xdr_stream xdr, uint32_t bitmap, str · 11 unmodified lines
4104	4101	return status;
4105	4102	}
4106	4103
4107		−static int decode_attr_time_metadata(struct xdr_stream xdr, uint32_t bitmap, struct timespec *time)
	4104	+static int decode_attr_time_metadata(struct xdr_stream xdr, uint32_t bitmap, struct timespec64 *time)
4108	4105	{
4109	4106	int status = 0;
4110	4107
@@ -4123,7 +4120,7 @@ static int decode_attr_time_metadata(struct xdr_stream xdr, uint32_t bitmap, s · 12 unmodified lines
4123	4120	}
4124	4121
4125	4122	static int decode_attr_time_delta(struct xdr_stream xdr, uint32_t bitmap,
4126		− struct timespec *time)
	4123	+ struct timespec64 *time)
4127	4124	{
4128	4125	int status = 0;
4129	4126
@@ -4186,7 +4183,7 @@ static int decode_attr_security_label(struct xdr_stream xdr, uint32_t bitmap, · 56 unmodified lines
4186	4183	return status;
4187	4184	}
4188	4185
4189		−static int decode_attr_time_modify(struct xdr_stream xdr, uint32_t bitmap, struct timespec *time)
	4186	+static int decode_attr_time_modify(struct xdr_stream xdr, uint32_t bitmap, struct timespec64 *time)
4190	4187	{
4191	4188	int status = 0;
4192	4189
@@ -7581,6 +7578,7 @@ const struct rpc_procinfo nfs4_procedures[] = { · 3388 unmodified lines
7581	7578	PROC42(CLONE, enc_clone, dec_clone),
7582	7579	PROC42(COPY, enc_copy, dec_copy),
7583	7580	PROC42(OFFLOAD_CANCEL, enc_offload_cancel, dec_offload_cancel),
	7581	+ PROC42(COPY_NOTIFY, enc_copy_notify, dec_copy_notify),
7584	7582	PROC(LOOKUPP, enc_lookupp, dec_lookupp),
7585	7583	PROC42(LAYOUTERROR, enc_layouterror, dec_layouterror),
7586	7584	};

fs/nfs/nfstrace.h+33−0

@@ -1065,6 +1065,39 @@ TRACE_EVENT(nfs_commit_done, · 1064 unmodified lines
1065	1065	)
1066	1066	);
1067	1067
	1068	+TRACE_EVENT(nfs_fh_to_dentry,
	1069	+ TP_PROTO(
	1070	+ const struct super_block *sb,
	1071	+ const struct nfs_fh *fh,
	1072	+ u64 fileid,
	1073	+ int error
	1074	+ ),
	1075	+
	1076	+ TP_ARGS(sb, fh, fileid, error),
	1077	+
	1078	+ TP_STRUCT__entry(
	1079	+ __field(int, error)
	1080	+ __field(dev_t, dev)
	1081	+ __field(u32, fhandle)
	1082	+ __field(u64, fileid)
	1083	+ ),
	1084	+
	1085	+ TP_fast_assign(
	1086	+ __entry->error = error;
	1087	+ __entry->dev = sb->s_dev;
	1088	+ __entry->fileid = fileid;
	1089	+ __entry->fhandle = nfs_fhandle_hash(fh);
	1090	+ ),
	1091	+
	1092	+ TP_printk(
	1093	+ "error=%d fileid=%02x:%02x:%llu fhandle=0x%08x ",
	1094	+ __entry->error,
	1095	+ MAJOR(__entry->dev), MINOR(__entry->dev),
	1096	+ (unsigned long long)__entry->fileid,
	1097	+ __entry->fhandle
	1098	+ )
	1099	+);
	1100	+
1068	1101	TRACE_DEFINE_ENUM(NFS_OK);
1069	1102	TRACE_DEFINE_ENUM(NFSERR_PERM);
1070	1103	TRACE_DEFINE_ENUM(NFSERR_NOENT);

fs/nfs/pnfs.c+0−2

@@ -2160,8 +2160,6 @@ out_unlock: · 2159 unmodified lines
2160	2160	return NULL;
2161	2161	}
2162	2162
2163		−extern const nfs4_stateid current_stateid;
2164		−
2165	2163	static void _lgopen_prepare_attached(struct nfs4_opendata *data,
2166	2164	struct nfs_open_context *ctx)
2167	2165	{

fs/nfs/super.c+3−3

@@ -1592,7 +1592,7 @@ static int nfs_parse_mount_options(char *raw, · 1591 unmodified lines
1592	1592	dfprintk(MOUNT, "NFS: invalid "
1593	1593	"lookupcache argument\n");
1594	1594	return 0;
1595		− };
	1595	+ }
1596	1596	break;
1597	1597	case Opt_fscache_uniq:
1598	1598	if (nfs_get_option_str(args, &mnt->fscache_uniq))
@@ -1625,7 +1625,7 @@ static int nfs_parse_mount_options(char *raw, · 26 unmodified lines
1625	1625	dfprintk(MOUNT, "NFS: invalid "
1626	1626	"local_lock argument\n");
1627	1627	return 0;
1628		− };
	1628	+ }
1629	1629	break;
1630	1630
1631	1631	/*
@@ -2585,7 +2585,7 @@ static void nfs_get_cache_cookie(struct super_block *sb, · 953 unmodified lines
2585	2585	if (mnt_s->fscache_key) {
2586	2586	uniq = mnt_s->fscache_key->key.uniquifier;
2587	2587	ulen = mnt_s->fscache_key->key.uniq_len;
2588		− };
	2588	+ }
2589	2589	} else
2590	2590	return;
2591	2591

fs/nfs/sysfs.c+1−2

@@ -121,8 +121,7 @@ static void nfs_netns_client_release(struct kobject *kobj) · 120 unmodified lines
121	121	struct nfs_netns_client,
122	122	kobject);
123	123
124		− if (c->identifier)
125		− kfree(c->identifier);
	124	+ kfree(c->identifier);
126	125	kfree(c);
127	126	}
128	127

include/linux/nfs4.h+26−0

@@ -16,6 +16,7 @@ · 15 unmodified lines
16	16	#include <linux/list.h>
17	17	#include <linux/uidgid.h>
18	18	#include <uapi/linux/nfs4.h>
	19	+#include <linux/sunrpc/msg_prot.h>
19	20
20	21	enum nfs4_acl_whotype {
21	22	NFS4_ACL_WHO_NAMED = 0,
@@ -539,6 +540,8 @@ enum { · 517 unmodified lines
539	540
540	541	NFSPROC4_CLNT_LOOKUPP,
541	542	NFSPROC4_CLNT_LAYOUTERROR,
	543	+
	544	+ NFSPROC4_CLNT_COPY_NOTIFY,
542	545	};
543	546
544	547	/* nfs41 types */
@@ -674,4 +677,27 @@ struct nfs4_op_map { · 129 unmodified lines
674	677	} u;
675	678	};
676	679
	680	+struct nfs42_netaddr {
	681	+ char netid[RPCBIND_MAXNETIDLEN];
	682	+ char addr[RPCBIND_MAXUADDRLEN + 1];
	683	+ u32 netid_len;
	684	+ u32 addr_len;
	685	+};
	686	+
	687	+enum netloc_type4 {
	688	+ NL4_NAME = 1,
	689	+ NL4_URL = 2,
	690	+ NL4_NETADDR = 3,
	691	+};
	692	+
	693	+struct nl4_server {
	694	+ enum netloc_type4 nl4_type;
	695	+ union {
	696	+ struct { /* NL4_NAME, NL4_URL */
	697	+ int nl4_str_sz;
	698	+ char nl4_str[NFS4_OPAQUE_LIMIT + 1];
	699	+ };
	700	+ struct nfs42_netaddr nl4_addr; /* NL4_NETADDR */
	701	+ } u;
	702	+};
677	703	#endif

include/linux/nfs_fs.h+3−1

@@ -189,13 +189,15 @@ struct nfs_inode { · 188 unmodified lines
189	189
190	190	struct nfs4_copy_state {
191	191	struct list_head copies;
	192	+ struct list_head src_copies;
192	193	nfs4_stateid stateid;
193	194	struct completion completion;
194	195	uint64_t count;
195	196	struct nfs_writeverf verf;
196	197	int error;
197	198	int flags;
198		− struct nfs4_state *parent_state;
	199	+ struct nfs4_state *parent_src_state;
	200	+ struct nfs4_state *parent_dst_state;
199	201	};
200	202
201	203	/*

include/linux/nfs_fs_sb.h+5−1

@@ -45,6 +45,9 @@ struct nfs_client { · 44 unmodified lines
45	45	#define NFS_CS_INFINITE_SLOTS 3 /* - don't limit TCP slots */
46	46	#define NFS_CS_NO_RETRANS_TIMEOUT 4 /* - Disable retransmit timeouts */
47	47	#define NFS_CS_TSM_POSSIBLE 5 /* - Maybe state migration */
	48	+#define NFS_CS_NOPING 6 /* - don't ping on connect */
	49	+#define NFS_CS_DS 7 /* - Server is a DS */
	50	+#define NFS_CS_REUSEPORT 8 /* - reuse src port on reconnect */
48	51	struct sockaddr_storage cl_addr; /* server identifier */
49	52	size_t cl_addrlen;
50	53	char * cl_hostname; /* hostname of server */
@@ -171,7 +174,7 @@ struct nfs_server { · 120 unmodified lines
171	174
172	175	struct nfs_fsid fsid;
173	176	__u64 maxfilesize; /* maximum file size */
174		− struct timespec time_delta; /* smallest time granularity */
	177	+ struct timespec64 time_delta; /* smallest time granularity */
175	178	unsigned long mount_time; /* when this fs was mounted */
176	179	struct super_block super; / VFS super block */
177	180	dev_t s_dev; /* superblock dev numbers */
@@ -276,5 +279,6 @@ struct nfs_server { · 98 unmodified lines
276	279	#define NFS_CAP_COPY (1U << 24)
277	280	#define NFS_CAP_OFFLOAD_CANCEL (1U << 25)
278	281	#define NFS_CAP_LAYOUTERROR (1U << 26)
	282	+#define NFS_CAP_COPY_NOTIFY (1U << 27)
279	283
280	284	#endif

include/linux/nfs_xdr.h+24−7

@@ -62,14 +62,14 @@ struct nfs_fattr { · 61 unmodified lines
62	62	struct nfs_fsid fsid;
63	63	__u64 fileid;
64	64	__u64 mounted_on_fileid;
65		− struct timespec atime;
66		− struct timespec mtime;
67		− struct timespec ctime;
	65	+ struct timespec64 atime;
	66	+ struct timespec64 mtime;
	67	+ struct timespec64 ctime;
68	68	__u64 change_attr; /* NFSv4 change attribute */
69	69	__u64 pre_change_attr;/* pre-op NFSv4 change attribute */
70	70	__u64 pre_size; /* pre_op_attr.size */
71		− struct timespec pre_mtime; /* pre_op_attr.mtime */
72		− struct timespec pre_ctime; /* pre_op_attr.ctime */
	71	+ struct timespec64 pre_mtime; /* pre_op_attr.mtime */
	72	+ struct timespec64 pre_ctime; /* pre_op_attr.ctime */
73	73	unsigned long time_start;
74	74	unsigned long gencount;
75	75	struct nfs4_string *owner_name;
@@ -143,7 +143,7 @@ struct nfs_fsinfo { · 67 unmodified lines
143	143	__u32 wtmult; /* writes should be multiple of this */
144	144	__u32 dtpref; /* pref. readdir transfer size */
145	145	__u64 maxfilesize;
146		− struct timespec time_delta; /* server time granularity */
	146	+ struct timespec64 time_delta; /* server time granularity */
147	147	__u32 lease_time; /* in seconds */
148	148	__u32 nlayouttypes; /* number of layouttypes */
149	149	__u32 layouttype[NFS_MAX_LAYOUT_TYPES]; /* supported pnfs layout driver */
@@ -869,7 +869,7 @@ struct nfs3_sattrargs { · 719 unmodified lines
869	869	struct nfs_fh * fh;
870	870	struct iattr * sattr;
871	871	unsigned int guard;
872		− struct timespec guardtime;
	872	+ struct timespec64 guardtime;
873	873	};
874	874
875	875	struct nfs3_diropargs {
@@ -1435,6 +1435,7 @@ struct nfs42_copy_args { · 559 unmodified lines
1435	1435
1436	1436	u64 count;
1437	1437	bool sync;
	1438	+ struct nl4_server *cp_src;
1438	1439	};
1439	1440
1440	1441	struct nfs42_write_res {
@@ -1463,6 +1464,22 @@ struct nfs42_offload_status_res { · 22 unmodified lines
1463	1464	int osr_status;
1464	1465	};
1465	1466
	1467	+struct nfs42_copy_notify_args {
	1468	+ struct nfs4_sequence_args cna_seq_args;
	1469	+
	1470	+ struct nfs_fh *cna_src_fh;
	1471	+ nfs4_stateid cna_src_stateid;
	1472	+ struct nl4_server cna_dst;
	1473	+};
	1474	+
	1475	+struct nfs42_copy_notify_res {
	1476	+ struct nfs4_sequence_res cnr_seq_res;
	1477	+
	1478	+ struct nfstime4 cnr_lease_time;
	1479	+ nfs4_stateid cnr_stateid;
	1480	+ struct nl4_server cnr_src;
	1481	+};
	1482	+
1466	1483	struct nfs42_seek_args {
1467	1484	struct nfs4_sequence_args seq_args;
1468	1485

include/linux/sunrpc/clnt.h+1−0

@@ -149,6 +149,7 @@ struct rpc_add_xprt_test { · 148 unmodified lines
149	149	#define RPC_CLNT_CREATE_NO_IDLE_TIMEOUT (1UL << 8)
150	150	#define RPC_CLNT_CREATE_NO_RETRANS_TIMEOUT (1UL << 9)
151	151	#define RPC_CLNT_CREATE_SOFTERR (1UL << 10)
	152	+#define RPC_CLNT_CREATE_REUSEPORT (1UL << 11)
152	153
153	154	struct rpc_clnt rpc_create(struct rpc_create_args args);
154	155	struct rpc_clnt rpc_bind_new_program(struct rpc_clnt ,

include/linux/sunrpc/xprt.h+2−1

@@ -207,7 +207,8 @@ struct rpc_xprt { · 206 unmodified lines
207	207	unsigned int min_reqs; /* min number of slots */
208	208	unsigned int num_reqs; /* total slots */
209	209	unsigned long state; /* transport state */
210		− unsigned char resvport : 1; /* use a reserved port */
	210	+ unsigned char resvport : 1, /* use a reserved port */
	211	+ reuseport : 1; /* reuse port on reconnect */
211	212	atomic_t swapper; /* we're swapping over this
212	213	transport */
213	214	unsigned int bind_index; /* bind function index */

include/trace/events/rpcrdma.h+132−66

@@ -85,6 +85,44 @@ DECLARE_EVENT_CLASS(xprtrdma_rxprt, · 84 unmodified lines
85	85	), \
86	86	TP_ARGS(r_xprt))
87	87
	88	+DECLARE_EVENT_CLASS(xprtrdma_connect_class,
	89	+ TP_PROTO(
	90	+ const struct rpcrdma_xprt *r_xprt,
	91	+ int rc
	92	+ ),
	93	+
	94	+ TP_ARGS(r_xprt, rc),
	95	+
	96	+ TP_STRUCT__entry(
	97	+ __field(const void *, r_xprt)
	98	+ __field(int, rc)
	99	+ __field(int, connect_status)
	100	+ __string(addr, rpcrdma_addrstr(r_xprt))
	101	+ __string(port, rpcrdma_portstr(r_xprt))
	102	+ ),
	103	+
	104	+ TP_fast_assign(
	105	+ __entry->r_xprt = r_xprt;
	106	+ __entry->rc = rc;
	107	+ __entry->connect_status = r_xprt->rx_ep.rep_connected;
	108	+ __assign_str(addr, rpcrdma_addrstr(r_xprt));
	109	+ __assign_str(port, rpcrdma_portstr(r_xprt));
	110	+ ),
	111	+
	112	+ TP_printk("peer=[%s]:%s r_xprt=%p: rc=%d connect status=%d",
	113	+ __get_str(addr), __get_str(port), __entry->r_xprt,
	114	+ __entry->rc, __entry->connect_status
	115	+ )
	116	+);
	117	+
	118	+#define DEFINE_CONN_EVENT(name) \
	119	+ DEFINE_EVENT(xprtrdma_connect_class, xprtrdma_##name, \
	120	+ TP_PROTO( \
	121	+ const struct rpcrdma_xprt *r_xprt, \
	122	+ int rc \
	123	+ ), \
	124	+ TP_ARGS(r_xprt, rc))
	125	+
88	126	DECLARE_EVENT_CLASS(xprtrdma_rdch_event,
89	127	TP_PROTO(
90	128	const struct rpc_task *task,
@@ -333,47 +371,81 @@ TRACE_EVENT(xprtrdma_cm_event, · 242 unmodified lines
333	371	)
334	372	);
335	373
336		−TRACE_EVENT(xprtrdma_disconnect,
	374	+TRACE_EVENT(xprtrdma_inline_thresh,
337	375	TP_PROTO(
338		− const struct rpcrdma_xprt *r_xprt,
339		− int status
	376	+ const struct rpcrdma_xprt *r_xprt
340	377	),
341	378
342		− TP_ARGS(r_xprt, status),
	379	+ TP_ARGS(r_xprt),
343	380
344	381	TP_STRUCT__entry(
345	382	__field(const void *, r_xprt)
346		− __field(int, status)
347		− __field(int, connected)
	383	+ __field(unsigned int, inline_send)
	384	+ __field(unsigned int, inline_recv)
	385	+ __field(unsigned int, max_send)
	386	+ __field(unsigned int, max_recv)
348	387	__string(addr, rpcrdma_addrstr(r_xprt))
349	388	__string(port, rpcrdma_portstr(r_xprt))
350	389	),
351	390
352	391	TP_fast_assign(
	392	+ const struct rpcrdma_ep *ep = &r_xprt->rx_ep;
	393	+
353	394	__entry->r_xprt = r_xprt;
354		− __entry->status = status;
355		− __entry->connected = r_xprt->rx_ep.rep_connected;
	395	+ __entry->inline_send = ep->rep_inline_send;
	396	+ __entry->inline_recv = ep->rep_inline_recv;
	397	+ __entry->max_send = ep->rep_max_inline_send;
	398	+ __entry->max_recv = ep->rep_max_inline_recv;
356	399	__assign_str(addr, rpcrdma_addrstr(r_xprt));
357	400	__assign_str(port, rpcrdma_portstr(r_xprt));
358	401	),
359	402
360		− TP_printk("peer=[%s]:%s r_xprt=%p: status=%d %sconnected",
361		− __get_str(addr), __get_str(port),
362		− __entry->r_xprt, __entry->status,
363		− __entry->connected == 1 ? "still " : "dis"
	403	+ TP_printk("peer=[%s]:%s r_xprt=%p neg send/recv=%u/%u, calc send/recv=%u/%u",
	404	+ __get_str(addr), __get_str(port), __entry->r_xprt,
	405	+ __entry->inline_send, __entry->inline_recv,
	406	+ __entry->max_send, __entry->max_recv
364	407	)
365	408	);
366	409
367		−DEFINE_RXPRT_EVENT(xprtrdma_conn_start);
368		−DEFINE_RXPRT_EVENT(xprtrdma_conn_tout);
	410	+DEFINE_CONN_EVENT(connect);
	411	+DEFINE_CONN_EVENT(disconnect);
	412	+
369	413	DEFINE_RXPRT_EVENT(xprtrdma_create);
370	414	DEFINE_RXPRT_EVENT(xprtrdma_op_destroy);
371	415	DEFINE_RXPRT_EVENT(xprtrdma_remove);
372	416	DEFINE_RXPRT_EVENT(xprtrdma_reinsert);
373		−DEFINE_RXPRT_EVENT(xprtrdma_reconnect);
374	417	DEFINE_RXPRT_EVENT(xprtrdma_op_inject_dsc);
375	418	DEFINE_RXPRT_EVENT(xprtrdma_op_close);
376		−DEFINE_RXPRT_EVENT(xprtrdma_op_connect);
	419	+DEFINE_RXPRT_EVENT(xprtrdma_op_setport);
	420	+
	421	+TRACE_EVENT(xprtrdma_op_connect,
	422	+ TP_PROTO(
	423	+ const struct rpcrdma_xprt *r_xprt,
	424	+ unsigned long delay
	425	+ ),
	426	+
	427	+ TP_ARGS(r_xprt, delay),
	428	+
	429	+ TP_STRUCT__entry(
	430	+ __field(const void *, r_xprt)
	431	+ __field(unsigned long, delay)
	432	+ __string(addr, rpcrdma_addrstr(r_xprt))
	433	+ __string(port, rpcrdma_portstr(r_xprt))
	434	+ ),
	435	+
	436	+ TP_fast_assign(
	437	+ __entry->r_xprt = r_xprt;
	438	+ __entry->delay = delay;
	439	+ __assign_str(addr, rpcrdma_addrstr(r_xprt));
	440	+ __assign_str(port, rpcrdma_portstr(r_xprt));
	441	+ ),
	442	+
	443	+ TP_printk("peer=[%s]:%s r_xprt=%p delay=%lu",
	444	+ __get_str(addr), __get_str(port), __entry->r_xprt,
	445	+ __entry->delay
	446	+ )
	447	+);
	448	+
377	449
378	450	TRACE_EVENT(xprtrdma_op_set_cto,
379	451	TP_PROTO(
@@ -532,6 +604,8 @@ DEFINE_WRCH_EVENT(write); · 152 unmodified lines
532	604	DEFINE_WRCH_EVENT(reply);
533	605
534	606	TRACE_DEFINE_ENUM(rpcrdma_noch);
	607	+TRACE_DEFINE_ENUM(rpcrdma_noch_pullup);
	608	+TRACE_DEFINE_ENUM(rpcrdma_noch_mapped);
535	609	TRACE_DEFINE_ENUM(rpcrdma_readch);
536	610	TRACE_DEFINE_ENUM(rpcrdma_areadch);
537	611	TRACE_DEFINE_ENUM(rpcrdma_writech);
@@ -540,6 +614,8 @@ TRACE_DEFINE_ENUM(rpcrdma_replych); · 2 unmodified lines
540	614	#define xprtrdma_show_chunktype(x) \
541	615	__print_symbolic(x, \
542	616	{ rpcrdma_noch, "inline" }, \
	617	+ { rpcrdma_noch_pullup, "pullup" }, \
	618	+ { rpcrdma_noch_mapped, "mapped" }, \
543	619	{ rpcrdma_readch, "read list" }, \
544	620	{ rpcrdma_areadch, "*read list" }, \
545	621	{ rpcrdma_writech, "write list" }, \
@@ -667,9 +743,8 @@ TRACE_EVENT(xprtrdma_post_send, · 121 unmodified lines
667	743	__entry->client_id = rqst->rq_task->tk_client ?
668	744	rqst->rq_task->tk_client->cl_clid : -1;
669	745	__entry->req = req;
670		− __entry->num_sge = req->rl_sendctx->sc_wr.num_sge;
671		− __entry->signaled = req->rl_sendctx->sc_wr.send_flags &
672		− IB_SEND_SIGNALED;
	746	+ __entry->num_sge = req->rl_wr.num_sge;
	747	+ __entry->signaled = req->rl_wr.send_flags & IB_SEND_SIGNALED;
673	748	__entry->status = status;
674	749	),
675	750
@@ -735,6 +810,31 @@ TRACE_EVENT(xprtrdma_post_recvs, · 59 unmodified lines
735	810	)
736	811	);
737	812
	813	+TRACE_EVENT(xprtrdma_post_linv,
	814	+ TP_PROTO(
	815	+ const struct rpcrdma_req *req,
	816	+ int status
	817	+ ),
	818	+
	819	+ TP_ARGS(req, status),
	820	+
	821	+ TP_STRUCT__entry(
	822	+ __field(const void *, req)
	823	+ __field(int, status)
	824	+ __field(u32, xid)
	825	+ ),
	826	+
	827	+ TP_fast_assign(
	828	+ __entry->req = req;
	829	+ __entry->status = status;
	830	+ __entry->xid = be32_to_cpu(req->rl_slot.rq_xid);
	831	+ ),
	832	+
	833	+ TP_printk("req=%p xid=0x%08x status=%d",
	834	+ __entry->req, __entry->xid, __entry->status
	835	+ )
	836	+);
	837	+
738	838	/**
739	839	** Completion events
740	840	**/
@@ -1021,66 +1121,32 @@ DEFINE_REPLY_EVENT(xprtrdma_reply_hdr); · 280 unmodified lines
1021	1121	TRACE_EVENT(xprtrdma_fixup,
1022	1122	TP_PROTO(
1023	1123	const struct rpc_rqst *rqst,
1024		− int len,
1025		− int hdrlen
	1124	+ unsigned long fixup
1026	1125	),
1027	1126
1028		− TP_ARGS(rqst, len, hdrlen),
	1127	+ TP_ARGS(rqst, fixup),
1029	1128
1030	1129	TP_STRUCT__entry(
1031	1130	__field(unsigned int, task_id)
1032	1131	__field(unsigned int, client_id)
1033		− __field(const void *, base)
1034		− __field(int, len)
1035		− __field(int, hdrlen)
1036		− ),
1037		−
1038		− TP_fast_assign(
1039		− __entry->task_id = rqst->rq_task->tk_pid;
1040		− __entry->client_id = rqst->rq_task->tk_client->cl_clid;
1041		− __entry->base = rqst->rq_rcv_buf.head[0].iov_base;
1042		− __entry->len = len;
1043		− __entry->hdrlen = hdrlen;
1044		− ),
1045		−
1046		− TP_printk("task:%u@%u base=%p len=%d hdrlen=%d",
1047		− __entry->task_id, __entry->client_id,
1048		− __entry->base, __entry->len, __entry->hdrlen
1049		− )
1050		−);
1051		−
1052		−TRACE_EVENT(xprtrdma_fixup_pg,
1053		− TP_PROTO(
1054		− const struct rpc_rqst *rqst,
1055		− int pageno,
1056		− const void *pos,
1057		− int len,
1058		− int curlen
1059		− ),
1060		−
1061		− TP_ARGS(rqst, pageno, pos, len, curlen),
1062		−
1063		− TP_STRUCT__entry(
1064		− __field(unsigned int, task_id)
1065		− __field(unsigned int, client_id)
1066		− __field(const void *, pos)
1067		− __field(int, pageno)
1068		− __field(int, len)
1069		− __field(int, curlen)
	1132	+ __field(unsigned long, fixup)
	1133	+ __field(size_t, headlen)
	1134	+ __field(unsigned int, pagelen)
	1135	+ __field(size_t, taillen)
1070	1136	),
1071	1137
1072	1138	TP_fast_assign(
1073	1139	__entry->task_id = rqst->rq_task->tk_pid;
1074	1140	__entry->client_id = rqst->rq_task->tk_client->cl_clid;
1075		− __entry->pos = pos;
1076		− __entry->pageno = pageno;
1077		− __entry->len = len;
1078		− __entry->curlen = curlen;
	1141	+ __entry->fixup = fixup;
	1142	+ __entry->headlen = rqst->rq_rcv_buf.head[0].iov_len;
	1143	+ __entry->pagelen = rqst->rq_rcv_buf.page_len;
	1144	+ __entry->taillen = rqst->rq_rcv_buf.tail[0].iov_len;
1079	1145	),
1080	1146
1081		− TP_printk("task:%u@%u pageno=%d pos=%p len=%d curlen=%d",
1082		− __entry->task_id, __entry->client_id,
1083		− __entry->pageno, __entry->pos, __entry->len, __entry->curlen
	1147	+ TP_printk("task:%u@%u fixup=%lu xdr=%zu/%u/%zu",
	1148	+ __entry->task_id, __entry->client_id, __entry->fixup,
	1149	+ __entry->headlen, __entry->pagelen, __entry->taillen
1084	1150	)
1085	1151	);
1086	1152

include/trace/events/sunrpc.h+94−0

@@ -165,6 +165,7 @@ DECLARE_EVENT_CLASS(rpc_task_running, · 164 unmodified lines
165	165	DEFINE_RPC_RUNNING_EVENT(begin);
166	166	DEFINE_RPC_RUNNING_EVENT(run_action);
167	167	DEFINE_RPC_RUNNING_EVENT(complete);
	168	+DEFINE_RPC_RUNNING_EVENT(end);
168	169
169	170	DECLARE_EVENT_CLASS(rpc_task_queued,
170	171
@@ -777,6 +778,99 @@ TRACE_EVENT(xprt_ping, · 606 unmodified lines
777	778	__get_str(addr), __get_str(port), __entry->status)
778	779	);
779	780
	781	+DECLARE_EVENT_CLASS(xprt_writelock_event,
	782	+ TP_PROTO(
	783	+ const struct rpc_xprt xprt, const struct rpc_task task
	784	+ ),
	785	+
	786	+ TP_ARGS(xprt, task),
	787	+
	788	+ TP_STRUCT__entry(
	789	+ __field(unsigned int, task_id)
	790	+ __field(unsigned int, client_id)
	791	+ __field(unsigned int, snd_task_id)
	792	+ ),
	793	+
	794	+ TP_fast_assign(
	795	+ if (task) {
	796	+ __entry->task_id = task->tk_pid;
	797	+ __entry->client_id = task->tk_client ?
	798	+ task->tk_client->cl_clid : -1;
	799	+ } else {
	800	+ __entry->task_id = -1;
	801	+ __entry->client_id = -1;
	802	+ }
	803	+ __entry->snd_task_id = xprt->snd_task ?
	804	+ xprt->snd_task->tk_pid : -1;
	805	+ ),
	806	+
	807	+ TP_printk("task:%u@%u snd_task:%u",
	808	+ __entry->task_id, __entry->client_id,
	809	+ __entry->snd_task_id)
	810	+);
	811	+
	812	+#define DEFINE_WRITELOCK_EVENT(name) \
	813	+ DEFINE_EVENT(xprt_writelock_event, xprt_##name, \
	814	+ TP_PROTO( \
	815	+ const struct rpc_xprt *xprt, \
	816	+ const struct rpc_task *task \
	817	+ ), \
	818	+ TP_ARGS(xprt, task))
	819	+
	820	+DEFINE_WRITELOCK_EVENT(reserve_xprt);
	821	+DEFINE_WRITELOCK_EVENT(release_xprt);
	822	+
	823	+DECLARE_EVENT_CLASS(xprt_cong_event,
	824	+ TP_PROTO(
	825	+ const struct rpc_xprt xprt, const struct rpc_task task
	826	+ ),
	827	+
	828	+ TP_ARGS(xprt, task),
	829	+
	830	+ TP_STRUCT__entry(
	831	+ __field(unsigned int, task_id)
	832	+ __field(unsigned int, client_id)
	833	+ __field(unsigned int, snd_task_id)
	834	+ __field(unsigned long, cong)
	835	+ __field(unsigned long, cwnd)
	836	+ __field(bool, wait)
	837	+ ),
	838	+
	839	+ TP_fast_assign(
	840	+ if (task) {
	841	+ __entry->task_id = task->tk_pid;
	842	+ __entry->client_id = task->tk_client ?
	843	+ task->tk_client->cl_clid : -1;
	844	+ } else {
	845	+ __entry->task_id = -1;
	846	+ __entry->client_id = -1;
	847	+ }
	848	+ __entry->snd_task_id = xprt->snd_task ?
	849	+ xprt->snd_task->tk_pid : -1;
	850	+ __entry->cong = xprt->cong;
	851	+ __entry->cwnd = xprt->cwnd;
	852	+ __entry->wait = test_bit(XPRT_CWND_WAIT, &xprt->state);
	853	+ ),
	854	+
	855	+ TP_printk("task:%u@%u snd_task:%u cong=%lu cwnd=%lu%s",
	856	+ __entry->task_id, __entry->client_id,
	857	+ __entry->snd_task_id, __entry->cong, __entry->cwnd,
	858	+ __entry->wait ? " (wait)" : "")
	859	+);
	860	+
	861	+#define DEFINE_CONG_EVENT(name) \
	862	+ DEFINE_EVENT(xprt_cong_event, xprt_##name, \
	863	+ TP_PROTO( \
	864	+ const struct rpc_xprt *xprt, \
	865	+ const struct rpc_task *task \
	866	+ ), \
	867	+ TP_ARGS(xprt, task))
	868	+
	869	+DEFINE_CONG_EVENT(reserve_cong);
	870	+DEFINE_CONG_EVENT(release_cong);
	871	+DEFINE_CONG_EVENT(get_cong);
	872	+DEFINE_CONG_EVENT(put_cong);
	873	+
780	874	TRACE_EVENT(xs_stream_read_data,
781	875	TP_PROTO(struct rpc_xprt *xprt, ssize_t err, size_t total),
782	876

net/sunrpc/clnt.c+8−13

@@ -591,6 +591,9 @@ struct rpc_clnt rpc_create(struct rpc_create_args args) · 590 unmodified lines
591	591	xprt->resvport = 1;
592	592	if (args->flags & RPC_CLNT_CREATE_NONPRIVPORT)
593	593	xprt->resvport = 0;
	594	+ xprt->reuseport = 0;
	595	+ if (args->flags & RPC_CLNT_CREATE_REUSEPORT)
	596	+ xprt->reuseport = 1;
594	597
595	598	clnt = rpc_create_xprt(args, xprt);
596	599	if (IS_ERR(clnt) \|\| args->nconnect <= 1)
@@ -1676,8 +1679,6 @@ call_reserveresult(struct rpc_task *task) · 1079 unmodified lines
1676	1679	return;
1677	1680	}
1678	1681
1679		− printk(KERN_ERR "%s: status=%d, but no request slot, exiting\n",
1680		− __func__, status);
1681	1682	rpc_call_rpcerror(task, -EIO);
1682	1683	return;
1683	1684	}
@@ -1686,11 +1687,8 @@ call_reserveresult(struct rpc_task *task) · 2 unmodified lines
1686	1687	* Even though there was an error, we may have acquired
1687	1688	* a request slot somehow. Make sure not to leak it.
1688	1689	*/
1689		− if (task->tk_rqstp) {
1690		− printk(KERN_ERR "%s: status=%d, request allocated anyway\n",
1691		− __func__, status);
	1690	+ if (task->tk_rqstp)
1692	1691	xprt_release(task);
1693		− }
1694	1692
1695	1693	switch (status) {
1696	1694	case -ENOMEM:
@@ -1699,14 +1697,9 @@ call_reserveresult(struct rpc_task *task) · 2 unmodified lines
1699	1697	case -EAGAIN: /* woken up; retry */
1700	1698	task->tk_action = call_retry_reserve;
1701	1699	return;
1702		− case -EIO: /* probably a shutdown */
1703		− break;
1704	1700	default:
1705		− printk(KERN_ERR "%s: unrecognized error %d, exiting\n",
1706		− __func__, status);
1707		− break;
	1701	+ rpc_call_rpcerror(task, status);
1708	1702	}
1709		− rpc_call_rpcerror(task, status);
1710	1703	}
1711	1704
1712	1705	/*
@@ -2906,7 +2899,7 @@ int rpc_clnt_add_xprt(struct rpc_clnt *clnt, · 1193 unmodified lines
2906	2899	struct rpc_xprt *xprt;
2907	2900	unsigned long connect_timeout;
2908	2901	unsigned long reconnect_timeout;
2909		− unsigned char resvport;
	2902	+ unsigned char resvport, reuseport;
2910	2903	int ret = 0;
2911	2904
2912	2905	rcu_read_lock();
@@ -2918,6 +2911,7 @@ int rpc_clnt_add_xprt(struct rpc_clnt *clnt, · 5 unmodified lines
2918	2911	return -EAGAIN;
2919	2912	}
2920	2913	resvport = xprt->resvport;
	2914	+ reuseport = xprt->reuseport;
2921	2915	connect_timeout = xprt->connect_timeout;
2922	2916	reconnect_timeout = xprt->max_reconnect_timeout;
2923	2917	rcu_read_unlock();
@@ -2928,6 +2922,7 @@ int rpc_clnt_add_xprt(struct rpc_clnt *clnt, · 4 unmodified lines
2928	2922	goto out_put_switch;
2929	2923	}
2930	2924	xprt->resvport = resvport;
	2925	+ xprt->reuseport = reuseport;
2931	2926	if (xprt->ops->set_connect_timeout != NULL)
2932	2927	xprt->ops->set_connect_timeout(xprt,
2933	2928	connect_timeout,

net/sunrpc/sched.c+2−1

@@ -260,7 +260,7 @@ static void __rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const c · 259 unmodified lines
260	260	rpc_reset_waitqueue_priority(queue);
261	261	queue->qlen = 0;
262	262	queue->timer_list.expires = 0;
263		− INIT_DEFERRABLE_WORK(&queue->timer_list.dwork, __rpc_queue_timer_fn);
	263	+ INIT_DELAYED_WORK(&queue->timer_list.dwork, __rpc_queue_timer_fn);
264	264	INIT_LIST_HEAD(&queue->timer_list.list);
265	265	rpc_assign_waitqueue_name(queue, qname);
266	266	}
@@ -824,6 +824,7 @@ rpc_reset_task_statistics(struct rpc_task *task) · 557 unmodified lines
824	824	*/
825	825	void rpc_exit_task(struct rpc_task *task)
826	826	{
	827	+ trace_rpc_task_end(task, task->tk_action);
827	828	task->tk_action = NULL;
828	829	if (task->tk_ops->rpc_count_stats)
829	830	task->tk_ops->rpc_count_stats(task, task->tk_calldata);

net/sunrpc/xdr.c+5−6

@@ -436,13 +436,12 @@ xdr_shrink_bufhead(struct xdr_buf *buf, size_t len) · 435 unmodified lines
436	436	}
437	437
438	438	/**
439		− * xdr_shrink_pagelen
	439	+ * xdr_shrink_pagelen - shrinks buf->pages by up to @len bytes
440	440	* @buf: xdr_buf
441	441	* @len: bytes to remove from buf->pages
442	442	*
443		− * Shrinks XDR buffer's page array buf->pages by
444		− * 'len' bytes. The extra data is not lost, but is instead
445		− * moved into the tail.
	443	+ * The extra data is not lost, but is instead moved into buf->tail.
	444	+ * Returns the actual number of bytes moved.
446	445	*/
447	446	static unsigned int
448	447	xdr_shrink_pagelen(struct xdr_buf *buf, size_t len)
@@ -455,8 +454,8 @@ xdr_shrink_pagelen(struct xdr_buf *buf, size_t len) · 6 unmodified lines
455	454
456	455	result = 0;
457	456	tail = buf->tail;
458		− BUG_ON (len > pglen);
459		−
	457	+ if (len > buf->page_len)
	458	+ len = buf-> page_len;
460	459	tailbuf_len = buf->buflen - buf->head->iov_len - buf->page_len;
461	460
462	461	/* Shift the tail first */

net/sunrpc/xprt.c+13−9

@@ -205,20 +205,20 @@ int xprt_reserve_xprt(struct rpc_xprt xprt, struct rpc_task task) · 204 unmodified lines
205	205
206	206	if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) {
207	207	if (task == xprt->snd_task)
208		− return 1;
	208	+ goto out_locked;
209	209	goto out_sleep;
210	210	}
211	211	if (test_bit(XPRT_WRITE_SPACE, &xprt->state))
212	212	goto out_unlock;
213	213	xprt->snd_task = task;
214	214
	215	+out_locked:
	216	+ trace_xprt_reserve_xprt(xprt, task);
215	217	return 1;
216	218
217	219	out_unlock:
218	220	xprt_clear_locked(xprt);
219	221	out_sleep:
220		− dprintk("RPC: %5u failed to lock transport %p\n",
221		− task->tk_pid, xprt);
222	222	task->tk_status = -EAGAIN;
223	223	if (RPC_IS_SOFT(task))
224	224	rpc_sleep_on_timeout(&xprt->sending, task, NULL,
@@ -269,23 +269,22 @@ int xprt_reserve_xprt_cong(struct rpc_xprt xprt, struct rpc_task task) · 44 unmodified lines
269	269
270	270	if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) {
271	271	if (task == xprt->snd_task)
272		− return 1;
	272	+ goto out_locked;
273	273	goto out_sleep;
274	274	}
275	275	if (req == NULL) {
276	276	xprt->snd_task = task;
277		− return 1;
	277	+ goto out_locked;
278	278	}
279	279	if (test_bit(XPRT_WRITE_SPACE, &xprt->state))
280	280	goto out_unlock;
281	281	if (!xprt_need_congestion_window_wait(xprt)) {
282	282	xprt->snd_task = task;
283		− return 1;
	283	+ goto out_locked;
284	284	}
285	285	out_unlock:
286	286	xprt_clear_locked(xprt);
287	287	out_sleep:
288		− dprintk("RPC: %5u failed to lock transport %p\n", task->tk_pid, xprt);
289	288	task->tk_status = -EAGAIN;
290	289	if (RPC_IS_SOFT(task))
291	290	rpc_sleep_on_timeout(&xprt->sending, task, NULL,
@@ -293,6 +292,9 @@ out_sleep: · 1 unmodified line
293	292	else
294	293	rpc_sleep_on(&xprt->sending, task, NULL);
295	294	return 0;
	295	+out_locked:
	296	+ trace_xprt_reserve_cong(xprt, task);
	297	+ return 1;
296	298	}
297	299	EXPORT_SYMBOL_GPL(xprt_reserve_xprt_cong);
298	300
@@ -357,6 +359,7 @@ void xprt_release_xprt(struct rpc_xprt xprt, struct rpc_task task) · 58 unmodified lines
357	359	xprt_clear_locked(xprt);
358	360	__xprt_lock_write_next(xprt);
359	361	}
	362	+ trace_xprt_release_xprt(xprt, task);
360	363	}
361	364	EXPORT_SYMBOL_GPL(xprt_release_xprt);
362	365
@@ -374,6 +377,7 @@ void xprt_release_xprt_cong(struct rpc_xprt xprt, struct rpc_task task) · 11 unmodified lines
374	377	xprt_clear_locked(xprt);
375	378	__xprt_lock_write_next_cong(xprt);
376	379	}
	380	+ trace_xprt_release_cong(xprt, task);
377	381	}
378	382	EXPORT_SYMBOL_GPL(xprt_release_xprt_cong);
379	383
@@ -395,8 +399,7 @@ __xprt_get_cong(struct rpc_xprt xprt, struct rpc_rqst req) · 15 unmodified lines
395	399	{
396	400	if (req->rq_cong)
397	401	return 1;
398		− dprintk("RPC: %5u xprt_cwnd_limited cong = %lu cwnd = %lu\n",
399		− req->rq_task->tk_pid, xprt->cong, xprt->cwnd);
	402	+ trace_xprt_get_cong(xprt, req->rq_task);
400	403	if (RPCXPRT_CONGESTED(xprt)) {
401	404	xprt_set_congestion_window_wait(xprt);
402	405	return 0;
@@ -418,6 +421,7 @@ __xprt_put_cong(struct rpc_xprt xprt, struct rpc_rqst req) · 15 unmodified lines
418	421	req->rq_cong = 0;
419	422	xprt->cong -= RPC_CWNDSCALE;
420	423	xprt_test_and_clear_congestion_window_wait(xprt);
	424	+ trace_xprt_put_cong(xprt, req->rq_task);
421	425	__xprt_lock_write_next_cong(xprt);
422	426	}
423	427

net/sunrpc/xprtrdma/backchannel.c+1−1

@@ -79,7 +79,7 @@ static int rpcrdma_bc_marshal_reply(struct rpc_rqst *rqst) · 78 unmodified lines
79	79	*p = xdr_zero;
80	80
81	81	if (rpcrdma_prepare_send_sges(r_xprt, req, RPCRDMA_HDRLEN_MIN,
82		− &rqst->rq_snd_buf, rpcrdma_noch))
	82	+ &rqst->rq_snd_buf, rpcrdma_noch_pullup))
83	83	return -EIO;
84	84
85	85	trace_xprtrdma_cb_reply(rqst);

net/sunrpc/xprtrdma/frwr_ops.c+11−42

@@ -36,8 +36,8 @@ · 35 unmodified lines
36	36	* connect worker from running concurrently.
37	37	*
38	38	* When the underlying transport disconnects, MRs that are in flight
39		− * are flushed and are likely unusable. Thus all flushed MRs are
40		− * destroyed. New MRs are created on demand.
	39	+ * are flushed and are likely unusable. Thus all MRs are destroyed.
	40	+ * New MRs are created on demand.
41	41	*/
42	42
43	43	#include <linux/sunrpc/rpc_rdma.h>
@@ -88,8 +88,10 @@ void frwr_release_mr(struct rpcrdma_mr *mr) · 44 unmodified lines
88	88	kfree(mr);
89	89	}
90	90
91		−static void frwr_mr_recycle(struct rpcrdma_xprt r_xprt, struct rpcrdma_mr mr)
	91	+static void frwr_mr_recycle(struct rpcrdma_mr *mr)
92	92	{
	93	+ struct rpcrdma_xprt *r_xprt = mr->mr_xprt;
	94	+
93	95	trace_xprtrdma_mr_recycle(mr);
94	96
95	97	if (mr->mr_dir != DMA_NONE) {
@@ -107,32 +109,6 @@ static void frwr_mr_recycle(struct rpcrdma_xprt r_xprt, struct rpcrdma_mr mr) · 11 unmodified lines
107	109	frwr_release_mr(mr);
108	110	}
109	111
110		−/* MRs are dynamically allocated, so simply clean up and release the MR.
111		− * A replacement MR will subsequently be allocated on demand.
112		− */
113		−static void
114		−frwr_mr_recycle_worker(struct work_struct *work)
115		−{
116		− struct rpcrdma_mr *mr = container_of(work, struct rpcrdma_mr,
117		− mr_recycle);
118		−
119		− frwr_mr_recycle(mr->mr_xprt, mr);
120		−}
121		−
122		−/* frwr_recycle - Discard MRs
123		− * @req: request to reset
124		− *
125		− * Used after a reconnect. These MRs could be in flight, we can't
126		− * tell. Safe thing to do is release them.
127		− */
128		−void frwr_recycle(struct rpcrdma_req *req)
129		−{
130		− struct rpcrdma_mr *mr;
131		−
132		− while ((mr = rpcrdma_mr_pop(&req->rl_registered)))
133		− frwr_mr_recycle(mr->mr_xprt, mr);
134		−}
135		−
136	112	/* frwr_reset - Place MRs back on the free list
137	113	* @req: request to reset
138	114	*
@@ -166,9 +142,6 @@ int frwr_init_mr(struct rpcrdma_ia ia, struct rpcrdma_mr mr) · 27 unmodified lines
166	142	struct ib_mr *frmr;
167	143	int rc;
168	144
169		− /* NB: ib_alloc_mr and device drivers typically allocate
170		− * memory with GFP_KERNEL.
171		− */
172	145	frmr = ib_alloc_mr(ia->ri_pd, ia->ri_mrtype, depth);
173	146	if (IS_ERR(frmr))
174	147	goto out_mr_err;
@@ -180,7 +153,6 @@ int frwr_init_mr(struct rpcrdma_ia ia, struct rpcrdma_mr mr) · 5 unmodified lines
180	153	mr->frwr.fr_mr = frmr;
181	154	mr->mr_dir = DMA_NONE;
182	155	INIT_LIST_HEAD(&mr->mr_list);
183		− INIT_WORK(&mr->mr_recycle, frwr_mr_recycle_worker);
184	156	init_completion(&mr->frwr.fr_linv_done);
185	157
186	158	sg_init_table(sg, depth);
@@ -424,7 +396,7 @@ int frwr_send(struct rpcrdma_ia ia, struct rpcrdma_req req) · 237 unmodified lines
424	396	struct ib_send_wr *post_wr;
425	397	struct rpcrdma_mr *mr;
426	398
427		− post_wr = &req->rl_sendctx->sc_wr;
	399	+ post_wr = &req->rl_wr;
428	400	list_for_each_entry(mr, &req->rl_registered, mr_list) {
429	401	struct rpcrdma_frwr *frwr;
430	402
@@ -440,9 +412,6 @@ int frwr_send(struct rpcrdma_ia ia, struct rpcrdma_req req) · 9 unmodified lines
440	412	post_wr = &frwr->fr_regwr.wr;
441	413	}
442	414
443		− /* If ib_post_send fails, the next ->send_request for
444		− * @req will queue these MRs for recovery.
445		− */
446	415	return ib_post_send(ia->ri_id->qp, post_wr, NULL);
447	416	}
448	417
@@ -468,7 +437,7 @@ void frwr_reminv(struct rpcrdma_rep rep, struct list_head mrs) · 19 unmodified lines
468	437	static void __frwr_release_mr(struct ib_wc wc, struct rpcrdma_mr mr)
469	438	{
470	439	if (wc->status != IB_WC_SUCCESS)
471		− rpcrdma_mr_recycle(mr);
	440	+ frwr_mr_recycle(mr);
472	441	else
473	442	rpcrdma_mr_put(mr);
474	443	}
@@ -570,7 +539,6 @@ void frwr_unmap_sync(struct rpcrdma_xprt r_xprt, struct rpcrdma_req req) · 95 unmodified lines
570	539	*/
571	540	bad_wr = NULL;
572	541	rc = ib_post_send(r_xprt->rx_ia.ri_id->qp, first, &bad_wr);
573		− trace_xprtrdma_post_send(req, rc);
574	542
575	543	/* The final LOCAL_INV WR in the chain is supposed to
576	544	* do the wake. If it was never posted, the wake will
@@ -583,6 +551,7 @@ void frwr_unmap_sync(struct rpcrdma_xprt r_xprt, struct rpcrdma_req req) · 6 unmodified lines
583	551
584	552	/* Recycle MRs in the LOCAL_INV chain that did not get posted.
585	553	*/
	554	+ trace_xprtrdma_post_linv(req, rc);
586	555	while (bad_wr) {
587	556	frwr = container_of(bad_wr, struct rpcrdma_frwr,
588	557	fr_invwr);
@@ -590,7 +559,7 @@ void frwr_unmap_sync(struct rpcrdma_xprt r_xprt, struct rpcrdma_req req) · 1 unmodified line
590	559	bad_wr = bad_wr->next;
591	560
592	561	list_del_init(&mr->mr_list);
593		− rpcrdma_mr_recycle(mr);
	562	+ frwr_mr_recycle(mr);
594	563	}
595	564	}
596	565
@@ -673,18 +642,18 @@ void frwr_unmap_async(struct rpcrdma_xprt r_xprt, struct rpcrdma_req req) · 76 unmodified lines
673	642	*/
674	643	bad_wr = NULL;
675	644	rc = ib_post_send(r_xprt->rx_ia.ri_id->qp, first, &bad_wr);
676		− trace_xprtrdma_post_send(req, rc);
677	645	if (!rc)
678	646	return;
679	647
680	648	/* Recycle MRs in the LOCAL_INV chain that did not get posted.
681	649	*/
	650	+ trace_xprtrdma_post_linv(req, rc);
682	651	while (bad_wr) {
683	652	frwr = container_of(bad_wr, struct rpcrdma_frwr, fr_invwr);
684	653	mr = container_of(frwr, struct rpcrdma_mr, frwr);
685	654	bad_wr = bad_wr->next;
686	655
687		− rpcrdma_mr_recycle(mr);
	656	+ frwr_mr_recycle(mr);
688	657	}
689	658
690	659	/* The final LOCAL_INV WR in the chain is supposed to

net/sunrpc/xprtrdma/rpc_rdma.c+268−145

@@ -78,8 +78,6 @@ static unsigned int rpcrdma_max_call_header_size(unsigned int maxsegs) · 77 unmodified lines
78	78	size += rpcrdma_segment_maxsz * sizeof(__be32);
79	79	size += sizeof(__be32); /* list discriminator */
80	80
81		− dprintk("RPC: %s: max call header size = %u\n",
82		− __func__, size);
83	81	return size;
84	82	}
85	83
@@ -100,8 +98,6 @@ static unsigned int rpcrdma_max_reply_header_size(unsigned int maxsegs) · 14 unmodified lines
100	98	size += maxsegs * rpcrdma_segment_maxsz * sizeof(__be32);
101	99	size += sizeof(__be32); /* list discriminator */
102	100
103		− dprintk("RPC: %s: max reply header size = %u\n",
104		− __func__, size);
105	101	return size;
106	102	}
107	103
@@ -363,8 +359,7 @@ static struct rpcrdma_mr_seg rpcrdma_mr_prepare(struct rpcrdma_xprt r_xprt, · 255 unmodified lines
363	359	out_getmr_err:
364	360	trace_xprtrdma_nomrs(req);
365	361	xprt_wait_for_buffer_space(&r_xprt->rx_xprt);
366		− if (r_xprt->rx_ep.rep_connected != -ENODEV)
367		− schedule_work(&r_xprt->rx_buf.rb_refresh_worker);
	362	+ rpcrdma_mrs_refresh(r_xprt);
368	363	return ERR_PTR(-EAGAIN);
369	364	}
370	365
@@ -393,7 +388,7 @@ static int rpcrdma_encode_read_list(struct rpcrdma_xprt *r_xprt, · 22 unmodified lines
393	388	unsigned int pos;
394	389	int nsegs;
395	390
396		− if (rtype == rpcrdma_noch)
	391	+ if (rtype == rpcrdma_noch_pullup \|\| rtype == rpcrdma_noch_mapped)
397	392	goto done;
398	393
399	394	pos = rqst->rq_snd_buf.head[0].iov_len;
@@ -565,6 +560,7 @@ static void rpcrdma_sendctx_done(struct kref *kref) · 165 unmodified lines
565	560	*/
566	561	void rpcrdma_sendctx_unmap(struct rpcrdma_sendctx *sc)
567	562	{
	563	+ struct rpcrdma_regbuf *rb = sc->sc_req->rl_sendbuf;
568	564	struct ib_sge *sge;
569	565
570	566	if (!sc->sc_unmap_count)
@@ -576,7 +572,7 @@ void rpcrdma_sendctx_unmap(struct rpcrdma_sendctx *sc) · 5 unmodified lines
576	572	*/
577	573	for (sge = &sc->sc_sges[2]; sc->sc_unmap_count;
578	574	++sge, --sc->sc_unmap_count)
579		− ib_dma_unmap_page(sc->sc_device, sge->addr, sge->length,
	575	+ ib_dma_unmap_page(rdmab_device(rb), sge->addr, sge->length,
580	576	DMA_TO_DEVICE);
581	577
582	578	kref_put(&sc->sc_req->rl_kref, rpcrdma_sendctx_done);
@@ -589,149 +585,228 @@ static bool rpcrdma_prepare_hdr_sge(struct rpcrdma_xprt *r_xprt, · 6 unmodified lines
589	585	{
590	586	struct rpcrdma_sendctx *sc = req->rl_sendctx;
591	587	struct rpcrdma_regbuf *rb = req->rl_rdmabuf;
592		− struct ib_sge *sge = sc->sc_sges;
	588	+ struct ib_sge *sge = &sc->sc_sges[req->rl_wr.num_sge++];
593	589
594	590	if (!rpcrdma_regbuf_dma_map(r_xprt, rb))
595		− goto out_regbuf;
	591	+ return false;
596	592	sge->addr = rdmab_addr(rb);
597	593	sge->length = len;
598	594	sge->lkey = rdmab_lkey(rb);
599	595
600	596	ib_dma_sync_single_for_device(rdmab_device(rb), sge->addr, sge->length,
601	597	DMA_TO_DEVICE);
602		− sc->sc_wr.num_sge++;
603	598	return true;
604		−
605		−out_regbuf:
606		− pr_err("rpcrdma: failed to DMA map a Send buffer\n");
607		− return false;
608	599	}
609	600
610		−/* Prepare the Send SGEs. The head and tail iovec, and each entry
611		− * in the page list, gets its own SGE.
	601	+/* The head iovec is straightforward, as it is usually already
	602	+ * DMA-mapped. Sync the content that has changed.
612	603	*/
613		−static bool rpcrdma_prepare_msg_sges(struct rpcrdma_xprt *r_xprt,
614		− struct rpcrdma_req *req,
615		− struct xdr_buf *xdr,
616		− enum rpcrdma_chunktype rtype)
	604	+static bool rpcrdma_prepare_head_iov(struct rpcrdma_xprt *r_xprt,
	605	+ struct rpcrdma_req *req, unsigned int len)
617	606	{
618	607	struct rpcrdma_sendctx *sc = req->rl_sendctx;
619		− unsigned int sge_no, page_base, len, remaining;
	608	+ struct ib_sge *sge = &sc->sc_sges[req->rl_wr.num_sge++];
620	609	struct rpcrdma_regbuf *rb = req->rl_sendbuf;
621		− struct ib_sge *sge = sc->sc_sges;
622		− struct page page, *ppages;
623	610
624		− /* The head iovec is straightforward, as it is already
625		− * DMA-mapped. Sync the content that has changed.
626		− */
627	611	if (!rpcrdma_regbuf_dma_map(r_xprt, rb))
628		− goto out_regbuf;
629		− sc->sc_device = rdmab_device(rb);
630		− sge_no = 1;
631		− sge[sge_no].addr = rdmab_addr(rb);
632		− sge[sge_no].length = xdr->head[0].iov_len;
633		− sge[sge_no].lkey = rdmab_lkey(rb);
634		− ib_dma_sync_single_for_device(rdmab_device(rb), sge[sge_no].addr,
635		− sge[sge_no].length, DMA_TO_DEVICE);
636		−
637		− /* If there is a Read chunk, the page list is being handled
638		− * via explicit RDMA, and thus is skipped here. However, the
639		− * tail iovec may include an XDR pad for the page list, as
640		− * well as additional content, and may not reside in the
641		− * same page as the head iovec.
642		− */
643		− if (rtype == rpcrdma_readch) {
644		− len = xdr->tail[0].iov_len;
	612	+ return false;
645	613
646		− /* Do not include the tail if it is only an XDR pad */
647		− if (len < 4)
648		− goto out;
	614	+ sge->addr = rdmab_addr(rb);
	615	+ sge->length = len;
	616	+ sge->lkey = rdmab_lkey(rb);
649	617
650		− page = virt_to_page(xdr->tail[0].iov_base);
651		− page_base = offset_in_page(xdr->tail[0].iov_base);
	618	+ ib_dma_sync_single_for_device(rdmab_device(rb), sge->addr, sge->length,
	619	+ DMA_TO_DEVICE);
	620	+ return true;
	621	+}
652	622
653		− /* If the content in the page list is an odd length,
654		− * xdr_write_pages() has added a pad at the beginning
655		− * of the tail iovec. Force the tail's non-pad content
656		− * to land at the next XDR position in the Send message.
657		− */
658		− page_base += len & 3;
659		− len -= len & 3;
660		− goto map_tail;
661		− }
	623	+/* If there is a page list present, DMA map and prepare an
	624	+ * SGE for each page to be sent.
	625	+ */
	626	+static bool rpcrdma_prepare_pagelist(struct rpcrdma_req *req,
	627	+ struct xdr_buf *xdr)
	628	+{
	629	+ struct rpcrdma_sendctx *sc = req->rl_sendctx;
	630	+ struct rpcrdma_regbuf *rb = req->rl_sendbuf;
	631	+ unsigned int page_base, len, remaining;
	632	+ struct page **ppages;
	633	+ struct ib_sge *sge;
662	634
663		− /* If there is a page list present, temporarily DMA map
664		− * and prepare an SGE for each page to be sent.
665		− */
666		− if (xdr->page_len) {
667		− ppages = xdr->pages + (xdr->page_base >> PAGE_SHIFT);
668		− page_base = offset_in_page(xdr->page_base);
669		− remaining = xdr->page_len;
670		− while (remaining) {
671		− sge_no++;
672		− if (sge_no > RPCRDMA_MAX_SEND_SGES - 2)
673		− goto out_mapping_overflow;
674		−
675		− len = min_t(u32, PAGE_SIZE - page_base, remaining);
676		− sge[sge_no].addr =
677		− ib_dma_map_page(rdmab_device(rb), *ppages,
678		− page_base, len, DMA_TO_DEVICE);
679		− if (ib_dma_mapping_error(rdmab_device(rb),
680		− sge[sge_no].addr))
681		− goto out_mapping_err;
682		− sge[sge_no].length = len;
683		− sge[sge_no].lkey = rdmab_lkey(rb);
684		−
685		− sc->sc_unmap_count++;
686		− ppages++;
687		− remaining -= len;
688		− page_base = 0;
689		− }
690		− }
	635	+ ppages = xdr->pages + (xdr->page_base >> PAGE_SHIFT);
	636	+ page_base = offset_in_page(xdr->page_base);
	637	+ remaining = xdr->page_len;
	638	+ while (remaining) {
	639	+ sge = &sc->sc_sges[req->rl_wr.num_sge++];
	640	+ len = min_t(unsigned int, PAGE_SIZE - page_base, remaining);
	641	+ sge->addr = ib_dma_map_page(rdmab_device(rb), *ppages,
	642	+ page_base, len, DMA_TO_DEVICE);
	643	+ if (ib_dma_mapping_error(rdmab_device(rb), sge->addr))
	644	+ goto out_mapping_err;
691	645
692		− /* The tail iovec is not always constructed in the same
693		− * page where the head iovec resides (see, for example,
694		− * gss_wrap_req_priv). To neatly accommodate that case,
695		− * DMA map it separately.
696		− */
697		− if (xdr->tail[0].iov_len) {
698		− page = virt_to_page(xdr->tail[0].iov_base);
699		− page_base = offset_in_page(xdr->tail[0].iov_base);
700		− len = xdr->tail[0].iov_len;
	646	+ sge->length = len;
	647	+ sge->lkey = rdmab_lkey(rb);
701	648
702		−map_tail:
703		− sge_no++;
704		− sge[sge_no].addr =
705		− ib_dma_map_page(rdmab_device(rb), page, page_base, len,
706		− DMA_TO_DEVICE);
707		− if (ib_dma_mapping_error(rdmab_device(rb), sge[sge_no].addr))
708		− goto out_mapping_err;
709		− sge[sge_no].length = len;
710		− sge[sge_no].lkey = rdmab_lkey(rb);
711	649	sc->sc_unmap_count++;
	650	+ ppages++;
	651	+ remaining -= len;
	652	+ page_base = 0;
712	653	}
713	654
714		−out:
715		− sc->sc_wr.num_sge += sge_no;
716		− if (sc->sc_unmap_count)
717		− kref_get(&req->rl_kref);
718	655	return true;
719	656
720		−out_regbuf:
721		− pr_err("rpcrdma: failed to DMA map a Send buffer\n");
	657	+out_mapping_err:
	658	+ trace_xprtrdma_dma_maperr(sge->addr);
722	659	return false;
	660	+}
723	661
724		−out_mapping_overflow:
725		− rpcrdma_sendctx_unmap(sc);
726		− pr_err("rpcrdma: too many Send SGEs (%u)\n", sge_no);
727		− return false;
	662	+/* The tail iovec may include an XDR pad for the page list,
	663	+ * as well as additional content, and may not reside in the
	664	+ * same page as the head iovec.
	665	+ */
	666	+static bool rpcrdma_prepare_tail_iov(struct rpcrdma_req *req,
	667	+ struct xdr_buf *xdr,
	668	+ unsigned int page_base, unsigned int len)
	669	+{
	670	+ struct rpcrdma_sendctx *sc = req->rl_sendctx;
	671	+ struct ib_sge *sge = &sc->sc_sges[req->rl_wr.num_sge++];
	672	+ struct rpcrdma_regbuf *rb = req->rl_sendbuf;
	673	+ struct page *page = virt_to_page(xdr->tail[0].iov_base);
	674	+
	675	+ sge->addr = ib_dma_map_page(rdmab_device(rb), page, page_base, len,
	676	+ DMA_TO_DEVICE);
	677	+ if (ib_dma_mapping_error(rdmab_device(rb), sge->addr))
	678	+ goto out_mapping_err;
	679	+
	680	+ sge->length = len;
	681	+ sge->lkey = rdmab_lkey(rb);
	682	+ ++sc->sc_unmap_count;
	683	+ return true;
728	684
729	685	out_mapping_err:
730		− rpcrdma_sendctx_unmap(sc);
731		− trace_xprtrdma_dma_maperr(sge[sge_no].addr);
	686	+ trace_xprtrdma_dma_maperr(sge->addr);
732	687	return false;
733	688	}
734	689
	690	+/* Copy the tail to the end of the head buffer.
	691	+ */
	692	+static void rpcrdma_pullup_tail_iov(struct rpcrdma_xprt *r_xprt,
	693	+ struct rpcrdma_req *req,
	694	+ struct xdr_buf *xdr)
	695	+{
	696	+ unsigned char *dst;
	697	+
	698	+ dst = (unsigned char *)xdr->head[0].iov_base;
	699	+ dst += xdr->head[0].iov_len + xdr->page_len;
	700	+ memmove(dst, xdr->tail[0].iov_base, xdr->tail[0].iov_len);
	701	+ r_xprt->rx_stats.pullup_copy_count += xdr->tail[0].iov_len;
	702	+}
	703	+
	704	+/* Copy pagelist content into the head buffer.
	705	+ */
	706	+static void rpcrdma_pullup_pagelist(struct rpcrdma_xprt *r_xprt,
	707	+ struct rpcrdma_req *req,
	708	+ struct xdr_buf *xdr)
	709	+{
	710	+ unsigned int len, page_base, remaining;
	711	+ struct page **ppages;
	712	+ unsigned char src, dst;
	713	+
	714	+ dst = (unsigned char *)xdr->head[0].iov_base;
	715	+ dst += xdr->head[0].iov_len;
	716	+ ppages = xdr->pages + (xdr->page_base >> PAGE_SHIFT);
	717	+ page_base = offset_in_page(xdr->page_base);
	718	+ remaining = xdr->page_len;
	719	+ while (remaining) {
	720	+ src = page_address(*ppages);
	721	+ src += page_base;
	722	+ len = min_t(unsigned int, PAGE_SIZE - page_base, remaining);
	723	+ memcpy(dst, src, len);
	724	+ r_xprt->rx_stats.pullup_copy_count += len;
	725	+
	726	+ ppages++;
	727	+ dst += len;
	728	+ remaining -= len;
	729	+ page_base = 0;
	730	+ }
	731	+}
	732	+
	733	+/* Copy the contents of @xdr into @rl_sendbuf and DMA sync it.
	734	+ * When the head, pagelist, and tail are small, a pull-up copy
	735	+ * is considerably less costly than DMA mapping the components
	736	+ * of @xdr.
	737	+ *
	738	+ * Assumptions:
	739	+ * - the caller has already verified that the total length
	740	+ * of the RPC Call body will fit into @rl_sendbuf.
	741	+ */
	742	+static bool rpcrdma_prepare_noch_pullup(struct rpcrdma_xprt *r_xprt,
	743	+ struct rpcrdma_req *req,
	744	+ struct xdr_buf *xdr)
	745	+{
	746	+ if (unlikely(xdr->tail[0].iov_len))
	747	+ rpcrdma_pullup_tail_iov(r_xprt, req, xdr);
	748	+
	749	+ if (unlikely(xdr->page_len))
	750	+ rpcrdma_pullup_pagelist(r_xprt, req, xdr);
	751	+
	752	+ /* The whole RPC message resides in the head iovec now */
	753	+ return rpcrdma_prepare_head_iov(r_xprt, req, xdr->len);
	754	+}
	755	+
	756	+static bool rpcrdma_prepare_noch_mapped(struct rpcrdma_xprt *r_xprt,
	757	+ struct rpcrdma_req *req,
	758	+ struct xdr_buf *xdr)
	759	+{
	760	+ struct kvec *tail = &xdr->tail[0];
	761	+
	762	+ if (!rpcrdma_prepare_head_iov(r_xprt, req, xdr->head[0].iov_len))
	763	+ return false;
	764	+ if (xdr->page_len)
	765	+ if (!rpcrdma_prepare_pagelist(req, xdr))
	766	+ return false;
	767	+ if (tail->iov_len)
	768	+ if (!rpcrdma_prepare_tail_iov(req, xdr,
	769	+ offset_in_page(tail->iov_base),
	770	+ tail->iov_len))
	771	+ return false;
	772	+
	773	+ if (req->rl_sendctx->sc_unmap_count)
	774	+ kref_get(&req->rl_kref);
	775	+ return true;
	776	+}
	777	+
	778	+static bool rpcrdma_prepare_readch(struct rpcrdma_xprt *r_xprt,
	779	+ struct rpcrdma_req *req,
	780	+ struct xdr_buf *xdr)
	781	+{
	782	+ if (!rpcrdma_prepare_head_iov(r_xprt, req, xdr->head[0].iov_len))
	783	+ return false;
	784	+
	785	+ /* If there is a Read chunk, the page list is being handled
	786	+ * via explicit RDMA, and thus is skipped here.
	787	+ */
	788	+
	789	+ /* Do not include the tail if it is only an XDR pad */
	790	+ if (xdr->tail[0].iov_len > 3) {
	791	+ unsigned int page_base, len;
	792	+
	793	+ /* If the content in the page list is an odd length,
	794	+ * xdr_write_pages() adds a pad at the beginning of
	795	+ * the tail iovec. Force the tail's non-pad content to
	796	+ * land at the next XDR position in the Send message.
	797	+ */
	798	+ page_base = offset_in_page(xdr->tail[0].iov_base);
	799	+ len = xdr->tail[0].iov_len;
	800	+ page_base += len & 3;
	801	+ len -= len & 3;
	802	+ if (!rpcrdma_prepare_tail_iov(req, xdr, page_base, len))
	803	+ return false;
	804	+ kref_get(&req->rl_kref);
	805	+ }
	806	+
	807	+ return true;
	808	+}
	809	+
735	810	/**
736	811	* rpcrdma_prepare_send_sges - Construct SGEs for a Send WR
737	812	* @r_xprt: controlling transport
@@ -742,31 +817,53 @@ out_mapping_err: · 4 unmodified lines
742	817	*
743	818	* Returns 0 on success; otherwise a negative errno is returned.
744	819	*/
745		−int
746		−rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt,
747		− struct rpcrdma_req *req, u32 hdrlen,
748		− struct xdr_buf *xdr, enum rpcrdma_chunktype rtype)
	820	+inline int rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt,
	821	+ struct rpcrdma_req *req, u32 hdrlen,
	822	+ struct xdr_buf *xdr,
	823	+ enum rpcrdma_chunktype rtype)
749	824	{
750	825	int ret;
751	826
752	827	ret = -EAGAIN;
753	828	req->rl_sendctx = rpcrdma_sendctx_get_locked(r_xprt);
754	829	if (!req->rl_sendctx)
755		− goto err;
756		− req->rl_sendctx->sc_wr.num_sge = 0;
	830	+ goto out_nosc;
757	831	req->rl_sendctx->sc_unmap_count = 0;
758	832	req->rl_sendctx->sc_req = req;
759	833	kref_init(&req->rl_kref);
	834	+ req->rl_wr.wr_cqe = &req->rl_sendctx->sc_cqe;
	835	+ req->rl_wr.sg_list = req->rl_sendctx->sc_sges;
	836	+ req->rl_wr.num_sge = 0;
	837	+ req->rl_wr.opcode = IB_WR_SEND;
760	838
761	839	ret = -EIO;
762	840	if (!rpcrdma_prepare_hdr_sge(r_xprt, req, hdrlen))
763		− goto err;
764		− if (rtype != rpcrdma_areadch)
765		− if (!rpcrdma_prepare_msg_sges(r_xprt, req, xdr, rtype))
766		− goto err;
	841	+ goto out_unmap;
	842	+
	843	+ switch (rtype) {
	844	+ case rpcrdma_noch_pullup:
	845	+ if (!rpcrdma_prepare_noch_pullup(r_xprt, req, xdr))
	846	+ goto out_unmap;
	847	+ break;
	848	+ case rpcrdma_noch_mapped:
	849	+ if (!rpcrdma_prepare_noch_mapped(r_xprt, req, xdr))
	850	+ goto out_unmap;
	851	+ break;
	852	+ case rpcrdma_readch:
	853	+ if (!rpcrdma_prepare_readch(r_xprt, req, xdr))
	854	+ goto out_unmap;
	855	+ break;
	856	+ case rpcrdma_areadch:
	857	+ break;
	858	+ default:
	859	+ goto out_unmap;
	860	+ }
	861	+
767	862	return 0;
768	863
769		−err:
	864	+out_unmap:
	865	+ rpcrdma_sendctx_unmap(req->rl_sendctx);
	866	+out_nosc:
770	867	trace_xprtrdma_prepsend_failed(&req->rl_slot, ret);
771	868	return ret;
772	869	}
@@ -796,6 +893,7 @@ rpcrdma_marshal_req(struct rpcrdma_xprt r_xprt, struct rpc_rqst rqst) · 23 unmodified lines
796	893	struct rpcrdma_req *req = rpcr_to_rdmar(rqst);
797	894	struct xdr_stream *xdr = &req->rl_stream;
798	895	enum rpcrdma_chunktype rtype, wtype;
	896	+ struct xdr_buf *buf = &rqst->rq_snd_buf;
799	897	bool ddp_allowed;
800	898	__be32 *p;
801	899	int ret;
@@ -853,8 +951,9 @@ rpcrdma_marshal_req(struct rpcrdma_xprt r_xprt, struct rpc_rqst rqst) · 51 unmodified lines
853	951	*/
854	952	if (rpcrdma_args_inline(r_xprt, rqst)) {
855	953	*p++ = rdma_msg;
856		− rtype = rpcrdma_noch;
857		− } else if (ddp_allowed && rqst->rq_snd_buf.flags & XDRBUF_WRITE) {
	954	+ rtype = buf->len < rdmab_length(req->rl_sendbuf) ?
	955	+ rpcrdma_noch_pullup : rpcrdma_noch_mapped;
	956	+ } else if (ddp_allowed && buf->flags & XDRBUF_WRITE) {
858	957	*p++ = rdma_msg;
859	958	rtype = rpcrdma_readch;
860	959	} else {
@@ -863,12 +962,6 @@ rpcrdma_marshal_req(struct rpcrdma_xprt r_xprt, struct rpc_rqst rqst) · 2 unmodified lines
863	962	rtype = rpcrdma_areadch;
864	963	}
865	964
866		− /* If this is a retransmit, discard previously registered
867		− * chunks. Very likely the connection has been replaced,
868		− * so these registrations are invalid and unusable.
869		− */
870		− frwr_recycle(req);
871		−
872	965	/* This implementation supports the following combinations
873	966	* of chunk lists in one RPC-over-RDMA Call message:
874	967	*
@@ -902,7 +995,7 @@ rpcrdma_marshal_req(struct rpcrdma_xprt r_xprt, struct rpc_rqst rqst) · 27 unmodified lines
902	995	goto out_err;
903	996
904	997	ret = rpcrdma_prepare_send_sges(r_xprt, req, req->rl_hdrbuf.len,
905		− &rqst->rq_snd_buf, rtype);
	998	+ buf, rtype);
906	999	if (ret)
907	1000	goto out_err;
908	1001
@@ -916,6 +1009,40 @@ out_err: · 7 unmodified lines
916	1009	return ret;
917	1010	}
918	1011
	1012	+static void __rpcrdma_update_cwnd_locked(struct rpc_xprt *xprt,
	1013	+ struct rpcrdma_buffer *buf,
	1014	+ u32 grant)
	1015	+{
	1016	+ buf->rb_credits = grant;
	1017	+ xprt->cwnd = grant << RPC_CWNDSHIFT;
	1018	+}
	1019	+
	1020	+static void rpcrdma_update_cwnd(struct rpcrdma_xprt *r_xprt, u32 grant)
	1021	+{
	1022	+ struct rpc_xprt *xprt = &r_xprt->rx_xprt;
	1023	+
	1024	+ spin_lock(&xprt->transport_lock);
	1025	+ __rpcrdma_update_cwnd_locked(xprt, &r_xprt->rx_buf, grant);
	1026	+ spin_unlock(&xprt->transport_lock);
	1027	+}
	1028	+
	1029	+/**
	1030	+ * rpcrdma_reset_cwnd - Reset the xprt's congestion window
	1031	+ * @r_xprt: controlling transport instance
	1032	+ *
	1033	+ * Prepare @r_xprt for the next connection by reinitializing
	1034	+ * its credit grant to one (see RFC 8166, Section 3.3.3).
	1035	+ */
	1036	+void rpcrdma_reset_cwnd(struct rpcrdma_xprt *r_xprt)
	1037	+{
	1038	+ struct rpc_xprt *xprt = &r_xprt->rx_xprt;
	1039	+
	1040	+ spin_lock(&xprt->transport_lock);
	1041	+ xprt->cong = 0;
	1042	+ __rpcrdma_update_cwnd_locked(xprt, &r_xprt->rx_buf, 1);
	1043	+ spin_unlock(&xprt->transport_lock);
	1044	+}
	1045	+
919	1046	/**
920	1047	* rpcrdma_inline_fixup - Scatter inline received data into rqst's iovecs
921	1048	* @rqst: controlling RPC request
@@ -955,7 +1082,6 @@ rpcrdma_inline_fixup(struct rpc_rqst rqst, char srcp, int copy_len, int pad) · 33 unmodified lines
955	1082	curlen = rqst->rq_rcv_buf.head[0].iov_len;
956	1083	if (curlen > copy_len)
957	1084	curlen = copy_len;
958		− trace_xprtrdma_fixup(rqst, copy_len, curlen);
959	1085	srcp += curlen;
960	1086	copy_len -= curlen;
961	1087
@@ -975,8 +1101,6 @@ rpcrdma_inline_fixup(struct rpc_rqst rqst, char srcp, int copy_len, int pad) · 13 unmodified lines
975	1101	if (curlen > pagelist_len)
976	1102	curlen = pagelist_len;
977	1103
978		− trace_xprtrdma_fixup_pg(rqst, i, srcp,
979		− copy_len, curlen);
980	1104	destp = kmap_atomic(ppages[i]);
981	1105	memcpy(destp + page_base, srcp, curlen);
982	1106	flush_dcache_page(ppages[i]);
@@ -1008,6 +1132,8 @@ rpcrdma_inline_fixup(struct rpc_rqst rqst, char srcp, int copy_len, int pad) · 25 unmodified lines
1008	1132	rqst->rq_private_buf.tail[0].iov_base = srcp;
1009	1133	}
1010	1134
	1135	+ if (fixup_copy_count)
	1136	+ trace_xprtrdma_fixup(rqst, fixup_copy_count);
1011	1137	return fixup_copy_count;
1012	1138	}
1013	1139
@@ -1356,12 +1482,9 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *rep) · 342 unmodified lines
1356	1482	credits = 1; /* don't deadlock */
1357	1483	else if (credits > buf->rb_max_requests)
1358	1484	credits = buf->rb_max_requests;
1359		− if (buf->rb_credits != credits) {
1360		− spin_lock(&xprt->transport_lock);
1361		− buf->rb_credits = credits;
1362		− xprt->cwnd = credits << RPC_CWNDSHIFT;
1363		− spin_unlock(&xprt->transport_lock);
1364		− }
	1485	+ if (buf->rb_credits != credits)
	1486	+ rpcrdma_update_cwnd(r_xprt, credits);
	1487	+ rpcrdma_post_recvs(r_xprt, false);
1365	1488
1366	1489	req = rpcr_to_rdmar(rqst);
1367	1490	if (req->rl_reply) {

net/sunrpc/xprtrdma/transport.c+10−23

@@ -243,16 +243,13 @@ xprt_rdma_connect_worker(struct work_struct *work) · 242 unmodified lines
243	243	rc = rpcrdma_ep_connect(&r_xprt->rx_ep, &r_xprt->rx_ia);
244	244	xprt_clear_connecting(xprt);
245	245	if (r_xprt->rx_ep.rep_connected > 0) {
246		− if (!xprt_test_and_set_connected(xprt)) {
247		− xprt->stat.connect_count++;
248		− xprt->stat.connect_time += (long)jiffies -
249		− xprt->stat.connect_start;
250		− xprt_wake_pending_tasks(xprt, -EAGAIN);
251		− }
252		− } else {
253		− if (xprt_test_and_clear_connected(xprt))
254		− xprt_wake_pending_tasks(xprt, rc);
	246	+ xprt->stat.connect_count++;
	247	+ xprt->stat.connect_time += (long)jiffies -
	248	+ xprt->stat.connect_start;
	249	+ xprt_set_connected(xprt);
	250	+ rc = -EAGAIN;
255	251	}
	252	+ xprt_wake_pending_tasks(xprt, rc);
256	253	}
257	254
258	255	/**
@@ -425,12 +422,6 @@ void xprt_rdma_close(struct rpc_xprt *xprt) · 166 unmodified lines
425	422	return;
426	423	rpcrdma_ep_disconnect(ep, ia);
427	424
428		− /* Prepare @xprt for the next connection by reinitializing
429		− * its credit grant to one (see RFC 8166, Section 3.3.3).
430		− */
431		− r_xprt->rx_buf.rb_credits = 1;
432		− xprt->cwnd = RPC_CWNDSHIFT;
433		−
434	425	out:
435	426	xprt->reestablish_timeout = 0;
436	427	++xprt->connect_cookie;
@@ -450,12 +441,6 @@ xprt_rdma_set_port(struct rpc_xprt *xprt, u16 port) · 13 unmodified lines
450	441	struct sockaddr sap = (struct sockaddr )&xprt->addr;
451	442	char buf[8];
452	443
453		− dprintk("RPC: %s: setting port for xprt %p (%s:%s) to %u\n",
454		− __func__, xprt,
455		− xprt->address_strings[RPC_DISPLAY_ADDR],
456		− xprt->address_strings[RPC_DISPLAY_PORT],
457		− port);
458		−
459	444	rpc_set_port(sap, port);
460	445
461	446	kfree(xprt->address_strings[RPC_DISPLAY_PORT]);
@@ -465,6 +450,9 @@ xprt_rdma_set_port(struct rpc_xprt *xprt, u16 port) · 3 unmodified lines
465	450	kfree(xprt->address_strings[RPC_DISPLAY_HEX_PORT]);
466	451	snprintf(buf, sizeof(buf), "%4hx", port);
467	452	xprt->address_strings[RPC_DISPLAY_HEX_PORT] = kstrdup(buf, GFP_KERNEL);
	453	+
	454	+ trace_xprtrdma_op_setport(container_of(xprt, struct rpcrdma_xprt,
	455	+ rx_xprt));
468	456	}
469	457
470	458	/**
@@ -536,13 +524,12 @@ xprt_rdma_connect(struct rpc_xprt xprt, struct rpc_task task) · 65 unmodified lines
536	524	struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);
537	525	unsigned long delay;
538	526
539		− trace_xprtrdma_op_connect(r_xprt);
540		−
541	527	delay = 0;
542	528	if (r_xprt->rx_ep.rep_connected != 0) {
543	529	delay = xprt_reconnect_delay(xprt);
544	530	xprt_reconnect_backoff(xprt, RPCRDMA_INIT_REEST_TO);
545	531	}
	532	+ trace_xprtrdma_op_connect(r_xprt, delay);
546	533	queue_delayed_work(xprtiod_workqueue, &r_xprt->rx_connect_worker,
547	534	delay);
548	535	}

net/sunrpc/xprtrdma/verbs.c+110−84

@@ -74,17 +74,17 @@ · 73 unmodified lines
74	74	/*
75	75	* internal functions
76	76	*/
77		−static void rpcrdma_sendctx_put_locked(struct rpcrdma_sendctx *sc);
	77	+static void rpcrdma_sendctx_put_locked(struct rpcrdma_xprt *r_xprt,
	78	+ struct rpcrdma_sendctx *sc);
	79	+static void rpcrdma_reqs_reset(struct rpcrdma_xprt *r_xprt);
78	80	static void rpcrdma_reps_destroy(struct rpcrdma_buffer *buf);
79	81	static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt);
80		−static void rpcrdma_mrs_destroy(struct rpcrdma_buffer *buf);
81		−static void rpcrdma_mr_free(struct rpcrdma_mr *mr);
	82	+static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt);
82	83	static struct rpcrdma_regbuf *
83	84	rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction,
84	85	gfp_t flags);
85	86	static void rpcrdma_regbuf_dma_unmap(struct rpcrdma_regbuf *rb);
86	87	static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb);
87		−static void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp);
88	88
89	89	/* Wait for outstanding transport work to finish. ib_drain_qp
90	90	* handles the drains in the wrong order for us, so open code
@@ -125,7 +125,7 @@ rpcrdma_qp_event_handler(struct ib_event event, void context) · 34 unmodified lines
125	125
126	126	/**
127	127	* rpcrdma_wc_send - Invoked by RDMA provider for each polled Send WC
128		− * @cq: completion queue (ignored)
	128	+ * @cq: completion queue
129	129	* @wc: completed WR
130	130	*
131	131	*/
@@ -138,7 +138,7 @@ rpcrdma_wc_send(struct ib_cq cq, struct ib_wc wc) · 6 unmodified lines
138	138
139	139	/* WARNING: Only wr_cqe and status are reliable at this point */
140	140	trace_xprtrdma_wc_send(sc, wc);
141		− rpcrdma_sendctx_put_locked(sc);
	141	+ rpcrdma_sendctx_put_locked((struct rpcrdma_xprt *)cq->cq_context, sc);
142	142	}
143	143
144	144	/**
@@ -170,7 +170,6 @@ rpcrdma_wc_receive(struct ib_cq cq, struct ib_wc wc) · 25 unmodified lines
170	170	rdmab_addr(rep->rr_rdmabuf),
171	171	wc->byte_len, DMA_FROM_DEVICE);
172	172
173		− rpcrdma_post_recvs(r_xprt, false);
174	173	rpcrdma_reply_handler(rep);
175	174	return;
176	175
@@ -178,11 +177,11 @@ out_flushed: · 1 unmodified line
178	177	rpcrdma_recv_buffer_put(rep);
179	178	}
180	179
181		−static void
182		−rpcrdma_update_connect_private(struct rpcrdma_xprt *r_xprt,
183		− struct rdma_conn_param *param)
	180	+static void rpcrdma_update_cm_private(struct rpcrdma_xprt *r_xprt,
	181	+ struct rdma_conn_param *param)
184	182	{
185	183	const struct rpcrdma_connect_private *pmsg = param->private_data;
	184	+ struct rpcrdma_ep *ep = &r_xprt->rx_ep;
186	185	unsigned int rsize, wsize;
187	186
188	187	/* Default settings for RPC-over-RDMA Version One */
@@ -198,13 +197,11 @@ rpcrdma_update_connect_private(struct rpcrdma_xprt *r_xprt, · 9 unmodified lines
198	197	wsize = rpcrdma_decode_buffer_size(pmsg->cp_recv_size);
199	198	}
200	199
201		− if (rsize < r_xprt->rx_ep.rep_inline_recv)
202		− r_xprt->rx_ep.rep_inline_recv = rsize;
203		− if (wsize < r_xprt->rx_ep.rep_inline_send)
204		− r_xprt->rx_ep.rep_inline_send = wsize;
205		− dprintk("RPC: %s: max send %u, max recv %u\n", __func__,
206		− r_xprt->rx_ep.rep_inline_send,
207		− r_xprt->rx_ep.rep_inline_recv);
	200	+ if (rsize < ep->rep_inline_recv)
	201	+ ep->rep_inline_recv = rsize;
	202	+ if (wsize < ep->rep_inline_send)
	203	+ ep->rep_inline_send = wsize;
	204	+
208	205	rpcrdma_set_max_header_sizes(r_xprt);
209	206	}
210	207
@@ -258,7 +255,8 @@ rpcrdma_cm_event_handler(struct rdma_cm_id id, struct rdma_cm_event event) · 47 unmodified lines
258	255	case RDMA_CM_EVENT_ESTABLISHED:
259	256	++xprt->connect_cookie;
260	257	ep->rep_connected = 1;
261		− rpcrdma_update_connect_private(r_xprt, &event->param.conn);
	258	+ rpcrdma_update_cm_private(r_xprt, &event->param.conn);
	259	+ trace_xprtrdma_inline_thresh(r_xprt);
262	260	wake_up_all(&ep->rep_connect_wait);
263	261	break;
264	262	case RDMA_CM_EVENT_CONNECT_ERROR:
@@ -298,8 +296,6 @@ rpcrdma_create_id(struct rpcrdma_xprt xprt, struct rpcrdma_ia ia) · 33 unmodified lines
298	296	struct rdma_cm_id *id;
299	297	int rc;
300	298
301		− trace_xprtrdma_conn_start(xprt);
302		−
303	299	init_completion(&ia->ri_done);
304	300	init_completion(&ia->ri_remove_done);
305	301
@@ -315,10 +311,8 @@ rpcrdma_create_id(struct rpcrdma_xprt xprt, struct rpcrdma_ia ia) · 9 unmodified lines
315	311	if (rc)
316	312	goto out;
317	313	rc = wait_for_completion_interruptible_timeout(&ia->ri_done, wtimeout);
318		− if (rc < 0) {
319		− trace_xprtrdma_conn_tout(xprt);
	314	+ if (rc < 0)
320	315	goto out;
321		− }
322	316
323	317	rc = ia->ri_async_rc;
324	318	if (rc)
@@ -329,10 +323,8 @@ rpcrdma_create_id(struct rpcrdma_xprt xprt, struct rpcrdma_ia ia) · 4 unmodified lines
329	323	if (rc)
330	324	goto out;
331	325	rc = wait_for_completion_interruptible_timeout(&ia->ri_done, wtimeout);
332		− if (rc < 0) {
333		− trace_xprtrdma_conn_tout(xprt);
	326	+ if (rc < 0)
334	327	goto out;
335		− }
336	328	rc = ia->ri_async_rc;
337	329	if (rc)
338	330	goto out;
@@ -409,8 +401,6 @@ rpcrdma_ia_remove(struct rpcrdma_ia *ia) · 70 unmodified lines
409	401	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
410	402	struct rpcrdma_req *req;
411	403
412		− cancel_work_sync(&buf->rb_refresh_worker);
413		−
414	404	/* This is similar to rpcrdma_ep_destroy, but:
415	405	* - Don't cancel the connect worker.
416	406	* - Don't call rpcrdma_ep_disconnect, which waits
@@ -437,7 +427,7 @@ rpcrdma_ia_remove(struct rpcrdma_ia *ia) · 20 unmodified lines
437	427	rpcrdma_regbuf_dma_unmap(req->rl_sendbuf);
438	428	rpcrdma_regbuf_dma_unmap(req->rl_recvbuf);
439	429	}
440		− rpcrdma_mrs_destroy(buf);
	430	+ rpcrdma_mrs_destroy(r_xprt);
441	431	ib_dealloc_pd(ia->ri_pd);
442	432	ia->ri_pd = NULL;
443	433
@@ -522,7 +512,7 @@ int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt) · 78 unmodified lines
522	512	init_waitqueue_head(&ep->rep_connect_wait);
523	513	ep->rep_receive_count = 0;
524	514
525		− sendcq = ib_alloc_cq_any(ia->ri_id->device, NULL,
	515	+ sendcq = ib_alloc_cq_any(ia->ri_id->device, r_xprt,
526	516	ep->rep_attr.cap.max_send_wr + 1,
527	517	IB_POLL_WORKQUEUE);
528	518	if (IS_ERR(sendcq)) {
@@ -630,8 +620,6 @@ static int rpcrdma_ep_recreate_xprt(struct rpcrdma_xprt *r_xprt, · 101 unmodified lines
630	620	pr_err("rpcrdma: rdma_create_qp returned %d\n", err);
631	621	goto out3;
632	622	}
633		−
634		− rpcrdma_mrs_create(r_xprt);
635	623	return 0;
636	624
637	625	out3:
@@ -649,8 +637,6 @@ static int rpcrdma_ep_reconnect(struct rpcrdma_xprt *r_xprt, · 11 unmodified lines
649	637	struct rdma_cm_id id, old;
650	638	int err, rc;
651	639
652		− trace_xprtrdma_reconnect(r_xprt);
653		−
654	640	rpcrdma_ep_disconnect(&r_xprt->rx_ep, ia);
655	641
656	642	rc = -EHOSTUNREACH;
@@ -705,7 +691,6 @@ retry: · 48 unmodified lines
705	691	memcpy(&qp_init_attr, &ep->rep_attr, sizeof(qp_init_attr));
706	692	switch (ep->rep_connected) {
707	693	case 0:
708		− dprintk("RPC: %s: connecting...\n", __func__);
709	694	rc = rdma_create_qp(ia->ri_id, ia->ri_pd, &qp_init_attr);
710	695	if (rc) {
711	696	rc = -ENETUNREACH;
@@ -726,6 +711,7 @@ retry: · 14 unmodified lines
726	711	ep->rep_connected = 0;
727	712	xprt_clear_connected(xprt);
728	713
	714	+ rpcrdma_reset_cwnd(r_xprt);
729	715	rpcrdma_post_recvs(r_xprt, true);
730	716
731	717	rc = rdma_connect(ia->ri_id, &ep->rep_remote_cma);
@@ -742,13 +728,14 @@ retry: · 10 unmodified lines
742	728	goto out;
743	729	}
744	730
745		− dprintk("RPC: %s: connected\n", __func__);
	731	+ rpcrdma_mrs_create(r_xprt);
746	732
747	733	out:
748	734	if (rc)
749	735	ep->rep_connected = rc;
750	736
751	737	out_noupdate:
	738	+ trace_xprtrdma_connect(r_xprt, rc);
752	739	return rc;
753	740	}
754	741
@@ -757,11 +744,8 @@ out_noupdate: · 2 unmodified lines
757	744	* @ep: endpoint to disconnect
758	745	* @ia: associated interface adapter
759	746	*
760		− * This is separate from destroy to facilitate the ability
761		− * to reconnect without recreating the endpoint.
762		− *
763		− * This call is not reentrant, and must not be made in parallel
764		− * on the same endpoint.
	747	+ * Caller serializes. Either the transport send lock is held,
	748	+ * or we're being called to destroy the transport.
765	749	*/
766	750	void
767	751	rpcrdma_ep_disconnect(struct rpcrdma_ep ep, struct rpcrdma_ia ia)
@@ -780,6 +764,8 @@ rpcrdma_ep_disconnect(struct rpcrdma_ep ep, struct rpcrdma_ia ia) · 12 unmodified lines
780	764	trace_xprtrdma_disconnect(r_xprt, rc);
781	765
782	766	rpcrdma_xprt_drain(r_xprt);
	767	+ rpcrdma_reqs_reset(r_xprt);
	768	+ rpcrdma_mrs_destroy(r_xprt);
783	769	}
784	770
785	771	/* Fixed-size circular FIFO queue. This implementation is wait-free and
@@ -817,9 +803,6 @@ static struct rpcrdma_sendctx rpcrdma_sendctx_create(struct rpcrdma_ia ia) · 31 unmodified lines
817	803	if (!sc)
818	804	return NULL;
819	805
820		− sc->sc_wr.wr_cqe = &sc->sc_cqe;
821		− sc->sc_wr.sg_list = sc->sc_sges;
822		− sc->sc_wr.opcode = IB_WR_SEND;
823	806	sc->sc_cqe.done = rpcrdma_wc_send;
824	807	return sc;
825	808	}
@@ -847,7 +830,6 @@ static int rpcrdma_sendctxs_create(struct rpcrdma_xprt *r_xprt) · 21 unmodified lines
847	830	if (!sc)
848	831	return -ENOMEM;
849	832
850		− sc->sc_xprt = r_xprt;
851	833	buf->rb_sc_ctxs[i] = sc;
852	834	}
853	835
@@ -910,6 +892,7 @@ out_emptyq: · 56 unmodified lines
910	892
911	893	/**
912	894	* rpcrdma_sendctx_put_locked - Release a send context
	895	+ * @r_xprt: controlling transport instance
913	896	* @sc: send context to release
914	897	*
915	898	* Usage: Called from Send completion to return a sendctxt
@@ -917,10 +900,10 @@ out_emptyq: · 1 unmodified line
917	900	*
918	901	* The caller serializes calls to this function (per transport).
919	902	*/
920		−static void
921		−rpcrdma_sendctx_put_locked(struct rpcrdma_sendctx *sc)
	903	+static void rpcrdma_sendctx_put_locked(struct rpcrdma_xprt *r_xprt,
	904	+ struct rpcrdma_sendctx *sc)
922	905	{
923		− struct rpcrdma_buffer *buf = &sc->sc_xprt->rx_buf;
	906	+ struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
924	907	unsigned long next_tail;
925	908
926	909	/* Unmap SGEs of previously completed but unsignaled
@@ -938,7 +921,7 @@ rpcrdma_sendctx_put_locked(struct rpcrdma_sendctx *sc) · 11 unmodified lines
938	921	/* Paired with READ_ONCE */
939	922	smp_store_release(&buf->rb_sc_tail, next_tail);
940	923
941		− xprt_write_space(&sc->sc_xprt->rx_xprt);
	924	+ xprt_write_space(&r_xprt->rx_xprt);
942	925	}
943	926
944	927	static void
@@ -965,7 +948,7 @@ rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt) · 20 unmodified lines
965	948	mr->mr_xprt = r_xprt;
966	949
967	950	spin_lock(&buf->rb_lock);
968		− list_add(&mr->mr_list, &buf->rb_mrs);
	951	+ rpcrdma_mr_push(mr, &buf->rb_mrs);
969	952	list_add(&mr->mr_all, &buf->rb_all_mrs);
970	953	spin_unlock(&buf->rb_lock);
971	954	}
@@ -986,6 +969,28 @@ rpcrdma_mr_refresh_worker(struct work_struct *work) · 14 unmodified lines
986	969	xprt_write_space(&r_xprt->rx_xprt);
987	970	}
988	971
	972	+/**
	973	+ * rpcrdma_mrs_refresh - Wake the MR refresh worker
	974	+ * @r_xprt: controlling transport instance
	975	+ *
	976	+ */
	977	+void rpcrdma_mrs_refresh(struct rpcrdma_xprt *r_xprt)
	978	+{
	979	+ struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
	980	+ struct rpcrdma_ep *ep = &r_xprt->rx_ep;
	981	+
	982	+ /* If there is no underlying device, it's no use to
	983	+ * wake the refresh worker.
	984	+ */
	985	+ if (ep->rep_connected != -ENODEV) {
	986	+ /* The work is scheduled on a WQ_MEM_RECLAIM
	987	+ * workqueue in order to prevent MR allocation
	988	+ * from recursing into NFS during direct reclaim.
	989	+ */
	990	+ queue_work(xprtiod_workqueue, &buf->rb_refresh_worker);
	991	+ }
	992	+}
	993	+
989	994	/**
990	995	* rpcrdma_req_create - Allocate an rpcrdma_req object
991	996	* @r_xprt: controlling r_xprt
@@ -1042,6 +1047,26 @@ out1: · 50 unmodified lines
1042	1047	return NULL;
1043	1048	}
1044	1049
	1050	+/**
	1051	+ * rpcrdma_reqs_reset - Reset all reqs owned by a transport
	1052	+ * @r_xprt: controlling transport instance
	1053	+ *
	1054	+ * ASSUMPTION: the rb_allreqs list is stable for the duration,
	1055	+ * and thus can be walked without holding rb_lock. Eg. the
	1056	+ * caller is holding the transport send lock to exclude
	1057	+ * device removal or disconnection.
	1058	+ */
	1059	+static void rpcrdma_reqs_reset(struct rpcrdma_xprt *r_xprt)
	1060	+{
	1061	+ struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
	1062	+ struct rpcrdma_req *req;
	1063	+
	1064	+ list_for_each_entry(req, &buf->rb_allreqs, rl_all) {
	1065	+ /* Credits are valid only for one connection */
	1066	+ req->rl_slot.rq_cong = 0;
	1067	+ }
	1068	+}
	1069	+
1045	1070	static struct rpcrdma_rep rpcrdma_rep_create(struct rpcrdma_xprt r_xprt,
1046	1071	bool temp)
1047	1072	{
@@ -1125,8 +1150,6 @@ int rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt) · 77 unmodified lines
1125	1150	INIT_LIST_HEAD(&buf->rb_all_mrs);
1126	1151	INIT_WORK(&buf->rb_refresh_worker, rpcrdma_mr_refresh_worker);
1127	1152
1128		− rpcrdma_mrs_create(r_xprt);
1129		−
1130	1153	INIT_LIST_HEAD(&buf->rb_send_bufs);
1131	1154	INIT_LIST_HEAD(&buf->rb_allreqs);
1132	1155
@@ -1134,14 +1157,13 @@ int rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt) · 1 unmodified line
1134	1157	for (i = 0; i < buf->rb_max_requests; i++) {
1135	1158	struct rpcrdma_req *req;
1136	1159
1137		− req = rpcrdma_req_create(r_xprt, RPCRDMA_V1_DEF_INLINE_SIZE,
	1160	+ req = rpcrdma_req_create(r_xprt, RPCRDMA_V1_DEF_INLINE_SIZE * 2,
1138	1161	GFP_KERNEL);
1139	1162	if (!req)
1140	1163	goto out;
1141	1164	list_add(&req->rl_list, &buf->rb_send_bufs);
1142	1165	}
1143	1166
1144		− buf->rb_credits = 1;
1145	1167	init_llist_head(&buf->rb_free_reps);
1146	1168
1147	1169	rc = rpcrdma_sendctxs_create(r_xprt);
@@ -1158,15 +1180,24 @@ out: · 10 unmodified lines
1158	1180	* rpcrdma_req_destroy - Destroy an rpcrdma_req object
1159	1181	* @req: unused object to be destroyed
1160	1182	*
1161		− * This function assumes that the caller prevents concurrent device
1162		− * unload and transport tear-down.
	1183	+ * Relies on caller holding the transport send lock to protect
	1184	+ * removing req->rl_all from buf->rb_all_reqs safely.
1163	1185	*/
1164	1186	void rpcrdma_req_destroy(struct rpcrdma_req *req)
1165	1187	{
	1188	+ struct rpcrdma_mr *mr;
	1189	+
1166	1190	list_del(&req->rl_all);
1167	1191
1168		− while (!list_empty(&req->rl_free_mrs))
1169		− rpcrdma_mr_free(rpcrdma_mr_pop(&req->rl_free_mrs));
	1192	+ while ((mr = rpcrdma_mr_pop(&req->rl_free_mrs))) {
	1193	+ struct rpcrdma_buffer *buf = &mr->mr_xprt->rx_buf;
	1194	+
	1195	+ spin_lock(&buf->rb_lock);
	1196	+ list_del(&mr->mr_all);
	1197	+ spin_unlock(&buf->rb_lock);
	1198	+
	1199	+ frwr_release_mr(mr);
	1200	+ }
1170	1201
1171	1202	rpcrdma_regbuf_free(req->rl_recvbuf);
1172	1203	rpcrdma_regbuf_free(req->rl_sendbuf);
@@ -1174,28 +1205,33 @@ void rpcrdma_req_destroy(struct rpcrdma_req *req) · 1 unmodified line
1174	1205	kfree(req);
1175	1206	}
1176	1207
1177		−static void
1178		−rpcrdma_mrs_destroy(struct rpcrdma_buffer *buf)
	1208	+/**
	1209	+ * rpcrdma_mrs_destroy - Release all of a transport's MRs
	1210	+ * @r_xprt: controlling transport instance
	1211	+ *
	1212	+ * Relies on caller holding the transport send lock to protect
	1213	+ * removing mr->mr_list from req->rl_free_mrs safely.
	1214	+ */
	1215	+static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt)
1179	1216	{
1180		− struct rpcrdma_xprt *r_xprt = container_of(buf, struct rpcrdma_xprt,
1181		− rx_buf);
	1217	+ struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
1182	1218	struct rpcrdma_mr *mr;
1183		− unsigned int count;
1184	1219
1185		− count = 0;
	1220	+ cancel_work_sync(&buf->rb_refresh_worker);
	1221	+
1186	1222	spin_lock(&buf->rb_lock);
1187	1223	while ((mr = list_first_entry_or_null(&buf->rb_all_mrs,
1188	1224	struct rpcrdma_mr,
1189	1225	mr_all)) != NULL) {
	1226	+ list_del(&mr->mr_list);
1190	1227	list_del(&mr->mr_all);
1191	1228	spin_unlock(&buf->rb_lock);
1192	1229
1193	1230	frwr_release_mr(mr);
1194		− count++;
	1231	+
1195	1232	spin_lock(&buf->rb_lock);
1196	1233	}
1197	1234	spin_unlock(&buf->rb_lock);
1198		− r_xprt->rx_stats.mrs_allocated = 0;
1199	1235	}
1200	1236
1201	1237	/**
@@ -1209,8 +1245,6 @@ rpcrdma_mrs_destroy(struct rpcrdma_buffer *buf) · 7 unmodified lines
1209	1245	void
1210	1246	rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf)
1211	1247	{
1212		− cancel_work_sync(&buf->rb_refresh_worker);
1213		−
1214	1248	rpcrdma_sendctxs_destroy(buf);
1215	1249	rpcrdma_reps_destroy(buf);
1216	1250
@@ -1222,8 +1256,6 @@ rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf) · 5 unmodified lines
1222	1256	list_del(&req->rl_list);
1223	1257	rpcrdma_req_destroy(req);
1224	1258	}
1225		−
1226		− rpcrdma_mrs_destroy(buf);
1227	1259	}
1228	1260
1229	1261	/**
@@ -1264,17 +1296,6 @@ void rpcrdma_mr_put(struct rpcrdma_mr *mr) · 34 unmodified lines
1264	1296	rpcrdma_mr_push(mr, &mr->mr_req->rl_free_mrs);
1265	1297	}
1266	1298
1267		−static void rpcrdma_mr_free(struct rpcrdma_mr *mr)
1268		−{
1269		− struct rpcrdma_xprt *r_xprt = mr->mr_xprt;
1270		− struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
1271		−
1272		− mr->mr_req = NULL;
1273		− spin_lock(&buf->rb_lock);
1274		− rpcrdma_mr_push(mr, &buf->rb_mrs);
1275		− spin_unlock(&buf->rb_lock);
1276		−}
1277		−
1278	1299	/**
1279	1300	* rpcrdma_buffer_get - Get a request buffer
1280	1301	* @buffers: Buffer pool from which to obtain a buffer
@@ -1437,7 +1458,7 @@ rpcrdma_ep_post(struct rpcrdma_ia *ia, · 156 unmodified lines
1437	1458	struct rpcrdma_ep *ep,
1438	1459	struct rpcrdma_req *req)
1439	1460	{
1440		− struct ib_send_wr *send_wr = &req->rl_sendctx->sc_wr;
	1461	+ struct ib_send_wr *send_wr = &req->rl_wr;
1441	1462	int rc;
1442	1463
1443	1464	if (!ep->rep_send_count \|\| kref_read(&req->rl_kref) > 1) {
@@ -1455,8 +1476,13 @@ rpcrdma_ep_post(struct rpcrdma_ia *ia, · 11 unmodified lines
1455	1476	return 0;
1456	1477	}
1457	1478
1458		−static void
1459		−rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp)
	1479	+/**
	1480	+ * rpcrdma_post_recvs - Refill the Receive Queue
	1481	+ * @r_xprt: controlling transport instance
	1482	+ * @temp: mark Receive buffers to be deleted after use
	1483	+ *
	1484	+ */
	1485	+void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp)
1460	1486	{
1461	1487	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
1462	1488	struct rpcrdma_ep *ep = &r_xprt->rx_ep;

net/sunrpc/xprtrdma/xprt_rdma.h+6−12

@@ -218,12 +218,8 @@ enum { · 217 unmodified lines
218	218	/* struct rpcrdma_sendctx - DMA mapped SGEs to unmap after Send completes
219	219	*/
220	220	struct rpcrdma_req;
221		−struct rpcrdma_xprt;
222	221	struct rpcrdma_sendctx {
223		− struct ib_send_wr sc_wr;
224	222	struct ib_cqe sc_cqe;
225		− struct ib_device *sc_device;
226		− struct rpcrdma_xprt *sc_xprt;
227	223	struct rpcrdma_req *sc_req;
228	224	unsigned int sc_unmap_count;
229	225	struct ib_sge sc_sges[];
@@ -257,7 +253,6 @@ struct rpcrdma_mr { · 27 unmodified lines
257	253	u32 mr_handle;
258	254	u32 mr_length;
259	255	u64 mr_offset;
260		− struct work_struct mr_recycle;
261	256	struct list_head mr_all;
262	257	};
263	258
@@ -318,6 +313,7 @@ struct rpcrdma_req { · 54 unmodified lines
318	313	struct rpcrdma_rep *rl_reply;
319	314	struct xdr_stream rl_stream;
320	315	struct xdr_buf rl_hdrbuf;
	316	+ struct ib_send_wr rl_wr;
321	317	struct rpcrdma_sendctx *rl_sendctx;
322	318	struct rpcrdma_regbuf rl_rdmabuf; / xprt header */
323	319	struct rpcrdma_regbuf rl_sendbuf; / rq_snd_buf */
@@ -474,6 +470,7 @@ void rpcrdma_ep_disconnect(struct rpcrdma_ep , struct rpcrdma_ia ); · 150 unmodified lines
474	470
475	471	int rpcrdma_ep_post(struct rpcrdma_ia , struct rpcrdma_ep ,
476	472	struct rpcrdma_req *);
	473	+void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp);
477	474
478	475	/*
479	476	* Buffer calls - xprtrdma/verbs.c
@@ -487,12 +484,7 @@ struct rpcrdma_sendctx rpcrdma_sendctx_get_locked(struct rpcrdma_xprt r_xprt); · 7 unmodified lines
487	484
488	485	struct rpcrdma_mr rpcrdma_mr_get(struct rpcrdma_xprt r_xprt);
489	486	void rpcrdma_mr_put(struct rpcrdma_mr *mr);
490		−
491		−static inline void
492		−rpcrdma_mr_recycle(struct rpcrdma_mr *mr)
493		−{
494		− schedule_work(&mr->mr_recycle);
495		−}
	487	+void rpcrdma_mrs_refresh(struct rpcrdma_xprt *r_xprt);
496	488
497	489	struct rpcrdma_req rpcrdma_buffer_get(struct rpcrdma_buffer );
498	490	void rpcrdma_buffer_put(struct rpcrdma_buffer *buffers,
@@ -542,7 +534,6 @@ rpcrdma_data_dir(bool writing) · 43 unmodified lines
542	534	/* Memory registration calls xprtrdma/frwr_ops.c
543	535	*/
544	536	bool frwr_is_supported(struct ib_device *device);
545		−void frwr_recycle(struct rpcrdma_req *req);
546	537	void frwr_reset(struct rpcrdma_req *req);
547	538	int frwr_open(struct rpcrdma_ia ia, struct rpcrdma_ep ep);
548	539	int frwr_init_mr(struct rpcrdma_ia ia, struct rpcrdma_mr mr);
@@ -563,6 +554,8 @@ void frwr_unmap_async(struct rpcrdma_xprt r_xprt, struct rpcrdma_req req); · 14 unmodified lines
563	554
564	555	enum rpcrdma_chunktype {
565	556	rpcrdma_noch = 0,
	557	+ rpcrdma_noch_pullup,
	558	+ rpcrdma_noch_mapped,
566	559	rpcrdma_readch,
567	560	rpcrdma_areadch,
568	561	rpcrdma_writech,
@@ -576,6 +569,7 @@ int rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt, · 7 unmodified lines
576	569	void rpcrdma_sendctx_unmap(struct rpcrdma_sendctx *sc);
577	570	int rpcrdma_marshal_req(struct rpcrdma_xprt r_xprt, struct rpc_rqst rqst);
578	571	void rpcrdma_set_max_header_sizes(struct rpcrdma_xprt *);
	572	+void rpcrdma_reset_cwnd(struct rpcrdma_xprt *r_xprt);
579	573	void rpcrdma_complete_rqst(struct rpcrdma_rep *rep);
580	574	void rpcrdma_reply_handler(struct rpcrdma_rep *rep);
581	575

net/sunrpc/xprtsock.c+1−1

@@ -1752,7 +1752,7 @@ static void xs_set_port(struct rpc_xprt *xprt, unsigned short port) · 1751 unmodified lines
1752	1752
1753	1753	static void xs_set_srcport(struct sock_xprt transport, struct socket sock)
1754	1754	{
1755		− if (transport->srcport == 0)
	1755	+ if (transport->srcport == 0 && transport->xprt.reuseport)
1756	1756	transport->srcport = xs_sock_getport(sock);
1757	1757	}
1758	1758

Merge tag 'nfs-for-5.5-1' of git://git.linux-nfs.org/projects/trondmy/linux-nfs

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code