1565 lines · 42.8 KB · C
RawDownload
// SPDX-License-Identifier: GPL-2.0/* * The NFSD open file cache. * * (c) 2015 - Jeff Layton <jeff.layton@primarydata.com> * * An nfsd_file object is a per-file collection of open state that binds * together: *   - a struct file * *   - a user credential *   - a network namespace *   - a read-ahead context *   - monitoring for writeback errors * * nfsd_file objects are reference-counted. Consumers acquire a new * object via the nfsd_file_acquire API. They manage their interest in * the acquired object, and hence the object's reference count, via * nfsd_file_get and nfsd_file_put. There are two varieties of nfsd_file * object: * *  * non-garbage-collected: When a consumer wants to precisely control *    the lifetime of a file's open state, it acquires a non-garbage- *    collected nfsd_file. The final nfsd_file_put releases the open *    state immediately. * *  * garbage-collected: When a consumer does not control the lifetime *    of open state, it acquires a garbage-collected nfsd_file. The *    final nfsd_file_put allows the open state to linger for a period *    during which it may be re-used. */ #include <linux/hash.h>#include <linux/slab.h>#include <linux/file.h>#include <linux/pagemap.h>#include <linux/sched.h>#include <linux/list_lru.h>#include <linux/fsnotify_backend.h>#include <linux/fsnotify.h>#include <linux/seq_file.h>#include <linux/rhashtable.h>#include <linux/nfslocalio.h> #include "vfs.h"#include "nfsd.h"#include "nfsfh.h"#include "netns.h"#include "filecache.h"#include "trace.h" #define NFSD_LAUNDRETTE_DELAY		     (2 * HZ) #define NFSD_FILE_CACHE_UP		     (0) /* We only care about NFSD_MAY_READ/WRITE for this cache */#define NFSD_FILE_MAY_MASK	(NFSD_MAY_READ|NFSD_MAY_WRITE|NFSD_MAY_LOCALIO) /* If the shrinker runs between calls to list_lru_walk_node() in * nfsd_file_gc(), the "remaining" count will be wrong.  This could * result in premature freeing of some files.  This may not matter much * but is easy to fix with this spinlock which temporarily disables * the shrinker. * * It also serializes callers of nfsd_file_dispose_list_delayed() * against per-net shutdown. */static DEFINE_SPINLOCK(nfsd_gc_lock); static DEFINE_PER_CPU(unsigned long, nfsd_file_cache_hits);static DEFINE_PER_CPU(unsigned long, nfsd_file_acquisitions);static DEFINE_PER_CPU(unsigned long, nfsd_file_allocations);static DEFINE_PER_CPU(unsigned long, nfsd_file_releases);static DEFINE_PER_CPU(unsigned long, nfsd_file_total_age);static DEFINE_PER_CPU(unsigned long, nfsd_file_evictions); static struct kmem_cache		*nfsd_file_slab;static struct kmem_cache		*nfsd_file_mark_slab;static struct list_lru			nfsd_file_lru;static unsigned long			nfsd_file_flags;static struct fsnotify_group		*nfsd_file_fsnotify_group;static struct fsnotify_group		*nfsd_dir_fsnotify_group;static struct delayed_work		nfsd_filecache_laundrette;static struct rhltable			nfsd_file_rhltable						____cacheline_aligned_in_smp; static boolnfsd_match_cred(const struct cred *c1, const struct cred *c2){	int i; 	if (!uid_eq(c1->fsuid, c2->fsuid))		return false;	if (!gid_eq(c1->fsgid, c2->fsgid))		return false;	if (c1->group_info == NULL || c2->group_info == NULL)		return c1->group_info == c2->group_info;	if (c1->group_info->ngroups != c2->group_info->ngroups)		return false;	for (i = 0; i < c1->group_info->ngroups; i++) {		if (!gid_eq(c1->group_info->gid[i], c2->group_info->gid[i]))			return false;	}	return true;} static const struct rhashtable_params nfsd_file_rhash_params = {	.key_len		= sizeof_field(struct nfsd_file, nf_inode),	.key_offset		= offsetof(struct nfsd_file, nf_inode),	.head_offset		= offsetof(struct nfsd_file, nf_rlist), 	/*	 * Start with a single page hash table to reduce resizing churn	 * on light workloads.	 */	.min_size		= 256,	.automatic_shrinking	= true,}; static voidnfsd_file_schedule_laundrette(void){	if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags))		queue_delayed_work(system_dfl_wq, &nfsd_filecache_laundrette,				   NFSD_LAUNDRETTE_DELAY);} static voidnfsd_file_slab_free(struct rcu_head *rcu){	struct nfsd_file *nf = container_of(rcu, struct nfsd_file, nf_rcu); 	put_cred(nf->nf_cred);	kmem_cache_free(nfsd_file_slab, nf);} static voidnfsd_file_mark_free(struct fsnotify_mark *mark){	struct nfsd_file_mark *nfm = container_of(mark, struct nfsd_file_mark,						  nfm_mark); 	kmem_cache_free(nfsd_file_mark_slab, nfm);} static struct nfsd_file_mark *nfsd_file_mark_get(struct nfsd_file_mark *nfm){	if (!refcount_inc_not_zero(&nfm->nfm_ref))		return NULL;	return nfm;} static voidnfsd_file_mark_put(struct nfsd_file_mark *nfm){	if (refcount_dec_and_test(&nfm->nfm_ref)) {		fsnotify_destroy_mark(&nfm->nfm_mark, nfm->nfm_mark.group);		fsnotify_put_mark(&nfm->nfm_mark);	}} static struct nfsd_file_mark *nfsd_file_mark_find_or_create(struct inode *inode){	struct nfsd_file_mark	*nfm = NULL, *new;	struct fsnotify_group	*group;	struct fsnotify_mark	*mark;	int			err; 	group = S_ISDIR(inode->i_mode) ? nfsd_dir_fsnotify_group : nfsd_file_fsnotify_group; 	do {		fsnotify_group_lock(group);		mark = fsnotify_find_inode_mark(inode, group);		if (mark) {			nfm = nfsd_file_mark_get(container_of(mark,						 struct nfsd_file_mark,						 nfm_mark));			fsnotify_group_unlock(group);			if (nfm) {				fsnotify_put_mark(mark);				break;			}			/* Avoid soft lockup race with nfsd_file_mark_put() */			fsnotify_destroy_mark(mark, group);			fsnotify_put_mark(mark);		} else {			fsnotify_group_unlock(group);		} 		/* allocate a new nfm */		new = kmem_cache_alloc(nfsd_file_mark_slab, GFP_KERNEL);		if (!new)			return NULL;		fsnotify_init_mark(&new->nfm_mark, group);		new->nfm_mark.mask = FS_ATTRIB|FS_DELETE_SELF;		refcount_set(&new->nfm_ref, 1);		mutex_init(&new->nfm_recalc_mutex); 		err = fsnotify_add_inode_mark(&new->nfm_mark, inode, 0); 		/*		 * If the add was successful, then return the object.		 * Otherwise, we need to put the reference we hold on the		 * nfm_mark. The fsnotify code will take a reference and put		 * it on failure, so we can't just free it directly. It's also		 * not safe to call fsnotify_destroy_mark on it as the		 * mark->group will be NULL. Thus, we can't let the nfm_ref		 * counter drive the destruction at this point.		 */		if (likely(!err))			nfm = new;		else			fsnotify_put_mark(&new->nfm_mark);	} while (unlikely(err == -EEXIST)); 	return nfm;} static struct nfsd_file *nfsd_file_alloc(struct net *net, struct inode *inode, unsigned char need,		bool want_gc){	struct nfsd_file *nf; 	nf = kmem_cache_alloc(nfsd_file_slab, GFP_KERNEL);	if (unlikely(!nf))		return NULL; 	this_cpu_inc(nfsd_file_allocations);	INIT_LIST_HEAD(&nf->nf_lru);	INIT_LIST_HEAD(&nf->nf_gc);	nf->nf_birthtime = ktime_get();	nf->nf_file = NULL;	nf->nf_cred = get_current_cred();	nf->nf_net = net;	nf->nf_flags = want_gc ?		BIT(NFSD_FILE_HASHED) | BIT(NFSD_FILE_PENDING) | BIT(NFSD_FILE_GC) :		BIT(NFSD_FILE_HASHED) | BIT(NFSD_FILE_PENDING);	nf->nf_inode = inode;	refcount_set(&nf->nf_ref, 1);	nf->nf_may = need;	nf->nf_mark = NULL;	nf->nf_dio_mem_align = 0;	nf->nf_dio_offset_align = 0;	nf->nf_dio_read_offset_align = 0;	return nf;} /** * nfsd_file_check_write_error - check for writeback errors on a file * @nf: nfsd_file to check for writeback errors * * Check whether a nfsd_file has an unseen error. Reset the write * verifier if so. */static voidnfsd_file_check_write_error(struct nfsd_file *nf){	struct file *file = nf->nf_file; 	if ((file->f_mode & FMODE_WRITE) &&	    filemap_check_wb_err(file->f_mapping, READ_ONCE(file->f_wb_err)))		nfsd_reset_write_verifier(net_generic(nf->nf_net, nfsd_net_id));} static voidnfsd_file_hash_remove(struct nfsd_file *nf){	trace_nfsd_file_unhash(nf);	rhltable_remove(&nfsd_file_rhltable, &nf->nf_rlist,			nfsd_file_rhash_params);} static boolnfsd_file_unhash(struct nfsd_file *nf){	if (test_and_clear_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {		nfsd_file_hash_remove(nf);		return true;	}	return false;} static voidnfsd_file_free(struct nfsd_file *nf){	s64 age = ktime_to_ms(ktime_sub(ktime_get(), nf->nf_birthtime)); 	trace_nfsd_file_free(nf); 	this_cpu_inc(nfsd_file_releases);	this_cpu_add(nfsd_file_total_age, age); 	nfsd_file_unhash(nf);	if (nf->nf_mark)		nfsd_file_mark_put(nf->nf_mark);	if (nf->nf_file) {		nfsd_file_check_write_error(nf);		nfsd_filp_close(nf->nf_file);	} 	/*	 * If this item is still linked via nf_lru, that's a bug.	 * WARN and leak it to preserve system stability.	 */	if (WARN_ON_ONCE(!list_empty(&nf->nf_lru)))		return; 	call_rcu(&nf->nf_rcu, nfsd_file_slab_free);} static boolnfsd_file_check_writeback(struct nfsd_file *nf){	struct file *file = nf->nf_file;	struct address_space *mapping; 	/* File not open for write? */	if (!(file->f_mode & FMODE_WRITE))		return false; 	/*	 * Some filesystems (e.g. NFS) flush all dirty data on close.	 * On others, there is no need to wait for writeback.	 */	if (!(file_inode(file)->i_sb->s_export_op->flags & EXPORT_OP_FLUSH_ON_CLOSE))		return false; 	mapping = file->f_mapping;	return mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) ||		mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK);} static void nfsd_file_lru_add(struct nfsd_file *nf){	refcount_inc(&nf->nf_ref);	if (list_lru_add_obj(&nfsd_file_lru, &nf->nf_lru))		trace_nfsd_file_lru_add(nf);	else {		refcount_dec(&nf->nf_ref);		WARN_ON_ONCE(1);		return;	}	nfsd_file_schedule_laundrette();} static bool nfsd_file_lru_remove(struct nfsd_file *nf){	if (list_lru_del_obj(&nfsd_file_lru, &nf->nf_lru)) {		trace_nfsd_file_lru_del(nf);		return true;	}	return false;} struct nfsd_file *nfsd_file_get(struct nfsd_file *nf){	if (nf && refcount_inc_not_zero(&nf->nf_ref))		return nf;	return NULL;} /** * nfsd_file_put - put the reference to a nfsd_file * @nf: nfsd_file of which to put the reference * * Put a reference to a nfsd_file. In the non-GC case, we just put the * reference immediately. In the GC case, if the reference would be * the last one, the put it on the LRU instead to be cleaned up later. */voidnfsd_file_put(struct nfsd_file *nf){	might_sleep();	trace_nfsd_file_put(nf); 	if (test_bit(NFSD_FILE_GC, &nf->nf_flags) &&	    test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {		set_bit(NFSD_FILE_REFERENCED, &nf->nf_flags);		set_bit(NFSD_FILE_RECENT, &nf->nf_flags);	} 	if (refcount_dec_and_test(&nf->nf_ref))		nfsd_file_free(nf);} /** * nfsd_file_put_local - put nfsd_file reference and arm nfsd_net_put in caller * @pnf: nfsd_file of which to put the reference * * First save the associated net to return to caller, then put * the reference of the nfsd_file. */struct net *nfsd_file_put_local(struct nfsd_file __rcu **pnf){	struct nfsd_file *nf;	struct net *net = NULL; 	nf = unrcu_pointer(xchg(pnf, NULL));	if (nf) {		net = nf->nf_net;		nfsd_file_put(nf);	}	return net;} /** * nfsd_file_file - get the backing file of an nfsd_file * @nf: nfsd_file of which to access the backing file. * * Return backing file for @nf. */struct file *nfsd_file_file(struct nfsd_file *nf){	return nf->nf_file;} static voidnfsd_file_dispose_list(struct list_head *dispose){	struct nfsd_file *nf; 	while (!list_empty(dispose)) {		nf = list_first_entry(dispose, struct nfsd_file, nf_gc);		list_del_init(&nf->nf_gc);		nfsd_file_free(nf);	}} /** * nfsd_file_dispose_list_delayed - queue dead files for nfsd thread disposal * @dispose: list of nfsd_files to be disposed * * Transfers each file to the dispose list in its nfsd_net and wakes an nfsd * thread to do the actual close.  This keeps the cost of fput() in the nfsd * threads rather than in the shrinker or GC worker. * * All callers must hold nfsd_gc_lock, so that nfsd_file_cache_shutdown_net() * can synchronize against them before draining the per-net dispose list. * This guarantees nf_net is still live when we call net_generic(). */static voidnfsd_file_dispose_list_delayed(struct list_head *dispose){	lockdep_assert_held(&nfsd_gc_lock); 	while (!list_empty(dispose)) {		struct nfsd_file *nf = list_first_entry(dispose,						struct nfsd_file, nf_gc);		struct nfsd_net *nn = net_generic(nf->nf_net, nfsd_net_id);		struct svc_serv *serv; 		spin_lock(&nn->fcache_dispose_lock);		list_move_tail(&nf->nf_gc, &nn->fcache_dispose_list);		spin_unlock(&nn->fcache_dispose_lock); 		/*		 * The filecache laundrette is shut down after the		 * nn->nfsd_serv pointer is cleared, but before the		 * svc_serv is freed.		 */		serv = nn->nfsd_serv;		if (serv)			svc_wake_up(serv);	}} /** * nfsd_file_net_dispose - deal with nfsd_files waiting to be disposed. * @nn: nfsd_net in which to find files to be disposed. * * When files held open for nfsv3 are removed from the filecache, whether * due to memory pressure or garbage collection, they are queued to * a per-net-ns queue.  This function completes the disposal, either * directly or by waking another nfsd thread to help with the work. */void nfsd_file_net_dispose(struct nfsd_net *nn){	if (!list_empty(&nn->fcache_dispose_list)) {		LIST_HEAD(dispose);		int i; 		spin_lock(&nn->fcache_dispose_lock);		for (i = 0; i < 8 && !list_empty(&nn->fcache_dispose_list); i++)			list_move(nn->fcache_dispose_list.next, &dispose);		spin_unlock(&nn->fcache_dispose_lock);		if (!list_empty(&nn->fcache_dispose_list)) {			/*			 * Wake up another thread to share the work			 * *before* doing any actual disposing.			 *			 * The filecache laundrette is shut down after			 * the nn->nfsd_serv pointer is cleared, but			 * before the svc_serv is freed.			 */			struct svc_serv *serv = nn->nfsd_serv;

Lines 1–500 of 1565.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t