// SPDX-License-Identifier: GPL-2.0/* * The NFSD open file cache. * * (c) 2015 - Jeff Layton <jeff.layton@primarydata.com> * * An nfsd_file object is a per-file collection of open state that binds * together: * - a struct file * * - a user credential * - a network namespace * - a read-ahead context * - monitoring for writeback errors * * nfsd_file objects are reference-counted. Consumers acquire a new * object via the nfsd_file_acquire API. They manage their interest in * the acquired object, and hence the object's reference count, via * nfsd_file_get and nfsd_file_put. There are two varieties of nfsd_file * object: * * * non-garbage-collected: When a consumer wants to precisely control * the lifetime of a file's open state, it acquires a non-garbage- * collected nfsd_file. The final nfsd_file_put releases the open * state immediately. * * * garbage-collected: When a consumer does not control the lifetime * of open state, it acquires a garbage-collected nfsd_file. The * final nfsd_file_put allows the open state to linger for a period * during which it may be re-used. */ #include <linux/hash.h>#include <linux/slab.h>#include <linux/file.h>#include <linux/pagemap.h>#include <linux/sched.h>#include <linux/list_lru.h>#include <linux/fsnotify_backend.h>#include <linux/fsnotify.h>#include <linux/seq_file.h>#include <linux/rhashtable.h>#include <linux/nfslocalio.h> #include "vfs.h"#include "nfsd.h"#include "nfsfh.h"#include "netns.h"#include "filecache.h"#include "trace.h" #define NFSD_LAUNDRETTE_DELAY (2 * HZ) #define NFSD_FILE_CACHE_UP (0) /* We only care about NFSD_MAY_READ/WRITE for this cache */#define NFSD_FILE_MAY_MASK (NFSD_MAY_READ|NFSD_MAY_WRITE|NFSD_MAY_LOCALIO) /* If the shrinker runs between calls to list_lru_walk_node() in * nfsd_file_gc(), the "remaining" count will be wrong. This could * result in premature freeing of some files. This may not matter much * but is easy to fix with this spinlock which temporarily disables * the shrinker. * * It also serializes callers of nfsd_file_dispose_list_delayed() * against per-net shutdown. */static DEFINE_SPINLOCK(nfsd_gc_lock); static DEFINE_PER_CPU(unsigned long, nfsd_file_cache_hits);static DEFINE_PER_CPU(unsigned long, nfsd_file_acquisitions);static DEFINE_PER_CPU(unsigned long, nfsd_file_allocations);static DEFINE_PER_CPU(unsigned long, nfsd_file_releases);static DEFINE_PER_CPU(unsigned long, nfsd_file_total_age);static DEFINE_PER_CPU(unsigned long, nfsd_file_evictions); static struct kmem_cache *nfsd_file_slab;static struct kmem_cache *nfsd_file_mark_slab;static struct list_lru nfsd_file_lru;static unsigned long nfsd_file_flags;static struct fsnotify_group *nfsd_file_fsnotify_group;static struct fsnotify_group *nfsd_dir_fsnotify_group;static struct delayed_work nfsd_filecache_laundrette;static struct rhltable nfsd_file_rhltable ____cacheline_aligned_in_smp; static boolnfsd_match_cred(const struct cred *c1, const struct cred *c2){ int i; if (!uid_eq(c1->fsuid, c2->fsuid)) return false; if (!gid_eq(c1->fsgid, c2->fsgid)) return false; if (c1->group_info == NULL || c2->group_info == NULL) return c1->group_info == c2->group_info; if (c1->group_info->ngroups != c2->group_info->ngroups) return false; for (i = 0; i < c1->group_info->ngroups; i++) { if (!gid_eq(c1->group_info->gid[i], c2->group_info->gid[i])) return false; } return true;} static const struct rhashtable_params nfsd_file_rhash_params = { .key_len = sizeof_field(struct nfsd_file, nf_inode), .key_offset = offsetof(struct nfsd_file, nf_inode), .head_offset = offsetof(struct nfsd_file, nf_rlist), /* * Start with a single page hash table to reduce resizing churn * on light workloads. */ .min_size = 256, .automatic_shrinking = true,}; static voidnfsd_file_schedule_laundrette(void){ if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags)) queue_delayed_work(system_dfl_wq, &nfsd_filecache_laundrette, NFSD_LAUNDRETTE_DELAY);} static voidnfsd_file_slab_free(struct rcu_head *rcu){ struct nfsd_file *nf = container_of(rcu, struct nfsd_file, nf_rcu); put_cred(nf->nf_cred); kmem_cache_free(nfsd_file_slab, nf);} static voidnfsd_file_mark_free(struct fsnotify_mark *mark){ struct nfsd_file_mark *nfm = container_of(mark, struct nfsd_file_mark, nfm_mark); kmem_cache_free(nfsd_file_mark_slab, nfm);} static struct nfsd_file_mark *nfsd_file_mark_get(struct nfsd_file_mark *nfm){ if (!refcount_inc_not_zero(&nfm->nfm_ref)) return NULL; return nfm;} static voidnfsd_file_mark_put(struct nfsd_file_mark *nfm){ if (refcount_dec_and_test(&nfm->nfm_ref)) { fsnotify_destroy_mark(&nfm->nfm_mark, nfm->nfm_mark.group); fsnotify_put_mark(&nfm->nfm_mark); }} static struct nfsd_file_mark *nfsd_file_mark_find_or_create(struct inode *inode){ struct nfsd_file_mark *nfm = NULL, *new; struct fsnotify_group *group; struct fsnotify_mark *mark; int err; group = S_ISDIR(inode->i_mode) ? nfsd_dir_fsnotify_group : nfsd_file_fsnotify_group; do { fsnotify_group_lock(group); mark = fsnotify_find_inode_mark(inode, group); if (mark) { nfm = nfsd_file_mark_get(container_of(mark, struct nfsd_file_mark, nfm_mark)); fsnotify_group_unlock(group); if (nfm) { fsnotify_put_mark(mark); break; } /* Avoid soft lockup race with nfsd_file_mark_put() */ fsnotify_destroy_mark(mark, group); fsnotify_put_mark(mark); } else { fsnotify_group_unlock(group); } /* allocate a new nfm */ new = kmem_cache_alloc(nfsd_file_mark_slab, GFP_KERNEL); if (!new) return NULL; fsnotify_init_mark(&new->nfm_mark, group); new->nfm_mark.mask = FS_ATTRIB|FS_DELETE_SELF; refcount_set(&new->nfm_ref, 1); mutex_init(&new->nfm_recalc_mutex); err = fsnotify_add_inode_mark(&new->nfm_mark, inode, 0); /* * If the add was successful, then return the object. * Otherwise, we need to put the reference we hold on the * nfm_mark. The fsnotify code will take a reference and put * it on failure, so we can't just free it directly. It's also * not safe to call fsnotify_destroy_mark on it as the * mark->group will be NULL. Thus, we can't let the nfm_ref * counter drive the destruction at this point. */ if (likely(!err)) nfm = new; else fsnotify_put_mark(&new->nfm_mark); } while (unlikely(err == -EEXIST)); return nfm;} static struct nfsd_file *nfsd_file_alloc(struct net *net, struct inode *inode, unsigned char need, bool want_gc){ struct nfsd_file *nf; nf = kmem_cache_alloc(nfsd_file_slab, GFP_KERNEL); if (unlikely(!nf)) return NULL; this_cpu_inc(nfsd_file_allocations); INIT_LIST_HEAD(&nf->nf_lru); INIT_LIST_HEAD(&nf->nf_gc); nf->nf_birthtime = ktime_get(); nf->nf_file = NULL; nf->nf_cred = get_current_cred(); nf->nf_net = net; nf->nf_flags = want_gc ? BIT(NFSD_FILE_HASHED) | BIT(NFSD_FILE_PENDING) | BIT(NFSD_FILE_GC) : BIT(NFSD_FILE_HASHED) | BIT(NFSD_FILE_PENDING); nf->nf_inode = inode; refcount_set(&nf->nf_ref, 1); nf->nf_may = need; nf->nf_mark = NULL; nf->nf_dio_mem_align = 0; nf->nf_dio_offset_align = 0; nf->nf_dio_read_offset_align = 0; return nf;} /** * nfsd_file_check_write_error - check for writeback errors on a file * @nf: nfsd_file to check for writeback errors * * Check whether a nfsd_file has an unseen error. Reset the write * verifier if so. */static voidnfsd_file_check_write_error(struct nfsd_file *nf){ struct file *file = nf->nf_file; if ((file->f_mode & FMODE_WRITE) && filemap_check_wb_err(file->f_mapping, READ_ONCE(file->f_wb_err))) nfsd_reset_write_verifier(net_generic(nf->nf_net, nfsd_net_id));} static voidnfsd_file_hash_remove(struct nfsd_file *nf){ trace_nfsd_file_unhash(nf); rhltable_remove(&nfsd_file_rhltable, &nf->nf_rlist, nfsd_file_rhash_params);} static boolnfsd_file_unhash(struct nfsd_file *nf){ if (test_and_clear_bit(NFSD_FILE_HASHED, &nf->nf_flags)) { nfsd_file_hash_remove(nf); return true; } return false;} static voidnfsd_file_free(struct nfsd_file *nf){ s64 age = ktime_to_ms(ktime_sub(ktime_get(), nf->nf_birthtime)); trace_nfsd_file_free(nf); this_cpu_inc(nfsd_file_releases); this_cpu_add(nfsd_file_total_age, age); nfsd_file_unhash(nf); if (nf->nf_mark) nfsd_file_mark_put(nf->nf_mark); if (nf->nf_file) { nfsd_file_check_write_error(nf); nfsd_filp_close(nf->nf_file); } /* * If this item is still linked via nf_lru, that's a bug. * WARN and leak it to preserve system stability. */ if (WARN_ON_ONCE(!list_empty(&nf->nf_lru))) return; call_rcu(&nf->nf_rcu, nfsd_file_slab_free);} static boolnfsd_file_check_writeback(struct nfsd_file *nf){ struct file *file = nf->nf_file; struct address_space *mapping; /* File not open for write? */ if (!(file->f_mode & FMODE_WRITE)) return false; /* * Some filesystems (e.g. NFS) flush all dirty data on close. * On others, there is no need to wait for writeback. */ if (!(file_inode(file)->i_sb->s_export_op->flags & EXPORT_OP_FLUSH_ON_CLOSE)) return false; mapping = file->f_mapping; return mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) || mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK);} static void nfsd_file_lru_add(struct nfsd_file *nf){ refcount_inc(&nf->nf_ref); if (list_lru_add_obj(&nfsd_file_lru, &nf->nf_lru)) trace_nfsd_file_lru_add(nf); else { refcount_dec(&nf->nf_ref); WARN_ON_ONCE(1); return; } nfsd_file_schedule_laundrette();} static bool nfsd_file_lru_remove(struct nfsd_file *nf){ if (list_lru_del_obj(&nfsd_file_lru, &nf->nf_lru)) { trace_nfsd_file_lru_del(nf); return true; } return false;} struct nfsd_file *nfsd_file_get(struct nfsd_file *nf){ if (nf && refcount_inc_not_zero(&nf->nf_ref)) return nf; return NULL;} /** * nfsd_file_put - put the reference to a nfsd_file * @nf: nfsd_file of which to put the reference * * Put a reference to a nfsd_file. In the non-GC case, we just put the * reference immediately. In the GC case, if the reference would be * the last one, the put it on the LRU instead to be cleaned up later. */voidnfsd_file_put(struct nfsd_file *nf){ might_sleep(); trace_nfsd_file_put(nf); if (test_bit(NFSD_FILE_GC, &nf->nf_flags) && test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) { set_bit(NFSD_FILE_REFERENCED, &nf->nf_flags); set_bit(NFSD_FILE_RECENT, &nf->nf_flags); } if (refcount_dec_and_test(&nf->nf_ref)) nfsd_file_free(nf);} /** * nfsd_file_put_local - put nfsd_file reference and arm nfsd_net_put in caller * @pnf: nfsd_file of which to put the reference * * First save the associated net to return to caller, then put * the reference of the nfsd_file. */struct net *nfsd_file_put_local(struct nfsd_file __rcu **pnf){ struct nfsd_file *nf; struct net *net = NULL; nf = unrcu_pointer(xchg(pnf, NULL)); if (nf) { net = nf->nf_net; nfsd_file_put(nf); } return net;} /** * nfsd_file_file - get the backing file of an nfsd_file * @nf: nfsd_file of which to access the backing file. * * Return backing file for @nf. */struct file *nfsd_file_file(struct nfsd_file *nf){ return nf->nf_file;} static voidnfsd_file_dispose_list(struct list_head *dispose){ struct nfsd_file *nf; while (!list_empty(dispose)) { nf = list_first_entry(dispose, struct nfsd_file, nf_gc); list_del_init(&nf->nf_gc); nfsd_file_free(nf); }} /** * nfsd_file_dispose_list_delayed - queue dead files for nfsd thread disposal * @dispose: list of nfsd_files to be disposed * * Transfers each file to the dispose list in its nfsd_net and wakes an nfsd * thread to do the actual close. This keeps the cost of fput() in the nfsd * threads rather than in the shrinker or GC worker. * * All callers must hold nfsd_gc_lock, so that nfsd_file_cache_shutdown_net() * can synchronize against them before draining the per-net dispose list. * This guarantees nf_net is still live when we call net_generic(). */static voidnfsd_file_dispose_list_delayed(struct list_head *dispose){ lockdep_assert_held(&nfsd_gc_lock); while (!list_empty(dispose)) { struct nfsd_file *nf = list_first_entry(dispose, struct nfsd_file, nf_gc); struct nfsd_net *nn = net_generic(nf->nf_net, nfsd_net_id); struct svc_serv *serv; spin_lock(&nn->fcache_dispose_lock); list_move_tail(&nf->nf_gc, &nn->fcache_dispose_list); spin_unlock(&nn->fcache_dispose_lock); /* * The filecache laundrette is shut down after the * nn->nfsd_serv pointer is cleared, but before the * svc_serv is freed. */ serv = nn->nfsd_serv; if (serv) svc_wake_up(serv); }} /** * nfsd_file_net_dispose - deal with nfsd_files waiting to be disposed. * @nn: nfsd_net in which to find files to be disposed. * * When files held open for nfsv3 are removed from the filecache, whether * due to memory pressure or garbage collection, they are queued to * a per-net-ns queue. This function completes the disposal, either * directly or by waking another nfsd thread to help with the work. */void nfsd_file_net_dispose(struct nfsd_net *nn){ if (!list_empty(&nn->fcache_dispose_list)) { LIST_HEAD(dispose); int i; spin_lock(&nn->fcache_dispose_lock); for (i = 0; i < 8 && !list_empty(&nn->fcache_dispose_list); i++) list_move(nn->fcache_dispose_list.next, &dispose); spin_unlock(&nn->fcache_dispose_lock); if (!list_empty(&nn->fcache_dispose_list)) { /* * Wake up another thread to share the work * *before* doing any actual disposing. * * The filecache laundrette is shut down after * the nn->nfsd_serv pointer is cleared, but * before the svc_serv is freed. */ struct svc_serv *serv = nn->nfsd_serv;Lines 1–500 of 1565. Load more
// SPDX-License-Identifier: GPL-2.0 /* * The NFSD open file cache. * * (c) 2015 - Jeff Layton <jeff.layton@primarydata.com> * * An nfsd_file object is a per-file collection of open state that binds * together: * - a struct file * * - a user credential * - a network namespace * - a read-ahead context * - monitoring for writeback errors * * nfsd_file objects are reference-counted. Consumers acquire a new * object via the nfsd_file_acquire API. They manage their interest in * the acquired object, and hence the object's reference count, via * nfsd_file_get and nfsd_file_put. There are two varieties of nfsd_file * object: * * * non-garbage-collected: When a consumer wants to precisely control * the lifetime of a file's open state, it acquires a non-garbage- * collected nfsd_file. The final nfsd_file_put releases the open * state immediately. * * * garbage-collected: When a consumer does not control the lifetime * of open state, it acquires a garbage-collected nfsd_file. The * final nfsd_file_put allows the open state to linger for a period * during which it may be re-used. */ #include <linux/hash.h> #include <linux/slab.h> #include <linux/file.h> #include <linux/pagemap.h> #include <linux/sched.h> #include <linux/list_lru.h> #include <linux/fsnotify_backend.h> #include <linux/fsnotify.h> #include <linux/seq_file.h> #include <linux/rhashtable.h> #include <linux/nfslocalio.h> #include "vfs.h" #include "nfsd.h" #include "nfsfh.h" #include "netns.h" #include "filecache.h" #include "trace.h" #define NFSD_LAUNDRETTE_DELAY (2 * HZ) #define NFSD_FILE_CACHE_UP (0) /* We only care about NFSD_MAY_READ/WRITE for this cache */ #define NFSD_FILE_MAY_MASK (NFSD_MAY_READ|NFSD_MAY_WRITE|NFSD_MAY_LOCALIO) /* If the shrinker runs between calls to list_lru_walk_node() in * nfsd_file_gc(), the "remaining" count will be wrong. This could * result in premature freeing of some files. This may not matter much * but is easy to fix with this spinlock which temporarily disables * the shrinker. * * It also serializes callers of nfsd_file_dispose_list_delayed() * against per-net shutdown. */ static DEFINE_SPINLOCK(nfsd_gc_lock); static DEFINE_PER_CPU(unsigned long, nfsd_file_cache_hits); static DEFINE_PER_CPU(unsigned long, nfsd_file_acquisitions); static DEFINE_PER_CPU(unsigned long, nfsd_file_allocations); static DEFINE_PER_CPU(unsigned long, nfsd_file_releases); static DEFINE_PER_CPU(unsigned long, nfsd_file_total_age); static DEFINE_PER_CPU(unsigned long, nfsd_file_evictions); static struct kmem_cache *nfsd_file_slab; static struct kmem_cache *nfsd_file_mark_slab; static struct list_lru nfsd_file_lru; static unsigned long nfsd_file_flags; static struct fsnotify_group *nfsd_file_fsnotify_group; static struct fsnotify_group *nfsd_dir_fsnotify_group; static struct delayed_work nfsd_filecache_laundrette; static struct rhltable nfsd_file_rhltable ____cacheline_aligned_in_smp; static bool nfsd_match_cred(const struct cred *c1, const struct cred *c2) { int i; if (!uid_eq(c1->fsuid, c2->fsuid)) return false; if (!gid_eq(c1->fsgid, c2->fsgid)) return false; if (c1->group_info == NULL || c2->group_info == NULL) return c1->group_info == c2->group_info; if (c1->group_info->ngroups != c2->group_info->ngroups) return false; for (i = 0; i < c1->group_info->ngroups; i++) { if (!gid_eq(c1->group_info->gid[i], c2->group_info->gid[i])) return false; } return true; } static const struct rhashtable_params nfsd_file_rhash_params = { .key_len = sizeof_field(struct nfsd_file, nf_inode), .key_offset = offsetof(struct nfsd_file, nf_inode), .head_offset = offsetof(struct nfsd_file, nf_rlist), /* * Start with a single page hash table to reduce resizing churn * on light workloads. */ .min_size = 256, .automatic_shrinking = true, }; static void nfsd_file_schedule_laundrette(void) { if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags)) queue_delayed_work(system_dfl_wq, &nfsd_filecache_laundrette, NFSD_LAUNDRETTE_DELAY); } static void nfsd_file_slab_free(struct rcu_head *rcu) { struct nfsd_file *nf = container_of(rcu, struct nfsd_file, nf_rcu); put_cred(nf->nf_cred); kmem_cache_free(nfsd_file_slab, nf); } static void nfsd_file_mark_free(struct fsnotify_mark *mark) { struct nfsd_file_mark *nfm = container_of(mark, struct nfsd_file_mark, nfm_mark); kmem_cache_free(nfsd_file_mark_slab, nfm); } static struct nfsd_file_mark * nfsd_file_mark_get(struct nfsd_file_mark *nfm) { if (!refcount_inc_not_zero(&nfm->nfm_ref)) return NULL; return nfm; } static void nfsd_file_mark_put(struct nfsd_file_mark *nfm) { if (refcount_dec_and_test(&nfm->nfm_ref)) { fsnotify_destroy_mark(&nfm->nfm_mark, nfm->nfm_mark.group); fsnotify_put_mark(&nfm->nfm_mark); } } static struct nfsd_file_mark * nfsd_file_mark_find_or_create(struct inode *inode) { struct nfsd_file_mark *nfm = NULL, *new; struct fsnotify_group *group; struct fsnotify_mark *mark; int err; group = S_ISDIR(inode->i_mode) ? nfsd_dir_fsnotify_group : nfsd_file_fsnotify_group; do { fsnotify_group_lock(group); mark = fsnotify_find_inode_mark(inode, group); if (mark) { nfm = nfsd_file_mark_get(container_of(mark, struct nfsd_file_mark, nfm_mark)); fsnotify_group_unlock(group); if (nfm) { fsnotify_put_mark(mark); break; } /* Avoid soft lockup race with nfsd_file_mark_put() */ fsnotify_destroy_mark(mark, group); fsnotify_put_mark(mark); } else { fsnotify_group_unlock(group); } /* allocate a new nfm */ new = kmem_cache_alloc(nfsd_file_mark_slab, GFP_KERNEL); if (!new) return NULL; fsnotify_init_mark(&new->nfm_mark, group); new->nfm_mark.mask = FS_ATTRIB|FS_DELETE_SELF; refcount_set(&new->nfm_ref, 1); mutex_init(&new->nfm_recalc_mutex); err = fsnotify_add_inode_mark(&new->nfm_mark, inode, 0); /* * If the add was successful, then return the object. * Otherwise, we need to put the reference we hold on the * nfm_mark. The fsnotify code will take a reference and put * it on failure, so we can't just free it directly. It's also * not safe to call fsnotify_destroy_mark on it as the * mark->group will be NULL. Thus, we can't let the nfm_ref * counter drive the destruction at this point. */ if (likely(!err)) nfm = new; else fsnotify_put_mark(&new->nfm_mark); } while (unlikely(err == -EEXIST)); return nfm; } static struct nfsd_file * nfsd_file_alloc(struct net *net, struct inode *inode, unsigned char need, bool want_gc) { struct nfsd_file *nf; nf = kmem_cache_alloc(nfsd_file_slab, GFP_KERNEL); if (unlikely(!nf)) return NULL; this_cpu_inc(nfsd_file_allocations); INIT_LIST_HEAD(&nf->nf_lru); INIT_LIST_HEAD(&nf->nf_gc); nf->nf_birthtime = ktime_get(); nf->nf_file = NULL; nf->nf_cred = get_current_cred(); nf->nf_net = net; nf->nf_flags = want_gc ? BIT(NFSD_FILE_HASHED) | BIT(NFSD_FILE_PENDING) | BIT(NFSD_FILE_GC) : BIT(NFSD_FILE_HASHED) | BIT(NFSD_FILE_PENDING); nf->nf_inode = inode; refcount_set(&nf->nf_ref, 1); nf->nf_may = need; nf->nf_mark = NULL; nf->nf_dio_mem_align = 0; nf->nf_dio_offset_align = 0; nf->nf_dio_read_offset_align = 0; return nf; } /** * nfsd_file_check_write_error - check for writeback errors on a file * @nf: nfsd_file to check for writeback errors * * Check whether a nfsd_file has an unseen error. Reset the write * verifier if so. */ static void nfsd_file_check_write_error(struct nfsd_file *nf) { struct file *file = nf->nf_file; if ((file->f_mode & FMODE_WRITE) && filemap_check_wb_err(file->f_mapping, READ_ONCE(file->f_wb_err))) nfsd_reset_write_verifier(net_generic(nf->nf_net, nfsd_net_id)); } static void nfsd_file_hash_remove(struct nfsd_file *nf) { trace_nfsd_file_unhash(nf); rhltable_remove(&nfsd_file_rhltable, &nf->nf_rlist, nfsd_file_rhash_params); } static bool nfsd_file_unhash(struct nfsd_file *nf) { if (test_and_clear_bit(NFSD_FILE_HASHED, &nf->nf_flags)) { nfsd_file_hash_remove(nf); return true; } return false; } static void nfsd_file_free(struct nfsd_file *nf) { s64 age = ktime_to_ms(ktime_sub(ktime_get(), nf->nf_birthtime)); trace_nfsd_file_free(nf); this_cpu_inc(nfsd_file_releases); this_cpu_add(nfsd_file_total_age, age); nfsd_file_unhash(nf); if (nf->nf_mark) nfsd_file_mark_put(nf->nf_mark); if (nf->nf_file) { nfsd_file_check_write_error(nf); nfsd_filp_close(nf->nf_file); } /* * If this item is still linked via nf_lru, that's a bug. * WARN and leak it to preserve system stability. */ if (WARN_ON_ONCE(!list_empty(&nf->nf_lru))) return; call_rcu(&nf->nf_rcu, nfsd_file_slab_free); } static bool nfsd_file_check_writeback(struct nfsd_file *nf) { struct file *file = nf->nf_file; struct address_space *mapping; /* File not open for write? */ if (!(file->f_mode & FMODE_WRITE)) return false; /* * Some filesystems (e.g. NFS) flush all dirty data on close. * On others, there is no need to wait for writeback. */ if (!(file_inode(file)->i_sb->s_export_op->flags & EXPORT_OP_FLUSH_ON_CLOSE)) return false; mapping = file->f_mapping; return mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) || mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK); } static void nfsd_file_lru_add(struct nfsd_file *nf) { refcount_inc(&nf->nf_ref); if (list_lru_add_obj(&nfsd_file_lru, &nf->nf_lru)) trace_nfsd_file_lru_add(nf); else { refcount_dec(&nf->nf_ref); WARN_ON_ONCE(1); return; } nfsd_file_schedule_laundrette(); } static bool nfsd_file_lru_remove(struct nfsd_file *nf) { if (list_lru_del_obj(&nfsd_file_lru, &nf->nf_lru)) { trace_nfsd_file_lru_del(nf); return true; } return false; } struct nfsd_file * nfsd_file_get(struct nfsd_file *nf) { if (nf && refcount_inc_not_zero(&nf->nf_ref)) return nf; return NULL; } /** * nfsd_file_put - put the reference to a nfsd_file * @nf: nfsd_file of which to put the reference * * Put a reference to a nfsd_file. In the non-GC case, we just put the * reference immediately. In the GC case, if the reference would be * the last one, the put it on the LRU instead to be cleaned up later. */ void nfsd_file_put(struct nfsd_file *nf) { might_sleep(); trace_nfsd_file_put(nf); if (test_bit(NFSD_FILE_GC, &nf->nf_flags) && test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) { set_bit(NFSD_FILE_REFERENCED, &nf->nf_flags); set_bit(NFSD_FILE_RECENT, &nf->nf_flags); } if (refcount_dec_and_test(&nf->nf_ref)) nfsd_file_free(nf); } /** * nfsd_file_put_local - put nfsd_file reference and arm nfsd_net_put in caller * @pnf: nfsd_file of which to put the reference * * First save the associated net to return to caller, then put * the reference of the nfsd_file. */ struct net * nfsd_file_put_local(struct nfsd_file __rcu **pnf) { struct nfsd_file *nf; struct net *net = NULL; nf = unrcu_pointer(xchg(pnf, NULL)); if (nf) { net = nf->nf_net; nfsd_file_put(nf); } return net; } /** * nfsd_file_file - get the backing file of an nfsd_file * @nf: nfsd_file of which to access the backing file. * * Return backing file for @nf. */ struct file * nfsd_file_file(struct nfsd_file *nf) { return nf->nf_file; } static void nfsd_file_dispose_list(struct list_head *dispose) { struct nfsd_file *nf; while (!list_empty(dispose)) { nf = list_first_entry(dispose, struct nfsd_file, nf_gc); list_del_init(&nf->nf_gc); nfsd_file_free(nf); } } /** * nfsd_file_dispose_list_delayed - queue dead files for nfsd thread disposal * @dispose: list of nfsd_files to be disposed * * Transfers each file to the dispose list in its nfsd_net and wakes an nfsd * thread to do the actual close. This keeps the cost of fput() in the nfsd * threads rather than in the shrinker or GC worker. * * All callers must hold nfsd_gc_lock, so that nfsd_file_cache_shutdown_net() * can synchronize against them before draining the per-net dispose list. * This guarantees nf_net is still live when we call net_generic(). */ static void nfsd_file_dispose_list_delayed(struct list_head *dispose) { lockdep_assert_held(&nfsd_gc_lock); while (!list_empty(dispose)) { struct nfsd_file *nf = list_first_entry(dispose, struct nfsd_file, nf_gc); struct nfsd_net *nn = net_generic(nf->nf_net, nfsd_net_id); struct svc_serv *serv; spin_lock(&nn->fcache_dispose_lock); list_move_tail(&nf->nf_gc, &nn->fcache_dispose_list); spin_unlock(&nn->fcache_dispose_lock); /* * The filecache laundrette is shut down after the * nn->nfsd_serv pointer is cleared, but before the * svc_serv is freed. */ serv = nn->nfsd_serv; if (serv) svc_wake_up(serv); } } /** * nfsd_file_net_dispose - deal with nfsd_files waiting to be disposed. * @nn: nfsd_net in which to find files to be disposed. * * When files held open for nfsv3 are removed from the filecache, whether * due to memory pressure or garbage collection, they are queued to * a per-net-ns queue. This function completes the disposal, either * directly or by waking another nfsd thread to help with the work. */ void nfsd_file_net_dispose(struct nfsd_net *nn) { if (!list_empty(&nn->fcache_dispose_list)) { LIST_HEAD(dispose); int i; spin_lock(&nn->fcache_dispose_lock); for (i = 0; i < 8 && !list_empty(&nn->fcache_dispose_list); i++) list_move(nn->fcache_dispose_list.next, &dispose); spin_unlock(&nn->fcache_dispose_lock); if (!list_empty(&nn->fcache_dispose_list)) { /* * Wake up another thread to share the work * *before* doing any actual disposing. * * The filecache laundrette is shut down after * the nn->nfsd_serv pointer is cleared, but * before the svc_serv is freed. */ struct svc_serv *serv = nn->nfsd_serv;