// SPDX-License-Identifier: GPL-2.0-only/* * linux/fs/nfs/write.c * * Write file data over NFS. * * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de> */ #include <linux/types.h>#include <linux/slab.h>#include <linux/mm.h>#include <linux/pagemap.h>#include <linux/file.h>#include <linux/writeback.h>#include <linux/swap.h>#include <linux/migrate.h> #include <linux/sunrpc/clnt.h>#include <linux/nfs_fs.h>#include <linux/nfs_mount.h>#include <linux/nfs_page.h>#include <linux/backing-dev.h>#include <linux/export.h>#include <linux/freezer.h>#include <linux/wait.h>#include <linux/iversion.h>#include <linux/filelock.h> #include <linux/uaccess.h>#include <linux/sched/mm.h> #include "delegation.h"#include "internal.h"#include "iostat.h"#include "nfs4_fs.h"#include "fscache.h"#include "pnfs.h" #include "nfstrace.h" #define NFSDBG_FACILITY NFSDBG_PAGECACHE #define MIN_POOL_WRITE (32)#define MIN_POOL_COMMIT (4) struct nfs_io_completion { void (*complete)(void *data); void *data; struct kref refcount;}; /* * Local function declarations */static void nfs_redirty_request(struct nfs_page *req);static const struct rpc_call_ops nfs_commit_ops;static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops;static const struct nfs_commit_completion_ops nfs_commit_completion_ops;static const struct nfs_rw_ops nfs_rw_write_ops;static void nfs_inode_remove_request(struct nfs_page *req);static void nfs_clear_request_commit(struct nfs_commit_info *cinfo, struct nfs_page *req);static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo, struct inode *inode); static struct kmem_cache *nfs_wdata_cachep;static mempool_t *nfs_wdata_mempool;static struct kmem_cache *nfs_cdata_cachep;static mempool_t *nfs_commit_mempool; struct nfs_commit_data *nfs_commitdata_alloc(void){ struct nfs_commit_data *p; p = kmem_cache_zalloc(nfs_cdata_cachep, nfs_io_gfp_mask()); if (!p) { p = mempool_alloc(nfs_commit_mempool, GFP_NOWAIT); if (!p) return NULL; memset(p, 0, sizeof(*p)); } INIT_LIST_HEAD(&p->pages); return p;}EXPORT_SYMBOL_GPL(nfs_commitdata_alloc); void nfs_commit_free(struct nfs_commit_data *p){ mempool_free(p, nfs_commit_mempool);}EXPORT_SYMBOL_GPL(nfs_commit_free); static struct nfs_pgio_header *nfs_writehdr_alloc(void){ struct nfs_pgio_header *p; p = kmem_cache_zalloc(nfs_wdata_cachep, nfs_io_gfp_mask()); if (!p) { p = mempool_alloc(nfs_wdata_mempool, GFP_NOWAIT); if (!p) return NULL; memset(p, 0, sizeof(*p)); } p->rw_mode = FMODE_WRITE; return p;} static void nfs_writehdr_free(struct nfs_pgio_header *hdr){ mempool_free(hdr, nfs_wdata_mempool);} static struct nfs_io_completion *nfs_io_completion_alloc(gfp_t gfp_flags){ return kmalloc_obj(struct nfs_io_completion, gfp_flags);} static void nfs_io_completion_init(struct nfs_io_completion *ioc, void (*complete)(void *), void *data){ ioc->complete = complete; ioc->data = data; kref_init(&ioc->refcount);} static void nfs_io_completion_release(struct kref *kref){ struct nfs_io_completion *ioc = container_of(kref, struct nfs_io_completion, refcount); ioc->complete(ioc->data); kfree(ioc);} static void nfs_io_completion_get(struct nfs_io_completion *ioc){ if (ioc != NULL) kref_get(&ioc->refcount);} static void nfs_io_completion_put(struct nfs_io_completion *ioc){ if (ioc != NULL) kref_put(&ioc->refcount, nfs_io_completion_release);} static voidnfs_page_set_inode_ref(struct nfs_page *req, struct inode *inode){ if (!test_and_set_bit(PG_INODE_REF, &req->wb_flags)) { kref_get(&req->wb_kref); atomic_long_inc(&NFS_I(inode)->nrequests); }} static void nfs_cancel_remove_inode(struct nfs_page *req, struct inode *inode){ if (test_and_clear_bit(PG_REMOVE, &req->wb_flags)) nfs_page_set_inode_ref(req, inode);} /** * nfs_folio_find_head_request - find head request associated with a folio * @folio: pointer to folio * * must be called while holding the inode lock. * * returns matching head request with reference held, or NULL if not found. */static struct nfs_page *nfs_folio_find_head_request(struct folio *folio){ struct address_space *mapping = folio->mapping; struct nfs_page *req; if (!folio_test_private(folio)) return NULL; spin_lock(&mapping->i_private_lock); req = folio->private; if (req) { WARN_ON_ONCE(req->wb_head != req); kref_get(&req->wb_kref); } spin_unlock(&mapping->i_private_lock); return req;} /* Adjust the file length if we're writing beyond the end */static void nfs_grow_file(struct folio *folio, unsigned int offset, unsigned int count){ struct inode *inode = folio->mapping->host; loff_t end, i_size; pgoff_t end_index; spin_lock(&inode->i_lock); i_size = i_size_read(inode); end_index = ((i_size - 1) >> folio_shift(folio)) << folio_order(folio); if (i_size > 0 && folio->index < end_index) goto out; end = folio_pos(folio) + (loff_t)offset + (loff_t)count; if (i_size >= end) goto out; trace_nfs_size_grow(inode, end); i_size_write(inode, end); NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_SIZE; nfs_inc_stats(inode, NFSIOS_EXTENDWRITE);out: /* Atomically update timestamps if they are delegated to us. */ nfs_update_delegated_mtime_locked(inode); spin_unlock(&inode->i_lock); nfs_fscache_invalidate(inode, 0);} /* A writeback failed: mark the page as bad, and invalidate the page cache */static void nfs_set_pageerror(struct address_space *mapping){ struct inode *inode = mapping->host; nfs_zap_mapping(mapping->host, mapping); /* Force file size revalidation */ spin_lock(&inode->i_lock); nfs_set_cache_invalid(inode, NFS_INO_REVAL_FORCED | NFS_INO_INVALID_CHANGE | NFS_INO_INVALID_SIZE); spin_unlock(&inode->i_lock);} static void nfs_mapping_set_error(struct folio *folio, int error){ struct address_space *mapping = folio->mapping; filemap_set_wb_err(mapping, error); if (mapping->host) errseq_set(&mapping->host->i_sb->s_wb_err, error == -ENOSPC ? -ENOSPC : -EIO); nfs_set_pageerror(mapping);} /* * nfs_page_covers_folio * @req: struct nfs_page * * Return true if the request covers the whole folio. * Note that the caller should ensure all subrequests have been joined */static bool nfs_page_group_covers_page(struct nfs_page *req){ unsigned int len = nfs_folio_length(nfs_page_to_folio(req)); return req->wb_pgbase == 0 && req->wb_bytes == len;} /* We can set the PG_uptodate flag if we see that a write request * covers the full page. */static void nfs_mark_uptodate(struct nfs_page *req){ struct folio *folio = nfs_page_to_folio(req); if (folio_test_uptodate(folio)) return; if (!nfs_page_group_covers_page(req)) return; folio_mark_uptodate(folio);} static int wb_priority(struct writeback_control *wbc){ int ret = 0; if (wbc->sync_mode == WB_SYNC_ALL) ret = FLUSH_COND_STABLE; return ret;} /* * NFS congestion control */ int nfs_congestion_kb; #define NFS_CONGESTION_ON_THRESH (nfs_congestion_kb >> (PAGE_SHIFT-10))#define NFS_CONGESTION_OFF_THRESH \ (NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2)) static void nfs_folio_set_writeback(struct folio *folio){ struct nfs_server *nfss = NFS_SERVER(folio->mapping->host); folio_start_writeback(folio); if (atomic_long_inc_return(&nfss->writeback) > NFS_CONGESTION_ON_THRESH) nfss->write_congested = 1;} static void nfs_folio_end_writeback(struct folio *folio){ struct nfs_server *nfss = NFS_SERVER(folio->mapping->host); folio_end_writeback_no_dropbehind(folio); if (atomic_long_dec_return(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH) { nfss->write_congested = 0; wake_up_all(&nfss->write_congestion_wait); }} static void nfs_page_end_writeback(struct nfs_page *req){ if (nfs_page_group_sync_on_bit(req, PG_WB_END)) { nfs_unlock_request(req); nfs_folio_end_writeback(nfs_page_to_folio(req)); } else nfs_unlock_request(req);} /* * nfs_destroy_unlinked_subrequests - destroy recently unlinked subrequests * * @destroy_list - request list (using wb_this_page) terminated by @old_head * @old_head - the old head of the list * * All subrequests must be locked and removed from all lists, so at this point * they are only "active" in this function, and possibly in nfs_wait_on_request * with a reference held by some other context. */static voidnfs_destroy_unlinked_subrequests(struct nfs_page *destroy_list, struct nfs_page *old_head, struct inode *inode){ while (destroy_list) { struct nfs_page *subreq = destroy_list; destroy_list = (subreq->wb_this_page == old_head) ? NULL : subreq->wb_this_page; /* Note: lock subreq in order to change subreq->wb_head */ nfs_page_set_headlock(subreq); WARN_ON_ONCE(old_head != subreq->wb_head); /* make sure old group is not used */ subreq->wb_this_page = subreq; subreq->wb_head = subreq; clear_bit(PG_REMOVE, &subreq->wb_flags); /* Note: races with nfs_page_group_destroy() */ if (!kref_read(&subreq->wb_kref)) { /* Check if we raced with nfs_page_group_destroy() */ if (test_and_clear_bit(PG_TEARDOWN, &subreq->wb_flags)) { nfs_page_clear_headlock(subreq); nfs_free_request(subreq); } else nfs_page_clear_headlock(subreq); continue; } nfs_page_clear_headlock(subreq); nfs_release_request(old_head); if (test_and_clear_bit(PG_INODE_REF, &subreq->wb_flags)) { nfs_release_request(subreq); atomic_long_dec(&NFS_I(inode)->nrequests); } /* subreq is now totally disconnected from page group or any * write / commit lists. last chance to wake any waiters */ nfs_unlock_and_release_request(subreq); }} /* * nfs_join_page_group - destroy subrequests of the head req * @head: the page used to lookup the "page group" of nfs_page structures * @inode: Inode to which the request belongs. * * This function joins all sub requests to the head request by first * locking all requests in the group, cancelling any pending operations * and finally updating the head request to cover the whole range covered by * the (former) group. All subrequests are removed from any write or commit * lists, unlinked from the group and destroyed. */void nfs_join_page_group(struct nfs_page *head, struct nfs_commit_info *cinfo, struct inode *inode){ struct nfs_page *subreq; struct nfs_page *destroy_list = NULL; unsigned int pgbase, off, bytes; pgbase = head->wb_pgbase; bytes = head->wb_bytes; off = head->wb_offset; for (subreq = head->wb_this_page; subreq != head; subreq = subreq->wb_this_page) { /* Subrequests should always form a contiguous range */ if (pgbase > subreq->wb_pgbase) { off -= pgbase - subreq->wb_pgbase; bytes += pgbase - subreq->wb_pgbase; pgbase = subreq->wb_pgbase; } bytes = max(subreq->wb_pgbase + subreq->wb_bytes - pgbase, bytes); } /* Set the head request's range to cover the former page group */ head->wb_pgbase = pgbase; head->wb_bytes = bytes; head->wb_offset = off; /* Now that all requests are locked, make sure they aren't on any list. * Commit list removal accounting is done after locks are dropped */ subreq = head; do { nfs_clear_request_commit(cinfo, subreq); subreq = subreq->wb_this_page; } while (subreq != head); /* unlink subrequests from head, destroy them later */ if (head->wb_this_page != head) { /* destroy list will be terminated by head */ destroy_list = head->wb_this_page; head->wb_this_page = head; } nfs_destroy_unlinked_subrequests(destroy_list, head, inode);} /** * nfs_wait_on_request - Wait for a request to complete. * @req: request to wait upon. * * Interruptible by fatal signals only. * The user is responsible for holding a count on the request. */static int nfs_wait_on_request(struct nfs_page *req){ if (!test_bit(PG_BUSY, &req->wb_flags)) return 0; set_bit(PG_CONTENDED2, &req->wb_flags); smp_mb__after_atomic(); return wait_on_bit_io(&req->wb_flags, PG_BUSY, TASK_UNINTERRUPTIBLE);} /* * nfs_unroll_locks - unlock all newly locked reqs and wait on @req * @head: head request of page group, must be holding head lock * @req: request that couldn't lock and needs to wait on the req bit lock * * This is a helper function for nfs_lock_and_join_requests * returns 0 on success, < 0 on error. */static voidnfs_unroll_locks(struct nfs_page *head, struct nfs_page *req){ struct nfs_page *tmp; /* relinquish all the locks successfully grabbed this run */ for (tmp = head->wb_this_page ; tmp != req; tmp = tmp->wb_this_page) { if (!kref_read(&tmp->wb_kref)) continue; nfs_unlock_and_release_request(tmp); }} /* * nfs_page_group_lock_subreq - try to lock a subrequest * @head: head request of page group * @subreq: request to lock * * This is a helper function for nfs_lock_and_join_requests which * must be called with the head request and page group both locked. * On error, it returns with the page group unlocked. */static intnfs_page_group_lock_subreq(struct nfs_page *head, struct nfs_page *subreq){ int ret; if (!kref_get_unless_zero(&subreq->wb_kref)) return 0; while (!nfs_lock_request(subreq)) { nfs_page_group_unlock(head); ret = nfs_wait_on_request(subreq); if (!ret) ret = nfs_page_group_lock(head); if (ret < 0) { nfs_unroll_locks(head, subreq); nfs_release_request(subreq); return ret; } } return 0;} /* * nfs_lock_and_join_requests - join all subreqs to the head req * @folio: the folio used to lookup the "page group" of nfs_page structures * * This function joins all sub requests to the head request by firstLines 1–500 of 2213. Load more
// SPDX-License-Identifier: GPL-2.0-only /* * linux/fs/nfs/write.c * * Write file data over NFS. * * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de> */ #include <linux/types.h> #include <linux/slab.h> #include <linux/mm.h> #include <linux/pagemap.h> #include <linux/file.h> #include <linux/writeback.h> #include <linux/swap.h> #include <linux/migrate.h> #include <linux/sunrpc/clnt.h> #include <linux/nfs_fs.h> #include <linux/nfs_mount.h> #include <linux/nfs_page.h> #include <linux/backing-dev.h> #include <linux/export.h> #include <linux/freezer.h> #include <linux/wait.h> #include <linux/iversion.h> #include <linux/filelock.h> #include <linux/uaccess.h> #include <linux/sched/mm.h> #include "delegation.h" #include "internal.h" #include "iostat.h" #include "nfs4_fs.h" #include "fscache.h" #include "pnfs.h" #include "nfstrace.h" #define NFSDBG_FACILITY NFSDBG_PAGECACHE #define MIN_POOL_WRITE (32) #define MIN_POOL_COMMIT (4) struct nfs_io_completion { void (*complete)(void *data); void *data; struct kref refcount; }; /* * Local function declarations */ static void nfs_redirty_request(struct nfs_page *req); static const struct rpc_call_ops nfs_commit_ops; static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops; static const struct nfs_commit_completion_ops nfs_commit_completion_ops; static const struct nfs_rw_ops nfs_rw_write_ops; static void nfs_inode_remove_request(struct nfs_page *req); static void nfs_clear_request_commit(struct nfs_commit_info *cinfo, struct nfs_page *req); static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo, struct inode *inode); static struct kmem_cache *nfs_wdata_cachep; static mempool_t *nfs_wdata_mempool; static struct kmem_cache *nfs_cdata_cachep; static mempool_t *nfs_commit_mempool; struct nfs_commit_data *nfs_commitdata_alloc(void) { struct nfs_commit_data *p; p = kmem_cache_zalloc(nfs_cdata_cachep, nfs_io_gfp_mask()); if (!p) { p = mempool_alloc(nfs_commit_mempool, GFP_NOWAIT); if (!p) return NULL; memset(p, 0, sizeof(*p)); } INIT_LIST_HEAD(&p->pages); return p; } EXPORT_SYMBOL_GPL(nfs_commitdata_alloc); void nfs_commit_free(struct nfs_commit_data *p) { mempool_free(p, nfs_commit_mempool); } EXPORT_SYMBOL_GPL(nfs_commit_free); static struct nfs_pgio_header *nfs_writehdr_alloc(void) { struct nfs_pgio_header *p; p = kmem_cache_zalloc(nfs_wdata_cachep, nfs_io_gfp_mask()); if (!p) { p = mempool_alloc(nfs_wdata_mempool, GFP_NOWAIT); if (!p) return NULL; memset(p, 0, sizeof(*p)); } p->rw_mode = FMODE_WRITE; return p; } static void nfs_writehdr_free(struct nfs_pgio_header *hdr) { mempool_free(hdr, nfs_wdata_mempool); } static struct nfs_io_completion *nfs_io_completion_alloc(gfp_t gfp_flags) { return kmalloc_obj(struct nfs_io_completion, gfp_flags); } static void nfs_io_completion_init(struct nfs_io_completion *ioc, void (*complete)(void *), void *data) { ioc->complete = complete; ioc->data = data; kref_init(&ioc->refcount); } static void nfs_io_completion_release(struct kref *kref) { struct nfs_io_completion *ioc = container_of(kref, struct nfs_io_completion, refcount); ioc->complete(ioc->data); kfree(ioc); } static void nfs_io_completion_get(struct nfs_io_completion *ioc) { if (ioc != NULL) kref_get(&ioc->refcount); } static void nfs_io_completion_put(struct nfs_io_completion *ioc) { if (ioc != NULL) kref_put(&ioc->refcount, nfs_io_completion_release); } static void nfs_page_set_inode_ref(struct nfs_page *req, struct inode *inode) { if (!test_and_set_bit(PG_INODE_REF, &req->wb_flags)) { kref_get(&req->wb_kref); atomic_long_inc(&NFS_I(inode)->nrequests); } } static void nfs_cancel_remove_inode(struct nfs_page *req, struct inode *inode) { if (test_and_clear_bit(PG_REMOVE, &req->wb_flags)) nfs_page_set_inode_ref(req, inode); } /** * nfs_folio_find_head_request - find head request associated with a folio * @folio: pointer to folio * * must be called while holding the inode lock. * * returns matching head request with reference held, or NULL if not found. */ static struct nfs_page *nfs_folio_find_head_request(struct folio *folio) { struct address_space *mapping = folio->mapping; struct nfs_page *req; if (!folio_test_private(folio)) return NULL; spin_lock(&mapping->i_private_lock); req = folio->private; if (req) { WARN_ON_ONCE(req->wb_head != req); kref_get(&req->wb_kref); } spin_unlock(&mapping->i_private_lock); return req; } /* Adjust the file length if we're writing beyond the end */ static void nfs_grow_file(struct folio *folio, unsigned int offset, unsigned int count) { struct inode *inode = folio->mapping->host; loff_t end, i_size; pgoff_t end_index; spin_lock(&inode->i_lock); i_size = i_size_read(inode); end_index = ((i_size - 1) >> folio_shift(folio)) << folio_order(folio); if (i_size > 0 && folio->index < end_index) goto out; end = folio_pos(folio) + (loff_t)offset + (loff_t)count; if (i_size >= end) goto out; trace_nfs_size_grow(inode, end); i_size_write(inode, end); NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_SIZE; nfs_inc_stats(inode, NFSIOS_EXTENDWRITE); out: /* Atomically update timestamps if they are delegated to us. */ nfs_update_delegated_mtime_locked(inode); spin_unlock(&inode->i_lock); nfs_fscache_invalidate(inode, 0); } /* A writeback failed: mark the page as bad, and invalidate the page cache */ static void nfs_set_pageerror(struct address_space *mapping) { struct inode *inode = mapping->host; nfs_zap_mapping(mapping->host, mapping); /* Force file size revalidation */ spin_lock(&inode->i_lock); nfs_set_cache_invalid(inode, NFS_INO_REVAL_FORCED | NFS_INO_INVALID_CHANGE | NFS_INO_INVALID_SIZE); spin_unlock(&inode->i_lock); } static void nfs_mapping_set_error(struct folio *folio, int error) { struct address_space *mapping = folio->mapping; filemap_set_wb_err(mapping, error); if (mapping->host) errseq_set(&mapping->host->i_sb->s_wb_err, error == -ENOSPC ? -ENOSPC : -EIO); nfs_set_pageerror(mapping); } /* * nfs_page_covers_folio * @req: struct nfs_page * * Return true if the request covers the whole folio. * Note that the caller should ensure all subrequests have been joined */ static bool nfs_page_group_covers_page(struct nfs_page *req) { unsigned int len = nfs_folio_length(nfs_page_to_folio(req)); return req->wb_pgbase == 0 && req->wb_bytes == len; } /* We can set the PG_uptodate flag if we see that a write request * covers the full page. */ static void nfs_mark_uptodate(struct nfs_page *req) { struct folio *folio = nfs_page_to_folio(req); if (folio_test_uptodate(folio)) return; if (!nfs_page_group_covers_page(req)) return; folio_mark_uptodate(folio); } static int wb_priority(struct writeback_control *wbc) { int ret = 0; if (wbc->sync_mode == WB_SYNC_ALL) ret = FLUSH_COND_STABLE; return ret; } /* * NFS congestion control */ int nfs_congestion_kb; #define NFS_CONGESTION_ON_THRESH (nfs_congestion_kb >> (PAGE_SHIFT-10)) #define NFS_CONGESTION_OFF_THRESH \ (NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2)) static void nfs_folio_set_writeback(struct folio *folio) { struct nfs_server *nfss = NFS_SERVER(folio->mapping->host); folio_start_writeback(folio); if (atomic_long_inc_return(&nfss->writeback) > NFS_CONGESTION_ON_THRESH) nfss->write_congested = 1; } static void nfs_folio_end_writeback(struct folio *folio) { struct nfs_server *nfss = NFS_SERVER(folio->mapping->host); folio_end_writeback_no_dropbehind(folio); if (atomic_long_dec_return(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH) { nfss->write_congested = 0; wake_up_all(&nfss->write_congestion_wait); } } static void nfs_page_end_writeback(struct nfs_page *req) { if (nfs_page_group_sync_on_bit(req, PG_WB_END)) { nfs_unlock_request(req); nfs_folio_end_writeback(nfs_page_to_folio(req)); } else nfs_unlock_request(req); } /* * nfs_destroy_unlinked_subrequests - destroy recently unlinked subrequests * * @destroy_list - request list (using wb_this_page) terminated by @old_head * @old_head - the old head of the list * * All subrequests must be locked and removed from all lists, so at this point * they are only "active" in this function, and possibly in nfs_wait_on_request * with a reference held by some other context. */ static void nfs_destroy_unlinked_subrequests(struct nfs_page *destroy_list, struct nfs_page *old_head, struct inode *inode) { while (destroy_list) { struct nfs_page *subreq = destroy_list; destroy_list = (subreq->wb_this_page == old_head) ? NULL : subreq->wb_this_page; /* Note: lock subreq in order to change subreq->wb_head */ nfs_page_set_headlock(subreq); WARN_ON_ONCE(old_head != subreq->wb_head); /* make sure old group is not used */ subreq->wb_this_page = subreq; subreq->wb_head = subreq; clear_bit(PG_REMOVE, &subreq->wb_flags); /* Note: races with nfs_page_group_destroy() */ if (!kref_read(&subreq->wb_kref)) { /* Check if we raced with nfs_page_group_destroy() */ if (test_and_clear_bit(PG_TEARDOWN, &subreq->wb_flags)) { nfs_page_clear_headlock(subreq); nfs_free_request(subreq); } else nfs_page_clear_headlock(subreq); continue; } nfs_page_clear_headlock(subreq); nfs_release_request(old_head); if (test_and_clear_bit(PG_INODE_REF, &subreq->wb_flags)) { nfs_release_request(subreq); atomic_long_dec(&NFS_I(inode)->nrequests); } /* subreq is now totally disconnected from page group or any * write / commit lists. last chance to wake any waiters */ nfs_unlock_and_release_request(subreq); } } /* * nfs_join_page_group - destroy subrequests of the head req * @head: the page used to lookup the "page group" of nfs_page structures * @inode: Inode to which the request belongs. * * This function joins all sub requests to the head request by first * locking all requests in the group, cancelling any pending operations * and finally updating the head request to cover the whole range covered by * the (former) group. All subrequests are removed from any write or commit * lists, unlinked from the group and destroyed. */ void nfs_join_page_group(struct nfs_page *head, struct nfs_commit_info *cinfo, struct inode *inode) { struct nfs_page *subreq; struct nfs_page *destroy_list = NULL; unsigned int pgbase, off, bytes; pgbase = head->wb_pgbase; bytes = head->wb_bytes; off = head->wb_offset; for (subreq = head->wb_this_page; subreq != head; subreq = subreq->wb_this_page) { /* Subrequests should always form a contiguous range */ if (pgbase > subreq->wb_pgbase) { off -= pgbase - subreq->wb_pgbase; bytes += pgbase - subreq->wb_pgbase; pgbase = subreq->wb_pgbase; } bytes = max(subreq->wb_pgbase + subreq->wb_bytes - pgbase, bytes); } /* Set the head request's range to cover the former page group */ head->wb_pgbase = pgbase; head->wb_bytes = bytes; head->wb_offset = off; /* Now that all requests are locked, make sure they aren't on any list. * Commit list removal accounting is done after locks are dropped */ subreq = head; do { nfs_clear_request_commit(cinfo, subreq); subreq = subreq->wb_this_page; } while (subreq != head); /* unlink subrequests from head, destroy them later */ if (head->wb_this_page != head) { /* destroy list will be terminated by head */ destroy_list = head->wb_this_page; head->wb_this_page = head; } nfs_destroy_unlinked_subrequests(destroy_list, head, inode); } /** * nfs_wait_on_request - Wait for a request to complete. * @req: request to wait upon. * * Interruptible by fatal signals only. * The user is responsible for holding a count on the request. */ static int nfs_wait_on_request(struct nfs_page *req) { if (!test_bit(PG_BUSY, &req->wb_flags)) return 0; set_bit(PG_CONTENDED2, &req->wb_flags); smp_mb__after_atomic(); return wait_on_bit_io(&req->wb_flags, PG_BUSY, TASK_UNINTERRUPTIBLE); } /* * nfs_unroll_locks - unlock all newly locked reqs and wait on @req * @head: head request of page group, must be holding head lock * @req: request that couldn't lock and needs to wait on the req bit lock * * This is a helper function for nfs_lock_and_join_requests * returns 0 on success, < 0 on error. */ static void nfs_unroll_locks(struct nfs_page *head, struct nfs_page *req) { struct nfs_page *tmp; /* relinquish all the locks successfully grabbed this run */ for (tmp = head->wb_this_page ; tmp != req; tmp = tmp->wb_this_page) { if (!kref_read(&tmp->wb_kref)) continue; nfs_unlock_and_release_request(tmp); } } /* * nfs_page_group_lock_subreq - try to lock a subrequest * @head: head request of page group * @subreq: request to lock * * This is a helper function for nfs_lock_and_join_requests which * must be called with the head request and page group both locked. * On error, it returns with the page group unlocked. */ static int nfs_page_group_lock_subreq(struct nfs_page *head, struct nfs_page *subreq) { int ret; if (!kref_get_unless_zero(&subreq->wb_kref)) return 0; while (!nfs_lock_request(subreq)) { nfs_page_group_unlock(head); ret = nfs_wait_on_request(subreq); if (!ret) ret = nfs_page_group_lock(head); if (ret < 0) { nfs_unroll_locks(head, subreq); nfs_release_request(subreq); return ret; } } return 0; } /* * nfs_lock_and_join_requests - join all subreqs to the head req * @folio: the folio used to lookup the "page group" of nfs_page structures * * This function joins all sub requests to the head request by first