/* * An async IO implementation for Linux * Written by Benjamin LaHaise <bcrl@kvack.org> * * Implements an efficient asynchronous io interface. * * Copyright 2000, 2001, 2002 Red Hat, Inc. All Rights Reserved. * Copyright 2018 Christoph Hellwig. * * See ../COPYING for licensing terms. */#define pr_fmt(fmt) "%s: " fmt, __func__ #include <linux/kernel.h>#include <linux/init.h>#include <linux/errno.h>#include <linux/time.h>#include <linux/aio_abi.h>#include <linux/export.h>#include <linux/syscalls.h>#include <linux/backing-dev.h>#include <linux/refcount.h>#include <linux/uio.h> #include <linux/sched/signal.h>#include <linux/fs.h>#include <linux/file.h>#include <linux/mm.h>#include <linux/mman.h>#include <linux/percpu.h>#include <linux/slab.h>#include <linux/timer.h>#include <linux/aio.h>#include <linux/highmem.h>#include <linux/workqueue.h>#include <linux/security.h>#include <linux/eventfd.h>#include <linux/blk_plug.h>#include <linux/compat.h>#include <linux/migrate.h>#include <linux/ramfs.h>#include <linux/percpu-refcount.h>#include <linux/mount.h>#include <linux/pseudo_fs.h> #include <linux/uaccess.h>#include <linux/nospec.h> #include "internal.h" #define KIOCB_KEY 0 #define AIO_RING_MAGIC 0xa10a10a1#define AIO_RING_COMPAT_FEATURES 1#define AIO_RING_INCOMPAT_FEATURES 0struct aio_ring { unsigned id; /* kernel internal index number */ unsigned nr; /* number of io_events */ unsigned head; /* Written to by userland or under ring_lock * mutex by aio_read_events_ring(). */ unsigned tail; unsigned magic; unsigned compat_features; unsigned incompat_features; unsigned header_length; /* size of aio_ring */ struct io_event io_events[];}; /* 128 bytes + ring size */ /* * Plugging is meant to work with larger batches of IOs. If we don't * have more than the below, then don't bother setting up a plug. */#define AIO_PLUG_THRESHOLD 2 #define AIO_RING_PAGES 8 struct kioctx_table { struct rcu_head rcu; unsigned nr; struct kioctx __rcu *table[] __counted_by(nr);}; struct kioctx_cpu { unsigned reqs_available;}; struct ctx_rq_wait { struct completion comp; atomic_t count;}; struct kioctx { struct percpu_ref users; atomic_t dead; struct percpu_ref reqs; unsigned long user_id; struct kioctx_cpu __percpu *cpu; /* * For percpu reqs_available, number of slots we move to/from global * counter at a time: */ unsigned req_batch; /* * This is what userspace passed to io_setup(), it's not used for * anything but counting against the global max_reqs quota. * * The real limit is nr_events - 1, which will be larger (see * aio_setup_ring()) */ unsigned max_reqs; /* Size of ringbuffer, in units of struct io_event */ unsigned nr_events; unsigned long mmap_base; unsigned long mmap_size; struct folio **ring_folios; long nr_pages; struct rcu_work free_rwork; /* see free_ioctx() */ /* * signals when all in-flight requests are done */ struct ctx_rq_wait *rq_wait; struct { /* * This counts the number of available slots in the ringbuffer, * so we avoid overflowing it: it's decremented (if positive) * when allocating a kiocb and incremented when the resulting * io_event is pulled off the ringbuffer. * * We batch accesses to it with a percpu version. */ atomic_t reqs_available; } ____cacheline_aligned_in_smp; struct { spinlock_t ctx_lock; struct list_head active_reqs; /* used for cancellation */ } ____cacheline_aligned_in_smp; struct { struct mutex ring_lock; wait_queue_head_t wait; } ____cacheline_aligned_in_smp; struct { unsigned tail; unsigned completed_events; spinlock_t completion_lock; } ____cacheline_aligned_in_smp; struct folio *internal_folios[AIO_RING_PAGES]; struct file *aio_ring_file; unsigned id;}; /* * First field must be the file pointer in all the * iocb unions! See also 'struct kiocb' in <linux/fs.h> */struct fsync_iocb { struct file *file; struct work_struct work; bool datasync; struct cred *creds;}; struct poll_iocb { struct file *file; struct wait_queue_head *head; __poll_t events; bool cancelled; bool work_scheduled; bool work_need_resched; struct wait_queue_entry wait; struct work_struct work;}; /* * NOTE! Each of the iocb union members has the file pointer * as the first entry in their struct definition. So you can * access the file pointer through any of the sub-structs, * or directly as just 'ki_filp' in this struct. */struct aio_kiocb { union { struct file *ki_filp; struct kiocb rw; struct fsync_iocb fsync; struct poll_iocb poll; }; struct kioctx *ki_ctx; kiocb_cancel_fn *ki_cancel; struct io_event ki_res; struct list_head ki_list; /* the aio core uses this * for cancellation */ refcount_t ki_refcnt; /* * If the aio_resfd field of the userspace iocb is not zero, * this is the underlying eventfd context to deliver events to. */ struct eventfd_ctx *ki_eventfd;}; struct aio_inode_info { struct inode vfs_inode; spinlock_t migrate_lock; struct kioctx *ctx;}; static inline struct aio_inode_info *AIO_I(struct inode *inode){ return container_of(inode, struct aio_inode_info, vfs_inode);} /*------ sysctl variables----*/static DEFINE_SPINLOCK(aio_nr_lock);static unsigned long aio_nr; /* current system wide number of aio requests */static unsigned long aio_max_nr = 0x10000; /* system wide maximum number of aio requests *//*----end sysctl variables---*/#ifdef CONFIG_SYSCTLstatic const struct ctl_table aio_sysctls[] = { { .procname = "aio-nr", .data = &aio_nr, .maxlen = sizeof(aio_nr), .mode = 0444, .proc_handler = proc_doulongvec_minmax, }, { .procname = "aio-max-nr", .data = &aio_max_nr, .maxlen = sizeof(aio_max_nr), .mode = 0644, .proc_handler = proc_doulongvec_minmax, },}; static void __init aio_sysctl_init(void){ register_sysctl_init("fs", aio_sysctls);}#else#define aio_sysctl_init() do { } while (0)#endif static struct kmem_cache *kiocb_cachep;static struct kmem_cache *kioctx_cachep;static struct kmem_cache *aio_inode_cachep; static struct vfsmount *aio_mnt; static const struct file_operations aio_ring_fops;static const struct address_space_operations aio_ctx_aops; static struct file *aio_private_file(struct kioctx *ctx, loff_t nr_pages){ struct file *file; struct inode *inode = alloc_anon_inode(aio_mnt->mnt_sb); if (IS_ERR(inode)) return ERR_CAST(inode); inode->i_mapping->a_ops = &aio_ctx_aops; AIO_I(inode)->ctx = ctx; inode->i_size = PAGE_SIZE * nr_pages; file = alloc_file_pseudo(inode, aio_mnt, "[aio]", O_RDWR, &aio_ring_fops); if (IS_ERR(file)) iput(inode); return file;} static struct inode *aio_alloc_inode(struct super_block *sb){ struct aio_inode_info *ai; ai = alloc_inode_sb(sb, aio_inode_cachep, GFP_KERNEL); if (!ai) return NULL; ai->ctx = NULL; return &ai->vfs_inode;} static void aio_free_inode(struct inode *inode){ kmem_cache_free(aio_inode_cachep, AIO_I(inode));} static const struct super_operations aio_super_operations = { .alloc_inode = aio_alloc_inode, .free_inode = aio_free_inode, .statfs = simple_statfs,}; static int aio_init_fs_context(struct fs_context *fc){ struct pseudo_fs_context *pfc; pfc = init_pseudo(fc, AIO_RING_MAGIC); if (!pfc) return -ENOMEM; pfc->ops = &aio_super_operations; return 0;} static void init_once(void *obj){ struct aio_inode_info *ai = obj; inode_init_once(&ai->vfs_inode); spin_lock_init(&ai->migrate_lock);} /* aio_setup * Creates the slab caches used by the aio routines, panic on * failure as this is done early during the boot sequence. */static int __init aio_setup(void){ static struct file_system_type aio_fs = { .name = "aio", .init_fs_context = aio_init_fs_context, .kill_sb = kill_anon_super, }; aio_inode_cachep = kmem_cache_create("aio_inode_cache", sizeof(struct aio_inode_info), 0, (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_ACCOUNT), init_once); aio_mnt = kern_mount(&aio_fs); if (IS_ERR(aio_mnt)) panic("Failed to create aio fs mount."); kiocb_cachep = KMEM_CACHE(aio_kiocb, SLAB_HWCACHE_ALIGN|SLAB_PANIC); kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC); aio_sysctl_init(); return 0;}__initcall(aio_setup); static void put_aio_ring_file(struct kioctx *ctx){ struct file *aio_ring_file = ctx->aio_ring_file; if (aio_ring_file) { struct inode *inode = file_inode(aio_ring_file); truncate_setsize(inode, 0); /* Prevent further access to the kioctx from migratepages */ spin_lock(&AIO_I(inode)->migrate_lock); AIO_I(inode)->ctx = NULL; ctx->aio_ring_file = NULL; spin_unlock(&AIO_I(inode)->migrate_lock); fput(aio_ring_file); }} static void aio_free_ring(struct kioctx *ctx){ int i; /* Disconnect the kiotx from the ring file. This prevents future * accesses to the kioctx from page migration. */ put_aio_ring_file(ctx); for (i = 0; i < ctx->nr_pages; i++) { struct folio *folio = ctx->ring_folios[i]; if (!folio) continue; pr_debug("pid(%d) [%d] folio->count=%d\n", current->pid, i, folio_ref_count(folio)); ctx->ring_folios[i] = NULL; folio_put(folio); } if (ctx->ring_folios && ctx->ring_folios != ctx->internal_folios) { kfree(ctx->ring_folios); ctx->ring_folios = NULL; }} static int aio_ring_mremap(struct vm_area_struct *vma){ struct file *file = vma->vm_file; struct mm_struct *mm = vma->vm_mm; struct kioctx_table *table; int i, res = -EINVAL; spin_lock(&mm->ioctx_lock); rcu_read_lock(); table = rcu_dereference(mm->ioctx_table); if (!table) goto out_unlock; for (i = 0; i < table->nr; i++) { struct kioctx *ctx; ctx = rcu_dereference(table->table[i]); if (ctx && ctx->aio_ring_file == file) { if (!atomic_read(&ctx->dead) && (ctx->mmap_size == (vma->vm_end - vma->vm_start))) { ctx->user_id = ctx->mmap_base = vma->vm_start; res = 0; } break; } } out_unlock: rcu_read_unlock(); spin_unlock(&mm->ioctx_lock); return res;} static const struct vm_operations_struct aio_ring_vm_ops = { .mremap = aio_ring_mremap,#if IS_ENABLED(CONFIG_MMU) .fault = filemap_fault, .map_pages = filemap_map_pages, .page_mkwrite = filemap_page_mkwrite,#endif}; static int aio_ring_mmap_prepare(struct vm_area_desc *desc){ vma_desc_set_flags(desc, VMA_DONTEXPAND_BIT, VMA_DONTCOPY_BIT); desc->vm_ops = &aio_ring_vm_ops; return 0;} static const struct file_operations aio_ring_fops = { .mmap_prepare = aio_ring_mmap_prepare,}; #if IS_ENABLED(CONFIG_MIGRATION)static int aio_migrate_folio(struct address_space *mapping, struct folio *dst, struct folio *src, enum migrate_mode mode){ struct kioctx *ctx; struct aio_inode_info *ai = AIO_I(mapping->host); unsigned long flags; pgoff_t idx; int rc = 0; /* ai->migrate_lock here protects against the kioctx teardown. */ spin_lock(&ai->migrate_lock); ctx = ai->ctx; if (!ctx) { rc = -EINVAL; goto out; } /* The ring_lock mutex. The prevents aio_read_events() from writing * to the ring's head, and prevents page migration from mucking in * a partially initialized kiotx. */ if (!mutex_trylock(&ctx->ring_lock)) { rc = -EAGAIN; goto out; } idx = src->index; if (idx < (pgoff_t)ctx->nr_pages) { /* Make sure the old folio hasn't already been changed */ if (ctx->ring_folios[idx] != src) rc = -EAGAIN; } else rc = -EINVAL; if (rc != 0) goto out_unlock; /* Writeback must be complete */ BUG_ON(folio_test_writeback(src)); folio_get(dst); Lines 1–500 of 2498. Load more
/* * An async IO implementation for Linux * Written by Benjamin LaHaise <bcrl@kvack.org> * * Implements an efficient asynchronous io interface. * * Copyright 2000, 2001, 2002 Red Hat, Inc. All Rights Reserved. * Copyright 2018 Christoph Hellwig. * * See ../COPYING for licensing terms. */ #define pr_fmt(fmt) "%s: " fmt, __func__ #include <linux/kernel.h> #include <linux/init.h> #include <linux/errno.h> #include <linux/time.h> #include <linux/aio_abi.h> #include <linux/export.h> #include <linux/syscalls.h> #include <linux/backing-dev.h> #include <linux/refcount.h> #include <linux/uio.h> #include <linux/sched/signal.h> #include <linux/fs.h> #include <linux/file.h> #include <linux/mm.h> #include <linux/mman.h> #include <linux/percpu.h> #include <linux/slab.h> #include <linux/timer.h> #include <linux/aio.h> #include <linux/highmem.h> #include <linux/workqueue.h> #include <linux/security.h> #include <linux/eventfd.h> #include <linux/blk_plug.h> #include <linux/compat.h> #include <linux/migrate.h> #include <linux/ramfs.h> #include <linux/percpu-refcount.h> #include <linux/mount.h> #include <linux/pseudo_fs.h> #include <linux/uaccess.h> #include <linux/nospec.h> #include "internal.h" #define KIOCB_KEY 0 #define AIO_RING_MAGIC 0xa10a10a1 #define AIO_RING_COMPAT_FEATURES 1 #define AIO_RING_INCOMPAT_FEATURES 0 struct aio_ring { unsigned id; /* kernel internal index number */ unsigned nr; /* number of io_events */ unsigned head; /* Written to by userland or under ring_lock * mutex by aio_read_events_ring(). */ unsigned tail; unsigned magic; unsigned compat_features; unsigned incompat_features; unsigned header_length; /* size of aio_ring */ struct io_event io_events[]; }; /* 128 bytes + ring size */ /* * Plugging is meant to work with larger batches of IOs. If we don't * have more than the below, then don't bother setting up a plug. */ #define AIO_PLUG_THRESHOLD 2 #define AIO_RING_PAGES 8 struct kioctx_table { struct rcu_head rcu; unsigned nr; struct kioctx __rcu *table[] __counted_by(nr); }; struct kioctx_cpu { unsigned reqs_available; }; struct ctx_rq_wait { struct completion comp; atomic_t count; }; struct kioctx { struct percpu_ref users; atomic_t dead; struct percpu_ref reqs; unsigned long user_id; struct kioctx_cpu __percpu *cpu; /* * For percpu reqs_available, number of slots we move to/from global * counter at a time: */ unsigned req_batch; /* * This is what userspace passed to io_setup(), it's not used for * anything but counting against the global max_reqs quota. * * The real limit is nr_events - 1, which will be larger (see * aio_setup_ring()) */ unsigned max_reqs; /* Size of ringbuffer, in units of struct io_event */ unsigned nr_events; unsigned long mmap_base; unsigned long mmap_size; struct folio **ring_folios; long nr_pages; struct rcu_work free_rwork; /* see free_ioctx() */ /* * signals when all in-flight requests are done */ struct ctx_rq_wait *rq_wait; struct { /* * This counts the number of available slots in the ringbuffer, * so we avoid overflowing it: it's decremented (if positive) * when allocating a kiocb and incremented when the resulting * io_event is pulled off the ringbuffer. * * We batch accesses to it with a percpu version. */ atomic_t reqs_available; } ____cacheline_aligned_in_smp; struct { spinlock_t ctx_lock; struct list_head active_reqs; /* used for cancellation */ } ____cacheline_aligned_in_smp; struct { struct mutex ring_lock; wait_queue_head_t wait; } ____cacheline_aligned_in_smp; struct { unsigned tail; unsigned completed_events; spinlock_t completion_lock; } ____cacheline_aligned_in_smp; struct folio *internal_folios[AIO_RING_PAGES]; struct file *aio_ring_file; unsigned id; }; /* * First field must be the file pointer in all the * iocb unions! See also 'struct kiocb' in <linux/fs.h> */ struct fsync_iocb { struct file *file; struct work_struct work; bool datasync; struct cred *creds; }; struct poll_iocb { struct file *file; struct wait_queue_head *head; __poll_t events; bool cancelled; bool work_scheduled; bool work_need_resched; struct wait_queue_entry wait; struct work_struct work; }; /* * NOTE! Each of the iocb union members has the file pointer * as the first entry in their struct definition. So you can * access the file pointer through any of the sub-structs, * or directly as just 'ki_filp' in this struct. */ struct aio_kiocb { union { struct file *ki_filp; struct kiocb rw; struct fsync_iocb fsync; struct poll_iocb poll; }; struct kioctx *ki_ctx; kiocb_cancel_fn *ki_cancel; struct io_event ki_res; struct list_head ki_list; /* the aio core uses this * for cancellation */ refcount_t ki_refcnt; /* * If the aio_resfd field of the userspace iocb is not zero, * this is the underlying eventfd context to deliver events to. */ struct eventfd_ctx *ki_eventfd; }; struct aio_inode_info { struct inode vfs_inode; spinlock_t migrate_lock; struct kioctx *ctx; }; static inline struct aio_inode_info *AIO_I(struct inode *inode) { return container_of(inode, struct aio_inode_info, vfs_inode); } /*------ sysctl variables----*/ static DEFINE_SPINLOCK(aio_nr_lock); static unsigned long aio_nr; /* current system wide number of aio requests */ static unsigned long aio_max_nr = 0x10000; /* system wide maximum number of aio requests */ /*----end sysctl variables---*/ #ifdef CONFIG_SYSCTL static const struct ctl_table aio_sysctls[] = { { .procname = "aio-nr", .data = &aio_nr, .maxlen = sizeof(aio_nr), .mode = 0444, .proc_handler = proc_doulongvec_minmax, }, { .procname = "aio-max-nr", .data = &aio_max_nr, .maxlen = sizeof(aio_max_nr), .mode = 0644, .proc_handler = proc_doulongvec_minmax, }, }; static void __init aio_sysctl_init(void) { register_sysctl_init("fs", aio_sysctls); } #else #define aio_sysctl_init() do { } while (0) #endif static struct kmem_cache *kiocb_cachep; static struct kmem_cache *kioctx_cachep; static struct kmem_cache *aio_inode_cachep; static struct vfsmount *aio_mnt; static const struct file_operations aio_ring_fops; static const struct address_space_operations aio_ctx_aops; static struct file *aio_private_file(struct kioctx *ctx, loff_t nr_pages) { struct file *file; struct inode *inode = alloc_anon_inode(aio_mnt->mnt_sb); if (IS_ERR(inode)) return ERR_CAST(inode); inode->i_mapping->a_ops = &aio_ctx_aops; AIO_I(inode)->ctx = ctx; inode->i_size = PAGE_SIZE * nr_pages; file = alloc_file_pseudo(inode, aio_mnt, "[aio]", O_RDWR, &aio_ring_fops); if (IS_ERR(file)) iput(inode); return file; } static struct inode *aio_alloc_inode(struct super_block *sb) { struct aio_inode_info *ai; ai = alloc_inode_sb(sb, aio_inode_cachep, GFP_KERNEL); if (!ai) return NULL; ai->ctx = NULL; return &ai->vfs_inode; } static void aio_free_inode(struct inode *inode) { kmem_cache_free(aio_inode_cachep, AIO_I(inode)); } static const struct super_operations aio_super_operations = { .alloc_inode = aio_alloc_inode, .free_inode = aio_free_inode, .statfs = simple_statfs, }; static int aio_init_fs_context(struct fs_context *fc) { struct pseudo_fs_context *pfc; pfc = init_pseudo(fc, AIO_RING_MAGIC); if (!pfc) return -ENOMEM; pfc->ops = &aio_super_operations; return 0; } static void init_once(void *obj) { struct aio_inode_info *ai = obj; inode_init_once(&ai->vfs_inode); spin_lock_init(&ai->migrate_lock); } /* aio_setup * Creates the slab caches used by the aio routines, panic on * failure as this is done early during the boot sequence. */ static int __init aio_setup(void) { static struct file_system_type aio_fs = { .name = "aio", .init_fs_context = aio_init_fs_context, .kill_sb = kill_anon_super, }; aio_inode_cachep = kmem_cache_create("aio_inode_cache", sizeof(struct aio_inode_info), 0, (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_ACCOUNT), init_once); aio_mnt = kern_mount(&aio_fs); if (IS_ERR(aio_mnt)) panic("Failed to create aio fs mount."); kiocb_cachep = KMEM_CACHE(aio_kiocb, SLAB_HWCACHE_ALIGN|SLAB_PANIC); kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC); aio_sysctl_init(); return 0; } __initcall(aio_setup); static void put_aio_ring_file(struct kioctx *ctx) { struct file *aio_ring_file = ctx->aio_ring_file; if (aio_ring_file) { struct inode *inode = file_inode(aio_ring_file); truncate_setsize(inode, 0); /* Prevent further access to the kioctx from migratepages */ spin_lock(&AIO_I(inode)->migrate_lock); AIO_I(inode)->ctx = NULL; ctx->aio_ring_file = NULL; spin_unlock(&AIO_I(inode)->migrate_lock); fput(aio_ring_file); } } static void aio_free_ring(struct kioctx *ctx) { int i; /* Disconnect the kiotx from the ring file. This prevents future * accesses to the kioctx from page migration. */ put_aio_ring_file(ctx); for (i = 0; i < ctx->nr_pages; i++) { struct folio *folio = ctx->ring_folios[i]; if (!folio) continue; pr_debug("pid(%d) [%d] folio->count=%d\n", current->pid, i, folio_ref_count(folio)); ctx->ring_folios[i] = NULL; folio_put(folio); } if (ctx->ring_folios && ctx->ring_folios != ctx->internal_folios) { kfree(ctx->ring_folios); ctx->ring_folios = NULL; } } static int aio_ring_mremap(struct vm_area_struct *vma) { struct file *file = vma->vm_file; struct mm_struct *mm = vma->vm_mm; struct kioctx_table *table; int i, res = -EINVAL; spin_lock(&mm->ioctx_lock); rcu_read_lock(); table = rcu_dereference(mm->ioctx_table); if (!table) goto out_unlock; for (i = 0; i < table->nr; i++) { struct kioctx *ctx; ctx = rcu_dereference(table->table[i]); if (ctx && ctx->aio_ring_file == file) { if (!atomic_read(&ctx->dead) && (ctx->mmap_size == (vma->vm_end - vma->vm_start))) { ctx->user_id = ctx->mmap_base = vma->vm_start; res = 0; } break; } } out_unlock: rcu_read_unlock(); spin_unlock(&mm->ioctx_lock); return res; } static const struct vm_operations_struct aio_ring_vm_ops = { .mremap = aio_ring_mremap, #if IS_ENABLED(CONFIG_MMU) .fault = filemap_fault, .map_pages = filemap_map_pages, .page_mkwrite = filemap_page_mkwrite, #endif }; static int aio_ring_mmap_prepare(struct vm_area_desc *desc) { vma_desc_set_flags(desc, VMA_DONTEXPAND_BIT, VMA_DONTCOPY_BIT); desc->vm_ops = &aio_ring_vm_ops; return 0; } static const struct file_operations aio_ring_fops = { .mmap_prepare = aio_ring_mmap_prepare, }; #if IS_ENABLED(CONFIG_MIGRATION) static int aio_migrate_folio(struct address_space *mapping, struct folio *dst, struct folio *src, enum migrate_mode mode) { struct kioctx *ctx; struct aio_inode_info *ai = AIO_I(mapping->host); unsigned long flags; pgoff_t idx; int rc = 0; /* ai->migrate_lock here protects against the kioctx teardown. */ spin_lock(&ai->migrate_lock); ctx = ai->ctx; if (!ctx) { rc = -EINVAL; goto out; } /* The ring_lock mutex. The prevents aio_read_events() from writing * to the ring's head, and prevents page migration from mucking in * a partially initialized kiotx. */ if (!mutex_trylock(&ctx->ring_lock)) { rc = -EAGAIN; goto out; } idx = src->index; if (idx < (pgoff_t)ctx->nr_pages) { /* Make sure the old folio hasn't already been changed */ if (ctx->ring_folios[idx] != src) rc = -EAGAIN; } else rc = -EINVAL; if (rc != 0) goto out_unlock; /* Writeback must be complete */ BUG_ON(folio_test_writeback(src)); folio_get(dst);