2278 lines · 56.5 KB · C
RawDownload
/* *	An async IO implementation for Linux *	Written by Benjamin LaHaise <bcrl@kvack.org> * *	Implements an efficient asynchronous io interface. * *	Copyright 2000, 2001, 2002 Red Hat, Inc.  All Rights Reserved. *	Copyright 2018 Christoph Hellwig. * *	See ../COPYING for licensing terms. */#define pr_fmt(fmt) "%s: " fmt, __func__ #include <linux/kernel.h>#include <linux/init.h>#include <linux/errno.h>#include <linux/time.h>#include <linux/aio_abi.h>#include <linux/export.h>#include <linux/syscalls.h>#include <linux/backing-dev.h>#include <linux/refcount.h>#include <linux/uio.h> #include <linux/sched/signal.h>#include <linux/fs.h>#include <linux/file.h>#include <linux/mm.h>#include <linux/mman.h>#include <linux/percpu.h>#include <linux/slab.h>#include <linux/timer.h>#include <linux/aio.h>#include <linux/highmem.h>#include <linux/workqueue.h>#include <linux/security.h>#include <linux/eventfd.h>#include <linux/blkdev.h>#include <linux/compat.h>#include <linux/migrate.h>#include <linux/ramfs.h>#include <linux/percpu-refcount.h>#include <linux/mount.h>#include <linux/pseudo_fs.h> #include <asm/kmap_types.h>#include <linux/uaccess.h>#include <linux/nospec.h> #include "internal.h" #define KIOCB_KEY		0 #define AIO_RING_MAGIC			0xa10a10a1#define AIO_RING_COMPAT_FEATURES	1#define AIO_RING_INCOMPAT_FEATURES	0struct aio_ring {	unsigned	id;	/* kernel internal index number */	unsigned	nr;	/* number of io_events */	unsigned	head;	/* Written to by userland or under ring_lock				 * mutex by aio_read_events_ring(). */	unsigned	tail; 	unsigned	magic;	unsigned	compat_features;	unsigned	incompat_features;	unsigned	header_length;	/* size of aio_ring */  	struct io_event		io_events[];}; /* 128 bytes + ring size */ /* * Plugging is meant to work with larger batches of IOs. If we don't * have more than the below, then don't bother setting up a plug. */#define AIO_PLUG_THRESHOLD	2 #define AIO_RING_PAGES	8 struct kioctx_table {	struct rcu_head		rcu;	unsigned		nr;	struct kioctx __rcu	*table[];}; struct kioctx_cpu {	unsigned		reqs_available;}; struct ctx_rq_wait {	struct completion comp;	atomic_t count;}; struct kioctx {	struct percpu_ref	users;	atomic_t		dead; 	struct percpu_ref	reqs; 	unsigned long		user_id; 	struct __percpu kioctx_cpu *cpu; 	/*	 * For percpu reqs_available, number of slots we move to/from global	 * counter at a time:	 */	unsigned		req_batch;	/*	 * This is what userspace passed to io_setup(), it's not used for	 * anything but counting against the global max_reqs quota.	 *	 * The real limit is nr_events - 1, which will be larger (see	 * aio_setup_ring())	 */	unsigned		max_reqs; 	/* Size of ringbuffer, in units of struct io_event */	unsigned		nr_events; 	unsigned long		mmap_base;	unsigned long		mmap_size; 	struct page		**ring_pages;	long			nr_pages; 	struct rcu_work		free_rwork;	/* see free_ioctx() */ 	/*	 * signals when all in-flight requests are done	 */	struct ctx_rq_wait	*rq_wait; 	struct {		/*		 * This counts the number of available slots in the ringbuffer,		 * so we avoid overflowing it: it's decremented (if positive)		 * when allocating a kiocb and incremented when the resulting		 * io_event is pulled off the ringbuffer.		 *		 * We batch accesses to it with a percpu version.		 */		atomic_t	reqs_available;	} ____cacheline_aligned_in_smp; 	struct {		spinlock_t	ctx_lock;		struct list_head active_reqs;	/* used for cancellation */	} ____cacheline_aligned_in_smp; 	struct {		struct mutex	ring_lock;		wait_queue_head_t wait;	} ____cacheline_aligned_in_smp; 	struct {		unsigned	tail;		unsigned	completed_events;		spinlock_t	completion_lock;	} ____cacheline_aligned_in_smp; 	struct page		*internal_pages[AIO_RING_PAGES];	struct file		*aio_ring_file; 	unsigned		id;}; /* * First field must be the file pointer in all the * iocb unions! See also 'struct kiocb' in <linux/fs.h> */struct fsync_iocb {	struct file		*file;	struct work_struct	work;	bool			datasync;	struct cred		*creds;}; struct poll_iocb {	struct file		*file;	struct wait_queue_head	*head;	__poll_t		events;	bool			done;	bool			cancelled;	struct wait_queue_entry	wait;	struct work_struct	work;}; /* * NOTE! Each of the iocb union members has the file pointer * as the first entry in their struct definition. So you can * access the file pointer through any of the sub-structs, * or directly as just 'ki_filp' in this struct. */struct aio_kiocb {	union {		struct file		*ki_filp;		struct kiocb		rw;		struct fsync_iocb	fsync;		struct poll_iocb	poll;	}; 	struct kioctx		*ki_ctx;	kiocb_cancel_fn		*ki_cancel; 	struct io_event		ki_res; 	struct list_head	ki_list;	/* the aio core uses this						 * for cancellation */	refcount_t		ki_refcnt; 	/*	 * If the aio_resfd field of the userspace iocb is not zero,	 * this is the underlying eventfd context to deliver events to.	 */	struct eventfd_ctx	*ki_eventfd;}; /*------ sysctl variables----*/static DEFINE_SPINLOCK(aio_nr_lock);unsigned long aio_nr;		/* current system wide number of aio requests */unsigned long aio_max_nr = 0x10000; /* system wide maximum number of aio requests *//*----end sysctl variables---*/ static struct kmem_cache	*kiocb_cachep;static struct kmem_cache	*kioctx_cachep; static struct vfsmount *aio_mnt; static const struct file_operations aio_ring_fops;static const struct address_space_operations aio_ctx_aops; static struct file *aio_private_file(struct kioctx *ctx, loff_t nr_pages){	struct file *file;	struct inode *inode = alloc_anon_inode(aio_mnt->mnt_sb);	if (IS_ERR(inode))		return ERR_CAST(inode); 	inode->i_mapping->a_ops = &aio_ctx_aops;	inode->i_mapping->private_data = ctx;	inode->i_size = PAGE_SIZE * nr_pages; 	file = alloc_file_pseudo(inode, aio_mnt, "[aio]",				O_RDWR, &aio_ring_fops);	if (IS_ERR(file))		iput(inode);	return file;} static int aio_init_fs_context(struct fs_context *fc){	if (!init_pseudo(fc, AIO_RING_MAGIC))		return -ENOMEM;	fc->s_iflags |= SB_I_NOEXEC;	return 0;} /* aio_setup *	Creates the slab caches used by the aio routines, panic on *	failure as this is done early during the boot sequence. */static int __init aio_setup(void){	static struct file_system_type aio_fs = {		.name		= "aio",		.init_fs_context = aio_init_fs_context,		.kill_sb	= kill_anon_super,	};	aio_mnt = kern_mount(&aio_fs);	if (IS_ERR(aio_mnt))		panic("Failed to create aio fs mount."); 	kiocb_cachep = KMEM_CACHE(aio_kiocb, SLAB_HWCACHE_ALIGN|SLAB_PANIC);	kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC);	return 0;}__initcall(aio_setup); static void put_aio_ring_file(struct kioctx *ctx){	struct file *aio_ring_file = ctx->aio_ring_file;	struct address_space *i_mapping; 	if (aio_ring_file) {		truncate_setsize(file_inode(aio_ring_file), 0); 		/* Prevent further access to the kioctx from migratepages */		i_mapping = aio_ring_file->f_mapping;		spin_lock(&i_mapping->private_lock);		i_mapping->private_data = NULL;		ctx->aio_ring_file = NULL;		spin_unlock(&i_mapping->private_lock); 		fput(aio_ring_file);	}} static void aio_free_ring(struct kioctx *ctx){	int i; 	/* Disconnect the kiotx from the ring file.  This prevents future	 * accesses to the kioctx from page migration.	 */	put_aio_ring_file(ctx); 	for (i = 0; i < ctx->nr_pages; i++) {		struct page *page;		pr_debug("pid(%d) [%d] page->count=%d\n", current->pid, i,				page_count(ctx->ring_pages[i]));		page = ctx->ring_pages[i];		if (!page)			continue;		ctx->ring_pages[i] = NULL;		put_page(page);	} 	if (ctx->ring_pages && ctx->ring_pages != ctx->internal_pages) {		kfree(ctx->ring_pages);		ctx->ring_pages = NULL;	}} static int aio_ring_mremap(struct vm_area_struct *vma){	struct file *file = vma->vm_file;	struct mm_struct *mm = vma->vm_mm;	struct kioctx_table *table;	int i, res = -EINVAL; 	spin_lock(&mm->ioctx_lock);	rcu_read_lock();	table = rcu_dereference(mm->ioctx_table);	for (i = 0; i < table->nr; i++) {		struct kioctx *ctx; 		ctx = rcu_dereference(table->table[i]);		if (ctx && ctx->aio_ring_file == file) {			if (!atomic_read(&ctx->dead)) {				ctx->user_id = ctx->mmap_base = vma->vm_start;				res = 0;			}			break;		}	} 	rcu_read_unlock();	spin_unlock(&mm->ioctx_lock);	return res;} static const struct vm_operations_struct aio_ring_vm_ops = {	.mremap		= aio_ring_mremap,#if IS_ENABLED(CONFIG_MMU)	.fault		= filemap_fault,	.map_pages	= filemap_map_pages,	.page_mkwrite	= filemap_page_mkwrite,#endif}; static int aio_ring_mmap(struct file *file, struct vm_area_struct *vma){	vma->vm_flags |= VM_DONTEXPAND;	vma->vm_ops = &aio_ring_vm_ops;	return 0;} static const struct file_operations aio_ring_fops = {	.mmap = aio_ring_mmap,}; #if IS_ENABLED(CONFIG_MIGRATION)static int aio_migratepage(struct address_space *mapping, struct page *new,			struct page *old, enum migrate_mode mode){	struct kioctx *ctx;	unsigned long flags;	pgoff_t idx;	int rc; 	/*	 * We cannot support the _NO_COPY case here, because copy needs to	 * happen under the ctx->completion_lock. That does not work with the	 * migration workflow of MIGRATE_SYNC_NO_COPY.	 */	if (mode == MIGRATE_SYNC_NO_COPY)		return -EINVAL; 	rc = 0; 	/* mapping->private_lock here protects against the kioctx teardown.  */	spin_lock(&mapping->private_lock);	ctx = mapping->private_data;	if (!ctx) {		rc = -EINVAL;		goto out;	} 	/* The ring_lock mutex.  The prevents aio_read_events() from writing	 * to the ring's head, and prevents page migration from mucking in	 * a partially initialized kiotx.	 */	if (!mutex_trylock(&ctx->ring_lock)) {		rc = -EAGAIN;		goto out;	} 	idx = old->index;	if (idx < (pgoff_t)ctx->nr_pages) {		/* Make sure the old page hasn't already been changed */		if (ctx->ring_pages[idx] != old)			rc = -EAGAIN;	} else		rc = -EINVAL; 	if (rc != 0)		goto out_unlock; 	/* Writeback must be complete */	BUG_ON(PageWriteback(old));	get_page(new); 	rc = migrate_page_move_mapping(mapping, new, old, 1);	if (rc != MIGRATEPAGE_SUCCESS) {		put_page(new);		goto out_unlock;	} 	/* Take completion_lock to prevent other writes to the ring buffer	 * while the old page is copied to the new.  This prevents new	 * events from being lost.	 */	spin_lock_irqsave(&ctx->completion_lock, flags);	migrate_page_copy(new, old);	BUG_ON(ctx->ring_pages[idx] != old);	ctx->ring_pages[idx] = new;	spin_unlock_irqrestore(&ctx->completion_lock, flags); 	/* The old page is no longer accessible. */	put_page(old); out_unlock:	mutex_unlock(&ctx->ring_lock);out:	spin_unlock(&mapping->private_lock);	return rc;}#endif static const struct address_space_operations aio_ctx_aops = {	.set_page_dirty = __set_page_dirty_no_writeback,#if IS_ENABLED(CONFIG_MIGRATION)	.migratepage	= aio_migratepage,#endif}; static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events){	struct aio_ring *ring;	struct mm_struct *mm = current->mm;	unsigned long size, unused;	int nr_pages;	int i;	struct file *file; 	/* Compensate for the ring buffer's head/tail overlap entry */	nr_events += 2;	/* 1 is required, 2 for good luck */ 	size = sizeof(struct aio_ring);	size += sizeof(struct io_event) * nr_events; 	nr_pages = PFN_UP(size);	if (nr_pages < 0)		return -EINVAL; 	file = aio_private_file(ctx, nr_pages);	if (IS_ERR(file)) {		ctx->aio_ring_file = NULL;		return -ENOMEM;	} 	ctx->aio_ring_file = file;	nr_events = (PAGE_SIZE * nr_pages - sizeof(struct aio_ring))			/ sizeof(struct io_event); 	ctx->ring_pages = ctx->internal_pages;	if (nr_pages > AIO_RING_PAGES) {		ctx->ring_pages = kcalloc(nr_pages, sizeof(struct page *),					  GFP_KERNEL);		if (!ctx->ring_pages) {			put_aio_ring_file(ctx);			return -ENOMEM;		}	} 	for (i = 0; i < nr_pages; i++) {		struct page *page;

Lines 1–500 of 2278.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t