// SPDX-License-Identifier: GPL-2.0-only/* * Copyright (C) 1991, 1992  Linus Torvalds * Copyright (C) 2001  Andrea Arcangeli <andrea@suse.de> SuSE * Copyright (C) 2016 - 2020 Christoph Hellwig */#include <linux/init.h>#include <linux/mm.h>#include <linux/blkdev.h>#include <linux/blk-integrity.h>#include <linux/buffer_head.h>#include <linux/mpage.h>#include <linux/uio.h>#include <linux/namei.h>#include <linux/task_io_accounting_ops.h>#include <linux/falloc.h>#include <linux/suspend.h>#include <linux/fs.h>#include <linux/iomap.h>#include <linux/module.h>#include <linux/io_uring/cmd.h>#include "blk.h" static inline struct inode *bdev_file_inode(struct file *file){	return file->f_mapping->host;} static blk_opf_t dio_bio_write_op(struct kiocb *iocb){	blk_opf_t opf = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE; 	/* avoid the need for a I/O completion work item */	if (iocb_is_dsync(iocb))		opf |= REQ_FUA;	return opf;} static bool blkdev_dio_invalid(struct block_device *bdev, struct kiocb *iocb,				struct iov_iter *iter){	return (iocb->ki_pos | iov_iter_count(iter)) &			(bdev_logical_block_size(bdev) - 1);} static inline int blkdev_iov_iter_get_pages(struct bio *bio,		struct iov_iter *iter, struct block_device *bdev){	return bio_iov_iter_get_pages(bio, iter, bdev_dma_alignment(bdev),			bdev_logical_block_size(bdev) - 1);} #define DIO_INLINE_BIO_VECS 4 static ssize_t __blkdev_direct_IO_simple(struct kiocb *iocb,		struct iov_iter *iter, struct block_device *bdev,		unsigned int nr_pages){	struct bio_vec inline_vecs[DIO_INLINE_BIO_VECS], *vecs;	loff_t pos = iocb->ki_pos;	bool should_dirty = false;	struct bio bio;	ssize_t ret; 	if (nr_pages <= DIO_INLINE_BIO_VECS)		vecs = inline_vecs;	else {		vecs = kmalloc_objs(struct bio_vec, nr_pages);		if (!vecs)			return -ENOMEM;	} 	if (iov_iter_rw(iter) == READ) {		bio_init(&bio, bdev, vecs, nr_pages, REQ_OP_READ);		if (user_backed_iter(iter))			should_dirty = true;	} else {		bio_init(&bio, bdev, vecs, nr_pages, dio_bio_write_op(iocb));	}	bio.bi_iter.bi_sector = pos >> SECTOR_SHIFT;	bio.bi_write_hint = file_inode(iocb->ki_filp)->i_write_hint;	bio.bi_write_stream = iocb->ki_write_stream;	bio.bi_ioprio = iocb->ki_ioprio;	if (iocb->ki_flags & IOCB_ATOMIC)		bio.bi_opf |= REQ_ATOMIC; 	ret = blkdev_iov_iter_get_pages(&bio, iter, bdev);	if (unlikely(ret))		goto out;	ret = bio.bi_iter.bi_size; 	if (iov_iter_rw(iter) == WRITE)		task_io_account_write(ret); 	if (iocb->ki_flags & IOCB_NOWAIT)		bio.bi_opf |= REQ_NOWAIT; 	submit_bio_wait(&bio); 	bio_release_pages(&bio, should_dirty);	if (unlikely(bio.bi_status))		ret = blk_status_to_errno(bio.bi_status); out:	if (vecs != inline_vecs)		kfree(vecs); 	bio_uninit(&bio); 	return ret;} enum {	DIO_SHOULD_DIRTY	= 1,	DIO_IS_SYNC		= 2,}; struct blkdev_dio {	union {		struct kiocb		*iocb;		struct task_struct	*waiter;	};	size_t			size;	atomic_t		ref;	unsigned int		flags;	struct bio		bio ____cacheline_aligned_in_smp;}; static struct bio_set blkdev_dio_pool; static void blkdev_bio_end_io(struct bio *bio){	struct blkdev_dio *dio = bio->bi_private;	bool should_dirty = dio->flags & DIO_SHOULD_DIRTY;	bool is_sync = dio->flags & DIO_IS_SYNC; 	if (bio->bi_status && !dio->bio.bi_status)		dio->bio.bi_status = bio->bi_status; 	if (bio_integrity(bio))		bio_integrity_unmap_user(bio); 	if (atomic_dec_and_test(&dio->ref)) {		if (!is_sync) {			struct kiocb *iocb = dio->iocb;			ssize_t ret; 			WRITE_ONCE(iocb->private, NULL); 			if (likely(!dio->bio.bi_status)) {				ret = dio->size;				iocb->ki_pos += ret;			} else {				ret = blk_status_to_errno(dio->bio.bi_status);			} 			dio->iocb->ki_complete(iocb, ret);			bio_put(&dio->bio);		} else {			struct task_struct *waiter = dio->waiter; 			WRITE_ONCE(dio->waiter, NULL);			blk_wake_io_task(waiter);		}	} 	if (should_dirty) {		bio_check_pages_dirty(bio);	} else {		bio_release_pages(bio, false);		bio_put(bio);	}} static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter,		struct block_device *bdev, unsigned int nr_pages){	struct blk_plug plug;	struct blkdev_dio *dio;	struct bio *bio;	bool is_read = (iov_iter_rw(iter) == READ), is_sync;	blk_opf_t opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb);	loff_t pos = iocb->ki_pos;	int ret = 0; 	bio = bio_alloc_bioset(bdev, nr_pages, opf, GFP_KERNEL,			       &blkdev_dio_pool);	dio = container_of(bio, struct blkdev_dio, bio);	atomic_set(&dio->ref, 1);	/*	 * Grab an extra reference to ensure the dio structure which is embedded	 * into the first bio stays around.	 */	bio_get(bio); 	is_sync = is_sync_kiocb(iocb);	if (is_sync) {		dio->flags = DIO_IS_SYNC;		dio->waiter = current;	} else {		dio->flags = 0;		dio->iocb = iocb;	} 	dio->size = 0;	if (is_read && user_backed_iter(iter))		dio->flags |= DIO_SHOULD_DIRTY; 	blk_start_plug(&plug); 	for (;;) {		bio->bi_iter.bi_sector = pos >> SECTOR_SHIFT;		bio->bi_write_hint = file_inode(iocb->ki_filp)->i_write_hint;		bio->bi_write_stream = iocb->ki_write_stream;		bio->bi_private = dio;		bio->bi_end_io = blkdev_bio_end_io;		bio->bi_ioprio = iocb->ki_ioprio; 		ret = blkdev_iov_iter_get_pages(bio, iter, bdev);		if (unlikely(ret)) {			bio_endio_status(bio, errno_to_blk_status(ret));			break;		}		if (iocb->ki_flags & IOCB_NOWAIT) {			/*			 * This is nonblocking IO, and we need to allocate			 * another bio if we have data left to map. As we			 * cannot guarantee that one of the sub bios will not			 * fail getting issued FOR NOWAIT and as error results			 * are coalesced across all of them, be safe and ask for			 * a retry of this from blocking context.			 */			if (unlikely(iov_iter_count(iter))) {				ret = -EAGAIN;				goto fail;			}			bio->bi_opf |= REQ_NOWAIT;		}		if (iocb->ki_flags & IOCB_HAS_METADATA) {			ret = bio_integrity_map_iter(bio, iocb->private);			if (unlikely(ret)) {				bio_endio_status(bio, errno_to_blk_status(ret));				break;			}		} 		if (is_read) {			if (dio->flags & DIO_SHOULD_DIRTY)				bio_set_pages_dirty(bio);		} else {			task_io_account_write(bio->bi_iter.bi_size);		}		dio->size += bio->bi_iter.bi_size;		pos += bio->bi_iter.bi_size; 		nr_pages = bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS);		if (!nr_pages) {			submit_bio(bio);			break;		}		atomic_inc(&dio->ref);		submit_bio(bio);		bio = bio_alloc(bdev, nr_pages, opf, GFP_KERNEL);	} 	blk_finish_plug(&plug); 	if (!is_sync)		return -EIOCBQUEUED; 	for (;;) {		set_current_state(TASK_UNINTERRUPTIBLE);		if (!READ_ONCE(dio->waiter))			break;		blk_io_schedule();	}	__set_current_state(TASK_RUNNING); 	if (!ret)		ret = blk_status_to_errno(dio->bio.bi_status);	if (likely(!ret))		ret = dio->size; 	bio_put(&dio->bio);	return ret;fail:	bio_release_pages(bio, false);	bio_clear_flag(bio, BIO_REFFED);	bio_put(bio);	blk_finish_plug(&plug);	return ret;} static void blkdev_bio_end_io_async(struct bio *bio){	struct blkdev_dio *dio = container_of(bio, struct blkdev_dio, bio);	struct kiocb *iocb = dio->iocb;	ssize_t ret; 	WRITE_ONCE(iocb->private, NULL); 	if (likely(!bio->bi_status)) {		ret = dio->size;		iocb->ki_pos += ret;	} else {		ret = blk_status_to_errno(bio->bi_status);	} 	if (bio_integrity(bio))		bio_integrity_unmap_user(bio); 	iocb->ki_complete(iocb, ret); 	if (dio->flags & DIO_SHOULD_DIRTY) {		bio_check_pages_dirty(bio);	} else {		bio_release_pages(bio, false);		bio_put(bio);	}} static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb,					struct iov_iter *iter,					struct block_device *bdev,					unsigned int nr_pages){	bool is_read = iov_iter_rw(iter) == READ;	blk_opf_t opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb);	struct blkdev_dio *dio;	struct bio *bio;	loff_t pos = iocb->ki_pos;	int ret = 0; 	bio = bio_alloc_bioset(bdev, nr_pages, opf, GFP_KERNEL,			       &blkdev_dio_pool);	dio = container_of(bio, struct blkdev_dio, bio);	dio->flags = 0;	dio->iocb = iocb;	bio->bi_iter.bi_sector = pos >> SECTOR_SHIFT;	bio->bi_write_hint = file_inode(iocb->ki_filp)->i_write_hint;	bio->bi_write_stream = iocb->ki_write_stream;	bio->bi_end_io = blkdev_bio_end_io_async;	bio->bi_ioprio = iocb->ki_ioprio; 	/*	 * Users don't rely on the iterator being in any particular	 * state for async I/O returning -EIOCBQUEUED, hence we can	 * avoid expensive iov_iter_advance(). Bypass	 * bio_iov_iter_get_pages() and set the bvec directly.	 */	if (!bio_iov_iter_set(bio, iter)) {		ret = blkdev_iov_iter_get_pages(bio, iter, bdev);		if (unlikely(ret))			goto out_bio_put;	}	dio->size = bio->bi_iter.bi_size; 	if (is_read) {		if (user_backed_iter(iter)) {			dio->flags |= DIO_SHOULD_DIRTY;			bio_set_pages_dirty(bio);		}	} else {		task_io_account_write(bio->bi_iter.bi_size);	} 	if (iocb->ki_flags & IOCB_HAS_METADATA) {		ret = bio_integrity_map_iter(bio, iocb->private);		WRITE_ONCE(iocb->private, NULL);		if (unlikely(ret))			goto out_bio_put;	} 	if (iocb->ki_flags & IOCB_ATOMIC)		bio->bi_opf |= REQ_ATOMIC; 	if (iocb->ki_flags & IOCB_NOWAIT)		bio->bi_opf |= REQ_NOWAIT; 	if (iocb->ki_flags & IOCB_HIPRI) {		bio->bi_opf |= REQ_POLLED;		submit_bio(bio);		WRITE_ONCE(iocb->private, bio);	} else {		submit_bio(bio);	}	return -EIOCBQUEUED; out_bio_put:	bio_put(bio);	return ret;} static ssize_t blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter){	struct block_device *bdev = I_BDEV(iocb->ki_filp->f_mapping->host);	unsigned int nr_pages; 	if (!iov_iter_count(iter))		return 0; 	if (blkdev_dio_invalid(bdev, iocb, iter))		return -EINVAL; 	if (iov_iter_rw(iter) == WRITE) {		u16 max_write_streams = bdev_max_write_streams(bdev); 		if (iocb->ki_write_stream) {			if (iocb->ki_write_stream > max_write_streams)				return -EINVAL;		} else if (max_write_streams) {			enum rw_hint write_hint =				file_inode(iocb->ki_filp)->i_write_hint; 			/*			 * Just use the write hint as write stream for block			 * device writes.  This assumes no file system is			 * mounted that would use the streams differently.			 */			if (write_hint <= max_write_streams)				iocb->ki_write_stream = write_hint;		}	} 	nr_pages = bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS + 1);	if (likely(nr_pages <= BIO_MAX_VECS &&		   !(iocb->ki_flags & IOCB_HAS_METADATA))) {		if (is_sync_kiocb(iocb))			return __blkdev_direct_IO_simple(iocb, iter, bdev,							nr_pages);		return __blkdev_direct_IO_async(iocb, iter, bdev, nr_pages);	} else if (iocb->ki_flags & IOCB_ATOMIC) {		return -EINVAL;	}	return __blkdev_direct_IO(iocb, iter, bdev, bio_max_segs(nr_pages));} static int blkdev_iomap_begin(struct inode *inode, loff_t offset, loff_t length,		unsigned int flags, struct iomap *iomap, struct iomap *srcmap){	struct block_device *bdev = I_BDEV(inode);	loff_t isize = i_size_read(inode); 	if (offset >= isize)		return -EIO; 	iomap->bdev = bdev;	iomap->offset = ALIGN_DOWN(offset, bdev_logical_block_size(bdev));	iomap->type = IOMAP_MAPPED;	iomap->addr = iomap->offset;	iomap->length = isize - iomap->offset;	iomap->flags |= IOMAP_F_BUFFER_HEAD; /* noop for !CONFIG_BUFFER_HEAD */	return 0;} static DEFINE_IOMAP_ITER_NEXT(blkdev_iomap_next, blkdev_iomap_begin); static const struct iomap_ops blkdev_iomap_ops = {	.iomap_next		= blkdev_iomap_next,}; #ifdef CONFIG_BUFFER_HEADstatic int blkdev_get_block(struct inode *inode, sector_t iblock,		struct buffer_head *bh, int create){	bh->b_bdev = I_BDEV(inode);	bh->b_blocknr = iblock;	set_buffer_mapped(bh);	return 0;} /* * We cannot call mpage_writepages() as it does not take the buffer lock. * We must use block_write_full_folio() directly which holds the buffer * lock.  The buffer lock provides the synchronisation with writeback * that filesystems rely on when they use the blockdev's mapping. */static int blkdev_writepages(struct address_space *mapping,		struct writeback_control *wbc){	struct folio *folio = NULL;	struct blk_plug plug;	int err; 	blk_start_plug(&plug);	while ((folio = writeback_iter(mapping, wbc, folio, &err)))		err = block_write_full_folio(folio, wbc, blkdev_get_block);	blk_finish_plug(&plug); 	return err;} static int blkdev_read_folio(struct file *file, struct folio *folio){	return block_read_full_folio(folio, blkdev_get_block);} static void blkdev_readahead(struct readahead_control *rac){	mpage_readahead(rac, blkdev_get_block);

Lines 1–500 of 955. Load more