// SPDX-License-Identifier: GPL-2.0/* * Functions related to mapping data to requests */#include <linux/kernel.h>#include <linux/sched/task_stack.h>#include <linux/module.h>#include <linux/bio.h>#include <linux/blkdev.h>#include <linux/uio.h> #include "blk.h" struct bio_map_data {	bool is_our_pages : 1;	bool is_null_mapped : 1;	struct iov_iter iter;	struct iovec iov[];}; static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data,					       gfp_t gfp_mask){	struct bio_map_data *bmd; 	if (data->nr_segs > UIO_MAXIOV)		return NULL; 	bmd = kmalloc_flex(*bmd, iov, data->nr_segs, gfp_mask);	if (!bmd)		return NULL;	bmd->iter = *data;	if (iter_is_iovec(data)) {		memcpy(bmd->iov, iter_iov(data), sizeof(struct iovec) * data->nr_segs);		bmd->iter.__iov = bmd->iov;	}	return bmd;} static inline void blk_mq_map_bio_put(struct bio *bio){	bio_put(bio);} static struct bio *blk_rq_map_bio_alloc(struct request *rq,		unsigned int nr_vecs, gfp_t gfp_mask){	struct block_device *bdev = rq->q->disk ? rq->q->disk->part0 : NULL;	struct bio *bio; 	bio = bio_alloc_bioset(bdev, nr_vecs, rq->cmd_flags, gfp_mask,				&fs_bio_set);	if (!bio)		return NULL; 	return bio;} /** * bio_copy_from_iter - copy all pages from iov_iter to bio * @bio: The &struct bio which describes the I/O as destination * @iter: iov_iter as source * * Copy all pages from iov_iter to bio. * Returns 0 on success, or error on failure. */static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter){	struct bio_vec *bvec;	struct bvec_iter_all iter_all; 	bio_for_each_segment_all(bvec, bio, iter_all) {		ssize_t ret; 		ret = copy_page_from_iter(bvec->bv_page,					  bvec->bv_offset,					  bvec->bv_len,					  iter); 		if (!iov_iter_count(iter))			break; 		if (ret < bvec->bv_len)			return -EFAULT;	} 	return 0;} /** * bio_copy_to_iter - copy all pages from bio to iov_iter * @bio: The &struct bio which describes the I/O as source * @iter: iov_iter as destination * * Copy all pages from bio to iov_iter. * Returns 0 on success, or error on failure. */static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter){	struct bio_vec *bvec;	struct bvec_iter_all iter_all; 	bio_for_each_segment_all(bvec, bio, iter_all) {		ssize_t ret; 		ret = copy_page_to_iter(bvec->bv_page,					bvec->bv_offset,					bvec->bv_len,					&iter); 		if (!iov_iter_count(&iter))			break; 		if (ret < bvec->bv_len)			return -EFAULT;	} 	return 0;} /** *	bio_uncopy_user	-	finish previously mapped bio *	@bio: bio being terminated * *	Free pages allocated from bio_copy_user_iov() and write back data *	to user space in case of a read. */static int bio_uncopy_user(struct bio *bio){	struct bio_map_data *bmd = bio->bi_private;	int ret = 0; 	if (!bmd->is_null_mapped) {		/*		 * if we're in a workqueue, the request is orphaned, so		 * don't copy into a random user address space, just free		 * and return -EINTR so user space doesn't expect any data.		 */		if (!current->mm)			ret = -EINTR;		else if (bio_data_dir(bio) == READ)			ret = bio_copy_to_iter(bio, bmd->iter);		if (bmd->is_our_pages)			bio_free_pages(bio);	}	kfree(bmd);	return ret;} static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data,		struct iov_iter *iter, gfp_t gfp_mask){	struct bio_map_data *bmd;	struct page *page;	struct bio *bio;	int i = 0, ret;	int nr_pages;	unsigned int len = iter->count;	unsigned int offset = map_data ? offset_in_page(map_data->offset) : 0; 	bmd = bio_alloc_map_data(iter, gfp_mask);	if (!bmd)		return -ENOMEM; 	/*	 * We need to do a deep copy of the iov_iter including the iovecs.	 * The caller provided iov might point to an on-stack or otherwise	 * shortlived one.	 */	bmd->is_our_pages = !map_data;	bmd->is_null_mapped = (map_data && map_data->null_mapped); 	nr_pages = bio_max_segs(DIV_ROUND_UP(offset + len, PAGE_SIZE)); 	ret = -ENOMEM;	bio = blk_rq_map_bio_alloc(rq, nr_pages, gfp_mask);	if (!bio)		goto out_bmd; 	if (map_data) {		nr_pages = 1U << map_data->page_order;		i = map_data->offset / PAGE_SIZE;	}	while (len) {		unsigned int bytes = PAGE_SIZE; 		bytes -= offset; 		if (bytes > len)			bytes = len; 		if (map_data) {			if (i == map_data->nr_entries * nr_pages) {				ret = -ENOMEM;				goto cleanup;			} 			page = map_data->pages[i / nr_pages];			page += (i % nr_pages); 			i++;		} else {			page = alloc_page(GFP_NOIO | gfp_mask);			if (!page) {				ret = -ENOMEM;				goto cleanup;			}		} 		if (bio_add_page(bio, page, bytes, offset) < bytes) {			if (!map_data)				__free_page(page);			break;		} 		len -= bytes;		offset = 0;	} 	if (map_data)		map_data->offset += bio->bi_iter.bi_size; 	/*	 * success	 */	if (iov_iter_rw(iter) == WRITE &&	     (!map_data || !map_data->null_mapped)) {		ret = bio_copy_from_iter(bio, iter);		if (ret)			goto cleanup;	} else if (map_data && map_data->from_user) {		struct iov_iter iter2 = *iter; 		/* This is the copy-in part of SG_DXFER_TO_FROM_DEV. */		iter2.data_source = ITER_SOURCE;		ret = bio_copy_from_iter(bio, &iter2);		if (ret)			goto cleanup;	} else {		if (bmd->is_our_pages)			zero_fill_bio(bio);		iov_iter_advance(iter, bio->bi_iter.bi_size);	} 	bio->bi_private = bmd; 	ret = blk_rq_append_bio(rq, bio);	if (ret)		goto cleanup;	return 0;cleanup:	if (!map_data)		bio_free_pages(bio);	blk_mq_map_bio_put(bio);out_bmd:	kfree(bmd);	return ret;} static int bio_map_user_iov(struct request *rq, struct iov_iter *iter,		gfp_t gfp_mask){	unsigned int nr_vecs = iov_iter_npages(iter, BIO_MAX_VECS);	struct bio *bio;	int ret; 	if (!iov_iter_count(iter))		return -EINVAL; 	bio = blk_rq_map_bio_alloc(rq, nr_vecs, gfp_mask);	if (!bio)		return -ENOMEM;	/*	 * No alignment requirements on our part to support arbitrary	 * passthrough commands.	 */	ret = bio_iov_iter_get_pages(bio, iter, 0, 0);	if (ret)		goto out_put;	ret = blk_rq_append_bio(rq, bio);	if (ret)		goto out_release;	return 0; out_release:	bio_release_pages(bio, false);out_put:	blk_mq_map_bio_put(bio);	return ret;} static void bio_invalidate_vmalloc_pages(struct bio *bio){#ifdef ARCH_IMPLEMENTS_FLUSH_KERNEL_VMAP_RANGE	if (bio->bi_private && !op_is_write(bio_op(bio))) {		unsigned long i, len = 0; 		for (i = 0; i < bio->bi_vcnt; i++)			len += bio->bi_io_vec[i].bv_len;		invalidate_kernel_vmap_range(bio->bi_private, len);	}#endif} static void bio_map_kern_endio(struct bio *bio){	bio_invalidate_vmalloc_pages(bio);	blk_mq_map_bio_put(bio);} static struct bio *bio_map_kern(struct request *rq, void *data, unsigned int len,		gfp_t gfp_mask){	unsigned int nr_vecs = bio_add_max_vecs(data, len);	struct bio *bio; 	bio = blk_rq_map_bio_alloc(rq, nr_vecs, gfp_mask);	if (!bio)		return ERR_PTR(-ENOMEM); 	if (is_vmalloc_addr(data)) {		bio->bi_private = data;		if (!bio_add_vmalloc(bio, data, len)) {			blk_mq_map_bio_put(bio);			return ERR_PTR(-EINVAL);		}	} else {		bio_add_virt_nofail(bio, data, len);	}	bio->bi_end_io = bio_map_kern_endio;	return bio;} static void bio_copy_kern_endio(struct bio *bio){	bio_free_pages(bio);	blk_mq_map_bio_put(bio);} static void bio_copy_kern_endio_read(struct bio *bio){	char *p = bio->bi_private;	struct bio_vec *bvec;	struct bvec_iter_all iter_all; 	bio_for_each_segment_all(bvec, bio, iter_all) {		memcpy_from_bvec(p, bvec);		p += bvec->bv_len;	} 	bio_copy_kern_endio(bio);} /** *	bio_copy_kern	-	copy kernel address into bio *	@rq: request to fill *	@data: pointer to buffer to copy *	@len: length in bytes *	@op: bio/request operation *	@gfp_mask: allocation flags for bio and page allocation * *	copy the kernel address into a bio suitable for io to a block *	device. Returns an error pointer in case of error. */static struct bio *bio_copy_kern(struct request *rq, void *data, unsigned int len,		gfp_t gfp_mask){	enum req_op op = req_op(rq);	unsigned long kaddr = (unsigned long)data;	unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;	unsigned long start = kaddr >> PAGE_SHIFT;	struct bio *bio;	void *p = data;	int nr_pages = 0; 	/*	 * Overflow, abort	 */	if (end < start)		return ERR_PTR(-EINVAL); 	nr_pages = end - start;	bio = blk_rq_map_bio_alloc(rq, nr_pages, gfp_mask);	if (!bio)		return ERR_PTR(-ENOMEM); 	while (len) {		struct page *page;		unsigned int bytes = PAGE_SIZE; 		if (bytes > len)			bytes = len; 		page = alloc_page(GFP_NOIO | __GFP_ZERO | gfp_mask);		if (!page)			goto cleanup; 		if (op_is_write(op))			memcpy(page_address(page), p, bytes); 		__bio_add_page(bio, page, bytes, 0); 		len -= bytes;		p += bytes;	} 	if (op_is_write(op)) {		bio->bi_end_io = bio_copy_kern_endio;	} else {		bio->bi_end_io = bio_copy_kern_endio_read;		bio->bi_private = data;	} 	return bio; cleanup:	bio_free_pages(bio);	blk_mq_map_bio_put(bio);	return ERR_PTR(-ENOMEM);} /* * Append a bio to a passthrough request.  Only works if the bio can be merged * into the request based on the driver constraints. */int blk_rq_append_bio(struct request *rq, struct bio *bio){	const struct queue_limits *lim = &rq->q->limits;	unsigned int max_bytes = lim->max_hw_sectors << SECTOR_SHIFT;	unsigned int nr_segs = 0;	int ret; 	/* check that the data layout matches the hardware restrictions */	ret = bio_split_io_at(bio, lim, &nr_segs, max_bytes, 0);	if (ret) {		/* if we would have to split the bio, copy instead */		if (ret > 0)			ret = -EREMOTEIO;		return ret;	} 	if (rq->bio) {		if (!ll_back_merge_fn(rq, bio, nr_segs))			return -EINVAL;		rq->phys_gap_bit = bio_seg_gap(rq->q, rq->biotail, bio,					       rq->phys_gap_bit);		rq->biotail->bi_next = bio;		rq->biotail = bio;		rq->__data_len += bio->bi_iter.bi_size;		bio_crypt_free_ctx(bio);		return 0;	} 	rq->nr_phys_segments = nr_segs;	rq->bio = rq->biotail = bio;	rq->__data_len = bio->bi_iter.bi_size;	rq->phys_gap_bit = bio->bi_bvec_gap_bit;	return 0;}EXPORT_SYMBOL(blk_rq_append_bio); /* Prepare bio for passthrough IO given ITER_BVEC iter */static int blk_rq_map_user_bvec(struct request *rq, const struct iov_iter *iter){	unsigned int max_bytes = rq->q->limits.max_hw_sectors << SECTOR_SHIFT;	struct bio *bio;	int ret; 	if (!iov_iter_count(iter) || iov_iter_count(iter) > max_bytes)		return -EINVAL; 	/* reuse the bvecs from the iterator instead of allocating new ones */	bio = blk_rq_map_bio_alloc(rq, 0, GFP_KERNEL);	if (!bio)		return -ENOMEM;	bio_iov_iter_set(bio, iter); 	ret = blk_rq_append_bio(rq, bio);	if (ret)		blk_mq_map_bio_put(bio);	return ret;} /** * blk_rq_map_user_iov - map user data to a request, for passthrough requests * @q:		request queue where request should be inserted * @rq:		request to map data to * @map_data:   pointer to the rq_map_data holding pages (if necessary) * @iter:	iovec iterator * @gfp_mask:	memory allocation flags * * Description: *    Data will be mapped directly for zero copy I/O, if possible. Otherwise *    a kernel bounce buffer is used. * *    A matching blk_rq_unmap_user() must be issued at the end of I/O, while *    still in process context. */int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,			struct rq_map_data *map_data,

Lines 1–500 of 682. Load more