#include <linux/export.h>#include <linux/bvec.h>#include <linux/uio.h>#include <linux/pagemap.h>#include <linux/slab.h>#include <linux/vmalloc.h>#include <linux/splice.h>#include <net/checksum.h> #define PIPE_PARANOIA /* for now */ #define iterate_iovec(i, n, __v, __p, skip, STEP) {	\	size_t left;					\	size_t wanted = n;				\	__p = i->iov;					\	__v.iov_len = min(n, __p->iov_len - skip);	\	if (likely(__v.iov_len)) {			\		__v.iov_base = __p->iov_base + skip;	\		left = (STEP);				\		__v.iov_len -= left;			\		skip += __v.iov_len;			\		n -= __v.iov_len;			\	} else {					\		left = 0;				\	}						\	while (unlikely(!left && n)) {			\		__p++;					\		__v.iov_len = min(n, __p->iov_len);	\		if (unlikely(!__v.iov_len))		\			continue;			\		__v.iov_base = __p->iov_base;		\		left = (STEP);				\		__v.iov_len -= left;			\		skip = __v.iov_len;			\		n -= __v.iov_len;			\	}						\	n = wanted - n;					\} #define iterate_kvec(i, n, __v, __p, skip, STEP) {	\	size_t wanted = n;				\	__p = i->kvec;					\	__v.iov_len = min(n, __p->iov_len - skip);	\	if (likely(__v.iov_len)) {			\		__v.iov_base = __p->iov_base + skip;	\		(void)(STEP);				\		skip += __v.iov_len;			\		n -= __v.iov_len;			\	}						\	while (unlikely(n)) {				\		__p++;					\		__v.iov_len = min(n, __p->iov_len);	\		if (unlikely(!__v.iov_len))		\			continue;			\		__v.iov_base = __p->iov_base;		\		(void)(STEP);				\		skip = __v.iov_len;			\		n -= __v.iov_len;			\	}						\	n = wanted;					\} #define iterate_bvec(i, n, __v, __bi, skip, STEP) {	\	struct bvec_iter __start;			\	__start.bi_size = n;				\	__start.bi_bvec_done = skip;			\	__start.bi_idx = 0;				\	for_each_bvec(__v, i->bvec, __bi, __start) {	\		if (!__v.bv_len)			\			continue;			\		(void)(STEP);				\	}						\} #define iterate_all_kinds(i, n, v, I, B, K) {			\	if (likely(n)) {					\		size_t skip = i->iov_offset;			\		if (unlikely(i->type & ITER_BVEC)) {		\			struct bio_vec v;			\			struct bvec_iter __bi;			\			iterate_bvec(i, n, v, __bi, skip, (B))	\		} else if (unlikely(i->type & ITER_KVEC)) {	\			const struct kvec *kvec;		\			struct kvec v;				\			iterate_kvec(i, n, v, kvec, skip, (K))	\		} else {					\			const struct iovec *iov;		\			struct iovec v;				\			iterate_iovec(i, n, v, iov, skip, (I))	\		}						\	}							\} #define iterate_and_advance(i, n, v, I, B, K) {			\	if (unlikely(i->count < n))				\		n = i->count;					\	if (i->count) {						\		size_t skip = i->iov_offset;			\		if (unlikely(i->type & ITER_BVEC)) {		\			const struct bio_vec *bvec = i->bvec;	\			struct bio_vec v;			\			struct bvec_iter __bi;			\			iterate_bvec(i, n, v, __bi, skip, (B))	\			i->bvec = __bvec_iter_bvec(i->bvec, __bi);	\			i->nr_segs -= i->bvec - bvec;		\			skip = __bi.bi_bvec_done;		\		} else if (unlikely(i->type & ITER_KVEC)) {	\			const struct kvec *kvec;		\			struct kvec v;				\			iterate_kvec(i, n, v, kvec, skip, (K))	\			if (skip == kvec->iov_len) {		\				kvec++;				\				skip = 0;			\			}					\			i->nr_segs -= kvec - i->kvec;		\			i->kvec = kvec;				\		} else {					\			const struct iovec *iov;		\			struct iovec v;				\			iterate_iovec(i, n, v, iov, skip, (I))	\			if (skip == iov->iov_len) {		\				iov++;				\				skip = 0;			\			}					\			i->nr_segs -= iov - i->iov;		\			i->iov = iov;				\		}						\		i->count -= n;					\		i->iov_offset = skip;				\	}							\} static int copyout(void __user *to, const void *from, size_t n){	if (access_ok(VERIFY_WRITE, to, n)) {		kasan_check_read(from, n);		n = raw_copy_to_user(to, from, n);	}	return n;} static int copyin(void *to, const void __user *from, size_t n){	if (access_ok(VERIFY_READ, from, n)) {		kasan_check_write(to, n);		n = raw_copy_from_user(to, from, n);	}	return n;} static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,			 struct iov_iter *i){	size_t skip, copy, left, wanted;	const struct iovec *iov;	char __user *buf;	void *kaddr, *from; 	if (unlikely(bytes > i->count))		bytes = i->count; 	if (unlikely(!bytes))		return 0; 	might_fault();	wanted = bytes;	iov = i->iov;	skip = i->iov_offset;	buf = iov->iov_base + skip;	copy = min(bytes, iov->iov_len - skip); 	if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_writeable(buf, copy)) {		kaddr = kmap_atomic(page);		from = kaddr + offset; 		/* first chunk, usually the only one */		left = copyout(buf, from, copy);		copy -= left;		skip += copy;		from += copy;		bytes -= copy; 		while (unlikely(!left && bytes)) {			iov++;			buf = iov->iov_base;			copy = min(bytes, iov->iov_len);			left = copyout(buf, from, copy);			copy -= left;			skip = copy;			from += copy;			bytes -= copy;		}		if (likely(!bytes)) {			kunmap_atomic(kaddr);			goto done;		}		offset = from - kaddr;		buf += copy;		kunmap_atomic(kaddr);		copy = min(bytes, iov->iov_len - skip);	}	/* Too bad - revert to non-atomic kmap */ 	kaddr = kmap(page);	from = kaddr + offset;	left = copyout(buf, from, copy);	copy -= left;	skip += copy;	from += copy;	bytes -= copy;	while (unlikely(!left && bytes)) {		iov++;		buf = iov->iov_base;		copy = min(bytes, iov->iov_len);		left = copyout(buf, from, copy);		copy -= left;		skip = copy;		from += copy;		bytes -= copy;	}	kunmap(page); done:	if (skip == iov->iov_len) {		iov++;		skip = 0;	}	i->count -= wanted - bytes;	i->nr_segs -= iov - i->iov;	i->iov = iov;	i->iov_offset = skip;	return wanted - bytes;} static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,			 struct iov_iter *i){	size_t skip, copy, left, wanted;	const struct iovec *iov;	char __user *buf;	void *kaddr, *to; 	if (unlikely(bytes > i->count))		bytes = i->count; 	if (unlikely(!bytes))		return 0; 	might_fault();	wanted = bytes;	iov = i->iov;	skip = i->iov_offset;	buf = iov->iov_base + skip;	copy = min(bytes, iov->iov_len - skip); 	if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_readable(buf, copy)) {		kaddr = kmap_atomic(page);		to = kaddr + offset; 		/* first chunk, usually the only one */		left = copyin(to, buf, copy);		copy -= left;		skip += copy;		to += copy;		bytes -= copy; 		while (unlikely(!left && bytes)) {			iov++;			buf = iov->iov_base;			copy = min(bytes, iov->iov_len);			left = copyin(to, buf, copy);			copy -= left;			skip = copy;			to += copy;			bytes -= copy;		}		if (likely(!bytes)) {			kunmap_atomic(kaddr);			goto done;		}		offset = to - kaddr;		buf += copy;		kunmap_atomic(kaddr);		copy = min(bytes, iov->iov_len - skip);	}	/* Too bad - revert to non-atomic kmap */ 	kaddr = kmap(page);	to = kaddr + offset;	left = copyin(to, buf, copy);	copy -= left;	skip += copy;	to += copy;	bytes -= copy;	while (unlikely(!left && bytes)) {		iov++;		buf = iov->iov_base;		copy = min(bytes, iov->iov_len);		left = copyin(to, buf, copy);		copy -= left;		skip = copy;		to += copy;		bytes -= copy;	}	kunmap(page); done:	if (skip == iov->iov_len) {		iov++;		skip = 0;	}	i->count -= wanted - bytes;	i->nr_segs -= iov - i->iov;	i->iov = iov;	i->iov_offset = skip;	return wanted - bytes;} #ifdef PIPE_PARANOIAstatic bool sanity(const struct iov_iter *i){	struct pipe_inode_info *pipe = i->pipe;	int idx = i->idx;	int next = pipe->curbuf + pipe->nrbufs;	if (i->iov_offset) {		struct pipe_buffer *p;		if (unlikely(!pipe->nrbufs))			goto Bad;	// pipe must be non-empty		if (unlikely(idx != ((next - 1) & (pipe->buffers - 1))))			goto Bad;	// must be at the last buffer... 		p = &pipe->bufs[idx];		if (unlikely(p->offset + p->len != i->iov_offset))			goto Bad;	// ... at the end of segment	} else {		if (idx != (next & (pipe->buffers - 1)))			goto Bad;	// must be right after the last buffer	}	return true;Bad:	printk(KERN_ERR "idx = %d, offset = %zd\n", i->idx, i->iov_offset);	printk(KERN_ERR "curbuf = %d, nrbufs = %d, buffers = %d\n",			pipe->curbuf, pipe->nrbufs, pipe->buffers);	for (idx = 0; idx < pipe->buffers; idx++)		printk(KERN_ERR "[%p %p %d %d]\n",			pipe->bufs[idx].ops,			pipe->bufs[idx].page,			pipe->bufs[idx].offset,			pipe->bufs[idx].len);	WARN_ON(1);	return false;}#else#define sanity(i) true#endif static inline int next_idx(int idx, struct pipe_inode_info *pipe){	return (idx + 1) & (pipe->buffers - 1);} static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,			 struct iov_iter *i){	struct pipe_inode_info *pipe = i->pipe;	struct pipe_buffer *buf;	size_t off;	int idx; 	if (unlikely(bytes > i->count))		bytes = i->count; 	if (unlikely(!bytes))		return 0; 	if (!sanity(i))		return 0; 	off = i->iov_offset;	idx = i->idx;	buf = &pipe->bufs[idx];	if (off) {		if (offset == off && buf->page == page) {			/* merge with the last one */			buf->len += bytes;			i->iov_offset += bytes;			goto out;		}		idx = next_idx(idx, pipe);		buf = &pipe->bufs[idx];	}	if (idx == pipe->curbuf && pipe->nrbufs)		return 0;	pipe->nrbufs++;	buf->ops = &page_cache_pipe_buf_ops;	get_page(buf->page = page);	buf->offset = offset;	buf->len = bytes;	i->iov_offset = offset + bytes;	i->idx = idx;out:	i->count -= bytes;	return bytes;} /* * Fault in one or more iovecs of the given iov_iter, to a maximum length of * bytes.  For each iovec, fault in each page that constitutes the iovec. * * Return 0 on success, or non-zero if the memory could not be accessed (i.e. * because it is an invalid address). */int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes){	size_t skip = i->iov_offset;	const struct iovec *iov;	int err;	struct iovec v; 	if (!(i->type & (ITER_BVEC|ITER_KVEC))) {		iterate_iovec(i, bytes, v, iov, skip, ({			err = fault_in_pages_readable(v.iov_base, v.iov_len);			if (unlikely(err))			return err;		0;}))	}	return 0;}EXPORT_SYMBOL(iov_iter_fault_in_readable); void iov_iter_init(struct iov_iter *i, int direction,			const struct iovec *iov, unsigned long nr_segs,			size_t count){	/* It will get better.  Eventually... */	if (uaccess_kernel()) {		direction |= ITER_KVEC;		i->type = direction;		i->kvec = (struct kvec *)iov;	} else {		i->type = direction;		i->iov = iov;	}	i->nr_segs = nr_segs;	i->iov_offset = 0;	i->count = count;}EXPORT_SYMBOL(iov_iter_init); static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len){	char *from = kmap_atomic(page);	memcpy(to, from + offset, len);	kunmap_atomic(from);} static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len){	char *to = kmap_atomic(page);	memcpy(to + offset, from, len);	kunmap_atomic(to);} static void memzero_page(struct page *page, size_t offset, size_t len){	char *addr = kmap_atomic(page);	memset(addr + offset, 0, len);	kunmap_atomic(addr);} static inline bool allocated(struct pipe_buffer *buf){	return buf->ops == &default_pipe_buf_ops;} static inline void data_start(const struct iov_iter *i, int *idxp, size_t *offp){	size_t off = i->iov_offset;	int idx = i->idx;	if (off && (!allocated(&i->pipe->bufs[idx]) || off == PAGE_SIZE)) {		idx = next_idx(idx, i->pipe);		off = 0;	}	*idxp = idx;	*offp = off;} static size_t push_pipe(struct iov_iter *i, size_t size,			int *idxp, size_t *offp){	struct pipe_inode_info *pipe = i->pipe;	size_t off;	int idx;	ssize_t left; 	if (unlikely(size > i->count))		size = i->count;	if (unlikely(!size))		return 0; 

Lines 1–500 of 1446. Load more