// SPDX-License-Identifier: GPL-2.0/* *  linux/fs/pipe.c * *  Copyright (C) 1991, 1992, 1999  Linus Torvalds */ #include <linux/mm.h>#include <linux/file.h>#include <linux/poll.h>#include <linux/slab.h>#include <linux/module.h>#include <linux/init.h>#include <linux/fs.h>#include <linux/log2.h>#include <linux/mount.h>#include <linux/pseudo_fs.h>#include <linux/magic.h>#include <linux/pipe_fs_i.h>#include <linux/uio.h>#include <linux/highmem.h>#include <linux/pagemap.h>#include <linux/audit.h>#include <linux/syscalls.h>#include <linux/fcntl.h>#include <linux/memcontrol.h>#include <linux/watch_queue.h> #include <linux/uaccess.h>#include <asm/ioctls.h> #include "internal.h" /* * The max size that a non-root user is allowed to grow the pipe. Can * be set by root in /proc/sys/fs/pipe-max-size */unsigned int pipe_max_size = 1048576; /* Maximum allocatable pages per user. Hard limit is unset by default, soft * matches default values. */unsigned long pipe_user_pages_hard;unsigned long pipe_user_pages_soft = PIPE_DEF_BUFFERS * INR_OPEN_CUR; /* * We use head and tail indices that aren't masked off, except at the point of * dereference, but rather they're allowed to wrap naturally.  This means there * isn't a dead spot in the buffer, but the ring has to be a power of two and * <= 2^31. * -- David Howells 2019-09-23. * * Reads with count = 0 should always return 0. * -- Julian Bradfield 1999-06-07. * * FIFOs and Pipes now generate SIGIO for both readers and writers. * -- Jeremy Elson <jelson@circlemud.org> 2001-08-16 * * pipe_read & write cleanup * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-09 */ static void pipe_lock_nested(struct pipe_inode_info *pipe, int subclass){	if (pipe->files)		mutex_lock_nested(&pipe->mutex, subclass);} void pipe_lock(struct pipe_inode_info *pipe){	/*	 * pipe_lock() nests non-pipe inode locks (for writing to a file)	 */	pipe_lock_nested(pipe, I_MUTEX_PARENT);}EXPORT_SYMBOL(pipe_lock); void pipe_unlock(struct pipe_inode_info *pipe){	if (pipe->files)		mutex_unlock(&pipe->mutex);}EXPORT_SYMBOL(pipe_unlock); static inline void __pipe_lock(struct pipe_inode_info *pipe){	mutex_lock_nested(&pipe->mutex, I_MUTEX_PARENT);} static inline void __pipe_unlock(struct pipe_inode_info *pipe){	mutex_unlock(&pipe->mutex);} void pipe_double_lock(struct pipe_inode_info *pipe1,		      struct pipe_inode_info *pipe2){	BUG_ON(pipe1 == pipe2); 	if (pipe1 < pipe2) {		pipe_lock_nested(pipe1, I_MUTEX_PARENT);		pipe_lock_nested(pipe2, I_MUTEX_CHILD);	} else {		pipe_lock_nested(pipe2, I_MUTEX_PARENT);		pipe_lock_nested(pipe1, I_MUTEX_CHILD);	}} /* Drop the inode semaphore and wait for a pipe event, atomically */void pipe_wait(struct pipe_inode_info *pipe){	DEFINE_WAIT(rdwait);	DEFINE_WAIT(wrwait); 	/*	 * Pipes are system-local resources, so sleeping on them	 * is considered a noninteractive wait:	 */	prepare_to_wait(&pipe->rd_wait, &rdwait, TASK_INTERRUPTIBLE);	prepare_to_wait(&pipe->wr_wait, &wrwait, TASK_INTERRUPTIBLE);	pipe_unlock(pipe);	schedule();	finish_wait(&pipe->rd_wait, &rdwait);	finish_wait(&pipe->wr_wait, &wrwait);	pipe_lock(pipe);} static void anon_pipe_buf_release(struct pipe_inode_info *pipe,				  struct pipe_buffer *buf){	struct page *page = buf->page; 	/*	 * If nobody else uses this page, and we don't already have a	 * temporary page, let's keep track of it as a one-deep	 * allocation cache. (Otherwise just release our reference to it)	 */	if (page_count(page) == 1 && !pipe->tmp_page)		pipe->tmp_page = page;	else		put_page(page);} static bool anon_pipe_buf_try_steal(struct pipe_inode_info *pipe,		struct pipe_buffer *buf){	struct page *page = buf->page; 	if (page_count(page) != 1)		return false;	memcg_kmem_uncharge_page(page, 0);	__SetPageLocked(page);	return true;} /** * generic_pipe_buf_try_steal - attempt to take ownership of a &pipe_buffer * @pipe:	the pipe that the buffer belongs to * @buf:	the buffer to attempt to steal * * Description: *	This function attempts to steal the &struct page attached to *	@buf. If successful, this function returns 0 and returns with *	the page locked. The caller may then reuse the page for whatever *	he wishes; the typical use is insertion into a different file *	page cache. */bool generic_pipe_buf_try_steal(struct pipe_inode_info *pipe,		struct pipe_buffer *buf){	struct page *page = buf->page; 	/*	 * A reference of one is golden, that means that the owner of this	 * page is the only one holding a reference to it. lock the page	 * and return OK.	 */	if (page_count(page) == 1) {		lock_page(page);		return true;	}	return false;}EXPORT_SYMBOL(generic_pipe_buf_try_steal); /** * generic_pipe_buf_get - get a reference to a &struct pipe_buffer * @pipe:	the pipe that the buffer belongs to * @buf:	the buffer to get a reference to * * Description: *	This function grabs an extra reference to @buf. It's used in *	in the tee() system call, when we duplicate the buffers in one *	pipe into another. */bool generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf){	return try_get_page(buf->page);}EXPORT_SYMBOL(generic_pipe_buf_get); /** * generic_pipe_buf_release - put a reference to a &struct pipe_buffer * @pipe:	the pipe that the buffer belongs to * @buf:	the buffer to put a reference to * * Description: *	This function releases a reference to @buf. */void generic_pipe_buf_release(struct pipe_inode_info *pipe,			      struct pipe_buffer *buf){	put_page(buf->page);}EXPORT_SYMBOL(generic_pipe_buf_release); static const struct pipe_buf_operations anon_pipe_buf_ops = {	.release	= anon_pipe_buf_release,	.try_steal	= anon_pipe_buf_try_steal,	.get		= generic_pipe_buf_get,}; /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */static inline bool pipe_readable(const struct pipe_inode_info *pipe){	unsigned int head = READ_ONCE(pipe->head);	unsigned int tail = READ_ONCE(pipe->tail);	unsigned int writers = READ_ONCE(pipe->writers); 	return !pipe_empty(head, tail) || !writers;} static ssize_tpipe_read(struct kiocb *iocb, struct iov_iter *to){	size_t total_len = iov_iter_count(to);	struct file *filp = iocb->ki_filp;	struct pipe_inode_info *pipe = filp->private_data;	bool was_full, wake_next_reader = false;	ssize_t ret; 	/* Null read succeeds. */	if (unlikely(total_len == 0))		return 0; 	ret = 0;	__pipe_lock(pipe); 	/*	 * We only wake up writers if the pipe was full when we started	 * reading in order to avoid unnecessary wakeups.	 *	 * But when we do wake up writers, we do so using a sync wakeup	 * (WF_SYNC), because we want them to get going and generate more	 * data for us.	 */	was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage);	for (;;) {		unsigned int head = pipe->head;		unsigned int tail = pipe->tail;		unsigned int mask = pipe->ring_size - 1; #ifdef CONFIG_WATCH_QUEUE		if (pipe->note_loss) {			struct watch_notification n; 			if (total_len < 8) {				if (ret == 0)					ret = -ENOBUFS;				break;			} 			n.type = WATCH_TYPE_META;			n.subtype = WATCH_META_LOSS_NOTIFICATION;			n.info = watch_sizeof(n);			if (copy_to_iter(&n, sizeof(n), to) != sizeof(n)) {				if (ret == 0)					ret = -EFAULT;				break;			}			ret += sizeof(n);			total_len -= sizeof(n);			pipe->note_loss = false;		}#endif 		if (!pipe_empty(head, tail)) {			struct pipe_buffer *buf = &pipe->bufs[tail & mask];			size_t chars = buf->len;			size_t written;			int error; 			if (chars > total_len) {				if (buf->flags & PIPE_BUF_FLAG_WHOLE) {					if (ret == 0)						ret = -ENOBUFS;					break;				}				chars = total_len;			} 			error = pipe_buf_confirm(pipe, buf);			if (error) {				if (!ret)					ret = error;				break;			} 			written = copy_page_to_iter(buf->page, buf->offset, chars, to);			if (unlikely(written < chars)) {				if (!ret)					ret = -EFAULT;				break;			}			ret += chars;			buf->offset += chars;			buf->len -= chars; 			/* Was it a packet buffer? Clean up and exit */			if (buf->flags & PIPE_BUF_FLAG_PACKET) {				total_len = chars;				buf->len = 0;			} 			if (!buf->len) {				pipe_buf_release(pipe, buf);				spin_lock_irq(&pipe->rd_wait.lock);#ifdef CONFIG_WATCH_QUEUE				if (buf->flags & PIPE_BUF_FLAG_LOSS)					pipe->note_loss = true;#endif				tail++;				pipe->tail = tail;				spin_unlock_irq(&pipe->rd_wait.lock);			}			total_len -= chars;			if (!total_len)				break;	/* common path: read succeeded */			if (!pipe_empty(head, tail))	/* More to do? */				continue;		} 		if (!pipe->writers)			break;		if (ret)			break;		if (filp->f_flags & O_NONBLOCK) {			ret = -EAGAIN;			break;		}		__pipe_unlock(pipe); 		/*		 * We only get here if we didn't actually read anything.		 *		 * However, we could have seen (and removed) a zero-sized		 * pipe buffer, and might have made space in the buffers		 * that way.		 *		 * You can't make zero-sized pipe buffers by doing an empty		 * write (not even in packet mode), but they can happen if		 * the writer gets an EFAULT when trying to fill a buffer		 * that already got allocated and inserted in the buffer		 * array.		 *		 * So we still need to wake up any pending writers in the		 * _very_ unlikely case that the pipe was full, but we got		 * no data.		 */		if (unlikely(was_full)) {			wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);			kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);		} 		/*		 * But because we didn't read anything, at this point we can		 * just return directly with -ERESTARTSYS if we're interrupted,		 * since we've done any required wakeups and there's no need		 * to mark anything accessed. And we've dropped the lock.		 */		if (wait_event_interruptible_exclusive(pipe->rd_wait, pipe_readable(pipe)) < 0)			return -ERESTARTSYS; 		__pipe_lock(pipe);		was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage);		wake_next_reader = true;	}	if (pipe_empty(pipe->head, pipe->tail))		wake_next_reader = false;	__pipe_unlock(pipe); 	if (was_full) {		wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);		kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);	}	if (wake_next_reader)		wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);	if (ret > 0)		file_accessed(filp);	return ret;} static inline int is_packetized(struct file *file){	return (file->f_flags & O_DIRECT) != 0;} /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */static inline bool pipe_writable(const struct pipe_inode_info *pipe){	unsigned int head = READ_ONCE(pipe->head);	unsigned int tail = READ_ONCE(pipe->tail);	unsigned int max_usage = READ_ONCE(pipe->max_usage); 	return !pipe_full(head, tail, max_usage) ||		!READ_ONCE(pipe->readers);} static ssize_tpipe_write(struct kiocb *iocb, struct iov_iter *from){	struct file *filp = iocb->ki_filp;	struct pipe_inode_info *pipe = filp->private_data;	unsigned int head;	ssize_t ret = 0;	size_t total_len = iov_iter_count(from);	ssize_t chars;	bool was_empty = false;	bool wake_next_writer = false; 	/* Null write succeeds. */	if (unlikely(total_len == 0))		return 0; 	__pipe_lock(pipe); 	if (!pipe->readers) {		send_sig(SIGPIPE, current, 0);		ret = -EPIPE;		goto out;	} #ifdef CONFIG_WATCH_QUEUE	if (pipe->watch_queue) {		ret = -EXDEV;		goto out;	}#endif 	/*	 * Only wake up if the pipe started out empty, since	 * otherwise there should be no readers waiting.	 *	 * If it wasn't empty we try to merge new data into	 * the last buffer.	 *	 * That naturally merges small writes, but it also	 * page-aligs the rest of the writes for large writes	 * spanning multiple pages.	 */	head = pipe->head;	was_empty = pipe_empty(head, pipe->tail);	chars = total_len & (PAGE_SIZE-1);	if (chars && !was_empty) {		unsigned int mask = pipe->ring_size - 1;		struct pipe_buffer *buf = &pipe->bufs[(head - 1) & mask];		int offset = buf->offset + buf->len; 		if ((buf->flags & PIPE_BUF_FLAG_CAN_MERGE) &&		    offset + chars <= PAGE_SIZE) {			ret = pipe_buf_confirm(pipe, buf);			if (ret)				goto out; 			ret = copy_page_from_iter(buf->page, offset, chars, from);			if (unlikely(ret < chars)) {				ret = -EFAULT;				goto out;			} 			buf->len += ret;			if (!iov_iter_count(from))				goto out;		}	} 	for (;;) {		if (!pipe->readers) {			send_sig(SIGPIPE, current, 0);			if (!ret)				ret = -EPIPE;			break;		} 		head = pipe->head;		if (!pipe_full(head, pipe->tail, pipe->max_usage)) {			unsigned int mask = pipe->ring_size - 1;			struct pipe_buffer *buf = &pipe->bufs[head & mask];			struct page *page = pipe->tmp_page;			int copied;

Lines 1–500 of 1412. Load more