// SPDX-License-Identifier: GPL-2.0/* *  linux/fs/pipe.c * *  Copyright (C) 1991, 1992, 1999  Linus Torvalds */ #include <linux/mm.h>#include <linux/file.h>#include <linux/poll.h>#include <linux/slab.h>#include <linux/module.h>#include <linux/init.h>#include <linux/fs.h>#include <linux/log2.h>#include <linux/mount.h>#include <linux/pseudo_fs.h>#include <linux/magic.h>#include <linux/pipe_fs_i.h>#include <linux/uio.h>#include <linux/highmem.h>#include <linux/pagemap.h>#include <linux/audit.h>#include <linux/syscalls.h>#include <linux/fcntl.h>#include <linux/memcontrol.h>#include <linux/watch_queue.h>#include <linux/sysctl.h> #include <linux/uaccess.h>#include <asm/ioctls.h> #include "internal.h" /* * New pipe buffers will be restricted to this size while the user is exceeding * their pipe buffer quota. The general pipe use case needs at least two * buffers: one for data yet to be read, and one for new data. If this is less * than two, then a write to a non-empty pipe may block even if the pipe is not * full. This can occur with GNU make jobserver or similar uses of pipes as * semaphores: multiple processes may be waiting to write tokens back to the * pipe before reading tokens: https://lore.kernel.org/lkml/1628086770.5rn8p04n6j.none@localhost/. * * Users can reduce their pipe buffers with F_SETPIPE_SZ below this at their * own risk, namely: pipe writes to non-full pipes may block until the pipe is * emptied. */#define PIPE_MIN_DEF_BUFFERS 2 /* * The max size that a non-root user is allowed to grow the pipe. Can * be set by root in /proc/sys/fs/pipe-max-size */static unsigned int pipe_max_size = 1048576; /* Maximum allocatable pages per user. Hard limit is unset by default, soft * matches default values. */static unsigned long pipe_user_pages_hard;static unsigned long pipe_user_pages_soft = PIPE_DEF_BUFFERS * INR_OPEN_CUR; /* * We use head and tail indices that aren't masked off, except at the point of * dereference, but rather they're allowed to wrap naturally.  This means there * isn't a dead spot in the buffer, but the ring has to be a power of two and * <= 2^31. * -- David Howells 2019-09-23. * * Reads with count = 0 should always return 0. * -- Julian Bradfield 1999-06-07. * * FIFOs and Pipes now generate SIGIO for both readers and writers. * -- Jeremy Elson <jelson@circlemud.org> 2001-08-16 * * pipe_read & write cleanup * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-09 */ #define cmp_int(l, r)		((l > r) - (l < r)) #ifdef CONFIG_PROVE_LOCKINGstatic int pipe_lock_cmp_fn(const struct lockdep_map *a,			    const struct lockdep_map *b){	return cmp_int((unsigned long) a, (unsigned long) b);}#endif void pipe_lock(struct pipe_inode_info *pipe){	if (pipe->files)		mutex_lock(&pipe->mutex);}EXPORT_SYMBOL(pipe_lock); void pipe_unlock(struct pipe_inode_info *pipe){	if (pipe->files)		mutex_unlock(&pipe->mutex);}EXPORT_SYMBOL(pipe_unlock); void pipe_double_lock(struct pipe_inode_info *pipe1,		      struct pipe_inode_info *pipe2){	BUG_ON(pipe1 == pipe2); 	if (pipe1 > pipe2)		swap(pipe1, pipe2); 	pipe_lock(pipe1);	pipe_lock(pipe2);} static void anon_pipe_buf_release(struct pipe_inode_info *pipe,				  struct pipe_buffer *buf){	struct page *page = buf->page; 	/*	 * If nobody else uses this page, and we don't already have a	 * temporary page, let's keep track of it as a one-deep	 * allocation cache. (Otherwise just release our reference to it)	 */	if (page_count(page) == 1 && !pipe->tmp_page)		pipe->tmp_page = page;	else		put_page(page);} static bool anon_pipe_buf_try_steal(struct pipe_inode_info *pipe,		struct pipe_buffer *buf){	struct page *page = buf->page; 	if (page_count(page) != 1)		return false;	memcg_kmem_uncharge_page(page, 0);	__SetPageLocked(page);	return true;} /** * generic_pipe_buf_try_steal - attempt to take ownership of a &pipe_buffer * @pipe:	the pipe that the buffer belongs to * @buf:	the buffer to attempt to steal * * Description: *	This function attempts to steal the &struct page attached to *	@buf. If successful, this function returns 0 and returns with *	the page locked. The caller may then reuse the page for whatever *	he wishes; the typical use is insertion into a different file *	page cache. */bool generic_pipe_buf_try_steal(struct pipe_inode_info *pipe,		struct pipe_buffer *buf){	struct page *page = buf->page; 	/*	 * A reference of one is golden, that means that the owner of this	 * page is the only one holding a reference to it. lock the page	 * and return OK.	 */	if (page_count(page) == 1) {		lock_page(page);		return true;	}	return false;}EXPORT_SYMBOL(generic_pipe_buf_try_steal); /** * generic_pipe_buf_get - get a reference to a &struct pipe_buffer * @pipe:	the pipe that the buffer belongs to * @buf:	the buffer to get a reference to * * Description: *	This function grabs an extra reference to @buf. It's used in *	the tee() system call, when we duplicate the buffers in one *	pipe into another. */bool generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf){	return try_get_page(buf->page);}EXPORT_SYMBOL(generic_pipe_buf_get); /** * generic_pipe_buf_release - put a reference to a &struct pipe_buffer * @pipe:	the pipe that the buffer belongs to * @buf:	the buffer to put a reference to * * Description: *	This function releases a reference to @buf. */void generic_pipe_buf_release(struct pipe_inode_info *pipe,			      struct pipe_buffer *buf){	put_page(buf->page);}EXPORT_SYMBOL(generic_pipe_buf_release); static const struct pipe_buf_operations anon_pipe_buf_ops = {	.release	= anon_pipe_buf_release,	.try_steal	= anon_pipe_buf_try_steal,	.get		= generic_pipe_buf_get,}; /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */static inline bool pipe_readable(const struct pipe_inode_info *pipe){	unsigned int head = READ_ONCE(pipe->head);	unsigned int tail = READ_ONCE(pipe->tail);	unsigned int writers = READ_ONCE(pipe->writers); 	return !pipe_empty(head, tail) || !writers;} static inline unsigned int pipe_update_tail(struct pipe_inode_info *pipe,					    struct pipe_buffer *buf,					    unsigned int tail){	pipe_buf_release(pipe, buf); 	/*	 * If the pipe has a watch_queue, we need additional protection	 * by the spinlock because notifications get posted with only	 * this spinlock, no mutex	 */	if (pipe_has_watch_queue(pipe)) {		spin_lock_irq(&pipe->rd_wait.lock);#ifdef CONFIG_WATCH_QUEUE		if (buf->flags & PIPE_BUF_FLAG_LOSS)			pipe->note_loss = true;#endif		pipe->tail = ++tail;		spin_unlock_irq(&pipe->rd_wait.lock);		return tail;	} 	/*	 * Without a watch_queue, we can simply increment the tail	 * without the spinlock - the mutex is enough.	 */	pipe->tail = ++tail;	return tail;} static ssize_tpipe_read(struct kiocb *iocb, struct iov_iter *to){	size_t total_len = iov_iter_count(to);	struct file *filp = iocb->ki_filp;	struct pipe_inode_info *pipe = filp->private_data;	bool was_full, wake_next_reader = false;	ssize_t ret; 	/* Null read succeeds. */	if (unlikely(total_len == 0))		return 0; 	ret = 0;	mutex_lock(&pipe->mutex); 	/*	 * We only wake up writers if the pipe was full when we started	 * reading in order to avoid unnecessary wakeups.	 *	 * But when we do wake up writers, we do so using a sync wakeup	 * (WF_SYNC), because we want them to get going and generate more	 * data for us.	 */	was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage);	for (;;) {		/* Read ->head with a barrier vs post_one_notification() */		unsigned int head = smp_load_acquire(&pipe->head);		unsigned int tail = pipe->tail;		unsigned int mask = pipe->ring_size - 1; #ifdef CONFIG_WATCH_QUEUE		if (pipe->note_loss) {			struct watch_notification n; 			if (total_len < 8) {				if (ret == 0)					ret = -ENOBUFS;				break;			} 			n.type = WATCH_TYPE_META;			n.subtype = WATCH_META_LOSS_NOTIFICATION;			n.info = watch_sizeof(n);			if (copy_to_iter(&n, sizeof(n), to) != sizeof(n)) {				if (ret == 0)					ret = -EFAULT;				break;			}			ret += sizeof(n);			total_len -= sizeof(n);			pipe->note_loss = false;		}#endif 		if (!pipe_empty(head, tail)) {			struct pipe_buffer *buf = &pipe->bufs[tail & mask];			size_t chars = buf->len;			size_t written;			int error; 			if (chars > total_len) {				if (buf->flags & PIPE_BUF_FLAG_WHOLE) {					if (ret == 0)						ret = -ENOBUFS;					break;				}				chars = total_len;			} 			error = pipe_buf_confirm(pipe, buf);			if (error) {				if (!ret)					ret = error;				break;			} 			written = copy_page_to_iter(buf->page, buf->offset, chars, to);			if (unlikely(written < chars)) {				if (!ret)					ret = -EFAULT;				break;			}			ret += chars;			buf->offset += chars;			buf->len -= chars; 			/* Was it a packet buffer? Clean up and exit */			if (buf->flags & PIPE_BUF_FLAG_PACKET) {				total_len = chars;				buf->len = 0;			} 			if (!buf->len)				tail = pipe_update_tail(pipe, buf, tail);			total_len -= chars;			if (!total_len)				break;	/* common path: read succeeded */			if (!pipe_empty(head, tail))	/* More to do? */				continue;		} 		if (!pipe->writers)			break;		if (ret)			break;		if ((filp->f_flags & O_NONBLOCK) ||		    (iocb->ki_flags & IOCB_NOWAIT)) {			ret = -EAGAIN;			break;		}		mutex_unlock(&pipe->mutex); 		/*		 * We only get here if we didn't actually read anything.		 *		 * However, we could have seen (and removed) a zero-sized		 * pipe buffer, and might have made space in the buffers		 * that way.		 *		 * You can't make zero-sized pipe buffers by doing an empty		 * write (not even in packet mode), but they can happen if		 * the writer gets an EFAULT when trying to fill a buffer		 * that already got allocated and inserted in the buffer		 * array.		 *		 * So we still need to wake up any pending writers in the		 * _very_ unlikely case that the pipe was full, but we got		 * no data.		 */		if (unlikely(was_full))			wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);		kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); 		/*		 * But because we didn't read anything, at this point we can		 * just return directly with -ERESTARTSYS if we're interrupted,		 * since we've done any required wakeups and there's no need		 * to mark anything accessed. And we've dropped the lock.		 */		if (wait_event_interruptible_exclusive(pipe->rd_wait, pipe_readable(pipe)) < 0)			return -ERESTARTSYS; 		mutex_lock(&pipe->mutex);		was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage);		wake_next_reader = true;	}	if (pipe_empty(pipe->head, pipe->tail))		wake_next_reader = false;	mutex_unlock(&pipe->mutex); 	if (was_full)		wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);	if (wake_next_reader)		wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);	kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);	if (ret > 0)		file_accessed(filp);	return ret;} static inline int is_packetized(struct file *file){	return (file->f_flags & O_DIRECT) != 0;} /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */static inline bool pipe_writable(const struct pipe_inode_info *pipe){	unsigned int head = READ_ONCE(pipe->head);	unsigned int tail = READ_ONCE(pipe->tail);	unsigned int max_usage = READ_ONCE(pipe->max_usage); 	return !pipe_full(head, tail, max_usage) ||		!READ_ONCE(pipe->readers);} static ssize_tpipe_write(struct kiocb *iocb, struct iov_iter *from){	struct file *filp = iocb->ki_filp;	struct pipe_inode_info *pipe = filp->private_data;	unsigned int head;	ssize_t ret = 0;	size_t total_len = iov_iter_count(from);	ssize_t chars;	bool was_empty = false;	bool wake_next_writer = false; 	/*	 * Reject writing to watch queue pipes before the point where we lock	 * the pipe.	 * Otherwise, lockdep would be unhappy if the caller already has another	 * pipe locked.	 * If we had to support locking a normal pipe and a notification pipe at	 * the same time, we could set up lockdep annotations for that, but	 * since we don't actually need that, it's simpler to just bail here.	 */	if (pipe_has_watch_queue(pipe))		return -EXDEV; 	/* Null write succeeds. */	if (unlikely(total_len == 0))		return 0; 	mutex_lock(&pipe->mutex); 	if (!pipe->readers) {		send_sig(SIGPIPE, current, 0);		ret = -EPIPE;		goto out;	} 	/*	 * If it wasn't empty we try to merge new data into	 * the last buffer.	 *	 * That naturally merges small writes, but it also	 * page-aligns the rest of the writes for large writes	 * spanning multiple pages.	 */	head = pipe->head;	was_empty = pipe_empty(head, pipe->tail);	chars = total_len & (PAGE_SIZE-1);	if (chars && !was_empty) {		unsigned int mask = pipe->ring_size - 1;		struct pipe_buffer *buf = &pipe->bufs[(head - 1) & mask];		int offset = buf->offset + buf->len; 		if ((buf->flags & PIPE_BUF_FLAG_CAN_MERGE) &&		    offset + chars <= PAGE_SIZE) {			ret = pipe_buf_confirm(pipe, buf);			if (ret)				goto out; 			ret = copy_page_from_iter(buf->page, offset, chars, from);			if (unlikely(ret < chars)) {				ret = -EFAULT;				goto out;			} 			buf->len += ret;			if (!iov_iter_count(from))				goto out;		}	} 	for (;;) {		if (!pipe->readers) {			send_sig(SIGPIPE, current, 0);			if (!ret)

Lines 1–500 of 1522. Load more