// SPDX-License-Identifier: GPL-2.0/* *  linux/fs/pipe.c * *  Copyright (C) 1991, 1992, 1999  Linus Torvalds */ #include <linux/mm.h>#include <linux/file.h>#include <linux/poll.h>#include <linux/slab.h>#include <linux/module.h>#include <linux/init.h>#include <linux/fs.h>#include <linux/log2.h>#include <linux/mount.h>#include <linux/pseudo_fs.h>#include <linux/magic.h>#include <linux/pipe_fs_i.h>#include <linux/uio.h>#include <linux/highmem.h>#include <linux/pagemap.h>#include <linux/audit.h>#include <linux/syscalls.h>#include <linux/fcntl.h>#include <linux/memcontrol.h>#include <linux/watch_queue.h>#include <linux/sysctl.h>#include <linux/sort.h> #include <linux/uaccess.h>#include <asm/ioctls.h> #include "internal.h" /* * New pipe buffers will be restricted to this size while the user is exceeding * their pipe buffer quota. The general pipe use case needs at least two * buffers: one for data yet to be read, and one for new data. If this is less * than two, then a write to a non-empty pipe may block even if the pipe is not * full. This can occur with GNU make jobserver or similar uses of pipes as * semaphores: multiple processes may be waiting to write tokens back to the * pipe before reading tokens: https://lore.kernel.org/lkml/1628086770.5rn8p04n6j.none@localhost/. * * Users can reduce their pipe buffers with F_SETPIPE_SZ below this at their * own risk, namely: pipe writes to non-full pipes may block until the pipe is * emptied. */#define PIPE_MIN_DEF_BUFFERS 2 /* * The max size that a non-root user is allowed to grow the pipe. Can * be set by root in /proc/sys/fs/pipe-max-size */static unsigned int pipe_max_size = 1048576; /* Maximum allocatable pages per user. Hard limit is unset by default, soft * matches default values. */static unsigned long pipe_user_pages_hard;static unsigned long pipe_user_pages_soft = PIPE_DEF_BUFFERS * INR_OPEN_CUR; /* * We use head and tail indices that aren't masked off, except at the point of * dereference, but rather they're allowed to wrap naturally.  This means there * isn't a dead spot in the buffer, but the ring has to be a power of two and * <= 2^31. * -- David Howells 2019-09-23. * * Reads with count = 0 should always return 0. * -- Julian Bradfield 1999-06-07. * * FIFOs and Pipes now generate SIGIO for both readers and writers. * -- Jeremy Elson <jelson@circlemud.org> 2001-08-16 * * pipe_read & write cleanup * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-09 */ #ifdef CONFIG_PROVE_LOCKINGstatic int pipe_lock_cmp_fn(const struct lockdep_map *a,			    const struct lockdep_map *b){	return cmp_int((unsigned long) a, (unsigned long) b);}#endif void pipe_lock(struct pipe_inode_info *pipe){	if (pipe->files)		mutex_lock(&pipe->mutex);}EXPORT_SYMBOL(pipe_lock); void pipe_unlock(struct pipe_inode_info *pipe){	if (pipe->files)		mutex_unlock(&pipe->mutex);}EXPORT_SYMBOL(pipe_unlock); void pipe_double_lock(struct pipe_inode_info *pipe1,		      struct pipe_inode_info *pipe2){	BUG_ON(pipe1 == pipe2); 	if (pipe1 > pipe2)		swap(pipe1, pipe2); 	pipe_lock(pipe1);	pipe_lock(pipe2);} static struct page *anon_pipe_prealloc_pop(struct anon_pipe_prealloc *prealloc){	if (!prealloc->count)		return NULL; 	prealloc->count--; 	return prealloc->pages[prealloc->count];} /* Push a page to the prealloc pool. Returns true if added, false if full. */static bool anon_pipe_prealloc_push(struct anon_pipe_prealloc *prealloc,				    struct page *page){	if (prealloc->count >= PIPE_PREALLOC_MAX)		return false;	prealloc->pages[prealloc->count++] = page;	return true;} /* * Top up the pipe's own pool, then take pipe->mutex and return with it held. * The shortfall is allocated outside the lock; the push and the caller's write * then run under a single lock acquisition, avoiding a separate prefill * lock/unlock cycle. anon_pipe_get_page() drains the pool instead of allocating * under the lock. */static void anon_pipe_prefill_and_lock(struct pipe_inode_info *pipe, size_t total_len){	struct page *pages[PIPE_PREALLOC_MAX];	unsigned int want, have, need, n = 0; 	want = min_t(unsigned int, DIV_ROUND_UP(total_len, PAGE_SIZE),		     PIPE_PREALLOC_MAX);	/* Unlocked read; the pool is refilled under the lock below. */	have = min_t(unsigned int, READ_ONCE(pipe->prealloc.count), want);	need = want - have; 	if (!need) {		mutex_lock(&pipe->mutex);		return;	} 	while (n < need) {		struct page *page = alloc_page(GFP_HIGHUSER | __GFP_ACCOUNT); 		if (!page)			break;		pages[n++] = page;	} 	mutex_lock(&pipe->mutex);	while (n && anon_pipe_prealloc_push(&pipe->prealloc, pages[n - 1]))		n--; 	/*	 * Just flush any extra page that got affected by the TOCTOU	 * effect	 */	while (n)		put_page(pages[--n]);} /* * Called with pipe->mutex held. Trim the pool down to PIPE_PREALLOC_KEEP under * the lock, drop it, then free the excess outside the critical section. */static void anon_pipe_trim_and_unlock(struct pipe_inode_info *pipe){	struct page *excess[PIPE_PREALLOC_MAX];	unsigned int nexcess = 0; 	while (pipe->prealloc.count > PIPE_PREALLOC_KEEP)		excess[nexcess++] = anon_pipe_prealloc_pop(&pipe->prealloc);	mutex_unlock(&pipe->mutex); 	while (nexcess)		put_page(excess[--nexcess]);} static struct page *anon_pipe_get_page(struct pipe_inode_info *pipe){	struct page *page; 	/* Drain the prealloc pool before allocating. Called with mutex held. */	page = anon_pipe_prealloc_pop(&pipe->prealloc);	if (page)		return page; 	/* FWIW: This is called with pipe->mutex held */	return alloc_page(GFP_HIGHUSER | __GFP_ACCOUNT);} static void anon_pipe_put_page(struct pipe_inode_info *pipe,			       struct page *page){	if (page_count(page) == 1 &&	    anon_pipe_prealloc_push(&pipe->prealloc, page))		return; 	put_page(page);} static void anon_pipe_buf_release(struct pipe_inode_info *pipe,				  struct pipe_buffer *buf){	struct page *page = buf->page; 	anon_pipe_put_page(pipe, page);} static bool anon_pipe_buf_try_steal(struct pipe_inode_info *pipe,		struct pipe_buffer *buf){	struct page *page = buf->page; 	if (page_count(page) != 1)		return false;	memcg_kmem_uncharge_page(page, 0);	__SetPageLocked(page);	return true;} /** * generic_pipe_buf_try_steal - attempt to take ownership of a &pipe_buffer * @pipe:	the pipe that the buffer belongs to * @buf:	the buffer to attempt to steal * * Description: *	This function attempts to steal the &struct page attached to *	@buf. If successful, this function returns 0 and returns with *	the page locked. The caller may then reuse the page for whatever *	he wishes; the typical use is insertion into a different file *	page cache. */bool generic_pipe_buf_try_steal(struct pipe_inode_info *pipe,		struct pipe_buffer *buf){	struct page *page = buf->page; 	/*	 * A reference of one is golden, that means that the owner of this	 * page is the only one holding a reference to it. lock the page	 * and return OK.	 */	if (page_count(page) == 1) {		lock_page(page);		return true;	}	return false;}EXPORT_SYMBOL(generic_pipe_buf_try_steal); /** * generic_pipe_buf_get - get a reference to a &struct pipe_buffer * @pipe:	the pipe that the buffer belongs to * @buf:	the buffer to get a reference to * * Description: *	This function grabs an extra reference to @buf. It's used in *	the tee() system call, when we duplicate the buffers in one *	pipe into another. */bool generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf){	return try_get_page(buf->page);}EXPORT_SYMBOL(generic_pipe_buf_get); /** * generic_pipe_buf_release - put a reference to a &struct pipe_buffer * @pipe:	the pipe that the buffer belongs to * @buf:	the buffer to put a reference to * * Description: *	This function releases a reference to @buf. */void generic_pipe_buf_release(struct pipe_inode_info *pipe,			      struct pipe_buffer *buf){	put_page(buf->page);}EXPORT_SYMBOL(generic_pipe_buf_release); static const struct pipe_buf_operations anon_pipe_buf_ops = {	.release	= anon_pipe_buf_release,	.try_steal	= anon_pipe_buf_try_steal,	.get		= generic_pipe_buf_get,}; /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */static inline bool pipe_readable(const struct pipe_inode_info *pipe){	union pipe_index idx = { .head_tail = READ_ONCE(pipe->head_tail) };	unsigned int writers = READ_ONCE(pipe->writers); 	return !pipe_empty(idx.head, idx.tail) || !writers;} static inline unsigned int pipe_update_tail(struct pipe_inode_info *pipe,					    struct pipe_buffer *buf,					    unsigned int tail){	pipe_buf_release(pipe, buf); 	/*	 * If the pipe has a watch_queue, we need additional protection	 * by the spinlock because notifications get posted with only	 * this spinlock, no mutex	 */	if (pipe_has_watch_queue(pipe)) {		spin_lock_irq(&pipe->rd_wait.lock);#ifdef CONFIG_WATCH_QUEUE		if (buf->flags & PIPE_BUF_FLAG_LOSS)			pipe->note_loss = true;#endif		pipe->tail = ++tail;		spin_unlock_irq(&pipe->rd_wait.lock);		return tail;	} 	/*	 * Without a watch_queue, we can simply increment the tail	 * without the spinlock - the mutex is enough.	 */	pipe->tail = ++tail;	return tail;} static ssize_tanon_pipe_read(struct kiocb *iocb, struct iov_iter *to){	size_t total_len = iov_iter_count(to);	struct file *filp = iocb->ki_filp;	struct pipe_inode_info *pipe = filp->private_data;	bool wake_writer = false, wake_next_reader = false;	ssize_t ret; 	/* Null read succeeds. */	if (unlikely(total_len == 0))		return 0; 	ret = 0;	mutex_lock(&pipe->mutex); 	/*	 * We only wake up writers if the pipe was full when we started reading	 * and it is no longer full after reading to avoid unnecessary wakeups.	 *	 * But when we do wake up writers, we do so using a sync wakeup	 * (WF_SYNC), because we want them to get going and generate more	 * data for us.	 */	for (;;) {		/* Read ->head with a barrier vs post_one_notification() */		unsigned int head = smp_load_acquire(&pipe->head);		unsigned int tail = pipe->tail; #ifdef CONFIG_WATCH_QUEUE		if (pipe->note_loss) {			struct watch_notification n; 			if (total_len < 8) {				if (ret == 0)					ret = -ENOBUFS;				break;			} 			n.type = WATCH_TYPE_META;			n.subtype = WATCH_META_LOSS_NOTIFICATION;			n.info = watch_sizeof(n);			if (copy_to_iter(&n, sizeof(n), to) != sizeof(n)) {				if (ret == 0)					ret = -EFAULT;				break;			}			ret += sizeof(n);			total_len -= sizeof(n);			pipe->note_loss = false;		}#endif 		if (!pipe_empty(head, tail)) {			struct pipe_buffer *buf = pipe_buf(pipe, tail);			size_t chars = buf->len;			size_t written;			int error; 			if (chars > total_len) {				if (buf->flags & PIPE_BUF_FLAG_WHOLE) {					if (ret == 0)						ret = -ENOBUFS;					break;				}				chars = total_len;			} 			error = pipe_buf_confirm(pipe, buf);			if (error) {				if (!ret)					ret = error;				break;			} 			written = copy_page_to_iter(buf->page, buf->offset, chars, to);			if (unlikely(written < chars)) {				if (!ret)					ret = -EFAULT;				break;			}			ret += chars;			buf->offset += chars;			buf->len -= chars; 			/* Was it a packet buffer? Clean up and exit */			if (buf->flags & PIPE_BUF_FLAG_PACKET) {				total_len = chars;				buf->len = 0;			} 			if (!buf->len) {				wake_writer |= pipe_full(head, tail, pipe->max_usage);				tail = pipe_update_tail(pipe, buf, tail);			}			total_len -= chars;			if (!total_len)				break;	/* common path: read succeeded */			if (!pipe_empty(head, tail))	/* More to do? */				continue;		} 		if (!pipe->writers)			break;		if (ret)			break;		if ((filp->f_flags & O_NONBLOCK) ||		    (iocb->ki_flags & IOCB_NOWAIT)) {			ret = -EAGAIN;			break;		}		mutex_unlock(&pipe->mutex);		/*		 * We only get here if we didn't actually read anything.		 *		 * But because we didn't read anything, at this point we can		 * just return directly with -ERESTARTSYS if we're interrupted,		 * since we've done any required wakeups and there's no need		 * to mark anything accessed. And we've dropped the lock.		 */		if (wait_event_interruptible_exclusive(pipe->rd_wait, pipe_readable(pipe)) < 0)			return -ERESTARTSYS; 		wake_next_reader = true;		mutex_lock(&pipe->mutex);	}	if (pipe_is_empty(pipe))		wake_next_reader = false;	/* Consumed buffers may have refilled the pool; trim it and unlock. */	anon_pipe_trim_and_unlock(pipe); 	if (wake_writer)		wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);	if (wake_next_reader)		wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);	kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);	return ret;} static ssize_tfifo_pipe_read(struct kiocb *iocb, struct iov_iter *to){	int ret = anon_pipe_read(iocb, to);	if (ret > 0)		file_accessed(iocb->ki_filp);	return ret;} static inline int is_packetized(struct file *file){	return (file->f_flags & O_DIRECT) != 0;} /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */static inline bool pipe_writable(const struct pipe_inode_info *pipe){	union pipe_index idx = { .head_tail = READ_ONCE(pipe->head_tail) };	unsigned int max_usage = READ_ONCE(pipe->max_usage);

Lines 1–500 of 1633. Load more