// SPDX-License-Identifier: GPL-2.0/* *  linux/fs/pipe.c * *  Copyright (C) 1991, 1992, 1999  Linus Torvalds */ #include <linux/mm.h>#include <linux/file.h>#include <linux/poll.h>#include <linux/slab.h>#include <linux/module.h>#include <linux/init.h>#include <linux/fs.h>#include <linux/log2.h>#include <linux/mount.h>#include <linux/pseudo_fs.h>#include <linux/magic.h>#include <linux/pipe_fs_i.h>#include <linux/uio.h>#include <linux/highmem.h>#include <linux/pagemap.h>#include <linux/audit.h>#include <linux/syscalls.h>#include <linux/fcntl.h>#include <linux/memcontrol.h>#include <linux/watch_queue.h>#include <linux/sysctl.h>#include <linux/sort.h> #include <linux/uaccess.h>#include <asm/ioctls.h> #include "internal.h" /* * New pipe buffers will be restricted to this size while the user is exceeding * their pipe buffer quota. The general pipe use case needs at least two * buffers: one for data yet to be read, and one for new data. If this is less * than two, then a write to a non-empty pipe may block even if the pipe is not * full. This can occur with GNU make jobserver or similar uses of pipes as * semaphores: multiple processes may be waiting to write tokens back to the * pipe before reading tokens: https://lore.kernel.org/lkml/1628086770.5rn8p04n6j.none@localhost/. * * Users can reduce their pipe buffers with F_SETPIPE_SZ below this at their * own risk, namely: pipe writes to non-full pipes may block until the pipe is * emptied. */#define PIPE_MIN_DEF_BUFFERS 2 /* * The max size that a non-root user is allowed to grow the pipe. Can * be set by root in /proc/sys/fs/pipe-max-size */static unsigned int pipe_max_size = 1048576; /* Maximum allocatable pages per user. Hard limit is unset by default, soft * matches default values. */static unsigned long pipe_user_pages_hard;static unsigned long pipe_user_pages_soft = PIPE_DEF_BUFFERS * INR_OPEN_CUR; /* * We use head and tail indices that aren't masked off, except at the point of * dereference, but rather they're allowed to wrap naturally.  This means there * isn't a dead spot in the buffer, but the ring has to be a power of two and * <= 2^31. * -- David Howells 2019-09-23. * * Reads with count = 0 should always return 0. * -- Julian Bradfield 1999-06-07. * * FIFOs and Pipes now generate SIGIO for both readers and writers. * -- Jeremy Elson <jelson@circlemud.org> 2001-08-16 * * pipe_read & write cleanup * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-09 */ #ifdef CONFIG_PROVE_LOCKINGstatic int pipe_lock_cmp_fn(const struct lockdep_map *a,			    const struct lockdep_map *b){	return cmp_int((unsigned long) a, (unsigned long) b);}#endif void pipe_lock(struct pipe_inode_info *pipe){	if (pipe->files)		mutex_lock(&pipe->mutex);}EXPORT_SYMBOL(pipe_lock); void pipe_unlock(struct pipe_inode_info *pipe){	if (pipe->files)		mutex_unlock(&pipe->mutex);}EXPORT_SYMBOL(pipe_unlock); void pipe_double_lock(struct pipe_inode_info *pipe1,		      struct pipe_inode_info *pipe2){	BUG_ON(pipe1 == pipe2); 	if (pipe1 > pipe2)		swap(pipe1, pipe2); 	pipe_lock(pipe1);	pipe_lock(pipe2);} #define PIPE_PREALLOC_MAX 8 struct anon_pipe_prealloc {	struct page *pages[PIPE_PREALLOC_MAX];	unsigned int count;}; /* * Pre-allocate pages outside pipe->mutex for multi-page writes. * alloc_page() with GFP_HIGHUSER can sleep in reclaim and runs memcg * charging; doing it under the mutex stalls a concurrent reader. * * Loop alloc_page() instead of alloc_pages_bulk_*(): the bulk path refuses * __GFP_ACCOUNT under memcg (see commit 8dcb3060d81d "memcg: page_alloc: * skip bulk allocator for __GFP_ACCOUNT") and silently degrades to a single * page. A per-page loop keeps memcg accounting and the task NUMA mempolicy * honoured for every page; the per-call overhead is small compared to the * pipe->mutex hold-time being shrunk. Any shortfall is covered by the * in-lock alloc_page() fallback in anon_pipe_get_page(). */static void anon_pipe_get_page_prealloc(struct anon_pipe_prealloc *prealloc,					size_t total_len){	unsigned int want, i;	struct page *page; 	prealloc->count = 0;	if (total_len <= PAGE_SIZE)		return; 	want = min_t(unsigned int, DIV_ROUND_UP(total_len, PAGE_SIZE),		     PIPE_PREALLOC_MAX); 	for (i = 0; i < want; i++) {		page = alloc_page(GFP_HIGHUSER | __GFP_ACCOUNT);		if (!page)			break;		prealloc->pages[prealloc->count++] = page;	}} static struct page *anon_pipe_prealloc_pop(struct anon_pipe_prealloc *prealloc){	if (!prealloc->count)		return NULL; 	prealloc->count--; 	return prealloc->pages[prealloc->count];} static struct page *anon_pipe_get_page(struct pipe_inode_info *pipe,				       struct anon_pipe_prealloc *prealloc){	struct page *page; 	/* Drain prealloc first to keep tmp_page[] hot for later small writes. */	page = anon_pipe_prealloc_pop(prealloc);	if (page)		return page; 	for (int i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) {		if (pipe->tmp_page[i]) {			page = pipe->tmp_page[i];			pipe->tmp_page[i] = NULL;			return page;		}	} 	/* FWIW: This is called with pipe->mutex held */	return alloc_page(GFP_HIGHUSER | __GFP_ACCOUNT);} static void anon_pipe_put_page(struct pipe_inode_info *pipe,			       struct page *page){	if (page_count(page) == 1) {		for (int i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) {			if (!pipe->tmp_page[i]) {				pipe->tmp_page[i] = page;				return;			}		}	} 	put_page(page);} /* * Stash leftover prealloc pages in tmp_page[] so the next write to this * pipe gets a hot page without entering the allocator. */static void anon_pipe_refill_tmp_pages(struct pipe_inode_info *pipe,				       struct anon_pipe_prealloc *prealloc){	int i, idx; 	if (!prealloc->count)		return; 	for (i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) {		if (pipe->tmp_page[i])			continue;		if (!prealloc->count)			return;		idx = --prealloc->count;		pipe->tmp_page[i] = prealloc->pages[idx];		prealloc->pages[idx] = NULL;	}} /* Runs after mutex_unlock() to keep put_page() out of the critical section. */static void anon_pipe_free_pages(struct anon_pipe_prealloc *prealloc){	while (prealloc->count) {		prealloc->count--;		put_page(prealloc->pages[prealloc->count]);	}} static void anon_pipe_buf_release(struct pipe_inode_info *pipe,				  struct pipe_buffer *buf){	struct page *page = buf->page; 	anon_pipe_put_page(pipe, page);} static bool anon_pipe_buf_try_steal(struct pipe_inode_info *pipe,		struct pipe_buffer *buf){	struct page *page = buf->page; 	if (page_count(page) != 1)		return false;	memcg_kmem_uncharge_page(page, 0);	__SetPageLocked(page);	return true;} /** * generic_pipe_buf_try_steal - attempt to take ownership of a &pipe_buffer * @pipe:	the pipe that the buffer belongs to * @buf:	the buffer to attempt to steal * * Description: *	This function attempts to steal the &struct page attached to *	@buf. If successful, this function returns 0 and returns with *	the page locked. The caller may then reuse the page for whatever *	he wishes; the typical use is insertion into a different file *	page cache. */bool generic_pipe_buf_try_steal(struct pipe_inode_info *pipe,		struct pipe_buffer *buf){	struct page *page = buf->page; 	/*	 * A reference of one is golden, that means that the owner of this	 * page is the only one holding a reference to it. lock the page	 * and return OK.	 */	if (page_count(page) == 1) {		lock_page(page);		return true;	}	return false;}EXPORT_SYMBOL(generic_pipe_buf_try_steal); /** * generic_pipe_buf_get - get a reference to a &struct pipe_buffer * @pipe:	the pipe that the buffer belongs to * @buf:	the buffer to get a reference to * * Description: *	This function grabs an extra reference to @buf. It's used in *	the tee() system call, when we duplicate the buffers in one *	pipe into another. */bool generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf){	return try_get_page(buf->page);}EXPORT_SYMBOL(generic_pipe_buf_get); /** * generic_pipe_buf_release - put a reference to a &struct pipe_buffer * @pipe:	the pipe that the buffer belongs to * @buf:	the buffer to put a reference to * * Description: *	This function releases a reference to @buf. */void generic_pipe_buf_release(struct pipe_inode_info *pipe,			      struct pipe_buffer *buf){	put_page(buf->page);}EXPORT_SYMBOL(generic_pipe_buf_release); static const struct pipe_buf_operations anon_pipe_buf_ops = {	.release	= anon_pipe_buf_release,	.try_steal	= anon_pipe_buf_try_steal,	.get		= generic_pipe_buf_get,}; /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */static inline bool pipe_readable(const struct pipe_inode_info *pipe){	union pipe_index idx = { .head_tail = READ_ONCE(pipe->head_tail) };	unsigned int writers = READ_ONCE(pipe->writers); 	return !pipe_empty(idx.head, idx.tail) || !writers;} static inline unsigned int pipe_update_tail(struct pipe_inode_info *pipe,					    struct pipe_buffer *buf,					    unsigned int tail){	pipe_buf_release(pipe, buf); 	/*	 * If the pipe has a watch_queue, we need additional protection	 * by the spinlock because notifications get posted with only	 * this spinlock, no mutex	 */	if (pipe_has_watch_queue(pipe)) {		spin_lock_irq(&pipe->rd_wait.lock);#ifdef CONFIG_WATCH_QUEUE		if (buf->flags & PIPE_BUF_FLAG_LOSS)			pipe->note_loss = true;#endif		pipe->tail = ++tail;		spin_unlock_irq(&pipe->rd_wait.lock);		return tail;	} 	/*	 * Without a watch_queue, we can simply increment the tail	 * without the spinlock - the mutex is enough.	 */	pipe->tail = ++tail;	return tail;} static ssize_tanon_pipe_read(struct kiocb *iocb, struct iov_iter *to){	size_t total_len = iov_iter_count(to);	struct file *filp = iocb->ki_filp;	struct pipe_inode_info *pipe = filp->private_data;	bool wake_writer = false, wake_next_reader = false;	ssize_t ret; 	/* Null read succeeds. */	if (unlikely(total_len == 0))		return 0; 	ret = 0;	mutex_lock(&pipe->mutex); 	/*	 * We only wake up writers if the pipe was full when we started reading	 * and it is no longer full after reading to avoid unnecessary wakeups.	 *	 * But when we do wake up writers, we do so using a sync wakeup	 * (WF_SYNC), because we want them to get going and generate more	 * data for us.	 */	for (;;) {		/* Read ->head with a barrier vs post_one_notification() */		unsigned int head = smp_load_acquire(&pipe->head);		unsigned int tail = pipe->tail; #ifdef CONFIG_WATCH_QUEUE		if (pipe->note_loss) {			struct watch_notification n; 			if (total_len < 8) {				if (ret == 0)					ret = -ENOBUFS;				break;			} 			n.type = WATCH_TYPE_META;			n.subtype = WATCH_META_LOSS_NOTIFICATION;			n.info = watch_sizeof(n);			if (copy_to_iter(&n, sizeof(n), to) != sizeof(n)) {				if (ret == 0)					ret = -EFAULT;				break;			}			ret += sizeof(n);			total_len -= sizeof(n);			pipe->note_loss = false;		}#endif 		if (!pipe_empty(head, tail)) {			struct pipe_buffer *buf = pipe_buf(pipe, tail);			size_t chars = buf->len;			size_t written;			int error; 			if (chars > total_len) {				if (buf->flags & PIPE_BUF_FLAG_WHOLE) {					if (ret == 0)						ret = -ENOBUFS;					break;				}				chars = total_len;			} 			error = pipe_buf_confirm(pipe, buf);			if (error) {				if (!ret)					ret = error;				break;			} 			written = copy_page_to_iter(buf->page, buf->offset, chars, to);			if (unlikely(written < chars)) {				if (!ret)					ret = -EFAULT;				break;			}			ret += chars;			buf->offset += chars;			buf->len -= chars; 			/* Was it a packet buffer? Clean up and exit */			if (buf->flags & PIPE_BUF_FLAG_PACKET) {				total_len = chars;				buf->len = 0;			} 			if (!buf->len) {				wake_writer |= pipe_full(head, tail, pipe->max_usage);				tail = pipe_update_tail(pipe, buf, tail);			}			total_len -= chars;			if (!total_len)				break;	/* common path: read succeeded */			if (!pipe_empty(head, tail))	/* More to do? */				continue;		} 		if (!pipe->writers)			break;		if (ret)			break;		if ((filp->f_flags & O_NONBLOCK) ||		    (iocb->ki_flags & IOCB_NOWAIT)) {			ret = -EAGAIN;			break;		}		mutex_unlock(&pipe->mutex);		/*		 * We only get here if we didn't actually read anything.		 *		 * But because we didn't read anything, at this point we can		 * just return directly with -ERESTARTSYS if we're interrupted,		 * since we've done any required wakeups and there's no need		 * to mark anything accessed. And we've dropped the lock.		 */		if (wait_event_interruptible_exclusive(pipe->rd_wait, pipe_readable(pipe)) < 0)			return -ERESTARTSYS; 		wake_next_reader = true;		mutex_lock(&pipe->mutex);	}	if (pipe_is_empty(pipe))		wake_next_reader = false;	mutex_unlock(&pipe->mutex); 	if (wake_writer)		wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);	if (wake_next_reader)		wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);	kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);	return ret;} static ssize_tfifo_pipe_read(struct kiocb *iocb, struct iov_iter *to){

Lines 1–500 of 1650. Load more