/* * linux/fs/pipe.c * * Copyright (C) 1991, 1992, 1999 Linus Torvalds */ #include <linux/mm.h>#include <linux/file.h>#include <linux/poll.h>#include <linux/slab.h>#include <linux/module.h>#include <linux/init.h>#include <linux/fs.h>#include <linux/mount.h>#include <linux/pipe_fs_i.h>#include <linux/uio.h>#include <linux/highmem.h>#include <linux/pagemap.h>#include <linux/audit.h> #include <asm/uaccess.h>#include <asm/ioctls.h> /* * We use a start+len construction, which provides full use of the * allocated memory. * -- Florian Coosmann (FGC) * * Reads with count = 0 should always return 0. * -- Julian Bradfield 1999-06-07. * * FIFOs and Pipes now generate SIGIO for both readers and writers. * -- Jeremy Elson <jelson@circlemud.org> 2001-08-16 * * pipe_read & write cleanup * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-09 */ /* Drop the inode semaphore and wait for a pipe event, atomically */void pipe_wait(struct pipe_inode_info *pipe){ DEFINE_WAIT(wait); /* * Pipes are system-local resources, so sleeping on them * is considered a noninteractive wait: */ prepare_to_wait(&pipe->wait, &wait, TASK_INTERRUPTIBLE | TASK_NONINTERACTIVE); if (pipe->inode) mutex_unlock(&pipe->inode->i_mutex); schedule(); finish_wait(&pipe->wait, &wait); if (pipe->inode) mutex_lock(&pipe->inode->i_mutex);} static intpipe_iov_copy_from_user(void *to, struct iovec *iov, unsigned long len, int atomic){ unsigned long copy; while (len > 0) { while (!iov->iov_len) iov++; copy = min_t(unsigned long, len, iov->iov_len); if (atomic) { if (__copy_from_user_inatomic(to, iov->iov_base, copy)) return -EFAULT; } else { if (copy_from_user(to, iov->iov_base, copy)) return -EFAULT; } to += copy; len -= copy; iov->iov_base += copy; iov->iov_len -= copy; } return 0;} static intpipe_iov_copy_to_user(struct iovec *iov, const void *from, unsigned long len, int atomic){ unsigned long copy; while (len > 0) { while (!iov->iov_len) iov++; copy = min_t(unsigned long, len, iov->iov_len); if (atomic) { if (__copy_to_user_inatomic(iov->iov_base, from, copy)) return -EFAULT; } else { if (copy_to_user(iov->iov_base, from, copy)) return -EFAULT; } from += copy; len -= copy; iov->iov_base += copy; iov->iov_len -= copy; } return 0;} /* * Attempt to pre-fault in the user memory, so we can use atomic copies. * Returns the number of bytes not faulted in. */static int iov_fault_in_pages_write(struct iovec *iov, unsigned long len){ while (!iov->iov_len) iov++; while (len > 0) { unsigned long this_len; this_len = min_t(unsigned long, len, iov->iov_len); if (fault_in_pages_writeable(iov->iov_base, this_len)) break; len -= this_len; iov++; } return len;} /* * Pre-fault in the user memory, so we can use atomic copies. */static void iov_fault_in_pages_read(struct iovec *iov, unsigned long len){ while (!iov->iov_len) iov++; while (len > 0) { unsigned long this_len; this_len = min_t(unsigned long, len, iov->iov_len); fault_in_pages_readable(iov->iov_base, this_len); len -= this_len; iov++; }} static void anon_pipe_buf_release(struct pipe_inode_info *pipe, struct pipe_buffer *buf){ struct page *page = buf->page; /* * If nobody else uses this page, and we don't already have a * temporary page, let's keep track of it as a one-deep * allocation cache. (Otherwise just release our reference to it) */ if (page_count(page) == 1 && !pipe->tmp_page) pipe->tmp_page = page; else page_cache_release(page);} void *generic_pipe_buf_map(struct pipe_inode_info *pipe, struct pipe_buffer *buf, int atomic){ if (atomic) { buf->flags |= PIPE_BUF_FLAG_ATOMIC; return kmap_atomic(buf->page, KM_USER0); } return kmap(buf->page);} void generic_pipe_buf_unmap(struct pipe_inode_info *pipe, struct pipe_buffer *buf, void *map_data){ if (buf->flags & PIPE_BUF_FLAG_ATOMIC) { buf->flags &= ~PIPE_BUF_FLAG_ATOMIC; kunmap_atomic(map_data, KM_USER0); } else kunmap(buf->page);} int generic_pipe_buf_steal(struct pipe_inode_info *pipe, struct pipe_buffer *buf){ struct page *page = buf->page; if (page_count(page) == 1) { lock_page(page); return 0; } return 1;} void generic_pipe_buf_get(struct pipe_inode_info *info, struct pipe_buffer *buf){ page_cache_get(buf->page);} int generic_pipe_buf_pin(struct pipe_inode_info *info, struct pipe_buffer *buf){ return 0;} static const struct pipe_buf_operations anon_pipe_buf_ops = { .can_merge = 1, .map = generic_pipe_buf_map, .unmap = generic_pipe_buf_unmap, .pin = generic_pipe_buf_pin, .release = anon_pipe_buf_release, .steal = generic_pipe_buf_steal, .get = generic_pipe_buf_get,}; static ssize_tpipe_read(struct kiocb *iocb, const struct iovec *_iov, unsigned long nr_segs, loff_t pos){ struct file *filp = iocb->ki_filp; struct inode *inode = filp->f_path.dentry->d_inode; struct pipe_inode_info *pipe; int do_wakeup; ssize_t ret; struct iovec *iov = (struct iovec *)_iov; size_t total_len; total_len = iov_length(iov, nr_segs); /* Null read succeeds. */ if (unlikely(total_len == 0)) return 0; do_wakeup = 0; ret = 0; mutex_lock(&inode->i_mutex); pipe = inode->i_pipe; for (;;) { int bufs = pipe->nrbufs; if (bufs) { int curbuf = pipe->curbuf; struct pipe_buffer *buf = pipe->bufs + curbuf; const struct pipe_buf_operations *ops = buf->ops; void *addr; size_t chars = buf->len; int error, atomic; if (chars > total_len) chars = total_len; error = ops->pin(pipe, buf); if (error) { if (!ret) error = ret; break; } atomic = !iov_fault_in_pages_write(iov, chars);redo: addr = ops->map(pipe, buf, atomic); error = pipe_iov_copy_to_user(iov, addr + buf->offset, chars, atomic); ops->unmap(pipe, buf, addr); if (unlikely(error)) { /* * Just retry with the slow path if we failed. */ if (atomic) { atomic = 0; goto redo; } if (!ret) ret = error; break; } ret += chars; buf->offset += chars; buf->len -= chars; if (!buf->len) { buf->ops = NULL; ops->release(pipe, buf); curbuf = (curbuf + 1) & (PIPE_BUFFERS-1); pipe->curbuf = curbuf; pipe->nrbufs = --bufs; do_wakeup = 1; } total_len -= chars; if (!total_len) break; /* common path: read succeeded */ } if (bufs) /* More to do? */ continue; if (!pipe->writers) break; if (!pipe->waiting_writers) { /* syscall merging: Usually we must not sleep * if O_NONBLOCK is set, or if we got some data. * But if a writer sleeps in kernel space, then * we can wait for that data without violating POSIX. */ if (ret) break; if (filp->f_flags & O_NONBLOCK) { ret = -EAGAIN; break; } } if (signal_pending(current)) { if (!ret) ret = -ERESTARTSYS; break; } if (do_wakeup) { wake_up_interruptible_sync(&pipe->wait); kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); } pipe_wait(pipe); } mutex_unlock(&inode->i_mutex); /* Signal writers asynchronously that there is more room. */ if (do_wakeup) { wake_up_interruptible(&pipe->wait); kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); } if (ret > 0) file_accessed(filp); return ret;} static ssize_tpipe_write(struct kiocb *iocb, const struct iovec *_iov, unsigned long nr_segs, loff_t ppos){ struct file *filp = iocb->ki_filp; struct inode *inode = filp->f_path.dentry->d_inode; struct pipe_inode_info *pipe; ssize_t ret; int do_wakeup; struct iovec *iov = (struct iovec *)_iov; size_t total_len; ssize_t chars; total_len = iov_length(iov, nr_segs); /* Null write succeeds. */ if (unlikely(total_len == 0)) return 0; do_wakeup = 0; ret = 0; mutex_lock(&inode->i_mutex); pipe = inode->i_pipe; if (!pipe->readers) { send_sig(SIGPIPE, current, 0); ret = -EPIPE; goto out; } /* We try to merge small writes */ chars = total_len & (PAGE_SIZE-1); /* size of the last buffer */ if (pipe->nrbufs && chars != 0) { int lastbuf = (pipe->curbuf + pipe->nrbufs - 1) & (PIPE_BUFFERS-1); struct pipe_buffer *buf = pipe->bufs + lastbuf; const struct pipe_buf_operations *ops = buf->ops; int offset = buf->offset + buf->len; if (ops->can_merge && offset + chars <= PAGE_SIZE) { int error, atomic = 1; void *addr; error = ops->pin(pipe, buf); if (error) goto out; iov_fault_in_pages_read(iov, chars);redo1: addr = ops->map(pipe, buf, atomic); error = pipe_iov_copy_from_user(offset + addr, iov, chars, atomic); ops->unmap(pipe, buf, addr); ret = error; do_wakeup = 1; if (error) { if (atomic) { atomic = 0; goto redo1; } goto out; } buf->len += chars; total_len -= chars; ret = chars; if (!total_len) goto out; } } for (;;) { int bufs; if (!pipe->readers) { send_sig(SIGPIPE, current, 0); if (!ret) ret = -EPIPE; break; } bufs = pipe->nrbufs; if (bufs < PIPE_BUFFERS) { int newbuf = (pipe->curbuf + bufs) & (PIPE_BUFFERS-1); struct pipe_buffer *buf = pipe->bufs + newbuf; struct page *page = pipe->tmp_page; char *src; int error, atomic = 1; if (!page) { page = alloc_page(GFP_HIGHUSER); if (unlikely(!page)) { ret = ret ? : -ENOMEM; break; } pipe->tmp_page = page; } /* Always wake up, even if the copy fails. Otherwise * we lock up (O_NONBLOCK-)readers that sleep due to * syscall merging. * FIXME! Is this really true? */ do_wakeup = 1; chars = PAGE_SIZE; if (chars > total_len) chars = total_len; iov_fault_in_pages_read(iov, chars);redo2: if (atomic) src = kmap_atomic(page, KM_USER0); else src = kmap(page); error = pipe_iov_copy_from_user(src, iov, chars, atomic); if (atomic) kunmap_atomic(src, KM_USER0); else kunmap(page); if (unlikely(error)) { if (atomic) { atomic = 0; goto redo2; } if (!ret) ret = error; break; } ret += chars; /* Insert it into the buffer array */ buf->page = page; buf->ops = &anon_pipe_buf_ops; buf->offset = 0; buf->len = chars; pipe->nrbufs = ++bufs; pipe->tmp_page = NULL; total_len -= chars; if (!total_len) break; } if (bufs < PIPE_BUFFERS) continue; if (filp->f_flags & O_NONBLOCK) { if (!ret) ret = -EAGAIN; break; } if (signal_pending(current)) { if (!ret) ret = -ERESTARTSYS; break; } if (do_wakeup) { wake_up_interruptible_sync(&pipe->wait); kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); do_wakeup = 0; } pipe->waiting_writers++; pipe_wait(pipe); pipe->waiting_writers--; }out: mutex_unlock(&inode->i_mutex); if (do_wakeup) { wake_up_interruptible(&pipe->wait); kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);Lines 1–500 of 1053. Load more
/* * linux/fs/pipe.c * * Copyright (C) 1991, 1992, 1999 Linus Torvalds */ #include <linux/mm.h> #include <linux/file.h> #include <linux/poll.h> #include <linux/slab.h> #include <linux/module.h> #include <linux/init.h> #include <linux/fs.h> #include <linux/mount.h> #include <linux/pipe_fs_i.h> #include <linux/uio.h> #include <linux/highmem.h> #include <linux/pagemap.h> #include <linux/audit.h> #include <asm/uaccess.h> #include <asm/ioctls.h> /* * We use a start+len construction, which provides full use of the * allocated memory. * -- Florian Coosmann (FGC) * * Reads with count = 0 should always return 0. * -- Julian Bradfield 1999-06-07. * * FIFOs and Pipes now generate SIGIO for both readers and writers. * -- Jeremy Elson <jelson@circlemud.org> 2001-08-16 * * pipe_read & write cleanup * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-09 */ /* Drop the inode semaphore and wait for a pipe event, atomically */ void pipe_wait(struct pipe_inode_info *pipe) { DEFINE_WAIT(wait); /* * Pipes are system-local resources, so sleeping on them * is considered a noninteractive wait: */ prepare_to_wait(&pipe->wait, &wait, TASK_INTERRUPTIBLE | TASK_NONINTERACTIVE); if (pipe->inode) mutex_unlock(&pipe->inode->i_mutex); schedule(); finish_wait(&pipe->wait, &wait); if (pipe->inode) mutex_lock(&pipe->inode->i_mutex); } static int pipe_iov_copy_from_user(void *to, struct iovec *iov, unsigned long len, int atomic) { unsigned long copy; while (len > 0) { while (!iov->iov_len) iov++; copy = min_t(unsigned long, len, iov->iov_len); if (atomic) { if (__copy_from_user_inatomic(to, iov->iov_base, copy)) return -EFAULT; } else { if (copy_from_user(to, iov->iov_base, copy)) return -EFAULT; } to += copy; len -= copy; iov->iov_base += copy; iov->iov_len -= copy; } return 0; } static int pipe_iov_copy_to_user(struct iovec *iov, const void *from, unsigned long len, int atomic) { unsigned long copy; while (len > 0) { while (!iov->iov_len) iov++; copy = min_t(unsigned long, len, iov->iov_len); if (atomic) { if (__copy_to_user_inatomic(iov->iov_base, from, copy)) return -EFAULT; } else { if (copy_to_user(iov->iov_base, from, copy)) return -EFAULT; } from += copy; len -= copy; iov->iov_base += copy; iov->iov_len -= copy; } return 0; } /* * Attempt to pre-fault in the user memory, so we can use atomic copies. * Returns the number of bytes not faulted in. */ static int iov_fault_in_pages_write(struct iovec *iov, unsigned long len) { while (!iov->iov_len) iov++; while (len > 0) { unsigned long this_len; this_len = min_t(unsigned long, len, iov->iov_len); if (fault_in_pages_writeable(iov->iov_base, this_len)) break; len -= this_len; iov++; } return len; } /* * Pre-fault in the user memory, so we can use atomic copies. */ static void iov_fault_in_pages_read(struct iovec *iov, unsigned long len) { while (!iov->iov_len) iov++; while (len > 0) { unsigned long this_len; this_len = min_t(unsigned long, len, iov->iov_len); fault_in_pages_readable(iov->iov_base, this_len); len -= this_len; iov++; } } static void anon_pipe_buf_release(struct pipe_inode_info *pipe, struct pipe_buffer *buf) { struct page *page = buf->page; /* * If nobody else uses this page, and we don't already have a * temporary page, let's keep track of it as a one-deep * allocation cache. (Otherwise just release our reference to it) */ if (page_count(page) == 1 && !pipe->tmp_page) pipe->tmp_page = page; else page_cache_release(page); } void *generic_pipe_buf_map(struct pipe_inode_info *pipe, struct pipe_buffer *buf, int atomic) { if (atomic) { buf->flags |= PIPE_BUF_FLAG_ATOMIC; return kmap_atomic(buf->page, KM_USER0); } return kmap(buf->page); } void generic_pipe_buf_unmap(struct pipe_inode_info *pipe, struct pipe_buffer *buf, void *map_data) { if (buf->flags & PIPE_BUF_FLAG_ATOMIC) { buf->flags &= ~PIPE_BUF_FLAG_ATOMIC; kunmap_atomic(map_data, KM_USER0); } else kunmap(buf->page); } int generic_pipe_buf_steal(struct pipe_inode_info *pipe, struct pipe_buffer *buf) { struct page *page = buf->page; if (page_count(page) == 1) { lock_page(page); return 0; } return 1; } void generic_pipe_buf_get(struct pipe_inode_info *info, struct pipe_buffer *buf) { page_cache_get(buf->page); } int generic_pipe_buf_pin(struct pipe_inode_info *info, struct pipe_buffer *buf) { return 0; } static const struct pipe_buf_operations anon_pipe_buf_ops = { .can_merge = 1, .map = generic_pipe_buf_map, .unmap = generic_pipe_buf_unmap, .pin = generic_pipe_buf_pin, .release = anon_pipe_buf_release, .steal = generic_pipe_buf_steal, .get = generic_pipe_buf_get, }; static ssize_t pipe_read(struct kiocb *iocb, const struct iovec *_iov, unsigned long nr_segs, loff_t pos) { struct file *filp = iocb->ki_filp; struct inode *inode = filp->f_path.dentry->d_inode; struct pipe_inode_info *pipe; int do_wakeup; ssize_t ret; struct iovec *iov = (struct iovec *)_iov; size_t total_len; total_len = iov_length(iov, nr_segs); /* Null read succeeds. */ if (unlikely(total_len == 0)) return 0; do_wakeup = 0; ret = 0; mutex_lock(&inode->i_mutex); pipe = inode->i_pipe; for (;;) { int bufs = pipe->nrbufs; if (bufs) { int curbuf = pipe->curbuf; struct pipe_buffer *buf = pipe->bufs + curbuf; const struct pipe_buf_operations *ops = buf->ops; void *addr; size_t chars = buf->len; int error, atomic; if (chars > total_len) chars = total_len; error = ops->pin(pipe, buf); if (error) { if (!ret) error = ret; break; } atomic = !iov_fault_in_pages_write(iov, chars); redo: addr = ops->map(pipe, buf, atomic); error = pipe_iov_copy_to_user(iov, addr + buf->offset, chars, atomic); ops->unmap(pipe, buf, addr); if (unlikely(error)) { /* * Just retry with the slow path if we failed. */ if (atomic) { atomic = 0; goto redo; } if (!ret) ret = error; break; } ret += chars; buf->offset += chars; buf->len -= chars; if (!buf->len) { buf->ops = NULL; ops->release(pipe, buf); curbuf = (curbuf + 1) & (PIPE_BUFFERS-1); pipe->curbuf = curbuf; pipe->nrbufs = --bufs; do_wakeup = 1; } total_len -= chars; if (!total_len) break; /* common path: read succeeded */ } if (bufs) /* More to do? */ continue; if (!pipe->writers) break; if (!pipe->waiting_writers) { /* syscall merging: Usually we must not sleep * if O_NONBLOCK is set, or if we got some data. * But if a writer sleeps in kernel space, then * we can wait for that data without violating POSIX. */ if (ret) break; if (filp->f_flags & O_NONBLOCK) { ret = -EAGAIN; break; } } if (signal_pending(current)) { if (!ret) ret = -ERESTARTSYS; break; } if (do_wakeup) { wake_up_interruptible_sync(&pipe->wait); kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); } pipe_wait(pipe); } mutex_unlock(&inode->i_mutex); /* Signal writers asynchronously that there is more room. */ if (do_wakeup) { wake_up_interruptible(&pipe->wait); kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); } if (ret > 0) file_accessed(filp); return ret; } static ssize_t pipe_write(struct kiocb *iocb, const struct iovec *_iov, unsigned long nr_segs, loff_t ppos) { struct file *filp = iocb->ki_filp; struct inode *inode = filp->f_path.dentry->d_inode; struct pipe_inode_info *pipe; ssize_t ret; int do_wakeup; struct iovec *iov = (struct iovec *)_iov; size_t total_len; ssize_t chars; total_len = iov_length(iov, nr_segs); /* Null write succeeds. */ if (unlikely(total_len == 0)) return 0; do_wakeup = 0; ret = 0; mutex_lock(&inode->i_mutex); pipe = inode->i_pipe; if (!pipe->readers) { send_sig(SIGPIPE, current, 0); ret = -EPIPE; goto out; } /* We try to merge small writes */ chars = total_len & (PAGE_SIZE-1); /* size of the last buffer */ if (pipe->nrbufs && chars != 0) { int lastbuf = (pipe->curbuf + pipe->nrbufs - 1) & (PIPE_BUFFERS-1); struct pipe_buffer *buf = pipe->bufs + lastbuf; const struct pipe_buf_operations *ops = buf->ops; int offset = buf->offset + buf->len; if (ops->can_merge && offset + chars <= PAGE_SIZE) { int error, atomic = 1; void *addr; error = ops->pin(pipe, buf); if (error) goto out; iov_fault_in_pages_read(iov, chars); redo1: addr = ops->map(pipe, buf, atomic); error = pipe_iov_copy_from_user(offset + addr, iov, chars, atomic); ops->unmap(pipe, buf, addr); ret = error; do_wakeup = 1; if (error) { if (atomic) { atomic = 0; goto redo1; } goto out; } buf->len += chars; total_len -= chars; ret = chars; if (!total_len) goto out; } } for (;;) { int bufs; if (!pipe->readers) { send_sig(SIGPIPE, current, 0); if (!ret) ret = -EPIPE; break; } bufs = pipe->nrbufs; if (bufs < PIPE_BUFFERS) { int newbuf = (pipe->curbuf + bufs) & (PIPE_BUFFERS-1); struct pipe_buffer *buf = pipe->bufs + newbuf; struct page *page = pipe->tmp_page; char *src; int error, atomic = 1; if (!page) { page = alloc_page(GFP_HIGHUSER); if (unlikely(!page)) { ret = ret ? : -ENOMEM; break; } pipe->tmp_page = page; } /* Always wake up, even if the copy fails. Otherwise * we lock up (O_NONBLOCK-)readers that sleep due to * syscall merging. * FIXME! Is this really true? */ do_wakeup = 1; chars = PAGE_SIZE; if (chars > total_len) chars = total_len; iov_fault_in_pages_read(iov, chars); redo2: if (atomic) src = kmap_atomic(page, KM_USER0); else src = kmap(page); error = pipe_iov_copy_from_user(src, iov, chars, atomic); if (atomic) kunmap_atomic(src, KM_USER0); else kunmap(page); if (unlikely(error)) { if (atomic) { atomic = 0; goto redo2; } if (!ret) ret = error; break; } ret += chars; /* Insert it into the buffer array */ buf->page = page; buf->ops = &anon_pipe_buf_ops; buf->offset = 0; buf->len = chars; pipe->nrbufs = ++bufs; pipe->tmp_page = NULL; total_len -= chars; if (!total_len) break; } if (bufs < PIPE_BUFFERS) continue; if (filp->f_flags & O_NONBLOCK) { if (!ret) ret = -EAGAIN; break; } if (signal_pending(current)) { if (!ret) ret = -ERESTARTSYS; break; } if (do_wakeup) { wake_up_interruptible_sync(&pipe->wait); kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); do_wakeup = 0; } pipe->waiting_writers++; pipe_wait(pipe); pipe->waiting_writers--; } out: mutex_unlock(&inode->i_mutex); if (do_wakeup) { wake_up_interruptible(&pipe->wait); kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);