/* * "splice": joining two ropes together by interweaving their strands. * * This is the "extended pipe" functionality, where a pipe is used as * an arbitrary in-memory buffer. Think of a pipe as a small kernel * buffer that you can use to transfer data from one end to the other. * * The traditional unix read/write is extended with a "splice()" operation * that transfers data buffers to or from a pipe buffer. * * Named by Larry McVoy, original implementation from Linus, extended by * Jens to support splicing to files, network, direct splicing, etc and * fixing lots of bugs. * * Copyright (C) 2005-2006 Jens Axboe <axboe@suse.de> * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org> * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu> * */#include <linux/fs.h>#include <linux/file.h>#include <linux/pagemap.h>#include <linux/pipe_fs_i.h>#include <linux/mm_inline.h>#include <linux/swap.h>#include <linux/writeback.h>#include <linux/buffer_head.h>#include <linux/module.h>#include <linux/syscalls.h>#include <linux/uio.h> struct partial_page { unsigned int offset; unsigned int len;}; /* * Passed to splice_to_pipe */struct splice_pipe_desc { struct page **pages; /* page map */ struct partial_page *partial; /* pages[] may not be contig */ int nr_pages; /* number of pages in map */ unsigned int flags; /* splice flags */ struct pipe_buf_operations *ops;/* ops associated with output pipe */}; /* * Attempt to steal a page from a pipe buffer. This should perhaps go into * a vm helper function, it's already simplified quite a bit by the * addition of remove_mapping(). If success is returned, the caller may * attempt to reuse this page for another destination. */static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe, struct pipe_buffer *buf){ struct page *page = buf->page; struct address_space *mapping = page_mapping(page); lock_page(page); WARN_ON(!PageUptodate(page)); /* * At least for ext2 with nobh option, we need to wait on writeback * completing on this page, since we'll remove it from the pagecache. * Otherwise truncate wont wait on the page, allowing the disk * blocks to be reused by someone else before we actually wrote our * data to them. fs corruption ensues. */ wait_on_page_writeback(page); if (PagePrivate(page)) try_to_release_page(page, mapping_gfp_mask(mapping)); if (!remove_mapping(mapping, page)) { unlock_page(page); return 1; } buf->flags |= PIPE_BUF_FLAG_LRU; return 0;} static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe, struct pipe_buffer *buf){ page_cache_release(buf->page); buf->flags &= ~PIPE_BUF_FLAG_LRU;} static int page_cache_pipe_buf_pin(struct pipe_inode_info *pipe, struct pipe_buffer *buf){ struct page *page = buf->page; int err; if (!PageUptodate(page)) { lock_page(page); /* * Page got truncated/unhashed. This will cause a 0-byte * splice, if this is the first page. */ if (!page->mapping) { err = -ENODATA; goto error; } /* * Uh oh, read-error from disk. */ if (!PageUptodate(page)) { err = -EIO; goto error; } /* * Page is ok afterall, we are done. */ unlock_page(page); } return 0;error: unlock_page(page); return err;} static struct pipe_buf_operations page_cache_pipe_buf_ops = { .can_merge = 0, .map = generic_pipe_buf_map, .unmap = generic_pipe_buf_unmap, .pin = page_cache_pipe_buf_pin, .release = page_cache_pipe_buf_release, .steal = page_cache_pipe_buf_steal, .get = generic_pipe_buf_get,}; static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe, struct pipe_buffer *buf){ if (!(buf->flags & PIPE_BUF_FLAG_GIFT)) return 1; buf->flags |= PIPE_BUF_FLAG_LRU; return generic_pipe_buf_steal(pipe, buf);} static struct pipe_buf_operations user_page_pipe_buf_ops = { .can_merge = 0, .map = generic_pipe_buf_map, .unmap = generic_pipe_buf_unmap, .pin = generic_pipe_buf_pin, .release = page_cache_pipe_buf_release, .steal = user_page_pipe_buf_steal, .get = generic_pipe_buf_get,}; /* * Pipe output worker. This sets up our pipe format with the page cache * pipe buffer operations. Otherwise very similar to the regular pipe_writev(). */static ssize_t splice_to_pipe(struct pipe_inode_info *pipe, struct splice_pipe_desc *spd){ int ret, do_wakeup, page_nr; ret = 0; do_wakeup = 0; page_nr = 0; if (pipe->inode) mutex_lock(&pipe->inode->i_mutex); for (;;) { if (!pipe->readers) { send_sig(SIGPIPE, current, 0); if (!ret) ret = -EPIPE; break; } if (pipe->nrbufs < PIPE_BUFFERS) { int newbuf = (pipe->curbuf + pipe->nrbufs) & (PIPE_BUFFERS - 1); struct pipe_buffer *buf = pipe->bufs + newbuf; buf->page = spd->pages[page_nr]; buf->offset = spd->partial[page_nr].offset; buf->len = spd->partial[page_nr].len; buf->ops = spd->ops; if (spd->flags & SPLICE_F_GIFT) buf->flags |= PIPE_BUF_FLAG_GIFT; pipe->nrbufs++; page_nr++; ret += buf->len; if (pipe->inode) do_wakeup = 1; if (!--spd->nr_pages) break; if (pipe->nrbufs < PIPE_BUFFERS) continue; break; } if (spd->flags & SPLICE_F_NONBLOCK) { if (!ret) ret = -EAGAIN; break; } if (signal_pending(current)) { if (!ret) ret = -ERESTARTSYS; break; } if (do_wakeup) { smp_mb(); if (waitqueue_active(&pipe->wait)) wake_up_interruptible_sync(&pipe->wait); kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); do_wakeup = 0; } pipe->waiting_writers++; pipe_wait(pipe); pipe->waiting_writers--; } if (pipe->inode) mutex_unlock(&pipe->inode->i_mutex); if (do_wakeup) { smp_mb(); if (waitqueue_active(&pipe->wait)) wake_up_interruptible(&pipe->wait); kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); } while (page_nr < spd->nr_pages) page_cache_release(spd->pages[page_nr++]); return ret;} static int__generic_file_splice_read(struct file *in, loff_t *ppos, struct pipe_inode_info *pipe, size_t len, unsigned int flags){ struct address_space *mapping = in->f_mapping; unsigned int loff, nr_pages; struct page *pages[PIPE_BUFFERS]; struct partial_page partial[PIPE_BUFFERS]; struct page *page; pgoff_t index, end_index; loff_t isize; size_t total_len; int error, page_nr; struct splice_pipe_desc spd = { .pages = pages, .partial = partial, .flags = flags, .ops = &page_cache_pipe_buf_ops, }; index = *ppos >> PAGE_CACHE_SHIFT; loff = *ppos & ~PAGE_CACHE_MASK; nr_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; if (nr_pages > PIPE_BUFFERS) nr_pages = PIPE_BUFFERS; /* * Initiate read-ahead on this page range. however, don't call into * read-ahead if this is a non-zero offset (we are likely doing small * chunk splice and the page is already there) for a single page. */ if (!loff || nr_pages > 1) page_cache_readahead(mapping, &in->f_ra, in, index, nr_pages); /* * Now fill in the holes: */ error = 0; total_len = 0; /* * Lookup the (hopefully) full range of pages we need. */ spd.nr_pages = find_get_pages_contig(mapping, index, nr_pages, pages); /* * If find_get_pages_contig() returned fewer pages than we needed, * allocate the rest. */ index += spd.nr_pages; while (spd.nr_pages < nr_pages) { /* * Page could be there, find_get_pages_contig() breaks on * the first hole. */ page = find_get_page(mapping, index); if (!page) { /* * Make sure the read-ahead engine is notified * about this failure. */ handle_ra_miss(mapping, &in->f_ra, index); /* * page didn't exist, allocate one. */ page = page_cache_alloc_cold(mapping); if (!page) break; error = add_to_page_cache_lru(page, mapping, index, mapping_gfp_mask(mapping)); if (unlikely(error)) { page_cache_release(page); if (error == -EEXIST) continue; break; } /* * add_to_page_cache() locks the page, unlock it * to avoid convoluting the logic below even more. */ unlock_page(page); } pages[spd.nr_pages++] = page; index++; } /* * Now loop over the map and see if we need to start IO on any * pages, fill in the partial map, etc. */ index = *ppos >> PAGE_CACHE_SHIFT; nr_pages = spd.nr_pages; spd.nr_pages = 0; for (page_nr = 0; page_nr < nr_pages; page_nr++) { unsigned int this_len; if (!len) break; /* * this_len is the max we'll use from this page */ this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff); page = pages[page_nr]; /* * If the page isn't uptodate, we may need to start io on it */ if (!PageUptodate(page)) { /* * If in nonblock mode then dont block on waiting * for an in-flight io page */ if (flags & SPLICE_F_NONBLOCK) break; lock_page(page); /* * page was truncated, stop here. if this isn't the * first page, we'll just complete what we already * added */ if (!page->mapping) { unlock_page(page); break; } /* * page was already under io and is now done, great */ if (PageUptodate(page)) { unlock_page(page); goto fill_it; } /* * need to read in the page */ error = mapping->a_ops->readpage(in, page); if (unlikely(error)) { /* * We really should re-lookup the page here, * but it complicates things a lot. Instead * lets just do what we already stored, and * we'll get it the next time we are called. */ if (error == AOP_TRUNCATED_PAGE) error = 0; break; } /* * i_size must be checked after ->readpage(). */ isize = i_size_read(mapping->host); end_index = (isize - 1) >> PAGE_CACHE_SHIFT; if (unlikely(!isize || index > end_index)) break; /* * if this is the last page, see if we need to shrink * the length and stop */ if (end_index == index) { loff = PAGE_CACHE_SIZE - (isize & ~PAGE_CACHE_MASK); if (total_len + loff > isize) break; /* * force quit after adding this page */ len = this_len; this_len = min(this_len, loff); loff = 0; } }fill_it: partial[page_nr].offset = loff; partial[page_nr].len = this_len; len -= this_len; total_len += this_len; loff = 0; spd.nr_pages++; index++; } /* * Release any pages at the end, if we quit early. 'i' is how far * we got, 'nr_pages' is how many pages are in the map. */ while (page_nr < nr_pages) page_cache_release(pages[page_nr++]); if (spd.nr_pages) return splice_to_pipe(pipe, &spd); return error;} /** * generic_file_splice_read - splice data from file to a pipe * @in: file to splice from * @pipe: pipe to splice to * @len: number of bytes to splice * @flags: splice modifier flags * * Will read pages from given file and fill them into a pipe. */ssize_t generic_file_splice_read(struct file *in, loff_t *ppos, struct pipe_inode_info *pipe, size_t len, unsigned int flags){ ssize_t spliced; int ret; ret = 0; spliced = 0; while (len) { ret = __generic_file_splice_read(in, ppos, pipe, len, flags); if (ret < 0) break; else if (!ret) { if (spliced) break; if (flags & SPLICE_F_NONBLOCK) { ret = -EAGAIN; break; } } *ppos += ret; len -= ret; spliced += ret; } if (spliced) return spliced; return ret;} EXPORT_SYMBOL(generic_file_splice_read); Lines 1–500 of 1490. Load more
/* * "splice": joining two ropes together by interweaving their strands. * * This is the "extended pipe" functionality, where a pipe is used as * an arbitrary in-memory buffer. Think of a pipe as a small kernel * buffer that you can use to transfer data from one end to the other. * * The traditional unix read/write is extended with a "splice()" operation * that transfers data buffers to or from a pipe buffer. * * Named by Larry McVoy, original implementation from Linus, extended by * Jens to support splicing to files, network, direct splicing, etc and * fixing lots of bugs. * * Copyright (C) 2005-2006 Jens Axboe <axboe@suse.de> * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org> * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu> * */ #include <linux/fs.h> #include <linux/file.h> #include <linux/pagemap.h> #include <linux/pipe_fs_i.h> #include <linux/mm_inline.h> #include <linux/swap.h> #include <linux/writeback.h> #include <linux/buffer_head.h> #include <linux/module.h> #include <linux/syscalls.h> #include <linux/uio.h> struct partial_page { unsigned int offset; unsigned int len; }; /* * Passed to splice_to_pipe */ struct splice_pipe_desc { struct page **pages; /* page map */ struct partial_page *partial; /* pages[] may not be contig */ int nr_pages; /* number of pages in map */ unsigned int flags; /* splice flags */ struct pipe_buf_operations *ops;/* ops associated with output pipe */ }; /* * Attempt to steal a page from a pipe buffer. This should perhaps go into * a vm helper function, it's already simplified quite a bit by the * addition of remove_mapping(). If success is returned, the caller may * attempt to reuse this page for another destination. */ static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe, struct pipe_buffer *buf) { struct page *page = buf->page; struct address_space *mapping = page_mapping(page); lock_page(page); WARN_ON(!PageUptodate(page)); /* * At least for ext2 with nobh option, we need to wait on writeback * completing on this page, since we'll remove it from the pagecache. * Otherwise truncate wont wait on the page, allowing the disk * blocks to be reused by someone else before we actually wrote our * data to them. fs corruption ensues. */ wait_on_page_writeback(page); if (PagePrivate(page)) try_to_release_page(page, mapping_gfp_mask(mapping)); if (!remove_mapping(mapping, page)) { unlock_page(page); return 1; } buf->flags |= PIPE_BUF_FLAG_LRU; return 0; } static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe, struct pipe_buffer *buf) { page_cache_release(buf->page); buf->flags &= ~PIPE_BUF_FLAG_LRU; } static int page_cache_pipe_buf_pin(struct pipe_inode_info *pipe, struct pipe_buffer *buf) { struct page *page = buf->page; int err; if (!PageUptodate(page)) { lock_page(page); /* * Page got truncated/unhashed. This will cause a 0-byte * splice, if this is the first page. */ if (!page->mapping) { err = -ENODATA; goto error; } /* * Uh oh, read-error from disk. */ if (!PageUptodate(page)) { err = -EIO; goto error; } /* * Page is ok afterall, we are done. */ unlock_page(page); } return 0; error: unlock_page(page); return err; } static struct pipe_buf_operations page_cache_pipe_buf_ops = { .can_merge = 0, .map = generic_pipe_buf_map, .unmap = generic_pipe_buf_unmap, .pin = page_cache_pipe_buf_pin, .release = page_cache_pipe_buf_release, .steal = page_cache_pipe_buf_steal, .get = generic_pipe_buf_get, }; static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe, struct pipe_buffer *buf) { if (!(buf->flags & PIPE_BUF_FLAG_GIFT)) return 1; buf->flags |= PIPE_BUF_FLAG_LRU; return generic_pipe_buf_steal(pipe, buf); } static struct pipe_buf_operations user_page_pipe_buf_ops = { .can_merge = 0, .map = generic_pipe_buf_map, .unmap = generic_pipe_buf_unmap, .pin = generic_pipe_buf_pin, .release = page_cache_pipe_buf_release, .steal = user_page_pipe_buf_steal, .get = generic_pipe_buf_get, }; /* * Pipe output worker. This sets up our pipe format with the page cache * pipe buffer operations. Otherwise very similar to the regular pipe_writev(). */ static ssize_t splice_to_pipe(struct pipe_inode_info *pipe, struct splice_pipe_desc *spd) { int ret, do_wakeup, page_nr; ret = 0; do_wakeup = 0; page_nr = 0; if (pipe->inode) mutex_lock(&pipe->inode->i_mutex); for (;;) { if (!pipe->readers) { send_sig(SIGPIPE, current, 0); if (!ret) ret = -EPIPE; break; } if (pipe->nrbufs < PIPE_BUFFERS) { int newbuf = (pipe->curbuf + pipe->nrbufs) & (PIPE_BUFFERS - 1); struct pipe_buffer *buf = pipe->bufs + newbuf; buf->page = spd->pages[page_nr]; buf->offset = spd->partial[page_nr].offset; buf->len = spd->partial[page_nr].len; buf->ops = spd->ops; if (spd->flags & SPLICE_F_GIFT) buf->flags |= PIPE_BUF_FLAG_GIFT; pipe->nrbufs++; page_nr++; ret += buf->len; if (pipe->inode) do_wakeup = 1; if (!--spd->nr_pages) break; if (pipe->nrbufs < PIPE_BUFFERS) continue; break; } if (spd->flags & SPLICE_F_NONBLOCK) { if (!ret) ret = -EAGAIN; break; } if (signal_pending(current)) { if (!ret) ret = -ERESTARTSYS; break; } if (do_wakeup) { smp_mb(); if (waitqueue_active(&pipe->wait)) wake_up_interruptible_sync(&pipe->wait); kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); do_wakeup = 0; } pipe->waiting_writers++; pipe_wait(pipe); pipe->waiting_writers--; } if (pipe->inode) mutex_unlock(&pipe->inode->i_mutex); if (do_wakeup) { smp_mb(); if (waitqueue_active(&pipe->wait)) wake_up_interruptible(&pipe->wait); kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); } while (page_nr < spd->nr_pages) page_cache_release(spd->pages[page_nr++]); return ret; } static int __generic_file_splice_read(struct file *in, loff_t *ppos, struct pipe_inode_info *pipe, size_t len, unsigned int flags) { struct address_space *mapping = in->f_mapping; unsigned int loff, nr_pages; struct page *pages[PIPE_BUFFERS]; struct partial_page partial[PIPE_BUFFERS]; struct page *page; pgoff_t index, end_index; loff_t isize; size_t total_len; int error, page_nr; struct splice_pipe_desc spd = { .pages = pages, .partial = partial, .flags = flags, .ops = &page_cache_pipe_buf_ops, }; index = *ppos >> PAGE_CACHE_SHIFT; loff = *ppos & ~PAGE_CACHE_MASK; nr_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; if (nr_pages > PIPE_BUFFERS) nr_pages = PIPE_BUFFERS; /* * Initiate read-ahead on this page range. however, don't call into * read-ahead if this is a non-zero offset (we are likely doing small * chunk splice and the page is already there) for a single page. */ if (!loff || nr_pages > 1) page_cache_readahead(mapping, &in->f_ra, in, index, nr_pages); /* * Now fill in the holes: */ error = 0; total_len = 0; /* * Lookup the (hopefully) full range of pages we need. */ spd.nr_pages = find_get_pages_contig(mapping, index, nr_pages, pages); /* * If find_get_pages_contig() returned fewer pages than we needed, * allocate the rest. */ index += spd.nr_pages; while (spd.nr_pages < nr_pages) { /* * Page could be there, find_get_pages_contig() breaks on * the first hole. */ page = find_get_page(mapping, index); if (!page) { /* * Make sure the read-ahead engine is notified * about this failure. */ handle_ra_miss(mapping, &in->f_ra, index); /* * page didn't exist, allocate one. */ page = page_cache_alloc_cold(mapping); if (!page) break; error = add_to_page_cache_lru(page, mapping, index, mapping_gfp_mask(mapping)); if (unlikely(error)) { page_cache_release(page); if (error == -EEXIST) continue; break; } /* * add_to_page_cache() locks the page, unlock it * to avoid convoluting the logic below even more. */ unlock_page(page); } pages[spd.nr_pages++] = page; index++; } /* * Now loop over the map and see if we need to start IO on any * pages, fill in the partial map, etc. */ index = *ppos >> PAGE_CACHE_SHIFT; nr_pages = spd.nr_pages; spd.nr_pages = 0; for (page_nr = 0; page_nr < nr_pages; page_nr++) { unsigned int this_len; if (!len) break; /* * this_len is the max we'll use from this page */ this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff); page = pages[page_nr]; /* * If the page isn't uptodate, we may need to start io on it */ if (!PageUptodate(page)) { /* * If in nonblock mode then dont block on waiting * for an in-flight io page */ if (flags & SPLICE_F_NONBLOCK) break; lock_page(page); /* * page was truncated, stop here. if this isn't the * first page, we'll just complete what we already * added */ if (!page->mapping) { unlock_page(page); break; } /* * page was already under io and is now done, great */ if (PageUptodate(page)) { unlock_page(page); goto fill_it; } /* * need to read in the page */ error = mapping->a_ops->readpage(in, page); if (unlikely(error)) { /* * We really should re-lookup the page here, * but it complicates things a lot. Instead * lets just do what we already stored, and * we'll get it the next time we are called. */ if (error == AOP_TRUNCATED_PAGE) error = 0; break; } /* * i_size must be checked after ->readpage(). */ isize = i_size_read(mapping->host); end_index = (isize - 1) >> PAGE_CACHE_SHIFT; if (unlikely(!isize || index > end_index)) break; /* * if this is the last page, see if we need to shrink * the length and stop */ if (end_index == index) { loff = PAGE_CACHE_SIZE - (isize & ~PAGE_CACHE_MASK); if (total_len + loff > isize) break; /* * force quit after adding this page */ len = this_len; this_len = min(this_len, loff); loff = 0; } } fill_it: partial[page_nr].offset = loff; partial[page_nr].len = this_len; len -= this_len; total_len += this_len; loff = 0; spd.nr_pages++; index++; } /* * Release any pages at the end, if we quit early. 'i' is how far * we got, 'nr_pages' is how many pages are in the map. */ while (page_nr < nr_pages) page_cache_release(pages[page_nr++]); if (spd.nr_pages) return splice_to_pipe(pipe, &spd); return error; } /** * generic_file_splice_read - splice data from file to a pipe * @in: file to splice from * @pipe: pipe to splice to * @len: number of bytes to splice * @flags: splice modifier flags * * Will read pages from given file and fill them into a pipe. */ ssize_t generic_file_splice_read(struct file *in, loff_t *ppos, struct pipe_inode_info *pipe, size_t len, unsigned int flags) { ssize_t spliced; int ret; ret = 0; spliced = 0; while (len) { ret = __generic_file_splice_read(in, ppos, pipe, len, flags); if (ret < 0) break; else if (!ret) { if (spliced) break; if (flags & SPLICE_F_NONBLOCK) { ret = -EAGAIN; break; } } *ppos += ret; len -= ret; spliced += ret; } if (spliced) return spliced; return ret; } EXPORT_SYMBOL(generic_file_splice_read);