History

Merge tag 'fuse-update-6.13' of git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse

Pull fuse updates from Miklos Szeredi: - Add page -> folio conversions (Joanne Koong, Josef Bacik) - Allow max size of fuse requests to be configurable with a sysctl (Joanne Koong) - Allow FOPEN_DIRECT_IO to take advantage of async code path (yangyun) - Fix large kernel reads (like a module load) in virtio_fs (Hou Tao) - Fix attribute inconsistency in case readdirplus (and plain lookup in corner cases) is racing with inode eviction (Zhang Tianci) - Fix a WARN_ON triggered by virtio_fs (Asahi Lina) * tag 'fuse-update-6.13' of git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse: (30 commits) virtiofs: dax: remove ->writepages() callback fuse: check attributes staleness on fuse_iget() fuse: remove pages for requests and exclusively use folios fuse: convert direct io to use folios mm/writeback: add folio_mark_dirty_lock() fuse: convert writebacks to use folios fuse: convert retrieves to use folios fuse: convert ioctls to use folios fuse: convert writes (non-writeback) to use folios fuse: convert reads to use folios fuse: convert readdir to use folios fuse: convert readlink to use folios fuse: convert cuse to use folios fuse: add support in virtio for requests using folios fuse: support folios in struct fuse_args_pages and fuse_copy_pages() fuse: convert fuse_notify_store to use folios fuse: convert fuse_retrieve to use folios fuse: use the folio based vmstat helpers fuse: convert fuse_writepage_need_send to take a folio fuse: convert fuse_do_readpage to use folios ...

Linus Torvalds committed 1 year ago

Commit fb527fc2 parents ff2a7a0 d1dfb5f16 files · +578 −374Browse files

Unified · Split

16 files
  1. Documentation/admin-guide/sysctl/fs.rst +10 −0
  2. fs/fuse/Makefile +1 −0
  3. fs/fuse/cuse.c +15 −14
  4. fs/fuse/dax.c +0 −11
  5. fs/fuse/dev.c +39 −27
  6. fs/fuse/dir.c +19 −18
  7. fs/fuse/file.c +258 −191
  8. fs/fuse/fuse_i.h +46 −22
  9. fs/fuse/inode.c +54 −13
  10. fs/fuse/ioctl.c +18 −17
  11. fs/fuse/readdir.c +18 −15
  12. fs/fuse/sysctl.c +40 −0
  13. fs/fuse/virtio_fs.c +42 −35
  14. include/linux/mm.h +1 −0
  15. mm/folio-compat.c +6 −0
  16. mm/page-writeback.c +11 −11
Documentation/admin-guide/sysctl/fs.rst+10−0
@@ -337,3 +337,13 @@ Each "watch" costs roughly 90 bytes on a 32-bit kernel, and roughly 160 bytes · 336 unmodified lines
337on a 64-bit one.337on a 64-bit one.
338The current default value for ``max_user_watches`` is 4% of the338The current default value for ``max_user_watches`` is 4% of the
339available low memory, divided by the "watch" cost in bytes.339available low memory, divided by the "watch" cost in bytes.
340
3415. /proc/sys/fs/fuse - Configuration options for FUSE filesystems
342=====================================================================
343
344This directory contains the following configuration options for FUSE
345filesystems:
346
347``/proc/sys/fs/fuse/max_pages_limit`` is a read/write file for
348setting/getting the maximum number of pages that can be used for servicing
349requests in FUSE.
fs/fuse/Makefile+1−0
@@ -14,5 +14,6 @@ fuse-y := dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o · 13 unmodified lines
14fuse-y += iomode.o14fuse-y += iomode.o
15fuse-$(CONFIG_FUSE_DAX) += dax.o15fuse-$(CONFIG_FUSE_DAX) += dax.o
16fuse-$(CONFIG_FUSE_PASSTHROUGH) += passthrough.o16fuse-$(CONFIG_FUSE_PASSTHROUGH) += passthrough.o
17fuse-$(CONFIG_SYSCTL) += sysctl.o
17 18
18virtiofs-y := virtio_fs.o19virtiofs-y := virtio_fs.o
fs/fuse/cuse.c+15−14
@@ -303,8 +303,8 @@ struct cuse_init_args { · 302 unmodified lines
303 struct fuse_args_pages ap;303 struct fuse_args_pages ap;
304 struct cuse_init_in in;304 struct cuse_init_in in;
305 struct cuse_init_out out;305 struct cuse_init_out out;
306 struct page *page;306 struct folio *folio;
307 struct fuse_page_desc desc;307 struct fuse_folio_desc desc;
308};308};
309 309
310/**310/**
@@ -326,7 +326,7 @@ static void cuse_process_init_reply(struct fuse_mount *fm, · 15 unmodified lines
326 struct fuse_args_pages *ap = &ia->ap;326 struct fuse_args_pages *ap = &ia->ap;
327 struct cuse_conn *cc = fc_to_cc(fc), *pos;327 struct cuse_conn *cc = fc_to_cc(fc), *pos;
328 struct cuse_init_out *arg = &ia->out;328 struct cuse_init_out *arg = &ia->out;
329 struct page *page = ap->pages[0];329 struct folio *folio = ap->folios[0];
330 struct cuse_devinfo devinfo = { };330 struct cuse_devinfo devinfo = { };
331 struct device *dev;331 struct device *dev;
332 struct cdev *cdev;332 struct cdev *cdev;
@@ -343,7 +343,7 @@ static void cuse_process_init_reply(struct fuse_mount *fm, · 10 unmodified lines
343 /* parse init reply */343 /* parse init reply */
344 cc->unrestricted_ioctl = arg->flags & CUSE_UNRESTRICTED_IOCTL;344 cc->unrestricted_ioctl = arg->flags & CUSE_UNRESTRICTED_IOCTL;
345 345
346 rc = cuse_parse_devinfo(page_address(page), ap->args.out_args[1].size,346 rc = cuse_parse_devinfo(folio_address(folio), ap->args.out_args[1].size,
347 &devinfo);347 &devinfo);
348 if (rc)348 if (rc)
349 goto err;349 goto err;
@@ -411,7 +411,7 @@ static void cuse_process_init_reply(struct fuse_mount *fm, · 61 unmodified lines
411 kobject_uevent(&dev->kobj, KOBJ_ADD);411 kobject_uevent(&dev->kobj, KOBJ_ADD);
412out:412out:
413 kfree(ia);413 kfree(ia);
414 __free_page(page);414 folio_put(folio);
415 return;415 return;
416 416
417err_cdev:417err_cdev:
@@ -429,7 +429,7 @@ err: · 11 unmodified lines
429static int cuse_send_init(struct cuse_conn *cc)429static int cuse_send_init(struct cuse_conn *cc)
430{430{
431 int rc;431 int rc;
432 struct page *page;432 struct folio *folio;
433 struct fuse_mount *fm = &cc->fm;433 struct fuse_mount *fm = &cc->fm;
434 struct cuse_init_args *ia;434 struct cuse_init_args *ia;
435 struct fuse_args_pages *ap;435 struct fuse_args_pages *ap;
@@ -437,13 +437,14 @@ static int cuse_send_init(struct cuse_conn *cc) · 1 unmodified line
437 BUILD_BUG_ON(CUSE_INIT_INFO_MAX > PAGE_SIZE);437 BUILD_BUG_ON(CUSE_INIT_INFO_MAX > PAGE_SIZE);
438 438
439 rc = -ENOMEM;439 rc = -ENOMEM;
440 page = alloc_page(GFP_KERNEL | __GFP_ZERO);440
441 if (!page)441 folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0);
442 if (!folio)
442 goto err;443 goto err;
443 444
444 ia = kzalloc(sizeof(*ia), GFP_KERNEL);445 ia = kzalloc(sizeof(*ia), GFP_KERNEL);
445 if (!ia)446 if (!ia)
446 goto err_free_page;447 goto err_free_folio;
447 448
448 ap = &ia->ap;449 ap = &ia->ap;
449 ia->in.major = FUSE_KERNEL_VERSION;450 ia->in.major = FUSE_KERNEL_VERSION;
@@ -459,18 +460,18 @@ static int cuse_send_init(struct cuse_conn *cc) · 9 unmodified lines
459 ap->args.out_args[1].size = CUSE_INIT_INFO_MAX;460 ap->args.out_args[1].size = CUSE_INIT_INFO_MAX;
460 ap->args.out_argvar = true;461 ap->args.out_argvar = true;
461 ap->args.out_pages = true;462 ap->args.out_pages = true;
462 ap->num_pages = 1;463 ap->num_folios = 1;
463 ap->pages = &ia->page;464 ap->folios = &ia->folio;
464 ap->descs = &ia->desc;465 ap->descs = &ia->desc;
465 ia->page = page;466 ia->folio = folio;
466 ia->desc.length = ap->args.out_args[1].size;467 ia->desc.length = ap->args.out_args[1].size;
467 ap->args.end = cuse_process_init_reply;468 ap->args.end = cuse_process_init_reply;
468 469
469 rc = fuse_simple_background(fm, &ap->args, GFP_KERNEL);470 rc = fuse_simple_background(fm, &ap->args, GFP_KERNEL);
470 if (rc) {471 if (rc) {
471 kfree(ia);472 kfree(ia);
472err_free_page:473err_free_folio:
473 __free_page(page);474 folio_put(folio);
474 }475 }
475err:476err:
476 return rc;477 return rc;
fs/fuse/dax.c+0−11
@@ -774,16 +774,6 @@ out: · 773 unmodified lines
774 return ret;774 return ret;
775}775}
776 776
777static int fuse_dax_writepages(struct address_space *mapping,
778 struct writeback_control *wbc)
779{
780
781 struct inode *inode = mapping->host;
782 struct fuse_conn *fc = get_fuse_conn(inode);
783
784 return dax_writeback_mapping_range(mapping, fc->dax->dev, wbc);
785}
786
787static vm_fault_t __fuse_dax_fault(struct vm_fault *vmf, unsigned int order,777static vm_fault_t __fuse_dax_fault(struct vm_fault *vmf, unsigned int order,
788 bool write)778 bool write)
789{779{
@@ -1323,7 +1313,6 @@ bool fuse_dax_inode_alloc(struct super_block *sb, struct fuse_inode *fi) · 533 unmodified lines
1323}1313}
1324 1314
1325static const struct address_space_operations fuse_dax_file_aops = {1315static const struct address_space_operations fuse_dax_file_aops = {
1326 .writepages = fuse_dax_writepages,
1327 .direct_IO = noop_direct_IO,1316 .direct_IO = noop_direct_IO,
1328 .dirty_folio = noop_dirty_folio,1317 .dirty_folio = noop_dirty_folio,
1329};1318};
fs/fuse/dev.c+39−27
@@ -1028,17 +1028,27 @@ static int fuse_copy_pages(struct fuse_copy_state *cs, unsigned nbytes, · 1027 unmodified lines
1028 struct fuse_req *req = cs->req;1028 struct fuse_req *req = cs->req;
1029 struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args);1029 struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args);
1030 1030
1031 1031 for (i = 0; i < ap->num_folios && (nbytes || zeroing); i++) {
1032 for (i = 0; i < ap->num_pages && (nbytes || zeroing); i++) {
1033 int err;1032 int err;
1034 unsigned int offset = ap->descs[i].offset;1033 unsigned int offset = ap->descs[i].offset;
1035 unsigned int count = min(nbytes, ap->descs[i].length);1034 unsigned int count = min(nbytes, ap->descs[i].length);
1035 struct page *orig, *pagep;
1036
1037 orig = pagep = &ap->folios[i]->page;
1036 1038
1037 err = fuse_copy_page(cs, &ap->pages[i], offset, count, zeroing);1039 err = fuse_copy_page(cs, &pagep, offset, count, zeroing);
1038 if (err)1040 if (err)
1039 return err;1041 return err;
1040 1042
1041 nbytes -= count;1043 nbytes -= count;
1044
1045 /*
1046 * fuse_copy_page may have moved a page from a pipe instead of
1047 * copying into our given page, so update the folios if it was
1048 * replaced.
1049 */
1050 if (pagep != orig)
1051 ap->folios[i] = page_folio(pagep);
1042 }1052 }
1043 return 0;1053 return 0;
1044}1054}
@@ -1654,24 +1664,25 @@ static int fuse_notify_store(struct fuse_conn *fc, unsigned int size, · 609 unmodified lines
1654 1664
1655 num = outarg.size;1665 num = outarg.size;
1656 while (num) {1666 while (num) {
1667 struct folio *folio;
1657 struct page *page;1668 struct page *page;
1658 unsigned int this_num;1669 unsigned int this_num;
1659 1670
1660 err = -ENOMEM;1671 folio = filemap_grab_folio(mapping, index);
1661 page = find_or_create_page(mapping, index,1672 err = PTR_ERR(folio);
1662 mapping_gfp_mask(mapping));1673 if (IS_ERR(folio))
1663 if (!page)
1664 goto out_iput;1674 goto out_iput;
1665 1675
1666 this_num = min_t(unsigned, num, PAGE_SIZE - offset);1676 page = &folio->page;
1677 this_num = min_t(unsigned, num, folio_size(folio) - offset);
1667 err = fuse_copy_page(cs, &page, offset, this_num, 0);1678 err = fuse_copy_page(cs, &page, offset, this_num, 0);
1668 if (!PageUptodate(page) && !err && offset == 0 &&1679 if (!folio_test_uptodate(folio) && !err && offset == 0 &&
1669 (this_num == PAGE_SIZE || file_size == end)) {1680 (this_num == folio_size(folio) || file_size == end)) {
1670 zero_user_segment(page, this_num, PAGE_SIZE);1681 folio_zero_segment(folio, this_num, folio_size(folio));
1671 SetPageUptodate(page);1682 folio_mark_uptodate(folio);
1672 }1683 }
1673 unlock_page(page);1684 folio_unlock(folio);
1674 put_page(page);1685 folio_put(folio);
1675 1686
1676 if (err)1687 if (err)
1677 goto out_iput;1688 goto out_iput;
@@ -1703,7 +1714,7 @@ static void fuse_retrieve_end(struct fuse_mount *fm, struct fuse_args *args, · 25 unmodified lines
1703 struct fuse_retrieve_args *ra =1714 struct fuse_retrieve_args *ra =
1704 container_of(args, typeof(*ra), ap.args);1715 container_of(args, typeof(*ra), ap.args);
1705 1716
1706 release_pages(ra->ap.pages, ra->ap.num_pages);1717 release_pages(ra->ap.folios, ra->ap.num_folios);
1707 kfree(ra);1718 kfree(ra);
1708}1719}
1709 1720
@@ -1717,7 +1728,7 @@ static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode, · 7 unmodified lines
1717 unsigned int num;1728 unsigned int num;
1718 unsigned int offset;1729 unsigned int offset;
1719 size_t total_len = 0;1730 size_t total_len = 0;
1720 unsigned int num_pages;1731 unsigned int num_pages, cur_pages = 0;
1721 struct fuse_conn *fc = fm->fc;1732 struct fuse_conn *fc = fm->fc;
1722 struct fuse_retrieve_args *ra;1733 struct fuse_retrieve_args *ra;
1723 size_t args_size = sizeof(*ra);1734 size_t args_size = sizeof(*ra);
@@ -1736,15 +1747,15 @@ static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode, · 12 unmodified lines
1736 num_pages = (num + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;1747 num_pages = (num + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
1737 num_pages = min(num_pages, fc->max_pages);1748 num_pages = min(num_pages, fc->max_pages);
1738 1749
1739 args_size += num_pages * (sizeof(ap->pages[0]) + sizeof(ap->descs[0]));1750 args_size += num_pages * (sizeof(ap->folios[0]) + sizeof(ap->descs[0]));
1740 1751
1741 ra = kzalloc(args_size, GFP_KERNEL);1752 ra = kzalloc(args_size, GFP_KERNEL);
1742 if (!ra)1753 if (!ra)
1743 return -ENOMEM;1754 return -ENOMEM;
1744 1755
1745 ap = &ra->ap;1756 ap = &ra->ap;
1746 ap->pages = (void *) (ra + 1);1757 ap->folios = (void *) (ra + 1);
1747 ap->descs = (void *) (ap->pages + num_pages);1758 ap->descs = (void *) (ap->folios + num_pages);
1748 1759
1749 args = &ap->args;1760 args = &ap->args;
1750 args->nodeid = outarg->nodeid;1761 args->nodeid = outarg->nodeid;
@@ -1755,19 +1766,20 @@ static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode, · 4 unmodified lines
1755 1766
1756 index = outarg->offset >> PAGE_SHIFT;1767 index = outarg->offset >> PAGE_SHIFT;
1757 1768
1758 while (num && ap->num_pages < num_pages) {1769 while (num && cur_pages < num_pages) {
1759 struct page *page;1770 struct folio *folio;
1760 unsigned int this_num;1771 unsigned int this_num;
1761 1772
1762 page = find_get_page(mapping, index);1773 folio = filemap_get_folio(mapping, index);
1763 if (!page)1774 if (IS_ERR(folio))
1764 break;1775 break;
1765 1776
1766 this_num = min_t(unsigned, num, PAGE_SIZE - offset);1777 this_num = min_t(unsigned, num, PAGE_SIZE - offset);
1767 ap->pages[ap->num_pages] = page;1778 ap->folios[ap->num_folios] = folio;
1768 ap->descs[ap->num_pages].offset = offset;1779 ap->descs[ap->num_folios].offset = offset;
1769 ap->descs[ap->num_pages].length = this_num;1780 ap->descs[ap->num_folios].length = this_num;
1770 ap->num_pages++;1781 ap->num_folios++;
1782 cur_pages++;
1771 1783
1772 offset = 0;1784 offset = 0;
1773 num -= this_num;1785 num -= this_num;
fs/fuse/dir.c+19−18
@@ -366,7 +366,7 @@ int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name · 365 unmodified lines
366 struct fuse_mount *fm = get_fuse_mount_super(sb);366 struct fuse_mount *fm = get_fuse_mount_super(sb);
367 FUSE_ARGS(args);367 FUSE_ARGS(args);
368 struct fuse_forget_link *forget;368 struct fuse_forget_link *forget;
369 u64 attr_version;369 u64 attr_version, evict_ctr;
370 int err;370 int err;
371 371
372 *inode = NULL;372 *inode = NULL;
@@ -381,6 +381,7 @@ int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name · 8 unmodified lines
381 goto out;381 goto out;
382 382
383 attr_version = fuse_get_attr_version(fm->fc);383 attr_version = fuse_get_attr_version(fm->fc);
384 evict_ctr = fuse_get_evict_ctr(fm->fc);
384 385
385 fuse_lookup_init(fm->fc, &args, nodeid, name, outarg);386 fuse_lookup_init(fm->fc, &args, nodeid, name, outarg);
386 err = fuse_simple_request(fm, &args);387 err = fuse_simple_request(fm, &args);
@@ -398,7 +399,7 @@ int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name · 11 unmodified lines
398 399
399 *inode = fuse_iget(sb, outarg->nodeid, outarg->generation,400 *inode = fuse_iget(sb, outarg->nodeid, outarg->generation,
400 &outarg->attr, ATTR_TIMEOUT(outarg),401 &outarg->attr, ATTR_TIMEOUT(outarg),
401 attr_version);402 attr_version, evict_ctr);
402 err = -ENOMEM;403 err = -ENOMEM;
403 if (!*inode) {404 if (!*inode) {
404 fuse_queue_forget(fm->fc, forget, outarg->nodeid, 1);405 fuse_queue_forget(fm->fc, forget, outarg->nodeid, 1);
@@ -691,7 +692,7 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, · 286 unmodified lines
691 ff->nodeid = outentry.nodeid;692 ff->nodeid = outentry.nodeid;
692 ff->open_flags = outopenp->open_flags;693 ff->open_flags = outopenp->open_flags;
693 inode = fuse_iget(dir->i_sb, outentry.nodeid, outentry.generation,694 inode = fuse_iget(dir->i_sb, outentry.nodeid, outentry.generation,
694 &outentry.attr, ATTR_TIMEOUT(&outentry), 0);695 &outentry.attr, ATTR_TIMEOUT(&outentry), 0, 0);
695 if (!inode) {696 if (!inode) {
696 flags &= ~(O_CREAT | O_EXCL | O_TRUNC);697 flags &= ~(O_CREAT | O_EXCL | O_TRUNC);
697 fuse_sync_release(NULL, ff, flags);698 fuse_sync_release(NULL, ff, flags);
@@ -822,7 +823,7 @@ static int create_new_entry(struct mnt_idmap *idmap, struct fuse_mount *fm, · 124 unmodified lines
822 goto out_put_forget_req;823 goto out_put_forget_req;
823 824
824 inode = fuse_iget(dir->i_sb, outarg.nodeid, outarg.generation,825 inode = fuse_iget(dir->i_sb, outarg.nodeid, outarg.generation,
825 &outarg.attr, ATTR_TIMEOUT(&outarg), 0);826 &outarg.attr, ATTR_TIMEOUT(&outarg), 0, 0);
826 if (!inode) {827 if (!inode) {
827 fuse_queue_forget(fm->fc, forget, outarg.nodeid, 1);828 fuse_queue_forget(fm->fc, forget, outarg.nodeid, 1);
828 return -ENOMEM;829 return -ENOMEM;
@@ -1585,13 +1586,13 @@ static int fuse_permission(struct mnt_idmap *idmap, · 756 unmodified lines
1585 return err;1586 return err;
1586}1587}
1587 1588
1588static int fuse_readlink_page(struct inode *inode, struct page *page)1589static int fuse_readlink_page(struct inode *inode, struct folio *folio)
1589{1590{
1590 struct fuse_mount *fm = get_fuse_mount(inode);1591 struct fuse_mount *fm = get_fuse_mount(inode);
1591 struct fuse_page_desc desc = { .length = PAGE_SIZE - 1 };1592 struct fuse_folio_desc desc = { .length = PAGE_SIZE - 1 };
1592 struct fuse_args_pages ap = {1593 struct fuse_args_pages ap = {
1593 .num_pages = 1,1594 .num_folios = 1,
1594 .pages = &page,1595 .folios = &folio,
1595 .descs = &desc,1596 .descs = &desc,
1596 };1597 };
1597 char *link;1598 char *link;
@@ -1614,7 +1615,7 @@ static int fuse_readlink_page(struct inode *inode, struct page *page) · 16 unmodified lines
1614 if (WARN_ON(res >= PAGE_SIZE))1615 if (WARN_ON(res >= PAGE_SIZE))
1615 return -EIO;1616 return -EIO;
1616 1617
1617 link = page_address(page);1618 link = folio_address(folio);
1618 link[res] = '\0';1619 link[res] = '\0';
1619 1620
1620 return 0;1621 return 0;
@@ -1624,7 +1625,7 @@ static const char *fuse_get_link(struct dentry *dentry, struct inode *inode, · 3 unmodified lines
1624 struct delayed_call *callback)1625 struct delayed_call *callback)
1625{1626{
1626 struct fuse_conn *fc = get_fuse_conn(inode);1627 struct fuse_conn *fc = get_fuse_conn(inode);
1627 struct page *page;1628 struct folio *folio;
1628 int err;1629 int err;
1629 1630
1630 err = -EIO;1631 err = -EIO;
@@ -1638,20 +1639,20 @@ static const char *fuse_get_link(struct dentry *dentry, struct inode *inode, · 7 unmodified lines
1638 if (!dentry)1639 if (!dentry)
1639 goto out_err;1640 goto out_err;
1640 1641
1641 page = alloc_page(GFP_KERNEL);1642 folio = folio_alloc(GFP_KERNEL, 0);
1642 err = -ENOMEM;1643 err = -ENOMEM;
1643 if (!page)1644 if (!folio)
1644 goto out_err;1645 goto out_err;
1645 1646
1646 err = fuse_readlink_page(inode, page);1647 err = fuse_readlink_page(inode, folio);
1647 if (err) {1648 if (err) {
1648 __free_page(page);1649 folio_put(folio);
1649 goto out_err;1650 goto out_err;
1650 }1651 }
1651 1652
1652 set_delayed_call(callback, page_put_link, page);1653 set_delayed_call(callback, page_put_link, &folio->page);
1653 1654
1654 return page_address(page);1655 return folio_address(folio);
1655 1656
1656out_err:1657out_err:
1657 return ERR_PTR(err);1658 return ERR_PTR(err);
@@ -2028,7 +2029,7 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, · 370 unmodified lines
2028 2029
2029 fuse_change_attributes_common(inode, &outarg.attr, NULL,2030 fuse_change_attributes_common(inode, &outarg.attr, NULL,
2030 ATTR_TIMEOUT(&outarg),2031 ATTR_TIMEOUT(&outarg),
2031 fuse_get_cache_mask(inode));2032 fuse_get_cache_mask(inode), 0);
2032 oldsize = inode->i_size;2033 oldsize = inode->i_size;
2033 /* see the comment in fuse_change_attributes() */2034 /* see the comment in fuse_change_attributes() */
2034 if (!is_wb || is_truncate)2035 if (!is_wb || is_truncate)
@@ -2231,7 +2232,7 @@ void fuse_init_dir(struct inode *inode) · 196 unmodified lines
2231 2232
2232static int fuse_symlink_read_folio(struct file *null, struct folio *folio)2233static int fuse_symlink_read_folio(struct file *null, struct folio *folio)
2233{2234{
2234 int err = fuse_readlink_page(folio->mapping->host, &folio->page);2235 int err = fuse_readlink_page(folio->mapping->host, folio);
2235 2236
2236 if (!err)2237 if (!err)
2237 folio_mark_uptodate(folio);2238 folio_mark_uptodate(folio);
fs/fuse/file.c+258−191
@@ -436,7 +436,7 @@ static struct fuse_writepage_args *fuse_find_writeback(struct fuse_inode *fi, · 435 unmodified lines
436 wpa = rb_entry(n, struct fuse_writepage_args, writepages_entry);436 wpa = rb_entry(n, struct fuse_writepage_args, writepages_entry);
437 WARN_ON(get_fuse_inode(wpa->inode) != fi);437 WARN_ON(get_fuse_inode(wpa->inode) != fi);
438 curr_index = wpa->ia.write.in.offset >> PAGE_SHIFT;438 curr_index = wpa->ia.write.in.offset >> PAGE_SHIFT;
439 if (idx_from >= curr_index + wpa->ia.ap.num_pages)439 if (idx_from >= curr_index + wpa->ia.ap.num_folios)
440 n = n->rb_right;440 n = n->rb_right;
441 else if (idx_to < curr_index)441 else if (idx_to < curr_index)
442 n = n->rb_left;442 n = n->rb_left;
@@ -483,6 +483,21 @@ static void fuse_wait_on_page_writeback(struct inode *inode, pgoff_t index) · 40 unmodified lines
483 wait_event(fi->page_waitq, !fuse_page_is_writeback(inode, index));483 wait_event(fi->page_waitq, !fuse_page_is_writeback(inode, index));
484}484}
485 485
486static inline bool fuse_folio_is_writeback(struct inode *inode,
487 struct folio *folio)
488{
489 pgoff_t last = folio_next_index(folio) - 1;
490 return fuse_range_is_writeback(inode, folio_index(folio), last);
491}
492
493static void fuse_wait_on_folio_writeback(struct inode *inode,
494 struct folio *folio)
495{
496 struct fuse_inode *fi = get_fuse_inode(inode);
497
498 wait_event(fi->page_waitq, !fuse_folio_is_writeback(inode, folio));
499}
500
486/*501/*
487 * Wait for all pending writepages on the inode to finish.502 * Wait for all pending writepages on the inode to finish.
488 *503 *
@@ -645,17 +660,20 @@ void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos, · 156 unmodified lines
645 args->out_args[0].size = count;660 args->out_args[0].size = count;
646}661}
647 662
648static void fuse_release_user_pages(struct fuse_args_pages *ap,663static void fuse_release_user_pages(struct fuse_args_pages *ap, ssize_t nres,
649 bool should_dirty)664 bool should_dirty)
650{665{
651 unsigned int i;666 unsigned int i;
652 667
653 for (i = 0; i < ap->num_pages; i++) {668 for (i = 0; i < ap->num_folios; i++) {
654 if (should_dirty)669 if (should_dirty)
655 set_page_dirty_lock(ap->pages[i]);670 folio_mark_dirty_lock(ap->folios[i]);
656 if (ap->args.is_pinned)671 if (ap->args.is_pinned)
657 unpin_user_page(ap->pages[i]);672 unpin_folio(ap->folios[i]);
658 }673 }
674
675 if (nres > 0 && ap->args.invalidate_vmap)
676 invalidate_kernel_vmap_range(ap->args.vmap_base, nres);
659}677}
660 678
661static void fuse_io_release(struct kref *kref)679static void fuse_io_release(struct kref *kref)
@@ -725,16 +743,16 @@ static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos) · 63 unmodified lines
725}743}
726 744
727static struct fuse_io_args *fuse_io_alloc(struct fuse_io_priv *io,745static struct fuse_io_args *fuse_io_alloc(struct fuse_io_priv *io,
728 unsigned int npages)746 unsigned int nfolios)
729{747{
730 struct fuse_io_args *ia;748 struct fuse_io_args *ia;
731 749
732 ia = kzalloc(sizeof(*ia), GFP_KERNEL);750 ia = kzalloc(sizeof(*ia), GFP_KERNEL);
733 if (ia) {751 if (ia) {
734 ia->io = io;752 ia->io = io;
735 ia->ap.pages = fuse_pages_alloc(npages, GFP_KERNEL,753 ia->ap.folios = fuse_folios_alloc(nfolios, GFP_KERNEL,
736 &ia->ap.descs);754 &ia->ap.descs);
737 if (!ia->ap.pages) {755 if (!ia->ap.folios) {
738 kfree(ia);756 kfree(ia);
739 ia = NULL;757 ia = NULL;
740 }758 }
@@ -744,7 +762,7 @@ static struct fuse_io_args *fuse_io_alloc(struct fuse_io_priv *io, · 3 unmodified lines
744 762
745static void fuse_io_free(struct fuse_io_args *ia)763static void fuse_io_free(struct fuse_io_args *ia)
746{764{
747 kfree(ia->ap.pages);765 kfree(ia->ap.folios);
748 kfree(ia);766 kfree(ia);
749}767}
750 768
@@ -754,25 +772,29 @@ static void fuse_aio_complete_req(struct fuse_mount *fm, struct fuse_args *args, · 3 unmodified lines
754 struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);772 struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);
755 struct fuse_io_priv *io = ia->io;773 struct fuse_io_priv *io = ia->io;
756 ssize_t pos = -1;774 ssize_t pos = -1;
757 775 size_t nres;
758 fuse_release_user_pages(&ia->ap, io->should_dirty);
759 776
760 if (err) {777 if (err) {
761 /* Nothing */778 /* Nothing */
762 } else if (io->write) {779 } else if (io->write) {
763 if (ia->write.out.size > ia->write.in.size) {780 if (ia->write.out.size > ia->write.in.size) {
764 err = -EIO;781 err = -EIO;
765 } else if (ia->write.in.size != ia->write.out.size) {782 } else {
766 pos = ia->write.in.offset - io->offset +783 nres = ia->write.out.size;
767 ia->write.out.size;784 if (ia->write.in.size != ia->write.out.size)
785 pos = ia->write.in.offset - io->offset +
786 ia->write.out.size;
768 }787 }
769 } else {788 } else {
770 u32 outsize = args->out_args[0].size;789 u32 outsize = args->out_args[0].size;
771 790
791 nres = outsize;
772 if (ia->read.in.size != outsize)792 if (ia->read.in.size != outsize)
773 pos = ia->read.in.offset - io->offset + outsize;793 pos = ia->read.in.offset - io->offset + outsize;
774 }794 }
775 795
796 fuse_release_user_pages(&ia->ap, err ?: nres, io->should_dirty);
797
776 fuse_aio_complete(io, err, pos);798 fuse_aio_complete(io, err, pos);
777 fuse_io_free(ia);799 fuse_io_free(ia);
778}800}
@@ -843,33 +865,33 @@ static void fuse_short_read(struct inode *inode, u64 attr_ver, size_t num_read, · 64 unmodified lines
843 * reached the client fs yet. So the hole is not present there.865 * reached the client fs yet. So the hole is not present there.
844 */866 */
845 if (!fc->writeback_cache) {867 if (!fc->writeback_cache) {
846 loff_t pos = page_offset(ap->pages[0]) + num_read;868 loff_t pos = folio_pos(ap->folios[0]) + num_read;
847 fuse_read_update_size(inode, pos, attr_ver);869 fuse_read_update_size(inode, pos, attr_ver);
848 }870 }
849}871}
850 872
851static int fuse_do_readpage(struct file *file, struct page *page)873static int fuse_do_readfolio(struct file *file, struct folio *folio)
852{874{
853 struct inode *inode = page->mapping->host;875 struct inode *inode = folio->mapping->host;
854 struct fuse_mount *fm = get_fuse_mount(inode);876 struct fuse_mount *fm = get_fuse_mount(inode);
855 loff_t pos = page_offset(page);877 loff_t pos = folio_pos(folio);
856 struct fuse_page_desc desc = { .length = PAGE_SIZE };878 struct fuse_folio_desc desc = { .length = PAGE_SIZE };
857 struct fuse_io_args ia = {879 struct fuse_io_args ia = {
858 .ap.args.page_zeroing = true,880 .ap.args.page_zeroing = true,
859 .ap.args.out_pages = true,881 .ap.args.out_pages = true,
860 .ap.num_pages = 1,882 .ap.num_folios = 1,
861 .ap.pages = &page,883 .ap.folios = &folio,
862 .ap.descs = &desc,884 .ap.descs = &desc,
863 };885 };
864 ssize_t res;886 ssize_t res;
865 u64 attr_ver;887 u64 attr_ver;
866 888
867 /*889 /*
868 * Page writeback can extend beyond the lifetime of the890 * With the temporary pages that are used to complete writeback, we can
869 * page-cache page, so make sure we read a properly synced891 * have writeback that extends beyond the lifetime of the folio. So
870 * page.892 * make sure we read a properly synced folio.
871 */893 */
872 fuse_wait_on_page_writeback(inode, page->index);894 fuse_wait_on_folio_writeback(inode, folio);
873 895
874 attr_ver = fuse_get_attr_version(fm->fc);896 attr_ver = fuse_get_attr_version(fm->fc);
875 897
@@ -887,25 +909,24 @@ static int fuse_do_readpage(struct file *file, struct page *page) · 11 unmodified lines
887 if (res < desc.length)909 if (res < desc.length)
888 fuse_short_read(inode, attr_ver, res, &ia.ap);910 fuse_short_read(inode, attr_ver, res, &ia.ap);
889 911
890 SetPageUptodate(page);912 folio_mark_uptodate(folio);
891 913
892 return 0;914 return 0;
893}915}
894 916
895static int fuse_read_folio(struct file *file, struct folio *folio)917static int fuse_read_folio(struct file *file, struct folio *folio)
896{918{
897 struct page *page = &folio->page;919 struct inode *inode = folio->mapping->host;
898 struct inode *inode = page->mapping->host;
899 int err;920 int err;
900 921
901 err = -EIO;922 err = -EIO;
902 if (fuse_is_bad(inode))923 if (fuse_is_bad(inode))
903 goto out;924 goto out;
904 925
905 err = fuse_do_readpage(file, page);926 err = fuse_do_readfolio(file, folio);
906 fuse_invalidate_atime(inode);927 fuse_invalidate_atime(inode);
907 out:928 out:
908 unlock_page(page);929 folio_unlock(folio);
909 return err;930 return err;
910}931}
911 932
@@ -919,8 +940,8 @@ static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args, · 7 unmodified lines
919 size_t num_read = args->out_args[0].size;940 size_t num_read = args->out_args[0].size;
920 struct address_space *mapping = NULL;941 struct address_space *mapping = NULL;
921 942
922 for (i = 0; mapping == NULL && i < ap->num_pages; i++)943 for (i = 0; mapping == NULL && i < ap->num_folios; i++)
923 mapping = ap->pages[i]->mapping;944 mapping = ap->folios[i]->mapping;
924 945
925 if (mapping) {946 if (mapping) {
926 struct inode *inode = mapping->host;947 struct inode *inode = mapping->host;
@@ -934,12 +955,8 @@ static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args, · 7 unmodified lines
934 fuse_invalidate_atime(inode);955 fuse_invalidate_atime(inode);
935 }956 }
936 957
937 for (i = 0; i < ap->num_pages; i++) {958 for (i = 0; i < ap->num_folios; i++)
938 struct folio *folio = page_folio(ap->pages[i]);959 folio_end_read(ap->folios[i], !err);
939
940 folio_end_read(folio, !err);
941 folio_put(folio);
942 }
943 if (ia->ff)960 if (ia->ff)
944 fuse_file_put(ia->ff, false);961 fuse_file_put(ia->ff, false);
945 962
@@ -951,8 +968,9 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file) · 5 unmodified lines
951 struct fuse_file *ff = file->private_data;968 struct fuse_file *ff = file->private_data;
952 struct fuse_mount *fm = ff->fm;969 struct fuse_mount *fm = ff->fm;
953 struct fuse_args_pages *ap = &ia->ap;970 struct fuse_args_pages *ap = &ia->ap;
954 loff_t pos = page_offset(ap->pages[0]);971 loff_t pos = folio_pos(ap->folios[0]);
955 size_t count = ap->num_pages << PAGE_SHIFT;972 /* Currently, all folios in FUSE are one page */
973 size_t count = ap->num_folios << PAGE_SHIFT;
956 ssize_t res;974 ssize_t res;
957 int err;975 int err;
958 976
@@ -963,7 +981,7 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file) · 4 unmodified lines
963 /* Don't overflow end offset */981 /* Don't overflow end offset */
964 if (pos + (count - 1) == LLONG_MAX) {982 if (pos + (count - 1) == LLONG_MAX) {
965 count--;983 count--;
966 ap->descs[ap->num_pages - 1].length--;984 ap->descs[ap->num_folios - 1].length--;
967 }985 }
968 WARN_ON((loff_t) (pos + count) < 0);986 WARN_ON((loff_t) (pos + count) < 0);
969 987
@@ -985,18 +1003,36 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file) · 15 unmodified lines
985static void fuse_readahead(struct readahead_control *rac)1003static void fuse_readahead(struct readahead_control *rac)
986{1004{
987 struct inode *inode = rac->mapping->host;1005 struct inode *inode = rac->mapping->host;
1006 struct fuse_inode *fi = get_fuse_inode(inode);
988 struct fuse_conn *fc = get_fuse_conn(inode);1007 struct fuse_conn *fc = get_fuse_conn(inode);
989 unsigned int i, max_pages, nr_pages = 0;1008 unsigned int max_pages, nr_pages;
1009 pgoff_t first = readahead_index(rac);
1010 pgoff_t last = first + readahead_count(rac) - 1;
990 1011
991 if (fuse_is_bad(inode))1012 if (fuse_is_bad(inode))
992 return;1013 return;
993 1014
1015 wait_event(fi->page_waitq, !fuse_range_is_writeback(inode, first, last));
1016
994 max_pages = min_t(unsigned int, fc->max_pages,1017 max_pages = min_t(unsigned int, fc->max_pages,
995 fc->max_read / PAGE_SIZE);1018 fc->max_read / PAGE_SIZE);
996 1019
997 for (;;) {1020 /*
1021 * This is only accurate the first time through, since readahead_folio()
1022 * doesn't update readahead_count() from the previous folio until the
1023 * next call. Grab nr_pages here so we know how many pages we're going
1024 * to have to process. This means that we will exit here with
1025 * readahead_count() == folio_nr_pages(last_folio), but we will have
1026 * consumed all of the folios, and read_pages() will call
1027 * readahead_folio() again which will clean up the rac.
1028 */
1029 nr_pages = readahead_count(rac);
1030
1031 while (nr_pages) {
998 struct fuse_io_args *ia;1032 struct fuse_io_args *ia;
999 struct fuse_args_pages *ap;1033 struct fuse_args_pages *ap;
1034 struct folio *folio;
1035 unsigned cur_pages = min(max_pages, nr_pages);
1000 1036
1001 if (fc->num_background >= fc->congestion_threshold &&1037 if (fc->num_background >= fc->congestion_threshold &&
1002 rac->ra->async_size >= readahead_count(rac))1038 rac->ra->async_size >= readahead_count(rac))
@@ -1006,23 +1042,19 @@ static void fuse_readahead(struct readahead_control *rac) · 3 unmodified lines
1006 */1042 */
1007 break;1043 break;
1008 1044
1009 nr_pages = readahead_count(rac) - nr_pages;1045 ia = fuse_io_alloc(NULL, cur_pages);
1010 if (nr_pages > max_pages)
1011 nr_pages = max_pages;
1012 if (nr_pages == 0)
1013 break;
1014 ia = fuse_io_alloc(NULL, nr_pages);
1015 if (!ia)1046 if (!ia)
1016 return;1047 return;
1017 ap = &ia->ap;1048 ap = &ia->ap;
1018 nr_pages = __readahead_batch(rac, ap->pages, nr_pages);1049
1019 for (i = 0; i < nr_pages; i++) {1050 while (ap->num_folios < cur_pages) {
1020 fuse_wait_on_page_writeback(inode,1051 folio = readahead_folio(rac);
1021 readahead_index(rac) + i);1052 ap->folios[ap->num_folios] = folio;
1022 ap->descs[i].length = PAGE_SIZE;1053 ap->descs[ap->num_folios].length = folio_size(folio);
1054 ap->num_folios++;
1023 }1055 }
1024 ap->num_pages = nr_pages;
1025 fuse_send_readpages(ia, rac->file);1056 fuse_send_readpages(ia, rac->file);
1057 nr_pages -= cur_pages;
1026 }1058 }
1027}1059}
1028 1060
@@ -1139,8 +1171,8 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, · 110 unmodified lines
1139 bool short_write;1171 bool short_write;
1140 int err;1172 int err;
1141 1173
1142 for (i = 0; i < ap->num_pages; i++)1174 for (i = 0; i < ap->num_folios; i++)
1143 fuse_wait_on_page_writeback(inode, ap->pages[i]->index);1175 fuse_wait_on_folio_writeback(inode, ap->folios[i]);
1144 1176
1145 fuse_write_args_fill(ia, ff, pos, count);1177 fuse_write_args_fill(ia, ff, pos, count);
1146 ia->write.in.flags = fuse_write_flags(iocb);1178 ia->write.in.flags = fuse_write_flags(iocb);
@@ -1154,24 +1186,24 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, · 7 unmodified lines
1154 short_write = ia->write.out.size < count;1186 short_write = ia->write.out.size < count;
1155 offset = ap->descs[0].offset;1187 offset = ap->descs[0].offset;
1156 count = ia->write.out.size;1188 count = ia->write.out.size;
1157 for (i = 0; i < ap->num_pages; i++) {1189 for (i = 0; i < ap->num_folios; i++) {
1158 struct page *page = ap->pages[i];1190 struct folio *folio = ap->folios[i];
1159 1191
1160 if (err) {1192 if (err) {
1161 ClearPageUptodate(page);1193 folio_clear_uptodate(folio);
1162 } else {1194 } else {
1163 if (count >= PAGE_SIZE - offset)1195 if (count >= folio_size(folio) - offset)
1164 count -= PAGE_SIZE - offset;1196 count -= folio_size(folio) - offset;
1165 else {1197 else {
1166 if (short_write)1198 if (short_write)
1167 ClearPageUptodate(page);1199 folio_clear_uptodate(folio);
1168 count = 0;1200 count = 0;
1169 }1201 }
1170 offset = 0;1202 offset = 0;
1171 }1203 }
1172 if (ia->write.page_locked && (i == ap->num_pages - 1))1204 if (ia->write.folio_locked && (i == ap->num_folios - 1))
1173 unlock_page(page);1205 folio_unlock(folio);
1174 put_page(page);1206 folio_put(folio);
1175 }1207 }
1176 1208
1177 return err;1209 return err;
@@ -1185,6 +1217,7 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, · 7 unmodified lines
1185 struct fuse_args_pages *ap = &ia->ap;1217 struct fuse_args_pages *ap = &ia->ap;
1186 struct fuse_conn *fc = get_fuse_conn(mapping->host);1218 struct fuse_conn *fc = get_fuse_conn(mapping->host);
1187 unsigned offset = pos & (PAGE_SIZE - 1);1219 unsigned offset = pos & (PAGE_SIZE - 1);
1220 unsigned int nr_pages = 0;
1188 size_t count = 0;1221 size_t count = 0;
1189 int err;1222 int err;
1190 1223
@@ -1193,7 +1226,7 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, · 2 unmodified lines
1193 1226
1194 do {1227 do {
1195 size_t tmp;1228 size_t tmp;
1196 struct page *page;1229 struct folio *folio;
1197 pgoff_t index = pos >> PAGE_SHIFT;1230 pgoff_t index = pos >> PAGE_SHIFT;
1198 size_t bytes = min_t(size_t, PAGE_SIZE - offset,1231 size_t bytes = min_t(size_t, PAGE_SIZE - offset,
1199 iov_iter_count(ii));1232 iov_iter_count(ii));
@@ -1205,27 +1238,30 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, · 5 unmodified lines
1205 if (fault_in_iov_iter_readable(ii, bytes))1238 if (fault_in_iov_iter_readable(ii, bytes))
1206 break;1239 break;
1207 1240
1208 err = -ENOMEM;1241 folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN,
1209 page = grab_cache_page_write_begin(mapping, index);1242 mapping_gfp_mask(mapping));
1210 if (!page)1243 if (IS_ERR(folio)) {
1244 err = PTR_ERR(folio);
1211 break;1245 break;
1246 }
1212 1247
1213 if (mapping_writably_mapped(mapping))1248 if (mapping_writably_mapped(mapping))
1214 flush_dcache_page(page);1249 flush_dcache_folio(folio);
1215 1250
1216 tmp = copy_page_from_iter_atomic(page, offset, bytes, ii);1251 tmp = copy_folio_from_iter_atomic(folio, offset, bytes, ii);
1217 flush_dcache_page(page);1252 flush_dcache_folio(folio);
1218 1253
1219 if (!tmp) {1254 if (!tmp) {
1220 unlock_page(page);1255 folio_unlock(folio);
1221 put_page(page);1256 folio_put(folio);
1222 goto again;1257 goto again;
1223 }1258 }
1224 1259
1225 err = 0;1260 err = 0;
1226 ap->pages[ap->num_pages] = page;1261 ap->folios[ap->num_folios] = folio;
1227 ap->descs[ap->num_pages].length = tmp;1262 ap->descs[ap->num_folios].length = tmp;
1228 ap->num_pages++;1263 ap->num_folios++;
1264 nr_pages++;
1229 1265
1230 count += tmp;1266 count += tmp;
1231 pos += tmp;1267 pos += tmp;
@@ -1235,18 +1271,18 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, · 3 unmodified lines
1235 1271
1236 /* If we copied full page, mark it uptodate */1272 /* If we copied full page, mark it uptodate */
1237 if (tmp == PAGE_SIZE)1273 if (tmp == PAGE_SIZE)
1238 SetPageUptodate(page);1274 folio_mark_uptodate(folio);
1239 1275
1240 if (PageUptodate(page)) {1276 if (folio_test_uptodate(folio)) {
1241 unlock_page(page);1277 folio_unlock(folio);
1242 } else {1278 } else {
1243 ia->write.page_locked = true;1279 ia->write.folio_locked = true;
1244 break;1280 break;
1245 }1281 }
1246 if (!fc->big_writes)1282 if (!fc->big_writes)
1247 break;1283 break;
1248 } while (iov_iter_count(ii) && count < fc->max_write &&1284 } while (iov_iter_count(ii) && count < fc->max_write &&
1249 ap->num_pages < max_pages && offset == 0);1285 nr_pages < max_pages && offset == 0);
1250 1286
1251 return count > 0 ? count : err;1287 return count > 0 ? count : err;
1252}1288}
@@ -1280,8 +1316,8 @@ static ssize_t fuse_perform_write(struct kiocb *iocb, struct iov_iter *ii) · 27 unmodified lines
1280 unsigned int nr_pages = fuse_wr_pages(pos, iov_iter_count(ii),1316 unsigned int nr_pages = fuse_wr_pages(pos, iov_iter_count(ii),
1281 fc->max_pages);1317 fc->max_pages);
1282 1318
1283 ap->pages = fuse_pages_alloc(nr_pages, GFP_KERNEL, &ap->descs);1319 ap->folios = fuse_folios_alloc(nr_pages, GFP_KERNEL, &ap->descs);
1284 if (!ap->pages) {1320 if (!ap->folios) {
1285 err = -ENOMEM;1321 err = -ENOMEM;
1286 break;1322 break;
1287 }1323 }
@@ -1303,7 +1339,7 @@ static ssize_t fuse_perform_write(struct kiocb *iocb, struct iov_iter *ii) · 15 unmodified lines
1303 err = -EIO;1339 err = -EIO;
1304 }1340 }
1305 }1341 }
1306 kfree(ap->pages);1342 kfree(ap->folios);
1307 } while (!err && iov_iter_count(ii));1343 } while (!err && iov_iter_count(ii));
1308 1344
1309 fuse_write_update_attr(inode, pos, res);1345 fuse_write_update_attr(inode, pos, res);
@@ -1430,11 +1466,7 @@ writethrough: · 120 unmodified lines
1430 1466
1431 task_io_account_write(count);1467 task_io_account_write(count);
1432 1468
1433 err = file_remove_privs(file);1469 err = kiocb_modified(iocb);
1434 if (err)
1435 goto out;
1436
1437 err = file_update_time(file);
1438 if (err)1470 if (err)
1439 goto out;1471 goto out;
1440 1472
@@ -1468,35 +1500,57 @@ static inline size_t fuse_get_frag_size(const struct iov_iter *ii, · 27 unmodified lines
1468 1500
1469static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii,1501static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii,
1470 size_t *nbytesp, int write,1502 size_t *nbytesp, int write,
1471 unsigned int max_pages)1503 unsigned int max_pages,
1504 bool use_pages_for_kvec_io)
1472{1505{
1506 bool flush_or_invalidate = false;
1507 unsigned int nr_pages = 0;
1473 size_t nbytes = 0; /* # bytes already packed in req */1508 size_t nbytes = 0; /* # bytes already packed in req */
1474 ssize_t ret = 0;1509 ssize_t ret = 0;
1475 1510
1476 /* Special case for kernel I/O: can copy directly into the buffer */1511 /* Special case for kernel I/O: can copy directly into the buffer.
1512 * However if the implementation of fuse_conn requires pages instead of
1513 * pointer (e.g., virtio-fs), use iov_iter_extract_pages() instead.
1514 */
1477 if (iov_iter_is_kvec(ii)) {1515 if (iov_iter_is_kvec(ii)) {
1478 unsigned long user_addr = fuse_get_user_addr(ii);1516 void *user_addr = (void *)fuse_get_user_addr(ii);
1479 size_t frag_size = fuse_get_frag_size(ii, *nbytesp);
1480 1517
1481 if (write)1518 if (!use_pages_for_kvec_io) {
1482 ap->args.in_args[1].value = (void *) user_addr;1519 size_t frag_size = fuse_get_frag_size(ii, *nbytesp);
1483 else
1484 ap->args.out_args[0].value = (void *) user_addr;
1485 1520
1486 iov_iter_advance(ii, frag_size);1521 if (write)
1487 *nbytesp = frag_size;1522 ap->args.in_args[1].value = user_addr;
1488 return 0;1523 else
1524 ap->args.out_args[0].value = user_addr;
1525
1526 iov_iter_advance(ii, frag_size);
1527 *nbytesp = frag_size;
1528 return 0;
1529 }
1530
1531 if (is_vmalloc_addr(user_addr)) {
1532 ap->args.vmap_base = user_addr;
1533 flush_or_invalidate = true;
1534 }
1489 }1535 }
1490 1536
1491 while (nbytes < *nbytesp && ap->num_pages < max_pages) {1537 /*
1492 unsigned npages;1538 * Until there is support for iov_iter_extract_folios(), we have to
1539 * manually extract pages using iov_iter_extract_pages() and then
1540 * copy that to a folios array.
1541 */
1542 struct page **pages = kzalloc(max_pages * sizeof(struct page *),
1543 GFP_KERNEL);
1544 if (!pages)
1545 return -ENOMEM;
1546
1547 while (nbytes < *nbytesp && nr_pages < max_pages) {
1548 unsigned nfolios, i;
1493 size_t start;1549 size_t start;
1494 struct page **pt_pages;
1495 1550
1496 pt_pages = &ap->pages[ap->num_pages];1551 ret = iov_iter_extract_pages(ii, &pages,
1497 ret = iov_iter_extract_pages(ii, &pt_pages,
1498 *nbytesp - nbytes,1552 *nbytesp - nbytes,
1499 max_pages - ap->num_pages,1553 max_pages - nr_pages,
1500 0, &start);1554 0, &start);
1501 if (ret < 0)1555 if (ret < 0)
1502 break;1556 break;
@@ -1504,16 +1558,25 @@ static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii, · 1 unmodified line
1504 nbytes += ret;1558 nbytes += ret;
1505 1559
1506 ret += start;1560 ret += start;
1507 npages = DIV_ROUND_UP(ret, PAGE_SIZE);1561 /* Currently, all folios in FUSE are one page */
1562 nfolios = DIV_ROUND_UP(ret, PAGE_SIZE);
1508 1563
1509 ap->descs[ap->num_pages].offset = start;1564 ap->descs[ap->num_folios].offset = start;
1510 fuse_page_descs_length_init(ap->descs, ap->num_pages, npages);1565 fuse_folio_descs_length_init(ap->descs, ap->num_folios, nfolios);
1566 for (i = 0; i < nfolios; i++)
1567 ap->folios[i + ap->num_folios] = page_folio(pages[i]);
1511 1568
1512 ap->num_pages += npages;1569 ap->num_folios += nfolios;
1513 ap->descs[ap->num_pages - 1].length -=1570 ap->descs[ap->num_folios - 1].length -=
1514 (PAGE_SIZE - ret) & (PAGE_SIZE - 1);1571 (PAGE_SIZE - ret) & (PAGE_SIZE - 1);
1572 nr_pages += nfolios;
1515 }1573 }
1574 kfree(pages);
1575
1576 if (write && flush_or_invalidate)
1577 flush_kernel_vmap_range(ap->args.vmap_base, nbytes);
1516 1578
1579 ap->args.invalidate_vmap = !write && flush_or_invalidate;
1517 ap->args.is_pinned = iov_iter_extract_will_pin(ii);1580 ap->args.is_pinned = iov_iter_extract_will_pin(ii);
1518 ap->args.user_pages = true;1581 ap->args.user_pages = true;
1519 if (write)1582 if (write)
@@ -1582,7 +1645,7 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, · 62 unmodified lines
1582 size_t nbytes = min(count, nmax);1645 size_t nbytes = min(count, nmax);
1583 1646
1584 err = fuse_get_user_pages(&ia->ap, iter, &nbytes, write,1647 err = fuse_get_user_pages(&ia->ap, iter, &nbytes, write,
1585 max_pages);1648 max_pages, fc->use_pages_for_kvec_io);
1586 if (err && !nbytes)1649 if (err && !nbytes)
1587 break;1650 break;
1588 1651
@@ -1596,7 +1659,7 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, · 7 unmodified lines
1596 }1659 }
1597 1660
1598 if (!io->async || nres < 0) {1661 if (!io->async || nres < 0) {
1599 fuse_release_user_pages(&ia->ap, io->should_dirty);1662 fuse_release_user_pages(&ia->ap, nres, io->should_dirty);
1600 fuse_io_free(ia);1663 fuse_io_free(ia);
1601 }1664 }
1602 ia = NULL;1665 ia = NULL;
@@ -1650,7 +1713,7 @@ static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to) · 47 unmodified lines
1650{1713{
1651 ssize_t res;1714 ssize_t res;
1652 1715
1653 if (!is_sync_kiocb(iocb) && iocb->ki_flags & IOCB_DIRECT) {1716 if (!is_sync_kiocb(iocb)) {
1654 res = fuse_direct_IO(iocb, to);1717 res = fuse_direct_IO(iocb, to);
1655 } else {1718 } else {
1656 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);1719 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
@@ -1664,7 +1727,6 @@ static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to) · 7 unmodified lines
1664static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from)1727static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from)
1665{1728{
1666 struct inode *inode = file_inode(iocb->ki_filp);1729 struct inode *inode = file_inode(iocb->ki_filp);
1667 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
1668 ssize_t res;1730 ssize_t res;
1669 bool exclusive;1731 bool exclusive;
1670 1732
@@ -1672,9 +1734,11 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) · 1 unmodified line
1672 res = generic_write_checks(iocb, from);1734 res = generic_write_checks(iocb, from);
1673 if (res > 0) {1735 if (res > 0) {
1674 task_io_account_write(res);1736 task_io_account_write(res);
1675 if (!is_sync_kiocb(iocb) && iocb->ki_flags & IOCB_DIRECT) {1737 if (!is_sync_kiocb(iocb)) {
1676 res = fuse_direct_IO(iocb, from);1738 res = fuse_direct_IO(iocb, from);
1677 } else {1739 } else {
1740 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
1741
1678 res = fuse_direct_io(&io, from, &iocb->ki_pos,1742 res = fuse_direct_io(&io, from, &iocb->ki_pos,
1679 FUSE_DIO_WRITE);1743 FUSE_DIO_WRITE);
1680 fuse_write_update_attr(inode, iocb->ki_pos, res);1744 fuse_write_update_attr(inode, iocb->ki_pos, res);
@@ -1760,21 +1824,21 @@ static void fuse_writepage_free(struct fuse_writepage_args *wpa) · 79 unmodified lines
1760 if (wpa->bucket)1824 if (wpa->bucket)
1761 fuse_sync_bucket_dec(wpa->bucket);1825 fuse_sync_bucket_dec(wpa->bucket);
1762 1826
1763 for (i = 0; i < ap->num_pages; i++)1827 for (i = 0; i < ap->num_folios; i++)
1764 __free_page(ap->pages[i]);1828 folio_put(ap->folios[i]);
1765 1829
1766 fuse_file_put(wpa->ia.ff, false);1830 fuse_file_put(wpa->ia.ff, false);
1767 1831
1768 kfree(ap->pages);1832 kfree(ap->folios);
1769 kfree(wpa);1833 kfree(wpa);
1770}1834}
1771 1835
1772static void fuse_writepage_finish_stat(struct inode *inode, struct page *page)1836static void fuse_writepage_finish_stat(struct inode *inode, struct folio *folio)
1773{1837{
1774 struct backing_dev_info *bdi = inode_to_bdi(inode);1838 struct backing_dev_info *bdi = inode_to_bdi(inode);
1775 1839
1776 dec_wb_stat(&bdi->wb, WB_WRITEBACK);1840 dec_wb_stat(&bdi->wb, WB_WRITEBACK);
1777 dec_node_page_state(page, NR_WRITEBACK_TEMP);1841 node_stat_sub_folio(folio, NR_WRITEBACK_TEMP);
1778 wb_writeout_inc(&bdi->wb);1842 wb_writeout_inc(&bdi->wb);
1779}1843}
1780 1844
@@ -1785,8 +1849,8 @@ static void fuse_writepage_finish(struct fuse_writepage_args *wpa) · 4 unmodified lines
1785 struct fuse_inode *fi = get_fuse_inode(inode);1849 struct fuse_inode *fi = get_fuse_inode(inode);
1786 int i;1850 int i;
1787 1851
1788 for (i = 0; i < ap->num_pages; i++)1852 for (i = 0; i < ap->num_folios; i++)
1789 fuse_writepage_finish_stat(inode, ap->pages[i]);1853 fuse_writepage_finish_stat(inode, ap->folios[i]);
1790 1854
1791 wake_up(&fi->page_waitq);1855 wake_up(&fi->page_waitq);
1792}1856}
@@ -1801,7 +1865,8 @@ __acquires(fi->lock) · 8 unmodified lines
1801 struct fuse_inode *fi = get_fuse_inode(wpa->inode);1865 struct fuse_inode *fi = get_fuse_inode(wpa->inode);
1802 struct fuse_write_in *inarg = &wpa->ia.write.in;1866 struct fuse_write_in *inarg = &wpa->ia.write.in;
1803 struct fuse_args *args = &wpa->ia.ap.args;1867 struct fuse_args *args = &wpa->ia.ap.args;
1804 __u64 data_size = wpa->ia.ap.num_pages * PAGE_SIZE;1868 /* Currently, all folios in FUSE are one page */
1869 __u64 data_size = wpa->ia.ap.num_folios * PAGE_SIZE;
1805 int err;1870 int err;
1806 1871
1807 fi->writectr++;1872 fi->writectr++;
@@ -1841,7 +1906,8 @@ __acquires(fi->lock) · 33 unmodified lines
1841 for (aux = wpa->next; aux; aux = next) {1906 for (aux = wpa->next; aux; aux = next) {
1842 next = aux->next;1907 next = aux->next;
1843 aux->next = NULL;1908 aux->next = NULL;
1844 fuse_writepage_finish_stat(aux->inode, aux->ia.ap.pages[0]);1909 fuse_writepage_finish_stat(aux->inode,
1910 aux->ia.ap.folios[0]);
1845 fuse_writepage_free(aux);1911 fuse_writepage_free(aux);
1846 }1912 }
1847 1913
@@ -1876,11 +1942,11 @@ static struct fuse_writepage_args *fuse_insert_writeback(struct rb_root *root, · 28 unmodified lines
1876 struct fuse_writepage_args *wpa)1942 struct fuse_writepage_args *wpa)
1877{1943{
1878 pgoff_t idx_from = wpa->ia.write.in.offset >> PAGE_SHIFT;1944 pgoff_t idx_from = wpa->ia.write.in.offset >> PAGE_SHIFT;
1879 pgoff_t idx_to = idx_from + wpa->ia.ap.num_pages - 1;1945 pgoff_t idx_to = idx_from + wpa->ia.ap.num_folios - 1;
1880 struct rb_node **p = &root->rb_node;1946 struct rb_node **p = &root->rb_node;
1881 struct rb_node *parent = NULL;1947 struct rb_node *parent = NULL;
1882 1948
1883 WARN_ON(!wpa->ia.ap.num_pages);1949 WARN_ON(!wpa->ia.ap.num_folios);
1884 while (*p) {1950 while (*p) {
1885 struct fuse_writepage_args *curr;1951 struct fuse_writepage_args *curr;
1886 pgoff_t curr_index;1952 pgoff_t curr_index;
@@ -1891,7 +1957,7 @@ static struct fuse_writepage_args *fuse_insert_writeback(struct rb_root *root, · 4 unmodified lines
1891 WARN_ON(curr->inode != wpa->inode);1957 WARN_ON(curr->inode != wpa->inode);
1892 curr_index = curr->ia.write.in.offset >> PAGE_SHIFT;1958 curr_index = curr->ia.write.in.offset >> PAGE_SHIFT;
1893 1959
1894 if (idx_from >= curr_index + curr->ia.ap.num_pages)1960 if (idx_from >= curr_index + curr->ia.ap.num_folios)
1895 p = &(*p)->rb_right;1961 p = &(*p)->rb_right;
1896 else if (idx_to < curr_index)1962 else if (idx_to < curr_index)
1897 p = &(*p)->rb_left;1963 p = &(*p)->rb_left;
@@ -2023,9 +2089,9 @@ static struct fuse_writepage_args *fuse_writepage_args_alloc(void) · 125 unmodified lines
2023 wpa = kzalloc(sizeof(*wpa), GFP_NOFS);2089 wpa = kzalloc(sizeof(*wpa), GFP_NOFS);
2024 if (wpa) {2090 if (wpa) {
2025 ap = &wpa->ia.ap;2091 ap = &wpa->ia.ap;
2026 ap->num_pages = 0;2092 ap->num_folios = 0;
2027 ap->pages = fuse_pages_alloc(1, GFP_NOFS, &ap->descs);2093 ap->folios = fuse_folios_alloc(1, GFP_NOFS, &ap->descs);
2028 if (!ap->pages) {2094 if (!ap->folios) {
2029 kfree(wpa);2095 kfree(wpa);
2030 wpa = NULL;2096 wpa = NULL;
2031 }2097 }
@@ -2049,19 +2115,19 @@ static void fuse_writepage_add_to_bucket(struct fuse_conn *fc, · 17 unmodified lines
2049}2115}
2050 2116
2051static void fuse_writepage_args_page_fill(struct fuse_writepage_args *wpa, struct folio *folio,2117static void fuse_writepage_args_page_fill(struct fuse_writepage_args *wpa, struct folio *folio,
2052 struct folio *tmp_folio, uint32_t page_index)2118 struct folio *tmp_folio, uint32_t folio_index)
2053{2119{
2054 struct inode *inode = folio->mapping->host;2120 struct inode *inode = folio->mapping->host;
2055 struct fuse_args_pages *ap = &wpa->ia.ap;2121 struct fuse_args_pages *ap = &wpa->ia.ap;
2056 2122
2057 folio_copy(tmp_folio, folio);2123 folio_copy(tmp_folio, folio);
2058 2124
2059 ap->pages[page_index] = &tmp_folio->page;2125 ap->folios[folio_index] = tmp_folio;
2060 ap->descs[page_index].offset = 0;2126 ap->descs[folio_index].offset = 0;
2061 ap->descs[page_index].length = PAGE_SIZE;2127 ap->descs[folio_index].length = PAGE_SIZE;
2062 2128
2063 inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK);2129 inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK);
2064 inc_node_page_state(&tmp_folio->page, NR_WRITEBACK_TEMP);2130 node_stat_add_folio(tmp_folio, NR_WRITEBACK_TEMP);
2065}2131}
2066 2132
2067static struct fuse_writepage_args *fuse_writepage_args_setup(struct folio *folio,2133static struct fuse_writepage_args *fuse_writepage_args_setup(struct folio *folio,
@@ -2115,7 +2181,7 @@ static int fuse_writepage_locked(struct folio *folio) · 47 unmodified lines
2115 goto err_writepage_args;2181 goto err_writepage_args;
2116 2182
2117 ap = &wpa->ia.ap;2183 ap = &wpa->ia.ap;
2118 ap->num_pages = 1;2184 ap->num_folios = 1;
2119 2185
2120 folio_start_writeback(folio);2186 folio_start_writeback(folio);
2121 fuse_writepage_args_page_fill(wpa, folio, tmp_folio, 0);2187 fuse_writepage_args_page_fill(wpa, folio, tmp_folio, 0);
@@ -2143,32 +2209,32 @@ struct fuse_fill_wb_data { · 21 unmodified lines
2143 struct fuse_writepage_args *wpa;2209 struct fuse_writepage_args *wpa;
2144 struct fuse_file *ff;2210 struct fuse_file *ff;
2145 struct inode *inode;2211 struct inode *inode;
2146 struct page **orig_pages;2212 struct folio **orig_folios;
2147 unsigned int max_pages;2213 unsigned int max_folios;
2148};2214};
2149 2215
2150static bool fuse_pages_realloc(struct fuse_fill_wb_data *data)2216static bool fuse_pages_realloc(struct fuse_fill_wb_data *data)
2151{2217{
2152 struct fuse_args_pages *ap = &data->wpa->ia.ap;2218 struct fuse_args_pages *ap = &data->wpa->ia.ap;
2153 struct fuse_conn *fc = get_fuse_conn(data->inode);2219 struct fuse_conn *fc = get_fuse_conn(data->inode);
2154 struct page **pages;2220 struct folio **folios;
2155 struct fuse_page_desc *descs;2221 struct fuse_folio_desc *descs;
2156 unsigned int npages = min_t(unsigned int,2222 unsigned int nfolios = min_t(unsigned int,
2157 max_t(unsigned int, data->max_pages * 2,2223 max_t(unsigned int, data->max_folios * 2,
2158 FUSE_DEFAULT_MAX_PAGES_PER_REQ),2224 FUSE_DEFAULT_MAX_PAGES_PER_REQ),
2159 fc->max_pages);2225 fc->max_pages);
2160 WARN_ON(npages <= data->max_pages);2226 WARN_ON(nfolios <= data->max_folios);
2161 2227
2162 pages = fuse_pages_alloc(npages, GFP_NOFS, &descs);2228 folios = fuse_folios_alloc(nfolios, GFP_NOFS, &descs);
2163 if (!pages)2229 if (!folios)
2164 return false;2230 return false;
2165 2231
2166 memcpy(pages, ap->pages, sizeof(struct page *) * ap->num_pages);2232 memcpy(folios, ap->folios, sizeof(struct folio *) * ap->num_folios);
2167 memcpy(descs, ap->descs, sizeof(struct fuse_page_desc) * ap->num_pages);2233 memcpy(descs, ap->descs, sizeof(struct fuse_folio_desc) * ap->num_folios);
2168 kfree(ap->pages);2234 kfree(ap->folios);
2169 ap->pages = pages;2235 ap->folios = folios;
2170 ap->descs = descs;2236 ap->descs = descs;
2171 data->max_pages = npages;2237 data->max_folios = nfolios;
2172 2238
2173 return true;2239 return true;
2174}2240}
@@ -2178,7 +2244,7 @@ static void fuse_writepages_send(struct fuse_fill_wb_data *data) · 3 unmodified lines
2178 struct fuse_writepage_args *wpa = data->wpa;2244 struct fuse_writepage_args *wpa = data->wpa;
2179 struct inode *inode = data->inode;2245 struct inode *inode = data->inode;
2180 struct fuse_inode *fi = get_fuse_inode(inode);2246 struct fuse_inode *fi = get_fuse_inode(inode);
2181 int num_pages = wpa->ia.ap.num_pages;2247 int num_folios = wpa->ia.ap.num_folios;
2182 int i;2248 int i;
2183 2249
2184 spin_lock(&fi->lock);2250 spin_lock(&fi->lock);
@@ -2186,8 +2252,8 @@ static void fuse_writepages_send(struct fuse_fill_wb_data *data) · 1 unmodified line
2186 fuse_flush_writepages(inode);2252 fuse_flush_writepages(inode);
2187 spin_unlock(&fi->lock);2253 spin_unlock(&fi->lock);
2188 2254
2189 for (i = 0; i < num_pages; i++)2255 for (i = 0; i < num_folios; i++)
2190 end_page_writeback(data->orig_pages[i]);2256 folio_end_writeback(data->orig_folios[i]);
2191}2257}
2192 2258
2193/*2259/*
@@ -2198,15 +2264,15 @@ static void fuse_writepages_send(struct fuse_fill_wb_data *data) · 4 unmodified lines
2198 * swapping the new temp page with the old one.2264 * swapping the new temp page with the old one.
2199 */2265 */
2200static bool fuse_writepage_add(struct fuse_writepage_args *new_wpa,2266static bool fuse_writepage_add(struct fuse_writepage_args *new_wpa,
2201 struct page *page)2267 struct folio *folio)
2202{2268{
2203 struct fuse_inode *fi = get_fuse_inode(new_wpa->inode);2269 struct fuse_inode *fi = get_fuse_inode(new_wpa->inode);
2204 struct fuse_writepage_args *tmp;2270 struct fuse_writepage_args *tmp;
2205 struct fuse_writepage_args *old_wpa;2271 struct fuse_writepage_args *old_wpa;
2206 struct fuse_args_pages *new_ap = &new_wpa->ia.ap;2272 struct fuse_args_pages *new_ap = &new_wpa->ia.ap;
2207 2273
2208 WARN_ON(new_ap->num_pages != 0);2274 WARN_ON(new_ap->num_folios != 0);
2209 new_ap->num_pages = 1;2275 new_ap->num_folios = 1;
2210 2276
2211 spin_lock(&fi->lock);2277 spin_lock(&fi->lock);
2212 old_wpa = fuse_insert_writeback(&fi->writepages, new_wpa);2278 old_wpa = fuse_insert_writeback(&fi->writepages, new_wpa);
@@ -2220,9 +2286,9 @@ static bool fuse_writepage_add(struct fuse_writepage_args *new_wpa, · 7 unmodified lines
2220 2286
2221 WARN_ON(tmp->inode != new_wpa->inode);2287 WARN_ON(tmp->inode != new_wpa->inode);
2222 curr_index = tmp->ia.write.in.offset >> PAGE_SHIFT;2288 curr_index = tmp->ia.write.in.offset >> PAGE_SHIFT;
2223 if (curr_index == page->index) {2289 if (curr_index == folio->index) {
2224 WARN_ON(tmp->ia.ap.num_pages != 1);2290 WARN_ON(tmp->ia.ap.num_folios != 1);
2225 swap(tmp->ia.ap.pages[0], new_ap->pages[0]);2291 swap(tmp->ia.ap.folios[0], new_ap->folios[0]);
2226 break;2292 break;
2227 }2293 }
2228 }2294 }
@@ -2235,18 +2301,19 @@ static bool fuse_writepage_add(struct fuse_writepage_args *new_wpa, · 6 unmodified lines
2235 spin_unlock(&fi->lock);2301 spin_unlock(&fi->lock);
2236 2302
2237 if (tmp) {2303 if (tmp) {
2238 fuse_writepage_finish_stat(new_wpa->inode, new_ap->pages[0]);2304 fuse_writepage_finish_stat(new_wpa->inode,
2305 folio);
2239 fuse_writepage_free(new_wpa);2306 fuse_writepage_free(new_wpa);
2240 }2307 }
2241 2308
2242 return false;2309 return false;
2243}2310}
2244 2311
2245static bool fuse_writepage_need_send(struct fuse_conn *fc, struct page *page,2312static bool fuse_writepage_need_send(struct fuse_conn *fc, struct folio *folio,
2246 struct fuse_args_pages *ap,2313 struct fuse_args_pages *ap,
2247 struct fuse_fill_wb_data *data)2314 struct fuse_fill_wb_data *data)
2248{2315{
2249 WARN_ON(!ap->num_pages);2316 WARN_ON(!ap->num_folios);
2250 2317
2251 /*2318 /*
2252 * Being under writeback is unlikely but possible. For example direct2319 * Being under writeback is unlikely but possible. For example direct
@@ -2254,23 +2321,23 @@ static bool fuse_writepage_need_send(struct fuse_conn *fc, struct page *page, · 1 unmodified line
2254 * the pages are faulted with get_user_pages(), and then after the read2321 * the pages are faulted with get_user_pages(), and then after the read
2255 * completed.2322 * completed.
2256 */2323 */
2257 if (fuse_page_is_writeback(data->inode, page->index))2324 if (fuse_folio_is_writeback(data->inode, folio))
2258 return true;2325 return true;
2259 2326
2260 /* Reached max pages */2327 /* Reached max pages */
2261 if (ap->num_pages == fc->max_pages)2328 if (ap->num_folios == fc->max_pages)
2262 return true;2329 return true;
2263 2330
2264 /* Reached max write bytes */2331 /* Reached max write bytes */
2265 if ((ap->num_pages + 1) * PAGE_SIZE > fc->max_write)2332 if ((ap->num_folios + 1) * PAGE_SIZE > fc->max_write)
2266 return true;2333 return true;
2267 2334
2268 /* Discontinuity */2335 /* Discontinuity */
2269 if (data->orig_pages[ap->num_pages - 1]->index + 1 != page->index)2336 if (data->orig_folios[ap->num_folios - 1]->index + 1 != folio_index(folio))
2270 return true;2337 return true;
2271 2338
2272 /* Need to grow the pages array? If so, did the expansion fail? */2339 /* Need to grow the pages array? If so, did the expansion fail? */
2273 if (ap->num_pages == data->max_pages && !fuse_pages_realloc(data))2340 if (ap->num_folios == data->max_folios && !fuse_pages_realloc(data))
2274 return true;2341 return true;
2275 2342
2276 return false;2343 return false;
@@ -2295,7 +2362,7 @@ static int fuse_writepages_fill(struct folio *folio, · 18 unmodified lines
2295 goto out_unlock;2362 goto out_unlock;
2296 }2363 }
2297 2364
2298 if (wpa && fuse_writepage_need_send(fc, &folio->page, ap, data)) {2365 if (wpa && fuse_writepage_need_send(fc, folio, ap, data)) {
2299 fuse_writepages_send(data);2366 fuse_writepages_send(data);
2300 data->wpa = NULL;2367 data->wpa = NULL;
2301 }2368 }
@@ -2314,7 +2381,7 @@ static int fuse_writepages_fill(struct folio *folio, · 12 unmodified lines
2314 * This is ensured by holding the page lock in page_mkwrite() while2381 * This is ensured by holding the page lock in page_mkwrite() while
2315 * checking fuse_page_is_writeback(). We already hold the page lock2382 * checking fuse_page_is_writeback(). We already hold the page lock
2316 * since clear_page_dirty_for_io() and keep it held until we add the2383 * since clear_page_dirty_for_io() and keep it held until we add the
2317 * request to the fi->writepages list and increment ap->num_pages.2384 * request to the fi->writepages list and increment ap->num_folios.
2318 * After this fuse_page_is_writeback() will indicate that the page is2385 * After this fuse_page_is_writeback() will indicate that the page is
2319 * under writeback, so we can release the page lock.2386 * under writeback, so we can release the page lock.
2320 */2387 */
@@ -2326,13 +2393,13 @@ static int fuse_writepages_fill(struct folio *folio, · 5 unmodified lines
2326 goto out_unlock;2393 goto out_unlock;
2327 }2394 }
2328 fuse_file_get(wpa->ia.ff);2395 fuse_file_get(wpa->ia.ff);
2329 data->max_pages = 1;2396 data->max_folios = 1;
2330 ap = &wpa->ia.ap;2397 ap = &wpa->ia.ap;
2331 }2398 }
2332 folio_start_writeback(folio);2399 folio_start_writeback(folio);
2333 2400
2334 fuse_writepage_args_page_fill(wpa, folio, tmp_folio, ap->num_pages);2401 fuse_writepage_args_page_fill(wpa, folio, tmp_folio, ap->num_folios);
2335 data->orig_pages[ap->num_pages] = &folio->page;2402 data->orig_folios[ap->num_folios] = folio;
2336 2403
2337 err = 0;2404 err = 0;
2338 if (data->wpa) {2405 if (data->wpa) {
@@ -2341,9 +2408,9 @@ static int fuse_writepages_fill(struct folio *folio, · 2 unmodified lines
2341 * fuse_page_is_writeback().2408 * fuse_page_is_writeback().
2342 */2409 */
2343 spin_lock(&fi->lock);2410 spin_lock(&fi->lock);
2344 ap->num_pages++;2411 ap->num_folios++;
2345 spin_unlock(&fi->lock);2412 spin_unlock(&fi->lock);
2346 } else if (fuse_writepage_add(wpa, &folio->page)) {2413 } else if (fuse_writepage_add(wpa, folio)) {
2347 data->wpa = wpa;2414 data->wpa = wpa;
2348 } else {2415 } else {
2349 folio_end_writeback(folio);2416 folio_end_writeback(folio);
@@ -2375,21 +2442,21 @@ static int fuse_writepages(struct address_space *mapping, · 25 unmodified lines
2375 data.ff = NULL;2442 data.ff = NULL;
2376 2443
2377 err = -ENOMEM;2444 err = -ENOMEM;
2378 data.orig_pages = kcalloc(fc->max_pages,2445 data.orig_folios = kcalloc(fc->max_pages,
2379 sizeof(struct page *),2446 sizeof(struct folio *),
2380 GFP_NOFS);2447 GFP_NOFS);
2381 if (!data.orig_pages)2448 if (!data.orig_folios)
2382 goto out;2449 goto out;
2383 2450
2384 err = write_cache_pages(mapping, wbc, fuse_writepages_fill, &data);2451 err = write_cache_pages(mapping, wbc, fuse_writepages_fill, &data);
2385 if (data.wpa) {2452 if (data.wpa) {
2386 WARN_ON(!data.wpa->ia.ap.num_pages);2453 WARN_ON(!data.wpa->ia.ap.num_folios);
2387 fuse_writepages_send(&data);2454 fuse_writepages_send(&data);
2388 }2455 }
2389 if (data.ff)2456 if (data.ff)
2390 fuse_file_put(data.ff, false);2457 fuse_file_put(data.ff, false);
2391 2458
2392 kfree(data.orig_pages);2459 kfree(data.orig_folios);
2393out:2460out:
2394 return err;2461 return err;
2395}2462}
@@ -2429,7 +2496,7 @@ static int fuse_write_begin(struct file *file, struct address_space *mapping, · 33 unmodified lines
2429 folio_zero_segment(folio, 0, off);2496 folio_zero_segment(folio, 0, off);
2430 goto success;2497 goto success;
2431 }2498 }
2432 err = fuse_do_readpage(file, &folio->page);2499 err = fuse_do_readfolio(file, folio);
2433 if (err)2500 if (err)
2434 goto cleanup;2501 goto cleanup;
2435success:2502success:
@@ -2518,17 +2585,17 @@ static void fuse_vma_close(struct vm_area_struct *vma) · 82 unmodified lines
2518 */2585 */
2519static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf)2586static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf)
2520{2587{
2521 struct page *page = vmf->page;2588 struct folio *folio = page_folio(vmf->page);
2522 struct inode *inode = file_inode(vmf->vma->vm_file);2589 struct inode *inode = file_inode(vmf->vma->vm_file);
2523 2590
2524 file_update_time(vmf->vma->vm_file);2591 file_update_time(vmf->vma->vm_file);
2525 lock_page(page);2592 folio_lock(folio);
2526 if (page->mapping != inode->i_mapping) {2593 if (folio->mapping != inode->i_mapping) {
2527 unlock_page(page);2594 folio_unlock(folio);
2528 return VM_FAULT_NOPAGE;2595 return VM_FAULT_NOPAGE;
2529 }2596 }
2530 2597
2531 fuse_wait_on_page_writeback(inode, page->index);2598 fuse_wait_on_folio_writeback(inode, folio);
2532 return VM_FAULT_LOCKED;2599 return VM_FAULT_LOCKED;
2533}2600}
2534 2601
fs/fuse/fuse_i.h+46−22
@@ -35,9 +35,6 @@ · 34 unmodified lines
35/** Default max number of pages that can be used in a single read request */35/** Default max number of pages that can be used in a single read request */
36#define FUSE_DEFAULT_MAX_PAGES_PER_REQ 3236#define FUSE_DEFAULT_MAX_PAGES_PER_REQ 32
37 37
38/** Maximum of max_pages received in init_out */
39#define FUSE_MAX_MAX_PAGES 256
40
41/** Bias for fi->writectr, meaning new writepages must not be sent */38/** Bias for fi->writectr, meaning new writepages must not be sent */
42#define FUSE_NOWRITE INT_MIN39#define FUSE_NOWRITE INT_MIN
43 40
@@ -47,6 +44,9 @@ · 3 unmodified lines
47/** Number of dentries for each connection in the control filesystem */44/** Number of dentries for each connection in the control filesystem */
48#define FUSE_CTL_NUM_DENTRIES 545#define FUSE_CTL_NUM_DENTRIES 5
49 46
47/** Maximum of max_pages received in init_out */
48extern unsigned int fuse_max_pages_limit;
49
50/** List of active connections */50/** List of active connections */
51extern struct list_head fuse_conn_list;51extern struct list_head fuse_conn_list;
52 52
@@ -285,8 +285,8 @@ struct fuse_arg { · 232 unmodified lines
285 void *value;285 void *value;
286};286};
287 287
288/** FUSE page descriptor */288/** FUSE folio descriptor */
289struct fuse_page_desc {289struct fuse_folio_desc {
290 unsigned int length;290 unsigned int length;
291 unsigned int offset;291 unsigned int offset;
292};292};
@@ -309,16 +309,19 @@ struct fuse_args { · 16 unmodified lines
309 bool may_block:1;309 bool may_block:1;
310 bool is_ext:1;310 bool is_ext:1;
311 bool is_pinned:1;311 bool is_pinned:1;
312 bool invalidate_vmap:1;
312 struct fuse_in_arg in_args[3];313 struct fuse_in_arg in_args[3];
313 struct fuse_arg out_args[2];314 struct fuse_arg out_args[2];
314 void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error);315 void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error);
316 /* Used for kvec iter backed by vmalloc address */
317 void *vmap_base;
315};318};
316 319
317struct fuse_args_pages {320struct fuse_args_pages {
318 struct fuse_args args;321 struct fuse_args args;
319 struct page **pages;322 struct folio **folios;
320 struct fuse_page_desc *descs;323 struct fuse_folio_desc *descs;
321 unsigned int num_pages;324 unsigned int num_folios;
322};325};
323 326
324struct fuse_release_args {327struct fuse_release_args {
@@ -857,6 +860,9 @@ struct fuse_conn { · 532 unmodified lines
857 /** Passthrough support for read/write IO */860 /** Passthrough support for read/write IO */
858 unsigned int passthrough:1;861 unsigned int passthrough:1;
859 862
863 /* Use pages instead of pointer for kernel I/O */
864 unsigned int use_pages_for_kvec_io:1;
865
860 /** Maximum stack depth for passthrough backing files */866 /** Maximum stack depth for passthrough backing files */
861 int max_stack_depth;867 int max_stack_depth;
862 868
@@ -884,6 +890,9 @@ struct fuse_conn { · 21 unmodified lines
884 /** Version counter for attribute changes */890 /** Version counter for attribute changes */
885 atomic64_t attr_version;891 atomic64_t attr_version;
886 892
893 /** Version counter for evict inode */
894 atomic64_t evict_ctr;
895
887 /** Called on final put */896 /** Called on final put */
888 void (*release)(struct fuse_conn *);897 void (*release)(struct fuse_conn *);
889 898
@@ -978,6 +987,11 @@ static inline u64 fuse_get_attr_version(struct fuse_conn *fc) · 88 unmodified lines
978 return atomic64_read(&fc->attr_version);987 return atomic64_read(&fc->attr_version);
979}988}
980 989
990static inline u64 fuse_get_evict_ctr(struct fuse_conn *fc)
991{
992 return atomic64_read(&fc->evict_ctr);
993}
994
981static inline bool fuse_stale_inode(const struct inode *inode, int generation,995static inline bool fuse_stale_inode(const struct inode *inode, int generation,
982 struct fuse_attr *attr)996 struct fuse_attr *attr)
983{997{
@@ -995,25 +1009,25 @@ static inline bool fuse_is_bad(struct inode *inode) · 11 unmodified lines
995 return unlikely(test_bit(FUSE_I_BAD, &get_fuse_inode(inode)->state));1009 return unlikely(test_bit(FUSE_I_BAD, &get_fuse_inode(inode)->state));
996}1010}
997 1011
998static inline struct page **fuse_pages_alloc(unsigned int npages, gfp_t flags,1012static inline struct folio **fuse_folios_alloc(unsigned int nfolios, gfp_t flags,
999 struct fuse_page_desc **desc)1013 struct fuse_folio_desc **desc)
1000{1014{
1001 struct page **pages;1015 struct folio **folios;
1002 1016
1003 pages = kzalloc(npages * (sizeof(struct page *) +1017 folios = kzalloc(nfolios * (sizeof(struct folio *) +
1004 sizeof(struct fuse_page_desc)), flags);1018 sizeof(struct fuse_folio_desc)), flags);
1005 *desc = (void *) (pages + npages);1019 *desc = (void *) (folios + nfolios);
1006 1020
1007 return pages;1021 return folios;
1008}1022}
1009 1023
1010static inline void fuse_page_descs_length_init(struct fuse_page_desc *descs,1024static inline void fuse_folio_descs_length_init(struct fuse_folio_desc *descs,
1011 unsigned int index,1025 unsigned int index,
1012 unsigned int nr_pages)1026 unsigned int nr_folios)
1013{1027{
1014 int i;1028 int i;
1015 1029
1016 for (i = index; i < index + nr_pages; i++)1030 for (i = index; i < index + nr_folios; i++)
1017 descs[i].length = PAGE_SIZE - descs[i].offset;1031 descs[i].length = PAGE_SIZE - descs[i].offset;
1018}1032}
1019 1033
@@ -1037,7 +1051,8 @@ extern const struct dentry_operations fuse_root_dentry_operations; · 17 unmodified lines
1037 */1051 */
1038struct inode *fuse_iget(struct super_block *sb, u64 nodeid,1052struct inode *fuse_iget(struct super_block *sb, u64 nodeid,
1039 int generation, struct fuse_attr *attr,1053 int generation, struct fuse_attr *attr,
1040 u64 attr_valid, u64 attr_version);1054 u64 attr_valid, u64 attr_version,
1055 u64 evict_ctr);
1041 1056
1042int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name,1057int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name,
1043 struct fuse_entry_out *outarg, struct inode **inode);1058 struct fuse_entry_out *outarg, struct inode **inode);
@@ -1062,7 +1077,7 @@ struct fuse_io_args { · 18 unmodified lines
1062 struct {1077 struct {
1063 struct fuse_write_in in;1078 struct fuse_write_in in;
1064 struct fuse_write_out out;1079 struct fuse_write_out out;
1065 bool page_locked;1080 bool folio_locked;
1066 } write;1081 } write;
1067 };1082 };
1068 struct fuse_args_pages ap;1083 struct fuse_args_pages ap;
@@ -1127,7 +1142,8 @@ void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr, · 58 unmodified lines
1127 1142
1128void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr,1143void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr,
1129 struct fuse_statx *sx,1144 struct fuse_statx *sx,
1130 u64 attr_valid, u32 cache_mask);1145 u64 attr_valid, u32 cache_mask,
1146 u64 evict_ctr);
1131 1147
1132u32 fuse_get_cache_mask(struct inode *inode);1148u32 fuse_get_cache_mask(struct inode *inode);
1133 1149
@@ -1480,4 +1496,12 @@ ssize_t fuse_passthrough_splice_write(struct pipe_inode_info *pipe, · 346 unmodified lines
1480 size_t len, unsigned int flags);1496 size_t len, unsigned int flags);
1481ssize_t fuse_passthrough_mmap(struct file *file, struct vm_area_struct *vma);1497ssize_t fuse_passthrough_mmap(struct file *file, struct vm_area_struct *vma);
1482 1498
1499#ifdef CONFIG_SYSCTL
1500extern int fuse_sysctl_register(void);
1501extern void fuse_sysctl_unregister(void);
1502#else
1503#define fuse_sysctl_register() (0)
1504#define fuse_sysctl_unregister() do { } while (0)
1505#endif /* CONFIG_SYSCTL */
1506
1483#endif /* _FS_FUSE_I_H */1507#endif /* _FS_FUSE_I_H */
fs/fuse/inode.c+54−13
@@ -35,6 +35,8 @@ DEFINE_MUTEX(fuse_mutex); · 34 unmodified lines
35 35
36static int set_global_limit(const char *val, const struct kernel_param *kp);36static int set_global_limit(const char *val, const struct kernel_param *kp);
37 37
38unsigned int fuse_max_pages_limit = 256;
39
38unsigned max_user_bgreq;40unsigned max_user_bgreq;
39module_param_call(max_user_bgreq, set_global_limit, param_get_uint,41module_param_call(max_user_bgreq, set_global_limit, param_get_uint,
40 &max_user_bgreq, 0644);42 &max_user_bgreq, 0644);
@@ -173,6 +175,14 @@ static void fuse_evict_inode(struct inode *inode) · 132 unmodified lines
173 fuse_cleanup_submount_lookup(fc, fi->submount_lookup);175 fuse_cleanup_submount_lookup(fc, fi->submount_lookup);
174 fi->submount_lookup = NULL;176 fi->submount_lookup = NULL;
175 }177 }
178 /*
179 * Evict of non-deleted inode may race with outstanding
180 * LOOKUP/READDIRPLUS requests and result in inconsistency when
181 * the request finishes. Deal with that here by bumping a
182 * counter that can be compared to the starting value.
183 */
184 if (inode->i_nlink > 0)
185 atomic64_inc(&fc->evict_ctr);
176 }186 }
177 if (S_ISREG(inode->i_mode) && !fuse_is_bad(inode)) {187 if (S_ISREG(inode->i_mode) && !fuse_is_bad(inode)) {
178 WARN_ON(fi->iocachectr != 0);188 WARN_ON(fi->iocachectr != 0);
@@ -206,17 +216,30 @@ static ino_t fuse_squash_ino(u64 ino64) · 27 unmodified lines
206 216
207void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr,217void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr,
208 struct fuse_statx *sx,218 struct fuse_statx *sx,
209 u64 attr_valid, u32 cache_mask)219 u64 attr_valid, u32 cache_mask,
220 u64 evict_ctr)
210{221{
211 struct fuse_conn *fc = get_fuse_conn(inode);222 struct fuse_conn *fc = get_fuse_conn(inode);
212 struct fuse_inode *fi = get_fuse_inode(inode);223 struct fuse_inode *fi = get_fuse_inode(inode);
213 224
214 lockdep_assert_held(&fi->lock);225 lockdep_assert_held(&fi->lock);
215 226
227 /*
228 * Clear basic stats from invalid mask.
229 *
230 * Don't do this if this is coming from a fuse_iget() call and there
231 * might have been a racing evict which would've invalidated the result
232 * if the attr_version would've been preserved.
233 *
234 * !evict_ctr -> this is create
235 * fi->attr_version != 0 -> this is not a new inode
236 * evict_ctr == fuse_get_evict_ctr() -> no evicts while during request
237 */
238 if (!evict_ctr || fi->attr_version || evict_ctr == fuse_get_evict_ctr(fc))
239 set_mask_bits(&fi->inval_mask, STATX_BASIC_STATS, 0);
240
216 fi->attr_version = atomic64_inc_return(&fc->attr_version);241 fi->attr_version = atomic64_inc_return(&fc->attr_version);
217 fi->i_time = attr_valid;242 fi->i_time = attr_valid;
218 /* Clear basic stats from invalid mask */
219 set_mask_bits(&fi->inval_mask, STATX_BASIC_STATS, 0);
220 243
221 inode->i_ino = fuse_squash_ino(attr->ino);244 inode->i_ino = fuse_squash_ino(attr->ino);
222 inode->i_mode = (inode->i_mode & S_IFMT) | (attr->mode & 07777);245 inode->i_mode = (inode->i_mode & S_IFMT) | (attr->mode & 07777);
@@ -295,9 +318,9 @@ u32 fuse_get_cache_mask(struct inode *inode) · 72 unmodified lines
295 return STATX_MTIME | STATX_CTIME | STATX_SIZE;318 return STATX_MTIME | STATX_CTIME | STATX_SIZE;
296}319}
297 320
298void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr,321static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr,
299 struct fuse_statx *sx,322 struct fuse_statx *sx, u64 attr_valid,
300 u64 attr_valid, u64 attr_version)323 u64 attr_version, u64 evict_ctr)
301{324{
302 struct fuse_conn *fc = get_fuse_conn(inode);325 struct fuse_conn *fc = get_fuse_conn(inode);
303 struct fuse_inode *fi = get_fuse_inode(inode);326 struct fuse_inode *fi = get_fuse_inode(inode);
@@ -331,7 +354,8 @@ void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr, · 27 unmodified lines
331 }354 }
332 355
333 old_mtime = inode_get_mtime(inode);356 old_mtime = inode_get_mtime(inode);
334 fuse_change_attributes_common(inode, attr, sx, attr_valid, cache_mask);357 fuse_change_attributes_common(inode, attr, sx, attr_valid, cache_mask,
358 evict_ctr);
335 359
336 oldsize = inode->i_size;360 oldsize = inode->i_size;
337 /*361 /*
@@ -372,6 +396,13 @@ void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr, · 34 unmodified lines
372 fuse_dax_dontcache(inode, attr->flags);396 fuse_dax_dontcache(inode, attr->flags);
373}397}
374 398
399void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr,
400 struct fuse_statx *sx, u64 attr_valid,
401 u64 attr_version)
402{
403 fuse_change_attributes_i(inode, attr, sx, attr_valid, attr_version, 0);
404}
405
375static void fuse_init_submount_lookup(struct fuse_submount_lookup *sl,406static void fuse_init_submount_lookup(struct fuse_submount_lookup *sl,
376 u64 nodeid)407 u64 nodeid)
377{408{
@@ -426,7 +457,8 @@ static int fuse_inode_set(struct inode *inode, void *_nodeidp) · 48 unmodified lines
426 457
427struct inode *fuse_iget(struct super_block *sb, u64 nodeid,458struct inode *fuse_iget(struct super_block *sb, u64 nodeid,
428 int generation, struct fuse_attr *attr,459 int generation, struct fuse_attr *attr,
429 u64 attr_valid, u64 attr_version)460 u64 attr_valid, u64 attr_version,
461 u64 evict_ctr)
430{462{
431 struct inode *inode;463 struct inode *inode;
432 struct fuse_inode *fi;464 struct fuse_inode *fi;
@@ -487,8 +519,8 @@ retry: · 54 unmodified lines
487 fi->nlookup++;519 fi->nlookup++;
488 spin_unlock(&fi->lock);520 spin_unlock(&fi->lock);
489done:521done:
490 fuse_change_attributes(inode, attr, NULL, attr_valid, attr_version);522 fuse_change_attributes_i(inode, attr, NULL, attr_valid, attr_version,
491 523 evict_ctr);
492 return inode;524 return inode;
493}525}
494 526
@@ -940,11 +972,12 @@ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm, · 445 unmodified lines
940 fc->initialized = 0;972 fc->initialized = 0;
941 fc->connected = 1;973 fc->connected = 1;
942 atomic64_set(&fc->attr_version, 1);974 atomic64_set(&fc->attr_version, 1);
975 atomic64_set(&fc->evict_ctr, 1);
943 get_random_bytes(&fc->scramble_key, sizeof(fc->scramble_key));976 get_random_bytes(&fc->scramble_key, sizeof(fc->scramble_key));
944 fc->pid_ns = get_pid_ns(task_active_pid_ns(current));977 fc->pid_ns = get_pid_ns(task_active_pid_ns(current));
945 fc->user_ns = get_user_ns(user_ns);978 fc->user_ns = get_user_ns(user_ns);
946 fc->max_pages = FUSE_DEFAULT_MAX_PAGES_PER_REQ;979 fc->max_pages = FUSE_DEFAULT_MAX_PAGES_PER_REQ;
947 fc->max_pages_limit = FUSE_MAX_MAX_PAGES;980 fc->max_pages_limit = fuse_max_pages_limit;
948 981
949 if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH))982 if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH))
950 fuse_backing_files_init(fc);983 fuse_backing_files_init(fc);
@@ -1001,7 +1034,7 @@ static struct inode *fuse_get_root_inode(struct super_block *sb, unsigned mode) · 50 unmodified lines
1001 attr.mode = mode;1034 attr.mode = mode;
1002 attr.ino = FUSE_ROOT_ID;1035 attr.ino = FUSE_ROOT_ID;
1003 attr.nlink = 1;1036 attr.nlink = 1;
1004 return fuse_iget(sb, FUSE_ROOT_ID, 0, &attr, 0, 0);1037 return fuse_iget(sb, FUSE_ROOT_ID, 0, &attr, 0, 0, 0);
1005}1038}
1006 1039
1007struct fuse_inode_handle {1040struct fuse_inode_handle {
@@ -1610,7 +1643,8 @@ static int fuse_fill_super_submount(struct super_block *sb, · 602 unmodified lines
1610 return -ENOMEM;1643 return -ENOMEM;
1611 1644
1612 fuse_fill_attr_from_inode(&root_attr, parent_fi);1645 fuse_fill_attr_from_inode(&root_attr, parent_fi);
1613 root = fuse_iget(sb, parent_fi->nodeid, 0, &root_attr, 0, 0);1646 root = fuse_iget(sb, parent_fi->nodeid, 0, &root_attr, 0, 0,
1647 fuse_get_evict_ctr(fm->fc));
1614 /*1648 /*
1615 * This inode is just a duplicate, so it is not looked up and1649 * This inode is just a duplicate, so it is not looked up and
1616 * its nlookup should not be incremented. fuse_iget() does1650 * its nlookup should not be incremented. fuse_iget() does
@@ -2063,8 +2097,14 @@ static int __init fuse_fs_init(void) · 446 unmodified lines
2063 if (err)2097 if (err)
2064 goto out3;2098 goto out3;
2065 2099
2100 err = fuse_sysctl_register();
2101 if (err)
2102 goto out4;
2103
2066 return 0;2104 return 0;
2067 2105
2106 out4:
2107 unregister_filesystem(&fuse_fs_type);
2068 out3:2108 out3:
2069 unregister_fuseblk();2109 unregister_fuseblk();
2070 out2:2110 out2:
@@ -2075,6 +2115,7 @@ static int __init fuse_fs_init(void) · 4 unmodified lines
2075 2115
2076static void fuse_fs_cleanup(void)2116static void fuse_fs_cleanup(void)
2077{2117{
2118 fuse_sysctl_unregister();
2078 unregister_filesystem(&fuse_fs_type);2119 unregister_filesystem(&fuse_fs_type);
2079 unregister_fuseblk();2120 unregister_fuseblk();
2080 2121
fs/fuse/ioctl.c+18−17
@@ -10,6 +10,8 @@ · 9 unmodified lines
10#include <linux/fileattr.h>10#include <linux/fileattr.h>
11#include <linux/fsverity.h>11#include <linux/fsverity.h>
12 12
13#define FUSE_VERITY_ENABLE_ARG_MAX_PAGES 256
14
13static ssize_t fuse_send_ioctl(struct fuse_mount *fm, struct fuse_args *args,15static ssize_t fuse_send_ioctl(struct fuse_mount *fm, struct fuse_args *args,
14 struct fuse_ioctl_out *outarg)16 struct fuse_ioctl_out *outarg)
15{17{
@@ -140,7 +142,7 @@ static int fuse_setup_enable_verity(unsigned long arg, struct iovec *iov, · 124 unmodified lines
140{142{
141 struct fsverity_enable_arg enable;143 struct fsverity_enable_arg enable;
142 struct fsverity_enable_arg __user *uarg = (void __user *)arg;144 struct fsverity_enable_arg __user *uarg = (void __user *)arg;
143 const __u32 max_buffer_len = FUSE_MAX_MAX_PAGES * PAGE_SIZE;145 const __u32 max_buffer_len = FUSE_VERITY_ENABLE_ARG_MAX_PAGES * PAGE_SIZE;
144 146
145 if (copy_from_user(&enable, uarg, sizeof(enable)))147 if (copy_from_user(&enable, uarg, sizeof(enable)))
146 return -EFAULT;148 return -EFAULT;
@@ -249,12 +251,12 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, · 102 unmodified lines
249 BUILD_BUG_ON(sizeof(struct fuse_ioctl_iovec) * FUSE_IOCTL_MAX_IOV > PAGE_SIZE);251 BUILD_BUG_ON(sizeof(struct fuse_ioctl_iovec) * FUSE_IOCTL_MAX_IOV > PAGE_SIZE);
250 252
251 err = -ENOMEM;253 err = -ENOMEM;
252 ap.pages = fuse_pages_alloc(fm->fc->max_pages, GFP_KERNEL, &ap.descs);254 ap.folios = fuse_folios_alloc(fm->fc->max_pages, GFP_KERNEL, &ap.descs);
253 iov_page = (struct iovec *) __get_free_page(GFP_KERNEL);255 iov_page = (struct iovec *) __get_free_page(GFP_KERNEL);
254 if (!ap.pages || !iov_page)256 if (!ap.folios || !iov_page)
255 goto out;257 goto out;
256 258
257 fuse_page_descs_length_init(ap.descs, 0, fm->fc->max_pages);259 fuse_folio_descs_length_init(ap.descs, 0, fm->fc->max_pages);
258 260
259 /*261 /*
260 * If restricted, initialize IO parameters as encoded in @cmd.262 * If restricted, initialize IO parameters as encoded in @cmd.
@@ -304,14 +306,13 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, · 43 unmodified lines
304 err = -ENOMEM;306 err = -ENOMEM;
305 if (max_pages > fm->fc->max_pages)307 if (max_pages > fm->fc->max_pages)
306 goto out;308 goto out;
307 while (ap.num_pages < max_pages) {309 while (ap.num_folios < max_pages) {
308 ap.pages[ap.num_pages] = alloc_page(GFP_KERNEL | __GFP_HIGHMEM);310 ap.folios[ap.num_folios] = folio_alloc(GFP_KERNEL | __GFP_HIGHMEM, 0);
309 if (!ap.pages[ap.num_pages])311 if (!ap.folios[ap.num_folios])
310 goto out;312 goto out;
311 ap.num_pages++;313 ap.num_folios++;
312 }314 }
313 315
314
315 /* okay, let's send it to the client */316 /* okay, let's send it to the client */
316 ap.args.opcode = FUSE_IOCTL;317 ap.args.opcode = FUSE_IOCTL;
317 ap.args.nodeid = ff->nodeid;318 ap.args.nodeid = ff->nodeid;
@@ -325,8 +326,8 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, · 7 unmodified lines
325 326
326 err = -EFAULT;327 err = -EFAULT;
327 iov_iter_init(&ii, ITER_SOURCE, in_iov, in_iovs, in_size);328 iov_iter_init(&ii, ITER_SOURCE, in_iov, in_iovs, in_size);
328 for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= ap.num_pages); i++) {329 for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= ap.num_folios); i++) {
329 c = copy_page_from_iter(ap.pages[i], 0, PAGE_SIZE, &ii);330 c = copy_folio_from_iter(ap.folios[i], 0, PAGE_SIZE, &ii);
330 if (c != PAGE_SIZE && iov_iter_count(&ii))331 if (c != PAGE_SIZE && iov_iter_count(&ii))
331 goto out;332 goto out;
332 }333 }
@@ -364,7 +365,7 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, · 31 unmodified lines
364 in_iovs + out_iovs > FUSE_IOCTL_MAX_IOV)365 in_iovs + out_iovs > FUSE_IOCTL_MAX_IOV)
365 goto out;366 goto out;
366 367
367 vaddr = kmap_local_page(ap.pages[0]);368 vaddr = kmap_local_folio(ap.folios[0], 0);
368 err = fuse_copy_ioctl_iovec(fm->fc, iov_page, vaddr,369 err = fuse_copy_ioctl_iovec(fm->fc, iov_page, vaddr,
369 transferred, in_iovs + out_iovs,370 transferred, in_iovs + out_iovs,
370 (flags & FUSE_IOCTL_COMPAT) != 0);371 (flags & FUSE_IOCTL_COMPAT) != 0);
@@ -392,17 +393,17 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, · 21 unmodified lines
392 393
393 err = -EFAULT;394 err = -EFAULT;
394 iov_iter_init(&ii, ITER_DEST, out_iov, out_iovs, transferred);395 iov_iter_init(&ii, ITER_DEST, out_iov, out_iovs, transferred);
395 for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= ap.num_pages); i++) {396 for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= ap.num_folios); i++) {
396 c = copy_page_to_iter(ap.pages[i], 0, PAGE_SIZE, &ii);397 c = copy_folio_to_iter(ap.folios[i], 0, PAGE_SIZE, &ii);
397 if (c != PAGE_SIZE && iov_iter_count(&ii))398 if (c != PAGE_SIZE && iov_iter_count(&ii))
398 goto out;399 goto out;
399 }400 }
400 err = 0;401 err = 0;
401 out:402 out:
402 free_page((unsigned long) iov_page);403 free_page((unsigned long) iov_page);
403 while (ap.num_pages)404 while (ap.num_folios)
404 __free_page(ap.pages[--ap.num_pages]);405 folio_put(ap.folios[--ap.num_folios]);
405 kfree(ap.pages);406 kfree(ap.folios);
406 407
407 return err ? err : outarg.result;408 return err ? err : outarg.result;
408}409}
fs/fuse/readdir.c+18−15
@@ -149,7 +149,7 @@ static int parse_dirfile(char *buf, size_t nbytes, struct file *file, · 148 unmodified lines
149 149
150static int fuse_direntplus_link(struct file *file,150static int fuse_direntplus_link(struct file *file,
151 struct fuse_direntplus *direntplus,151 struct fuse_direntplus *direntplus,
152 u64 attr_version)152 u64 attr_version, u64 evict_ctr)
153{153{
154 struct fuse_entry_out *o = &direntplus->entry_out;154 struct fuse_entry_out *o = &direntplus->entry_out;
155 struct fuse_dirent *dirent = &direntplus->dirent;155 struct fuse_dirent *dirent = &direntplus->dirent;
@@ -233,7 +233,7 @@ retry: · 77 unmodified lines
233 } else {233 } else {
234 inode = fuse_iget(dir->i_sb, o->nodeid, o->generation,234 inode = fuse_iget(dir->i_sb, o->nodeid, o->generation,
235 &o->attr, ATTR_TIMEOUT(o),235 &o->attr, ATTR_TIMEOUT(o),
236 attr_version);236 attr_version, evict_ctr);
237 if (!inode)237 if (!inode)
238 inode = ERR_PTR(-ENOMEM);238 inode = ERR_PTR(-ENOMEM);
239 239
@@ -284,7 +284,8 @@ static void fuse_force_forget(struct file *file, u64 nodeid) · 44 unmodified lines
284}284}
285 285
286static int parse_dirplusfile(char *buf, size_t nbytes, struct file *file,286static int parse_dirplusfile(char *buf, size_t nbytes, struct file *file,
287 struct dir_context *ctx, u64 attr_version)287 struct dir_context *ctx, u64 attr_version,
288 u64 evict_ctr)
288{289{
289 struct fuse_direntplus *direntplus;290 struct fuse_direntplus *direntplus;
290 struct fuse_dirent *dirent;291 struct fuse_dirent *dirent;
@@ -319,7 +320,7 @@ static int parse_dirplusfile(char *buf, size_t nbytes, struct file *file, · 28 unmodified lines
319 buf += reclen;320 buf += reclen;
320 nbytes -= reclen;321 nbytes -= reclen;
321 322
322 ret = fuse_direntplus_link(file, direntplus, attr_version);323 ret = fuse_direntplus_link(file, direntplus, attr_version, evict_ctr);
323 if (ret)324 if (ret)
324 fuse_force_forget(file, direntplus->entry_out.nodeid);325 fuse_force_forget(file, direntplus->entry_out.nodeid);
325 }326 }
@@ -331,26 +332,27 @@ static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) · 5 unmodified lines
331{332{
332 int plus;333 int plus;
333 ssize_t res;334 ssize_t res;
334 struct page *page;335 struct folio *folio;
335 struct inode *inode = file_inode(file);336 struct inode *inode = file_inode(file);
336 struct fuse_mount *fm = get_fuse_mount(inode);337 struct fuse_mount *fm = get_fuse_mount(inode);
337 struct fuse_io_args ia = {};338 struct fuse_io_args ia = {};
338 struct fuse_args_pages *ap = &ia.ap;339 struct fuse_args_pages *ap = &ia.ap;
339 struct fuse_page_desc desc = { .length = PAGE_SIZE };340 struct fuse_folio_desc desc = { .length = PAGE_SIZE };
340 u64 attr_version = 0;341 u64 attr_version = 0, evict_ctr = 0;
341 bool locked;342 bool locked;
342 343
343 page = alloc_page(GFP_KERNEL);344 folio = folio_alloc(GFP_KERNEL, 0);
344 if (!page)345 if (!folio)
345 return -ENOMEM;346 return -ENOMEM;
346 347
347 plus = fuse_use_readdirplus(inode, ctx);348 plus = fuse_use_readdirplus(inode, ctx);
348 ap->args.out_pages = true;349 ap->args.out_pages = true;
349 ap->num_pages = 1;350 ap->num_folios = 1;
350 ap->pages = &page;351 ap->folios = &folio;
351 ap->descs = &desc;352 ap->descs = &desc;
352 if (plus) {353 if (plus) {
353 attr_version = fuse_get_attr_version(fm->fc);354 attr_version = fuse_get_attr_version(fm->fc);
355 evict_ctr = fuse_get_evict_ctr(fm->fc);
354 fuse_read_args_fill(&ia, file, ctx->pos, PAGE_SIZE,356 fuse_read_args_fill(&ia, file, ctx->pos, PAGE_SIZE,
355 FUSE_READDIRPLUS);357 FUSE_READDIRPLUS);
356 } else {358 } else {
@@ -367,15 +369,16 @@ static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) · 10 unmodified lines
367 if (ff->open_flags & FOPEN_CACHE_DIR)369 if (ff->open_flags & FOPEN_CACHE_DIR)
368 fuse_readdir_cache_end(file, ctx->pos);370 fuse_readdir_cache_end(file, ctx->pos);
369 } else if (plus) {371 } else if (plus) {
370 res = parse_dirplusfile(page_address(page), res,372 res = parse_dirplusfile(folio_address(folio), res,
371 file, ctx, attr_version);373 file, ctx, attr_version,
374 evict_ctr);
372 } else {375 } else {
373 res = parse_dirfile(page_address(page), res, file,376 res = parse_dirfile(folio_address(folio), res, file,
374 ctx);377 ctx);
375 }378 }
376 }379 }
377 380
378 __free_page(page);381 folio_put(folio);
379 fuse_invalidate_atime(inode);382 fuse_invalidate_atime(inode);
380 return res;383 return res;
381}384}
fs/fuse/sysctl.c+40−0
@@ -0,0 +1,40 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * linux/fs/fuse/fuse_sysctl.c
4 *
5 * Sysctl interface to fuse parameters
6 */
7#include <linux/sysctl.h>
8
9#include "fuse_i.h"
10
11static struct ctl_table_header *fuse_table_header;
12
13/* Bound by fuse_init_out max_pages, which is a u16 */
14static unsigned int sysctl_fuse_max_pages_limit = 65535;
15
16static struct ctl_table fuse_sysctl_table[] = {
17 {
18 .procname = "max_pages_limit",
19 .data = &fuse_max_pages_limit,
20 .maxlen = sizeof(fuse_max_pages_limit),
21 .mode = 0644,
22 .proc_handler = proc_douintvec_minmax,
23 .extra1 = SYSCTL_ONE,
24 .extra2 = &sysctl_fuse_max_pages_limit,
25 },
26};
27
28int fuse_sysctl_register(void)
29{
30 fuse_table_header = register_sysctl("fs/fuse", fuse_sysctl_table);
31 if (!fuse_table_header)
32 return -ENOMEM;
33 return 0;
34}
35
36void fuse_sysctl_unregister(void)
37{
38 unregister_sysctl_table(fuse_table_header);
39 fuse_table_header = NULL;
40}
fs/fuse/virtio_fs.c+42−35
@@ -97,7 +97,8 @@ struct virtio_fs_req_work { · 96 unmodified lines
97};97};
98 98
99static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,99static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,
100 struct fuse_req *req, bool in_flight);100 struct fuse_req *req, bool in_flight,
101 gfp_t gfp);
101 102
102static const struct constant_table dax_param_enums[] = {103static const struct constant_table dax_param_enums[] = {
103 {"always", FUSE_DAX_ALWAYS },104 {"always", FUSE_DAX_ALWAYS },
@@ -575,6 +576,8 @@ static void virtio_fs_request_dispatch_work(struct work_struct *work) · 471 unmodified lines
575 576
576 /* Dispatch pending requests */577 /* Dispatch pending requests */
577 while (1) {578 while (1) {
579 unsigned int flags;
580
578 spin_lock(&fsvq->lock);581 spin_lock(&fsvq->lock);
579 req = list_first_entry_or_null(&fsvq->queued_reqs,582 req = list_first_entry_or_null(&fsvq->queued_reqs,
580 struct fuse_req, list);583 struct fuse_req, list);
@@ -585,7 +588,9 @@ static void virtio_fs_request_dispatch_work(struct work_struct *work) · 4 unmodified lines
585 list_del_init(&req->list);588 list_del_init(&req->list);
586 spin_unlock(&fsvq->lock);589 spin_unlock(&fsvq->lock);
587 590
588 ret = virtio_fs_enqueue_req(fsvq, req, true);591 flags = memalloc_nofs_save();
592 ret = virtio_fs_enqueue_req(fsvq, req, true, GFP_KERNEL);
593 memalloc_nofs_restore(flags);
589 if (ret < 0) {594 if (ret < 0) {
590 if (ret == -ENOSPC) {595 if (ret == -ENOSPC) {
591 spin_lock(&fsvq->lock);596 spin_lock(&fsvq->lock);
@@ -686,7 +691,7 @@ static void virtio_fs_hiprio_dispatch_work(struct work_struct *work) · 94 unmodified lines
686}691}
687 692
688/* Allocate and copy args into req->argbuf */693/* Allocate and copy args into req->argbuf */
689static int copy_args_to_argbuf(struct fuse_req *req)694static int copy_args_to_argbuf(struct fuse_req *req, gfp_t gfp)
690{695{
691 struct fuse_args *args = req->args;696 struct fuse_args *args = req->args;
692 unsigned int offset = 0;697 unsigned int offset = 0;
@@ -700,7 +705,7 @@ static int copy_args_to_argbuf(struct fuse_req *req) · 7 unmodified lines
700 len = fuse_len_args(num_in, (struct fuse_arg *) args->in_args) +705 len = fuse_len_args(num_in, (struct fuse_arg *) args->in_args) +
701 fuse_len_args(num_out, args->out_args);706 fuse_len_args(num_out, args->out_args);
702 707
703 req->argbuf = kmalloc(len, GFP_ATOMIC);708 req->argbuf = kmalloc(len, gfp);
704 if (!req->argbuf)709 if (!req->argbuf)
705 return -ENOMEM;710 return -ENOMEM;
706 711
@@ -760,7 +765,7 @@ static void virtio_fs_request_complete(struct fuse_req *req, · 53 unmodified lines
760 struct fuse_args *args;765 struct fuse_args *args;
761 struct fuse_args_pages *ap;766 struct fuse_args_pages *ap;
762 unsigned int len, i, thislen;767 unsigned int len, i, thislen;
763 struct page *page;768 struct folio *folio;
764 769
765 /*770 /*
766 * TODO verify that server properly follows FUSE protocol771 * TODO verify that server properly follows FUSE protocol
@@ -772,12 +777,12 @@ static void virtio_fs_request_complete(struct fuse_req *req, · 5 unmodified lines
772 if (args->out_pages && args->page_zeroing) {777 if (args->out_pages && args->page_zeroing) {
773 len = args->out_args[args->out_numargs - 1].size;778 len = args->out_args[args->out_numargs - 1].size;
774 ap = container_of(args, typeof(*ap), args);779 ap = container_of(args, typeof(*ap), args);
775 for (i = 0; i < ap->num_pages; i++) {780 for (i = 0; i < ap->num_folios; i++) {
776 thislen = ap->descs[i].length;781 thislen = ap->descs[i].length;
777 if (len < thislen) {782 if (len < thislen) {
778 WARN_ON(ap->descs[i].offset);783 WARN_ON(ap->descs[i].offset);
779 page = ap->pages[i];784 folio = ap->folios[i];
780 zero_user_segment(page, len, thislen);785 folio_zero_segment(folio, len, thislen);
781 len = 0;786 len = 0;
782 } else {787 } else {
783 len -= thislen;788 len -= thislen;
@@ -1267,15 +1272,15 @@ static void virtio_fs_send_interrupt(struct fuse_iqueue *fiq, struct fuse_req *r · 483 unmodified lines
1267}1272}
1268 1273
1269/* Count number of scatter-gather elements required */1274/* Count number of scatter-gather elements required */
1270static unsigned int sg_count_fuse_pages(struct fuse_page_desc *page_descs,1275static unsigned int sg_count_fuse_folios(struct fuse_folio_desc *folio_descs,
1271 unsigned int num_pages,1276 unsigned int num_folios,
1272 unsigned int total_len)1277 unsigned int total_len)
1273{1278{
1274 unsigned int i;1279 unsigned int i;
1275 unsigned int this_len;1280 unsigned int this_len;
1276 1281
1277 for (i = 0; i < num_pages && total_len; i++) {1282 for (i = 0; i < num_folios && total_len; i++) {
1278 this_len = min(page_descs[i].length, total_len);1283 this_len = min(folio_descs[i].length, total_len);
1279 total_len -= this_len;1284 total_len -= this_len;
1280 }1285 }
1281 1286
@@ -1294,8 +1299,8 @@ static unsigned int sg_count_fuse_req(struct fuse_req *req) · 12 unmodified lines
1294 1299
1295 if (args->in_pages) {1300 if (args->in_pages) {
1296 size = args->in_args[args->in_numargs - 1].size;1301 size = args->in_args[args->in_numargs - 1].size;
1297 total_sgs += sg_count_fuse_pages(ap->descs, ap->num_pages,1302 total_sgs += sg_count_fuse_folios(ap->descs, ap->num_folios,
1298 size);1303 size);
1299 }1304 }
1300 1305
1301 if (!test_bit(FR_ISREPLY, &req->flags))1306 if (!test_bit(FR_ISREPLY, &req->flags))
@@ -1308,27 +1313,27 @@ static unsigned int sg_count_fuse_req(struct fuse_req *req) · 6 unmodified lines
1308 1313
1309 if (args->out_pages) {1314 if (args->out_pages) {
1310 size = args->out_args[args->out_numargs - 1].size;1315 size = args->out_args[args->out_numargs - 1].size;
1311 total_sgs += sg_count_fuse_pages(ap->descs, ap->num_pages,1316 total_sgs += sg_count_fuse_folios(ap->descs, ap->num_folios,
1312 size);1317 size);
1313 }1318 }
1314 1319
1315 return total_sgs;1320 return total_sgs;
1316}1321}
1317 1322
1318/* Add pages to scatter-gather list and return number of elements used */1323/* Add folios to scatter-gather list and return number of elements used */
1319static unsigned int sg_init_fuse_pages(struct scatterlist *sg,1324static unsigned int sg_init_fuse_folios(struct scatterlist *sg,
1320 struct page **pages,1325 struct folio **folios,
1321 struct fuse_page_desc *page_descs,1326 struct fuse_folio_desc *folio_descs,
1322 unsigned int num_pages,1327 unsigned int num_folios,
1323 unsigned int total_len)1328 unsigned int total_len)
1324{1329{
1325 unsigned int i;1330 unsigned int i;
1326 unsigned int this_len;1331 unsigned int this_len;
1327 1332
1328 for (i = 0; i < num_pages && total_len; i++) {1333 for (i = 0; i < num_folios && total_len; i++) {
1329 sg_init_table(&sg[i], 1);1334 sg_init_table(&sg[i], 1);
1330 this_len = min(page_descs[i].length, total_len);1335 this_len = min(folio_descs[i].length, total_len);
1331 sg_set_page(&sg[i], pages[i], this_len, page_descs[i].offset);1336 sg_set_folio(&sg[i], folios[i], this_len, folio_descs[i].offset);
1332 total_len -= this_len;1337 total_len -= this_len;
1333 }1338 }
1334 1339
@@ -1353,10 +1358,10 @@ static unsigned int sg_init_fuse_args(struct scatterlist *sg, · 18 unmodified lines
1353 sg_init_one(&sg[total_sgs++], argbuf, len);1358 sg_init_one(&sg[total_sgs++], argbuf, len);
1354 1359
1355 if (argpages)1360 if (argpages)
1356 total_sgs += sg_init_fuse_pages(&sg[total_sgs],1361 total_sgs += sg_init_fuse_folios(&sg[total_sgs],
1357 ap->pages, ap->descs,1362 ap->folios, ap->descs,
1358 ap->num_pages,1363 ap->num_folios,
1359 args[numargs - 1].size);1364 args[numargs - 1].size);
1360 1365
1361 if (len_used)1366 if (len_used)
1362 *len_used = len;1367 *len_used = len;
@@ -1366,7 +1371,8 @@ static unsigned int sg_init_fuse_args(struct scatterlist *sg, · 3 unmodified lines
1366 1371
1367/* Add a request to a virtqueue and kick the device */1372/* Add a request to a virtqueue and kick the device */
1368static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,1373static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,
1369 struct fuse_req *req, bool in_flight)1374 struct fuse_req *req, bool in_flight,
1375 gfp_t gfp)
1370{1376{
1371 /* requests need at least 4 elements */1377 /* requests need at least 4 elements */
1372 struct scatterlist *stack_sgs[6];1378 struct scatterlist *stack_sgs[6];
@@ -1387,8 +1393,8 @@ static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, · 14 unmodified lines
1387 /* Does the sglist fit on the stack? */1393 /* Does the sglist fit on the stack? */
1388 total_sgs = sg_count_fuse_req(req);1394 total_sgs = sg_count_fuse_req(req);
1389 if (total_sgs > ARRAY_SIZE(stack_sgs)) {1395 if (total_sgs > ARRAY_SIZE(stack_sgs)) {
1390 sgs = kmalloc_array(total_sgs, sizeof(sgs[0]), GFP_ATOMIC);1396 sgs = kmalloc_array(total_sgs, sizeof(sgs[0]), gfp);
1391 sg = kmalloc_array(total_sgs, sizeof(sg[0]), GFP_ATOMIC);1397 sg = kmalloc_array(total_sgs, sizeof(sg[0]), gfp);
1392 if (!sgs || !sg) {1398 if (!sgs || !sg) {
1393 ret = -ENOMEM;1399 ret = -ENOMEM;
1394 goto out;1400 goto out;
@@ -1396,7 +1402,7 @@ static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, · 1 unmodified line
1396 }1402 }
1397 1403
1398 /* Use a bounce buffer since stack args cannot be mapped */1404 /* Use a bounce buffer since stack args cannot be mapped */
1399 ret = copy_args_to_argbuf(req);1405 ret = copy_args_to_argbuf(req, gfp);
1400 if (ret < 0)1406 if (ret < 0)
1401 goto out;1407 goto out;
1402 1408
@@ -1490,7 +1496,7 @@ static void virtio_fs_send_req(struct fuse_iqueue *fiq, struct fuse_req *req) · 87 unmodified lines
1490 queue_id);1496 queue_id);
1491 1497
1492 fsvq = &fs->vqs[queue_id];1498 fsvq = &fs->vqs[queue_id];
1493 ret = virtio_fs_enqueue_req(fsvq, req, false);1499 ret = virtio_fs_enqueue_req(fsvq, req, false, GFP_ATOMIC);
1494 if (ret < 0) {1500 if (ret < 0) {
1495 if (ret == -ENOSPC) {1501 if (ret == -ENOSPC) {
1496 /*1502 /*
@@ -1691,6 +1697,7 @@ static int virtio_fs_get_tree(struct fs_context *fsc) · 194 unmodified lines
1691 fc->delete_stale = true;1697 fc->delete_stale = true;
1692 fc->auto_submounts = true;1698 fc->auto_submounts = true;
1693 fc->sync_fs = true;1699 fc->sync_fs = true;
1700 fc->use_pages_for_kvec_io = true;
1694 1701
1695 /* Tell FUSE to split requests that exceed the virtqueue's size */1702 /* Tell FUSE to split requests that exceed the virtqueue's size */
1696 fc->max_pages_limit = min_t(unsigned int, fc->max_pages_limit,1703 fc->max_pages_limit = min_t(unsigned int, fc->max_pages_limit,
include/linux/mm.h+1−0
@@ -2550,6 +2550,7 @@ struct kvec; · 2549 unmodified lines
2550struct page *get_dump_page(unsigned long addr);2550struct page *get_dump_page(unsigned long addr);
2551 2551
2552bool folio_mark_dirty(struct folio *folio);2552bool folio_mark_dirty(struct folio *folio);
2553bool folio_mark_dirty_lock(struct folio *folio);
2553bool set_page_dirty(struct page *page);2554bool set_page_dirty(struct page *page);
2554int set_page_dirty_lock(struct page *page);2555int set_page_dirty_lock(struct page *page);
2555 2556
mm/folio-compat.c+6−0
@@ -52,6 +52,12 @@ bool set_page_dirty(struct page *page) · 51 unmodified lines
52}52}
53EXPORT_SYMBOL(set_page_dirty);53EXPORT_SYMBOL(set_page_dirty);
54 54
55int set_page_dirty_lock(struct page *page)
56{
57 return folio_mark_dirty_lock(page_folio(page));
58}
59EXPORT_SYMBOL(set_page_dirty_lock);
60
55bool clear_page_dirty_for_io(struct page *page)61bool clear_page_dirty_for_io(struct page *page)
56{62{
57 return folio_clear_dirty_for_io(page_folio(page));63 return folio_clear_dirty_for_io(page_folio(page));
mm/page-writeback.c+11−11
@@ -2925,25 +2925,25 @@ bool folio_mark_dirty(struct folio *folio) · 2924 unmodified lines
2925EXPORT_SYMBOL(folio_mark_dirty);2925EXPORT_SYMBOL(folio_mark_dirty);
2926 2926
2927/*2927/*
2928 * set_page_dirty() is racy if the caller has no reference against2928 * folio_mark_dirty() is racy if the caller has no reference against
2929 * page->mapping->host, and if the page is unlocked. This is because another2929 * folio->mapping->host, and if the folio is unlocked. This is because another
2930 * CPU could truncate the page off the mapping and then free the mapping.2930 * CPU could truncate the folio off the mapping and then free the mapping.
2931 *2931 *
2932 * Usually, the page _is_ locked, or the caller is a user-space process which2932 * Usually, the folio _is_ locked, or the caller is a user-space process which
2933 * holds a reference on the inode by having an open file.2933 * holds a reference on the inode by having an open file.
2934 *2934 *
2935 * In other cases, the page should be locked before running set_page_dirty().2935 * In other cases, the folio should be locked before running folio_mark_dirty().
2936 */2936 */
2937int set_page_dirty_lock(struct page *page)2937bool folio_mark_dirty_lock(struct folio *folio)
2938{2938{
2939 int ret;2939 bool ret;
2940 2940
2941 lock_page(page);2941 folio_lock(folio);
2942 ret = set_page_dirty(page);2942 ret = folio_mark_dirty(folio);
2943 unlock_page(page);2943 folio_unlock(folio);
2944 return ret;2944 return ret;
2945}2945}
2946EXPORT_SYMBOL(set_page_dirty_lock);2946EXPORT_SYMBOL(folio_mark_dirty_lock);
2947 2947
2948/*2948/*
2949 * This cancels just the dirty bit on the kernel page itself, it does NOT2949 * This cancels just the dirty bit on the kernel page itself, it does NOT

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t