Pull fuse updates from Miklos Szeredi: - Add page -> folio conversions (Joanne Koong, Josef Bacik) - Allow max size of fuse requests to be configurable with a sysctl (Joanne Koong) - Allow FOPEN_DIRECT_IO to take advantage of async code path (yangyun) - Fix large kernel reads (like a module load) in virtio_fs (Hou Tao) - Fix attribute inconsistency in case readdirplus (and plain lookup in corner cases) is racing with inode eviction (Zhang Tianci) - Fix a WARN_ON triggered by virtio_fs (Asahi Lina) * tag 'fuse-update-6.13' of git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse: (30 commits) virtiofs: dax: remove ->writepages() callback fuse: check attributes staleness on fuse_iget() fuse: remove pages for requests and exclusively use folios fuse: convert direct io to use folios mm/writeback: add folio_mark_dirty_lock() fuse: convert writebacks to use folios fuse: convert retrieves to use folios fuse: convert ioctls to use folios fuse: convert writes (non-writeback) to use folios fuse: convert reads to use folios fuse: convert readdir to use folios fuse: convert readlink to use folios fuse: convert cuse to use folios fuse: add support in virtio for requests using folios fuse: support folios in struct fuse_args_pages and fuse_copy_pages() fuse: convert fuse_notify_store to use folios fuse: convert fuse_retrieve to use folios fuse: use the folio based vmstat helpers fuse: convert fuse_writepage_need_send to take a folio fuse: convert fuse_do_readpage to use folios ...
Unified · Split
| @@ -337,3 +337,13 @@ Each "watch" costs roughly 90 bytes on a 32-bit kernel, and roughly 160 bytes · 336 unmodified lines | ||
| 337 | 337 | on a 64-bit one. |
| 338 | 338 | The current default value for ``max_user_watches`` is 4% of the |
| 339 | 339 | available low memory, divided by the "watch" cost in bytes. |
| 340 | + | |
| 341 | +5. /proc/sys/fs/fuse - Configuration options for FUSE filesystems | |
| 342 | +===================================================================== | |
| 343 | + | |
| 344 | +This directory contains the following configuration options for FUSE | |
| 345 | +filesystems: | |
| 346 | + | |
| 347 | +``/proc/sys/fs/fuse/max_pages_limit`` is a read/write file for | |
| 348 | +setting/getting the maximum number of pages that can be used for servicing | |
| 349 | +requests in FUSE. | |
| @@ -14,5 +14,6 @@ fuse-y := dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o · 13 unmodified lines | ||
| 14 | 14 | fuse-y += iomode.o |
| 15 | 15 | fuse-$(CONFIG_FUSE_DAX) += dax.o |
| 16 | 16 | fuse-$(CONFIG_FUSE_PASSTHROUGH) += passthrough.o |
| 17 | +fuse-$(CONFIG_SYSCTL) += sysctl.o | |
| 17 | 18 | |
| 18 | 19 | virtiofs-y := virtio_fs.o |
| @@ -303,8 +303,8 @@ struct cuse_init_args { · 302 unmodified lines | ||
| 303 | 303 | struct fuse_args_pages ap; |
| 304 | 304 | struct cuse_init_in in; |
| 305 | 305 | struct cuse_init_out out; |
| 306 | − struct page *page; | |
| 307 | − struct fuse_page_desc desc; | |
| 306 | + struct folio *folio; | |
| 307 | + struct fuse_folio_desc desc; | |
| 308 | 308 | }; |
| 309 | 309 | |
| 310 | 310 | /** |
| @@ -326,7 +326,7 @@ static void cuse_process_init_reply(struct fuse_mount *fm, · 15 unmodified lines | ||
| 326 | 326 | struct fuse_args_pages *ap = &ia->ap; |
| 327 | 327 | struct cuse_conn *cc = fc_to_cc(fc), *pos; |
| 328 | 328 | struct cuse_init_out *arg = &ia->out; |
| 329 | − struct page *page = ap->pages[0]; | |
| 329 | + struct folio *folio = ap->folios[0]; | |
| 330 | 330 | struct cuse_devinfo devinfo = { }; |
| 331 | 331 | struct device *dev; |
| 332 | 332 | struct cdev *cdev; |
| @@ -343,7 +343,7 @@ static void cuse_process_init_reply(struct fuse_mount *fm, · 10 unmodified lines | ||
| 343 | 343 | /* parse init reply */ |
| 344 | 344 | cc->unrestricted_ioctl = arg->flags & CUSE_UNRESTRICTED_IOCTL; |
| 345 | 345 | |
| 346 | − rc = cuse_parse_devinfo(page_address(page), ap->args.out_args[1].size, | |
| 346 | + rc = cuse_parse_devinfo(folio_address(folio), ap->args.out_args[1].size, | |
| 347 | 347 | &devinfo); |
| 348 | 348 | if (rc) |
| 349 | 349 | goto err; |
| @@ -411,7 +411,7 @@ static void cuse_process_init_reply(struct fuse_mount *fm, · 61 unmodified lines | ||
| 411 | 411 | kobject_uevent(&dev->kobj, KOBJ_ADD); |
| 412 | 412 | out: |
| 413 | 413 | kfree(ia); |
| 414 | − __free_page(page); | |
| 414 | + folio_put(folio); | |
| 415 | 415 | return; |
| 416 | 416 | |
| 417 | 417 | err_cdev: |
| @@ -429,7 +429,7 @@ err: · 11 unmodified lines | ||
| 429 | 429 | static int cuse_send_init(struct cuse_conn *cc) |
| 430 | 430 | { |
| 431 | 431 | int rc; |
| 432 | − struct page *page; | |
| 432 | + struct folio *folio; | |
| 433 | 433 | struct fuse_mount *fm = &cc->fm; |
| 434 | 434 | struct cuse_init_args *ia; |
| 435 | 435 | struct fuse_args_pages *ap; |
| @@ -437,13 +437,14 @@ static int cuse_send_init(struct cuse_conn *cc) · 1 unmodified line | ||
| 437 | 437 | BUILD_BUG_ON(CUSE_INIT_INFO_MAX > PAGE_SIZE); |
| 438 | 438 | |
| 439 | 439 | rc = -ENOMEM; |
| 440 | − page = alloc_page(GFP_KERNEL | __GFP_ZERO); | |
| 441 | − if (!page) | |
| 440 | + | |
| 441 | + folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0); | |
| 442 | + if (!folio) | |
| 442 | 443 | goto err; |
| 443 | 444 | |
| 444 | 445 | ia = kzalloc(sizeof(*ia), GFP_KERNEL); |
| 445 | 446 | if (!ia) |
| 446 | − goto err_free_page; | |
| 447 | + goto err_free_folio; | |
| 447 | 448 | |
| 448 | 449 | ap = &ia->ap; |
| 449 | 450 | ia->in.major = FUSE_KERNEL_VERSION; |
| @@ -459,18 +460,18 @@ static int cuse_send_init(struct cuse_conn *cc) · 9 unmodified lines | ||
| 459 | 460 | ap->args.out_args[1].size = CUSE_INIT_INFO_MAX; |
| 460 | 461 | ap->args.out_argvar = true; |
| 461 | 462 | ap->args.out_pages = true; |
| 462 | − ap->num_pages = 1; | |
| 463 | − ap->pages = &ia->page; | |
| 463 | + ap->num_folios = 1; | |
| 464 | + ap->folios = &ia->folio; | |
| 464 | 465 | ap->descs = &ia->desc; |
| 465 | − ia->page = page; | |
| 466 | + ia->folio = folio; | |
| 466 | 467 | ia->desc.length = ap->args.out_args[1].size; |
| 467 | 468 | ap->args.end = cuse_process_init_reply; |
| 468 | 469 | |
| 469 | 470 | rc = fuse_simple_background(fm, &ap->args, GFP_KERNEL); |
| 470 | 471 | if (rc) { |
| 471 | 472 | kfree(ia); |
| 472 | −err_free_page: | |
| 473 | − __free_page(page); | |
| 473 | +err_free_folio: | |
| 474 | + folio_put(folio); | |
| 474 | 475 | } |
| 475 | 476 | err: |
| 476 | 477 | return rc; |
| @@ -774,16 +774,6 @@ out: · 773 unmodified lines | ||
| 774 | 774 | return ret; |
| 775 | 775 | } |
| 776 | 776 | |
| 777 | −static int fuse_dax_writepages(struct address_space *mapping, | |
| 778 | − struct writeback_control *wbc) | |
| 779 | −{ | |
| 780 | − | |
| 781 | − struct inode *inode = mapping->host; | |
| 782 | − struct fuse_conn *fc = get_fuse_conn(inode); | |
| 783 | − | |
| 784 | − return dax_writeback_mapping_range(mapping, fc->dax->dev, wbc); | |
| 785 | −} | |
| 786 | − | |
| 787 | 777 | static vm_fault_t __fuse_dax_fault(struct vm_fault *vmf, unsigned int order, |
| 788 | 778 | bool write) |
| 789 | 779 | { |
| @@ -1323,7 +1313,6 @@ bool fuse_dax_inode_alloc(struct super_block *sb, struct fuse_inode *fi) · 533 unmodified lines | ||
| 1323 | 1313 | } |
| 1324 | 1314 | |
| 1325 | 1315 | static const struct address_space_operations fuse_dax_file_aops = { |
| 1326 | − .writepages = fuse_dax_writepages, | |
| 1327 | 1316 | .direct_IO = noop_direct_IO, |
| 1328 | 1317 | .dirty_folio = noop_dirty_folio, |
| 1329 | 1318 | }; |
| @@ -1028,17 +1028,27 @@ static int fuse_copy_pages(struct fuse_copy_state *cs, unsigned nbytes, · 1027 unmodified lines | ||
| 1028 | 1028 | struct fuse_req *req = cs->req; |
| 1029 | 1029 | struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args); |
| 1030 | 1030 | |
| 1031 | − | |
| 1032 | − for (i = 0; i < ap->num_pages && (nbytes || zeroing); i++) { | |
| 1031 | + for (i = 0; i < ap->num_folios && (nbytes || zeroing); i++) { | |
| 1033 | 1032 | int err; |
| 1034 | 1033 | unsigned int offset = ap->descs[i].offset; |
| 1035 | 1034 | unsigned int count = min(nbytes, ap->descs[i].length); |
| 1035 | + struct page *orig, *pagep; | |
| 1036 | + | |
| 1037 | + orig = pagep = &ap->folios[i]->page; | |
| 1036 | 1038 | |
| 1037 | − err = fuse_copy_page(cs, &ap->pages[i], offset, count, zeroing); | |
| 1039 | + err = fuse_copy_page(cs, &pagep, offset, count, zeroing); | |
| 1038 | 1040 | if (err) |
| 1039 | 1041 | return err; |
| 1040 | 1042 | |
| 1041 | 1043 | nbytes -= count; |
| 1044 | + | |
| 1045 | + /* | |
| 1046 | + * fuse_copy_page may have moved a page from a pipe instead of | |
| 1047 | + * copying into our given page, so update the folios if it was | |
| 1048 | + * replaced. | |
| 1049 | + */ | |
| 1050 | + if (pagep != orig) | |
| 1051 | + ap->folios[i] = page_folio(pagep); | |
| 1042 | 1052 | } |
| 1043 | 1053 | return 0; |
| 1044 | 1054 | } |
| @@ -1654,24 +1664,25 @@ static int fuse_notify_store(struct fuse_conn *fc, unsigned int size, · 609 unmodified lines | ||
| 1654 | 1664 | |
| 1655 | 1665 | num = outarg.size; |
| 1656 | 1666 | while (num) { |
| 1667 | + struct folio *folio; | |
| 1657 | 1668 | struct page *page; |
| 1658 | 1669 | unsigned int this_num; |
| 1659 | 1670 | |
| 1660 | − err = -ENOMEM; | |
| 1661 | − page = find_or_create_page(mapping, index, | |
| 1662 | − mapping_gfp_mask(mapping)); | |
| 1663 | − if (!page) | |
| 1671 | + folio = filemap_grab_folio(mapping, index); | |
| 1672 | + err = PTR_ERR(folio); | |
| 1673 | + if (IS_ERR(folio)) | |
| 1664 | 1674 | goto out_iput; |
| 1665 | 1675 | |
| 1666 | − this_num = min_t(unsigned, num, PAGE_SIZE - offset); | |
| 1676 | + page = &folio->page; | |
| 1677 | + this_num = min_t(unsigned, num, folio_size(folio) - offset); | |
| 1667 | 1678 | err = fuse_copy_page(cs, &page, offset, this_num, 0); |
| 1668 | − if (!PageUptodate(page) && !err && offset == 0 && | |
| 1669 | − (this_num == PAGE_SIZE || file_size == end)) { | |
| 1670 | − zero_user_segment(page, this_num, PAGE_SIZE); | |
| 1671 | − SetPageUptodate(page); | |
| 1679 | + if (!folio_test_uptodate(folio) && !err && offset == 0 && | |
| 1680 | + (this_num == folio_size(folio) || file_size == end)) { | |
| 1681 | + folio_zero_segment(folio, this_num, folio_size(folio)); | |
| 1682 | + folio_mark_uptodate(folio); | |
| 1672 | 1683 | } |
| 1673 | − unlock_page(page); | |
| 1674 | − put_page(page); | |
| 1684 | + folio_unlock(folio); | |
| 1685 | + folio_put(folio); | |
| 1675 | 1686 | |
| 1676 | 1687 | if (err) |
| 1677 | 1688 | goto out_iput; |
| @@ -1703,7 +1714,7 @@ static void fuse_retrieve_end(struct fuse_mount *fm, struct fuse_args *args, · 25 unmodified lines | ||
| 1703 | 1714 | struct fuse_retrieve_args *ra = |
| 1704 | 1715 | container_of(args, typeof(*ra), ap.args); |
| 1705 | 1716 | |
| 1706 | − release_pages(ra->ap.pages, ra->ap.num_pages); | |
| 1717 | + release_pages(ra->ap.folios, ra->ap.num_folios); | |
| 1707 | 1718 | kfree(ra); |
| 1708 | 1719 | } |
| 1709 | 1720 | |
| @@ -1717,7 +1728,7 @@ static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode, · 7 unmodified lines | ||
| 1717 | 1728 | unsigned int num; |
| 1718 | 1729 | unsigned int offset; |
| 1719 | 1730 | size_t total_len = 0; |
| 1720 | − unsigned int num_pages; | |
| 1731 | + unsigned int num_pages, cur_pages = 0; | |
| 1721 | 1732 | struct fuse_conn *fc = fm->fc; |
| 1722 | 1733 | struct fuse_retrieve_args *ra; |
| 1723 | 1734 | size_t args_size = sizeof(*ra); |
| @@ -1736,15 +1747,15 @@ static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode, · 12 unmodified lines | ||
| 1736 | 1747 | num_pages = (num + offset + PAGE_SIZE - 1) >> PAGE_SHIFT; |
| 1737 | 1748 | num_pages = min(num_pages, fc->max_pages); |
| 1738 | 1749 | |
| 1739 | − args_size += num_pages * (sizeof(ap->pages[0]) + sizeof(ap->descs[0])); | |
| 1750 | + args_size += num_pages * (sizeof(ap->folios[0]) + sizeof(ap->descs[0])); | |
| 1740 | 1751 | |
| 1741 | 1752 | ra = kzalloc(args_size, GFP_KERNEL); |
| 1742 | 1753 | if (!ra) |
| 1743 | 1754 | return -ENOMEM; |
| 1744 | 1755 | |
| 1745 | 1756 | ap = &ra->ap; |
| 1746 | − ap->pages = (void *) (ra + 1); | |
| 1747 | − ap->descs = (void *) (ap->pages + num_pages); | |
| 1757 | + ap->folios = (void *) (ra + 1); | |
| 1758 | + ap->descs = (void *) (ap->folios + num_pages); | |
| 1748 | 1759 | |
| 1749 | 1760 | args = &ap->args; |
| 1750 | 1761 | args->nodeid = outarg->nodeid; |
| @@ -1755,19 +1766,20 @@ static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode, · 4 unmodified lines | ||
| 1755 | 1766 | |
| 1756 | 1767 | index = outarg->offset >> PAGE_SHIFT; |
| 1757 | 1768 | |
| 1758 | − while (num && ap->num_pages < num_pages) { | |
| 1759 | − struct page *page; | |
| 1769 | + while (num && cur_pages < num_pages) { | |
| 1770 | + struct folio *folio; | |
| 1760 | 1771 | unsigned int this_num; |
| 1761 | 1772 | |
| 1762 | − page = find_get_page(mapping, index); | |
| 1763 | − if (!page) | |
| 1773 | + folio = filemap_get_folio(mapping, index); | |
| 1774 | + if (IS_ERR(folio)) | |
| 1764 | 1775 | break; |
| 1765 | 1776 | |
| 1766 | 1777 | this_num = min_t(unsigned, num, PAGE_SIZE - offset); |
| 1767 | − ap->pages[ap->num_pages] = page; | |
| 1768 | − ap->descs[ap->num_pages].offset = offset; | |
| 1769 | − ap->descs[ap->num_pages].length = this_num; | |
| 1770 | − ap->num_pages++; | |
| 1778 | + ap->folios[ap->num_folios] = folio; | |
| 1779 | + ap->descs[ap->num_folios].offset = offset; | |
| 1780 | + ap->descs[ap->num_folios].length = this_num; | |
| 1781 | + ap->num_folios++; | |
| 1782 | + cur_pages++; | |
| 1771 | 1783 | |
| 1772 | 1784 | offset = 0; |
| 1773 | 1785 | num -= this_num; |
| @@ -366,7 +366,7 @@ int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name · 365 unmodified lines | ||
| 366 | 366 | struct fuse_mount *fm = get_fuse_mount_super(sb); |
| 367 | 367 | FUSE_ARGS(args); |
| 368 | 368 | struct fuse_forget_link *forget; |
| 369 | − u64 attr_version; | |
| 369 | + u64 attr_version, evict_ctr; | |
| 370 | 370 | int err; |
| 371 | 371 | |
| 372 | 372 | *inode = NULL; |
| @@ -381,6 +381,7 @@ int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name · 8 unmodified lines | ||
| 381 | 381 | goto out; |
| 382 | 382 | |
| 383 | 383 | attr_version = fuse_get_attr_version(fm->fc); |
| 384 | + evict_ctr = fuse_get_evict_ctr(fm->fc); | |
| 384 | 385 | |
| 385 | 386 | fuse_lookup_init(fm->fc, &args, nodeid, name, outarg); |
| 386 | 387 | err = fuse_simple_request(fm, &args); |
| @@ -398,7 +399,7 @@ int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name · 11 unmodified lines | ||
| 398 | 399 | |
| 399 | 400 | *inode = fuse_iget(sb, outarg->nodeid, outarg->generation, |
| 400 | 401 | &outarg->attr, ATTR_TIMEOUT(outarg), |
| 401 | − attr_version); | |
| 402 | + attr_version, evict_ctr); | |
| 402 | 403 | err = -ENOMEM; |
| 403 | 404 | if (!*inode) { |
| 404 | 405 | fuse_queue_forget(fm->fc, forget, outarg->nodeid, 1); |
| @@ -691,7 +692,7 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, · 286 unmodified lines | ||
| 691 | 692 | ff->nodeid = outentry.nodeid; |
| 692 | 693 | ff->open_flags = outopenp->open_flags; |
| 693 | 694 | inode = fuse_iget(dir->i_sb, outentry.nodeid, outentry.generation, |
| 694 | − &outentry.attr, ATTR_TIMEOUT(&outentry), 0); | |
| 695 | + &outentry.attr, ATTR_TIMEOUT(&outentry), 0, 0); | |
| 695 | 696 | if (!inode) { |
| 696 | 697 | flags &= ~(O_CREAT | O_EXCL | O_TRUNC); |
| 697 | 698 | fuse_sync_release(NULL, ff, flags); |
| @@ -822,7 +823,7 @@ static int create_new_entry(struct mnt_idmap *idmap, struct fuse_mount *fm, · 124 unmodified lines | ||
| 822 | 823 | goto out_put_forget_req; |
| 823 | 824 | |
| 824 | 825 | inode = fuse_iget(dir->i_sb, outarg.nodeid, outarg.generation, |
| 825 | − &outarg.attr, ATTR_TIMEOUT(&outarg), 0); | |
| 826 | + &outarg.attr, ATTR_TIMEOUT(&outarg), 0, 0); | |
| 826 | 827 | if (!inode) { |
| 827 | 828 | fuse_queue_forget(fm->fc, forget, outarg.nodeid, 1); |
| 828 | 829 | return -ENOMEM; |
| @@ -1585,13 +1586,13 @@ static int fuse_permission(struct mnt_idmap *idmap, · 756 unmodified lines | ||
| 1585 | 1586 | return err; |
| 1586 | 1587 | } |
| 1587 | 1588 | |
| 1588 | −static int fuse_readlink_page(struct inode *inode, struct page *page) | |
| 1589 | +static int fuse_readlink_page(struct inode *inode, struct folio *folio) | |
| 1589 | 1590 | { |
| 1590 | 1591 | struct fuse_mount *fm = get_fuse_mount(inode); |
| 1591 | − struct fuse_page_desc desc = { .length = PAGE_SIZE - 1 }; | |
| 1592 | + struct fuse_folio_desc desc = { .length = PAGE_SIZE - 1 }; | |
| 1592 | 1593 | struct fuse_args_pages ap = { |
| 1593 | − .num_pages = 1, | |
| 1594 | − .pages = &page, | |
| 1594 | + .num_folios = 1, | |
| 1595 | + .folios = &folio, | |
| 1595 | 1596 | .descs = &desc, |
| 1596 | 1597 | }; |
| 1597 | 1598 | char *link; |
| @@ -1614,7 +1615,7 @@ static int fuse_readlink_page(struct inode *inode, struct page *page) · 16 unmodified lines | ||
| 1614 | 1615 | if (WARN_ON(res >= PAGE_SIZE)) |
| 1615 | 1616 | return -EIO; |
| 1616 | 1617 | |
| 1617 | − link = page_address(page); | |
| 1618 | + link = folio_address(folio); | |
| 1618 | 1619 | link[res] = '\0'; |
| 1619 | 1620 | |
| 1620 | 1621 | return 0; |
| @@ -1624,7 +1625,7 @@ static const char *fuse_get_link(struct dentry *dentry, struct inode *inode, · 3 unmodified lines | ||
| 1624 | 1625 | struct delayed_call *callback) |
| 1625 | 1626 | { |
| 1626 | 1627 | struct fuse_conn *fc = get_fuse_conn(inode); |
| 1627 | − struct page *page; | |
| 1628 | + struct folio *folio; | |
| 1628 | 1629 | int err; |
| 1629 | 1630 | |
| 1630 | 1631 | err = -EIO; |
| @@ -1638,20 +1639,20 @@ static const char *fuse_get_link(struct dentry *dentry, struct inode *inode, · 7 unmodified lines | ||
| 1638 | 1639 | if (!dentry) |
| 1639 | 1640 | goto out_err; |
| 1640 | 1641 | |
| 1641 | − page = alloc_page(GFP_KERNEL); | |
| 1642 | + folio = folio_alloc(GFP_KERNEL, 0); | |
| 1642 | 1643 | err = -ENOMEM; |
| 1643 | − if (!page) | |
| 1644 | + if (!folio) | |
| 1644 | 1645 | goto out_err; |
| 1645 | 1646 | |
| 1646 | − err = fuse_readlink_page(inode, page); | |
| 1647 | + err = fuse_readlink_page(inode, folio); | |
| 1647 | 1648 | if (err) { |
| 1648 | − __free_page(page); | |
| 1649 | + folio_put(folio); | |
| 1649 | 1650 | goto out_err; |
| 1650 | 1651 | } |
| 1651 | 1652 | |
| 1652 | − set_delayed_call(callback, page_put_link, page); | |
| 1653 | + set_delayed_call(callback, page_put_link, &folio->page); | |
| 1653 | 1654 | |
| 1654 | − return page_address(page); | |
| 1655 | + return folio_address(folio); | |
| 1655 | 1656 | |
| 1656 | 1657 | out_err: |
| 1657 | 1658 | return ERR_PTR(err); |
| @@ -2028,7 +2029,7 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, · 370 unmodified lines | ||
| 2028 | 2029 | |
| 2029 | 2030 | fuse_change_attributes_common(inode, &outarg.attr, NULL, |
| 2030 | 2031 | ATTR_TIMEOUT(&outarg), |
| 2031 | − fuse_get_cache_mask(inode)); | |
| 2032 | + fuse_get_cache_mask(inode), 0); | |
| 2032 | 2033 | oldsize = inode->i_size; |
| 2033 | 2034 | /* see the comment in fuse_change_attributes() */ |
| 2034 | 2035 | if (!is_wb || is_truncate) |
| @@ -2231,7 +2232,7 @@ void fuse_init_dir(struct inode *inode) · 196 unmodified lines | ||
| 2231 | 2232 | |
| 2232 | 2233 | static int fuse_symlink_read_folio(struct file *null, struct folio *folio) |
| 2233 | 2234 | { |
| 2234 | − int err = fuse_readlink_page(folio->mapping->host, &folio->page); | |
| 2235 | + int err = fuse_readlink_page(folio->mapping->host, folio); | |
| 2235 | 2236 | |
| 2236 | 2237 | if (!err) |
| 2237 | 2238 | folio_mark_uptodate(folio); |
| @@ -436,7 +436,7 @@ static struct fuse_writepage_args *fuse_find_writeback(struct fuse_inode *fi, · 435 unmodified lines | ||
| 436 | 436 | wpa = rb_entry(n, struct fuse_writepage_args, writepages_entry); |
| 437 | 437 | WARN_ON(get_fuse_inode(wpa->inode) != fi); |
| 438 | 438 | curr_index = wpa->ia.write.in.offset >> PAGE_SHIFT; |
| 439 | − if (idx_from >= curr_index + wpa->ia.ap.num_pages) | |
| 439 | + if (idx_from >= curr_index + wpa->ia.ap.num_folios) | |
| 440 | 440 | n = n->rb_right; |
| 441 | 441 | else if (idx_to < curr_index) |
| 442 | 442 | n = n->rb_left; |
| @@ -483,6 +483,21 @@ static void fuse_wait_on_page_writeback(struct inode *inode, pgoff_t index) · 40 unmodified lines | ||
| 483 | 483 | wait_event(fi->page_waitq, !fuse_page_is_writeback(inode, index)); |
| 484 | 484 | } |
| 485 | 485 | |
| 486 | +static inline bool fuse_folio_is_writeback(struct inode *inode, | |
| 487 | + struct folio *folio) | |
| 488 | +{ | |
| 489 | + pgoff_t last = folio_next_index(folio) - 1; | |
| 490 | + return fuse_range_is_writeback(inode, folio_index(folio), last); | |
| 491 | +} | |
| 492 | + | |
| 493 | +static void fuse_wait_on_folio_writeback(struct inode *inode, | |
| 494 | + struct folio *folio) | |
| 495 | +{ | |
| 496 | + struct fuse_inode *fi = get_fuse_inode(inode); | |
| 497 | + | |
| 498 | + wait_event(fi->page_waitq, !fuse_folio_is_writeback(inode, folio)); | |
| 499 | +} | |
| 500 | + | |
| 486 | 501 | /* |
| 487 | 502 | * Wait for all pending writepages on the inode to finish. |
| 488 | 503 | * |
| @@ -645,17 +660,20 @@ void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos, · 156 unmodified lines | ||
| 645 | 660 | args->out_args[0].size = count; |
| 646 | 661 | } |
| 647 | 662 | |
| 648 | −static void fuse_release_user_pages(struct fuse_args_pages *ap, | |
| 663 | +static void fuse_release_user_pages(struct fuse_args_pages *ap, ssize_t nres, | |
| 649 | 664 | bool should_dirty) |
| 650 | 665 | { |
| 651 | 666 | unsigned int i; |
| 652 | 667 | |
| 653 | − for (i = 0; i < ap->num_pages; i++) { | |
| 668 | + for (i = 0; i < ap->num_folios; i++) { | |
| 654 | 669 | if (should_dirty) |
| 655 | − set_page_dirty_lock(ap->pages[i]); | |
| 670 | + folio_mark_dirty_lock(ap->folios[i]); | |
| 656 | 671 | if (ap->args.is_pinned) |
| 657 | − unpin_user_page(ap->pages[i]); | |
| 672 | + unpin_folio(ap->folios[i]); | |
| 658 | 673 | } |
| 674 | + | |
| 675 | + if (nres > 0 && ap->args.invalidate_vmap) | |
| 676 | + invalidate_kernel_vmap_range(ap->args.vmap_base, nres); | |
| 659 | 677 | } |
| 660 | 678 | |
| 661 | 679 | static void fuse_io_release(struct kref *kref) |
| @@ -725,16 +743,16 @@ static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos) · 63 unmodified lines | ||
| 725 | 743 | } |
| 726 | 744 | |
| 727 | 745 | static struct fuse_io_args *fuse_io_alloc(struct fuse_io_priv *io, |
| 728 | − unsigned int npages) | |
| 746 | + unsigned int nfolios) | |
| 729 | 747 | { |
| 730 | 748 | struct fuse_io_args *ia; |
| 731 | 749 | |
| 732 | 750 | ia = kzalloc(sizeof(*ia), GFP_KERNEL); |
| 733 | 751 | if (ia) { |
| 734 | 752 | ia->io = io; |
| 735 | − ia->ap.pages = fuse_pages_alloc(npages, GFP_KERNEL, | |
| 736 | − &ia->ap.descs); | |
| 737 | − if (!ia->ap.pages) { | |
| 753 | + ia->ap.folios = fuse_folios_alloc(nfolios, GFP_KERNEL, | |
| 754 | + &ia->ap.descs); | |
| 755 | + if (!ia->ap.folios) { | |
| 738 | 756 | kfree(ia); |
| 739 | 757 | ia = NULL; |
| 740 | 758 | } |
| @@ -744,7 +762,7 @@ static struct fuse_io_args *fuse_io_alloc(struct fuse_io_priv *io, · 3 unmodified lines | ||
| 744 | 762 | |
| 745 | 763 | static void fuse_io_free(struct fuse_io_args *ia) |
| 746 | 764 | { |
| 747 | − kfree(ia->ap.pages); | |
| 765 | + kfree(ia->ap.folios); | |
| 748 | 766 | kfree(ia); |
| 749 | 767 | } |
| 750 | 768 | |
| @@ -754,25 +772,29 @@ static void fuse_aio_complete_req(struct fuse_mount *fm, struct fuse_args *args, · 3 unmodified lines | ||
| 754 | 772 | struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args); |
| 755 | 773 | struct fuse_io_priv *io = ia->io; |
| 756 | 774 | ssize_t pos = -1; |
| 757 | − | |
| 758 | − fuse_release_user_pages(&ia->ap, io->should_dirty); | |
| 775 | + size_t nres; | |
| 759 | 776 | |
| 760 | 777 | if (err) { |
| 761 | 778 | /* Nothing */ |
| 762 | 779 | } else if (io->write) { |
| 763 | 780 | if (ia->write.out.size > ia->write.in.size) { |
| 764 | 781 | err = -EIO; |
| 765 | − } else if (ia->write.in.size != ia->write.out.size) { | |
| 766 | − pos = ia->write.in.offset - io->offset + | |
| 767 | − ia->write.out.size; | |
| 782 | + } else { | |
| 783 | + nres = ia->write.out.size; | |
| 784 | + if (ia->write.in.size != ia->write.out.size) | |
| 785 | + pos = ia->write.in.offset - io->offset + | |
| 786 | + ia->write.out.size; | |
| 768 | 787 | } |
| 769 | 788 | } else { |
| 770 | 789 | u32 outsize = args->out_args[0].size; |
| 771 | 790 | |
| 791 | + nres = outsize; | |
| 772 | 792 | if (ia->read.in.size != outsize) |
| 773 | 793 | pos = ia->read.in.offset - io->offset + outsize; |
| 774 | 794 | } |
| 775 | 795 | |
| 796 | + fuse_release_user_pages(&ia->ap, err ?: nres, io->should_dirty); | |
| 797 | + | |
| 776 | 798 | fuse_aio_complete(io, err, pos); |
| 777 | 799 | fuse_io_free(ia); |
| 778 | 800 | } |
| @@ -843,33 +865,33 @@ static void fuse_short_read(struct inode *inode, u64 attr_ver, size_t num_read, · 64 unmodified lines | ||
| 843 | 865 | * reached the client fs yet. So the hole is not present there. |
| 844 | 866 | */ |
| 845 | 867 | if (!fc->writeback_cache) { |
| 846 | − loff_t pos = page_offset(ap->pages[0]) + num_read; | |
| 868 | + loff_t pos = folio_pos(ap->folios[0]) + num_read; | |
| 847 | 869 | fuse_read_update_size(inode, pos, attr_ver); |
| 848 | 870 | } |
| 849 | 871 | } |
| 850 | 872 | |
| 851 | −static int fuse_do_readpage(struct file *file, struct page *page) | |
| 873 | +static int fuse_do_readfolio(struct file *file, struct folio *folio) | |
| 852 | 874 | { |
| 853 | − struct inode *inode = page->mapping->host; | |
| 875 | + struct inode *inode = folio->mapping->host; | |
| 854 | 876 | struct fuse_mount *fm = get_fuse_mount(inode); |
| 855 | − loff_t pos = page_offset(page); | |
| 856 | − struct fuse_page_desc desc = { .length = PAGE_SIZE }; | |
| 877 | + loff_t pos = folio_pos(folio); | |
| 878 | + struct fuse_folio_desc desc = { .length = PAGE_SIZE }; | |
| 857 | 879 | struct fuse_io_args ia = { |
| 858 | 880 | .ap.args.page_zeroing = true, |
| 859 | 881 | .ap.args.out_pages = true, |
| 860 | − .ap.num_pages = 1, | |
| 861 | − .ap.pages = &page, | |
| 882 | + .ap.num_folios = 1, | |
| 883 | + .ap.folios = &folio, | |
| 862 | 884 | .ap.descs = &desc, |
| 863 | 885 | }; |
| 864 | 886 | ssize_t res; |
| 865 | 887 | u64 attr_ver; |
| 866 | 888 | |
| 867 | 889 | /* |
| 868 | − * Page writeback can extend beyond the lifetime of the | |
| 869 | − * page-cache page, so make sure we read a properly synced | |
| 870 | − * page. | |
| 890 | + * With the temporary pages that are used to complete writeback, we can | |
| 891 | + * have writeback that extends beyond the lifetime of the folio. So | |
| 892 | + * make sure we read a properly synced folio. | |
| 871 | 893 | */ |
| 872 | − fuse_wait_on_page_writeback(inode, page->index); | |
| 894 | + fuse_wait_on_folio_writeback(inode, folio); | |
| 873 | 895 | |
| 874 | 896 | attr_ver = fuse_get_attr_version(fm->fc); |
| 875 | 897 | |
| @@ -887,25 +909,24 @@ static int fuse_do_readpage(struct file *file, struct page *page) · 11 unmodified lines | ||
| 887 | 909 | if (res < desc.length) |
| 888 | 910 | fuse_short_read(inode, attr_ver, res, &ia.ap); |
| 889 | 911 | |
| 890 | − SetPageUptodate(page); | |
| 912 | + folio_mark_uptodate(folio); | |
| 891 | 913 | |
| 892 | 914 | return 0; |
| 893 | 915 | } |
| 894 | 916 | |
| 895 | 917 | static int fuse_read_folio(struct file *file, struct folio *folio) |
| 896 | 918 | { |
| 897 | − struct page *page = &folio->page; | |
| 898 | − struct inode *inode = page->mapping->host; | |
| 919 | + struct inode *inode = folio->mapping->host; | |
| 899 | 920 | int err; |
| 900 | 921 | |
| 901 | 922 | err = -EIO; |
| 902 | 923 | if (fuse_is_bad(inode)) |
| 903 | 924 | goto out; |
| 904 | 925 | |
| 905 | − err = fuse_do_readpage(file, page); | |
| 926 | + err = fuse_do_readfolio(file, folio); | |
| 906 | 927 | fuse_invalidate_atime(inode); |
| 907 | 928 | out: |
| 908 | − unlock_page(page); | |
| 929 | + folio_unlock(folio); | |
| 909 | 930 | return err; |
| 910 | 931 | } |
| 911 | 932 | |
| @@ -919,8 +940,8 @@ static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args, · 7 unmodified lines | ||
| 919 | 940 | size_t num_read = args->out_args[0].size; |
| 920 | 941 | struct address_space *mapping = NULL; |
| 921 | 942 | |
| 922 | − for (i = 0; mapping == NULL && i < ap->num_pages; i++) | |
| 923 | − mapping = ap->pages[i]->mapping; | |
| 943 | + for (i = 0; mapping == NULL && i < ap->num_folios; i++) | |
| 944 | + mapping = ap->folios[i]->mapping; | |
| 924 | 945 | |
| 925 | 946 | if (mapping) { |
| 926 | 947 | struct inode *inode = mapping->host; |
| @@ -934,12 +955,8 @@ static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args, · 7 unmodified lines | ||
| 934 | 955 | fuse_invalidate_atime(inode); |
| 935 | 956 | } |
| 936 | 957 | |
| 937 | − for (i = 0; i < ap->num_pages; i++) { | |
| 938 | − struct folio *folio = page_folio(ap->pages[i]); | |
| 939 | − | |
| 940 | − folio_end_read(folio, !err); | |
| 941 | − folio_put(folio); | |
| 942 | − } | |
| 958 | + for (i = 0; i < ap->num_folios; i++) | |
| 959 | + folio_end_read(ap->folios[i], !err); | |
| 943 | 960 | if (ia->ff) |
| 944 | 961 | fuse_file_put(ia->ff, false); |
| 945 | 962 | |
| @@ -951,8 +968,9 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file) · 5 unmodified lines | ||
| 951 | 968 | struct fuse_file *ff = file->private_data; |
| 952 | 969 | struct fuse_mount *fm = ff->fm; |
| 953 | 970 | struct fuse_args_pages *ap = &ia->ap; |
| 954 | − loff_t pos = page_offset(ap->pages[0]); | |
| 955 | − size_t count = ap->num_pages << PAGE_SHIFT; | |
| 971 | + loff_t pos = folio_pos(ap->folios[0]); | |
| 972 | + /* Currently, all folios in FUSE are one page */ | |
| 973 | + size_t count = ap->num_folios << PAGE_SHIFT; | |
| 956 | 974 | ssize_t res; |
| 957 | 975 | int err; |
| 958 | 976 | |
| @@ -963,7 +981,7 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file) · 4 unmodified lines | ||
| 963 | 981 | /* Don't overflow end offset */ |
| 964 | 982 | if (pos + (count - 1) == LLONG_MAX) { |
| 965 | 983 | count--; |
| 966 | − ap->descs[ap->num_pages - 1].length--; | |
| 984 | + ap->descs[ap->num_folios - 1].length--; | |
| 967 | 985 | } |
| 968 | 986 | WARN_ON((loff_t) (pos + count) < 0); |
| 969 | 987 | |
| @@ -985,18 +1003,36 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file) · 15 unmodified lines | ||
| 985 | 1003 | static void fuse_readahead(struct readahead_control *rac) |
| 986 | 1004 | { |
| 987 | 1005 | struct inode *inode = rac->mapping->host; |
| 1006 | + struct fuse_inode *fi = get_fuse_inode(inode); | |
| 988 | 1007 | struct fuse_conn *fc = get_fuse_conn(inode); |
| 989 | − unsigned int i, max_pages, nr_pages = 0; | |
| 1008 | + unsigned int max_pages, nr_pages; | |
| 1009 | + pgoff_t first = readahead_index(rac); | |
| 1010 | + pgoff_t last = first + readahead_count(rac) - 1; | |
| 990 | 1011 | |
| 991 | 1012 | if (fuse_is_bad(inode)) |
| 992 | 1013 | return; |
| 993 | 1014 | |
| 1015 | + wait_event(fi->page_waitq, !fuse_range_is_writeback(inode, first, last)); | |
| 1016 | + | |
| 994 | 1017 | max_pages = min_t(unsigned int, fc->max_pages, |
| 995 | 1018 | fc->max_read / PAGE_SIZE); |
| 996 | 1019 | |
| 997 | − for (;;) { | |
| 1020 | + /* | |
| 1021 | + * This is only accurate the first time through, since readahead_folio() | |
| 1022 | + * doesn't update readahead_count() from the previous folio until the | |
| 1023 | + * next call. Grab nr_pages here so we know how many pages we're going | |
| 1024 | + * to have to process. This means that we will exit here with | |
| 1025 | + * readahead_count() == folio_nr_pages(last_folio), but we will have | |
| 1026 | + * consumed all of the folios, and read_pages() will call | |
| 1027 | + * readahead_folio() again which will clean up the rac. | |
| 1028 | + */ | |
| 1029 | + nr_pages = readahead_count(rac); | |
| 1030 | + | |
| 1031 | + while (nr_pages) { | |
| 998 | 1032 | struct fuse_io_args *ia; |
| 999 | 1033 | struct fuse_args_pages *ap; |
| 1034 | + struct folio *folio; | |
| 1035 | + unsigned cur_pages = min(max_pages, nr_pages); | |
| 1000 | 1036 | |
| 1001 | 1037 | if (fc->num_background >= fc->congestion_threshold && |
| 1002 | 1038 | rac->ra->async_size >= readahead_count(rac)) |
| @@ -1006,23 +1042,19 @@ static void fuse_readahead(struct readahead_control *rac) · 3 unmodified lines | ||
| 1006 | 1042 | */ |
| 1007 | 1043 | break; |
| 1008 | 1044 | |
| 1009 | − nr_pages = readahead_count(rac) - nr_pages; | |
| 1010 | − if (nr_pages > max_pages) | |
| 1011 | − nr_pages = max_pages; | |
| 1012 | − if (nr_pages == 0) | |
| 1013 | − break; | |
| 1014 | − ia = fuse_io_alloc(NULL, nr_pages); | |
| 1045 | + ia = fuse_io_alloc(NULL, cur_pages); | |
| 1015 | 1046 | if (!ia) |
| 1016 | 1047 | return; |
| 1017 | 1048 | ap = &ia->ap; |
| 1018 | − nr_pages = __readahead_batch(rac, ap->pages, nr_pages); | |
| 1019 | − for (i = 0; i < nr_pages; i++) { | |
| 1020 | − fuse_wait_on_page_writeback(inode, | |
| 1021 | − readahead_index(rac) + i); | |
| 1022 | − ap->descs[i].length = PAGE_SIZE; | |
| 1049 | + | |
| 1050 | + while (ap->num_folios < cur_pages) { | |
| 1051 | + folio = readahead_folio(rac); | |
| 1052 | + ap->folios[ap->num_folios] = folio; | |
| 1053 | + ap->descs[ap->num_folios].length = folio_size(folio); | |
| 1054 | + ap->num_folios++; | |
| 1023 | 1055 | } |
| 1024 | − ap->num_pages = nr_pages; | |
| 1025 | 1056 | fuse_send_readpages(ia, rac->file); |
| 1057 | + nr_pages -= cur_pages; | |
| 1026 | 1058 | } |
| 1027 | 1059 | } |
| 1028 | 1060 | |
| @@ -1139,8 +1171,8 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, · 110 unmodified lines | ||
| 1139 | 1171 | bool short_write; |
| 1140 | 1172 | int err; |
| 1141 | 1173 | |
| 1142 | − for (i = 0; i < ap->num_pages; i++) | |
| 1143 | − fuse_wait_on_page_writeback(inode, ap->pages[i]->index); | |
| 1174 | + for (i = 0; i < ap->num_folios; i++) | |
| 1175 | + fuse_wait_on_folio_writeback(inode, ap->folios[i]); | |
| 1144 | 1176 | |
| 1145 | 1177 | fuse_write_args_fill(ia, ff, pos, count); |
| 1146 | 1178 | ia->write.in.flags = fuse_write_flags(iocb); |
| @@ -1154,24 +1186,24 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, · 7 unmodified lines | ||
| 1154 | 1186 | short_write = ia->write.out.size < count; |
| 1155 | 1187 | offset = ap->descs[0].offset; |
| 1156 | 1188 | count = ia->write.out.size; |
| 1157 | − for (i = 0; i < ap->num_pages; i++) { | |
| 1158 | − struct page *page = ap->pages[i]; | |
| 1189 | + for (i = 0; i < ap->num_folios; i++) { | |
| 1190 | + struct folio *folio = ap->folios[i]; | |
| 1159 | 1191 | |
| 1160 | 1192 | if (err) { |
| 1161 | − ClearPageUptodate(page); | |
| 1193 | + folio_clear_uptodate(folio); | |
| 1162 | 1194 | } else { |
| 1163 | − if (count >= PAGE_SIZE - offset) | |
| 1164 | − count -= PAGE_SIZE - offset; | |
| 1195 | + if (count >= folio_size(folio) - offset) | |
| 1196 | + count -= folio_size(folio) - offset; | |
| 1165 | 1197 | else { |
| 1166 | 1198 | if (short_write) |
| 1167 | − ClearPageUptodate(page); | |
| 1199 | + folio_clear_uptodate(folio); | |
| 1168 | 1200 | count = 0; |
| 1169 | 1201 | } |
| 1170 | 1202 | offset = 0; |
| 1171 | 1203 | } |
| 1172 | − if (ia->write.page_locked && (i == ap->num_pages - 1)) | |
| 1173 | − unlock_page(page); | |
| 1174 | − put_page(page); | |
| 1204 | + if (ia->write.folio_locked && (i == ap->num_folios - 1)) | |
| 1205 | + folio_unlock(folio); | |
| 1206 | + folio_put(folio); | |
| 1175 | 1207 | } |
| 1176 | 1208 | |
| 1177 | 1209 | return err; |
| @@ -1185,6 +1217,7 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, · 7 unmodified lines | ||
| 1185 | 1217 | struct fuse_args_pages *ap = &ia->ap; |
| 1186 | 1218 | struct fuse_conn *fc = get_fuse_conn(mapping->host); |
| 1187 | 1219 | unsigned offset = pos & (PAGE_SIZE - 1); |
| 1220 | + unsigned int nr_pages = 0; | |
| 1188 | 1221 | size_t count = 0; |
| 1189 | 1222 | int err; |
| 1190 | 1223 | |
| @@ -1193,7 +1226,7 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, · 2 unmodified lines | ||
| 1193 | 1226 | |
| 1194 | 1227 | do { |
| 1195 | 1228 | size_t tmp; |
| 1196 | − struct page *page; | |
| 1229 | + struct folio *folio; | |
| 1197 | 1230 | pgoff_t index = pos >> PAGE_SHIFT; |
| 1198 | 1231 | size_t bytes = min_t(size_t, PAGE_SIZE - offset, |
| 1199 | 1232 | iov_iter_count(ii)); |
| @@ -1205,27 +1238,30 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, · 5 unmodified lines | ||
| 1205 | 1238 | if (fault_in_iov_iter_readable(ii, bytes)) |
| 1206 | 1239 | break; |
| 1207 | 1240 | |
| 1208 | − err = -ENOMEM; | |
| 1209 | − page = grab_cache_page_write_begin(mapping, index); | |
| 1210 | − if (!page) | |
| 1241 | + folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN, | |
| 1242 | + mapping_gfp_mask(mapping)); | |
| 1243 | + if (IS_ERR(folio)) { | |
| 1244 | + err = PTR_ERR(folio); | |
| 1211 | 1245 | break; |
| 1246 | + } | |
| 1212 | 1247 | |
| 1213 | 1248 | if (mapping_writably_mapped(mapping)) |
| 1214 | − flush_dcache_page(page); | |
| 1249 | + flush_dcache_folio(folio); | |
| 1215 | 1250 | |
| 1216 | − tmp = copy_page_from_iter_atomic(page, offset, bytes, ii); | |
| 1217 | − flush_dcache_page(page); | |
| 1251 | + tmp = copy_folio_from_iter_atomic(folio, offset, bytes, ii); | |
| 1252 | + flush_dcache_folio(folio); | |
| 1218 | 1253 | |
| 1219 | 1254 | if (!tmp) { |
| 1220 | − unlock_page(page); | |
| 1221 | − put_page(page); | |
| 1255 | + folio_unlock(folio); | |
| 1256 | + folio_put(folio); | |
| 1222 | 1257 | goto again; |
| 1223 | 1258 | } |
| 1224 | 1259 | |
| 1225 | 1260 | err = 0; |
| 1226 | − ap->pages[ap->num_pages] = page; | |
| 1227 | − ap->descs[ap->num_pages].length = tmp; | |
| 1228 | − ap->num_pages++; | |
| 1261 | + ap->folios[ap->num_folios] = folio; | |
| 1262 | + ap->descs[ap->num_folios].length = tmp; | |
| 1263 | + ap->num_folios++; | |
| 1264 | + nr_pages++; | |
| 1229 | 1265 | |
| 1230 | 1266 | count += tmp; |
| 1231 | 1267 | pos += tmp; |
| @@ -1235,18 +1271,18 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, · 3 unmodified lines | ||
| 1235 | 1271 | |
| 1236 | 1272 | /* If we copied full page, mark it uptodate */ |
| 1237 | 1273 | if (tmp == PAGE_SIZE) |
| 1238 | − SetPageUptodate(page); | |
| 1274 | + folio_mark_uptodate(folio); | |
| 1239 | 1275 | |
| 1240 | − if (PageUptodate(page)) { | |
| 1241 | − unlock_page(page); | |
| 1276 | + if (folio_test_uptodate(folio)) { | |
| 1277 | + folio_unlock(folio); | |
| 1242 | 1278 | } else { |
| 1243 | − ia->write.page_locked = true; | |
| 1279 | + ia->write.folio_locked = true; | |
| 1244 | 1280 | break; |
| 1245 | 1281 | } |
| 1246 | 1282 | if (!fc->big_writes) |
| 1247 | 1283 | break; |
| 1248 | 1284 | } while (iov_iter_count(ii) && count < fc->max_write && |
| 1249 | − ap->num_pages < max_pages && offset == 0); | |
| 1285 | + nr_pages < max_pages && offset == 0); | |
| 1250 | 1286 | |
| 1251 | 1287 | return count > 0 ? count : err; |
| 1252 | 1288 | } |
| @@ -1280,8 +1316,8 @@ static ssize_t fuse_perform_write(struct kiocb *iocb, struct iov_iter *ii) · 27 unmodified lines | ||
| 1280 | 1316 | unsigned int nr_pages = fuse_wr_pages(pos, iov_iter_count(ii), |
| 1281 | 1317 | fc->max_pages); |
| 1282 | 1318 | |
| 1283 | − ap->pages = fuse_pages_alloc(nr_pages, GFP_KERNEL, &ap->descs); | |
| 1284 | − if (!ap->pages) { | |
| 1319 | + ap->folios = fuse_folios_alloc(nr_pages, GFP_KERNEL, &ap->descs); | |
| 1320 | + if (!ap->folios) { | |
| 1285 | 1321 | err = -ENOMEM; |
| 1286 | 1322 | break; |
| 1287 | 1323 | } |
| @@ -1303,7 +1339,7 @@ static ssize_t fuse_perform_write(struct kiocb *iocb, struct iov_iter *ii) · 15 unmodified lines | ||
| 1303 | 1339 | err = -EIO; |
| 1304 | 1340 | } |
| 1305 | 1341 | } |
| 1306 | − kfree(ap->pages); | |
| 1342 | + kfree(ap->folios); | |
| 1307 | 1343 | } while (!err && iov_iter_count(ii)); |
| 1308 | 1344 | |
| 1309 | 1345 | fuse_write_update_attr(inode, pos, res); |
| @@ -1430,11 +1466,7 @@ writethrough: · 120 unmodified lines | ||
| 1430 | 1466 | |
| 1431 | 1467 | task_io_account_write(count); |
| 1432 | 1468 | |
| 1433 | − err = file_remove_privs(file); | |
| 1434 | − if (err) | |
| 1435 | − goto out; | |
| 1436 | − | |
| 1437 | − err = file_update_time(file); | |
| 1469 | + err = kiocb_modified(iocb); | |
| 1438 | 1470 | if (err) |
| 1439 | 1471 | goto out; |
| 1440 | 1472 | |
| @@ -1468,35 +1500,57 @@ static inline size_t fuse_get_frag_size(const struct iov_iter *ii, · 27 unmodified lines | ||
| 1468 | 1500 | |
| 1469 | 1501 | static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii, |
| 1470 | 1502 | size_t *nbytesp, int write, |
| 1471 | − unsigned int max_pages) | |
| 1503 | + unsigned int max_pages, | |
| 1504 | + bool use_pages_for_kvec_io) | |
| 1472 | 1505 | { |
| 1506 | + bool flush_or_invalidate = false; | |
| 1507 | + unsigned int nr_pages = 0; | |
| 1473 | 1508 | size_t nbytes = 0; /* # bytes already packed in req */ |
| 1474 | 1509 | ssize_t ret = 0; |
| 1475 | 1510 | |
| 1476 | − /* Special case for kernel I/O: can copy directly into the buffer */ | |
| 1511 | + /* Special case for kernel I/O: can copy directly into the buffer. | |
| 1512 | + * However if the implementation of fuse_conn requires pages instead of | |
| 1513 | + * pointer (e.g., virtio-fs), use iov_iter_extract_pages() instead. | |
| 1514 | + */ | |
| 1477 | 1515 | if (iov_iter_is_kvec(ii)) { |
| 1478 | − unsigned long user_addr = fuse_get_user_addr(ii); | |
| 1479 | − size_t frag_size = fuse_get_frag_size(ii, *nbytesp); | |
| 1516 | + void *user_addr = (void *)fuse_get_user_addr(ii); | |
| 1480 | 1517 | |
| 1481 | − if (write) | |
| 1482 | − ap->args.in_args[1].value = (void *) user_addr; | |
| 1483 | − else | |
| 1484 | − ap->args.out_args[0].value = (void *) user_addr; | |
| 1518 | + if (!use_pages_for_kvec_io) { | |
| 1519 | + size_t frag_size = fuse_get_frag_size(ii, *nbytesp); | |
| 1485 | 1520 | |
| 1486 | − iov_iter_advance(ii, frag_size); | |
| 1487 | − *nbytesp = frag_size; | |
| 1488 | − return 0; | |
| 1521 | + if (write) | |
| 1522 | + ap->args.in_args[1].value = user_addr; | |
| 1523 | + else | |
| 1524 | + ap->args.out_args[0].value = user_addr; | |
| 1525 | + | |
| 1526 | + iov_iter_advance(ii, frag_size); | |
| 1527 | + *nbytesp = frag_size; | |
| 1528 | + return 0; | |
| 1529 | + } | |
| 1530 | + | |
| 1531 | + if (is_vmalloc_addr(user_addr)) { | |
| 1532 | + ap->args.vmap_base = user_addr; | |
| 1533 | + flush_or_invalidate = true; | |
| 1534 | + } | |
| 1489 | 1535 | } |
| 1490 | 1536 | |
| 1491 | − while (nbytes < *nbytesp && ap->num_pages < max_pages) { | |
| 1492 | − unsigned npages; | |
| 1537 | + /* | |
| 1538 | + * Until there is support for iov_iter_extract_folios(), we have to | |
| 1539 | + * manually extract pages using iov_iter_extract_pages() and then | |
| 1540 | + * copy that to a folios array. | |
| 1541 | + */ | |
| 1542 | + struct page **pages = kzalloc(max_pages * sizeof(struct page *), | |
| 1543 | + GFP_KERNEL); | |
| 1544 | + if (!pages) | |
| 1545 | + return -ENOMEM; | |
| 1546 | + | |
| 1547 | + while (nbytes < *nbytesp && nr_pages < max_pages) { | |
| 1548 | + unsigned nfolios, i; | |
| 1493 | 1549 | size_t start; |
| 1494 | − struct page **pt_pages; | |
| 1495 | 1550 | |
| 1496 | − pt_pages = &ap->pages[ap->num_pages]; | |
| 1497 | − ret = iov_iter_extract_pages(ii, &pt_pages, | |
| 1551 | + ret = iov_iter_extract_pages(ii, &pages, | |
| 1498 | 1552 | *nbytesp - nbytes, |
| 1499 | − max_pages - ap->num_pages, | |
| 1553 | + max_pages - nr_pages, | |
| 1500 | 1554 | 0, &start); |
| 1501 | 1555 | if (ret < 0) |
| 1502 | 1556 | break; |
| @@ -1504,16 +1558,25 @@ static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii, · 1 unmodified line | ||
| 1504 | 1558 | nbytes += ret; |
| 1505 | 1559 | |
| 1506 | 1560 | ret += start; |
| 1507 | − npages = DIV_ROUND_UP(ret, PAGE_SIZE); | |
| 1561 | + /* Currently, all folios in FUSE are one page */ | |
| 1562 | + nfolios = DIV_ROUND_UP(ret, PAGE_SIZE); | |
| 1508 | 1563 | |
| 1509 | − ap->descs[ap->num_pages].offset = start; | |
| 1510 | − fuse_page_descs_length_init(ap->descs, ap->num_pages, npages); | |
| 1564 | + ap->descs[ap->num_folios].offset = start; | |
| 1565 | + fuse_folio_descs_length_init(ap->descs, ap->num_folios, nfolios); | |
| 1566 | + for (i = 0; i < nfolios; i++) | |
| 1567 | + ap->folios[i + ap->num_folios] = page_folio(pages[i]); | |
| 1511 | 1568 | |
| 1512 | − ap->num_pages += npages; | |
| 1513 | − ap->descs[ap->num_pages - 1].length -= | |
| 1569 | + ap->num_folios += nfolios; | |
| 1570 | + ap->descs[ap->num_folios - 1].length -= | |
| 1514 | 1571 | (PAGE_SIZE - ret) & (PAGE_SIZE - 1); |
| 1572 | + nr_pages += nfolios; | |
| 1515 | 1573 | } |
| 1574 | + kfree(pages); | |
| 1575 | + | |
| 1576 | + if (write && flush_or_invalidate) | |
| 1577 | + flush_kernel_vmap_range(ap->args.vmap_base, nbytes); | |
| 1516 | 1578 | |
| 1579 | + ap->args.invalidate_vmap = !write && flush_or_invalidate; | |
| 1517 | 1580 | ap->args.is_pinned = iov_iter_extract_will_pin(ii); |
| 1518 | 1581 | ap->args.user_pages = true; |
| 1519 | 1582 | if (write) |
| @@ -1582,7 +1645,7 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, · 62 unmodified lines | ||
| 1582 | 1645 | size_t nbytes = min(count, nmax); |
| 1583 | 1646 | |
| 1584 | 1647 | err = fuse_get_user_pages(&ia->ap, iter, &nbytes, write, |
| 1585 | − max_pages); | |
| 1648 | + max_pages, fc->use_pages_for_kvec_io); | |
| 1586 | 1649 | if (err && !nbytes) |
| 1587 | 1650 | break; |
| 1588 | 1651 | |
| @@ -1596,7 +1659,7 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, · 7 unmodified lines | ||
| 1596 | 1659 | } |
| 1597 | 1660 | |
| 1598 | 1661 | if (!io->async || nres < 0) { |
| 1599 | − fuse_release_user_pages(&ia->ap, io->should_dirty); | |
| 1662 | + fuse_release_user_pages(&ia->ap, nres, io->should_dirty); | |
| 1600 | 1663 | fuse_io_free(ia); |
| 1601 | 1664 | } |
| 1602 | 1665 | ia = NULL; |
| @@ -1650,7 +1713,7 @@ static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to) · 47 unmodified lines | ||
| 1650 | 1713 | { |
| 1651 | 1714 | ssize_t res; |
| 1652 | 1715 | |
| 1653 | − if (!is_sync_kiocb(iocb) && iocb->ki_flags & IOCB_DIRECT) { | |
| 1716 | + if (!is_sync_kiocb(iocb)) { | |
| 1654 | 1717 | res = fuse_direct_IO(iocb, to); |
| 1655 | 1718 | } else { |
| 1656 | 1719 | struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); |
| @@ -1664,7 +1727,6 @@ static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to) · 7 unmodified lines | ||
| 1664 | 1727 | static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) |
| 1665 | 1728 | { |
| 1666 | 1729 | struct inode *inode = file_inode(iocb->ki_filp); |
| 1667 | − struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); | |
| 1668 | 1730 | ssize_t res; |
| 1669 | 1731 | bool exclusive; |
| 1670 | 1732 | |
| @@ -1672,9 +1734,11 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) · 1 unmodified line | ||
| 1672 | 1734 | res = generic_write_checks(iocb, from); |
| 1673 | 1735 | if (res > 0) { |
| 1674 | 1736 | task_io_account_write(res); |
| 1675 | − if (!is_sync_kiocb(iocb) && iocb->ki_flags & IOCB_DIRECT) { | |
| 1737 | + if (!is_sync_kiocb(iocb)) { | |
| 1676 | 1738 | res = fuse_direct_IO(iocb, from); |
| 1677 | 1739 | } else { |
| 1740 | + struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); | |
| 1741 | + | |
| 1678 | 1742 | res = fuse_direct_io(&io, from, &iocb->ki_pos, |
| 1679 | 1743 | FUSE_DIO_WRITE); |
| 1680 | 1744 | fuse_write_update_attr(inode, iocb->ki_pos, res); |
| @@ -1760,21 +1824,21 @@ static void fuse_writepage_free(struct fuse_writepage_args *wpa) · 79 unmodified lines | ||
| 1760 | 1824 | if (wpa->bucket) |
| 1761 | 1825 | fuse_sync_bucket_dec(wpa->bucket); |
| 1762 | 1826 | |
| 1763 | − for (i = 0; i < ap->num_pages; i++) | |
| 1764 | − __free_page(ap->pages[i]); | |
| 1827 | + for (i = 0; i < ap->num_folios; i++) | |
| 1828 | + folio_put(ap->folios[i]); | |
| 1765 | 1829 | |
| 1766 | 1830 | fuse_file_put(wpa->ia.ff, false); |
| 1767 | 1831 | |
| 1768 | − kfree(ap->pages); | |
| 1832 | + kfree(ap->folios); | |
| 1769 | 1833 | kfree(wpa); |
| 1770 | 1834 | } |
| 1771 | 1835 | |
| 1772 | −static void fuse_writepage_finish_stat(struct inode *inode, struct page *page) | |
| 1836 | +static void fuse_writepage_finish_stat(struct inode *inode, struct folio *folio) | |
| 1773 | 1837 | { |
| 1774 | 1838 | struct backing_dev_info *bdi = inode_to_bdi(inode); |
| 1775 | 1839 | |
| 1776 | 1840 | dec_wb_stat(&bdi->wb, WB_WRITEBACK); |
| 1777 | − dec_node_page_state(page, NR_WRITEBACK_TEMP); | |
| 1841 | + node_stat_sub_folio(folio, NR_WRITEBACK_TEMP); | |
| 1778 | 1842 | wb_writeout_inc(&bdi->wb); |
| 1779 | 1843 | } |
| 1780 | 1844 | |
| @@ -1785,8 +1849,8 @@ static void fuse_writepage_finish(struct fuse_writepage_args *wpa) · 4 unmodified lines | ||
| 1785 | 1849 | struct fuse_inode *fi = get_fuse_inode(inode); |
| 1786 | 1850 | int i; |
| 1787 | 1851 | |
| 1788 | − for (i = 0; i < ap->num_pages; i++) | |
| 1789 | − fuse_writepage_finish_stat(inode, ap->pages[i]); | |
| 1852 | + for (i = 0; i < ap->num_folios; i++) | |
| 1853 | + fuse_writepage_finish_stat(inode, ap->folios[i]); | |
| 1790 | 1854 | |
| 1791 | 1855 | wake_up(&fi->page_waitq); |
| 1792 | 1856 | } |
| @@ -1801,7 +1865,8 @@ __acquires(fi->lock) · 8 unmodified lines | ||
| 1801 | 1865 | struct fuse_inode *fi = get_fuse_inode(wpa->inode); |
| 1802 | 1866 | struct fuse_write_in *inarg = &wpa->ia.write.in; |
| 1803 | 1867 | struct fuse_args *args = &wpa->ia.ap.args; |
| 1804 | − __u64 data_size = wpa->ia.ap.num_pages * PAGE_SIZE; | |
| 1868 | + /* Currently, all folios in FUSE are one page */ | |
| 1869 | + __u64 data_size = wpa->ia.ap.num_folios * PAGE_SIZE; | |
| 1805 | 1870 | int err; |
| 1806 | 1871 | |
| 1807 | 1872 | fi->writectr++; |
| @@ -1841,7 +1906,8 @@ __acquires(fi->lock) · 33 unmodified lines | ||
| 1841 | 1906 | for (aux = wpa->next; aux; aux = next) { |
| 1842 | 1907 | next = aux->next; |
| 1843 | 1908 | aux->next = NULL; |
| 1844 | − fuse_writepage_finish_stat(aux->inode, aux->ia.ap.pages[0]); | |
| 1909 | + fuse_writepage_finish_stat(aux->inode, | |
| 1910 | + aux->ia.ap.folios[0]); | |
| 1845 | 1911 | fuse_writepage_free(aux); |
| 1846 | 1912 | } |
| 1847 | 1913 | |
| @@ -1876,11 +1942,11 @@ static struct fuse_writepage_args *fuse_insert_writeback(struct rb_root *root, · 28 unmodified lines | ||
| 1876 | 1942 | struct fuse_writepage_args *wpa) |
| 1877 | 1943 | { |
| 1878 | 1944 | pgoff_t idx_from = wpa->ia.write.in.offset >> PAGE_SHIFT; |
| 1879 | − pgoff_t idx_to = idx_from + wpa->ia.ap.num_pages - 1; | |
| 1945 | + pgoff_t idx_to = idx_from + wpa->ia.ap.num_folios - 1; | |
| 1880 | 1946 | struct rb_node **p = &root->rb_node; |
| 1881 | 1947 | struct rb_node *parent = NULL; |
| 1882 | 1948 | |
| 1883 | − WARN_ON(!wpa->ia.ap.num_pages); | |
| 1949 | + WARN_ON(!wpa->ia.ap.num_folios); | |
| 1884 | 1950 | while (*p) { |
| 1885 | 1951 | struct fuse_writepage_args *curr; |
| 1886 | 1952 | pgoff_t curr_index; |
| @@ -1891,7 +1957,7 @@ static struct fuse_writepage_args *fuse_insert_writeback(struct rb_root *root, · 4 unmodified lines | ||
| 1891 | 1957 | WARN_ON(curr->inode != wpa->inode); |
| 1892 | 1958 | curr_index = curr->ia.write.in.offset >> PAGE_SHIFT; |
| 1893 | 1959 | |
| 1894 | − if (idx_from >= curr_index + curr->ia.ap.num_pages) | |
| 1960 | + if (idx_from >= curr_index + curr->ia.ap.num_folios) | |
| 1895 | 1961 | p = &(*p)->rb_right; |
| 1896 | 1962 | else if (idx_to < curr_index) |
| 1897 | 1963 | p = &(*p)->rb_left; |
| @@ -2023,9 +2089,9 @@ static struct fuse_writepage_args *fuse_writepage_args_alloc(void) · 125 unmodified lines | ||
| 2023 | 2089 | wpa = kzalloc(sizeof(*wpa), GFP_NOFS); |
| 2024 | 2090 | if (wpa) { |
| 2025 | 2091 | ap = &wpa->ia.ap; |
| 2026 | − ap->num_pages = 0; | |
| 2027 | − ap->pages = fuse_pages_alloc(1, GFP_NOFS, &ap->descs); | |
| 2028 | − if (!ap->pages) { | |
| 2092 | + ap->num_folios = 0; | |
| 2093 | + ap->folios = fuse_folios_alloc(1, GFP_NOFS, &ap->descs); | |
| 2094 | + if (!ap->folios) { | |
| 2029 | 2095 | kfree(wpa); |
| 2030 | 2096 | wpa = NULL; |
| 2031 | 2097 | } |
| @@ -2049,19 +2115,19 @@ static void fuse_writepage_add_to_bucket(struct fuse_conn *fc, · 17 unmodified lines | ||
| 2049 | 2115 | } |
| 2050 | 2116 | |
| 2051 | 2117 | static void fuse_writepage_args_page_fill(struct fuse_writepage_args *wpa, struct folio *folio, |
| 2052 | − struct folio *tmp_folio, uint32_t page_index) | |
| 2118 | + struct folio *tmp_folio, uint32_t folio_index) | |
| 2053 | 2119 | { |
| 2054 | 2120 | struct inode *inode = folio->mapping->host; |
| 2055 | 2121 | struct fuse_args_pages *ap = &wpa->ia.ap; |
| 2056 | 2122 | |
| 2057 | 2123 | folio_copy(tmp_folio, folio); |
| 2058 | 2124 | |
| 2059 | − ap->pages[page_index] = &tmp_folio->page; | |
| 2060 | − ap->descs[page_index].offset = 0; | |
| 2061 | − ap->descs[page_index].length = PAGE_SIZE; | |
| 2125 | + ap->folios[folio_index] = tmp_folio; | |
| 2126 | + ap->descs[folio_index].offset = 0; | |
| 2127 | + ap->descs[folio_index].length = PAGE_SIZE; | |
| 2062 | 2128 | |
| 2063 | 2129 | inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK); |
| 2064 | − inc_node_page_state(&tmp_folio->page, NR_WRITEBACK_TEMP); | |
| 2130 | + node_stat_add_folio(tmp_folio, NR_WRITEBACK_TEMP); | |
| 2065 | 2131 | } |
| 2066 | 2132 | |
| 2067 | 2133 | static struct fuse_writepage_args *fuse_writepage_args_setup(struct folio *folio, |
| @@ -2115,7 +2181,7 @@ static int fuse_writepage_locked(struct folio *folio) · 47 unmodified lines | ||
| 2115 | 2181 | goto err_writepage_args; |
| 2116 | 2182 | |
| 2117 | 2183 | ap = &wpa->ia.ap; |
| 2118 | − ap->num_pages = 1; | |
| 2184 | + ap->num_folios = 1; | |
| 2119 | 2185 | |
| 2120 | 2186 | folio_start_writeback(folio); |
| 2121 | 2187 | fuse_writepage_args_page_fill(wpa, folio, tmp_folio, 0); |
| @@ -2143,32 +2209,32 @@ struct fuse_fill_wb_data { · 21 unmodified lines | ||
| 2143 | 2209 | struct fuse_writepage_args *wpa; |
| 2144 | 2210 | struct fuse_file *ff; |
| 2145 | 2211 | struct inode *inode; |
| 2146 | − struct page **orig_pages; | |
| 2147 | − unsigned int max_pages; | |
| 2212 | + struct folio **orig_folios; | |
| 2213 | + unsigned int max_folios; | |
| 2148 | 2214 | }; |
| 2149 | 2215 | |
| 2150 | 2216 | static bool fuse_pages_realloc(struct fuse_fill_wb_data *data) |
| 2151 | 2217 | { |
| 2152 | 2218 | struct fuse_args_pages *ap = &data->wpa->ia.ap; |
| 2153 | 2219 | struct fuse_conn *fc = get_fuse_conn(data->inode); |
| 2154 | − struct page **pages; | |
| 2155 | − struct fuse_page_desc *descs; | |
| 2156 | − unsigned int npages = min_t(unsigned int, | |
| 2157 | − max_t(unsigned int, data->max_pages * 2, | |
| 2158 | − FUSE_DEFAULT_MAX_PAGES_PER_REQ), | |
| 2220 | + struct folio **folios; | |
| 2221 | + struct fuse_folio_desc *descs; | |
| 2222 | + unsigned int nfolios = min_t(unsigned int, | |
| 2223 | + max_t(unsigned int, data->max_folios * 2, | |
| 2224 | + FUSE_DEFAULT_MAX_PAGES_PER_REQ), | |
| 2159 | 2225 | fc->max_pages); |
| 2160 | − WARN_ON(npages <= data->max_pages); | |
| 2226 | + WARN_ON(nfolios <= data->max_folios); | |
| 2161 | 2227 | |
| 2162 | − pages = fuse_pages_alloc(npages, GFP_NOFS, &descs); | |
| 2163 | − if (!pages) | |
| 2228 | + folios = fuse_folios_alloc(nfolios, GFP_NOFS, &descs); | |
| 2229 | + if (!folios) | |
| 2164 | 2230 | return false; |
| 2165 | 2231 | |
| 2166 | − memcpy(pages, ap->pages, sizeof(struct page *) * ap->num_pages); | |
| 2167 | − memcpy(descs, ap->descs, sizeof(struct fuse_page_desc) * ap->num_pages); | |
| 2168 | − kfree(ap->pages); | |
| 2169 | − ap->pages = pages; | |
| 2232 | + memcpy(folios, ap->folios, sizeof(struct folio *) * ap->num_folios); | |
| 2233 | + memcpy(descs, ap->descs, sizeof(struct fuse_folio_desc) * ap->num_folios); | |
| 2234 | + kfree(ap->folios); | |
| 2235 | + ap->folios = folios; | |
| 2170 | 2236 | ap->descs = descs; |
| 2171 | − data->max_pages = npages; | |
| 2237 | + data->max_folios = nfolios; | |
| 2172 | 2238 | |
| 2173 | 2239 | return true; |
| 2174 | 2240 | } |
| @@ -2178,7 +2244,7 @@ static void fuse_writepages_send(struct fuse_fill_wb_data *data) · 3 unmodified lines | ||
| 2178 | 2244 | struct fuse_writepage_args *wpa = data->wpa; |
| 2179 | 2245 | struct inode *inode = data->inode; |
| 2180 | 2246 | struct fuse_inode *fi = get_fuse_inode(inode); |
| 2181 | − int num_pages = wpa->ia.ap.num_pages; | |
| 2247 | + int num_folios = wpa->ia.ap.num_folios; | |
| 2182 | 2248 | int i; |
| 2183 | 2249 | |
| 2184 | 2250 | spin_lock(&fi->lock); |
| @@ -2186,8 +2252,8 @@ static void fuse_writepages_send(struct fuse_fill_wb_data *data) · 1 unmodified line | ||
| 2186 | 2252 | fuse_flush_writepages(inode); |
| 2187 | 2253 | spin_unlock(&fi->lock); |
| 2188 | 2254 | |
| 2189 | − for (i = 0; i < num_pages; i++) | |
| 2190 | − end_page_writeback(data->orig_pages[i]); | |
| 2255 | + for (i = 0; i < num_folios; i++) | |
| 2256 | + folio_end_writeback(data->orig_folios[i]); | |
| 2191 | 2257 | } |
| 2192 | 2258 | |
| 2193 | 2259 | /* |
| @@ -2198,15 +2264,15 @@ static void fuse_writepages_send(struct fuse_fill_wb_data *data) · 4 unmodified lines | ||
| 2198 | 2264 | * swapping the new temp page with the old one. |
| 2199 | 2265 | */ |
| 2200 | 2266 | static bool fuse_writepage_add(struct fuse_writepage_args *new_wpa, |
| 2201 | − struct page *page) | |
| 2267 | + struct folio *folio) | |
| 2202 | 2268 | { |
| 2203 | 2269 | struct fuse_inode *fi = get_fuse_inode(new_wpa->inode); |
| 2204 | 2270 | struct fuse_writepage_args *tmp; |
| 2205 | 2271 | struct fuse_writepage_args *old_wpa; |
| 2206 | 2272 | struct fuse_args_pages *new_ap = &new_wpa->ia.ap; |
| 2207 | 2273 | |
| 2208 | − WARN_ON(new_ap->num_pages != 0); | |
| 2209 | − new_ap->num_pages = 1; | |
| 2274 | + WARN_ON(new_ap->num_folios != 0); | |
| 2275 | + new_ap->num_folios = 1; | |
| 2210 | 2276 | |
| 2211 | 2277 | spin_lock(&fi->lock); |
| 2212 | 2278 | old_wpa = fuse_insert_writeback(&fi->writepages, new_wpa); |
| @@ -2220,9 +2286,9 @@ static bool fuse_writepage_add(struct fuse_writepage_args *new_wpa, · 7 unmodified lines | ||
| 2220 | 2286 | |
| 2221 | 2287 | WARN_ON(tmp->inode != new_wpa->inode); |
| 2222 | 2288 | curr_index = tmp->ia.write.in.offset >> PAGE_SHIFT; |
| 2223 | − if (curr_index == page->index) { | |
| 2224 | − WARN_ON(tmp->ia.ap.num_pages != 1); | |
| 2225 | − swap(tmp->ia.ap.pages[0], new_ap->pages[0]); | |
| 2289 | + if (curr_index == folio->index) { | |
| 2290 | + WARN_ON(tmp->ia.ap.num_folios != 1); | |
| 2291 | + swap(tmp->ia.ap.folios[0], new_ap->folios[0]); | |
| 2226 | 2292 | break; |
| 2227 | 2293 | } |
| 2228 | 2294 | } |
| @@ -2235,18 +2301,19 @@ static bool fuse_writepage_add(struct fuse_writepage_args *new_wpa, · 6 unmodified lines | ||
| 2235 | 2301 | spin_unlock(&fi->lock); |
| 2236 | 2302 | |
| 2237 | 2303 | if (tmp) { |
| 2238 | − fuse_writepage_finish_stat(new_wpa->inode, new_ap->pages[0]); | |
| 2304 | + fuse_writepage_finish_stat(new_wpa->inode, | |
| 2305 | + folio); | |
| 2239 | 2306 | fuse_writepage_free(new_wpa); |
| 2240 | 2307 | } |
| 2241 | 2308 | |
| 2242 | 2309 | return false; |
| 2243 | 2310 | } |
| 2244 | 2311 | |
| 2245 | −static bool fuse_writepage_need_send(struct fuse_conn *fc, struct page *page, | |
| 2312 | +static bool fuse_writepage_need_send(struct fuse_conn *fc, struct folio *folio, | |
| 2246 | 2313 | struct fuse_args_pages *ap, |
| 2247 | 2314 | struct fuse_fill_wb_data *data) |
| 2248 | 2315 | { |
| 2249 | − WARN_ON(!ap->num_pages); | |
| 2316 | + WARN_ON(!ap->num_folios); | |
| 2250 | 2317 | |
| 2251 | 2318 | /* |
| 2252 | 2319 | * Being under writeback is unlikely but possible. For example direct |
| @@ -2254,23 +2321,23 @@ static bool fuse_writepage_need_send(struct fuse_conn *fc, struct page *page, · 1 unmodified line | ||
| 2254 | 2321 | * the pages are faulted with get_user_pages(), and then after the read |
| 2255 | 2322 | * completed. |
| 2256 | 2323 | */ |
| 2257 | − if (fuse_page_is_writeback(data->inode, page->index)) | |
| 2324 | + if (fuse_folio_is_writeback(data->inode, folio)) | |
| 2258 | 2325 | return true; |
| 2259 | 2326 | |
| 2260 | 2327 | /* Reached max pages */ |
| 2261 | − if (ap->num_pages == fc->max_pages) | |
| 2328 | + if (ap->num_folios == fc->max_pages) | |
| 2262 | 2329 | return true; |
| 2263 | 2330 | |
| 2264 | 2331 | /* Reached max write bytes */ |
| 2265 | − if ((ap->num_pages + 1) * PAGE_SIZE > fc->max_write) | |
| 2332 | + if ((ap->num_folios + 1) * PAGE_SIZE > fc->max_write) | |
| 2266 | 2333 | return true; |
| 2267 | 2334 | |
| 2268 | 2335 | /* Discontinuity */ |
| 2269 | − if (data->orig_pages[ap->num_pages - 1]->index + 1 != page->index) | |
| 2336 | + if (data->orig_folios[ap->num_folios - 1]->index + 1 != folio_index(folio)) | |
| 2270 | 2337 | return true; |
| 2271 | 2338 | |
| 2272 | 2339 | /* Need to grow the pages array? If so, did the expansion fail? */ |
| 2273 | − if (ap->num_pages == data->max_pages && !fuse_pages_realloc(data)) | |
| 2340 | + if (ap->num_folios == data->max_folios && !fuse_pages_realloc(data)) | |
| 2274 | 2341 | return true; |
| 2275 | 2342 | |
| 2276 | 2343 | return false; |
| @@ -2295,7 +2362,7 @@ static int fuse_writepages_fill(struct folio *folio, · 18 unmodified lines | ||
| 2295 | 2362 | goto out_unlock; |
| 2296 | 2363 | } |
| 2297 | 2364 | |
| 2298 | − if (wpa && fuse_writepage_need_send(fc, &folio->page, ap, data)) { | |
| 2365 | + if (wpa && fuse_writepage_need_send(fc, folio, ap, data)) { | |
| 2299 | 2366 | fuse_writepages_send(data); |
| 2300 | 2367 | data->wpa = NULL; |
| 2301 | 2368 | } |
| @@ -2314,7 +2381,7 @@ static int fuse_writepages_fill(struct folio *folio, · 12 unmodified lines | ||
| 2314 | 2381 | * This is ensured by holding the page lock in page_mkwrite() while |
| 2315 | 2382 | * checking fuse_page_is_writeback(). We already hold the page lock |
| 2316 | 2383 | * since clear_page_dirty_for_io() and keep it held until we add the |
| 2317 | − * request to the fi->writepages list and increment ap->num_pages. | |
| 2384 | + * request to the fi->writepages list and increment ap->num_folios. | |
| 2318 | 2385 | * After this fuse_page_is_writeback() will indicate that the page is |
| 2319 | 2386 | * under writeback, so we can release the page lock. |
| 2320 | 2387 | */ |
| @@ -2326,13 +2393,13 @@ static int fuse_writepages_fill(struct folio *folio, · 5 unmodified lines | ||
| 2326 | 2393 | goto out_unlock; |
| 2327 | 2394 | } |
| 2328 | 2395 | fuse_file_get(wpa->ia.ff); |
| 2329 | − data->max_pages = 1; | |
| 2396 | + data->max_folios = 1; | |
| 2330 | 2397 | ap = &wpa->ia.ap; |
| 2331 | 2398 | } |
| 2332 | 2399 | folio_start_writeback(folio); |
| 2333 | 2400 | |
| 2334 | − fuse_writepage_args_page_fill(wpa, folio, tmp_folio, ap->num_pages); | |
| 2335 | − data->orig_pages[ap->num_pages] = &folio->page; | |
| 2401 | + fuse_writepage_args_page_fill(wpa, folio, tmp_folio, ap->num_folios); | |
| 2402 | + data->orig_folios[ap->num_folios] = folio; | |
| 2336 | 2403 | |
| 2337 | 2404 | err = 0; |
| 2338 | 2405 | if (data->wpa) { |
| @@ -2341,9 +2408,9 @@ static int fuse_writepages_fill(struct folio *folio, · 2 unmodified lines | ||
| 2341 | 2408 | * fuse_page_is_writeback(). |
| 2342 | 2409 | */ |
| 2343 | 2410 | spin_lock(&fi->lock); |
| 2344 | − ap->num_pages++; | |
| 2411 | + ap->num_folios++; | |
| 2345 | 2412 | spin_unlock(&fi->lock); |
| 2346 | − } else if (fuse_writepage_add(wpa, &folio->page)) { | |
| 2413 | + } else if (fuse_writepage_add(wpa, folio)) { | |
| 2347 | 2414 | data->wpa = wpa; |
| 2348 | 2415 | } else { |
| 2349 | 2416 | folio_end_writeback(folio); |
| @@ -2375,21 +2442,21 @@ static int fuse_writepages(struct address_space *mapping, · 25 unmodified lines | ||
| 2375 | 2442 | data.ff = NULL; |
| 2376 | 2443 | |
| 2377 | 2444 | err = -ENOMEM; |
| 2378 | − data.orig_pages = kcalloc(fc->max_pages, | |
| 2379 | − sizeof(struct page *), | |
| 2380 | − GFP_NOFS); | |
| 2381 | − if (!data.orig_pages) | |
| 2445 | + data.orig_folios = kcalloc(fc->max_pages, | |
| 2446 | + sizeof(struct folio *), | |
| 2447 | + GFP_NOFS); | |
| 2448 | + if (!data.orig_folios) | |
| 2382 | 2449 | goto out; |
| 2383 | 2450 | |
| 2384 | 2451 | err = write_cache_pages(mapping, wbc, fuse_writepages_fill, &data); |
| 2385 | 2452 | if (data.wpa) { |
| 2386 | − WARN_ON(!data.wpa->ia.ap.num_pages); | |
| 2453 | + WARN_ON(!data.wpa->ia.ap.num_folios); | |
| 2387 | 2454 | fuse_writepages_send(&data); |
| 2388 | 2455 | } |
| 2389 | 2456 | if (data.ff) |
| 2390 | 2457 | fuse_file_put(data.ff, false); |
| 2391 | 2458 | |
| 2392 | − kfree(data.orig_pages); | |
| 2459 | + kfree(data.orig_folios); | |
| 2393 | 2460 | out: |
| 2394 | 2461 | return err; |
| 2395 | 2462 | } |
| @@ -2429,7 +2496,7 @@ static int fuse_write_begin(struct file *file, struct address_space *mapping, · 33 unmodified lines | ||
| 2429 | 2496 | folio_zero_segment(folio, 0, off); |
| 2430 | 2497 | goto success; |
| 2431 | 2498 | } |
| 2432 | − err = fuse_do_readpage(file, &folio->page); | |
| 2499 | + err = fuse_do_readfolio(file, folio); | |
| 2433 | 2500 | if (err) |
| 2434 | 2501 | goto cleanup; |
| 2435 | 2502 | success: |
| @@ -2518,17 +2585,17 @@ static void fuse_vma_close(struct vm_area_struct *vma) · 82 unmodified lines | ||
| 2518 | 2585 | */ |
| 2519 | 2586 | static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) |
| 2520 | 2587 | { |
| 2521 | − struct page *page = vmf->page; | |
| 2588 | + struct folio *folio = page_folio(vmf->page); | |
| 2522 | 2589 | struct inode *inode = file_inode(vmf->vma->vm_file); |
| 2523 | 2590 | |
| 2524 | 2591 | file_update_time(vmf->vma->vm_file); |
| 2525 | − lock_page(page); | |
| 2526 | − if (page->mapping != inode->i_mapping) { | |
| 2527 | − unlock_page(page); | |
| 2592 | + folio_lock(folio); | |
| 2593 | + if (folio->mapping != inode->i_mapping) { | |
| 2594 | + folio_unlock(folio); | |
| 2528 | 2595 | return VM_FAULT_NOPAGE; |
| 2529 | 2596 | } |
| 2530 | 2597 | |
| 2531 | − fuse_wait_on_page_writeback(inode, page->index); | |
| 2598 | + fuse_wait_on_folio_writeback(inode, folio); | |
| 2532 | 2599 | return VM_FAULT_LOCKED; |
| 2533 | 2600 | } |
| 2534 | 2601 | |
| @@ -35,9 +35,6 @@ · 34 unmodified lines | ||
| 35 | 35 | /** Default max number of pages that can be used in a single read request */ |
| 36 | 36 | #define FUSE_DEFAULT_MAX_PAGES_PER_REQ 32 |
| 37 | 37 | |
| 38 | −/** Maximum of max_pages received in init_out */ | |
| 39 | −#define FUSE_MAX_MAX_PAGES 256 | |
| 40 | − | |
| 41 | 38 | /** Bias for fi->writectr, meaning new writepages must not be sent */ |
| 42 | 39 | #define FUSE_NOWRITE INT_MIN |
| 43 | 40 | |
| @@ -47,6 +44,9 @@ · 3 unmodified lines | ||
| 47 | 44 | /** Number of dentries for each connection in the control filesystem */ |
| 48 | 45 | #define FUSE_CTL_NUM_DENTRIES 5 |
| 49 | 46 | |
| 47 | +/** Maximum of max_pages received in init_out */ | |
| 48 | +extern unsigned int fuse_max_pages_limit; | |
| 49 | + | |
| 50 | 50 | /** List of active connections */ |
| 51 | 51 | extern struct list_head fuse_conn_list; |
| 52 | 52 | |
| @@ -285,8 +285,8 @@ struct fuse_arg { · 232 unmodified lines | ||
| 285 | 285 | void *value; |
| 286 | 286 | }; |
| 287 | 287 | |
| 288 | −/** FUSE page descriptor */ | |
| 289 | −struct fuse_page_desc { | |
| 288 | +/** FUSE folio descriptor */ | |
| 289 | +struct fuse_folio_desc { | |
| 290 | 290 | unsigned int length; |
| 291 | 291 | unsigned int offset; |
| 292 | 292 | }; |
| @@ -309,16 +309,19 @@ struct fuse_args { · 16 unmodified lines | ||
| 309 | 309 | bool may_block:1; |
| 310 | 310 | bool is_ext:1; |
| 311 | 311 | bool is_pinned:1; |
| 312 | + bool invalidate_vmap:1; | |
| 312 | 313 | struct fuse_in_arg in_args[3]; |
| 313 | 314 | struct fuse_arg out_args[2]; |
| 314 | 315 | void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error); |
| 316 | + /* Used for kvec iter backed by vmalloc address */ | |
| 317 | + void *vmap_base; | |
| 315 | 318 | }; |
| 316 | 319 | |
| 317 | 320 | struct fuse_args_pages { |
| 318 | 321 | struct fuse_args args; |
| 319 | − struct page **pages; | |
| 320 | − struct fuse_page_desc *descs; | |
| 321 | − unsigned int num_pages; | |
| 322 | + struct folio **folios; | |
| 323 | + struct fuse_folio_desc *descs; | |
| 324 | + unsigned int num_folios; | |
| 322 | 325 | }; |
| 323 | 326 | |
| 324 | 327 | struct fuse_release_args { |
| @@ -857,6 +860,9 @@ struct fuse_conn { · 532 unmodified lines | ||
| 857 | 860 | /** Passthrough support for read/write IO */ |
| 858 | 861 | unsigned int passthrough:1; |
| 859 | 862 | |
| 863 | + /* Use pages instead of pointer for kernel I/O */ | |
| 864 | + unsigned int use_pages_for_kvec_io:1; | |
| 865 | + | |
| 860 | 866 | /** Maximum stack depth for passthrough backing files */ |
| 861 | 867 | int max_stack_depth; |
| 862 | 868 | |
| @@ -884,6 +890,9 @@ struct fuse_conn { · 21 unmodified lines | ||
| 884 | 890 | /** Version counter for attribute changes */ |
| 885 | 891 | atomic64_t attr_version; |
| 886 | 892 | |
| 893 | + /** Version counter for evict inode */ | |
| 894 | + atomic64_t evict_ctr; | |
| 895 | + | |
| 887 | 896 | /** Called on final put */ |
| 888 | 897 | void (*release)(struct fuse_conn *); |
| 889 | 898 | |
| @@ -978,6 +987,11 @@ static inline u64 fuse_get_attr_version(struct fuse_conn *fc) · 88 unmodified lines | ||
| 978 | 987 | return atomic64_read(&fc->attr_version); |
| 979 | 988 | } |
| 980 | 989 | |
| 990 | +static inline u64 fuse_get_evict_ctr(struct fuse_conn *fc) | |
| 991 | +{ | |
| 992 | + return atomic64_read(&fc->evict_ctr); | |
| 993 | +} | |
| 994 | + | |
| 981 | 995 | static inline bool fuse_stale_inode(const struct inode *inode, int generation, |
| 982 | 996 | struct fuse_attr *attr) |
| 983 | 997 | { |
| @@ -995,25 +1009,25 @@ static inline bool fuse_is_bad(struct inode *inode) · 11 unmodified lines | ||
| 995 | 1009 | return unlikely(test_bit(FUSE_I_BAD, &get_fuse_inode(inode)->state)); |
| 996 | 1010 | } |
| 997 | 1011 | |
| 998 | −static inline struct page **fuse_pages_alloc(unsigned int npages, gfp_t flags, | |
| 999 | − struct fuse_page_desc **desc) | |
| 1012 | +static inline struct folio **fuse_folios_alloc(unsigned int nfolios, gfp_t flags, | |
| 1013 | + struct fuse_folio_desc **desc) | |
| 1000 | 1014 | { |
| 1001 | − struct page **pages; | |
| 1015 | + struct folio **folios; | |
| 1002 | 1016 | |
| 1003 | − pages = kzalloc(npages * (sizeof(struct page *) + | |
| 1004 | − sizeof(struct fuse_page_desc)), flags); | |
| 1005 | − *desc = (void *) (pages + npages); | |
| 1017 | + folios = kzalloc(nfolios * (sizeof(struct folio *) + | |
| 1018 | + sizeof(struct fuse_folio_desc)), flags); | |
| 1019 | + *desc = (void *) (folios + nfolios); | |
| 1006 | 1020 | |
| 1007 | − return pages; | |
| 1021 | + return folios; | |
| 1008 | 1022 | } |
| 1009 | 1023 | |
| 1010 | −static inline void fuse_page_descs_length_init(struct fuse_page_desc *descs, | |
| 1011 | − unsigned int index, | |
| 1012 | − unsigned int nr_pages) | |
| 1024 | +static inline void fuse_folio_descs_length_init(struct fuse_folio_desc *descs, | |
| 1025 | + unsigned int index, | |
| 1026 | + unsigned int nr_folios) | |
| 1013 | 1027 | { |
| 1014 | 1028 | int i; |
| 1015 | 1029 | |
| 1016 | − for (i = index; i < index + nr_pages; i++) | |
| 1030 | + for (i = index; i < index + nr_folios; i++) | |
| 1017 | 1031 | descs[i].length = PAGE_SIZE - descs[i].offset; |
| 1018 | 1032 | } |
| 1019 | 1033 | |
| @@ -1037,7 +1051,8 @@ extern const struct dentry_operations fuse_root_dentry_operations; · 17 unmodified lines | ||
| 1037 | 1051 | */ |
| 1038 | 1052 | struct inode *fuse_iget(struct super_block *sb, u64 nodeid, |
| 1039 | 1053 | int generation, struct fuse_attr *attr, |
| 1040 | − u64 attr_valid, u64 attr_version); | |
| 1054 | + u64 attr_valid, u64 attr_version, | |
| 1055 | + u64 evict_ctr); | |
| 1041 | 1056 | |
| 1042 | 1057 | int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name, |
| 1043 | 1058 | struct fuse_entry_out *outarg, struct inode **inode); |
| @@ -1062,7 +1077,7 @@ struct fuse_io_args { · 18 unmodified lines | ||
| 1062 | 1077 | struct { |
| 1063 | 1078 | struct fuse_write_in in; |
| 1064 | 1079 | struct fuse_write_out out; |
| 1065 | − bool page_locked; | |
| 1080 | + bool folio_locked; | |
| 1066 | 1081 | } write; |
| 1067 | 1082 | }; |
| 1068 | 1083 | struct fuse_args_pages ap; |
| @@ -1127,7 +1142,8 @@ void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr, · 58 unmodified lines | ||
| 1127 | 1142 | |
| 1128 | 1143 | void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr, |
| 1129 | 1144 | struct fuse_statx *sx, |
| 1130 | − u64 attr_valid, u32 cache_mask); | |
| 1145 | + u64 attr_valid, u32 cache_mask, | |
| 1146 | + u64 evict_ctr); | |
| 1131 | 1147 | |
| 1132 | 1148 | u32 fuse_get_cache_mask(struct inode *inode); |
| 1133 | 1149 | |
| @@ -1480,4 +1496,12 @@ ssize_t fuse_passthrough_splice_write(struct pipe_inode_info *pipe, · 346 unmodified lines | ||
| 1480 | 1496 | size_t len, unsigned int flags); |
| 1481 | 1497 | ssize_t fuse_passthrough_mmap(struct file *file, struct vm_area_struct *vma); |
| 1482 | 1498 | |
| 1499 | +#ifdef CONFIG_SYSCTL | |
| 1500 | +extern int fuse_sysctl_register(void); | |
| 1501 | +extern void fuse_sysctl_unregister(void); | |
| 1502 | +#else | |
| 1503 | +#define fuse_sysctl_register() (0) | |
| 1504 | +#define fuse_sysctl_unregister() do { } while (0) | |
| 1505 | +#endif /* CONFIG_SYSCTL */ | |
| 1506 | + | |
| 1483 | 1507 | #endif /* _FS_FUSE_I_H */ |
| @@ -35,6 +35,8 @@ DEFINE_MUTEX(fuse_mutex); · 34 unmodified lines | ||
| 35 | 35 | |
| 36 | 36 | static int set_global_limit(const char *val, const struct kernel_param *kp); |
| 37 | 37 | |
| 38 | +unsigned int fuse_max_pages_limit = 256; | |
| 39 | + | |
| 38 | 40 | unsigned max_user_bgreq; |
| 39 | 41 | module_param_call(max_user_bgreq, set_global_limit, param_get_uint, |
| 40 | 42 | &max_user_bgreq, 0644); |
| @@ -173,6 +175,14 @@ static void fuse_evict_inode(struct inode *inode) · 132 unmodified lines | ||
| 173 | 175 | fuse_cleanup_submount_lookup(fc, fi->submount_lookup); |
| 174 | 176 | fi->submount_lookup = NULL; |
| 175 | 177 | } |
| 178 | + /* | |
| 179 | + * Evict of non-deleted inode may race with outstanding | |
| 180 | + * LOOKUP/READDIRPLUS requests and result in inconsistency when | |
| 181 | + * the request finishes. Deal with that here by bumping a | |
| 182 | + * counter that can be compared to the starting value. | |
| 183 | + */ | |
| 184 | + if (inode->i_nlink > 0) | |
| 185 | + atomic64_inc(&fc->evict_ctr); | |
| 176 | 186 | } |
| 177 | 187 | if (S_ISREG(inode->i_mode) && !fuse_is_bad(inode)) { |
| 178 | 188 | WARN_ON(fi->iocachectr != 0); |
| @@ -206,17 +216,30 @@ static ino_t fuse_squash_ino(u64 ino64) · 27 unmodified lines | ||
| 206 | 216 | |
| 207 | 217 | void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr, |
| 208 | 218 | struct fuse_statx *sx, |
| 209 | − u64 attr_valid, u32 cache_mask) | |
| 219 | + u64 attr_valid, u32 cache_mask, | |
| 220 | + u64 evict_ctr) | |
| 210 | 221 | { |
| 211 | 222 | struct fuse_conn *fc = get_fuse_conn(inode); |
| 212 | 223 | struct fuse_inode *fi = get_fuse_inode(inode); |
| 213 | 224 | |
| 214 | 225 | lockdep_assert_held(&fi->lock); |
| 215 | 226 | |
| 227 | + /* | |
| 228 | + * Clear basic stats from invalid mask. | |
| 229 | + * | |
| 230 | + * Don't do this if this is coming from a fuse_iget() call and there | |
| 231 | + * might have been a racing evict which would've invalidated the result | |
| 232 | + * if the attr_version would've been preserved. | |
| 233 | + * | |
| 234 | + * !evict_ctr -> this is create | |
| 235 | + * fi->attr_version != 0 -> this is not a new inode | |
| 236 | + * evict_ctr == fuse_get_evict_ctr() -> no evicts while during request | |
| 237 | + */ | |
| 238 | + if (!evict_ctr || fi->attr_version || evict_ctr == fuse_get_evict_ctr(fc)) | |
| 239 | + set_mask_bits(&fi->inval_mask, STATX_BASIC_STATS, 0); | |
| 240 | + | |
| 216 | 241 | fi->attr_version = atomic64_inc_return(&fc->attr_version); |
| 217 | 242 | fi->i_time = attr_valid; |
| 218 | − /* Clear basic stats from invalid mask */ | |
| 219 | − set_mask_bits(&fi->inval_mask, STATX_BASIC_STATS, 0); | |
| 220 | 243 | |
| 221 | 244 | inode->i_ino = fuse_squash_ino(attr->ino); |
| 222 | 245 | inode->i_mode = (inode->i_mode & S_IFMT) | (attr->mode & 07777); |
| @@ -295,9 +318,9 @@ u32 fuse_get_cache_mask(struct inode *inode) · 72 unmodified lines | ||
| 295 | 318 | return STATX_MTIME | STATX_CTIME | STATX_SIZE; |
| 296 | 319 | } |
| 297 | 320 | |
| 298 | −void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr, | |
| 299 | − struct fuse_statx *sx, | |
| 300 | − u64 attr_valid, u64 attr_version) | |
| 321 | +static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr, | |
| 322 | + struct fuse_statx *sx, u64 attr_valid, | |
| 323 | + u64 attr_version, u64 evict_ctr) | |
| 301 | 324 | { |
| 302 | 325 | struct fuse_conn *fc = get_fuse_conn(inode); |
| 303 | 326 | struct fuse_inode *fi = get_fuse_inode(inode); |
| @@ -331,7 +354,8 @@ void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr, · 27 unmodified lines | ||
| 331 | 354 | } |
| 332 | 355 | |
| 333 | 356 | old_mtime = inode_get_mtime(inode); |
| 334 | − fuse_change_attributes_common(inode, attr, sx, attr_valid, cache_mask); | |
| 357 | + fuse_change_attributes_common(inode, attr, sx, attr_valid, cache_mask, | |
| 358 | + evict_ctr); | |
| 335 | 359 | |
| 336 | 360 | oldsize = inode->i_size; |
| 337 | 361 | /* |
| @@ -372,6 +396,13 @@ void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr, · 34 unmodified lines | ||
| 372 | 396 | fuse_dax_dontcache(inode, attr->flags); |
| 373 | 397 | } |
| 374 | 398 | |
| 399 | +void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr, | |
| 400 | + struct fuse_statx *sx, u64 attr_valid, | |
| 401 | + u64 attr_version) | |
| 402 | +{ | |
| 403 | + fuse_change_attributes_i(inode, attr, sx, attr_valid, attr_version, 0); | |
| 404 | +} | |
| 405 | + | |
| 375 | 406 | static void fuse_init_submount_lookup(struct fuse_submount_lookup *sl, |
| 376 | 407 | u64 nodeid) |
| 377 | 408 | { |
| @@ -426,7 +457,8 @@ static int fuse_inode_set(struct inode *inode, void *_nodeidp) · 48 unmodified lines | ||
| 426 | 457 | |
| 427 | 458 | struct inode *fuse_iget(struct super_block *sb, u64 nodeid, |
| 428 | 459 | int generation, struct fuse_attr *attr, |
| 429 | − u64 attr_valid, u64 attr_version) | |
| 460 | + u64 attr_valid, u64 attr_version, | |
| 461 | + u64 evict_ctr) | |
| 430 | 462 | { |
| 431 | 463 | struct inode *inode; |
| 432 | 464 | struct fuse_inode *fi; |
| @@ -487,8 +519,8 @@ retry: · 54 unmodified lines | ||
| 487 | 519 | fi->nlookup++; |
| 488 | 520 | spin_unlock(&fi->lock); |
| 489 | 521 | done: |
| 490 | − fuse_change_attributes(inode, attr, NULL, attr_valid, attr_version); | |
| 491 | − | |
| 522 | + fuse_change_attributes_i(inode, attr, NULL, attr_valid, attr_version, | |
| 523 | + evict_ctr); | |
| 492 | 524 | return inode; |
| 493 | 525 | } |
| 494 | 526 | |
| @@ -940,11 +972,12 @@ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm, · 445 unmodified lines | ||
| 940 | 972 | fc->initialized = 0; |
| 941 | 973 | fc->connected = 1; |
| 942 | 974 | atomic64_set(&fc->attr_version, 1); |
| 975 | + atomic64_set(&fc->evict_ctr, 1); | |
| 943 | 976 | get_random_bytes(&fc->scramble_key, sizeof(fc->scramble_key)); |
| 944 | 977 | fc->pid_ns = get_pid_ns(task_active_pid_ns(current)); |
| 945 | 978 | fc->user_ns = get_user_ns(user_ns); |
| 946 | 979 | fc->max_pages = FUSE_DEFAULT_MAX_PAGES_PER_REQ; |
| 947 | − fc->max_pages_limit = FUSE_MAX_MAX_PAGES; | |
| 980 | + fc->max_pages_limit = fuse_max_pages_limit; | |
| 948 | 981 | |
| 949 | 982 | if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH)) |
| 950 | 983 | fuse_backing_files_init(fc); |
| @@ -1001,7 +1034,7 @@ static struct inode *fuse_get_root_inode(struct super_block *sb, unsigned mode) · 50 unmodified lines | ||
| 1001 | 1034 | attr.mode = mode; |
| 1002 | 1035 | attr.ino = FUSE_ROOT_ID; |
| 1003 | 1036 | attr.nlink = 1; |
| 1004 | − return fuse_iget(sb, FUSE_ROOT_ID, 0, &attr, 0, 0); | |
| 1037 | + return fuse_iget(sb, FUSE_ROOT_ID, 0, &attr, 0, 0, 0); | |
| 1005 | 1038 | } |
| 1006 | 1039 | |
| 1007 | 1040 | struct fuse_inode_handle { |
| @@ -1610,7 +1643,8 @@ static int fuse_fill_super_submount(struct super_block *sb, · 602 unmodified lines | ||
| 1610 | 1643 | return -ENOMEM; |
| 1611 | 1644 | |
| 1612 | 1645 | fuse_fill_attr_from_inode(&root_attr, parent_fi); |
| 1613 | − root = fuse_iget(sb, parent_fi->nodeid, 0, &root_attr, 0, 0); | |
| 1646 | + root = fuse_iget(sb, parent_fi->nodeid, 0, &root_attr, 0, 0, | |
| 1647 | + fuse_get_evict_ctr(fm->fc)); | |
| 1614 | 1648 | /* |
| 1615 | 1649 | * This inode is just a duplicate, so it is not looked up and |
| 1616 | 1650 | * its nlookup should not be incremented. fuse_iget() does |
| @@ -2063,8 +2097,14 @@ static int __init fuse_fs_init(void) · 446 unmodified lines | ||
| 2063 | 2097 | if (err) |
| 2064 | 2098 | goto out3; |
| 2065 | 2099 | |
| 2100 | + err = fuse_sysctl_register(); | |
| 2101 | + if (err) | |
| 2102 | + goto out4; | |
| 2103 | + | |
| 2066 | 2104 | return 0; |
| 2067 | 2105 | |
| 2106 | + out4: | |
| 2107 | + unregister_filesystem(&fuse_fs_type); | |
| 2068 | 2108 | out3: |
| 2069 | 2109 | unregister_fuseblk(); |
| 2070 | 2110 | out2: |
| @@ -2075,6 +2115,7 @@ static int __init fuse_fs_init(void) · 4 unmodified lines | ||
| 2075 | 2115 | |
| 2076 | 2116 | static void fuse_fs_cleanup(void) |
| 2077 | 2117 | { |
| 2118 | + fuse_sysctl_unregister(); | |
| 2078 | 2119 | unregister_filesystem(&fuse_fs_type); |
| 2079 | 2120 | unregister_fuseblk(); |
| 2080 | 2121 | |
| @@ -10,6 +10,8 @@ · 9 unmodified lines | ||
| 10 | 10 | #include <linux/fileattr.h> |
| 11 | 11 | #include <linux/fsverity.h> |
| 12 | 12 | |
| 13 | +#define FUSE_VERITY_ENABLE_ARG_MAX_PAGES 256 | |
| 14 | + | |
| 13 | 15 | static ssize_t fuse_send_ioctl(struct fuse_mount *fm, struct fuse_args *args, |
| 14 | 16 | struct fuse_ioctl_out *outarg) |
| 15 | 17 | { |
| @@ -140,7 +142,7 @@ static int fuse_setup_enable_verity(unsigned long arg, struct iovec *iov, · 124 unmodified lines | ||
| 140 | 142 | { |
| 141 | 143 | struct fsverity_enable_arg enable; |
| 142 | 144 | struct fsverity_enable_arg __user *uarg = (void __user *)arg; |
| 143 | − const __u32 max_buffer_len = FUSE_MAX_MAX_PAGES * PAGE_SIZE; | |
| 145 | + const __u32 max_buffer_len = FUSE_VERITY_ENABLE_ARG_MAX_PAGES * PAGE_SIZE; | |
| 144 | 146 | |
| 145 | 147 | if (copy_from_user(&enable, uarg, sizeof(enable))) |
| 146 | 148 | return -EFAULT; |
| @@ -249,12 +251,12 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, · 102 unmodified lines | ||
| 249 | 251 | BUILD_BUG_ON(sizeof(struct fuse_ioctl_iovec) * FUSE_IOCTL_MAX_IOV > PAGE_SIZE); |
| 250 | 252 | |
| 251 | 253 | err = -ENOMEM; |
| 252 | − ap.pages = fuse_pages_alloc(fm->fc->max_pages, GFP_KERNEL, &ap.descs); | |
| 254 | + ap.folios = fuse_folios_alloc(fm->fc->max_pages, GFP_KERNEL, &ap.descs); | |
| 253 | 255 | iov_page = (struct iovec *) __get_free_page(GFP_KERNEL); |
| 254 | − if (!ap.pages || !iov_page) | |
| 256 | + if (!ap.folios || !iov_page) | |
| 255 | 257 | goto out; |
| 256 | 258 | |
| 257 | − fuse_page_descs_length_init(ap.descs, 0, fm->fc->max_pages); | |
| 259 | + fuse_folio_descs_length_init(ap.descs, 0, fm->fc->max_pages); | |
| 258 | 260 | |
| 259 | 261 | /* |
| 260 | 262 | * If restricted, initialize IO parameters as encoded in @cmd. |
| @@ -304,14 +306,13 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, · 43 unmodified lines | ||
| 304 | 306 | err = -ENOMEM; |
| 305 | 307 | if (max_pages > fm->fc->max_pages) |
| 306 | 308 | goto out; |
| 307 | − while (ap.num_pages < max_pages) { | |
| 308 | − ap.pages[ap.num_pages] = alloc_page(GFP_KERNEL | __GFP_HIGHMEM); | |
| 309 | − if (!ap.pages[ap.num_pages]) | |
| 309 | + while (ap.num_folios < max_pages) { | |
| 310 | + ap.folios[ap.num_folios] = folio_alloc(GFP_KERNEL | __GFP_HIGHMEM, 0); | |
| 311 | + if (!ap.folios[ap.num_folios]) | |
| 310 | 312 | goto out; |
| 311 | − ap.num_pages++; | |
| 313 | + ap.num_folios++; | |
| 312 | 314 | } |
| 313 | 315 | |
| 314 | − | |
| 315 | 316 | /* okay, let's send it to the client */ |
| 316 | 317 | ap.args.opcode = FUSE_IOCTL; |
| 317 | 318 | ap.args.nodeid = ff->nodeid; |
| @@ -325,8 +326,8 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, · 7 unmodified lines | ||
| 325 | 326 | |
| 326 | 327 | err = -EFAULT; |
| 327 | 328 | iov_iter_init(&ii, ITER_SOURCE, in_iov, in_iovs, in_size); |
| 328 | − for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= ap.num_pages); i++) { | |
| 329 | − c = copy_page_from_iter(ap.pages[i], 0, PAGE_SIZE, &ii); | |
| 329 | + for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= ap.num_folios); i++) { | |
| 330 | + c = copy_folio_from_iter(ap.folios[i], 0, PAGE_SIZE, &ii); | |
| 330 | 331 | if (c != PAGE_SIZE && iov_iter_count(&ii)) |
| 331 | 332 | goto out; |
| 332 | 333 | } |
| @@ -364,7 +365,7 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, · 31 unmodified lines | ||
| 364 | 365 | in_iovs + out_iovs > FUSE_IOCTL_MAX_IOV) |
| 365 | 366 | goto out; |
| 366 | 367 | |
| 367 | − vaddr = kmap_local_page(ap.pages[0]); | |
| 368 | + vaddr = kmap_local_folio(ap.folios[0], 0); | |
| 368 | 369 | err = fuse_copy_ioctl_iovec(fm->fc, iov_page, vaddr, |
| 369 | 370 | transferred, in_iovs + out_iovs, |
| 370 | 371 | (flags & FUSE_IOCTL_COMPAT) != 0); |
| @@ -392,17 +393,17 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, · 21 unmodified lines | ||
| 392 | 393 | |
| 393 | 394 | err = -EFAULT; |
| 394 | 395 | iov_iter_init(&ii, ITER_DEST, out_iov, out_iovs, transferred); |
| 395 | − for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= ap.num_pages); i++) { | |
| 396 | − c = copy_page_to_iter(ap.pages[i], 0, PAGE_SIZE, &ii); | |
| 396 | + for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= ap.num_folios); i++) { | |
| 397 | + c = copy_folio_to_iter(ap.folios[i], 0, PAGE_SIZE, &ii); | |
| 397 | 398 | if (c != PAGE_SIZE && iov_iter_count(&ii)) |
| 398 | 399 | goto out; |
| 399 | 400 | } |
| 400 | 401 | err = 0; |
| 401 | 402 | out: |
| 402 | 403 | free_page((unsigned long) iov_page); |
| 403 | − while (ap.num_pages) | |
| 404 | − __free_page(ap.pages[--ap.num_pages]); | |
| 405 | − kfree(ap.pages); | |
| 404 | + while (ap.num_folios) | |
| 405 | + folio_put(ap.folios[--ap.num_folios]); | |
| 406 | + kfree(ap.folios); | |
| 406 | 407 | |
| 407 | 408 | return err ? err : outarg.result; |
| 408 | 409 | } |
| @@ -149,7 +149,7 @@ static int parse_dirfile(char *buf, size_t nbytes, struct file *file, · 148 unmodified lines | ||
| 149 | 149 | |
| 150 | 150 | static int fuse_direntplus_link(struct file *file, |
| 151 | 151 | struct fuse_direntplus *direntplus, |
| 152 | − u64 attr_version) | |
| 152 | + u64 attr_version, u64 evict_ctr) | |
| 153 | 153 | { |
| 154 | 154 | struct fuse_entry_out *o = &direntplus->entry_out; |
| 155 | 155 | struct fuse_dirent *dirent = &direntplus->dirent; |
| @@ -233,7 +233,7 @@ retry: · 77 unmodified lines | ||
| 233 | 233 | } else { |
| 234 | 234 | inode = fuse_iget(dir->i_sb, o->nodeid, o->generation, |
| 235 | 235 | &o->attr, ATTR_TIMEOUT(o), |
| 236 | − attr_version); | |
| 236 | + attr_version, evict_ctr); | |
| 237 | 237 | if (!inode) |
| 238 | 238 | inode = ERR_PTR(-ENOMEM); |
| 239 | 239 | |
| @@ -284,7 +284,8 @@ static void fuse_force_forget(struct file *file, u64 nodeid) · 44 unmodified lines | ||
| 284 | 284 | } |
| 285 | 285 | |
| 286 | 286 | static int parse_dirplusfile(char *buf, size_t nbytes, struct file *file, |
| 287 | − struct dir_context *ctx, u64 attr_version) | |
| 287 | + struct dir_context *ctx, u64 attr_version, | |
| 288 | + u64 evict_ctr) | |
| 288 | 289 | { |
| 289 | 290 | struct fuse_direntplus *direntplus; |
| 290 | 291 | struct fuse_dirent *dirent; |
| @@ -319,7 +320,7 @@ static int parse_dirplusfile(char *buf, size_t nbytes, struct file *file, · 28 unmodified lines | ||
| 319 | 320 | buf += reclen; |
| 320 | 321 | nbytes -= reclen; |
| 321 | 322 | |
| 322 | − ret = fuse_direntplus_link(file, direntplus, attr_version); | |
| 323 | + ret = fuse_direntplus_link(file, direntplus, attr_version, evict_ctr); | |
| 323 | 324 | if (ret) |
| 324 | 325 | fuse_force_forget(file, direntplus->entry_out.nodeid); |
| 325 | 326 | } |
| @@ -331,26 +332,27 @@ static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) · 5 unmodified lines | ||
| 331 | 332 | { |
| 332 | 333 | int plus; |
| 333 | 334 | ssize_t res; |
| 334 | − struct page *page; | |
| 335 | + struct folio *folio; | |
| 335 | 336 | struct inode *inode = file_inode(file); |
| 336 | 337 | struct fuse_mount *fm = get_fuse_mount(inode); |
| 337 | 338 | struct fuse_io_args ia = {}; |
| 338 | 339 | struct fuse_args_pages *ap = &ia.ap; |
| 339 | − struct fuse_page_desc desc = { .length = PAGE_SIZE }; | |
| 340 | − u64 attr_version = 0; | |
| 340 | + struct fuse_folio_desc desc = { .length = PAGE_SIZE }; | |
| 341 | + u64 attr_version = 0, evict_ctr = 0; | |
| 341 | 342 | bool locked; |
| 342 | 343 | |
| 343 | − page = alloc_page(GFP_KERNEL); | |
| 344 | − if (!page) | |
| 344 | + folio = folio_alloc(GFP_KERNEL, 0); | |
| 345 | + if (!folio) | |
| 345 | 346 | return -ENOMEM; |
| 346 | 347 | |
| 347 | 348 | plus = fuse_use_readdirplus(inode, ctx); |
| 348 | 349 | ap->args.out_pages = true; |
| 349 | − ap->num_pages = 1; | |
| 350 | − ap->pages = &page; | |
| 350 | + ap->num_folios = 1; | |
| 351 | + ap->folios = &folio; | |
| 351 | 352 | ap->descs = &desc; |
| 352 | 353 | if (plus) { |
| 353 | 354 | attr_version = fuse_get_attr_version(fm->fc); |
| 355 | + evict_ctr = fuse_get_evict_ctr(fm->fc); | |
| 354 | 356 | fuse_read_args_fill(&ia, file, ctx->pos, PAGE_SIZE, |
| 355 | 357 | FUSE_READDIRPLUS); |
| 356 | 358 | } else { |
| @@ -367,15 +369,16 @@ static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) · 10 unmodified lines | ||
| 367 | 369 | if (ff->open_flags & FOPEN_CACHE_DIR) |
| 368 | 370 | fuse_readdir_cache_end(file, ctx->pos); |
| 369 | 371 | } else if (plus) { |
| 370 | − res = parse_dirplusfile(page_address(page), res, | |
| 371 | − file, ctx, attr_version); | |
| 372 | + res = parse_dirplusfile(folio_address(folio), res, | |
| 373 | + file, ctx, attr_version, | |
| 374 | + evict_ctr); | |
| 372 | 375 | } else { |
| 373 | − res = parse_dirfile(page_address(page), res, file, | |
| 376 | + res = parse_dirfile(folio_address(folio), res, file, | |
| 374 | 377 | ctx); |
| 375 | 378 | } |
| 376 | 379 | } |
| 377 | 380 | |
| 378 | − __free_page(page); | |
| 381 | + folio_put(folio); | |
| 379 | 382 | fuse_invalidate_atime(inode); |
| 380 | 383 | return res; |
| 381 | 384 | } |
| @@ -0,0 +1,40 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0 | |
| 2 | +/* | |
| 3 | + * linux/fs/fuse/fuse_sysctl.c | |
| 4 | + * | |
| 5 | + * Sysctl interface to fuse parameters | |
| 6 | + */ | |
| 7 | +#include <linux/sysctl.h> | |
| 8 | + | |
| 9 | +#include "fuse_i.h" | |
| 10 | + | |
| 11 | +static struct ctl_table_header *fuse_table_header; | |
| 12 | + | |
| 13 | +/* Bound by fuse_init_out max_pages, which is a u16 */ | |
| 14 | +static unsigned int sysctl_fuse_max_pages_limit = 65535; | |
| 15 | + | |
| 16 | +static struct ctl_table fuse_sysctl_table[] = { | |
| 17 | + { | |
| 18 | + .procname = "max_pages_limit", | |
| 19 | + .data = &fuse_max_pages_limit, | |
| 20 | + .maxlen = sizeof(fuse_max_pages_limit), | |
| 21 | + .mode = 0644, | |
| 22 | + .proc_handler = proc_douintvec_minmax, | |
| 23 | + .extra1 = SYSCTL_ONE, | |
| 24 | + .extra2 = &sysctl_fuse_max_pages_limit, | |
| 25 | + }, | |
| 26 | +}; | |
| 27 | + | |
| 28 | +int fuse_sysctl_register(void) | |
| 29 | +{ | |
| 30 | + fuse_table_header = register_sysctl("fs/fuse", fuse_sysctl_table); | |
| 31 | + if (!fuse_table_header) | |
| 32 | + return -ENOMEM; | |
| 33 | + return 0; | |
| 34 | +} | |
| 35 | + | |
| 36 | +void fuse_sysctl_unregister(void) | |
| 37 | +{ | |
| 38 | + unregister_sysctl_table(fuse_table_header); | |
| 39 | + fuse_table_header = NULL; | |
| 40 | +} | |
| @@ -97,7 +97,8 @@ struct virtio_fs_req_work { · 96 unmodified lines | ||
| 97 | 97 | }; |
| 98 | 98 | |
| 99 | 99 | static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, |
| 100 | − struct fuse_req *req, bool in_flight); | |
| 100 | + struct fuse_req *req, bool in_flight, | |
| 101 | + gfp_t gfp); | |
| 101 | 102 | |
| 102 | 103 | static const struct constant_table dax_param_enums[] = { |
| 103 | 104 | {"always", FUSE_DAX_ALWAYS }, |
| @@ -575,6 +576,8 @@ static void virtio_fs_request_dispatch_work(struct work_struct *work) · 471 unmodified lines | ||
| 575 | 576 | |
| 576 | 577 | /* Dispatch pending requests */ |
| 577 | 578 | while (1) { |
| 579 | + unsigned int flags; | |
| 580 | + | |
| 578 | 581 | spin_lock(&fsvq->lock); |
| 579 | 582 | req = list_first_entry_or_null(&fsvq->queued_reqs, |
| 580 | 583 | struct fuse_req, list); |
| @@ -585,7 +588,9 @@ static void virtio_fs_request_dispatch_work(struct work_struct *work) · 4 unmodified lines | ||
| 585 | 588 | list_del_init(&req->list); |
| 586 | 589 | spin_unlock(&fsvq->lock); |
| 587 | 590 | |
| 588 | − ret = virtio_fs_enqueue_req(fsvq, req, true); | |
| 591 | + flags = memalloc_nofs_save(); | |
| 592 | + ret = virtio_fs_enqueue_req(fsvq, req, true, GFP_KERNEL); | |
| 593 | + memalloc_nofs_restore(flags); | |
| 589 | 594 | if (ret < 0) { |
| 590 | 595 | if (ret == -ENOSPC) { |
| 591 | 596 | spin_lock(&fsvq->lock); |
| @@ -686,7 +691,7 @@ static void virtio_fs_hiprio_dispatch_work(struct work_struct *work) · 94 unmodified lines | ||
| 686 | 691 | } |
| 687 | 692 | |
| 688 | 693 | /* Allocate and copy args into req->argbuf */ |
| 689 | −static int copy_args_to_argbuf(struct fuse_req *req) | |
| 694 | +static int copy_args_to_argbuf(struct fuse_req *req, gfp_t gfp) | |
| 690 | 695 | { |
| 691 | 696 | struct fuse_args *args = req->args; |
| 692 | 697 | unsigned int offset = 0; |
| @@ -700,7 +705,7 @@ static int copy_args_to_argbuf(struct fuse_req *req) · 7 unmodified lines | ||
| 700 | 705 | len = fuse_len_args(num_in, (struct fuse_arg *) args->in_args) + |
| 701 | 706 | fuse_len_args(num_out, args->out_args); |
| 702 | 707 | |
| 703 | − req->argbuf = kmalloc(len, GFP_ATOMIC); | |
| 708 | + req->argbuf = kmalloc(len, gfp); | |
| 704 | 709 | if (!req->argbuf) |
| 705 | 710 | return -ENOMEM; |
| 706 | 711 | |
| @@ -760,7 +765,7 @@ static void virtio_fs_request_complete(struct fuse_req *req, · 53 unmodified lines | ||
| 760 | 765 | struct fuse_args *args; |
| 761 | 766 | struct fuse_args_pages *ap; |
| 762 | 767 | unsigned int len, i, thislen; |
| 763 | − struct page *page; | |
| 768 | + struct folio *folio; | |
| 764 | 769 | |
| 765 | 770 | /* |
| 766 | 771 | * TODO verify that server properly follows FUSE protocol |
| @@ -772,12 +777,12 @@ static void virtio_fs_request_complete(struct fuse_req *req, · 5 unmodified lines | ||
| 772 | 777 | if (args->out_pages && args->page_zeroing) { |
| 773 | 778 | len = args->out_args[args->out_numargs - 1].size; |
| 774 | 779 | ap = container_of(args, typeof(*ap), args); |
| 775 | − for (i = 0; i < ap->num_pages; i++) { | |
| 780 | + for (i = 0; i < ap->num_folios; i++) { | |
| 776 | 781 | thislen = ap->descs[i].length; |
| 777 | 782 | if (len < thislen) { |
| 778 | 783 | WARN_ON(ap->descs[i].offset); |
| 779 | − page = ap->pages[i]; | |
| 780 | − zero_user_segment(page, len, thislen); | |
| 784 | + folio = ap->folios[i]; | |
| 785 | + folio_zero_segment(folio, len, thislen); | |
| 781 | 786 | len = 0; |
| 782 | 787 | } else { |
| 783 | 788 | len -= thislen; |
| @@ -1267,15 +1272,15 @@ static void virtio_fs_send_interrupt(struct fuse_iqueue *fiq, struct fuse_req *r · 483 unmodified lines | ||
| 1267 | 1272 | } |
| 1268 | 1273 | |
| 1269 | 1274 | /* Count number of scatter-gather elements required */ |
| 1270 | −static unsigned int sg_count_fuse_pages(struct fuse_page_desc *page_descs, | |
| 1271 | − unsigned int num_pages, | |
| 1272 | − unsigned int total_len) | |
| 1275 | +static unsigned int sg_count_fuse_folios(struct fuse_folio_desc *folio_descs, | |
| 1276 | + unsigned int num_folios, | |
| 1277 | + unsigned int total_len) | |
| 1273 | 1278 | { |
| 1274 | 1279 | unsigned int i; |
| 1275 | 1280 | unsigned int this_len; |
| 1276 | 1281 | |
| 1277 | − for (i = 0; i < num_pages && total_len; i++) { | |
| 1278 | − this_len = min(page_descs[i].length, total_len); | |
| 1282 | + for (i = 0; i < num_folios && total_len; i++) { | |
| 1283 | + this_len = min(folio_descs[i].length, total_len); | |
| 1279 | 1284 | total_len -= this_len; |
| 1280 | 1285 | } |
| 1281 | 1286 | |
| @@ -1294,8 +1299,8 @@ static unsigned int sg_count_fuse_req(struct fuse_req *req) · 12 unmodified lines | ||
| 1294 | 1299 | |
| 1295 | 1300 | if (args->in_pages) { |
| 1296 | 1301 | size = args->in_args[args->in_numargs - 1].size; |
| 1297 | − total_sgs += sg_count_fuse_pages(ap->descs, ap->num_pages, | |
| 1298 | − size); | |
| 1302 | + total_sgs += sg_count_fuse_folios(ap->descs, ap->num_folios, | |
| 1303 | + size); | |
| 1299 | 1304 | } |
| 1300 | 1305 | |
| 1301 | 1306 | if (!test_bit(FR_ISREPLY, &req->flags)) |
| @@ -1308,27 +1313,27 @@ static unsigned int sg_count_fuse_req(struct fuse_req *req) · 6 unmodified lines | ||
| 1308 | 1313 | |
| 1309 | 1314 | if (args->out_pages) { |
| 1310 | 1315 | size = args->out_args[args->out_numargs - 1].size; |
| 1311 | − total_sgs += sg_count_fuse_pages(ap->descs, ap->num_pages, | |
| 1312 | − size); | |
| 1316 | + total_sgs += sg_count_fuse_folios(ap->descs, ap->num_folios, | |
| 1317 | + size); | |
| 1313 | 1318 | } |
| 1314 | 1319 | |
| 1315 | 1320 | return total_sgs; |
| 1316 | 1321 | } |
| 1317 | 1322 | |
| 1318 | −/* Add pages to scatter-gather list and return number of elements used */ | |
| 1319 | −static unsigned int sg_init_fuse_pages(struct scatterlist *sg, | |
| 1320 | − struct page **pages, | |
| 1321 | − struct fuse_page_desc *page_descs, | |
| 1322 | − unsigned int num_pages, | |
| 1323 | − unsigned int total_len) | |
| 1323 | +/* Add folios to scatter-gather list and return number of elements used */ | |
| 1324 | +static unsigned int sg_init_fuse_folios(struct scatterlist *sg, | |
| 1325 | + struct folio **folios, | |
| 1326 | + struct fuse_folio_desc *folio_descs, | |
| 1327 | + unsigned int num_folios, | |
| 1328 | + unsigned int total_len) | |
| 1324 | 1329 | { |
| 1325 | 1330 | unsigned int i; |
| 1326 | 1331 | unsigned int this_len; |
| 1327 | 1332 | |
| 1328 | − for (i = 0; i < num_pages && total_len; i++) { | |
| 1333 | + for (i = 0; i < num_folios && total_len; i++) { | |
| 1329 | 1334 | sg_init_table(&sg[i], 1); |
| 1330 | − this_len = min(page_descs[i].length, total_len); | |
| 1331 | − sg_set_page(&sg[i], pages[i], this_len, page_descs[i].offset); | |
| 1335 | + this_len = min(folio_descs[i].length, total_len); | |
| 1336 | + sg_set_folio(&sg[i], folios[i], this_len, folio_descs[i].offset); | |
| 1332 | 1337 | total_len -= this_len; |
| 1333 | 1338 | } |
| 1334 | 1339 | |
| @@ -1353,10 +1358,10 @@ static unsigned int sg_init_fuse_args(struct scatterlist *sg, · 18 unmodified lines | ||
| 1353 | 1358 | sg_init_one(&sg[total_sgs++], argbuf, len); |
| 1354 | 1359 | |
| 1355 | 1360 | if (argpages) |
| 1356 | − total_sgs += sg_init_fuse_pages(&sg[total_sgs], | |
| 1357 | − ap->pages, ap->descs, | |
| 1358 | − ap->num_pages, | |
| 1359 | − args[numargs - 1].size); | |
| 1361 | + total_sgs += sg_init_fuse_folios(&sg[total_sgs], | |
| 1362 | + ap->folios, ap->descs, | |
| 1363 | + ap->num_folios, | |
| 1364 | + args[numargs - 1].size); | |
| 1360 | 1365 | |
| 1361 | 1366 | if (len_used) |
| 1362 | 1367 | *len_used = len; |
| @@ -1366,7 +1371,8 @@ static unsigned int sg_init_fuse_args(struct scatterlist *sg, · 3 unmodified lines | ||
| 1366 | 1371 | |
| 1367 | 1372 | /* Add a request to a virtqueue and kick the device */ |
| 1368 | 1373 | static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, |
| 1369 | − struct fuse_req *req, bool in_flight) | |
| 1374 | + struct fuse_req *req, bool in_flight, | |
| 1375 | + gfp_t gfp) | |
| 1370 | 1376 | { |
| 1371 | 1377 | /* requests need at least 4 elements */ |
| 1372 | 1378 | struct scatterlist *stack_sgs[6]; |
| @@ -1387,8 +1393,8 @@ static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, · 14 unmodified lines | ||
| 1387 | 1393 | /* Does the sglist fit on the stack? */ |
| 1388 | 1394 | total_sgs = sg_count_fuse_req(req); |
| 1389 | 1395 | if (total_sgs > ARRAY_SIZE(stack_sgs)) { |
| 1390 | − sgs = kmalloc_array(total_sgs, sizeof(sgs[0]), GFP_ATOMIC); | |
| 1391 | − sg = kmalloc_array(total_sgs, sizeof(sg[0]), GFP_ATOMIC); | |
| 1396 | + sgs = kmalloc_array(total_sgs, sizeof(sgs[0]), gfp); | |
| 1397 | + sg = kmalloc_array(total_sgs, sizeof(sg[0]), gfp); | |
| 1392 | 1398 | if (!sgs || !sg) { |
| 1393 | 1399 | ret = -ENOMEM; |
| 1394 | 1400 | goto out; |
| @@ -1396,7 +1402,7 @@ static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, · 1 unmodified line | ||
| 1396 | 1402 | } |
| 1397 | 1403 | |
| 1398 | 1404 | /* Use a bounce buffer since stack args cannot be mapped */ |
| 1399 | − ret = copy_args_to_argbuf(req); | |
| 1405 | + ret = copy_args_to_argbuf(req, gfp); | |
| 1400 | 1406 | if (ret < 0) |
| 1401 | 1407 | goto out; |
| 1402 | 1408 | |
| @@ -1490,7 +1496,7 @@ static void virtio_fs_send_req(struct fuse_iqueue *fiq, struct fuse_req *req) · 87 unmodified lines | ||
| 1490 | 1496 | queue_id); |
| 1491 | 1497 | |
| 1492 | 1498 | fsvq = &fs->vqs[queue_id]; |
| 1493 | − ret = virtio_fs_enqueue_req(fsvq, req, false); | |
| 1499 | + ret = virtio_fs_enqueue_req(fsvq, req, false, GFP_ATOMIC); | |
| 1494 | 1500 | if (ret < 0) { |
| 1495 | 1501 | if (ret == -ENOSPC) { |
| 1496 | 1502 | /* |
| @@ -1691,6 +1697,7 @@ static int virtio_fs_get_tree(struct fs_context *fsc) · 194 unmodified lines | ||
| 1691 | 1697 | fc->delete_stale = true; |
| 1692 | 1698 | fc->auto_submounts = true; |
| 1693 | 1699 | fc->sync_fs = true; |
| 1700 | + fc->use_pages_for_kvec_io = true; | |
| 1694 | 1701 | |
| 1695 | 1702 | /* Tell FUSE to split requests that exceed the virtqueue's size */ |
| 1696 | 1703 | fc->max_pages_limit = min_t(unsigned int, fc->max_pages_limit, |
| @@ -2550,6 +2550,7 @@ struct kvec; · 2549 unmodified lines | ||
| 2550 | 2550 | struct page *get_dump_page(unsigned long addr); |
| 2551 | 2551 | |
| 2552 | 2552 | bool folio_mark_dirty(struct folio *folio); |
| 2553 | +bool folio_mark_dirty_lock(struct folio *folio); | |
| 2553 | 2554 | bool set_page_dirty(struct page *page); |
| 2554 | 2555 | int set_page_dirty_lock(struct page *page); |
| 2555 | 2556 | |
| @@ -52,6 +52,12 @@ bool set_page_dirty(struct page *page) · 51 unmodified lines | ||
| 52 | 52 | } |
| 53 | 53 | EXPORT_SYMBOL(set_page_dirty); |
| 54 | 54 | |
| 55 | +int set_page_dirty_lock(struct page *page) | |
| 56 | +{ | |
| 57 | + return folio_mark_dirty_lock(page_folio(page)); | |
| 58 | +} | |
| 59 | +EXPORT_SYMBOL(set_page_dirty_lock); | |
| 60 | + | |
| 55 | 61 | bool clear_page_dirty_for_io(struct page *page) |
| 56 | 62 | { |
| 57 | 63 | return folio_clear_dirty_for_io(page_folio(page)); |
| @@ -2925,25 +2925,25 @@ bool folio_mark_dirty(struct folio *folio) · 2924 unmodified lines | ||
| 2925 | 2925 | EXPORT_SYMBOL(folio_mark_dirty); |
| 2926 | 2926 | |
| 2927 | 2927 | /* |
| 2928 | − * set_page_dirty() is racy if the caller has no reference against | |
| 2929 | − * page->mapping->host, and if the page is unlocked. This is because another | |
| 2930 | − * CPU could truncate the page off the mapping and then free the mapping. | |
| 2928 | + * folio_mark_dirty() is racy if the caller has no reference against | |
| 2929 | + * folio->mapping->host, and if the folio is unlocked. This is because another | |
| 2930 | + * CPU could truncate the folio off the mapping and then free the mapping. | |
| 2931 | 2931 | * |
| 2932 | − * Usually, the page _is_ locked, or the caller is a user-space process which | |
| 2932 | + * Usually, the folio _is_ locked, or the caller is a user-space process which | |
| 2933 | 2933 | * holds a reference on the inode by having an open file. |
| 2934 | 2934 | * |
| 2935 | − * In other cases, the page should be locked before running set_page_dirty(). | |
| 2935 | + * In other cases, the folio should be locked before running folio_mark_dirty(). | |
| 2936 | 2936 | */ |
| 2937 | −int set_page_dirty_lock(struct page *page) | |
| 2937 | +bool folio_mark_dirty_lock(struct folio *folio) | |
| 2938 | 2938 | { |
| 2939 | − int ret; | |
| 2939 | + bool ret; | |
| 2940 | 2940 | |
| 2941 | − lock_page(page); | |
| 2942 | − ret = set_page_dirty(page); | |
| 2943 | − unlock_page(page); | |
| 2941 | + folio_lock(folio); | |
| 2942 | + ret = folio_mark_dirty(folio); | |
| 2943 | + folio_unlock(folio); | |
| 2944 | 2944 | return ret; |
| 2945 | 2945 | } |
| 2946 | −EXPORT_SYMBOL(set_page_dirty_lock); | |
| 2946 | +EXPORT_SYMBOL(folio_mark_dirty_lock); | |
| 2947 | 2947 | |
| 2948 | 2948 | /* |
| 2949 | 2949 | * This cancels just the dirty bit on the kernel page itself, it does NOT |