Pull fuse updates from Miklos Szeredi: - Add page -> folio conversions (Joanne Koong, Josef Bacik) - Allow max size of fuse requests to be configurable with a sysctl (Joanne Koong) - Allow FOPEN_DIRECT_IO to take advantage of async code path (yangyun) - Fix large kernel reads (like a module load) in virtio_fs (Hou Tao) - Fix attribute inconsistency in case readdirplus (and plain lookup in corner cases) is racing with inode eviction (Zhang Tianci) - Fix a WARN_ON triggered by virtio_fs (Asahi Lina) * tag 'fuse-update-6.13' of git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse: (30 commits) virtiofs: dax: remove ->writepages() callback fuse: check attributes staleness on fuse_iget() fuse: remove pages for requests and exclusively use folios fuse: convert direct io to use folios mm/writeback: add folio_mark_dirty_lock() fuse: convert writebacks to use folios fuse: convert retrieves to use folios fuse: convert ioctls to use folios fuse: convert writes (non-writeback) to use folios fuse: convert reads to use folios fuse: convert readdir to use folios fuse: convert readlink to use folios fuse: convert cuse to use folios fuse: add support in virtio for requests using folios fuse: support folios in struct fuse_args_pages and fuse_copy_pages() fuse: convert fuse_notify_store to use folios fuse: convert fuse_retrieve to use folios fuse: use the folio based vmstat helpers fuse: convert fuse_writepage_need_send to take a folio fuse: convert fuse_do_readpage to use folios ...
Unified · Split
| @@ -337,3 +337,13 @@ Each "watch" costs roughly 90 bytes on a 32-bit kernel, and roughly 160 bytes · 336 unmodified lines | |||
| 337 | on a 64-bit one. | 337 | on a 64-bit one. |
| 338 | The current default value for ``max_user_watches`` is 4% of the | 338 | The current default value for ``max_user_watches`` is 4% of the |
| 339 | available low memory, divided by the "watch" cost in bytes. | 339 | available low memory, divided by the "watch" cost in bytes. |
| 340 | |||
| 341 | 5. /proc/sys/fs/fuse - Configuration options for FUSE filesystems | ||
| 342 | ===================================================================== | ||
| 343 | |||
| 344 | This directory contains the following configuration options for FUSE | ||
| 345 | filesystems: | ||
| 346 | |||
| 347 | ``/proc/sys/fs/fuse/max_pages_limit`` is a read/write file for | ||
| 348 | setting/getting the maximum number of pages that can be used for servicing | ||
| 349 | requests in FUSE. | ||
| @@ -14,5 +14,6 @@ fuse-y := dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o · 13 unmodified lines | |||
| 14 | fuse-y += iomode.o | 14 | fuse-y += iomode.o |
| 15 | fuse-$(CONFIG_FUSE_DAX) += dax.o | 15 | fuse-$(CONFIG_FUSE_DAX) += dax.o |
| 16 | fuse-$(CONFIG_FUSE_PASSTHROUGH) += passthrough.o | 16 | fuse-$(CONFIG_FUSE_PASSTHROUGH) += passthrough.o |
| 17 | fuse-$(CONFIG_SYSCTL) += sysctl.o | ||
| 17 | 18 | ||
| 18 | virtiofs-y := virtio_fs.o | 19 | virtiofs-y := virtio_fs.o |
| @@ -303,8 +303,8 @@ struct cuse_init_args { · 302 unmodified lines | |||
| 303 | struct fuse_args_pages ap; | 303 | struct fuse_args_pages ap; |
| 304 | struct cuse_init_in in; | 304 | struct cuse_init_in in; |
| 305 | struct cuse_init_out out; | 305 | struct cuse_init_out out; |
| 306 | struct page *page; | 306 | struct folio *folio; |
| 307 | struct fuse_page_desc desc; | 307 | struct fuse_folio_desc desc; |
| 308 | }; | 308 | }; |
| 309 | 309 | ||
| 310 | /** | 310 | /** |
| @@ -326,7 +326,7 @@ static void cuse_process_init_reply(struct fuse_mount *fm, · 15 unmodified lines | |||
| 326 | struct fuse_args_pages *ap = &ia->ap; | 326 | struct fuse_args_pages *ap = &ia->ap; |
| 327 | struct cuse_conn *cc = fc_to_cc(fc), *pos; | 327 | struct cuse_conn *cc = fc_to_cc(fc), *pos; |
| 328 | struct cuse_init_out *arg = &ia->out; | 328 | struct cuse_init_out *arg = &ia->out; |
| 329 | struct page *page = ap->pages[0]; | 329 | struct folio *folio = ap->folios[0]; |
| 330 | struct cuse_devinfo devinfo = { }; | 330 | struct cuse_devinfo devinfo = { }; |
| 331 | struct device *dev; | 331 | struct device *dev; |
| 332 | struct cdev *cdev; | 332 | struct cdev *cdev; |
| @@ -343,7 +343,7 @@ static void cuse_process_init_reply(struct fuse_mount *fm, · 10 unmodified lines | |||
| 343 | /* parse init reply */ | 343 | /* parse init reply */ |
| 344 | cc->unrestricted_ioctl = arg->flags & CUSE_UNRESTRICTED_IOCTL; | 344 | cc->unrestricted_ioctl = arg->flags & CUSE_UNRESTRICTED_IOCTL; |
| 345 | 345 | ||
| 346 | rc = cuse_parse_devinfo(page_address(page), ap->args.out_args[1].size, | 346 | rc = cuse_parse_devinfo(folio_address(folio), ap->args.out_args[1].size, |
| 347 | &devinfo); | 347 | &devinfo); |
| 348 | if (rc) | 348 | if (rc) |
| 349 | goto err; | 349 | goto err; |
| @@ -411,7 +411,7 @@ static void cuse_process_init_reply(struct fuse_mount *fm, · 61 unmodified lines | |||
| 411 | kobject_uevent(&dev->kobj, KOBJ_ADD); | 411 | kobject_uevent(&dev->kobj, KOBJ_ADD); |
| 412 | out: | 412 | out: |
| 413 | kfree(ia); | 413 | kfree(ia); |
| 414 | __free_page(page); | 414 | folio_put(folio); |
| 415 | return; | 415 | return; |
| 416 | 416 | ||
| 417 | err_cdev: | 417 | err_cdev: |
| @@ -429,7 +429,7 @@ err: · 11 unmodified lines | |||
| 429 | static int cuse_send_init(struct cuse_conn *cc) | 429 | static int cuse_send_init(struct cuse_conn *cc) |
| 430 | { | 430 | { |
| 431 | int rc; | 431 | int rc; |
| 432 | struct page *page; | 432 | struct folio *folio; |
| 433 | struct fuse_mount *fm = &cc->fm; | 433 | struct fuse_mount *fm = &cc->fm; |
| 434 | struct cuse_init_args *ia; | 434 | struct cuse_init_args *ia; |
| 435 | struct fuse_args_pages *ap; | 435 | struct fuse_args_pages *ap; |
| @@ -437,13 +437,14 @@ static int cuse_send_init(struct cuse_conn *cc) · 1 unmodified line | |||
| 437 | BUILD_BUG_ON(CUSE_INIT_INFO_MAX > PAGE_SIZE); | 437 | BUILD_BUG_ON(CUSE_INIT_INFO_MAX > PAGE_SIZE); |
| 438 | 438 | ||
| 439 | rc = -ENOMEM; | 439 | rc = -ENOMEM; |
| 440 | page = alloc_page(GFP_KERNEL | __GFP_ZERO); | 440 | |
| 441 | if (!page) | 441 | folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0); |
| 442 | if (!folio) | ||
| 442 | goto err; | 443 | goto err; |
| 443 | 444 | ||
| 444 | ia = kzalloc(sizeof(*ia), GFP_KERNEL); | 445 | ia = kzalloc(sizeof(*ia), GFP_KERNEL); |
| 445 | if (!ia) | 446 | if (!ia) |
| 446 | goto err_free_page; | 447 | goto err_free_folio; |
| 447 | 448 | ||
| 448 | ap = &ia->ap; | 449 | ap = &ia->ap; |
| 449 | ia->in.major = FUSE_KERNEL_VERSION; | 450 | ia->in.major = FUSE_KERNEL_VERSION; |
| @@ -459,18 +460,18 @@ static int cuse_send_init(struct cuse_conn *cc) · 9 unmodified lines | |||
| 459 | ap->args.out_args[1].size = CUSE_INIT_INFO_MAX; | 460 | ap->args.out_args[1].size = CUSE_INIT_INFO_MAX; |
| 460 | ap->args.out_argvar = true; | 461 | ap->args.out_argvar = true; |
| 461 | ap->args.out_pages = true; | 462 | ap->args.out_pages = true; |
| 462 | ap->num_pages = 1; | 463 | ap->num_folios = 1; |
| 463 | ap->pages = &ia->page; | 464 | ap->folios = &ia->folio; |
| 464 | ap->descs = &ia->desc; | 465 | ap->descs = &ia->desc; |
| 465 | ia->page = page; | 466 | ia->folio = folio; |
| 466 | ia->desc.length = ap->args.out_args[1].size; | 467 | ia->desc.length = ap->args.out_args[1].size; |
| 467 | ap->args.end = cuse_process_init_reply; | 468 | ap->args.end = cuse_process_init_reply; |
| 468 | 469 | ||
| 469 | rc = fuse_simple_background(fm, &ap->args, GFP_KERNEL); | 470 | rc = fuse_simple_background(fm, &ap->args, GFP_KERNEL); |
| 470 | if (rc) { | 471 | if (rc) { |
| 471 | kfree(ia); | 472 | kfree(ia); |
| 472 | err_free_page: | 473 | err_free_folio: |
| 473 | __free_page(page); | 474 | folio_put(folio); |
| 474 | } | 475 | } |
| 475 | err: | 476 | err: |
| 476 | return rc; | 477 | return rc; |
| @@ -774,16 +774,6 @@ out: · 773 unmodified lines | |||
| 774 | return ret; | 774 | return ret; |
| 775 | } | 775 | } |
| 776 | 776 | ||
| 777 | static int fuse_dax_writepages(struct address_space *mapping, | ||
| 778 | struct writeback_control *wbc) | ||
| 779 | { | ||
| 780 | |||
| 781 | struct inode *inode = mapping->host; | ||
| 782 | struct fuse_conn *fc = get_fuse_conn(inode); | ||
| 783 | |||
| 784 | return dax_writeback_mapping_range(mapping, fc->dax->dev, wbc); | ||
| 785 | } | ||
| 786 | |||
| 787 | static vm_fault_t __fuse_dax_fault(struct vm_fault *vmf, unsigned int order, | 777 | static vm_fault_t __fuse_dax_fault(struct vm_fault *vmf, unsigned int order, |
| 788 | bool write) | 778 | bool write) |
| 789 | { | 779 | { |
| @@ -1323,7 +1313,6 @@ bool fuse_dax_inode_alloc(struct super_block *sb, struct fuse_inode *fi) · 533 unmodified lines | |||
| 1323 | } | 1313 | } |
| 1324 | 1314 | ||
| 1325 | static const struct address_space_operations fuse_dax_file_aops = { | 1315 | static const struct address_space_operations fuse_dax_file_aops = { |
| 1326 | .writepages = fuse_dax_writepages, | ||
| 1327 | .direct_IO = noop_direct_IO, | 1316 | .direct_IO = noop_direct_IO, |
| 1328 | .dirty_folio = noop_dirty_folio, | 1317 | .dirty_folio = noop_dirty_folio, |
| 1329 | }; | 1318 | }; |
| @@ -1028,17 +1028,27 @@ static int fuse_copy_pages(struct fuse_copy_state *cs, unsigned nbytes, · 1027 unmodified lines | |||
| 1028 | struct fuse_req *req = cs->req; | 1028 | struct fuse_req *req = cs->req; |
| 1029 | struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args); | 1029 | struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args); |
| 1030 | 1030 | ||
| 1031 | 1031 | for (i = 0; i < ap->num_folios && (nbytes || zeroing); i++) { | |
| 1032 | for (i = 0; i < ap->num_pages && (nbytes || zeroing); i++) { | ||
| 1033 | int err; | 1032 | int err; |
| 1034 | unsigned int offset = ap->descs[i].offset; | 1033 | unsigned int offset = ap->descs[i].offset; |
| 1035 | unsigned int count = min(nbytes, ap->descs[i].length); | 1034 | unsigned int count = min(nbytes, ap->descs[i].length); |
| 1035 | struct page *orig, *pagep; | ||
| 1036 | |||
| 1037 | orig = pagep = &ap->folios[i]->page; | ||
| 1036 | 1038 | ||
| 1037 | err = fuse_copy_page(cs, &ap->pages[i], offset, count, zeroing); | 1039 | err = fuse_copy_page(cs, &pagep, offset, count, zeroing); |
| 1038 | if (err) | 1040 | if (err) |
| 1039 | return err; | 1041 | return err; |
| 1040 | 1042 | ||
| 1041 | nbytes -= count; | 1043 | nbytes -= count; |
| 1044 | |||
| 1045 | /* | ||
| 1046 | * fuse_copy_page may have moved a page from a pipe instead of | ||
| 1047 | * copying into our given page, so update the folios if it was | ||
| 1048 | * replaced. | ||
| 1049 | */ | ||
| 1050 | if (pagep != orig) | ||
| 1051 | ap->folios[i] = page_folio(pagep); | ||
| 1042 | } | 1052 | } |
| 1043 | return 0; | 1053 | return 0; |
| 1044 | } | 1054 | } |
| @@ -1654,24 +1664,25 @@ static int fuse_notify_store(struct fuse_conn *fc, unsigned int size, · 609 unmodified lines | |||
| 1654 | 1664 | ||
| 1655 | num = outarg.size; | 1665 | num = outarg.size; |
| 1656 | while (num) { | 1666 | while (num) { |
| 1667 | struct folio *folio; | ||
| 1657 | struct page *page; | 1668 | struct page *page; |
| 1658 | unsigned int this_num; | 1669 | unsigned int this_num; |
| 1659 | 1670 | ||
| 1660 | err = -ENOMEM; | 1671 | folio = filemap_grab_folio(mapping, index); |
| 1661 | page = find_or_create_page(mapping, index, | 1672 | err = PTR_ERR(folio); |
| 1662 | mapping_gfp_mask(mapping)); | 1673 | if (IS_ERR(folio)) |
| 1663 | if (!page) | ||
| 1664 | goto out_iput; | 1674 | goto out_iput; |
| 1665 | 1675 | ||
| 1666 | this_num = min_t(unsigned, num, PAGE_SIZE - offset); | 1676 | page = &folio->page; |
| 1677 | this_num = min_t(unsigned, num, folio_size(folio) - offset); | ||
| 1667 | err = fuse_copy_page(cs, &page, offset, this_num, 0); | 1678 | err = fuse_copy_page(cs, &page, offset, this_num, 0); |
| 1668 | if (!PageUptodate(page) && !err && offset == 0 && | 1679 | if (!folio_test_uptodate(folio) && !err && offset == 0 && |
| 1669 | (this_num == PAGE_SIZE || file_size == end)) { | 1680 | (this_num == folio_size(folio) || file_size == end)) { |
| 1670 | zero_user_segment(page, this_num, PAGE_SIZE); | 1681 | folio_zero_segment(folio, this_num, folio_size(folio)); |
| 1671 | SetPageUptodate(page); | 1682 | folio_mark_uptodate(folio); |
| 1672 | } | 1683 | } |
| 1673 | unlock_page(page); | 1684 | folio_unlock(folio); |
| 1674 | put_page(page); | 1685 | folio_put(folio); |
| 1675 | 1686 | ||
| 1676 | if (err) | 1687 | if (err) |
| 1677 | goto out_iput; | 1688 | goto out_iput; |
| @@ -1703,7 +1714,7 @@ static void fuse_retrieve_end(struct fuse_mount *fm, struct fuse_args *args, · 25 unmodified lines | |||
| 1703 | struct fuse_retrieve_args *ra = | 1714 | struct fuse_retrieve_args *ra = |
| 1704 | container_of(args, typeof(*ra), ap.args); | 1715 | container_of(args, typeof(*ra), ap.args); |
| 1705 | 1716 | ||
| 1706 | release_pages(ra->ap.pages, ra->ap.num_pages); | 1717 | release_pages(ra->ap.folios, ra->ap.num_folios); |
| 1707 | kfree(ra); | 1718 | kfree(ra); |
| 1708 | } | 1719 | } |
| 1709 | 1720 | ||
| @@ -1717,7 +1728,7 @@ static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode, · 7 unmodified lines | |||
| 1717 | unsigned int num; | 1728 | unsigned int num; |
| 1718 | unsigned int offset; | 1729 | unsigned int offset; |
| 1719 | size_t total_len = 0; | 1730 | size_t total_len = 0; |
| 1720 | unsigned int num_pages; | 1731 | unsigned int num_pages, cur_pages = 0; |
| 1721 | struct fuse_conn *fc = fm->fc; | 1732 | struct fuse_conn *fc = fm->fc; |
| 1722 | struct fuse_retrieve_args *ra; | 1733 | struct fuse_retrieve_args *ra; |
| 1723 | size_t args_size = sizeof(*ra); | 1734 | size_t args_size = sizeof(*ra); |
| @@ -1736,15 +1747,15 @@ static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode, · 12 unmodified lines | |||
| 1736 | num_pages = (num + offset + PAGE_SIZE - 1) >> PAGE_SHIFT; | 1747 | num_pages = (num + offset + PAGE_SIZE - 1) >> PAGE_SHIFT; |
| 1737 | num_pages = min(num_pages, fc->max_pages); | 1748 | num_pages = min(num_pages, fc->max_pages); |
| 1738 | 1749 | ||
| 1739 | args_size += num_pages * (sizeof(ap->pages[0]) + sizeof(ap->descs[0])); | 1750 | args_size += num_pages * (sizeof(ap->folios[0]) + sizeof(ap->descs[0])); |
| 1740 | 1751 | ||
| 1741 | ra = kzalloc(args_size, GFP_KERNEL); | 1752 | ra = kzalloc(args_size, GFP_KERNEL); |
| 1742 | if (!ra) | 1753 | if (!ra) |
| 1743 | return -ENOMEM; | 1754 | return -ENOMEM; |
| 1744 | 1755 | ||
| 1745 | ap = &ra->ap; | 1756 | ap = &ra->ap; |
| 1746 | ap->pages = (void *) (ra + 1); | 1757 | ap->folios = (void *) (ra + 1); |
| 1747 | ap->descs = (void *) (ap->pages + num_pages); | 1758 | ap->descs = (void *) (ap->folios + num_pages); |
| 1748 | 1759 | ||
| 1749 | args = &ap->args; | 1760 | args = &ap->args; |
| 1750 | args->nodeid = outarg->nodeid; | 1761 | args->nodeid = outarg->nodeid; |
| @@ -1755,19 +1766,20 @@ static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode, · 4 unmodified lines | |||
| 1755 | 1766 | ||
| 1756 | index = outarg->offset >> PAGE_SHIFT; | 1767 | index = outarg->offset >> PAGE_SHIFT; |
| 1757 | 1768 | ||
| 1758 | while (num && ap->num_pages < num_pages) { | 1769 | while (num && cur_pages < num_pages) { |
| 1759 | struct page *page; | 1770 | struct folio *folio; |
| 1760 | unsigned int this_num; | 1771 | unsigned int this_num; |
| 1761 | 1772 | ||
| 1762 | page = find_get_page(mapping, index); | 1773 | folio = filemap_get_folio(mapping, index); |
| 1763 | if (!page) | 1774 | if (IS_ERR(folio)) |
| 1764 | break; | 1775 | break; |
| 1765 | 1776 | ||
| 1766 | this_num = min_t(unsigned, num, PAGE_SIZE - offset); | 1777 | this_num = min_t(unsigned, num, PAGE_SIZE - offset); |
| 1767 | ap->pages[ap->num_pages] = page; | 1778 | ap->folios[ap->num_folios] = folio; |
| 1768 | ap->descs[ap->num_pages].offset = offset; | 1779 | ap->descs[ap->num_folios].offset = offset; |
| 1769 | ap->descs[ap->num_pages].length = this_num; | 1780 | ap->descs[ap->num_folios].length = this_num; |
| 1770 | ap->num_pages++; | 1781 | ap->num_folios++; |
| 1782 | cur_pages++; | ||
| 1771 | 1783 | ||
| 1772 | offset = 0; | 1784 | offset = 0; |
| 1773 | num -= this_num; | 1785 | num -= this_num; |
| @@ -366,7 +366,7 @@ int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name · 365 unmodified lines | |||
| 366 | struct fuse_mount *fm = get_fuse_mount_super(sb); | 366 | struct fuse_mount *fm = get_fuse_mount_super(sb); |
| 367 | FUSE_ARGS(args); | 367 | FUSE_ARGS(args); |
| 368 | struct fuse_forget_link *forget; | 368 | struct fuse_forget_link *forget; |
| 369 | u64 attr_version; | 369 | u64 attr_version, evict_ctr; |
| 370 | int err; | 370 | int err; |
| 371 | 371 | ||
| 372 | *inode = NULL; | 372 | *inode = NULL; |
| @@ -381,6 +381,7 @@ int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name · 8 unmodified lines | |||
| 381 | goto out; | 381 | goto out; |
| 382 | 382 | ||
| 383 | attr_version = fuse_get_attr_version(fm->fc); | 383 | attr_version = fuse_get_attr_version(fm->fc); |
| 384 | evict_ctr = fuse_get_evict_ctr(fm->fc); | ||
| 384 | 385 | ||
| 385 | fuse_lookup_init(fm->fc, &args, nodeid, name, outarg); | 386 | fuse_lookup_init(fm->fc, &args, nodeid, name, outarg); |
| 386 | err = fuse_simple_request(fm, &args); | 387 | err = fuse_simple_request(fm, &args); |
| @@ -398,7 +399,7 @@ int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name · 11 unmodified lines | |||
| 398 | 399 | ||
| 399 | *inode = fuse_iget(sb, outarg->nodeid, outarg->generation, | 400 | *inode = fuse_iget(sb, outarg->nodeid, outarg->generation, |
| 400 | &outarg->attr, ATTR_TIMEOUT(outarg), | 401 | &outarg->attr, ATTR_TIMEOUT(outarg), |
| 401 | attr_version); | 402 | attr_version, evict_ctr); |
| 402 | err = -ENOMEM; | 403 | err = -ENOMEM; |
| 403 | if (!*inode) { | 404 | if (!*inode) { |
| 404 | fuse_queue_forget(fm->fc, forget, outarg->nodeid, 1); | 405 | fuse_queue_forget(fm->fc, forget, outarg->nodeid, 1); |
| @@ -691,7 +692,7 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, · 286 unmodified lines | |||
| 691 | ff->nodeid = outentry.nodeid; | 692 | ff->nodeid = outentry.nodeid; |
| 692 | ff->open_flags = outopenp->open_flags; | 693 | ff->open_flags = outopenp->open_flags; |
| 693 | inode = fuse_iget(dir->i_sb, outentry.nodeid, outentry.generation, | 694 | inode = fuse_iget(dir->i_sb, outentry.nodeid, outentry.generation, |
| 694 | &outentry.attr, ATTR_TIMEOUT(&outentry), 0); | 695 | &outentry.attr, ATTR_TIMEOUT(&outentry), 0, 0); |
| 695 | if (!inode) { | 696 | if (!inode) { |
| 696 | flags &= ~(O_CREAT | O_EXCL | O_TRUNC); | 697 | flags &= ~(O_CREAT | O_EXCL | O_TRUNC); |
| 697 | fuse_sync_release(NULL, ff, flags); | 698 | fuse_sync_release(NULL, ff, flags); |
| @@ -822,7 +823,7 @@ static int create_new_entry(struct mnt_idmap *idmap, struct fuse_mount *fm, · 124 unmodified lines | |||
| 822 | goto out_put_forget_req; | 823 | goto out_put_forget_req; |
| 823 | 824 | ||
| 824 | inode = fuse_iget(dir->i_sb, outarg.nodeid, outarg.generation, | 825 | inode = fuse_iget(dir->i_sb, outarg.nodeid, outarg.generation, |
| 825 | &outarg.attr, ATTR_TIMEOUT(&outarg), 0); | 826 | &outarg.attr, ATTR_TIMEOUT(&outarg), 0, 0); |
| 826 | if (!inode) { | 827 | if (!inode) { |
| 827 | fuse_queue_forget(fm->fc, forget, outarg.nodeid, 1); | 828 | fuse_queue_forget(fm->fc, forget, outarg.nodeid, 1); |
| 828 | return -ENOMEM; | 829 | return -ENOMEM; |
| @@ -1585,13 +1586,13 @@ static int fuse_permission(struct mnt_idmap *idmap, · 756 unmodified lines | |||
| 1585 | return err; | 1586 | return err; |
| 1586 | } | 1587 | } |
| 1587 | 1588 | ||
| 1588 | static int fuse_readlink_page(struct inode *inode, struct page *page) | 1589 | static int fuse_readlink_page(struct inode *inode, struct folio *folio) |
| 1589 | { | 1590 | { |
| 1590 | struct fuse_mount *fm = get_fuse_mount(inode); | 1591 | struct fuse_mount *fm = get_fuse_mount(inode); |
| 1591 | struct fuse_page_desc desc = { .length = PAGE_SIZE - 1 }; | 1592 | struct fuse_folio_desc desc = { .length = PAGE_SIZE - 1 }; |
| 1592 | struct fuse_args_pages ap = { | 1593 | struct fuse_args_pages ap = { |
| 1593 | .num_pages = 1, | 1594 | .num_folios = 1, |
| 1594 | .pages = &page, | 1595 | .folios = &folio, |
| 1595 | .descs = &desc, | 1596 | .descs = &desc, |
| 1596 | }; | 1597 | }; |
| 1597 | char *link; | 1598 | char *link; |
| @@ -1614,7 +1615,7 @@ static int fuse_readlink_page(struct inode *inode, struct page *page) · 16 unmodified lines | |||
| 1614 | if (WARN_ON(res >= PAGE_SIZE)) | 1615 | if (WARN_ON(res >= PAGE_SIZE)) |
| 1615 | return -EIO; | 1616 | return -EIO; |
| 1616 | 1617 | ||
| 1617 | link = page_address(page); | 1618 | link = folio_address(folio); |
| 1618 | link[res] = '\0'; | 1619 | link[res] = '\0'; |
| 1619 | 1620 | ||
| 1620 | return 0; | 1621 | return 0; |
| @@ -1624,7 +1625,7 @@ static const char *fuse_get_link(struct dentry *dentry, struct inode *inode, · 3 unmodified lines | |||
| 1624 | struct delayed_call *callback) | 1625 | struct delayed_call *callback) |
| 1625 | { | 1626 | { |
| 1626 | struct fuse_conn *fc = get_fuse_conn(inode); | 1627 | struct fuse_conn *fc = get_fuse_conn(inode); |
| 1627 | struct page *page; | 1628 | struct folio *folio; |
| 1628 | int err; | 1629 | int err; |
| 1629 | 1630 | ||
| 1630 | err = -EIO; | 1631 | err = -EIO; |
| @@ -1638,20 +1639,20 @@ static const char *fuse_get_link(struct dentry *dentry, struct inode *inode, · 7 unmodified lines | |||
| 1638 | if (!dentry) | 1639 | if (!dentry) |
| 1639 | goto out_err; | 1640 | goto out_err; |
| 1640 | 1641 | ||
| 1641 | page = alloc_page(GFP_KERNEL); | 1642 | folio = folio_alloc(GFP_KERNEL, 0); |
| 1642 | err = -ENOMEM; | 1643 | err = -ENOMEM; |
| 1643 | if (!page) | 1644 | if (!folio) |
| 1644 | goto out_err; | 1645 | goto out_err; |
| 1645 | 1646 | ||
| 1646 | err = fuse_readlink_page(inode, page); | 1647 | err = fuse_readlink_page(inode, folio); |
| 1647 | if (err) { | 1648 | if (err) { |
| 1648 | __free_page(page); | 1649 | folio_put(folio); |
| 1649 | goto out_err; | 1650 | goto out_err; |
| 1650 | } | 1651 | } |
| 1651 | 1652 | ||
| 1652 | set_delayed_call(callback, page_put_link, page); | 1653 | set_delayed_call(callback, page_put_link, &folio->page); |
| 1653 | 1654 | ||
| 1654 | return page_address(page); | 1655 | return folio_address(folio); |
| 1655 | 1656 | ||
| 1656 | out_err: | 1657 | out_err: |
| 1657 | return ERR_PTR(err); | 1658 | return ERR_PTR(err); |
| @@ -2028,7 +2029,7 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, · 370 unmodified lines | |||
| 2028 | 2029 | ||
| 2029 | fuse_change_attributes_common(inode, &outarg.attr, NULL, | 2030 | fuse_change_attributes_common(inode, &outarg.attr, NULL, |
| 2030 | ATTR_TIMEOUT(&outarg), | 2031 | ATTR_TIMEOUT(&outarg), |
| 2031 | fuse_get_cache_mask(inode)); | 2032 | fuse_get_cache_mask(inode), 0); |
| 2032 | oldsize = inode->i_size; | 2033 | oldsize = inode->i_size; |
| 2033 | /* see the comment in fuse_change_attributes() */ | 2034 | /* see the comment in fuse_change_attributes() */ |
| 2034 | if (!is_wb || is_truncate) | 2035 | if (!is_wb || is_truncate) |
| @@ -2231,7 +2232,7 @@ void fuse_init_dir(struct inode *inode) · 196 unmodified lines | |||
| 2231 | 2232 | ||
| 2232 | static int fuse_symlink_read_folio(struct file *null, struct folio *folio) | 2233 | static int fuse_symlink_read_folio(struct file *null, struct folio *folio) |
| 2233 | { | 2234 | { |
| 2234 | int err = fuse_readlink_page(folio->mapping->host, &folio->page); | 2235 | int err = fuse_readlink_page(folio->mapping->host, folio); |
| 2235 | 2236 | ||
| 2236 | if (!err) | 2237 | if (!err) |
| 2237 | folio_mark_uptodate(folio); | 2238 | folio_mark_uptodate(folio); |
| @@ -436,7 +436,7 @@ static struct fuse_writepage_args *fuse_find_writeback(struct fuse_inode *fi, · 435 unmodified lines | |||
| 436 | wpa = rb_entry(n, struct fuse_writepage_args, writepages_entry); | 436 | wpa = rb_entry(n, struct fuse_writepage_args, writepages_entry); |
| 437 | WARN_ON(get_fuse_inode(wpa->inode) != fi); | 437 | WARN_ON(get_fuse_inode(wpa->inode) != fi); |
| 438 | curr_index = wpa->ia.write.in.offset >> PAGE_SHIFT; | 438 | curr_index = wpa->ia.write.in.offset >> PAGE_SHIFT; |
| 439 | if (idx_from >= curr_index + wpa->ia.ap.num_pages) | 439 | if (idx_from >= curr_index + wpa->ia.ap.num_folios) |
| 440 | n = n->rb_right; | 440 | n = n->rb_right; |
| 441 | else if (idx_to < curr_index) | 441 | else if (idx_to < curr_index) |
| 442 | n = n->rb_left; | 442 | n = n->rb_left; |
| @@ -483,6 +483,21 @@ static void fuse_wait_on_page_writeback(struct inode *inode, pgoff_t index) · 40 unmodified lines | |||
| 483 | wait_event(fi->page_waitq, !fuse_page_is_writeback(inode, index)); | 483 | wait_event(fi->page_waitq, !fuse_page_is_writeback(inode, index)); |
| 484 | } | 484 | } |
| 485 | 485 | ||
| 486 | static inline bool fuse_folio_is_writeback(struct inode *inode, | ||
| 487 | struct folio *folio) | ||
| 488 | { | ||
| 489 | pgoff_t last = folio_next_index(folio) - 1; | ||
| 490 | return fuse_range_is_writeback(inode, folio_index(folio), last); | ||
| 491 | } | ||
| 492 | |||
| 493 | static void fuse_wait_on_folio_writeback(struct inode *inode, | ||
| 494 | struct folio *folio) | ||
| 495 | { | ||
| 496 | struct fuse_inode *fi = get_fuse_inode(inode); | ||
| 497 | |||
| 498 | wait_event(fi->page_waitq, !fuse_folio_is_writeback(inode, folio)); | ||
| 499 | } | ||
| 500 | |||
| 486 | /* | 501 | /* |
| 487 | * Wait for all pending writepages on the inode to finish. | 502 | * Wait for all pending writepages on the inode to finish. |
| 488 | * | 503 | * |
| @@ -645,17 +660,20 @@ void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos, · 156 unmodified lines | |||
| 645 | args->out_args[0].size = count; | 660 | args->out_args[0].size = count; |
| 646 | } | 661 | } |
| 647 | 662 | ||
| 648 | static void fuse_release_user_pages(struct fuse_args_pages *ap, | 663 | static void fuse_release_user_pages(struct fuse_args_pages *ap, ssize_t nres, |
| 649 | bool should_dirty) | 664 | bool should_dirty) |
| 650 | { | 665 | { |
| 651 | unsigned int i; | 666 | unsigned int i; |
| 652 | 667 | ||
| 653 | for (i = 0; i < ap->num_pages; i++) { | 668 | for (i = 0; i < ap->num_folios; i++) { |
| 654 | if (should_dirty) | 669 | if (should_dirty) |
| 655 | set_page_dirty_lock(ap->pages[i]); | 670 | folio_mark_dirty_lock(ap->folios[i]); |
| 656 | if (ap->args.is_pinned) | 671 | if (ap->args.is_pinned) |
| 657 | unpin_user_page(ap->pages[i]); | 672 | unpin_folio(ap->folios[i]); |
| 658 | } | 673 | } |
| 674 | |||
| 675 | if (nres > 0 && ap->args.invalidate_vmap) | ||
| 676 | invalidate_kernel_vmap_range(ap->args.vmap_base, nres); | ||
| 659 | } | 677 | } |
| 660 | 678 | ||
| 661 | static void fuse_io_release(struct kref *kref) | 679 | static void fuse_io_release(struct kref *kref) |
| @@ -725,16 +743,16 @@ static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos) · 63 unmodified lines | |||
| 725 | } | 743 | } |
| 726 | 744 | ||
| 727 | static struct fuse_io_args *fuse_io_alloc(struct fuse_io_priv *io, | 745 | static struct fuse_io_args *fuse_io_alloc(struct fuse_io_priv *io, |
| 728 | unsigned int npages) | 746 | unsigned int nfolios) |
| 729 | { | 747 | { |
| 730 | struct fuse_io_args *ia; | 748 | struct fuse_io_args *ia; |
| 731 | 749 | ||
| 732 | ia = kzalloc(sizeof(*ia), GFP_KERNEL); | 750 | ia = kzalloc(sizeof(*ia), GFP_KERNEL); |
| 733 | if (ia) { | 751 | if (ia) { |
| 734 | ia->io = io; | 752 | ia->io = io; |
| 735 | ia->ap.pages = fuse_pages_alloc(npages, GFP_KERNEL, | 753 | ia->ap.folios = fuse_folios_alloc(nfolios, GFP_KERNEL, |
| 736 | &ia->ap.descs); | 754 | &ia->ap.descs); |
| 737 | if (!ia->ap.pages) { | 755 | if (!ia->ap.folios) { |
| 738 | kfree(ia); | 756 | kfree(ia); |
| 739 | ia = NULL; | 757 | ia = NULL; |
| 740 | } | 758 | } |
| @@ -744,7 +762,7 @@ static struct fuse_io_args *fuse_io_alloc(struct fuse_io_priv *io, · 3 unmodified lines | |||
| 744 | 762 | ||
| 745 | static void fuse_io_free(struct fuse_io_args *ia) | 763 | static void fuse_io_free(struct fuse_io_args *ia) |
| 746 | { | 764 | { |
| 747 | kfree(ia->ap.pages); | 765 | kfree(ia->ap.folios); |
| 748 | kfree(ia); | 766 | kfree(ia); |
| 749 | } | 767 | } |
| 750 | 768 | ||
| @@ -754,25 +772,29 @@ static void fuse_aio_complete_req(struct fuse_mount *fm, struct fuse_args *args, · 3 unmodified lines | |||
| 754 | struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args); | 772 | struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args); |
| 755 | struct fuse_io_priv *io = ia->io; | 773 | struct fuse_io_priv *io = ia->io; |
| 756 | ssize_t pos = -1; | 774 | ssize_t pos = -1; |
| 757 | 775 | size_t nres; | |
| 758 | fuse_release_user_pages(&ia->ap, io->should_dirty); | ||
| 759 | 776 | ||
| 760 | if (err) { | 777 | if (err) { |
| 761 | /* Nothing */ | 778 | /* Nothing */ |
| 762 | } else if (io->write) { | 779 | } else if (io->write) { |
| 763 | if (ia->write.out.size > ia->write.in.size) { | 780 | if (ia->write.out.size > ia->write.in.size) { |
| 764 | err = -EIO; | 781 | err = -EIO; |
| 765 | } else if (ia->write.in.size != ia->write.out.size) { | 782 | } else { |
| 766 | pos = ia->write.in.offset - io->offset + | 783 | nres = ia->write.out.size; |
| 767 | ia->write.out.size; | 784 | if (ia->write.in.size != ia->write.out.size) |
| 785 | pos = ia->write.in.offset - io->offset + | ||
| 786 | ia->write.out.size; | ||
| 768 | } | 787 | } |
| 769 | } else { | 788 | } else { |
| 770 | u32 outsize = args->out_args[0].size; | 789 | u32 outsize = args->out_args[0].size; |
| 771 | 790 | ||
| 791 | nres = outsize; | ||
| 772 | if (ia->read.in.size != outsize) | 792 | if (ia->read.in.size != outsize) |
| 773 | pos = ia->read.in.offset - io->offset + outsize; | 793 | pos = ia->read.in.offset - io->offset + outsize; |
| 774 | } | 794 | } |
| 775 | 795 | ||
| 796 | fuse_release_user_pages(&ia->ap, err ?: nres, io->should_dirty); | ||
| 797 | |||
| 776 | fuse_aio_complete(io, err, pos); | 798 | fuse_aio_complete(io, err, pos); |
| 777 | fuse_io_free(ia); | 799 | fuse_io_free(ia); |
| 778 | } | 800 | } |
| @@ -843,33 +865,33 @@ static void fuse_short_read(struct inode *inode, u64 attr_ver, size_t num_read, · 64 unmodified lines | |||
| 843 | * reached the client fs yet. So the hole is not present there. | 865 | * reached the client fs yet. So the hole is not present there. |
| 844 | */ | 866 | */ |
| 845 | if (!fc->writeback_cache) { | 867 | if (!fc->writeback_cache) { |
| 846 | loff_t pos = page_offset(ap->pages[0]) + num_read; | 868 | loff_t pos = folio_pos(ap->folios[0]) + num_read; |
| 847 | fuse_read_update_size(inode, pos, attr_ver); | 869 | fuse_read_update_size(inode, pos, attr_ver); |
| 848 | } | 870 | } |
| 849 | } | 871 | } |
| 850 | 872 | ||
| 851 | static int fuse_do_readpage(struct file *file, struct page *page) | 873 | static int fuse_do_readfolio(struct file *file, struct folio *folio) |
| 852 | { | 874 | { |
| 853 | struct inode *inode = page->mapping->host; | 875 | struct inode *inode = folio->mapping->host; |
| 854 | struct fuse_mount *fm = get_fuse_mount(inode); | 876 | struct fuse_mount *fm = get_fuse_mount(inode); |
| 855 | loff_t pos = page_offset(page); | 877 | loff_t pos = folio_pos(folio); |
| 856 | struct fuse_page_desc desc = { .length = PAGE_SIZE }; | 878 | struct fuse_folio_desc desc = { .length = PAGE_SIZE }; |
| 857 | struct fuse_io_args ia = { | 879 | struct fuse_io_args ia = { |
| 858 | .ap.args.page_zeroing = true, | 880 | .ap.args.page_zeroing = true, |
| 859 | .ap.args.out_pages = true, | 881 | .ap.args.out_pages = true, |
| 860 | .ap.num_pages = 1, | 882 | .ap.num_folios = 1, |
| 861 | .ap.pages = &page, | 883 | .ap.folios = &folio, |
| 862 | .ap.descs = &desc, | 884 | .ap.descs = &desc, |
| 863 | }; | 885 | }; |
| 864 | ssize_t res; | 886 | ssize_t res; |
| 865 | u64 attr_ver; | 887 | u64 attr_ver; |
| 866 | 888 | ||
| 867 | /* | 889 | /* |
| 868 | * Page writeback can extend beyond the lifetime of the | 890 | * With the temporary pages that are used to complete writeback, we can |
| 869 | * page-cache page, so make sure we read a properly synced | 891 | * have writeback that extends beyond the lifetime of the folio. So |
| 870 | * page. | 892 | * make sure we read a properly synced folio. |
| 871 | */ | 893 | */ |
| 872 | fuse_wait_on_page_writeback(inode, page->index); | 894 | fuse_wait_on_folio_writeback(inode, folio); |
| 873 | 895 | ||
| 874 | attr_ver = fuse_get_attr_version(fm->fc); | 896 | attr_ver = fuse_get_attr_version(fm->fc); |
| 875 | 897 | ||
| @@ -887,25 +909,24 @@ static int fuse_do_readpage(struct file *file, struct page *page) · 11 unmodified lines | |||
| 887 | if (res < desc.length) | 909 | if (res < desc.length) |
| 888 | fuse_short_read(inode, attr_ver, res, &ia.ap); | 910 | fuse_short_read(inode, attr_ver, res, &ia.ap); |
| 889 | 911 | ||
| 890 | SetPageUptodate(page); | 912 | folio_mark_uptodate(folio); |
| 891 | 913 | ||
| 892 | return 0; | 914 | return 0; |
| 893 | } | 915 | } |
| 894 | 916 | ||
| 895 | static int fuse_read_folio(struct file *file, struct folio *folio) | 917 | static int fuse_read_folio(struct file *file, struct folio *folio) |
| 896 | { | 918 | { |
| 897 | struct page *page = &folio->page; | 919 | struct inode *inode = folio->mapping->host; |
| 898 | struct inode *inode = page->mapping->host; | ||
| 899 | int err; | 920 | int err; |
| 900 | 921 | ||
| 901 | err = -EIO; | 922 | err = -EIO; |
| 902 | if (fuse_is_bad(inode)) | 923 | if (fuse_is_bad(inode)) |
| 903 | goto out; | 924 | goto out; |
| 904 | 925 | ||
| 905 | err = fuse_do_readpage(file, page); | 926 | err = fuse_do_readfolio(file, folio); |
| 906 | fuse_invalidate_atime(inode); | 927 | fuse_invalidate_atime(inode); |
| 907 | out: | 928 | out: |
| 908 | unlock_page(page); | 929 | folio_unlock(folio); |
| 909 | return err; | 930 | return err; |
| 910 | } | 931 | } |
| 911 | 932 | ||
| @@ -919,8 +940,8 @@ static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args, · 7 unmodified lines | |||
| 919 | size_t num_read = args->out_args[0].size; | 940 | size_t num_read = args->out_args[0].size; |
| 920 | struct address_space *mapping = NULL; | 941 | struct address_space *mapping = NULL; |
| 921 | 942 | ||
| 922 | for (i = 0; mapping == NULL && i < ap->num_pages; i++) | 943 | for (i = 0; mapping == NULL && i < ap->num_folios; i++) |
| 923 | mapping = ap->pages[i]->mapping; | 944 | mapping = ap->folios[i]->mapping; |
| 924 | 945 | ||
| 925 | if (mapping) { | 946 | if (mapping) { |
| 926 | struct inode *inode = mapping->host; | 947 | struct inode *inode = mapping->host; |
| @@ -934,12 +955,8 @@ static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args, · 7 unmodified lines | |||
| 934 | fuse_invalidate_atime(inode); | 955 | fuse_invalidate_atime(inode); |
| 935 | } | 956 | } |
| 936 | 957 | ||
| 937 | for (i = 0; i < ap->num_pages; i++) { | 958 | for (i = 0; i < ap->num_folios; i++) |
| 938 | struct folio *folio = page_folio(ap->pages[i]); | 959 | folio_end_read(ap->folios[i], !err); |
| 939 | |||
| 940 | folio_end_read(folio, !err); | ||
| 941 | folio_put(folio); | ||
| 942 | } | ||
| 943 | if (ia->ff) | 960 | if (ia->ff) |
| 944 | fuse_file_put(ia->ff, false); | 961 | fuse_file_put(ia->ff, false); |
| 945 | 962 | ||
| @@ -951,8 +968,9 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file) · 5 unmodified lines | |||
| 951 | struct fuse_file *ff = file->private_data; | 968 | struct fuse_file *ff = file->private_data; |
| 952 | struct fuse_mount *fm = ff->fm; | 969 | struct fuse_mount *fm = ff->fm; |
| 953 | struct fuse_args_pages *ap = &ia->ap; | 970 | struct fuse_args_pages *ap = &ia->ap; |
| 954 | loff_t pos = page_offset(ap->pages[0]); | 971 | loff_t pos = folio_pos(ap->folios[0]); |
| 955 | size_t count = ap->num_pages << PAGE_SHIFT; | 972 | /* Currently, all folios in FUSE are one page */ |
| 973 | size_t count = ap->num_folios << PAGE_SHIFT; | ||
| 956 | ssize_t res; | 974 | ssize_t res; |
| 957 | int err; | 975 | int err; |
| 958 | 976 | ||
| @@ -963,7 +981,7 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file) · 4 unmodified lines | |||
| 963 | /* Don't overflow end offset */ | 981 | /* Don't overflow end offset */ |
| 964 | if (pos + (count - 1) == LLONG_MAX) { | 982 | if (pos + (count - 1) == LLONG_MAX) { |
| 965 | count--; | 983 | count--; |
| 966 | ap->descs[ap->num_pages - 1].length--; | 984 | ap->descs[ap->num_folios - 1].length--; |
| 967 | } | 985 | } |
| 968 | WARN_ON((loff_t) (pos + count) < 0); | 986 | WARN_ON((loff_t) (pos + count) < 0); |
| 969 | 987 | ||
| @@ -985,18 +1003,36 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file) · 15 unmodified lines | |||
| 985 | static void fuse_readahead(struct readahead_control *rac) | 1003 | static void fuse_readahead(struct readahead_control *rac) |
| 986 | { | 1004 | { |
| 987 | struct inode *inode = rac->mapping->host; | 1005 | struct inode *inode = rac->mapping->host; |
| 1006 | struct fuse_inode *fi = get_fuse_inode(inode); | ||
| 988 | struct fuse_conn *fc = get_fuse_conn(inode); | 1007 | struct fuse_conn *fc = get_fuse_conn(inode); |
| 989 | unsigned int i, max_pages, nr_pages = 0; | 1008 | unsigned int max_pages, nr_pages; |
| 1009 | pgoff_t first = readahead_index(rac); | ||
| 1010 | pgoff_t last = first + readahead_count(rac) - 1; | ||
| 990 | 1011 | ||
| 991 | if (fuse_is_bad(inode)) | 1012 | if (fuse_is_bad(inode)) |
| 992 | return; | 1013 | return; |
| 993 | 1014 | ||
| 1015 | wait_event(fi->page_waitq, !fuse_range_is_writeback(inode, first, last)); | ||
| 1016 | |||
| 994 | max_pages = min_t(unsigned int, fc->max_pages, | 1017 | max_pages = min_t(unsigned int, fc->max_pages, |
| 995 | fc->max_read / PAGE_SIZE); | 1018 | fc->max_read / PAGE_SIZE); |
| 996 | 1019 | ||
| 997 | for (;;) { | 1020 | /* |
| 1021 | * This is only accurate the first time through, since readahead_folio() | ||
| 1022 | * doesn't update readahead_count() from the previous folio until the | ||
| 1023 | * next call. Grab nr_pages here so we know how many pages we're going | ||
| 1024 | * to have to process. This means that we will exit here with | ||
| 1025 | * readahead_count() == folio_nr_pages(last_folio), but we will have | ||
| 1026 | * consumed all of the folios, and read_pages() will call | ||
| 1027 | * readahead_folio() again which will clean up the rac. | ||
| 1028 | */ | ||
| 1029 | nr_pages = readahead_count(rac); | ||
| 1030 | |||
| 1031 | while (nr_pages) { | ||
| 998 | struct fuse_io_args *ia; | 1032 | struct fuse_io_args *ia; |
| 999 | struct fuse_args_pages *ap; | 1033 | struct fuse_args_pages *ap; |
| 1034 | struct folio *folio; | ||
| 1035 | unsigned cur_pages = min(max_pages, nr_pages); | ||
| 1000 | 1036 | ||
| 1001 | if (fc->num_background >= fc->congestion_threshold && | 1037 | if (fc->num_background >= fc->congestion_threshold && |
| 1002 | rac->ra->async_size >= readahead_count(rac)) | 1038 | rac->ra->async_size >= readahead_count(rac)) |
| @@ -1006,23 +1042,19 @@ static void fuse_readahead(struct readahead_control *rac) · 3 unmodified lines | |||
| 1006 | */ | 1042 | */ |
| 1007 | break; | 1043 | break; |
| 1008 | 1044 | ||
| 1009 | nr_pages = readahead_count(rac) - nr_pages; | 1045 | ia = fuse_io_alloc(NULL, cur_pages); |
| 1010 | if (nr_pages > max_pages) | ||
| 1011 | nr_pages = max_pages; | ||
| 1012 | if (nr_pages == 0) | ||
| 1013 | break; | ||
| 1014 | ia = fuse_io_alloc(NULL, nr_pages); | ||
| 1015 | if (!ia) | 1046 | if (!ia) |
| 1016 | return; | 1047 | return; |
| 1017 | ap = &ia->ap; | 1048 | ap = &ia->ap; |
| 1018 | nr_pages = __readahead_batch(rac, ap->pages, nr_pages); | 1049 | |
| 1019 | for (i = 0; i < nr_pages; i++) { | 1050 | while (ap->num_folios < cur_pages) { |
| 1020 | fuse_wait_on_page_writeback(inode, | 1051 | folio = readahead_folio(rac); |
| 1021 | readahead_index(rac) + i); | 1052 | ap->folios[ap->num_folios] = folio; |
| 1022 | ap->descs[i].length = PAGE_SIZE; | 1053 | ap->descs[ap->num_folios].length = folio_size(folio); |
| 1054 | ap->num_folios++; | ||
| 1023 | } | 1055 | } |
| 1024 | ap->num_pages = nr_pages; | ||
| 1025 | fuse_send_readpages(ia, rac->file); | 1056 | fuse_send_readpages(ia, rac->file); |
| 1057 | nr_pages -= cur_pages; | ||
| 1026 | } | 1058 | } |
| 1027 | } | 1059 | } |
| 1028 | 1060 | ||
| @@ -1139,8 +1171,8 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, · 110 unmodified lines | |||
| 1139 | bool short_write; | 1171 | bool short_write; |
| 1140 | int err; | 1172 | int err; |
| 1141 | 1173 | ||
| 1142 | for (i = 0; i < ap->num_pages; i++) | 1174 | for (i = 0; i < ap->num_folios; i++) |
| 1143 | fuse_wait_on_page_writeback(inode, ap->pages[i]->index); | 1175 | fuse_wait_on_folio_writeback(inode, ap->folios[i]); |
| 1144 | 1176 | ||
| 1145 | fuse_write_args_fill(ia, ff, pos, count); | 1177 | fuse_write_args_fill(ia, ff, pos, count); |
| 1146 | ia->write.in.flags = fuse_write_flags(iocb); | 1178 | ia->write.in.flags = fuse_write_flags(iocb); |
| @@ -1154,24 +1186,24 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, · 7 unmodified lines | |||
| 1154 | short_write = ia->write.out.size < count; | 1186 | short_write = ia->write.out.size < count; |
| 1155 | offset = ap->descs[0].offset; | 1187 | offset = ap->descs[0].offset; |
| 1156 | count = ia->write.out.size; | 1188 | count = ia->write.out.size; |
| 1157 | for (i = 0; i < ap->num_pages; i++) { | 1189 | for (i = 0; i < ap->num_folios; i++) { |
| 1158 | struct page *page = ap->pages[i]; | 1190 | struct folio *folio = ap->folios[i]; |
| 1159 | 1191 | ||
| 1160 | if (err) { | 1192 | if (err) { |
| 1161 | ClearPageUptodate(page); | 1193 | folio_clear_uptodate(folio); |
| 1162 | } else { | 1194 | } else { |
| 1163 | if (count >= PAGE_SIZE - offset) | 1195 | if (count >= folio_size(folio) - offset) |
| 1164 | count -= PAGE_SIZE - offset; | 1196 | count -= folio_size(folio) - offset; |
| 1165 | else { | 1197 | else { |
| 1166 | if (short_write) | 1198 | if (short_write) |
| 1167 | ClearPageUptodate(page); | 1199 | folio_clear_uptodate(folio); |
| 1168 | count = 0; | 1200 | count = 0; |
| 1169 | } | 1201 | } |
| 1170 | offset = 0; | 1202 | offset = 0; |
| 1171 | } | 1203 | } |
| 1172 | if (ia->write.page_locked && (i == ap->num_pages - 1)) | 1204 | if (ia->write.folio_locked && (i == ap->num_folios - 1)) |
| 1173 | unlock_page(page); | 1205 | folio_unlock(folio); |
| 1174 | put_page(page); | 1206 | folio_put(folio); |
| 1175 | } | 1207 | } |
| 1176 | 1208 | ||
| 1177 | return err; | 1209 | return err; |
| @@ -1185,6 +1217,7 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, · 7 unmodified lines | |||
| 1185 | struct fuse_args_pages *ap = &ia->ap; | 1217 | struct fuse_args_pages *ap = &ia->ap; |
| 1186 | struct fuse_conn *fc = get_fuse_conn(mapping->host); | 1218 | struct fuse_conn *fc = get_fuse_conn(mapping->host); |
| 1187 | unsigned offset = pos & (PAGE_SIZE - 1); | 1219 | unsigned offset = pos & (PAGE_SIZE - 1); |
| 1220 | unsigned int nr_pages = 0; | ||
| 1188 | size_t count = 0; | 1221 | size_t count = 0; |
| 1189 | int err; | 1222 | int err; |
| 1190 | 1223 | ||
| @@ -1193,7 +1226,7 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, · 2 unmodified lines | |||
| 1193 | 1226 | ||
| 1194 | do { | 1227 | do { |
| 1195 | size_t tmp; | 1228 | size_t tmp; |
| 1196 | struct page *page; | 1229 | struct folio *folio; |
| 1197 | pgoff_t index = pos >> PAGE_SHIFT; | 1230 | pgoff_t index = pos >> PAGE_SHIFT; |
| 1198 | size_t bytes = min_t(size_t, PAGE_SIZE - offset, | 1231 | size_t bytes = min_t(size_t, PAGE_SIZE - offset, |
| 1199 | iov_iter_count(ii)); | 1232 | iov_iter_count(ii)); |
| @@ -1205,27 +1238,30 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, · 5 unmodified lines | |||
| 1205 | if (fault_in_iov_iter_readable(ii, bytes)) | 1238 | if (fault_in_iov_iter_readable(ii, bytes)) |
| 1206 | break; | 1239 | break; |
| 1207 | 1240 | ||
| 1208 | err = -ENOMEM; | 1241 | folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN, |
| 1209 | page = grab_cache_page_write_begin(mapping, index); | 1242 | mapping_gfp_mask(mapping)); |
| 1210 | if (!page) | 1243 | if (IS_ERR(folio)) { |
| 1244 | err = PTR_ERR(folio); | ||
| 1211 | break; | 1245 | break; |
| 1246 | } | ||
| 1212 | 1247 | ||
| 1213 | if (mapping_writably_mapped(mapping)) | 1248 | if (mapping_writably_mapped(mapping)) |
| 1214 | flush_dcache_page(page); | 1249 | flush_dcache_folio(folio); |
| 1215 | 1250 | ||
| 1216 | tmp = copy_page_from_iter_atomic(page, offset, bytes, ii); | 1251 | tmp = copy_folio_from_iter_atomic(folio, offset, bytes, ii); |
| 1217 | flush_dcache_page(page); | 1252 | flush_dcache_folio(folio); |
| 1218 | 1253 | ||
| 1219 | if (!tmp) { | 1254 | if (!tmp) { |
| 1220 | unlock_page(page); | 1255 | folio_unlock(folio); |
| 1221 | put_page(page); | 1256 | folio_put(folio); |
| 1222 | goto again; | 1257 | goto again; |
| 1223 | } | 1258 | } |
| 1224 | 1259 | ||
| 1225 | err = 0; | 1260 | err = 0; |
| 1226 | ap->pages[ap->num_pages] = page; | 1261 | ap->folios[ap->num_folios] = folio; |
| 1227 | ap->descs[ap->num_pages].length = tmp; | 1262 | ap->descs[ap->num_folios].length = tmp; |
| 1228 | ap->num_pages++; | 1263 | ap->num_folios++; |
| 1264 | nr_pages++; | ||
| 1229 | 1265 | ||
| 1230 | count += tmp; | 1266 | count += tmp; |
| 1231 | pos += tmp; | 1267 | pos += tmp; |
| @@ -1235,18 +1271,18 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, · 3 unmodified lines | |||
| 1235 | 1271 | ||
| 1236 | /* If we copied full page, mark it uptodate */ | 1272 | /* If we copied full page, mark it uptodate */ |
| 1237 | if (tmp == PAGE_SIZE) | 1273 | if (tmp == PAGE_SIZE) |
| 1238 | SetPageUptodate(page); | 1274 | folio_mark_uptodate(folio); |
| 1239 | 1275 | ||
| 1240 | if (PageUptodate(page)) { | 1276 | if (folio_test_uptodate(folio)) { |
| 1241 | unlock_page(page); | 1277 | folio_unlock(folio); |
| 1242 | } else { | 1278 | } else { |
| 1243 | ia->write.page_locked = true; | 1279 | ia->write.folio_locked = true; |
| 1244 | break; | 1280 | break; |
| 1245 | } | 1281 | } |
| 1246 | if (!fc->big_writes) | 1282 | if (!fc->big_writes) |
| 1247 | break; | 1283 | break; |
| 1248 | } while (iov_iter_count(ii) && count < fc->max_write && | 1284 | } while (iov_iter_count(ii) && count < fc->max_write && |
| 1249 | ap->num_pages < max_pages && offset == 0); | 1285 | nr_pages < max_pages && offset == 0); |
| 1250 | 1286 | ||
| 1251 | return count > 0 ? count : err; | 1287 | return count > 0 ? count : err; |
| 1252 | } | 1288 | } |
| @@ -1280,8 +1316,8 @@ static ssize_t fuse_perform_write(struct kiocb *iocb, struct iov_iter *ii) · 27 unmodified lines | |||
| 1280 | unsigned int nr_pages = fuse_wr_pages(pos, iov_iter_count(ii), | 1316 | unsigned int nr_pages = fuse_wr_pages(pos, iov_iter_count(ii), |
| 1281 | fc->max_pages); | 1317 | fc->max_pages); |
| 1282 | 1318 | ||
| 1283 | ap->pages = fuse_pages_alloc(nr_pages, GFP_KERNEL, &ap->descs); | 1319 | ap->folios = fuse_folios_alloc(nr_pages, GFP_KERNEL, &ap->descs); |
| 1284 | if (!ap->pages) { | 1320 | if (!ap->folios) { |
| 1285 | err = -ENOMEM; | 1321 | err = -ENOMEM; |
| 1286 | break; | 1322 | break; |
| 1287 | } | 1323 | } |
| @@ -1303,7 +1339,7 @@ static ssize_t fuse_perform_write(struct kiocb *iocb, struct iov_iter *ii) · 15 unmodified lines | |||
| 1303 | err = -EIO; | 1339 | err = -EIO; |
| 1304 | } | 1340 | } |
| 1305 | } | 1341 | } |
| 1306 | kfree(ap->pages); | 1342 | kfree(ap->folios); |
| 1307 | } while (!err && iov_iter_count(ii)); | 1343 | } while (!err && iov_iter_count(ii)); |
| 1308 | 1344 | ||
| 1309 | fuse_write_update_attr(inode, pos, res); | 1345 | fuse_write_update_attr(inode, pos, res); |
| @@ -1430,11 +1466,7 @@ writethrough: · 120 unmodified lines | |||
| 1430 | 1466 | ||
| 1431 | task_io_account_write(count); | 1467 | task_io_account_write(count); |
| 1432 | 1468 | ||
| 1433 | err = file_remove_privs(file); | 1469 | err = kiocb_modified(iocb); |
| 1434 | if (err) | ||
| 1435 | goto out; | ||
| 1436 | |||
| 1437 | err = file_update_time(file); | ||
| 1438 | if (err) | 1470 | if (err) |
| 1439 | goto out; | 1471 | goto out; |
| 1440 | 1472 | ||
| @@ -1468,35 +1500,57 @@ static inline size_t fuse_get_frag_size(const struct iov_iter *ii, · 27 unmodified lines | |||
| 1468 | 1500 | ||
| 1469 | static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii, | 1501 | static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii, |
| 1470 | size_t *nbytesp, int write, | 1502 | size_t *nbytesp, int write, |
| 1471 | unsigned int max_pages) | 1503 | unsigned int max_pages, |
| 1504 | bool use_pages_for_kvec_io) | ||
| 1472 | { | 1505 | { |
| 1506 | bool flush_or_invalidate = false; | ||
| 1507 | unsigned int nr_pages = 0; | ||
| 1473 | size_t nbytes = 0; /* # bytes already packed in req */ | 1508 | size_t nbytes = 0; /* # bytes already packed in req */ |
| 1474 | ssize_t ret = 0; | 1509 | ssize_t ret = 0; |
| 1475 | 1510 | ||
| 1476 | /* Special case for kernel I/O: can copy directly into the buffer */ | 1511 | /* Special case for kernel I/O: can copy directly into the buffer. |
| 1512 | * However if the implementation of fuse_conn requires pages instead of | ||
| 1513 | * pointer (e.g., virtio-fs), use iov_iter_extract_pages() instead. | ||
| 1514 | */ | ||
| 1477 | if (iov_iter_is_kvec(ii)) { | 1515 | if (iov_iter_is_kvec(ii)) { |
| 1478 | unsigned long user_addr = fuse_get_user_addr(ii); | 1516 | void *user_addr = (void *)fuse_get_user_addr(ii); |
| 1479 | size_t frag_size = fuse_get_frag_size(ii, *nbytesp); | ||
| 1480 | 1517 | ||
| 1481 | if (write) | 1518 | if (!use_pages_for_kvec_io) { |
| 1482 | ap->args.in_args[1].value = (void *) user_addr; | 1519 | size_t frag_size = fuse_get_frag_size(ii, *nbytesp); |
| 1483 | else | ||
| 1484 | ap->args.out_args[0].value = (void *) user_addr; | ||
| 1485 | 1520 | ||
| 1486 | iov_iter_advance(ii, frag_size); | 1521 | if (write) |
| 1487 | *nbytesp = frag_size; | 1522 | ap->args.in_args[1].value = user_addr; |
| 1488 | return 0; | 1523 | else |
| 1524 | ap->args.out_args[0].value = user_addr; | ||
| 1525 | |||
| 1526 | iov_iter_advance(ii, frag_size); | ||
| 1527 | *nbytesp = frag_size; | ||
| 1528 | return 0; | ||
| 1529 | } | ||
| 1530 | |||
| 1531 | if (is_vmalloc_addr(user_addr)) { | ||
| 1532 | ap->args.vmap_base = user_addr; | ||
| 1533 | flush_or_invalidate = true; | ||
| 1534 | } | ||
| 1489 | } | 1535 | } |
| 1490 | 1536 | ||
| 1491 | while (nbytes < *nbytesp && ap->num_pages < max_pages) { | 1537 | /* |
| 1492 | unsigned npages; | 1538 | * Until there is support for iov_iter_extract_folios(), we have to |
| 1539 | * manually extract pages using iov_iter_extract_pages() and then | ||
| 1540 | * copy that to a folios array. | ||
| 1541 | */ | ||
| 1542 | struct page **pages = kzalloc(max_pages * sizeof(struct page *), | ||
| 1543 | GFP_KERNEL); | ||
| 1544 | if (!pages) | ||
| 1545 | return -ENOMEM; | ||
| 1546 | |||
| 1547 | while (nbytes < *nbytesp && nr_pages < max_pages) { | ||
| 1548 | unsigned nfolios, i; | ||
| 1493 | size_t start; | 1549 | size_t start; |
| 1494 | struct page **pt_pages; | ||
| 1495 | 1550 | ||
| 1496 | pt_pages = &ap->pages[ap->num_pages]; | 1551 | ret = iov_iter_extract_pages(ii, &pages, |
| 1497 | ret = iov_iter_extract_pages(ii, &pt_pages, | ||
| 1498 | *nbytesp - nbytes, | 1552 | *nbytesp - nbytes, |
| 1499 | max_pages - ap->num_pages, | 1553 | max_pages - nr_pages, |
| 1500 | 0, &start); | 1554 | 0, &start); |
| 1501 | if (ret < 0) | 1555 | if (ret < 0) |
| 1502 | break; | 1556 | break; |
| @@ -1504,16 +1558,25 @@ static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii, · 1 unmodified line | |||
| 1504 | nbytes += ret; | 1558 | nbytes += ret; |
| 1505 | 1559 | ||
| 1506 | ret += start; | 1560 | ret += start; |
| 1507 | npages = DIV_ROUND_UP(ret, PAGE_SIZE); | 1561 | /* Currently, all folios in FUSE are one page */ |
| 1562 | nfolios = DIV_ROUND_UP(ret, PAGE_SIZE); | ||
| 1508 | 1563 | ||
| 1509 | ap->descs[ap->num_pages].offset = start; | 1564 | ap->descs[ap->num_folios].offset = start; |
| 1510 | fuse_page_descs_length_init(ap->descs, ap->num_pages, npages); | 1565 | fuse_folio_descs_length_init(ap->descs, ap->num_folios, nfolios); |
| 1566 | for (i = 0; i < nfolios; i++) | ||
| 1567 | ap->folios[i + ap->num_folios] = page_folio(pages[i]); | ||
| 1511 | 1568 | ||
| 1512 | ap->num_pages += npages; | 1569 | ap->num_folios += nfolios; |
| 1513 | ap->descs[ap->num_pages - 1].length -= | 1570 | ap->descs[ap->num_folios - 1].length -= |
| 1514 | (PAGE_SIZE - ret) & (PAGE_SIZE - 1); | 1571 | (PAGE_SIZE - ret) & (PAGE_SIZE - 1); |
| 1572 | nr_pages += nfolios; | ||
| 1515 | } | 1573 | } |
| 1574 | kfree(pages); | ||
| 1575 | |||
| 1576 | if (write && flush_or_invalidate) | ||
| 1577 | flush_kernel_vmap_range(ap->args.vmap_base, nbytes); | ||
| 1516 | 1578 | ||
| 1579 | ap->args.invalidate_vmap = !write && flush_or_invalidate; | ||
| 1517 | ap->args.is_pinned = iov_iter_extract_will_pin(ii); | 1580 | ap->args.is_pinned = iov_iter_extract_will_pin(ii); |
| 1518 | ap->args.user_pages = true; | 1581 | ap->args.user_pages = true; |
| 1519 | if (write) | 1582 | if (write) |
| @@ -1582,7 +1645,7 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, · 62 unmodified lines | |||
| 1582 | size_t nbytes = min(count, nmax); | 1645 | size_t nbytes = min(count, nmax); |
| 1583 | 1646 | ||
| 1584 | err = fuse_get_user_pages(&ia->ap, iter, &nbytes, write, | 1647 | err = fuse_get_user_pages(&ia->ap, iter, &nbytes, write, |
| 1585 | max_pages); | 1648 | max_pages, fc->use_pages_for_kvec_io); |
| 1586 | if (err && !nbytes) | 1649 | if (err && !nbytes) |
| 1587 | break; | 1650 | break; |
| 1588 | 1651 | ||
| @@ -1596,7 +1659,7 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, · 7 unmodified lines | |||
| 1596 | } | 1659 | } |
| 1597 | 1660 | ||
| 1598 | if (!io->async || nres < 0) { | 1661 | if (!io->async || nres < 0) { |
| 1599 | fuse_release_user_pages(&ia->ap, io->should_dirty); | 1662 | fuse_release_user_pages(&ia->ap, nres, io->should_dirty); |
| 1600 | fuse_io_free(ia); | 1663 | fuse_io_free(ia); |
| 1601 | } | 1664 | } |
| 1602 | ia = NULL; | 1665 | ia = NULL; |
| @@ -1650,7 +1713,7 @@ static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to) · 47 unmodified lines | |||
| 1650 | { | 1713 | { |
| 1651 | ssize_t res; | 1714 | ssize_t res; |
| 1652 | 1715 | ||
| 1653 | if (!is_sync_kiocb(iocb) && iocb->ki_flags & IOCB_DIRECT) { | 1716 | if (!is_sync_kiocb(iocb)) { |
| 1654 | res = fuse_direct_IO(iocb, to); | 1717 | res = fuse_direct_IO(iocb, to); |
| 1655 | } else { | 1718 | } else { |
| 1656 | struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); | 1719 | struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); |
| @@ -1664,7 +1727,6 @@ static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to) · 7 unmodified lines | |||
| 1664 | static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) | 1727 | static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) |
| 1665 | { | 1728 | { |
| 1666 | struct inode *inode = file_inode(iocb->ki_filp); | 1729 | struct inode *inode = file_inode(iocb->ki_filp); |
| 1667 | struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); | ||
| 1668 | ssize_t res; | 1730 | ssize_t res; |
| 1669 | bool exclusive; | 1731 | bool exclusive; |
| 1670 | 1732 | ||
| @@ -1672,9 +1734,11 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) · 1 unmodified line | |||
| 1672 | res = generic_write_checks(iocb, from); | 1734 | res = generic_write_checks(iocb, from); |
| 1673 | if (res > 0) { | 1735 | if (res > 0) { |
| 1674 | task_io_account_write(res); | 1736 | task_io_account_write(res); |
| 1675 | if (!is_sync_kiocb(iocb) && iocb->ki_flags & IOCB_DIRECT) { | 1737 | if (!is_sync_kiocb(iocb)) { |
| 1676 | res = fuse_direct_IO(iocb, from); | 1738 | res = fuse_direct_IO(iocb, from); |
| 1677 | } else { | 1739 | } else { |
| 1740 | struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); | ||
| 1741 | |||
| 1678 | res = fuse_direct_io(&io, from, &iocb->ki_pos, | 1742 | res = fuse_direct_io(&io, from, &iocb->ki_pos, |
| 1679 | FUSE_DIO_WRITE); | 1743 | FUSE_DIO_WRITE); |
| 1680 | fuse_write_update_attr(inode, iocb->ki_pos, res); | 1744 | fuse_write_update_attr(inode, iocb->ki_pos, res); |
| @@ -1760,21 +1824,21 @@ static void fuse_writepage_free(struct fuse_writepage_args *wpa) · 79 unmodified lines | |||
| 1760 | if (wpa->bucket) | 1824 | if (wpa->bucket) |
| 1761 | fuse_sync_bucket_dec(wpa->bucket); | 1825 | fuse_sync_bucket_dec(wpa->bucket); |
| 1762 | 1826 | ||
| 1763 | for (i = 0; i < ap->num_pages; i++) | 1827 | for (i = 0; i < ap->num_folios; i++) |
| 1764 | __free_page(ap->pages[i]); | 1828 | folio_put(ap->folios[i]); |
| 1765 | 1829 | ||
| 1766 | fuse_file_put(wpa->ia.ff, false); | 1830 | fuse_file_put(wpa->ia.ff, false); |
| 1767 | 1831 | ||
| 1768 | kfree(ap->pages); | 1832 | kfree(ap->folios); |
| 1769 | kfree(wpa); | 1833 | kfree(wpa); |
| 1770 | } | 1834 | } |
| 1771 | 1835 | ||
| 1772 | static void fuse_writepage_finish_stat(struct inode *inode, struct page *page) | 1836 | static void fuse_writepage_finish_stat(struct inode *inode, struct folio *folio) |
| 1773 | { | 1837 | { |
| 1774 | struct backing_dev_info *bdi = inode_to_bdi(inode); | 1838 | struct backing_dev_info *bdi = inode_to_bdi(inode); |
| 1775 | 1839 | ||
| 1776 | dec_wb_stat(&bdi->wb, WB_WRITEBACK); | 1840 | dec_wb_stat(&bdi->wb, WB_WRITEBACK); |
| 1777 | dec_node_page_state(page, NR_WRITEBACK_TEMP); | 1841 | node_stat_sub_folio(folio, NR_WRITEBACK_TEMP); |
| 1778 | wb_writeout_inc(&bdi->wb); | 1842 | wb_writeout_inc(&bdi->wb); |
| 1779 | } | 1843 | } |
| 1780 | 1844 | ||
| @@ -1785,8 +1849,8 @@ static void fuse_writepage_finish(struct fuse_writepage_args *wpa) · 4 unmodified lines | |||
| 1785 | struct fuse_inode *fi = get_fuse_inode(inode); | 1849 | struct fuse_inode *fi = get_fuse_inode(inode); |
| 1786 | int i; | 1850 | int i; |
| 1787 | 1851 | ||
| 1788 | for (i = 0; i < ap->num_pages; i++) | 1852 | for (i = 0; i < ap->num_folios; i++) |
| 1789 | fuse_writepage_finish_stat(inode, ap->pages[i]); | 1853 | fuse_writepage_finish_stat(inode, ap->folios[i]); |
| 1790 | 1854 | ||
| 1791 | wake_up(&fi->page_waitq); | 1855 | wake_up(&fi->page_waitq); |
| 1792 | } | 1856 | } |
| @@ -1801,7 +1865,8 @@ __acquires(fi->lock) · 8 unmodified lines | |||
| 1801 | struct fuse_inode *fi = get_fuse_inode(wpa->inode); | 1865 | struct fuse_inode *fi = get_fuse_inode(wpa->inode); |
| 1802 | struct fuse_write_in *inarg = &wpa->ia.write.in; | 1866 | struct fuse_write_in *inarg = &wpa->ia.write.in; |
| 1803 | struct fuse_args *args = &wpa->ia.ap.args; | 1867 | struct fuse_args *args = &wpa->ia.ap.args; |
| 1804 | __u64 data_size = wpa->ia.ap.num_pages * PAGE_SIZE; | 1868 | /* Currently, all folios in FUSE are one page */ |
| 1869 | __u64 data_size = wpa->ia.ap.num_folios * PAGE_SIZE; | ||
| 1805 | int err; | 1870 | int err; |
| 1806 | 1871 | ||
| 1807 | fi->writectr++; | 1872 | fi->writectr++; |
| @@ -1841,7 +1906,8 @@ __acquires(fi->lock) · 33 unmodified lines | |||
| 1841 | for (aux = wpa->next; aux; aux = next) { | 1906 | for (aux = wpa->next; aux; aux = next) { |
| 1842 | next = aux->next; | 1907 | next = aux->next; |
| 1843 | aux->next = NULL; | 1908 | aux->next = NULL; |
| 1844 | fuse_writepage_finish_stat(aux->inode, aux->ia.ap.pages[0]); | 1909 | fuse_writepage_finish_stat(aux->inode, |
| 1910 | aux->ia.ap.folios[0]); | ||
| 1845 | fuse_writepage_free(aux); | 1911 | fuse_writepage_free(aux); |
| 1846 | } | 1912 | } |
| 1847 | 1913 | ||
| @@ -1876,11 +1942,11 @@ static struct fuse_writepage_args *fuse_insert_writeback(struct rb_root *root, · 28 unmodified lines | |||
| 1876 | struct fuse_writepage_args *wpa) | 1942 | struct fuse_writepage_args *wpa) |
| 1877 | { | 1943 | { |
| 1878 | pgoff_t idx_from = wpa->ia.write.in.offset >> PAGE_SHIFT; | 1944 | pgoff_t idx_from = wpa->ia.write.in.offset >> PAGE_SHIFT; |
| 1879 | pgoff_t idx_to = idx_from + wpa->ia.ap.num_pages - 1; | 1945 | pgoff_t idx_to = idx_from + wpa->ia.ap.num_folios - 1; |
| 1880 | struct rb_node **p = &root->rb_node; | 1946 | struct rb_node **p = &root->rb_node; |
| 1881 | struct rb_node *parent = NULL; | 1947 | struct rb_node *parent = NULL; |
| 1882 | 1948 | ||
| 1883 | WARN_ON(!wpa->ia.ap.num_pages); | 1949 | WARN_ON(!wpa->ia.ap.num_folios); |
| 1884 | while (*p) { | 1950 | while (*p) { |
| 1885 | struct fuse_writepage_args *curr; | 1951 | struct fuse_writepage_args *curr; |
| 1886 | pgoff_t curr_index; | 1952 | pgoff_t curr_index; |
| @@ -1891,7 +1957,7 @@ static struct fuse_writepage_args *fuse_insert_writeback(struct rb_root *root, · 4 unmodified lines | |||
| 1891 | WARN_ON(curr->inode != wpa->inode); | 1957 | WARN_ON(curr->inode != wpa->inode); |
| 1892 | curr_index = curr->ia.write.in.offset >> PAGE_SHIFT; | 1958 | curr_index = curr->ia.write.in.offset >> PAGE_SHIFT; |
| 1893 | 1959 | ||
| 1894 | if (idx_from >= curr_index + curr->ia.ap.num_pages) | 1960 | if (idx_from >= curr_index + curr->ia.ap.num_folios) |
| 1895 | p = &(*p)->rb_right; | 1961 | p = &(*p)->rb_right; |
| 1896 | else if (idx_to < curr_index) | 1962 | else if (idx_to < curr_index) |
| 1897 | p = &(*p)->rb_left; | 1963 | p = &(*p)->rb_left; |
| @@ -2023,9 +2089,9 @@ static struct fuse_writepage_args *fuse_writepage_args_alloc(void) · 125 unmodified lines | |||
| 2023 | wpa = kzalloc(sizeof(*wpa), GFP_NOFS); | 2089 | wpa = kzalloc(sizeof(*wpa), GFP_NOFS); |
| 2024 | if (wpa) { | 2090 | if (wpa) { |
| 2025 | ap = &wpa->ia.ap; | 2091 | ap = &wpa->ia.ap; |
| 2026 | ap->num_pages = 0; | 2092 | ap->num_folios = 0; |
| 2027 | ap->pages = fuse_pages_alloc(1, GFP_NOFS, &ap->descs); | 2093 | ap->folios = fuse_folios_alloc(1, GFP_NOFS, &ap->descs); |
| 2028 | if (!ap->pages) { | 2094 | if (!ap->folios) { |
| 2029 | kfree(wpa); | 2095 | kfree(wpa); |
| 2030 | wpa = NULL; | 2096 | wpa = NULL; |
| 2031 | } | 2097 | } |
| @@ -2049,19 +2115,19 @@ static void fuse_writepage_add_to_bucket(struct fuse_conn *fc, · 17 unmodified lines | |||
| 2049 | } | 2115 | } |
| 2050 | 2116 | ||
| 2051 | static void fuse_writepage_args_page_fill(struct fuse_writepage_args *wpa, struct folio *folio, | 2117 | static void fuse_writepage_args_page_fill(struct fuse_writepage_args *wpa, struct folio *folio, |
| 2052 | struct folio *tmp_folio, uint32_t page_index) | 2118 | struct folio *tmp_folio, uint32_t folio_index) |
| 2053 | { | 2119 | { |
| 2054 | struct inode *inode = folio->mapping->host; | 2120 | struct inode *inode = folio->mapping->host; |
| 2055 | struct fuse_args_pages *ap = &wpa->ia.ap; | 2121 | struct fuse_args_pages *ap = &wpa->ia.ap; |
| 2056 | 2122 | ||
| 2057 | folio_copy(tmp_folio, folio); | 2123 | folio_copy(tmp_folio, folio); |
| 2058 | 2124 | ||
| 2059 | ap->pages[page_index] = &tmp_folio->page; | 2125 | ap->folios[folio_index] = tmp_folio; |
| 2060 | ap->descs[page_index].offset = 0; | 2126 | ap->descs[folio_index].offset = 0; |
| 2061 | ap->descs[page_index].length = PAGE_SIZE; | 2127 | ap->descs[folio_index].length = PAGE_SIZE; |
| 2062 | 2128 | ||
| 2063 | inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK); | 2129 | inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK); |
| 2064 | inc_node_page_state(&tmp_folio->page, NR_WRITEBACK_TEMP); | 2130 | node_stat_add_folio(tmp_folio, NR_WRITEBACK_TEMP); |
| 2065 | } | 2131 | } |
| 2066 | 2132 | ||
| 2067 | static struct fuse_writepage_args *fuse_writepage_args_setup(struct folio *folio, | 2133 | static struct fuse_writepage_args *fuse_writepage_args_setup(struct folio *folio, |
| @@ -2115,7 +2181,7 @@ static int fuse_writepage_locked(struct folio *folio) · 47 unmodified lines | |||
| 2115 | goto err_writepage_args; | 2181 | goto err_writepage_args; |
| 2116 | 2182 | ||
| 2117 | ap = &wpa->ia.ap; | 2183 | ap = &wpa->ia.ap; |
| 2118 | ap->num_pages = 1; | 2184 | ap->num_folios = 1; |
| 2119 | 2185 | ||
| 2120 | folio_start_writeback(folio); | 2186 | folio_start_writeback(folio); |
| 2121 | fuse_writepage_args_page_fill(wpa, folio, tmp_folio, 0); | 2187 | fuse_writepage_args_page_fill(wpa, folio, tmp_folio, 0); |
| @@ -2143,32 +2209,32 @@ struct fuse_fill_wb_data { · 21 unmodified lines | |||
| 2143 | struct fuse_writepage_args *wpa; | 2209 | struct fuse_writepage_args *wpa; |
| 2144 | struct fuse_file *ff; | 2210 | struct fuse_file *ff; |
| 2145 | struct inode *inode; | 2211 | struct inode *inode; |
| 2146 | struct page **orig_pages; | 2212 | struct folio **orig_folios; |
| 2147 | unsigned int max_pages; | 2213 | unsigned int max_folios; |
| 2148 | }; | 2214 | }; |
| 2149 | 2215 | ||
| 2150 | static bool fuse_pages_realloc(struct fuse_fill_wb_data *data) | 2216 | static bool fuse_pages_realloc(struct fuse_fill_wb_data *data) |
| 2151 | { | 2217 | { |
| 2152 | struct fuse_args_pages *ap = &data->wpa->ia.ap; | 2218 | struct fuse_args_pages *ap = &data->wpa->ia.ap; |
| 2153 | struct fuse_conn *fc = get_fuse_conn(data->inode); | 2219 | struct fuse_conn *fc = get_fuse_conn(data->inode); |
| 2154 | struct page **pages; | 2220 | struct folio **folios; |
| 2155 | struct fuse_page_desc *descs; | 2221 | struct fuse_folio_desc *descs; |
| 2156 | unsigned int npages = min_t(unsigned int, | 2222 | unsigned int nfolios = min_t(unsigned int, |
| 2157 | max_t(unsigned int, data->max_pages * 2, | 2223 | max_t(unsigned int, data->max_folios * 2, |
| 2158 | FUSE_DEFAULT_MAX_PAGES_PER_REQ), | 2224 | FUSE_DEFAULT_MAX_PAGES_PER_REQ), |
| 2159 | fc->max_pages); | 2225 | fc->max_pages); |
| 2160 | WARN_ON(npages <= data->max_pages); | 2226 | WARN_ON(nfolios <= data->max_folios); |
| 2161 | 2227 | ||
| 2162 | pages = fuse_pages_alloc(npages, GFP_NOFS, &descs); | 2228 | folios = fuse_folios_alloc(nfolios, GFP_NOFS, &descs); |
| 2163 | if (!pages) | 2229 | if (!folios) |
| 2164 | return false; | 2230 | return false; |
| 2165 | 2231 | ||
| 2166 | memcpy(pages, ap->pages, sizeof(struct page *) * ap->num_pages); | 2232 | memcpy(folios, ap->folios, sizeof(struct folio *) * ap->num_folios); |
| 2167 | memcpy(descs, ap->descs, sizeof(struct fuse_page_desc) * ap->num_pages); | 2233 | memcpy(descs, ap->descs, sizeof(struct fuse_folio_desc) * ap->num_folios); |
| 2168 | kfree(ap->pages); | 2234 | kfree(ap->folios); |
| 2169 | ap->pages = pages; | 2235 | ap->folios = folios; |
| 2170 | ap->descs = descs; | 2236 | ap->descs = descs; |
| 2171 | data->max_pages = npages; | 2237 | data->max_folios = nfolios; |
| 2172 | 2238 | ||
| 2173 | return true; | 2239 | return true; |
| 2174 | } | 2240 | } |
| @@ -2178,7 +2244,7 @@ static void fuse_writepages_send(struct fuse_fill_wb_data *data) · 3 unmodified lines | |||
| 2178 | struct fuse_writepage_args *wpa = data->wpa; | 2244 | struct fuse_writepage_args *wpa = data->wpa; |
| 2179 | struct inode *inode = data->inode; | 2245 | struct inode *inode = data->inode; |
| 2180 | struct fuse_inode *fi = get_fuse_inode(inode); | 2246 | struct fuse_inode *fi = get_fuse_inode(inode); |
| 2181 | int num_pages = wpa->ia.ap.num_pages; | 2247 | int num_folios = wpa->ia.ap.num_folios; |
| 2182 | int i; | 2248 | int i; |
| 2183 | 2249 | ||
| 2184 | spin_lock(&fi->lock); | 2250 | spin_lock(&fi->lock); |
| @@ -2186,8 +2252,8 @@ static void fuse_writepages_send(struct fuse_fill_wb_data *data) · 1 unmodified line | |||
| 2186 | fuse_flush_writepages(inode); | 2252 | fuse_flush_writepages(inode); |
| 2187 | spin_unlock(&fi->lock); | 2253 | spin_unlock(&fi->lock); |
| 2188 | 2254 | ||
| 2189 | for (i = 0; i < num_pages; i++) | 2255 | for (i = 0; i < num_folios; i++) |
| 2190 | end_page_writeback(data->orig_pages[i]); | 2256 | folio_end_writeback(data->orig_folios[i]); |
| 2191 | } | 2257 | } |
| 2192 | 2258 | ||
| 2193 | /* | 2259 | /* |
| @@ -2198,15 +2264,15 @@ static void fuse_writepages_send(struct fuse_fill_wb_data *data) · 4 unmodified lines | |||
| 2198 | * swapping the new temp page with the old one. | 2264 | * swapping the new temp page with the old one. |
| 2199 | */ | 2265 | */ |
| 2200 | static bool fuse_writepage_add(struct fuse_writepage_args *new_wpa, | 2266 | static bool fuse_writepage_add(struct fuse_writepage_args *new_wpa, |
| 2201 | struct page *page) | 2267 | struct folio *folio) |
| 2202 | { | 2268 | { |
| 2203 | struct fuse_inode *fi = get_fuse_inode(new_wpa->inode); | 2269 | struct fuse_inode *fi = get_fuse_inode(new_wpa->inode); |
| 2204 | struct fuse_writepage_args *tmp; | 2270 | struct fuse_writepage_args *tmp; |
| 2205 | struct fuse_writepage_args *old_wpa; | 2271 | struct fuse_writepage_args *old_wpa; |
| 2206 | struct fuse_args_pages *new_ap = &new_wpa->ia.ap; | 2272 | struct fuse_args_pages *new_ap = &new_wpa->ia.ap; |
| 2207 | 2273 | ||
| 2208 | WARN_ON(new_ap->num_pages != 0); | 2274 | WARN_ON(new_ap->num_folios != 0); |
| 2209 | new_ap->num_pages = 1; | 2275 | new_ap->num_folios = 1; |
| 2210 | 2276 | ||
| 2211 | spin_lock(&fi->lock); | 2277 | spin_lock(&fi->lock); |
| 2212 | old_wpa = fuse_insert_writeback(&fi->writepages, new_wpa); | 2278 | old_wpa = fuse_insert_writeback(&fi->writepages, new_wpa); |
| @@ -2220,9 +2286,9 @@ static bool fuse_writepage_add(struct fuse_writepage_args *new_wpa, · 7 unmodified lines | |||
| 2220 | 2286 | ||
| 2221 | WARN_ON(tmp->inode != new_wpa->inode); | 2287 | WARN_ON(tmp->inode != new_wpa->inode); |
| 2222 | curr_index = tmp->ia.write.in.offset >> PAGE_SHIFT; | 2288 | curr_index = tmp->ia.write.in.offset >> PAGE_SHIFT; |
| 2223 | if (curr_index == page->index) { | 2289 | if (curr_index == folio->index) { |
| 2224 | WARN_ON(tmp->ia.ap.num_pages != 1); | 2290 | WARN_ON(tmp->ia.ap.num_folios != 1); |
| 2225 | swap(tmp->ia.ap.pages[0], new_ap->pages[0]); | 2291 | swap(tmp->ia.ap.folios[0], new_ap->folios[0]); |
| 2226 | break; | 2292 | break; |
| 2227 | } | 2293 | } |
| 2228 | } | 2294 | } |
| @@ -2235,18 +2301,19 @@ static bool fuse_writepage_add(struct fuse_writepage_args *new_wpa, · 6 unmodified lines | |||
| 2235 | spin_unlock(&fi->lock); | 2301 | spin_unlock(&fi->lock); |
| 2236 | 2302 | ||
| 2237 | if (tmp) { | 2303 | if (tmp) { |
| 2238 | fuse_writepage_finish_stat(new_wpa->inode, new_ap->pages[0]); | 2304 | fuse_writepage_finish_stat(new_wpa->inode, |
| 2305 | folio); | ||
| 2239 | fuse_writepage_free(new_wpa); | 2306 | fuse_writepage_free(new_wpa); |
| 2240 | } | 2307 | } |
| 2241 | 2308 | ||
| 2242 | return false; | 2309 | return false; |
| 2243 | } | 2310 | } |
| 2244 | 2311 | ||
| 2245 | static bool fuse_writepage_need_send(struct fuse_conn *fc, struct page *page, | 2312 | static bool fuse_writepage_need_send(struct fuse_conn *fc, struct folio *folio, |
| 2246 | struct fuse_args_pages *ap, | 2313 | struct fuse_args_pages *ap, |
| 2247 | struct fuse_fill_wb_data *data) | 2314 | struct fuse_fill_wb_data *data) |
| 2248 | { | 2315 | { |
| 2249 | WARN_ON(!ap->num_pages); | 2316 | WARN_ON(!ap->num_folios); |
| 2250 | 2317 | ||
| 2251 | /* | 2318 | /* |
| 2252 | * Being under writeback is unlikely but possible. For example direct | 2319 | * Being under writeback is unlikely but possible. For example direct |
| @@ -2254,23 +2321,23 @@ static bool fuse_writepage_need_send(struct fuse_conn *fc, struct page *page, · 1 unmodified line | |||
| 2254 | * the pages are faulted with get_user_pages(), and then after the read | 2321 | * the pages are faulted with get_user_pages(), and then after the read |
| 2255 | * completed. | 2322 | * completed. |
| 2256 | */ | 2323 | */ |
| 2257 | if (fuse_page_is_writeback(data->inode, page->index)) | 2324 | if (fuse_folio_is_writeback(data->inode, folio)) |
| 2258 | return true; | 2325 | return true; |
| 2259 | 2326 | ||
| 2260 | /* Reached max pages */ | 2327 | /* Reached max pages */ |
| 2261 | if (ap->num_pages == fc->max_pages) | 2328 | if (ap->num_folios == fc->max_pages) |
| 2262 | return true; | 2329 | return true; |
| 2263 | 2330 | ||
| 2264 | /* Reached max write bytes */ | 2331 | /* Reached max write bytes */ |
| 2265 | if ((ap->num_pages + 1) * PAGE_SIZE > fc->max_write) | 2332 | if ((ap->num_folios + 1) * PAGE_SIZE > fc->max_write) |
| 2266 | return true; | 2333 | return true; |
| 2267 | 2334 | ||
| 2268 | /* Discontinuity */ | 2335 | /* Discontinuity */ |
| 2269 | if (data->orig_pages[ap->num_pages - 1]->index + 1 != page->index) | 2336 | if (data->orig_folios[ap->num_folios - 1]->index + 1 != folio_index(folio)) |
| 2270 | return true; | 2337 | return true; |
| 2271 | 2338 | ||
| 2272 | /* Need to grow the pages array? If so, did the expansion fail? */ | 2339 | /* Need to grow the pages array? If so, did the expansion fail? */ |
| 2273 | if (ap->num_pages == data->max_pages && !fuse_pages_realloc(data)) | 2340 | if (ap->num_folios == data->max_folios && !fuse_pages_realloc(data)) |
| 2274 | return true; | 2341 | return true; |
| 2275 | 2342 | ||
| 2276 | return false; | 2343 | return false; |
| @@ -2295,7 +2362,7 @@ static int fuse_writepages_fill(struct folio *folio, · 18 unmodified lines | |||
| 2295 | goto out_unlock; | 2362 | goto out_unlock; |
| 2296 | } | 2363 | } |
| 2297 | 2364 | ||
| 2298 | if (wpa && fuse_writepage_need_send(fc, &folio->page, ap, data)) { | 2365 | if (wpa && fuse_writepage_need_send(fc, folio, ap, data)) { |
| 2299 | fuse_writepages_send(data); | 2366 | fuse_writepages_send(data); |
| 2300 | data->wpa = NULL; | 2367 | data->wpa = NULL; |
| 2301 | } | 2368 | } |
| @@ -2314,7 +2381,7 @@ static int fuse_writepages_fill(struct folio *folio, · 12 unmodified lines | |||
| 2314 | * This is ensured by holding the page lock in page_mkwrite() while | 2381 | * This is ensured by holding the page lock in page_mkwrite() while |
| 2315 | * checking fuse_page_is_writeback(). We already hold the page lock | 2382 | * checking fuse_page_is_writeback(). We already hold the page lock |
| 2316 | * since clear_page_dirty_for_io() and keep it held until we add the | 2383 | * since clear_page_dirty_for_io() and keep it held until we add the |
| 2317 | * request to the fi->writepages list and increment ap->num_pages. | 2384 | * request to the fi->writepages list and increment ap->num_folios. |
| 2318 | * After this fuse_page_is_writeback() will indicate that the page is | 2385 | * After this fuse_page_is_writeback() will indicate that the page is |
| 2319 | * under writeback, so we can release the page lock. | 2386 | * under writeback, so we can release the page lock. |
| 2320 | */ | 2387 | */ |
| @@ -2326,13 +2393,13 @@ static int fuse_writepages_fill(struct folio *folio, · 5 unmodified lines | |||
| 2326 | goto out_unlock; | 2393 | goto out_unlock; |
| 2327 | } | 2394 | } |
| 2328 | fuse_file_get(wpa->ia.ff); | 2395 | fuse_file_get(wpa->ia.ff); |
| 2329 | data->max_pages = 1; | 2396 | data->max_folios = 1; |
| 2330 | ap = &wpa->ia.ap; | 2397 | ap = &wpa->ia.ap; |
| 2331 | } | 2398 | } |
| 2332 | folio_start_writeback(folio); | 2399 | folio_start_writeback(folio); |
| 2333 | 2400 | ||
| 2334 | fuse_writepage_args_page_fill(wpa, folio, tmp_folio, ap->num_pages); | 2401 | fuse_writepage_args_page_fill(wpa, folio, tmp_folio, ap->num_folios); |
| 2335 | data->orig_pages[ap->num_pages] = &folio->page; | 2402 | data->orig_folios[ap->num_folios] = folio; |
| 2336 | 2403 | ||
| 2337 | err = 0; | 2404 | err = 0; |
| 2338 | if (data->wpa) { | 2405 | if (data->wpa) { |
| @@ -2341,9 +2408,9 @@ static int fuse_writepages_fill(struct folio *folio, · 2 unmodified lines | |||
| 2341 | * fuse_page_is_writeback(). | 2408 | * fuse_page_is_writeback(). |
| 2342 | */ | 2409 | */ |
| 2343 | spin_lock(&fi->lock); | 2410 | spin_lock(&fi->lock); |
| 2344 | ap->num_pages++; | 2411 | ap->num_folios++; |
| 2345 | spin_unlock(&fi->lock); | 2412 | spin_unlock(&fi->lock); |
| 2346 | } else if (fuse_writepage_add(wpa, &folio->page)) { | 2413 | } else if (fuse_writepage_add(wpa, folio)) { |
| 2347 | data->wpa = wpa; | 2414 | data->wpa = wpa; |
| 2348 | } else { | 2415 | } else { |
| 2349 | folio_end_writeback(folio); | 2416 | folio_end_writeback(folio); |
| @@ -2375,21 +2442,21 @@ static int fuse_writepages(struct address_space *mapping, · 25 unmodified lines | |||
| 2375 | data.ff = NULL; | 2442 | data.ff = NULL; |
| 2376 | 2443 | ||
| 2377 | err = -ENOMEM; | 2444 | err = -ENOMEM; |
| 2378 | data.orig_pages = kcalloc(fc->max_pages, | 2445 | data.orig_folios = kcalloc(fc->max_pages, |
| 2379 | sizeof(struct page *), | 2446 | sizeof(struct folio *), |
| 2380 | GFP_NOFS); | 2447 | GFP_NOFS); |
| 2381 | if (!data.orig_pages) | 2448 | if (!data.orig_folios) |
| 2382 | goto out; | 2449 | goto out; |
| 2383 | 2450 | ||
| 2384 | err = write_cache_pages(mapping, wbc, fuse_writepages_fill, &data); | 2451 | err = write_cache_pages(mapping, wbc, fuse_writepages_fill, &data); |
| 2385 | if (data.wpa) { | 2452 | if (data.wpa) { |
| 2386 | WARN_ON(!data.wpa->ia.ap.num_pages); | 2453 | WARN_ON(!data.wpa->ia.ap.num_folios); |
| 2387 | fuse_writepages_send(&data); | 2454 | fuse_writepages_send(&data); |
| 2388 | } | 2455 | } |
| 2389 | if (data.ff) | 2456 | if (data.ff) |
| 2390 | fuse_file_put(data.ff, false); | 2457 | fuse_file_put(data.ff, false); |
| 2391 | 2458 | ||
| 2392 | kfree(data.orig_pages); | 2459 | kfree(data.orig_folios); |
| 2393 | out: | 2460 | out: |
| 2394 | return err; | 2461 | return err; |
| 2395 | } | 2462 | } |
| @@ -2429,7 +2496,7 @@ static int fuse_write_begin(struct file *file, struct address_space *mapping, · 33 unmodified lines | |||
| 2429 | folio_zero_segment(folio, 0, off); | 2496 | folio_zero_segment(folio, 0, off); |
| 2430 | goto success; | 2497 | goto success; |
| 2431 | } | 2498 | } |
| 2432 | err = fuse_do_readpage(file, &folio->page); | 2499 | err = fuse_do_readfolio(file, folio); |
| 2433 | if (err) | 2500 | if (err) |
| 2434 | goto cleanup; | 2501 | goto cleanup; |
| 2435 | success: | 2502 | success: |
| @@ -2518,17 +2585,17 @@ static void fuse_vma_close(struct vm_area_struct *vma) · 82 unmodified lines | |||
| 2518 | */ | 2585 | */ |
| 2519 | static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) | 2586 | static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) |
| 2520 | { | 2587 | { |
| 2521 | struct page *page = vmf->page; | 2588 | struct folio *folio = page_folio(vmf->page); |
| 2522 | struct inode *inode = file_inode(vmf->vma->vm_file); | 2589 | struct inode *inode = file_inode(vmf->vma->vm_file); |
| 2523 | 2590 | ||
| 2524 | file_update_time(vmf->vma->vm_file); | 2591 | file_update_time(vmf->vma->vm_file); |
| 2525 | lock_page(page); | 2592 | folio_lock(folio); |
| 2526 | if (page->mapping != inode->i_mapping) { | 2593 | if (folio->mapping != inode->i_mapping) { |
| 2527 | unlock_page(page); | 2594 | folio_unlock(folio); |
| 2528 | return VM_FAULT_NOPAGE; | 2595 | return VM_FAULT_NOPAGE; |
| 2529 | } | 2596 | } |
| 2530 | 2597 | ||
| 2531 | fuse_wait_on_page_writeback(inode, page->index); | 2598 | fuse_wait_on_folio_writeback(inode, folio); |
| 2532 | return VM_FAULT_LOCKED; | 2599 | return VM_FAULT_LOCKED; |
| 2533 | } | 2600 | } |
| 2534 | 2601 | ||
| @@ -35,9 +35,6 @@ · 34 unmodified lines | |||
| 35 | /** Default max number of pages that can be used in a single read request */ | 35 | /** Default max number of pages that can be used in a single read request */ |
| 36 | #define FUSE_DEFAULT_MAX_PAGES_PER_REQ 32 | 36 | #define FUSE_DEFAULT_MAX_PAGES_PER_REQ 32 |
| 37 | 37 | ||
| 38 | /** Maximum of max_pages received in init_out */ | ||
| 39 | #define FUSE_MAX_MAX_PAGES 256 | ||
| 40 | |||
| 41 | /** Bias for fi->writectr, meaning new writepages must not be sent */ | 38 | /** Bias for fi->writectr, meaning new writepages must not be sent */ |
| 42 | #define FUSE_NOWRITE INT_MIN | 39 | #define FUSE_NOWRITE INT_MIN |
| 43 | 40 | ||
| @@ -47,6 +44,9 @@ · 3 unmodified lines | |||
| 47 | /** Number of dentries for each connection in the control filesystem */ | 44 | /** Number of dentries for each connection in the control filesystem */ |
| 48 | #define FUSE_CTL_NUM_DENTRIES 5 | 45 | #define FUSE_CTL_NUM_DENTRIES 5 |
| 49 | 46 | ||
| 47 | /** Maximum of max_pages received in init_out */ | ||
| 48 | extern unsigned int fuse_max_pages_limit; | ||
| 49 | |||
| 50 | /** List of active connections */ | 50 | /** List of active connections */ |
| 51 | extern struct list_head fuse_conn_list; | 51 | extern struct list_head fuse_conn_list; |
| 52 | 52 | ||
| @@ -285,8 +285,8 @@ struct fuse_arg { · 232 unmodified lines | |||
| 285 | void *value; | 285 | void *value; |
| 286 | }; | 286 | }; |
| 287 | 287 | ||
| 288 | /** FUSE page descriptor */ | 288 | /** FUSE folio descriptor */ |
| 289 | struct fuse_page_desc { | 289 | struct fuse_folio_desc { |
| 290 | unsigned int length; | 290 | unsigned int length; |
| 291 | unsigned int offset; | 291 | unsigned int offset; |
| 292 | }; | 292 | }; |
| @@ -309,16 +309,19 @@ struct fuse_args { · 16 unmodified lines | |||
| 309 | bool may_block:1; | 309 | bool may_block:1; |
| 310 | bool is_ext:1; | 310 | bool is_ext:1; |
| 311 | bool is_pinned:1; | 311 | bool is_pinned:1; |
| 312 | bool invalidate_vmap:1; | ||
| 312 | struct fuse_in_arg in_args[3]; | 313 | struct fuse_in_arg in_args[3]; |
| 313 | struct fuse_arg out_args[2]; | 314 | struct fuse_arg out_args[2]; |
| 314 | void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error); | 315 | void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error); |
| 316 | /* Used for kvec iter backed by vmalloc address */ | ||
| 317 | void *vmap_base; | ||
| 315 | }; | 318 | }; |
| 316 | 319 | ||
| 317 | struct fuse_args_pages { | 320 | struct fuse_args_pages { |
| 318 | struct fuse_args args; | 321 | struct fuse_args args; |
| 319 | struct page **pages; | 322 | struct folio **folios; |
| 320 | struct fuse_page_desc *descs; | 323 | struct fuse_folio_desc *descs; |
| 321 | unsigned int num_pages; | 324 | unsigned int num_folios; |
| 322 | }; | 325 | }; |
| 323 | 326 | ||
| 324 | struct fuse_release_args { | 327 | struct fuse_release_args { |
| @@ -857,6 +860,9 @@ struct fuse_conn { · 532 unmodified lines | |||
| 857 | /** Passthrough support for read/write IO */ | 860 | /** Passthrough support for read/write IO */ |
| 858 | unsigned int passthrough:1; | 861 | unsigned int passthrough:1; |
| 859 | 862 | ||
| 863 | /* Use pages instead of pointer for kernel I/O */ | ||
| 864 | unsigned int use_pages_for_kvec_io:1; | ||
| 865 | |||
| 860 | /** Maximum stack depth for passthrough backing files */ | 866 | /** Maximum stack depth for passthrough backing files */ |
| 861 | int max_stack_depth; | 867 | int max_stack_depth; |
| 862 | 868 | ||
| @@ -884,6 +890,9 @@ struct fuse_conn { · 21 unmodified lines | |||
| 884 | /** Version counter for attribute changes */ | 890 | /** Version counter for attribute changes */ |
| 885 | atomic64_t attr_version; | 891 | atomic64_t attr_version; |
| 886 | 892 | ||
| 893 | /** Version counter for evict inode */ | ||
| 894 | atomic64_t evict_ctr; | ||
| 895 | |||
| 887 | /** Called on final put */ | 896 | /** Called on final put */ |
| 888 | void (*release)(struct fuse_conn *); | 897 | void (*release)(struct fuse_conn *); |
| 889 | 898 | ||
| @@ -978,6 +987,11 @@ static inline u64 fuse_get_attr_version(struct fuse_conn *fc) · 88 unmodified lines | |||
| 978 | return atomic64_read(&fc->attr_version); | 987 | return atomic64_read(&fc->attr_version); |
| 979 | } | 988 | } |
| 980 | 989 | ||
| 990 | static inline u64 fuse_get_evict_ctr(struct fuse_conn *fc) | ||
| 991 | { | ||
| 992 | return atomic64_read(&fc->evict_ctr); | ||
| 993 | } | ||
| 994 | |||
| 981 | static inline bool fuse_stale_inode(const struct inode *inode, int generation, | 995 | static inline bool fuse_stale_inode(const struct inode *inode, int generation, |
| 982 | struct fuse_attr *attr) | 996 | struct fuse_attr *attr) |
| 983 | { | 997 | { |
| @@ -995,25 +1009,25 @@ static inline bool fuse_is_bad(struct inode *inode) · 11 unmodified lines | |||
| 995 | return unlikely(test_bit(FUSE_I_BAD, &get_fuse_inode(inode)->state)); | 1009 | return unlikely(test_bit(FUSE_I_BAD, &get_fuse_inode(inode)->state)); |
| 996 | } | 1010 | } |
| 997 | 1011 | ||
| 998 | static inline struct page **fuse_pages_alloc(unsigned int npages, gfp_t flags, | 1012 | static inline struct folio **fuse_folios_alloc(unsigned int nfolios, gfp_t flags, |
| 999 | struct fuse_page_desc **desc) | 1013 | struct fuse_folio_desc **desc) |
| 1000 | { | 1014 | { |
| 1001 | struct page **pages; | 1015 | struct folio **folios; |
| 1002 | 1016 | ||
| 1003 | pages = kzalloc(npages * (sizeof(struct page *) + | 1017 | folios = kzalloc(nfolios * (sizeof(struct folio *) + |
| 1004 | sizeof(struct fuse_page_desc)), flags); | 1018 | sizeof(struct fuse_folio_desc)), flags); |
| 1005 | *desc = (void *) (pages + npages); | 1019 | *desc = (void *) (folios + nfolios); |
| 1006 | 1020 | ||
| 1007 | return pages; | 1021 | return folios; |
| 1008 | } | 1022 | } |
| 1009 | 1023 | ||
| 1010 | static inline void fuse_page_descs_length_init(struct fuse_page_desc *descs, | 1024 | static inline void fuse_folio_descs_length_init(struct fuse_folio_desc *descs, |
| 1011 | unsigned int index, | 1025 | unsigned int index, |
| 1012 | unsigned int nr_pages) | 1026 | unsigned int nr_folios) |
| 1013 | { | 1027 | { |
| 1014 | int i; | 1028 | int i; |
| 1015 | 1029 | ||
| 1016 | for (i = index; i < index + nr_pages; i++) | 1030 | for (i = index; i < index + nr_folios; i++) |
| 1017 | descs[i].length = PAGE_SIZE - descs[i].offset; | 1031 | descs[i].length = PAGE_SIZE - descs[i].offset; |
| 1018 | } | 1032 | } |
| 1019 | 1033 | ||
| @@ -1037,7 +1051,8 @@ extern const struct dentry_operations fuse_root_dentry_operations; · 17 unmodified lines | |||
| 1037 | */ | 1051 | */ |
| 1038 | struct inode *fuse_iget(struct super_block *sb, u64 nodeid, | 1052 | struct inode *fuse_iget(struct super_block *sb, u64 nodeid, |
| 1039 | int generation, struct fuse_attr *attr, | 1053 | int generation, struct fuse_attr *attr, |
| 1040 | u64 attr_valid, u64 attr_version); | 1054 | u64 attr_valid, u64 attr_version, |
| 1055 | u64 evict_ctr); | ||
| 1041 | 1056 | ||
| 1042 | int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name, | 1057 | int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name, |
| 1043 | struct fuse_entry_out *outarg, struct inode **inode); | 1058 | struct fuse_entry_out *outarg, struct inode **inode); |
| @@ -1062,7 +1077,7 @@ struct fuse_io_args { · 18 unmodified lines | |||
| 1062 | struct { | 1077 | struct { |
| 1063 | struct fuse_write_in in; | 1078 | struct fuse_write_in in; |
| 1064 | struct fuse_write_out out; | 1079 | struct fuse_write_out out; |
| 1065 | bool page_locked; | 1080 | bool folio_locked; |
| 1066 | } write; | 1081 | } write; |
| 1067 | }; | 1082 | }; |
| 1068 | struct fuse_args_pages ap; | 1083 | struct fuse_args_pages ap; |
| @@ -1127,7 +1142,8 @@ void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr, · 58 unmodified lines | |||
| 1127 | 1142 | ||
| 1128 | void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr, | 1143 | void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr, |
| 1129 | struct fuse_statx *sx, | 1144 | struct fuse_statx *sx, |
| 1130 | u64 attr_valid, u32 cache_mask); | 1145 | u64 attr_valid, u32 cache_mask, |
| 1146 | u64 evict_ctr); | ||
| 1131 | 1147 | ||
| 1132 | u32 fuse_get_cache_mask(struct inode *inode); | 1148 | u32 fuse_get_cache_mask(struct inode *inode); |
| 1133 | 1149 | ||
| @@ -1480,4 +1496,12 @@ ssize_t fuse_passthrough_splice_write(struct pipe_inode_info *pipe, · 346 unmodified lines | |||
| 1480 | size_t len, unsigned int flags); | 1496 | size_t len, unsigned int flags); |
| 1481 | ssize_t fuse_passthrough_mmap(struct file *file, struct vm_area_struct *vma); | 1497 | ssize_t fuse_passthrough_mmap(struct file *file, struct vm_area_struct *vma); |
| 1482 | 1498 | ||
| 1499 | #ifdef CONFIG_SYSCTL | ||
| 1500 | extern int fuse_sysctl_register(void); | ||
| 1501 | extern void fuse_sysctl_unregister(void); | ||
| 1502 | #else | ||
| 1503 | #define fuse_sysctl_register() (0) | ||
| 1504 | #define fuse_sysctl_unregister() do { } while (0) | ||
| 1505 | #endif /* CONFIG_SYSCTL */ | ||
| 1506 | |||
| 1483 | #endif /* _FS_FUSE_I_H */ | 1507 | #endif /* _FS_FUSE_I_H */ |
| @@ -35,6 +35,8 @@ DEFINE_MUTEX(fuse_mutex); · 34 unmodified lines | |||
| 35 | 35 | ||
| 36 | static int set_global_limit(const char *val, const struct kernel_param *kp); | 36 | static int set_global_limit(const char *val, const struct kernel_param *kp); |
| 37 | 37 | ||
| 38 | unsigned int fuse_max_pages_limit = 256; | ||
| 39 | |||
| 38 | unsigned max_user_bgreq; | 40 | unsigned max_user_bgreq; |
| 39 | module_param_call(max_user_bgreq, set_global_limit, param_get_uint, | 41 | module_param_call(max_user_bgreq, set_global_limit, param_get_uint, |
| 40 | &max_user_bgreq, 0644); | 42 | &max_user_bgreq, 0644); |
| @@ -173,6 +175,14 @@ static void fuse_evict_inode(struct inode *inode) · 132 unmodified lines | |||
| 173 | fuse_cleanup_submount_lookup(fc, fi->submount_lookup); | 175 | fuse_cleanup_submount_lookup(fc, fi->submount_lookup); |
| 174 | fi->submount_lookup = NULL; | 176 | fi->submount_lookup = NULL; |
| 175 | } | 177 | } |
| 178 | /* | ||
| 179 | * Evict of non-deleted inode may race with outstanding | ||
| 180 | * LOOKUP/READDIRPLUS requests and result in inconsistency when | ||
| 181 | * the request finishes. Deal with that here by bumping a | ||
| 182 | * counter that can be compared to the starting value. | ||
| 183 | */ | ||
| 184 | if (inode->i_nlink > 0) | ||
| 185 | atomic64_inc(&fc->evict_ctr); | ||
| 176 | } | 186 | } |
| 177 | if (S_ISREG(inode->i_mode) && !fuse_is_bad(inode)) { | 187 | if (S_ISREG(inode->i_mode) && !fuse_is_bad(inode)) { |
| 178 | WARN_ON(fi->iocachectr != 0); | 188 | WARN_ON(fi->iocachectr != 0); |
| @@ -206,17 +216,30 @@ static ino_t fuse_squash_ino(u64 ino64) · 27 unmodified lines | |||
| 206 | 216 | ||
| 207 | void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr, | 217 | void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr, |
| 208 | struct fuse_statx *sx, | 218 | struct fuse_statx *sx, |
| 209 | u64 attr_valid, u32 cache_mask) | 219 | u64 attr_valid, u32 cache_mask, |
| 220 | u64 evict_ctr) | ||
| 210 | { | 221 | { |
| 211 | struct fuse_conn *fc = get_fuse_conn(inode); | 222 | struct fuse_conn *fc = get_fuse_conn(inode); |
| 212 | struct fuse_inode *fi = get_fuse_inode(inode); | 223 | struct fuse_inode *fi = get_fuse_inode(inode); |
| 213 | 224 | ||
| 214 | lockdep_assert_held(&fi->lock); | 225 | lockdep_assert_held(&fi->lock); |
| 215 | 226 | ||
| 227 | /* | ||
| 228 | * Clear basic stats from invalid mask. | ||
| 229 | * | ||
| 230 | * Don't do this if this is coming from a fuse_iget() call and there | ||
| 231 | * might have been a racing evict which would've invalidated the result | ||
| 232 | * if the attr_version would've been preserved. | ||
| 233 | * | ||
| 234 | * !evict_ctr -> this is create | ||
| 235 | * fi->attr_version != 0 -> this is not a new inode | ||
| 236 | * evict_ctr == fuse_get_evict_ctr() -> no evicts while during request | ||
| 237 | */ | ||
| 238 | if (!evict_ctr || fi->attr_version || evict_ctr == fuse_get_evict_ctr(fc)) | ||
| 239 | set_mask_bits(&fi->inval_mask, STATX_BASIC_STATS, 0); | ||
| 240 | |||
| 216 | fi->attr_version = atomic64_inc_return(&fc->attr_version); | 241 | fi->attr_version = atomic64_inc_return(&fc->attr_version); |
| 217 | fi->i_time = attr_valid; | 242 | fi->i_time = attr_valid; |
| 218 | /* Clear basic stats from invalid mask */ | ||
| 219 | set_mask_bits(&fi->inval_mask, STATX_BASIC_STATS, 0); | ||
| 220 | 243 | ||
| 221 | inode->i_ino = fuse_squash_ino(attr->ino); | 244 | inode->i_ino = fuse_squash_ino(attr->ino); |
| 222 | inode->i_mode = (inode->i_mode & S_IFMT) | (attr->mode & 07777); | 245 | inode->i_mode = (inode->i_mode & S_IFMT) | (attr->mode & 07777); |
| @@ -295,9 +318,9 @@ u32 fuse_get_cache_mask(struct inode *inode) · 72 unmodified lines | |||
| 295 | return STATX_MTIME | STATX_CTIME | STATX_SIZE; | 318 | return STATX_MTIME | STATX_CTIME | STATX_SIZE; |
| 296 | } | 319 | } |
| 297 | 320 | ||
| 298 | void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr, | 321 | static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr, |
| 299 | struct fuse_statx *sx, | 322 | struct fuse_statx *sx, u64 attr_valid, |
| 300 | u64 attr_valid, u64 attr_version) | 323 | u64 attr_version, u64 evict_ctr) |
| 301 | { | 324 | { |
| 302 | struct fuse_conn *fc = get_fuse_conn(inode); | 325 | struct fuse_conn *fc = get_fuse_conn(inode); |
| 303 | struct fuse_inode *fi = get_fuse_inode(inode); | 326 | struct fuse_inode *fi = get_fuse_inode(inode); |
| @@ -331,7 +354,8 @@ void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr, · 27 unmodified lines | |||
| 331 | } | 354 | } |
| 332 | 355 | ||
| 333 | old_mtime = inode_get_mtime(inode); | 356 | old_mtime = inode_get_mtime(inode); |
| 334 | fuse_change_attributes_common(inode, attr, sx, attr_valid, cache_mask); | 357 | fuse_change_attributes_common(inode, attr, sx, attr_valid, cache_mask, |
| 358 | evict_ctr); | ||
| 335 | 359 | ||
| 336 | oldsize = inode->i_size; | 360 | oldsize = inode->i_size; |
| 337 | /* | 361 | /* |
| @@ -372,6 +396,13 @@ void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr, · 34 unmodified lines | |||
| 372 | fuse_dax_dontcache(inode, attr->flags); | 396 | fuse_dax_dontcache(inode, attr->flags); |
| 373 | } | 397 | } |
| 374 | 398 | ||
| 399 | void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr, | ||
| 400 | struct fuse_statx *sx, u64 attr_valid, | ||
| 401 | u64 attr_version) | ||
| 402 | { | ||
| 403 | fuse_change_attributes_i(inode, attr, sx, attr_valid, attr_version, 0); | ||
| 404 | } | ||
| 405 | |||
| 375 | static void fuse_init_submount_lookup(struct fuse_submount_lookup *sl, | 406 | static void fuse_init_submount_lookup(struct fuse_submount_lookup *sl, |
| 376 | u64 nodeid) | 407 | u64 nodeid) |
| 377 | { | 408 | { |
| @@ -426,7 +457,8 @@ static int fuse_inode_set(struct inode *inode, void *_nodeidp) · 48 unmodified lines | |||
| 426 | 457 | ||
| 427 | struct inode *fuse_iget(struct super_block *sb, u64 nodeid, | 458 | struct inode *fuse_iget(struct super_block *sb, u64 nodeid, |
| 428 | int generation, struct fuse_attr *attr, | 459 | int generation, struct fuse_attr *attr, |
| 429 | u64 attr_valid, u64 attr_version) | 460 | u64 attr_valid, u64 attr_version, |
| 461 | u64 evict_ctr) | ||
| 430 | { | 462 | { |
| 431 | struct inode *inode; | 463 | struct inode *inode; |
| 432 | struct fuse_inode *fi; | 464 | struct fuse_inode *fi; |
| @@ -487,8 +519,8 @@ retry: · 54 unmodified lines | |||
| 487 | fi->nlookup++; | 519 | fi->nlookup++; |
| 488 | spin_unlock(&fi->lock); | 520 | spin_unlock(&fi->lock); |
| 489 | done: | 521 | done: |
| 490 | fuse_change_attributes(inode, attr, NULL, attr_valid, attr_version); | 522 | fuse_change_attributes_i(inode, attr, NULL, attr_valid, attr_version, |
| 491 | 523 | evict_ctr); | |
| 492 | return inode; | 524 | return inode; |
| 493 | } | 525 | } |
| 494 | 526 | ||
| @@ -940,11 +972,12 @@ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm, · 445 unmodified lines | |||
| 940 | fc->initialized = 0; | 972 | fc->initialized = 0; |
| 941 | fc->connected = 1; | 973 | fc->connected = 1; |
| 942 | atomic64_set(&fc->attr_version, 1); | 974 | atomic64_set(&fc->attr_version, 1); |
| 975 | atomic64_set(&fc->evict_ctr, 1); | ||
| 943 | get_random_bytes(&fc->scramble_key, sizeof(fc->scramble_key)); | 976 | get_random_bytes(&fc->scramble_key, sizeof(fc->scramble_key)); |
| 944 | fc->pid_ns = get_pid_ns(task_active_pid_ns(current)); | 977 | fc->pid_ns = get_pid_ns(task_active_pid_ns(current)); |
| 945 | fc->user_ns = get_user_ns(user_ns); | 978 | fc->user_ns = get_user_ns(user_ns); |
| 946 | fc->max_pages = FUSE_DEFAULT_MAX_PAGES_PER_REQ; | 979 | fc->max_pages = FUSE_DEFAULT_MAX_PAGES_PER_REQ; |
| 947 | fc->max_pages_limit = FUSE_MAX_MAX_PAGES; | 980 | fc->max_pages_limit = fuse_max_pages_limit; |
| 948 | 981 | ||
| 949 | if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH)) | 982 | if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH)) |
| 950 | fuse_backing_files_init(fc); | 983 | fuse_backing_files_init(fc); |
| @@ -1001,7 +1034,7 @@ static struct inode *fuse_get_root_inode(struct super_block *sb, unsigned mode) · 50 unmodified lines | |||
| 1001 | attr.mode = mode; | 1034 | attr.mode = mode; |
| 1002 | attr.ino = FUSE_ROOT_ID; | 1035 | attr.ino = FUSE_ROOT_ID; |
| 1003 | attr.nlink = 1; | 1036 | attr.nlink = 1; |
| 1004 | return fuse_iget(sb, FUSE_ROOT_ID, 0, &attr, 0, 0); | 1037 | return fuse_iget(sb, FUSE_ROOT_ID, 0, &attr, 0, 0, 0); |
| 1005 | } | 1038 | } |
| 1006 | 1039 | ||
| 1007 | struct fuse_inode_handle { | 1040 | struct fuse_inode_handle { |
| @@ -1610,7 +1643,8 @@ static int fuse_fill_super_submount(struct super_block *sb, · 602 unmodified lines | |||
| 1610 | return -ENOMEM; | 1643 | return -ENOMEM; |
| 1611 | 1644 | ||
| 1612 | fuse_fill_attr_from_inode(&root_attr, parent_fi); | 1645 | fuse_fill_attr_from_inode(&root_attr, parent_fi); |
| 1613 | root = fuse_iget(sb, parent_fi->nodeid, 0, &root_attr, 0, 0); | 1646 | root = fuse_iget(sb, parent_fi->nodeid, 0, &root_attr, 0, 0, |
| 1647 | fuse_get_evict_ctr(fm->fc)); | ||
| 1614 | /* | 1648 | /* |
| 1615 | * This inode is just a duplicate, so it is not looked up and | 1649 | * This inode is just a duplicate, so it is not looked up and |
| 1616 | * its nlookup should not be incremented. fuse_iget() does | 1650 | * its nlookup should not be incremented. fuse_iget() does |
| @@ -2063,8 +2097,14 @@ static int __init fuse_fs_init(void) · 446 unmodified lines | |||
| 2063 | if (err) | 2097 | if (err) |
| 2064 | goto out3; | 2098 | goto out3; |
| 2065 | 2099 | ||
| 2100 | err = fuse_sysctl_register(); | ||
| 2101 | if (err) | ||
| 2102 | goto out4; | ||
| 2103 | |||
| 2066 | return 0; | 2104 | return 0; |
| 2067 | 2105 | ||
| 2106 | out4: | ||
| 2107 | unregister_filesystem(&fuse_fs_type); | ||
| 2068 | out3: | 2108 | out3: |
| 2069 | unregister_fuseblk(); | 2109 | unregister_fuseblk(); |
| 2070 | out2: | 2110 | out2: |
| @@ -2075,6 +2115,7 @@ static int __init fuse_fs_init(void) · 4 unmodified lines | |||
| 2075 | 2115 | ||
| 2076 | static void fuse_fs_cleanup(void) | 2116 | static void fuse_fs_cleanup(void) |
| 2077 | { | 2117 | { |
| 2118 | fuse_sysctl_unregister(); | ||
| 2078 | unregister_filesystem(&fuse_fs_type); | 2119 | unregister_filesystem(&fuse_fs_type); |
| 2079 | unregister_fuseblk(); | 2120 | unregister_fuseblk(); |
| 2080 | 2121 | ||
| @@ -10,6 +10,8 @@ · 9 unmodified lines | |||
| 10 | #include <linux/fileattr.h> | 10 | #include <linux/fileattr.h> |
| 11 | #include <linux/fsverity.h> | 11 | #include <linux/fsverity.h> |
| 12 | 12 | ||
| 13 | #define FUSE_VERITY_ENABLE_ARG_MAX_PAGES 256 | ||
| 14 | |||
| 13 | static ssize_t fuse_send_ioctl(struct fuse_mount *fm, struct fuse_args *args, | 15 | static ssize_t fuse_send_ioctl(struct fuse_mount *fm, struct fuse_args *args, |
| 14 | struct fuse_ioctl_out *outarg) | 16 | struct fuse_ioctl_out *outarg) |
| 15 | { | 17 | { |
| @@ -140,7 +142,7 @@ static int fuse_setup_enable_verity(unsigned long arg, struct iovec *iov, · 124 unmodified lines | |||
| 140 | { | 142 | { |
| 141 | struct fsverity_enable_arg enable; | 143 | struct fsverity_enable_arg enable; |
| 142 | struct fsverity_enable_arg __user *uarg = (void __user *)arg; | 144 | struct fsverity_enable_arg __user *uarg = (void __user *)arg; |
| 143 | const __u32 max_buffer_len = FUSE_MAX_MAX_PAGES * PAGE_SIZE; | 145 | const __u32 max_buffer_len = FUSE_VERITY_ENABLE_ARG_MAX_PAGES * PAGE_SIZE; |
| 144 | 146 | ||
| 145 | if (copy_from_user(&enable, uarg, sizeof(enable))) | 147 | if (copy_from_user(&enable, uarg, sizeof(enable))) |
| 146 | return -EFAULT; | 148 | return -EFAULT; |
| @@ -249,12 +251,12 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, · 102 unmodified lines | |||
| 249 | BUILD_BUG_ON(sizeof(struct fuse_ioctl_iovec) * FUSE_IOCTL_MAX_IOV > PAGE_SIZE); | 251 | BUILD_BUG_ON(sizeof(struct fuse_ioctl_iovec) * FUSE_IOCTL_MAX_IOV > PAGE_SIZE); |
| 250 | 252 | ||
| 251 | err = -ENOMEM; | 253 | err = -ENOMEM; |
| 252 | ap.pages = fuse_pages_alloc(fm->fc->max_pages, GFP_KERNEL, &ap.descs); | 254 | ap.folios = fuse_folios_alloc(fm->fc->max_pages, GFP_KERNEL, &ap.descs); |
| 253 | iov_page = (struct iovec *) __get_free_page(GFP_KERNEL); | 255 | iov_page = (struct iovec *) __get_free_page(GFP_KERNEL); |
| 254 | if (!ap.pages || !iov_page) | 256 | if (!ap.folios || !iov_page) |
| 255 | goto out; | 257 | goto out; |
| 256 | 258 | ||
| 257 | fuse_page_descs_length_init(ap.descs, 0, fm->fc->max_pages); | 259 | fuse_folio_descs_length_init(ap.descs, 0, fm->fc->max_pages); |
| 258 | 260 | ||
| 259 | /* | 261 | /* |
| 260 | * If restricted, initialize IO parameters as encoded in @cmd. | 262 | * If restricted, initialize IO parameters as encoded in @cmd. |
| @@ -304,14 +306,13 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, · 43 unmodified lines | |||
| 304 | err = -ENOMEM; | 306 | err = -ENOMEM; |
| 305 | if (max_pages > fm->fc->max_pages) | 307 | if (max_pages > fm->fc->max_pages) |
| 306 | goto out; | 308 | goto out; |
| 307 | while (ap.num_pages < max_pages) { | 309 | while (ap.num_folios < max_pages) { |
| 308 | ap.pages[ap.num_pages] = alloc_page(GFP_KERNEL | __GFP_HIGHMEM); | 310 | ap.folios[ap.num_folios] = folio_alloc(GFP_KERNEL | __GFP_HIGHMEM, 0); |
| 309 | if (!ap.pages[ap.num_pages]) | 311 | if (!ap.folios[ap.num_folios]) |
| 310 | goto out; | 312 | goto out; |
| 311 | ap.num_pages++; | 313 | ap.num_folios++; |
| 312 | } | 314 | } |
| 313 | 315 | ||
| 314 | |||
| 315 | /* okay, let's send it to the client */ | 316 | /* okay, let's send it to the client */ |
| 316 | ap.args.opcode = FUSE_IOCTL; | 317 | ap.args.opcode = FUSE_IOCTL; |
| 317 | ap.args.nodeid = ff->nodeid; | 318 | ap.args.nodeid = ff->nodeid; |
| @@ -325,8 +326,8 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, · 7 unmodified lines | |||
| 325 | 326 | ||
| 326 | err = -EFAULT; | 327 | err = -EFAULT; |
| 327 | iov_iter_init(&ii, ITER_SOURCE, in_iov, in_iovs, in_size); | 328 | iov_iter_init(&ii, ITER_SOURCE, in_iov, in_iovs, in_size); |
| 328 | for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= ap.num_pages); i++) { | 329 | for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= ap.num_folios); i++) { |
| 329 | c = copy_page_from_iter(ap.pages[i], 0, PAGE_SIZE, &ii); | 330 | c = copy_folio_from_iter(ap.folios[i], 0, PAGE_SIZE, &ii); |
| 330 | if (c != PAGE_SIZE && iov_iter_count(&ii)) | 331 | if (c != PAGE_SIZE && iov_iter_count(&ii)) |
| 331 | goto out; | 332 | goto out; |
| 332 | } | 333 | } |
| @@ -364,7 +365,7 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, · 31 unmodified lines | |||
| 364 | in_iovs + out_iovs > FUSE_IOCTL_MAX_IOV) | 365 | in_iovs + out_iovs > FUSE_IOCTL_MAX_IOV) |
| 365 | goto out; | 366 | goto out; |
| 366 | 367 | ||
| 367 | vaddr = kmap_local_page(ap.pages[0]); | 368 | vaddr = kmap_local_folio(ap.folios[0], 0); |
| 368 | err = fuse_copy_ioctl_iovec(fm->fc, iov_page, vaddr, | 369 | err = fuse_copy_ioctl_iovec(fm->fc, iov_page, vaddr, |
| 369 | transferred, in_iovs + out_iovs, | 370 | transferred, in_iovs + out_iovs, |
| 370 | (flags & FUSE_IOCTL_COMPAT) != 0); | 371 | (flags & FUSE_IOCTL_COMPAT) != 0); |
| @@ -392,17 +393,17 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, · 21 unmodified lines | |||
| 392 | 393 | ||
| 393 | err = -EFAULT; | 394 | err = -EFAULT; |
| 394 | iov_iter_init(&ii, ITER_DEST, out_iov, out_iovs, transferred); | 395 | iov_iter_init(&ii, ITER_DEST, out_iov, out_iovs, transferred); |
| 395 | for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= ap.num_pages); i++) { | 396 | for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= ap.num_folios); i++) { |
| 396 | c = copy_page_to_iter(ap.pages[i], 0, PAGE_SIZE, &ii); | 397 | c = copy_folio_to_iter(ap.folios[i], 0, PAGE_SIZE, &ii); |
| 397 | if (c != PAGE_SIZE && iov_iter_count(&ii)) | 398 | if (c != PAGE_SIZE && iov_iter_count(&ii)) |
| 398 | goto out; | 399 | goto out; |
| 399 | } | 400 | } |
| 400 | err = 0; | 401 | err = 0; |
| 401 | out: | 402 | out: |
| 402 | free_page((unsigned long) iov_page); | 403 | free_page((unsigned long) iov_page); |
| 403 | while (ap.num_pages) | 404 | while (ap.num_folios) |
| 404 | __free_page(ap.pages[--ap.num_pages]); | 405 | folio_put(ap.folios[--ap.num_folios]); |
| 405 | kfree(ap.pages); | 406 | kfree(ap.folios); |
| 406 | 407 | ||
| 407 | return err ? err : outarg.result; | 408 | return err ? err : outarg.result; |
| 408 | } | 409 | } |
| @@ -149,7 +149,7 @@ static int parse_dirfile(char *buf, size_t nbytes, struct file *file, · 148 unmodified lines | |||
| 149 | 149 | ||
| 150 | static int fuse_direntplus_link(struct file *file, | 150 | static int fuse_direntplus_link(struct file *file, |
| 151 | struct fuse_direntplus *direntplus, | 151 | struct fuse_direntplus *direntplus, |
| 152 | u64 attr_version) | 152 | u64 attr_version, u64 evict_ctr) |
| 153 | { | 153 | { |
| 154 | struct fuse_entry_out *o = &direntplus->entry_out; | 154 | struct fuse_entry_out *o = &direntplus->entry_out; |
| 155 | struct fuse_dirent *dirent = &direntplus->dirent; | 155 | struct fuse_dirent *dirent = &direntplus->dirent; |
| @@ -233,7 +233,7 @@ retry: · 77 unmodified lines | |||
| 233 | } else { | 233 | } else { |
| 234 | inode = fuse_iget(dir->i_sb, o->nodeid, o->generation, | 234 | inode = fuse_iget(dir->i_sb, o->nodeid, o->generation, |
| 235 | &o->attr, ATTR_TIMEOUT(o), | 235 | &o->attr, ATTR_TIMEOUT(o), |
| 236 | attr_version); | 236 | attr_version, evict_ctr); |
| 237 | if (!inode) | 237 | if (!inode) |
| 238 | inode = ERR_PTR(-ENOMEM); | 238 | inode = ERR_PTR(-ENOMEM); |
| 239 | 239 | ||
| @@ -284,7 +284,8 @@ static void fuse_force_forget(struct file *file, u64 nodeid) · 44 unmodified lines | |||
| 284 | } | 284 | } |
| 285 | 285 | ||
| 286 | static int parse_dirplusfile(char *buf, size_t nbytes, struct file *file, | 286 | static int parse_dirplusfile(char *buf, size_t nbytes, struct file *file, |
| 287 | struct dir_context *ctx, u64 attr_version) | 287 | struct dir_context *ctx, u64 attr_version, |
| 288 | u64 evict_ctr) | ||
| 288 | { | 289 | { |
| 289 | struct fuse_direntplus *direntplus; | 290 | struct fuse_direntplus *direntplus; |
| 290 | struct fuse_dirent *dirent; | 291 | struct fuse_dirent *dirent; |
| @@ -319,7 +320,7 @@ static int parse_dirplusfile(char *buf, size_t nbytes, struct file *file, · 28 unmodified lines | |||
| 319 | buf += reclen; | 320 | buf += reclen; |
| 320 | nbytes -= reclen; | 321 | nbytes -= reclen; |
| 321 | 322 | ||
| 322 | ret = fuse_direntplus_link(file, direntplus, attr_version); | 323 | ret = fuse_direntplus_link(file, direntplus, attr_version, evict_ctr); |
| 323 | if (ret) | 324 | if (ret) |
| 324 | fuse_force_forget(file, direntplus->entry_out.nodeid); | 325 | fuse_force_forget(file, direntplus->entry_out.nodeid); |
| 325 | } | 326 | } |
| @@ -331,26 +332,27 @@ static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) · 5 unmodified lines | |||
| 331 | { | 332 | { |
| 332 | int plus; | 333 | int plus; |
| 333 | ssize_t res; | 334 | ssize_t res; |
| 334 | struct page *page; | 335 | struct folio *folio; |
| 335 | struct inode *inode = file_inode(file); | 336 | struct inode *inode = file_inode(file); |
| 336 | struct fuse_mount *fm = get_fuse_mount(inode); | 337 | struct fuse_mount *fm = get_fuse_mount(inode); |
| 337 | struct fuse_io_args ia = {}; | 338 | struct fuse_io_args ia = {}; |
| 338 | struct fuse_args_pages *ap = &ia.ap; | 339 | struct fuse_args_pages *ap = &ia.ap; |
| 339 | struct fuse_page_desc desc = { .length = PAGE_SIZE }; | 340 | struct fuse_folio_desc desc = { .length = PAGE_SIZE }; |
| 340 | u64 attr_version = 0; | 341 | u64 attr_version = 0, evict_ctr = 0; |
| 341 | bool locked; | 342 | bool locked; |
| 342 | 343 | ||
| 343 | page = alloc_page(GFP_KERNEL); | 344 | folio = folio_alloc(GFP_KERNEL, 0); |
| 344 | if (!page) | 345 | if (!folio) |
| 345 | return -ENOMEM; | 346 | return -ENOMEM; |
| 346 | 347 | ||
| 347 | plus = fuse_use_readdirplus(inode, ctx); | 348 | plus = fuse_use_readdirplus(inode, ctx); |
| 348 | ap->args.out_pages = true; | 349 | ap->args.out_pages = true; |
| 349 | ap->num_pages = 1; | 350 | ap->num_folios = 1; |
| 350 | ap->pages = &page; | 351 | ap->folios = &folio; |
| 351 | ap->descs = &desc; | 352 | ap->descs = &desc; |
| 352 | if (plus) { | 353 | if (plus) { |
| 353 | attr_version = fuse_get_attr_version(fm->fc); | 354 | attr_version = fuse_get_attr_version(fm->fc); |
| 355 | evict_ctr = fuse_get_evict_ctr(fm->fc); | ||
| 354 | fuse_read_args_fill(&ia, file, ctx->pos, PAGE_SIZE, | 356 | fuse_read_args_fill(&ia, file, ctx->pos, PAGE_SIZE, |
| 355 | FUSE_READDIRPLUS); | 357 | FUSE_READDIRPLUS); |
| 356 | } else { | 358 | } else { |
| @@ -367,15 +369,16 @@ static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) · 10 unmodified lines | |||
| 367 | if (ff->open_flags & FOPEN_CACHE_DIR) | 369 | if (ff->open_flags & FOPEN_CACHE_DIR) |
| 368 | fuse_readdir_cache_end(file, ctx->pos); | 370 | fuse_readdir_cache_end(file, ctx->pos); |
| 369 | } else if (plus) { | 371 | } else if (plus) { |
| 370 | res = parse_dirplusfile(page_address(page), res, | 372 | res = parse_dirplusfile(folio_address(folio), res, |
| 371 | file, ctx, attr_version); | 373 | file, ctx, attr_version, |
| 374 | evict_ctr); | ||
| 372 | } else { | 375 | } else { |
| 373 | res = parse_dirfile(page_address(page), res, file, | 376 | res = parse_dirfile(folio_address(folio), res, file, |
| 374 | ctx); | 377 | ctx); |
| 375 | } | 378 | } |
| 376 | } | 379 | } |
| 377 | 380 | ||
| 378 | __free_page(page); | 381 | folio_put(folio); |
| 379 | fuse_invalidate_atime(inode); | 382 | fuse_invalidate_atime(inode); |
| 380 | return res; | 383 | return res; |
| 381 | } | 384 | } |
| @@ -0,0 +1,40 @@ | |||
| 1 | // SPDX-License-Identifier: GPL-2.0 | ||
| 2 | /* | ||
| 3 | * linux/fs/fuse/fuse_sysctl.c | ||
| 4 | * | ||
| 5 | * Sysctl interface to fuse parameters | ||
| 6 | */ | ||
| 7 | #include <linux/sysctl.h> | ||
| 8 | |||
| 9 | #include "fuse_i.h" | ||
| 10 | |||
| 11 | static struct ctl_table_header *fuse_table_header; | ||
| 12 | |||
| 13 | /* Bound by fuse_init_out max_pages, which is a u16 */ | ||
| 14 | static unsigned int sysctl_fuse_max_pages_limit = 65535; | ||
| 15 | |||
| 16 | static struct ctl_table fuse_sysctl_table[] = { | ||
| 17 | { | ||
| 18 | .procname = "max_pages_limit", | ||
| 19 | .data = &fuse_max_pages_limit, | ||
| 20 | .maxlen = sizeof(fuse_max_pages_limit), | ||
| 21 | .mode = 0644, | ||
| 22 | .proc_handler = proc_douintvec_minmax, | ||
| 23 | .extra1 = SYSCTL_ONE, | ||
| 24 | .extra2 = &sysctl_fuse_max_pages_limit, | ||
| 25 | }, | ||
| 26 | }; | ||
| 27 | |||
| 28 | int fuse_sysctl_register(void) | ||
| 29 | { | ||
| 30 | fuse_table_header = register_sysctl("fs/fuse", fuse_sysctl_table); | ||
| 31 | if (!fuse_table_header) | ||
| 32 | return -ENOMEM; | ||
| 33 | return 0; | ||
| 34 | } | ||
| 35 | |||
| 36 | void fuse_sysctl_unregister(void) | ||
| 37 | { | ||
| 38 | unregister_sysctl_table(fuse_table_header); | ||
| 39 | fuse_table_header = NULL; | ||
| 40 | } | ||
| @@ -97,7 +97,8 @@ struct virtio_fs_req_work { · 96 unmodified lines | |||
| 97 | }; | 97 | }; |
| 98 | 98 | ||
| 99 | static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, | 99 | static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, |
| 100 | struct fuse_req *req, bool in_flight); | 100 | struct fuse_req *req, bool in_flight, |
| 101 | gfp_t gfp); | ||
| 101 | 102 | ||
| 102 | static const struct constant_table dax_param_enums[] = { | 103 | static const struct constant_table dax_param_enums[] = { |
| 103 | {"always", FUSE_DAX_ALWAYS }, | 104 | {"always", FUSE_DAX_ALWAYS }, |
| @@ -575,6 +576,8 @@ static void virtio_fs_request_dispatch_work(struct work_struct *work) · 471 unmodified lines | |||
| 575 | 576 | ||
| 576 | /* Dispatch pending requests */ | 577 | /* Dispatch pending requests */ |
| 577 | while (1) { | 578 | while (1) { |
| 579 | unsigned int flags; | ||
| 580 | |||
| 578 | spin_lock(&fsvq->lock); | 581 | spin_lock(&fsvq->lock); |
| 579 | req = list_first_entry_or_null(&fsvq->queued_reqs, | 582 | req = list_first_entry_or_null(&fsvq->queued_reqs, |
| 580 | struct fuse_req, list); | 583 | struct fuse_req, list); |
| @@ -585,7 +588,9 @@ static void virtio_fs_request_dispatch_work(struct work_struct *work) · 4 unmodified lines | |||
| 585 | list_del_init(&req->list); | 588 | list_del_init(&req->list); |
| 586 | spin_unlock(&fsvq->lock); | 589 | spin_unlock(&fsvq->lock); |
| 587 | 590 | ||
| 588 | ret = virtio_fs_enqueue_req(fsvq, req, true); | 591 | flags = memalloc_nofs_save(); |
| 592 | ret = virtio_fs_enqueue_req(fsvq, req, true, GFP_KERNEL); | ||
| 593 | memalloc_nofs_restore(flags); | ||
| 589 | if (ret < 0) { | 594 | if (ret < 0) { |
| 590 | if (ret == -ENOSPC) { | 595 | if (ret == -ENOSPC) { |
| 591 | spin_lock(&fsvq->lock); | 596 | spin_lock(&fsvq->lock); |
| @@ -686,7 +691,7 @@ static void virtio_fs_hiprio_dispatch_work(struct work_struct *work) · 94 unmodified lines | |||
| 686 | } | 691 | } |
| 687 | 692 | ||
| 688 | /* Allocate and copy args into req->argbuf */ | 693 | /* Allocate and copy args into req->argbuf */ |
| 689 | static int copy_args_to_argbuf(struct fuse_req *req) | 694 | static int copy_args_to_argbuf(struct fuse_req *req, gfp_t gfp) |
| 690 | { | 695 | { |
| 691 | struct fuse_args *args = req->args; | 696 | struct fuse_args *args = req->args; |
| 692 | unsigned int offset = 0; | 697 | unsigned int offset = 0; |
| @@ -700,7 +705,7 @@ static int copy_args_to_argbuf(struct fuse_req *req) · 7 unmodified lines | |||
| 700 | len = fuse_len_args(num_in, (struct fuse_arg *) args->in_args) + | 705 | len = fuse_len_args(num_in, (struct fuse_arg *) args->in_args) + |
| 701 | fuse_len_args(num_out, args->out_args); | 706 | fuse_len_args(num_out, args->out_args); |
| 702 | 707 | ||
| 703 | req->argbuf = kmalloc(len, GFP_ATOMIC); | 708 | req->argbuf = kmalloc(len, gfp); |
| 704 | if (!req->argbuf) | 709 | if (!req->argbuf) |
| 705 | return -ENOMEM; | 710 | return -ENOMEM; |
| 706 | 711 | ||
| @@ -760,7 +765,7 @@ static void virtio_fs_request_complete(struct fuse_req *req, · 53 unmodified lines | |||
| 760 | struct fuse_args *args; | 765 | struct fuse_args *args; |
| 761 | struct fuse_args_pages *ap; | 766 | struct fuse_args_pages *ap; |
| 762 | unsigned int len, i, thislen; | 767 | unsigned int len, i, thislen; |
| 763 | struct page *page; | 768 | struct folio *folio; |
| 764 | 769 | ||
| 765 | /* | 770 | /* |
| 766 | * TODO verify that server properly follows FUSE protocol | 771 | * TODO verify that server properly follows FUSE protocol |
| @@ -772,12 +777,12 @@ static void virtio_fs_request_complete(struct fuse_req *req, · 5 unmodified lines | |||
| 772 | if (args->out_pages && args->page_zeroing) { | 777 | if (args->out_pages && args->page_zeroing) { |
| 773 | len = args->out_args[args->out_numargs - 1].size; | 778 | len = args->out_args[args->out_numargs - 1].size; |
| 774 | ap = container_of(args, typeof(*ap), args); | 779 | ap = container_of(args, typeof(*ap), args); |
| 775 | for (i = 0; i < ap->num_pages; i++) { | 780 | for (i = 0; i < ap->num_folios; i++) { |
| 776 | thislen = ap->descs[i].length; | 781 | thislen = ap->descs[i].length; |
| 777 | if (len < thislen) { | 782 | if (len < thislen) { |
| 778 | WARN_ON(ap->descs[i].offset); | 783 | WARN_ON(ap->descs[i].offset); |
| 779 | page = ap->pages[i]; | 784 | folio = ap->folios[i]; |
| 780 | zero_user_segment(page, len, thislen); | 785 | folio_zero_segment(folio, len, thislen); |
| 781 | len = 0; | 786 | len = 0; |
| 782 | } else { | 787 | } else { |
| 783 | len -= thislen; | 788 | len -= thislen; |
| @@ -1267,15 +1272,15 @@ static void virtio_fs_send_interrupt(struct fuse_iqueue *fiq, struct fuse_req *r · 483 unmodified lines | |||
| 1267 | } | 1272 | } |
| 1268 | 1273 | ||
| 1269 | /* Count number of scatter-gather elements required */ | 1274 | /* Count number of scatter-gather elements required */ |
| 1270 | static unsigned int sg_count_fuse_pages(struct fuse_page_desc *page_descs, | 1275 | static unsigned int sg_count_fuse_folios(struct fuse_folio_desc *folio_descs, |
| 1271 | unsigned int num_pages, | 1276 | unsigned int num_folios, |
| 1272 | unsigned int total_len) | 1277 | unsigned int total_len) |
| 1273 | { | 1278 | { |
| 1274 | unsigned int i; | 1279 | unsigned int i; |
| 1275 | unsigned int this_len; | 1280 | unsigned int this_len; |
| 1276 | 1281 | ||
| 1277 | for (i = 0; i < num_pages && total_len; i++) { | 1282 | for (i = 0; i < num_folios && total_len; i++) { |
| 1278 | this_len = min(page_descs[i].length, total_len); | 1283 | this_len = min(folio_descs[i].length, total_len); |
| 1279 | total_len -= this_len; | 1284 | total_len -= this_len; |
| 1280 | } | 1285 | } |
| 1281 | 1286 | ||
| @@ -1294,8 +1299,8 @@ static unsigned int sg_count_fuse_req(struct fuse_req *req) · 12 unmodified lines | |||
| 1294 | 1299 | ||
| 1295 | if (args->in_pages) { | 1300 | if (args->in_pages) { |
| 1296 | size = args->in_args[args->in_numargs - 1].size; | 1301 | size = args->in_args[args->in_numargs - 1].size; |
| 1297 | total_sgs += sg_count_fuse_pages(ap->descs, ap->num_pages, | 1302 | total_sgs += sg_count_fuse_folios(ap->descs, ap->num_folios, |
| 1298 | size); | 1303 | size); |
| 1299 | } | 1304 | } |
| 1300 | 1305 | ||
| 1301 | if (!test_bit(FR_ISREPLY, &req->flags)) | 1306 | if (!test_bit(FR_ISREPLY, &req->flags)) |
| @@ -1308,27 +1313,27 @@ static unsigned int sg_count_fuse_req(struct fuse_req *req) · 6 unmodified lines | |||
| 1308 | 1313 | ||
| 1309 | if (args->out_pages) { | 1314 | if (args->out_pages) { |
| 1310 | size = args->out_args[args->out_numargs - 1].size; | 1315 | size = args->out_args[args->out_numargs - 1].size; |
| 1311 | total_sgs += sg_count_fuse_pages(ap->descs, ap->num_pages, | 1316 | total_sgs += sg_count_fuse_folios(ap->descs, ap->num_folios, |
| 1312 | size); | 1317 | size); |
| 1313 | } | 1318 | } |
| 1314 | 1319 | ||
| 1315 | return total_sgs; | 1320 | return total_sgs; |
| 1316 | } | 1321 | } |
| 1317 | 1322 | ||
| 1318 | /* Add pages to scatter-gather list and return number of elements used */ | 1323 | /* Add folios to scatter-gather list and return number of elements used */ |
| 1319 | static unsigned int sg_init_fuse_pages(struct scatterlist *sg, | 1324 | static unsigned int sg_init_fuse_folios(struct scatterlist *sg, |
| 1320 | struct page **pages, | 1325 | struct folio **folios, |
| 1321 | struct fuse_page_desc *page_descs, | 1326 | struct fuse_folio_desc *folio_descs, |
| 1322 | unsigned int num_pages, | 1327 | unsigned int num_folios, |
| 1323 | unsigned int total_len) | 1328 | unsigned int total_len) |
| 1324 | { | 1329 | { |
| 1325 | unsigned int i; | 1330 | unsigned int i; |
| 1326 | unsigned int this_len; | 1331 | unsigned int this_len; |
| 1327 | 1332 | ||
| 1328 | for (i = 0; i < num_pages && total_len; i++) { | 1333 | for (i = 0; i < num_folios && total_len; i++) { |
| 1329 | sg_init_table(&sg[i], 1); | 1334 | sg_init_table(&sg[i], 1); |
| 1330 | this_len = min(page_descs[i].length, total_len); | 1335 | this_len = min(folio_descs[i].length, total_len); |
| 1331 | sg_set_page(&sg[i], pages[i], this_len, page_descs[i].offset); | 1336 | sg_set_folio(&sg[i], folios[i], this_len, folio_descs[i].offset); |
| 1332 | total_len -= this_len; | 1337 | total_len -= this_len; |
| 1333 | } | 1338 | } |
| 1334 | 1339 | ||
| @@ -1353,10 +1358,10 @@ static unsigned int sg_init_fuse_args(struct scatterlist *sg, · 18 unmodified lines | |||
| 1353 | sg_init_one(&sg[total_sgs++], argbuf, len); | 1358 | sg_init_one(&sg[total_sgs++], argbuf, len); |
| 1354 | 1359 | ||
| 1355 | if (argpages) | 1360 | if (argpages) |
| 1356 | total_sgs += sg_init_fuse_pages(&sg[total_sgs], | 1361 | total_sgs += sg_init_fuse_folios(&sg[total_sgs], |
| 1357 | ap->pages, ap->descs, | 1362 | ap->folios, ap->descs, |
| 1358 | ap->num_pages, | 1363 | ap->num_folios, |
| 1359 | args[numargs - 1].size); | 1364 | args[numargs - 1].size); |
| 1360 | 1365 | ||
| 1361 | if (len_used) | 1366 | if (len_used) |
| 1362 | *len_used = len; | 1367 | *len_used = len; |
| @@ -1366,7 +1371,8 @@ static unsigned int sg_init_fuse_args(struct scatterlist *sg, · 3 unmodified lines | |||
| 1366 | 1371 | ||
| 1367 | /* Add a request to a virtqueue and kick the device */ | 1372 | /* Add a request to a virtqueue and kick the device */ |
| 1368 | static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, | 1373 | static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, |
| 1369 | struct fuse_req *req, bool in_flight) | 1374 | struct fuse_req *req, bool in_flight, |
| 1375 | gfp_t gfp) | ||
| 1370 | { | 1376 | { |
| 1371 | /* requests need at least 4 elements */ | 1377 | /* requests need at least 4 elements */ |
| 1372 | struct scatterlist *stack_sgs[6]; | 1378 | struct scatterlist *stack_sgs[6]; |
| @@ -1387,8 +1393,8 @@ static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, · 14 unmodified lines | |||
| 1387 | /* Does the sglist fit on the stack? */ | 1393 | /* Does the sglist fit on the stack? */ |
| 1388 | total_sgs = sg_count_fuse_req(req); | 1394 | total_sgs = sg_count_fuse_req(req); |
| 1389 | if (total_sgs > ARRAY_SIZE(stack_sgs)) { | 1395 | if (total_sgs > ARRAY_SIZE(stack_sgs)) { |
| 1390 | sgs = kmalloc_array(total_sgs, sizeof(sgs[0]), GFP_ATOMIC); | 1396 | sgs = kmalloc_array(total_sgs, sizeof(sgs[0]), gfp); |
| 1391 | sg = kmalloc_array(total_sgs, sizeof(sg[0]), GFP_ATOMIC); | 1397 | sg = kmalloc_array(total_sgs, sizeof(sg[0]), gfp); |
| 1392 | if (!sgs || !sg) { | 1398 | if (!sgs || !sg) { |
| 1393 | ret = -ENOMEM; | 1399 | ret = -ENOMEM; |
| 1394 | goto out; | 1400 | goto out; |
| @@ -1396,7 +1402,7 @@ static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, · 1 unmodified line | |||
| 1396 | } | 1402 | } |
| 1397 | 1403 | ||
| 1398 | /* Use a bounce buffer since stack args cannot be mapped */ | 1404 | /* Use a bounce buffer since stack args cannot be mapped */ |
| 1399 | ret = copy_args_to_argbuf(req); | 1405 | ret = copy_args_to_argbuf(req, gfp); |
| 1400 | if (ret < 0) | 1406 | if (ret < 0) |
| 1401 | goto out; | 1407 | goto out; |
| 1402 | 1408 | ||
| @@ -1490,7 +1496,7 @@ static void virtio_fs_send_req(struct fuse_iqueue *fiq, struct fuse_req *req) · 87 unmodified lines | |||
| 1490 | queue_id); | 1496 | queue_id); |
| 1491 | 1497 | ||
| 1492 | fsvq = &fs->vqs[queue_id]; | 1498 | fsvq = &fs->vqs[queue_id]; |
| 1493 | ret = virtio_fs_enqueue_req(fsvq, req, false); | 1499 | ret = virtio_fs_enqueue_req(fsvq, req, false, GFP_ATOMIC); |
| 1494 | if (ret < 0) { | 1500 | if (ret < 0) { |
| 1495 | if (ret == -ENOSPC) { | 1501 | if (ret == -ENOSPC) { |
| 1496 | /* | 1502 | /* |
| @@ -1691,6 +1697,7 @@ static int virtio_fs_get_tree(struct fs_context *fsc) · 194 unmodified lines | |||
| 1691 | fc->delete_stale = true; | 1697 | fc->delete_stale = true; |
| 1692 | fc->auto_submounts = true; | 1698 | fc->auto_submounts = true; |
| 1693 | fc->sync_fs = true; | 1699 | fc->sync_fs = true; |
| 1700 | fc->use_pages_for_kvec_io = true; | ||
| 1694 | 1701 | ||
| 1695 | /* Tell FUSE to split requests that exceed the virtqueue's size */ | 1702 | /* Tell FUSE to split requests that exceed the virtqueue's size */ |
| 1696 | fc->max_pages_limit = min_t(unsigned int, fc->max_pages_limit, | 1703 | fc->max_pages_limit = min_t(unsigned int, fc->max_pages_limit, |
| @@ -2550,6 +2550,7 @@ struct kvec; · 2549 unmodified lines | |||
| 2550 | struct page *get_dump_page(unsigned long addr); | 2550 | struct page *get_dump_page(unsigned long addr); |
| 2551 | 2551 | ||
| 2552 | bool folio_mark_dirty(struct folio *folio); | 2552 | bool folio_mark_dirty(struct folio *folio); |
| 2553 | bool folio_mark_dirty_lock(struct folio *folio); | ||
| 2553 | bool set_page_dirty(struct page *page); | 2554 | bool set_page_dirty(struct page *page); |
| 2554 | int set_page_dirty_lock(struct page *page); | 2555 | int set_page_dirty_lock(struct page *page); |
| 2555 | 2556 | ||
| @@ -52,6 +52,12 @@ bool set_page_dirty(struct page *page) · 51 unmodified lines | |||
| 52 | } | 52 | } |
| 53 | EXPORT_SYMBOL(set_page_dirty); | 53 | EXPORT_SYMBOL(set_page_dirty); |
| 54 | 54 | ||
| 55 | int set_page_dirty_lock(struct page *page) | ||
| 56 | { | ||
| 57 | return folio_mark_dirty_lock(page_folio(page)); | ||
| 58 | } | ||
| 59 | EXPORT_SYMBOL(set_page_dirty_lock); | ||
| 60 | |||
| 55 | bool clear_page_dirty_for_io(struct page *page) | 61 | bool clear_page_dirty_for_io(struct page *page) |
| 56 | { | 62 | { |
| 57 | return folio_clear_dirty_for_io(page_folio(page)); | 63 | return folio_clear_dirty_for_io(page_folio(page)); |
| @@ -2925,25 +2925,25 @@ bool folio_mark_dirty(struct folio *folio) · 2924 unmodified lines | |||
| 2925 | EXPORT_SYMBOL(folio_mark_dirty); | 2925 | EXPORT_SYMBOL(folio_mark_dirty); |
| 2926 | 2926 | ||
| 2927 | /* | 2927 | /* |
| 2928 | * set_page_dirty() is racy if the caller has no reference against | 2928 | * folio_mark_dirty() is racy if the caller has no reference against |
| 2929 | * page->mapping->host, and if the page is unlocked. This is because another | 2929 | * folio->mapping->host, and if the folio is unlocked. This is because another |
| 2930 | * CPU could truncate the page off the mapping and then free the mapping. | 2930 | * CPU could truncate the folio off the mapping and then free the mapping. |
| 2931 | * | 2931 | * |
| 2932 | * Usually, the page _is_ locked, or the caller is a user-space process which | 2932 | * Usually, the folio _is_ locked, or the caller is a user-space process which |
| 2933 | * holds a reference on the inode by having an open file. | 2933 | * holds a reference on the inode by having an open file. |
| 2934 | * | 2934 | * |
| 2935 | * In other cases, the page should be locked before running set_page_dirty(). | 2935 | * In other cases, the folio should be locked before running folio_mark_dirty(). |
| 2936 | */ | 2936 | */ |
| 2937 | int set_page_dirty_lock(struct page *page) | 2937 | bool folio_mark_dirty_lock(struct folio *folio) |
| 2938 | { | 2938 | { |
| 2939 | int ret; | 2939 | bool ret; |
| 2940 | 2940 | ||
| 2941 | lock_page(page); | 2941 | folio_lock(folio); |
| 2942 | ret = set_page_dirty(page); | 2942 | ret = folio_mark_dirty(folio); |
| 2943 | unlock_page(page); | 2943 | folio_unlock(folio); |
| 2944 | return ret; | 2944 | return ret; |
| 2945 | } | 2945 | } |
| 2946 | EXPORT_SYMBOL(set_page_dirty_lock); | 2946 | EXPORT_SYMBOL(folio_mark_dirty_lock); |
| 2947 | 2947 | ||
| 2948 | /* | 2948 | /* |
| 2949 | * This cancels just the dirty bit on the kernel page itself, it does NOT | 2949 | * This cancels just the dirty bit on the kernel page itself, it does NOT |