History

Merge tag 'for-7.2-rc6-fixup-worker-tag' of git://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux

Pull Btrfs Fixes 2: Electric Boogaloo from David Sterba: "This brings back the fixup worker infrastructure. It's a mechanism to detect pages/folios that are marked dirty without filesystem knowledge and require COW fixup. The consequence of not doing so is silent data loss. The first patch covers the scenarios in detail, also reflecting folio API port and subpage block size support added in recent years. The original fixup worker was only for pages. The patch is relatively big, half of the code is debugging and support code, the rest is the core design around the detection and fix. The second patch handles an unlikely case when there's work left during unmount" * tag 'for-7.2-rc6-fixup-worker-tag' of git://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux: btrfs: flush the fixup workers during close_ctree btrfs: trigger cow fixup via dirty_folio()

Linus Torvalds committed 3 months ago

Commit 6c68fa62 parents 6335463 ae2567b8 files · +651 −12Browse files

Unified · Split

8 files
  1. fs/btrfs/btrfs_inode.h +1 −0
  2. fs/btrfs/disk-io.c +27 −1
  3. fs/btrfs/extent_io.c +113 −0
  4. fs/btrfs/fs.h +12 −0
  5. fs/btrfs/inode.c +216 −1
  6. fs/btrfs/subpage.c +211 −5
  7. fs/btrfs/subpage.h +36 −5
  8. include/trace/events/btrfs.h +35 −0
fs/btrfs/btrfs_inode.h+1−0
@@ -600,6 +600,7 @@ int btrfs_prealloc_file_range_trans(struct inode *inode, · 599 unmodified lines
600 loff_t actual_len, u64 *alloc_hint);600 loff_t actual_len, u64 *alloc_hint);
601int btrfs_run_delalloc_range(struct btrfs_inode *inode, struct folio *locked_folio,601int btrfs_run_delalloc_range(struct btrfs_inode *inode, struct folio *locked_folio,
602 u64 start, u64 end, struct writeback_control *wbc);602 u64 start, u64 end, struct writeback_control *wbc);
603void btrfs_queue_writepage_fixup(struct btrfs_inode *inode, struct folio *folio);
603int btrfs_encoded_io_compression_from_extent(struct btrfs_fs_info *fs_info,604int btrfs_encoded_io_compression_from_extent(struct btrfs_fs_info *fs_info,
604 int compress_type);605 int compress_type);
605int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode,606int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode,
fs/btrfs/disk-io.c+27−1
@@ -1760,6 +1760,8 @@ static int read_backup_root(struct btrfs_fs_info *fs_info, u8 priority) · 1759 unmodified lines
1760/* helper to cleanup workers */1760/* helper to cleanup workers */
1761static void btrfs_stop_all_workers(struct btrfs_fs_info *fs_info)1761static void btrfs_stop_all_workers(struct btrfs_fs_info *fs_info)
1762{1762{
1763 if (fs_info->fixup_workers)
1764 destroy_workqueue(fs_info->fixup_workers);
1763 btrfs_destroy_workqueue(fs_info->delalloc_workers);1765 btrfs_destroy_workqueue(fs_info->delalloc_workers);
1764 btrfs_destroy_workqueue(fs_info->workers);1766 btrfs_destroy_workqueue(fs_info->workers);
1765 if (fs_info->endio_workers)1767 if (fs_info->endio_workers)
@@ -1967,6 +1969,9 @@ static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info) · 201 unmodified lines
1967 fs_info->caching_workers =1969 fs_info->caching_workers =
1968 btrfs_alloc_workqueue(fs_info, "cache", flags, max_active, 0);1970 btrfs_alloc_workqueue(fs_info, "cache", flags, max_active, 0);
1969 1971
1972 fs_info->fixup_workers =
1973 alloc_ordered_workqueue("btrfs-fixup", ordered_flags);
1974
1970 fs_info->endio_workers =1975 fs_info->endio_workers =
1971 alloc_workqueue("btrfs-endio", flags, max_active);1976 alloc_workqueue("btrfs-endio", flags, max_active);
1972 fs_info->endio_meta_workers =1977 fs_info->endio_meta_workers =
@@ -1992,7 +1997,7 @@ static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info) · 19 unmodified lines
1992 fs_info->endio_workers && fs_info->endio_meta_workers &&1997 fs_info->endio_workers && fs_info->endio_meta_workers &&
1993 fs_info->endio_write_workers &&1998 fs_info->endio_write_workers &&
1994 fs_info->endio_freespace_worker && fs_info->rmw_workers &&1999 fs_info->endio_freespace_worker && fs_info->rmw_workers &&
1995 fs_info->caching_workers &&2000 fs_info->caching_workers && fs_info->fixup_workers &&
1996 fs_info->delayed_workers && fs_info->qgroup_rescan_workers &&2001 fs_info->delayed_workers && fs_info->qgroup_rescan_workers &&
1997 fs_info->discard_ctl.discard_workers)) {2002 fs_info->discard_ctl.discard_workers)) {
1998 return -ENOMEM;2003 return -ENOMEM;
@@ -4364,6 +4369,18 @@ void __cold close_ctree(struct btrfs_fs_info *fs_info) · 2365 unmodified lines
4364 /* clear out the rbtree of defraggable inodes */4369 /* clear out the rbtree of defraggable inodes */
4365 btrfs_cleanup_defrag_inodes(fs_info);4370 btrfs_cleanup_defrag_inodes(fs_info);
4366 4371
4372 /*
4373 * Before the unmount, we sync down all the writeback which can
4374 * generate fixup work. We are about to run delalloc for autodefrag so
4375 * piggy back on that by also flushing the fixup work which can also
4376 * generate delalloc we would like to get run.
4377 *
4378 * After this, it is still possible that some thread doing writeback is
4379 * in btrfs_queue_writepage_fixup() and might finish queueing some final
4380 * work, racing the btrfs_fs_closing() check there.
4381 */
4382 flush_workqueue(fs_info->fixup_workers);
4383
4367 /*4384 /*
4368 * Handle the error fs first, as it will flush and wait for all ordered4385 * Handle the error fs first, as it will flush and wait for all ordered
4369 * extents. This will generate delayed iputs, thus we want to handle4386 * extents. This will generate delayed iputs, thus we want to handle
@@ -4441,6 +4458,15 @@ void __cold close_ctree(struct btrfs_fs_info *fs_info) · 71 unmodified lines
4441 cancel_work_sync(&fs_info->preempt_reclaim_work);4458 cancel_work_sync(&fs_info->preempt_reclaim_work);
4442 cancel_work_sync(&fs_info->em_shrinker_work);4459 cancel_work_sync(&fs_info->em_shrinker_work);
4443 4460
4461 /*
4462 * Reclaim workers can run writeback which can queue fixup.
4463 * After the above cancel_work_sync() calls, any such queueing attempts are
4464 * guaranteed to see btrfs_fs_closing(), so at this point we can genuinely fully
4465 * flush the fixup workqueue. This relies on the belief that *now* no thread can
4466 * still be sitting in btrfs_queue_writepage_fixup().
4467 */
4468 flush_workqueue(fs_info->fixup_workers);
4469
4444 /*4470 /*
4445 * Run delayed iputs again because an async reclaim worker may have4471 * Run delayed iputs again because an async reclaim worker may have
4446 * added new ones if it was flushing delalloc:4472 * added new ones if it was flushing delalloc:
fs/btrfs/extent_io.c+113−0
@@ -1440,6 +1440,115 @@ static bool find_next_delalloc_bitmap(struct folio *folio, · 1439 unmodified lines
1440 return true;1440 return true;
1441}1441}
1442 1442
1443/*
1444 * Debug checks for fixup selection logic to help ensure the invariants
1445 * we expect for fixup marking hold in practice.
1446 *
1447 * - A dirty block without a fixup bit is covered by delalloc or a running
1448 * ordered extent (it was dirtied by a reserving write path).
1449 * - A block with a fixup bit is never covered by delalloc: every delalloc
1450 * setter holds the folio lock and cancels the fixup state of the blocks
1451 * it covers (btrfs_folio_set_dirty()) before releasing it.
1452 */
1453static void debug_check_writepage_fixup(struct btrfs_inode *inode, u64 start,
1454 u32 len, bool needs_fixup)
1455{
1456 struct btrfs_ordered_extent *ordered;
1457 bool delalloc;
1458
1459 if (!IS_ENABLED(CONFIG_BTRFS_DEBUG))
1460 return;
1461
1462 delalloc = btrfs_test_range_bit_exists(&inode->io_tree, start,
1463 start + len - 1, EXTENT_DELALLOC);
1464 if (needs_fixup) {
1465 if (unlikely(delalloc))
1466 DEBUG_WARN("writeback: delalloc and fixup conflict. ino %llu start %llu",
1467 btrfs_ino(inode), start);
1468 } else {
1469 if (delalloc)
1470 return;
1471
1472 ordered = btrfs_lookup_ordered_range(inode, start, len);
1473 if (unlikely(!ordered))
1474 DEBUG_WARN("dirty block, no delalloc, fixup, ordered. ino %llu start %llu",
1475 btrfs_ino(inode), start);
1476 else
1477 btrfs_put_ordered_extent(ordered);
1478 }
1479}
1480
1481/*
1482 * Handle folios dirtied without a delalloc reservation, e.g.
1483 * O_DIRECT read into a MAP_SHARED mapping dirtying via set_page_dirty_lock().
1484 *
1485 * btrfs_data_dirty_folio() records the affected blocks in the fixup bitmap
1486 * and the folio fixup flag and we check them here in writeback.
1487 *
1488 * Don't submit such blocks and queue work for the fixup worker to reserve
1489 * space for them so that they can be submitted properly by writeback.
1490 *
1491 * Return 1 if the folio needed fixup, 0 if not, and a negative error code
1492 * on error.
1493 */
1494static noinline_for_stack int writepage_fixup(struct btrfs_inode *inode,
1495 struct folio *folio,
1496 struct btrfs_bio_ctrl *bio_ctrl)
1497{
1498 struct btrfs_fs_info *fs_info = inode_to_fs_info(&inode->vfs_inode);
1499 const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio);
1500 const u32 sectorsize = fs_info->sectorsize;
1501 const u64 page_start = folio_pos(folio);
1502 bool found_fixup = false;
1503 unsigned int bit;
1504
1505 /*
1506 * A folio was dirtied without calling aops->dirty_folio() which we
1507 * explicitly assert is not allowed.
1508 */
1509 if (unlikely(bitmap_empty(bio_ctrl->submit_bitmap, blocks_per_folio))) {
1510 DEBUG_WARN();
1511 btrfs_err_rl(fs_info,
1512 "root %lld ino %llu folio %llu is dirty with an empty dirty bitmap",
1513 btrfs_root_id(inode->root), btrfs_ino(inode),
1514 folio_pos(folio));
1515 return -EUCLEAN;
1516 }
1517
1518 /* Cheap check on the folio flag. Set iff the fixup bitmap is non-empty. */
1519 if (likely(!folio_test_fixup_pending(folio)))
1520 return 0;
1521
1522 for_each_set_bit(bit, bio_ctrl->submit_bitmap, blocks_per_folio) {
1523 const u64 start = page_start + (bit << fs_info->sectorsize_bits);
1524 const bool needs_fixup = btrfs_folio_test_fixup(fs_info, folio,
1525 start, sectorsize);
1526
1527 debug_check_writepage_fixup(inode, start, sectorsize, needs_fixup);
1528 if (needs_fixup) {
1529 bitmap_clear(bio_ctrl->submit_bitmap, bit, 1);
1530 found_fixup = true;
1531 }
1532 }
1533 if (likely(found_fixup)) {
1534 btrfs_queue_writepage_fixup(inode, folio);
1535 folio_redirty_for_writepage(bio_ctrl->wbc, folio);
1536 if (bitmap_empty(bio_ctrl->submit_bitmap, blocks_per_folio)) {
1537 folio_unlock(folio);
1538 return 1;
1539 }
1540 return 0;
1541 }
1542 /* We should always find fixup if the folio fixup flag was set. */
1543 DEBUG_WARN();
1544 btrfs_err_rl(fs_info,
1545 "root %lld ino %llu folio %llu is fixup with an empty fixup bitmap",
1546 btrfs_root_id(inode->root), btrfs_ino(inode),
1547 folio_pos(folio));
1548
1549 return -EUCLEAN;
1550}
1551
1443/*1552/*
1444 * Do all of the delayed allocation setup.1553 * Do all of the delayed allocation setup.
1445 *1554 *
@@ -1492,6 +1601,10 @@ static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode, · 46 unmodified lines
1492 /* Save the dirty bitmap as our submission bitmap will be a subset of it. */1601 /* Save the dirty bitmap as our submission bitmap will be a subset of it. */
1493 btrfs_copy_subpage_dirty_bitmap(fs_info, folio, bio_ctrl->submit_bitmap);1602 btrfs_copy_subpage_dirty_bitmap(fs_info, folio, bio_ctrl->submit_bitmap);
1494 1603
1604 ret = writepage_fixup(inode, folio, bio_ctrl);
1605 if (ret)
1606 return ret;
1607
1495 for_each_set_bitrange(start_bit, end_bit, bio_ctrl->submit_bitmap,1608 for_each_set_bitrange(start_bit, end_bit, bio_ctrl->submit_bitmap,
1496 blocks_per_folio) {1609 blocks_per_folio) {
1497 u64 start = page_start + (start_bit << fs_info->sectorsize_bits);1610 u64 start = page_start + (start_bit << fs_info->sectorsize_bits);
fs/btrfs/fs.h+12−0
@@ -713,6 +713,8 @@ struct btrfs_fs_info { · 712 unmodified lines
713 struct btrfs_workqueue *endio_write_workers;713 struct btrfs_workqueue *endio_write_workers;
714 struct btrfs_workqueue *endio_freespace_worker;714 struct btrfs_workqueue *endio_freespace_worker;
715 struct btrfs_workqueue *caching_workers;715 struct btrfs_workqueue *caching_workers;
716
717 struct workqueue_struct *fixup_workers;
716 struct btrfs_workqueue *delayed_workers;718 struct btrfs_workqueue *delayed_workers;
717 719
718 struct task_struct *transaction_kthread;720 struct task_struct *transaction_kthread;
@@ -1200,6 +1202,16 @@ static inline void btrfs_wake_unfinished_drop(struct btrfs_fs_info *fs_info) · 481 unmodified lines
1200 clear_and_wake_up_bit(BTRFS_FS_UNFINISHED_DROPS, &fs_info->flags);1202 clear_and_wake_up_bit(BTRFS_FS_UNFINISHED_DROPS, &fs_info->flags);
1201}1203}
1202 1204
1205/*
1206 * We use the folio owner_2 flag to indicate the folio has blocks that were
1207 * dirtied without a space reservation and need the writepage fixup before
1208 * writeback. For bs < folio_size the fixup bitmap tracks the affected
1209 * blocks.
1210 */
1211#define folio_test_fixup_pending(folio) folio_test_owner_2(folio)
1212#define folio_set_fixup_pending(folio) folio_set_owner_2(folio)
1213#define folio_clear_fixup_pending(folio) folio_clear_owner_2(folio)
1214
1203#define BTRFS_FS_ERROR(fs_info) (READ_ONCE((fs_info)->fs_error))1215#define BTRFS_FS_ERROR(fs_info) (READ_ONCE((fs_info)->fs_error))
1204 1216
1205#define BTRFS_FS_LOG_CLEANUP_ERROR(fs_info) \1217#define BTRFS_FS_LOG_CLEANUP_ERROR(fs_info) \
fs/btrfs/inode.c+216−1
@@ -2812,6 +2812,180 @@ int btrfs_set_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, · 2811 unmodified lines
2812 EXTENT_DELALLOC | extra_bits, cached_state);2812 EXTENT_DELALLOC | extra_bits, cached_state);
2813}2813}
2814 2814
2815struct btrfs_writepage_fixup {
2816 struct folio *folio;
2817 struct btrfs_inode *inode;
2818 struct work_struct work;
2819};
2820
2821/*
2822 * Do the real fixup work of reserving space for the blocks a folio's fixup
2823 * state records. Queued by writepage_fixup() when writeback found the bits set.
2824 *
2825 * Since the fixup can be cancelled by a task dirtying with a reservation, we must
2826 * re-check the state of fixup under the folio lock.
2827 */
2828static void btrfs_writepage_fixup_worker(struct work_struct *work)
2829{
2830 struct btrfs_writepage_fixup *fixup =
2831 container_of(work, struct btrfs_writepage_fixup, work);
2832 struct extent_state *cached_state = NULL;
2833 struct extent_changeset *data_reserved = NULL;
2834 unsigned long delalloc_bitmap[BITS_TO_LONGS(BTRFS_MAX_BLOCKS_PER_FOLIO)] = { 0 };
2835 struct folio *folio = fixup->folio;
2836 struct btrfs_inode *inode = fixup->inode;
2837 struct btrfs_fs_info *fs_info = inode->root->fs_info;
2838 const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio);
2839 const u32 sectorsize = fs_info->sectorsize;
2840 const u64 page_start = folio_pos(folio);
2841 const u64 page_end = folio_next_pos(folio) - 1;
2842 unsigned int start_bit;
2843 unsigned int end_bit;
2844 unsigned int bit;
2845 bool reserved;
2846 int ret;
2847
2848 /*
2849 * We would prefer to reserve under the folio lock when we know exactly
2850 * which blocks need a reservation. Unfortunately, since the reservation
2851 * can go into flushers which can go into writeback, which takes folio
2852 * locks, that is not possible. Therefore, we have to reserve for the
2853 * whole folio here, then release what we didn't end up needing once we
2854 * figure it out.
2855 *
2856 * Also note the slightly strange error checking. If fixup is actually
2857 * not set, we don't need to mark an error on the mapping. So hang on to
2858 * ret until after we lock and find out if we actually care.
2859 */
2860 ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start,
2861 folio_size(folio));
2862 reserved = (ret == 0);
2863again:
2864 folio_lock(folio);
2865
2866 if (!folio->mapping || !folio_test_fixup_pending(folio)) {
2867 ret = 0;
2868 goto out;
2869 }
2870 if (ret)
2871 goto out;
2872
2873 btrfs_lock_extent(&inode->io_tree, page_start, page_end, &cached_state);
2874
2875 for (bit = 0; bit < blocks_per_folio; bit++) {
2876 struct btrfs_ordered_extent *ordered;
2877 const u64 start = page_start + (bit << fs_info->sectorsize_bits);
2878
2879 if (test_bit(bit, delalloc_bitmap))
2880 continue;
2881 if (!btrfs_folio_test_fixup(fs_info, folio, start, sectorsize))
2882 continue;
2883 /*
2884 * Any task that sets EXTENT_DELALLOC clears the fixup bits
2885 * under the folio lock, so it should be impossible to observe
2886 * both under the lock. Setting delalloc twice would wrongly
2887 * double account the space.
2888 */
2889 if (IS_ENABLED(CONFIG_BTRFS_DEBUG) &&
2890 unlikely(btrfs_test_range_bit_exists(&inode->io_tree, start,
2891 start + sectorsize - 1,
2892 EXTENT_DELALLOC))) {
2893 DEBUG_WARN("fixup worker: delalloc and fixup conflict. ino %llu start %llu",
2894 btrfs_ino(inode), start);
2895 btrfs_folio_clear_fixup(fs_info, folio, start, sectorsize);
2896 continue;
2897 }
2898 ordered = btrfs_lookup_ordered_range(inode, start, sectorsize);
2899 if (ordered) {
2900 trace_btrfs_writepage_fixup_defer(inode, ordered);
2901 btrfs_unlock_extent(&inode->io_tree, page_start,
2902 page_end, &cached_state);
2903 folio_unlock(folio);
2904 btrfs_start_ordered_extent(ordered);
2905 btrfs_put_ordered_extent(ordered);
2906 goto again;
2907 }
2908 ret = btrfs_set_extent_delalloc(inode, start,
2909 start + sectorsize - 1, 0,
2910 &cached_state);
2911 if (ret)
2912 break;
2913 trace_btrfs_writepage_fixup_reserve(inode, start, sectorsize);
2914 btrfs_folio_clear_fixup(fs_info, folio, start, sectorsize);
2915 set_bit(bit, delalloc_bitmap);
2916 }
2917
2918 btrfs_unlock_extent(&inode->io_tree, page_start, page_end, &cached_state);
2919out:
2920 if (ret < 0) {
2921 /* Failure here is analogous to failure in writeback. */
2922 mapping_set_error(folio->mapping, ret);
2923 btrfs_folio_clear_fixup_dirty(fs_info, folio, page_start,
2924 folio_size(folio));
2925 }
2926 if (reserved) {
2927 btrfs_delalloc_release_extents(inode, folio_size(folio));
2928 for_each_clear_bitrange(start_bit, end_bit, delalloc_bitmap,
2929 blocks_per_folio)
2930 btrfs_delalloc_release_space(inode, data_reserved,
2931 page_start + (start_bit << fs_info->sectorsize_bits),
2932 (end_bit - start_bit) << fs_info->sectorsize_bits,
2933 true);
2934 }
2935 folio_unlock(folio);
2936 folio_put(folio);
2937 kfree(fixup);
2938 extent_changeset_free(data_reserved);
2939 btrfs_add_delayed_iput(inode);
2940}
2941
2942/*
2943 * Queue space reservation fixup work for blocks dirtied without a space reservation.
2944 *
2945 * Should be used by writeback while holding the folio locked.
2946 *
2947 * If we fail to queue fixup, then the folio state is unchanged and a future
2948 * writeback pass will still see it.
2949 */
2950void btrfs_queue_writepage_fixup(struct btrfs_inode *inode, struct folio *folio)
2951{
2952 struct btrfs_fs_info *fs_info = inode->root->fs_info;
2953 struct btrfs_writepage_fixup *fixup;
2954
2955 /*
2956 * Disallow queueing more fixup during unmount to break the cycle
2957 * of writeback queuing fixup queuing writeback etc.
2958 *
2959 * If it actually hit, then something which was fixup wasn't written
2960 * which we should warn about.
2961 */
2962 if (btrfs_fs_closing(fs_info)) {
2963 btrfs_warn_rl(fs_info,
2964 "dropping unqueued fixup blocks at unmount. root %lld ino %llu folio %llu",
2965 btrfs_root_id(inode->root), btrfs_ino(inode),
2966 folio_pos(folio));
2967 btrfs_folio_clear_fixup_dirty(fs_info, folio,
2968 folio_pos(folio), folio_size(folio));
2969 return;
2970 }
2971
2972 fixup = kzalloc_obj(*fixup, GFP_NOFS);
2973 if (!fixup)
2974 return;
2975
2976 /*
2977 * This is called from within extent_write_cache_pages() which
2978 * has successfully done an igrab(). But that will be released at the
2979 * end of the writeback pass. We need to extend it for the worker as well.
2980 */
2981 ihold(&inode->vfs_inode);
2982 folio_get(folio);
2983 INIT_WORK(&fixup->work, btrfs_writepage_fixup_worker);
2984 fixup->folio = folio;
2985 fixup->inode = inode;
2986 queue_work(fs_info->fixup_workers, &fixup->work);
2987}
2988
2815/*2989/*
2816 * Clear the old accounting flags and set EXTENT_DELALLOC for the range.2990 * Clear the old accounting flags and set EXTENT_DELALLOC for the range.
2817 *2991 *
@@ -7508,6 +7682,12 @@ static void btrfs_invalidate_folio(struct folio *folio, size_t offset, · 4690 unmodified lines
7508 folio_wait_writeback(folio);7682 folio_wait_writeback(folio);
7509 wait_subpage_spinlock(folio);7683 wait_subpage_spinlock(folio);
7510 7684
7685 /*
7686 * The invalidated blocks are going away; drop any fixup blocks among
7687 * them, data included, as they have no space reservation.
7688 */
7689 btrfs_folio_clear_fixup_dirty(fs_info, folio, page_start + offset, length);
7690
7511 /*7691 /*
7512 * For subpage case, we have call sites like7692 * For subpage case, we have call sites like
7513 * btrfs_punch_hole_lock_range() which passes range not aligned to7693 * btrfs_punch_hole_lock_range() which passes range not aligned to
@@ -10550,6 +10730,41 @@ static const struct file_operations btrfs_dir_file_operations = { · 3036 unmodified lines
10550 .setlease = generic_setlease,10730 .setlease = generic_setlease,
10551};10731};
10552 10732
10733/*
10734 * The folio is going dirty without a btrfs delalloc space reservation.
10735 * This requires a fixup before writeback which we might sleep so cannot
10736 * run in this context, so we merely set state on the folio indicating it
10737 * needs fixup before writeback.
10738 *
10739 * Note that there is no range in the input, so the whole folio is marked
10740 * dirty and fixup.
10741 *
10742 * We believe that all callers of dirty_folio either:
10743 * - take the folio lock (e.g. pinned folio release notification).
10744 * - take the pte lock but must be running on a dirty pte which means
10745 * page_mkwrite() ran on it and reserved the space. zap_pte_range() cannot
10746 * race with writeback cleaning the folio because writeback runs
10747 * folio_mkclean() which also uses the pte lock and revokes outstanding
10748 * writable mappings.
10749 * Therefore, an additional folio private lock (a la bfs->lock for all cases,
10750 * not just subpage) is not necessary.
10751 */
10752static bool btrfs_data_dirty_folio(struct address_space *mapping,
10753 struct folio *folio)
10754{
10755 struct btrfs_inode *inode = BTRFS_I(mapping->host);
10756 struct btrfs_fs_info *fs_info = inode->root->fs_info;
10757 const u64 page_start = folio_pos(folio);
10758 const u64 range_end = min_t(u64, folio_next_pos(folio),
10759 round_up(i_size_read(&inode->vfs_inode),
10760 fs_info->sectorsize));
10761
10762 if (range_end > page_start)
10763 btrfs_folio_set_fixup_dirty(fs_info, folio, page_start,
10764 range_end - page_start);
10765 return filemap_dirty_folio(mapping, folio);
10766}
10767
10553/*10768/*
10554 * btrfs doesn't support the bmap operation because swapfiles10769 * btrfs doesn't support the bmap operation because swapfiles
10555 * use bmap to make a mapping of extents in the file. They assume10770 * use bmap to make a mapping of extents in the file. They assume
@@ -10570,7 +10785,7 @@ static const struct address_space_operations btrfs_aops = { · 14 unmodified lines
10570 .launder_folio = btrfs_launder_folio,10785 .launder_folio = btrfs_launder_folio,
10571 .release_folio = btrfs_release_folio,10786 .release_folio = btrfs_release_folio,
10572 .migrate_folio = btrfs_migrate_folio,10787 .migrate_folio = btrfs_migrate_folio,
10573 .dirty_folio = filemap_dirty_folio,10788 .dirty_folio = btrfs_data_dirty_folio,
10574 .error_remove_folio = generic_error_remove_folio,10789 .error_remove_folio = generic_error_remove_folio,
10575 .swap_activate = btrfs_swap_activate,10790 .swap_activate = btrfs_swap_activate,
10576 .swap_deactivate = btrfs_swap_deactivate,10791 .swap_deactivate = btrfs_swap_deactivate,
fs/btrfs/subpage.c+211−5
@@ -345,18 +345,57 @@ void btrfs_subpage_clear_uptodate(const struct btrfs_fs_info *fs_info, · 344 unmodified lines
345 spin_unlock_irqrestore(&bfs->lock, flags);345 spin_unlock_irqrestore(&bfs->lock, flags);
346}346}
347 347
348/*
349 * folio_mark_dirty() for a folio we are dirtying with a space reservation.
350 *
351 * Dirtiers without a reservation use btrfs_data_dirty_folio().
352 */
353static void btrfs_folio_mark_dirty(struct folio *folio)
354{
355 struct address_space *mapping = folio_mapping(folio);
356
357 if (!mapping || !mapping->host || !is_data_inode(BTRFS_I(mapping->host))) {
358 folio_mark_dirty(folio);
359 return;
360 }
361 if (folio_test_reclaim(folio))
362 folio_clear_reclaim(folio);
363 filemap_dirty_folio(mapping, folio);
364}
365
366/*
367 * The set helper of the dirty ops, so it only runs for folios without a
368 * fixup bitmap: for those the folio flag is the whole fixup state, and this
369 * reserving write covers the block, so retire it. Metadata never has the
370 * flag set and only pays the test.
371 */
372static void btrfs_folio_mark_dirty_reserved(struct folio *folio)
373{
374 if (folio_test_fixup_pending(folio))
375 folio_clear_fixup_pending(folio);
376 btrfs_folio_mark_dirty(folio);
377}
378
348void btrfs_subpage_set_dirty(const struct btrfs_fs_info *fs_info,379void btrfs_subpage_set_dirty(const struct btrfs_fs_info *fs_info,
349 struct folio *folio, u64 start, u32 len)380 struct folio *folio, u64 start, u32 len)
350{381{
351 struct btrfs_folio_state *bfs = folio_get_private(folio);382 struct btrfs_folio_state *bfs = folio_get_private(folio);
352 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,383 unsigned int dirty_bit = subpage_calc_start_bit(fs_info, folio,
353 dirty, start, len);384 dirty, start, len);
385 unsigned int fixup_bit = subpage_calc_start_bit(fs_info, folio,
386 fixup, start, len);
387 const unsigned int nbits = len >> fs_info->sectorsize_bits;
354 unsigned long flags;388 unsigned long flags;
355 389
356 spin_lock_irqsave(&bfs->lock, flags);390 spin_lock_irqsave(&bfs->lock, flags);
357 bitmap_set(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits);391 bitmap_set(bfs->bitmaps, dirty_bit, nbits);
392 /* Proper dirtying obviates the need for fixup. */
393 bitmap_clear(bfs->bitmaps, fixup_bit, nbits);
394 if (folio_test_fixup_pending(folio) &&
395 subpage_test_bitmap_all_zero(fs_info, folio, fixup))
396 folio_clear_fixup_pending(folio);
358 spin_unlock_irqrestore(&bfs->lock, flags);397 spin_unlock_irqrestore(&bfs->lock, flags);
359 folio_mark_dirty(folio);398 btrfs_folio_mark_dirty(folio);
360}399}
361 400
362static void folio_clear_tags(struct folio *folio)401static void folio_clear_tags(struct folio *folio)
@@ -457,6 +496,172 @@ void btrfs_subpage_clear_writeback(const struct btrfs_fs_info *fs_info, · 94 unmodified lines
457 spin_unlock_irqrestore(&bfs->lock, flags);496 spin_unlock_irqrestore(&bfs->lock, flags);
458}497}
459 498
499void btrfs_subpage_clear_fixup(const struct btrfs_fs_info *fs_info,
500 struct folio *folio, u64 start, u32 len)
501{
502 struct btrfs_folio_state *bfs = folio_get_private(folio);
503 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,
504 fixup, start, len);
505 unsigned long flags;
506
507 spin_lock_irqsave(&bfs->lock, flags);
508 bitmap_clear(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
509 if (subpage_test_bitmap_all_zero(fs_info, folio, fixup))
510 folio_clear_fixup_pending(folio);
511 spin_unlock_irqrestore(&bfs->lock, flags);
512}
513
514/*
515 * In one pass under bfs->lock, mark every block with a clear dirty bit in the
516 * range both dirty and needing fixup.
517 *
518 * Only called from the dirty_folio callback, which owns the folio-level
519 * dirty flag; calling folio_mark_dirty() here would recurse.
520 *
521 * The folio fixup flag and bits are both set under bfs->lock so that a
522 * writeback pass observing the new bits also observes the flag.
523 */
524static void btrfs_subpage_set_fixup_dirty(const struct btrfs_fs_info *fs_info,
525 struct folio *folio, u64 start, u32 len)
526{
527 struct btrfs_folio_state *bfs = folio_get_private(folio);
528 unsigned int dirty_bit = subpage_calc_start_bit(fs_info, folio,
529 dirty, start, len);
530 unsigned int fixup_bit = subpage_calc_start_bit(fs_info, folio,
531 fixup, start, len);
532 const unsigned int nbits = len >> fs_info->sectorsize_bits;
533 unsigned long flags;
534 bool marked = false;
535
536 spin_lock_irqsave(&bfs->lock, flags);
537 for (unsigned int i = 0; i < nbits; i++) {
538 if (test_bit(dirty_bit + i, bfs->bitmaps))
539 continue;
540 set_bit(dirty_bit + i, bfs->bitmaps);
541 set_bit(fixup_bit + i, bfs->bitmaps);
542 marked = true;
543 }
544 if (marked)
545 folio_set_fixup_pending(folio);
546 spin_unlock_irqrestore(&bfs->lock, flags);
547}
548
549/*
550 * Mark the still-clean blocks of a folio dirty and needing fixup, for
551 * btrfs_data_dirty_folio().
552 *
553 * A subpage block size folio that is not uptodate is left alone: its clean
554 * blocks may hold content that was never read in, which must not be marked
555 * dirty.
556 */
557void btrfs_folio_set_fixup_dirty(const struct btrfs_fs_info *fs_info,
558 struct folio *folio, u64 start, u32 len)
559{
560 if (!btrfs_is_subpage(fs_info, folio)) {
561 if (!folio_test_dirty(folio))
562 folio_set_fixup_pending(folio);
563 return;
564 }
565 if (!folio_test_uptodate(folio))
566 return;
567 btrfs_subpage_set_fixup_dirty(fs_info, folio, start, len);
568}
569
570/*
571 * Drop the fixup blocks inside the range: clear both their fixup and dirty
572 * bits.
573 *
574 * Fixup blocks carry no space reservation, so their fixup and dirty bits
575 * must be dropped together. Clearing only the fixup bit would leave a
576 * dirty block without a reservation which is not a valid state.
577 *
578 * Returns true if the folio has no dirty blocks left.
579 */
580static bool btrfs_subpage_clear_fixup_dirty(const struct btrfs_fs_info *fs_info,
581 struct folio *folio, u64 start, u32 len)
582{
583 struct btrfs_folio_state *bfs = folio_get_private(folio);
584 unsigned int dirty_bit = subpage_calc_start_bit(fs_info, folio,
585 dirty, start, len);
586 unsigned int fixup_bit = subpage_calc_start_bit(fs_info, folio,
587 fixup, start, len);
588 const unsigned int nbits = len >> fs_info->sectorsize_bits;
589 unsigned long flags;
590 bool last;
591
592 spin_lock_irqsave(&bfs->lock, flags);
593 for (unsigned int i = 0; i < nbits; i++) {
594 if (!test_bit(fixup_bit + i, bfs->bitmaps))
595 continue;
596 clear_bit(fixup_bit + i, bfs->bitmaps);
597 clear_bit(dirty_bit + i, bfs->bitmaps);
598 }
599 if (subpage_test_bitmap_all_zero(fs_info, folio, fixup))
600 folio_clear_fixup_pending(folio);
601 last = subpage_test_bitmap_all_zero(fs_info, folio, dirty);
602 spin_unlock_irqrestore(&bfs->lock, flags);
603 return last;
604}
605
606/*
607 * Drop the fixup blocks inside the range, for callers discarding their data:
608 * btrfs_invalidate_folio() and the writepage fixup worker's error path.
609 *
610 * Callers that have just reserved space for a block want
611 * btrfs_folio_clear_fixup() instead - there the block stays dirty and gets
612 * written.
613 *
614 * The range can be byte-granular (an unaligned truncate through
615 * btrfs_invalidate_folio()); only blocks fully inside it are dropped, as a
616 * partially covered block still holds live data outside the range. For
617 * single-block folios the folio flag is the fixup state, so it is dropped
618 * only when the range covers the whole folio.
619 */
620void btrfs_folio_clear_fixup_dirty(const struct btrfs_fs_info *fs_info,
621 struct folio *folio, u64 start, u32 len)
622{
623 u64 aligned_start;
624 u64 aligned_end;
625
626 /* The folio flag is set whenever any fixup bitmap bit is. */
627 if (!folio_test_fixup_pending(folio))
628 return;
629 if (!btrfs_is_subpage(fs_info, folio)) {
630 if (start <= folio_pos(folio) &&
631 start + len >= folio_next_pos(folio)) {
632 folio_clear_fixup_pending(folio);
633 folio_clear_dirty_for_io(folio);
634 }
635 return;
636 }
637 btrfs_subpage_clamp_range(folio, &start, &len);
638 aligned_start = round_up(start, fs_info->sectorsize);
639 aligned_end = round_down(start + len, fs_info->sectorsize);
640 if (aligned_end <= aligned_start)
641 return;
642 if (btrfs_subpage_clear_fixup_dirty(fs_info, folio, aligned_start,
643 aligned_end - aligned_start))
644 folio_clear_dirty_for_io(folio);
645}
646
647bool btrfs_folio_test_fixup(const struct btrfs_fs_info *fs_info,
648 struct folio *folio, u64 start, u32 len)
649{
650 if (!btrfs_is_subpage(fs_info, folio))
651 return folio_test_fixup_pending(folio);
652 return btrfs_subpage_test_fixup(fs_info, folio, start, len);
653}
654
655void btrfs_folio_clear_fixup(const struct btrfs_fs_info *fs_info,
656 struct folio *folio, u64 start, u32 len)
657{
658 if (!btrfs_is_subpage(fs_info, folio)) {
659 folio_clear_fixup_pending(folio);
660 return;
661 }
662 btrfs_subpage_clear_fixup(fs_info, folio, start, len);
663}
664
460/*665/*
461 * Unlike set/clear which is dependent on each page status, for test all bits666 * Unlike set/clear which is dependent on each page status, for test all bits
462 * are tested in the same way.667 * are tested in the same way.
@@ -480,6 +685,7 @@ bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info, \ · 17 unmodified lines
480IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(uptodate);685IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(uptodate);
481IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(dirty);686IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(dirty);
482IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(writeback);687IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(writeback);
688IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(fixup);
483 689
484/*690/*
485 * Note that, in selftests (extent-io-tests), we can have empty fs_info passed691 * Note that, in selftests (extent-io-tests), we can have empty fs_info passed
@@ -571,8 +777,8 @@ bool btrfs_meta_folio_test_##name(struct folio *folio, const struct extent_buffe · 85 unmodified lines
571}777}
572IMPLEMENT_BTRFS_PAGE_OPS(uptodate, folio_mark_uptodate, folio_clear_uptodate,778IMPLEMENT_BTRFS_PAGE_OPS(uptodate, folio_mark_uptodate, folio_clear_uptodate,
573 folio_test_uptodate);779 folio_test_uptodate);
574IMPLEMENT_BTRFS_PAGE_OPS(dirty, folio_mark_dirty, folio_clear_dirty_for_io,780IMPLEMENT_BTRFS_PAGE_OPS(dirty, btrfs_folio_mark_dirty_reserved,
575 folio_test_dirty);781 folio_clear_dirty_for_io, folio_test_dirty);
576IMPLEMENT_BTRFS_PAGE_OPS(writeback, folio_start_writeback, folio_end_writeback,782IMPLEMENT_BTRFS_PAGE_OPS(writeback, folio_start_writeback, folio_end_writeback,
577 folio_test_writeback);783 folio_test_writeback);
578 784
fs/btrfs/subpage.h+36−5
@@ -14,15 +14,15 @@ struct folio; · 13 unmodified lines
14/*14/*
15 * Extra info for subpage bitmap.15 * Extra info for subpage bitmap.
16 *16 *
17 * For subpage we pack all uptodate/dirty/writeback bitmaps into17 * For subpage we pack all uptodate/dirty/writeback/fixup bitmaps into
18 * one larger bitmap.18 * one larger bitmap.
19 *19 *
20 * This structure records how they are organized in the bitmap:20 * This structure records how they are organized in the bitmap:
21 *21 *
22 * /- uptodate /- dirty /- writeback22 * /- uptodate /- dirty /- writeback /- fixup
23 * | | |23 * | | | |
24 * v v v24 * v v v v
25 * |u|u|u|u|........|u|u|d|d|.......|d|d|w|w|.......|w|w|25 * |u|u|u|u|........|u|u|d|d|.......|d|d|w|w|.....|w|w|f|f|.....|f|f|
26 * |< sectors_per_page >|26 * |< sectors_per_page >|
27 *27 *
28 * Unlike regular macro-like enums, here we do not go upper-case names, as28 * Unlike regular macro-like enums, here we do not go upper-case names, as
@@ -40,6 +40,14 @@ enum { · 11 unmodified lines
40 */40 */
41 btrfs_bitmap_nr_writeback,41 btrfs_bitmap_nr_writeback,
42 42
43 /*
44 * Blocks dirtied by the dirty_folio callback instead of a reserving
45 * write path (e.g. set_page_dirty_lock() on a GUP pin). They have
46 * no space reservation and need the writepage fixup before they can
47 * be submitted.
48 */
49 btrfs_bitmap_nr_fixup,
50
43 btrfs_bitmap_nr_max51 btrfs_bitmap_nr_max
44};52};
45 53
@@ -165,6 +173,29 @@ DECLARE_BTRFS_SUBPAGE_OPS(uptodate); · 119 unmodified lines
165DECLARE_BTRFS_SUBPAGE_OPS(dirty);173DECLARE_BTRFS_SUBPAGE_OPS(dirty);
166DECLARE_BTRFS_SUBPAGE_OPS(writeback);174DECLARE_BTRFS_SUBPAGE_OPS(writeback);
167 175
176/*
177 * Fixup bit helpers.
178 *
179 * The fixup bit is data-only and has no plain set helper (setting happens
180 * together with dirtying in btrfs_subpage_set_fixup_dirty()), so it does not
181 * go through DECLARE_BTRFS_SUBPAGE_OPS(). For single-block folios the
182 * folio_*_fixup_pending() flag takes the place of the bitmap.
183 */
184void btrfs_subpage_clear_fixup(const struct btrfs_fs_info *fs_info,
185 struct folio *folio, u64 start, u32 len);
186bool btrfs_subpage_test_fixup(const struct btrfs_fs_info *fs_info,
187 struct folio *folio, u64 start, u32 len);
188bool btrfs_folio_test_fixup(const struct btrfs_fs_info *fs_info,
189 struct folio *folio, u64 start, u32 len);
190void btrfs_folio_set_fixup_dirty(const struct btrfs_fs_info *fs_info,
191 struct folio *folio, u64 start, u32 len);
192/* For a block that just got its space reserved; it stays dirty. */
193void btrfs_folio_clear_fixup(const struct btrfs_fs_info *fs_info,
194 struct folio *folio, u64 start, u32 len);
195/* For callers discarding the data; clears the dirty bits too. */
196void btrfs_folio_clear_fixup_dirty(const struct btrfs_fs_info *fs_info,
197 struct folio *folio, u64 start, u32 len);
198
168/*199/*
169 * Helper for error cleanup, where a folio will have its dirty flag cleared,200 * Helper for error cleanup, where a folio will have its dirty flag cleared,
170 * with writeback started and finished.201 * with writeback started and finished.
include/trace/events/btrfs.h+35−0
@@ -689,6 +689,41 @@ DEFINE_EVENT(btrfs__ordered_extent, btrfs_ordered_extent_lookup_first, · 688 unmodified lines
689 TP_ARGS(inode, ordered)689 TP_ARGS(inode, ordered)
690);690);
691 691
692/*
693 * The writepage fixup worker deferred a block because this still-running
694 * ordered extent covers it.
695 */
696DEFINE_EVENT(btrfs__ordered_extent, btrfs_writepage_fixup_defer,
697
698 TP_PROTO(const struct btrfs_inode *inode,
699 const struct btrfs_ordered_extent *ordered),
700
701 TP_ARGS(inode, ordered)
702);
703
704/* The writepage fixup worker reserved space for a block and set delalloc. */
705TRACE_EVENT(btrfs_writepage_fixup_reserve,
706
707 TP_PROTO(const struct btrfs_inode *inode, u64 start, u32 len),
708
709 TP_ARGS(inode, start, len),
710
711 TP_STRUCT__entry_btrfs(
712 __field( u64, ino )
713 __field( u64, start )
714 __field( u32, len )
715 ),
716
717 TP_fast_assign_btrfs(inode->root->fs_info,
718 __entry->ino = btrfs_ino(inode);
719 __entry->start = start;
720 __entry->len = len;
721 ),
722
723 TP_printk_btrfs("ino=%llu start=%llu len=%u",
724 __entry->ino, __entry->start, __entry->len)
725);
726
692DEFINE_EVENT(btrfs__ordered_extent, btrfs_ordered_extent_split,727DEFINE_EVENT(btrfs__ordered_extent, btrfs_ordered_extent_split,
693 728
694 TP_PROTO(const struct btrfs_inode *inode,729 TP_PROTO(const struct btrfs_inode *inode,