Merge tag 'for-7.2-rc6-fixup-worker-tag' of git://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux

Pull Btrfs Fixes 2: Electric Boogaloo from David Sterba: "This brings back the fixup worker infrastructure. It's a mechanism to detect pages/folios that are marked dirty without filesystem knowledge and require COW fixup. The consequence of not doing so is silent data loss. The first patch covers the scenarios in detail, also reflecting folio API port and subpage block size support added in recent years. The original fixup worker was only for pages. The patch is relatively big, half of the code is debugging and support code, the rest is the core design around the detection and fix. The second patch handles an unlikely case when there's work left during unmount" * tag 'for-7.2-rc6-fixup-worker-tag' of git://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux: btrfs: flush the fixup workers during close_ctree btrfs: trigger cow fixup via dirty_folio()

Linus Torvalds committed 3 months ago

Commit 6c68fa62 parents 6335463 ae2567b8 files · +651 −12Browse files

Unified · Split

8 files

fs/btrfs/btrfs_inode.h +1 −0
fs/btrfs/disk-io.c +27 −1
fs/btrfs/extent_io.c +113 −0
fs/btrfs/fs.h +12 −0
fs/btrfs/inode.c +216 −1
fs/btrfs/subpage.c +211 −5
fs/btrfs/subpage.h +36 −5
include/trace/events/btrfs.h +35 −0

fs/btrfs/btrfs_inode.h+1−0

@@ -600,6 +600,7 @@ int btrfs_prealloc_file_range_trans(struct inode *inode, · 599 unmodified lines
600	600	loff_t actual_len, u64 *alloc_hint);
601	601	int btrfs_run_delalloc_range(struct btrfs_inode inode, struct folio locked_folio,
602	602	u64 start, u64 end, struct writeback_control *wbc);
	603	+void btrfs_queue_writepage_fixup(struct btrfs_inode inode, struct folio folio);
603	604	int btrfs_encoded_io_compression_from_extent(struct btrfs_fs_info *fs_info,
604	605	int compress_type);
605	606	int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode,

fs/btrfs/disk-io.c+27−1

@@ -1760,6 +1760,8 @@ static int read_backup_root(struct btrfs_fs_info *fs_info, u8 priority) · 1759 unmodified lines
1760	1760	/* helper to cleanup workers */
1761	1761	static void btrfs_stop_all_workers(struct btrfs_fs_info *fs_info)
1762	1762	{
	1763	+ if (fs_info->fixup_workers)
	1764	+ destroy_workqueue(fs_info->fixup_workers);
1763	1765	btrfs_destroy_workqueue(fs_info->delalloc_workers);
1764	1766	btrfs_destroy_workqueue(fs_info->workers);
1765	1767	if (fs_info->endio_workers)
@@ -1967,6 +1969,9 @@ static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info) · 201 unmodified lines
1967	1969	fs_info->caching_workers =
1968	1970	btrfs_alloc_workqueue(fs_info, "cache", flags, max_active, 0);
1969	1971
	1972	+ fs_info->fixup_workers =
	1973	+ alloc_ordered_workqueue("btrfs-fixup", ordered_flags);
	1974	+
1970	1975	fs_info->endio_workers =
1971	1976	alloc_workqueue("btrfs-endio", flags, max_active);
1972	1977	fs_info->endio_meta_workers =
@@ -1992,7 +1997,7 @@ static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info) · 19 unmodified lines
1992	1997	fs_info->endio_workers && fs_info->endio_meta_workers &&
1993	1998	fs_info->endio_write_workers &&
1994	1999	fs_info->endio_freespace_worker && fs_info->rmw_workers &&
1995		− fs_info->caching_workers &&
	2000	+ fs_info->caching_workers && fs_info->fixup_workers &&
1996	2001	fs_info->delayed_workers && fs_info->qgroup_rescan_workers &&
1997	2002	fs_info->discard_ctl.discard_workers)) {
1998	2003	return -ENOMEM;
@@ -4364,6 +4369,18 @@ void __cold close_ctree(struct btrfs_fs_info *fs_info) · 2365 unmodified lines
4364	4369	/* clear out the rbtree of defraggable inodes */
4365	4370	btrfs_cleanup_defrag_inodes(fs_info);
4366	4371
	4372	+ /*
	4373	+ * Before the unmount, we sync down all the writeback which can
	4374	+ * generate fixup work. We are about to run delalloc for autodefrag so
	4375	+ * piggy back on that by also flushing the fixup work which can also
	4376	+ * generate delalloc we would like to get run.
	4377	+ *
	4378	+ * After this, it is still possible that some thread doing writeback is
	4379	+ * in btrfs_queue_writepage_fixup() and might finish queueing some final
	4380	+ * work, racing the btrfs_fs_closing() check there.
	4381	+ */
	4382	+ flush_workqueue(fs_info->fixup_workers);
	4383	+
4367	4384	/*
4368	4385	* Handle the error fs first, as it will flush and wait for all ordered
4369	4386	* extents. This will generate delayed iputs, thus we want to handle
@@ -4441,6 +4458,15 @@ void __cold close_ctree(struct btrfs_fs_info *fs_info) · 71 unmodified lines
4441	4458	cancel_work_sync(&fs_info->preempt_reclaim_work);
4442	4459	cancel_work_sync(&fs_info->em_shrinker_work);
4443	4460
	4461	+ /*
	4462	+ * Reclaim workers can run writeback which can queue fixup.
	4463	+ * After the above cancel_work_sync() calls, any such queueing attempts are
	4464	+ * guaranteed to see btrfs_fs_closing(), so at this point we can genuinely fully
	4465	+ * flush the fixup workqueue. This relies on the belief that now no thread can
	4466	+ * still be sitting in btrfs_queue_writepage_fixup().
	4467	+ */
	4468	+ flush_workqueue(fs_info->fixup_workers);
	4469	+
4444	4470	/*
4445	4471	* Run delayed iputs again because an async reclaim worker may have
4446	4472	* added new ones if it was flushing delalloc:

fs/btrfs/extent_io.c+113−0

@@ -1440,6 +1440,115 @@ static bool find_next_delalloc_bitmap(struct folio *folio, · 1439 unmodified lines
1440	1440	return true;
1441	1441	}
1442	1442
	1443	+/*
	1444	+ * Debug checks for fixup selection logic to help ensure the invariants
	1445	+ * we expect for fixup marking hold in practice.
	1446	+ *
	1447	+ * - A dirty block without a fixup bit is covered by delalloc or a running
	1448	+ * ordered extent (it was dirtied by a reserving write path).
	1449	+ * - A block with a fixup bit is never covered by delalloc: every delalloc
	1450	+ * setter holds the folio lock and cancels the fixup state of the blocks
	1451	+ * it covers (btrfs_folio_set_dirty()) before releasing it.
	1452	+ */
	1453	+static void debug_check_writepage_fixup(struct btrfs_inode *inode, u64 start,
	1454	+ u32 len, bool needs_fixup)
	1455	+{
	1456	+ struct btrfs_ordered_extent *ordered;
	1457	+ bool delalloc;
	1458	+
	1459	+ if (!IS_ENABLED(CONFIG_BTRFS_DEBUG))
	1460	+ return;
	1461	+
	1462	+ delalloc = btrfs_test_range_bit_exists(&inode->io_tree, start,
	1463	+ start + len - 1, EXTENT_DELALLOC);
	1464	+ if (needs_fixup) {
	1465	+ if (unlikely(delalloc))
	1466	+ DEBUG_WARN("writeback: delalloc and fixup conflict. ino %llu start %llu",
	1467	+ btrfs_ino(inode), start);
	1468	+ } else {
	1469	+ if (delalloc)
	1470	+ return;
	1471	+
	1472	+ ordered = btrfs_lookup_ordered_range(inode, start, len);
	1473	+ if (unlikely(!ordered))
	1474	+ DEBUG_WARN("dirty block, no delalloc, fixup, ordered. ino %llu start %llu",
	1475	+ btrfs_ino(inode), start);
	1476	+ else
	1477	+ btrfs_put_ordered_extent(ordered);
	1478	+ }
	1479	+}
	1480	+
	1481	+/*
	1482	+ * Handle folios dirtied without a delalloc reservation, e.g.
	1483	+ * O_DIRECT read into a MAP_SHARED mapping dirtying via set_page_dirty_lock().
	1484	+ *
	1485	+ * btrfs_data_dirty_folio() records the affected blocks in the fixup bitmap
	1486	+ * and the folio fixup flag and we check them here in writeback.
	1487	+ *
	1488	+ * Don't submit such blocks and queue work for the fixup worker to reserve
	1489	+ * space for them so that they can be submitted properly by writeback.
	1490	+ *
	1491	+ * Return 1 if the folio needed fixup, 0 if not, and a negative error code
	1492	+ * on error.
	1493	+ */
	1494	+static noinline_for_stack int writepage_fixup(struct btrfs_inode *inode,
	1495	+ struct folio *folio,
	1496	+ struct btrfs_bio_ctrl *bio_ctrl)
	1497	+{
	1498	+ struct btrfs_fs_info *fs_info = inode_to_fs_info(&inode->vfs_inode);
	1499	+ const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio);
	1500	+ const u32 sectorsize = fs_info->sectorsize;
	1501	+ const u64 page_start = folio_pos(folio);
	1502	+ bool found_fixup = false;
	1503	+ unsigned int bit;
	1504	+
	1505	+ /*
	1506	+ * A folio was dirtied without calling aops->dirty_folio() which we
	1507	+ * explicitly assert is not allowed.
	1508	+ */
	1509	+ if (unlikely(bitmap_empty(bio_ctrl->submit_bitmap, blocks_per_folio))) {
	1510	+ DEBUG_WARN();
	1511	+ btrfs_err_rl(fs_info,
	1512	+ "root %lld ino %llu folio %llu is dirty with an empty dirty bitmap",
	1513	+ btrfs_root_id(inode->root), btrfs_ino(inode),
	1514	+ folio_pos(folio));
	1515	+ return -EUCLEAN;
	1516	+ }
	1517	+
	1518	+ /* Cheap check on the folio flag. Set iff the fixup bitmap is non-empty. */
	1519	+ if (likely(!folio_test_fixup_pending(folio)))
	1520	+ return 0;
	1521	+
	1522	+ for_each_set_bit(bit, bio_ctrl->submit_bitmap, blocks_per_folio) {
	1523	+ const u64 start = page_start + (bit << fs_info->sectorsize_bits);
	1524	+ const bool needs_fixup = btrfs_folio_test_fixup(fs_info, folio,
	1525	+ start, sectorsize);
	1526	+
	1527	+ debug_check_writepage_fixup(inode, start, sectorsize, needs_fixup);
	1528	+ if (needs_fixup) {
	1529	+ bitmap_clear(bio_ctrl->submit_bitmap, bit, 1);
	1530	+ found_fixup = true;
	1531	+ }
	1532	+ }
	1533	+ if (likely(found_fixup)) {
	1534	+ btrfs_queue_writepage_fixup(inode, folio);
	1535	+ folio_redirty_for_writepage(bio_ctrl->wbc, folio);
	1536	+ if (bitmap_empty(bio_ctrl->submit_bitmap, blocks_per_folio)) {
	1537	+ folio_unlock(folio);
	1538	+ return 1;
	1539	+ }
	1540	+ return 0;
	1541	+ }
	1542	+ /* We should always find fixup if the folio fixup flag was set. */
	1543	+ DEBUG_WARN();
	1544	+ btrfs_err_rl(fs_info,
	1545	+ "root %lld ino %llu folio %llu is fixup with an empty fixup bitmap",
	1546	+ btrfs_root_id(inode->root), btrfs_ino(inode),
	1547	+ folio_pos(folio));
	1548	+
	1549	+ return -EUCLEAN;
	1550	+}
	1551	+
1443	1552	/*
1444	1553	* Do all of the delayed allocation setup.
1445	1554	*
@@ -1492,6 +1601,10 @@ static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode, · 46 unmodified lines
1492	1601	/* Save the dirty bitmap as our submission bitmap will be a subset of it. */
1493	1602	btrfs_copy_subpage_dirty_bitmap(fs_info, folio, bio_ctrl->submit_bitmap);
1494	1603
	1604	+ ret = writepage_fixup(inode, folio, bio_ctrl);
	1605	+ if (ret)
	1606	+ return ret;
	1607	+
1495	1608	for_each_set_bitrange(start_bit, end_bit, bio_ctrl->submit_bitmap,
1496	1609	blocks_per_folio) {
1497	1610	u64 start = page_start + (start_bit << fs_info->sectorsize_bits);

fs/btrfs/fs.h+12−0

@@ -713,6 +713,8 @@ struct btrfs_fs_info { · 712 unmodified lines
713	713	struct btrfs_workqueue *endio_write_workers;
714	714	struct btrfs_workqueue *endio_freespace_worker;
715	715	struct btrfs_workqueue *caching_workers;
	716	+
	717	+ struct workqueue_struct *fixup_workers;
716	718	struct btrfs_workqueue *delayed_workers;
717	719
718	720	struct task_struct *transaction_kthread;
@@ -1200,6 +1202,16 @@ static inline void btrfs_wake_unfinished_drop(struct btrfs_fs_info *fs_info) · 481 unmodified lines
1200	1202	clear_and_wake_up_bit(BTRFS_FS_UNFINISHED_DROPS, &fs_info->flags);
1201	1203	}
1202	1204
	1205	+/*
	1206	+ * We use the folio owner_2 flag to indicate the folio has blocks that were
	1207	+ * dirtied without a space reservation and need the writepage fixup before
	1208	+ * writeback. For bs < folio_size the fixup bitmap tracks the affected
	1209	+ * blocks.
	1210	+ */
	1211	+#define folio_test_fixup_pending(folio) folio_test_owner_2(folio)
	1212	+#define folio_set_fixup_pending(folio) folio_set_owner_2(folio)
	1213	+#define folio_clear_fixup_pending(folio) folio_clear_owner_2(folio)
	1214	+
1203	1215	#define BTRFS_FS_ERROR(fs_info) (READ_ONCE((fs_info)->fs_error))
1204	1216
1205	1217	#define BTRFS_FS_LOG_CLEANUP_ERROR(fs_info) \

fs/btrfs/inode.c+216−1

@@ -2812,6 +2812,180 @@ int btrfs_set_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, · 2811 unmodified lines
2812	2812	EXTENT_DELALLOC \| extra_bits, cached_state);
2813	2813	}
2814	2814
	2815	+struct btrfs_writepage_fixup {
	2816	+ struct folio *folio;
	2817	+ struct btrfs_inode *inode;
	2818	+ struct work_struct work;
	2819	+};
	2820	+
	2821	+/*
	2822	+ * Do the real fixup work of reserving space for the blocks a folio's fixup
	2823	+ * state records. Queued by writepage_fixup() when writeback found the bits set.
	2824	+ *
	2825	+ * Since the fixup can be cancelled by a task dirtying with a reservation, we must
	2826	+ * re-check the state of fixup under the folio lock.
	2827	+ */
	2828	+static void btrfs_writepage_fixup_worker(struct work_struct *work)
	2829	+{
	2830	+ struct btrfs_writepage_fixup *fixup =
	2831	+ container_of(work, struct btrfs_writepage_fixup, work);
	2832	+ struct extent_state *cached_state = NULL;
	2833	+ struct extent_changeset *data_reserved = NULL;
	2834	+ unsigned long delalloc_bitmap[BITS_TO_LONGS(BTRFS_MAX_BLOCKS_PER_FOLIO)] = { 0 };
	2835	+ struct folio *folio = fixup->folio;
	2836	+ struct btrfs_inode *inode = fixup->inode;
	2837	+ struct btrfs_fs_info *fs_info = inode->root->fs_info;
	2838	+ const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio);
	2839	+ const u32 sectorsize = fs_info->sectorsize;
	2840	+ const u64 page_start = folio_pos(folio);
	2841	+ const u64 page_end = folio_next_pos(folio) - 1;
	2842	+ unsigned int start_bit;
	2843	+ unsigned int end_bit;
	2844	+ unsigned int bit;
	2845	+ bool reserved;
	2846	+ int ret;
	2847	+
	2848	+ /*
	2849	+ * We would prefer to reserve under the folio lock when we know exactly
	2850	+ * which blocks need a reservation. Unfortunately, since the reservation
	2851	+ * can go into flushers which can go into writeback, which takes folio
	2852	+ * locks, that is not possible. Therefore, we have to reserve for the
	2853	+ * whole folio here, then release what we didn't end up needing once we
	2854	+ * figure it out.
	2855	+ *
	2856	+ * Also note the slightly strange error checking. If fixup is actually
	2857	+ * not set, we don't need to mark an error on the mapping. So hang on to
	2858	+ * ret until after we lock and find out if we actually care.
	2859	+ */
	2860	+ ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start,
	2861	+ folio_size(folio));
	2862	+ reserved = (ret == 0);
	2863	+again:
	2864	+ folio_lock(folio);
	2865	+
	2866	+ if (!folio->mapping \|\| !folio_test_fixup_pending(folio)) {
	2867	+ ret = 0;
	2868	+ goto out;
	2869	+ }
	2870	+ if (ret)
	2871	+ goto out;
	2872	+
	2873	+ btrfs_lock_extent(&inode->io_tree, page_start, page_end, &cached_state);
	2874	+
	2875	+ for (bit = 0; bit < blocks_per_folio; bit++) {
	2876	+ struct btrfs_ordered_extent *ordered;
	2877	+ const u64 start = page_start + (bit << fs_info->sectorsize_bits);
	2878	+
	2879	+ if (test_bit(bit, delalloc_bitmap))
	2880	+ continue;
	2881	+ if (!btrfs_folio_test_fixup(fs_info, folio, start, sectorsize))
	2882	+ continue;
	2883	+ /*
	2884	+ * Any task that sets EXTENT_DELALLOC clears the fixup bits
	2885	+ * under the folio lock, so it should be impossible to observe
	2886	+ * both under the lock. Setting delalloc twice would wrongly
	2887	+ * double account the space.
	2888	+ */
	2889	+ if (IS_ENABLED(CONFIG_BTRFS_DEBUG) &&
	2890	+ unlikely(btrfs_test_range_bit_exists(&inode->io_tree, start,
	2891	+ start + sectorsize - 1,
	2892	+ EXTENT_DELALLOC))) {
	2893	+ DEBUG_WARN("fixup worker: delalloc and fixup conflict. ino %llu start %llu",
	2894	+ btrfs_ino(inode), start);
	2895	+ btrfs_folio_clear_fixup(fs_info, folio, start, sectorsize);
	2896	+ continue;
	2897	+ }
	2898	+ ordered = btrfs_lookup_ordered_range(inode, start, sectorsize);
	2899	+ if (ordered) {
	2900	+ trace_btrfs_writepage_fixup_defer(inode, ordered);
	2901	+ btrfs_unlock_extent(&inode->io_tree, page_start,
	2902	+ page_end, &cached_state);
	2903	+ folio_unlock(folio);
	2904	+ btrfs_start_ordered_extent(ordered);
	2905	+ btrfs_put_ordered_extent(ordered);
	2906	+ goto again;
	2907	+ }
	2908	+ ret = btrfs_set_extent_delalloc(inode, start,
	2909	+ start + sectorsize - 1, 0,
	2910	+ &cached_state);
	2911	+ if (ret)
	2912	+ break;
	2913	+ trace_btrfs_writepage_fixup_reserve(inode, start, sectorsize);
	2914	+ btrfs_folio_clear_fixup(fs_info, folio, start, sectorsize);
	2915	+ set_bit(bit, delalloc_bitmap);
	2916	+ }
	2917	+
	2918	+ btrfs_unlock_extent(&inode->io_tree, page_start, page_end, &cached_state);
	2919	+out:
	2920	+ if (ret < 0) {
	2921	+ /* Failure here is analogous to failure in writeback. */
	2922	+ mapping_set_error(folio->mapping, ret);
	2923	+ btrfs_folio_clear_fixup_dirty(fs_info, folio, page_start,
	2924	+ folio_size(folio));
	2925	+ }
	2926	+ if (reserved) {
	2927	+ btrfs_delalloc_release_extents(inode, folio_size(folio));
	2928	+ for_each_clear_bitrange(start_bit, end_bit, delalloc_bitmap,
	2929	+ blocks_per_folio)
	2930	+ btrfs_delalloc_release_space(inode, data_reserved,
	2931	+ page_start + (start_bit << fs_info->sectorsize_bits),
	2932	+ (end_bit - start_bit) << fs_info->sectorsize_bits,
	2933	+ true);
	2934	+ }
	2935	+ folio_unlock(folio);
	2936	+ folio_put(folio);
	2937	+ kfree(fixup);
	2938	+ extent_changeset_free(data_reserved);
	2939	+ btrfs_add_delayed_iput(inode);
	2940	+}
	2941	+
	2942	+/*
	2943	+ * Queue space reservation fixup work for blocks dirtied without a space reservation.
	2944	+ *
	2945	+ * Should be used by writeback while holding the folio locked.
	2946	+ *
	2947	+ * If we fail to queue fixup, then the folio state is unchanged and a future
	2948	+ * writeback pass will still see it.
	2949	+ */
	2950	+void btrfs_queue_writepage_fixup(struct btrfs_inode inode, struct folio folio)
	2951	+{
	2952	+ struct btrfs_fs_info *fs_info = inode->root->fs_info;
	2953	+ struct btrfs_writepage_fixup *fixup;
	2954	+
	2955	+ /*
	2956	+ * Disallow queueing more fixup during unmount to break the cycle
	2957	+ * of writeback queuing fixup queuing writeback etc.
	2958	+ *
	2959	+ * If it actually hit, then something which was fixup wasn't written
	2960	+ * which we should warn about.
	2961	+ */
	2962	+ if (btrfs_fs_closing(fs_info)) {
	2963	+ btrfs_warn_rl(fs_info,
	2964	+ "dropping unqueued fixup blocks at unmount. root %lld ino %llu folio %llu",
	2965	+ btrfs_root_id(inode->root), btrfs_ino(inode),
	2966	+ folio_pos(folio));
	2967	+ btrfs_folio_clear_fixup_dirty(fs_info, folio,
	2968	+ folio_pos(folio), folio_size(folio));
	2969	+ return;
	2970	+ }
	2971	+
	2972	+ fixup = kzalloc_obj(*fixup, GFP_NOFS);
	2973	+ if (!fixup)
	2974	+ return;
	2975	+
	2976	+ /*
	2977	+ * This is called from within extent_write_cache_pages() which
	2978	+ * has successfully done an igrab(). But that will be released at the
	2979	+ * end of the writeback pass. We need to extend it for the worker as well.
	2980	+ */
	2981	+ ihold(&inode->vfs_inode);
	2982	+ folio_get(folio);
	2983	+ INIT_WORK(&fixup->work, btrfs_writepage_fixup_worker);
	2984	+ fixup->folio = folio;
	2985	+ fixup->inode = inode;
	2986	+ queue_work(fs_info->fixup_workers, &fixup->work);
	2987	+}
	2988	+
2815	2989	/*
2816	2990	* Clear the old accounting flags and set EXTENT_DELALLOC for the range.
2817	2991	*
@@ -7508,6 +7682,12 @@ static void btrfs_invalidate_folio(struct folio *folio, size_t offset, · 4690 unmodified lines
7508	7682	folio_wait_writeback(folio);
7509	7683	wait_subpage_spinlock(folio);
7510	7684
	7685	+ /*
	7686	+ * The invalidated blocks are going away; drop any fixup blocks among
	7687	+ * them, data included, as they have no space reservation.
	7688	+ */
	7689	+ btrfs_folio_clear_fixup_dirty(fs_info, folio, page_start + offset, length);
	7690	+
7511	7691	/*
7512	7692	* For subpage case, we have call sites like
7513	7693	* btrfs_punch_hole_lock_range() which passes range not aligned to
@@ -10550,6 +10730,41 @@ static const struct file_operations btrfs_dir_file_operations = { · 3036 unmodified lines
10550	10730	.setlease = generic_setlease,
10551	10731	};
10552	10732
	10733	+/*
	10734	+ * The folio is going dirty without a btrfs delalloc space reservation.
	10735	+ * This requires a fixup before writeback which we might sleep so cannot
	10736	+ * run in this context, so we merely set state on the folio indicating it
	10737	+ * needs fixup before writeback.
	10738	+ *
	10739	+ * Note that there is no range in the input, so the whole folio is marked
	10740	+ * dirty and fixup.
	10741	+ *
	10742	+ * We believe that all callers of dirty_folio either:
	10743	+ * - take the folio lock (e.g. pinned folio release notification).
	10744	+ * - take the pte lock but must be running on a dirty pte which means
	10745	+ * page_mkwrite() ran on it and reserved the space. zap_pte_range() cannot
	10746	+ * race with writeback cleaning the folio because writeback runs
	10747	+ * folio_mkclean() which also uses the pte lock and revokes outstanding
	10748	+ * writable mappings.
	10749	+ * Therefore, an additional folio private lock (a la bfs->lock for all cases,
	10750	+ * not just subpage) is not necessary.
	10751	+ */
	10752	+static bool btrfs_data_dirty_folio(struct address_space *mapping,
	10753	+ struct folio *folio)
	10754	+{
	10755	+ struct btrfs_inode *inode = BTRFS_I(mapping->host);
	10756	+ struct btrfs_fs_info *fs_info = inode->root->fs_info;
	10757	+ const u64 page_start = folio_pos(folio);
	10758	+ const u64 range_end = min_t(u64, folio_next_pos(folio),
	10759	+ round_up(i_size_read(&inode->vfs_inode),
	10760	+ fs_info->sectorsize));
	10761	+
	10762	+ if (range_end > page_start)
	10763	+ btrfs_folio_set_fixup_dirty(fs_info, folio, page_start,
	10764	+ range_end - page_start);
	10765	+ return filemap_dirty_folio(mapping, folio);
	10766	+}
	10767	+
10553	10768	/*
10554	10769	* btrfs doesn't support the bmap operation because swapfiles
10555	10770	* use bmap to make a mapping of extents in the file. They assume
@@ -10570,7 +10785,7 @@ static const struct address_space_operations btrfs_aops = { · 14 unmodified lines
10570	10785	.launder_folio = btrfs_launder_folio,
10571	10786	.release_folio = btrfs_release_folio,
10572	10787	.migrate_folio = btrfs_migrate_folio,
10573		− .dirty_folio = filemap_dirty_folio,
	10788	+ .dirty_folio = btrfs_data_dirty_folio,
10574	10789	.error_remove_folio = generic_error_remove_folio,
10575	10790	.swap_activate = btrfs_swap_activate,
10576	10791	.swap_deactivate = btrfs_swap_deactivate,

fs/btrfs/subpage.c+211−5

@@ -345,18 +345,57 @@ void btrfs_subpage_clear_uptodate(const struct btrfs_fs_info *fs_info, · 344 unmodified lines
345	345	spin_unlock_irqrestore(&bfs->lock, flags);
346	346	}
347	347
	348	+/*
	349	+ * folio_mark_dirty() for a folio we are dirtying with a space reservation.
	350	+ *
	351	+ * Dirtiers without a reservation use btrfs_data_dirty_folio().
	352	+ */
	353	+static void btrfs_folio_mark_dirty(struct folio *folio)
	354	+{
	355	+ struct address_space *mapping = folio_mapping(folio);
	356	+
	357	+ if (!mapping \|\| !mapping->host \|\| !is_data_inode(BTRFS_I(mapping->host))) {
	358	+ folio_mark_dirty(folio);
	359	+ return;
	360	+ }
	361	+ if (folio_test_reclaim(folio))
	362	+ folio_clear_reclaim(folio);
	363	+ filemap_dirty_folio(mapping, folio);
	364	+}
	365	+
	366	+/*
	367	+ * The set helper of the dirty ops, so it only runs for folios without a
	368	+ * fixup bitmap: for those the folio flag is the whole fixup state, and this
	369	+ * reserving write covers the block, so retire it. Metadata never has the
	370	+ * flag set and only pays the test.
	371	+ */
	372	+static void btrfs_folio_mark_dirty_reserved(struct folio *folio)
	373	+{
	374	+ if (folio_test_fixup_pending(folio))
	375	+ folio_clear_fixup_pending(folio);
	376	+ btrfs_folio_mark_dirty(folio);
	377	+}
	378	+
348	379	void btrfs_subpage_set_dirty(const struct btrfs_fs_info *fs_info,
349	380	struct folio *folio, u64 start, u32 len)
350	381	{
351	382	struct btrfs_folio_state *bfs = folio_get_private(folio);
352		− unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,
	383	+ unsigned int dirty_bit = subpage_calc_start_bit(fs_info, folio,
353	384	dirty, start, len);
	385	+ unsigned int fixup_bit = subpage_calc_start_bit(fs_info, folio,
	386	+ fixup, start, len);
	387	+ const unsigned int nbits = len >> fs_info->sectorsize_bits;
354	388	unsigned long flags;
355	389
356	390	spin_lock_irqsave(&bfs->lock, flags);
357		− bitmap_set(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
	391	+ bitmap_set(bfs->bitmaps, dirty_bit, nbits);
	392	+ /* Proper dirtying obviates the need for fixup. */
	393	+ bitmap_clear(bfs->bitmaps, fixup_bit, nbits);
	394	+ if (folio_test_fixup_pending(folio) &&
	395	+ subpage_test_bitmap_all_zero(fs_info, folio, fixup))
	396	+ folio_clear_fixup_pending(folio);
358	397	spin_unlock_irqrestore(&bfs->lock, flags);
359		− folio_mark_dirty(folio);
	398	+ btrfs_folio_mark_dirty(folio);
360	399	}
361	400
362	401	static void folio_clear_tags(struct folio *folio)
@@ -457,6 +496,172 @@ void btrfs_subpage_clear_writeback(const struct btrfs_fs_info *fs_info, · 94 unmodified lines
457	496	spin_unlock_irqrestore(&bfs->lock, flags);
458	497	}
459	498
	499	+void btrfs_subpage_clear_fixup(const struct btrfs_fs_info *fs_info,
	500	+ struct folio *folio, u64 start, u32 len)
	501	+{
	502	+ struct btrfs_folio_state *bfs = folio_get_private(folio);
	503	+ unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,
	504	+ fixup, start, len);
	505	+ unsigned long flags;
	506	+
	507	+ spin_lock_irqsave(&bfs->lock, flags);
	508	+ bitmap_clear(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
	509	+ if (subpage_test_bitmap_all_zero(fs_info, folio, fixup))
	510	+ folio_clear_fixup_pending(folio);
	511	+ spin_unlock_irqrestore(&bfs->lock, flags);
	512	+}
	513	+
	514	+/*
	515	+ * In one pass under bfs->lock, mark every block with a clear dirty bit in the
	516	+ * range both dirty and needing fixup.
	517	+ *
	518	+ * Only called from the dirty_folio callback, which owns the folio-level
	519	+ * dirty flag; calling folio_mark_dirty() here would recurse.
	520	+ *
	521	+ * The folio fixup flag and bits are both set under bfs->lock so that a
	522	+ * writeback pass observing the new bits also observes the flag.
	523	+ */
	524	+static void btrfs_subpage_set_fixup_dirty(const struct btrfs_fs_info *fs_info,
	525	+ struct folio *folio, u64 start, u32 len)
	526	+{
	527	+ struct btrfs_folio_state *bfs = folio_get_private(folio);
	528	+ unsigned int dirty_bit = subpage_calc_start_bit(fs_info, folio,
	529	+ dirty, start, len);
	530	+ unsigned int fixup_bit = subpage_calc_start_bit(fs_info, folio,
	531	+ fixup, start, len);
	532	+ const unsigned int nbits = len >> fs_info->sectorsize_bits;
	533	+ unsigned long flags;
	534	+ bool marked = false;
	535	+
	536	+ spin_lock_irqsave(&bfs->lock, flags);
	537	+ for (unsigned int i = 0; i < nbits; i++) {
	538	+ if (test_bit(dirty_bit + i, bfs->bitmaps))
	539	+ continue;
	540	+ set_bit(dirty_bit + i, bfs->bitmaps);
	541	+ set_bit(fixup_bit + i, bfs->bitmaps);
	542	+ marked = true;
	543	+ }
	544	+ if (marked)
	545	+ folio_set_fixup_pending(folio);
	546	+ spin_unlock_irqrestore(&bfs->lock, flags);
	547	+}
	548	+
	549	+/*
	550	+ * Mark the still-clean blocks of a folio dirty and needing fixup, for
	551	+ * btrfs_data_dirty_folio().
	552	+ *
	553	+ * A subpage block size folio that is not uptodate is left alone: its clean
	554	+ * blocks may hold content that was never read in, which must not be marked
	555	+ * dirty.
	556	+ */
	557	+void btrfs_folio_set_fixup_dirty(const struct btrfs_fs_info *fs_info,
	558	+ struct folio *folio, u64 start, u32 len)
	559	+{
	560	+ if (!btrfs_is_subpage(fs_info, folio)) {
	561	+ if (!folio_test_dirty(folio))
	562	+ folio_set_fixup_pending(folio);
	563	+ return;
	564	+ }
	565	+ if (!folio_test_uptodate(folio))
	566	+ return;
	567	+ btrfs_subpage_set_fixup_dirty(fs_info, folio, start, len);
	568	+}
	569	+
	570	+/*
	571	+ * Drop the fixup blocks inside the range: clear both their fixup and dirty
	572	+ * bits.
	573	+ *
	574	+ * Fixup blocks carry no space reservation, so their fixup and dirty bits
	575	+ * must be dropped together. Clearing only the fixup bit would leave a
	576	+ * dirty block without a reservation which is not a valid state.
	577	+ *
	578	+ * Returns true if the folio has no dirty blocks left.
	579	+ */
	580	+static bool btrfs_subpage_clear_fixup_dirty(const struct btrfs_fs_info *fs_info,
	581	+ struct folio *folio, u64 start, u32 len)
	582	+{
	583	+ struct btrfs_folio_state *bfs = folio_get_private(folio);
	584	+ unsigned int dirty_bit = subpage_calc_start_bit(fs_info, folio,
	585	+ dirty, start, len);
	586	+ unsigned int fixup_bit = subpage_calc_start_bit(fs_info, folio,
	587	+ fixup, start, len);
	588	+ const unsigned int nbits = len >> fs_info->sectorsize_bits;
	589	+ unsigned long flags;
	590	+ bool last;
	591	+
	592	+ spin_lock_irqsave(&bfs->lock, flags);
	593	+ for (unsigned int i = 0; i < nbits; i++) {
	594	+ if (!test_bit(fixup_bit + i, bfs->bitmaps))
	595	+ continue;
	596	+ clear_bit(fixup_bit + i, bfs->bitmaps);
	597	+ clear_bit(dirty_bit + i, bfs->bitmaps);
	598	+ }
	599	+ if (subpage_test_bitmap_all_zero(fs_info, folio, fixup))
	600	+ folio_clear_fixup_pending(folio);
	601	+ last = subpage_test_bitmap_all_zero(fs_info, folio, dirty);
	602	+ spin_unlock_irqrestore(&bfs->lock, flags);
	603	+ return last;
	604	+}
	605	+
	606	+/*
	607	+ * Drop the fixup blocks inside the range, for callers discarding their data:
	608	+ * btrfs_invalidate_folio() and the writepage fixup worker's error path.
	609	+ *
	610	+ * Callers that have just reserved space for a block want
	611	+ * btrfs_folio_clear_fixup() instead - there the block stays dirty and gets
	612	+ * written.
	613	+ *
	614	+ * The range can be byte-granular (an unaligned truncate through
	615	+ * btrfs_invalidate_folio()); only blocks fully inside it are dropped, as a
	616	+ * partially covered block still holds live data outside the range. For
	617	+ * single-block folios the folio flag is the fixup state, so it is dropped
	618	+ * only when the range covers the whole folio.
	619	+ */
	620	+void btrfs_folio_clear_fixup_dirty(const struct btrfs_fs_info *fs_info,
	621	+ struct folio *folio, u64 start, u32 len)
	622	+{
	623	+ u64 aligned_start;
	624	+ u64 aligned_end;
	625	+
	626	+ /* The folio flag is set whenever any fixup bitmap bit is. */
	627	+ if (!folio_test_fixup_pending(folio))
	628	+ return;
	629	+ if (!btrfs_is_subpage(fs_info, folio)) {
	630	+ if (start <= folio_pos(folio) &&
	631	+ start + len >= folio_next_pos(folio)) {
	632	+ folio_clear_fixup_pending(folio);
	633	+ folio_clear_dirty_for_io(folio);
	634	+ }
	635	+ return;
	636	+ }
	637	+ btrfs_subpage_clamp_range(folio, &start, &len);
	638	+ aligned_start = round_up(start, fs_info->sectorsize);
	639	+ aligned_end = round_down(start + len, fs_info->sectorsize);
	640	+ if (aligned_end <= aligned_start)
	641	+ return;
	642	+ if (btrfs_subpage_clear_fixup_dirty(fs_info, folio, aligned_start,
	643	+ aligned_end - aligned_start))
	644	+ folio_clear_dirty_for_io(folio);
	645	+}
	646	+
	647	+bool btrfs_folio_test_fixup(const struct btrfs_fs_info *fs_info,
	648	+ struct folio *folio, u64 start, u32 len)
	649	+{
	650	+ if (!btrfs_is_subpage(fs_info, folio))
	651	+ return folio_test_fixup_pending(folio);
	652	+ return btrfs_subpage_test_fixup(fs_info, folio, start, len);
	653	+}
	654	+
	655	+void btrfs_folio_clear_fixup(const struct btrfs_fs_info *fs_info,
	656	+ struct folio *folio, u64 start, u32 len)
	657	+{
	658	+ if (!btrfs_is_subpage(fs_info, folio)) {
	659	+ folio_clear_fixup_pending(folio);
	660	+ return;
	661	+ }
	662	+ btrfs_subpage_clear_fixup(fs_info, folio, start, len);
	663	+}
	664	+
460	665	/*
461	666	* Unlike set/clear which is dependent on each page status, for test all bits
462	667	* are tested in the same way.
@@ -480,6 +685,7 @@ bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info, \ · 17 unmodified lines
480	685	IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(uptodate);
481	686	IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(dirty);
482	687	IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(writeback);
	688	+IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(fixup);
483	689
484	690	/*
485	691	* Note that, in selftests (extent-io-tests), we can have empty fs_info passed
@@ -571,8 +777,8 @@ bool btrfs_meta_folio_test_##name(struct folio *folio, const struct extent_buffe · 85 unmodified lines
571	777	}
572	778	IMPLEMENT_BTRFS_PAGE_OPS(uptodate, folio_mark_uptodate, folio_clear_uptodate,
573	779	folio_test_uptodate);
574		−IMPLEMENT_BTRFS_PAGE_OPS(dirty, folio_mark_dirty, folio_clear_dirty_for_io,
575		− folio_test_dirty);
	780	+IMPLEMENT_BTRFS_PAGE_OPS(dirty, btrfs_folio_mark_dirty_reserved,
	781	+ folio_clear_dirty_for_io, folio_test_dirty);
576	782	IMPLEMENT_BTRFS_PAGE_OPS(writeback, folio_start_writeback, folio_end_writeback,
577	783	folio_test_writeback);
578	784

fs/btrfs/subpage.h+36−5

@@ -14,15 +14,15 @@ struct folio; · 13 unmodified lines
14	14	/*
15	15	* Extra info for subpage bitmap.
16	16	*
17		− * For subpage we pack all uptodate/dirty/writeback bitmaps into
	17	+ * For subpage we pack all uptodate/dirty/writeback/fixup bitmaps into
18	18	* one larger bitmap.
19	19	*
20	20	* This structure records how they are organized in the bitmap:
21	21	*
22		− * /- uptodate /- dirty /- writeback
23		− * \| \| \|
24		− * v v v
25		− * \|u\|u\|u\|u\|........\|u\|u\|d\|d\|.......\|d\|d\|w\|w\|.......\|w\|w\|
	22	+ * /- uptodate /- dirty /- writeback /- fixup
	23	+ * \| \| \| \|
	24	+ * v v v v
	25	+ * \|u\|u\|u\|u\|........\|u\|u\|d\|d\|.......\|d\|d\|w\|w\|.....\|w\|w\|f\|f\|.....\|f\|f\|
26	26	* \|< sectors_per_page >\|
27	27	*
28	28	* Unlike regular macro-like enums, here we do not go upper-case names, as
@@ -40,6 +40,14 @@ enum { · 11 unmodified lines
40	40	*/
41	41	btrfs_bitmap_nr_writeback,
42	42
	43	+ /*
	44	+ * Blocks dirtied by the dirty_folio callback instead of a reserving
	45	+ * write path (e.g. set_page_dirty_lock() on a GUP pin). They have
	46	+ * no space reservation and need the writepage fixup before they can
	47	+ * be submitted.
	48	+ */
	49	+ btrfs_bitmap_nr_fixup,
	50	+
43	51	btrfs_bitmap_nr_max
44	52	};
45	53
@@ -165,6 +173,29 @@ DECLARE_BTRFS_SUBPAGE_OPS(uptodate); · 119 unmodified lines
165	173	DECLARE_BTRFS_SUBPAGE_OPS(dirty);
166	174	DECLARE_BTRFS_SUBPAGE_OPS(writeback);
167	175
	176	+/*
	177	+ * Fixup bit helpers.
	178	+ *
	179	+ * The fixup bit is data-only and has no plain set helper (setting happens
	180	+ * together with dirtying in btrfs_subpage_set_fixup_dirty()), so it does not
	181	+ * go through DECLARE_BTRFS_SUBPAGE_OPS(). For single-block folios the
	182	+ * folio_*_fixup_pending() flag takes the place of the bitmap.
	183	+ */
	184	+void btrfs_subpage_clear_fixup(const struct btrfs_fs_info *fs_info,
	185	+ struct folio *folio, u64 start, u32 len);
	186	+bool btrfs_subpage_test_fixup(const struct btrfs_fs_info *fs_info,
	187	+ struct folio *folio, u64 start, u32 len);
	188	+bool btrfs_folio_test_fixup(const struct btrfs_fs_info *fs_info,
	189	+ struct folio *folio, u64 start, u32 len);
	190	+void btrfs_folio_set_fixup_dirty(const struct btrfs_fs_info *fs_info,
	191	+ struct folio *folio, u64 start, u32 len);
	192	+/* For a block that just got its space reserved; it stays dirty. */
	193	+void btrfs_folio_clear_fixup(const struct btrfs_fs_info *fs_info,
	194	+ struct folio *folio, u64 start, u32 len);
	195	+/* For callers discarding the data; clears the dirty bits too. */
	196	+void btrfs_folio_clear_fixup_dirty(const struct btrfs_fs_info *fs_info,
	197	+ struct folio *folio, u64 start, u32 len);
	198	+
168	199	/*
169	200	* Helper for error cleanup, where a folio will have its dirty flag cleared,
170	201	* with writeback started and finished.

include/trace/events/btrfs.h+35−0

@@ -689,6 +689,41 @@ DEFINE_EVENT(btrfs__ordered_extent, btrfs_ordered_extent_lookup_first, · 688 unmodified lines
689	689	TP_ARGS(inode, ordered)
690	690	);
691	691
	692	+/*
	693	+ * The writepage fixup worker deferred a block because this still-running
	694	+ * ordered extent covers it.
	695	+ */
	696	+DEFINE_EVENT(btrfs__ordered_extent, btrfs_writepage_fixup_defer,
	697	+
	698	+ TP_PROTO(const struct btrfs_inode *inode,
	699	+ const struct btrfs_ordered_extent *ordered),
	700	+
	701	+ TP_ARGS(inode, ordered)
	702	+);
	703	+
	704	+/* The writepage fixup worker reserved space for a block and set delalloc. */
	705	+TRACE_EVENT(btrfs_writepage_fixup_reserve,
	706	+
	707	+ TP_PROTO(const struct btrfs_inode *inode, u64 start, u32 len),
	708	+
	709	+ TP_ARGS(inode, start, len),
	710	+
	711	+ TP_STRUCT__entry_btrfs(
	712	+ __field( u64, ino )
	713	+ __field( u64, start )
	714	+ __field( u32, len )
	715	+ ),
	716	+
	717	+ TP_fast_assign_btrfs(inode->root->fs_info,
	718	+ __entry->ino = btrfs_ino(inode);
	719	+ __entry->start = start;
	720	+ __entry->len = len;
	721	+ ),
	722	+
	723	+ TP_printk_btrfs("ino=%llu start=%llu len=%u",
	724	+ __entry->ino, __entry->start, __entry->len)
	725	+);
	726	+
692	727	DEFINE_EVENT(btrfs__ordered_extent, btrfs_ordered_extent_split,
693	728
694	729	TP_PROTO(const struct btrfs_inode *inode,