History

Merge tag 'for-5.8-tag' of git://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux

Pull btrfs updates from David Sterba: "Highlights: - speedup dead root detection during orphan cleanup, eg. when there are many deleted subvolumes waiting to be cleaned, the trees are now looked up in radix tree instead of a O(N^2) search - snapshot creation with inherited qgroup will mark the qgroup inconsistent, requires a rescan - send will emit file capabilities after chown, this produces a stream that does not need postprocessing to set the capabilities again - direct io ported to iomap infrastructure, cleaned up and simplified code, notably removing last use of struct buffer_head in btrfs code Core changes: - factor out backreference iteration, to be used by ordinary backreferences and relocation code - improved global block reserve utilization * better logic to serialize requests * increased maximum available for unlink * improved handling on large pages (64K) - direct io cleanups and fixes * simplify layering, where cloned bios were unnecessarily created for some cases * error handling fixes (submit, endio) * remove repair worker thread, used to avoid deadlocks during repair - refactored block group reading code, preparatory work for new type of block group storage that should improve mount time on large filesystems Cleanups: - cleaned up (and slightly sped up) set/get helpers for metadata data structure members - root bit REF_COWS got renamed to SHAREABLE to reflect the that the blocks of the tree get shared either among subvolumes or with the relocation trees Fixes: - when subvolume deletion fails due to ENOSPC, the filesystem is not turned read-only - device scan deals with devices from other filesystems that changed ownership due to overwrite (mkfs) - fix a race between scrub and block group removal/allocation - fix long standing bug of a runaway balance operation, printing the same line to the syslog, caused by a stale status bit on a reloc tree that prevented progress - fix corrupt log due to concurrent fsync of inodes with shared extents - fix space underflow for NODATACOW and buffered writes when it for some reason needs to fallback to COW mode" * tag 'for-5.8-tag' of git://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux: (133 commits) btrfs: fix space_info bytes_may_use underflow during space cache writeout btrfs: fix space_info bytes_may_use underflow after nocow buffered write btrfs: fix wrong file range cleanup after an error filling dealloc range btrfs: remove redundant local variable in read_block_for_search btrfs: open code key_search btrfs: split btrfs_direct_IO to read and write part btrfs: remove BTRFS_INODE_READDIO_NEED_LOCK fs: remove dio_end_io() btrfs: switch to iomap_dio_rw() for dio iomap: remove lockdep_assert_held() iomap: add a filesystem hook for direct I/O bio submission fs: export generic_file_buffered_read() btrfs: turn space cache writeout failure messages into debug messages btrfs: include error on messages about failure to write space/inode caches btrfs: remove useless 'fail_unlock' label from btrfs_csum_file_blocks() btrfs: do not ignore error from btrfs_next_leaf() when inserting checksums btrfs: make checksum item extension more efficient btrfs: fix corrupt log due to concurrent fsync of inodes with shared extents btrfs: unexport btrfs_compress_set_level() btrfs: simplify iget helpers ...

Linus Torvalds committed 6 years ago

Commit f3cdc8a2 parents 8eeae5b 2166e5e52 files · +3224 −3045Browse files

Unified · Split

52 files
  1. .clang-format +1 −0
  2. Documentation/block/biovecs.rst +2 −0
  3. fs/btrfs/Kconfig +1 −0
  4. fs/btrfs/backref.c +831 −6
  5. fs/btrfs/backref.h +297 −0
  6. fs/btrfs/block-group.c +154 −79
  7. fs/btrfs/block-group.h +13 −1
  8. fs/btrfs/block-rsv.c +4 −1
  9. fs/btrfs/btrfs_inode.h +8 −36
  10. fs/btrfs/compression.c +17 −19
  11. fs/btrfs/compression.h +0 −2
  12. fs/btrfs/ctree.c +76 −104
  13. fs/btrfs/ctree.h +66 −55
  14. fs/btrfs/disk-io.c +51 −42
  15. fs/btrfs/disk-io.h +1 −3
  16. fs/btrfs/export.c +3 −14
  17. fs/btrfs/extent-io-tree.h +1 −0
  18. fs/btrfs/extent-tree.c +3 −20
  19. fs/btrfs/extent_io.c +132 −156
  20. fs/btrfs/extent_io.h +35 −32
  21. fs/btrfs/file-item.c +35 −27
  22. fs/btrfs/file.c +94 −17
  23. fs/btrfs/free-space-cache.c +17 −64
  24. fs/btrfs/inode.c +452 −867
  25. fs/btrfs/ioctl.c +49 −53
  26. fs/btrfs/locking.c +1 −0
  27. fs/btrfs/misc.h +54 −0
  28. fs/btrfs/props.c +2 −7
  29. fs/btrfs/qgroup.c +14 −0
  30. fs/btrfs/relocation.c +288 −1031
  31. fs/btrfs/root-tree.c +5 −7
  32. fs/btrfs/scrub.c +45 −14
  33. fs/btrfs/send.c +72 −17
  34. fs/btrfs/space-info.c +73 −8
  35. fs/btrfs/space-info.h +1 −0
  36. fs/btrfs/struct-funcs.c +112 −111
  37. fs/btrfs/super.c +20 −18
  38. fs/btrfs/transaction.c +32 −46
  39. fs/btrfs/transaction.h +1 −2
  40. fs/btrfs/tree-checker.c +0 −4
  41. fs/btrfs/tree-defrag.c +1 −1
  42. fs/btrfs/tree-log.c +90 −102
  43. fs/btrfs/uuid-tree.c +1 −5
  44. fs/btrfs/volumes.c +44 −36
  45. fs/direct-io.c +0 −19
  46. fs/iomap/direct-io.c +10 −7
  47. include/linux/bio.h +8 −0
  48. include/linux/fs.h +2 −2
  49. include/linux/iomap.h +2 −0
  50. include/trace/events/btrfs.h +1 −0
  51. include/uapi/linux/btrfs_tree.h +0 −9
  52. mm/filemap.c +2 −1
.clang-format+1−0
@@ -80,6 +80,7 @@ ForEachMacros: · 79 unmodified lines
8080 - 'ax25_uid_for_each'
8181 - '__bio_for_each_bvec'
8282 - 'bio_for_each_bvec'
83+ - 'bio_for_each_bvec_all'
8384 - 'bio_for_each_integrity_vec'
8485 - '__bio_for_each_segment'
8586 - 'bio_for_each_segment'
Documentation/block/biovecs.rst+2−0
@@ -129,6 +129,7 @@ Usage of helpers: · 128 unmodified lines
129129 ::
130130
131131 bio_for_each_segment_all()
132+ bio_for_each_bvec_all()
132133 bio_first_bvec_all()
133134 bio_first_page_all()
134135 bio_last_bvec_all()
@@ -143,4 +144,5 @@ Usage of helpers: · 8 unmodified lines
143144 bio_vec' will contain a multi-page IO vector during the iteration::
144145
145146 bio_for_each_bvec()
147+ bio_for_each_bvec_all()
146148 rq_for_each_bvec()
fs/btrfs/Kconfig+1−0
@@ -14,6 +14,7 @@ config BTRFS_FS · 13 unmodified lines
1414 select LZO_DECOMPRESS
1515 select ZSTD_COMPRESS
1616 select ZSTD_DECOMPRESS
17+ select FS_IOMAP
1718 select RAID6_PQ
1819 select XOR_BLOCKS
1920 select SRCU
fs/btrfs/backref.c+831−6
@@ -13,6 +13,7 @@ · 12 unmodified lines
1313 #include "transaction.h"
1414 #include "delayed-ref.h"
1515 #include "locking.h"
16+#include "misc.h"
1617
1718 /* Just an arbitrary number so we can be sure this happened */
1819 #define BACKREF_FOUND_SHARED 6
@@ -537,18 +538,13 @@ static int resolve_indirect_ref(struct btrfs_fs_info *fs_info, · 518 unmodified lines
537538 const u64 *extent_item_pos, bool ignore_offset)
538539 {
539540 struct btrfs_root *root;
540− struct btrfs_key root_key;
541541 struct extent_buffer *eb;
542542 int ret = 0;
543543 int root_level;
544544 int level = ref->level;
545545 struct btrfs_key search_key = ref->key_for_search;
546546
547− root_key.objectid = ref->root_id;
548− root_key.type = BTRFS_ROOT_ITEM_KEY;
549− root_key.offset = (u64)-1;
550−
551− root = btrfs_get_fs_root(fs_info, &root_key, false);
547+ root = btrfs_get_fs_root(fs_info, ref->root_id, false);
552548 if (IS_ERR(root)) {
553549 ret = PTR_ERR(root);
554550 goto out_free;
@@ -2295,3 +2291,832 @@ void free_ipath(struct inode_fs_paths *ipath) · 1740 unmodified lines
22952291 kvfree(ipath->fspath);
22962292 kfree(ipath);
22972293 }
2294+
2295+struct btrfs_backref_iter *btrfs_backref_iter_alloc(
2296+ struct btrfs_fs_info *fs_info, gfp_t gfp_flag)
2297+{
2298+ struct btrfs_backref_iter *ret;
2299+
2300+ ret = kzalloc(sizeof(*ret), gfp_flag);
2301+ if (!ret)
2302+ return NULL;
2303+
2304+ ret->path = btrfs_alloc_path();
2305+ if (!ret) {
2306+ kfree(ret);
2307+ return NULL;
2308+ }
2309+
2310+ /* Current backref iterator only supports iteration in commit root */
2311+ ret->path->search_commit_root = 1;
2312+ ret->path->skip_locking = 1;
2313+ ret->fs_info = fs_info;
2314+
2315+ return ret;
2316+}
2317+
2318+int btrfs_backref_iter_start(struct btrfs_backref_iter *iter, u64 bytenr)
2319+{
2320+ struct btrfs_fs_info *fs_info = iter->fs_info;
2321+ struct btrfs_path *path = iter->path;
2322+ struct btrfs_extent_item *ei;
2323+ struct btrfs_key key;
2324+ int ret;
2325+
2326+ key.objectid = bytenr;
2327+ key.type = BTRFS_METADATA_ITEM_KEY;
2328+ key.offset = (u64)-1;
2329+ iter->bytenr = bytenr;
2330+
2331+ ret = btrfs_search_slot(NULL, fs_info->extent_root, &key, path, 0, 0);
2332+ if (ret < 0)
2333+ return ret;
2334+ if (ret == 0) {
2335+ ret = -EUCLEAN;
2336+ goto release;
2337+ }
2338+ if (path->slots[0] == 0) {
2339+ WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
2340+ ret = -EUCLEAN;
2341+ goto release;
2342+ }
2343+ path->slots[0]--;
2344+
2345+ btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
2346+ if ((key.type != BTRFS_EXTENT_ITEM_KEY &&
2347+ key.type != BTRFS_METADATA_ITEM_KEY) || key.objectid != bytenr) {
2348+ ret = -ENOENT;
2349+ goto release;
2350+ }
2351+ memcpy(&iter->cur_key, &key, sizeof(key));
2352+ iter->item_ptr = (u32)btrfs_item_ptr_offset(path->nodes[0],
2353+ path->slots[0]);
2354+ iter->end_ptr = (u32)(iter->item_ptr +
2355+ btrfs_item_size_nr(path->nodes[0], path->slots[0]));
2356+ ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
2357+ struct btrfs_extent_item);
2358+
2359+ /*
2360+ * Only support iteration on tree backref yet.
2361+ *
2362+ * This is an extra precaution for non skinny-metadata, where
2363+ * EXTENT_ITEM is also used for tree blocks, that we can only use
2364+ * extent flags to determine if it's a tree block.
2365+ */
2366+ if (btrfs_extent_flags(path->nodes[0], ei) & BTRFS_EXTENT_FLAG_DATA) {
2367+ ret = -ENOTSUPP;
2368+ goto release;
2369+ }
2370+ iter->cur_ptr = (u32)(iter->item_ptr + sizeof(*ei));
2371+
2372+ /* If there is no inline backref, go search for keyed backref */
2373+ if (iter->cur_ptr >= iter->end_ptr) {
2374+ ret = btrfs_next_item(fs_info->extent_root, path);
2375+
2376+ /* No inline nor keyed ref */
2377+ if (ret > 0) {
2378+ ret = -ENOENT;
2379+ goto release;
2380+ }
2381+ if (ret < 0)
2382+ goto release;
2383+
2384+ btrfs_item_key_to_cpu(path->nodes[0], &iter->cur_key,
2385+ path->slots[0]);
2386+ if (iter->cur_key.objectid != bytenr ||
2387+ (iter->cur_key.type != BTRFS_SHARED_BLOCK_REF_KEY &&
2388+ iter->cur_key.type != BTRFS_TREE_BLOCK_REF_KEY)) {
2389+ ret = -ENOENT;
2390+ goto release;
2391+ }
2392+ iter->cur_ptr = (u32)btrfs_item_ptr_offset(path->nodes[0],
2393+ path->slots[0]);
2394+ iter->item_ptr = iter->cur_ptr;
2395+ iter->end_ptr = (u32)(iter->item_ptr + btrfs_item_size_nr(
2396+ path->nodes[0], path->slots[0]));
2397+ }
2398+
2399+ return 0;
2400+release:
2401+ btrfs_backref_iter_release(iter);
2402+ return ret;
2403+}
2404+
2405+/*
2406+ * Go to the next backref item of current bytenr, can be either inlined or
2407+ * keyed.
2408+ *
2409+ * Caller needs to check whether it's inline ref or not by iter->cur_key.
2410+ *
2411+ * Return 0 if we get next backref without problem.
2412+ * Return >0 if there is no extra backref for this bytenr.
2413+ * Return <0 if there is something wrong happened.
2414+ */
2415+int btrfs_backref_iter_next(struct btrfs_backref_iter *iter)
2416+{
2417+ struct extent_buffer *eb = btrfs_backref_get_eb(iter);
2418+ struct btrfs_path *path = iter->path;
2419+ struct btrfs_extent_inline_ref *iref;
2420+ int ret;
2421+ u32 size;
2422+
2423+ if (btrfs_backref_iter_is_inline_ref(iter)) {
2424+ /* We're still inside the inline refs */
2425+ ASSERT(iter->cur_ptr < iter->end_ptr);
2426+
2427+ if (btrfs_backref_has_tree_block_info(iter)) {
2428+ /* First tree block info */
2429+ size = sizeof(struct btrfs_tree_block_info);
2430+ } else {
2431+ /* Use inline ref type to determine the size */
2432+ int type;
2433+
2434+ iref = (struct btrfs_extent_inline_ref *)
2435+ ((unsigned long)iter->cur_ptr);
2436+ type = btrfs_extent_inline_ref_type(eb, iref);
2437+
2438+ size = btrfs_extent_inline_ref_size(type);
2439+ }
2440+ iter->cur_ptr += size;
2441+ if (iter->cur_ptr < iter->end_ptr)
2442+ return 0;
2443+
2444+ /* All inline items iterated, fall through */
2445+ }
2446+
2447+ /* We're at keyed items, there is no inline item, go to the next one */
2448+ ret = btrfs_next_item(iter->fs_info->extent_root, iter->path);
2449+ if (ret)
2450+ return ret;
2451+
2452+ btrfs_item_key_to_cpu(path->nodes[0], &iter->cur_key, path->slots[0]);
2453+ if (iter->cur_key.objectid != iter->bytenr ||
2454+ (iter->cur_key.type != BTRFS_TREE_BLOCK_REF_KEY &&
2455+ iter->cur_key.type != BTRFS_SHARED_BLOCK_REF_KEY))
2456+ return 1;
2457+ iter->item_ptr = (u32)btrfs_item_ptr_offset(path->nodes[0],
2458+ path->slots[0]);
2459+ iter->cur_ptr = iter->item_ptr;
2460+ iter->end_ptr = iter->item_ptr + (u32)btrfs_item_size_nr(path->nodes[0],
2461+ path->slots[0]);
2462+ return 0;
2463+}
2464+
2465+void btrfs_backref_init_cache(struct btrfs_fs_info *fs_info,
2466+ struct btrfs_backref_cache *cache, int is_reloc)
2467+{
2468+ int i;
2469+
2470+ cache->rb_root = RB_ROOT;
2471+ for (i = 0; i < BTRFS_MAX_LEVEL; i++)
2472+ INIT_LIST_HEAD(&cache->pending[i]);
2473+ INIT_LIST_HEAD(&cache->changed);
2474+ INIT_LIST_HEAD(&cache->detached);
2475+ INIT_LIST_HEAD(&cache->leaves);
2476+ INIT_LIST_HEAD(&cache->pending_edge);
2477+ INIT_LIST_HEAD(&cache->useless_node);
2478+ cache->fs_info = fs_info;
2479+ cache->is_reloc = is_reloc;
2480+}
2481+
2482+struct btrfs_backref_node *btrfs_backref_alloc_node(
2483+ struct btrfs_backref_cache *cache, u64 bytenr, int level)
2484+{
2485+ struct btrfs_backref_node *node;
2486+
2487+ ASSERT(level >= 0 && level < BTRFS_MAX_LEVEL);
2488+ node = kzalloc(sizeof(*node), GFP_NOFS);
2489+ if (!node)
2490+ return node;
2491+
2492+ INIT_LIST_HEAD(&node->list);
2493+ INIT_LIST_HEAD(&node->upper);
2494+ INIT_LIST_HEAD(&node->lower);
2495+ RB_CLEAR_NODE(&node->rb_node);
2496+ cache->nr_nodes++;
2497+ node->level = level;
2498+ node->bytenr = bytenr;
2499+
2500+ return node;
2501+}
2502+
2503+struct btrfs_backref_edge *btrfs_backref_alloc_edge(
2504+ struct btrfs_backref_cache *cache)
2505+{
2506+ struct btrfs_backref_edge *edge;
2507+
2508+ edge = kzalloc(sizeof(*edge), GFP_NOFS);
2509+ if (edge)
2510+ cache->nr_edges++;
2511+ return edge;
2512+}
2513+
2514+/*
2515+ * Drop the backref node from cache, also cleaning up all its
2516+ * upper edges and any uncached nodes in the path.
2517+ *
2518+ * This cleanup happens bottom up, thus the node should either
2519+ * be the lowest node in the cache or a detached node.
2520+ */
2521+void btrfs_backref_cleanup_node(struct btrfs_backref_cache *cache,
2522+ struct btrfs_backref_node *node)
2523+{
2524+ struct btrfs_backref_node *upper;
2525+ struct btrfs_backref_edge *edge;
2526+
2527+ if (!node)
2528+ return;
2529+
2530+ BUG_ON(!node->lowest && !node->detached);
2531+ while (!list_empty(&node->upper)) {
2532+ edge = list_entry(node->upper.next, struct btrfs_backref_edge,
2533+ list[LOWER]);
2534+ upper = edge->node[UPPER];
2535+ list_del(&edge->list[LOWER]);
2536+ list_del(&edge->list[UPPER]);
2537+ btrfs_backref_free_edge(cache, edge);
2538+
2539+ if (RB_EMPTY_NODE(&upper->rb_node)) {
2540+ BUG_ON(!list_empty(&node->upper));
2541+ btrfs_backref_drop_node(cache, node);
2542+ node = upper;
2543+ node->lowest = 1;
2544+ continue;
2545+ }
2546+ /*
2547+ * Add the node to leaf node list if no other child block
2548+ * cached.
2549+ */
2550+ if (list_empty(&upper->lower)) {
2551+ list_add_tail(&upper->lower, &cache->leaves);
2552+ upper->lowest = 1;
2553+ }
2554+ }
2555+
2556+ btrfs_backref_drop_node(cache, node);
2557+}
2558+
2559+/*
2560+ * Release all nodes/edges from current cache
2561+ */
2562+void btrfs_backref_release_cache(struct btrfs_backref_cache *cache)
2563+{
2564+ struct btrfs_backref_node *node;
2565+ int i;
2566+
2567+ while (!list_empty(&cache->detached)) {
2568+ node = list_entry(cache->detached.next,
2569+ struct btrfs_backref_node, list);
2570+ btrfs_backref_cleanup_node(cache, node);
2571+ }
2572+
2573+ while (!list_empty(&cache->leaves)) {
2574+ node = list_entry(cache->leaves.next,
2575+ struct btrfs_backref_node, lower);
2576+ btrfs_backref_cleanup_node(cache, node);
2577+ }
2578+
2579+ cache->last_trans = 0;
2580+
2581+ for (i = 0; i < BTRFS_MAX_LEVEL; i++)
2582+ ASSERT(list_empty(&cache->pending[i]));
2583+ ASSERT(list_empty(&cache->pending_edge));
2584+ ASSERT(list_empty(&cache->useless_node));
2585+ ASSERT(list_empty(&cache->changed));
2586+ ASSERT(list_empty(&cache->detached));
2587+ ASSERT(RB_EMPTY_ROOT(&cache->rb_root));
2588+ ASSERT(!cache->nr_nodes);
2589+ ASSERT(!cache->nr_edges);
2590+}
2591+
2592+/*
2593+ * Handle direct tree backref
2594+ *
2595+ * Direct tree backref means, the backref item shows its parent bytenr
2596+ * directly. This is for SHARED_BLOCK_REF backref (keyed or inlined).
2597+ *
2598+ * @ref_key: The converted backref key.
2599+ * For keyed backref, it's the item key.
2600+ * For inlined backref, objectid is the bytenr,
2601+ * type is btrfs_inline_ref_type, offset is
2602+ * btrfs_inline_ref_offset.
2603+ */
2604+static int handle_direct_tree_backref(struct btrfs_backref_cache *cache,
2605+ struct btrfs_key *ref_key,
2606+ struct btrfs_backref_node *cur)
2607+{
2608+ struct btrfs_backref_edge *edge;
2609+ struct btrfs_backref_node *upper;
2610+ struct rb_node *rb_node;
2611+
2612+ ASSERT(ref_key->type == BTRFS_SHARED_BLOCK_REF_KEY);
2613+
2614+ /* Only reloc root uses backref pointing to itself */
2615+ if (ref_key->objectid == ref_key->offset) {
2616+ struct btrfs_root *root;
2617+
2618+ cur->is_reloc_root = 1;
2619+ /* Only reloc backref cache cares about a specific root */
2620+ if (cache->is_reloc) {
2621+ root = find_reloc_root(cache->fs_info, cur->bytenr);
2622+ if (WARN_ON(!root))
2623+ return -ENOENT;
2624+ cur->root = root;
2625+ } else {
2626+ /*
2627+ * For generic purpose backref cache, reloc root node
2628+ * is useless.
2629+ */
2630+ list_add(&cur->list, &cache->useless_node);
2631+ }
2632+ return 0;
2633+ }
2634+
2635+ edge = btrfs_backref_alloc_edge(cache);
2636+ if (!edge)
2637+ return -ENOMEM;
2638+
2639+ rb_node = rb_simple_search(&cache->rb_root, ref_key->offset);
2640+ if (!rb_node) {
2641+ /* Parent node not yet cached */
2642+ upper = btrfs_backref_alloc_node(cache, ref_key->offset,
2643+ cur->level + 1);
2644+ if (!upper) {
2645+ btrfs_backref_free_edge(cache, edge);
2646+ return -ENOMEM;
2647+ }
2648+
2649+ /*
2650+ * Backrefs for the upper level block isn't cached, add the
2651+ * block to pending list
2652+ */
2653+ list_add_tail(&edge->list[UPPER], &cache->pending_edge);
2654+ } else {
2655+ /* Parent node already cached */
2656+ upper = rb_entry(rb_node, struct btrfs_backref_node, rb_node);
2657+ ASSERT(upper->checked);
2658+ INIT_LIST_HEAD(&edge->list[UPPER]);
2659+ }
2660+ btrfs_backref_link_edge(edge, cur, upper, LINK_LOWER);
2661+ return 0;
2662+}
2663+
2664+/*
2665+ * Handle indirect tree backref
2666+ *
2667+ * Indirect tree backref means, we only know which tree the node belongs to.
2668+ * We still need to do a tree search to find out the parents. This is for
2669+ * TREE_BLOCK_REF backref (keyed or inlined).
2670+ *
2671+ * @ref_key: The same as @ref_key in handle_direct_tree_backref()
2672+ * @tree_key: The first key of this tree block.
2673+ * @path: A clean (released) path, to avoid allocating path everytime
2674+ * the function get called.
2675+ */
2676+static int handle_indirect_tree_backref(struct btrfs_backref_cache *cache,
2677+ struct btrfs_path *path,
2678+ struct btrfs_key *ref_key,
2679+ struct btrfs_key *tree_key,
2680+ struct btrfs_backref_node *cur)
2681+{
2682+ struct btrfs_fs_info *fs_info = cache->fs_info;
2683+ struct btrfs_backref_node *upper;
2684+ struct btrfs_backref_node *lower;
2685+ struct btrfs_backref_edge *edge;
2686+ struct extent_buffer *eb;
2687+ struct btrfs_root *root;
2688+ struct rb_node *rb_node;
2689+ int level;
2690+ bool need_check = true;
2691+ int ret;
2692+
2693+ root = btrfs_get_fs_root(fs_info, ref_key->offset, false);
2694+ if (IS_ERR(root))
2695+ return PTR_ERR(root);
2696+ if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state))
2697+ cur->cowonly = 1;
2698+
2699+ if (btrfs_root_level(&root->root_item) == cur->level) {
2700+ /* Tree root */
2701+ ASSERT(btrfs_root_bytenr(&root->root_item) == cur->bytenr);
2702+ /*
2703+ * For reloc backref cache, we may ignore reloc root. But for
2704+ * general purpose backref cache, we can't rely on
2705+ * btrfs_should_ignore_reloc_root() as it may conflict with
2706+ * current running relocation and lead to missing root.
2707+ *
2708+ * For general purpose backref cache, reloc root detection is
2709+ * completely relying on direct backref (key->offset is parent
2710+ * bytenr), thus only do such check for reloc cache.
2711+ */
2712+ if (btrfs_should_ignore_reloc_root(root) && cache->is_reloc) {
2713+ btrfs_put_root(root);
2714+ list_add(&cur->list, &cache->useless_node);
2715+ } else {
2716+ cur->root = root;
2717+ }
2718+ return 0;
2719+ }
2720+
2721+ level = cur->level + 1;
2722+
2723+ /* Search the tree to find parent blocks referring to the block */
2724+ path->search_commit_root = 1;
2725+ path->skip_locking = 1;
2726+ path->lowest_level = level;
2727+ ret = btrfs_search_slot(NULL, root, tree_key, path, 0, 0);
2728+ path->lowest_level = 0;
2729+ if (ret < 0) {
2730+ btrfs_put_root(root);
2731+ return ret;
2732+ }
2733+ if (ret > 0 && path->slots[level] > 0)
2734+ path->slots[level]--;
2735+
2736+ eb = path->nodes[level];
2737+ if (btrfs_node_blockptr(eb, path->slots[level]) != cur->bytenr) {
2738+ btrfs_err(fs_info,
2739+"couldn't find block (%llu) (level %d) in tree (%llu) with key (%llu %u %llu)",
2740+ cur->bytenr, level - 1, root->root_key.objectid,
2741+ tree_key->objectid, tree_key->type, tree_key->offset);
2742+ btrfs_put_root(root);
2743+ ret = -ENOENT;
2744+ goto out;
2745+ }
2746+ lower = cur;
2747+
2748+ /* Add all nodes and edges in the path */
2749+ for (; level < BTRFS_MAX_LEVEL; level++) {
2750+ if (!path->nodes[level]) {
2751+ ASSERT(btrfs_root_bytenr(&root->root_item) ==
2752+ lower->bytenr);
2753+ /* Same as previous should_ignore_reloc_root() call */
2754+ if (btrfs_should_ignore_reloc_root(root) &&
2755+ cache->is_reloc) {
2756+ btrfs_put_root(root);
2757+ list_add(&lower->list, &cache->useless_node);
2758+ } else {
2759+ lower->root = root;
2760+ }
2761+ break;
2762+ }
2763+
2764+ edge = btrfs_backref_alloc_edge(cache);
2765+ if (!edge) {
2766+ btrfs_put_root(root);
2767+ ret = -ENOMEM;
2768+ goto out;
2769+ }
2770+
2771+ eb = path->nodes[level];
2772+ rb_node = rb_simple_search(&cache->rb_root, eb->start);
2773+ if (!rb_node) {
2774+ upper = btrfs_backref_alloc_node(cache, eb->start,
2775+ lower->level + 1);
2776+ if (!upper) {
2777+ btrfs_put_root(root);
2778+ btrfs_backref_free_edge(cache, edge);
2779+ ret = -ENOMEM;
2780+ goto out;
2781+ }
2782+ upper->owner = btrfs_header_owner(eb);
2783+ if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state))
2784+ upper->cowonly = 1;
2785+
2786+ /*
2787+ * If we know the block isn't shared we can avoid
2788+ * checking its backrefs.
2789+ */
2790+ if (btrfs_block_can_be_shared(root, eb))
2791+ upper->checked = 0;
2792+ else
2793+ upper->checked = 1;
2794+
2795+ /*
2796+ * Add the block to pending list if we need to check its
2797+ * backrefs, we only do this once while walking up a
2798+ * tree as we will catch anything else later on.
2799+ */
2800+ if (!upper->checked && need_check) {
2801+ need_check = false;
2802+ list_add_tail(&edge->list[UPPER],
2803+ &cache->pending_edge);
2804+ } else {
2805+ if (upper->checked)
2806+ need_check = true;
2807+ INIT_LIST_HEAD(&edge->list[UPPER]);
2808+ }
2809+ } else {
2810+ upper = rb_entry(rb_node, struct btrfs_backref_node,
2811+ rb_node);
2812+ ASSERT(upper->checked);
2813+ INIT_LIST_HEAD(&edge->list[UPPER]);
2814+ if (!upper->owner)
2815+ upper->owner = btrfs_header_owner(eb);
2816+ }
2817+ btrfs_backref_link_edge(edge, lower, upper, LINK_LOWER);
2818+
2819+ if (rb_node) {
2820+ btrfs_put_root(root);
2821+ break;
2822+ }
2823+ lower = upper;
2824+ upper = NULL;
2825+ }
2826+out:
2827+ btrfs_release_path(path);
2828+ return ret;
2829+}
2830+
2831+/*
2832+ * Add backref node @cur into @cache.
2833+ *
2834+ * NOTE: Even if the function returned 0, @cur is not yet cached as its upper
2835+ * links aren't yet bi-directional. Needs to finish such links.
2836+ * Use btrfs_backref_finish_upper_links() to finish such linkage.
2837+ *
2838+ * @path: Released path for indirect tree backref lookup
2839+ * @iter: Released backref iter for extent tree search
2840+ * @node_key: The first key of the tree block
2841+ */
2842+int btrfs_backref_add_tree_node(struct btrfs_backref_cache *cache,
2843+ struct btrfs_path *path,
2844+ struct btrfs_backref_iter *iter,
2845+ struct btrfs_key *node_key,
2846+ struct btrfs_backref_node *cur)
2847+{
2848+ struct btrfs_fs_info *fs_info = cache->fs_info;
2849+ struct btrfs_backref_edge *edge;
2850+ struct btrfs_backref_node *exist;
2851+ int ret;
2852+
2853+ ret = btrfs_backref_iter_start(iter, cur->bytenr);
2854+ if (ret < 0)
2855+ return ret;
2856+ /*
2857+ * We skip the first btrfs_tree_block_info, as we don't use the key
2858+ * stored in it, but fetch it from the tree block
2859+ */
2860+ if (btrfs_backref_has_tree_block_info(iter)) {
2861+ ret = btrfs_backref_iter_next(iter);
2862+ if (ret < 0)
2863+ goto out;
2864+ /* No extra backref? This means the tree block is corrupted */
2865+ if (ret > 0) {
2866+ ret = -EUCLEAN;
2867+ goto out;
2868+ }
2869+ }
2870+ WARN_ON(cur->checked);
2871+ if (!list_empty(&cur->upper)) {
2872+ /*
2873+ * The backref was added previously when processing backref of
2874+ * type BTRFS_TREE_BLOCK_REF_KEY
2875+ */
2876+ ASSERT(list_is_singular(&cur->upper));
2877+ edge = list_entry(cur->upper.next, struct btrfs_backref_edge,
2878+ list[LOWER]);
2879+ ASSERT(list_empty(&edge->list[UPPER]));
2880+ exist = edge->node[UPPER];
2881+ /*
2882+ * Add the upper level block to pending list if we need check
2883+ * its backrefs
2884+ */
2885+ if (!exist->checked)
2886+ list_add_tail(&edge->list[UPPER], &cache->pending_edge);
2887+ } else {
2888+ exist = NULL;
2889+ }
2890+
2891+ for (; ret == 0; ret = btrfs_backref_iter_next(iter)) {
2892+ struct extent_buffer *eb;
2893+ struct btrfs_key key;
2894+ int type;
2895+
2896+ cond_resched();
2897+ eb = btrfs_backref_get_eb(iter);
2898+
2899+ key.objectid = iter->bytenr;
2900+ if (btrfs_backref_iter_is_inline_ref(iter)) {
2901+ struct btrfs_extent_inline_ref *iref;
2902+
2903+ /* Update key for inline backref */
2904+ iref = (struct btrfs_extent_inline_ref *)
2905+ ((unsigned long)iter->cur_ptr);
2906+ type = btrfs_get_extent_inline_ref_type(eb, iref,
2907+ BTRFS_REF_TYPE_BLOCK);
2908+ if (type == BTRFS_REF_TYPE_INVALID) {
2909+ ret = -EUCLEAN;
2910+ goto out;
2911+ }
2912+ key.type = type;
2913+ key.offset = btrfs_extent_inline_ref_offset(eb, iref);
2914+ } else {
2915+ key.type = iter->cur_key.type;
2916+ key.offset = iter->cur_key.offset;
2917+ }
2918+
2919+ /*
2920+ * Parent node found and matches current inline ref, no need to
2921+ * rebuild this node for this inline ref
2922+ */
2923+ if (exist &&
2924+ ((key.type == BTRFS_TREE_BLOCK_REF_KEY &&
2925+ exist->owner == key.offset) ||
2926+ (key.type == BTRFS_SHARED_BLOCK_REF_KEY &&
2927+ exist->bytenr == key.offset))) {
2928+ exist = NULL;
2929+ continue;
2930+ }
2931+
2932+ /* SHARED_BLOCK_REF means key.offset is the parent bytenr */
2933+ if (key.type == BTRFS_SHARED_BLOCK_REF_KEY) {
2934+ ret = handle_direct_tree_backref(cache, &key, cur);
2935+ if (ret < 0)
2936+ goto out;
2937+ continue;
2938+ } else if (unlikely(key.type == BTRFS_EXTENT_REF_V0_KEY)) {
2939+ ret = -EINVAL;
2940+ btrfs_print_v0_err(fs_info);
2941+ btrfs_handle_fs_error(fs_info, ret, NULL);
2942+ goto out;
2943+ } else if (key.type != BTRFS_TREE_BLOCK_REF_KEY) {
2944+ continue;
2945+ }
2946+
2947+ /*
2948+ * key.type == BTRFS_TREE_BLOCK_REF_KEY, inline ref offset
2949+ * means the root objectid. We need to search the tree to get
2950+ * its parent bytenr.
2951+ */
2952+ ret = handle_indirect_tree_backref(cache, path, &key, node_key,
2953+ cur);
2954+ if (ret < 0)
2955+ goto out;
2956+ }
2957+ ret = 0;
2958+ cur->checked = 1;
2959+ WARN_ON(exist);
2960+out:
2961+ btrfs_backref_iter_release(iter);
2962+ return ret;
2963+}
2964+
2965+/*
2966+ * Finish the upwards linkage created by btrfs_backref_add_tree_node()
2967+ */
2968+int btrfs_backref_finish_upper_links(struct btrfs_backref_cache *cache,
2969+ struct btrfs_backref_node *start)
2970+{
2971+ struct list_head *useless_node = &cache->useless_node;
2972+ struct btrfs_backref_edge *edge;
2973+ struct rb_node *rb_node;
2974+ LIST_HEAD(pending_edge);
2975+
2976+ ASSERT(start->checked);
2977+
2978+ /* Insert this node to cache if it's not COW-only */
2979+ if (!start->cowonly) {
2980+ rb_node = rb_simple_insert(&cache->rb_root, start->bytenr,
2981+ &start->rb_node);
2982+ if (rb_node)
2983+ btrfs_backref_panic(cache->fs_info, start->bytenr,
2984+ -EEXIST);
2985+ list_add_tail(&start->lower, &cache->leaves);
2986+ }
2987+
2988+ /*
2989+ * Use breadth first search to iterate all related edges.
2990+ *
2991+ * The starting points are all the edges of this node
2992+ */
2993+ list_for_each_entry(edge, &start->upper, list[LOWER])
2994+ list_add_tail(&edge->list[UPPER], &pending_edge);
2995+
2996+ while (!list_empty(&pending_edge)) {
2997+ struct btrfs_backref_node *upper;
2998+ struct btrfs_backref_node *lower;
2999+ struct rb_node *rb_node;
3000+
3001+ edge = list_first_entry(&pending_edge,
3002+ struct btrfs_backref_edge, list[UPPER]);
3003+ list_del_init(&edge->list[UPPER]);
3004+ upper = edge->node[UPPER];
3005+ lower = edge->node[LOWER];
3006+
3007+ /* Parent is detached, no need to keep any edges */
3008+ if (upper->detached) {
3009+ list_del(&edge->list[LOWER]);
3010+ btrfs_backref_free_edge(cache, edge);
3011+
3012+ /* Lower node is orphan, queue for cleanup */
3013+ if (list_empty(&lower->upper))
3014+ list_add(&lower->list, useless_node);
3015+ continue;
3016+ }
3017+
3018+ /*
3019+ * All new nodes added in current build_backref_tree() haven't
3020+ * been linked to the cache rb tree.
3021+ * So if we have upper->rb_node populated, this means a cache
3022+ * hit. We only need to link the edge, as @upper and all its
3023+ * parents have already been linked.
3024+ */
3025+ if (!RB_EMPTY_NODE(&upper->rb_node)) {
3026+ if (upper->lowest) {
3027+ list_del_init(&upper->lower);
3028+ upper->lowest = 0;
3029+ }
3030+
3031+ list_add_tail(&edge->list[UPPER], &upper->lower);
3032+ continue;
3033+ }
3034+
3035+ /* Sanity check, we shouldn't have any unchecked nodes */
3036+ if (!upper->checked) {
3037+ ASSERT(0);
3038+ return -EUCLEAN;
3039+ }
3040+
3041+ /* Sanity check, COW-only node has non-COW-only parent */
3042+ if (start->cowonly != upper->cowonly) {
3043+ ASSERT(0);
3044+ return -EUCLEAN;
3045+ }
3046+
3047+ /* Only cache non-COW-only (subvolume trees) tree blocks */
3048+ if (!upper->cowonly) {
3049+ rb_node = rb_simple_insert(&cache->rb_root, upper->bytenr,
3050+ &upper->rb_node);
3051+ if (rb_node) {
3052+ btrfs_backref_panic(cache->fs_info,
3053+ upper->bytenr, -EEXIST);
3054+ return -EUCLEAN;
3055+ }
3056+ }
3057+
3058+ list_add_tail(&edge->list[UPPER], &upper->lower);
3059+
3060+ /*
3061+ * Also queue all the parent edges of this uncached node
3062+ * to finish the upper linkage
3063+ */
3064+ list_for_each_entry(edge, &upper->upper, list[LOWER])
3065+ list_add_tail(&edge->list[UPPER], &pending_edge);
3066+ }
3067+ return 0;
3068+}
3069+
3070+void btrfs_backref_error_cleanup(struct btrfs_backref_cache *cache,
3071+ struct btrfs_backref_node *node)
3072+{
3073+ struct btrfs_backref_node *lower;
3074+ struct btrfs_backref_node *upper;
3075+ struct btrfs_backref_edge *edge;
3076+
3077+ while (!list_empty(&cache->useless_node)) {
3078+ lower = list_first_entry(&cache->useless_node,
3079+ struct btrfs_backref_node, list);
3080+ list_del_init(&lower->list);
3081+ }
3082+ while (!list_empty(&cache->pending_edge)) {
3083+ edge = list_first_entry(&cache->pending_edge,
3084+ struct btrfs_backref_edge, list[UPPER]);
3085+ list_del(&edge->list[UPPER]);
3086+ list_del(&edge->list[LOWER]);
3087+ lower = edge->node[LOWER];
3088+ upper = edge->node[UPPER];
3089+ btrfs_backref_free_edge(cache, edge);
3090+
3091+ /*
3092+ * Lower is no longer linked to any upper backref nodes and
3093+ * isn't in the cache, we can free it ourselves.
3094+ */
3095+ if (list_empty(&lower->upper) &&
3096+ RB_EMPTY_NODE(&lower->rb_node))
3097+ list_add(&lower->list, &cache->useless_node);
3098+
3099+ if (!RB_EMPTY_NODE(&upper->rb_node))
3100+ continue;
3101+
3102+ /* Add this guy's upper edges to the list to process */
3103+ list_for_each_entry(edge, &upper->upper, list[LOWER])
3104+ list_add_tail(&edge->list[UPPER],
3105+ &cache->pending_edge);
3106+ if (list_empty(&upper->upper))
3107+ list_add(&upper->list, &cache->useless_node);
3108+ }
3109+
3110+ while (!list_empty(&cache->useless_node)) {
3111+ lower = list_first_entry(&cache->useless_node,
3112+ struct btrfs_backref_node, list);
3113+ list_del_init(&lower->list);
3114+ if (lower == node)
3115+ node = NULL;
3116+ btrfs_backref_free_node(cache, lower);
3117+ }
3118+
3119+ btrfs_backref_cleanup_node(cache, node);
3120+ ASSERT(list_empty(&cache->useless_node) &&
3121+ list_empty(&cache->pending_edge));
3122+}
fs/btrfs/backref.h+297−0
@@ -8,6 +8,7 @@ · 7 unmodified lines
88
99 #include <linux/btrfs.h>
1010 #include "ulist.h"
11+#include "disk-io.h"
1112 #include "extent_io.h"
1213
1314 struct inode_fs_paths {
@@ -78,4 +79,300 @@ struct prelim_ref { · 64 unmodified lines
7879 u64 wanted_disk_byte;
7980 };
8081
82+/*
83+ * Iterate backrefs of one extent.
84+ *
85+ * Now it only supports iteration of tree block in commit root.
86+ */
87+struct btrfs_backref_iter {
88+ u64 bytenr;
89+ struct btrfs_path *path;
90+ struct btrfs_fs_info *fs_info;
91+ struct btrfs_key cur_key;
92+ u32 item_ptr;
93+ u32 cur_ptr;
94+ u32 end_ptr;
95+};
96+
97+struct btrfs_backref_iter *btrfs_backref_iter_alloc(
98+ struct btrfs_fs_info *fs_info, gfp_t gfp_flag);
99+
100+static inline void btrfs_backref_iter_free(struct btrfs_backref_iter *iter)
101+{
102+ if (!iter)
103+ return;
104+ btrfs_free_path(iter->path);
105+ kfree(iter);
106+}
107+
108+static inline struct extent_buffer *btrfs_backref_get_eb(
109+ struct btrfs_backref_iter *iter)
110+{
111+ if (!iter)
112+ return NULL;
113+ return iter->path->nodes[0];
114+}
115+
116+/*
117+ * For metadata with EXTENT_ITEM key (non-skinny) case, the first inline data
118+ * is btrfs_tree_block_info, without a btrfs_extent_inline_ref header.
119+ *
120+ * This helper determines if that's the case.
121+ */
122+static inline bool btrfs_backref_has_tree_block_info(
123+ struct btrfs_backref_iter *iter)
124+{
125+ if (iter->cur_key.type == BTRFS_EXTENT_ITEM_KEY &&
126+ iter->cur_ptr - iter->item_ptr == sizeof(struct btrfs_extent_item))
127+ return true;
128+ return false;
129+}
130+
131+int btrfs_backref_iter_start(struct btrfs_backref_iter *iter, u64 bytenr);
132+
133+int btrfs_backref_iter_next(struct btrfs_backref_iter *iter);
134+
135+static inline bool btrfs_backref_iter_is_inline_ref(
136+ struct btrfs_backref_iter *iter)
137+{
138+ if (iter->cur_key.type == BTRFS_EXTENT_ITEM_KEY ||
139+ iter->cur_key.type == BTRFS_METADATA_ITEM_KEY)
140+ return true;
141+ return false;
142+}
143+
144+static inline void btrfs_backref_iter_release(struct btrfs_backref_iter *iter)
145+{
146+ iter->bytenr = 0;
147+ iter->item_ptr = 0;
148+ iter->cur_ptr = 0;
149+ iter->end_ptr = 0;
150+ btrfs_release_path(iter->path);
151+ memset(&iter->cur_key, 0, sizeof(iter->cur_key));
152+}
153+
154+/*
155+ * Backref cache related structures
156+ *
157+ * The whole objective of backref_cache is to build a bi-directional map
158+ * of tree blocks (represented by backref_node) and all their parents.
159+ */
160+
161+/*
162+ * Represent a tree block in the backref cache
163+ */
164+struct btrfs_backref_node {
165+ struct {
166+ struct rb_node rb_node;
167+ u64 bytenr;
168+ }; /* Use rb_simple_node for search/insert */
169+
170+ u64 new_bytenr;
171+ /* Objectid of tree block owner, can be not uptodate */
172+ u64 owner;
173+ /* Link to pending, changed or detached list */
174+ struct list_head list;
175+
176+ /* List of upper level edges, which link this node to its parents */
177+ struct list_head upper;
178+ /* List of lower level edges, which link this node to its children */
179+ struct list_head lower;
180+
181+ /* NULL if this node is not tree root */
182+ struct btrfs_root *root;
183+ /* Extent buffer got by COWing the block */
184+ struct extent_buffer *eb;
185+ /* Level of the tree block */
186+ unsigned int level:8;
187+ /* Is the block in a non-shareable tree */
188+ unsigned int cowonly:1;
189+ /* 1 if no child node is in the cache */
190+ unsigned int lowest:1;
191+ /* Is the extent buffer locked */
192+ unsigned int locked:1;
193+ /* Has the block been processed */
194+ unsigned int processed:1;
195+ /* Have backrefs of this block been checked */
196+ unsigned int checked:1;
197+ /*
198+ * 1 if corresponding block has been COWed but some upper level block
199+ * pointers may not point to the new location
200+ */
201+ unsigned int pending:1;
202+ /* 1 if the backref node isn't connected to any other backref node */
203+ unsigned int detached:1;
204+
205+ /*
206+ * For generic purpose backref cache, where we only care if it's a reloc
207+ * root, doesn't care the source subvolid.
208+ */
209+ unsigned int is_reloc_root:1;
210+};
211+
212+#define LOWER 0
213+#define UPPER 1
214+
215+/*
216+ * Represent an edge connecting upper and lower backref nodes.
217+ */
218+struct btrfs_backref_edge {
219+ /*
220+ * list[LOWER] is linked to btrfs_backref_node::upper of lower level
221+ * node, and list[UPPER] is linked to btrfs_backref_node::lower of
222+ * upper level node.
223+ *
224+ * Also, build_backref_tree() uses list[UPPER] for pending edges, before
225+ * linking list[UPPER] to its upper level nodes.
226+ */
227+ struct list_head list[2];
228+
229+ /* Two related nodes */
230+ struct btrfs_backref_node *node[2];
231+};
232+
233+struct btrfs_backref_cache {
234+ /* Red black tree of all backref nodes in the cache */
235+ struct rb_root rb_root;
236+ /* For passing backref nodes to btrfs_reloc_cow_block */
237+ struct btrfs_backref_node *path[BTRFS_MAX_LEVEL];
238+ /*
239+ * List of blocks that have been COWed but some block pointers in upper
240+ * level blocks may not reflect the new location
241+ */
242+ struct list_head pending[BTRFS_MAX_LEVEL];
243+ /* List of backref nodes with no child node */
244+ struct list_head leaves;
245+ /* List of blocks that have been COWed in current transaction */
246+ struct list_head changed;
247+ /* List of detached backref node. */
248+ struct list_head detached;
249+
250+ u64 last_trans;
251+
252+ int nr_nodes;
253+ int nr_edges;
254+
255+ /* List of unchecked backref edges during backref cache build */
256+ struct list_head pending_edge;
257+
258+ /* List of useless backref nodes during backref cache build */
259+ struct list_head useless_node;
260+
261+ struct btrfs_fs_info *fs_info;
262+
263+ /*
264+ * Whether this cache is for relocation
265+ *
266+ * Reloction backref cache require more info for reloc root compared
267+ * to generic backref cache.
268+ */
269+ unsigned int is_reloc;
270+};
271+
272+void btrfs_backref_init_cache(struct btrfs_fs_info *fs_info,
273+ struct btrfs_backref_cache *cache, int is_reloc);
274+struct btrfs_backref_node *btrfs_backref_alloc_node(
275+ struct btrfs_backref_cache *cache, u64 bytenr, int level);
276+struct btrfs_backref_edge *btrfs_backref_alloc_edge(
277+ struct btrfs_backref_cache *cache);
278+
279+#define LINK_LOWER (1 << 0)
280+#define LINK_UPPER (1 << 1)
281+static inline void btrfs_backref_link_edge(struct btrfs_backref_edge *edge,
282+ struct btrfs_backref_node *lower,
283+ struct btrfs_backref_node *upper,
284+ int link_which)
285+{
286+ ASSERT(upper && lower && upper->level == lower->level + 1);
287+ edge->node[LOWER] = lower;
288+ edge->node[UPPER] = upper;
289+ if (link_which & LINK_LOWER)
290+ list_add_tail(&edge->list[LOWER], &lower->upper);
291+ if (link_which & LINK_UPPER)
292+ list_add_tail(&edge->list[UPPER], &upper->lower);
293+}
294+
295+static inline void btrfs_backref_free_node(struct btrfs_backref_cache *cache,
296+ struct btrfs_backref_node *node)
297+{
298+ if (node) {
299+ cache->nr_nodes--;
300+ btrfs_put_root(node->root);
301+ kfree(node);
302+ }
303+}
304+
305+static inline void btrfs_backref_free_edge(struct btrfs_backref_cache *cache,
306+ struct btrfs_backref_edge *edge)
307+{
308+ if (edge) {
309+ cache->nr_edges--;
310+ kfree(edge);
311+ }
312+}
313+
314+static inline void btrfs_backref_unlock_node_buffer(
315+ struct btrfs_backref_node *node)
316+{
317+ if (node->locked) {
318+ btrfs_tree_unlock(node->eb);
319+ node->locked = 0;
320+ }
321+}
322+
323+static inline void btrfs_backref_drop_node_buffer(
324+ struct btrfs_backref_node *node)
325+{
326+ if (node->eb) {
327+ btrfs_backref_unlock_node_buffer(node);
328+ free_extent_buffer(node->eb);
329+ node->eb = NULL;
330+ }
331+}
332+
333+/*
334+ * Drop the backref node from cache without cleaning up its children
335+ * edges.
336+ *
337+ * This can only be called on node without parent edges.
338+ * The children edges are still kept as is.
339+ */
340+static inline void btrfs_backref_drop_node(struct btrfs_backref_cache *tree,
341+ struct btrfs_backref_node *node)
342+{
343+ BUG_ON(!list_empty(&node->upper));
344+
345+ btrfs_backref_drop_node_buffer(node);
346+ list_del(&node->list);
347+ list_del(&node->lower);
348+ if (!RB_EMPTY_NODE(&node->rb_node))
349+ rb_erase(&node->rb_node, &tree->rb_root);
350+ btrfs_backref_free_node(tree, node);
351+}
352+
353+void btrfs_backref_cleanup_node(struct btrfs_backref_cache *cache,
354+ struct btrfs_backref_node *node);
355+
356+void btrfs_backref_release_cache(struct btrfs_backref_cache *cache);
357+
358+static inline void btrfs_backref_panic(struct btrfs_fs_info *fs_info,
359+ u64 bytenr, int errno)
360+{
361+ btrfs_panic(fs_info, errno,
362+ "Inconsistency in backref cache found at offset %llu",
363+ bytenr);
364+}
365+
366+int btrfs_backref_add_tree_node(struct btrfs_backref_cache *cache,
367+ struct btrfs_path *path,
368+ struct btrfs_backref_iter *iter,
369+ struct btrfs_key *node_key,
370+ struct btrfs_backref_node *cur);
371+
372+int btrfs_backref_finish_upper_links(struct btrfs_backref_cache *cache,
373+ struct btrfs_backref_node *start);
374+
375+void btrfs_backref_error_cleanup(struct btrfs_backref_cache *cache,
376+ struct btrfs_backref_node *node);
377+
81378 #endif
fs/btrfs/block-group.c+154−79
@@ -7,7 +7,6 @@ · 6 unmodified lines
77 #include "disk-io.h"
88 #include "free-space-cache.h"
99 #include "free-space-tree.h"
10−#include "disk-io.h"
1110 #include "volumes.h"
1211 #include "transaction.h"
1312 #include "ref-verify.h"
@@ -161,6 +160,8 @@ static int btrfs_add_block_group_cache(struct btrfs_fs_info *info, · 147 unmodified lines
161160 struct rb_node *parent = NULL;
162161 struct btrfs_block_group *cache;
163162
163+ ASSERT(block_group->length != 0);
164+
164165 spin_lock(&info->block_group_cache_lock);
165166 p = &info->block_group_cache_tree.rb_node;
166167
@@ -863,11 +864,34 @@ static void clear_incompat_bg_bits(struct btrfs_fs_info *fs_info, u64 flags) · 696 unmodified lines
863864 }
864865 }
865866
867+static int remove_block_group_item(struct btrfs_trans_handle *trans,
868+ struct btrfs_path *path,
869+ struct btrfs_block_group *block_group)
870+{
871+ struct btrfs_fs_info *fs_info = trans->fs_info;
872+ struct btrfs_root *root;
873+ struct btrfs_key key;
874+ int ret;
875+
876+ root = fs_info->extent_root;
877+ key.objectid = block_group->start;
878+ key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
879+ key.offset = block_group->length;
880+
881+ ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
882+ if (ret > 0)
883+ ret = -ENOENT;
884+ if (ret < 0)
885+ return ret;
886+
887+ ret = btrfs_del_item(trans, root, path);
888+ return ret;
889+}
890+
866891 int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
867892 u64 group_start, struct extent_map *em)
868893 {
869894 struct btrfs_fs_info *fs_info = trans->fs_info;
870− struct btrfs_root *root = fs_info->extent_root;
871895 struct btrfs_path *path;
872896 struct btrfs_block_group *block_group;
873897 struct btrfs_free_cluster *cluster;
@@ -1065,26 +1089,25 @@ int btrfs_remove_block_group(struct btrfs_trans_handle *trans, · 191 unmodified lines
10651089
10661090 spin_unlock(&block_group->space_info->lock);
10671091
1068− key.objectid = block_group->start;
1069− key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
1070− key.offset = block_group->length;
1071−
10721092 mutex_lock(&fs_info->chunk_mutex);
10731093 spin_lock(&block_group->lock);
10741094 block_group->removed = 1;
10751095 /*
1076− * At this point trimming can't start on this block group, because we
1077− * removed the block group from the tree fs_info->block_group_cache_tree
1078− * so no one can't find it anymore and even if someone already got this
1079− * block group before we removed it from the rbtree, they have already
1080− * incremented block_group->trimming - if they didn't, they won't find
1081− * any free space entries because we already removed them all when we
1082− * called btrfs_remove_free_space_cache().
1096+ * At this point trimming or scrub can't start on this block group,
1097+ * because we removed the block group from the rbtree
1098+ * fs_info->block_group_cache_tree so no one can't find it anymore and
1099+ * even if someone already got this block group before we removed it
1100+ * from the rbtree, they have already incremented block_group->frozen -
1101+ * if they didn't, for the trimming case they won't find any free space
1102+ * entries because we already removed them all when we called
1103+ * btrfs_remove_free_space_cache().
10831104 *
10841105 * And we must not remove the extent map from the fs_info->mapping_tree
10851106 * to prevent the same logical address range and physical device space
1086− * ranges from being reused for a new block group. This is because our
1087− * fs trim operation (btrfs_trim_fs() / btrfs_ioctl_fitrim()) is
1107+ * ranges from being reused for a new block group. This is needed to
1108+ * avoid races with trimming and scrub.
1109+ *
1110+ * An fs trim operation (btrfs_trim_fs() / btrfs_ioctl_fitrim()) is
10881111 * completely transactionless, so while it is trimming a range the
10891112 * currently running transaction might finish and a new one start,
10901113 * allowing for new block groups to be created that can reuse the same
@@ -1095,7 +1118,7 @@ int btrfs_remove_block_group(struct btrfs_trans_handle *trans, · 4 unmodified lines
10951118 * in place until the extents have been discarded completely when
10961119 * the transaction commit has completed.
10971120 */
1098− remove_em = (atomic_read(&block_group->trimming) == 0);
1121+ remove_em = (atomic_read(&block_group->frozen) == 0);
10991122 spin_unlock(&block_group->lock);
11001123
11011124 mutex_unlock(&fs_info->chunk_mutex);
@@ -1107,16 +1130,10 @@ int btrfs_remove_block_group(struct btrfs_trans_handle *trans, · 5 unmodified lines
11071130 /* Once for the block groups rbtree */
11081131 btrfs_put_block_group(block_group);
11091132
1110− ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1111− if (ret > 0)
1112− ret = -EIO;
1133+ ret = remove_block_group_item(trans, path, block_group);
11131134 if (ret < 0)
11141135 goto out;
11151136
1116− ret = btrfs_del_item(trans, root, path);
1117− if (ret)
1118− goto out;
1119−
11201137 if (remove_em) {
11211138 struct extent_map_tree *em_tree;
11221139
@@ -1175,7 +1192,7 @@ struct btrfs_trans_handle *btrfs_start_trans_remove_block_group( · 52 unmodified lines
11751192 free_extent_map(em);
11761193
11771194 return btrfs_start_transaction_fallback_global_rsv(fs_info->extent_root,
1178− num_items, 1);
1195+ num_items);
11791196 }
11801197
11811198 /*
@@ -1284,25 +1301,17 @@ static bool clean_pinned_extents(struct btrfs_trans_handle *trans, · 102 unmodified lines
12841301 ret = clear_extent_bits(&prev_trans->pinned_extents, start, end,
12851302 EXTENT_DIRTY);
12861303 if (ret)
1287− goto err;
1304+ goto out;
12881305 }
12891306
12901307 ret = clear_extent_bits(&trans->transaction->pinned_extents, start, end,
12911308 EXTENT_DIRTY);
1292− if (ret)
1293− goto err;
1309+out:
12941310 mutex_unlock(&fs_info->unused_bg_unpin_mutex);
12951311 if (prev_trans)
12961312 btrfs_put_transaction(prev_trans);
12971313
1298− return true;
1299−
1300−err:
1301− mutex_unlock(&fs_info->unused_bg_unpin_mutex);
1302− if (prev_trans)
1303− btrfs_put_transaction(prev_trans);
1304− btrfs_dec_block_group_ro(bg);
1305− return false;
1314+ return ret == 0;
13061315 }
13071316
13081317 /*
@@ -1400,8 +1409,10 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info) · 91 unmodified lines
14001409 * We could have pending pinned extents for this block group,
14011410 * just delete them, we don't care about them anymore.
14021411 */
1403− if (!clean_pinned_extents(trans, block_group))
1412+ if (!clean_pinned_extents(trans, block_group)) {
1413+ btrfs_dec_block_group_ro(block_group);
14041414 goto end_trans;
1415+ }
14051416
14061417 /*
14071418 * At this point, the block_group is read only and should fail
@@ -1450,7 +1461,7 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info) · 42 unmodified lines
14501461
14511462 /* Implicit trim during transaction commit. */
14521463 if (trimming)
1453− btrfs_get_block_group_trimming(block_group);
1464+ btrfs_freeze_block_group(block_group);
14541465
14551466 /*
14561467 * Btrfs_remove_chunk will abort the transaction if things go
@@ -1460,7 +1471,7 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info) · 3 unmodified lines
14601471
14611472 if (ret) {
14621473 if (trimming)
1463− btrfs_put_block_group_trimming(block_group);
1474+ btrfs_unfreeze_block_group(block_group);
14641475 goto end_trans;
14651476 }
14661477
@@ -1774,7 +1785,7 @@ static void link_block_group(struct btrfs_block_group *cache) · 307 unmodified lines
17741785 }
17751786
17761787 static struct btrfs_block_group *btrfs_create_block_group_cache(
1777− struct btrfs_fs_info *fs_info, u64 start, u64 size)
1788+ struct btrfs_fs_info *fs_info, u64 start)
17781789 {
17791790 struct btrfs_block_group *cache;
17801791
@@ -1790,7 +1801,6 @@ static struct btrfs_block_group *btrfs_create_block_group_cache( · 9 unmodified lines
17901801 }
17911802
17921803 cache->start = start;
1793− cache->length = size;
17941804
17951805 cache->fs_info = fs_info;
17961806 cache->full_stripe_len = btrfs_full_stripe_len(fs_info, start);
@@ -1809,7 +1819,7 @@ static struct btrfs_block_group *btrfs_create_block_group_cache( · 12 unmodified lines
18091819 INIT_LIST_HEAD(&cache->dirty_list);
18101820 INIT_LIST_HEAD(&cache->io_list);
18111821 btrfs_init_free_space_ctl(cache);
1812− atomic_set(&cache->trimming, 0);
1822+ atomic_set(&cache->frozen, 0);
18131823 mutex_init(&cache->free_space_lock);
18141824 btrfs_init_full_stripe_locks_tree(&cache->full_stripe_locks_root);
18151825
@@ -1870,25 +1880,44 @@ static int check_chunk_block_group_mappings(struct btrfs_fs_info *fs_info) · 54 unmodified lines
18701880 return ret;
18711881 }
18721882
1883+static int read_block_group_item(struct btrfs_block_group *cache,
1884+ struct btrfs_path *path,
1885+ const struct btrfs_key *key)
1886+{
1887+ struct extent_buffer *leaf = path->nodes[0];
1888+ struct btrfs_block_group_item bgi;
1889+ int slot = path->slots[0];
1890+
1891+ cache->length = key->offset;
1892+
1893+ read_extent_buffer(leaf, &bgi, btrfs_item_ptr_offset(leaf, slot),
1894+ sizeof(bgi));
1895+ cache->used = btrfs_stack_block_group_used(&bgi);
1896+ cache->flags = btrfs_stack_block_group_flags(&bgi);
1897+
1898+ return 0;
1899+}
1900+
18731901 static int read_one_block_group(struct btrfs_fs_info *info,
18741902 struct btrfs_path *path,
18751903 const struct btrfs_key *key,
18761904 int need_clear)
18771905 {
1878− struct extent_buffer *leaf = path->nodes[0];
18791906 struct btrfs_block_group *cache;
18801907 struct btrfs_space_info *space_info;
1881− struct btrfs_block_group_item bgi;
18821908 const bool mixed = btrfs_fs_incompat(info, MIXED_GROUPS);
1883− int slot = path->slots[0];
18841909 int ret;
18851910
18861911 ASSERT(key->type == BTRFS_BLOCK_GROUP_ITEM_KEY);
18871912
1888− cache = btrfs_create_block_group_cache(info, key->objectid, key->offset);
1913+ cache = btrfs_create_block_group_cache(info, key->objectid);
18891914 if (!cache)
18901915 return -ENOMEM;
18911916
1917+ ret = read_block_group_item(cache, path, key);
1918+ if (ret < 0)
1919+ goto error;
1920+
18921921 if (need_clear) {
18931922 /*
18941923 * When we mount with old space cache, we need to
@@ -1903,10 +1932,6 @@ static int read_one_block_group(struct btrfs_fs_info *info, · 8 unmodified lines
19031932 if (btrfs_test_opt(info, SPACE_CACHE))
19041933 cache->disk_cache_state = BTRFS_DC_CLEAR;
19051934 }
1906− read_extent_buffer(leaf, &bgi, btrfs_item_ptr_offset(leaf, slot),
1907− sizeof(bgi));
1908− cache->used = btrfs_stack_block_group_used(&bgi);
1909− cache->flags = btrfs_stack_block_group_flags(&bgi);
19101935 if (!mixed && ((cache->flags & BTRFS_BLOCK_GROUP_METADATA) &&
19111936 (cache->flags & BTRFS_BLOCK_GROUP_DATA))) {
19121937 btrfs_err(info,
@@ -1934,15 +1959,15 @@ static int read_one_block_group(struct btrfs_fs_info *info, · 21 unmodified lines
19341959 * are empty, and we can just add all the space in and be done with it.
19351960 * This saves us _a_lot_ of time, particularly in the full case.
19361961 */
1937− if (key->offset == cache->used) {
1962+ if (cache->length == cache->used) {
19381963 cache->last_byte_to_unpin = (u64)-1;
19391964 cache->cached = BTRFS_CACHE_FINISHED;
19401965 btrfs_free_excluded_extents(cache);
19411966 } else if (cache->used == 0) {
19421967 cache->last_byte_to_unpin = (u64)-1;
19431968 cache->cached = BTRFS_CACHE_FINISHED;
1944− add_new_free_space(cache, key->objectid,
1945− key->objectid + key->offset);
1969+ add_new_free_space(cache, cache->start,
1970+ cache->start + cache->length);
19461971 btrfs_free_excluded_extents(cache);
19471972 }
19481973
@@ -1952,7 +1977,7 @@ static int read_one_block_group(struct btrfs_fs_info *info, · 3 unmodified lines
19521977 goto error;
19531978 }
19541979 trace_btrfs_add_block_group(info, cache, 0);
1955− btrfs_update_space_info(info, cache->flags, key->offset,
1980+ btrfs_update_space_info(info, cache->flags, cache->length,
19561981 cache->used, cache->bytes_super, &space_info);
19571982
19581983 cache->space_info = space_info;
@@ -1991,7 +2016,6 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info) · 32 unmodified lines
19912016 path = btrfs_alloc_path();
19922017 if (!path)
19932018 return -ENOMEM;
1994− path->reada = READA_FORWARD;
19952019
19962020 cache_gen = btrfs_super_cache_generation(info->super_copy);
19972021 if (btrfs_test_opt(info, SPACE_CACHE) &&
@@ -2046,13 +2070,32 @@ error: · 48 unmodified lines
20462070 return ret;
20472071 }
20482072
2073+static int insert_block_group_item(struct btrfs_trans_handle *trans,
2074+ struct btrfs_block_group *block_group)
2075+{
2076+ struct btrfs_fs_info *fs_info = trans->fs_info;
2077+ struct btrfs_block_group_item bgi;
2078+ struct btrfs_root *root;
2079+ struct btrfs_key key;
2080+
2081+ spin_lock(&block_group->lock);
2082+ btrfs_set_stack_block_group_used(&bgi, block_group->used);
2083+ btrfs_set_stack_block_group_chunk_objectid(&bgi,
2084+ BTRFS_FIRST_CHUNK_TREE_OBJECTID);
2085+ btrfs_set_stack_block_group_flags(&bgi, block_group->flags);
2086+ key.objectid = block_group->start;
2087+ key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
2088+ key.offset = block_group->length;
2089+ spin_unlock(&block_group->lock);
2090+
2091+ root = fs_info->extent_root;
2092+ return btrfs_insert_item(trans, root, &key, &bgi, sizeof(bgi));
2093+}
2094+
20492095 void btrfs_create_pending_block_groups(struct btrfs_trans_handle *trans)
20502096 {
20512097 struct btrfs_fs_info *fs_info = trans->fs_info;
20522098 struct btrfs_block_group *block_group;
2053− struct btrfs_root *extent_root = fs_info->extent_root;
2054− struct btrfs_block_group_item item;
2055− struct btrfs_key key;
20562099 int ret = 0;
20572100
20582101 if (!trans->can_flush_pending_bgs)
@@ -2065,21 +2108,11 @@ void btrfs_create_pending_block_groups(struct btrfs_trans_handle *trans) · 6 unmodified lines
20652108 if (ret)
20662109 goto next;
20672110
2068− spin_lock(&block_group->lock);
2069− btrfs_set_stack_block_group_used(&item, block_group->used);
2070− btrfs_set_stack_block_group_chunk_objectid(&item,
2071− BTRFS_FIRST_CHUNK_TREE_OBJECTID);
2072− btrfs_set_stack_block_group_flags(&item, block_group->flags);
2073− key.objectid = block_group->start;
2074− key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
2075− key.offset = block_group->length;
2076− spin_unlock(&block_group->lock);
2077−
2078− ret = btrfs_insert_item(trans, extent_root, &key, &item,
2079− sizeof(item));
2111+ ret = insert_block_group_item(trans, block_group);
20802112 if (ret)
20812113 btrfs_abort_transaction(trans, ret);
2082− ret = btrfs_finish_chunk_alloc(trans, key.objectid, key.offset);
2114+ ret = btrfs_finish_chunk_alloc(trans, block_group->start,
2115+ block_group->length);
20832116 if (ret)
20842117 btrfs_abort_transaction(trans, ret);
20852118 add_block_group_free_space(trans, block_group);
@@ -2100,10 +2133,11 @@ int btrfs_make_block_group(struct btrfs_trans_handle *trans, u64 bytes_used, · 14 unmodified lines
21002133
21012134 btrfs_set_log_full_commit(trans);
21022135
2103− cache = btrfs_create_block_group_cache(fs_info, chunk_offset, size);
2136+ cache = btrfs_create_block_group_cache(fs_info, chunk_offset);
21042137 if (!cache)
21052138 return -ENOMEM;
21062139
2140+ cache->length = size;
21072141 cache->used = bytes_used;
21082142 cache->flags = type;
21092143 cache->last_byte_to_unpin = (u64)-1;
@@ -2314,13 +2348,13 @@ void btrfs_dec_block_group_ro(struct btrfs_block_group *cache) · 204 unmodified lines
23142348 spin_unlock(&sinfo->lock);
23152349 }
23162350
2317−static int write_one_cache_group(struct btrfs_trans_handle *trans,
2318− struct btrfs_path *path,
2319− struct btrfs_block_group *cache)
2351+static int update_block_group_item(struct btrfs_trans_handle *trans,
2352+ struct btrfs_path *path,
2353+ struct btrfs_block_group *cache)
23202354 {
23212355 struct btrfs_fs_info *fs_info = trans->fs_info;
23222356 int ret;
2323− struct btrfs_root *extent_root = fs_info->extent_root;
2357+ struct btrfs_root *root = fs_info->extent_root;
23242358 unsigned long bi;
23252359 struct extent_buffer *leaf;
23262360 struct btrfs_block_group_item bgi;
@@ -2330,7 +2364,7 @@ static int write_one_cache_group(struct btrfs_trans_handle *trans, · 3 unmodified lines
23302364 key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
23312365 key.offset = cache->length;
23322366
2333− ret = btrfs_search_slot(trans, extent_root, &key, path, 0, 1);
2367+ ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
23342368 if (ret) {
23352369 if (ret > 0)
23362370 ret = -ENOENT;
@@ -2642,7 +2676,7 @@ again: · 305 unmodified lines
26422676 }
26432677 }
26442678 if (!ret) {
2645− ret = write_one_cache_group(trans, path, cache);
2679+ ret = update_block_group_item(trans, path, cache);
26462680 /*
26472681 * Our block group might still be attached to the list
26482682 * of new block groups in the transaction handle of some
@@ -2791,7 +2825,7 @@ int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans) · 142 unmodified lines
27912825 }
27922826 }
27932827 if (!ret) {
2794− ret = write_one_cache_group(trans, path, cache);
2828+ ret = update_block_group_item(trans, path, cache);
27952829 /*
27962830 * One of the free space endio workers might have
27972831 * created a new block group while updating a free space
@@ -2808,7 +2842,7 @@ int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans) · 10 unmodified lines
28082842 if (ret == -ENOENT) {
28092843 wait_event(cur_trans->writer_wait,
28102844 atomic_read(&cur_trans->num_writers) == 1);
2811− ret = write_one_cache_group(trans, path, cache);
2845+ ret = update_block_group_item(trans, path, cache);
28122846 }
28132847 if (ret)
28142848 btrfs_abort_transaction(trans, ret);
@@ -3384,3 +3418,44 @@ int btrfs_free_block_groups(struct btrfs_fs_info *info) · 569 unmodified lines
33843418 }
33853419 return 0;
33863420 }
3421+
3422+void btrfs_freeze_block_group(struct btrfs_block_group *cache)
3423+{
3424+ atomic_inc(&cache->frozen);
3425+}
3426+
3427+void btrfs_unfreeze_block_group(struct btrfs_block_group *block_group)
3428+{
3429+ struct btrfs_fs_info *fs_info = block_group->fs_info;
3430+ struct extent_map_tree *em_tree;
3431+ struct extent_map *em;
3432+ bool cleanup;
3433+
3434+ spin_lock(&block_group->lock);
3435+ cleanup = (atomic_dec_and_test(&block_group->frozen) &&
3436+ block_group->removed);
3437+ spin_unlock(&block_group->lock);
3438+
3439+ if (cleanup) {
3440+ mutex_lock(&fs_info->chunk_mutex);
3441+ em_tree = &fs_info->mapping_tree;
3442+ write_lock(&em_tree->lock);
3443+ em = lookup_extent_mapping(em_tree, block_group->start,
3444+ 1);
3445+ BUG_ON(!em); /* logic error, can't happen */
3446+ remove_extent_mapping(em_tree, em);
3447+ write_unlock(&em_tree->lock);
3448+ mutex_unlock(&fs_info->chunk_mutex);
3449+
3450+ /* once for us and once for the tree */
3451+ free_extent_map(em);
3452+ free_extent_map(em);
3453+
3454+ /*
3455+ * We may have left one free space entry and other possible
3456+ * tasks trimming this block group have left 1 entry each one.
3457+ * Free them if any.
3458+ */
3459+ __btrfs_remove_free_space_cache(block_group->free_space_ctl);
3460+ }
3461+}
fs/btrfs/block-group.h+13−1
@@ -129,8 +129,17 @@ struct btrfs_block_group { · 128 unmodified lines
129129 /* For read-only block groups */
130130 struct list_head ro_list;
131131
132+ /*
133+ * When non-zero it means the block group's logical address and its
134+ * device extents can not be reused for future block group allocations
135+ * until the counter goes down to 0. This is to prevent them from being
136+ * reused while some task is still using the block group after it was
137+ * deleted - we want to make sure they can only be reused for new block
138+ * groups after that task is done with the deleted block group.
139+ */
140+ atomic_t frozen;
141+
132142 /* For discard operations */
133− atomic_t trimming;
134143 struct list_head discard_list;
135144 int discard_index;
136145 u64 discard_eligible_time;
@@ -283,6 +292,9 @@ static inline int btrfs_block_group_done(struct btrfs_block_group *cache) · 146 unmodified lines
283292 cache->cached == BTRFS_CACHE_ERROR;
284293 }
285294
295+void btrfs_freeze_block_group(struct btrfs_block_group *cache);
296+void btrfs_unfreeze_block_group(struct btrfs_block_group *cache);
297+
286298 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
287299 int btrfs_rmap_block(struct btrfs_fs_info *fs_info, u64 chunk_start,
288300 u64 physical, u64 **logical, int *naddrs, int *stripe_len);
fs/btrfs/block-rsv.c+4−1
@@ -5,6 +5,7 @@ · 4 unmodified lines
55 #include "block-rsv.h"
66 #include "space-info.h"
77 #include "transaction.h"
8+#include "block-group.h"
89
910 /*
1011 * HOW DO BLOCK RESERVES WORK
@@ -405,6 +406,8 @@ void btrfs_update_global_block_rsv(struct btrfs_fs_info *fs_info) · 394 unmodified lines
405406 else
406407 block_rsv->full = 0;
407408
409+ if (block_rsv->size >= sinfo->total_bytes)
410+ sinfo->force_alloc = CHUNK_ALLOC_FORCE;
408411 spin_unlock(&block_rsv->lock);
409412 spin_unlock(&sinfo->lock);
410413 }
@@ -455,7 +458,7 @@ static struct btrfs_block_rsv *get_block_rsv( · 44 unmodified lines
455458 struct btrfs_fs_info *fs_info = root->fs_info;
456459 struct btrfs_block_rsv *block_rsv = NULL;
457460
458− if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
461+ if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state) ||
459462 (root == fs_info->csum_root && trans->adding_csums) ||
460463 (root == fs_info->uuid_root))
461464 block_rsv = trans->block_rsv;
fs/btrfs/btrfs_inode.h+8−36
@@ -7,6 +7,7 @@ · 6 unmodified lines
77 #define BTRFS_INODE_H
88
99 #include <linux/hash.h>
10+#include <linux/refcount.h>
1011 #include "extent_map.h"
1112 #include "extent_io.h"
1213 #include "ordered-data.h"
@@ -27,7 +28,6 @@ enum { · 14 unmodified lines
2728 BTRFS_INODE_NEEDS_FULL_SYNC,
2829 BTRFS_INODE_COPY_EVERYTHING,
2930 BTRFS_INODE_IN_DELALLOC_LIST,
30− BTRFS_INODE_READDIO_NEED_LOCK,
3131 BTRFS_INODE_HAS_PROPS,
3232 BTRFS_INODE_SNAPSHOT_FLUSH,
3333 };
@@ -293,53 +293,25 @@ static inline int btrfs_inode_in_log(struct btrfs_inode *inode, u64 generation) · 259 unmodified lines
293293 return ret;
294294 }
295295
296−#define BTRFS_DIO_ORIG_BIO_SUBMITTED 0x1
297−
298296 struct btrfs_dio_private {
299297 struct inode *inode;
300− unsigned long flags;
301298 u64 logical_offset;
302299 u64 disk_bytenr;
303300 u64 bytes;
304− void *private;
305−
306− /* number of bios pending for this dio */
307− atomic_t pending_bios;
308301
309− /* IO errors */
310− int errors;
311−
312− /* orig_bio is our btrfs_io_bio */
313− struct bio *orig_bio;
302+ /*
303+ * References to this structure. There is one reference per in-flight
304+ * bio plus one while we're still setting up.
305+ */
306+ refcount_t refs;
314307
315308 /* dio_bio came from fs/direct-io.c */
316309 struct bio *dio_bio;
317310
318− /*
319− * The original bio may be split to several sub-bios, this is
320− * done during endio of sub-bios
321− */
322− blk_status_t (*subio_endio)(struct inode *, struct btrfs_io_bio *,
323− blk_status_t);
311+ /* Array of checksums */
312+ u8 csums[];
324313 };
325314
326−/*
327− * Disable DIO read nolock optimization, so new dio readers will be forced
328− * to grab i_mutex. It is used to avoid the endless truncate due to
329− * nonlocked dio read.
330− */
331−static inline void btrfs_inode_block_unlocked_dio(struct btrfs_inode *inode)
332−{
333− set_bit(BTRFS_INODE_READDIO_NEED_LOCK, &inode->runtime_flags);
334− smp_mb();
335−}
336−
337−static inline void btrfs_inode_resume_unlocked_dio(struct btrfs_inode *inode)
338−{
339− smp_mb__before_atomic();
340− clear_bit(BTRFS_INODE_READDIO_NEED_LOCK, &inode->runtime_flags);
341−}
342−
343315 /* Array of bytes with variable length, hexadecimal format 0x1234 */
344316 #define CSUM_FMT "0x%*phN"
345317 #define CSUM_FMT_VALUE(size, bytes) size, bytes
fs/btrfs/compression.c+17−19
@@ -194,11 +194,9 @@ static int check_compressed_csum(struct btrfs_inode *inode, · 193 unmodified lines
194194 for (i = 0; i < cb->nr_pages; i++) {
195195 page = cb->compressed_pages[i];
196196
197− crypto_shash_init(shash);
198197 kaddr = kmap_atomic(page);
199− crypto_shash_update(shash, kaddr, PAGE_SIZE);
198+ crypto_shash_digest(shash, kaddr, PAGE_SIZE, csum);
200199 kunmap_atomic(kaddr);
201− crypto_shash_final(shash, (u8 *)&csum);
202200
203201 if (memcmp(&csum, cb_sum, csum_size)) {
204202 btrfs_print_data_csum_error(inode, disk_start,
@@ -1141,6 +1139,22 @@ static void put_workspace(int type, struct list_head *ws) · 936 unmodified lines
11411139 }
11421140 }
11431141
1142+/*
1143+ * Adjust @level according to the limits of the compression algorithm or
1144+ * fallback to default
1145+ */
1146+static unsigned int btrfs_compress_set_level(int type, unsigned level)
1147+{
1148+ const struct btrfs_compress_op *ops = btrfs_compress_op[type];
1149+
1150+ if (level == 0)
1151+ level = ops->default_level;
1152+ else
1153+ level = min(level, ops->max_level);
1154+
1155+ return level;
1156+}
1157+
11441158 /*
11451159 * Given an address space and start and length, compress the bytes into @pages
11461160 * that are allocated on demand.
@@ -1748,19 +1762,3 @@ unsigned int btrfs_compress_str2level(unsigned int type, const char *str) · 601 unmodified lines
17481762
17491763 return level;
17501764 }
1751−
1752−/*
1753− * Adjust @level according to the limits of the compression algorithm or
1754− * fallback to default
1755− */
1756−unsigned int btrfs_compress_set_level(int type, unsigned level)
1757−{
1758− const struct btrfs_compress_op *ops = btrfs_compress_op[type];
1759−
1760− if (level == 0)
1761− level = ops->default_level;
1762− else
1763− level = min(level, ops->max_level);
1764−
1765− return level;
1766−}
fs/btrfs/compression.h+0−2
@@ -140,8 +140,6 @@ extern const struct btrfs_compress_op btrfs_zstd_compress; · 139 unmodified lines
140140 const char* btrfs_compress_type2str(enum btrfs_compression_type type);
141141 bool btrfs_compress_is_valid_type(const char *str, size_t len);
142142
143−unsigned int btrfs_compress_set_level(int type, unsigned level);
144−
145143 int btrfs_compress_heuristic(struct inode *inode, u64 start, u64 end);
146144
147145 #endif
fs/btrfs/ctree.c+76−104
@@ -144,9 +144,10 @@ struct extent_buffer *btrfs_root_node(struct btrfs_root *root) · 143 unmodified lines
144144 return eb;
145145 }
146146
147−/* cowonly root (everything not a reference counted cow subvolume), just get
148− * put onto a simple dirty list. transaction.c walks this to make sure they
149− * get properly updated on disk.
147+/*
148+ * Cowonly root (not-shareable trees, everything not subvolume or reloc roots),
149+ * just get put onto a simple dirty list. Transaction walks this list to make
150+ * sure they get properly updated on disk.
150151 */
151152 static void add_root_to_dirty_list(struct btrfs_root *root)
152153 {
@@ -185,9 +186,9 @@ int btrfs_copy_root(struct btrfs_trans_handle *trans, · 32 unmodified lines
185186 int level;
186187 struct btrfs_disk_key disk_key;
187188
188− WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
189+ WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) &&
189190 trans->transid != fs_info->running_transaction->transid);
190− WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
191+ WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) &&
191192 trans->transid != root->last_trans);
192193
193194 level = btrfs_header_level(buf);
@@ -826,12 +827,11 @@ int btrfs_block_can_be_shared(struct btrfs_root *root, · 632 unmodified lines
826827 struct extent_buffer *buf)
827828 {
828829 /*
829− * Tree blocks not in reference counted trees and tree roots
830− * are never shared. If a block was allocated after the last
831− * snapshot and the block was not allocated by tree relocation,
832− * we know the block is not shared.
830+ * Tree blocks not in shareable trees and tree roots are never shared.
831+ * If a block was allocated after the last snapshot and the block was
832+ * not allocated by tree relocation, we know the block is not shared.
833833 */
834− if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
834+ if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state) &&
835835 buf != root->node && buf != root->commit_root &&
836836 (btrfs_header_generation(buf) <=
837837 btrfs_root_last_snapshot(&root->root_item) ||
@@ -1024,9 +1024,9 @@ static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans, · 186 unmodified lines
10241024
10251025 btrfs_assert_tree_locked(buf);
10261026
1027− WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
1027+ WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) &&
10281028 trans->transid != fs_info->running_transaction->transid);
1029− WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
1029+ WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) &&
10301030 trans->transid != root->last_trans);
10311031
10321032 level = btrfs_header_level(buf);
@@ -1065,7 +1065,7 @@ static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans, · 32 unmodified lines
10651065 return ret;
10661066 }
10671067
1068− if (test_bit(BTRFS_ROOT_REF_COWS, &root->state)) {
1068+ if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) {
10691069 ret = btrfs_reloc_cow_block(trans, root, buf, cow);
10701070 if (ret) {
10711071 btrfs_abort_transaction(trans, ret);
@@ -1668,15 +1668,8 @@ static noinline int generic_bin_search(struct extent_buffer *eb, · 596 unmodified lines
16681668 {
16691669 int low = 0;
16701670 int high = max;
1671− int mid;
16721671 int ret;
1673− struct btrfs_disk_key *tmp = NULL;
1674− struct btrfs_disk_key unaligned;
1675− unsigned long offset;
1676− char *kaddr = NULL;
1677− unsigned long map_start = 0;
1678− unsigned long map_len = 0;
1679− int err;
1672+ const int key_size = sizeof(struct btrfs_disk_key);
16801673
16811674 if (low > high) {
16821675 btrfs_err(eb->fs_info,
@@ -1687,32 +1680,26 @@ static noinline int generic_bin_search(struct extent_buffer *eb, · 4 unmodified lines
16871680 }
16881681
16891682 while (low < high) {
1683+ unsigned long oip;
1684+ unsigned long offset;
1685+ struct btrfs_disk_key *tmp;
1686+ struct btrfs_disk_key unaligned;
1687+ int mid;
1688+
16901689 mid = (low + high) / 2;
16911690 offset = p + mid * item_size;
1691+ oip = offset_in_page(offset);
16921692
1693− if (!kaddr || offset < map_start ||
1694− (offset + sizeof(struct btrfs_disk_key)) >
1695− map_start + map_len) {
1696−
1697− err = map_private_extent_buffer(eb, offset,
1698− sizeof(struct btrfs_disk_key),
1699− &kaddr, &map_start, &map_len);
1700−
1701− if (!err) {
1702− tmp = (struct btrfs_disk_key *)(kaddr + offset -
1703− map_start);
1704− } else if (err == 1) {
1705− read_extent_buffer(eb, &unaligned,
1706− offset, sizeof(unaligned));
1707− tmp = &unaligned;
1708− } else {
1709− return err;
1710− }
1693+ if (oip + key_size <= PAGE_SIZE) {
1694+ const unsigned long idx = offset >> PAGE_SHIFT;
1695+ char *kaddr = page_address(eb->pages[idx]);
17111696
1697+ tmp = (struct btrfs_disk_key *)(kaddr + oip);
17121698 } else {
1713− tmp = (struct btrfs_disk_key *)(kaddr + offset -
1714− map_start);
1699+ read_extent_buffer(eb, &unaligned, offset, key_size);
1700+ tmp = &unaligned;
17151701 }
1702+
17161703 ret = comp_keys(tmp, key);
17171704
17181705 if (ret < 0)
@@ -1733,9 +1720,9 @@ static noinline int generic_bin_search(struct extent_buffer *eb, · 14 unmodified lines
17331720 * leaves vs nodes
17341721 */
17351722 int btrfs_bin_search(struct extent_buffer *eb, const struct btrfs_key *key,
1736− int level, int *slot)
1723+ int *slot)
17371724 {
1738− if (level == 0)
1725+ if (btrfs_header_level(eb) == 0)
17391726 return generic_bin_search(eb,
17401727 offsetof(struct btrfs_leaf, items),
17411728 sizeof(struct btrfs_item),
@@ -2348,16 +2335,15 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · 606 unmodified lines
23482335 struct btrfs_fs_info *fs_info = root->fs_info;
23492336 u64 blocknr;
23502337 u64 gen;
2351− struct extent_buffer *b = *eb_ret;
23522338 struct extent_buffer *tmp;
23532339 struct btrfs_key first_key;
23542340 int ret;
23552341 int parent_level;
23562342
2357− blocknr = btrfs_node_blockptr(b, slot);
2358− gen = btrfs_node_ptr_generation(b, slot);
2359− parent_level = btrfs_header_level(b);
2360− btrfs_node_key_to_cpu(b, &first_key, slot);
2343+ blocknr = btrfs_node_blockptr(*eb_ret, slot);
2344+ gen = btrfs_node_ptr_generation(*eb_ret, slot);
2345+ parent_level = btrfs_header_level(*eb_ret);
2346+ btrfs_node_key_to_cpu(*eb_ret, &first_key, slot);
23612347
23622348 tmp = find_extent_buffer(fs_info, blocknr);
23632349 if (tmp) {
@@ -2501,19 +2487,6 @@ done: · 137 unmodified lines
25012487 return ret;
25022488 }
25032489
2504−static int key_search(struct extent_buffer *b, const struct btrfs_key *key,
2505− int level, int *prev_cmp, int *slot)
2506−{
2507− if (*prev_cmp != 0) {
2508− *prev_cmp = btrfs_bin_search(b, key, level, slot);
2509− return *prev_cmp;
2510− }
2511−
2512− *slot = 0;
2513−
2514− return 0;
2515−}
2516−
25172490 int btrfs_find_item(struct btrfs_root *fs_root, struct btrfs_path *path,
25182491 u64 iobjectid, u64 ioff, u8 key_type,
25192492 struct btrfs_key *found_key)
@@ -2783,9 +2756,23 @@ cow_done: · 263 unmodified lines
27832756 }
27842757 }
27852758
2786− ret = key_search(b, key, level, &prev_cmp, &slot);
2787− if (ret < 0)
2788− goto done;
2759+ /*
2760+ * If btrfs_bin_search returns an exact match (prev_cmp == 0)
2761+ * we can safely assume the target key will always be in slot 0
2762+ * on lower levels due to the invariants BTRFS' btree provides,
2763+ * namely that a btrfs_key_ptr entry always points to the
2764+ * lowest key in the child node, thus we can skip searching
2765+ * lower levels
2766+ */
2767+ if (prev_cmp == 0) {
2768+ slot = 0;
2769+ ret = 0;
2770+ } else {
2771+ ret = btrfs_bin_search(b, key, &slot);
2772+ prev_cmp = ret;
2773+ if (ret < 0)
2774+ goto done;
2775+ }
27892776
27902777 if (level == 0) {
27912778 p->slots[level] = slot;
@@ -2909,7 +2896,6 @@ int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key, · 117 unmodified lines
29092896 int level;
29102897 int lowest_unlock = 1;
29112898 u8 lowest_level = 0;
2912− int prev_cmp = -1;
29132899
29142900 lowest_level = p->lowest_level;
29152901 WARN_ON(p->nodes[0] != NULL);
@@ -2942,12 +2928,7 @@ again: · 26 unmodified lines
29422928 */
29432929 btrfs_unlock_up_safe(p, level + 1);
29442930
2945− /*
2946− * Since we can unwind ebs we want to do a real search every
2947− * time.
2948− */
2949− prev_cmp = -1;
2950− ret = key_search(b, key, level, &prev_cmp, &slot);
2931+ ret = btrfs_bin_search(b, key, &slot);
29512932 if (ret < 0)
29522933 goto done;
29532934
@@ -3507,19 +3488,17 @@ static int leaf_space_used(struct extent_buffer *l, int start, int nr) · 553 unmodified lines
35073488 {
35083489 struct btrfs_item *start_item;
35093490 struct btrfs_item *end_item;
3510− struct btrfs_map_token token;
35113491 int data_len;
35123492 int nritems = btrfs_header_nritems(l);
35133493 int end = min(nritems, start + nr) - 1;
35143494
35153495 if (!nr)
35163496 return 0;
3517− btrfs_init_map_token(&token, l);
35183497 start_item = btrfs_item_nr(start);
35193498 end_item = btrfs_item_nr(end);
3520− data_len = btrfs_token_item_offset(l, start_item, &token) +
3521− btrfs_token_item_size(l, start_item, &token);
3522− data_len = data_len - btrfs_token_item_offset(l, end_item, &token);
3499+ data_len = btrfs_item_offset(l, start_item) +
3500+ btrfs_item_size(l, start_item);
3501+ data_len = data_len - btrfs_item_offset(l, end_item);
35233502 data_len += sizeof(struct btrfs_item) * nr;
35243503 WARN_ON(data_len < 0);
35253504 return data_len;
@@ -3650,8 +3629,8 @@ static noinline int __push_leaf_right(struct btrfs_path *path, · 124 unmodified lines
36503629 push_space = BTRFS_LEAF_DATA_SIZE(fs_info);
36513630 for (i = 0; i < right_nritems; i++) {
36523631 item = btrfs_item_nr(i);
3653− push_space -= btrfs_token_item_size(right, item, &token);
3654− btrfs_set_token_item_offset(right, item, push_space, &token);
3632+ push_space -= btrfs_token_item_size(&token, item);
3633+ btrfs_set_token_item_offset(&token, item, push_space);
36553634 }
36563635
36573636 left_nritems -= push_items;
@@ -3859,10 +3838,9 @@ static noinline int __push_leaf_left(struct btrfs_path *path, int data_size, · 201 unmodified lines
38593838
38603839 item = btrfs_item_nr(i);
38613840
3862− ioff = btrfs_token_item_offset(left, item, &token);
3863− btrfs_set_token_item_offset(left, item,
3864− ioff - (BTRFS_LEAF_DATA_SIZE(fs_info) - old_left_item_size),
3865− &token);
3841+ ioff = btrfs_token_item_offset(&token, item);
3842+ btrfs_set_token_item_offset(&token, item,
3843+ ioff - (BTRFS_LEAF_DATA_SIZE(fs_info) - old_left_item_size));
38663844 }
38673845 btrfs_set_header_nritems(left, old_left_nritems + push_items);
38683846
@@ -3892,9 +3870,8 @@ static noinline int __push_leaf_left(struct btrfs_path *path, int data_size, · 23 unmodified lines
38923870 for (i = 0; i < right_nritems; i++) {
38933871 item = btrfs_item_nr(i);
38943872
3895− push_space = push_space - btrfs_token_item_size(right,
3896− item, &token);
3897− btrfs_set_token_item_offset(right, item, push_space, &token);
3873+ push_space = push_space - btrfs_token_item_size(&token, item);
3874+ btrfs_set_token_item_offset(&token, item, push_space);
38983875 }
38993876
39003877 btrfs_mark_buffer_dirty(left);
@@ -4036,9 +4013,8 @@ static noinline void copy_for_split(struct btrfs_trans_handle *trans, · 135 unmodified lines
40364013 struct btrfs_item *item = btrfs_item_nr(i);
40374014 u32 ioff;
40384015
4039− ioff = btrfs_token_item_offset(right, item, &token);
4040− btrfs_set_token_item_offset(right, item,
4041− ioff + rt_data_off, &token);
4016+ ioff = btrfs_token_item_offset(&token, item);
4017+ btrfs_set_token_item_offset(&token, item, ioff + rt_data_off);
40424018 }
40434019
40444020 btrfs_set_header_nritems(l, mid);
@@ -4541,9 +4517,8 @@ void btrfs_truncate_item(struct btrfs_path *path, u32 new_size, int from_end) · 496 unmodified lines
45414517 u32 ioff;
45424518 item = btrfs_item_nr(i);
45434519
4544− ioff = btrfs_token_item_offset(leaf, item, &token);
4545− btrfs_set_token_item_offset(leaf, item,
4546− ioff + size_diff, &token);
4520+ ioff = btrfs_token_item_offset(&token, item);
4521+ btrfs_set_token_item_offset(&token, item, ioff + size_diff);
45474522 }
45484523
45494524 /* shift the data */
@@ -4640,9 +4615,8 @@ void btrfs_extend_item(struct btrfs_path *path, u32 data_size) · 90 unmodified lines
46404615 u32 ioff;
46414616 item = btrfs_item_nr(i);
46424617
4643− ioff = btrfs_token_item_offset(leaf, item, &token);
4644− btrfs_set_token_item_offset(leaf, item,
4645− ioff - data_size, &token);
4618+ ioff = btrfs_token_item_offset(&token, item);
4619+ btrfs_set_token_item_offset(&token, item, ioff - data_size);
46464620 }
46474621
46484622 /* shift the data */
@@ -4718,9 +4692,9 @@ void setup_items_for_insert(struct btrfs_root *root, struct btrfs_path *path, · 69 unmodified lines
47184692 u32 ioff;
47194693
47204694 item = btrfs_item_nr(i);
4721− ioff = btrfs_token_item_offset(leaf, item, &token);
4722− btrfs_set_token_item_offset(leaf, item,
4723− ioff - total_data, &token);
4695+ ioff = btrfs_token_item_offset(&token, item);
4696+ btrfs_set_token_item_offset(&token, item,
4697+ ioff - total_data);
47244698 }
47254699 /* shift the items */
47264700 memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + nr),
@@ -4739,10 +4713,9 @@ void setup_items_for_insert(struct btrfs_root *root, struct btrfs_path *path, · 12 unmodified lines
47394713 btrfs_cpu_key_to_disk(&disk_key, cpu_key + i);
47404714 btrfs_set_item_key(leaf, &disk_key, slot + i);
47414715 item = btrfs_item_nr(slot + i);
4742− btrfs_set_token_item_offset(leaf, item,
4743− data_end - data_size[i], &token);
4716+ btrfs_set_token_item_offset(&token, item, data_end - data_size[i]);
47444717 data_end -= data_size[i];
4745− btrfs_set_token_item_size(leaf, item, data_size[i], &token);
4718+ btrfs_set_token_item_size(&token, item, data_size[i]);
47464719 }
47474720
47484721 btrfs_set_header_nritems(leaf, nritems + nr);
@@ -4930,9 +4903,8 @@ int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root, · 181 unmodified lines
49304903 u32 ioff;
49314904
49324905 item = btrfs_item_nr(i);
4933− ioff = btrfs_token_item_offset(leaf, item, &token);
4934− btrfs_set_token_item_offset(leaf, item,
4935− ioff + dsize, &token);
4906+ ioff = btrfs_token_item_offset(&token, item);
4907+ btrfs_set_token_item_offset(&token, item, ioff + dsize);
49364908 }
49374909
49384910 memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot),
@@ -5103,7 +5075,7 @@ again: · 164 unmodified lines
51035075 while (1) {
51045076 nritems = btrfs_header_nritems(cur);
51055077 level = btrfs_header_level(cur);
5106− sret = btrfs_bin_search(cur, min_key, level, &slot);
5078+ sret = btrfs_bin_search(cur, min_key, &slot);
51075079 if (sret < 0) {
51085080 ret = sret;
51095081 goto out;
fs/btrfs/ctree.h+66−55
@@ -28,6 +28,7 @@ · 27 unmodified lines
2828 #include <linux/dynamic_debug.h>
2929 #include <linux/refcount.h>
3030 #include <linux/crc32c.h>
31+#include <linux/iomap.h>
3132 #include "extent-io-tree.h"
3233 #include "extent_io.h"
3334 #include "extent_map.h"
@@ -582,6 +583,7 @@ struct btrfs_fs_info { · 548 unmodified lines
582583 struct btrfs_root *quota_root;
583584 struct btrfs_root *uuid_root;
584585 struct btrfs_root *free_space_root;
586+ struct btrfs_root *data_reloc_root;
585587
586588 /* the log root tree is a directory of all the other log roots */
587589 struct btrfs_root *log_root_tree;
@@ -758,7 +760,6 @@ struct btrfs_fs_info { · 170 unmodified lines
758760 struct btrfs_workqueue *endio_workers;
759761 struct btrfs_workqueue *endio_meta_workers;
760762 struct btrfs_workqueue *endio_raid56_workers;
761− struct btrfs_workqueue *endio_repair_workers;
762763 struct btrfs_workqueue *rmw_workers;
763764 struct btrfs_workqueue *endio_meta_write_workers;
764765 struct btrfs_workqueue *endio_write_workers;
@@ -970,7 +971,28 @@ enum { · 205 unmodified lines
970971 * is used to tell us when more checks are required
971972 */
972973 BTRFS_ROOT_IN_TRANS_SETUP,
973− BTRFS_ROOT_REF_COWS,
974+
975+ /*
976+ * Set if tree blocks of this root can be shared by other roots.
977+ * Only subvolume trees and their reloc trees have this bit set.
978+ * Conflicts with TRACK_DIRTY bit.
979+ *
980+ * This affects two things:
981+ *
982+ * - How balance works
983+ * For shareable roots, we need to use reloc tree and do path
984+ * replacement for balance, and need various pre/post hooks for
985+ * snapshot creation to handle them.
986+ *
987+ * While for non-shareable trees, we just simply do a tree search
988+ * with COW.
989+ *
990+ * - How dirty roots are tracked
991+ * For shareable roots, btrfs_record_root_in_trans() is needed to
992+ * track them, while non-subvolume roots have TRACK_DIRTY bit, they
993+ * don't need to set this manually.
994+ */
995+ BTRFS_ROOT_SHAREABLE,
974996 BTRFS_ROOT_TRACK_DIRTY,
975997 BTRFS_ROOT_IN_RADIX,
976998 BTRFS_ROOT_ORPHAN_ITEM_INSERTED,
@@ -1056,7 +1078,7 @@ struct btrfs_root { · 79 unmodified lines
10561078 struct btrfs_key defrag_progress;
10571079 struct btrfs_key defrag_max;
10581080
1059− /* the dirty list is only used by non-reference counted roots */
1081+ /* The dirty list is only used by non-shareable roots */
10601082 struct list_head dirty_list;
10611083
10621084 struct list_head root_list;
@@ -1146,6 +1168,9 @@ struct btrfs_root { · 83 unmodified lines
11461168 /* Record pairs of swapped blocks for qgroup */
11471169 struct btrfs_qgroup_swapped_blocks swapped_blocks;
11481170
1171+ /* Used only by log trees, when logging csum items */
1172+ struct extent_io_tree log_csum_range;
1173+
11491174 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
11501175 u64 alloc_bytenr;
11511176 #endif
@@ -1341,7 +1366,7 @@ do { \ · 189 unmodified lines
13411366 BTRFS_INODE_ROOT_ITEM_INIT)
13421367
13431368 struct btrfs_map_token {
1344− const struct extent_buffer *eb;
1369+ struct extent_buffer *eb;
13451370 char *kaddr;
13461371 unsigned long offset;
13471372 };
@@ -1353,7 +1378,8 @@ static inline void btrfs_init_map_token(struct btrfs_map_token *token, · 5 unmodified lines
13531378 struct extent_buffer *eb)
13541379 {
13551380 token->eb = eb;
1356− token->kaddr = NULL;
1381+ token->kaddr = page_address(eb->pages[0]);
1382+ token->offset = 0;
13571383 }
13581384
13591385 /* some macros to generate set/get functions for the struct fields. This
@@ -1377,15 +1403,14 @@ static inline void btrfs_init_map_token(struct btrfs_map_token *token, · 17 unmodified lines
13771403 sizeof(((type *)0)->member)))
13781404
13791405 #define DECLARE_BTRFS_SETGET_BITS(bits) \
1380−u##bits btrfs_get_token_##bits(const struct extent_buffer *eb, \
1381− const void *ptr, unsigned long off, \
1382− struct btrfs_map_token *token); \
1383−void btrfs_set_token_##bits(struct extent_buffer *eb, const void *ptr, \
1384− unsigned long off, u##bits val, \
1385− struct btrfs_map_token *token); \
1406+u##bits btrfs_get_token_##bits(struct btrfs_map_token *token, \
1407+ const void *ptr, unsigned long off); \
1408+void btrfs_set_token_##bits(struct btrfs_map_token *token, \
1409+ const void *ptr, unsigned long off, \
1410+ u##bits val); \
13861411 u##bits btrfs_get_##bits(const struct extent_buffer *eb, \
13871412 const void *ptr, unsigned long off); \
1388−void btrfs_set_##bits(struct extent_buffer *eb, void *ptr, \
1413+void btrfs_set_##bits(const struct extent_buffer *eb, void *ptr, \
13891414 unsigned long off, u##bits val);
13901415
13911416 DECLARE_BTRFS_SETGET_BITS(8)
@@ -1400,25 +1425,23 @@ static inline u##bits btrfs_##name(const struct extent_buffer *eb, \ · 8 unmodified lines
14001425 BUILD_BUG_ON(sizeof(u##bits) != sizeof(((type *)0))->member); \
14011426 return btrfs_get_##bits(eb, s, offsetof(type, member)); \
14021427 } \
1403−static inline void btrfs_set_##name(struct extent_buffer *eb, type *s, \
1428+static inline void btrfs_set_##name(const struct extent_buffer *eb, type *s, \
14041429 u##bits val) \
14051430 { \
14061431 BUILD_BUG_ON(sizeof(u##bits) != sizeof(((type *)0))->member); \
14071432 btrfs_set_##bits(eb, s, offsetof(type, member), val); \
14081433 } \
1409−static inline u##bits btrfs_token_##name(const struct extent_buffer *eb,\
1410− const type *s, \
1411− struct btrfs_map_token *token) \
1434+static inline u##bits btrfs_token_##name(struct btrfs_map_token *token, \
1435+ const type *s) \
14121436 { \
14131437 BUILD_BUG_ON(sizeof(u##bits) != sizeof(((type *)0))->member); \
1414− return btrfs_get_token_##bits(eb, s, offsetof(type, member), token); \
1438+ return btrfs_get_token_##bits(token, s, offsetof(type, member));\
14151439 } \
1416−static inline void btrfs_set_token_##name(struct extent_buffer *eb, \
1417− type *s, u##bits val, \
1418− struct btrfs_map_token *token) \
1440+static inline void btrfs_set_token_##name(struct btrfs_map_token *token,\
1441+ type *s, u##bits val) \
14191442 { \
14201443 BUILD_BUG_ON(sizeof(u##bits) != sizeof(((type *)0))->member); \
1421− btrfs_set_token_##bits(eb, s, offsetof(type, member), val, token); \
1444+ btrfs_set_token_##bits(token, s, offsetof(type, member), val); \
14221445 }
14231446
14241447 #define BTRFS_SETGET_HEADER_FUNCS(name, type, member, bits) \
@@ -1428,7 +1451,7 @@ static inline u##bits btrfs_##name(const struct extent_buffer *eb) \ · 3 unmodified lines
14281451 u##bits res = le##bits##_to_cpu(p->member); \
14291452 return res; \
14301453 } \
1431−static inline void btrfs_set_##name(struct extent_buffer *eb, \
1454+static inline void btrfs_set_##name(const struct extent_buffer *eb, \
14321455 u##bits val) \
14331456 { \
14341457 type *p = page_address(eb->pages[0]); \
@@ -1446,7 +1469,7 @@ static inline void btrfs_set_##name(type *s, u##bits val) \ · 11 unmodified lines
14461469 }
14471470
14481471
1449−static inline u64 btrfs_device_total_bytes(struct extent_buffer *eb,
1472+static inline u64 btrfs_device_total_bytes(const struct extent_buffer *eb,
14501473 struct btrfs_dev_item *s)
14511474 {
14521475 BUILD_BUG_ON(sizeof(u64) !=
@@ -1454,7 +1477,7 @@ static inline u64 btrfs_device_total_bytes(struct extent_buffer *eb, · 1 unmodified line
14541477 return btrfs_get_64(eb, s, offsetof(struct btrfs_dev_item,
14551478 total_bytes));
14561479 }
1457−static inline void btrfs_set_device_total_bytes(struct extent_buffer *eb,
1480+static inline void btrfs_set_device_total_bytes(const struct extent_buffer *eb,
14581481 struct btrfs_dev_item *s,
14591482 u64 val)
14601483 {
@@ -1558,13 +1581,13 @@ static inline char *btrfs_stripe_dev_uuid_nr(struct btrfs_chunk *c, int nr) · 97 unmodified lines
15581581 return btrfs_stripe_dev_uuid(btrfs_stripe_nr(c, nr));
15591582 }
15601583
1561−static inline u64 btrfs_stripe_offset_nr(struct extent_buffer *eb,
1584+static inline u64 btrfs_stripe_offset_nr(const struct extent_buffer *eb,
15621585 struct btrfs_chunk *c, int nr)
15631586 {
15641587 return btrfs_stripe_offset(eb, btrfs_stripe_nr(c, nr));
15651588 }
15661589
1567−static inline u64 btrfs_stripe_devid_nr(struct extent_buffer *eb,
1590+static inline u64 btrfs_stripe_devid_nr(const struct extent_buffer *eb,
15681591 struct btrfs_chunk *c, int nr)
15691592 {
15701593 return btrfs_stripe_devid(eb, btrfs_stripe_nr(c, nr));
@@ -1644,31 +1667,21 @@ BTRFS_SETGET_FUNCS(dev_extent_chunk_objectid, struct btrfs_dev_extent, · 73 unmodified lines
16441667 BTRFS_SETGET_FUNCS(dev_extent_chunk_offset, struct btrfs_dev_extent,
16451668 chunk_offset, 64);
16461669 BTRFS_SETGET_FUNCS(dev_extent_length, struct btrfs_dev_extent, length, 64);
1647−
1648−static inline unsigned long btrfs_dev_extent_chunk_tree_uuid(struct btrfs_dev_extent *dev)
1649−{
1650− unsigned long ptr = offsetof(struct btrfs_dev_extent, chunk_tree_uuid);
1651− return (unsigned long)dev + ptr;
1652−}
1653−
16541670 BTRFS_SETGET_FUNCS(extent_refs, struct btrfs_extent_item, refs, 64);
16551671 BTRFS_SETGET_FUNCS(extent_generation, struct btrfs_extent_item,
16561672 generation, 64);
16571673 BTRFS_SETGET_FUNCS(extent_flags, struct btrfs_extent_item, flags, 64);
16581674
1659−BTRFS_SETGET_FUNCS(extent_refs_v0, struct btrfs_extent_item_v0, refs, 32);
1660−
1661−
16621675 BTRFS_SETGET_FUNCS(tree_block_level, struct btrfs_tree_block_info, level, 8);
16631676
1664−static inline void btrfs_tree_block_key(struct extent_buffer *eb,
1677+static inline void btrfs_tree_block_key(const struct extent_buffer *eb,
16651678 struct btrfs_tree_block_info *item,
16661679 struct btrfs_disk_key *key)
16671680 {
16681681 read_eb_member(eb, item, struct btrfs_tree_block_info, key, key);
16691682 }
16701683
1671−static inline void btrfs_set_tree_block_key(struct extent_buffer *eb,
1684+static inline void btrfs_set_tree_block_key(const struct extent_buffer *eb,
16721685 struct btrfs_tree_block_info *item,
16731686 struct btrfs_disk_key *key)
16741687 {
@@ -1706,12 +1719,6 @@ static inline u32 btrfs_extent_inline_ref_size(int type) · 31 unmodified lines
17061719 return 0;
17071720 }
17081721
1709−BTRFS_SETGET_FUNCS(ref_root_v0, struct btrfs_extent_ref_v0, root, 64);
1710−BTRFS_SETGET_FUNCS(ref_generation_v0, struct btrfs_extent_ref_v0,
1711− generation, 64);
1712−BTRFS_SETGET_FUNCS(ref_objectid_v0, struct btrfs_extent_ref_v0, objectid, 64);
1713−BTRFS_SETGET_FUNCS(ref_count_v0, struct btrfs_extent_ref_v0, count, 32);
1714−
17151722 /* struct btrfs_node */
17161723 BTRFS_SETGET_FUNCS(key_blockptr, struct btrfs_key_ptr, blockptr, 64);
17171724 BTRFS_SETGET_FUNCS(key_generation, struct btrfs_key_ptr, generation, 64);
@@ -1720,7 +1727,7 @@ BTRFS_SETGET_STACK_FUNCS(stack_key_blockptr, struct btrfs_key_ptr, · 2 unmodified lines
17201727 BTRFS_SETGET_STACK_FUNCS(stack_key_generation, struct btrfs_key_ptr,
17211728 generation, 64);
17221729
1723−static inline u64 btrfs_node_blockptr(struct extent_buffer *eb, int nr)
1730+static inline u64 btrfs_node_blockptr(const struct extent_buffer *eb, int nr)
17241731 {
17251732 unsigned long ptr;
17261733 ptr = offsetof(struct btrfs_node, ptrs) +
@@ -1728,7 +1735,7 @@ static inline u64 btrfs_node_blockptr(struct extent_buffer *eb, int nr) · 1 unmodified line
17281735 return btrfs_key_blockptr(eb, (struct btrfs_key_ptr *)ptr);
17291736 }
17301737
1731−static inline void btrfs_set_node_blockptr(struct extent_buffer *eb,
1738+static inline void btrfs_set_node_blockptr(const struct extent_buffer *eb,
17321739 int nr, u64 val)
17331740 {
17341741 unsigned long ptr;
@@ -1737,7 +1744,7 @@ static inline void btrfs_set_node_blockptr(struct extent_buffer *eb, · 2 unmodified lines
17371744 btrfs_set_key_blockptr(eb, (struct btrfs_key_ptr *)ptr, val);
17381745 }
17391746
1740−static inline u64 btrfs_node_ptr_generation(struct extent_buffer *eb, int nr)
1747+static inline u64 btrfs_node_ptr_generation(const struct extent_buffer *eb, int nr)
17411748 {
17421749 unsigned long ptr;
17431750 ptr = offsetof(struct btrfs_node, ptrs) +
@@ -1745,7 +1752,7 @@ static inline u64 btrfs_node_ptr_generation(struct extent_buffer *eb, int nr) · 1 unmodified line
17451752 return btrfs_key_generation(eb, (struct btrfs_key_ptr *)ptr);
17461753 }
17471754
1748−static inline void btrfs_set_node_ptr_generation(struct extent_buffer *eb,
1755+static inline void btrfs_set_node_ptr_generation(const struct extent_buffer *eb,
17491756 int nr, u64 val)
17501757 {
17511758 unsigned long ptr;
@@ -1763,7 +1770,7 @@ static inline unsigned long btrfs_node_key_ptr_offset(int nr) · 11 unmodified lines
17631770 void btrfs_node_key(const struct extent_buffer *eb,
17641771 struct btrfs_disk_key *disk_key, int nr);
17651772
1766−static inline void btrfs_set_node_key(struct extent_buffer *eb,
1773+static inline void btrfs_set_node_key(const struct extent_buffer *eb,
17671774 struct btrfs_disk_key *disk_key, int nr)
17681775 {
17691776 unsigned long ptr;
@@ -2498,8 +2505,6 @@ int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans, · 728 unmodified lines
24982505 struct btrfs_ref *generic_ref);
24992506
25002507 int btrfs_extent_readonly(struct btrfs_fs_info *fs_info, u64 bytenr);
2501−void btrfs_get_block_group_trimming(struct btrfs_block_group *cache);
2502−void btrfs_put_block_group_trimming(struct btrfs_block_group *cache);
25032508 void btrfs_clear_space_info_full(struct btrfs_fs_info *info);
25042509
25052510 enum btrfs_reserve_flush_enum {
@@ -2512,6 +2517,7 @@ enum btrfs_reserve_flush_enum { · 6 unmodified lines
25122517 BTRFS_RESERVE_FLUSH_LIMIT,
25132518 BTRFS_RESERVE_FLUSH_EVICT,
25142519 BTRFS_RESERVE_FLUSH_ALL,
2520+ BTRFS_RESERVE_FLUSH_ALL_STEAL,
25152521 };
25162522
25172523 enum btrfs_flush_state {
@@ -2551,7 +2557,7 @@ void btrfs_wait_for_snapshot_creation(struct btrfs_root *root); · 33 unmodified lines
25512557
25522558 /* ctree.c */
25532559 int btrfs_bin_search(struct extent_buffer *eb, const struct btrfs_key *key,
2554− int level, int *slot);
2560+ int *slot);
25552561 int __pure btrfs_comp_cpu_keys(const struct btrfs_key *k1, const struct btrfs_key *k2);
25562562 int btrfs_previous_item(struct btrfs_root *root,
25572563 struct btrfs_path *path, u64 min_objectid,
@@ -2896,10 +2902,9 @@ void btrfs_free_inode(struct inode *inode); · 338 unmodified lines
28962902 int btrfs_drop_inode(struct inode *inode);
28972903 int __init btrfs_init_cachep(void);
28982904 void __cold btrfs_destroy_cachep(void);
2899−struct inode *btrfs_iget_path(struct super_block *s, struct btrfs_key *location,
2905+struct inode *btrfs_iget_path(struct super_block *s, u64 ino,
29002906 struct btrfs_root *root, struct btrfs_path *path);
2901−struct inode *btrfs_iget(struct super_block *s, struct btrfs_key *location,
2902− struct btrfs_root *root);
2907+struct inode *btrfs_iget(struct super_block *s, u64 ino, struct btrfs_root *root);
29032908 struct extent_map *btrfs_get_extent(struct btrfs_inode *inode,
29042909 struct page *page, size_t pg_offset,
29052910 u64 start, u64 end);
@@ -2929,6 +2934,9 @@ int btrfs_writepage_cow_fixup(struct page *page, u64 start, u64 end); · 23 unmodified lines
29292934 void btrfs_writepage_endio_finish_ordered(struct page *page, u64 start,
29302935 u64 end, int uptodate);
29312936 extern const struct dentry_operations btrfs_dentry_operations;
2937+ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter);
2938+extern const struct iomap_ops btrfs_dio_iomap_ops;
2939+extern const struct iomap_dio_ops btrfs_dops;
29322940
29332941 /* ioctl.c */
29342942 long btrfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
@@ -3381,6 +3389,9 @@ void btrfs_reloc_pre_snapshot(struct btrfs_pending_snapshot *pending, · 446 unmodified lines
33813389 int btrfs_reloc_post_snapshot(struct btrfs_trans_handle *trans,
33823390 struct btrfs_pending_snapshot *pending);
33833391 int btrfs_should_cancel_balance(struct btrfs_fs_info *fs_info);
3392+struct btrfs_root *find_reloc_root(struct btrfs_fs_info *fs_info,
3393+ u64 bytenr);
3394+int btrfs_should_ignore_reloc_root(struct btrfs_root *root);
33843395
33853396 /* scrub.c */
33863397 int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start,
fs/btrfs/disk-io.c+51−42
@@ -358,16 +358,14 @@ static int btrfs_check_super_csum(struct btrfs_fs_info *fs_info, · 357 unmodified lines
358358 SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
359359
360360 shash->tfm = fs_info->csum_shash;
361− crypto_shash_init(shash);
362361
363362 /*
364363 * The super_block structure does not span the whole
365364 * BTRFS_SUPER_INFO_SIZE range, we expect that the unused space is
366365 * filled with zeros and is included in the checksum.
367366 */
368− crypto_shash_update(shash, raw_disk_sb + BTRFS_CSUM_SIZE,
369− BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE);
370− crypto_shash_final(shash, result);
367+ crypto_shash_digest(shash, raw_disk_sb + BTRFS_CSUM_SIZE,
368+ BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE, result);
371369
372370 if (memcmp(disk_sb->csum, result, btrfs_super_csum_size(disk_sb)))
373371 return 1;
@@ -709,9 +707,7 @@ static void end_workqueue_bio(struct bio *bio) · 335 unmodified lines
709707 else
710708 wq = fs_info->endio_write_workers;
711709 } else {
712− if (unlikely(end_io_wq->metadata == BTRFS_WQ_ENDIO_DIO_REPAIR))
713− wq = fs_info->endio_repair_workers;
714− else if (end_io_wq->metadata == BTRFS_WQ_ENDIO_RAID56)
710+ if (end_io_wq->metadata == BTRFS_WQ_ENDIO_RAID56)
715711 wq = fs_info->endio_raid56_workers;
716712 else if (end_io_wq->metadata)
717713 wq = fs_info->endio_meta_workers;
@@ -1135,9 +1131,12 @@ static void __setup_root(struct btrfs_root *root, struct btrfs_fs_info *fs_info, · 417 unmodified lines
11351131 root->log_transid = 0;
11361132 root->log_transid_committed = -1;
11371133 root->last_log_commit = 0;
1138− if (!dummy)
1134+ if (!dummy) {
11391135 extent_io_tree_init(fs_info, &root->dirty_log_pages,
11401136 IO_TREE_ROOT_DIRTY_LOG_PAGES, NULL);
1137+ extent_io_tree_init(fs_info, &root->log_csum_range,
1138+ IO_TREE_LOG_CSUM_RANGE, NULL);
1139+ }
11411140
11421141 memset(&root->root_key, 0, sizeof(root->root_key));
11431142 memset(&root->root_item, 0, sizeof(root->root_item));
@@ -1275,12 +1274,13 @@ static struct btrfs_root *alloc_log_tree(struct btrfs_trans_handle *trans, · 131 unmodified lines
12751274 root->root_key.offset = BTRFS_TREE_LOG_OBJECTID;
12761275
12771276 /*
1278− * DON'T set REF_COWS for log trees
1277+ * DON'T set SHAREABLE bit for log trees.
12791278 *
1280− * log trees do not get reference counted because they go away
1281− * before a real commit is actually done. They do store pointers
1282− * to file data extents, and those reference counts still get
1283− * updated (along with back refs to the log tree).
1279+ * Log trees are not exposed to user space thus can't be snapshotted,
1280+ * and they go away before a real commit is actually done.
1281+ *
1282+ * They do store pointers to file data extents, and those reference
1283+ * counts still get updated (along with back refs to the log tree).
12841284 */
12851285
12861286 leaf = btrfs_alloc_tree_block(trans, root, 0, BTRFS_TREE_LOG_OBJECTID,
@@ -1418,8 +1418,9 @@ static int btrfs_init_fs_root(struct btrfs_root *root) · 131 unmodified lines
14181418 if (ret)
14191419 goto fail;
14201420
1421− if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
1422− set_bit(BTRFS_ROOT_REF_COWS, &root->state);
1421+ if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID &&
1422+ root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID) {
1423+ set_bit(BTRFS_ROOT_SHAREABLE, &root->state);
14231424 btrfs_check_and_init_root_item(&root->root_item);
14241425 }
14251426
@@ -1524,6 +1525,7 @@ void btrfs_free_fs_info(struct btrfs_fs_info *fs_info) · 98 unmodified lines
15241525 btrfs_put_root(fs_info->uuid_root);
15251526 btrfs_put_root(fs_info->free_space_root);
15261527 btrfs_put_root(fs_info->fs_root);
1528+ btrfs_put_root(fs_info->data_reloc_root);
15271529 btrfs_check_leaked_roots(fs_info);
15281530 btrfs_extent_buffer_leak_debug_check(fs_info);
15291531 kfree(fs_info->super_copy);
@@ -1533,35 +1535,34 @@ void btrfs_free_fs_info(struct btrfs_fs_info *fs_info) · 3 unmodified lines
15331535
15341536
15351537 struct btrfs_root *btrfs_get_fs_root(struct btrfs_fs_info *fs_info,
1536− struct btrfs_key *location,
1537− bool check_ref)
1538+ u64 objectid, bool check_ref)
15381539 {
15391540 struct btrfs_root *root;
15401541 struct btrfs_path *path;
15411542 struct btrfs_key key;
15421543 int ret;
15431544
1544− if (location->objectid == BTRFS_ROOT_TREE_OBJECTID)
1545+ if (objectid == BTRFS_ROOT_TREE_OBJECTID)
15451546 return btrfs_grab_root(fs_info->tree_root);
1546− if (location->objectid == BTRFS_EXTENT_TREE_OBJECTID)
1547+ if (objectid == BTRFS_EXTENT_TREE_OBJECTID)
15471548 return btrfs_grab_root(fs_info->extent_root);
1548− if (location->objectid == BTRFS_CHUNK_TREE_OBJECTID)
1549+ if (objectid == BTRFS_CHUNK_TREE_OBJECTID)
15491550 return btrfs_grab_root(fs_info->chunk_root);
1550− if (location->objectid == BTRFS_DEV_TREE_OBJECTID)
1551+ if (objectid == BTRFS_DEV_TREE_OBJECTID)
15511552 return btrfs_grab_root(fs_info->dev_root);
1552− if (location->objectid == BTRFS_CSUM_TREE_OBJECTID)
1553+ if (objectid == BTRFS_CSUM_TREE_OBJECTID)
15531554 return btrfs_grab_root(fs_info->csum_root);
1554− if (location->objectid == BTRFS_QUOTA_TREE_OBJECTID)
1555+ if (objectid == BTRFS_QUOTA_TREE_OBJECTID)
15551556 return btrfs_grab_root(fs_info->quota_root) ?
15561557 fs_info->quota_root : ERR_PTR(-ENOENT);
1557− if (location->objectid == BTRFS_UUID_TREE_OBJECTID)
1558+ if (objectid == BTRFS_UUID_TREE_OBJECTID)
15581559 return btrfs_grab_root(fs_info->uuid_root) ?
15591560 fs_info->uuid_root : ERR_PTR(-ENOENT);
1560− if (location->objectid == BTRFS_FREE_SPACE_TREE_OBJECTID)
1561+ if (objectid == BTRFS_FREE_SPACE_TREE_OBJECTID)
15611562 return btrfs_grab_root(fs_info->free_space_root) ?
15621563 fs_info->free_space_root : ERR_PTR(-ENOENT);
15631564 again:
1564− root = btrfs_lookup_fs_root(fs_info, location->objectid);
1565+ root = btrfs_lookup_fs_root(fs_info, objectid);
15651566 if (root) {
15661567 if (check_ref && btrfs_root_refs(&root->root_item) == 0) {
15671568 btrfs_put_root(root);
@@ -1570,7 +1571,10 @@ again: · 2 unmodified lines
15701571 return root;
15711572 }
15721573
1573− root = btrfs_read_tree_root(fs_info->tree_root, location);
1574+ key.objectid = objectid;
1575+ key.type = BTRFS_ROOT_ITEM_KEY;
1576+ key.offset = (u64)-1;
1577+ root = btrfs_read_tree_root(fs_info->tree_root, &key);
15741578 if (IS_ERR(root))
15751579 return root;
15761580
@@ -1590,7 +1594,7 @@ again: · 13 unmodified lines
15901594 }
15911595 key.objectid = BTRFS_ORPHAN_OBJECTID;
15921596 key.type = BTRFS_ORPHAN_ITEM_KEY;
1593− key.offset = location->objectid;
1597+ key.offset = objectid;
15941598
15951599 ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
15961600 btrfs_free_path(path);
@@ -1940,7 +1944,6 @@ static void btrfs_stop_all_workers(struct btrfs_fs_info *fs_info) · 343 unmodified lines
19401944 btrfs_destroy_workqueue(fs_info->workers);
19411945 btrfs_destroy_workqueue(fs_info->endio_workers);
19421946 btrfs_destroy_workqueue(fs_info->endio_raid56_workers);
1943− btrfs_destroy_workqueue(fs_info->endio_repair_workers);
19441947 btrfs_destroy_workqueue(fs_info->rmw_workers);
19451948 btrfs_destroy_workqueue(fs_info->endio_write_workers);
19461949 btrfs_destroy_workqueue(fs_info->endio_freespace_worker);
@@ -1981,6 +1984,7 @@ static void free_root_pointers(struct btrfs_fs_info *info, bool free_chunk_root) · 34 unmodified lines
19811984 free_root_extent_buffers(info->quota_root);
19821985 free_root_extent_buffers(info->uuid_root);
19831986 free_root_extent_buffers(info->fs_root);
1987+ free_root_extent_buffers(info->data_reloc_root);
19841988 if (free_chunk_root)
19851989 free_root_extent_buffers(info->chunk_root);
19861990 free_root_extent_buffers(info->free_space_root);
@@ -1993,6 +1997,7 @@ void btrfs_put_root(struct btrfs_root *root) · 6 unmodified lines
19931997
19941998 if (refcount_dec_and_test(&root->refs)) {
19951999 WARN_ON(!RB_EMPTY_ROOT(&root->inode_tree));
2000+ WARN_ON(test_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state));
19962001 if (root->anon_dev)
19972002 free_anon_bdev(root->anon_dev);
19982003 btrfs_drew_lock_destroy(&root->snapshot_lock);
@@ -2143,8 +2148,6 @@ static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info, · 144 unmodified lines
21432148 fs_info->endio_raid56_workers =
21442149 btrfs_alloc_workqueue(fs_info, "endio-raid56", flags,
21452150 max_active, 4);
2146− fs_info->endio_repair_workers =
2147− btrfs_alloc_workqueue(fs_info, "endio-repair", flags, 1, 0);
21482151 fs_info->rmw_workers =
21492152 btrfs_alloc_workqueue(fs_info, "rmw", flags, max_active, 2);
21502153 fs_info->endio_write_workers =
@@ -2168,7 +2171,6 @@ static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info, · 17 unmodified lines
21682171 fs_info->flush_workers &&
21692172 fs_info->endio_workers && fs_info->endio_meta_workers &&
21702173 fs_info->endio_meta_write_workers &&
2171− fs_info->endio_repair_workers &&
21722174 fs_info->endio_write_workers && fs_info->endio_raid56_workers &&
21732175 fs_info->endio_freespace_worker && fs_info->rmw_workers &&
21742176 fs_info->caching_workers && fs_info->readahead_workers &&
@@ -2290,6 +2292,19 @@ static int btrfs_read_roots(struct btrfs_fs_info *fs_info) · 115 unmodified lines
22902292 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
22912293 fs_info->csum_root = root;
22922294
2295+ /*
2296+ * This tree can share blocks with some other fs tree during relocation
2297+ * and we need a proper setup by btrfs_get_fs_root
2298+ */
2299+ root = btrfs_get_fs_root(tree_root->fs_info,
2300+ BTRFS_DATA_RELOC_TREE_OBJECTID, true);
2301+ if (IS_ERR(root)) {
2302+ ret = PTR_ERR(root);
2303+ goto out;
2304+ }
2305+ set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2306+ fs_info->data_reloc_root = root;
2307+
22932308 location.objectid = BTRFS_QUOTA_TREE_OBJECTID;
22942309 root = btrfs_read_tree_root(tree_root, &location);
22952310 if (!IS_ERR(root)) {
@@ -2827,7 +2842,6 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device · 531 unmodified lines
28272842 u64 generation;
28282843 u64 features;
28292844 u16 csum_type;
2830− struct btrfs_key location;
28312845 struct btrfs_super_block *disk_super;
28322846 struct btrfs_fs_info *fs_info = btrfs_sb(sb);
28332847 struct btrfs_root *tree_root;
@@ -3241,11 +3255,7 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device · 407 unmodified lines
32413255 }
32423256 }
32433257
3244− location.objectid = BTRFS_FS_TREE_OBJECTID;
3245− location.type = BTRFS_ROOT_ITEM_KEY;
3246− location.offset = 0;
3247−
3248− fs_info->fs_root = btrfs_get_fs_root(fs_info, &location, true);
3258+ fs_info->fs_root = btrfs_get_fs_root(fs_info, BTRFS_FS_TREE_OBJECTID, true);
32493259 if (IS_ERR(fs_info->fs_root)) {
32503260 err = PTR_ERR(fs_info->fs_root);
32513261 btrfs_warn(fs_info, "failed to read fs tree: %d", err);
@@ -3508,10 +3518,9 @@ static int write_dev_supers(struct btrfs_device *device, · 256 unmodified lines
35083518
35093519 btrfs_set_super_bytenr(sb, bytenr);
35103520
3511− crypto_shash_init(shash);
3512− crypto_shash_update(shash, (const char *)sb + BTRFS_CSUM_SIZE,
3513− BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE);
3514− crypto_shash_final(shash, sb->csum);
3521+ crypto_shash_digest(shash, (const char *)sb + BTRFS_CSUM_SIZE,
3522+ BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE,
3523+ sb->csum);
35153524
35163525 page = find_or_create_page(mapping, bytenr >> PAGE_SHIFT,
35173526 GFP_NOFS);
fs/btrfs/disk-io.h+1−3
@@ -25,7 +25,6 @@ enum btrfs_wq_endio_type { · 24 unmodified lines
2525 BTRFS_WQ_ENDIO_METADATA,
2626 BTRFS_WQ_ENDIO_FREE_SPACE,
2727 BTRFS_WQ_ENDIO_RAID56,
28− BTRFS_WQ_ENDIO_DIO_REPAIR,
2928 };
3029
3130 static inline u64 btrfs_sb_offset(int mirror)
@@ -67,8 +66,7 @@ int btrfs_insert_fs_root(struct btrfs_fs_info *fs_info, · 35 unmodified lines
6766 void btrfs_free_fs_roots(struct btrfs_fs_info *fs_info);
6867
6968 struct btrfs_root *btrfs_get_fs_root(struct btrfs_fs_info *fs_info,
70− struct btrfs_key *key,
71− bool check_ref);
69+ u64 objectid, bool check_ref);
7270
7371 void btrfs_free_fs_info(struct btrfs_fs_info *fs_info);
7472 int btrfs_cleanup_fs_roots(struct btrfs_fs_info *fs_info);
fs/btrfs/export.c+3−14
@@ -64,24 +64,15 @@ struct dentry *btrfs_get_dentry(struct super_block *sb, u64 objectid, · 63 unmodified lines
6464 struct btrfs_fs_info *fs_info = btrfs_sb(sb);
6565 struct btrfs_root *root;
6666 struct inode *inode;
67− struct btrfs_key key;
6867
6968 if (objectid < BTRFS_FIRST_FREE_OBJECTID)
7069 return ERR_PTR(-ESTALE);
7170
72− key.objectid = root_objectid;
73− key.type = BTRFS_ROOT_ITEM_KEY;
74− key.offset = (u64)-1;
75−
76− root = btrfs_get_fs_root(fs_info, &key, true);
71+ root = btrfs_get_fs_root(fs_info, root_objectid, true);
7772 if (IS_ERR(root))
7873 return ERR_CAST(root);
7974
80− key.objectid = objectid;
81− key.type = BTRFS_INODE_ITEM_KEY;
82− key.offset = 0;
83−
84− inode = btrfs_iget(sb, &key, root);
75+ inode = btrfs_iget(sb, objectid, root);
8576 btrfs_put_root(root);
8677 if (IS_ERR(inode))
8778 return ERR_CAST(inode);
@@ -200,9 +191,7 @@ struct dentry *btrfs_get_parent(struct dentry *child) · 112 unmodified lines
200191 found_key.offset, 0, 0);
201192 }
202193
203− key.type = BTRFS_INODE_ITEM_KEY;
204− key.offset = 0;
205− return d_obtain_alias(btrfs_iget(fs_info->sb, &key, root));
194+ return d_obtain_alias(btrfs_iget(fs_info->sb, key.objectid, root));
206195 fail:
207196 btrfs_free_path(path);
208197 return ERR_PTR(ret);
fs/btrfs/extent-io-tree.h+1−0
@@ -44,6 +44,7 @@ enum { · 43 unmodified lines
4444 IO_TREE_TRANS_DIRTY_PAGES,
4545 IO_TREE_ROOT_DIRTY_LOG_PAGES,
4646 IO_TREE_INODE_FILE_EXTENT,
47+ IO_TREE_LOG_CSUM_RANGE,
4748 IO_TREE_SELFTEST,
4849 };
4950
fs/btrfs/extent-tree.c+3−20
@@ -2114,22 +2114,6 @@ static u64 find_middle(struct rb_root *root) · 2113 unmodified lines
21142114 }
21152115 #endif
21162116
2117−static inline u64 heads_to_leaves(struct btrfs_fs_info *fs_info, u64 heads)
2118−{
2119− u64 num_bytes;
2120−
2121− num_bytes = heads * (sizeof(struct btrfs_extent_item) +
2122− sizeof(struct btrfs_extent_inline_ref));
2123− if (!btrfs_fs_incompat(fs_info, SKINNY_METADATA))
2124− num_bytes += heads * sizeof(struct btrfs_tree_block_info);
2125−
2126− /*
2127− * We don't ever fill up leaves all the way so multiply by 2 just to be
2128− * closer to what we're really going to want to use.
2129− */
2130− return div_u64(num_bytes, BTRFS_LEAF_DATA_SIZE(fs_info));
2131−}
2132−
21332117 /*
21342118 * Takes the number of bytes to be csumm'ed and figures out how many leaves it
21352119 * would require to store the csums for that many bytes.
@@ -2442,7 +2426,7 @@ static int __btrfs_mod_ref(struct btrfs_trans_handle *trans, · 306 unmodified lines
24422426 nritems = btrfs_header_nritems(buf);
24432427 level = btrfs_header_level(buf);
24442428
2445− if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state) && level == 0)
2429+ if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && level == 0)
24462430 return 0;
24472431
24482432 if (full_backref)
@@ -2932,7 +2916,7 @@ int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans) · 483 unmodified lines
29322916 &trimmed);
29332917
29342918 list_del_init(&block_group->bg_list);
2935− btrfs_put_block_group_trimming(block_group);
2919+ btrfs_unfreeze_block_group(block_group);
29362920 btrfs_put_block_group(block_group);
29372921
29382922 if (ret) {
@@ -3369,6 +3353,7 @@ static struct btrfs_block_group *btrfs_lock_cluster( · 430 unmodified lines
33693353 struct btrfs_block_group *block_group,
33703354 struct btrfs_free_cluster *cluster,
33713355 int delalloc)
3356+ __acquires(&cluster->refill_lock)
33723357 {
33733358 struct btrfs_block_group *used_bg = NULL;
33743359
@@ -5501,8 +5486,6 @@ out: · 2126 unmodified lines
55015486 */
55025487 if (!for_reloc && !root_dropped)
55035488 btrfs_add_dead_root(root);
5504− if (err && err != -EAGAIN)
5505− btrfs_handle_fs_error(fs_info, err, NULL);
55065489 return err;
55075490 }
55085491
fs/btrfs/extent_io.c+132−156
@@ -2333,7 +2333,7 @@ int repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start, · 2332 unmodified lines
23332333 return 0;
23342334 }
23352335
2336−int btrfs_repair_eb_io_failure(struct extent_buffer *eb, int mirror_num)
2336+int btrfs_repair_eb_io_failure(const struct extent_buffer *eb, int mirror_num)
23372337 {
23382338 struct btrfs_fs_info *fs_info = eb->fs_info;
23392339 u64 start = eb->start;
@@ -2537,8 +2537,9 @@ int btrfs_get_io_failure_record(struct inode *inode, u64 start, u64 end, · 197 unmodified lines
25372537 return 0;
25382538 }
25392539
2540−bool btrfs_check_repairable(struct inode *inode, unsigned failed_bio_pages,
2541− struct io_failure_record *failrec, int failed_mirror)
2540+static bool btrfs_check_repairable(struct inode *inode, bool needs_validation,
2541+ struct io_failure_record *failrec,
2542+ int failed_mirror)
25422543 {
25432544 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
25442545 int num_copies;
@@ -2561,7 +2562,7 @@ bool btrfs_check_repairable(struct inode *inode, unsigned failed_bio_pages, · 16 unmodified lines
25612562 * a) deliver good data to the caller
25622563 * b) correct the bad sectors on disk
25632564 */
2564− if (failed_bio_pages > 1) {
2565+ if (needs_validation) {
25652566 /*
25662567 * to fulfill b), we need to know the exact failing sectors, as
25672568 * we don't want to rewrite any more than the failed ones. thus,
@@ -2600,94 +2601,115 @@ bool btrfs_check_repairable(struct inode *inode, unsigned failed_bio_pages, · 32 unmodified lines
26002601 return true;
26012602 }
26022603
2603−
2604−struct bio *btrfs_create_repair_bio(struct inode *inode, struct bio *failed_bio,
2605− struct io_failure_record *failrec,
2606− struct page *page, int pg_offset, int icsum,
2607− bio_end_io_t *endio_func, void *data)
2604+static bool btrfs_io_needs_validation(struct inode *inode, struct bio *bio)
26082605 {
2609− struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2610− struct bio *bio;
2611− struct btrfs_io_bio *btrfs_failed_bio;
2612− struct btrfs_io_bio *btrfs_bio;
2606+ u64 len = 0;
2607+ const u32 blocksize = inode->i_sb->s_blocksize;
26132608
2614− bio = btrfs_io_bio_alloc(1);
2615− bio->bi_end_io = endio_func;
2616− bio->bi_iter.bi_sector = failrec->logical >> 9;
2617− bio->bi_iter.bi_size = 0;
2618− bio->bi_private = data;
2609+ /*
2610+ * If bi_status is BLK_STS_OK, then this was a checksum error, not an
2611+ * I/O error. In this case, we already know exactly which sector was
2612+ * bad, so we don't need to validate.
2613+ */
2614+ if (bio->bi_status == BLK_STS_OK)
2615+ return false;
26192616
2620− btrfs_failed_bio = btrfs_io_bio(failed_bio);
2621− if (btrfs_failed_bio->csum) {
2622− u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
2617+ /*
2618+ * We need to validate each sector individually if the failed I/O was
2619+ * for multiple sectors.
2620+ *
2621+ * There are a few possible bios that can end up here:
2622+ * 1. A buffered read bio, which is not cloned.
2623+ * 2. A direct I/O read bio, which is cloned.
2624+ * 3. A (buffered or direct) repair bio, which is not cloned.
2625+ *
2626+ * For cloned bios (case 2), we can get the size from
2627+ * btrfs_io_bio->iter; for non-cloned bios (cases 1 and 3), we can get
2628+ * it from the bvecs.
2629+ */
2630+ if (bio_flagged(bio, BIO_CLONED)) {
2631+ if (btrfs_io_bio(bio)->iter.bi_size > blocksize)
2632+ return true;
2633+ } else {
2634+ struct bio_vec *bvec;
2635+ int i;
26232636
2624− btrfs_bio = btrfs_io_bio(bio);
2625− btrfs_bio->csum = btrfs_bio->csum_inline;
2626− icsum *= csum_size;
2627− memcpy(btrfs_bio->csum, btrfs_failed_bio->csum + icsum,
2628− csum_size);
2637+ bio_for_each_bvec_all(bvec, bio, i) {
2638+ len += bvec->bv_len;
2639+ if (len > blocksize)
2640+ return true;
2641+ }
26292642 }
2630−
2631− bio_add_page(bio, page, failrec->len, pg_offset);
2632−
2633− return bio;
2643+ return false;
26342644 }
26352645
2636−/*
2637− * This is a generic handler for readpage errors. If other copies exist, read
2638− * those and write back good data to the failed position. Does not investigate
2639− * in remapping the failed extent elsewhere, hoping the device will be smart
2640− * enough to do this as needed
2641− */
2642−static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset,
2643− struct page *page, u64 start, u64 end,
2644− int failed_mirror)
2646+blk_status_t btrfs_submit_read_repair(struct inode *inode,
2647+ struct bio *failed_bio, u64 phy_offset,
2648+ struct page *page, unsigned int pgoff,
2649+ u64 start, u64 end, int failed_mirror,
2650+ submit_bio_hook_t *submit_bio_hook)
26452651 {
26462652 struct io_failure_record *failrec;
2647− struct inode *inode = page->mapping->host;
2653+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
26482654 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
26492655 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2650− struct bio *bio;
2651− int read_mode = 0;
2656+ struct btrfs_io_bio *failed_io_bio = btrfs_io_bio(failed_bio);
2657+ const int icsum = phy_offset >> inode->i_sb->s_blocksize_bits;
2658+ bool need_validation;
2659+ struct bio *repair_bio;
2660+ struct btrfs_io_bio *repair_io_bio;
26522661 blk_status_t status;
26532662 int ret;
2654− unsigned failed_bio_pages = failed_bio->bi_iter.bi_size >> PAGE_SHIFT;
2663+
2664+ btrfs_debug(fs_info,
2665+ "repair read error: read error at %llu", start);
26552666
26562667 BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);
26572668
26582669 ret = btrfs_get_io_failure_record(inode, start, end, &failrec);
26592670 if (ret)
2660− return ret;
2671+ return errno_to_blk_status(ret);
2672+
2673+ need_validation = btrfs_io_needs_validation(inode, failed_bio);
26612674
2662− if (!btrfs_check_repairable(inode, failed_bio_pages, failrec,
2675+ if (!btrfs_check_repairable(inode, need_validation, failrec,
26632676 failed_mirror)) {
26642677 free_io_failure(failure_tree, tree, failrec);
2665− return -EIO;
2678+ return BLK_STS_IOERR;
26662679 }
26672680
2668− if (failed_bio_pages > 1)
2669− read_mode |= REQ_FAILFAST_DEV;
2681+ repair_bio = btrfs_io_bio_alloc(1);
2682+ repair_io_bio = btrfs_io_bio(repair_bio);
2683+ repair_bio->bi_opf = REQ_OP_READ;
2684+ if (need_validation)
2685+ repair_bio->bi_opf |= REQ_FAILFAST_DEV;
2686+ repair_bio->bi_end_io = failed_bio->bi_end_io;
2687+ repair_bio->bi_iter.bi_sector = failrec->logical >> 9;
2688+ repair_bio->bi_private = failed_bio->bi_private;
26702689
2671− phy_offset >>= inode->i_sb->s_blocksize_bits;
2672− bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,
2673− start - page_offset(page),
2674− (int)phy_offset, failed_bio->bi_end_io,
2675− NULL);
2676− bio->bi_opf = REQ_OP_READ | read_mode;
2690+ if (failed_io_bio->csum) {
2691+ const u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
2692+
2693+ repair_io_bio->csum = repair_io_bio->csum_inline;
2694+ memcpy(repair_io_bio->csum,
2695+ failed_io_bio->csum + csum_size * icsum, csum_size);
2696+ }
2697+
2698+ bio_add_page(repair_bio, page, failrec->len, pgoff);
2699+ repair_io_bio->logical = failrec->start;
2700+ repair_io_bio->iter = repair_bio->bi_iter;
26772701
26782702 btrfs_debug(btrfs_sb(inode->i_sb),
2679− "Repair Read Error: submitting new read[%#x] to this_mirror=%d, in_validation=%d",
2680− read_mode, failrec->this_mirror, failrec->in_validation);
2703+"repair read error: submitting new read to mirror %d, in_validation=%d",
2704+ failrec->this_mirror, failrec->in_validation);
26812705
2682− status = tree->ops->submit_bio_hook(tree->private_data, bio, failrec->this_mirror,
2683− failrec->bio_flags);
2706+ status = submit_bio_hook(inode, repair_bio, failrec->this_mirror,
2707+ failrec->bio_flags);
26842708 if (status) {
26852709 free_io_failure(failure_tree, tree, failrec);
2686− bio_put(bio);
2687− ret = blk_status_to_errno(status);
2710+ bio_put(repair_bio);
26882711 }
2689−
2690− return ret;
2712+ return status;
26912713 }
26922714
26932715 /* lots and lots of room for performance fixes in the end_bio funcs */
@@ -2859,9 +2881,10 @@ static void end_bio_extent_readpage(struct bio *bio) · 165 unmodified lines
28592881 * If it can't handle the error it will return -EIO and
28602882 * we remain responsible for that page.
28612883 */
2862− ret = bio_readpage_error(bio, offset, page, start, end,
2863− mirror);
2864− if (ret == 0) {
2884+ if (!btrfs_submit_read_repair(inode, bio, offset, page,
2885+ start - page_offset(page),
2886+ start, end, mirror,
2887+ tree->ops->submit_bio_hook)) {
28652888 uptodate = !bio->bi_status;
28662889 offset += len;
28672890 continue;
@@ -4862,7 +4885,7 @@ static void __free_extent_buffer(struct extent_buffer *eb) · 1994 unmodified lines
48624885 kmem_cache_free(extent_buffer_cache, eb);
48634886 }
48644887
4865−int extent_buffer_under_io(struct extent_buffer *eb)
4888+int extent_buffer_under_io(const struct extent_buffer *eb)
48664889 {
48674890 return (atomic_read(&eb->io_pages) ||
48684891 test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
@@ -4967,7 +4990,7 @@ __alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start, · 98 unmodified lines
49674990 return eb;
49684991 }
49694992
4970−struct extent_buffer *btrfs_clone_extent_buffer(struct extent_buffer *src)
4993+struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src)
49714994 {
49724995 int i;
49734996 struct page *p;
@@ -5373,7 +5396,7 @@ void free_extent_buffer_stale(struct extent_buffer *eb) · 399 unmodified lines
53735396 release_extent_buffer(eb);
53745397 }
53755398
5376−void clear_extent_buffer_dirty(struct extent_buffer *eb)
5399+void clear_extent_buffer_dirty(const struct extent_buffer *eb)
53775400 {
53785401 int i;
53795402 int num_pages;
@@ -5571,8 +5594,7 @@ void read_extent_buffer(const struct extent_buffer *eb, void *dstv, · 191 unmodified lines
55715594 struct page *page;
55725595 char *kaddr;
55735596 char *dst = (char *)dstv;
5574− size_t start_offset = offset_in_page(eb->start);
5575− unsigned long i = (start_offset + start) >> PAGE_SHIFT;
5597+ unsigned long i = start >> PAGE_SHIFT;
55765598
55775599 if (start + len > eb->len) {
55785600 WARN(1, KERN_ERR "btrfs bad mapping eb start %llu len %lu, wanted %lu %lu\n",
@@ -5581,7 +5603,7 @@ void read_extent_buffer(const struct extent_buffer *eb, void *dstv, · 2 unmodified lines
55815603 return;
55825604 }
55835605
5584− offset = offset_in_page(start_offset + start);
5606+ offset = offset_in_page(start);
55855607
55865608 while (len > 0) {
55875609 page = eb->pages[i];
@@ -5606,14 +5628,13 @@ int read_extent_buffer_to_user(const struct extent_buffer *eb, · 18 unmodified lines
56065628 struct page *page;
56075629 char *kaddr;
56085630 char __user *dst = (char __user *)dstv;
5609− size_t start_offset = offset_in_page(eb->start);
5610− unsigned long i = (start_offset + start) >> PAGE_SHIFT;
5631+ unsigned long i = start >> PAGE_SHIFT;
56115632 int ret = 0;
56125633
56135634 WARN_ON(start > eb->len);
56145635 WARN_ON(start + len > eb->start + eb->len);
56155636
5616− offset = offset_in_page(start_offset + start);
5637+ offset = offset_in_page(start);
56175638
56185639 while (len > 0) {
56195640 page = eb->pages[i];
@@ -5634,48 +5655,6 @@ int read_extent_buffer_to_user(const struct extent_buffer *eb, · 14 unmodified lines
56345655 return ret;
56355656 }
56365657
5637−/*
5638− * return 0 if the item is found within a page.
5639− * return 1 if the item spans two pages.
5640− * return -EINVAL otherwise.
5641− */
5642−int map_private_extent_buffer(const struct extent_buffer *eb,
5643− unsigned long start, unsigned long min_len,
5644− char **map, unsigned long *map_start,
5645− unsigned long *map_len)
5646−{
5647− size_t offset;
5648− char *kaddr;
5649− struct page *p;
5650− size_t start_offset = offset_in_page(eb->start);
5651− unsigned long i = (start_offset + start) >> PAGE_SHIFT;
5652− unsigned long end_i = (start_offset + start + min_len - 1) >>
5653− PAGE_SHIFT;
5654−
5655− if (start + min_len > eb->len) {
5656− WARN(1, KERN_ERR "btrfs bad mapping eb start %llu len %lu, wanted %lu %lu\n",
5657− eb->start, eb->len, start, min_len);
5658− return -EINVAL;
5659− }
5660−
5661− if (i != end_i)
5662− return 1;
5663−
5664− if (i == 0) {
5665− offset = start_offset;
5666− *map_start = 0;
5667− } else {
5668− offset = 0;
5669− *map_start = ((u64)i << PAGE_SHIFT) - start_offset;
5670− }
5671−
5672− p = eb->pages[i];
5673− kaddr = page_address(p);
5674− *map = kaddr + offset;
5675− *map_len = PAGE_SIZE - offset;
5676− return 0;
5677−}
5678−
56795658 int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv,
56805659 unsigned long start, unsigned long len)
56815660 {
@@ -5684,14 +5663,13 @@ int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv, · 2 unmodified lines
56845663 struct page *page;
56855664 char *kaddr;
56865665 char *ptr = (char *)ptrv;
5687− size_t start_offset = offset_in_page(eb->start);
5688− unsigned long i = (start_offset + start) >> PAGE_SHIFT;
5666+ unsigned long i = start >> PAGE_SHIFT;
56895667 int ret = 0;
56905668
56915669 WARN_ON(start > eb->len);
56925670 WARN_ON(start + len > eb->start + eb->len);
56935671
5694− offset = offset_in_page(start_offset + start);
5672+ offset = offset_in_page(start);
56955673
56965674 while (len > 0) {
56975675 page = eb->pages[i];
@@ -5711,7 +5689,7 @@ int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv, · 13 unmodified lines
57115689 return ret;
57125690 }
57135691
5714−void write_extent_buffer_chunk_tree_uuid(struct extent_buffer *eb,
5692+void write_extent_buffer_chunk_tree_uuid(const struct extent_buffer *eb,
57155693 const void *srcv)
57165694 {
57175695 char *kaddr;
@@ -5722,7 +5700,7 @@ void write_extent_buffer_chunk_tree_uuid(struct extent_buffer *eb, · 4 unmodified lines
57225700 BTRFS_FSID_SIZE);
57235701 }
57245702
5725−void write_extent_buffer_fsid(struct extent_buffer *eb, const void *srcv)
5703+void write_extent_buffer_fsid(const struct extent_buffer *eb, const void *srcv)
57265704 {
57275705 char *kaddr;
57285706
@@ -5732,7 +5710,7 @@ void write_extent_buffer_fsid(struct extent_buffer *eb, const void *srcv) · 3 unmodified lines
57325710 BTRFS_FSID_SIZE);
57335711 }
57345712
5735−void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
5713+void write_extent_buffer(const struct extent_buffer *eb, const void *srcv,
57365714 unsigned long start, unsigned long len)
57375715 {
57385716 size_t cur;
@@ -5740,13 +5718,12 @@ void write_extent_buffer(struct extent_buffer *eb, const void *srcv, · 1 unmodified line
57405718 struct page *page;
57415719 char *kaddr;
57425720 char *src = (char *)srcv;
5743− size_t start_offset = offset_in_page(eb->start);
5744− unsigned long i = (start_offset + start) >> PAGE_SHIFT;
5721+ unsigned long i = start >> PAGE_SHIFT;
57455722
57465723 WARN_ON(start > eb->len);
57475724 WARN_ON(start + len > eb->start + eb->len);
57485725
5749− offset = offset_in_page(start_offset + start);
5726+ offset = offset_in_page(start);
57505727
57515728 while (len > 0) {
57525729 page = eb->pages[i];
@@ -5763,20 +5740,19 @@ void write_extent_buffer(struct extent_buffer *eb, const void *srcv, · 10 unmodified lines
57635740 }
57645741 }
57655742
5766−void memzero_extent_buffer(struct extent_buffer *eb, unsigned long start,
5743+void memzero_extent_buffer(const struct extent_buffer *eb, unsigned long start,
57675744 unsigned long len)
57685745 {
57695746 size_t cur;
57705747 size_t offset;
57715748 struct page *page;
57725749 char *kaddr;
5773− size_t start_offset = offset_in_page(eb->start);
5774− unsigned long i = (start_offset + start) >> PAGE_SHIFT;
5750+ unsigned long i = start >> PAGE_SHIFT;
57755751
57765752 WARN_ON(start > eb->len);
57775753 WARN_ON(start + len > eb->start + eb->len);
57785754
5779− offset = offset_in_page(start_offset + start);
5755+ offset = offset_in_page(start);
57805756
57815757 while (len > 0) {
57825758 page = eb->pages[i];
@@ -5792,8 +5768,8 @@ void memzero_extent_buffer(struct extent_buffer *eb, unsigned long start, · 9 unmodified lines
57925768 }
57935769 }
57945770
5795−void copy_extent_buffer_full(struct extent_buffer *dst,
5796− struct extent_buffer *src)
5771+void copy_extent_buffer_full(const struct extent_buffer *dst,
5772+ const struct extent_buffer *src)
57975773 {
57985774 int i;
57995775 int num_pages;
@@ -5806,7 +5782,8 @@ void copy_extent_buffer_full(struct extent_buffer *dst, · 6 unmodified lines
58065782 page_address(src->pages[i]));
58075783 }
58085784
5809−void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
5785+void copy_extent_buffer(const struct extent_buffer *dst,
5786+ const struct extent_buffer *src,
58105787 unsigned long dst_offset, unsigned long src_offset,
58115788 unsigned long len)
58125789 {
@@ -5815,12 +5792,11 @@ void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src, · 2 unmodified lines
58155792 size_t offset;
58165793 struct page *page;
58175794 char *kaddr;
5818− size_t start_offset = offset_in_page(dst->start);
5819− unsigned long i = (start_offset + dst_offset) >> PAGE_SHIFT;
5795+ unsigned long i = dst_offset >> PAGE_SHIFT;
58205796
58215797 WARN_ON(src->len != dst_len);
58225798
5823− offset = offset_in_page(start_offset + dst_offset);
5799+ offset = offset_in_page(dst_offset);
58245800
58255801 while (len > 0) {
58265802 page = dst->pages[i];
@@ -5851,12 +5827,11 @@ void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src, · 24 unmodified lines
58515827 * This helper hides the ugliness of finding the byte in an extent buffer which
58525828 * contains a given bit.
58535829 */
5854−static inline void eb_bitmap_offset(struct extent_buffer *eb,
5830+static inline void eb_bitmap_offset(const struct extent_buffer *eb,
58555831 unsigned long start, unsigned long nr,
58565832 unsigned long *page_index,
58575833 size_t *page_offset)
58585834 {
5859− size_t start_offset = offset_in_page(eb->start);
58605835 size_t byte_offset = BIT_BYTE(nr);
58615836 size_t offset;
58625837
@@ -5865,7 +5840,7 @@ static inline void eb_bitmap_offset(struct extent_buffer *eb, · 2 unmodified lines
58655840 * the bitmap item in the extent buffer + the offset of the byte in the
58665841 * bitmap item.
58675842 */
5868− offset = start_offset + start + byte_offset;
5843+ offset = start + byte_offset;
58695844
58705845 *page_index = offset >> PAGE_SHIFT;
58715846 *page_offset = offset_in_page(offset);
@@ -5877,7 +5852,7 @@ static inline void eb_bitmap_offset(struct extent_buffer *eb, · 5 unmodified lines
58775852 * @start: offset of the bitmap item in the extent buffer
58785853 * @nr: bit number to test
58795854 */
5880−int extent_buffer_test_bit(struct extent_buffer *eb, unsigned long start,
5855+int extent_buffer_test_bit(const struct extent_buffer *eb, unsigned long start,
58815856 unsigned long nr)
58825857 {
58835858 u8 *kaddr;
@@ -5899,7 +5874,7 @@ int extent_buffer_test_bit(struct extent_buffer *eb, unsigned long start, · 15 unmodified lines
58995874 * @pos: bit number of the first bit
59005875 * @len: number of bits to set
59015876 */
5902−void extent_buffer_bitmap_set(struct extent_buffer *eb, unsigned long start,
5877+void extent_buffer_bitmap_set(const struct extent_buffer *eb, unsigned long start,
59035878 unsigned long pos, unsigned long len)
59045879 {
59055880 u8 *kaddr;
@@ -5941,8 +5916,9 @@ void extent_buffer_bitmap_set(struct extent_buffer *eb, unsigned long start, · 35 unmodified lines
59415916 * @pos: bit number of the first bit
59425917 * @len: number of bits to clear
59435918 */
5944−void extent_buffer_bitmap_clear(struct extent_buffer *eb, unsigned long start,
5945− unsigned long pos, unsigned long len)
5919+void extent_buffer_bitmap_clear(const struct extent_buffer *eb,
5920+ unsigned long start, unsigned long pos,
5921+ unsigned long len)
59465922 {
59475923 u8 *kaddr;
59485924 struct page *page;
@@ -6003,14 +5979,14 @@ static void copy_pages(struct page *dst_page, struct page *src_page, · 54 unmodified lines
60035979 memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
60045980 }
60055981
6006−void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
6007− unsigned long src_offset, unsigned long len)
5982+void memcpy_extent_buffer(const struct extent_buffer *dst,
5983+ unsigned long dst_offset, unsigned long src_offset,
5984+ unsigned long len)
60085985 {
60095986 struct btrfs_fs_info *fs_info = dst->fs_info;
60105987 size_t cur;
60115988 size_t dst_off_in_page;
60125989 size_t src_off_in_page;
6013− size_t start_offset = offset_in_page(dst->start);
60145990 unsigned long dst_i;
60155991 unsigned long src_i;
60165992
@@ -6028,11 +6004,11 @@ void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset, · 11 unmodified lines
60286004 }
60296005
60306006 while (len > 0) {
6031− dst_off_in_page = offset_in_page(start_offset + dst_offset);
6032− src_off_in_page = offset_in_page(start_offset + src_offset);
6007+ dst_off_in_page = offset_in_page(dst_offset);
6008+ src_off_in_page = offset_in_page(src_offset);
60336009
6034− dst_i = (start_offset + dst_offset) >> PAGE_SHIFT;
6035− src_i = (start_offset + src_offset) >> PAGE_SHIFT;
6010+ dst_i = dst_offset >> PAGE_SHIFT;
6011+ src_i = src_offset >> PAGE_SHIFT;
60366012
60376013 cur = min(len, (unsigned long)(PAGE_SIZE -
60386014 src_off_in_page));
@@ -6048,8 +6024,9 @@ void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset, · 9 unmodified lines
60486024 }
60496025 }
60506026
6051−void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
6052− unsigned long src_offset, unsigned long len)
6027+void memmove_extent_buffer(const struct extent_buffer *dst,
6028+ unsigned long dst_offset, unsigned long src_offset,
6029+ unsigned long len)
60536030 {
60546031 struct btrfs_fs_info *fs_info = dst->fs_info;
60556032 size_t cur;
@@ -6057,7 +6034,6 @@ void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset, · 1 unmodified line
60576034 size_t src_off_in_page;
60586035 unsigned long dst_end = dst_offset + len - 1;
60596036 unsigned long src_end = src_offset + len - 1;
6060− size_t start_offset = offset_in_page(dst->start);
60616037 unsigned long dst_i;
60626038 unsigned long src_i;
60636039
@@ -6078,11 +6054,11 @@ void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset, · 14 unmodified lines
60786054 return;
60796055 }
60806056 while (len > 0) {
6081− dst_i = (start_offset + dst_end) >> PAGE_SHIFT;
6082− src_i = (start_offset + src_end) >> PAGE_SHIFT;
6057+ dst_i = dst_end >> PAGE_SHIFT;
6058+ src_i = src_end >> PAGE_SHIFT;
60836059
6084− dst_off_in_page = offset_in_page(start_offset + dst_end);
6085− src_off_in_page = offset_in_page(start_offset + src_end);
6060+ dst_off_in_page = offset_in_page(dst_end);
6061+ src_off_in_page = offset_in_page(src_end);
60866062
60876063 cur = min_t(unsigned long, len, src_off_in_page + 1);
60886064 cur = min(cur, dst_off_in_page + 1);
fs/btrfs/extent_io.h+35−32
@@ -66,6 +66,10 @@ struct btrfs_io_bio; · 65 unmodified lines
6666 struct io_failure_record;
6767 struct extent_io_tree;
6868
69+typedef blk_status_t (submit_bio_hook_t)(struct inode *inode, struct bio *bio,
70+ int mirror_num,
71+ unsigned long bio_flags);
72+
6973 typedef blk_status_t (extent_submit_bio_start_t)(void *private_data,
7074 struct bio *bio, u64 bio_offset);
7175
@@ -74,8 +78,7 @@ struct extent_io_ops { · 2 unmodified lines
7478 * The following callbacks must be always defined, the function
7579 * pointer will be called unconditionally.
7680 */
77− blk_status_t (*submit_bio_hook)(struct inode *inode, struct bio *bio,
78− int mirror_num, unsigned long bio_flags);
81+ submit_bio_hook_t *submit_bio_hook;
7982 int (*readpage_end_io_hook)(struct btrfs_io_bio *io_bio, u64 phy_offset,
8083 struct page *page, u64 start, u64 end,
8184 int mirror);
@@ -209,7 +212,7 @@ struct extent_buffer *__alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info, · 127 unmodified lines
209212 u64 start, unsigned long len);
210213 struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
211214 u64 start);
212−struct extent_buffer *btrfs_clone_extent_buffer(struct extent_buffer *src);
215+struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src);
213216 struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info,
214217 u64 start);
215218 void free_extent_buffer(struct extent_buffer *eb);
@@ -227,7 +230,7 @@ static inline int num_extent_pages(const struct extent_buffer *eb) · 11 unmodified lines
227230 (eb->start >> PAGE_SHIFT);
228231 }
229232
230−static inline int extent_buffer_uptodate(struct extent_buffer *eb)
233+static inline int extent_buffer_uptodate(const struct extent_buffer *eb)
231234 {
232235 return test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
233236 }
@@ -240,37 +243,37 @@ void read_extent_buffer(const struct extent_buffer *eb, void *dst, · 6 unmodified lines
240243 int read_extent_buffer_to_user(const struct extent_buffer *eb,
241244 void __user *dst, unsigned long start,
242245 unsigned long len);
243−void write_extent_buffer_fsid(struct extent_buffer *eb, const void *src);
244−void write_extent_buffer_chunk_tree_uuid(struct extent_buffer *eb,
246+void write_extent_buffer_fsid(const struct extent_buffer *eb, const void *src);
247+void write_extent_buffer_chunk_tree_uuid(const struct extent_buffer *eb,
245248 const void *src);
246−void write_extent_buffer(struct extent_buffer *eb, const void *src,
249+void write_extent_buffer(const struct extent_buffer *eb, const void *src,
247250 unsigned long start, unsigned long len);
248−void copy_extent_buffer_full(struct extent_buffer *dst,
249− struct extent_buffer *src);
250−void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
251+void copy_extent_buffer_full(const struct extent_buffer *dst,
252+ const struct extent_buffer *src);
253+void copy_extent_buffer(const struct extent_buffer *dst,
254+ const struct extent_buffer *src,
251255 unsigned long dst_offset, unsigned long src_offset,
252256 unsigned long len);
253−void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
254− unsigned long src_offset, unsigned long len);
255−void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
256− unsigned long src_offset, unsigned long len);
257−void memzero_extent_buffer(struct extent_buffer *eb, unsigned long start,
257+void memcpy_extent_buffer(const struct extent_buffer *dst,
258+ unsigned long dst_offset, unsigned long src_offset,
259+ unsigned long len);
260+void memmove_extent_buffer(const struct extent_buffer *dst,
261+ unsigned long dst_offset, unsigned long src_offset,
262+ unsigned long len);
263+void memzero_extent_buffer(const struct extent_buffer *eb, unsigned long start,
258264 unsigned long len);
259−int extent_buffer_test_bit(struct extent_buffer *eb, unsigned long start,
265+int extent_buffer_test_bit(const struct extent_buffer *eb, unsigned long start,
260266 unsigned long pos);
261−void extent_buffer_bitmap_set(struct extent_buffer *eb, unsigned long start,
267+void extent_buffer_bitmap_set(const struct extent_buffer *eb, unsigned long start,
262268 unsigned long pos, unsigned long len);
263−void extent_buffer_bitmap_clear(struct extent_buffer *eb, unsigned long start,
264− unsigned long pos, unsigned long len);
265−void clear_extent_buffer_dirty(struct extent_buffer *eb);
269+void extent_buffer_bitmap_clear(const struct extent_buffer *eb,
270+ unsigned long start, unsigned long pos,
271+ unsigned long len);
272+void clear_extent_buffer_dirty(const struct extent_buffer *eb);
266273 bool set_extent_buffer_dirty(struct extent_buffer *eb);
267274 void set_extent_buffer_uptodate(struct extent_buffer *eb);
268275 void clear_extent_buffer_uptodate(struct extent_buffer *eb);
269−int extent_buffer_under_io(struct extent_buffer *eb);
270−int map_private_extent_buffer(const struct extent_buffer *eb,
271− unsigned long offset, unsigned long min_len,
272− char **map, unsigned long *map_start,
273− unsigned long *map_len);
276+int extent_buffer_under_io(const struct extent_buffer *eb);
274277 void extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end);
275278 void extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end);
276279 void extent_clear_unlock_delalloc(struct inode *inode, u64 start, u64 end,
@@ -289,7 +292,7 @@ int repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start, · 12 unmodified lines
289292 u64 length, u64 logical, struct page *page,
290293 unsigned int pg_offset, int mirror_num);
291294 void end_extent_writepage(struct page *page, int err, u64 start, u64 end);
292−int btrfs_repair_eb_io_failure(struct extent_buffer *eb, int mirror_num);
295+int btrfs_repair_eb_io_failure(const struct extent_buffer *eb, int mirror_num);
293296
294297 /*
295298 * When IO fails, either with EIO or csum verification fails, we
@@ -311,12 +314,12 @@ struct io_failure_record { · 15 unmodified lines
311314 };
312315
313316
314−bool btrfs_check_repairable(struct inode *inode, unsigned failed_bio_pages,
315− struct io_failure_record *failrec, int fail_mirror);
316−struct bio *btrfs_create_repair_bio(struct inode *inode, struct bio *failed_bio,
317− struct io_failure_record *failrec,
318− struct page *page, int pg_offset, int icsum,
319− bio_end_io_t *endio_func, void *data);
317+blk_status_t btrfs_submit_read_repair(struct inode *inode,
318+ struct bio *failed_bio, u64 phy_offset,
319+ struct page *page, unsigned int pgoff,
320+ u64 start, u64 end, int failed_mirror,
321+ submit_bio_hook_t *submit_bio_hook);
322+
320323 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
321324 bool find_lock_delalloc_range(struct inode *inode,
322325 struct page *locked_page, u64 *start,
fs/btrfs/file-item.c+35−27
@@ -242,11 +242,13 @@ int btrfs_lookup_file_extent(struct btrfs_trans_handle *trans, · 241 unmodified lines
242242 /**
243243 * btrfs_lookup_bio_sums - Look up checksums for a bio.
244244 * @inode: inode that the bio is for.
245− * @bio: bio embedded in btrfs_io_bio.
245+ * @bio: bio to look up.
246246 * @offset: Unless (u64)-1, look up checksums for this offset in the file.
247247 * If (u64)-1, use the page offsets from the bio instead.
248− * @dst: Buffer of size btrfs_super_csum_size() used to return checksum. If
249− * NULL, the checksum is returned in btrfs_io_bio(bio)->csum instead.
248+ * @dst: Buffer of size nblocks * btrfs_super_csum_size() used to return
249+ * checksum (nblocks = bio->bi_iter.bi_size / fs_info->sectorsize). If
250+ * NULL, the checksum buffer is allocated and returned in
251+ * btrfs_io_bio(bio)->csum instead.
250252 *
251253 * Return: BLK_STS_RESOURCE if allocating memory fails, BLK_STS_OK otherwise.
252254 */
@@ -256,7 +258,6 @@ blk_status_t btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio, · 3 unmodified lines
256258 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
257259 struct bio_vec bvec;
258260 struct bvec_iter iter;
259− struct btrfs_io_bio *btrfs_bio = btrfs_io_bio(bio);
260261 struct btrfs_csum_item *item = NULL;
261262 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
262263 struct btrfs_path *path;
@@ -277,6 +278,8 @@ blk_status_t btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio, · 14 unmodified lines
277278
278279 nblocks = bio->bi_iter.bi_size >> inode->i_sb->s_blocksize_bits;
279280 if (!dst) {
281+ struct btrfs_io_bio *btrfs_bio = btrfs_io_bio(bio);
282+
280283 if (nblocks * csum_size > BTRFS_BIO_INLINE_CSUM_SIZE) {
281284 btrfs_bio->csum = kmalloc_array(nblocks, csum_size,
282285 GFP_NOFS);
@@ -598,13 +601,12 @@ blk_status_t btrfs_csum_one_bio(struct inode *inode, struct bio *bio, · 315 unmodified lines
598601 index = 0;
599602 }
600603
601− crypto_shash_init(shash);
602604 data = kmap_atomic(bvec.bv_page);
603− crypto_shash_update(shash, data + bvec.bv_offset
605+ crypto_shash_digest(shash, data + bvec.bv_offset
604606 + (i * fs_info->sectorsize),
605− fs_info->sectorsize);
607+ fs_info->sectorsize,
608+ sums->sums + index);
606609 kunmap_atomic(data);
607− crypto_shash_final(shash, (char *)(sums->sums + index));
608610 index += csum_size;
609611 offset += fs_info->sectorsize;
610612 this_sum_bytes += fs_info->sectorsize;
@@ -869,7 +871,7 @@ again: · 258 unmodified lines
869871 }
870872 ret = PTR_ERR(item);
871873 if (ret != -EFBIG && ret != -ENOENT)
872− goto fail_unlock;
874+ goto out;
873875
874876 if (ret == -EFBIG) {
875877 u32 item_size;
@@ -887,10 +889,12 @@ again: · 11 unmodified lines
887889 nritems = btrfs_header_nritems(path->nodes[0]);
888890 if (!nritems || (path->slots[0] >= nritems - 1)) {
889891 ret = btrfs_next_leaf(root, path);
890− if (ret == 1)
892+ if (ret < 0) {
893+ goto out;
894+ } else if (ret > 0) {
891895 found_next = 1;
892− if (ret != 0)
893896 goto insert;
897+ }
894898 slot = path->slots[0];
895899 }
896900 btrfs_item_key_to_cpu(path->nodes[0], &found_key, slot);
@@ -905,14 +909,27 @@ again: · 8 unmodified lines
905909 }
906910
907911 /*
908− * at this point, we know the tree has an item, but it isn't big
909− * enough yet to put our csum in. Grow it
912+ * At this point, we know the tree has a checksum item that ends at an
913+ * offset matching the start of the checksum range we want to insert.
914+ * We try to extend that item as much as possible and then add as many
915+ * checksums to it as they fit.
916+ *
917+ * First check if the leaf has enough free space for at least one
918+ * checksum. If it has go directly to the item extension code, otherwise
919+ * release the path and do a search for insertion before the extension.
910920 */
921+ if (btrfs_leaf_free_space(leaf) >= csum_size) {
922+ btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
923+ csum_offset = (bytenr - found_key.offset) >>
924+ fs_info->sb->s_blocksize_bits;
925+ goto extend_csum;
926+ }
927+
911928 btrfs_release_path(path);
912929 ret = btrfs_search_slot(trans, root, &file_key, path,
913930 csum_size, 1);
914931 if (ret < 0)
915− goto fail_unlock;
932+ goto out;
916933
917934 if (ret > 0) {
918935 if (path->slots[0] == 0)
@@ -931,19 +948,13 @@ again: · 12 unmodified lines
931948 goto insert;
932949 }
933950
951+extend_csum:
934952 if (csum_offset == btrfs_item_size_nr(leaf, path->slots[0]) /
935953 csum_size) {
936954 int extend_nr;
937955 u64 tmp;
938956 u32 diff;
939− u32 free_space;
940−
941− if (btrfs_leaf_free_space(leaf) <
942− sizeof(struct btrfs_item) + csum_size * 2)
943− goto insert;
944957
945− free_space = btrfs_leaf_free_space(leaf) -
946− sizeof(struct btrfs_item) - csum_size;
947958 tmp = sums->len - total_bytes;
948959 tmp >>= fs_info->sb->s_blocksize_bits;
949960 WARN_ON(tmp < 1);
@@ -954,7 +965,7 @@ again: · 4 unmodified lines
954965 MAX_CSUM_ITEMS(fs_info, csum_size) * csum_size);
955966
956967 diff = diff - btrfs_item_size_nr(leaf, path->slots[0]);
957− diff = min(free_space, diff);
968+ diff = min_t(u32, btrfs_leaf_free_space(leaf), diff);
958969 diff /= csum_size;
959970 diff *= csum_size;
960971
@@ -985,9 +996,9 @@ insert: · 24 unmodified lines
985996 ins_size);
986997 path->leave_spinning = 0;
987998 if (ret < 0)
988− goto fail_unlock;
999+ goto out;
9891000 if (WARN_ON(ret != 0))
990− goto fail_unlock;
1001+ goto out;
9911002 leaf = path->nodes[0];
9921003 csum:
9931004 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_csum_item);
@@ -1017,9 +1028,6 @@ found: · 23 unmodified lines
10171028 out:
10181029 btrfs_free_path(path);
10191030 return ret;
1020−
1021−fail_unlock:
1022− goto out;
10231031 }
10241032
10251033 void btrfs_extent_item_to_extent_map(struct btrfs_inode *inode,
fs/btrfs/file.c+94−17
@@ -275,26 +275,18 @@ static int __btrfs_run_defrag_inode(struct btrfs_fs_info *fs_info, · 274 unmodified lines
275275 {
276276 struct btrfs_root *inode_root;
277277 struct inode *inode;
278− struct btrfs_key key;
279278 struct btrfs_ioctl_defrag_range_args range;
280279 int num_defrag;
281280 int ret;
282281
283282 /* get the inode */
284− key.objectid = defrag->root;
285− key.type = BTRFS_ROOT_ITEM_KEY;
286− key.offset = (u64)-1;
287−
288− inode_root = btrfs_get_fs_root(fs_info, &key, true);
283+ inode_root = btrfs_get_fs_root(fs_info, defrag->root, true);
289284 if (IS_ERR(inode_root)) {
290285 ret = PTR_ERR(inode_root);
291286 goto cleanup;
292287 }
293288
294− key.objectid = defrag->ino;
295− key.type = BTRFS_INODE_ITEM_KEY;
296− key.offset = 0;
297− inode = btrfs_iget(fs_info->sb, &key, inode_root);
289+ inode = btrfs_iget(fs_info->sb, defrag->ino, inode_root);
298290 btrfs_put_root(inode_root);
299291 if (IS_ERR(inode)) {
300292 ret = PTR_ERR(inode);
@@ -775,7 +767,7 @@ int __btrfs_drop_extents(struct btrfs_trans_handle *trans, · 474 unmodified lines
775767 if (start >= BTRFS_I(inode)->disk_i_size && !replace_extent)
776768 modify_tree = 0;
777769
778− update_refs = (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
770+ update_refs = (test_bit(BTRFS_ROOT_SHAREABLE, &root->state) ||
779771 root == fs_info->tree_root);
780772 while (1) {
781773 recow = 0;
@@ -1817,21 +1809,61 @@ again: · 1035 unmodified lines
18171809 return num_written ? num_written : ret;
18181810 }
18191811
1820−static ssize_t __btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from)
1812+static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info,
1813+ const struct iov_iter *iter, loff_t offset)
1814+{
1815+ const unsigned int blocksize_mask = fs_info->sectorsize - 1;
1816+
1817+ if (offset & blocksize_mask)
1818+ return -EINVAL;
1819+
1820+ if (iov_iter_alignment(iter) & blocksize_mask)
1821+ return -EINVAL;
1822+
1823+ return 0;
1824+}
1825+
1826+static ssize_t btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from)
18211827 {
18221828 struct file *file = iocb->ki_filp;
18231829 struct inode *inode = file_inode(file);
1824− loff_t pos;
1825− ssize_t written;
1830+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1831+ loff_t pos = iocb->ki_pos;
1832+ ssize_t written = 0;
18261833 ssize_t written_buffered;
18271834 loff_t endbyte;
18281835 int err;
1836+ size_t count = 0;
1837+ bool relock = false;
18291838
1830− written = generic_file_direct_write(iocb, from);
1839+ if (check_direct_IO(fs_info, from, pos))
1840+ goto buffered;
1841+
1842+ count = iov_iter_count(from);
1843+ /*
1844+ * If the write DIO is beyond the EOF, we need update the isize, but it
1845+ * is protected by i_mutex. So we can not unlock the i_mutex at this
1846+ * case.
1847+ */
1848+ if (pos + count <= inode->i_size) {
1849+ inode_unlock(inode);
1850+ relock = true;
1851+ } else if (iocb->ki_flags & IOCB_NOWAIT) {
1852+ return -EAGAIN;
1853+ }
1854+
1855+ down_read(&BTRFS_I(inode)->dio_sem);
1856+ written = iomap_dio_rw(iocb, from, &btrfs_dio_iomap_ops, &btrfs_dops,
1857+ is_sync_kiocb(iocb));
1858+ up_read(&BTRFS_I(inode)->dio_sem);
1859+
1860+ if (relock)
1861+ inode_lock(inode);
18311862
18321863 if (written < 0 || !iov_iter_count(from))
18331864 return written;
18341865
1866+buffered:
18351867 pos = iocb->ki_pos;
18361868 written_buffered = btrfs_buffered_write(iocb, from);
18371869 if (written_buffered < 0) {
@@ -1970,7 +2002,7 @@ static ssize_t btrfs_file_write_iter(struct kiocb *iocb, · 132 unmodified lines
19702002 atomic_inc(&BTRFS_I(inode)->sync_writers);
19712003
19722004 if (iocb->ki_flags & IOCB_DIRECT) {
1973− num_written = __btrfs_direct_write(iocb, from);
2005+ num_written = btrfs_direct_write(iocb, from);
19742006 } else {
19752007 num_written = btrfs_buffered_write(iocb, from);
19762008 if (num_written > 0)
@@ -3484,9 +3516,54 @@ static int btrfs_file_open(struct inode *inode, struct file *filp) · 1507 unmodified lines
34843516 return generic_file_open(inode, filp);
34853517 }
34863518
3519+static int check_direct_read(struct btrfs_fs_info *fs_info,
3520+ const struct iov_iter *iter, loff_t offset)
3521+{
3522+ int ret;
3523+ int i, seg;
3524+
3525+ ret = check_direct_IO(fs_info, iter, offset);
3526+ if (ret < 0)
3527+ return ret;
3528+
3529+ for (seg = 0; seg < iter->nr_segs; seg++)
3530+ for (i = seg + 1; i < iter->nr_segs; i++)
3531+ if (iter->iov[seg].iov_base == iter->iov[i].iov_base)
3532+ return -EINVAL;
3533+ return 0;
3534+}
3535+
3536+static ssize_t btrfs_direct_read(struct kiocb *iocb, struct iov_iter *to)
3537+{
3538+ struct inode *inode = file_inode(iocb->ki_filp);
3539+ ssize_t ret;
3540+
3541+ if (check_direct_read(btrfs_sb(inode->i_sb), to, iocb->ki_pos))
3542+ return 0;
3543+
3544+ inode_lock_shared(inode);
3545+ ret = iomap_dio_rw(iocb, to, &btrfs_dio_iomap_ops, &btrfs_dops,
3546+ is_sync_kiocb(iocb));
3547+ inode_unlock_shared(inode);
3548+ return ret;
3549+}
3550+
3551+static ssize_t btrfs_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
3552+{
3553+ ssize_t ret = 0;
3554+
3555+ if (iocb->ki_flags & IOCB_DIRECT) {
3556+ ret = btrfs_direct_read(iocb, to);
3557+ if (ret < 0)
3558+ return ret;
3559+ }
3560+
3561+ return generic_file_buffered_read(iocb, to, ret);
3562+}
3563+
34873564 const struct file_operations btrfs_file_operations = {
34883565 .llseek = btrfs_file_llseek,
3489− .read_iter = generic_file_read_iter,
3566+ .read_iter = btrfs_file_read_iter,
34903567 .splice_read = generic_file_splice_read,
34913568 .write_iter = btrfs_file_write_iter,
34923569 .mmap = btrfs_file_mmap,
fs/btrfs/free-space-cache.c+17−64
@@ -82,7 +82,7 @@ static struct inode *__lookup_free_space_inode(struct btrfs_root *root, · 81 unmodified lines
8282 * sure NOFS is set to keep us from deadlocking.
8383 */
8484 nofs_flag = memalloc_nofs_save();
85− inode = btrfs_iget_path(fs_info->sb, &location, root, path);
85+ inode = btrfs_iget_path(fs_info->sb, location.objectid, root, path);
8686 btrfs_release_path(path);
8787 memalloc_nofs_restore(nofs_flag);
8888 if (IS_ERR(inode))
@@ -1190,13 +1190,10 @@ out: · 1101 unmodified lines
11901190 if (ret) {
11911191 invalidate_inode_pages2(inode->i_mapping);
11921192 BTRFS_I(inode)->generation = 0;
1193− if (block_group) {
1194−#ifdef CONFIG_BTRFS_DEBUG
1195− btrfs_err(root->fs_info,
1196− "failed to write free space cache for block group %llu",
1197− block_group->start);
1198−#endif
1199− }
1193+ if (block_group)
1194+ btrfs_debug(root->fs_info,
1195+ "failed to write free space cache for block group %llu error %d",
1196+ block_group->start, ret);
12001197 }
12011198 btrfs_update_inode(trans, root, inode);
12021199
@@ -1415,11 +1412,9 @@ int btrfs_write_out_cache(struct btrfs_trans_handle *trans, · 212 unmodified lines
14151412 ret = __btrfs_write_out_cache(fs_info->tree_root, inode, ctl,
14161413 block_group, &block_group->io_ctl, trans);
14171414 if (ret) {
1418−#ifdef CONFIG_BTRFS_DEBUG
1419− btrfs_err(fs_info,
1420− "failed to write free space cache for block group %llu",
1421− block_group->start);
1422−#endif
1415+ btrfs_debug(fs_info,
1416+ "failed to write free space cache for block group %llu error %d",
1417+ block_group->start, ret);
14231418 spin_lock(&block_group->lock);
14241419 block_group->disk_cache_state = BTRFS_DC_ERROR;
14251420 spin_unlock(&block_group->lock);
@@ -3762,46 +3757,6 @@ out: · 2336 unmodified lines
37623757 return ret;
37633758 }
37643759
3765−void btrfs_get_block_group_trimming(struct btrfs_block_group *cache)
3766−{
3767− atomic_inc(&cache->trimming);
3768−}
3769−
3770−void btrfs_put_block_group_trimming(struct btrfs_block_group *block_group)
3771−{
3772− struct btrfs_fs_info *fs_info = block_group->fs_info;
3773− struct extent_map_tree *em_tree;
3774− struct extent_map *em;
3775− bool cleanup;
3776−
3777− spin_lock(&block_group->lock);
3778− cleanup = (atomic_dec_and_test(&block_group->trimming) &&
3779− block_group->removed);
3780− spin_unlock(&block_group->lock);
3781−
3782− if (cleanup) {
3783− mutex_lock(&fs_info->chunk_mutex);
3784− em_tree = &fs_info->mapping_tree;
3785− write_lock(&em_tree->lock);
3786− em = lookup_extent_mapping(em_tree, block_group->start,
3787− 1);
3788− BUG_ON(!em); /* logic error, can't happen */
3789− remove_extent_mapping(em_tree, em);
3790− write_unlock(&em_tree->lock);
3791− mutex_unlock(&fs_info->chunk_mutex);
3792−
3793− /* once for us and once for the tree */
3794− free_extent_map(em);
3795− free_extent_map(em);
3796−
3797− /*
3798− * We've left one free space entry and other tasks trimming
3799− * this block group have left 1 entry each one. Free them.
3800− */
3801− __btrfs_remove_free_space_cache(block_group->free_space_ctl);
3802− }
3803−}
3804−
38053760 int btrfs_trim_block_group(struct btrfs_block_group *block_group,
38063761 u64 *trimmed, u64 start, u64 end, u64 minlen)
38073762 {
@@ -3816,7 +3771,7 @@ int btrfs_trim_block_group(struct btrfs_block_group *block_group, · 8 unmodified lines
38163771 spin_unlock(&block_group->lock);
38173772 return 0;
38183773 }
3819− btrfs_get_block_group_trimming(block_group);
3774+ btrfs_freeze_block_group(block_group);
38203775 spin_unlock(&block_group->lock);
38213776
38223777 ret = trim_no_bitmap(block_group, trimmed, start, end, minlen, false);
@@ -3829,7 +3784,7 @@ int btrfs_trim_block_group(struct btrfs_block_group *block_group, · 6 unmodified lines
38293784 if (rem)
38303785 reset_trimming_bitmap(ctl, offset_to_bitmap(ctl, end));
38313786 out:
3832− btrfs_put_block_group_trimming(block_group);
3787+ btrfs_unfreeze_block_group(block_group);
38333788 return ret;
38343789 }
38353790
@@ -3846,11 +3801,11 @@ int btrfs_trim_block_group_extents(struct btrfs_block_group *block_group, · 10 unmodified lines
38463801 spin_unlock(&block_group->lock);
38473802 return 0;
38483803 }
3849− btrfs_get_block_group_trimming(block_group);
3804+ btrfs_freeze_block_group(block_group);
38503805 spin_unlock(&block_group->lock);
38513806
38523807 ret = trim_no_bitmap(block_group, trimmed, start, end, minlen, async);
3853− btrfs_put_block_group_trimming(block_group);
3808+ btrfs_unfreeze_block_group(block_group);
38543809
38553810 return ret;
38563811 }
@@ -3868,13 +3823,13 @@ int btrfs_trim_block_group_bitmaps(struct btrfs_block_group *block_group, · 11 unmodified lines
38683823 spin_unlock(&block_group->lock);
38693824 return 0;
38703825 }
3871− btrfs_get_block_group_trimming(block_group);
3826+ btrfs_freeze_block_group(block_group);
38723827 spin_unlock(&block_group->lock);
38733828
38743829 ret = trim_bitmaps(block_group, trimmed, start, end, minlen, maxlen,
38753830 async);
38763831
3877− btrfs_put_block_group_trimming(block_group);
3832+ btrfs_unfreeze_block_group(block_group);
38783833
38793834 return ret;
38803835 }
@@ -4035,11 +3990,9 @@ int btrfs_write_out_ino_cache(struct btrfs_root *root, · 154 unmodified lines
40353990 if (release_metadata)
40363991 btrfs_delalloc_release_metadata(BTRFS_I(inode),
40373992 inode->i_size, true);
4038−#ifdef CONFIG_BTRFS_DEBUG
4039− btrfs_err(fs_info,
4040− "failed to write free ino cache for root %llu",
4041− root->root_key.objectid);
4042−#endif
3993+ btrfs_debug(fs_info,
3994+ "failed to write free ino cache for root %llu error %d",
3995+ root->root_key.objectid, ret);
40433996 }
40443997
40453998 return ret;
fs/btrfs/inode.c+452−867
@@ -5,7 +5,6 @@ · 4 unmodified lines
55
66 #include <linux/kernel.h>
77 #include <linux/bio.h>
8−#include <linux/buffer_head.h>
98 #include <linux/file.h>
109 #include <linux/fs.h>
1110 #include <linux/pagemap.h>
@@ -49,17 +48,18 @@ · 37 unmodified lines
4948 #include "qgroup.h"
5049 #include "delalloc-space.h"
5150 #include "block-group.h"
51+#include "space-info.h"
5252
5353 struct btrfs_iget_args {
54− struct btrfs_key *location;
54+ u64 ino;
5555 struct btrfs_root *root;
5656 };
5757
5858 struct btrfs_dio_data {
5959 u64 reserve;
60− u64 unsubmitted_oe_range_start;
61− u64 unsubmitted_oe_range_end;
62− int overwrite;
60+ loff_t length;
61+ ssize_t submitted;
62+ struct extent_changeset *data_reserved;
6363 };
6464
6565 static const struct inode_operations btrfs_dir_inode_operations;
@@ -1142,7 +1142,7 @@ out_unlock: · 1076 unmodified lines
11421142 */
11431143 if (extent_reserved) {
11441144 extent_clear_unlock_delalloc(inode, start,
1145− start + cur_alloc_size,
1145+ start + cur_alloc_size - 1,
11461146 locked_page,
11471147 clear_bits,
11481148 page_ops);
@@ -1355,6 +1355,66 @@ static noinline int csum_exist_in_range(struct btrfs_fs_info *fs_info, · 206 unmodified lines
13551355 return 1;
13561356 }
13571357
1358+static int fallback_to_cow(struct inode *inode, struct page *locked_page,
1359+ const u64 start, const u64 end,
1360+ int *page_started, unsigned long *nr_written)
1361+{
1362+ const bool is_space_ino = btrfs_is_free_space_inode(BTRFS_I(inode));
1363+ const u64 range_bytes = end + 1 - start;
1364+ struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
1365+ u64 range_start = start;
1366+ u64 count;
1367+
1368+ /*
1369+ * If EXTENT_NORESERVE is set it means that when the buffered write was
1370+ * made we had not enough available data space and therefore we did not
1371+ * reserve data space for it, since we though we could do NOCOW for the
1372+ * respective file range (either there is prealloc extent or the inode
1373+ * has the NOCOW bit set).
1374+ *
1375+ * However when we need to fallback to COW mode (because for example the
1376+ * block group for the corresponding extent was turned to RO mode by a
1377+ * scrub or relocation) we need to do the following:
1378+ *
1379+ * 1) We increment the bytes_may_use counter of the data space info.
1380+ * If COW succeeds, it allocates a new data extent and after doing
1381+ * that it decrements the space info's bytes_may_use counter and
1382+ * increments its bytes_reserved counter by the same amount (we do
1383+ * this at btrfs_add_reserved_bytes()). So we need to increment the
1384+ * bytes_may_use counter to compensate (when space is reserved at
1385+ * buffered write time, the bytes_may_use counter is incremented);
1386+ *
1387+ * 2) We clear the EXTENT_NORESERVE bit from the range. We do this so
1388+ * that if the COW path fails for any reason, it decrements (through
1389+ * extent_clear_unlock_delalloc()) the bytes_may_use counter of the
1390+ * data space info, which we incremented in the step above.
1391+ *
1392+ * If we need to fallback to cow and the inode corresponds to a free
1393+ * space cache inode, we must also increment bytes_may_use of the data
1394+ * space_info for the same reason. Space caches always get a prealloc
1395+ * extent for them, however scrub or balance may have set the block
1396+ * group that contains that extent to RO mode.
1397+ */
1398+ count = count_range_bits(io_tree, &range_start, end, range_bytes,
1399+ EXTENT_NORESERVE, 0);
1400+ if (count > 0 || is_space_ino) {
1401+ const u64 bytes = is_space_ino ? range_bytes : count;
1402+ struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
1403+ struct btrfs_space_info *sinfo = fs_info->data_sinfo;
1404+
1405+ spin_lock(&sinfo->lock);
1406+ btrfs_space_info_update_bytes_may_use(fs_info, sinfo, bytes);
1407+ spin_unlock(&sinfo->lock);
1408+
1409+ if (count > 0)
1410+ clear_extent_bit(io_tree, start, end, EXTENT_NORESERVE,
1411+ 0, 0, NULL);
1412+ }
1413+
1414+ return cow_file_range(inode, locked_page, start, end, page_started,
1415+ nr_written, 1);
1416+}
1417+
13581418 /*
13591419 * when nowcow writeback call back. This checks for snapshots or COW copies
13601420 * of the extents that exist in the file, and COWs the file as required.
@@ -1602,9 +1662,9 @@ out_check: · 241 unmodified lines
16021662 * NOCOW, following one which needs to be COW'ed
16031663 */
16041664 if (cow_start != (u64)-1) {
1605− ret = cow_file_range(inode, locked_page,
1606− cow_start, found_key.offset - 1,
1607− page_started, nr_written, 1);
1665+ ret = fallback_to_cow(inode, locked_page, cow_start,
1666+ found_key.offset - 1,
1667+ page_started, nr_written);
16081668 if (ret) {
16091669 if (nocow)
16101670 btrfs_dec_nocow_writers(fs_info,
@@ -1693,8 +1753,8 @@ out_check: · 82 unmodified lines
16931753
16941754 if (cow_start != (u64)-1) {
16951755 cur_offset = end;
1696− ret = cow_file_range(inode, locked_page, cow_start, end,
1697− page_started, nr_written, 1);
1756+ ret = fallback_to_cow(inode, locked_page, cow_start, end,
1757+ page_started, nr_written);
16981758 if (ret)
16991759 goto error;
17001760 }
@@ -2726,10 +2786,9 @@ void btrfs_writepage_endio_finish_ordered(struct page *page, u64 start, · 1025 unmodified lines
27262786 btrfs_queue_work(wq, &ordered_extent->work);
27272787 }
27282788
2729−static int __readpage_endio_check(struct inode *inode,
2730− struct btrfs_io_bio *io_bio,
2731− int icsum, struct page *page,
2732− int pgoff, u64 start, size_t len)
2789+static int check_data_csum(struct inode *inode, struct btrfs_io_bio *io_bio,
2790+ int icsum, struct page *page, int pgoff, u64 start,
2791+ size_t len)
27332792 {
27342793 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
27352794 SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
@@ -2743,9 +2802,7 @@ static int __readpage_endio_check(struct inode *inode, · 7 unmodified lines
27432802 kaddr = kmap_atomic(page);
27442803 shash->tfm = fs_info->csum_shash;
27452804
2746− crypto_shash_init(shash);
2747− crypto_shash_update(shash, kaddr + pgoff, len);
2748− crypto_shash_final(shash, csum);
2805+ crypto_shash_digest(shash, kaddr + pgoff, len, csum);
27492806
27502807 if (memcmp(csum, csum_expected, csum_size))
27512808 goto zeroit;
@@ -2790,8 +2847,8 @@ static int btrfs_readpage_end_io_hook(struct btrfs_io_bio *io_bio, · 38 unmodified lines
27902847 }
27912848
27922849 phy_offset >>= inode->i_sb->s_blocksize_bits;
2793− return __readpage_endio_check(inode, io_bio, phy_offset, page, offset,
2794− start, (size_t)(end - start + 1));
2850+ return check_data_csum(inode, io_bio, phy_offset, page, offset, start,
2851+ (size_t)(end - start + 1));
27952852 }
27962853
27972854 /*
@@ -2981,7 +3038,7 @@ int btrfs_orphan_cleanup(struct btrfs_root *root) · 183 unmodified lines
29813038 found_key.objectid = found_key.offset;
29823039 found_key.type = BTRFS_INODE_ITEM_KEY;
29833040 found_key.offset = 0;
2984− inode = btrfs_iget(fs_info->sb, &found_key, root);
3041+ inode = btrfs_iget(fs_info->sb, last_objectid, root);
29853042 ret = PTR_ERR_OR_ZERO(inode);
29863043 if (ret && ret != -ENOENT)
29873044 goto out;
@@ -3000,18 +3057,16 @@ int btrfs_orphan_cleanup(struct btrfs_root *root) · 12 unmodified lines
30003057 * orphan must not get deleted.
30013058 * find_dead_roots already ran before us, so if this
30023059 * is a snapshot deletion, we should find the root
3003− * in the dead_roots list
3060+ * in the fs_roots radix tree.
30043061 */
3005− spin_lock(&fs_info->trans_lock);
3006− list_for_each_entry(dead_root, &fs_info->dead_roots,
3007− root_list) {
3008− if (dead_root->root_key.objectid ==
3009− found_key.objectid) {
3010− is_dead_root = 1;
3011− break;
3012− }
3013− }
3014− spin_unlock(&fs_info->trans_lock);
3062+
3063+ spin_lock(&fs_info->fs_roots_radix_lock);
3064+ dead_root = radix_tree_lookup(&fs_info->fs_roots_radix,
3065+ (unsigned long)found_key.objectid);
3066+ if (dead_root && btrfs_root_refs(&dead_root->root_item) == 0)
3067+ is_dead_root = 1;
3068+ spin_unlock(&fs_info->fs_roots_radix_lock);
3069+
30153070 if (is_dead_root) {
30163071 /* prevent this orphan from being found again */
30173072 key.offset = found_key.objectid - 1;
@@ -3357,43 +3412,40 @@ static void fill_inode_item(struct btrfs_trans_handle *trans, · 339 unmodified lines
33573412
33583413 btrfs_init_map_token(&token, leaf);
33593414
3360− btrfs_set_token_inode_uid(leaf, item, i_uid_read(inode), &token);
3361− btrfs_set_token_inode_gid(leaf, item, i_gid_read(inode), &token);
3362− btrfs_set_token_inode_size(leaf, item, BTRFS_I(inode)->disk_i_size,
3363− &token);
3364− btrfs_set_token_inode_mode(leaf, item, inode->i_mode, &token);
3365− btrfs_set_token_inode_nlink(leaf, item, inode->i_nlink, &token);
3366−
3367− btrfs_set_token_timespec_sec(leaf, &item->atime,
3368− inode->i_atime.tv_sec, &token);
3369− btrfs_set_token_timespec_nsec(leaf, &item->atime,
3370− inode->i_atime.tv_nsec, &token);
3371−
3372− btrfs_set_token_timespec_sec(leaf, &item->mtime,
3373− inode->i_mtime.tv_sec, &token);
3374− btrfs_set_token_timespec_nsec(leaf, &item->mtime,
3375− inode->i_mtime.tv_nsec, &token);
3376−
3377− btrfs_set_token_timespec_sec(leaf, &item->ctime,
3378− inode->i_ctime.tv_sec, &token);
3379− btrfs_set_token_timespec_nsec(leaf, &item->ctime,
3380− inode->i_ctime.tv_nsec, &token);
3381−
3382− btrfs_set_token_timespec_sec(leaf, &item->otime,
3383− BTRFS_I(inode)->i_otime.tv_sec, &token);
3384− btrfs_set_token_timespec_nsec(leaf, &item->otime,
3385− BTRFS_I(inode)->i_otime.tv_nsec, &token);
3386−
3387− btrfs_set_token_inode_nbytes(leaf, item, inode_get_bytes(inode),
3388− &token);
3389− btrfs_set_token_inode_generation(leaf, item, BTRFS_I(inode)->generation,
3390− &token);
3391− btrfs_set_token_inode_sequence(leaf, item, inode_peek_iversion(inode),
3392− &token);
3393− btrfs_set_token_inode_transid(leaf, item, trans->transid, &token);
3394− btrfs_set_token_inode_rdev(leaf, item, inode->i_rdev, &token);
3395− btrfs_set_token_inode_flags(leaf, item, BTRFS_I(inode)->flags, &token);
3396− btrfs_set_token_inode_block_group(leaf, item, 0, &token);
3415+ btrfs_set_token_inode_uid(&token, item, i_uid_read(inode));
3416+ btrfs_set_token_inode_gid(&token, item, i_gid_read(inode));
3417+ btrfs_set_token_inode_size(&token, item, BTRFS_I(inode)->disk_i_size);
3418+ btrfs_set_token_inode_mode(&token, item, inode->i_mode);
3419+ btrfs_set_token_inode_nlink(&token, item, inode->i_nlink);
3420+
3421+ btrfs_set_token_timespec_sec(&token, &item->atime,
3422+ inode->i_atime.tv_sec);
3423+ btrfs_set_token_timespec_nsec(&token, &item->atime,
3424+ inode->i_atime.tv_nsec);
3425+
3426+ btrfs_set_token_timespec_sec(&token, &item->mtime,
3427+ inode->i_mtime.tv_sec);
3428+ btrfs_set_token_timespec_nsec(&token, &item->mtime,
3429+ inode->i_mtime.tv_nsec);
3430+
3431+ btrfs_set_token_timespec_sec(&token, &item->ctime,
3432+ inode->i_ctime.tv_sec);
3433+ btrfs_set_token_timespec_nsec(&token, &item->ctime,
3434+ inode->i_ctime.tv_nsec);
3435+
3436+ btrfs_set_token_timespec_sec(&token, &item->otime,
3437+ BTRFS_I(inode)->i_otime.tv_sec);
3438+ btrfs_set_token_timespec_nsec(&token, &item->otime,
3439+ BTRFS_I(inode)->i_otime.tv_nsec);
3440+
3441+ btrfs_set_token_inode_nbytes(&token, item, inode_get_bytes(inode));
3442+ btrfs_set_token_inode_generation(&token, item,
3443+ BTRFS_I(inode)->generation);
3444+ btrfs_set_token_inode_sequence(&token, item, inode_peek_iversion(inode));
3445+ btrfs_set_token_inode_transid(&token, item, trans->transid);
3446+ btrfs_set_token_inode_rdev(&token, item, inode->i_rdev);
3447+ btrfs_set_token_inode_flags(&token, item, BTRFS_I(inode)->flags);
3448+ btrfs_set_token_inode_block_group(&token, item, 0);
33973449 }
33983450
33993451 /*
@@ -3618,7 +3670,7 @@ static struct btrfs_trans_handle *__unlink_start_trans(struct inode *dir) · 218 unmodified lines
36183670 * 1 for the inode ref
36193671 * 1 for the inode
36203672 */
3621− return btrfs_start_transaction_fallback_global_rsv(root, 5, 5);
3673+ return btrfs_start_transaction_fallback_global_rsv(root, 5);
36223674 }
36233675
36243676 static int btrfs_unlink(struct inode *dir, struct dentry *dentry)
@@ -4108,11 +4160,12 @@ int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans, · 483 unmodified lines
41084160 BUG_ON(new_size > 0 && min_type != BTRFS_EXTENT_DATA_KEY);
41094161
41104162 /*
4111− * for non-free space inodes and ref cows, we want to back off from
4112− * time to time
4163+ * For non-free space inodes and non-shareable roots, we want to back
4164+ * off from time to time. This means all inodes in subvolume roots,
4165+ * reloc roots, and data reloc roots.
41134166 */
41144167 if (!btrfs_is_free_space_inode(BTRFS_I(inode)) &&
4115− test_bit(BTRFS_ROOT_REF_COWS, &root->state))
4168+ test_bit(BTRFS_ROOT_SHAREABLE, &root->state))
41164169 be_nice = true;
41174170
41184171 path = btrfs_alloc_path();
@@ -4120,20 +4173,19 @@ int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans, · 1 unmodified line
41204173 return -ENOMEM;
41214174 path->reada = READA_BACK;
41224175
4123− if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID)
4176+ if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
41244177 lock_extent_bits(&BTRFS_I(inode)->io_tree, lock_start, (u64)-1,
41254178 &cached_state);
41264179
4127− /*
4128− * We want to drop from the next block forward in case this new size is
4129− * not block aligned since we will be keeping the last block of the
4130− * extent just the way it is.
4131− */
4132− if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
4133− root == fs_info->tree_root)
4180+ /*
4181+ * We want to drop from the next block forward in case this
4182+ * new size is not block aligned since we will be keeping the
4183+ * last block of the extent just the way it is.
4184+ */
41344185 btrfs_drop_extent_cache(BTRFS_I(inode), ALIGN(new_size,
41354186 fs_info->sectorsize),
41364187 (u64)-1, 0);
4188+ }
41374189
41384190 /*
41394191 * This function is also used to drop the items in the log tree before
@@ -4241,7 +4293,7 @@ search_again: · 101 unmodified lines
42414293 extent_num_bytes);
42424294 num_dec = (orig_num_bytes -
42434295 extent_num_bytes);
4244− if (test_bit(BTRFS_ROOT_REF_COWS,
4296+ if (test_bit(BTRFS_ROOT_SHAREABLE,
42454297 &root->state) &&
42464298 extent_start != 0)
42474299 inode_sub_bytes(inode, num_dec);
@@ -4257,7 +4309,7 @@ search_again: · 9 unmodified lines
42574309 num_dec = btrfs_file_extent_num_bytes(leaf, fi);
42584310 if (extent_start != 0) {
42594311 found_extent = 1;
4260− if (test_bit(BTRFS_ROOT_REF_COWS,
4312+ if (test_bit(BTRFS_ROOT_SHAREABLE,
42614313 &root->state))
42624314 inode_sub_bytes(inode, num_dec);
42634315 }
@@ -4293,7 +4345,7 @@ search_again: · 29 unmodified lines
42934345 clear_len = fs_info->sectorsize;
42944346 }
42954347
4296− if (test_bit(BTRFS_ROOT_REF_COWS, &root->state))
4348+ if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state))
42974349 inode_sub_bytes(inode, item_end + 1 - new_size);
42984350 }
42994351 delete:
@@ -4334,8 +4386,7 @@ delete: · 34 unmodified lines
43344386 should_throttle = false;
43354387
43364388 if (found_extent &&
4337− (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
4338− root == fs_info->tree_root)) {
4389+ root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
43394390 struct btrfs_ref ref = { 0 };
43404391
43414392 bytes_deleted += extent_num_bytes;
@@ -4759,10 +4810,7 @@ static int btrfs_setsize(struct inode *inode, struct iattr *attr) · 417 unmodified lines
47594810
47604811 truncate_setsize(inode, newsize);
47614812
4762− /* Disable nonlocked read DIO to avoid the endless truncate */
4763− btrfs_inode_block_unlocked_dio(BTRFS_I(inode));
47644813 inode_dio_wait(inode);
4765− btrfs_inode_resume_unlocked_dio(BTRFS_I(inode));
47664814
47674815 ret = btrfs_truncate(inode, newsize == oldsize);
47684816 if (ret && inode->i_nlink) {
@@ -5154,7 +5202,7 @@ static int fixup_tree_root_location(struct btrfs_fs_info *fs_info, · 385 unmodified lines
51545202
51555203 btrfs_release_path(path);
51565204
5157− new_root = btrfs_get_fs_root(fs_info, location, true);
5205+ new_root = btrfs_get_fs_root(fs_info, location->objectid, true);
51585206 if (IS_ERR(new_root)) {
51595207 err = PTR_ERR(new_root);
51605208 goto out;
@@ -5232,9 +5280,11 @@ static void inode_tree_del(struct inode *inode) · 71 unmodified lines
52325280 static int btrfs_init_locked_inode(struct inode *inode, void *p)
52335281 {
52345282 struct btrfs_iget_args *args = p;
5235− inode->i_ino = args->location->objectid;
5236− memcpy(&BTRFS_I(inode)->location, args->location,
5237− sizeof(*args->location));
5283+
5284+ inode->i_ino = args->ino;
5285+ BTRFS_I(inode)->location.objectid = args->ino;
5286+ BTRFS_I(inode)->location.type = BTRFS_INODE_ITEM_KEY;
5287+ BTRFS_I(inode)->location.offset = 0;
52385288 BTRFS_I(inode)->root = btrfs_grab_root(args->root);
52395289 BUG_ON(args->root && !BTRFS_I(inode)->root);
52405290 return 0;
@@ -5243,19 +5293,19 @@ static int btrfs_init_locked_inode(struct inode *inode, void *p) · 2 unmodified lines
52435293 static int btrfs_find_actor(struct inode *inode, void *opaque)
52445294 {
52455295 struct btrfs_iget_args *args = opaque;
5246− return args->location->objectid == BTRFS_I(inode)->location.objectid &&
5296+
5297+ return args->ino == BTRFS_I(inode)->location.objectid &&
52475298 args->root == BTRFS_I(inode)->root;
52485299 }
52495300
5250−static struct inode *btrfs_iget_locked(struct super_block *s,
5251− struct btrfs_key *location,
5301+static struct inode *btrfs_iget_locked(struct super_block *s, u64 ino,
52525302 struct btrfs_root *root)
52535303 {
52545304 struct inode *inode;
52555305 struct btrfs_iget_args args;
5256− unsigned long hashval = btrfs_inode_hash(location->objectid, root);
5306+ unsigned long hashval = btrfs_inode_hash(ino, root);
52575307
5258− args.location = location;
5308+ args.ino = ino;
52595309 args.root = root;
52605310
52615311 inode = iget5_locked(s, hashval, btrfs_find_actor,
@@ -5265,17 +5315,17 @@ static struct inode *btrfs_iget_locked(struct super_block *s, · 3 unmodified lines
52655315 }
52665316
52675317 /*
5268− * Get an inode object given its location and corresponding root.
5318+ * Get an inode object given its inode number and corresponding root.
52695319 * Path can be preallocated to prevent recursing back to iget through
52705320 * allocator. NULL is also valid but may require an additional allocation
52715321 * later.
52725322 */
5273−struct inode *btrfs_iget_path(struct super_block *s, struct btrfs_key *location,
5323+struct inode *btrfs_iget_path(struct super_block *s, u64 ino,
52745324 struct btrfs_root *root, struct btrfs_path *path)
52755325 {
52765326 struct inode *inode;
52775327
5278− inode = btrfs_iget_locked(s, location, root);
5328+ inode = btrfs_iget_locked(s, ino, root);
52795329 if (!inode)
52805330 return ERR_PTR(-ENOMEM);
52815331
@@ -5302,10 +5352,9 @@ struct inode *btrfs_iget_path(struct super_block *s, struct btrfs_key *location, · 20 unmodified lines
53025352 return inode;
53035353 }
53045354
5305−struct inode *btrfs_iget(struct super_block *s, struct btrfs_key *location,
5306− struct btrfs_root *root)
5355+struct inode *btrfs_iget(struct super_block *s, u64 ino, struct btrfs_root *root)
53075356 {
5308− return btrfs_iget_path(s, location, root, NULL);
5357+ return btrfs_iget_path(s, ino, root, NULL);
53095358 }
53105359
53115360 static struct inode *new_simple_dir(struct super_block *s,
@@ -5374,7 +5423,7 @@ struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry) · 62 unmodified lines
53745423 return ERR_PTR(ret);
53755424
53765425 if (location.type == BTRFS_INODE_ITEM_KEY) {
5377− inode = btrfs_iget(dir->i_sb, &location, root);
5426+ inode = btrfs_iget(dir->i_sb, location.objectid, root);
53785427 if (IS_ERR(inode))
53795428 return inode;
53805429
@@ -5398,7 +5447,7 @@ struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry) · 17 unmodified lines
53985447 else
53995448 inode = new_simple_dir(dir->i_sb, &location, sub_root);
54005449 } else {
5401− inode = btrfs_iget(dir->i_sb, &location, sub_root);
5450+ inode = btrfs_iget(dir->i_sb, location.objectid, sub_root);
54025451 }
54035452 if (root != sub_root)
54045453 btrfs_put_root(sub_root);
@@ -5779,7 +5828,8 @@ int btrfs_set_inode_index(struct btrfs_inode *dir, u64 *index) · 374 unmodified lines
57795828 static int btrfs_insert_inode_locked(struct inode *inode)
57805829 {
57815830 struct btrfs_iget_args args;
5782− args.location = &BTRFS_I(inode)->location;
5831+
5832+ args.ino = BTRFS_I(inode)->location.objectid;
57835833 args.root = BTRFS_I(inode)->root;
57845834
57855835 return insert_inode_locked4(inode,
@@ -6991,7 +7041,7 @@ out: · 1205 unmodified lines
69917041 }
69927042
69937043 static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend,
6994− struct extent_state **cached_state, int writing)
7044+ struct extent_state **cached_state, bool writing)
69957045 {
69967046 struct btrfs_ordered_extent *ordered;
69977047 int ret = 0;
@@ -7129,30 +7179,7 @@ static struct extent_map *create_io_em(struct inode *inode, u64 start, u64 len, · 131 unmodified lines
71297179 }
71307180
71317181
7132−static int btrfs_get_blocks_direct_read(struct extent_map *em,
7133− struct buffer_head *bh_result,
7134− struct inode *inode,
7135− u64 start, u64 len)
7136−{
7137− struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7138−
7139− if (em->block_start == EXTENT_MAP_HOLE ||
7140− test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
7141− return -ENOENT;
7142−
7143− len = min(len, em->len - (start - em->start));
7144−
7145− bh_result->b_blocknr = (em->block_start + (start - em->start)) >>
7146− inode->i_blkbits;
7147− bh_result->b_size = len;
7148− bh_result->b_bdev = fs_info->fs_devices->latest_bdev;
7149− set_buffer_mapped(bh_result);
7150−
7151− return 0;
7152−}
7153−
71547182 static int btrfs_get_blocks_direct_write(struct extent_map **map,
7155− struct buffer_head *bh_result,
71567183 struct inode *inode,
71577184 struct btrfs_dio_data *dio_data,
71587185 u64 start, u64 len)
@@ -7214,7 +7241,6 @@ static int btrfs_get_blocks_direct_write(struct extent_map **map, · 55 unmodified lines
72147241 }
72157242
72167243 /* this will cow the extent */
7217− len = bh_result->b_size;
72187244 free_extent_map(em);
72197245 *map = em = btrfs_new_extent_direct(inode, start, len);
72207246 if (IS_ERR(em)) {
@@ -7225,64 +7251,73 @@ static int btrfs_get_blocks_direct_write(struct extent_map **map, · 4 unmodified lines
72257251 len = min(len, em->len - (start - em->start));
72267252
72277253 skip_cow:
7228− bh_result->b_blocknr = (em->block_start + (start - em->start)) >>
7229− inode->i_blkbits;
7230− bh_result->b_size = len;
7231− bh_result->b_bdev = fs_info->fs_devices->latest_bdev;
7232− set_buffer_mapped(bh_result);
7233−
7234− if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
7235− set_buffer_new(bh_result);
7236−
72377254 /*
72387255 * Need to update the i_size under the extent lock so buffered
72397256 * readers will get the updated i_size when we unlock.
72407257 */
7241− if (!dio_data->overwrite && start + len > i_size_read(inode))
7258+ if (start + len > i_size_read(inode))
72427259 i_size_write(inode, start + len);
72437260
7244− WARN_ON(dio_data->reserve < len);
72457261 dio_data->reserve -= len;
7246− dio_data->unsubmitted_oe_range_end = start + len;
7247− current->journal_info = dio_data;
72487262 out:
72497263 return ret;
72507264 }
72517265
7252−static int btrfs_get_blocks_direct(struct inode *inode, sector_t iblock,
7253− struct buffer_head *bh_result, int create)
7266+static int btrfs_dio_iomap_begin(struct inode *inode, loff_t start,
7267+ loff_t length, unsigned flags, struct iomap *iomap,
7268+ struct iomap *srcmap)
72547269 {
72557270 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
72567271 struct extent_map *em;
72577272 struct extent_state *cached_state = NULL;
72587273 struct btrfs_dio_data *dio_data = NULL;
7259− u64 start = iblock << inode->i_blkbits;
72607274 u64 lockstart, lockend;
7261− u64 len = bh_result->b_size;
7275+ const bool write = !!(flags & IOMAP_WRITE);
72627276 int ret = 0;
7277+ u64 len = length;
7278+ bool unlock_extents = false;
72637279
7264− if (!create)
7280+ if (!write)
72657281 len = min_t(u64, len, fs_info->sectorsize);
72667282
72677283 lockstart = start;
72687284 lockend = start + len - 1;
72697285
7270− if (current->journal_info) {
7271− /*
7272− * Need to pull our outstanding extents and set journal_info to NULL so
7273− * that anything that needs to check if there's a transaction doesn't get
7274− * confused.
7275− */
7276− dio_data = current->journal_info;
7277− current->journal_info = NULL;
7286+ /*
7287+ * The generic stuff only does filemap_write_and_wait_range, which
7288+ * isn't enough if we've written compressed pages to this area, so we
7289+ * need to flush the dirty pages again to make absolutely sure that any
7290+ * outstanding dirty pages are on disk.
7291+ */
7292+ if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
7293+ &BTRFS_I(inode)->runtime_flags))
7294+ ret = filemap_fdatawrite_range(inode->i_mapping, start,
7295+ start + length - 1);
7296+
7297+ dio_data = kzalloc(sizeof(*dio_data), GFP_NOFS);
7298+ if (!dio_data)
7299+ return -ENOMEM;
7300+
7301+ dio_data->length = length;
7302+ if (write) {
7303+ dio_data->reserve = round_up(length, fs_info->sectorsize);
7304+ ret = btrfs_delalloc_reserve_space(inode,
7305+ &dio_data->data_reserved,
7306+ start, dio_data->reserve);
7307+ if (ret) {
7308+ extent_changeset_free(dio_data->data_reserved);
7309+ kfree(dio_data);
7310+ return ret;
7311+ }
72787312 }
7313+ iomap->private = dio_data;
7314+
72797315
72807316 /*
72817317 * If this errors out it's because we couldn't invalidate pagecache for
72827318 * this range and we need to fallback to buffered.
72837319 */
7284− if (lock_extent_direct(inode, lockstart, lockend, &cached_state,
7285− create)) {
7320+ if (lock_extent_direct(inode, lockstart, lockend, &cached_state, write)) {
72867321 ret = -ENOTBLK;
72877322 goto err;
72887323 }
@@ -7314,36 +7349,48 @@ static int btrfs_get_blocks_direct(struct inode *inode, sector_t iblock, · 25 unmodified lines
73147349 goto unlock_err;
73157350 }
73167351
7317− if (create) {
7318− ret = btrfs_get_blocks_direct_write(&em, bh_result, inode,
7319− dio_data, start, len);
7352+ len = min(len, em->len - (start - em->start));
7353+ if (write) {
7354+ ret = btrfs_get_blocks_direct_write(&em, inode, dio_data,
7355+ start, len);
73207356 if (ret < 0)
73217357 goto unlock_err;
7322−
7323− unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart,
7324− lockend, &cached_state);
7358+ unlock_extents = true;
7359+ /* Recalc len in case the new em is smaller than requested */
7360+ len = min(len, em->len - (start - em->start));
73257361 } else {
7326− ret = btrfs_get_blocks_direct_read(em, bh_result, inode,
7327− start, len);
7328− /* Can be negative only if we read from a hole */
7329− if (ret < 0) {
7330− ret = 0;
7331− free_extent_map(em);
7332− goto unlock_err;
7333− }
73347362 /*
73357363 * We need to unlock only the end area that we aren't using.
73367364 * The rest is going to be unlocked by the endio routine.
73377365 */
7338− lockstart = start + bh_result->b_size;
7339− if (lockstart < lockend) {
7340− unlock_extent_cached(&BTRFS_I(inode)->io_tree,
7341− lockstart, lockend, &cached_state);
7342− } else {
7343− free_extent_state(cached_state);
7344− }
7366+ lockstart = start + len;
7367+ if (lockstart < lockend)
7368+ unlock_extents = true;
73457369 }
73467370
7371+ if (unlock_extents)
7372+ unlock_extent_cached(&BTRFS_I(inode)->io_tree,
7373+ lockstart, lockend, &cached_state);
7374+ else
7375+ free_extent_state(cached_state);
7376+
7377+ /*
7378+ * Translate extent map information to iomap.
7379+ * We trim the extents (and move the addr) even though iomap code does
7380+ * that, since we have locked only the parts we are performing I/O in.
7381+ */
7382+ if ((em->block_start == EXTENT_MAP_HOLE) ||
7383+ (test_bit(EXTENT_FLAG_PREALLOC, &em->flags) && !write)) {
7384+ iomap->addr = IOMAP_NULL_ADDR;
7385+ iomap->type = IOMAP_HOLE;
7386+ } else {
7387+ iomap->addr = em->block_start + (start - em->start);
7388+ iomap->type = IOMAP_MAPPED;
7389+ }
7390+ iomap->offset = start;
7391+ iomap->bdev = fs_info->fs_devices->latest_bdev;
7392+ iomap->length = len;
7393+
73477394 free_extent_map(em);
73487395
73497396 return 0;
@@ -7352,370 +7399,152 @@ unlock_err: · 2 unmodified lines
73527399 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
73537400 &cached_state);
73547401 err:
7355− if (dio_data)
7356− current->journal_info = dio_data;
7402+ if (dio_data) {
7403+ btrfs_delalloc_release_space(inode, dio_data->data_reserved,
7404+ start, dio_data->reserve, true);
7405+ btrfs_delalloc_release_extents(BTRFS_I(inode), dio_data->reserve);
7406+ extent_changeset_free(dio_data->data_reserved);
7407+ kfree(dio_data);
7408+ }
73577409 return ret;
73587410 }
73597411
7360−static inline blk_status_t submit_dio_repair_bio(struct inode *inode,
7361− struct bio *bio,
7362− int mirror_num)
7412+static int btrfs_dio_iomap_end(struct inode *inode, loff_t pos, loff_t length,
7413+ ssize_t written, unsigned flags, struct iomap *iomap)
73637414 {
7364− struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7365− blk_status_t ret;
7415+ int ret = 0;
7416+ struct btrfs_dio_data *dio_data = iomap->private;
7417+ size_t submitted = dio_data->submitted;
7418+ const bool write = !!(flags & IOMAP_WRITE);
73667419
7367− BUG_ON(bio_op(bio) == REQ_OP_WRITE);
7420+ if (!write && (iomap->type == IOMAP_HOLE)) {
7421+ /* If reading from a hole, unlock and return */
7422+ unlock_extent(&BTRFS_I(inode)->io_tree, pos, pos + length - 1);
7423+ goto out;
7424+ }
73687425
7369− ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DIO_REPAIR);
7370− if (ret)
7371− return ret;
7426+ if (submitted < length) {
7427+ pos += submitted;
7428+ length -= submitted;
7429+ if (write)
7430+ __endio_write_update_ordered(inode, pos, length, false);
7431+ else
7432+ unlock_extent(&BTRFS_I(inode)->io_tree, pos,
7433+ pos + length - 1);
7434+ ret = -ENOTBLK;
7435+ }
73727436
7373− ret = btrfs_map_bio(fs_info, bio, mirror_num);
7437+ if (write) {
7438+ if (dio_data->reserve)
7439+ btrfs_delalloc_release_space(inode,
7440+ dio_data->data_reserved, pos,
7441+ dio_data->reserve, true);
7442+ btrfs_delalloc_release_extents(BTRFS_I(inode), dio_data->length);
7443+ extent_changeset_free(dio_data->data_reserved);
7444+ }
7445+out:
7446+ kfree(dio_data);
7447+ iomap->private = NULL;
73747448
73757449 return ret;
73767450 }
73777451
7378−static int btrfs_check_dio_repairable(struct inode *inode,
7379− struct bio *failed_bio,
7380− struct io_failure_record *failrec,
7381− int failed_mirror)
7452+static void btrfs_dio_private_put(struct btrfs_dio_private *dip)
73827453 {
7383− struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7384− int num_copies;
7385−
7386− num_copies = btrfs_num_copies(fs_info, failrec->logical, failrec->len);
7387− if (num_copies == 1) {
7388− /*
7389− * we only have a single copy of the data, so don't bother with
7390− * all the retry and error correction code that follows. no
7391− * matter what the error is, it is very likely to persist.
7392− */
7393− btrfs_debug(fs_info,
7394− "Check DIO Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d",
7395− num_copies, failrec->this_mirror, failed_mirror);
7396− return 0;
7397− }
7398−
7399− failrec->failed_mirror = failed_mirror;
7400− failrec->this_mirror++;
7401− if (failrec->this_mirror == failed_mirror)
7402− failrec->this_mirror++;
7454+ /*
7455+ * This implies a barrier so that stores to dio_bio->bi_status before
7456+ * this and loads of dio_bio->bi_status after this are fully ordered.
7457+ */
7458+ if (!refcount_dec_and_test(&dip->refs))
7459+ return;
74037460
7404− if (failrec->this_mirror > num_copies) {
7405− btrfs_debug(fs_info,
7406− "Check DIO Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d",
7407− num_copies, failrec->this_mirror, failed_mirror);
7408− return 0;
7461+ if (bio_op(dip->dio_bio) == REQ_OP_WRITE) {
7462+ __endio_write_update_ordered(dip->inode, dip->logical_offset,
7463+ dip->bytes,
7464+ !dip->dio_bio->bi_status);
7465+ } else {
7466+ unlock_extent(&BTRFS_I(dip->inode)->io_tree,
7467+ dip->logical_offset,
7468+ dip->logical_offset + dip->bytes - 1);
74097469 }
74107470
7411− return 1;
7471+ bio_endio(dip->dio_bio);
7472+ kfree(dip);
74127473 }
74137474
7414−static blk_status_t dio_read_error(struct inode *inode, struct bio *failed_bio,
7415− struct page *page, unsigned int pgoff,
7416− u64 start, u64 end, int failed_mirror,
7417− bio_end_io_t *repair_endio, void *repair_arg)
7475+static blk_status_t submit_dio_repair_bio(struct inode *inode, struct bio *bio,
7476+ int mirror_num,
7477+ unsigned long bio_flags)
74187478 {
7419− struct io_failure_record *failrec;
7420− struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
7421− struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
7422− struct bio *bio;
7423− int isector;
7424− unsigned int read_mode = 0;
7425− int segs;
7426− int ret;
7427− blk_status_t status;
7428− struct bio_vec bvec;
7479+ struct btrfs_dio_private *dip = bio->bi_private;
7480+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7481+ blk_status_t ret;
74297482
7430− BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);
7483+ BUG_ON(bio_op(bio) == REQ_OP_WRITE);
74317484
7432− ret = btrfs_get_io_failure_record(inode, start, end, &failrec);
7485+ ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DATA);
74337486 if (ret)
7434− return errno_to_blk_status(ret);
7435−
7436− ret = btrfs_check_dio_repairable(inode, failed_bio, failrec,
7437− failed_mirror);
7438− if (!ret) {
7439− free_io_failure(failure_tree, io_tree, failrec);
7440− return BLK_STS_IOERR;
7441− }
7442−
7443− segs = bio_segments(failed_bio);
7444− bio_get_first_bvec(failed_bio, &bvec);
7445− if (segs > 1 ||
7446− (bvec.bv_len > btrfs_inode_sectorsize(inode)))
7447− read_mode |= REQ_FAILFAST_DEV;
7448−
7449− isector = start - btrfs_io_bio(failed_bio)->logical;
7450− isector >>= inode->i_sb->s_blocksize_bits;
7451− bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,
7452− pgoff, isector, repair_endio, repair_arg);
7453− bio->bi_opf = REQ_OP_READ | read_mode;
7454−
7455− btrfs_debug(BTRFS_I(inode)->root->fs_info,
7456− "repair DIO read error: submitting new dio read[%#x] to this_mirror=%d, in_validation=%d",
7457− read_mode, failrec->this_mirror, failrec->in_validation);
7458−
7459− status = submit_dio_repair_bio(inode, bio, failrec->this_mirror);
7460− if (status) {
7461− free_io_failure(failure_tree, io_tree, failrec);
7462− bio_put(bio);
7463− }
7464−
7465− return status;
7466−}
7467−
7468−struct btrfs_retry_complete {
7469− struct completion done;
7470− struct inode *inode;
7471− u64 start;
7472− int uptodate;
7473−};
7487+ return ret;
74747488
7475−static void btrfs_retry_endio_nocsum(struct bio *bio)
7476−{
7477− struct btrfs_retry_complete *done = bio->bi_private;
7478− struct inode *inode = done->inode;
7479− struct bio_vec *bvec;
7480− struct extent_io_tree *io_tree, *failure_tree;
7481− struct bvec_iter_all iter_all;
7482−
7483− if (bio->bi_status)
7484− goto end;
7485−
7486− ASSERT(bio->bi_vcnt == 1);
7487− io_tree = &BTRFS_I(inode)->io_tree;
7488− failure_tree = &BTRFS_I(inode)->io_failure_tree;
7489− ASSERT(bio_first_bvec_all(bio)->bv_len == btrfs_inode_sectorsize(inode));
7490−
7491− done->uptodate = 1;
7492− ASSERT(!bio_flagged(bio, BIO_CLONED));
7493− bio_for_each_segment_all(bvec, bio, iter_all)
7494− clean_io_failure(BTRFS_I(inode)->root->fs_info, failure_tree,
7495− io_tree, done->start, bvec->bv_page,
7496− btrfs_ino(BTRFS_I(inode)), 0);
7497−end:
7498− complete(&done->done);
7499− bio_put(bio);
7489+ refcount_inc(&dip->refs);
7490+ ret = btrfs_map_bio(fs_info, bio, mirror_num);
7491+ if (ret)
7492+ refcount_dec(&dip->refs);
7493+ return ret;
75007494 }
75017495
7502−static blk_status_t __btrfs_correct_data_nocsum(struct inode *inode,
7503− struct btrfs_io_bio *io_bio)
7496+static blk_status_t btrfs_check_read_dio_bio(struct inode *inode,
7497+ struct btrfs_io_bio *io_bio,
7498+ const bool uptodate)
75047499 {
7505− struct btrfs_fs_info *fs_info;
7500+ struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
7501+ const u32 sectorsize = fs_info->sectorsize;
7502+ struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
7503+ struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
7504+ const bool csum = !(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM);
75067505 struct bio_vec bvec;
75077506 struct bvec_iter iter;
7508− struct btrfs_retry_complete done;
7509− u64 start;
7510− unsigned int pgoff;
7511− u32 sectorsize;
7512− int nr_sectors;
7513− blk_status_t ret;
7507+ u64 start = io_bio->logical;
7508+ int icsum = 0;
75147509 blk_status_t err = BLK_STS_OK;
75157510
7516− fs_info = BTRFS_I(inode)->root->fs_info;
7517− sectorsize = fs_info->sectorsize;
7518−
7519− start = io_bio->logical;
7520− done.inode = inode;
7521− io_bio->bio.bi_iter = io_bio->iter;
7511+ __bio_for_each_segment(bvec, &io_bio->bio, iter, io_bio->iter) {
7512+ unsigned int i, nr_sectors, pgoff;
75227513
7523− bio_for_each_segment(bvec, &io_bio->bio, iter) {
75247514 nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec.bv_len);
75257515 pgoff = bvec.bv_offset;
7526−
7527−next_block_or_try_again:
7528− done.uptodate = 0;
7529− done.start = start;
7530− init_completion(&done.done);
7531−
7532− ret = dio_read_error(inode, &io_bio->bio, bvec.bv_page,
7533− pgoff, start, start + sectorsize - 1,
7534− io_bio->mirror_num,
7535− btrfs_retry_endio_nocsum, &done);
7536− if (ret) {
7537− err = ret;
7538− goto next;
7539− }
7540−
7541− wait_for_completion_io(&done.done);
7542−
7543− if (!done.uptodate) {
7544− /* We might have another mirror, so try again */
7545− goto next_block_or_try_again;
7546− }
7547−
7548−next:
7549− start += sectorsize;
7550−
7551− nr_sectors--;
7552− if (nr_sectors) {
7553− pgoff += sectorsize;
7516+ for (i = 0; i < nr_sectors; i++) {
75547517 ASSERT(pgoff < PAGE_SIZE);
7555− goto next_block_or_try_again;
7556− }
7557− }
7558−
7559− return err;
7560−}
7561−
7562−static void btrfs_retry_endio(struct bio *bio)
7563−{
7564− struct btrfs_retry_complete *done = bio->bi_private;
7565− struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
7566− struct extent_io_tree *io_tree, *failure_tree;
7567− struct inode *inode = done->inode;
7568− struct bio_vec *bvec;
7569− int uptodate;
7570− int ret;
7571− int i = 0;
7572− struct bvec_iter_all iter_all;
7573−
7574− if (bio->bi_status)
7575− goto end;
7576−
7577− uptodate = 1;
7578−
7579− ASSERT(bio->bi_vcnt == 1);
7580− ASSERT(bio_first_bvec_all(bio)->bv_len == btrfs_inode_sectorsize(done->inode));
7581−
7582− io_tree = &BTRFS_I(inode)->io_tree;
7583− failure_tree = &BTRFS_I(inode)->io_failure_tree;
7584−
7585− ASSERT(!bio_flagged(bio, BIO_CLONED));
7586− bio_for_each_segment_all(bvec, bio, iter_all) {
7587− ret = __readpage_endio_check(inode, io_bio, i, bvec->bv_page,
7588− bvec->bv_offset, done->start,
7589− bvec->bv_len);
7590− if (!ret)
7591− clean_io_failure(BTRFS_I(inode)->root->fs_info,
7592− failure_tree, io_tree, done->start,
7593− bvec->bv_page,
7594− btrfs_ino(BTRFS_I(inode)),
7595− bvec->bv_offset);
7596− else
7597− uptodate = 0;
7598− i++;
7599− }
7600−
7601− done->uptodate = uptodate;
7602−end:
7603− complete(&done->done);
7604− bio_put(bio);
7605−}
7606−
7607−static blk_status_t __btrfs_subio_endio_read(struct inode *inode,
7608− struct btrfs_io_bio *io_bio, blk_status_t err)
7609−{
7610− struct btrfs_fs_info *fs_info;
7611− struct bio_vec bvec;
7612− struct bvec_iter iter;
7613− struct btrfs_retry_complete done;
7614− u64 start;
7615− u64 offset = 0;
7616− u32 sectorsize;
7617− int nr_sectors;
7618− unsigned int pgoff;
7619− int csum_pos;
7620− bool uptodate = (err == 0);
7621− int ret;
7622− blk_status_t status;
7623−
7624− fs_info = BTRFS_I(inode)->root->fs_info;
7625− sectorsize = fs_info->sectorsize;
7626−
7627− err = BLK_STS_OK;
7628− start = io_bio->logical;
7629− done.inode = inode;
7630− io_bio->bio.bi_iter = io_bio->iter;
7631−
7632− bio_for_each_segment(bvec, &io_bio->bio, iter) {
7633− nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec.bv_len);
7634−
7635− pgoff = bvec.bv_offset;
7636−next_block:
7637− if (uptodate) {
7638− csum_pos = BTRFS_BYTES_TO_BLKS(fs_info, offset);
7639− ret = __readpage_endio_check(inode, io_bio, csum_pos,
7640− bvec.bv_page, pgoff, start, sectorsize);
7641− if (likely(!ret))
7642− goto next;
7643− }
7644−try_again:
7645− done.uptodate = 0;
7646− done.start = start;
7647− init_completion(&done.done);
7648−
7649− status = dio_read_error(inode, &io_bio->bio, bvec.bv_page,
7650− pgoff, start, start + sectorsize - 1,
7651− io_bio->mirror_num, btrfs_retry_endio,
7652− &done);
7653− if (status) {
7654− err = status;
7655− goto next;
7656− }
7657−
7658− wait_for_completion_io(&done.done);
7659−
7660− if (!done.uptodate) {
7661− /* We might have another mirror, so try again */
7662− goto try_again;
7663− }
7664−next:
7665− offset += sectorsize;
7666− start += sectorsize;
7667−
7668− ASSERT(nr_sectors);
7669−
7670− nr_sectors--;
7671− if (nr_sectors) {
7518+ if (uptodate &&
7519+ (!csum || !check_data_csum(inode, io_bio, icsum,
7520+ bvec.bv_page, pgoff,
7521+ start, sectorsize))) {
7522+ clean_io_failure(fs_info, failure_tree, io_tree,
7523+ start, bvec.bv_page,
7524+ btrfs_ino(BTRFS_I(inode)),
7525+ pgoff);
7526+ } else {
7527+ blk_status_t status;
7528+
7529+ status = btrfs_submit_read_repair(inode,
7530+ &io_bio->bio,
7531+ start - io_bio->logical,
7532+ bvec.bv_page, pgoff,
7533+ start,
7534+ start + sectorsize - 1,
7535+ io_bio->mirror_num,
7536+ submit_dio_repair_bio);
7537+ if (status)
7538+ err = status;
7539+ }
7540+ start += sectorsize;
7541+ icsum++;
76727542 pgoff += sectorsize;
7673− ASSERT(pgoff < PAGE_SIZE);
7674− goto next_block;
76757543 }
76767544 }
7677−
76787545 return err;
76797546 }
76807547
7681−static blk_status_t btrfs_subio_endio_read(struct inode *inode,
7682− struct btrfs_io_bio *io_bio, blk_status_t err)
7683−{
7684− bool skip_csum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;
7685−
7686− if (skip_csum) {
7687− if (unlikely(err))
7688− return __btrfs_correct_data_nocsum(inode, io_bio);
7689− else
7690− return BLK_STS_OK;
7691− } else {
7692− return __btrfs_subio_endio_read(inode, io_bio, err);
7693− }
7694−}
7695−
7696−static void btrfs_endio_direct_read(struct bio *bio)
7697−{
7698− struct btrfs_dio_private *dip = bio->bi_private;
7699− struct inode *inode = dip->inode;
7700− struct bio *dio_bio;
7701− struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
7702− blk_status_t err = bio->bi_status;
7703−
7704− if (dip->flags & BTRFS_DIO_ORIG_BIO_SUBMITTED)
7705− err = btrfs_subio_endio_read(inode, io_bio, err);
7706−
7707− unlock_extent(&BTRFS_I(inode)->io_tree, dip->logical_offset,
7708− dip->logical_offset + dip->bytes - 1);
7709− dio_bio = dip->dio_bio;
7710−
7711− kfree(dip);
7712−
7713− dio_bio->bi_status = err;
7714− dio_end_io(dio_bio);
7715− btrfs_io_bio_free_csum(io_bio);
7716− bio_put(bio);
7717−}
7718−
77197548 static void __endio_write_update_ordered(struct inode *inode,
77207549 const u64 offset, const u64 bytes,
77217550 const bool uptodate)
@@ -7759,21 +7588,6 @@ static void __endio_write_update_ordered(struct inode *inode, · 37 unmodified lines
77597588 }
77607589 }
77617590
7762−static void btrfs_endio_direct_write(struct bio *bio)
7763−{
7764− struct btrfs_dio_private *dip = bio->bi_private;
7765− struct bio *dio_bio = dip->dio_bio;
7766−
7767− __endio_write_update_ordered(dip->inode, dip->logical_offset,
7768− dip->bytes, !bio->bi_status);
7769−
7770− kfree(dip);
7771−
7772− dio_bio->bi_status = bio->bi_status;
7773− dio_end_io(dio_bio);
7774− bio_put(bio);
7775−}
7776−
77777591 static blk_status_t btrfs_submit_bio_start_direct_io(void *private_data,
77787592 struct bio *bio, u64 offset)
77797593 {
@@ -7797,64 +7611,16 @@ static void btrfs_end_dio_bio(struct bio *bio) · 17 unmodified lines
77977611 (unsigned long long)bio->bi_iter.bi_sector,
77987612 bio->bi_iter.bi_size, err);
77997613
7800− if (dip->subio_endio)
7801− err = dip->subio_endio(dip->inode, btrfs_io_bio(bio), err);
7802−
7803− if (err) {
7804− /*
7805− * We want to perceive the errors flag being set before
7806− * decrementing the reference count. We don't need a barrier
7807− * since atomic operations with a return value are fully
7808− * ordered as per atomic_t.txt
7809− */
7810− dip->errors = 1;
7614+ if (bio_op(bio) == REQ_OP_READ) {
7615+ err = btrfs_check_read_dio_bio(dip->inode, btrfs_io_bio(bio),
7616+ !err);
78117617 }
78127618
7813− /* if there are more bios still pending for this dio, just exit */
7814− if (!atomic_dec_and_test(&dip->pending_bios))
7815− goto out;
7619+ if (err)
7620+ dip->dio_bio->bi_status = err;
78167621
7817− if (dip->errors) {
7818− bio_io_error(dip->orig_bio);
7819− } else {
7820− dip->dio_bio->bi_status = BLK_STS_OK;
7821− bio_endio(dip->orig_bio);
7822− }
7823−out:
78247622 bio_put(bio);
7825−}
7826−
7827−static inline blk_status_t btrfs_lookup_and_bind_dio_csum(struct inode *inode,
7828− struct btrfs_dio_private *dip,
7829− struct bio *bio,
7830− u64 file_offset)
7831−{
7832− struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
7833− struct btrfs_io_bio *orig_io_bio = btrfs_io_bio(dip->orig_bio);
7834− u16 csum_size;
7835− blk_status_t ret;
7836−
7837− /*
7838− * We load all the csum data we need when we submit
7839− * the first bio to reduce the csum tree search and
7840− * contention.
7841− */
7842− if (dip->logical_offset == file_offset) {
7843− ret = btrfs_lookup_bio_sums(inode, dip->orig_bio, file_offset,
7844− NULL);
7845− if (ret)
7846− return ret;
7847− }
7848−
7849− if (bio == dip->orig_bio)
7850− return 0;
7851−
7852− file_offset -= dip->logical_offset;
7853− file_offset >>= inode->i_sb->s_blocksize_bits;
7854− csum_size = btrfs_super_csum_size(btrfs_sb(inode->i_sb)->super_copy);
7855− io_bio->csum = orig_io_bio->csum + csum_size * file_offset;
7856−
7857− return 0;
7623+ btrfs_dio_private_put(dip);
78587624 }
78597625
78607626 static inline blk_status_t btrfs_submit_dio_bio(struct bio *bio,
@@ -7892,10 +7658,12 @@ static inline blk_status_t btrfs_submit_dio_bio(struct bio *bio, · 31 unmodified lines
78927658 if (ret)
78937659 goto err;
78947660 } else {
7895− ret = btrfs_lookup_and_bind_dio_csum(inode, dip, bio,
7896− file_offset);
7897− if (ret)
7898− goto err;
7661+ u64 csum_offset;
7662+
7663+ csum_offset = file_offset - dip->logical_offset;
7664+ csum_offset >>= inode->i_sb->s_blocksize_bits;
7665+ csum_offset *= btrfs_super_csum_size(fs_info->super_copy);
7666+ btrfs_io_bio(bio)->csum = dip->csums + csum_offset;
78997667 }
79007668 map:
79017669 ret = btrfs_map_bio(fs_info, bio, 0);
@@ -7903,14 +7671,53 @@ err: · 1 unmodified line
79037671 return ret;
79047672 }
79057673
7906−static int btrfs_submit_direct_hook(struct btrfs_dio_private *dip)
7674+/*
7675+ * If this succeeds, the btrfs_dio_private is responsible for cleaning up locked
7676+ * or ordered extents whether or not we submit any bios.
7677+ */
7678+static struct btrfs_dio_private *btrfs_create_dio_private(struct bio *dio_bio,
7679+ struct inode *inode,
7680+ loff_t file_offset)
79077681 {
7908− struct inode *inode = dip->inode;
7682+ const bool write = (bio_op(dio_bio) == REQ_OP_WRITE);
7683+ const bool csum = !(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM);
7684+ size_t dip_size;
7685+ struct btrfs_dio_private *dip;
7686+
7687+ dip_size = sizeof(*dip);
7688+ if (!write && csum) {
7689+ struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7690+ const u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
7691+ size_t nblocks;
7692+
7693+ nblocks = dio_bio->bi_iter.bi_size >> inode->i_sb->s_blocksize_bits;
7694+ dip_size += csum_size * nblocks;
7695+ }
7696+
7697+ dip = kzalloc(dip_size, GFP_NOFS);
7698+ if (!dip)
7699+ return NULL;
7700+
7701+ dip->inode = inode;
7702+ dip->logical_offset = file_offset;
7703+ dip->bytes = dio_bio->bi_iter.bi_size;
7704+ dip->disk_bytenr = (u64)dio_bio->bi_iter.bi_sector << 9;
7705+ dip->dio_bio = dio_bio;
7706+ refcount_set(&dip->refs, 1);
7707+ return dip;
7708+}
7709+
7710+static blk_qc_t btrfs_submit_direct(struct inode *inode, struct iomap *iomap,
7711+ struct bio *dio_bio, loff_t file_offset)
7712+{
7713+ const bool write = (bio_op(dio_bio) == REQ_OP_WRITE);
7714+ const bool csum = !(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM);
79097715 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7716+ const bool raid56 = (btrfs_data_alloc_profile(fs_info) &
7717+ BTRFS_BLOCK_GROUP_RAID56_MASK);
7718+ struct btrfs_dio_private *dip;
79107719 struct bio *bio;
7911− struct bio *orig_bio = dip->orig_bio;
7912− u64 start_sector = orig_bio->bi_iter.bi_sector;
7913− u64 file_offset = dip->logical_offset;
7720+ u64 start_sector;
79147721 int async_submit = 0;
79157722 u64 submit_len;
79167723 int clone_offset = 0;
@@ -7918,330 +7725,108 @@ static int btrfs_submit_direct_hook(struct btrfs_dio_private *dip) · 1 unmodified line
79187725 int ret;
79197726 blk_status_t status;
79207727 struct btrfs_io_geometry geom;
7728+ struct btrfs_dio_data *dio_data = iomap->private;
79217729
7922− submit_len = orig_bio->bi_iter.bi_size;
7923− ret = btrfs_get_io_geometry(fs_info, btrfs_op(orig_bio),
7924− start_sector << 9, submit_len, &geom);
7925− if (ret)
7926− return -EIO;
7730+ dip = btrfs_create_dio_private(dio_bio, inode, file_offset);
7731+ if (!dip) {
7732+ if (!write) {
7733+ unlock_extent(&BTRFS_I(inode)->io_tree, file_offset,
7734+ file_offset + dio_bio->bi_iter.bi_size - 1);
7735+ }
7736+ dio_bio->bi_status = BLK_STS_RESOURCE;
7737+ bio_endio(dio_bio);
7738+ return BLK_QC_T_NONE;
7739+ }
79277740
7928− if (geom.len >= submit_len) {
7929− bio = orig_bio;
7930− dip->flags |= BTRFS_DIO_ORIG_BIO_SUBMITTED;
7931− goto submit;
7741+ if (!write && csum) {
7742+ /*
7743+ * Load the csums up front to reduce csum tree searches and
7744+ * contention when submitting bios.
7745+ */
7746+ status = btrfs_lookup_bio_sums(inode, dio_bio, file_offset,
7747+ dip->csums);
7748+ if (status != BLK_STS_OK)
7749+ goto out_err;
79327750 }
79337751
7934− /* async crcs make it difficult to collect full stripe writes. */
7935− if (btrfs_data_alloc_profile(fs_info) & BTRFS_BLOCK_GROUP_RAID56_MASK)
7936− async_submit = 0;
7937− else
7938− async_submit = 1;
7752+ start_sector = dio_bio->bi_iter.bi_sector;
7753+ submit_len = dio_bio->bi_iter.bi_size;
79397754
7940− /* bio split */
7941− ASSERT(geom.len <= INT_MAX);
7942− atomic_inc(&dip->pending_bios);
79437755 do {
7756+ ret = btrfs_get_io_geometry(fs_info, btrfs_op(dio_bio),
7757+ start_sector << 9, submit_len,
7758+ &geom);
7759+ if (ret) {
7760+ status = errno_to_blk_status(ret);
7761+ goto out_err;
7762+ }
7763+ ASSERT(geom.len <= INT_MAX);
7764+
79447765 clone_len = min_t(int, submit_len, geom.len);
79457766
79467767 /*
79477768 * This will never fail as it's passing GPF_NOFS and
79487769 * the allocation is backed by btrfs_bioset.
79497770 */
7950− bio = btrfs_bio_clone_partial(orig_bio, clone_offset,
7951− clone_len);
7771+ bio = btrfs_bio_clone_partial(dio_bio, clone_offset, clone_len);
79527772 bio->bi_private = dip;
79537773 bio->bi_end_io = btrfs_end_dio_bio;
79547774 btrfs_io_bio(bio)->logical = file_offset;
79557775
79567776 ASSERT(submit_len >= clone_len);
79577777 submit_len -= clone_len;
7958− if (submit_len == 0)
7959− break;
79607778
79617779 /*
79627780 * Increase the count before we submit the bio so we know
79637781 * the end IO handler won't happen before we increase the
79647782 * count. Otherwise, the dip might get freed before we're
79657783 * done setting it up.
7784+ *
7785+ * We transfer the initial reference to the last bio, so we
7786+ * don't need to increment the reference count for the last one.
79667787 */
7967− atomic_inc(&dip->pending_bios);
7788+ if (submit_len > 0) {
7789+ refcount_inc(&dip->refs);
7790+ /*
7791+ * If we are submitting more than one bio, submit them
7792+ * all asynchronously. The exception is RAID 5 or 6, as
7793+ * asynchronous checksums make it difficult to collect
7794+ * full stripe writes.
7795+ */
7796+ if (!raid56)
7797+ async_submit = 1;
7798+ }
79687799
79697800 status = btrfs_submit_dio_bio(bio, inode, file_offset,
79707801 async_submit);
79717802 if (status) {
79727803 bio_put(bio);
7973− atomic_dec(&dip->pending_bios);
7804+ if (submit_len > 0)
7805+ refcount_dec(&dip->refs);
79747806 goto out_err;
79757807 }
79767808
7809+ dio_data->submitted += clone_len;
79777810 clone_offset += clone_len;
79787811 start_sector += clone_len >> 9;
79797812 file_offset += clone_len;
7980−
7981− ret = btrfs_get_io_geometry(fs_info, btrfs_op(orig_bio),
7982− start_sector << 9, submit_len, &geom);
7983− if (ret)
7984− goto out_err;
79857813 } while (submit_len > 0);
7814+ return BLK_QC_T_NONE;
79867815
7987−submit:
7988− status = btrfs_submit_dio_bio(bio, inode, file_offset, async_submit);
7989− if (!status)
7990− return 0;
7991−
7992− bio_put(bio);
79937816 out_err:
7994− dip->errors = 1;
7995− /*
7996− * Before atomic variable goto zero, we must make sure dip->errors is
7997− * perceived to be set. This ordering is ensured by the fact that an
7998− * atomic operations with a return value are fully ordered as per
7999− * atomic_t.txt
8000− */
8001− if (atomic_dec_and_test(&dip->pending_bios))
8002− bio_io_error(dip->orig_bio);
8003−
8004− /* bio_end_io() will handle error, so we needn't return it */
8005− return 0;
8006−}
8007−
8008−static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode,
8009− loff_t file_offset)
8010−{
8011− struct btrfs_dio_private *dip = NULL;
8012− struct bio *bio = NULL;
8013− struct btrfs_io_bio *io_bio;
8014− bool write = (bio_op(dio_bio) == REQ_OP_WRITE);
8015− int ret = 0;
8016−
8017− bio = btrfs_bio_clone(dio_bio);
8018−
8019− dip = kzalloc(sizeof(*dip), GFP_NOFS);
8020− if (!dip) {
8021− ret = -ENOMEM;
8022− goto free_ordered;
8023− }
8024−
8025− dip->private = dio_bio->bi_private;
8026− dip->inode = inode;
8027− dip->logical_offset = file_offset;
8028− dip->bytes = dio_bio->bi_iter.bi_size;
8029− dip->disk_bytenr = (u64)dio_bio->bi_iter.bi_sector << 9;
8030− bio->bi_private = dip;
8031− dip->orig_bio = bio;
8032− dip->dio_bio = dio_bio;
8033− atomic_set(&dip->pending_bios, 0);
8034− io_bio = btrfs_io_bio(bio);
8035− io_bio->logical = file_offset;
8036−
8037− if (write) {
8038− bio->bi_end_io = btrfs_endio_direct_write;
8039− } else {
8040− bio->bi_end_io = btrfs_endio_direct_read;
8041− dip->subio_endio = btrfs_subio_endio_read;
8042− }
8043−
8044− /*
8045− * Reset the range for unsubmitted ordered extents (to a 0 length range)
8046− * even if we fail to submit a bio, because in such case we do the
8047− * corresponding error handling below and it must not be done a second
8048− * time by btrfs_direct_IO().
8049− */
8050− if (write) {
8051− struct btrfs_dio_data *dio_data = current->journal_info;
8052−
8053− dio_data->unsubmitted_oe_range_end = dip->logical_offset +
8054− dip->bytes;
8055− dio_data->unsubmitted_oe_range_start =
8056− dio_data->unsubmitted_oe_range_end;
8057− }
8058−
8059− ret = btrfs_submit_direct_hook(dip);
8060− if (!ret)
8061− return;
8062−
8063− btrfs_io_bio_free_csum(io_bio);
8064−
8065−free_ordered:
8066− /*
8067− * If we arrived here it means either we failed to submit the dip
8068− * or we either failed to clone the dio_bio or failed to allocate the
8069− * dip. If we cloned the dio_bio and allocated the dip, we can just
8070− * call bio_endio against our io_bio so that we get proper resource
8071− * cleanup if we fail to submit the dip, otherwise, we must do the
8072− * same as btrfs_endio_direct_[write|read] because we can't call these
8073− * callbacks - they require an allocated dip and a clone of dio_bio.
8074− */
8075− if (bio && dip) {
8076− bio_io_error(bio);
8077− /*
8078− * The end io callbacks free our dip, do the final put on bio
8079− * and all the cleanup and final put for dio_bio (through
8080− * dio_end_io()).
8081− */
8082− dip = NULL;
8083− bio = NULL;
8084− } else {
8085− if (write)
8086− __endio_write_update_ordered(inode,
8087− file_offset,
8088− dio_bio->bi_iter.bi_size,
8089− false);
8090− else
8091− unlock_extent(&BTRFS_I(inode)->io_tree, file_offset,
8092− file_offset + dio_bio->bi_iter.bi_size - 1);
8093−
8094− dio_bio->bi_status = BLK_STS_IOERR;
8095− /*
8096− * Releases and cleans up our dio_bio, no need to bio_put()
8097− * nor bio_endio()/bio_io_error() against dio_bio.
8098− */
8099− dio_end_io(dio_bio);
8100− }
8101− if (bio)
8102− bio_put(bio);
8103− kfree(dip);
7817+ dip->dio_bio->bi_status = status;
7818+ btrfs_dio_private_put(dip);
7819+ return BLK_QC_T_NONE;
81047820 }
81057821
8106−static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info,
8107− const struct iov_iter *iter, loff_t offset)
8108−{
8109− int seg;
8110− int i;
8111− unsigned int blocksize_mask = fs_info->sectorsize - 1;
8112− ssize_t retval = -EINVAL;
8113−
8114− if (offset & blocksize_mask)
8115− goto out;
8116−
8117− if (iov_iter_alignment(iter) & blocksize_mask)
8118− goto out;
8119−
8120− /* If this is a write we don't need to check anymore */
8121− if (iov_iter_rw(iter) != READ || !iter_is_iovec(iter))
8122− return 0;
8123− /*
8124− * Check to make sure we don't have duplicate iov_base's in this
8125− * iovec, if so return EINVAL, otherwise we'll get csum errors
8126− * when reading back.
8127− */
8128− for (seg = 0; seg < iter->nr_segs; seg++) {
8129− for (i = seg + 1; i < iter->nr_segs; i++) {
8130− if (iter->iov[seg].iov_base == iter->iov[i].iov_base)
8131− goto out;
8132− }
8133− }
8134− retval = 0;
8135−out:
8136− return retval;
8137−}
8138−
8139−static ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
8140−{
8141− struct file *file = iocb->ki_filp;
8142− struct inode *inode = file->f_mapping->host;
8143− struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
8144− struct btrfs_dio_data dio_data = { 0 };
8145− struct extent_changeset *data_reserved = NULL;
8146− loff_t offset = iocb->ki_pos;
8147− size_t count = 0;
8148− int flags = 0;
8149− bool wakeup = true;
8150− bool relock = false;
8151− ssize_t ret;
8152−
8153− if (check_direct_IO(fs_info, iter, offset))
8154− return 0;
8155−
8156− inode_dio_begin(inode);
8157−
8158− /*
8159− * The generic stuff only does filemap_write_and_wait_range, which
8160− * isn't enough if we've written compressed pages to this area, so
8161− * we need to flush the dirty pages again to make absolutely sure
8162− * that any outstanding dirty pages are on disk.
8163− */
8164− count = iov_iter_count(iter);
8165− if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
8166− &BTRFS_I(inode)->runtime_flags))
8167− filemap_fdatawrite_range(inode->i_mapping, offset,
8168− offset + count - 1);
8169−
8170− if (iov_iter_rw(iter) == WRITE) {
8171− /*
8172− * If the write DIO is beyond the EOF, we need update
8173− * the isize, but it is protected by i_mutex. So we can
8174− * not unlock the i_mutex at this case.
8175− */
8176− if (offset + count <= inode->i_size) {
8177− dio_data.overwrite = 1;
8178− inode_unlock(inode);
8179− relock = true;
8180− } else if (iocb->ki_flags & IOCB_NOWAIT) {
8181− ret = -EAGAIN;
8182− goto out;
8183− }
8184− ret = btrfs_delalloc_reserve_space(inode, &data_reserved,
8185− offset, count);
8186− if (ret)
8187− goto out;
8188−
8189− /*
8190− * We need to know how many extents we reserved so that we can
8191− * do the accounting properly if we go over the number we
8192− * originally calculated. Abuse current->journal_info for this.
8193− */
8194− dio_data.reserve = round_up(count,
8195− fs_info->sectorsize);
8196− dio_data.unsubmitted_oe_range_start = (u64)offset;
8197− dio_data.unsubmitted_oe_range_end = (u64)offset;
8198− current->journal_info = &dio_data;
8199− down_read(&BTRFS_I(inode)->dio_sem);
8200− } else if (test_bit(BTRFS_INODE_READDIO_NEED_LOCK,
8201− &BTRFS_I(inode)->runtime_flags)) {
8202− inode_dio_end(inode);
8203− flags = DIO_LOCKING | DIO_SKIP_HOLES;
8204− wakeup = false;
8205− }
8206−
8207− ret = __blockdev_direct_IO(iocb, inode,
8208− fs_info->fs_devices->latest_bdev,
8209− iter, btrfs_get_blocks_direct, NULL,
8210− btrfs_submit_direct, flags);
8211− if (iov_iter_rw(iter) == WRITE) {
8212− up_read(&BTRFS_I(inode)->dio_sem);
8213− current->journal_info = NULL;
8214− if (ret < 0 && ret != -EIOCBQUEUED) {
8215− if (dio_data.reserve)
8216− btrfs_delalloc_release_space(inode, data_reserved,
8217− offset, dio_data.reserve, true);
8218− /*
8219− * On error we might have left some ordered extents
8220− * without submitting corresponding bios for them, so
8221− * cleanup them up to avoid other tasks getting them
8222− * and waiting for them to complete forever.
8223− */
8224− if (dio_data.unsubmitted_oe_range_start <
8225− dio_data.unsubmitted_oe_range_end)
8226− __endio_write_update_ordered(inode,
8227− dio_data.unsubmitted_oe_range_start,
8228− dio_data.unsubmitted_oe_range_end -
8229− dio_data.unsubmitted_oe_range_start,
8230− false);
8231− } else if (ret >= 0 && (size_t)ret < count)
8232− btrfs_delalloc_release_space(inode, data_reserved,
8233− offset, count - (size_t)ret, true);
8234− btrfs_delalloc_release_extents(BTRFS_I(inode), count);
8235− }
8236−out:
8237− if (wakeup)
8238− inode_dio_end(inode);
8239− if (relock)
8240− inode_lock(inode);
7822+const struct iomap_ops btrfs_dio_iomap_ops = {
7823+ .iomap_begin = btrfs_dio_iomap_begin,
7824+ .iomap_end = btrfs_dio_iomap_end,
7825+};
82417826
8242− extent_changeset_free(data_reserved);
8243− return ret;
8244−}
7827+const struct iomap_dio_ops btrfs_dops = {
7828+ .submit_io = btrfs_submit_direct,
7829+};
82457830
82467831 #define BTRFS_FIEMAP_FLAGS (FIEMAP_FLAG_SYNC)
82477832
@@ -10539,7 +10124,7 @@ static const struct address_space_operations btrfs_aops = { · 2291 unmodified lines
1053910124 .writepage = btrfs_writepage,
1054010125 .writepages = btrfs_writepages,
1054110126 .readahead = btrfs_readahead,
10542− .direct_IO = btrfs_direct_IO,
10127+ .direct_IO = noop_direct_IO,
1054310128 .invalidatepage = btrfs_invalidatepage,
1054410129 .releasepage = btrfs_releasepage,
1054510130 #ifdef CONFIG_MIGRATION
fs/btrfs/ioctl.c+49−53
@@ -660,7 +660,7 @@ static noinline int create_subvol(struct inode *dir, · 659 unmodified lines
660660 goto fail;
661661
662662 key.offset = (u64)-1;
663− new_root = btrfs_get_fs_root(fs_info, &key, true);
663+ new_root = btrfs_get_fs_root(fs_info, objectid, true);
664664 if (IS_ERR(new_root)) {
665665 ret = PTR_ERR(new_root);
666666 btrfs_abort_transaction(trans, ret);
@@ -748,9 +748,8 @@ static int create_snapshot(struct btrfs_root *root, struct inode *dir, · 81 unmodified lines
748748 struct btrfs_pending_snapshot *pending_snapshot;
749749 struct btrfs_trans_handle *trans;
750750 int ret;
751− bool snapshot_force_cow = false;
752751
753− if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state))
752+ if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state))
754753 return -EINVAL;
755754
756755 if (atomic_read(&root->nr_swapfiles)) {
@@ -771,27 +770,6 @@ static int create_snapshot(struct btrfs_root *root, struct inode *dir, · 14 unmodified lines
771770 goto free_pending;
772771 }
773772
774− /*
775− * Force new buffered writes to reserve space even when NOCOW is
776− * possible. This is to avoid later writeback (running dealloc) to
777− * fallback to COW mode and unexpectedly fail with ENOSPC.
778− */
779− btrfs_drew_read_lock(&root->snapshot_lock);
780−
781− ret = btrfs_start_delalloc_snapshot(root);
782− if (ret)
783− goto dec_and_free;
784−
785− /*
786− * All previous writes have started writeback in NOCOW mode, so now
787− * we force future writes to fallback to COW mode during snapshot
788− * creation.
789− */
790− atomic_inc(&root->snapshot_force_cow);
791− snapshot_force_cow = true;
792−
793− btrfs_wait_ordered_extents(root, U64_MAX, 0, (u64)-1);
794−
795773 btrfs_init_block_rsv(&pending_snapshot->block_rsv,
796774 BTRFS_BLOCK_RSV_TEMP);
797775 /*
@@ -806,7 +784,7 @@ static int create_snapshot(struct btrfs_root *root, struct inode *dir, · 8 unmodified lines
806784 &pending_snapshot->block_rsv, 8,
807785 false);
808786 if (ret)
809− goto dec_and_free;
787+ goto free_pending;
810788
811789 pending_snapshot->dentry = dentry;
812790 pending_snapshot->root = root;
@@ -848,11 +826,6 @@ static int create_snapshot(struct btrfs_root *root, struct inode *dir, · 35 unmodified lines
848826 fail:
849827 btrfs_put_root(pending_snapshot->snap);
850828 btrfs_subvolume_release_metadata(fs_info, &pending_snapshot->block_rsv);
851−dec_and_free:
852− if (snapshot_force_cow)
853− atomic_dec(&root->snapshot_force_cow);
854− btrfs_drew_read_unlock(&root->snapshot_lock);
855−
856829 free_pending:
857830 kfree(pending_snapshot->root_item);
858831 btrfs_free_path(pending_snapshot->path);
@@ -983,6 +956,45 @@ out_unlock: · 124 unmodified lines
983956 return error;
984957 }
985958
959+static noinline int btrfs_mksnapshot(const struct path *parent,
960+ const char *name, int namelen,
961+ struct btrfs_root *root,
962+ bool readonly,
963+ struct btrfs_qgroup_inherit *inherit)
964+{
965+ int ret;
966+ bool snapshot_force_cow = false;
967+
968+ /*
969+ * Force new buffered writes to reserve space even when NOCOW is
970+ * possible. This is to avoid later writeback (running dealloc) to
971+ * fallback to COW mode and unexpectedly fail with ENOSPC.
972+ */
973+ btrfs_drew_read_lock(&root->snapshot_lock);
974+
975+ ret = btrfs_start_delalloc_snapshot(root);
976+ if (ret)
977+ goto out;
978+
979+ /*
980+ * All previous writes have started writeback in NOCOW mode, so now
981+ * we force future writes to fallback to COW mode during snapshot
982+ * creation.
983+ */
984+ atomic_inc(&root->snapshot_force_cow);
985+ snapshot_force_cow = true;
986+
987+ btrfs_wait_ordered_extents(root, U64_MAX, 0, (u64)-1);
988+
989+ ret = btrfs_mksubvol(parent, name, namelen,
990+ root, readonly, inherit);
991+out:
992+ if (snapshot_force_cow)
993+ atomic_dec(&root->snapshot_force_cow);
994+ btrfs_drew_read_unlock(&root->snapshot_lock);
995+ return ret;
996+}
997+
986998 /*
987999 * When we're defragging a range, we don't want to kick it off again
9881000 * if it is really just waiting for delalloc to send it down.
@@ -1762,7 +1774,7 @@ static noinline int __btrfs_ioctl_snap_create(struct file *file, · 773 unmodified lines
17621774 */
17631775 ret = -EPERM;
17641776 } else {
1765− ret = btrfs_mksubvol(&file->f_path, name, namelen,
1777+ ret = btrfs_mksnapshot(&file->f_path, name, namelen,
17661778 BTRFS_I(src_inode)->root,
17671779 readonly, inherit);
17681780 }
@@ -2127,10 +2139,7 @@ static noinline int search_ioctl(struct inode *inode, · 358 unmodified lines
21272139 /* search the root of the inode that was passed */
21282140 root = btrfs_grab_root(BTRFS_I(inode)->root);
21292141 } else {
2130− key.objectid = sk->tree_id;
2131− key.type = BTRFS_ROOT_ITEM_KEY;
2132− key.offset = (u64)-1;
2133− root = btrfs_get_fs_root(info, &key, true);
2142+ root = btrfs_get_fs_root(info, sk->tree_id, true);
21342143 if (IS_ERR(root)) {
21352144 btrfs_free_path(path);
21362145 return PTR_ERR(root);
@@ -2263,10 +2272,7 @@ static noinline int btrfs_search_path_in_tree(struct btrfs_fs_info *info, · 126 unmodified lines
22632272
22642273 ptr = &name[BTRFS_INO_LOOKUP_PATH_MAX - 1];
22652274
2266− key.objectid = tree_id;
2267− key.type = BTRFS_ROOT_ITEM_KEY;
2268− key.offset = (u64)-1;
2269− root = btrfs_get_fs_root(info, &key, true);
2275+ root = btrfs_get_fs_root(info, tree_id, true);
22702276 if (IS_ERR(root)) {
22712277 ret = PTR_ERR(root);
22722278 root = NULL;
@@ -2359,10 +2365,7 @@ static int btrfs_search_path_in_tree_user(struct inode *inode, · 86 unmodified lines
23592365 if (dirid != upper_limit.objectid) {
23602366 ptr = &args->path[BTRFS_INO_LOOKUP_USER_PATH_MAX - 1];
23612367
2362− key.objectid = treeid;
2363− key.type = BTRFS_ROOT_ITEM_KEY;
2364− key.offset = (u64)-1;
2365− root = btrfs_get_fs_root(fs_info, &key, true);
2368+ root = btrfs_get_fs_root(fs_info, treeid, true);
23662369 if (IS_ERR(root)) {
23672370 ret = PTR_ERR(root);
23682371 goto out;
@@ -2421,7 +2424,7 @@ static int btrfs_search_path_in_tree_user(struct inode *inode, · 52 unmodified lines
24212424 goto out_put;
24222425 }
24232426
2424− temp_inode = btrfs_iget(sb, &key2, root);
2427+ temp_inode = btrfs_iget(sb, key2.objectid, root);
24252428 if (IS_ERR(temp_inode)) {
24262429 ret = PTR_ERR(temp_inode);
24272430 goto out_put;
@@ -2608,9 +2611,7 @@ static int btrfs_ioctl_get_subvol_info(struct file *file, void __user *argp) · 180 unmodified lines
26082611
26092612 /* Get root_item of inode's subvolume */
26102613 key.objectid = BTRFS_I(inode)->root->root_key.objectid;
2611− key.type = BTRFS_ROOT_ITEM_KEY;
2612− key.offset = (u64)-1;
2613− root = btrfs_get_fs_root(fs_info, &key, true);
2614+ root = btrfs_get_fs_root(fs_info, key.objectid, true);
26142615 if (IS_ERR(root)) {
26152616 ret = PTR_ERR(root);
26162617 goto out_free;
@@ -3278,7 +3279,6 @@ static long btrfs_ioctl_default_subvol(struct file *file, void __user *argp) · 661 unmodified lines
32783279 struct btrfs_dir_item *di;
32793280 struct btrfs_trans_handle *trans;
32803281 struct btrfs_path *path = NULL;
3281− struct btrfs_key location;
32823282 struct btrfs_disk_key disk_key;
32833283 u64 objectid = 0;
32843284 u64 dir_id;
@@ -3299,11 +3299,7 @@ static long btrfs_ioctl_default_subvol(struct file *file, void __user *argp) · 14 unmodified lines
32993299 if (!objectid)
33003300 objectid = BTRFS_FS_TREE_OBJECTID;
33013301
3302− location.objectid = objectid;
3303− location.type = BTRFS_ROOT_ITEM_KEY;
3304− location.offset = (u64)-1;
3305−
3306− new_root = btrfs_get_fs_root(fs_info, &location, true);
3302+ new_root = btrfs_get_fs_root(fs_info, objectid, true);
33073303 if (IS_ERR(new_root)) {
33083304 ret = PTR_ERR(new_root);
33093305 goto out;
fs/btrfs/locking.c+1−0
@@ -410,6 +410,7 @@ void btrfs_tree_read_unlock_blocking(struct extent_buffer *eb) · 409 unmodified lines
410410 * The rwlock is held for write upon exit.
411411 */
412412 void btrfs_tree_lock(struct extent_buffer *eb)
413+ __acquires(&eb->lock)
413414 {
414415 u64 start_ns = 0;
415416
fs/btrfs/misc.h+54−0
@@ -6,6 +6,7 @@ · 5 unmodified lines
66 #include <linux/sched.h>
77 #include <linux/wait.h>
88 #include <asm/div64.h>
9+#include <linux/rbtree.h>
910
1011 #define in_range(b, first, len) ((b) >= (first) && (b) < (first) + (len))
1112
@@ -58,4 +59,57 @@ static inline bool has_single_bit_set(u64 n) · 46 unmodified lines
5859 return is_power_of_two_u64(n);
5960 }
6061
62+/*
63+ * Simple bytenr based rb_tree relate structures
64+ *
65+ * Any structure wants to use bytenr as single search index should have their
66+ * structure start with these members.
67+ */
68+struct rb_simple_node {
69+ struct rb_node rb_node;
70+ u64 bytenr;
71+};
72+
73+static inline struct rb_node *rb_simple_search(struct rb_root *root, u64 bytenr)
74+{
75+ struct rb_node *node = root->rb_node;
76+ struct rb_simple_node *entry;
77+
78+ while (node) {
79+ entry = rb_entry(node, struct rb_simple_node, rb_node);
80+
81+ if (bytenr < entry->bytenr)
82+ node = node->rb_left;
83+ else if (bytenr > entry->bytenr)
84+ node = node->rb_right;
85+ else
86+ return node;
87+ }
88+ return NULL;
89+}
90+
91+static inline struct rb_node *rb_simple_insert(struct rb_root *root, u64 bytenr,
92+ struct rb_node *node)
93+{
94+ struct rb_node **p = &root->rb_node;
95+ struct rb_node *parent = NULL;
96+ struct rb_simple_node *entry;
97+
98+ while (*p) {
99+ parent = *p;
100+ entry = rb_entry(parent, struct rb_simple_node, rb_node);
101+
102+ if (bytenr < entry->bytenr)
103+ p = &(*p)->rb_left;
104+ else if (bytenr > entry->bytenr)
105+ p = &(*p)->rb_right;
106+ else
107+ return parent;
108+ }
109+
110+ rb_link_node(node, parent, p);
111+ rb_insert_color(node, root);
112+ return NULL;
113+}
114+
61115 #endif
fs/btrfs/props.c+2−7
@@ -408,19 +408,14 @@ int btrfs_subvol_inherit_props(struct btrfs_trans_handle *trans, · 407 unmodified lines
408408 struct btrfs_root *parent_root)
409409 {
410410 struct super_block *sb = root->fs_info->sb;
411− struct btrfs_key key;
412411 struct inode *parent_inode, *child_inode;
413412 int ret;
414413
415− key.objectid = BTRFS_FIRST_FREE_OBJECTID;
416− key.type = BTRFS_INODE_ITEM_KEY;
417− key.offset = 0;
418−
419− parent_inode = btrfs_iget(sb, &key, parent_root);
414+ parent_inode = btrfs_iget(sb, BTRFS_FIRST_FREE_OBJECTID, parent_root);
420415 if (IS_ERR(parent_inode))
421416 return PTR_ERR(parent_inode);
422417
423− child_inode = btrfs_iget(sb, &key, root);
418+ child_inode = btrfs_iget(sb, BTRFS_FIRST_FREE_OBJECTID, root);
424419 if (IS_ERR(child_inode)) {
425420 iput(parent_inode);
426421 return PTR_ERR(child_inode);
fs/btrfs/qgroup.c+14−0
@@ -2622,6 +2622,7 @@ int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans, u64 srcid, · 2621 unmodified lines
26222622 struct btrfs_root *quota_root;
26232623 struct btrfs_qgroup *srcgroup;
26242624 struct btrfs_qgroup *dstgroup;
2625+ bool need_rescan = false;
26252626 u32 level_size = 0;
26262627 u64 nums;
26272628
@@ -2765,6 +2766,13 @@ int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans, u64 srcid, · 137 unmodified lines
27652766 goto unlock;
27662767 }
27672768 ++i_qgroups;
2769+
2770+ /*
2771+ * If we're doing a snapshot, and adding the snapshot to a new
2772+ * qgroup, the numbers are guaranteed to be incorrect.
2773+ */
2774+ if (srcid)
2775+ need_rescan = true;
27682776 }
27692777
27702778 for (i = 0; i < inherit->num_ref_copies; ++i, i_qgroups += 2) {
@@ -2784,6 +2792,9 @@ int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans, u64 srcid, · 13 unmodified lines
27842792
27852793 dst->rfer = src->rfer - level_size;
27862794 dst->rfer_cmpr = src->rfer_cmpr - level_size;
2795+
2796+ /* Manually tweaking numbers certainly needs a rescan */
2797+ need_rescan = true;
27872798 }
27882799 for (i = 0; i < inherit->num_excl_copies; ++i, i_qgroups += 2) {
27892800 struct btrfs_qgroup *src;
@@ -2802,6 +2813,7 @@ int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans, u64 srcid, · 12 unmodified lines
28022813
28032814 dst->excl = src->excl + level_size;
28042815 dst->excl_cmpr = src->excl_cmpr + level_size;
2816+ need_rescan = true;
28052817 }
28062818
28072819 unlock:
@@ -2809,6 +2821,8 @@ unlock: · 1 unmodified line
28092821 out:
28102822 if (!committing)
28112823 mutex_unlock(&fs_info->qgroup_ioctl_lock);
2824+ if (need_rescan)
2825+ fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT;
28122826 return ret;
28132827 }
28142828
fs/btrfs/relocation.c+288−1031
@@ -24,6 +24,7 @@ · 23 unmodified lines
2424 #include "delalloc-space.h"
2525 #include "block-group.h"
2626 #include "backref.h"
27+#include "misc.h"
2728
2829 /*
2930 * Relocation overview
@@ -72,100 +73,15 @@ · 42 unmodified lines
7273 * The entry point of relocation is relocate_block_group() function.
7374 */
7475
75−/*
76− * backref_node, mapping_node and tree_block start with this
77− */
78−struct tree_entry {
79− struct rb_node rb_node;
80− u64 bytenr;
81−};
82−
83−/*
84− * present a tree block in the backref cache
85− */
86−struct backref_node {
87− struct rb_node rb_node;
88− u64 bytenr;
89−
90− u64 new_bytenr;
91− /* objectid of tree block owner, can be not uptodate */
92− u64 owner;
93− /* link to pending, changed or detached list */
94− struct list_head list;
95− /* list of upper level blocks reference this block */
96− struct list_head upper;
97− /* list of child blocks in the cache */
98− struct list_head lower;
99− /* NULL if this node is not tree root */
100− struct btrfs_root *root;
101− /* extent buffer got by COW the block */
102− struct extent_buffer *eb;
103− /* level of tree block */
104− unsigned int level:8;
105− /* is the block in non-reference counted tree */
106− unsigned int cowonly:1;
107− /* 1 if no child node in the cache */
108− unsigned int lowest:1;
109− /* is the extent buffer locked */
110− unsigned int locked:1;
111− /* has the block been processed */
112− unsigned int processed:1;
113− /* have backrefs of this block been checked */
114− unsigned int checked:1;
115− /*
116− * 1 if corresponding block has been cowed but some upper
117− * level block pointers may not point to the new location
118− */
119− unsigned int pending:1;
120− /*
121− * 1 if the backref node isn't connected to any other
122− * backref node.
123− */
124− unsigned int detached:1;
125−};
126−
127−/*
128− * present a block pointer in the backref cache
129− */
130−struct backref_edge {
131− struct list_head list[2];
132− struct backref_node *node[2];
133−};
134−
135−#define LOWER 0
136−#define UPPER 1
13776 #define RELOCATION_RESERVED_NODES 256
138−
139−struct backref_cache {
140− /* red black tree of all backref nodes in the cache */
141− struct rb_root rb_root;
142− /* for passing backref nodes to btrfs_reloc_cow_block */
143− struct backref_node *path[BTRFS_MAX_LEVEL];
144− /*
145− * list of blocks that have been cowed but some block
146− * pointers in upper level blocks may not reflect the
147− * new location
148− */
149− struct list_head pending[BTRFS_MAX_LEVEL];
150− /* list of backref nodes with no child node */
151− struct list_head leaves;
152− /* list of blocks that have been cowed in current transaction */
153− struct list_head changed;
154− /* list of detached backref node. */
155− struct list_head detached;
156−
157− u64 last_trans;
158−
159− int nr_nodes;
160− int nr_edges;
161−};
162−
16377 /*
16478 * map address of tree root to tree
16579 */
16680 struct mapping_node {
167− struct rb_node rb_node;
168− u64 bytenr;
81+ struct {
82+ struct rb_node rb_node;
83+ u64 bytenr;
84+ }; /* Use rb_simle_node for search/insert */
16985 void *data;
17086 };
17187
@@ -178,8 +94,10 @@ struct mapping_tree { · 6 unmodified lines
17894 * present a tree block to process
17995 */
18096 struct tree_block {
181− struct rb_node rb_node;
182− u64 bytenr;
97+ struct {
98+ struct rb_node rb_node;
99+ u64 bytenr;
100+ }; /* Use rb_simple_node for search/insert */
183101 struct btrfs_key key;
184102 unsigned int level:8;
185103 unsigned int key_ready:1;
@@ -204,7 +122,7 @@ struct reloc_control { · 18 unmodified lines
204122
205123 struct btrfs_block_rsv *block_rsv;
206124
207− struct backref_cache backref_cache;
125+ struct btrfs_backref_cache backref_cache;
208126
209127 struct file_extent_cluster cluster;
210128 /* tree blocks have been processed */
@@ -235,168 +153,41 @@ struct reloc_control { · 24 unmodified lines
235153 #define MOVE_DATA_EXTENTS 0
236154 #define UPDATE_DATA_PTRS 1
237155
238−static void remove_backref_node(struct backref_cache *cache,
239− struct backref_node *node);
240−static void __mark_block_processed(struct reloc_control *rc,
241− struct backref_node *node);
242−
243−static void mapping_tree_init(struct mapping_tree *tree)
244−{
245− tree->rb_root = RB_ROOT;
246− spin_lock_init(&tree->lock);
247−}
248−
249−static void backref_cache_init(struct backref_cache *cache)
250−{
251− int i;
252− cache->rb_root = RB_ROOT;
253− for (i = 0; i < BTRFS_MAX_LEVEL; i++)
254− INIT_LIST_HEAD(&cache->pending[i]);
255− INIT_LIST_HEAD(&cache->changed);
256− INIT_LIST_HEAD(&cache->detached);
257− INIT_LIST_HEAD(&cache->leaves);
258−}
259−
260−static void backref_cache_cleanup(struct backref_cache *cache)
261−{
262− struct backref_node *node;
263− int i;
264−
265− while (!list_empty(&cache->detached)) {
266− node = list_entry(cache->detached.next,
267− struct backref_node, list);
268− remove_backref_node(cache, node);
269− }
270−
271− while (!list_empty(&cache->leaves)) {
272− node = list_entry(cache->leaves.next,
273− struct backref_node, lower);
274− remove_backref_node(cache, node);
275− }
276−
277− cache->last_trans = 0;
278−
279− for (i = 0; i < BTRFS_MAX_LEVEL; i++)
280− ASSERT(list_empty(&cache->pending[i]));
281− ASSERT(list_empty(&cache->changed));
282− ASSERT(list_empty(&cache->detached));
283− ASSERT(RB_EMPTY_ROOT(&cache->rb_root));
284− ASSERT(!cache->nr_nodes);
285− ASSERT(!cache->nr_edges);
286−}
287−
288−static struct backref_node *alloc_backref_node(struct backref_cache *cache)
289−{
290− struct backref_node *node;
291−
292− node = kzalloc(sizeof(*node), GFP_NOFS);
293− if (node) {
294− INIT_LIST_HEAD(&node->list);
295− INIT_LIST_HEAD(&node->upper);
296− INIT_LIST_HEAD(&node->lower);
297− RB_CLEAR_NODE(&node->rb_node);
298− cache->nr_nodes++;
299− }
300− return node;
301−}
302−
303−static void free_backref_node(struct backref_cache *cache,
304− struct backref_node *node)
305−{
306− if (node) {
307− cache->nr_nodes--;
308− btrfs_put_root(node->root);
309− kfree(node);
310− }
311−}
312−
313−static struct backref_edge *alloc_backref_edge(struct backref_cache *cache)
314−{
315− struct backref_edge *edge;
316−
317− edge = kzalloc(sizeof(*edge), GFP_NOFS);
318− if (edge)
319− cache->nr_edges++;
320− return edge;
321−}
322−
323−static void free_backref_edge(struct backref_cache *cache,
324− struct backref_edge *edge)
156+static void mark_block_processed(struct reloc_control *rc,
157+ struct btrfs_backref_node *node)
325158 {
326− if (edge) {
327− cache->nr_edges--;
328− kfree(edge);
329− }
330−}
159+ u32 blocksize;
331160
332−static struct rb_node *tree_insert(struct rb_root *root, u64 bytenr,
333− struct rb_node *node)
334−{
335− struct rb_node **p = &root->rb_node;
336− struct rb_node *parent = NULL;
337− struct tree_entry *entry;
338−
339− while (*p) {
340− parent = *p;
341− entry = rb_entry(parent, struct tree_entry, rb_node);
342−
343− if (bytenr < entry->bytenr)
344− p = &(*p)->rb_left;
345− else if (bytenr > entry->bytenr)
346− p = &(*p)->rb_right;
347− else
348− return parent;
161+ if (node->level == 0 ||
162+ in_range(node->bytenr, rc->block_group->start,
163+ rc->block_group->length)) {
164+ blocksize = rc->extent_root->fs_info->nodesize;
165+ set_extent_bits(&rc->processed_blocks, node->bytenr,
166+ node->bytenr + blocksize - 1, EXTENT_DIRTY);
349167 }
350−
351− rb_link_node(node, parent, p);
352− rb_insert_color(node, root);
353− return NULL;
168+ node->processed = 1;
354169 }
355170
356−static struct rb_node *tree_search(struct rb_root *root, u64 bytenr)
357−{
358− struct rb_node *n = root->rb_node;
359− struct tree_entry *entry;
360−
361− while (n) {
362− entry = rb_entry(n, struct tree_entry, rb_node);
363171
364− if (bytenr < entry->bytenr)
365− n = n->rb_left;
366− else if (bytenr > entry->bytenr)
367− n = n->rb_right;
368− else
369− return n;
370− }
371− return NULL;
372−}
373−
374−static void backref_tree_panic(struct rb_node *rb_node, int errno, u64 bytenr)
172+static void mapping_tree_init(struct mapping_tree *tree)
375173 {
376−
377− struct btrfs_fs_info *fs_info = NULL;
378− struct backref_node *bnode = rb_entry(rb_node, struct backref_node,
379− rb_node);
380− if (bnode->root)
381− fs_info = bnode->root->fs_info;
382− btrfs_panic(fs_info, errno,
383− "Inconsistency in backref cache found at offset %llu",
384− bytenr);
174+ tree->rb_root = RB_ROOT;
175+ spin_lock_init(&tree->lock);
385176 }
386177
387178 /*
388179 * walk up backref nodes until reach node presents tree root
389180 */
390−static struct backref_node *walk_up_backref(struct backref_node *node,
391− struct backref_edge *edges[],
392− int *index)
181+static struct btrfs_backref_node *walk_up_backref(
182+ struct btrfs_backref_node *node,
183+ struct btrfs_backref_edge *edges[], int *index)
393184 {
394− struct backref_edge *edge;
185+ struct btrfs_backref_edge *edge;
395186 int idx = *index;
396187
397188 while (!list_empty(&node->upper)) {
398189 edge = list_entry(node->upper.next,
399− struct backref_edge, list[LOWER]);
190+ struct btrfs_backref_edge, list[LOWER]);
400191 edges[idx++] = edge;
401192 node = edge->node[UPPER];
402193 }
@@ -408,11 +199,11 @@ static struct backref_node *walk_up_backref(struct backref_node *node, · 5 unmodified lines
408199 /*
409200 * walk down backref nodes to find start of next reference path
410201 */
411−static struct backref_node *walk_down_backref(struct backref_edge *edges[],
412− int *index)
202+static struct btrfs_backref_node *walk_down_backref(
203+ struct btrfs_backref_edge *edges[], int *index)
413204 {
414− struct backref_edge *edge;
415− struct backref_node *lower;
205+ struct btrfs_backref_edge *edge;
206+ struct btrfs_backref_node *lower;
416207 int idx = *index;
417208
418209 while (idx > 0) {
@@ -423,7 +214,7 @@ static struct backref_node *walk_down_backref(struct backref_edge *edges[], · 4 unmodified lines
423214 continue;
424215 }
425216 edge = list_entry(edge->list[LOWER].next,
426− struct backref_edge, list[LOWER]);
217+ struct btrfs_backref_edge, list[LOWER]);
427218 edges[idx - 1] = edge;
428219 *index = idx;
429220 return edge->node[UPPER];
@@ -432,95 +223,24 @@ static struct backref_node *walk_down_backref(struct backref_edge *edges[], · 2 unmodified lines
432223 return NULL;
433224 }
434225
435−static void unlock_node_buffer(struct backref_node *node)
436−{
437− if (node->locked) {
438− btrfs_tree_unlock(node->eb);
439− node->locked = 0;
440− }
441−}
442−
443−static void drop_node_buffer(struct backref_node *node)
444−{
445− if (node->eb) {
446− unlock_node_buffer(node);
447− free_extent_buffer(node->eb);
448− node->eb = NULL;
449− }
450−}
451−
452−static void drop_backref_node(struct backref_cache *tree,
453− struct backref_node *node)
454−{
455− BUG_ON(!list_empty(&node->upper));
456−
457− drop_node_buffer(node);
458− list_del(&node->list);
459− list_del(&node->lower);
460− if (!RB_EMPTY_NODE(&node->rb_node))
461− rb_erase(&node->rb_node, &tree->rb_root);
462− free_backref_node(tree, node);
463−}
464−
465−/*
466− * remove a backref node from the backref cache
467− */
468−static void remove_backref_node(struct backref_cache *cache,
469− struct backref_node *node)
470−{
471− struct backref_node *upper;
472− struct backref_edge *edge;
473−
474− if (!node)
475− return;
476−
477− BUG_ON(!node->lowest && !node->detached);
478− while (!list_empty(&node->upper)) {
479− edge = list_entry(node->upper.next, struct backref_edge,
480− list[LOWER]);
481− upper = edge->node[UPPER];
482− list_del(&edge->list[LOWER]);
483− list_del(&edge->list[UPPER]);
484− free_backref_edge(cache, edge);
485−
486− if (RB_EMPTY_NODE(&upper->rb_node)) {
487− BUG_ON(!list_empty(&node->upper));
488− drop_backref_node(cache, node);
489− node = upper;
490− node->lowest = 1;
491− continue;
492− }
493− /*
494− * add the node to leaf node list if no other
495− * child block cached.
496− */
497− if (list_empty(&upper->lower)) {
498− list_add_tail(&upper->lower, &cache->leaves);
499− upper->lowest = 1;
500− }
501− }
502−
503− drop_backref_node(cache, node);
504−}
505−
506−static void update_backref_node(struct backref_cache *cache,
507− struct backref_node *node, u64 bytenr)
226+static void update_backref_node(struct btrfs_backref_cache *cache,
227+ struct btrfs_backref_node *node, u64 bytenr)
508228 {
509229 struct rb_node *rb_node;
510230 rb_erase(&node->rb_node, &cache->rb_root);
511231 node->bytenr = bytenr;
512− rb_node = tree_insert(&cache->rb_root, node->bytenr, &node->rb_node);
232+ rb_node = rb_simple_insert(&cache->rb_root, node->bytenr, &node->rb_node);
513233 if (rb_node)
514− backref_tree_panic(rb_node, -EEXIST, bytenr);
234+ btrfs_backref_panic(cache->fs_info, bytenr, -EEXIST);
515235 }
516236
517237 /*
518238 * update backref cache after a transaction commit
519239 */
520240 static int update_backref_cache(struct btrfs_trans_handle *trans,
521− struct backref_cache *cache)
241+ struct btrfs_backref_cache *cache)
522242 {
523− struct backref_node *node;
243+ struct btrfs_backref_node *node;
524244 int level = 0;
525245
526246 if (cache->last_trans == 0) {
@@ -538,13 +258,13 @@ static int update_backref_cache(struct btrfs_trans_handle *trans, · 11 unmodified lines
538258 */
539259 while (!list_empty(&cache->detached)) {
540260 node = list_entry(cache->detached.next,
541− struct backref_node, list);
542− remove_backref_node(cache, node);
261+ struct btrfs_backref_node, list);
262+ btrfs_backref_cleanup_node(cache, node);
543263 }
544264
545265 while (!list_empty(&cache->changed)) {
546266 node = list_entry(cache->changed.next,
547− struct backref_node, list);
267+ struct btrfs_backref_node, list);
548268 list_del_init(&node->list);
549269 BUG_ON(node->pending);
550270 update_backref_node(cache, node, node->new_bytenr);
@@ -585,7 +305,8 @@ static bool reloc_root_is_dead(struct btrfs_root *root) · 34 unmodified lines
585305 *
586306 * Reloc tree after swap is considered dead, thus not considered as valid.
587307 * This is enough for most callers, as they don't distinguish dead reloc root
588− * from no reloc root. But should_ignore_root() below is a special case.
308+ * from no reloc root. But btrfs_should_ignore_reloc_root() below is a
309+ * special case.
589310 */
590311 static bool have_reloc_root(struct btrfs_root *root)
591312 {
@@ -596,11 +317,11 @@ static bool have_reloc_root(struct btrfs_root *root) · 4 unmodified lines
596317 return true;
597318 }
598319
599−static int should_ignore_root(struct btrfs_root *root)
320+int btrfs_should_ignore_reloc_root(struct btrfs_root *root)
600321 {
601322 struct btrfs_root *reloc_root;
602323
603− if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state))
324+ if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state))
604325 return 0;
605326
606327 /* This root has been merged with its reloc tree, we can ignore it */
@@ -622,18 +343,20 @@ static int should_ignore_root(struct btrfs_root *root) · 15 unmodified lines
622343 */
623344 return 1;
624345 }
346+
625347 /*
626348 * find reloc tree by address of tree root
627349 */
628−static struct btrfs_root *find_reloc_root(struct reloc_control *rc,
629− u64 bytenr)
350+struct btrfs_root *find_reloc_root(struct btrfs_fs_info *fs_info, u64 bytenr)
630351 {
352+ struct reloc_control *rc = fs_info->reloc_ctl;
631353 struct rb_node *rb_node;
632354 struct mapping_node *node;
633355 struct btrfs_root *root = NULL;
634356
357+ ASSERT(rc);
635358 spin_lock(&rc->reloc_root_tree.lock);
636− rb_node = tree_search(&rc->reloc_root_tree.rb_root, bytenr);
359+ rb_node = rb_simple_search(&rc->reloc_root_tree.rb_root, bytenr);
637360 if (rb_node) {
638361 node = rb_entry(rb_node, struct mapping_node, rb_node);
639362 root = (struct btrfs_root *)node->data;
@@ -642,594 +365,165 @@ static struct btrfs_root *find_reloc_root(struct reloc_control *rc, · 2 unmodified lines
642365 return btrfs_grab_root(root);
643366 }
644367
645−static struct btrfs_root *read_fs_root(struct btrfs_fs_info *fs_info,
646− u64 root_objectid)
368+/*
369+ * For useless nodes, do two major clean ups:
370+ *
371+ * - Cleanup the children edges and nodes
372+ * If child node is also orphan (no parent) during cleanup, then the child
373+ * node will also be cleaned up.
374+ *
375+ * - Freeing up leaves (level 0), keeps nodes detached
376+ * For nodes, the node is still cached as "detached"
377+ *
378+ * Return false if @node is not in the @useless_nodes list.
379+ * Return true if @node is in the @useless_nodes list.
380+ */
381+static bool handle_useless_nodes(struct reloc_control *rc,
382+ struct btrfs_backref_node *node)
647383 {
648− struct btrfs_key key;
384+ struct btrfs_backref_cache *cache = &rc->backref_cache;
385+ struct list_head *useless_node = &cache->useless_node;
386+ bool ret = false;
649387
650− key.objectid = root_objectid;
651− key.type = BTRFS_ROOT_ITEM_KEY;
652− key.offset = (u64)-1;
388+ while (!list_empty(useless_node)) {
389+ struct btrfs_backref_node *cur;
653390
654− return btrfs_get_fs_root(fs_info, &key, false);
655−}
391+ cur = list_first_entry(useless_node, struct btrfs_backref_node,
392+ list);
393+ list_del_init(&cur->list);
656394
657−static noinline_for_stack
658−int find_inline_backref(struct extent_buffer *leaf, int slot,
659− unsigned long *ptr, unsigned long *end)
660−{
661− struct btrfs_key key;
662− struct btrfs_extent_item *ei;
663− struct btrfs_tree_block_info *bi;
664− u32 item_size;
395+ /* Only tree root nodes can be added to @useless_nodes */
396+ ASSERT(list_empty(&cur->upper));
665397
666− btrfs_item_key_to_cpu(leaf, &key, slot);
398+ if (cur == node)
399+ ret = true;
667400
668− item_size = btrfs_item_size_nr(leaf, slot);
669− if (item_size < sizeof(*ei)) {
670− btrfs_print_v0_err(leaf->fs_info);
671− btrfs_handle_fs_error(leaf->fs_info, -EINVAL, NULL);
672− return 1;
673− }
674− ei = btrfs_item_ptr(leaf, slot, struct btrfs_extent_item);
675− WARN_ON(!(btrfs_extent_flags(leaf, ei) &
676− BTRFS_EXTENT_FLAG_TREE_BLOCK));
401+ /* The node is the lowest node */
402+ if (cur->lowest) {
403+ list_del_init(&cur->lower);
404+ cur->lowest = 0;
405+ }
677406
678− if (key.type == BTRFS_EXTENT_ITEM_KEY &&
679− item_size <= sizeof(*ei) + sizeof(*bi)) {
680− WARN_ON(item_size < sizeof(*ei) + sizeof(*bi));
681− return 1;
682− }
683− if (key.type == BTRFS_METADATA_ITEM_KEY &&
684− item_size <= sizeof(*ei)) {
685− WARN_ON(item_size < sizeof(*ei));
686− return 1;
687− }
407+ /* Cleanup the lower edges */
408+ while (!list_empty(&cur->lower)) {
409+ struct btrfs_backref_edge *edge;
410+ struct btrfs_backref_node *lower;
688411
689− if (key.type == BTRFS_EXTENT_ITEM_KEY) {
690− bi = (struct btrfs_tree_block_info *)(ei + 1);
691− *ptr = (unsigned long)(bi + 1);
692− } else {
693− *ptr = (unsigned long)(ei + 1);
412+ edge = list_entry(cur->lower.next,
413+ struct btrfs_backref_edge, list[UPPER]);
414+ list_del(&edge->list[UPPER]);
415+ list_del(&edge->list[LOWER]);
416+ lower = edge->node[LOWER];
417+ btrfs_backref_free_edge(cache, edge);
418+
419+ /* Child node is also orphan, queue for cleanup */
420+ if (list_empty(&lower->upper))
421+ list_add(&lower->list, useless_node);
422+ }
423+ /* Mark this block processed for relocation */
424+ mark_block_processed(rc, cur);
425+
426+ /*
427+ * Backref nodes for tree leaves are deleted from the cache.
428+ * Backref nodes for upper level tree blocks are left in the
429+ * cache to avoid unnecessary backref lookup.
430+ */
431+ if (cur->level > 0) {
432+ list_add(&cur->list, &cache->detached);
433+ cur->detached = 1;
434+ } else {
435+ rb_erase(&cur->rb_node, &cache->rb_root);
436+ btrfs_backref_free_node(cache, cur);
437+ }
694438 }
695− *end = (unsigned long)ei + item_size;
696− return 0;
439+ return ret;
697440 }
698441
699442 /*
700− * build backref tree for a given tree block. root of the backref tree
701− * corresponds the tree block, leaves of the backref tree correspond
702− * roots of b-trees that reference the tree block.
443+ * Build backref tree for a given tree block. Root of the backref tree
444+ * corresponds the tree block, leaves of the backref tree correspond roots of
445+ * b-trees that reference the tree block.
703446 *
704− * the basic idea of this function is check backrefs of a given block
705− * to find upper level blocks that reference the block, and then check
706− * backrefs of these upper level blocks recursively. the recursion stop
707− * when tree root is reached or backrefs for the block is cached.
447+ * The basic idea of this function is check backrefs of a given block to find
448+ * upper level blocks that reference the block, and then check backrefs of
449+ * these upper level blocks recursively. The recursion stops when tree root is
450+ * reached or backrefs for the block is cached.
708451 *
709− * NOTE: if we find backrefs for a block are cached, we know backrefs
710− * for all upper level blocks that directly/indirectly reference the
711− * block are also cached.
452+ * NOTE: if we find that backrefs for a block are cached, we know backrefs for
453+ * all upper level blocks that directly/indirectly reference the block are also
454+ * cached.
712455 */
713−static noinline_for_stack
714−struct backref_node *build_backref_tree(struct reloc_control *rc,
715− struct btrfs_key *node_key,
716− int level, u64 bytenr)
456+static noinline_for_stack struct btrfs_backref_node *build_backref_tree(
457+ struct reloc_control *rc, struct btrfs_key *node_key,
458+ int level, u64 bytenr)
717459 {
718− struct backref_cache *cache = &rc->backref_cache;
719− struct btrfs_path *path1; /* For searching extent root */
720− struct btrfs_path *path2; /* For searching parent of TREE_BLOCK_REF */
721− struct extent_buffer *eb;
722− struct btrfs_root *root;
723− struct backref_node *cur;
724− struct backref_node *upper;
725− struct backref_node *lower;
726− struct backref_node *node = NULL;
727− struct backref_node *exist = NULL;
728− struct backref_edge *edge;
729− struct rb_node *rb_node;
730− struct btrfs_key key;
731− unsigned long end;
732− unsigned long ptr;
733− LIST_HEAD(list); /* Pending edge list, upper node needs to be checked */
734− LIST_HEAD(useless);
735− int cowonly;
460+ struct btrfs_backref_iter *iter;
461+ struct btrfs_backref_cache *cache = &rc->backref_cache;
462+ /* For searching parent of TREE_BLOCK_REF */
463+ struct btrfs_path *path;
464+ struct btrfs_backref_node *cur;
465+ struct btrfs_backref_node *node = NULL;
466+ struct btrfs_backref_edge *edge;
736467 int ret;
737468 int err = 0;
738− bool need_check = true;
739469
740− path1 = btrfs_alloc_path();
741− path2 = btrfs_alloc_path();
742− if (!path1 || !path2) {
470+ iter = btrfs_backref_iter_alloc(rc->extent_root->fs_info, GFP_NOFS);
471+ if (!iter)
472+ return ERR_PTR(-ENOMEM);
473+ path = btrfs_alloc_path();
474+ if (!path) {
743475 err = -ENOMEM;
744476 goto out;
745477 }
746478
747− node = alloc_backref_node(cache);
479+ node = btrfs_backref_alloc_node(cache, bytenr, level);
748480 if (!node) {
749481 err = -ENOMEM;
750482 goto out;
751483 }
752484
753− node->bytenr = bytenr;
754− node->level = level;
755485 node->lowest = 1;
756486 cur = node;
757−again:
758− end = 0;
759− ptr = 0;
760− key.objectid = cur->bytenr;
761− key.type = BTRFS_METADATA_ITEM_KEY;
762− key.offset = (u64)-1;
763−
764− path1->search_commit_root = 1;
765− path1->skip_locking = 1;
766− ret = btrfs_search_slot(NULL, rc->extent_root, &key, path1,
767− 0, 0);
768− if (ret < 0) {
769− err = ret;
770− goto out;
771− }
772− ASSERT(ret);
773− ASSERT(path1->slots[0]);
774−
775− path1->slots[0]--;
776487
777− WARN_ON(cur->checked);
778− if (!list_empty(&cur->upper)) {
779− /*
780− * the backref was added previously when processing
781− * backref of type BTRFS_TREE_BLOCK_REF_KEY
782− */
783− ASSERT(list_is_singular(&cur->upper));
784− edge = list_entry(cur->upper.next, struct backref_edge,
785− list[LOWER]);
786− ASSERT(list_empty(&edge->list[UPPER]));
787− exist = edge->node[UPPER];
788− /*
789− * add the upper level block to pending list if we need
790− * check its backrefs
791− */
792− if (!exist->checked)
793− list_add_tail(&edge->list[UPPER], &list);
794− } else {
795− exist = NULL;
796− }
797−
798− while (1) {
799− cond_resched();
800− eb = path1->nodes[0];
801−
802− if (ptr >= end) {
803− if (path1->slots[0] >= btrfs_header_nritems(eb)) {
804− ret = btrfs_next_leaf(rc->extent_root, path1);
805− if (ret < 0) {
806− err = ret;
807− goto out;
808− }
809− if (ret > 0)
810− break;
811− eb = path1->nodes[0];
812− }
813−
814− btrfs_item_key_to_cpu(eb, &key, path1->slots[0]);
815− if (key.objectid != cur->bytenr) {
816− WARN_ON(exist);
817− break;
818− }
819−
820− if (key.type == BTRFS_EXTENT_ITEM_KEY ||
821− key.type == BTRFS_METADATA_ITEM_KEY) {
822− ret = find_inline_backref(eb, path1->slots[0],
823− &ptr, &end);
824− if (ret)
825− goto next;
826− }
827− }
828−
829− if (ptr < end) {
830− /* update key for inline back ref */
831− struct btrfs_extent_inline_ref *iref;
832− int type;
833− iref = (struct btrfs_extent_inline_ref *)ptr;
834− type = btrfs_get_extent_inline_ref_type(eb, iref,
835− BTRFS_REF_TYPE_BLOCK);
836− if (type == BTRFS_REF_TYPE_INVALID) {
837− err = -EUCLEAN;
838− goto out;
839− }
840− key.type = type;
841− key.offset = btrfs_extent_inline_ref_offset(eb, iref);
842−
843− WARN_ON(key.type != BTRFS_TREE_BLOCK_REF_KEY &&
844− key.type != BTRFS_SHARED_BLOCK_REF_KEY);
845− }
846−
847− /*
848− * Parent node found and matches current inline ref, no need to
849− * rebuild this node for this inline ref.
850− */
851− if (exist &&
852− ((key.type == BTRFS_TREE_BLOCK_REF_KEY &&
853− exist->owner == key.offset) ||
854− (key.type == BTRFS_SHARED_BLOCK_REF_KEY &&
855− exist->bytenr == key.offset))) {
856− exist = NULL;
857− goto next;
858− }
859−
860− /* SHARED_BLOCK_REF means key.offset is the parent bytenr */
861− if (key.type == BTRFS_SHARED_BLOCK_REF_KEY) {
862− if (key.objectid == key.offset) {
863− /*
864− * Only root blocks of reloc trees use backref
865− * pointing to itself.
866− */
867− root = find_reloc_root(rc, cur->bytenr);
868− ASSERT(root);
869− cur->root = root;
870− break;
871− }
872−
873− edge = alloc_backref_edge(cache);
874− if (!edge) {
875− err = -ENOMEM;
876− goto out;
877− }
878− rb_node = tree_search(&cache->rb_root, key.offset);
879− if (!rb_node) {
880− upper = alloc_backref_node(cache);
881− if (!upper) {
882− free_backref_edge(cache, edge);
883− err = -ENOMEM;
884− goto out;
885− }
886− upper->bytenr = key.offset;
887− upper->level = cur->level + 1;
888− /*
889− * backrefs for the upper level block isn't
890− * cached, add the block to pending list
891− */
892− list_add_tail(&edge->list[UPPER], &list);
893− } else {
894− upper = rb_entry(rb_node, struct backref_node,
895− rb_node);
896− ASSERT(upper->checked);
897− INIT_LIST_HEAD(&edge->list[UPPER]);
898− }
899− list_add_tail(&edge->list[LOWER], &cur->upper);
900− edge->node[LOWER] = cur;
901− edge->node[UPPER] = upper;
902−
903− goto next;
904− } else if (unlikely(key.type == BTRFS_EXTENT_REF_V0_KEY)) {
905− err = -EINVAL;
906− btrfs_print_v0_err(rc->extent_root->fs_info);
907− btrfs_handle_fs_error(rc->extent_root->fs_info, err,
908− NULL);
909− goto out;
910− } else if (key.type != BTRFS_TREE_BLOCK_REF_KEY) {
911− goto next;
912− }
913−
914− /*
915− * key.type == BTRFS_TREE_BLOCK_REF_KEY, inline ref offset
916− * means the root objectid. We need to search the tree to get
917− * its parent bytenr.
918− */
919− root = read_fs_root(rc->extent_root->fs_info, key.offset);
920− if (IS_ERR(root)) {
921− err = PTR_ERR(root);
922− goto out;
923− }
924−
925− if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state))
926− cur->cowonly = 1;
927−
928− if (btrfs_root_level(&root->root_item) == cur->level) {
929− /* tree root */
930− ASSERT(btrfs_root_bytenr(&root->root_item) ==
931− cur->bytenr);
932− if (should_ignore_root(root)) {
933− btrfs_put_root(root);
934− list_add(&cur->list, &useless);
935− } else {
936− cur->root = root;
937− }
938− break;
939− }
940−
941− level = cur->level + 1;
942−
943− /* Search the tree to find parent blocks referring the block. */
944− path2->search_commit_root = 1;
945− path2->skip_locking = 1;
946− path2->lowest_level = level;
947− ret = btrfs_search_slot(NULL, root, node_key, path2, 0, 0);
948− path2->lowest_level = 0;
488+ /* Breadth-first search to build backref cache */
489+ do {
490+ ret = btrfs_backref_add_tree_node(cache, path, iter, node_key,
491+ cur);
949492 if (ret < 0) {
950− btrfs_put_root(root);
951493 err = ret;
952494 goto out;
953495 }
954− if (ret > 0 && path2->slots[level] > 0)
955− path2->slots[level]--;
956−
957− eb = path2->nodes[level];
958− if (btrfs_node_blockptr(eb, path2->slots[level]) !=
959− cur->bytenr) {
960− btrfs_err(root->fs_info,
961− "couldn't find block (%llu) (level %d) in tree (%llu) with key (%llu %u %llu)",
962− cur->bytenr, level - 1,
963− root->root_key.objectid,
964− node_key->objectid, node_key->type,
965− node_key->offset);
966− btrfs_put_root(root);
967− err = -ENOENT;
968− goto out;
969− }
970− lower = cur;
971− need_check = true;
972−
973− /* Add all nodes and edges in the path */
974− for (; level < BTRFS_MAX_LEVEL; level++) {
975− if (!path2->nodes[level]) {
976− ASSERT(btrfs_root_bytenr(&root->root_item) ==
977− lower->bytenr);
978− if (should_ignore_root(root)) {
979− btrfs_put_root(root);
980− list_add(&lower->list, &useless);
981− } else {
982− lower->root = root;
983− }
984− break;
985− }
986−
987− edge = alloc_backref_edge(cache);
988− if (!edge) {
989− btrfs_put_root(root);
990− err = -ENOMEM;
991− goto out;
992− }
993−
994− eb = path2->nodes[level];
995− rb_node = tree_search(&cache->rb_root, eb->start);
996− if (!rb_node) {
997− upper = alloc_backref_node(cache);
998− if (!upper) {
999− btrfs_put_root(root);
1000− free_backref_edge(cache, edge);
1001− err = -ENOMEM;
1002− goto out;
1003− }
1004− upper->bytenr = eb->start;
1005− upper->owner = btrfs_header_owner(eb);
1006− upper->level = lower->level + 1;
1007− if (!test_bit(BTRFS_ROOT_REF_COWS,
1008− &root->state))
1009− upper->cowonly = 1;
1010−
1011− /*
1012− * if we know the block isn't shared
1013− * we can void checking its backrefs.
1014− */
1015− if (btrfs_block_can_be_shared(root, eb))
1016− upper->checked = 0;
1017− else
1018− upper->checked = 1;
1019−
1020− /*
1021− * add the block to pending list if we
1022− * need check its backrefs, we only do this once
1023− * while walking up a tree as we will catch
1024− * anything else later on.
1025− */
1026− if (!upper->checked && need_check) {
1027− need_check = false;
1028− list_add_tail(&edge->list[UPPER],
1029− &list);
1030− } else {
1031− if (upper->checked)
1032− need_check = true;
1033− INIT_LIST_HEAD(&edge->list[UPPER]);
1034− }
1035− } else {
1036− upper = rb_entry(rb_node, struct backref_node,
1037− rb_node);
1038− ASSERT(upper->checked);
1039− INIT_LIST_HEAD(&edge->list[UPPER]);
1040− if (!upper->owner)
1041− upper->owner = btrfs_header_owner(eb);
1042− }
1043− list_add_tail(&edge->list[LOWER], &lower->upper);
1044− edge->node[LOWER] = lower;
1045− edge->node[UPPER] = upper;
1046−
1047− if (rb_node) {
1048− btrfs_put_root(root);
1049− break;
1050− }
1051− lower = upper;
1052− upper = NULL;
1053− }
1054− btrfs_release_path(path2);
1055−next:
1056− if (ptr < end) {
1057− ptr += btrfs_extent_inline_ref_size(key.type);
1058− if (ptr >= end) {
1059− WARN_ON(ptr > end);
1060− ptr = 0;
1061− end = 0;
1062− }
1063− }
1064− if (ptr >= end)
1065− path1->slots[0]++;
1066− }
1067− btrfs_release_path(path1);
1068−
1069− cur->checked = 1;
1070− WARN_ON(exist);
1071−
1072− /* the pending list isn't empty, take the first block to process */
1073− if (!list_empty(&list)) {
1074− edge = list_entry(list.next, struct backref_edge, list[UPPER]);
1075− list_del_init(&edge->list[UPPER]);
1076− cur = edge->node[UPPER];
1077− goto again;
1078− }
1079−
1080− /*
1081− * everything goes well, connect backref nodes and insert backref nodes
1082− * into the cache.
1083− */
1084− ASSERT(node->checked);
1085− cowonly = node->cowonly;
1086− if (!cowonly) {
1087− rb_node = tree_insert(&cache->rb_root, node->bytenr,
1088− &node->rb_node);
1089− if (rb_node)
1090− backref_tree_panic(rb_node, -EEXIST, node->bytenr);
1091− list_add_tail(&node->lower, &cache->leaves);
1092− }
1093−
1094− list_for_each_entry(edge, &node->upper, list[LOWER])
1095− list_add_tail(&edge->list[UPPER], &list);
1096−
1097− while (!list_empty(&list)) {
1098− edge = list_entry(list.next, struct backref_edge, list[UPPER]);
1099− list_del_init(&edge->list[UPPER]);
1100− upper = edge->node[UPPER];
1101− if (upper->detached) {
1102− list_del(&edge->list[LOWER]);
1103− lower = edge->node[LOWER];
1104− free_backref_edge(cache, edge);
1105− if (list_empty(&lower->upper))
1106− list_add(&lower->list, &useless);
1107− continue;
1108− }
1109−
1110− if (!RB_EMPTY_NODE(&upper->rb_node)) {
1111− if (upper->lowest) {
1112− list_del_init(&upper->lower);
1113− upper->lowest = 0;
1114− }
1115−
1116− list_add_tail(&edge->list[UPPER], &upper->lower);
1117− continue;
1118− }
1119−
1120− if (!upper->checked) {
1121− /*
1122− * Still want to blow up for developers since this is a
1123− * logic bug.
1124− */
1125− ASSERT(0);
1126− err = -EINVAL;
1127− goto out;
1128− }
1129− if (cowonly != upper->cowonly) {
1130− ASSERT(0);
1131− err = -EINVAL;
1132− goto out;
1133− }
1134−
1135− if (!cowonly) {
1136− rb_node = tree_insert(&cache->rb_root, upper->bytenr,
1137− &upper->rb_node);
1138− if (rb_node)
1139− backref_tree_panic(rb_node, -EEXIST,
1140− upper->bytenr);
496+ edge = list_first_entry_or_null(&cache->pending_edge,
497+ struct btrfs_backref_edge, list[UPPER]);
498+ /*
499+ * The pending list isn't empty, take the first block to
500+ * process
501+ */
502+ if (edge) {
503+ list_del_init(&edge->list[UPPER]);
504+ cur = edge->node[UPPER];
1141505 }
506+ } while (edge);
1142507
1143− list_add_tail(&edge->list[UPPER], &upper->lower);
1144−
1145− list_for_each_entry(edge, &upper->upper, list[LOWER])
1146− list_add_tail(&edge->list[UPPER], &list);
508+ /* Finish the upper linkage of newly added edges/nodes */
509+ ret = btrfs_backref_finish_upper_links(cache, node);
510+ if (ret < 0) {
511+ err = ret;
512+ goto out;
1147513 }
1148− /*
1149− * process useless backref nodes. backref nodes for tree leaves
1150− * are deleted from the cache. backref nodes for upper level
1151− * tree blocks are left in the cache to avoid unnecessary backref
1152− * lookup.
1153− */
1154− while (!list_empty(&useless)) {
1155− upper = list_entry(useless.next, struct backref_node, list);
1156− list_del_init(&upper->list);
1157− ASSERT(list_empty(&upper->upper));
1158− if (upper == node)
1159− node = NULL;
1160− if (upper->lowest) {
1161− list_del_init(&upper->lower);
1162− upper->lowest = 0;
1163− }
1164− while (!list_empty(&upper->lower)) {
1165− edge = list_entry(upper->lower.next,
1166− struct backref_edge, list[UPPER]);
1167− list_del(&edge->list[UPPER]);
1168− list_del(&edge->list[LOWER]);
1169− lower = edge->node[LOWER];
1170− free_backref_edge(cache, edge);
1171514
1172− if (list_empty(&lower->upper))
1173− list_add(&lower->list, &useless);
1174− }
1175− __mark_block_processed(rc, upper);
1176− if (upper->level > 0) {
1177− list_add(&upper->list, &cache->detached);
1178− upper->detached = 1;
1179− } else {
1180− rb_erase(&upper->rb_node, &cache->rb_root);
1181− free_backref_node(cache, upper);
1182− }
1183− }
515+ if (handle_useless_nodes(rc, node))
516+ node = NULL;
1184517 out:
1185− btrfs_free_path(path1);
1186− btrfs_free_path(path2);
518+ btrfs_backref_iter_free(iter);
519+ btrfs_free_path(path);
1187520 if (err) {
1188− while (!list_empty(&useless)) {
1189− lower = list_entry(useless.next,
1190− struct backref_node, list);
1191− list_del_init(&lower->list);
1192− }
1193− while (!list_empty(&list)) {
1194− edge = list_first_entry(&list, struct backref_edge,
1195− list[UPPER]);
1196− list_del(&edge->list[UPPER]);
1197− list_del(&edge->list[LOWER]);
1198− lower = edge->node[LOWER];
1199− upper = edge->node[UPPER];
1200− free_backref_edge(cache, edge);
1201−
1202− /*
1203− * Lower is no longer linked to any upper backref nodes
1204− * and isn't in the cache, we can free it ourselves.
1205− */
1206− if (list_empty(&lower->upper) &&
1207− RB_EMPTY_NODE(&lower->rb_node))
1208− list_add(&lower->list, &useless);
1209−
1210− if (!RB_EMPTY_NODE(&upper->rb_node))
1211− continue;
1212−
1213− /* Add this guy's upper edges to the list to process */
1214− list_for_each_entry(edge, &upper->upper, list[LOWER])
1215− list_add_tail(&edge->list[UPPER], &list);
1216− if (list_empty(&upper->upper))
1217− list_add(&upper->list, &useless);
1218− }
1219−
1220− while (!list_empty(&useless)) {
1221− lower = list_entry(useless.next,
1222− struct backref_node, list);
1223− list_del_init(&lower->list);
1224− if (lower == node)
1225− node = NULL;
1226− free_backref_node(cache, lower);
1227− }
1228−
1229− remove_backref_node(cache, node);
521+ btrfs_backref_error_cleanup(cache, node);
1230522 return ERR_PTR(err);
1231523 }
1232524 ASSERT(!node || !node->detached);
525+ ASSERT(list_empty(&cache->useless_node) &&
526+ list_empty(&cache->pending_edge));
1233527 return node;
1234528 }
1235529
@@ -1244,19 +538,19 @@ static int clone_backref_node(struct btrfs_trans_handle *trans, · 8 unmodified lines
1244538 struct btrfs_root *dest)
1245539 {
1246540 struct btrfs_root *reloc_root = src->reloc_root;
1247− struct backref_cache *cache = &rc->backref_cache;
1248− struct backref_node *node = NULL;
1249− struct backref_node *new_node;
1250− struct backref_edge *edge;
1251− struct backref_edge *new_edge;
541+ struct btrfs_backref_cache *cache = &rc->backref_cache;
542+ struct btrfs_backref_node *node = NULL;
543+ struct btrfs_backref_node *new_node;
544+ struct btrfs_backref_edge *edge;
545+ struct btrfs_backref_edge *new_edge;
1252546 struct rb_node *rb_node;
1253547
1254548 if (cache->last_trans > 0)
1255549 update_backref_cache(trans, cache);
1256550
1257− rb_node = tree_search(&cache->rb_root, src->commit_root->start);
551+ rb_node = rb_simple_search(&cache->rb_root, src->commit_root->start);
1258552 if (rb_node) {
1259− node = rb_entry(rb_node, struct backref_node, rb_node);
553+ node = rb_entry(rb_node, struct btrfs_backref_node, rb_node);
1260554 if (node->detached)
1261555 node = NULL;
1262556 else
@@ -1264,10 +558,10 @@ static int clone_backref_node(struct btrfs_trans_handle *trans, · 1 unmodified line
1264558 }
1265559
1266560 if (!node) {
1267− rb_node = tree_search(&cache->rb_root,
1268− reloc_root->commit_root->start);
561+ rb_node = rb_simple_search(&cache->rb_root,
562+ reloc_root->commit_root->start);
1269563 if (rb_node) {
1270− node = rb_entry(rb_node, struct backref_node,
564+ node = rb_entry(rb_node, struct btrfs_backref_node,
1271565 rb_node);
1272566 BUG_ON(node->detached);
1273567 }
@@ -1276,12 +570,11 @@ static int clone_backref_node(struct btrfs_trans_handle *trans, · 2 unmodified lines
1276570 if (!node)
1277571 return 0;
1278572
1279− new_node = alloc_backref_node(cache);
573+ new_node = btrfs_backref_alloc_node(cache, dest->node->start,
574+ node->level);
1280575 if (!new_node)
1281576 return -ENOMEM;
1282577
1283− new_node->bytenr = dest->node->start;
1284− new_node->level = node->level;
1285578 new_node->lowest = node->lowest;
1286579 new_node->checked = 1;
1287580 new_node->root = btrfs_grab_root(dest);
@@ -1289,23 +582,21 @@ static int clone_backref_node(struct btrfs_trans_handle *trans, · 1 unmodified line
1289582
1290583 if (!node->lowest) {
1291584 list_for_each_entry(edge, &node->lower, list[UPPER]) {
1292− new_edge = alloc_backref_edge(cache);
585+ new_edge = btrfs_backref_alloc_edge(cache);
1293586 if (!new_edge)
1294587 goto fail;
1295588
1296− new_edge->node[UPPER] = new_node;
1297− new_edge->node[LOWER] = edge->node[LOWER];
1298− list_add_tail(&new_edge->list[UPPER],
1299− &new_node->lower);
589+ btrfs_backref_link_edge(new_edge, edge->node[LOWER],
590+ new_node, LINK_UPPER);
1300591 }
1301592 } else {
1302593 list_add_tail(&new_node->lower, &cache->leaves);
1303594 }
1304595
1305− rb_node = tree_insert(&cache->rb_root, new_node->bytenr,
1306− &new_node->rb_node);
596+ rb_node = rb_simple_insert(&cache->rb_root, new_node->bytenr,
597+ &new_node->rb_node);
1307598 if (rb_node)
1308− backref_tree_panic(rb_node, -EEXIST, new_node->bytenr);
599+ btrfs_backref_panic(trans->fs_info, new_node->bytenr, -EEXIST);
1309600
1310601 if (!new_node->lowest) {
1311602 list_for_each_entry(new_edge, &new_node->lower, list[UPPER]) {
@@ -1317,11 +608,11 @@ static int clone_backref_node(struct btrfs_trans_handle *trans, · 5 unmodified lines
1317608 fail:
1318609 while (!list_empty(&new_node->lower)) {
1319610 new_edge = list_entry(new_node->lower.next,
1320− struct backref_edge, list[UPPER]);
611+ struct btrfs_backref_edge, list[UPPER]);
1321612 list_del(&new_edge->list[UPPER]);
1322− free_backref_edge(cache, new_edge);
613+ btrfs_backref_free_edge(cache, new_edge);
1323614 }
1324− free_backref_node(cache, new_node);
615+ btrfs_backref_free_node(cache, new_node);
1325616 return -ENOMEM;
1326617 }
1327618
@@ -1343,8 +634,8 @@ static int __must_check __add_reloc_root(struct btrfs_root *root) · 15 unmodified lines
1343634 node->data = root;
1344635
1345636 spin_lock(&rc->reloc_root_tree.lock);
1346− rb_node = tree_insert(&rc->reloc_root_tree.rb_root,
1347− node->bytenr, &node->rb_node);
637+ rb_node = rb_simple_insert(&rc->reloc_root_tree.rb_root,
638+ node->bytenr, &node->rb_node);
1348639 spin_unlock(&rc->reloc_root_tree.lock);
1349640 if (rb_node) {
1350641 btrfs_panic(fs_info, -EEXIST,
@@ -1370,8 +661,8 @@ static void __del_reloc_root(struct btrfs_root *root) · 19 unmodified lines
1370661
1371662 if (rc && root->node) {
1372663 spin_lock(&rc->reloc_root_tree.lock);
1373− rb_node = tree_search(&rc->reloc_root_tree.rb_root,
1374− root->commit_root->start);
664+ rb_node = rb_simple_search(&rc->reloc_root_tree.rb_root,
665+ root->commit_root->start);
1375666 if (rb_node) {
1376667 node = rb_entry(rb_node, struct mapping_node, rb_node);
1377668 rb_erase(&node->rb_node, &rc->reloc_root_tree.rb_root);
@@ -1414,8 +705,8 @@ static int __update_reloc_root(struct btrfs_root *root) · 36 unmodified lines
1414705 struct reloc_control *rc = fs_info->reloc_ctl;
1415706
1416707 spin_lock(&rc->reloc_root_tree.lock);
1417− rb_node = tree_search(&rc->reloc_root_tree.rb_root,
1418− root->commit_root->start);
708+ rb_node = rb_simple_search(&rc->reloc_root_tree.rb_root,
709+ root->commit_root->start);
1419710 if (rb_node) {
1420711 node = rb_entry(rb_node, struct mapping_node, rb_node);
1421712 rb_erase(&node->rb_node, &rc->reloc_root_tree.rb_root);
@@ -1428,11 +719,11 @@ static int __update_reloc_root(struct btrfs_root *root) · 6 unmodified lines
1428719
1429720 spin_lock(&rc->reloc_root_tree.lock);
1430721 node->bytenr = root->node->start;
1431− rb_node = tree_insert(&rc->reloc_root_tree.rb_root,
1432− node->bytenr, &node->rb_node);
722+ rb_node = rb_simple_insert(&rc->reloc_root_tree.rb_root,
723+ node->bytenr, &node->rb_node);
1433724 spin_unlock(&rc->reloc_root_tree.lock);
1434725 if (rb_node)
1435− backref_tree_panic(rb_node, -EEXIST, node->bytenr);
726+ btrfs_backref_panic(fs_info, node->bytenr, -EEXIST);
1436727 return 0;
1437728 }
1438729
@@ -1505,7 +796,7 @@ static struct btrfs_root *create_reloc_root(struct btrfs_trans_handle *trans, · 66 unmodified lines
1505796
1506797 reloc_root = btrfs_read_tree_root(fs_info->tree_root, &root_key);
1507798 BUG_ON(IS_ERR(reloc_root));
1508− set_bit(BTRFS_ROOT_REF_COWS, &reloc_root->state);
799+ set_bit(BTRFS_ROOT_SHAREABLE, &reloc_root->state);
1509800 reloc_root->last_trans = trans->transid;
1510801 return reloc_root;
1511802 }
@@ -1679,14 +970,6 @@ again: · 167 unmodified lines
1679970 return NULL;
1680971 }
1681972
1682−static int in_block_group(u64 bytenr, struct btrfs_block_group *block_group)
1683−{
1684− if (bytenr >= block_group->start &&
1685− bytenr < block_group->start + block_group->length)
1686− return 1;
1687− return 0;
1688−}
1689−
1690973 /*
1691974 * get new location of data
1692975 */
@@ -1784,7 +1067,8 @@ int replace_file_extents(struct btrfs_trans_handle *trans, · 91 unmodified lines
17841067 num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
17851068 if (bytenr == 0)
17861069 continue;
1787− if (!in_block_group(bytenr, rc->block_group))
1070+ if (!in_range(bytenr, rc->block_group->start,
1071+ rc->block_group->length))
17881072 continue;
17891073
17901074 /*
@@ -1940,7 +1224,7 @@ again: · 149 unmodified lines
19401224 level = btrfs_header_level(parent);
19411225 BUG_ON(level < lowest_level);
19421226
1943− ret = btrfs_bin_search(parent, &key, level, &slot);
1227+ ret = btrfs_bin_search(parent, &key, &slot);
19441228 if (ret < 0)
19451229 break;
19461230 if (ret && slot > 0)
@@ -2560,7 +1844,8 @@ again: · 613 unmodified lines
25601844 struct btrfs_root, root_list);
25611845 list_del_init(&reloc_root->root_list);
25621846
2563− root = read_fs_root(fs_info, reloc_root->root_key.offset);
1847+ root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset,
1848+ false);
25641849 BUG_ON(IS_ERR(root));
25651850 BUG_ON(root->reloc_root != reloc_root);
25661851
@@ -2588,13 +1873,10 @@ again: · 21 unmodified lines
25881873 static noinline_for_stack
25891874 void free_reloc_roots(struct list_head *list)
25901875 {
2591− struct btrfs_root *reloc_root;
1876+ struct btrfs_root *reloc_root, *tmp;
25921877
2593− while (!list_empty(list)) {
2594− reloc_root = list_entry(list->next, struct btrfs_root,
2595− root_list);
1878+ list_for_each_entry_safe(reloc_root, tmp, list, root_list)
25961879 __del_reloc_root(reloc_root);
2597− }
25981880 }
25991881
26001882 static noinline_for_stack
@@ -2624,12 +1906,11 @@ again: · 23 unmodified lines
26241906 reloc_root = list_entry(reloc_roots.next,
26251907 struct btrfs_root, root_list);
26261908
1909+ root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset,
1910+ false);
26271911 if (btrfs_root_refs(&reloc_root->root_item) > 0) {
2628− root = read_fs_root(fs_info,
2629− reloc_root->root_key.offset);
26301912 BUG_ON(IS_ERR(root));
26311913 BUG_ON(root->reloc_root != reloc_root);
2632−
26331914 ret = merge_reloc_root(rc, root);
26341915 btrfs_put_root(root);
26351916 if (ret) {
@@ -2639,6 +1920,16 @@ again: · 3 unmodified lines
26391920 goto out;
26401921 }
26411922 } else {
1923+ if (!IS_ERR(root)) {
1924+ if (root->reloc_root == reloc_root) {
1925+ root->reloc_root = NULL;
1926+ btrfs_put_root(reloc_root);
1927+ }
1928+ clear_bit(BTRFS_ROOT_DEAD_RELOC_TREE,
1929+ &root->state);
1930+ btrfs_put_root(root);
1931+ }
1932+
26421933 list_del_init(&reloc_root->root_list);
26431934 /* Don't forget to queue this reloc root for cleanup */
26441935 list_add_tail(&reloc_root->reloc_dirty_list,
@@ -2653,15 +1944,13 @@ again: · 8 unmodified lines
26531944 out:
26541945 if (ret) {
26551946 btrfs_handle_fs_error(fs_info, ret, NULL);
2656− if (!list_empty(&reloc_roots))
2657− free_reloc_roots(&reloc_roots);
1947+ free_reloc_roots(&reloc_roots);
26581948
26591949 /* new reloc root may be added */
26601950 mutex_lock(&fs_info->reloc_mutex);
26611951 list_splice_init(&rc->reloc_roots, &reloc_roots);
26621952 mutex_unlock(&fs_info->reloc_mutex);
2663− if (!list_empty(&reloc_roots))
2664− free_reloc_roots(&reloc_roots);
1953+ free_reloc_roots(&reloc_roots);
26651954 }
26661955
26671956 /*
@@ -2702,7 +1991,7 @@ static int record_reloc_root_in_trans(struct btrfs_trans_handle *trans, · 34 unmodified lines
27021991 if (reloc_root->last_trans == trans->transid)
27031992 return 0;
27041993
2705− root = read_fs_root(fs_info, reloc_root->root_key.offset);
1994+ root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset, false);
27061995 BUG_ON(IS_ERR(root));
27071996 BUG_ON(root->reloc_root != reloc_root);
27081997 ret = btrfs_record_root_in_trans(trans, root);
@@ -2714,10 +2003,10 @@ static int record_reloc_root_in_trans(struct btrfs_trans_handle *trans, · 5 unmodified lines
27142003 static noinline_for_stack
27152004 struct btrfs_root *select_reloc_root(struct btrfs_trans_handle *trans,
27162005 struct reloc_control *rc,
2717− struct backref_node *node,
2718− struct backref_edge *edges[])
2006+ struct btrfs_backref_node *node,
2007+ struct btrfs_backref_edge *edges[])
27192008 {
2720− struct backref_node *next;
2009+ struct btrfs_backref_node *next;
27212010 struct btrfs_root *root;
27222011 int index = 0;
27232012
@@ -2727,7 +2016,7 @@ struct btrfs_root *select_reloc_root(struct btrfs_trans_handle *trans, · 3 unmodified lines
27272016 next = walk_up_backref(next, edges, &index);
27282017 root = next->root;
27292018 BUG_ON(!root);
2730− BUG_ON(!test_bit(BTRFS_ROOT_REF_COWS, &root->state));
2019+ BUG_ON(!test_bit(BTRFS_ROOT_SHAREABLE, &root->state));
27312020
27322021 if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) {
27332022 record_reloc_root_in_trans(trans, root);
@@ -2746,7 +2035,7 @@ struct btrfs_root *select_reloc_root(struct btrfs_trans_handle *trans, · 12 unmodified lines
27462035 ASSERT(next->root);
27472036 list_add_tail(&next->list,
27482037 &rc->backref_cache.changed);
2749− __mark_block_processed(rc, next);
2038+ mark_block_processed(rc, next);
27502039 break;
27512040 }
27522041
@@ -2771,18 +2060,21 @@ struct btrfs_root *select_reloc_root(struct btrfs_trans_handle *trans, · 18 unmodified lines
27712060 }
27722061
27732062 /*
2774− * select a tree root for relocation. return NULL if the block
2775− * is reference counted. we should use do_relocation() in this
2776− * case. return a tree root pointer if the block isn't reference
2777− * counted. return -ENOENT if the block is root of reloc tree.
2063+ * Select a tree root for relocation.
2064+ *
2065+ * Return NULL if the block is not shareable. We should use do_relocation() in
2066+ * this case.
2067+ *
2068+ * Return a tree root pointer if the block is shareable.
2069+ * Return -ENOENT if the block is root of reloc tree.
27782070 */
27792071 static noinline_for_stack
2780−struct btrfs_root *select_one_root(struct backref_node *node)
2072+struct btrfs_root *select_one_root(struct btrfs_backref_node *node)
27812073 {
2782− struct backref_node *next;
2074+ struct btrfs_backref_node *next;
27832075 struct btrfs_root *root;
27842076 struct btrfs_root *fs_root = NULL;
2785− struct backref_edge *edges[BTRFS_MAX_LEVEL - 1];
2077+ struct btrfs_backref_edge *edges[BTRFS_MAX_LEVEL - 1];
27862078 int index = 0;
27872079
27882080 next = node;
@@ -2792,8 +2084,8 @@ struct btrfs_root *select_one_root(struct backref_node *node) · 3 unmodified lines
27922084 root = next->root;
27932085 BUG_ON(!root);
27942086
2795− /* no other choice for non-references counted tree */
2796− if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state))
2087+ /* No other choice for non-shareable tree */
2088+ if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state))
27972089 return root;
27982090
27992091 if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID)
@@ -2814,12 +2106,12 @@ struct btrfs_root *select_one_root(struct backref_node *node) · 14 unmodified lines
28142106
28152107 static noinline_for_stack
28162108 u64 calcu_metadata_size(struct reloc_control *rc,
2817− struct backref_node *node, int reserve)
2109+ struct btrfs_backref_node *node, int reserve)
28182110 {
28192111 struct btrfs_fs_info *fs_info = rc->extent_root->fs_info;
2820− struct backref_node *next = node;
2821− struct backref_edge *edge;
2822− struct backref_edge *edges[BTRFS_MAX_LEVEL - 1];
2112+ struct btrfs_backref_node *next = node;
2113+ struct btrfs_backref_edge *edge;
2114+ struct btrfs_backref_edge *edges[BTRFS_MAX_LEVEL - 1];
28232115 u64 num_bytes = 0;
28242116 int index = 0;
28252117
@@ -2837,7 +2129,7 @@ u64 calcu_metadata_size(struct reloc_control *rc, · 11 unmodified lines
28372129 break;
28382130
28392131 edge = list_entry(next->upper.next,
2840− struct backref_edge, list[LOWER]);
2132+ struct btrfs_backref_edge, list[LOWER]);
28412133 edges[index++] = edge;
28422134 next = edge->node[UPPER];
28432135 }
@@ -2848,7 +2140,7 @@ u64 calcu_metadata_size(struct reloc_control *rc, · 4 unmodified lines
28482140
28492141 static int reserve_metadata_space(struct btrfs_trans_handle *trans,
28502142 struct reloc_control *rc,
2851− struct backref_node *node)
2143+ struct btrfs_backref_node *node)
28522144 {
28532145 struct btrfs_root *root = rc->extent_root;
28542146 struct btrfs_fs_info *fs_info = root->fs_info;
@@ -2896,14 +2188,14 @@ static int reserve_metadata_space(struct btrfs_trans_handle *trans, · 41 unmodified lines
28962188 */
28972189 static int do_relocation(struct btrfs_trans_handle *trans,
28982190 struct reloc_control *rc,
2899− struct backref_node *node,
2191+ struct btrfs_backref_node *node,
29002192 struct btrfs_key *key,
29012193 struct btrfs_path *path, int lowest)
29022194 {
29032195 struct btrfs_fs_info *fs_info = rc->extent_root->fs_info;
2904− struct backref_node *upper;
2905− struct backref_edge *edge;
2906− struct backref_edge *edges[BTRFS_MAX_LEVEL - 1];
2196+ struct btrfs_backref_node *upper;
2197+ struct btrfs_backref_edge *edge;
2198+ struct btrfs_backref_edge *edges[BTRFS_MAX_LEVEL - 1];
29072199 struct btrfs_root *root;
29082200 struct extent_buffer *eb;
29092201 u32 blocksize;
@@ -2929,8 +2221,7 @@ static int do_relocation(struct btrfs_trans_handle *trans, · 19 unmodified lines
29292221
29302222 if (upper->eb && !upper->locked) {
29312223 if (!lowest) {
2932− ret = btrfs_bin_search(upper->eb, key,
2933− upper->level, &slot);
2224+ ret = btrfs_bin_search(upper->eb, key, &slot);
29342225 if (ret < 0) {
29352226 err = ret;
29362227 goto next;
@@ -2940,7 +2231,7 @@ static int do_relocation(struct btrfs_trans_handle *trans, · 3 unmodified lines
29402231 if (node->eb->start == bytenr)
29412232 goto next;
29422233 }
2943− drop_node_buffer(upper);
2234+ btrfs_backref_drop_node_buffer(upper);
29442235 }
29452236
29462237 if (!upper->eb) {
@@ -2968,8 +2259,7 @@ static int do_relocation(struct btrfs_trans_handle *trans, · 21 unmodified lines
29682259 slot = path->slots[upper->level];
29692260 btrfs_release_path(path);
29702261 } else {
2971− ret = btrfs_bin_search(upper->eb, key, upper->level,
2972− &slot);
2262+ ret = btrfs_bin_search(upper->eb, key, &slot);
29732263 if (ret < 0) {
29742264 err = ret;
29752265 goto next;
@@ -3039,15 +2329,15 @@ static int do_relocation(struct btrfs_trans_handle *trans, · 63 unmodified lines
30392329 }
30402330 next:
30412331 if (!upper->pending)
3042− drop_node_buffer(upper);
2332+ btrfs_backref_drop_node_buffer(upper);
30432333 else
3044− unlock_node_buffer(upper);
2334+ btrfs_backref_unlock_node_buffer(upper);
30452335 if (err)
30462336 break;
30472337 }
30482338
30492339 if (!err && node->pending) {
3050− drop_node_buffer(node);
2340+ btrfs_backref_drop_node_buffer(node);
30512341 list_move_tail(&node->list, &rc->backref_cache.changed);
30522342 node->pending = 0;
30532343 }
@@ -3059,7 +2349,7 @@ next: · 5 unmodified lines
30592349
30602350 static int link_to_upper(struct btrfs_trans_handle *trans,
30612351 struct reloc_control *rc,
3062− struct backref_node *node,
2352+ struct btrfs_backref_node *node,
30632353 struct btrfs_path *path)
30642354 {
30652355 struct btrfs_key key;
@@ -3073,15 +2363,15 @@ static int finish_pending_nodes(struct btrfs_trans_handle *trans, · 7 unmodified lines
30732363 struct btrfs_path *path, int err)
30742364 {
30752365 LIST_HEAD(list);
3076− struct backref_cache *cache = &rc->backref_cache;
3077− struct backref_node *node;
2366+ struct btrfs_backref_cache *cache = &rc->backref_cache;
2367+ struct btrfs_backref_node *node;
30782368 int level;
30792369 int ret;
30802370
30812371 for (level = 0; level < BTRFS_MAX_LEVEL; level++) {
30822372 while (!list_empty(&cache->pending[level])) {
30832373 node = list_entry(cache->pending[level].next,
3084− struct backref_node, list);
2374+ struct btrfs_backref_node, list);
30852375 list_move_tail(&node->list, &list);
30862376 BUG_ON(!node->pending);
30872377
@@ -3096,35 +2386,16 @@ static int finish_pending_nodes(struct btrfs_trans_handle *trans, · 8 unmodified lines
30962386 return err;
30972387 }
30982388
3099−static void mark_block_processed(struct reloc_control *rc,
3100− u64 bytenr, u32 blocksize)
3101−{
3102− set_extent_bits(&rc->processed_blocks, bytenr, bytenr + blocksize - 1,
3103− EXTENT_DIRTY);
3104−}
3105−
3106−static void __mark_block_processed(struct reloc_control *rc,
3107− struct backref_node *node)
3108−{
3109− u32 blocksize;
3110− if (node->level == 0 ||
3111− in_block_group(node->bytenr, rc->block_group)) {
3112− blocksize = rc->extent_root->fs_info->nodesize;
3113− mark_block_processed(rc, node->bytenr, blocksize);
3114− }
3115− node->processed = 1;
3116−}
3117−
31182389 /*
31192390 * mark a block and all blocks directly/indirectly reference the block
31202391 * as processed.
31212392 */
31222393 static void update_processed_blocks(struct reloc_control *rc,
3123− struct backref_node *node)
2394+ struct btrfs_backref_node *node)
31242395 {
3125− struct backref_node *next = node;
3126− struct backref_edge *edge;
3127− struct backref_edge *edges[BTRFS_MAX_LEVEL - 1];
2396+ struct btrfs_backref_node *next = node;
2397+ struct btrfs_backref_edge *edge;
2398+ struct btrfs_backref_edge *edges[BTRFS_MAX_LEVEL - 1];
31282399 int index = 0;
31292400
31302401 while (next) {
@@ -3133,13 +2404,13 @@ static void update_processed_blocks(struct reloc_control *rc, · 2 unmodified lines
31332404 if (next->processed)
31342405 break;
31352406
3136− __mark_block_processed(rc, next);
2407+ mark_block_processed(rc, next);
31372408
31382409 if (list_empty(&next->upper))
31392410 break;
31402411
31412412 edge = list_entry(next->upper.next,
3142− struct backref_edge, list[LOWER]);
2413+ struct btrfs_backref_edge, list[LOWER]);
31432414 edges[index++] = edge;
31442415 next = edge->node[UPPER];
31452416 }
@@ -3184,7 +2455,7 @@ static int get_tree_block_key(struct btrfs_fs_info *fs_info, · 38 unmodified lines
31842455 */
31852456 static int relocate_tree_block(struct btrfs_trans_handle *trans,
31862457 struct reloc_control *rc,
3187− struct backref_node *node,
2458+ struct btrfs_backref_node *node,
31882459 struct btrfs_key *key,
31892460 struct btrfs_path *path)
31902461 {
@@ -3210,7 +2481,7 @@ static int relocate_tree_block(struct btrfs_trans_handle *trans, · 19 unmodified lines
32102481 }
32112482
32122483 if (root) {
3213− if (test_bit(BTRFS_ROOT_REF_COWS, &root->state)) {
2484+ if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) {
32142485 BUG_ON(node->new_bytenr);
32152486 BUG_ON(!list_empty(&node->list));
32162487 btrfs_record_root_in_trans(trans, root);
@@ -3234,7 +2505,7 @@ static int relocate_tree_block(struct btrfs_trans_handle *trans, · 17 unmodified lines
32342505 }
32352506 out:
32362507 if (ret || node->level == 0 || node->cowonly)
3237− remove_backref_node(&rc->backref_cache, node);
2508+ btrfs_backref_cleanup_node(&rc->backref_cache, node);
32382509 return ret;
32392510 }
32402511
@@ -3246,7 +2517,7 @@ int relocate_tree_blocks(struct btrfs_trans_handle *trans, · 5 unmodified lines
32462517 struct reloc_control *rc, struct rb_root *blocks)
32472518 {
32482519 struct btrfs_fs_info *fs_info = rc->extent_root->fs_info;
3249− struct backref_node *node;
2520+ struct btrfs_backref_node *node;
32502521 struct btrfs_path *path;
32512522 struct tree_block *block;
32522523 struct tree_block *next;
@@ -3613,9 +2884,10 @@ static int add_tree_block(struct reloc_control *rc, · 360 unmodified lines
36132884 block->level = level;
36142885 block->key_ready = 0;
36152886
3616− rb_node = tree_insert(blocks, block->bytenr, &block->rb_node);
2887+ rb_node = rb_simple_insert(blocks, block->bytenr, &block->rb_node);
36172888 if (rb_node)
3618− backref_tree_panic(rb_node, -EEXIST, block->bytenr);
2889+ btrfs_backref_panic(rc->extent_root->fs_info, block->bytenr,
2890+ -EEXIST);
36192891
36202892 return 0;
36212893 }
@@ -3636,7 +2908,7 @@ static int __add_tree_block(struct reloc_control *rc, · 14 unmodified lines
36362908 if (tree_block_processed(bytenr, rc))
36372909 return 0;
36382910
3639− if (tree_search(blocks, bytenr))
2911+ if (rb_simple_search(blocks, bytenr))
36402912 return 0;
36412913
36422914 path = btrfs_alloc_path();
@@ -3698,7 +2970,6 @@ static int delete_block_group_cache(struct btrfs_fs_info *fs_info, · 55 unmodified lines
36982970 struct inode *inode,
36992971 u64 ino)
37002972 {
3701− struct btrfs_key key;
37022973 struct btrfs_root *root = fs_info->tree_root;
37032974 struct btrfs_trans_handle *trans;
37042975 int ret = 0;
@@ -3706,11 +2977,7 @@ static int delete_block_group_cache(struct btrfs_fs_info *fs_info, · 1 unmodified line
37062977 if (inode)
37072978 goto truncate;
37082979
3709− key.objectid = ino;
3710− key.type = BTRFS_INODE_ITEM_KEY;
3711− key.offset = 0;
3712−
3713− inode = btrfs_iget(fs_info->sb, &key, root);
2980+ inode = btrfs_iget(fs_info->sb, ino, root);
37142981 if (IS_ERR(inode))
37152982 return -ENOENT;
37162983
@@ -4122,7 +3389,7 @@ restart: · 405 unmodified lines
41223389 rc->create_reloc_tree = 0;
41233390 set_reloc_control(rc);
41243391
4125− backref_cache_cleanup(&rc->backref_cache);
3392+ btrfs_backref_release_cache(&rc->backref_cache);
41263393 btrfs_block_rsv_release(fs_info, rc->block_rsv, (u64)-1, NULL);
41273394
41283395 /*
@@ -4198,14 +3465,10 @@ struct inode *create_reloc_inode(struct btrfs_fs_info *fs_info, · 69 unmodified lines
41983465 struct inode *inode = NULL;
41993466 struct btrfs_trans_handle *trans;
42003467 struct btrfs_root *root;
4201− struct btrfs_key key;
42023468 u64 objectid;
42033469 int err = 0;
42043470
4205− root = read_fs_root(fs_info, BTRFS_DATA_RELOC_TREE_OBJECTID);
4206− if (IS_ERR(root))
4207− return ERR_CAST(root);
4208−
3471+ root = btrfs_grab_root(fs_info->data_reloc_root);
42093472 trans = btrfs_start_transaction(root, 6);
42103473 if (IS_ERR(trans)) {
42113474 btrfs_put_root(root);
@@ -4219,10 +3482,7 @@ struct inode *create_reloc_inode(struct btrfs_fs_info *fs_info, · 7 unmodified lines
42193482 err = __insert_orphan_inode(trans, root, objectid);
42203483 BUG_ON(err);
42213484
4222− key.objectid = objectid;
4223− key.type = BTRFS_INODE_ITEM_KEY;
4224− key.offset = 0;
4225− inode = btrfs_iget(fs_info->sb, &key, root);
3485+ inode = btrfs_iget(fs_info->sb, objectid, root);
42263486 BUG_ON(IS_ERR(inode));
42273487 BTRFS_I(inode)->index_cnt = group->start;
42283488
@@ -4249,7 +3509,7 @@ static struct reloc_control *alloc_reloc_control(struct btrfs_fs_info *fs_info) · 20 unmodified lines
42493509
42503510 INIT_LIST_HEAD(&rc->reloc_roots);
42513511 INIT_LIST_HEAD(&rc->dirty_subvol_roots);
4252− backref_cache_init(&rc->backref_cache);
3512+ btrfs_backref_init_cache(fs_info, &rc->backref_cache, 1);
42533513 mapping_tree_init(&rc->reloc_root_tree);
42543514 extent_io_tree_init(fs_info, &rc->processed_blocks,
42553515 IO_TREE_RELOC_BLOCKS, NULL);
@@ -4494,12 +3754,12 @@ int btrfs_recover_relocation(struct btrfs_root *root) · 238 unmodified lines
44943754 goto out;
44953755 }
44963756
4497− set_bit(BTRFS_ROOT_REF_COWS, &reloc_root->state);
3757+ set_bit(BTRFS_ROOT_SHAREABLE, &reloc_root->state);
44983758 list_add(&reloc_root->root_list, &reloc_roots);
44993759
45003760 if (btrfs_root_refs(&reloc_root->root_item) > 0) {
4501− fs_root = read_fs_root(fs_info,
4502− reloc_root->root_key.offset);
3761+ fs_root = btrfs_get_fs_root(fs_info,
3762+ reloc_root->root_key.offset, false);
45033763 if (IS_ERR(fs_root)) {
45043764 ret = PTR_ERR(fs_root);
45053765 if (ret != -ENOENT) {
@@ -4555,7 +3815,8 @@ int btrfs_recover_relocation(struct btrfs_root *root) · 49 unmodified lines
45553815 continue;
45563816 }
45573817
4558− fs_root = read_fs_root(fs_info, reloc_root->root_key.offset);
3818+ fs_root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset,
3819+ false);
45593820 if (IS_ERR(fs_root)) {
45603821 err = PTR_ERR(fs_root);
45613822 list_add_tail(&reloc_root->root_list, &reloc_roots);
@@ -4591,20 +3852,16 @@ out_unset: · 29 unmodified lines
45913852 unset_reloc_control(rc);
45923853 free_reloc_control(rc);
45933854 out:
4594− if (!list_empty(&reloc_roots))
4595− free_reloc_roots(&reloc_roots);
3855+ free_reloc_roots(&reloc_roots);
45963856
45973857 btrfs_free_path(path);
45983858
45993859 if (err == 0) {
46003860 /* cleanup orphan inode in data relocation tree */
4601− fs_root = read_fs_root(fs_info, BTRFS_DATA_RELOC_TREE_OBJECTID);
4602− if (IS_ERR(fs_root)) {
4603− err = PTR_ERR(fs_root);
4604− } else {
4605− err = btrfs_orphan_cleanup(fs_root);
4606− btrfs_put_root(fs_root);
4607− }
3861+ fs_root = btrfs_grab_root(fs_info->data_reloc_root);
3862+ ASSERT(fs_root);
3863+ err = btrfs_orphan_cleanup(fs_root);
3864+ btrfs_put_root(fs_root);
46083865 }
46093866 return err;
46103867 }
@@ -4666,7 +3923,7 @@ int btrfs_reloc_cow_block(struct btrfs_trans_handle *trans, · 55 unmodified lines
46663923 {
46673924 struct btrfs_fs_info *fs_info = root->fs_info;
46683925 struct reloc_control *rc;
4669− struct backref_node *node;
3926+ struct btrfs_backref_node *node;
46703927 int first_cow = 0;
46713928 int level;
46723929 int ret = 0;
@@ -4691,7 +3948,7 @@ int btrfs_reloc_cow_block(struct btrfs_trans_handle *trans, · 18 unmodified lines
46913948 BUG_ON(node->bytenr != buf->start &&
46923949 node->new_bytenr != buf->start);
46933950
4694− drop_node_buffer(node);
3951+ btrfs_backref_drop_node_buffer(node);
46953952 atomic_inc(&cow->refs);
46963953 node->eb = cow;
46973954 node->new_bytenr = cow->start;
@@ -4703,7 +3960,7 @@ int btrfs_reloc_cow_block(struct btrfs_trans_handle *trans, · 5 unmodified lines
47033960 }
47043961
47053962 if (first_cow)
4706− __mark_block_processed(rc, node);
3963+ mark_block_processed(rc, node);
47073964
47083965 if (first_cow && level > 0)
47093966 rc->nodes_relocated += buf->len;
fs/btrfs/root-tree.c+5−7
@@ -210,7 +210,6 @@ int btrfs_find_orphan_roots(struct btrfs_fs_info *fs_info) · 209 unmodified lines
210210 struct extent_buffer *leaf;
211211 struct btrfs_path *path;
212212 struct btrfs_key key;
213− struct btrfs_key root_key;
214213 struct btrfs_root *root;
215214 int err = 0;
216215 int ret;
@@ -223,10 +222,9 @@ int btrfs_find_orphan_roots(struct btrfs_fs_info *fs_info) · 6 unmodified lines
223222 key.type = BTRFS_ORPHAN_ITEM_KEY;
224223 key.offset = 0;
225224
226− root_key.type = BTRFS_ROOT_ITEM_KEY;
227− root_key.offset = (u64)-1;
228−
229225 while (1) {
226+ u64 root_objectid;
227+
230228 ret = btrfs_search_slot(NULL, tree_root, &key, path, 0, 0);
231229 if (ret < 0) {
232230 err = ret;
@@ -250,10 +248,10 @@ int btrfs_find_orphan_roots(struct btrfs_fs_info *fs_info) · 17 unmodified lines
250248 key.type != BTRFS_ORPHAN_ITEM_KEY)
251249 break;
252250
253− root_key.objectid = key.offset;
251+ root_objectid = key.offset;
254252 key.offset++;
255253
256− root = btrfs_get_fs_root(fs_info, &root_key, false);
254+ root = btrfs_get_fs_root(fs_info, root_objectid, false);
257255 err = PTR_ERR_OR_ZERO(root);
258256 if (err && err != -ENOENT) {
259257 break;
@@ -270,7 +268,7 @@ int btrfs_find_orphan_roots(struct btrfs_fs_info *fs_info) · 10 unmodified lines
270268 break;
271269 }
272270 err = btrfs_del_orphan_item(trans, tree_root,
273− root_key.objectid);
271+ root_objectid);
274272 btrfs_end_transaction(trans);
275273 if (err) {
276274 btrfs_handle_fs_error(fs_info, err,
fs/btrfs/scrub.c+45−14
@@ -647,13 +647,9 @@ static int scrub_print_warning_inode(u64 inum, u64 offset, u64 root, · 646 unmodified lines
647647 struct btrfs_fs_info *fs_info = swarn->dev->fs_info;
648648 struct inode_fs_paths *ipath = NULL;
649649 struct btrfs_root *local_root;
650− struct btrfs_key root_key;
651650 struct btrfs_key key;
652651
653− root_key.objectid = root;
654− root_key.type = BTRFS_ROOT_ITEM_KEY;
655− root_key.offset = (u64)-1;
656− local_root = btrfs_get_fs_root(fs_info, &root_key, true);
652+ local_root = btrfs_get_fs_root(fs_info, root, true);
657653 if (IS_ERR(local_root)) {
658654 ret = PTR_ERR(local_root);
659655 goto err;
@@ -3046,7 +3042,8 @@ out: · 2386 unmodified lines
30463042 static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx,
30473043 struct map_lookup *map,
30483044 struct btrfs_device *scrub_dev,
3049− int num, u64 base, u64 length)
3045+ int num, u64 base, u64 length,
3046+ struct btrfs_block_group *cache)
30503047 {
30513048 struct btrfs_path *path, *ppath;
30523049 struct btrfs_fs_info *fs_info = sctx->fs_info;
@@ -3284,6 +3281,20 @@ static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx, · 231 unmodified lines
32843281 break;
32853282 }
32863283
3284+ /*
3285+ * If our block group was removed in the meanwhile, just
3286+ * stop scrubbing since there is no point in continuing.
3287+ * Continuing would prevent reusing its device extents
3288+ * for new block groups for a long time.
3289+ */
3290+ spin_lock(&cache->lock);
3291+ if (cache->removed) {
3292+ spin_unlock(&cache->lock);
3293+ ret = 0;
3294+ goto out;
3295+ }
3296+ spin_unlock(&cache->lock);
3297+
32873298 extent = btrfs_item_ptr(l, slot,
32883299 struct btrfs_extent_item);
32893300 flags = btrfs_extent_flags(l, extent);
@@ -3328,13 +3339,14 @@ again: · 38 unmodified lines
33283339 &extent_dev,
33293340 &extent_mirror_num);
33303341
3331− ret = btrfs_lookup_csums_range(csum_root,
3332− extent_logical,
3333− extent_logical +
3334− extent_len - 1,
3335− &sctx->csum_list, 1);
3336− if (ret)
3337− goto out;
3342+ if (flags & BTRFS_EXTENT_FLAG_DATA) {
3343+ ret = btrfs_lookup_csums_range(csum_root,
3344+ extent_logical,
3345+ extent_logical + extent_len - 1,
3346+ &sctx->csum_list, 1);
3347+ if (ret)
3348+ goto out;
3349+ }
33383350
33393351 ret = scrub_extent(sctx, map, extent_logical, extent_len,
33403352 extent_physical, extent_dev, flags,
@@ -3457,7 +3469,7 @@ static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx, · 116 unmodified lines
34573469 if (map->stripes[i].dev->bdev == scrub_dev->bdev &&
34583470 map->stripes[i].physical == dev_offset) {
34593471 ret = scrub_stripe(sctx, map, scrub_dev, i,
3460− chunk_offset, length);
3472+ chunk_offset, length, cache);
34613473 if (ret)
34623474 goto out;
34633475 }
@@ -3554,6 +3566,23 @@ int scrub_enumerate_chunks(struct scrub_ctx *sctx, · 90 unmodified lines
35543566 if (!cache)
35553567 goto skip;
35563568
3569+ /*
3570+ * Make sure that while we are scrubbing the corresponding block
3571+ * group doesn't get its logical address and its device extents
3572+ * reused for another block group, which can possibly be of a
3573+ * different type and different profile. We do this to prevent
3574+ * false error detections and crashes due to bogus attempts to
3575+ * repair extents.
3576+ */
3577+ spin_lock(&cache->lock);
3578+ if (cache->removed) {
3579+ spin_unlock(&cache->lock);
3580+ btrfs_put_block_group(cache);
3581+ goto skip;
3582+ }
3583+ btrfs_freeze_block_group(cache);
3584+ spin_unlock(&cache->lock);
3585+
35573586 /*
35583587 * we need call btrfs_inc_block_group_ro() with scrubs_paused,
35593588 * to avoid deadlock caused by:
@@ -3609,6 +3638,7 @@ int scrub_enumerate_chunks(struct scrub_ctx *sctx, · 49 unmodified lines
36093638 } else {
36103639 btrfs_warn(fs_info,
36113640 "failed setting block group ro: %d", ret);
3641+ btrfs_unfreeze_block_group(cache);
36123642 btrfs_put_block_group(cache);
36133643 scrub_pause_off(fs_info);
36143644 break;
@@ -3695,6 +3725,7 @@ int scrub_enumerate_chunks(struct scrub_ctx *sctx, · 80 unmodified lines
36953725 spin_unlock(&cache->lock);
36963726 }
36973727
3728+ btrfs_unfreeze_block_group(cache);
36983729 btrfs_put_block_group(cache);
36993730 if (ret)
37003731 break;
fs/btrfs/send.c+72−17
@@ -23,6 +23,7 @@ · 22 unmodified lines
2323 #include "btrfs_inode.h"
2424 #include "transaction.h"
2525 #include "compression.h"
26+#include "xattr.h"
2627
2728 /*
2829 * Maximum number of references an extent can have in order for us to attempt to
@@ -4545,6 +4546,10 @@ static int __process_new_xattr(int num, struct btrfs_key *di_key, · 4516 unmodified lines
45454546 struct fs_path *p;
45464547 struct posix_acl_xattr_header dummy_acl;
45474548
4549+ /* Capabilities are emitted by finish_inode_if_needed */
4550+ if (!strncmp(name, XATTR_NAME_CAPS, name_len))
4551+ return 0;
4552+
45484553 p = fs_path_alloc();
45494554 if (!p)
45504555 return -ENOMEM;
@@ -4801,17 +4806,12 @@ static ssize_t fill_read_buf(struct send_ctx *sctx, u64 offset, u32 len) · 250 unmodified lines
48014806 struct inode *inode;
48024807 struct page *page;
48034808 char *addr;
4804− struct btrfs_key key;
48054809 pgoff_t index = offset >> PAGE_SHIFT;
48064810 pgoff_t last_index;
48074811 unsigned pg_offset = offset_in_page(offset);
48084812 ssize_t ret = 0;
48094813
4810− key.objectid = sctx->cur_ino;
4811− key.type = BTRFS_INODE_ITEM_KEY;
4812− key.offset = 0;
4813−
4814− inode = btrfs_iget(fs_info->sb, &key, root);
4814+ inode = btrfs_iget(fs_info->sb, sctx->cur_ino, root);
48154815 if (IS_ERR(inode))
48164816 return PTR_ERR(inode);
48174817
@@ -5107,6 +5107,64 @@ static int send_extent_data(struct send_ctx *sctx, · 289 unmodified lines
51075107 return 0;
51085108 }
51095109
5110+/*
5111+ * Search for a capability xattr related to sctx->cur_ino. If the capability is
5112+ * found, call send_set_xattr function to emit it.
5113+ *
5114+ * Return 0 if there isn't a capability, or when the capability was emitted
5115+ * successfully, or < 0 if an error occurred.
5116+ */
5117+static int send_capabilities(struct send_ctx *sctx)
5118+{
5119+ struct fs_path *fspath = NULL;
5120+ struct btrfs_path *path;
5121+ struct btrfs_dir_item *di;
5122+ struct extent_buffer *leaf;
5123+ unsigned long data_ptr;
5124+ char *buf = NULL;
5125+ int buf_len;
5126+ int ret = 0;
5127+
5128+ path = alloc_path_for_send();
5129+ if (!path)
5130+ return -ENOMEM;
5131+
5132+ di = btrfs_lookup_xattr(NULL, sctx->send_root, path, sctx->cur_ino,
5133+ XATTR_NAME_CAPS, strlen(XATTR_NAME_CAPS), 0);
5134+ if (!di) {
5135+ /* There is no xattr for this inode */
5136+ goto out;
5137+ } else if (IS_ERR(di)) {
5138+ ret = PTR_ERR(di);
5139+ goto out;
5140+ }
5141+
5142+ leaf = path->nodes[0];
5143+ buf_len = btrfs_dir_data_len(leaf, di);
5144+
5145+ fspath = fs_path_alloc();
5146+ buf = kmalloc(buf_len, GFP_KERNEL);
5147+ if (!fspath || !buf) {
5148+ ret = -ENOMEM;
5149+ goto out;
5150+ }
5151+
5152+ ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, fspath);
5153+ if (ret < 0)
5154+ goto out;
5155+
5156+ data_ptr = (unsigned long)(di + 1) + btrfs_dir_name_len(leaf, di);
5157+ read_extent_buffer(leaf, buf, data_ptr, buf_len);
5158+
5159+ ret = send_set_xattr(sctx, fspath, XATTR_NAME_CAPS,
5160+ strlen(XATTR_NAME_CAPS), buf, buf_len);
5161+out:
5162+ kfree(buf);
5163+ fs_path_free(fspath);
5164+ btrfs_free_path(path);
5165+ return ret;
5166+}
5167+
51105168 static int clone_range(struct send_ctx *sctx,
51115169 struct clone_root *clone_root,
51125170 const u64 disk_byte,
@@ -5972,6 +6030,10 @@ static int finish_inode_if_needed(struct send_ctx *sctx, int at_end) · 859 unmodified lines
59726030 goto out;
59736031 }
59746032
6033+ ret = send_capabilities(sctx);
6034+ if (ret < 0)
6035+ goto out;
6036+
59756037 /*
59766038 * If other directory inodes depended on our current directory
59776039 * inode's move/rename, now do their move/rename operations.
@@ -7021,7 +7083,6 @@ long btrfs_ioctl_send(struct file *mnt_file, struct btrfs_ioctl_send_args *arg) · 1043 unmodified lines
70217083 struct btrfs_root *send_root = BTRFS_I(file_inode(mnt_file))->root;
70227084 struct btrfs_fs_info *fs_info = send_root->fs_info;
70237085 struct btrfs_root *clone_root;
7024− struct btrfs_key key;
70257086 struct send_ctx *sctx = NULL;
70267087 u32 i;
70277088 u64 *clone_sources_tmp = NULL;
@@ -7143,11 +7204,8 @@ long btrfs_ioctl_send(struct file *mnt_file, struct btrfs_ioctl_send_args *arg) · 115 unmodified lines
71437204 }
71447205
71457206 for (i = 0; i < arg->clone_sources_count; i++) {
7146− key.objectid = clone_sources_tmp[i];
7147− key.type = BTRFS_ROOT_ITEM_KEY;
7148− key.offset = (u64)-1;
7149−
7150− clone_root = btrfs_get_fs_root(fs_info, &key, true);
7207+ clone_root = btrfs_get_fs_root(fs_info,
7208+ clone_sources_tmp[i], true);
71517209 if (IS_ERR(clone_root)) {
71527210 ret = PTR_ERR(clone_root);
71537211 goto out;
@@ -7178,11 +7236,8 @@ long btrfs_ioctl_send(struct file *mnt_file, struct btrfs_ioctl_send_args *arg) · 24 unmodified lines
71787236 }
71797237
71807238 if (arg->parent_root) {
7181− key.objectid = arg->parent_root;
7182− key.type = BTRFS_ROOT_ITEM_KEY;
7183− key.offset = (u64)-1;
7184−
7185− sctx->parent_root = btrfs_get_fs_root(fs_info, &key, true);
7239+ sctx->parent_root = btrfs_get_fs_root(fs_info, arg->parent_root,
7240+ true);
71867241 if (IS_ERR(sctx->parent_root)) {
71877242 ret = PTR_ERR(sctx->parent_root);
71887243 goto out;
fs/btrfs/space-info.c+73−8
@@ -626,6 +626,7 @@ static int may_commit_transaction(struct btrfs_fs_info *fs_info, · 625 unmodified lines
626626 struct reserve_ticket *ticket = NULL;
627627 struct btrfs_block_rsv *delayed_rsv = &fs_info->delayed_block_rsv;
628628 struct btrfs_block_rsv *delayed_refs_rsv = &fs_info->delayed_refs_rsv;
629+ struct btrfs_block_rsv *trans_rsv = &fs_info->trans_block_rsv;
629630 struct btrfs_trans_handle *trans;
630631 u64 bytes_needed;
631632 u64 reclaim_bytes = 0;
@@ -688,6 +689,11 @@ static int may_commit_transaction(struct btrfs_fs_info *fs_info, · 56 unmodified lines
688689 spin_lock(&delayed_refs_rsv->lock);
689690 reclaim_bytes += delayed_refs_rsv->reserved;
690691 spin_unlock(&delayed_refs_rsv->lock);
692+
693+ spin_lock(&trans_rsv->lock);
694+ reclaim_bytes += trans_rsv->reserved;
695+ spin_unlock(&trans_rsv->lock);
696+
691697 if (reclaim_bytes >= bytes_needed)
692698 goto commit;
693699 bytes_needed -= reclaim_bytes;
@@ -856,6 +862,34 @@ static inline int need_do_async_reclaim(struct btrfs_fs_info *fs_info, · 162 unmodified lines
856862 !test_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state));
857863 }
858864
865+static bool steal_from_global_rsv(struct btrfs_fs_info *fs_info,
866+ struct btrfs_space_info *space_info,
867+ struct reserve_ticket *ticket)
868+{
869+ struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
870+ u64 min_bytes;
871+
872+ if (global_rsv->space_info != space_info)
873+ return false;
874+
875+ spin_lock(&global_rsv->lock);
876+ min_bytes = div_factor(global_rsv->size, 1);
877+ if (global_rsv->reserved < min_bytes + ticket->bytes) {
878+ spin_unlock(&global_rsv->lock);
879+ return false;
880+ }
881+ global_rsv->reserved -= ticket->bytes;
882+ ticket->bytes = 0;
883+ list_del_init(&ticket->list);
884+ wake_up(&ticket->wait);
885+ space_info->tickets_id++;
886+ if (global_rsv->reserved < global_rsv->size)
887+ global_rsv->full = 0;
888+ spin_unlock(&global_rsv->lock);
889+
890+ return true;
891+}
892+
859893 /*
860894 * maybe_fail_all_tickets - we've exhausted our flushing, start failing tickets
861895 * @fs_info - fs_info for this fs
@@ -888,6 +922,10 @@ static bool maybe_fail_all_tickets(struct btrfs_fs_info *fs_info, · 26 unmodified lines
888922 ticket = list_first_entry(&space_info->tickets,
889923 struct reserve_ticket, list);
890924
925+ if (ticket->steal &&
926+ steal_from_global_rsv(fs_info, space_info, ticket))
927+ return true;
928+
891929 /*
892930 * may_commit_transaction will avoid committing the transaction
893931 * if it doesn't feel like the space reclaimed by the commit
@@ -1104,6 +1142,7 @@ static int handle_reserve_ticket(struct btrfs_fs_info *fs_info, · 210 unmodified lines
11041142
11051143 switch (flush) {
11061144 case BTRFS_RESERVE_FLUSH_ALL:
1145+ case BTRFS_RESERVE_FLUSH_ALL_STEAL:
11071146 wait_reserve_ticket(fs_info, space_info, ticket);
11081147 break;
11091148 case BTRFS_RESERVE_FLUSH_LIMIT:
@@ -1125,11 +1164,17 @@ static int handle_reserve_ticket(struct btrfs_fs_info *fs_info, · 15 unmodified lines
11251164 ret = ticket->error;
11261165 if (ticket->bytes || ticket->error) {
11271166 /*
1128− * Need to delete here for priority tickets. For regular tickets
1129− * either the async reclaim job deletes the ticket from the list
1130− * or we delete it ourselves at wait_reserve_ticket().
1167+ * We were a priority ticket, so we need to delete ourselves
1168+ * from the list. Because we could have other priority tickets
1169+ * behind us that require less space, run
1170+ * btrfs_try_granting_tickets() to see if their reservations can
1171+ * now be made.
11311172 */
1132− remove_ticket(space_info, ticket);
1173+ if (!list_empty(&ticket->list)) {
1174+ remove_ticket(space_info, ticket);
1175+ btrfs_try_granting_tickets(fs_info, space_info);
1176+ }
1177+
11331178 if (!ret)
11341179 ret = -ENOSPC;
11351180 }
@@ -1145,6 +1190,16 @@ static int handle_reserve_ticket(struct btrfs_fs_info *fs_info, · 9 unmodified lines
11451190 return ret;
11461191 }
11471192
1193+/*
1194+ * This returns true if this flush state will go through the ordinary flushing
1195+ * code.
1196+ */
1197+static inline bool is_normal_flushing(enum btrfs_reserve_flush_enum flush)
1198+{
1199+ return (flush == BTRFS_RESERVE_FLUSH_ALL) ||
1200+ (flush == BTRFS_RESERVE_FLUSH_ALL_STEAL);
1201+}
1202+
11481203 /**
11491204 * reserve_metadata_bytes - try to reserve bytes from the block_rsv's space
11501205 * @root - the root we're allocating for
@@ -1175,8 +1230,17 @@ static int __reserve_metadata_bytes(struct btrfs_fs_info *fs_info, · 24 unmodified lines
11751230 spin_lock(&space_info->lock);
11761231 ret = -ENOSPC;
11771232 used = btrfs_space_info_used(space_info, true);
1178− pending_tickets = !list_empty(&space_info->tickets) ||
1179− !list_empty(&space_info->priority_tickets);
1233+
1234+ /*
1235+ * We don't want NO_FLUSH allocations to jump everybody, they can
1236+ * generally handle ENOSPC in a different way, so treat them the same as
1237+ * normal flushers when it comes to skipping pending tickets.
1238+ */
1239+ if (is_normal_flushing(flush) || (flush == BTRFS_RESERVE_NO_FLUSH))
1240+ pending_tickets = !list_empty(&space_info->tickets) ||
1241+ !list_empty(&space_info->priority_tickets);
1242+ else
1243+ pending_tickets = !list_empty(&space_info->priority_tickets);
11801244
11811245 /*
11821246 * Carry on if we have enough space (short-circuit) OR call
@@ -1198,12 +1262,13 @@ static int __reserve_metadata_bytes(struct btrfs_fs_info *fs_info, · 15 unmodified lines
11981262 * the list and we will do our own flushing further down.
11991263 */
12001264 if (ret && flush != BTRFS_RESERVE_NO_FLUSH) {
1201− ASSERT(space_info->reclaim_size >= 0);
12021265 ticket.bytes = orig_bytes;
12031266 ticket.error = 0;
12041267 space_info->reclaim_size += ticket.bytes;
12051268 init_waitqueue_head(&ticket.wait);
1206− if (flush == BTRFS_RESERVE_FLUSH_ALL) {
1269+ ticket.steal = (flush == BTRFS_RESERVE_FLUSH_ALL_STEAL);
1270+ if (flush == BTRFS_RESERVE_FLUSH_ALL ||
1271+ flush == BTRFS_RESERVE_FLUSH_ALL_STEAL) {
12071272 list_add_tail(&ticket.list, &space_info->tickets);
12081273 if (!space_info->flush) {
12091274 space_info->flush = 1;
fs/btrfs/space-info.h+1−0
@@ -78,6 +78,7 @@ struct btrfs_space_info { · 77 unmodified lines
7878 struct reserve_ticket {
7979 u64 bytes;
8080 int error;
81+ bool steal;
8182 struct list_head list;
8283 wait_queue_head_t wait;
8384 };
fs/btrfs/struct-funcs.c+112−111
@@ -17,151 +17,152 @@ static inline void put_unaligned_le8(u8 val, void *p) · 16 unmodified lines
1717 *(u8 *)p = val;
1818 }
1919
20+static bool check_setget_bounds(const struct extent_buffer *eb,
21+ const void *ptr, unsigned off, int size)
22+{
23+ const unsigned long member_offset = (unsigned long)ptr + off;
24+
25+ if (member_offset > eb->len) {
26+ btrfs_warn(eb->fs_info,
27+ "bad eb member start: ptr 0x%lx start %llu member offset %lu size %d",
28+ (unsigned long)ptr, eb->start, member_offset, size);
29+ return false;
30+ }
31+ if (member_offset + size > eb->len) {
32+ btrfs_warn(eb->fs_info,
33+ "bad eb member end: ptr 0x%lx start %llu member offset %lu size %d",
34+ (unsigned long)ptr, eb->start, member_offset, size);
35+ return false;
36+ }
37+
38+ return true;
39+}
40+
2041 /*
21− * this is some deeply nasty code.
42+ * Macro templates that define helpers to read/write extent buffer data of a
43+ * given size, that are also used via ctree.h for access to item members by
44+ * specialized helpers.
2245 *
23− * The end result is that anyone who #includes ctree.h gets a
24− * declaration for the btrfs_set_foo functions and btrfs_foo functions,
25− * which are wrappers of btrfs_set_token_#bits functions and
26− * btrfs_get_token_#bits functions, which are defined in this file.
46+ * Generic helpers:
47+ * - btrfs_set_8 (for 8/16/32/64)
48+ * - btrfs_get_8 (for 8/16/32/64)
2749 *
28− * These setget functions do all the extent_buffer related mapping
29− * required to efficiently read and write specific fields in the extent
30− * buffers. Every pointer to metadata items in btrfs is really just
31− * an unsigned long offset into the extent buffer which has been
32− * cast to a specific type. This gives us all the gcc type checking.
50+ * Generic helpers with a token (cached address of the most recently accessed
51+ * page):
52+ * - btrfs_set_token_8 (for 8/16/32/64)
53+ * - btrfs_get_token_8 (for 8/16/32/64)
3354 *
34− * The extent buffer api is used to do the page spanning work required to
35− * have a metadata blocksize different from the page size.
55+ * The set/get functions handle data spanning two pages transparently, in case
56+ * metadata block size is larger than page. Every pointer to metadata items is
57+ * an offset into the extent buffer page array, cast to a specific type. This
58+ * gives us all the type checking.
3659 *
37− * There are 2 variants defined, one with a token pointer and one without.
60+ * The extent buffer pages stored in the array pages do not form a contiguous
61+ * phyusical range, but the API functions assume the linear offset to the range
62+ * from 0 to metadata node size.
3863 */
3964
4065 #define DEFINE_BTRFS_SETGET_BITS(bits) \
41−u##bits btrfs_get_token_##bits(const struct extent_buffer *eb, \
42− const void *ptr, unsigned long off, \
43− struct btrfs_map_token *token) \
66+u##bits btrfs_get_token_##bits(struct btrfs_map_token *token, \
67+ const void *ptr, unsigned long off) \
4468 { \
45− unsigned long part_offset = (unsigned long)ptr; \
46− unsigned long offset = part_offset + off; \
47− void *p; \
48− int err; \
49− char *kaddr; \
50− unsigned long map_start; \
51− unsigned long map_len; \
52− int size = sizeof(u##bits); \
53− u##bits res; \
69+ const unsigned long member_offset = (unsigned long)ptr + off; \
70+ const unsigned long idx = member_offset >> PAGE_SHIFT; \
71+ const unsigned long oip = offset_in_page(member_offset); \
72+ const int size = sizeof(u##bits); \
73+ u8 lebytes[sizeof(u##bits)]; \
74+ const int part = PAGE_SIZE - oip; \
5475 \
5576 ASSERT(token); \
56− ASSERT(token->eb == eb); \
57− \
58− if (token->kaddr && token->offset <= offset && \
59− (token->offset + PAGE_SIZE >= offset + size)) { \
60− kaddr = token->kaddr; \
61− p = kaddr + part_offset - token->offset; \
62− res = get_unaligned_le##bits(p + off); \
63− return res; \
77+ ASSERT(token->kaddr); \
78+ ASSERT(check_setget_bounds(token->eb, ptr, off, size)); \
79+ if (token->offset <= member_offset && \
80+ member_offset + size <= token->offset + PAGE_SIZE) { \
81+ return get_unaligned_le##bits(token->kaddr + oip); \
6482 } \
65− err = map_private_extent_buffer(eb, offset, size, \
66− &kaddr, &map_start, &map_len); \
67− if (err) { \
68− __le##bits leres; \
83+ token->kaddr = page_address(token->eb->pages[idx]); \
84+ token->offset = idx << PAGE_SHIFT; \
85+ if (oip + size <= PAGE_SIZE) \
86+ return get_unaligned_le##bits(token->kaddr + oip); \
6987 \
70− read_extent_buffer(eb, &leres, offset, size); \
71− return le##bits##_to_cpu(leres); \
72− } \
73− p = kaddr + part_offset - map_start; \
74− res = get_unaligned_le##bits(p + off); \
75− token->kaddr = kaddr; \
76− token->offset = map_start; \
77− return res; \
88+ memcpy(lebytes, token->kaddr + oip, part); \
89+ token->kaddr = page_address(token->eb->pages[idx + 1]); \
90+ token->offset = (idx + 1) << PAGE_SHIFT; \
91+ memcpy(lebytes + part, token->kaddr, size - part); \
92+ return get_unaligned_le##bits(lebytes); \
7893 } \
7994 u##bits btrfs_get_##bits(const struct extent_buffer *eb, \
8095 const void *ptr, unsigned long off) \
8196 { \
82− unsigned long part_offset = (unsigned long)ptr; \
83− unsigned long offset = part_offset + off; \
84− void *p; \
85− int err; \
86− char *kaddr; \
87− unsigned long map_start; \
88− unsigned long map_len; \
89− int size = sizeof(u##bits); \
90− u##bits res; \
97+ const unsigned long member_offset = (unsigned long)ptr + off; \
98+ const unsigned long oip = offset_in_page(member_offset); \
99+ const unsigned long idx = member_offset >> PAGE_SHIFT; \
100+ char *kaddr = page_address(eb->pages[idx]); \
101+ const int size = sizeof(u##bits); \
102+ const int part = PAGE_SIZE - oip; \
103+ u8 lebytes[sizeof(u##bits)]; \
91104 \
92− err = map_private_extent_buffer(eb, offset, size, \
93− &kaddr, &map_start, &map_len); \
94− if (err) { \
95− __le##bits leres; \
105+ ASSERT(check_setget_bounds(eb, ptr, off, size)); \
106+ if (oip + size <= PAGE_SIZE) \
107+ return get_unaligned_le##bits(kaddr + oip); \
96108 \
97− read_extent_buffer(eb, &leres, offset, size); \
98− return le##bits##_to_cpu(leres); \
99− } \
100− p = kaddr + part_offset - map_start; \
101− res = get_unaligned_le##bits(p + off); \
102− return res; \
109+ memcpy(lebytes, kaddr + oip, part); \
110+ kaddr = page_address(eb->pages[idx + 1]); \
111+ memcpy(lebytes + part, kaddr, size - part); \
112+ return get_unaligned_le##bits(lebytes); \
103113 } \
104−void btrfs_set_token_##bits(struct extent_buffer *eb, \
114+void btrfs_set_token_##bits(struct btrfs_map_token *token, \
105115 const void *ptr, unsigned long off, \
106− u##bits val, \
107− struct btrfs_map_token *token) \
116+ u##bits val) \
108117 { \
109− unsigned long part_offset = (unsigned long)ptr; \
110− unsigned long offset = part_offset + off; \
111− void *p; \
112− int err; \
113− char *kaddr; \
114− unsigned long map_start; \
115− unsigned long map_len; \
116− int size = sizeof(u##bits); \
118+ const unsigned long member_offset = (unsigned long)ptr + off; \
119+ const unsigned long idx = member_offset >> PAGE_SHIFT; \
120+ const unsigned long oip = offset_in_page(member_offset); \
121+ const int size = sizeof(u##bits); \
122+ u8 lebytes[sizeof(u##bits)]; \
123+ const int part = PAGE_SIZE - oip; \
117124 \
118125 ASSERT(token); \
119− ASSERT(token->eb == eb); \
120− \
121− if (token->kaddr && token->offset <= offset && \
122− (token->offset + PAGE_SIZE >= offset + size)) { \
123− kaddr = token->kaddr; \
124− p = kaddr + part_offset - token->offset; \
125− put_unaligned_le##bits(val, p + off); \
126+ ASSERT(token->kaddr); \
127+ ASSERT(check_setget_bounds(token->eb, ptr, off, size)); \
128+ if (token->offset <= member_offset && \
129+ member_offset + size <= token->offset + PAGE_SIZE) { \
130+ put_unaligned_le##bits(val, token->kaddr + oip); \
126131 return; \
127132 } \
128− err = map_private_extent_buffer(eb, offset, size, \
129− &kaddr, &map_start, &map_len); \
130− if (err) { \
131− __le##bits val2; \
132− \
133− val2 = cpu_to_le##bits(val); \
134− write_extent_buffer(eb, &val2, offset, size); \
133+ token->kaddr = page_address(token->eb->pages[idx]); \
134+ token->offset = idx << PAGE_SHIFT; \
135+ if (oip + size <= PAGE_SIZE) { \
136+ put_unaligned_le##bits(val, token->kaddr + oip); \
135137 return; \
136138 } \
137− p = kaddr + part_offset - map_start; \
138− put_unaligned_le##bits(val, p + off); \
139− token->kaddr = kaddr; \
140− token->offset = map_start; \
139+ put_unaligned_le##bits(val, lebytes); \
140+ memcpy(token->kaddr + oip, lebytes, part); \
141+ token->kaddr = page_address(token->eb->pages[idx + 1]); \
142+ token->offset = (idx + 1) << PAGE_SHIFT; \
143+ memcpy(token->kaddr, lebytes + part, size - part); \
141144 } \
142−void btrfs_set_##bits(struct extent_buffer *eb, void *ptr, \
145+void btrfs_set_##bits(const struct extent_buffer *eb, void *ptr, \
143146 unsigned long off, u##bits val) \
144147 { \
145− unsigned long part_offset = (unsigned long)ptr; \
146− unsigned long offset = part_offset + off; \
147− void *p; \
148− int err; \
149− char *kaddr; \
150− unsigned long map_start; \
151− unsigned long map_len; \
152− int size = sizeof(u##bits); \
153− \
154− err = map_private_extent_buffer(eb, offset, size, \
155− &kaddr, &map_start, &map_len); \
156− if (err) { \
157− __le##bits val2; \
148+ const unsigned long member_offset = (unsigned long)ptr + off; \
149+ const unsigned long oip = offset_in_page(member_offset); \
150+ const unsigned long idx = member_offset >> PAGE_SHIFT; \
151+ char *kaddr = page_address(eb->pages[idx]); \
152+ const int size = sizeof(u##bits); \
153+ const int part = PAGE_SIZE - oip; \
154+ u8 lebytes[sizeof(u##bits)]; \
158155 \
159− val2 = cpu_to_le##bits(val); \
160− write_extent_buffer(eb, &val2, offset, size); \
156+ ASSERT(check_setget_bounds(eb, ptr, off, size)); \
157+ if (oip + size <= PAGE_SIZE) { \
158+ put_unaligned_le##bits(val, kaddr + oip); \
161159 return; \
162160 } \
163− p = kaddr + part_offset - map_start; \
164− put_unaligned_le##bits(val, p + off); \
161+ \
162+ put_unaligned_le##bits(val, lebytes); \
163+ memcpy(kaddr + oip, lebytes, part); \
164+ kaddr = page_address(eb->pages[idx + 1]); \
165+ memcpy(kaddr, lebytes + part, size - part); \
165166 }
166167
167168 DEFINE_BTRFS_SETGET_BITS(8)
fs/btrfs/super.c+20−18
@@ -72,23 +72,32 @@ const char * __attribute_const__ btrfs_decode_error(int errno) · 71 unmodified lines
7272 char *errstr = "unknown";
7373
7474 switch (errno) {
75− case -EIO:
75+ case -ENOENT: /* -2 */
76+ errstr = "No such entry";
77+ break;
78+ case -EIO: /* -5 */
7679 errstr = "IO failure";
7780 break;
78− case -ENOMEM:
81+ case -ENOMEM: /* -12*/
7982 errstr = "Out of memory";
8083 break;
81− case -EROFS:
82− errstr = "Readonly filesystem";
83− break;
84− case -EEXIST:
84+ case -EEXIST: /* -17 */
8585 errstr = "Object already exists";
8686 break;
87− case -ENOSPC:
87+ case -ENOSPC: /* -28 */
8888 errstr = "No space left";
8989 break;
90− case -ENOENT:
91− errstr = "No such entry";
90+ case -EROFS: /* -30 */
91+ errstr = "Readonly filesystem";
92+ break;
93+ case -EOPNOTSUPP: /* -95 */
94+ errstr = "Operation not supported";
95+ break;
96+ case -EUCLEAN: /* -117 */
97+ errstr = "Filesystem corrupted";
98+ break;
99+ case -EDQUOT: /* -122 */
100+ errstr = "Quota exceeded";
92101 break;
93102 }
94103
@@ -1093,10 +1102,7 @@ char *btrfs_get_subvol_name_from_objectid(struct btrfs_fs_info *fs_info, · 998 unmodified lines
10931102 dirid = btrfs_root_ref_dirid(path->nodes[0], root_ref);
10941103 btrfs_release_path(path);
10951104
1096− key.objectid = subvol_objectid;
1097− key.type = BTRFS_ROOT_ITEM_KEY;
1098− key.offset = (u64)-1;
1099− fs_root = btrfs_get_fs_root(fs_info, &key, true);
1105+ fs_root = btrfs_get_fs_root(fs_info, subvol_objectid, true);
11001106 if (IS_ERR(fs_root)) {
11011107 ret = PTR_ERR(fs_root);
11021108 fs_root = NULL;
@@ -1211,7 +1217,6 @@ static int btrfs_fill_super(struct super_block *sb, · 108 unmodified lines
12111217 {
12121218 struct inode *inode;
12131219 struct btrfs_fs_info *fs_info = btrfs_sb(sb);
1214− struct btrfs_key key;
12151220 int err;
12161221
12171222 sb->s_maxbytes = MAX_LFS_FILESIZE;
@@ -1239,10 +1244,7 @@ static int btrfs_fill_super(struct super_block *sb, · 21 unmodified lines
12391244 return err;
12401245 }
12411246
1242− key.objectid = BTRFS_FIRST_FREE_OBJECTID;
1243− key.type = BTRFS_INODE_ITEM_KEY;
1244− key.offset = 0;
1245− inode = btrfs_iget(sb, &key, fs_info->fs_root);
1247+ inode = btrfs_iget(sb, BTRFS_FIRST_FREE_OBJECTID, fs_info->fs_root);
12461248 if (IS_ERR(inode)) {
12471249 err = PTR_ERR(inode);
12481250 goto fail_close;
fs/btrfs/transaction.c+32−46
@@ -21,6 +21,7 @@ · 20 unmodified lines
2121 #include "dev-replace.h"
2222 #include "qgroup.h"
2323 #include "block-group.h"
24+#include "space-info.h"
2425
2526 #define BTRFS_ROOT_TRANS_TAG 0
2627
@@ -141,7 +142,7 @@ void btrfs_put_transaction(struct btrfs_transaction *transaction) · 114 unmodified lines
141142 struct btrfs_block_group,
142143 bg_list);
143144 list_del_init(&cache->bg_list);
144− btrfs_put_block_group_trimming(cache);
145+ btrfs_unfreeze_block_group(cache);
145146 btrfs_put_block_group(cache);
146147 }
147148 WARN_ON(!list_empty(&transaction->dev_update_list));
@@ -348,10 +349,10 @@ loop: · 200 unmodified lines
348349 }
349350
350351 /*
351− * this does all the record keeping required to make sure that a reference
352− * counted root is properly recorded in a given transaction. This is required
353− * to make sure the old root from before we joined the transaction is deleted
354− * when the transaction commits
352+ * This does all the record keeping required to make sure that a shareable root
353+ * is properly recorded in a given transaction. This is required to make sure
354+ * the old root from before we joined the transaction is deleted when the
355+ * transaction commits.
355356 */
356357 static int record_root_in_trans(struct btrfs_trans_handle *trans,
357358 struct btrfs_root *root,
@@ -359,7 +360,7 @@ static int record_root_in_trans(struct btrfs_trans_handle *trans, · 1 unmodified line
359360 {
360361 struct btrfs_fs_info *fs_info = root->fs_info;
361362
362− if ((test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
363+ if ((test_bit(BTRFS_ROOT_SHAREABLE, &root->state) &&
363364 root->last_trans < trans->transid) || force) {
364365 WARN_ON(root == fs_info->extent_root);
365366 WARN_ON(!force && root->commit_root != root->node);
@@ -438,7 +439,7 @@ int btrfs_record_root_in_trans(struct btrfs_trans_handle *trans, · 72 unmodified lines
438439 {
439440 struct btrfs_fs_info *fs_info = root->fs_info;
440441
441− if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state))
442+ if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state))
442443 return 0;
443444
444445 /*
@@ -503,7 +504,7 @@ static inline bool need_reserve_reloc_root(struct btrfs_root *root) · 58 unmodified lines
503504 struct btrfs_fs_info *fs_info = root->fs_info;
504505
505506 if (!fs_info->reloc_ctl ||
506− !test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
507+ !test_bit(BTRFS_ROOT_SHAREABLE, &root->state) ||
507508 root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID ||
508509 root->reloc_root)
509510 return false;
@@ -523,6 +524,7 @@ start_transaction(struct btrfs_root *root, unsigned int num_items, · 13 unmodified lines
523524 u64 num_bytes = 0;
524525 u64 qgroup_reserved = 0;
525526 bool reloc_reserved = false;
527+ bool do_chunk_alloc = false;
526528 int ret;
527529
528530 /* Send isn't supposed to start transactions. */
@@ -563,7 +565,8 @@ start_transaction(struct btrfs_root *root, unsigned int num_items, · 34 unmodified lines
563565 * refill that amount for whatever is missing in the reserve.
564566 */
565567 num_bytes = btrfs_calc_insert_metadata_size(fs_info, num_items);
566− if (delayed_refs_rsv->full == 0) {
568+ if (flush == BTRFS_RESERVE_FLUSH_ALL &&
569+ delayed_refs_rsv->full == 0) {
567570 delayed_refs_bytes = num_bytes;
568571 num_bytes <<= 1;
569572 }
@@ -584,6 +587,9 @@ start_transaction(struct btrfs_root *root, unsigned int num_items, · 14 unmodified lines
584587 delayed_refs_bytes);
585588 num_bytes -= delayed_refs_bytes;
586589 }
590+
591+ if (rsv->space_info->force_alloc)
592+ do_chunk_alloc = true;
587593 } else if (num_items == 0 && flush == BTRFS_RESERVE_FLUSH_ALL &&
588594 !delayed_refs_rsv->full) {
589595 /*
@@ -665,6 +671,19 @@ got_it: · 75 unmodified lines
665671 if (!current->journal_info)
666672 current->journal_info = h;
667673
674+ /*
675+ * If the space_info is marked ALLOC_FORCE then we'll get upgraded to
676+ * ALLOC_FORCE the first run through, and then we won't allocate for
677+ * anybody else who races in later. We don't care about the return
678+ * value here.
679+ */
680+ if (do_chunk_alloc && num_bytes) {
681+ u64 flags = h->block_rsv->space_info->flags;
682+
683+ btrfs_chunk_alloc(h, btrfs_get_alloc_profile(fs_info, flags),
684+ CHUNK_ALLOC_NO_FORCE);
685+ }
686+
668687 /*
669688 * btrfs_record_root_in_trans() needs to alloc new extents, and may
670689 * call btrfs_join_transaction() while we're also starting a
@@ -699,43 +718,10 @@ struct btrfs_trans_handle *btrfs_start_transaction(struct btrfs_root *root, · 28 unmodified lines
699718
700719 struct btrfs_trans_handle *btrfs_start_transaction_fallback_global_rsv(
701720 struct btrfs_root *root,
702− unsigned int num_items,
703− int min_factor)
721+ unsigned int num_items)
704722 {
705− struct btrfs_fs_info *fs_info = root->fs_info;
706− struct btrfs_trans_handle *trans;
707− u64 num_bytes;
708− int ret;
709−
710− /*
711− * We have two callers: unlink and block group removal. The
712− * former should succeed even if we will temporarily exceed
713− * quota and the latter operates on the extent root so
714− * qgroup enforcement is ignored anyway.
715− */
716− trans = start_transaction(root, num_items, TRANS_START,
717− BTRFS_RESERVE_FLUSH_ALL, false);
718− if (!IS_ERR(trans) || PTR_ERR(trans) != -ENOSPC)
719− return trans;
720−
721− trans = btrfs_start_transaction(root, 0);
722− if (IS_ERR(trans))
723− return trans;
724−
725− num_bytes = btrfs_calc_insert_metadata_size(fs_info, num_items);
726− ret = btrfs_cond_migrate_bytes(fs_info, &fs_info->trans_block_rsv,
727− num_bytes, min_factor);
728− if (ret) {
729− btrfs_end_transaction(trans);
730− return ERR_PTR(ret);
731− }
732−
733− trans->block_rsv = &fs_info->trans_block_rsv;
734− trans->bytes_reserved = num_bytes;
735− trace_btrfs_space_reservation(fs_info, "transaction",
736− trans->transid, num_bytes, 1);
737−
738− return trans;
723+ return start_transaction(root, num_items, TRANS_START,
724+ BTRFS_RESERVE_FLUSH_ALL_STEAL, false);
739725 }
740726
741727 struct btrfs_trans_handle *btrfs_join_transaction(struct btrfs_root *root)
@@ -1644,7 +1630,7 @@ static noinline int create_pending_snapshot(struct btrfs_trans_handle *trans, · 902 unmodified lines
16441630 }
16451631
16461632 key.offset = (u64)-1;
1647− pending->snap = btrfs_get_fs_root(fs_info, &key, true);
1633+ pending->snap = btrfs_get_fs_root(fs_info, objectid, true);
16481634 if (IS_ERR(pending->snap)) {
16491635 ret = PTR_ERR(pending->snap);
16501636 btrfs_abort_transaction(trans, ret);
fs/btrfs/transaction.h+1−2
@@ -193,8 +193,7 @@ struct btrfs_trans_handle *btrfs_start_transaction(struct btrfs_root *root, · 192 unmodified lines
193193 unsigned int num_items);
194194 struct btrfs_trans_handle *btrfs_start_transaction_fallback_global_rsv(
195195 struct btrfs_root *root,
196− unsigned int num_items,
197− int min_factor);
196+ unsigned int num_items);
198197 struct btrfs_trans_handle *btrfs_join_transaction(struct btrfs_root *root);
199198 struct btrfs_trans_handle *btrfs_join_transaction_spacecache(struct btrfs_root *root);
200199 struct btrfs_trans_handle *btrfs_join_transaction_nostart(struct btrfs_root *root);
fs/btrfs/tree-checker.c+0−4
@@ -957,10 +957,6 @@ static int check_dev_item(struct extent_buffer *leaf, · 956 unmodified lines
957957 return 0;
958958 }
959959
960−/* Inode item error output has the same format as dir_item_err() */
961−#define inode_item_err(eb, slot, fmt, ...) \
962− dir_item_err(eb, slot, fmt, __VA_ARGS__)
963−
964960 static int check_inode_item(struct extent_buffer *leaf,
965961 struct btrfs_key *key, int slot)
966962 {
fs/btrfs/tree-defrag.c+1−1
@@ -35,7 +35,7 @@ int btrfs_defrag_leaves(struct btrfs_trans_handle *trans, · 34 unmodified lines
3535 goto out;
3636 }
3737
38− if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state))
38+ if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state))
3939 goto out;
4040
4141 path = btrfs_alloc_path();
fs/btrfs/tree-log.c+90−102
@@ -505,13 +505,8 @@ insert: · 504 unmodified lines
505505 */
506506 if (S_ISREG(btrfs_inode_mode(eb, src_item)) &&
507507 S_ISREG(btrfs_inode_mode(dst_eb, dst_item)) &&
508− ino_size != 0) {
509− struct btrfs_map_token token;
510−
511− btrfs_init_map_token(&token, dst_eb);
512− btrfs_set_token_inode_size(dst_eb, dst_item,
513− ino_size, &token);
514− }
508+ ino_size != 0)
509+ btrfs_set_inode_size(dst_eb, dst_item, ino_size);
515510 goto no_copy;
516511 }
517512
@@ -555,13 +550,9 @@ no_copy: · 37 unmodified lines
555550 static noinline struct inode *read_one_inode(struct btrfs_root *root,
556551 u64 objectid)
557552 {
558− struct btrfs_key key;
559553 struct inode *inode;
560554
561− key.objectid = objectid;
562− key.type = BTRFS_INODE_ITEM_KEY;
563− key.offset = 0;
564− inode = btrfs_iget(root->fs_info->sb, &key, root);
555+ inode = btrfs_iget(root->fs_info->sb, objectid, root);
565556 if (IS_ERR(inode))
566557 inode = NULL;
567558 return inode;
@@ -3299,6 +3290,7 @@ static void free_log_tree(struct btrfs_trans_handle *trans, · 2731 unmodified lines
32993290
33003291 clear_extent_bits(&log->dirty_log_pages, 0, (u64)-1,
33013292 EXTENT_DIRTY | EXTENT_NEW | EXTENT_NEED_WAIT);
3293+ extent_io_tree_release(&log->log_csum_range);
33023294 btrfs_put_root(log);
33033295 }
33043296
@@ -3816,8 +3808,7 @@ static int drop_objectid_items(struct btrfs_trans_handle *trans, · 511 unmodified lines
38163808
38173809 found_key.offset = 0;
38183810 found_key.type = 0;
3819− ret = btrfs_bin_search(path->nodes[0], &found_key, 0,
3820− &start_slot);
3811+ ret = btrfs_bin_search(path->nodes[0], &found_key, &start_slot);
38213812 if (ret < 0)
38223813 break;
38233814
@@ -3853,44 +3844,41 @@ static void fill_inode_item(struct btrfs_trans_handle *trans, · 29 unmodified lines
38533844 * just to say 'this inode exists' and a logging
38543845 * to say 'update this inode with these values'
38553846 */
3856− btrfs_set_token_inode_generation(leaf, item, 0, &token);
3857− btrfs_set_token_inode_size(leaf, item, logged_isize, &token);
3847+ btrfs_set_token_inode_generation(&token, item, 0);
3848+ btrfs_set_token_inode_size(&token, item, logged_isize);
38583849 } else {
3859− btrfs_set_token_inode_generation(leaf, item,
3860− BTRFS_I(inode)->generation,
3861− &token);
3862− btrfs_set_token_inode_size(leaf, item, inode->i_size, &token);
3863− }
3864−
3865− btrfs_set_token_inode_uid(leaf, item, i_uid_read(inode), &token);
3866− btrfs_set_token_inode_gid(leaf, item, i_gid_read(inode), &token);
3867− btrfs_set_token_inode_mode(leaf, item, inode->i_mode, &token);
3868− btrfs_set_token_inode_nlink(leaf, item, inode->i_nlink, &token);
3869−
3870− btrfs_set_token_timespec_sec(leaf, &item->atime,
3871− inode->i_atime.tv_sec, &token);
3872− btrfs_set_token_timespec_nsec(leaf, &item->atime,
3873− inode->i_atime.tv_nsec, &token);
3874−
3875− btrfs_set_token_timespec_sec(leaf, &item->mtime,
3876− inode->i_mtime.tv_sec, &token);
3877− btrfs_set_token_timespec_nsec(leaf, &item->mtime,
3878− inode->i_mtime.tv_nsec, &token);
3879−
3880− btrfs_set_token_timespec_sec(leaf, &item->ctime,
3881− inode->i_ctime.tv_sec, &token);
3882− btrfs_set_token_timespec_nsec(leaf, &item->ctime,
3883− inode->i_ctime.tv_nsec, &token);
3884−
3885− btrfs_set_token_inode_nbytes(leaf, item, inode_get_bytes(inode),
3886− &token);
3887−
3888− btrfs_set_token_inode_sequence(leaf, item,
3889− inode_peek_iversion(inode), &token);
3890− btrfs_set_token_inode_transid(leaf, item, trans->transid, &token);
3891− btrfs_set_token_inode_rdev(leaf, item, inode->i_rdev, &token);
3892− btrfs_set_token_inode_flags(leaf, item, BTRFS_I(inode)->flags, &token);
3893− btrfs_set_token_inode_block_group(leaf, item, 0, &token);
3850+ btrfs_set_token_inode_generation(&token, item,
3851+ BTRFS_I(inode)->generation);
3852+ btrfs_set_token_inode_size(&token, item, inode->i_size);
3853+ }
3854+
3855+ btrfs_set_token_inode_uid(&token, item, i_uid_read(inode));
3856+ btrfs_set_token_inode_gid(&token, item, i_gid_read(inode));
3857+ btrfs_set_token_inode_mode(&token, item, inode->i_mode);
3858+ btrfs_set_token_inode_nlink(&token, item, inode->i_nlink);
3859+
3860+ btrfs_set_token_timespec_sec(&token, &item->atime,
3861+ inode->i_atime.tv_sec);
3862+ btrfs_set_token_timespec_nsec(&token, &item->atime,
3863+ inode->i_atime.tv_nsec);
3864+
3865+ btrfs_set_token_timespec_sec(&token, &item->mtime,
3866+ inode->i_mtime.tv_sec);
3867+ btrfs_set_token_timespec_nsec(&token, &item->mtime,
3868+ inode->i_mtime.tv_nsec);
3869+
3870+ btrfs_set_token_timespec_sec(&token, &item->ctime,
3871+ inode->i_ctime.tv_sec);
3872+ btrfs_set_token_timespec_nsec(&token, &item->ctime,
3873+ inode->i_ctime.tv_nsec);
3874+
3875+ btrfs_set_token_inode_nbytes(&token, item, inode_get_bytes(inode));
3876+
3877+ btrfs_set_token_inode_sequence(&token, item, inode_peek_iversion(inode));
3878+ btrfs_set_token_inode_transid(&token, item, trans->transid);
3879+ btrfs_set_token_inode_rdev(&token, item, inode->i_rdev);
3880+ btrfs_set_token_inode_flags(&token, item, BTRFS_I(inode)->flags);
3881+ btrfs_set_token_inode_block_group(&token, item, 0);
38943882 }
38953883
38963884 static int log_inode_item(struct btrfs_trans_handle *trans,
@@ -3916,8 +3904,20 @@ static int log_csums(struct btrfs_trans_handle *trans, · 19 unmodified lines
39163904 struct btrfs_root *log_root,
39173905 struct btrfs_ordered_sum *sums)
39183906 {
3907+ const u64 lock_end = sums->bytenr + sums->len - 1;
3908+ struct extent_state *cached_state = NULL;
39193909 int ret;
39203910
3911+ /*
3912+ * Serialize logging for checksums. This is to avoid racing with the
3913+ * same checksum being logged by another task that is logging another
3914+ * file which happens to refer to the same extent as well. Such races
3915+ * can leave checksum items in the log with overlapping ranges.
3916+ */
3917+ ret = lock_extent_bits(&log_root->log_csum_range, sums->bytenr,
3918+ lock_end, &cached_state);
3919+ if (ret)
3920+ return ret;
39213921 /*
39223922 * Due to extent cloning, we might have logged a csum item that covers a
39233923 * subrange of a cloned extent, and later we can end up logging a csum
@@ -3928,10 +3928,13 @@ static int log_csums(struct btrfs_trans_handle *trans, · 4 unmodified lines
39283928 * trim and adjust) any existing csum items in the log for this range.
39293929 */
39303930 ret = btrfs_del_csums(trans, log_root, sums->bytenr, sums->len);
3931− if (ret)
3932− return ret;
3931+ if (!ret)
3932+ ret = btrfs_csum_file_blocks(trans, log_root, sums);
39333933
3934− return btrfs_csum_file_blocks(trans, log_root, sums);
3934+ unlock_extent_cached(&log_root->log_csum_range, sums->bytenr, lock_end,
3935+ &cached_state);
3936+
3937+ return ret;
39353938 }
39363939
39373940 static noinline int copy_items(struct btrfs_trans_handle *trans,
@@ -4164,43 +4167,35 @@ static int log_one_extent(struct btrfs_trans_handle *trans, · 226 unmodified lines
41644167 fi = btrfs_item_ptr(leaf, path->slots[0],
41654168 struct btrfs_file_extent_item);
41664169
4167− btrfs_set_token_file_extent_generation(leaf, fi, trans->transid,
4168− &token);
4170+ btrfs_set_token_file_extent_generation(&token, fi, trans->transid);
41694171 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
4170− btrfs_set_token_file_extent_type(leaf, fi,
4171− BTRFS_FILE_EXTENT_PREALLOC,
4172− &token);
4172+ btrfs_set_token_file_extent_type(&token, fi,
4173+ BTRFS_FILE_EXTENT_PREALLOC);
41734174 else
4174− btrfs_set_token_file_extent_type(leaf, fi,
4175− BTRFS_FILE_EXTENT_REG,
4176− &token);
4175+ btrfs_set_token_file_extent_type(&token, fi,
4176+ BTRFS_FILE_EXTENT_REG);
41774177
41784178 block_len = max(em->block_len, em->orig_block_len);
41794179 if (em->compress_type != BTRFS_COMPRESS_NONE) {
4180− btrfs_set_token_file_extent_disk_bytenr(leaf, fi,
4181− em->block_start,
4182− &token);
4183− btrfs_set_token_file_extent_disk_num_bytes(leaf, fi, block_len,
4184− &token);
4180+ btrfs_set_token_file_extent_disk_bytenr(&token, fi,
4181+ em->block_start);
4182+ btrfs_set_token_file_extent_disk_num_bytes(&token, fi, block_len);
41854183 } else if (em->block_start < EXTENT_MAP_LAST_BYTE) {
4186− btrfs_set_token_file_extent_disk_bytenr(leaf, fi,
4184+ btrfs_set_token_file_extent_disk_bytenr(&token, fi,
41874185 em->block_start -
4188− extent_offset, &token);
4189− btrfs_set_token_file_extent_disk_num_bytes(leaf, fi, block_len,
4190− &token);
4186+ extent_offset);
4187+ btrfs_set_token_file_extent_disk_num_bytes(&token, fi, block_len);
41914188 } else {
4192− btrfs_set_token_file_extent_disk_bytenr(leaf, fi, 0, &token);
4193− btrfs_set_token_file_extent_disk_num_bytes(leaf, fi, 0,
4194− &token);
4195− }
4196−
4197− btrfs_set_token_file_extent_offset(leaf, fi, extent_offset, &token);
4198− btrfs_set_token_file_extent_num_bytes(leaf, fi, em->len, &token);
4199− btrfs_set_token_file_extent_ram_bytes(leaf, fi, em->ram_bytes, &token);
4200− btrfs_set_token_file_extent_compression(leaf, fi, em->compress_type,
4201− &token);
4202− btrfs_set_token_file_extent_encryption(leaf, fi, 0, &token);
4203− btrfs_set_token_file_extent_other_encoding(leaf, fi, 0, &token);
4189+ btrfs_set_token_file_extent_disk_bytenr(&token, fi, 0);
4190+ btrfs_set_token_file_extent_disk_num_bytes(&token, fi, 0);
4191+ }
4192+
4193+ btrfs_set_token_file_extent_offset(&token, fi, extent_offset);
4194+ btrfs_set_token_file_extent_num_bytes(&token, fi, em->len);
4195+ btrfs_set_token_file_extent_ram_bytes(&token, fi, em->ram_bytes);
4196+ btrfs_set_token_file_extent_compression(&token, fi, em->compress_type);
4197+ btrfs_set_token_file_extent_encryption(&token, fi, 0);
4198+ btrfs_set_token_file_extent_other_encoding(&token, fi, 0);
42044199 btrfs_mark_buffer_dirty(leaf);
42054200
42064201 btrfs_release_path(path);
@@ -4336,12 +4331,9 @@ static int btrfs_log_prealloc_extents(struct btrfs_trans_handle *trans, · 129 unmodified lines
43364331 }
43374332 }
43384333 }
4339− if (ins_nr > 0) {
4334+ if (ins_nr > 0)
43404335 ret = copy_items(trans, inode, dst_path, path,
43414336 start_slot, ins_nr, 1, 0);
4342− if (ret > 0)
4343− ret = 0;
4344− }
43454337 out:
43464338 btrfs_release_path(path);
43474339 btrfs_free_path(dst_path);
@@ -4835,10 +4827,7 @@ static int log_conflicting_inodes(struct btrfs_trans_handle *trans, · 487 unmodified lines
48354827
48364828 btrfs_release_path(path);
48374829
4838− key.objectid = ino;
4839− key.type = BTRFS_INODE_ITEM_KEY;
4840− key.offset = 0;
4841− inode = btrfs_iget(fs_info->sb, &key, root);
4830+ inode = btrfs_iget(fs_info->sb, ino, root);
48424831 /*
48434832 * If the other inode that had a conflicting dir entry was
48444833 * deleted in the current transaction, we need to log its parent
@@ -4847,8 +4836,7 @@ static int log_conflicting_inodes(struct btrfs_trans_handle *trans, · 2 unmodified lines
48474836 if (IS_ERR(inode)) {
48484837 ret = PTR_ERR(inode);
48494838 if (ret == -ENOENT) {
4850− key.objectid = parent;
4851− inode = btrfs_iget(fs_info->sb, &key, root);
4839+ inode = btrfs_iget(fs_info->sb, parent, root);
48524840 if (IS_ERR(inode)) {
48534841 ret = PTR_ERR(inode);
48544842 } else {
@@ -5587,7 +5575,7 @@ process_leaf: · 732 unmodified lines
55875575 continue;
55885576
55895577 btrfs_release_path(path);
5590− di_inode = btrfs_iget(fs_info->sb, &di_key, root);
5578+ di_inode = btrfs_iget(fs_info->sb, di_key.objectid, root);
55915579 if (IS_ERR(di_inode)) {
55925580 ret = PTR_ERR(di_inode);
55935581 goto next_dir_inode;
@@ -5713,7 +5701,8 @@ static int btrfs_log_all_parents(struct btrfs_trans_handle *trans, · 119 unmodified lines
57135701 cur_offset = item_size;
57145702 }
57155703
5716− dir_inode = btrfs_iget(fs_info->sb, &inode_key, root);
5704+ dir_inode = btrfs_iget(fs_info->sb, inode_key.objectid,
5705+ root);
57175706 /*
57185707 * If the parent inode was deleted, return an error to
57195708 * fallback to a transaction commit. This is to prevent
@@ -5780,14 +5769,17 @@ static int log_new_ancestors(struct btrfs_trans_handle *trans, · 60 unmodified lines
57805769 int slot = path->slots[0];
57815770 struct btrfs_key search_key;
57825771 struct inode *inode;
5772+ u64 ino;
57835773 int ret = 0;
57845774
57855775 btrfs_release_path(path);
57865776
5777+ ino = found_key.offset;
5778+
57875779 search_key.objectid = found_key.offset;
57885780 search_key.type = BTRFS_INODE_ITEM_KEY;
57895781 search_key.offset = 0;
5790− inode = btrfs_iget(fs_info->sb, &search_key, root);
5782+ inode = btrfs_iget(fs_info->sb, ino, root);
57915783 if (IS_ERR(inode))
57925784 return PTR_ERR(inode);
57935785
@@ -6132,7 +6124,6 @@ int btrfs_recover_log_trees(struct btrfs_root *log_root_tree) · 338 unmodified lines
61326124 struct btrfs_trans_handle *trans;
61336125 struct btrfs_key key;
61346126 struct btrfs_key found_key;
6135− struct btrfs_key tmp_key;
61366127 struct btrfs_root *log;
61376128 struct btrfs_fs_info *fs_info = log_root_tree->fs_info;
61386129 struct walk_control wc = {
@@ -6194,11 +6185,8 @@ again: · 55 unmodified lines
61946185 goto error;
61956186 }
61966187
6197− tmp_key.objectid = found_key.offset;
6198− tmp_key.type = BTRFS_ROOT_ITEM_KEY;
6199− tmp_key.offset = (u64)-1;
6200−
6201− wc.replay_dest = btrfs_get_fs_root(fs_info, &tmp_key, true);
6188+ wc.replay_dest = btrfs_get_fs_root(fs_info, found_key.offset,
6189+ true);
62026190 if (IS_ERR(wc.replay_dest)) {
62036191 ret = PTR_ERR(wc.replay_dest);
62046192
fs/btrfs/uuid-tree.c+1−5
@@ -257,7 +257,6 @@ out: · 256 unmodified lines
257257 static int btrfs_check_uuid_tree_entry(struct btrfs_fs_info *fs_info,
258258 u8 *uuid, u8 type, u64 subvolid)
259259 {
260− struct btrfs_key key;
261260 int ret = 0;
262261 struct btrfs_root *subvol_root;
263262
@@ -265,10 +264,7 @@ static int btrfs_check_uuid_tree_entry(struct btrfs_fs_info *fs_info, · 1 unmodified line
265264 type != BTRFS_UUID_KEY_RECEIVED_SUBVOL)
266265 goto out;
267266
268− key.objectid = subvolid;
269− key.type = BTRFS_ROOT_ITEM_KEY;
270− key.offset = (u64)-1;
271− subvol_root = btrfs_get_fs_root(fs_info, &key, true);
267+ subvol_root = btrfs_get_fs_root(fs_info, subvolid, true);
272268 if (IS_ERR(subvol_root)) {
273269 ret = PTR_ERR(subvol_root);
274270 if (ret == -ENOENT)
fs/btrfs/volumes.c+44−36
@@ -280,10 +280,9 @@ static int __btrfs_map_block(struct btrfs_fs_info *fs_info, · 279 unmodified lines
280280 * ============
281281 *
282282 * uuid_mutex
283− * volume_mutex
284− * device_list_mutex
285− * chunk_mutex
286− * balance_mutex
283+ * device_list_mutex
284+ * chunk_mutex
285+ * balance_mutex
287286 *
288287 *
289288 * Exclusive operations, BTRFS_FS_EXCL_OP
@@ -1042,6 +1041,8 @@ again: · 752 unmodified lines
10421041 &device->dev_state)) {
10431042 if (!test_bit(BTRFS_DEV_STATE_REPLACE_TGT,
10441043 &device->dev_state) &&
1044+ !test_bit(BTRFS_DEV_STATE_MISSING,
1045+ &device->dev_state) &&
10451046 (!latest_dev ||
10461047 device->generation > latest_dev->generation)) {
10471048 latest_dev = device;
@@ -1185,7 +1186,6 @@ static int open_fs_devices(struct btrfs_fs_devices *fs_devices, · 137 unmodified lines
11851186 {
11861187 struct btrfs_device *device;
11871188 struct btrfs_device *latest_dev = NULL;
1188− int ret = 0;
11891189
11901190 flags |= FMODE_EXCL;
11911191
@@ -1198,16 +1198,15 @@ static int open_fs_devices(struct btrfs_fs_devices *fs_devices, · 6 unmodified lines
11981198 device->generation > latest_dev->generation)
11991199 latest_dev = device;
12001200 }
1201− if (fs_devices->open_devices == 0) {
1202− ret = -EINVAL;
1203− goto out;
1204− }
1201+ if (fs_devices->open_devices == 0)
1202+ return -EINVAL;
1203+
12051204 fs_devices->opened = 1;
12061205 fs_devices->latest_bdev = latest_dev->bdev;
12071206 fs_devices->total_rw_bytes = 0;
12081207 fs_devices->chunk_alloc_policy = BTRFS_CHUNK_ALLOC_REGULAR;
1209−out:
1210− return ret;
1208+
1209+ return 0;
12111210 }
12121211
12131212 static int devid_cmp(void *priv, struct list_head *a, struct list_head *b)
@@ -1251,49 +1250,48 @@ void btrfs_release_disk_super(struct btrfs_super_block *super) · 37 unmodified lines
12511250 put_page(page);
12521251 }
12531252
1254−static int btrfs_read_disk_super(struct block_device *bdev, u64 bytenr,
1255− struct page **page,
1256− struct btrfs_super_block **disk_super)
1253+static struct btrfs_super_block *btrfs_read_disk_super(struct block_device *bdev,
1254+ u64 bytenr)
12571255 {
1256+ struct btrfs_super_block *disk_super;
1257+ struct page *page;
12581258 void *p;
12591259 pgoff_t index;
12601260
12611261 /* make sure our super fits in the device */
12621262 if (bytenr + PAGE_SIZE >= i_size_read(bdev->bd_inode))
1263− return 1;
1263+ return ERR_PTR(-EINVAL);
12641264
12651265 /* make sure our super fits in the page */
1266− if (sizeof(**disk_super) > PAGE_SIZE)
1267− return 1;
1266+ if (sizeof(*disk_super) > PAGE_SIZE)
1267+ return ERR_PTR(-EINVAL);
12681268
12691269 /* make sure our super doesn't straddle pages on disk */
12701270 index = bytenr >> PAGE_SHIFT;
1271− if ((bytenr + sizeof(**disk_super) - 1) >> PAGE_SHIFT != index)
1272− return 1;
1271+ if ((bytenr + sizeof(*disk_super) - 1) >> PAGE_SHIFT != index)
1272+ return ERR_PTR(-EINVAL);
12731273
12741274 /* pull in the page with our super */
1275− *page = read_cache_page_gfp(bdev->bd_inode->i_mapping,
1276− index, GFP_KERNEL);
1275+ page = read_cache_page_gfp(bdev->bd_inode->i_mapping, index, GFP_KERNEL);
12771276
1278− if (IS_ERR(*page))
1279− return 1;
1277+ if (IS_ERR(page))
1278+ return ERR_CAST(page);
12801279
1281− p = page_address(*page);
1280+ p = page_address(page);
12821281
12831282 /* align our pointer to the offset of the super block */
1284− *disk_super = p + offset_in_page(bytenr);
1283+ disk_super = p + offset_in_page(bytenr);
12851284
1286− if (btrfs_super_bytenr(*disk_super) != bytenr ||
1287− btrfs_super_magic(*disk_super) != BTRFS_MAGIC) {
1285+ if (btrfs_super_bytenr(disk_super) != bytenr ||
1286+ btrfs_super_magic(disk_super) != BTRFS_MAGIC) {
12881287 btrfs_release_disk_super(p);
1289− return 1;
1288+ return ERR_PTR(-EINVAL);
12901289 }
12911290
1292− if ((*disk_super)->label[0] &&
1293− (*disk_super)->label[BTRFS_LABEL_SIZE - 1])
1294− (*disk_super)->label[BTRFS_LABEL_SIZE - 1] = '\0';
1291+ if (disk_super->label[0] && disk_super->label[BTRFS_LABEL_SIZE - 1])
1292+ disk_super->label[BTRFS_LABEL_SIZE - 1] = 0;
12951293
1296− return 0;
1294+ return disk_super;
12971295 }
12981296
12991297 int btrfs_forget_devices(const char *path)
@@ -1319,7 +1317,6 @@ struct btrfs_device *btrfs_scan_one_device(const char *path, fmode_t flags, · 19 unmodified lines
13191317 bool new_device_added = false;
13201318 struct btrfs_device *device = NULL;
13211319 struct block_device *bdev;
1322− struct page *page;
13231320 u64 bytenr;
13241321
13251322 lockdep_assert_held(&uuid_mutex);
@@ -1337,8 +1334,9 @@ struct btrfs_device *btrfs_scan_one_device(const char *path, fmode_t flags, · 11 unmodified lines
13371334 if (IS_ERR(bdev))
13381335 return ERR_CAST(bdev);
13391336
1340− if (btrfs_read_disk_super(bdev, bytenr, &page, &disk_super)) {
1341− device = ERR_PTR(-EINVAL);
1337+ disk_super = btrfs_read_disk_super(bdev, bytenr);
1338+ if (IS_ERR(disk_super)) {
1339+ device = ERR_CAST(disk_super);
13421340 goto error_bdev_put;
13431341 }
13441342
@@ -2663,8 +2661,18 @@ int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path · 1318 unmodified lines
26632661 ret = btrfs_commit_transaction(trans);
26642662 }
26652663
2666− /* Update ctime/mtime for libblkid */
2664+ /*
2665+ * Now that we have written a new super block to this device, check all
2666+ * other fs_devices list if device_path alienates any other scanned
2667+ * device.
2668+ * We can ignore the return value as it typically returns -EINVAL and
2669+ * only succeeds if the device was an alien.
2670+ */
2671+ btrfs_forget_devices(device_path);
2672+
2673+ /* Update ctime/mtime for blkid or udev */
26672674 update_dev_time(device_path);
2675+
26682676 return ret;
26692677
26702678 error_sysfs:
fs/direct-io.c+0−19
@@ -386,25 +386,6 @@ static void dio_bio_end_io(struct bio *bio) · 385 unmodified lines
386386 spin_unlock_irqrestore(&dio->bio_lock, flags);
387387 }
388388
389−/**
390− * dio_end_io - handle the end io action for the given bio
391− * @bio: The direct io bio thats being completed
392− *
393− * This is meant to be called by any filesystem that uses their own dio_submit_t
394− * so that the DIO specific endio actions are dealt with after the filesystem
395− * has done it's completion work.
396− */
397−void dio_end_io(struct bio *bio)
398−{
399− struct dio *dio = bio->bi_private;
400−
401− if (dio->is_async)
402− dio_bio_end_aio(bio);
403− else
404− dio_bio_end_io(bio);
405−}
406−EXPORT_SYMBOL_GPL(dio_end_io);
407−
408389 static inline void
409390 dio_bio_alloc(struct dio *dio, struct dio_submit *sdio,
410391 struct block_device *bdev,
fs/iomap/direct-io.c+10−7
@@ -59,7 +59,7 @@ int iomap_dio_iopoll(struct kiocb *kiocb, bool spin) · 58 unmodified lines
5959 EXPORT_SYMBOL_GPL(iomap_dio_iopoll);
6060
6161 static void iomap_dio_submit_bio(struct iomap_dio *dio, struct iomap *iomap,
62− struct bio *bio)
62+ struct bio *bio, loff_t pos)
6363 {
6464 atomic_inc(&dio->ref);
6565
@@ -67,7 +67,12 @@ static void iomap_dio_submit_bio(struct iomap_dio *dio, struct iomap *iomap, · 1 unmodified line
6767 bio_set_polled(bio, dio->iocb);
6868
6969 dio->submit.last_queue = bdev_get_queue(iomap->bdev);
70− dio->submit.cookie = submit_bio(bio);
70+ if (dio->dops && dio->dops->submit_io)
71+ dio->submit.cookie = dio->dops->submit_io(
72+ file_inode(dio->iocb->ki_filp),
73+ iomap, bio, pos);
74+ else
75+ dio->submit.cookie = submit_bio(bio);
7176 }
7277
7378 static ssize_t iomap_dio_complete(struct iomap_dio *dio)
@@ -191,7 +196,7 @@ iomap_dio_zero(struct iomap_dio *dio, struct iomap *iomap, loff_t pos, · 117 unmodified lines
191196 get_page(page);
192197 __bio_add_page(bio, page, len, 0);
193198 bio_set_op_attrs(bio, REQ_OP_WRITE, flags);
194− iomap_dio_submit_bio(dio, iomap, bio);
199+ iomap_dio_submit_bio(dio, iomap, bio, pos);
195200 }
196201
197202 static loff_t
@@ -299,11 +304,11 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length, · 101 unmodified lines
299304 }
300305
301306 dio->size += n;
302− pos += n;
303307 copied += n;
304308
305309 nr_pages = iov_iter_npages(dio->submit.iter, BIO_MAX_PAGES);
306− iomap_dio_submit_bio(dio, iomap, bio);
310+ iomap_dio_submit_bio(dio, iomap, bio, pos);
311+ pos += n;
307312 } while (nr_pages);
308313
309314 /*
@@ -411,8 +416,6 @@ iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, · 101 unmodified lines
411416 struct blk_plug plug;
412417 struct iomap_dio *dio;
413418
414− lockdep_assert_held(&inode->i_rwsem);
415−
416419 if (!count)
417420 return 0;
418421
include/linux/bio.h+8−0
@@ -169,6 +169,14 @@ static inline void bio_advance_iter(const struct bio *bio, · 168 unmodified lines
169169 #define bio_for_each_bvec(bvl, bio, iter) \
170170 __bio_for_each_bvec(bvl, bio, iter, (bio)->bi_iter)
171171
172+/*
173+ * Iterate over all multi-page bvecs. Drivers shouldn't use this version for the
174+ * same reasons as bio_for_each_segment_all().
175+ */
176+#define bio_for_each_bvec_all(bvl, bio, i) \
177+ for (i = 0, bvl = bio_first_bvec_all(bio); \
178+ i < (bio)->bi_vcnt; i++, bvl++) \
179+
172180 #define bio_iter_last(bvec, iter) ((iter).bi_size == (bvec).bv_len)
173181
174182 static inline unsigned bio_segments(struct bio *bio)
include/linux/fs.h+2−2
@@ -3148,6 +3148,8 @@ extern int generic_file_rw_checks(struct file *file_in, struct file *file_out); · 3147 unmodified lines
31483148 extern int generic_copy_file_checks(struct file *file_in, loff_t pos_in,
31493149 struct file *file_out, loff_t pos_out,
31503150 size_t *count, unsigned int flags);
3151+extern ssize_t generic_file_buffered_read(struct kiocb *iocb,
3152+ struct iov_iter *to, ssize_t already_read);
31513153 extern ssize_t generic_file_read_iter(struct kiocb *, struct iov_iter *);
31523154 extern ssize_t __generic_file_write_iter(struct kiocb *, struct iov_iter *);
31533155 extern ssize_t generic_file_write_iter(struct kiocb *, struct iov_iter *);
@@ -3209,8 +3211,6 @@ enum { · 55 unmodified lines
32093211 DIO_SKIP_HOLES = 0x02,
32103212 };
32113213
3212−void dio_end_io(struct bio *bio);
3213−
32143214 ssize_t __blockdev_direct_IO(struct kiocb *iocb, struct inode *inode,
32153215 struct block_device *bdev, struct iov_iter *iter,
32163216 get_block_t get_block,
include/linux/iomap.h+2−0
@@ -251,6 +251,8 @@ int iomap_writepages(struct address_space *mapping, · 250 unmodified lines
251251 struct iomap_dio_ops {
252252 int (*end_io)(struct kiocb *iocb, ssize_t size, int error,
253253 unsigned flags);
254+ blk_qc_t (*submit_io)(struct inode *inode, struct iomap *iomap,
255+ struct bio *bio, loff_t file_offset);
254256 };
255257
256258 ssize_t iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
include/trace/events/btrfs.h+1−0
@@ -89,6 +89,7 @@ TRACE_DEFINE_ENUM(COMMIT_TRANS); · 88 unmodified lines
8989 { IO_TREE_TRANS_DIRTY_PAGES, "TRANS_DIRTY_PAGES" }, \
9090 { IO_TREE_ROOT_DIRTY_LOG_PAGES, "ROOT_DIRTY_LOG_PAGES" }, \
9191 { IO_TREE_INODE_FILE_EXTENT, "INODE_FILE_EXTENT" }, \
92+ { IO_TREE_LOG_CSUM_RANGE, "LOG_CSUM_RANGE" }, \
9293 { IO_TREE_SELFTEST, "SELFTEST" })
9394
9495 #define BTRFS_GROUP_FLAGS \
include/uapi/linux/btrfs_tree.h+0−9
@@ -519,15 +519,6 @@ struct btrfs_extent_inline_ref { · 518 unmodified lines
519519 __le64 offset;
520520 } __attribute__ ((__packed__));
521521
522−/* old style backrefs item */
523−struct btrfs_extent_ref_v0 {
524− __le64 root;
525− __le64 generation;
526− __le64 objectid;
527− __le32 count;
528−} __attribute__ ((__packed__));
529−
530−
531522 /* dev extents record free space on individual devices. The owner
532523 * field points back to the chunk allocation mapping tree that allocated
533524 * the extent. The chunk tree uuid field is a way to double check the owner
mm/filemap.c+2−1
@@ -1991,7 +1991,7 @@ static void shrink_readahead_size_eio(struct file_ra_state *ra) · 1990 unmodified lines
19911991 * * total number of bytes copied, including those the were already @written
19921992 * * negative error code if nothing was copied
19931993 */
1994−static ssize_t generic_file_buffered_read(struct kiocb *iocb,
1994+ssize_t generic_file_buffered_read(struct kiocb *iocb,
19951995 struct iov_iter *iter, ssize_t written)
19961996 {
19971997 struct file *filp = iocb->ki_filp;
@@ -2243,6 +2243,7 @@ out: · 245 unmodified lines
22432243 file_accessed(filp);
22442244 return written ? written : error;
22452245 }
2246+EXPORT_SYMBOL_GPL(generic_file_buffered_read);
22462247
22472248 /**
22482249 * generic_file_read_iter - generic filesystem read routine