| @@ -5,6 +5,7 @@ · 4 unmodified lines |
| 5 | 5 | |
| 6 | 6 | #include <linux/fs.h> |
| 7 | 7 | #include <linux/blkdev.h> |
| 8 | +#include <linux/radix-tree.h> |
| 8 | 9 | #include <linux/writeback.h> |
| 9 | 10 | #include <linux/workqueue.h> |
| 10 | 11 | #include <linux/kthread.h> |
| @@ -1210,9 +1211,9 @@ static void __setup_root(struct btrfs_root *root, struct btrfs_fs_info *fs_info, · 1199 unmodified lines |
| 1210 | 1211 | btrfs_qgroup_init_swapped_blocks(&root->swapped_blocks); |
| 1211 | 1212 | #ifdef CONFIG_BTRFS_DEBUG |
| 1212 | 1213 | INIT_LIST_HEAD(&root->leak_list); |
| 1213 | | − spin_lock(&fs_info->fs_roots_lock); |
| 1214 | + spin_lock(&fs_info->fs_roots_radix_lock); |
| 1214 | 1215 | list_add_tail(&root->leak_list, &fs_info->allocated_roots); |
| 1215 | | − spin_unlock(&fs_info->fs_roots_lock); |
| 1216 | + spin_unlock(&fs_info->fs_roots_radix_lock); |
| 1216 | 1217 | #endif |
| 1217 | 1218 | } |
| 1218 | 1219 | |
| @@ -1659,11 +1660,12 @@ static struct btrfs_root *btrfs_lookup_fs_root(struct btrfs_fs_info *fs_info, · 440 unmodified lines |
| 1659 | 1660 | { |
| 1660 | 1661 | struct btrfs_root *root; |
| 1661 | 1662 | |
| 1662 | | − spin_lock(&fs_info->fs_roots_lock); |
| 1663 | | − root = xa_load(&fs_info->fs_roots, (unsigned long)root_id); |
| 1663 | + spin_lock(&fs_info->fs_roots_radix_lock); |
| 1664 | + root = radix_tree_lookup(&fs_info->fs_roots_radix, |
| 1665 | + (unsigned long)root_id); |
| 1664 | 1666 | if (root) |
| 1665 | 1667 | root = btrfs_grab_root(root); |
| 1666 | | − spin_unlock(&fs_info->fs_roots_lock); |
| 1668 | + spin_unlock(&fs_info->fs_roots_radix_lock); |
| 1667 | 1669 | return root; |
| 1668 | 1670 | } |
| 1669 | 1671 | |
| @@ -1705,14 +1707,20 @@ int btrfs_insert_fs_root(struct btrfs_fs_info *fs_info, · 35 unmodified lines |
| 1705 | 1707 | { |
| 1706 | 1708 | int ret; |
| 1707 | 1709 | |
| 1708 | | − spin_lock(&fs_info->fs_roots_lock); |
| 1709 | | − ret = xa_insert(&fs_info->fs_roots, (unsigned long)root->root_key.objectid, |
| 1710 | | − root, GFP_NOFS); |
| 1710 | + ret = radix_tree_preload(GFP_NOFS); |
| 1711 | + if (ret) |
| 1712 | + return ret; |
| 1713 | + |
| 1714 | + spin_lock(&fs_info->fs_roots_radix_lock); |
| 1715 | + ret = radix_tree_insert(&fs_info->fs_roots_radix, |
| 1716 | + (unsigned long)root->root_key.objectid, |
| 1717 | + root); |
| 1711 | 1718 | if (ret == 0) { |
| 1712 | 1719 | btrfs_grab_root(root); |
| 1713 | | − set_bit(BTRFS_ROOT_REGISTERED, &root->state); |
| 1720 | + set_bit(BTRFS_ROOT_IN_RADIX, &root->state); |
| 1714 | 1721 | } |
| 1715 | | − spin_unlock(&fs_info->fs_roots_lock); |
| 1722 | + spin_unlock(&fs_info->fs_roots_radix_lock); |
| 1723 | + radix_tree_preload_end(); |
| 1716 | 1724 | |
| 1717 | 1725 | return ret; |
| 1718 | 1726 | } |
| @@ -2342,9 +2350,9 @@ void btrfs_put_root(struct btrfs_root *root) · 623 unmodified lines |
| 2342 | 2350 | btrfs_drew_lock_destroy(&root->snapshot_lock); |
| 2343 | 2351 | free_root_extent_buffers(root); |
| 2344 | 2352 | #ifdef CONFIG_BTRFS_DEBUG |
| 2345 | | − spin_lock(&root->fs_info->fs_roots_lock); |
| 2353 | + spin_lock(&root->fs_info->fs_roots_radix_lock); |
| 2346 | 2354 | list_del_init(&root->leak_list); |
| 2347 | | − spin_unlock(&root->fs_info->fs_roots_lock); |
| 2355 | + spin_unlock(&root->fs_info->fs_roots_radix_lock); |
| 2348 | 2356 | #endif |
| 2349 | 2357 | kfree(root); |
| 2350 | 2358 | } |
| @@ -2352,21 +2360,28 @@ void btrfs_put_root(struct btrfs_root *root) · 1 unmodified line |
| 2352 | 2360 | |
| 2353 | 2361 | void btrfs_free_fs_roots(struct btrfs_fs_info *fs_info) |
| 2354 | 2362 | { |
| 2355 | | − struct btrfs_root *root; |
| 2356 | | − unsigned long index = 0; |
| 2363 | + int ret; |
| 2364 | + struct btrfs_root *gang[8]; |
| 2365 | + int i; |
| 2357 | 2366 | |
| 2358 | 2367 | while (!list_empty(&fs_info->dead_roots)) { |
| 2359 | | − root = list_entry(fs_info->dead_roots.next, |
| 2360 | | − struct btrfs_root, root_list); |
| 2361 | | − list_del(&root->root_list); |
| 2368 | + gang[0] = list_entry(fs_info->dead_roots.next, |
| 2369 | + struct btrfs_root, root_list); |
| 2370 | + list_del(&gang[0]->root_list); |
| 2362 | 2371 | |
| 2363 | | − if (test_bit(BTRFS_ROOT_REGISTERED, &root->state)) |
| 2364 | | − btrfs_drop_and_free_fs_root(fs_info, root); |
| 2365 | | − btrfs_put_root(root); |
| 2372 | + if (test_bit(BTRFS_ROOT_IN_RADIX, &gang[0]->state)) |
| 2373 | + btrfs_drop_and_free_fs_root(fs_info, gang[0]); |
| 2374 | + btrfs_put_root(gang[0]); |
| 2366 | 2375 | } |
| 2367 | 2376 | |
| 2368 | | − xa_for_each(&fs_info->fs_roots, index, root) { |
| 2369 | | − btrfs_drop_and_free_fs_root(fs_info, root); |
| 2377 | + while (1) { |
| 2378 | + ret = radix_tree_gang_lookup(&fs_info->fs_roots_radix, |
| 2379 | + (void **)gang, 0, |
| 2380 | + ARRAY_SIZE(gang)); |
| 2381 | + if (!ret) |
| 2382 | + break; |
| 2383 | + for (i = 0; i < ret; i++) |
| 2384 | + btrfs_drop_and_free_fs_root(fs_info, gang[i]); |
| 2370 | 2385 | } |
| 2371 | 2386 | } |
| 2372 | 2387 | |
| @@ -3134,7 +3149,7 @@ static int __cold init_tree_roots(struct btrfs_fs_info *fs_info) · 761 unmodified lines |
| 3134 | 3149 | |
| 3135 | 3150 | void btrfs_init_fs_info(struct btrfs_fs_info *fs_info) |
| 3136 | 3151 | { |
| 3137 | | − xa_init_flags(&fs_info->fs_roots, GFP_ATOMIC); |
| 3152 | + INIT_RADIX_TREE(&fs_info->fs_roots_radix, GFP_ATOMIC); |
| 3138 | 3153 | xa_init_flags(&fs_info->extent_buffers, GFP_ATOMIC); |
| 3139 | 3154 | INIT_LIST_HEAD(&fs_info->trans_list); |
| 3140 | 3155 | INIT_LIST_HEAD(&fs_info->dead_roots); |
| @@ -3143,7 +3158,7 @@ void btrfs_init_fs_info(struct btrfs_fs_info *fs_info) · 2 unmodified lines |
| 3143 | 3158 | INIT_LIST_HEAD(&fs_info->caching_block_groups); |
| 3144 | 3159 | spin_lock_init(&fs_info->delalloc_root_lock); |
| 3145 | 3160 | spin_lock_init(&fs_info->trans_lock); |
| 3146 | | − spin_lock_init(&fs_info->fs_roots_lock); |
| 3161 | + spin_lock_init(&fs_info->fs_roots_radix_lock); |
| 3147 | 3162 | spin_lock_init(&fs_info->delayed_iput_lock); |
| 3148 | 3163 | spin_lock_init(&fs_info->defrag_inodes_lock); |
| 3149 | 3164 | spin_lock_init(&fs_info->super_lock); |
| @@ -3374,7 +3389,7 @@ int btrfs_start_pre_rw_mount(struct btrfs_fs_info *fs_info) · 224 unmodified lines |
| 3374 | 3389 | /* |
| 3375 | 3390 | * btrfs_find_orphan_roots() is responsible for finding all the dead |
| 3376 | 3391 | * roots (with 0 refs), flag them with BTRFS_ROOT_DEAD_TREE and load |
| 3377 | | − * them into the fs_info->fs_roots. This must be done before |
| 3392 | + * them into the fs_info->fs_roots_radix tree. This must be done before |
| 3378 | 3393 | * calling btrfs_orphan_cleanup() on the tree root. If we don't do it |
| 3379 | 3394 | * first, then btrfs_orphan_cleanup() will delete a dead root's orphan |
| 3380 | 3395 | * item before the root's tree is deleted - this means that if we unmount |
| @@ -4498,11 +4513,12 @@ void btrfs_drop_and_free_fs_root(struct btrfs_fs_info *fs_info, · 1117 unmodified lines |
| 4498 | 4513 | { |
| 4499 | 4514 | bool drop_ref = false; |
| 4500 | 4515 | |
| 4501 | | − spin_lock(&fs_info->fs_roots_lock); |
| 4502 | | − xa_erase(&fs_info->fs_roots, (unsigned long)root->root_key.objectid); |
| 4503 | | − if (test_and_clear_bit(BTRFS_ROOT_REGISTERED, &root->state)) |
| 4516 | + spin_lock(&fs_info->fs_roots_radix_lock); |
| 4517 | + radix_tree_delete(&fs_info->fs_roots_radix, |
| 4518 | + (unsigned long)root->root_key.objectid); |
| 4519 | + if (test_and_clear_bit(BTRFS_ROOT_IN_RADIX, &root->state)) |
| 4504 | 4520 | drop_ref = true; |
| 4505 | | − spin_unlock(&fs_info->fs_roots_lock); |
| 4521 | + spin_unlock(&fs_info->fs_roots_radix_lock); |
| 4506 | 4522 | |
| 4507 | 4523 | if (BTRFS_FS_ERROR(fs_info)) { |
| 4508 | 4524 | ASSERT(root->log_root == NULL); |
| @@ -4518,48 +4534,50 @@ void btrfs_drop_and_free_fs_root(struct btrfs_fs_info *fs_info, · 9 unmodified lines |
| 4518 | 4534 | |
| 4519 | 4535 | int btrfs_cleanup_fs_roots(struct btrfs_fs_info *fs_info) |
| 4520 | 4536 | { |
| 4521 | | − struct btrfs_root *roots[8]; |
| 4522 | | − unsigned long index = 0; |
| 4523 | | − int i; |
| 4537 | + u64 root_objectid = 0; |
| 4538 | + struct btrfs_root *gang[8]; |
| 4539 | + int i = 0; |
| 4524 | 4540 | int err = 0; |
| 4525 | | − int grabbed; |
| 4541 | + unsigned int ret = 0; |
| 4526 | 4542 | |
| 4527 | 4543 | while (1) { |
| 4528 | | − struct btrfs_root *root; |
| 4529 | | − |
| 4530 | | − spin_lock(&fs_info->fs_roots_lock); |
| 4531 | | − if (!xa_find(&fs_info->fs_roots, &index, ULONG_MAX, XA_PRESENT)) { |
| 4532 | | − spin_unlock(&fs_info->fs_roots_lock); |
| 4533 | | − return err; |
| 4544 | + spin_lock(&fs_info->fs_roots_radix_lock); |
| 4545 | + ret = radix_tree_gang_lookup(&fs_info->fs_roots_radix, |
| 4546 | + (void **)gang, root_objectid, |
| 4547 | + ARRAY_SIZE(gang)); |
| 4548 | + if (!ret) { |
| 4549 | + spin_unlock(&fs_info->fs_roots_radix_lock); |
| 4550 | + break; |
| 4534 | 4551 | } |
| 4552 | + root_objectid = gang[ret - 1]->root_key.objectid + 1; |
| 4535 | 4553 | |
| 4536 | | − grabbed = 0; |
| 4537 | | − xa_for_each_start(&fs_info->fs_roots, index, root, index) { |
| 4538 | | − /* Avoid grabbing roots in dead_roots */ |
| 4539 | | − if (btrfs_root_refs(&root->root_item) > 0) |
| 4540 | | − roots[grabbed++] = btrfs_grab_root(root); |
| 4541 | | − if (grabbed >= ARRAY_SIZE(roots)) |
| 4542 | | − break; |
| 4554 | + for (i = 0; i < ret; i++) { |
| 4555 | + /* Avoid to grab roots in dead_roots */ |
| 4556 | + if (btrfs_root_refs(&gang[i]->root_item) == 0) { |
| 4557 | + gang[i] = NULL; |
| 4558 | + continue; |
| 4559 | + } |
| 4560 | + /* grab all the search result for later use */ |
| 4561 | + gang[i] = btrfs_grab_root(gang[i]); |
| 4543 | 4562 | } |
| 4544 | | − spin_unlock(&fs_info->fs_roots_lock); |
| 4563 | + spin_unlock(&fs_info->fs_roots_radix_lock); |
| 4545 | 4564 | |
| 4546 | | − for (i = 0; i < grabbed; i++) { |
| 4547 | | − if (!roots[i]) |
| 4565 | + for (i = 0; i < ret; i++) { |
| 4566 | + if (!gang[i]) |
| 4548 | 4567 | continue; |
| 4549 | | − index = roots[i]->root_key.objectid; |
| 4550 | | − err = btrfs_orphan_cleanup(roots[i]); |
| 4568 | + root_objectid = gang[i]->root_key.objectid; |
| 4569 | + err = btrfs_orphan_cleanup(gang[i]); |
| 4551 | 4570 | if (err) |
| 4552 | | − goto out; |
| 4553 | | − btrfs_put_root(roots[i]); |
| 4571 | + break; |
| 4572 | + btrfs_put_root(gang[i]); |
| 4554 | 4573 | } |
| 4555 | | − index++; |
| 4574 | + root_objectid++; |
| 4556 | 4575 | } |
| 4557 | 4576 | |
| 4558 | | −out: |
| 4559 | | − /* Release the roots that remain uncleaned due to error */ |
| 4560 | | − for (; i < grabbed; i++) { |
| 4561 | | − if (roots[i]) |
| 4562 | | − btrfs_put_root(roots[i]); |
| 4577 | + /* release the uncleaned roots due to error */ |
| 4578 | + for (; i < ret; i++) { |
| 4579 | + if (gang[i]) |
| 4580 | + btrfs_put_root(gang[i]); |
| 4563 | 4581 | } |
| 4564 | 4582 | return err; |
| 4565 | 4583 | } |
| @@ -4878,28 +4896,31 @@ static void btrfs_error_commit_super(struct btrfs_fs_info *fs_info) · 312 unmodified lines |
| 4878 | 4896 | |
| 4879 | 4897 | static void btrfs_drop_all_logs(struct btrfs_fs_info *fs_info) |
| 4880 | 4898 | { |
| 4881 | | − unsigned long index = 0; |
| 4882 | | − int grabbed = 0; |
| 4883 | | − struct btrfs_root *roots[8]; |
| 4899 | + struct btrfs_root *gang[8]; |
| 4900 | + u64 root_objectid = 0; |
| 4901 | + int ret; |
| 4902 | + |
| 4903 | + spin_lock(&fs_info->fs_roots_radix_lock); |
| 4904 | + while ((ret = radix_tree_gang_lookup(&fs_info->fs_roots_radix, |
| 4905 | + (void **)gang, root_objectid, |
| 4906 | + ARRAY_SIZE(gang))) != 0) { |
| 4907 | + int i; |
| 4884 | 4908 | |
| 4885 | | − spin_lock(&fs_info->fs_roots_lock); |
| 4886 | | − while ((grabbed = xa_extract(&fs_info->fs_roots, (void **)roots, index, |
| 4887 | | − ULONG_MAX, 8, XA_PRESENT))) { |
| 4888 | | − for (int i = 0; i < grabbed; i++) |
| 4889 | | − roots[i] = btrfs_grab_root(roots[i]); |
| 4890 | | − spin_unlock(&fs_info->fs_roots_lock); |
| 4909 | + for (i = 0; i < ret; i++) |
| 4910 | + gang[i] = btrfs_grab_root(gang[i]); |
| 4911 | + spin_unlock(&fs_info->fs_roots_radix_lock); |
| 4891 | 4912 | |
| 4892 | | − for (int i = 0; i < grabbed; i++) { |
| 4893 | | − if (!roots[i]) |
| 4913 | + for (i = 0; i < ret; i++) { |
| 4914 | + if (!gang[i]) |
| 4894 | 4915 | continue; |
| 4895 | | − index = roots[i]->root_key.objectid; |
| 4896 | | − btrfs_free_log(NULL, roots[i]); |
| 4897 | | − btrfs_put_root(roots[i]); |
| 4916 | + root_objectid = gang[i]->root_key.objectid; |
| 4917 | + btrfs_free_log(NULL, gang[i]); |
| 4918 | + btrfs_put_root(gang[i]); |
| 4898 | 4919 | } |
| 4899 | | − index++; |
| 4900 | | − spin_lock(&fs_info->fs_roots_lock); |
| 4920 | + root_objectid++; |
| 4921 | + spin_lock(&fs_info->fs_roots_radix_lock); |
| 4901 | 4922 | } |
| 4902 | | − spin_unlock(&fs_info->fs_roots_lock); |
| 4923 | + spin_unlock(&fs_info->fs_roots_radix_lock); |
| 4903 | 4924 | btrfs_free_log_root_tree(NULL, fs_info); |
| 4904 | 4925 | } |
| 4905 | 4926 | |