The throttle path doesn't take cleaner_delayed_iput_mutex, which means we could think we're done flushing iputs in the data space reservation path when we could have a throttler doing an iput. There's no real reason to serialize the delayed iput flushing, so instead of taking the cleaner_delayed_iput_mutex whenever we flush the delayed iputs just replace it with an atomic counter and a waitqueue. This removes the short (or long depending on how big the inode is) window where we think there are no more pending iputs when there really are some. The waiting is killable as it could be indirectly called from user operations like fallocate or zero-range. Such call sites should handle the error but otherwise it's not necessary. Eg. flush_space just needs to attempt to make space by waiting on iputs.
Unified · Split
| @@ -934,7 +934,8 @@ struct btrfs_fs_info { · 933 unmodified lines | |||
| 934 | 934 | ||
| 935 | spinlock_t delayed_iput_lock; | 935 | spinlock_t delayed_iput_lock; |
| 936 | struct list_head delayed_iputs; | 936 | struct list_head delayed_iputs; |
| 937 | struct mutex cleaner_delayed_iput_mutex; | 937 | atomic_t nr_delayed_iputs; |
| 938 | wait_queue_head_t delayed_iputs_wait; | ||
| 938 | 939 | ||
| 939 | /* this protects tree_mod_seq_list */ | 940 | /* this protects tree_mod_seq_list */ |
| 940 | spinlock_t tree_mod_seq_lock; | 941 | spinlock_t tree_mod_seq_lock; |
| @@ -3282,6 +3283,7 @@ int btrfs_orphan_cleanup(struct btrfs_root *root); · 2341 unmodified lines | |||
| 3282 | int btrfs_cont_expand(struct inode *inode, loff_t oldsize, loff_t size); | 3283 | int btrfs_cont_expand(struct inode *inode, loff_t oldsize, loff_t size); |
| 3283 | void btrfs_add_delayed_iput(struct inode *inode); | 3284 | void btrfs_add_delayed_iput(struct inode *inode); |
| 3284 | void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info); | 3285 | void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info); |
| 3286 | int btrfs_wait_on_delayed_iputs(struct btrfs_fs_info *fs_info); | ||
| 3285 | int btrfs_prealloc_file_range(struct inode *inode, int mode, | 3287 | int btrfs_prealloc_file_range(struct inode *inode, int mode, |
| 3286 | u64 start, u64 num_bytes, u64 min_size, | 3288 | u64 start, u64 num_bytes, u64 min_size, |
| 3287 | loff_t actual_len, u64 *alloc_hint); | 3289 | loff_t actual_len, u64 *alloc_hint); |
| @@ -1717,9 +1717,7 @@ static int cleaner_kthread(void *arg) · 1716 unmodified lines | |||
| 1717 | goto sleep; | 1717 | goto sleep; |
| 1718 | } | 1718 | } |
| 1719 | 1719 | ||
| 1720 | mutex_lock(&fs_info->cleaner_delayed_iput_mutex); | ||
| 1721 | btrfs_run_delayed_iputs(fs_info); | 1720 | btrfs_run_delayed_iputs(fs_info); |
| 1722 | mutex_unlock(&fs_info->cleaner_delayed_iput_mutex); | ||
| 1723 | 1721 | ||
| 1724 | again = btrfs_clean_one_deleted_snapshot(root); | 1722 | again = btrfs_clean_one_deleted_snapshot(root); |
| 1725 | mutex_unlock(&fs_info->cleaner_mutex); | 1723 | mutex_unlock(&fs_info->cleaner_mutex); |
| @@ -2676,7 +2674,6 @@ int open_ctree(struct super_block *sb, · 950 unmodified lines | |||
| 2676 | mutex_init(&fs_info->delete_unused_bgs_mutex); | 2674 | mutex_init(&fs_info->delete_unused_bgs_mutex); |
| 2677 | mutex_init(&fs_info->reloc_mutex); | 2675 | mutex_init(&fs_info->reloc_mutex); |
| 2678 | mutex_init(&fs_info->delalloc_root_mutex); | 2676 | mutex_init(&fs_info->delalloc_root_mutex); |
| 2679 | mutex_init(&fs_info->cleaner_delayed_iput_mutex); | ||
| 2680 | seqlock_init(&fs_info->profiles_lock); | 2677 | seqlock_init(&fs_info->profiles_lock); |
| 2681 | 2678 | ||
| 2682 | INIT_LIST_HEAD(&fs_info->dirty_cowonly_roots); | 2679 | INIT_LIST_HEAD(&fs_info->dirty_cowonly_roots); |
| @@ -2698,6 +2695,7 @@ int open_ctree(struct super_block *sb, · 15 unmodified lines | |||
| 2698 | atomic_set(&fs_info->defrag_running, 0); | 2695 | atomic_set(&fs_info->defrag_running, 0); |
| 2699 | atomic_set(&fs_info->qgroup_op_seq, 0); | 2696 | atomic_set(&fs_info->qgroup_op_seq, 0); |
| 2700 | atomic_set(&fs_info->reada_works_cnt, 0); | 2697 | atomic_set(&fs_info->reada_works_cnt, 0); |
| 2698 | atomic_set(&fs_info->nr_delayed_iputs, 0); | ||
| 2701 | atomic64_set(&fs_info->tree_mod_seq, 0); | 2699 | atomic64_set(&fs_info->tree_mod_seq, 0); |
| 2702 | fs_info->sb = sb; | 2700 | fs_info->sb = sb; |
| 2703 | fs_info->max_inline = BTRFS_DEFAULT_MAX_INLINE; | 2701 | fs_info->max_inline = BTRFS_DEFAULT_MAX_INLINE; |
| @@ -2775,6 +2773,7 @@ int open_ctree(struct super_block *sb, · 71 unmodified lines | |||
| 2775 | init_waitqueue_head(&fs_info->transaction_wait); | 2773 | init_waitqueue_head(&fs_info->transaction_wait); |
| 2776 | init_waitqueue_head(&fs_info->transaction_blocked_wait); | 2774 | init_waitqueue_head(&fs_info->transaction_blocked_wait); |
| 2777 | init_waitqueue_head(&fs_info->async_submit_wait); | 2775 | init_waitqueue_head(&fs_info->async_submit_wait); |
| 2776 | init_waitqueue_head(&fs_info->delayed_iputs_wait); | ||
| 2778 | 2777 | ||
| 2779 | INIT_LIST_HEAD(&fs_info->pinned_chunks); | 2778 | INIT_LIST_HEAD(&fs_info->pinned_chunks); |
| 2780 | 2779 | ||
| @@ -4279,10 +4279,14 @@ commit_trans: · 4278 unmodified lines | |||
| 4279 | /* | 4279 | /* |
| 4280 | * The cleaner kthread might still be doing iput | 4280 | * The cleaner kthread might still be doing iput |
| 4281 | * operations. Wait for it to finish so that | 4281 | * operations. Wait for it to finish so that |
| 4282 | * more space is released. | 4282 | * more space is released. We don't need to |
| 4283 | * explicitly run the delayed iputs here because | ||
| 4284 | * the commit_transaction would have woken up | ||
| 4285 | * the cleaner. | ||
| 4283 | */ | 4286 | */ |
| 4284 | mutex_lock(&fs_info->cleaner_delayed_iput_mutex); | 4287 | ret = btrfs_wait_on_delayed_iputs(fs_info); |
| 4285 | mutex_unlock(&fs_info->cleaner_delayed_iput_mutex); | 4288 | if (ret) |
| 4289 | return ret; | ||
| 4286 | goto again; | 4290 | goto again; |
| 4287 | } else { | 4291 | } else { |
| 4288 | btrfs_end_transaction(trans); | 4292 | btrfs_end_transaction(trans); |
| @@ -4967,9 +4971,8 @@ static void flush_space(struct btrfs_fs_info *fs_info, · 678 unmodified lines | |||
| 4967 | * bunch of pinned space, so make sure we run the iputs before | 4971 | * bunch of pinned space, so make sure we run the iputs before |
| 4968 | * we do our pinned bytes check below. | 4972 | * we do our pinned bytes check below. |
| 4969 | */ | 4973 | */ |
| 4970 | mutex_lock(&fs_info->cleaner_delayed_iput_mutex); | ||
| 4971 | btrfs_run_delayed_iputs(fs_info); | 4974 | btrfs_run_delayed_iputs(fs_info); |
| 4972 | mutex_unlock(&fs_info->cleaner_delayed_iput_mutex); | 4975 | btrfs_wait_on_delayed_iputs(fs_info); |
| 4973 | 4976 | ||
| 4974 | ret = may_commit_transaction(fs_info, space_info); | 4977 | ret = may_commit_transaction(fs_info, space_info); |
| 4975 | break; | 4978 | break; |
| @@ -3256,6 +3256,7 @@ void btrfs_add_delayed_iput(struct inode *inode) · 3255 unmodified lines | |||
| 3256 | if (atomic_add_unless(&inode->i_count, -1, 1)) | 3256 | if (atomic_add_unless(&inode->i_count, -1, 1)) |
| 3257 | return; | 3257 | return; |
| 3258 | 3258 | ||
| 3259 | atomic_inc(&fs_info->nr_delayed_iputs); | ||
| 3259 | spin_lock(&fs_info->delayed_iput_lock); | 3260 | spin_lock(&fs_info->delayed_iput_lock); |
| 3260 | ASSERT(list_empty(&binode->delayed_iput)); | 3261 | ASSERT(list_empty(&binode->delayed_iput)); |
| 3261 | list_add_tail(&binode->delayed_iput, &fs_info->delayed_iputs); | 3262 | list_add_tail(&binode->delayed_iput, &fs_info->delayed_iputs); |
| @@ -3276,11 +3277,32 @@ void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info) · 14 unmodified lines | |||
| 3276 | list_del_init(&inode->delayed_iput); | 3277 | list_del_init(&inode->delayed_iput); |
| 3277 | spin_unlock(&fs_info->delayed_iput_lock); | 3278 | spin_unlock(&fs_info->delayed_iput_lock); |
| 3278 | iput(&inode->vfs_inode); | 3279 | iput(&inode->vfs_inode); |
| 3280 | if (atomic_dec_and_test(&fs_info->nr_delayed_iputs)) | ||
| 3281 | wake_up(&fs_info->delayed_iputs_wait); | ||
| 3279 | spin_lock(&fs_info->delayed_iput_lock); | 3282 | spin_lock(&fs_info->delayed_iput_lock); |
| 3280 | } | 3283 | } |
| 3281 | spin_unlock(&fs_info->delayed_iput_lock); | 3284 | spin_unlock(&fs_info->delayed_iput_lock); |
| 3282 | } | 3285 | } |
| 3283 | 3286 | ||
| 3287 | /** | ||
| 3288 | * btrfs_wait_on_delayed_iputs - wait on the delayed iputs to be done running | ||
| 3289 | * @fs_info - the fs_info for this fs | ||
| 3290 | * @return - EINTR if we were killed, 0 if nothing's pending | ||
| 3291 | * | ||
| 3292 | * This will wait on any delayed iputs that are currently running with KILLABLE | ||
| 3293 | * set. Once they are all done running we will return, unless we are killed in | ||
| 3294 | * which case we return EINTR. This helps in user operations like fallocate etc | ||
| 3295 | * that might get blocked on the iputs. | ||
| 3296 | */ | ||
| 3297 | int btrfs_wait_on_delayed_iputs(struct btrfs_fs_info *fs_info) | ||
| 3298 | { | ||
| 3299 | int ret = wait_event_killable(fs_info->delayed_iputs_wait, | ||
| 3300 | atomic_read(&fs_info->nr_delayed_iputs) == 0); | ||
| 3301 | if (ret) | ||
| 3302 | return -EINTR; | ||
| 3303 | return 0; | ||
| 3304 | } | ||
| 3305 | |||
| 3284 | /* | 3306 | /* |
| 3285 | * This creates an orphan entry for the given inode in case something goes wrong | 3307 | * This creates an orphan entry for the given inode in case something goes wrong |
| 3286 | * in the middle of an unlink. | 3308 | * in the middle of an unlink. |