lael-labs/linux

arch/parisc/include/asm/cacheflush.h+3−3

arch/parisc/kernel/cache.c+1−1

@@ -549,7 +549,7 @@ extern void purge_kernel_dcache_page_asm(unsigned long); · 548 unmodified lines
549	extern void clear_user_page_asm(void *, unsigned long);	549	extern void clear_user_page_asm(void *, unsigned long);
550	extern void copy_user_page_asm(void , void , unsigned long);	550	extern void copy_user_page_asm(void , void , unsigned long);
551		551
552	void flush_kernel_dcache_page_addr(void *addr)	552	void flush_kernel_dcache_page_addr(const void *addr)
553	{	553	{
554	unsigned long flags;	554	unsigned long flags;
555		555

fs/btrfs/async-thread.h+0−1

@@ -13,7 +13,6 @@ struct btrfs_fs_info; · 12 unmodified lines
13	struct btrfs_workqueue;	13	struct btrfs_workqueue;
14	struct btrfs_work;	14	struct btrfs_work;
15	typedef void (btrfs_func_t)(struct btrfs_work arg);	15	typedef void (btrfs_func_t)(struct btrfs_work arg);
16	typedef void (btrfs_work_func_t)(struct work_struct arg);
17		16
18	struct btrfs_work {	17	struct btrfs_work {
19	btrfs_func_t func;	18	btrfs_func_t func;

fs/btrfs/backref.c+49−39

 	return ret;
 }
+static int build_ino_list(u64 inum, u64 offset, u64 root, void *ctx)
+{
+	struct btrfs_data_container *inodes = ctx;
+	const size_t c = 3 * sizeof(u64);
+	if (inodes->bytes_left >= c) {
+		inodes->bytes_left -= c;
+		inodes->val[inodes->elem_cnt] = inum;
+		inodes->val[inodes->elem_cnt + 1] = offset;
+		inodes->val[inodes->elem_cnt + 2] = root;
+		inodes->elem_cnt += 3;
+	} else {
+		inodes->bytes_missing += c - inodes->bytes_left;
+		inodes->bytes_left = 0;
+		inodes->elem_missed += 3;
+	}
+	return 0;
+}
 int iterate_inodes_from_logical(u64 logical, struct btrfs_fs_info *fs_info,
 				struct btrfs_path *path,
-				iterate_extent_inodes_t *iterate, void *ctx,
+				void *ctx, bool ignore_offset)
-				bool ignore_offset)
 {
 	int ret;
 	u64 extent_item_pos;
 	extent_item_pos = logical - found_key.objectid;
 	ret = iterate_extent_inodes(fs_info, found_key.objectid,
 					extent_item_pos, search_commit_root,
-					iterate, ctx, ignore_offset);
+					build_ino_list, ctx, ignore_offset);
 	return ret;
 }
-typedef int (iterate_irefs_t)(u64 parent, u32 name_len, unsigned long name_off,
+static int inode_to_path(u64 inum, u32 name_len, unsigned long name_off,
-			      struct extent_buffer *eb, void *ctx);
+			 struct extent_buffer *eb, struct inode_fs_paths *ipath);
-static int iterate_inode_refs(u64 inum, struct btrfs_root *fs_root,
+static int iterate_inode_refs(u64 inum, struct inode_fs_paths *ipath)
-			      struct btrfs_path *path,
-			      iterate_irefs_t *iterate, void *ctx)
 {
 	int ret = 0;
 	int slot;
 	u32 name_len;
 	u64 parent = 0;
 	int found = 0;
+	struct btrfs_root *fs_root = ipath->fs_root;
+	struct btrfs_path *path = ipath->btrfs_path;
 	struct extent_buffer *eb;
 	struct btrfs_inode_ref *iref;
 	struct btrfs_key found_key;
 				"following ref at offset %u for inode %llu in tree %llu",
 				cur, found_key.objectid,
 				fs_root->root_key.objectid);
-			ret = iterate(parent, name_len,
+			ret = inode_to_path(parent, name_len,
-				      (unsigned long)(iref + 1), eb, ctx);
+				      (unsigned long)(iref + 1), eb, ipath);
 			if (ret)
 				break;
 			len = sizeof(*iref) + name_len;
 	return ret;
 }
-static int iterate_inode_extrefs(u64 inum, struct btrfs_root *fs_root,
+static int iterate_inode_extrefs(u64 inum, struct inode_fs_paths *ipath)
-				 struct btrfs_path *path,
-				 iterate_irefs_t *iterate, void *ctx)
 {
 	int ret;
 	int slot;
 	u64 offset = 0;
 	u64 parent;
 	int found = 0;
+	struct btrfs_root *fs_root = ipath->fs_root;
+	struct btrfs_path *path = ipath->btrfs_path;
 	struct extent_buffer *eb;
 	struct btrfs_inode_extref *extref;
 	u32 item_size;
 			extref = (struct btrfs_inode_extref *)(ptr + cur_offset);
 			parent = btrfs_inode_extref_parent(eb, extref);
 			name_len = btrfs_inode_extref_name_len(eb, extref);
-			ret = iterate(parent, name_len,
+			ret = inode_to_path(parent, name_len,
-				      (unsigned long)&extref->name, eb, ctx);
+				      (unsigned long)&extref->name, eb, ipath);
 			if (ret)
 				break;
 	return ret;
 }
-static int iterate_irefs(u64 inum, struct btrfs_root *fs_root,
-			 struct btrfs_path *path, iterate_irefs_t *iterate,
-			 void *ctx)
-{
-	int ret;
-	int found_refs = 0;
-	ret = iterate_inode_refs(inum, fs_root, path, iterate, ctx);
-	if (!ret)
-		++found_refs;
-	else if (ret != -ENOENT)
-		return ret;
-	ret = iterate_inode_extrefs(inum, fs_root, path, iterate, ctx);
-	if (ret == -ENOENT && found_refs)
-		return 0;
-	return ret;
-}
 /*
  * returns 0 if the path could be dumped (probably truncated)
  * returns <0 in case of an error
  */
 static int inode_to_path(u64 inum, u32 name_len, unsigned long name_off,
-			 struct extent_buffer *eb, void *ctx)
+			 struct extent_buffer *eb, struct inode_fs_paths *ipath)
 {
-	struct inode_fs_paths *ipath = ctx;
 	char *fspath;
 	char *fspath_min;
 	int i = ipath->fspath->elem_cnt;
  */
 int paths_from_inode(u64 inum, struct inode_fs_paths *ipath)
 {
-	return iterate_irefs(inum, ipath->fs_root, ipath->btrfs_path,
+	int ret;
-			     inode_to_path, ipath);
+	int found_refs = 0;
+	ret = iterate_inode_refs(inum, ipath);
+	if (!ret)
+		++found_refs;
+	else if (ret != -ENOENT)
+		return ret;
+	ret = iterate_inode_extrefs(inum, ipath);
+	if (ret == -ENOENT && found_refs)
+		return 0;
+	return ret;
 }
 struct btrfs_data_container *init_data_container(u32 total_bytes)

fs/btrfs/backref.h+1−2

@@ -35,8 +35,7 @@ int iterate_extent_inodes(struct btrfs_fs_info *fs_info, · 34 unmodified lines
35	bool ignore_offset);	35	bool ignore_offset);
36		36
37	int iterate_inodes_from_logical(u64 logical, struct btrfs_fs_info *fs_info,	37	int iterate_inodes_from_logical(u64 logical, struct btrfs_fs_info *fs_info,
38	struct btrfs_path *path,	38	struct btrfs_path path, void ctx,
39	iterate_extent_inodes_t iterate, void ctx,
40	bool ignore_offset);	39	bool ignore_offset);
41		40
42	int paths_from_inode(u64 inum, struct inode_fs_paths *ipath);	41	int paths_from_inode(u64 inum, struct inode_fs_paths *ipath);

fs/btrfs/block-group.c+28−6

 			< block_group->zone_unusable);
 		WARN_ON(block_group->space_info->disk_total
 			< block_group->length * factor);
+		WARN_ON(block_group->zone_is_active &&
+			block_group->space_info->active_total_bytes
+			< block_group->length);
 	}
 	block_group->space_info->total_bytes -= block_group->length;
+	if (block_group->zone_is_active)
+		block_group->space_info->active_total_bytes -= block_group->length;
 	block_group->space_info->bytes_readonly -=
 		(block_group->length - block_group->zone_unusable);
 	block_group->space_info->bytes_zone_unusable -=
 		stripe_nr = physical - map->stripes[i].physical;
 		stripe_nr = div64_u64_rem(stripe_nr, map->stripe_len, &offset);
-		if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
+		if (map->type & (BTRFS_BLOCK_GROUP_RAID0 |
+				 BTRFS_BLOCK_GROUP_RAID10)) {
 			stripe_nr = stripe_nr * map->num_stripes + i;
 			stripe_nr = div_u64(stripe_nr, map->sub_stripes);
-		} else if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
-			stripe_nr = stripe_nr * map->num_stripes + i;
 		}
 		/*
 		 * The remaining case would be for RAID56, multiply by
 	trace_btrfs_add_block_group(info, cache, 0);
 	btrfs_update_space_info(info, cache->flags, cache->length,
 				cache->used, cache->bytes_super,
-				cache->zone_unusable, &space_info);
+				cache->zone_unusable, cache->zone_is_active,
+				&space_info);
 	cache->space_info = space_info;
 		}
 		btrfs_update_space_info(fs_info, bg->flags, em->len, em->len,
-					0, 0, &space_info);
+					0, 0, false, &space_info);
 		bg->space_info = space_info;
 		link_block_group(bg);
 	trace_btrfs_add_block_group(fs_info, cache, 1);
 	btrfs_update_space_info(fs_info, cache->flags, size, bytes_used,
 				cache->bytes_super, cache->zone_unusable,
-				&cache->space_info);
+				cache->zone_is_active, &cache->space_info);
 	btrfs_update_global_block_rsv(fs_info);
 	link_block_group(cache);
 	ret = btrfs_chunk_alloc(trans, alloc_flags, CHUNK_ALLOC_FORCE);
 	if (ret < 0)
 		goto out;
+	/*
+	 * We have allocated a new chunk. We also need to activate that chunk to
+	 * grant metadata tickets for zoned filesystem.
+	 */
+	ret = btrfs_zoned_activate_one_bg(fs_info, cache->space_info, true);
+	if (ret < 0)
+		goto out;
 	ret = inc_block_group_ro(cache, 0);
 	if (ret == -ETXTBSY)
 		goto unlock_out;
 			 * attempt.
 			 */
 			wait_for_alloc = true;
+			force = CHUNK_ALLOC_NO_FORCE;
 			spin_unlock(&space_info->lock);
 			mutex_lock(&fs_info->chunk_mutex);
 			mutex_unlock(&fs_info->chunk_mutex);
 		if (IS_ERR(bg)) {
 			ret = PTR_ERR(bg);
 		} else {
+			/*
+			 * We have a new chunk. We also need to activate it for
+			 * zoned filesystem.
+			 */
+			ret = btrfs_zoned_activate_one_bg(fs_info, info, true);
+			if (ret < 0)
+				return;
 			/*
 			 * If we fail to add the chunk item here, we end up
 			 * trying again at phase 2 of chunk allocation, at

fs/btrfs/block-rsv.c+9−12

 	if (block_rsv->reserved >= block_rsv->size) {
 		num_bytes = block_rsv->reserved - block_rsv->size;
 		block_rsv->reserved = block_rsv->size;
-		block_rsv->full = 1;
+		block_rsv->full = true;
 	} else {
 		num_bytes = 0;
 	}
 				bytes_to_add = min(num_bytes, bytes_to_add);
 				dest->reserved += bytes_to_add;
 				if (dest->reserved >= dest->size)
-					dest->full = 1;
+					dest->full = true;
 				num_bytes -= bytes_to_add;
 			}
 			spin_unlock(&dest->lock);
 	return 0;
 }
-void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv, unsigned short type)
+void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv, enum btrfs_rsv_type type)
 {
 	memset(rsv, 0, sizeof(*rsv));
 	spin_lock_init(&rsv->lock);
 void btrfs_init_metadata_block_rsv(struct btrfs_fs_info *fs_info,
 				   struct btrfs_block_rsv *rsv,
-				   unsigned short type)
+				   enum btrfs_rsv_type type)
 {
 	btrfs_init_block_rsv(rsv, type);
 	rsv->space_info = btrfs_find_space_info(fs_info,
 }
 struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_fs_info *fs_info,
-					      unsigned short type)
+					      enum btrfs_rsv_type type)
 {
 	struct btrfs_block_rsv *block_rsv;
 	if (block_rsv->reserved >= num_bytes) {
 		block_rsv->reserved -= num_bytes;
 		if (block_rsv->reserved < block_rsv->size)
-			block_rsv->full = 0;
+			block_rsv->full = false;
 		ret = 0;
 	}
 	spin_unlock(&block_rsv->lock);
 	if (update_size)
 		block_rsv->size += num_bytes;
 	else if (block_rsv->reserved >= block_rsv->size)
-		block_rsv->full = 1;
+		block_rsv->full = true;
 	spin_unlock(&block_rsv->lock);
 }
 	}
 	global_rsv->reserved -= num_bytes;
 	if (global_rsv->reserved < global_rsv->size)
-		global_rsv->full = 0;
+		global_rsv->full = false;
 	spin_unlock(&global_rsv->lock);
 	btrfs_block_rsv_add_bytes(dest, num_bytes, true);
 		btrfs_try_granting_tickets(fs_info, sinfo);
 	}
-	if (block_rsv->reserved == block_rsv->size)
+	block_rsv->full = (block_rsv->reserved == block_rsv->size);
-		block_rsv->full = 1;
-	else
-		block_rsv->full = 0;
 	if (block_rsv->size >= sinfo->total_bytes)
 		sinfo->force_alloc = CHUNK_ALLOC_FORCE;

fs/btrfs/block-rsv.h+8−7

 /*
  * Types of block reserves
  */
-enum {
+enum btrfs_rsv_type {
 	BTRFS_BLOCK_RSV_GLOBAL,
 	BTRFS_BLOCK_RSV_DELALLOC,
 	BTRFS_BLOCK_RSV_TRANS,
 	u64 reserved;
 	struct btrfs_space_info *space_info;
 	spinlock_t lock;
-	unsigned short full;
+	bool full;
-	unsigned short type;
+	bool failfast;
-	unsigned short failfast;
+	/* Block reserve type, one of BTRFS_BLOCK_RSV_* */
+	enum btrfs_rsv_type type:8;
 	/*
 	 * Qgroup equivalent for @size @reserved
 	u64 qgroup_rsv_reserved;
 };
-void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv, unsigned short type);
+void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv, enum btrfs_rsv_type type);
 void btrfs_init_root_block_rsv(struct btrfs_root *root);
 struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_fs_info *fs_info,
-					      unsigned short type);
+					      enum btrfs_rsv_type type);
 void btrfs_init_metadata_block_rsv(struct btrfs_fs_info *fs_info,
 				   struct btrfs_block_rsv *rsv,
-				   unsigned short type);
+				   enum btrfs_rsv_type type);
 void btrfs_free_block_rsv(struct btrfs_fs_info *fs_info,
 			  struct btrfs_block_rsv *rsv);
 int btrfs_block_rsv_add(struct btrfs_fs_info *fs_info,

fs/btrfs/btrfs_inode.h+18−7

@@ -279,19 +279,31 @@ static inline void btrfs_insert_inode_hash(struct inode *inode) · 278 unmodified lines
279	__insert_inode_hash(inode, h);	279	__insert_inode_hash(inode, h);
280	}	280	}
281		281
		282	#if BITS_PER_LONG == 32
		283
		284	/*
		285	* On 32 bit systems the i_ino of struct inode is 32 bits (unsigned long), so
		286	* we use the inode's location objectid which is a u64 to avoid truncation.
		287	*/
282	static inline u64 btrfs_ino(const struct btrfs_inode *inode)	288	static inline u64 btrfs_ino(const struct btrfs_inode *inode)
283	{	289	{
284	u64 ino = inode->location.objectid;	290	u64 ino = inode->location.objectid;
285		291
286	/*	292	/* type == BTRFS_ROOT_ITEM_KEY: subvol dir */
287	* !ino: btree_inode	293	if (inode->location.type == BTRFS_ROOT_ITEM_KEY)
288	* type == BTRFS_ROOT_ITEM_KEY: subvol dir
289	*/
290	if (!ino \|\| inode->location.type == BTRFS_ROOT_ITEM_KEY)
291	ino = inode->vfs_inode.i_ino;	294	ino = inode->vfs_inode.i_ino;
292	return ino;	295	return ino;
293	}	296	}
294		297
		298	#else
		299
		300	static inline u64 btrfs_ino(const struct btrfs_inode *inode)
		301	{
		302	return inode->vfs_inode.i_ino;
		303	}
		304
		305	#endif
		306
295	static inline void btrfs_i_size_write(struct btrfs_inode *inode, u64 size)	307	static inline void btrfs_i_size_write(struct btrfs_inode *inode, u64 size)
296	{	308	{
297	i_size_write(&inode->vfs_inode, size);	309	i_size_write(&inode->vfs_inode, size);
@@ -305,8 +317,7 @@ static inline bool btrfs_is_free_space_inode(struct btrfs_inode *inode) · 7 unmodified lines
305	if (root == root->fs_info->tree_root &&	317	if (root == root->fs_info->tree_root &&
306	btrfs_ino(inode) != BTRFS_BTREE_INODE_OBJECTID)	318	btrfs_ino(inode) != BTRFS_BTREE_INODE_OBJECTID)
307	return true;	319	return true;
308	if (inode->location.objectid == BTRFS_FREE_INO_OBJECTID)	320
309	return true;
310	return false;	321	return false;
311	}	322	}
312		323

fs/btrfs/compression.c+102−257

@@ -136,109 +136,14 @@ static int compression_decompress(int type, struct list_head *ws, · 135 unmodified lines
136		136
137	static int btrfs_decompress_bio(struct compressed_bio *cb);	137	static int btrfs_decompress_bio(struct compressed_bio *cb);
138		138
139	static inline int compressed_bio_size(struct btrfs_fs_info *fs_info,
140	unsigned long disk_size)
141	{
142	return sizeof(struct compressed_bio) +
143	(DIV_ROUND_UP(disk_size, fs_info->sectorsize)) * fs_info->csum_size;
144	}
145
146	static int check_compressed_csum(struct btrfs_inode inode, struct bio bio,
147	u64 disk_start)
148	{
149	struct btrfs_fs_info *fs_info = inode->root->fs_info;
150	SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
151	const u32 csum_size = fs_info->csum_size;
152	const u32 sectorsize = fs_info->sectorsize;
153	struct page *page;
154	unsigned int i;
155	char *kaddr;
156	u8 csum[BTRFS_CSUM_SIZE];
157	struct compressed_bio *cb = bio->bi_private;
158	u8 *cb_sum = cb->sums;
159
160	if ((inode->flags & BTRFS_INODE_NODATASUM) \|\|
161	test_bit(BTRFS_FS_STATE_NO_CSUMS, &fs_info->fs_state))
162	return 0;
163
164	shash->tfm = fs_info->csum_shash;
165
166	for (i = 0; i < cb->nr_pages; i++) {
167	u32 pg_offset;
168	u32 bytes_left = PAGE_SIZE;
169	page = cb->compressed_pages[i];
170
171	/* Determine the remaining bytes inside the page first */
172	if (i == cb->nr_pages - 1)
173	bytes_left = cb->compressed_len - i * PAGE_SIZE;
174
175	/* Hash through the page sector by sector */
176	for (pg_offset = 0; pg_offset < bytes_left;
177	pg_offset += sectorsize) {
178	kaddr = kmap_atomic(page);
179	crypto_shash_digest(shash, kaddr + pg_offset,
180	sectorsize, csum);
181	kunmap_atomic(kaddr);
182
183	if (memcmp(&csum, cb_sum, csum_size) != 0) {
184	btrfs_print_data_csum_error(inode, disk_start,
185	csum, cb_sum, cb->mirror_num);
186	if (btrfs_bio(bio)->device)
187	btrfs_dev_stat_inc_and_print(
188	btrfs_bio(bio)->device,
189	BTRFS_DEV_STAT_CORRUPTION_ERRS);
190	return -EIO;
191	}
192	cb_sum += csum_size;
193	disk_start += sectorsize;
194	}
195	}
196	return 0;
197	}
198
199	/*
200	* Reduce bio and io accounting for a compressed_bio with its corresponding bio.
201	*
202	* Return true if there is no pending bio nor io.
203	* Return false otherwise.
204	*/
205	static bool dec_and_test_compressed_bio(struct compressed_bio cb, struct bio bio)
206	{
207	struct btrfs_fs_info *fs_info = btrfs_sb(cb->inode->i_sb);
208	unsigned int bi_size = 0;
209	bool last_io = false;
210	struct bio_vec *bvec;
211	struct bvec_iter_all iter_all;
212
213	/*
214	* At endio time, bi_iter.bi_size doesn't represent the real bio size.
215	* Thus here we have to iterate through all segments to grab correct
216	* bio size.
217	*/
218	bio_for_each_segment_all(bvec, bio, iter_all)
219	bi_size += bvec->bv_len;
220
221	if (bio->bi_status)
222	cb->status = bio->bi_status;
223
224	ASSERT(bi_size && bi_size <= cb->compressed_len);
225	last_io = refcount_sub_and_test(bi_size >> fs_info->sectorsize_bits,
226	&cb->pending_sectors);
227	/*
228	* Here we must wake up the possible error handler after all other
229	* operations on @cb finished, or we can race with
230	* finish_compressed_bio_*() which may free @cb.
231	*/
232	wake_up_var(cb);
233
234	return last_io;
235	}
236
237	static void finish_compressed_bio_read(struct compressed_bio *cb)	139	static void finish_compressed_bio_read(struct compressed_bio *cb)
238	{	140	{
239	unsigned int index;	141	unsigned int index;
240	struct page *page;	142	struct page *page;
241		143
		144	if (cb->status == BLK_STS_OK)
		145	cb->status = errno_to_blk_status(btrfs_decompress_bio(cb));
		146
242	/* Release the compressed pages */	147	/* Release the compressed pages */
243	for (index = 0; index < cb->nr_pages; index++) {	148	for (index = 0; index < cb->nr_pages; index++) {
244	page = cb->compressed_pages[index];	149	page = cb->compressed_pages[index];
@@ -247,85 +152,63 @@ static void finish_compressed_bio_read(struct compressed_bio *cb) · 2 unmodified lines
247	}	152	}
248		153
249	/* Do io completion on the original bio */	154	/* Do io completion on the original bio */
250	if (cb->status != BLK_STS_OK) {	155	if (cb->status != BLK_STS_OK)
251	cb->orig_bio->bi_status = cb->status;	156	cb->orig_bio->bi_status = cb->status;
252	bio_endio(cb->orig_bio);	157	bio_endio(cb->orig_bio);
253	} else {
254	struct bio_vec *bvec;
255	struct bvec_iter_all iter_all;
256
257	/*
258	* We have verified the checksum already, set page checked so
259	* the end_io handlers know about it
260	*/
261	ASSERT(!bio_flagged(cb->orig_bio, BIO_CLONED));
262	bio_for_each_segment_all(bvec, cb->orig_bio, iter_all) {
263	u64 bvec_start = page_offset(bvec->bv_page) +
264	bvec->bv_offset;
265
266	btrfs_page_set_checked(btrfs_sb(cb->inode->i_sb),
267	bvec->bv_page, bvec_start,
268	bvec->bv_len);
269	}
270
271	bio_endio(cb->orig_bio);
272	}
273		158
274	/* Finally free the cb struct */	159	/* Finally free the cb struct */
275	kfree(cb->compressed_pages);	160	kfree(cb->compressed_pages);
276	kfree(cb);	161	kfree(cb);
277	}	162	}
278		163
279	/* when we finish reading compressed pages from the disk, we	164	/*
280	* decompress them and then run the bio end_io routines on the	165	* Verify the checksums and kick off repair if needed on the uncompressed data
281	* decompressed pages (in the inode address space).	166	* before decompressing it into the original bio and freeing the uncompressed
282	*	167	* pages.
283	* This allows the checksumming and other IO error handling routines
284	* to work normally
285	*
286	* The compressed pages are freed here, and it must be run
287	* in process context
288	*/	168	*/
289	static void end_compressed_bio_read(struct bio *bio)	169	static void end_compressed_bio_read(struct bio *bio)
290	{	170	{
291	struct compressed_bio *cb = bio->bi_private;	171	struct compressed_bio *cb = bio->bi_private;
292	struct inode *inode;	172	struct inode *inode = cb->inode;
293	unsigned int mirror = btrfs_bio(bio)->mirror_num;	173	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
294	int ret = 0;	174	struct btrfs_inode *bi = BTRFS_I(inode);
295		175	bool csum = !(bi->flags & BTRFS_INODE_NODATASUM) &&
296	if (!dec_and_test_compressed_bio(cb, bio))	176	!test_bit(BTRFS_FS_STATE_NO_CSUMS, &fs_info->fs_state);
297	goto out;	177	blk_status_t status = bio->bi_status;
298		178	struct btrfs_bio *bbio = btrfs_bio(bio);
299	/*	179	struct bvec_iter iter;
300	* Record the correct mirror_num in cb->orig_bio so that	180	struct bio_vec bv;
301	* read-repair can work properly.	181	u32 offset;
302	*/	182
303	btrfs_bio(cb->orig_bio)->mirror_num = mirror;	183	btrfs_bio_for_each_sector(fs_info, bv, bbio, iter, offset) {
304	cb->mirror_num = mirror;	184	u64 start = bbio->file_offset + offset;
305		185
306	/*	186	if (!status &&
307	* Some IO in this cb have failed, just skip checksum as there	187	(!csum \|\| !btrfs_check_data_csum(inode, bbio, offset,
308	* is no way it could be correct.	188	bv.bv_page, bv.bv_offset))) {
309	*/	189	clean_io_failure(fs_info, &bi->io_failure_tree,
310	if (cb->status != BLK_STS_OK)	190	&bi->io_tree, start, bv.bv_page,
311	goto csum_failed;	191	btrfs_ino(bi), bv.bv_offset);
312		192	} else {
313	inode = cb->inode;	193	int ret;
314	ret = check_compressed_csum(BTRFS_I(inode), bio,	194
315	bio->bi_iter.bi_sector << 9);	195	refcount_inc(&cb->pending_ios);
316	if (ret)	196	ret = btrfs_repair_one_sector(inode, bbio, offset,
317	goto csum_failed;	197	bv.bv_page, bv.bv_offset,
		198	btrfs_submit_data_read_bio);
		199	if (ret) {
		200	refcount_dec(&cb->pending_ios);
		201	status = errno_to_blk_status(ret);
		202	}
		203	}
		204	}
318		205
319	/* ok, we're the last bio for this extent, lets start	206	if (status)
320	* the decompression.	207	cb->status = status;
321	*/
322	ret = btrfs_decompress_bio(cb);
323		208
324	csum_failed:	209	if (refcount_dec_and_test(&cb->pending_ios))
325	if (ret)	210	finish_compressed_bio_read(cb);
326	cb->status = errno_to_blk_status(ret);	211	btrfs_bio_free_csum(bbio);
327	finish_compressed_bio_read(cb);
328	out:
329	bio_put(bio);	212	bio_put(bio);
330	}	213	}
331		214
@@ -403,6 +286,14 @@ static void finish_compressed_bio_write(struct compressed_bio *cb) · 71 unmodified lines
403	kfree(cb);	286	kfree(cb);
404	}	287	}
405		288
		289	static void btrfs_finish_compressed_write_work(struct work_struct *work)
		290	{
		291	struct compressed_bio *cb =
		292	container_of(work, struct compressed_bio, write_end_work);
		293
		294	finish_compressed_bio_write(cb);
		295	}
		296
406	/*	297	/*
407	* Do the cleanup once all the compressed pages hit the disk. This will clear	298	* Do the cleanup once all the compressed pages hit the disk. This will clear
408	* writeback on the file pages and free the compressed pages.	299	* writeback on the file pages and free the compressed pages.
@@ -414,29 +305,18 @@ static void end_compressed_bio_write(struct bio *bio) · 5 unmodified lines
414	{	305	{
415	struct compressed_bio *cb = bio->bi_private;	306	struct compressed_bio *cb = bio->bi_private;
416		307
417	if (!dec_and_test_compressed_bio(cb, bio))	308	if (bio->bi_status)
418	goto out;	309	cb->status = bio->bi_status;
419		310
420	btrfs_record_physical_zoned(cb->inode, cb->start, bio);	311	if (refcount_dec_and_test(&cb->pending_ios)) {
		312	struct btrfs_fs_info *fs_info = btrfs_sb(cb->inode->i_sb);
421		313
422	finish_compressed_bio_write(cb);	314	btrfs_record_physical_zoned(cb->inode, cb->start, bio);
423	out:	315	queue_work(fs_info->compressed_write_workers, &cb->write_end_work);
		316	}
424	bio_put(bio);	317	bio_put(bio);
425	}	318	}
426		319
427	static blk_status_t submit_compressed_bio(struct btrfs_fs_info *fs_info,
428	struct bio *bio, int mirror_num)
429	{
430	blk_status_t ret;
431
432	ASSERT(bio->bi_iter.bi_size);
433	ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DATA);
434	if (ret)
435	return ret;
436	ret = btrfs_map_bio(fs_info, bio, mirror_num);
437	return ret;
438	}
439
440	/*	320	/*
441	* Allocate a compressed_bio, which will be used to read/write on-disk	321	* Allocate a compressed_bio, which will be used to read/write on-disk
442	* (aka, compressed) * data.	322	* (aka, compressed) * data.
@@ -487,7 +367,7 @@ static struct bio alloc_compressed_bio(struct compressed_bio cb, u64 disk_byte · 44 unmodified lines
487	return ERR_PTR(ret);	367	return ERR_PTR(ret);
488	}	368	}
489	*next_stripe_start = disk_bytenr + geom.len;	369	*next_stripe_start = disk_bytenr + geom.len;
490		370	refcount_inc(&cb->pending_ios);
491	return bio;	371	return bio;
492	}	372	}
493		373
@@ -514,26 +394,25 @@ blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start, · 20 unmodified lines
514	struct compressed_bio *cb;	394	struct compressed_bio *cb;
515	u64 cur_disk_bytenr = disk_start;	395	u64 cur_disk_bytenr = disk_start;
516	u64 next_stripe_start;	396	u64 next_stripe_start;
517	blk_status_t ret;	397	blk_status_t ret = BLK_STS_OK;
518	int skip_sum = inode->flags & BTRFS_INODE_NODATASUM;	398	int skip_sum = inode->flags & BTRFS_INODE_NODATASUM;
519	const bool use_append = btrfs_use_zone_append(inode, disk_start);	399	const bool use_append = btrfs_use_zone_append(inode, disk_start);
520	const enum req_op bio_op = use_append ? REQ_OP_ZONE_APPEND : REQ_OP_WRITE;	400	const enum req_op bio_op = use_append ? REQ_OP_ZONE_APPEND : REQ_OP_WRITE;
521		401
522	ASSERT(IS_ALIGNED(start, fs_info->sectorsize) &&	402	ASSERT(IS_ALIGNED(start, fs_info->sectorsize) &&
523	IS_ALIGNED(len, fs_info->sectorsize));	403	IS_ALIGNED(len, fs_info->sectorsize));
524	cb = kmalloc(compressed_bio_size(fs_info, compressed_len), GFP_NOFS);	404	cb = kmalloc(sizeof(struct compressed_bio), GFP_NOFS);
525	if (!cb)	405	if (!cb)
526	return BLK_STS_RESOURCE;	406	return BLK_STS_RESOURCE;
527	refcount_set(&cb->pending_sectors, compressed_len >> fs_info->sectorsize_bits);	407	refcount_set(&cb->pending_ios, 1);
528	cb->status = BLK_STS_OK;	408	cb->status = BLK_STS_OK;
529	cb->inode = &inode->vfs_inode;	409	cb->inode = &inode->vfs_inode;
530	cb->start = start;	410	cb->start = start;
531	cb->len = len;	411	cb->len = len;
532	cb->mirror_num = 0;
533	cb->compressed_pages = compressed_pages;	412	cb->compressed_pages = compressed_pages;
534	cb->compressed_len = compressed_len;	413	cb->compressed_len = compressed_len;
535	cb->writeback = writeback;	414	cb->writeback = writeback;
536	cb->orig_bio = NULL;	415	INIT_WORK(&cb->write_end_work, btrfs_finish_compressed_write_work);
537	cb->nr_pages = nr_pages;	416	cb->nr_pages = nr_pages;
538		417
539	if (blkcg_css)	418	if (blkcg_css)
@@ -554,8 +433,7 @@ blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start, · 14 unmodified lines
554	&next_stripe_start);	433	&next_stripe_start);
555	if (IS_ERR(bio)) {	434	if (IS_ERR(bio)) {
556	ret = errno_to_blk_status(PTR_ERR(bio));	435	ret = errno_to_blk_status(PTR_ERR(bio));
557	bio = NULL;	436	break;
558	goto finish_cb;
559	}	437	}
560	if (blkcg_css)	438	if (blkcg_css)
561	bio->bi_opf \|= REQ_CGROUP_PUNT;	439	bio->bi_opf \|= REQ_CGROUP_PUNT;
@@ -599,44 +477,25 @@ blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start, · 37 unmodified lines
599	if (submit) {	477	if (submit) {
600	if (!skip_sum) {	478	if (!skip_sum) {
601	ret = btrfs_csum_one_bio(inode, bio, start, true);	479	ret = btrfs_csum_one_bio(inode, bio, start, true);
602	if (ret)	480	if (ret) {
603	goto finish_cb;	481	bio->bi_status = ret;
		482	bio_endio(bio);
		483	break;
		484	}
604	}	485	}
605		486
606	ret = submit_compressed_bio(fs_info, bio, 0);	487	ASSERT(bio->bi_iter.bi_size);
607	if (ret)	488	btrfs_submit_bio(fs_info, bio, 0);
608	goto finish_cb;
609	bio = NULL;	489	bio = NULL;
610	}	490	}
611	cond_resched();	491	cond_resched();
612	}	492	}
613	if (blkcg_css)
614	kthread_associate_blkcg(NULL);
615
616	return 0;
617		493
618	finish_cb:
619	if (blkcg_css)	494	if (blkcg_css)
620	kthread_associate_blkcg(NULL);	495	kthread_associate_blkcg(NULL);
621		496
622	if (bio) {	497	if (refcount_dec_and_test(&cb->pending_ios))
623	bio->bi_status = ret;	498	finish_compressed_bio_write(cb);
624	bio_endio(bio);
625	}
626	/* Last byte of @cb is submitted, endio will free @cb */
627	if (cur_disk_bytenr == disk_start + compressed_len)
628	return ret;
629
630	wait_var_event(cb, refcount_read(&cb->pending_sectors) ==
631	(disk_start + compressed_len - cur_disk_bytenr) >>
632	fs_info->sectorsize_bits);
633	/*
634	* Even with previous bio ended, we should still have io not yet
635	* submitted, thus need to finish manually.
636	*/
637	ASSERT(refcount_read(&cb->pending_sectors));
638	/* Now we are the only one referring @cb, can finish it safely. */
639	finish_compressed_bio_write(cb);
640	return ret;	499	return ret;
641	}	500	}
642		501
@@ -765,7 +624,6 @@ static noinline int add_ra_bio_pages(struct inode *inode, · 122 unmodified lines
765	int zeros;	624	int zeros;
766	zeros = PAGE_SIZE - zero_offset;	625	zeros = PAGE_SIZE - zero_offset;
767	memzero_page(page, zero_offset, zeros);	626	memzero_page(page, zero_offset, zeros);
768	flush_dcache_page(page);
769	}	627	}
770	}	628	}
771		629
@@ -819,7 +677,6 @@ void btrfs_submit_compressed_read(struct inode inode, struct bio bio, · 47 unmodified lines
819	blk_status_t ret;	677	blk_status_t ret;
820	int ret2;	678	int ret2;
821	int i;	679	int i;
822	u8 *sums;
823		680
824	em_tree = &BTRFS_I(inode)->extent_tree;	681	em_tree = &BTRFS_I(inode)->extent_tree;
825		682
@@ -837,17 +694,15 @@ void btrfs_submit_compressed_read(struct inode inode, struct bio bio, · 11 unmodified lines
837		694
838	ASSERT(em->compress_type != BTRFS_COMPRESS_NONE);	695	ASSERT(em->compress_type != BTRFS_COMPRESS_NONE);
839	compressed_len = em->block_len;	696	compressed_len = em->block_len;
840	cb = kmalloc(compressed_bio_size(fs_info, compressed_len), GFP_NOFS);	697	cb = kmalloc(sizeof(struct compressed_bio), GFP_NOFS);
841	if (!cb) {	698	if (!cb) {
842	ret = BLK_STS_RESOURCE;	699	ret = BLK_STS_RESOURCE;
843	goto out;	700	goto out;
844	}	701	}
845		702
846	refcount_set(&cb->pending_sectors, compressed_len >> fs_info->sectorsize_bits);	703	refcount_set(&cb->pending_ios, 1);
847	cb->status = BLK_STS_OK;	704	cb->status = BLK_STS_OK;
848	cb->inode = inode;	705	cb->inode = inode;
849	cb->mirror_num = mirror_num;
850	sums = cb->sums;
851		706
852	cb->start = em->orig_start;	707	cb->start = em->orig_start;
853	em_len = em->len;	708	em_len = em->len;
@@ -893,9 +748,8 @@ void btrfs_submit_compressed_read(struct inode inode, struct bio bio, · 39 unmodified lines
893	REQ_OP_READ, end_compressed_bio_read,	748	REQ_OP_READ, end_compressed_bio_read,
894	&next_stripe_start);	749	&next_stripe_start);
895	if (IS_ERR(comp_bio)) {	750	if (IS_ERR(comp_bio)) {
896	ret = errno_to_blk_status(PTR_ERR(comp_bio));	751	cb->status = errno_to_blk_status(PTR_ERR(comp_bio));
897	comp_bio = NULL;	752	break;
898	goto finish_cb;
899	}	753	}
900	}	754	}
901	/*	755	/*
@@ -931,22 +785,33 @@ void btrfs_submit_compressed_read(struct inode inode, struct bio bio, · 29 unmodified lines
931	submit = true;	785	submit = true;
932		786
933	if (submit) {	787	if (submit) {
934	unsigned int nr_sectors;	788	/* Save the original iter for read repair */
		789	if (bio_op(comp_bio) == REQ_OP_READ)
		790	btrfs_bio(comp_bio)->iter = comp_bio->bi_iter;
935		791
936	ret = btrfs_lookup_bio_sums(inode, comp_bio, sums);	792	/*
937	if (ret)	793	* Save the initial offset of this chunk, as there
938	goto finish_cb;	794	* is no direct correlation between compressed pages and
		795	* the original file offset. The field is only used for
		796	* priting error messages.
		797	*/
		798	btrfs_bio(comp_bio)->file_offset = file_offset;
939		799
940	nr_sectors = DIV_ROUND_UP(comp_bio->bi_iter.bi_size,	800	ret = btrfs_lookup_bio_sums(inode, comp_bio, NULL);
941	fs_info->sectorsize);	801	if (ret) {
942	sums += fs_info->csum_size * nr_sectors;	802	comp_bio->bi_status = ret;
		803	bio_endio(comp_bio);
		804	break;
		805	}
943		806
944	ret = submit_compressed_bio(fs_info, comp_bio, mirror_num);	807	ASSERT(comp_bio->bi_iter.bi_size);
945	if (ret)	808	btrfs_submit_bio(fs_info, comp_bio, mirror_num);
946	goto finish_cb;
947	comp_bio = NULL;	809	comp_bio = NULL;
948	}	810	}
949	}	811	}
		812
		813	if (refcount_dec_and_test(&cb->pending_ios))
		814	finish_compressed_bio_read(cb);
950	return;	815	return;
951		816
952	fail:	817	fail:
@@ -964,25 +829,6 @@ out: · 11 unmodified lines
964	bio->bi_status = ret;	829	bio->bi_status = ret;
965	bio_endio(bio);	830	bio_endio(bio);
966	return;	831	return;
967	finish_cb:
968	if (comp_bio) {
969	comp_bio->bi_status = ret;
970	bio_endio(comp_bio);
971	}
972	/* All bytes of @cb is submitted, endio will free @cb */
973	if (cur_disk_byte == disk_bytenr + compressed_len)
974	return;
975
976	wait_var_event(cb, refcount_read(&cb->pending_sectors) ==
977	(disk_bytenr + compressed_len - cur_disk_byte) >>
978	fs_info->sectorsize_bits);
979	/*
980	* Even with previous bio ended, we should still have io not yet
981	* submitted, thus need to finish @cb manually.
982	*/
983	ASSERT(refcount_read(&cb->pending_sectors));
984	/* Now we are the only one referring @cb, can finish it safely. */
985	finish_compressed_bio_read(cb);
986	}	832	}
987		833
988	/*	834	/*
@@ -1481,7 +1327,6 @@ int btrfs_decompress_buf2page(const char *buf, u32 buf_len, · 492 unmodified lines
1481	ASSERT(copy_start - decompressed < buf_len);	1327	ASSERT(copy_start - decompressed < buf_len);
1482	memcpy_to_page(bvec.bv_page, bvec.bv_offset,	1328	memcpy_to_page(bvec.bv_page, bvec.bv_offset,
1483	buf + copy_start - decompressed, copy_len);	1329	buf + copy_start - decompressed, copy_len);
1484	flush_dcache_page(bvec.bv_page);
1485	cur_offset += copy_len;	1330	cur_offset += copy_len;
1486		1331
1487	bio_advance(orig_bio, copy_len);	1332	bio_advance(orig_bio, copy_len);

fs/btrfs/compression.h+7−11

@@ -30,8 +30,8 @@ static_assert((BTRFS_MAX_COMPRESSED % PAGE_SIZE) == 0); · 29 unmodified lines
30	#define BTRFS_ZLIB_DEFAULT_LEVEL 3	30	#define BTRFS_ZLIB_DEFAULT_LEVEL 3
31		31
32	struct compressed_bio {	32	struct compressed_bio {
33	/* Number of sectors with unfinished IO (unsubmitted or unfinished) */	33	/* Number of outstanding bios */
34	refcount_t pending_sectors;	34	refcount_t pending_ios;
35		35
36	/* Number of compressed pages in the array */	36	/* Number of compressed pages in the array */
37	unsigned int nr_pages;	37	unsigned int nr_pages;
@@ -59,16 +59,12 @@ struct compressed_bio { · 21 unmodified lines
59		59
60	/* IO errors */	60	/* IO errors */
61	blk_status_t status;	61	blk_status_t status;
62	int mirror_num;
63		62
64	/* for reads, this is the bio we are copying the data into */	63	union {
65	struct bio *orig_bio;	64	/* For reads, this is the bio we are copying the data into */
66		65	struct bio *orig_bio;
67	/*	66	struct work_struct write_end_work;
68	* the start of a variable length array of checksums only	67	};
69	* used by reads
70	*/
71	u8 sums[];
72	};	68	};
73		69
74	static inline unsigned int btrfs_compress_type(unsigned int type_level)	70	static inline unsigned int btrfs_compress_type(unsigned int type_level)

fs/btrfs/ctree.h+84−21

 #define BTRFS_STAT_CURR		0
 #define BTRFS_STAT_PREV		1
-/*
- * Count how many BTRFS_MAX_EXTENT_SIZE cover the @size
- */
-static inline u32 count_max_extents(u64 size)
-{
-	return div_u64(size + BTRFS_MAX_EXTENT_SIZE - 1, BTRFS_MAX_EXTENT_SIZE);
-}
 static inline unsigned long btrfs_chunk_item_size(int num_stripes)
 {
 	BUG_ON(num_stripes == 0);
 #define BTRFS_SUPER_INFO_OFFSET			SZ_64K
 #define BTRFS_SUPER_INFO_SIZE			4096
+/*
+ * The reserved space at the beginning of each device.
+ * It covers the primary super block and leaves space for potential use by other
+ * tools like bootloaders or to lower potential damage of accidental overwrite.
+ */
+#define BTRFS_DEVICE_RANGE_RESERVED			(SZ_1M)
 /*
  * the super block basically lists the main trees of the FS
  * it currently lacks any block count etc etc
 	__le64 chunk_root;
 	__le64 log_root;
-	/* this will help find the new super based on the log root */
+	/*
-	__le64 log_root_transid;
+	 * This member has never been utilized since the very beginning, thus
+	 * it's always 0 regardless of kernel version.  We always use
+	 * generation + 1 to read log tree root.  So here we mark it deprecated.
+	 */
+	__le64 __unused_log_root_transid;
 	__le64 total_bytes;
 	__le64 bytes_used;
 	__le64 root_dir_objectid;
 	/* Indicate we have half completed snapshot deletions pending. */
 	BTRFS_FS_UNFINISHED_DROPS,
+	/* Indicate we have to finish a zone to do next allocation. */
+	BTRFS_FS_NEED_ZONE_FINISH,
 #if BITS_PER_LONG == 32
 	/* Indicate if we have error/warn message printed on 32bit systems */
 	BTRFS_FS_32BIT_ERROR,
 	BTRFS_EXCLOP_SWAP_ACTIVATE,
 };
+/* Store data about transaction commits, exported via sysfs. */
+struct btrfs_commit_stats {
+	/* Total number of commits */
+	u64 commit_count;
+	/* The maximum commit duration so far in ns */
+	u64 max_commit_dur;
+	/* The last commit duration in ns */
+	u64 last_commit_dur;
+	/* The total commit duration in ns */
+	u64 total_commit_dur;
+};
 struct btrfs_fs_info {
 	u8 chunk_tree_uuid[BTRFS_UUID_SIZE];
 	unsigned long flags;
 	struct btrfs_workqueue *hipri_workers;
 	struct btrfs_workqueue *delalloc_workers;
 	struct btrfs_workqueue *flush_workers;
-	struct btrfs_workqueue *endio_workers;
+	struct workqueue_struct *endio_workers;
-	struct btrfs_workqueue *endio_meta_workers;
+	struct workqueue_struct *endio_meta_workers;
-	struct btrfs_workqueue *endio_raid56_workers;
+	struct workqueue_struct *endio_raid56_workers;
 	struct workqueue_struct *rmw_workers;
-	struct btrfs_workqueue *endio_meta_write_workers;
+	struct workqueue_struct *compressed_write_workers;
 	struct btrfs_workqueue *endio_write_workers;
 	struct btrfs_workqueue *endio_freespace_worker;
 	struct btrfs_workqueue *caching_workers;
 	u32 csums_per_leaf;
 	u32 stripesize;
+	/*
+	 * Maximum size of an extent. BTRFS_MAX_EXTENT_SIZE on regular
+	 * filesystem, on zoned it depends on the device constraints.
+	 */
+	u64 max_extent_size;
 	/* Block groups and devices containing active swapfiles. */
 	spinlock_t swapfile_pins_lock;
 	struct rb_root swapfile_pins;
 	 */
 	u64 zone_size;
+	/* Max size to emit ZONE_APPEND write command */
+	u64 max_zone_append_size;
 	struct mutex zoned_meta_io_lock;
 	spinlock_t treelog_bg_lock;
 	u64 treelog_bg;
 	spinlock_t zone_active_bgs_lock;
 	struct list_head zone_active_bgs;
+	/* Waiters when BTRFS_FS_NEED_ZONE_FINISH is set */
+	wait_queue_head_t zone_finish_wait;
+	/* Updates are not protected by any lock */
+	struct btrfs_commit_stats commit_stats;
 #ifdef CONFIG_BTRFS_FS_REF_VERIFY
 	spinlock_t ref_verify_lock;
 			 chunk_root_level, 8);
 BTRFS_SETGET_STACK_FUNCS(super_log_root, struct btrfs_super_block,
 			 log_root, 64);
-BTRFS_SETGET_STACK_FUNCS(super_log_root_transid, struct btrfs_super_block,
-			 log_root_transid, 64);
 BTRFS_SETGET_STACK_FUNCS(super_log_root_level, struct btrfs_super_block,
 			 log_root_level, 8);
 BTRFS_SETGET_STACK_FUNCS(super_total_bytes, struct btrfs_super_block,
 				     enum btrfs_inline_ref_type is_data);
 u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset);
+static inline u8 *btrfs_csum_ptr(const struct btrfs_fs_info *fs_info, u8 *csums,
+				 u64 offset)
+{
+	u64 offset_in_sectors = offset >> fs_info->sectorsize_bits;
+	return csums + offset_in_sectors * fs_info->csum_size;
+}
 /*
- * Take the number of bytes to be checksummmed and figure out how many leaves
+ * Take the number of bytes to be checksummed and figure out how many leaves
  * it would require to store the csums for that many bytes.
  */
 static inline u64 btrfs_csum_bytes_to_leaves(
 u64 btrfs_file_extent_end(const struct btrfs_path *path);
 /* inode.c */
-void btrfs_submit_data_bio(struct inode *inode, struct bio *bio,
+void btrfs_submit_data_write_bio(struct inode *inode, struct bio *bio, int mirror_num);
-			   int mirror_num, enum btrfs_compression_type compress_type);
+void btrfs_submit_data_read_bio(struct inode *inode, struct bio *bio,
+			int mirror_num, enum btrfs_compression_type compress_type);
+int btrfs_check_sector_csum(struct btrfs_fs_info *fs_info, struct page *page,
+			    u32 pgoff, u8 *csum, const u8 * const csum_expected);
+int btrfs_check_data_csum(struct inode *inode, struct btrfs_bio *bbio,
+			  u32 bio_offset, struct page *page, u32 pgoff);
 unsigned int btrfs_verify_data_csum(struct btrfs_bio *bbio,
 				    u32 bio_offset, struct page *page,
 				    u64 start, u64 end);
+int btrfs_check_data_csum(struct inode *inode, struct btrfs_bio *bbio,
+			  u32 bio_offset, struct page *page, u32 pgoff);
 struct extent_map *btrfs_get_extent_fiemap(struct btrfs_inode *inode,
 					   u64 start, u64 len);
 noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len,
 struct inode *btrfs_new_subvol_inode(struct user_namespace *mnt_userns,
 				     struct inode *dir);
  void btrfs_set_delalloc_extent(struct inode *inode, struct extent_state *state,
-			       unsigned *bits);
+			        u32 bits);
 void btrfs_clear_delalloc_extent(struct inode *inode,
-				 struct extent_state *state, unsigned *bits);
+				 struct extent_state *state, u32 bits);
 void btrfs_merge_delalloc_extent(struct inode *inode, struct extent_state *new,
 				 struct extent_state *other);
 void btrfs_split_delalloc_extent(struct inode *inode,
 void btrfs_writepage_endio_finish_ordered(struct btrfs_inode *inode,
 					  struct page *page, u64 start,
 					  u64 end, bool uptodate);
+int btrfs_encoded_io_compression_from_extent(struct btrfs_fs_info *fs_info,
+					     int compress_type);
+int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode,
+					  u64 file_offset, u64 disk_bytenr,
+					  u64 disk_io_size,
+					  struct page **pages);
 ssize_t btrfs_encoded_read(struct kiocb *iocb, struct iov_iter *iter,
 			   struct btrfs_ioctl_encoded_io_args *encoded);
 ssize_t btrfs_do_encoded_write(struct kiocb *iocb, struct iov_iter *from,
 	return fs_info->zone_size > 0;
 }
+/*
+ * Count how many fs_info->max_extent_size cover the @size
+ */
+static inline u32 count_max_extents(struct btrfs_fs_info *fs_info, u64 size)
+{
+#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
+	if (!fs_info)
+		return div_u64(size + BTRFS_MAX_EXTENT_SIZE - 1, BTRFS_MAX_EXTENT_SIZE);
+#endif
+	return div_u64(size + fs_info->max_extent_size - 1, fs_info->max_extent_size);
+}
 static inline bool btrfs_is_data_reloc_root(const struct btrfs_root *root)
 {
 	return root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID;

fs/btrfs/delalloc-space.c+3−3

@@ -273,7 +273,7 @@ static void calc_inode_reservations(struct btrfs_fs_info *fs_info, · 272 unmodified lines
273	u64 num_bytes, u64 disk_num_bytes,	273	u64 num_bytes, u64 disk_num_bytes,
274	u64 meta_reserve, u64 qgroup_reserve)	274	u64 meta_reserve, u64 qgroup_reserve)
275	{	275	{
276	u64 nr_extents = count_max_extents(num_bytes);	276	u64 nr_extents = count_max_extents(fs_info, num_bytes);
277	u64 csum_leaves = btrfs_csum_bytes_to_leaves(fs_info, disk_num_bytes);	277	u64 csum_leaves = btrfs_csum_bytes_to_leaves(fs_info, disk_num_bytes);
278	u64 inode_update = btrfs_calc_metadata_size(fs_info, 1);	278	u64 inode_update = btrfs_calc_metadata_size(fs_info, 1);
279		279
@@ -350,7 +350,7 @@ int btrfs_delalloc_reserve_metadata(struct btrfs_inode *inode, u64 num_bytes, · 70 unmodified lines
350	* needs to free the reservation we just made.	350	* needs to free the reservation we just made.
351	*/	351	*/
352	spin_lock(&inode->lock);	352	spin_lock(&inode->lock);
353	nr_extents = count_max_extents(num_bytes);	353	nr_extents = count_max_extents(fs_info, num_bytes);
354	btrfs_mod_outstanding_extents(inode, nr_extents);	354	btrfs_mod_outstanding_extents(inode, nr_extents);
355	inode->csum_bytes += disk_num_bytes;	355	inode->csum_bytes += disk_num_bytes;
356	btrfs_calculate_inode_block_rsv_size(fs_info, inode);	356	btrfs_calculate_inode_block_rsv_size(fs_info, inode);
@@ -413,7 +413,7 @@ void btrfs_delalloc_release_extents(struct btrfs_inode *inode, u64 num_bytes) · 56 unmodified lines
413	unsigned num_extents;	413	unsigned num_extents;
414		414
415	spin_lock(&inode->lock);	415	spin_lock(&inode->lock);
416	num_extents = count_max_extents(num_bytes);	416	num_extents = count_max_extents(fs_info, num_bytes);
417	btrfs_mod_outstanding_extents(inode, -num_extents);	417	btrfs_mod_outstanding_extents(inode, -num_extents);
418	btrfs_calculate_inode_block_rsv_size(fs_info, inode);	418	btrfs_calculate_inode_block_rsv_size(fs_info, inode);
419	spin_unlock(&inode->lock);	419	spin_unlock(&inode->lock);

fs/btrfs/delayed-inode.c+281−114

@@ -52,18 +52,6 @@ static inline void btrfs_init_delayed_node( · 51 unmodified lines
52	INIT_LIST_HEAD(&delayed_node->p_list);	52	INIT_LIST_HEAD(&delayed_node->p_list);
53	}	53	}
54		54
55	static inline int btrfs_is_continuous_delayed_item(
56	struct btrfs_delayed_item *item1,
57	struct btrfs_delayed_item *item2)
58	{
59	if (item1->key.type == BTRFS_DIR_INDEX_KEY &&
60	item1->key.objectid == item2->key.objectid &&
61	item1->key.type == item2->key.type &&
62	item1->key.offset + 1 == item2->key.offset)
63	return 1;
64	return 0;
65	}
66
67	static struct btrfs_delayed_node *btrfs_get_delayed_node(	55	static struct btrfs_delayed_node *btrfs_get_delayed_node(
68	struct btrfs_inode *btrfs_inode)	56	struct btrfs_inode *btrfs_inode)
69	{	57	{
@@ -398,8 +386,7 @@ static struct btrfs_delayed_item *__btrfs_lookup_delayed_insertion_item( · 328 unmodified lines
398	}	386	}
399		387
400	static int __btrfs_add_delayed_item(struct btrfs_delayed_node *delayed_node,	388	static int __btrfs_add_delayed_item(struct btrfs_delayed_node *delayed_node,
401	struct btrfs_delayed_item *ins,	389	struct btrfs_delayed_item *ins)
402	int action)
403	{	390	{
404	struct rb_node *p, node;	391	struct rb_node *p, node;
405	struct rb_node *parent_node = NULL;	392	struct rb_node *parent_node = NULL;
@@ -408,9 +395,9 @@ static int __btrfs_add_delayed_item(struct btrfs_delayed_node *delayed_node, · 2 unmodified lines
408	int cmp;	395	int cmp;
409	bool leftmost = true;	396	bool leftmost = true;
410		397
411	if (action == BTRFS_DELAYED_INSERTION_ITEM)	398	if (ins->ins_or_del == BTRFS_DELAYED_INSERTION_ITEM)
412	root = &delayed_node->ins_root;	399	root = &delayed_node->ins_root;
413	else if (action == BTRFS_DELAYED_DELETION_ITEM)	400	else if (ins->ins_or_del == BTRFS_DELAYED_DELETION_ITEM)
414	root = &delayed_node->del_root;	401	root = &delayed_node->del_root;
415	else	402	else
416	BUG();	403	BUG();
@@ -436,32 +423,19 @@ static int __btrfs_add_delayed_item(struct btrfs_delayed_node *delayed_node, · 19 unmodified lines
436	rb_link_node(node, parent_node, p);	423	rb_link_node(node, parent_node, p);
437	rb_insert_color_cached(node, root, leftmost);	424	rb_insert_color_cached(node, root, leftmost);
438	ins->delayed_node = delayed_node;	425	ins->delayed_node = delayed_node;
439	ins->ins_or_del = action;
440		426
441	if (ins->key.type == BTRFS_DIR_INDEX_KEY &&	427	/* Delayed items are always for dir index items. */
442	action == BTRFS_DELAYED_INSERTION_ITEM &&	428	ASSERT(ins->key.type == BTRFS_DIR_INDEX_KEY);
		429
		430	if (ins->ins_or_del == BTRFS_DELAYED_INSERTION_ITEM &&
443	ins->key.offset >= delayed_node->index_cnt)	431	ins->key.offset >= delayed_node->index_cnt)
444	delayed_node->index_cnt = ins->key.offset + 1;	432	delayed_node->index_cnt = ins->key.offset + 1;
445		433
446	delayed_node->count++;	434	delayed_node->count++;
447	atomic_inc(&delayed_node->root->fs_info->delayed_root->items);	435	atomic_inc(&delayed_node->root->fs_info->delayed_root->items);
448	return 0;	436	return 0;
449	}	437	}
450		438
451	static int __btrfs_add_delayed_insertion_item(struct btrfs_delayed_node *node,
452	struct btrfs_delayed_item *item)
453	{
454	return __btrfs_add_delayed_item(node, item,
455	BTRFS_DELAYED_INSERTION_ITEM);
456	}
457
458	static int __btrfs_add_delayed_deletion_item(struct btrfs_delayed_node *node,
459	struct btrfs_delayed_item *item)
460	{
461	return __btrfs_add_delayed_item(node, item,
462	BTRFS_DELAYED_DELETION_ITEM);
463	}
464
465	static void finish_one_item(struct btrfs_delayed_root *delayed_root)	439	static void finish_one_item(struct btrfs_delayed_root *delayed_root)
466	{	440	{
467	int seq = atomic_inc_return(&delayed_root->items_seq);	441	int seq = atomic_inc_return(&delayed_root->items_seq);
@@ -573,7 +547,13 @@ static int btrfs_delayed_item_reserve_metadata(struct btrfs_trans_handle *trans, · 105 unmodified lines
573	trace_btrfs_space_reservation(fs_info, "delayed_item",	547	trace_btrfs_space_reservation(fs_info, "delayed_item",
574	item->key.objectid,	548	item->key.objectid,
575	num_bytes, 1);	549	num_bytes, 1);
576	item->bytes_reserved = num_bytes;	550	/*
		551	* For insertions we track reserved metadata space by accounting
		552	* for the number of leaves that will be used, based on the delayed
		553	* node's index_items_size field.
		554	*/
		555	if (item->ins_or_del == BTRFS_DELAYED_DELETION_ITEM)
		556	item->bytes_reserved = num_bytes;
577	}	557	}
578		558
579	return ret;	559	return ret;
@@ -599,6 +579,21 @@ static void btrfs_delayed_item_release_metadata(struct btrfs_root *root, · 19 unmodified lines
599	btrfs_block_rsv_release(fs_info, rsv, item->bytes_reserved, NULL);	579	btrfs_block_rsv_release(fs_info, rsv, item->bytes_reserved, NULL);
600	}	580	}
601		581
		582	static void btrfs_delayed_item_release_leaves(struct btrfs_delayed_node *node,
		583	unsigned int num_leaves)
		584	{
		585	struct btrfs_fs_info *fs_info = node->root->fs_info;
		586	const u64 bytes = btrfs_calc_insert_metadata_size(fs_info, num_leaves);
		587
		588	/* There are no space reservations during log replay, bail out. */
		589	if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags))
		590	return;
		591
		592	trace_btrfs_space_reservation(fs_info, "delayed_item", node->inode_id,
		593	bytes, 0);
		594	btrfs_block_rsv_release(fs_info, &fs_info->delayed_block_rsv, bytes, NULL);
		595	}
		596
602	static int btrfs_delayed_inode_reserve_metadata(	597	static int btrfs_delayed_inode_reserve_metadata(
603	struct btrfs_trans_handle *trans,	598	struct btrfs_trans_handle *trans,
604	struct btrfs_root *root,	599	struct btrfs_root *root,
@@ -672,22 +667,53 @@ static void btrfs_delayed_inode_release_metadata(struct btrfs_fs_info *fs_info, · 67 unmodified lines
672	}	667	}
673		668
674	/*	669	/*
675	* Insert a single delayed item or a batch of delayed items that have consecutive	670	* Insert a single delayed item or a batch of delayed items, as many as possible
676	* keys if they exist.	671	* that fit in a leaf. The delayed items (dir index keys) are sorted by their key
		672	* in the rbtree, and if there's a gap between two consecutive dir index items,
		673	* then it means at some point we had delayed dir indexes to add but they got
		674	* removed (by btrfs_delete_delayed_dir_index()) before we attempted to flush them
		675	* into the subvolume tree. Dir index keys also have their offsets coming from a
		676	* monotonically increasing counter, so we can't get new keys with an offset that
		677	* fits within a gap between delayed dir index items.
677	*/	678	*/
678	static int btrfs_insert_delayed_item(struct btrfs_trans_handle *trans,	679	static int btrfs_insert_delayed_item(struct btrfs_trans_handle *trans,
679	struct btrfs_root *root,	680	struct btrfs_root *root,
680	struct btrfs_path *path,	681	struct btrfs_path *path,
681	struct btrfs_delayed_item *first_item)	682	struct btrfs_delayed_item *first_item)
682	{	683	{
		684	struct btrfs_fs_info *fs_info = root->fs_info;
		685	struct btrfs_delayed_node *node = first_item->delayed_node;
683	LIST_HEAD(item_list);	686	LIST_HEAD(item_list);
684	struct btrfs_delayed_item *curr;	687	struct btrfs_delayed_item *curr;
685	struct btrfs_delayed_item *next;	688	struct btrfs_delayed_item *next;
686	const int max_size = BTRFS_LEAF_DATA_SIZE(root->fs_info);	689	const int max_size = BTRFS_LEAF_DATA_SIZE(fs_info);
687	struct btrfs_item_batch batch;	690	struct btrfs_item_batch batch;
688	int total_size;	691	int total_size;
689	char *ins_data = NULL;	692	char *ins_data = NULL;
690	int ret;	693	int ret;
		694	bool continuous_keys_only = false;
		695
		696	lockdep_assert_held(&node->mutex);
		697
		698	/*
		699	* During normal operation the delayed index offset is continuously
		700	* increasing, so we can batch insert all items as there will not be any
		701	* overlapping keys in the tree.
		702	*
		703	* The exception to this is log replay, where we may have interleaved
		704	* offsets in the tree, so our batch needs to be continuous keys only in
		705	* order to ensure we do not end up with out of order items in our leaf.
		706	*/
		707	if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags))
		708	continuous_keys_only = true;
		709
		710	/*
		711	* For delayed items to insert, we track reserved metadata bytes based
		712	* on the number of leaves that we will use.
		713	* See btrfs_insert_delayed_dir_index() and
		714	* btrfs_delayed_item_reserve_metadata()).
		715	*/
		716	ASSERT(first_item->bytes_reserved == 0);
691		717
692	list_add_tail(&first_item->tree_list, &item_list);	718	list_add_tail(&first_item->tree_list, &item_list);
693	batch.total_data_size = first_item->data_len;	719	batch.total_data_size = first_item->data_len;
@@ -699,9 +725,19 @@ static int btrfs_insert_delayed_item(struct btrfs_trans_handle *trans, · 5 unmodified lines
699	int next_size;	725	int next_size;
700		726
701	next = __btrfs_next_delayed_item(curr);	727	next = __btrfs_next_delayed_item(curr);
702	if (!next \|\| !btrfs_is_continuous_delayed_item(curr, next))	728	if (!next)
		729	break;
		730
		731	/*
		732	* We cannot allow gaps in the key space if we're doing log
		733	* replay.
		734	*/
		735	if (continuous_keys_only &&
		736	(next->key.offset != curr->key.offset + 1))
703	break;	737	break;
704		738
		739	ASSERT(next->bytes_reserved == 0);
		740
705	next_size = next->data_len + sizeof(struct btrfs_item);	741	next_size = next->data_len + sizeof(struct btrfs_item);
706	if (total_size + next_size > max_size)	742	if (total_size + next_size > max_size)
707	break;	743	break;
@@ -758,9 +794,41 @@ static int btrfs_insert_delayed_item(struct btrfs_trans_handle *trans, · 50 unmodified lines
758	*/	794	*/
759	btrfs_release_path(path);	795	btrfs_release_path(path);
760		796
		797	ASSERT(node->index_item_leaves > 0);
		798
		799	/*
		800	* For normal operations we will batch an entire leaf's worth of delayed
		801	* items, so if there are more items to process we can decrement
		802	* index_item_leaves by 1 as we inserted 1 leaf's worth of items.
		803	*
		804	* However for log replay we may not have inserted an entire leaf's
		805	* worth of items, we may have not had continuous items, so decrementing
		806	* here would mess up the index_item_leaves accounting. For this case
		807	* only clean up the accounting when there are no items left.
		808	*/
		809	if (next && !continuous_keys_only) {
		810	/*
		811	* We inserted one batch of items into a leaf a there are more
		812	* items to flush in a future batch, now release one unit of
		813	* metadata space from the delayed block reserve, corresponding
		814	* the leaf we just flushed to.
		815	*/
		816	btrfs_delayed_item_release_leaves(node, 1);
		817	node->index_item_leaves--;
		818	} else if (!next) {
		819	/*
		820	* There are no more items to insert. We can have a number of
		821	* reserved leaves > 1 here - this happens when many dir index
		822	* items are added and then removed before they are flushed (file
		823	* names with a very short life, never span a transaction). So
		824	* release all remaining leaves.
		825	*/
		826	btrfs_delayed_item_release_leaves(node, node->index_item_leaves);
		827	node->index_item_leaves = 0;
		828	}
		829
761	list_for_each_entry_safe(curr, next, &item_list, tree_list) {	830	list_for_each_entry_safe(curr, next, &item_list, tree_list) {
762	list_del(&curr->tree_list);	831	list_del(&curr->tree_list);
763	btrfs_delayed_item_release_metadata(root, curr);
764	btrfs_release_delayed_item(curr);	832	btrfs_release_delayed_item(curr);
765	}	833	}
766	out:	834	out:
@@ -796,62 +864,75 @@ static int btrfs_batch_delete_items(struct btrfs_trans_handle *trans, · 29 unmodified lines
796	struct btrfs_path *path,	864	struct btrfs_path *path,
797	struct btrfs_delayed_item *item)	865	struct btrfs_delayed_item *item)
798	{	866	{
		867	struct btrfs_fs_info *fs_info = root->fs_info;
799	struct btrfs_delayed_item curr, next;	868	struct btrfs_delayed_item curr, next;
800	struct extent_buffer *leaf;	869	struct extent_buffer *leaf = path->nodes[0];
801	struct btrfs_key key;	870	LIST_HEAD(batch_list);
802	struct list_head head;	871	int nitems, slot, last_slot;
803	int nitems, i, last_item;	872	int ret;
804	int ret = 0;	873	u64 total_reserved_size = item->bytes_reserved;
805		874
806	BUG_ON(!path->nodes[0]);	875	ASSERT(leaf != NULL);
807		876
808	leaf = path->nodes[0];	877	slot = path->slots[0];
		878	last_slot = btrfs_header_nritems(leaf) - 1;
		879	/*
		880	* Our caller always gives us a path pointing to an existing item, so
		881	* this can not happen.
		882	*/
		883	ASSERT(slot <= last_slot);
		884	if (WARN_ON(slot > last_slot))
		885	return -ENOENT;
809		886
810	i = path->slots[0];	887	nitems = 1;
811	last_item = btrfs_header_nritems(leaf) - 1;	888	curr = item;
812	if (i > last_item)	889	list_add_tail(&curr->tree_list, &batch_list);
813	return -ENOENT; /* FIXME: Is errno suitable? */
814		890
815	next = item;
816	INIT_LIST_HEAD(&head);
817	btrfs_item_key_to_cpu(leaf, &key, i);
818	nitems = 0;
819	/*	891	/*
820	* count the number of the dir index items that we can delete in batch	892	* Keep checking if the next delayed item matches the next item in the
		893	* leaf - if so, we can add it to the batch of items to delete from the
		894	* leaf.
821	*/	895	*/
822	while (btrfs_comp_cpu_keys(&next->key, &key) == 0) {	896	while (slot < last_slot) {
823	list_add_tail(&next->tree_list, &head);	897	struct btrfs_key key;
824	nitems++;
825		898
826	curr = next;
827	next = __btrfs_next_delayed_item(curr);	899	next = __btrfs_next_delayed_item(curr);
828	if (!next)	900	if (!next)
829	break;	901	break;
830		902
831	if (!btrfs_is_continuous_delayed_item(curr, next))	903	slot++;
832	break;	904	btrfs_item_key_to_cpu(leaf, &key, slot);
833		905	if (btrfs_comp_cpu_keys(&next->key, &key) != 0)
834	i++;
835	if (i > last_item)
836	break;	906	break;
837	btrfs_item_key_to_cpu(leaf, &key, i);	907	nitems++;
		908	curr = next;
		909	list_add_tail(&curr->tree_list, &batch_list);
		910	total_reserved_size += curr->bytes_reserved;
838	}	911	}
839		912
840	if (!nitems)
841	return 0;
842
843	ret = btrfs_del_items(trans, root, path, path->slots[0], nitems);	913	ret = btrfs_del_items(trans, root, path, path->slots[0], nitems);
844	if (ret)	914	if (ret)
845	goto out;	915	return ret;
		916
		917	/* In case of BTRFS_FS_LOG_RECOVERING items won't have reserved space */
		918	if (total_reserved_size > 0) {
		919	/*
		920	* Check btrfs_delayed_item_reserve_metadata() to see why we
		921	* don't need to release/reserve qgroup space.
		922	*/
		923	trace_btrfs_space_reservation(fs_info, "delayed_item",
		924	item->key.objectid, total_reserved_size,
		925	0);
		926	btrfs_block_rsv_release(fs_info, &fs_info->delayed_block_rsv,
		927	total_reserved_size, NULL);
		928	}
846		929
847	list_for_each_entry_safe(curr, next, &head, tree_list) {	930	list_for_each_entry_safe(curr, next, &batch_list, tree_list) {
848	btrfs_delayed_item_release_metadata(root, curr);
849	list_del(&curr->tree_list);	931	list_del(&curr->tree_list);
850	btrfs_release_delayed_item(curr);	932	btrfs_release_delayed_item(curr);
851	}	933	}
852		934
853	out:	935	return 0;
854	return ret;
855	}	936	}
856		937
857	static int btrfs_delete_delayed_items(struct btrfs_trans_handle *trans,	938	static int btrfs_delete_delayed_items(struct btrfs_trans_handle *trans,
@@ -859,43 +940,52 @@ static int btrfs_delete_delayed_items(struct btrfs_trans_handle *trans, · 1 unmodified line
859	struct btrfs_root *root,	940	struct btrfs_root *root,
860	struct btrfs_delayed_node *node)	941	struct btrfs_delayed_node *node)
861	{	942	{
862	struct btrfs_delayed_item curr, prev;
863	int ret = 0;	943	int ret = 0;
864		944
865	do_again:	945	while (ret == 0) {
866	mutex_lock(&node->mutex);	946	struct btrfs_delayed_item *item;
867	curr = __btrfs_first_delayed_deletion_item(node);	947
868	if (!curr)	948	mutex_lock(&node->mutex);
869	goto delete_fail;	949	item = __btrfs_first_delayed_deletion_item(node);
		950	if (!item) {
		951	mutex_unlock(&node->mutex);
		952	break;
		953	}
		954
		955	ret = btrfs_search_slot(trans, root, &item->key, path, -1, 1);
		956	if (ret > 0) {
		957	/*
		958	* There's no matching item in the leaf. This means we
		959	* have already deleted this item in a past run of the
		960	* delayed items. We ignore errors when running delayed
		961	* items from an async context, through a work queue job
		962	* running btrfs_async_run_delayed_root(), and don't
		963	* release delayed items that failed to complete. This
		964	* is because we will retry later, and at transaction
		965	* commit time we always run delayed items and will
		966	* then deal with errors if they fail to run again.
		967	*
		968	* So just release delayed items for which we can't find
		969	* an item in the tree, and move to the next item.
		970	*/
		971	btrfs_release_path(path);
		972	btrfs_release_delayed_item(item);
		973	ret = 0;
		974	} else if (ret == 0) {
		975	ret = btrfs_batch_delete_items(trans, root, path, item);
		976	btrfs_release_path(path);
		977	}
870		978
871	ret = btrfs_search_slot(trans, root, &curr->key, path, -1, 1);
872	if (ret < 0)
873	goto delete_fail;
874	else if (ret > 0) {
875	/*	979	/*
876	* can't find the item which the node points to, so this node	980	* We unlock and relock on each iteration, this is to prevent
877	* is invalid, just drop it.	981	* blocking other tasks for too long while we are being run from
		982	* the async context (work queue job). Those tasks are typically
		983	* running system calls like creat/mkdir/rename/unlink/etc which
		984	* need to add delayed items to this delayed node.
878	*/	985	*/
879	prev = curr;	986	mutex_unlock(&node->mutex);
880	curr = __btrfs_next_delayed_item(prev);
881	btrfs_release_delayed_item(prev);
882	ret = 0;
883	btrfs_release_path(path);
884	if (curr) {
885	mutex_unlock(&node->mutex);
886	goto do_again;
887	} else
888	goto delete_fail;
889	}	987	}
890		988
891	btrfs_batch_delete_items(trans, root, path, curr);
892	btrfs_release_path(path);
893	mutex_unlock(&node->mutex);
894	goto do_again;
895
896	delete_fail:
897	btrfs_release_path(path);
898	mutex_unlock(&node->mutex);
899	return ret;	989	return ret;
900	}	990	}
901		991
@@ -1354,9 +1444,13 @@ int btrfs_insert_delayed_dir_index(struct btrfs_trans_handle *trans, · 452 unmodified lines
1354	struct btrfs_disk_key *disk_key, u8 type,	1444	struct btrfs_disk_key *disk_key, u8 type,
1355	u64 index)	1445	u64 index)
1356	{	1446	{
		1447	struct btrfs_fs_info *fs_info = trans->fs_info;
		1448	const unsigned int leaf_data_size = BTRFS_LEAF_DATA_SIZE(fs_info);
1357	struct btrfs_delayed_node *delayed_node;	1449	struct btrfs_delayed_node *delayed_node;
1358	struct btrfs_delayed_item *delayed_item;	1450	struct btrfs_delayed_item *delayed_item;
1359	struct btrfs_dir_item *dir_item;	1451	struct btrfs_dir_item *dir_item;
		1452	bool reserve_leaf_space;
		1453	u32 data_len;
1360	int ret;	1454	int ret;
1361		1455
1362	delayed_node = btrfs_get_or_create_delayed_node(dir);	1456	delayed_node = btrfs_get_or_create_delayed_node(dir);
@@ -1372,6 +1466,7 @@ int btrfs_insert_delayed_dir_index(struct btrfs_trans_handle *trans, · 9 unmodified lines
1372	delayed_item->key.objectid = btrfs_ino(dir);	1466	delayed_item->key.objectid = btrfs_ino(dir);
1373	delayed_item->key.type = BTRFS_DIR_INDEX_KEY;	1467	delayed_item->key.type = BTRFS_DIR_INDEX_KEY;
1374	delayed_item->key.offset = index;	1468	delayed_item->key.offset = index;
		1469	delayed_item->ins_or_del = BTRFS_DELAYED_INSERTION_ITEM;
1375		1470
1376	dir_item = (struct btrfs_dir_item *)delayed_item->data;	1471	dir_item = (struct btrfs_dir_item *)delayed_item->data;
1377	dir_item->location = *disk_key;	1472	dir_item->location = *disk_key;
@@ -1381,15 +1476,52 @@ int btrfs_insert_delayed_dir_index(struct btrfs_trans_handle *trans, · 3 unmodified lines
1381	btrfs_set_stack_dir_type(dir_item, type);	1476	btrfs_set_stack_dir_type(dir_item, type);
1382	memcpy((char *)(dir_item + 1), name, name_len);	1477	memcpy((char *)(dir_item + 1), name, name_len);
1383		1478
1384	ret = btrfs_delayed_item_reserve_metadata(trans, dir->root, delayed_item);	1479	data_len = delayed_item->data_len + sizeof(struct btrfs_item);
1385	/*
1386	* we have reserved enough space when we start a new transaction,
1387	* so reserving metadata failure is impossible
1388	*/
1389	BUG_ON(ret);
1390		1480
1391	mutex_lock(&delayed_node->mutex);	1481	mutex_lock(&delayed_node->mutex);
1392	ret = __btrfs_add_delayed_insertion_item(delayed_node, delayed_item);	1482
		1483	if (delayed_node->index_item_leaves == 0 \|\|
		1484	delayed_node->curr_index_batch_size + data_len > leaf_data_size) {
		1485	delayed_node->curr_index_batch_size = data_len;
		1486	reserve_leaf_space = true;
		1487	} else {
		1488	delayed_node->curr_index_batch_size += data_len;
		1489	reserve_leaf_space = false;
		1490	}
		1491
		1492	if (reserve_leaf_space) {
		1493	ret = btrfs_delayed_item_reserve_metadata(trans, dir->root,
		1494	delayed_item);
		1495	/*
		1496	* Space was reserved for a dir index item insertion when we
		1497	* started the transaction, so getting a failure here should be
		1498	* impossible.
		1499	*/
		1500	if (WARN_ON(ret)) {
		1501	mutex_unlock(&delayed_node->mutex);
		1502	btrfs_release_delayed_item(delayed_item);
		1503	goto release_node;
		1504	}
		1505
		1506	delayed_node->index_item_leaves++;
		1507	} else if (!test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) {
		1508	const u64 bytes = btrfs_calc_insert_metadata_size(fs_info, 1);
		1509
		1510	/*
		1511	* Adding the new dir index item does not require touching another
		1512	* leaf, so we can release 1 unit of metadata that was previously
		1513	* reserved when starting the transaction. This applies only to
		1514	* the case where we had a transaction start and excludes the
		1515	* transaction join case (when replaying log trees).
		1516	*/
		1517	trace_btrfs_space_reservation(fs_info, "transaction",
		1518	trans->transid, bytes, 0);
		1519	btrfs_block_rsv_release(fs_info, trans->block_rsv, bytes, NULL);
		1520	ASSERT(trans->bytes_reserved >= bytes);
		1521	trans->bytes_reserved -= bytes;
		1522	}
		1523
		1524	ret = __btrfs_add_delayed_item(delayed_node, delayed_item);
1393	if (unlikely(ret)) {	1525	if (unlikely(ret)) {
1394	btrfs_err(trans->fs_info,	1526	btrfs_err(trans->fs_info,
1395	"err add delayed dir index item(name: %.*s) into the insertion tree of the delayed node(root id: %llu, inode id: %llu, errno: %d)",	1527	"err add delayed dir index item(name: %.*s) into the insertion tree of the delayed node(root id: %llu, inode id: %llu, errno: %d)",
@@ -1417,8 +1549,37 @@ static int btrfs_delete_delayed_insertion_item(struct btrfs_fs_info *fs_info, · 21 unmodified lines
1417	return 1;	1549	return 1;
1418	}	1550	}
1419		1551
1420	btrfs_delayed_item_release_metadata(node->root, item);	1552	/*
		1553	* For delayed items to insert, we track reserved metadata bytes based
		1554	* on the number of leaves that we will use.
		1555	* See btrfs_insert_delayed_dir_index() and
		1556	* btrfs_delayed_item_reserve_metadata()).
		1557	*/
		1558	ASSERT(item->bytes_reserved == 0);
		1559	ASSERT(node->index_item_leaves > 0);
		1560
		1561	/*
		1562	* If there's only one leaf reserved, we can decrement this item from the
		1563	* current batch, otherwise we can not because we don't know which leaf
		1564	* it belongs to. With the current limit on delayed items, we rarely
		1565	* accumulate enough dir index items to fill more than one leaf (even
		1566	* when using a leaf size of 4K).
		1567	*/
		1568	if (node->index_item_leaves == 1) {
		1569	const u32 data_len = item->data_len + sizeof(struct btrfs_item);
		1570
		1571	ASSERT(node->curr_index_batch_size >= data_len);
		1572	node->curr_index_batch_size -= data_len;
		1573	}
		1574
1421	btrfs_release_delayed_item(item);	1575	btrfs_release_delayed_item(item);
		1576
		1577	/* If we now have no more dir index items, we can release all leaves. */
		1578	if (RB_EMPTY_ROOT(&node->ins_root.rb_root)) {
		1579	btrfs_delayed_item_release_leaves(node, node->index_item_leaves);
		1580	node->index_item_leaves = 0;
		1581	}
		1582
1422	mutex_unlock(&node->mutex);	1583	mutex_unlock(&node->mutex);
1423	return 0;	1584	return 0;
1424	}	1585	}
@@ -1451,6 +1612,7 @@ int btrfs_delete_delayed_dir_index(struct btrfs_trans_handle *trans, · 26 unmodified lines
1451	}	1612	}
1452		1613
1453	item->key = item_key;	1614	item->key = item_key;
		1615	item->ins_or_del = BTRFS_DELAYED_DELETION_ITEM;
1454		1616
1455	ret = btrfs_delayed_item_reserve_metadata(trans, dir->root, item);	1617	ret = btrfs_delayed_item_reserve_metadata(trans, dir->root, item);
1456	/*	1618	/*
@@ -1465,7 +1627,7 @@ int btrfs_delete_delayed_dir_index(struct btrfs_trans_handle *trans, · 8 unmodified lines
1465	}	1627	}
1466		1628
1467	mutex_lock(&node->mutex);	1629	mutex_lock(&node->mutex);
1468	ret = __btrfs_add_delayed_deletion_item(node, item);	1630	ret = __btrfs_add_delayed_item(node, item);
1469	if (unlikely(ret)) {	1631	if (unlikely(ret)) {
1470	btrfs_err(trans->fs_info,	1632	btrfs_err(trans->fs_info,
1471	"err add delayed dir index item(index: %llu) into the deletion tree of the delayed node(root id: %llu, inode id: %llu, errno: %d)",	1633	"err add delayed dir index item(index: %llu) into the deletion tree of the delayed node(root id: %llu, inode id: %llu, errno: %d)",
@@ -1833,12 +1995,17 @@ static void __btrfs_kill_delayed_node(struct btrfs_delayed_node *delayed_node) · 361 unmodified lines
1833	mutex_lock(&delayed_node->mutex);	1995	mutex_lock(&delayed_node->mutex);
1834	curr_item = __btrfs_first_delayed_insertion_item(delayed_node);	1996	curr_item = __btrfs_first_delayed_insertion_item(delayed_node);
1835	while (curr_item) {	1997	while (curr_item) {
1836	btrfs_delayed_item_release_metadata(root, curr_item);
1837	prev_item = curr_item;	1998	prev_item = curr_item;
1838	curr_item = __btrfs_next_delayed_item(prev_item);	1999	curr_item = __btrfs_next_delayed_item(prev_item);
1839	btrfs_release_delayed_item(prev_item);	2000	btrfs_release_delayed_item(prev_item);
1840	}	2001	}
1841		2002
		2003	if (delayed_node->index_item_leaves > 0) {
		2004	btrfs_delayed_item_release_leaves(delayed_node,
		2005	delayed_node->index_item_leaves);
		2006	delayed_node->index_item_leaves = 0;
		2007	}
		2008
1842	curr_item = __btrfs_first_delayed_deletion_item(delayed_node);	2009	curr_item = __btrfs_first_delayed_deletion_item(delayed_node);
1843	while (curr_item) {	2010	while (curr_item) {
1844	btrfs_delayed_item_release_metadata(root, curr_item);	2011	btrfs_delayed_item_release_metadata(root, curr_item);

fs/btrfs/delayed-inode.h+11−0

@@ -58,6 +58,17 @@ struct btrfs_delayed_node { · 57 unmodified lines
58	u64 index_cnt;	58	u64 index_cnt;
59	unsigned long flags;	59	unsigned long flags;
60	int count;	60	int count;
		61	/*
		62	* The size of the next batch of dir index items to insert (if this
		63	* node is from a directory inode). Protected by @mutex.
		64	*/
		65	u32 curr_index_batch_size;
		66	/*
		67	* Number of leaves reserved for inserting dir index items (if this
		68	* node belongs to a directory inode). This may be larger then the
		69	* actual number of leaves we end up using. Protected by @mutex.
		70	*/
		71	u32 index_item_leaves;
61	};	72	};
62		73
63	struct btrfs_delayed_item {	74	struct btrfs_delayed_item {

fs/btrfs/delayed-ref.c+2−2

@@ -132,7 +132,7 @@ void btrfs_update_delayed_refs_rsv(struct btrfs_trans_handle *trans) · 131 unmodified lines
132		132
133	spin_lock(&delayed_rsv->lock);	133	spin_lock(&delayed_rsv->lock);
134	delayed_rsv->size += num_bytes;	134	delayed_rsv->size += num_bytes;
135	delayed_rsv->full = 0;	135	delayed_rsv->full = false;
136	spin_unlock(&delayed_rsv->lock);	136	spin_unlock(&delayed_rsv->lock);
137	trans->delayed_ref_updates = 0;	137	trans->delayed_ref_updates = 0;
138	}	138	}
@@ -175,7 +175,7 @@ void btrfs_migrate_to_delayed_refs_rsv(struct btrfs_fs_info *fs_info, · 36 unmodified lines
175	if (num_bytes)	175	if (num_bytes)
176	delayed_refs_rsv->reserved += num_bytes;	176	delayed_refs_rsv->reserved += num_bytes;
177	if (delayed_refs_rsv->reserved >= delayed_refs_rsv->size)	177	if (delayed_refs_rsv->reserved >= delayed_refs_rsv->size)
178	delayed_refs_rsv->full = 1;	178	delayed_refs_rsv->full = true;
179	spin_unlock(&delayed_refs_rsv->lock);	179	spin_unlock(&delayed_refs_rsv->lock);
180		180
181	if (num_bytes)	181	if (num_bytes)

fs/btrfs/dev-replace.c+2−1

@@ -587,7 +587,8 @@ bool btrfs_finish_block_group_to_copy(struct btrfs_device *srcdev, · 586 unmodified lines
587	ASSERT(!IS_ERR(em));	587	ASSERT(!IS_ERR(em));
588	map = em->map_lookup;	588	map = em->map_lookup;
589		589
590	num_extents = cur_extent = 0;	590	num_extents = 0;
		591	cur_extent = 0;
591	for (i = 0; i < map->num_stripes; i++) {	592	for (i = 0; i < map->num_stripes; i++) {
592	/* We have more device extent to copy */	593	/* We have more device extent to copy */
593	if (srcdev != map->stripes[i].dev)	594	if (srcdev != map->stripes[i].dev)

fs/btrfs/disk-io.c+88−180

 				 BTRFS_SUPER_FLAG_METADUMP |\
 				 BTRFS_SUPER_FLAG_METADUMP_V2)
-static void end_workqueue_fn(struct btrfs_work *work);
 static void btrfs_destroy_ordered_extents(struct btrfs_root *root);
 static int btrfs_destroy_delayed_refs(struct btrfs_transaction *trans,
 				      struct btrfs_fs_info *fs_info);
 static int btrfs_cleanup_transaction(struct btrfs_fs_info *fs_info);
 static void btrfs_error_commit_super(struct btrfs_fs_info *fs_info);
-/*
- * btrfs_end_io_wq structs are used to do processing in task context when an IO
- * is complete.  This is used during reads to verify checksums, and it is used
- * by writes to insert metadata for new file extents after IO is complete.
- */
-struct btrfs_end_io_wq {
-	struct bio *bio;
-	bio_end_io_t *end_io;
-	void *private;
-	struct btrfs_fs_info *info;
-	blk_status_t status;
-	enum btrfs_wq_endio_type metadata;
-	struct btrfs_work work;
-};
-static struct kmem_cache *btrfs_end_io_wq_cache;
-int __init btrfs_end_io_wq_init(void)
-{
-	btrfs_end_io_wq_cache = kmem_cache_create("btrfs_end_io_wq",
-					sizeof(struct btrfs_end_io_wq),
-					0,
-					SLAB_MEM_SPREAD,
-					NULL);
-	if (!btrfs_end_io_wq_cache)
-		return -ENOMEM;
-	return 0;
-}
-void __cold btrfs_end_io_wq_exit(void)
-{
-	kmem_cache_destroy(btrfs_end_io_wq_cache);
-}
 static void btrfs_free_csum_hash(struct btrfs_fs_info *fs_info)
 {
 	if (fs_info->csum_shash)
 		goto out;
 	}
 	btrfs_err_rl(eb->fs_info,
-		"parent transid verify failed on %llu wanted %llu found %llu",
+"parent transid verify failed on logical %llu mirror %u wanted %llu found %llu",
-			eb->start,
+			eb->start, eb->read_mirror,
 			parent_transid, btrfs_header_generation(eb));
 	ret = 1;
 	clear_extent_buffer_uptodate(eb);
 	found_start = btrfs_header_bytenr(eb);
 	if (found_start != eb->start) {
-		btrfs_err_rl(fs_info, "bad tree block start, want %llu have %llu",
+		btrfs_err_rl(fs_info,
-			     eb->start, found_start);
+			"bad tree block start, mirror %u want %llu have %llu",
+			     eb->read_mirror, eb->start, found_start);
 		ret = -EIO;
 		goto out;
 	}
 	if (check_tree_block_fsid(eb)) {
-		btrfs_err_rl(fs_info, "bad fsid on block %llu",
+		btrfs_err_rl(fs_info, "bad fsid on logical %llu mirror %u",
-			     eb->start);
+			     eb->start, eb->read_mirror);
 		ret = -EIO;
 		goto out;
 	}
 	found_level = btrfs_header_level(eb);
 	if (found_level >= BTRFS_MAX_LEVEL) {
-		btrfs_err(fs_info, "bad tree block level %d on %llu",
+		btrfs_err(fs_info,
-			  (int)btrfs_header_level(eb), eb->start);
+			"bad tree block level, mirror %u level %d on logical %llu",
+			eb->read_mirror, btrfs_header_level(eb), eb->start);
 		ret = -EIO;
 		goto out;
 	}
 	if (memcmp(result, header_csum, csum_size) != 0) {
 		btrfs_warn_rl(fs_info,
-	"checksum verify failed on %llu wanted " CSUM_FMT " found " CSUM_FMT " level %d",
+"checksum verify failed on logical %llu mirror %u wanted " CSUM_FMT " found " CSUM_FMT " level %d",
-			      eb->start,
+			      eb->start, eb->read_mirror,
 			      CSUM_FMT_VALUE(csum_size, header_csum),
 			      CSUM_FMT_VALUE(csum_size, result),
 			      btrfs_header_level(eb));
 		set_extent_buffer_uptodate(eb);
 	else
 		btrfs_err(fs_info,
-			  "block=%llu read time tree block corruption detected",
+		"read time tree block corruption detected on logical %llu mirror %u",
-			  eb->start);
+			  eb->start, eb->read_mirror);
 out:
 	return ret;
 }
 	return ret;
 }
-static void end_workqueue_bio(struct bio *bio)
-{
-	struct btrfs_end_io_wq *end_io_wq = bio->bi_private;
-	struct btrfs_fs_info *fs_info;
-	struct btrfs_workqueue *wq;
-	fs_info = end_io_wq->info;
-	end_io_wq->status = bio->bi_status;
-	if (btrfs_op(bio) == BTRFS_MAP_WRITE) {
-		if (end_io_wq->metadata == BTRFS_WQ_ENDIO_METADATA)
-			wq = fs_info->endio_meta_write_workers;
-		else if (end_io_wq->metadata == BTRFS_WQ_ENDIO_FREE_SPACE)
-			wq = fs_info->endio_freespace_worker;
-		else if (end_io_wq->metadata == BTRFS_WQ_ENDIO_RAID56)
-			wq = fs_info->endio_raid56_workers;
-		else
-			wq = fs_info->endio_write_workers;
-	} else {
-		if (end_io_wq->metadata == BTRFS_WQ_ENDIO_RAID56)
-			wq = fs_info->endio_raid56_workers;
-		else if (end_io_wq->metadata)
-			wq = fs_info->endio_meta_workers;
-		else
-			wq = fs_info->endio_workers;
-	}
-	btrfs_init_work(&end_io_wq->work, end_workqueue_fn, NULL, NULL);
-	btrfs_queue_work(wq, &end_io_wq->work);
-}
-blk_status_t btrfs_bio_wq_end_io(struct btrfs_fs_info *info, struct bio *bio,
-			enum btrfs_wq_endio_type metadata)
-{
-	struct btrfs_end_io_wq *end_io_wq;
-	end_io_wq = kmem_cache_alloc(btrfs_end_io_wq_cache, GFP_NOFS);
-	if (!end_io_wq)
-		return BLK_STS_RESOURCE;
-	end_io_wq->private = bio->bi_private;
-	end_io_wq->end_io = bio->bi_end_io;
-	end_io_wq->info = info;
-	end_io_wq->status = 0;
-	end_io_wq->bio = bio;
-	end_io_wq->metadata = metadata;
-	bio->bi_private = end_io_wq;
-	bio->bi_end_io = end_workqueue_bio;
-	return 0;
-}
 static void run_one_async_start(struct btrfs_work *work)
 {
 	struct async_submit_bio *async;
 {
 	struct async_submit_bio *async;
 	struct inode *inode;
-	blk_status_t ret;
 	async = container_of(work, struct  async_submit_bio, work);
 	inode = async->inode;
 	 * This changes nothing when cgroups aren't in use.
 	 */
 	async->bio->bi_opf |= REQ_CGROUP_PUNT;
-	ret = btrfs_map_bio(btrfs_sb(inode->i_sb), async->bio, async->mirror_num);
+	btrfs_submit_bio(btrfs_sb(inode->i_sb), async->bio, async->mirror_num);
-	if (ret) {
-		async->bio->bi_status = ret;
-		bio_endio(async->bio);
-	}
 }
 static void run_one_async_free(struct btrfs_work *work)
 	kfree(async);
 }
-blk_status_t btrfs_wq_submit_bio(struct inode *inode, struct bio *bio,
+/*
-				 int mirror_num, u64 dio_file_offset,
+ * Submit bio to an async queue.
-				 extent_submit_bio_start_t *submit_bio_start)
+ *
+ * Retrun:
+ * - true if the work has been succesfuly submitted
+ * - false in case of error
+ */
+bool btrfs_wq_submit_bio(struct inode *inode, struct bio *bio, int mirror_num,
+			 u64 dio_file_offset,
+			 extent_submit_bio_start_t *submit_bio_start)
 {
 	struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
 	struct async_submit_bio *async;
 	async = kmalloc(sizeof(*async), GFP_NOFS);
 	if (!async)
-		return BLK_STS_RESOURCE;
+		return false;
 	async->inode = inode;
 	async->bio = bio;
 		btrfs_queue_work(fs_info->hipri_workers, &async->work);
 	else
 		btrfs_queue_work(fs_info->workers, &async->work);
-	return 0;
+	return true;
 }
 static blk_status_t btree_csum_one_bio(struct bio *bio)
 {
 	/*
 	 * when we're called for a write, we're already in the async
-	 * submission context.  Just jump into btrfs_map_bio
+	 * submission context.  Just jump into btrfs_submit_bio.
 	 */
 	return btree_csum_one_bio(bio);
 }
 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
 	blk_status_t ret;
+	bio->bi_opf |= REQ_META;
 	if (btrfs_op(bio) != BTRFS_MAP_WRITE) {
-		/*
+		btrfs_submit_bio(fs_info, bio, mirror_num);
-		 * called for a read, do the setup so that checksum validation
+		return;
-		 * can happen in the async kernel threads
-		 */
-		ret = btrfs_bio_wq_end_io(fs_info, bio,
-					  BTRFS_WQ_ENDIO_METADATA);
-		if (!ret)
-			ret = btrfs_map_bio(fs_info, bio, mirror_num);
-	} else if (!should_async_write(fs_info, BTRFS_I(inode))) {
-		ret = btree_csum_one_bio(bio);
-		if (!ret)
-			ret = btrfs_map_bio(fs_info, bio, mirror_num);
-	} else {
-		/*
-		 * kthread helpers are used to submit writes so that
-		 * checksumming can happen in parallel across all CPUs
-		 */
-		ret = btrfs_wq_submit_bio(inode, bio, mirror_num, 0,
-					  btree_submit_bio_start);
 	}
+	/*
+	 * Kthread helpers are used to submit writes so that checksumming can
+	 * happen in parallel across all CPUs.
+	 */
+	if (should_async_write(fs_info, BTRFS_I(inode)) &&
+	    btrfs_wq_submit_bio(inode, bio, mirror_num, 0, btree_submit_bio_start))
+		return;
+	ret = btree_csum_one_bio(bio);
 	if (ret) {
 		bio->bi_status = ret;
 		bio_endio(bio);
+		return;
 	}
+	btrfs_submit_bio(fs_info, bio, mirror_num);
 }
 #ifdef CONFIG_MIGRATION
 fail:
 	/*
 	 * If our caller provided us an anonymous device, then it's his
-	 * responsability to free it in case we fail. So we have to set our
+	 * responsibility to free it in case we fail. So we have to set our
 	 * root's anon_dev to 0 to avoid a double free, once by btrfs_put_root()
 	 * and once again by our caller.
 	 */
 	return root;
 }
-/*
- * called by the kthread helper functions to finally call the bio end_io
- * functions.  This is where read checksum verification actually happens
- */
-static void end_workqueue_fn(struct btrfs_work *work)
-{
-	struct bio *bio;
-	struct btrfs_end_io_wq *end_io_wq;
-	end_io_wq = container_of(work, struct btrfs_end_io_wq, work);
-	bio = end_io_wq->bio;
-	bio->bi_status = end_io_wq->status;
-	bio->bi_private = end_io_wq->private;
-	bio->bi_end_io = end_io_wq->end_io;
-	bio_endio(bio);
-	kmem_cache_free(btrfs_end_io_wq_cache, end_io_wq);
-}
 static int cleaner_kthread(void *arg)
 {
 	struct btrfs_fs_info *fs_info = arg;
 	btrfs_destroy_workqueue(fs_info->delalloc_workers);
 	btrfs_destroy_workqueue(fs_info->hipri_workers);
 	btrfs_destroy_workqueue(fs_info->workers);
-	btrfs_destroy_workqueue(fs_info->endio_workers);
+	if (fs_info->endio_workers)
-	btrfs_destroy_workqueue(fs_info->endio_raid56_workers);
+		destroy_workqueue(fs_info->endio_workers);
+	if (fs_info->endio_raid56_workers)
+		destroy_workqueue(fs_info->endio_raid56_workers);
 	if (fs_info->rmw_workers)
 		destroy_workqueue(fs_info->rmw_workers);
+	if (fs_info->compressed_write_workers)
+		destroy_workqueue(fs_info->compressed_write_workers);
 	btrfs_destroy_workqueue(fs_info->endio_write_workers);
 	btrfs_destroy_workqueue(fs_info->endio_freespace_worker);
 	btrfs_destroy_workqueue(fs_info->delayed_workers);
 	 * the queues used for metadata I/O, since tasks from those other work
 	 * queues can do metadata I/O operations.
 	 */
-	btrfs_destroy_workqueue(fs_info->endio_meta_workers);
+	if (fs_info->endio_meta_workers)
-	btrfs_destroy_workqueue(fs_info->endio_meta_write_workers);
+		destroy_workqueue(fs_info->endio_meta_workers);
 }
 static void free_root_extent_buffers(struct btrfs_root *root)
 	extent_map_tree_init(&BTRFS_I(inode)->extent_tree);
 	BTRFS_I(inode)->root = btrfs_grab_root(fs_info->tree_root);
-	memset(&BTRFS_I(inode)->location, 0, sizeof(struct btrfs_key));
+	BTRFS_I(inode)->location.objectid = BTRFS_BTREE_INODE_OBJECTID;
+	BTRFS_I(inode)->location.type = 0;
+	BTRFS_I(inode)->location.offset = 0;
 	set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags);
 	btrfs_insert_inode_hash(inode);
 }
 	fs_info->fixup_workers =
 		btrfs_alloc_workqueue(fs_info, "fixup", flags, 1, 0);
-	/*
-	 * endios are largely parallel and should have a very
-	 * low idle thresh
-	 */
 	fs_info->endio_workers =
-		btrfs_alloc_workqueue(fs_info, "endio", flags, max_active, 4);
+		alloc_workqueue("btrfs-endio", flags, max_active);
 	fs_info->endio_meta_workers =
-		btrfs_alloc_workqueue(fs_info, "endio-meta", flags,
+		alloc_workqueue("btrfs-endio-meta", flags, max_active);
-				      max_active, 4);
-	fs_info->endio_meta_write_workers =
-		btrfs_alloc_workqueue(fs_info, "endio-meta-write", flags,
-				      max_active, 2);
 	fs_info->endio_raid56_workers =
-		btrfs_alloc_workqueue(fs_info, "endio-raid56", flags,
+		alloc_workqueue("btrfs-endio-raid56", flags, max_active);
-				      max_active, 4);
 	fs_info->rmw_workers = alloc_workqueue("btrfs-rmw", flags, max_active);
 	fs_info->endio_write_workers =
 		btrfs_alloc_workqueue(fs_info, "endio-write", flags,
 				      max_active, 2);
+	fs_info->compressed_write_workers =
+		alloc_workqueue("btrfs-compressed-write", flags, max_active);
 	fs_info->endio_freespace_worker =
 		btrfs_alloc_workqueue(fs_info, "freespace-write", flags,
 				      max_active, 0);
 	if (!(fs_info->workers && fs_info->hipri_workers &&
 	      fs_info->delalloc_workers && fs_info->flush_workers &&
 	      fs_info->endio_workers && fs_info->endio_meta_workers &&
-	      fs_info->endio_meta_write_workers &&
+	      fs_info->compressed_write_workers &&
 	      fs_info->endio_write_workers && fs_info->endio_raid56_workers &&
 	      fs_info->endio_freespace_worker && fs_info->rmw_workers &&
 	      fs_info->caching_workers && fs_info->fixup_workers &&
 	fs_info->csum_shash = csum_shash;
+	btrfs_info(fs_info, "using %s (%s) checksum algorithm",
+			btrfs_super_csum_name(csum_type),
+			crypto_shash_driver_name(csum_shash));
 	return 0;
 }
 	init_waitqueue_head(&fs_info->transaction_blocked_wait);
 	init_waitqueue_head(&fs_info->async_submit_wait);
 	init_waitqueue_head(&fs_info->delayed_iputs_wait);
+	init_waitqueue_head(&fs_info->zone_finish_wait);
 	/* Usable values until the real ones are cached from the superblock */
 	fs_info->nodesize = 4096;
 	fs_info->sectorsize_bits = ilog2(4096);
 	fs_info->stripesize = 4096;
+	fs_info->max_extent_size = BTRFS_MAX_EXTENT_SIZE;
 	spin_lock_init(&fs_info->swapfile_pins_lock);
 	fs_info->swapfile_pins = RB_ROOT;
 	 */
 	fs_info->compress_type = BTRFS_COMPRESS_ZLIB;
-	/*
-	 * Flag our filesystem as having big metadata blocks if they are bigger
-	 * than the page size.
-	 */
-	if (btrfs_super_nodesize(disk_super) > PAGE_SIZE) {
-		if (!(features & BTRFS_FEATURE_INCOMPAT_BIG_METADATA))
-			btrfs_info(fs_info,
-				"flagging fs with big metadata feature");
-		features |= BTRFS_FEATURE_INCOMPAT_BIG_METADATA;
-	}
 	/* Set up fs_info before parsing mount options */
 	nodesize = btrfs_super_nodesize(disk_super);
 	else if (fs_info->compress_type == BTRFS_COMPRESS_ZSTD)
 		features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD;
-	if (features & BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA)
+	/*
-		btrfs_info(fs_info, "has skinny extents");
+	 * Flag our filesystem as having big metadata blocks if they are bigger
+	 * than the page size.
+	 */
+	if (btrfs_super_nodesize(disk_super) > PAGE_SIZE)
+		features |= BTRFS_FEATURE_INCOMPAT_BIG_METADATA;
 	/*
 	 * mixed block groups end up with duplicate but slightly offset
 		err = -EINVAL;
 		goto fail_alloc;
 	}
+	/*
+	 * We have unsupported RO compat features, although RO mounted, we
+	 * should not cause any metadata write, including log replay.
+	 * Or we could screw up whatever the new feature requires.
+	 */
+	if (unlikely(features && btrfs_super_log_root(disk_super) &&
+		     !btrfs_test_opt(fs_info, NOLOGREPLAY))) {
+		btrfs_err(fs_info,
+"cannot replay dirty log with unsupported compat_ro features (0x%llx), try rescue=nologreplay",
+			  features);
+		err = -EINVAL;
+		goto fail_alloc;
+	}
 	if (sectorsize < PAGE_SIZE) {
 		struct btrfs_subpage_info *subpage_info;

fs/btrfs/disk-io.h+3−14

  */
 #define BTRFS_BDEV_BLOCKSIZE	(4096)
-enum btrfs_wq_endio_type {
-	BTRFS_WQ_ENDIO_DATA,
-	BTRFS_WQ_ENDIO_METADATA,
-	BTRFS_WQ_ENDIO_FREE_SPACE,
-	BTRFS_WQ_ENDIO_RAID56,
-};
 static inline u64 btrfs_sb_offset(int mirror)
 {
 	u64 start = SZ_16K;
 			  int atomic);
 int btrfs_read_extent_buffer(struct extent_buffer *buf, u64 parent_transid,
 			     int level, struct btrfs_key *first_key);
-blk_status_t btrfs_bio_wq_end_io(struct btrfs_fs_info *info, struct bio *bio,
+bool btrfs_wq_submit_bio(struct inode *inode, struct bio *bio, int mirror_num,
-			enum btrfs_wq_endio_type metadata);
+			 u64 dio_file_offset,
-blk_status_t btrfs_wq_submit_bio(struct inode *inode, struct bio *bio,
+			 extent_submit_bio_start_t *submit_bio_start);
-				 int mirror_num, u64 dio_file_offset,
-				 extent_submit_bio_start_t *submit_bio_start);
 blk_status_t btrfs_submit_bio_done(void *private_data, struct bio *bio,
 			  int mirror_num);
 int btrfs_alloc_log_tree_node(struct btrfs_trans_handle *trans,
 int btrfs_get_num_tolerated_disk_barrier_failures(u64 flags);
 int btrfs_get_free_objectid(struct btrfs_root *root, u64 *objectid);
 int btrfs_init_root_free_objectid(struct btrfs_root *root);
-int __init btrfs_end_io_wq_init(void);
-void __cold btrfs_end_io_wq_exit(void);
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 void btrfs_set_buffer_lockdep_class(u64 objectid,

fs/btrfs/extent-tree.c+87−62

 	return ret;
 }
-static int do_discard_extent(struct btrfs_io_stripe *stripe, u64 *bytes)
+static int do_discard_extent(struct btrfs_discard_stripe *stripe, u64 *bytes)
 {
 	struct btrfs_device *dev = stripe->dev;
 	struct btrfs_fs_info *fs_info = dev->fs_info;
 	u64 discarded_bytes = 0;
 	u64 end = bytenr + num_bytes;
 	u64 cur = bytenr;
-	struct btrfs_io_context *bioc = NULL;
 	/*
-	 * Avoid races with device replace and make sure our bioc has devices
+	 * Avoid races with device replace and make sure the devices in the
-	 * associated to its stripes that don't go away while we are discarding.
+	 * stripes don't go away while we are discarding.
 	 */
 	btrfs_bio_counter_inc_blocked(fs_info);
 	while (cur < end) {
-		struct btrfs_io_stripe *stripe;
+		struct btrfs_discard_stripe *stripes;
+		unsigned int num_stripes;
 		int i;
 		num_bytes = end - cur;
-		/* Tell the block device(s) that the sectors can be discarded */
+		stripes = btrfs_map_discard(fs_info, cur, &num_bytes, &num_stripes);
-		ret = btrfs_map_block(fs_info, BTRFS_MAP_DISCARD, cur,
+		if (IS_ERR(stripes)) {
-				      &num_bytes, &bioc, 0);
+			ret = PTR_ERR(stripes);
-		/*
+			if (ret == -EOPNOTSUPP)
-		 * Error can be -ENOMEM, -ENOENT (no such chunk mapping) or
+				ret = 0;
-		 * -EOPNOTSUPP. For any such error, @num_bytes is not updated,
+			break;
-		 * thus we can't continue anyway.
+		}
-		 */
-		if (ret < 0)
-			goto out;
-		stripe = bioc->stripes;
+		for (i = 0; i < num_stripes; i++) {
-		for (i = 0; i < bioc->num_stripes; i++, stripe++) {
+			struct btrfs_discard_stripe *stripe = stripes + i;
 			u64 bytes;
-			struct btrfs_device *device = stripe->dev;
-			if (!device->bdev) {
+			if (!stripe->dev->bdev) {
 				ASSERT(btrfs_test_opt(fs_info, DEGRADED));
 				continue;
 			}
-			if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state))
+			if (!test_bit(BTRFS_DEV_STATE_WRITEABLE,
+					&stripe->dev->dev_state))
 				continue;
 			ret = do_discard_extent(stripe, &bytes);
-			if (!ret) {
+			if (ret) {
-				discarded_bytes += bytes;
-			} else if (ret != -EOPNOTSUPP) {
 				/*
-				 * Logic errors or -ENOMEM, or -EIO, but
+				 * Keep going if discard is not supported by the
-				 * unlikely to happen.
+				 * device.
-				 *
-				 * And since there are two loops, explicitly
-				 * go to out to avoid confusion.
 				 */
-				btrfs_put_bioc(bioc);
+				if (ret != -EOPNOTSUPP)
-				goto out;
+					break;
+				ret = 0;
+			} else {
+				discarded_bytes += bytes;
 			}
-			/*
-			 * Just in case we get back EOPNOTSUPP for some reason,
-			 * just ignore the return value so we don't screw up
-			 * people calling discard_extent.
-			 */
-			ret = 0;
 		}
-		btrfs_put_bioc(bioc);
+		kfree(stripes);
+		if (ret)
+			break;
 		cur += num_bytes;
 	}
-out:
 	btrfs_bio_counter_dec(fs_info);
 	if (actual_bytes)
 		*actual_bytes = discarded_bytes;
-	if (ret == -EOPNOTSUPP)
-		ret = 0;
 	return ret;
 }
 	}
 }
-static bool can_allocate_chunk(struct btrfs_fs_info *fs_info,
+static int can_allocate_chunk_zoned(struct btrfs_fs_info *fs_info,
-			       struct find_free_extent_ctl *ffe_ctl)
+				    struct find_free_extent_ctl *ffe_ctl)
+{
+	/* If we can activate new zone, just allocate a chunk and use it */
+	if (btrfs_can_activate_zone(fs_info->fs_devices, ffe_ctl->flags))
+		return 0;
+	/*
+	 * We already reached the max active zones. Try to finish one block
+	 * group to make a room for a new block group. This is only possible
+	 * for a data block group because btrfs_zone_finish() may need to wait
+	 * for a running transaction which can cause a deadlock for metadata
+	 * allocation.
+	 */
+	if (ffe_ctl->flags & BTRFS_BLOCK_GROUP_DATA) {
+		int ret = btrfs_zone_finish_one_bg(fs_info);
+		if (ret == 1)
+			return 0;
+		else if (ret < 0)
+			return ret;
+	}
+	/*
+	 * If we have enough free space left in an already active block group
+	 * and we can't activate any other zone now, do not allow allocating a
+	 * new chunk and let find_free_extent() retry with a smaller size.
+	 */
+	if (ffe_ctl->max_extent_size >= ffe_ctl->min_alloc_size)
+		return -ENOSPC;
+	/*
+	 * Even min_alloc_size is not left in any block groups. Since we cannot
+	 * activate a new block group, allocating it may not help. Let's tell a
+	 * caller to try again and hope it progress something by writing some
+	 * parts of the region. That is only possible for data block groups,
+	 * where a part of the region can be written.
+	 */
+	if (ffe_ctl->flags & BTRFS_BLOCK_GROUP_DATA)
+		return -EAGAIN;
+	/*
+	 * We cannot activate a new block group and no enough space left in any
+	 * block groups. So, allocating a new block group may not help. But,
+	 * there is nothing to do anyway, so let's go with it.
+	 */
+	return 0;
+}
+static int can_allocate_chunk(struct btrfs_fs_info *fs_info,
+			      struct find_free_extent_ctl *ffe_ctl)
 {
 	switch (ffe_ctl->policy) {
 	case BTRFS_EXTENT_ALLOC_CLUSTERED:
-		return true;
+		return 0;
 	case BTRFS_EXTENT_ALLOC_ZONED:
-		/*
+		return can_allocate_chunk_zoned(fs_info, ffe_ctl);
-		 * If we have enough free space left in an already
-		 * active block group and we can't activate any other
-		 * zone now, do not allow allocating a new chunk and
-		 * let find_free_extent() retry with a smaller size.
-		 */
-		if (ffe_ctl->max_extent_size >= ffe_ctl->min_alloc_size &&
-		    !btrfs_can_activate_zone(fs_info->fs_devices, ffe_ctl->flags))
-			return false;
-		return true;
 	default:
 		BUG();
 	}
 			int exist = 0;
 			/*Check if allocation policy allows to create a new chunk */
-			if (!can_allocate_chunk(fs_info, ffe_ctl))
+			ret = can_allocate_chunk(fs_info, ffe_ctl);
-				return -ENOSPC;
+			if (ret)
+				return ret;
 			trans = current->journal_info;
 			if (trans)
  */
 static int btrfs_trim_free_extents(struct btrfs_device *device, u64 *trimmed)
 {
-	u64 start = SZ_1M, len = 0, end = 0;
+	u64 start = BTRFS_DEVICE_RANGE_RESERVED, len = 0, end = 0;
 	int ret;
 	*trimmed = 0;
 			break;
 		}
-		/* Ensure we skip the reserved area in the first 1M */
+		/* Ensure we skip the reserved space on each device. */
-		start = max_t(u64, start, SZ_1M);
+		start = max_t(u64, start, BTRFS_DEVICE_RANGE_RESERVED);
 		/*
 		 * If find_first_clear_extent_bit find a range that spans the

fs/btrfs/extent_io.c+392−481

@@ -144,6 +144,7 @@ struct tree_entry { · 143 unmodified lines
144	*/	144	*/
145	struct btrfs_bio_ctrl {	145	struct btrfs_bio_ctrl {
146	struct bio *bio;	146	struct bio *bio;
		147	int mirror_num;
147	enum btrfs_compression_type compress_type;	148	enum btrfs_compression_type compress_type;
148	u32 len_to_stripe_boundary;	149	u32 len_to_stripe_boundary;
149	u32 len_to_oe_boundary;	150	u32 len_to_oe_boundary;
@@ -178,61 +179,56 @@ static int add_extent_changeset(struct extent_state *state, u32 bits, · 28 unmodified lines
178	return ret;	179	return ret;
179	}	180	}
180		181
181	static void submit_one_bio(struct bio *bio, int mirror_num,	182	static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl)
182	enum btrfs_compression_type compress_type)
183	{	183	{
184	struct extent_io_tree *tree = bio->bi_private;	184	struct bio *bio;
		185	struct bio_vec *bv;
		186	struct inode *inode;
		187	int mirror_num;
		188
		189	if (!bio_ctrl->bio)
		190	return;
185		191
186	bio->bi_private = NULL;	192	bio = bio_ctrl->bio;
		193	bv = bio_first_bvec_all(bio);
		194	inode = bv->bv_page->mapping->host;
		195	mirror_num = bio_ctrl->mirror_num;
187		196
188	/* Caller should ensure the bio has at least some range added */	197	/* Caller should ensure the bio has at least some range added */
189	ASSERT(bio->bi_iter.bi_size);	198	ASSERT(bio->bi_iter.bi_size);
190		199
191	if (is_data_inode(tree->private_data))	200	btrfs_bio(bio)->file_offset = page_offset(bv->bv_page) + bv->bv_offset;
192	btrfs_submit_data_bio(tree->private_data, bio, mirror_num,
193	compress_type);
194	else
195	btrfs_submit_metadata_bio(tree->private_data, bio, mirror_num);
196	/*
197	* Above submission hooks will handle the error by ending the bio,
198	* which will do the cleanup properly. So here we should not return
199	* any error, or the caller of submit_extent_page() will do cleanup
200	* again, causing problems.
201	*/
202	}
203		201
204	/* Cleanup unsubmitted bios */	202	if (!is_data_inode(inode))
205	static void end_write_bio(struct extent_page_data *epd, int ret)	203	btrfs_submit_metadata_bio(inode, bio, mirror_num);
206	{	204	else if (btrfs_op(bio) == BTRFS_MAP_WRITE)
207	struct bio *bio = epd->bio_ctrl.bio;	205	btrfs_submit_data_write_bio(inode, bio, mirror_num);
		206	else
		207	btrfs_submit_data_read_bio(inode, bio, mirror_num,
		208	bio_ctrl->compress_type);
208		209
209	if (bio) {	210	/* The bio is owned by the bi_end_io handler now */
210	bio->bi_status = errno_to_blk_status(ret);	211	bio_ctrl->bio = NULL;
211	bio_endio(bio);
212	epd->bio_ctrl.bio = NULL;
213	}
214	}	212	}
215		213
216	/*	214	/*
217	* Submit bio from extent page data via submit_one_bio	215	* Submit or fail the current bio in an extent_page_data structure.
218	*
219	* Return 0 if everything is OK.
220	* Return <0 for error.
221	*/	216	*/
222	static void flush_write_bio(struct extent_page_data *epd)	217	static void submit_write_bio(struct extent_page_data *epd, int ret)
223	{	218	{
224	struct bio *bio = epd->bio_ctrl.bio;	219	struct bio *bio = epd->bio_ctrl.bio;
225		220
226	if (bio) {	221	if (!bio)
227	submit_one_bio(bio, 0, 0);	222	return;
228	/*	223
229	* Clean up of epd->bio is handled by its endio function.	224	if (ret) {
230	* And endio is either triggered by successful bio execution	225	ASSERT(ret < 0);
231	* or the error handler of submit bio hook.	226	bio->bi_status = errno_to_blk_status(ret);
232	* So at this point, no matter what happened, we don't need	227	bio_endio(bio);
233	* to clean up epd->bio.	228	/* The bio is owned by the bi_end_io handler now */
234	*/
235	epd->bio_ctrl.bio = NULL;	229	epd->bio_ctrl.bio = NULL;
		230	} else {
		231	submit_one_bio(&epd->bio_ctrl);
236	}	232	}
237	}	233	}
238		234
@@ -376,131 +372,121 @@ void free_extent_state(struct extent_state *state) · 137 unmodified lines
376	}	372	}
377	}	373	}
378		374
379	static struct rb_node tree_insert(struct rb_root root,
380	struct rb_node *search_start,
381	u64 offset,
382	struct rb_node *node,
383	struct rb_node ***p_in,
384	struct rb_node **parent_in)
385	{
386	struct rb_node **p;
387	struct rb_node *parent = NULL;
388	struct tree_entry *entry;
389
390	if (p_in && parent_in) {
391	p = *p_in;
392	parent = *parent_in;
393	goto do_insert;
394	}
395
396	p = search_start ? &search_start : &root->rb_node;
397	while (*p) {
398	parent = *p;
399	entry = rb_entry(parent, struct tree_entry, rb_node);
400
401	if (offset < entry->start)
402	p = &(*p)->rb_left;
403	else if (offset > entry->end)
404	p = &(*p)->rb_right;
405	else
406	return parent;
407	}
408
409	do_insert:
410	rb_link_node(node, parent, p);
411	rb_insert_color(node, root);
412	return NULL;
413	}
414
415	/**	375	/**
416	* Search @tree for an entry that contains @offset. Such entry would have	376	* Search @tree for an entry that contains @offset. Such entry would have
417	* entry->start <= offset && entry->end >= offset.	377	* entry->start <= offset && entry->end >= offset.
418	*	378	*
419	* @tree: the tree to search	379	* @tree: the tree to search
420	* @offset: offset that should fall within an entry in @tree	380	* @offset: offset that should fall within an entry in @tree
421	* @next_ret: pointer to the first entry whose range ends after @offset	381	* @node_ret: pointer where new node should be anchored (used when inserting an
422	* @prev_ret: pointer to the first entry whose range begins before @offset
423	* @p_ret: pointer where new node should be anchored (used when inserting an
424	* entry in the tree)	382	* entry in the tree)
425	* @parent_ret: points to entry which would have been the parent of the entry,	383	* @parent_ret: points to entry which would have been the parent of the entry,
426	* containing @offset	384	* containing @offset
427	*	385	*
428	* This function returns a pointer to the entry that contains @offset byte	386	* Return a pointer to the entry that contains @offset byte address and don't change
429	* address. If no such entry exists, then NULL is returned and the other	387	* @node_ret and @parent_ret.
430	* pointer arguments to the function are filled, otherwise the found entry is	388	*
431	* returned and other pointers are left untouched.	389	* If no such entry exists, return pointer to entry that ends before @offset
		390	* and fill parameters @node_ret and @parent_ret, ie. does not return NULL.
432	*/	391	*/
433	static struct rb_node __etree_search(struct extent_io_tree tree, u64 offset,	392	static inline struct rb_node tree_search_for_insert(struct extent_io_tree tree,
434	struct rb_node **next_ret,	393	u64 offset,
435	struct rb_node **prev_ret,	394	struct rb_node ***node_ret,
436	struct rb_node ***p_ret,	395	struct rb_node **parent_ret)
437	struct rb_node **parent_ret)
438	{	396	{
439	struct rb_root *root = &tree->state;	397	struct rb_root *root = &tree->state;
440	struct rb_node **n = &root->rb_node;	398	struct rb_node **node = &root->rb_node;
441	struct rb_node *prev = NULL;	399	struct rb_node *prev = NULL;
442	struct rb_node *orig_prev = NULL;
443	struct tree_entry *entry;	400	struct tree_entry *entry;
444	struct tree_entry *prev_entry = NULL;
445		401
446	while (*n) {	402	while (*node) {
447	prev = *n;	403	prev = *node;
448	entry = rb_entry(prev, struct tree_entry, rb_node);	404	entry = rb_entry(prev, struct tree_entry, rb_node);
449	prev_entry = entry;
450		405
451	if (offset < entry->start)	406	if (offset < entry->start)
452	n = &(*n)->rb_left;	407	node = &(*node)->rb_left;
453	else if (offset > entry->end)	408	else if (offset > entry->end)
454	n = &(*n)->rb_right;	409	node = &(*node)->rb_right;
455	else	410	else
456	return *n;	411	return *node;
457	}	412	}
458		413
459	if (p_ret)	414	if (node_ret)
460	*p_ret = n;	415	*node_ret = node;
461	if (parent_ret)	416	if (parent_ret)
462	*parent_ret = prev;	417	*parent_ret = prev;
463		418
464	if (next_ret) {	419	/* Search neighbors until we find the first one past the end */
465	orig_prev = prev;	420	while (prev && offset > entry->end) {
466	while (prev && offset > prev_entry->end) {	421	prev = rb_next(prev);
467	prev = rb_next(prev);	422	entry = rb_entry(prev, struct tree_entry, rb_node);
468	prev_entry = rb_entry(prev, struct tree_entry, rb_node);
469	}
470	*next_ret = prev;
471	prev = orig_prev;
472	}	423	}
473		424
474	if (prev_ret) {	425	return prev;
475	prev_entry = rb_entry(prev, struct tree_entry, rb_node);
476	while (prev && offset < prev_entry->start) {
477	prev = rb_prev(prev);
478	prev_entry = rb_entry(prev, struct tree_entry, rb_node);
479	}
480	*prev_ret = prev;
481	}
482	return NULL;
483	}	426	}
484		427
485	static inline struct rb_node *	428	/*
486	tree_search_for_insert(struct extent_io_tree *tree,	429	* Inexact rb-tree search, return the next entry if @offset is not found
487	u64 offset,	430	*/
488	struct rb_node ***p_ret,	431	static inline struct rb_node tree_search(struct extent_io_tree tree, u64 offset)
489	struct rb_node **parent_ret)
490	{	432	{
491	struct rb_node *next= NULL;	433	return tree_search_for_insert(tree, offset, NULL, NULL);
492	struct rb_node *ret;
493
494	ret = __etree_search(tree, offset, &next, NULL, p_ret, parent_ret);
495	if (!ret)
496	return next;
497	return ret;
498	}	434	}
499		435
500	static inline struct rb_node tree_search(struct extent_io_tree tree,	436	/**
501	u64 offset)	437	* Search offset in the tree or fill neighbor rbtree node pointers.
		438	*
		439	* @tree: the tree to search
		440	* @offset: offset that should fall within an entry in @tree
		441	* @next_ret: pointer to the first entry whose range ends after @offset
		442	* @prev_ret: pointer to the first entry whose range begins before @offset
		443	*
		444	* Return a pointer to the entry that contains @offset byte address. If no
		445	* such entry exists, then return NULL and fill @prev_ret and @next_ret.
		446	* Otherwise return the found entry and other pointers are left untouched.
		447	*/
		448	static struct rb_node tree_search_prev_next(struct extent_io_tree tree,
		449	u64 offset,
		450	struct rb_node **prev_ret,
		451	struct rb_node **next_ret)
502	{	452	{
503	return tree_search_for_insert(tree, offset, NULL, NULL);	453	struct rb_root *root = &tree->state;
		454	struct rb_node **node = &root->rb_node;
		455	struct rb_node *prev = NULL;
		456	struct rb_node *orig_prev = NULL;
		457	struct tree_entry *entry;
		458
		459	ASSERT(prev_ret);
		460	ASSERT(next_ret);
		461
		462	while (*node) {
		463	prev = *node;
		464	entry = rb_entry(prev, struct tree_entry, rb_node);
		465
		466	if (offset < entry->start)
		467	node = &(*node)->rb_left;
		468	else if (offset > entry->end)
		469	node = &(*node)->rb_right;
		470	else
		471	return *node;
		472	}
		473
		474	orig_prev = prev;
		475	while (prev && offset > entry->end) {
		476	prev = rb_next(prev);
		477	entry = rb_entry(prev, struct tree_entry, rb_node);
		478	}
		479	*next_ret = prev;
		480	prev = orig_prev;
		481
		482	entry = rb_entry(prev, struct tree_entry, rb_node);
		483	while (prev && offset < entry->start) {
		484	prev = rb_prev(prev);
		485	entry = rb_entry(prev, struct tree_entry, rb_node);
		486	}
		487	*prev_ret = prev;
		488
		489	return NULL;
504	}	490	}
505		491
506	/*	492	/*
@@ -554,7 +540,7 @@ static void merge_state(struct extent_io_tree *tree, · 47 unmodified lines
554	}	540	}
555		541
556	static void set_state_bits(struct extent_io_tree *tree,	542	static void set_state_bits(struct extent_io_tree *tree,
557	struct extent_state state, u32 bits,	543	struct extent_state *state, u32 bits,
558	struct extent_changeset *changeset);	544	struct extent_changeset *changeset);
559		545
560	/*	546	/*
@@ -568,36 +554,55 @@ static void set_state_bits(struct extent_io_tree *tree, · 7 unmodified lines
568	* probably isn't what you want to call (see set/clear_extent_bit).	554	* probably isn't what you want to call (see set/clear_extent_bit).
569	*/	555	*/
570	static int insert_state(struct extent_io_tree *tree,	556	static int insert_state(struct extent_io_tree *tree,
571	struct extent_state *state, u64 start, u64 end,	557	struct extent_state *state,
572	struct rb_node ***p,	558	u32 bits, struct extent_changeset *changeset)
573	struct rb_node **parent,
574	u32 bits, struct extent_changeset changeset)
575	{	559	{
576	struct rb_node *node;	560	struct rb_node **node;
577		561	struct rb_node *parent;
578	if (end < start) {	562	const u64 end = state->end;
579	btrfs_err(tree->fs_info,
580	"insert state: end < start %llu %llu", end, start);
581	WARN_ON(1);
582	}
583	state->start = start;
584	state->end = end;
585		563
586	set_state_bits(tree, state, bits, changeset);	564	set_state_bits(tree, state, bits, changeset);
587		565
588	node = tree_insert(&tree->state, NULL, end, &state->rb_node, p, parent);	566	node = &tree->state.rb_node;
589	if (node) {	567	while (*node) {
590	struct extent_state *found;	568	struct tree_entry *entry;
591	found = rb_entry(node, struct extent_state, rb_node);	569
592	btrfs_err(tree->fs_info,	570	parent = *node;
593	"found node %llu %llu on insert of %llu %llu",	571	entry = rb_entry(parent, struct tree_entry, rb_node);
594	found->start, found->end, start, end);	572
595	return -EEXIST;	573	if (end < entry->start) {
		574	node = &(*node)->rb_left;
		575	} else if (end > entry->end) {
		576	node = &(*node)->rb_right;
		577	} else {
		578	btrfs_err(tree->fs_info,
		579	"found node %llu %llu on insert of %llu %llu",
		580	entry->start, entry->end, state->start, end);
		581	return -EEXIST;
		582	}
596	}	583	}
		584
		585	rb_link_node(&state->rb_node, parent, node);
		586	rb_insert_color(&state->rb_node, &tree->state);
		587
597	merge_state(tree, state);	588	merge_state(tree, state);
598	return 0;	589	return 0;
599	}	590	}
600		591
		592	/*
		593	* Insert state to @tree to the location given by @node and @parent.
		594	*/
		595	static void insert_state_fast(struct extent_io_tree *tree,
		596	struct extent_state state, struct rb_node *node,
		597	struct rb_node *parent, unsigned bits,
		598	struct extent_changeset *changeset)
		599	{
		600	set_state_bits(tree, state, bits, changeset);
		601	rb_link_node(&state->rb_node, parent, node);
		602	rb_insert_color(&state->rb_node, &tree->state);
		603	merge_state(tree, state);
		604	}
		605
601	/*	606	/*
602	* split a given extent state struct in two, inserting the preallocated	607	* split a given extent state struct in two, inserting the preallocated
603	* struct 'prealloc' as the newly created second half. 'split' indicates an	608	* struct 'prealloc' as the newly created second half. 'split' indicates an
@@ -615,7 +620,8 @@ static int insert_state(struct extent_io_tree *tree, · 11 unmodified lines
615	static int split_state(struct extent_io_tree tree, struct extent_state orig,	620	static int split_state(struct extent_io_tree tree, struct extent_state orig,
616	struct extent_state *prealloc, u64 split)	621	struct extent_state *prealloc, u64 split)
617	{	622	{
618	struct rb_node *node;	623	struct rb_node *parent = NULL;
		624	struct rb_node **node;
619		625
620	if (tree->private_data && is_data_inode(tree->private_data))	626	if (tree->private_data && is_data_inode(tree->private_data))
621	btrfs_split_delalloc_extent(tree->private_data, orig, split);	627	btrfs_split_delalloc_extent(tree->private_data, orig, split);
@@ -625,12 +631,27 @@ static int split_state(struct extent_io_tree tree, struct extent_state orig, · 3 unmodified lines
625	prealloc->state = orig->state;	631	prealloc->state = orig->state;
626	orig->start = split;	632	orig->start = split;
627		633
628	node = tree_insert(&tree->state, &orig->rb_node, prealloc->end,	634	parent = &orig->rb_node;
629	&prealloc->rb_node, NULL, NULL);	635	node = &parent;
630	if (node) {	636	while (*node) {
631	free_extent_state(prealloc);	637	struct tree_entry *entry;
632	return -EEXIST;	638
		639	parent = *node;
		640	entry = rb_entry(parent, struct tree_entry, rb_node);
		641
		642	if (prealloc->end < entry->start) {
		643	node = &(*node)->rb_left;
		644	} else if (prealloc->end > entry->end) {
		645	node = &(*node)->rb_right;
		646	} else {
		647	free_extent_state(prealloc);
		648	return -EEXIST;
		649	}
633	}	650	}
		651
		652	rb_link_node(&prealloc->rb_node, parent, node);
		653	rb_insert_color(&prealloc->rb_node, &tree->state);
		654
634	return 0;	655	return 0;
635	}	656	}
636		657
@@ -652,11 +673,11 @@ static struct extent_state next_state(struct extent_state state) · 15 unmodified lines
652	*/	673	*/
653	static struct extent_state clear_state_bit(struct extent_io_tree tree,	674	static struct extent_state clear_state_bit(struct extent_io_tree tree,
654	struct extent_state *state,	675	struct extent_state *state,
655	u32 *bits, int wake,	676	u32 bits, int wake,
656	struct extent_changeset *changeset)	677	struct extent_changeset *changeset)
657	{	678	{
658	struct extent_state *next;	679	struct extent_state *next;
659	u32 bits_to_clear = *bits & ~EXTENT_CTLBITS;	680	u32 bits_to_clear = bits & ~EXTENT_CTLBITS;
660	int ret;	681	int ret;
661		682
662	if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {	683	if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
@@ -818,8 +839,7 @@ hit_next: · 155 unmodified lines
818	if (err)	839	if (err)
819	goto out;	840	goto out;
820	if (state->end <= end) {	841	if (state->end <= end) {
821	state = clear_state_bit(tree, state, &bits, wake,	842	state = clear_state_bit(tree, state, bits, wake, changeset);
822	changeset);
823	goto next;	843	goto next;
824	}	844	}
825	goto search_again;	845	goto search_again;
@@ -840,13 +860,13 @@ hit_next: · 14 unmodified lines
840	if (wake)	860	if (wake)
841	wake_up(&state->wq);	861	wake_up(&state->wq);
842		862
843	clear_state_bit(tree, prealloc, &bits, wake, changeset);	863	clear_state_bit(tree, prealloc, bits, wake, changeset);
844		864
845	prealloc = NULL;	865	prealloc = NULL;
846	goto out;	866	goto out;
847	}	867	}
848		868
849	state = clear_state_bit(tree, state, &bits, wake, changeset);	869	state = clear_state_bit(tree, state, bits, wake, changeset);
850	next:	870	next:
851	if (last_end == (u64)-1)	871	if (last_end == (u64)-1)
852	goto out;	872	goto out;
@@ -937,9 +957,9 @@ out: · 84 unmodified lines
937		957
938	static void set_state_bits(struct extent_io_tree *tree,	958	static void set_state_bits(struct extent_io_tree *tree,
939	struct extent_state *state,	959	struct extent_state *state,
940	u32 bits, struct extent_changeset changeset)	960	u32 bits, struct extent_changeset *changeset)
941	{	961	{
942	u32 bits_to_set = *bits & ~EXTENT_CTLBITS;	962	u32 bits_to_set = bits & ~EXTENT_CTLBITS;
943	int ret;	963	int ret;
944		964
945	if (tree->private_data && is_data_inode(tree->private_data))	965	if (tree->private_data && is_data_inode(tree->private_data))
@@ -1033,11 +1053,9 @@ again: · 87 unmodified lines
1033	if (!node) {	1053	if (!node) {
1034	prealloc = alloc_extent_state_atomic(prealloc);	1054	prealloc = alloc_extent_state_atomic(prealloc);
1035	BUG_ON(!prealloc);	1055	BUG_ON(!prealloc);
1036	err = insert_state(tree, prealloc, start, end,	1056	prealloc->start = start;
1037	&p, &parent, &bits, changeset);	1057	prealloc->end = end;
1038	if (err)	1058	insert_state_fast(tree, prealloc, p, parent, bits, changeset);
1039	extent_io_tree_panic(tree, err);
1040
1041	cache_state(prealloc, cached_state);	1059	cache_state(prealloc, cached_state);
1042	prealloc = NULL;	1060	prealloc = NULL;
1043	goto out;	1061	goto out;
@@ -1060,7 +1078,7 @@ hit_next: · 16 unmodified lines
1060	goto out;	1078	goto out;
1061	}	1079	}
1062		1080
1063	set_state_bits(tree, state, &bits, changeset);	1081	set_state_bits(tree, state, bits, changeset);
1064	cache_state(state, cached_state);	1082	cache_state(state, cached_state);
1065	merge_state(tree, state);	1083	merge_state(tree, state);
1066	if (last_end == (u64)-1)	1084	if (last_end == (u64)-1)
@@ -1116,7 +1134,7 @@ hit_next: · 49 unmodified lines
1116	if (err)	1134	if (err)
1117	goto out;	1135	goto out;
1118	if (state->end <= end) {	1136	if (state->end <= end) {
1119	set_state_bits(tree, state, &bits, changeset);	1137	set_state_bits(tree, state, bits, changeset);
1120	cache_state(state, cached_state);	1138	cache_state(state, cached_state);
1121	merge_state(tree, state);	1139	merge_state(tree, state);
1122	if (last_end == (u64)-1)	1140	if (last_end == (u64)-1)
@@ -1150,8 +1168,9 @@ hit_next: · 27 unmodified lines
1150	* Avoid to free 'prealloc' if it can be merged with	1168	* Avoid to free 'prealloc' if it can be merged with
1151	* the later extent.	1169	* the later extent.
1152	*/	1170	*/
1153	err = insert_state(tree, prealloc, start, this_end,	1171	prealloc->start = start;
1154	NULL, NULL, &bits, changeset);	1172	prealloc->end = this_end;
		1173	err = insert_state(tree, prealloc, bits, changeset);
1155	if (err)	1174	if (err)
1156	extent_io_tree_panic(tree, err);	1175	extent_io_tree_panic(tree, err);
1157		1176
@@ -1179,7 +1198,7 @@ hit_next: · 21 unmodified lines
1179	if (err)	1198	if (err)
1180	extent_io_tree_panic(tree, err);	1199	extent_io_tree_panic(tree, err);
1181		1200
1182	set_state_bits(tree, prealloc, &bits, changeset);	1201	set_state_bits(tree, prealloc, bits, changeset);
1183	cache_state(prealloc, cached_state);	1202	cache_state(prealloc, cached_state);
1184	merge_state(tree, prealloc);	1203	merge_state(tree, prealloc);
1185	prealloc = NULL;	1204	prealloc = NULL;
@@ -1274,10 +1293,9 @@ again: · 88 unmodified lines
1274	err = -ENOMEM;	1293	err = -ENOMEM;
1275	goto out;	1294	goto out;
1276	}	1295	}
1277	err = insert_state(tree, prealloc, start, end,	1296	prealloc->start = start;
1278	&p, &parent, &bits, NULL);	1297	prealloc->end = end;
1279	if (err)	1298	insert_state_fast(tree, prealloc, p, parent, bits, NULL);
1280	extent_io_tree_panic(tree, err);
1281	cache_state(prealloc, cached_state);	1299	cache_state(prealloc, cached_state);
1282	prealloc = NULL;	1300	prealloc = NULL;
1283	goto out;	1301	goto out;
@@ -1294,9 +1312,9 @@ hit_next: · 10 unmodified lines
1294	* Just lock what we found and keep going	1312	* Just lock what we found and keep going
1295	*/	1313	*/
1296	if (state->start == start && state->end <= end) {	1314	if (state->start == start && state->end <= end) {
1297	set_state_bits(tree, state, &bits, NULL);	1315	set_state_bits(tree, state, bits, NULL);
1298	cache_state(state, cached_state);	1316	cache_state(state, cached_state);
1299	state = clear_state_bit(tree, state, &clear_bits, 0, NULL);	1317	state = clear_state_bit(tree, state, clear_bits, 0, NULL);
1300	if (last_end == (u64)-1)	1318	if (last_end == (u64)-1)
1301	goto out;	1319	goto out;
1302	start = last_end + 1;	1320	start = last_end + 1;
@@ -1335,10 +1353,9 @@ hit_next: · 32 unmodified lines
1335	if (err)	1353	if (err)
1336	goto out;	1354	goto out;
1337	if (state->end <= end) {	1355	if (state->end <= end) {
1338	set_state_bits(tree, state, &bits, NULL);	1356	set_state_bits(tree, state, bits, NULL);
1339	cache_state(state, cached_state);	1357	cache_state(state, cached_state);
1340	state = clear_state_bit(tree, state, &clear_bits, 0,	1358	state = clear_state_bit(tree, state, clear_bits, 0, NULL);
1341	NULL);
1342	if (last_end == (u64)-1)	1359	if (last_end == (u64)-1)
1343	goto out;	1360	goto out;
1344	start = last_end + 1;	1361	start = last_end + 1;
@@ -1372,8 +1389,9 @@ hit_next: · 27 unmodified lines
1372	* Avoid to free 'prealloc' if it can be merged with	1389	* Avoid to free 'prealloc' if it can be merged with
1373	* the later extent.	1390	* the later extent.
1374	*/	1391	*/
1375	err = insert_state(tree, prealloc, start, this_end,	1392	prealloc->start = start;
1376	NULL, NULL, &bits, NULL);	1393	prealloc->end = this_end;
		1394	err = insert_state(tree, prealloc, bits, NULL);
1377	if (err)	1395	if (err)
1378	extent_io_tree_panic(tree, err);	1396	extent_io_tree_panic(tree, err);
1379	cache_state(prealloc, cached_state);	1397	cache_state(prealloc, cached_state);
@@ -1398,9 +1416,9 @@ hit_next: · 18 unmodified lines
1398	if (err)	1416	if (err)
1399	extent_io_tree_panic(tree, err);	1417	extent_io_tree_panic(tree, err);
1400		1418
1401	set_state_bits(tree, prealloc, &bits, NULL);	1419	set_state_bits(tree, prealloc, bits, NULL);
1402	cache_state(prealloc, cached_state);	1420	cache_state(prealloc, cached_state);
1403	clear_state_bit(tree, prealloc, &clear_bits, 0, NULL);	1421	clear_state_bit(tree, prealloc, clear_bits, 0, NULL);
1404	prealloc = NULL;	1422	prealloc = NULL;
1405	goto out;	1423	goto out;
1406	}	1424	}
@@ -1674,7 +1692,7 @@ void find_first_clear_extent_bit(struct extent_io_tree *tree, u64 start, · 267 unmodified lines
1674		1692
1675	/* Find first extent with bits cleared */	1693	/* Find first extent with bits cleared */
1676	while (1) {	1694	while (1) {
1677	node = __etree_search(tree, start, &next, &prev, NULL, NULL);	1695	node = tree_search_prev_next(tree, start, &prev, &next);
1678	if (!node && !next && !prev) {	1696	if (!node && !next && !prev) {
1679	/*	1697	/*
1680	* Tree is completely empty, send full range and let	1698	* Tree is completely empty, send full range and let
@@ -2007,10 +2025,12 @@ noinline_for_stack bool find_lock_delalloc_range(struct inode *inode, · 326 unmodified lines
2007	struct page locked_page, u64 start,	2025	struct page locked_page, u64 start,
2008	u64 *end)	2026	u64 *end)
2009	{	2027	{
		2028	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2010	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;	2029	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2011	const u64 orig_start = *start;	2030	const u64 orig_start = *start;
2012	const u64 orig_end = *end;	2031	const u64 orig_end = *end;
2013	u64 max_bytes = BTRFS_MAX_EXTENT_SIZE;	2032	/* The sanity tests may not set a valid fs_info. */
		2033	u64 max_bytes = fs_info ? fs_info->max_extent_size : BTRFS_MAX_EXTENT_SIZE;
2014	u64 delalloc_start;	2034	u64 delalloc_start;
2015	u64 delalloc_end;	2035	u64 delalloc_end;
2016	bool found;	2036	bool found;
@@ -2418,6 +2438,20 @@ int btrfs_repair_eb_io_failure(const struct extent_buffer *eb, int mirror_num) · 401 unmodified lines
2418	return ret;	2438	return ret;
2419	}	2439	}
2420		2440
		2441	static int next_mirror(const struct io_failure_record *failrec, int cur_mirror)
		2442	{
		2443	if (cur_mirror == failrec->num_copies)
		2444	return cur_mirror + 1 - failrec->num_copies;
		2445	return cur_mirror + 1;
		2446	}
		2447
		2448	static int prev_mirror(const struct io_failure_record *failrec, int cur_mirror)
		2449	{
		2450	if (cur_mirror == 1)
		2451	return failrec->num_copies;
		2452	return cur_mirror - 1;
		2453	}
		2454
2421	/*	2455	/*
2422	* each time an IO finishes, we do a fast check in the IO failure tree	2456	* each time an IO finishes, we do a fast check in the IO failure tree
2423	* to see if we need to process or clean up an io_failure_record	2457	* to see if we need to process or clean up an io_failure_record
@@ -2430,7 +2464,7 @@ int clean_io_failure(struct btrfs_fs_info *fs_info, · 6 unmodified lines
2430	u64 private;	2464	u64 private;
2431	struct io_failure_record *failrec;	2465	struct io_failure_record *failrec;
2432	struct extent_state *state;	2466	struct extent_state *state;
2433	int num_copies;	2467	int mirror;
2434	int ret;	2468	int ret;
2435		2469
2436	private = 0;	2470	private = 0;
@@ -2454,20 +2488,19 @@ int clean_io_failure(struct btrfs_fs_info *fs_info, · 17 unmodified lines
2454	EXTENT_LOCKED);	2488	EXTENT_LOCKED);
2455	spin_unlock(&io_tree->lock);	2489	spin_unlock(&io_tree->lock);
2456		2490
2457	if (state && state->start <= failrec->start &&	2491	if (!state \|\| state->start > failrec->start \|\|
2458	state->end >= failrec->start + failrec->len - 1) {	2492	state->end < failrec->start + failrec->len - 1)
2459	num_copies = btrfs_num_copies(fs_info, failrec->logical,	2493	goto out;
2460	failrec->len);	2494
2461	if (num_copies > 1) {	2495	mirror = failrec->this_mirror;
2462	repair_io_failure(fs_info, ino, start, failrec->len,	2496	do {
2463	failrec->logical, page, pg_offset,	2497	mirror = prev_mirror(failrec, mirror);
2464	failrec->failed_mirror);	2498	repair_io_failure(fs_info, ino, start, failrec->len,
2465	}	2499	failrec->logical, page, pg_offset, mirror);
2466	}	2500	} while (mirror != failrec->failed_mirror);
2467		2501
2468	out:	2502	out:
2469	free_io_failure(failure_tree, io_tree, failrec);	2503	free_io_failure(failure_tree, io_tree, failrec);
2470
2471	return 0;	2504	return 0;
2472	}	2505	}
2473		2506
@@ -2506,17 +2539,16 @@ void btrfs_free_io_failure_record(struct btrfs_inode *inode, u64 start, u64 end) · 32 unmodified lines
2506	}	2539	}
2507		2540
2508	static struct io_failure_record btrfs_get_io_failure_record(struct inode inode,	2541	static struct io_failure_record btrfs_get_io_failure_record(struct inode inode,
2509	u64 start)	2542	struct btrfs_bio *bbio,
		2543	unsigned int bio_offset)
2510	{	2544	{
2511	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);	2545	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
		2546	u64 start = bbio->file_offset + bio_offset;
2512	struct io_failure_record *failrec;	2547	struct io_failure_record *failrec;
2513	struct extent_map *em;
2514	struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;	2548	struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2515	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;	2549	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2516	struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
2517	const u32 sectorsize = fs_info->sectorsize;	2550	const u32 sectorsize = fs_info->sectorsize;
2518	int ret;	2551	int ret;
2519	u64 logical;
2520		2552
2521	failrec = get_state_failrec(failure_tree, start);	2553	failrec = get_state_failrec(failure_tree, start);
2522	if (!IS_ERR(failrec)) {	2554	if (!IS_ERR(failrec)) {
@@ -2528,7 +2560,8 @@ static struct io_failure_record btrfs_get_io_failure_record(struct inode inode · 5 unmodified lines
2528	* (e.g. with a list for failed_mirror) to make	2560	* (e.g. with a list for failed_mirror) to make
2529	* clean_io_failure() clean all those errors at once.	2561	* clean_io_failure() clean all those errors at once.
2530	*/	2562	*/
2531		2563	ASSERT(failrec->this_mirror == bbio->mirror_num);
		2564	ASSERT(failrec->len == fs_info->sectorsize);
2532	return failrec;	2565	return failrec;
2533	}	2566	}
2534		2567
@@ -2538,41 +2571,28 @@ static struct io_failure_record btrfs_get_io_failure_record(struct inode inode · 3 unmodified lines
2538		2571
2539	failrec->start = start;	2572	failrec->start = start;
2540	failrec->len = sectorsize;	2573	failrec->len = sectorsize;
2541	failrec->this_mirror = 0;	2574	failrec->failed_mirror = bbio->mirror_num;
2542	failrec->compress_type = BTRFS_COMPRESS_NONE;	2575	failrec->this_mirror = bbio->mirror_num;
		2576	failrec->logical = (bbio->iter.bi_sector << SECTOR_SHIFT) + bio_offset;
2543		2577
2544	read_lock(&em_tree->lock);	2578	btrfs_debug(fs_info,
2545	em = lookup_extent_mapping(em_tree, start, failrec->len);	2579	"new io failure record logical %llu start %llu",
2546	if (!em) {	2580	failrec->logical, start);
2547	read_unlock(&em_tree->lock);
2548	kfree(failrec);
2549	return ERR_PTR(-EIO);
2550	}
2551		2581
2552	if (em->start > start \|\| em->start + em->len <= start) {	2582	failrec->num_copies = btrfs_num_copies(fs_info, failrec->logical, sectorsize);
2553	free_extent_map(em);	2583	if (failrec->num_copies == 1) {
2554	em = NULL;	2584	/*
2555	}	2585	* We only have a single copy of the data, so don't bother with
2556	read_unlock(&em_tree->lock);	2586	* all the retry and error correction code that follows. No
2557	if (!em) {	2587	* matter what the error is, it is very likely to persist.
		2588	*/
		2589	btrfs_debug(fs_info,
		2590	"cannot repair logical %llu num_copies %d",
		2591	failrec->logical, failrec->num_copies);
2558	kfree(failrec);	2592	kfree(failrec);
2559	return ERR_PTR(-EIO);	2593	return ERR_PTR(-EIO);
2560	}	2594	}
2561		2595
2562	logical = start - em->start;
2563	logical = em->block_start + logical;
2564	if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2565	logical = em->block_start;
2566	failrec->compress_type = em->compress_type;
2567	}
2568
2569	btrfs_debug(fs_info,
2570	"Get IO Failure Record: (new) logical=%llu, start=%llu, len=%llu",
2571	logical, start, failrec->len);
2572
2573	failrec->logical = logical;
2574	free_extent_map(em);
2575
2576	/* Set the bits in the private failure tree */	2596	/* Set the bits in the private failure tree */
2577	ret = set_extent_bits(failure_tree, start, start + sectorsize - 1,	2597	ret = set_extent_bits(failure_tree, start, start + sectorsize - 1,
2578	EXTENT_LOCKED \| EXTENT_DIRTY);	2598	EXTENT_LOCKED \| EXTENT_DIRTY);
@@ -2589,65 +2609,16 @@ static struct io_failure_record btrfs_get_io_failure_record(struct inode inode · 10 unmodified lines
2589	return failrec;	2609	return failrec;
2590	}	2610	}
2591		2611
2592	static bool btrfs_check_repairable(struct inode *inode,	2612	int btrfs_repair_one_sector(struct inode inode, struct btrfs_bio failed_bbio,
2593	struct io_failure_record *failrec,	2613	u32 bio_offset, struct page *page, unsigned int pgoff,
2594	int failed_mirror)
2595	{
2596	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2597	int num_copies;
2598
2599	num_copies = btrfs_num_copies(fs_info, failrec->logical, failrec->len);
2600	if (num_copies == 1) {
2601	/*
2602	* we only have a single copy of the data, so don't bother with
2603	* all the retry and error correction code that follows. no
2604	* matter what the error is, it is very likely to persist.
2605	*/
2606	btrfs_debug(fs_info,
2607	"Check Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d",
2608	num_copies, failrec->this_mirror, failed_mirror);
2609	return false;
2610	}
2611
2612	/* The failure record should only contain one sector */
2613	ASSERT(failrec->len == fs_info->sectorsize);
2614
2615	/*
2616	* There are two premises:
2617	* a) deliver good data to the caller
2618	* b) correct the bad sectors on disk
2619	*
2620	* Since we're only doing repair for one sector, we only need to get
2621	* a good copy of the failed sector and if we succeed, we have setup
2622	* everything for repair_io_failure to do the rest for us.
2623	*/
2624	ASSERT(failed_mirror);
2625	failrec->failed_mirror = failed_mirror;
2626	failrec->this_mirror++;
2627	if (failrec->this_mirror == failed_mirror)
2628	failrec->this_mirror++;
2629
2630	if (failrec->this_mirror > num_copies) {
2631	btrfs_debug(fs_info,
2632	"Check Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d",
2633	num_copies, failrec->this_mirror, failed_mirror);
2634	return false;
2635	}
2636
2637	return true;
2638	}
2639
2640	int btrfs_repair_one_sector(struct inode *inode,
2641	struct bio *failed_bio, u32 bio_offset,
2642	struct page *page, unsigned int pgoff,
2643	u64 start, int failed_mirror,
2644	submit_bio_hook_t *submit_bio_hook)	2614	submit_bio_hook_t *submit_bio_hook)
2645	{	2615	{
		2616	u64 start = failed_bbio->file_offset + bio_offset;
2646	struct io_failure_record *failrec;	2617	struct io_failure_record *failrec;
2647	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);	2618	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2648	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;	2619	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2649	struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;	2620	struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2650	struct btrfs_bio *failed_bbio = btrfs_bio(failed_bio);	2621	struct bio *failed_bio = &failed_bbio->bio;
2651	const int icsum = bio_offset >> fs_info->sectorsize_bits;	2622	const int icsum = bio_offset >> fs_info->sectorsize_bits;
2652	struct bio *repair_bio;	2623	struct bio *repair_bio;
2653	struct btrfs_bio *repair_bbio;	2624	struct btrfs_bio *repair_bbio;
@@ -2657,12 +2628,24 @@ int btrfs_repair_one_sector(struct inode *inode, · 3 unmodified lines
2657		2628
2658	BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);	2629	BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);
2659		2630
2660	failrec = btrfs_get_io_failure_record(inode, start);	2631	failrec = btrfs_get_io_failure_record(inode, failed_bbio, bio_offset);
2661	if (IS_ERR(failrec))	2632	if (IS_ERR(failrec))
2662	return PTR_ERR(failrec);	2633	return PTR_ERR(failrec);
2663		2634
2664		2635	/*
2665	if (!btrfs_check_repairable(inode, failrec, failed_mirror)) {	2636	* There are two premises:
		2637	* a) deliver good data to the caller
		2638	* b) correct the bad sectors on disk
		2639	*
		2640	* Since we're only doing repair for one sector, we only need to get
		2641	* a good copy of the failed sector and if we succeed, we have setup
		2642	* everything for repair_io_failure to do the rest for us.
		2643	*/
		2644	failrec->this_mirror = next_mirror(failrec, failrec->this_mirror);
		2645	if (failrec->this_mirror == failrec->failed_mirror) {
		2646	btrfs_debug(fs_info,
		2647	"failed to repair num_copies %d this_mirror %d failed_mirror %d",
		2648	failrec->num_copies, failrec->this_mirror, failrec->failed_mirror);
2666	free_io_failure(failure_tree, tree, failrec);	2649	free_io_failure(failure_tree, tree, failrec);
2667	return -EIO;	2650	return -EIO;
2668	}	2651	}
@@ -2695,7 +2678,7 @@ int btrfs_repair_one_sector(struct inode *inode, · 26 unmodified lines
2695	* will be handled by the endio on the repair_bio, so we can't return an	2678	* will be handled by the endio on the repair_bio, so we can't return an
2696	* error here.	2679	* error here.
2697	*/	2680	*/
2698	submit_bio_hook(inode, repair_bio, failrec->this_mirror, failrec->compress_type);	2681	submit_bio_hook(inode, repair_bio, failrec->this_mirror, 0);
2699	return BLK_STS_OK;	2682	return BLK_STS_OK;
2700	}	2683	}
2701		2684
@@ -2727,21 +2710,35 @@ static void end_page_read(struct page *page, bool uptodate, u64 start, u32 len) · 25 unmodified lines
2727	btrfs_subpage_end_reader(fs_info, page, start, len);	2710	btrfs_subpage_end_reader(fs_info, page, start, len);
2728	}	2711	}
2729		2712
2730	static blk_status_t submit_data_read_repair(struct inode *inode,	2713	static void end_sector_io(struct page *page, u64 offset, bool uptodate)
2731	struct bio *failed_bio,	2714	{
2732	u32 bio_offset, struct page *page,	2715	struct btrfs_inode *inode = BTRFS_I(page->mapping->host);
2733	unsigned int pgoff,	2716	const u32 sectorsize = inode->root->fs_info->sectorsize;
2734	u64 start, u64 end,	2717	struct extent_state *cached = NULL;
2735	int failed_mirror,	2718
2736	unsigned int error_bitmap)	2719	end_page_read(page, uptodate, offset, sectorsize);
		2720	if (uptodate)
		2721	set_extent_uptodate(&inode->io_tree, offset,
		2722	offset + sectorsize - 1, &cached, GFP_ATOMIC);
		2723	unlock_extent_cached_atomic(&inode->io_tree, offset,
		2724	offset + sectorsize - 1, &cached);
		2725	}
		2726
		2727	static void submit_data_read_repair(struct inode *inode,
		2728	struct btrfs_bio *failed_bbio,
		2729	u32 bio_offset, const struct bio_vec *bvec,
		2730	unsigned int error_bitmap)
2737	{	2731	{
		2732	const unsigned int pgoff = bvec->bv_offset;
2738	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);	2733	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
		2734	struct page *page = bvec->bv_page;
		2735	const u64 start = page_offset(bvec->bv_page) + bvec->bv_offset;
		2736	const u64 end = start + bvec->bv_len - 1;
2739	const u32 sectorsize = fs_info->sectorsize;	2737	const u32 sectorsize = fs_info->sectorsize;
2740	const int nr_bits = (end + 1 - start) >> fs_info->sectorsize_bits;	2738	const int nr_bits = (end + 1 - start) >> fs_info->sectorsize_bits;
2741	int error = 0;
2742	int i;	2739	int i;
2743		2740
2744	BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);	2741	BUG_ON(bio_op(&failed_bbio->bio) == REQ_OP_WRITE);
2745		2742
2746	/* This repair is only for data */	2743	/* This repair is only for data */
2747	ASSERT(is_data_inode(inode));	2744	ASSERT(is_data_inode(inode));
@@ -2753,12 +2750,11 @@ static blk_status_t submit_data_read_repair(struct inode *inode, · 5 unmodified lines
2753	* We only get called on buffered IO, thus page must be mapped and bio	2750	* We only get called on buffered IO, thus page must be mapped and bio
2754	* must not be cloned.	2751	* must not be cloned.
2755	*/	2752	*/
2756	ASSERT(page->mapping && !bio_flagged(failed_bio, BIO_CLONED));	2753	ASSERT(page->mapping && !bio_flagged(&failed_bbio->bio, BIO_CLONED));
2757		2754
2758	/* Iterate through all the sectors in the range */	2755	/* Iterate through all the sectors in the range */
2759	for (i = 0; i < nr_bits; i++) {	2756	for (i = 0; i < nr_bits; i++) {
2760	const unsigned int offset = i * sectorsize;	2757	const unsigned int offset = i * sectorsize;
2761	struct extent_state *cached = NULL;
2762	bool uptodate = false;	2758	bool uptodate = false;
2763	int ret;	2759	int ret;
2764		2760
@@ -2771,10 +2767,9 @@ static blk_status_t submit_data_read_repair(struct inode *inode, · 6 unmodified lines
2771	goto next;	2767	goto next;
2772	}	2768	}
2773		2769
2774	ret = btrfs_repair_one_sector(inode, failed_bio,	2770	ret = btrfs_repair_one_sector(inode, failed_bbio,
2775	bio_offset + offset,	2771	bio_offset + offset, page, pgoff + offset,
2776	page, pgoff + offset, start + offset,	2772	btrfs_submit_data_read_bio);
2777	failed_mirror, btrfs_submit_data_bio);
2778	if (!ret) {	2773	if (!ret) {
2779	/*	2774	/*
2780	* We have submitted the read repair, the page release	2775	* We have submitted the read repair, the page release
@@ -2785,24 +2780,12 @@ static blk_status_t submit_data_read_repair(struct inode *inode, · 4 unmodified lines
2785	continue;	2780	continue;
2786	}	2781	}
2787	/*	2782	/*
2788	* Repair failed, just record the error but still continue.	2783	* Continue on failed repair, otherwise the remaining sectors
2789	* Or the remaining sectors will not be properly unlocked.	2784	* will not be properly unlocked.
2790	*/	2785	*/
2791	if (!error)
2792	error = ret;
2793	next:	2786	next:
2794	end_page_read(page, uptodate, start + offset, sectorsize);	2787	end_sector_io(page, start + offset, uptodate);
2795	if (uptodate)
2796	set_extent_uptodate(&BTRFS_I(inode)->io_tree,
2797	start + offset,
2798	start + offset + sectorsize - 1,
2799	&cached, GFP_ATOMIC);
2800	unlock_extent_cached_atomic(&BTRFS_I(inode)->io_tree,
2801	start + offset,
2802	start + offset + sectorsize - 1,
2803	&cached);
2804	}	2788	}
2805	return errno_to_blk_status(error);
2806	}	2789	}
2807		2790
2808	/* lots and lots of room for performance fixes in the end_bio funcs */	2791	/* lots and lots of room for performance fixes in the end_bio funcs */
@@ -3017,7 +3000,6 @@ static void end_bio_extent_readpage(struct bio *bio) · 208 unmodified lines
3017	*/	3000	*/
3018	u32 bio_offset = 0;	3001	u32 bio_offset = 0;
3019	int mirror;	3002	int mirror;
3020	int ret;
3021	struct bvec_iter_all iter_all;	3003	struct bvec_iter_all iter_all;
3022		3004
3023	ASSERT(!bio_flagged(bio, BIO_CLONED));	3005	ASSERT(!bio_flagged(bio, BIO_CLONED));
@@ -3028,6 +3010,7 @@ static void end_bio_extent_readpage(struct bio *bio) · 4 unmodified lines
3028	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);	3010	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3029	const u32 sectorsize = fs_info->sectorsize;	3011	const u32 sectorsize = fs_info->sectorsize;
3030	unsigned int error_bitmap = (unsigned int)-1;	3012	unsigned int error_bitmap = (unsigned int)-1;
		3013	bool repair = false;
3031	u64 start;	3014	u64 start;
3032	u64 end;	3015	u64 end;
3033	u32 len;	3016	u32 len;
@@ -3065,57 +3048,23 @@ static void end_bio_extent_readpage(struct bio *bio) · 31 unmodified lines
3065	if (is_data_inode(inode)) {	3048	if (is_data_inode(inode)) {
3066	error_bitmap = btrfs_verify_data_csum(bbio,	3049	error_bitmap = btrfs_verify_data_csum(bbio,
3067	bio_offset, page, start, end);	3050	bio_offset, page, start, end);
3068	ret = error_bitmap;	3051	if (error_bitmap)
		3052	uptodate = false;
3069	} else {	3053	} else {
3070	ret = btrfs_validate_metadata_buffer(bbio,	3054	if (btrfs_validate_metadata_buffer(bbio,
3071	page, start, end, mirror);	3055	page, start, end, mirror))
		3056	uptodate = false;
3072	}	3057	}
3073	if (ret)
3074	uptodate = false;
3075	else
3076	clean_io_failure(BTRFS_I(inode)->root->fs_info,
3077	failure_tree, tree, start,
3078	page,
3079	btrfs_ino(BTRFS_I(inode)), 0);
3080	}	3058	}
3081		3059
3082	if (likely(uptodate))
3083	goto readpage_ok;
3084
3085	if (is_data_inode(inode)) {
3086	/*
3087	* If we failed to submit the IO at all we'll have a
3088	* mirror_num == 0, in which case we need to just mark
3089	* the page with an error and unlock it and carry on.
3090	*/
3091	if (mirror == 0)
3092	goto readpage_ok;
3093
3094	/*
3095	* submit_data_read_repair() will handle all the good
3096	* and bad sectors, we just continue to the next bvec.
3097	*/
3098	submit_data_read_repair(inode, bio, bio_offset, page,
3099	start - page_offset(page),
3100	start, end, mirror,
3101	error_bitmap);
3102
3103	ASSERT(bio_offset + len > bio_offset);
3104	bio_offset += len;
3105	continue;
3106	} else {
3107	struct extent_buffer *eb;
3108
3109	eb = find_extent_buffer_readpage(fs_info, page, start);
3110	set_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
3111	eb->read_mirror = mirror;
3112	atomic_dec(&eb->io_pages);
3113	}
3114	readpage_ok:
3115	if (likely(uptodate)) {	3060	if (likely(uptodate)) {
3116	loff_t i_size = i_size_read(inode);	3061	loff_t i_size = i_size_read(inode);
3117	pgoff_t end_index = i_size >> PAGE_SHIFT;	3062	pgoff_t end_index = i_size >> PAGE_SHIFT;
3118		3063
		3064	clean_io_failure(BTRFS_I(inode)->root->fs_info,
		3065	failure_tree, tree, start, page,
		3066	btrfs_ino(BTRFS_I(inode)), 0);
		3067
3119	/*	3068	/*
3120	* Zero out the remaining part if this range straddles	3069	* Zero out the remaining part if this range straddles
3121	* i_size.	3070	* i_size.
@@ -3132,14 +3081,44 @@ readpage_ok: · 10 unmodified lines
3132	zero_user_segment(page, zero_start,	3081	zero_user_segment(page, zero_start,
3133	offset_in_page(end) + 1);	3082	offset_in_page(end) + 1);
3134	}	3083	}
		3084	} else if (is_data_inode(inode)) {
		3085	/*
		3086	* Only try to repair bios that actually made it to a
		3087	* device. If the bio failed to be submitted mirror
		3088	* is 0 and we need to fail it without retrying.
		3089	*
		3090	* This also includes the high level bios for compressed
		3091	* extents - these never make it to a device and repair
		3092	* is already handled on the lower compressed bio.
		3093	*/
		3094	if (mirror > 0)
		3095	repair = true;
		3096	} else {
		3097	struct extent_buffer *eb;
		3098
		3099	eb = find_extent_buffer_readpage(fs_info, page, start);
		3100	set_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
		3101	eb->read_mirror = mirror;
		3102	atomic_dec(&eb->io_pages);
		3103	}
		3104
		3105	if (repair) {
		3106	/*
		3107	* submit_data_read_repair() will handle all the good
		3108	* and bad sectors, we just continue to the next bvec.
		3109	*/
		3110	submit_data_read_repair(inode, bbio, bio_offset, bvec,
		3111	error_bitmap);
		3112	} else {
		3113	/* Update page status and unlock */
		3114	end_page_read(page, uptodate, start, len);
		3115	endio_readpage_release_extent(&processed, BTRFS_I(inode),
		3116	start, end, PageUptodate(page));
3135	}	3117	}
		3118
3136	ASSERT(bio_offset + len > bio_offset);	3119	ASSERT(bio_offset + len > bio_offset);
3137	bio_offset += len;	3120	bio_offset += len;
3138		3121
3139	/* Update page status and unlock */
3140	end_page_read(page, uptodate, start, len);
3141	endio_readpage_release_extent(&processed, BTRFS_I(inode),
3142	start, end, PageUptodate(page));
3143	}	3122	}
3144	/* Release the last extent */	3123	/* Release the last extent */
3145	endio_readpage_release_extent(&processed, NULL, 0, 0, false);	3124	endio_readpage_release_extent(&processed, NULL, 0, 0, false);
@@ -3208,19 +3187,6 @@ struct bio *btrfs_bio_alloc(unsigned int nr_iovecs) · 62 unmodified lines
3208	return bio;	3187	return bio;
3209	}	3188	}
3210		3189
3211	struct bio btrfs_bio_clone(struct block_device bdev, struct bio *bio)
3212	{
3213	struct btrfs_bio *bbio;
3214	struct bio *new;
3215
3216	/* Bio allocation backed by a bioset does not fail */
3217	new = bio_alloc_clone(bdev, bio, GFP_NOFS, &btrfs_bioset);
3218	bbio = btrfs_bio(new);
3219	btrfs_bio_init(bbio);
3220	bbio->iter = bio->bi_iter;
3221	return new;
3222	}
3223
3224	struct bio btrfs_bio_clone_partial(struct bio orig, u64 offset, u64 size)	3190	struct bio btrfs_bio_clone_partial(struct bio orig, u64 offset, u64 size)
3225	{	3191	{
3226	struct bio *bio;	3192	struct bio *bio;
@@ -3380,7 +3346,6 @@ static int alloc_new_bio(struct btrfs_inode *inode, · 153 unmodified lines
3380	bio_ctrl->bio = bio;	3346	bio_ctrl->bio = bio;
3381	bio_ctrl->compress_type = compress_type;	3347	bio_ctrl->compress_type = compress_type;
3382	bio->bi_end_io = end_io_func;	3348	bio->bi_end_io = end_io_func;
3383	bio->bi_private = &inode->io_tree;
3384	bio->bi_opf = opf;	3349	bio->bi_opf = opf;
3385	ret = calc_bio_boundaries(bio_ctrl, inode, file_offset);	3350	ret = calc_bio_boundaries(bio_ctrl, inode, file_offset);
3386	if (ret < 0)	3351	if (ret < 0)
@@ -3445,7 +3410,6 @@ static int submit_extent_page(blk_opf_t opf, · 58 unmodified lines
3445	struct page *page, u64 disk_bytenr,	3410	struct page *page, u64 disk_bytenr,
3446	size_t size, unsigned long pg_offset,	3411	size_t size, unsigned long pg_offset,
3447	bio_end_io_t end_io_func,	3412	bio_end_io_t end_io_func,
3448	int mirror_num,
3449	enum btrfs_compression_type compress_type,	3413	enum btrfs_compression_type compress_type,
3450	bool force_bio_submit)	3414	bool force_bio_submit)
3451	{	3415	{
@@ -3457,10 +3421,8 @@ static int submit_extent_page(blk_opf_t opf, · 5 unmodified lines
3457		3421
3458	ASSERT(pg_offset < PAGE_SIZE && size <= PAGE_SIZE &&	3422	ASSERT(pg_offset < PAGE_SIZE && size <= PAGE_SIZE &&
3459	pg_offset + size <= PAGE_SIZE);	3423	pg_offset + size <= PAGE_SIZE);
3460	if (force_bio_submit && bio_ctrl->bio) {	3424	if (force_bio_submit)
3461	submit_one_bio(bio_ctrl->bio, mirror_num, bio_ctrl->compress_type);	3425	submit_one_bio(bio_ctrl);
3462	bio_ctrl->bio = NULL;
3463	}
3464		3426
3465	while (cur < pg_offset + size) {	3427	while (cur < pg_offset + size) {
3466	u32 offset = cur - pg_offset;	3428	u32 offset = cur - pg_offset;
@@ -3500,8 +3462,7 @@ static int submit_extent_page(blk_opf_t opf, · 33 unmodified lines
3500	if (added < size - offset) {	3462	if (added < size - offset) {
3501	/* The bio should contain some page(s) */	3463	/* The bio should contain some page(s) */
3502	ASSERT(bio_ctrl->bio->bi_iter.bi_size);	3464	ASSERT(bio_ctrl->bio->bi_iter.bi_size);
3503	submit_one_bio(bio_ctrl->bio, mirror_num, bio_ctrl->compress_type);	3465	submit_one_bio(bio_ctrl);
3504	bio_ctrl->bio = NULL;
3505	}	3466	}
3506	cur += added;	3467	cur += added;
3507	}	3468	}
@@ -3649,7 +3610,6 @@ static int btrfs_do_readpage(struct page page, struct extent_map *em_cached, · 141 unmodified lines
3649	if (zero_offset) {	3610	if (zero_offset) {
3650	iosize = PAGE_SIZE - zero_offset;	3611	iosize = PAGE_SIZE - zero_offset;
3651	memzero_page(page, zero_offset, iosize);	3612	memzero_page(page, zero_offset, iosize);
3652	flush_dcache_page(page);
3653	}	3613	}
3654	}	3614	}
3655	begin_page_read(fs_info, page);	3615	begin_page_read(fs_info, page);
@@ -3664,7 +3624,6 @@ static int btrfs_do_readpage(struct page page, struct extent_map *em_cached, · 8 unmodified lines
3664		3624
3665	iosize = PAGE_SIZE - pg_offset;	3625	iosize = PAGE_SIZE - pg_offset;
3666	memzero_page(page, pg_offset, iosize);	3626	memzero_page(page, pg_offset, iosize);
3667	flush_dcache_page(page);
3668	set_extent_uptodate(tree, cur, cur + iosize - 1,	3627	set_extent_uptodate(tree, cur, cur + iosize - 1,
3669	&cached, GFP_NOFS);	3628	&cached, GFP_NOFS);
3670	unlock_extent_cached(tree, cur,	3629	unlock_extent_cached(tree, cur,
@@ -3748,7 +3707,6 @@ static int btrfs_do_readpage(struct page page, struct extent_map *em_cached, · 77 unmodified lines
3748	struct extent_state *cached = NULL;	3707	struct extent_state *cached = NULL;
3749		3708
3750	memzero_page(page, pg_offset, iosize);	3709	memzero_page(page, pg_offset, iosize);
3751	flush_dcache_page(page);
3752		3710
3753	set_extent_uptodate(tree, cur, cur + iosize - 1,	3711	set_extent_uptodate(tree, cur, cur + iosize - 1,
3754	&cached, GFP_NOFS);	3712	&cached, GFP_NOFS);
@@ -3781,10 +3739,8 @@ static int btrfs_do_readpage(struct page page, struct extent_map *em_cached, · 26 unmodified lines
3781		3739
3782	ret = submit_extent_page(REQ_OP_READ \| read_flags, NULL,	3740	ret = submit_extent_page(REQ_OP_READ \| read_flags, NULL,
3783	bio_ctrl, page, disk_bytenr, iosize,	3741	bio_ctrl, page, disk_bytenr, iosize,
3784	pg_offset,	3742	pg_offset, end_bio_extent_readpage,
3785	end_bio_extent_readpage, 0,	3743	this_bio_flag, force_bio_submit);
3786	this_bio_flag,
3787	force_bio_submit);
3788	if (ret) {	3744	if (ret) {
3789	/*	3745	/*
3790	* We have to unlock the remaining range, or the page	3746	* We have to unlock the remaining range, or the page
@@ -3817,8 +3773,7 @@ int btrfs_read_folio(struct file file, struct folio folio) · 26 unmodified lines
3817	* If btrfs_do_readpage() failed we will want to submit the assembled	3773	* If btrfs_do_readpage() failed we will want to submit the assembled
3818	* bio to do the cleanup.	3774	* bio to do the cleanup.
3819	*/	3775	*/
3820	if (bio_ctrl.bio)	3776	submit_one_bio(&bio_ctrl);
3821	submit_one_bio(bio_ctrl.bio, 0, bio_ctrl.compress_type);
3822	return ret;	3777	return ret;
3823	}	3778	}
3824		3779
@@ -4101,7 +4056,7 @@ static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode, · 276 unmodified lines
4101	disk_bytenr, iosize,	4056	disk_bytenr, iosize,
4102	cur - page_offset(page),	4057	cur - page_offset(page),
4103	end_bio_extent_writepage,	4058	end_bio_extent_writepage,
4104	0, 0, false);	4059	0, false);
4105	if (ret) {	4060	if (ret) {
4106	has_error = true;	4061	has_error = true;
4107	if (!saved_ret)	4062	if (!saved_ret)
@@ -4166,10 +4121,8 @@ static int __extent_writepage(struct page page, struct writeback_control wbc, · 58 unmodified lines
4166	return 0;	4121	return 0;
4167	}	4122	}
4168		4123
4169	if (page->index == end_index) {	4124	if (page->index == end_index)
4170	memzero_page(page, pg_offset, PAGE_SIZE - pg_offset);	4125	memzero_page(page, pg_offset, PAGE_SIZE - pg_offset);
4171	flush_dcache_page(page);
4172	}
4173		4126
4174	ret = set_page_extent_mapped(page);	4127	ret = set_page_extent_mapped(page);
4175	if (ret < 0) {	4128	if (ret < 0) {
@@ -4278,7 +4231,7 @@ static noinline_for_stack int lock_extent_buffer_for_io(struct extent_buffer *eb · 102 unmodified lines
4278	int ret = 0;	4231	int ret = 0;
4279		4232
4280	if (!btrfs_try_tree_write_lock(eb)) {	4233	if (!btrfs_try_tree_write_lock(eb)) {
4281	flush_write_bio(epd);	4234	submit_write_bio(epd, 0);
4282	flush = 1;	4235	flush = 1;
4283	btrfs_tree_lock(eb);	4236	btrfs_tree_lock(eb);
4284	}	4237	}
@@ -4288,7 +4241,7 @@ static noinline_for_stack int lock_extent_buffer_for_io(struct extent_buffer *eb · 3 unmodified lines
4288	if (!epd->sync_io)	4241	if (!epd->sync_io)
4289	return 0;	4242	return 0;
4290	if (!flush) {	4243	if (!flush) {
4291	flush_write_bio(epd);	4244	submit_write_bio(epd, 0);
4292	flush = 1;	4245	flush = 1;
4293	}	4246	}
4294	while (1) {	4247	while (1) {
@@ -4335,7 +4288,7 @@ static noinline_for_stack int lock_extent_buffer_for_io(struct extent_buffer *eb · 40 unmodified lines
4335		4288
4336	if (!trylock_page(p)) {	4289	if (!trylock_page(p)) {
4337	if (!flush) {	4290	if (!flush) {
4338	flush_write_bio(epd);	4291	submit_write_bio(epd, 0);
4339	flush = 1;	4292	flush = 1;
4340	}	4293	}
4341	lock_page(p);	4294	lock_page(p);
@@ -4577,7 +4530,7 @@ static int write_one_subpage_eb(struct extent_buffer *eb, · 235 unmodified lines
4577	{	4530	{
4578	struct btrfs_fs_info *fs_info = eb->fs_info;	4531	struct btrfs_fs_info *fs_info = eb->fs_info;
4579	struct page *page = eb->pages[0];	4532	struct page *page = eb->pages[0];
4580	blk_opf_t write_flags = wbc_to_write_flags(wbc) \| REQ_META;	4533	blk_opf_t write_flags = wbc_to_write_flags(wbc);
4581	bool no_dirty_ebs = false;	4534	bool no_dirty_ebs = false;
4582	int ret;	4535	int ret;
4583		4536
@@ -4596,7 +4549,7 @@ static int write_one_subpage_eb(struct extent_buffer *eb, · 12 unmodified lines
4596	ret = submit_extent_page(REQ_OP_WRITE \| write_flags, wbc,	4549	ret = submit_extent_page(REQ_OP_WRITE \| write_flags, wbc,
4597	&epd->bio_ctrl, page, eb->start, eb->len,	4550	&epd->bio_ctrl, page, eb->start, eb->len,
4598	eb->start - page_offset(page),	4551	eb->start - page_offset(page),
4599	end_bio_subpage_eb_writepage, 0, 0, false);	4552	end_bio_subpage_eb_writepage, 0, false);
4600	if (ret) {	4553	if (ret) {
4601	btrfs_subpage_clear_writeback(fs_info, page, eb->start, eb->len);	4554	btrfs_subpage_clear_writeback(fs_info, page, eb->start, eb->len);
4602	set_btree_ioerr(page, eb);	4555	set_btree_ioerr(page, eb);
@@ -4622,7 +4575,7 @@ static noinline_for_stack int write_one_eb(struct extent_buffer *eb, · 19 unmodified lines
4622	{	4575	{
4623	u64 disk_bytenr = eb->start;	4576	u64 disk_bytenr = eb->start;
4624	int i, num_pages;	4577	int i, num_pages;
4625	blk_opf_t write_flags = wbc_to_write_flags(wbc) \| REQ_META;	4578	blk_opf_t write_flags = wbc_to_write_flags(wbc);
4626	int ret = 0;	4579	int ret = 0;
4627		4580
4628	prepare_eb_write(eb);	4581	prepare_eb_write(eb);
@@ -4637,7 +4590,7 @@ static noinline_for_stack int write_one_eb(struct extent_buffer *eb, · 8 unmodified lines
4637	&epd->bio_ctrl, p, disk_bytenr,	4590	&epd->bio_ctrl, p, disk_bytenr,
4638	PAGE_SIZE, 0,	4591	PAGE_SIZE, 0,
4639	end_bio_extent_buffer_writepage,	4592	end_bio_extent_buffer_writepage,
4640	0, 0, false);	4593	0, false);
4641	if (ret) {	4594	if (ret) {
4642	set_btree_ioerr(p, eb);	4595	set_btree_ioerr(p, eb);
4643	if (PageWriteback(p))	4596	if (PageWriteback(p))
@@ -4751,7 +4704,7 @@ static int submit_eb_subpage(struct page *page, · 107 unmodified lines
4751		4704
4752	cleanup:	4705	cleanup:
4753	/* We hit error, end bio for the submitted extent buffers */	4706	/* We hit error, end bio for the submitted extent buffers */
4754	end_write_bio(epd, ret);	4707	submit_write_bio(epd, ret);
4755	return ret;	4708	return ret;
4756	}	4709	}
4757		4710
@@ -4930,10 +4883,6 @@ retry: · 172 unmodified lines
4930	index = 0;	4883	index = 0;
4931	goto retry;	4884	goto retry;
4932	}	4885	}
4933	if (ret < 0) {
4934	end_write_bio(&epd, ret);
4935	goto out;
4936	}
4937	/*	4886	/*
4938	* If something went wrong, don't allow any metadata write bio to be	4887	* If something went wrong, don't allow any metadata write bio to be
4939	* submitted.	4888	* submitted.
@@ -4960,21 +4909,17 @@ retry: · 20 unmodified lines
4960	* Now such dirty tree block will not be cleaned by any dirty	4909	* Now such dirty tree block will not be cleaned by any dirty
4961	* extent io tree. Thus we don't want to submit such wild eb	4910	* extent io tree. Thus we don't want to submit such wild eb
4962	* if the fs already has error.	4911	* if the fs already has error.
4963	*/	4912	*
4964	if (!BTRFS_FS_ERROR(fs_info)) {
4965	flush_write_bio(&epd);
4966	} else {
4967	ret = -EROFS;
4968	end_write_bio(&epd, ret);
4969	}
4970	out:
4971	btrfs_zoned_meta_io_unlock(fs_info);
4972	/*
4973	* We can get ret > 0 from submit_extent_page() indicating how many ebs	4913	* We can get ret > 0 from submit_extent_page() indicating how many ebs
4974	* were submitted. Reset it to 0 to avoid false alerts for the caller.	4914	* were submitted. Reset it to 0 to avoid false alerts for the caller.
4975	*/	4915	*/
4976	if (ret > 0)	4916	if (ret > 0)
4977	ret = 0;	4917	ret = 0;
		4918	if (!ret && BTRFS_FS_ERROR(fs_info))
		4919	ret = -EROFS;
		4920	submit_write_bio(&epd, ret);
		4921
		4922	btrfs_zoned_meta_io_unlock(fs_info);
4978	return ret;	4923	return ret;
4979	}	4924	}
4980		4925
@@ -5076,7 +5021,7 @@ retry: · 95 unmodified lines
5076	* tmpfs file mapping	5021	* tmpfs file mapping
5077	*/	5022	*/
5078	if (!trylock_page(page)) {	5023	if (!trylock_page(page)) {
5079	flush_write_bio(epd);	5024	submit_write_bio(epd, 0);
5080	lock_page(page);	5025	lock_page(page);
5081	}	5026	}
5082		5027
@@ -5087,7 +5032,7 @@ retry: · 4 unmodified lines
5087		5032
5088	if (wbc->sync_mode != WB_SYNC_NONE) {	5033	if (wbc->sync_mode != WB_SYNC_NONE) {
5089	if (PageWriteback(page))	5034	if (PageWriteback(page))
5090	flush_write_bio(epd);	5035	submit_write_bio(epd, 0);
5091	wait_on_page_writeback(page);	5036	wait_on_page_writeback(page);
5092	}	5037	}
5093		5038
@@ -5127,7 +5072,7 @@ retry: · 33 unmodified lines
5127	* page in our current bio, and thus deadlock, so flush the	5072	* page in our current bio, and thus deadlock, so flush the
5128	* write bio here.	5073	* write bio here.
5129	*/	5074	*/
5130	flush_write_bio(epd);	5075	submit_write_bio(epd, 0);
5131	goto retry;	5076	goto retry;
5132	}	5077	}
5133		5078
@@ -5138,26 +5083,6 @@ retry: · 4 unmodified lines
5138	return ret;	5083	return ret;
5139	}	5084	}
5140		5085
5141	int extent_write_full_page(struct page page, struct writeback_control wbc)
5142	{
5143	int ret;
5144	struct extent_page_data epd = {
5145	.bio_ctrl = { 0 },
5146	.extent_locked = 0,
5147	.sync_io = wbc->sync_mode == WB_SYNC_ALL,
5148	};
5149
5150	ret = __extent_writepage(page, wbc, &epd);
5151	ASSERT(ret <= 0);
5152	if (ret < 0) {
5153	end_write_bio(&epd, ret);
5154	return ret;
5155	}
5156
5157	flush_write_bio(&epd);
5158	return ret;
5159	}
5160
5161	/*	5086	/*
5162	* Submit the pages in the range to bio for call sites which delalloc range has	5087	* Submit the pages in the range to bio for call sites which delalloc range has
5163	* already been ran (aka, ordered extent inserted) and all pages are still	5088	* already been ran (aka, ordered extent inserted) and all pages are still
@@ -5215,10 +5140,7 @@ int extent_write_locked_range(struct inode *inode, u64 start, u64 end) · 51 unmodified lines
5215	cur = cur_end + 1;	5140	cur = cur_end + 1;
5216	}	5141	}
5217		5142
5218	if (!found_error)	5143	submit_write_bio(&epd, found_error ? ret : 0);
5219	flush_write_bio(&epd);
5220	else
5221	end_write_bio(&epd, ret);
5222		5144
5223	wbc_detach_inode(&wbc_writepages);	5145	wbc_detach_inode(&wbc_writepages);
5224	if (found_error)	5146	if (found_error)
@@ -5243,13 +5165,7 @@ int extent_writepages(struct address_space *mapping, · 18 unmodified lines
5243	*/	5165	*/
5244	btrfs_zoned_data_reloc_lock(BTRFS_I(inode));	5166	btrfs_zoned_data_reloc_lock(BTRFS_I(inode));
5245	ret = extent_write_cache_pages(mapping, wbc, &epd);	5167	ret = extent_write_cache_pages(mapping, wbc, &epd);
5246	ASSERT(ret <= 0);	5168	submit_write_bio(&epd, ret);
5247	if (ret < 0) {
5248	btrfs_zoned_data_reloc_unlock(BTRFS_I(inode));
5249	end_write_bio(&epd, ret);
5250	return ret;
5251	}
5252	flush_write_bio(&epd);
5253	btrfs_zoned_data_reloc_unlock(BTRFS_I(inode));	5169	btrfs_zoned_data_reloc_unlock(BTRFS_I(inode));
5254	return ret;	5170	return ret;
5255	}	5171	}
@@ -5272,9 +5188,7 @@ void extent_readahead(struct readahead_control *rac) · 16 unmodified lines
5272		5188
5273	if (em_cached)	5189	if (em_cached)
5274	free_extent_map(em_cached);	5190	free_extent_map(em_cached);
5275		5191	submit_one_bio(&bio_ctrl);
5276	if (bio_ctrl.bio)
5277	submit_one_bio(bio_ctrl.bio, 0, bio_ctrl.compress_type);
5278	}	5192	}
5279		5193
5280	/*	5194	/*
@@ -6206,7 +6120,7 @@ static int check_eb_alignment(struct btrfs_fs_info *fs_info, u64 start) · 925 unmodified lines
6206	return -EINVAL;	6120	return -EINVAL;
6207	}	6121	}
6208	if (fs_info->nodesize >= PAGE_SIZE &&	6122	if (fs_info->nodesize >= PAGE_SIZE &&
6209	!IS_ALIGNED(start, PAGE_SIZE)) {	6123	!PAGE_ALIGNED(start)) {
6210	btrfs_err(fs_info,	6124	btrfs_err(fs_info,
6211	"tree block is not page aligned, start %llu nodesize %u",	6125	"tree block is not page aligned, start %llu nodesize %u",
6212	start, fs_info->nodesize);	6126	start, fs_info->nodesize);
@@ -6608,7 +6522,9 @@ static int read_extent_buffer_subpage(struct extent_buffer *eb, int wait, · 395 unmodified lines
6608	struct btrfs_fs_info *fs_info = eb->fs_info;	6522	struct btrfs_fs_info *fs_info = eb->fs_info;
6609	struct extent_io_tree *io_tree;	6523	struct extent_io_tree *io_tree;
6610	struct page *page = eb->pages[0];	6524	struct page *page = eb->pages[0];
6611	struct btrfs_bio_ctrl bio_ctrl = { 0 };	6525	struct btrfs_bio_ctrl bio_ctrl = {
		6526	.mirror_num = mirror_num,
		6527	};
6612	int ret = 0;	6528	int ret = 0;
6613		6529
6614	ASSERT(!test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags));	6530	ASSERT(!test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags));
@@ -6640,11 +6556,10 @@ static int read_extent_buffer_subpage(struct extent_buffer *eb, int wait, · 25 unmodified lines
6640	btrfs_subpage_clear_error(fs_info, page, eb->start, eb->len);	6556	btrfs_subpage_clear_error(fs_info, page, eb->start, eb->len);
6641		6557
6642	btrfs_subpage_start_reader(fs_info, page, eb->start, eb->len);	6558	btrfs_subpage_start_reader(fs_info, page, eb->start, eb->len);
6643	ret = submit_extent_page(REQ_OP_READ \| REQ_META, NULL, &bio_ctrl,	6559	ret = submit_extent_page(REQ_OP_READ, NULL, &bio_ctrl,
6644	page, eb->start, eb->len,	6560	page, eb->start, eb->len,
6645	eb->start - page_offset(page),	6561	eb->start - page_offset(page),
6646	end_bio_extent_readpage, mirror_num, 0,	6562	end_bio_extent_readpage, 0, true);
6647	true);
6648	if (ret) {	6563	if (ret) {
6649	/*	6564	/*
6650	* In the endio function, if we hit something wrong we will	6565	* In the endio function, if we hit something wrong we will
@@ -6653,10 +6568,7 @@ static int read_extent_buffer_subpage(struct extent_buffer *eb, int wait, · 2 unmodified lines
6653	*/	6568	*/
6654	atomic_dec(&eb->io_pages);	6569	atomic_dec(&eb->io_pages);
6655	}	6570	}
6656	if (bio_ctrl.bio) {	6571	submit_one_bio(&bio_ctrl);
6657	submit_one_bio(bio_ctrl.bio, mirror_num, 0);
6658	bio_ctrl.bio = NULL;
6659	}
6660	if (ret \|\| wait != WAIT_COMPLETE)	6572	if (ret \|\| wait != WAIT_COMPLETE)
6661	return ret;	6573	return ret;
6662		6574
@@ -6676,7 +6588,9 @@ int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num) · 13 unmodified lines
6676	int all_uptodate = 1;	6588	int all_uptodate = 1;
6677	int num_pages;	6589	int num_pages;
6678	unsigned long num_reads = 0;	6590	unsigned long num_reads = 0;
6679	struct btrfs_bio_ctrl bio_ctrl = { 0 };	6591	struct btrfs_bio_ctrl bio_ctrl = {
		6592	.mirror_num = mirror_num,
		6593	};
6680		6594
6681	if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))	6595	if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
6682	return 0;	6596	return 0;
@@ -6747,10 +6661,10 @@ int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num) · 64 unmodified lines
6747	}	6661	}
6748		6662
6749	ClearPageError(page);	6663	ClearPageError(page);
6750	err = submit_extent_page(REQ_OP_READ \| REQ_META, NULL,	6664	err = submit_extent_page(REQ_OP_READ, NULL,
6751	&bio_ctrl, page, page_offset(page),	6665	&bio_ctrl, page, page_offset(page),
6752	PAGE_SIZE, 0, end_bio_extent_readpage,	6666	PAGE_SIZE, 0, end_bio_extent_readpage,
6753	mirror_num, 0, false);	6667	0, false);
6754	if (err) {	6668	if (err) {
6755	/*	6669	/*
6756	* We failed to submit the bio so it's the	6670	* We failed to submit the bio so it's the
@@ -6767,10 +6681,7 @@ int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num) · 10 unmodified lines
6767	}	6681	}
6768	}	6682	}
6769		6683
6770	if (bio_ctrl.bio) {	6684	submit_one_bio(&bio_ctrl);
6771	submit_one_bio(bio_ctrl.bio, mirror_num, bio_ctrl.compress_type);
6772	bio_ctrl.bio = NULL;
6773	}
6774		6685
6775	if (ret \|\| wait != WAIT_COMPLETE)	6686	if (ret \|\| wait != WAIT_COMPLETE)
6776	return ret;	6687	return ret;

fs/btrfs/extent_io.h+4−11

 #define BITMAP_LAST_BYTE_MASK(nbits) \
 	(BYTE_MASK >> (-(nbits) & (BITS_PER_BYTE - 1)))
+struct btrfs_bio;
 struct btrfs_root;
 struct btrfs_inode;
 struct btrfs_io_bio;
 struct extent_map_tree;
-typedef struct extent_map *(get_extent_t)(struct btrfs_inode *inode,
-					  struct page *page, size_t pg_offset,
-					  u64 start, u64 len);
 int try_release_extent_mapping(struct page *page, gfp_t mask);
 int try_release_extent_buffer(struct page *page);
 int btrfs_read_folio(struct file *file, struct folio *folio);
-int extent_write_full_page(struct page *page, struct writeback_control *wbc);
 int extent_write_locked_range(struct inode *inode, u64 start, u64 end);
 int extent_writepages(struct address_space *mapping,
 		      struct writeback_control *wbc);
 int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array);
 struct bio *btrfs_bio_alloc(unsigned int nr_iovecs);
-struct bio *btrfs_bio_clone(struct block_device *bdev, struct bio *bio);
 struct bio *btrfs_bio_clone_partial(struct bio *orig, u64 offset, u64 size);
 void end_extent_writepage(struct page *page, int err, u64 start, u64 end);
 	u64 start;
 	u64 len;
 	u64 logical;
-	enum btrfs_compression_type compress_type;
 	int this_mirror;
 	int failed_mirror;
+	int num_copies;
 };
-int btrfs_repair_one_sector(struct inode *inode,
+int btrfs_repair_one_sector(struct inode *inode, struct btrfs_bio *failed_bbio,
-			    struct bio *failed_bio, u32 bio_offset,
+			    u32 bio_offset, struct page *page, unsigned int pgoff,
-			    struct page *page, unsigned int pgoff,
-			    u64 start, int failed_mirror,
 			    submit_bio_hook_t *submit_bio_hook);
 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS

fs/btrfs/file.c+23−6

 	btrfs_inode_unlock(inode, ilock_flags);
-	/* If 'err' is -ENOTBLK then it means we must fallback to buffered IO. */
+	/*
+	 * If 'err' is -ENOTBLK or we have not written all data, then it means
+	 * we must fallback to buffered IO.
+	 */
 	if ((err < 0 && err != -ENOTBLK) || !iov_iter_count(from))
 		goto out;
 buffered:
+	/*
+	 * If we are in a NOWAIT context, then return -EAGAIN to signal the caller
+	 * it must retry the operation in a context where blocking is acceptable,
+	 * since we currently don't have NOWAIT semantics support for buffered IO
+	 * and may block there for many reasons (reserving space for example).
+	 */
+	if (iocb->ki_flags & IOCB_NOWAIT) {
+		err = -EAGAIN;
+		goto out;
+	}
 	pos = iocb->ki_pos;
 	written_buffered = btrfs_buffered_write(iocb, from);
 	if (written_buffered < 0) {
 		num_written = btrfs_encoded_write(iocb, from, encoded);
 		num_sync = encoded->len;
 	} else if (iocb->ki_flags & IOCB_DIRECT) {
-		num_written = num_sync = btrfs_direct_write(iocb, from);
+		num_written = btrfs_direct_write(iocb, from);
+		num_sync = num_written;
 	} else {
-		num_written = num_sync = btrfs_buffered_write(iocb, from);
+		num_written = btrfs_buffered_write(iocb, from);
+		num_sync = num_written;
 	}
 	btrfs_set_inode_last_sub_trans(inode);
 	btrfs_release_log_ctx_extents(&ctx);
 	if (ret < 0) {
 		/* Fallthrough and commit/free transaction. */
-		ret = 1;
+		ret = BTRFS_LOG_FORCE_COMMIT;
 	}
 	/* we've logged all the items and now have a consistent
 		goto out;
 	}
 	rsv->size = btrfs_calc_insert_metadata_size(fs_info, 1);
-	rsv->failfast = 1;
+	rsv->failfast = true;
 	/*
 	 * 1 - update the inode
 	ASSERT(trans != NULL);
 	inode_inc_iversion(inode);
-	inode->i_mtime = inode->i_ctime = current_time(inode);
+	inode->i_mtime = current_time(inode);
+	inode->i_ctime = inode->i_mtime;
 	ret = btrfs_update_inode(trans, root, BTRFS_I(inode));
 	updated_inode = true;
 	btrfs_end_transaction(trans);

fs/btrfs/free-space-cache.c+2−1

 	 * data, keep it dense.
 	 */
 	if (btrfs_test_opt(fs_info, SSD_SPREAD)) {
-		cont1_bytes = min_bytes = bytes + empty_size;
+		cont1_bytes = bytes + empty_size;
+		min_bytes = cont1_bytes;
 	} else if (block_group->flags & BTRFS_BLOCK_GROUP_METADATA) {
 		cont1_bytes = bytes;
 		min_bytes = fs_info->sectorsize;

fs/btrfs/inode.c+400−364

@@ -114,21 +114,17 @@ struct kmem_cache *btrfs_free_space_bitmap_cachep; · 113 unmodified lines
114		114
115	static int btrfs_setsize(struct inode inode, struct iattr attr);	115	static int btrfs_setsize(struct inode inode, struct iattr attr);
116	static int btrfs_truncate(struct inode *inode, bool skip_writeback);	116	static int btrfs_truncate(struct inode *inode, bool skip_writeback);
117	static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent);
118	static noinline int cow_file_range(struct btrfs_inode *inode,	117	static noinline int cow_file_range(struct btrfs_inode *inode,
119	struct page *locked_page,	118	struct page *locked_page,
120	u64 start, u64 end, int *page_started,	119	u64 start, u64 end, int *page_started,
121	unsigned long *nr_written, int unlock);	120	unsigned long *nr_written, int unlock,
		121	u64 *done_offset);
122	static struct extent_map create_io_em(struct btrfs_inode inode, u64 start,	122	static struct extent_map create_io_em(struct btrfs_inode inode, u64 start,
123	u64 len, u64 orig_start, u64 block_start,	123	u64 len, u64 orig_start, u64 block_start,
124	u64 block_len, u64 orig_block_len,	124	u64 block_len, u64 orig_block_len,
125	u64 ram_bytes, int compress_type,	125	u64 ram_bytes, int compress_type,
126	int type);	126	int type);
127		127
128	static void __endio_write_update_ordered(struct btrfs_inode *inode,
129	const u64 offset, const u64 bytes,
130	const bool uptodate);
131
132	/*	128	/*
133	* btrfs_inode_lock - lock inode i_rwsem based on arguments passed	129	* btrfs_inode_lock - lock inode i_rwsem based on arguments passed
134	*	130	*
@@ -195,11 +191,14 @@ static inline void btrfs_cleanup_ordered_extents(struct btrfs_inode *inode, · 60 unmodified lines
195	{	191	{
196	unsigned long index = offset >> PAGE_SHIFT;	192	unsigned long index = offset >> PAGE_SHIFT;
197	unsigned long end_index = (offset + bytes - 1) >> PAGE_SHIFT;	193	unsigned long end_index = (offset + bytes - 1) >> PAGE_SHIFT;
198	u64 page_start = page_offset(locked_page);	194	u64 page_start, page_end;
199	u64 page_end = page_start + PAGE_SIZE - 1;
200
201	struct page *page;	195	struct page *page;
202		196
		197	if (locked_page) {
		198	page_start = page_offset(locked_page);
		199	page_end = page_start + PAGE_SIZE - 1;
		200	}
		201
203	while (index <= end_index) {	202	while (index <= end_index) {
204	/*	203	/*
205	* For locked page, we will call end_extent_writepage() on it	204	* For locked page, we will call end_extent_writepage() on it
@@ -212,7 +211,7 @@ static inline void btrfs_cleanup_ordered_extents(struct btrfs_inode *inode, · 6 unmodified lines
212	* btrfs_mark_ordered_io_finished() would skip the accounting	211	* btrfs_mark_ordered_io_finished() would skip the accounting
213	* for the page range, and the ordered extent will never finish.	212	* for the page range, and the ordered extent will never finish.
214	*/	213	*/
215	if (index == (page_offset(locked_page) >> PAGE_SHIFT)) {	214	if (locked_page && index == (page_start >> PAGE_SHIFT)) {
216	index++;	215	index++;
217	continue;	216	continue;
218	}	217	}
@@ -223,7 +222,7 @@ static inline void btrfs_cleanup_ordered_extents(struct btrfs_inode *inode, · 4 unmodified lines
223		222
224	/*	223	/*
225	* Here we just clear all Ordered bits for every page in the	224	* Here we just clear all Ordered bits for every page in the
226	* range, then __endio_write_update_ordered() will handle	225	* range, then btrfs_mark_ordered_io_finished() will handle
227	* the ordered extent accounting for the range.	226	* the ordered extent accounting for the range.
228	*/	227	*/
229	btrfs_page_clamp_clear_ordered(inode->root->fs_info, page,	228	btrfs_page_clamp_clear_ordered(inode->root->fs_info, page,
@@ -231,20 +230,23 @@ static inline void btrfs_cleanup_ordered_extents(struct btrfs_inode *inode, · 1 unmodified line
231	put_page(page);	230	put_page(page);
232	}	231	}
233		232
234	/* The locked page covers the full range, nothing needs to be done */	233	if (locked_page) {
235	if (bytes + offset <= page_offset(locked_page) + PAGE_SIZE)	234	/* The locked page covers the full range, nothing needs to be done */
236	return;	235	if (bytes + offset <= page_start + PAGE_SIZE)
237	/*	236	return;
238	* In case this page belongs to the delalloc range being instantiated	237	/*
239	* then skip it, since the first page of a range is going to be	238	* In case this page belongs to the delalloc range being
240	* properly cleaned up by the caller of run_delalloc_range	239	* instantiated then skip it, since the first page of a range is
241	*/	240	* going to be properly cleaned up by the caller of
242	if (page_start >= offset && page_end <= (offset + bytes - 1)) {	241	* run_delalloc_range
243	bytes = offset + bytes - page_offset(locked_page) - PAGE_SIZE;	242	*/
244	offset = page_offset(locked_page) + PAGE_SIZE;	243	if (page_start >= offset && page_end <= (offset + bytes - 1)) {
		244	bytes = offset + bytes - page_offset(locked_page) - PAGE_SIZE;
		245	offset = page_offset(locked_page) + PAGE_SIZE;
		246	}
245	}	247	}
246		248
247	return __endio_write_update_ordered(inode, offset, bytes, false);	249	return btrfs_mark_ordered_io_finished(inode, NULL, offset, bytes, false);
248	}	250	}
249		251
250	static int btrfs_dirty_inode(struct inode *inode);	252	static int btrfs_dirty_inode(struct inode *inode);
@@ -332,9 +334,9 @@ static int insert_inline_extent(struct btrfs_trans_handle *trans, · 81 unmodified lines
332	cur_size = min_t(unsigned long, compressed_size,	334	cur_size = min_t(unsigned long, compressed_size,
333	PAGE_SIZE);	335	PAGE_SIZE);
334		336
335	kaddr = kmap_atomic(cpage);	337	kaddr = kmap_local_page(cpage);
336	write_extent_buffer(leaf, kaddr, ptr, cur_size);	338	write_extent_buffer(leaf, kaddr, ptr, cur_size);
337	kunmap_atomic(kaddr);	339	kunmap_local(kaddr);
338		340
339	i++;	341	i++;
340	ptr += cur_size;	342	ptr += cur_size;
@@ -345,9 +347,9 @@ static int insert_inline_extent(struct btrfs_trans_handle *trans, · 4 unmodified lines
345	} else {	347	} else {
346	page = find_get_page(inode->vfs_inode.i_mapping, 0);	348	page = find_get_page(inode->vfs_inode.i_mapping, 0);
347	btrfs_set_file_extent_compression(leaf, ei, 0);	349	btrfs_set_file_extent_compression(leaf, ei, 0);
348	kaddr = kmap_atomic(page);	350	kaddr = kmap_local_page(page);
349	write_extent_buffer(leaf, kaddr, ptr, size);	351	write_extent_buffer(leaf, kaddr, ptr, size);
350	kunmap_atomic(kaddr);	352	kunmap_local(kaddr);
351	put_page(page);	353	put_page(page);
352	}	354	}
353	btrfs_mark_buffer_dirty(leaf);	355	btrfs_mark_buffer_dirty(leaf);
@@ -560,8 +562,8 @@ static inline int inode_need_compress(struct btrfs_inode *inode, u64 start, · 206 unmodified lines
560	* will unlock the full page.	562	* will unlock the full page.
561	*/	563	*/
562	if (fs_info->sectorsize < PAGE_SIZE) {	564	if (fs_info->sectorsize < PAGE_SIZE) {
563	if (!IS_ALIGNED(start, PAGE_SIZE) \|\|	565	if (!PAGE_ALIGNED(start) \|\|
564	!IS_ALIGNED(end + 1, PAGE_SIZE))	566	!PAGE_ALIGNED(end + 1))
565	return 0;	567	return 0;
566	}	568	}
567		569
@@ -678,8 +680,8 @@ again: · 110 unmodified lines
678	* Thus we must also check against @actual_end, not just @end.	680	* Thus we must also check against @actual_end, not just @end.
679	*/	681	*/
680	if (blocksize < PAGE_SIZE) {	682	if (blocksize < PAGE_SIZE) {
681	if (!IS_ALIGNED(start, PAGE_SIZE) \|\|	683	if (!PAGE_ALIGNED(start) \|\|
682	!IS_ALIGNED(round_up(actual_end, blocksize), PAGE_SIZE))	684	!PAGE_ALIGNED(round_up(actual_end, blocksize)))
683	goto cleanup_and_bail_uncompressed;	685	goto cleanup_and_bail_uncompressed;
684	}	686	}
685		687
@@ -920,15 +922,25 @@ static int submit_uncompressed_range(struct btrfs_inode *inode, · 234 unmodified lines
920	* can directly submit them without interruption.	922	* can directly submit them without interruption.
921	*/	923	*/
922	ret = cow_file_range(inode, locked_page, start, end, &page_started,	924	ret = cow_file_range(inode, locked_page, start, end, &page_started,
923	&nr_written, 0);	925	&nr_written, 0, NULL);
924	/* Inline extent inserted, page gets unlocked and everything is done */	926	/* Inline extent inserted, page gets unlocked and everything is done */
925	if (page_started) {	927	if (page_started) {
926	ret = 0;	928	ret = 0;
927	goto out;	929	goto out;
928	}	930	}
929	if (ret < 0) {	931	if (ret < 0) {
930	if (locked_page)	932	btrfs_cleanup_ordered_extents(inode, locked_page, start, end - start + 1);
		933	if (locked_page) {
		934	const u64 page_start = page_offset(locked_page);
		935	const u64 page_end = page_start + PAGE_SIZE - 1;
		936
		937	btrfs_page_set_error(inode->root->fs_info, locked_page,
		938	page_start, PAGE_SIZE);
		939	set_page_writeback(locked_page);
		940	end_page_writeback(locked_page);
		941	end_extent_writepage(locked_page, ret, page_start, page_end);
931	unlock_page(locked_page);	942	unlock_page(locked_page);
		943	}
932	goto out;	944	goto out;
933	}	945	}
934		946
@@ -1133,15 +1145,39 @@ static u64 get_extent_allocation_hint(struct btrfs_inode *inode, u64 start, · 198 unmodified lines
1133	* *page_started is set to one if we unlock locked_page and do everything	1145	* *page_started is set to one if we unlock locked_page and do everything
1134	* required to start IO on it. It may be clean and already done with	1146	* required to start IO on it. It may be clean and already done with
1135	* IO when we return.	1147	* IO when we return.
		1148	*
		1149	* When unlock == 1, we unlock the pages in successfully allocated regions.
		1150	* When unlock == 0, we leave them locked for writing them out.
		1151	*
		1152	* However, we unlock all the pages except @locked_page in case of failure.
		1153	*
		1154	* In summary, page locking state will be as follow:
		1155	*
		1156	* - page_started == 1 (return value)
		1157	* - All the pages are unlocked. IO is started.
		1158	* - Note that this can happen only on success
		1159	* - unlock == 1
		1160	* - All the pages except @locked_page are unlocked in any case
		1161	* - unlock == 0
		1162	* - On success, all the pages are locked for writing out them
		1163	* - On failure, all the pages except @locked_page are unlocked
		1164	*
		1165	* When a failure happens in the second or later iteration of the
		1166	* while-loop, the ordered extents created in previous iterations are kept
		1167	* intact. So, the caller must clean them up by calling
		1168	* btrfs_cleanup_ordered_extents(). See btrfs_run_delalloc_range() for
		1169	* example.
1136	*/	1170	*/
1137	static noinline int cow_file_range(struct btrfs_inode *inode,	1171	static noinline int cow_file_range(struct btrfs_inode *inode,
1138	struct page *locked_page,	1172	struct page *locked_page,
1139	u64 start, u64 end, int *page_started,	1173	u64 start, u64 end, int *page_started,
1140	unsigned long *nr_written, int unlock)	1174	unsigned long *nr_written, int unlock,
		1175	u64 *done_offset)
1141	{	1176	{
1142	struct btrfs_root *root = inode->root;	1177	struct btrfs_root *root = inode->root;
1143	struct btrfs_fs_info *fs_info = root->fs_info;	1178	struct btrfs_fs_info *fs_info = root->fs_info;
1144	u64 alloc_hint = 0;	1179	u64 alloc_hint = 0;
		1180	u64 orig_start = start;
1145	u64 num_bytes;	1181	u64 num_bytes;
1146	unsigned long ram_size;	1182	unsigned long ram_size;
1147	u64 cur_alloc_size = 0;	1183	u64 cur_alloc_size = 0;
@@ -1329,18 +1365,62 @@ out_reserve: · 181 unmodified lines
1329	btrfs_dec_block_group_reservations(fs_info, ins.objectid);	1365	btrfs_dec_block_group_reservations(fs_info, ins.objectid);
1330	btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1);	1366	btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1);
1331	out_unlock:	1367	out_unlock:
		1368	/*
		1369	* If done_offset is non-NULL and ret == -EAGAIN, we expect the
		1370	* caller to write out the successfully allocated region and retry.
		1371	*/
		1372	if (done_offset && ret == -EAGAIN) {
		1373	if (orig_start < start)
		1374	*done_offset = start - 1;
		1375	else
		1376	*done_offset = start;
		1377	return ret;
		1378	} else if (ret == -EAGAIN) {
		1379	/* Convert to -ENOSPC since the caller cannot retry. */
		1380	ret = -ENOSPC;
		1381	}
		1382
		1383	/*
		1384	* Now, we have three regions to clean up:
		1385	*
		1386	* \|-------(1)----\|---(2)---\|-------------(3)----------\|
		1387	* `- orig_start `- start `- start + cur_alloc_size `- end
		1388	*
		1389	* We process each region below.
		1390	*/
		1391
1332	clear_bits = EXTENT_LOCKED \| EXTENT_DELALLOC \| EXTENT_DELALLOC_NEW \|	1392	clear_bits = EXTENT_LOCKED \| EXTENT_DELALLOC \| EXTENT_DELALLOC_NEW \|
1333	EXTENT_DEFRAG \| EXTENT_CLEAR_META_RESV;	1393	EXTENT_DEFRAG \| EXTENT_CLEAR_META_RESV;
1334	page_ops = PAGE_UNLOCK \| PAGE_START_WRITEBACK \| PAGE_END_WRITEBACK;	1394	page_ops = PAGE_UNLOCK \| PAGE_START_WRITEBACK \| PAGE_END_WRITEBACK;
		1395
		1396	/*
		1397	* For the range (1). We have already instantiated the ordered extents
		1398	* for this region. They are cleaned up by
		1399	* btrfs_cleanup_ordered_extents() in e.g,
		1400	* btrfs_run_delalloc_range(). EXTENT_LOCKED \| EXTENT_DELALLOC are
		1401	* already cleared in the above loop. And, EXTENT_DELALLOC_NEW \|
		1402	* EXTENT_DEFRAG \| EXTENT_CLEAR_META_RESV are handled by the cleanup
		1403	* function.
		1404	*
		1405	* However, in case of unlock == 0, we still need to unlock the pages
		1406	* (except @locked_page) to ensure all the pages are unlocked.
		1407	*/
		1408	if (!unlock && orig_start < start) {
		1409	if (!locked_page)
		1410	mapping_set_error(inode->vfs_inode.i_mapping, ret);
		1411	extent_clear_unlock_delalloc(inode, orig_start, start - 1,
		1412	locked_page, 0, page_ops);
		1413	}
		1414
1335	/*	1415	/*
1336	* If we reserved an extent for our delalloc range (or a subrange) and	1416	* For the range (2). If we reserved an extent for our delalloc range
1337	* failed to create the respective ordered extent, then it means that	1417	* (or a subrange) and failed to create the respective ordered extent,
1338	* when we reserved the extent we decremented the extent's size from	1418	* then it means that when we reserved the extent we decremented the
1339	* the data space_info's bytes_may_use counter and incremented the	1419	* extent's size from the data space_info's bytes_may_use counter and
1340	* space_info's bytes_reserved counter by the same amount. We must make	1420	* incremented the space_info's bytes_reserved counter by the same
1341	* sure extent_clear_unlock_delalloc() does not try to decrement again	1421	* amount. We must make sure extent_clear_unlock_delalloc() does not try
1342	* the data space_info's bytes_may_use counter, therefore we do not pass	1422	* to decrement again the data space_info's bytes_may_use counter,
1343	* it the flag EXTENT_CLEAR_DATA_RESV.	1423	* therefore we do not pass it the flag EXTENT_CLEAR_DATA_RESV.
1344	*/	1424	*/
1345	if (extent_reserved) {	1425	if (extent_reserved) {
1346	extent_clear_unlock_delalloc(inode, start,	1426	extent_clear_unlock_delalloc(inode, start,
@@ -1350,12 +1430,19 @@ out_unlock: · 3 unmodified lines
1350	page_ops);	1430	page_ops);
1351	start += cur_alloc_size;	1431	start += cur_alloc_size;
1352	if (start >= end)	1432	if (start >= end)
1353	goto out;	1433	return ret;
1354	}	1434	}
		1435
		1436	/*
		1437	* For the range (3). We never touched the region. In addition to the
		1438	* clear_bits above, we add EXTENT_CLEAR_DATA_RESV to release the data
		1439	* space_info's bytes_may_use counter, reserved in
		1440	* btrfs_check_data_free_space().
		1441	*/
1355	extent_clear_unlock_delalloc(inode, start, end, locked_page,	1442	extent_clear_unlock_delalloc(inode, start, end, locked_page,
1356	clear_bits \| EXTENT_CLEAR_DATA_RESV,	1443	clear_bits \| EXTENT_CLEAR_DATA_RESV,
1357	page_ops);	1444	page_ops);
1358	goto out;	1445	return ret;
1359	}	1446	}
1360		1447
1361	/*	1448	/*
@@ -1538,19 +1625,42 @@ static noinline int run_delalloc_zoned(struct btrfs_inode *inode, · 176 unmodified lines
1538	u64 end, int *page_started,	1625	u64 end, int *page_started,
1539	unsigned long *nr_written)	1626	unsigned long *nr_written)
1540	{	1627	{
		1628	u64 done_offset = end;
1541	int ret;	1629	int ret;
		1630	bool locked_page_done = false;
1542		1631
1543	ret = cow_file_range(inode, locked_page, start, end, page_started,	1632	while (start <= end) {
1544	nr_written, 0);	1633	ret = cow_file_range(inode, locked_page, start, end, page_started,
1545	if (ret)	1634	nr_written, 0, &done_offset);
1546	return ret;	1635	if (ret && ret != -EAGAIN)
		1636	return ret;
1547		1637
1548	if (*page_started)	1638	if (*page_started) {
1549	return 0;	1639	ASSERT(ret == 0);
		1640	return 0;
		1641	}
		1642
		1643	if (ret == 0)
		1644	done_offset = end;
		1645
		1646	if (done_offset == start) {
		1647	struct btrfs_fs_info *info = inode->root->fs_info;
		1648
		1649	wait_var_event(&info->zone_finish_wait,
		1650	!test_bit(BTRFS_FS_NEED_ZONE_FINISH, &info->flags));
		1651	continue;
		1652	}
		1653
		1654	if (!locked_page_done) {
		1655	__set_page_dirty_nobuffers(locked_page);
		1656	account_page_redirty(locked_page);
		1657	}
		1658	locked_page_done = true;
		1659	extent_write_locked_range(&inode->vfs_inode, start, done_offset);
		1660
		1661	start = done_offset + 1;
		1662	}
1550		1663
1551	__set_page_dirty_nobuffers(locked_page);
1552	account_page_redirty(locked_page);
1553	extent_write_locked_range(&inode->vfs_inode, start, end);
1554	*page_started = 1;	1664	*page_started = 1;
1555		1665
1556	return 0;	1666	return 0;
@@ -1642,7 +1752,7 @@ static int fallback_to_cow(struct btrfs_inode inode, struct page locked_page, · 85 unmodified lines
1642	}	1752	}
1643		1753
1644	return cow_file_range(inode, locked_page, start, end, page_started,	1754	return cow_file_range(inode, locked_page, start, end, page_started,
1645	nr_written, 1);	1755	nr_written, 1, NULL);
1646	}	1756	}
1647		1757
1648	struct can_nocow_file_extent_args {	1758	struct can_nocow_file_extent_args {
@@ -2115,7 +2225,7 @@ int btrfs_run_delalloc_range(struct btrfs_inode inode, struct page locked_page · 466 unmodified lines
2115	page_started, nr_written);	2225	page_started, nr_written);
2116	else	2226	else
2117	ret = cow_file_range(inode, locked_page, start, end,	2227	ret = cow_file_range(inode, locked_page, start, end,
2118	page_started, nr_written, 1);	2228	page_started, nr_written, 1, NULL);
2119	} else {	2229	} else {
2120	set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, &inode->runtime_flags);	2230	set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, &inode->runtime_flags);
2121	ret = cow_file_range_async(inode, wbc, locked_page, start, end,	2231	ret = cow_file_range_async(inode, wbc, locked_page, start, end,
@@ -2131,6 +2241,7 @@ int btrfs_run_delalloc_range(struct btrfs_inode inode, struct page locked_page · 9 unmodified lines
2131	void btrfs_split_delalloc_extent(struct inode *inode,	2241	void btrfs_split_delalloc_extent(struct inode *inode,
2132	struct extent_state *orig, u64 split)	2242	struct extent_state *orig, u64 split)
2133	{	2243	{
		2244	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2134	u64 size;	2245	u64 size;
2135		2246
2136	/* not delalloc, ignore it */	2247	/* not delalloc, ignore it */
@@ -2138,7 +2249,7 @@ void btrfs_split_delalloc_extent(struct inode *inode, · 1 unmodified line
2138	return;	2249	return;
2139		2250
2140	size = orig->end - orig->start + 1;	2251	size = orig->end - orig->start + 1;
2141	if (size > BTRFS_MAX_EXTENT_SIZE) {	2252	if (size > fs_info->max_extent_size) {
2142	u32 num_extents;	2253	u32 num_extents;
2143	u64 new_size;	2254	u64 new_size;
2144		2255
@@ -2147,10 +2258,10 @@ void btrfs_split_delalloc_extent(struct inode *inode, · 2 unmodified lines
2147	* applies here, just in reverse.	2258	* applies here, just in reverse.
2148	*/	2259	*/
2149	new_size = orig->end - split + 1;	2260	new_size = orig->end - split + 1;
2150	num_extents = count_max_extents(new_size);	2261	num_extents = count_max_extents(fs_info, new_size);
2151	new_size = split - orig->start;	2262	new_size = split - orig->start;
2152	num_extents += count_max_extents(new_size);	2263	num_extents += count_max_extents(fs_info, new_size);
2153	if (count_max_extents(size) >= num_extents)	2264	if (count_max_extents(fs_info, size) >= num_extents)
2154	return;	2265	return;
2155	}	2266	}
2156		2267
@@ -2167,6 +2278,7 @@ void btrfs_split_delalloc_extent(struct inode *inode, · 10 unmodified lines
2167	void btrfs_merge_delalloc_extent(struct inode inode, struct extent_state new,	2278	void btrfs_merge_delalloc_extent(struct inode inode, struct extent_state new,
2168	struct extent_state *other)	2279	struct extent_state *other)
2169	{	2280	{
		2281	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2170	u64 new_size, old_size;	2282	u64 new_size, old_size;
2171	u32 num_extents;	2283	u32 num_extents;
2172		2284
@@ -2180,7 +2292,7 @@ void btrfs_merge_delalloc_extent(struct inode inode, struct extent_state new, · 7 unmodified lines
2180	new_size = other->end - new->start + 1;	2292	new_size = other->end - new->start + 1;
2181		2293
2182	/* we're not bigger than the max, unreserve the space and go */	2294	/* we're not bigger than the max, unreserve the space and go */
2183	if (new_size <= BTRFS_MAX_EXTENT_SIZE) {	2295	if (new_size <= fs_info->max_extent_size) {
2184	spin_lock(&BTRFS_I(inode)->lock);	2296	spin_lock(&BTRFS_I(inode)->lock);
2185	btrfs_mod_outstanding_extents(BTRFS_I(inode), -1);	2297	btrfs_mod_outstanding_extents(BTRFS_I(inode), -1);
2186	spin_unlock(&BTRFS_I(inode)->lock);	2298	spin_unlock(&BTRFS_I(inode)->lock);
@@ -2206,10 +2318,10 @@ void btrfs_merge_delalloc_extent(struct inode inode, struct extent_state new, · 19 unmodified lines
2206	* this case.	2318	* this case.
2207	*/	2319	*/
2208	old_size = other->end - other->start + 1;	2320	old_size = other->end - other->start + 1;
2209	num_extents = count_max_extents(old_size);	2321	num_extents = count_max_extents(fs_info, old_size);
2210	old_size = new->end - new->start + 1;	2322	old_size = new->end - new->start + 1;
2211	num_extents += count_max_extents(old_size);	2323	num_extents += count_max_extents(fs_info, old_size);
2212	if (count_max_extents(new_size) >= num_extents)	2324	if (count_max_extents(fs_info, new_size) >= num_extents)
2213	return;	2325	return;
2214		2326
2215	spin_lock(&BTRFS_I(inode)->lock);	2327	spin_lock(&BTRFS_I(inode)->lock);
@@ -2274,21 +2386,21 @@ static void btrfs_del_delalloc_inode(struct btrfs_root *root, · 58 unmodified lines
2274	* list of inodes that have pending delalloc work to be done.	2386	* list of inodes that have pending delalloc work to be done.
2275	*/	2387	*/
2276	void btrfs_set_delalloc_extent(struct inode inode, struct extent_state state,	2388	void btrfs_set_delalloc_extent(struct inode inode, struct extent_state state,
2277	unsigned *bits)	2389	u32 bits)
2278	{	2390	{
2279	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);	2391	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2280		2392
2281	if ((bits & EXTENT_DEFRAG) && !(bits & EXTENT_DELALLOC))	2393	if ((bits & EXTENT_DEFRAG) && !(bits & EXTENT_DELALLOC))
2282	WARN_ON(1);	2394	WARN_ON(1);
2283	/*	2395	/*
2284	* set_bit and clear bit hooks normally require _irqsave/restore	2396	* set_bit and clear bit hooks normally require _irqsave/restore
2285	* but in this case, we are only testing for the DELALLOC	2397	* but in this case, we are only testing for the DELALLOC
2286	* bit, which is only set or cleared with irqs on	2398	* bit, which is only set or cleared with irqs on
2287	*/	2399	*/
2288	if (!(state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {	2400	if (!(state->state & EXTENT_DELALLOC) && (bits & EXTENT_DELALLOC)) {
2289	struct btrfs_root *root = BTRFS_I(inode)->root;	2401	struct btrfs_root *root = BTRFS_I(inode)->root;
2290	u64 len = state->end + 1 - state->start;	2402	u64 len = state->end + 1 - state->start;
2291	u32 num_extents = count_max_extents(len);	2403	u32 num_extents = count_max_extents(fs_info, len);
2292	bool do_list = !btrfs_is_free_space_inode(BTRFS_I(inode));	2404	bool do_list = !btrfs_is_free_space_inode(BTRFS_I(inode));
2293		2405
2294	spin_lock(&BTRFS_I(inode)->lock);	2406	spin_lock(&BTRFS_I(inode)->lock);
@@ -2303,7 +2415,7 @@ void btrfs_set_delalloc_extent(struct inode inode, struct extent_state state, · 8 unmodified lines
2303	fs_info->delalloc_batch);	2415	fs_info->delalloc_batch);
2304	spin_lock(&BTRFS_I(inode)->lock);	2416	spin_lock(&BTRFS_I(inode)->lock);
2305	BTRFS_I(inode)->delalloc_bytes += len;	2417	BTRFS_I(inode)->delalloc_bytes += len;
2306	if (*bits & EXTENT_DEFRAG)	2418	if (bits & EXTENT_DEFRAG)
2307	BTRFS_I(inode)->defrag_bytes += len;	2419	BTRFS_I(inode)->defrag_bytes += len;
2308	if (do_list && !test_bit(BTRFS_INODE_IN_DELALLOC_LIST,	2420	if (do_list && !test_bit(BTRFS_INODE_IN_DELALLOC_LIST,
2309	&BTRFS_I(inode)->runtime_flags))	2421	&BTRFS_I(inode)->runtime_flags))
@@ -2312,7 +2424,7 @@ void btrfs_set_delalloc_extent(struct inode inode, struct extent_state state, · 2 unmodified lines
2312	}	2424	}
2313		2425
2314	if (!(state->state & EXTENT_DELALLOC_NEW) &&	2426	if (!(state->state & EXTENT_DELALLOC_NEW) &&
2315	(*bits & EXTENT_DELALLOC_NEW)) {	2427	(bits & EXTENT_DELALLOC_NEW)) {
2316	spin_lock(&BTRFS_I(inode)->lock);	2428	spin_lock(&BTRFS_I(inode)->lock);
2317	BTRFS_I(inode)->new_delalloc_bytes += state->end + 1 -	2429	BTRFS_I(inode)->new_delalloc_bytes += state->end + 1 -
2318	state->start;	2430	state->start;
@@ -2325,14 +2437,14 @@ void btrfs_set_delalloc_extent(struct inode inode, struct extent_state state, · 6 unmodified lines
2325	* accounting happens.	2437	* accounting happens.
2326	*/	2438	*/
2327	void btrfs_clear_delalloc_extent(struct inode *vfs_inode,	2439	void btrfs_clear_delalloc_extent(struct inode *vfs_inode,
2328	struct extent_state state, unsigned bits)	2440	struct extent_state *state, u32 bits)
2329	{	2441	{
2330	struct btrfs_inode *inode = BTRFS_I(vfs_inode);	2442	struct btrfs_inode *inode = BTRFS_I(vfs_inode);
2331	struct btrfs_fs_info *fs_info = btrfs_sb(vfs_inode->i_sb);	2443	struct btrfs_fs_info *fs_info = btrfs_sb(vfs_inode->i_sb);
2332	u64 len = state->end + 1 - state->start;	2444	u64 len = state->end + 1 - state->start;
2333	u32 num_extents = count_max_extents(len);	2445	u32 num_extents = count_max_extents(fs_info, len);
2334		2446
2335	if ((state->state & EXTENT_DEFRAG) && (*bits & EXTENT_DEFRAG)) {	2447	if ((state->state & EXTENT_DEFRAG) && (bits & EXTENT_DEFRAG)) {
2336	spin_lock(&inode->lock);	2448	spin_lock(&inode->lock);
2337	inode->defrag_bytes -= len;	2449	inode->defrag_bytes -= len;
2338	spin_unlock(&inode->lock);	2450	spin_unlock(&inode->lock);
@@ -2343,7 +2455,7 @@ void btrfs_clear_delalloc_extent(struct inode *vfs_inode, · 4 unmodified lines
2343	* but in this case, we are only testing for the DELALLOC	2455	* but in this case, we are only testing for the DELALLOC
2344	* bit, which is only set or cleared with irqs on	2456	* bit, which is only set or cleared with irqs on
2345	*/	2457	*/
2346	if ((state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {	2458	if ((state->state & EXTENT_DELALLOC) && (bits & EXTENT_DELALLOC)) {
2347	struct btrfs_root *root = inode->root;	2459	struct btrfs_root *root = inode->root;
2348	bool do_list = !btrfs_is_free_space_inode(inode);	2460	bool do_list = !btrfs_is_free_space_inode(inode);
2349		2461
@@ -2356,7 +2468,7 @@ void btrfs_clear_delalloc_extent(struct inode *vfs_inode, · 6 unmodified lines
2356	* don't need to call delalloc_release_metadata if there is an	2468	* don't need to call delalloc_release_metadata if there is an
2357	* error.	2469	* error.
2358	*/	2470	*/
2359	if (*bits & EXTENT_CLEAR_META_RESV &&	2471	if (bits & EXTENT_CLEAR_META_RESV &&
2360	root != fs_info->tree_root)	2472	root != fs_info->tree_root)
2361	btrfs_delalloc_release_metadata(inode, len, false);	2473	btrfs_delalloc_release_metadata(inode, len, false);
2362		2474
@@ -2366,7 +2478,7 @@ void btrfs_clear_delalloc_extent(struct inode *vfs_inode, · 3 unmodified lines
2366		2478
2367	if (!btrfs_is_data_reloc_root(root) &&	2479	if (!btrfs_is_data_reloc_root(root) &&
2368	do_list && !(state->state & EXTENT_NORESERVE) &&	2480	do_list && !(state->state & EXTENT_NORESERVE) &&
2369	(*bits & EXTENT_CLEAR_DATA_RESV))	2481	(bits & EXTENT_CLEAR_DATA_RESV))
2370	btrfs_free_reserved_data_space_noquota(fs_info, len);	2482	btrfs_free_reserved_data_space_noquota(fs_info, len);
2371		2483
2372	percpu_counter_add_batch(&fs_info->delalloc_bytes, -len,	2484	percpu_counter_add_batch(&fs_info->delalloc_bytes, -len,
@@ -2381,11 +2493,11 @@ void btrfs_clear_delalloc_extent(struct inode *vfs_inode, · 8 unmodified lines
2381	}	2493	}
2382		2494
2383	if ((state->state & EXTENT_DELALLOC_NEW) &&	2495	if ((state->state & EXTENT_DELALLOC_NEW) &&
2384	(*bits & EXTENT_DELALLOC_NEW)) {	2496	(bits & EXTENT_DELALLOC_NEW)) {
2385	spin_lock(&inode->lock);	2497	spin_lock(&inode->lock);
2386	ASSERT(inode->new_delalloc_bytes >= len);	2498	ASSERT(inode->new_delalloc_bytes >= len);
2387	inode->new_delalloc_bytes -= len;	2499	inode->new_delalloc_bytes -= len;
2388	if (*bits & EXTENT_ADD_INODE_BYTES)	2500	if (bits & EXTENT_ADD_INODE_BYTES)
2389	inode_add_bytes(&inode->vfs_inode, len);	2501	inode_add_bytes(&inode->vfs_inode, len);
2390	spin_unlock(&inode->lock);	2502	spin_unlock(&inode->lock);
2391	}	2503	}
@@ -2580,95 +2692,78 @@ out: · 188 unmodified lines
2580	return errno_to_blk_status(ret);	2692	return errno_to_blk_status(ret);
2581	}	2693	}
2582		2694
2583	/*	2695	void btrfs_submit_data_write_bio(struct inode inode, struct bio bio, int mirror_num)
2584	* extent_io.c submission hook. This does the right thing for csum calculation
2585	* on write, or reading the csums from the tree before a read.
2586	*
2587	* Rules about async/sync submit,
2588	* a) read: sync submit
2589	*
2590	* b) write without checksum: sync submit
2591	*
2592	* c) write with checksum:
2593	* c-1) if bio is issued by fsync: sync submit
2594	* (sync_writers != 0)
2595	*
2596	* c-2) if root is reloc root: sync submit
2597	* (only in case of buffered IO)
2598	*
2599	* c-3) otherwise: async submit
2600	*/
2601	void btrfs_submit_data_bio(struct inode inode, struct bio bio,
2602	int mirror_num, enum btrfs_compression_type compress_type)
2603	{	2696	{
2604	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);	2697	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2605	struct btrfs_root *root = BTRFS_I(inode)->root;	2698	struct btrfs_inode *bi = BTRFS_I(inode);
2606	enum btrfs_wq_endio_type metadata = BTRFS_WQ_ENDIO_DATA;	2699	blk_status_t ret;
2607	blk_status_t ret = 0;
2608	int skip_sum;
2609	int async = !atomic_read(&BTRFS_I(inode)->sync_writers);
2610
2611	skip_sum = (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM) \|\|
2612	test_bit(BTRFS_FS_STATE_NO_CSUMS, &fs_info->fs_state);
2613
2614	if (btrfs_is_free_space_inode(BTRFS_I(inode)))
2615	metadata = BTRFS_WQ_ENDIO_FREE_SPACE;
2616		2700
2617	if (bio_op(bio) == REQ_OP_ZONE_APPEND) {	2701	if (bio_op(bio) == REQ_OP_ZONE_APPEND) {
2618	struct page *page = bio_first_bvec_all(bio)->bv_page;	2702	ret = extract_ordered_extent(bi, bio,
2619	loff_t file_offset = page_offset(page);	2703	page_offset(bio_first_bvec_all(bio)->bv_page));
2620
2621	ret = extract_ordered_extent(BTRFS_I(inode), bio, file_offset);
2622	if (ret)	2704	if (ret)
2623	goto out;	2705	goto out;
2624	}	2706	}
2625		2707
2626	if (btrfs_op(bio) != BTRFS_MAP_WRITE) {	2708	/*
2627	ret = btrfs_bio_wq_end_io(fs_info, bio, metadata);	2709	* If we need to checksum, and the I/O is not issued by fsync and
2628	if (ret)	2710	* friends, that is ->sync_writers != 0, defer the submission to a
2629	goto out;	2711	* workqueue to parallelize it.
2630		2712	*
2631	if (compress_type != BTRFS_COMPRESS_NONE) {	2713	* Csum items for reloc roots have already been cloned at this point,
2632	/*	2714	* so they are handled as part of the no-checksum case.
2633	* btrfs_submit_compressed_read will handle completing	2715	*/
2634	* the bio if there were any errors, so just return	2716	if (!(bi->flags & BTRFS_INODE_NODATASUM) &&
2635	* here.	2717	!test_bit(BTRFS_FS_STATE_NO_CSUMS, &fs_info->fs_state) &&
2636	*/	2718	!btrfs_is_data_reloc_root(bi->root)) {
2637	btrfs_submit_compressed_read(inode, bio, mirror_num);	2719	if (!atomic_read(&bi->sync_writers) &&
		2720	btrfs_wq_submit_bio(inode, bio, mirror_num, 0,
		2721	btrfs_submit_bio_start))
2638	return;	2722	return;
2639	} else {	2723
2640	/*	2724	ret = btrfs_csum_one_bio(bi, bio, (u64)-1, false);
2641	* Lookup bio sums does extra checks around whether we
2642	* need to csum or not, which is why we ignore skip_sum
2643	* here.
2644	*/
2645	ret = btrfs_lookup_bio_sums(inode, bio, NULL);
2646	if (ret)
2647	goto out;
2648	}
2649	goto mapit;
2650	} else if (async && !skip_sum) {
2651	/* csum items have already been cloned */
2652	if (btrfs_is_data_reloc_root(root))
2653	goto mapit;
2654	/* we're doing a write, do the async checksumming */
2655	ret = btrfs_wq_submit_bio(inode, bio, mirror_num,
2656	0, btrfs_submit_bio_start);
2657	goto out;
2658	} else if (!skip_sum) {
2659	ret = btrfs_csum_one_bio(BTRFS_I(inode), bio, (u64)-1, false);
2660	if (ret)	2725	if (ret)
2661	goto out;	2726	goto out;
2662	}	2727	}
		2728	btrfs_submit_bio(fs_info, bio, mirror_num);
		2729	return;
		2730	out:
		2731	if (ret) {
		2732	bio->bi_status = ret;
		2733	bio_endio(bio);
		2734	}
		2735	}
2663		2736
2664	mapit:	2737	void btrfs_submit_data_read_bio(struct inode inode, struct bio bio,
2665	ret = btrfs_map_bio(fs_info, bio, mirror_num);	2738	int mirror_num, enum btrfs_compression_type compress_type)
		2739	{
		2740	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
		2741	blk_status_t ret;
2666		2742
2667	out:	2743	if (compress_type != BTRFS_COMPRESS_NONE) {
		2744	/*
		2745	* btrfs_submit_compressed_read will handle completing the bio
		2746	* if there were any errors, so just return here.
		2747	*/
		2748	btrfs_submit_compressed_read(inode, bio, mirror_num);
		2749	return;
		2750	}
		2751
		2752	/* Save the original iter for read repair */
		2753	btrfs_bio(bio)->iter = bio->bi_iter;
		2754
		2755	/*
		2756	* Lookup bio sums does extra checks around whether we need to csum or
		2757	* not, which is why we ignore skip_sum here.
		2758	*/
		2759	ret = btrfs_lookup_bio_sums(inode, bio, NULL);
2668	if (ret) {	2760	if (ret) {
2669	bio->bi_status = ret;	2761	bio->bi_status = ret;
2670	bio_endio(bio);	2762	bio_endio(bio);
		2763	return;
2671	}	2764	}
		2765
		2766	btrfs_submit_bio(fs_info, bio, mirror_num);
2672	}	2767	}
2673		2768
2674	/*	2769	/*
@@ -3075,8 +3170,10 @@ static int insert_ordered_extent_file_extent(struct btrfs_trans_handle *trans, · 400 unmodified lines
3075	btrfs_set_stack_file_extent_disk_num_bytes(&stack_fi,	3170	btrfs_set_stack_file_extent_disk_num_bytes(&stack_fi,
3076	oe->disk_num_bytes);	3171	oe->disk_num_bytes);
3077	btrfs_set_stack_file_extent_offset(&stack_fi, oe->offset);	3172	btrfs_set_stack_file_extent_offset(&stack_fi, oe->offset);
3078	if (test_bit(BTRFS_ORDERED_TRUNCATED, &oe->flags))	3173	if (test_bit(BTRFS_ORDERED_TRUNCATED, &oe->flags)) {
3079	num_bytes = ram_bytes = oe->truncated_len;	3174	num_bytes = oe->truncated_len;
		3175	ram_bytes = num_bytes;
		3176	}
3080	btrfs_set_stack_file_extent_num_bytes(&stack_fi, num_bytes);	3177	btrfs_set_stack_file_extent_num_bytes(&stack_fi, num_bytes);
3081	btrfs_set_stack_file_extent_ram_bytes(&stack_fi, ram_bytes);	3178	btrfs_set_stack_file_extent_ram_bytes(&stack_fi, ram_bytes);
3082	btrfs_set_stack_file_extent_compression(&stack_fi, oe->compress_type);	3179	btrfs_set_stack_file_extent_compression(&stack_fi, oe->compress_type);
@@ -3102,7 +3199,7 @@ static int insert_ordered_extent_file_extent(struct btrfs_trans_handle *trans, · 19 unmodified lines
3102	* an ordered extent if the range of bytes in the file it covers are	3199	* an ordered extent if the range of bytes in the file it covers are
3103	* fully written.	3200	* fully written.
3104	*/	3201	*/
3105	static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent)	3202	int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent)
3106	{	3203	{
3107	struct btrfs_inode *inode = BTRFS_I(ordered_extent->inode);	3204	struct btrfs_inode *inode = BTRFS_I(ordered_extent->inode);
3108	struct btrfs_root *root = inode->root;	3205	struct btrfs_root *root = inode->root;
@@ -3311,65 +3408,71 @@ out: · 202 unmodified lines
3311	return ret;	3408	return ret;
3312	}	3409	}
3313		3410
3314	static void finish_ordered_fn(struct btrfs_work *work)
3315	{
3316	struct btrfs_ordered_extent *ordered_extent;
3317	ordered_extent = container_of(work, struct btrfs_ordered_extent, work);
3318	btrfs_finish_ordered_io(ordered_extent);
3319	}
3320
3321	void btrfs_writepage_endio_finish_ordered(struct btrfs_inode *inode,	3411	void btrfs_writepage_endio_finish_ordered(struct btrfs_inode *inode,
3322	struct page *page, u64 start,	3412	struct page *page, u64 start,
3323	u64 end, bool uptodate)	3413	u64 end, bool uptodate)
3324	{	3414	{
3325	trace_btrfs_writepage_end_io_hook(inode, start, end, uptodate);	3415	trace_btrfs_writepage_end_io_hook(inode, start, end, uptodate);
3326		3416
3327	btrfs_mark_ordered_io_finished(inode, page, start, end + 1 - start,	3417	btrfs_mark_ordered_io_finished(inode, page, start, end + 1 - start, uptodate);
3328	finish_ordered_fn, uptodate);	3418	}
		3419
		3420	/*
		3421	* Verify the checksum for a single sector without any extra action that depend
		3422	* on the type of I/O.
		3423	*/
		3424	int btrfs_check_sector_csum(struct btrfs_fs_info fs_info, struct page page,
		3425	u32 pgoff, u8 csum, const u8 const csum_expected)
		3426	{
		3427	SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
		3428	char *kaddr;
		3429
		3430	ASSERT(pgoff + fs_info->sectorsize <= PAGE_SIZE);
		3431
		3432	shash->tfm = fs_info->csum_shash;
		3433
		3434	kaddr = kmap_local_page(page) + pgoff;
		3435	crypto_shash_digest(shash, kaddr, fs_info->sectorsize, csum);
		3436	kunmap_local(kaddr);
		3437
		3438	if (memcmp(csum, csum_expected, fs_info->csum_size))
		3439	return -EIO;
		3440	return 0;
3329	}	3441	}
3330		3442
3331	/*	3443	/*
3332	* check_data_csum - verify checksum of one sector of uncompressed data	3444	* check_data_csum - verify checksum of one sector of uncompressed data
3333	* @inode: inode	3445	* @inode: inode
3334	* @io_bio: btrfs_io_bio which contains the csum	3446	* @bbio: btrfs_bio which contains the csum
3335	* @bio_offset: offset to the beginning of the bio (in bytes)	3447	* @bio_offset: offset to the beginning of the bio (in bytes)
3336	* @page: page where is the data to be verified	3448	* @page: page where is the data to be verified
3337	* @pgoff: offset inside the page	3449	* @pgoff: offset inside the page
3338	* @start: logical offset in the file
3339	*	3450	*
3340	* The length of such check is always one sector size.	3451	* The length of such check is always one sector size.
		3452	*
		3453	* When csum mismatch is detected, we will also report the error and fill the
		3454	* corrupted range with zero. (Thus it needs the extra parameters)
3341	*/	3455	*/
3342	static int check_data_csum(struct inode inode, struct btrfs_bio bbio,	3456	int btrfs_check_data_csum(struct inode inode, struct btrfs_bio bbio,
3343	u32 bio_offset, struct page *page, u32 pgoff,	3457	u32 bio_offset, struct page *page, u32 pgoff)
3344	u64 start)
3345	{	3458	{
3346	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);	3459	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3347	SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
3348	char *kaddr;
3349	u32 len = fs_info->sectorsize;	3460	u32 len = fs_info->sectorsize;
3350	const u32 csum_size = fs_info->csum_size;
3351	unsigned int offset_sectors;
3352	u8 *csum_expected;	3461	u8 *csum_expected;
3353	u8 csum[BTRFS_CSUM_SIZE];	3462	u8 csum[BTRFS_CSUM_SIZE];
3354		3463
3355	ASSERT(pgoff + len <= PAGE_SIZE);	3464	ASSERT(pgoff + len <= PAGE_SIZE);
3356		3465
3357	offset_sectors = bio_offset >> fs_info->sectorsize_bits;	3466	csum_expected = btrfs_csum_ptr(fs_info, bbio->csum, bio_offset);
3358	csum_expected = ((u8 )bbio->csum) + offset_sectors csum_size;
3359
3360	kaddr = kmap_atomic(page);
3361	shash->tfm = fs_info->csum_shash;
3362
3363	crypto_shash_digest(shash, kaddr + pgoff, len, csum);
3364	kunmap_atomic(kaddr);
3365		3467
3366	if (memcmp(csum, csum_expected, csum_size))	3468	if (btrfs_check_sector_csum(fs_info, page, pgoff, csum, csum_expected))
3367	goto zeroit;	3469	goto zeroit;
3368
3369	return 0;	3470	return 0;
		3471
3370	zeroit:	3472	zeroit:
3371	btrfs_print_data_csum_error(BTRFS_I(inode), start, csum, csum_expected,	3473	btrfs_print_data_csum_error(BTRFS_I(inode),
3372	bbio->mirror_num);	3474	bbio->file_offset + bio_offset,
		3475	csum, csum_expected, bbio->mirror_num);
3373	if (bbio->device)	3476	if (bbio->device)
3374	btrfs_dev_stat_inc_and_print(bbio->device,	3477	btrfs_dev_stat_inc_and_print(bbio->device,
3375	BTRFS_DEV_STAT_CORRUPTION_ERRS);	3478	BTRFS_DEV_STAT_CORRUPTION_ERRS);
@@ -3401,11 +3504,6 @@ unsigned int btrfs_verify_data_csum(struct btrfs_bio *bbio, · 25 unmodified lines
3401	u32 pg_off;	3504	u32 pg_off;
3402	unsigned int result = 0;	3505	unsigned int result = 0;
3403		3506
3404	if (btrfs_page_test_checked(fs_info, page, start, end + 1 - start)) {
3405	btrfs_page_clear_checked(fs_info, page, start, end + 1 - start);
3406	return 0;
3407	}
3408
3409	/*	3507	/*
3410	* This only happens for NODATASUM or compressed read.	3508	* This only happens for NODATASUM or compressed read.
3411	* Normally this should be covered by above check for compressed read	3509	* Normally this should be covered by above check for compressed read
@@ -3438,8 +3536,7 @@ unsigned int btrfs_verify_data_csum(struct btrfs_bio *bbio, · 26 unmodified lines
3438	EXTENT_NODATASUM);	3536	EXTENT_NODATASUM);
3439	continue;	3537	continue;
3440	}	3538	}
3441	ret = check_data_csum(inode, bbio, bio_offset, page, pg_off,	3539	ret = btrfs_check_data_csum(inode, bbio, bio_offset, page, pg_off);
3442	page_offset(page) + pg_off);
3443	if (ret < 0) {	3540	if (ret < 0) {
3444	const int nr_bit = (pg_off - offset_in_page(start)) >>	3541	const int nr_bit = (pg_off - offset_in_page(start)) >>
3445	root->fs_info->sectorsize_bits;	3542	root->fs_info->sectorsize_bits;
@@ -4228,7 +4325,7 @@ skip_backref: · 782 unmodified lines
4228	/*	4325	/*
4229	* If we are in a rename context, we don't need to update anything in the	4326	* If we are in a rename context, we don't need to update anything in the
4230	* log. That will be done later during the rename by btrfs_log_new_name().	4327	* log. That will be done later during the rename by btrfs_log_new_name().
4231	* Besides that, doing it here would only cause extra unncessary btree	4328	* Besides that, doing it here would only cause extra unnecessary btree
4232	* operations on the log tree, increasing latency for applications.	4329	* operations on the log tree, increasing latency for applications.
4233	*/	4330	*/
4234	if (!rename_ctx) {	4331	if (!rename_ctx) {
@@ -4256,8 +4353,9 @@ err: · 21 unmodified lines
4256	btrfs_i_size_write(dir, dir->vfs_inode.i_size - name_len * 2);	4353	btrfs_i_size_write(dir, dir->vfs_inode.i_size - name_len * 2);
4257	inode_inc_iversion(&inode->vfs_inode);	4354	inode_inc_iversion(&inode->vfs_inode);
4258	inode_inc_iversion(&dir->vfs_inode);	4355	inode_inc_iversion(&dir->vfs_inode);
4259	inode->vfs_inode.i_ctime = dir->vfs_inode.i_mtime =	4356	inode->vfs_inode.i_ctime = current_time(&inode->vfs_inode);
4260	dir->vfs_inode.i_ctime = current_time(&inode->vfs_inode);	4357	dir->vfs_inode.i_mtime = inode->vfs_inode.i_ctime;
		4358	dir->vfs_inode.i_ctime = inode->vfs_inode.i_ctime;
4261	ret = btrfs_update_inode(trans, root, dir);	4359	ret = btrfs_update_inode(trans, root, dir);
4262	out:	4360	out:
4263	return ret;	4361	return ret;
@@ -4419,7 +4517,8 @@ static int btrfs_unlink_subvol(struct btrfs_trans_handle *trans, · 155 unmodified lines
4419		4517
4420	btrfs_i_size_write(BTRFS_I(dir), dir->i_size - name_len * 2);	4518	btrfs_i_size_write(BTRFS_I(dir), dir->i_size - name_len * 2);
4421	inode_inc_iversion(dir);	4519	inode_inc_iversion(dir);
4422	dir->i_mtime = dir->i_ctime = current_time(dir);	4520	dir->i_mtime = current_time(dir);
		4521	dir->i_ctime = dir->i_mtime;
4423	ret = btrfs_update_inode_fallback(trans, root, BTRFS_I(dir));	4522	ret = btrfs_update_inode_fallback(trans, root, BTRFS_I(dir));
4424	if (ret)	4523	if (ret)
4425	btrfs_abort_transaction(trans, ret);	4524	btrfs_abort_transaction(trans, ret);
@@ -4858,7 +4957,6 @@ again: · 432 unmodified lines
4858	else	4957	else
4859	memzero_page(page, (block_start - page_offset(page)) + offset,	4958	memzero_page(page, (block_start - page_offset(page)) + offset,
4860	len);	4959	len);
4861	flush_dcache_page(page);
4862	}	4960	}
4863	btrfs_page_clear_checked(fs_info, page, block_start,	4961	btrfs_page_clear_checked(fs_info, page, block_start,
4864	block_end + 1 - block_start);	4962	block_end + 1 - block_start);
@@ -5061,9 +5159,10 @@ static int btrfs_setsize(struct inode inode, struct iattr attr) · 196 unmodified lines
5061	*/	5159	*/
5062	if (newsize != oldsize) {	5160	if (newsize != oldsize) {
5063	inode_inc_iversion(inode);	5161	inode_inc_iversion(inode);
5064	if (!(mask & (ATTR_CTIME \| ATTR_MTIME)))	5162	if (!(mask & (ATTR_CTIME \| ATTR_MTIME))) {
5065	inode->i_ctime = inode->i_mtime =	5163	inode->i_mtime = current_time(inode);
5066	current_time(inode);	5164	inode->i_ctime = inode->i_mtime;
		5165	}
5067	}	5166	}
5068		5167
5069	if (newsize > oldsize) {	5168	if (newsize > oldsize) {
@@ -5371,7 +5470,7 @@ void btrfs_evict_inode(struct inode *inode) · 301 unmodified lines
5371	if (!rsv)	5470	if (!rsv)
5372	goto no_delete;	5471	goto no_delete;
5373	rsv->size = btrfs_calc_metadata_size(fs_info, 1);	5472	rsv->size = btrfs_calc_metadata_size(fs_info, 1);
5374	rsv->failfast = 1;	5473	rsv->failfast = true;
5375		5474
5376	btrfs_i_size_write(BTRFS_I(inode), 0);	5475	btrfs_i_size_write(BTRFS_I(inode), 0);
5377		5476
@@ -5763,14 +5862,14 @@ struct inode btrfs_lookup_dentry(struct inode dir, struct dentry *dentry) · 385 unmodified lines
5763	if (ret != -ENOENT)	5862	if (ret != -ENOENT)
5764	inode = ERR_PTR(ret);	5863	inode = ERR_PTR(ret);
5765	else	5864	else
5766	inode = new_simple_dir(dir->i_sb, &location, sub_root);	5865	inode = new_simple_dir(dir->i_sb, &location, root);
5767	} else {	5866	} else {
5768	inode = btrfs_iget(dir->i_sb, location.objectid, sub_root);	5867	inode = btrfs_iget(dir->i_sb, location.objectid, sub_root);
5769	}
5770	if (root != sub_root)
5771	btrfs_put_root(sub_root);	5868	btrfs_put_root(sub_root);
5772		5869
5773	if (!IS_ERR(inode) && root != sub_root) {	5870	if (IS_ERR(inode))
		5871	return inode;
		5872
5774	down_read(&fs_info->cleanup_work_sem);	5873	down_read(&fs_info->cleanup_work_sem);
5775	if (!sb_rdonly(inode->i_sb))	5874	if (!sb_rdonly(inode->i_sb))
5776	ret = btrfs_orphan_cleanup(sub_root);	5875	ret = btrfs_orphan_cleanup(sub_root);
@@ -6366,7 +6465,13 @@ int btrfs_create_new_inode(struct btrfs_trans_handle *trans, · 589 unmodified lines
6366	}	6465	}
6367		6466
6368	btrfs_mark_buffer_dirty(path->nodes[0]);	6467	btrfs_mark_buffer_dirty(path->nodes[0]);
6369	btrfs_release_path(path);	6468	/*
		6469	* We don't need the path anymore, plus inheriting properties, adding
		6470	* ACLs, security xattrs, orphan item or adding the link, will result in
		6471	* allocating yet another path. So just free our path.
		6472	*/
		6473	btrfs_free_path(path);
		6474	path = NULL;
6370		6475
6371	if (args->subvol) {	6476	if (args->subvol) {
6372	struct inode *parent;	6477	struct inode *parent;
@@ -6423,8 +6528,7 @@ int btrfs_create_new_inode(struct btrfs_trans_handle *trans, · 50 unmodified lines
6423	goto discard;	6528	goto discard;
6424	}	6529	}
6425		6530
6426	ret = 0;	6531	return 0;
6427	goto out;
6428		6532
6429	discard:	6533	discard:
6430	/*	6534	/*
@@ -7506,7 +7610,8 @@ static int btrfs_get_blocks_direct_write(struct extent_map **map, · 1075 unmodified lines
7506	btrfs_dec_nocow_writers(bg);	7610	btrfs_dec_nocow_writers(bg);
7507	if (type == BTRFS_ORDERED_PREALLOC) {	7611	if (type == BTRFS_ORDERED_PREALLOC) {
7508	free_extent_map(em);	7612	free_extent_map(em);
7509	*map = em = em2;	7613	*map = em2;
		7614	em = em2;
7510	}	7615	}
7511		7616
7512	if (IS_ERR(em2)) {	7617	if (IS_ERR(em2)) {
@@ -7588,8 +7693,12 @@ static int btrfs_dio_iomap_begin(struct inode *inode, loff_t start, · 75 unmodified lines
7588	const u64 data_alloc_len = length;	7693	const u64 data_alloc_len = length;
7589	bool unlock_extents = false;	7694	bool unlock_extents = false;
7590		7695
		7696	/*
		7697	* Cap the size of reads to that usually seen in buffered I/O as we need
		7698	* to allocate a contiguous array for the checksums.
		7699	*/
7591	if (!write)	7700	if (!write)
7592	len = min_t(u64, len, fs_info->sectorsize);	7701	len = min_t(u64, len, fs_info->sectorsize * BTRFS_MAX_BIO_SECTORS);
7593		7702
7594	lockstart = start;	7703	lockstart = start;
7595	lockend = start + len - 1;	7704	lockend = start + len - 1;
@@ -7824,8 +7933,8 @@ static int btrfs_dio_iomap_end(struct inode *inode, loff_t pos, loff_t length, · 228 unmodified lines
7824	pos += submitted;	7933	pos += submitted;
7825	length -= submitted;	7934	length -= submitted;
7826	if (write)	7935	if (write)
7827	__endio_write_update_ordered(BTRFS_I(inode), pos,	7936	btrfs_mark_ordered_io_finished(BTRFS_I(inode), NULL,
7828	length, false);	7937	pos, length, false);
7829	else	7938	else
7830	unlock_extent(&BTRFS_I(inode)->io_tree, pos,	7939	unlock_extent(&BTRFS_I(inode)->io_tree, pos,
7831	pos + length - 1);	7940	pos + length - 1);
@@ -7847,10 +7956,9 @@ static void btrfs_dio_private_put(struct btrfs_dio_private *dip) · 15 unmodified lines
7847	return;	7956	return;
7848		7957
7849	if (btrfs_op(&dip->bio) == BTRFS_MAP_WRITE) {	7958	if (btrfs_op(&dip->bio) == BTRFS_MAP_WRITE) {
7850	__endio_write_update_ordered(BTRFS_I(dip->inode),	7959	btrfs_mark_ordered_io_finished(BTRFS_I(dip->inode), NULL,
7851	dip->file_offset,	7960	dip->file_offset, dip->bytes,
7852	dip->bytes,	7961	!dip->bio.bi_status);
7853	!dip->bio.bi_status);
7854	} else {	7962	} else {
7855	unlock_extent(&BTRFS_I(dip->inode)->io_tree,	7963	unlock_extent(&BTRFS_I(dip->inode)->io_tree,
7856	dip->file_offset,	7964	dip->file_offset,
@@ -7870,12 +7978,8 @@ static void submit_dio_repair_bio(struct inode inode, struct bio bio, · 13 unmodified lines
7870		7978
7871	BUG_ON(bio_op(bio) == REQ_OP_WRITE);	7979	BUG_ON(bio_op(bio) == REQ_OP_WRITE);
7872		7980
7873	if (btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DATA))
7874	return;
7875
7876	refcount_inc(&dip->refs);	7981	refcount_inc(&dip->refs);
7877	if (btrfs_map_bio(fs_info, bio, mirror_num))	7982	btrfs_submit_bio(fs_info, bio, mirror_num);
7878	refcount_dec(&dip->refs);
7879	}	7983	}
7880		7984
7881	static blk_status_t btrfs_check_read_dio_bio(struct btrfs_dio_private *dip,	7985	static blk_status_t btrfs_check_read_dio_bio(struct btrfs_dio_private *dip,
@@ -7884,56 +7988,35 @@ static blk_status_t btrfs_check_read_dio_bio(struct btrfs_dio_private *dip, · 2 unmodified lines
7884	{	7988	{
7885	struct inode *inode = dip->inode;	7989	struct inode *inode = dip->inode;
7886	struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;	7990	struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
7887	const u32 sectorsize = fs_info->sectorsize;
7888	struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;	7991	struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
7889	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;	7992	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
7890	const bool csum = !(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM);	7993	const bool csum = !(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM);
7891	struct bio_vec bvec;
7892	struct bvec_iter iter;
7893	u32 bio_offset = 0;
7894	blk_status_t err = BLK_STS_OK;	7994	blk_status_t err = BLK_STS_OK;
		7995	struct bvec_iter iter;
		7996	struct bio_vec bv;
		7997	u32 offset;
		7998
		7999	btrfs_bio_for_each_sector(fs_info, bv, bbio, iter, offset) {
		8000	u64 start = bbio->file_offset + offset;
		8001
		8002	if (uptodate &&
		8003	(!csum \|\| !btrfs_check_data_csum(inode, bbio, offset, bv.bv_page,
		8004	bv.bv_offset))) {
		8005	clean_io_failure(fs_info, failure_tree, io_tree, start,
		8006	bv.bv_page, btrfs_ino(BTRFS_I(inode)),
		8007	bv.bv_offset);
		8008	} else {
		8009	int ret;
7895		8010
7896	__bio_for_each_segment(bvec, &bbio->bio, iter, bbio->iter) {	8011	ret = btrfs_repair_one_sector(inode, bbio, offset,
7897	unsigned int i, nr_sectors, pgoff;	8012	bv.bv_page, bv.bv_offset,
7898		8013	submit_dio_repair_bio);
7899	nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec.bv_len);	8014	if (ret)
7900	pgoff = bvec.bv_offset;	8015	err = errno_to_blk_status(ret);
7901	for (i = 0; i < nr_sectors; i++) {
7902	u64 start = bbio->file_offset + bio_offset;
7903
7904	ASSERT(pgoff < PAGE_SIZE);
7905	if (uptodate &&
7906	(!csum \|\| !check_data_csum(inode, bbio,
7907	bio_offset, bvec.bv_page,
7908	pgoff, start))) {
7909	clean_io_failure(fs_info, failure_tree, io_tree,
7910	start, bvec.bv_page,
7911	btrfs_ino(BTRFS_I(inode)),
7912	pgoff);
7913	} else {
7914	int ret;
7915
7916	ret = btrfs_repair_one_sector(inode, &bbio->bio,
7917	bio_offset, bvec.bv_page, pgoff,
7918	start, bbio->mirror_num,
7919	submit_dio_repair_bio);
7920	if (ret)
7921	err = errno_to_blk_status(ret);
7922	}
7923	ASSERT(bio_offset + sectorsize > bio_offset);
7924	bio_offset += sectorsize;
7925	pgoff += sectorsize;
7926	}	8016	}
7927	}	8017	}
7928	return err;
7929	}
7930		8018
7931	static void __endio_write_update_ordered(struct btrfs_inode *inode,	8019	return err;
7932	const u64 offset, const u64 bytes,
7933	const bool uptodate)
7934	{
7935	btrfs_mark_ordered_io_finished(inode, NULL, offset, bytes,
7936	finish_ordered_fn, uptodate);
7937	}	8020	}
7938		8021
7939	static blk_status_t btrfs_submit_bio_start_direct_io(struct inode *inode,	8022	static blk_status_t btrfs_submit_bio_start_direct_io(struct inode *inode,
@@ -7968,51 +8051,43 @@ static void btrfs_end_dio_bio(struct bio *bio) · 28 unmodified lines
7968	btrfs_dio_private_put(dip);	8051	btrfs_dio_private_put(dip);
7969	}	8052	}
7970		8053
7971	static inline blk_status_t btrfs_submit_dio_bio(struct bio *bio,	8054	static void btrfs_submit_dio_bio(struct bio bio, struct inode inode,
7972	struct inode *inode, u64 file_offset, int async_submit)	8055	u64 file_offset, int async_submit)
7973	{	8056	{
7974	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);	8057	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7975	struct btrfs_dio_private *dip = bio->bi_private;	8058	struct btrfs_dio_private *dip = bio->bi_private;
7976	bool write = btrfs_op(bio) == BTRFS_MAP_WRITE;
7977	blk_status_t ret;	8059	blk_status_t ret;
7978		8060
7979	/* Check btrfs_submit_bio_hook() for rules about async submit. */	8061	/* Save the original iter for read repair */
7980	if (async_submit)	8062	if (btrfs_op(bio) == BTRFS_MAP_READ)
7981	async_submit = !atomic_read(&BTRFS_I(inode)->sync_writers);	8063	btrfs_bio(bio)->iter = bio->bi_iter;
7982
7983	if (!write) {
7984	ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DATA);
7985	if (ret)
7986	goto err;
7987	}
7988		8064
7989	if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)	8065	if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)
7990	goto map;	8066	goto map;
7991		8067
7992	if (write && async_submit) {	8068	if (btrfs_op(bio) == BTRFS_MAP_WRITE) {
7993	ret = btrfs_wq_submit_bio(inode, bio, 0, file_offset,	8069	/* Check btrfs_submit_data_write_bio() for async submit rules */
7994	btrfs_submit_bio_start_direct_io);	8070	if (async_submit && !atomic_read(&BTRFS_I(inode)->sync_writers) &&
7995	goto err;	8071	btrfs_wq_submit_bio(inode, bio, 0, file_offset,
7996	} else if (write) {	8072	btrfs_submit_bio_start_direct_io))
		8073	return;
		8074
7997	/*	8075	/*
7998	* If we aren't doing async submit, calculate the csum of the	8076	* If we aren't doing async submit, calculate the csum of the
7999	* bio now.	8077	* bio now.
8000	*/	8078	*/
8001	ret = btrfs_csum_one_bio(BTRFS_I(inode), bio, file_offset, false);	8079	ret = btrfs_csum_one_bio(BTRFS_I(inode), bio, file_offset, false);
8002	if (ret)	8080	if (ret) {
8003	goto err;	8081	bio->bi_status = ret;
		8082	bio_endio(bio);
		8083	return;
		8084	}
8004	} else {	8085	} else {
8005	u64 csum_offset;	8086	btrfs_bio(bio)->csum = btrfs_csum_ptr(fs_info, dip->csums,
8006		8087	file_offset - dip->file_offset);
8007	csum_offset = file_offset - dip->file_offset;
8008	csum_offset >>= fs_info->sectorsize_bits;
8009	csum_offset *= fs_info->csum_size;
8010	btrfs_bio(bio)->csum = dip->csums + csum_offset;
8011	}	8088	}
8012	map:	8089	map:
8013	ret = btrfs_map_bio(fs_info, bio, 0);	8090	btrfs_submit_bio(fs_info, bio, 0);
8014	err:
8015	return ret;
8016	}	8091	}
8017		8092
8018	static void btrfs_submit_direct(const struct iomap_iter *iter,	8093	static void btrfs_submit_direct(const struct iomap_iter *iter,
@@ -8125,14 +8200,7 @@ static void btrfs_submit_direct(const struct iomap_iter *iter, · 106 unmodified lines
8125	async_submit = 1;	8200	async_submit = 1;
8126	}	8201	}
8127		8202
8128	status = btrfs_submit_dio_bio(bio, inode, file_offset,	8203	btrfs_submit_dio_bio(bio, inode, file_offset, async_submit);
8129	async_submit);
8130	if (status) {
8131	bio_put(bio);
8132	if (submit_len > 0)
8133	refcount_dec(&dip->refs);
8134	goto out_err_em;
8135	}
8136		8204
8137	dio_data->submitted += clone_len;	8205	dio_data->submitted += clone_len;
8138	clone_offset += clone_len;	8206	clone_offset += clone_len;
@@ -8181,31 +8249,6 @@ static int btrfs_fiemap(struct inode inode, struct fiemap_extent_info fieinfo, · 42 unmodified lines
8181	return extent_fiemap(BTRFS_I(inode), fieinfo, start, len);	8249	return extent_fiemap(BTRFS_I(inode), fieinfo, start, len);
8182	}	8250	}
8183		8251
8184	static int btrfs_writepage(struct page page, struct writeback_control wbc)
8185	{
8186	struct inode *inode = page->mapping->host;
8187	int ret;
8188
8189	if (current->flags & PF_MEMALLOC) {
8190	redirty_page_for_writepage(wbc, page);
8191	unlock_page(page);
8192	return 0;
8193	}
8194
8195	/*
8196	* If we are under memory pressure we will call this directly from the
8197	* VM, we need to make sure we have the inode referenced for the ordered
8198	* extent. If not just return like we didn't do anything.
8199	*/
8200	if (!igrab(inode)) {
8201	redirty_page_for_writepage(wbc, page);
8202	return AOP_WRITEPAGE_ACTIVATE;
8203	}
8204	ret = extent_write_full_page(page, wbc);
8205	btrfs_add_delayed_iput(inode);
8206	return ret;
8207	}
8208
8209	static int btrfs_writepages(struct address_space *mapping,	8252	static int btrfs_writepages(struct address_space *mapping,
8210	struct writeback_control *wbc)	8253	struct writeback_control *wbc)
8211	{	8254	{
@@ -8503,7 +8546,7 @@ vm_fault_t btrfs_page_mkwrite(struct vm_fault *vmf) · 291 unmodified lines
8503	* Reserving delalloc space after obtaining the page lock can lead to	8546	* Reserving delalloc space after obtaining the page lock can lead to
8504	* deadlock. For example, if a dirty page is locked by this function	8547	* deadlock. For example, if a dirty page is locked by this function
8505	* and the call to btrfs_delalloc_reserve_space() ends up triggering	8548	* and the call to btrfs_delalloc_reserve_space() ends up triggering
8506	* dirty page write out, then the btrfs_writepage() function could	8549	* dirty page write out, then the btrfs_writepages() function could
8507	* end up waiting indefinitely to get a lock on the page currently	8550	* end up waiting indefinitely to get a lock on the page currently
8508	* being processed by btrfs_page_mkwrite() function.	8551	* being processed by btrfs_page_mkwrite() function.
8509	*/	8552	*/
@@ -8594,10 +8637,9 @@ again: · 84 unmodified lines
8594	else	8637	else
8595	zero_start = PAGE_SIZE;	8638	zero_start = PAGE_SIZE;
8596		8639
8597	if (zero_start != PAGE_SIZE) {	8640	if (zero_start != PAGE_SIZE)
8598	memzero_page(page, zero_start, PAGE_SIZE - zero_start);	8641	memzero_page(page, zero_start, PAGE_SIZE - zero_start);
8599	flush_dcache_page(page);	8642
8600	}
8601	btrfs_page_clear_checked(fs_info, page, page_start, PAGE_SIZE);	8643	btrfs_page_clear_checked(fs_info, page, page_start, PAGE_SIZE);
8602	btrfs_page_set_dirty(fs_info, page, page_start, end + 1 - page_start);	8644	btrfs_page_set_dirty(fs_info, page, page_start, end + 1 - page_start);
8603	btrfs_page_set_uptodate(fs_info, page, page_start, end + 1 - page_start);	8645	btrfs_page_set_uptodate(fs_info, page, page_start, end + 1 - page_start);
@@ -8680,7 +8722,7 @@ static int btrfs_truncate(struct inode *inode, bool skip_writeback) · 76 unmodified lines
8680	if (!rsv)	8722	if (!rsv)
8681	return -ENOMEM;	8723	return -ENOMEM;
8682	rsv->size = min_size;	8724	rsv->size = min_size;
8683	rsv->failfast = 1;	8725	rsv->failfast = true;
8684		8726
8685	/*	8727	/*
8686	* 1 for the truncate slack space	8728	* 1 for the truncate slack space
@@ -9201,8 +9243,10 @@ static int btrfs_rename_exchange(struct inode *old_dir, · 514 unmodified lines
9201	inode_inc_iversion(new_dir);	9243	inode_inc_iversion(new_dir);
9202	inode_inc_iversion(old_inode);	9244	inode_inc_iversion(old_inode);
9203	inode_inc_iversion(new_inode);	9245	inode_inc_iversion(new_inode);
9204	old_dir->i_ctime = old_dir->i_mtime = ctime;	9246	old_dir->i_mtime = ctime;
9205	new_dir->i_ctime = new_dir->i_mtime = ctime;	9247	old_dir->i_ctime = ctime;
		9248	new_dir->i_mtime = ctime;
		9249	new_dir->i_ctime = ctime;
9206	old_inode->i_ctime = ctime;	9250	old_inode->i_ctime = ctime;
9207	new_inode->i_ctime = ctime;	9251	new_inode->i_ctime = ctime;
9208		9252
@@ -9465,9 +9509,11 @@ static int btrfs_rename(struct user_namespace *mnt_userns, · 256 unmodified lines
9465	inode_inc_iversion(old_dir);	9509	inode_inc_iversion(old_dir);
9466	inode_inc_iversion(new_dir);	9510	inode_inc_iversion(new_dir);
9467	inode_inc_iversion(old_inode);	9511	inode_inc_iversion(old_inode);
9468	old_dir->i_ctime = old_dir->i_mtime =	9512	old_dir->i_mtime = current_time(old_dir);
9469	new_dir->i_ctime = new_dir->i_mtime =	9513	old_dir->i_ctime = old_dir->i_mtime;
9470	old_inode->i_ctime = current_time(old_dir);	9514	new_dir->i_mtime = old_dir->i_mtime;
		9515	new_dir->i_ctime = old_dir->i_mtime;
		9516	old_inode->i_ctime = old_dir->i_mtime;
9471		9517
9472	if (old_dentry->d_parent != new_dentry->d_parent)	9518	if (old_dentry->d_parent != new_dentry->d_parent)
9473	btrfs_record_unlink_dir(trans, BTRFS_I(old_dir),	9519	btrfs_record_unlink_dir(trans, BTRFS_I(old_dir),
@@ -9555,15 +9601,21 @@ static int btrfs_rename2(struct user_namespace mnt_userns, struct inode old_di · 81 unmodified lines
9555	struct dentry old_dentry, struct inode new_dir,	9601	struct dentry old_dentry, struct inode new_dir,
9556	struct dentry *new_dentry, unsigned int flags)	9602	struct dentry *new_dentry, unsigned int flags)
9557	{	9603	{
		9604	int ret;
		9605
9558	if (flags & ~(RENAME_NOREPLACE \| RENAME_EXCHANGE \| RENAME_WHITEOUT))	9606	if (flags & ~(RENAME_NOREPLACE \| RENAME_EXCHANGE \| RENAME_WHITEOUT))
9559	return -EINVAL;	9607	return -EINVAL;
9560		9608
9561	if (flags & RENAME_EXCHANGE)	9609	if (flags & RENAME_EXCHANGE)
9562	return btrfs_rename_exchange(old_dir, old_dentry, new_dir,	9610	ret = btrfs_rename_exchange(old_dir, old_dentry, new_dir,
9563	new_dentry);	9611	new_dentry);
		9612	else
		9613	ret = btrfs_rename(mnt_userns, old_dir, old_dentry, new_dir,
		9614	new_dentry, flags);
9564		9615
9565	return btrfs_rename(mnt_userns, old_dir, old_dentry, new_dir,	9616	btrfs_btree_balance_dirty(BTRFS_I(new_dir)->root->fs_info);
9566	new_dentry, flags);	9617
		9618	return ret;
9567	}	9619	}
9568		9620
9569	struct btrfs_delalloc_work {	9621	struct btrfs_delalloc_work {
@@ -10183,9 +10235,8 @@ void btrfs_set_range_writeback(struct btrfs_inode *inode, u64 start, u64 end) · 613 unmodified lines
10183	}	10235	}
10184	}	10236	}
10185		10237
10186	static int btrfs_encoded_io_compression_from_extent(	10238	int btrfs_encoded_io_compression_from_extent(struct btrfs_fs_info *fs_info,
10187	struct btrfs_fs_info *fs_info,	10239	int compress_type)
10188	int compress_type)
10189	{	10240	{
10190	switch (compress_type) {	10241	switch (compress_type) {
10191	case BTRFS_COMPRESS_NONE:	10242	case BTRFS_COMPRESS_NONE:
@@ -10308,7 +10359,6 @@ static blk_status_t submit_encoded_read_bio(struct btrfs_inode *inode, · 116 unmodified lines
10308	struct bio *bio, int mirror_num)	10359	struct bio *bio, int mirror_num)
10309	{	10360	{
10310	struct btrfs_encoded_read_private *priv = bio->bi_private;	10361	struct btrfs_encoded_read_private *priv = bio->bi_private;
10311	struct btrfs_bio *bbio = btrfs_bio(bio);
10312	struct btrfs_fs_info *fs_info = inode->root->fs_info;	10362	struct btrfs_fs_info *fs_info = inode->root->fs_info;
10313	blk_status_t ret;	10363	blk_status_t ret;
10314		10364
@@ -10318,19 +10368,9 @@ static blk_status_t submit_encoded_read_bio(struct btrfs_inode *inode, · 3 unmodified lines
10318	return ret;	10368	return ret;
10319	}	10369	}
10320		10370
10321	ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DATA);
10322	if (ret) {
10323	btrfs_bio_free_csum(bbio);
10324	return ret;
10325	}
10326
10327	atomic_inc(&priv->pending);	10371	atomic_inc(&priv->pending);
10328	ret = btrfs_map_bio(fs_info, bio, mirror_num);	10372	btrfs_submit_bio(fs_info, bio, mirror_num);
10329	if (ret) {	10373	return BLK_STS_OK;
10330	atomic_dec(&priv->pending);
10331	btrfs_bio_free_csum(bbio);
10332	}
10333	return ret;
10334	}	10374	}
10335		10375
10336	static blk_status_t btrfs_encoded_read_verify_csum(struct btrfs_bio *bbio)	10376	static blk_status_t btrfs_encoded_read_verify_csum(struct btrfs_bio *bbio)
@@ -10342,7 +10382,6 @@ static blk_status_t btrfs_encoded_read_verify_csum(struct btrfs_bio *bbio) · 5 unmodified lines
10342	u32 sectorsize = fs_info->sectorsize;	10382	u32 sectorsize = fs_info->sectorsize;
10343	struct bio_vec *bvec;	10383	struct bio_vec *bvec;
10344	struct bvec_iter_all iter_all;	10384	struct bvec_iter_all iter_all;
10345	u64 start = priv->file_offset;
10346	u32 bio_offset = 0;	10385	u32 bio_offset = 0;
10347		10386
10348	if (priv->skip_csum \|\| !uptodate)	10387	if (priv->skip_csum \|\| !uptodate)
@@ -10355,10 +10394,9 @@ static blk_status_t btrfs_encoded_read_verify_csum(struct btrfs_bio *bbio) · 6 unmodified lines
10355	pgoff = bvec->bv_offset;	10394	pgoff = bvec->bv_offset;
10356	for (i = 0; i < nr_sectors; i++) {	10395	for (i = 0; i < nr_sectors; i++) {
10357	ASSERT(pgoff < PAGE_SIZE);	10396	ASSERT(pgoff < PAGE_SIZE);
10358	if (check_data_csum(&inode->vfs_inode, bbio, bio_offset,	10397	if (btrfs_check_data_csum(&inode->vfs_inode, bbio, bio_offset,
10359	bvec->bv_page, pgoff, start))	10398	bvec->bv_page, pgoff))
10360	return BLK_STS_IOERR;	10399	return BLK_STS_IOERR;
10361	start += sectorsize;
10362	bio_offset += sectorsize;	10400	bio_offset += sectorsize;
10363	pgoff += sectorsize;	10401	pgoff += sectorsize;
10364	}	10402	}
@@ -10390,11 +10428,9 @@ static void btrfs_encoded_read_endio(struct bio *bio) · 25 unmodified lines
10390	bio_put(bio);	10428	bio_put(bio);
10391	}	10429	}
10392		10430
10393	static int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode,	10431	int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode,
10394	u64 file_offset,	10432	u64 file_offset, u64 disk_bytenr,
10395	u64 disk_bytenr,	10433	u64 disk_io_size, struct page **pages)
10396	u64 disk_io_size,
10397	struct page **pages)
10398	{	10434	{
10399	struct btrfs_fs_info *fs_info = inode->root->fs_info;	10435	struct btrfs_fs_info *fs_info = inode->root->fs_info;
10400	struct btrfs_encoded_read_private priv = {	10436	struct btrfs_encoded_read_private priv = {
@@ -10625,7 +10661,8 @@ ssize_t btrfs_encoded_read(struct kiocb iocb, struct iov_iter iter, · 224 unmodified lines
10625	ret = -ENOBUFS;	10661	ret = -ENOBUFS;
10626	goto out_em;	10662	goto out_em;
10627	}	10663	}
10628	disk_io_size = count = em->block_len;	10664	disk_io_size = em->block_len;
		10665	count = em->block_len;
10629	encoded->unencoded_len = em->ram_bytes;	10666	encoded->unencoded_len = em->ram_bytes;
10630	encoded->unencoded_offset = iocb->ki_pos - em->orig_start;	10667	encoded->unencoded_offset = iocb->ki_pos - em->orig_start;
10631	ret = btrfs_encoded_io_compression_from_extent(fs_info,	10668	ret = btrfs_encoded_io_compression_from_extent(fs_info,
@@ -10788,15 +10825,15 @@ ssize_t btrfs_do_encoded_write(struct kiocb iocb, struct iov_iter from, · 156 unmodified lines
10788	ret = -ENOMEM;	10825	ret = -ENOMEM;
10789	goto out_pages;	10826	goto out_pages;
10790	}	10827	}
10791	kaddr = kmap(pages[i]);	10828	kaddr = kmap_local_page(pages[i]);
10792	if (copy_from_iter(kaddr, bytes, from) != bytes) {	10829	if (copy_from_iter(kaddr, bytes, from) != bytes) {
10793	kunmap(pages[i]);	10830	kunmap_local(kaddr);
10794	ret = -EFAULT;	10831	ret = -EFAULT;
10795	goto out_pages;	10832	goto out_pages;
10796	}	10833	}
10797	if (bytes < PAGE_SIZE)	10834	if (bytes < PAGE_SIZE)
10798	memset(kaddr + bytes, 0, PAGE_SIZE - bytes);	10835	memset(kaddr + bytes, 0, PAGE_SIZE - bytes);
10799	kunmap(pages[i]);	10836	kunmap_local(kaddr);
10800	}	10837	}
10801		10838
10802	for (;;) {	10839	for (;;) {
@@ -11425,7 +11462,6 @@ static const struct file_operations btrfs_dir_file_operations = { · 622 unmodified lines
11425	*/	11462	*/
11426	static const struct address_space_operations btrfs_aops = {	11463	static const struct address_space_operations btrfs_aops = {
11427	.read_folio = btrfs_read_folio,	11464	.read_folio = btrfs_read_folio,
11428	.writepage = btrfs_writepage,
11429	.writepages = btrfs_writepages,	11465	.writepages = btrfs_writepages,
11430	.readahead = btrfs_readahead,	11466	.readahead = btrfs_readahead,
11431	.direct_IO = noop_direct_IO,	11467	.direct_IO = noop_direct_IO,

fs/btrfs/ioctl.c+79−71

 	return em;
 }
-static u32 get_extent_max_capacity(const struct extent_map *em)
+static u32 get_extent_max_capacity(const struct btrfs_fs_info *fs_info,
+				   const struct extent_map *em)
 {
 	if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
 		return BTRFS_MAX_COMPRESSED;
-	return BTRFS_MAX_EXTENT_SIZE;
+	return fs_info->max_extent_size;
 }
 static bool defrag_check_next_extent(struct inode *inode, struct extent_map *em,
 				     u32 extent_thresh, u64 newer_than, bool locked)
 {
+	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
 	struct extent_map *next;
 	bool ret = false;
 	 * If the next extent is at its max capacity, defragging current extent
 	 * makes no sense, as the total number of extents won't change.
 	 */
-	if (next->len >= get_extent_max_capacity(em))
+	if (next->len >= get_extent_max_capacity(fs_info, em))
 		goto out;
 	/* Skip older extent */
 	if (next->generation < newer_than)
 				  bool locked, struct list_head *target_list,
 				  u64 *last_scanned_ret)
 {
+	struct btrfs_fs_info *fs_info = inode->root->fs_info;
 	bool last_is_target = false;
 	u64 cur = start;
 	int ret = 0;
 		 * Skip extents already at its max capacity, this is mostly for
 		 * compressed extents, which max cap is only 128K.
 		 */
-		if (em->len >= get_extent_max_capacity(em))
+		if (em->len >= get_extent_max_capacity(fs_info, em))
 			goto next;
 		/*
 	return ret;
 }
-static int build_ino_list(u64 inum, u64 offset, u64 root, void *ctx)
-{
-	struct btrfs_data_container *inodes = ctx;
-	const size_t c = 3 * sizeof(u64);
-	if (inodes->bytes_left >= c) {
-		inodes->bytes_left -= c;
-		inodes->val[inodes->elem_cnt] = inum;
-		inodes->val[inodes->elem_cnt + 1] = offset;
-		inodes->val[inodes->elem_cnt + 2] = root;
-		inodes->elem_cnt += 3;
-	} else {
-		inodes->bytes_missing += c - inodes->bytes_left;
-		inodes->bytes_left = 0;
-		inodes->elem_missed += 3;
-	}
-	return 0;
-}
 static long btrfs_ioctl_logical_to_ino(struct btrfs_fs_info *fs_info,
 					void __user *arg, int version)
 {
 	}
 	ret = iterate_inodes_from_logical(loi->logical, fs_info, path,
-					  build_ino_list, inodes, ignore_offset);
+					  inodes, ignore_offset);
 	if (ret == -EINVAL)
 		ret = -ENOENT;
 	if (ret < 0)
 	spin_unlock(&fs_info->balance_lock);
 }
+/**
+ * Try to acquire fs_info::balance_mutex as well as set BTRFS_EXLCOP_BALANCE as
+ * required.
+ *
+ * @fs_info:       the filesystem
+ * @excl_acquired: ptr to boolean value which is set to false in case balance
+ *                 is being resumed
+ *
+ * Return 0 on success in which case both fs_info::balance is acquired as well
+ * as exclusive ops are blocked. In case of failure return an error code.
+ */
+static int btrfs_try_lock_balance(struct btrfs_fs_info *fs_info, bool *excl_acquired)
+{
+	int ret;
+	/*
+	 * Exclusive operation is locked. Three possibilities:
+	 *   (1) some other op is running
+	 *   (2) balance is running
+	 *   (3) balance is paused -- special case (think resume)
+	 */
+	while (1) {
+		if (btrfs_exclop_start(fs_info, BTRFS_EXCLOP_BALANCE)) {
+			*excl_acquired = true;
+			mutex_lock(&fs_info->balance_mutex);
+			return 0;
+		}
+		mutex_lock(&fs_info->balance_mutex);
+		if (fs_info->balance_ctl) {
+			/* This is either (2) or (3) */
+			if (test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) {
+				/* This is (2) */
+				ret = -EINPROGRESS;
+				goto out_failure;
+			} else {
+				mutex_unlock(&fs_info->balance_mutex);
+				/*
+				 * Lock released to allow other waiters to
+				 * continue, we'll reexamine the status again.
+				 */
+				mutex_lock(&fs_info->balance_mutex);
+				if (fs_info->balance_ctl &&
+				    !test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) {
+					/* This is (3) */
+					*excl_acquired = false;
+					return 0;
+				}
+			}
+		} else {
+			/* This is (1) */
+			ret = BTRFS_ERROR_DEV_EXCL_RUN_IN_PROGRESS;
+			goto out_failure;
+		}
+		mutex_unlock(&fs_info->balance_mutex);
+	}
+out_failure:
+	mutex_unlock(&fs_info->balance_mutex);
+	*excl_acquired = false;
+	return ret;
+}
 static long btrfs_ioctl_balance(struct file *file, void __user *arg)
 {
 	struct btrfs_root *root = BTRFS_I(file_inode(file))->root;
 	struct btrfs_fs_info *fs_info = root->fs_info;
 	struct btrfs_ioctl_balance_args *bargs;
 	struct btrfs_balance_control *bctl;
-	bool need_unlock; /* for mut. excl. ops lock */
+	bool need_unlock = true;
 	int ret;
 	if (!capable(CAP_SYS_ADMIN))
 		goto out;
 	}
-again:
+	ret = btrfs_try_lock_balance(fs_info, &need_unlock);
-	if (btrfs_exclop_start(fs_info, BTRFS_EXCLOP_BALANCE)) {
+	if (ret)
-		mutex_lock(&fs_info->balance_mutex);
-		need_unlock = true;
-		goto locked;
-	}
-	/*
-	 * mut. excl. ops lock is locked.  Three possibilities:
-	 *   (1) some other op is running
-	 *   (2) balance is running
-	 *   (3) balance is paused -- special case (think resume)
-	 */
-	mutex_lock(&fs_info->balance_mutex);
-	if (fs_info->balance_ctl) {
-		/* this is either (2) or (3) */
-		if (!test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) {
-			mutex_unlock(&fs_info->balance_mutex);
-			/*
-			 * Lock released to allow other waiters to continue,
-			 * we'll reexamine the status again.
-			 */
-			mutex_lock(&fs_info->balance_mutex);
-			if (fs_info->balance_ctl &&
-			    !test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) {
-				/* this is (3) */
-				need_unlock = false;
-				goto locked;
-			}
-			mutex_unlock(&fs_info->balance_mutex);
-			goto again;
-		} else {
-			/* this is (2) */
-			mutex_unlock(&fs_info->balance_mutex);
-			ret = -EINPROGRESS;
-			goto out;
-		}
-	} else {
-		/* this is (1) */
-		mutex_unlock(&fs_info->balance_mutex);
-		ret = BTRFS_ERROR_DEV_EXCL_RUN_IN_PROGRESS;
 		goto out;
-	}
-locked:
+	lockdep_assert_held(&fs_info->balance_mutex);
 	if (bargs->flags & BTRFS_BALANCE_RESUME) {
 		if (!fs_info->balance_ctl) {
 			ret = -ENOTCONN;

fs/btrfs/lzo.c+12−16

 		out_pages[*cur_out / PAGE_SIZE] = cur_page;
 	}
-	kaddr = kmap(cur_page);
+	kaddr = kmap_local_page(cur_page);
 	write_compress_length(kaddr + offset_in_page(*cur_out),
 			      compressed_size);
 	*cur_out += LZO_LEN;
 		u32 copy_len = min_t(u32, sectorsize - *cur_out % sectorsize,
 				     orig_out + compressed_size - *cur_out);
-		kunmap(cur_page);
+		kunmap_local(kaddr);
 		if ((*cur_out / PAGE_SIZE) >= max_nr_page)
 			return -E2BIG;
 				return -ENOMEM;
 			out_pages[*cur_out / PAGE_SIZE] = cur_page;
 		}
-		kaddr = kmap(cur_page);
+		kaddr = kmap_local_page(cur_page);
 		memcpy(kaddr + offset_in_page(*cur_out),
 		       compressed_data + *cur_out - orig_out, copy_len);
 	*cur_out += sector_bytes_left;
 out:
-	kunmap(cur_page);
+	kunmap_local(kaddr);
 	return 0;
 }
 		/* Compress at most one sector of data each time */
 		in_len = min_t(u32, start + len - cur_in, sectorsize - sector_off);
 		ASSERT(in_len);
-		data_in = kmap(page_in);
+		data_in = kmap_local_page(page_in);
 		ret = lzo1x_1_compress(data_in +
 				       offset_in_page(cur_in), in_len,
 				       workspace->cbuf, &out_len,
 				       workspace->mem);
-		kunmap(page_in);
+		kunmap_local(data_in);
 		if (ret < 0) {
 			pr_debug("BTRFS: lzo in loop returned %d\n", ret);
 			ret = -EIO;
 	u32 orig_in = *cur_in;
 	while (*cur_in < orig_in + len) {
-		char *kaddr;
 		struct page *cur_page;
 		u32 copy_len = min_t(u32, PAGE_SIZE - offset_in_page(*cur_in),
 					  orig_in + len - *cur_in);
 		ASSERT(copy_len);
 		cur_page = cb->compressed_pages[*cur_in / PAGE_SIZE];
-		kaddr = kmap(cur_page);
+		memcpy_from_page(dest + *cur_in - orig_in, cur_page,
-		memcpy(dest + *cur_in - orig_in,
+				 offset_in_page(*cur_in), copy_len);
-			kaddr + offset_in_page(*cur_in),
-			copy_len);
-		kunmap(cur_page);
 		*cur_in += copy_len;
 	}
 	/* Bytes decompressed so far */
 	u32 cur_out = 0;
-	kaddr = kmap(cb->compressed_pages[0]);
+	kaddr = kmap_local_page(cb->compressed_pages[0]);
 	len_in = read_compress_length(kaddr);
-	kunmap(cb->compressed_pages[0]);
+	kunmap_local(kaddr);
 	cur_in += LZO_LEN;
 	/*
 		       (cur_in + LZO_LEN - 1) / sectorsize);
 		cur_page = cb->compressed_pages[cur_in / PAGE_SIZE];
 		ASSERT(cur_page);
-		kaddr = kmap(cur_page);
+		kaddr = kmap_local_page(cur_page);
 		seg_len = read_compress_length(kaddr + offset_in_page(cur_in));
-		kunmap(cur_page);
+		kunmap_local(kaddr);
 		cur_in += LZO_LEN;
 		if (seg_len > WORKSPACE_CBUF_LENGTH) {

fs/btrfs/ordered-data.c+29−11

 	spin_unlock_irq(&tree->lock);
 }
+static void finish_ordered_fn(struct btrfs_work *work)
+{
+	struct btrfs_ordered_extent *ordered_extent;
+	ordered_extent = container_of(work, struct btrfs_ordered_extent, work);
+	btrfs_finish_ordered_io(ordered_extent);
+}
 /*
  * Mark all ordered extents io inside the specified range finished.
  *
- * @page:	 The invovled page for the opeartion.
+ * @page:	 The involved page for the operation.
  *		 For uncompressed buffered IO, the page status also needs to be
  *		 updated to indicate whether the pending ordered io is finished.
  *		 Can be NULL for direct IO and compressed write.
  *		 For these cases, callers are ensured they won't execute the
  *		 endio function twice.
- * @finish_func: The function to be executed when all the IO of an ordered
- *		 extent are finished.
  *
  * This function is called for endio, thus the range must have ordered
- * extent(s) coveri it.
+ * extent(s) covering it.
  */
 void btrfs_mark_ordered_io_finished(struct btrfs_inode *inode,
-				struct page *page, u64 file_offset,
+				    struct page *page, u64 file_offset,
-				u64 num_bytes, btrfs_func_t finish_func,
+				    u64 num_bytes, bool uptodate)
-				bool uptodate)
 {
 	struct btrfs_ordered_inode_tree *tree = &inode->ordered_tree;
 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
 			set_bit(BTRFS_ORDERED_IO_DONE, &entry->flags);
 			cond_wake_up(&entry->wait);
 			refcount_inc(&entry->refs);
+			trace_btrfs_ordered_extent_mark_finished(inode, entry);
 			spin_unlock_irqrestore(&tree->lock, flags);
-			btrfs_init_work(&entry->work, finish_func, NULL, NULL);
+			btrfs_init_work(&entry->work, finish_ordered_fn, NULL, NULL);
 			btrfs_queue_work(wq, &entry->work);
 			spin_lock_irqsave(&tree->lock, flags);
 		}
 	if (finished && cached && entry) {
 		*cached = entry;
 		refcount_inc(&entry->refs);
+		trace_btrfs_ordered_extent_dec_test_pending(inode, entry);
 	}
 	spin_unlock_irqrestore(&tree->lock, flags);
 	return finished;
 	entry = rb_entry(node, struct btrfs_ordered_extent, rb_node);
 	if (!in_range(file_offset, entry->file_offset, entry->num_bytes))
 		entry = NULL;
-	if (entry)
+	if (entry) {
 		refcount_inc(&entry->refs);
+		trace_btrfs_ordered_extent_lookup(inode, entry);
+	}
 out:
 	spin_unlock_irqrestore(&tree->lock, flags);
 	return entry;
 			break;
 	}
 out:
-	if (entry)
+	if (entry) {
 		refcount_inc(&entry->refs);
+		trace_btrfs_ordered_extent_lookup_range(inode, entry);
+	}
 	spin_unlock_irq(&tree->lock);
 	return entry;
 }
 		ASSERT(list_empty(&ordered->log_list));
 		list_add_tail(&ordered->log_list, list);
 		refcount_inc(&ordered->refs);
+		trace_btrfs_ordered_extent_lookup_for_logging(inode, ordered);
 	}
 	spin_unlock_irq(&tree->lock);
 }
 	entry = rb_entry(node, struct btrfs_ordered_extent, rb_node);
 	refcount_inc(&entry->refs);
+	trace_btrfs_ordered_extent_lookup_first(inode, entry);
 out:
 	spin_unlock_irq(&tree->lock);
 	return entry;
 	/* No ordered extent in the range */
 	entry = NULL;
 out:
-	if (entry)
+	if (entry) {
 		refcount_inc(&entry->refs);
+		trace_btrfs_ordered_extent_lookup_first_range(inode, entry);
+	}
 	spin_unlock_irq(&tree->lock);
 	return entry;
 }
 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
 	int ret = 0;
+	trace_btrfs_ordered_extent_split(BTRFS_I(inode), ordered);
 	spin_lock_irq(&tree->lock);
 	/* Remove from tree once */
 	node = &ordered->rb_node;

fs/btrfs/ordered-data.h+3−2

 	t->last = NULL;
 }
+int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent);
 void btrfs_put_ordered_extent(struct btrfs_ordered_extent *entry);
 void btrfs_remove_ordered_extent(struct btrfs_inode *btrfs_inode,
 				struct btrfs_ordered_extent *entry);
 void btrfs_mark_ordered_io_finished(struct btrfs_inode *inode,
 				struct page *page, u64 file_offset,
-				u64 num_bytes, btrfs_func_t finish_func,
+				u64 num_bytes, bool uptodate);
-				bool uptodate);
 bool btrfs_dec_test_ordered_pending(struct btrfs_inode *inode,
 				    struct btrfs_ordered_extent **cached,
 				    u64 file_offset, u64 io_size);

fs/btrfs/raid56.c+371−421

@@ -63,137 +63,6 @@ struct sector_ptr { · 62 unmodified lines
63	unsigned int uptodate:8;	63	unsigned int uptodate:8;
64	};	64	};
65		65
66	enum btrfs_rbio_ops {
67	BTRFS_RBIO_WRITE,
68	BTRFS_RBIO_READ_REBUILD,
69	BTRFS_RBIO_PARITY_SCRUB,
70	BTRFS_RBIO_REBUILD_MISSING,
71	};
72
73	struct btrfs_raid_bio {
74	struct btrfs_io_context *bioc;
75
76	/* while we're doing rmw on a stripe
77	* we put it into a hash table so we can
78	* lock the stripe and merge more rbios
79	* into it.
80	*/
81	struct list_head hash_list;
82
83	/*
84	* LRU list for the stripe cache
85	*/
86	struct list_head stripe_cache;
87
88	/*
89	* for scheduling work in the helper threads
90	*/
91	struct work_struct work;
92
93	/*
94	* bio list and bio_list_lock are used
95	* to add more bios into the stripe
96	* in hopes of avoiding the full rmw
97	*/
98	struct bio_list bio_list;
99	spinlock_t bio_list_lock;
100
101	/* also protected by the bio_list_lock, the
102	* plug list is used by the plugging code
103	* to collect partial bios while plugged. The
104	* stripe locking code also uses it to hand off
105	* the stripe lock to the next pending IO
106	*/
107	struct list_head plug_list;
108
109	/*
110	* flags that tell us if it is safe to
111	* merge with this bio
112	*/
113	unsigned long flags;
114
115	/*
116	* set if we're doing a parity rebuild
117	* for a read from higher up, which is handled
118	* differently from a parity rebuild as part of
119	* rmw
120	*/
121	enum btrfs_rbio_ops operation;
122
123	/* Size of each individual stripe on disk */
124	u32 stripe_len;
125
126	/* How many pages there are for the full stripe including P/Q */
127	u16 nr_pages;
128
129	/* How many sectors there are for the full stripe including P/Q */
130	u16 nr_sectors;
131
132	/* Number of data stripes (no p/q) */
133	u8 nr_data;
134
135	/* Numer of all stripes (including P/Q) */
136	u8 real_stripes;
137
138	/* How many pages there are for each stripe */
139	u8 stripe_npages;
140
141	/* How many sectors there are for each stripe */
142	u8 stripe_nsectors;
143
144	/* First bad stripe, -1 means no corruption */
145	s8 faila;
146
147	/* Second bad stripe (for RAID6 use) */
148	s8 failb;
149
150	/* Stripe number that we're scrubbing */
151	u8 scrubp;
152
153	/*
154	* size of all the bios in the bio_list. This
155	* helps us decide if the rbio maps to a full
156	* stripe or not
157	*/
158	int bio_list_bytes;
159
160	int generic_bio_cnt;
161
162	refcount_t refs;
163
164	atomic_t stripes_pending;
165
166	atomic_t error;
167	/*
168	* these are two arrays of pointers. We allocate the
169	* rbio big enough to hold them both and setup their
170	* locations when the rbio is allocated
171	*/
172
173	/* pointers to pages that we allocated for
174	* reading/writing stripes directly from the disk (including P/Q)
175	*/
176	struct page **stripe_pages;
177
178	/* Pointers to the sectors in the bio_list, for faster lookup */
179	struct sector_ptr *bio_sectors;
180
181	/*
182	* For subpage support, we need to map each sector to above
183	* stripe_pages.
184	*/
185	struct sector_ptr *stripe_sectors;
186
187	/* Bitmap to record which horizontal stripe has data */
188	unsigned long *dbitmap;
189
190	/* allocated with real_stripes-many pointers for finish_() calls /
191	void **finish_pointers;
192
193	/* Allocated with stripe_nsectors-many bits for finish_() calls /
194	unsigned long *finish_pbitmap;
195	};
196
197	static int __raid56_parity_recover(struct btrfs_raid_bio *rbio);	66	static int __raid56_parity_recover(struct btrfs_raid_bio *rbio);
198	static noinline void finish_rmw(struct btrfs_raid_bio *rbio);	67	static noinline void finish_rmw(struct btrfs_raid_bio *rbio);
199	static void rmw_work(struct work_struct *work);	68	static void rmw_work(struct work_struct *work);
@@ -347,6 +216,24 @@ static void index_stripe_sectors(struct btrfs_raid_bio *rbio) · 147 unmodified lines
347	}	216	}
348	}	217	}
349		218
		219	static void steal_rbio_page(struct btrfs_raid_bio *src,
		220	struct btrfs_raid_bio *dest, int page_nr)
		221	{
		222	const u32 sectorsize = src->bioc->fs_info->sectorsize;
		223	const u32 sectors_per_page = PAGE_SIZE / sectorsize;
		224	int i;
		225
		226	if (dest->stripe_pages[page_nr])
		227	__free_page(dest->stripe_pages[page_nr]);
		228	dest->stripe_pages[page_nr] = src->stripe_pages[page_nr];
		229	src->stripe_pages[page_nr] = NULL;
		230
		231	/* Also update the sector->uptodate bits. */
		232	for (i = sectors_per_page * page_nr;
		233	i < sectors_per_page * page_nr + sectors_per_page; i++)
		234	dest->stripe_sectors[i].uptodate = true;
		235	}
		236
350	/*	237	/*
351	* Stealing an rbio means taking all the uptodate pages from the stripe array	238	* Stealing an rbio means taking all the uptodate pages from the stripe array
352	* in the source rbio and putting them into the destination rbio.	239	* in the source rbio and putting them into the destination rbio.
@@ -358,7 +245,6 @@ static void steal_rbio(struct btrfs_raid_bio src, struct btrfs_raid_bio dest) · 5 unmodified lines
358	{	245	{
359	int i;	246	int i;
360	struct page *s;	247	struct page *s;
361	struct page *d;
362		248
363	if (!test_bit(RBIO_CACHE_READY_BIT, &src->flags))	249	if (!test_bit(RBIO_CACHE_READY_BIT, &src->flags))
364	return;	250	return;
@@ -368,12 +254,7 @@ static void steal_rbio(struct btrfs_raid_bio src, struct btrfs_raid_bio dest) · 3 unmodified lines
368	if (!s \|\| !full_page_sectors_uptodate(src, i))	254	if (!s \|\| !full_page_sectors_uptodate(src, i))
369	continue;	255	continue;
370		256
371	d = dest->stripe_pages[i];	257	steal_rbio_page(src, dest, i);
372	if (d)
373	__free_page(d);
374
375	dest->stripe_pages[i] = s;
376	src->stripe_pages[i] = NULL;
377	}	258	}
378	index_stripe_sectors(dest);	259	index_stripe_sectors(dest);
379	index_stripe_sectors(src);	260	index_stripe_sectors(src);
@@ -391,6 +272,9 @@ static void merge_rbio(struct btrfs_raid_bio *dest, · 11 unmodified lines
391	{	272	{
392	bio_list_merge(&dest->bio_list, &victim->bio_list);	273	bio_list_merge(&dest->bio_list, &victim->bio_list);
393	dest->bio_list_bytes += victim->bio_list_bytes;	274	dest->bio_list_bytes += victim->bio_list_bytes;
		275	/* Also inherit the bitmaps from @victim. */
		276	bitmap_or(&dest->dbitmap, &victim->dbitmap, &dest->dbitmap,
		277	dest->stripe_nsectors);
394	dest->generic_bio_cnt += victim->generic_bio_cnt;	278	dest->generic_bio_cnt += victim->generic_bio_cnt;
395	bio_list_init(&victim->bio_list);	279	bio_list_init(&victim->bio_list);
396	}	280	}
@@ -590,9 +474,9 @@ static int rbio_is_full(struct btrfs_raid_bio *rbio) · 193 unmodified lines
590	int ret = 1;	474	int ret = 1;
591		475
592	spin_lock_irqsave(&rbio->bio_list_lock, flags);	476	spin_lock_irqsave(&rbio->bio_list_lock, flags);
593	if (size != rbio->nr_data * rbio->stripe_len)	477	if (size != rbio->nr_data * BTRFS_STRIPE_LEN)
594	ret = 0;	478	ret = 0;
595	BUG_ON(size > rbio->nr_data * rbio->stripe_len);	479	BUG_ON(size > rbio->nr_data * BTRFS_STRIPE_LEN);
596	spin_unlock_irqrestore(&rbio->bio_list_lock, flags);	480	spin_unlock_irqrestore(&rbio->bio_list_lock, flags);
597		481
598	return ret;	482	return ret;
@@ -932,6 +816,12 @@ static void rbio_orig_end_io(struct btrfs_raid_bio *rbio, blk_status_t err) · 333 unmodified lines
932		816
933	if (rbio->generic_bio_cnt)	817	if (rbio->generic_bio_cnt)
934	btrfs_bio_counter_sub(rbio->bioc->fs_info, rbio->generic_bio_cnt);	818	btrfs_bio_counter_sub(rbio->bioc->fs_info, rbio->generic_bio_cnt);
		819	/*
		820	* Clear the data bitmap, as the rbio may be cached for later usage.
		821	* do this before before unlock_stripe() so there will be no new bio
		822	* for this bio.
		823	*/
		824	bitmap_clear(&rbio->dbitmap, 0, rbio->stripe_nsectors);
935		825
936	/*	826	/*
937	* At this moment, rbio->bio_list is empty, however since rbio does not	827	* At this moment, rbio->bio_list is empty, however since rbio does not
@@ -1023,29 +913,30 @@ static struct sector_ptr sector_in_rbio(struct btrfs_raid_bio rbio, · 85 unmodified lines
1023	* this does not allocate any pages for rbio->pages.	913	* this does not allocate any pages for rbio->pages.
1024	*/	914	*/
1025	static struct btrfs_raid_bio alloc_rbio(struct btrfs_fs_info fs_info,	915	static struct btrfs_raid_bio alloc_rbio(struct btrfs_fs_info fs_info,
1026	struct btrfs_io_context *bioc,	916	struct btrfs_io_context *bioc)
1027	u32 stripe_len)
1028	{	917	{
1029	const unsigned int real_stripes = bioc->num_stripes - bioc->num_tgtdevs;	918	const unsigned int real_stripes = bioc->num_stripes - bioc->num_tgtdevs;
1030	const unsigned int stripe_npages = stripe_len >> PAGE_SHIFT;	919	const unsigned int stripe_npages = BTRFS_STRIPE_LEN >> PAGE_SHIFT;
1031	const unsigned int num_pages = stripe_npages * real_stripes;	920	const unsigned int num_pages = stripe_npages * real_stripes;
1032	const unsigned int stripe_nsectors = stripe_len >> fs_info->sectorsize_bits;	921	const unsigned int stripe_nsectors =
		922	BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits;
1033	const unsigned int num_sectors = stripe_nsectors * real_stripes;	923	const unsigned int num_sectors = stripe_nsectors * real_stripes;
1034	struct btrfs_raid_bio *rbio;	924	struct btrfs_raid_bio *rbio;
1035	int nr_data = 0;
1036	void *p;	925	void *p;
1037		926
1038	ASSERT(IS_ALIGNED(stripe_len, PAGE_SIZE));
1039	/* PAGE_SIZE must also be aligned to sectorsize for subpage support */	927	/* PAGE_SIZE must also be aligned to sectorsize for subpage support */
1040	ASSERT(IS_ALIGNED(PAGE_SIZE, fs_info->sectorsize));	928	ASSERT(IS_ALIGNED(PAGE_SIZE, fs_info->sectorsize));
		929	/*
		930	* Our current stripe len should be fixed to 64k thus stripe_nsectors
		931	* (at most 16) should be no larger than BITS_PER_LONG.
		932	*/
		933	ASSERT(stripe_nsectors <= BITS_PER_LONG);
1041		934
1042	rbio = kzalloc(sizeof(*rbio) +	935	rbio = kzalloc(sizeof(*rbio) +
1043	sizeof(rbio->stripe_pages) num_pages +	936	sizeof(rbio->stripe_pages) num_pages +
1044	sizeof(rbio->bio_sectors) num_sectors +	937	sizeof(rbio->bio_sectors) num_sectors +
1045	sizeof(rbio->stripe_sectors) num_sectors +	938	sizeof(rbio->stripe_sectors) num_sectors +
1046	sizeof(rbio->finish_pointers) real_stripes +	939	sizeof(rbio->finish_pointers) real_stripes,
1047	sizeof(rbio->dbitmap) BITS_TO_LONGS(stripe_nsectors) +
1048	sizeof(rbio->finish_pbitmap) BITS_TO_LONGS(stripe_nsectors),
1049	GFP_NOFS);	940	GFP_NOFS);
1050	if (!rbio)	941	if (!rbio)
1051	return ERR_PTR(-ENOMEM);	942	return ERR_PTR(-ENOMEM);
@@ -1056,7 +947,6 @@ static struct btrfs_raid_bio alloc_rbio(struct btrfs_fs_info fs_info, · 4 unmodified lines
1056	INIT_LIST_HEAD(&rbio->stripe_cache);	947	INIT_LIST_HEAD(&rbio->stripe_cache);
1057	INIT_LIST_HEAD(&rbio->hash_list);	948	INIT_LIST_HEAD(&rbio->hash_list);
1058	rbio->bioc = bioc;	949	rbio->bioc = bioc;
1059	rbio->stripe_len = stripe_len;
1060	rbio->nr_pages = num_pages;	950	rbio->nr_pages = num_pages;
1061	rbio->nr_sectors = num_sectors;	951	rbio->nr_sectors = num_sectors;
1062	rbio->real_stripes = real_stripes;	952	rbio->real_stripes = real_stripes;
@@ -1081,18 +971,11 @@ static struct btrfs_raid_bio alloc_rbio(struct btrfs_fs_info fs_info, · 18 unmodified lines
1081	CONSUME_ALLOC(rbio->bio_sectors, num_sectors);	971	CONSUME_ALLOC(rbio->bio_sectors, num_sectors);
1082	CONSUME_ALLOC(rbio->stripe_sectors, num_sectors);	972	CONSUME_ALLOC(rbio->stripe_sectors, num_sectors);
1083	CONSUME_ALLOC(rbio->finish_pointers, real_stripes);	973	CONSUME_ALLOC(rbio->finish_pointers, real_stripes);
1084	CONSUME_ALLOC(rbio->dbitmap, BITS_TO_LONGS(stripe_nsectors));
1085	CONSUME_ALLOC(rbio->finish_pbitmap, BITS_TO_LONGS(stripe_nsectors));
1086	#undef CONSUME_ALLOC	974	#undef CONSUME_ALLOC
1087		975
1088	if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID5)	976	ASSERT(btrfs_nr_parity_stripes(bioc->map_type));
1089	nr_data = real_stripes - 1;	977	rbio->nr_data = real_stripes - btrfs_nr_parity_stripes(bioc->map_type);
1090	else if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID6)
1091	nr_data = real_stripes - 2;
1092	else
1093	BUG();
1094		978
1095	rbio->nr_data = nr_data;
1096	return rbio;	979	return rbio;
1097	}	980	}
1098		981
@@ -1135,7 +1018,6 @@ static int rbio_add_io_sector(struct btrfs_raid_bio *rbio, · 36 unmodified lines
1135	struct sector_ptr *sector,	1018	struct sector_ptr *sector,
1136	unsigned int stripe_nr,	1019	unsigned int stripe_nr,
1137	unsigned int sector_nr,	1020	unsigned int sector_nr,
1138	unsigned long bio_max_len,
1139	enum req_op op)	1021	enum req_op op)
1140	{	1022	{
1141	const u32 sectorsize = rbio->bioc->fs_info->sectorsize;	1023	const u32 sectorsize = rbio->bioc->fs_info->sectorsize;
@@ -1180,7 +1062,8 @@ static int rbio_add_io_sector(struct btrfs_raid_bio *rbio, · 38 unmodified lines
1180	}	1062	}
1181		1063
1182	/* put a new bio on the list */	1064	/* put a new bio on the list */
1183	bio = bio_alloc(stripe->dev->bdev, max(bio_max_len >> PAGE_SHIFT, 1UL),	1065	bio = bio_alloc(stripe->dev->bdev,
		1066	max(BTRFS_STRIPE_LEN >> PAGE_SHIFT, 1),
1184	op, GFP_NOFS);	1067	op, GFP_NOFS);
1185	bio->bi_iter.bi_sector = disk_start >> 9;	1068	bio->bi_iter.bi_sector = disk_start >> 9;
1186	bio->bi_private = rbio;	1069	bio->bi_private = rbio;
@@ -1215,9 +1098,6 @@ static void index_one_bio(struct btrfs_raid_bio rbio, struct bio bio) · 28 unmodified lines
1215	u32 offset = (bio->bi_iter.bi_sector << SECTOR_SHIFT) -	1098	u32 offset = (bio->bi_iter.bi_sector << SECTOR_SHIFT) -
1216	rbio->bioc->raid_map[0];	1099	rbio->bioc->raid_map[0];
1217		1100
1218	if (bio_flagged(bio, BIO_CLONED))
1219	bio->bi_iter = btrfs_bio(bio)->iter;
1220
1221	bio_for_each_segment(bvec, bio, iter) {	1101	bio_for_each_segment(bvec, bio, iter) {
1222	u32 bvec_offset;	1102	u32 bvec_offset;
1223		1103
@@ -1252,6 +1132,34 @@ static void index_rbio_pages(struct btrfs_raid_bio *rbio) · 28 unmodified lines
1252	spin_unlock_irq(&rbio->bio_list_lock);	1132	spin_unlock_irq(&rbio->bio_list_lock);
1253	}	1133	}
1254		1134
		1135	static void bio_get_trace_info(struct btrfs_raid_bio rbio, struct bio bio,
		1136	struct raid56_bio_trace_info *trace_info)
		1137	{
		1138	const struct btrfs_io_context *bioc = rbio->bioc;
		1139	int i;
		1140
		1141	ASSERT(bioc);
		1142
		1143	/* We rely on bio->bi_bdev to find the stripe number. */
		1144	if (!bio->bi_bdev)
		1145	goto not_found;
		1146
		1147	for (i = 0; i < bioc->num_stripes; i++) {
		1148	if (bio->bi_bdev != bioc->stripes[i].dev->bdev)
		1149	continue;
		1150	trace_info->stripe_nr = i;
		1151	trace_info->devid = bioc->stripes[i].dev->devid;
		1152	trace_info->offset = (bio->bi_iter.bi_sector << SECTOR_SHIFT) -
		1153	bioc->stripes[i].physical;
		1154	return;
		1155	}
		1156
		1157	not_found:
		1158	trace_info->devid = -1;
		1159	trace_info->offset = -1;
		1160	trace_info->stripe_nr = -1;
		1161	}
		1162
1255	/*	1163	/*
1256	* this is called from one of two situations. We either	1164	* this is called from one of two situations. We either
1257	* have a full stripe from the higher layers, or we've read all	1165	* have a full stripe from the higher layers, or we've read all
@@ -1266,7 +1174,10 @@ static noinline void finish_rmw(struct btrfs_raid_bio *rbio) · 8 unmodified lines
1266	const u32 sectorsize = bioc->fs_info->sectorsize;	1174	const u32 sectorsize = bioc->fs_info->sectorsize;
1267	void **pointers = rbio->finish_pointers;	1175	void **pointers = rbio->finish_pointers;
1268	int nr_data = rbio->nr_data;	1176	int nr_data = rbio->nr_data;
		1177	/* The total sector number inside the full stripe. */
		1178	int total_sector_nr;
1269	int stripe;	1179	int stripe;
		1180	/* Sector number inside a stripe. */
1270	int sectornr;	1181	int sectornr;
1271	bool has_qstripe;	1182	bool has_qstripe;
1272	struct bio_list bio_list;	1183	struct bio_list bio_list;
@@ -1282,6 +1193,9 @@ static noinline void finish_rmw(struct btrfs_raid_bio *rbio) · 9 unmodified lines
1282	else	1193	else
1283	BUG();	1194	BUG();
1284		1195
		1196	/* We should have at least one data sector. */
		1197	ASSERT(bitmap_weight(&rbio->dbitmap, rbio->stripe_nsectors));
		1198
1285	/* at this point we either have a full stripe,	1199	/* at this point we either have a full stripe,
1286	* or we've read the full stripe from the drive.	1200	* or we've read the full stripe from the drive.
1287	* recalculate the parity and write the new results.	1201	* recalculate the parity and write the new results.
@@ -1348,55 +1262,71 @@ static noinline void finish_rmw(struct btrfs_raid_bio *rbio) · 60 unmodified lines
1348	}	1262	}
1349		1263
1350	/*	1264	/*
1351	* time to start writing. Make bios for everything from the	1265	* Start writing. Make bios for everything from the higher layers (the
1352	* higher layers (the bio_list in our rbio) and our p/q. Ignore	1266	* bio_list in our rbio) and our P/Q. Ignore everything else.
1353	* everything else.
1354	*/	1267	*/
1355	for (stripe = 0; stripe < rbio->real_stripes; stripe++) {	1268	for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors;
1356	for (sectornr = 0; sectornr < rbio->stripe_nsectors; sectornr++) {	1269	total_sector_nr++) {
1357	struct sector_ptr *sector;	1270	struct sector_ptr *sector;
1358		1271
1359	if (stripe < rbio->nr_data) {	1272	stripe = total_sector_nr / rbio->stripe_nsectors;
1360	sector = sector_in_rbio(rbio, stripe, sectornr, 1);	1273	sectornr = total_sector_nr % rbio->stripe_nsectors;
1361	if (!sector)
1362	continue;
1363	} else {
1364	sector = rbio_stripe_sector(rbio, stripe, sectornr);
1365	}
1366		1274
1367	ret = rbio_add_io_sector(rbio, &bio_list, sector, stripe,	1275	/* This vertical stripe has no data, skip it. */
1368	sectornr, rbio->stripe_len,	1276	if (!test_bit(sectornr, &rbio->dbitmap))
1369	REQ_OP_WRITE);	1277	continue;
1370	if (ret)	1278
1371	goto cleanup;	1279	if (stripe < rbio->nr_data) {
		1280	sector = sector_in_rbio(rbio, stripe, sectornr, 1);
		1281	if (!sector)
		1282	continue;
		1283	} else {
		1284	sector = rbio_stripe_sector(rbio, stripe, sectornr);
1372	}	1285	}
		1286
		1287	ret = rbio_add_io_sector(rbio, &bio_list, sector, stripe,
		1288	sectornr, REQ_OP_WRITE);
		1289	if (ret)
		1290	goto cleanup;
1373	}	1291	}
1374		1292
1375	if (likely(!bioc->num_tgtdevs))	1293	if (likely(!bioc->num_tgtdevs))
1376	goto write_data;	1294	goto write_data;
1377		1295
1378	for (stripe = 0; stripe < rbio->real_stripes; stripe++) {	1296	for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors;
1379	if (!bioc->tgtdev_map[stripe])	1297	total_sector_nr++) {
1380	continue;	1298	struct sector_ptr *sector;
1381		1299
1382	for (sectornr = 0; sectornr < rbio->stripe_nsectors; sectornr++) {	1300	stripe = total_sector_nr / rbio->stripe_nsectors;
1383	struct sector_ptr *sector;	1301	sectornr = total_sector_nr % rbio->stripe_nsectors;
1384		1302
1385	if (stripe < rbio->nr_data) {	1303	if (!bioc->tgtdev_map[stripe]) {
1386	sector = sector_in_rbio(rbio, stripe, sectornr, 1);	1304	/*
1387	if (!sector)	1305	* We can skip the whole stripe completely, note
1388	continue;	1306	* total_sector_nr will be increased by one anyway.
1389	} else {	1307	*/
1390	sector = rbio_stripe_sector(rbio, stripe, sectornr);	1308	ASSERT(sectornr == 0);
1391	}	1309	total_sector_nr += rbio->stripe_nsectors - 1;
		1310	continue;
		1311	}
1392		1312
1393	ret = rbio_add_io_sector(rbio, &bio_list, sector,	1313	/* This vertical stripe has no data, skip it. */
1394	rbio->bioc->tgtdev_map[stripe],	1314	if (!test_bit(sectornr, &rbio->dbitmap))
1395	sectornr, rbio->stripe_len,	1315	continue;
1396	REQ_OP_WRITE);	1316
1397	if (ret)	1317	if (stripe < rbio->nr_data) {
1398	goto cleanup;	1318	sector = sector_in_rbio(rbio, stripe, sectornr, 1);
		1319	if (!sector)
		1320	continue;
		1321	} else {
		1322	sector = rbio_stripe_sector(rbio, stripe, sectornr);
1399	}	1323	}
		1324
		1325	ret = rbio_add_io_sector(rbio, &bio_list, sector,
		1326	rbio->bioc->tgtdev_map[stripe],
		1327	sectornr, REQ_OP_WRITE);
		1328	if (ret)
		1329	goto cleanup;
1400	}	1330	}
1401		1331
1402	write_data:	1332	write_data:
@@ -1406,6 +1336,12 @@ write_data: · 3 unmodified lines
1406	while ((bio = bio_list_pop(&bio_list))) {	1336	while ((bio = bio_list_pop(&bio_list))) {
1407	bio->bi_end_io = raid_write_end_io;	1337	bio->bi_end_io = raid_write_end_io;
1408		1338
		1339	if (trace_raid56_write_stripe_enabled()) {
		1340	struct raid56_bio_trace_info trace_info = { 0 };
		1341
		1342	bio_get_trace_info(rbio, bio, &trace_info);
		1343	trace_raid56_write_stripe(rbio, bio, &trace_info);
		1344	}
1409	submit_bio(bio);	1345	submit_bio(bio);
1410	}	1346	}
1411	return;	1347	return;
@@ -1433,7 +1369,7 @@ static int find_bio_stripe(struct btrfs_raid_bio *rbio, · 21 unmodified lines
1433		1369
1434	for (i = 0; i < rbio->bioc->num_stripes; i++) {	1370	for (i = 0; i < rbio->bioc->num_stripes; i++) {
1435	stripe = &rbio->bioc->stripes[i];	1371	stripe = &rbio->bioc->stripes[i];
1436	if (in_range(physical, stripe->physical, rbio->stripe_len) &&	1372	if (in_range(physical, stripe->physical, BTRFS_STRIPE_LEN) &&
1437	stripe->dev->bdev && bio->bi_bdev == stripe->dev->bdev) {	1373	stripe->dev->bdev && bio->bi_bdev == stripe->dev->bdev) {
1438	return i;	1374	return i;
1439	}	1375	}
@@ -1455,7 +1391,7 @@ static int find_logical_bio_stripe(struct btrfs_raid_bio *rbio, · 15 unmodified lines
1455	for (i = 0; i < rbio->nr_data; i++) {	1391	for (i = 0; i < rbio->nr_data; i++) {
1456	u64 stripe_start = rbio->bioc->raid_map[i];	1392	u64 stripe_start = rbio->bioc->raid_map[i];
1457		1393
1458	if (in_range(logical, stripe_start, rbio->stripe_len))	1394	if (in_range(logical, stripe_start, BTRFS_STRIPE_LEN))
1459	return i;	1395	return i;
1460	}	1396	}
1461	return -1;	1397	return -1;
@@ -1552,15 +1488,7 @@ static void set_bio_pages_uptodate(struct btrfs_raid_bio rbio, struct bio bio) · 90 unmodified lines
1552	}	1488	}
1553	}	1489	}
1554		1490
1555	/*	1491	static void raid56_bio_end_io(struct bio *bio)
1556	* end io for the read phase of the rmw cycle. All the bios here are physical
1557	* stripe bios we've read from the disk so we can recalculate the parity of the
1558	* stripe.
1559	*
1560	* This will usually kick off finish_rmw once all the bios are read in, but it
1561	* may trigger parity reconstruction if we had any errors along the way
1562	*/
1563	static void raid_rmw_end_io(struct bio *bio)
1564	{	1492	{
1565	struct btrfs_raid_bio *rbio = bio->bi_private;	1493	struct btrfs_raid_bio *rbio = bio->bi_private;
1566		1494
@@ -1571,23 +1499,34 @@ static void raid_rmw_end_io(struct bio *bio) · 4 unmodified lines
1571		1499
1572	bio_put(bio);	1500	bio_put(bio);
1573		1501
1574	if (!atomic_dec_and_test(&rbio->stripes_pending))	1502	if (atomic_dec_and_test(&rbio->stripes_pending))
1575	return;	1503	queue_work(rbio->bioc->fs_info->endio_raid56_workers,
		1504	&rbio->end_io_work);
		1505	}
1576		1506
1577	if (atomic_read(&rbio->error) > rbio->bioc->max_errors)	1507	/*
1578	goto cleanup;	1508	* End io handler for the read phase of the RMW cycle. All the bios here are
		1509	* physical stripe bios we've read from the disk so we can recalculate the
		1510	* parity of the stripe.
		1511	*
		1512	* This will usually kick off finish_rmw once all the bios are read in, but it
		1513	* may trigger parity reconstruction if we had any errors along the way
		1514	*/
		1515	static void raid56_rmw_end_io_work(struct work_struct *work)
		1516	{
		1517	struct btrfs_raid_bio *rbio =
		1518	container_of(work, struct btrfs_raid_bio, end_io_work);
		1519
		1520	if (atomic_read(&rbio->error) > rbio->bioc->max_errors) {
		1521	rbio_orig_end_io(rbio, BLK_STS_IOERR);
		1522	return;
		1523	}
1579		1524
1580	/*	1525	/*
1581	* this will normally call finish_rmw to start our write	1526	* This will normally call finish_rmw to start our write but if there
1582	* but if there are any failed stripes we'll reconstruct	1527	* are any failed stripes we'll reconstruct from parity first.
1583	* from parity first
1584	*/	1528	*/
1585	validate_rbio_for_rmw(rbio);	1529	validate_rbio_for_rmw(rbio);
1586	return;
1587
1588	cleanup:
1589
1590	rbio_orig_end_io(rbio, BLK_STS_IOERR);
1591	}	1530	}
1592		1531
1593	/*	1532	/*
@@ -1598,9 +1537,9 @@ static int raid56_rmw_stripe(struct btrfs_raid_bio *rbio) · 4 unmodified lines
1598	{	1537	{
1599	int bios_to_read = 0;	1538	int bios_to_read = 0;
1600	struct bio_list bio_list;	1539	struct bio_list bio_list;
		1540	const int nr_data_sectors = rbio->stripe_nsectors * rbio->nr_data;
1601	int ret;	1541	int ret;
1602	int sectornr;	1542	int total_sector_nr;
1603	int stripe;
1604	struct bio *bio;	1543	struct bio *bio;
1605		1544
1606	bio_list_init(&bio_list);	1545	bio_list_init(&bio_list);
@@ -1612,38 +1551,34 @@ static int raid56_rmw_stripe(struct btrfs_raid_bio *rbio) · 5 unmodified lines
1612	index_rbio_pages(rbio);	1551	index_rbio_pages(rbio);
1613		1552
1614	atomic_set(&rbio->error, 0);	1553	atomic_set(&rbio->error, 0);
1615	/*	1554	/* Build a list of bios to read all the missing data sectors. */
1616	* build a list of bios to read all the missing parts of this	1555	for (total_sector_nr = 0; total_sector_nr < nr_data_sectors;
1617	* stripe	1556	total_sector_nr++) {
1618	*/	1557	struct sector_ptr *sector;
1619	for (stripe = 0; stripe < rbio->nr_data; stripe++) {	1558	int stripe = total_sector_nr / rbio->stripe_nsectors;
1620	for (sectornr = 0; sectornr < rbio->stripe_nsectors; sectornr++) {	1559	int sectornr = total_sector_nr % rbio->stripe_nsectors;
1621	struct sector_ptr *sector;
1622		1560
1623	/*	1561	/*
1624	* We want to find all the sectors missing from the	1562	* We want to find all the sectors missing from the rbio and
1625	* rbio and read them from the disk. If * sector_in_rbio()	1563	* read them from the disk. If sector_in_rbio() finds a page
1626	* finds a page in the bio list we don't need to read	1564	* in the bio list we don't need to read it off the stripe.
1627	* it off the stripe.	1565	*/
1628	*/	1566	sector = sector_in_rbio(rbio, stripe, sectornr, 1);
1629	sector = sector_in_rbio(rbio, stripe, sectornr, 1);	1567	if (sector)
1630	if (sector)	1568	continue;
1631	continue;
1632		1569
1633	sector = rbio_stripe_sector(rbio, stripe, sectornr);	1570	sector = rbio_stripe_sector(rbio, stripe, sectornr);
1634	/*	1571	/*
1635	* The bio cache may have handed us an uptodate page.	1572	* The bio cache may have handed us an uptodate page. If so,
1636	* If so, be happy and use it.	1573	* use it.
1637	*/	1574	*/
1638	if (sector->uptodate)	1575	if (sector->uptodate)
1639	continue;	1576	continue;
1640		1577
1641	ret = rbio_add_io_sector(rbio, &bio_list, sector,	1578	ret = rbio_add_io_sector(rbio, &bio_list, sector,
1642	stripe, sectornr, rbio->stripe_len,	1579	stripe, sectornr, REQ_OP_READ);
1643	REQ_OP_READ);	1580	if (ret)
1644	if (ret)	1581	goto cleanup;
1645	goto cleanup;
1646	}
1647	}	1582	}
1648		1583
1649	bios_to_read = bio_list_size(&bio_list);	1584	bios_to_read = bio_list_size(&bio_list);
@@ -1662,11 +1597,16 @@ static int raid56_rmw_stripe(struct btrfs_raid_bio *rbio) · 12 unmodified lines
1662	* touch it after that.	1597	* touch it after that.
1663	*/	1598	*/
1664	atomic_set(&rbio->stripes_pending, bios_to_read);	1599	atomic_set(&rbio->stripes_pending, bios_to_read);
		1600	INIT_WORK(&rbio->end_io_work, raid56_rmw_end_io_work);
1665	while ((bio = bio_list_pop(&bio_list))) {	1601	while ((bio = bio_list_pop(&bio_list))) {
1666	bio->bi_end_io = raid_rmw_end_io;	1602	bio->bi_end_io = raid56_bio_end_io;
1667		1603
1668	btrfs_bio_wq_end_io(rbio->bioc->fs_info, bio, BTRFS_WQ_ENDIO_RAID56);	1604	if (trace_raid56_read_partial_enabled()) {
		1605	struct raid56_bio_trace_info trace_info = { 0 };
1669		1606
		1607	bio_get_trace_info(rbio, bio, &trace_info);
		1608	trace_raid56_read_partial(rbio, bio, &trace_info);
		1609	}
1670	submit_bio(bio);	1610	submit_bio(bio);
1671	}	1611	}
1672	/* the actual write will happen once the reads are done */	1612	/* the actual write will happen once the reads are done */
@@ -1833,27 +1773,53 @@ static void btrfs_raid_unplug(struct blk_plug_cb *cb, bool from_schedule) · 160 unmodified lines
1833	run_plug(plug);	1773	run_plug(plug);
1834	}	1774	}
1835		1775
		1776	/* Add the original bio into rbio->bio_list, and update rbio::dbitmap. */
		1777	static void rbio_add_bio(struct btrfs_raid_bio rbio, struct bio orig_bio)
		1778	{
		1779	const struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
		1780	const u64 orig_logical = orig_bio->bi_iter.bi_sector << SECTOR_SHIFT;
		1781	const u64 full_stripe_start = rbio->bioc->raid_map[0];
		1782	const u32 orig_len = orig_bio->bi_iter.bi_size;
		1783	const u32 sectorsize = fs_info->sectorsize;
		1784	u64 cur_logical;
		1785
		1786	ASSERT(orig_logical >= full_stripe_start &&
		1787	orig_logical + orig_len <= full_stripe_start +
		1788	rbio->nr_data * BTRFS_STRIPE_LEN);
		1789
		1790	bio_list_add(&rbio->bio_list, orig_bio);
		1791	rbio->bio_list_bytes += orig_bio->bi_iter.bi_size;
		1792
		1793	/* Update the dbitmap. */
		1794	for (cur_logical = orig_logical; cur_logical < orig_logical + orig_len;
		1795	cur_logical += sectorsize) {
		1796	int bit = ((u32)(cur_logical - full_stripe_start) >>
		1797	fs_info->sectorsize_bits) % rbio->stripe_nsectors;
		1798
		1799	set_bit(bit, &rbio->dbitmap);
		1800	}
		1801	}
		1802
1836	/*	1803	/*
1837	* our main entry point for writes from the rest of the FS.	1804	* our main entry point for writes from the rest of the FS.
1838	*/	1805	*/
1839	int raid56_parity_write(struct bio bio, struct btrfs_io_context bioc, u32 stripe_len)	1806	void raid56_parity_write(struct bio bio, struct btrfs_io_context bioc)
1840	{	1807	{
1841	struct btrfs_fs_info *fs_info = bioc->fs_info;	1808	struct btrfs_fs_info *fs_info = bioc->fs_info;
1842	struct btrfs_raid_bio *rbio;	1809	struct btrfs_raid_bio *rbio;
1843	struct btrfs_plug_cb *plug = NULL;	1810	struct btrfs_plug_cb *plug = NULL;
1844	struct blk_plug_cb *cb;	1811	struct blk_plug_cb *cb;
1845	int ret;	1812	int ret = 0;
1846		1813
1847	rbio = alloc_rbio(fs_info, bioc, stripe_len);	1814	rbio = alloc_rbio(fs_info, bioc);
1848	if (IS_ERR(rbio)) {	1815	if (IS_ERR(rbio)) {
1849	btrfs_put_bioc(bioc);	1816	btrfs_put_bioc(bioc);
1850	return PTR_ERR(rbio);	1817	ret = PTR_ERR(rbio);
		1818	goto out_dec_counter;
1851	}	1819	}
1852	bio_list_add(&rbio->bio_list, bio);
1853	rbio->bio_list_bytes = bio->bi_iter.bi_size;
1854	rbio->operation = BTRFS_RBIO_WRITE;	1820	rbio->operation = BTRFS_RBIO_WRITE;
		1821	rbio_add_bio(rbio, bio);
1855		1822
1856	btrfs_bio_counter_inc_noblocked(fs_info);
1857	rbio->generic_bio_cnt = 1;	1823	rbio->generic_bio_cnt = 1;
1858		1824
1859	/*	1825	/*
@@ -1863,8 +1829,8 @@ int raid56_parity_write(struct bio bio, struct btrfs_io_context bioc, u32 stri · 3 unmodified lines
1863	if (rbio_is_full(rbio)) {	1829	if (rbio_is_full(rbio)) {
1864	ret = full_stripe_write(rbio);	1830	ret = full_stripe_write(rbio);
1865	if (ret)	1831	if (ret)
1866	btrfs_bio_counter_dec(fs_info);	1832	goto out_dec_counter;
1867	return ret;	1833	return;
1868	}	1834	}
1869		1835
1870	cb = blk_check_plugged(btrfs_raid_unplug, fs_info, sizeof(*plug));	1836	cb = blk_check_plugged(btrfs_raid_unplug, fs_info, sizeof(*plug));
@@ -1875,13 +1841,18 @@ int raid56_parity_write(struct bio bio, struct btrfs_io_context bioc, u32 stri · 4 unmodified lines
1875	INIT_LIST_HEAD(&plug->rbio_list);	1841	INIT_LIST_HEAD(&plug->rbio_list);
1876	}	1842	}
1877	list_add_tail(&rbio->plug_list, &plug->rbio_list);	1843	list_add_tail(&rbio->plug_list, &plug->rbio_list);
1878	ret = 0;
1879	} else {	1844	} else {
1880	ret = __raid56_parity_write(rbio);	1845	ret = __raid56_parity_write(rbio);
1881	if (ret)	1846	if (ret)
1882	btrfs_bio_counter_dec(fs_info);	1847	goto out_dec_counter;
1883	}	1848	}
1884	return ret;	1849
		1850	return;
		1851
		1852	out_dec_counter:
		1853	btrfs_bio_counter_dec(fs_info);
		1854	bio->bi_status = errno_to_blk_status(ret);
		1855	bio_endio(bio);
1885	}	1856	}
1886		1857
1887	/*	1858	/*
@@ -1939,7 +1910,7 @@ static void __raid_recover_end_io(struct btrfs_raid_bio *rbio) · 51 unmodified lines
1939	* which we have data when doing parity scrub.	1910	* which we have data when doing parity scrub.
1940	*/	1911	*/
1941	if (rbio->operation == BTRFS_RBIO_PARITY_SCRUB &&	1912	if (rbio->operation == BTRFS_RBIO_PARITY_SCRUB &&
1942	!test_bit(sectornr, rbio->dbitmap))	1913	!test_bit(sectornr, &rbio->dbitmap))
1943	continue;	1914	continue;
1944		1915
1945	/*	1916	/*
@@ -2108,25 +2079,13 @@ cleanup_io: · 162 unmodified lines
2108	}	2079	}
2109		2080
2110	/*	2081	/*
2111	* This is called only for stripes we've read from disk to	2082	* This is called only for stripes we've read from disk to reconstruct the
2112	* reconstruct the parity.	2083	* parity.
2113	*/	2084	*/
2114	static void raid_recover_end_io(struct bio *bio)	2085	static void raid_recover_end_io_work(struct work_struct *work)
2115	{	2086	{
2116	struct btrfs_raid_bio *rbio = bio->bi_private;	2087	struct btrfs_raid_bio *rbio =
2117		2088	container_of(work, struct btrfs_raid_bio, end_io_work);
2118	/*
2119	* we only read stripe pages off the disk, set them
2120	* up to date if there were no errors
2121	*/
2122	if (bio->bi_status)
2123	fail_bio_stripe(rbio, bio);
2124	else
2125	set_bio_pages_uptodate(rbio, bio);
2126	bio_put(bio);
2127
2128	if (!atomic_dec_and_test(&rbio->stripes_pending))
2129	return;
2130		2089
2131	if (atomic_read(&rbio->error) > rbio->bioc->max_errors)	2090	if (atomic_read(&rbio->error) > rbio->bioc->max_errors)
2132	rbio_orig_end_io(rbio, BLK_STS_IOERR);	2091	rbio_orig_end_io(rbio, BLK_STS_IOERR);
@@ -2147,8 +2106,7 @@ static int __raid56_parity_recover(struct btrfs_raid_bio *rbio) · 14 unmodified lines
2147	int bios_to_read = 0;	2106	int bios_to_read = 0;
2148	struct bio_list bio_list;	2107	struct bio_list bio_list;
2149	int ret;	2108	int ret;
2150	int sectornr;	2109	int total_sector_nr;
2151	int stripe;
2152	struct bio *bio;	2110	struct bio *bio;
2153		2111
2154	bio_list_init(&bio_list);	2112	bio_list_init(&bio_list);
@@ -2160,33 +2118,31 @@ static int __raid56_parity_recover(struct btrfs_raid_bio *rbio) · 5 unmodified lines
2160	atomic_set(&rbio->error, 0);	2118	atomic_set(&rbio->error, 0);
2161		2119
2162	/*	2120	/*
2163	* read everything that hasn't failed. Thanks to the	2121	* Read everything that hasn't failed. However this time we will
2164	* stripe cache, it is possible that some or all of these	2122	* not trust any cached sector.
2165	* pages are going to be uptodate.	2123	* As we may read out some stale data but higher layer is not reading
		2124	* that stale part.
		2125	*
		2126	* So here we always re-read everything in recovery path.
2166	*/	2127	*/
2167	for (stripe = 0; stripe < rbio->real_stripes; stripe++) {	2128	for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors;
		2129	total_sector_nr++) {
		2130	int stripe = total_sector_nr / rbio->stripe_nsectors;
		2131	int sectornr = total_sector_nr % rbio->stripe_nsectors;
		2132	struct sector_ptr *sector;
		2133
2168	if (rbio->faila == stripe \|\| rbio->failb == stripe) {	2134	if (rbio->faila == stripe \|\| rbio->failb == stripe) {
2169	atomic_inc(&rbio->error);	2135	atomic_inc(&rbio->error);
		2136	/* Skip the current stripe. */
		2137	ASSERT(sectornr == 0);
		2138	total_sector_nr += rbio->stripe_nsectors - 1;
2170	continue;	2139	continue;
2171	}	2140	}
2172		2141	sector = rbio_stripe_sector(rbio, stripe, sectornr);
2173	for (sectornr = 0; sectornr < rbio->stripe_nsectors; sectornr++) {	2142	ret = rbio_add_io_sector(rbio, &bio_list, sector, stripe,
2174	struct sector_ptr *sector;	2143	sectornr, REQ_OP_READ);
2175		2144	if (ret < 0)
2176	/*	2145	goto cleanup;
2177	* the rmw code may have already read this
2178	* page in
2179	*/
2180	sector = rbio_stripe_sector(rbio, stripe, sectornr);
2181	if (sector->uptodate)
2182	continue;
2183
2184	ret = rbio_add_io_sector(rbio, &bio_list, sector,
2185	stripe, sectornr, rbio->stripe_len,
2186	REQ_OP_READ);
2187	if (ret < 0)
2188	goto cleanup;
2189	}
2190	}	2146	}
2191		2147
2192	bios_to_read = bio_list_size(&bio_list);	2148	bios_to_read = bio_list_size(&bio_list);
@@ -2209,11 +2165,16 @@ static int __raid56_parity_recover(struct btrfs_raid_bio *rbio) · 16 unmodified lines
2209	* touch it after that.	2165	* touch it after that.
2210	*/	2166	*/
2211	atomic_set(&rbio->stripes_pending, bios_to_read);	2167	atomic_set(&rbio->stripes_pending, bios_to_read);
		2168	INIT_WORK(&rbio->end_io_work, raid_recover_end_io_work);
2212	while ((bio = bio_list_pop(&bio_list))) {	2169	while ((bio = bio_list_pop(&bio_list))) {
2213	bio->bi_end_io = raid_recover_end_io;	2170	bio->bi_end_io = raid56_bio_end_io;
2214		2171
2215	btrfs_bio_wq_end_io(rbio->bioc->fs_info, bio, BTRFS_WQ_ENDIO_RAID56);	2172	if (trace_raid56_scrub_read_recover_enabled()) {
		2173	struct raid56_bio_trace_info trace_info = { 0 };
2216		2174
		2175	bio_get_trace_info(rbio, bio, &trace_info);
		2176	trace_raid56_scrub_read_recover(rbio, bio, &trace_info);
		2177	}
2217	submit_bio(bio);	2178	submit_bio(bio);
2218	}	2179	}
2219		2180
@@ -2236,28 +2197,27 @@ cleanup: · 16 unmodified lines
2236	* so we assume the bio they send down corresponds to a failed part	2197	* so we assume the bio they send down corresponds to a failed part
2237	* of the drive.	2198	* of the drive.
2238	*/	2199	*/
2239	int raid56_parity_recover(struct bio bio, struct btrfs_io_context bioc,	2200	void raid56_parity_recover(struct bio bio, struct btrfs_io_context bioc,
2240	u32 stripe_len, int mirror_num, int generic_io)	2201	int mirror_num, bool generic_io)
2241	{	2202	{
2242	struct btrfs_fs_info *fs_info = bioc->fs_info;	2203	struct btrfs_fs_info *fs_info = bioc->fs_info;
2243	struct btrfs_raid_bio *rbio;	2204	struct btrfs_raid_bio *rbio;
2244	int ret;
2245		2205
2246	if (generic_io) {	2206	if (generic_io) {
2247	ASSERT(bioc->mirror_num == mirror_num);	2207	ASSERT(bioc->mirror_num == mirror_num);
2248	btrfs_bio(bio)->mirror_num = mirror_num;	2208	btrfs_bio(bio)->mirror_num = mirror_num;
		2209	} else {
		2210	btrfs_get_bioc(bioc);
2249	}	2211	}
2250		2212
2251	rbio = alloc_rbio(fs_info, bioc, stripe_len);	2213	rbio = alloc_rbio(fs_info, bioc);
2252	if (IS_ERR(rbio)) {	2214	if (IS_ERR(rbio)) {
2253	if (generic_io)	2215	bio->bi_status = errno_to_blk_status(PTR_ERR(rbio));
2254	btrfs_put_bioc(bioc);	2216	goto out_end_bio;
2255	return PTR_ERR(rbio);
2256	}	2217	}
2257		2218
2258	rbio->operation = BTRFS_RBIO_READ_REBUILD;	2219	rbio->operation = BTRFS_RBIO_READ_REBUILD;
2259	bio_list_add(&rbio->bio_list, bio);	2220	rbio_add_bio(rbio, bio);
2260	rbio->bio_list_bytes = bio->bi_iter.bi_size;
2261		2221
2262	rbio->faila = find_logical_bio_stripe(rbio, bio);	2222	rbio->faila = find_logical_bio_stripe(rbio, bio);
2263	if (rbio->faila == -1) {	2223	if (rbio->faila == -1) {
@@ -2265,18 +2225,13 @@ int raid56_parity_recover(struct bio bio, struct btrfs_io_context bioc, · 1 unmodified line
2265	"%s could not find the bad stripe in raid56 so that we cannot recover any more (bio has logical %llu len %llu, bioc has map_type %llu)",	2225	"%s could not find the bad stripe in raid56 so that we cannot recover any more (bio has logical %llu len %llu, bioc has map_type %llu)",
2266	__func__, bio->bi_iter.bi_sector << 9,	2226	__func__, bio->bi_iter.bi_sector << 9,
2267	(u64)bio->bi_iter.bi_size, bioc->map_type);	2227	(u64)bio->bi_iter.bi_size, bioc->map_type);
2268	if (generic_io)
2269	btrfs_put_bioc(bioc);
2270	kfree(rbio);	2228	kfree(rbio);
2271	return -EIO;	2229	bio->bi_status = BLK_STS_IOERR;
		2230	goto out_end_bio;
2272	}	2231	}
2273		2232
2274	if (generic_io) {	2233	if (generic_io)
2275	btrfs_bio_counter_inc_noblocked(fs_info);
2276	rbio->generic_bio_cnt = 1;	2234	rbio->generic_bio_cnt = 1;
2277	} else {
2278	btrfs_get_bioc(bioc);
2279	}
2280		2235
2281	/*	2236	/*
2282	* Loop retry:	2237	* Loop retry:
@@ -2295,24 +2250,20 @@ int raid56_parity_recover(struct bio bio, struct btrfs_io_context bioc, · 12 unmodified lines
2295	rbio->failb--;	2250	rbio->failb--;
2296	}	2251	}
2297		2252
2298	ret = lock_stripe_add(rbio);	2253	if (lock_stripe_add(rbio))
		2254	return;
2299		2255
2300	/*	2256	/*
2301	* __raid56_parity_recover will end the bio with	2257	* This adds our rbio to the list of rbios that will be handled after
2302	* any errors it hits. We don't want to return	2258	* the current lock owner is done.
2303	* its error value up the stack because our caller
2304	* will end up calling bio_endio with any nonzero
2305	* return
2306	*/
2307	if (ret == 0)
2308	__raid56_parity_recover(rbio);
2309	/*
2310	* our rbio has been added to the list of
2311	* rbios that will be handled after the
2312	* currently lock owner is done
2313	*/	2259	*/
2314	return 0;	2260	__raid56_parity_recover(rbio);
		2261	return;
2315		2262
		2263	out_end_bio:
		2264	btrfs_bio_counter_dec(fs_info);
		2265	btrfs_put_bioc(bioc);
		2266	bio_endio(bio);
2316	}	2267	}
2317		2268
2318	static void rmw_work(struct work_struct *work)	2269	static void rmw_work(struct work_struct *work)
@@ -2343,14 +2294,14 @@ static void read_rebuild_work(struct work_struct *work) · 24 unmodified lines
2343		2294
2344	struct btrfs_raid_bio raid56_parity_alloc_scrub_rbio(struct bio bio,	2295	struct btrfs_raid_bio raid56_parity_alloc_scrub_rbio(struct bio bio,
2345	struct btrfs_io_context *bioc,	2296	struct btrfs_io_context *bioc,
2346	u32 stripe_len, struct btrfs_device *scrub_dev,	2297	struct btrfs_device *scrub_dev,
2347	unsigned long *dbitmap, int stripe_nsectors)	2298	unsigned long *dbitmap, int stripe_nsectors)
2348	{	2299	{
2349	struct btrfs_fs_info *fs_info = bioc->fs_info;	2300	struct btrfs_fs_info *fs_info = bioc->fs_info;
2350	struct btrfs_raid_bio *rbio;	2301	struct btrfs_raid_bio *rbio;
2351	int i;	2302	int i;
2352		2303
2353	rbio = alloc_rbio(fs_info, bioc, stripe_len);	2304	rbio = alloc_rbio(fs_info, bioc);
2354	if (IS_ERR(rbio))	2305	if (IS_ERR(rbio))
2355	return NULL;	2306	return NULL;
2356	bio_list_add(&rbio->bio_list, bio);	2307	bio_list_add(&rbio->bio_list, bio);
@@ -2374,7 +2325,7 @@ struct btrfs_raid_bio raid56_parity_alloc_scrub_rbio(struct bio bio, · 17 unmodified lines
2374	}	2325	}
2375	ASSERT(i < rbio->real_stripes);	2326	ASSERT(i < rbio->real_stripes);
2376		2327
2377	bitmap_copy(rbio->dbitmap, dbitmap, stripe_nsectors);	2328	bitmap_copy(&rbio->dbitmap, dbitmap, stripe_nsectors);
2378		2329
2379	/*	2330	/*
2380	* We have already increased bio_counter when getting bioc, record it	2331	* We have already increased bio_counter when getting bioc, record it
@@ -2395,7 +2346,7 @@ void raid56_add_scrub_pages(struct btrfs_raid_bio rbio, struct page page, · 14 unmodified lines
2395		2346
2396	ASSERT(logical >= rbio->bioc->raid_map[0]);	2347	ASSERT(logical >= rbio->bioc->raid_map[0]);
2397	ASSERT(logical + sectorsize <= rbio->bioc->raid_map[0] +	2348	ASSERT(logical + sectorsize <= rbio->bioc->raid_map[0] +
2398	rbio->stripe_len * rbio->nr_data);	2349	BTRFS_STRIPE_LEN * rbio->nr_data);
2399	stripe_offset = (int)(logical - rbio->bioc->raid_map[0]);	2350	stripe_offset = (int)(logical - rbio->bioc->raid_map[0]);
2400	index = stripe_offset / sectorsize;	2351	index = stripe_offset / sectorsize;
2401	rbio->bio_sectors[index].page = page;	2352	rbio->bio_sectors[index].page = page;
@@ -2409,23 +2360,22 @@ void raid56_add_scrub_pages(struct btrfs_raid_bio rbio, struct page page, · 7 unmodified lines
2409	static int alloc_rbio_essential_pages(struct btrfs_raid_bio *rbio)	2360	static int alloc_rbio_essential_pages(struct btrfs_raid_bio *rbio)
2410	{	2361	{
2411	const u32 sectorsize = rbio->bioc->fs_info->sectorsize;	2362	const u32 sectorsize = rbio->bioc->fs_info->sectorsize;
2412	int stripe;	2363	int total_sector_nr;
2413	int sectornr;
2414
2415	for_each_set_bit(sectornr, rbio->dbitmap, rbio->stripe_nsectors) {
2416	for (stripe = 0; stripe < rbio->real_stripes; stripe++) {
2417	struct page *page;
2418	int index = (stripe * rbio->stripe_nsectors + sectornr) *
2419	sectorsize >> PAGE_SHIFT;
2420		2364
2421	if (rbio->stripe_pages[index])	2365	for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors;
2422	continue;	2366	total_sector_nr++) {
		2367	struct page *page;
		2368	int sectornr = total_sector_nr % rbio->stripe_nsectors;
		2369	int index = (total_sector_nr * sectorsize) >> PAGE_SHIFT;
2423		2370
2424	page = alloc_page(GFP_NOFS);	2371	if (!test_bit(sectornr, &rbio->dbitmap))
2425	if (!page)	2372	continue;
2426	return -ENOMEM;	2373	if (rbio->stripe_pages[index])
2427	rbio->stripe_pages[index] = page;	2374	continue;
2428	}	2375	page = alloc_page(GFP_NOFS);
		2376	if (!page)
		2377	return -ENOMEM;
		2378	rbio->stripe_pages[index] = page;
2429	}	2379	}
2430	index_stripe_sectors(rbio);	2380	index_stripe_sectors(rbio);
2431	return 0;	2381	return 0;
@@ -2437,7 +2387,7 @@ static noinline void finish_parity_scrub(struct btrfs_raid_bio *rbio, · 5 unmodified lines
2437	struct btrfs_io_context *bioc = rbio->bioc;	2387	struct btrfs_io_context *bioc = rbio->bioc;
2438	const u32 sectorsize = bioc->fs_info->sectorsize;	2388	const u32 sectorsize = bioc->fs_info->sectorsize;
2439	void **pointers = rbio->finish_pointers;	2389	void **pointers = rbio->finish_pointers;
2440	unsigned long *pbitmap = rbio->finish_pbitmap;	2390	unsigned long *pbitmap = &rbio->finish_pbitmap;
2441	int nr_data = rbio->nr_data;	2391	int nr_data = rbio->nr_data;
2442	int stripe;	2392	int stripe;
2443	int sectornr;	2393	int sectornr;
@@ -2460,7 +2410,7 @@ static noinline void finish_parity_scrub(struct btrfs_raid_bio *rbio, · 16 unmodified lines
2460		2410
2461	if (bioc->num_tgtdevs && bioc->tgtdev_map[rbio->scrubp]) {	2411	if (bioc->num_tgtdevs && bioc->tgtdev_map[rbio->scrubp]) {
2462	is_replace = 1;	2412	is_replace = 1;
2463	bitmap_copy(pbitmap, rbio->dbitmap, rbio->stripe_nsectors);	2413	bitmap_copy(pbitmap, &rbio->dbitmap, rbio->stripe_nsectors);
2464	}	2414	}
2465		2415
2466	/*	2416	/*
@@ -2497,7 +2447,7 @@ static noinline void finish_parity_scrub(struct btrfs_raid_bio *rbio, · 30 unmodified lines
2497	/* Map the parity stripe just once */	2447	/* Map the parity stripe just once */
2498	pointers[nr_data] = kmap_local_page(p_sector.page);	2448	pointers[nr_data] = kmap_local_page(p_sector.page);
2499		2449
2500	for_each_set_bit(sectornr, rbio->dbitmap, rbio->stripe_nsectors) {	2450	for_each_set_bit(sectornr, &rbio->dbitmap, rbio->stripe_nsectors) {
2501	struct sector_ptr *sector;	2451	struct sector_ptr *sector;
2502	void *parity;	2452	void *parity;
2503		2453
@@ -2525,7 +2475,7 @@ static noinline void finish_parity_scrub(struct btrfs_raid_bio *rbio, · 21 unmodified lines
2525	memcpy(parity, pointers[rbio->scrubp], sectorsize);	2475	memcpy(parity, pointers[rbio->scrubp], sectorsize);
2526	else	2476	else
2527	/* Parity is right, needn't writeback */	2477	/* Parity is right, needn't writeback */
2528	bitmap_clear(rbio->dbitmap, sectornr, 1);	2478	bitmap_clear(&rbio->dbitmap, sectornr, 1);
2529	kunmap_local(parity);	2479	kunmap_local(parity);
2530		2480
2531	for (stripe = nr_data - 1; stripe >= 0; stripe--)	2481	for (stripe = nr_data - 1; stripe >= 0; stripe--)
@@ -2547,12 +2497,12 @@ writeback: · 15 unmodified lines
2547	* higher layers (the bio_list in our rbio) and our p/q. Ignore	2497	* higher layers (the bio_list in our rbio) and our p/q. Ignore
2548	* everything else.	2498	* everything else.
2549	*/	2499	*/
2550	for_each_set_bit(sectornr, rbio->dbitmap, rbio->stripe_nsectors) {	2500	for_each_set_bit(sectornr, &rbio->dbitmap, rbio->stripe_nsectors) {
2551	struct sector_ptr *sector;	2501	struct sector_ptr *sector;
2552		2502
2553	sector = rbio_stripe_sector(rbio, rbio->scrubp, sectornr);	2503	sector = rbio_stripe_sector(rbio, rbio->scrubp, sectornr);
2554	ret = rbio_add_io_sector(rbio, &bio_list, sector, rbio->scrubp,	2504	ret = rbio_add_io_sector(rbio, &bio_list, sector, rbio->scrubp,
2555	sectornr, rbio->stripe_len, REQ_OP_WRITE);	2505	sectornr, REQ_OP_WRITE);
2556	if (ret)	2506	if (ret)
2557	goto cleanup;	2507	goto cleanup;
2558	}	2508	}
@@ -2566,7 +2516,7 @@ writeback: · 7 unmodified lines
2566	sector = rbio_stripe_sector(rbio, rbio->scrubp, sectornr);	2516	sector = rbio_stripe_sector(rbio, rbio->scrubp, sectornr);
2567	ret = rbio_add_io_sector(rbio, &bio_list, sector,	2517	ret = rbio_add_io_sector(rbio, &bio_list, sector,
2568	bioc->tgtdev_map[rbio->scrubp],	2518	bioc->tgtdev_map[rbio->scrubp],
2569	sectornr, rbio->stripe_len, REQ_OP_WRITE);	2519	sectornr, REQ_OP_WRITE);
2570	if (ret)	2520	if (ret)
2571	goto cleanup;	2521	goto cleanup;
2572	}	2522	}
@@ -2584,6 +2534,12 @@ submit_write: · 11 unmodified lines
2584	while ((bio = bio_list_pop(&bio_list))) {	2534	while ((bio = bio_list_pop(&bio_list))) {
2585	bio->bi_end_io = raid_write_end_io;	2535	bio->bi_end_io = raid_write_end_io;
2586		2536
		2537	if (trace_raid56_scrub_write_stripe_enabled()) {
		2538	struct raid56_bio_trace_info trace_info = { 0 };
		2539
		2540	bio_get_trace_info(rbio, bio, &trace_info);
		2541	trace_raid56_scrub_write_stripe(rbio, bio, &trace_info);
		2542	}
2587	submit_bio(bio);	2543	submit_bio(bio);
2588	}	2544	}
2589	return;	2545	return;
@@ -2671,24 +2627,14 @@ cleanup: · 81 unmodified lines
2671	* This will usually kick off finish_rmw once all the bios are read in, but it	2627	* This will usually kick off finish_rmw once all the bios are read in, but it
2672	* may trigger parity reconstruction if we had any errors along the way	2628	* may trigger parity reconstruction if we had any errors along the way
2673	*/	2629	*/
2674	static void raid56_parity_scrub_end_io(struct bio *bio)	2630	static void raid56_parity_scrub_end_io_work(struct work_struct *work)
2675	{	2631	{
2676	struct btrfs_raid_bio *rbio = bio->bi_private;	2632	struct btrfs_raid_bio *rbio =
2677		2633	container_of(work, struct btrfs_raid_bio, end_io_work);
2678	if (bio->bi_status)
2679	fail_bio_stripe(rbio, bio);
2680	else
2681	set_bio_pages_uptodate(rbio, bio);
2682
2683	bio_put(bio);
2684
2685	if (!atomic_dec_and_test(&rbio->stripes_pending))
2686	return;
2687		2634
2688	/*	2635	/*
2689	* this will normally call finish_rmw to start our write	2636	* This will normally call finish_rmw to start our write, but if there
2690	* but if there are any failed stripes we'll reconstruct	2637	* are any failed stripes we'll reconstruct from parity first
2691	* from parity first
2692	*/	2638	*/
2693	validate_rbio_for_parity_scrub(rbio);	2639	validate_rbio_for_parity_scrub(rbio);
2694	}	2640	}
@@ -2698,8 +2644,7 @@ static void raid56_parity_scrub_stripe(struct btrfs_raid_bio *rbio) · 3 unmodified lines
2698	int bios_to_read = 0;	2644	int bios_to_read = 0;
2699	struct bio_list bio_list;	2645	struct bio_list bio_list;
2700	int ret;	2646	int ret;
2701	int sectornr;	2647	int total_sector_nr;
2702	int stripe;
2703	struct bio *bio;	2648	struct bio *bio;
2704		2649
2705	bio_list_init(&bio_list);	2650	bio_list_init(&bio_list);
@@ -2709,37 +2654,38 @@ static void raid56_parity_scrub_stripe(struct btrfs_raid_bio *rbio) · 3 unmodified lines
2709	goto cleanup;	2654	goto cleanup;
2710		2655
2711	atomic_set(&rbio->error, 0);	2656	atomic_set(&rbio->error, 0);
2712	/*	2657	/* Build a list of bios to read all the missing parts. */
2713	* build a list of bios to read all the missing parts of this	2658	for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors;
2714	* stripe	2659	total_sector_nr++) {
2715	*/	2660	int sectornr = total_sector_nr % rbio->stripe_nsectors;
2716	for (stripe = 0; stripe < rbio->real_stripes; stripe++) {	2661	int stripe = total_sector_nr / rbio->stripe_nsectors;
2717	for_each_set_bit(sectornr , rbio->dbitmap, rbio->stripe_nsectors) {	2662	struct sector_ptr *sector;
2718	struct sector_ptr *sector;
2719	/*
2720	* We want to find all the sectors missing from the
2721	* rbio and read them from the disk. If * sector_in_rbio()
2722	* finds a sector in the bio list we don't need to read
2723	* it off the stripe.
2724	*/
2725	sector = sector_in_rbio(rbio, stripe, sectornr, 1);
2726	if (sector)
2727	continue;
2728		2663
2729	sector = rbio_stripe_sector(rbio, stripe, sectornr);	2664	/* No data in the vertical stripe, no need to read. */
2730	/*	2665	if (!test_bit(sectornr, &rbio->dbitmap))
2731	* The bio cache may have handed us an uptodate sector.	2666	continue;
2732	* If so, be happy and use it.
2733	*/
2734	if (sector->uptodate)
2735	continue;
2736		2667
2737	ret = rbio_add_io_sector(rbio, &bio_list, sector,	2668	/*
2738	stripe, sectornr, rbio->stripe_len,	2669	* We want to find all the sectors missing from the rbio and
2739	REQ_OP_READ);	2670	* read them from the disk. If sector_in_rbio() finds a sector
2740	if (ret)	2671	* in the bio list we don't need to read it off the stripe.
2741	goto cleanup;	2672	*/
2742	}	2673	sector = sector_in_rbio(rbio, stripe, sectornr, 1);
		2674	if (sector)
		2675	continue;
		2676
		2677	sector = rbio_stripe_sector(rbio, stripe, sectornr);
		2678	/*
		2679	* The bio cache may have handed us an uptodate sector. If so,
		2680	* use it.
		2681	*/
		2682	if (sector->uptodate)
		2683	continue;
		2684
		2685	ret = rbio_add_io_sector(rbio, &bio_list, sector, stripe,
		2686	sectornr, REQ_OP_READ);
		2687	if (ret)
		2688	goto cleanup;
2743	}	2689	}
2744		2690
2745	bios_to_read = bio_list_size(&bio_list);	2691	bios_to_read = bio_list_size(&bio_list);
@@ -2758,11 +2704,16 @@ static void raid56_parity_scrub_stripe(struct btrfs_raid_bio *rbio) · 12 unmodified lines
2758	* touch it after that.	2704	* touch it after that.
2759	*/	2705	*/
2760	atomic_set(&rbio->stripes_pending, bios_to_read);	2706	atomic_set(&rbio->stripes_pending, bios_to_read);
		2707	INIT_WORK(&rbio->end_io_work, raid56_parity_scrub_end_io_work);
2761	while ((bio = bio_list_pop(&bio_list))) {	2708	while ((bio = bio_list_pop(&bio_list))) {
2762	bio->bi_end_io = raid56_parity_scrub_end_io;	2709	bio->bi_end_io = raid56_bio_end_io;
2763		2710
2764	btrfs_bio_wq_end_io(rbio->bioc->fs_info, bio, BTRFS_WQ_ENDIO_RAID56);	2711	if (trace_raid56_scrub_read_enabled()) {
		2712	struct raid56_bio_trace_info trace_info = { 0 };
2765		2713
		2714	bio_get_trace_info(rbio, bio, &trace_info);
		2715	trace_raid56_scrub_read(rbio, bio, &trace_info);
		2716	}
2766	submit_bio(bio);	2717	submit_bio(bio);
2767	}	2718	}
2768	/* the actual write will happen once the reads are done */	2719	/* the actual write will happen once the reads are done */
@@ -2797,13 +2748,12 @@ void raid56_parity_submit_scrub_rbio(struct btrfs_raid_bio *rbio) · 28 unmodified lines
2797	/* The following code is used for dev replace of a missing RAID 5/6 device. */	2748	/* The following code is used for dev replace of a missing RAID 5/6 device. */
2798		2749
2799	struct btrfs_raid_bio *	2750	struct btrfs_raid_bio *
2800	raid56_alloc_missing_rbio(struct bio bio, struct btrfs_io_context bioc,	2751	raid56_alloc_missing_rbio(struct bio bio, struct btrfs_io_context bioc)
2801	u64 length)
2802	{	2752	{
2803	struct btrfs_fs_info *fs_info = bioc->fs_info;	2753	struct btrfs_fs_info *fs_info = bioc->fs_info;
2804	struct btrfs_raid_bio *rbio;	2754	struct btrfs_raid_bio *rbio;
2805		2755
2806	rbio = alloc_rbio(fs_info, bioc, length);	2756	rbio = alloc_rbio(fs_info, bioc);
2807	if (IS_ERR(rbio))	2757	if (IS_ERR(rbio))
2808	return NULL;	2758	return NULL;
2809		2759

fs/btrfs/raid56.h+151−17

@@ -7,45 +7,179 @@ · 6 unmodified lines
7	#ifndef BTRFS_RAID56_H	7	#ifndef BTRFS_RAID56_H
8	#define BTRFS_RAID56_H	8	#define BTRFS_RAID56_H
9		9
10	static inline int nr_parity_stripes(const struct map_lookup *map)	10	#include <linux/workqueue.h>
11	{	11	#include "volumes.h"
12	if (map->type & BTRFS_BLOCK_GROUP_RAID5)	12
13	return 1;	13	enum btrfs_rbio_ops {
14	else if (map->type & BTRFS_BLOCK_GROUP_RAID6)	14	BTRFS_RBIO_WRITE,
15	return 2;	15	BTRFS_RBIO_READ_REBUILD,
16	else	16	BTRFS_RBIO_PARITY_SCRUB,
17	return 0;	17	BTRFS_RBIO_REBUILD_MISSING,
18	}	18	};
		19
		20	struct btrfs_raid_bio {
		21	struct btrfs_io_context *bioc;
		22
		23	/*
		24	* While we're doing RMW on a stripe we put it into a hash table so we
		25	* can lock the stripe and merge more rbios into it.
		26	*/
		27	struct list_head hash_list;
		28
		29	/* LRU list for the stripe cache */
		30	struct list_head stripe_cache;
		31
		32	/* For scheduling work in the helper threads */
		33	struct work_struct work;
		34
		35	/*
		36	* bio_list and bio_list_lock are used to add more bios into the stripe
		37	* in hopes of avoiding the full RMW
		38	*/
		39	struct bio_list bio_list;
		40	spinlock_t bio_list_lock;
		41
		42	/*
		43	* Also protected by the bio_list_lock, the plug list is used by the
		44	* plugging code to collect partial bios while plugged. The stripe
		45	* locking code also uses it to hand off the stripe lock to the next
		46	* pending IO.
		47	*/
		48	struct list_head plug_list;
		49
		50	/* Flags that tell us if it is safe to merge with this bio. */
		51	unsigned long flags;
		52
		53	/*
		54	* Set if we're doing a parity rebuild for a read from higher up, which
		55	* is handled differently from a parity rebuild as part of RMW.
		56	*/
		57	enum btrfs_rbio_ops operation;
		58
		59	/* How many pages there are for the full stripe including P/Q */
		60	u16 nr_pages;
		61
		62	/* How many sectors there are for the full stripe including P/Q */
		63	u16 nr_sectors;
		64
		65	/* Number of data stripes (no p/q) */
		66	u8 nr_data;
		67
		68	/* Numer of all stripes (including P/Q) */
		69	u8 real_stripes;
		70
		71	/* How many pages there are for each stripe */
		72	u8 stripe_npages;
		73
		74	/* How many sectors there are for each stripe */
		75	u8 stripe_nsectors;
		76
		77	/* First bad stripe, -1 means no corruption */
		78	s8 faila;
		79
		80	/* Second bad stripe (for RAID6 use) */
		81	s8 failb;
		82
		83	/* Stripe number that we're scrubbing */
		84	u8 scrubp;
		85
		86	/*
		87	* Size of all the bios in the bio_list. This helps us decide if the
		88	* rbio maps to a full stripe or not.
		89	*/
		90	int bio_list_bytes;
		91
		92	int generic_bio_cnt;
		93
		94	refcount_t refs;
		95
		96	atomic_t stripes_pending;
		97
		98	atomic_t error;
		99
		100	struct work_struct end_io_work;
		101
		102	/* Bitmap to record which horizontal stripe has data */
		103	unsigned long dbitmap;
		104
		105	/* Allocated with stripe_nsectors-many bits for finish_() calls /
		106	unsigned long finish_pbitmap;
		107
		108	/*
		109	* These are two arrays of pointers. We allocate the rbio big enough
		110	* to hold them both and setup their locations when the rbio is
		111	* allocated.
		112	*/
		113
		114	/*
		115	* Pointers to pages that we allocated for reading/writing stripes
		116	* directly from the disk (including P/Q).
		117	*/
		118	struct page **stripe_pages;
		119
		120	/* Pointers to the sectors in the bio_list, for faster lookup */
		121	struct sector_ptr *bio_sectors;
		122
		123	/*
		124	* For subpage support, we need to map each sector to above
		125	* stripe_pages.
		126	*/
		127	struct sector_ptr *stripe_sectors;
		128
		129	/* Allocated with real_stripes-many pointers for finish_() calls /
		130	void **finish_pointers;
		131	};
		132
		133	/*
		134	* For trace event usage only. Records useful debug info for each bio submitted
		135	* by RAID56 to each physical device.
		136	*
		137	* No matter signed or not, (-1) is always the one indicating we can not grab
		138	* the proper stripe number.
		139	*/
		140	struct raid56_bio_trace_info {
		141	u64 devid;
		142
		143	/* The offset inside the stripe. (<= STRIPE_LEN) */
		144	u32 offset;
		145
		146	/*
		147	* Stripe number.
		148	* 0 is the first data stripe, and nr_data for P stripe,
		149	* nr_data + 1 for Q stripe.
		150	* >= real_stripes for
		151	*/
		152	u8 stripe_nr;
		153	};
19		154
20	static inline int nr_data_stripes(const struct map_lookup *map)	155	static inline int nr_data_stripes(const struct map_lookup *map)
21	{	156	{
22	return map->num_stripes - nr_parity_stripes(map);	157	return map->num_stripes - btrfs_nr_parity_stripes(map->type);
23	}	158	}
		159
24	#define RAID5_P_STRIPE ((u64)-2)	160	#define RAID5_P_STRIPE ((u64)-2)
25	#define RAID6_Q_STRIPE ((u64)-1)	161	#define RAID6_Q_STRIPE ((u64)-1)
26		162
27	#define is_parity_stripe(x) (((x) == RAID5_P_STRIPE) \|\| \	163	#define is_parity_stripe(x) (((x) == RAID5_P_STRIPE) \|\| \
28	((x) == RAID6_Q_STRIPE))	164	((x) == RAID6_Q_STRIPE))
29		165
30	struct btrfs_raid_bio;
31	struct btrfs_device;	166	struct btrfs_device;
32		167
33	int raid56_parity_recover(struct bio bio, struct btrfs_io_context bioc,	168	void raid56_parity_recover(struct bio bio, struct btrfs_io_context bioc,
34	u32 stripe_len, int mirror_num, int generic_io);	169	int mirror_num, bool generic_io);
35	int raid56_parity_write(struct bio bio, struct btrfs_io_context bioc, u32 stripe_len);	170	void raid56_parity_write(struct bio bio, struct btrfs_io_context bioc);
36		171
37	void raid56_add_scrub_pages(struct btrfs_raid_bio rbio, struct page page,	172	void raid56_add_scrub_pages(struct btrfs_raid_bio rbio, struct page page,
38	unsigned int pgoff, u64 logical);	173	unsigned int pgoff, u64 logical);
39		174
40	struct btrfs_raid_bio raid56_parity_alloc_scrub_rbio(struct bio bio,	175	struct btrfs_raid_bio raid56_parity_alloc_scrub_rbio(struct bio bio,
41	struct btrfs_io_context *bioc, u32 stripe_len,	176	struct btrfs_io_context *bioc,
42	struct btrfs_device *scrub_dev,	177	struct btrfs_device *scrub_dev,
43	unsigned long *dbitmap, int stripe_nsectors);	178	unsigned long *dbitmap, int stripe_nsectors);
44	void raid56_parity_submit_scrub_rbio(struct btrfs_raid_bio *rbio);	179	void raid56_parity_submit_scrub_rbio(struct btrfs_raid_bio *rbio);
45		180
46	struct btrfs_raid_bio *	181	struct btrfs_raid_bio *
47	raid56_alloc_missing_rbio(struct bio bio, struct btrfs_io_context bioc,	182	raid56_alloc_missing_rbio(struct bio bio, struct btrfs_io_context bioc);
48	u64 length);
49	void raid56_submit_missing_rbio(struct btrfs_raid_bio *rbio);	183	void raid56_submit_missing_rbio(struct btrfs_raid_bio *rbio);
50		184
51	int btrfs_alloc_stripe_hash_table(struct btrfs_fs_info *info);	185	int btrfs_alloc_stripe_hash_table(struct btrfs_fs_info *info);

fs/btrfs/reflink.c+12−7

 #include "compression.h"
 #include "ctree.h"
 #include "delalloc-space.h"
+#include "disk-io.h"
 #include "reflink.h"
 #include "transaction.h"
 #include "subpage.h"
 	int ret;
 	inode_inc_iversion(inode);
-	if (!no_time_update)
+	if (!no_time_update) {
-		inode->i_mtime = inode->i_ctime = current_time(inode);
+		inode->i_mtime = current_time(inode);
+		inode->i_ctime = inode->i_mtime;
+	}
 	/*
 	 * We round up to the block size at eof when determining which
 	 * extents to clone above, but shouldn't round up the file size.
 	if (comp_type == BTRFS_COMPRESS_NONE) {
 		memcpy_to_page(page, offset_in_page(file_offset), data_start,
 			       datal);
-		flush_dcache_page(page);
 	} else {
 		ret = btrfs_decompress(comp_type, data_start, page,
 				       offset_in_page(file_offset),
 	 *
 	 * So what's in the range [500, 4095] corresponds to zeroes.
 	 */
-	if (datal < block_size) {
+	if (datal < block_size)
 		memzero_page(page, datal, block_size - datal);
-		flush_dcache_page(page);
-	}
 	btrfs_page_set_uptodate(fs_info, page, file_offset, block_size);
 	btrfs_page_clear_checked(fs_info, page, file_offset, block_size);
 static int btrfs_extent_same_range(struct inode *src, u64 loff, u64 len,
 				   struct inode *dst, u64 dst_loff)
 {
-	const u64 bs = BTRFS_I(src)->root->fs_info->sb->s_blocksize;
+	struct btrfs_fs_info *fs_info = BTRFS_I(src)->root->fs_info;
+	const u64 bs = fs_info->sb->s_blocksize;
 	int ret;
 	/*
 	ret = btrfs_clone(src, dst, loff, len, ALIGN(len, bs), dst_loff, 1);
 	btrfs_double_extent_unlock(src, loff, dst, dst_loff, len);
+	btrfs_btree_balance_dirty(fs_info);
 	return ret;
 }
 				round_down(destoff, PAGE_SIZE),
 				round_up(destoff + len, PAGE_SIZE) - 1);
+	btrfs_btree_balance_dirty(fs_info);
 	return ret;
 }

fs/btrfs/scrub.c+26−45

@@ -135,15 +135,13 @@ struct scrub_parity { · 134 unmodified lines
135	struct work_struct work;	135	struct work_struct work;
136		136
137	/* Mark the parity blocks which have data */	137	/* Mark the parity blocks which have data */
138	unsigned long *dbitmap;	138	unsigned long dbitmap;
139		139
140	/*	140	/*
141	* Mark the parity blocks which have data, but errors happen when	141	* Mark the parity blocks which have data, but errors happen when
142	* read data or check data	142	* read data or check data
143	*/	143	*/
144	unsigned long *ebitmap;	144	unsigned long ebitmap;
145
146	unsigned long bitmap[];
147	};	145	};
148		146
149	struct scrub_ctx {	147	struct scrub_ctx {
@@ -1218,7 +1216,6 @@ static inline int scrub_nr_raid_mirrors(struct btrfs_io_context *bioc) · 1068 unmodified lines
1218		1216
1219	static inline void scrub_stripe_index_and_offset(u64 logical, u64 map_type,	1217	static inline void scrub_stripe_index_and_offset(u64 logical, u64 map_type,
1220	u64 *raid_map,	1218	u64 *raid_map,
1221	u64 mapped_length,
1222	int nstripes, int mirror,	1219	int nstripes, int mirror,
1223	int *stripe_index,	1220	int *stripe_index,
1224	u64 *stripe_offset)	1221	u64 *stripe_offset)
@@ -1233,7 +1230,7 @@ static inline void scrub_stripe_index_and_offset(u64 logical, u64 map_type, · 8 unmodified lines
1233	continue;	1230	continue;
1234		1231
1235	if (logical >= raid_map[i] &&	1232	if (logical >= raid_map[i] &&
1236	logical < raid_map[i] + mapped_length)	1233	logical < raid_map[i] + BTRFS_STRIPE_LEN)
1237	break;	1234	break;
1238	}	1235	}
1239		1236
@@ -1337,7 +1334,6 @@ leave_nomem: · 97 unmodified lines
1337	scrub_stripe_index_and_offset(logical,	1334	scrub_stripe_index_and_offset(logical,
1338	bioc->map_type,	1335	bioc->map_type,
1339	bioc->raid_map,	1336	bioc->raid_map,
1340	mapped_length,
1341	bioc->num_stripes -	1337	bioc->num_stripes -
1342	bioc->num_tgtdevs,	1338	bioc->num_tgtdevs,
1343	mirror_index,	1339	mirror_index,
@@ -1380,19 +1376,12 @@ static int scrub_submit_raid56_bio_wait(struct btrfs_fs_info *fs_info, · 36 unmodified lines
1380	struct scrub_sector *sector)	1376	struct scrub_sector *sector)
1381	{	1377	{
1382	DECLARE_COMPLETION_ONSTACK(done);	1378	DECLARE_COMPLETION_ONSTACK(done);
1383	int ret;
1384	int mirror_num;
1385		1379
1386	bio->bi_iter.bi_sector = sector->logical >> 9;	1380	bio->bi_iter.bi_sector = sector->logical >> 9;
1387	bio->bi_private = &done;	1381	bio->bi_private = &done;
1388	bio->bi_end_io = scrub_bio_wait_endio;	1382	bio->bi_end_io = scrub_bio_wait_endio;
1389		1383	raid56_parity_recover(bio, sector->recover->bioc,
1390	mirror_num = sector->sblock->sectors[0]->mirror_num;	1384	sector->sblock->sectors[0]->mirror_num, false);
1391	ret = raid56_parity_recover(bio, sector->recover->bioc,
1392	sector->recover->map_length,
1393	mirror_num, 0);
1394	if (ret)
1395	return ret;
1396		1385
1397	wait_for_completion_io(&done);	1386	wait_for_completion_io(&done);
1398	return blk_status_to_errno(bio->bi_status);	1387	return blk_status_to_errno(bio->bi_status);
@@ -2197,7 +2186,7 @@ static void scrub_missing_raid56_pages(struct scrub_block *sblock) · 798 unmodified lines
2197	bio->bi_private = sblock;	2186	bio->bi_private = sblock;
2198	bio->bi_end_io = scrub_missing_raid56_end_io;	2187	bio->bi_end_io = scrub_missing_raid56_end_io;
2199		2188
2200	rbio = raid56_alloc_missing_rbio(bio, bioc, length);	2189	rbio = raid56_alloc_missing_rbio(bio, bioc);
2201	if (!rbio)	2190	if (!rbio)
2202	goto rbio_out;	2191	goto rbio_out;
2203		2192
@@ -2406,13 +2395,13 @@ static inline void __scrub_mark_bitmap(struct scrub_parity *sparity, · 202 unmodified lines
2406	static inline void scrub_parity_mark_sectors_error(struct scrub_parity *sparity,	2395	static inline void scrub_parity_mark_sectors_error(struct scrub_parity *sparity,
2407	u64 start, u32 len)	2396	u64 start, u32 len)
2408	{	2397	{
2409	__scrub_mark_bitmap(sparity, sparity->ebitmap, start, len);	2398	__scrub_mark_bitmap(sparity, &sparity->ebitmap, start, len);
2410	}	2399	}
2411		2400
2412	static inline void scrub_parity_mark_sectors_data(struct scrub_parity *sparity,	2401	static inline void scrub_parity_mark_sectors_data(struct scrub_parity *sparity,
2413	u64 start, u32 len)	2402	u64 start, u32 len)
2414	{	2403	{
2415	__scrub_mark_bitmap(sparity, sparity->dbitmap, start, len);	2404	__scrub_mark_bitmap(sparity, &sparity->dbitmap, start, len);
2416	}	2405	}
2417		2406
2418	static void scrub_block_complete(struct scrub_block *sblock)	2407	static void scrub_block_complete(struct scrub_block *sblock)
@@ -2763,7 +2752,7 @@ static void scrub_free_parity(struct scrub_parity *sparity) · 344 unmodified lines
2763	struct scrub_sector curr, next;	2752	struct scrub_sector curr, next;
2764	int nbits;	2753	int nbits;
2765		2754
2766	nbits = bitmap_weight(sparity->ebitmap, sparity->nsectors);	2755	nbits = bitmap_weight(&sparity->ebitmap, sparity->nsectors);
2767	if (nbits) {	2756	if (nbits) {
2768	spin_lock(&sctx->stat_lock);	2757	spin_lock(&sctx->stat_lock);
2769	sctx->stat.read_errors += nbits;	2758	sctx->stat.read_errors += nbits;
@@ -2795,8 +2784,8 @@ static void scrub_parity_bio_endio(struct bio *bio) · 25 unmodified lines
2795	struct btrfs_fs_info *fs_info = sparity->sctx->fs_info;	2784	struct btrfs_fs_info *fs_info = sparity->sctx->fs_info;
2796		2785
2797	if (bio->bi_status)	2786	if (bio->bi_status)
2798	bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,	2787	bitmap_or(&sparity->ebitmap, &sparity->ebitmap,
2799	sparity->nsectors);	2788	&sparity->dbitmap, sparity->nsectors);
2800		2789
2801	bio_put(bio);	2790	bio_put(bio);
2802		2791
@@ -2814,8 +2803,8 @@ static void scrub_parity_check_and_repair(struct scrub_parity *sparity) · 11 unmodified lines
2814	u64 length;	2803	u64 length;
2815	int ret;	2804	int ret;
2816		2805
2817	if (!bitmap_andnot(sparity->dbitmap, sparity->dbitmap, sparity->ebitmap,	2806	if (!bitmap_andnot(&sparity->dbitmap, &sparity->dbitmap,
2818	sparity->nsectors))	2807	&sparity->ebitmap, sparity->nsectors))
2819	goto out;	2808	goto out;
2820		2809
2821	length = sparity->logic_end - sparity->logic_start;	2810	length = sparity->logic_end - sparity->logic_start;
@@ -2831,9 +2820,9 @@ static void scrub_parity_check_and_repair(struct scrub_parity *sparity) · 9 unmodified lines
2831	bio->bi_private = sparity;	2820	bio->bi_private = sparity;
2832	bio->bi_end_io = scrub_parity_bio_endio;	2821	bio->bi_end_io = scrub_parity_bio_endio;
2833		2822
2834	rbio = raid56_parity_alloc_scrub_rbio(bio, bioc, length,	2823	rbio = raid56_parity_alloc_scrub_rbio(bio, bioc,
2835	sparity->scrub_dev,	2824	sparity->scrub_dev,
2836	sparity->dbitmap,	2825	&sparity->dbitmap,
2837	sparity->nsectors);	2826	sparity->nsectors);
2838	if (!rbio)	2827	if (!rbio)
2839	goto rbio_out;	2828	goto rbio_out;
@@ -2847,7 +2836,7 @@ rbio_out: · 7 unmodified lines
2847	bioc_out:	2836	bioc_out:
2848	btrfs_bio_counter_dec(fs_info);	2837	btrfs_bio_counter_dec(fs_info);
2849	btrfs_put_bioc(bioc);	2838	btrfs_put_bioc(bioc);
2850	bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,	2839	bitmap_or(&sparity->ebitmap, &sparity->ebitmap, &sparity->dbitmap,
2851	sparity->nsectors);	2840	sparity->nsectors);
2852	spin_lock(&sctx->stat_lock);	2841	spin_lock(&sctx->stat_lock);
2853	sctx->stat.malloc_errors++;	2842	sctx->stat.malloc_errors++;
@@ -2856,11 +2845,6 @@ out: · 2 unmodified lines
2856	scrub_free_parity(sparity);	2845	scrub_free_parity(sparity);
2857	}	2846	}
2858		2847
2859	static inline int scrub_calc_parity_bitmap_len(int nsectors)
2860	{
2861	return DIV_ROUND_UP(nsectors, BITS_PER_LONG) * sizeof(long);
2862	}
2863
2864	static void scrub_parity_get(struct scrub_parity *sparity)	2848	static void scrub_parity_get(struct scrub_parity *sparity)
2865	{	2849	{
2866	refcount_inc(&sparity->refs);	2850	refcount_inc(&sparity->refs);
@@ -3131,7 +3115,6 @@ static noinline_for_stack int scrub_raid56_parity(struct scrub_ctx *sctx, · 264 unmodified lines
3131	int ret;	3115	int ret;
3132	struct scrub_parity *sparity;	3116	struct scrub_parity *sparity;
3133	int nsectors;	3117	int nsectors;
3134	int bitmap_len;
3135		3118
3136	path = btrfs_alloc_path();	3119	path = btrfs_alloc_path();
3137	if (!path) {	3120	if (!path) {
@@ -3145,9 +3128,8 @@ static noinline_for_stack int scrub_raid56_parity(struct scrub_ctx *sctx, · 7 unmodified lines
3145		3128
3146	ASSERT(map->stripe_len <= U32_MAX);	3129	ASSERT(map->stripe_len <= U32_MAX);
3147	nsectors = map->stripe_len >> fs_info->sectorsize_bits;	3130	nsectors = map->stripe_len >> fs_info->sectorsize_bits;
3148	bitmap_len = scrub_calc_parity_bitmap_len(nsectors);	3131	ASSERT(nsectors <= BITS_PER_LONG);
3149	sparity = kzalloc(sizeof(struct scrub_parity) + 2 * bitmap_len,	3132	sparity = kzalloc(sizeof(struct scrub_parity), GFP_NOFS);
3150	GFP_NOFS);
3151	if (!sparity) {	3133	if (!sparity) {
3152	spin_lock(&sctx->stat_lock);	3134	spin_lock(&sctx->stat_lock);
3153	sctx->stat.malloc_errors++;	3135	sctx->stat.malloc_errors++;
@@ -3165,8 +3147,6 @@ static noinline_for_stack int scrub_raid56_parity(struct scrub_ctx *sctx, · 11 unmodified lines
3165	sparity->logic_end = logic_end;	3147	sparity->logic_end = logic_end;
3166	refcount_set(&sparity->refs, 1);	3148	refcount_set(&sparity->refs, 1);
3167	INIT_LIST_HEAD(&sparity->sectors_list);	3149	INIT_LIST_HEAD(&sparity->sectors_list);
3168	sparity->dbitmap = sparity->bitmap;
3169	sparity->ebitmap = (void *)sparity->bitmap + bitmap_len;
3170		3150
3171	ret = 0;	3151	ret = 0;
3172	for (cur_logical = logic_start; cur_logical < logic_end;	3152	for (cur_logical = logic_start; cur_logical < logic_end;
@@ -3429,20 +3409,22 @@ static int scrub_simple_stripe(struct scrub_ctx *sctx, · 256 unmodified lines
3429		3409
3430	static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx,	3410	static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx,
3431	struct btrfs_block_group *bg,	3411	struct btrfs_block_group *bg,
3432	struct map_lookup *map,	3412	struct extent_map *em,
3433	struct btrfs_device *scrub_dev,	3413	struct btrfs_device *scrub_dev,
3434	int stripe_index, u64 dev_extent_len)	3414	int stripe_index)
3435	{	3415	{
3436	struct btrfs_path *path;	3416	struct btrfs_path *path;
3437	struct btrfs_fs_info *fs_info = sctx->fs_info;	3417	struct btrfs_fs_info *fs_info = sctx->fs_info;
3438	struct btrfs_root *root;	3418	struct btrfs_root *root;
3439	struct btrfs_root *csum_root;	3419	struct btrfs_root *csum_root;
3440	struct blk_plug plug;	3420	struct blk_plug plug;
		3421	struct map_lookup *map = em->map_lookup;
3441	const u64 profile = map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK;	3422	const u64 profile = map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK;
3442	const u64 chunk_logical = bg->start;	3423	const u64 chunk_logical = bg->start;
3443	int ret;	3424	int ret;
3444	u64 physical = map->stripes[stripe_index].physical;	3425	u64 physical = map->stripes[stripe_index].physical;
3445	const u64 physical_end = physical + dev_extent_len;	3426	const u64 dev_stripe_len = btrfs_calc_stripe_length(em);
		3427	const u64 physical_end = physical + dev_stripe_len;
3446	u64 logical;	3428	u64 logical;
3447	u64 logic_end;	3429	u64 logic_end;
3448	/* The logical increment after finishing one stripe */	3430	/* The logical increment after finishing one stripe */
@@ -3569,8 +3551,8 @@ next: · 120 unmodified lines
3569	physical += map->stripe_len;	3551	physical += map->stripe_len;
3570	spin_lock(&sctx->stat_lock);	3552	spin_lock(&sctx->stat_lock);
3571	if (stop_loop)	3553	if (stop_loop)
3572	sctx->stat.last_physical = map->stripes[stripe_index].physical +	3554	sctx->stat.last_physical =
3573	dev_extent_len;	3555	map->stripes[stripe_index].physical + dev_stripe_len;
3574	else	3556	else
3575	sctx->stat.last_physical = physical;	3557	sctx->stat.last_physical = physical;
3576	spin_unlock(&sctx->stat_lock);	3558	spin_unlock(&sctx->stat_lock);
@@ -3639,8 +3621,7 @@ static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx, · 62 unmodified lines
3639	for (i = 0; i < map->num_stripes; ++i) {	3621	for (i = 0; i < map->num_stripes; ++i) {
3640	if (map->stripes[i].dev->bdev == scrub_dev->bdev &&	3622	if (map->stripes[i].dev->bdev == scrub_dev->bdev &&
3641	map->stripes[i].physical == dev_offset) {	3623	map->stripes[i].physical == dev_offset) {
3642	ret = scrub_stripe(sctx, bg, map, scrub_dev, i,	3624	ret = scrub_stripe(sctx, bg, em, scrub_dev, i);
3643	dev_extent_len);
3644	if (ret)	3625	if (ret)
3645	goto out;	3626	goto out;
3646	}	3627	}

fs/btrfs/send.c+533−248

@@ -17,6 +17,7 @@ · 16 unmodified lines
17	#include <linux/crc32c.h>	17	#include <linux/crc32c.h>
18		18
19	#include "send.h"	19	#include "send.h"
		20	#include "ctree.h"
20	#include "backref.h"	21	#include "backref.h"
21	#include "locking.h"	22	#include "locking.h"
22	#include "disk-io.h"	23	#include "disk-io.h"
@@ -82,8 +83,12 @@ struct send_ctx { · 59 unmodified lines
82	char *send_buf;	83	char *send_buf;
83	u32 send_size;	84	u32 send_size;
84	u32 send_max_size;	85	u32 send_max_size;
85	u64 total_send_size;	86	/*
86	u64 cmd_send_size[BTRFS_SEND_C_MAX + 1];	87	* Whether BTRFS_SEND_A_DATA attribute was already added to current
		88	* command (since protocol v2, data must be the last attribute).
		89	*/
		90	bool put_data;
		91	struct page **send_buf_pages;
87	u64 flags; /* 'flags' member of btrfs_ioctl_send_args is u64 */	92	u64 flags; /* 'flags' member of btrfs_ioctl_send_args is u64 */
88	/* Protocol version compatibility requested */	93	/* Protocol version compatibility requested */
89	u32 proto;	94	u32 proto;
@@ -113,14 +118,14 @@ struct send_ctx { · 23 unmodified lines
113	*/	118	*/
114	u64 cur_ino;	119	u64 cur_ino;
115	u64 cur_inode_gen;	120	u64 cur_inode_gen;
116	int cur_inode_new;
117	int cur_inode_new_gen;
118	int cur_inode_deleted;
119	u64 cur_inode_size;	121	u64 cur_inode_size;
120	u64 cur_inode_mode;	122	u64 cur_inode_mode;
121	u64 cur_inode_rdev;	123	u64 cur_inode_rdev;
122	u64 cur_inode_last_extent;	124	u64 cur_inode_last_extent;
123	u64 cur_inode_next_write_offset;	125	u64 cur_inode_next_write_offset;
		126	bool cur_inode_new;
		127	bool cur_inode_new_gen;
		128	bool cur_inode_deleted;
124	bool ignore_cur_inode;	129	bool ignore_cur_inode;
125		130
126	u64 send_progress;	131	u64 send_progress;
@@ -235,6 +240,9 @@ struct send_ctx { · 108 unmodified lines
235	* Indexed by the inode number of the directory to be deleted.	240	* Indexed by the inode number of the directory to be deleted.
236	*/	241	*/
237	struct rb_root orphan_dirs;	242	struct rb_root orphan_dirs;
		243
		244	struct rb_root rbtree_new_refs;
		245	struct rb_root rbtree_deleted_refs;
238	};	246	};
239		247
240	struct pending_dir_move {	248	struct pending_dir_move {
@@ -335,8 +343,8 @@ __maybe_unused · 94 unmodified lines
335	static bool proto_cmd_ok(const struct send_ctx *sctx, int cmd)	343	static bool proto_cmd_ok(const struct send_ctx *sctx, int cmd)
336	{	344	{
337	switch (sctx->proto) {	345	switch (sctx->proto) {
338	case 1: return cmd < __BTRFS_SEND_C_MAX_V1;	346	case 1: return cmd <= BTRFS_SEND_C_MAX_V1;
339	case 2: return cmd < __BTRFS_SEND_C_MAX_V2;	347	case 2: return cmd <= BTRFS_SEND_C_MAX_V2;
340	default: return false;	348	default: return false;
341	}	349	}
342	}	350	}
@@ -577,15 +585,10 @@ static int write_buf(struct file filp, const void buf, u32 len, loff_t *off) · 234 unmodified lines
577		585
578	while (pos < len) {	586	while (pos < len) {
579	ret = kernel_write(filp, buf + pos, len - pos, off);	587	ret = kernel_write(filp, buf + pos, len - pos, off);
580	/* TODO handle that correctly */
581	/*if (ret == -ERESTARTSYS) {
582	continue;
583	}*/
584	if (ret < 0)	588	if (ret < 0)
585	return ret;	589	return ret;
586	if (ret == 0) {	590	if (ret == 0)
587	return -EIO;	591	return -EIO;
588	}
589	pos += ret;	592	pos += ret;
590	}	593	}
591		594
@@ -598,6 +601,9 @@ static int tlv_put(struct send_ctx sctx, u16 attr, const void data, int len) · 6 unmodified lines
598	int total_len = sizeof(*hdr) + len;	601	int total_len = sizeof(*hdr) + len;
599	int left = sctx->send_max_size - sctx->send_size;	602	int left = sctx->send_max_size - sctx->send_size;
600		603
		604	if (WARN_ON_ONCE(sctx->put_data))
		605	return -EINVAL;
		606
601	if (unlikely(left < total_len))	607	if (unlikely(left < total_len))
602	return -EOVERFLOW;	608	return -EOVERFLOW;
603		609
@@ -618,6 +624,7 @@ static int tlv_put(struct send_ctx sctx, u16 attr, const void data, int len) · 14 unmodified lines
618	return tlv_put(sctx, attr, &__tmp, sizeof(__tmp)); \	624	return tlv_put(sctx, attr, &__tmp, sizeof(__tmp)); \
619	}	625	}
620		626
		627	TLV_PUT_DEFINE_INT(32)
621	TLV_PUT_DEFINE_INT(64)	628	TLV_PUT_DEFINE_INT(64)
622		629
623	static int tlv_put_string(struct send_ctx *sctx, u16 attr,	630	static int tlv_put_string(struct send_ctx *sctx, u16 attr,
@@ -693,8 +700,7 @@ static int send_header(struct send_ctx *sctx) · 69 unmodified lines
693	struct btrfs_stream_header hdr;	700	struct btrfs_stream_header hdr;
694		701
695	strcpy(hdr.magic, BTRFS_SEND_STREAM_MAGIC);	702	strcpy(hdr.magic, BTRFS_SEND_STREAM_MAGIC);
696	hdr.version = cpu_to_le32(BTRFS_SEND_STREAM_VERSION);	703	hdr.version = cpu_to_le32(sctx->proto);
697
698	return write_buf(sctx->send_filp, &hdr, sizeof(hdr),	704	return write_buf(sctx->send_filp, &hdr, sizeof(hdr),
699	&sctx->send_off);	705	&sctx->send_off);
700	}	706	}
@@ -734,9 +740,8 @@ static int send_cmd(struct send_ctx *sctx) · 33 unmodified lines
734	ret = write_buf(sctx->send_filp, sctx->send_buf, sctx->send_size,	740	ret = write_buf(sctx->send_filp, sctx->send_buf, sctx->send_size,
735	&sctx->send_off);	741	&sctx->send_off);
736		742
737	sctx->total_send_size += sctx->send_size;
738	sctx->cmd_send_size[get_unaligned_le16(&hdr->cmd)] += sctx->send_size;
739	sctx->send_size = 0;	743	sctx->send_size = 0;
		744	sctx->put_data = false;
740		745
741	return ret;	746	return ret;
742	}	747	}
@@ -842,7 +847,7 @@ out: · 99 unmodified lines
842	*/	847	*/
843	static int __get_inode_info(struct btrfs_root root, struct btrfs_path path,	848	static int __get_inode_info(struct btrfs_root root, struct btrfs_path path,
844	u64 ino, u64 size, u64 gen, u64 mode, u64 uid,	849	u64 ino, u64 size, u64 gen, u64 mode, u64 uid,
845	u64 gid, u64 rdev)	850	u64 gid, u64 rdev, u64 *fileattr)
846	{	851	{
847	int ret;	852	int ret;
848	struct btrfs_inode_item *ii;	853	struct btrfs_inode_item *ii;
@@ -872,6 +877,12 @@ static int __get_inode_info(struct btrfs_root root, struct btrfs_path path, · 23 unmodified lines
872	*gid = btrfs_inode_gid(path->nodes[0], ii);	877	*gid = btrfs_inode_gid(path->nodes[0], ii);
873	if (rdev)	878	if (rdev)
874	*rdev = btrfs_inode_rdev(path->nodes[0], ii);	879	*rdev = btrfs_inode_rdev(path->nodes[0], ii);
		880	/*
		881	* Transfer the unchanged u64 value of btrfs_inode_item::flags, that's
		882	* otherwise logically split to 32/32 parts.
		883	*/
		884	if (fileattr)
		885	*fileattr = btrfs_inode_flags(path->nodes[0], ii);
875		886
876	return ret;	887	return ret;
877	}	888	}
@@ -879,7 +890,7 @@ static int __get_inode_info(struct btrfs_root root, struct btrfs_path path, · 1 unmodified line
879	static int get_inode_info(struct btrfs_root *root,	890	static int get_inode_info(struct btrfs_root *root,
880	u64 ino, u64 size, u64 gen,	891	u64 ino, u64 size, u64 gen,
881	u64 mode, u64 uid, u64 *gid,	892	u64 mode, u64 uid, u64 *gid,
882	u64 *rdev)	893	u64 rdev, u64 fileattr)
883	{	894	{
884	struct btrfs_path *path;	895	struct btrfs_path *path;
885	int ret;	896	int ret;
@@ -888,7 +899,7 @@ static int get_inode_info(struct btrfs_root *root, · 2 unmodified lines
888	if (!path)	899	if (!path)
889	return -ENOMEM;	900	return -ENOMEM;
890	ret = __get_inode_info(root, path, ino, size, gen, mode, uid, gid,	901	ret = __get_inode_info(root, path, ino, size, gen, mode, uid, gid,
891	rdev);	902	rdev, fileattr);
892	btrfs_free_path(path);	903	btrfs_free_path(path);
893	return ret;	904	return ret;
894	}	905	}
@@ -1634,7 +1645,7 @@ static int get_cur_inode_state(struct send_ctx *sctx, u64 ino, u64 gen) · 739 unmodified lines
1634	u64 right_gen;	1645	u64 right_gen;
1635		1646
1636	ret = get_inode_info(sctx->send_root, ino, NULL, &left_gen, NULL, NULL,	1647	ret = get_inode_info(sctx->send_root, ino, NULL, &left_gen, NULL, NULL,
1637	NULL, NULL);	1648	NULL, NULL, NULL);
1638	if (ret < 0 && ret != -ENOENT)	1649	if (ret < 0 && ret != -ENOENT)
1639	goto out;	1650	goto out;
1640	left_ret = ret;	1651	left_ret = ret;
@@ -1643,7 +1654,7 @@ static int get_cur_inode_state(struct send_ctx *sctx, u64 ino, u64 gen) · 2 unmodified lines
1643	right_ret = -ENOENT;	1654	right_ret = -ENOENT;
1644	} else {	1655	} else {
1645	ret = get_inode_info(sctx->parent_root, ino, NULL, &right_gen,	1656	ret = get_inode_info(sctx->parent_root, ino, NULL, &right_gen,
1646	NULL, NULL, NULL, NULL);	1657	NULL, NULL, NULL, NULL, NULL);
1647	if (ret < 0 && ret != -ENOENT)	1658	if (ret < 0 && ret != -ENOENT)
1648	goto out;	1659	goto out;
1649	right_ret = ret;	1660	right_ret = ret;
@@ -1806,7 +1817,7 @@ static int get_first_ref(struct btrfs_root *root, u64 ino, · 156 unmodified lines
1806		1817
1807	if (dir_gen) {	1818	if (dir_gen) {
1808	ret = get_inode_info(root, parent_dir, NULL, dir_gen, NULL,	1819	ret = get_inode_info(root, parent_dir, NULL, dir_gen, NULL,
1809	NULL, NULL, NULL);	1820	NULL, NULL, NULL, NULL);
1810	if (ret < 0)	1821	if (ret < 0)
1811	goto out;	1822	goto out;
1812	}	1823	}
@@ -1878,7 +1889,7 @@ static int will_overwrite_ref(struct send_ctx *sctx, u64 dir, u64 dir_gen, · 65 unmodified lines
1878	*/	1889	*/
1879	if (sctx->parent_root && dir != BTRFS_FIRST_FREE_OBJECTID) {	1890	if (sctx->parent_root && dir != BTRFS_FIRST_FREE_OBJECTID) {
1880	ret = get_inode_info(sctx->parent_root, dir, NULL, &gen, NULL,	1891	ret = get_inode_info(sctx->parent_root, dir, NULL, &gen, NULL,
1881	NULL, NULL, NULL);	1892	NULL, NULL, NULL, NULL);
1882	if (ret < 0 && ret != -ENOENT)	1893	if (ret < 0 && ret != -ENOENT)
1883	goto out;	1894	goto out;
1884	if (ret) {	1895	if (ret) {
@@ -1906,7 +1917,7 @@ static int will_overwrite_ref(struct send_ctx *sctx, u64 dir, u64 dir_gen, · 21 unmodified lines
1906	if (other_inode > sctx->send_progress \|\|	1917	if (other_inode > sctx->send_progress \|\|
1907	is_waiting_for_move(sctx, other_inode)) {	1918	is_waiting_for_move(sctx, other_inode)) {
1908	ret = get_inode_info(sctx->parent_root, other_inode, NULL,	1919	ret = get_inode_info(sctx->parent_root, other_inode, NULL,
1909	who_gen, who_mode, NULL, NULL, NULL);	1920	who_gen, who_mode, NULL, NULL, NULL, NULL);
1910	if (ret < 0)	1921	if (ret < 0)
1911	goto out;	1922	goto out;
1912		1923
@@ -1945,7 +1956,7 @@ static int did_overwrite_ref(struct send_ctx *sctx, · 32 unmodified lines
1945		1956
1946	if (dir != BTRFS_FIRST_FREE_OBJECTID) {	1957	if (dir != BTRFS_FIRST_FREE_OBJECTID) {
1947	ret = get_inode_info(sctx->send_root, dir, NULL, &gen, NULL,	1958	ret = get_inode_info(sctx->send_root, dir, NULL, &gen, NULL,
1948	NULL, NULL, NULL);	1959	NULL, NULL, NULL, NULL);
1949	if (ret < 0 && ret != -ENOENT)	1960	if (ret < 0 && ret != -ENOENT)
1950	goto out;	1961	goto out;
1951	if (ret) {	1962	if (ret) {
@@ -1968,7 +1979,7 @@ static int did_overwrite_ref(struct send_ctx *sctx, · 16 unmodified lines
1968	}	1979	}
1969		1980
1970	ret = get_inode_info(sctx->send_root, ow_inode, NULL, &gen, NULL, NULL,	1981	ret = get_inode_info(sctx->send_root, ow_inode, NULL, &gen, NULL, NULL,
1971	NULL, NULL);	1982	NULL, NULL, NULL);
1972	if (ret < 0)	1983	if (ret < 0)
1973	goto out;	1984	goto out;
1974		1985
@@ -2184,7 +2195,7 @@ static int __get_cur_name_and_parent(struct send_ctx *sctx, · 209 unmodified lines
2184	/*	2195	/*
2185	* If the inode is not existent yet, add the orphan name and return 1.	2196	* If the inode is not existent yet, add the orphan name and return 1.
2186	* This should only happen for the parent dir that we determine in	2197	* This should only happen for the parent dir that we determine in
2187	* __record_new_ref	2198	* record_new_ref_if_needed().
2188	*/	2199	*/
2189	ret = is_inode_existent(sctx, ino, gen);	2200	ret = is_inode_existent(sctx, ino, gen);
2190	if (ret < 0)	2201	if (ret < 0)
@@ -2499,6 +2510,39 @@ out: · 308 unmodified lines
2499	return ret;	2510	return ret;
2500	}	2511	}
2501		2512
		2513	static int send_fileattr(struct send_ctx *sctx, u64 ino, u64 gen, u64 fileattr)
		2514	{
		2515	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
		2516	int ret = 0;
		2517	struct fs_path *p;
		2518
		2519	if (sctx->proto < 2)
		2520	return 0;
		2521
		2522	btrfs_debug(fs_info, "send_fileattr %llu fileattr=%llu", ino, fileattr);
		2523
		2524	p = fs_path_alloc();
		2525	if (!p)
		2526	return -ENOMEM;
		2527
		2528	ret = begin_cmd(sctx, BTRFS_SEND_C_FILEATTR);
		2529	if (ret < 0)
		2530	goto out;
		2531
		2532	ret = get_cur_path(sctx, ino, gen, p);
		2533	if (ret < 0)
		2534	goto out;
		2535	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
		2536	TLV_PUT_U64(sctx, BTRFS_SEND_A_FILEATTR, fileattr);
		2537
		2538	ret = send_cmd(sctx);
		2539
		2540	tlv_put_failure:
		2541	out:
		2542	fs_path_free(p);
		2543	return ret;
		2544	}
		2545
2502	static int send_chown(struct send_ctx *sctx, u64 ino, u64 gen, u64 uid, u64 gid)	2546	static int send_chown(struct send_ctx *sctx, u64 ino, u64 gen, u64 uid, u64 gid)
2503	{	2547	{
2504	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;	2548	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
@@ -2578,7 +2622,8 @@ static int send_utimes(struct send_ctx *sctx, u64 ino, u64 gen) · 73 unmodified lines
2578	TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_ATIME, eb, &ii->atime);	2622	TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_ATIME, eb, &ii->atime);
2579	TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_MTIME, eb, &ii->mtime);	2623	TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_MTIME, eb, &ii->mtime);
2580	TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_CTIME, eb, &ii->ctime);	2624	TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_CTIME, eb, &ii->ctime);
2581	/* TODO Add otime support when the otime patches get into upstream */	2625	if (sctx->proto >= 2)
		2626	TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_OTIME, eb, &ii->otime);
2582		2627
2583	ret = send_cmd(sctx);	2628	ret = send_cmd(sctx);
2584		2629
@@ -2612,7 +2657,7 @@ static int send_create_inode(struct send_ctx *sctx, u64 ino) · 27 unmodified lines
2612		2657
2613	if (ino != sctx->cur_ino) {	2658	if (ino != sctx->cur_ino) {
2614	ret = get_inode_info(sctx->send_root, ino, NULL, &gen, &mode,	2659	ret = get_inode_info(sctx->send_root, ino, NULL, &gen, &mode,
2615	NULL, NULL, &rdev);	2660	NULL, NULL, &rdev, NULL);
2616	if (ret < 0)	2661	if (ret < 0)
2617	goto out;	2662	goto out;
2618	} else {	2663	} else {
@@ -2751,48 +2796,50 @@ struct recorded_ref { · 132 unmodified lines
2751	u64 dir;	2796	u64 dir;
2752	u64 dir_gen;	2797	u64 dir_gen;
2753	int name_len;	2798	int name_len;
		2799	struct rb_node node;
		2800	struct rb_root *root;
2754	};	2801	};
2755		2802
2756	static void set_ref_path(struct recorded_ref ref, struct fs_path path)	2803	static struct recorded_ref *recorded_ref_alloc(void)
2757	{	2804	{
2758	ref->full_path = path;	2805	struct recorded_ref *ref;
2759	ref->name = (char *)kbasename(ref->full_path->start);	2806
2760	ref->name_len = ref->full_path->end - ref->name;	2807	ref = kzalloc(sizeof(*ref), GFP_KERNEL);
		2808	if (!ref)
		2809	return NULL;
		2810	RB_CLEAR_NODE(&ref->node);
		2811	INIT_LIST_HEAD(&ref->list);
		2812	return ref;
2761	}	2813	}
2762		2814
2763	/*	2815	static void recorded_ref_free(struct recorded_ref *ref)
2764	* We need to process new refs before deleted refs, but compare_tree gives us
2765	* everything mixed. So we first record all refs and later process them.
2766	* This function is a helper to record one ref.
2767	*/
2768	static int __record_ref(struct list_head *head, u64 dir,
2769	u64 dir_gen, struct fs_path *path)
2770	{	2816	{
2771	struct recorded_ref *ref;
2772
2773	ref = kmalloc(sizeof(*ref), GFP_KERNEL);
2774	if (!ref)	2817	if (!ref)
2775	return -ENOMEM;	2818	return;
		2819	if (!RB_EMPTY_NODE(&ref->node))
		2820	rb_erase(&ref->node, ref->root);
		2821	list_del(&ref->list);
		2822	fs_path_free(ref->full_path);
		2823	kfree(ref);
		2824	}
2776		2825
2777	ref->dir = dir;	2826	static void set_ref_path(struct recorded_ref ref, struct fs_path path)
2778	ref->dir_gen = dir_gen;	2827	{
2779	set_ref_path(ref, path);	2828	ref->full_path = path;
2780	list_add_tail(&ref->list, head);	2829	ref->name = (char *)kbasename(ref->full_path->start);
2781	return 0;	2830	ref->name_len = ref->full_path->end - ref->name;
2782	}	2831	}
2783		2832
2784	static int dup_ref(struct recorded_ref ref, struct list_head list)	2833	static int dup_ref(struct recorded_ref ref, struct list_head list)
2785	{	2834	{
2786	struct recorded_ref *new;	2835	struct recorded_ref *new;
2787		2836
2788	new = kmalloc(sizeof(*ref), GFP_KERNEL);	2837	new = recorded_ref_alloc();
2789	if (!new)	2838	if (!new)
2790	return -ENOMEM;	2839	return -ENOMEM;
2791		2840
2792	new->dir = ref->dir;	2841	new->dir = ref->dir;
2793	new->dir_gen = ref->dir_gen;	2842	new->dir_gen = ref->dir_gen;
2794	new->full_path = NULL;
2795	INIT_LIST_HEAD(&new->list);
2796	list_add_tail(&new->list, list);	2843	list_add_tail(&new->list, list);
2797	return 0;	2844	return 0;
2798	}	2845	}
@@ -2803,9 +2850,7 @@ static void __free_recorded_refs(struct list_head *head) · 4 unmodified lines
2803		2850
2804	while (!list_empty(head)) {	2851	while (!list_empty(head)) {
2805	cur = list_entry(head->next, struct recorded_ref, list);	2852	cur = list_entry(head->next, struct recorded_ref, list);
2806	fs_path_free(cur->full_path);	2853	recorded_ref_free(cur);
2807	list_del(&cur->list);
2808	kfree(cur);
2809	}	2854	}
2810	}	2855	}
2811		2856
@@ -3315,7 +3360,7 @@ finish: · 503 unmodified lines
3315	* The parent inode might have been deleted in the send snapshot	3360	* The parent inode might have been deleted in the send snapshot
3316	*/	3361	*/
3317	ret = get_inode_info(sctx->send_root, cur->dir, NULL,	3362	ret = get_inode_info(sctx->send_root, cur->dir, NULL,
3318	NULL, NULL, NULL, NULL, NULL);	3363	NULL, NULL, NULL, NULL, NULL, NULL);
3319	if (ret == -ENOENT) {	3364	if (ret == -ENOENT) {
3320	ret = 0;	3365	ret = 0;
3321	continue;	3366	continue;
@@ -3490,11 +3535,11 @@ static int wait_for_dest_dir_move(struct send_ctx *sctx, · 168 unmodified lines
3490	}	3535	}
3491		3536
3492	ret = get_inode_info(sctx->parent_root, di_key.objectid, NULL,	3537	ret = get_inode_info(sctx->parent_root, di_key.objectid, NULL,
3493	&left_gen, NULL, NULL, NULL, NULL);	3538	&left_gen, NULL, NULL, NULL, NULL, NULL);
3494	if (ret < 0)	3539	if (ret < 0)
3495	goto out;	3540	goto out;
3496	ret = get_inode_info(sctx->send_root, di_key.objectid, NULL,	3541	ret = get_inode_info(sctx->send_root, di_key.objectid, NULL,
3497	&right_gen, NULL, NULL, NULL, NULL);	3542	&right_gen, NULL, NULL, NULL, NULL, NULL);
3498	if (ret < 0) {	3543	if (ret < 0) {
3499	if (ret == -ENOENT)	3544	if (ret == -ENOENT)
3500	ret = 0;	3545	ret = 0;
@@ -3625,7 +3670,7 @@ static int is_ancestor(struct btrfs_root *root, · 124 unmodified lines
3625	}	3670	}
3626		3671
3627	ret = get_inode_info(root, parent, NULL, &parent_gen,	3672	ret = get_inode_info(root, parent, NULL, &parent_gen,
3628	NULL, NULL, NULL, NULL);	3673	NULL, NULL, NULL, NULL, NULL);
3629	if (ret < 0)	3674	if (ret < 0)
3630	goto out;	3675	goto out;
3631	ret = check_ino_in_path(root, ino1, ino1_gen,	3676	ret = check_ino_in_path(root, ino1, ino1_gen,
@@ -3717,7 +3762,7 @@ static int wait_for_parent_move(struct send_ctx *sctx, · 85 unmodified lines
3717		3762
3718	ret = get_inode_info(sctx->parent_root, ino, NULL,	3763	ret = get_inode_info(sctx->parent_root, ino, NULL,
3719	&parent_ino_gen, NULL, NULL, NULL,	3764	&parent_ino_gen, NULL, NULL, NULL,
3720	NULL);	3765	NULL, NULL);
3721	if (ret < 0)	3766	if (ret < 0)
3722	goto out;	3767	goto out;
3723	if (ino_gen == parent_ino_gen) {	3768	if (ino_gen == parent_ino_gen) {
@@ -4311,185 +4356,169 @@ out: · 587 unmodified lines
4311	return ret;	4356	return ret;
4312	}	4357	}
4313		4358
4314	static int record_ref(struct btrfs_root root, u64 dir, struct fs_path name,	4359	static int rbtree_ref_comp(const void k, const struct rb_node node)
4315	void ctx, struct list_head refs)	4360	{
		4361	const struct recorded_ref *data = k;
		4362	const struct recorded_ref *ref = rb_entry(node, struct recorded_ref, node);
		4363	int result;
		4364
		4365	if (data->dir > ref->dir)
		4366	return 1;
		4367	if (data->dir < ref->dir)
		4368	return -1;
		4369	if (data->dir_gen > ref->dir_gen)
		4370	return 1;
		4371	if (data->dir_gen < ref->dir_gen)
		4372	return -1;
		4373	if (data->name_len > ref->name_len)
		4374	return 1;
		4375	if (data->name_len < ref->name_len)
		4376	return -1;
		4377	result = strcmp(data->name, ref->name);
		4378	if (result > 0)
		4379	return 1;
		4380	if (result < 0)
		4381	return -1;
		4382	return 0;
		4383	}
		4384
		4385	static bool rbtree_ref_less(struct rb_node node, const struct rb_node parent)
		4386	{
		4387	const struct recorded_ref *entry = rb_entry(node, struct recorded_ref, node);
		4388
		4389	return rbtree_ref_comp(entry, parent) < 0;
		4390	}
		4391
		4392	static int record_ref_in_tree(struct rb_root root, struct list_head refs,
		4393	struct fs_path *name, u64 dir, u64 dir_gen,
		4394	struct send_ctx *sctx)
4316	{	4395	{
4317	int ret = 0;	4396	int ret = 0;
4318	struct send_ctx *sctx = ctx;	4397	struct fs_path *path = NULL;
4319	struct fs_path *p;	4398	struct recorded_ref *ref = NULL;
4320	u64 gen;
4321		4399
4322	p = fs_path_alloc();	4400	path = fs_path_alloc();
4323	if (!p)	4401	if (!path) {
4324	return -ENOMEM;	4402	ret = -ENOMEM;
		4403	goto out;
		4404	}
4325		4405
4326	ret = get_inode_info(root, dir, NULL, &gen, NULL, NULL,	4406	ref = recorded_ref_alloc();
4327	NULL, NULL);	4407	if (!ref) {
4328	if (ret < 0)	4408	ret = -ENOMEM;
4329	goto out;	4409	goto out;
		4410	}
4330		4411
4331	ret = get_cur_path(sctx, dir, gen, p);	4412	ret = get_cur_path(sctx, dir, dir_gen, path);
4332	if (ret < 0)	4413	if (ret < 0)
4333	goto out;	4414	goto out;
4334	ret = fs_path_add_path(p, name);	4415	ret = fs_path_add_path(path, name);
4335	if (ret < 0)	4416	if (ret < 0)
4336	goto out;	4417	goto out;
4337		4418
4338	ret = __record_ref(refs, dir, gen, p);	4419	ref->dir = dir;
4339		4420	ref->dir_gen = dir_gen;
		4421	set_ref_path(ref, path);
		4422	list_add_tail(&ref->list, refs);
		4423	rb_add(&ref->node, root, rbtree_ref_less);
		4424	ref->root = root;
4340	out:	4425	out:
4341	if (ret)	4426	if (ret) {
4342	fs_path_free(p);	4427	if (path && (!ref \|\| !ref->full_path))
		4428	fs_path_free(path);
		4429	recorded_ref_free(ref);
		4430	}
4343	return ret;	4431	return ret;
4344	}	4432	}
4345		4433
4346	static int __record_new_ref(int num, u64 dir, int index,	4434	static int record_new_ref_if_needed(int num, u64 dir, int index,
4347	struct fs_path *name,	4435	struct fs_path name, void ctx)
4348	void *ctx)
4349	{
4350	struct send_ctx *sctx = ctx;
4351	return record_ref(sctx->send_root, dir, name, ctx, &sctx->new_refs);
4352	}
4353
4354
4355	static int __record_deleted_ref(int num, u64 dir, int index,
4356	struct fs_path *name,
4357	void *ctx)
4358	{	4436	{
		4437	int ret = 0;
4359	struct send_ctx *sctx = ctx;	4438	struct send_ctx *sctx = ctx;
4360	return record_ref(sctx->parent_root, dir, name, ctx,	4439	struct rb_node *node = NULL;
4361	&sctx->deleted_refs);	4440	struct recorded_ref data;
4362	}	4441	struct recorded_ref *ref;
4363		4442	u64 dir_gen;
4364	static int record_new_ref(struct send_ctx *sctx)
4365	{
4366	int ret;
4367		4443
4368	ret = iterate_inode_ref(sctx->send_root, sctx->left_path,	4444	ret = get_inode_info(sctx->send_root, dir, NULL, &dir_gen, NULL,
4369	sctx->cmp_key, 0, __record_new_ref, sctx);	4445	NULL, NULL, NULL, NULL);
4370	if (ret < 0)	4446	if (ret < 0)
4371	goto out;	4447	goto out;
4372	ret = 0;
4373		4448
		4449	data.dir = dir;
		4450	data.dir_gen = dir_gen;
		4451	set_ref_path(&data, name);
		4452	node = rb_find(&data, &sctx->rbtree_deleted_refs, rbtree_ref_comp);
		4453	if (node) {
		4454	ref = rb_entry(node, struct recorded_ref, node);
		4455	recorded_ref_free(ref);
		4456	} else {
		4457	ret = record_ref_in_tree(&sctx->rbtree_new_refs,
		4458	&sctx->new_refs, name, dir, dir_gen,
		4459	sctx);
		4460	}
4374	out:	4461	out:
4375	return ret;	4462	return ret;
4376	}	4463	}
4377		4464
4378	static int record_deleted_ref(struct send_ctx *sctx)	4465	static int record_deleted_ref_if_needed(int num, u64 dir, int index,
		4466	struct fs_path name, void ctx)
4379	{	4467	{
4380	int ret;	4468	int ret = 0;
		4469	struct send_ctx *sctx = ctx;
		4470	struct rb_node *node = NULL;
		4471	struct recorded_ref data;
		4472	struct recorded_ref *ref;
		4473	u64 dir_gen;
4381		4474
4382	ret = iterate_inode_ref(sctx->parent_root, sctx->right_path,	4475	ret = get_inode_info(sctx->parent_root, dir, NULL, &dir_gen, NULL,
4383	sctx->cmp_key, 0, __record_deleted_ref, sctx);	4476	NULL, NULL, NULL, NULL);
4384	if (ret < 0)	4477	if (ret < 0)
4385	goto out;	4478	goto out;
4386	ret = 0;
4387		4479
		4480	data.dir = dir;
		4481	data.dir_gen = dir_gen;
		4482	set_ref_path(&data, name);
		4483	node = rb_find(&data, &sctx->rbtree_new_refs, rbtree_ref_comp);
		4484	if (node) {
		4485	ref = rb_entry(node, struct recorded_ref, node);
		4486	recorded_ref_free(ref);
		4487	} else {
		4488	ret = record_ref_in_tree(&sctx->rbtree_deleted_refs,
		4489	&sctx->deleted_refs, name, dir,
		4490	dir_gen, sctx);
		4491	}
4388	out:	4492	out:
4389	return ret;	4493	return ret;
4390	}	4494	}
4391		4495
4392	struct find_ref_ctx {	4496	static int record_new_ref(struct send_ctx *sctx)
4393	u64 dir;
4394	u64 dir_gen;
4395	struct btrfs_root *root;
4396	struct fs_path *name;
4397	int found_idx;
4398	};
4399
4400	static int __find_iref(int num, u64 dir, int index,
4401	struct fs_path *name,
4402	void *ctx_)
4403	{
4404	struct find_ref_ctx *ctx = ctx_;
4405	u64 dir_gen;
4406	int ret;
4407
4408	if (dir == ctx->dir && fs_path_len(name) == fs_path_len(ctx->name) &&
4409	strncmp(name->start, ctx->name->start, fs_path_len(name)) == 0) {
4410	/*
4411	* To avoid doing extra lookups we'll only do this if everything
4412	* else matches.
4413	*/
4414	ret = get_inode_info(ctx->root, dir, NULL, &dir_gen, NULL,
4415	NULL, NULL, NULL);
4416	if (ret)
4417	return ret;
4418	if (dir_gen != ctx->dir_gen)
4419	return 0;
4420	ctx->found_idx = num;
4421	return 1;
4422	}
4423	return 0;
4424	}
4425
4426	static int find_iref(struct btrfs_root *root,
4427	struct btrfs_path *path,
4428	struct btrfs_key *key,
4429	u64 dir, u64 dir_gen, struct fs_path *name)
4430	{	4497	{
4431	int ret;	4498	int ret;
4432	struct find_ref_ctx ctx;
4433
4434	ctx.dir = dir;
4435	ctx.name = name;
4436	ctx.dir_gen = dir_gen;
4437	ctx.found_idx = -1;
4438	ctx.root = root;
4439		4499
4440	ret = iterate_inode_ref(root, path, key, 0, __find_iref, &ctx);	4500	ret = iterate_inode_ref(sctx->send_root, sctx->left_path,
		4501	sctx->cmp_key, 0, record_new_ref_if_needed, sctx);
4441	if (ret < 0)	4502	if (ret < 0)
4442	return ret;	4503	goto out;
4443		4504	ret = 0;
4444	if (ctx.found_idx == -1)
4445	return -ENOENT;
4446
4447	return ctx.found_idx;
4448	}
4449
4450	static int __record_changed_new_ref(int num, u64 dir, int index,
4451	struct fs_path *name,
4452	void *ctx)
4453	{
4454	u64 dir_gen;
4455	int ret;
4456	struct send_ctx *sctx = ctx;
4457
4458	ret = get_inode_info(sctx->send_root, dir, NULL, &dir_gen, NULL,
4459	NULL, NULL, NULL);
4460	if (ret)
4461	return ret;
4462
4463	ret = find_iref(sctx->parent_root, sctx->right_path,
4464	sctx->cmp_key, dir, dir_gen, name);
4465	if (ret == -ENOENT)
4466	ret = __record_new_ref(num, dir, index, name, sctx);
4467	else if (ret > 0)
4468	ret = 0;
4469		4505
		4506	out:
4470	return ret;	4507	return ret;
4471	}	4508	}
4472		4509
4473	static int __record_changed_deleted_ref(int num, u64 dir, int index,	4510	static int record_deleted_ref(struct send_ctx *sctx)
4474	struct fs_path *name,
4475	void *ctx)
4476	{	4511	{
4477	u64 dir_gen;
4478	int ret;	4512	int ret;
4479	struct send_ctx *sctx = ctx;
4480
4481	ret = get_inode_info(sctx->parent_root, dir, NULL, &dir_gen, NULL,
4482	NULL, NULL, NULL);
4483	if (ret)
4484	return ret;
4485		4513
4486	ret = find_iref(sctx->send_root, sctx->left_path, sctx->cmp_key,	4514	ret = iterate_inode_ref(sctx->parent_root, sctx->right_path,
4487	dir, dir_gen, name);	4515	sctx->cmp_key, 0, record_deleted_ref_if_needed,
4488	if (ret == -ENOENT)	4516	sctx);
4489	ret = __record_deleted_ref(num, dir, index, name, sctx);	4517	if (ret < 0)
4490	else if (ret > 0)	4518	goto out;
4491	ret = 0;	4519	ret = 0;
4492		4520
		4521	out:
4493	return ret;	4522	return ret;
4494	}	4523	}
4495		4524
@@ -4498,11 +4527,11 @@ static int record_changed_ref(struct send_ctx *sctx) · 2 unmodified lines
4498	int ret = 0;	4527	int ret = 0;
4499		4528
4500	ret = iterate_inode_ref(sctx->send_root, sctx->left_path,	4529	ret = iterate_inode_ref(sctx->send_root, sctx->left_path,
4501	sctx->cmp_key, 0, __record_changed_new_ref, sctx);	4530	sctx->cmp_key, 0, record_new_ref_if_needed, sctx);
4502	if (ret < 0)	4531	if (ret < 0)
4503	goto out;	4532	goto out;
4504	ret = iterate_inode_ref(sctx->parent_root, sctx->right_path,	4533	ret = iterate_inode_ref(sctx->parent_root, sctx->right_path,
4505	sctx->cmp_key, 0, __record_changed_deleted_ref, sctx);	4534	sctx->cmp_key, 0, record_deleted_ref_if_needed, sctx);
4506	if (ret < 0)	4535	if (ret < 0)
4507	goto out;	4536	goto out;
4508	ret = 0;	4537	ret = 0;
@@ -4533,10 +4562,10 @@ static int process_all_refs(struct send_ctx *sctx, · 24 unmodified lines
4533		4562
4534	if (cmd == BTRFS_COMPARE_TREE_NEW) {	4563	if (cmd == BTRFS_COMPARE_TREE_NEW) {
4535	root = sctx->send_root;	4564	root = sctx->send_root;
4536	cb = __record_new_ref;	4565	cb = record_new_ref_if_needed;
4537	} else if (cmd == BTRFS_COMPARE_TREE_DELETED) {	4566	} else if (cmd == BTRFS_COMPARE_TREE_DELETED) {
4538	root = sctx->parent_root;	4567	root = sctx->parent_root;
4539	cb = __record_deleted_ref;	4568	cb = record_deleted_ref_if_needed;
4540	} else {	4569	} else {
4541	btrfs_err(sctx->send_root->fs_info,	4570	btrfs_err(sctx->send_root->fs_info,
4542	"Wrong command %d in process_all_refs", cmd);	4571	"Wrong command %d in process_all_refs", cmd);
@@ -4864,14 +4893,28 @@ static inline u64 max_send_read_size(const struct send_ctx *sctx) · 321 unmodified lines
4864		4893
4865	static int put_data_header(struct send_ctx *sctx, u32 len)	4894	static int put_data_header(struct send_ctx *sctx, u32 len)
4866	{	4895	{
4867	struct btrfs_tlv_header *hdr;	4896	if (WARN_ON_ONCE(sctx->put_data))
		4897	return -EINVAL;
		4898	sctx->put_data = true;
		4899	if (sctx->proto >= 2) {
		4900	/*
		4901	* Since v2, the data attribute header doesn't include a length,
		4902	* it is implicitly to the end of the command.
		4903	*/
		4904	if (sctx->send_max_size - sctx->send_size < sizeof(__le16) + len)
		4905	return -EOVERFLOW;
		4906	put_unaligned_le16(BTRFS_SEND_A_DATA, sctx->send_buf + sctx->send_size);
		4907	sctx->send_size += sizeof(__le16);
		4908	} else {
		4909	struct btrfs_tlv_header *hdr;
4868		4910
4869	if (sctx->send_max_size - sctx->send_size < sizeof(*hdr) + len)	4911	if (sctx->send_max_size - sctx->send_size < sizeof(*hdr) + len)
4870	return -EOVERFLOW;	4912	return -EOVERFLOW;
4871	hdr = (struct btrfs_tlv_header *)(sctx->send_buf + sctx->send_size);	4913	hdr = (struct btrfs_tlv_header *)(sctx->send_buf + sctx->send_size);
4872	put_unaligned_le16(BTRFS_SEND_A_DATA, &hdr->tlv_type);	4914	put_unaligned_le16(BTRFS_SEND_A_DATA, &hdr->tlv_type);
4873	put_unaligned_le16(len, &hdr->tlv_len);	4915	put_unaligned_le16(len, &hdr->tlv_len);
4874	sctx->send_size += sizeof(*hdr);	4916	sctx->send_size += sizeof(*hdr);
		4917	}
4875	return 0;	4918	return 0;
4876	}	4919	}
4877		4920
@@ -5014,7 +5057,7 @@ static int send_clone(struct send_ctx *sctx, · 136 unmodified lines
5014		5057
5015	if (clone_root->root == sctx->send_root) {	5058	if (clone_root->root == sctx->send_root) {
5016	ret = get_inode_info(sctx->send_root, clone_root->ino, NULL,	5059	ret = get_inode_info(sctx->send_root, clone_root->ino, NULL,
5017	&gen, NULL, NULL, NULL, NULL);	5060	&gen, NULL, NULL, NULL, NULL, NULL);
5018	if (ret < 0)	5061	if (ret < 0)
5019	goto out;	5062	goto out;
5020	ret = get_cur_path(sctx, clone_root->ino, gen, p);	5063	ret = get_cur_path(sctx, clone_root->ino, gen, p);
@@ -5141,17 +5184,214 @@ tlv_put_failure: · 120 unmodified lines
5141	return ret;	5184	return ret;
5142	}	5185	}
5143		5186
5144	static int send_extent_data(struct send_ctx *sctx,	5187	static int send_encoded_inline_extent(struct send_ctx *sctx,
5145	const u64 offset,	5188	struct btrfs_path *path, u64 offset,
5146	const u64 len)	5189	u64 len)
		5190	{
		5191	struct btrfs_root *root = sctx->send_root;
		5192	struct btrfs_fs_info *fs_info = root->fs_info;
		5193	struct inode *inode;
		5194	struct fs_path *fspath;
		5195	struct extent_buffer *leaf = path->nodes[0];
		5196	struct btrfs_key key;
		5197	struct btrfs_file_extent_item *ei;
		5198	u64 ram_bytes;
		5199	size_t inline_size;
		5200	int ret;
		5201
		5202	inode = btrfs_iget(fs_info->sb, sctx->cur_ino, root);
		5203	if (IS_ERR(inode))
		5204	return PTR_ERR(inode);
		5205
		5206	fspath = fs_path_alloc();
		5207	if (!fspath) {
		5208	ret = -ENOMEM;
		5209	goto out;
		5210	}
		5211
		5212	ret = begin_cmd(sctx, BTRFS_SEND_C_ENCODED_WRITE);
		5213	if (ret < 0)
		5214	goto out;
		5215
		5216	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, fspath);
		5217	if (ret < 0)
		5218	goto out;
		5219
		5220	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
		5221	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
		5222	ram_bytes = btrfs_file_extent_ram_bytes(leaf, ei);
		5223	inline_size = btrfs_file_extent_inline_item_len(leaf, path->slots[0]);
		5224
		5225	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, fspath);
		5226	TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
		5227	TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_FILE_LEN,
		5228	min(key.offset + ram_bytes - offset, len));
		5229	TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_LEN, ram_bytes);
		5230	TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_OFFSET, offset - key.offset);
		5231	ret = btrfs_encoded_io_compression_from_extent(fs_info,
		5232	btrfs_file_extent_compression(leaf, ei));
		5233	if (ret < 0)
		5234	goto out;
		5235	TLV_PUT_U32(sctx, BTRFS_SEND_A_COMPRESSION, ret);
		5236
		5237	ret = put_data_header(sctx, inline_size);
		5238	if (ret < 0)
		5239	goto out;
		5240	read_extent_buffer(leaf, sctx->send_buf + sctx->send_size,
		5241	btrfs_file_extent_inline_start(ei), inline_size);
		5242	sctx->send_size += inline_size;
		5243
		5244	ret = send_cmd(sctx);
		5245
		5246	tlv_put_failure:
		5247	out:
		5248	fs_path_free(fspath);
		5249	iput(inode);
		5250	return ret;
		5251	}
		5252
		5253	static int send_encoded_extent(struct send_ctx sctx, struct btrfs_path path,
		5254	u64 offset, u64 len)
		5255	{
		5256	struct btrfs_root *root = sctx->send_root;
		5257	struct btrfs_fs_info *fs_info = root->fs_info;
		5258	struct inode *inode;
		5259	struct fs_path *fspath;
		5260	struct extent_buffer *leaf = path->nodes[0];
		5261	struct btrfs_key key;
		5262	struct btrfs_file_extent_item *ei;
		5263	u64 disk_bytenr, disk_num_bytes;
		5264	u32 data_offset;
		5265	struct btrfs_cmd_header *hdr;
		5266	u32 crc;
		5267	int ret;
		5268
		5269	inode = btrfs_iget(fs_info->sb, sctx->cur_ino, root);
		5270	if (IS_ERR(inode))
		5271	return PTR_ERR(inode);
		5272
		5273	fspath = fs_path_alloc();
		5274	if (!fspath) {
		5275	ret = -ENOMEM;
		5276	goto out;
		5277	}
		5278
		5279	ret = begin_cmd(sctx, BTRFS_SEND_C_ENCODED_WRITE);
		5280	if (ret < 0)
		5281	goto out;
		5282
		5283	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, fspath);
		5284	if (ret < 0)
		5285	goto out;
		5286
		5287	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
		5288	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
		5289	disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei);
		5290	disk_num_bytes = btrfs_file_extent_disk_num_bytes(leaf, ei);
		5291
		5292	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, fspath);
		5293	TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
		5294	TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_FILE_LEN,
		5295	min(key.offset + btrfs_file_extent_num_bytes(leaf, ei) - offset,
		5296	len));
		5297	TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_LEN,
		5298	btrfs_file_extent_ram_bytes(leaf, ei));
		5299	TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_OFFSET,
		5300	offset - key.offset + btrfs_file_extent_offset(leaf, ei));
		5301	ret = btrfs_encoded_io_compression_from_extent(fs_info,
		5302	btrfs_file_extent_compression(leaf, ei));
		5303	if (ret < 0)
		5304	goto out;
		5305	TLV_PUT_U32(sctx, BTRFS_SEND_A_COMPRESSION, ret);
		5306	TLV_PUT_U32(sctx, BTRFS_SEND_A_ENCRYPTION, 0);
		5307
		5308	ret = put_data_header(sctx, disk_num_bytes);
		5309	if (ret < 0)
		5310	goto out;
		5311
		5312	/*
		5313	* We want to do I/O directly into the send buffer, so get the next page
		5314	* boundary in the send buffer. This means that there may be a gap
		5315	* between the beginning of the command and the file data.
		5316	*/
		5317	data_offset = ALIGN(sctx->send_size, PAGE_SIZE);
		5318	if (data_offset > sctx->send_max_size \|\|
		5319	sctx->send_max_size - data_offset < disk_num_bytes) {
		5320	ret = -EOVERFLOW;
		5321	goto out;
		5322	}
		5323
		5324	/*
		5325	* Note that send_buf is a mapping of send_buf_pages, so this is really
		5326	* reading into send_buf.
		5327	*/
		5328	ret = btrfs_encoded_read_regular_fill_pages(BTRFS_I(inode), offset,
		5329	disk_bytenr, disk_num_bytes,
		5330	sctx->send_buf_pages +
		5331	(data_offset >> PAGE_SHIFT));
		5332	if (ret)
		5333	goto out;
		5334
		5335	hdr = (struct btrfs_cmd_header *)sctx->send_buf;
		5336	hdr->len = cpu_to_le32(sctx->send_size + disk_num_bytes - sizeof(*hdr));
		5337	hdr->crc = 0;
		5338	crc = btrfs_crc32c(0, sctx->send_buf, sctx->send_size);
		5339	crc = btrfs_crc32c(crc, sctx->send_buf + data_offset, disk_num_bytes);
		5340	hdr->crc = cpu_to_le32(crc);
		5341
		5342	ret = write_buf(sctx->send_filp, sctx->send_buf, sctx->send_size,
		5343	&sctx->send_off);
		5344	if (!ret) {
		5345	ret = write_buf(sctx->send_filp, sctx->send_buf + data_offset,
		5346	disk_num_bytes, &sctx->send_off);
		5347	}
		5348	sctx->send_size = 0;
		5349	sctx->put_data = false;
		5350
		5351	tlv_put_failure:
		5352	out:
		5353	fs_path_free(fspath);
		5354	iput(inode);
		5355	return ret;
		5356	}
		5357
		5358	static int send_extent_data(struct send_ctx sctx, struct btrfs_path path,
		5359	const u64 offset, const u64 len)
5147	{	5360	{
5148	const u64 end = offset + len;	5361	const u64 end = offset + len;
		5362	struct extent_buffer *leaf = path->nodes[0];
		5363	struct btrfs_file_extent_item *ei;
5149	u64 read_size = max_send_read_size(sctx);	5364	u64 read_size = max_send_read_size(sctx);
5150	u64 sent = 0;	5365	u64 sent = 0;
5151		5366
5152	if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA)	5367	if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA)
5153	return send_update_extent(sctx, offset, len);	5368	return send_update_extent(sctx, offset, len);
5154		5369
		5370	ei = btrfs_item_ptr(leaf, path->slots[0],
		5371	struct btrfs_file_extent_item);
		5372	if ((sctx->flags & BTRFS_SEND_FLAG_COMPRESSED) &&
		5373	btrfs_file_extent_compression(leaf, ei) != BTRFS_COMPRESS_NONE) {
		5374	bool is_inline = (btrfs_file_extent_type(leaf, ei) ==
		5375	BTRFS_FILE_EXTENT_INLINE);
		5376
		5377	/*
		5378	* Send the compressed extent unless the compressed data is
		5379	* larger than the decompressed data. This can happen if we're
		5380	* not sending the entire extent, either because it has been
		5381	* partially overwritten/truncated or because this is a part of
		5382	* the extent that we couldn't clone in clone_range().
		5383	*/
		5384	if (is_inline &&
		5385	btrfs_file_extent_inline_item_len(leaf,
		5386	path->slots[0]) <= len) {
		5387	return send_encoded_inline_extent(sctx, path, offset,
		5388	len);
		5389	} else if (!is_inline &&
		5390	btrfs_file_extent_disk_num_bytes(leaf, ei) <= len) {
		5391	return send_encoded_extent(sctx, path, offset, len);
		5392	}
		5393	}
		5394
5155	if (sctx->cur_inode == NULL) {	5395	if (sctx->cur_inode == NULL) {
5156	struct btrfs_root *root = sctx->send_root;	5396	struct btrfs_root *root = sctx->send_root;
5157		5397
@@ -5289,12 +5529,9 @@ out: · 131 unmodified lines
5289	return ret;	5529	return ret;
5290	}	5530	}
5291		5531
5292	static int clone_range(struct send_ctx *sctx,	5532	static int clone_range(struct send_ctx sctx, struct btrfs_path dst_path,
5293	struct clone_root *clone_root,	5533	struct clone_root *clone_root, const u64 disk_byte,
5294	const u64 disk_byte,	5534	u64 data_offset, u64 offset, u64 len)
5295	u64 data_offset,
5296	u64 offset,
5297	u64 len)
5298	{	5535	{
5299	struct btrfs_path *path;	5536	struct btrfs_path *path;
5300	struct btrfs_key key;	5537	struct btrfs_key key;
@@ -5318,7 +5555,7 @@ static int clone_range(struct send_ctx *sctx, · 17 unmodified lines
5318	*/	5555	*/
5319	if (clone_root->offset == 0 &&	5556	if (clone_root->offset == 0 &&
5320	len == sctx->send_root->fs_info->sectorsize)	5557	len == sctx->send_root->fs_info->sectorsize)
5321	return send_extent_data(sctx, offset, len);	5558	return send_extent_data(sctx, dst_path, offset, len);
5322		5559
5323	path = alloc_path_for_send();	5560	path = alloc_path_for_send();
5324	if (!path)	5561	if (!path)
@@ -5329,7 +5566,8 @@ static int clone_range(struct send_ctx *sctx, · 4 unmodified lines
5329	* accept clones from these extents.	5566	* accept clones from these extents.
5330	*/	5567	*/
5331	ret = __get_inode_info(clone_root->root, path, clone_root->ino,	5568	ret = __get_inode_info(clone_root->root, path, clone_root->ino,
5332	&clone_src_i_size, NULL, NULL, NULL, NULL, NULL);	5569	&clone_src_i_size, NULL, NULL, NULL, NULL, NULL,
		5570	NULL);
5333	btrfs_release_path(path);	5571	btrfs_release_path(path);
5334	if (ret < 0)	5572	if (ret < 0)
5335	goto out;	5573	goto out;
@@ -5415,7 +5653,8 @@ static int clone_range(struct send_ctx *sctx, · 79 unmodified lines
5415		5653
5416	if (hole_len > len)	5654	if (hole_len > len)
5417	hole_len = len;	5655	hole_len = len;
5418	ret = send_extent_data(sctx, offset, hole_len);	5656	ret = send_extent_data(sctx, dst_path, offset,
		5657	hole_len);
5419	if (ret < 0)	5658	if (ret < 0)
5420	goto out;	5659	goto out;
5421		5660
@@ -5488,14 +5727,16 @@ static int clone_range(struct send_ctx *sctx, · 66 unmodified lines
5488	if (ret < 0)	5727	if (ret < 0)
5489	goto out;	5728	goto out;
5490	}	5729	}
5491	ret = send_extent_data(sctx, offset + slen,	5730	ret = send_extent_data(sctx, dst_path,
		5731	offset + slen,
5492	clone_len - slen);	5732	clone_len - slen);
5493	} else {	5733	} else {
5494	ret = send_clone(sctx, offset, clone_len,	5734	ret = send_clone(sctx, offset, clone_len,
5495	clone_root);	5735	clone_root);
5496	}	5736	}
5497	} else {	5737	} else {
5498	ret = send_extent_data(sctx, offset, clone_len);	5738	ret = send_extent_data(sctx, dst_path, offset,
		5739	clone_len);
5499	}	5740	}
5500		5741
5501	if (ret < 0)	5742	if (ret < 0)
@@ -5527,7 +5768,7 @@ next: · 25 unmodified lines
5527	}	5768	}
5528		5769
5529	if (len > 0)	5770	if (len > 0)
5530	ret = send_extent_data(sctx, offset, len);	5771	ret = send_extent_data(sctx, dst_path, offset, len);
5531	else	5772	else
5532	ret = 0;	5773	ret = 0;
5533	out:	5774	out:
@@ -5558,10 +5799,10 @@ static int send_write_or_clone(struct send_ctx *sctx, · 24 unmodified lines
5558	struct btrfs_file_extent_item);	5799	struct btrfs_file_extent_item);
5559	disk_byte = btrfs_file_extent_disk_bytenr(path->nodes[0], ei);	5800	disk_byte = btrfs_file_extent_disk_bytenr(path->nodes[0], ei);
5560	data_offset = btrfs_file_extent_offset(path->nodes[0], ei);	5801	data_offset = btrfs_file_extent_offset(path->nodes[0], ei);
5561	ret = clone_range(sctx, clone_root, disk_byte, data_offset,	5802	ret = clone_range(sctx, path, clone_root, disk_byte,
5562	offset, end - offset);	5803	data_offset, offset, end - offset);
5563	} else {	5804	} else {
5564	ret = send_extent_data(sctx, offset, end - offset);	5805	ret = send_extent_data(sctx, path, offset, end - offset);
5565	}	5806	}
5566	sctx->cur_inode_next_write_offset = end;	5807	sctx->cur_inode_next_write_offset = end;
5567	return ret;	5808	return ret;
@@ -6021,11 +6262,14 @@ static int finish_inode_if_needed(struct send_ctx *sctx, int at_end) · 453 unmodified lines
6021	u64 left_mode;	6262	u64 left_mode;
6022	u64 left_uid;	6263	u64 left_uid;
6023	u64 left_gid;	6264	u64 left_gid;
		6265	u64 left_fileattr;
6024	u64 right_mode;	6266	u64 right_mode;
6025	u64 right_uid;	6267	u64 right_uid;
6026	u64 right_gid;	6268	u64 right_gid;
		6269	u64 right_fileattr;
6027	int need_chmod = 0;	6270	int need_chmod = 0;
6028	int need_chown = 0;	6271	int need_chown = 0;
		6272	bool need_fileattr = false;
6029	int need_truncate = 1;	6273	int need_truncate = 1;
6030	int pending_move = 0;	6274	int pending_move = 0;
6031	int refs_processed = 0;	6275	int refs_processed = 0;
@@ -6059,7 +6303,7 @@ static int finish_inode_if_needed(struct send_ctx *sctx, int at_end) · 27 unmodified lines
6059	goto out;	6303	goto out;
6060		6304
6061	ret = get_inode_info(sctx->send_root, sctx->cur_ino, NULL, NULL,	6305	ret = get_inode_info(sctx->send_root, sctx->cur_ino, NULL, NULL,
6062	&left_mode, &left_uid, &left_gid, NULL);	6306	&left_mode, &left_uid, &left_gid, NULL, &left_fileattr);
6063	if (ret < 0)	6307	if (ret < 0)
6064	goto out;	6308	goto out;
6065		6309
@@ -6074,7 +6318,7 @@ static int finish_inode_if_needed(struct send_ctx *sctx, int at_end) · 8 unmodified lines
6074		6318
6075	ret = get_inode_info(sctx->parent_root, sctx->cur_ino,	6319	ret = get_inode_info(sctx->parent_root, sctx->cur_ino,
6076	&old_size, NULL, &right_mode, &right_uid,	6320	&old_size, NULL, &right_mode, &right_uid,
6077	&right_gid, NULL);	6321	&right_gid, NULL, &right_fileattr);
6078	if (ret < 0)	6322	if (ret < 0)
6079	goto out;	6323	goto out;
6080		6324
@@ -6082,6 +6326,8 @@ static int finish_inode_if_needed(struct send_ctx *sctx, int at_end) · 1 unmodified line
6082	need_chown = 1;	6326	need_chown = 1;
6083	if (!S_ISLNK(sctx->cur_inode_mode) && left_mode != right_mode)	6327	if (!S_ISLNK(sctx->cur_inode_mode) && left_mode != right_mode)
6084	need_chmod = 1;	6328	need_chmod = 1;
		6329	if (!S_ISLNK(sctx->cur_inode_mode) && left_fileattr != right_fileattr)
		6330	need_fileattr = true;
6085	if ((old_size == sctx->cur_inode_size) \|\|	6331	if ((old_size == sctx->cur_inode_size) \|\|
6086	(sctx->cur_inode_size > old_size &&	6332	(sctx->cur_inode_size > old_size &&
6087	sctx->cur_inode_next_write_offset == sctx->cur_inode_size))	6333	sctx->cur_inode_next_write_offset == sctx->cur_inode_size))
@@ -6125,6 +6371,12 @@ static int finish_inode_if_needed(struct send_ctx *sctx, int at_end) · 37 unmodified lines
6125	if (ret < 0)	6371	if (ret < 0)
6126	goto out;	6372	goto out;
6127	}	6373	}
		6374	if (need_fileattr) {
		6375	ret = send_fileattr(sctx, sctx->cur_ino, sctx->cur_inode_gen,
		6376	left_fileattr);
		6377	if (ret < 0)
		6378	goto out;
		6379	}
6128		6380
6129	ret = send_capabilities(sctx);	6381	ret = send_capabilities(sctx);
6130	if (ret < 0)	6382	if (ret < 0)
@@ -6165,8 +6417,13 @@ static int record_parent_ref(int num, u64 dir, int index, struct fs_path *name, · 34 unmodified lines
6165	{	6417	{
6166	struct parent_paths_ctx *ppctx = ctx;	6418	struct parent_paths_ctx *ppctx = ctx;
6167		6419
6168	return record_ref(ppctx->sctx->parent_root, dir, name, ppctx->sctx,	6420	/*
6169	ppctx->refs);	6421	* Pass 0 as the generation for the directory, we don't care about it
		6422	* here as we have no new references to add, we just want to delete all
		6423	* references for an inode.
		6424	*/
		6425	return record_ref_in_tree(&ppctx->sctx->rbtree_deleted_refs, ppctx->refs,
		6426	name, dir, 0, ppctx->sctx);
6170	}	6427	}
6171		6428
6172	/*	6429	/*
@@ -6220,9 +6477,7 @@ static int btrfs_unlink_all_paths(struct send_ctx *sctx) · 47 unmodified lines
6220	ret = send_unlink(sctx, ref->full_path);	6477	ret = send_unlink(sctx, ref->full_path);
6221	if (ret < 0)	6478	if (ret < 0)
6222	goto out;	6479	goto out;
6223	fs_path_free(ref->full_path);	6480	recorded_ref_free(ref);
6224	list_del(&ref->list);
6225	kfree(ref);
6226	}	6481	}
6227	ret = 0;	6482	ret = 0;
6228	out:	6483	out:
@@ -6269,7 +6524,7 @@ static int changed_inode(struct send_ctx *sctx, · 40 unmodified lines
6269	close_current_inode(sctx);	6524	close_current_inode(sctx);
6270		6525
6271	sctx->cur_ino = key->objectid;	6526	sctx->cur_ino = key->objectid;
6272	sctx->cur_inode_new_gen = 0;	6527	sctx->cur_inode_new_gen = false;
6273	sctx->cur_inode_last_extent = (u64)-1;	6528	sctx->cur_inode_last_extent = (u64)-1;
6274	sctx->cur_inode_next_write_offset = 0;	6529	sctx->cur_inode_next_write_offset = 0;
6275	sctx->ignore_cur_inode = false;	6530	sctx->ignore_cur_inode = false;
@@ -6310,7 +6565,7 @@ static int changed_inode(struct send_ctx *sctx, · 34 unmodified lines
6310	*/	6565	*/
6311	if (left_gen != right_gen &&	6566	if (left_gen != right_gen &&
6312	sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID)	6567	sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID)
6313	sctx->cur_inode_new_gen = 1;	6568	sctx->cur_inode_new_gen = true;
6314	}	6569	}
6315		6570
6316	/*	6571	/*
@@ -6342,8 +6597,8 @@ static int changed_inode(struct send_ctx *sctx, · 25 unmodified lines
6342		6597
6343	if (result == BTRFS_COMPARE_TREE_NEW) {	6598	if (result == BTRFS_COMPARE_TREE_NEW) {
6344	sctx->cur_inode_gen = left_gen;	6599	sctx->cur_inode_gen = left_gen;
6345	sctx->cur_inode_new = 1;	6600	sctx->cur_inode_new = true;
6346	sctx->cur_inode_deleted = 0;	6601	sctx->cur_inode_deleted = false;
6347	sctx->cur_inode_size = btrfs_inode_size(	6602	sctx->cur_inode_size = btrfs_inode_size(
6348	sctx->left_path->nodes[0], left_ii);	6603	sctx->left_path->nodes[0], left_ii);
6349	sctx->cur_inode_mode = btrfs_inode_mode(	6604	sctx->cur_inode_mode = btrfs_inode_mode(
@@ -6354,8 +6609,8 @@ static int changed_inode(struct send_ctx *sctx, · 4 unmodified lines
6354	ret = send_create_inode_if_needed(sctx);	6609	ret = send_create_inode_if_needed(sctx);
6355	} else if (result == BTRFS_COMPARE_TREE_DELETED) {	6610	} else if (result == BTRFS_COMPARE_TREE_DELETED) {
6356	sctx->cur_inode_gen = right_gen;	6611	sctx->cur_inode_gen = right_gen;
6357	sctx->cur_inode_new = 0;	6612	sctx->cur_inode_new = false;
6358	sctx->cur_inode_deleted = 1;	6613	sctx->cur_inode_deleted = true;
6359	sctx->cur_inode_size = btrfs_inode_size(	6614	sctx->cur_inode_size = btrfs_inode_size(
6360	sctx->right_path->nodes[0], right_ii);	6615	sctx->right_path->nodes[0], right_ii);
6361	sctx->cur_inode_mode = btrfs_inode_mode(	6616	sctx->cur_inode_mode = btrfs_inode_mode(
@@ -6373,8 +6628,8 @@ static int changed_inode(struct send_ctx *sctx, · 11 unmodified lines
6373	* First, process the inode as if it was deleted.	6628	* First, process the inode as if it was deleted.
6374	*/	6629	*/
6375	sctx->cur_inode_gen = right_gen;	6630	sctx->cur_inode_gen = right_gen;
6376	sctx->cur_inode_new = 0;	6631	sctx->cur_inode_new = false;
6377	sctx->cur_inode_deleted = 1;	6632	sctx->cur_inode_deleted = true;
6378	sctx->cur_inode_size = btrfs_inode_size(	6633	sctx->cur_inode_size = btrfs_inode_size(
6379	sctx->right_path->nodes[0], right_ii);	6634	sctx->right_path->nodes[0], right_ii);
6380	sctx->cur_inode_mode = btrfs_inode_mode(	6635	sctx->cur_inode_mode = btrfs_inode_mode(
@@ -6388,8 +6643,8 @@ static int changed_inode(struct send_ctx *sctx, · 7 unmodified lines
6388	* Now process the inode as if it was new.	6643	* Now process the inode as if it was new.
6389	*/	6644	*/
6390	sctx->cur_inode_gen = left_gen;	6645	sctx->cur_inode_gen = left_gen;
6391	sctx->cur_inode_new = 1;	6646	sctx->cur_inode_new = true;
6392	sctx->cur_inode_deleted = 0;	6647	sctx->cur_inode_deleted = false;
6393	sctx->cur_inode_size = btrfs_inode_size(	6648	sctx->cur_inode_size = btrfs_inode_size(
6394	sctx->left_path->nodes[0], left_ii);	6649	sctx->left_path->nodes[0], left_ii);
6395	sctx->cur_inode_mode = btrfs_inode_mode(	6650	sctx->cur_inode_mode = btrfs_inode_mode(
@@ -6421,9 +6676,9 @@ static int changed_inode(struct send_ctx *sctx, · 25 unmodified lines
6421	goto out;	6676	goto out;
6422	} else {	6677	} else {
6423	sctx->cur_inode_gen = left_gen;	6678	sctx->cur_inode_gen = left_gen;
6424	sctx->cur_inode_new = 0;	6679	sctx->cur_inode_new = false;
6425	sctx->cur_inode_new_gen = 0;	6680	sctx->cur_inode_new_gen = false;
6426	sctx->cur_inode_deleted = 0;	6681	sctx->cur_inode_deleted = false;
6427	sctx->cur_inode_size = btrfs_inode_size(	6682	sctx->cur_inode_size = btrfs_inode_size(
6428	sctx->left_path->nodes[0], left_ii);	6683	sctx->left_path->nodes[0], left_ii);
6429	sctx->cur_inode_mode = btrfs_inode_mode(	6684	sctx->cur_inode_mode = btrfs_inode_mode(
@@ -6536,12 +6791,12 @@ static int dir_changed(struct send_ctx *sctx, u64 dir) · 106 unmodified lines
6536	int ret;	6791	int ret;
6537		6792
6538	ret = get_inode_info(sctx->send_root, dir, NULL, &new_gen, NULL, NULL,	6793	ret = get_inode_info(sctx->send_root, dir, NULL, &new_gen, NULL, NULL,
6539	NULL, NULL);	6794	NULL, NULL, NULL);
6540	if (ret)	6795	if (ret)
6541	return ret;	6796	return ret;
6542		6797
6543	ret = get_inode_info(sctx->parent_root, dir, NULL, &orig_gen, NULL,	6798	ret = get_inode_info(sctx->parent_root, dir, NULL, &orig_gen, NULL,
6544	NULL, NULL, NULL);	6799	NULL, NULL, NULL, NULL);
6545	if (ret)	6800	if (ret)
6546	return ret;	6801	return ret;
6547		6802
@@ -7537,6 +7792,10 @@ long btrfs_ioctl_send(struct inode inode, struct btrfs_ioctl_send_args arg) · 989 unmodified lines
7537	} else {	7792	} else {
7538	sctx->proto = 1;	7793	sctx->proto = 1;
7539	}	7794	}
		7795	if ((arg->flags & BTRFS_SEND_FLAG_COMPRESSED) && sctx->proto < 2) {
		7796	ret = -EINVAL;
		7797	goto out;
		7798	}
7540		7799
7541	sctx->send_filp = fget(arg->send_fd);	7800	sctx->send_filp = fget(arg->send_fd);
7542	if (!sctx->send_filp) {	7801	if (!sctx->send_filp) {
@@ -7556,8 +7815,31 @@ long btrfs_ioctl_send(struct inode inode, struct btrfs_ioctl_send_args arg) · 13 unmodified lines
7556		7815
7557	sctx->clone_roots_cnt = arg->clone_sources_count;	7816	sctx->clone_roots_cnt = arg->clone_sources_count;
7558		7817
7559	sctx->send_max_size = BTRFS_SEND_BUF_SIZE;	7818	if (sctx->proto >= 2) {
7560	sctx->send_buf = kvmalloc(sctx->send_max_size, GFP_KERNEL);	7819	u32 send_buf_num_pages;
		7820
		7821	sctx->send_max_size = ALIGN(SZ_16K + BTRFS_MAX_COMPRESSED, PAGE_SIZE);
		7822	sctx->send_buf = vmalloc(sctx->send_max_size);
		7823	if (!sctx->send_buf) {
		7824	ret = -ENOMEM;
		7825	goto out;
		7826	}
		7827	send_buf_num_pages = sctx->send_max_size >> PAGE_SHIFT;
		7828	sctx->send_buf_pages = kcalloc(send_buf_num_pages,
		7829	sizeof(*sctx->send_buf_pages),
		7830	GFP_KERNEL);
		7831	if (!sctx->send_buf_pages) {
		7832	ret = -ENOMEM;
		7833	goto out;
		7834	}
		7835	for (i = 0; i < send_buf_num_pages; i++) {
		7836	sctx->send_buf_pages[i] =
		7837	vmalloc_to_page(sctx->send_buf + (i << PAGE_SHIFT));
		7838	}
		7839	} else {
		7840	sctx->send_max_size = BTRFS_SEND_BUF_SIZE_V1;
		7841	sctx->send_buf = kvmalloc(sctx->send_max_size, GFP_KERNEL);
		7842	}
7561	if (!sctx->send_buf) {	7843	if (!sctx->send_buf) {
7562	ret = -ENOMEM;	7844	ret = -ENOMEM;
7563	goto out;	7845	goto out;
@@ -7566,6 +7848,8 @@ long btrfs_ioctl_send(struct inode inode, struct btrfs_ioctl_send_args arg) · 2 unmodified lines
7566	sctx->pending_dir_moves = RB_ROOT;	7848	sctx->pending_dir_moves = RB_ROOT;
7567	sctx->waiting_dir_moves = RB_ROOT;	7849	sctx->waiting_dir_moves = RB_ROOT;
7568	sctx->orphan_dirs = RB_ROOT;	7850	sctx->orphan_dirs = RB_ROOT;
		7851	sctx->rbtree_new_refs = RB_ROOT;
		7852	sctx->rbtree_deleted_refs = RB_ROOT;
7569		7853
7570	sctx->clone_roots = kvcalloc(sizeof(*sctx->clone_roots),	7854	sctx->clone_roots = kvcalloc(sizeof(*sctx->clone_roots),
7571	arg->clone_sources_count + 1,	7855	arg->clone_sources_count + 1,
@@ -7750,6 +8034,7 @@ out: · 178 unmodified lines
7750	fput(sctx->send_filp);	8034	fput(sctx->send_filp);
7751		8035
7752	kvfree(sctx->clone_roots);	8036	kvfree(sctx->clone_roots);
		8037	kfree(sctx->send_buf_pages);
7753	kvfree(sctx->send_buf);	8038	kvfree(sctx->send_buf);
7754		8039
7755	name_cache_free(sctx);	8040	name_cache_free(sctx);

fs/btrfs/send.h+103−66

@@ -7,12 +7,19 @@ · 6 unmodified lines
7	#ifndef BTRFS_SEND_H	7	#ifndef BTRFS_SEND_H
8	#define BTRFS_SEND_H	8	#define BTRFS_SEND_H
9		9
10	#include "ctree.h"	10	#include <linux/types.h>
11		11
12	#define BTRFS_SEND_STREAM_MAGIC "btrfs-stream"	12	#define BTRFS_SEND_STREAM_MAGIC "btrfs-stream"
13	#define BTRFS_SEND_STREAM_VERSION 1	13	#define BTRFS_SEND_STREAM_VERSION 2
14		14
15	#define BTRFS_SEND_BUF_SIZE SZ_64K	15	/*
		16	* In send stream v1, no command is larger than 64K. In send stream v2, no limit
		17	* should be assumed.
		18	*/
		19	#define BTRFS_SEND_BUF_SIZE_V1 SZ_64K
		20
		21	struct inode;
		22	struct btrfs_ioctl_send_args;
16		23
17	enum btrfs_tlv_type {	24	enum btrfs_tlv_type {
18	BTRFS_TLV_U8,	25	BTRFS_TLV_U8,
@@ -46,87 +53,117 @@ struct btrfs_tlv_header { · 27 unmodified lines
46		53
47	/* commands */	54	/* commands */
48	enum btrfs_send_cmd {	55	enum btrfs_send_cmd {
49	BTRFS_SEND_C_UNSPEC,	56	BTRFS_SEND_C_UNSPEC = 0,
50		57
51	/* Version 1 */	58	/* Version 1 */
52	BTRFS_SEND_C_SUBVOL,	59	BTRFS_SEND_C_SUBVOL = 1,
53	BTRFS_SEND_C_SNAPSHOT,	60	BTRFS_SEND_C_SNAPSHOT = 2,
54		61
55	BTRFS_SEND_C_MKFILE,	62	BTRFS_SEND_C_MKFILE = 3,
56	BTRFS_SEND_C_MKDIR,	63	BTRFS_SEND_C_MKDIR = 4,
57	BTRFS_SEND_C_MKNOD,	64	BTRFS_SEND_C_MKNOD = 5,
58	BTRFS_SEND_C_MKFIFO,	65	BTRFS_SEND_C_MKFIFO = 6,
59	BTRFS_SEND_C_MKSOCK,	66	BTRFS_SEND_C_MKSOCK = 7,
60	BTRFS_SEND_C_SYMLINK,	67	BTRFS_SEND_C_SYMLINK = 8,
61		68
62	BTRFS_SEND_C_RENAME,	69	BTRFS_SEND_C_RENAME = 9,
63	BTRFS_SEND_C_LINK,	70	BTRFS_SEND_C_LINK = 10,
64	BTRFS_SEND_C_UNLINK,	71	BTRFS_SEND_C_UNLINK = 11,
65	BTRFS_SEND_C_RMDIR,	72	BTRFS_SEND_C_RMDIR = 12,
66		73
67	BTRFS_SEND_C_SET_XATTR,	74	BTRFS_SEND_C_SET_XATTR = 13,
68	BTRFS_SEND_C_REMOVE_XATTR,	75	BTRFS_SEND_C_REMOVE_XATTR = 14,
69		76
70	BTRFS_SEND_C_WRITE,	77	BTRFS_SEND_C_WRITE = 15,
71	BTRFS_SEND_C_CLONE,	78	BTRFS_SEND_C_CLONE = 16,
72		79
73	BTRFS_SEND_C_TRUNCATE,	80	BTRFS_SEND_C_TRUNCATE = 17,
74	BTRFS_SEND_C_CHMOD,	81	BTRFS_SEND_C_CHMOD = 18,
75	BTRFS_SEND_C_CHOWN,	82	BTRFS_SEND_C_CHOWN = 19,
76	BTRFS_SEND_C_UTIMES,	83	BTRFS_SEND_C_UTIMES = 20,
77		84
78	BTRFS_SEND_C_END,	85	BTRFS_SEND_C_END = 21,
79	BTRFS_SEND_C_UPDATE_EXTENT,	86	BTRFS_SEND_C_UPDATE_EXTENT = 22,
80	__BTRFS_SEND_C_MAX_V1,	87	BTRFS_SEND_C_MAX_V1 = 22,
81		88
82	/* Version 2 */	89	/* Version 2 */
83	__BTRFS_SEND_C_MAX_V2,	90	BTRFS_SEND_C_FALLOCATE = 23,
		91	BTRFS_SEND_C_FILEATTR = 24,
		92	BTRFS_SEND_C_ENCODED_WRITE = 25,
		93	BTRFS_SEND_C_MAX_V2 = 25,
84		94
85	/* End */	95	/* End */
86	__BTRFS_SEND_C_MAX,	96	BTRFS_SEND_C_MAX = 25,
87	};	97	};
88	#define BTRFS_SEND_C_MAX (__BTRFS_SEND_C_MAX - 1)
89		98
90	/* attributes in send stream */	99	/* attributes in send stream */
91	enum {	100	enum {
92	BTRFS_SEND_A_UNSPEC,	101	BTRFS_SEND_A_UNSPEC = 0,
93		102
94	BTRFS_SEND_A_UUID,	103	/* Version 1 */
95	BTRFS_SEND_A_CTRANSID,	104	BTRFS_SEND_A_UUID = 1,
96		105	BTRFS_SEND_A_CTRANSID = 2,
97	BTRFS_SEND_A_INO,	106
98	BTRFS_SEND_A_SIZE,	107	BTRFS_SEND_A_INO = 3,
99	BTRFS_SEND_A_MODE,	108	BTRFS_SEND_A_SIZE = 4,
100	BTRFS_SEND_A_UID,	109	BTRFS_SEND_A_MODE = 5,
101	BTRFS_SEND_A_GID,	110	BTRFS_SEND_A_UID = 6,
102	BTRFS_SEND_A_RDEV,	111	BTRFS_SEND_A_GID = 7,
103	BTRFS_SEND_A_CTIME,	112	BTRFS_SEND_A_RDEV = 8,
104	BTRFS_SEND_A_MTIME,	113	BTRFS_SEND_A_CTIME = 9,
105	BTRFS_SEND_A_ATIME,	114	BTRFS_SEND_A_MTIME = 10,
106	BTRFS_SEND_A_OTIME,	115	BTRFS_SEND_A_ATIME = 11,
107		116	BTRFS_SEND_A_OTIME = 12,
108	BTRFS_SEND_A_XATTR_NAME,	117
109	BTRFS_SEND_A_XATTR_DATA,	118	BTRFS_SEND_A_XATTR_NAME = 13,
110		119	BTRFS_SEND_A_XATTR_DATA = 14,
111	BTRFS_SEND_A_PATH,	120
112	BTRFS_SEND_A_PATH_TO,	121	BTRFS_SEND_A_PATH = 15,
113	BTRFS_SEND_A_PATH_LINK,	122	BTRFS_SEND_A_PATH_TO = 16,
114		123	BTRFS_SEND_A_PATH_LINK = 17,
115	BTRFS_SEND_A_FILE_OFFSET,	124
116	BTRFS_SEND_A_DATA,	125	BTRFS_SEND_A_FILE_OFFSET = 18,
117		126	/*
118	BTRFS_SEND_A_CLONE_UUID,	127	* As of send stream v2, this attribute is special: it must be the last
119	BTRFS_SEND_A_CLONE_CTRANSID,	128	* attribute in a command, its header contains only the type, and its
120	BTRFS_SEND_A_CLONE_PATH,	129	* length is implicitly the remaining length of the command.
121	BTRFS_SEND_A_CLONE_OFFSET,	130	*/
122	BTRFS_SEND_A_CLONE_LEN,	131	BTRFS_SEND_A_DATA = 19,
123		132
124	__BTRFS_SEND_A_MAX,	133	BTRFS_SEND_A_CLONE_UUID = 20,
		134	BTRFS_SEND_A_CLONE_CTRANSID = 21,
		135	BTRFS_SEND_A_CLONE_PATH = 22,
		136	BTRFS_SEND_A_CLONE_OFFSET = 23,
		137	BTRFS_SEND_A_CLONE_LEN = 24,
		138
		139	BTRFS_SEND_A_MAX_V1 = 24,
		140
		141	/* Version 2 */
		142	BTRFS_SEND_A_FALLOCATE_MODE = 25,
		143
		144	/*
		145	* File attributes from the FS_*_FL namespace (i_flags, xflags),
		146	* translated to BTRFS_INODE_* bits (BTRFS_INODE_FLAG_MASK) and stored
		147	* in btrfs_inode_item::flags (represented by btrfs_inode::flags and
		148	* btrfs_inode::ro_flags).
		149	*/
		150	BTRFS_SEND_A_FILEATTR = 26,
		151
		152	BTRFS_SEND_A_UNENCODED_FILE_LEN = 27,
		153	BTRFS_SEND_A_UNENCODED_LEN = 28,
		154	BTRFS_SEND_A_UNENCODED_OFFSET = 29,
		155	/*
		156	* COMPRESSION and ENCRYPTION default to NONE (0) if omitted from
		157	* BTRFS_SEND_C_ENCODED_WRITE.
		158	*/
		159	BTRFS_SEND_A_COMPRESSION = 30,
		160	BTRFS_SEND_A_ENCRYPTION = 31,
		161	BTRFS_SEND_A_MAX_V2 = 31,
		162
		163	/* End */
		164	BTRFS_SEND_A_MAX = 31,
125	};	165	};
126	#define BTRFS_SEND_A_MAX (__BTRFS_SEND_A_MAX - 1)
127		166
128	#ifdef __KERNEL__
129	long btrfs_ioctl_send(struct inode inode, struct btrfs_ioctl_send_args arg);	167	long btrfs_ioctl_send(struct inode inode, struct btrfs_ioctl_send_args arg);
130	#endif
131		168
132	#endif	169	#endif

fs/btrfs/space-info.c+99−11

 #include "ordered-data.h"
 #include "transaction.h"
 #include "block-group.h"
+#include "zoned.h"
 /*
  * HOW DOES SPACE RESERVATION WORK
  */
 #define BTRFS_DEFAULT_ZONED_RECLAIM_THRESH			(75)
+/*
+ * Calculate chunk size depending on volume type (regular or zoned).
+ */
+static u64 calc_chunk_size(const struct btrfs_fs_info *fs_info, u64 flags)
+{
+	if (btrfs_is_zoned(fs_info))
+		return fs_info->zone_size;
+	ASSERT(flags & BTRFS_BLOCK_GROUP_TYPE_MASK);
+	if (flags & BTRFS_BLOCK_GROUP_DATA)
+		return SZ_1G;
+	else if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
+		return SZ_32M;
+	/* Handle BTRFS_BLOCK_GROUP_METADATA */
+	if (fs_info->fs_devices->total_rw_bytes > 50ULL * SZ_1G)
+		return SZ_1G;
+	return SZ_256M;
+}
+/*
+ * Update default chunk size.
+ */
+void btrfs_update_space_info_chunk_size(struct btrfs_space_info *space_info,
+					u64 chunk_size)
+{
+	WRITE_ONCE(space_info->chunk_size, chunk_size);
+}
 static int create_space_info(struct btrfs_fs_info *info, u64 flags)
 {
 	INIT_LIST_HEAD(&space_info->tickets);
 	INIT_LIST_HEAD(&space_info->priority_tickets);
 	space_info->clamp = 1;
+	btrfs_update_space_info_chunk_size(space_info, calc_chunk_size(info, flags));
 	if (btrfs_is_zoned(info))
 		space_info->bg_reclaim_threshold = BTRFS_DEFAULT_ZONED_RECLAIM_THRESH;
 void btrfs_update_space_info(struct btrfs_fs_info *info, u64 flags,
 			     u64 total_bytes, u64 bytes_used,
 			     u64 bytes_readonly, u64 bytes_zone_unusable,
-			     struct btrfs_space_info **space_info)
+			     bool active, struct btrfs_space_info **space_info)
 {
 	struct btrfs_space_info *found;
 	int factor;
 	ASSERT(found);
 	spin_lock(&found->lock);
 	found->total_bytes += total_bytes;
+	if (active)
+		found->active_total_bytes += total_bytes;
 	found->disk_total += total_bytes * factor;
 	found->bytes_used += bytes_used;
 	found->disk_used += bytes_used * factor;
 	return avail;
 }
+static inline u64 writable_total_bytes(struct btrfs_fs_info *fs_info,
+				       struct btrfs_space_info *space_info)
+{
+	/*
+	 * On regular filesystem, all total_bytes are always writable. On zoned
+	 * filesystem, there may be a limitation imposed by max_active_zones.
+	 * For metadata allocation, we cannot finish an existing active block
+	 * group to avoid a deadlock. Thus, we need to consider only the active
+	 * groups to be writable for metadata space.
+	 */
+	if (!btrfs_is_zoned(fs_info) || (space_info->flags & BTRFS_BLOCK_GROUP_DATA))
+		return space_info->total_bytes;
+	return space_info->active_total_bytes;
+}
 int btrfs_can_overcommit(struct btrfs_fs_info *fs_info,
 			 struct btrfs_space_info *space_info, u64 bytes,
 			 enum btrfs_reserve_flush_enum flush)
 		return 0;
 	used = btrfs_space_info_used(space_info, true);
-	avail = calc_available_free_space(fs_info, space_info, flush);
+	if (btrfs_is_zoned(fs_info) && (space_info->flags & BTRFS_BLOCK_GROUP_METADATA))
+		avail = 0;
+	else
+		avail = calc_available_free_space(fs_info, space_info, flush);
-	if (used + bytes < space_info->total_bytes + avail)
+	if (used + bytes < writable_total_bytes(fs_info, space_info) + avail)
 		return 1;
 	return 0;
 }
 		ticket = list_first_entry(head, struct reserve_ticket, list);
 		/* Check and see if our ticket can be satisfied now. */
-		if ((used + ticket->bytes <= space_info->total_bytes) ||
+		if ((used + ticket->bytes <= writable_total_bytes(fs_info, space_info)) ||
 		    btrfs_can_overcommit(fs_info, space_info, ticket->bytes,
 					 flush)) {
 			btrfs_space_info_update_bytes_may_use(fs_info,
 		break;
 	case ALLOC_CHUNK:
 	case ALLOC_CHUNK_FORCE:
+		/*
+		 * For metadata space on zoned filesystem, reaching here means we
+		 * don't have enough space left in active_total_bytes. Try to
+		 * activate a block group first, because we may have inactive
+		 * block group already allocated.
+		 */
+		ret = btrfs_zoned_activate_one_bg(fs_info, space_info, false);
+		if (ret < 0)
+			break;
+		else if (ret == 1)
+			break;
 		trans = btrfs_join_transaction(root);
 		if (IS_ERR(trans)) {
 			ret = PTR_ERR(trans);
 				(state == ALLOC_CHUNK) ? CHUNK_ALLOC_NO_FORCE :
 					CHUNK_ALLOC_FORCE);
 		btrfs_end_transaction(trans);
+		/*
+		 * For metadata space on zoned filesystem, allocating a new chunk
+		 * is not enough. We still need to activate the block * group.
+		 * Active the newly allocated block group by (maybe) finishing
+		 * a block group.
+		 */
+		if (ret == 1) {
+			ret = btrfs_zoned_activate_one_bg(fs_info, space_info, true);
+			/*
+			 * Revert to the original ret regardless we could finish
+			 * one block group or not.
+			 */
+			if (ret >= 0)
+				ret = 1;
+		}
 		if (ret > 0 || ret == -ENOSPC)
 			ret = 0;
 		break;
 {
 	u64 used;
 	u64 avail;
+	u64 total;
 	u64 to_reclaim = space_info->reclaim_size;
 	lockdep_assert_held(&space_info->lock);
 	 * space.  If that's the case add in our overage so we make sure to put
 	 * appropriate pressure on the flushing state machine.
 	 */
-	if (space_info->total_bytes + avail < used)
+	total = writable_total_bytes(fs_info, space_info);
-		to_reclaim += used - (space_info->total_bytes + avail);
+	if (total + avail < used)
+		to_reclaim += used - (total + avail);
 	return to_reclaim;
 }
 {
 	u64 global_rsv_size = fs_info->global_block_rsv.reserved;
 	u64 ordered, delalloc;
-	u64 thresh = div_factor_fine(space_info->total_bytes, 90);
+	u64 total = writable_total_bytes(fs_info, space_info);
+	u64 thresh;
 	u64 used;
+	thresh = div_factor_fine(total, 90);
 	lockdep_assert_held(&space_info->lock);
 	/* If we're just plain full then async reclaim just slows us down. */
 					   BTRFS_RESERVE_FLUSH_ALL);
 	used = space_info->bytes_used + space_info->bytes_reserved +
 	       space_info->bytes_readonly + global_rsv_size;
-	if (used < space_info->total_bytes)
+	if (used < total)
-		thresh += space_info->total_bytes - used;
+		thresh += total - used;
 	thresh >>= space_info->clamp;
 	used = space_info->bytes_pinned;
 	to_reclaim = btrfs_calc_reclaim_metadata_size(fs_info, space_info);
 	/*
 	 * This is the priority reclaim path, so to_reclaim could be >0 still
-	 * because we may have only satisified the priority tickets and still
+	 * because we may have only satisfied the priority tickets and still
 	 * left non priority tickets on the list.  We would then have
 	 * to_reclaim but ->bytes == 0.
 	 */
 	 * can_overcommit() to ensure we can overcommit to continue.
 	 */
 	if (!pending_tickets &&
-	    ((used + orig_bytes <= space_info->total_bytes) ||
+	    ((used + orig_bytes <= writable_total_bytes(fs_info, space_info)) ||
 	     btrfs_can_overcommit(fs_info, space_info, orig_bytes, flush))) {
 		btrfs_space_info_update_bytes_may_use(fs_info, space_info,
 						      orig_bytes);

fs/btrfs/space-info.h+7−1

 	u64 bytes_may_use;	/* number of bytes that may be used for
 				   delalloc/allocations */
 	u64 bytes_readonly;	/* total bytes that are read only */
+	/* Total bytes in the space, but only accounts active block groups. */
+	u64 active_total_bytes;
 	u64 bytes_zone_unusable;	/* total bytes that are unusable until
 					   resetting the device zone */
 	u64 max_extent_size;	/* This will hold the maximum extent size of
 				   the space info if we had an ENOSPC in the
 				   allocator. */
+	/* Chunk size in bytes */
+	u64 chunk_size;
 	/*
 	 * Once a block group drops below this threshold (percents) we'll
 void btrfs_update_space_info(struct btrfs_fs_info *info, u64 flags,
 			     u64 total_bytes, u64 bytes_used,
 			     u64 bytes_readonly, u64 bytes_zone_unusable,
-			     struct btrfs_space_info **space_info);
+			     bool active, struct btrfs_space_info **space_info);
+void btrfs_update_space_info_chunk_size(struct btrfs_space_info *space_info,
+					u64 chunk_size);
 struct btrfs_space_info *btrfs_find_space_info(struct btrfs_fs_info *info,
 					       u64 flags);
 u64 __pure btrfs_space_info_used(struct btrfs_space_info *s_info,

fs/btrfs/struct-funcs.c+3−8

 {
 	const unsigned long member_offset = (unsigned long)ptr + off;
-	if (member_offset > eb->len) {
+	if (unlikely(member_offset + size > eb->len)) {
 		btrfs_warn(eb->fs_info,
-	"bad eb member start: ptr 0x%lx start %llu member offset %lu size %d",
+		"bad eb member %s: ptr 0x%lx start %llu member offset %lu size %d",
-			(unsigned long)ptr, eb->start, member_offset, size);
+			(member_offset > eb->len ? "start" : "end"),
-		return false;
-	}
-	if (member_offset + size > eb->len) {
-		btrfs_warn(eb->fs_info,
-	"bad eb member end: ptr 0x%lx start %llu member offset %lu size %d",
 			(unsigned long)ptr, eb->start, member_offset, size);
 		return false;
 	}

fs/btrfs/subpage.c+2−2

@@ -123,7 +123,7 @@ int btrfs_attach_subpage(const struct btrfs_fs_info *fs_info, · 122 unmodified lines
123	struct btrfs_subpage *subpage;	123	struct btrfs_subpage *subpage;
124		124
125	/*	125	/*
126	* We have cases like a dummy extent buffer page, which is not mappped	126	* We have cases like a dummy extent buffer page, which is not mapped
127	* and doesn't need to be locked.	127	* and doesn't need to be locked.
128	*/	128	*/
129	if (page->mapping)	129	if (page->mapping)
@@ -731,7 +731,7 @@ void btrfs_page_assert_not_dirty(const struct btrfs_fs_info *fs_info, · 601 unmodified lines
731	* It should not have any subpage::writers count.	731	* It should not have any subpage::writers count.
732	* Can be unlocked by unlock_page().	732	* Can be unlocked by unlock_page().
733	* This is the most common locked page for __extent_writepage() called	733	* This is the most common locked page for __extent_writepage() called
734	* inside extent_write_cache_pages() or extent_write_full_page().	734	* inside extent_write_cache_pages().
735	* Rarer cases include the @locked_page from extent_write_locked_range().	735	* Rarer cases include the @locked_page from extent_write_locked_range().
736	*	736	*
737	* - Page locked by lock_delalloc_pages()	737	* - Page locked by lock_delalloc_pages()

fs/btrfs/super.c+9−27

 #include "block-group.h"
 #include "discard.h"
 #include "qgroup.h"
+#include "raid56.h"
 #define CREATE_TRACE_POINTS
 #include <trace/events/btrfs.h>
 #define STATE_STRING_BUF_LEN	(sizeof(STATE_STRING_PREFACE) + BTRFS_FS_STATE_COUNT)
 /*
- * Characters to print to indicate error conditions or uncommon filesystem sate.
+ * Characters to print to indicate error conditions or uncommon filesystem state.
  * RO is not an error.
  */
 static const char fs_state_chars[] = {
 	btrfs_workqueue_set_max(fs_info->hipri_workers, new_pool_size);
 	btrfs_workqueue_set_max(fs_info->delalloc_workers, new_pool_size);
 	btrfs_workqueue_set_max(fs_info->caching_workers, new_pool_size);
-	btrfs_workqueue_set_max(fs_info->endio_workers, new_pool_size);
-	btrfs_workqueue_set_max(fs_info->endio_meta_workers, new_pool_size);
-	btrfs_workqueue_set_max(fs_info->endio_meta_write_workers,
-				new_pool_size);
 	btrfs_workqueue_set_max(fs_info->endio_write_workers, new_pool_size);
 	btrfs_workqueue_set_max(fs_info->endio_freespace_worker, new_pool_size);
 	btrfs_workqueue_set_max(fs_info->delayed_workers, new_pool_size);
 	if (type & BTRFS_BLOCK_GROUP_RAID0)
 		num_stripes = nr_devices;
-	else if (type & BTRFS_BLOCK_GROUP_RAID1)
+	else if (type & BTRFS_BLOCK_GROUP_RAID1_MASK)
-		num_stripes = 2;
+		num_stripes = rattr->ncopies;
-	else if (type & BTRFS_BLOCK_GROUP_RAID1C3)
-		num_stripes = 3;
-	else if (type & BTRFS_BLOCK_GROUP_RAID1C4)
-		num_stripes = 4;
 	else if (type & BTRFS_BLOCK_GROUP_RAID10)
 		num_stripes = 4;
 		avail_space = rounddown(avail_space, BTRFS_STRIPE_LEN);
 		/*
-		 * In order to avoid overwriting the superblock on the drive,
+		 * Ensure we have at least min_stripe_size on top of the
-		 * btrfs starts at an offset of at least 1MB when doing chunk
+		 * reserved space on the device.
-		 * allocation.
-		 *
-		 * This ensures we have at least min_stripe_size free space
-		 * after excluding 1MB.
 		 */
-		if (avail_space <= SZ_1M + min_stripe_size)
+		if (avail_space <= BTRFS_DEVICE_RANGE_RESERVED + min_stripe_size)
 			continue;
-		avail_space -= SZ_1M;
+		avail_space -= BTRFS_DEVICE_RANGE_RESERVED;
 		devices_info[i].dev = device;
 		devices_info[i].max_avail = avail_space;
 	if (err)
 		goto free_delayed_ref;
-	err = btrfs_end_io_wq_init();
-	if (err)
-		goto free_prelim_ref;
 	err = btrfs_interface_init();
 	if (err)
-		goto free_end_io_wq;
+		goto free_prelim_ref;
 	btrfs_print_mod_info();
 unregister_ioctl:
 	btrfs_interface_exit();
-free_end_io_wq:
-	btrfs_end_io_wq_exit();
 free_prelim_ref:
 	btrfs_prelim_ref_exit();
 free_delayed_ref:
 	extent_state_cache_exit();
 	extent_io_exit();
 	btrfs_interface_exit();
-	btrfs_end_io_wq_exit();
 	unregister_filesystem(&btrfs_fs_type);
 	btrfs_exit_sysfs();
 	btrfs_cleanup_fs_uuids();

fs/btrfs/sysfs.c+177−9

@@ -21,6 +21,7 @@ · 20 unmodified lines
21	#include "space-info.h"	21	#include "space-info.h"
22	#include "block-group.h"	22	#include "block-group.h"
23	#include "qgroup.h"	23	#include "qgroup.h"
		24	#include "misc.h"
24		25
25	/*	26	/*
26	* Structure name Path	27	* Structure name Path
@@ -61,6 +62,10 @@ struct raid_kobject { · 34 unmodified lines
61	.store = _store, \	62	.store = _store, \
62	}	63	}
63		64
		65	#define BTRFS_ATTR_W(_prefix, _name, _store) \
		66	static struct kobj_attribute btrfs_attr_##_prefix##_##_name = \
		67	__INIT_KOBJ_ATTR(_name, 0200, NULL, _store)
		68
64	#define BTRFS_ATTR_RW(_prefix, _name, _show, _store) \	69	#define BTRFS_ATTR_RW(_prefix, _name, _show, _store) \
65	static struct kobj_attribute btrfs_attr_##_prefix##_##_name = \	70	static struct kobj_attribute btrfs_attr_##_prefix##_##_name = \
66	__INIT_KOBJ_ATTR(_name, 0644, _show, _store)	71	__INIT_KOBJ_ATTR(_name, 0644, _show, _store)
@@ -92,6 +97,7 @@ static struct btrfs_feature_attr btrfs_attr_features_##_name = { \ · 25 unmodified lines
92		97
93	static inline struct btrfs_fs_info to_fs_info(struct kobject kobj);	98	static inline struct btrfs_fs_info to_fs_info(struct kobject kobj);
94	static inline struct btrfs_fs_devices to_fs_devs(struct kobject kobj);	99	static inline struct btrfs_fs_devices to_fs_devs(struct kobject kobj);
		100	static struct kobject get_btrfs_kobj(struct kobject kobj);
95		101
96	static struct btrfs_feature_attr to_btrfs_feature_attr(struct kobj_attribute a)	102	static struct btrfs_feature_attr to_btrfs_feature_attr(struct kobj_attribute a)
97	{	103	{
@@ -270,12 +276,10 @@ static umode_t btrfs_feature_visible(struct kobject *kobj, · 172 unmodified lines
270	return mode;	276	return mode;
271	}	277	}
272		278
273	BTRFS_FEAT_ATTR_INCOMPAT(mixed_backref, MIXED_BACKREF);
274	BTRFS_FEAT_ATTR_INCOMPAT(default_subvol, DEFAULT_SUBVOL);	279	BTRFS_FEAT_ATTR_INCOMPAT(default_subvol, DEFAULT_SUBVOL);
275	BTRFS_FEAT_ATTR_INCOMPAT(mixed_groups, MIXED_GROUPS);	280	BTRFS_FEAT_ATTR_INCOMPAT(mixed_groups, MIXED_GROUPS);
276	BTRFS_FEAT_ATTR_INCOMPAT(compress_lzo, COMPRESS_LZO);	281	BTRFS_FEAT_ATTR_INCOMPAT(compress_lzo, COMPRESS_LZO);
277	BTRFS_FEAT_ATTR_INCOMPAT(compress_zstd, COMPRESS_ZSTD);	282	BTRFS_FEAT_ATTR_INCOMPAT(compress_zstd, COMPRESS_ZSTD);
278	BTRFS_FEAT_ATTR_INCOMPAT(big_metadata, BIG_METADATA);
279	BTRFS_FEAT_ATTR_INCOMPAT(extended_iref, EXTENDED_IREF);	283	BTRFS_FEAT_ATTR_INCOMPAT(extended_iref, EXTENDED_IREF);
280	BTRFS_FEAT_ATTR_INCOMPAT(raid56, RAID56);	284	BTRFS_FEAT_ATTR_INCOMPAT(raid56, RAID56);
281	BTRFS_FEAT_ATTR_INCOMPAT(skinny_metadata, SKINNY_METADATA);	285	BTRFS_FEAT_ATTR_INCOMPAT(skinny_metadata, SKINNY_METADATA);
@@ -283,9 +287,10 @@ BTRFS_FEAT_ATTR_INCOMPAT(no_holes, NO_HOLES); · 1 unmodified line
283	BTRFS_FEAT_ATTR_INCOMPAT(metadata_uuid, METADATA_UUID);	287	BTRFS_FEAT_ATTR_INCOMPAT(metadata_uuid, METADATA_UUID);
284	BTRFS_FEAT_ATTR_COMPAT_RO(free_space_tree, FREE_SPACE_TREE);	288	BTRFS_FEAT_ATTR_COMPAT_RO(free_space_tree, FREE_SPACE_TREE);
285	BTRFS_FEAT_ATTR_INCOMPAT(raid1c34, RAID1C34);	289	BTRFS_FEAT_ATTR_INCOMPAT(raid1c34, RAID1C34);
286	#ifdef CONFIG_BTRFS_DEBUG	290	#ifdef CONFIG_BLK_DEV_ZONED
287	/* Remove once support for zoned allocation is feature complete */
288	BTRFS_FEAT_ATTR_INCOMPAT(zoned, ZONED);	291	BTRFS_FEAT_ATTR_INCOMPAT(zoned, ZONED);
		292	#endif
		293	#ifdef CONFIG_BTRFS_DEBUG
289	/* Remove once support for extent tree v2 is feature complete */	294	/* Remove once support for extent tree v2 is feature complete */
290	BTRFS_FEAT_ATTR_INCOMPAT(extent_tree_v2, EXTENT_TREE_V2);	295	BTRFS_FEAT_ATTR_INCOMPAT(extent_tree_v2, EXTENT_TREE_V2);
291	#endif	296	#endif
@@ -296,17 +301,15 @@ BTRFS_FEAT_ATTR_COMPAT_RO(verity, VERITY); · 4 unmodified lines
296	/*	301	/*
297	* Features which depend on feature bits and may differ between each fs.	302	* Features which depend on feature bits and may differ between each fs.
298	*	303	*
299	* /sys/fs/btrfs/features - all available features implemeted by this version	304	* /sys/fs/btrfs/features - all available features implemented by this version
300	* /sys/fs/btrfs/UUID/features - features of the fs which are enabled or	305	* /sys/fs/btrfs/UUID/features - features of the fs which are enabled or
301	* can be changed on a mounted filesystem.	306	* can be changed on a mounted filesystem.
302	*/	307	*/
303	static struct attribute *btrfs_supported_feature_attrs[] = {	308	static struct attribute *btrfs_supported_feature_attrs[] = {
304	BTRFS_FEAT_ATTR_PTR(mixed_backref),
305	BTRFS_FEAT_ATTR_PTR(default_subvol),	309	BTRFS_FEAT_ATTR_PTR(default_subvol),
306	BTRFS_FEAT_ATTR_PTR(mixed_groups),	310	BTRFS_FEAT_ATTR_PTR(mixed_groups),
307	BTRFS_FEAT_ATTR_PTR(compress_lzo),	311	BTRFS_FEAT_ATTR_PTR(compress_lzo),
308	BTRFS_FEAT_ATTR_PTR(compress_zstd),	312	BTRFS_FEAT_ATTR_PTR(compress_zstd),
309	BTRFS_FEAT_ATTR_PTR(big_metadata),
310	BTRFS_FEAT_ATTR_PTR(extended_iref),	313	BTRFS_FEAT_ATTR_PTR(extended_iref),
311	BTRFS_FEAT_ATTR_PTR(raid56),	314	BTRFS_FEAT_ATTR_PTR(raid56),
312	BTRFS_FEAT_ATTR_PTR(skinny_metadata),	315	BTRFS_FEAT_ATTR_PTR(skinny_metadata),
@@ -314,8 +317,10 @@ static struct attribute *btrfs_supported_feature_attrs[] = { · 1 unmodified line
314	BTRFS_FEAT_ATTR_PTR(metadata_uuid),	317	BTRFS_FEAT_ATTR_PTR(metadata_uuid),
315	BTRFS_FEAT_ATTR_PTR(free_space_tree),	318	BTRFS_FEAT_ATTR_PTR(free_space_tree),
316	BTRFS_FEAT_ATTR_PTR(raid1c34),	319	BTRFS_FEAT_ATTR_PTR(raid1c34),
317	#ifdef CONFIG_BTRFS_DEBUG	320	#ifdef CONFIG_BLK_DEV_ZONED
318	BTRFS_FEAT_ATTR_PTR(zoned),	321	BTRFS_FEAT_ATTR_PTR(zoned),
		322	#endif
		323	#ifdef CONFIG_BTRFS_DEBUG
319	BTRFS_FEAT_ATTR_PTR(extent_tree_v2),	324	BTRFS_FEAT_ATTR_PTR(extent_tree_v2),
320	#endif	325	#endif
321	#ifdef CONFIG_FS_VERITY	326	#ifdef CONFIG_FS_VERITY
@@ -709,6 +714,112 @@ static ssize_t btrfs_space_info_show_##field(struct kobject *kobj, \ · 387 unmodified lines
709	} \	714	} \
710	BTRFS_ATTR(space_info, field, btrfs_space_info_show_##field)	715	BTRFS_ATTR(space_info, field, btrfs_space_info_show_##field)
711		716
		717	static ssize_t btrfs_chunk_size_show(struct kobject *kobj,
		718	struct kobj_attribute a, char buf)
		719	{
		720	struct btrfs_space_info *sinfo = to_space_info(kobj);
		721
		722	return sysfs_emit(buf, "%llu\n", READ_ONCE(sinfo->chunk_size));
		723	}
		724
		725	/*
		726	* Store new chunk size in space info. Can be called on a read-only filesystem.
		727	*
		728	* If the new chunk size value is larger than 10% of free space it is reduced
		729	* to match that limit. Alignment must be to 256M and the system chunk size
		730	* cannot be set.
		731	*/
		732	static ssize_t btrfs_chunk_size_store(struct kobject *kobj,
		733	struct kobj_attribute *a,
		734	const char *buf, size_t len)
		735	{
		736	struct btrfs_space_info *space_info = to_space_info(kobj);
		737	struct btrfs_fs_info *fs_info = to_fs_info(get_btrfs_kobj(kobj));
		738	char *retptr;
		739	u64 val;
		740
		741	if (!capable(CAP_SYS_ADMIN))
		742	return -EPERM;
		743
		744	if (!fs_info->fs_devices)
		745	return -EINVAL;
		746
		747	if (btrfs_is_zoned(fs_info))
		748	return -EINVAL;
		749
		750	/* System block type must not be changed. */
		751	if (space_info->flags & BTRFS_BLOCK_GROUP_SYSTEM)
		752	return -EPERM;
		753
		754	val = memparse(buf, &retptr);
		755	/* There could be trailing '\n', also catch any typos after the value */
		756	retptr = skip_spaces(retptr);
		757	if (*retptr != 0 \|\| val == 0)
		758	return -EINVAL;
		759
		760	val = min(val, BTRFS_MAX_DATA_CHUNK_SIZE);
		761
		762	/* Limit stripe size to 10% of available space. */
		763	val = min(div_factor(fs_info->fs_devices->total_rw_bytes, 1), val);
		764
		765	/* Must be multiple of 256M. */
		766	val &= ~((u64)SZ_256M - 1);
		767
		768	/* Must be at least 256M. */
		769	if (val < SZ_256M)
		770	return -EINVAL;
		771
		772	btrfs_update_space_info_chunk_size(space_info, val);
		773
		774	return len;
		775	}
		776
		777	#ifdef CONFIG_BTRFS_DEBUG
		778	/*
		779	* Request chunk allocation with current chunk size.
		780	*/
		781	static ssize_t btrfs_force_chunk_alloc_store(struct kobject *kobj,
		782	struct kobj_attribute *a,
		783	const char *buf, size_t len)
		784	{
		785	struct btrfs_space_info *space_info = to_space_info(kobj);
		786	struct btrfs_fs_info *fs_info = to_fs_info(get_btrfs_kobj(kobj));
		787	struct btrfs_trans_handle *trans;
		788	bool val;
		789	int ret;
		790
		791	if (!capable(CAP_SYS_ADMIN))
		792	return -EPERM;
		793
		794	if (sb_rdonly(fs_info->sb))
		795	return -EROFS;
		796
		797	ret = kstrtobool(buf, &val);
		798	if (ret)
		799	return ret;
		800
		801	if (!val)
		802	return -EINVAL;
		803
		804	/*
		805	* This is unsafe to be called from sysfs context and may cause
		806	* unexpected problems.
		807	*/
		808	trans = btrfs_start_transaction(fs_info->tree_root, 0);
		809	if (IS_ERR(trans))
		810	return PTR_ERR(trans);
		811	ret = btrfs_force_chunk_alloc(trans, space_info->flags);
		812	btrfs_end_transaction(trans);
		813
		814	if (ret == 1)
		815	return len;
		816
		817	return -ENOSPC;
		818	}
		819	BTRFS_ATTR_W(space_info, force_chunk_alloc, btrfs_force_chunk_alloc_store);
		820
		821	#endif
		822
712	SPACE_INFO_ATTR(flags);	823	SPACE_INFO_ATTR(flags);
713	SPACE_INFO_ATTR(total_bytes);	824	SPACE_INFO_ATTR(total_bytes);
714	SPACE_INFO_ATTR(bytes_used);	825	SPACE_INFO_ATTR(bytes_used);
@@ -719,6 +830,7 @@ SPACE_INFO_ATTR(bytes_readonly); · 4 unmodified lines
719	SPACE_INFO_ATTR(bytes_zone_unusable);	830	SPACE_INFO_ATTR(bytes_zone_unusable);
720	SPACE_INFO_ATTR(disk_used);	831	SPACE_INFO_ATTR(disk_used);
721	SPACE_INFO_ATTR(disk_total);	832	SPACE_INFO_ATTR(disk_total);
		833	BTRFS_ATTR_RW(space_info, chunk_size, btrfs_chunk_size_show, btrfs_chunk_size_store);
722		834
723	static ssize_t btrfs_sinfo_bg_reclaim_threshold_show(struct kobject *kobj,	835	static ssize_t btrfs_sinfo_bg_reclaim_threshold_show(struct kobject *kobj,
724	struct kobj_attribute *a,	836	struct kobj_attribute *a,
@@ -773,6 +885,10 @@ static struct attribute *space_info_attrs[] = { · 48 unmodified lines
773	BTRFS_ATTR_PTR(space_info, disk_used),	885	BTRFS_ATTR_PTR(space_info, disk_used),
774	BTRFS_ATTR_PTR(space_info, disk_total),	886	BTRFS_ATTR_PTR(space_info, disk_total),
775	BTRFS_ATTR_PTR(space_info, bg_reclaim_threshold),	887	BTRFS_ATTR_PTR(space_info, bg_reclaim_threshold),
		888	BTRFS_ATTR_PTR(space_info, chunk_size),
		889	#ifdef CONFIG_BTRFS_DEBUG
		890	BTRFS_ATTR_PTR(space_info, force_chunk_alloc),
		891	#endif
776	NULL,	892	NULL,
777	};	893	};
778	ATTRIBUTE_GROUPS(space_info);	894	ATTRIBUTE_GROUPS(space_info);
@@ -871,6 +987,48 @@ static ssize_t btrfs_sectorsize_show(struct kobject *kobj, · 92 unmodified lines
871		987
872	BTRFS_ATTR(, sectorsize, btrfs_sectorsize_show);	988	BTRFS_ATTR(, sectorsize, btrfs_sectorsize_show);
873		989
		990	static ssize_t btrfs_commit_stats_show(struct kobject *kobj,
		991	struct kobj_attribute a, char buf)
		992	{
		993	struct btrfs_fs_info *fs_info = to_fs_info(kobj);
		994
		995	return sysfs_emit(buf,
		996	"commits %llu\n"
		997	"last_commit_ms %llu\n"
		998	"max_commit_ms %llu\n"
		999	"total_commit_ms %llu\n",
		1000	fs_info->commit_stats.commit_count,
		1001	div_u64(fs_info->commit_stats.last_commit_dur, NSEC_PER_MSEC),
		1002	div_u64(fs_info->commit_stats.max_commit_dur, NSEC_PER_MSEC),
		1003	div_u64(fs_info->commit_stats.total_commit_dur, NSEC_PER_MSEC));
		1004	}
		1005
		1006	static ssize_t btrfs_commit_stats_store(struct kobject *kobj,
		1007	struct kobj_attribute *a,
		1008	const char *buf, size_t len)
		1009	{
		1010	struct btrfs_fs_info *fs_info = to_fs_info(kobj);
		1011	unsigned long val;
		1012	int ret;
		1013
		1014	if (!fs_info)
		1015	return -EPERM;
		1016
		1017	if (!capable(CAP_SYS_RESOURCE))
		1018	return -EPERM;
		1019
		1020	ret = kstrtoul(buf, 10, &val);
		1021	if (ret)
		1022	return ret;
		1023	if (val)
		1024	return -EINVAL;
		1025
		1026	WRITE_ONCE(fs_info->commit_stats.max_commit_dur, 0);
		1027
		1028	return len;
		1029	}
		1030	BTRFS_ATTR_RW(, commit_stats, btrfs_commit_stats_show, btrfs_commit_stats_store);
		1031
874	static ssize_t btrfs_clone_alignment_show(struct kobject *kobj,	1032	static ssize_t btrfs_clone_alignment_show(struct kobject *kobj,
875	struct kobj_attribute a, char buf)	1033	struct kobj_attribute a, char buf)
876	{	1034	{
@@ -1110,6 +1268,7 @@ static const struct attribute *btrfs_attrs[] = { · 233 unmodified lines
1110	BTRFS_ATTR_PTR(, generation),	1268	BTRFS_ATTR_PTR(, generation),
1111	BTRFS_ATTR_PTR(, read_policy),	1269	BTRFS_ATTR_PTR(, read_policy),
1112	BTRFS_ATTR_PTR(, bg_reclaim_threshold),	1270	BTRFS_ATTR_PTR(, bg_reclaim_threshold),
		1271	BTRFS_ATTR_PTR(, commit_stats),
1113	NULL,	1272	NULL,
1114	};	1273	};
1115		1274
@@ -1140,6 +1299,16 @@ static inline struct btrfs_fs_info to_fs_info(struct kobject kobj) · 24 unmodified lines
1140	return to_fs_devs(kobj)->fs_info;	1299	return to_fs_devs(kobj)->fs_info;
1141	}	1300	}
1142		1301
		1302	static struct kobject get_btrfs_kobj(struct kobject kobj)
		1303	{
		1304	while (kobj) {
		1305	if (kobj->ktype == &btrfs_ktype)
		1306	return kobj;
		1307	kobj = kobj->parent;
		1308	}
		1309	return NULL;
		1310	}
		1311
1143	#define NUM_FEATURE_BITS 64	1312	#define NUM_FEATURE_BITS 64
1144	#define BTRFS_FEATURE_NAME_MAX 13	1313	#define BTRFS_FEATURE_NAME_MAX 13
1145	static char btrfs_unknown_feature_names[FEAT_MAX][NUM_FEATURE_BITS][BTRFS_FEATURE_NAME_MAX];	1314	static char btrfs_unknown_feature_names[FEAT_MAX][NUM_FEATURE_BITS][BTRFS_FEATURE_NAME_MAX];
@@ -2106,4 +2275,3 @@ void __cold btrfs_exit_sysfs(void) · 960 unmodified lines
2106	#endif	2275	#endif
2107	kset_unregister(btrfs_kset);	2276	kset_unregister(btrfs_kset);
2108	}	2277	}
2109

fs/btrfs/tests/btrfs-tests.c+1−0

@@ -59,6 +59,7 @@ struct inode *btrfs_new_test_inode(void) · 58 unmodified lines
59	return NULL;	59	return NULL;
60		60
61	inode->i_mode = S_IFREG;	61	inode->i_mode = S_IFREG;
		62	inode->i_ino = BTRFS_FIRST_FREE_OBJECTID;
62	BTRFS_I(inode)->location.type = BTRFS_INODE_ITEM_KEY;	63	BTRFS_I(inode)->location.type = BTRFS_INODE_ITEM_KEY;
63	BTRFS_I(inode)->location.objectid = BTRFS_FIRST_FREE_OBJECTID;	64	BTRFS_I(inode)->location.objectid = BTRFS_FIRST_FREE_OBJECTID;
64	BTRFS_I(inode)->location.offset = 0;	65	BTRFS_I(inode)->location.offset = 0;

fs/btrfs/tests/extent-buffer-tests.c+2−1

@@ -47,7 +47,8 @@ static int test_btrfs_split_item(u32 sectorsize, u32 nodesize) · 46 unmodified lines
47	goto out;	47	goto out;
48	}	48	}
49		49
50	path->nodes[0] = eb = alloc_dummy_extent_buffer(fs_info, nodesize);	50	eb = alloc_dummy_extent_buffer(fs_info, nodesize);
		51	path->nodes[0] = eb;
51	if (!eb) {	52	if (!eb) {
52	test_std_err(TEST_ALLOC_EXTENT_BUFFER);	53	test_std_err(TEST_ALLOC_EXTENT_BUFFER);
53	ret = -ENOMEM;	54	ret = -ENOMEM;

fs/btrfs/transaction.c+24−2

 #include <linux/pagemap.h>
 #include <linux/blkdev.h>
 #include <linux/uuid.h>
+#include <linux/timekeeping.h>
 #include "misc.h"
 #include "ctree.h"
 #include "disk-io.h"
 	btrfs_i_size_write(BTRFS_I(parent_inode), parent_inode->i_size +
 					 dentry->d_name.len * 2);
-	parent_inode->i_mtime = parent_inode->i_ctime =
+	parent_inode->i_mtime = current_time(parent_inode);
-		current_time(parent_inode);
+	parent_inode->i_ctime = parent_inode->i_mtime;
 	ret = btrfs_update_inode_fallback(trans, parent_root, BTRFS_I(parent_inode));
 	if (ret) {
 		btrfs_abort_transaction(trans, ret);
 	list_add(&trans->pending_snapshot->list, &cur_trans->pending_snapshots);
 }
+static void update_commit_stats(struct btrfs_fs_info *fs_info, ktime_t interval)
+{
+	fs_info->commit_stats.commit_count++;
+	fs_info->commit_stats.last_commit_dur = interval;
+	fs_info->commit_stats.max_commit_dur =
+			max_t(u64, fs_info->commit_stats.max_commit_dur, interval);
+	fs_info->commit_stats.total_commit_dur += interval;
+}
 int btrfs_commit_transaction(struct btrfs_trans_handle *trans)
 {
 	struct btrfs_fs_info *fs_info = trans->fs_info;
 	struct btrfs_transaction *cur_trans = trans->transaction;
 	struct btrfs_transaction *prev_trans = NULL;
 	int ret;
+	ktime_t start_time;
+	ktime_t interval;
 	ASSERT(refcount_read(&trans->use_count) == 1);
 		}
 	}
+	/*
+	 * Get the time spent on the work done by the commit thread and not
+	 * the time spent waiting on a previous commit
+	 */
+	start_time = ktime_get_ns();
 	extwriter_counter_dec(cur_trans, trans->type);
 	ret = btrfs_start_delalloc_flush(fs_info);
 	trace_btrfs_transaction_commit(fs_info);
+	interval = ktime_get_ns() - start_time;
 	btrfs_scrub_continue(fs_info);
 	if (current->journal_info == trans)
 	kmem_cache_free(btrfs_trans_handle_cachep, trans);
+	update_commit_stats(fs_info, interval);
 	return ret;
 unlock_reloc:

fs/btrfs/tree-log.c+18−11

 		int index = (root->log_transid + 1) % 2;
 		if (btrfs_need_log_full_commit(trans)) {
-			ret = -EAGAIN;
+			ret = BTRFS_LOG_FORCE_COMMIT;
 			goto out;
 		}
 		 * writing.
 		 */
 		if (zoned && !created) {
-			ret = -EAGAIN;
+			ret = BTRFS_LOG_FORCE_COMMIT;
 			goto out;
 		}
 	struct btrfs_key location;
 	/*
-	 * Currenly we only log dir index keys. Even if we replay a log created
+	 * Currently we only log dir index keys. Even if we replay a log created
 	 * by an older kernel that logged both dir index and dir item keys, all
 	 * we need to do is process the dir index keys, we (and our caller) can
 	 * safely ignore dir item keys (key type BTRFS_DIR_ITEM_KEY).
 	/* bail out if we need to do a full commit */
 	if (btrfs_need_log_full_commit(trans)) {
-		ret = -EAGAIN;
+		ret = BTRFS_LOG_FORCE_COMMIT;
 		mutex_unlock(&root->log_mutex);
 		goto out;
 	}
 		}
 		btrfs_wait_tree_log_extents(log, mark);
 		mutex_unlock(&log_root_tree->log_mutex);
-		ret = -EAGAIN;
+		ret = BTRFS_LOG_FORCE_COMMIT;
 		goto out;
 	}
 		blk_finish_plug(&plug);
 		btrfs_wait_tree_log_extents(log, mark);
 		mutex_unlock(&log_root_tree->log_mutex);
-		ret = -EAGAIN;
+		ret = BTRFS_LOG_FORCE_COMMIT;
 		goto out_wake_log_root;
 	}
 	    inode_only == LOG_INODE_ALL &&
 	    inode->last_unlink_trans >= trans->transid) {
 		btrfs_set_log_full_commit(trans);
-		ret = 1;
+		ret = BTRFS_LOG_FORCE_COMMIT;
 		goto out_unlock;
 	}
 	bool log_dentries = false;
 	if (btrfs_test_opt(fs_info, NOTREELOG)) {
-		ret = 1;
+		ret = BTRFS_LOG_FORCE_COMMIT;
 		goto end_no_trans;
 	}
 	if (btrfs_root_refs(&root->root_item) == 0) {
-		ret = 1;
+		ret = BTRFS_LOG_FORCE_COMMIT;
 		goto end_no_trans;
 	}
 end_trans:
 	if (ret < 0) {
 		btrfs_set_log_full_commit(trans);
-		ret = 1;
+		ret = BTRFS_LOG_FORCE_COMMIT;
 	}
 	if (ret)
 		 * anyone from syncing the log until we have updated both inodes
 		 * in the log.
 		 */
+		ret = join_running_log_trans(root);
+		/*
+		 * At least one of the inodes was logged before, so this should
+		 * not fail, but if it does, it's not serious, just bail out and
+		 * mark the log for a full commit.
+		 */
+		if (WARN_ON_ONCE(ret < 0))
+			goto out;
 		log_pinned = true;
-		btrfs_pin_log_trans(root);
 		path = btrfs_alloc_path();
 		if (!path) {

fs/btrfs/tree-log.h+3−0

@@ -12,6 +12,9 @@ · 11 unmodified lines
12	/* return value for btrfs_log_dentry_safe that means we don't need to log it at all */	12	/* return value for btrfs_log_dentry_safe that means we don't need to log it at all */
13	#define BTRFS_NO_LOG_SYNC 256	13	#define BTRFS_NO_LOG_SYNC 256
14		14
		15	/* We can't use the tree log for whatever reason, force a transaction commit */
		16	#define BTRFS_LOG_FORCE_COMMIT (1)
		17
15	struct btrfs_log_ctx {	18	struct btrfs_log_ctx {
16	int log_ret;	19	int log_ret;
17	int log_transid;	20	int log_transid;

fs/btrfs/volumes.c+172−190

@@ -182,6 +182,13 @@ const char *btrfs_bg_type_to_raid_name(u64 flags) · 181 unmodified lines
182	return btrfs_raid_array[index].raid_name;	182	return btrfs_raid_array[index].raid_name;
183	}	183	}
184		184
		185	int btrfs_nr_parity_stripes(u64 type)
		186	{
		187	enum btrfs_raid_types index = btrfs_bg_flags_to_raid_index(type);
		188
		189	return btrfs_raid_array[index].nparity;
		190	}
		191
185	/*	192	/*
186	* Fill @buf with textual description of @bg_flags, no more than @size_buf	193	* Fill @buf with textual description of @bg_flags, no more than @size_buf
187	* bytes including terminating null byte.	194	* bytes including terminating null byte.
@@ -238,7 +245,6 @@ out_overflow:; · 50 unmodified lines
238		245
239	static int init_first_rw_device(struct btrfs_trans_handle *trans);	246	static int init_first_rw_device(struct btrfs_trans_handle *trans);
240	static int btrfs_relocate_sys_chunks(struct btrfs_fs_info *fs_info);	247	static int btrfs_relocate_sys_chunks(struct btrfs_fs_info *fs_info);
241	static void btrfs_dev_stat_print_on_error(struct btrfs_device *dev);
242	static void btrfs_dev_stat_print_on_load(struct btrfs_device *device);	248	static void btrfs_dev_stat_print_on_load(struct btrfs_device *device);
243	static int __btrfs_map_block(struct btrfs_fs_info *fs_info,	249	static int __btrfs_map_block(struct btrfs_fs_info *fs_info,
244	enum btrfs_map_op op,	250	enum btrfs_map_op op,
@@ -1396,12 +1402,7 @@ static u64 dev_extent_search_start(struct btrfs_device *device, u64 start) · 1151 unmodified lines
1396	{	1402	{
1397	switch (device->fs_devices->chunk_alloc_policy) {	1403	switch (device->fs_devices->chunk_alloc_policy) {
1398	case BTRFS_CHUNK_ALLOC_REGULAR:	1404	case BTRFS_CHUNK_ALLOC_REGULAR:
1399	/*	1405	return max_t(u64, start, BTRFS_DEVICE_RANGE_RESERVED);
1400	* We don't want to overwrite the superblock on the drive nor
1401	* any area used by the boot loader (grub for example), so we
1402	* make sure to start at an offset of at least 1MB.
1403	*/
1404	return max_t(u64, start, SZ_1M);
1405	case BTRFS_CHUNK_ALLOC_ZONED:	1406	case BTRFS_CHUNK_ALLOC_ZONED:
1406	/*	1407	/*
1407	* We don't care about the starting region like regular	1408	* We don't care about the starting region like regular
@@ -5071,26 +5072,16 @@ static void init_alloc_chunk_ctl_policy_regular( · 3663 unmodified lines
5071	struct btrfs_fs_devices *fs_devices,	5072	struct btrfs_fs_devices *fs_devices,
5072	struct alloc_chunk_ctl *ctl)	5073	struct alloc_chunk_ctl *ctl)
5073	{	5074	{
5074	u64 type = ctl->type;	5075	struct btrfs_space_info *space_info;
5075		5076
5076	if (type & BTRFS_BLOCK_GROUP_DATA) {	5077	space_info = btrfs_find_space_info(fs_devices->fs_info, ctl->type);
5077	ctl->max_stripe_size = SZ_1G;	5078	ASSERT(space_info);
5078	ctl->max_chunk_size = BTRFS_MAX_DATA_CHUNK_SIZE;	5079
5079	} else if (type & BTRFS_BLOCK_GROUP_METADATA) {	5080	ctl->max_chunk_size = READ_ONCE(space_info->chunk_size);
5080	/* For larger filesystems, use larger metadata chunks */	5081	ctl->max_stripe_size = ctl->max_chunk_size;
5081	if (fs_devices->total_rw_bytes > 50ULL * SZ_1G)	5082
5082	ctl->max_stripe_size = SZ_1G;	5083	if (ctl->type & BTRFS_BLOCK_GROUP_SYSTEM)
5083	else	5084	ctl->devs_max = min_t(int, ctl->devs_max, BTRFS_MAX_DEVS_SYS_CHUNK);
5084	ctl->max_stripe_size = SZ_256M;
5085	ctl->max_chunk_size = ctl->max_stripe_size;
5086	} else if (type & BTRFS_BLOCK_GROUP_SYSTEM) {
5087	ctl->max_stripe_size = SZ_32M;
5088	ctl->max_chunk_size = 2 * ctl->max_stripe_size;
5089	ctl->devs_max = min_t(int, ctl->devs_max,
5090	BTRFS_MAX_DEVS_SYS_CHUNK);
5091	} else {
5092	BUG();
5093	}
5094		5085
5095	/* We don't want a chunk larger than 10% of writable space */	5086	/* We don't want a chunk larger than 10% of writable space */
5096	ctl->max_chunk_size = min(div_factor(fs_devices->total_rw_bytes, 1),	5087	ctl->max_chunk_size = min(div_factor(fs_devices->total_rw_bytes, 1),
@@ -5720,7 +5711,8 @@ int btrfs_num_copies(struct btrfs_fs_info *fs_info, u64 logical, u64 len) · 623 unmodified lines
5720	{	5711	{
5721	struct extent_map *em;	5712	struct extent_map *em;
5722	struct map_lookup *map;	5713	struct map_lookup *map;
5723	int ret;	5714	enum btrfs_raid_types index;
		5715	int ret = 1;
5724		5716
5725	em = btrfs_get_chunk_map(fs_info, logical, len);	5717	em = btrfs_get_chunk_map(fs_info, logical, len);
5726	if (IS_ERR(em))	5718	if (IS_ERR(em))
@@ -5733,10 +5725,11 @@ int btrfs_num_copies(struct btrfs_fs_info *fs_info, u64 logical, u64 len) · 6 unmodified lines
5733	return 1;	5725	return 1;
5734		5726
5735	map = em->map_lookup;	5727	map = em->map_lookup;
5736	if (map->type & (BTRFS_BLOCK_GROUP_DUP \| BTRFS_BLOCK_GROUP_RAID1_MASK))	5728	index = btrfs_bg_flags_to_raid_index(map->type);
5737	ret = map->num_stripes;	5729
5738	else if (map->type & BTRFS_BLOCK_GROUP_RAID10)	5730	/* Non-RAID56, use their ncopies from btrfs_raid_array. */
5739	ret = map->sub_stripes;	5731	if (!(map->type & BTRFS_BLOCK_GROUP_RAID56_MASK))
		5732	ret = btrfs_raid_array[index].ncopies;
5740	else if (map->type & BTRFS_BLOCK_GROUP_RAID5)	5733	else if (map->type & BTRFS_BLOCK_GROUP_RAID5)
5741	ret = 2;	5734	ret = 2;
5742	else if (map->type & BTRFS_BLOCK_GROUP_RAID6)	5735	else if (map->type & BTRFS_BLOCK_GROUP_RAID6)
@@ -5748,8 +5741,6 @@ int btrfs_num_copies(struct btrfs_fs_info *fs_info, u64 logical, u64 len) · 5 unmodified lines
5748	* stripe under reconstruction.	5741	* stripe under reconstruction.
5749	*/	5742	*/
5750	ret = map->num_stripes;	5743	ret = map->num_stripes;
5751	else
5752	ret = 1;
5753	free_extent_map(em);	5744	free_extent_map(em);
5754		5745
5755	down_read(&fs_info->dev_replace.rwsem);	5746	down_read(&fs_info->dev_replace.rwsem);
@@ -5768,6 +5759,9 @@ unsigned long btrfs_full_stripe_len(struct btrfs_fs_info *fs_info, · 12 unmodified lines
5768	struct map_lookup *map;	5759	struct map_lookup *map;
5769	unsigned long len = fs_info->sectorsize;	5760	unsigned long len = fs_info->sectorsize;
5770		5761
		5762	if (!btrfs_fs_incompat(fs_info, RAID56))
		5763	return len;
		5764
5771	em = btrfs_get_chunk_map(fs_info, logical, len);	5765	em = btrfs_get_chunk_map(fs_info, logical, len);
5772		5766
5773	if (!WARN_ON(IS_ERR(em))) {	5767	if (!WARN_ON(IS_ERR(em))) {
@@ -5785,6 +5779,9 @@ int btrfs_is_parity_mirror(struct btrfs_fs_info *fs_info, u64 logical, u64 len) · 11 unmodified lines
5785	struct map_lookup *map;	5779	struct map_lookup *map;
5786	int ret = 0;	5780	int ret = 0;
5787		5781
		5782	if (!btrfs_fs_incompat(fs_info, RAID56))
		5783	return 0;
		5784
5788	em = btrfs_get_chunk_map(fs_info, logical, len);	5785	em = btrfs_get_chunk_map(fs_info, logical, len);
5789		5786
5790	if(!WARN_ON(IS_ERR(em))) {	5787	if(!WARN_ON(IS_ERR(em))) {
@@ -5917,18 +5914,17 @@ void btrfs_put_bioc(struct btrfs_io_context *bioc) · 126 unmodified lines
5917	kfree(bioc);	5914	kfree(bioc);
5918	}	5915	}
5919		5916
5920	/* can REQ_OP_DISCARD be sent with other REQ like REQ_OP_WRITE? */
5921	/*	5917	/*
5922	* Please note that, discard won't be sent to target device of device	5918	* Please note that, discard won't be sent to target device of device
5923	* replace.	5919	* replace.
5924	*/	5920	*/
5925	static int __btrfs_map_block_for_discard(struct btrfs_fs_info *fs_info,	5921	struct btrfs_discard_stripe btrfs_map_discard(struct btrfs_fs_info fs_info,
5926	u64 logical, u64 *length_ret,	5922	u64 logical, u64 *length_ret,
5927	struct btrfs_io_context **bioc_ret)	5923	u32 *num_stripes)
5928	{	5924	{
5929	struct extent_map *em;	5925	struct extent_map *em;
5930	struct map_lookup *map;	5926	struct map_lookup *map;
5931	struct btrfs_io_context *bioc;	5927	struct btrfs_discard_stripe *stripes;
5932	u64 length = *length_ret;	5928	u64 length = *length_ret;
5933	u64 offset;	5929	u64 offset;
5934	u64 stripe_nr;	5930	u64 stripe_nr;
@@ -5937,29 +5933,26 @@ static int __btrfs_map_block_for_discard(struct btrfs_fs_info *fs_info, · 2 unmodified lines
5937	u64 stripe_cnt;	5933	u64 stripe_cnt;
5938	u64 stripe_len;	5934	u64 stripe_len;
5939	u64 stripe_offset;	5935	u64 stripe_offset;
5940	u64 num_stripes;
5941	u32 stripe_index;	5936	u32 stripe_index;
5942	u32 factor = 0;	5937	u32 factor = 0;
5943	u32 sub_stripes = 0;	5938	u32 sub_stripes = 0;
5944	u64 stripes_per_dev = 0;	5939	u64 stripes_per_dev = 0;
5945	u32 remaining_stripes = 0;	5940	u32 remaining_stripes = 0;
5946	u32 last_stripe = 0;	5941	u32 last_stripe = 0;
5947	int ret = 0;	5942	int ret;
5948	int i;	5943	int i;
5949		5944
5950	/* Discard always returns a bioc. */
5951	ASSERT(bioc_ret);
5952
5953	em = btrfs_get_chunk_map(fs_info, logical, length);	5945	em = btrfs_get_chunk_map(fs_info, logical, length);
5954	if (IS_ERR(em))	5946	if (IS_ERR(em))
5955	return PTR_ERR(em);	5947	return ERR_CAST(em);
5956		5948
5957	map = em->map_lookup;	5949	map = em->map_lookup;
		5950
5958	/* we don't discard raid56 yet */	5951	/* we don't discard raid56 yet */
5959	if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {	5952	if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
5960	ret = -EOPNOTSUPP;	5953	ret = -EOPNOTSUPP;
5961	goto out;	5954	goto out_free_map;
5962	}	5955	}
5963		5956
5964	offset = logical - em->start;	5957	offset = logical - em->start;
5965	length = min_t(u64, em->start + em->len - logical, length);	5958	length = min_t(u64, em->start + em->len - logical, length);
@@ -5985,7 +5978,7 @@ static int __btrfs_map_block_for_discard(struct btrfs_fs_info *fs_info, · 19 unmodified lines
5985	* device we have to walk to find the data, and stripe_index is	5978	* device we have to walk to find the data, and stripe_index is
5986	* the number of our device in the stripe array	5979	* the number of our device in the stripe array
5987	*/	5980	*/
5988	num_stripes = 1;	5981	*num_stripes = 1;
5989	stripe_index = 0;	5982	stripe_index = 0;
5990	if (map->type & (BTRFS_BLOCK_GROUP_RAID0 \|	5983	if (map->type & (BTRFS_BLOCK_GROUP_RAID0 \|
5991	BTRFS_BLOCK_GROUP_RAID10)) {	5984	BTRFS_BLOCK_GROUP_RAID10)) {
@@ -5995,7 +5988,7 @@ static int __btrfs_map_block_for_discard(struct btrfs_fs_info *fs_info, · 3 unmodified lines
5995	sub_stripes = map->sub_stripes;	5988	sub_stripes = map->sub_stripes;
5996		5989
5997	factor = map->num_stripes / sub_stripes;	5990	factor = map->num_stripes / sub_stripes;
5998	num_stripes = min_t(u64, map->num_stripes,	5991	*num_stripes = min_t(u64, map->num_stripes,
5999	sub_stripes * stripe_cnt);	5992	sub_stripes * stripe_cnt);
6000	stripe_nr = div_u64_rem(stripe_nr, factor, &stripe_index);	5993	stripe_nr = div_u64_rem(stripe_nr, factor, &stripe_index);
6001	stripe_index *= sub_stripes;	5994	stripe_index *= sub_stripes;
@@ -6005,31 +5998,30 @@ static int __btrfs_map_block_for_discard(struct btrfs_fs_info *fs_info, · 3 unmodified lines
6005	last_stripe *= sub_stripes;	5998	last_stripe *= sub_stripes;
6006	} else if (map->type & (BTRFS_BLOCK_GROUP_RAID1_MASK \|	5999	} else if (map->type & (BTRFS_BLOCK_GROUP_RAID1_MASK \|
6007	BTRFS_BLOCK_GROUP_DUP)) {	6000	BTRFS_BLOCK_GROUP_DUP)) {
6008	num_stripes = map->num_stripes;	6001	*num_stripes = map->num_stripes;
6009	} else {	6002	} else {
6010	stripe_nr = div_u64_rem(stripe_nr, map->num_stripes,	6003	stripe_nr = div_u64_rem(stripe_nr, map->num_stripes,
6011	&stripe_index);	6004	&stripe_index);
6012	}	6005	}
6013		6006
6014	bioc = alloc_btrfs_io_context(fs_info, num_stripes, 0);	6007	stripes = kcalloc(num_stripes, sizeof(stripes), GFP_NOFS);
6015	if (!bioc) {	6008	if (!stripes) {
6016	ret = -ENOMEM;	6009	ret = -ENOMEM;
6017	goto out;	6010	goto out_free_map;
6018	}	6011	}
6019		6012
6020	for (i = 0; i < num_stripes; i++) {	6013	for (i = 0; i < *num_stripes; i++) {
6021	bioc->stripes[i].physical =	6014	stripes[i].physical =
6022	map->stripes[stripe_index].physical +	6015	map->stripes[stripe_index].physical +
6023	stripe_offset + stripe_nr * map->stripe_len;	6016	stripe_offset + stripe_nr * map->stripe_len;
6024	bioc->stripes[i].dev = map->stripes[stripe_index].dev;	6017	stripes[i].dev = map->stripes[stripe_index].dev;
6025		6018
6026	if (map->type & (BTRFS_BLOCK_GROUP_RAID0 \|	6019	if (map->type & (BTRFS_BLOCK_GROUP_RAID0 \|
6027	BTRFS_BLOCK_GROUP_RAID10)) {	6020	BTRFS_BLOCK_GROUP_RAID10)) {
6028	bioc->stripes[i].length = stripes_per_dev *	6021	stripes[i].length = stripes_per_dev * map->stripe_len;
6029	map->stripe_len;
6030		6022
6031	if (i / sub_stripes < remaining_stripes)	6023	if (i / sub_stripes < remaining_stripes)
6032	bioc->stripes[i].length += map->stripe_len;	6024	stripes[i].length += map->stripe_len;
6033		6025
6034	/*	6026	/*
6035	* Special for the first stripe and	6027	* Special for the first stripe and
@@ -6040,17 +6032,17 @@ static int __btrfs_map_block_for_discard(struct btrfs_fs_info *fs_info, · 4 unmodified lines
6040	* off end_off	6032	* off end_off
6041	*/	6033	*/
6042	if (i < sub_stripes)	6034	if (i < sub_stripes)
6043	bioc->stripes[i].length -= stripe_offset;	6035	stripes[i].length -= stripe_offset;
6044		6036
6045	if (stripe_index >= last_stripe &&	6037	if (stripe_index >= last_stripe &&
6046	stripe_index <= (last_stripe +	6038	stripe_index <= (last_stripe +
6047	sub_stripes - 1))	6039	sub_stripes - 1))
6048	bioc->stripes[i].length -= stripe_end_offset;	6040	stripes[i].length -= stripe_end_offset;
6049		6041
6050	if (i == sub_stripes - 1)	6042	if (i == sub_stripes - 1)
6051	stripe_offset = 0;	6043	stripe_offset = 0;
6052	} else {	6044	} else {
6053	bioc->stripes[i].length = length;	6045	stripes[i].length = length;
6054	}	6046	}
6055		6047
6056	stripe_index++;	6048	stripe_index++;
@@ -6060,12 +6052,11 @@ static int __btrfs_map_block_for_discard(struct btrfs_fs_info *fs_info, · 3 unmodified lines
6060	}	6052	}
6061	}	6053	}
6062		6054
6063	*bioc_ret = bioc;
6064	bioc->map_type = map->type;
6065	bioc->num_stripes = num_stripes;
6066	out:
6067	free_extent_map(em);	6055	free_extent_map(em);
6068	return ret;	6056	return stripes;
		6057	out_free_map:
		6058	free_extent_map(em);
		6059	return ERR_PTR(ret);
6069	}	6060	}
6070		6061
6071	/*	6062	/*
@@ -6208,7 +6199,6 @@ static void handle_ops_on_dev_replace(enum btrfs_map_op op, · 136 unmodified lines
6208	bioc->stripes + i;	6199	bioc->stripes + i;
6209		6200
6210	new->physical = old->physical;	6201	new->physical = old->physical;
6211	new->length = old->length;
6212	new->dev = dev_replace->tgtdev;	6202	new->dev = dev_replace->tgtdev;
6213	bioc->tgtdev_map[i] = index_where_to_add;	6203	bioc->tgtdev_map[i] = index_where_to_add;
6214	index_where_to_add++;	6204	index_where_to_add++;
@@ -6249,8 +6239,6 @@ static void handle_ops_on_dev_replace(enum btrfs_map_op op, · 34 unmodified lines
6249	bioc->stripes + num_stripes;	6239	bioc->stripes + num_stripes;
6250		6240
6251	tgtdev_stripe->physical = physical_of_found;	6241	tgtdev_stripe->physical = physical_of_found;
6252	tgtdev_stripe->length =
6253	bioc->stripes[index_srcdev].length;
6254	tgtdev_stripe->dev = dev_replace->tgtdev;	6242	tgtdev_stripe->dev = dev_replace->tgtdev;
6255	bioc->tgtdev_map[index_srcdev] = num_stripes;	6243	bioc->tgtdev_map[index_srcdev] = num_stripes;
6256		6244
@@ -6472,6 +6460,7 @@ static int __btrfs_map_block(struct btrfs_fs_info *fs_info, · 215 unmodified lines
6472	}	6460	}
6473		6461
6474	} else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {	6462	} else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
		6463	ASSERT(map->stripe_len == BTRFS_STRIPE_LEN);
6475	if (need_raid_map && (need_full_stripe(op) \|\| mirror_num > 1)) {	6464	if (need_raid_map && (need_full_stripe(op) \|\| mirror_num > 1)) {
6476	/* push stripe_nr back to the start of the full stripe */	6465	/* push stripe_nr back to the start of the full stripe */
6477	stripe_nr = div64_u64(raid56_full_stripe_start,	6466	stripe_nr = div64_u64(raid56_full_stripe_start,
@@ -6479,9 +6468,12 @@ static int __btrfs_map_block(struct btrfs_fs_info *fs_info, · 1 unmodified line
6479		6468
6480	/* RAID[56] write or recovery. Return all stripes */	6469	/* RAID[56] write or recovery. Return all stripes */
6481	num_stripes = map->num_stripes;	6470	num_stripes = map->num_stripes;
6482	max_errors = nr_parity_stripes(map);	6471	max_errors = btrfs_chunk_max_errors(map);
6483		6472
6484	*length = map->stripe_len;	6473	/* Return the length to the full stripe end */
		6474	length = min(logical + length,
		6475	raid56_full_stripe_start + em->start +
		6476	data_stripes * stripe_len) - logical;
6485	stripe_index = 0;	6477	stripe_index = 0;
6486	stripe_offset = 0;	6478	stripe_offset = 0;
6487	} else {	6479	} else {
@@ -6604,10 +6596,6 @@ int btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op, · 116 unmodified lines
6604	u64 logical, u64 *length,	6596	u64 logical, u64 *length,
6605	struct btrfs_io_context **bioc_ret, int mirror_num)	6597	struct btrfs_io_context **bioc_ret, int mirror_num)
6606	{	6598	{
6607	if (op == BTRFS_MAP_DISCARD)
6608	return __btrfs_map_block_for_discard(fs_info, logical,
6609	length, bioc_ret);
6610
6611	return __btrfs_map_block(fs_info, op, logical, length, bioc_ret,	6599	return __btrfs_map_block(fs_info, op, logical, length, bioc_ret,
6612	mirror_num, 0);	6600	mirror_num, 0);
6613	}	6601	}
@@ -6620,77 +6608,106 @@ int btrfs_map_sblock(struct btrfs_fs_info *fs_info, enum btrfs_map_op op, · 6 unmodified lines
6620	return __btrfs_map_block(fs_info, op, logical, length, bioc_ret, 0, 1);	6608	return __btrfs_map_block(fs_info, op, logical, length, bioc_ret, 0, 1);
6621	}	6609	}
6622		6610
6623	static inline void btrfs_end_bioc(struct btrfs_io_context bioc, struct bio bio)	6611	static struct workqueue_struct btrfs_end_io_wq(struct btrfs_io_context bioc)
		6612	{
		6613	if (bioc->orig_bio->bi_opf & REQ_META)
		6614	return bioc->fs_info->endio_meta_workers;
		6615	return bioc->fs_info->endio_workers;
		6616	}
		6617
		6618	static void btrfs_end_bio_work(struct work_struct *work)
		6619	{
		6620	struct btrfs_bio *bbio =
		6621	container_of(work, struct btrfs_bio, end_io_work);
		6622
		6623	bio_endio(&bbio->bio);
		6624	}
		6625
		6626	static void btrfs_end_bioc(struct btrfs_io_context *bioc, bool async)
6624	{	6627	{
6625	bio->bi_private = bioc->private;	6628	struct bio *orig_bio = bioc->orig_bio;
6626	bio->bi_end_io = bioc->end_io;	6629	struct btrfs_bio *bbio = btrfs_bio(orig_bio);
6627	bio_endio(bio);	6630
		6631	bbio->mirror_num = bioc->mirror_num;
		6632	orig_bio->bi_private = bioc->private;
		6633	orig_bio->bi_end_io = bioc->end_io;
		6634
		6635	/*
		6636	* Only send an error to the higher layers if it is beyond the tolerance
		6637	* threshold.
		6638	*/
		6639	if (atomic_read(&bioc->error) > bioc->max_errors)
		6640	orig_bio->bi_status = BLK_STS_IOERR;
		6641	else
		6642	orig_bio->bi_status = BLK_STS_OK;
		6643
		6644	if (btrfs_op(orig_bio) == BTRFS_MAP_READ && async) {
		6645	INIT_WORK(&bbio->end_io_work, btrfs_end_bio_work);
		6646	queue_work(btrfs_end_io_wq(bioc), &bbio->end_io_work);
		6647	} else {
		6648	bio_endio(orig_bio);
		6649	}
6628		6650
6629	btrfs_put_bioc(bioc);	6651	btrfs_put_bioc(bioc);
6630	}	6652	}
6631		6653
6632	static void btrfs_end_bio(struct bio *bio)	6654	static void btrfs_end_bio(struct bio *bio)
6633	{	6655	{
6634	struct btrfs_io_context *bioc = bio->bi_private;	6656	struct btrfs_io_stripe *stripe = bio->bi_private;
6635	int is_orig_bio = 0;	6657	struct btrfs_io_context *bioc = stripe->bioc;
6636		6658
6637	if (bio->bi_status) {	6659	if (bio->bi_status) {
6638	atomic_inc(&bioc->error);	6660	atomic_inc(&bioc->error);
6639	if (bio->bi_status == BLK_STS_IOERR \|\|	6661	if (bio->bi_status == BLK_STS_IOERR \|\|
6640	bio->bi_status == BLK_STS_TARGET) {	6662	bio->bi_status == BLK_STS_TARGET) {
6641	struct btrfs_device *dev = btrfs_bio(bio)->device;
6642
6643	ASSERT(dev->bdev);
6644	if (btrfs_op(bio) == BTRFS_MAP_WRITE)	6663	if (btrfs_op(bio) == BTRFS_MAP_WRITE)
6645	btrfs_dev_stat_inc_and_print(dev,	6664	btrfs_dev_stat_inc_and_print(stripe->dev,
6646	BTRFS_DEV_STAT_WRITE_ERRS);	6665	BTRFS_DEV_STAT_WRITE_ERRS);
6647	else if (!(bio->bi_opf & REQ_RAHEAD))	6666	else if (!(bio->bi_opf & REQ_RAHEAD))
6648	btrfs_dev_stat_inc_and_print(dev,	6667	btrfs_dev_stat_inc_and_print(stripe->dev,
6649	BTRFS_DEV_STAT_READ_ERRS);	6668	BTRFS_DEV_STAT_READ_ERRS);
6650	if (bio->bi_opf & REQ_PREFLUSH)	6669	if (bio->bi_opf & REQ_PREFLUSH)
6651	btrfs_dev_stat_inc_and_print(dev,	6670	btrfs_dev_stat_inc_and_print(stripe->dev,
6652	BTRFS_DEV_STAT_FLUSH_ERRS);	6671	BTRFS_DEV_STAT_FLUSH_ERRS);
6653	}	6672	}
6654	}	6673	}
6655		6674
6656	if (bio == bioc->orig_bio)	6675	if (bio != bioc->orig_bio)
6657	is_orig_bio = 1;	6676	bio_put(bio);
6658		6677
6659	btrfs_bio_counter_dec(bioc->fs_info);	6678	btrfs_bio_counter_dec(bioc->fs_info);
6660		6679	if (atomic_dec_and_test(&bioc->stripes_pending))
6661	if (atomic_dec_and_test(&bioc->stripes_pending)) {	6680	btrfs_end_bioc(bioc, true);
6662	if (!is_orig_bio) {
6663	bio_put(bio);
6664	bio = bioc->orig_bio;
6665	}
6666
6667	btrfs_bio(bio)->mirror_num = bioc->mirror_num;
6668	/* only send an error to the higher layers if it is
6669	* beyond the tolerance of the btrfs bio
6670	*/
6671	if (atomic_read(&bioc->error) > bioc->max_errors) {
6672	bio->bi_status = BLK_STS_IOERR;
6673	} else {
6674	/*
6675	* this bio is actually up to date, we didn't
6676	* go over the max number of errors
6677	*/
6678	bio->bi_status = BLK_STS_OK;
6679	}
6680
6681	btrfs_end_bioc(bioc, bio);
6682	} else if (!is_orig_bio) {
6683	bio_put(bio);
6684	}
6685	}	6681	}
6686		6682
6687	static void submit_stripe_bio(struct btrfs_io_context bioc, struct bio bio,	6683	static void submit_stripe_bio(struct btrfs_io_context *bioc,
6688	u64 physical, struct btrfs_device *dev)	6684	struct bio *orig_bio, int dev_nr, bool clone)
6689	{	6685	{
6690	struct btrfs_fs_info *fs_info = bioc->fs_info;	6686	struct btrfs_fs_info *fs_info = bioc->fs_info;
		6687	struct btrfs_device *dev = bioc->stripes[dev_nr].dev;
		6688	u64 physical = bioc->stripes[dev_nr].physical;
		6689	struct bio *bio;
		6690
		6691	if (!dev \|\| !dev->bdev \|\|
		6692	test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) \|\|
		6693	(btrfs_op(orig_bio) == BTRFS_MAP_WRITE &&
		6694	!test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))) {
		6695	atomic_inc(&bioc->error);
		6696	if (atomic_dec_and_test(&bioc->stripes_pending))
		6697	btrfs_end_bioc(bioc, false);
		6698	return;
		6699	}
		6700
		6701	if (clone) {
		6702	bio = bio_alloc_clone(dev->bdev, orig_bio, GFP_NOFS, &fs_bio_set);
		6703	} else {
		6704	bio = orig_bio;
		6705	bio_set_dev(bio, dev->bdev);
		6706	btrfs_bio(bio)->device = dev;
		6707	}
6691		6708
6692	bio->bi_private = bioc;	6709	bioc->stripes[dev_nr].bioc = bioc;
6693	btrfs_bio(bio)->device = dev;	6710	bio->bi_private = &bioc->stripes[dev_nr];
6694	bio->bi_end_io = btrfs_end_bio;	6711	bio->bi_end_io = btrfs_end_bio;
6695	bio->bi_iter.bi_sector = physical >> 9;	6712	bio->bi_iter.bi_sector = physical >> 9;
6696	/*	6713	/*
@@ -6708,8 +6725,8 @@ static void submit_stripe_bio(struct btrfs_io_context bioc, struct bio bio, · 11 unmodified lines
6708	}	6725	}
6709	}	6726	}
6710	btrfs_debug_in_rcu(fs_info,	6727	btrfs_debug_in_rcu(fs_info,
6711	"btrfs_map_bio: rw %d 0x%x, sector=%llu, dev=%lu (%s id %llu), size=%u",	6728	"%s: rw %d 0x%x, sector=%llu, dev=%lu (%s id %llu), size=%u",
6712	bio_op(bio), bio->bi_opf, bio->bi_iter.bi_sector,	6729	__func__, bio_op(bio), bio->bi_opf, bio->bi_iter.bi_sector,
6713	(unsigned long)dev->bdev->bd_dev, rcu_str_deref(dev->name),	6730	(unsigned long)dev->bdev->bd_dev, rcu_str_deref(dev->name),
6714	dev->devid, bio->bi_iter.bi_size);	6731	dev->devid, bio->bi_iter.bi_size);
6715		6732
@@ -6719,66 +6736,39 @@ static void submit_stripe_bio(struct btrfs_io_context bioc, struct bio bio, · 3 unmodified lines
6719	submit_bio(bio);	6736	submit_bio(bio);
6720	}	6737	}
6721		6738
6722	static void bioc_error(struct btrfs_io_context bioc, struct bio bio, u64 logical)	6739	void btrfs_submit_bio(struct btrfs_fs_info fs_info, struct bio bio, int mirror_num)
6723	{
6724	atomic_inc(&bioc->error);
6725	if (atomic_dec_and_test(&bioc->stripes_pending)) {
6726	/* Should be the original bio. */
6727	WARN_ON(bio != bioc->orig_bio);
6728
6729	btrfs_bio(bio)->mirror_num = bioc->mirror_num;
6730	bio->bi_iter.bi_sector = logical >> 9;
6731	if (atomic_read(&bioc->error) > bioc->max_errors)
6732	bio->bi_status = BLK_STS_IOERR;
6733	else
6734	bio->bi_status = BLK_STS_OK;
6735	btrfs_end_bioc(bioc, bio);
6736	}
6737	}
6738
6739	blk_status_t btrfs_map_bio(struct btrfs_fs_info fs_info, struct bio bio,
6740	int mirror_num)
6741	{	6740	{
6742	struct btrfs_device *dev;
6743	struct bio *first_bio = bio;
6744	u64 logical = bio->bi_iter.bi_sector << 9;	6741	u64 logical = bio->bi_iter.bi_sector << 9;
6745	u64 length = 0;	6742	u64 length = bio->bi_iter.bi_size;
6746	u64 map_length;	6743	u64 map_length = length;
6747	int ret;	6744	int ret;
6748	int dev_nr;	6745	int dev_nr;
6749	int total_devs;	6746	int total_devs;
6750	struct btrfs_io_context *bioc = NULL;	6747	struct btrfs_io_context *bioc = NULL;
6751		6748
6752	length = bio->bi_iter.bi_size;
6753	map_length = length;
6754
6755	btrfs_bio_counter_inc_blocked(fs_info);	6749	btrfs_bio_counter_inc_blocked(fs_info);
6756	ret = __btrfs_map_block(fs_info, btrfs_op(bio), logical,	6750	ret = __btrfs_map_block(fs_info, btrfs_op(bio), logical,
6757	&map_length, &bioc, mirror_num, 1);	6751	&map_length, &bioc, mirror_num, 1);
6758	if (ret) {	6752	if (ret) {
6759	btrfs_bio_counter_dec(fs_info);	6753	btrfs_bio_counter_dec(fs_info);
6760	return errno_to_blk_status(ret);	6754	bio->bi_status = errno_to_blk_status(ret);
		6755	bio_endio(bio);
		6756	return;
6761	}	6757	}
6762		6758
6763	total_devs = bioc->num_stripes;	6759	total_devs = bioc->num_stripes;
6764	bioc->orig_bio = first_bio;	6760	bioc->orig_bio = bio;
6765	bioc->private = first_bio->bi_private;	6761	bioc->private = bio->bi_private;
6766	bioc->end_io = first_bio->bi_end_io;	6762	bioc->end_io = bio->bi_end_io;
6767	atomic_set(&bioc->stripes_pending, bioc->num_stripes);	6763	atomic_set(&bioc->stripes_pending, total_devs);
6768		6764
6769	if ((bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) &&	6765	if ((bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) &&
6770	((btrfs_op(bio) == BTRFS_MAP_WRITE) \|\| (mirror_num > 1))) {	6766	((btrfs_op(bio) == BTRFS_MAP_WRITE) \|\| (mirror_num > 1))) {
6771	/* In this case, map_length has been set to the length of	6767	if (btrfs_op(bio) == BTRFS_MAP_WRITE)
6772	a single stripe; not the whole write */	6768	raid56_parity_write(bio, bioc);
6773	if (btrfs_op(bio) == BTRFS_MAP_WRITE) {	6769	else
6774	ret = raid56_parity_write(bio, bioc, map_length);	6770	raid56_parity_recover(bio, bioc, mirror_num, true);
6775	} else {	6771	return;
6776	ret = raid56_parity_recover(bio, bioc, map_length,
6777	mirror_num, 1);
6778	}
6779
6780	btrfs_bio_counter_dec(fs_info);
6781	return errno_to_blk_status(ret);
6782	}	6772	}
6783		6773
6784	if (map_length < length) {	6774	if (map_length < length) {
@@ -6789,26 +6779,11 @@ blk_status_t btrfs_map_bio(struct btrfs_fs_info fs_info, struct bio bio, · 4 unmodified lines
6789	}	6779	}
6790		6780
6791	for (dev_nr = 0; dev_nr < total_devs; dev_nr++) {	6781	for (dev_nr = 0; dev_nr < total_devs; dev_nr++) {
6792	dev = bioc->stripes[dev_nr].dev;	6782	const bool should_clone = (dev_nr < total_devs - 1);
6793	if (!dev \|\| !dev->bdev \|\| test_bit(BTRFS_DEV_STATE_MISSING,
6794	&dev->dev_state) \|\|
6795	(btrfs_op(first_bio) == BTRFS_MAP_WRITE &&
6796	!test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))) {
6797	bioc_error(bioc, first_bio, logical);
6798	continue;
6799	}
6800
6801	if (dev_nr < total_devs - 1) {
6802	bio = btrfs_bio_clone(dev->bdev, first_bio);
6803	} else {
6804	bio = first_bio;
6805	bio_set_dev(bio, dev->bdev);
6806	}
6807		6783
6808	submit_stripe_bio(bioc, bio, bioc->stripes[dev_nr].physical, dev);	6784	submit_stripe_bio(bioc, bio, dev_nr, should_clone);
6809	}	6785	}
6810	btrfs_bio_counter_dec(fs_info);	6786	btrfs_bio_counter_dec(fs_info);
6811	return BLK_STS_OK;
6812	}	6787	}
6813		6788
6814	static bool dev_args_match_fs_devices(const struct btrfs_dev_lookup_args *args,	6789	static bool dev_args_match_fs_devices(const struct btrfs_dev_lookup_args *args,
@@ -6966,11 +6941,12 @@ static void btrfs_report_missing_device(struct btrfs_fs_info *fs_info, · 151 unmodified lines
6966	devid, uuid);	6941	devid, uuid);
6967	}	6942	}
6968		6943
6969	static u64 calc_stripe_length(u64 type, u64 chunk_len, int num_stripes)	6944	u64 btrfs_calc_stripe_length(const struct extent_map *em)
6970	{	6945	{
6971	const int data_stripes = calc_data_stripes(type, num_stripes);	6946	const struct map_lookup *map = em->map_lookup;
		6947	const int data_stripes = calc_data_stripes(map->type, map->num_stripes);
6972		6948
6973	return div_u64(chunk_len, data_stripes);	6949	return div_u64(em->len, data_stripes);
6974	}	6950	}
6975		6951
6976	#if BITS_PER_LONG == 32	6952	#if BITS_PER_LONG == 32
@@ -7109,8 +7085,7 @@ static int read_one_chunk(struct btrfs_key key, struct extent_buffer leaf, · 132 unmodified lines
7109	map->type = type;	7085	map->type = type;
7110	map->sub_stripes = btrfs_chunk_sub_stripes(leaf, chunk);	7086	map->sub_stripes = btrfs_chunk_sub_stripes(leaf, chunk);
7111	map->verified_stripes = 0;	7087	map->verified_stripes = 0;
7112	em->orig_block_len = calc_stripe_length(type, em->len,	7088	em->orig_block_len = btrfs_calc_stripe_length(em);
7113	map->num_stripes);
7114	for (i = 0; i < num_stripes; i++) {	7089	for (i = 0; i < num_stripes; i++) {
7115	map->stripes[i].physical =	7090	map->stripes[i].physical =
7116	btrfs_stripe_offset_nr(leaf, chunk, i);	7091	btrfs_stripe_offset_nr(leaf, chunk, i);
@@ -7236,7 +7211,8 @@ static int read_one_dev(struct extent_buffer *leaf, · 119 unmodified lines
7236	u8 fs_uuid[BTRFS_FSID_SIZE];	7211	u8 fs_uuid[BTRFS_FSID_SIZE];
7237	u8 dev_uuid[BTRFS_UUID_SIZE];	7212	u8 dev_uuid[BTRFS_UUID_SIZE];
7238		7213
7239	devid = args.devid = btrfs_device_id(leaf, dev_item);	7214	devid = btrfs_device_id(leaf, dev_item);
		7215	args.devid = devid;
7240	read_extent_buffer(leaf, dev_uuid, btrfs_device_uuid(dev_item),	7216	read_extent_buffer(leaf, dev_uuid, btrfs_device_uuid(dev_item),
7241	BTRFS_UUID_SIZE);	7217	BTRFS_UUID_SIZE);
7242	read_extent_buffer(leaf, fs_uuid, btrfs_device_fsid(dev_item),	7218	read_extent_buffer(leaf, fs_uuid, btrfs_device_fsid(dev_item),
@@ -7865,11 +7841,7 @@ int btrfs_run_dev_stats(struct btrfs_trans_handle *trans) · 622 unmodified lines
7865	void btrfs_dev_stat_inc_and_print(struct btrfs_device *dev, int index)	7841	void btrfs_dev_stat_inc_and_print(struct btrfs_device *dev, int index)
7866	{	7842	{
7867	btrfs_dev_stat_inc(dev, index);	7843	btrfs_dev_stat_inc(dev, index);
7868	btrfs_dev_stat_print_on_error(dev);
7869	}
7870		7844
7871	static void btrfs_dev_stat_print_on_error(struct btrfs_device *dev)
7872	{
7873	if (!dev->dev_stats_valid)	7845	if (!dev->dev_stats_valid)
7874	return;	7846	return;
7875	btrfs_err_rl_in_rcu(dev->fs_info,	7847	btrfs_err_rl_in_rcu(dev->fs_info,
@@ -8011,7 +7983,7 @@ static int verify_one_dev_extent(struct btrfs_fs_info *fs_info, · 135 unmodified lines
8011	}	7983	}
8012		7984
8013	map = em->map_lookup;	7985	map = em->map_lookup;
8014	stripe_len = calc_stripe_length(map->type, em->len, map->num_stripes);	7986	stripe_len = btrfs_calc_stripe_length(em);
8015	if (physical_len != stripe_len) {	7987	if (physical_len != stripe_len) {
8016	btrfs_err(fs_info,	7988	btrfs_err(fs_info,
8017	"dev extent physical offset %llu on devid %llu length doesn't match chunk %llu, have %llu expect %llu",	7989	"dev extent physical offset %llu on devid %llu length doesn't match chunk %llu, have %llu expect %llu",
@@ -8021,6 +7993,16 @@ static int verify_one_dev_extent(struct btrfs_fs_info *fs_info, · 3 unmodified lines
8021	goto out;	7993	goto out;
8022	}	7994	}
8023		7995
		7996	/*
		7997	* Very old mkfs.btrfs (before v4.1) will not respect the reserved
		7998	* space. Although kernel can handle it without problem, better to warn
		7999	* the users.
		8000	*/
		8001	if (physical_offset < BTRFS_DEVICE_RANGE_RESERVED)
		8002	btrfs_warn(fs_info,
		8003	"devid %llu physical %llu len %llu inside the reserved space",
		8004	devid, physical_offset, physical_len);
		8005
8024	for (i = 0; i < map->num_stripes; i++) {	8006	for (i = 0; i < map->num_stripes; i++) {
8025	if (map->stripes[i].dev->devid == devid &&	8007	if (map->stripes[i].dev->devid == devid &&
8026	map->stripes[i].physical == physical_offset) {	8008	map->stripes[i].physical == physical_offset) {

fs/btrfs/volumes.h+43−3

 				- 2 * sizeof(struct btrfs_chunk))	\
 				/ sizeof(struct btrfs_stripe) + 1)
+/*
+ * Maximum number of sectors for a single bio to limit the size of the
+ * checksum array.  This matches the number of bio_vecs per bio and thus the
+ * I/O size for buffered I/O.
+ */
+#define BTRFS_MAX_BIO_SECTORS				(256)
 /*
  * Additional info to pass along bio.
  *
 	u8 csum_inline[BTRFS_BIO_INLINE_CSUM_SIZE];
 	struct bvec_iter iter;
+	/* For read end I/O handling */
+	struct work_struct end_io_work;
 	/*
 	 * This member must come last, bio_alloc_bioset will allocate enough
 	 * bytes for entire btrfs_bio but relies on bio being last.
 	}
 }
+/*
+ * Iterate through a btrfs_bio (@bbio) on a per-sector basis.
+ *
+ * bvl        - struct bio_vec
+ * bbio       - struct btrfs_bio
+ * iters      - struct bvec_iter
+ * bio_offset - unsigned int
+ */
+#define btrfs_bio_for_each_sector(fs_info, bvl, bbio, iter, bio_offset)	\
+	for ((iter) = (bbio)->iter, (bio_offset) = 0;			\
+	     (iter).bi_size &&					\
+	     (((bvl) = bio_iter_iovec((&(bbio)->bio), (iter))), 1);	\
+	     (bio_offset) += fs_info->sectorsize,			\
+	     bio_advance_iter_single(&(bbio)->bio, &(iter),		\
+	     (fs_info)->sectorsize))
 struct btrfs_io_stripe {
+	struct btrfs_device *dev;
+	union {
+		/* Block mapping */
+		u64 physical;
+		/* For the endio handler */
+		struct btrfs_io_context *bioc;
+	};
+};
+struct btrfs_discard_stripe {
 	struct btrfs_device *dev;
 	u64 physical;
-	u64 length; /* only used for discard mappings */
+	u64 length;
 };
 /*
 int btrfs_map_sblock(struct btrfs_fs_info *fs_info, enum btrfs_map_op op,
 		     u64 logical, u64 *length,
 		     struct btrfs_io_context **bioc_ret);
+struct btrfs_discard_stripe *btrfs_map_discard(struct btrfs_fs_info *fs_info,
+					       u64 logical, u64 *length_ret,
+					       u32 *num_stripes);
 int btrfs_get_io_geometry(struct btrfs_fs_info *fs_info, struct extent_map *map,
 			  enum btrfs_map_op op, u64 logical,
 			  struct btrfs_io_geometry *io_geom);
 struct btrfs_block_group *btrfs_create_chunk(struct btrfs_trans_handle *trans,
 					    u64 type);
 void btrfs_mapping_tree_free(struct extent_map_tree *tree);
-blk_status_t btrfs_map_bio(struct btrfs_fs_info *fs_info, struct bio *bio,
+void btrfs_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio, int mirror_num);
-			   int mirror_num);
 int btrfs_open_devices(struct btrfs_fs_devices *fs_devices,
 		       fmode_t flags, void *holder);
 struct btrfs_device *btrfs_scan_one_device(const char *path,
 			   u64 logical, u64 len);
 unsigned long btrfs_full_stripe_len(struct btrfs_fs_info *fs_info,
 				    u64 logical);
+u64 btrfs_calc_stripe_length(const struct extent_map *em);
+int btrfs_nr_parity_stripes(u64 type);
 int btrfs_chunk_alloc_add_chunk_item(struct btrfs_trans_handle *trans,
 				     struct btrfs_block_group *bg);
 int btrfs_remove_chunk(struct btrfs_trans_handle *trans, u64 chunk_offset);

fs/btrfs/zlib.c+18−24

 {
 	struct workspace *workspace = list_entry(ws, struct workspace, list);
 	int ret;
-	char *data_in;
+	char *data_in = NULL;
 	char *cpage_out;
 	int nr_pages = 0;
 	struct page *in_page = NULL;
 		ret = -ENOMEM;
 		goto out;
 	}
-	cpage_out = kmap(out_page);
+	cpage_out = page_address(out_page);
 	pages[0] = out_page;
 	nr_pages = 1;
 				int i;
 				for (i = 0; i < in_buf_pages; i++) {
-					if (in_page) {
+					if (data_in) {
-						kunmap(in_page);
+						kunmap_local(data_in);
 						put_page(in_page);
 					}
 					in_page = find_get_page(mapping,
 								start >> PAGE_SHIFT);
-					data_in = kmap(in_page);
+					data_in = kmap_local_page(in_page);
 					memcpy(workspace->buf + i * PAGE_SIZE,
 					       data_in, PAGE_SIZE);
 					start += PAGE_SIZE;
 				}
 				workspace->strm.next_in = workspace->buf;
 			} else {
-				if (in_page) {
+				if (data_in) {
-					kunmap(in_page);
+					kunmap_local(data_in);
 					put_page(in_page);
 				}
 				in_page = find_get_page(mapping,
 							start >> PAGE_SHIFT);
-				data_in = kmap(in_page);
+				data_in = kmap_local_page(in_page);
 				start += PAGE_SIZE;
 				workspace->strm.next_in = data_in;
 			}
 		 * the stream end if required
 		 */
 		if (workspace->strm.avail_out == 0) {
-			kunmap(out_page);
 			if (nr_pages == nr_dest_pages) {
-				out_page = NULL;
 				ret = -E2BIG;
 				goto out;
 			}
 				ret = -ENOMEM;
 				goto out;
 			}
-			cpage_out = kmap(out_page);
+			cpage_out = page_address(out_page);
 			pages[nr_pages] = out_page;
 			nr_pages++;
 			workspace->strm.avail_out = PAGE_SIZE;
 			goto out;
 		} else if (workspace->strm.avail_out == 0) {
 			/* get another page for the stream end */
-			kunmap(out_page);
 			if (nr_pages == nr_dest_pages) {
-				out_page = NULL;
 				ret = -E2BIG;
 				goto out;
 			}
 				ret = -ENOMEM;
 				goto out;
 			}
-			cpage_out = kmap(out_page);
+			cpage_out = page_address(out_page);
 			pages[nr_pages] = out_page;
 			nr_pages++;
 			workspace->strm.avail_out = PAGE_SIZE;
 	*total_in = workspace->strm.total_in;
 out:
 	*out_pages = nr_pages;
-	if (out_page)
+	if (data_in) {
-		kunmap(out_page);
+		kunmap_local(data_in);
-	if (in_page) {
-		kunmap(in_page);
 		put_page(in_page);
 	}
 	return ret;
 }
 	unsigned long buf_start;
 	struct page **pages_in = cb->compressed_pages;
-	data_in = kmap(pages_in[page_in_index]);
+	data_in = kmap_local_page(pages_in[page_in_index]);
 	workspace->strm.next_in = data_in;
 	workspace->strm.avail_in = min_t(size_t, srclen, PAGE_SIZE);
 	workspace->strm.total_in = 0;
 	if (Z_OK != zlib_inflateInit2(&workspace->strm, wbits)) {
 		pr_warn("BTRFS: inflateInit failed\n");
-		kunmap(pages_in[page_in_index]);
+		kunmap_local(data_in);
 		return -EIO;
 	}
 	while (workspace->strm.total_in < srclen) {
 		if (workspace->strm.avail_in == 0) {
 			unsigned long tmp;
-			kunmap(pages_in[page_in_index]);
+			kunmap_local(data_in);
 			page_in_index++;
 			if (page_in_index >= total_pages_in) {
 				data_in = NULL;
 				break;
 			}
-			data_in = kmap(pages_in[page_in_index]);
+			data_in = kmap_local_page(pages_in[page_in_index]);
 			workspace->strm.next_in = data_in;
 			tmp = srclen - workspace->strm.total_in;
 			workspace->strm.avail_in = min(tmp, PAGE_SIZE);
 done:
 	zlib_inflateEnd(&workspace->strm);
 	if (data_in)
-		kunmap(pages_in[page_in_index]);
+		kunmap_local(data_in);
 	if (!ret)
 		zero_fill_bio(cb->orig_bio);
 	return ret;

fs/btrfs/zoned.c+128−3

@@ -94,9 +94,9 @@ static int sb_write_pointer(struct block_device bdev, struct blk_zone zones, · 93 unmodified lines
94	* Possible states of log buffer zones	94	* Possible states of log buffer zones
95	*	95	*
96	* Empty[0] In use[0] Full[0]	96	* Empty[0] In use[0] Full[0]
97	* Empty[1] * x 0	97	* Empty[1] * 0 1
98	* In use[1] 0 x 0	98	* In use[1] x x 1
99	* Full[1] 1 1 C	99	* Full[1] 0 0 C
100	*	100	*
101	* Log position:	101	* Log position:
102	* *: Special case, no superblock is written	102	* *: Special case, no superblock is written
@@ -415,6 +415,16 @@ int btrfs_get_dev_zone_info(struct btrfs_device *device, bool populate_cache) · 312 unmodified lines
415	nr_sectors = bdev_nr_sectors(bdev);	415	nr_sectors = bdev_nr_sectors(bdev);
416	zone_info->zone_size_shift = ilog2(zone_info->zone_size);	416	zone_info->zone_size_shift = ilog2(zone_info->zone_size);
417	zone_info->nr_zones = nr_sectors >> ilog2(zone_sectors);	417	zone_info->nr_zones = nr_sectors >> ilog2(zone_sectors);
		418	/*
		419	* We limit max_zone_append_size also by max_segments *
		420	* PAGE_SIZE. Technically, we can have multiple pages per segment. But,
		421	* since btrfs adds the pages one by one to a bio, and btrfs cannot
		422	* increase the metadata reservation even if it increases the number of
		423	* extents, it is safe to stick with the limit.
		424	*/
		425	zone_info->max_zone_append_size =
		426	min_t(u64, (u64)bdev_max_zone_append_sectors(bdev) << SECTOR_SHIFT,
		427	(u64)bdev_max_segments(bdev) << PAGE_SHIFT);
418	if (!IS_ALIGNED(nr_sectors, zone_sectors))	428	if (!IS_ALIGNED(nr_sectors, zone_sectors))
419	zone_info->nr_zones++;	429	zone_info->nr_zones++;
420		430
@@ -640,6 +650,7 @@ int btrfs_check_zoned_mode(struct btrfs_fs_info *fs_info) · 219 unmodified lines
640	u64 zoned_devices = 0;	650	u64 zoned_devices = 0;
641	u64 nr_devices = 0;	651	u64 nr_devices = 0;
642	u64 zone_size = 0;	652	u64 zone_size = 0;
		653	u64 max_zone_append_size = 0;
643	const bool incompat_zoned = btrfs_fs_incompat(fs_info, ZONED);	654	const bool incompat_zoned = btrfs_fs_incompat(fs_info, ZONED);
644	int ret = 0;	655	int ret = 0;
645		656
@@ -674,6 +685,11 @@ int btrfs_check_zoned_mode(struct btrfs_fs_info *fs_info) · 28 unmodified lines
674	ret = -EINVAL;	685	ret = -EINVAL;
675	goto out;	686	goto out;
676	}	687	}
		688	if (!max_zone_append_size \|\|
		689	(zone_info->max_zone_append_size &&
		690	zone_info->max_zone_append_size < max_zone_append_size))
		691	max_zone_append_size =
		692	zone_info->max_zone_append_size;
677	}	693	}
678	nr_devices++;	694	nr_devices++;
679	}	695	}
@@ -723,7 +739,11 @@ int btrfs_check_zoned_mode(struct btrfs_fs_info *fs_info) · 43 unmodified lines
723	}	739	}
724		740
725	fs_info->zone_size = zone_size;	741	fs_info->zone_size = zone_size;
		742	fs_info->max_zone_append_size = ALIGN_DOWN(max_zone_append_size,
		743	fs_info->sectorsize);
726	fs_info->fs_devices->chunk_alloc_policy = BTRFS_CHUNK_ALLOC_ZONED;	744	fs_info->fs_devices->chunk_alloc_policy = BTRFS_CHUNK_ALLOC_ZONED;
		745	if (fs_info->max_zone_append_size < fs_info->max_extent_size)
		746	fs_info->max_extent_size = fs_info->max_zone_append_size;
727		747
728	/*	748	/*
729	* Check mount options here, because we might change fs_info->zoned	749	* Check mount options here, because we might change fs_info->zoned
@@ -1829,6 +1849,7 @@ struct btrfs_device btrfs_zoned_get_device(struct btrfs_fs_info fs_info, · 1099 unmodified lines
1829	bool btrfs_zone_activate(struct btrfs_block_group *block_group)	1849	bool btrfs_zone_activate(struct btrfs_block_group *block_group)
1830	{	1850	{
1831	struct btrfs_fs_info *fs_info = block_group->fs_info;	1851	struct btrfs_fs_info *fs_info = block_group->fs_info;
		1852	struct btrfs_space_info *space_info = block_group->space_info;
1832	struct map_lookup *map;	1853	struct map_lookup *map;
1833	struct btrfs_device *device;	1854	struct btrfs_device *device;
1834	u64 physical;	1855	u64 physical;
@@ -1840,6 +1861,7 @@ bool btrfs_zone_activate(struct btrfs_block_group *block_group) · 5 unmodified lines
1840		1861
1841	map = block_group->physical_map;	1862	map = block_group->physical_map;
1842		1863
		1864	spin_lock(&space_info->lock);
1843	spin_lock(&block_group->lock);	1865	spin_lock(&block_group->lock);
1844	if (block_group->zone_is_active) {	1866	if (block_group->zone_is_active) {
1845	ret = true;	1867	ret = true;
@@ -1868,7 +1890,10 @@ bool btrfs_zone_activate(struct btrfs_block_group *block_group) · 22 unmodified lines
1868		1890
1869	/* Successfully activated all the zones */	1891	/* Successfully activated all the zones */
1870	block_group->zone_is_active = 1;	1892	block_group->zone_is_active = 1;
		1893	space_info->active_total_bytes += block_group->length;
1871	spin_unlock(&block_group->lock);	1894	spin_unlock(&block_group->lock);
		1895	btrfs_try_granting_tickets(fs_info, space_info);
		1896	spin_unlock(&space_info->lock);
1872		1897
1873	/* For the active block group list */	1898	/* For the active block group list */
1874	btrfs_get_block_group(block_group);	1899	btrfs_get_block_group(block_group);
@@ -1881,6 +1906,7 @@ bool btrfs_zone_activate(struct btrfs_block_group *block_group) · 6 unmodified lines
1881		1906
1882	out_unlock:	1907	out_unlock:
1883	spin_unlock(&block_group->lock);	1908	spin_unlock(&block_group->lock);
		1909	spin_unlock(&space_info->lock);
1884	return ret;	1910	return ret;
1885	}	1911	}
1886		1912
@@ -1981,6 +2007,9 @@ static int do_zone_finish(struct btrfs_block_group *block_group, bool fully_writ · 94 unmodified lines
1981	/* For active_bg_list */	2007	/* For active_bg_list */
1982	btrfs_put_block_group(block_group);	2008	btrfs_put_block_group(block_group);
1983		2009
		2010	clear_bit(BTRFS_FS_NEED_ZONE_FINISH, &fs_info->flags);
		2011	wake_up_all(&fs_info->zone_finish_wait);
		2012
1984	return 0;	2013	return 0;
1985	}	2014	}
1986		2015
@@ -2017,6 +2046,9 @@ bool btrfs_can_activate_zone(struct btrfs_fs_devices *fs_devices, u64 flags) · 30 unmodified lines
2017	}	2046	}
2018	mutex_unlock(&fs_info->chunk_mutex);	2047	mutex_unlock(&fs_info->chunk_mutex);
2019		2048
		2049	if (!ret)
		2050	set_bit(BTRFS_FS_NEED_ZONE_FINISH, &fs_info->flags);
		2051
2020	return ret;	2052	return ret;
2021	}	2053	}
2022		2054
@@ -2160,3 +2192,96 @@ out: · 137 unmodified lines
2160	spin_unlock(&block_group->lock);	2192	spin_unlock(&block_group->lock);
2161	btrfs_put_block_group(block_group);	2193	btrfs_put_block_group(block_group);
2162	}	2194	}
		2195
		2196	int btrfs_zone_finish_one_bg(struct btrfs_fs_info *fs_info)
		2197	{
		2198	struct btrfs_block_group *block_group;
		2199	struct btrfs_block_group *min_bg = NULL;
		2200	u64 min_avail = U64_MAX;
		2201	int ret;
		2202
		2203	spin_lock(&fs_info->zone_active_bgs_lock);
		2204	list_for_each_entry(block_group, &fs_info->zone_active_bgs,
		2205	active_bg_list) {
		2206	u64 avail;
		2207
		2208	spin_lock(&block_group->lock);
		2209	if (block_group->reserved \|\|
		2210	(block_group->flags & BTRFS_BLOCK_GROUP_SYSTEM)) {
		2211	spin_unlock(&block_group->lock);
		2212	continue;
		2213	}
		2214
		2215	avail = block_group->zone_capacity - block_group->alloc_offset;
		2216	if (min_avail > avail) {
		2217	if (min_bg)
		2218	btrfs_put_block_group(min_bg);
		2219	min_bg = block_group;
		2220	min_avail = avail;
		2221	btrfs_get_block_group(min_bg);
		2222	}
		2223	spin_unlock(&block_group->lock);
		2224	}
		2225	spin_unlock(&fs_info->zone_active_bgs_lock);
		2226
		2227	if (!min_bg)
		2228	return 0;
		2229
		2230	ret = btrfs_zone_finish(min_bg);
		2231	btrfs_put_block_group(min_bg);
		2232
		2233	return ret < 0 ? ret : 1;
		2234	}
		2235
		2236	int btrfs_zoned_activate_one_bg(struct btrfs_fs_info *fs_info,
		2237	struct btrfs_space_info *space_info,
		2238	bool do_finish)
		2239	{
		2240	struct btrfs_block_group *bg;
		2241	int index;
		2242
		2243	if (!btrfs_is_zoned(fs_info) \|\| (space_info->flags & BTRFS_BLOCK_GROUP_DATA))
		2244	return 0;
		2245
		2246	/* No more block groups to activate */
		2247	if (space_info->active_total_bytes == space_info->total_bytes)
		2248	return 0;
		2249
		2250	for (;;) {
		2251	int ret;
		2252	bool need_finish = false;
		2253
		2254	down_read(&space_info->groups_sem);
		2255	for (index = 0; index < BTRFS_NR_RAID_TYPES; index++) {
		2256	list_for_each_entry(bg, &space_info->block_groups[index],
		2257	list) {
		2258	if (!spin_trylock(&bg->lock))
		2259	continue;
		2260	if (btrfs_zoned_bg_is_full(bg) \|\| bg->zone_is_active) {
		2261	spin_unlock(&bg->lock);
		2262	continue;
		2263	}
		2264	spin_unlock(&bg->lock);
		2265
		2266	if (btrfs_zone_activate(bg)) {
		2267	up_read(&space_info->groups_sem);
		2268	return 1;
		2269	}
		2270
		2271	need_finish = true;
		2272	}
		2273	}
		2274	up_read(&space_info->groups_sem);
		2275
		2276	if (!do_finish \|\| !need_finish)
		2277	break;
		2278
		2279	ret = btrfs_zone_finish_one_bg(fs_info);
		2280	if (ret == 0)
		2281	break;
		2282	if (ret < 0)
		2283	return ret;
		2284	}
		2285
		2286	return 0;
		2287	}

fs/btrfs/zoned.h+18−0

 	 */
 	u64 zone_size;
 	u8  zone_size_shift;
+	u64 max_zone_append_size;
 	u32 nr_zones;
 	unsigned int max_active_zones;
 	atomic_t active_zones_left;
 bool btrfs_zoned_should_reclaim(struct btrfs_fs_info *fs_info);
 void btrfs_zoned_release_data_reloc_bg(struct btrfs_fs_info *fs_info, u64 logical,
 				       u64 length);
+int btrfs_zone_finish_one_bg(struct btrfs_fs_info *fs_info);
+int btrfs_zoned_activate_one_bg(struct btrfs_fs_info *fs_info,
+				struct btrfs_space_info *space_info, bool do_finish);
 #else /* CONFIG_BLK_DEV_ZONED */
 static inline int btrfs_get_dev_zone(struct btrfs_device *device, u64 pos,
 				     struct blk_zone *zone)
 static inline void btrfs_zoned_release_data_reloc_bg(struct btrfs_fs_info *fs_info,
 						     u64 logical, u64 length) { }
+static inline int btrfs_zone_finish_one_bg(struct btrfs_fs_info *fs_info)
+{
+	return 1;
+}
+static inline int btrfs_zoned_activate_one_bg(struct btrfs_fs_info *fs_info,
+					      struct btrfs_space_info *space_info,
+					      bool do_finish)
+{
+	/* Consider all the block groups are active */
+	return 0;
+}
 #endif
 static inline bool btrfs_dev_is_sequential(struct btrfs_device *device, u64 pos)

fs/btrfs/zstd.c+13−20

 	/* map in the first page of input data */
 	in_page = find_get_page(mapping, start >> PAGE_SHIFT);
-	workspace->in_buf.src = kmap(in_page);
+	workspace->in_buf.src = kmap_local_page(in_page);
 	workspace->in_buf.pos = 0;
 	workspace->in_buf.size = min_t(size_t, len, PAGE_SIZE);
 		goto out;
 	}
 	pages[nr_pages++] = out_page;
-	workspace->out_buf.dst = kmap(out_page);
+	workspace->out_buf.dst = page_address(out_page);
 	workspace->out_buf.pos = 0;
 	workspace->out_buf.size = min_t(size_t, max_out, PAGE_SIZE);
 		if (workspace->out_buf.pos == workspace->out_buf.size) {
 			tot_out += PAGE_SIZE;
 			max_out -= PAGE_SIZE;
-			kunmap(out_page);
 			if (nr_pages == nr_dest_pages) {
-				out_page = NULL;
 				ret = -E2BIG;
 				goto out;
 			}
 				goto out;
 			}
 			pages[nr_pages++] = out_page;
-			workspace->out_buf.dst = kmap(out_page);
+			workspace->out_buf.dst = page_address(out_page);
 			workspace->out_buf.pos = 0;
 			workspace->out_buf.size = min_t(size_t, max_out,
 							PAGE_SIZE);
 		/* Check if we need more input */
 		if (workspace->in_buf.pos == workspace->in_buf.size) {
 			tot_in += PAGE_SIZE;
-			kunmap(in_page);
+			kunmap_local(workspace->in_buf.src);
 			put_page(in_page);
 			start += PAGE_SIZE;
 			len -= PAGE_SIZE;
 			in_page = find_get_page(mapping, start >> PAGE_SHIFT);
-			workspace->in_buf.src = kmap(in_page);
+			workspace->in_buf.src = kmap_local_page(in_page);
 			workspace->in_buf.pos = 0;
 			workspace->in_buf.size = min_t(size_t, len, PAGE_SIZE);
 		}
 		tot_out += PAGE_SIZE;
 		max_out -= PAGE_SIZE;
-		kunmap(out_page);
 		if (nr_pages == nr_dest_pages) {
-			out_page = NULL;
 			ret = -E2BIG;
 			goto out;
 		}
 			goto out;
 		}
 		pages[nr_pages++] = out_page;
-		workspace->out_buf.dst = kmap(out_page);
+		workspace->out_buf.dst = page_address(out_page);
 		workspace->out_buf.pos = 0;
 		workspace->out_buf.size = min_t(size_t, max_out, PAGE_SIZE);
 	}
 	*total_out = tot_out;
 out:
 	*out_pages = nr_pages;
-	/* Cleanup */
+	if (workspace->in_buf.src) {
-	if (in_page) {
+		kunmap_local(workspace->in_buf.src);
-		kunmap(in_page);
 		put_page(in_page);
 	}
-	if (out_page)
-		kunmap(out_page);
 	return ret;
 }
 		goto done;
 	}
-	workspace->in_buf.src = kmap(pages_in[page_in_index]);
+	workspace->in_buf.src = kmap_local_page(pages_in[page_in_index]);
 	workspace->in_buf.pos = 0;
 	workspace->in_buf.size = min_t(size_t, srclen, PAGE_SIZE);
 			break;
 		if (workspace->in_buf.pos == workspace->in_buf.size) {
-			kunmap(pages_in[page_in_index++]);
+			kunmap_local(workspace->in_buf.src);
+			page_in_index++;
 			if (page_in_index >= total_pages_in) {
 				workspace->in_buf.src = NULL;
 				ret = -EIO;
 				goto done;
 			}
 			srclen -= PAGE_SIZE;
-			workspace->in_buf.src = kmap(pages_in[page_in_index]);
+			workspace->in_buf.src = kmap_local_page(pages_in[page_in_index]);
 			workspace->in_buf.pos = 0;
 			workspace->in_buf.size = min_t(size_t, srclen, PAGE_SIZE);
 		}
 	zero_fill_bio(cb->orig_bio);
 done:
 	if (workspace->in_buf.src)
-		kunmap(pages_in[page_in_index]);
+		kunmap_local(workspace->in_buf.src);
 	return ret;
 }

include/linux/blkdev.h+5−0

 	return queue_max_zone_append_sectors(bdev_get_queue(bdev));
 }
+static inline unsigned int bdev_max_segments(struct block_device *bdev)
+{
+	return queue_max_segments(bdev_get_queue(bdev));
+}
 static inline unsigned queue_logical_block_size(const struct request_queue *q)
 {
 	int retval = 512;

include/linux/highmem-internal.h+5−5

@@ -8,7 +8,7 @@ · 7 unmodified lines
8	#ifdef CONFIG_KMAP_LOCAL	8	#ifdef CONFIG_KMAP_LOCAL
9	void *__kmap_local_pfn_prot(unsigned long pfn, pgprot_t prot);	9	void *__kmap_local_pfn_prot(unsigned long pfn, pgprot_t prot);
10	void __kmap_local_page_prot(struct page page, pgprot_t prot);	10	void __kmap_local_page_prot(struct page page, pgprot_t prot);
11	void kunmap_local_indexed(void *vaddr);	11	void kunmap_local_indexed(const void *vaddr);
12	void kmap_local_fork(struct task_struct *tsk);	12	void kmap_local_fork(struct task_struct *tsk);
13	void __kmap_local_sched_out(void);	13	void __kmap_local_sched_out(void);
14	void __kmap_local_sched_in(void);	14	void __kmap_local_sched_in(void);
@@ -89,7 +89,7 @@ static inline void *kmap_local_pfn(unsigned long pfn) · 74 unmodified lines
89	return __kmap_local_pfn_prot(pfn, kmap_prot);	89	return __kmap_local_pfn_prot(pfn, kmap_prot);
90	}	90	}
91		91
92	static inline void __kunmap_local(void *vaddr)	92	static inline void __kunmap_local(const void *vaddr)
93	{	93	{
94	kunmap_local_indexed(vaddr);	94	kunmap_local_indexed(vaddr);
95	}	95	}
@@ -121,7 +121,7 @@ static inline void *kmap_atomic_pfn(unsigned long pfn) · 25 unmodified lines
121	return __kmap_local_pfn_prot(pfn, kmap_prot);	121	return __kmap_local_pfn_prot(pfn, kmap_prot);
122	}	122	}
123		123
124	static inline void __kunmap_atomic(void *addr)	124	static inline void __kunmap_atomic(const void *addr)
125	{	125	{
126	kunmap_local_indexed(addr);	126	kunmap_local_indexed(addr);
127	pagefault_enable();	127	pagefault_enable();
@@ -197,7 +197,7 @@ static inline void *kmap_local_pfn(unsigned long pfn) · 69 unmodified lines
197	return kmap_local_page(pfn_to_page(pfn));	197	return kmap_local_page(pfn_to_page(pfn));
198	}	198	}
199		199
200	static inline void __kunmap_local(void *addr)	200	static inline void __kunmap_local(const void *addr)
201	{	201	{
202	#ifdef ARCH_HAS_FLUSH_ON_KUNMAP	202	#ifdef ARCH_HAS_FLUSH_ON_KUNMAP
203	kunmap_flush_on_unmap(addr);	203	kunmap_flush_on_unmap(addr);
@@ -224,7 +224,7 @@ static inline void *kmap_atomic_pfn(unsigned long pfn) · 20 unmodified lines
224	return kmap_atomic(pfn_to_page(pfn));	224	return kmap_atomic(pfn_to_page(pfn));
225	}	225	}
226		226
227	static inline void __kunmap_atomic(void *addr)	227	static inline void __kunmap_atomic(const void *addr)
228	{	228	{
229	#ifdef ARCH_HAS_FLUSH_ON_KUNMAP	229	#ifdef ARCH_HAS_FLUSH_ON_KUNMAP
230	kunmap_flush_on_unmap(addr);	230	kunmap_flush_on_unmap(addr);

include/trace/events/btrfs.h+158−0

@@ -30,6 +30,8 @@ struct btrfs_qgroup; · 29 unmodified lines
30	struct extent_io_tree;	30	struct extent_io_tree;
31	struct prelim_ref;	31	struct prelim_ref;
32	struct btrfs_space_info;	32	struct btrfs_space_info;
		33	struct btrfs_raid_bio;
		34	struct raid56_bio_trace_info;
33		35
34	#define show_ref_type(type) \	36	#define show_ref_type(type) \
35	__print_symbolic(type, \	37	__print_symbolic(type, \
@@ -596,6 +598,70 @@ DEFINE_EVENT(btrfs__ordered_extent, btrfs_ordered_extent_put, · 560 unmodified lines
596	TP_ARGS(inode, ordered)	598	TP_ARGS(inode, ordered)
597	);	599	);
598		600
		601	DEFINE_EVENT(btrfs__ordered_extent, btrfs_ordered_extent_lookup,
		602
		603	TP_PROTO(const struct btrfs_inode *inode,
		604	const struct btrfs_ordered_extent *ordered),
		605
		606	TP_ARGS(inode, ordered)
		607	);
		608
		609	DEFINE_EVENT(btrfs__ordered_extent, btrfs_ordered_extent_lookup_range,
		610
		611	TP_PROTO(const struct btrfs_inode *inode,
		612	const struct btrfs_ordered_extent *ordered),
		613
		614	TP_ARGS(inode, ordered)
		615	);
		616
		617	DEFINE_EVENT(btrfs__ordered_extent, btrfs_ordered_extent_lookup_first_range,
		618
		619	TP_PROTO(const struct btrfs_inode *inode,
		620	const struct btrfs_ordered_extent *ordered),
		621
		622	TP_ARGS(inode, ordered)
		623	);
		624
		625	DEFINE_EVENT(btrfs__ordered_extent, btrfs_ordered_extent_lookup_for_logging,
		626
		627	TP_PROTO(const struct btrfs_inode *inode,
		628	const struct btrfs_ordered_extent *ordered),
		629
		630	TP_ARGS(inode, ordered)
		631	);
		632
		633	DEFINE_EVENT(btrfs__ordered_extent, btrfs_ordered_extent_lookup_first,
		634
		635	TP_PROTO(const struct btrfs_inode *inode,
		636	const struct btrfs_ordered_extent *ordered),
		637
		638	TP_ARGS(inode, ordered)
		639	);
		640
		641	DEFINE_EVENT(btrfs__ordered_extent, btrfs_ordered_extent_split,
		642
		643	TP_PROTO(const struct btrfs_inode *inode,
		644	const struct btrfs_ordered_extent *ordered),
		645
		646	TP_ARGS(inode, ordered)
		647	);
		648
		649	DEFINE_EVENT(btrfs__ordered_extent, btrfs_ordered_extent_dec_test_pending,
		650
		651	TP_PROTO(const struct btrfs_inode *inode,
		652	const struct btrfs_ordered_extent *ordered),
		653
		654	TP_ARGS(inode, ordered)
		655	);
		656
		657	DEFINE_EVENT(btrfs__ordered_extent, btrfs_ordered_extent_mark_finished,
		658
		659	TP_PROTO(const struct btrfs_inode *inode,
		660	const struct btrfs_ordered_extent *ordered),
		661
		662	TP_ARGS(inode, ordered)
		663	);
		664
599	DECLARE_EVENT_CLASS(btrfs__writepage,	665	DECLARE_EVENT_CLASS(btrfs__writepage,
600		666
601	TP_PROTO(const struct page page, const struct inode inode,	667	TP_PROTO(const struct page page, const struct inode inode,
@@ -2258,6 +2324,98 @@ DEFINE_EVENT(btrfs__space_info_update, update_bytes_pinned, · 1656 unmodified lines
2258	TP_ARGS(fs_info, sinfo, old, diff)	2324	TP_ARGS(fs_info, sinfo, old, diff)
2259	);	2325	);
2260		2326
		2327	DECLARE_EVENT_CLASS(btrfs_raid56_bio,
		2328
		2329	TP_PROTO(const struct btrfs_raid_bio *rbio,
		2330	const struct bio *bio,
		2331	const struct raid56_bio_trace_info *trace_info),
		2332
		2333	TP_ARGS(rbio, bio, trace_info),
		2334
		2335	TP_STRUCT__entry_btrfs(
		2336	__field( u64, full_stripe )
		2337	__field( u64, physical )
		2338	__field( u64, devid )
		2339	__field( u32, offset )
		2340	__field( u32, len )
		2341	__field( u8, opf )
		2342	__field( u8, total_stripes )
		2343	__field( u8, real_stripes )
		2344	__field( u8, nr_data )
		2345	__field( u8, stripe_nr )
		2346	),
		2347
		2348	TP_fast_assign_btrfs(rbio->bioc->fs_info,
		2349	__entry->full_stripe = rbio->bioc->raid_map[0];
		2350	__entry->physical = bio->bi_iter.bi_sector << SECTOR_SHIFT;
		2351	__entry->len = bio->bi_iter.bi_size;
		2352	__entry->opf = bio_op(bio);
		2353	__entry->devid = trace_info->devid;
		2354	__entry->offset = trace_info->offset;
		2355	__entry->stripe_nr = trace_info->stripe_nr;
		2356	__entry->total_stripes = rbio->bioc->num_stripes;
		2357	__entry->real_stripes = rbio->real_stripes;
		2358	__entry->nr_data = rbio->nr_data;
		2359	),
		2360	/*
		2361	* For type output, we need to output things like "DATA1"
		2362	* (the first data stripe), "DATA2" (the second data stripe),
		2363	* "PQ1" (P stripe),"PQ2" (Q stripe), "REPLACE0" (replace target device).
		2364	*/
		2365	TP_printk_btrfs(
		2366	"full_stripe=%llu devid=%lld type=%s%d offset=%d opf=0x%x physical=%llu len=%u",
		2367	__entry->full_stripe, __entry->devid,
		2368	(__entry->stripe_nr < __entry->nr_data) ? "DATA" :
		2369	((__entry->stripe_nr < __entry->real_stripes) ? "PQ" :
		2370	"REPLACE"),
		2371	(__entry->stripe_nr < __entry->nr_data) ?
		2372	(__entry->stripe_nr + 1) :
		2373	((__entry->stripe_nr < __entry->real_stripes) ?
		2374	(__entry->stripe_nr - __entry->nr_data + 1) : 0),
		2375	__entry->offset, __entry->opf, __entry->physical, __entry->len)
		2376	);
		2377
		2378	DEFINE_EVENT(btrfs_raid56_bio, raid56_read_partial,
		2379	TP_PROTO(const struct btrfs_raid_bio *rbio,
		2380	const struct bio *bio,
		2381	const struct raid56_bio_trace_info *trace_info),
		2382
		2383	TP_ARGS(rbio, bio, trace_info)
		2384	);
		2385
		2386	DEFINE_EVENT(btrfs_raid56_bio, raid56_write_stripe,
		2387	TP_PROTO(const struct btrfs_raid_bio *rbio,
		2388	const struct bio *bio,
		2389	const struct raid56_bio_trace_info *trace_info),
		2390
		2391	TP_ARGS(rbio, bio, trace_info)
		2392	);
		2393
		2394
		2395	DEFINE_EVENT(btrfs_raid56_bio, raid56_scrub_write_stripe,
		2396	TP_PROTO(const struct btrfs_raid_bio *rbio,
		2397	const struct bio *bio,
		2398	const struct raid56_bio_trace_info *trace_info),
		2399
		2400	TP_ARGS(rbio, bio, trace_info)
		2401	);
		2402
		2403	DEFINE_EVENT(btrfs_raid56_bio, raid56_scrub_read,
		2404	TP_PROTO(const struct btrfs_raid_bio *rbio,
		2405	const struct bio *bio,
		2406	const struct raid56_bio_trace_info *trace_info),
		2407
		2408	TP_ARGS(rbio, bio, trace_info)
		2409	);
		2410
		2411	DEFINE_EVENT(btrfs_raid56_bio, raid56_scrub_read_recover,
		2412	TP_PROTO(const struct btrfs_raid_bio *rbio,
		2413	const struct bio *bio,
		2414	const struct raid56_bio_trace_info *trace_info),
		2415
		2416	TP_ARGS(rbio, bio, trace_info)
		2417	);
		2418
2261	#endif /* _TRACE_BTRFS_H */	2419	#endif /* _TRACE_BTRFS_H */
2262		2420
2263	/* This part must be outside protection */	2421	/* This part must be outside protection */

include/uapi/linux/btrfs.h+9−1

  */
 #define BTRFS_SEND_FLAG_VERSION			0x8
+/*
+ * Send compressed data using the ENCODED_WRITE command instead of decompressing
+ * the data and sending it with the WRITE command. This requires protocol
+ * version >= 2.
+ */
+#define BTRFS_SEND_FLAG_COMPRESSED		0x10
 #define BTRFS_SEND_FLAG_MASK \
 	(BTRFS_SEND_FLAG_NO_FILE_DATA | \
 	 BTRFS_SEND_FLAG_OMIT_STREAM_HEADER | \
 	 BTRFS_SEND_FLAG_OMIT_END_CMD | \
-	 BTRFS_SEND_FLAG_VERSION)
+	 BTRFS_SEND_FLAG_VERSION | \
+	 BTRFS_SEND_FLAG_COMPRESSED)
 struct btrfs_ioctl_send_args {
 	__s64 send_fd;			/* in */

mm/highmem.c+1−1

@@ -561,7 +561,7 @@ void __kmap_local_page_prot(struct page page, pgprot_t prot) · 560 unmodified lines
561	}	561	}
562	EXPORT_SYMBOL(__kmap_local_page_prot);	562	EXPORT_SYMBOL(__kmap_local_page_prot);
563		563
564	void kunmap_local_indexed(void *vaddr)	564	void kunmap_local_indexed(const void *vaddr)
565	{	565	{
566	unsigned long addr = (unsigned long) vaddr & PAGE_MASK;	566	unsigned long addr = (unsigned long) vaddr & PAGE_MASK;
567	pte_t *kmap_pte;	567	pte_t *kmap_pte;

Merge tag 'for-5.20-tag' of git://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code

@@ -22,7 +22,7 @@ void flush_kernel_icache_range_asm(unsigned long, unsigned long); · 21 unmodified lines
22	void flush_user_dcache_range_asm(unsigned long, unsigned long);	22	void flush_user_dcache_range_asm(unsigned long, unsigned long);
23	void flush_kernel_dcache_range_asm(unsigned long, unsigned long);	23	void flush_kernel_dcache_range_asm(unsigned long, unsigned long);
24	void purge_kernel_dcache_range_asm(unsigned long, unsigned long);	24	void purge_kernel_dcache_range_asm(unsigned long, unsigned long);
25	void flush_kernel_dcache_page_asm(void *);	25	void flush_kernel_dcache_page_asm(const void *addr);
26	void flush_kernel_icache_page(void *);	26	void flush_kernel_icache_page(void *);
27		27
28	/* Cache flush operations */	28	/* Cache flush operations */
@@ -31,7 +31,7 @@ void flush_cache_all_local(void); · 2 unmodified lines
31	void flush_cache_all(void);	31	void flush_cache_all(void);
32	void flush_cache_mm(struct mm_struct *mm);	32	void flush_cache_mm(struct mm_struct *mm);
33		33
34	void flush_kernel_dcache_page_addr(void *addr);	34	void flush_kernel_dcache_page_addr(const void *addr);
35		35
36	#define flush_kernel_dcache_range(start,size) \	36	#define flush_kernel_dcache_range(start,size) \
37	flush_kernel_dcache_range_asm((start), (start)+(size));	37	flush_kernel_dcache_range_asm((start), (start)+(size));
@@ -75,7 +75,7 @@ void flush_dcache_page_asm(unsigned long phys_addr, unsigned long vaddr); · 37 unmodified lines
75	void flush_anon_page(struct vm_area_struct vma, struct page page, unsigned long vmaddr);	75	void flush_anon_page(struct vm_area_struct vma, struct page page, unsigned long vmaddr);
76		76
77	#define ARCH_HAS_FLUSH_ON_KUNMAP	77	#define ARCH_HAS_FLUSH_ON_KUNMAP
78	static inline void kunmap_flush_on_unmap(void *addr)	78	static inline void kunmap_flush_on_unmap(const void *addr)
79	{	79	{
80	flush_kernel_dcache_page_addr(addr);	80	flush_kernel_dcache_page_addr(addr);
81	}	81	}