lael-labs/linux

drivers/md/dm-bio-prison.c+0−25

@@ -207,31 +207,6 @@ void dm_cell_release(struct dm_bio_prison_cell cell, struct bio_list bios) · 206 unmodified lines
207	}	207	}
208	EXPORT_SYMBOL_GPL(dm_cell_release);	208	EXPORT_SYMBOL_GPL(dm_cell_release);
209		209
210	/*
211	* There are a couple of places where we put a bio into a cell briefly
212	* before taking it out again. In these situations we know that no other
213	* bio may be in the cell. This function releases the cell, and also does
214	* a sanity check.
215	*/
216	static void __cell_release_singleton(struct dm_bio_prison_cell cell, struct bio bio)
217	{
218	BUG_ON(cell->holder != bio);
219	BUG_ON(!bio_list_empty(&cell->bios));
220
221	__cell_release(cell, NULL);
222	}
223
224	void dm_cell_release_singleton(struct dm_bio_prison_cell cell, struct bio bio)
225	{
226	unsigned long flags;
227	struct dm_bio_prison *prison = cell->prison;
228
229	spin_lock_irqsave(&prison->lock, flags);
230	__cell_release_singleton(cell, bio);
231	spin_unlock_irqrestore(&prison->lock, flags);
232	}
233	EXPORT_SYMBOL_GPL(dm_cell_release_singleton);
234
235	/*	210	/*
236	* Sometimes we don't want the holder, just the additional bios.	211	* Sometimes we don't want the holder, just the additional bios.
237	*/	212	*/

drivers/md/dm-bio-prison.h+0−1

@@ -44,7 +44,6 @@ int dm_bio_detain(struct dm_bio_prison prison, struct dm_cell_key key, · 43 unmodified lines
44	struct bio inmate, struct dm_bio_prison_cell *ref);	44	struct bio inmate, struct dm_bio_prison_cell *ref);
45		45
46	void dm_cell_release(struct dm_bio_prison_cell cell, struct bio_list bios);	46	void dm_cell_release(struct dm_bio_prison_cell cell, struct bio_list bios);
47	void dm_cell_release_singleton(struct dm_bio_prison_cell cell, struct bio bio); // FIXME: bio arg not needed
48	void dm_cell_release_no_holder(struct dm_bio_prison_cell cell, struct bio_list inmates);	47	void dm_cell_release_no_holder(struct dm_bio_prison_cell cell, struct bio_list inmates);
49	void dm_cell_error(struct dm_bio_prison_cell *cell);	48	void dm_cell_error(struct dm_bio_prison_cell *cell);
50		49

drivers/md/dm-crypt.c+2−3

 	return ret;
 }
-static int crypt_map(struct dm_target *ti, struct bio *bio,
+static int crypt_map(struct dm_target *ti, struct bio *bio)
-		     union map_info *map_context)
 {
 	struct dm_crypt_io *io;
 	struct crypt_config *cc = ti->private;
 static struct target_type crypt_target = {
 	.name   = "crypt",
-	.version = {1, 11, 0},
+	.version = {1, 12, 0},
 	.module = THIS_MODULE,
 	.ctr    = crypt_ctr,
 	.dtr    = crypt_dtr,

drivers/md/dm-delay.c+2−3

 	atomic_set(&dc->may_delay, 1);
 }
-static int delay_map(struct dm_target *ti, struct bio *bio,
+static int delay_map(struct dm_target *ti, struct bio *bio)
-		     union map_info *map_context)
 {
 	struct delay_c *dc = ti->private;
 static struct target_type delay_target = {
 	.name	     = "delay",
-	.version     = {1, 1, 0},
+	.version     = {1, 2, 0},
 	.module      = THIS_MODULE,
 	.ctr	     = delay_ctr,
 	.dtr	     = delay_dtr,

drivers/md/dm-flakey.c+13−8

 	DROP_WRITES
 };
+struct per_bio_data {
+	bool bio_submitted;
+};
 static int parse_features(struct dm_arg_set *as, struct flakey_c *fc,
 			  struct dm_target *ti)
 {
 	ti->num_flush_requests = 1;
 	ti->num_discard_requests = 1;
+	ti->per_bio_data_size = sizeof(struct per_bio_data);
 	ti->private = fc;
 	return 0;
 	}
 }
-static int flakey_map(struct dm_target *ti, struct bio *bio,
+static int flakey_map(struct dm_target *ti, struct bio *bio)
-		      union map_info *map_context)
 {
 	struct flakey_c *fc = ti->private;
 	unsigned elapsed;
+	struct per_bio_data *pb = dm_per_bio_data(bio, sizeof(struct per_bio_data));
+	pb->bio_submitted = false;
 	/* Are we alive ? */
 	elapsed = (jiffies - fc->start_time) / HZ;
 		/*
 		 * Flag this bio as submitted while down.
 		 */
-		map_context->ll = 1;
+		pb->bio_submitted = true;
 		/*
 		 * Map reads as normal.
 	return DM_MAPIO_REMAPPED;
 }
-static int flakey_end_io(struct dm_target *ti, struct bio *bio,
+static int flakey_end_io(struct dm_target *ti, struct bio *bio, int error)
-			 int error, union map_info *map_context)
 {
 	struct flakey_c *fc = ti->private;
-	unsigned bio_submitted_while_down = map_context->ll;
+	struct per_bio_data *pb = dm_per_bio_data(bio, sizeof(struct per_bio_data));
 	/*
 	 * Corrupt successful READs while in down state.
 	 * If flags were specified, only corrupt those that match.
 	 */
-	if (fc->corrupt_bio_byte && !error && bio_submitted_while_down &&
+	if (fc->corrupt_bio_byte && !error && pb->bio_submitted &&
 	    (bio_data_dir(bio) == READ) && (fc->corrupt_bio_rw == READ) &&
 	    all_corrupt_bio_flags_match(bio, fc))
 		corrupt_bio_data(bio, fc);
 static struct target_type flakey_target = {
 	.name   = "flakey",
-	.version = {1, 2, 0},
+	.version = {1, 3, 0},
 	.module = THIS_MODULE,
 	.ctr    = flakey_ctr,
 	.dtr    = flakey_dtr,

drivers/md/dm-io.c+18−5

 	unsigned num_bvecs;
 	sector_t remaining = where->count;
 	struct request_queue *q = bdev_get_queue(where->bdev);
-	sector_t discard_sectors;
+	unsigned short logical_block_size = queue_logical_block_size(q);
+	sector_t num_sectors;
 	/*
 	 * where->count may be zero if rw holds a flush and we need to
 		/*
 		 * Allocate a suitably sized-bio.
 		 */
-		if (rw & REQ_DISCARD)
+		if ((rw & REQ_DISCARD) || (rw & REQ_WRITE_SAME))
 			num_bvecs = 1;
 		else
 			num_bvecs = min_t(int, bio_get_nr_vecs(where->bdev),
 		store_io_and_region_in_bio(bio, io, region);
 		if (rw & REQ_DISCARD) {
-			discard_sectors = min_t(sector_t, q->limits.max_discard_sectors, remaining);
+			num_sectors = min_t(sector_t, q->limits.max_discard_sectors, remaining);
-			bio->bi_size = discard_sectors << SECTOR_SHIFT;
+			bio->bi_size = num_sectors << SECTOR_SHIFT;
-			remaining -= discard_sectors;
+			remaining -= num_sectors;
+		} else if (rw & REQ_WRITE_SAME) {
+			/*
+			 * WRITE SAME only uses a single page.
+			 */
+			dp->get_page(dp, &page, &len, &offset);
+			bio_add_page(bio, page, logical_block_size, offset);
+			num_sectors = min_t(sector_t, q->limits.max_write_same_sectors, remaining);
+			bio->bi_size = num_sectors << SECTOR_SHIFT;
+			offset = 0;
+			remaining -= num_sectors;
+			dp->next_page(dp);
 		} else while (remaining) {
 			/*
 			 * Try and add as many pages as possible.

drivers/md/dm-ioctl.c+43−21

@@ -1543,7 +1543,21 @@ static int check_version(unsigned int cmd, struct dm_ioctl __user *user) · 1542 unmodified lines
1543	return r;	1543	return r;
1544	}	1544	}
1545		1545
1546	static int copy_params(struct dm_ioctl __user user, struct dm_ioctl *param)	1546	#define DM_PARAMS_VMALLOC 0x0001 /* Params alloced with vmalloc not kmalloc */
		1547	#define DM_WIPE_BUFFER 0x0010 /* Wipe input buffer before returning from ioctl */
		1548
		1549	static void free_params(struct dm_ioctl *param, size_t param_size, int param_flags)
		1550	{
		1551	if (param_flags & DM_WIPE_BUFFER)
		1552	memset(param, 0, param_size);
		1553
		1554	if (param_flags & DM_PARAMS_VMALLOC)
		1555	vfree(param);
		1556	else
		1557	kfree(param);
		1558	}
		1559
		1560	static int copy_params(struct dm_ioctl __user user, struct dm_ioctl param, int param_flags)
1547	{	1561	{
1548	struct dm_ioctl tmp, *dmi;	1562	struct dm_ioctl tmp, *dmi;
1549	int secure_data;	1563	int secure_data;
@@ -1556,7 +1570,21 @@ static int copy_params(struct dm_ioctl __user user, struct dm_ioctl *param) · 6 unmodified lines
1556		1570
1557	secure_data = tmp.flags & DM_SECURE_DATA_FLAG;	1571	secure_data = tmp.flags & DM_SECURE_DATA_FLAG;
1558		1572
1559	dmi = vmalloc(tmp.data_size);	1573	*param_flags = secure_data ? DM_WIPE_BUFFER : 0;
		1574
		1575	/*
		1576	* Try to avoid low memory issues when a device is suspended.
		1577	* Use kmalloc() rather than vmalloc() when we can.
		1578	*/
		1579	dmi = NULL;
		1580	if (tmp.data_size <= KMALLOC_MAX_SIZE)
		1581	dmi = kmalloc(tmp.data_size, GFP_NOIO \| __GFP_NORETRY \| __GFP_NOMEMALLOC \| __GFP_NOWARN);
		1582
		1583	if (!dmi) {
		1584	dmi = __vmalloc(tmp.data_size, GFP_NOIO \| __GFP_REPEAT \| __GFP_HIGH, PAGE_KERNEL);
		1585	*param_flags \|= DM_PARAMS_VMALLOC;
		1586	}
		1587
1560	if (!dmi) {	1588	if (!dmi) {
1561	if (secure_data && clear_user(user, tmp.data_size))	1589	if (secure_data && clear_user(user, tmp.data_size))
1562	return -EFAULT;	1590	return -EFAULT;
@@ -1566,6 +1594,14 @@ static int copy_params(struct dm_ioctl __user user, struct dm_ioctl *param) · 3 unmodified lines
1566	if (copy_from_user(dmi, user, tmp.data_size))	1594	if (copy_from_user(dmi, user, tmp.data_size))
1567	goto bad;	1595	goto bad;
1568		1596
		1597	/*
		1598	* Abort if something changed the ioctl data while it was being copied.
		1599	*/
		1600	if (dmi->data_size != tmp.data_size) {
		1601	DMERR("rejecting ioctl: data size modified while processing parameters");
		1602	goto bad;
		1603	}
		1604
1569	/* Wipe the user buffer so we do not return it to userspace */	1605	/* Wipe the user buffer so we do not return it to userspace */
1570	if (secure_data && clear_user(user, tmp.data_size))	1606	if (secure_data && clear_user(user, tmp.data_size))
1571	goto bad;	1607	goto bad;
@@ -1574,9 +1610,8 @@ static int copy_params(struct dm_ioctl __user user, struct dm_ioctl *param) · 2 unmodified lines
1574	return 0;	1610	return 0;
1575		1611
1576	bad:	1612	bad:
1577	if (secure_data)	1613	free_params(dmi, tmp.data_size, *param_flags);
1578	memset(dmi, 0, tmp.data_size);	1614
1579	vfree(dmi);
1580	return -EFAULT;	1615	return -EFAULT;
1581	}	1616	}
1582		1617
@@ -1613,7 +1648,7 @@ static int validate_params(uint cmd, struct dm_ioctl *param) · 30 unmodified lines
1613	static int ctl_ioctl(uint command, struct dm_ioctl __user *user)	1648	static int ctl_ioctl(uint command, struct dm_ioctl __user *user)
1614	{	1649	{
1615	int r = 0;	1650	int r = 0;
1616	int wipe_buffer;	1651	int param_flags;
1617	unsigned int cmd;	1652	unsigned int cmd;
1618	struct dm_ioctl *uninitialized_var(param);	1653	struct dm_ioctl *uninitialized_var(param);
1619	ioctl_fn fn = NULL;	1654	ioctl_fn fn = NULL;
@@ -1648,25 +1683,15 @@ static int ctl_ioctl(uint command, struct dm_ioctl __user *user) · 28 unmodified lines
1648	return -ENOTTY;	1683	return -ENOTTY;
1649	}	1684	}
1650		1685
1651	/*
1652	* Trying to avoid low memory issues when a device is
1653	* suspended.
1654	*/
1655	current->flags \|= PF_MEMALLOC;
1656
1657	/*	1686	/*
1658	* Copy the parameters into kernel space.	1687	* Copy the parameters into kernel space.
1659	*/	1688	*/
1660	r = copy_params(user, &param);	1689	r = copy_params(user, &param, &param_flags);
1661
1662	current->flags &= ~PF_MEMALLOC;
1663		1690
1664	if (r)	1691	if (r)
1665	return r;	1692	return r;
1666		1693
1667	input_param_size = param->data_size;	1694	input_param_size = param->data_size;
1668	wipe_buffer = param->flags & DM_SECURE_DATA_FLAG;
1669
1670	r = validate_params(cmd, param);	1695	r = validate_params(cmd, param);
1671	if (r)	1696	if (r)
1672	goto out;	1697	goto out;
@@ -1681,10 +1706,7 @@ static int ctl_ioctl(uint command, struct dm_ioctl __user *user) · 8 unmodified lines
1681	r = -EFAULT;	1706	r = -EFAULT;
1682		1707
1683	out:	1708	out:
1684	if (wipe_buffer)	1709	free_params(param, input_param_size, param_flags);
1685	memset(param, 0, input_param_size);
1686
1687	vfree(param);
1688	return r;	1710	return r;
1689	}	1711	}
1690		1712

drivers/md/dm-kcopyd.c+14−4

@@ -349,7 +349,7 @@ static void complete_io(unsigned long error, void *context) · 348 unmodified lines
349	struct dm_kcopyd_client *kc = job->kc;	349	struct dm_kcopyd_client *kc = job->kc;
350		350
351	if (error) {	351	if (error) {
352	if (job->rw == WRITE)	352	if (job->rw & WRITE)
353	job->write_err \|= error;	353	job->write_err \|= error;
354	else	354	else
355	job->read_err = 1;	355	job->read_err = 1;
@@ -361,7 +361,7 @@ static void complete_io(unsigned long error, void *context) · 5 unmodified lines
361	}	361	}
362	}	362	}
363		363
364	if (job->rw == WRITE)	364	if (job->rw & WRITE)
365	push(&kc->complete_jobs, job);	365	push(&kc->complete_jobs, job);
366		366
367	else {	367	else {
@@ -432,7 +432,7 @@ static int process_jobs(struct list_head jobs, struct dm_kcopyd_client kc, · 64 unmodified lines
432		432
433	if (r < 0) {	433	if (r < 0) {
434	/* error this rogue job */	434	/* error this rogue job */
435	if (job->rw == WRITE)	435	if (job->rw & WRITE)
436	job->write_err = (unsigned long) -1L;	436	job->write_err = (unsigned long) -1L;
437	else	437	else
438	job->read_err = 1;	438	job->read_err = 1;
@@ -585,6 +585,7 @@ int dm_kcopyd_copy(struct dm_kcopyd_client kc, struct dm_io_region from, · 146 unmodified lines
585	unsigned int flags, dm_kcopyd_notify_fn fn, void *context)	585	unsigned int flags, dm_kcopyd_notify_fn fn, void *context)
586	{	586	{
587	struct kcopyd_job *job;	587	struct kcopyd_job *job;
		588	int i;
588		589
589	/*	590	/*
590	* Allocate an array of jobs consisting of one master job	591	* Allocate an array of jobs consisting of one master job
@@ -611,7 +612,16 @@ int dm_kcopyd_copy(struct dm_kcopyd_client kc, struct dm_io_region from, · 20 unmodified lines
611	memset(&job->source, 0, sizeof job->source);	612	memset(&job->source, 0, sizeof job->source);
612	job->source.count = job->dests[0].count;	613	job->source.count = job->dests[0].count;
613	job->pages = &zero_page_list;	614	job->pages = &zero_page_list;
614	job->rw = WRITE;	615
		616	/*
		617	* Use WRITE SAME to optimize zeroing if all dests support it.
		618	*/
		619	job->rw = WRITE \| REQ_WRITE_SAME;
		620	for (i = 0; i < job->num_dests; i++)
		621	if (!bdev_write_same(job->dests[i].bdev)) {
		622	job->rw = WRITE;
		623	break;
		624	}
615	}	625	}
616		626
617	job->fn = fn;	627	job->fn = fn;

drivers/md/dm-linear.c+3−3

 	ti->num_flush_requests = 1;
 	ti->num_discard_requests = 1;
+	ti->num_write_same_requests = 1;
 	ti->private = lc;
 	return 0;
 		bio->bi_sector = linear_map_sector(ti, bio->bi_sector);
 }
-static int linear_map(struct dm_target *ti, struct bio *bio,
+static int linear_map(struct dm_target *ti, struct bio *bio)
-		      union map_info *map_context)
 {
 	linear_map_bio(ti, bio);
 static struct target_type linear_target = {
 	.name   = "linear",
-	.version = {1, 1, 0},
+	.version = {1, 2, 0},
 	.module = THIS_MODULE,
 	.ctr    = linear_ctr,
 	.dtr    = linear_dtr,

drivers/md/dm-raid.c+5−3

 		 * Choose a reasonable default.  All figures in sectors.
 		 */
 		if (min_region_size > (1 << 13)) {
+			/* If not a power of 2, make it the next power of 2 */
+			if (min_region_size & (min_region_size - 1))
+				region_size = 1 << fls(region_size);
 			DMINFO("Choosing default region size of %lu sectors",
 			       region_size);
-			region_size = min_region_size;
 		} else {
 			DMINFO("Choosing default region size of 4MiB");
 			region_size = 1 << 13; /* sectors */
 	context_free(rs);
 }
-static int raid_map(struct dm_target *ti, struct bio *bio, union map_info *map_context)
+static int raid_map(struct dm_target *ti, struct bio *bio)
 {
 	struct raid_set *rs = ti->private;
 	struct mddev *mddev = &rs->md;
 static struct target_type raid_target = {
 	.name = "raid",
-	.version = {1, 3, 1},
+	.version = {1, 4, 0},
 	.module = THIS_MODULE,
 	.ctr = raid_ctr,
 	.dtr = raid_dtr,

drivers/md/dm-raid1.c+23−52

 	struct dm_region_hash *rh;
 	struct dm_kcopyd_client *kcopyd_client;
 	struct dm_io_client *io_client;
-	mempool_t *read_record_pool;
 	/* recovery */
 	region_t nr_regions;
 		queue_bio(ms, bio, WRITE);
 }
-#define MIN_READ_RECORDS 20
+struct dm_raid1_bio_record {
-struct dm_raid1_read_record {
 	struct mirror *m;
+	/* if details->bi_bdev == NULL, details were not saved */
 	struct dm_bio_details details;
+	region_t write_region;
 };
-static struct kmem_cache *_dm_raid1_read_record_cache;
 /*
  * Every mirror should look like this one.
  */
 	atomic_set(&ms->suspend, 0);
 	atomic_set(&ms->default_mirror, DEFAULT_MIRROR);
-	ms->read_record_pool = mempool_create_slab_pool(MIN_READ_RECORDS,
-						_dm_raid1_read_record_cache);
-	if (!ms->read_record_pool) {
-		ti->error = "Error creating mirror read_record_pool";
-		kfree(ms);
-		return NULL;
-	}
 	ms->io_client = dm_io_client_create();
 	if (IS_ERR(ms->io_client)) {
 		ti->error = "Error creating dm_io client";
-		mempool_destroy(ms->read_record_pool);
 		kfree(ms);
  		return NULL;
 	}
 	if (IS_ERR(ms->rh)) {
 		ti->error = "Error creating dirty region hash";
 		dm_io_client_destroy(ms->io_client);
-		mempool_destroy(ms->read_record_pool);
 		kfree(ms);
 		return NULL;
 	}
 	dm_io_client_destroy(ms->io_client);
 	dm_region_hash_destroy(ms->rh);
-	mempool_destroy(ms->read_record_pool);
 	kfree(ms);
 }
 	ti->num_flush_requests = 1;
 	ti->num_discard_requests = 1;
+	ti->per_bio_data_size = sizeof(struct dm_raid1_bio_record);
 	ti->discard_zeroes_data_unsupported = true;
 	ms->kmirrord_wq = alloc_workqueue("kmirrord",
 /*
  * Mirror mapping function
  */
-static int mirror_map(struct dm_target *ti, struct bio *bio,
+static int mirror_map(struct dm_target *ti, struct bio *bio)
-		      union map_info *map_context)
 {
 	int r, rw = bio_rw(bio);
 	struct mirror *m;
 	struct mirror_set *ms = ti->private;
-	struct dm_raid1_read_record *read_record = NULL;
 	struct dm_dirty_log *log = dm_rh_dirty_log(ms->rh);
+	struct dm_raid1_bio_record *bio_record =
+	  dm_per_bio_data(bio, sizeof(struct dm_raid1_bio_record));
+	bio_record->details.bi_bdev = NULL;
 	if (rw == WRITE) {
 		/* Save region for mirror_end_io() handler */
-		map_context->ll = dm_rh_bio_to_region(ms->rh, bio);
+		bio_record->write_region = dm_rh_bio_to_region(ms->rh, bio);
 		queue_bio(ms, bio, rw);
 		return DM_MAPIO_SUBMITTED;
 	}
 	if (unlikely(!m))
 		return -EIO;
-	read_record = mempool_alloc(ms->read_record_pool, GFP_NOIO);
+	dm_bio_record(&bio_record->details, bio);
-	if (likely(read_record)) {
+	bio_record->m = m;
-		dm_bio_record(&read_record->details, bio);
-		map_context->ptr = read_record;
-		read_record->m = m;
-	}
 	map_bio(m, bio);
 	return DM_MAPIO_REMAPPED;
 }
-static int mirror_end_io(struct dm_target *ti, struct bio *bio,
+static int mirror_end_io(struct dm_target *ti, struct bio *bio, int error)
-			 int error, union map_info *map_context)
 {
 	int rw = bio_rw(bio);
 	struct mirror_set *ms = (struct mirror_set *) ti->private;
 	struct mirror *m = NULL;
 	struct dm_bio_details *bd = NULL;
-	struct dm_raid1_read_record *read_record = map_context->ptr;
+	struct dm_raid1_bio_record *bio_record =
+	  dm_per_bio_data(bio, sizeof(struct dm_raid1_bio_record));
 	/*
 	 * We need to dec pending if this was a write.
 	 */
 	if (rw == WRITE) {
 		if (!(bio->bi_rw & (REQ_FLUSH | REQ_DISCARD)))
-			dm_rh_dec(ms->rh, map_context->ll);
+			dm_rh_dec(ms->rh, bio_record->write_region);
 		return error;
 	}
 		goto out;
 	if (unlikely(error)) {
-		if (!read_record) {
+		if (!bio_record->details.bi_bdev) {
 			/*
 			 * There wasn't enough memory to record necessary
 			 * information for a retry or there was no other
 			return -EIO;
 		}
-		m = read_record->m;
+		m = bio_record->m;
 		DMERR("Mirror read failed from %s. Trying alternative device.",
 		      m->dev->name);
 		 * mirror.
 		 */
 		if (default_ok(m) || mirror_available(ms, bio)) {
-			bd = &read_record->details;
+			bd = &bio_record->details;
 			dm_bio_restore(bd, bio);
-			mempool_free(read_record, ms->read_record_pool);
+			bio_record->details.bi_bdev = NULL;
-			map_context->ptr = NULL;
 			queue_bio(ms, bio, rw);
-			return 1;
+			return DM_ENDIO_INCOMPLETE;
 		}
 		DMERR("All replicated volumes dead, failing I/O");
 	}
 out:
-	if (read_record) {
+	bio_record->details.bi_bdev = NULL;
-		mempool_free(read_record, ms->read_record_pool);
-		map_context->ptr = NULL;
-	}
 	return error;
 }
 static struct target_type mirror_target = {
 	.name	 = "mirror",
-	.version = {1, 12, 1},
+	.version = {1, 13, 1},
 	.module	 = THIS_MODULE,
 	.ctr	 = mirror_ctr,
 	.dtr	 = mirror_dtr,
 {
 	int r;
-	_dm_raid1_read_record_cache = KMEM_CACHE(dm_raid1_read_record, 0);
-	if (!_dm_raid1_read_record_cache) {
-		DMERR("Can't allocate dm_raid1_read_record cache");
-		r = -ENOMEM;
-		goto bad_cache;
-	}
 	r = dm_register_target(&mirror_target);
 	if (r < 0) {
 		DMERR("Failed to register mirror target");
 	return 0;
 bad_target:
-	kmem_cache_destroy(_dm_raid1_read_record_cache);
-bad_cache:
 	return r;
 }
 static void __exit dm_mirror_exit(void)
 {
 	dm_unregister_target(&mirror_target);
-	kmem_cache_destroy(_dm_raid1_read_record_cache);
 }
 /* Module hooks */

drivers/md/dm-snap.c+34−56

 	/* Chunks with outstanding reads */
 	spinlock_t tracked_chunk_lock;
-	mempool_t *tracked_chunk_pool;
 	struct hlist_head tracked_chunk_hash[DM_TRACKED_CHUNK_HASH_SIZE];
 	/* The on disk metadata handler */
 	chunk_t chunk;
 };
-static struct kmem_cache *tracked_chunk_cache;
+static void init_tracked_chunk(struct bio *bio)
+{
+	struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk));
+	INIT_HLIST_NODE(&c->node);
+}
-static struct dm_snap_tracked_chunk *track_chunk(struct dm_snapshot *s,
+static bool is_bio_tracked(struct bio *bio)
-						 chunk_t chunk)
 {
-	struct dm_snap_tracked_chunk *c = mempool_alloc(s->tracked_chunk_pool,
+	struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk));
-							GFP_NOIO);
+	return !hlist_unhashed(&c->node);
-	unsigned long flags;
+}
+static void track_chunk(struct dm_snapshot *s, struct bio *bio, chunk_t chunk)
+{
+	struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk));
 	c->chunk = chunk;
-	spin_lock_irqsave(&s->tracked_chunk_lock, flags);
+	spin_lock_irq(&s->tracked_chunk_lock);
 	hlist_add_head(&c->node,
 		       &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)]);
-	spin_unlock_irqrestore(&s->tracked_chunk_lock, flags);
+	spin_unlock_irq(&s->tracked_chunk_lock);
-	return c;
 }
-static void stop_tracking_chunk(struct dm_snapshot *s,
+static void stop_tracking_chunk(struct dm_snapshot *s, struct bio *bio)
-				struct dm_snap_tracked_chunk *c)
 {
+	struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk));
 	unsigned long flags;
 	spin_lock_irqsave(&s->tracked_chunk_lock, flags);
 	hlist_del(&c->node);
 	spin_unlock_irqrestore(&s->tracked_chunk_lock, flags);
-	mempool_free(c, s->tracked_chunk_pool);
 }
 static int __chunk_is_tracked(struct dm_snapshot *s, chunk_t chunk)
 		goto bad_pending_pool;
 	}
-	s->tracked_chunk_pool = mempool_create_slab_pool(MIN_IOS,
-							 tracked_chunk_cache);
-	if (!s->tracked_chunk_pool) {
-		ti->error = "Could not allocate tracked_chunk mempool for "
-			    "tracking reads";
-		goto bad_tracked_chunk_pool;
-	}
 	for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++)
 		INIT_HLIST_HEAD(&s->tracked_chunk_hash[i]);
 	ti->private = s;
 	ti->num_flush_requests = num_flush_requests;
+	ti->per_bio_data_size = sizeof(struct dm_snap_tracked_chunk);
 	/* Add snapshot to the list of snapshots for this origin */
 	/* Exceptions aren't triggered till snapshot_resume() is called */
 	unregister_snapshot(s);
 bad_load_and_register:
-	mempool_destroy(s->tracked_chunk_pool);
-bad_tracked_chunk_pool:
 	mempool_destroy(s->pending_pool);
 bad_pending_pool:
 		BUG_ON(!hlist_empty(&s->tracked_chunk_hash[i]));
 #endif
-	mempool_destroy(s->tracked_chunk_pool);
 	__free_exceptions(s);
 	mempool_destroy(s->pending_pool);
 					  s->store->chunk_mask);
 }
-static int snapshot_map(struct dm_target *ti, struct bio *bio,
+static int snapshot_map(struct dm_target *ti, struct bio *bio)
-			union map_info *map_context)
 {
 	struct dm_exception *e;
 	struct dm_snapshot *s = ti->private;
 	chunk_t chunk;
 	struct dm_snap_pending_exception *pe = NULL;
+	init_tracked_chunk(bio);
 	if (bio->bi_rw & REQ_FLUSH) {
 		bio->bi_bdev = s->cow->bdev;
 		return DM_MAPIO_REMAPPED;
 		}
 	} else {
 		bio->bi_bdev = s->origin->bdev;
-		map_context->ptr = track_chunk(s, chunk);
+		track_chunk(s, bio, chunk);
 	}
 out_unlock:
  * If merging is currently taking place on the chunk in question, the
  * I/O is deferred by adding it to s->bios_queued_during_merge.
  */
-static int snapshot_merge_map(struct dm_target *ti, struct bio *bio,
+static int snapshot_merge_map(struct dm_target *ti, struct bio *bio)
-			      union map_info *map_context)
 {
 	struct dm_exception *e;
 	struct dm_snapshot *s = ti->private;
 	int r = DM_MAPIO_REMAPPED;
 	chunk_t chunk;
+	init_tracked_chunk(bio);
 	if (bio->bi_rw & REQ_FLUSH) {
-		if (!map_context->target_request_nr)
+		if (!dm_bio_get_target_request_nr(bio))
 			bio->bi_bdev = s->origin->bdev;
 		else
 			bio->bi_bdev = s->cow->bdev;
-		map_context->ptr = NULL;
 		return DM_MAPIO_REMAPPED;
 	}
 		remap_exception(s, e, bio, chunk);
 		if (bio_rw(bio) == WRITE)
-			map_context->ptr = track_chunk(s, chunk);
+			track_chunk(s, bio, chunk);
 		goto out_unlock;
 	}
 	return r;
 }
-static int snapshot_end_io(struct dm_target *ti, struct bio *bio,
+static int snapshot_end_io(struct dm_target *ti, struct bio *bio, int error)
-			   int error, union map_info *map_context)
 {
 	struct dm_snapshot *s = ti->private;
-	struct dm_snap_tracked_chunk *c = map_context->ptr;
-	if (c)
+	if (is_bio_tracked(bio))
-		stop_tracking_chunk(s, c);
+		stop_tracking_chunk(s, bio);
 	return 0;
 }
 	dm_put_device(ti, dev);
 }
-static int origin_map(struct dm_target *ti, struct bio *bio,
+static int origin_map(struct dm_target *ti, struct bio *bio)
-		      union map_info *map_context)
 {
 	struct dm_dev *dev = ti->private;
 	bio->bi_bdev = dev->bdev;
 static struct target_type origin_target = {
 	.name    = "snapshot-origin",
-	.version = {1, 7, 1},
+	.version = {1, 8, 0},
 	.module  = THIS_MODULE,
 	.ctr     = origin_ctr,
 	.dtr     = origin_dtr,
 static struct target_type snapshot_target = {
 	.name    = "snapshot",
-	.version = {1, 10, 0},
+	.version = {1, 11, 0},
 	.module  = THIS_MODULE,
 	.ctr     = snapshot_ctr,
 	.dtr     = snapshot_dtr,
 static struct target_type merge_target = {
 	.name    = dm_snapshot_merge_target_name,
-	.version = {1, 1, 0},
+	.version = {1, 2, 0},
 	.module  = THIS_MODULE,
 	.ctr     = snapshot_ctr,
 	.dtr     = snapshot_dtr,
 		goto bad_pending_cache;
 	}
-	tracked_chunk_cache = KMEM_CACHE(dm_snap_tracked_chunk, 0);
-	if (!tracked_chunk_cache) {
-		DMERR("Couldn't create cache to track chunks in use.");
-		r = -ENOMEM;
-		goto bad_tracked_chunk_cache;
-	}
 	return 0;
-bad_tracked_chunk_cache:
-	kmem_cache_destroy(pending_cache);
 bad_pending_cache:
 	kmem_cache_destroy(exception_cache);
 bad_exception_cache:
 	exit_origin_hash();
 	kmem_cache_destroy(pending_cache);
 	kmem_cache_destroy(exception_cache);
-	kmem_cache_destroy(tracked_chunk_cache);
 	dm_exception_store_exit();
 }

drivers/md/dm-stripe.c+10−10

 	ti->num_flush_requests = stripes;
 	ti->num_discard_requests = stripes;
+	ti->num_write_same_requests = stripes;
 	sc->chunk_size = chunk_size;
 	if (chunk_size & (chunk_size - 1))
 		*result += sc->chunk_size;		/* next chunk */
 }
-static int stripe_map_discard(struct stripe_c *sc, struct bio *bio,
+static int stripe_map_range(struct stripe_c *sc, struct bio *bio,
-			      uint32_t target_stripe)
+			    uint32_t target_stripe)
 {
 	sector_t begin, end;
 	}
 }
-static int stripe_map(struct dm_target *ti, struct bio *bio,
+static int stripe_map(struct dm_target *ti, struct bio *bio)
-		      union map_info *map_context)
 {
 	struct stripe_c *sc = ti->private;
 	uint32_t stripe;
 	unsigned target_request_nr;
 	if (bio->bi_rw & REQ_FLUSH) {
-		target_request_nr = map_context->target_request_nr;
+		target_request_nr = dm_bio_get_target_request_nr(bio);
 		BUG_ON(target_request_nr >= sc->stripes);
 		bio->bi_bdev = sc->stripe[target_request_nr].dev->bdev;
 		return DM_MAPIO_REMAPPED;
 	}
-	if (unlikely(bio->bi_rw & REQ_DISCARD)) {
+	if (unlikely(bio->bi_rw & REQ_DISCARD) ||
-		target_request_nr = map_context->target_request_nr;
+	    unlikely(bio->bi_rw & REQ_WRITE_SAME)) {
+		target_request_nr = dm_bio_get_target_request_nr(bio);
 		BUG_ON(target_request_nr >= sc->stripes);
-		return stripe_map_discard(sc, bio, target_request_nr);
+		return stripe_map_range(sc, bio, target_request_nr);
 	}
 	stripe_map_sector(sc, bio->bi_sector, &stripe, &bio->bi_sector);
 	return 0;
 }
-static int stripe_end_io(struct dm_target *ti, struct bio *bio,
+static int stripe_end_io(struct dm_target *ti, struct bio *bio, int error)
-			 int error, union map_info *map_context)
 {
 	unsigned i;
 	char major_minor[16];

drivers/md/dm-table.c+40−1

@@ -967,13 +967,22 @@ bool dm_table_request_based(struct dm_table *t) · 966 unmodified lines
967	int dm_table_alloc_md_mempools(struct dm_table *t)	967	int dm_table_alloc_md_mempools(struct dm_table *t)
968	{	968	{
969	unsigned type = dm_table_get_type(t);	969	unsigned type = dm_table_get_type(t);
		970	unsigned per_bio_data_size = 0;
		971	struct dm_target *tgt;
		972	unsigned i;
970		973
971	if (unlikely(type == DM_TYPE_NONE)) {	974	if (unlikely(type == DM_TYPE_NONE)) {
972	DMWARN("no table type is set, can't allocate mempools");	975	DMWARN("no table type is set, can't allocate mempools");
973	return -EINVAL;	976	return -EINVAL;
974	}	977	}
975		978
976	t->mempools = dm_alloc_md_mempools(type, t->integrity_supported);	979	if (type == DM_TYPE_BIO_BASED)
		980	for (i = 0; i < t->num_targets; i++) {
		981	tgt = t->targets + i;
		982	per_bio_data_size = max(per_bio_data_size, tgt->per_bio_data_size);
		983	}
		984
		985	t->mempools = dm_alloc_md_mempools(type, t->integrity_supported, per_bio_data_size);
977	if (!t->mempools)	986	if (!t->mempools)
978	return -ENOMEM;	987	return -ENOMEM;
979		988
@@ -1414,6 +1423,33 @@ static bool dm_table_all_devices_attribute(struct dm_table *t, · 434 unmodified lines
1414	return 1;	1423	return 1;
1415	}	1424	}
1416		1425
		1426	static int device_not_write_same_capable(struct dm_target ti, struct dm_dev dev,
		1427	sector_t start, sector_t len, void *data)
		1428	{
		1429	struct request_queue *q = bdev_get_queue(dev->bdev);
		1430
		1431	return q && !q->limits.max_write_same_sectors;
		1432	}
		1433
		1434	static bool dm_table_supports_write_same(struct dm_table *t)
		1435	{
		1436	struct dm_target *ti;
		1437	unsigned i = 0;
		1438
		1439	while (i < dm_table_get_num_targets(t)) {
		1440	ti = dm_table_get_target(t, i++);
		1441
		1442	if (!ti->num_write_same_requests)
		1443	return false;
		1444
		1445	if (!ti->type->iterate_devices \|\|
		1446	!ti->type->iterate_devices(ti, device_not_write_same_capable, NULL))
		1447	return false;
		1448	}
		1449
		1450	return true;
		1451	}
		1452
1417	void dm_table_set_restrictions(struct dm_table t, struct request_queue q,	1453	void dm_table_set_restrictions(struct dm_table t, struct request_queue q,
1418	struct queue_limits *limits)	1454	struct queue_limits *limits)
1419	{	1455	{
@@ -1445,6 +1481,9 @@ void dm_table_set_restrictions(struct dm_table t, struct request_queue q, · 25 unmodified lines
1445	else	1481	else
1446	queue_flag_clear_unlocked(QUEUE_FLAG_NONROT, q);	1482	queue_flag_clear_unlocked(QUEUE_FLAG_NONROT, q);
1447		1483
		1484	if (!dm_table_supports_write_same(t))
		1485	q->limits.max_write_same_sectors = 0;
		1486
1448	dm_table_set_integrity(t);	1487	dm_table_set_integrity(t);
1449		1488
1450	/*	1489	/*

drivers/md/dm-target.c+2−3

 	/* empty */
 }
-static int io_err_map(struct dm_target *tt, struct bio *bio,
+static int io_err_map(struct dm_target *tt, struct bio *bio)
-		      union map_info *map_context)
 {
 	return -EIO;
 }
 static struct target_type error_target = {
 	.name = "error",
-	.version = {1, 0, 1},
+	.version = {1, 1, 0},
 	.ctr  = io_err_ctr,
 	.dtr  = io_err_dtr,
 	.map  = io_err_map,

drivers/md/dm-thin-metadata.c+1−1

@@ -408,7 +408,7 @@ static void __setup_btree_details(struct dm_pool_metadata *pmd) · 407 unmodified lines
408		408
409	pmd->tl_info.tm = pmd->tm;	409	pmd->tl_info.tm = pmd->tm;
410	pmd->tl_info.levels = 1;	410	pmd->tl_info.levels = 1;
411	pmd->tl_info.value_type.context = &pmd->info;	411	pmd->tl_info.value_type.context = &pmd->bl_info;
412	pmd->tl_info.value_type.size = sizeof(__le64);	412	pmd->tl_info.value_type.size = sizeof(__le64);
413	pmd->tl_info.value_type.inc = subtree_inc;	413	pmd->tl_info.value_type.inc = subtree_inc;
414	pmd->tl_info.value_type.dec = subtree_dec;	414	pmd->tl_info.value_type.dec = subtree_dec;

drivers/md/dm-thin.c+120−114

@@ -186,7 +186,6 @@ struct pool { · 185 unmodified lines
186		186
187	struct dm_thin_new_mapping *next_mapping;	187	struct dm_thin_new_mapping *next_mapping;
188	mempool_t *mapping_pool;	188	mempool_t *mapping_pool;
189	mempool_t *endio_hook_pool;
190		189
191	process_bio_fn process_bio;	190	process_bio_fn process_bio;
192	process_bio_fn process_discard;	191	process_bio_fn process_discard;
@@ -304,7 +303,7 @@ static void __requeue_bio_list(struct thin_c tc, struct bio_list master) · 111 unmodified lines
304	bio_list_init(master);	303	bio_list_init(master);
305		304
306	while ((bio = bio_list_pop(&bios))) {	305	while ((bio = bio_list_pop(&bios))) {
307	struct dm_thin_endio_hook *h = dm_get_mapinfo(bio)->ptr;	306	struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
308		307
309	if (h->tc == tc)	308	if (h->tc == tc)
310	bio_endio(bio, DM_ENDIO_REQUEUE);	309	bio_endio(bio, DM_ENDIO_REQUEUE);
@@ -368,6 +367,17 @@ static int bio_triggers_commit(struct thin_c tc, struct bio bio) · 57 unmodified lines
368	dm_thin_changed_this_transaction(tc->td);	367	dm_thin_changed_this_transaction(tc->td);
369	}	368	}
370		369
		370	static void inc_all_io_entry(struct pool pool, struct bio bio)
		371	{
		372	struct dm_thin_endio_hook *h;
		373
		374	if (bio->bi_rw & REQ_DISCARD)
		375	return;
		376
		377	h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
		378	h->all_io_entry = dm_deferred_entry_inc(pool->all_io_ds);
		379	}
		380
371	static void issue(struct thin_c tc, struct bio bio)	381	static void issue(struct thin_c tc, struct bio bio)
372	{	382	{
373	struct pool *pool = tc->pool;	383	struct pool *pool = tc->pool;
@@ -474,7 +484,7 @@ static void copy_complete(int read_err, unsigned long write_err, void *context) · 100 unmodified lines
474	static void overwrite_endio(struct bio *bio, int err)	484	static void overwrite_endio(struct bio *bio, int err)
475	{	485	{
476	unsigned long flags;	486	unsigned long flags;
477	struct dm_thin_endio_hook *h = dm_get_mapinfo(bio)->ptr;	487	struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
478	struct dm_thin_new_mapping *m = h->overwrite_mapping;	488	struct dm_thin_new_mapping *m = h->overwrite_mapping;
479	struct pool *pool = m->tc->pool;	489	struct pool *pool = m->tc->pool;
480		490
@@ -499,8 +509,7 @@ static void overwrite_endio(struct bio *bio, int err) · 18 unmodified lines
499	/*	509	/*
500	* This sends the bios in the cell back to the deferred_bios list.	510	* This sends the bios in the cell back to the deferred_bios list.
501	*/	511	*/
502	static void cell_defer(struct thin_c tc, struct dm_bio_prison_cell cell,	512	static void cell_defer(struct thin_c tc, struct dm_bio_prison_cell cell)
503	dm_block_t data_block)
504	{	513	{
505	struct pool *pool = tc->pool;	514	struct pool *pool = tc->pool;
506	unsigned long flags;	515	unsigned long flags;
@@ -513,17 +522,13 @@ static void cell_defer(struct thin_c tc, struct dm_bio_prison_cell cell, · 6 unmodified lines
513	}	522	}
514		523
515	/*	524	/*
516	* Same as cell_defer above, except it omits one particular detainee,	525	* Same as cell_defer except it omits the original holder of the cell.
517	* a write bio that covers the block and has already been processed.
518	*/	526	*/
519	static void cell_defer_except(struct thin_c tc, struct dm_bio_prison_cell cell)	527	static void cell_defer_no_holder(struct thin_c tc, struct dm_bio_prison_cell cell)
520	{	528	{
521	struct bio_list bios;
522	struct pool *pool = tc->pool;	529	struct pool *pool = tc->pool;
523	unsigned long flags;	530	unsigned long flags;
524		531
525	bio_list_init(&bios);
526
527	spin_lock_irqsave(&pool->lock, flags);	532	spin_lock_irqsave(&pool->lock, flags);
528	dm_cell_release_no_holder(cell, &pool->deferred_bios);	533	dm_cell_release_no_holder(cell, &pool->deferred_bios);
529	spin_unlock_irqrestore(&pool->lock, flags);	534	spin_unlock_irqrestore(&pool->lock, flags);
@@ -561,7 +566,7 @@ static void process_prepared_mapping(struct dm_thin_new_mapping *m) · 31 unmodified lines
561	*/	566	*/
562	r = dm_thin_insert_block(tc->td, m->virt_block, m->data_block);	567	r = dm_thin_insert_block(tc->td, m->virt_block, m->data_block);
563	if (r) {	568	if (r) {
564	DMERR("dm_thin_insert_block() failed");	569	DMERR_LIMIT("dm_thin_insert_block() failed");
565	dm_cell_error(m->cell);	570	dm_cell_error(m->cell);
566	goto out;	571	goto out;
567	}	572	}
@@ -573,10 +578,10 @@ static void process_prepared_mapping(struct dm_thin_new_mapping *m) · 5 unmodified lines
573	* the bios in the cell.	578	* the bios in the cell.
574	*/	579	*/
575	if (bio) {	580	if (bio) {
576	cell_defer_except(tc, m->cell);	581	cell_defer_no_holder(tc, m->cell);
577	bio_endio(bio, 0);	582	bio_endio(bio, 0);
578	} else	583	} else
579	cell_defer(tc, m->cell, m->data_block);	584	cell_defer(tc, m->cell);
580		585
581	out:	586	out:
582	list_del(&m->list);	587	list_del(&m->list);
@@ -588,8 +593,8 @@ static void process_prepared_discard_fail(struct dm_thin_new_mapping *m) · 5 unmodified lines
588	struct thin_c *tc = m->tc;	593	struct thin_c *tc = m->tc;
589		594
590	bio_io_error(m->bio);	595	bio_io_error(m->bio);
591	cell_defer_except(tc, m->cell);	596	cell_defer_no_holder(tc, m->cell);
592	cell_defer_except(tc, m->cell2);	597	cell_defer_no_holder(tc, m->cell2);
593	mempool_free(m, tc->pool->mapping_pool);	598	mempool_free(m, tc->pool->mapping_pool);
594	}	599	}
595		600
@@ -597,13 +602,15 @@ static void process_prepared_discard_passdown(struct dm_thin_new_mapping *m) · 1 unmodified line
597	{	602	{
598	struct thin_c *tc = m->tc;	603	struct thin_c *tc = m->tc;
599		604
		605	inc_all_io_entry(tc->pool, m->bio);
		606	cell_defer_no_holder(tc, m->cell);
		607	cell_defer_no_holder(tc, m->cell2);
		608
600	if (m->pass_discard)	609	if (m->pass_discard)
601	remap_and_issue(tc, m->bio, m->data_block);	610	remap_and_issue(tc, m->bio, m->data_block);
602	else	611	else
603	bio_endio(m->bio, 0);	612	bio_endio(m->bio, 0);
604		613
605	cell_defer_except(tc, m->cell);
606	cell_defer_except(tc, m->cell2);
607	mempool_free(m, tc->pool->mapping_pool);	614	mempool_free(m, tc->pool->mapping_pool);
608	}	615	}
609		616
@@ -614,7 +621,7 @@ static void process_prepared_discard(struct dm_thin_new_mapping *m) · 4 unmodified lines
614		621
615	r = dm_thin_remove_block(tc->td, m->virt_block);	622	r = dm_thin_remove_block(tc->td, m->virt_block);
616	if (r)	623	if (r)
617	DMERR("dm_thin_remove_block() failed");	624	DMERR_LIMIT("dm_thin_remove_block() failed");
618		625
619	process_prepared_discard_passdown(m);	626	process_prepared_discard_passdown(m);
620	}	627	}
@@ -706,11 +713,12 @@ static void schedule_copy(struct thin_c *tc, dm_block_t virt_block, · 85 unmodified lines
706	* bio immediately. Otherwise we use kcopyd to clone the data first.	713	* bio immediately. Otherwise we use kcopyd to clone the data first.
707	*/	714	*/
708	if (io_overwrites_block(pool, bio)) {	715	if (io_overwrites_block(pool, bio)) {
709	struct dm_thin_endio_hook *h = dm_get_mapinfo(bio)->ptr;	716	struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
710		717
711	h->overwrite_mapping = m;	718	h->overwrite_mapping = m;
712	m->bio = bio;	719	m->bio = bio;
713	save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);	720	save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);
		721	inc_all_io_entry(pool, bio);
714	remap_and_issue(tc, bio, data_dest);	722	remap_and_issue(tc, bio, data_dest);
715	} else {	723	} else {
716	struct dm_io_region from, to;	724	struct dm_io_region from, to;
@@ -727,7 +735,7 @@ static void schedule_copy(struct thin_c *tc, dm_block_t virt_block, · 10 unmodified lines
727	0, copy_complete, m);	735	0, copy_complete, m);
728	if (r < 0) {	736	if (r < 0) {
729	mempool_free(m, pool->mapping_pool);	737	mempool_free(m, pool->mapping_pool);
730	DMERR("dm_kcopyd_copy() failed");	738	DMERR_LIMIT("dm_kcopyd_copy() failed");
731	dm_cell_error(cell);	739	dm_cell_error(cell);
732	}	740	}
733	}	741	}
@@ -775,11 +783,12 @@ static void schedule_zero(struct thin_c *tc, dm_block_t virt_block, · 41 unmodified lines
775	process_prepared_mapping(m);	783	process_prepared_mapping(m);
776		784
777	else if (io_overwrites_block(pool, bio)) {	785	else if (io_overwrites_block(pool, bio)) {
778	struct dm_thin_endio_hook *h = dm_get_mapinfo(bio)->ptr;	786	struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
779		787
780	h->overwrite_mapping = m;	788	h->overwrite_mapping = m;
781	m->bio = bio;	789	m->bio = bio;
782	save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);	790	save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);
		791	inc_all_io_entry(pool, bio);
783	remap_and_issue(tc, bio, data_block);	792	remap_and_issue(tc, bio, data_block);
784	} else {	793	} else {
785	int r;	794	int r;
@@ -792,7 +801,7 @@ static void schedule_zero(struct thin_c *tc, dm_block_t virt_block, · 6 unmodified lines
792	r = dm_kcopyd_zero(pool->copier, 1, &to, 0, copy_complete, m);	801	r = dm_kcopyd_zero(pool->copier, 1, &to, 0, copy_complete, m);
793	if (r < 0) {	802	if (r < 0) {
794	mempool_free(m, pool->mapping_pool);	803	mempool_free(m, pool->mapping_pool);
795	DMERR("dm_kcopyd_zero() failed");	804	DMERR_LIMIT("dm_kcopyd_zero() failed");
796	dm_cell_error(cell);	805	dm_cell_error(cell);
797	}	806	}
798	}	807	}
@@ -804,7 +813,7 @@ static int commit(struct pool *pool) · 5 unmodified lines
804		813
805	r = dm_pool_commit_metadata(pool->pmd);	814	r = dm_pool_commit_metadata(pool->pmd);
806	if (r)	815	if (r)
807	DMERR("commit failed, error = %d", r);	816	DMERR_LIMIT("commit failed: error = %d", r);
808		817
809	return r;	818	return r;
810	}	819	}
@@ -889,7 +898,7 @@ static int alloc_data_block(struct thin_c tc, dm_block_t result) · 78 unmodified lines
889	*/	898	*/
890	static void retry_on_resume(struct bio *bio)	899	static void retry_on_resume(struct bio *bio)
891	{	900	{
892	struct dm_thin_endio_hook *h = dm_get_mapinfo(bio)->ptr;	901	struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
893	struct thin_c *tc = h->tc;	902	struct thin_c *tc = h->tc;
894	struct pool *pool = tc->pool;	903	struct pool *pool = tc->pool;
895	unsigned long flags;	904	unsigned long flags;
@@ -936,7 +945,7 @@ static void process_discard(struct thin_c tc, struct bio bio) · 40 unmodified lines
936	*/	945	*/
937	build_data_key(tc->td, lookup_result.block, &key2);	946	build_data_key(tc->td, lookup_result.block, &key2);
938	if (dm_bio_detain(tc->pool->prison, &key2, bio, &cell2)) {	947	if (dm_bio_detain(tc->pool->prison, &key2, bio, &cell2)) {
939	dm_cell_release_singleton(cell, bio);	948	cell_defer_no_holder(tc, cell);
940	break;	949	break;
941	}	950	}
942		951
@@ -962,13 +971,15 @@ static void process_discard(struct thin_c tc, struct bio bio) · 19 unmodified lines
962	wake_worker(pool);	971	wake_worker(pool);
963	}	972	}
964	} else {	973	} else {
		974	inc_all_io_entry(pool, bio);
		975	cell_defer_no_holder(tc, cell);
		976	cell_defer_no_holder(tc, cell2);
		977
965	/*	978	/*
966	* The DM core makes sure that the discard doesn't span	979	* The DM core makes sure that the discard doesn't span
967	* a block boundary. So we submit the discard of a	980	* a block boundary. So we submit the discard of a
968	* partial block appropriately.	981	* partial block appropriately.
969	*/	982	*/
970	dm_cell_release_singleton(cell, bio);
971	dm_cell_release_singleton(cell2, bio);
972	if ((!lookup_result.shared) && pool->pf.discard_passdown)	983	if ((!lookup_result.shared) && pool->pf.discard_passdown)
973	remap_and_issue(tc, bio, lookup_result.block);	984	remap_and_issue(tc, bio, lookup_result.block);
974	else	985	else
@@ -980,13 +991,14 @@ static void process_discard(struct thin_c tc, struct bio bio) · 5 unmodified lines
980	/*	991	/*
981	* It isn't provisioned, just forget it.	992	* It isn't provisioned, just forget it.
982	*/	993	*/
983	dm_cell_release_singleton(cell, bio);	994	cell_defer_no_holder(tc, cell);
984	bio_endio(bio, 0);	995	bio_endio(bio, 0);
985	break;	996	break;
986		997
987	default:	998	default:
988	DMERR("discard: find block unexpectedly returned %d", r);	999	DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d",
989	dm_cell_release_singleton(cell, bio);	1000	__func__, r);
		1001	cell_defer_no_holder(tc, cell);
990	bio_io_error(bio);	1002	bio_io_error(bio);
991	break;	1003	break;
992	}	1004	}
@@ -1012,7 +1024,8 @@ static void break_sharing(struct thin_c tc, struct bio bio, dm_block_t block, · 19 unmodified lines
1012	break;	1024	break;
1013		1025
1014	default:	1026	default:
1015	DMERR("%s: alloc_data_block() failed, error = %d", __func__, r);	1027	DMERR_LIMIT("%s: alloc_data_block() failed: error = %d",
		1028	__func__, r);
1016	dm_cell_error(cell);	1029	dm_cell_error(cell);
1017	break;	1030	break;
1018	}	1031	}
@@ -1037,11 +1050,12 @@ static void process_shared_bio(struct thin_c tc, struct bio bio, · 18 unmodified lines
1037	if (bio_data_dir(bio) == WRITE && bio->bi_size)	1050	if (bio_data_dir(bio) == WRITE && bio->bi_size)
1038	break_sharing(tc, bio, block, &key, lookup_result, cell);	1051	break_sharing(tc, bio, block, &key, lookup_result, cell);
1039	else {	1052	else {
1040	struct dm_thin_endio_hook *h = dm_get_mapinfo(bio)->ptr;	1053	struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
1041		1054
1042	h->shared_read_entry = dm_deferred_entry_inc(pool->shared_read_ds);	1055	h->shared_read_entry = dm_deferred_entry_inc(pool->shared_read_ds);
		1056	inc_all_io_entry(pool, bio);
		1057	cell_defer_no_holder(tc, cell);
1043		1058
1044	dm_cell_release_singleton(cell, bio);
1045	remap_and_issue(tc, bio, lookup_result->block);	1059	remap_and_issue(tc, bio, lookup_result->block);
1046	}	1060	}
1047	}	1061	}
@@ -1056,7 +1070,9 @@ static void provision_block(struct thin_c tc, struct bio bio, dm_block_t block · 8 unmodified lines
1056	* Remap empty bios (flushes) immediately, without provisioning.	1070	* Remap empty bios (flushes) immediately, without provisioning.
1057	*/	1071	*/
1058	if (!bio->bi_size) {	1072	if (!bio->bi_size) {
1059	dm_cell_release_singleton(cell, bio);	1073	inc_all_io_entry(tc->pool, bio);
		1074	cell_defer_no_holder(tc, cell);
		1075
1060	remap_and_issue(tc, bio, 0);	1076	remap_and_issue(tc, bio, 0);
1061	return;	1077	return;
1062	}	1078	}
@@ -1066,7 +1082,7 @@ static void provision_block(struct thin_c tc, struct bio bio, dm_block_t block · 3 unmodified lines
1066	*/	1082	*/
1067	if (bio_data_dir(bio) == READ) {	1083	if (bio_data_dir(bio) == READ) {
1068	zero_fill_bio(bio);	1084	zero_fill_bio(bio);
1069	dm_cell_release_singleton(cell, bio);	1085	cell_defer_no_holder(tc, cell);
1070	bio_endio(bio, 0);	1086	bio_endio(bio, 0);
1071	return;	1087	return;
1072	}	1088	}
@@ -1085,7 +1101,8 @@ static void provision_block(struct thin_c tc, struct bio bio, dm_block_t block · 12 unmodified lines
1085	break;	1101	break;
1086		1102
1087	default:	1103	default:
1088	DMERR("%s: alloc_data_block() failed, error = %d", __func__, r);	1104	DMERR_LIMIT("%s: alloc_data_block() failed: error = %d",
		1105	__func__, r);
1089	set_pool_mode(tc->pool, PM_READ_ONLY);	1106	set_pool_mode(tc->pool, PM_READ_ONLY);
1090	dm_cell_error(cell);	1107	dm_cell_error(cell);
1091	break;	1108	break;
@@ -1111,34 +1128,31 @@ static void process_bio(struct thin_c tc, struct bio bio) · 19 unmodified lines
1111	r = dm_thin_find_block(tc->td, block, 1, &lookup_result);	1128	r = dm_thin_find_block(tc->td, block, 1, &lookup_result);
1112	switch (r) {	1129	switch (r) {
1113	case 0:	1130	case 0:
1114	/*	1131	if (lookup_result.shared) {
1115	* We can release this cell now. This thread is the only
1116	* one that puts bios into a cell, and we know there were
1117	* no preceding bios.
1118	*/
1119	/*
1120	* TODO: this will probably have to change when discard goes
1121	* back in.
1122	*/
1123	dm_cell_release_singleton(cell, bio);
1124
1125	if (lookup_result.shared)
1126	process_shared_bio(tc, bio, block, &lookup_result);	1132	process_shared_bio(tc, bio, block, &lookup_result);
1127	else	1133	cell_defer_no_holder(tc, cell);
		1134	} else {
		1135	inc_all_io_entry(tc->pool, bio);
		1136	cell_defer_no_holder(tc, cell);
		1137
1128	remap_and_issue(tc, bio, lookup_result.block);	1138	remap_and_issue(tc, bio, lookup_result.block);
		1139	}
1129	break;	1140	break;
1130		1141
1131	case -ENODATA:	1142	case -ENODATA:
1132	if (bio_data_dir(bio) == READ && tc->origin_dev) {	1143	if (bio_data_dir(bio) == READ && tc->origin_dev) {
1133	dm_cell_release_singleton(cell, bio);	1144	inc_all_io_entry(tc->pool, bio);
		1145	cell_defer_no_holder(tc, cell);
		1146
1134	remap_to_origin_and_issue(tc, bio);	1147	remap_to_origin_and_issue(tc, bio);
1135	} else	1148	} else
1136	provision_block(tc, bio, block, cell);	1149	provision_block(tc, bio, block, cell);
1137	break;	1150	break;
1138		1151
1139	default:	1152	default:
1140	DMERR("dm_thin_find_block() failed, error = %d", r);	1153	DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d",
1141	dm_cell_release_singleton(cell, bio);	1154	__func__, r);
		1155	cell_defer_no_holder(tc, cell);
1142	bio_io_error(bio);	1156	bio_io_error(bio);
1143	break;	1157	break;
1144	}	1158	}
@@ -1156,8 +1170,10 @@ static void process_bio_read_only(struct thin_c tc, struct bio bio) · 11 unmodified lines
1156	case 0:	1170	case 0:
1157	if (lookup_result.shared && (rw == WRITE) && bio->bi_size)	1171	if (lookup_result.shared && (rw == WRITE) && bio->bi_size)
1158	bio_io_error(bio);	1172	bio_io_error(bio);
1159	else	1173	else {
		1174	inc_all_io_entry(tc->pool, bio);
1160	remap_and_issue(tc, bio, lookup_result.block);	1175	remap_and_issue(tc, bio, lookup_result.block);
		1176	}
1161	break;	1177	break;
1162		1178
1163	case -ENODATA:	1179	case -ENODATA:
@@ -1167,6 +1183,7 @@ static void process_bio_read_only(struct thin_c tc, struct bio bio) · 3 unmodified lines
1167	}	1183	}
1168		1184
1169	if (tc->origin_dev) {	1185	if (tc->origin_dev) {
		1186	inc_all_io_entry(tc->pool, bio);
1170	remap_to_origin_and_issue(tc, bio);	1187	remap_to_origin_and_issue(tc, bio);
1171	break;	1188	break;
1172	}	1189	}
@@ -1176,7 +1193,8 @@ static void process_bio_read_only(struct thin_c tc, struct bio bio) · 3 unmodified lines
1176	break;	1193	break;
1177		1194
1178	default:	1195	default:
1179	DMERR("dm_thin_find_block() failed, error = %d", r);	1196	DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d",
		1197	__func__, r);
1180	bio_io_error(bio);	1198	bio_io_error(bio);
1181	break;	1199	break;
1182	}	1200	}
@@ -1207,7 +1225,7 @@ static void process_deferred_bios(struct pool *pool) · 24 unmodified lines
1207	spin_unlock_irqrestore(&pool->lock, flags);	1225	spin_unlock_irqrestore(&pool->lock, flags);
1208		1226
1209	while ((bio = bio_list_pop(&bios))) {	1227	while ((bio = bio_list_pop(&bios))) {
1210	struct dm_thin_endio_hook *h = dm_get_mapinfo(bio)->ptr;	1228	struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
1211	struct thin_c *tc = h->tc;	1229	struct thin_c *tc = h->tc;
1212		1230
1213	/*	1231	/*
@@ -1340,32 +1358,30 @@ static void thin_defer_bio(struct thin_c tc, struct bio bio) · 126 unmodified lines
1340	wake_worker(pool);	1358	wake_worker(pool);
1341	}	1359	}
1342		1360
1343	static struct dm_thin_endio_hook thin_hook_bio(struct thin_c tc, struct bio *bio)	1361	static void thin_hook_bio(struct thin_c tc, struct bio bio)
1344	{	1362	{
1345	struct pool *pool = tc->pool;	1363	struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
1346	struct dm_thin_endio_hook *h = mempool_alloc(pool->endio_hook_pool, GFP_NOIO);
1347		1364
1348	h->tc = tc;	1365	h->tc = tc;
1349	h->shared_read_entry = NULL;	1366	h->shared_read_entry = NULL;
1350	h->all_io_entry = bio->bi_rw & REQ_DISCARD ? NULL : dm_deferred_entry_inc(pool->all_io_ds);	1367	h->all_io_entry = NULL;
1351	h->overwrite_mapping = NULL;	1368	h->overwrite_mapping = NULL;
1352
1353	return h;
1354	}	1369	}
1355		1370
1356	/*	1371	/*
1357	* Non-blocking function called from the thin target's map function.	1372	* Non-blocking function called from the thin target's map function.
1358	*/	1373	*/
1359	static int thin_bio_map(struct dm_target ti, struct bio bio,	1374	static int thin_bio_map(struct dm_target ti, struct bio bio)
1360	union map_info *map_context)
1361	{	1375	{
1362	int r;	1376	int r;
1363	struct thin_c *tc = ti->private;	1377	struct thin_c *tc = ti->private;
1364	dm_block_t block = get_bio_block(tc, bio);	1378	dm_block_t block = get_bio_block(tc, bio);
1365	struct dm_thin_device *td = tc->td;	1379	struct dm_thin_device *td = tc->td;
1366	struct dm_thin_lookup_result result;	1380	struct dm_thin_lookup_result result;
		1381	struct dm_bio_prison_cell cell1, cell2;
		1382	struct dm_cell_key key;
1367		1383
1368	map_context->ptr = thin_hook_bio(tc, bio);	1384	thin_hook_bio(tc, bio);
1369		1385
1370	if (get_pool_mode(tc->pool) == PM_FAIL) {	1386	if (get_pool_mode(tc->pool) == PM_FAIL) {
1371	bio_io_error(bio);	1387	bio_io_error(bio);
@@ -1400,12 +1416,25 @@ static int thin_bio_map(struct dm_target ti, struct bio bio, · 28 unmodified lines
1400	* shared flag will be set in their case.	1416	* shared flag will be set in their case.
1401	*/	1417	*/
1402	thin_defer_bio(tc, bio);	1418	thin_defer_bio(tc, bio);
1403	r = DM_MAPIO_SUBMITTED;	1419	return DM_MAPIO_SUBMITTED;
1404	} else {
1405	remap(tc, bio, result.block);
1406	r = DM_MAPIO_REMAPPED;
1407	}	1420	}
1408	break;	1421
		1422	build_virtual_key(tc->td, block, &key);
		1423	if (dm_bio_detain(tc->pool->prison, &key, bio, &cell1))
		1424	return DM_MAPIO_SUBMITTED;
		1425
		1426	build_data_key(tc->td, result.block, &key);
		1427	if (dm_bio_detain(tc->pool->prison, &key, bio, &cell2)) {
		1428	cell_defer_no_holder(tc, cell1);
		1429	return DM_MAPIO_SUBMITTED;
		1430	}
		1431
		1432	inc_all_io_entry(tc->pool, bio);
		1433	cell_defer_no_holder(tc, cell2);
		1434	cell_defer_no_holder(tc, cell1);
		1435
		1436	remap(tc, bio, result.block);
		1437	return DM_MAPIO_REMAPPED;
1409		1438
1410	case -ENODATA:	1439	case -ENODATA:
1411	if (get_pool_mode(tc->pool) == PM_READ_ONLY) {	1440	if (get_pool_mode(tc->pool) == PM_READ_ONLY) {
@@ -1414,8 +1443,7 @@ static int thin_bio_map(struct dm_target ti, struct bio bio, · 2 unmodified lines
1414	* of doing so. Just error it.	1443	* of doing so. Just error it.
1415	*/	1444	*/
1416	bio_io_error(bio);	1445	bio_io_error(bio);
1417	r = DM_MAPIO_SUBMITTED;	1446	return DM_MAPIO_SUBMITTED;
1418	break;
1419	}	1447	}
1420	/* fall through */	1448	/* fall through */
1421		1449
@@ -1425,8 +1453,7 @@ static int thin_bio_map(struct dm_target ti, struct bio bio, · 3 unmodified lines
1425	* provide the hint to load the metadata into cache.	1453	* provide the hint to load the metadata into cache.
1426	*/	1454	*/
1427	thin_defer_bio(tc, bio);	1455	thin_defer_bio(tc, bio);
1428	r = DM_MAPIO_SUBMITTED;	1456	return DM_MAPIO_SUBMITTED;
1429	break;
1430		1457
1431	default:	1458	default:
1432	/*	1459	/*
@@ -1435,11 +1462,8 @@ static int thin_bio_map(struct dm_target ti, struct bio bio, · 2 unmodified lines
1435	* pool is switched to fail-io mode.	1462	* pool is switched to fail-io mode.
1436	*/	1463	*/
1437	bio_io_error(bio);	1464	bio_io_error(bio);
1438	r = DM_MAPIO_SUBMITTED;	1465	return DM_MAPIO_SUBMITTED;
1439	break;
1440	}	1466	}
1441
1442	return r;
1443	}	1467	}
1444		1468
1445	static int pool_is_congested(struct dm_target_callbacks *cb, int bdi_bits)	1469	static int pool_is_congested(struct dm_target_callbacks *cb, int bdi_bits)
@@ -1566,14 +1590,12 @@ static void __pool_destroy(struct pool *pool) · 120 unmodified lines
1566	if (pool->next_mapping)	1590	if (pool->next_mapping)
1567	mempool_free(pool->next_mapping, pool->mapping_pool);	1591	mempool_free(pool->next_mapping, pool->mapping_pool);
1568	mempool_destroy(pool->mapping_pool);	1592	mempool_destroy(pool->mapping_pool);
1569	mempool_destroy(pool->endio_hook_pool);
1570	dm_deferred_set_destroy(pool->shared_read_ds);	1593	dm_deferred_set_destroy(pool->shared_read_ds);
1571	dm_deferred_set_destroy(pool->all_io_ds);	1594	dm_deferred_set_destroy(pool->all_io_ds);
1572	kfree(pool);	1595	kfree(pool);
1573	}	1596	}
1574		1597
1575	static struct kmem_cache *_new_mapping_cache;	1598	static struct kmem_cache *_new_mapping_cache;
1576	static struct kmem_cache *_endio_hook_cache;
1577		1599
1578	static struct pool pool_create(struct mapped_device pool_md,	1600	static struct pool pool_create(struct mapped_device pool_md,
1579	struct block_device *metadata_dev,	1601	struct block_device *metadata_dev,
@@ -1667,13 +1689,6 @@ static struct pool pool_create(struct mapped_device pool_md, · 87 unmodified lines
1667	goto bad_mapping_pool;	1689	goto bad_mapping_pool;
1668	}	1690	}
1669		1691
1670	pool->endio_hook_pool = mempool_create_slab_pool(ENDIO_HOOK_POOL_SIZE,
1671	_endio_hook_cache);
1672	if (!pool->endio_hook_pool) {
1673	*error = "Error creating pool's endio_hook mempool";
1674	err_p = ERR_PTR(-ENOMEM);
1675	goto bad_endio_hook_pool;
1676	}
1677	pool->ref_count = 1;	1692	pool->ref_count = 1;
1678	pool->last_commit_jiffies = jiffies;	1693	pool->last_commit_jiffies = jiffies;
1679	pool->pool_md = pool_md;	1694	pool->pool_md = pool_md;
@@ -1682,8 +1697,6 @@ static struct pool pool_create(struct mapped_device pool_md, · 2 unmodified lines
1682		1697
1683	return pool;	1698	return pool;
1684		1699
1685	bad_endio_hook_pool:
1686	mempool_destroy(pool->mapping_pool);
1687	bad_mapping_pool:	1700	bad_mapping_pool:
1688	dm_deferred_set_destroy(pool->all_io_ds);	1701	dm_deferred_set_destroy(pool->all_io_ds);
1689	bad_all_io_ds:	1702	bad_all_io_ds:
@@ -1966,8 +1979,7 @@ out_unlock: · 276 unmodified lines
1966	return r;	1979	return r;
1967	}	1980	}
1968		1981
1969	static int pool_map(struct dm_target ti, struct bio bio,	1982	static int pool_map(struct dm_target ti, struct bio bio)
1970	union map_info *map_context)
1971	{	1983	{
1972	int r;	1984	int r;
1973	struct pool_c *pt = ti->private;	1985	struct pool_c *pt = ti->private;
@@ -2358,7 +2370,9 @@ static int pool_status(struct dm_target *ti, status_type_t type, · 384 unmodified lines
2358	else	2370	else
2359	DMEMIT("rw ");	2371	DMEMIT("rw ");
2360		2372
2361	if (pool->pf.discard_enabled && pool->pf.discard_passdown)	2373	if (!pool->pf.discard_enabled)
		2374	DMEMIT("ignore_discard");
		2375	else if (pool->pf.discard_passdown)
2362	DMEMIT("discard_passdown");	2376	DMEMIT("discard_passdown");
2363	else	2377	else
2364	DMEMIT("no_discard_passdown");	2378	DMEMIT("no_discard_passdown");
@@ -2454,7 +2468,7 @@ static struct target_type pool_target = { · 89 unmodified lines
2454	.name = "thin-pool",	2468	.name = "thin-pool",
2455	.features = DM_TARGET_SINGLETON \| DM_TARGET_ALWAYS_WRITEABLE \|	2469	.features = DM_TARGET_SINGLETON \| DM_TARGET_ALWAYS_WRITEABLE \|
2456	DM_TARGET_IMMUTABLE,	2470	DM_TARGET_IMMUTABLE,
2457	.version = {1, 5, 0},	2471	.version = {1, 6, 0},
2458	.module = THIS_MODULE,	2472	.module = THIS_MODULE,
2459	.ctr = pool_ctr,	2473	.ctr = pool_ctr,
2460	.dtr = pool_dtr,	2474	.dtr = pool_dtr,
@@ -2576,6 +2590,7 @@ static int thin_ctr(struct dm_target ti, unsigned argc, char *argv) · 115 unmodified lines
2576		2590
2577	ti->num_flush_requests = 1;	2591	ti->num_flush_requests = 1;
2578	ti->flush_supported = true;	2592	ti->flush_supported = true;
		2593	ti->per_bio_data_size = sizeof(struct dm_thin_endio_hook);
2579		2594
2580	/* In case the pool supports discards, pass them on. */	2595	/* In case the pool supports discards, pass them on. */
2581	if (tc->pool->pf.discard_enabled) {	2596	if (tc->pool->pf.discard_enabled) {
@@ -2609,20 +2624,17 @@ out_unlock: · 27 unmodified lines
2609	return r;	2624	return r;
2610	}	2625	}
2611		2626
2612	static int thin_map(struct dm_target ti, struct bio bio,	2627	static int thin_map(struct dm_target ti, struct bio bio)
2613	union map_info *map_context)
2614	{	2628	{
2615	bio->bi_sector = dm_target_offset(ti, bio->bi_sector);	2629	bio->bi_sector = dm_target_offset(ti, bio->bi_sector);
2616		2630
2617	return thin_bio_map(ti, bio, map_context);	2631	return thin_bio_map(ti, bio);
2618	}	2632	}
2619		2633
2620	static int thin_endio(struct dm_target *ti,	2634	static int thin_endio(struct dm_target ti, struct bio bio, int err)
2621	struct bio *bio, int err,
2622	union map_info *map_context)
2623	{	2635	{
2624	unsigned long flags;	2636	unsigned long flags;
2625	struct dm_thin_endio_hook *h = map_context->ptr;	2637	struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
2626	struct list_head work;	2638	struct list_head work;
2627	struct dm_thin_new_mapping m, tmp;	2639	struct dm_thin_new_mapping m, tmp;
2628	struct pool *pool = h->tc->pool;	2640	struct pool *pool = h->tc->pool;
@@ -2643,14 +2655,15 @@ static int thin_endio(struct dm_target *ti, · 14 unmodified lines
2643	if (h->all_io_entry) {	2655	if (h->all_io_entry) {
2644	INIT_LIST_HEAD(&work);	2656	INIT_LIST_HEAD(&work);
2645	dm_deferred_entry_dec(h->all_io_entry, &work);	2657	dm_deferred_entry_dec(h->all_io_entry, &work);
2646	spin_lock_irqsave(&pool->lock, flags);	2658	if (!list_empty(&work)) {
2647	list_for_each_entry_safe(m, tmp, &work, list)	2659	spin_lock_irqsave(&pool->lock, flags);
2648	list_add(&m->list, &pool->prepared_discards);	2660	list_for_each_entry_safe(m, tmp, &work, list)
2649	spin_unlock_irqrestore(&pool->lock, flags);	2661	list_add(&m->list, &pool->prepared_discards);
		2662	spin_unlock_irqrestore(&pool->lock, flags);
		2663	wake_worker(pool);
		2664	}
2650	}	2665	}
2651		2666
2652	mempool_free(h, pool->endio_hook_pool);
2653
2654	return 0;	2667	return 0;
2655	}	2668	}
2656		2669
@@ -2745,7 +2758,7 @@ static void thin_io_hints(struct dm_target ti, struct queue_limits limits) · 88 unmodified lines
2745		2758
2746	static struct target_type thin_target = {	2759	static struct target_type thin_target = {
2747	.name = "thin",	2760	.name = "thin",
2748	.version = {1, 5, 0},	2761	.version = {1, 6, 0},
2749	.module = THIS_MODULE,	2762	.module = THIS_MODULE,
2750	.ctr = thin_ctr,	2763	.ctr = thin_ctr,
2751	.dtr = thin_dtr,	2764	.dtr = thin_dtr,
@@ -2779,14 +2792,8 @@ static int __init dm_thin_init(void) · 27 unmodified lines
2779	if (!_new_mapping_cache)	2792	if (!_new_mapping_cache)
2780	goto bad_new_mapping_cache;	2793	goto bad_new_mapping_cache;
2781		2794
2782	_endio_hook_cache = KMEM_CACHE(dm_thin_endio_hook, 0);
2783	if (!_endio_hook_cache)
2784	goto bad_endio_hook_cache;
2785
2786	return 0;	2795	return 0;
2787		2796
2788	bad_endio_hook_cache:
2789	kmem_cache_destroy(_new_mapping_cache);
2790	bad_new_mapping_cache:	2797	bad_new_mapping_cache:
2791	dm_unregister_target(&pool_target);	2798	dm_unregister_target(&pool_target);
2792	bad_pool_target:	2799	bad_pool_target:
@@ -2801,7 +2808,6 @@ static void dm_thin_exit(void) · 8 unmodified lines
2801	dm_unregister_target(&pool_target);	2808	dm_unregister_target(&pool_target);
2802		2809
2803	kmem_cache_destroy(_new_mapping_cache);	2810	kmem_cache_destroy(_new_mapping_cache);
2804	kmem_cache_destroy(_endio_hook_cache);
2805	}	2811	}
2806		2812
2807	module_init(dm_thin_init);	2813	module_init(dm_thin_init);

drivers/md/dm-verity.c+5−20

 	unsigned shash_descsize;/* the size of temporary space for crypto */
 	int hash_failed;	/* set to 1 if hash of any block failed */
-	mempool_t *io_mempool;	/* mempool of struct dm_verity_io */
 	mempool_t *vec_mempool;	/* mempool of bio vector */
 	struct workqueue_struct *verify_wq;
 struct dm_verity_io {
 	struct dm_verity *v;
-	struct bio *bio;
 	/* original values of bio->bi_end_io and bio->bi_private */
 	bio_end_io_t *orig_bi_end_io;
  */
 static void verity_finish_io(struct dm_verity_io *io, int error)
 {
-	struct bio *bio = io->bio;
 	struct dm_verity *v = io->v;
+	struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_bio_data_size);
 	bio->bi_end_io = io->orig_bi_end_io;
 	bio->bi_private = io->orig_bi_private;
 	if (io->io_vec != io->io_vec_inline)
 		mempool_free(io->io_vec, v->vec_mempool);
-	mempool_free(io, v->io_mempool);
 	bio_endio(bio, error);
 }
  * Bio map function. It allocates dm_verity_io structure and bio vector and
  * fills them. Then it issues prefetches and the I/O.
  */
-static int verity_map(struct dm_target *ti, struct bio *bio,
+static int verity_map(struct dm_target *ti, struct bio *bio)
-		      union map_info *map_context)
 {
 	struct dm_verity *v = ti->private;
 	struct dm_verity_io *io;
 	if (bio_data_dir(bio) == WRITE)
 		return -EIO;
-	io = mempool_alloc(v->io_mempool, GFP_NOIO);
+	io = dm_per_bio_data(bio, ti->per_bio_data_size);
 	io->v = v;
-	io->bio = bio;
 	io->orig_bi_end_io = bio->bi_end_io;
 	io->orig_bi_private = bio->bi_private;
 	io->block = bio->bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT);
 	if (v->vec_mempool)
 		mempool_destroy(v->vec_mempool);
-	if (v->io_mempool)
-		mempool_destroy(v->io_mempool);
 	if (v->bufio)
 		dm_bufio_client_destroy(v->bufio);
 		goto bad;
 	}
-	v->io_mempool = mempool_create_kmalloc_pool(DM_VERITY_MEMPOOL_SIZE,
+	ti->per_bio_data_size = roundup(sizeof(struct dm_verity_io) + v->shash_descsize + v->digest_size * 2, __alignof__(struct dm_verity_io));
-	  sizeof(struct dm_verity_io) + v->shash_descsize + v->digest_size * 2);
-	if (!v->io_mempool) {
-		ti->error = "Cannot allocate io mempool";
-		r = -ENOMEM;
-		goto bad;
-	}
 	v->vec_mempool = mempool_create_kmalloc_pool(DM_VERITY_MEMPOOL_SIZE,
 					BIO_MAX_PAGES * sizeof(struct bio_vec));
 static struct target_type verity_target = {
 	.name		= "verity",
-	.version	= {1, 0, 0},
+	.version	= {1, 1, 0},
 	.module		= THIS_MODULE,
 	.ctr		= verity_ctr,
 	.dtr		= verity_dtr,

drivers/md/dm-zero.c+2−3

 /*
  * Return zeros only on reads
  */
-static int zero_map(struct dm_target *ti, struct bio *bio,
+static int zero_map(struct dm_target *ti, struct bio *bio)
-		      union map_info *map_context)
 {
 	switch(bio_rw(bio)) {
 	case READ:
 static struct target_type zero_target = {
 	.name   = "zero",
-	.version = {1, 0, 0},
+	.version = {1, 1, 0},
 	.module = THIS_MODULE,
 	.ctr    = zero_ctr,
 	.map    = zero_map,

drivers/md/dm.c+58−26

 	spinlock_t endio_lock;
 };
-/*
- * For bio-based dm.
- * One of these is allocated per target within a bio.  Hopefully
- * this will be simplified out one day.
- */
-struct dm_target_io {
-	struct dm_io *io;
-	struct dm_target *ti;
-	union map_info info;
-	struct bio clone;
-};
 /*
  * For request-based dm.
  * One of these is allocated per request.
 		error = -EIO;
 	if (endio) {
-		r = endio(tio->ti, bio, error, &tio->info);
+		r = endio(tio->ti, bio, error);
 		if (r < 0 || r == DM_ENDIO_REQUEUE)
 			/*
 			 * error and requeue request are handled
 	 */
 	atomic_inc(&tio->io->io_count);
 	sector = clone->bi_sector;
-	r = ti->type->map(ti, clone, &tio->info);
+	r = ti->type->map(ti, clone);
 	if (r == DM_MAPIO_REMAPPED) {
 		/* the bio has been remapped so dispatch it */
 	tio->io = ci->io;
 	tio->ti = ti;
 	memset(&tio->info, 0, sizeof(tio->info));
+	tio->target_request_nr = 0;
 	return tio;
 }
 	struct dm_target_io *tio = alloc_tio(ci, ti, ci->bio->bi_max_vecs);
 	struct bio *clone = &tio->clone;
-	tio->info.target_request_nr = request_nr;
+	tio->target_request_nr = request_nr;
 	/*
 	 * Discard requests require the bio's inline iovecs be initialized.
 	ci->sector_count = 0;
 }
-static int __clone_and_map_discard(struct clone_info *ci)
+typedef unsigned (*get_num_requests_fn)(struct dm_target *ti);
+static unsigned get_num_discard_requests(struct dm_target *ti)
+{
+	return ti->num_discard_requests;
+}
+static unsigned get_num_write_same_requests(struct dm_target *ti)
+{
+	return ti->num_write_same_requests;
+}
+typedef bool (*is_split_required_fn)(struct dm_target *ti);
+static bool is_split_required_for_discard(struct dm_target *ti)
+{
+	return ti->split_discard_requests;
+}
+static int __clone_and_map_changing_extent_only(struct clone_info *ci,
+						get_num_requests_fn get_num_requests,
+						is_split_required_fn is_split_required)
 {
 	struct dm_target *ti;
 	sector_t len;
 			return -EIO;
 		/*
-		 * Even though the device advertised discard support,
+		 * Even though the device advertised support for this type of
-		 * that does not mean every target supports it, and
+		 * request, that does not mean every target supports it, and
 		 * reconfiguration might also have changed that since the
 		 * check was performed.
 		 */
-		if (!ti->num_discard_requests)
+		if (!get_num_requests || !get_num_requests(ti))
 			return -EOPNOTSUPP;
-		if (!ti->split_discard_requests)
+		if (is_split_required && !is_split_required(ti))
 			len = min(ci->sector_count, max_io_len_target_boundary(ci->sector, ti));
 		else
 			len = min(ci->sector_count, max_io_len(ci->sector, ti));
 	return 0;
 }
+static int __clone_and_map_discard(struct clone_info *ci)
+{
+	return __clone_and_map_changing_extent_only(ci, get_num_discard_requests,
+						    is_split_required_for_discard);
+}
+static int __clone_and_map_write_same(struct clone_info *ci)
+{
+	return __clone_and_map_changing_extent_only(ci, get_num_write_same_requests, NULL);
+}
 static int __clone_and_map(struct clone_info *ci)
 {
 	struct bio *bio = ci->bio;
 	if (unlikely(bio->bi_rw & REQ_DISCARD))
 		return __clone_and_map_discard(ci);
+	else if (unlikely(bio->bi_rw & REQ_WRITE_SAME))
+		return __clone_and_map_write_same(ci);
 	ti = dm_table_find_target(ci->map, ci->sector);
 	if (!dm_target_is_valid(ti))
 static void __bind_mempools(struct mapped_device *md, struct dm_table *t)
 {
-	struct dm_md_mempools *p;
+	struct dm_md_mempools *p = dm_table_get_md_mempools(t);
-	if (md->io_pool && (md->tio_pool || dm_table_get_type(t) == DM_TYPE_BIO_BASED) && md->bs)
+	if (md->io_pool && (md->tio_pool || dm_table_get_type(t) == DM_TYPE_BIO_BASED) && md->bs) {
-		/* the md already has necessary mempools */
+		/*
+		 * The md already has necessary mempools. Reload just the
+		 * bioset because front_pad may have changed because
+		 * a different table was loaded.
+		 */
+		bioset_free(md->bs);
+		md->bs = p->bs;
+		p->bs = NULL;
 		goto out;
+	}
-	p = dm_table_get_md_mempools(t);
 	BUG_ON(!p || md->io_pool || md->tio_pool || md->bs);
 	md->io_pool = p->io_pool;
 }
 EXPORT_SYMBOL_GPL(dm_noflush_suspending);
-struct dm_md_mempools *dm_alloc_md_mempools(unsigned type, unsigned integrity)
+struct dm_md_mempools *dm_alloc_md_mempools(unsigned type, unsigned integrity, unsigned per_bio_data_size)
 {
 	struct dm_md_mempools *pools = kmalloc(sizeof(*pools), GFP_KERNEL);
 	unsigned int pool_size = (type == DM_TYPE_BIO_BASED) ? 16 : MIN_IOS;
 	if (!pools)
 		return NULL;
+	per_bio_data_size = roundup(per_bio_data_size, __alignof__(struct dm_target_io));
 	pools->io_pool = (type == DM_TYPE_BIO_BASED) ?
 			 mempool_create_slab_pool(MIN_IOS, _io_cache) :
 			 mempool_create_slab_pool(MIN_IOS, _rq_bio_info_cache);
 	pools->bs = (type == DM_TYPE_BIO_BASED) ?
 		bioset_create(pool_size,
-			      offsetof(struct dm_target_io, clone)) :
+			      per_bio_data_size + offsetof(struct dm_target_io, clone)) :
 		bioset_create(pool_size,
 			      offsetof(struct dm_rq_clone_bio_info, clone));
 	if (!pools->bs)

drivers/md/dm.h+1−1

@@ -159,7 +159,7 @@ void dm_kcopyd_exit(void); · 158 unmodified lines
159	/*	159	/*
160	* Mempool operations	160	* Mempool operations
161	*/	161	*/
162	struct dm_md_mempools *dm_alloc_md_mempools(unsigned type, unsigned integrity);	162	struct dm_md_mempools *dm_alloc_md_mempools(unsigned type, unsigned integrity, unsigned per_bio_data_size);
163	void dm_free_md_mempools(struct dm_md_mempools *pools);	163	void dm_free_md_mempools(struct dm_md_mempools *pools);
164		164
165	#endif	165	#endif

drivers/md/persistent-data/dm-block-manager.c+7−5

 		if (!v)
 			return 0;
 		r = v->check(v, (struct dm_block *) buf, dm_bufio_get_block_size(bm->bufio));
-		if (unlikely(r))
+		if (unlikely(r)) {
+			DMERR_LIMIT("%s validator check failed for block %llu", v->name,
+				    (unsigned long long) dm_bufio_get_block_number(buf));
 			return r;
+		}
 		aux->validator = v;
 	} else {
 		if (unlikely(aux->validator != v)) {
-			DMERR("validator mismatch (old=%s vs new=%s) for block %llu",
+			DMERR_LIMIT("validator mismatch (old=%s vs new=%s) for block %llu",
-				aux->validator->name, v ? v->name : "NULL",
+				    aux->validator->name, v ? v->name : "NULL",
-				(unsigned long long)
+				    (unsigned long long) dm_bufio_get_block_number(buf));
-					dm_bufio_get_block_number(buf));
 			return -EINVAL;
 		}
 	}

drivers/md/persistent-data/dm-btree-internal.h+8−8

@@ -36,13 +36,13 @@ struct node_header { · 35 unmodified lines
36	__le32 padding;	36	__le32 padding;
37	} __packed;	37	} __packed;
38		38
39	struct node {	39	struct btree_node {
40	struct node_header header;	40	struct node_header header;
41	__le64 keys[0];	41	__le64 keys[0];
42	} __packed;	42	} __packed;
43		43
44		44
45	void inc_children(struct dm_transaction_manager tm, struct node n,	45	void inc_children(struct dm_transaction_manager tm, struct btree_node n,
46	struct dm_btree_value_type *vt);	46	struct dm_btree_value_type *vt);
47		47
48	int new_block(struct dm_btree_info info, struct dm_block *result);	48	int new_block(struct dm_btree_info info, struct dm_block *result);
@@ -64,7 +64,7 @@ struct ro_spine { · 15 unmodified lines
64	void init_ro_spine(struct ro_spine s, struct dm_btree_info info);	64	void init_ro_spine(struct ro_spine s, struct dm_btree_info info);
65	int exit_ro_spine(struct ro_spine *s);	65	int exit_ro_spine(struct ro_spine *s);
66	int ro_step(struct ro_spine *s, dm_block_t new_child);	66	int ro_step(struct ro_spine *s, dm_block_t new_child);
67	struct node ro_node(struct ro_spine s);	67	struct btree_node ro_node(struct ro_spine s);
68		68
69	struct shadow_spine {	69	struct shadow_spine {
70	struct dm_btree_info *info;	70	struct dm_btree_info *info;
@@ -98,17 +98,17 @@ int shadow_root(struct shadow_spine *s); · 27 unmodified lines
98	/*	98	/*
99	* Some inlines.	99	* Some inlines.
100	*/	100	*/
101	static inline __le64 key_ptr(struct node n, uint32_t index)	101	static inline __le64 key_ptr(struct btree_node n, uint32_t index)
102	{	102	{
103	return n->keys + index;	103	return n->keys + index;
104	}	104	}
105		105
106	static inline void value_base(struct node n)	106	static inline void value_base(struct btree_node n)
107	{	107	{
108	return &n->keys[le32_to_cpu(n->header.max_entries)];	108	return &n->keys[le32_to_cpu(n->header.max_entries)];
109	}	109	}
110		110
111	static inline void value_ptr(struct node n, uint32_t index)	111	static inline void value_ptr(struct btree_node n, uint32_t index)
112	{	112	{
113	uint32_t value_size = le32_to_cpu(n->header.value_size);	113	uint32_t value_size = le32_to_cpu(n->header.value_size);
114	return value_base(n) + (value_size * index);	114	return value_base(n) + (value_size * index);
@@ -117,7 +117,7 @@ static inline void value_ptr(struct node n, uint32_t index) · 2 unmodified lines
117	/*	117	/*
118	* Assumes the values are suitably-aligned and converts to core format.	118	* Assumes the values are suitably-aligned and converts to core format.
119	*/	119	*/
120	static inline uint64_t value64(struct node *n, uint32_t index)	120	static inline uint64_t value64(struct btree_node *n, uint32_t index)
121	{	121	{
122	__le64 *values_le = value_base(n);	122	__le64 *values_le = value_base(n);
123		123
@@ -127,7 +127,7 @@ static inline uint64_t value64(struct node *n, uint32_t index) · 3 unmodified lines
127	/*	127	/*
128	* Searching for a key within a single node.	128	* Searching for a key within a single node.
129	*/	129	*/
130	int lower_bound(struct node *n, uint64_t key);	130	int lower_bound(struct btree_node *n, uint64_t key);
131		131
132	extern struct dm_block_validator btree_node_validator;	132	extern struct dm_block_validator btree_node_validator;
133		133

drivers/md/persistent-data/dm-btree-remove.c+25−25

@@ -53,7 +53,7 @@ · 52 unmodified lines
53	/*	53	/*
54	* Some little utilities for moving node data around.	54	* Some little utilities for moving node data around.
55	*/	55	*/
56	static void node_shift(struct node *n, int shift)	56	static void node_shift(struct btree_node *n, int shift)
57	{	57	{
58	uint32_t nr_entries = le32_to_cpu(n->header.nr_entries);	58	uint32_t nr_entries = le32_to_cpu(n->header.nr_entries);
59	uint32_t value_size = le32_to_cpu(n->header.value_size);	59	uint32_t value_size = le32_to_cpu(n->header.value_size);
@@ -79,7 +79,7 @@ static void node_shift(struct node *n, int shift) · 19 unmodified lines
79	}	79	}
80	}	80	}
81		81
82	static void node_copy(struct node left, struct node right, int shift)	82	static void node_copy(struct btree_node left, struct btree_node right, int shift)
83	{	83	{
84	uint32_t nr_left = le32_to_cpu(left->header.nr_entries);	84	uint32_t nr_left = le32_to_cpu(left->header.nr_entries);
85	uint32_t value_size = le32_to_cpu(left->header.value_size);	85	uint32_t value_size = le32_to_cpu(left->header.value_size);
@@ -108,7 +108,7 @@ static void node_copy(struct node left, struct node right, int shift) · 22 unmodified lines
108	/*	108	/*
109	* Delete a specific entry from a leaf node.	109	* Delete a specific entry from a leaf node.
110	*/	110	*/
111	static void delete_at(struct node *n, unsigned index)	111	static void delete_at(struct btree_node *n, unsigned index)
112	{	112	{
113	unsigned nr_entries = le32_to_cpu(n->header.nr_entries);	113	unsigned nr_entries = le32_to_cpu(n->header.nr_entries);
114	unsigned nr_to_copy = nr_entries - (index + 1);	114	unsigned nr_to_copy = nr_entries - (index + 1);
@@ -128,7 +128,7 @@ static void delete_at(struct node *n, unsigned index) · 13 unmodified lines
128	n->header.nr_entries = cpu_to_le32(nr_entries - 1);	128	n->header.nr_entries = cpu_to_le32(nr_entries - 1);
129	}	129	}
130		130
131	static unsigned merge_threshold(struct node *n)	131	static unsigned merge_threshold(struct btree_node *n)
132	{	132	{
133	return le32_to_cpu(n->header.max_entries) / 3;	133	return le32_to_cpu(n->header.max_entries) / 3;
134	}	134	}
@@ -136,7 +136,7 @@ static unsigned merge_threshold(struct node *n) · 1 unmodified line
136	struct child {	136	struct child {
137	unsigned index;	137	unsigned index;
138	struct dm_block *block;	138	struct dm_block *block;
139	struct node *n;	139	struct btree_node *n;
140	};	140	};
141		141
142	static struct dm_btree_value_type le64_type = {	142	static struct dm_btree_value_type le64_type = {
@@ -147,7 +147,7 @@ static struct dm_btree_value_type le64_type = { · 4 unmodified lines
147	.equal = NULL	147	.equal = NULL
148	};	148	};
149		149
150	static int init_child(struct dm_btree_info info, struct node parent,	150	static int init_child(struct dm_btree_info info, struct btree_node parent,
151	unsigned index, struct child *result)	151	unsigned index, struct child *result)
152	{	152	{
153	int r, inc;	153	int r, inc;
@@ -177,7 +177,7 @@ static int exit_child(struct dm_btree_info info, struct child c) · 23 unmodified lines
177	return dm_tm_unlock(info->tm, c->block);	177	return dm_tm_unlock(info->tm, c->block);
178	}	178	}
179		179
180	static void shift(struct node left, struct node right, int count)	180	static void shift(struct btree_node left, struct btree_node right, int count)
181	{	181	{
182	uint32_t nr_left = le32_to_cpu(left->header.nr_entries);	182	uint32_t nr_left = le32_to_cpu(left->header.nr_entries);
183	uint32_t nr_right = le32_to_cpu(right->header.nr_entries);	183	uint32_t nr_right = le32_to_cpu(right->header.nr_entries);
@@ -203,11 +203,11 @@ static void shift(struct node left, struct node right, int count) · 19 unmodified lines
203	right->header.nr_entries = cpu_to_le32(nr_right + count);	203	right->header.nr_entries = cpu_to_le32(nr_right + count);
204	}	204	}
205		205
206	static void __rebalance2(struct dm_btree_info info, struct node parent,	206	static void __rebalance2(struct dm_btree_info info, struct btree_node parent,
207	struct child l, struct child r)	207	struct child l, struct child r)
208	{	208	{
209	struct node *left = l->n;	209	struct btree_node *left = l->n;
210	struct node *right = r->n;	210	struct btree_node *right = r->n;
211	uint32_t nr_left = le32_to_cpu(left->header.nr_entries);	211	uint32_t nr_left = le32_to_cpu(left->header.nr_entries);
212	uint32_t nr_right = le32_to_cpu(right->header.nr_entries);	212	uint32_t nr_right = le32_to_cpu(right->header.nr_entries);
213	unsigned threshold = 2 * merge_threshold(left) + 1;	213	unsigned threshold = 2 * merge_threshold(left) + 1;
@@ -239,7 +239,7 @@ static int rebalance2(struct shadow_spine s, struct dm_btree_info info, · 25 unmodified lines
239	unsigned left_index)	239	unsigned left_index)
240	{	240	{
241	int r;	241	int r;
242	struct node *parent;	242	struct btree_node *parent;
243	struct child left, right;	243	struct child left, right;
244		244
245	parent = dm_block_data(shadow_current(s));	245	parent = dm_block_data(shadow_current(s));
@@ -270,9 +270,9 @@ static int rebalance2(struct shadow_spine s, struct dm_btree_info info, · 24 unmodified lines
270	* in right, then rebalance2. This wastes some cpu, but I want something	270	* in right, then rebalance2. This wastes some cpu, but I want something
271	* simple atm.	271	* simple atm.
272	*/	272	*/
273	static void delete_center_node(struct dm_btree_info info, struct node parent,	273	static void delete_center_node(struct dm_btree_info info, struct btree_node parent,
274	struct child l, struct child c, struct child *r,	274	struct child l, struct child c, struct child *r,
275	struct node left, struct node center, struct node *right,	275	struct btree_node left, struct btree_node center, struct btree_node *right,
276	uint32_t nr_left, uint32_t nr_center, uint32_t nr_right)	276	uint32_t nr_left, uint32_t nr_center, uint32_t nr_right)
277	{	277	{
278	uint32_t max_entries = le32_to_cpu(left->header.max_entries);	278	uint32_t max_entries = le32_to_cpu(left->header.max_entries);
@@ -301,9 +301,9 @@ static void delete_center_node(struct dm_btree_info info, struct node parent, · 22 unmodified lines
301	/*	301	/*
302	* Redistributes entries among 3 sibling nodes.	302	* Redistributes entries among 3 sibling nodes.
303	*/	303	*/
304	static void redistribute3(struct dm_btree_info info, struct node parent,	304	static void redistribute3(struct dm_btree_info info, struct btree_node parent,
305	struct child l, struct child c, struct child *r,	305	struct child l, struct child c, struct child *r,
306	struct node left, struct node center, struct node *right,	306	struct btree_node left, struct btree_node center, struct btree_node *right,
307	uint32_t nr_left, uint32_t nr_center, uint32_t nr_right)	307	uint32_t nr_left, uint32_t nr_center, uint32_t nr_right)
308	{	308	{
309	int s;	309	int s;
@@ -343,12 +343,12 @@ static void redistribute3(struct dm_btree_info info, struct node parent, · 33 unmodified lines
343	*key_ptr(parent, r->index) = right->keys[0];	343	*key_ptr(parent, r->index) = right->keys[0];
344	}	344	}
345		345
346	static void __rebalance3(struct dm_btree_info info, struct node parent,	346	static void __rebalance3(struct dm_btree_info info, struct btree_node parent,
347	struct child l, struct child c, struct child *r)	347	struct child l, struct child c, struct child *r)
348	{	348	{
349	struct node *left = l->n;	349	struct btree_node *left = l->n;
350	struct node *center = c->n;	350	struct btree_node *center = c->n;
351	struct node *right = r->n;	351	struct btree_node *right = r->n;
352		352
353	uint32_t nr_left = le32_to_cpu(left->header.nr_entries);	353	uint32_t nr_left = le32_to_cpu(left->header.nr_entries);
354	uint32_t nr_center = le32_to_cpu(center->header.nr_entries);	354	uint32_t nr_center = le32_to_cpu(center->header.nr_entries);
@@ -371,7 +371,7 @@ static int rebalance3(struct shadow_spine s, struct dm_btree_info info, · 16 unmodified lines
371	unsigned left_index)	371	unsigned left_index)
372	{	372	{
373	int r;	373	int r;
374	struct node *parent = dm_block_data(shadow_current(s));	374	struct btree_node *parent = dm_block_data(shadow_current(s));
375	struct child left, center, right;	375	struct child left, center, right;
376		376
377	/*	377	/*
@@ -421,7 +421,7 @@ static int get_nr_entries(struct dm_transaction_manager *tm, · 43 unmodified lines
421	{	421	{
422	int r;	422	int r;
423	struct dm_block *block;	423	struct dm_block *block;
424	struct node *n;	424	struct btree_node *n;
425		425
426	r = dm_tm_read_lock(tm, b, &btree_node_validator, &block);	426	r = dm_tm_read_lock(tm, b, &btree_node_validator, &block);
427	if (r)	427	if (r)
@@ -438,7 +438,7 @@ static int rebalance_children(struct shadow_spine *s, · 10 unmodified lines
438	{	438	{
439	int i, r, has_left_sibling, has_right_sibling;	439	int i, r, has_left_sibling, has_right_sibling;
440	uint32_t child_entries;	440	uint32_t child_entries;
441	struct node *n;	441	struct btree_node *n;
442		442
443	n = dm_block_data(shadow_current(s));	443	n = dm_block_data(shadow_current(s));
444		444
@@ -483,7 +483,7 @@ static int rebalance_children(struct shadow_spine *s, · 38 unmodified lines
483	return r;	483	return r;
484	}	484	}
485		485
486	static int do_leaf(struct node n, uint64_t key, unsigned index)	486	static int do_leaf(struct btree_node n, uint64_t key, unsigned index)
487	{	487	{
488	int i = lower_bound(n, key);	488	int i = lower_bound(n, key);
489		489
@@ -506,7 +506,7 @@ static int remove_raw(struct shadow_spine s, struct dm_btree_info info, · 16 unmodified lines
506	uint64_t key, unsigned *index)	506	uint64_t key, unsigned *index)
507	{	507	{
508	int i = *index, r;	508	int i = *index, r;
509	struct node *n;	509	struct btree_node *n;
510		510
511	for (;;) {	511	for (;;) {
512	r = shadow_step(s, root, vt);	512	r = shadow_step(s, root, vt);
@@ -556,7 +556,7 @@ int dm_btree_remove(struct dm_btree_info *info, dm_block_t root, · 43 unmodified lines
556	unsigned level, last_level = info->levels - 1;	556	unsigned level, last_level = info->levels - 1;
557	int index = 0, r = 0;	557	int index = 0, r = 0;
558	struct shadow_spine spine;	558	struct shadow_spine spine;
559	struct node *n;	559	struct btree_node *n;
560		560
561	init_shadow_spine(&spine, info);	561	init_shadow_spine(&spine, info);
562	for (level = 0; level < info->levels; level++) {	562	for (level = 0; level < info->levels; level++) {

drivers/md/persistent-data/dm-btree-spine.c+10−10

@@ -23,7 +23,7 @@ static void node_prepare_for_write(struct dm_block_validator *v, · 22 unmodified lines
23	struct dm_block *b,	23	struct dm_block *b,
24	size_t block_size)	24	size_t block_size)
25	{	25	{
26	struct node *n = dm_block_data(b);	26	struct btree_node *n = dm_block_data(b);
27	struct node_header *h = &n->header;	27	struct node_header *h = &n->header;
28		28
29	h->blocknr = cpu_to_le64(dm_block_location(b));	29	h->blocknr = cpu_to_le64(dm_block_location(b));
@@ -38,15 +38,15 @@ static int node_check(struct dm_block_validator *v, · 8 unmodified lines
38	struct dm_block *b,	38	struct dm_block *b,
39	size_t block_size)	39	size_t block_size)
40	{	40	{
41	struct node *n = dm_block_data(b);	41	struct btree_node *n = dm_block_data(b);
42	struct node_header *h = &n->header;	42	struct node_header *h = &n->header;
43	size_t value_size;	43	size_t value_size;
44	__le32 csum_disk;	44	__le32 csum_disk;
45	uint32_t flags;	45	uint32_t flags;
46		46
47	if (dm_block_location(b) != le64_to_cpu(h->blocknr)) {	47	if (dm_block_location(b) != le64_to_cpu(h->blocknr)) {
48	DMERR("node_check failed blocknr %llu wanted %llu",	48	DMERR_LIMIT("node_check failed: blocknr %llu != wanted %llu",
49	le64_to_cpu(h->blocknr), dm_block_location(b));	49	le64_to_cpu(h->blocknr), dm_block_location(b));
50	return -ENOTBLK;	50	return -ENOTBLK;
51	}	51	}
52		52
@@ -54,8 +54,8 @@ static int node_check(struct dm_block_validator *v, · 1 unmodified line
54	block_size - sizeof(__le32),	54	block_size - sizeof(__le32),
55	BTREE_CSUM_XOR));	55	BTREE_CSUM_XOR));
56	if (csum_disk != h->csum) {	56	if (csum_disk != h->csum) {
57	DMERR("node_check failed csum %u wanted %u",	57	DMERR_LIMIT("node_check failed: csum %u != wanted %u",
58	le32_to_cpu(csum_disk), le32_to_cpu(h->csum));	58	le32_to_cpu(csum_disk), le32_to_cpu(h->csum));
59	return -EILSEQ;	59	return -EILSEQ;
60	}	60	}
61		61
@@ -63,12 +63,12 @@ static int node_check(struct dm_block_validator *v, · 1 unmodified line
63		63
64	if (sizeof(struct node_header) +	64	if (sizeof(struct node_header) +
65	(sizeof(__le64) + value_size) * le32_to_cpu(h->max_entries) > block_size) {	65	(sizeof(__le64) + value_size) * le32_to_cpu(h->max_entries) > block_size) {
66	DMERR("node_check failed: max_entries too large");	66	DMERR_LIMIT("node_check failed: max_entries too large");
67	return -EILSEQ;	67	return -EILSEQ;
68	}	68	}
69		69
70	if (le32_to_cpu(h->nr_entries) > le32_to_cpu(h->max_entries)) {	70	if (le32_to_cpu(h->nr_entries) > le32_to_cpu(h->max_entries)) {
71	DMERR("node_check failed, too many entries");	71	DMERR_LIMIT("node_check failed: too many entries");
72	return -EILSEQ;	72	return -EILSEQ;
73	}	73	}
74		74
@@ -77,7 +77,7 @@ static int node_check(struct dm_block_validator *v, · 2 unmodified lines
77	*/	77	*/
78	flags = le32_to_cpu(h->flags);	78	flags = le32_to_cpu(h->flags);
79	if (!(flags & INTERNAL_NODE) && !(flags & LEAF_NODE)) {	79	if (!(flags & INTERNAL_NODE) && !(flags & LEAF_NODE)) {
80	DMERR("node_check failed, node is neither INTERNAL or LEAF");	80	DMERR_LIMIT("node_check failed: node is neither INTERNAL or LEAF");
81	return -EILSEQ;	81	return -EILSEQ;
82	}	82	}
83		83
@@ -164,7 +164,7 @@ int ro_step(struct ro_spine *s, dm_block_t new_child) · 80 unmodified lines
164	return r;	164	return r;
165	}	165	}
166		166
167	struct node ro_node(struct ro_spine s)	167	struct btree_node ro_node(struct ro_spine s)
168	{	168	{
169	struct dm_block *block;	169	struct dm_block *block;
170		170

drivers/md/persistent-data/dm-btree.c+18−13

 /*----------------------------------------------------------------*/
 /* makes the assumption that no two keys are the same. */
-static int bsearch(struct node *n, uint64_t key, int want_hi)
+static int bsearch(struct btree_node *n, uint64_t key, int want_hi)
 {
 	int lo = -1, hi = le32_to_cpu(n->header.nr_entries);
 	return want_hi ? hi : lo;
 }
-int lower_bound(struct node *n, uint64_t key)
+int lower_bound(struct btree_node *n, uint64_t key)
 {
 	return bsearch(n, key, 0);
 }
-void inc_children(struct dm_transaction_manager *tm, struct node *n,
+void inc_children(struct dm_transaction_manager *tm, struct btree_node *n,
 		  struct dm_btree_value_type *vt)
 {
 	unsigned i;
 			vt->inc(vt->context, value_ptr(n, i));
 }
-static int insert_at(size_t value_size, struct node *node, unsigned index,
+static int insert_at(size_t value_size, struct btree_node *node, unsigned index,
 		      uint64_t key, void *value)
 		      __dm_written_to_disk(value)
 {
 {
 	int r;
 	struct dm_block *b;
-	struct node *n;
+	struct btree_node *n;
 	size_t block_size;
 	uint32_t max_entries;
 #define MAX_SPINE_DEPTH 64
 struct frame {
 	struct dm_block *b;
-	struct node *n;
+	struct btree_node *n;
 	unsigned level;
 	unsigned nr_children;
 	unsigned current_child;
 	dm_tm_unlock(s->tm, f->b);
 }
+static bool is_internal_level(struct dm_btree_info *info, struct frame *f)
+{
+	return f->level < (info->levels - 1);
+}
 int dm_btree_del(struct dm_btree_info *info, dm_block_t root)
 {
 	int r;
 	s->tm = info->tm;
 	s->top = -1;
-	r = push_frame(s, root, 1);
+	r = push_frame(s, root, 0);
 	if (r)
 		goto out;
 			if (r)
 				goto out;
-		} else if (f->level != (info->levels - 1)) {
+		} else if (is_internal_level(info, f)) {
 			b = value64(f->n, f->current_child);
 			f->current_child++;
 			r = push_frame(s, b, f->level + 1);
 /*----------------------------------------------------------------*/
 static int btree_lookup_raw(struct ro_spine *s, dm_block_t block, uint64_t key,
-			    int (*search_fn)(struct node *, uint64_t),
+			    int (*search_fn)(struct btree_node *, uint64_t),
 			    uint64_t *result_key, void *v, size_t value_size)
 {
 	int i, r;
 	size_t size;
 	unsigned nr_left, nr_right;
 	struct dm_block *left, *right, *parent;
-	struct node *ln, *rn, *pn;
+	struct btree_node *ln, *rn, *pn;
 	__le64 location;
 	left = shadow_current(s);
 	size_t size;
 	unsigned nr_left, nr_right;
 	struct dm_block *left, *right, *new_parent;
-	struct node *pn, *ln, *rn;
+	struct btree_node *pn, *ln, *rn;
 	__le64 val;
 	new_parent = shadow_current(s);
 			    uint64_t key, unsigned *index)
 {
 	int r, i = *index, top = 1;
-	struct node *node;
+	struct btree_node *node;
 	for (;;) {
 		r = shadow_step(s, root, vt);
 	unsigned level, index = -1, last_level = info->levels - 1;
 	dm_block_t block = root;
 	struct shadow_spine spine;
-	struct node *n;
+	struct btree_node *n;
 	struct dm_btree_value_type le64_type;
 	le64_type.context = NULL;

drivers/md/persistent-data/dm-space-map-common.c+8−8

@@ -39,8 +39,8 @@ static int index_check(struct dm_block_validator *v, · 38 unmodified lines
39	__le32 csum_disk;	39	__le32 csum_disk;
40		40
41	if (dm_block_location(b) != le64_to_cpu(mi_le->blocknr)) {	41	if (dm_block_location(b) != le64_to_cpu(mi_le->blocknr)) {
42	DMERR("index_check failed blocknr %llu wanted %llu",	42	DMERR_LIMIT("index_check failed: blocknr %llu != wanted %llu",
43	le64_to_cpu(mi_le->blocknr), dm_block_location(b));	43	le64_to_cpu(mi_le->blocknr), dm_block_location(b));
44	return -ENOTBLK;	44	return -ENOTBLK;
45	}	45	}
46		46
@@ -48,8 +48,8 @@ static int index_check(struct dm_block_validator *v, · 1 unmodified line
48	block_size - sizeof(__le32),	48	block_size - sizeof(__le32),
49	INDEX_CSUM_XOR));	49	INDEX_CSUM_XOR));
50	if (csum_disk != mi_le->csum) {	50	if (csum_disk != mi_le->csum) {
51	DMERR("index_check failed csum %u wanted %u",	51	DMERR_LIMIT("index_check failed: csum %u != wanted %u",
52	le32_to_cpu(csum_disk), le32_to_cpu(mi_le->csum));	52	le32_to_cpu(csum_disk), le32_to_cpu(mi_le->csum));
53	return -EILSEQ;	53	return -EILSEQ;
54	}	54	}
55		55
@@ -89,8 +89,8 @@ static int bitmap_check(struct dm_block_validator *v, · 33 unmodified lines
89	__le32 csum_disk;	89	__le32 csum_disk;
90		90
91	if (dm_block_location(b) != le64_to_cpu(disk_header->blocknr)) {	91	if (dm_block_location(b) != le64_to_cpu(disk_header->blocknr)) {
92	DMERR("bitmap check failed blocknr %llu wanted %llu",	92	DMERR_LIMIT("bitmap check failed: blocknr %llu != wanted %llu",
93	le64_to_cpu(disk_header->blocknr), dm_block_location(b));	93	le64_to_cpu(disk_header->blocknr), dm_block_location(b));
94	return -ENOTBLK;	94	return -ENOTBLK;
95	}	95	}
96		96
@@ -98,8 +98,8 @@ static int bitmap_check(struct dm_block_validator *v, · 1 unmodified line
98	block_size - sizeof(__le32),	98	block_size - sizeof(__le32),
99	BITMAP_CSUM_XOR));	99	BITMAP_CSUM_XOR));
100	if (csum_disk != disk_header->csum) {	100	if (csum_disk != disk_header->csum) {
101	DMERR("bitmap check failed csum %u wanted %u",	101	DMERR_LIMIT("bitmap check failed: csum %u != wanted %u",
102	le32_to_cpu(csum_disk), le32_to_cpu(disk_header->csum));	102	le32_to_cpu(csum_disk), le32_to_cpu(disk_header->csum));
103	return -EILSEQ;	103	return -EILSEQ;
104	}	104	}
105		105

drivers/md/persistent-data/dm-space-map-metadata.c+1−1

@@ -337,7 +337,7 @@ static int sm_metadata_new_block(struct dm_space_map sm, dm_block_t b) · 336 unmodified lines
337	{	337	{
338	int r = sm_metadata_new_block_(sm, b);	338	int r = sm_metadata_new_block_(sm, b);
339	if (r)	339	if (r)
340	DMERR("out of metadata space");	340	DMERR("unable to allocate new metadata block");
341	return r;	341	return r;
342	}	342	}
343		343

include/linux/device-mapper.h+47−8

 union map_info {
 	void *ptr;
 	unsigned long long ll;
-	unsigned target_request_nr;
 };
 /*
  * = 1: simple remap complete
  * = 2: The target wants to push back the io
  */
-typedef int (*dm_map_fn) (struct dm_target *ti, struct bio *bio,
+typedef int (*dm_map_fn) (struct dm_target *ti, struct bio *bio);
-			  union map_info *map_context);
 typedef int (*dm_map_request_fn) (struct dm_target *ti, struct request *clone,
 				  union map_info *map_context);
  * 2   : The target wants to push back the io
  */
 typedef int (*dm_endio_fn) (struct dm_target *ti,
-			    struct bio *bio, int error,
+			    struct bio *bio, int error);
-			    union map_info *map_context);
 typedef int (*dm_request_endio_fn) (struct dm_target *ti,
 				    struct request *clone, int error,
 				    union map_info *map_context);
 	 * A number of zero-length barrier requests that will be submitted
 	 * to the target for the purpose of flushing cache.
 	 *
-	 * The request number will be placed in union map_info->target_request_nr.
+	 * The request number can be accessed with dm_bio_get_target_request_nr.
 	 * It is a responsibility of the target driver to remap these requests
 	 * to the real underlying devices.
 	 */
 	unsigned num_flush_requests;
 	/*
-	 * The number of discard requests that will be submitted to the
+	 * The number of discard requests that will be submitted to the target.
-	 * target.  map_info->request_nr is used just like num_flush_requests.
+	 * The request number can be accessed with dm_bio_get_target_request_nr.
 	 */
 	unsigned num_discard_requests;
+	/*
+	 * The number of WRITE SAME requests that will be submitted to the target.
+	 * The request number can be accessed with dm_bio_get_target_request_nr.
+	 */
+	unsigned num_write_same_requests;
+	/*
+	 * The minimum number of extra bytes allocated in each bio for the
+	 * target to use.  dm_per_bio_data returns the data location.
+	 */
+	unsigned per_bio_data_size;
 	/* target specific data */
 	void *private;
 	int (*congested_fn) (struct dm_target_callbacks *, int);
 };
+/*
+ * For bio-based dm.
+ * One of these is allocated for each bio.
+ * This structure shouldn't be touched directly by target drivers.
+ * It is here so that we can inline dm_per_bio_data and
+ * dm_bio_from_per_bio_data
+ */
+struct dm_target_io {
+	struct dm_io *io;
+	struct dm_target *ti;
+	union map_info info;
+	unsigned target_request_nr;
+	struct bio clone;
+};
+static inline void *dm_per_bio_data(struct bio *bio, size_t data_size)
+{
+	return (char *)bio - offsetof(struct dm_target_io, clone) - data_size;
+}
+static inline struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size)
+{
+	return (struct bio *)((char *)data + data_size + offsetof(struct dm_target_io, clone));
+}
+static inline unsigned dm_bio_get_target_request_nr(const struct bio *bio)
+{
+	return container_of(bio, struct dm_target_io, clone)->target_request_nr;
+}
 int dm_register_target(struct target_type *t);
 void dm_unregister_target(struct target_type *t);

include/uapi/linux/dm-ioctl.h+2−2

@@ -268,8 +268,8 @@ enum { · 267 unmodified lines
268		268
269	#define DM_VERSION_MAJOR 4	269	#define DM_VERSION_MAJOR 4
270	#define DM_VERSION_MINOR 23	270	#define DM_VERSION_MINOR 23
271	#define DM_VERSION_PATCHLEVEL 0	271	#define DM_VERSION_PATCHLEVEL 1
272	#define DM_VERSION_EXTRA "-ioctl (2012-07-25)"	272	#define DM_VERSION_EXTRA "-ioctl (2012-12-18)"
273		273
274	/* Status bits */	274	/* Status bits */
275	#define DM_READONLY_FLAG (1 << 0) /* In/Out */	275	#define DM_READONLY_FLAG (1 << 0) /* In/Out */