History

block: split struct bio_integrity_payload

Many of the fields in struct bio_integrity_payload are only needed for the default integrity buffer in the block layer, and the variable sized array at the end of the structure makes it very hard to embed into caller allocated structures. Reduce struct bio_integrity_payload to the minimal structure needed in common code and create two separate containing structures for the automatically generated payload and the caller allocated payload. The latter is a simple wrapper for struct bio_integrity_payload and the bvecs, while the former contains the additional fields moved out of struct bio_integrity_payload. Always use a dedicated mempool for automatic integrity metadata instead of depending on bio_set that is submitter controlled and thus often doesn't have the mempool initialized and stop using mempools for the submitter buffers as they aren't in the NOIO I/O submission path where we need to guarantee forward progress.

Christoph Hellwig committed 1 year ago

Commit 105ca2a1 parent e51679111 files · +80 −188Browse files

Unified · Split

11 files
  1. block/bio-integrity-auto.c +52 −23
  2. block/bio-integrity.c +22 −85
  3. block/bio.c +0 −6
  4. block/blk.h +1 −1
  5. block/t10-pi.c +3 −3
  6. drivers/md/dm-integrity.c +0 −12
  7. drivers/md/dm-table.c +0 −6
  8. drivers/md/md.c +0 −13
  9. drivers/target/target_core_iblock.c +0 −12
  10. include/linux/bio-integrity.h +2 −23
  11. include/linux/bio.h +0 −4
block/bio-integrity-auto.c+52−23
@@ -12,18 +12,34 @@ · 11 unmodified lines
1212 #include <linux/workqueue.h>
1313 #include "blk.h"
1414
15+struct bio_integrity_data {
16+ struct bio *bio;
17+ struct bvec_iter saved_bio_iter;
18+ struct work_struct work;
19+ struct bio_integrity_payload bip;
20+ struct bio_vec bvec;
21+};
22+
23+static struct kmem_cache *bid_slab;
24+static mempool_t bid_pool;
1525 static struct workqueue_struct *kintegrityd_wq;
1626
17−static void bio_integrity_verify_fn(struct work_struct *work)
27+static void bio_integrity_finish(struct bio_integrity_data *bid)
1828 {
19− struct bio_integrity_payload *bip =
20− container_of(work, struct bio_integrity_payload, bip_work);
21− struct bio *bio = bip->bip_bio;
29+ bid->bio->bi_integrity = NULL;
30+ bid->bio->bi_opf &= ~REQ_INTEGRITY;
31+ kfree(bvec_virt(bid->bip.bip_vec));
32+ mempool_free(bid, &bid_pool);
33+}
2234
23− blk_integrity_verify(bio);
35+static void bio_integrity_verify_fn(struct work_struct *work)
36+{
37+ struct bio_integrity_data *bid =
38+ container_of(work, struct bio_integrity_data, work);
39+ struct bio *bio = bid->bio;
2440
25− kfree(bvec_virt(bip->bip_vec));
26− bio_integrity_free(bio);
41+ blk_integrity_verify_iter(bio, &bid->saved_bio_iter);
42+ bio_integrity_finish(bid);
2743 bio_endio(bio);
2844 }
2945
@@ -40,15 +56,16 @@ bool __bio_integrity_endio(struct bio *bio) · 10 unmodified lines
4056 {
4157 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
4258 struct bio_integrity_payload *bip = bio_integrity(bio);
59+ struct bio_integrity_data *bid =
60+ container_of(bip, struct bio_integrity_data, bip);
4361
4462 if (bio_op(bio) == REQ_OP_READ && !bio->bi_status && bi->csum_type) {
45− INIT_WORK(&bip->bip_work, bio_integrity_verify_fn);
46− queue_work(kintegrityd_wq, &bip->bip_work);
63+ INIT_WORK(&bid->work, bio_integrity_verify_fn);
64+ queue_work(kintegrityd_wq, &bid->work);
4765 return false;
4866 }
4967
50− kfree(bvec_virt(bip->bip_vec));
51− bio_integrity_free(bio);
68+ bio_integrity_finish(bid);
5269 return true;
5370 }
5471
@@ -65,8 +82,8 @@ bool __bio_integrity_endio(struct bio *bio) · 10 unmodified lines
6582 */
6683 bool bio_integrity_prep(struct bio *bio)
6784 {
68− struct bio_integrity_payload *bip;
6985 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
86+ struct bio_integrity_data *bid;
7087 gfp_t gfp = GFP_NOIO;
7188 unsigned int len;
7289 void *buf;
@@ -102,27 +119,30 @@ bool bio_integrity_prep(struct bio *bio) · 29 unmodified lines
102119 return true;
103120 }
104121
122+ if (WARN_ON_ONCE(bio_has_crypt_ctx(bio)))
123+ return true;
124+
105125 /* Allocate kernel buffer for protection data */
106126 len = bio_integrity_bytes(bi, bio_sectors(bio));
107127 buf = kmalloc(len, gfp);
108128 if (!buf)
109129 goto err_end_io;
130+ bid = mempool_alloc(&bid_pool, GFP_NOIO);
131+ if (!bid)
132+ goto err_free_buf;
133+ bio_integrity_init(bio, &bid->bip, &bid->bvec, 1);
110134
111− bip = bio_integrity_alloc(bio, GFP_NOIO, 1);
112− if (IS_ERR(bip)) {
113− kfree(buf);
114− goto err_end_io;
115− }
135+ bid->bio = bio;
116136
117− bip->bip_flags |= BIP_BLOCK_INTEGRITY;
118− bip_set_seed(bip, bio->bi_iter.bi_sector);
137+ bid->bip.bip_flags |= BIP_BLOCK_INTEGRITY;
138+ bip_set_seed(&bid->bip, bio->bi_iter.bi_sector);
119139
120140 if (bi->csum_type == BLK_INTEGRITY_CSUM_IP)
121− bip->bip_flags |= BIP_IP_CHECKSUM;
141+ bid->bip.bip_flags |= BIP_IP_CHECKSUM;
122142 if (bi->csum_type)
123− bip->bip_flags |= BIP_CHECK_GUARD;
143+ bid->bip.bip_flags |= BIP_CHECK_GUARD;
124144 if (bi->flags & BLK_INTEGRITY_REF_TAG)
125− bip->bip_flags |= BIP_CHECK_REFTAG;
145+ bid->bip.bip_flags |= BIP_CHECK_REFTAG;
126146
127147 if (bio_integrity_add_page(bio, virt_to_page(buf), len,
128148 offset_in_page(buf)) < len)
@@ -132,9 +152,11 @@ bool bio_integrity_prep(struct bio *bio) · 3 unmodified lines
132152 if (bio_data_dir(bio) == WRITE)
133153 blk_integrity_generate(bio);
134154 else
135− bip->bio_iter = bio->bi_iter;
155+ bid->saved_bio_iter = bio->bi_iter;
136156 return true;
137157
158+err_free_buf:
159+ kfree(buf);
138160 err_end_io:
139161 bio->bi_status = BLK_STS_RESOURCE;
140162 bio_endio(bio);
@@ -149,6 +171,13 @@ void blk_flush_integrity(void) · 8 unmodified lines
149171
150172 static int __init blk_integrity_auto_init(void)
151173 {
174+ bid_slab = kmem_cache_create("bio_integrity_data",
175+ sizeof(struct bio_integrity_data), 0,
176+ SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
177+
178+ if (mempool_init_slab_pool(&bid_pool, BIO_POOL_SIZE, bid_slab))
179+ panic("bio: can't create integrity pool\n");
180+
152181 /*
153182 * kintegrityd won't block much but may burn a lot of CPU cycles.
154183 * Make it highpri CPU intensive wq with max concurrency of 1.
block/bio-integrity.c+22−85
@@ -7,13 +7,12 @@ · 6 unmodified lines
77 */
88
99 #include <linux/blk-integrity.h>
10−#include <linux/mempool.h>
11−#include <linux/export.h>
12−#include <linux/bio.h>
13−#include <linux/slab.h>
1410 #include "blk.h"
1511
16−static struct kmem_cache *bip_slab;
12+struct bio_integrity_alloc {
13+ struct bio_integrity_payload bip;
14+ struct bio_vec bvecs[];
15+};
1716
1817 /**
1918 * bio_integrity_free - Free bio integrity payload
@@ -23,21 +22,23 @@ static struct kmem_cache *bip_slab; · 3 unmodified lines
2322 */
2423 void bio_integrity_free(struct bio *bio)
2524 {
26− struct bio_integrity_payload *bip = bio_integrity(bio);
27− struct bio_set *bs = bio->bi_pool;
28−
29− if (bs && mempool_initialized(&bs->bio_integrity_pool)) {
30− if (bip->bip_vec)
31− bvec_free(&bs->bvec_integrity_pool, bip->bip_vec,
32− bip->bip_max_vcnt);
33− mempool_free(bip, &bs->bio_integrity_pool);
34− } else {
35− kfree(bip);
36− }
25+ kfree(bio_integrity(bio));
3726 bio->bi_integrity = NULL;
3827 bio->bi_opf &= ~REQ_INTEGRITY;
3928 }
4029
30+void bio_integrity_init(struct bio *bio, struct bio_integrity_payload *bip,
31+ struct bio_vec *bvecs, unsigned int nr_vecs)
32+{
33+ memset(bip, 0, sizeof(*bip));
34+ bip->bip_max_vcnt = nr_vecs;
35+ if (nr_vecs)
36+ bip->bip_vec = bvecs;
37+
38+ bio->bi_integrity = bip;
39+ bio->bi_opf |= REQ_INTEGRITY;
40+}
41+
4142 /**
4243 * bio_integrity_alloc - Allocate integrity payload and attach it to bio
4344 * @bio: bio to attach integrity metadata to
@@ -52,48 +53,16 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, · 8 unmodified lines
5253 gfp_t gfp_mask,
5354 unsigned int nr_vecs)
5455 {
55− struct bio_integrity_payload *bip;
56− struct bio_set *bs = bio->bi_pool;
57− unsigned inline_vecs;
56+ struct bio_integrity_alloc *bia;
5857
5958 if (WARN_ON_ONCE(bio_has_crypt_ctx(bio)))
6059 return ERR_PTR(-EOPNOTSUPP);
6160
62− if (!bs || !mempool_initialized(&bs->bio_integrity_pool)) {
63− bip = kmalloc(struct_size(bip, bip_inline_vecs, nr_vecs), gfp_mask);
64− inline_vecs = nr_vecs;
65− } else {
66− bip = mempool_alloc(&bs->bio_integrity_pool, gfp_mask);
67− inline_vecs = BIO_INLINE_VECS;
68− }
69−
70− if (unlikely(!bip))
61+ bia = kmalloc(struct_size(bia, bvecs, nr_vecs), gfp_mask);
62+ if (unlikely(!bia))
7163 return ERR_PTR(-ENOMEM);
72−
73− memset(bip, 0, sizeof(*bip));
74−
75− /* always report as many vecs as asked explicitly, not inline vecs */
76− bip->bip_max_vcnt = nr_vecs;
77− if (nr_vecs > inline_vecs) {
78− bip->bip_vec = bvec_alloc(&bs->bvec_integrity_pool,
79− &bip->bip_max_vcnt, gfp_mask);
80− if (!bip->bip_vec)
81− goto err;
82− } else if (nr_vecs) {
83− bip->bip_vec = bip->bip_inline_vecs;
84− }
85−
86− bip->bip_bio = bio;
87− bio->bi_integrity = bip;
88− bio->bi_opf |= REQ_INTEGRITY;
89−
90− return bip;
91−err:
92− if (bs && mempool_initialized(&bs->bio_integrity_pool))
93− mempool_free(bip, &bs->bio_integrity_pool);
94− else
95− kfree(bip);
96− return ERR_PTR(-ENOMEM);
64+ bio_integrity_init(bio, &bia->bip, bia->bvecs, nr_vecs);
65+ return &bia->bip;
9766 }
9867 EXPORT_SYMBOL(bio_integrity_alloc);
9968
@@ -467,35 +436,3 @@ int bio_integrity_clone(struct bio *bio, struct bio *bio_src, · 367 unmodified lines
467436
468437 return 0;
469438 }
470−
471−int bioset_integrity_create(struct bio_set *bs, int pool_size)
472−{
473− if (mempool_initialized(&bs->bio_integrity_pool))
474− return 0;
475−
476− if (mempool_init_slab_pool(&bs->bio_integrity_pool,
477− pool_size, bip_slab))
478− return -1;
479−
480− if (biovec_init_pool(&bs->bvec_integrity_pool, pool_size)) {
481− mempool_exit(&bs->bio_integrity_pool);
482− return -1;
483− }
484−
485− return 0;
486−}
487−EXPORT_SYMBOL(bioset_integrity_create);
488−
489−void bioset_integrity_free(struct bio_set *bs)
490−{
491− mempool_exit(&bs->bio_integrity_pool);
492− mempool_exit(&bs->bvec_integrity_pool);
493−}
494−
495−void __init bio_integrity_init(void)
496−{
497− bip_slab = kmem_cache_create("bio_integrity_payload",
498− sizeof(struct bio_integrity_payload) +
499− sizeof(struct bio_vec) * BIO_INLINE_VECS,
500− 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
501−}
block/bio.c+0−6
@@ -1657,7 +1657,6 @@ void bioset_exit(struct bio_set *bs) · 1656 unmodified lines
16571657 mempool_exit(&bs->bio_pool);
16581658 mempool_exit(&bs->bvec_pool);
16591659
1660− bioset_integrity_free(bs);
16611660 if (bs->bio_slab)
16621661 bio_put_slab(bs);
16631662 bs->bio_slab = NULL;
@@ -1737,8 +1736,6 @@ static int __init init_bio(void) · 73 unmodified lines
17371736
17381737 BUILD_BUG_ON(BIO_FLAG_LAST > 8 * sizeof_field(struct bio, bi_flags));
17391738
1740− bio_integrity_init();
1741−
17421739 for (i = 0; i < ARRAY_SIZE(bvec_slabs); i++) {
17431740 struct biovec_slab *bvs = bvec_slabs + i;
17441741
@@ -1754,9 +1751,6 @@ static int __init init_bio(void) · 9 unmodified lines
17541751 BIOSET_NEED_BVECS | BIOSET_PERCPU_CACHE))
17551752 panic("bio: can't allocate bios\n");
17561753
1757− if (bioset_integrity_create(&fs_bio_set, BIO_POOL_SIZE))
1758− panic("bio: can't create integrity pool\n");
1759−
17601754 return 0;
17611755 }
17621756 subsys_initcall(init_bio);
block/blk.h+1−1
@@ -710,7 +710,7 @@ int bdev_open(struct block_device *bdev, blk_mode_t mode, void *holder, · 709 unmodified lines
710710 int bdev_permission(dev_t dev, blk_mode_t mode, void *holder);
711711
712712 void blk_integrity_generate(struct bio *bio);
713−void blk_integrity_verify(struct bio *bio);
713+void blk_integrity_verify_iter(struct bio *bio, struct bvec_iter *saved_iter);
714714 void blk_integrity_prepare(struct request *rq);
715715 void blk_integrity_complete(struct request *rq, unsigned int nr_bytes);
716716
block/t10-pi.c+3−3
@@ -404,7 +404,7 @@ void blk_integrity_generate(struct bio *bio) · 403 unmodified lines
404404 }
405405 }
406406
407−void blk_integrity_verify(struct bio *bio)
407+void blk_integrity_verify_iter(struct bio *bio, struct bvec_iter *saved_iter)
408408 {
409409 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
410410 struct bio_integrity_payload *bip = bio_integrity(bio);
@@ -418,9 +418,9 @@ void blk_integrity_verify(struct bio *bio) · 7 unmodified lines
418418 */
419419 iter.disk_name = bio->bi_bdev->bd_disk->disk_name;
420420 iter.interval = 1 << bi->interval_exp;
421− iter.seed = bip->bio_iter.bi_sector;
421+ iter.seed = saved_iter->bi_sector;
422422 iter.prot_buf = bvec_virt(bip->bip_vec);
423− __bio_for_each_segment(bv, bio, bviter, bip->bio_iter) {
423+ __bio_for_each_segment(bv, bio, bviter, *saved_iter) {
424424 void *kaddr = bvec_kmap_local(&bv);
425425 blk_status_t ret = BLK_STS_OK;
426426
drivers/md/dm-integrity.c+0−12
@@ -4808,23 +4808,11 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv · 4807 unmodified lines
48084808 ti->error = "Cannot allocate bio set";
48094809 goto bad;
48104810 }
4811− r = bioset_integrity_create(&ic->recheck_bios, RECHECK_POOL_SIZE);
4812− if (r) {
4813− ti->error = "Cannot allocate bio integrity set";
4814− r = -ENOMEM;
4815− goto bad;
4816− }
48174811 r = bioset_init(&ic->recalc_bios, 1, 0, BIOSET_NEED_BVECS);
48184812 if (r) {
48194813 ti->error = "Cannot allocate bio set";
48204814 goto bad;
48214815 }
4822− r = bioset_integrity_create(&ic->recalc_bios, 1);
4823− if (r) {
4824− ti->error = "Cannot allocate bio integrity set";
4825− r = -ENOMEM;
4826− goto bad;
4827− }
48284816 }
48294817
48304818 ic->metadata_wq = alloc_workqueue("dm-integrity-metadata",
drivers/md/dm-table.c+0−6
@@ -1081,15 +1081,9 @@ static int dm_table_alloc_md_mempools(struct dm_table *t, struct mapped_device * · 1080 unmodified lines
10811081 __alignof__(struct dm_io)) + DM_IO_BIO_OFFSET;
10821082 if (bioset_init(&pools->io_bs, pool_size, io_front_pad, bioset_flags))
10831083 goto out_free_pools;
1084− if (mempool_needs_integrity &&
1085− bioset_integrity_create(&pools->io_bs, pool_size))
1086− goto out_free_pools;
10871084 init_bs:
10881085 if (bioset_init(&pools->bs, pool_size, front_pad, 0))
10891086 goto out_free_pools;
1090− if (mempool_needs_integrity &&
1091− bioset_integrity_create(&pools->bs, pool_size))
1092− goto out_free_pools;
10931087
10941088 t->mempools = pools;
10951089 return 0;
drivers/md/md.c+0−13
@@ -2359,19 +2359,6 @@ int md_integrity_register(struct mddev *mddev) · 2358 unmodified lines
23592359 return 0; /* shouldn't register */
23602360
23612361 pr_debug("md: data integrity enabled on %s\n", mdname(mddev));
2362− if (bioset_integrity_create(&mddev->bio_set, BIO_POOL_SIZE) ||
2363− (mddev->level != 1 && mddev->level != 10 &&
2364− bioset_integrity_create(&mddev->io_clone_set, BIO_POOL_SIZE))) {
2365− /*
2366− * No need to handle the failure of bioset_integrity_create,
2367− * because the function is called by md_run() -> pers->run(),
2368− * md_run calls bioset_exit -> bioset_integrity_free in case
2369− * of failure case.
2370− */
2371− pr_err("md: failed to create integrity pool for %s\n",
2372− mdname(mddev));
2373− return -EINVAL;
2374− }
23752362 return 0;
23762363 }
23772364 EXPORT_SYMBOL(md_integrity_register);
drivers/target/target_core_iblock.c+0−12
@@ -167,18 +167,6 @@ static int iblock_configure_device(struct se_device *dev) · 166 unmodified lines
167167 break;
168168 }
169169
170− if (dev->dev_attrib.pi_prot_type) {
171− struct bio_set *bs = &ib_dev->ibd_bio_set;
172−
173− if (bioset_integrity_create(bs, IBLOCK_BIO_POOL_SIZE) < 0) {
174− pr_err("Unable to allocate bioset for PI\n");
175− ret = -ENOMEM;
176− goto out_blkdev_put;
177− }
178− pr_debug("IBLOCK setup BIP bs->bio_integrity_pool: %p\n",
179− &bs->bio_integrity_pool);
180− }
181−
182170 dev->dev_attrib.hw_pi_prot_type = dev->dev_attrib.pi_prot_type;
183171 return 0;
184172
include/linux/bio-integrity.h+2−23
@@ -16,8 +16,6 @@ enum bip_flags { · 15 unmodified lines
1616 };
1717
1818 struct bio_integrity_payload {
19− struct bio *bip_bio; /* parent bio */
20−
2119 struct bvec_iter bip_iter;
2220
2321 unsigned short bip_vcnt; /* # of integrity bio_vecs */
@@ -25,12 +23,7 @@ struct bio_integrity_payload { · 1 unmodified line
2523 unsigned short bip_flags; /* control flags */
2624 u16 app_tag; /* application tag value */
2725
28− struct bvec_iter bio_iter; /* for rewinding parent bio */
29−
30− struct work_struct bip_work; /* I/O completion */
31−
3226 struct bio_vec *bip_vec;
33− struct bio_vec bip_inline_vecs[];/* embedded bvec array */
3427 };
3528
3629 #define BIP_CLONE_FLAGS (BIP_MAPPED_INTEGRITY | BIP_IP_CHECKSUM | \
@@ -74,6 +67,8 @@ static inline void bip_set_seed(struct bio_integrity_payload *bip, · 37 unmodified lines
7467 bip->bip_iter.bi_sector = seed;
7568 }
7669
70+void bio_integrity_init(struct bio *bio, struct bio_integrity_payload *bip,
71+ struct bio_vec *bvecs, unsigned int nr_vecs);
7772 struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, gfp_t gfp,
7873 unsigned int nr);
7974 int bio_integrity_add_page(struct bio *bio, struct page *page, unsigned int len,
@@ -85,9 +80,6 @@ bool bio_integrity_prep(struct bio *bio); · 5 unmodified lines
8580 void bio_integrity_advance(struct bio *bio, unsigned int bytes_done);
8681 void bio_integrity_trim(struct bio *bio);
8782 int bio_integrity_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp_mask);
88−int bioset_integrity_create(struct bio_set *bs, int pool_size);
89−void bioset_integrity_free(struct bio_set *bs);
90−void bio_integrity_init(void);
9183
9284 #else /* CONFIG_BLK_DEV_INTEGRITY */
9385
@@ -96,15 +88,6 @@ static inline struct bio_integrity_payload *bio_integrity(struct bio *bio) · 2 unmodified lines
9688 return NULL;
9789 }
9890
99−static inline int bioset_integrity_create(struct bio_set *bs, int pool_size)
100−{
101− return 0;
102−}
103−
104−static inline void bioset_integrity_free(struct bio_set *bs)
105−{
106−}
107−
10891 static inline int bio_integrity_map_user(struct bio *bio, struct iov_iter *iter)
10992 {
11093 return -EINVAL;
@@ -139,10 +122,6 @@ static inline void bio_integrity_trim(struct bio *bio) · 28 unmodified lines
139122 {
140123 }
141124
142−static inline void bio_integrity_init(void)
143−{
144−}
145−
146125 static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag)
147126 {
148127 return false;
include/linux/bio.h+0−4
@@ -625,10 +625,6 @@ struct bio_set { · 624 unmodified lines
625625
626626 mempool_t bio_pool;
627627 mempool_t bvec_pool;
628−#if defined(CONFIG_BLK_DEV_INTEGRITY)
629− mempool_t bio_integrity_pool;
630− mempool_t bvec_integrity_pool;
631−#endif
632628
633629 unsigned int back_pad;
634630 /*