History

fs-verity: implement readahead of Merkle tree pages

When fs-verity verifies data pages, currently it reads each Merkle tree page synchronously using read_mapping_page(). Therefore, when the Merkle tree pages aren't already cached, fs-verity causes an extra 4 KiB I/O request for every 512 KiB of data (assuming that the Merkle tree uses SHA-256 and 4 KiB blocks). This results in more I/O requests and performance loss than is strictly necessary. Therefore, implement readahead of the Merkle tree pages. For simplicity, we take advantage of the fact that the kernel already does readahead of the file's *data*, just like it does for any other file. Due to this, we don't really need a separate readahead state (struct file_ra_state) just for the Merkle tree, but rather we just need to piggy-back on the existing data readahead requests. We also only really need to bother with the first level of the Merkle tree, since the usual fan-out factor is 128, so normally over 99% of Merkle tree I/O requests are for the first level. Therefore, make fsverity_verify_bio() enable readahead of the first Merkle tree level, for up to 1/4 the number of pages in the bio, when it sees that the REQ_RAHEAD flag is set on the bio. The readahead size is then passed down to ->read_merkle_tree_page() for the filesystem to (optionally) implement if it sees that the requested page is uncached. While we're at it, also make build_merkle_tree_level() set the Merkle tree readahead size, since it's easy to do there. However, for now don't set the readahead size in fsverity_verify_page(), since currently it's only used to verify holes on ext4 and f2fs, and it would need parameters added to know how much to read ahead. This patch significantly improves fs-verity sequential read performance. Some quick benchmarks with 'cat'-ing a 250MB file after dropping caches: On an ARM64 phone (using sha256-ce): Before: 217 MB/s After: 263 MB/s (compare to sha256sum of non-verity file: 357 MB/s) In an x86_64 VM (using sha256-avx2): Before: 173 MB/s After: 215 MB/s (compare to sha256sum of non-verity file: 223 MB/s)

Eric Biggers committed 6 years ago

Commit fd390731 parent c22415d9 files · +137 −13Browse files

Unified · Split

9 files
  1. fs/ext4/verity.c +45 −2
  2. fs/f2fs/data.c +1 −1
  3. fs/f2fs/f2fs.h +3 −0
  4. fs/f2fs/verity.c +45 −2
  5. fs/verity/enable.c +7 −1
  6. fs/verity/fsverity_private.h +1 −0
  7. fs/verity/open.c +1 −0
  8. fs/verity/verify.c +28 −6
  9. include/linux/fsverity.h +6 −1
fs/ext4/verity.c+45−2
@@ -342,12 +342,55 @@ static int ext4_get_verity_descriptor(struct inode *inode, void *buf, · 341 unmodified lines
342 return desc_size;342 return desc_size;
343}343}
344 344
345/*
346 * Prefetch some pages from the file's Merkle tree.
347 *
348 * This is basically a stripped-down version of __do_page_cache_readahead()
349 * which works on pages past i_size.
350 */
351static void ext4_merkle_tree_readahead(struct address_space *mapping,
352 pgoff_t start_index, unsigned long count)
353{
354 LIST_HEAD(pages);
355 unsigned int nr_pages = 0;
356 struct page *page;
357 pgoff_t index;
358 struct blk_plug plug;
359
360 for (index = start_index; index < start_index + count; index++) {
361 page = xa_load(&mapping->i_pages, index);
362 if (!page || xa_is_value(page)) {
363 page = __page_cache_alloc(readahead_gfp_mask(mapping));
364 if (!page)
365 break;
366 page->index = index;
367 list_add(&page->lru, &pages);
368 nr_pages++;
369 }
370 }
371 blk_start_plug(&plug);
372 ext4_mpage_readpages(mapping, &pages, NULL, nr_pages, true);
373 blk_finish_plug(&plug);
374}
375
345static struct page *ext4_read_merkle_tree_page(struct inode *inode,376static struct page *ext4_read_merkle_tree_page(struct inode *inode,
346 pgoff_t index)377 pgoff_t index,
378 unsigned long num_ra_pages)
347{379{
380 struct page *page;
381
348 index += ext4_verity_metadata_pos(inode) >> PAGE_SHIFT;382 index += ext4_verity_metadata_pos(inode) >> PAGE_SHIFT;
349 383
350 return read_mapping_page(inode->i_mapping, index, NULL);384 page = find_get_page_flags(inode->i_mapping, index, FGP_ACCESSED);
385 if (!page || !PageUptodate(page)) {
386 if (page)
387 put_page(page);
388 else if (num_ra_pages > 1)
389 ext4_merkle_tree_readahead(inode->i_mapping, index,
390 num_ra_pages);
391 page = read_mapping_page(inode->i_mapping, index, NULL);
392 }
393 return page;
351}394}
352 395
353static int ext4_write_merkle_tree_block(struct inode *inode, const void *buf,396static int ext4_write_merkle_tree_block(struct inode *inode, const void *buf,
fs/f2fs/data.c+1−1
@@ -1881,7 +1881,7 @@ out: · 1880 unmodified lines
1881 * use ->readpage() or do the necessary surgery to decouple ->readpages()1881 * use ->readpage() or do the necessary surgery to decouple ->readpages()
1882 * from read-ahead.1882 * from read-ahead.
1883 */1883 */
1884static int f2fs_mpage_readpages(struct address_space *mapping,1884int f2fs_mpage_readpages(struct address_space *mapping,
1885 struct list_head *pages, struct page *page,1885 struct list_head *pages, struct page *page,
1886 unsigned nr_pages, bool is_readahead)1886 unsigned nr_pages, bool is_readahead)
1887{1887{
fs/f2fs/f2fs.h+3−0
@@ -3229,6 +3229,9 @@ int f2fs_reserve_new_block(struct dnode_of_data *dn); · 3228 unmodified lines
3229int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index);3229int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index);
3230int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from);3230int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from);
3231int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index);3231int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index);
3232int f2fs_mpage_readpages(struct address_space *mapping,
3233 struct list_head *pages, struct page *page,
3234 unsigned nr_pages, bool is_readahead);
3232struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index,3235struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index,
3233 int op_flags, bool for_write);3236 int op_flags, bool for_write);
3234struct page *f2fs_find_data_page(struct inode *inode, pgoff_t index);3237struct page *f2fs_find_data_page(struct inode *inode, pgoff_t index);
fs/f2fs/verity.c+45−2
@@ -222,12 +222,55 @@ static int f2fs_get_verity_descriptor(struct inode *inode, void *buf, · 221 unmodified lines
222 return size;222 return size;
223}223}
224 224
225/*
226 * Prefetch some pages from the file's Merkle tree.
227 *
228 * This is basically a stripped-down version of __do_page_cache_readahead()
229 * which works on pages past i_size.
230 */
231static void f2fs_merkle_tree_readahead(struct address_space *mapping,
232 pgoff_t start_index, unsigned long count)
233{
234 LIST_HEAD(pages);
235 unsigned int nr_pages = 0;
236 struct page *page;
237 pgoff_t index;
238 struct blk_plug plug;
239
240 for (index = start_index; index < start_index + count; index++) {
241 page = xa_load(&mapping->i_pages, index);
242 if (!page || xa_is_value(page)) {
243 page = __page_cache_alloc(readahead_gfp_mask(mapping));
244 if (!page)
245 break;
246 page->index = index;
247 list_add(&page->lru, &pages);
248 nr_pages++;
249 }
250 }
251 blk_start_plug(&plug);
252 f2fs_mpage_readpages(mapping, &pages, NULL, nr_pages, true);
253 blk_finish_plug(&plug);
254}
255
225static struct page *f2fs_read_merkle_tree_page(struct inode *inode,256static struct page *f2fs_read_merkle_tree_page(struct inode *inode,
226 pgoff_t index)257 pgoff_t index,
258 unsigned long num_ra_pages)
227{259{
260 struct page *page;
261
228 index += f2fs_verity_metadata_pos(inode) >> PAGE_SHIFT;262 index += f2fs_verity_metadata_pos(inode) >> PAGE_SHIFT;
229 263
230 return read_mapping_page(inode->i_mapping, index, NULL);264 page = find_get_page_flags(inode->i_mapping, index, FGP_ACCESSED);
265 if (!page || !PageUptodate(page)) {
266 if (page)
267 put_page(page);
268 else if (num_ra_pages > 1)
269 f2fs_merkle_tree_readahead(inode->i_mapping, index,
270 num_ra_pages);
271 page = read_mapping_page(inode->i_mapping, index, NULL);
272 }
273 return page;
231}274}
232 275
233static int f2fs_write_merkle_tree_block(struct inode *inode, const void *buf,276static int f2fs_write_merkle_tree_block(struct inode *inode, const void *buf,
fs/verity/enable.c+7−1
@@ -8,6 +8,7 @@ · 7 unmodified lines
8#include "fsverity_private.h"8#include "fsverity_private.h"
9 9
10#include <crypto/hash.h>10#include <crypto/hash.h>
11#include <linux/backing-dev.h>
11#include <linux/mount.h>12#include <linux/mount.h>
12#include <linux/pagemap.h>13#include <linux/pagemap.h>
13#include <linux/sched/signal.h>14#include <linux/sched/signal.h>
@@ -86,9 +87,14 @@ static int build_merkle_tree_level(struct file *filp, unsigned int level, · 72 unmodified lines
86 return err;87 return err;
87 }88 }
88 } else {89 } else {
90 unsigned long num_ra_pages =
91 min_t(unsigned long, num_blocks_to_hash - i,
92 inode->i_sb->s_bdi->io_pages);
93
89 /* Non-leaf: hashing hash block from level below */94 /* Non-leaf: hashing hash block from level below */
90 src_page = vops->read_merkle_tree_page(inode,95 src_page = vops->read_merkle_tree_page(inode,
91 params->level_start[level - 1] + i);96 params->level_start[level - 1] + i,
97 num_ra_pages);
92 if (IS_ERR(src_page)) {98 if (IS_ERR(src_page)) {
93 err = PTR_ERR(src_page);99 err = PTR_ERR(src_page);
94 fsverity_err(inode,100 fsverity_err(inode,
fs/verity/fsverity_private.h+1−0
@@ -50,6 +50,7 @@ struct merkle_tree_params { · 49 unmodified lines
50 unsigned int log_arity; /* log2(hashes_per_block) */50 unsigned int log_arity; /* log2(hashes_per_block) */
51 unsigned int num_levels; /* number of levels in Merkle tree */51 unsigned int num_levels; /* number of levels in Merkle tree */
52 u64 tree_size; /* Merkle tree size in bytes */52 u64 tree_size; /* Merkle tree size in bytes */
53 unsigned long level0_blocks; /* number of blocks in tree level 0 */
53 54
54 /*55 /*
55 * Starting block index for each tree level, ordered from leaf level (0)56 * Starting block index for each tree level, ordered from leaf level (0)
fs/verity/open.c+1−0
@@ -102,6 +102,7 @@ int fsverity_init_merkle_tree_params(struct merkle_tree_params *params, · 101 unmodified lines
102 /* temporarily using level_start[] to store blocks in level */102 /* temporarily using level_start[] to store blocks in level */
103 params->level_start[params->num_levels++] = blocks;103 params->level_start[params->num_levels++] = blocks;
104 }104 }
105 params->level0_blocks = params->level_start[0];
105 106
106 /* Compute the starting block of each level */107 /* Compute the starting block of each level */
107 offset = 0;108 offset = 0;
fs/verity/verify.c+28−6
@@ -84,7 +84,8 @@ static inline int cmp_hashes(const struct fsverity_info *vi, · 83 unmodified lines
84 * Return: true if the page is valid, else false.84 * Return: true if the page is valid, else false.
85 */85 */
86static bool verify_page(struct inode *inode, const struct fsverity_info *vi,86static bool verify_page(struct inode *inode, const struct fsverity_info *vi,
87 struct ahash_request *req, struct page *data_page)87 struct ahash_request *req, struct page *data_page,
88 unsigned long level0_ra_pages)
88{89{
89 const struct merkle_tree_params *params = &vi->tree_params;90 const struct merkle_tree_params *params = &vi->tree_params;
90 const unsigned int hsize = params->digest_size;91 const unsigned int hsize = params->digest_size;
@@ -117,8 +118,8 @@ static bool verify_page(struct inode *inode, const struct fsverity_info *vi, · 26 unmodified lines
117 pr_debug_ratelimited("Level %d: hindex=%lu, hoffset=%u\n",118 pr_debug_ratelimited("Level %d: hindex=%lu, hoffset=%u\n",
118 level, hindex, hoffset);119 level, hindex, hoffset);
119 120
120 hpage = inode->i_sb->s_vop->read_merkle_tree_page(inode,121 hpage = inode->i_sb->s_vop->read_merkle_tree_page(inode, hindex,
121 hindex);122 level == 0 ? level0_ra_pages : 0);
122 if (IS_ERR(hpage)) {123 if (IS_ERR(hpage)) {
123 err = PTR_ERR(hpage);124 err = PTR_ERR(hpage);
124 fsverity_err(inode,125 fsverity_err(inode,
@@ -195,7 +196,7 @@ bool fsverity_verify_page(struct page *page) · 70 unmodified lines
195 if (unlikely(!req))196 if (unlikely(!req))
196 return false;197 return false;
197 198
198 valid = verify_page(inode, vi, req, page);199 valid = verify_page(inode, vi, req, page, 0);
199 200
200 ahash_request_free(req);201 ahash_request_free(req);
201 202
@@ -222,21 +223,42 @@ void fsverity_verify_bio(struct bio *bio) · 20 unmodified lines
222{223{
223 struct inode *inode = bio_first_page_all(bio)->mapping->host;224 struct inode *inode = bio_first_page_all(bio)->mapping->host;
224 const struct fsverity_info *vi = inode->i_verity_info;225 const struct fsverity_info *vi = inode->i_verity_info;
226 const struct merkle_tree_params *params = &vi->tree_params;
225 struct ahash_request *req;227 struct ahash_request *req;
226 struct bio_vec *bv;228 struct bio_vec *bv;
227 struct bvec_iter_all iter_all;229 struct bvec_iter_all iter_all;
230 unsigned long max_ra_pages = 0;
228 231
229 req = ahash_request_alloc(vi->tree_params.hash_alg->tfm, GFP_NOFS);232 req = ahash_request_alloc(params->hash_alg->tfm, GFP_NOFS);
230 if (unlikely(!req)) {233 if (unlikely(!req)) {
231 bio_for_each_segment_all(bv, bio, iter_all)234 bio_for_each_segment_all(bv, bio, iter_all)
232 SetPageError(bv->bv_page);235 SetPageError(bv->bv_page);
233 return;236 return;
234 }237 }
235 238
239 if (bio->bi_opf & REQ_RAHEAD) {
240 /*
241 * If this bio is for data readahead, then we also do readahead
242 * of the first (largest) level of the Merkle tree. Namely,
243 * when a Merkle tree page is read, we also try to piggy-back on
244 * some additional pages -- up to 1/4 the number of data pages.
245 *
246 * This improves sequential read performance, as it greatly
247 * reduces the number of I/O requests made to the Merkle tree.
248 */
249 bio_for_each_segment_all(bv, bio, iter_all)
250 max_ra_pages++;
251 max_ra_pages /= 4;
252 }
253
236 bio_for_each_segment_all(bv, bio, iter_all) {254 bio_for_each_segment_all(bv, bio, iter_all) {
237 struct page *page = bv->bv_page;255 struct page *page = bv->bv_page;
256 unsigned long level0_index = page->index >> params->log_arity;
257 unsigned long level0_ra_pages =
258 min(max_ra_pages, params->level0_blocks - level0_index);
238 259
239 if (!PageError(page) && !verify_page(inode, vi, req, page))260 if (!PageError(page) &&
261 !verify_page(inode, vi, req, page, level0_ra_pages))
240 SetPageError(page);262 SetPageError(page);
241 }263 }
242 264
include/linux/fsverity.h+6−1
@@ -77,6 +77,10 @@ struct fsverity_operations { · 76 unmodified lines
77 *77 *
78 * @inode: the inode78 * @inode: the inode
79 * @index: 0-based index of the page within the Merkle tree79 * @index: 0-based index of the page within the Merkle tree
80 * @num_ra_pages: The number of Merkle tree pages that should be
81 * prefetched starting at @index if the page at @index
82 * isn't already cached. Implementations may ignore this
83 * argument; it's only a performance optimization.
80 *84 *
81 * This can be called at any time on an open verity file, as well as85 * This can be called at any time on an open verity file, as well as
82 * between ->begin_enable_verity() and ->end_enable_verity(). It may be86 * between ->begin_enable_verity() and ->end_enable_verity(). It may be
@@ -87,7 +91,8 @@ struct fsverity_operations { · 4 unmodified lines
87 * Return: the page on success, ERR_PTR() on failure91 * Return: the page on success, ERR_PTR() on failure
88 */92 */
89 struct page *(*read_merkle_tree_page)(struct inode *inode,93 struct page *(*read_merkle_tree_page)(struct inode *inode,
90 pgoff_t index);94 pgoff_t index,
95 unsigned long num_ra_pages);
91 96
92 /**97 /**
93 * Write a Merkle tree block to the given inode.98 * Write a Merkle tree block to the given inode.