fscrypt: add inline encryption support

Add support for inline encryption to fs/crypto/. With "inline encryption", the block layer handles the decryption/encryption as part of the bio, instead of the filesystem doing the crypto itself via Linux's crypto API. This model is needed in order to take advantage of the inline encryption hardware present on most modern mobile SoCs. To use inline encryption, the filesystem needs to be mounted with '-o inlinecrypt'. Blk-crypto will then be used instead of the traditional filesystem-layer crypto whenever possible to encrypt the contents of any encrypted files in that filesystem. Fscrypt still provides the key and IV to use, and the actual ciphertext on-disk is still the same; therefore it's testable using the existing fscrypt ciphertext verification tests. Note that since blk-crypto has a fallback to Linux's crypto API, and also supports all the encryption modes currently supported by fscrypt, this feature is usable and testable even without actual inline encryption hardware. Per-filesystem changes will be needed to set encryption contexts when submitting bios and to implement the 'inlinecrypt' mount option. This patch just adds the common code.

Satya Tangirala committed 6 years ago

Commit 5fee3601 parent 457e7a112 files · +673 −47Browse files

Unified · Split

12 files

Documentation/filesystems/fscrypt.rst +3 −0
fs/crypto/Kconfig +6 −0
fs/crypto/Makefile +1 −0
fs/crypto/bio.c +51 −0
fs/crypto/crypto.c +1 −1
fs/crypto/fname.c +2 −2
fs/crypto/fscrypt_private.h +105 −10
fs/crypto/inline_crypt.c +364 −0
fs/crypto/keyring.c +3 −3
fs/crypto/keysetup.c +48 −22
fs/crypto/keysetup_v1.c +7 −9
include/linux/fscrypt.h +82 −0

Documentation/filesystems/fscrypt.rst+3−0

@@ -1255,6 +1255,7 @@ f2fs encryption using `kvm-xfstests · 1254 unmodified lines
1255	1255	<https://github.com/tytso/xfstests-bld/blob/master/Documentation/kvm-quickstart.md>`_::
1256	1256
1257	1257	kvm-xfstests -c ext4,f2fs -g encrypt
	1258	+ kvm-xfstests -c ext4,f2fs -g encrypt -m inlinecrypt
1258	1259
1259	1260	UBIFS encryption can also be tested this way, but it should be done in
1260	1261	a separate command, and it takes some time for kvm-xfstests to set up
@@ -1276,6 +1277,7 @@ This tests the encrypted I/O paths more thoroughly. To do this with · 15 unmodified lines
1276	1277	kvm-xfstests, use the "encrypt" filesystem configuration::
1277	1278
1278	1279	kvm-xfstests -c ext4/encrypt,f2fs/encrypt -g auto
	1280	+ kvm-xfstests -c ext4/encrypt,f2fs/encrypt -g auto -m inlinecrypt
1279	1281
1280	1282	Because this runs many more tests than "-g encrypt" does, it takes
1281	1283	much longer to run; so also consider using `gce-xfstests
@@ -1283,3 +1285,4 @@ much longer to run; so also consider using `gce-xfstests · 1 unmodified line
1283	1285	instead of kvm-xfstests::
1284	1286
1285	1287	gce-xfstests -c ext4/encrypt,f2fs/encrypt -g auto
	1288	+ gce-xfstests -c ext4/encrypt,f2fs/encrypt -g auto -m inlinecrypt

fs/crypto/Kconfig+6−0

@@ -24,3 +24,9 @@ config FS_ENCRYPTION_ALGS · 23 unmodified lines
24	24	select CRYPTO_SHA256
25	25	select CRYPTO_SHA512
26	26	select CRYPTO_XTS
	27	+
	28	+config FS_ENCRYPTION_INLINE_CRYPT
	29	+ bool "Enable fscrypt to use inline crypto"
	30	+ depends on FS_ENCRYPTION && BLK_INLINE_ENCRYPTION
	31	+ help
	32	+ Enable fscrypt to use inline encryption hardware if available.

fs/crypto/Makefile+1−0

@@ -11,3 +11,4 @@ fscrypto-y := crypto.o \ · 10 unmodified lines
11	11	policy.o
12	12
13	13	fscrypto-$(CONFIG_BLOCK) += bio.o
	14	+fscrypto-$(CONFIG_FS_ENCRYPTION_INLINE_CRYPT) += inline_crypt.o

fs/crypto/bio.c+51−0

@@ -41,6 +41,53 @@ void fscrypt_decrypt_bio(struct bio *bio) · 40 unmodified lines
41	41	}
42	42	EXPORT_SYMBOL(fscrypt_decrypt_bio);
43	43
	44	+static int fscrypt_zeroout_range_inline_crypt(const struct inode *inode,
	45	+ pgoff_t lblk, sector_t pblk,
	46	+ unsigned int len)
	47	+{
	48	+ const unsigned int blockbits = inode->i_blkbits;
	49	+ const unsigned int blocks_per_page = 1 << (PAGE_SHIFT - blockbits);
	50	+ struct bio *bio;
	51	+ int ret, err = 0;
	52	+ int num_pages = 0;
	53	+
	54	+ /* This always succeeds since __GFP_DIRECT_RECLAIM is set. */
	55	+ bio = bio_alloc(GFP_NOFS, BIO_MAX_PAGES);
	56	+
	57	+ while (len) {
	58	+ unsigned int blocks_this_page = min(len, blocks_per_page);
	59	+ unsigned int bytes_this_page = blocks_this_page << blockbits;
	60	+
	61	+ if (num_pages == 0) {
	62	+ fscrypt_set_bio_crypt_ctx(bio, inode, lblk, GFP_NOFS);
	63	+ bio_set_dev(bio, inode->i_sb->s_bdev);
	64	+ bio->bi_iter.bi_sector =
	65	+ pblk << (blockbits - SECTOR_SHIFT);
	66	+ bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
	67	+ }
	68	+ ret = bio_add_page(bio, ZERO_PAGE(0), bytes_this_page, 0);
	69	+ if (WARN_ON(ret != bytes_this_page)) {
	70	+ err = -EIO;
	71	+ goto out;
	72	+ }
	73	+ num_pages++;
	74	+ len -= blocks_this_page;
	75	+ lblk += blocks_this_page;
	76	+ pblk += blocks_this_page;
	77	+ if (num_pages == BIO_MAX_PAGES \|\| !len \|\|
	78	+ !fscrypt_mergeable_bio(bio, inode, lblk)) {
	79	+ err = submit_bio_wait(bio);
	80	+ if (err)
	81	+ goto out;
	82	+ bio_reset(bio);
	83	+ num_pages = 0;
	84	+ }
	85	+ }
	86	+out:
	87	+ bio_put(bio);
	88	+ return err;
	89	+}
	90	+
44	91	/**
45	92	* fscrypt_zeroout_range() - zero out a range of blocks in an encrypted file
46	93	* @inode: the file's inode
@@ -75,6 +122,10 @@ int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk, · 28 unmodified lines
75	122	if (len == 0)
76	123	return 0;
77	124
	125	+ if (fscrypt_inode_uses_inline_crypto(inode))
	126	+ return fscrypt_zeroout_range_inline_crypt(inode, lblk, pblk,
	127	+ len);
	128	+
78	129	BUILD_BUG_ON(ARRAY_SIZE(pages) > BIO_MAX_PAGES);
79	130	nr_pages = min_t(unsigned int, ARRAY_SIZE(pages),
80	131	(len + blocks_per_page - 1) >> blocks_per_page_bits);

fs/crypto/crypto.c+1−1

@@ -100,7 +100,7 @@ int fscrypt_crypt_block(const struct inode *inode, fscrypt_direction_t rw, · 99 unmodified lines
100	100	DECLARE_CRYPTO_WAIT(wait);
101	101	struct scatterlist dst, src;
102	102	struct fscrypt_info *ci = inode->i_crypt_info;
103		− struct crypto_skcipher *tfm = ci->ci_ctfm;
	103	+ struct crypto_skcipher *tfm = ci->ci_enc_key.tfm;
104	104	int res = 0;
105	105
106	106	if (WARN_ON_ONCE(len <= 0))

fs/crypto/fname.c+2−2

@@ -115,7 +115,7 @@ int fscrypt_fname_encrypt(const struct inode inode, const struct qstr iname, · 114 unmodified lines
115	115	struct skcipher_request *req = NULL;
116	116	DECLARE_CRYPTO_WAIT(wait);
117	117	const struct fscrypt_info *ci = inode->i_crypt_info;
118		− struct crypto_skcipher *tfm = ci->ci_ctfm;
	118	+ struct crypto_skcipher *tfm = ci->ci_enc_key.tfm;
119	119	union fscrypt_iv iv;
120	120	struct scatterlist sg;
121	121	int res;
@@ -171,7 +171,7 @@ static int fname_decrypt(const struct inode *inode, · 49 unmodified lines
171	171	DECLARE_CRYPTO_WAIT(wait);
172	172	struct scatterlist src_sg, dst_sg;
173	173	const struct fscrypt_info *ci = inode->i_crypt_info;
174		− struct crypto_skcipher *tfm = ci->ci_ctfm;
	174	+ struct crypto_skcipher *tfm = ci->ci_enc_key.tfm;
175	175	union fscrypt_iv iv;
176	176	int res;
177	177

fs/crypto/fscrypt_private.h+105−10

@@ -14,6 +14,7 @@ · 13 unmodified lines
14	14	#include <linux/fscrypt.h>
15	15	#include <linux/siphash.h>
16	16	#include <crypto/hash.h>
	17	+#include <linux/blk-crypto.h>
17	18
18	19	#define CONST_STRLEN(str) (sizeof(str) - 1)
19	20
@@ -166,6 +167,20 @@ struct fscrypt_symlink_data { · 146 unmodified lines
166	167	char encrypted_path[1];
167	168	} __packed;
168	169
	170	+/**
	171	+ * struct fscrypt_prepared_key - a key prepared for actual encryption/decryption
	172	+ * @tfm: crypto API transform object
	173	+ * @blk_key: key for blk-crypto
	174	+ *
	175	+ * Normally only one of the fields will be non-NULL.
	176	+ */
	177	+struct fscrypt_prepared_key {
	178	+ struct crypto_skcipher *tfm;
	179	+#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT
	180	+ struct fscrypt_blk_crypto_key *blk_key;
	181	+#endif
	182	+};
	183	+
169	184	/*
170	185	* fscrypt_info - the "encryption key" for an inode
171	186	*
@@ -175,12 +190,20 @@ struct fscrypt_symlink_data { · 3 unmodified lines
175	190	*/
176	191	struct fscrypt_info {
177	192
178		− /* The actual crypto transform used for encryption and decryption */
179		− struct crypto_skcipher *ci_ctfm;
	193	+ /* The key in a form prepared for actual encryption/decryption */
	194	+ struct fscrypt_prepared_key ci_enc_key;
180	195
181		− /* True if the key should be freed when this fscrypt_info is freed */
	196	+ /* True if ci_enc_key should be freed when this fscrypt_info is freed */
182	197	bool ci_owns_key;
183	198
	199	+#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT
	200	+ /*
	201	+ * True if this inode will use inline encryption (blk-crypto) instead of
	202	+ * the traditional filesystem-layer encryption.
	203	+ */
	204	+ bool ci_inlinecrypt;
	205	+#endif
	206	+
184	207	/*
185	208	* Encryption mode used for this inode. It corresponds to either the
186	209	* contents or filenames encryption mode, depending on the inode type.
@@ -205,7 +228,7 @@ struct fscrypt_info { · 18 unmodified lines
205	228
206	229	/*
207	230	* If non-NULL, then encryption is done using the master key directly
208		− * and ci_ctfm will equal ci_direct_key->dk_ctfm.
	231	+ * and ci_enc_key will equal ci_direct_key->dk_key.
209	232	*/
210	233	struct fscrypt_direct_key *ci_direct_key;
211	234
@@ -260,6 +283,7 @@ union fscrypt_iv { · 48 unmodified lines
260	283	u8 nonce[FS_KEY_DERIVATION_NONCE_SIZE];
261	284	};
262	285	u8 raw[FSCRYPT_MAX_IV_SIZE];
	286	+ __le64 dun[FSCRYPT_MAX_IV_SIZE / sizeof(__le64)];
263	287	};
264	288
265	289	void fscrypt_generate_iv(union fscrypt_iv *iv, u64 lblk_num,
@@ -302,6 +326,75 @@ int fscrypt_hkdf_expand(const struct fscrypt_hkdf *hkdf, u8 context, · 36 unmodified lines
302	326
303	327	void fscrypt_destroy_hkdf(struct fscrypt_hkdf *hkdf);
304	328
	329	+/* inline_crypt.c */
	330	+#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT
	331	+int fscrypt_select_encryption_impl(struct fscrypt_info *ci);
	332	+
	333	+static inline bool
	334	+fscrypt_using_inline_encryption(const struct fscrypt_info *ci)
	335	+{
	336	+ return ci->ci_inlinecrypt;
	337	+}
	338	+
	339	+int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key,
	340	+ const u8 *raw_key,
	341	+ const struct fscrypt_info *ci);
	342	+
	343	+void fscrypt_destroy_inline_crypt_key(struct fscrypt_prepared_key *prep_key);
	344	+
	345	+/*
	346	+ * Check whether the crypto transform or blk-crypto key has been allocated in
	347	+ * @prep_key, depending on which encryption implementation the file will use.
	348	+ */
	349	+static inline bool
	350	+fscrypt_is_key_prepared(struct fscrypt_prepared_key *prep_key,
	351	+ const struct fscrypt_info *ci)
	352	+{
	353	+ /*
	354	+ * The READ_ONCE() here pairs with the smp_store_release() in
	355	+ * fscrypt_prepare_key(). (This only matters for the per-mode keys,
	356	+ * which are shared by multiple inodes.)
	357	+ */
	358	+ if (fscrypt_using_inline_encryption(ci))
	359	+ return READ_ONCE(prep_key->blk_key) != NULL;
	360	+ return READ_ONCE(prep_key->tfm) != NULL;
	361	+}
	362	+
	363	+#else /* CONFIG_FS_ENCRYPTION_INLINE_CRYPT */
	364	+
	365	+static inline int fscrypt_select_encryption_impl(struct fscrypt_info *ci)
	366	+{
	367	+ return 0;
	368	+}
	369	+
	370	+static inline bool
	371	+fscrypt_using_inline_encryption(const struct fscrypt_info *ci)
	372	+{
	373	+ return false;
	374	+}
	375	+
	376	+static inline int
	377	+fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key,
	378	+ const u8 *raw_key,
	379	+ const struct fscrypt_info *ci)
	380	+{
	381	+ WARN_ON(1);
	382	+ return -EOPNOTSUPP;
	383	+}
	384	+
	385	+static inline void
	386	+fscrypt_destroy_inline_crypt_key(struct fscrypt_prepared_key *prep_key)
	387	+{
	388	+}
	389	+
	390	+static inline bool
	391	+fscrypt_is_key_prepared(struct fscrypt_prepared_key *prep_key,
	392	+ const struct fscrypt_info *ci)
	393	+{
	394	+ return READ_ONCE(prep_key->tfm) != NULL;
	395	+}
	396	+#endif /* !CONFIG_FS_ENCRYPTION_INLINE_CRYPT */
	397	+
305	398	/* keyring.c */
306	399
307	400	/*
@@ -395,9 +488,9 @@ struct fscrypt_master_key { · 87 unmodified lines
395	488	* Per-mode encryption keys for the various types of encryption policies
396	489	* that use them. Allocated and derived on-demand.
397	490	*/
398		− struct crypto_skcipher *mk_direct_keys[__FSCRYPT_MODE_MAX + 1];
399		− struct crypto_skcipher *mk_iv_ino_lblk_64_keys[__FSCRYPT_MODE_MAX + 1];
400		− struct crypto_skcipher *mk_iv_ino_lblk_32_keys[__FSCRYPT_MODE_MAX + 1];
	491	+ struct fscrypt_prepared_key mk_direct_keys[__FSCRYPT_MODE_MAX + 1];
	492	+ struct fscrypt_prepared_key mk_iv_ino_lblk_64_keys[__FSCRYPT_MODE_MAX + 1];
	493	+ struct fscrypt_prepared_key mk_iv_ino_lblk_32_keys[__FSCRYPT_MODE_MAX + 1];
401	494
402	495	/* Hash key for inode numbers. Initialized only when needed. */
403	496	siphash_key_t mk_ino_hash_key;
@@ -461,13 +554,15 @@ struct fscrypt_mode { · 57 unmodified lines
461	554	int keysize;
462	555	int ivsize;
463	556	int logged_impl_name;
	557	+ enum blk_crypto_mode_num blk_crypto_mode;
464	558	};
465	559
466	560	extern struct fscrypt_mode fscrypt_modes[];
467	561
468		−struct crypto_skcipher fscrypt_allocate_skcipher(struct fscrypt_mode mode,
469		− const u8 *raw_key,
470		− const struct inode *inode);
	562	+int fscrypt_prepare_key(struct fscrypt_prepared_key *prep_key,
	563	+ const u8 raw_key, const struct fscrypt_info ci);
	564	+
	565	+void fscrypt_destroy_prepared_key(struct fscrypt_prepared_key *prep_key);
471	566
472	567	int fscrypt_set_per_file_enc_key(struct fscrypt_info ci, const u8 raw_key);
473	568

fs/crypto/inline_crypt.c+364−0

@@ -0,0 +1,364 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/*
	3	+ * Inline encryption support for fscrypt
	4	+ *
	5	+ * Copyright 2019 Google LLC
	6	+ */
	7	+
	8	+/*
	9	+ * With "inline encryption", the block layer handles the decryption/encryption
	10	+ * as part of the bio, instead of the filesystem doing the crypto itself via
	11	+ * crypto API. See Documentation/block/inline-encryption.rst. fscrypt still
	12	+ * provides the key and IV to use.
	13	+ */
	14	+
	15	+#include <linux/blk-crypto.h>
	16	+#include <linux/blkdev.h>
	17	+#include <linux/buffer_head.h>
	18	+#include <linux/sched/mm.h>
	19	+
	20	+#include "fscrypt_private.h"
	21	+
	22	+struct fscrypt_blk_crypto_key {
	23	+ struct blk_crypto_key base;
	24	+ int num_devs;
	25	+ struct request_queue *devs[];
	26	+};
	27	+
	28	+static int fscrypt_get_num_devices(struct super_block *sb)
	29	+{
	30	+ if (sb->s_cop->get_num_devices)
	31	+ return sb->s_cop->get_num_devices(sb);
	32	+ return 1;
	33	+}
	34	+
	35	+static void fscrypt_get_devices(struct super_block *sb, int num_devs,
	36	+ struct request_queue **devs)
	37	+{
	38	+ if (num_devs == 1)
	39	+ devs[0] = bdev_get_queue(sb->s_bdev);
	40	+ else
	41	+ sb->s_cop->get_devices(sb, devs);
	42	+}
	43	+
	44	+static unsigned int fscrypt_get_dun_bytes(const struct fscrypt_info *ci)
	45	+{
	46	+ struct super_block *sb = ci->ci_inode->i_sb;
	47	+ unsigned int flags = fscrypt_policy_flags(&ci->ci_policy);
	48	+ int ino_bits = 64, lblk_bits = 64;
	49	+
	50	+ if (flags & FSCRYPT_POLICY_FLAG_DIRECT_KEY)
	51	+ return offsetofend(union fscrypt_iv, nonce);
	52	+
	53	+ if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64)
	54	+ return sizeof(__le64);
	55	+
	56	+ if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32)
	57	+ return sizeof(__le32);
	58	+
	59	+ /* Default case: IVs are just the file logical block number */
	60	+ if (sb->s_cop->get_ino_and_lblk_bits)
	61	+ sb->s_cop->get_ino_and_lblk_bits(sb, &ino_bits, &lblk_bits);
	62	+ return DIV_ROUND_UP(lblk_bits, 8);
	63	+}
	64	+
	65	+/* Enable inline encryption for this file if supported. */
	66	+int fscrypt_select_encryption_impl(struct fscrypt_info *ci)
	67	+{
	68	+ const struct inode *inode = ci->ci_inode;
	69	+ struct super_block *sb = inode->i_sb;
	70	+ struct blk_crypto_config crypto_cfg;
	71	+ int num_devs;
	72	+ struct request_queue **devs;
	73	+ int i;
	74	+
	75	+ /* The file must need contents encryption, not filenames encryption */
	76	+ if (!fscrypt_needs_contents_encryption(inode))
	77	+ return 0;
	78	+
	79	+ /* The crypto mode must have a blk-crypto counterpart */
	80	+ if (ci->ci_mode->blk_crypto_mode == BLK_ENCRYPTION_MODE_INVALID)
	81	+ return 0;
	82	+
	83	+ /* The filesystem must be mounted with -o inlinecrypt */
	84	+ if (!(sb->s_flags & SB_INLINECRYPT))
	85	+ return 0;
	86	+
	87	+ /*
	88	+ * When a page contains multiple logically contiguous filesystem blocks,
	89	+ * some filesystem code only calls fscrypt_mergeable_bio() for the first
	90	+ * block in the page. This is fine for most of fscrypt's IV generation
	91	+ * strategies, where contiguous blocks imply contiguous IVs. But it
	92	+ * doesn't work with IV_INO_LBLK_32. For now, simply exclude
	93	+ * IV_INO_LBLK_32 with blocksize != PAGE_SIZE from inline encryption.
	94	+ */
	95	+ if ((fscrypt_policy_flags(&ci->ci_policy) &
	96	+ FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32) &&
	97	+ sb->s_blocksize != PAGE_SIZE)
	98	+ return 0;
	99	+
	100	+ /*
	101	+ * On all the filesystem's devices, blk-crypto must support the crypto
	102	+ * configuration that the file would use.
	103	+ */
	104	+ crypto_cfg.crypto_mode = ci->ci_mode->blk_crypto_mode;
	105	+ crypto_cfg.data_unit_size = sb->s_blocksize;
	106	+ crypto_cfg.dun_bytes = fscrypt_get_dun_bytes(ci);
	107	+ num_devs = fscrypt_get_num_devices(sb);
	108	+ devs = kmalloc_array(num_devs, sizeof(*devs), GFP_NOFS);
	109	+ if (!devs)
	110	+ return -ENOMEM;
	111	+ fscrypt_get_devices(sb, num_devs, devs);
	112	+
	113	+ for (i = 0; i < num_devs; i++) {
	114	+ if (!blk_crypto_config_supported(devs[i], &crypto_cfg))
	115	+ goto out_free_devs;
	116	+ }
	117	+
	118	+ ci->ci_inlinecrypt = true;
	119	+out_free_devs:
	120	+ kfree(devs);
	121	+
	122	+ return 0;
	123	+}
	124	+
	125	+int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key,
	126	+ const u8 *raw_key,
	127	+ const struct fscrypt_info *ci)
	128	+{
	129	+ const struct inode *inode = ci->ci_inode;
	130	+ struct super_block *sb = inode->i_sb;
	131	+ enum blk_crypto_mode_num crypto_mode = ci->ci_mode->blk_crypto_mode;
	132	+ int num_devs = fscrypt_get_num_devices(sb);
	133	+ int queue_refs = 0;
	134	+ struct fscrypt_blk_crypto_key *blk_key;
	135	+ int err;
	136	+ int i;
	137	+ unsigned int flags;
	138	+
	139	+ blk_key = kzalloc(struct_size(blk_key, devs, num_devs), GFP_NOFS);
	140	+ if (!blk_key)
	141	+ return -ENOMEM;
	142	+
	143	+ blk_key->num_devs = num_devs;
	144	+ fscrypt_get_devices(sb, num_devs, blk_key->devs);
	145	+
	146	+ err = blk_crypto_init_key(&blk_key->base, raw_key, crypto_mode,
	147	+ fscrypt_get_dun_bytes(ci), sb->s_blocksize);
	148	+ if (err) {
	149	+ fscrypt_err(inode, "error %d initializing blk-crypto key", err);
	150	+ goto fail;
	151	+ }
	152	+
	153	+ /*
	154	+ * We have to start using blk-crypto on all the filesystem's devices.
	155	+ * We also have to save all the request_queue's for later so that the
	156	+ * key can be evicted from them. This is needed because some keys
	157	+ * aren't destroyed until after the filesystem was already unmounted
	158	+ * (namely, the per-mode keys in struct fscrypt_master_key).
	159	+ */
	160	+ for (i = 0; i < num_devs; i++) {
	161	+ if (!blk_get_queue(blk_key->devs[i])) {
	162	+ fscrypt_err(inode, "couldn't get request_queue");
	163	+ err = -EAGAIN;
	164	+ goto fail;
	165	+ }
	166	+ queue_refs++;
	167	+
	168	+ flags = memalloc_nofs_save();
	169	+ err = blk_crypto_start_using_key(&blk_key->base,
	170	+ blk_key->devs[i]);
	171	+ memalloc_nofs_restore(flags);
	172	+ if (err) {
	173	+ fscrypt_err(inode,
	174	+ "error %d starting to use blk-crypto", err);
	175	+ goto fail;
	176	+ }
	177	+ }
	178	+ /*
	179	+ * Pairs with READ_ONCE() in fscrypt_is_key_prepared(). (Only matters
	180	+ * for the per-mode keys, which are shared by multiple inodes.)
	181	+ */
	182	+ smp_store_release(&prep_key->blk_key, blk_key);
	183	+ return 0;
	184	+
	185	+fail:
	186	+ for (i = 0; i < queue_refs; i++)
	187	+ blk_put_queue(blk_key->devs[i]);
	188	+ kzfree(blk_key);
	189	+ return err;
	190	+}
	191	+
	192	+void fscrypt_destroy_inline_crypt_key(struct fscrypt_prepared_key *prep_key)
	193	+{
	194	+ struct fscrypt_blk_crypto_key *blk_key = prep_key->blk_key;
	195	+ int i;
	196	+
	197	+ if (blk_key) {
	198	+ for (i = 0; i < blk_key->num_devs; i++) {
	199	+ blk_crypto_evict_key(blk_key->devs[i], &blk_key->base);
	200	+ blk_put_queue(blk_key->devs[i]);
	201	+ }
	202	+ kzfree(blk_key);
	203	+ }
	204	+}
	205	+
	206	+bool __fscrypt_inode_uses_inline_crypto(const struct inode *inode)
	207	+{
	208	+ return inode->i_crypt_info->ci_inlinecrypt;
	209	+}
	210	+EXPORT_SYMBOL_GPL(__fscrypt_inode_uses_inline_crypto);
	211	+
	212	+static void fscrypt_generate_dun(const struct fscrypt_info *ci, u64 lblk_num,
	213	+ u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE])
	214	+{
	215	+ union fscrypt_iv iv;
	216	+ int i;
	217	+
	218	+ fscrypt_generate_iv(&iv, lblk_num, ci);
	219	+
	220	+ BUILD_BUG_ON(FSCRYPT_MAX_IV_SIZE > BLK_CRYPTO_MAX_IV_SIZE);
	221	+ memset(dun, 0, BLK_CRYPTO_MAX_IV_SIZE);
	222	+ for (i = 0; i < ci->ci_mode->ivsize/sizeof(dun[0]); i++)
	223	+ dun[i] = le64_to_cpu(iv.dun[i]);
	224	+}
	225	+
	226	+/**
	227	+ * fscrypt_set_bio_crypt_ctx() - prepare a file contents bio for inline crypto
	228	+ * @bio: a bio which will eventually be submitted to the file
	229	+ * @inode: the file's inode
	230	+ * @first_lblk: the first file logical block number in the I/O
	231	+ * @gfp_mask: memory allocation flags - these must be a waiting mask so that
	232	+ * bio_crypt_set_ctx can't fail.
	233	+ *
	234	+ * If the contents of the file should be encrypted (or decrypted) with inline
	235	+ * encryption, then assign the appropriate encryption context to the bio.
	236	+ *
	237	+ * Normally the bio should be newly allocated (i.e. no pages added yet), as
	238	+ * otherwise fscrypt_mergeable_bio() won't work as intended.
	239	+ *
	240	+ * The encryption context will be freed automatically when the bio is freed.
	241	+ */
	242	+void fscrypt_set_bio_crypt_ctx(struct bio bio, const struct inode inode,
	243	+ u64 first_lblk, gfp_t gfp_mask)
	244	+{
	245	+ const struct fscrypt_info *ci = inode->i_crypt_info;
	246	+ u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE];
	247	+
	248	+ if (!fscrypt_inode_uses_inline_crypto(inode))
	249	+ return;
	250	+
	251	+ fscrypt_generate_dun(ci, first_lblk, dun);
	252	+ bio_crypt_set_ctx(bio, &ci->ci_enc_key.blk_key->base, dun, gfp_mask);
	253	+}
	254	+EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx);
	255	+
	256	+/* Extract the inode and logical block number from a buffer_head. */
	257	+static bool bh_get_inode_and_lblk_num(const struct buffer_head *bh,
	258	+ const struct inode **inode_ret,
	259	+ u64 *lblk_num_ret)
	260	+{
	261	+ struct page *page = bh->b_page;
	262	+ const struct address_space *mapping;
	263	+ const struct inode *inode;
	264	+
	265	+ /*
	266	+ * The ext4 journal (jbd2) can submit a buffer_head it directly created
	267	+ * for a non-pagecache page. fscrypt doesn't care about these.
	268	+ */
	269	+ mapping = page_mapping(page);
	270	+ if (!mapping)
	271	+ return false;
	272	+ inode = mapping->host;
	273	+
	274	+ *inode_ret = inode;
	275	+ *lblk_num_ret = ((u64)page->index << (PAGE_SHIFT - inode->i_blkbits)) +
	276	+ (bh_offset(bh) >> inode->i_blkbits);
	277	+ return true;
	278	+}
	279	+
	280	+/**
	281	+ * fscrypt_set_bio_crypt_ctx_bh() - prepare a file contents bio for inline
	282	+ * crypto
	283	+ * @bio: a bio which will eventually be submitted to the file
	284	+ * @first_bh: the first buffer_head for which I/O will be submitted
	285	+ * @gfp_mask: memory allocation flags
	286	+ *
	287	+ * Same as fscrypt_set_bio_crypt_ctx(), except this takes a buffer_head instead
	288	+ * of an inode and block number directly.
	289	+ */
	290	+void fscrypt_set_bio_crypt_ctx_bh(struct bio *bio,
	291	+ const struct buffer_head *first_bh,
	292	+ gfp_t gfp_mask)
	293	+{
	294	+ const struct inode *inode;
	295	+ u64 first_lblk;
	296	+
	297	+ if (bh_get_inode_and_lblk_num(first_bh, &inode, &first_lblk))
	298	+ fscrypt_set_bio_crypt_ctx(bio, inode, first_lblk, gfp_mask);
	299	+}
	300	+EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx_bh);
	301	+
	302	+/**
	303	+ * fscrypt_mergeable_bio() - test whether data can be added to a bio
	304	+ * @bio: the bio being built up
	305	+ * @inode: the inode for the next part of the I/O
	306	+ * @next_lblk: the next file logical block number in the I/O
	307	+ *
	308	+ * When building a bio which may contain data which should undergo inline
	309	+ * encryption (or decryption) via fscrypt, filesystems should call this function
	310	+ * to ensure that the resulting bio contains only contiguous data unit numbers.
	311	+ * This will return false if the next part of the I/O cannot be merged with the
	312	+ * bio because either the encryption key would be different or the encryption
	313	+ * data unit numbers would be discontiguous.
	314	+ *
	315	+ * fscrypt_set_bio_crypt_ctx() must have already been called on the bio.
	316	+ *
	317	+ * Return: true iff the I/O is mergeable
	318	+ */
	319	+bool fscrypt_mergeable_bio(struct bio bio, const struct inode inode,
	320	+ u64 next_lblk)
	321	+{
	322	+ const struct bio_crypt_ctx *bc = bio->bi_crypt_context;
	323	+ u64 next_dun[BLK_CRYPTO_DUN_ARRAY_SIZE];
	324	+
	325	+ if (!!bc != fscrypt_inode_uses_inline_crypto(inode))
	326	+ return false;
	327	+ if (!bc)
	328	+ return true;
	329	+
	330	+ /*
	331	+ * Comparing the key pointers is good enough, as all I/O for each key
	332	+ * uses the same pointer. I.e., there's currently no need to support
	333	+ * merging requests where the keys are the same but the pointers differ.
	334	+ */
	335	+ if (bc->bc_key != &inode->i_crypt_info->ci_enc_key.blk_key->base)
	336	+ return false;
	337	+
	338	+ fscrypt_generate_dun(inode->i_crypt_info, next_lblk, next_dun);
	339	+ return bio_crypt_dun_is_contiguous(bc, bio->bi_iter.bi_size, next_dun);
	340	+}
	341	+EXPORT_SYMBOL_GPL(fscrypt_mergeable_bio);
	342	+
	343	+/**
	344	+ * fscrypt_mergeable_bio_bh() - test whether data can be added to a bio
	345	+ * @bio: the bio being built up
	346	+ * @next_bh: the next buffer_head for which I/O will be submitted
	347	+ *
	348	+ * Same as fscrypt_mergeable_bio(), except this takes a buffer_head instead of
	349	+ * an inode and block number directly.
	350	+ *
	351	+ * Return: true iff the I/O is mergeable
	352	+ */
	353	+bool fscrypt_mergeable_bio_bh(struct bio *bio,
	354	+ const struct buffer_head *next_bh)
	355	+{
	356	+ const struct inode *inode;
	357	+ u64 next_lblk;
	358	+
	359	+ if (!bh_get_inode_and_lblk_num(next_bh, &inode, &next_lblk))
	360	+ return !bio->bi_crypt_context;
	361	+
	362	+ return fscrypt_mergeable_bio(bio, inode, next_lblk);
	363	+}
	364	+EXPORT_SYMBOL_GPL(fscrypt_mergeable_bio_bh);

fs/crypto/keyring.c+3−3

@@ -45,9 +45,9 @@ static void free_master_key(struct fscrypt_master_key *mk) · 44 unmodified lines
45	45	wipe_master_key_secret(&mk->mk_secret);
46	46
47	47	for (i = 0; i <= __FSCRYPT_MODE_MAX; i++) {
48		− crypto_free_skcipher(mk->mk_direct_keys[i]);
49		− crypto_free_skcipher(mk->mk_iv_ino_lblk_64_keys[i]);
50		− crypto_free_skcipher(mk->mk_iv_ino_lblk_32_keys[i]);
	48	+ fscrypt_destroy_prepared_key(&mk->mk_direct_keys[i]);
	49	+ fscrypt_destroy_prepared_key(&mk->mk_iv_ino_lblk_64_keys[i]);
	50	+ fscrypt_destroy_prepared_key(&mk->mk_iv_ino_lblk_32_keys[i]);
51	51	}
52	52
53	53	key_put(mk->mk_users);

fs/crypto/keysetup.c+48−22

@@ -19,6 +19,7 @@ struct fscrypt_mode fscrypt_modes[] = { · 18 unmodified lines
19	19	.cipher_str = "xts(aes)",
20	20	.keysize = 64,
21	21	.ivsize = 16,
	22	+ .blk_crypto_mode = BLK_ENCRYPTION_MODE_AES_256_XTS,
22	23	},
23	24	[FSCRYPT_MODE_AES_256_CTS] = {
24	25	.friendly_name = "AES-256-CTS-CBC",
@@ -31,6 +32,7 @@ struct fscrypt_mode fscrypt_modes[] = { · 6 unmodified lines
31	32	.cipher_str = "essiv(cbc(aes),sha256)",
32	33	.keysize = 16,
33	34	.ivsize = 16,
	35	+ .blk_crypto_mode = BLK_ENCRYPTION_MODE_AES_128_CBC_ESSIV,
34	36	},
35	37	[FSCRYPT_MODE_AES_128_CTS] = {
36	38	.friendly_name = "AES-128-CTS-CBC",
@@ -43,6 +45,7 @@ struct fscrypt_mode fscrypt_modes[] = { · 6 unmodified lines
43	45	.cipher_str = "adiantum(xchacha12,aes)",
44	46	.keysize = 32,
45	47	.ivsize = 32,
	48	+ .blk_crypto_mode = BLK_ENCRYPTION_MODE_ADIANTUM,
46	49	},
47	50	};
48	51
@@ -64,9 +67,9 @@ select_encryption_mode(const union fscrypt_policy *policy, · 15 unmodified lines
64	67	}
65	68
66	69	/* Create a symmetric cipher object for the given encryption mode and key */
67		−struct crypto_skcipher fscrypt_allocate_skcipher(struct fscrypt_mode mode,
68		− const u8 *raw_key,
69		− const struct inode *inode)
	70	+static struct crypto_skcipher *
	71	+fscrypt_allocate_skcipher(struct fscrypt_mode mode, const u8 raw_key,
	72	+ const struct inode *inode)
70	73	{
71	74	struct crypto_skcipher *tfm;
72	75	int err;
@@ -109,30 +112,54 @@ err_free_tfm: · 36 unmodified lines
109	112	return ERR_PTR(err);
110	113	}
111	114
112		−/* Given a per-file encryption key, set up the file's crypto transform object */
113		−int fscrypt_set_per_file_enc_key(struct fscrypt_info ci, const u8 raw_key)
	115	+/*
	116	+ * Prepare the crypto transform object or blk-crypto key in @prep_key, given the
	117	+ * raw key, encryption mode, and flag indicating which encryption implementation
	118	+ * (fs-layer or blk-crypto) will be used.
	119	+ */
	120	+int fscrypt_prepare_key(struct fscrypt_prepared_key *prep_key,
	121	+ const u8 raw_key, const struct fscrypt_info ci)
114	122	{
115	123	struct crypto_skcipher *tfm;
116	124
	125	+ if (fscrypt_using_inline_encryption(ci))
	126	+ return fscrypt_prepare_inline_crypt_key(prep_key, raw_key, ci);
	127	+
117	128	tfm = fscrypt_allocate_skcipher(ci->ci_mode, raw_key, ci->ci_inode);
118	129	if (IS_ERR(tfm))
119	130	return PTR_ERR(tfm);
	131	+ /*
	132	+ * Pairs with READ_ONCE() in fscrypt_is_key_prepared(). (Only matters
	133	+ * for the per-mode keys, which are shared by multiple inodes.)
	134	+ */
	135	+ smp_store_release(&prep_key->tfm, tfm);
	136	+ return 0;
	137	+}
	138	+
	139	+/* Destroy a crypto transform object and/or blk-crypto key. */
	140	+void fscrypt_destroy_prepared_key(struct fscrypt_prepared_key *prep_key)
	141	+{
	142	+ crypto_free_skcipher(prep_key->tfm);
	143	+ fscrypt_destroy_inline_crypt_key(prep_key);
	144	+}
120	145
121		− ci->ci_ctfm = tfm;
	146	+/* Given a per-file encryption key, set up the file's crypto transform object */
	147	+int fscrypt_set_per_file_enc_key(struct fscrypt_info ci, const u8 raw_key)
	148	+{
122	149	ci->ci_owns_key = true;
123		− return 0;
	150	+ return fscrypt_prepare_key(&ci->ci_enc_key, raw_key, ci);
124	151	}
125	152
126	153	static int setup_per_mode_enc_key(struct fscrypt_info *ci,
127	154	struct fscrypt_master_key *mk,
128		− struct crypto_skcipher **tfms,
	155	+ struct fscrypt_prepared_key *keys,
129	156	u8 hkdf_context, bool include_fs_uuid)
130	157	{
131	158	const struct inode *inode = ci->ci_inode;
132	159	const struct super_block *sb = inode->i_sb;
133	160	struct fscrypt_mode *mode = ci->ci_mode;
134	161	const u8 mode_num = mode - fscrypt_modes;
135		− struct crypto_skcipher *tfm;
	162	+ struct fscrypt_prepared_key *prep_key;
136	163	u8 mode_key[FSCRYPT_MAX_KEY_SIZE];
137	164	u8 hkdf_info[sizeof(mode_num) + sizeof(sb->s_uuid)];
138	165	unsigned int hkdf_infolen = 0;
@@ -141,16 +168,15 @@ static int setup_per_mode_enc_key(struct fscrypt_info *ci, · 2 unmodified lines
141	168	if (WARN_ON(mode_num > __FSCRYPT_MODE_MAX))
142	169	return -EINVAL;
143	170
144		− /* pairs with smp_store_release() below */
145		− tfm = READ_ONCE(tfms[mode_num]);
146		− if (likely(tfm != NULL)) {
147		− ci->ci_ctfm = tfm;
	171	+ prep_key = &keys[mode_num];
	172	+ if (fscrypt_is_key_prepared(prep_key, ci)) {
	173	+ ci->ci_enc_key = *prep_key;
148	174	return 0;
149	175	}
150	176
151	177	mutex_lock(&fscrypt_mode_key_setup_mutex);
152	178
153		− if (tfms[mode_num])
	179	+ if (fscrypt_is_key_prepared(prep_key, ci))
154	180	goto done_unlock;
155	181
156	182	BUILD_BUG_ON(sizeof(mode_num) != 1);
@@ -167,16 +193,12 @@ static int setup_per_mode_enc_key(struct fscrypt_info *ci, · 10 unmodified lines
167	193	mode_key, mode->keysize);
168	194	if (err)
169	195	goto out_unlock;
170		− tfm = fscrypt_allocate_skcipher(mode, mode_key, inode);
	196	+ err = fscrypt_prepare_key(prep_key, mode_key, ci);
171	197	memzero_explicit(mode_key, mode->keysize);
172		− if (IS_ERR(tfm)) {
173		− err = PTR_ERR(tfm);
	198	+ if (err)
174	199	goto out_unlock;
175		− }
176		− /* pairs with READ_ONCE() above */
177		− smp_store_release(&tfms[mode_num], tfm);
178	200	done_unlock:
179		− ci->ci_ctfm = tfm;
	201	+ ci->ci_enc_key = *prep_key;
180	202	err = 0;
181	203	out_unlock:
182	204	mutex_unlock(&fscrypt_mode_key_setup_mutex);
@@ -310,6 +332,10 @@ static int setup_file_encryption_key(struct fscrypt_info *ci, · 127 unmodified lines
310	332	struct fscrypt_key_specifier mk_spec;
311	333	int err;
312	334
	335	+ err = fscrypt_select_encryption_impl(ci);
	336	+ if (err)
	337	+ return err;
	338	+
313	339	switch (ci->ci_policy.version) {
314	340	case FSCRYPT_POLICY_V1:
315	341	mk_spec.type = FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR;
@@ -402,7 +428,7 @@ static void put_crypt_info(struct fscrypt_info *ci) · 86 unmodified lines
402	428	if (ci->ci_direct_key)
403	429	fscrypt_put_direct_key(ci->ci_direct_key);
404	430	else if (ci->ci_owns_key)
405		− crypto_free_skcipher(ci->ci_ctfm);
	431	+ fscrypt_destroy_prepared_key(&ci->ci_enc_key);
406	432
407	433	key = ci->ci_master_key;
408	434	if (key) {

fs/crypto/keysetup_v1.c+7−9

@@ -146,7 +146,7 @@ struct fscrypt_direct_key { · 145 unmodified lines
146	146	struct hlist_node dk_node;
147	147	refcount_t dk_refcount;
148	148	const struct fscrypt_mode *dk_mode;
149		− struct crypto_skcipher *dk_ctfm;
	149	+ struct fscrypt_prepared_key dk_key;
150	150	u8 dk_descriptor[FSCRYPT_KEY_DESCRIPTOR_SIZE];
151	151	u8 dk_raw[FSCRYPT_MAX_KEY_SIZE];
152	152	};
@@ -154,7 +154,7 @@ struct fscrypt_direct_key { · 1 unmodified line
154	154	static void free_direct_key(struct fscrypt_direct_key *dk)
155	155	{
156	156	if (dk) {
157		− crypto_free_skcipher(dk->dk_ctfm);
	157	+ fscrypt_destroy_prepared_key(&dk->dk_key);
158	158	kzfree(dk);
159	159	}
160	160	}
@@ -199,6 +199,8 @@ find_or_insert_direct_key(struct fscrypt_direct_key *to_insert, · 38 unmodified lines
199	199	continue;
200	200	if (ci->ci_mode != dk->dk_mode)
201	201	continue;
	202	+ if (!fscrypt_is_key_prepared(&dk->dk_key, ci))
	203	+ continue;
202	204	if (crypto_memneq(raw_key, dk->dk_raw, ci->ci_mode->keysize))
203	205	continue;
204	206	/* using existing tfm with same (descriptor, mode, raw_key) */
@@ -231,13 +233,9 @@ fscrypt_get_direct_key(const struct fscrypt_info ci, const u8 raw_key) · 26 unmodified lines
231	233	return ERR_PTR(-ENOMEM);
232	234	refcount_set(&dk->dk_refcount, 1);
233	235	dk->dk_mode = ci->ci_mode;
234		− dk->dk_ctfm = fscrypt_allocate_skcipher(ci->ci_mode, raw_key,
235		− ci->ci_inode);
236		− if (IS_ERR(dk->dk_ctfm)) {
237		− err = PTR_ERR(dk->dk_ctfm);
238		− dk->dk_ctfm = NULL;
	236	+ err = fscrypt_prepare_key(&dk->dk_key, raw_key, ci);
	237	+ if (err)
239	238	goto err_free_dk;
240		− }
241	239	memcpy(dk->dk_descriptor, ci->ci_policy.v1.master_key_descriptor,
242	240	FSCRYPT_KEY_DESCRIPTOR_SIZE);
243	241	memcpy(dk->dk_raw, raw_key, ci->ci_mode->keysize);
@@ -259,7 +257,7 @@ static int setup_v1_file_key_direct(struct fscrypt_info *ci, · 15 unmodified lines
259	257	if (IS_ERR(dk))
260	258	return PTR_ERR(dk);
261	259	ci->ci_direct_key = dk;
262		− ci->ci_ctfm = dk->dk_ctfm;
	260	+ ci->ci_enc_key = dk->dk_key;
263	261	return 0;
264	262	}
265	263

include/linux/fscrypt.h+82−0

@@ -69,6 +69,9 @@ struct fscrypt_operations { · 68 unmodified lines
69	69	bool (has_stable_inodes)(struct super_block sb);
70	70	void (get_ino_and_lblk_bits)(struct super_block sb,
71	71	int ino_bits_ret, int lblk_bits_ret);
	72	+ int (get_num_devices)(struct super_block sb);
	73	+ void (get_devices)(struct super_block sb,
	74	+ struct request_queue **devs);
72	75	};
73	76
74	77	static inline bool fscrypt_has_encryption_key(const struct inode *inode)
@@ -537,6 +540,85 @@ static inline void fscrypt_set_ops(struct super_block *sb, · 462 unmodified lines
537	540
538	541	#endif /* !CONFIG_FS_ENCRYPTION */
539	542
	543	+/* inline_crypt.c */
	544	+#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT
	545	+
	546	+bool __fscrypt_inode_uses_inline_crypto(const struct inode *inode);
	547	+
	548	+void fscrypt_set_bio_crypt_ctx(struct bio *bio,
	549	+ const struct inode *inode, u64 first_lblk,
	550	+ gfp_t gfp_mask);
	551	+
	552	+void fscrypt_set_bio_crypt_ctx_bh(struct bio *bio,
	553	+ const struct buffer_head *first_bh,
	554	+ gfp_t gfp_mask);
	555	+
	556	+bool fscrypt_mergeable_bio(struct bio bio, const struct inode inode,
	557	+ u64 next_lblk);
	558	+
	559	+bool fscrypt_mergeable_bio_bh(struct bio *bio,
	560	+ const struct buffer_head *next_bh);
	561	+
	562	+#else /* CONFIG_FS_ENCRYPTION_INLINE_CRYPT */
	563	+
	564	+static inline bool __fscrypt_inode_uses_inline_crypto(const struct inode *inode)
	565	+{
	566	+ return false;
	567	+}
	568	+
	569	+static inline void fscrypt_set_bio_crypt_ctx(struct bio *bio,
	570	+ const struct inode *inode,
	571	+ u64 first_lblk, gfp_t gfp_mask) { }
	572	+
	573	+static inline void fscrypt_set_bio_crypt_ctx_bh(
	574	+ struct bio *bio,
	575	+ const struct buffer_head *first_bh,
	576	+ gfp_t gfp_mask) { }
	577	+
	578	+static inline bool fscrypt_mergeable_bio(struct bio *bio,
	579	+ const struct inode *inode,
	580	+ u64 next_lblk)
	581	+{
	582	+ return true;
	583	+}
	584	+
	585	+static inline bool fscrypt_mergeable_bio_bh(struct bio *bio,
	586	+ const struct buffer_head *next_bh)
	587	+{
	588	+ return true;
	589	+}
	590	+#endif /* !CONFIG_FS_ENCRYPTION_INLINE_CRYPT */
	591	+
	592	+/**
	593	+ * fscrypt_inode_uses_inline_crypto() - test whether an inode uses inline
	594	+ * encryption
	595	+ * @inode: an inode. If encrypted, its key must be set up.
	596	+ *
	597	+ * Return: true if the inode requires file contents encryption and if the
	598	+ * encryption should be done in the block layer via blk-crypto rather
	599	+ * than in the filesystem layer.
	600	+ */
	601	+static inline bool fscrypt_inode_uses_inline_crypto(const struct inode *inode)
	602	+{
	603	+ return fscrypt_needs_contents_encryption(inode) &&
	604	+ __fscrypt_inode_uses_inline_crypto(inode);
	605	+}
	606	+
	607	+/**
	608	+ * fscrypt_inode_uses_fs_layer_crypto() - test whether an inode uses fs-layer
	609	+ * encryption
	610	+ * @inode: an inode. If encrypted, its key must be set up.
	611	+ *
	612	+ * Return: true if the inode requires file contents encryption and if the
	613	+ * encryption should be done in the filesystem layer rather than in the
	614	+ * block layer via blk-crypto.
	615	+ */
	616	+static inline bool fscrypt_inode_uses_fs_layer_crypto(const struct inode *inode)
	617	+{
	618	+ return fscrypt_needs_contents_encryption(inode) &&
	619	+ !__fscrypt_inode_uses_inline_crypto(inode);
	620	+}
	621	+
540	622	/**
541	623	* fscrypt_require_key() - require an inode's encryption key
542	624	* @inode: the inode we need the key for