lael-labs/linux

Documentation/filesystems/fscrypt.rst+3−0

 <https://github.com/tytso/xfstests-bld/blob/master/Documentation/kvm-quickstart.md>`_::
     kvm-xfstests -c ext4,f2fs -g encrypt
+    kvm-xfstests -c ext4,f2fs -g encrypt -m inlinecrypt
 UBIFS encryption can also be tested this way, but it should be done in
 a separate command, and it takes some time for kvm-xfstests to set up
 kvm-xfstests, use the "encrypt" filesystem configuration::
     kvm-xfstests -c ext4/encrypt,f2fs/encrypt -g auto
+    kvm-xfstests -c ext4/encrypt,f2fs/encrypt -g auto -m inlinecrypt
 Because this runs many more tests than "-g encrypt" does, it takes
 much longer to run; so also consider using `gce-xfstests
 instead of kvm-xfstests::
     gce-xfstests -c ext4/encrypt,f2fs/encrypt -g auto
+    gce-xfstests -c ext4/encrypt,f2fs/encrypt -g auto -m inlinecrypt

fs/crypto/Kconfig+6−0

@@ -24,3 +24,9 @@ config FS_ENCRYPTION_ALGS · 23 unmodified lines
24	select CRYPTO_SHA256	24	select CRYPTO_SHA256
25	select CRYPTO_SHA512	25	select CRYPTO_SHA512
26	select CRYPTO_XTS	26	select CRYPTO_XTS
		27
		28	config FS_ENCRYPTION_INLINE_CRYPT
		29	bool "Enable fscrypt to use inline crypto"
		30	depends on FS_ENCRYPTION && BLK_INLINE_ENCRYPTION
		31	help
		32	Enable fscrypt to use inline encryption hardware if available.

fs/crypto/Makefile+1−0

@@ -11,3 +11,4 @@ fscrypto-y := crypto.o \ · 10 unmodified lines
11	policy.o	11	policy.o
12		12
13	fscrypto-$(CONFIG_BLOCK) += bio.o	13	fscrypto-$(CONFIG_BLOCK) += bio.o
		14	fscrypto-$(CONFIG_FS_ENCRYPTION_INLINE_CRYPT) += inline_crypt.o

fs/crypto/bio.c+51−0

@@ -41,6 +41,53 @@ void fscrypt_decrypt_bio(struct bio *bio) · 40 unmodified lines
41	}	41	}
42	EXPORT_SYMBOL(fscrypt_decrypt_bio);	42	EXPORT_SYMBOL(fscrypt_decrypt_bio);
43		43
		44	static int fscrypt_zeroout_range_inline_crypt(const struct inode *inode,
		45	pgoff_t lblk, sector_t pblk,
		46	unsigned int len)
		47	{
		48	const unsigned int blockbits = inode->i_blkbits;
		49	const unsigned int blocks_per_page = 1 << (PAGE_SHIFT - blockbits);
		50	struct bio *bio;
		51	int ret, err = 0;
		52	int num_pages = 0;
		53
		54	/* This always succeeds since __GFP_DIRECT_RECLAIM is set. */
		55	bio = bio_alloc(GFP_NOFS, BIO_MAX_PAGES);
		56
		57	while (len) {
		58	unsigned int blocks_this_page = min(len, blocks_per_page);
		59	unsigned int bytes_this_page = blocks_this_page << blockbits;
		60
		61	if (num_pages == 0) {
		62	fscrypt_set_bio_crypt_ctx(bio, inode, lblk, GFP_NOFS);
		63	bio_set_dev(bio, inode->i_sb->s_bdev);
		64	bio->bi_iter.bi_sector =
		65	pblk << (blockbits - SECTOR_SHIFT);
		66	bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
		67	}
		68	ret = bio_add_page(bio, ZERO_PAGE(0), bytes_this_page, 0);
		69	if (WARN_ON(ret != bytes_this_page)) {
		70	err = -EIO;
		71	goto out;
		72	}
		73	num_pages++;
		74	len -= blocks_this_page;
		75	lblk += blocks_this_page;
		76	pblk += blocks_this_page;
		77	if (num_pages == BIO_MAX_PAGES \|\| !len \|\|
		78	!fscrypt_mergeable_bio(bio, inode, lblk)) {
		79	err = submit_bio_wait(bio);
		80	if (err)
		81	goto out;
		82	bio_reset(bio);
		83	num_pages = 0;
		84	}
		85	}
		86	out:
		87	bio_put(bio);
		88	return err;
		89	}
		90
44	/**	91	/**
45	* fscrypt_zeroout_range() - zero out a range of blocks in an encrypted file	92	* fscrypt_zeroout_range() - zero out a range of blocks in an encrypted file
46	* @inode: the file's inode	93	* @inode: the file's inode
@@ -75,6 +122,10 @@ int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk, · 28 unmodified lines
75	if (len == 0)	122	if (len == 0)
76	return 0;	123	return 0;
77		124
		125	if (fscrypt_inode_uses_inline_crypto(inode))
		126	return fscrypt_zeroout_range_inline_crypt(inode, lblk, pblk,
		127	len);
		128
78	BUILD_BUG_ON(ARRAY_SIZE(pages) > BIO_MAX_PAGES);	129	BUILD_BUG_ON(ARRAY_SIZE(pages) > BIO_MAX_PAGES);
79	nr_pages = min_t(unsigned int, ARRAY_SIZE(pages),	130	nr_pages = min_t(unsigned int, ARRAY_SIZE(pages),
80	(len + blocks_per_page - 1) >> blocks_per_page_bits);	131	(len + blocks_per_page - 1) >> blocks_per_page_bits);

fs/crypto/crypto.c+1−1

@@ -100,7 +100,7 @@ int fscrypt_crypt_block(const struct inode *inode, fscrypt_direction_t rw, · 99 unmodified lines
100	DECLARE_CRYPTO_WAIT(wait);	100	DECLARE_CRYPTO_WAIT(wait);
101	struct scatterlist dst, src;	101	struct scatterlist dst, src;
102	struct fscrypt_info *ci = inode->i_crypt_info;	102	struct fscrypt_info *ci = inode->i_crypt_info;
103	struct crypto_skcipher *tfm = ci->ci_ctfm;	103	struct crypto_skcipher *tfm = ci->ci_enc_key.tfm;
104	int res = 0;	104	int res = 0;
105		105
106	if (WARN_ON_ONCE(len <= 0))	106	if (WARN_ON_ONCE(len <= 0))

fs/crypto/fname.c+2−2

@@ -115,7 +115,7 @@ int fscrypt_fname_encrypt(const struct inode inode, const struct qstr iname, · 114 unmodified lines
115	struct skcipher_request *req = NULL;	115	struct skcipher_request *req = NULL;
116	DECLARE_CRYPTO_WAIT(wait);	116	DECLARE_CRYPTO_WAIT(wait);
117	const struct fscrypt_info *ci = inode->i_crypt_info;	117	const struct fscrypt_info *ci = inode->i_crypt_info;
118	struct crypto_skcipher *tfm = ci->ci_ctfm;	118	struct crypto_skcipher *tfm = ci->ci_enc_key.tfm;
119	union fscrypt_iv iv;	119	union fscrypt_iv iv;
120	struct scatterlist sg;	120	struct scatterlist sg;
121	int res;	121	int res;
@@ -171,7 +171,7 @@ static int fname_decrypt(const struct inode *inode, · 49 unmodified lines
171	DECLARE_CRYPTO_WAIT(wait);	171	DECLARE_CRYPTO_WAIT(wait);
172	struct scatterlist src_sg, dst_sg;	172	struct scatterlist src_sg, dst_sg;
173	const struct fscrypt_info *ci = inode->i_crypt_info;	173	const struct fscrypt_info *ci = inode->i_crypt_info;
174	struct crypto_skcipher *tfm = ci->ci_ctfm;	174	struct crypto_skcipher *tfm = ci->ci_enc_key.tfm;
175	union fscrypt_iv iv;	175	union fscrypt_iv iv;
176	int res;	176	int res;
177		177

fs/crypto/fscrypt_private.h+105−10

@@ -14,6 +14,7 @@ · 13 unmodified lines
14	#include <linux/fscrypt.h>	14	#include <linux/fscrypt.h>
15	#include <linux/siphash.h>	15	#include <linux/siphash.h>
16	#include <crypto/hash.h>	16	#include <crypto/hash.h>
		17	#include <linux/blk-crypto.h>
17		18
18	#define CONST_STRLEN(str) (sizeof(str) - 1)	19	#define CONST_STRLEN(str) (sizeof(str) - 1)
19		20
@@ -166,6 +167,20 @@ struct fscrypt_symlink_data { · 146 unmodified lines
166	char encrypted_path[1];	167	char encrypted_path[1];
167	} __packed;	168	} __packed;
168		169
		170	/**
		171	* struct fscrypt_prepared_key - a key prepared for actual encryption/decryption
		172	* @tfm: crypto API transform object
		173	* @blk_key: key for blk-crypto
		174	*
		175	* Normally only one of the fields will be non-NULL.
		176	*/
		177	struct fscrypt_prepared_key {
		178	struct crypto_skcipher *tfm;
		179	#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT
		180	struct fscrypt_blk_crypto_key *blk_key;
		181	#endif
		182	};
		183
169	/*	184	/*
170	* fscrypt_info - the "encryption key" for an inode	185	* fscrypt_info - the "encryption key" for an inode
171	*	186	*
@@ -175,12 +190,20 @@ struct fscrypt_symlink_data { · 3 unmodified lines
175	*/	190	*/
176	struct fscrypt_info {	191	struct fscrypt_info {
177		192
178	/* The actual crypto transform used for encryption and decryption */	193	/* The key in a form prepared for actual encryption/decryption */
179	struct crypto_skcipher *ci_ctfm;	194	struct fscrypt_prepared_key ci_enc_key;
180		195
181	/* True if the key should be freed when this fscrypt_info is freed */	196	/* True if ci_enc_key should be freed when this fscrypt_info is freed */
182	bool ci_owns_key;	197	bool ci_owns_key;
183		198
		199	#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT
		200	/*
		201	* True if this inode will use inline encryption (blk-crypto) instead of
		202	* the traditional filesystem-layer encryption.
		203	*/
		204	bool ci_inlinecrypt;
		205	#endif
		206
184	/*	207	/*
185	* Encryption mode used for this inode. It corresponds to either the	208	* Encryption mode used for this inode. It corresponds to either the
186	* contents or filenames encryption mode, depending on the inode type.	209	* contents or filenames encryption mode, depending on the inode type.
@@ -205,7 +228,7 @@ struct fscrypt_info { · 18 unmodified lines
205		228
206	/*	229	/*
207	* If non-NULL, then encryption is done using the master key directly	230	* If non-NULL, then encryption is done using the master key directly
208	* and ci_ctfm will equal ci_direct_key->dk_ctfm.	231	* and ci_enc_key will equal ci_direct_key->dk_key.
209	*/	232	*/
210	struct fscrypt_direct_key *ci_direct_key;	233	struct fscrypt_direct_key *ci_direct_key;
211		234
@@ -260,6 +283,7 @@ union fscrypt_iv { · 48 unmodified lines
260	u8 nonce[FS_KEY_DERIVATION_NONCE_SIZE];	283	u8 nonce[FS_KEY_DERIVATION_NONCE_SIZE];
261	};	284	};
262	u8 raw[FSCRYPT_MAX_IV_SIZE];	285	u8 raw[FSCRYPT_MAX_IV_SIZE];
		286	__le64 dun[FSCRYPT_MAX_IV_SIZE / sizeof(__le64)];
263	};	287	};
264		288
265	void fscrypt_generate_iv(union fscrypt_iv *iv, u64 lblk_num,	289	void fscrypt_generate_iv(union fscrypt_iv *iv, u64 lblk_num,
@@ -302,6 +326,75 @@ int fscrypt_hkdf_expand(const struct fscrypt_hkdf *hkdf, u8 context, · 36 unmodified lines
302		326
303	void fscrypt_destroy_hkdf(struct fscrypt_hkdf *hkdf);	327	void fscrypt_destroy_hkdf(struct fscrypt_hkdf *hkdf);
304		328
		329	/* inline_crypt.c */
		330	#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT
		331	int fscrypt_select_encryption_impl(struct fscrypt_info *ci);
		332
		333	static inline bool
		334	fscrypt_using_inline_encryption(const struct fscrypt_info *ci)
		335	{
		336	return ci->ci_inlinecrypt;
		337	}
		338
		339	int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key,
		340	const u8 *raw_key,
		341	const struct fscrypt_info *ci);
		342
		343	void fscrypt_destroy_inline_crypt_key(struct fscrypt_prepared_key *prep_key);
		344
		345	/*
		346	* Check whether the crypto transform or blk-crypto key has been allocated in
		347	* @prep_key, depending on which encryption implementation the file will use.
		348	*/
		349	static inline bool
		350	fscrypt_is_key_prepared(struct fscrypt_prepared_key *prep_key,
		351	const struct fscrypt_info *ci)
		352	{
		353	/*
		354	* The READ_ONCE() here pairs with the smp_store_release() in
		355	* fscrypt_prepare_key(). (This only matters for the per-mode keys,
		356	* which are shared by multiple inodes.)
		357	*/
		358	if (fscrypt_using_inline_encryption(ci))
		359	return READ_ONCE(prep_key->blk_key) != NULL;
		360	return READ_ONCE(prep_key->tfm) != NULL;
		361	}
		362
		363	#else /* CONFIG_FS_ENCRYPTION_INLINE_CRYPT */
		364
		365	static inline int fscrypt_select_encryption_impl(struct fscrypt_info *ci)
		366	{
		367	return 0;
		368	}
		369
		370	static inline bool
		371	fscrypt_using_inline_encryption(const struct fscrypt_info *ci)
		372	{
		373	return false;
		374	}
		375
		376	static inline int
		377	fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key,
		378	const u8 *raw_key,
		379	const struct fscrypt_info *ci)
		380	{
		381	WARN_ON(1);
		382	return -EOPNOTSUPP;
		383	}
		384
		385	static inline void
		386	fscrypt_destroy_inline_crypt_key(struct fscrypt_prepared_key *prep_key)
		387	{
		388	}
		389
		390	static inline bool
		391	fscrypt_is_key_prepared(struct fscrypt_prepared_key *prep_key,
		392	const struct fscrypt_info *ci)
		393	{
		394	return READ_ONCE(prep_key->tfm) != NULL;
		395	}
		396	#endif /* !CONFIG_FS_ENCRYPTION_INLINE_CRYPT */
		397
305	/* keyring.c */	398	/* keyring.c */
306		399
307	/*	400	/*
@@ -395,9 +488,9 @@ struct fscrypt_master_key { · 87 unmodified lines
395	* Per-mode encryption keys for the various types of encryption policies	488	* Per-mode encryption keys for the various types of encryption policies
396	* that use them. Allocated and derived on-demand.	489	* that use them. Allocated and derived on-demand.
397	*/	490	*/
398	struct crypto_skcipher *mk_direct_keys[__FSCRYPT_MODE_MAX + 1];	491	struct fscrypt_prepared_key mk_direct_keys[__FSCRYPT_MODE_MAX + 1];
399	struct crypto_skcipher *mk_iv_ino_lblk_64_keys[__FSCRYPT_MODE_MAX + 1];	492	struct fscrypt_prepared_key mk_iv_ino_lblk_64_keys[__FSCRYPT_MODE_MAX + 1];
400	struct crypto_skcipher *mk_iv_ino_lblk_32_keys[__FSCRYPT_MODE_MAX + 1];	493	struct fscrypt_prepared_key mk_iv_ino_lblk_32_keys[__FSCRYPT_MODE_MAX + 1];
401		494
402	/* Hash key for inode numbers. Initialized only when needed. */	495	/* Hash key for inode numbers. Initialized only when needed. */
403	siphash_key_t mk_ino_hash_key;	496	siphash_key_t mk_ino_hash_key;
@@ -461,13 +554,15 @@ struct fscrypt_mode { · 57 unmodified lines
461	int keysize;	554	int keysize;
462	int ivsize;	555	int ivsize;
463	int logged_impl_name;	556	int logged_impl_name;
		557	enum blk_crypto_mode_num blk_crypto_mode;
464	};	558	};
465		559
466	extern struct fscrypt_mode fscrypt_modes[];	560	extern struct fscrypt_mode fscrypt_modes[];
467		561
468	struct crypto_skcipher fscrypt_allocate_skcipher(struct fscrypt_mode mode,	562	int fscrypt_prepare_key(struct fscrypt_prepared_key *prep_key,
469	const u8 *raw_key,	563	const u8 raw_key, const struct fscrypt_info ci);
470	const struct inode *inode);	564
		565	void fscrypt_destroy_prepared_key(struct fscrypt_prepared_key *prep_key);
471		566
472	int fscrypt_set_per_file_enc_key(struct fscrypt_info ci, const u8 raw_key);	567	int fscrypt_set_per_file_enc_key(struct fscrypt_info ci, const u8 raw_key);
473		568

fs/crypto/inline_crypt.c+364−0

@@ -0,0 +1,364 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Inline encryption support for fscrypt
	4	*
	5	* Copyright 2019 Google LLC
	6	*/
	7
	8	/*
	9	* With "inline encryption", the block layer handles the decryption/encryption
	10	* as part of the bio, instead of the filesystem doing the crypto itself via
	11	* crypto API. See Documentation/block/inline-encryption.rst. fscrypt still
	12	* provides the key and IV to use.
	13	*/
	14
	15	#include <linux/blk-crypto.h>
	16	#include <linux/blkdev.h>
	17	#include <linux/buffer_head.h>
	18	#include <linux/sched/mm.h>
	19
	20	#include "fscrypt_private.h"
	21
	22	struct fscrypt_blk_crypto_key {
	23	struct blk_crypto_key base;
	24	int num_devs;
	25	struct request_queue *devs[];
	26	};
	27
	28	static int fscrypt_get_num_devices(struct super_block *sb)
	29	{
	30	if (sb->s_cop->get_num_devices)
	31	return sb->s_cop->get_num_devices(sb);
	32	return 1;
	33	}
	34
	35	static void fscrypt_get_devices(struct super_block *sb, int num_devs,
	36	struct request_queue **devs)
	37	{
	38	if (num_devs == 1)
	39	devs[0] = bdev_get_queue(sb->s_bdev);
	40	else
	41	sb->s_cop->get_devices(sb, devs);
	42	}
	43
	44	static unsigned int fscrypt_get_dun_bytes(const struct fscrypt_info *ci)
	45	{
	46	struct super_block *sb = ci->ci_inode->i_sb;
	47	unsigned int flags = fscrypt_policy_flags(&ci->ci_policy);
	48	int ino_bits = 64, lblk_bits = 64;
	49
	50	if (flags & FSCRYPT_POLICY_FLAG_DIRECT_KEY)
	51	return offsetofend(union fscrypt_iv, nonce);
	52
	53	if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64)
	54	return sizeof(__le64);
	55
	56	if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32)
	57	return sizeof(__le32);
	58
	59	/* Default case: IVs are just the file logical block number */
	60	if (sb->s_cop->get_ino_and_lblk_bits)
	61	sb->s_cop->get_ino_and_lblk_bits(sb, &ino_bits, &lblk_bits);
	62	return DIV_ROUND_UP(lblk_bits, 8);
	63	}
	64
	65	/* Enable inline encryption for this file if supported. */
	66	int fscrypt_select_encryption_impl(struct fscrypt_info *ci)
	67	{
	68	const struct inode *inode = ci->ci_inode;
	69	struct super_block *sb = inode->i_sb;
	70	struct blk_crypto_config crypto_cfg;
	71	int num_devs;
	72	struct request_queue **devs;
	73	int i;
	74
	75	/* The file must need contents encryption, not filenames encryption */
	76	if (!fscrypt_needs_contents_encryption(inode))
	77	return 0;
	78
	79	/* The crypto mode must have a blk-crypto counterpart */
	80	if (ci->ci_mode->blk_crypto_mode == BLK_ENCRYPTION_MODE_INVALID)
	81	return 0;
	82
	83	/* The filesystem must be mounted with -o inlinecrypt */
	84	if (!(sb->s_flags & SB_INLINECRYPT))
	85	return 0;
	86
	87	/*
	88	* When a page contains multiple logically contiguous filesystem blocks,
	89	* some filesystem code only calls fscrypt_mergeable_bio() for the first
	90	* block in the page. This is fine for most of fscrypt's IV generation
	91	* strategies, where contiguous blocks imply contiguous IVs. But it
	92	* doesn't work with IV_INO_LBLK_32. For now, simply exclude
	93	* IV_INO_LBLK_32 with blocksize != PAGE_SIZE from inline encryption.
	94	*/
	95	if ((fscrypt_policy_flags(&ci->ci_policy) &
	96	FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32) &&
	97	sb->s_blocksize != PAGE_SIZE)
	98	return 0;
	99
	100	/*
	101	* On all the filesystem's devices, blk-crypto must support the crypto
	102	* configuration that the file would use.
	103	*/
	104	crypto_cfg.crypto_mode = ci->ci_mode->blk_crypto_mode;
	105	crypto_cfg.data_unit_size = sb->s_blocksize;
	106	crypto_cfg.dun_bytes = fscrypt_get_dun_bytes(ci);
	107	num_devs = fscrypt_get_num_devices(sb);
	108	devs = kmalloc_array(num_devs, sizeof(*devs), GFP_NOFS);
	109	if (!devs)
	110	return -ENOMEM;
	111	fscrypt_get_devices(sb, num_devs, devs);
	112
	113	for (i = 0; i < num_devs; i++) {
	114	if (!blk_crypto_config_supported(devs[i], &crypto_cfg))
	115	goto out_free_devs;
	116	}
	117
	118	ci->ci_inlinecrypt = true;
	119	out_free_devs:
	120	kfree(devs);
	121
	122	return 0;
	123	}
	124
	125	int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key,
	126	const u8 *raw_key,
	127	const struct fscrypt_info *ci)
	128	{
	129	const struct inode *inode = ci->ci_inode;
	130	struct super_block *sb = inode->i_sb;
	131	enum blk_crypto_mode_num crypto_mode = ci->ci_mode->blk_crypto_mode;
	132	int num_devs = fscrypt_get_num_devices(sb);
	133	int queue_refs = 0;
	134	struct fscrypt_blk_crypto_key *blk_key;
	135	int err;
	136	int i;
	137	unsigned int flags;
	138
	139	blk_key = kzalloc(struct_size(blk_key, devs, num_devs), GFP_NOFS);
	140	if (!blk_key)
	141	return -ENOMEM;
	142
	143	blk_key->num_devs = num_devs;
	144	fscrypt_get_devices(sb, num_devs, blk_key->devs);
	145
	146	err = blk_crypto_init_key(&blk_key->base, raw_key, crypto_mode,
	147	fscrypt_get_dun_bytes(ci), sb->s_blocksize);
	148	if (err) {
	149	fscrypt_err(inode, "error %d initializing blk-crypto key", err);
	150	goto fail;
	151	}
	152
	153	/*
	154	* We have to start using blk-crypto on all the filesystem's devices.
	155	* We also have to save all the request_queue's for later so that the
	156	* key can be evicted from them. This is needed because some keys
	157	* aren't destroyed until after the filesystem was already unmounted
	158	* (namely, the per-mode keys in struct fscrypt_master_key).
	159	*/
	160	for (i = 0; i < num_devs; i++) {
	161	if (!blk_get_queue(blk_key->devs[i])) {
	162	fscrypt_err(inode, "couldn't get request_queue");
	163	err = -EAGAIN;
	164	goto fail;
	165	}
	166	queue_refs++;
	167
	168	flags = memalloc_nofs_save();
	169	err = blk_crypto_start_using_key(&blk_key->base,
	170	blk_key->devs[i]);
	171	memalloc_nofs_restore(flags);
	172	if (err) {
	173	fscrypt_err(inode,
	174	"error %d starting to use blk-crypto", err);
	175	goto fail;
	176	}
	177	}
	178	/*
	179	* Pairs with READ_ONCE() in fscrypt_is_key_prepared(). (Only matters
	180	* for the per-mode keys, which are shared by multiple inodes.)
	181	*/
	182	smp_store_release(&prep_key->blk_key, blk_key);
	183	return 0;
	184
	185	fail:
	186	for (i = 0; i < queue_refs; i++)
	187	blk_put_queue(blk_key->devs[i]);
	188	kzfree(blk_key);
	189	return err;
	190	}
	191
	192	void fscrypt_destroy_inline_crypt_key(struct fscrypt_prepared_key *prep_key)
	193	{
	194	struct fscrypt_blk_crypto_key *blk_key = prep_key->blk_key;
	195	int i;
	196
	197	if (blk_key) {
	198	for (i = 0; i < blk_key->num_devs; i++) {
	199	blk_crypto_evict_key(blk_key->devs[i], &blk_key->base);
	200	blk_put_queue(blk_key->devs[i]);
	201	}
	202	kzfree(blk_key);
	203	}
	204	}
	205
	206	bool __fscrypt_inode_uses_inline_crypto(const struct inode *inode)
	207	{
	208	return inode->i_crypt_info->ci_inlinecrypt;
	209	}
	210	EXPORT_SYMBOL_GPL(__fscrypt_inode_uses_inline_crypto);
	211
	212	static void fscrypt_generate_dun(const struct fscrypt_info *ci, u64 lblk_num,
	213	u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE])
	214	{
	215	union fscrypt_iv iv;
	216	int i;
	217
	218	fscrypt_generate_iv(&iv, lblk_num, ci);
	219
	220	BUILD_BUG_ON(FSCRYPT_MAX_IV_SIZE > BLK_CRYPTO_MAX_IV_SIZE);
	221	memset(dun, 0, BLK_CRYPTO_MAX_IV_SIZE);
	222	for (i = 0; i < ci->ci_mode->ivsize/sizeof(dun[0]); i++)
	223	dun[i] = le64_to_cpu(iv.dun[i]);
	224	}
	225
	226	/**
	227	* fscrypt_set_bio_crypt_ctx() - prepare a file contents bio for inline crypto
	228	* @bio: a bio which will eventually be submitted to the file
	229	* @inode: the file's inode
	230	* @first_lblk: the first file logical block number in the I/O
	231	* @gfp_mask: memory allocation flags - these must be a waiting mask so that
	232	* bio_crypt_set_ctx can't fail.
	233	*
	234	* If the contents of the file should be encrypted (or decrypted) with inline
	235	* encryption, then assign the appropriate encryption context to the bio.
	236	*
	237	* Normally the bio should be newly allocated (i.e. no pages added yet), as
	238	* otherwise fscrypt_mergeable_bio() won't work as intended.
	239	*
	240	* The encryption context will be freed automatically when the bio is freed.
	241	*/
	242	void fscrypt_set_bio_crypt_ctx(struct bio bio, const struct inode inode,
	243	u64 first_lblk, gfp_t gfp_mask)
	244	{
	245	const struct fscrypt_info *ci = inode->i_crypt_info;
	246	u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE];
	247
	248	if (!fscrypt_inode_uses_inline_crypto(inode))
	249	return;
	250
	251	fscrypt_generate_dun(ci, first_lblk, dun);
	252	bio_crypt_set_ctx(bio, &ci->ci_enc_key.blk_key->base, dun, gfp_mask);
	253	}
	254	EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx);
	255
	256	/* Extract the inode and logical block number from a buffer_head. */
	257	static bool bh_get_inode_and_lblk_num(const struct buffer_head *bh,
	258	const struct inode **inode_ret,
	259	u64 *lblk_num_ret)
	260	{
	261	struct page *page = bh->b_page;
	262	const struct address_space *mapping;
	263	const struct inode *inode;
	264
	265	/*
	266	* The ext4 journal (jbd2) can submit a buffer_head it directly created
	267	* for a non-pagecache page. fscrypt doesn't care about these.
	268	*/
	269	mapping = page_mapping(page);
	270	if (!mapping)
	271	return false;
	272	inode = mapping->host;
	273
	274	*inode_ret = inode;
	275	*lblk_num_ret = ((u64)page->index << (PAGE_SHIFT - inode->i_blkbits)) +
	276	(bh_offset(bh) >> inode->i_blkbits);
	277	return true;
	278	}
	279
	280	/**
	281	* fscrypt_set_bio_crypt_ctx_bh() - prepare a file contents bio for inline
	282	* crypto
	283	* @bio: a bio which will eventually be submitted to the file
	284	* @first_bh: the first buffer_head for which I/O will be submitted
	285	* @gfp_mask: memory allocation flags
	286	*
	287	* Same as fscrypt_set_bio_crypt_ctx(), except this takes a buffer_head instead
	288	* of an inode and block number directly.
	289	*/
	290	void fscrypt_set_bio_crypt_ctx_bh(struct bio *bio,
	291	const struct buffer_head *first_bh,
	292	gfp_t gfp_mask)
	293	{
	294	const struct inode *inode;
	295	u64 first_lblk;
	296
	297	if (bh_get_inode_and_lblk_num(first_bh, &inode, &first_lblk))
	298	fscrypt_set_bio_crypt_ctx(bio, inode, first_lblk, gfp_mask);
	299	}
	300	EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx_bh);
	301
	302	/**
	303	* fscrypt_mergeable_bio() - test whether data can be added to a bio
	304	* @bio: the bio being built up
	305	* @inode: the inode for the next part of the I/O
	306	* @next_lblk: the next file logical block number in the I/O
	307	*
	308	* When building a bio which may contain data which should undergo inline
	309	* encryption (or decryption) via fscrypt, filesystems should call this function
	310	* to ensure that the resulting bio contains only contiguous data unit numbers.
	311	* This will return false if the next part of the I/O cannot be merged with the
	312	* bio because either the encryption key would be different or the encryption
	313	* data unit numbers would be discontiguous.
	314	*
	315	* fscrypt_set_bio_crypt_ctx() must have already been called on the bio.
	316	*
	317	* Return: true iff the I/O is mergeable
	318	*/
	319	bool fscrypt_mergeable_bio(struct bio bio, const struct inode inode,
	320	u64 next_lblk)
	321	{
	322	const struct bio_crypt_ctx *bc = bio->bi_crypt_context;
	323	u64 next_dun[BLK_CRYPTO_DUN_ARRAY_SIZE];
	324
	325	if (!!bc != fscrypt_inode_uses_inline_crypto(inode))
	326	return false;
	327	if (!bc)
	328	return true;
	329
	330	/*
	331	* Comparing the key pointers is good enough, as all I/O for each key
	332	* uses the same pointer. I.e., there's currently no need to support
	333	* merging requests where the keys are the same but the pointers differ.
	334	*/
	335	if (bc->bc_key != &inode->i_crypt_info->ci_enc_key.blk_key->base)
	336	return false;
	337
	338	fscrypt_generate_dun(inode->i_crypt_info, next_lblk, next_dun);
	339	return bio_crypt_dun_is_contiguous(bc, bio->bi_iter.bi_size, next_dun);
	340	}
	341	EXPORT_SYMBOL_GPL(fscrypt_mergeable_bio);
	342
	343	/**
	344	* fscrypt_mergeable_bio_bh() - test whether data can be added to a bio
	345	* @bio: the bio being built up
	346	* @next_bh: the next buffer_head for which I/O will be submitted
	347	*
	348	* Same as fscrypt_mergeable_bio(), except this takes a buffer_head instead of
	349	* an inode and block number directly.
	350	*
	351	* Return: true iff the I/O is mergeable
	352	*/
	353	bool fscrypt_mergeable_bio_bh(struct bio *bio,
	354	const struct buffer_head *next_bh)
	355	{
	356	const struct inode *inode;
	357	u64 next_lblk;
	358
	359	if (!bh_get_inode_and_lblk_num(next_bh, &inode, &next_lblk))
	360	return !bio->bi_crypt_context;
	361
	362	return fscrypt_mergeable_bio(bio, inode, next_lblk);
	363	}
	364	EXPORT_SYMBOL_GPL(fscrypt_mergeable_bio_bh);

fs/crypto/keyring.c+3−3

@@ -45,9 +45,9 @@ static void free_master_key(struct fscrypt_master_key *mk) · 44 unmodified lines
45	wipe_master_key_secret(&mk->mk_secret);	45	wipe_master_key_secret(&mk->mk_secret);
46		46
47	for (i = 0; i <= __FSCRYPT_MODE_MAX; i++) {	47	for (i = 0; i <= __FSCRYPT_MODE_MAX; i++) {
48	crypto_free_skcipher(mk->mk_direct_keys[i]);	48	fscrypt_destroy_prepared_key(&mk->mk_direct_keys[i]);
49	crypto_free_skcipher(mk->mk_iv_ino_lblk_64_keys[i]);	49	fscrypt_destroy_prepared_key(&mk->mk_iv_ino_lblk_64_keys[i]);
50	crypto_free_skcipher(mk->mk_iv_ino_lblk_32_keys[i]);	50	fscrypt_destroy_prepared_key(&mk->mk_iv_ino_lblk_32_keys[i]);
51	}	51	}
52		52
53	key_put(mk->mk_users);	53	key_put(mk->mk_users);

fs/crypto/keysetup.c+48−22

 		.cipher_str = "xts(aes)",
 		.keysize = 64,
 		.ivsize = 16,
+		.blk_crypto_mode = BLK_ENCRYPTION_MODE_AES_256_XTS,
 	},
 	[FSCRYPT_MODE_AES_256_CTS] = {
 		.friendly_name = "AES-256-CTS-CBC",
 		.cipher_str = "essiv(cbc(aes),sha256)",
 		.keysize = 16,
 		.ivsize = 16,
+		.blk_crypto_mode = BLK_ENCRYPTION_MODE_AES_128_CBC_ESSIV,
 	},
 	[FSCRYPT_MODE_AES_128_CTS] = {
 		.friendly_name = "AES-128-CTS-CBC",
 		.cipher_str = "adiantum(xchacha12,aes)",
 		.keysize = 32,
 		.ivsize = 32,
+		.blk_crypto_mode = BLK_ENCRYPTION_MODE_ADIANTUM,
 	},
 };
 }
 /* Create a symmetric cipher object for the given encryption mode and key */
-struct crypto_skcipher *fscrypt_allocate_skcipher(struct fscrypt_mode *mode,
+static struct crypto_skcipher *
-						  const u8 *raw_key,
+fscrypt_allocate_skcipher(struct fscrypt_mode *mode, const u8 *raw_key,
-						  const struct inode *inode)
+			  const struct inode *inode)
 {
 	struct crypto_skcipher *tfm;
 	int err;
 	return ERR_PTR(err);
 }
-/* Given a per-file encryption key, set up the file's crypto transform object */
+/*
-int fscrypt_set_per_file_enc_key(struct fscrypt_info *ci, const u8 *raw_key)
+ * Prepare the crypto transform object or blk-crypto key in @prep_key, given the
+ * raw key, encryption mode, and flag indicating which encryption implementation
+ * (fs-layer or blk-crypto) will be used.
+ */
+int fscrypt_prepare_key(struct fscrypt_prepared_key *prep_key,
+			const u8 *raw_key, const struct fscrypt_info *ci)
 {
 	struct crypto_skcipher *tfm;
+	if (fscrypt_using_inline_encryption(ci))
+		return fscrypt_prepare_inline_crypt_key(prep_key, raw_key, ci);
 	tfm = fscrypt_allocate_skcipher(ci->ci_mode, raw_key, ci->ci_inode);
 	if (IS_ERR(tfm))
 		return PTR_ERR(tfm);
+	/*
+	 * Pairs with READ_ONCE() in fscrypt_is_key_prepared().  (Only matters
+	 * for the per-mode keys, which are shared by multiple inodes.)
+	 */
+	smp_store_release(&prep_key->tfm, tfm);
+	return 0;
+}
+/* Destroy a crypto transform object and/or blk-crypto key. */
+void fscrypt_destroy_prepared_key(struct fscrypt_prepared_key *prep_key)
+{
+	crypto_free_skcipher(prep_key->tfm);
+	fscrypt_destroy_inline_crypt_key(prep_key);
+}
-	ci->ci_ctfm = tfm;
+/* Given a per-file encryption key, set up the file's crypto transform object */
+int fscrypt_set_per_file_enc_key(struct fscrypt_info *ci, const u8 *raw_key)
+{
 	ci->ci_owns_key = true;
-	return 0;
+	return fscrypt_prepare_key(&ci->ci_enc_key, raw_key, ci);
 }
 static int setup_per_mode_enc_key(struct fscrypt_info *ci,
 				  struct fscrypt_master_key *mk,
-				  struct crypto_skcipher **tfms,
+				  struct fscrypt_prepared_key *keys,
 				  u8 hkdf_context, bool include_fs_uuid)
 {
 	const struct inode *inode = ci->ci_inode;
 	const struct super_block *sb = inode->i_sb;
 	struct fscrypt_mode *mode = ci->ci_mode;
 	const u8 mode_num = mode - fscrypt_modes;
-	struct crypto_skcipher *tfm;
+	struct fscrypt_prepared_key *prep_key;
 	u8 mode_key[FSCRYPT_MAX_KEY_SIZE];
 	u8 hkdf_info[sizeof(mode_num) + sizeof(sb->s_uuid)];
 	unsigned int hkdf_infolen = 0;
 	if (WARN_ON(mode_num > __FSCRYPT_MODE_MAX))
 		return -EINVAL;
-	/* pairs with smp_store_release() below */
+	prep_key = &keys[mode_num];
-	tfm = READ_ONCE(tfms[mode_num]);
+	if (fscrypt_is_key_prepared(prep_key, ci)) {
-	if (likely(tfm != NULL)) {
+		ci->ci_enc_key = *prep_key;
-		ci->ci_ctfm = tfm;
 		return 0;
 	}
 	mutex_lock(&fscrypt_mode_key_setup_mutex);
-	if (tfms[mode_num])
+	if (fscrypt_is_key_prepared(prep_key, ci))
 		goto done_unlock;
 	BUILD_BUG_ON(sizeof(mode_num) != 1);
 				  mode_key, mode->keysize);
 	if (err)
 		goto out_unlock;
-	tfm = fscrypt_allocate_skcipher(mode, mode_key, inode);
+	err = fscrypt_prepare_key(prep_key, mode_key, ci);
 	memzero_explicit(mode_key, mode->keysize);
-	if (IS_ERR(tfm)) {
+	if (err)
-		err = PTR_ERR(tfm);
 		goto out_unlock;
-	}
-	/* pairs with READ_ONCE() above */
-	smp_store_release(&tfms[mode_num], tfm);
 done_unlock:
-	ci->ci_ctfm = tfm;
+	ci->ci_enc_key = *prep_key;
 	err = 0;
 out_unlock:
 	mutex_unlock(&fscrypt_mode_key_setup_mutex);
 	struct fscrypt_key_specifier mk_spec;
 	int err;
+	err = fscrypt_select_encryption_impl(ci);
+	if (err)
+		return err;
 	switch (ci->ci_policy.version) {
 	case FSCRYPT_POLICY_V1:
 		mk_spec.type = FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR;
 	if (ci->ci_direct_key)
 		fscrypt_put_direct_key(ci->ci_direct_key);
 	else if (ci->ci_owns_key)
-		crypto_free_skcipher(ci->ci_ctfm);
+		fscrypt_destroy_prepared_key(&ci->ci_enc_key);
 	key = ci->ci_master_key;
 	if (key) {

fs/crypto/keysetup_v1.c+7−9

 	struct hlist_node		dk_node;
 	refcount_t			dk_refcount;
 	const struct fscrypt_mode	*dk_mode;
-	struct crypto_skcipher		*dk_ctfm;
+	struct fscrypt_prepared_key	dk_key;
 	u8				dk_descriptor[FSCRYPT_KEY_DESCRIPTOR_SIZE];
 	u8				dk_raw[FSCRYPT_MAX_KEY_SIZE];
 };
 static void free_direct_key(struct fscrypt_direct_key *dk)
 {
 	if (dk) {
-		crypto_free_skcipher(dk->dk_ctfm);
+		fscrypt_destroy_prepared_key(&dk->dk_key);
 		kzfree(dk);
 	}
 }
 			continue;
 		if (ci->ci_mode != dk->dk_mode)
 			continue;
+		if (!fscrypt_is_key_prepared(&dk->dk_key, ci))
+			continue;
 		if (crypto_memneq(raw_key, dk->dk_raw, ci->ci_mode->keysize))
 			continue;
 		/* using existing tfm with same (descriptor, mode, raw_key) */
 		return ERR_PTR(-ENOMEM);
 	refcount_set(&dk->dk_refcount, 1);
 	dk->dk_mode = ci->ci_mode;
-	dk->dk_ctfm = fscrypt_allocate_skcipher(ci->ci_mode, raw_key,
+	err = fscrypt_prepare_key(&dk->dk_key, raw_key, ci);
-						ci->ci_inode);
+	if (err)
-	if (IS_ERR(dk->dk_ctfm)) {
-		err = PTR_ERR(dk->dk_ctfm);
-		dk->dk_ctfm = NULL;
 		goto err_free_dk;
-	}
 	memcpy(dk->dk_descriptor, ci->ci_policy.v1.master_key_descriptor,
 	       FSCRYPT_KEY_DESCRIPTOR_SIZE);
 	memcpy(dk->dk_raw, raw_key, ci->ci_mode->keysize);
 	if (IS_ERR(dk))
 		return PTR_ERR(dk);
 	ci->ci_direct_key = dk;
-	ci->ci_ctfm = dk->dk_ctfm;
+	ci->ci_enc_key = dk->dk_key;
 	return 0;
 }

include/linux/fscrypt.h+82−0

@@ -69,6 +69,9 @@ struct fscrypt_operations { · 68 unmodified lines
69	bool (has_stable_inodes)(struct super_block sb);	69	bool (has_stable_inodes)(struct super_block sb);
70	void (get_ino_and_lblk_bits)(struct super_block sb,	70	void (get_ino_and_lblk_bits)(struct super_block sb,
71	int ino_bits_ret, int lblk_bits_ret);	71	int ino_bits_ret, int lblk_bits_ret);
		72	int (get_num_devices)(struct super_block sb);
		73	void (get_devices)(struct super_block sb,
		74	struct request_queue **devs);
72	};	75	};
73		76
74	static inline bool fscrypt_has_encryption_key(const struct inode *inode)	77	static inline bool fscrypt_has_encryption_key(const struct inode *inode)
@@ -537,6 +540,85 @@ static inline void fscrypt_set_ops(struct super_block *sb, · 462 unmodified lines
537		540
538	#endif /* !CONFIG_FS_ENCRYPTION */	541	#endif /* !CONFIG_FS_ENCRYPTION */
539		542
		543	/* inline_crypt.c */
		544	#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT
		545
		546	bool __fscrypt_inode_uses_inline_crypto(const struct inode *inode);
		547
		548	void fscrypt_set_bio_crypt_ctx(struct bio *bio,
		549	const struct inode *inode, u64 first_lblk,
		550	gfp_t gfp_mask);
		551
		552	void fscrypt_set_bio_crypt_ctx_bh(struct bio *bio,
		553	const struct buffer_head *first_bh,
		554	gfp_t gfp_mask);
		555
		556	bool fscrypt_mergeable_bio(struct bio bio, const struct inode inode,
		557	u64 next_lblk);
		558
		559	bool fscrypt_mergeable_bio_bh(struct bio *bio,
		560	const struct buffer_head *next_bh);
		561
		562	#else /* CONFIG_FS_ENCRYPTION_INLINE_CRYPT */
		563
		564	static inline bool __fscrypt_inode_uses_inline_crypto(const struct inode *inode)
		565	{
		566	return false;
		567	}
		568
		569	static inline void fscrypt_set_bio_crypt_ctx(struct bio *bio,
		570	const struct inode *inode,
		571	u64 first_lblk, gfp_t gfp_mask) { }
		572
		573	static inline void fscrypt_set_bio_crypt_ctx_bh(
		574	struct bio *bio,
		575	const struct buffer_head *first_bh,
		576	gfp_t gfp_mask) { }
		577
		578	static inline bool fscrypt_mergeable_bio(struct bio *bio,
		579	const struct inode *inode,
		580	u64 next_lblk)
		581	{
		582	return true;
		583	}
		584
		585	static inline bool fscrypt_mergeable_bio_bh(struct bio *bio,
		586	const struct buffer_head *next_bh)
		587	{
		588	return true;
		589	}
		590	#endif /* !CONFIG_FS_ENCRYPTION_INLINE_CRYPT */
		591
		592	/**
		593	* fscrypt_inode_uses_inline_crypto() - test whether an inode uses inline
		594	* encryption
		595	* @inode: an inode. If encrypted, its key must be set up.
		596	*
		597	* Return: true if the inode requires file contents encryption and if the
		598	* encryption should be done in the block layer via blk-crypto rather
		599	* than in the filesystem layer.
		600	*/
		601	static inline bool fscrypt_inode_uses_inline_crypto(const struct inode *inode)
		602	{
		603	return fscrypt_needs_contents_encryption(inode) &&
		604	__fscrypt_inode_uses_inline_crypto(inode);
		605	}
		606
		607	/**
		608	* fscrypt_inode_uses_fs_layer_crypto() - test whether an inode uses fs-layer
		609	* encryption
		610	* @inode: an inode. If encrypted, its key must be set up.
		611	*
		612	* Return: true if the inode requires file contents encryption and if the
		613	* encryption should be done in the filesystem layer rather than in the
		614	* block layer via blk-crypto.
		615	*/
		616	static inline bool fscrypt_inode_uses_fs_layer_crypto(const struct inode *inode)
		617	{
		618	return fscrypt_needs_contents_encryption(inode) &&
		619	!__fscrypt_inode_uses_inline_crypto(inode);
		620	}
		621
540	/**	622	/**
541	* fscrypt_require_key() - require an inode's encryption key	623	* fscrypt_require_key() - require an inode's encryption key
542	* @inode: the inode we need the key for	624	* @inode: the inode we need the key for