crypto: twofish-avx - remove duplicated glue code and use shared glue code from glue_helper

Now that shared glue code is available, convert twofish-avx to use it.

Jussi Kivilinna committed 14 years ago

Commit a7378d41 parent 414cb5e4 files · +162 −487Browse files

Unified · Split

4 files

arch/x86/crypto/twofish_avx_glue.c +99 −457
arch/x86/crypto/twofish_glue_3way.c +16 −30
arch/x86/include/asm/crypto/twofish.h +46 −0
crypto/Kconfig +1 −0

arch/x86/crypto/twofish_avx_glue.c+99−457

@@ -4,9 +4,6 @@ · 3 unmodified lines
4	4	* Copyright (C) 2012 Johannes Goetzfried
5	5	* <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
6	6	*
7		− * Glue code based on serpent_sse2_glue.c by:
8		− * Copyright (C) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
9		− *
10	7	* This program is free software; you can redistribute it and/or modify
11	8	* it under the terms of the GNU General Public License as published by
12	9	* the Free Software Foundation; either version 2 of the License, or
@@ -39,38 +36,21 @@ · 26 unmodified lines
39	36	#include <asm/i387.h>
40	37	#include <asm/xcr.h>
41	38	#include <asm/xsave.h>
	39	+#include <asm/crypto/twofish.h>
42	40	#include <asm/crypto/ablk_helper.h>
	41	+#include <asm/crypto/glue_helper.h>
43	42	#include <crypto/scatterwalk.h>
44	43	#include <linux/workqueue.h>
45	44	#include <linux/spinlock.h>
46	45
47		−
48	46	#define TWOFISH_PARALLEL_BLOCKS 8
49	47
50		−/* regular block cipher functions from twofish_x86_64 module */
51		−asmlinkage void twofish_enc_blk(struct twofish_ctx ctx, u8 dst,
52		− const u8 *src);
53		−asmlinkage void twofish_dec_blk(struct twofish_ctx ctx, u8 dst,
54		− const u8 *src);
55		−
56		−/* 3-way parallel cipher functions from twofish_x86_64-3way module */
57		−asmlinkage void __twofish_enc_blk_3way(struct twofish_ctx ctx, u8 dst,
58		− const u8 *src, bool xor);
59		−asmlinkage void twofish_dec_blk_3way(struct twofish_ctx ctx, u8 dst,
60		− const u8 *src);
61		−
62	48	static inline void twofish_enc_blk_3way(struct twofish_ctx ctx, u8 dst,
63	49	const u8 *src)
64	50	{
65	51	__twofish_enc_blk_3way(ctx, dst, src, false);
66	52	}
67	53
68		−static inline void twofish_enc_blk_3way_xor(struct twofish_ctx ctx, u8 dst,
69		− const u8 *src)
70		−{
71		− __twofish_enc_blk_3way(ctx, dst, src, true);
72		−}
73		−
74	54	/* 8-way parallel cipher functions */
75	55	asmlinkage void __twofish_enc_blk_8way(struct twofish_ctx ctx, u8 dst,
76	56	const u8 *src, bool xor);
@@ -95,423 +75,142 @@ static inline void twofish_dec_blk_xway(struct twofish_ctx ctx, u8 dst, · 18 unmodified lines
95	75	twofish_dec_blk_8way(ctx, dst, src);
96	76	}
97	77
98		−
99		−static inline bool twofish_fpu_begin(bool fpu_enabled, unsigned int nbytes)
	78	+static void twofish_dec_blk_cbc_xway(void ctx, u128 dst, const u128 *src)
100	79	{
101		− if (fpu_enabled)
102		− return true;
	80	+ u128 ivs[TWOFISH_PARALLEL_BLOCKS - 1];
	81	+ unsigned int j;
103	82
104		− /* AVX is only used when chunk to be processed is large enough, so
105		− * do not enable FPU until it is necessary.
106		− */
107		− if (nbytes < TF_BLOCK_SIZE * TWOFISH_PARALLEL_BLOCKS)
108		− return false;
	83	+ for (j = 0; j < TWOFISH_PARALLEL_BLOCKS - 1; j++)
	84	+ ivs[j] = src[j];
109	85
110		− kernel_fpu_begin();
111		− return true;
112		−}
	86	+ twofish_dec_blk_xway(ctx, (u8 )dst, (u8 )src);
113	87
114		−static inline void twofish_fpu_end(bool fpu_enabled)
115		−{
116		− if (fpu_enabled)
117		− kernel_fpu_end();
	88	+ for (j = 0; j < TWOFISH_PARALLEL_BLOCKS - 1; j++)
	89	+ u128_xor(dst + (j + 1), dst + (j + 1), ivs + j);
118	90	}
119	91
120		−static int ecb_crypt(struct blkcipher_desc desc, struct blkcipher_walk walk,
121		− bool enc)
	92	+static void twofish_enc_blk_ctr_xway(void ctx, u128 dst, const u128 *src,
	93	+ u128 *iv)
122	94	{
123		− bool fpu_enabled = false;
124		− struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
125		− const unsigned int bsize = TF_BLOCK_SIZE;
126		− unsigned int nbytes;
127		− int err;
	95	+ be128 ctrblks[TWOFISH_PARALLEL_BLOCKS];
	96	+ unsigned int i;
128	97
129		− err = blkcipher_walk_virt(desc, walk);
130		− desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
	98	+ for (i = 0; i < TWOFISH_PARALLEL_BLOCKS; i++) {
	99	+ if (dst != src)
	100	+ dst[i] = src[i];
131	101
132		− while ((nbytes = walk->nbytes)) {
133		− u8 *wsrc = walk->src.virt.addr;
134		− u8 *wdst = walk->dst.virt.addr;
135		−
136		− fpu_enabled = twofish_fpu_begin(fpu_enabled, nbytes);
137		−
138		− /* Process multi-block batch */
139		− if (nbytes >= bsize * TWOFISH_PARALLEL_BLOCKS) {
140		− do {
141		− if (enc)
142		− twofish_enc_blk_xway(ctx, wdst, wsrc);
143		− else
144		− twofish_dec_blk_xway(ctx, wdst, wsrc);
145		−
146		− wsrc += bsize * TWOFISH_PARALLEL_BLOCKS;
147		− wdst += bsize * TWOFISH_PARALLEL_BLOCKS;
148		− nbytes -= bsize * TWOFISH_PARALLEL_BLOCKS;
149		− } while (nbytes >= bsize * TWOFISH_PARALLEL_BLOCKS);
150		−
151		− if (nbytes < bsize)
152		− goto done;
153		− }
154		−
155		− /* Process three block batch */
156		− if (nbytes >= bsize * 3) {
157		− do {
158		− if (enc)
159		− twofish_enc_blk_3way(ctx, wdst, wsrc);
160		− else
161		− twofish_dec_blk_3way(ctx, wdst, wsrc);
162		−
163		− wsrc += bsize * 3;
164		− wdst += bsize * 3;
165		− nbytes -= bsize * 3;
166		− } while (nbytes >= bsize * 3);
167		−
168		− if (nbytes < bsize)
169		− goto done;
170		− }
171		−
172		− /* Handle leftovers */
173		− do {
174		− if (enc)
175		− twofish_enc_blk(ctx, wdst, wsrc);
176		− else
177		− twofish_dec_blk(ctx, wdst, wsrc);
178		−
179		− wsrc += bsize;
180		− wdst += bsize;
181		− nbytes -= bsize;
182		− } while (nbytes >= bsize);
183		−
184		−done:
185		− err = blkcipher_walk_done(desc, walk, nbytes);
	102	+ u128_to_be128(&ctrblks[i], iv);
	103	+ u128_inc(iv);
186	104	}
187	105
188		− twofish_fpu_end(fpu_enabled);
189		− return err;
	106	+ twofish_enc_blk_xway_xor(ctx, (u8 )dst, (u8 )ctrblks);
190	107	}
191	108
	109	+static const struct common_glue_ctx twofish_enc = {
	110	+ .num_funcs = 3,
	111	+ .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
	112	+
	113	+ .funcs = { {
	114	+ .num_blocks = TWOFISH_PARALLEL_BLOCKS,
	115	+ .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_xway) }
	116	+ }, {
	117	+ .num_blocks = 3,
	118	+ .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
	119	+ }, {
	120	+ .num_blocks = 1,
	121	+ .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
	122	+ } }
	123	+};
	124	+
	125	+static const struct common_glue_ctx twofish_ctr = {
	126	+ .num_funcs = 3,
	127	+ .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
	128	+
	129	+ .funcs = { {
	130	+ .num_blocks = TWOFISH_PARALLEL_BLOCKS,
	131	+ .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_xway) }
	132	+ }, {
	133	+ .num_blocks = 3,
	134	+ .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) }
	135	+ }, {
	136	+ .num_blocks = 1,
	137	+ .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) }
	138	+ } }
	139	+};
	140	+
	141	+static const struct common_glue_ctx twofish_dec = {
	142	+ .num_funcs = 3,
	143	+ .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
	144	+
	145	+ .funcs = { {
	146	+ .num_blocks = TWOFISH_PARALLEL_BLOCKS,
	147	+ .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_xway) }
	148	+ }, {
	149	+ .num_blocks = 3,
	150	+ .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
	151	+ }, {
	152	+ .num_blocks = 1,
	153	+ .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
	154	+ } }
	155	+};
	156	+
	157	+static const struct common_glue_ctx twofish_dec_cbc = {
	158	+ .num_funcs = 3,
	159	+ .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
	160	+
	161	+ .funcs = { {
	162	+ .num_blocks = TWOFISH_PARALLEL_BLOCKS,
	163	+ .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_xway) }
	164	+ }, {
	165	+ .num_blocks = 3,
	166	+ .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
	167	+ }, {
	168	+ .num_blocks = 1,
	169	+ .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
	170	+ } }
	171	+};
	172	+
192	173	static int ecb_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
193	174	struct scatterlist *src, unsigned int nbytes)
194	175	{
195		− struct blkcipher_walk walk;
196		−
197		− blkcipher_walk_init(&walk, dst, src, nbytes);
198		− return ecb_crypt(desc, &walk, true);
	176	+ return glue_ecb_crypt_128bit(&twofish_enc, desc, dst, src, nbytes);
199	177	}
200	178
201	179	static int ecb_decrypt(struct blkcipher_desc desc, struct scatterlist dst,
202	180	struct scatterlist *src, unsigned int nbytes)
203	181	{
204		− struct blkcipher_walk walk;
205		−
206		− blkcipher_walk_init(&walk, dst, src, nbytes);
207		− return ecb_crypt(desc, &walk, false);
208		−}
209		−
210		−static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,
211		− struct blkcipher_walk *walk)
212		−{
213		− struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
214		− const unsigned int bsize = TF_BLOCK_SIZE;
215		− unsigned int nbytes = walk->nbytes;
216		− u128 src = (u128 )walk->src.virt.addr;
217		− u128 dst = (u128 )walk->dst.virt.addr;
218		− u128 iv = (u128 )walk->iv;
219		−
220		− do {
221		− u128_xor(dst, src, iv);
222		− twofish_enc_blk(ctx, (u8 )dst, (u8 )dst);
223		− iv = dst;
224		−
225		− src += 1;
226		− dst += 1;
227		− nbytes -= bsize;
228		− } while (nbytes >= bsize);
229		−
230		− u128_xor((u128 )walk->iv, (u128 )walk->iv, iv);
231		− return nbytes;
	182	+ return glue_ecb_crypt_128bit(&twofish_dec, desc, dst, src, nbytes);
232	183	}
233	184
234	185	static int cbc_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
235	186	struct scatterlist *src, unsigned int nbytes)
236	187	{
237		− struct blkcipher_walk walk;
238		− int err;
239		−
240		− blkcipher_walk_init(&walk, dst, src, nbytes);
241		− err = blkcipher_walk_virt(desc, &walk);
242		−
243		− while ((nbytes = walk.nbytes)) {
244		− nbytes = __cbc_encrypt(desc, &walk);
245		− err = blkcipher_walk_done(desc, &walk, nbytes);
246		− }
247		−
248		− return err;
249		−}
250		−
251		−static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
252		− struct blkcipher_walk *walk)
253		−{
254		− struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
255		− const unsigned int bsize = TF_BLOCK_SIZE;
256		− unsigned int nbytes = walk->nbytes;
257		− u128 src = (u128 )walk->src.virt.addr;
258		− u128 dst = (u128 )walk->dst.virt.addr;
259		− u128 ivs[TWOFISH_PARALLEL_BLOCKS - 1];
260		− u128 last_iv;
261		− int i;
262		−
263		− /* Start of the last block. */
264		− src += nbytes / bsize - 1;
265		− dst += nbytes / bsize - 1;
266		−
267		− last_iv = *src;
268		−
269		− /* Process multi-block batch */
270		− if (nbytes >= bsize * TWOFISH_PARALLEL_BLOCKS) {
271		− do {
272		− nbytes -= bsize * (TWOFISH_PARALLEL_BLOCKS - 1);
273		− src -= TWOFISH_PARALLEL_BLOCKS - 1;
274		− dst -= TWOFISH_PARALLEL_BLOCKS - 1;
275		−
276		− for (i = 0; i < TWOFISH_PARALLEL_BLOCKS - 1; i++)
277		− ivs[i] = src[i];
278		−
279		− twofish_dec_blk_xway(ctx, (u8 )dst, (u8 )src);
280		−
281		− for (i = 0; i < TWOFISH_PARALLEL_BLOCKS - 1; i++)
282		− u128_xor(dst + (i + 1), dst + (i + 1), ivs + i);
283		−
284		− nbytes -= bsize;
285		− if (nbytes < bsize)
286		− goto done;
287		−
288		− u128_xor(dst, dst, src - 1);
289		− src -= 1;
290		− dst -= 1;
291		− } while (nbytes >= bsize * TWOFISH_PARALLEL_BLOCKS);
292		−
293		− if (nbytes < bsize)
294		− goto done;
295		− }
296		−
297		− /* Process three block batch */
298		− if (nbytes >= bsize * 3) {
299		− do {
300		− nbytes -= bsize * (3 - 1);
301		− src -= 3 - 1;
302		− dst -= 3 - 1;
303		−
304		− ivs[0] = src[0];
305		− ivs[1] = src[1];
306		−
307		− twofish_dec_blk_3way(ctx, (u8 )dst, (u8 )src);
308		−
309		− u128_xor(dst + 1, dst + 1, ivs + 0);
310		− u128_xor(dst + 2, dst + 2, ivs + 1);
311		−
312		− nbytes -= bsize;
313		− if (nbytes < bsize)
314		− goto done;
315		−
316		− u128_xor(dst, dst, src - 1);
317		− src -= 1;
318		− dst -= 1;
319		− } while (nbytes >= bsize * 3);
320		−
321		− if (nbytes < bsize)
322		− goto done;
323		− }
324		−
325		− /* Handle leftovers */
326		− for (;;) {
327		− twofish_dec_blk(ctx, (u8 )dst, (u8 )src);
328		−
329		− nbytes -= bsize;
330		− if (nbytes < bsize)
331		− break;
332		−
333		− u128_xor(dst, dst, src - 1);
334		− src -= 1;
335		− dst -= 1;
336		− }
337		−
338		−done:
339		− u128_xor(dst, dst, (u128 *)walk->iv);
340		− (u128 )walk->iv = last_iv;
341		−
342		− return nbytes;
	188	+ return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk), desc,
	189	+ dst, src, nbytes);
343	190	}
344	191
345	192	static int cbc_decrypt(struct blkcipher_desc desc, struct scatterlist dst,
346	193	struct scatterlist *src, unsigned int nbytes)
347	194	{
348		− bool fpu_enabled = false;
349		− struct blkcipher_walk walk;
350		− int err;
351		−
352		− blkcipher_walk_init(&walk, dst, src, nbytes);
353		− err = blkcipher_walk_virt(desc, &walk);
354		− desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
355		−
356		− while ((nbytes = walk.nbytes)) {
357		− fpu_enabled = twofish_fpu_begin(fpu_enabled, nbytes);
358		− nbytes = __cbc_decrypt(desc, &walk);
359		− err = blkcipher_walk_done(desc, &walk, nbytes);
360		− }
361		−
362		− twofish_fpu_end(fpu_enabled);
363		− return err;
364		−}
365		−
366		−static inline void u128_to_be128(be128 dst, const u128 src)
367		−{
368		− dst->a = cpu_to_be64(src->a);
369		− dst->b = cpu_to_be64(src->b);
	195	+ return glue_cbc_decrypt_128bit(&twofish_dec_cbc, desc, dst, src,
	196	+ nbytes);
370	197	}
371	198
372		−static inline void be128_to_u128(u128 dst, const be128 src)
373		−{
374		− dst->a = be64_to_cpu(src->a);
375		− dst->b = be64_to_cpu(src->b);
376		−}
377		−
378		−static inline void u128_inc(u128 *i)
	199	+static int ctr_crypt(struct blkcipher_desc desc, struct scatterlist dst,
	200	+ struct scatterlist *src, unsigned int nbytes)
379	201	{
380		− i->b++;
381		− if (!i->b)
382		− i->a++;
	202	+ return glue_ctr_crypt_128bit(&twofish_ctr, desc, dst, src, nbytes);
383	203	}
384	204
385		−static void ctr_crypt_final(struct blkcipher_desc *desc,
386		− struct blkcipher_walk *walk)
	205	+static inline bool twofish_fpu_begin(bool fpu_enabled, unsigned int nbytes)
387	206	{
388		− struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
389		− u8 *ctrblk = walk->iv;
390		− u8 keystream[TF_BLOCK_SIZE];
391		− u8 *src = walk->src.virt.addr;
392		− u8 *dst = walk->dst.virt.addr;
393		− unsigned int nbytes = walk->nbytes;
394		−
395		− twofish_enc_blk(ctx, keystream, ctrblk);
396		− crypto_xor(keystream, src, nbytes);
397		− memcpy(dst, keystream, nbytes);
398		−
399		− crypto_inc(ctrblk, TF_BLOCK_SIZE);
	207	+ return glue_fpu_begin(TF_BLOCK_SIZE, TWOFISH_PARALLEL_BLOCKS, NULL,
	208	+ fpu_enabled, nbytes);
400	209	}
401	210
402		−static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
403		− struct blkcipher_walk *walk)
404		−{
405		− struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
406		− const unsigned int bsize = TF_BLOCK_SIZE;
407		− unsigned int nbytes = walk->nbytes;
408		− u128 src = (u128 )walk->src.virt.addr;
409		− u128 dst = (u128 )walk->dst.virt.addr;
410		− u128 ctrblk;
411		− be128 ctrblocks[TWOFISH_PARALLEL_BLOCKS];
412		− int i;
413		−
414		− be128_to_u128(&ctrblk, (be128 *)walk->iv);
415		−
416		− /* Process multi-block batch */
417		− if (nbytes >= bsize * TWOFISH_PARALLEL_BLOCKS) {
418		− do {
419		− /* create ctrblks for parallel encrypt */
420		− for (i = 0; i < TWOFISH_PARALLEL_BLOCKS; i++) {
421		− if (dst != src)
422		− dst[i] = src[i];
423		−
424		− u128_to_be128(&ctrblocks[i], &ctrblk);
425		− u128_inc(&ctrblk);
426		− }
427		−
428		− twofish_enc_blk_xway_xor(ctx, (u8 *)dst,
429		− (u8 *)ctrblocks);
430		−
431		− src += TWOFISH_PARALLEL_BLOCKS;
432		− dst += TWOFISH_PARALLEL_BLOCKS;
433		− nbytes -= bsize * TWOFISH_PARALLEL_BLOCKS;
434		− } while (nbytes >= bsize * TWOFISH_PARALLEL_BLOCKS);
435		−
436		− if (nbytes < bsize)
437		− goto done;
438		− }
439		−
440		− /* Process three block batch */
441		− if (nbytes >= bsize * 3) {
442		− do {
443		− if (dst != src) {
444		− dst[0] = src[0];
445		− dst[1] = src[1];
446		− dst[2] = src[2];
447		− }
448		−
449		− /* create ctrblks for parallel encrypt */
450		− u128_to_be128(&ctrblocks[0], &ctrblk);
451		− u128_inc(&ctrblk);
452		− u128_to_be128(&ctrblocks[1], &ctrblk);
453		− u128_inc(&ctrblk);
454		− u128_to_be128(&ctrblocks[2], &ctrblk);
455		− u128_inc(&ctrblk);
456		−
457		− twofish_enc_blk_3way_xor(ctx, (u8 *)dst,
458		− (u8 *)ctrblocks);
459		−
460		− src += 3;
461		− dst += 3;
462		− nbytes -= bsize * 3;
463		− } while (nbytes >= bsize * 3);
464		−
465		− if (nbytes < bsize)
466		− goto done;
467		− }
468		−
469		− /* Handle leftovers */
470		− do {
471		− if (dst != src)
472		− dst = src;
473		−
474		− u128_to_be128(&ctrblocks[0], &ctrblk);
475		− u128_inc(&ctrblk);
476		−
477		− twofish_enc_blk(ctx, (u8 )ctrblocks, (u8 )ctrblocks);
478		− u128_xor(dst, dst, (u128 *)ctrblocks);
479		−
480		− src += 1;
481		− dst += 1;
482		− nbytes -= bsize;
483		− } while (nbytes >= bsize);
484		−
485		−done:
486		− u128_to_be128((be128 *)walk->iv, &ctrblk);
487		− return nbytes;
488		−}
489		−
490		−static int ctr_crypt(struct blkcipher_desc desc, struct scatterlist dst,
491		− struct scatterlist *src, unsigned int nbytes)
	211	+static inline void twofish_fpu_end(bool fpu_enabled)
492	212	{
493		− bool fpu_enabled = false;
494		− struct blkcipher_walk walk;
495		− int err;
496		−
497		− blkcipher_walk_init(&walk, dst, src, nbytes);
498		− err = blkcipher_walk_virt_block(desc, &walk, TF_BLOCK_SIZE);
499		− desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
500		−
501		− while ((nbytes = walk.nbytes) >= TF_BLOCK_SIZE) {
502		− fpu_enabled = twofish_fpu_begin(fpu_enabled, nbytes);
503		− nbytes = __ctr_crypt(desc, &walk);
504		− err = blkcipher_walk_done(desc, &walk, nbytes);
505		− }
506		−
507		− twofish_fpu_end(fpu_enabled);
508		−
509		− if (walk.nbytes) {
510		− ctr_crypt_final(desc, &walk);
511		− err = blkcipher_walk_done(desc, &walk, 0);
512		− }
513		−
514		− return err;
	213	+ glue_fpu_end(fpu_enabled);
515	214	}
516	215
517	216	struct crypt_priv {
@@ -563,26 +262,6 @@ static void decrypt_callback(void priv, u8 srcdst, unsigned int nbytes) · 45 unmodified lines
563	262	twofish_dec_blk(ctx->ctx, srcdst, srcdst);
564	263	}
565	264
566		−struct twofish_lrw_ctx {
567		− struct lrw_table_ctx lrw_table;
568		− struct twofish_ctx twofish_ctx;
569		−};
570		−
571		−static int lrw_twofish_setkey(struct crypto_tfm tfm, const u8 key,
572		− unsigned int keylen)
573		−{
574		− struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
575		− int err;
576		−
577		− err = __twofish_setkey(&ctx->twofish_ctx, key,
578		− keylen - TF_BLOCK_SIZE, &tfm->crt_flags);
579		− if (err)
580		− return err;
581		−
582		− return lrw_init_table(&ctx->lrw_table, key + keylen -
583		− TF_BLOCK_SIZE);
584		−}
585		−
586	265	static int lrw_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
587	266	struct scatterlist *src, unsigned int nbytes)
588	267	{
@@ -635,43 +314,6 @@ static int lrw_decrypt(struct blkcipher_desc desc, struct scatterlist dst, · 46 unmodified lines
635	314	return ret;
636	315	}
637	316
638		−static void lrw_exit_tfm(struct crypto_tfm *tfm)
639		−{
640		− struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
641		−
642		− lrw_free_table(&ctx->lrw_table);
643		−}
644		−
645		−struct twofish_xts_ctx {
646		− struct twofish_ctx tweak_ctx;
647		− struct twofish_ctx crypt_ctx;
648		−};
649		−
650		−static int xts_twofish_setkey(struct crypto_tfm tfm, const u8 key,
651		− unsigned int keylen)
652		−{
653		− struct twofish_xts_ctx *ctx = crypto_tfm_ctx(tfm);
654		− u32 *flags = &tfm->crt_flags;
655		− int err;
656		−
657		− /* key consists of keys of equal size concatenated, therefore
658		− * the length must be even
659		− */
660		− if (keylen % 2) {
661		− *flags \|= CRYPTO_TFM_RES_BAD_KEY_LEN;
662		− return -EINVAL;
663		− }
664		−
665		− /* first half of xts-key is for crypt */
666		− err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2, flags);
667		− if (err)
668		− return err;
669		−
670		− /* second half of xts-key is for tweak */
671		− return __twofish_setkey(&ctx->tweak_ctx,
672		− key + keylen / 2, keylen / 2, flags);
673		−}
674		−
675	317	static int xts_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
676	318	struct scatterlist *src, unsigned int nbytes)
677	319	{
@@ -798,7 +440,7 @@ static struct crypto_alg twofish_algs[10] = { { · 120 unmodified lines
798	440	.cra_type = &crypto_blkcipher_type,
799	441	.cra_module = THIS_MODULE,
800	442	.cra_list = LIST_HEAD_INIT(twofish_algs[3].cra_list),
801		− .cra_exit = lrw_exit_tfm,
	443	+ .cra_exit = lrw_twofish_exit_tfm,
802	444	.cra_u = {
803	445	.blkcipher = {
804	446	.min_keysize = TF_MIN_KEY_SIZE +

arch/x86/crypto/twofish_glue_3way.c+16−30

@@ -28,22 +28,12 @@ · 27 unmodified lines
28	28	#include <crypto/algapi.h>
29	29	#include <crypto/twofish.h>
30	30	#include <crypto/b128ops.h>
	31	+#include <asm/crypto/twofish.h>
31	32	#include <asm/crypto/glue_helper.h>
32	33	#include <crypto/lrw.h>
33	34	#include <crypto/xts.h>
34	35
35		−/* regular block cipher functions from twofish_x86_64 module */
36		−asmlinkage void twofish_enc_blk(struct twofish_ctx ctx, u8 dst,
37		− const u8 *src);
38		−asmlinkage void twofish_dec_blk(struct twofish_ctx ctx, u8 dst,
39		− const u8 *src);
40		−
41		−/* 3-way parallel cipher functions */
42		−asmlinkage void __twofish_enc_blk_3way(struct twofish_ctx ctx, u8 dst,
43		− const u8 *src, bool xor);
44	36	EXPORT_SYMBOL_GPL(__twofish_enc_blk_3way);
45		−asmlinkage void twofish_dec_blk_3way(struct twofish_ctx ctx, u8 dst,
46		− const u8 *src);
47	37	EXPORT_SYMBOL_GPL(twofish_dec_blk_3way);
48	38
49	39	static inline void twofish_enc_blk_3way(struct twofish_ctx ctx, u8 dst,
@@ -58,7 +48,7 @@ static inline void twofish_enc_blk_xor_3way(struct twofish_ctx ctx, u8 dst, · 8 unmodified lines
58	48	__twofish_enc_blk_3way(ctx, dst, src, true);
59	49	}
60	50
61		−static void twofish_dec_blk_cbc_3way(void ctx, u128 dst, const u128 *src)
	51	+void twofish_dec_blk_cbc_3way(void ctx, u128 dst, const u128 *src)
62	52	{
63	53	u128 ivs[2];
64	54
@@ -70,8 +60,9 @@ static void twofish_dec_blk_cbc_3way(void ctx, u128 dst, const u128 *src) · 5 unmodified lines
70	60	u128_xor(&dst[1], &dst[1], &ivs[0]);
71	61	u128_xor(&dst[2], &dst[2], &ivs[1]);
72	62	}
	63	+EXPORT_SYMBOL_GPL(twofish_dec_blk_cbc_3way);
73	64
74		−static void twofish_enc_blk_ctr(void ctx, u128 dst, const u128 src, u128 iv)
	65	+void twofish_enc_blk_ctr(void ctx, u128 dst, const u128 src, u128 iv)
75	66	{
76	67	be128 ctrblk;
77	68
@@ -84,8 +75,9 @@ static void twofish_enc_blk_ctr(void ctx, u128 dst, const u128 src, u128 iv) · 6 unmodified lines
84	75	twofish_enc_blk(ctx, (u8 )&ctrblk, (u8 )&ctrblk);
85	76	u128_xor(dst, dst, (u128 *)&ctrblk);
86	77	}
	78	+EXPORT_SYMBOL_GPL(twofish_enc_blk_ctr);
87	79
88		−static void twofish_enc_blk_ctr_3way(void ctx, u128 dst, const u128 *src,
	80	+void twofish_enc_blk_ctr_3way(void ctx, u128 dst, const u128 *src,
89	81	u128 *iv)
90	82	{
91	83	be128 ctrblks[3];
@@ -105,6 +97,7 @@ static void twofish_enc_blk_ctr_3way(void ctx, u128 dst, const u128 *src, · 13 unmodified lines
105	97
106	98	twofish_enc_blk_xor_3way(ctx, (u8 )dst, (u8 )ctrblks);
107	99	}
	100	+EXPORT_SYMBOL_GPL(twofish_enc_blk_ctr_3way);
108	101
109	102	static const struct common_glue_ctx twofish_enc = {
110	103	.num_funcs = 2,
@@ -220,13 +213,8 @@ static void decrypt_callback(void priv, u8 srcdst, unsigned int nbytes) · 109 unmodified lines
220	213	twofish_dec_blk(ctx, srcdst, srcdst);
221	214	}
222	215
223		−struct twofish_lrw_ctx {
224		− struct lrw_table_ctx lrw_table;
225		− struct twofish_ctx twofish_ctx;
226		−};
227		−
228		−static int lrw_twofish_setkey(struct crypto_tfm tfm, const u8 key,
229		− unsigned int keylen)
	216	+int lrw_twofish_setkey(struct crypto_tfm tfm, const u8 key,
	217	+ unsigned int keylen)
230	218	{
231	219	struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
232	220	int err;
@@ -238,6 +226,7 @@ static int lrw_twofish_setkey(struct crypto_tfm tfm, const u8 key, · 5 unmodified lines
238	226
239	227	return lrw_init_table(&ctx->lrw_table, key + keylen - TF_BLOCK_SIZE);
240	228	}
	229	+EXPORT_SYMBOL_GPL(lrw_twofish_setkey);
241	230
242	231	static int lrw_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
243	232	struct scatterlist *src, unsigned int nbytes)
@@ -273,20 +262,16 @@ static int lrw_decrypt(struct blkcipher_desc desc, struct scatterlist dst, · 29 unmodified lines
273	262	return lrw_crypt(desc, dst, src, nbytes, &req);
274	263	}
275	264
276		−static void lrw_exit_tfm(struct crypto_tfm *tfm)
	265	+void lrw_twofish_exit_tfm(struct crypto_tfm *tfm)
277	266	{
278	267	struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
279	268
280	269	lrw_free_table(&ctx->lrw_table);
281	270	}
	271	+EXPORT_SYMBOL_GPL(lrw_twofish_exit_tfm);
282	272
283		−struct twofish_xts_ctx {
284		− struct twofish_ctx tweak_ctx;
285		− struct twofish_ctx crypt_ctx;
286		−};
287		−
288		−static int xts_twofish_setkey(struct crypto_tfm tfm, const u8 key,
289		− unsigned int keylen)
	273	+int xts_twofish_setkey(struct crypto_tfm tfm, const u8 key,
	274	+ unsigned int keylen)
290	275	{
291	276	struct twofish_xts_ctx *ctx = crypto_tfm_ctx(tfm);
292	277	u32 *flags = &tfm->crt_flags;
@@ -309,6 +294,7 @@ static int xts_twofish_setkey(struct crypto_tfm tfm, const u8 key, · 16 unmodified lines
309	294	return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
310	295	flags);
311	296	}
	297	+EXPORT_SYMBOL_GPL(xts_twofish_setkey);
312	298
313	299	static int xts_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
314	300	struct scatterlist *src, unsigned int nbytes)
@@ -419,7 +405,7 @@ static struct crypto_alg tf_algs[5] = { { · 104 unmodified lines
419	405	.cra_type = &crypto_blkcipher_type,
420	406	.cra_module = THIS_MODULE,
421	407	.cra_list = LIST_HEAD_INIT(tf_algs[3].cra_list),
422		− .cra_exit = lrw_exit_tfm,
	408	+ .cra_exit = lrw_twofish_exit_tfm,
423	409	.cra_u = {
424	410	.blkcipher = {
425	411	.min_keysize = TF_MIN_KEY_SIZE + TF_BLOCK_SIZE,

arch/x86/include/asm/crypto/twofish.h+46−0

@@ -0,0 +1,46 @@
	1	+#ifndef ASM_X86_TWOFISH_H
	2	+#define ASM_X86_TWOFISH_H
	3	+
	4	+#include <linux/crypto.h>
	5	+#include <crypto/twofish.h>
	6	+#include <crypto/lrw.h>
	7	+#include <crypto/b128ops.h>
	8	+
	9	+struct twofish_lrw_ctx {
	10	+ struct lrw_table_ctx lrw_table;
	11	+ struct twofish_ctx twofish_ctx;
	12	+};
	13	+
	14	+struct twofish_xts_ctx {
	15	+ struct twofish_ctx tweak_ctx;
	16	+ struct twofish_ctx crypt_ctx;
	17	+};
	18	+
	19	+/* regular block cipher functions from twofish_x86_64 module */
	20	+asmlinkage void twofish_enc_blk(struct twofish_ctx ctx, u8 dst,
	21	+ const u8 *src);
	22	+asmlinkage void twofish_dec_blk(struct twofish_ctx ctx, u8 dst,
	23	+ const u8 *src);
	24	+
	25	+/* 3-way parallel cipher functions */
	26	+asmlinkage void __twofish_enc_blk_3way(struct twofish_ctx ctx, u8 dst,
	27	+ const u8 *src, bool xor);
	28	+asmlinkage void twofish_dec_blk_3way(struct twofish_ctx ctx, u8 dst,
	29	+ const u8 *src);
	30	+
	31	+/* helpers from twofish_x86_64-3way module */
	32	+extern void twofish_dec_blk_cbc_3way(void ctx, u128 dst, const u128 *src);
	33	+extern void twofish_enc_blk_ctr(void ctx, u128 dst, const u128 *src,
	34	+ u128 *iv);
	35	+extern void twofish_enc_blk_ctr_3way(void ctx, u128 dst, const u128 *src,
	36	+ u128 *iv);
	37	+
	38	+extern int lrw_twofish_setkey(struct crypto_tfm tfm, const u8 key,
	39	+ unsigned int keylen);
	40	+
	41	+extern void lrw_twofish_exit_tfm(struct crypto_tfm *tfm);
	42	+
	43	+extern int xts_twofish_setkey(struct crypto_tfm tfm, const u8 key,
	44	+ unsigned int keylen);
	45	+
	46	+#endif /* ASM_X86_TWOFISH_H */

crypto/Kconfig+1−0

@@ -958,6 +958,7 @@ config CRYPTO_TWOFISH_AVX_X86_64 · 957 unmodified lines
958	958	select CRYPTO_ALGAPI
959	959	select CRYPTO_CRYPTD
960	960	select CRYPTO_ABLK_HELPER_X86
	961	+ select CRYPTO_GLUE_HELPER_X86
961	962	select CRYPTO_TWOFISH_COMMON
962	963	select CRYPTO_TWOFISH_X86_64
963	964	select CRYPTO_TWOFISH_X86_64_3WAY