lael-labs/linux

Documentation/DocBook/crypto-API.tmpl+2−2

@@ -440,8 +440,8 @@ · 439 unmodified lines
440	The type flag specifies the type of the cipher algorithm.	440	The type flag specifies the type of the cipher algorithm.
441	The caller usually provides a 0 when the caller wants the	441	The caller usually provides a 0 when the caller wants the
442	default handling. Otherwise, the caller may provide the	442	default handling. Otherwise, the caller may provide the
443	following selections which match the the aforementioned	443	following selections which match the aforementioned cipher
444	cipher types:	444	types:
445	</para>	445	</para>
446		446
447	<itemizedlist>	447	<itemizedlist>

Documentation/crypto/asymmetric-keys.txt+1−1

@@ -76,7 +76,7 @@ the criterion string: · 75 unmodified lines
76	Looking in /proc/keys, the last 8 hex digits of the key fingerprint are	76	Looking in /proc/keys, the last 8 hex digits of the key fingerprint are
77	displayed, along with the subtype:	77	displayed, along with the subtype:
78		78
79	1a39e171 I----- 1 perm 3f010000 0 0 asymmetri modsign.0: DSA 5acc2142 []	79	1a39e171 I----- 1 perm 3f010000 0 0 asymmetric modsign.0: DSA 5acc2142 []
80		80
81		81
82	=========================	82	=========================

Documentation/devicetree/bindings/rng/brcm,bcm2835.txt+7−1

 Required properties:
-- compatible : should be "brcm,bcm2835-rng"
+- compatible : should be "brcm,bcm2835-rng"  or "brcm,bcm-nsp-rng" or
+  "brcm,bcm5301x-rng"
 - reg : Specifies base physical address and size of the registers.
 Example:
         compatible = "brcm,bcm2835-rng";
         reg = <0x7e104000 0x10>;
 };
+rng@18033000 {
+	compatible = "brcm,bcm-nsp-rng";
+	reg = <0x18033000 0x14>;
+};

MAINTAINERS+4−1

@@ -3286,6 +3286,7 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/herbert/cryptodev-2.6.git · 3285 unmodified lines
3286	T: git git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6.git	3286	T: git git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6.git
3287	S: Maintained	3287	S: Maintained
3288	F: Documentation/crypto/	3288	F: Documentation/crypto/
		3289	F: Documentation/devicetree/bindings/crypto/
3289	F: Documentation/DocBook/crypto-API.tmpl	3290	F: Documentation/DocBook/crypto-API.tmpl
3290	F: arch/*/crypto/	3291	F: arch/*/crypto/
3291	F: crypto/	3292	F: crypto/
@@ -5273,6 +5274,7 @@ M: Matt Mackall <mpm@selenic.com> · 1981 unmodified lines
5273	M: Herbert Xu <herbert@gondor.apana.org.au>	5274	M: Herbert Xu <herbert@gondor.apana.org.au>
5274	L: linux-crypto@vger.kernel.org	5275	L: linux-crypto@vger.kernel.org
5275	S: Odd fixes	5276	S: Odd fixes
		5277	F: Documentation/devicetree/bindings/rng/
5276	F: Documentation/hw_random.txt	5278	F: Documentation/hw_random.txt
5277	F: drivers/char/hw_random/	5279	F: drivers/char/hw_random/
5278	F: include/linux/hw_random.h	5280	F: include/linux/hw_random.h
@@ -9318,7 +9320,8 @@ L: rtc-linux@googlegroups.com · 4039 unmodified lines
9318	S: Maintained	9320	S: Maintained
9319		9321
9320	QAT DRIVER	9322	QAT DRIVER
9321	M: Tadeusz Struk <tadeusz.struk@intel.com>	9323	M: Giovanni Cabiddu <giovanni.cabiddu@intel.com>
		9324	M: Salvatore Benedetto <salvatore.benedetto@intel.com>
9322	L: qat-linux@intel.com	9325	L: qat-linux@intel.com
9323	S: Supported	9326	S: Supported
9324	F: drivers/crypto/qat/	9327	F: drivers/crypto/qat/

arch/arm/boot/dts/bcm-nsp.dtsi+5−0

 			brcm,nand-has-wp;
 		};
+		rng: rng@33000 {
+			compatible = "brcm,bcm-nsp-rng";
+			reg = <0x33000 0x14>;
+		};
 		ccbtimer0: timer@34000 {
 			compatible = "arm,sp804";
 			reg = <0x34000 0x1000>;

arch/arm/crypto/ghash-ce-glue.c+17−23

 	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
 	struct ahash_request *cryptd_req = ahash_request_ctx(req);
 	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
+	struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
+	struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
-	if (!may_use_simd()) {
+	desc->tfm = child;
-		memcpy(cryptd_req, req, sizeof(*req));
+	desc->flags = req->base.flags;
-		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
+	return crypto_shash_init(desc);
-		return crypto_ahash_init(cryptd_req);
-	} else {
-		struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
-		struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
-		desc->tfm = child;
-		desc->flags = req->base.flags;
-		return crypto_shash_init(desc);
-	}
 }
 static int ghash_async_update(struct ahash_request *req)
 {
 	struct ahash_request *cryptd_req = ahash_request_ctx(req);
+	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
+	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
-	if (!may_use_simd()) {
+	if (!may_use_simd() ||
-		struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+	    (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
-		struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
-		struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
 		memcpy(cryptd_req, req, sizeof(*req));
 		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
 		return crypto_ahash_update(cryptd_req);
 static int ghash_async_final(struct ahash_request *req)
 {
 	struct ahash_request *cryptd_req = ahash_request_ctx(req);
+	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
+	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
-	if (!may_use_simd()) {
+	if (!may_use_simd() ||
-		struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+	    (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
-		struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
-		struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
 		memcpy(cryptd_req, req, sizeof(*req));
 		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
 		return crypto_ahash_final(cryptd_req);
 	struct ahash_request *cryptd_req = ahash_request_ctx(req);
 	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
-	if (!may_use_simd()) {
+	if (!may_use_simd() ||
+	    (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
 		memcpy(cryptd_req, req, sizeof(*req));
 		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
 		return crypto_ahash_digest(cryptd_req);

arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts+4−0

@@ -49,6 +49,10 @@ · 48 unmodified lines
49		49
50	/ {	50	/ {
51	model = "LS1043A RDB Board";	51	model = "LS1043A RDB Board";
		52
		53	aliases {
		54	crypto = &crypto;
		55	};
52	};	56	};
53		57
54	&i2c0 {	58	&i2c0 {

arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi+43−0

@@ -159,6 +159,49 @@ · 158 unmodified lines
159	big-endian;	159	big-endian;
160	};	160	};
161		161
		162	crypto: crypto@1700000 {
		163	compatible = "fsl,sec-v5.4", "fsl,sec-v5.0",
		164	"fsl,sec-v4.0";
		165	fsl,sec-era = <3>;
		166	#address-cells = <1>;
		167	#size-cells = <1>;
		168	ranges = <0x0 0x00 0x1700000 0x100000>;
		169	reg = <0x00 0x1700000 0x0 0x100000>;
		170	interrupts = <0 75 0x4>;
		171
		172	sec_jr0: jr@10000 {
		173	compatible = "fsl,sec-v5.4-job-ring",
		174	"fsl,sec-v5.0-job-ring",
		175	"fsl,sec-v4.0-job-ring";
		176	reg = <0x10000 0x10000>;
		177	interrupts = <0 71 0x4>;
		178	};
		179
		180	sec_jr1: jr@20000 {
		181	compatible = "fsl,sec-v5.4-job-ring",
		182	"fsl,sec-v5.0-job-ring",
		183	"fsl,sec-v4.0-job-ring";
		184	reg = <0x20000 0x10000>;
		185	interrupts = <0 72 0x4>;
		186	};
		187
		188	sec_jr2: jr@30000 {
		189	compatible = "fsl,sec-v5.4-job-ring",
		190	"fsl,sec-v5.0-job-ring",
		191	"fsl,sec-v4.0-job-ring";
		192	reg = <0x30000 0x10000>;
		193	interrupts = <0 73 0x4>;
		194	};
		195
		196	sec_jr3: jr@40000 {
		197	compatible = "fsl,sec-v5.4-job-ring",
		198	"fsl,sec-v5.0-job-ring",
		199	"fsl,sec-v4.0-job-ring";
		200	reg = <0x40000 0x10000>;
		201	interrupts = <0 74 0x4>;
		202	};
		203	};
		204
162	dcfg: dcfg@1ee0000 {	205	dcfg: dcfg@1ee0000 {
163	compatible = "fsl,ls1043a-dcfg", "syscon";	206	compatible = "fsl,ls1043a-dcfg", "syscon";
164	reg = <0x0 0x1ee0000 0x0 0x10000>;	207	reg = <0x0 0x1ee0000 0x0 0x10000>;

arch/arm64/include/asm/io.h+3−1

 #define iounmap				__iounmap
 /*
- * io{read,write}{16,32}be() macros
+ * io{read,write}{16,32,64}be() macros
  */
 #define ioread16be(p)		({ __u16 __v = be16_to_cpu((__force __be16)__raw_readw(p)); __iormb(); __v; })
 #define ioread32be(p)		({ __u32 __v = be32_to_cpu((__force __be32)__raw_readl(p)); __iormb(); __v; })
+#define ioread64be(p)		({ __u64 __v = be64_to_cpu((__force __be64)__raw_readq(p)); __iormb(); __v; })
 #define iowrite16be(v,p)	({ __iowmb(); __raw_writew((__force __u16)cpu_to_be16(v), p); })
 #define iowrite32be(v,p)	({ __iowmb(); __raw_writel((__force __u32)cpu_to_be32(v), p); })
+#define iowrite64be(v,p)	({ __iowmb(); __raw_writeq((__force __u64)cpu_to_be64(v), p); })
 /*
  * Convert a physical pointer to a virtual kernel pointer for /dev/mem

arch/powerpc/crypto/Makefile+2−0

 obj-$(CONFIG_CRYPTO_SHA1_PPC) += sha1-powerpc.o
 obj-$(CONFIG_CRYPTO_SHA1_PPC_SPE) += sha1-ppc-spe.o
 obj-$(CONFIG_CRYPTO_SHA256_PPC_SPE) += sha256-ppc-spe.o
+obj-$(CONFIG_CRYPT_CRC32C_VPMSUM) += crc32c-vpmsum.o
 aes-ppc-spe-y := aes-spe-core.o aes-spe-keys.o aes-tab-4k.o aes-spe-modes.o aes-spe-glue.o
 md5-ppc-y := md5-asm.o md5-glue.o
 sha1-powerpc-y := sha1-powerpc-asm.o sha1.o
 sha1-ppc-spe-y := sha1-spe-asm.o sha1-spe-glue.o
 sha256-ppc-spe-y := sha256-spe-asm.o sha256-spe-glue.o
+crc32c-vpmsum-y := crc32c-vpmsum_asm.o crc32c-vpmsum_glue.o

arch/powerpc/crypto/aes-spe-regs.h+1−1

@@ -18,7 +18,7 @@ · 17 unmodified lines
18	#define rLN r7 /* length of data to be processed */	18	#define rLN r7 /* length of data to be processed */
19	#define rIP r8 /* potiner to IV (CBC/CTR/XTS modes) */	19	#define rIP r8 /* potiner to IV (CBC/CTR/XTS modes) */
20	#define rKT r9 /* pointer to tweak key (XTS mode) */	20	#define rKT r9 /* pointer to tweak key (XTS mode) */
21	#define rT0 r11 /* pointers to en-/decrpytion tables */	21	#define rT0 r11 /* pointers to en-/decryption tables */
22	#define rT1 r10	22	#define rT1 r10
23	#define rD0 r9 /* data */	23	#define rD0 r9 /* data */
24	#define rD1 r14	24	#define rD1 r14

arch/powerpc/crypto/crc32c-vpmsum_asm.S+1553−0

@@ -0,0 +1,1553 @@
	1	/*
	2	* Calculate the checksum of data that is 16 byte aligned and a multiple of
	3	* 16 bytes.
	4	*
	5	* The first step is to reduce it to 1024 bits. We do this in 8 parallel
	6	* chunks in order to mask the latency of the vpmsum instructions. If we
	7	* have more than 32 kB of data to checksum we repeat this step multiple
	8	* times, passing in the previous 1024 bits.
	9	*
	10	* The next step is to reduce the 1024 bits to 64 bits. This step adds
	11	* 32 bits of 0s to the end - this matches what a CRC does. We just
	12	* calculate constants that land the data in this 32 bits.
	13	*
	14	* We then use fixed point Barrett reduction to compute a mod n over GF(2)
	15	* for n = CRC using POWER8 instructions. We use x = 32.
	16	*
	17	* http://en.wikipedia.org/wiki/Barrett_reduction
	18	*
	19	* Copyright (C) 2015 Anton Blanchard <anton@au.ibm.com>, IBM
	20	*
	21	* This program is free software; you can redistribute it and/or
	22	* modify it under the terms of the GNU General Public License
	23	* as published by the Free Software Foundation; either version
	24	* 2 of the License, or (at your option) any later version.
	25	*/
	26	#include <asm/ppc_asm.h>
	27	#include <asm/ppc-opcode.h>
	28
	29	.section .rodata
	30	.balign 16
	31
	32	.byteswap_constant:
	33	/* byte reverse permute constant */
	34	.octa 0x0F0E0D0C0B0A09080706050403020100
	35
	36	#define MAX_SIZE 32768
	37	.constants:
	38
	39	/* Reduce 262144 kbits to 1024 bits */
	40	/* x^261120 mod p(x)` << 1, x^261184 mod p(x)` << 1 */
	41	.octa 0x00000000b6ca9e20000000009c37c408
	42
	43	/* x^260096 mod p(x)` << 1, x^260160 mod p(x)` << 1 */
	44	.octa 0x00000000350249a800000001b51df26c
	45
	46	/* x^259072 mod p(x)` << 1, x^259136 mod p(x)` << 1 */
	47	.octa 0x00000001862dac54000000000724b9d0
	48
	49	/* x^258048 mod p(x)` << 1, x^258112 mod p(x)` << 1 */
	50	.octa 0x00000001d87fb48c00000001c00532fe
	51
	52	/* x^257024 mod p(x)` << 1, x^257088 mod p(x)` << 1 */
	53	.octa 0x00000001f39b699e00000000f05a9362
	54
	55	/* x^256000 mod p(x)` << 1, x^256064 mod p(x)` << 1 */
	56	.octa 0x0000000101da11b400000001e1007970
	57
	58	/* x^254976 mod p(x)` << 1, x^255040 mod p(x)` << 1 */
	59	.octa 0x00000001cab571e000000000a57366ee
	60
	61	/* x^253952 mod p(x)` << 1, x^254016 mod p(x)` << 1 */
	62	.octa 0x00000000c7020cfe0000000192011284
	63
	64	/* x^252928 mod p(x)` << 1, x^252992 mod p(x)` << 1 */
	65	.octa 0x00000000cdaed1ae0000000162716d9a
	66
	67	/* x^251904 mod p(x)` << 1, x^251968 mod p(x)` << 1 */
	68	.octa 0x00000001e804effc00000000cd97ecde
	69
	70	/* x^250880 mod p(x)` << 1, x^250944 mod p(x)` << 1 */
	71	.octa 0x0000000077c3ea3a0000000058812bc0
	72
	73	/* x^249856 mod p(x)` << 1, x^249920 mod p(x)` << 1 */
	74	.octa 0x0000000068df31b40000000088b8c12e
	75
	76	/* x^248832 mod p(x)` << 1, x^248896 mod p(x)` << 1 */
	77	.octa 0x00000000b059b6c200000001230b234c
	78
	79	/* x^247808 mod p(x)` << 1, x^247872 mod p(x)` << 1 */
	80	.octa 0x0000000145fb8ed800000001120b416e
	81
	82	/* x^246784 mod p(x)` << 1, x^246848 mod p(x)` << 1 */
	83	.octa 0x00000000cbc0916800000001974aecb0
	84
	85	/* x^245760 mod p(x)` << 1, x^245824 mod p(x)` << 1 */
	86	.octa 0x000000005ceeedc2000000008ee3f226
	87
	88	/* x^244736 mod p(x)` << 1, x^244800 mod p(x)` << 1 */
	89	.octa 0x0000000047d74e8600000001089aba9a
	90
	91	/* x^243712 mod p(x)` << 1, x^243776 mod p(x)` << 1 */
	92	.octa 0x00000001407e9e220000000065113872
	93
	94	/* x^242688 mod p(x)` << 1, x^242752 mod p(x)` << 1 */
	95	.octa 0x00000001da967bda000000005c07ec10
	96
	97	/* x^241664 mod p(x)` << 1, x^241728 mod p(x)` << 1 */
	98	.octa 0x000000006c8983680000000187590924
	99
	100	/* x^240640 mod p(x)` << 1, x^240704 mod p(x)` << 1 */
	101	.octa 0x00000000f2d14c9800000000e35da7c6
	102
	103	/* x^239616 mod p(x)` << 1, x^239680 mod p(x)` << 1 */
	104	.octa 0x00000001993c6ad4000000000415855a
	105
	106	/* x^238592 mod p(x)` << 1, x^238656 mod p(x)` << 1 */
	107	.octa 0x000000014683d1ac0000000073617758
	108
	109	/* x^237568 mod p(x)` << 1, x^237632 mod p(x)` << 1 */
	110	.octa 0x00000001a7c93e6c0000000176021d28
	111
	112	/* x^236544 mod p(x)` << 1, x^236608 mod p(x)` << 1 */
	113	.octa 0x000000010211e90a00000001c358fd0a
	114
	115	/* x^235520 mod p(x)` << 1, x^235584 mod p(x)` << 1 */
	116	.octa 0x000000001119403e00000001ff7a2c18
	117
	118	/* x^234496 mod p(x)` << 1, x^234560 mod p(x)` << 1 */
	119	.octa 0x000000001c3261aa00000000f2d9f7e4
	120
	121	/* x^233472 mod p(x)` << 1, x^233536 mod p(x)` << 1 */
	122	.octa 0x000000014e37a634000000016cf1f9c8
	123
	124	/* x^232448 mod p(x)` << 1, x^232512 mod p(x)` << 1 */
	125	.octa 0x0000000073786c0c000000010af9279a
	126
	127	/* x^231424 mod p(x)` << 1, x^231488 mod p(x)` << 1 */
	128	.octa 0x000000011dc037f80000000004f101e8
	129
	130	/* x^230400 mod p(x)` << 1, x^230464 mod p(x)` << 1 */
	131	.octa 0x0000000031433dfc0000000070bcf184
	132
	133	/* x^229376 mod p(x)` << 1, x^229440 mod p(x)` << 1 */
	134	.octa 0x000000009cde8348000000000a8de642
	135
	136	/* x^228352 mod p(x)` << 1, x^228416 mod p(x)` << 1 */
	137	.octa 0x0000000038d3c2a60000000062ea130c
	138
	139	/* x^227328 mod p(x)` << 1, x^227392 mod p(x)` << 1 */
	140	.octa 0x000000011b25f26000000001eb31cbb2
	141
	142	/* x^226304 mod p(x)` << 1, x^226368 mod p(x)` << 1 */
	143	.octa 0x000000001629e6f00000000170783448
	144
	145	/* x^225280 mod p(x)` << 1, x^225344 mod p(x)` << 1 */
	146	.octa 0x0000000160838b4c00000001a684b4c6
	147
	148	/* x^224256 mod p(x)` << 1, x^224320 mod p(x)` << 1 */
	149	.octa 0x000000007a44011c00000000253ca5b4
	150
	151	/* x^223232 mod p(x)` << 1, x^223296 mod p(x)` << 1 */
	152	.octa 0x00000000226f417a0000000057b4b1e2
	153
	154	/* x^222208 mod p(x)` << 1, x^222272 mod p(x)` << 1 */
	155	.octa 0x0000000045eb2eb400000000b6bd084c
	156
	157	/* x^221184 mod p(x)` << 1, x^221248 mod p(x)` << 1 */
	158	.octa 0x000000014459d70c0000000123c2d592
	159
	160	/* x^220160 mod p(x)` << 1, x^220224 mod p(x)` << 1 */
	161	.octa 0x00000001d406ed8200000000159dafce
	162
	163	/* x^219136 mod p(x)` << 1, x^219200 mod p(x)` << 1 */
	164	.octa 0x0000000160c8e1a80000000127e1a64e
	165
	166	/* x^218112 mod p(x)` << 1, x^218176 mod p(x)` << 1 */
	167	.octa 0x0000000027ba80980000000056860754
	168
	169	/* x^217088 mod p(x)` << 1, x^217152 mod p(x)` << 1 */
	170	.octa 0x000000006d92d01800000001e661aae8
	171
	172	/* x^216064 mod p(x)` << 1, x^216128 mod p(x)` << 1 */
	173	.octa 0x000000012ed7e3f200000000f82c6166
	174
	175	/* x^215040 mod p(x)` << 1, x^215104 mod p(x)` << 1 */
	176	.octa 0x000000002dc8778800000000c4f9c7ae
	177
	178	/* x^214016 mod p(x)` << 1, x^214080 mod p(x)` << 1 */
	179	.octa 0x0000000018240bb80000000074203d20
	180
	181	/* x^212992 mod p(x)` << 1, x^213056 mod p(x)` << 1 */
	182	.octa 0x000000001ad381580000000198173052
	183
	184	/* x^211968 mod p(x)` << 1, x^212032 mod p(x)` << 1 */
	185	.octa 0x00000001396b78f200000001ce8aba54
	186
	187	/* x^210944 mod p(x)` << 1, x^211008 mod p(x)` << 1 */
	188	.octa 0x000000011a68133400000001850d5d94
	189
	190	/* x^209920 mod p(x)` << 1, x^209984 mod p(x)` << 1 */
	191	.octa 0x000000012104732e00000001d609239c
	192
	193	/* x^208896 mod p(x)` << 1, x^208960 mod p(x)` << 1 */
	194	.octa 0x00000000a140d90c000000001595f048
	195
	196	/* x^207872 mod p(x)` << 1, x^207936 mod p(x)` << 1 */
	197	.octa 0x00000001b7215eda0000000042ccee08
	198
	199	/* x^206848 mod p(x)` << 1, x^206912 mod p(x)` << 1 */
	200	.octa 0x00000001aaf1df3c000000010a389d74
	201
	202	/* x^205824 mod p(x)` << 1, x^205888 mod p(x)` << 1 */
	203	.octa 0x0000000029d15b8a000000012a840da6
	204
	205	/* x^204800 mod p(x)` << 1, x^204864 mod p(x)` << 1 */
	206	.octa 0x00000000f1a96922000000001d181c0c
	207
	208	/* x^203776 mod p(x)` << 1, x^203840 mod p(x)` << 1 */
	209	.octa 0x00000001ac80d03c0000000068b7d1f6
	210
	211	/* x^202752 mod p(x)` << 1, x^202816 mod p(x)` << 1 */
	212	.octa 0x000000000f11d56a000000005b0f14fc
	213
	214	/* x^201728 mod p(x)` << 1, x^201792 mod p(x)` << 1 */
	215	.octa 0x00000001f1c022a20000000179e9e730
	216
	217	/* x^200704 mod p(x)` << 1, x^200768 mod p(x)` << 1 */
	218	.octa 0x0000000173d00ae200000001ce1368d6
	219
	220	/* x^199680 mod p(x)` << 1, x^199744 mod p(x)` << 1 */
	221	.octa 0x00000001d4ffe4ac0000000112c3a84c
	222
	223	/* x^198656 mod p(x)` << 1, x^198720 mod p(x)` << 1 */
	224	.octa 0x000000016edc5ae400000000de940fee
	225
	226	/* x^197632 mod p(x)` << 1, x^197696 mod p(x)` << 1 */
	227	.octa 0x00000001f1a0214000000000fe896b7e
	228
	229	/* x^196608 mod p(x)` << 1, x^196672 mod p(x)` << 1 */
	230	.octa 0x00000000ca0b28a000000001f797431c
	231
	232	/* x^195584 mod p(x)` << 1, x^195648 mod p(x)` << 1 */
	233	.octa 0x00000001928e30a20000000053e989ba
	234
	235	/* x^194560 mod p(x)` << 1, x^194624 mod p(x)` << 1 */
	236	.octa 0x0000000097b1b002000000003920cd16
	237
	238	/* x^193536 mod p(x)` << 1, x^193600 mod p(x)` << 1 */
	239	.octa 0x00000000b15bf90600000001e6f579b8
	240
	241	/* x^192512 mod p(x)` << 1, x^192576 mod p(x)` << 1 */
	242	.octa 0x00000000411c5d52000000007493cb0a
	243
	244	/* x^191488 mod p(x)` << 1, x^191552 mod p(x)` << 1 */
	245	.octa 0x00000001c36f330000000001bdd376d8
	246
	247	/* x^190464 mod p(x)` << 1, x^190528 mod p(x)` << 1 */
	248	.octa 0x00000001119227e0000000016badfee6
	249
	250	/* x^189440 mod p(x)` << 1, x^189504 mod p(x)` << 1 */
	251	.octa 0x00000000114d47020000000071de5c58
	252
	253	/* x^188416 mod p(x)` << 1, x^188480 mod p(x)` << 1 */
	254	.octa 0x00000000458b5b9800000000453f317c
	255
	256	/* x^187392 mod p(x)` << 1, x^187456 mod p(x)` << 1 */
	257	.octa 0x000000012e31fb8e0000000121675cce
	258
	259	/* x^186368 mod p(x)` << 1, x^186432 mod p(x)` << 1 */
	260	.octa 0x000000005cf619d800000001f409ee92
	261
	262	/* x^185344 mod p(x)` << 1, x^185408 mod p(x)` << 1 */
	263	.octa 0x0000000063f4d8b200000000f36b9c88
	264
	265	/* x^184320 mod p(x)` << 1, x^184384 mod p(x)` << 1 */
	266	.octa 0x000000004138dc8a0000000036b398f4
	267
	268	/* x^183296 mod p(x)` << 1, x^183360 mod p(x)` << 1 */
	269	.octa 0x00000001d29ee8e000000001748f9adc
	270
	271	/* x^182272 mod p(x)` << 1, x^182336 mod p(x)` << 1 */
	272	.octa 0x000000006a08ace800000001be94ec00
	273
	274	/* x^181248 mod p(x)` << 1, x^181312 mod p(x)` << 1 */
	275	.octa 0x0000000127d4201000000000b74370d6
	276
	277	/* x^180224 mod p(x)` << 1, x^180288 mod p(x)` << 1 */
	278	.octa 0x0000000019d76b6200000001174d0b98
	279
	280	/* x^179200 mod p(x)` << 1, x^179264 mod p(x)` << 1 */
	281	.octa 0x00000001b1471f6e00000000befc06a4
	282
	283	/* x^178176 mod p(x)` << 1, x^178240 mod p(x)` << 1 */
	284	.octa 0x00000001f64c19cc00000001ae125288
	285
	286	/* x^177152 mod p(x)` << 1, x^177216 mod p(x)` << 1 */
	287	.octa 0x00000000003c0ea00000000095c19b34
	288
	289	/* x^176128 mod p(x)` << 1, x^176192 mod p(x)` << 1 */
	290	.octa 0x000000014d73abf600000001a78496f2
	291
	292	/* x^175104 mod p(x)` << 1, x^175168 mod p(x)` << 1 */
	293	.octa 0x00000001620eb84400000001ac5390a0
	294
	295	/* x^174080 mod p(x)` << 1, x^174144 mod p(x)` << 1 */
	296	.octa 0x0000000147655048000000002a80ed6e
	297
	298	/* x^173056 mod p(x)` << 1, x^173120 mod p(x)` << 1 */
	299	.octa 0x0000000067b5077e00000001fa9b0128
	300
	301	/* x^172032 mod p(x)` << 1, x^172096 mod p(x)` << 1 */
	302	.octa 0x0000000010ffe20600000001ea94929e
	303
	304	/* x^171008 mod p(x)` << 1, x^171072 mod p(x)` << 1 */
	305	.octa 0x000000000fee8f1e0000000125f4305c
	306
	307	/* x^169984 mod p(x)` << 1, x^170048 mod p(x)` << 1 */
	308	.octa 0x00000001da26fbae00000001471e2002
	309
	310	/* x^168960 mod p(x)` << 1, x^169024 mod p(x)` << 1 */
	311	.octa 0x00000001b3a8bd880000000132d2253a
	312
	313	/* x^167936 mod p(x)` << 1, x^168000 mod p(x)` << 1 */
	314	.octa 0x00000000e8f3898e00000000f26b3592
	315
	316	/* x^166912 mod p(x)` << 1, x^166976 mod p(x)` << 1 */
	317	.octa 0x00000000b0d0d28c00000000bc8b67b0
	318
	319	/* x^165888 mod p(x)` << 1, x^165952 mod p(x)` << 1 */
	320	.octa 0x0000000030f2a798000000013a826ef2
	321
	322	/* x^164864 mod p(x)` << 1, x^164928 mod p(x)` << 1 */
	323	.octa 0x000000000fba10020000000081482c84
	324
	325	/* x^163840 mod p(x)` << 1, x^163904 mod p(x)` << 1 */
	326	.octa 0x00000000bdb9bd7200000000e77307c2
	327
	328	/* x^162816 mod p(x)` << 1, x^162880 mod p(x)` << 1 */
	329	.octa 0x0000000075d3bf5a00000000d4a07ec8
	330
	331	/* x^161792 mod p(x)` << 1, x^161856 mod p(x)` << 1 */
	332	.octa 0x00000000ef1f98a00000000017102100
	333
	334	/* x^160768 mod p(x)` << 1, x^160832 mod p(x)` << 1 */
	335	.octa 0x00000000689c760200000000db406486
	336
	337	/* x^159744 mod p(x)` << 1, x^159808 mod p(x)` << 1 */
	338	.octa 0x000000016d5fa5fe0000000192db7f88
	339
	340	/* x^158720 mod p(x)` << 1, x^158784 mod p(x)` << 1 */
	341	.octa 0x00000001d0d2b9ca000000018bf67b1e
	342
	343	/* x^157696 mod p(x)` << 1, x^157760 mod p(x)` << 1 */
	344	.octa 0x0000000041e7b470000000007c09163e
	345
	346	/* x^156672 mod p(x)` << 1, x^156736 mod p(x)` << 1 */
	347	.octa 0x00000001cbb6495e000000000adac060
	348
	349	/* x^155648 mod p(x)` << 1, x^155712 mod p(x)` << 1 */
	350	.octa 0x000000010052a0b000000000bd8316ae
	351
	352	/* x^154624 mod p(x)` << 1, x^154688 mod p(x)` << 1 */
	353	.octa 0x00000001d8effb5c000000019f09ab54
	354
	355	/* x^153600 mod p(x)` << 1, x^153664 mod p(x)` << 1 */
	356	.octa 0x00000001d969853c0000000125155542
	357
	358	/* x^152576 mod p(x)` << 1, x^152640 mod p(x)` << 1 */
	359	.octa 0x00000000523ccce2000000018fdb5882
	360
	361	/* x^151552 mod p(x)` << 1, x^151616 mod p(x)` << 1 */
	362	.octa 0x000000001e2436bc00000000e794b3f4
	363
	364	/* x^150528 mod p(x)` << 1, x^150592 mod p(x)` << 1 */
	365	.octa 0x00000000ddd1c3a2000000016f9bb022
	366
	367	/* x^149504 mod p(x)` << 1, x^149568 mod p(x)` << 1 */
	368	.octa 0x0000000019fcfe3800000000290c9978
	369
	370	/* x^148480 mod p(x)` << 1, x^148544 mod p(x)` << 1 */
	371	.octa 0x00000001ce95db640000000083c0f350
	372
	373	/* x^147456 mod p(x)` << 1, x^147520 mod p(x)` << 1 */
	374	.octa 0x00000000af5828060000000173ea6628
	375
	376	/* x^146432 mod p(x)` << 1, x^146496 mod p(x)` << 1 */
	377	.octa 0x00000001006388f600000001c8b4e00a
	378
	379	/* x^145408 mod p(x)` << 1, x^145472 mod p(x)` << 1 */
	380	.octa 0x0000000179eca00a00000000de95d6aa
	381
	382	/* x^144384 mod p(x)` << 1, x^144448 mod p(x)` << 1 */
	383	.octa 0x0000000122410a6a000000010b7f7248
	384
	385	/* x^143360 mod p(x)` << 1, x^143424 mod p(x)` << 1 */
	386	.octa 0x000000004288e87c00000001326e3a06
	387
	388	/* x^142336 mod p(x)` << 1, x^142400 mod p(x)` << 1 */
	389	.octa 0x000000016c5490da00000000bb62c2e6
	390
	391	/* x^141312 mod p(x)` << 1, x^141376 mod p(x)` << 1 */
	392	.octa 0x00000000d1c71f6e0000000156a4b2c2
	393
	394	/* x^140288 mod p(x)` << 1, x^140352 mod p(x)` << 1 */
	395	.octa 0x00000001b4ce08a6000000011dfe763a
	396
	397	/* x^139264 mod p(x)` << 1, x^139328 mod p(x)` << 1 */
	398	.octa 0x00000001466ba60c000000007bcca8e2
	399
	400	/* x^138240 mod p(x)` << 1, x^138304 mod p(x)` << 1 */
	401	.octa 0x00000001f6c488a40000000186118faa
	402
	403	/* x^137216 mod p(x)` << 1, x^137280 mod p(x)` << 1 */
	404	.octa 0x000000013bfb06820000000111a65a88
	405
	406	/* x^136192 mod p(x)` << 1, x^136256 mod p(x)` << 1 */
	407	.octa 0x00000000690e9e54000000003565e1c4
	408
	409	/* x^135168 mod p(x)` << 1, x^135232 mod p(x)` << 1 */
	410	.octa 0x00000000281346b6000000012ed02a82
	411
	412	/* x^134144 mod p(x)` << 1, x^134208 mod p(x)` << 1 */
	413	.octa 0x000000015646402400000000c486ecfc
	414
	415	/* x^133120 mod p(x)` << 1, x^133184 mod p(x)` << 1 */
	416	.octa 0x000000016063a8dc0000000001b951b2
	417
	418	/* x^132096 mod p(x)` << 1, x^132160 mod p(x)` << 1 */
	419	.octa 0x0000000116a663620000000048143916
	420
	421	/* x^131072 mod p(x)` << 1, x^131136 mod p(x)` << 1 */
	422	.octa 0x000000017e8aa4d200000001dc2ae124
	423
	424	/* x^130048 mod p(x)` << 1, x^130112 mod p(x)` << 1 */
	425	.octa 0x00000001728eb10c00000001416c58d6
	426
	427	/* x^129024 mod p(x)` << 1, x^129088 mod p(x)` << 1 */
	428	.octa 0x00000001b08fd7fa00000000a479744a
	429
	430	/* x^128000 mod p(x)` << 1, x^128064 mod p(x)` << 1 */
	431	.octa 0x00000001092a16e80000000096ca3a26
	432
	433	/* x^126976 mod p(x)` << 1, x^127040 mod p(x)` << 1 */
	434	.octa 0x00000000a505637c00000000ff223d4e
	435
	436	/* x^125952 mod p(x)` << 1, x^126016 mod p(x)` << 1 */
	437	.octa 0x00000000d94869b2000000010e84da42
	438
	439	/* x^124928 mod p(x)` << 1, x^124992 mod p(x)` << 1 */
	440	.octa 0x00000001c8b203ae00000001b61ba3d0
	441
	442	/* x^123904 mod p(x)` << 1, x^123968 mod p(x)` << 1 */
	443	.octa 0x000000005704aea000000000680f2de8
	444
	445	/* x^122880 mod p(x)` << 1, x^122944 mod p(x)` << 1 */
	446	.octa 0x000000012e295fa2000000008772a9a8
	447
	448	/* x^121856 mod p(x)` << 1, x^121920 mod p(x)` << 1 */
	449	.octa 0x000000011d0908bc0000000155f295bc
	450
	451	/* x^120832 mod p(x)` << 1, x^120896 mod p(x)` << 1 */
	452	.octa 0x0000000193ed97ea00000000595f9282
	453
	454	/* x^119808 mod p(x)` << 1, x^119872 mod p(x)` << 1 */
	455	.octa 0x000000013a0f1c520000000164b1c25a
	456
	457	/* x^118784 mod p(x)` << 1, x^118848 mod p(x)` << 1 */
	458	.octa 0x000000010c2c40c000000000fbd67c50
	459
	460	/* x^117760 mod p(x)` << 1, x^117824 mod p(x)` << 1 */
	461	.octa 0x00000000ff6fac3e0000000096076268
	462
	463	/* x^116736 mod p(x)` << 1, x^116800 mod p(x)` << 1 */
	464	.octa 0x000000017b3609c000000001d288e4cc
	465
	466	/* x^115712 mod p(x)` << 1, x^115776 mod p(x)` << 1 */
	467	.octa 0x0000000088c8c92200000001eaac1bdc
	468
	469	/* x^114688 mod p(x)` << 1, x^114752 mod p(x)` << 1 */
	470	.octa 0x00000001751baae600000001f1ea39e2
	471
	472	/* x^113664 mod p(x)` << 1, x^113728 mod p(x)` << 1 */
	473	.octa 0x000000010795297200000001eb6506fc
	474
	475	/* x^112640 mod p(x)` << 1, x^112704 mod p(x)` << 1 */
	476	.octa 0x0000000162b00abe000000010f806ffe
	477
	478	/* x^111616 mod p(x)` << 1, x^111680 mod p(x)` << 1 */
	479	.octa 0x000000000d7b404c000000010408481e
	480
	481	/* x^110592 mod p(x)` << 1, x^110656 mod p(x)` << 1 */
	482	.octa 0x00000000763b13d40000000188260534
	483
	484	/* x^109568 mod p(x)` << 1, x^109632 mod p(x)` << 1 */
	485	.octa 0x00000000f6dc22d80000000058fc73e0
	486
	487	/* x^108544 mod p(x)` << 1, x^108608 mod p(x)` << 1 */
	488	.octa 0x000000007daae06000000000391c59b8
	489
	490	/* x^107520 mod p(x)` << 1, x^107584 mod p(x)` << 1 */
	491	.octa 0x000000013359ab7c000000018b638400
	492
	493	/* x^106496 mod p(x)` << 1, x^106560 mod p(x)` << 1 */
	494	.octa 0x000000008add438a000000011738f5c4
	495
	496	/* x^105472 mod p(x)` << 1, x^105536 mod p(x)` << 1 */
	497	.octa 0x00000001edbefdea000000008cf7c6da
	498
	499	/* x^104448 mod p(x)` << 1, x^104512 mod p(x)` << 1 */
	500	.octa 0x000000004104e0f800000001ef97fb16
	501
	502	/* x^103424 mod p(x)` << 1, x^103488 mod p(x)` << 1 */
	503	.octa 0x00000000b48a82220000000102130e20
	504
	505	/* x^102400 mod p(x)` << 1, x^102464 mod p(x)` << 1 */
	506	.octa 0x00000001bcb4684400000000db968898
	507
	508	/* x^101376 mod p(x)` << 1, x^101440 mod p(x)` << 1 */
	509	.octa 0x000000013293ce0a00000000b5047b5e
	510
	511	/* x^100352 mod p(x)` << 1, x^100416 mod p(x)` << 1 */
	512	.octa 0x00000001710d0844000000010b90fdb2
	513
	514	/* x^99328 mod p(x)` << 1, x^99392 mod p(x)` << 1 */
	515	.octa 0x0000000117907f6e000000004834a32e
	516
	517	/* x^98304 mod p(x)` << 1, x^98368 mod p(x)` << 1 */
	518	.octa 0x0000000087ddf93e0000000059c8f2b0
	519
	520	/* x^97280 mod p(x)` << 1, x^97344 mod p(x)` << 1 */
	521	.octa 0x000000005970e9b00000000122cec508
	522
	523	/* x^96256 mod p(x)` << 1, x^96320 mod p(x)` << 1 */
	524	.octa 0x0000000185b2b7d0000000000a330cda
	525
	526	/* x^95232 mod p(x)` << 1, x^95296 mod p(x)` << 1 */
	527	.octa 0x00000001dcee0efc000000014a47148c
	528
	529	/* x^94208 mod p(x)` << 1, x^94272 mod p(x)` << 1 */
	530	.octa 0x0000000030da27220000000042c61cb8
	531
	532	/* x^93184 mod p(x)` << 1, x^93248 mod p(x)` << 1 */
	533	.octa 0x000000012f925a180000000012fe6960
	534
	535	/* x^92160 mod p(x)` << 1, x^92224 mod p(x)` << 1 */
	536	.octa 0x00000000dd2e357c00000000dbda2c20
	537
	538	/* x^91136 mod p(x)` << 1, x^91200 mod p(x)` << 1 */
	539	.octa 0x00000000071c80de000000011122410c
	540
	541	/* x^90112 mod p(x)` << 1, x^90176 mod p(x)` << 1 */
	542	.octa 0x000000011513140a00000000977b2070
	543
	544	/* x^89088 mod p(x)` << 1, x^89152 mod p(x)` << 1 */
	545	.octa 0x00000001df876e8e000000014050438e
	546
	547	/* x^88064 mod p(x)` << 1, x^88128 mod p(x)` << 1 */
	548	.octa 0x000000015f81d6ce0000000147c840e8
	549
	550	/* x^87040 mod p(x)` << 1, x^87104 mod p(x)` << 1 */
	551	.octa 0x000000019dd94dbe00000001cc7c88ce
	552
	553	/* x^86016 mod p(x)` << 1, x^86080 mod p(x)` << 1 */
	554	.octa 0x00000001373d206e00000001476b35a4
	555
	556	/* x^84992 mod p(x)` << 1, x^85056 mod p(x)` << 1 */
	557	.octa 0x00000000668ccade000000013d52d508
	558
	559	/* x^83968 mod p(x)` << 1, x^84032 mod p(x)` << 1 */
	560	.octa 0x00000001b192d268000000008e4be32e
	561
	562	/* x^82944 mod p(x)` << 1, x^83008 mod p(x)` << 1 */
	563	.octa 0x00000000e30f3a7800000000024120fe
	564
	565	/* x^81920 mod p(x)` << 1, x^81984 mod p(x)` << 1 */
	566	.octa 0x000000010ef1f7bc00000000ddecddb4
	567
	568	/* x^80896 mod p(x)` << 1, x^80960 mod p(x)` << 1 */
	569	.octa 0x00000001f5ac738000000000d4d403bc
	570
	571	/* x^79872 mod p(x)` << 1, x^79936 mod p(x)` << 1 */
	572	.octa 0x000000011822ea7000000001734b89aa
	573
	574	/* x^78848 mod p(x)` << 1, x^78912 mod p(x)` << 1 */
	575	.octa 0x00000000c3a33848000000010e7a58d6
	576
	577	/* x^77824 mod p(x)` << 1, x^77888 mod p(x)` << 1 */
	578	.octa 0x00000001bd151c2400000001f9f04e9c
	579
	580	/* x^76800 mod p(x)` << 1, x^76864 mod p(x)` << 1 */
	581	.octa 0x0000000056002d7600000000b692225e
	582
	583	/* x^75776 mod p(x)` << 1, x^75840 mod p(x)` << 1 */
	584	.octa 0x000000014657c4f4000000019b8d3f3e
	585
	586	/* x^74752 mod p(x)` << 1, x^74816 mod p(x)` << 1 */
	587	.octa 0x0000000113742d7c00000001a874f11e
	588
	589	/* x^73728 mod p(x)` << 1, x^73792 mod p(x)` << 1 */
	590	.octa 0x000000019c5920ba000000010d5a4254
	591
	592	/* x^72704 mod p(x)` << 1, x^72768 mod p(x)` << 1 */
	593	.octa 0x000000005216d2d600000000bbb2f5d6
	594
	595	/* x^71680 mod p(x)` << 1, x^71744 mod p(x)` << 1 */
	596	.octa 0x0000000136f5ad8a0000000179cc0e36
	597
	598	/* x^70656 mod p(x)` << 1, x^70720 mod p(x)` << 1 */
	599	.octa 0x000000018b07beb600000001dca1da4a
	600
	601	/* x^69632 mod p(x)` << 1, x^69696 mod p(x)` << 1 */
	602	.octa 0x00000000db1e93b000000000feb1a192
	603
	604	/* x^68608 mod p(x)` << 1, x^68672 mod p(x)` << 1 */
	605	.octa 0x000000000b96fa3a00000000d1eeedd6
	606
	607	/* x^67584 mod p(x)` << 1, x^67648 mod p(x)` << 1 */
	608	.octa 0x00000001d9968af0000000008fad9bb4
	609
	610	/* x^66560 mod p(x)` << 1, x^66624 mod p(x)` << 1 */
	611	.octa 0x000000000e4a77a200000001884938e4
	612
	613	/* x^65536 mod p(x)` << 1, x^65600 mod p(x)` << 1 */
	614	.octa 0x00000000508c2ac800000001bc2e9bc0
	615
	616	/* x^64512 mod p(x)` << 1, x^64576 mod p(x)` << 1 */
	617	.octa 0x0000000021572a8000000001f9658a68
	618
	619	/* x^63488 mod p(x)` << 1, x^63552 mod p(x)` << 1 */
	620	.octa 0x00000001b859daf2000000001b9224fc
	621
	622	/* x^62464 mod p(x)` << 1, x^62528 mod p(x)` << 1 */
	623	.octa 0x000000016f7884740000000055b2fb84
	624
	625	/* x^61440 mod p(x)` << 1, x^61504 mod p(x)` << 1 */
	626	.octa 0x00000001b438810e000000018b090348
	627
	628	/* x^60416 mod p(x)` << 1, x^60480 mod p(x)` << 1 */
	629	.octa 0x0000000095ddc6f2000000011ccbd5ea
	630
	631	/* x^59392 mod p(x)` << 1, x^59456 mod p(x)` << 1 */
	632	.octa 0x00000001d977c20c0000000007ae47f8
	633
	634	/* x^58368 mod p(x)` << 1, x^58432 mod p(x)` << 1 */
	635	.octa 0x00000000ebedb99a0000000172acbec0
	636
	637	/* x^57344 mod p(x)` << 1, x^57408 mod p(x)` << 1 */
	638	.octa 0x00000001df9e9e9200000001c6e3ff20
	639
	640	/* x^56320 mod p(x)` << 1, x^56384 mod p(x)` << 1 */
	641	.octa 0x00000001a4a3f95200000000e1b38744
	642
	643	/* x^55296 mod p(x)` << 1, x^55360 mod p(x)` << 1 */
	644	.octa 0x00000000e2f5122000000000791585b2
	645
	646	/* x^54272 mod p(x)` << 1, x^54336 mod p(x)` << 1 */
	647	.octa 0x000000004aa01f3e00000000ac53b894
	648
	649	/* x^53248 mod p(x)` << 1, x^53312 mod p(x)` << 1 */
	650	.octa 0x00000000b3e90a5800000001ed5f2cf4
	651
	652	/* x^52224 mod p(x)` << 1, x^52288 mod p(x)` << 1 */
	653	.octa 0x000000000c9ca2aa00000001df48b2e0
	654
	655	/* x^51200 mod p(x)` << 1, x^51264 mod p(x)` << 1 */
	656	.octa 0x000000015168231600000000049c1c62
	657
	658	/* x^50176 mod p(x)` << 1, x^50240 mod p(x)` << 1 */
	659	.octa 0x0000000036fce78c000000017c460c12
	660
	661	/* x^49152 mod p(x)` << 1, x^49216 mod p(x)` << 1 */
	662	.octa 0x000000009037dc10000000015be4da7e
	663
	664	/* x^48128 mod p(x)` << 1, x^48192 mod p(x)` << 1 */
	665	.octa 0x00000000d3298582000000010f38f668
	666
	667	/* x^47104 mod p(x)` << 1, x^47168 mod p(x)` << 1 */
	668	.octa 0x00000001b42e8ad60000000039f40a00
	669
	670	/* x^46080 mod p(x)` << 1, x^46144 mod p(x)` << 1 */
	671	.octa 0x00000000142a983800000000bd4c10c4
	672
	673	/* x^45056 mod p(x)` << 1, x^45120 mod p(x)` << 1 */
	674	.octa 0x0000000109c7f1900000000042db1d98
	675
	676	/* x^44032 mod p(x)` << 1, x^44096 mod p(x)` << 1 */
	677	.octa 0x0000000056ff931000000001c905bae6
	678
	679	/* x^43008 mod p(x)` << 1, x^43072 mod p(x)` << 1 */
	680	.octa 0x00000001594513aa00000000069d40ea
	681
	682	/* x^41984 mod p(x)` << 1, x^42048 mod p(x)` << 1 */
	683	.octa 0x00000001e3b5b1e8000000008e4fbad0
	684
	685	/* x^40960 mod p(x)` << 1, x^41024 mod p(x)` << 1 */
	686	.octa 0x000000011dd5fc080000000047bedd46
	687
	688	/* x^39936 mod p(x)` << 1, x^40000 mod p(x)` << 1 */
	689	.octa 0x00000001675f0cc20000000026396bf8
	690
	691	/* x^38912 mod p(x)` << 1, x^38976 mod p(x)` << 1 */
	692	.octa 0x00000000d1c8dd4400000000379beb92
	693
	694	/* x^37888 mod p(x)` << 1, x^37952 mod p(x)` << 1 */
	695	.octa 0x0000000115ebd3d8000000000abae54a
	696
	697	/* x^36864 mod p(x)` << 1, x^36928 mod p(x)` << 1 */
	698	.octa 0x00000001ecbd0dac0000000007e6a128
	699
	700	/* x^35840 mod p(x)` << 1, x^35904 mod p(x)` << 1 */
	701	.octa 0x00000000cdf67af2000000000ade29d2
	702
	703	/* x^34816 mod p(x)` << 1, x^34880 mod p(x)` << 1 */
	704	.octa 0x000000004c01ff4c00000000f974c45c
	705
	706	/* x^33792 mod p(x)` << 1, x^33856 mod p(x)` << 1 */
	707	.octa 0x00000000f2d8657e00000000e77ac60a
	708
	709	/* x^32768 mod p(x)` << 1, x^32832 mod p(x)` << 1 */
	710	.octa 0x000000006bae74c40000000145895816
	711
	712	/* x^31744 mod p(x)` << 1, x^31808 mod p(x)` << 1 */
	713	.octa 0x0000000152af8aa00000000038e362be
	714
	715	/* x^30720 mod p(x)` << 1, x^30784 mod p(x)` << 1 */
	716	.octa 0x0000000004663802000000007f991a64
	717
	718	/* x^29696 mod p(x)` << 1, x^29760 mod p(x)` << 1 */
	719	.octa 0x00000001ab2f5afc00000000fa366d3a
	720
	721	/* x^28672 mod p(x)` << 1, x^28736 mod p(x)` << 1 */
	722	.octa 0x0000000074a4ebd400000001a2bb34f0
	723
	724	/* x^27648 mod p(x)` << 1, x^27712 mod p(x)` << 1 */
	725	.octa 0x00000001d7ab3a4c0000000028a9981e
	726
	727	/* x^26624 mod p(x)` << 1, x^26688 mod p(x)` << 1 */
	728	.octa 0x00000001a8da60c600000001dbc672be
	729
	730	/* x^25600 mod p(x)` << 1, x^25664 mod p(x)` << 1 */
	731	.octa 0x000000013cf6382000000000b04d77f6
	732
	733	/* x^24576 mod p(x)` << 1, x^24640 mod p(x)` << 1 */
	734	.octa 0x00000000bec12e1e0000000124400d96
	735
	736	/* x^23552 mod p(x)` << 1, x^23616 mod p(x)` << 1 */
	737	.octa 0x00000001c6368010000000014ca4b414
	738
	739	/* x^22528 mod p(x)` << 1, x^22592 mod p(x)` << 1 */
	740	.octa 0x00000001e6e78758000000012fe2c938
	741
	742	/* x^21504 mod p(x)` << 1, x^21568 mod p(x)` << 1 */
	743	.octa 0x000000008d7f2b3c00000001faed01e6
	744
	745	/* x^20480 mod p(x)` << 1, x^20544 mod p(x)` << 1 */
	746	.octa 0x000000016b4a156e000000007e80ecfe
	747
	748	/* x^19456 mod p(x)` << 1, x^19520 mod p(x)` << 1 */
	749	.octa 0x00000001c63cfeb60000000098daee94
	750
	751	/* x^18432 mod p(x)` << 1, x^18496 mod p(x)` << 1 */
	752	.octa 0x000000015f902670000000010a04edea
	753
	754	/* x^17408 mod p(x)` << 1, x^17472 mod p(x)` << 1 */
	755	.octa 0x00000001cd5de11e00000001c00b4524
	756
	757	/* x^16384 mod p(x)` << 1, x^16448 mod p(x)` << 1 */
	758	.octa 0x000000001acaec540000000170296550
	759
	760	/* x^15360 mod p(x)` << 1, x^15424 mod p(x)` << 1 */
	761	.octa 0x000000002bd0ca780000000181afaa48
	762
	763	/* x^14336 mod p(x)` << 1, x^14400 mod p(x)` << 1 */
	764	.octa 0x0000000032d63d5c0000000185a31ffa
	765
	766	/* x^13312 mod p(x)` << 1, x^13376 mod p(x)` << 1 */
	767	.octa 0x000000001c6d4e4c000000002469f608
	768
	769	/* x^12288 mod p(x)` << 1, x^12352 mod p(x)` << 1 */
	770	.octa 0x0000000106a60b92000000006980102a
	771
	772	/* x^11264 mod p(x)` << 1, x^11328 mod p(x)` << 1 */
	773	.octa 0x00000000d3855e120000000111ea9ca8
	774
	775	/* x^10240 mod p(x)` << 1, x^10304 mod p(x)` << 1 */
	776	.octa 0x00000000e312563600000001bd1d29ce
	777
	778	/* x^9216 mod p(x)` << 1, x^9280 mod p(x)` << 1 */
	779	.octa 0x000000009e8f7ea400000001b34b9580
	780
	781	/* x^8192 mod p(x)` << 1, x^8256 mod p(x)` << 1 */
	782	.octa 0x00000001c82e562c000000003076054e
	783
	784	/* x^7168 mod p(x)` << 1, x^7232 mod p(x)` << 1 */
	785	.octa 0x00000000ca9f09ce000000012a608ea4
	786
	787	/* x^6144 mod p(x)` << 1, x^6208 mod p(x)` << 1 */
	788	.octa 0x00000000c63764e600000000784d05fe
	789
	790	/* x^5120 mod p(x)` << 1, x^5184 mod p(x)` << 1 */
	791	.octa 0x0000000168d2e49e000000016ef0d82a
	792
	793	/* x^4096 mod p(x)` << 1, x^4160 mod p(x)` << 1 */
	794	.octa 0x00000000e986c1480000000075bda454
	795
	796	/* x^3072 mod p(x)` << 1, x^3136 mod p(x)` << 1 */
	797	.octa 0x00000000cfb65894000000003dc0a1c4
	798
	799	/* x^2048 mod p(x)` << 1, x^2112 mod p(x)` << 1 */
	800	.octa 0x0000000111cadee400000000e9a5d8be
	801
	802	/* x^1024 mod p(x)` << 1, x^1088 mod p(x)` << 1 */
	803	.octa 0x0000000171fb63ce00000001609bc4b4
	804
	805	.short_constants:
	806
	807	/* Reduce final 1024-2048 bits to 64 bits, shifting 32 bits to include the trailing 32 bits of zeros */
	808	/* x^1952 mod p(x)`, x^1984 mod p(x)`, x^2016 mod p(x)`, x^2048 mod p(x)` */
	809	.octa 0x7fec2963e5bf80485cf015c388e56f72
	810
	811	/* x^1824 mod p(x)`, x^1856 mod p(x)`, x^1888 mod p(x)`, x^1920 mod p(x)` */
	812	.octa 0x38e888d4844752a9963a18920246e2e6
	813
	814	/* x^1696 mod p(x)`, x^1728 mod p(x)`, x^1760 mod p(x)`, x^1792 mod p(x)` */
	815	.octa 0x42316c00730206ad419a441956993a31
	816
	817	/* x^1568 mod p(x)`, x^1600 mod p(x)`, x^1632 mod p(x)`, x^1664 mod p(x)` */
	818	.octa 0x543d5c543e65ddf9924752ba2b830011
	819
	820	/* x^1440 mod p(x)`, x^1472 mod p(x)`, x^1504 mod p(x)`, x^1536 mod p(x)` */
	821	.octa 0x78e87aaf56767c9255bd7f9518e4a304
	822
	823	/* x^1312 mod p(x)`, x^1344 mod p(x)`, x^1376 mod p(x)`, x^1408 mod p(x)` */
	824	.octa 0x8f68fcec1903da7f6d76739fe0553f1e
	825
	826	/* x^1184 mod p(x)`, x^1216 mod p(x)`, x^1248 mod p(x)`, x^1280 mod p(x)` */
	827	.octa 0x3f4840246791d588c133722b1fe0b5c3
	828
	829	/* x^1056 mod p(x)`, x^1088 mod p(x)`, x^1120 mod p(x)`, x^1152 mod p(x)` */
	830	.octa 0x34c96751b04de25a64b67ee0e55ef1f3
	831
	832	/* x^928 mod p(x)`, x^960 mod p(x)`, x^992 mod p(x)`, x^1024 mod p(x)` */
	833	.octa 0x156c8e180b4a395b069db049b8fdb1e7
	834
	835	/* x^800 mod p(x)`, x^832 mod p(x)`, x^864 mod p(x)`, x^896 mod p(x)` */
	836	.octa 0xe0b99ccbe661f7bea11bfaf3c9e90b9e
	837
	838	/* x^672 mod p(x)`, x^704 mod p(x)`, x^736 mod p(x)`, x^768 mod p(x)` */
	839	.octa 0x041d37768cd75659817cdc5119b29a35
	840
	841	/* x^544 mod p(x)`, x^576 mod p(x)`, x^608 mod p(x)`, x^640 mod p(x)` */
	842	.octa 0x3a0777818cfaa9651ce9d94b36c41f1c
	843
	844	/* x^416 mod p(x)`, x^448 mod p(x)`, x^480 mod p(x)`, x^512 mod p(x)` */
	845	.octa 0x0e148e8252377a554f256efcb82be955
	846
	847	/* x^288 mod p(x)`, x^320 mod p(x)`, x^352 mod p(x)`, x^384 mod p(x)` */
	848	.octa 0x9c25531d19e65ddeec1631edb2dea967
	849
	850	/* x^160 mod p(x)`, x^192 mod p(x)`, x^224 mod p(x)`, x^256 mod p(x)` */
	851	.octa 0x790606ff9957c0a65d27e147510ac59a
	852
	853	/* x^32 mod p(x)`, x^64 mod p(x)`, x^96 mod p(x)`, x^128 mod p(x)` */
	854	.octa 0x82f63b786ea2d55ca66805eb18b8ea18
	855
	856
	857	.barrett_constants:
	858	/* 33 bit reflected Barrett constant m - (4^32)/n */
	859	.octa 0x000000000000000000000000dea713f1 /* x^64 div p(x)` */
	860	/* 33 bit reflected Barrett constant n */
	861	.octa 0x00000000000000000000000105ec76f1
	862
	863	.text
	864
	865	#if defined(__BIG_ENDIAN__)
	866	#define BYTESWAP_DATA
	867	#else
	868	#undef BYTESWAP_DATA
	869	#endif
	870
	871	#define off16 r25
	872	#define off32 r26
	873	#define off48 r27
	874	#define off64 r28
	875	#define off80 r29
	876	#define off96 r30
	877	#define off112 r31
	878
	879	#define const1 v24
	880	#define const2 v25
	881
	882	#define byteswap v26
	883	#define mask_32bit v27
	884	#define mask_64bit v28
	885	#define zeroes v29
	886
	887	#ifdef BYTESWAP_DATA
	888	#define VPERM(A, B, C, D) vperm A, B, C, D
	889	#else
	890	#define VPERM(A, B, C, D)
	891	#endif
	892
	893	/* unsigned int __crc32c_vpmsum(unsigned int crc, void p, unsigned long len) /
	894	FUNC_START(__crc32c_vpmsum)
	895	std r31,-8(r1)
	896	std r30,-16(r1)
	897	std r29,-24(r1)
	898	std r28,-32(r1)
	899	std r27,-40(r1)
	900	std r26,-48(r1)
	901	std r25,-56(r1)
	902
	903	li off16,16
	904	li off32,32
	905	li off48,48
	906	li off64,64
	907	li off80,80
	908	li off96,96
	909	li off112,112
	910	li r0,0
	911
	912	/* Enough room for saving 10 non volatile VMX registers */
	913	subi r6,r1,56+10*16
	914	subi r7,r1,56+2*16
	915
	916	stvx v20,0,r6
	917	stvx v21,off16,r6
	918	stvx v22,off32,r6
	919	stvx v23,off48,r6
	920	stvx v24,off64,r6
	921	stvx v25,off80,r6
	922	stvx v26,off96,r6
	923	stvx v27,off112,r6
	924	stvx v28,0,r7
	925	stvx v29,off16,r7
	926
	927	mr r10,r3
	928
	929	vxor zeroes,zeroes,zeroes
	930	vspltisw v0,-1
	931
	932	vsldoi mask_32bit,zeroes,v0,4
	933	vsldoi mask_64bit,zeroes,v0,8
	934
	935	/* Get the initial value into v8 */
	936	vxor v8,v8,v8
	937	MTVRD(v8, R3)
	938	vsldoi v8,zeroes,v8,8 /* shift into bottom 32 bits */
	939
	940	#ifdef BYTESWAP_DATA
	941	addis r3,r2,.byteswap_constant@toc@ha
	942	addi r3,r3,.byteswap_constant@toc@l
	943
	944	lvx byteswap,0,r3
	945	addi r3,r3,16
	946	#endif
	947
	948	cmpdi r5,256
	949	blt .Lshort
	950
	951	rldicr r6,r5,0,56
	952
	953	/* Checksum in blocks of MAX_SIZE */
	954	1: lis r7,MAX_SIZE@h
	955	ori r7,r7,MAX_SIZE@l
	956	mr r9,r7
	957	cmpd r6,r7
	958	bgt 2f
	959	mr r7,r6
	960	2: subf r6,r7,r6
	961
	962	/* our main loop does 128 bytes at a time */
	963	srdi r7,r7,7
	964
	965	/*
	966	* Work out the offset into the constants table to start at. Each
	967	* constant is 16 bytes, and it is used against 128 bytes of input
	968	* data - 128 / 16 = 8
	969	*/
	970	sldi r8,r7,4
	971	srdi r9,r9,3
	972	subf r8,r8,r9
	973
	974	/* We reduce our final 128 bytes in a separate step */
	975	addi r7,r7,-1
	976	mtctr r7
	977
	978	addis r3,r2,.constants@toc@ha
	979	addi r3,r3,.constants@toc@l
	980
	981	/* Find the start of our constants */
	982	add r3,r3,r8
	983
	984	/* zero v0-v7 which will contain our checksums */
	985	vxor v0,v0,v0
	986	vxor v1,v1,v1
	987	vxor v2,v2,v2
	988	vxor v3,v3,v3
	989	vxor v4,v4,v4
	990	vxor v5,v5,v5
	991	vxor v6,v6,v6
	992	vxor v7,v7,v7
	993
	994	lvx const1,0,r3
	995
	996	/*
	997	* If we are looping back to consume more data we use the values
	998	* already in v16-v23.
	999	*/
	1000	cmpdi r0,1
	1001	beq 2f
	1002
	1003	/* First warm up pass */
	1004	lvx v16,0,r4
	1005	lvx v17,off16,r4
	1006	VPERM(v16,v16,v16,byteswap)
	1007	VPERM(v17,v17,v17,byteswap)
	1008	lvx v18,off32,r4
	1009	lvx v19,off48,r4
	1010	VPERM(v18,v18,v18,byteswap)
	1011	VPERM(v19,v19,v19,byteswap)
	1012	lvx v20,off64,r4
	1013	lvx v21,off80,r4
	1014	VPERM(v20,v20,v20,byteswap)
	1015	VPERM(v21,v21,v21,byteswap)
	1016	lvx v22,off96,r4
	1017	lvx v23,off112,r4
	1018	VPERM(v22,v22,v22,byteswap)
	1019	VPERM(v23,v23,v23,byteswap)
	1020	addi r4,r4,8*16
	1021
	1022	/* xor in initial value */
	1023	vxor v16,v16,v8
	1024
	1025	2: bdz .Lfirst_warm_up_done
	1026
	1027	addi r3,r3,16
	1028	lvx const2,0,r3
	1029
	1030	/* Second warm up pass */
	1031	VPMSUMD(v8,v16,const1)
	1032	lvx v16,0,r4
	1033	VPERM(v16,v16,v16,byteswap)
	1034	ori r2,r2,0
	1035
	1036	VPMSUMD(v9,v17,const1)
	1037	lvx v17,off16,r4
	1038	VPERM(v17,v17,v17,byteswap)
	1039	ori r2,r2,0
	1040
	1041	VPMSUMD(v10,v18,const1)
	1042	lvx v18,off32,r4
	1043	VPERM(v18,v18,v18,byteswap)
	1044	ori r2,r2,0
	1045
	1046	VPMSUMD(v11,v19,const1)
	1047	lvx v19,off48,r4
	1048	VPERM(v19,v19,v19,byteswap)
	1049	ori r2,r2,0
	1050
	1051	VPMSUMD(v12,v20,const1)
	1052	lvx v20,off64,r4
	1053	VPERM(v20,v20,v20,byteswap)
	1054	ori r2,r2,0
	1055
	1056	VPMSUMD(v13,v21,const1)
	1057	lvx v21,off80,r4
	1058	VPERM(v21,v21,v21,byteswap)
	1059	ori r2,r2,0
	1060
	1061	VPMSUMD(v14,v22,const1)
	1062	lvx v22,off96,r4
	1063	VPERM(v22,v22,v22,byteswap)
	1064	ori r2,r2,0
	1065
	1066	VPMSUMD(v15,v23,const1)
	1067	lvx v23,off112,r4
	1068	VPERM(v23,v23,v23,byteswap)
	1069
	1070	addi r4,r4,8*16
	1071
	1072	bdz .Lfirst_cool_down
	1073
	1074	/*
	1075	* main loop. We modulo schedule it such that it takes three iterations
	1076	* to complete - first iteration load, second iteration vpmsum, third
	1077	* iteration xor.
	1078	*/
	1079	.balign 16
	1080	4: lvx const1,0,r3
	1081	addi r3,r3,16
	1082	ori r2,r2,0
	1083
	1084	vxor v0,v0,v8
	1085	VPMSUMD(v8,v16,const2)
	1086	lvx v16,0,r4
	1087	VPERM(v16,v16,v16,byteswap)
	1088	ori r2,r2,0
	1089
	1090	vxor v1,v1,v9
	1091	VPMSUMD(v9,v17,const2)
	1092	lvx v17,off16,r4
	1093	VPERM(v17,v17,v17,byteswap)
	1094	ori r2,r2,0
	1095
	1096	vxor v2,v2,v10
	1097	VPMSUMD(v10,v18,const2)
	1098	lvx v18,off32,r4
	1099	VPERM(v18,v18,v18,byteswap)
	1100	ori r2,r2,0
	1101
	1102	vxor v3,v3,v11
	1103	VPMSUMD(v11,v19,const2)
	1104	lvx v19,off48,r4
	1105	VPERM(v19,v19,v19,byteswap)
	1106	lvx const2,0,r3
	1107	ori r2,r2,0
	1108
	1109	vxor v4,v4,v12
	1110	VPMSUMD(v12,v20,const1)
	1111	lvx v20,off64,r4
	1112	VPERM(v20,v20,v20,byteswap)
	1113	ori r2,r2,0
	1114
	1115	vxor v5,v5,v13
	1116	VPMSUMD(v13,v21,const1)
	1117	lvx v21,off80,r4
	1118	VPERM(v21,v21,v21,byteswap)
	1119	ori r2,r2,0
	1120
	1121	vxor v6,v6,v14
	1122	VPMSUMD(v14,v22,const1)
	1123	lvx v22,off96,r4
	1124	VPERM(v22,v22,v22,byteswap)
	1125	ori r2,r2,0
	1126
	1127	vxor v7,v7,v15
	1128	VPMSUMD(v15,v23,const1)
	1129	lvx v23,off112,r4
	1130	VPERM(v23,v23,v23,byteswap)
	1131
	1132	addi r4,r4,8*16
	1133
	1134	bdnz 4b
	1135
	1136	.Lfirst_cool_down:
	1137	/* First cool down pass */
	1138	lvx const1,0,r3
	1139	addi r3,r3,16
	1140
	1141	vxor v0,v0,v8
	1142	VPMSUMD(v8,v16,const1)
	1143	ori r2,r2,0
	1144
	1145	vxor v1,v1,v9
	1146	VPMSUMD(v9,v17,const1)
	1147	ori r2,r2,0
	1148
	1149	vxor v2,v2,v10
	1150	VPMSUMD(v10,v18,const1)
	1151	ori r2,r2,0
	1152
	1153	vxor v3,v3,v11
	1154	VPMSUMD(v11,v19,const1)
	1155	ori r2,r2,0
	1156
	1157	vxor v4,v4,v12
	1158	VPMSUMD(v12,v20,const1)
	1159	ori r2,r2,0
	1160
	1161	vxor v5,v5,v13
	1162	VPMSUMD(v13,v21,const1)
	1163	ori r2,r2,0
	1164
	1165	vxor v6,v6,v14
	1166	VPMSUMD(v14,v22,const1)
	1167	ori r2,r2,0
	1168
	1169	vxor v7,v7,v15
	1170	VPMSUMD(v15,v23,const1)
	1171	ori r2,r2,0
	1172
	1173	.Lsecond_cool_down:
	1174	/* Second cool down pass */
	1175	vxor v0,v0,v8
	1176	vxor v1,v1,v9
	1177	vxor v2,v2,v10
	1178	vxor v3,v3,v11
	1179	vxor v4,v4,v12
	1180	vxor v5,v5,v13
	1181	vxor v6,v6,v14
	1182	vxor v7,v7,v15
	1183
	1184	/*
	1185	* vpmsumd produces a 96 bit result in the least significant bits
	1186	* of the register. Since we are bit reflected we have to shift it
	1187	* left 32 bits so it occupies the least significant bits in the
	1188	* bit reflected domain.
	1189	*/
	1190	vsldoi v0,v0,zeroes,4
	1191	vsldoi v1,v1,zeroes,4
	1192	vsldoi v2,v2,zeroes,4
	1193	vsldoi v3,v3,zeroes,4
	1194	vsldoi v4,v4,zeroes,4
	1195	vsldoi v5,v5,zeroes,4
	1196	vsldoi v6,v6,zeroes,4
	1197	vsldoi v7,v7,zeroes,4
	1198
	1199	/* xor with last 1024 bits */
	1200	lvx v8,0,r4
	1201	lvx v9,off16,r4
	1202	VPERM(v8,v8,v8,byteswap)
	1203	VPERM(v9,v9,v9,byteswap)
	1204	lvx v10,off32,r4
	1205	lvx v11,off48,r4
	1206	VPERM(v10,v10,v10,byteswap)
	1207	VPERM(v11,v11,v11,byteswap)
	1208	lvx v12,off64,r4
	1209	lvx v13,off80,r4
	1210	VPERM(v12,v12,v12,byteswap)
	1211	VPERM(v13,v13,v13,byteswap)
	1212	lvx v14,off96,r4
	1213	lvx v15,off112,r4
	1214	VPERM(v14,v14,v14,byteswap)
	1215	VPERM(v15,v15,v15,byteswap)
	1216
	1217	addi r4,r4,8*16
	1218
	1219	vxor v16,v0,v8
	1220	vxor v17,v1,v9
	1221	vxor v18,v2,v10
	1222	vxor v19,v3,v11
	1223	vxor v20,v4,v12
	1224	vxor v21,v5,v13
	1225	vxor v22,v6,v14
	1226	vxor v23,v7,v15
	1227
	1228	li r0,1
	1229	cmpdi r6,0
	1230	addi r6,r6,128
	1231	bne 1b
	1232
	1233	/* Work out how many bytes we have left */
	1234	andi. r5,r5,127
	1235
	1236	/* Calculate where in the constant table we need to start */
	1237	subfic r6,r5,128
	1238	add r3,r3,r6
	1239
	1240	/* How many 16 byte chunks are in the tail */
	1241	srdi r7,r5,4
	1242	mtctr r7
	1243
	1244	/*
	1245	* Reduce the previously calculated 1024 bits to 64 bits, shifting
	1246	* 32 bits to include the trailing 32 bits of zeros
	1247	*/
	1248	lvx v0,0,r3
	1249	lvx v1,off16,r3
	1250	lvx v2,off32,r3
	1251	lvx v3,off48,r3
	1252	lvx v4,off64,r3
	1253	lvx v5,off80,r3
	1254	lvx v6,off96,r3
	1255	lvx v7,off112,r3
	1256	addi r3,r3,8*16
	1257
	1258	VPMSUMW(v0,v16,v0)
	1259	VPMSUMW(v1,v17,v1)
	1260	VPMSUMW(v2,v18,v2)
	1261	VPMSUMW(v3,v19,v3)
	1262	VPMSUMW(v4,v20,v4)
	1263	VPMSUMW(v5,v21,v5)
	1264	VPMSUMW(v6,v22,v6)
	1265	VPMSUMW(v7,v23,v7)
	1266
	1267	/* Now reduce the tail (0 - 112 bytes) */
	1268	cmpdi r7,0
	1269	beq 1f
	1270
	1271	lvx v16,0,r4
	1272	lvx v17,0,r3
	1273	VPERM(v16,v16,v16,byteswap)
	1274	VPMSUMW(v16,v16,v17)
	1275	vxor v0,v0,v16
	1276	bdz 1f
	1277
	1278	lvx v16,off16,r4
	1279	lvx v17,off16,r3
	1280	VPERM(v16,v16,v16,byteswap)
	1281	VPMSUMW(v16,v16,v17)
	1282	vxor v0,v0,v16
	1283	bdz 1f
	1284
	1285	lvx v16,off32,r4
	1286	lvx v17,off32,r3
	1287	VPERM(v16,v16,v16,byteswap)
	1288	VPMSUMW(v16,v16,v17)
	1289	vxor v0,v0,v16
	1290	bdz 1f
	1291
	1292	lvx v16,off48,r4
	1293	lvx v17,off48,r3
	1294	VPERM(v16,v16,v16,byteswap)
	1295	VPMSUMW(v16,v16,v17)
	1296	vxor v0,v0,v16
	1297	bdz 1f
	1298
	1299	lvx v16,off64,r4
	1300	lvx v17,off64,r3
	1301	VPERM(v16,v16,v16,byteswap)
	1302	VPMSUMW(v16,v16,v17)
	1303	vxor v0,v0,v16
	1304	bdz 1f
	1305
	1306	lvx v16,off80,r4
	1307	lvx v17,off80,r3
	1308	VPERM(v16,v16,v16,byteswap)
	1309	VPMSUMW(v16,v16,v17)
	1310	vxor v0,v0,v16
	1311	bdz 1f
	1312
	1313	lvx v16,off96,r4
	1314	lvx v17,off96,r3
	1315	VPERM(v16,v16,v16,byteswap)
	1316	VPMSUMW(v16,v16,v17)
	1317	vxor v0,v0,v16
	1318
	1319	/* Now xor all the parallel chunks together */
	1320	1: vxor v0,v0,v1
	1321	vxor v2,v2,v3
	1322	vxor v4,v4,v5
	1323	vxor v6,v6,v7
	1324
	1325	vxor v0,v0,v2
	1326	vxor v4,v4,v6
	1327
	1328	vxor v0,v0,v4
	1329
	1330	.Lbarrett_reduction:
	1331	/* Barrett constants */
	1332	addis r3,r2,.barrett_constants@toc@ha
	1333	addi r3,r3,.barrett_constants@toc@l
	1334
	1335	lvx const1,0,r3
	1336	lvx const2,off16,r3
	1337
	1338	vsldoi v1,v0,v0,8
	1339	vxor v0,v0,v1 /* xor two 64 bit results together */
	1340
	1341	/* shift left one bit */
	1342	vspltisb v1,1
	1343	vsl v0,v0,v1
	1344
	1345	vand v0,v0,mask_64bit
	1346
	1347	/*
	1348	* The reflected version of Barrett reduction. Instead of bit
	1349	* reflecting our data (which is expensive to do), we bit reflect our
	1350	* constants and our algorithm, which means the intermediate data in
	1351	* our vector registers goes from 0-63 instead of 63-0. We can reflect
	1352	* the algorithm because we don't carry in mod 2 arithmetic.
	1353	*/
	1354	vand v1,v0,mask_32bit /* bottom 32 bits of a */
	1355	VPMSUMD(v1,v1,const1) /* ma */
	1356	vand v1,v1,mask_32bit /* bottom 32bits of ma */
	1357	VPMSUMD(v1,v1,const2) /* qn */
	1358	vxor v0,v0,v1 /* a - qn, subtraction is xor in GF(2) */
	1359
	1360	/*
	1361	* Since we are bit reflected, the result (ie the low 32 bits) is in
	1362	* the high 32 bits. We just need to shift it left 4 bytes
	1363	* V0 [ 0 1 X 3 ]
	1364	* V0 [ 0 X 2 3 ]
	1365	*/
	1366	vsldoi v0,v0,zeroes,4 /* shift result into top 64 bits of */
	1367
	1368	/* Get it into r3 */
	1369	MFVRD(R3, v0)
	1370
	1371	.Lout:
	1372	subi r6,r1,56+10*16
	1373	subi r7,r1,56+2*16
	1374
	1375	lvx v20,0,r6
	1376	lvx v21,off16,r6
	1377	lvx v22,off32,r6
	1378	lvx v23,off48,r6
	1379	lvx v24,off64,r6
	1380	lvx v25,off80,r6
	1381	lvx v26,off96,r6
	1382	lvx v27,off112,r6
	1383	lvx v28,0,r7
	1384	lvx v29,off16,r7
	1385
	1386	ld r31,-8(r1)
	1387	ld r30,-16(r1)
	1388	ld r29,-24(r1)
	1389	ld r28,-32(r1)
	1390	ld r27,-40(r1)
	1391	ld r26,-48(r1)
	1392	ld r25,-56(r1)
	1393
	1394	blr
	1395
	1396	.Lfirst_warm_up_done:
	1397	lvx const1,0,r3
	1398	addi r3,r3,16
	1399
	1400	VPMSUMD(v8,v16,const1)
	1401	VPMSUMD(v9,v17,const1)
	1402	VPMSUMD(v10,v18,const1)
	1403	VPMSUMD(v11,v19,const1)
	1404	VPMSUMD(v12,v20,const1)
	1405	VPMSUMD(v13,v21,const1)
	1406	VPMSUMD(v14,v22,const1)
	1407	VPMSUMD(v15,v23,const1)
	1408
	1409	b .Lsecond_cool_down
	1410
	1411	.Lshort:
	1412	cmpdi r5,0
	1413	beq .Lzero
	1414
	1415	addis r3,r2,.short_constants@toc@ha
	1416	addi r3,r3,.short_constants@toc@l
	1417
	1418	/* Calculate where in the constant table we need to start */
	1419	subfic r6,r5,256
	1420	add r3,r3,r6
	1421
	1422	/* How many 16 byte chunks? */
	1423	srdi r7,r5,4
	1424	mtctr r7
	1425
	1426	vxor v19,v19,v19
	1427	vxor v20,v20,v20
	1428
	1429	lvx v0,0,r4
	1430	lvx v16,0,r3
	1431	VPERM(v0,v0,v16,byteswap)
	1432	vxor v0,v0,v8 /* xor in initial value */
	1433	VPMSUMW(v0,v0,v16)
	1434	bdz .Lv0
	1435
	1436	lvx v1,off16,r4
	1437	lvx v17,off16,r3
	1438	VPERM(v1,v1,v17,byteswap)
	1439	VPMSUMW(v1,v1,v17)
	1440	bdz .Lv1
	1441
	1442	lvx v2,off32,r4
	1443	lvx v16,off32,r3
	1444	VPERM(v2,v2,v16,byteswap)
	1445	VPMSUMW(v2,v2,v16)
	1446	bdz .Lv2
	1447
	1448	lvx v3,off48,r4
	1449	lvx v17,off48,r3
	1450	VPERM(v3,v3,v17,byteswap)
	1451	VPMSUMW(v3,v3,v17)
	1452	bdz .Lv3
	1453
	1454	lvx v4,off64,r4
	1455	lvx v16,off64,r3
	1456	VPERM(v4,v4,v16,byteswap)
	1457	VPMSUMW(v4,v4,v16)
	1458	bdz .Lv4
	1459
	1460	lvx v5,off80,r4
	1461	lvx v17,off80,r3
	1462	VPERM(v5,v5,v17,byteswap)
	1463	VPMSUMW(v5,v5,v17)
	1464	bdz .Lv5
	1465
	1466	lvx v6,off96,r4
	1467	lvx v16,off96,r3
	1468	VPERM(v6,v6,v16,byteswap)
	1469	VPMSUMW(v6,v6,v16)
	1470	bdz .Lv6
	1471
	1472	lvx v7,off112,r4
	1473	lvx v17,off112,r3
	1474	VPERM(v7,v7,v17,byteswap)
	1475	VPMSUMW(v7,v7,v17)
	1476	bdz .Lv7
	1477
	1478	addi r3,r3,128
	1479	addi r4,r4,128
	1480
	1481	lvx v8,0,r4
	1482	lvx v16,0,r3
	1483	VPERM(v8,v8,v16,byteswap)
	1484	VPMSUMW(v8,v8,v16)
	1485	bdz .Lv8
	1486
	1487	lvx v9,off16,r4
	1488	lvx v17,off16,r3
	1489	VPERM(v9,v9,v17,byteswap)
	1490	VPMSUMW(v9,v9,v17)
	1491	bdz .Lv9
	1492
	1493	lvx v10,off32,r4
	1494	lvx v16,off32,r3
	1495	VPERM(v10,v10,v16,byteswap)
	1496	VPMSUMW(v10,v10,v16)
	1497	bdz .Lv10
	1498
	1499	lvx v11,off48,r4
	1500	lvx v17,off48,r3
	1501	VPERM(v11,v11,v17,byteswap)
	1502	VPMSUMW(v11,v11,v17)
	1503	bdz .Lv11
	1504
	1505	lvx v12,off64,r4
	1506	lvx v16,off64,r3
	1507	VPERM(v12,v12,v16,byteswap)
	1508	VPMSUMW(v12,v12,v16)
	1509	bdz .Lv12
	1510
	1511	lvx v13,off80,r4
	1512	lvx v17,off80,r3
	1513	VPERM(v13,v13,v17,byteswap)
	1514	VPMSUMW(v13,v13,v17)
	1515	bdz .Lv13
	1516
	1517	lvx v14,off96,r4
	1518	lvx v16,off96,r3
	1519	VPERM(v14,v14,v16,byteswap)
	1520	VPMSUMW(v14,v14,v16)
	1521	bdz .Lv14
	1522
	1523	lvx v15,off112,r4
	1524	lvx v17,off112,r3
	1525	VPERM(v15,v15,v17,byteswap)
	1526	VPMSUMW(v15,v15,v17)
	1527
	1528	.Lv15: vxor v19,v19,v15
	1529	.Lv14: vxor v20,v20,v14
	1530	.Lv13: vxor v19,v19,v13
	1531	.Lv12: vxor v20,v20,v12
	1532	.Lv11: vxor v19,v19,v11
	1533	.Lv10: vxor v20,v20,v10
	1534	.Lv9: vxor v19,v19,v9
	1535	.Lv8: vxor v20,v20,v8
	1536	.Lv7: vxor v19,v19,v7
	1537	.Lv6: vxor v20,v20,v6
	1538	.Lv5: vxor v19,v19,v5
	1539	.Lv4: vxor v20,v20,v4
	1540	.Lv3: vxor v19,v19,v3
	1541	.Lv2: vxor v20,v20,v2
	1542	.Lv1: vxor v19,v19,v1
	1543	.Lv0: vxor v20,v20,v0
	1544
	1545	vxor v0,v19,v20
	1546
	1547	b .Lbarrett_reduction
	1548
	1549	.Lzero:
	1550	mr r3,r10
	1551	b .Lout
	1552
	1553	FUNC_END(__crc32_vpmsum)

arch/powerpc/crypto/crc32c-vpmsum_glue.c+167−0

@@ -0,0 +1,167 @@
	1	#include <linux/crc32.h>
	2	#include <crypto/internal/hash.h>
	3	#include <linux/init.h>
	4	#include <linux/module.h>
	5	#include <linux/string.h>
	6	#include <linux/kernel.h>
	7	#include <asm/switch_to.h>
	8
	9	#define CHKSUM_BLOCK_SIZE 1
	10	#define CHKSUM_DIGEST_SIZE 4
	11
	12	#define VMX_ALIGN 16
	13	#define VMX_ALIGN_MASK (VMX_ALIGN-1)
	14
	15	#define VECTOR_BREAKPOINT 512
	16
	17	u32 __crc32c_vpmsum(u32 crc, unsigned char const *p, size_t len);
	18
	19	static u32 crc32c_vpmsum(u32 crc, unsigned char const *p, size_t len)
	20	{
	21	unsigned int prealign;
	22	unsigned int tail;
	23
	24	if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) \|\| in_interrupt())
	25	return __crc32c_le(crc, p, len);
	26
	27	if ((unsigned long)p & VMX_ALIGN_MASK) {
	28	prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK);
	29	crc = __crc32c_le(crc, p, prealign);
	30	len -= prealign;
	31	p += prealign;
	32	}
	33
	34	if (len & ~VMX_ALIGN_MASK) {
	35	pagefault_disable();
	36	enable_kernel_altivec();
	37	crc = __crc32c_vpmsum(crc, p, len & ~VMX_ALIGN_MASK);
	38	pagefault_enable();
	39	}
	40
	41	tail = len & VMX_ALIGN_MASK;
	42	if (tail) {
	43	p += len & ~VMX_ALIGN_MASK;
	44	crc = __crc32c_le(crc, p, tail);
	45	}
	46
	47	return crc;
	48	}
	49
	50	static int crc32c_vpmsum_cra_init(struct crypto_tfm *tfm)
	51	{
	52	u32 *key = crypto_tfm_ctx(tfm);
	53
	54	*key = 0;
	55
	56	return 0;
	57	}
	58
	59	/*
	60	* Setting the seed allows arbitrary accumulators and flexible XOR policy
	61	* If your algorithm starts with ~0, then XOR with ~0 before you set
	62	* the seed.
	63	*/
	64	static int crc32c_vpmsum_setkey(struct crypto_shash hash, const u8 key,
	65	unsigned int keylen)
	66	{
	67	u32 *mctx = crypto_shash_ctx(hash);
	68
	69	if (keylen != sizeof(u32)) {
	70	crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN);
	71	return -EINVAL;
	72	}
	73	mctx = le32_to_cpup((__le32 )key);
	74	return 0;
	75	}
	76
	77	static int crc32c_vpmsum_init(struct shash_desc *desc)
	78	{
	79	u32 *mctx = crypto_shash_ctx(desc->tfm);
	80	u32 *crcp = shash_desc_ctx(desc);
	81
	82	crcp = mctx;
	83
	84	return 0;
	85	}
	86
	87	static int crc32c_vpmsum_update(struct shash_desc desc, const u8 data,
	88	unsigned int len)
	89	{
	90	u32 *crcp = shash_desc_ctx(desc);
	91
	92	crcp = crc32c_vpmsum(crcp, data, len);
	93
	94	return 0;
	95	}
	96
	97	static int __crc32c_vpmsum_finup(u32 crcp, const u8 data, unsigned int len,
	98	u8 *out)
	99	{
	100	(__le32 )out = ~cpu_to_le32(crc32c_vpmsum(*crcp, data, len));
	101
	102	return 0;
	103	}
	104
	105	static int crc32c_vpmsum_finup(struct shash_desc desc, const u8 data,
	106	unsigned int len, u8 *out)
	107	{
	108	return __crc32c_vpmsum_finup(shash_desc_ctx(desc), data, len, out);
	109	}
	110
	111	static int crc32c_vpmsum_final(struct shash_desc desc, u8 out)
	112	{
	113	u32 *crcp = shash_desc_ctx(desc);
	114
	115	(__le32 )out = ~cpu_to_le32p(crcp);
	116
	117	return 0;
	118	}
	119
	120	static int crc32c_vpmsum_digest(struct shash_desc desc, const u8 data,
	121	unsigned int len, u8 *out)
	122	{
	123	return __crc32c_vpmsum_finup(crypto_shash_ctx(desc->tfm), data, len,
	124	out);
	125	}
	126
	127	static struct shash_alg alg = {
	128	.setkey = crc32c_vpmsum_setkey,
	129	.init = crc32c_vpmsum_init,
	130	.update = crc32c_vpmsum_update,
	131	.final = crc32c_vpmsum_final,
	132	.finup = crc32c_vpmsum_finup,
	133	.digest = crc32c_vpmsum_digest,
	134	.descsize = sizeof(u32),
	135	.digestsize = CHKSUM_DIGEST_SIZE,
	136	.base = {
	137	.cra_name = "crc32c",
	138	.cra_driver_name = "crc32c-vpmsum",
	139	.cra_priority = 200,
	140	.cra_blocksize = CHKSUM_BLOCK_SIZE,
	141	.cra_ctxsize = sizeof(u32),
	142	.cra_module = THIS_MODULE,
	143	.cra_init = crc32c_vpmsum_cra_init,
	144	}
	145	};
	146
	147	static int __init crc32c_vpmsum_mod_init(void)
	148	{
	149	if (!cpu_has_feature(CPU_FTR_ARCH_207S))
	150	return -ENODEV;
	151
	152	return crypto_register_shash(&alg);
	153	}
	154
	155	static void __exit crc32c_vpmsum_mod_fini(void)
	156	{
	157	crypto_unregister_shash(&alg);
	158	}
	159
	160	module_init(crc32c_vpmsum_mod_init);
	161	module_exit(crc32c_vpmsum_mod_fini);
	162
	163	MODULE_AUTHOR("Anton Blanchard <anton@samba.org>");
	164	MODULE_DESCRIPTION("CRC32C using vector polynomial multiply-sum instructions");
	165	MODULE_LICENSE("GPL");
	166	MODULE_ALIAS_CRYPTO("crc32c");
	167	MODULE_ALIAS_CRYPTO("crc32c-vpmsum");

arch/powerpc/include/asm/ppc-opcode.h+12−0

 #define PPC_INST_MFSPR_DSCR_USER_MASK	0xfc1fffff
 #define PPC_INST_MTSPR_DSCR_USER	0x7c0303a6
 #define PPC_INST_MTSPR_DSCR_USER_MASK	0xfc1fffff
+#define PPC_INST_MFVSRD			0x7c000066
+#define PPC_INST_MTVSRD			0x7c000166
 #define PPC_INST_SLBFEE			0x7c0007a7
 #define PPC_INST_STRING			0x7c00042a
 #define PPC_INST_WAIT			0x7c00007c
 #define PPC_INST_TLBIVAX		0x7c000624
 #define PPC_INST_TLBSRX_DOT		0x7c0006a5
+#define PPC_INST_VPMSUMW		0x10000488
+#define PPC_INST_VPMSUMD		0x100004c8
 #define PPC_INST_XXLOR			0xf0000510
 #define PPC_INST_XXSWAPD		0xf0000250
 #define PPC_INST_XVCPSGNDP		0xf0000780
 					       VSX_XX1((s), a, b))
 #define LXVD2X(s, a, b)		stringify_in_c(.long PPC_INST_LXVD2X | \
 					       VSX_XX1((s), a, b))
+#define MFVRD(a, t)		stringify_in_c(.long PPC_INST_MFVSRD | \
+					       VSX_XX1((t)+32, a, R0))
+#define MTVRD(t, a)		stringify_in_c(.long PPC_INST_MTVSRD | \
+					       VSX_XX1((t)+32, a, R0))
+#define VPMSUMW(t, a, b)	stringify_in_c(.long PPC_INST_VPMSUMW | \
+					       VSX_XX3((t), a, b))
+#define VPMSUMD(t, a, b)	stringify_in_c(.long PPC_INST_VPMSUMD | \
+					       VSX_XX3((t), a, b))
 #define XXLOR(t, a, b)		stringify_in_c(.long PPC_INST_XXLOR | \
 					       VSX_XX3((t), a, b))
 #define XXSWAPD(t, a)		stringify_in_c(.long PPC_INST_XXSWAPD | \

arch/powerpc/include/asm/ppc_asm.h+3−0

@@ -286,6 +286,9 @@ n: · 285 unmodified lines
286		286
287	#endif	287	#endif
288		288
		289	#define FUNC_START(name) _GLOBAL(name)
		290	#define FUNC_END(name)
		291
289	/*	292	/*
290	* LOAD_REG_IMMEDIATE(rn, expr)	293	* LOAD_REG_IMMEDIATE(rn, expr)
291	* Loads the value of the constant expression 'expr' into register 'rn'	294	* Loads the value of the constant expression 'expr' into register 'rn'

arch/powerpc/kernel/iomap.c+24−0

@@ -38,6 +38,18 @@ EXPORT_SYMBOL(ioread16); · 37 unmodified lines
38	EXPORT_SYMBOL(ioread16be);	38	EXPORT_SYMBOL(ioread16be);
39	EXPORT_SYMBOL(ioread32);	39	EXPORT_SYMBOL(ioread32);
40	EXPORT_SYMBOL(ioread32be);	40	EXPORT_SYMBOL(ioread32be);
		41	#ifdef __powerpc64__
		42	u64 ioread64(void __iomem *addr)
		43	{
		44	return readq(addr);
		45	}
		46	u64 ioread64be(void __iomem *addr)
		47	{
		48	return readq_be(addr);
		49	}
		50	EXPORT_SYMBOL(ioread64);
		51	EXPORT_SYMBOL(ioread64be);
		52	#endif /* __powerpc64__ */
41		53
42	void iowrite8(u8 val, void __iomem *addr)	54	void iowrite8(u8 val, void __iomem *addr)
43	{	55	{
@@ -64,6 +76,18 @@ EXPORT_SYMBOL(iowrite16); · 20 unmodified lines
64	EXPORT_SYMBOL(iowrite16be);	76	EXPORT_SYMBOL(iowrite16be);
65	EXPORT_SYMBOL(iowrite32);	77	EXPORT_SYMBOL(iowrite32);
66	EXPORT_SYMBOL(iowrite32be);	78	EXPORT_SYMBOL(iowrite32be);
		79	#ifdef __powerpc64__
		80	void iowrite64(u64 val, void __iomem *addr)
		81	{
		82	writeq(val, addr);
		83	}
		84	void iowrite64be(u64 val, void __iomem *addr)
		85	{
		86	writeq_be(val, addr);
		87	}
		88	EXPORT_SYMBOL(iowrite64);
		89	EXPORT_SYMBOL(iowrite64be);
		90	#endif /* __powerpc64__ */
67		91
68	/*	92	/*
69	* These are the "repeat read/write" functions. Note the	93	* These are the "repeat read/write" functions. Note the

arch/s390/crypto/aes_s390.c+60−53

 #include <crypto/aes.h>
 #include <crypto/algapi.h>
+#include <crypto/internal/skcipher.h>
 #include <linux/err.h>
 #include <linux/module.h>
 #include <linux/cpufeature.h>
 	long dec;
 	int key_len;
 	union {
-		struct crypto_blkcipher *blk;
+		struct crypto_skcipher *blk;
 		struct crypto_cipher *cip;
 	} fallback;
 };
 	long enc;
 	long dec;
 	int key_len;
-	struct crypto_blkcipher *fallback;
+	struct crypto_skcipher *fallback;
 };
 /*
 	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
 	unsigned int ret;
-	sctx->fallback.blk->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
+	crypto_skcipher_clear_flags(sctx->fallback.blk, CRYPTO_TFM_REQ_MASK);
-	sctx->fallback.blk->base.crt_flags |= (tfm->crt_flags &
+	crypto_skcipher_set_flags(sctx->fallback.blk, tfm->crt_flags &
-			CRYPTO_TFM_REQ_MASK);
+						      CRYPTO_TFM_REQ_MASK);
+	ret = crypto_skcipher_setkey(sctx->fallback.blk, key, len);
+	tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
+	tfm->crt_flags |= crypto_skcipher_get_flags(sctx->fallback.blk) &
+			  CRYPTO_TFM_RES_MASK;
-	ret = crypto_blkcipher_setkey(sctx->fallback.blk, key, len);
-	if (ret) {
-		tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
-		tfm->crt_flags |= (sctx->fallback.blk->base.crt_flags &
-				CRYPTO_TFM_RES_MASK);
-	}
 	return ret;
 }
 		unsigned int nbytes)
 {
 	unsigned int ret;
-	struct crypto_blkcipher *tfm;
+	struct crypto_blkcipher *tfm = desc->tfm;
-	struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm);
+	struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(tfm);
+	SKCIPHER_REQUEST_ON_STACK(req, sctx->fallback.blk);
-	tfm = desc->tfm;
+	skcipher_request_set_tfm(req, sctx->fallback.blk);
-	desc->tfm = sctx->fallback.blk;
+	skcipher_request_set_callback(req, desc->flags, NULL, NULL);
+	skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
-	ret = crypto_blkcipher_decrypt_iv(desc, dst, src, nbytes);
+	ret = crypto_skcipher_decrypt(req);
-	desc->tfm = tfm;
+	skcipher_request_zero(req);
 	return ret;
 }
 		unsigned int nbytes)
 {
 	unsigned int ret;
-	struct crypto_blkcipher *tfm;
+	struct crypto_blkcipher *tfm = desc->tfm;
-	struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm);
+	struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(tfm);
+	SKCIPHER_REQUEST_ON_STACK(req, sctx->fallback.blk);
-	tfm = desc->tfm;
+	skcipher_request_set_tfm(req, sctx->fallback.blk);
-	desc->tfm = sctx->fallback.blk;
+	skcipher_request_set_callback(req, desc->flags, NULL, NULL);
+	skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
-	ret = crypto_blkcipher_encrypt_iv(desc, dst, src, nbytes);
+	ret = crypto_skcipher_encrypt(req);
-	desc->tfm = tfm;
 	return ret;
 }
 	const char *name = tfm->__crt_alg->cra_name;
 	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
-	sctx->fallback.blk = crypto_alloc_blkcipher(name, 0,
+	sctx->fallback.blk = crypto_alloc_skcipher(name, 0,
-			CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
+						   CRYPTO_ALG_ASYNC |
+						   CRYPTO_ALG_NEED_FALLBACK);
 	if (IS_ERR(sctx->fallback.blk)) {
 		pr_err("Allocating AES fallback algorithm %s failed\n",
 {
 	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
-	crypto_free_blkcipher(sctx->fallback.blk);
+	crypto_free_skcipher(sctx->fallback.blk);
-	sctx->fallback.blk = NULL;
 }
 static struct crypto_alg ecb_aes_alg = {
 	struct s390_xts_ctx *xts_ctx = crypto_tfm_ctx(tfm);
 	unsigned int ret;
-	xts_ctx->fallback->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
+	crypto_skcipher_clear_flags(xts_ctx->fallback, CRYPTO_TFM_REQ_MASK);
-	xts_ctx->fallback->base.crt_flags |= (tfm->crt_flags &
+	crypto_skcipher_set_flags(xts_ctx->fallback, tfm->crt_flags &
-			CRYPTO_TFM_REQ_MASK);
+						     CRYPTO_TFM_REQ_MASK);
+	ret = crypto_skcipher_setkey(xts_ctx->fallback, key, len);
+	tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
+	tfm->crt_flags |= crypto_skcipher_get_flags(xts_ctx->fallback) &
+			  CRYPTO_TFM_RES_MASK;
-	ret = crypto_blkcipher_setkey(xts_ctx->fallback, key, len);
-	if (ret) {
-		tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
-		tfm->crt_flags |= (xts_ctx->fallback->base.crt_flags &
-				CRYPTO_TFM_RES_MASK);
-	}
 	return ret;
 }
 		struct scatterlist *dst, struct scatterlist *src,
 		unsigned int nbytes)
 {
-	struct s390_xts_ctx *xts_ctx = crypto_blkcipher_ctx(desc->tfm);
+	struct crypto_blkcipher *tfm = desc->tfm;
-	struct crypto_blkcipher *tfm;
+	struct s390_xts_ctx *xts_ctx = crypto_blkcipher_ctx(tfm);
+	SKCIPHER_REQUEST_ON_STACK(req, xts_ctx->fallback);
 	unsigned int ret;
-	tfm = desc->tfm;
+	skcipher_request_set_tfm(req, xts_ctx->fallback);
-	desc->tfm = xts_ctx->fallback;
+	skcipher_request_set_callback(req, desc->flags, NULL, NULL);
+	skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
-	ret = crypto_blkcipher_decrypt_iv(desc, dst, src, nbytes);
+	ret = crypto_skcipher_decrypt(req);
-	desc->tfm = tfm;
+	skcipher_request_zero(req);
 	return ret;
 }
 		struct scatterlist *dst, struct scatterlist *src,
 		unsigned int nbytes)
 {
-	struct s390_xts_ctx *xts_ctx = crypto_blkcipher_ctx(desc->tfm);
+	struct crypto_blkcipher *tfm = desc->tfm;
-	struct crypto_blkcipher *tfm;
+	struct s390_xts_ctx *xts_ctx = crypto_blkcipher_ctx(tfm);
+	SKCIPHER_REQUEST_ON_STACK(req, xts_ctx->fallback);
 	unsigned int ret;
-	tfm = desc->tfm;
+	skcipher_request_set_tfm(req, xts_ctx->fallback);
-	desc->tfm = xts_ctx->fallback;
+	skcipher_request_set_callback(req, desc->flags, NULL, NULL);
+	skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
-	ret = crypto_blkcipher_encrypt_iv(desc, dst, src, nbytes);
+	ret = crypto_skcipher_encrypt(req);
-	desc->tfm = tfm;
+	skcipher_request_zero(req);
 	return ret;
 }
 	const char *name = tfm->__crt_alg->cra_name;
 	struct s390_xts_ctx *xts_ctx = crypto_tfm_ctx(tfm);
-	xts_ctx->fallback = crypto_alloc_blkcipher(name, 0,
+	xts_ctx->fallback = crypto_alloc_skcipher(name, 0,
-			CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
+						  CRYPTO_ALG_ASYNC |
+						  CRYPTO_ALG_NEED_FALLBACK);
 	if (IS_ERR(xts_ctx->fallback)) {
 		pr_err("Allocating XTS fallback algorithm %s failed\n",
 {
 	struct s390_xts_ctx *xts_ctx = crypto_tfm_ctx(tfm);
-	crypto_free_blkcipher(xts_ctx->fallback);
+	crypto_free_skcipher(xts_ctx->fallback);
-	xts_ctx->fallback = NULL;
 }
 static struct crypto_alg xts_aes_alg = {

arch/x86/crypto/Makefile+3−1

 ifeq ($(avx2_supported),yes)
 	obj-$(CONFIG_CRYPTO_CAMELLIA_AESNI_AVX2_X86_64) += camellia-aesni-avx2.o
 	obj-$(CONFIG_CRYPTO_SERPENT_AVX2_X86_64) += serpent-avx2.o
-	obj-$(CONFIG_CRYPTO_SHA1_MB) += sha-mb/
+	obj-$(CONFIG_CRYPTO_SHA1_MB) += sha1-mb/
+	obj-$(CONFIG_CRYPTO_SHA256_MB) += sha256-mb/
+	obj-$(CONFIG_CRYPTO_SHA512_MB) += sha512-mb/
 endif
 aes-i586-y := aes-i586-asm_32.o aes_glue.o

arch/x86/crypto/aesni-intel_glue.c+23−71

@@ -59,17 +59,6 @@ struct aesni_rfc4106_gcm_ctx { · 58 unmodified lines
59	u8 nonce[4];	59	u8 nonce[4];
60	};	60	};
61		61
62	struct aesni_gcm_set_hash_subkey_result {
63	int err;
64	struct completion completion;
65	};
66
67	struct aesni_hash_subkey_req_data {
68	u8 iv[16];
69	struct aesni_gcm_set_hash_subkey_result result;
70	struct scatterlist sg;
71	};
72
73	struct aesni_lrw_ctx {	62	struct aesni_lrw_ctx {
74	struct lrw_table_ctx lrw_table;	63	struct lrw_table_ctx lrw_table;
75	u8 raw_aes_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1];	64	u8 raw_aes_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1];
@@ -809,71 +798,28 @@ static void rfc4106_exit(struct crypto_aead *aead) · 733 unmodified lines
809	cryptd_free_aead(*ctx);	798	cryptd_free_aead(*ctx);
810	}	799	}
811		800
812	static void
813	rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err)
814	{
815	struct aesni_gcm_set_hash_subkey_result *result = req->data;
816
817	if (err == -EINPROGRESS)
818	return;
819	result->err = err;
820	complete(&result->completion);
821	}
822
823	static int	801	static int
824	rfc4106_set_hash_subkey(u8 hash_subkey, const u8 key, unsigned int key_len)	802	rfc4106_set_hash_subkey(u8 hash_subkey, const u8 key, unsigned int key_len)
825	{	803	{
826	struct crypto_ablkcipher *ctr_tfm;	804	struct crypto_cipher *tfm;
827	struct ablkcipher_request *req;	805	int ret;
828	int ret = -EINVAL;
829	struct aesni_hash_subkey_req_data *req_data;
830		806
831	ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0);	807	tfm = crypto_alloc_cipher("aes", 0, 0);
832	if (IS_ERR(ctr_tfm))	808	if (IS_ERR(tfm))
833	return PTR_ERR(ctr_tfm);	809	return PTR_ERR(tfm);
834		810
835	ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len);	811	ret = crypto_cipher_setkey(tfm, key, key_len);
836	if (ret)	812	if (ret)
837	goto out_free_ablkcipher;	813	goto out_free_cipher;
838
839	ret = -ENOMEM;
840	req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL);
841	if (!req)
842	goto out_free_ablkcipher;
843
844	req_data = kmalloc(sizeof(*req_data), GFP_KERNEL);
845	if (!req_data)
846	goto out_free_request;
847
848	memset(req_data->iv, 0, sizeof(req_data->iv));
849		814
850	/* Clear the data in the hash sub key container to zero.*/	815	/* Clear the data in the hash sub key container to zero.*/
851	/* We want to cipher all zeros to create the hash sub key. */	816	/* We want to cipher all zeros to create the hash sub key. */
852	memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);	817	memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
853		818
854	init_completion(&req_data->result.completion);	819	crypto_cipher_encrypt_one(tfm, hash_subkey, hash_subkey);
855	sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE);	820
856	ablkcipher_request_set_tfm(req, ctr_tfm);	821	out_free_cipher:
857	ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP \|	822	crypto_free_cipher(tfm);
858	CRYPTO_TFM_REQ_MAY_BACKLOG,
859	rfc4106_set_hash_subkey_done,
860	&req_data->result);
861
862	ablkcipher_request_set_crypt(req, &req_data->sg,
863	&req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv);
864
865	ret = crypto_ablkcipher_encrypt(req);
866	if (ret == -EINPROGRESS \|\| ret == -EBUSY) {
867	ret = wait_for_completion_interruptible
868	(&req_data->result.completion);
869	if (!ret)
870	ret = req_data->result.err;
871	}
872	kfree(req_data);
873	out_free_request:
874	ablkcipher_request_free(req);
875	out_free_ablkcipher:
876	crypto_free_ablkcipher(ctr_tfm);
877	return ret;	823	return ret;
878	}	824	}
879		825
@@ -1098,9 +1044,12 @@ static int rfc4106_encrypt(struct aead_request *req) · 218 unmodified lines
1098	struct cryptd_aead **ctx = crypto_aead_ctx(tfm);	1044	struct cryptd_aead **ctx = crypto_aead_ctx(tfm);
1099	struct cryptd_aead cryptd_tfm = ctx;	1045	struct cryptd_aead cryptd_tfm = ctx;
1100		1046
1101	aead_request_set_tfm(req, irq_fpu_usable() ?	1047	tfm = &cryptd_tfm->base;
1102	cryptd_aead_child(cryptd_tfm) :	1048	if (irq_fpu_usable() && (!in_atomic() \|\|
1103	&cryptd_tfm->base);	1049	!cryptd_aead_queued(cryptd_tfm)))
		1050	tfm = cryptd_aead_child(cryptd_tfm);
		1051
		1052	aead_request_set_tfm(req, tfm);
1104		1053
1105	return crypto_aead_encrypt(req);	1054	return crypto_aead_encrypt(req);
1106	}	1055	}
@@ -1111,9 +1060,12 @@ static int rfc4106_decrypt(struct aead_request *req) · 4 unmodified lines
1111	struct cryptd_aead **ctx = crypto_aead_ctx(tfm);	1060	struct cryptd_aead **ctx = crypto_aead_ctx(tfm);
1112	struct cryptd_aead cryptd_tfm = ctx;	1061	struct cryptd_aead cryptd_tfm = ctx;
1113		1062
1114	aead_request_set_tfm(req, irq_fpu_usable() ?	1063	tfm = &cryptd_tfm->base;
1115	cryptd_aead_child(cryptd_tfm) :	1064	if (irq_fpu_usable() && (!in_atomic() \|\|
1116	&cryptd_tfm->base);	1065	!cryptd_aead_queued(cryptd_tfm)))
		1066	tfm = cryptd_aead_child(cryptd_tfm);
		1067
		1068	aead_request_set_tfm(req, tfm);
1117		1069
1118	return crypto_aead_decrypt(req);	1070	return crypto_aead_decrypt(req);
1119	}	1071	}

arch/x86/crypto/chacha20_glue.c+1−1

@@ -70,7 +70,7 @@ static int chacha20_simd(struct blkcipher_desc desc, struct scatterlist dst, · 69 unmodified lines
70	struct blkcipher_walk walk;	70	struct blkcipher_walk walk;
71	int err;	71	int err;
72		72
73	if (!may_use_simd())	73	if (nbytes <= CHACHA20_BLOCK_SIZE \|\| !may_use_simd())
74	return crypto_chacha20_crypt(desc, dst, src, nbytes);	74	return crypto_chacha20_crypt(desc, dst, src, nbytes);
75		75
76	state = (u32 *)roundup((uintptr_t)state_buf, CHACHA20_STATE_ALIGN);	76	state = (u32 *)roundup((uintptr_t)state_buf, CHACHA20_STATE_ALIGN);

arch/x86/crypto/ghash-clmulni-intel_glue.c+17−23

 	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
 	struct ahash_request *cryptd_req = ahash_request_ctx(req);
 	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
+	struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
+	struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
-	if (!irq_fpu_usable()) {
+	desc->tfm = child;
-		memcpy(cryptd_req, req, sizeof(*req));
+	desc->flags = req->base.flags;
-		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
+	return crypto_shash_init(desc);
-		return crypto_ahash_init(cryptd_req);
-	} else {
-		struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
-		struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
-		desc->tfm = child;
-		desc->flags = req->base.flags;
-		return crypto_shash_init(desc);
-	}
 }
 static int ghash_async_update(struct ahash_request *req)
 {
 	struct ahash_request *cryptd_req = ahash_request_ctx(req);
+	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
+	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
-	if (!irq_fpu_usable()) {
+	if (!irq_fpu_usable() ||
-		struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+	    (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
-		struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
-		struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
 		memcpy(cryptd_req, req, sizeof(*req));
 		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
 		return crypto_ahash_update(cryptd_req);
 static int ghash_async_final(struct ahash_request *req)
 {
 	struct ahash_request *cryptd_req = ahash_request_ctx(req);
+	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
+	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
-	if (!irq_fpu_usable()) {
+	if (!irq_fpu_usable() ||
-		struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+	    (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
-		struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
-		struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
 		memcpy(cryptd_req, req, sizeof(*req));
 		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
 		return crypto_ahash_final(cryptd_req);
 	struct ahash_request *cryptd_req = ahash_request_ctx(req);
 	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
-	if (!irq_fpu_usable()) {
+	if (!irq_fpu_usable() ||
+	    (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
 		memcpy(cryptd_req, req, sizeof(*req));
 		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
 		return crypto_ahash_digest(cryptd_req);

arch/x86/crypto/sha1-mb/Makefilerenamed from arch/x86/crypto/sha-mb/Makefile+0−0

File renamed without changes.

arch/x86/crypto/sha1-mb/sha1_mb.crenamed from arch/x86/crypto/sha-mb/sha1_mb.c+173−115

 #include <asm/byteorder.h>
 #include <linux/hardirq.h>
 #include <asm/fpu/api.h>
-#include "sha_mb_ctx.h"
+#include "sha1_mb_ctx.h"
 #define FLUSH_INTERVAL 1000 /* in usec */
 	struct mcryptd_ahash *mcryptd_tfm;
 };
-static inline struct mcryptd_hash_request_ctx *cast_hash_to_mcryptd_ctx(struct sha1_hash_ctx *hash_ctx)
+static inline struct mcryptd_hash_request_ctx
+		*cast_hash_to_mcryptd_ctx(struct sha1_hash_ctx *hash_ctx)
 {
-	struct shash_desc *desc;
+	struct ahash_request *areq;
-	desc = container_of((void *) hash_ctx, struct shash_desc, __ctx);
+	areq = container_of((void *) hash_ctx, struct ahash_request, __ctx);
-	return container_of(desc, struct mcryptd_hash_request_ctx, desc);
+	return container_of(areq, struct mcryptd_hash_request_ctx, areq);
 }
-static inline struct ahash_request *cast_mcryptd_ctx_to_req(struct mcryptd_hash_request_ctx *ctx)
+static inline struct ahash_request
+		*cast_mcryptd_ctx_to_req(struct mcryptd_hash_request_ctx *ctx)
 {
 	return container_of((void *) ctx, struct ahash_request, __ctx);
 }
 static void req_ctx_init(struct mcryptd_hash_request_ctx *rctx,
-				struct shash_desc *desc)
+				struct ahash_request *areq)
 {
 	rctx->flag = HASH_UPDATE;
 }
 static asmlinkage void (*sha1_job_mgr_init)(struct sha1_mb_mgr *state);
-static asmlinkage struct job_sha1* (*sha1_job_mgr_submit)(struct sha1_mb_mgr *state,
+static asmlinkage struct job_sha1* (*sha1_job_mgr_submit)
-							  struct job_sha1 *job);
+			(struct sha1_mb_mgr *state, struct job_sha1 *job);
-static asmlinkage struct job_sha1* (*sha1_job_mgr_flush)(struct sha1_mb_mgr *state);
+static asmlinkage struct job_sha1* (*sha1_job_mgr_flush)
-static asmlinkage struct job_sha1* (*sha1_job_mgr_get_comp_job)(struct sha1_mb_mgr *state);
+						(struct sha1_mb_mgr *state);
+static asmlinkage struct job_sha1* (*sha1_job_mgr_get_comp_job)
+						(struct sha1_mb_mgr *state);
 static inline void sha1_init_digest(uint32_t *digest)
 {
 	return i >> SHA1_LOG2_BLOCK_SIZE;
 }
-static struct sha1_hash_ctx *sha1_ctx_mgr_resubmit(struct sha1_ctx_mgr *mgr, struct sha1_hash_ctx *ctx)
+static struct sha1_hash_ctx *sha1_ctx_mgr_resubmit(struct sha1_ctx_mgr *mgr,
+						struct sha1_hash_ctx *ctx)
 {
 	while (ctx) {
 		if (ctx->status & HASH_CTX_STS_COMPLETE) {
 				ctx->job.buffer = (uint8_t *) buffer;
 				ctx->job.len = len;
-				ctx = (struct sha1_hash_ctx *) sha1_job_mgr_submit(&mgr->mgr,
+				ctx = (struct sha1_hash_ctx *)sha1_job_mgr_submit(&mgr->mgr,
-										  &ctx->job);
+										&ctx->job);
 				continue;
 			}
 		}
 		if (ctx->status & HASH_CTX_STS_LAST) {
 			uint8_t *buf = ctx->partial_block_buffer;
-			uint32_t n_extra_blocks = sha1_pad(buf, ctx->total_length);
+			uint32_t n_extra_blocks =
+					sha1_pad(buf, ctx->total_length);
 			ctx->status = (HASH_CTX_STS_PROCESSING |
 				       HASH_CTX_STS_COMPLETE);
 			ctx->job.buffer = buf;
 			ctx->job.len = (uint32_t) n_extra_blocks;
-			ctx = (struct sha1_hash_ctx *) sha1_job_mgr_submit(&mgr->mgr, &ctx->job);
+			ctx = (struct sha1_hash_ctx *)
+				sha1_job_mgr_submit(&mgr->mgr, &ctx->job);
 			continue;
 		}
 	return NULL;
 }
-static struct sha1_hash_ctx *sha1_ctx_mgr_get_comp_ctx(struct sha1_ctx_mgr *mgr)
+static struct sha1_hash_ctx
+			*sha1_ctx_mgr_get_comp_ctx(struct sha1_ctx_mgr *mgr)
 {
 	/*
 	 * If get_comp_job returns NULL, there are no jobs complete.
-	 * If get_comp_job returns a job, verify that it is safe to return to the user.
+	 * If get_comp_job returns a job, verify that it is safe to return to
+	 * the user.
 	 * If it is not ready, resubmit the job to finish processing.
 	 * If sha1_ctx_mgr_resubmit returned a job, it is ready to be returned.
-	 * Otherwise, all jobs currently being managed by the hash_ctx_mgr still need processing.
+	 * Otherwise, all jobs currently being managed by the hash_ctx_mgr
+	 * still need processing.
 	 */
 	struct sha1_hash_ctx *ctx;
 					  int flags)
 {
 	if (flags & (~HASH_ENTIRE)) {
-		/* User should not pass anything other than FIRST, UPDATE, or LAST */
+		/*
+		 * User should not pass anything other than FIRST, UPDATE, or
+		 * LAST
+		 */
 		ctx->error = HASH_CTX_ERROR_INVALID_FLAGS;
 		return ctx;
 	}
 		ctx->partial_block_buffer_length = 0;
 	}
-	/* If we made it here, there were no errors during this call to submit */
+	/*
+	 * If we made it here, there were no errors during this call to
+	 * submit
+	 */
 	ctx->error = HASH_CTX_ERROR_NONE;
 	/* Store buffer ptr info from user */
 	ctx->incoming_buffer = buffer;
 	ctx->incoming_buffer_length = len;
-	/* Store the user's request flags and mark this ctx as currently being processed. */
+	/*
+	 * Store the user's request flags and mark this ctx as currently
+	 * being processed.
+	 */
 	ctx->status = (flags & HASH_LAST) ?
 			(HASH_CTX_STS_PROCESSING | HASH_CTX_STS_LAST) :
 			HASH_CTX_STS_PROCESSING;
 	 * Or if the user's buffer contains less than a whole block,
 	 * append as much as possible to the extra block.
 	 */
-	if ((ctx->partial_block_buffer_length) | (len < SHA1_BLOCK_SIZE)) {
+	if (ctx->partial_block_buffer_length || len < SHA1_BLOCK_SIZE) {
-		/* Compute how many bytes to copy from user buffer into extra block */
+		/*
-		uint32_t copy_len = SHA1_BLOCK_SIZE - ctx->partial_block_buffer_length;
+		 * Compute how many bytes to copy from user buffer into
+		 * extra block
+		 */
+		uint32_t copy_len = SHA1_BLOCK_SIZE -
+					ctx->partial_block_buffer_length;
 		if (len < copy_len)
 			copy_len = len;
 				buffer, copy_len);
 			ctx->partial_block_buffer_length += copy_len;
-			ctx->incoming_buffer = (const void *)((const char *)buffer + copy_len);
+			ctx->incoming_buffer = (const void *)
+					((const char *)buffer + copy_len);
 			ctx->incoming_buffer_length = len - copy_len;
 		}
-		/* The extra block should never contain more than 1 block here */
+		/*
+		 * The extra block should never contain more than 1 block
+		 * here
+		 */
 		assert(ctx->partial_block_buffer_length <= SHA1_BLOCK_SIZE);
-		/* If the extra block buffer contains exactly 1 block, it can be hashed. */
+		/*
+		 * If the extra block buffer contains exactly 1 block, it can
+		 * be hashed.
+		 */
 		if (ctx->partial_block_buffer_length >= SHA1_BLOCK_SIZE) {
 			ctx->partial_block_buffer_length = 0;
 			ctx->job.buffer = ctx->partial_block_buffer;
 			ctx->job.len = 1;
-			ctx = (struct sha1_hash_ctx *) sha1_job_mgr_submit(&mgr->mgr, &ctx->job);
+			ctx = (struct sha1_hash_ctx *)
+				sha1_job_mgr_submit(&mgr->mgr, &ctx->job);
 		}
 	}
 			return NULL;
 		/*
-		 * If flush returned a job, resubmit the job to finish processing.
+		 * If flush returned a job, resubmit the job to finish
+		 * processing.
 		 */
 		ctx = sha1_ctx_mgr_resubmit(mgr, ctx);
 		/*
-		 * If sha1_ctx_mgr_resubmit returned a job, it is ready to be returned.
+		 * If sha1_ctx_mgr_resubmit returned a job, it is ready to be
-		 * Otherwise, all jobs currently being managed by the sha1_ctx_mgr
+		 * returned. Otherwise, all jobs currently being managed by the
-		 * still need processing. Loop.
+		 * sha1_ctx_mgr still need processing. Loop.
 		 */
 		if (ctx)
 			return ctx;
 	}
 }
-static int sha1_mb_init(struct shash_desc *desc)
+static int sha1_mb_init(struct ahash_request *areq)
 {
-	struct sha1_hash_ctx *sctx = shash_desc_ctx(desc);
+	struct sha1_hash_ctx *sctx = ahash_request_ctx(areq);
 	hash_ctx_init(sctx);
 	sctx->job.result_digest[0] = SHA1_H0;
 static int sha1_mb_set_results(struct mcryptd_hash_request_ctx *rctx)
 {
 	int	i;
-	struct	sha1_hash_ctx *sctx = shash_desc_ctx(&rctx->desc);
+	struct	sha1_hash_ctx *sctx = ahash_request_ctx(&rctx->areq);
 	__be32	*dst = (__be32 *) rctx->out;
 	for (i = 0; i < 5; ++i)
 				flag |= HASH_LAST;
 		}
-		sha_ctx = (struct sha1_hash_ctx *) shash_desc_ctx(&rctx->desc);
+		sha_ctx = (struct sha1_hash_ctx *)
+						ahash_request_ctx(&rctx->areq);
 		kernel_fpu_begin();
-		sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, nbytes, flag);
+		sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx,
+						rctx->walk.data, nbytes, flag);
 		if (!sha_ctx) {
 			if (flush)
 				sha_ctx = sha1_ctx_mgr_flush(cstate->mgr);
 	mcryptd_arm_flusher(cstate, delay);
 }
-static int sha1_mb_update(struct shash_desc *desc, const u8 *data,
+static int sha1_mb_update(struct ahash_request *areq)
-			  unsigned int len)
 {
 	struct mcryptd_hash_request_ctx *rctx =
-			container_of(desc, struct mcryptd_hash_request_ctx, desc);
+		container_of(areq, struct mcryptd_hash_request_ctx, areq);
 	struct mcryptd_alg_cstate *cstate =
 				this_cpu_ptr(sha1_mb_alg_state.alg_cstate);
 	}
 	/* need to init context */
-	req_ctx_init(rctx, desc);
+	req_ctx_init(rctx, areq);
 	nbytes = crypto_ahash_walk_first(req, &rctx->walk);
 		rctx->flag |= HASH_DONE;
 	/* submit */
-	sha_ctx = (struct sha1_hash_ctx *) shash_desc_ctx(desc);
+	sha_ctx = (struct sha1_hash_ctx *) ahash_request_ctx(areq);
 	sha1_mb_add_list(rctx, cstate);
 	kernel_fpu_begin();
-	sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, nbytes, HASH_UPDATE);
+	sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data,
+							nbytes, HASH_UPDATE);
 	kernel_fpu_end();
 	/* check if anything is returned */
 	return ret;
 }
-static int sha1_mb_finup(struct shash_desc *desc, const u8 *data,
+static int sha1_mb_finup(struct ahash_request *areq)
-			     unsigned int len, u8 *out)
 {
 	struct mcryptd_hash_request_ctx *rctx =
-			container_of(desc, struct mcryptd_hash_request_ctx, desc);
+		container_of(areq, struct mcryptd_hash_request_ctx, areq);
 	struct mcryptd_alg_cstate *cstate =
 				this_cpu_ptr(sha1_mb_alg_state.alg_cstate);
 	}
 	/* need to init context */
-	req_ctx_init(rctx, desc);
+	req_ctx_init(rctx, areq);
 	nbytes = crypto_ahash_walk_first(req, &rctx->walk);
 		rctx->flag |= HASH_DONE;
 		flag = HASH_LAST;
 	}
-	rctx->out = out;
 	/* submit */
 	rctx->flag |= HASH_FINAL;
-	sha_ctx = (struct sha1_hash_ctx *) shash_desc_ctx(desc);
+	sha_ctx = (struct sha1_hash_ctx *) ahash_request_ctx(areq);
 	sha1_mb_add_list(rctx, cstate);
 	kernel_fpu_begin();
-	sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, nbytes, flag);
+	sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data,
+								nbytes, flag);
 	kernel_fpu_end();
 	/* check if anything is returned */
 	return ret;
 }
-static int sha1_mb_final(struct shash_desc *desc, u8 *out)
+static int sha1_mb_final(struct ahash_request *areq)
 {
 	struct mcryptd_hash_request_ctx *rctx =
-			container_of(desc, struct mcryptd_hash_request_ctx, desc);
+		container_of(areq, struct mcryptd_hash_request_ctx, areq);
 	struct mcryptd_alg_cstate *cstate =
 				this_cpu_ptr(sha1_mb_alg_state.alg_cstate);
 	}
 	/* need to init context */
-	req_ctx_init(rctx, desc);
+	req_ctx_init(rctx, areq);
-	rctx->out = out;
 	rctx->flag |= HASH_DONE | HASH_FINAL;
-	sha_ctx = (struct sha1_hash_ctx *) shash_desc_ctx(desc);
+	sha_ctx = (struct sha1_hash_ctx *) ahash_request_ctx(areq);
 	/* flag HASH_FINAL and 0 data size */
 	sha1_mb_add_list(rctx, cstate);
 	kernel_fpu_begin();
-	sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, &data, 0, HASH_LAST);
+	sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, &data, 0,
+								HASH_LAST);
 	kernel_fpu_end();
 	/* check if anything is returned */
 	return ret;
 }
-static int sha1_mb_export(struct shash_desc *desc, void *out)
+static int sha1_mb_export(struct ahash_request *areq, void *out)
 {
-	struct sha1_hash_ctx *sctx = shash_desc_ctx(desc);
+	struct sha1_hash_ctx *sctx = ahash_request_ctx(areq);
 	memcpy(out, sctx, sizeof(*sctx));
 	return 0;
 }
-static int sha1_mb_import(struct shash_desc *desc, const void *in)
+static int sha1_mb_import(struct ahash_request *areq, const void *in)
 {
-	struct sha1_hash_ctx *sctx = shash_desc_ctx(desc);
+	struct sha1_hash_ctx *sctx = ahash_request_ctx(areq);
 	memcpy(sctx, in, sizeof(*sctx));
 	return 0;
 }
+static int sha1_mb_async_init_tfm(struct crypto_tfm *tfm)
+{
+	struct mcryptd_ahash *mcryptd_tfm;
+	struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct mcryptd_hash_ctx *mctx;
-static struct shash_alg sha1_mb_shash_alg = {
+	mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha1-mb",
-	.digestsize	=	SHA1_DIGEST_SIZE,
+						CRYPTO_ALG_INTERNAL,
+						CRYPTO_ALG_INTERNAL);
+	if (IS_ERR(mcryptd_tfm))
+		return PTR_ERR(mcryptd_tfm);
+	mctx = crypto_ahash_ctx(&mcryptd_tfm->base);
+	mctx->alg_state = &sha1_mb_alg_state;
+	ctx->mcryptd_tfm = mcryptd_tfm;
+	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+				sizeof(struct ahash_request) +
+				crypto_ahash_reqsize(&mcryptd_tfm->base));
+	return 0;
+}
+static void sha1_mb_async_exit_tfm(struct crypto_tfm *tfm)
+{
+	struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm);
+	mcryptd_free_ahash(ctx->mcryptd_tfm);
+}
+static int sha1_mb_areq_init_tfm(struct crypto_tfm *tfm)
+{
+	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+				sizeof(struct ahash_request) +
+				sizeof(struct sha1_hash_ctx));
+	return 0;
+}
+static void sha1_mb_areq_exit_tfm(struct crypto_tfm *tfm)
+{
+	struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm);
+	mcryptd_free_ahash(ctx->mcryptd_tfm);
+}
+static struct ahash_alg sha1_mb_areq_alg = {
 	.init		=	sha1_mb_init,
 	.update		=	sha1_mb_update,
 	.final		=	sha1_mb_final,
 	.finup		=	sha1_mb_finup,
 	.export		=	sha1_mb_export,
 	.import		=	sha1_mb_import,
-	.descsize	=	sizeof(struct sha1_hash_ctx),
+	.halg		=	{
-	.statesize	=	sizeof(struct sha1_hash_ctx),
+		.digestsize	=	SHA1_DIGEST_SIZE,
-	.base		=	{
+		.statesize	=	sizeof(struct sha1_hash_ctx),
-		.cra_name	 = "__sha1-mb",
+		.base		=	{
-		.cra_driver_name = "__intel_sha1-mb",
+			.cra_name	 = "__sha1-mb",
-		.cra_priority	 = 100,
+			.cra_driver_name = "__intel_sha1-mb",
-		/*
+			.cra_priority	 = 100,
-		 * use ASYNC flag as some buffers in multi-buffer
+			/*
-		 * algo may not have completed before hashing thread sleep
+			 * use ASYNC flag as some buffers in multi-buffer
-		 */
+			 * algo may not have completed before hashing thread
-		.cra_flags	 = CRYPTO_ALG_TYPE_SHASH | CRYPTO_ALG_ASYNC |
+			 * sleep
-				   CRYPTO_ALG_INTERNAL,
+			 */
-		.cra_blocksize	 = SHA1_BLOCK_SIZE,
+			.cra_flags	= CRYPTO_ALG_TYPE_AHASH |
-		.cra_module	 = THIS_MODULE,
+						CRYPTO_ALG_ASYNC |
-		.cra_list	 = LIST_HEAD_INIT(sha1_mb_shash_alg.base.cra_list),
+						CRYPTO_ALG_INTERNAL,
+			.cra_blocksize	= SHA1_BLOCK_SIZE,
+			.cra_module	= THIS_MODULE,
+			.cra_list	= LIST_HEAD_INIT
+					(sha1_mb_areq_alg.halg.base.cra_list),
+			.cra_init	= sha1_mb_areq_init_tfm,
+			.cra_exit	= sha1_mb_areq_exit_tfm,
+			.cra_ctxsize	= sizeof(struct sha1_hash_ctx),
+		}
 	}
 };
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 	struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
 	struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
-	struct crypto_shash *child = mcryptd_ahash_child(mcryptd_tfm);
+	struct crypto_ahash *child = mcryptd_ahash_child(mcryptd_tfm);
 	struct mcryptd_hash_request_ctx *rctx;
-	struct shash_desc *desc;
+	struct ahash_request *areq;
 	memcpy(mcryptd_req, req, sizeof(*req));
 	ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
 	rctx = ahash_request_ctx(mcryptd_req);
-	desc = &rctx->desc;
+	areq = &rctx->areq;
-	desc->tfm = child;
-	desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
-	return crypto_ahash_import(mcryptd_req, in);
-}
-static int sha1_mb_async_init_tfm(struct crypto_tfm *tfm)
-{
-	struct mcryptd_ahash *mcryptd_tfm;
-	struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm);
-	struct mcryptd_hash_ctx *mctx;
-	mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha1-mb",
+	ahash_request_set_tfm(areq, child);
-					  CRYPTO_ALG_INTERNAL,
+	ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_SLEEP,
-					  CRYPTO_ALG_INTERNAL);
+					rctx->complete, req);
-	if (IS_ERR(mcryptd_tfm))
-		return PTR_ERR(mcryptd_tfm);
-	mctx = crypto_ahash_ctx(&mcryptd_tfm->base);
-	mctx->alg_state = &sha1_mb_alg_state;
-	ctx->mcryptd_tfm = mcryptd_tfm;
-	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
-				 sizeof(struct ahash_request) +
-				 crypto_ahash_reqsize(&mcryptd_tfm->base));
-	return 0;
+	return crypto_ahash_import(mcryptd_req, in);
-}
-static void sha1_mb_async_exit_tfm(struct crypto_tfm *tfm)
-{
-	struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm);
-	mcryptd_free_ahash(ctx->mcryptd_tfm);
 }
 static struct ahash_alg sha1_mb_async_alg = {
 		if (time_before(cur_time, rctx->tag.expire))
 			break;
 		kernel_fpu_begin();
-		sha_ctx = (struct sha1_hash_ctx *) sha1_ctx_mgr_flush(cstate->mgr);
+		sha_ctx = (struct sha1_hash_ctx *)
+					sha1_ctx_mgr_flush(cstate->mgr);
 		kernel_fpu_end();
 		if (!sha_ctx) {
 			pr_err("sha1_mb error: nothing got flushed for non-empty list\n");
 	}
 	sha1_mb_alg_state.flusher = &sha1_mb_flusher;
-	err = crypto_register_shash(&sha1_mb_shash_alg);
+	err = crypto_register_ahash(&sha1_mb_areq_alg);
 	if (err)
 		goto err2;
 	err = crypto_register_ahash(&sha1_mb_async_alg);
 	return 0;
 err1:
-	crypto_unregister_shash(&sha1_mb_shash_alg);
+	crypto_unregister_ahash(&sha1_mb_areq_alg);
 err2:
 	for_each_possible_cpu(cpu) {
 		cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu);
 	struct mcryptd_alg_cstate *cpu_state;
 	crypto_unregister_ahash(&sha1_mb_async_alg);
-	crypto_unregister_shash(&sha1_mb_shash_alg);
+	crypto_unregister_ahash(&sha1_mb_areq_alg);
 	for_each_possible_cpu(cpu) {
 		cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu);
 		kfree(cpu_state->mgr);

arch/x86/crypto/sha1-mb/sha1_mb_ctx.hrenamed from arch/x86/crypto/sha-mb/sha_mb_ctx.h+1−1

@@ -54,7 +54,7 @@ · 53 unmodified lines
54	#ifndef _SHA_MB_CTX_INTERNAL_H	54	#ifndef _SHA_MB_CTX_INTERNAL_H
55	#define _SHA_MB_CTX_INTERNAL_H	55	#define _SHA_MB_CTX_INTERNAL_H
56		56
57	#include "sha_mb_mgr.h"	57	#include "sha1_mb_mgr.h"
58		58
59	#define HASH_UPDATE 0x00	59	#define HASH_UPDATE 0x00
60	#define HASH_FIRST 0x01	60	#define HASH_FIRST 0x01

arch/x86/crypto/sha1-mb/sha1_mb_mgr.hrenamed from arch/x86/crypto/sha-mb/sha_mb_mgr.h+0−0

File renamed without changes.

arch/x86/crypto/sha1-mb/sha1_mb_mgr_datastruct.Srenamed from arch/x86/crypto/sha-mb/sha1_mb_mgr_datastruct.S+0−0

File renamed without changes.

arch/x86/crypto/sha1-mb/sha1_mb_mgr_flush_avx2.Srenamed from arch/x86/crypto/sha-mb/sha1_mb_mgr_flush_avx2.S+0−0

File renamed without changes.

arch/x86/crypto/sha1-mb/sha1_mb_mgr_init_avx2.crenamed from arch/x86/crypto/sha-mb/sha1_mb_mgr_init_avx2.c+1−1

@@ -51,7 +51,7 @@ · 50 unmodified lines
51	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.	51	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
52	*/	52	*/
53		53
54	#include "sha_mb_mgr.h"	54	#include "sha1_mb_mgr.h"
55		55
56	void sha1_mb_mgr_init_avx2(struct sha1_mb_mgr *state)	56	void sha1_mb_mgr_init_avx2(struct sha1_mb_mgr *state)
57	{	57	{

arch/x86/crypto/sha1-mb/sha1_mb_mgr_submit_avx2.Srenamed from arch/x86/crypto/sha-mb/sha1_mb_mgr_submit_avx2.S+0−0

File renamed without changes.

arch/x86/crypto/sha1-mb/sha1_x8_avx2.Srenamed from arch/x86/crypto/sha-mb/sha1_x8_avx2.S+0−0

File renamed without changes.

arch/x86/crypto/sha1_ssse3_glue.c+6−0

@@ -374,3 +374,9 @@ MODULE_LICENSE("GPL"); · 373 unmodified lines
374	MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, Supplemental SSE3 accelerated");	374	MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, Supplemental SSE3 accelerated");
375		375
376	MODULE_ALIAS_CRYPTO("sha1");	376	MODULE_ALIAS_CRYPTO("sha1");
		377	MODULE_ALIAS_CRYPTO("sha1-ssse3");
		378	MODULE_ALIAS_CRYPTO("sha1-avx");
		379	MODULE_ALIAS_CRYPTO("sha1-avx2");
		380	#ifdef CONFIG_AS_SHA1_NI
		381	MODULE_ALIAS_CRYPTO("sha1-ni");
		382	#endif

arch/x86/crypto/sha256-mb/Makefile+11−0

@@ -0,0 +1,11 @@
	1	#
	2	# Arch-specific CryptoAPI modules.
	3	#
	4
	5	avx2_supported := $(call as-instr,vpgatherdd %ymm0$(comma)(%eax$(comma)%ymm1\
	6	$(comma)4)$(comma)%ymm2,yes,no)
	7	ifeq ($(avx2_supported),yes)
	8	obj-$(CONFIG_CRYPTO_SHA256_MB) += sha256-mb.o
	9	sha256-mb-y := sha256_mb.o sha256_mb_mgr_flush_avx2.o \
	10	sha256_mb_mgr_init_avx2.o sha256_mb_mgr_submit_avx2.o sha256_x8_avx2.o
	11	endif

arch/x86/crypto/sha256-mb/sha256_mb.c+1030−0

@@ -0,0 +1,1030 @@
	1	/*
	2	* Multi buffer SHA256 algorithm Glue Code
	3	*
	4	* This file is provided under a dual BSD/GPLv2 license. When using or
	5	* redistributing this file, you may do so under either license.
	6	*
	7	* GPL LICENSE SUMMARY
	8	*
	9	* Copyright(c) 2016 Intel Corporation.
	10	*
	11	* This program is free software; you can redistribute it and/or modify
	12	* it under the terms of version 2 of the GNU General Public License as
	13	* published by the Free Software Foundation.
	14	*
	15	* This program is distributed in the hope that it will be useful, but
	16	* WITHOUT ANY WARRANTY; without even the implied warranty of
	17	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	18	* General Public License for more details.
	19	*
	20	* Contact Information:
	21	* Megha Dey <megha.dey@linux.intel.com>
	22	*
	23	* BSD LICENSE
	24	*
	25	* Copyright(c) 2016 Intel Corporation.
	26	*
	27	* Redistribution and use in source and binary forms, with or without
	28	* modification, are permitted provided that the following conditions
	29	* are met:
	30	*
	31	* * Redistributions of source code must retain the above copyright
	32	* notice, this list of conditions and the following disclaimer.
	33	* * Redistributions in binary form must reproduce the above copyright
	34	* notice, this list of conditions and the following disclaimer in
	35	* the documentation and/or other materials provided with the
	36	* distribution.
	37	* * Neither the name of Intel Corporation nor the names of its
	38	* contributors may be used to endorse or promote products derived
	39	* from this software without specific prior written permission.
	40	*
	41	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	42	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	43	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	44	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	45	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	46	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	47	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	48	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	49	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	50	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	51	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	52	*/
	53
	54	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
	55
	56	#include <crypto/internal/hash.h>
	57	#include <linux/init.h>
	58	#include <linux/module.h>
	59	#include <linux/mm.h>
	60	#include <linux/cryptohash.h>
	61	#include <linux/types.h>
	62	#include <linux/list.h>
	63	#include <crypto/scatterwalk.h>
	64	#include <crypto/sha.h>
	65	#include <crypto/mcryptd.h>
	66	#include <crypto/crypto_wq.h>
	67	#include <asm/byteorder.h>
	68	#include <linux/hardirq.h>
	69	#include <asm/fpu/api.h>
	70	#include "sha256_mb_ctx.h"
	71
	72	#define FLUSH_INTERVAL 1000 /* in usec */
	73
	74	static struct mcryptd_alg_state sha256_mb_alg_state;
	75
	76	struct sha256_mb_ctx {
	77	struct mcryptd_ahash *mcryptd_tfm;
	78	};
	79
	80	static inline struct mcryptd_hash_request_ctx
	81	cast_hash_to_mcryptd_ctx(struct sha256_hash_ctx hash_ctx)
	82	{
	83	struct ahash_request *areq;
	84
	85	areq = container_of((void *) hash_ctx, struct ahash_request, __ctx);
	86	return container_of(areq, struct mcryptd_hash_request_ctx, areq);
	87	}
	88
	89	static inline struct ahash_request
	90	cast_mcryptd_ctx_to_req(struct mcryptd_hash_request_ctx ctx)
	91	{
	92	return container_of((void *) ctx, struct ahash_request, __ctx);
	93	}
	94
	95	static void req_ctx_init(struct mcryptd_hash_request_ctx *rctx,
	96	struct ahash_request *areq)
	97	{
	98	rctx->flag = HASH_UPDATE;
	99	}
	100
	101	static asmlinkage void (sha256_job_mgr_init)(struct sha256_mb_mgr state);
	102	static asmlinkage struct job_sha256* (*sha256_job_mgr_submit)
	103	(struct sha256_mb_mgr state, struct job_sha256 job);
	104	static asmlinkage struct job_sha256* (*sha256_job_mgr_flush)
	105	(struct sha256_mb_mgr *state);
	106	static asmlinkage struct job_sha256* (*sha256_job_mgr_get_comp_job)
	107	(struct sha256_mb_mgr *state);
	108
	109	inline void sha256_init_digest(uint32_t *digest)
	110	{
	111	static const uint32_t initial_digest[SHA256_DIGEST_LENGTH] = {
	112	SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3,
	113	SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7};
	114	memcpy(digest, initial_digest, sizeof(initial_digest));
	115	}
	116
	117	inline uint32_t sha256_pad(uint8_t padblock[SHA256_BLOCK_SIZE * 2],
	118	uint32_t total_len)
	119	{
	120	uint32_t i = total_len & (SHA256_BLOCK_SIZE - 1);
	121
	122	memset(&padblock[i], 0, SHA256_BLOCK_SIZE);
	123	padblock[i] = 0x80;
	124
	125	i += ((SHA256_BLOCK_SIZE - 1) &
	126	(0 - (total_len + SHA256_PADLENGTHFIELD_SIZE + 1)))
	127	+ 1 + SHA256_PADLENGTHFIELD_SIZE;
	128
	129	#if SHA256_PADLENGTHFIELD_SIZE == 16
	130	((uint64_t ) &padblock[i - 16]) = 0;
	131	#endif
	132
	133	((uint64_t ) &padblock[i - 8]) = cpu_to_be64(total_len << 3);
	134
	135	/* Number of extra blocks to hash */
	136	return i >> SHA256_LOG2_BLOCK_SIZE;
	137	}
	138
	139	static struct sha256_hash_ctx
	140	sha256_ctx_mgr_resubmit(struct sha256_ctx_mgr mgr,
	141	struct sha256_hash_ctx *ctx)
	142	{
	143	while (ctx) {
	144	if (ctx->status & HASH_CTX_STS_COMPLETE) {
	145	/* Clear PROCESSING bit */
	146	ctx->status = HASH_CTX_STS_COMPLETE;
	147	return ctx;
	148	}
	149
	150	/*
	151	* If the extra blocks are empty, begin hashing what remains
	152	* in the user's buffer.
	153	*/
	154	if (ctx->partial_block_buffer_length == 0 &&
	155	ctx->incoming_buffer_length) {
	156
	157	const void *buffer = ctx->incoming_buffer;
	158	uint32_t len = ctx->incoming_buffer_length;
	159	uint32_t copy_len;
	160
	161	/*
	162	* Only entire blocks can be hashed.
	163	* Copy remainder to extra blocks buffer.
	164	*/
	165	copy_len = len & (SHA256_BLOCK_SIZE-1);
	166
	167	if (copy_len) {
	168	len -= copy_len;
	169	memcpy(ctx->partial_block_buffer,
	170	((const char *) buffer + len),
	171	copy_len);
	172	ctx->partial_block_buffer_length = copy_len;
	173	}
	174
	175	ctx->incoming_buffer_length = 0;
	176
	177	/* len should be a multiple of the block size now */
	178	assert((len % SHA256_BLOCK_SIZE) == 0);
	179
	180	/* Set len to the number of blocks to be hashed */
	181	len >>= SHA256_LOG2_BLOCK_SIZE;
	182
	183	if (len) {
	184
	185	ctx->job.buffer = (uint8_t *) buffer;
	186	ctx->job.len = len;
	187	ctx = (struct sha256_hash_ctx *)
	188	sha256_job_mgr_submit(&mgr->mgr, &ctx->job);
	189	continue;
	190	}
	191	}
	192
	193	/*
	194	* If the extra blocks are not empty, then we are
	195	* either on the last block(s) or we need more
	196	* user input before continuing.
	197	*/
	198	if (ctx->status & HASH_CTX_STS_LAST) {
	199
	200	uint8_t *buf = ctx->partial_block_buffer;
	201	uint32_t n_extra_blocks =
	202	sha256_pad(buf, ctx->total_length);
	203
	204	ctx->status = (HASH_CTX_STS_PROCESSING \|
	205	HASH_CTX_STS_COMPLETE);
	206	ctx->job.buffer = buf;
	207	ctx->job.len = (uint32_t) n_extra_blocks;
	208	ctx = (struct sha256_hash_ctx *)
	209	sha256_job_mgr_submit(&mgr->mgr, &ctx->job);
	210	continue;
	211	}
	212
	213	ctx->status = HASH_CTX_STS_IDLE;
	214	return ctx;
	215	}
	216
	217	return NULL;
	218	}
	219
	220	static struct sha256_hash_ctx
	221	sha256_ctx_mgr_get_comp_ctx(struct sha256_ctx_mgr mgr)
	222	{
	223	/*
	224	* If get_comp_job returns NULL, there are no jobs complete.
	225	* If get_comp_job returns a job, verify that it is safe to return to
	226	* the user. If it is not ready, resubmit the job to finish processing.
	227	* If sha256_ctx_mgr_resubmit returned a job, it is ready to be
	228	* returned. Otherwise, all jobs currently being managed by the
	229	* hash_ctx_mgr still need processing.
	230	*/
	231	struct sha256_hash_ctx *ctx;
	232
	233	ctx = (struct sha256_hash_ctx *) sha256_job_mgr_get_comp_job(&mgr->mgr);
	234	return sha256_ctx_mgr_resubmit(mgr, ctx);
	235	}
	236
	237	static void sha256_ctx_mgr_init(struct sha256_ctx_mgr *mgr)
	238	{
	239	sha256_job_mgr_init(&mgr->mgr);
	240	}
	241
	242	static struct sha256_hash_ctx sha256_ctx_mgr_submit(struct sha256_ctx_mgr mgr,
	243	struct sha256_hash_ctx *ctx,
	244	const void *buffer,
	245	uint32_t len,
	246	int flags)
	247	{
	248	if (flags & (~HASH_ENTIRE)) {
	249	/* User should not pass anything other than FIRST, UPDATE
	250	* or LAST
	251	*/
	252	ctx->error = HASH_CTX_ERROR_INVALID_FLAGS;
	253	return ctx;
	254	}
	255
	256	if (ctx->status & HASH_CTX_STS_PROCESSING) {
	257	/* Cannot submit to a currently processing job. */
	258	ctx->error = HASH_CTX_ERROR_ALREADY_PROCESSING;
	259	return ctx;
	260	}
	261
	262	if ((ctx->status & HASH_CTX_STS_COMPLETE) && !(flags & HASH_FIRST)) {
	263	/* Cannot update a finished job. */
	264	ctx->error = HASH_CTX_ERROR_ALREADY_COMPLETED;
	265	return ctx;
	266	}
	267
	268	if (flags & HASH_FIRST) {
	269	/* Init digest */
	270	sha256_init_digest(ctx->job.result_digest);
	271
	272	/* Reset byte counter */
	273	ctx->total_length = 0;
	274
	275	/* Clear extra blocks */
	276	ctx->partial_block_buffer_length = 0;
	277	}
	278
	279	/* If we made it here, there was no error during this call to submit */
	280	ctx->error = HASH_CTX_ERROR_NONE;
	281
	282	/* Store buffer ptr info from user */
	283	ctx->incoming_buffer = buffer;
	284	ctx->incoming_buffer_length = len;
	285
	286	/*
	287	* Store the user's request flags and mark this ctx as currently
	288	* being processed.
	289	*/
	290	ctx->status = (flags & HASH_LAST) ?
	291	(HASH_CTX_STS_PROCESSING \| HASH_CTX_STS_LAST) :
	292	HASH_CTX_STS_PROCESSING;
	293
	294	/* Advance byte counter */
	295	ctx->total_length += len;
	296
	297	/*
	298	* If there is anything currently buffered in the extra blocks,
	299	* append to it until it contains a whole block.
	300	* Or if the user's buffer contains less than a whole block,
	301	* append as much as possible to the extra block.
	302	*/
	303	if (ctx->partial_block_buffer_length \|\| len < SHA256_BLOCK_SIZE) {
	304	/*
	305	* Compute how many bytes to copy from user buffer into
	306	* extra block
	307	*/
	308	uint32_t copy_len = SHA256_BLOCK_SIZE -
	309	ctx->partial_block_buffer_length;
	310	if (len < copy_len)
	311	copy_len = len;
	312
	313	if (copy_len) {
	314	/* Copy and update relevant pointers and counters */
	315	memcpy(
	316	&ctx->partial_block_buffer[ctx->partial_block_buffer_length],
	317	buffer, copy_len);
	318
	319	ctx->partial_block_buffer_length += copy_len;
	320	ctx->incoming_buffer = (const void *)
	321	((const char *)buffer + copy_len);
	322	ctx->incoming_buffer_length = len - copy_len;
	323	}
	324
	325	/* The extra block should never contain more than 1 block */
	326	assert(ctx->partial_block_buffer_length <= SHA256_BLOCK_SIZE);
	327
	328	/*
	329	* If the extra block buffer contains exactly 1 block,
	330	* it can be hashed.
	331	*/
	332	if (ctx->partial_block_buffer_length >= SHA256_BLOCK_SIZE) {
	333	ctx->partial_block_buffer_length = 0;
	334
	335	ctx->job.buffer = ctx->partial_block_buffer;
	336	ctx->job.len = 1;
	337	ctx = (struct sha256_hash_ctx *)
	338	sha256_job_mgr_submit(&mgr->mgr, &ctx->job);
	339	}
	340	}
	341
	342	return sha256_ctx_mgr_resubmit(mgr, ctx);
	343	}
	344
	345	static struct sha256_hash_ctx sha256_ctx_mgr_flush(struct sha256_ctx_mgr mgr)
	346	{
	347	struct sha256_hash_ctx *ctx;
	348
	349	while (1) {
	350	ctx = (struct sha256_hash_ctx *)
	351	sha256_job_mgr_flush(&mgr->mgr);
	352
	353	/* If flush returned 0, there are no more jobs in flight. */
	354	if (!ctx)
	355	return NULL;
	356
	357	/*
	358	* If flush returned a job, resubmit the job to finish
	359	* processing.
	360	*/
	361	ctx = sha256_ctx_mgr_resubmit(mgr, ctx);
	362
	363	/*
	364	* If sha256_ctx_mgr_resubmit returned a job, it is ready to
	365	* be returned. Otherwise, all jobs currently being managed by
	366	* the sha256_ctx_mgr still need processing. Loop.
	367	*/
	368	if (ctx)
	369	return ctx;
	370	}
	371	}
	372
	373	static int sha256_mb_init(struct ahash_request *areq)
	374	{
	375	struct sha256_hash_ctx *sctx = ahash_request_ctx(areq);
	376
	377	hash_ctx_init(sctx);
	378	sctx->job.result_digest[0] = SHA256_H0;
	379	sctx->job.result_digest[1] = SHA256_H1;
	380	sctx->job.result_digest[2] = SHA256_H2;
	381	sctx->job.result_digest[3] = SHA256_H3;
	382	sctx->job.result_digest[4] = SHA256_H4;
	383	sctx->job.result_digest[5] = SHA256_H5;
	384	sctx->job.result_digest[6] = SHA256_H6;
	385	sctx->job.result_digest[7] = SHA256_H7;
	386	sctx->total_length = 0;
	387	sctx->partial_block_buffer_length = 0;
	388	sctx->status = HASH_CTX_STS_IDLE;
	389
	390	return 0;
	391	}
	392
	393	static int sha256_mb_set_results(struct mcryptd_hash_request_ctx *rctx)
	394	{
	395	int i;
	396	struct sha256_hash_ctx *sctx = ahash_request_ctx(&rctx->areq);
	397	__be32 dst = (__be32 ) rctx->out;
	398
	399	for (i = 0; i < 8; ++i)
	400	dst[i] = cpu_to_be32(sctx->job.result_digest[i]);
	401
	402	return 0;
	403	}
	404
	405	static int sha_finish_walk(struct mcryptd_hash_request_ctx **ret_rctx,
	406	struct mcryptd_alg_cstate *cstate, bool flush)
	407	{
	408	int flag = HASH_UPDATE;
	409	int nbytes, err = 0;
	410	struct mcryptd_hash_request_ctx rctx = ret_rctx;
	411	struct sha256_hash_ctx *sha_ctx;
	412
	413	/* more work ? */
	414	while (!(rctx->flag & HASH_DONE)) {
	415	nbytes = crypto_ahash_walk_done(&rctx->walk, 0);
	416	if (nbytes < 0) {
	417	err = nbytes;
	418	goto out;
	419	}
	420	/* check if the walk is done */
	421	if (crypto_ahash_walk_last(&rctx->walk)) {
	422	rctx->flag \|= HASH_DONE;
	423	if (rctx->flag & HASH_FINAL)
	424	flag \|= HASH_LAST;
	425
	426	}
	427	sha_ctx = (struct sha256_hash_ctx *)
	428	ahash_request_ctx(&rctx->areq);
	429	kernel_fpu_begin();
	430	sha_ctx = sha256_ctx_mgr_submit(cstate->mgr, sha_ctx,
	431	rctx->walk.data, nbytes, flag);
	432	if (!sha_ctx) {
	433	if (flush)
	434	sha_ctx = sha256_ctx_mgr_flush(cstate->mgr);
	435	}
	436	kernel_fpu_end();
	437	if (sha_ctx)
	438	rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
	439	else {
	440	rctx = NULL;
	441	goto out;
	442	}
	443	}
	444
	445	/* copy the results */
	446	if (rctx->flag & HASH_FINAL)
	447	sha256_mb_set_results(rctx);
	448
	449	out:
	450	*ret_rctx = rctx;
	451	return err;
	452	}
	453
	454	static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx,
	455	struct mcryptd_alg_cstate *cstate,
	456	int err)
	457	{
	458	struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
	459	struct sha256_hash_ctx *sha_ctx;
	460	struct mcryptd_hash_request_ctx *req_ctx;
	461	int ret;
	462
	463	/* remove from work list */
	464	spin_lock(&cstate->work_lock);
	465	list_del(&rctx->waiter);
	466	spin_unlock(&cstate->work_lock);
	467
	468	if (irqs_disabled())
	469	rctx->complete(&req->base, err);
	470	else {
	471	local_bh_disable();
	472	rctx->complete(&req->base, err);
	473	local_bh_enable();
	474	}
	475
	476	/* check to see if there are other jobs that are done */
	477	sha_ctx = sha256_ctx_mgr_get_comp_ctx(cstate->mgr);
	478	while (sha_ctx) {
	479	req_ctx = cast_hash_to_mcryptd_ctx(sha_ctx);
	480	ret = sha_finish_walk(&req_ctx, cstate, false);
	481	if (req_ctx) {
	482	spin_lock(&cstate->work_lock);
	483	list_del(&req_ctx->waiter);
	484	spin_unlock(&cstate->work_lock);
	485
	486	req = cast_mcryptd_ctx_to_req(req_ctx);
	487	if (irqs_disabled())
	488	rctx->complete(&req->base, ret);
	489	else {
	490	local_bh_disable();
	491	rctx->complete(&req->base, ret);
	492	local_bh_enable();
	493	}
	494	}
	495	sha_ctx = sha256_ctx_mgr_get_comp_ctx(cstate->mgr);
	496	}
	497
	498	return 0;
	499	}
	500
	501	static void sha256_mb_add_list(struct mcryptd_hash_request_ctx *rctx,
	502	struct mcryptd_alg_cstate *cstate)
	503	{
	504	unsigned long next_flush;
	505	unsigned long delay = usecs_to_jiffies(FLUSH_INTERVAL);
	506
	507	/* initialize tag */
	508	rctx->tag.arrival = jiffies; /* tag the arrival time */
	509	rctx->tag.seq_num = cstate->next_seq_num++;
	510	next_flush = rctx->tag.arrival + delay;
	511	rctx->tag.expire = next_flush;
	512
	513	spin_lock(&cstate->work_lock);
	514	list_add_tail(&rctx->waiter, &cstate->work_list);
	515	spin_unlock(&cstate->work_lock);
	516
	517	mcryptd_arm_flusher(cstate, delay);
	518	}
	519
	520	static int sha256_mb_update(struct ahash_request *areq)
	521	{
	522	struct mcryptd_hash_request_ctx *rctx =
	523	container_of(areq, struct mcryptd_hash_request_ctx, areq);
	524	struct mcryptd_alg_cstate *cstate =
	525	this_cpu_ptr(sha256_mb_alg_state.alg_cstate);
	526
	527	struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
	528	struct sha256_hash_ctx *sha_ctx;
	529	int ret = 0, nbytes;
	530
	531	/* sanity check */
	532	if (rctx->tag.cpu != smp_processor_id()) {
	533	pr_err("mcryptd error: cpu clash\n");
	534	goto done;
	535	}
	536
	537	/* need to init context */
	538	req_ctx_init(rctx, areq);
	539
	540	nbytes = crypto_ahash_walk_first(req, &rctx->walk);
	541
	542	if (nbytes < 0) {
	543	ret = nbytes;
	544	goto done;
	545	}
	546
	547	if (crypto_ahash_walk_last(&rctx->walk))
	548	rctx->flag \|= HASH_DONE;
	549
	550	/* submit */
	551	sha_ctx = (struct sha256_hash_ctx *) ahash_request_ctx(areq);
	552	sha256_mb_add_list(rctx, cstate);
	553	kernel_fpu_begin();
	554	sha_ctx = sha256_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data,
	555	nbytes, HASH_UPDATE);
	556	kernel_fpu_end();
	557
	558	/* check if anything is returned */
	559	if (!sha_ctx)
	560	return -EINPROGRESS;
	561
	562	if (sha_ctx->error) {
	563	ret = sha_ctx->error;
	564	rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
	565	goto done;
	566	}
	567
	568	rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
	569	ret = sha_finish_walk(&rctx, cstate, false);
	570
	571	if (!rctx)
	572	return -EINPROGRESS;
	573	done:
	574	sha_complete_job(rctx, cstate, ret);
	575	return ret;
	576	}
	577
	578	static int sha256_mb_finup(struct ahash_request *areq)
	579	{
	580	struct mcryptd_hash_request_ctx *rctx =
	581	container_of(areq, struct mcryptd_hash_request_ctx, areq);
	582	struct mcryptd_alg_cstate *cstate =
	583	this_cpu_ptr(sha256_mb_alg_state.alg_cstate);
	584
	585	struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
	586	struct sha256_hash_ctx *sha_ctx;
	587	int ret = 0, flag = HASH_UPDATE, nbytes;
	588
	589	/* sanity check */
	590	if (rctx->tag.cpu != smp_processor_id()) {
	591	pr_err("mcryptd error: cpu clash\n");
	592	goto done;
	593	}
	594
	595	/* need to init context */
	596	req_ctx_init(rctx, areq);
	597
	598	nbytes = crypto_ahash_walk_first(req, &rctx->walk);
	599
	600	if (nbytes < 0) {
	601	ret = nbytes;
	602	goto done;
	603	}
	604
	605	if (crypto_ahash_walk_last(&rctx->walk)) {
	606	rctx->flag \|= HASH_DONE;
	607	flag = HASH_LAST;
	608	}
	609
	610	/* submit */
	611	rctx->flag \|= HASH_FINAL;
	612	sha_ctx = (struct sha256_hash_ctx *) ahash_request_ctx(areq);
	613	sha256_mb_add_list(rctx, cstate);
	614
	615	kernel_fpu_begin();
	616	sha_ctx = sha256_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data,
	617	nbytes, flag);
	618	kernel_fpu_end();
	619
	620	/* check if anything is returned */
	621	if (!sha_ctx)
	622	return -EINPROGRESS;
	623
	624	if (sha_ctx->error) {
	625	ret = sha_ctx->error;
	626	goto done;
	627	}
	628
	629	rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
	630	ret = sha_finish_walk(&rctx, cstate, false);
	631	if (!rctx)
	632	return -EINPROGRESS;
	633	done:
	634	sha_complete_job(rctx, cstate, ret);
	635	return ret;
	636	}
	637
	638	static int sha256_mb_final(struct ahash_request *areq)
	639	{
	640	struct mcryptd_hash_request_ctx *rctx =
	641	container_of(areq, struct mcryptd_hash_request_ctx,
	642	areq);
	643	struct mcryptd_alg_cstate *cstate =
	644	this_cpu_ptr(sha256_mb_alg_state.alg_cstate);
	645
	646	struct sha256_hash_ctx *sha_ctx;
	647	int ret = 0;
	648	u8 data;
	649
	650	/* sanity check */
	651	if (rctx->tag.cpu != smp_processor_id()) {
	652	pr_err("mcryptd error: cpu clash\n");
	653	goto done;
	654	}
	655
	656	/* need to init context */
	657	req_ctx_init(rctx, areq);
	658
	659	rctx->flag \|= HASH_DONE \| HASH_FINAL;
	660
	661	sha_ctx = (struct sha256_hash_ctx *) ahash_request_ctx(areq);
	662	/* flag HASH_FINAL and 0 data size */
	663	sha256_mb_add_list(rctx, cstate);
	664	kernel_fpu_begin();
	665	sha_ctx = sha256_ctx_mgr_submit(cstate->mgr, sha_ctx, &data, 0,
	666	HASH_LAST);
	667	kernel_fpu_end();
	668
	669	/* check if anything is returned */
	670	if (!sha_ctx)
	671	return -EINPROGRESS;
	672
	673	if (sha_ctx->error) {
	674	ret = sha_ctx->error;
	675	rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
	676	goto done;
	677	}
	678
	679	rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
	680	ret = sha_finish_walk(&rctx, cstate, false);
	681	if (!rctx)
	682	return -EINPROGRESS;
	683	done:
	684	sha_complete_job(rctx, cstate, ret);
	685	return ret;
	686	}
	687
	688	static int sha256_mb_export(struct ahash_request areq, void out)
	689	{
	690	struct sha256_hash_ctx *sctx = ahash_request_ctx(areq);
	691
	692	memcpy(out, sctx, sizeof(*sctx));
	693
	694	return 0;
	695	}
	696
	697	static int sha256_mb_import(struct ahash_request areq, const void in)
	698	{
	699	struct sha256_hash_ctx *sctx = ahash_request_ctx(areq);
	700
	701	memcpy(sctx, in, sizeof(*sctx));
	702
	703	return 0;
	704	}
	705
	706	static int sha256_mb_async_init_tfm(struct crypto_tfm *tfm)
	707	{
	708	struct mcryptd_ahash *mcryptd_tfm;
	709	struct sha256_mb_ctx *ctx = crypto_tfm_ctx(tfm);
	710	struct mcryptd_hash_ctx *mctx;
	711
	712	mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha256-mb",
	713	CRYPTO_ALG_INTERNAL,
	714	CRYPTO_ALG_INTERNAL);
	715	if (IS_ERR(mcryptd_tfm))
	716	return PTR_ERR(mcryptd_tfm);
	717	mctx = crypto_ahash_ctx(&mcryptd_tfm->base);
	718	mctx->alg_state = &sha256_mb_alg_state;
	719	ctx->mcryptd_tfm = mcryptd_tfm;
	720	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
	721	sizeof(struct ahash_request) +
	722	crypto_ahash_reqsize(&mcryptd_tfm->base));
	723
	724	return 0;
	725	}
	726
	727	static void sha256_mb_async_exit_tfm(struct crypto_tfm *tfm)
	728	{
	729	struct sha256_mb_ctx *ctx = crypto_tfm_ctx(tfm);
	730
	731	mcryptd_free_ahash(ctx->mcryptd_tfm);
	732	}
	733
	734	static int sha256_mb_areq_init_tfm(struct crypto_tfm *tfm)
	735	{
	736	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
	737	sizeof(struct ahash_request) +
	738	sizeof(struct sha256_hash_ctx));
	739
	740	return 0;
	741	}
	742
	743	static void sha256_mb_areq_exit_tfm(struct crypto_tfm *tfm)
	744	{
	745	struct sha256_mb_ctx *ctx = crypto_tfm_ctx(tfm);
	746
	747	mcryptd_free_ahash(ctx->mcryptd_tfm);
	748	}
	749
	750	static struct ahash_alg sha256_mb_areq_alg = {
	751	.init = sha256_mb_init,
	752	.update = sha256_mb_update,
	753	.final = sha256_mb_final,
	754	.finup = sha256_mb_finup,
	755	.export = sha256_mb_export,
	756	.import = sha256_mb_import,
	757	.halg = {
	758	.digestsize = SHA256_DIGEST_SIZE,
	759	.statesize = sizeof(struct sha256_hash_ctx),
	760	.base = {
	761	.cra_name = "__sha256-mb",
	762	.cra_driver_name = "__intel_sha256-mb",
	763	.cra_priority = 100,
	764	/*
	765	* use ASYNC flag as some buffers in multi-buffer
	766	* algo may not have completed before hashing thread
	767	* sleep
	768	*/
	769	.cra_flags = CRYPTO_ALG_TYPE_AHASH \|
	770	CRYPTO_ALG_ASYNC \|
	771	CRYPTO_ALG_INTERNAL,
	772	.cra_blocksize = SHA256_BLOCK_SIZE,
	773	.cra_module = THIS_MODULE,
	774	.cra_list = LIST_HEAD_INIT
	775	(sha256_mb_areq_alg.halg.base.cra_list),
	776	.cra_init = sha256_mb_areq_init_tfm,
	777	.cra_exit = sha256_mb_areq_exit_tfm,
	778	.cra_ctxsize = sizeof(struct sha256_hash_ctx),
	779	}
	780	}
	781	};
	782
	783	static int sha256_mb_async_init(struct ahash_request *req)
	784	{
	785	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	786	struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm);
	787	struct ahash_request *mcryptd_req = ahash_request_ctx(req);
	788	struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
	789
	790	memcpy(mcryptd_req, req, sizeof(*req));
	791	ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
	792	return crypto_ahash_init(mcryptd_req);
	793	}
	794
	795	static int sha256_mb_async_update(struct ahash_request *req)
	796	{
	797	struct ahash_request *mcryptd_req = ahash_request_ctx(req);
	798
	799	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	800	struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm);
	801	struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
	802
	803	memcpy(mcryptd_req, req, sizeof(*req));
	804	ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
	805	return crypto_ahash_update(mcryptd_req);
	806	}
	807
	808	static int sha256_mb_async_finup(struct ahash_request *req)
	809	{
	810	struct ahash_request *mcryptd_req = ahash_request_ctx(req);
	811
	812	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	813	struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm);
	814	struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
	815
	816	memcpy(mcryptd_req, req, sizeof(*req));
	817	ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
	818	return crypto_ahash_finup(mcryptd_req);
	819	}
	820
	821	static int sha256_mb_async_final(struct ahash_request *req)
	822	{
	823	struct ahash_request *mcryptd_req = ahash_request_ctx(req);
	824
	825	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	826	struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm);
	827	struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
	828
	829	memcpy(mcryptd_req, req, sizeof(*req));
	830	ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
	831	return crypto_ahash_final(mcryptd_req);
	832	}
	833
	834	static int sha256_mb_async_digest(struct ahash_request *req)
	835	{
	836	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	837	struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm);
	838	struct ahash_request *mcryptd_req = ahash_request_ctx(req);
	839	struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
	840
	841	memcpy(mcryptd_req, req, sizeof(*req));
	842	ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
	843	return crypto_ahash_digest(mcryptd_req);
	844	}
	845
	846	static int sha256_mb_async_export(struct ahash_request req, void out)
	847	{
	848	struct ahash_request *mcryptd_req = ahash_request_ctx(req);
	849	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	850	struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm);
	851	struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
	852
	853	memcpy(mcryptd_req, req, sizeof(*req));
	854	ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
	855	return crypto_ahash_export(mcryptd_req, out);
	856	}
	857
	858	static int sha256_mb_async_import(struct ahash_request req, const void in)
	859	{
	860	struct ahash_request *mcryptd_req = ahash_request_ctx(req);
	861	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	862	struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm);
	863	struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
	864	struct crypto_ahash *child = mcryptd_ahash_child(mcryptd_tfm);
	865	struct mcryptd_hash_request_ctx *rctx;
	866	struct ahash_request *areq;
	867
	868	memcpy(mcryptd_req, req, sizeof(*req));
	869	ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
	870	rctx = ahash_request_ctx(mcryptd_req);
	871	areq = &rctx->areq;
	872
	873	ahash_request_set_tfm(areq, child);
	874	ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_SLEEP,
	875	rctx->complete, req);
	876
	877	return crypto_ahash_import(mcryptd_req, in);
	878	}
	879
	880	static struct ahash_alg sha256_mb_async_alg = {
	881	.init = sha256_mb_async_init,
	882	.update = sha256_mb_async_update,
	883	.final = sha256_mb_async_final,
	884	.finup = sha256_mb_async_finup,
	885	.export = sha256_mb_async_export,
	886	.import = sha256_mb_async_import,
	887	.digest = sha256_mb_async_digest,
	888	.halg = {
	889	.digestsize = SHA256_DIGEST_SIZE,
	890	.statesize = sizeof(struct sha256_hash_ctx),
	891	.base = {
	892	.cra_name = "sha256",
	893	.cra_driver_name = "sha256_mb",
	894	.cra_priority = 200,
	895	.cra_flags = CRYPTO_ALG_TYPE_AHASH \|
	896	CRYPTO_ALG_ASYNC,
	897	.cra_blocksize = SHA256_BLOCK_SIZE,
	898	.cra_type = &crypto_ahash_type,
	899	.cra_module = THIS_MODULE,
	900	.cra_list = LIST_HEAD_INIT
	901	(sha256_mb_async_alg.halg.base.cra_list),
	902	.cra_init = sha256_mb_async_init_tfm,
	903	.cra_exit = sha256_mb_async_exit_tfm,
	904	.cra_ctxsize = sizeof(struct sha256_mb_ctx),
	905	.cra_alignmask = 0,
	906	},
	907	},
	908	};
	909
	910	static unsigned long sha256_mb_flusher(struct mcryptd_alg_cstate *cstate)
	911	{
	912	struct mcryptd_hash_request_ctx *rctx;
	913	unsigned long cur_time;
	914	unsigned long next_flush = 0;
	915	struct sha256_hash_ctx *sha_ctx;
	916
	917
	918	cur_time = jiffies;
	919
	920	while (!list_empty(&cstate->work_list)) {
	921	rctx = list_entry(cstate->work_list.next,
	922	struct mcryptd_hash_request_ctx, waiter);
	923	if (time_before(cur_time, rctx->tag.expire))
	924	break;
	925	kernel_fpu_begin();
	926	sha_ctx = (struct sha256_hash_ctx *)
	927	sha256_ctx_mgr_flush(cstate->mgr);
	928	kernel_fpu_end();
	929	if (!sha_ctx) {
	930	pr_err("sha256_mb error: nothing got"
	931	" flushed for non-empty list\n");
	932	break;
	933	}
	934	rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
	935	sha_finish_walk(&rctx, cstate, true);
	936	sha_complete_job(rctx, cstate, 0);
	937	}
	938
	939	if (!list_empty(&cstate->work_list)) {
	940	rctx = list_entry(cstate->work_list.next,
	941	struct mcryptd_hash_request_ctx, waiter);
	942	/* get the hash context and then flush time */
	943	next_flush = rctx->tag.expire;
	944	mcryptd_arm_flusher(cstate, get_delay(next_flush));
	945	}
	946	return next_flush;
	947	}
	948
	949	static int __init sha256_mb_mod_init(void)
	950	{
	951
	952	int cpu;
	953	int err;
	954	struct mcryptd_alg_cstate *cpu_state;
	955
	956	/* check for dependent cpu features */
	957	if (!boot_cpu_has(X86_FEATURE_AVX2) \|\|
	958	!boot_cpu_has(X86_FEATURE_BMI2))
	959	return -ENODEV;
	960
	961	/* initialize multibuffer structures */
	962	sha256_mb_alg_state.alg_cstate = alloc_percpu
	963	(struct mcryptd_alg_cstate);
	964
	965	sha256_job_mgr_init = sha256_mb_mgr_init_avx2;
	966	sha256_job_mgr_submit = sha256_mb_mgr_submit_avx2;
	967	sha256_job_mgr_flush = sha256_mb_mgr_flush_avx2;
	968	sha256_job_mgr_get_comp_job = sha256_mb_mgr_get_comp_job_avx2;
	969
	970	if (!sha256_mb_alg_state.alg_cstate)
	971	return -ENOMEM;
	972	for_each_possible_cpu(cpu) {
	973	cpu_state = per_cpu_ptr(sha256_mb_alg_state.alg_cstate, cpu);
	974	cpu_state->next_flush = 0;
	975	cpu_state->next_seq_num = 0;
	976	cpu_state->flusher_engaged = false;
	977	INIT_DELAYED_WORK(&cpu_state->flush, mcryptd_flusher);
	978	cpu_state->cpu = cpu;
	979	cpu_state->alg_state = &sha256_mb_alg_state;
	980	cpu_state->mgr = kzalloc(sizeof(struct sha256_ctx_mgr),
	981	GFP_KERNEL);
	982	if (!cpu_state->mgr)
	983	goto err2;
	984	sha256_ctx_mgr_init(cpu_state->mgr);
	985	INIT_LIST_HEAD(&cpu_state->work_list);
	986	spin_lock_init(&cpu_state->work_lock);
	987	}
	988	sha256_mb_alg_state.flusher = &sha256_mb_flusher;
	989
	990	err = crypto_register_ahash(&sha256_mb_areq_alg);
	991	if (err)
	992	goto err2;
	993	err = crypto_register_ahash(&sha256_mb_async_alg);
	994	if (err)
	995	goto err1;
	996
	997
	998	return 0;
	999	err1:
	1000	crypto_unregister_ahash(&sha256_mb_areq_alg);
	1001	err2:
	1002	for_each_possible_cpu(cpu) {
	1003	cpu_state = per_cpu_ptr(sha256_mb_alg_state.alg_cstate, cpu);
	1004	kfree(cpu_state->mgr);
	1005	}
	1006	free_percpu(sha256_mb_alg_state.alg_cstate);
	1007	return -ENODEV;
	1008	}
	1009
	1010	static void __exit sha256_mb_mod_fini(void)
	1011	{
	1012	int cpu;
	1013	struct mcryptd_alg_cstate *cpu_state;
	1014
	1015	crypto_unregister_ahash(&sha256_mb_async_alg);
	1016	crypto_unregister_ahash(&sha256_mb_areq_alg);
	1017	for_each_possible_cpu(cpu) {
	1018	cpu_state = per_cpu_ptr(sha256_mb_alg_state.alg_cstate, cpu);
	1019	kfree(cpu_state->mgr);
	1020	}
	1021	free_percpu(sha256_mb_alg_state.alg_cstate);
	1022	}
	1023
	1024	module_init(sha256_mb_mod_init);
	1025	module_exit(sha256_mb_mod_fini);
	1026
	1027	MODULE_LICENSE("GPL");
	1028	MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, multi buffer accelerated");
	1029
	1030	MODULE_ALIAS_CRYPTO("sha256");

arch/x86/crypto/sha256-mb/sha256_mb_ctx.h+136−0

@@ -0,0 +1,136 @@
	1	/*
	2	* Header file for multi buffer SHA256 context
	3	*
	4	* This file is provided under a dual BSD/GPLv2 license. When using or
	5	* redistributing this file, you may do so under either license.
	6	*
	7	* GPL LICENSE SUMMARY
	8	*
	9	* Copyright(c) 2016 Intel Corporation.
	10	*
	11	* This program is free software; you can redistribute it and/or modify
	12	* it under the terms of version 2 of the GNU General Public License as
	13	* published by the Free Software Foundation.
	14	*
	15	* This program is distributed in the hope that it will be useful, but
	16	* WITHOUT ANY WARRANTY; without even the implied warranty of
	17	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	18	* General Public License for more details.
	19	*
	20	* Contact Information:
	21	* Megha Dey <megha.dey@linux.intel.com>
	22	*
	23	* BSD LICENSE
	24	*
	25	* Copyright(c) 2016 Intel Corporation.
	26	*
	27	* Redistribution and use in source and binary forms, with or without
	28	* modification, are permitted provided that the following conditions
	29	* are met:
	30	*
	31	* * Redistributions of source code must retain the above copyright
	32	* notice, this list of conditions and the following disclaimer.
	33	* * Redistributions in binary form must reproduce the above copyright
	34	* notice, this list of conditions and the following disclaimer in
	35	* the documentation and/or other materials provided with the
	36	* distribution.
	37	* * Neither the name of Intel Corporation nor the names of its
	38	* contributors may be used to endorse or promote products derived
	39	* from this software without specific prior written permission.
	40	*
	41	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	42	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	43	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	44	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	45	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	46	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	47	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	48	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	49	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	50	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	51	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	52	*/
	53
	54	#ifndef _SHA_MB_CTX_INTERNAL_H
	55	#define _SHA_MB_CTX_INTERNAL_H
	56
	57	#include "sha256_mb_mgr.h"
	58
	59	#define HASH_UPDATE 0x00
	60	#define HASH_FIRST 0x01
	61	#define HASH_LAST 0x02
	62	#define HASH_ENTIRE 0x03
	63	#define HASH_DONE 0x04
	64	#define HASH_FINAL 0x08
	65
	66	#define HASH_CTX_STS_IDLE 0x00
	67	#define HASH_CTX_STS_PROCESSING 0x01
	68	#define HASH_CTX_STS_LAST 0x02
	69	#define HASH_CTX_STS_COMPLETE 0x04
	70
	71	enum hash_ctx_error {
	72	HASH_CTX_ERROR_NONE = 0,
	73	HASH_CTX_ERROR_INVALID_FLAGS = -1,
	74	HASH_CTX_ERROR_ALREADY_PROCESSING = -2,
	75	HASH_CTX_ERROR_ALREADY_COMPLETED = -3,
	76
	77	#ifdef HASH_CTX_DEBUG
	78	HASH_CTX_ERROR_DEBUG_DIGEST_MISMATCH = -4,
	79	#endif
	80	};
	81
	82
	83	#define hash_ctx_user_data(ctx) ((ctx)->user_data)
	84	#define hash_ctx_digest(ctx) ((ctx)->job.result_digest)
	85	#define hash_ctx_processing(ctx) ((ctx)->status & HASH_CTX_STS_PROCESSING)
	86	#define hash_ctx_complete(ctx) ((ctx)->status == HASH_CTX_STS_COMPLETE)
	87	#define hash_ctx_status(ctx) ((ctx)->status)
	88	#define hash_ctx_error(ctx) ((ctx)->error)
	89	#define hash_ctx_init(ctx) \
	90	do { \
	91	(ctx)->error = HASH_CTX_ERROR_NONE; \
	92	(ctx)->status = HASH_CTX_STS_COMPLETE; \
	93	} while (0)
	94
	95
	96	/* Hash Constants and Typedefs */
	97	#define SHA256_DIGEST_LENGTH 8
	98	#define SHA256_LOG2_BLOCK_SIZE 6
	99
	100	#define SHA256_PADLENGTHFIELD_SIZE 8
	101
	102	#ifdef SHA_MB_DEBUG
	103	#define assert(expr) \
	104	do { \
	105	if (unlikely(!(expr))) { \
	106	printk(KERN_ERR "Assertion failed! %s,%s,%s,line=%d\n", \
	107	#expr, __FILE__, __func__, __LINE__); \
	108	} \
	109	} while (0)
	110	#else
	111	#define assert(expr) do {} while (0)
	112	#endif
	113
	114	struct sha256_ctx_mgr {
	115	struct sha256_mb_mgr mgr;
	116	};
	117
	118	/* typedef struct sha256_ctx_mgr sha256_ctx_mgr; */
	119
	120	struct sha256_hash_ctx {
	121	/* Must be at struct offset 0 */
	122	struct job_sha256 job;
	123	/* status flag */
	124	int status;
	125	/* error flag */
	126	int error;
	127
	128	uint32_t total_length;
	129	const void *incoming_buffer;
	130	uint32_t incoming_buffer_length;
	131	uint8_t partial_block_buffer[SHA256_BLOCK_SIZE * 2];
	132	uint32_t partial_block_buffer_length;
	133	void *user_data;
	134	};
	135
	136	#endif

arch/x86/crypto/sha256-mb/sha256_mb_mgr.h+108−0

@@ -0,0 +1,108 @@
	1	/*
	2	* Header file for multi buffer SHA256 algorithm manager
	3	*
	4	* This file is provided under a dual BSD/GPLv2 license. When using or
	5	* redistributing this file, you may do so under either license.
	6	*
	7	* GPL LICENSE SUMMARY
	8	*
	9	* Copyright(c) 2016 Intel Corporation.
	10	*
	11	* This program is free software; you can redistribute it and/or modify
	12	* it under the terms of version 2 of the GNU General Public License as
	13	* published by the Free Software Foundation.
	14	*
	15	* This program is distributed in the hope that it will be useful, but
	16	* WITHOUT ANY WARRANTY; without even the implied warranty of
	17	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	18	* General Public License for more details.
	19	*
	20	* Contact Information:
	21	* Megha Dey <megha.dey@linux.intel.com>
	22	*
	23	* BSD LICENSE
	24	*
	25	* Copyright(c) 2016 Intel Corporation.
	26	*
	27	* Redistribution and use in source and binary forms, with or without
	28	* modification, are permitted provided that the following conditions
	29	* are met:
	30	*
	31	* * Redistributions of source code must retain the above copyright
	32	* notice, this list of conditions and the following disclaimer.
	33	* * Redistributions in binary form must reproduce the above copyright
	34	* notice, this list of conditions and the following disclaimer in
	35	* the documentation and/or other materials provided with the
	36	* distribution.
	37	* * Neither the name of Intel Corporation nor the names of its
	38	* contributors may be used to endorse or promote products derived
	39	* from this software without specific prior written permission.
	40	*
	41	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	42	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	43	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	44	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	45	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	46	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	47	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	48	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	49	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	50	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	51	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	52	*/
	53	#ifndef __SHA_MB_MGR_H
	54	#define __SHA_MB_MGR_H
	55
	56	#include <linux/types.h>
	57
	58	#define NUM_SHA256_DIGEST_WORDS 8
	59
	60	enum job_sts { STS_UNKNOWN = 0,
	61	STS_BEING_PROCESSED = 1,
	62	STS_COMPLETED = 2,
	63	STS_INTERNAL_ERROR = 3,
	64	STS_ERROR = 4
	65	};
	66
	67	struct job_sha256 {
	68	u8 *buffer;
	69	u32 len;
	70	u32 result_digest[NUM_SHA256_DIGEST_WORDS] __aligned(32);
	71	enum job_sts status;
	72	void *user_data;
	73	};
	74
	75	/* SHA256 out-of-order scheduler */
	76
	77	/* typedef uint32_t sha8_digest_array[8][8]; */
	78
	79	struct sha256_args_x8 {
	80	uint32_t digest[8][8];
	81	uint8_t *data_ptr[8];
	82	};
	83
	84	struct sha256_lane_data {
	85	struct job_sha256 *job_in_lane;
	86	};
	87
	88	struct sha256_mb_mgr {
	89	struct sha256_args_x8 args;
	90
	91	uint32_t lens[8];
	92
	93	/* each byte is index (0...7) of unused lanes */
	94	uint64_t unused_lanes;
	95	/* byte 4 is set to FF as a flag */
	96	struct sha256_lane_data ldata[8];
	97	};
	98
	99
	100	#define SHA256_MB_MGR_NUM_LANES_AVX2 8
	101
	102	void sha256_mb_mgr_init_avx2(struct sha256_mb_mgr *state);
	103	struct job_sha256 sha256_mb_mgr_submit_avx2(struct sha256_mb_mgr state,
	104	struct job_sha256 *job);
	105	struct job_sha256 sha256_mb_mgr_flush_avx2(struct sha256_mb_mgr state);
	106	struct job_sha256 sha256_mb_mgr_get_comp_job_avx2(struct sha256_mb_mgr state);
	107
	108	#endif

arch/x86/crypto/sha256-mb/sha256_mb_mgr_datastruct.S+304−0

@@ -0,0 +1,304 @@
	1	/*
	2	* Header file for multi buffer SHA256 algorithm data structure
	3	*
	4	* This file is provided under a dual BSD/GPLv2 license. When using or
	5	* redistributing this file, you may do so under either license.
	6	*
	7	* GPL LICENSE SUMMARY
	8	*
	9	* Copyright(c) 2016 Intel Corporation.
	10	*
	11	* This program is free software; you can redistribute it and/or modify
	12	* it under the terms of version 2 of the GNU General Public License as
	13	* published by the Free Software Foundation.
	14	*
	15	* This program is distributed in the hope that it will be useful, but
	16	* WITHOUT ANY WARRANTY; without even the implied warranty of
	17	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	18	* General Public License for more details.
	19	*
	20	* Contact Information:
	21	* Megha Dey <megha.dey@linux.intel.com>
	22	*
	23	* BSD LICENSE
	24	*
	25	* Copyright(c) 2016 Intel Corporation.
	26	*
	27	* Redistribution and use in source and binary forms, with or without
	28	* modification, are permitted provided that the following conditions
	29	* are met:
	30	*
	31	* * Redistributions of source code must retain the above copyright
	32	* notice, this list of conditions and the following disclaimer.
	33	* * Redistributions in binary form must reproduce the above copyright
	34	* notice, this list of conditions and the following disclaimer in
	35	* the documentation and/or other materials provided with the
	36	* distribution.
	37	* * Neither the name of Intel Corporation nor the names of its
	38	* contributors may be used to endorse or promote products derived
	39	* from this software without specific prior written permission.
	40	*
	41	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	42	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	43	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	44	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	45	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	46	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	47	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	48	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	49	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	50	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	51	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	52	*/
	53
	54	# Macros for defining data structures
	55
	56	# Usage example
	57
	58	#START_FIELDS # JOB_AES
	59	### name size align
	60	#FIELD _plaintext, 8, 8 # pointer to plaintext
	61	#FIELD _ciphertext, 8, 8 # pointer to ciphertext
	62	#FIELD _IV, 16, 8 # IV
	63	#FIELD _keys, 8, 8 # pointer to keys
	64	#FIELD _len, 4, 4 # length in bytes
	65	#FIELD _status, 4, 4 # status enumeration
	66	#FIELD _user_data, 8, 8 # pointer to user data
	67	#UNION _union, size1, align1, \
	68	# size2, align2, \
	69	# size3, align3, \
	70	# ...
	71	#END_FIELDS
	72	#%assign _JOB_AES_size _FIELD_OFFSET
	73	#%assign _JOB_AES_align _STRUCT_ALIGN
	74
	75	#########################################################################
	76
	77	# Alternate "struc-like" syntax:
	78	# STRUCT job_aes2
	79	# RES_Q .plaintext, 1
	80	# RES_Q .ciphertext, 1
	81	# RES_DQ .IV, 1
	82	# RES_B .nested, _JOB_AES_SIZE, _JOB_AES_ALIGN
	83	# RES_U .union, size1, align1, \
	84	# size2, align2, \
	85	# ...
	86	# ENDSTRUCT
	87	# # Following only needed if nesting
	88	# %assign job_aes2_size _FIELD_OFFSET
	89	# %assign job_aes2_align _STRUCT_ALIGN
	90	#
	91	# RES_* macros take a name, a count and an optional alignment.
	92	# The count in in terms of the base size of the macro, and the
	93	# default alignment is the base size.
	94	# The macros are:
	95	# Macro Base size
	96	# RES_B 1
	97	# RES_W 2
	98	# RES_D 4
	99	# RES_Q 8
	100	# RES_DQ 16
	101	# RES_Y 32
	102	# RES_Z 64
	103	#
	104	# RES_U defines a union. It's arguments are a name and two or more
	105	# pairs of "size, alignment"
	106	#
	107	# The two assigns are only needed if this structure is being nested
	108	# within another. Even if the assigns are not done, one can still use
	109	# STRUCT_NAME_size as the size of the structure.
	110	#
	111	# Note that for nesting, you still need to assign to STRUCT_NAME_size.
	112	#
	113	# The differences between this and using "struc" directly are that each
	114	# type is implicitly aligned to its natural length (although this can be
	115	# over-ridden with an explicit third parameter), and that the structure
	116	# is padded at the end to its overall alignment.
	117	#
	118
	119	#########################################################################
	120
	121	#ifndef _DATASTRUCT_ASM_
	122	#define _DATASTRUCT_ASM_
	123
	124	#define SZ8 8*SHA256_DIGEST_WORD_SIZE
	125	#define ROUNDS 64*SZ8
	126	#define PTR_SZ 8
	127	#define SHA256_DIGEST_WORD_SIZE 4
	128	#define MAX_SHA256_LANES 8
	129	#define SHA256_DIGEST_WORDS 8
	130	#define SHA256_DIGEST_ROW_SIZE (MAX_SHA256_LANES * SHA256_DIGEST_WORD_SIZE)
	131	#define SHA256_DIGEST_SIZE (SHA256_DIGEST_ROW_SIZE * SHA256_DIGEST_WORDS)
	132	#define SHA256_BLK_SZ 64
	133
	134	# START_FIELDS
	135	.macro START_FIELDS
	136	_FIELD_OFFSET = 0
	137	_STRUCT_ALIGN = 0
	138	.endm
	139
	140	# FIELD name size align
	141	.macro FIELD name size align
	142	_FIELD_OFFSET = (_FIELD_OFFSET + (\align) - 1) & (~ ((\align)-1))
	143	\name = _FIELD_OFFSET
	144	_FIELD_OFFSET = _FIELD_OFFSET + (\size)
	145	.if (\align > _STRUCT_ALIGN)
	146	_STRUCT_ALIGN = \align
	147	.endif
	148	.endm
	149
	150	# END_FIELDS
	151	.macro END_FIELDS
	152	_FIELD_OFFSET = (_FIELD_OFFSET + _STRUCT_ALIGN-1) & (~ (_STRUCT_ALIGN-1))
	153	.endm
	154
	155	########################################################################
	156
	157	.macro STRUCT p1
	158	START_FIELDS
	159	.struc \p1
	160	.endm
	161
	162	.macro ENDSTRUCT
	163	tmp = _FIELD_OFFSET
	164	END_FIELDS
	165	tmp = (_FIELD_OFFSET - %%tmp)
	166	.if (tmp > 0)
	167	.lcomm tmp
	168	.endif
	169	.endstruc
	170	.endm
	171
	172	## RES_int name size align
	173	.macro RES_int p1 p2 p3
	174	name = \p1
	175	size = \p2
	176	align = .\p3
	177
	178	_FIELD_OFFSET = (_FIELD_OFFSET + (align) - 1) & (~ ((align)-1))
	179	.align align
	180	.lcomm name size
	181	_FIELD_OFFSET = _FIELD_OFFSET + (size)
	182	.if (align > _STRUCT_ALIGN)
	183	_STRUCT_ALIGN = align
	184	.endif
	185	.endm
	186
	187	# macro RES_B name, size [, align]
	188	.macro RES_B _name, _size, _align=1
	189	RES_int _name _size _align
	190	.endm
	191
	192	# macro RES_W name, size [, align]
	193	.macro RES_W _name, _size, _align=2
	194	RES_int _name 2*(_size) _align
	195	.endm
	196
	197	# macro RES_D name, size [, align]
	198	.macro RES_D _name, _size, _align=4
	199	RES_int _name 4*(_size) _align
	200	.endm
	201
	202	# macro RES_Q name, size [, align]
	203	.macro RES_Q _name, _size, _align=8
	204	RES_int _name 8*(_size) _align
	205	.endm
	206
	207	# macro RES_DQ name, size [, align]
	208	.macro RES_DQ _name, _size, _align=16
	209	RES_int _name 16*(_size) _align
	210	.endm
	211
	212	# macro RES_Y name, size [, align]
	213	.macro RES_Y _name, _size, _align=32
	214	RES_int _name 32*(_size) _align
	215	.endm
	216
	217	# macro RES_Z name, size [, align]
	218	.macro RES_Z _name, _size, _align=64
	219	RES_int _name 64*(_size) _align
	220	.endm
	221
	222	#endif
	223
	224
	225	########################################################################
	226	#### Define SHA256 Out Of Order Data Structures
	227	########################################################################
	228
	229	START_FIELDS # LANE_DATA
	230	### name size align
	231	FIELD _job_in_lane, 8, 8 # pointer to job object
	232	END_FIELDS
	233
	234	_LANE_DATA_size = _FIELD_OFFSET
	235	_LANE_DATA_align = _STRUCT_ALIGN
	236
	237	########################################################################
	238
	239	START_FIELDS # SHA256_ARGS_X4
	240	### name size align
	241	FIELD _digest, 488, 4 # transposed digest
	242	FIELD _data_ptr, 8*8, 8 # array of pointers to data
	243	END_FIELDS
	244
	245	_SHA256_ARGS_X4_size = _FIELD_OFFSET
	246	_SHA256_ARGS_X4_align = _STRUCT_ALIGN
	247	_SHA256_ARGS_X8_size = _FIELD_OFFSET
	248	_SHA256_ARGS_X8_align = _STRUCT_ALIGN
	249
	250	#######################################################################
	251
	252	START_FIELDS # MB_MGR
	253	### name size align
	254	FIELD _args, _SHA256_ARGS_X4_size, _SHA256_ARGS_X4_align
	255	FIELD _lens, 4*8, 8
	256	FIELD _unused_lanes, 8, 8
	257	FIELD _ldata, _LANE_DATA_size*8, _LANE_DATA_align
	258	END_FIELDS
	259
	260	_MB_MGR_size = _FIELD_OFFSET
	261	_MB_MGR_align = _STRUCT_ALIGN
	262
	263	_args_digest = _args + _digest
	264	_args_data_ptr = _args + _data_ptr
	265
	266	#######################################################################
	267
	268	START_FIELDS #STACK_FRAME
	269	### name size align
	270	FIELD _data, 16*SZ8, 1 # transposed digest
	271	FIELD _digest, 8*SZ8, 1 # array of pointers to data
	272	FIELD _ytmp, 4*SZ8, 1
	273	FIELD _rsp, 8, 1
	274	END_FIELDS
	275
	276	_STACK_FRAME_size = _FIELD_OFFSET
	277	_STACK_FRAME_align = _STRUCT_ALIGN
	278
	279	#######################################################################
	280
	281	########################################################################
	282	#### Define constants
	283	########################################################################
	284
	285	#define STS_UNKNOWN 0
	286	#define STS_BEING_PROCESSED 1
	287	#define STS_COMPLETED 2
	288
	289	########################################################################
	290	#### Define JOB_SHA256 structure
	291	########################################################################
	292
	293	START_FIELDS # JOB_SHA256
	294
	295	### name size align
	296	FIELD _buffer, 8, 8 # pointer to buffer
	297	FIELD _len, 8, 8 # length in bytes
	298	FIELD _result_digest, 8*4, 32 # Digest (output)
	299	FIELD _status, 4, 4
	300	FIELD _user_data, 8, 8
	301	END_FIELDS
	302
	303	_JOB_SHA256_size = _FIELD_OFFSET
	304	_JOB_SHA256_align = _STRUCT_ALIGN

arch/x86/crypto/sha256-mb/sha256_mb_mgr_flush_avx2.S+304−0

@@ -0,0 +1,304 @@
	1	/*
	2	* Flush routine for SHA256 multibuffer
	3	*
	4	* This file is provided under a dual BSD/GPLv2 license. When using or
	5	* redistributing this file, you may do so under either license.
	6	*
	7	* GPL LICENSE SUMMARY
	8	*
	9	* Copyright(c) 2016 Intel Corporation.
	10	*
	11	* This program is free software; you can redistribute it and/or modify
	12	* it under the terms of version 2 of the GNU General Public License as
	13	* published by the Free Software Foundation.
	14	*
	15	* This program is distributed in the hope that it will be useful, but
	16	* WITHOUT ANY WARRANTY; without even the implied warranty of
	17	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	18	* General Public License for more details.
	19	*
	20	* Contact Information:
	21	* Megha Dey <megha.dey@linux.intel.com>
	22	*
	23	* BSD LICENSE
	24	*
	25	* Copyright(c) 2016 Intel Corporation.
	26	*
	27	* Redistribution and use in source and binary forms, with or without
	28	* modification, are permitted provided that the following conditions
	29	* are met:
	30	*
	31	* * Redistributions of source code must retain the above copyright
	32	* notice, this list of conditions and the following disclaimer.
	33	* * Redistributions in binary form must reproduce the above copyright
	34	* notice, this list of conditions and the following disclaimer in
	35	* the documentation and/or other materials provided with the
	36	* distribution.
	37	* * Neither the name of Intel Corporation nor the names of its
	38	* contributors may be used to endorse or promote products derived
	39	* from this software without specific prior written permission.
	40	*
	41	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	42	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	43	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	44	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	45	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	46	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	47	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	48	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	49	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	50	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	51	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	52	*/
	53	#include <linux/linkage.h>
	54	#include <asm/frame.h>
	55	#include "sha256_mb_mgr_datastruct.S"
	56
	57	.extern sha256_x8_avx2
	58
	59	#LINUX register definitions
	60	#define arg1 %rdi
	61	#define arg2 %rsi
	62
	63	# Common register definitions
	64	#define state arg1
	65	#define job arg2
	66	#define len2 arg2
	67
	68	# idx must be a register not clobberred by sha1_mult
	69	#define idx %r8
	70	#define DWORD_idx %r8d
	71
	72	#define unused_lanes %rbx
	73	#define lane_data %rbx
	74	#define tmp2 %rbx
	75	#define tmp2_w %ebx
	76
	77	#define job_rax %rax
	78	#define tmp1 %rax
	79	#define size_offset %rax
	80	#define tmp %rax
	81	#define start_offset %rax
	82
	83	#define tmp3 %arg1
	84
	85	#define extra_blocks %arg2
	86	#define p %arg2
	87
	88	.macro LABEL prefix n
	89	\prefix\n\():
	90	.endm
	91
	92	.macro JNE_SKIP i
	93	jne skip_\i
	94	.endm
	95
	96	.altmacro
	97	.macro SET_OFFSET _offset
	98	offset = \_offset
	99	.endm
	100	.noaltmacro
	101
	102	# JOB_SHA256* sha256_mb_mgr_flush_avx2(MB_MGR *state)
	103	# arg 1 : rcx : state
	104	ENTRY(sha256_mb_mgr_flush_avx2)
	105	FRAME_BEGIN
	106	push %rbx
	107
	108	# If bit (32+3) is set, then all lanes are empty
	109	mov _unused_lanes(state), unused_lanes
	110	bt $32+3, unused_lanes
	111	jc return_null
	112
	113	# find a lane with a non-null job
	114	xor idx, idx
	115	offset = (_ldata + 1 * _LANE_DATA_size + _job_in_lane)
	116	cmpq $0, offset(state)
	117	cmovne one(%rip), idx
	118	offset = (_ldata + 2 * _LANE_DATA_size + _job_in_lane)
	119	cmpq $0, offset(state)
	120	cmovne two(%rip), idx
	121	offset = (_ldata + 3 * _LANE_DATA_size + _job_in_lane)
	122	cmpq $0, offset(state)
	123	cmovne three(%rip), idx
	124	offset = (_ldata + 4 * _LANE_DATA_size + _job_in_lane)
	125	cmpq $0, offset(state)
	126	cmovne four(%rip), idx
	127	offset = (_ldata + 5 * _LANE_DATA_size + _job_in_lane)
	128	cmpq $0, offset(state)
	129	cmovne five(%rip), idx
	130	offset = (_ldata + 6 * _LANE_DATA_size + _job_in_lane)
	131	cmpq $0, offset(state)
	132	cmovne six(%rip), idx
	133	offset = (_ldata + 7 * _LANE_DATA_size + _job_in_lane)
	134	cmpq $0, offset(state)
	135	cmovne seven(%rip), idx
	136
	137	# copy idx to empty lanes
	138	copy_lane_data:
	139	offset = (_args + _data_ptr)
	140	mov offset(state,idx,8), tmp
	141
	142	I = 0
	143	.rep 8
	144	offset = (_ldata + I * _LANE_DATA_size + _job_in_lane)
	145	cmpq $0, offset(state)
	146	.altmacro
	147	JNE_SKIP %I
	148	offset = (_args + _data_ptr + 8*I)
	149	mov tmp, offset(state)
	150	offset = (_lens + 4*I)
	151	movl $0xFFFFFFFF, offset(state)
	152	LABEL skip_ %I
	153	I = (I+1)
	154	.noaltmacro
	155	.endr
	156
	157	# Find min length
	158	vmovdqa _lens+0*16(state), %xmm0
	159	vmovdqa _lens+1*16(state), %xmm1
	160
	161	vpminud %xmm1, %xmm0, %xmm2 # xmm2 has {D,C,B,A}
	162	vpalignr $8, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,D,C}
	163	vpminud %xmm3, %xmm2, %xmm2 # xmm2 has {x,x,E,F}
	164	vpalignr $4, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,x,E}
	165	vpminud %xmm3, %xmm2, %xmm2 # xmm2 has min val in low dword
	166
	167	vmovd %xmm2, DWORD_idx
	168	mov idx, len2
	169	and $0xF, idx
	170	shr $4, len2
	171	jz len_is_0
	172
	173	vpand clear_low_nibble(%rip), %xmm2, %xmm2
	174	vpshufd $0, %xmm2, %xmm2
	175
	176	vpsubd %xmm2, %xmm0, %xmm0
	177	vpsubd %xmm2, %xmm1, %xmm1
	178
	179	vmovdqa %xmm0, _lens+0*16(state)
	180	vmovdqa %xmm1, _lens+1*16(state)
	181
	182	# "state" and "args" are the same address, arg1
	183	# len is arg2
	184	call sha256_x8_avx2
	185	# state and idx are intact
	186
	187	len_is_0:
	188	# process completed job "idx"
	189	imul $_LANE_DATA_size, idx, lane_data
	190	lea _ldata(state, lane_data), lane_data
	191
	192	mov _job_in_lane(lane_data), job_rax
	193	movq $0, _job_in_lane(lane_data)
	194	movl $STS_COMPLETED, _status(job_rax)
	195	mov _unused_lanes(state), unused_lanes
	196	shl $4, unused_lanes
	197	or idx, unused_lanes
	198
	199	mov unused_lanes, _unused_lanes(state)
	200	movl $0xFFFFFFFF, _lens(state,idx,4)
	201
	202	vmovd _args_digest(state , idx, 4) , %xmm0
	203	vpinsrd $1, _args_digest+1*32(state, idx, 4), %xmm0, %xmm0
	204	vpinsrd $2, _args_digest+2*32(state, idx, 4), %xmm0, %xmm0
	205	vpinsrd $3, _args_digest+3*32(state, idx, 4), %xmm0, %xmm0
	206	vmovd _args_digest+4*32(state, idx, 4), %xmm1
	207	vpinsrd $1, _args_digest+5*32(state, idx, 4), %xmm1, %xmm1
	208	vpinsrd $2, _args_digest+6*32(state, idx, 4), %xmm1, %xmm1
	209	vpinsrd $3, _args_digest+7*32(state, idx, 4), %xmm1, %xmm1
	210
	211	vmovdqu %xmm0, _result_digest(job_rax)
	212	offset = (_result_digest + 1*16)
	213	vmovdqu %xmm1, offset(job_rax)
	214
	215	return:
	216	pop %rbx
	217	FRAME_END
	218	ret
	219
	220	return_null:
	221	xor job_rax, job_rax
	222	jmp return
	223	ENDPROC(sha256_mb_mgr_flush_avx2)
	224
	225	##############################################################################
	226
	227	.align 16
	228	ENTRY(sha256_mb_mgr_get_comp_job_avx2)
	229	push %rbx
	230
	231	## if bit 32+3 is set, then all lanes are empty
	232	mov _unused_lanes(state), unused_lanes
	233	bt $(32+3), unused_lanes
	234	jc .return_null
	235
	236	# Find min length
	237	vmovdqa _lens(state), %xmm0
	238	vmovdqa _lens+1*16(state), %xmm1
	239
	240	vpminud %xmm1, %xmm0, %xmm2 # xmm2 has {D,C,B,A}
	241	vpalignr $8, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,D,C}
	242	vpminud %xmm3, %xmm2, %xmm2 # xmm2 has {x,x,E,F}
	243	vpalignr $4, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,x,E}
	244	vpminud %xmm3, %xmm2, %xmm2 # xmm2 has min val in low dword
	245
	246	vmovd %xmm2, DWORD_idx
	247	test $~0xF, idx
	248	jnz .return_null
	249
	250	# process completed job "idx"
	251	imul $_LANE_DATA_size, idx, lane_data
	252	lea _ldata(state, lane_data), lane_data
	253
	254	mov _job_in_lane(lane_data), job_rax
	255	movq $0, _job_in_lane(lane_data)
	256	movl $STS_COMPLETED, _status(job_rax)
	257	mov _unused_lanes(state), unused_lanes
	258	shl $4, unused_lanes
	259	or idx, unused_lanes
	260	mov unused_lanes, _unused_lanes(state)
	261
	262	movl $0xFFFFFFFF, _lens(state, idx, 4)
	263
	264	vmovd _args_digest(state, idx, 4), %xmm0
	265	vpinsrd $1, _args_digest+1*32(state, idx, 4), %xmm0, %xmm0
	266	vpinsrd $2, _args_digest+2*32(state, idx, 4), %xmm0, %xmm0
	267	vpinsrd $3, _args_digest+3*32(state, idx, 4), %xmm0, %xmm0
	268	movl _args_digest+4*32(state, idx, 4), tmp2_w
	269	vpinsrd $1, _args_digest+5*32(state, idx, 4), %xmm1, %xmm1
	270	vpinsrd $2, _args_digest+6*32(state, idx, 4), %xmm1, %xmm1
	271	vpinsrd $3, _args_digest+7*32(state, idx, 4), %xmm1, %xmm1
	272
	273	vmovdqu %xmm0, _result_digest(job_rax)
	274	movl tmp2_w, _result_digest+1*16(job_rax)
	275
	276	pop %rbx
	277
	278	ret
	279
	280	.return_null:
	281	xor job_rax, job_rax
	282	pop %rbx
	283	ret
	284	ENDPROC(sha256_mb_mgr_get_comp_job_avx2)
	285
	286	.data
	287
	288	.align 16
	289	clear_low_nibble:
	290	.octa 0x000000000000000000000000FFFFFFF0
	291	one:
	292	.quad 1
	293	two:
	294	.quad 2
	295	three:
	296	.quad 3
	297	four:
	298	.quad 4
	299	five:
	300	.quad 5
	301	six:
	302	.quad 6
	303	seven:
	304	.quad 7

arch/x86/crypto/sha256-mb/sha256_mb_mgr_init_avx2.c+65−0

@@ -0,0 +1,65 @@
	1	/*
	2	* Initialization code for multi buffer SHA256 algorithm for AVX2
	3	*
	4	* This file is provided under a dual BSD/GPLv2 license. When using or
	5	* redistributing this file, you may do so under either license.
	6	*
	7	* GPL LICENSE SUMMARY
	8	*
	9	* Copyright(c) 2016 Intel Corporation.
	10	*
	11	* This program is free software; you can redistribute it and/or modify
	12	* it under the terms of version 2 of the GNU General Public License as
	13	* published by the Free Software Foundation.
	14	*
	15	* This program is distributed in the hope that it will be useful, but
	16	* WITHOUT ANY WARRANTY; without even the implied warranty of
	17	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	18	* General Public License for more details.
	19	*
	20	* Contact Information:
	21	* Megha Dey <megha.dey@linux.intel.com>
	22	*
	23	* BSD LICENSE
	24	*
	25	* Copyright(c) 2016 Intel Corporation.
	26	*
	27	* Redistribution and use in source and binary forms, with or without
	28	* modification, are permitted provided that the following conditions
	29	* are met:
	30	*
	31	* * Redistributions of source code must retain the above copyright
	32	* notice, this list of conditions and the following disclaimer.
	33	* * Redistributions in binary form must reproduce the above copyright
	34	* notice, this list of conditions and the following disclaimer in
	35	* the documentation and/or other materials provided with the
	36	* distribution.
	37	* * Neither the name of Intel Corporation nor the names of its
	38	* contributors may be used to endorse or promote products derived
	39	* from this software without specific prior written permission.
	40	*
	41	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	42	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	43	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	44	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	45	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	46	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	47	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	48	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	49	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	50	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	51	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	52	*/
	53
	54	#include "sha256_mb_mgr.h"
	55
	56	void sha256_mb_mgr_init_avx2(struct sha256_mb_mgr *state)
	57	{
	58	unsigned int j;
	59
	60	state->unused_lanes = 0xF76543210ULL;
	61	for (j = 0; j < 8; j++) {
	62	state->lens[j] = 0xFFFFFFFF;
	63	state->ldata[j].job_in_lane = NULL;
	64	}
	65	}

arch/x86/crypto/sha256-mb/sha256_mb_mgr_submit_avx2.S+215−0

@@ -0,0 +1,215 @@
	1	/*
	2	* Buffer submit code for multi buffer SHA256 algorithm
	3	*
	4	* This file is provided under a dual BSD/GPLv2 license. When using or
	5	* redistributing this file, you may do so under either license.
	6	*
	7	* GPL LICENSE SUMMARY
	8	*
	9	* Copyright(c) 2016 Intel Corporation.
	10	*
	11	* This program is free software; you can redistribute it and/or modify
	12	* it under the terms of version 2 of the GNU General Public License as
	13	* published by the Free Software Foundation.
	14	*
	15	* This program is distributed in the hope that it will be useful, but
	16	* WITHOUT ANY WARRANTY; without even the implied warranty of
	17	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	18	* General Public License for more details.
	19	*
	20	* Contact Information:
	21	* Megha Dey <megha.dey@linux.intel.com>
	22	*
	23	* BSD LICENSE
	24	*
	25	* Copyright(c) 2016 Intel Corporation.
	26	*
	27	* Redistribution and use in source and binary forms, with or without
	28	* modification, are permitted provided that the following conditions
	29	* are met:
	30	*
	31	* * Redistributions of source code must retain the above copyright
	32	* notice, this list of conditions and the following disclaimer.
	33	* * Redistributions in binary form must reproduce the above copyright
	34	* notice, this list of conditions and the following disclaimer in
	35	* the documentation and/or other materials provided with the
	36	* distribution.
	37	* * Neither the name of Intel Corporation nor the names of its
	38	* contributors may be used to endorse or promote products derived
	39	* from this software without specific prior written permission.
	40	*
	41	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	42	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	43	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	44	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	45	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	46	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	47	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	48	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	49	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	50	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	51	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	52	*/
	53
	54	#include <linux/linkage.h>
	55	#include <asm/frame.h>
	56	#include "sha256_mb_mgr_datastruct.S"
	57
	58	.extern sha256_x8_avx2
	59
	60	# LINUX register definitions
	61	arg1 = %rdi
	62	arg2 = %rsi
	63	size_offset = %rcx
	64	tmp2 = %rcx
	65	extra_blocks = %rdx
	66
	67	# Common definitions
	68	#define state arg1
	69	#define job %rsi
	70	#define len2 arg2
	71	#define p2 arg2
	72
	73	# idx must be a register not clobberred by sha1_x8_avx2
	74	idx = %r8
	75	DWORD_idx = %r8d
	76	last_len = %r8
	77
	78	p = %r11
	79	start_offset = %r11
	80
	81	unused_lanes = %rbx
	82	BYTE_unused_lanes = %bl
	83
	84	job_rax = %rax
	85	len = %rax
	86	DWORD_len = %eax
	87
	88	lane = %r12
	89	tmp3 = %r12
	90
	91	tmp = %r9
	92	DWORD_tmp = %r9d
	93
	94	lane_data = %r10
	95
	96	# JOB* sha256_mb_mgr_submit_avx2(MB_MGR state, JOB_SHA256 job)
	97	# arg 1 : rcx : state
	98	# arg 2 : rdx : job
	99	ENTRY(sha256_mb_mgr_submit_avx2)
	100	FRAME_BEGIN
	101	push %rbx
	102	push %r12
	103
	104	mov _unused_lanes(state), unused_lanes
	105	mov unused_lanes, lane
	106	and $0xF, lane
	107	shr $4, unused_lanes
	108	imul $_LANE_DATA_size, lane, lane_data
	109	movl $STS_BEING_PROCESSED, _status(job)
	110	lea _ldata(state, lane_data), lane_data
	111	mov unused_lanes, _unused_lanes(state)
	112	movl _len(job), DWORD_len
	113
	114	mov job, _job_in_lane(lane_data)
	115	shl $4, len
	116	or lane, len
	117
	118	movl DWORD_len, _lens(state , lane, 4)
	119
	120	# Load digest words from result_digest
	121	vmovdqu _result_digest(job), %xmm0
	122	vmovdqu _result_digest+1*16(job), %xmm1
	123	vmovd %xmm0, _args_digest(state, lane, 4)
	124	vpextrd $1, %xmm0, _args_digest+1*32(state , lane, 4)
	125	vpextrd $2, %xmm0, _args_digest+2*32(state , lane, 4)
	126	vpextrd $3, %xmm0, _args_digest+3*32(state , lane, 4)
	127	vmovd %xmm1, _args_digest+4*32(state , lane, 4)
	128
	129	vpextrd $1, %xmm1, _args_digest+5*32(state , lane, 4)
	130	vpextrd $2, %xmm1, _args_digest+6*32(state , lane, 4)
	131	vpextrd $3, %xmm1, _args_digest+7*32(state , lane, 4)
	132
	133	mov _buffer(job), p
	134	mov p, _args_data_ptr(state, lane, 8)
	135
	136	cmp $0xF, unused_lanes
	137	jne return_null
	138
	139	start_loop:
	140	# Find min length
	141	vmovdqa _lens(state), %xmm0
	142	vmovdqa _lens+1*16(state), %xmm1
	143
	144	vpminud %xmm1, %xmm0, %xmm2 # xmm2 has {D,C,B,A}
	145	vpalignr $8, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,D,C}
	146	vpminud %xmm3, %xmm2, %xmm2 # xmm2 has {x,x,E,F}
	147	vpalignr $4, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,x,E}
	148	vpminud %xmm3, %xmm2, %xmm2 # xmm2 has min val in low dword
	149
	150	vmovd %xmm2, DWORD_idx
	151	mov idx, len2
	152	and $0xF, idx
	153	shr $4, len2
	154	jz len_is_0
	155
	156	vpand clear_low_nibble(%rip), %xmm2, %xmm2
	157	vpshufd $0, %xmm2, %xmm2
	158
	159	vpsubd %xmm2, %xmm0, %xmm0
	160	vpsubd %xmm2, %xmm1, %xmm1
	161
	162	vmovdqa %xmm0, _lens + 0*16(state)
	163	vmovdqa %xmm1, _lens + 1*16(state)
	164
	165	# "state" and "args" are the same address, arg1
	166	# len is arg2
	167	call sha256_x8_avx2
	168
	169	# state and idx are intact
	170
	171	len_is_0:
	172	# process completed job "idx"
	173	imul $_LANE_DATA_size, idx, lane_data
	174	lea _ldata(state, lane_data), lane_data
	175
	176	mov _job_in_lane(lane_data), job_rax
	177	mov _unused_lanes(state), unused_lanes
	178	movq $0, _job_in_lane(lane_data)
	179	movl $STS_COMPLETED, _status(job_rax)
	180	shl $4, unused_lanes
	181	or idx, unused_lanes
	182	mov unused_lanes, _unused_lanes(state)
	183
	184	movl $0xFFFFFFFF, _lens(state,idx,4)
	185
	186	vmovd _args_digest(state, idx, 4), %xmm0
	187	vpinsrd $1, _args_digest+1*32(state , idx, 4), %xmm0, %xmm0
	188	vpinsrd $2, _args_digest+2*32(state , idx, 4), %xmm0, %xmm0
	189	vpinsrd $3, _args_digest+3*32(state , idx, 4), %xmm0, %xmm0
	190	vmovd _args_digest+4*32(state, idx, 4), %xmm1
	191
	192	vpinsrd $1, _args_digest+5*32(state , idx, 4), %xmm1, %xmm1
	193	vpinsrd $2, _args_digest+6*32(state , idx, 4), %xmm1, %xmm1
	194	vpinsrd $3, _args_digest+7*32(state , idx, 4), %xmm1, %xmm1
	195
	196	vmovdqu %xmm0, _result_digest(job_rax)
	197	vmovdqu %xmm1, _result_digest+1*16(job_rax)
	198
	199	return:
	200	pop %r12
	201	pop %rbx
	202	FRAME_END
	203	ret
	204
	205	return_null:
	206	xor job_rax, job_rax
	207	jmp return
	208
	209	ENDPROC(sha256_mb_mgr_submit_avx2)
	210
	211	.data
	212
	213	.align 16
	214	clear_low_nibble:
	215	.octa 0x000000000000000000000000FFFFFFF0

arch/x86/crypto/sha256-mb/sha256_x8_avx2.S+593−0

@@ -0,0 +1,593 @@
	1	/*
	2	* Multi-buffer SHA256 algorithm hash compute routine
	3	*
	4	* This file is provided under a dual BSD/GPLv2 license. When using or
	5	* redistributing this file, you may do so under either license.
	6	*
	7	* GPL LICENSE SUMMARY
	8	*
	9	* Copyright(c) 2016 Intel Corporation.
	10	*
	11	* This program is free software; you can redistribute it and/or modify
	12	* it under the terms of version 2 of the GNU General Public License as
	13	* published by the Free Software Foundation.
	14	*
	15	* This program is distributed in the hope that it will be useful, but
	16	* WITHOUT ANY WARRANTY; without even the implied warranty of
	17	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	18	* General Public License for more details.
	19	*
	20	* Contact Information:
	21	* Megha Dey <megha.dey@linux.intel.com>
	22	*
	23	* BSD LICENSE
	24	*
	25	* Copyright(c) 2016 Intel Corporation.
	26	*
	27	* Redistribution and use in source and binary forms, with or without
	28	* modification, are permitted provided that the following conditions
	29	* are met:
	30	*
	31	* * Redistributions of source code must retain the above copyright
	32	* notice, this list of conditions and the following disclaimer.
	33	* * Redistributions in binary form must reproduce the above copyright
	34	* notice, this list of conditions and the following disclaimer in
	35	* the documentation and/or other materials provided with the
	36	* distribution.
	37	* * Neither the name of Intel Corporation nor the names of its
	38	* contributors may be used to endorse or promote products derived
	39	* from this software without specific prior written permission.
	40	*
	41	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	42	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	43	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	44	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	45	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	46	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	47	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	48	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	49	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	50	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	51	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	52	*/
	53
	54	#include <linux/linkage.h>
	55	#include "sha256_mb_mgr_datastruct.S"
	56
	57	## code to compute oct SHA256 using SSE-256
	58	## outer calling routine takes care of save and restore of XMM registers
	59	## Logic designed/laid out by JDG
	60
	61	## Function clobbers: rax, rcx, rdx, rbx, rsi, rdi, r9-r15; %ymm0-15
	62	## Linux clobbers: rax rbx rcx rdx rsi r9 r10 r11 r12 r13 r14 r15
	63	## Linux preserves: rdi rbp r8
	64	##
	65	## clobbers %ymm0-15
	66
	67	arg1 = %rdi
	68	arg2 = %rsi
	69	reg3 = %rcx
	70	reg4 = %rdx
	71
	72	# Common definitions
	73	STATE = arg1
	74	INP_SIZE = arg2
	75
	76	IDX = %rax
	77	ROUND = %rbx
	78	TBL = reg3
	79
	80	inp0 = %r9
	81	inp1 = %r10
	82	inp2 = %r11
	83	inp3 = %r12
	84	inp4 = %r13
	85	inp5 = %r14
	86	inp6 = %r15
	87	inp7 = reg4
	88
	89	a = %ymm0
	90	b = %ymm1
	91	c = %ymm2
	92	d = %ymm3
	93	e = %ymm4
	94	f = %ymm5
	95	g = %ymm6
	96	h = %ymm7
	97
	98	T1 = %ymm8
	99
	100	a0 = %ymm12
	101	a1 = %ymm13
	102	a2 = %ymm14
	103	TMP = %ymm15
	104	TMP0 = %ymm6
	105	TMP1 = %ymm7
	106
	107	TT0 = %ymm8
	108	TT1 = %ymm9
	109	TT2 = %ymm10
	110	TT3 = %ymm11
	111	TT4 = %ymm12
	112	TT5 = %ymm13
	113	TT6 = %ymm14
	114	TT7 = %ymm15
	115
	116	# Define stack usage
	117
	118	# Assume stack aligned to 32 bytes before call
	119	# Therefore FRAMESZ mod 32 must be 32-8 = 24
	120
	121	#define FRAMESZ 0x388
	122
	123	#define VMOVPS vmovups
	124
	125	# TRANSPOSE8 r0, r1, r2, r3, r4, r5, r6, r7, t0, t1
	126	# "transpose" data in {r0...r7} using temps {t0...t1}
	127	# Input looks like: {r0 r1 r2 r3 r4 r5 r6 r7}
	128	# r0 = {a7 a6 a5 a4 a3 a2 a1 a0}
	129	# r1 = {b7 b6 b5 b4 b3 b2 b1 b0}
	130	# r2 = {c7 c6 c5 c4 c3 c2 c1 c0}
	131	# r3 = {d7 d6 d5 d4 d3 d2 d1 d0}
	132	# r4 = {e7 e6 e5 e4 e3 e2 e1 e0}
	133	# r5 = {f7 f6 f5 f4 f3 f2 f1 f0}
	134	# r6 = {g7 g6 g5 g4 g3 g2 g1 g0}
	135	# r7 = {h7 h6 h5 h4 h3 h2 h1 h0}
	136	#
	137	# Output looks like: {r0 r1 r2 r3 r4 r5 r6 r7}
	138	# r0 = {h0 g0 f0 e0 d0 c0 b0 a0}
	139	# r1 = {h1 g1 f1 e1 d1 c1 b1 a1}
	140	# r2 = {h2 g2 f2 e2 d2 c2 b2 a2}
	141	# r3 = {h3 g3 f3 e3 d3 c3 b3 a3}
	142	# r4 = {h4 g4 f4 e4 d4 c4 b4 a4}
	143	# r5 = {h5 g5 f5 e5 d5 c5 b5 a5}
	144	# r6 = {h6 g6 f6 e6 d6 c6 b6 a6}
	145	# r7 = {h7 g7 f7 e7 d7 c7 b7 a7}
	146	#
	147
	148	.macro TRANSPOSE8 r0 r1 r2 r3 r4 r5 r6 r7 t0 t1
	149	# process top half (r0..r3) {a...d}
	150	vshufps $0x44, \r1, \r0, \t0 # t0 = {b5 b4 a5 a4 b1 b0 a1 a0}
	151	vshufps $0xEE, \r1, \r0, \r0 # r0 = {b7 b6 a7 a6 b3 b2 a3 a2}
	152	vshufps $0x44, \r3, \r2, \t1 # t1 = {d5 d4 c5 c4 d1 d0 c1 c0}
	153	vshufps $0xEE, \r3, \r2, \r2 # r2 = {d7 d6 c7 c6 d3 d2 c3 c2}
	154	vshufps $0xDD, \t1, \t0, \r3 # r3 = {d5 c5 b5 a5 d1 c1 b1 a1}
	155	vshufps $0x88, \r2, \r0, \r1 # r1 = {d6 c6 b6 a6 d2 c2 b2 a2}
	156	vshufps $0xDD, \r2, \r0, \r0 # r0 = {d7 c7 b7 a7 d3 c3 b3 a3}
	157	vshufps $0x88, \t1, \t0, \t0 # t0 = {d4 c4 b4 a4 d0 c0 b0 a0}
	158
	159	# use r2 in place of t0
	160	# process bottom half (r4..r7) {e...h}
	161	vshufps $0x44, \r5, \r4, \r2 # r2 = {f5 f4 e5 e4 f1 f0 e1 e0}
	162	vshufps $0xEE, \r5, \r4, \r4 # r4 = {f7 f6 e7 e6 f3 f2 e3 e2}
	163	vshufps $0x44, \r7, \r6, \t1 # t1 = {h5 h4 g5 g4 h1 h0 g1 g0}
	164	vshufps $0xEE, \r7, \r6, \r6 # r6 = {h7 h6 g7 g6 h3 h2 g3 g2}
	165	vshufps $0xDD, \t1, \r2, \r7 # r7 = {h5 g5 f5 e5 h1 g1 f1 e1}
	166	vshufps $0x88, \r6, \r4, \r5 # r5 = {h6 g6 f6 e6 h2 g2 f2 e2}
	167	vshufps $0xDD, \r6, \r4, \r4 # r4 = {h7 g7 f7 e7 h3 g3 f3 e3}
	168	vshufps $0x88, \t1, \r2, \t1 # t1 = {h4 g4 f4 e4 h0 g0 f0 e0}
	169
	170	vperm2f128 $0x13, \r1, \r5, \r6 # h6...a6
	171	vperm2f128 $0x02, \r1, \r5, \r2 # h2...a2
	172	vperm2f128 $0x13, \r3, \r7, \r5 # h5...a5
	173	vperm2f128 $0x02, \r3, \r7, \r1 # h1...a1
	174	vperm2f128 $0x13, \r0, \r4, \r7 # h7...a7
	175	vperm2f128 $0x02, \r0, \r4, \r3 # h3...a3
	176	vperm2f128 $0x13, \t0, \t1, \r4 # h4...a4
	177	vperm2f128 $0x02, \t0, \t1, \r0 # h0...a0
	178
	179	.endm
	180
	181	.macro ROTATE_ARGS
	182	TMP_ = h
	183	h = g
	184	g = f
	185	f = e
	186	e = d
	187	d = c
	188	c = b
	189	b = a
	190	a = TMP_
	191	.endm
	192
	193	.macro _PRORD reg imm tmp
	194	vpslld $(32-\imm),\reg,\tmp
	195	vpsrld $\imm,\reg, \reg
	196	vpor \tmp,\reg, \reg
	197	.endm
	198
	199	# PRORD_nd reg, imm, tmp, src
	200	.macro _PRORD_nd reg imm tmp src
	201	vpslld $(32-\imm), \src, \tmp
	202	vpsrld $\imm, \src, \reg
	203	vpor \tmp, \reg, \reg
	204	.endm
	205
	206	# PRORD dst/src, amt
	207	.macro PRORD reg imm
	208	_PRORD \reg,\imm,TMP
	209	.endm
	210
	211	# PRORD_nd dst, src, amt
	212	.macro PRORD_nd reg tmp imm
	213	_PRORD_nd \reg, \imm, TMP, \tmp
	214	.endm
	215
	216	# arguments passed implicitly in preprocessor symbols i, a...h
	217	.macro ROUND_00_15 _T1 i
	218	PRORD_nd a0,e,5 # sig1: a0 = (e >> 5)
	219
	220	vpxor g, f, a2 # ch: a2 = f^g
	221	vpand e,a2, a2 # ch: a2 = (f^g)&e
	222	vpxor g, a2, a2 # a2 = ch
	223
	224	PRORD_nd a1,e,25 # sig1: a1 = (e >> 25)
	225
	226	vmovdqu \_T1,(SZ8*(\i & 0xf))(%rsp)
	227	vpaddd (TBL,ROUND,1), \_T1, \_T1 # T1 = W + K
	228	vpxor e,a0, a0 # sig1: a0 = e ^ (e >> 5)
	229	PRORD a0, 6 # sig1: a0 = (e >> 6) ^ (e >> 11)
	230	vpaddd a2, h, h # h = h + ch
	231	PRORD_nd a2,a,11 # sig0: a2 = (a >> 11)
	232	vpaddd \_T1,h, h # h = h + ch + W + K
	233	vpxor a1, a0, a0 # a0 = sigma1
	234	PRORD_nd a1,a,22 # sig0: a1 = (a >> 22)
	235	vpxor c, a, \_T1 # maj: T1 = a^c
	236	add $SZ8, ROUND # ROUND++
	237	vpand b, \_T1, \_T1 # maj: T1 = (a^c)&b
	238	vpaddd a0, h, h
	239	vpaddd h, d, d
	240	vpxor a, a2, a2 # sig0: a2 = a ^ (a >> 11)
	241	PRORD a2,2 # sig0: a2 = (a >> 2) ^ (a >> 13)
	242	vpxor a1, a2, a2 # a2 = sig0
	243	vpand c, a, a1 # maj: a1 = a&c
	244	vpor \_T1, a1, a1 # a1 = maj
	245	vpaddd a1, h, h # h = h + ch + W + K + maj
	246	vpaddd a2, h, h # h = h + ch + W + K + maj + sigma0
	247	ROTATE_ARGS
	248	.endm
	249
	250	# arguments passed implicitly in preprocessor symbols i, a...h
	251	.macro ROUND_16_XX _T1 i
	252	vmovdqu (SZ8*((\i-15)&0xf))(%rsp), \_T1
	253	vmovdqu (SZ8*((\i-2)&0xf))(%rsp), a1
	254	vmovdqu \_T1, a0
	255	PRORD \_T1,11
	256	vmovdqu a1, a2
	257	PRORD a1,2
	258	vpxor a0, \_T1, \_T1
	259	PRORD \_T1, 7
	260	vpxor a2, a1, a1
	261	PRORD a1, 17
	262	vpsrld $3, a0, a0
	263	vpxor a0, \_T1, \_T1
	264	vpsrld $10, a2, a2
	265	vpxor a2, a1, a1
	266	vpaddd (SZ8*((\i-16)&0xf))(%rsp), \_T1, \_T1
	267	vpaddd (SZ8*((\i-7)&0xf))(%rsp), a1, a1
	268	vpaddd a1, \_T1, \_T1
	269
	270	ROUND_00_15 \_T1,\i
	271	.endm
	272
	273	# SHA256_ARGS:
	274	# UINT128 digest[8]; // transposed digests
	275	# UINT8 *data_ptr[4];
	276
	277	# void sha256_x8_avx2(SHA256_ARGS *args, UINT64 bytes);
	278	# arg 1 : STATE : pointer to array of pointers to input data
	279	# arg 2 : INP_SIZE : size of input in blocks
	280	# general registers preserved in outer calling routine
	281	# outer calling routine saves all the XMM registers
	282	# save rsp, allocate 32-byte aligned for local variables
	283	ENTRY(sha256_x8_avx2)
	284
	285	# save callee-saved clobbered registers to comply with C function ABI
	286	push %r12
	287	push %r13
	288	push %r14
	289	push %r15
	290
	291	mov %rsp, IDX
	292	sub $FRAMESZ, %rsp
	293	and $~0x1F, %rsp
	294	mov IDX, _rsp(%rsp)
	295
	296	# Load the pre-transposed incoming digest.
	297	vmovdqu 0*SHA256_DIGEST_ROW_SIZE(STATE),a
	298	vmovdqu 1*SHA256_DIGEST_ROW_SIZE(STATE),b
	299	vmovdqu 2*SHA256_DIGEST_ROW_SIZE(STATE),c
	300	vmovdqu 3*SHA256_DIGEST_ROW_SIZE(STATE),d
	301	vmovdqu 4*SHA256_DIGEST_ROW_SIZE(STATE),e
	302	vmovdqu 5*SHA256_DIGEST_ROW_SIZE(STATE),f
	303	vmovdqu 6*SHA256_DIGEST_ROW_SIZE(STATE),g
	304	vmovdqu 7*SHA256_DIGEST_ROW_SIZE(STATE),h
	305
	306	lea K256_8(%rip),TBL
	307
	308	# load the address of each of the 4 message lanes
	309	# getting ready to transpose input onto stack
	310	mov _args_data_ptr+0*PTR_SZ(STATE),inp0
	311	mov _args_data_ptr+1*PTR_SZ(STATE),inp1
	312	mov _args_data_ptr+2*PTR_SZ(STATE),inp2
	313	mov _args_data_ptr+3*PTR_SZ(STATE),inp3
	314	mov _args_data_ptr+4*PTR_SZ(STATE),inp4
	315	mov _args_data_ptr+5*PTR_SZ(STATE),inp5
	316	mov _args_data_ptr+6*PTR_SZ(STATE),inp6
	317	mov _args_data_ptr+7*PTR_SZ(STATE),inp7
	318
	319	xor IDX, IDX
	320	lloop:
	321	xor ROUND, ROUND
	322
	323	# save old digest
	324	vmovdqu a, _digest(%rsp)
	325	vmovdqu b, _digest+1*SZ8(%rsp)
	326	vmovdqu c, _digest+2*SZ8(%rsp)
	327	vmovdqu d, _digest+3*SZ8(%rsp)
	328	vmovdqu e, _digest+4*SZ8(%rsp)
	329	vmovdqu f, _digest+5*SZ8(%rsp)
	330	vmovdqu g, _digest+6*SZ8(%rsp)
	331	vmovdqu h, _digest+7*SZ8(%rsp)
	332	i = 0
	333	.rep 2
	334	VMOVPS i*32(inp0, IDX), TT0
	335	VMOVPS i*32(inp1, IDX), TT1
	336	VMOVPS i*32(inp2, IDX), TT2
	337	VMOVPS i*32(inp3, IDX), TT3
	338	VMOVPS i*32(inp4, IDX), TT4
	339	VMOVPS i*32(inp5, IDX), TT5
	340	VMOVPS i*32(inp6, IDX), TT6
	341	VMOVPS i*32(inp7, IDX), TT7
	342	vmovdqu g, _ytmp(%rsp)
	343	vmovdqu h, _ytmp+1*SZ8(%rsp)
	344	TRANSPOSE8 TT0, TT1, TT2, TT3, TT4, TT5, TT6, TT7, TMP0, TMP1
	345	vmovdqu PSHUFFLE_BYTE_FLIP_MASK(%rip), TMP1
	346	vmovdqu _ytmp(%rsp), g
	347	vpshufb TMP1, TT0, TT0
	348	vpshufb TMP1, TT1, TT1
	349	vpshufb TMP1, TT2, TT2
	350	vpshufb TMP1, TT3, TT3
	351	vpshufb TMP1, TT4, TT4
	352	vpshufb TMP1, TT5, TT5
	353	vpshufb TMP1, TT6, TT6
	354	vpshufb TMP1, TT7, TT7
	355	vmovdqu _ytmp+1*SZ8(%rsp), h
	356	vmovdqu TT4, _ytmp(%rsp)
	357	vmovdqu TT5, _ytmp+1*SZ8(%rsp)
	358	vmovdqu TT6, _ytmp+2*SZ8(%rsp)
	359	vmovdqu TT7, _ytmp+3*SZ8(%rsp)
	360	ROUND_00_15 TT0,(i*8+0)
	361	vmovdqu _ytmp(%rsp), TT0
	362	ROUND_00_15 TT1,(i*8+1)
	363	vmovdqu _ytmp+1*SZ8(%rsp), TT1
	364	ROUND_00_15 TT2,(i*8+2)
	365	vmovdqu _ytmp+2*SZ8(%rsp), TT2
	366	ROUND_00_15 TT3,(i*8+3)
	367	vmovdqu _ytmp+3*SZ8(%rsp), TT3
	368	ROUND_00_15 TT0,(i*8+4)
	369	ROUND_00_15 TT1,(i*8+5)
	370	ROUND_00_15 TT2,(i*8+6)
	371	ROUND_00_15 TT3,(i*8+7)
	372	i = (i+1)
	373	.endr
	374	add $64, IDX
	375	i = (i*8)
	376
	377	jmp Lrounds_16_xx
	378	.align 16
	379	Lrounds_16_xx:
	380	.rep 16
	381	ROUND_16_XX T1, i
	382	i = (i+1)
	383	.endr
	384
	385	cmp $ROUNDS,ROUND
	386	jb Lrounds_16_xx
	387
	388	# add old digest
	389	vpaddd _digest+0*SZ8(%rsp), a, a
	390	vpaddd _digest+1*SZ8(%rsp), b, b
	391	vpaddd _digest+2*SZ8(%rsp), c, c
	392	vpaddd _digest+3*SZ8(%rsp), d, d
	393	vpaddd _digest+4*SZ8(%rsp), e, e
	394	vpaddd _digest+5*SZ8(%rsp), f, f
	395	vpaddd _digest+6*SZ8(%rsp), g, g
	396	vpaddd _digest+7*SZ8(%rsp), h, h
	397
	398	sub $1, INP_SIZE # unit is blocks
	399	jne lloop
	400
	401	# write back to memory (state object) the transposed digest
	402	vmovdqu a, 0*SHA256_DIGEST_ROW_SIZE(STATE)
	403	vmovdqu b, 1*SHA256_DIGEST_ROW_SIZE(STATE)
	404	vmovdqu c, 2*SHA256_DIGEST_ROW_SIZE(STATE)
	405	vmovdqu d, 3*SHA256_DIGEST_ROW_SIZE(STATE)
	406	vmovdqu e, 4*SHA256_DIGEST_ROW_SIZE(STATE)
	407	vmovdqu f, 5*SHA256_DIGEST_ROW_SIZE(STATE)
	408	vmovdqu g, 6*SHA256_DIGEST_ROW_SIZE(STATE)
	409	vmovdqu h, 7*SHA256_DIGEST_ROW_SIZE(STATE)
	410
	411	# update input pointers
	412	add IDX, inp0
	413	mov inp0, _args_data_ptr+0*8(STATE)
	414	add IDX, inp1
	415	mov inp1, _args_data_ptr+1*8(STATE)
	416	add IDX, inp2
	417	mov inp2, _args_data_ptr+2*8(STATE)
	418	add IDX, inp3
	419	mov inp3, _args_data_ptr+3*8(STATE)
	420	add IDX, inp4
	421	mov inp4, _args_data_ptr+4*8(STATE)
	422	add IDX, inp5
	423	mov inp5, _args_data_ptr+5*8(STATE)
	424	add IDX, inp6
	425	mov inp6, _args_data_ptr+6*8(STATE)
	426	add IDX, inp7
	427	mov inp7, _args_data_ptr+7*8(STATE)
	428
	429	# Postamble
	430	mov _rsp(%rsp), %rsp
	431
	432	# restore callee-saved clobbered registers
	433	pop %r15
	434	pop %r14
	435	pop %r13
	436	pop %r12
	437
	438	ret
	439	ENDPROC(sha256_x8_avx2)
	440	.data
	441	.align 64
	442	K256_8:
	443	.octa 0x428a2f98428a2f98428a2f98428a2f98
	444	.octa 0x428a2f98428a2f98428a2f98428a2f98
	445	.octa 0x71374491713744917137449171374491
	446	.octa 0x71374491713744917137449171374491
	447	.octa 0xb5c0fbcfb5c0fbcfb5c0fbcfb5c0fbcf
	448	.octa 0xb5c0fbcfb5c0fbcfb5c0fbcfb5c0fbcf
	449	.octa 0xe9b5dba5e9b5dba5e9b5dba5e9b5dba5
	450	.octa 0xe9b5dba5e9b5dba5e9b5dba5e9b5dba5
	451	.octa 0x3956c25b3956c25b3956c25b3956c25b
	452	.octa 0x3956c25b3956c25b3956c25b3956c25b
	453	.octa 0x59f111f159f111f159f111f159f111f1
	454	.octa 0x59f111f159f111f159f111f159f111f1
	455	.octa 0x923f82a4923f82a4923f82a4923f82a4
	456	.octa 0x923f82a4923f82a4923f82a4923f82a4
	457	.octa 0xab1c5ed5ab1c5ed5ab1c5ed5ab1c5ed5
	458	.octa 0xab1c5ed5ab1c5ed5ab1c5ed5ab1c5ed5
	459	.octa 0xd807aa98d807aa98d807aa98d807aa98
	460	.octa 0xd807aa98d807aa98d807aa98d807aa98
	461	.octa 0x12835b0112835b0112835b0112835b01
	462	.octa 0x12835b0112835b0112835b0112835b01
	463	.octa 0x243185be243185be243185be243185be
	464	.octa 0x243185be243185be243185be243185be
	465	.octa 0x550c7dc3550c7dc3550c7dc3550c7dc3
	466	.octa 0x550c7dc3550c7dc3550c7dc3550c7dc3
	467	.octa 0x72be5d7472be5d7472be5d7472be5d74
	468	.octa 0x72be5d7472be5d7472be5d7472be5d74
	469	.octa 0x80deb1fe80deb1fe80deb1fe80deb1fe
	470	.octa 0x80deb1fe80deb1fe80deb1fe80deb1fe
	471	.octa 0x9bdc06a79bdc06a79bdc06a79bdc06a7
	472	.octa 0x9bdc06a79bdc06a79bdc06a79bdc06a7
	473	.octa 0xc19bf174c19bf174c19bf174c19bf174
	474	.octa 0xc19bf174c19bf174c19bf174c19bf174
	475	.octa 0xe49b69c1e49b69c1e49b69c1e49b69c1
	476	.octa 0xe49b69c1e49b69c1e49b69c1e49b69c1
	477	.octa 0xefbe4786efbe4786efbe4786efbe4786
	478	.octa 0xefbe4786efbe4786efbe4786efbe4786
	479	.octa 0x0fc19dc60fc19dc60fc19dc60fc19dc6
	480	.octa 0x0fc19dc60fc19dc60fc19dc60fc19dc6
	481	.octa 0x240ca1cc240ca1cc240ca1cc240ca1cc
	482	.octa 0x240ca1cc240ca1cc240ca1cc240ca1cc
	483	.octa 0x2de92c6f2de92c6f2de92c6f2de92c6f
	484	.octa 0x2de92c6f2de92c6f2de92c6f2de92c6f
	485	.octa 0x4a7484aa4a7484aa4a7484aa4a7484aa
	486	.octa 0x4a7484aa4a7484aa4a7484aa4a7484aa
	487	.octa 0x5cb0a9dc5cb0a9dc5cb0a9dc5cb0a9dc
	488	.octa 0x5cb0a9dc5cb0a9dc5cb0a9dc5cb0a9dc
	489	.octa 0x76f988da76f988da76f988da76f988da
	490	.octa 0x76f988da76f988da76f988da76f988da
	491	.octa 0x983e5152983e5152983e5152983e5152
	492	.octa 0x983e5152983e5152983e5152983e5152
	493	.octa 0xa831c66da831c66da831c66da831c66d
	494	.octa 0xa831c66da831c66da831c66da831c66d
	495	.octa 0xb00327c8b00327c8b00327c8b00327c8
	496	.octa 0xb00327c8b00327c8b00327c8b00327c8
	497	.octa 0xbf597fc7bf597fc7bf597fc7bf597fc7
	498	.octa 0xbf597fc7bf597fc7bf597fc7bf597fc7
	499	.octa 0xc6e00bf3c6e00bf3c6e00bf3c6e00bf3
	500	.octa 0xc6e00bf3c6e00bf3c6e00bf3c6e00bf3
	501	.octa 0xd5a79147d5a79147d5a79147d5a79147
	502	.octa 0xd5a79147d5a79147d5a79147d5a79147
	503	.octa 0x06ca635106ca635106ca635106ca6351
	504	.octa 0x06ca635106ca635106ca635106ca6351
	505	.octa 0x14292967142929671429296714292967
	506	.octa 0x14292967142929671429296714292967
	507	.octa 0x27b70a8527b70a8527b70a8527b70a85
	508	.octa 0x27b70a8527b70a8527b70a8527b70a85
	509	.octa 0x2e1b21382e1b21382e1b21382e1b2138
	510	.octa 0x2e1b21382e1b21382e1b21382e1b2138
	511	.octa 0x4d2c6dfc4d2c6dfc4d2c6dfc4d2c6dfc
	512	.octa 0x4d2c6dfc4d2c6dfc4d2c6dfc4d2c6dfc
	513	.octa 0x53380d1353380d1353380d1353380d13
	514	.octa 0x53380d1353380d1353380d1353380d13
	515	.octa 0x650a7354650a7354650a7354650a7354
	516	.octa 0x650a7354650a7354650a7354650a7354
	517	.octa 0x766a0abb766a0abb766a0abb766a0abb
	518	.octa 0x766a0abb766a0abb766a0abb766a0abb
	519	.octa 0x81c2c92e81c2c92e81c2c92e81c2c92e
	520	.octa 0x81c2c92e81c2c92e81c2c92e81c2c92e
	521	.octa 0x92722c8592722c8592722c8592722c85
	522	.octa 0x92722c8592722c8592722c8592722c85
	523	.octa 0xa2bfe8a1a2bfe8a1a2bfe8a1a2bfe8a1
	524	.octa 0xa2bfe8a1a2bfe8a1a2bfe8a1a2bfe8a1
	525	.octa 0xa81a664ba81a664ba81a664ba81a664b
	526	.octa 0xa81a664ba81a664ba81a664ba81a664b
	527	.octa 0xc24b8b70c24b8b70c24b8b70c24b8b70
	528	.octa 0xc24b8b70c24b8b70c24b8b70c24b8b70
	529	.octa 0xc76c51a3c76c51a3c76c51a3c76c51a3
	530	.octa 0xc76c51a3c76c51a3c76c51a3c76c51a3
	531	.octa 0xd192e819d192e819d192e819d192e819
	532	.octa 0xd192e819d192e819d192e819d192e819
	533	.octa 0xd6990624d6990624d6990624d6990624
	534	.octa 0xd6990624d6990624d6990624d6990624
	535	.octa 0xf40e3585f40e3585f40e3585f40e3585
	536	.octa 0xf40e3585f40e3585f40e3585f40e3585
	537	.octa 0x106aa070106aa070106aa070106aa070
	538	.octa 0x106aa070106aa070106aa070106aa070
	539	.octa 0x19a4c11619a4c11619a4c11619a4c116
	540	.octa 0x19a4c11619a4c11619a4c11619a4c116
	541	.octa 0x1e376c081e376c081e376c081e376c08
	542	.octa 0x1e376c081e376c081e376c081e376c08
	543	.octa 0x2748774c2748774c2748774c2748774c
	544	.octa 0x2748774c2748774c2748774c2748774c
	545	.octa 0x34b0bcb534b0bcb534b0bcb534b0bcb5
	546	.octa 0x34b0bcb534b0bcb534b0bcb534b0bcb5
	547	.octa 0x391c0cb3391c0cb3391c0cb3391c0cb3
	548	.octa 0x391c0cb3391c0cb3391c0cb3391c0cb3
	549	.octa 0x4ed8aa4a4ed8aa4a4ed8aa4a4ed8aa4a
	550	.octa 0x4ed8aa4a4ed8aa4a4ed8aa4a4ed8aa4a
	551	.octa 0x5b9cca4f5b9cca4f5b9cca4f5b9cca4f
	552	.octa 0x5b9cca4f5b9cca4f5b9cca4f5b9cca4f
	553	.octa 0x682e6ff3682e6ff3682e6ff3682e6ff3
	554	.octa 0x682e6ff3682e6ff3682e6ff3682e6ff3
	555	.octa 0x748f82ee748f82ee748f82ee748f82ee
	556	.octa 0x748f82ee748f82ee748f82ee748f82ee
	557	.octa 0x78a5636f78a5636f78a5636f78a5636f
	558	.octa 0x78a5636f78a5636f78a5636f78a5636f
	559	.octa 0x84c8781484c8781484c8781484c87814
	560	.octa 0x84c8781484c8781484c8781484c87814
	561	.octa 0x8cc702088cc702088cc702088cc70208
	562	.octa 0x8cc702088cc702088cc702088cc70208
	563	.octa 0x90befffa90befffa90befffa90befffa
	564	.octa 0x90befffa90befffa90befffa90befffa
	565	.octa 0xa4506ceba4506ceba4506ceba4506ceb
	566	.octa 0xa4506ceba4506ceba4506ceba4506ceb
	567	.octa 0xbef9a3f7bef9a3f7bef9a3f7bef9a3f7
	568	.octa 0xbef9a3f7bef9a3f7bef9a3f7bef9a3f7
	569	.octa 0xc67178f2c67178f2c67178f2c67178f2
	570	.octa 0xc67178f2c67178f2c67178f2c67178f2
	571	PSHUFFLE_BYTE_FLIP_MASK:
	572	.octa 0x0c0d0e0f08090a0b0405060700010203
	573	.octa 0x0c0d0e0f08090a0b0405060700010203
	574
	575	.align 64
	576	.global K256
	577	K256:
	578	.int 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
	579	.int 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
	580	.int 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
	581	.int 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
	582	.int 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
	583	.int 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
	584	.int 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
	585	.int 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
	586	.int 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
	587	.int 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
	588	.int 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
	589	.int 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
	590	.int 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
	591	.int 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
	592	.int 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
	593	.int 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2

arch/x86/crypto/sha256_ssse3_glue.c+10−0

@@ -427,4 +427,14 @@ MODULE_LICENSE("GPL"); · 426 unmodified lines
427	MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated");	427	MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated");
428		428
429	MODULE_ALIAS_CRYPTO("sha256");	429	MODULE_ALIAS_CRYPTO("sha256");
		430	MODULE_ALIAS_CRYPTO("sha256-ssse3");
		431	MODULE_ALIAS_CRYPTO("sha256-avx");
		432	MODULE_ALIAS_CRYPTO("sha256-avx2");
430	MODULE_ALIAS_CRYPTO("sha224");	433	MODULE_ALIAS_CRYPTO("sha224");
		434	MODULE_ALIAS_CRYPTO("sha224-ssse3");
		435	MODULE_ALIAS_CRYPTO("sha224-avx");
		436	MODULE_ALIAS_CRYPTO("sha224-avx2");
		437	#ifdef CONFIG_AS_SHA256_NI
		438	MODULE_ALIAS_CRYPTO("sha256-ni");
		439	MODULE_ALIAS_CRYPTO("sha224-ni");
		440	#endif

arch/x86/crypto/sha512-mb/Makefile+11−0

@@ -0,0 +1,11 @@
	1	#
	2	# Arch-specific CryptoAPI modules.
	3	#
	4
	5	avx2_supported := $(call as-instr,vpgatherdd %ymm0$(comma)(%eax$(comma)%ymm1\
	6	$(comma)4)$(comma)%ymm2,yes,no)
	7	ifeq ($(avx2_supported),yes)
	8	obj-$(CONFIG_CRYPTO_SHA512_MB) += sha512-mb.o
	9	sha512-mb-y := sha512_mb.o sha512_mb_mgr_flush_avx2.o \
	10	sha512_mb_mgr_init_avx2.o sha512_mb_mgr_submit_avx2.o sha512_x4_avx2.o
	11	endif

arch/x86/crypto/sha512-mb/sha512_mb.c+1046−0

@@ -0,0 +1,1046 @@
	1	/*
	2	* Multi buffer SHA512 algorithm Glue Code
	3	*
	4	* This file is provided under a dual BSD/GPLv2 license. When using or
	5	* redistributing this file, you may do so under either license.
	6	*
	7	* GPL LICENSE SUMMARY
	8	*
	9	* Copyright(c) 2016 Intel Corporation.
	10	*
	11	* This program is free software; you can redistribute it and/or modify
	12	* it under the terms of version 2 of the GNU General Public License as
	13	* published by the Free Software Foundation.
	14	*
	15	* This program is distributed in the hope that it will be useful, but
	16	* WITHOUT ANY WARRANTY; without even the implied warranty of
	17	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	18	* General Public License for more details.
	19	*
	20	* Contact Information:
	21	* Megha Dey <megha.dey@linux.intel.com>
	22	*
	23	* BSD LICENSE
	24	*
	25	* Copyright(c) 2016 Intel Corporation.
	26	*
	27	* Redistribution and use in source and binary forms, with or without
	28	* modification, are permitted provided that the following conditions
	29	* are met:
	30	*
	31	* * Redistributions of source code must retain the above copyright
	32	* notice, this list of conditions and the following disclaimer.
	33	* * Redistributions in binary form must reproduce the above copyright
	34	* notice, this list of conditions and the following disclaimer in
	35	* the documentation and/or other materials provided with the
	36	* distribution.
	37	* * Neither the name of Intel Corporation nor the names of its
	38	* contributors may be used to endorse or promote products derived
	39	* from this software without specific prior written permission.
	40	*
	41	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	42	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	43	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	44	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	45	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	46	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	47	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	48	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	49	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	50	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	51	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	52	*/
	53
	54	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
	55
	56	#include <crypto/internal/hash.h>
	57	#include <linux/init.h>
	58	#include <linux/module.h>
	59	#include <linux/mm.h>
	60	#include <linux/cryptohash.h>
	61	#include <linux/types.h>
	62	#include <linux/list.h>
	63	#include <crypto/scatterwalk.h>
	64	#include <crypto/sha.h>
	65	#include <crypto/mcryptd.h>
	66	#include <crypto/crypto_wq.h>
	67	#include <asm/byteorder.h>
	68	#include <linux/hardirq.h>
	69	#include <asm/fpu/api.h>
	70	#include "sha512_mb_ctx.h"
	71
	72	#define FLUSH_INTERVAL 1000 /* in usec */
	73
	74	static struct mcryptd_alg_state sha512_mb_alg_state;
	75
	76	struct sha512_mb_ctx {
	77	struct mcryptd_ahash *mcryptd_tfm;
	78	};
	79
	80	static inline struct mcryptd_hash_request_ctx
	81	cast_hash_to_mcryptd_ctx(struct sha512_hash_ctx hash_ctx)
	82	{
	83	struct ahash_request *areq;
	84
	85	areq = container_of((void *) hash_ctx, struct ahash_request, __ctx);
	86	return container_of(areq, struct mcryptd_hash_request_ctx, areq);
	87	}
	88
	89	static inline struct ahash_request
	90	cast_mcryptd_ctx_to_req(struct mcryptd_hash_request_ctx ctx)
	91	{
	92	return container_of((void *) ctx, struct ahash_request, __ctx);
	93	}
	94
	95	static void req_ctx_init(struct mcryptd_hash_request_ctx *rctx,
	96	struct ahash_request *areq)
	97	{
	98	rctx->flag = HASH_UPDATE;
	99	}
	100
	101	static asmlinkage void (sha512_job_mgr_init)(struct sha512_mb_mgr state);
	102	static asmlinkage struct job_sha512* (*sha512_job_mgr_submit)
	103	(struct sha512_mb_mgr *state,
	104	struct job_sha512 *job);
	105	static asmlinkage struct job_sha512* (*sha512_job_mgr_flush)
	106	(struct sha512_mb_mgr *state);
	107	static asmlinkage struct job_sha512* (*sha512_job_mgr_get_comp_job)
	108	(struct sha512_mb_mgr *state);
	109
	110	inline void sha512_init_digest(uint64_t *digest)
	111	{
	112	static const uint64_t initial_digest[SHA512_DIGEST_LENGTH] = {
	113	SHA512_H0, SHA512_H1, SHA512_H2,
	114	SHA512_H3, SHA512_H4, SHA512_H5,
	115	SHA512_H6, SHA512_H7 };
	116	memcpy(digest, initial_digest, sizeof(initial_digest));
	117	}
	118
	119	inline uint32_t sha512_pad(uint8_t padblock[SHA512_BLOCK_SIZE * 2],
	120	uint32_t total_len)
	121	{
	122	uint32_t i = total_len & (SHA512_BLOCK_SIZE - 1);
	123
	124	memset(&padblock[i], 0, SHA512_BLOCK_SIZE);
	125	padblock[i] = 0x80;
	126
	127	i += ((SHA512_BLOCK_SIZE - 1) &
	128	(0 - (total_len + SHA512_PADLENGTHFIELD_SIZE + 1)))
	129	+ 1 + SHA512_PADLENGTHFIELD_SIZE;
	130
	131	#if SHA512_PADLENGTHFIELD_SIZE == 16
	132	((uint64_t ) &padblock[i - 16]) = 0;
	133	#endif
	134
	135	((uint64_t ) &padblock[i - 8]) = cpu_to_be64(total_len << 3);
	136
	137	/* Number of extra blocks to hash */
	138	return i >> SHA512_LOG2_BLOCK_SIZE;
	139	}
	140
	141	static struct sha512_hash_ctx *sha512_ctx_mgr_resubmit
	142	(struct sha512_ctx_mgr mgr, struct sha512_hash_ctx ctx)
	143	{
	144	while (ctx) {
	145	if (ctx->status & HASH_CTX_STS_COMPLETE) {
	146	/* Clear PROCESSING bit */
	147	ctx->status = HASH_CTX_STS_COMPLETE;
	148	return ctx;
	149	}
	150
	151	/*
	152	* If the extra blocks are empty, begin hashing what remains
	153	* in the user's buffer.
	154	*/
	155	if (ctx->partial_block_buffer_length == 0 &&
	156	ctx->incoming_buffer_length) {
	157
	158	const void *buffer = ctx->incoming_buffer;
	159	uint32_t len = ctx->incoming_buffer_length;
	160	uint32_t copy_len;
	161
	162	/*
	163	* Only entire blocks can be hashed.
	164	* Copy remainder to extra blocks buffer.
	165	*/
	166	copy_len = len & (SHA512_BLOCK_SIZE-1);
	167
	168	if (copy_len) {
	169	len -= copy_len;
	170	memcpy(ctx->partial_block_buffer,
	171	((const char *) buffer + len),
	172	copy_len);
	173	ctx->partial_block_buffer_length = copy_len;
	174	}
	175
	176	ctx->incoming_buffer_length = 0;
	177
	178	/* len should be a multiple of the block size now */
	179	assert((len % SHA512_BLOCK_SIZE) == 0);
	180
	181	/* Set len to the number of blocks to be hashed */
	182	len >>= SHA512_LOG2_BLOCK_SIZE;
	183
	184	if (len) {
	185
	186	ctx->job.buffer = (uint8_t *) buffer;
	187	ctx->job.len = len;
	188	ctx = (struct sha512_hash_ctx *)
	189	sha512_job_mgr_submit(&mgr->mgr,
	190	&ctx->job);
	191	continue;
	192	}
	193	}
	194
	195	/*
	196	* If the extra blocks are not empty, then we are
	197	* either on the last block(s) or we need more
	198	* user input before continuing.
	199	*/
	200	if (ctx->status & HASH_CTX_STS_LAST) {
	201
	202	uint8_t *buf = ctx->partial_block_buffer;
	203	uint32_t n_extra_blocks =
	204	sha512_pad(buf, ctx->total_length);
	205
	206	ctx->status = (HASH_CTX_STS_PROCESSING \|
	207	HASH_CTX_STS_COMPLETE);
	208	ctx->job.buffer = buf;
	209	ctx->job.len = (uint32_t) n_extra_blocks;
	210	ctx = (struct sha512_hash_ctx *)
	211	sha512_job_mgr_submit(&mgr->mgr, &ctx->job);
	212	continue;
	213	}
	214
	215	if (ctx)
	216	ctx->status = HASH_CTX_STS_IDLE;
	217	return ctx;
	218	}
	219
	220	return NULL;
	221	}
	222
	223	static struct sha512_hash_ctx
	224	sha512_ctx_mgr_get_comp_ctx(struct sha512_ctx_mgr mgr)
	225	{
	226	/*
	227	* If get_comp_job returns NULL, there are no jobs complete.
	228	* If get_comp_job returns a job, verify that it is safe to return to
	229	* the user.
	230	* If it is not ready, resubmit the job to finish processing.
	231	* If sha512_ctx_mgr_resubmit returned a job, it is ready to be
	232	* returned.
	233	* Otherwise, all jobs currently being managed by the hash_ctx_mgr
	234	* still need processing.
	235	*/
	236	struct sha512_hash_ctx *ctx;
	237
	238	ctx = (struct sha512_hash_ctx *)
	239	sha512_job_mgr_get_comp_job(&mgr->mgr);
	240	return sha512_ctx_mgr_resubmit(mgr, ctx);
	241	}
	242
	243	static void sha512_ctx_mgr_init(struct sha512_ctx_mgr *mgr)
	244	{
	245	sha512_job_mgr_init(&mgr->mgr);
	246	}
	247
	248	static struct sha512_hash_ctx
	249	sha512_ctx_mgr_submit(struct sha512_ctx_mgr mgr,
	250	struct sha512_hash_ctx *ctx,
	251	const void *buffer,
	252	uint32_t len,
	253	int flags)
	254	{
	255	if (flags & (~HASH_ENTIRE)) {
	256	/*
	257	* User should not pass anything other than FIRST, UPDATE, or
	258	* LAST
	259	*/
	260	ctx->error = HASH_CTX_ERROR_INVALID_FLAGS;
	261	return ctx;
	262	}
	263
	264	if (ctx->status & HASH_CTX_STS_PROCESSING) {
	265	/* Cannot submit to a currently processing job. */
	266	ctx->error = HASH_CTX_ERROR_ALREADY_PROCESSING;
	267	return ctx;
	268	}
	269
	270	if ((ctx->status & HASH_CTX_STS_COMPLETE) && !(flags & HASH_FIRST)) {
	271	/* Cannot update a finished job. */
	272	ctx->error = HASH_CTX_ERROR_ALREADY_COMPLETED;
	273	return ctx;
	274	}
	275
	276
	277	if (flags & HASH_FIRST) {
	278	/* Init digest */
	279	sha512_init_digest(ctx->job.result_digest);
	280
	281	/* Reset byte counter */
	282	ctx->total_length = 0;
	283
	284	/* Clear extra blocks */
	285	ctx->partial_block_buffer_length = 0;
	286	}
	287
	288	/*
	289	* If we made it here, there were no errors during this call to
	290	* submit
	291	*/
	292	ctx->error = HASH_CTX_ERROR_NONE;
	293
	294	/* Store buffer ptr info from user */
	295	ctx->incoming_buffer = buffer;
	296	ctx->incoming_buffer_length = len;
	297
	298	/*
	299	* Store the user's request flags and mark this ctx as currently being
	300	* processed.
	301	*/
	302	ctx->status = (flags & HASH_LAST) ?
	303	(HASH_CTX_STS_PROCESSING \| HASH_CTX_STS_LAST) :
	304	HASH_CTX_STS_PROCESSING;
	305
	306	/* Advance byte counter */
	307	ctx->total_length += len;
	308
	309	/*
	310	* If there is anything currently buffered in the extra blocks,
	311	* append to it until it contains a whole block.
	312	* Or if the user's buffer contains less than a whole block,
	313	* append as much as possible to the extra block.
	314	*/
	315	if (ctx->partial_block_buffer_length \|\| len < SHA512_BLOCK_SIZE) {
	316	/* Compute how many bytes to copy from user buffer into extra
	317	* block
	318	*/
	319	uint32_t copy_len = SHA512_BLOCK_SIZE -
	320	ctx->partial_block_buffer_length;
	321	if (len < copy_len)
	322	copy_len = len;
	323
	324	if (copy_len) {
	325	/* Copy and update relevant pointers and counters */
	326	memcpy
	327	(&ctx->partial_block_buffer[ctx->partial_block_buffer_length],
	328	buffer, copy_len);
	329
	330	ctx->partial_block_buffer_length += copy_len;
	331	ctx->incoming_buffer = (const void *)
	332	((const char *)buffer + copy_len);
	333	ctx->incoming_buffer_length = len - copy_len;
	334	}
	335
	336	/* The extra block should never contain more than 1 block
	337	* here
	338	*/
	339	assert(ctx->partial_block_buffer_length <= SHA512_BLOCK_SIZE);
	340
	341	/* If the extra block buffer contains exactly 1 block, it can
	342	* be hashed.
	343	*/
	344	if (ctx->partial_block_buffer_length >= SHA512_BLOCK_SIZE) {
	345	ctx->partial_block_buffer_length = 0;
	346
	347	ctx->job.buffer = ctx->partial_block_buffer;
	348	ctx->job.len = 1;
	349	ctx = (struct sha512_hash_ctx *)
	350	sha512_job_mgr_submit(&mgr->mgr, &ctx->job);
	351	}
	352	}
	353
	354	return sha512_ctx_mgr_resubmit(mgr, ctx);
	355	}
	356
	357	static struct sha512_hash_ctx sha512_ctx_mgr_flush(struct sha512_ctx_mgr mgr)
	358	{
	359	struct sha512_hash_ctx *ctx;
	360
	361	while (1) {
	362	ctx = (struct sha512_hash_ctx *)
	363	sha512_job_mgr_flush(&mgr->mgr);
	364
	365	/* If flush returned 0, there are no more jobs in flight. */
	366	if (!ctx)
	367	return NULL;
	368
	369	/*
	370	* If flush returned a job, resubmit the job to finish
	371	* processing.
	372	*/
	373	ctx = sha512_ctx_mgr_resubmit(mgr, ctx);
	374
	375	/*
	376	* If sha512_ctx_mgr_resubmit returned a job, it is ready to
	377	* be returned. Otherwise, all jobs currently being managed by
	378	* the sha512_ctx_mgr still need processing. Loop.
	379	*/
	380	if (ctx)
	381	return ctx;
	382	}
	383	}
	384
	385	static int sha512_mb_init(struct ahash_request *areq)
	386	{
	387	struct sha512_hash_ctx *sctx = ahash_request_ctx(areq);
	388
	389	hash_ctx_init(sctx);
	390	sctx->job.result_digest[0] = SHA512_H0;
	391	sctx->job.result_digest[1] = SHA512_H1;
	392	sctx->job.result_digest[2] = SHA512_H2;
	393	sctx->job.result_digest[3] = SHA512_H3;
	394	sctx->job.result_digest[4] = SHA512_H4;
	395	sctx->job.result_digest[5] = SHA512_H5;
	396	sctx->job.result_digest[6] = SHA512_H6;
	397	sctx->job.result_digest[7] = SHA512_H7;
	398	sctx->total_length = 0;
	399	sctx->partial_block_buffer_length = 0;
	400	sctx->status = HASH_CTX_STS_IDLE;
	401
	402	return 0;
	403	}
	404
	405	static int sha512_mb_set_results(struct mcryptd_hash_request_ctx *rctx)
	406	{
	407	int i;
	408	struct sha512_hash_ctx *sctx = ahash_request_ctx(&rctx->areq);
	409	__be64 dst = (__be64 ) rctx->out;
	410
	411	for (i = 0; i < 8; ++i)
	412	dst[i] = cpu_to_be64(sctx->job.result_digest[i]);
	413
	414	return 0;
	415	}
	416
	417	static int sha_finish_walk(struct mcryptd_hash_request_ctx **ret_rctx,
	418	struct mcryptd_alg_cstate *cstate, bool flush)
	419	{
	420	int flag = HASH_UPDATE;
	421	int nbytes, err = 0;
	422	struct mcryptd_hash_request_ctx rctx = ret_rctx;
	423	struct sha512_hash_ctx *sha_ctx;
	424
	425	/* more work ? */
	426	while (!(rctx->flag & HASH_DONE)) {
	427	nbytes = crypto_ahash_walk_done(&rctx->walk, 0);
	428	if (nbytes < 0) {
	429	err = nbytes;
	430	goto out;
	431	}
	432	/* check if the walk is done */
	433	if (crypto_ahash_walk_last(&rctx->walk)) {
	434	rctx->flag \|= HASH_DONE;
	435	if (rctx->flag & HASH_FINAL)
	436	flag \|= HASH_LAST;
	437
	438	}
	439	sha_ctx = (struct sha512_hash_ctx *)
	440	ahash_request_ctx(&rctx->areq);
	441	kernel_fpu_begin();
	442	sha_ctx = sha512_ctx_mgr_submit(cstate->mgr, sha_ctx,
	443	rctx->walk.data, nbytes, flag);
	444	if (!sha_ctx) {
	445	if (flush)
	446	sha_ctx = sha512_ctx_mgr_flush(cstate->mgr);
	447	}
	448	kernel_fpu_end();
	449	if (sha_ctx)
	450	rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
	451	else {
	452	rctx = NULL;
	453	goto out;
	454	}
	455	}
	456
	457	/* copy the results */
	458	if (rctx->flag & HASH_FINAL)
	459	sha512_mb_set_results(rctx);
	460
	461	out:
	462	*ret_rctx = rctx;
	463	return err;
	464	}
	465
	466	static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx,
	467	struct mcryptd_alg_cstate *cstate,
	468	int err)
	469	{
	470	struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
	471	struct sha512_hash_ctx *sha_ctx;
	472	struct mcryptd_hash_request_ctx *req_ctx;
	473	int ret;
	474
	475	/* remove from work list */
	476	spin_lock(&cstate->work_lock);
	477	list_del(&rctx->waiter);
	478	spin_unlock(&cstate->work_lock);
	479
	480	if (irqs_disabled())
	481	rctx->complete(&req->base, err);
	482	else {
	483	local_bh_disable();
	484	rctx->complete(&req->base, err);
	485	local_bh_enable();
	486	}
	487
	488	/* check to see if there are other jobs that are done */
	489	sha_ctx = sha512_ctx_mgr_get_comp_ctx(cstate->mgr);
	490	while (sha_ctx) {
	491	req_ctx = cast_hash_to_mcryptd_ctx(sha_ctx);
	492	ret = sha_finish_walk(&req_ctx, cstate, false);
	493	if (req_ctx) {
	494	spin_lock(&cstate->work_lock);
	495	list_del(&req_ctx->waiter);
	496	spin_unlock(&cstate->work_lock);
	497
	498	req = cast_mcryptd_ctx_to_req(req_ctx);
	499	if (irqs_disabled())
	500	rctx->complete(&req->base, ret);
	501	else {
	502	local_bh_disable();
	503	rctx->complete(&req->base, ret);
	504	local_bh_enable();
	505	}
	506	}
	507	sha_ctx = sha512_ctx_mgr_get_comp_ctx(cstate->mgr);
	508	}
	509
	510	return 0;
	511	}
	512
	513	static void sha512_mb_add_list(struct mcryptd_hash_request_ctx *rctx,
	514	struct mcryptd_alg_cstate *cstate)
	515	{
	516	unsigned long next_flush;
	517	unsigned long delay = usecs_to_jiffies(FLUSH_INTERVAL);
	518
	519	/* initialize tag */
	520	rctx->tag.arrival = jiffies; /* tag the arrival time */
	521	rctx->tag.seq_num = cstate->next_seq_num++;
	522	next_flush = rctx->tag.arrival + delay;
	523	rctx->tag.expire = next_flush;
	524
	525	spin_lock(&cstate->work_lock);
	526	list_add_tail(&rctx->waiter, &cstate->work_list);
	527	spin_unlock(&cstate->work_lock);
	528
	529	mcryptd_arm_flusher(cstate, delay);
	530	}
	531
	532	static int sha512_mb_update(struct ahash_request *areq)
	533	{
	534	struct mcryptd_hash_request_ctx *rctx =
	535	container_of(areq, struct mcryptd_hash_request_ctx,
	536	areq);
	537	struct mcryptd_alg_cstate *cstate =
	538	this_cpu_ptr(sha512_mb_alg_state.alg_cstate);
	539
	540	struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
	541	struct sha512_hash_ctx *sha_ctx;
	542	int ret = 0, nbytes;
	543
	544
	545	/* sanity check */
	546	if (rctx->tag.cpu != smp_processor_id()) {
	547	pr_err("mcryptd error: cpu clash\n");
	548	goto done;
	549	}
	550
	551	/* need to init context */
	552	req_ctx_init(rctx, areq);
	553
	554	nbytes = crypto_ahash_walk_first(req, &rctx->walk);
	555
	556	if (nbytes < 0) {
	557	ret = nbytes;
	558	goto done;
	559	}
	560
	561	if (crypto_ahash_walk_last(&rctx->walk))
	562	rctx->flag \|= HASH_DONE;
	563
	564	/* submit */
	565	sha_ctx = (struct sha512_hash_ctx *) ahash_request_ctx(areq);
	566	sha512_mb_add_list(rctx, cstate);
	567	kernel_fpu_begin();
	568	sha_ctx = sha512_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data,
	569	nbytes, HASH_UPDATE);
	570	kernel_fpu_end();
	571
	572	/* check if anything is returned */
	573	if (!sha_ctx)
	574	return -EINPROGRESS;
	575
	576	if (sha_ctx->error) {
	577	ret = sha_ctx->error;
	578	rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
	579	goto done;
	580	}
	581
	582	rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
	583	ret = sha_finish_walk(&rctx, cstate, false);
	584
	585	if (!rctx)
	586	return -EINPROGRESS;
	587	done:
	588	sha_complete_job(rctx, cstate, ret);
	589	return ret;
	590	}
	591
	592	static int sha512_mb_finup(struct ahash_request *areq)
	593	{
	594	struct mcryptd_hash_request_ctx *rctx =
	595	container_of(areq, struct mcryptd_hash_request_ctx,
	596	areq);
	597	struct mcryptd_alg_cstate *cstate =
	598	this_cpu_ptr(sha512_mb_alg_state.alg_cstate);
	599
	600	struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
	601	struct sha512_hash_ctx *sha_ctx;
	602	int ret = 0, flag = HASH_UPDATE, nbytes;
	603
	604	/* sanity check */
	605	if (rctx->tag.cpu != smp_processor_id()) {
	606	pr_err("mcryptd error: cpu clash\n");
	607	goto done;
	608	}
	609
	610	/* need to init context */
	611	req_ctx_init(rctx, areq);
	612
	613	nbytes = crypto_ahash_walk_first(req, &rctx->walk);
	614
	615	if (nbytes < 0) {
	616	ret = nbytes;
	617	goto done;
	618	}
	619
	620	if (crypto_ahash_walk_last(&rctx->walk)) {
	621	rctx->flag \|= HASH_DONE;
	622	flag = HASH_LAST;
	623	}
	624
	625	/* submit */
	626	rctx->flag \|= HASH_FINAL;
	627	sha_ctx = (struct sha512_hash_ctx *) ahash_request_ctx(areq);
	628	sha512_mb_add_list(rctx, cstate);
	629
	630	kernel_fpu_begin();
	631	sha_ctx = sha512_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data,
	632	nbytes, flag);
	633	kernel_fpu_end();
	634
	635	/* check if anything is returned */
	636	if (!sha_ctx)
	637	return -EINPROGRESS;
	638
	639	if (sha_ctx->error) {
	640	ret = sha_ctx->error;
	641	goto done;
	642	}
	643
	644	rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
	645	ret = sha_finish_walk(&rctx, cstate, false);
	646	if (!rctx)
	647	return -EINPROGRESS;
	648	done:
	649	sha_complete_job(rctx, cstate, ret);
	650	return ret;
	651	}
	652
	653	static int sha512_mb_final(struct ahash_request *areq)
	654	{
	655	struct mcryptd_hash_request_ctx *rctx =
	656	container_of(areq, struct mcryptd_hash_request_ctx,
	657	areq);
	658	struct mcryptd_alg_cstate *cstate =
	659	this_cpu_ptr(sha512_mb_alg_state.alg_cstate);
	660
	661	struct sha512_hash_ctx *sha_ctx;
	662	int ret = 0;
	663	u8 data;
	664
	665	/* sanity check */
	666	if (rctx->tag.cpu != smp_processor_id()) {
	667	pr_err("mcryptd error: cpu clash\n");
	668	goto done;
	669	}
	670
	671	/* need to init context */
	672	req_ctx_init(rctx, areq);
	673
	674	rctx->flag \|= HASH_DONE \| HASH_FINAL;
	675
	676	sha_ctx = (struct sha512_hash_ctx *) ahash_request_ctx(areq);
	677	/* flag HASH_FINAL and 0 data size */
	678	sha512_mb_add_list(rctx, cstate);
	679	kernel_fpu_begin();
	680	sha_ctx = sha512_ctx_mgr_submit(cstate->mgr, sha_ctx, &data, 0,
	681	HASH_LAST);
	682	kernel_fpu_end();
	683
	684	/* check if anything is returned */
	685	if (!sha_ctx)
	686	return -EINPROGRESS;
	687
	688	if (sha_ctx->error) {
	689	ret = sha_ctx->error;
	690	rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
	691	goto done;
	692	}
	693
	694	rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
	695	ret = sha_finish_walk(&rctx, cstate, false);
	696	if (!rctx)
	697	return -EINPROGRESS;
	698	done:
	699	sha_complete_job(rctx, cstate, ret);
	700	return ret;
	701	}
	702
	703	static int sha512_mb_export(struct ahash_request areq, void out)
	704	{
	705	struct sha512_hash_ctx *sctx = ahash_request_ctx(areq);
	706
	707	memcpy(out, sctx, sizeof(*sctx));
	708
	709	return 0;
	710	}
	711
	712	static int sha512_mb_import(struct ahash_request areq, const void in)
	713	{
	714	struct sha512_hash_ctx *sctx = ahash_request_ctx(areq);
	715
	716	memcpy(sctx, in, sizeof(*sctx));
	717
	718	return 0;
	719	}
	720
	721	static int sha512_mb_async_init_tfm(struct crypto_tfm *tfm)
	722	{
	723	struct mcryptd_ahash *mcryptd_tfm;
	724	struct sha512_mb_ctx *ctx = crypto_tfm_ctx(tfm);
	725	struct mcryptd_hash_ctx *mctx;
	726
	727	mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha512-mb",
	728	CRYPTO_ALG_INTERNAL,
	729	CRYPTO_ALG_INTERNAL);
	730	if (IS_ERR(mcryptd_tfm))
	731	return PTR_ERR(mcryptd_tfm);
	732	mctx = crypto_ahash_ctx(&mcryptd_tfm->base);
	733	mctx->alg_state = &sha512_mb_alg_state;
	734	ctx->mcryptd_tfm = mcryptd_tfm;
	735	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
	736	sizeof(struct ahash_request) +
	737	crypto_ahash_reqsize(&mcryptd_tfm->base));
	738
	739	return 0;
	740	}
	741
	742	static void sha512_mb_async_exit_tfm(struct crypto_tfm *tfm)
	743	{
	744	struct sha512_mb_ctx *ctx = crypto_tfm_ctx(tfm);
	745
	746	mcryptd_free_ahash(ctx->mcryptd_tfm);
	747	}
	748
	749	static int sha512_mb_areq_init_tfm(struct crypto_tfm *tfm)
	750	{
	751	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
	752	sizeof(struct ahash_request) +
	753	sizeof(struct sha512_hash_ctx));
	754
	755	return 0;
	756	}
	757
	758	static void sha512_mb_areq_exit_tfm(struct crypto_tfm *tfm)
	759	{
	760	struct sha512_mb_ctx *ctx = crypto_tfm_ctx(tfm);
	761
	762	mcryptd_free_ahash(ctx->mcryptd_tfm);
	763	}
	764
	765	static struct ahash_alg sha512_mb_areq_alg = {
	766	.init = sha512_mb_init,
	767	.update = sha512_mb_update,
	768	.final = sha512_mb_final,
	769	.finup = sha512_mb_finup,
	770	.export = sha512_mb_export,
	771	.import = sha512_mb_import,
	772	.halg = {
	773	.digestsize = SHA512_DIGEST_SIZE,
	774	.statesize = sizeof(struct sha512_hash_ctx),
	775	.base = {
	776	.cra_name = "__sha512-mb",
	777	.cra_driver_name = "__intel_sha512-mb",
	778	.cra_priority = 100,
	779	/*
	780	* use ASYNC flag as some buffers in multi-buffer
	781	* algo may not have completed before hashing thread
	782	* sleep
	783	*/
	784	.cra_flags = CRYPTO_ALG_TYPE_AHASH \|
	785	CRYPTO_ALG_ASYNC \|
	786	CRYPTO_ALG_INTERNAL,
	787	.cra_blocksize = SHA512_BLOCK_SIZE,
	788	.cra_module = THIS_MODULE,
	789	.cra_list = LIST_HEAD_INIT
	790	(sha512_mb_areq_alg.halg.base.cra_list),
	791	.cra_init = sha512_mb_areq_init_tfm,
	792	.cra_exit = sha512_mb_areq_exit_tfm,
	793	.cra_ctxsize = sizeof(struct sha512_hash_ctx),
	794	}
	795	}
	796	};
	797
	798	static int sha512_mb_async_init(struct ahash_request *req)
	799	{
	800	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	801	struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm);
	802	struct ahash_request *mcryptd_req = ahash_request_ctx(req);
	803	struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
	804
	805	memcpy(mcryptd_req, req, sizeof(*req));
	806	ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
	807	return crypto_ahash_init(mcryptd_req);
	808	}
	809
	810	static int sha512_mb_async_update(struct ahash_request *req)
	811	{
	812	struct ahash_request *mcryptd_req = ahash_request_ctx(req);
	813
	814	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	815	struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm);
	816	struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
	817
	818	memcpy(mcryptd_req, req, sizeof(*req));
	819	ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
	820	return crypto_ahash_update(mcryptd_req);
	821	}
	822
	823	static int sha512_mb_async_finup(struct ahash_request *req)
	824	{
	825	struct ahash_request *mcryptd_req = ahash_request_ctx(req);
	826
	827	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	828	struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm);
	829	struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
	830
	831	memcpy(mcryptd_req, req, sizeof(*req));
	832	ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
	833	return crypto_ahash_finup(mcryptd_req);
	834	}
	835
	836	static int sha512_mb_async_final(struct ahash_request *req)
	837	{
	838	struct ahash_request *mcryptd_req = ahash_request_ctx(req);
	839
	840	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	841	struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm);
	842	struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
	843
	844	memcpy(mcryptd_req, req, sizeof(*req));
	845	ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
	846	return crypto_ahash_final(mcryptd_req);
	847	}
	848
	849	static int sha512_mb_async_digest(struct ahash_request *req)
	850	{
	851	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	852	struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm);
	853	struct ahash_request *mcryptd_req = ahash_request_ctx(req);
	854	struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
	855
	856	memcpy(mcryptd_req, req, sizeof(*req));
	857	ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
	858	return crypto_ahash_digest(mcryptd_req);
	859	}
	860
	861	static int sha512_mb_async_export(struct ahash_request req, void out)
	862	{
	863	struct ahash_request *mcryptd_req = ahash_request_ctx(req);
	864	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	865	struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm);
	866	struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
	867
	868	memcpy(mcryptd_req, req, sizeof(*req));
	869	ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
	870	return crypto_ahash_export(mcryptd_req, out);
	871	}
	872
	873	static int sha512_mb_async_import(struct ahash_request req, const void in)
	874	{
	875	struct ahash_request *mcryptd_req = ahash_request_ctx(req);
	876	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	877	struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm);
	878	struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
	879	struct crypto_ahash *child = mcryptd_ahash_child(mcryptd_tfm);
	880	struct mcryptd_hash_request_ctx *rctx;
	881	struct ahash_request *areq;
	882
	883	memcpy(mcryptd_req, req, sizeof(*req));
	884	ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
	885	rctx = ahash_request_ctx(mcryptd_req);
	886
	887	areq = &rctx->areq;
	888
	889	ahash_request_set_tfm(areq, child);
	890	ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_SLEEP,
	891	rctx->complete, req);
	892
	893	return crypto_ahash_import(mcryptd_req, in);
	894	}
	895
	896	static struct ahash_alg sha512_mb_async_alg = {
	897	.init = sha512_mb_async_init,
	898	.update = sha512_mb_async_update,
	899	.final = sha512_mb_async_final,
	900	.finup = sha512_mb_async_finup,
	901	.digest = sha512_mb_async_digest,
	902	.export = sha512_mb_async_export,
	903	.import = sha512_mb_async_import,
	904	.halg = {
	905	.digestsize = SHA512_DIGEST_SIZE,
	906	.statesize = sizeof(struct sha512_hash_ctx),
	907	.base = {
	908	.cra_name = "sha512",
	909	.cra_driver_name = "sha512_mb",
	910	.cra_priority = 200,
	911	.cra_flags = CRYPTO_ALG_TYPE_AHASH \|
	912	CRYPTO_ALG_ASYNC,
	913	.cra_blocksize = SHA512_BLOCK_SIZE,
	914	.cra_type = &crypto_ahash_type,
	915	.cra_module = THIS_MODULE,
	916	.cra_list = LIST_HEAD_INIT
	917	(sha512_mb_async_alg.halg.base.cra_list),
	918	.cra_init = sha512_mb_async_init_tfm,
	919	.cra_exit = sha512_mb_async_exit_tfm,
	920	.cra_ctxsize = sizeof(struct sha512_mb_ctx),
	921	.cra_alignmask = 0,
	922	},
	923	},
	924	};
	925
	926	static unsigned long sha512_mb_flusher(struct mcryptd_alg_cstate *cstate)
	927	{
	928	struct mcryptd_hash_request_ctx *rctx;
	929	unsigned long cur_time;
	930	unsigned long next_flush = 0;
	931	struct sha512_hash_ctx *sha_ctx;
	932
	933
	934	cur_time = jiffies;
	935
	936	while (!list_empty(&cstate->work_list)) {
	937	rctx = list_entry(cstate->work_list.next,
	938	struct mcryptd_hash_request_ctx, waiter);
	939	if time_before(cur_time, rctx->tag.expire)
	940	break;
	941	kernel_fpu_begin();
	942	sha_ctx = (struct sha512_hash_ctx *)
	943	sha512_ctx_mgr_flush(cstate->mgr);
	944	kernel_fpu_end();
	945	if (!sha_ctx) {
	946	pr_err("sha512_mb error: nothing got flushed for"
	947	" non-empty list\n");
	948	break;
	949	}
	950	rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
	951	sha_finish_walk(&rctx, cstate, true);
	952	sha_complete_job(rctx, cstate, 0);
	953	}
	954
	955	if (!list_empty(&cstate->work_list)) {
	956	rctx = list_entry(cstate->work_list.next,
	957	struct mcryptd_hash_request_ctx, waiter);
	958	/* get the hash context and then flush time */
	959	next_flush = rctx->tag.expire;
	960	mcryptd_arm_flusher(cstate, get_delay(next_flush));
	961	}
	962	return next_flush;
	963	}
	964
	965	static int __init sha512_mb_mod_init(void)
	966	{
	967
	968	int cpu;
	969	int err;
	970	struct mcryptd_alg_cstate *cpu_state;
	971
	972	/* check for dependent cpu features */
	973	if (!boot_cpu_has(X86_FEATURE_AVX2) \|\|
	974	!boot_cpu_has(X86_FEATURE_BMI2))
	975	return -ENODEV;
	976
	977	/* initialize multibuffer structures */
	978	sha512_mb_alg_state.alg_cstate =
	979	alloc_percpu(struct mcryptd_alg_cstate);
	980
	981	sha512_job_mgr_init = sha512_mb_mgr_init_avx2;
	982	sha512_job_mgr_submit = sha512_mb_mgr_submit_avx2;
	983	sha512_job_mgr_flush = sha512_mb_mgr_flush_avx2;
	984	sha512_job_mgr_get_comp_job = sha512_mb_mgr_get_comp_job_avx2;
	985
	986	if (!sha512_mb_alg_state.alg_cstate)
	987	return -ENOMEM;
	988	for_each_possible_cpu(cpu) {
	989	cpu_state = per_cpu_ptr(sha512_mb_alg_state.alg_cstate, cpu);
	990	cpu_state->next_flush = 0;
	991	cpu_state->next_seq_num = 0;
	992	cpu_state->flusher_engaged = false;
	993	INIT_DELAYED_WORK(&cpu_state->flush, mcryptd_flusher);
	994	cpu_state->cpu = cpu;
	995	cpu_state->alg_state = &sha512_mb_alg_state;
	996	cpu_state->mgr = kzalloc(sizeof(struct sha512_ctx_mgr),
	997	GFP_KERNEL);
	998	if (!cpu_state->mgr)
	999	goto err2;
	1000	sha512_ctx_mgr_init(cpu_state->mgr);
	1001	INIT_LIST_HEAD(&cpu_state->work_list);
	1002	spin_lock_init(&cpu_state->work_lock);
	1003	}
	1004	sha512_mb_alg_state.flusher = &sha512_mb_flusher;
	1005
	1006	err = crypto_register_ahash(&sha512_mb_areq_alg);
	1007	if (err)
	1008	goto err2;
	1009	err = crypto_register_ahash(&sha512_mb_async_alg);
	1010	if (err)
	1011	goto err1;
	1012
	1013
	1014	return 0;
	1015	err1:
	1016	crypto_unregister_ahash(&sha512_mb_areq_alg);
	1017	err2:
	1018	for_each_possible_cpu(cpu) {
	1019	cpu_state = per_cpu_ptr(sha512_mb_alg_state.alg_cstate, cpu);
	1020	kfree(cpu_state->mgr);
	1021	}
	1022	free_percpu(sha512_mb_alg_state.alg_cstate);
	1023	return -ENODEV;
	1024	}
	1025
	1026	static void __exit sha512_mb_mod_fini(void)
	1027	{
	1028	int cpu;
	1029	struct mcryptd_alg_cstate *cpu_state;
	1030
	1031	crypto_unregister_ahash(&sha512_mb_async_alg);
	1032	crypto_unregister_ahash(&sha512_mb_areq_alg);
	1033	for_each_possible_cpu(cpu) {
	1034	cpu_state = per_cpu_ptr(sha512_mb_alg_state.alg_cstate, cpu);
	1035	kfree(cpu_state->mgr);
	1036	}
	1037	free_percpu(sha512_mb_alg_state.alg_cstate);
	1038	}
	1039
	1040	module_init(sha512_mb_mod_init);
	1041	module_exit(sha512_mb_mod_fini);
	1042
	1043	MODULE_LICENSE("GPL");
	1044	MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, multi buffer accelerated");
	1045
	1046	MODULE_ALIAS("sha512");

arch/x86/crypto/sha512-mb/sha512_mb_ctx.h+130−0

@@ -0,0 +1,130 @@
	1	/*
	2	* Header file for multi buffer SHA512 context
	3	*
	4	* This file is provided under a dual BSD/GPLv2 license. When using or
	5	* redistributing this file, you may do so under either license.
	6	*
	7	* GPL LICENSE SUMMARY
	8	*
	9	* Copyright(c) 2016 Intel Corporation.
	10	*
	11	* This program is free software; you can redistribute it and/or modify
	12	* it under the terms of version 2 of the GNU General Public License as
	13	* published by the Free Software Foundation.
	14	*
	15	* This program is distributed in the hope that it will be useful, but
	16	* WITHOUT ANY WARRANTY; without even the implied warranty of
	17	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	18	* General Public License for more details.
	19	*
	20	* Contact Information:
	21	* Megha Dey <megha.dey@linux.intel.com>
	22	*
	23	* BSD LICENSE
	24	*
	25	* Copyright(c) 2016 Intel Corporation.
	26	*
	27	* Redistribution and use in source and binary forms, with or without
	28	* modification, are permitted provided that the following conditions
	29	* are met:
	30	*
	31	* * Redistributions of source code must retain the above copyright
	32	* notice, this list of conditions and the following disclaimer.
	33	* * Redistributions in binary form must reproduce the above copyright
	34	* notice, this list of conditions and the following disclaimer in
	35	* the documentation and/or other materials provided with the
	36	* distribution.
	37	* * Neither the name of Intel Corporation nor the names of its
	38	* contributors may be used to endorse or promote products derived
	39	* from this software without specific prior written permission.
	40	*
	41	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	42	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	43	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	44	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	45	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	46	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	47	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	48	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	49	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	50	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	51	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	52	*/
	53
	54	#ifndef _SHA_MB_CTX_INTERNAL_H
	55	#define _SHA_MB_CTX_INTERNAL_H
	56
	57	#include "sha512_mb_mgr.h"
	58
	59	#define HASH_UPDATE 0x00
	60	#define HASH_FIRST 0x01
	61	#define HASH_LAST 0x02
	62	#define HASH_ENTIRE 0x03
	63	#define HASH_DONE 0x04
	64	#define HASH_FINAL 0x08
	65
	66	#define HASH_CTX_STS_IDLE 0x00
	67	#define HASH_CTX_STS_PROCESSING 0x01
	68	#define HASH_CTX_STS_LAST 0x02
	69	#define HASH_CTX_STS_COMPLETE 0x04
	70
	71	enum hash_ctx_error {
	72	HASH_CTX_ERROR_NONE = 0,
	73	HASH_CTX_ERROR_INVALID_FLAGS = -1,
	74	HASH_CTX_ERROR_ALREADY_PROCESSING = -2,
	75	HASH_CTX_ERROR_ALREADY_COMPLETED = -3,
	76	};
	77
	78	#define hash_ctx_user_data(ctx) ((ctx)->user_data)
	79	#define hash_ctx_digest(ctx) ((ctx)->job.result_digest)
	80	#define hash_ctx_processing(ctx) ((ctx)->status & HASH_CTX_STS_PROCESSING)
	81	#define hash_ctx_complete(ctx) ((ctx)->status == HASH_CTX_STS_COMPLETE)
	82	#define hash_ctx_status(ctx) ((ctx)->status)
	83	#define hash_ctx_error(ctx) ((ctx)->error)
	84	#define hash_ctx_init(ctx) \
	85	do { \
	86	(ctx)->error = HASH_CTX_ERROR_NONE; \
	87	(ctx)->status = HASH_CTX_STS_COMPLETE; \
	88	} while (0)
	89
	90	/* Hash Constants and Typedefs */
	91	#define SHA512_DIGEST_LENGTH 8
	92	#define SHA512_LOG2_BLOCK_SIZE 7
	93
	94	#define SHA512_PADLENGTHFIELD_SIZE 16
	95
	96	#ifdef SHA_MB_DEBUG
	97	#define assert(expr) \
	98	do { \
	99	if (unlikely(!(expr))) { \
	100	printk(KERN_ERR "Assertion failed! %s,%s,%s,line=%d\n", \
	101	#expr, __FILE__, __func__, __LINE__); \
	102	} \
	103	} while (0)
	104	#else
	105	#define assert(expr) do {} while (0)
	106	#endif
	107
	108	struct sha512_ctx_mgr {
	109	struct sha512_mb_mgr mgr;
	110	};
	111
	112	/* typedef struct sha512_ctx_mgr sha512_ctx_mgr; */
	113
	114	struct sha512_hash_ctx {
	115	/* Must be at struct offset 0 */
	116	struct job_sha512 job;
	117	/* status flag */
	118	int status;
	119	/* error flag */
	120	int error;
	121
	122	uint32_t total_length;
	123	const void *incoming_buffer;
	124	uint32_t incoming_buffer_length;
	125	uint8_t partial_block_buffer[SHA512_BLOCK_SIZE * 2];
	126	uint32_t partial_block_buffer_length;
	127	void *user_data;
	128	};
	129
	130	#endif

arch/x86/crypto/sha512-mb/sha512_mb_mgr.h+104−0

@@ -0,0 +1,104 @@
	1	/*
	2	* Header file for multi buffer SHA512 algorithm manager
	3	*
	4	* This file is provided under a dual BSD/GPLv2 license. When using or
	5	* redistributing this file, you may do so under either license.
	6	*
	7	* GPL LICENSE SUMMARY
	8	*
	9	* Copyright(c) 2016 Intel Corporation.
	10	*
	11	* This program is free software; you can redistribute it and/or modify
	12	* it under the terms of version 2 of the GNU General Public License as
	13	* published by the Free Software Foundation.
	14	*
	15	* This program is distributed in the hope that it will be useful, but
	16	* WITHOUT ANY WARRANTY; without even the implied warranty of
	17	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	18	* General Public License for more details.
	19	*
	20	* Contact Information:
	21	* Megha Dey <megha.dey@linux.intel.com>
	22	*
	23	* BSD LICENSE
	24	*
	25	* Copyright(c) 2016 Intel Corporation.
	26	*
	27	* Redistribution and use in source and binary forms, with or without
	28	* modification, are permitted provided that the following conditions
	29	* are met:
	30	*
	31	* * Redistributions of source code must retain the above copyright
	32	* notice, this list of conditions and the following disclaimer.
	33	* * Redistributions in binary form must reproduce the above copyright
	34	* notice, this list of conditions and the following disclaimer in
	35	* the documentation and/or other materials provided with the
	36	* distribution.
	37	* * Neither the name of Intel Corporation nor the names of its
	38	* contributors may be used to endorse or promote products derived
	39	* from this software without specific prior written permission.
	40	*
	41	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	42	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	43	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	44	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	45	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	46	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	47	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	48	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	49	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	50	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	51	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	52	*/
	53
	54	#ifndef __SHA_MB_MGR_H
	55	#define __SHA_MB_MGR_H
	56
	57	#include <linux/types.h>
	58
	59	#define NUM_SHA512_DIGEST_WORDS 8
	60
	61	enum job_sts {STS_UNKNOWN = 0,
	62	STS_BEING_PROCESSED = 1,
	63	STS_COMPLETED = 2,
	64	STS_INTERNAL_ERROR = 3,
	65	STS_ERROR = 4
	66	};
	67
	68	struct job_sha512 {
	69	u8 *buffer;
	70	u64 len;
	71	u64 result_digest[NUM_SHA512_DIGEST_WORDS] __aligned(32);
	72	enum job_sts status;
	73	void *user_data;
	74	};
	75
	76	struct sha512_args_x4 {
	77	uint64_t digest[8][4];
	78	uint8_t *data_ptr[4];
	79	};
	80
	81	struct sha512_lane_data {
	82	struct job_sha512 *job_in_lane;
	83	};
	84
	85	struct sha512_mb_mgr {
	86	struct sha512_args_x4 args;
	87
	88	uint64_t lens[4];
	89
	90	/* each byte is index (0...7) of unused lanes */
	91	uint64_t unused_lanes;
	92	/* byte 4 is set to FF as a flag */
	93	struct sha512_lane_data ldata[4];
	94	};
	95
	96	#define SHA512_MB_MGR_NUM_LANES_AVX2 4
	97
	98	void sha512_mb_mgr_init_avx2(struct sha512_mb_mgr *state);
	99	struct job_sha512 sha512_mb_mgr_submit_avx2(struct sha512_mb_mgr state,
	100	struct job_sha512 *job);
	101	struct job_sha512 sha512_mb_mgr_flush_avx2(struct sha512_mb_mgr state);
	102	struct job_sha512 sha512_mb_mgr_get_comp_job_avx2(struct sha512_mb_mgr state);
	103
	104	#endif

arch/x86/crypto/sha512-mb/sha512_mb_mgr_datastruct.S+281−0

@@ -0,0 +1,281 @@
	1	/*
	2	* Header file for multi buffer SHA256 algorithm data structure
	3	*
	4	* This file is provided under a dual BSD/GPLv2 license. When using or
	5	* redistributing this file, you may do so under either license.
	6	*
	7	* GPL LICENSE SUMMARY
	8	*
	9	* Copyright(c) 2016 Intel Corporation.
	10	*
	11	* This program is free software; you can redistribute it and/or modify
	12	* it under the terms of version 2 of the GNU General Public License as
	13	* published by the Free Software Foundation.
	14	*
	15	* This program is distributed in the hope that it will be useful, but
	16	* WITHOUT ANY WARRANTY; without even the implied warranty of
	17	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	18	* General Public License for more details.
	19	*
	20	* Contact Information:
	21	* Megha Dey <megha.dey@linux.intel.com>
	22	*
	23	* BSD LICENSE
	24	*
	25	* Copyright(c) 2016 Intel Corporation.
	26	*
	27	* Redistribution and use in source and binary forms, with or without
	28	* modification, are permitted provided that the following conditions
	29	* are met:
	30	*
	31	* * Redistributions of source code must retain the above copyright
	32	* notice, this list of conditions and the following disclaimer.
	33	* * Redistributions in binary form must reproduce the above copyright
	34	* notice, this list of conditions and the following disclaimer in
	35	* the documentation and/or other materials provided with the
	36	* distribution.
	37	* * Neither the name of Intel Corporation nor the names of its
	38	* contributors may be used to endorse or promote products derived
	39	* from this software without specific prior written permission.
	40	*
	41	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	42	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	43	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	44	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	45	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	46	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	47	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	48	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	49	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	50	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	51	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	52	*/
	53
	54	# Macros for defining data structures
	55
	56	# Usage example
	57
	58	#START_FIELDS # JOB_AES
	59	### name size align
	60	#FIELD _plaintext, 8, 8 # pointer to plaintext
	61	#FIELD _ciphertext, 8, 8 # pointer to ciphertext
	62	#FIELD _IV, 16, 8 # IV
	63	#FIELD _keys, 8, 8 # pointer to keys
	64	#FIELD _len, 4, 4 # length in bytes
	65	#FIELD _status, 4, 4 # status enumeration
	66	#FIELD _user_data, 8, 8 # pointer to user data
	67	#UNION _union, size1, align1, \
	68	# size2, align2, \
	69	# size3, align3, \
	70	# ...
	71	#END_FIELDS
	72	#%assign _JOB_AES_size _FIELD_OFFSET
	73	#%assign _JOB_AES_align _STRUCT_ALIGN
	74
	75	#########################################################################
	76
	77	# Alternate "struc-like" syntax:
	78	# STRUCT job_aes2
	79	# RES_Q .plaintext, 1
	80	# RES_Q .ciphertext, 1
	81	# RES_DQ .IV, 1
	82	# RES_B .nested, _JOB_AES_SIZE, _JOB_AES_ALIGN
	83	# RES_U .union, size1, align1, \
	84	# size2, align2, \
	85	# ...
	86	# ENDSTRUCT
	87	# # Following only needed if nesting
	88	# %assign job_aes2_size _FIELD_OFFSET
	89	# %assign job_aes2_align _STRUCT_ALIGN
	90	#
	91	# RES_* macros take a name, a count and an optional alignment.
	92	# The count in in terms of the base size of the macro, and the
	93	# default alignment is the base size.
	94	# The macros are:
	95	# Macro Base size
	96	# RES_B 1
	97	# RES_W 2
	98	# RES_D 4
	99	# RES_Q 8
	100	# RES_DQ 16
	101	# RES_Y 32
	102	# RES_Z 64
	103	#
	104	# RES_U defines a union. It's arguments are a name and two or more
	105	# pairs of "size, alignment"
	106	#
	107	# The two assigns are only needed if this structure is being nested
	108	# within another. Even if the assigns are not done, one can still use
	109	# STRUCT_NAME_size as the size of the structure.
	110	#
	111	# Note that for nesting, you still need to assign to STRUCT_NAME_size.
	112	#
	113	# The differences between this and using "struc" directly are that each
	114	# type is implicitly aligned to its natural length (although this can be
	115	# over-ridden with an explicit third parameter), and that the structure
	116	# is padded at the end to its overall alignment.
	117	#
	118
	119	#########################################################################
	120
	121	#ifndef _DATASTRUCT_ASM_
	122	#define _DATASTRUCT_ASM_
	123
	124	#define PTR_SZ 8
	125	#define SHA512_DIGEST_WORD_SIZE 8
	126	#define SHA512_MB_MGR_NUM_LANES_AVX2 4
	127	#define NUM_SHA512_DIGEST_WORDS 8
	128	#define SZ4 4*SHA512_DIGEST_WORD_SIZE
	129	#define ROUNDS 80*SZ4
	130	#define SHA512_DIGEST_ROW_SIZE (SHA512_MB_MGR_NUM_LANES_AVX2 * 8)
	131
	132	# START_FIELDS
	133	.macro START_FIELDS
	134	_FIELD_OFFSET = 0
	135	_STRUCT_ALIGN = 0
	136	.endm
	137
	138	# FIELD name size align
	139	.macro FIELD name size align
	140	_FIELD_OFFSET = (_FIELD_OFFSET + (\align) - 1) & (~ ((\align)-1))
	141	\name = _FIELD_OFFSET
	142	_FIELD_OFFSET = _FIELD_OFFSET + (\size)
	143	.if (\align > _STRUCT_ALIGN)
	144	_STRUCT_ALIGN = \align
	145	.endif
	146	.endm
	147
	148	# END_FIELDS
	149	.macro END_FIELDS
	150	_FIELD_OFFSET = (_FIELD_OFFSET + _STRUCT_ALIGN-1) & (~ (_STRUCT_ALIGN-1))
	151	.endm
	152
	153	.macro STRUCT p1
	154	START_FIELDS
	155	.struc \p1
	156	.endm
	157
	158	.macro ENDSTRUCT
	159	tmp = _FIELD_OFFSET
	160	END_FIELDS
	161	tmp = (_FIELD_OFFSET - ##tmp)
	162	.if (tmp > 0)
	163	.lcomm tmp
	164	.endm
	165
	166	## RES_int name size align
	167	.macro RES_int p1 p2 p3
	168	name = \p1
	169	size = \p2
	170	align = .\p3
	171
	172	_FIELD_OFFSET = (_FIELD_OFFSET + (align) - 1) & (~ ((align)-1))
	173	.align align
	174	.lcomm name size
	175	_FIELD_OFFSET = _FIELD_OFFSET + (size)
	176	.if (align > _STRUCT_ALIGN)
	177	_STRUCT_ALIGN = align
	178	.endif
	179	.endm
	180
	181	# macro RES_B name, size [, align]
	182	.macro RES_B _name, _size, _align=1
	183	RES_int _name _size _align
	184	.endm
	185
	186	# macro RES_W name, size [, align]
	187	.macro RES_W _name, _size, _align=2
	188	RES_int _name 2*(_size) _align
	189	.endm
	190
	191	# macro RES_D name, size [, align]
	192	.macro RES_D _name, _size, _align=4
	193	RES_int _name 4*(_size) _align
	194	.endm
	195
	196	# macro RES_Q name, size [, align]
	197	.macro RES_Q _name, _size, _align=8
	198	RES_int _name 8*(_size) _align
	199	.endm
	200
	201	# macro RES_DQ name, size [, align]
	202	.macro RES_DQ _name, _size, _align=16
	203	RES_int _name 16*(_size) _align
	204	.endm
	205
	206	# macro RES_Y name, size [, align]
	207	.macro RES_Y _name, _size, _align=32
	208	RES_int _name 32*(_size) _align
	209	.endm
	210
	211	# macro RES_Z name, size [, align]
	212	.macro RES_Z _name, _size, _align=64
	213	RES_int _name 64*(_size) _align
	214	.endm
	215
	216	#endif
	217
	218	###################################################################
	219	### Define SHA512 Out Of Order Data Structures
	220	###################################################################
	221
	222	START_FIELDS # LANE_DATA
	223	### name size align
	224	FIELD _job_in_lane, 8, 8 # pointer to job object
	225	END_FIELDS
	226
	227	_LANE_DATA_size = _FIELD_OFFSET
	228	_LANE_DATA_align = _STRUCT_ALIGN
	229
	230	####################################################################
	231
	232	START_FIELDS # SHA512_ARGS_X4
	233	### name size align
	234	FIELD _digest, 884, 4 # transposed digest
	235	FIELD _data_ptr, 8*4, 8 # array of pointers to data
	236	END_FIELDS
	237
	238	_SHA512_ARGS_X4_size = _FIELD_OFFSET
	239	_SHA512_ARGS_X4_align = _STRUCT_ALIGN
	240
	241	#####################################################################
	242
	243	START_FIELDS # MB_MGR
	244	### name size align
	245	FIELD _args, _SHA512_ARGS_X4_size, _SHA512_ARGS_X4_align
	246	FIELD _lens, 8*4, 8
	247	FIELD _unused_lanes, 8, 8
	248	FIELD _ldata, _LANE_DATA_size*4, _LANE_DATA_align
	249	END_FIELDS
	250
	251	_MB_MGR_size = _FIELD_OFFSET
	252	_MB_MGR_align = _STRUCT_ALIGN
	253
	254	_args_digest = _args + _digest
	255	_args_data_ptr = _args + _data_ptr
	256
	257	#######################################################################
	258
	259	#######################################################################
	260	#### Define constants
	261	#######################################################################
	262
	263	#define STS_UNKNOWN 0
	264	#define STS_BEING_PROCESSED 1
	265	#define STS_COMPLETED 2
	266
	267	#######################################################################
	268	#### Define JOB_SHA512 structure
	269	#######################################################################
	270
	271	START_FIELDS # JOB_SHA512
	272	### name size align
	273	FIELD _buffer, 8, 8 # pointer to buffer
	274	FIELD _len, 8, 8 # length in bytes
	275	FIELD _result_digest, 8*8, 32 # Digest (output)
	276	FIELD _status, 4, 4
	277	FIELD _user_data, 8, 8
	278	END_FIELDS
	279
	280	_JOB_SHA512_size = _FIELD_OFFSET
	281	_JOB_SHA512_align = _STRUCT_ALIGN

arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S+291−0

@@ -0,0 +1,291 @@
	1	/*
	2	* Flush routine for SHA512 multibuffer
	3	*
	4	* This file is provided under a dual BSD/GPLv2 license. When using or
	5	* redistributing this file, you may do so under either license.
	6	*
	7	* GPL LICENSE SUMMARY
	8	*
	9	* Copyright(c) 2016 Intel Corporation.
	10	*
	11	* This program is free software; you can redistribute it and/or modify
	12	* it under the terms of version 2 of the GNU General Public License as
	13	* published by the Free Software Foundation.
	14	*
	15	* This program is distributed in the hope that it will be useful, but
	16	* WITHOUT ANY WARRANTY; without even the implied warranty of
	17	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	18	* General Public License for more details.
	19	*
	20	* Contact Information:
	21	* Megha Dey <megha.dey@linux.intel.com>
	22	*
	23	* BSD LICENSE
	24	*
	25	* Copyright(c) 2016 Intel Corporation.
	26	*
	27	* Redistribution and use in source and binary forms, with or without
	28	* modification, are permitted provided that the following conditions
	29	* are met:
	30	*
	31	* * Redistributions of source code must retain the above copyright
	32	* notice, this list of conditions and the following disclaimer.
	33	* * Redistributions in binary form must reproduce the above copyright
	34	* notice, this list of conditions and the following disclaimer in
	35	* the documentation and/or other materials provided with the
	36	* distribution.
	37	* * Neither the name of Intel Corporation nor the names of its
	38	* contributors may be used to endorse or promote products derived
	39	* from this software without specific prior written permission.
	40	*
	41	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	42	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	43	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	44	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	45	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	46	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	47	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	48	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	49	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	50	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	51	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	52	*/
	53
	54	#include <linux/linkage.h>
	55	#include <asm/frame.h>
	56	#include "sha512_mb_mgr_datastruct.S"
	57
	58	.extern sha512_x4_avx2
	59
	60	# LINUX register definitions
	61	#define arg1 %rdi
	62	#define arg2 %rsi
	63
	64	# idx needs to be other than arg1, arg2, rbx, r12
	65	#define idx %rdx
	66
	67	# Common definitions
	68	#define state arg1
	69	#define job arg2
	70	#define len2 arg2
	71
	72	#define unused_lanes %rbx
	73	#define lane_data %rbx
	74	#define tmp2 %rbx
	75
	76	#define job_rax %rax
	77	#define tmp1 %rax
	78	#define size_offset %rax
	79	#define tmp %rax
	80	#define start_offset %rax
	81
	82	#define tmp3 arg1
	83
	84	#define extra_blocks arg2
	85	#define p arg2
	86
	87	#define tmp4 %r8
	88	#define lens0 %r8
	89
	90	#define lens1 %r9
	91	#define lens2 %r10
	92	#define lens3 %r11
	93
	94	.macro LABEL prefix n
	95	\prefix\n\():
	96	.endm
	97
	98	.macro JNE_SKIP i
	99	jne skip_\i
	100	.endm
	101
	102	.altmacro
	103	.macro SET_OFFSET _offset
	104	offset = \_offset
	105	.endm
	106	.noaltmacro
	107
	108	# JOB* sha512_mb_mgr_flush_avx2(MB_MGR *state)
	109	# arg 1 : rcx : state
	110	ENTRY(sha512_mb_mgr_flush_avx2)
	111	FRAME_BEGIN
	112	push %rbx
	113
	114	# If bit (32+3) is set, then all lanes are empty
	115	mov _unused_lanes(state), unused_lanes
	116	bt $32+7, unused_lanes
	117	jc return_null
	118
	119	# find a lane with a non-null job
	120	xor idx, idx
	121	offset = (_ldata + 1*_LANE_DATA_size + _job_in_lane)
	122	cmpq $0, offset(state)
	123	cmovne one(%rip), idx
	124	offset = (_ldata + 2*_LANE_DATA_size + _job_in_lane)
	125	cmpq $0, offset(state)
	126	cmovne two(%rip), idx
	127	offset = (_ldata + 3*_LANE_DATA_size + _job_in_lane)
	128	cmpq $0, offset(state)
	129	cmovne three(%rip), idx
	130
	131	# copy idx to empty lanes
	132	copy_lane_data:
	133	offset = (_args + _data_ptr)
	134	mov offset(state,idx,8), tmp
	135
	136	I = 0
	137	.rep 4
	138	offset = (_ldata + I * _LANE_DATA_size + _job_in_lane)
	139	cmpq $0, offset(state)
	140	.altmacro
	141	JNE_SKIP %I
	142	offset = (_args + _data_ptr + 8*I)
	143	mov tmp, offset(state)
	144	offset = (_lens + 8*I +4)
	145	movl $0xFFFFFFFF, offset(state)
	146	LABEL skip_ %I
	147	I = (I+1)
	148	.noaltmacro
	149	.endr
	150
	151	# Find min length
	152	mov _lens + 0*8(state),lens0
	153	mov lens0,idx
	154	mov _lens + 1*8(state),lens1
	155	cmp idx,lens1
	156	cmovb lens1,idx
	157	mov _lens + 2*8(state),lens2
	158	cmp idx,lens2
	159	cmovb lens2,idx
	160	mov _lens + 3*8(state),lens3
	161	cmp idx,lens3
	162	cmovb lens3,idx
	163	mov idx,len2
	164	and $0xF,idx
	165	and $~0xFF,len2
	166	jz len_is_0
	167
	168	sub len2, lens0
	169	sub len2, lens1
	170	sub len2, lens2
	171	sub len2, lens3
	172	shr $32,len2
	173	mov lens0, _lens + 0*8(state)
	174	mov lens1, _lens + 1*8(state)
	175	mov lens2, _lens + 2*8(state)
	176	mov lens3, _lens + 3*8(state)
	177
	178	# "state" and "args" are the same address, arg1
	179	# len is arg2
	180	call sha512_x4_avx2
	181	# state and idx are intact
	182
	183	len_is_0:
	184	# process completed job "idx"
	185	imul $_LANE_DATA_size, idx, lane_data
	186	lea _ldata(state, lane_data), lane_data
	187
	188	mov _job_in_lane(lane_data), job_rax
	189	movq $0, _job_in_lane(lane_data)
	190	movl $STS_COMPLETED, _status(job_rax)
	191	mov _unused_lanes(state), unused_lanes
	192	shl $8, unused_lanes
	193	or idx, unused_lanes
	194	mov unused_lanes, _unused_lanes(state)
	195
	196	movl $0xFFFFFFFF, _lens+4(state, idx, 8)
	197
	198	vmovq _args_digest+0*32(state, idx, 8), %xmm0
	199	vpinsrq $1, _args_digest+1*32(state, idx, 8), %xmm0, %xmm0
	200	vmovq _args_digest+2*32(state, idx, 8), %xmm1
	201	vpinsrq $1, _args_digest+3*32(state, idx, 8), %xmm1, %xmm1
	202	vmovq _args_digest+4*32(state, idx, 8), %xmm2
	203	vpinsrq $1, _args_digest+5*32(state, idx, 8), %xmm2, %xmm2
	204	vmovq _args_digest+6*32(state, idx, 8), %xmm3
	205	vpinsrq $1, _args_digest+7*32(state, idx, 8), %xmm3, %xmm3
	206
	207	vmovdqu %xmm0, _result_digest(job_rax)
	208	vmovdqu %xmm1, _result_digest+1*16(job_rax)
	209	vmovdqu %xmm2, _result_digest+2*16(job_rax)
	210	vmovdqu %xmm3, _result_digest+3*16(job_rax)
	211
	212	return:
	213	pop %rbx
	214	FRAME_END
	215	ret
	216
	217	return_null:
	218	xor job_rax, job_rax
	219	jmp return
	220	ENDPROC(sha512_mb_mgr_flush_avx2)
	221	.align 16
	222
	223	ENTRY(sha512_mb_mgr_get_comp_job_avx2)
	224	push %rbx
	225
	226	mov _unused_lanes(state), unused_lanes
	227	bt $(32+7), unused_lanes
	228	jc .return_null
	229
	230	# Find min length
	231	mov _lens(state),lens0
	232	mov lens0,idx
	233	mov _lens+1*8(state),lens1
	234	cmp idx,lens1
	235	cmovb lens1,idx
	236	mov _lens+2*8(state),lens2
	237	cmp idx,lens2
	238	cmovb lens2,idx
	239	mov _lens+3*8(state),lens3
	240	cmp idx,lens3
	241	cmovb lens3,idx
	242	test $~0xF,idx
	243	jnz .return_null
	244	and $0xF,idx
	245
	246	#process completed job "idx"
	247	imul $_LANE_DATA_size, idx, lane_data
	248	lea _ldata(state, lane_data), lane_data
	249
	250	mov _job_in_lane(lane_data), job_rax
	251	movq $0, _job_in_lane(lane_data)
	252	movl $STS_COMPLETED, _status(job_rax)
	253	mov _unused_lanes(state), unused_lanes
	254	shl $8, unused_lanes
	255	or idx, unused_lanes
	256	mov unused_lanes, _unused_lanes(state)
	257
	258	movl $0xFFFFFFFF, _lens+4(state, idx, 8)
	259
	260	vmovq _args_digest(state, idx, 8), %xmm0
	261	vpinsrq $1, _args_digest+1*32(state, idx, 8), %xmm0, %xmm0
	262	vmovq _args_digest+2*32(state, idx, 8), %xmm1
	263	vpinsrq $1, _args_digest+3*32(state, idx, 8), %xmm1, %xmm1
	264	vmovq _args_digest+4*32(state, idx, 8), %xmm2
	265	vpinsrq $1, _args_digest+5*32(state, idx, 8), %xmm2, %xmm2
	266	vmovq _args_digest+6*32(state, idx, 8), %xmm3
	267	vpinsrq $1, _args_digest+7*32(state, idx, 8), %xmm3, %xmm3
	268
	269	vmovdqu %xmm0, _result_digest+0*16(job_rax)
	270	vmovdqu %xmm1, _result_digest+1*16(job_rax)
	271	vmovdqu %xmm2, _result_digest+2*16(job_rax)
	272	vmovdqu %xmm3, _result_digest+3*16(job_rax)
	273
	274	pop %rbx
	275
	276	ret
	277
	278	.return_null:
	279	xor job_rax, job_rax
	280	pop %rbx
	281	ret
	282	ENDPROC(sha512_mb_mgr_get_comp_job_avx2)
	283	.data
	284
	285	.align 16
	286	one:
	287	.quad 1
	288	two:
	289	.quad 2
	290	three:
	291	.quad 3

arch/x86/crypto/sha512-mb/sha512_mb_mgr_init_avx2.c+67−0

@@ -0,0 +1,67 @@
	1	/*
	2	* Initialization code for multi buffer SHA256 algorithm for AVX2
	3	*
	4	* This file is provided under a dual BSD/GPLv2 license. When using or
	5	* redistributing this file, you may do so under either license.
	6	*
	7	* GPL LICENSE SUMMARY
	8	*
	9	* Copyright(c) 2016 Intel Corporation.
	10	*
	11	* This program is free software; you can redistribute it and/or modify
	12	* it under the terms of version 2 of the GNU General Public License as
	13	* published by the Free Software Foundation.
	14	*
	15	* This program is distributed in the hope that it will be useful, but
	16	* WITHOUT ANY WARRANTY; without even the implied warranty of
	17	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	18	* General Public License for more details.
	19	*
	20	* Contact Information:
	21	* Megha Dey <megha.dey@linux.intel.com>
	22	*
	23	* BSD LICENSE
	24	*
	25	* Copyright(c) 2016 Intel Corporation.
	26	*
	27	* Redistribution and use in source and binary forms, with or without
	28	* modification, are permitted provided that the following conditions
	29	* are met:
	30	*
	31	* * Redistributions of source code must retain the above copyright
	32	* notice, this list of conditions and the following disclaimer.
	33	* * Redistributions in binary form must reproduce the above copyright
	34	* notice, this list of conditions and the following disclaimer in
	35	* the documentation and/or other materials provided with the
	36	* distribution.
	37	* * Neither the name of Intel Corporation nor the names of its
	38	* contributors may be used to endorse or promote products derived
	39	* from this software without specific prior written permission.
	40	*
	41	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	42	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	43	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	44	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	45	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	46	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	47	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	48	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	49	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	50	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	51	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	52	*/
	53
	54	#include "sha512_mb_mgr.h"
	55
	56	void sha512_mb_mgr_init_avx2(struct sha512_mb_mgr *state)
	57	{
	58	unsigned int j;
	59
	60	state->lens[0] = 0;
	61	state->lens[1] = 1;
	62	state->lens[2] = 2;
	63	state->lens[3] = 3;
	64	state->unused_lanes = 0xFF03020100;
	65	for (j = 0; j < 4; j++)
	66	state->ldata[j].job_in_lane = NULL;
	67	}

arch/x86/crypto/sha512-mb/sha512_mb_mgr_submit_avx2.S+222−0

@@ -0,0 +1,222 @@
	1	/*
	2	* Buffer submit code for multi buffer SHA512 algorithm
	3	*
	4	* This file is provided under a dual BSD/GPLv2 license. When using or
	5	* redistributing this file, you may do so under either license.
	6	*
	7	* GPL LICENSE SUMMARY
	8	*
	9	* Copyright(c) 2016 Intel Corporation.
	10	*
	11	* This program is free software; you can redistribute it and/or modify
	12	* it under the terms of version 2 of the GNU General Public License as
	13	* published by the Free Software Foundation.
	14	*
	15	* This program is distributed in the hope that it will be useful, but
	16	* WITHOUT ANY WARRANTY; without even the implied warranty of
	17	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	18	* General Public License for more details.
	19	*
	20	* Contact Information:
	21	* Megha Dey <megha.dey@linux.intel.com>
	22	*
	23	* BSD LICENSE
	24	*
	25	* Copyright(c) 2016 Intel Corporation.
	26	*
	27	* Redistribution and use in source and binary forms, with or without
	28	* modification, are permitted provided that the following conditions
	29	* are met:
	30	*
	31	* * Redistributions of source code must retain the above copyright
	32	* notice, this list of conditions and the following disclaimer.
	33	* * Redistributions in binary form must reproduce the above copyright
	34	* notice, this list of conditions and the following disclaimer in
	35	* the documentation and/or other materials provided with the
	36	* distribution.
	37	* * Neither the name of Intel Corporation nor the names of its
	38	* contributors may be used to endorse or promote products derived
	39	* from this software without specific prior written permission.
	40	*
	41	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	42	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	43	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	44	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	45	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	46	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	47	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	48	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	49	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	50	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	51	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	52	*/
	53
	54	#include <linux/linkage.h>
	55	#include <asm/frame.h>
	56	#include "sha512_mb_mgr_datastruct.S"
	57
	58	.extern sha512_x4_avx2
	59
	60	#define arg1 %rdi
	61	#define arg2 %rsi
	62
	63	#define idx %rdx
	64	#define last_len %rdx
	65
	66	#define size_offset %rcx
	67	#define tmp2 %rcx
	68
	69	# Common definitions
	70	#define state arg1
	71	#define job arg2
	72	#define len2 arg2
	73	#define p2 arg2
	74
	75	#define p %r11
	76	#define start_offset %r11
	77
	78	#define unused_lanes %rbx
	79
	80	#define job_rax %rax
	81	#define len %rax
	82
	83	#define lane %r12
	84	#define tmp3 %r12
	85	#define lens3 %r12
	86
	87	#define extra_blocks %r8
	88	#define lens0 %r8
	89
	90	#define tmp %r9
	91	#define lens1 %r9
	92
	93	#define lane_data %r10
	94	#define lens2 %r10
	95
	96	#define DWORD_len %eax
	97
	98	# JOB* sha512_mb_mgr_submit_avx2(MB_MGR state, JOB job)
	99	# arg 1 : rcx : state
	100	# arg 2 : rdx : job
	101	ENTRY(sha512_mb_mgr_submit_avx2)
	102	FRAME_BEGIN
	103	push %rbx
	104	push %r12
	105
	106	mov _unused_lanes(state), unused_lanes
	107	movzb %bl,lane
	108	shr $8, unused_lanes
	109	imul $_LANE_DATA_size, lane,lane_data
	110	movl $STS_BEING_PROCESSED, _status(job)
	111	lea _ldata(state, lane_data), lane_data
	112	mov unused_lanes, _unused_lanes(state)
	113	movl _len(job), DWORD_len
	114
	115	mov job, _job_in_lane(lane_data)
	116	movl DWORD_len,_lens+4(state , lane, 8)
	117
	118	# Load digest words from result_digest
	119	vmovdqu _result_digest+0*16(job), %xmm0
	120	vmovdqu _result_digest+1*16(job), %xmm1
	121	vmovdqu _result_digest+2*16(job), %xmm2
	122	vmovdqu _result_digest+3*16(job), %xmm3
	123
	124	vmovq %xmm0, _args_digest(state, lane, 8)
	125	vpextrq $1, %xmm0, _args_digest+1*32(state , lane, 8)
	126	vmovq %xmm1, _args_digest+2*32(state , lane, 8)
	127	vpextrq $1, %xmm1, _args_digest+3*32(state , lane, 8)
	128	vmovq %xmm2, _args_digest+4*32(state , lane, 8)
	129	vpextrq $1, %xmm2, _args_digest+5*32(state , lane, 8)
	130	vmovq %xmm3, _args_digest+6*32(state , lane, 8)
	131	vpextrq $1, %xmm3, _args_digest+7*32(state , lane, 8)
	132
	133	mov _buffer(job), p
	134	mov p, _args_data_ptr(state, lane, 8)
	135
	136	cmp $0xFF, unused_lanes
	137	jne return_null
	138
	139	start_loop:
	140
	141	# Find min length
	142	mov _lens+0*8(state),lens0
	143	mov lens0,idx
	144	mov _lens+1*8(state),lens1
	145	cmp idx,lens1
	146	cmovb lens1, idx
	147	mov _lens+2*8(state),lens2
	148	cmp idx,lens2
	149	cmovb lens2,idx
	150	mov _lens+3*8(state),lens3
	151	cmp idx,lens3
	152	cmovb lens3,idx
	153	mov idx,len2
	154	and $0xF,idx
	155	and $~0xFF,len2
	156	jz len_is_0
	157
	158	sub len2,lens0
	159	sub len2,lens1
	160	sub len2,lens2
	161	sub len2,lens3
	162	shr $32,len2
	163	mov lens0, _lens + 0*8(state)
	164	mov lens1, _lens + 1*8(state)
	165	mov lens2, _lens + 2*8(state)
	166	mov lens3, _lens + 3*8(state)
	167
	168	# "state" and "args" are the same address, arg1
	169	# len is arg2
	170	call sha512_x4_avx2
	171	# state and idx are intact
	172
	173	len_is_0:
	174
	175	# process completed job "idx"
	176	imul $_LANE_DATA_size, idx, lane_data
	177	lea _ldata(state, lane_data), lane_data
	178
	179	mov _job_in_lane(lane_data), job_rax
	180	mov _unused_lanes(state), unused_lanes
	181	movq $0, _job_in_lane(lane_data)
	182	movl $STS_COMPLETED, _status(job_rax)
	183	shl $8, unused_lanes
	184	or idx, unused_lanes
	185	mov unused_lanes, _unused_lanes(state)
	186
	187	movl $0xFFFFFFFF,_lens+4(state,idx,8)
	188	vmovq _args_digest+0*32(state , idx, 8), %xmm0
	189	vpinsrq $1, _args_digest+1*32(state , idx, 8), %xmm0, %xmm0
	190	vmovq _args_digest+2*32(state , idx, 8), %xmm1
	191	vpinsrq $1, _args_digest+3*32(state , idx, 8), %xmm1, %xmm1
	192	vmovq _args_digest+4*32(state , idx, 8), %xmm2
	193	vpinsrq $1, _args_digest+5*32(state , idx, 8), %xmm2, %xmm2
	194	vmovq _args_digest+6*32(state , idx, 8), %xmm3
	195	vpinsrq $1, _args_digest+7*32(state , idx, 8), %xmm3, %xmm3
	196
	197	vmovdqu %xmm0, _result_digest + 0*16(job_rax)
	198	vmovdqu %xmm1, _result_digest + 1*16(job_rax)
	199	vmovdqu %xmm2, _result_digest + 2*16(job_rax)
	200	vmovdqu %xmm3, _result_digest + 3*16(job_rax)
	201
	202	return:
	203	pop %r12
	204	pop %rbx
	205	FRAME_END
	206	ret
	207
	208	return_null:
	209	xor job_rax, job_rax
	210	jmp return
	211	ENDPROC(sha512_mb_mgr_submit_avx2)
	212	.data
	213
	214	.align 16
	215	H0: .int 0x6a09e667
	216	H1: .int 0xbb67ae85
	217	H2: .int 0x3c6ef372
	218	H3: .int 0xa54ff53a
	219	H4: .int 0x510e527f
	220	H5: .int 0x9b05688c
	221	H6: .int 0x1f83d9ab
	222	H7: .int 0x5be0cd19

arch/x86/crypto/sha512-mb/sha512_x4_avx2.S+529−0

@@ -0,0 +1,529 @@
	1	/*
	2	* Multi-buffer SHA512 algorithm hash compute routine
	3	*
	4	* This file is provided under a dual BSD/GPLv2 license. When using or
	5	* redistributing this file, you may do so under either license.
	6	*
	7	* GPL LICENSE SUMMARY
	8	*
	9	* Copyright(c) 2016 Intel Corporation.
	10	*
	11	* This program is free software; you can redistribute it and/or modify
	12	* it under the terms of version 2 of the GNU General Public License as
	13	* published by the Free Software Foundation.
	14	*
	15	* This program is distributed in the hope that it will be useful, but
	16	* WITHOUT ANY WARRANTY; without even the implied warranty of
	17	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	18	* General Public License for more details.
	19	*
	20	* Contact Information:
	21	* Megha Dey <megha.dey@linux.intel.com>
	22	*
	23	* BSD LICENSE
	24	*
	25	* Copyright(c) 2016 Intel Corporation.
	26	*
	27	* Redistribution and use in source and binary forms, with or without
	28	* modification, are permitted provided that the following conditions
	29	* are met:
	30	*
	31	* * Redistributions of source code must retain the above copyright
	32	* notice, this list of conditions and the following disclaimer.
	33	* * Redistributions in binary form must reproduce the above copyright
	34	* notice, this list of conditions and the following disclaimer in
	35	* the documentation and/or other materials provided with the
	36	* distribution.
	37	* * Neither the name of Intel Corporation nor the names of its
	38	* contributors may be used to endorse or promote products derived
	39	* from this software without specific prior written permission.
	40	*
	41	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	42	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	43	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	44	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	45	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	46	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	47	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	48	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	49	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	50	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	51	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	52	*/
	53
	54	# code to compute quad SHA512 using AVX2
	55	# use YMMs to tackle the larger digest size
	56	# outer calling routine takes care of save and restore of XMM registers
	57	# Logic designed/laid out by JDG
	58
	59	# Function clobbers: rax, rcx, rdx, rbx, rsi, rdi, r9-r15; ymm0-15
	60	# Stack must be aligned to 32 bytes before call
	61	# Linux clobbers: rax rbx rcx rsi r8 r9 r10 r11 r12
	62	# Linux preserves: rcx rdx rdi rbp r13 r14 r15
	63	# clobbers ymm0-15
	64
	65	#include <linux/linkage.h>
	66	#include "sha512_mb_mgr_datastruct.S"
	67
	68	arg1 = %rdi
	69	arg2 = %rsi
	70
	71	# Common definitions
	72	STATE = arg1
	73	INP_SIZE = arg2
	74
	75	IDX = %rax
	76	ROUND = %rbx
	77	TBL = %r8
	78
	79	inp0 = %r9
	80	inp1 = %r10
	81	inp2 = %r11
	82	inp3 = %r12
	83
	84	a = %ymm0
	85	b = %ymm1
	86	c = %ymm2
	87	d = %ymm3
	88	e = %ymm4
	89	f = %ymm5
	90	g = %ymm6
	91	h = %ymm7
	92
	93	a0 = %ymm8
	94	a1 = %ymm9
	95	a2 = %ymm10
	96
	97	TT0 = %ymm14
	98	TT1 = %ymm13
	99	TT2 = %ymm12
	100	TT3 = %ymm11
	101	TT4 = %ymm10
	102	TT5 = %ymm9
	103
	104	T1 = %ymm14
	105	TMP = %ymm15
	106
	107	# Define stack usage
	108	STACK_SPACE1 = SZ416 + NUM_SHA512_DIGEST_WORDSSZ4 + 24
	109
	110	#define VMOVPD vmovupd
	111	_digest = SZ4*16
	112
	113	# transpose r0, r1, r2, r3, t0, t1
	114	# "transpose" data in {r0..r3} using temps {t0..t3}
	115	# Input looks like: {r0 r1 r2 r3}
	116	# r0 = {a7 a6 a5 a4 a3 a2 a1 a0}
	117	# r1 = {b7 b6 b5 b4 b3 b2 b1 b0}
	118	# r2 = {c7 c6 c5 c4 c3 c2 c1 c0}
	119	# r3 = {d7 d6 d5 d4 d3 d2 d1 d0}
	120	#
	121	# output looks like: {t0 r1 r0 r3}
	122	# t0 = {d1 d0 c1 c0 b1 b0 a1 a0}
	123	# r1 = {d3 d2 c3 c2 b3 b2 a3 a2}
	124	# r0 = {d5 d4 c5 c4 b5 b4 a5 a4}
	125	# r3 = {d7 d6 c7 c6 b7 b6 a7 a6}
	126
	127	.macro TRANSPOSE r0 r1 r2 r3 t0 t1
	128	vshufps $0x44, \r1, \r0, \t0 # t0 = {b5 b4 a5 a4 b1 b0 a1 a0}
	129	vshufps $0xEE, \r1, \r0, \r0 # r0 = {b7 b6 a7 a6 b3 b2 a3 a2}
	130	vshufps $0x44, \r3, \r2, \t1 # t1 = {d5 d4 c5 c4 d1 d0 c1 c0}
	131	vshufps $0xEE, \r3, \r2, \r2 # r2 = {d7 d6 c7 c6 d3 d2 c3 c2}
	132
	133	vperm2f128 $0x20, \r2, \r0, \r1 # h6...a6
	134	vperm2f128 $0x31, \r2, \r0, \r3 # h2...a2
	135	vperm2f128 $0x31, \t1, \t0, \r0 # h5...a5
	136	vperm2f128 $0x20, \t1, \t0, \t0 # h1...a1
	137	.endm
	138
	139	.macro ROTATE_ARGS
	140	TMP_ = h
	141	h = g
	142	g = f
	143	f = e
	144	e = d
	145	d = c
	146	c = b
	147	b = a
	148	a = TMP_
	149	.endm
	150
	151	# PRORQ reg, imm, tmp
	152	# packed-rotate-right-double
	153	# does a rotate by doing two shifts and an or
	154	.macro _PRORQ reg imm tmp
	155	vpsllq $(64-\imm),\reg,\tmp
	156	vpsrlq $\imm,\reg, \reg
	157	vpor \tmp,\reg, \reg
	158	.endm
	159
	160	# non-destructive
	161	# PRORQ_nd reg, imm, tmp, src
	162	.macro _PRORQ_nd reg imm tmp src
	163	vpsllq $(64-\imm), \src, \tmp
	164	vpsrlq $\imm, \src, \reg
	165	vpor \tmp, \reg, \reg
	166	.endm
	167
	168	# PRORQ dst/src, amt
	169	.macro PRORQ reg imm
	170	_PRORQ \reg, \imm, TMP
	171	.endm
	172
	173	# PRORQ_nd dst, src, amt
	174	.macro PRORQ_nd reg tmp imm
	175	_PRORQ_nd \reg, \imm, TMP, \tmp
	176	.endm
	177
	178	#; arguments passed implicitly in preprocessor symbols i, a...h
	179	.macro ROUND_00_15 _T1 i
	180	PRORQ_nd a0, e, (18-14) # sig1: a0 = (e >> 4)
	181
	182	vpxor g, f, a2 # ch: a2 = f^g
	183	vpand e,a2, a2 # ch: a2 = (f^g)&e
	184	vpxor g, a2, a2 # a2 = ch
	185
	186	PRORQ_nd a1,e,41 # sig1: a1 = (e >> 25)
	187
	188	offset = SZ4*(\i & 0xf)
	189	vmovdqu \_T1,offset(%rsp)
	190	vpaddq (TBL,ROUND,1), \_T1, \_T1 # T1 = W + K
	191	vpxor e,a0, a0 # sig1: a0 = e ^ (e >> 5)
	192	PRORQ a0, 14 # sig1: a0 = (e >> 6) ^ (e >> 11)
	193	vpaddq a2, h, h # h = h + ch
	194	PRORQ_nd a2,a,6 # sig0: a2 = (a >> 11)
	195	vpaddq \_T1,h, h # h = h + ch + W + K
	196	vpxor a1, a0, a0 # a0 = sigma1
	197	vmovdqu a,\_T1
	198	PRORQ_nd a1,a,39 # sig0: a1 = (a >> 22)
	199	vpxor c, \_T1, \_T1 # maj: T1 = a^c
	200	add $SZ4, ROUND # ROUND++
	201	vpand b, \_T1, \_T1 # maj: T1 = (a^c)&b
	202	vpaddq a0, h, h
	203	vpaddq h, d, d
	204	vpxor a, a2, a2 # sig0: a2 = a ^ (a >> 11)
	205	PRORQ a2,28 # sig0: a2 = (a >> 2) ^ (a >> 13)
	206	vpxor a1, a2, a2 # a2 = sig0
	207	vpand c, a, a1 # maj: a1 = a&c
	208	vpor \_T1, a1, a1 # a1 = maj
	209	vpaddq a1, h, h # h = h + ch + W + K + maj
	210	vpaddq a2, h, h # h = h + ch + W + K + maj + sigma0
	211	ROTATE_ARGS
	212	.endm
	213
	214
	215	#; arguments passed implicitly in preprocessor symbols i, a...h
	216	.macro ROUND_16_XX _T1 i
	217	vmovdqu SZ4*((\i-15)&0xf)(%rsp), \_T1
	218	vmovdqu SZ4*((\i-2)&0xf)(%rsp), a1
	219	vmovdqu \_T1, a0
	220	PRORQ \_T1,7
	221	vmovdqu a1, a2
	222	PRORQ a1,42
	223	vpxor a0, \_T1, \_T1
	224	PRORQ \_T1, 1
	225	vpxor a2, a1, a1
	226	PRORQ a1, 19
	227	vpsrlq $7, a0, a0
	228	vpxor a0, \_T1, \_T1
	229	vpsrlq $6, a2, a2
	230	vpxor a2, a1, a1
	231	vpaddq SZ4*((\i-16)&0xf)(%rsp), \_T1, \_T1
	232	vpaddq SZ4*((\i-7)&0xf)(%rsp), a1, a1
	233	vpaddq a1, \_T1, \_T1
	234
	235	ROUND_00_15 \_T1,\i
	236	.endm
	237
	238
	239	# void sha512_x4_avx2(void *STATE, const int INP_SIZE)
	240	# arg 1 : STATE : pointer to input data
	241	# arg 2 : INP_SIZE : size of data in blocks (assumed >= 1)
	242	ENTRY(sha512_x4_avx2)
	243	# general registers preserved in outer calling routine
	244	# outer calling routine saves all the XMM registers
	245	# save callee-saved clobbered registers to comply with C function ABI
	246	push %r12
	247	push %r13
	248	push %r14
	249	push %r15
	250
	251	sub $STACK_SPACE1, %rsp
	252
	253	# Load the pre-transposed incoming digest.
	254	vmovdqu 0*SHA512_DIGEST_ROW_SIZE(STATE),a
	255	vmovdqu 1*SHA512_DIGEST_ROW_SIZE(STATE),b
	256	vmovdqu 2*SHA512_DIGEST_ROW_SIZE(STATE),c
	257	vmovdqu 3*SHA512_DIGEST_ROW_SIZE(STATE),d
	258	vmovdqu 4*SHA512_DIGEST_ROW_SIZE(STATE),e
	259	vmovdqu 5*SHA512_DIGEST_ROW_SIZE(STATE),f
	260	vmovdqu 6*SHA512_DIGEST_ROW_SIZE(STATE),g
	261	vmovdqu 7*SHA512_DIGEST_ROW_SIZE(STATE),h
	262
	263	lea K512_4(%rip),TBL
	264
	265	# load the address of each of the 4 message lanes
	266	# getting ready to transpose input onto stack
	267	mov _data_ptr+0*PTR_SZ(STATE),inp0
	268	mov _data_ptr+1*PTR_SZ(STATE),inp1
	269	mov _data_ptr+2*PTR_SZ(STATE),inp2
	270	mov _data_ptr+3*PTR_SZ(STATE),inp3
	271
	272	xor IDX, IDX
	273	lloop:
	274	xor ROUND, ROUND
	275
	276	# save old digest
	277	vmovdqu a, _digest(%rsp)
	278	vmovdqu b, _digest+1*SZ4(%rsp)
	279	vmovdqu c, _digest+2*SZ4(%rsp)
	280	vmovdqu d, _digest+3*SZ4(%rsp)
	281	vmovdqu e, _digest+4*SZ4(%rsp)
	282	vmovdqu f, _digest+5*SZ4(%rsp)
	283	vmovdqu g, _digest+6*SZ4(%rsp)
	284	vmovdqu h, _digest+7*SZ4(%rsp)
	285	i = 0
	286	.rep 4
	287	vmovdqu PSHUFFLE_BYTE_FLIP_MASK(%rip), TMP
	288	VMOVPD i*32(inp0, IDX), TT2
	289	VMOVPD i*32(inp1, IDX), TT1
	290	VMOVPD i*32(inp2, IDX), TT4
	291	VMOVPD i*32(inp3, IDX), TT3
	292	TRANSPOSE TT2, TT1, TT4, TT3, TT0, TT5
	293	vpshufb TMP, TT0, TT0
	294	vpshufb TMP, TT1, TT1
	295	vpshufb TMP, TT2, TT2
	296	vpshufb TMP, TT3, TT3
	297	ROUND_00_15 TT0,(i*4+0)
	298	ROUND_00_15 TT1,(i*4+1)
	299	ROUND_00_15 TT2,(i*4+2)
	300	ROUND_00_15 TT3,(i*4+3)
	301	i = (i+1)
	302	.endr
	303	add $128, IDX
	304
	305	i = (i*4)
	306
	307	jmp Lrounds_16_xx
	308	.align 16
	309	Lrounds_16_xx:
	310	.rep 16
	311	ROUND_16_XX T1, i
	312	i = (i+1)
	313	.endr
	314	cmp $0xa00,ROUND
	315	jb Lrounds_16_xx
	316
	317	# add old digest
	318	vpaddq _digest(%rsp), a, a
	319	vpaddq _digest+1*SZ4(%rsp), b, b
	320	vpaddq _digest+2*SZ4(%rsp), c, c
	321	vpaddq _digest+3*SZ4(%rsp), d, d
	322	vpaddq _digest+4*SZ4(%rsp), e, e
	323	vpaddq _digest+5*SZ4(%rsp), f, f
	324	vpaddq _digest+6*SZ4(%rsp), g, g
	325	vpaddq _digest+7*SZ4(%rsp), h, h
	326
	327	sub $1, INP_SIZE # unit is blocks
	328	jne lloop
	329
	330	# write back to memory (state object) the transposed digest
	331	vmovdqu a, 0*SHA512_DIGEST_ROW_SIZE(STATE)
	332	vmovdqu b, 1*SHA512_DIGEST_ROW_SIZE(STATE)
	333	vmovdqu c, 2*SHA512_DIGEST_ROW_SIZE(STATE)
	334	vmovdqu d, 3*SHA512_DIGEST_ROW_SIZE(STATE)
	335	vmovdqu e, 4*SHA512_DIGEST_ROW_SIZE(STATE)
	336	vmovdqu f, 5*SHA512_DIGEST_ROW_SIZE(STATE)
	337	vmovdqu g, 6*SHA512_DIGEST_ROW_SIZE(STATE)
	338	vmovdqu h, 7*SHA512_DIGEST_ROW_SIZE(STATE)
	339
	340	# update input data pointers
	341	add IDX, inp0
	342	mov inp0, _data_ptr+0*PTR_SZ(STATE)
	343	add IDX, inp1
	344	mov inp1, _data_ptr+1*PTR_SZ(STATE)
	345	add IDX, inp2
	346	mov inp2, _data_ptr+2*PTR_SZ(STATE)
	347	add IDX, inp3
	348	mov inp3, _data_ptr+3*PTR_SZ(STATE)
	349
	350	#;;;;;;;;;;;;;;;
	351	#; Postamble
	352	add $STACK_SPACE1, %rsp
	353	# restore callee-saved clobbered registers
	354
	355	pop %r15
	356	pop %r14
	357	pop %r13
	358	pop %r12
	359
	360	# outer calling routine restores XMM and other GP registers
	361	ret
	362	ENDPROC(sha512_x4_avx2)
	363
	364	.data
	365	.align 64
	366	K512_4:
	367	.octa 0x428a2f98d728ae22428a2f98d728ae22,\
	368	0x428a2f98d728ae22428a2f98d728ae22
	369	.octa 0x7137449123ef65cd7137449123ef65cd,\
	370	0x7137449123ef65cd7137449123ef65cd
	371	.octa 0xb5c0fbcfec4d3b2fb5c0fbcfec4d3b2f,\
	372	0xb5c0fbcfec4d3b2fb5c0fbcfec4d3b2f
	373	.octa 0xe9b5dba58189dbbce9b5dba58189dbbc,\
	374	0xe9b5dba58189dbbce9b5dba58189dbbc
	375	.octa 0x3956c25bf348b5383956c25bf348b538,\
	376	0x3956c25bf348b5383956c25bf348b538
	377	.octa 0x59f111f1b605d01959f111f1b605d019,\
	378	0x59f111f1b605d01959f111f1b605d019
	379	.octa 0x923f82a4af194f9b923f82a4af194f9b,\
	380	0x923f82a4af194f9b923f82a4af194f9b
	381	.octa 0xab1c5ed5da6d8118ab1c5ed5da6d8118,\
	382	0xab1c5ed5da6d8118ab1c5ed5da6d8118
	383	.octa 0xd807aa98a3030242d807aa98a3030242,\
	384	0xd807aa98a3030242d807aa98a3030242
	385	.octa 0x12835b0145706fbe12835b0145706fbe,\
	386	0x12835b0145706fbe12835b0145706fbe
	387	.octa 0x243185be4ee4b28c243185be4ee4b28c,\
	388	0x243185be4ee4b28c243185be4ee4b28c
	389	.octa 0x550c7dc3d5ffb4e2550c7dc3d5ffb4e2,\
	390	0x550c7dc3d5ffb4e2550c7dc3d5ffb4e2
	391	.octa 0x72be5d74f27b896f72be5d74f27b896f,\
	392	0x72be5d74f27b896f72be5d74f27b896f
	393	.octa 0x80deb1fe3b1696b180deb1fe3b1696b1,\
	394	0x80deb1fe3b1696b180deb1fe3b1696b1
	395	.octa 0x9bdc06a725c712359bdc06a725c71235,\
	396	0x9bdc06a725c712359bdc06a725c71235
	397	.octa 0xc19bf174cf692694c19bf174cf692694,\
	398	0xc19bf174cf692694c19bf174cf692694
	399	.octa 0xe49b69c19ef14ad2e49b69c19ef14ad2,\
	400	0xe49b69c19ef14ad2e49b69c19ef14ad2
	401	.octa 0xefbe4786384f25e3efbe4786384f25e3,\
	402	0xefbe4786384f25e3efbe4786384f25e3
	403	.octa 0x0fc19dc68b8cd5b50fc19dc68b8cd5b5,\
	404	0x0fc19dc68b8cd5b50fc19dc68b8cd5b5
	405	.octa 0x240ca1cc77ac9c65240ca1cc77ac9c65,\
	406	0x240ca1cc77ac9c65240ca1cc77ac9c65
	407	.octa 0x2de92c6f592b02752de92c6f592b0275,\
	408	0x2de92c6f592b02752de92c6f592b0275
	409	.octa 0x4a7484aa6ea6e4834a7484aa6ea6e483,\
	410	0x4a7484aa6ea6e4834a7484aa6ea6e483
	411	.octa 0x5cb0a9dcbd41fbd45cb0a9dcbd41fbd4,\
	412	0x5cb0a9dcbd41fbd45cb0a9dcbd41fbd4
	413	.octa 0x76f988da831153b576f988da831153b5,\
	414	0x76f988da831153b576f988da831153b5
	415	.octa 0x983e5152ee66dfab983e5152ee66dfab,\
	416	0x983e5152ee66dfab983e5152ee66dfab
	417	.octa 0xa831c66d2db43210a831c66d2db43210,\
	418	0xa831c66d2db43210a831c66d2db43210
	419	.octa 0xb00327c898fb213fb00327c898fb213f,\
	420	0xb00327c898fb213fb00327c898fb213f
	421	.octa 0xbf597fc7beef0ee4bf597fc7beef0ee4,\
	422	0xbf597fc7beef0ee4bf597fc7beef0ee4
	423	.octa 0xc6e00bf33da88fc2c6e00bf33da88fc2,\
	424	0xc6e00bf33da88fc2c6e00bf33da88fc2
	425	.octa 0xd5a79147930aa725d5a79147930aa725,\
	426	0xd5a79147930aa725d5a79147930aa725
	427	.octa 0x06ca6351e003826f06ca6351e003826f,\
	428	0x06ca6351e003826f06ca6351e003826f
	429	.octa 0x142929670a0e6e70142929670a0e6e70,\
	430	0x142929670a0e6e70142929670a0e6e70
	431	.octa 0x27b70a8546d22ffc27b70a8546d22ffc,\
	432	0x27b70a8546d22ffc27b70a8546d22ffc
	433	.octa 0x2e1b21385c26c9262e1b21385c26c926,\
	434	0x2e1b21385c26c9262e1b21385c26c926
	435	.octa 0x4d2c6dfc5ac42aed4d2c6dfc5ac42aed,\
	436	0x4d2c6dfc5ac42aed4d2c6dfc5ac42aed
	437	.octa 0x53380d139d95b3df53380d139d95b3df,\
	438	0x53380d139d95b3df53380d139d95b3df
	439	.octa 0x650a73548baf63de650a73548baf63de,\
	440	0x650a73548baf63de650a73548baf63de
	441	.octa 0x766a0abb3c77b2a8766a0abb3c77b2a8,\
	442	0x766a0abb3c77b2a8766a0abb3c77b2a8
	443	.octa 0x81c2c92e47edaee681c2c92e47edaee6,\
	444	0x81c2c92e47edaee681c2c92e47edaee6
	445	.octa 0x92722c851482353b92722c851482353b,\
	446	0x92722c851482353b92722c851482353b
	447	.octa 0xa2bfe8a14cf10364a2bfe8a14cf10364,\
	448	0xa2bfe8a14cf10364a2bfe8a14cf10364
	449	.octa 0xa81a664bbc423001a81a664bbc423001,\
	450	0xa81a664bbc423001a81a664bbc423001
	451	.octa 0xc24b8b70d0f89791c24b8b70d0f89791,\
	452	0xc24b8b70d0f89791c24b8b70d0f89791
	453	.octa 0xc76c51a30654be30c76c51a30654be30,\
	454	0xc76c51a30654be30c76c51a30654be30
	455	.octa 0xd192e819d6ef5218d192e819d6ef5218,\
	456	0xd192e819d6ef5218d192e819d6ef5218
	457	.octa 0xd69906245565a910d69906245565a910,\
	458	0xd69906245565a910d69906245565a910
	459	.octa 0xf40e35855771202af40e35855771202a,\
	460	0xf40e35855771202af40e35855771202a
	461	.octa 0x106aa07032bbd1b8106aa07032bbd1b8,\
	462	0x106aa07032bbd1b8106aa07032bbd1b8
	463	.octa 0x19a4c116b8d2d0c819a4c116b8d2d0c8,\
	464	0x19a4c116b8d2d0c819a4c116b8d2d0c8
	465	.octa 0x1e376c085141ab531e376c085141ab53,\
	466	0x1e376c085141ab531e376c085141ab53
	467	.octa 0x2748774cdf8eeb992748774cdf8eeb99,\
	468	0x2748774cdf8eeb992748774cdf8eeb99
	469	.octa 0x34b0bcb5e19b48a834b0bcb5e19b48a8,\
	470	0x34b0bcb5e19b48a834b0bcb5e19b48a8
	471	.octa 0x391c0cb3c5c95a63391c0cb3c5c95a63,\
	472	0x391c0cb3c5c95a63391c0cb3c5c95a63
	473	.octa 0x4ed8aa4ae3418acb4ed8aa4ae3418acb,\
	474	0x4ed8aa4ae3418acb4ed8aa4ae3418acb
	475	.octa 0x5b9cca4f7763e3735b9cca4f7763e373,\
	476	0x5b9cca4f7763e3735b9cca4f7763e373
	477	.octa 0x682e6ff3d6b2b8a3682e6ff3d6b2b8a3,\
	478	0x682e6ff3d6b2b8a3682e6ff3d6b2b8a3
	479	.octa 0x748f82ee5defb2fc748f82ee5defb2fc,\
	480	0x748f82ee5defb2fc748f82ee5defb2fc
	481	.octa 0x78a5636f43172f6078a5636f43172f60,\
	482	0x78a5636f43172f6078a5636f43172f60
	483	.octa 0x84c87814a1f0ab7284c87814a1f0ab72,\
	484	0x84c87814a1f0ab7284c87814a1f0ab72
	485	.octa 0x8cc702081a6439ec8cc702081a6439ec,\
	486	0x8cc702081a6439ec8cc702081a6439ec
	487	.octa 0x90befffa23631e2890befffa23631e28,\
	488	0x90befffa23631e2890befffa23631e28
	489	.octa 0xa4506cebde82bde9a4506cebde82bde9,\
	490	0xa4506cebde82bde9a4506cebde82bde9
	491	.octa 0xbef9a3f7b2c67915bef9a3f7b2c67915,\
	492	0xbef9a3f7b2c67915bef9a3f7b2c67915
	493	.octa 0xc67178f2e372532bc67178f2e372532b,\
	494	0xc67178f2e372532bc67178f2e372532b
	495	.octa 0xca273eceea26619cca273eceea26619c,\
	496	0xca273eceea26619cca273eceea26619c
	497	.octa 0xd186b8c721c0c207d186b8c721c0c207,\
	498	0xd186b8c721c0c207d186b8c721c0c207
	499	.octa 0xeada7dd6cde0eb1eeada7dd6cde0eb1e,\
	500	0xeada7dd6cde0eb1eeada7dd6cde0eb1e
	501	.octa 0xf57d4f7fee6ed178f57d4f7fee6ed178,\
	502	0xf57d4f7fee6ed178f57d4f7fee6ed178
	503	.octa 0x06f067aa72176fba06f067aa72176fba,\
	504	0x06f067aa72176fba06f067aa72176fba
	505	.octa 0x0a637dc5a2c898a60a637dc5a2c898a6,\
	506	0x0a637dc5a2c898a60a637dc5a2c898a6
	507	.octa 0x113f9804bef90dae113f9804bef90dae,\
	508	0x113f9804bef90dae113f9804bef90dae
	509	.octa 0x1b710b35131c471b1b710b35131c471b,\
	510	0x1b710b35131c471b1b710b35131c471b
	511	.octa 0x28db77f523047d8428db77f523047d84,\
	512	0x28db77f523047d8428db77f523047d84
	513	.octa 0x32caab7b40c7249332caab7b40c72493,\
	514	0x32caab7b40c7249332caab7b40c72493
	515	.octa 0x3c9ebe0a15c9bebc3c9ebe0a15c9bebc,\
	516	0x3c9ebe0a15c9bebc3c9ebe0a15c9bebc
	517	.octa 0x431d67c49c100d4c431d67c49c100d4c,\
	518	0x431d67c49c100d4c431d67c49c100d4c
	519	.octa 0x4cc5d4becb3e42b64cc5d4becb3e42b6,\
	520	0x4cc5d4becb3e42b64cc5d4becb3e42b6
	521	.octa 0x597f299cfc657e2a597f299cfc657e2a,\
	522	0x597f299cfc657e2a597f299cfc657e2a
	523	.octa 0x5fcb6fab3ad6faec5fcb6fab3ad6faec,\
	524	0x5fcb6fab3ad6faec5fcb6fab3ad6faec
	525	.octa 0x6c44198c4a4758176c44198c4a475817,\
	526	0x6c44198c4a4758176c44198c4a475817
	527
	528	PSHUFFLE_BYTE_FLIP_MASK: .octa 0x08090a0b0c0d0e0f0001020304050607
	529	.octa 0x18191a1b1c1d1e1f1011121314151617

arch/x86/crypto/sha512_ssse3_glue.c+6−0

@@ -346,4 +346,10 @@ MODULE_LICENSE("GPL"); · 345 unmodified lines
346	MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, Supplemental SSE3 accelerated");	346	MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, Supplemental SSE3 accelerated");
347		347
348	MODULE_ALIAS_CRYPTO("sha512");	348	MODULE_ALIAS_CRYPTO("sha512");
		349	MODULE_ALIAS_CRYPTO("sha512-ssse3");
		350	MODULE_ALIAS_CRYPTO("sha512-avx");
		351	MODULE_ALIAS_CRYPTO("sha512-avx2");
349	MODULE_ALIAS_CRYPTO("sha384");	352	MODULE_ALIAS_CRYPTO("sha384");
		353	MODULE_ALIAS_CRYPTO("sha384-ssse3");
		354	MODULE_ALIAS_CRYPTO("sha384-avx");
		355	MODULE_ALIAS_CRYPTO("sha384-avx2");

crypto/Kconfig+77−0

@@ -93,6 +93,15 @@ config CRYPTO_AKCIPHER · 92 unmodified lines
93	select CRYPTO_AKCIPHER2	93	select CRYPTO_AKCIPHER2
94	select CRYPTO_ALGAPI	94	select CRYPTO_ALGAPI
95		95
		96	config CRYPTO_KPP2
		97	tristate
		98	select CRYPTO_ALGAPI2
		99
		100	config CRYPTO_KPP
		101	tristate
		102	select CRYPTO_ALGAPI
		103	select CRYPTO_KPP2
		104
96	config CRYPTO_RSA	105	config CRYPTO_RSA
97	tristate "RSA algorithm"	106	tristate "RSA algorithm"
98	select CRYPTO_AKCIPHER	107	select CRYPTO_AKCIPHER
@@ -102,6 +111,19 @@ config CRYPTO_RSA · 3 unmodified lines
102	help	111	help
103	Generic implementation of the RSA public key algorithm.	112	Generic implementation of the RSA public key algorithm.
104		113
		114	config CRYPTO_DH
		115	tristate "Diffie-Hellman algorithm"
		116	select CRYPTO_KPP
		117	select MPILIB
		118	help
		119	Generic implementation of the Diffie-Hellman algorithm.
		120
		121	config CRYPTO_ECDH
		122	tristate "ECDH algorithm"
		123	select CRYTPO_KPP
		124	help
		125	Generic implementation of the ECDH algorithm
		126
105	config CRYPTO_MANAGER	127	config CRYPTO_MANAGER
106	tristate "Cryptographic algorithm manager"	128	tristate "Cryptographic algorithm manager"
107	select CRYPTO_MANAGER2	129	select CRYPTO_MANAGER2
@@ -115,6 +137,7 @@ config CRYPTO_MANAGER2 · 7 unmodified lines
115	select CRYPTO_HASH2	137	select CRYPTO_HASH2
116	select CRYPTO_BLKCIPHER2	138	select CRYPTO_BLKCIPHER2
117	select CRYPTO_AKCIPHER2	139	select CRYPTO_AKCIPHER2
		140	select CRYPTO_KPP2
118		141
119	config CRYPTO_USER	142	config CRYPTO_USER
120	tristate "Userspace cryptographic algorithm configuration"	143	tristate "Userspace cryptographic algorithm configuration"
@@ -414,6 +437,17 @@ config CRYPTO_CRC32C_INTEL · 293 unmodified lines
414	gain performance compared with software implementation.	437	gain performance compared with software implementation.
415	Module will be crc32c-intel.	438	Module will be crc32c-intel.
416		439
		440	config CRYPT_CRC32C_VPMSUM
		441	tristate "CRC32c CRC algorithm (powerpc64)"
		442	depends on PPC64
		443	select CRYPTO_HASH
		444	select CRC32
		445	help
		446	CRC32c algorithm implemented using vector polynomial multiply-sum
		447	(vpmsum) instructions, introduced in POWER8. Enable on POWER8
		448	and newer processors for improved performance.
		449
		450
417	config CRYPTO_CRC32C_SPARC64	451	config CRYPTO_CRC32C_SPARC64
418	tristate "CRC32c CRC algorithm (SPARC64)"	452	tristate "CRC32c CRC algorithm (SPARC64)"
419	depends on SPARC64	453	depends on SPARC64
@@ -681,6 +715,38 @@ config CRYPTO_SHA1_MB · 261 unmodified lines
681	lanes remain unfilled, a flush operation will be initiated to	715	lanes remain unfilled, a flush operation will be initiated to
682	process the crypto jobs, adding a slight latency.	716	process the crypto jobs, adding a slight latency.
683		717
		718	config CRYPTO_SHA256_MB
		719	tristate "SHA256 digest algorithm (x86_64 Multi-Buffer, Experimental)"
		720	depends on X86 && 64BIT
		721	select CRYPTO_SHA256
		722	select CRYPTO_HASH
		723	select CRYPTO_MCRYPTD
		724	help
		725	SHA-256 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented
		726	using multi-buffer technique. This algorithm computes on
		727	multiple data lanes concurrently with SIMD instructions for
		728	better throughput. It should not be enabled by default but
		729	used when there is significant amount of work to keep the keep
		730	the data lanes filled to get performance benefit. If the data
		731	lanes remain unfilled, a flush operation will be initiated to
		732	process the crypto jobs, adding a slight latency.
		733
		734	config CRYPTO_SHA512_MB
		735	tristate "SHA512 digest algorithm (x86_64 Multi-Buffer, Experimental)"
		736	depends on X86 && 64BIT
		737	select CRYPTO_SHA512
		738	select CRYPTO_HASH
		739	select CRYPTO_MCRYPTD
		740	help
		741	SHA-512 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented
		742	using multi-buffer technique. This algorithm computes on
		743	multiple data lanes concurrently with SIMD instructions for
		744	better throughput. It should not be enabled by default but
		745	used when there is significant amount of work to keep the keep
		746	the data lanes filled to get performance benefit. If the data
		747	lanes remain unfilled, a flush operation will be initiated to
		748	process the crypto jobs, adding a slight latency.
		749
684	config CRYPTO_SHA256	750	config CRYPTO_SHA256
685	tristate "SHA224 and SHA256 digest algorithm"	751	tristate "SHA224 and SHA256 digest algorithm"
686	select CRYPTO_HASH	752	select CRYPTO_HASH
@@ -750,6 +816,16 @@ config CRYPTO_SHA512_SPARC64 · 63 unmodified lines
750	SHA-512 secure hash standard (DFIPS 180-2) implemented	816	SHA-512 secure hash standard (DFIPS 180-2) implemented
751	using sparc64 crypto instructions, when available.	817	using sparc64 crypto instructions, when available.
752		818
		819	config CRYPTO_SHA3
		820	tristate "SHA3 digest algorithm"
		821	select CRYPTO_HASH
		822	help
		823	SHA-3 secure hash standard (DFIPS 202). It's based on
		824	cryptographic sponge function family called Keccak.
		825
		826	References:
		827	http://keccak.noekeon.org/
		828
753	config CRYPTO_TGR192	829	config CRYPTO_TGR192
754	tristate "Tiger digest algorithms"	830	tristate "Tiger digest algorithms"
755	select CRYPTO_HASH	831	select CRYPTO_HASH
@@ -1567,6 +1643,7 @@ config CRYPTO_DRBG_HASH · 811 unmodified lines
1567	config CRYPTO_DRBG_CTR	1643	config CRYPTO_DRBG_CTR
1568	bool "Enable CTR DRBG"	1644	bool "Enable CTR DRBG"
1569	select CRYPTO_AES	1645	select CRYPTO_AES
		1646	depends on CRYPTO_CTR
1570	help	1647	help
1571	Enable the CTR DRBG variant as defined in NIST SP800-90A.	1648	Enable the CTR DRBG variant as defined in NIST SP800-90A.
1572		1649

crypto/Makefile+10−2

 crypto_blkcipher-y += blkcipher.o
 crypto_blkcipher-y += skcipher.o
 obj-$(CONFIG_CRYPTO_BLKCIPHER2) += crypto_blkcipher.o
-obj-$(CONFIG_CRYPTO_BLKCIPHER2) += chainiv.o
-obj-$(CONFIG_CRYPTO_BLKCIPHER2) += eseqiv.o
 obj-$(CONFIG_CRYPTO_SEQIV) += seqiv.o
 obj-$(CONFIG_CRYPTO_ECHAINIV) += echainiv.o
 obj-$(CONFIG_CRYPTO_HASH2) += crypto_hash.o
 obj-$(CONFIG_CRYPTO_AKCIPHER2) += akcipher.o
+obj-$(CONFIG_CRYPTO_KPP2) += kpp.o
+dh_generic-y := dh.o
+dh_generic-y += dh_helper.o
+obj-$(CONFIG_CRYPTO_DH) += dh_generic.o
+ecdh_generic-y := ecc.o
+ecdh_generic-y += ecdh.o
+ecdh_generic-y += ecdh_helper.o
+obj-$(CONFIG_CRYPTO_ECDH) += ecdh_generic.o
 $(obj)/rsapubkey-asn1.o: $(obj)/rsapubkey-asn1.c $(obj)/rsapubkey-asn1.h
 $(obj)/rsaprivkey-asn1.o: $(obj)/rsaprivkey-asn1.c $(obj)/rsaprivkey-asn1.h
 obj-$(CONFIG_CRYPTO_SHA1) += sha1_generic.o
 obj-$(CONFIG_CRYPTO_SHA256) += sha256_generic.o
 obj-$(CONFIG_CRYPTO_SHA512) += sha512_generic.o
+obj-$(CONFIG_CRYPTO_SHA3) += sha3_generic.o
 obj-$(CONFIG_CRYPTO_WP512) += wp512.o
 obj-$(CONFIG_CRYPTO_TGR192) += tgr192.o
 obj-$(CONFIG_CRYPTO_GF128MUL) += gf128mul.o

crypto/ablk_helper.c+4−2

 	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 	struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm);
-	if (!may_use_simd()) {
+	if (!may_use_simd() ||
+	    (in_atomic() && cryptd_ablkcipher_queued(ctx->cryptd_tfm))) {
 		struct ablkcipher_request *cryptd_req =
 			ablkcipher_request_ctx(req);
 	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 	struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm);
-	if (!may_use_simd()) {
+	if (!may_use_simd() ||
+	    (in_atomic() && cryptd_ablkcipher_queued(ctx->cryptd_tfm))) {
 		struct ablkcipher_request *cryptd_req =
 			ablkcipher_request_ctx(req);

crypto/ablkcipher.c+0−223

@@ -14,11 +14,8 @@ · 13 unmodified lines
14	*/	14	*/
15		15
16	#include <crypto/internal/skcipher.h>	16	#include <crypto/internal/skcipher.h>
17	#include <linux/cpumask.h>
18	#include <linux/err.h>	17	#include <linux/err.h>
19	#include <linux/kernel.h>	18	#include <linux/kernel.h>
20	#include <linux/rtnetlink.h>
21	#include <linux/sched.h>
22	#include <linux/slab.h>	19	#include <linux/slab.h>
23	#include <linux/seq_file.h>	20	#include <linux/seq_file.h>
24	#include <linux/cryptouser.h>	21	#include <linux/cryptouser.h>
@@ -349,16 +346,6 @@ static unsigned int crypto_ablkcipher_ctxsize(struct crypto_alg *alg, u32 type, · 324 unmodified lines
349	return alg->cra_ctxsize;	346	return alg->cra_ctxsize;
350	}	347	}
351		348
352	int skcipher_null_givencrypt(struct skcipher_givcrypt_request *req)
353	{
354	return crypto_ablkcipher_encrypt(&req->creq);
355	}
356
357	int skcipher_null_givdecrypt(struct skcipher_givcrypt_request *req)
358	{
359	return crypto_ablkcipher_decrypt(&req->creq);
360	}
361
362	static int crypto_init_ablkcipher_ops(struct crypto_tfm *tfm, u32 type,	349	static int crypto_init_ablkcipher_ops(struct crypto_tfm *tfm, u32 type,
363	u32 mask)	350	u32 mask)
364	{	351	{
@@ -371,10 +358,6 @@ static int crypto_init_ablkcipher_ops(struct crypto_tfm *tfm, u32 type, · 6 unmodified lines
371	crt->setkey = setkey;	358	crt->setkey = setkey;
372	crt->encrypt = alg->encrypt;	359	crt->encrypt = alg->encrypt;
373	crt->decrypt = alg->decrypt;	360	crt->decrypt = alg->decrypt;
374	if (!alg->ivsize) {
375	crt->givencrypt = skcipher_null_givencrypt;
376	crt->givdecrypt = skcipher_null_givdecrypt;
377	}
378	crt->base = __crypto_ablkcipher_cast(tfm);	361	crt->base = __crypto_ablkcipher_cast(tfm);
379	crt->ivsize = alg->ivsize;	362	crt->ivsize = alg->ivsize;
380		363
@@ -436,11 +419,6 @@ const struct crypto_type crypto_ablkcipher_type = { · 55 unmodified lines
436	};	419	};
437	EXPORT_SYMBOL_GPL(crypto_ablkcipher_type);	420	EXPORT_SYMBOL_GPL(crypto_ablkcipher_type);
438		421
439	static int no_givdecrypt(struct skcipher_givcrypt_request *req)
440	{
441	return -ENOSYS;
442	}
443
444	static int crypto_init_givcipher_ops(struct crypto_tfm *tfm, u32 type,	422	static int crypto_init_givcipher_ops(struct crypto_tfm *tfm, u32 type,
445	u32 mask)	423	u32 mask)
446	{	424	{
@@ -454,8 +432,6 @@ static int crypto_init_givcipher_ops(struct crypto_tfm *tfm, u32 type, · 7 unmodified lines
454	alg->setkey : setkey;	432	alg->setkey : setkey;
455	crt->encrypt = alg->encrypt;	433	crt->encrypt = alg->encrypt;
456	crt->decrypt = alg->decrypt;	434	crt->decrypt = alg->decrypt;
457	crt->givencrypt = alg->givencrypt ?: no_givdecrypt;
458	crt->givdecrypt = alg->givdecrypt ?: no_givdecrypt;
459	crt->base = __crypto_ablkcipher_cast(tfm);	435	crt->base = __crypto_ablkcipher_cast(tfm);
460	crt->ivsize = alg->ivsize;	436	crt->ivsize = alg->ivsize;
461		437
@@ -516,202 +492,3 @@ const struct crypto_type crypto_givcipher_type = { · 54 unmodified lines
516	.report = crypto_givcipher_report,	492	.report = crypto_givcipher_report,
517	};	493	};
518	EXPORT_SYMBOL_GPL(crypto_givcipher_type);	494	EXPORT_SYMBOL_GPL(crypto_givcipher_type);
519
520	const char crypto_default_geniv(const struct crypto_alg alg)
521	{
522	if (((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) ==
523	CRYPTO_ALG_TYPE_BLKCIPHER ? alg->cra_blkcipher.ivsize :
524	alg->cra_ablkcipher.ivsize) !=
525	alg->cra_blocksize)
526	return "chainiv";
527
528	return "eseqiv";
529	}
530
531	static int crypto_givcipher_default(struct crypto_alg *alg, u32 type, u32 mask)
532	{
533	struct rtattr *tb[3];
534	struct {
535	struct rtattr attr;
536	struct crypto_attr_type data;
537	} ptype;
538	struct {
539	struct rtattr attr;
540	struct crypto_attr_alg data;
541	} palg;
542	struct crypto_template *tmpl;
543	struct crypto_instance *inst;
544	struct crypto_alg *larval;
545	const char *geniv;
546	int err;
547
548	larval = crypto_larval_lookup(alg->cra_driver_name,
549	(type & ~CRYPTO_ALG_TYPE_MASK) \|
550	CRYPTO_ALG_TYPE_GIVCIPHER,
551	mask \| CRYPTO_ALG_TYPE_MASK);
552	err = PTR_ERR(larval);
553	if (IS_ERR(larval))
554	goto out;
555
556	err = -EAGAIN;
557	if (!crypto_is_larval(larval))
558	goto drop_larval;
559
560	ptype.attr.rta_len = sizeof(ptype);
561	ptype.attr.rta_type = CRYPTOA_TYPE;
562	ptype.data.type = type \| CRYPTO_ALG_GENIV;
563	/* GENIV tells the template that we're making a default geniv. */
564	ptype.data.mask = mask \| CRYPTO_ALG_GENIV;
565	tb[0] = &ptype.attr;
566
567	palg.attr.rta_len = sizeof(palg);
568	palg.attr.rta_type = CRYPTOA_ALG;
569	/* Must use the exact name to locate ourselves. */
570	memcpy(palg.data.name, alg->cra_driver_name, CRYPTO_MAX_ALG_NAME);
571	tb[1] = &palg.attr;
572
573	tb[2] = NULL;
574
575	if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) ==
576	CRYPTO_ALG_TYPE_BLKCIPHER)
577	geniv = alg->cra_blkcipher.geniv;
578	else
579	geniv = alg->cra_ablkcipher.geniv;
580
581	if (!geniv)
582	geniv = crypto_default_geniv(alg);
583
584	tmpl = crypto_lookup_template(geniv);
585	err = -ENOENT;
586	if (!tmpl)
587	goto kill_larval;
588
589	if (tmpl->create) {
590	err = tmpl->create(tmpl, tb);
591	if (err)
592	goto put_tmpl;
593	goto ok;
594	}
595
596	inst = tmpl->alloc(tb);
597	err = PTR_ERR(inst);
598	if (IS_ERR(inst))
599	goto put_tmpl;
600
601	err = crypto_register_instance(tmpl, inst);
602	if (err) {
603	tmpl->free(inst);
604	goto put_tmpl;
605	}
606
607	ok:
608	/* Redo the lookup to use the instance we just registered. */
609	err = -EAGAIN;
610
611	put_tmpl:
612	crypto_tmpl_put(tmpl);
613	kill_larval:
614	crypto_larval_kill(larval);
615	drop_larval:
616	crypto_mod_put(larval);
617	out:
618	crypto_mod_put(alg);
619	return err;
620	}
621
622	struct crypto_alg crypto_lookup_skcipher(const char name, u32 type, u32 mask)
623	{
624	struct crypto_alg *alg;
625
626	alg = crypto_alg_mod_lookup(name, type, mask);
627	if (IS_ERR(alg))
628	return alg;
629
630	if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) ==
631	CRYPTO_ALG_TYPE_GIVCIPHER)
632	return alg;
633
634	if (!((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) ==
635	CRYPTO_ALG_TYPE_BLKCIPHER ? alg->cra_blkcipher.ivsize :
636	alg->cra_ablkcipher.ivsize))
637	return alg;
638
639	crypto_mod_put(alg);
640	alg = crypto_alg_mod_lookup(name, type \| CRYPTO_ALG_TESTED,
641	mask & ~CRYPTO_ALG_TESTED);
642	if (IS_ERR(alg))
643	return alg;
644
645	if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) ==
646	CRYPTO_ALG_TYPE_GIVCIPHER) {
647	if (~alg->cra_flags & (type ^ ~mask) & CRYPTO_ALG_TESTED) {
648	crypto_mod_put(alg);
649	alg = ERR_PTR(-ENOENT);
650	}
651	return alg;
652	}
653
654	BUG_ON(!((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) ==
655	CRYPTO_ALG_TYPE_BLKCIPHER ? alg->cra_blkcipher.ivsize :
656	alg->cra_ablkcipher.ivsize));
657
658	return ERR_PTR(crypto_givcipher_default(alg, type, mask));
659	}
660	EXPORT_SYMBOL_GPL(crypto_lookup_skcipher);
661
662	int crypto_grab_skcipher(struct crypto_skcipher_spawn spawn, const char name,
663	u32 type, u32 mask)
664	{
665	struct crypto_alg *alg;
666	int err;
667
668	type = crypto_skcipher_type(type);
669	mask = crypto_skcipher_mask(mask);
670
671	alg = crypto_lookup_skcipher(name, type, mask);
672	if (IS_ERR(alg))
673	return PTR_ERR(alg);
674
675	err = crypto_init_spawn(&spawn->base, alg, spawn->base.inst, mask);
676	crypto_mod_put(alg);
677	return err;
678	}
679	EXPORT_SYMBOL_GPL(crypto_grab_skcipher);
680
681	struct crypto_ablkcipher crypto_alloc_ablkcipher(const char alg_name,
682	u32 type, u32 mask)
683	{
684	struct crypto_tfm *tfm;
685	int err;
686
687	type = crypto_skcipher_type(type);
688	mask = crypto_skcipher_mask(mask);
689
690	for (;;) {
691	struct crypto_alg *alg;
692
693	alg = crypto_lookup_skcipher(alg_name, type, mask);
694	if (IS_ERR(alg)) {
695	err = PTR_ERR(alg);
696	goto err;
697	}
698
699	tfm = __crypto_alloc_tfm(alg, type, mask);
700	if (!IS_ERR(tfm))
701	return __crypto_ablkcipher_cast(tfm);
702
703	crypto_mod_put(alg);
704	err = PTR_ERR(tfm);
705
706	err:
707	if (err != -EAGAIN)
708	break;
709	if (fatal_signal_pending(current)) {
710	err = -EINTR;
711	break;
712	}
713	}
714
715	return ERR_PTR(err);
716	}
717	EXPORT_SYMBOL_GPL(crypto_alloc_ablkcipher);

crypto/aead.c+10−6

 	if (err)
 		goto out;
-	ctx->null = crypto_get_default_null_skcipher();
+	ctx->sknull = crypto_get_default_null_skcipher2();
-	err = PTR_ERR(ctx->null);
+	err = PTR_ERR(ctx->sknull);
-	if (IS_ERR(ctx->null))
+	if (IS_ERR(ctx->sknull))
 		goto out;
 	child = crypto_spawn_aead(aead_instance_ctx(inst));
 	return err;
 drop_null:
-	crypto_put_default_null_skcipher();
+	crypto_put_default_null_skcipher2();
 	goto out;
 }
 EXPORT_SYMBOL_GPL(aead_init_geniv);
 	struct aead_geniv_ctx *ctx = crypto_aead_ctx(tfm);
 	crypto_free_aead(ctx->child);
-	crypto_put_default_null_skcipher();
+	crypto_put_default_null_skcipher2();
 }
 EXPORT_SYMBOL_GPL(aead_exit_geniv);
 {
 	struct crypto_alg *base = &alg->base;
-	if (max(alg->maxauthsize, alg->ivsize) > PAGE_SIZE / 8)
+	if (max3(alg->maxauthsize, alg->ivsize, alg->chunksize) >
+	    PAGE_SIZE / 8)
 		return -EINVAL;
+	if (!alg->chunksize)
+		alg->chunksize = base->cra_blocksize;
 	base->cra_type = &crypto_aead_type;
 	base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK;
 	base->cra_flags |= CRYPTO_ALG_TYPE_AEAD;

crypto/ahash.c+3−3

@@ -461,10 +461,10 @@ static int crypto_ahash_init_tfm(struct crypto_tfm *tfm) · 460 unmodified lines
461		461
462	static unsigned int crypto_ahash_extsize(struct crypto_alg *alg)	462	static unsigned int crypto_ahash_extsize(struct crypto_alg *alg)
463	{	463	{
464	if (alg->cra_type == &crypto_ahash_type)	464	if (alg->cra_type != &crypto_ahash_type)
465	return alg->cra_ctxsize;	465	return sizeof(struct crypto_shash *);
466		466
467	return sizeof(struct crypto_shash *);	467	return crypto_alg_extsize(alg);
468	}	468	}
469		469
470	#ifdef CONFIG_NET	470	#ifdef CONFIG_NET

crypto/algapi.c+17−7

 }
 EXPORT_SYMBOL_GPL(crypto_attr_u32);
+int crypto_inst_setname(struct crypto_instance *inst, const char *name,
+			struct crypto_alg *alg)
+{
+	if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", name,
+		     alg->cra_name) >= CRYPTO_MAX_ALG_NAME)
+		return -ENAMETOOLONG;
+	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s(%s)",
+		     name, alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
+		return -ENAMETOOLONG;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(crypto_inst_setname);
 void *crypto_alloc_instance2(const char *name, struct crypto_alg *alg,
 			     unsigned int head)
 {
 	inst = (void *)(p + head);
-	err = -ENAMETOOLONG;
+	err = crypto_inst_setname(inst, name, alg);
-	if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", name,
+	if (err)
-		     alg->cra_name) >= CRYPTO_MAX_ALG_NAME)
-		goto err_free_inst;
-	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s(%s)",
-		     name, alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
 		goto err_free_inst;
 	return p;

crypto/authenc.c+59−57

 struct crypto_authenc_ctx {
 	struct crypto_ahash *auth;
-	struct crypto_ablkcipher *enc;
+	struct crypto_skcipher *enc;
-	struct crypto_blkcipher *null;
+	struct crypto_skcipher *null;
 };
 struct authenc_request_ctx {
 {
 	struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc);
 	struct crypto_ahash *auth = ctx->auth;
-	struct crypto_ablkcipher *enc = ctx->enc;
+	struct crypto_skcipher *enc = ctx->enc;
 	struct crypto_authenc_keys keys;
 	int err = -EINVAL;
 	if (err)
 		goto out;
-	crypto_ablkcipher_clear_flags(enc, CRYPTO_TFM_REQ_MASK);
+	crypto_skcipher_clear_flags(enc, CRYPTO_TFM_REQ_MASK);
-	crypto_ablkcipher_set_flags(enc, crypto_aead_get_flags(authenc) &
+	crypto_skcipher_set_flags(enc, crypto_aead_get_flags(authenc) &
-					 CRYPTO_TFM_REQ_MASK);
+				       CRYPTO_TFM_REQ_MASK);
-	err = crypto_ablkcipher_setkey(enc, keys.enckey, keys.enckeylen);
+	err = crypto_skcipher_setkey(enc, keys.enckey, keys.enckeylen);
-	crypto_aead_set_flags(authenc, crypto_ablkcipher_get_flags(enc) &
+	crypto_aead_set_flags(authenc, crypto_skcipher_get_flags(enc) &
 				       CRYPTO_TFM_RES_MASK);
 out:
 {
 	struct crypto_aead *authenc = crypto_aead_reqtfm(req);
 	struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc);
-	struct blkcipher_desc desc = {
+	SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null);
-		.tfm = ctx->null,
-	};
-	return crypto_blkcipher_encrypt(&desc, req->dst, req->src,
+	skcipher_request_set_tfm(skreq, ctx->null);
-					req->assoclen);
+	skcipher_request_set_callback(skreq, aead_request_flags(req),
+				      NULL, NULL);
+	skcipher_request_set_crypt(skreq, req->src, req->dst, req->assoclen,
+				   NULL);
+	return crypto_skcipher_encrypt(skreq);
 }
 static int crypto_authenc_encrypt(struct aead_request *req)
 	struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc);
 	struct authenc_instance_ctx *ictx = aead_instance_ctx(inst);
 	struct authenc_request_ctx *areq_ctx = aead_request_ctx(req);
-	struct crypto_ablkcipher *enc = ctx->enc;
+	struct crypto_skcipher *enc = ctx->enc;
 	unsigned int cryptlen = req->cryptlen;
-	struct ablkcipher_request *abreq = (void *)(areq_ctx->tail +
+	struct skcipher_request *skreq = (void *)(areq_ctx->tail +
-						    ictx->reqoff);
+						  ictx->reqoff);
 	struct scatterlist *src, *dst;
 	int err;
-	sg_init_table(areq_ctx->src, 2);
 	src = scatterwalk_ffwd(areq_ctx->src, req->src, req->assoclen);
 	dst = src;
 		if (err)
 			return err;
-		sg_init_table(areq_ctx->dst, 2);
 		dst = scatterwalk_ffwd(areq_ctx->dst, req->dst, req->assoclen);
 	}
-	ablkcipher_request_set_tfm(abreq, enc);
+	skcipher_request_set_tfm(skreq, enc);
-	ablkcipher_request_set_callback(abreq, aead_request_flags(req),
+	skcipher_request_set_callback(skreq, aead_request_flags(req),
-					crypto_authenc_encrypt_done, req);
+				      crypto_authenc_encrypt_done, req);
-	ablkcipher_request_set_crypt(abreq, src, dst, cryptlen, req->iv);
+	skcipher_request_set_crypt(skreq, src, dst, cryptlen, req->iv);
-	err = crypto_ablkcipher_encrypt(abreq);
+	err = crypto_skcipher_encrypt(skreq);
 	if (err)
 		return err;
 	struct authenc_instance_ctx *ictx = aead_instance_ctx(inst);
 	struct authenc_request_ctx *areq_ctx = aead_request_ctx(req);
 	struct ahash_request *ahreq = (void *)(areq_ctx->tail + ictx->reqoff);
-	struct ablkcipher_request *abreq = (void *)(areq_ctx->tail +
+	struct skcipher_request *skreq = (void *)(areq_ctx->tail +
-						    ictx->reqoff);
+						  ictx->reqoff);
 	unsigned int authsize = crypto_aead_authsize(authenc);
 	u8 *ihash = ahreq->result + authsize;
 	struct scatterlist *src, *dst;
 	if (crypto_memneq(ihash, ahreq->result, authsize))
 		return -EBADMSG;
-	sg_init_table(areq_ctx->src, 2);
 	src = scatterwalk_ffwd(areq_ctx->src, req->src, req->assoclen);
 	dst = src;
-	if (req->src != req->dst) {
+	if (req->src != req->dst)
-		sg_init_table(areq_ctx->dst, 2);
 		dst = scatterwalk_ffwd(areq_ctx->dst, req->dst, req->assoclen);
-	}
-	ablkcipher_request_set_tfm(abreq, ctx->enc);
+	skcipher_request_set_tfm(skreq, ctx->enc);
-	ablkcipher_request_set_callback(abreq, aead_request_flags(req),
+	skcipher_request_set_callback(skreq, aead_request_flags(req),
-					req->base.complete, req->base.data);
+				      req->base.complete, req->base.data);
-	ablkcipher_request_set_crypt(abreq, src, dst,
+	skcipher_request_set_crypt(skreq, src, dst,
-				     req->cryptlen - authsize, req->iv);
+				   req->cryptlen - authsize, req->iv);
-	return crypto_ablkcipher_decrypt(abreq);
+	return crypto_skcipher_decrypt(skreq);
 }
 static void authenc_verify_ahash_done(struct crypto_async_request *areq,
 	struct authenc_instance_ctx *ictx = aead_instance_ctx(inst);
 	struct crypto_authenc_ctx *ctx = crypto_aead_ctx(tfm);
 	struct crypto_ahash *auth;
-	struct crypto_ablkcipher *enc;
+	struct crypto_skcipher *enc;
-	struct crypto_blkcipher *null;
+	struct crypto_skcipher *null;
 	int err;
 	auth = crypto_spawn_ahash(&ictx->auth);
 	if (IS_ERR(auth))
 		return PTR_ERR(auth);
-	enc = crypto_spawn_skcipher(&ictx->enc);
+	enc = crypto_spawn_skcipher2(&ictx->enc);
 	err = PTR_ERR(enc);
 	if (IS_ERR(enc))
 		goto err_free_ahash;
-	null = crypto_get_default_null_skcipher();
+	null = crypto_get_default_null_skcipher2();
 	err = PTR_ERR(null);
 	if (IS_ERR(null))
 		goto err_free_skcipher;
 		max_t(unsigned int,
 		      crypto_ahash_reqsize(auth) +
 		      sizeof(struct ahash_request),
-		      sizeof(struct ablkcipher_request) +
+		      sizeof(struct skcipher_request) +
-		      crypto_ablkcipher_reqsize(enc)));
+		      crypto_skcipher_reqsize(enc)));
 	return 0;
 err_free_skcipher:
-	crypto_free_ablkcipher(enc);
+	crypto_free_skcipher(enc);
 err_free_ahash:
 	crypto_free_ahash(auth);
 	return err;
 	struct crypto_authenc_ctx *ctx = crypto_aead_ctx(tfm);
 	crypto_free_ahash(ctx->auth);
-	crypto_free_ablkcipher(ctx->enc);
+	crypto_free_skcipher(ctx->enc);
-	crypto_put_default_null_skcipher();
+	crypto_put_default_null_skcipher2();
 }
 static void crypto_authenc_free(struct aead_instance *inst)
 	struct aead_instance *inst;
 	struct hash_alg_common *auth;
 	struct crypto_alg *auth_base;
-	struct crypto_alg *enc;
+	struct skcipher_alg *enc;
 	struct authenc_instance_ctx *ctx;
 	const char *enc_name;
 	int err;
 		return -EINVAL;
 	auth = ahash_attr_alg(tb[1], CRYPTO_ALG_TYPE_HASH,
-			       CRYPTO_ALG_TYPE_AHASH_MASK);
+			      CRYPTO_ALG_TYPE_AHASH_MASK |
+			      crypto_requires_sync(algt->type, algt->mask));
 	if (IS_ERR(auth))
 		return PTR_ERR(auth);
 		goto err_free_inst;
 	crypto_set_skcipher_spawn(&ctx->enc, aead_crypto_instance(inst));
-	err = crypto_grab_skcipher(&ctx->enc, enc_name, 0,
+	err = crypto_grab_skcipher2(&ctx->enc, enc_name, 0,
-				   crypto_requires_sync(algt->type,
+				    crypto_requires_sync(algt->type,
-							algt->mask));
+							 algt->mask));
 	if (err)
 		goto err_drop_auth;
-	enc = crypto_skcipher_spawn_alg(&ctx->enc);
+	enc = crypto_spawn_skcipher_alg(&ctx->enc);
 	ctx->reqoff = ALIGN(2 * auth->digestsize + auth_base->cra_alignmask,
 			    auth_base->cra_alignmask + 1);
 	err = -ENAMETOOLONG;
 	if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
-		     "authenc(%s,%s)", auth_base->cra_name, enc->cra_name) >=
+		     "authenc(%s,%s)", auth_base->cra_name,
+		     enc->base.cra_name) >=
 	    CRYPTO_MAX_ALG_NAME)
 		goto err_drop_enc;
 	if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
 		     "authenc(%s,%s)", auth_base->cra_driver_name,
-		     enc->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
+		     enc->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
 		goto err_drop_enc;
-	inst->alg.base.cra_flags = enc->cra_flags & CRYPTO_ALG_ASYNC;
+	inst->alg.base.cra_flags = (auth_base->cra_flags |
-	inst->alg.base.cra_priority = enc->cra_priority * 10 +
+				    enc->base.cra_flags) & CRYPTO_ALG_ASYNC;
+	inst->alg.base.cra_priority = enc->base.cra_priority * 10 +
 				      auth_base->cra_priority;
-	inst->alg.base.cra_blocksize = enc->cra_blocksize;
+	inst->alg.base.cra_blocksize = enc->base.cra_blocksize;
 	inst->alg.base.cra_alignmask = auth_base->cra_alignmask |
-				       enc->cra_alignmask;
+				       enc->base.cra_alignmask;
 	inst->alg.base.cra_ctxsize = sizeof(struct crypto_authenc_ctx);
-	inst->alg.ivsize = enc->cra_ablkcipher.ivsize;
+	inst->alg.ivsize = crypto_skcipher_alg_ivsize(enc);
+	inst->alg.chunksize = crypto_skcipher_alg_chunksize(enc);
 	inst->alg.maxauthsize = auth->digestsize;
 	inst->alg.init = crypto_authenc_init_tfm;

crypto/authencesn.c+56−50

 struct crypto_authenc_esn_ctx {
 	unsigned int reqoff;
 	struct crypto_ahash *auth;
-	struct crypto_ablkcipher *enc;
+	struct crypto_skcipher *enc;
-	struct crypto_blkcipher *null;
+	struct crypto_skcipher *null;
 };
 struct authenc_esn_request_ctx {
 {
 	struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
 	struct crypto_ahash *auth = ctx->auth;
-	struct crypto_ablkcipher *enc = ctx->enc;
+	struct crypto_skcipher *enc = ctx->enc;
 	struct crypto_authenc_keys keys;
 	int err = -EINVAL;
 	if (err)
 		goto out;
-	crypto_ablkcipher_clear_flags(enc, CRYPTO_TFM_REQ_MASK);
+	crypto_skcipher_clear_flags(enc, CRYPTO_TFM_REQ_MASK);
-	crypto_ablkcipher_set_flags(enc, crypto_aead_get_flags(authenc_esn) &
+	crypto_skcipher_set_flags(enc, crypto_aead_get_flags(authenc_esn) &
 					 CRYPTO_TFM_REQ_MASK);
-	err = crypto_ablkcipher_setkey(enc, keys.enckey, keys.enckeylen);
+	err = crypto_skcipher_setkey(enc, keys.enckey, keys.enckeylen);
-	crypto_aead_set_flags(authenc_esn, crypto_ablkcipher_get_flags(enc) &
+	crypto_aead_set_flags(authenc_esn, crypto_skcipher_get_flags(enc) &
 					   CRYPTO_TFM_RES_MASK);
 out:
 {
 	struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req);
 	struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
-	struct blkcipher_desc desc = {
+	SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null);
-		.tfm = ctx->null,
-	};
-	return crypto_blkcipher_encrypt(&desc, req->dst, req->src, len);
+	skcipher_request_set_tfm(skreq, ctx->null);
+	skcipher_request_set_callback(skreq, aead_request_flags(req),
+				      NULL, NULL);
+	skcipher_request_set_crypt(skreq, req->src, req->dst, len, NULL);
+	return crypto_skcipher_encrypt(skreq);
 }
 static int crypto_authenc_esn_encrypt(struct aead_request *req)
 	struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req);
 	struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req);
 	struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
-	struct ablkcipher_request *abreq = (void *)(areq_ctx->tail
+	struct skcipher_request *skreq = (void *)(areq_ctx->tail +
-						    + ctx->reqoff);
+						  ctx->reqoff);
-	struct crypto_ablkcipher *enc = ctx->enc;
+	struct crypto_skcipher *enc = ctx->enc;
 	unsigned int assoclen = req->assoclen;
 	unsigned int cryptlen = req->cryptlen;
 	struct scatterlist *src, *dst;
 		dst = scatterwalk_ffwd(areq_ctx->dst, req->dst, assoclen);
 	}
-	ablkcipher_request_set_tfm(abreq, enc);
+	skcipher_request_set_tfm(skreq, enc);
-	ablkcipher_request_set_callback(abreq, aead_request_flags(req),
+	skcipher_request_set_callback(skreq, aead_request_flags(req),
-					crypto_authenc_esn_encrypt_done, req);
+				      crypto_authenc_esn_encrypt_done, req);
-	ablkcipher_request_set_crypt(abreq, src, dst, cryptlen, req->iv);
+	skcipher_request_set_crypt(skreq, src, dst, cryptlen, req->iv);
-	err = crypto_ablkcipher_encrypt(abreq);
+	err = crypto_skcipher_encrypt(skreq);
 	if (err)
 		return err;
 	unsigned int authsize = crypto_aead_authsize(authenc_esn);
 	struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req);
 	struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
-	struct ablkcipher_request *abreq = (void *)(areq_ctx->tail
+	struct skcipher_request *skreq = (void *)(areq_ctx->tail +
-						    + ctx->reqoff);
+						  ctx->reqoff);
 	struct crypto_ahash *auth = ctx->auth;
 	u8 *ohash = PTR_ALIGN((u8 *)areq_ctx->tail,
 			      crypto_ahash_alignmask(auth) + 1);
 	sg_init_table(areq_ctx->dst, 2);
 	dst = scatterwalk_ffwd(areq_ctx->dst, dst, assoclen);
-	ablkcipher_request_set_tfm(abreq, ctx->enc);
+	skcipher_request_set_tfm(skreq, ctx->enc);
-	ablkcipher_request_set_callback(abreq, flags,
+	skcipher_request_set_callback(skreq, flags,
-					req->base.complete, req->base.data);
+				      req->base.complete, req->base.data);
-	ablkcipher_request_set_crypt(abreq, dst, dst, cryptlen, req->iv);
+	skcipher_request_set_crypt(skreq, dst, dst, cryptlen, req->iv);
-	return crypto_ablkcipher_decrypt(abreq);
+	return crypto_skcipher_decrypt(skreq);
 }
 static void authenc_esn_verify_ahash_done(struct crypto_async_request *areq,
 	struct authenc_esn_instance_ctx *ictx = aead_instance_ctx(inst);
 	struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(tfm);
 	struct crypto_ahash *auth;
-	struct crypto_ablkcipher *enc;
+	struct crypto_skcipher *enc;
-	struct crypto_blkcipher *null;
+	struct crypto_skcipher *null;
 	int err;
 	auth = crypto_spawn_ahash(&ictx->auth);
 	if (IS_ERR(auth))
 		return PTR_ERR(auth);
-	enc = crypto_spawn_skcipher(&ictx->enc);
+	enc = crypto_spawn_skcipher2(&ictx->enc);
 	err = PTR_ERR(enc);
 	if (IS_ERR(enc))
 		goto err_free_ahash;
-	null = crypto_get_default_null_skcipher();
+	null = crypto_get_default_null_skcipher2();
 	err = PTR_ERR(null);
 	if (IS_ERR(null))
 		goto err_free_skcipher;
 		sizeof(struct authenc_esn_request_ctx) +
 		ctx->reqoff +
 		max_t(unsigned int,
-			crypto_ahash_reqsize(auth) +
+		      crypto_ahash_reqsize(auth) +
-			sizeof(struct ahash_request),
+		      sizeof(struct ahash_request),
-			sizeof(struct skcipher_givcrypt_request) +
+		      sizeof(struct skcipher_request) +
-			crypto_ablkcipher_reqsize(enc)));
+		      crypto_skcipher_reqsize(enc)));
 	return 0;
 err_free_skcipher:
-	crypto_free_ablkcipher(enc);
+	crypto_free_skcipher(enc);
 err_free_ahash:
 	crypto_free_ahash(auth);
 	return err;
 	struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(tfm);
 	crypto_free_ahash(ctx->auth);
-	crypto_free_ablkcipher(ctx->enc);
+	crypto_free_skcipher(ctx->enc);
-	crypto_put_default_null_skcipher();
+	crypto_put_default_null_skcipher2();
 }
 static void crypto_authenc_esn_free(struct aead_instance *inst)
 	struct aead_instance *inst;
 	struct hash_alg_common *auth;
 	struct crypto_alg *auth_base;
-	struct crypto_alg *enc;
+	struct skcipher_alg *enc;
 	struct authenc_esn_instance_ctx *ctx;
 	const char *enc_name;
 	int err;
 		return -EINVAL;
 	auth = ahash_attr_alg(tb[1], CRYPTO_ALG_TYPE_HASH,
-			       CRYPTO_ALG_TYPE_AHASH_MASK);
+			      CRYPTO_ALG_TYPE_AHASH_MASK |
+			      crypto_requires_sync(algt->type, algt->mask));
 	if (IS_ERR(auth))
 		return PTR_ERR(auth);
 		goto err_free_inst;
 	crypto_set_skcipher_spawn(&ctx->enc, aead_crypto_instance(inst));
-	err = crypto_grab_skcipher(&ctx->enc, enc_name, 0,
+	err = crypto_grab_skcipher2(&ctx->enc, enc_name, 0,
-				   crypto_requires_sync(algt->type,
+				    crypto_requires_sync(algt->type,
-							algt->mask));
+							 algt->mask));
 	if (err)
 		goto err_drop_auth;
-	enc = crypto_skcipher_spawn_alg(&ctx->enc);
+	enc = crypto_spawn_skcipher_alg(&ctx->enc);
 	err = -ENAMETOOLONG;
 	if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
 		     "authencesn(%s,%s)", auth_base->cra_name,
-		     enc->cra_name) >= CRYPTO_MAX_ALG_NAME)
+		     enc->base.cra_name) >= CRYPTO_MAX_ALG_NAME)
 		goto err_drop_enc;
 	if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
 		     "authencesn(%s,%s)", auth_base->cra_driver_name,
-		     enc->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
+		     enc->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
 		goto err_drop_enc;
-	inst->alg.base.cra_flags = enc->cra_flags & CRYPTO_ALG_ASYNC;
+	inst->alg.base.cra_flags = (auth_base->cra_flags |
-	inst->alg.base.cra_priority = enc->cra_priority * 10 +
+				    enc->base.cra_flags) & CRYPTO_ALG_ASYNC;
+	inst->alg.base.cra_priority = enc->base.cra_priority * 10 +
 				      auth_base->cra_priority;
-	inst->alg.base.cra_blocksize = enc->cra_blocksize;
+	inst->alg.base.cra_blocksize = enc->base.cra_blocksize;
 	inst->alg.base.cra_alignmask = auth_base->cra_alignmask |
-				       enc->cra_alignmask;
+				       enc->base.cra_alignmask;
 	inst->alg.base.cra_ctxsize = sizeof(struct crypto_authenc_esn_ctx);
-	inst->alg.ivsize = enc->cra_ablkcipher.ivsize;
+	inst->alg.ivsize = crypto_skcipher_alg_ivsize(enc);
+	inst->alg.chunksize = crypto_skcipher_alg_chunksize(enc);
 	inst->alg.maxauthsize = auth->digestsize;
 	inst->alg.init = crypto_authenc_esn_init_tfm;

crypto/blkcipher.c+0−185

@@ -21,7 +21,6 @@ · 20 unmodified lines
21	#include <linux/hardirq.h>	21	#include <linux/hardirq.h>
22	#include <linux/kernel.h>	22	#include <linux/kernel.h>
23	#include <linux/module.h>	23	#include <linux/module.h>
24	#include <linux/scatterlist.h>
25	#include <linux/seq_file.h>	24	#include <linux/seq_file.h>
26	#include <linux/slab.h>	25	#include <linux/slab.h>
27	#include <linux/string.h>	26	#include <linux/string.h>
@@ -466,10 +465,6 @@ static int crypto_init_blkcipher_ops_async(struct crypto_tfm *tfm) · 438 unmodified lines
466	crt->setkey = async_setkey;	465	crt->setkey = async_setkey;
467	crt->encrypt = async_encrypt;	466	crt->encrypt = async_encrypt;
468	crt->decrypt = async_decrypt;	467	crt->decrypt = async_decrypt;
469	if (!alg->ivsize) {
470	crt->givencrypt = skcipher_null_givencrypt;
471	crt->givdecrypt = skcipher_null_givdecrypt;
472	}
473	crt->base = __crypto_ablkcipher_cast(tfm);	468	crt->base = __crypto_ablkcipher_cast(tfm);
474	crt->ivsize = alg->ivsize;	469	crt->ivsize = alg->ivsize;
475		470
@@ -560,185 +555,5 @@ const struct crypto_type crypto_blkcipher_type = { · 84 unmodified lines
560	};	555	};
561	EXPORT_SYMBOL_GPL(crypto_blkcipher_type);	556	EXPORT_SYMBOL_GPL(crypto_blkcipher_type);
562		557
563	static int crypto_grab_nivcipher(struct crypto_skcipher_spawn *spawn,
564	const char *name, u32 type, u32 mask)
565	{
566	struct crypto_alg *alg;
567	int err;
568
569	type = crypto_skcipher_type(type);
570	mask = crypto_skcipher_mask(mask)\| CRYPTO_ALG_GENIV;
571
572	alg = crypto_alg_mod_lookup(name, type, mask);
573	if (IS_ERR(alg))
574	return PTR_ERR(alg);
575
576	err = crypto_init_spawn(&spawn->base, alg, spawn->base.inst, mask);
577	crypto_mod_put(alg);
578	return err;
579	}
580
581	struct crypto_instance skcipher_geniv_alloc(struct crypto_template tmpl,
582	struct rtattr **tb, u32 type,
583	u32 mask)
584	{
585	struct {
586	int (setkey)(struct crypto_ablkcipher tfm, const u8 *key,
587	unsigned int keylen);
588	int (encrypt)(struct ablkcipher_request req);
589	int (decrypt)(struct ablkcipher_request req);
590
591	unsigned int min_keysize;
592	unsigned int max_keysize;
593	unsigned int ivsize;
594
595	const char *geniv;
596	} balg;
597	const char *name;
598	struct crypto_skcipher_spawn *spawn;
599	struct crypto_attr_type *algt;
600	struct crypto_instance *inst;
601	struct crypto_alg *alg;
602	int err;
603
604	algt = crypto_get_attr_type(tb);
605	if (IS_ERR(algt))
606	return ERR_CAST(algt);
607
608	if ((algt->type ^ (CRYPTO_ALG_TYPE_GIVCIPHER \| CRYPTO_ALG_GENIV)) &
609	algt->mask)
610	return ERR_PTR(-EINVAL);
611
612	name = crypto_attr_alg_name(tb[1]);
613	if (IS_ERR(name))
614	return ERR_CAST(name);
615
616	inst = kzalloc(sizeof(inst) + sizeof(spawn), GFP_KERNEL);
617	if (!inst)
618	return ERR_PTR(-ENOMEM);
619
620	spawn = crypto_instance_ctx(inst);
621
622	/* Ignore async algorithms if necessary. */
623	mask \|= crypto_requires_sync(algt->type, algt->mask);
624
625	crypto_set_skcipher_spawn(spawn, inst);
626	err = crypto_grab_nivcipher(spawn, name, type, mask);
627	if (err)
628	goto err_free_inst;
629
630	alg = crypto_skcipher_spawn_alg(spawn);
631
632	if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) ==
633	CRYPTO_ALG_TYPE_BLKCIPHER) {
634	balg.ivsize = alg->cra_blkcipher.ivsize;
635	balg.min_keysize = alg->cra_blkcipher.min_keysize;
636	balg.max_keysize = alg->cra_blkcipher.max_keysize;
637
638	balg.setkey = async_setkey;
639	balg.encrypt = async_encrypt;
640	balg.decrypt = async_decrypt;
641
642	balg.geniv = alg->cra_blkcipher.geniv;
643	} else {
644	balg.ivsize = alg->cra_ablkcipher.ivsize;
645	balg.min_keysize = alg->cra_ablkcipher.min_keysize;
646	balg.max_keysize = alg->cra_ablkcipher.max_keysize;
647
648	balg.setkey = alg->cra_ablkcipher.setkey;
649	balg.encrypt = alg->cra_ablkcipher.encrypt;
650	balg.decrypt = alg->cra_ablkcipher.decrypt;
651
652	balg.geniv = alg->cra_ablkcipher.geniv;
653	}
654
655	err = -EINVAL;
656	if (!balg.ivsize)
657	goto err_drop_alg;
658
659	/*
660	* This is only true if we're constructing an algorithm with its
661	* default IV generator. For the default generator we elide the
662	* template name and double-check the IV generator.
663	*/
664	if (algt->mask & CRYPTO_ALG_GENIV) {
665	if (!balg.geniv)
666	balg.geniv = crypto_default_geniv(alg);
667	err = -EAGAIN;
668	if (strcmp(tmpl->name, balg.geniv))
669	goto err_drop_alg;
670
671	memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
672	memcpy(inst->alg.cra_driver_name, alg->cra_driver_name,
673	CRYPTO_MAX_ALG_NAME);
674	} else {
675	err = -ENAMETOOLONG;
676	if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME,
677	"%s(%s)", tmpl->name, alg->cra_name) >=
678	CRYPTO_MAX_ALG_NAME)
679	goto err_drop_alg;
680	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
681	"%s(%s)", tmpl->name, alg->cra_driver_name) >=
682	CRYPTO_MAX_ALG_NAME)
683	goto err_drop_alg;
684	}
685
686	inst->alg.cra_flags = CRYPTO_ALG_TYPE_GIVCIPHER \| CRYPTO_ALG_GENIV;
687	inst->alg.cra_flags \|= alg->cra_flags & CRYPTO_ALG_ASYNC;
688	inst->alg.cra_priority = alg->cra_priority;
689	inst->alg.cra_blocksize = alg->cra_blocksize;
690	inst->alg.cra_alignmask = alg->cra_alignmask;
691	inst->alg.cra_type = &crypto_givcipher_type;
692
693	inst->alg.cra_ablkcipher.ivsize = balg.ivsize;
694	inst->alg.cra_ablkcipher.min_keysize = balg.min_keysize;
695	inst->alg.cra_ablkcipher.max_keysize = balg.max_keysize;
696	inst->alg.cra_ablkcipher.geniv = balg.geniv;
697
698	inst->alg.cra_ablkcipher.setkey = balg.setkey;
699	inst->alg.cra_ablkcipher.encrypt = balg.encrypt;
700	inst->alg.cra_ablkcipher.decrypt = balg.decrypt;
701
702	out:
703	return inst;
704
705	err_drop_alg:
706	crypto_drop_skcipher(spawn);
707	err_free_inst:
708	kfree(inst);
709	inst = ERR_PTR(err);
710	goto out;
711	}
712	EXPORT_SYMBOL_GPL(skcipher_geniv_alloc);
713
714	void skcipher_geniv_free(struct crypto_instance *inst)
715	{
716	crypto_drop_skcipher(crypto_instance_ctx(inst));
717	kfree(inst);
718	}
719	EXPORT_SYMBOL_GPL(skcipher_geniv_free);
720
721	int skcipher_geniv_init(struct crypto_tfm *tfm)
722	{
723	struct crypto_instance inst = (void )tfm->__crt_alg;
724	struct crypto_ablkcipher *cipher;
725
726	cipher = crypto_spawn_skcipher(crypto_instance_ctx(inst));
727	if (IS_ERR(cipher))
728	return PTR_ERR(cipher);
729
730	tfm->crt_ablkcipher.base = cipher;
731	tfm->crt_ablkcipher.reqsize += crypto_ablkcipher_reqsize(cipher);
732
733	return 0;
734	}
735	EXPORT_SYMBOL_GPL(skcipher_geniv_init);
736
737	void skcipher_geniv_exit(struct crypto_tfm *tfm)
738	{
739	crypto_free_ablkcipher(tfm->crt_ablkcipher.base);
740	}
741	EXPORT_SYMBOL_GPL(skcipher_geniv_exit);
742
743	MODULE_LICENSE("GPL");	558	MODULE_LICENSE("GPL");
744	MODULE_DESCRIPTION("Generic block chaining cipher type");	559	MODULE_DESCRIPTION("Generic block chaining cipher type");

crypto/ccm.c+37−35

 struct crypto_ccm_ctx {
 	struct crypto_cipher *cipher;
-	struct crypto_ablkcipher *ctr;
+	struct crypto_skcipher *ctr;
 };
 struct crypto_rfc4309_ctx {
 	u32 flags;
 	struct scatterlist src[3];
 	struct scatterlist dst[3];
-	struct ablkcipher_request abreq;
+	struct skcipher_request skreq;
 };
 static inline struct crypto_ccm_req_priv_ctx *crypto_ccm_reqctx(
 			     unsigned int keylen)
 {
 	struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead);
-	struct crypto_ablkcipher *ctr = ctx->ctr;
+	struct crypto_skcipher *ctr = ctx->ctr;
 	struct crypto_cipher *tfm = ctx->cipher;
 	int err = 0;
-	crypto_ablkcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
+	crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
-	crypto_ablkcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
+	crypto_skcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
-				    CRYPTO_TFM_REQ_MASK);
+				       CRYPTO_TFM_REQ_MASK);
-	err = crypto_ablkcipher_setkey(ctr, key, keylen);
+	err = crypto_skcipher_setkey(ctr, key, keylen);
-	crypto_aead_set_flags(aead, crypto_ablkcipher_get_flags(ctr) &
+	crypto_aead_set_flags(aead, crypto_skcipher_get_flags(ctr) &
 			      CRYPTO_TFM_RES_MASK);
 	if (err)
 		goto out;
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
 	struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead);
 	struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req);
-	struct ablkcipher_request *abreq = &pctx->abreq;
+	struct skcipher_request *skreq = &pctx->skreq;
 	struct scatterlist *dst;
 	unsigned int cryptlen = req->cryptlen;
 	u8 *odata = pctx->odata;
 	if (req->src != req->dst)
 		dst = pctx->dst;
-	ablkcipher_request_set_tfm(abreq, ctx->ctr);
+	skcipher_request_set_tfm(skreq, ctx->ctr);
-	ablkcipher_request_set_callback(abreq, pctx->flags,
+	skcipher_request_set_callback(skreq, pctx->flags,
-					crypto_ccm_encrypt_done, req);
+				      crypto_ccm_encrypt_done, req);
-	ablkcipher_request_set_crypt(abreq, pctx->src, dst, cryptlen + 16, iv);
+	skcipher_request_set_crypt(skreq, pctx->src, dst, cryptlen + 16, iv);
-	err = crypto_ablkcipher_encrypt(abreq);
+	err = crypto_skcipher_encrypt(skreq);
 	if (err)
 		return err;
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
 	struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead);
 	struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req);
-	struct ablkcipher_request *abreq = &pctx->abreq;
+	struct skcipher_request *skreq = &pctx->skreq;
 	struct scatterlist *dst;
 	unsigned int authsize = crypto_aead_authsize(aead);
 	unsigned int cryptlen = req->cryptlen;
 	if (req->src != req->dst)
 		dst = pctx->dst;
-	ablkcipher_request_set_tfm(abreq, ctx->ctr);
+	skcipher_request_set_tfm(skreq, ctx->ctr);
-	ablkcipher_request_set_callback(abreq, pctx->flags,
+	skcipher_request_set_callback(skreq, pctx->flags,
-					crypto_ccm_decrypt_done, req);
+				      crypto_ccm_decrypt_done, req);
-	ablkcipher_request_set_crypt(abreq, pctx->src, dst, cryptlen + 16, iv);
+	skcipher_request_set_crypt(skreq, pctx->src, dst, cryptlen + 16, iv);
-	err = crypto_ablkcipher_decrypt(abreq);
+	err = crypto_skcipher_decrypt(skreq);
 	if (err)
 		return err;
 	struct ccm_instance_ctx *ictx = aead_instance_ctx(inst);
 	struct crypto_ccm_ctx *ctx = crypto_aead_ctx(tfm);
 	struct crypto_cipher *cipher;
-	struct crypto_ablkcipher *ctr;
+	struct crypto_skcipher *ctr;
 	unsigned long align;
 	int err;
 	if (IS_ERR(cipher))
 		return PTR_ERR(cipher);
-	ctr = crypto_spawn_skcipher(&ictx->ctr);
+	ctr = crypto_spawn_skcipher2(&ictx->ctr);
 	err = PTR_ERR(ctr);
 	if (IS_ERR(ctr))
 		goto err_free_cipher;
 	crypto_aead_set_reqsize(
 		tfm,
 		align + sizeof(struct crypto_ccm_req_priv_ctx) +
-		crypto_ablkcipher_reqsize(ctr));
+		crypto_skcipher_reqsize(ctr));
 	return 0;
 	struct crypto_ccm_ctx *ctx = crypto_aead_ctx(tfm);
 	crypto_free_cipher(ctx->cipher);
-	crypto_free_ablkcipher(ctx->ctr);
+	crypto_free_skcipher(ctx->ctr);
 }
 static void crypto_ccm_free(struct aead_instance *inst)
 {
 	struct crypto_attr_type *algt;
 	struct aead_instance *inst;
-	struct crypto_alg *ctr;
+	struct skcipher_alg *ctr;
 	struct crypto_alg *cipher;
 	struct ccm_instance_ctx *ictx;
 	int err;
 		goto err_free_inst;
 	crypto_set_skcipher_spawn(&ictx->ctr, aead_crypto_instance(inst));
-	err = crypto_grab_skcipher(&ictx->ctr, ctr_name, 0,
+	err = crypto_grab_skcipher2(&ictx->ctr, ctr_name, 0,
-				   crypto_requires_sync(algt->type,
+				    crypto_requires_sync(algt->type,
-							algt->mask));
+							 algt->mask));
 	if (err)
 		goto err_drop_cipher;
-	ctr = crypto_skcipher_spawn_alg(&ictx->ctr);
+	ctr = crypto_spawn_skcipher_alg(&ictx->ctr);
 	/* Not a stream cipher? */
 	err = -EINVAL;
-	if (ctr->cra_blocksize != 1)
+	if (ctr->base.cra_blocksize != 1)
 		goto err_drop_ctr;
 	/* We want the real thing! */
-	if (ctr->cra_ablkcipher.ivsize != 16)
+	if (crypto_skcipher_alg_ivsize(ctr) != 16)
 		goto err_drop_ctr;
 	err = -ENAMETOOLONG;
 	if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
-		     "ccm_base(%s,%s)", ctr->cra_driver_name,
+		     "ccm_base(%s,%s)", ctr->base.cra_driver_name,
 		     cipher->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
 		goto err_drop_ctr;
 	memcpy(inst->alg.base.cra_name, full_name, CRYPTO_MAX_ALG_NAME);
-	inst->alg.base.cra_flags = ctr->cra_flags & CRYPTO_ALG_ASYNC;
+	inst->alg.base.cra_flags = ctr->base.cra_flags & CRYPTO_ALG_ASYNC;
 	inst->alg.base.cra_priority = (cipher->cra_priority +
-				       ctr->cra_priority) / 2;
+				       ctr->base.cra_priority) / 2;
 	inst->alg.base.cra_blocksize = 1;
 	inst->alg.base.cra_alignmask = cipher->cra_alignmask |
-				       ctr->cra_alignmask |
+				       ctr->base.cra_alignmask |
 				       (__alignof__(u32) - 1);
 	inst->alg.ivsize = 16;
+	inst->alg.chunksize = crypto_skcipher_alg_chunksize(ctr);
 	inst->alg.maxauthsize = 16;
 	inst->alg.base.cra_ctxsize = sizeof(struct crypto_ccm_ctx);
 	inst->alg.init = crypto_ccm_init_tfm;
 	inst->alg.base.cra_alignmask = alg->base.cra_alignmask;
 	inst->alg.ivsize = 8;
+	inst->alg.chunksize = crypto_aead_alg_chunksize(alg);
 	inst->alg.maxauthsize = 16;
 	inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4309_ctx);

crypto/chacha20poly1305.c+46−43

 };
 struct chachapoly_ctx {
-	struct crypto_ablkcipher *chacha;
+	struct crypto_skcipher *chacha;
 	struct crypto_ahash *poly;
 	/* key bytes we use for the ChaCha20 IV */
 	unsigned int saltlen;
 struct chacha_req {
 	u8 iv[CHACHA20_IV_SIZE];
 	struct scatterlist src[1];
-	struct ablkcipher_request req; /* must be last member */
+	struct skcipher_request req; /* must be last member */
 };
 struct chachapoly_req_ctx {
 		dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen);
 	}
-	ablkcipher_request_set_callback(&creq->req, aead_request_flags(req),
+	skcipher_request_set_callback(&creq->req, aead_request_flags(req),
-					chacha_decrypt_done, req);
+				      chacha_decrypt_done, req);
-	ablkcipher_request_set_tfm(&creq->req, ctx->chacha);
+	skcipher_request_set_tfm(&creq->req, ctx->chacha);
-	ablkcipher_request_set_crypt(&creq->req, src, dst,
+	skcipher_request_set_crypt(&creq->req, src, dst,
-				     rctx->cryptlen, creq->iv);
+				   rctx->cryptlen, creq->iv);
-	err = crypto_ablkcipher_decrypt(&creq->req);
+	err = crypto_skcipher_decrypt(&creq->req);
 	if (err)
 		return err;
 	chacha_iv(creq->iv, req, 0);
-	ablkcipher_request_set_callback(&creq->req, aead_request_flags(req),
+	skcipher_request_set_callback(&creq->req, aead_request_flags(req),
-					poly_genkey_done, req);
+				      poly_genkey_done, req);
-	ablkcipher_request_set_tfm(&creq->req, ctx->chacha);
+	skcipher_request_set_tfm(&creq->req, ctx->chacha);
-	ablkcipher_request_set_crypt(&creq->req, creq->src, creq->src,
+	skcipher_request_set_crypt(&creq->req, creq->src, creq->src,
-				     POLY1305_KEY_SIZE, creq->iv);
+				   POLY1305_KEY_SIZE, creq->iv);
-	err = crypto_ablkcipher_decrypt(&creq->req);
+	err = crypto_skcipher_decrypt(&creq->req);
 	if (err)
 		return err;
 		dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen);
 	}
-	ablkcipher_request_set_callback(&creq->req, aead_request_flags(req),
+	skcipher_request_set_callback(&creq->req, aead_request_flags(req),
-					chacha_encrypt_done, req);
+				      chacha_encrypt_done, req);
-	ablkcipher_request_set_tfm(&creq->req, ctx->chacha);
+	skcipher_request_set_tfm(&creq->req, ctx->chacha);
-	ablkcipher_request_set_crypt(&creq->req, src, dst,
+	skcipher_request_set_crypt(&creq->req, src, dst,
-				     req->cryptlen, creq->iv);
+				   req->cryptlen, creq->iv);
-	err = crypto_ablkcipher_encrypt(&creq->req);
+	err = crypto_skcipher_encrypt(&creq->req);
 	if (err)
 		return err;
 	keylen -= ctx->saltlen;
 	memcpy(ctx->salt, key + keylen, ctx->saltlen);
-	crypto_ablkcipher_clear_flags(ctx->chacha, CRYPTO_TFM_REQ_MASK);
+	crypto_skcipher_clear_flags(ctx->chacha, CRYPTO_TFM_REQ_MASK);
-	crypto_ablkcipher_set_flags(ctx->chacha, crypto_aead_get_flags(aead) &
+	crypto_skcipher_set_flags(ctx->chacha, crypto_aead_get_flags(aead) &
-				    CRYPTO_TFM_REQ_MASK);
+					       CRYPTO_TFM_REQ_MASK);
-	err = crypto_ablkcipher_setkey(ctx->chacha, key, keylen);
+	err = crypto_skcipher_setkey(ctx->chacha, key, keylen);
-	crypto_aead_set_flags(aead, crypto_ablkcipher_get_flags(ctx->chacha) &
+	crypto_aead_set_flags(aead, crypto_skcipher_get_flags(ctx->chacha) &
-			      CRYPTO_TFM_RES_MASK);
+				    CRYPTO_TFM_RES_MASK);
 	return err;
 }
 	struct aead_instance *inst = aead_alg_instance(tfm);
 	struct chachapoly_instance_ctx *ictx = aead_instance_ctx(inst);
 	struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);
-	struct crypto_ablkcipher *chacha;
+	struct crypto_skcipher *chacha;
 	struct crypto_ahash *poly;
 	unsigned long align;
 	if (IS_ERR(poly))
 		return PTR_ERR(poly);
-	chacha = crypto_spawn_skcipher(&ictx->chacha);
+	chacha = crypto_spawn_skcipher2(&ictx->chacha);
 	if (IS_ERR(chacha)) {
 		crypto_free_ahash(poly);
 		return PTR_ERR(chacha);
 		tfm,
 		align + offsetof(struct chachapoly_req_ctx, u) +
 		max(offsetof(struct chacha_req, req) +
-		    sizeof(struct ablkcipher_request) +
+		    sizeof(struct skcipher_request) +
-		    crypto_ablkcipher_reqsize(chacha),
+		    crypto_skcipher_reqsize(chacha),
 		    offsetof(struct poly_req, req) +
 		    sizeof(struct ahash_request) +
 		    crypto_ahash_reqsize(poly)));
 	struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);
 	crypto_free_ahash(ctx->poly);
-	crypto_free_ablkcipher(ctx->chacha);
+	crypto_free_skcipher(ctx->chacha);
 }
 static void chachapoly_free(struct aead_instance *inst)
 {
 	struct crypto_attr_type *algt;
 	struct aead_instance *inst;
-	struct crypto_alg *chacha;
+	struct skcipher_alg *chacha;
 	struct crypto_alg *poly;
 	struct hash_alg_common *poly_hash;
 	struct chachapoly_instance_ctx *ctx;
 	poly = crypto_find_alg(poly_name, &crypto_ahash_type,
 			       CRYPTO_ALG_TYPE_HASH,
-			       CRYPTO_ALG_TYPE_AHASH_MASK);
+			       CRYPTO_ALG_TYPE_AHASH_MASK |
+			       crypto_requires_sync(algt->type,
+						    algt->mask));
 	if (IS_ERR(poly))
 		return PTR_ERR(poly);
 		goto err_free_inst;
 	crypto_set_skcipher_spawn(&ctx->chacha, aead_crypto_instance(inst));
-	err = crypto_grab_skcipher(&ctx->chacha, chacha_name, 0,
+	err = crypto_grab_skcipher2(&ctx->chacha, chacha_name, 0,
-				   crypto_requires_sync(algt->type,
+				    crypto_requires_sync(algt->type,
-							algt->mask));
+							 algt->mask));
 	if (err)
 		goto err_drop_poly;
-	chacha = crypto_skcipher_spawn_alg(&ctx->chacha);
+	chacha = crypto_spawn_skcipher_alg(&ctx->chacha);
 	err = -EINVAL;
 	/* Need 16-byte IV size, including Initial Block Counter value */
-	if (chacha->cra_ablkcipher.ivsize != CHACHA20_IV_SIZE)
+	if (crypto_skcipher_alg_ivsize(chacha) != CHACHA20_IV_SIZE)
 		goto out_drop_chacha;
 	/* Not a stream cipher? */
-	if (chacha->cra_blocksize != 1)
+	if (chacha->base.cra_blocksize != 1)
 		goto out_drop_chacha;
 	err = -ENAMETOOLONG;
 		     poly_name) >= CRYPTO_MAX_ALG_NAME)
 		goto out_drop_chacha;
 	if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
-		     "%s(%s,%s)", name, chacha->cra_driver_name,
+		     "%s(%s,%s)", name, chacha->base.cra_driver_name,
 		     poly->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
 		goto out_drop_chacha;
-	inst->alg.base.cra_flags = (chacha->cra_flags | poly->cra_flags) &
+	inst->alg.base.cra_flags = (chacha->base.cra_flags | poly->cra_flags) &
 				   CRYPTO_ALG_ASYNC;
-	inst->alg.base.cra_priority = (chacha->cra_priority +
+	inst->alg.base.cra_priority = (chacha->base.cra_priority +
 				       poly->cra_priority) / 2;
 	inst->alg.base.cra_blocksize = 1;
-	inst->alg.base.cra_alignmask = chacha->cra_alignmask |
+	inst->alg.base.cra_alignmask = chacha->base.cra_alignmask |
 				       poly->cra_alignmask;
 	inst->alg.base.cra_ctxsize = sizeof(struct chachapoly_ctx) +
 				     ctx->saltlen;
 	inst->alg.ivsize = ivsize;
+	inst->alg.chunksize = crypto_skcipher_alg_chunksize(chacha);
 	inst->alg.maxauthsize = POLY1305_DIGEST_SIZE;
 	inst->alg.init = chachapoly_init;
 	inst->alg.exit = chachapoly_exit;

crypto/chainiv.c+0−317

@@ -1,317 +0,0 @@
1	/*
2	* chainiv: Chain IV Generator
3	*
4	* Generate IVs simply be using the last block of the previous encryption.
5	* This is mainly useful for CBC with a synchronous algorithm.
6	*
7	* Copyright (c) 2007 Herbert Xu <herbert@gondor.apana.org.au>
8	*
9	* This program is free software; you can redistribute it and/or modify it
10	* under the terms of the GNU General Public License as published by the Free
11	* Software Foundation; either version 2 of the License, or (at your option)
12	* any later version.
13	*
14	*/
15
16	#include <crypto/internal/skcipher.h>
17	#include <crypto/rng.h>
18	#include <crypto/crypto_wq.h>
19	#include <linux/err.h>
20	#include <linux/init.h>
21	#include <linux/kernel.h>
22	#include <linux/module.h>
23	#include <linux/spinlock.h>
24	#include <linux/string.h>
25	#include <linux/workqueue.h>
26
27	enum {
28	CHAINIV_STATE_INUSE = 0,
29	};
30
31	struct chainiv_ctx {
32	spinlock_t lock;
33	char iv[];
34	};
35
36	struct async_chainiv_ctx {
37	unsigned long state;
38
39	spinlock_t lock;
40	int err;
41
42	struct crypto_queue queue;
43	struct work_struct postponed;
44
45	char iv[];
46	};
47
48	static int chainiv_givencrypt(struct skcipher_givcrypt_request *req)
49	{
50	struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
51	struct chainiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
52	struct ablkcipher_request *subreq = skcipher_givcrypt_reqctx(req);
53	unsigned int ivsize;
54	int err;
55
56	ablkcipher_request_set_tfm(subreq, skcipher_geniv_cipher(geniv));
57	ablkcipher_request_set_callback(subreq, req->creq.base.flags &
58	~CRYPTO_TFM_REQ_MAY_SLEEP,
59	req->creq.base.complete,
60	req->creq.base.data);
61	ablkcipher_request_set_crypt(subreq, req->creq.src, req->creq.dst,
62	req->creq.nbytes, req->creq.info);
63
64	spin_lock_bh(&ctx->lock);
65
66	ivsize = crypto_ablkcipher_ivsize(geniv);
67
68	memcpy(req->giv, ctx->iv, ivsize);
69	memcpy(subreq->info, ctx->iv, ivsize);
70
71	err = crypto_ablkcipher_encrypt(subreq);
72	if (err)
73	goto unlock;
74
75	memcpy(ctx->iv, subreq->info, ivsize);
76
77	unlock:
78	spin_unlock_bh(&ctx->lock);
79
80	return err;
81	}
82
83	static int chainiv_init_common(struct crypto_tfm *tfm, char iv[])
84	{
85	struct crypto_ablkcipher *geniv = __crypto_ablkcipher_cast(tfm);
86	int err = 0;
87
88	tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request);
89
90	if (iv) {
91	err = crypto_rng_get_bytes(crypto_default_rng, iv,
92	crypto_ablkcipher_ivsize(geniv));
93	crypto_put_default_rng();
94	}
95
96	return err ?: skcipher_geniv_init(tfm);
97	}
98
99	static int chainiv_init(struct crypto_tfm *tfm)
100	{
101	struct crypto_ablkcipher *geniv = __crypto_ablkcipher_cast(tfm);
102	struct chainiv_ctx *ctx = crypto_tfm_ctx(tfm);
103	char *iv;
104
105	spin_lock_init(&ctx->lock);
106
107	iv = NULL;
108	if (!crypto_get_default_rng()) {
109	crypto_ablkcipher_crt(geniv)->givencrypt = chainiv_givencrypt;
110	iv = ctx->iv;
111	}
112
113	return chainiv_init_common(tfm, iv);
114	}
115
116	static int async_chainiv_schedule_work(struct async_chainiv_ctx *ctx)
117	{
118	int queued;
119	int err = ctx->err;
120
121	if (!ctx->queue.qlen) {
122	smp_mb__before_atomic();
123	clear_bit(CHAINIV_STATE_INUSE, &ctx->state);
124
125	if (!ctx->queue.qlen \|\|
126	test_and_set_bit(CHAINIV_STATE_INUSE, &ctx->state))
127	goto out;
128	}
129
130	queued = queue_work(kcrypto_wq, &ctx->postponed);
131	BUG_ON(!queued);
132
133	out:
134	return err;
135	}
136
137	static int async_chainiv_postpone_request(struct skcipher_givcrypt_request *req)
138	{
139	struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
140	struct async_chainiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
141	int err;
142
143	spin_lock_bh(&ctx->lock);
144	err = skcipher_enqueue_givcrypt(&ctx->queue, req);
145	spin_unlock_bh(&ctx->lock);
146
147	if (test_and_set_bit(CHAINIV_STATE_INUSE, &ctx->state))
148	return err;
149
150	ctx->err = err;
151	return async_chainiv_schedule_work(ctx);
152	}
153
154	static int async_chainiv_givencrypt_tail(struct skcipher_givcrypt_request *req)
155	{
156	struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
157	struct async_chainiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
158	struct ablkcipher_request *subreq = skcipher_givcrypt_reqctx(req);
159	unsigned int ivsize = crypto_ablkcipher_ivsize(geniv);
160
161	memcpy(req->giv, ctx->iv, ivsize);
162	memcpy(subreq->info, ctx->iv, ivsize);
163
164	ctx->err = crypto_ablkcipher_encrypt(subreq);
165	if (ctx->err)
166	goto out;
167
168	memcpy(ctx->iv, subreq->info, ivsize);
169
170	out:
171	return async_chainiv_schedule_work(ctx);
172	}
173
174	static int async_chainiv_givencrypt(struct skcipher_givcrypt_request *req)
175	{
176	struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
177	struct async_chainiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
178	struct ablkcipher_request *subreq = skcipher_givcrypt_reqctx(req);
179
180	ablkcipher_request_set_tfm(subreq, skcipher_geniv_cipher(geniv));
181	ablkcipher_request_set_callback(subreq, req->creq.base.flags,
182	req->creq.base.complete,
183	req->creq.base.data);
184	ablkcipher_request_set_crypt(subreq, req->creq.src, req->creq.dst,
185	req->creq.nbytes, req->creq.info);
186
187	if (test_and_set_bit(CHAINIV_STATE_INUSE, &ctx->state))
188	goto postpone;
189
190	if (ctx->queue.qlen) {
191	clear_bit(CHAINIV_STATE_INUSE, &ctx->state);
192	goto postpone;
193	}
194
195	return async_chainiv_givencrypt_tail(req);
196
197	postpone:
198	return async_chainiv_postpone_request(req);
199	}
200
201	static void async_chainiv_do_postponed(struct work_struct *work)
202	{
203	struct async_chainiv_ctx *ctx = container_of(work,
204	struct async_chainiv_ctx,
205	postponed);
206	struct skcipher_givcrypt_request *req;
207	struct ablkcipher_request *subreq;
208	int err;
209
210	/* Only handle one request at a time to avoid hogging keventd. */
211	spin_lock_bh(&ctx->lock);
212	req = skcipher_dequeue_givcrypt(&ctx->queue);
213	spin_unlock_bh(&ctx->lock);
214
215	if (!req) {
216	async_chainiv_schedule_work(ctx);
217	return;
218	}
219
220	subreq = skcipher_givcrypt_reqctx(req);
221	subreq->base.flags \|= CRYPTO_TFM_REQ_MAY_SLEEP;
222
223	err = async_chainiv_givencrypt_tail(req);
224
225	local_bh_disable();
226	skcipher_givcrypt_complete(req, err);
227	local_bh_enable();
228	}
229
230	static int async_chainiv_init(struct crypto_tfm *tfm)
231	{
232	struct crypto_ablkcipher *geniv = __crypto_ablkcipher_cast(tfm);
233	struct async_chainiv_ctx *ctx = crypto_tfm_ctx(tfm);
234	char *iv;
235
236	spin_lock_init(&ctx->lock);
237
238	crypto_init_queue(&ctx->queue, 100);
239	INIT_WORK(&ctx->postponed, async_chainiv_do_postponed);
240
241	iv = NULL;
242	if (!crypto_get_default_rng()) {
243	crypto_ablkcipher_crt(geniv)->givencrypt =
244	async_chainiv_givencrypt;
245	iv = ctx->iv;
246	}
247
248	return chainiv_init_common(tfm, iv);
249	}
250
251	static void async_chainiv_exit(struct crypto_tfm *tfm)
252	{
253	struct async_chainiv_ctx *ctx = crypto_tfm_ctx(tfm);
254
255	BUG_ON(test_bit(CHAINIV_STATE_INUSE, &ctx->state) \|\| ctx->queue.qlen);
256
257	skcipher_geniv_exit(tfm);
258	}
259
260	static struct crypto_template chainiv_tmpl;
261
262	static struct crypto_instance chainiv_alloc(struct rtattr *tb)
263	{
264	struct crypto_attr_type *algt;
265	struct crypto_instance *inst;
266
267	algt = crypto_get_attr_type(tb);
268	if (IS_ERR(algt))
269	return ERR_CAST(algt);
270
271	inst = skcipher_geniv_alloc(&chainiv_tmpl, tb, 0, 0);
272	if (IS_ERR(inst))
273	goto out;
274
275	inst->alg.cra_init = chainiv_init;
276	inst->alg.cra_exit = skcipher_geniv_exit;
277
278	inst->alg.cra_ctxsize = sizeof(struct chainiv_ctx);
279
280	if (!crypto_requires_sync(algt->type, algt->mask)) {
281	inst->alg.cra_flags \|= CRYPTO_ALG_ASYNC;
282
283	inst->alg.cra_init = async_chainiv_init;
284	inst->alg.cra_exit = async_chainiv_exit;
285
286	inst->alg.cra_ctxsize = sizeof(struct async_chainiv_ctx);
287	}
288
289	inst->alg.cra_ctxsize += inst->alg.cra_ablkcipher.ivsize;
290
291	out:
292	return inst;
293	}
294
295	static struct crypto_template chainiv_tmpl = {
296	.name = "chainiv",
297	.alloc = chainiv_alloc,
298	.free = skcipher_geniv_free,
299	.module = THIS_MODULE,
300	};
301
302	static int __init chainiv_module_init(void)
303	{
304	return crypto_register_template(&chainiv_tmpl);
305	}
306
307	static void chainiv_module_exit(void)
308	{
309	crypto_unregister_template(&chainiv_tmpl);
310	}
311
312	module_init(chainiv_module_init);
313	module_exit(chainiv_module_exit);
314
315	MODULE_LICENSE("GPL");
316	MODULE_DESCRIPTION("Chain IV Generator");
317	MODULE_ALIAS_CRYPTO("chainiv");

crypto/cryptd.c+113−19

 #include <crypto/internal/aead.h>
 #include <crypto/cryptd.h>
 #include <crypto/crypto_wq.h>
+#include <linux/atomic.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
-#define CRYPTD_MAX_CPU_QLEN 100
+#define CRYPTD_MAX_CPU_QLEN 1000
 struct cryptd_cpu_queue {
 	struct crypto_queue queue;
 };
 struct cryptd_blkcipher_ctx {
+	atomic_t refcnt;
 	struct crypto_blkcipher *child;
 };
 };
 struct cryptd_hash_ctx {
+	atomic_t refcnt;
 	struct crypto_shash *child;
 };
 };
 struct cryptd_aead_ctx {
+	atomic_t refcnt;
 	struct crypto_aead *child;
 };
 {
 	int cpu, err;
 	struct cryptd_cpu_queue *cpu_queue;
+	struct crypto_tfm *tfm;
+	atomic_t *refcnt;
+	bool may_backlog;
 	cpu = get_cpu();
 	cpu_queue = this_cpu_ptr(queue->cpu_queue);
 	err = crypto_enqueue_request(&cpu_queue->queue, request);
+	refcnt = crypto_tfm_ctx(request->tfm);
+	may_backlog = request->flags & CRYPTO_TFM_REQ_MAY_BACKLOG;
+	if (err == -EBUSY && !may_backlog)
+		goto out_put_cpu;
 	queue_work_on(cpu, kcrypto_wq, &cpu_queue->work);
+	if (!atomic_read(refcnt))
+		goto out_put_cpu;
+	tfm = request->tfm;
+	atomic_inc(refcnt);
+out_put_cpu:
 	put_cpu();
 	return err;
 						unsigned int len))
 {
 	struct cryptd_blkcipher_request_ctx *rctx;
+	struct cryptd_blkcipher_ctx *ctx;
+	struct crypto_ablkcipher *tfm;
 	struct blkcipher_desc desc;
+	int refcnt;
 	rctx = ablkcipher_request_ctx(req);
 	req->base.complete = rctx->complete;
 out:
+	tfm = crypto_ablkcipher_reqtfm(req);
+	ctx = crypto_ablkcipher_ctx(tfm);
+	refcnt = atomic_read(&ctx->refcnt);
 	local_bh_disable();
 	rctx->complete(&req->base, err);
 	local_bh_enable();
+	if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt))
+		crypto_free_ablkcipher(tfm);
 }
 static void cryptd_blkcipher_encrypt(struct crypto_async_request *req, int err)
 	return cryptd_enqueue_request(queue, &req->base);
 }
+static void cryptd_hash_complete(struct ahash_request *req, int err)
+{
+	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+	struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+	struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
+	int refcnt = atomic_read(&ctx->refcnt);
+	local_bh_disable();
+	rctx->complete(&req->base, err);
+	local_bh_enable();
+	if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt))
+		crypto_free_ahash(tfm);
+}
 static void cryptd_hash_init(struct crypto_async_request *req_async, int err)
 {
 	struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
 	req->base.complete = rctx->complete;
 out:
-	local_bh_disable();
+	cryptd_hash_complete(req, err);
-	rctx->complete(&req->base, err);
-	local_bh_enable();
 }
 static int cryptd_hash_init_enqueue(struct ahash_request *req)
 	req->base.complete = rctx->complete;
 out:
-	local_bh_disable();
+	cryptd_hash_complete(req, err);
-	rctx->complete(&req->base, err);
-	local_bh_enable();
 }
 static int cryptd_hash_update_enqueue(struct ahash_request *req)
 	req->base.complete = rctx->complete;
 out:
-	local_bh_disable();
+	cryptd_hash_complete(req, err);
-	rctx->complete(&req->base, err);
-	local_bh_enable();
 }
 static int cryptd_hash_final_enqueue(struct ahash_request *req)
 	req->base.complete = rctx->complete;
 out:
-	local_bh_disable();
+	cryptd_hash_complete(req, err);
-	rctx->complete(&req->base, err);
-	local_bh_enable();
 }
 static int cryptd_hash_finup_enqueue(struct ahash_request *req)
 	req->base.complete = rctx->complete;
 out:
-	local_bh_disable();
+	cryptd_hash_complete(req, err);
-	rctx->complete(&req->base, err);
-	local_bh_enable();
 }
 static int cryptd_hash_digest_enqueue(struct ahash_request *req)
 			int (*crypt)(struct aead_request *req))
 {
 	struct cryptd_aead_request_ctx *rctx;
+	struct cryptd_aead_ctx *ctx;
 	crypto_completion_t compl;
+	struct crypto_aead *tfm;
+	int refcnt;
 	rctx = aead_request_ctx(req);
 	compl = rctx->complete;
 		goto out;
 	aead_request_set_tfm(req, child);
 	err = crypt( req );
 out:
+	tfm = crypto_aead_reqtfm(req);
+	ctx = crypto_aead_ctx(tfm);
+	refcnt = atomic_read(&ctx->refcnt);
 	local_bh_disable();
 	compl(&req->base, err);
 	local_bh_enable();
+	if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt))
+		crypto_free_aead(tfm);
 }
 static void cryptd_aead_encrypt(struct crypto_async_request *areq, int err)
 						  u32 type, u32 mask)
 {
 	char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
+	struct cryptd_blkcipher_ctx *ctx;
 	struct crypto_tfm *tfm;
 	if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
 		return ERR_PTR(-EINVAL);
 	}
+	ctx = crypto_tfm_ctx(tfm);
+	atomic_set(&ctx->refcnt, 1);
 	return __cryptd_ablkcipher_cast(__crypto_ablkcipher_cast(tfm));
 }
 EXPORT_SYMBOL_GPL(cryptd_alloc_ablkcipher);
 }
 EXPORT_SYMBOL_GPL(cryptd_ablkcipher_child);
+bool cryptd_ablkcipher_queued(struct cryptd_ablkcipher *tfm)
+{
+	struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base);
+	return atomic_read(&ctx->refcnt) - 1;
+}
+EXPORT_SYMBOL_GPL(cryptd_ablkcipher_queued);
 void cryptd_free_ablkcipher(struct cryptd_ablkcipher *tfm)
 {
-	crypto_free_ablkcipher(&tfm->base);
+	struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base);
+	if (atomic_dec_and_test(&ctx->refcnt))
+		crypto_free_ablkcipher(&tfm->base);
 }
 EXPORT_SYMBOL_GPL(cryptd_free_ablkcipher);
 					u32 type, u32 mask)
 {
 	char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
+	struct cryptd_hash_ctx *ctx;
 	struct crypto_ahash *tfm;
 	if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
 		return ERR_PTR(-EINVAL);
 	}
+	ctx = crypto_ahash_ctx(tfm);
+	atomic_set(&ctx->refcnt, 1);
 	return __cryptd_ahash_cast(tfm);
 }
 EXPORT_SYMBOL_GPL(cryptd_alloc_ahash);
 }
 EXPORT_SYMBOL_GPL(cryptd_shash_desc);
+bool cryptd_ahash_queued(struct cryptd_ahash *tfm)
+{
+	struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
+	return atomic_read(&ctx->refcnt) - 1;
+}
+EXPORT_SYMBOL_GPL(cryptd_ahash_queued);
 void cryptd_free_ahash(struct cryptd_ahash *tfm)
 {
-	crypto_free_ahash(&tfm->base);
+	struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
+	if (atomic_dec_and_test(&ctx->refcnt))
+		crypto_free_ahash(&tfm->base);
 }
 EXPORT_SYMBOL_GPL(cryptd_free_ahash);
 						  u32 type, u32 mask)
 {
 	char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
+	struct cryptd_aead_ctx *ctx;
 	struct crypto_aead *tfm;
 	if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
 		crypto_free_aead(tfm);
 		return ERR_PTR(-EINVAL);
 	}
+	ctx = crypto_aead_ctx(tfm);
+	atomic_set(&ctx->refcnt, 1);
 	return __cryptd_aead_cast(tfm);
 }
 EXPORT_SYMBOL_GPL(cryptd_alloc_aead);
 }
 EXPORT_SYMBOL_GPL(cryptd_aead_child);
+bool cryptd_aead_queued(struct cryptd_aead *tfm)
+{
+	struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base);
+	return atomic_read(&ctx->refcnt) - 1;
+}
+EXPORT_SYMBOL_GPL(cryptd_aead_queued);
 void cryptd_free_aead(struct cryptd_aead *tfm)
 {
-	crypto_free_aead(&tfm->base);
+	struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base);
+	if (atomic_dec_and_test(&ctx->refcnt))
+		crypto_free_aead(&tfm->base);
 }
 EXPORT_SYMBOL_GPL(cryptd_free_aead);

crypto/crypto_null.c+6−5

 #include <linux/string.h>
 static DEFINE_MUTEX(crypto_default_null_skcipher_lock);
-static struct crypto_blkcipher *crypto_default_null_skcipher;
+static struct crypto_skcipher *crypto_default_null_skcipher;
 static int crypto_default_null_skcipher_refcnt;
 static int null_compress(struct crypto_tfm *tfm, const u8 *src,
 MODULE_ALIAS_CRYPTO("digest_null");
 MODULE_ALIAS_CRYPTO("cipher_null");
-struct crypto_blkcipher *crypto_get_default_null_skcipher(void)
+struct crypto_skcipher *crypto_get_default_null_skcipher(void)
 {
-	struct crypto_blkcipher *tfm;
+	struct crypto_skcipher *tfm;
 	mutex_lock(&crypto_default_null_skcipher_lock);
 	tfm = crypto_default_null_skcipher;
 	if (!tfm) {
-		tfm = crypto_alloc_blkcipher("ecb(cipher_null)", 0, 0);
+		tfm = crypto_alloc_skcipher("ecb(cipher_null)",
+					    0, CRYPTO_ALG_ASYNC);
 		if (IS_ERR(tfm))
 			goto unlock;
 {
 	mutex_lock(&crypto_default_null_skcipher_lock);
 	if (!--crypto_default_null_skcipher_refcnt) {
-		crypto_free_blkcipher(crypto_default_null_skcipher);
+		crypto_free_skcipher(crypto_default_null_skcipher);
 		crypto_default_null_skcipher = NULL;
 	}
 	mutex_unlock(&crypto_default_null_skcipher_lock);

crypto/crypto_user.c+21−36

 #include <crypto/internal/skcipher.h>
 #include <crypto/internal/rng.h>
 #include <crypto/akcipher.h>
+#include <crypto/kpp.h>
 #include "internal.h"
 	return -EMSGSIZE;
 }
+static int crypto_report_kpp(struct sk_buff *skb, struct crypto_alg *alg)
+{
+	struct crypto_report_kpp rkpp;
+	strncpy(rkpp.type, "kpp", sizeof(rkpp.type));
+	if (nla_put(skb, CRYPTOCFGA_REPORT_KPP,
+		    sizeof(struct crypto_report_kpp), &rkpp))
+		goto nla_put_failure;
+	return 0;
+nla_put_failure:
+	return -EMSGSIZE;
+}
 static int crypto_report_one(struct crypto_alg *alg,
 			     struct crypto_user_alg *ualg, struct sk_buff *skb)
 {
 			goto nla_put_failure;
 		break;
+	case CRYPTO_ALG_TYPE_KPP:
+		if (crypto_report_kpp(skb, alg))
+			goto nla_put_failure;
+		break;
 	}
 out:
 	return err;
 }
-static struct crypto_alg *crypto_user_skcipher_alg(const char *name, u32 type,
-						   u32 mask)
-{
-	int err;
-	struct crypto_alg *alg;
-	type = crypto_skcipher_type(type);
-	mask = crypto_skcipher_mask(mask);
-	for (;;) {
-		alg = crypto_lookup_skcipher(name,  type, mask);
-		if (!IS_ERR(alg))
-			return alg;
-		err = PTR_ERR(alg);
-		if (err != -EAGAIN)
-			break;
-		if (fatal_signal_pending(current)) {
-			err = -EINTR;
-			break;
-		}
-	}
-	return ERR_PTR(err);
-}
 static int crypto_add_alg(struct sk_buff *skb, struct nlmsghdr *nlh,
 			  struct nlattr **attrs)
 {
 	else
 		name = p->cru_name;
-	switch (p->cru_type & p->cru_mask & CRYPTO_ALG_TYPE_MASK) {
+	alg = crypto_alg_mod_lookup(name, p->cru_type, p->cru_mask);
-	case CRYPTO_ALG_TYPE_GIVCIPHER:
-	case CRYPTO_ALG_TYPE_BLKCIPHER:
-	case CRYPTO_ALG_TYPE_ABLKCIPHER:
-		alg = crypto_user_skcipher_alg(name, p->cru_type, p->cru_mask);
-		break;
-	default:
-		alg = crypto_alg_mod_lookup(name, p->cru_type, p->cru_mask);
-	}
 	if (IS_ERR(alg))
 		return PTR_ERR(alg);

crypto/ctr.c+94−89

@@ -26,13 +26,13 @@ struct crypto_ctr_ctx { · 25 unmodified lines
26	};	26	};
27		27
28	struct crypto_rfc3686_ctx {	28	struct crypto_rfc3686_ctx {
29	struct crypto_ablkcipher *child;	29	struct crypto_skcipher *child;
30	u8 nonce[CTR_RFC3686_NONCE_SIZE];	30	u8 nonce[CTR_RFC3686_NONCE_SIZE];
31	};	31	};
32		32
33	struct crypto_rfc3686_req_ctx {	33	struct crypto_rfc3686_req_ctx {
34	u8 iv[CTR_RFC3686_BLOCK_SIZE];	34	u8 iv[CTR_RFC3686_BLOCK_SIZE];
35	struct ablkcipher_request subreq CRYPTO_MINALIGN_ATTR;	35	struct skcipher_request subreq CRYPTO_MINALIGN_ATTR;
36	};	36	};
37		37
38	static int crypto_ctr_setkey(struct crypto_tfm parent, const u8 key,	38	static int crypto_ctr_setkey(struct crypto_tfm parent, const u8 key,
@@ -249,11 +249,11 @@ static struct crypto_template crypto_ctr_tmpl = { · 210 unmodified lines
249	.module = THIS_MODULE,	249	.module = THIS_MODULE,
250	};	250	};
251		251
252	static int crypto_rfc3686_setkey(struct crypto_ablkcipher *parent,	252	static int crypto_rfc3686_setkey(struct crypto_skcipher *parent,
253	const u8 *key, unsigned int keylen)	253	const u8 *key, unsigned int keylen)
254	{	254	{
255	struct crypto_rfc3686_ctx *ctx = crypto_ablkcipher_ctx(parent);	255	struct crypto_rfc3686_ctx *ctx = crypto_skcipher_ctx(parent);
256	struct crypto_ablkcipher *child = ctx->child;	256	struct crypto_skcipher *child = ctx->child;
257	int err;	257	int err;
258		258
259	/* the nonce is stored in bytes at end of key */	259	/* the nonce is stored in bytes at end of key */
@@ -265,173 +265,178 @@ static int crypto_rfc3686_setkey(struct crypto_ablkcipher *parent, · 5 unmodified lines
265		265
266	keylen -= CTR_RFC3686_NONCE_SIZE;	266	keylen -= CTR_RFC3686_NONCE_SIZE;
267		267
268	crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);	268	crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
269	crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(parent) &	269	crypto_skcipher_set_flags(child, crypto_skcipher_get_flags(parent) &
270	CRYPTO_TFM_REQ_MASK);	270	CRYPTO_TFM_REQ_MASK);
271	err = crypto_ablkcipher_setkey(child, key, keylen);	271	err = crypto_skcipher_setkey(child, key, keylen);
272	crypto_ablkcipher_set_flags(parent, crypto_ablkcipher_get_flags(child) &	272	crypto_skcipher_set_flags(parent, crypto_skcipher_get_flags(child) &
273	CRYPTO_TFM_RES_MASK);	273	CRYPTO_TFM_RES_MASK);
274		274
275	return err;	275	return err;
276	}	276	}
277		277
278	static int crypto_rfc3686_crypt(struct ablkcipher_request *req)	278	static int crypto_rfc3686_crypt(struct skcipher_request *req)
279	{	279	{
280	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);	280	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
281	struct crypto_rfc3686_ctx *ctx = crypto_ablkcipher_ctx(tfm);	281	struct crypto_rfc3686_ctx *ctx = crypto_skcipher_ctx(tfm);
282	struct crypto_ablkcipher *child = ctx->child;	282	struct crypto_skcipher *child = ctx->child;
283	unsigned long align = crypto_ablkcipher_alignmask(tfm);	283	unsigned long align = crypto_skcipher_alignmask(tfm);
284	struct crypto_rfc3686_req_ctx *rctx =	284	struct crypto_rfc3686_req_ctx *rctx =
285	(void )PTR_ALIGN((u8 )ablkcipher_request_ctx(req), align + 1);	285	(void )PTR_ALIGN((u8 )skcipher_request_ctx(req), align + 1);
286	struct ablkcipher_request *subreq = &rctx->subreq;	286	struct skcipher_request *subreq = &rctx->subreq;
287	u8 *iv = rctx->iv;	287	u8 *iv = rctx->iv;
288		288
289	/* set up counter block */	289	/* set up counter block */
290	memcpy(iv, ctx->nonce, CTR_RFC3686_NONCE_SIZE);	290	memcpy(iv, ctx->nonce, CTR_RFC3686_NONCE_SIZE);
291	memcpy(iv + CTR_RFC3686_NONCE_SIZE, req->info, CTR_RFC3686_IV_SIZE);	291	memcpy(iv + CTR_RFC3686_NONCE_SIZE, req->iv, CTR_RFC3686_IV_SIZE);
292		292
293	/* initialize counter portion of counter block */	293	/* initialize counter portion of counter block */
294	(__be32 )(iv + CTR_RFC3686_NONCE_SIZE + CTR_RFC3686_IV_SIZE) =	294	(__be32 )(iv + CTR_RFC3686_NONCE_SIZE + CTR_RFC3686_IV_SIZE) =
295	cpu_to_be32(1);	295	cpu_to_be32(1);
296		296
297	ablkcipher_request_set_tfm(subreq, child);	297	skcipher_request_set_tfm(subreq, child);
298	ablkcipher_request_set_callback(subreq, req->base.flags,	298	skcipher_request_set_callback(subreq, req->base.flags,
299	req->base.complete, req->base.data);	299	req->base.complete, req->base.data);
300	ablkcipher_request_set_crypt(subreq, req->src, req->dst, req->nbytes,	300	skcipher_request_set_crypt(subreq, req->src, req->dst,
301	iv);	301	req->cryptlen, iv);
302		302
303	return crypto_ablkcipher_encrypt(subreq);	303	return crypto_skcipher_encrypt(subreq);
304	}	304	}
305		305
306	static int crypto_rfc3686_init_tfm(struct crypto_tfm *tfm)	306	static int crypto_rfc3686_init_tfm(struct crypto_skcipher *tfm)
307	{	307	{
308	struct crypto_instance inst = (void )tfm->__crt_alg;	308	struct skcipher_instance *inst = skcipher_alg_instance(tfm);
309	struct crypto_skcipher_spawn *spawn = crypto_instance_ctx(inst);	309	struct crypto_skcipher_spawn *spawn = skcipher_instance_ctx(inst);
310	struct crypto_rfc3686_ctx *ctx = crypto_tfm_ctx(tfm);	310	struct crypto_rfc3686_ctx *ctx = crypto_skcipher_ctx(tfm);
311	struct crypto_ablkcipher *cipher;	311	struct crypto_skcipher *cipher;
312	unsigned long align;	312	unsigned long align;
		313	unsigned int reqsize;
313		314
314	cipher = crypto_spawn_skcipher(spawn);	315	cipher = crypto_spawn_skcipher2(spawn);
315	if (IS_ERR(cipher))	316	if (IS_ERR(cipher))
316	return PTR_ERR(cipher);	317	return PTR_ERR(cipher);
317		318
318	ctx->child = cipher;	319	ctx->child = cipher;
319		320
320	align = crypto_tfm_alg_alignmask(tfm);	321	align = crypto_skcipher_alignmask(tfm);
321	align &= ~(crypto_tfm_ctx_alignment() - 1);	322	align &= ~(crypto_tfm_ctx_alignment() - 1);
322	tfm->crt_ablkcipher.reqsize = align +	323	reqsize = align + sizeof(struct crypto_rfc3686_req_ctx) +
323	sizeof(struct crypto_rfc3686_req_ctx) +	324	crypto_skcipher_reqsize(cipher);
324	crypto_ablkcipher_reqsize(cipher);	325	crypto_skcipher_set_reqsize(tfm, reqsize);
325		326
326	return 0;	327	return 0;
327	}	328	}
328		329
329	static void crypto_rfc3686_exit_tfm(struct crypto_tfm *tfm)	330	static void crypto_rfc3686_exit_tfm(struct crypto_skcipher *tfm)
330	{	331	{
331	struct crypto_rfc3686_ctx *ctx = crypto_tfm_ctx(tfm);	332	struct crypto_rfc3686_ctx *ctx = crypto_skcipher_ctx(tfm);
		333
		334	crypto_free_skcipher(ctx->child);
		335	}
332		336
333	crypto_free_ablkcipher(ctx->child);	337	static void crypto_rfc3686_free(struct skcipher_instance *inst)
		338	{
		339	struct crypto_skcipher_spawn *spawn = skcipher_instance_ctx(inst);
		340
		341	crypto_drop_skcipher(spawn);
		342	kfree(inst);
334	}	343	}
335		344
336	static struct crypto_instance crypto_rfc3686_alloc(struct rtattr *tb)	345	static int crypto_rfc3686_create(struct crypto_template *tmpl,
		346	struct rtattr **tb)
337	{	347	{
338	struct crypto_attr_type *algt;	348	struct crypto_attr_type *algt;
339	struct crypto_instance *inst;	349	struct skcipher_instance *inst;
340	struct crypto_alg *alg;	350	struct skcipher_alg *alg;
341	struct crypto_skcipher_spawn *spawn;	351	struct crypto_skcipher_spawn *spawn;
342	const char *cipher_name;	352	const char *cipher_name;
343	int err;	353	int err;
344		354
345	algt = crypto_get_attr_type(tb);	355	algt = crypto_get_attr_type(tb);
346	if (IS_ERR(algt))	356	if (IS_ERR(algt))
347	return ERR_CAST(algt);	357	return PTR_ERR(algt);
348		358
349	if ((algt->type ^ CRYPTO_ALG_TYPE_BLKCIPHER) & algt->mask)	359	if ((algt->type ^ CRYPTO_ALG_TYPE_SKCIPHER) & algt->mask)
350	return ERR_PTR(-EINVAL);	360	return -EINVAL;
351		361
352	cipher_name = crypto_attr_alg_name(tb[1]);	362	cipher_name = crypto_attr_alg_name(tb[1]);
353	if (IS_ERR(cipher_name))	363	if (IS_ERR(cipher_name))
354	return ERR_CAST(cipher_name);	364	return PTR_ERR(cipher_name);
355		365
356	inst = kzalloc(sizeof(inst) + sizeof(spawn), GFP_KERNEL);	366	inst = kzalloc(sizeof(inst) + sizeof(spawn), GFP_KERNEL);
357	if (!inst)	367	if (!inst)
358	return ERR_PTR(-ENOMEM);	368	return -ENOMEM;
359		369
360	spawn = crypto_instance_ctx(inst);	370	spawn = skcipher_instance_ctx(inst);
361		371
362	crypto_set_skcipher_spawn(spawn, inst);	372	crypto_set_skcipher_spawn(spawn, skcipher_crypto_instance(inst));
363	err = crypto_grab_skcipher(spawn, cipher_name, 0,	373	err = crypto_grab_skcipher2(spawn, cipher_name, 0,
364	crypto_requires_sync(algt->type,	374	crypto_requires_sync(algt->type,
365	algt->mask));	375	algt->mask));
366	if (err)	376	if (err)
367	goto err_free_inst;	377	goto err_free_inst;
368		378
369	alg = crypto_skcipher_spawn_alg(spawn);	379	alg = crypto_spawn_skcipher_alg(spawn);
370		380
371	/* We only support 16-byte blocks. */	381	/* We only support 16-byte blocks. */
372	err = -EINVAL;	382	err = -EINVAL;
373	if (alg->cra_ablkcipher.ivsize != CTR_RFC3686_BLOCK_SIZE)	383	if (crypto_skcipher_alg_ivsize(alg) != CTR_RFC3686_BLOCK_SIZE)
374	goto err_drop_spawn;	384	goto err_drop_spawn;
375		385
376	/* Not a stream cipher? */	386	/* Not a stream cipher? */
377	if (alg->cra_blocksize != 1)	387	if (alg->base.cra_blocksize != 1)
378	goto err_drop_spawn;	388	goto err_drop_spawn;
379		389
380	err = -ENAMETOOLONG;	390	err = -ENAMETOOLONG;
381	if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, "rfc3686(%s)",	391	if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
382	alg->cra_name) >= CRYPTO_MAX_ALG_NAME)	392	"rfc3686(%s)", alg->base.cra_name) >= CRYPTO_MAX_ALG_NAME)
383	goto err_drop_spawn;	393	goto err_drop_spawn;
384	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,	394	if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
385	"rfc3686(%s)", alg->cra_driver_name) >=	395	"rfc3686(%s)", alg->base.cra_driver_name) >=
386	CRYPTO_MAX_ALG_NAME)	396	CRYPTO_MAX_ALG_NAME)
387	goto err_drop_spawn;	397	goto err_drop_spawn;
388		398
389	inst->alg.cra_priority = alg->cra_priority;	399	inst->alg.base.cra_priority = alg->base.cra_priority;
390	inst->alg.cra_blocksize = 1;	400	inst->alg.base.cra_blocksize = 1;
391	inst->alg.cra_alignmask = alg->cra_alignmask;	401	inst->alg.base.cra_alignmask = alg->base.cra_alignmask;
392		402
393	inst->alg.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \|	403	inst->alg.base.cra_flags = alg->base.cra_flags & CRYPTO_ALG_ASYNC;
394	(alg->cra_flags & CRYPTO_ALG_ASYNC);
395	inst->alg.cra_type = &crypto_ablkcipher_type;
396		404
397	inst->alg.cra_ablkcipher.ivsize = CTR_RFC3686_IV_SIZE;	405	inst->alg.ivsize = CTR_RFC3686_IV_SIZE;
398	inst->alg.cra_ablkcipher.min_keysize =	406	inst->alg.chunksize = crypto_skcipher_alg_chunksize(alg);
399	alg->cra_ablkcipher.min_keysize + CTR_RFC3686_NONCE_SIZE;	407	inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg) +
400	inst->alg.cra_ablkcipher.max_keysize =	408	CTR_RFC3686_NONCE_SIZE;
401	alg->cra_ablkcipher.max_keysize + CTR_RFC3686_NONCE_SIZE;	409	inst->alg.max_keysize = crypto_skcipher_alg_max_keysize(alg) +
		410	CTR_RFC3686_NONCE_SIZE;
402		411
403	inst->alg.cra_ablkcipher.geniv = "seqiv";	412	inst->alg.setkey = crypto_rfc3686_setkey;
		413	inst->alg.encrypt = crypto_rfc3686_crypt;
		414	inst->alg.decrypt = crypto_rfc3686_crypt;
404		415
405	inst->alg.cra_ablkcipher.setkey = crypto_rfc3686_setkey;	416	inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc3686_ctx);
406	inst->alg.cra_ablkcipher.encrypt = crypto_rfc3686_crypt;
407	inst->alg.cra_ablkcipher.decrypt = crypto_rfc3686_crypt;
408		417
409	inst->alg.cra_ctxsize = sizeof(struct crypto_rfc3686_ctx);	418	inst->alg.init = crypto_rfc3686_init_tfm;
		419	inst->alg.exit = crypto_rfc3686_exit_tfm;
410		420
411	inst->alg.cra_init = crypto_rfc3686_init_tfm;	421	inst->free = crypto_rfc3686_free;
412	inst->alg.cra_exit = crypto_rfc3686_exit_tfm;
413		422
414	return inst;	423	err = skcipher_register_instance(tmpl, inst);
		424	if (err)
		425	goto err_drop_spawn;
		426
		427	out:
		428	return err;
415		429
416	err_drop_spawn:	430	err_drop_spawn:
417	crypto_drop_skcipher(spawn);	431	crypto_drop_skcipher(spawn);
418	err_free_inst:	432	err_free_inst:
419	kfree(inst);	433	kfree(inst);
420	return ERR_PTR(err);	434	goto out;
421	}
422
423	static void crypto_rfc3686_free(struct crypto_instance *inst)
424	{
425	struct crypto_skcipher_spawn *spawn = crypto_instance_ctx(inst);
426
427	crypto_drop_skcipher(spawn);
428	kfree(inst);
429	}	435	}
430		436
431	static struct crypto_template crypto_rfc3686_tmpl = {	437	static struct crypto_template crypto_rfc3686_tmpl = {
432	.name = "rfc3686",	438	.name = "rfc3686",
433	.alloc = crypto_rfc3686_alloc,	439	.create = crypto_rfc3686_create,
434	.free = crypto_rfc3686_free,
435	.module = THIS_MODULE,	440	.module = THIS_MODULE,
436	};	441	};
437		442

crypto/cts.c+285−210

@@ -40,7 +40,7 @@ · 39 unmodified lines
40	* rfc3962 includes errata information in its Appendix A.	40	* rfc3962 includes errata information in its Appendix A.
41	*/	41	*/
42		42
43	#include <crypto/algapi.h>	43	#include <crypto/internal/skcipher.h>
44	#include <linux/err.h>	44	#include <linux/err.h>
45	#include <linux/init.h>	45	#include <linux/init.h>
46	#include <linux/kernel.h>	46	#include <linux/kernel.h>
@@ -51,289 +51,364 @@ · 4 unmodified lines
51	#include <linux/slab.h>	51	#include <linux/slab.h>
52		52
53	struct crypto_cts_ctx {	53	struct crypto_cts_ctx {
54	struct crypto_blkcipher *child;	54	struct crypto_skcipher *child;
55	};	55	};
56		56
57	static int crypto_cts_setkey(struct crypto_tfm parent, const u8 key,	57	struct crypto_cts_reqctx {
58	unsigned int keylen)	58	struct scatterlist sg[2];
		59	unsigned offset;
		60	struct skcipher_request subreq;
		61	};
		62
		63	static inline u8 crypto_cts_reqctx_space(struct skcipher_request req)
59	{	64	{
60	struct crypto_cts_ctx *ctx = crypto_tfm_ctx(parent);	65	struct crypto_cts_reqctx *rctx = skcipher_request_ctx(req);
61	struct crypto_blkcipher *child = ctx->child;	66	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
62	int err;	67	struct crypto_cts_ctx *ctx = crypto_skcipher_ctx(tfm);
		68	struct crypto_skcipher *child = ctx->child;
63		69
64	crypto_blkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);	70	return PTR_ALIGN((u8 *)(rctx + 1) + crypto_skcipher_reqsize(child),
65	crypto_blkcipher_set_flags(child, crypto_tfm_get_flags(parent) &	71	crypto_skcipher_alignmask(tfm) + 1);
66	CRYPTO_TFM_REQ_MASK);
67	err = crypto_blkcipher_setkey(child, key, keylen);
68	crypto_tfm_set_flags(parent, crypto_blkcipher_get_flags(child) &
69	CRYPTO_TFM_RES_MASK);
70	return err;
71	}	72	}
72		73
73	static int cts_cbc_encrypt(struct crypto_cts_ctx *ctx,	74	static int crypto_cts_setkey(struct crypto_skcipher parent, const u8 key,
74	struct blkcipher_desc *desc,	75	unsigned int keylen)
75	struct scatterlist *dst,
76	struct scatterlist *src,
77	unsigned int offset,
78	unsigned int nbytes)
79	{	76	{
80	int bsize = crypto_blkcipher_blocksize(desc->tfm);	77	struct crypto_cts_ctx *ctx = crypto_skcipher_ctx(parent);
81	u8 tmp[bsize], tmp2[bsize];	78	struct crypto_skcipher *child = ctx->child;
82	struct blkcipher_desc lcldesc;
83	struct scatterlist sgsrc[1], sgdst[1];
84	int lastn = nbytes - bsize;
85	u8 iv[bsize];
86	u8 s[bsize * 2], d[bsize * 2];
87	int err;	79	int err;
88		80
89	if (lastn < 0)	81	crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
90	return -EINVAL;	82	crypto_skcipher_set_flags(child, crypto_skcipher_get_flags(parent) &
		83	CRYPTO_TFM_REQ_MASK);
		84	err = crypto_skcipher_setkey(child, key, keylen);
		85	crypto_skcipher_set_flags(parent, crypto_skcipher_get_flags(child) &
		86	CRYPTO_TFM_RES_MASK);
		87	return err;
		88	}
91		89
92	sg_init_table(sgsrc, 1);	90	static void cts_cbc_crypt_done(struct crypto_async_request *areq, int err)
93	sg_init_table(sgdst, 1);	91	{
		92	struct skcipher_request *req = areq->data;
94		93
95	memset(s, 0, sizeof(s));	94	if (err == -EINPROGRESS)
96	scatterwalk_map_and_copy(s, src, offset, nbytes, 0);	95	return;
97		96
98	memcpy(iv, desc->info, bsize);	97	skcipher_request_complete(req, err);
		98	}
99		99
100	lcldesc.tfm = ctx->child;	100	static int cts_cbc_encrypt(struct skcipher_request *req)
101	lcldesc.info = iv;	101	{
102	lcldesc.flags = desc->flags;	102	struct crypto_cts_reqctx *rctx = skcipher_request_ctx(req);
		103	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
		104	struct skcipher_request *subreq = &rctx->subreq;
		105	int bsize = crypto_skcipher_blocksize(tfm);
		106	u8 d[bsize * 2] __attribute__ ((aligned(__alignof__(u32))));
		107	struct scatterlist *sg;
		108	unsigned int offset;
		109	int lastn;
		110
		111	offset = rctx->offset;
		112	lastn = req->cryptlen - offset;
		113
		114	sg = scatterwalk_ffwd(rctx->sg, req->dst, offset - bsize);
		115	scatterwalk_map_and_copy(d + bsize, sg, 0, bsize, 0);
		116
		117	memset(d, 0, bsize);
		118	scatterwalk_map_and_copy(d, req->src, offset, lastn, 0);
		119
		120	scatterwalk_map_and_copy(d, sg, 0, bsize + lastn, 1);
		121	memzero_explicit(d, sizeof(d));
		122
		123	skcipher_request_set_callback(subreq, req->base.flags &
		124	CRYPTO_TFM_REQ_MAY_BACKLOG,
		125	cts_cbc_crypt_done, req);
		126	skcipher_request_set_crypt(subreq, sg, sg, bsize, req->iv);
		127	return crypto_skcipher_encrypt(subreq);
		128	}
103		129
104	sg_set_buf(&sgsrc[0], s, bsize);	130	static void crypto_cts_encrypt_done(struct crypto_async_request *areq, int err)
105	sg_set_buf(&sgdst[0], tmp, bsize);	131	{
106	err = crypto_blkcipher_encrypt_iv(&lcldesc, sgdst, sgsrc, bsize);	132	struct skcipher_request *req = areq->data;
107		133
108	memcpy(d + bsize, tmp, lastn);	134	if (err)
		135	goto out;
109		136
110	lcldesc.info = tmp;	137	err = cts_cbc_encrypt(req);
		138	if (err == -EINPROGRESS \|\|
		139	(err == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
		140	return;
111		141
112	sg_set_buf(&sgsrc[0], s + bsize, bsize);	142	out:
113	sg_set_buf(&sgdst[0], tmp2, bsize);	143	skcipher_request_complete(req, err);
114	err = crypto_blkcipher_encrypt_iv(&lcldesc, sgdst, sgsrc, bsize);	144	}
115		145
116	memcpy(d, tmp2, bsize);	146	static int crypto_cts_encrypt(struct skcipher_request *req)
		147	{
		148	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
		149	struct crypto_cts_reqctx *rctx = skcipher_request_ctx(req);
		150	struct crypto_cts_ctx *ctx = crypto_skcipher_ctx(tfm);
		151	struct skcipher_request *subreq = &rctx->subreq;
		152	int bsize = crypto_skcipher_blocksize(tfm);
		153	unsigned int nbytes = req->cryptlen;
		154	int cbc_blocks = (nbytes + bsize - 1) / bsize - 1;
		155	unsigned int offset;
		156
		157	skcipher_request_set_tfm(subreq, ctx->child);
		158
		159	if (cbc_blocks <= 0) {
		160	skcipher_request_set_callback(subreq, req->base.flags,
		161	req->base.complete,
		162	req->base.data);
		163	skcipher_request_set_crypt(subreq, req->src, req->dst, nbytes,
		164	req->iv);
		165	return crypto_skcipher_encrypt(subreq);
		166	}
117		167
118	scatterwalk_map_and_copy(d, dst, offset, nbytes, 1);	168	offset = cbc_blocks * bsize;
		169	rctx->offset = offset;
119		170
120	memcpy(desc->info, tmp2, bsize);	171	skcipher_request_set_callback(subreq, req->base.flags,
		172	crypto_cts_encrypt_done, req);
		173	skcipher_request_set_crypt(subreq, req->src, req->dst,
		174	offset, req->iv);
121		175
122	return err;	176	return crypto_skcipher_encrypt(subreq) ?:
		177	cts_cbc_encrypt(req);
123	}	178	}
124		179
125	static int crypto_cts_encrypt(struct blkcipher_desc *desc,	180	static int cts_cbc_decrypt(struct skcipher_request *req)
126	struct scatterlist dst, struct scatterlist src,
127	unsigned int nbytes)
128	{	181	{
129	struct crypto_cts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);	182	struct crypto_cts_reqctx *rctx = skcipher_request_ctx(req);
130	int bsize = crypto_blkcipher_blocksize(desc->tfm);	183	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
131	int tot_blocks = (nbytes + bsize - 1) / bsize;	184	struct skcipher_request *subreq = &rctx->subreq;
132	int cbc_blocks = tot_blocks > 2 ? tot_blocks - 2 : 0;	185	int bsize = crypto_skcipher_blocksize(tfm);
133	struct blkcipher_desc lcldesc;	186	u8 d[bsize * 2] __attribute__ ((aligned(__alignof__(u32))));
134	int err;	187	struct scatterlist *sg;
		188	unsigned int offset;
		189	u8 *space;
		190	int lastn;
		191
		192	offset = rctx->offset;
		193	lastn = req->cryptlen - offset;
		194
		195	sg = scatterwalk_ffwd(rctx->sg, req->dst, offset - bsize);
		196
		197	/* 1. Decrypt Cn-1 (s) to create Dn */
		198	scatterwalk_map_and_copy(d + bsize, sg, 0, bsize, 0);
		199	space = crypto_cts_reqctx_space(req);
		200	crypto_xor(d + bsize, space, bsize);
		201	/* 2. Pad Cn with zeros at the end to create C of length BB */
		202	memset(d, 0, bsize);
		203	scatterwalk_map_and_copy(d, req->src, offset, lastn, 0);
		204	/* 3. Exclusive-or Dn with C to create Xn */
		205	/* 4. Select the first Ln bytes of Xn to create Pn */
		206	crypto_xor(d + bsize, d, lastn);
		207
		208	/* 5. Append the tail (BB - Ln) bytes of Xn to Cn to create En */
		209	memcpy(d + lastn, d + bsize + lastn, bsize - lastn);
		210	/* 6. Decrypt En to create Pn-1 */
135		211
136	lcldesc.tfm = ctx->child;	212	scatterwalk_map_and_copy(d, sg, 0, bsize + lastn, 1);
137	lcldesc.info = desc->info;	213	memzero_explicit(d, sizeof(d));
138	lcldesc.flags = desc->flags;
139
140	if (tot_blocks == 1) {
141	err = crypto_blkcipher_encrypt_iv(&lcldesc, dst, src, bsize);
142	} else if (nbytes <= bsize * 2) {
143	err = cts_cbc_encrypt(ctx, desc, dst, src, 0, nbytes);
144	} else {
145	/* do normal function for tot_blocks - 2 */
146	err = crypto_blkcipher_encrypt_iv(&lcldesc, dst, src,
147	cbc_blocks * bsize);
148	if (err == 0) {
149	/* do cts for final two blocks */
150	err = cts_cbc_encrypt(ctx, desc, dst, src,
151	cbc_blocks * bsize,
152	nbytes - (cbc_blocks * bsize));
153	}
154	}
155		214
156	return err;	215	skcipher_request_set_callback(subreq, req->base.flags &
		216	CRYPTO_TFM_REQ_MAY_BACKLOG,
		217	cts_cbc_crypt_done, req);
		218
		219	skcipher_request_set_crypt(subreq, sg, sg, bsize, space);
		220	return crypto_skcipher_decrypt(subreq);
157	}	221	}
158		222
159	static int cts_cbc_decrypt(struct crypto_cts_ctx *ctx,	223	static void crypto_cts_decrypt_done(struct crypto_async_request *areq, int err)
160	struct blkcipher_desc *desc,
161	struct scatterlist *dst,
162	struct scatterlist *src,
163	unsigned int offset,
164	unsigned int nbytes)
165	{	224	{
166	int bsize = crypto_blkcipher_blocksize(desc->tfm);	225	struct skcipher_request *req = areq->data;
167	u8 tmp[bsize];
168	struct blkcipher_desc lcldesc;
169	struct scatterlist sgsrc[1], sgdst[1];
170	int lastn = nbytes - bsize;
171	u8 iv[bsize];
172	u8 s[bsize * 2], d[bsize * 2];
173	int err;
174
175	if (lastn < 0)
176	return -EINVAL;
177		226
178	sg_init_table(sgsrc, 1);	227	if (err)
179	sg_init_table(sgdst, 1);	228	goto out;
180		229
181	scatterwalk_map_and_copy(s, src, offset, nbytes, 0);	230	err = cts_cbc_decrypt(req);
		231	if (err == -EINPROGRESS \|\|
		232	(err == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
		233	return;
182		234
183	lcldesc.tfm = ctx->child;	235	out:
184	lcldesc.info = iv;	236	skcipher_request_complete(req, err);
185	lcldesc.flags = desc->flags;	237	}
186		238
187	/* 1. Decrypt Cn-1 (s) to create Dn (tmp)*/	239	static int crypto_cts_decrypt(struct skcipher_request *req)
188	memset(iv, 0, sizeof(iv));	240	{
189	sg_set_buf(&sgsrc[0], s, bsize);	241	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
190	sg_set_buf(&sgdst[0], tmp, bsize);	242	struct crypto_cts_reqctx *rctx = skcipher_request_ctx(req);
191	err = crypto_blkcipher_decrypt_iv(&lcldesc, sgdst, sgsrc, bsize);	243	struct crypto_cts_ctx *ctx = crypto_skcipher_ctx(tfm);
192	if (err)	244	struct skcipher_request *subreq = &rctx->subreq;
193	return err;	245	int bsize = crypto_skcipher_blocksize(tfm);
194	/* 2. Pad Cn with zeros at the end to create C of length BB */	246	unsigned int nbytes = req->cryptlen;
195	memset(iv, 0, sizeof(iv));	247	int cbc_blocks = (nbytes + bsize - 1) / bsize - 1;
196	memcpy(iv, s + bsize, lastn);	248	unsigned int offset;
197	/* 3. Exclusive-or Dn (tmp) with C (iv) to create Xn (tmp) */	249	u8 *space;
198	crypto_xor(tmp, iv, bsize);	250
199	/* 4. Select the first Ln bytes of Xn (tmp) to create Pn */	251	skcipher_request_set_tfm(subreq, ctx->child);
200	memcpy(d + bsize, tmp, lastn);	252
201		253	if (cbc_blocks <= 0) {
202	/* 5. Append the tail (BB - Ln) bytes of Xn (tmp) to Cn to create En */	254	skcipher_request_set_callback(subreq, req->base.flags,
203	memcpy(s + bsize + lastn, tmp + lastn, bsize - lastn);	255	req->base.complete,
204	/* 6. Decrypt En to create Pn-1 */	256	req->base.data);
205	memzero_explicit(iv, sizeof(iv));	257	skcipher_request_set_crypt(subreq, req->src, req->dst, nbytes,
		258	req->iv);
		259	return crypto_skcipher_decrypt(subreq);
		260	}
206		261
207	sg_set_buf(&sgsrc[0], s + bsize, bsize);	262	skcipher_request_set_callback(subreq, req->base.flags,
208	sg_set_buf(&sgdst[0], d, bsize);	263	crypto_cts_decrypt_done, req);
209	err = crypto_blkcipher_decrypt_iv(&lcldesc, sgdst, sgsrc, bsize);
210		264
211	/* XOR with previous block */	265	space = crypto_cts_reqctx_space(req);
212	crypto_xor(d, desc->info, bsize);
213		266
214	scatterwalk_map_and_copy(d, dst, offset, nbytes, 1);	267	offset = cbc_blocks * bsize;
		268	rctx->offset = offset;
215		269
216	memcpy(desc->info, s, bsize);	270	if (cbc_blocks <= 1)
217	return err;	271	memcpy(space, req->iv, bsize);
218	}	272	else
		273	scatterwalk_map_and_copy(space, req->src, offset - 2 * bsize,
		274	bsize, 0);
219		275
220	static int crypto_cts_decrypt(struct blkcipher_desc *desc,	276	skcipher_request_set_crypt(subreq, req->src, req->dst,
221	struct scatterlist dst, struct scatterlist src,	277	offset, req->iv);
222	unsigned int nbytes)
223	{
224	struct crypto_cts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
225	int bsize = crypto_blkcipher_blocksize(desc->tfm);
226	int tot_blocks = (nbytes + bsize - 1) / bsize;
227	int cbc_blocks = tot_blocks > 2 ? tot_blocks - 2 : 0;
228	struct blkcipher_desc lcldesc;
229	int err;
230		278
231	lcldesc.tfm = ctx->child;	279	return crypto_skcipher_decrypt(subreq) ?:
232	lcldesc.info = desc->info;	280	cts_cbc_decrypt(req);
233	lcldesc.flags = desc->flags;
234
235	if (tot_blocks == 1) {
236	err = crypto_blkcipher_decrypt_iv(&lcldesc, dst, src, bsize);
237	} else if (nbytes <= bsize * 2) {
238	err = cts_cbc_decrypt(ctx, desc, dst, src, 0, nbytes);
239	} else {
240	/* do normal function for tot_blocks - 2 */
241	err = crypto_blkcipher_decrypt_iv(&lcldesc, dst, src,
242	cbc_blocks * bsize);
243	if (err == 0) {
244	/* do cts for final two blocks */
245	err = cts_cbc_decrypt(ctx, desc, dst, src,
246	cbc_blocks * bsize,
247	nbytes - (cbc_blocks * bsize));
248	}
249	}
250	return err;
251	}	281	}
252		282
253	static int crypto_cts_init_tfm(struct crypto_tfm *tfm)	283	static int crypto_cts_init_tfm(struct crypto_skcipher *tfm)
254	{	284	{
255	struct crypto_instance inst = (void )tfm->__crt_alg;	285	struct skcipher_instance *inst = skcipher_alg_instance(tfm);
256	struct crypto_spawn *spawn = crypto_instance_ctx(inst);	286	struct crypto_skcipher_spawn *spawn = skcipher_instance_ctx(inst);
257	struct crypto_cts_ctx *ctx = crypto_tfm_ctx(tfm);	287	struct crypto_cts_ctx *ctx = crypto_skcipher_ctx(tfm);
258	struct crypto_blkcipher *cipher;	288	struct crypto_skcipher *cipher;
259		289	unsigned reqsize;
260	cipher = crypto_spawn_blkcipher(spawn);	290	unsigned bsize;
		291	unsigned align;
		292
		293	cipher = crypto_spawn_skcipher2(spawn);
261	if (IS_ERR(cipher))	294	if (IS_ERR(cipher))
262	return PTR_ERR(cipher);	295	return PTR_ERR(cipher);
263		296
264	ctx->child = cipher;	297	ctx->child = cipher;
		298
		299	align = crypto_skcipher_alignmask(tfm);
		300	bsize = crypto_skcipher_blocksize(cipher);
		301	reqsize = ALIGN(sizeof(struct crypto_cts_reqctx) +
		302	crypto_skcipher_reqsize(cipher),
		303	crypto_tfm_ctx_alignment()) +
		304	(align & ~(crypto_tfm_ctx_alignment() - 1)) + bsize;
		305
		306	crypto_skcipher_set_reqsize(tfm, reqsize);
		307
265	return 0;	308	return 0;
266	}	309	}
267		310
268	static void crypto_cts_exit_tfm(struct crypto_tfm *tfm)	311	static void crypto_cts_exit_tfm(struct crypto_skcipher *tfm)
269	{	312	{
270	struct crypto_cts_ctx *ctx = crypto_tfm_ctx(tfm);	313	struct crypto_cts_ctx *ctx = crypto_skcipher_ctx(tfm);
271	crypto_free_blkcipher(ctx->child);	314
		315	crypto_free_skcipher(ctx->child);
272	}	316	}
273		317
274	static struct crypto_instance crypto_cts_alloc(struct rtattr *tb)	318	static void crypto_cts_free(struct skcipher_instance *inst)
275	{	319	{
276	struct crypto_instance *inst;	320	crypto_drop_skcipher(skcipher_instance_ctx(inst));
277	struct crypto_alg *alg;	321	kfree(inst);
		322	}
		323
		324	static int crypto_cts_create(struct crypto_template tmpl, struct rtattr *tb)
		325	{
		326	struct crypto_skcipher_spawn *spawn;
		327	struct skcipher_instance *inst;
		328	struct crypto_attr_type *algt;
		329	struct skcipher_alg *alg;
		330	const char *cipher_name;
278	int err;	331	int err;
279		332
280	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER);	333	algt = crypto_get_attr_type(tb);
		334	if (IS_ERR(algt))
		335	return PTR_ERR(algt);
		336
		337	if ((algt->type ^ CRYPTO_ALG_TYPE_SKCIPHER) & algt->mask)
		338	return -EINVAL;
		339
		340	cipher_name = crypto_attr_alg_name(tb[1]);
		341	if (IS_ERR(cipher_name))
		342	return PTR_ERR(cipher_name);
		343
		344	inst = kzalloc(sizeof(inst) + sizeof(spawn), GFP_KERNEL);
		345	if (!inst)
		346	return -ENOMEM;
		347
		348	spawn = skcipher_instance_ctx(inst);
		349
		350	crypto_set_skcipher_spawn(spawn, skcipher_crypto_instance(inst));
		351	err = crypto_grab_skcipher2(spawn, cipher_name, 0,
		352	crypto_requires_sync(algt->type,
		353	algt->mask));
281	if (err)	354	if (err)
282	return ERR_PTR(err);	355	goto err_free_inst;
283		356
284	alg = crypto_attr_alg(tb[1], CRYPTO_ALG_TYPE_BLKCIPHER,	357	alg = crypto_spawn_skcipher_alg(spawn);
285	CRYPTO_ALG_TYPE_MASK);
286	if (IS_ERR(alg))
287	return ERR_CAST(alg);
288		358
289	inst = ERR_PTR(-EINVAL);	359	err = -EINVAL;
290	if (!is_power_of_2(alg->cra_blocksize))	360	if (crypto_skcipher_alg_ivsize(alg) != alg->base.cra_blocksize)
291	goto out_put_alg;	361	goto err_drop_spawn;
292		362
293	if (strncmp(alg->cra_name, "cbc(", 4))	363	if (strncmp(alg->base.cra_name, "cbc(", 4))
294	goto out_put_alg;	364	goto err_drop_spawn;
295		365
296	inst = crypto_alloc_instance("cts", alg);	366	err = crypto_inst_setname(skcipher_crypto_instance(inst), "cts",
297	if (IS_ERR(inst))	367	&alg->base);
298	goto out_put_alg;	368	if (err)
		369	goto err_drop_spawn;
299		370
300	inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER;	371	inst->alg.base.cra_flags = alg->base.cra_flags & CRYPTO_ALG_ASYNC;
301	inst->alg.cra_priority = alg->cra_priority;	372	inst->alg.base.cra_priority = alg->base.cra_priority;
302	inst->alg.cra_blocksize = alg->cra_blocksize;	373	inst->alg.base.cra_blocksize = alg->base.cra_blocksize;
303	inst->alg.cra_alignmask = alg->cra_alignmask;	374	inst->alg.base.cra_alignmask = alg->base.cra_alignmask;
304	inst->alg.cra_type = &crypto_blkcipher_type;
305		375
306	/* We access the data as u32s when xoring. */	376	/* We access the data as u32s when xoring. */
307	inst->alg.cra_alignmask \|= __alignof__(u32) - 1;	377	inst->alg.base.cra_alignmask \|= __alignof__(u32) - 1;
308		378
309	inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize;	379	inst->alg.ivsize = alg->base.cra_blocksize;
310	inst->alg.cra_blkcipher.min_keysize = alg->cra_blkcipher.min_keysize;	380	inst->alg.chunksize = crypto_skcipher_alg_chunksize(alg);
311	inst->alg.cra_blkcipher.max_keysize = alg->cra_blkcipher.max_keysize;	381	inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg);
		382	inst->alg.max_keysize = crypto_skcipher_alg_max_keysize(alg);
312		383
313	inst->alg.cra_ctxsize = sizeof(struct crypto_cts_ctx);	384	inst->alg.base.cra_ctxsize = sizeof(struct crypto_cts_ctx);
314		385
315	inst->alg.cra_init = crypto_cts_init_tfm;	386	inst->alg.init = crypto_cts_init_tfm;
316	inst->alg.cra_exit = crypto_cts_exit_tfm;	387	inst->alg.exit = crypto_cts_exit_tfm;
317		388
318	inst->alg.cra_blkcipher.setkey = crypto_cts_setkey;	389	inst->alg.setkey = crypto_cts_setkey;
319	inst->alg.cra_blkcipher.encrypt = crypto_cts_encrypt;	390	inst->alg.encrypt = crypto_cts_encrypt;
320	inst->alg.cra_blkcipher.decrypt = crypto_cts_decrypt;	391	inst->alg.decrypt = crypto_cts_decrypt;
321		392
322	out_put_alg:	393	inst->free = crypto_cts_free;
323	crypto_mod_put(alg);
324	return inst;
325	}
326		394
327	static void crypto_cts_free(struct crypto_instance *inst)	395	err = skcipher_register_instance(tmpl, inst);
328	{	396	if (err)
329	crypto_drop_spawn(crypto_instance_ctx(inst));	397	goto err_drop_spawn;
		398
		399	out:
		400	return err;
		401
		402	err_drop_spawn:
		403	crypto_drop_skcipher(spawn);
		404	err_free_inst:
330	kfree(inst);	405	kfree(inst);
		406	goto out;
331	}	407	}
332		408
333	static struct crypto_template crypto_cts_tmpl = {	409	static struct crypto_template crypto_cts_tmpl = {
334	.name = "cts",	410	.name = "cts",
335	.alloc = crypto_cts_alloc,	411	.create = crypto_cts_create,
336	.free = crypto_cts_free,
337	.module = THIS_MODULE,	412	.module = THIS_MODULE,
338	};	413	};
339		414

crypto/dh.c+189−0

@@ -0,0 +1,189 @@
	1	/* Diffie-Hellman Key Agreement Method [RFC2631]
	2	*
	3	* Copyright (c) 2016, Intel Corporation
	4	* Authors: Salvatore Benedetto <salvatore.benedetto@intel.com>
	5	*
	6	* This program is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU General Public Licence
	8	* as published by the Free Software Foundation; either version
	9	* 2 of the Licence, or (at your option) any later version.
	10	*/
	11
	12	#include <linux/module.h>
	13	#include <crypto/internal/kpp.h>
	14	#include <crypto/kpp.h>
	15	#include <crypto/dh.h>
	16	#include <linux/mpi.h>
	17
	18	struct dh_ctx {
	19	MPI p;
	20	MPI g;
	21	MPI xa;
	22	};
	23
	24	static inline void dh_clear_params(struct dh_ctx *ctx)
	25	{
	26	mpi_free(ctx->p);
	27	mpi_free(ctx->g);
	28	ctx->p = NULL;
	29	ctx->g = NULL;
	30	}
	31
	32	static void dh_free_ctx(struct dh_ctx *ctx)
	33	{
	34	dh_clear_params(ctx);
	35	mpi_free(ctx->xa);
	36	ctx->xa = NULL;
	37	}
	38
	39	/*
	40	* If base is g we compute the public key
	41	* ya = g^xa mod p; [RFC2631 sec 2.1.1]
	42	* else if base if the counterpart public key we compute the shared secret
	43	* ZZ = yb^xa mod p; [RFC2631 sec 2.1.1]
	44	*/
	45	static int _compute_val(const struct dh_ctx *ctx, MPI base, MPI val)
	46	{
	47	/* val = base^xa mod p */
	48	return mpi_powm(val, base, ctx->xa, ctx->p);
	49	}
	50
	51	static inline struct dh_ctx dh_get_ctx(struct crypto_kpp tfm)
	52	{
	53	return kpp_tfm_ctx(tfm);
	54	}
	55
	56	static int dh_check_params_length(unsigned int p_len)
	57	{
	58	return (p_len < 1536) ? -EINVAL : 0;
	59	}
	60
	61	static int dh_set_params(struct dh_ctx ctx, struct dh params)
	62	{
	63	if (unlikely(!params->p \|\| !params->g))
	64	return -EINVAL;
	65
	66	if (dh_check_params_length(params->p_size << 3))
	67	return -EINVAL;
	68
	69	ctx->p = mpi_read_raw_data(params->p, params->p_size);
	70	if (!ctx->p)
	71	return -EINVAL;
	72
	73	ctx->g = mpi_read_raw_data(params->g, params->g_size);
	74	if (!ctx->g) {
	75	mpi_free(ctx->p);
	76	return -EINVAL;
	77	}
	78
	79	return 0;
	80	}
	81
	82	static int dh_set_secret(struct crypto_kpp tfm, void buf, unsigned int len)
	83	{
	84	struct dh_ctx *ctx = dh_get_ctx(tfm);
	85	struct dh params;
	86
	87	if (crypto_dh_decode_key(buf, len, &params) < 0)
	88	return -EINVAL;
	89
	90	if (dh_set_params(ctx, &params) < 0)
	91	return -EINVAL;
	92
	93	ctx->xa = mpi_read_raw_data(params.key, params.key_size);
	94	if (!ctx->xa) {
	95	dh_clear_params(ctx);
	96	return -EINVAL;
	97	}
	98
	99	return 0;
	100	}
	101
	102	static int dh_compute_value(struct kpp_request *req)
	103	{
	104	struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
	105	struct dh_ctx *ctx = dh_get_ctx(tfm);
	106	MPI base, val = mpi_alloc(0);
	107	int ret = 0;
	108	int sign;
	109
	110	if (!val)
	111	return -ENOMEM;
	112
	113	if (unlikely(!ctx->xa)) {
	114	ret = -EINVAL;
	115	goto err_free_val;
	116	}
	117
	118	if (req->src) {
	119	base = mpi_read_raw_from_sgl(req->src, req->src_len);
	120	if (!base) {
	121	ret = EINVAL;
	122	goto err_free_val;
	123	}
	124	} else {
	125	base = ctx->g;
	126	}
	127
	128	ret = _compute_val(ctx, base, val);
	129	if (ret)
	130	goto err_free_base;
	131
	132	ret = mpi_write_to_sgl(val, req->dst, req->dst_len, &sign);
	133	if (ret)
	134	goto err_free_base;
	135
	136	if (sign < 0)
	137	ret = -EBADMSG;
	138	err_free_base:
	139	if (req->src)
	140	mpi_free(base);
	141	err_free_val:
	142	mpi_free(val);
	143	return ret;
	144	}
	145
	146	static int dh_max_size(struct crypto_kpp *tfm)
	147	{
	148	struct dh_ctx *ctx = dh_get_ctx(tfm);
	149
	150	return mpi_get_size(ctx->p);
	151	}
	152
	153	static void dh_exit_tfm(struct crypto_kpp *tfm)
	154	{
	155	struct dh_ctx *ctx = dh_get_ctx(tfm);
	156
	157	dh_free_ctx(ctx);
	158	}
	159
	160	static struct kpp_alg dh = {
	161	.set_secret = dh_set_secret,
	162	.generate_public_key = dh_compute_value,
	163	.compute_shared_secret = dh_compute_value,
	164	.max_size = dh_max_size,
	165	.exit = dh_exit_tfm,
	166	.base = {
	167	.cra_name = "dh",
	168	.cra_driver_name = "dh-generic",
	169	.cra_priority = 100,
	170	.cra_module = THIS_MODULE,
	171	.cra_ctxsize = sizeof(struct dh_ctx),
	172	},
	173	};
	174
	175	static int dh_init(void)
	176	{
	177	return crypto_register_kpp(&dh);
	178	}
	179
	180	static void dh_exit(void)
	181	{
	182	crypto_unregister_kpp(&dh);
	183	}
	184
	185	module_init(dh_init);
	186	module_exit(dh_exit);
	187	MODULE_ALIAS_CRYPTO("dh");
	188	MODULE_LICENSE("GPL");
	189	MODULE_DESCRIPTION("DH generic algorithm");

crypto/dh_helper.c+95−0

@@ -0,0 +1,95 @@
	1	/*
	2	* Copyright (c) 2016, Intel Corporation
	3	* Authors: Salvatore Benedetto <salvatore.benedetto@intel.com>
	4	*
	5	* This program is free software; you can redistribute it and/or
	6	* modify it under the terms of the GNU General Public Licence
	7	* as published by the Free Software Foundation; either version
	8	* 2 of the Licence, or (at your option) any later version.
	9	*/
	10	#include <linux/kernel.h>
	11	#include <linux/export.h>
	12	#include <linux/err.h>
	13	#include <linux/string.h>
	14	#include <crypto/dh.h>
	15	#include <crypto/kpp.h>
	16
	17	#define DH_KPP_SECRET_MIN_SIZE (sizeof(struct kpp_secret) + 3 * sizeof(int))
	18
	19	static inline u8 dh_pack_data(void dst, const void *src, size_t size)
	20	{
	21	memcpy(dst, src, size);
	22	return dst + size;
	23	}
	24
	25	static inline const u8 dh_unpack_data(void dst, const void *src, size_t size)
	26	{
	27	memcpy(dst, src, size);
	28	return src + size;
	29	}
	30
	31	static inline int dh_data_size(const struct dh *p)
	32	{
	33	return p->key_size + p->p_size + p->g_size;
	34	}
	35
	36	int crypto_dh_key_len(const struct dh *p)
	37	{
	38	return DH_KPP_SECRET_MIN_SIZE + dh_data_size(p);
	39	}
	40	EXPORT_SYMBOL_GPL(crypto_dh_key_len);
	41
	42	int crypto_dh_encode_key(char buf, unsigned int len, const struct dh params)
	43	{
	44	u8 *ptr = buf;
	45	struct kpp_secret secret = {
	46	.type = CRYPTO_KPP_SECRET_TYPE_DH,
	47	.len = len
	48	};
	49
	50	if (unlikely(!buf))
	51	return -EINVAL;
	52
	53	if (len != crypto_dh_key_len(params))
	54	return -EINVAL;
	55
	56	ptr = dh_pack_data(ptr, &secret, sizeof(secret));
	57	ptr = dh_pack_data(ptr, &params->key_size, sizeof(params->key_size));
	58	ptr = dh_pack_data(ptr, &params->p_size, sizeof(params->p_size));
	59	ptr = dh_pack_data(ptr, &params->g_size, sizeof(params->g_size));
	60	ptr = dh_pack_data(ptr, params->key, params->key_size);
	61	ptr = dh_pack_data(ptr, params->p, params->p_size);
	62	dh_pack_data(ptr, params->g, params->g_size);
	63
	64	return 0;
	65	}
	66	EXPORT_SYMBOL_GPL(crypto_dh_encode_key);
	67
	68	int crypto_dh_decode_key(const char buf, unsigned int len, struct dh params)
	69	{
	70	const u8 *ptr = buf;
	71	struct kpp_secret secret;
	72
	73	if (unlikely(!buf \|\| len < DH_KPP_SECRET_MIN_SIZE))
	74	return -EINVAL;
	75
	76	ptr = dh_unpack_data(&secret, ptr, sizeof(secret));
	77	if (secret.type != CRYPTO_KPP_SECRET_TYPE_DH)
	78	return -EINVAL;
	79
	80	ptr = dh_unpack_data(&params->key_size, ptr, sizeof(params->key_size));
	81	ptr = dh_unpack_data(&params->p_size, ptr, sizeof(params->p_size));
	82	ptr = dh_unpack_data(&params->g_size, ptr, sizeof(params->g_size));
	83	if (secret.len != crypto_dh_key_len(params))
	84	return -EINVAL;
	85
	86	/* Don't allocate memory. Set pointers to data within
	87	* the given buffer
	88	*/
	89	params->key = (void *)ptr;
	90	params->p = (void *)(ptr + params->key_size);
	91	params->g = (void *)(ptr + params->key_size + params->p_size);
	92
	93	return 0;
	94	}
	95	EXPORT_SYMBOL_GPL(crypto_dh_decode_key);

crypto/drbg.c+184−85

@@ -252,10 +252,16 @@ MODULE_ALIAS_CRYPTO("drbg_nopr_ctr_aes192"); · 251 unmodified lines
252	MODULE_ALIAS_CRYPTO("drbg_pr_ctr_aes128");	252	MODULE_ALIAS_CRYPTO("drbg_pr_ctr_aes128");
253	MODULE_ALIAS_CRYPTO("drbg_nopr_ctr_aes128");	253	MODULE_ALIAS_CRYPTO("drbg_nopr_ctr_aes128");
254		254
255	static int drbg_kcapi_sym(struct drbg_state drbg, const unsigned char key,	255	static void drbg_kcapi_symsetkey(struct drbg_state *drbg,
256	unsigned char outval, const struct drbg_string in);	256	const unsigned char *key);
		257	static int drbg_kcapi_sym(struct drbg_state drbg, unsigned char outval,
		258	const struct drbg_string *in);
257	static int drbg_init_sym_kernel(struct drbg_state *drbg);	259	static int drbg_init_sym_kernel(struct drbg_state *drbg);
258	static int drbg_fini_sym_kernel(struct drbg_state *drbg);	260	static int drbg_fini_sym_kernel(struct drbg_state *drbg);
		261	static int drbg_kcapi_sym_ctr(struct drbg_state *drbg,
		262	u8 *inbuf, u32 inbuflen,
		263	u8 *outbuf, u32 outlen);
		264	#define DRBG_CTR_NULL_LEN 128
259		265
260	/* BCC function for CTR DRBG as defined in 10.4.3 */	266	/* BCC function for CTR DRBG as defined in 10.4.3 */
261	static int drbg_ctr_bcc(struct drbg_state *drbg,	267	static int drbg_ctr_bcc(struct drbg_state *drbg,
@@ -270,6 +276,7 @@ static int drbg_ctr_bcc(struct drbg_state *drbg, · 8 unmodified lines
270	drbg_string_fill(&data, out, drbg_blocklen(drbg));	276	drbg_string_fill(&data, out, drbg_blocklen(drbg));
271		277
272	/* 10.4.3 step 2 / 4 */	278	/* 10.4.3 step 2 / 4 */
		279	drbg_kcapi_symsetkey(drbg, key);
273	list_for_each_entry(curr, in, list) {	280	list_for_each_entry(curr, in, list) {
274	const unsigned char *pos = curr->buf;	281	const unsigned char *pos = curr->buf;
275	size_t len = curr->len;	282	size_t len = curr->len;
@@ -278,7 +285,7 @@ static int drbg_ctr_bcc(struct drbg_state *drbg, · 2 unmodified lines
278	/* 10.4.3 step 4.2 */	285	/* 10.4.3 step 4.2 */
279	if (drbg_blocklen(drbg) == cnt) {	286	if (drbg_blocklen(drbg) == cnt) {
280	cnt = 0;	287	cnt = 0;
281	ret = drbg_kcapi_sym(drbg, key, out, &data);	288	ret = drbg_kcapi_sym(drbg, out, &data);
282	if (ret)	289	if (ret)
283	return ret;	290	return ret;
284	}	291	}
@@ -290,7 +297,7 @@ static int drbg_ctr_bcc(struct drbg_state *drbg, · 5 unmodified lines
290	}	297	}
291	/* 10.4.3 step 4.2 for last block */	298	/* 10.4.3 step 4.2 for last block */
292	if (cnt)	299	if (cnt)
293	ret = drbg_kcapi_sym(drbg, key, out, &data);	300	ret = drbg_kcapi_sym(drbg, out, &data);
294		301
295	return ret;	302	return ret;
296	}	303	}
@@ -425,6 +432,7 @@ static int drbg_ctr_df(struct drbg_state *drbg, · 128 unmodified lines
425	/* 10.4.2 step 12: overwriting of outval is implemented in next step */	432	/* 10.4.2 step 12: overwriting of outval is implemented in next step */
426		433
427	/* 10.4.2 step 13 */	434	/* 10.4.2 step 13 */
		435	drbg_kcapi_symsetkey(drbg, temp);
428	while (generated_len < bytes_to_return) {	436	while (generated_len < bytes_to_return) {
429	short blocklen = 0;	437	short blocklen = 0;
430	/*	438	/*
@@ -432,7 +440,7 @@ static int drbg_ctr_df(struct drbg_state *drbg, · 1 unmodified line
432	* implicit as the key is only drbg_blocklen in size based on	440	* implicit as the key is only drbg_blocklen in size based on
433	* the implementation of the cipher function callback	441	* the implementation of the cipher function callback
434	*/	442	*/
435	ret = drbg_kcapi_sym(drbg, temp, X, &cipherin);	443	ret = drbg_kcapi_sym(drbg, X, &cipherin);
436	if (ret)	444	if (ret)
437	goto out;	445	goto out;
438	blocklen = (drbg_blocklen(drbg) <	446	blocklen = (drbg_blocklen(drbg) <
@@ -476,49 +484,47 @@ static int drbg_ctr_update(struct drbg_state drbg, struct list_head seed, · 37 unmodified lines
476	unsigned char *temp = drbg->scratchpad;	484	unsigned char *temp = drbg->scratchpad;
477	unsigned char *df_data = drbg->scratchpad + drbg_statelen(drbg) +	485	unsigned char *df_data = drbg->scratchpad + drbg_statelen(drbg) +
478	drbg_blocklen(drbg);	486	drbg_blocklen(drbg);
479	unsigned char temp_p, df_data_p; /* pointer to iterate over buffers */
480	unsigned int len = 0;
481	struct drbg_string cipherin;
482		487
483	if (3 > reseed)	488	if (3 > reseed)
484	memset(df_data, 0, drbg_statelen(drbg));	489	memset(df_data, 0, drbg_statelen(drbg));
485		490
486	/* 10.2.1.3.2 step 2 and 10.2.1.4.2 step 2 */	491	if (!reseed) {
487	if (seed) {	492	/*
488	ret = drbg_ctr_df(drbg, df_data, drbg_statelen(drbg), seed);	493	* The DRBG uses the CTR mode of the underlying AES cipher. The
		494	* CTR mode increments the counter value after the AES operation
		495	* but SP800-90A requires that the counter is incremented before
		496	* the AES operation. Hence, we increment it at the time we set
		497	* it by one.
		498	*/
		499	crypto_inc(drbg->V, drbg_blocklen(drbg));
		500
		501	ret = crypto_skcipher_setkey(drbg->ctr_handle, drbg->C,
		502	drbg_keylen(drbg));
489	if (ret)	503	if (ret)
490	goto out;	504	goto out;
491	}	505	}
492		506
493	drbg_string_fill(&cipherin, drbg->V, drbg_blocklen(drbg));	507	/* 10.2.1.3.2 step 2 and 10.2.1.4.2 step 2 */
494	/*	508	if (seed) {
495	* 10.2.1.3.2 steps 2 and 3 are already covered as the allocation	509	ret = drbg_ctr_df(drbg, df_data, drbg_statelen(drbg), seed);
496	* zeroizes all memory during initialization
497	*/
498	while (len < (drbg_statelen(drbg))) {
499	/* 10.2.1.2 step 2.1 */
500	crypto_inc(drbg->V, drbg_blocklen(drbg));
501	/*
502	* 10.2.1.2 step 2.2 */
503	ret = drbg_kcapi_sym(drbg, drbg->C, temp + len, &cipherin);
504	if (ret)	510	if (ret)
505	goto out;	511	goto out;
506	/* 10.2.1.2 step 2.3 and 3 */
507	len += drbg_blocklen(drbg);
508	}	512	}
509		513
510	/* 10.2.1.2 step 4 */	514	ret = drbg_kcapi_sym_ctr(drbg, df_data, drbg_statelen(drbg),
511	temp_p = temp;	515	temp, drbg_statelen(drbg));
512	df_data_p = df_data;	516	if (ret)
513	for (len = 0; len < drbg_statelen(drbg); len++) {	517	return ret;
514	temp_p ^= df_data_p;
515	df_data_p++; temp_p++;
516	}
517		518
518	/* 10.2.1.2 step 5 */	519	/* 10.2.1.2 step 5 */
519	memcpy(drbg->C, temp, drbg_keylen(drbg));	520	ret = crypto_skcipher_setkey(drbg->ctr_handle, temp,
		521	drbg_keylen(drbg));
		522	if (ret)
		523	goto out;
520	/* 10.2.1.2 step 6 */	524	/* 10.2.1.2 step 6 */
521	memcpy(drbg->V, temp + drbg_keylen(drbg), drbg_blocklen(drbg));	525	memcpy(drbg->V, temp + drbg_keylen(drbg), drbg_blocklen(drbg));
		526	/* See above: increment counter by one to compensate timing of CTR op */
		527	crypto_inc(drbg->V, drbg_blocklen(drbg));
522	ret = 0;	528	ret = 0;
523		529
524	out:	530	out:
@@ -537,9 +543,8 @@ static int drbg_ctr_generate(struct drbg_state *drbg, · 12 unmodified lines
537	unsigned char *buf, unsigned int buflen,	543	unsigned char *buf, unsigned int buflen,
538	struct list_head *addtl)	544	struct list_head *addtl)
539	{	545	{
540	int len = 0;	546	int ret;
541	int ret = 0;	547	int len = min_t(int, buflen, INT_MAX);
542	struct drbg_string data;
543		548
544	/* 10.2.1.5.2 step 2 */	549	/* 10.2.1.5.2 step 2 */
545	if (addtl && !list_empty(addtl)) {	550	if (addtl && !list_empty(addtl)) {
@@ -549,33 +554,16 @@ static int drbg_ctr_generate(struct drbg_state *drbg, · 3 unmodified lines
549	}	554	}
550		555
551	/* 10.2.1.5.2 step 4.1 */	556	/* 10.2.1.5.2 step 4.1 */
552	crypto_inc(drbg->V, drbg_blocklen(drbg));	557	ret = drbg_kcapi_sym_ctr(drbg, drbg->ctr_null_value, DRBG_CTR_NULL_LEN,
553	drbg_string_fill(&data, drbg->V, drbg_blocklen(drbg));	558	buf, len);
554	while (len < buflen) {	559	if (ret)
555	int outlen = 0;	560	return ret;
556	/* 10.2.1.5.2 step 4.2 */
557	ret = drbg_kcapi_sym(drbg, drbg->C, drbg->scratchpad, &data);
558	if (ret) {
559	len = ret;
560	goto out;
561	}
562	outlen = (drbg_blocklen(drbg) < (buflen - len)) ?
563	drbg_blocklen(drbg) : (buflen - len);
564	/* 10.2.1.5.2 step 4.3 */
565	memcpy(buf + len, drbg->scratchpad, outlen);
566	len += outlen;
567	/* 10.2.1.5.2 step 6 */
568	if (len < buflen)
569	crypto_inc(drbg->V, drbg_blocklen(drbg));
570	}
571		561
572	/* 10.2.1.5.2 step 6 */	562	/* 10.2.1.5.2 step 6 */
573	ret = drbg_ctr_update(drbg, NULL, 3);	563	ret = drbg_ctr_update(drbg, NULL, 3);
574	if (ret)	564	if (ret)
575	len = ret;	565	len = ret;
576		566
577	out:
578	memset(drbg->scratchpad, 0, drbg_blocklen(drbg));
579	return len;	567	return len;
580	}	568	}
581		569
@@ -1145,11 +1133,11 @@ static inline void drbg_dealloc_state(struct drbg_state *drbg) · 563 unmodified lines
1145	if (!drbg)	1133	if (!drbg)
1146	return;	1134	return;
1147	kzfree(drbg->V);	1135	kzfree(drbg->V);
1148	drbg->V = NULL;	1136	drbg->Vbuf = NULL;
1149	kzfree(drbg->C);	1137	kzfree(drbg->C);
1150	drbg->C = NULL;	1138	drbg->Cbuf = NULL;
1151	kzfree(drbg->scratchpad);	1139	kzfree(drbg->scratchpadbuf);
1152	drbg->scratchpad = NULL;	1140	drbg->scratchpadbuf = NULL;
1153	drbg->reseed_ctr = 0;	1141	drbg->reseed_ctr = 0;
1154	drbg->d_ops = NULL;	1142	drbg->d_ops = NULL;
1155	drbg->core = NULL;	1143	drbg->core = NULL;
@@ -1185,12 +1173,18 @@ static inline int drbg_alloc_state(struct drbg_state *drbg) · 29 unmodified lines
1185	goto err;	1173	goto err;
1186	}	1174	}
1187		1175
1188	drbg->V = kmalloc(drbg_statelen(drbg), GFP_KERNEL);	1176	ret = drbg->d_ops->crypto_init(drbg);
1189	if (!drbg->V)	1177	if (ret < 0)
1190	goto err;
1191	drbg->C = kmalloc(drbg_statelen(drbg), GFP_KERNEL);
1192	if (!drbg->C)
1193	goto err;	1178	goto err;
		1179
		1180	drbg->Vbuf = kmalloc(drbg_statelen(drbg) + ret, GFP_KERNEL);
		1181	if (!drbg->Vbuf)
		1182	goto fini;
		1183	drbg->V = PTR_ALIGN(drbg->Vbuf, ret + 1);
		1184	drbg->Cbuf = kmalloc(drbg_statelen(drbg) + ret, GFP_KERNEL);
		1185	if (!drbg->Cbuf)
		1186	goto fini;
		1187	drbg->C = PTR_ALIGN(drbg->Cbuf, ret + 1);
1194	/* scratchpad is only generated for CTR and Hash */	1188	/* scratchpad is only generated for CTR and Hash */
1195	if (drbg->core->flags & DRBG_HMAC)	1189	if (drbg->core->flags & DRBG_HMAC)
1196	sb_size = 0;	1190	sb_size = 0;
@@ -1204,13 +1198,16 @@ static inline int drbg_alloc_state(struct drbg_state *drbg) · 7 unmodified lines
1204	sb_size = drbg_statelen(drbg) + drbg_blocklen(drbg);	1198	sb_size = drbg_statelen(drbg) + drbg_blocklen(drbg);
1205		1199
1206	if (0 < sb_size) {	1200	if (0 < sb_size) {
1207	drbg->scratchpad = kzalloc(sb_size, GFP_KERNEL);	1201	drbg->scratchpadbuf = kzalloc(sb_size + ret, GFP_KERNEL);
1208	if (!drbg->scratchpad)	1202	if (!drbg->scratchpadbuf)
1209	goto err;	1203	goto fini;
		1204	drbg->scratchpad = PTR_ALIGN(drbg->scratchpadbuf, ret + 1);
1210	}	1205	}
1211		1206
1212	return 0;	1207	return 0;
1213		1208
		1209	fini:
		1210	drbg->d_ops->crypto_fini(drbg);
1214	err:	1211	err:
1215	drbg_dealloc_state(drbg);	1212	drbg_dealloc_state(drbg);
1216	return ret;	1213	return ret;
@@ -1478,10 +1475,6 @@ static int drbg_instantiate(struct drbg_state drbg, struct drbg_string pers, · 261 unmodified lines
1478	if (ret)	1475	if (ret)
1479	goto unlock;	1476	goto unlock;
1480		1477
1481	ret = -EFAULT;
1482	if (drbg->d_ops->crypto_init(drbg))
1483	goto err;
1484
1485	ret = drbg_prepare_hrng(drbg);	1478	ret = drbg_prepare_hrng(drbg);
1486	if (ret)	1479	if (ret)
1487	goto free_everything;	1480	goto free_everything;
@@ -1505,8 +1498,6 @@ static int drbg_instantiate(struct drbg_state drbg, struct drbg_string pers, · 17 unmodified lines
1505	mutex_unlock(&drbg->drbg_mutex);	1498	mutex_unlock(&drbg->drbg_mutex);
1506	return ret;	1499	return ret;
1507		1500
1508	err:
1509	drbg_dealloc_state(drbg);
1510	unlock:	1501	unlock:
1511	mutex_unlock(&drbg->drbg_mutex);	1502	mutex_unlock(&drbg->drbg_mutex);
1512	return ret;	1503	return ret;
@@ -1591,7 +1582,8 @@ static int drbg_init_hash_kernel(struct drbg_state *drbg) · 78 unmodified lines
1591	sdesc->shash.tfm = tfm;	1582	sdesc->shash.tfm = tfm;
1592	sdesc->shash.flags = 0;	1583	sdesc->shash.flags = 0;
1593	drbg->priv_data = sdesc;	1584	drbg->priv_data = sdesc;
1594	return 0;	1585
		1586	return crypto_shash_alignmask(tfm);
1595	}	1587	}
1596		1588
1597	static int drbg_fini_hash_kernel(struct drbg_state *drbg)	1589	static int drbg_fini_hash_kernel(struct drbg_state *drbg)
@@ -1627,10 +1619,45 @@ static int drbg_kcapi_hash(struct drbg_state drbg, unsigned char outval, · 29 unmodified lines
1627	#endif /* (CONFIG_CRYPTO_DRBG_HASH \|\| CONFIG_CRYPTO_DRBG_HMAC) */	1619	#endif /* (CONFIG_CRYPTO_DRBG_HASH \|\| CONFIG_CRYPTO_DRBG_HMAC) */
1628		1620
1629	#ifdef CONFIG_CRYPTO_DRBG_CTR	1621	#ifdef CONFIG_CRYPTO_DRBG_CTR
		1622	static int drbg_fini_sym_kernel(struct drbg_state *drbg)
		1623	{
		1624	struct crypto_cipher *tfm =
		1625	(struct crypto_cipher *)drbg->priv_data;
		1626	if (tfm)
		1627	crypto_free_cipher(tfm);
		1628	drbg->priv_data = NULL;
		1629
		1630	if (drbg->ctr_handle)
		1631	crypto_free_skcipher(drbg->ctr_handle);
		1632	drbg->ctr_handle = NULL;
		1633
		1634	if (drbg->ctr_req)
		1635	skcipher_request_free(drbg->ctr_req);
		1636	drbg->ctr_req = NULL;
		1637
		1638	kfree(drbg->ctr_null_value_buf);
		1639	drbg->ctr_null_value = NULL;
		1640
		1641	return 0;
		1642	}
		1643
		1644	static void drbg_skcipher_cb(struct crypto_async_request *req, int error)
		1645	{
		1646	struct drbg_state *drbg = req->data;
		1647
		1648	if (error == -EINPROGRESS)
		1649	return;
		1650	drbg->ctr_async_err = error;
		1651	complete(&drbg->ctr_completion);
		1652	}
		1653
1630	static int drbg_init_sym_kernel(struct drbg_state *drbg)	1654	static int drbg_init_sym_kernel(struct drbg_state *drbg)
1631	{	1655	{
1632	int ret = 0;
1633	struct crypto_cipher *tfm;	1656	struct crypto_cipher *tfm;
		1657	struct crypto_skcipher *sk_tfm;
		1658	struct skcipher_request *req;
		1659	unsigned int alignmask;
		1660	char ctr_name[CRYPTO_MAX_ALG_NAME];
1634		1661
1635	tfm = crypto_alloc_cipher(drbg->core->backend_cra_name, 0, 0);	1662	tfm = crypto_alloc_cipher(drbg->core->backend_cra_name, 0, 0);
1636	if (IS_ERR(tfm)) {	1663	if (IS_ERR(tfm)) {
@@ -1640,31 +1667,103 @@ static int drbg_init_sym_kernel(struct drbg_state *drbg) · 3 unmodified lines
1640	}	1667	}
1641	BUG_ON(drbg_blocklen(drbg) != crypto_cipher_blocksize(tfm));	1668	BUG_ON(drbg_blocklen(drbg) != crypto_cipher_blocksize(tfm));
1642	drbg->priv_data = tfm;	1669	drbg->priv_data = tfm;
1643	return ret;	1670
		1671	if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)",
		1672	drbg->core->backend_cra_name) >= CRYPTO_MAX_ALG_NAME) {
		1673	drbg_fini_sym_kernel(drbg);
		1674	return -EINVAL;
		1675	}
		1676	sk_tfm = crypto_alloc_skcipher(ctr_name, 0, 0);
		1677	if (IS_ERR(sk_tfm)) {
		1678	pr_info("DRBG: could not allocate CTR cipher TFM handle: %s\n",
		1679	ctr_name);
		1680	drbg_fini_sym_kernel(drbg);
		1681	return PTR_ERR(sk_tfm);
		1682	}
		1683	drbg->ctr_handle = sk_tfm;
		1684
		1685	req = skcipher_request_alloc(sk_tfm, GFP_KERNEL);
		1686	if (!req) {
		1687	pr_info("DRBG: could not allocate request queue\n");
		1688	drbg_fini_sym_kernel(drbg);
		1689	return -ENOMEM;
		1690	}
		1691	drbg->ctr_req = req;
		1692	skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
		1693	drbg_skcipher_cb, drbg);
		1694
		1695	alignmask = crypto_skcipher_alignmask(sk_tfm);
		1696	drbg->ctr_null_value_buf = kzalloc(DRBG_CTR_NULL_LEN + alignmask,
		1697	GFP_KERNEL);
		1698	if (!drbg->ctr_null_value_buf) {
		1699	drbg_fini_sym_kernel(drbg);
		1700	return -ENOMEM;
		1701	}
		1702	drbg->ctr_null_value = (u8 *)PTR_ALIGN(drbg->ctr_null_value_buf,
		1703	alignmask + 1);
		1704
		1705	return alignmask;
1644	}	1706	}
1645		1707
1646	static int drbg_fini_sym_kernel(struct drbg_state *drbg)	1708	static void drbg_kcapi_symsetkey(struct drbg_state *drbg,
		1709	const unsigned char *key)
1647	{	1710	{
1648	struct crypto_cipher *tfm =	1711	struct crypto_cipher *tfm =
1649	(struct crypto_cipher *)drbg->priv_data;	1712	(struct crypto_cipher *)drbg->priv_data;
1650	if (tfm)	1713
1651	crypto_free_cipher(tfm);	1714	crypto_cipher_setkey(tfm, key, (drbg_keylen(drbg)));
1652	drbg->priv_data = NULL;
1653	return 0;
1654	}	1715	}
1655		1716
1656	static int drbg_kcapi_sym(struct drbg_state drbg, const unsigned char key,	1717	static int drbg_kcapi_sym(struct drbg_state drbg, unsigned char outval,
1657	unsigned char outval, const struct drbg_string in)	1718	const struct drbg_string *in)
1658	{	1719	{
1659	struct crypto_cipher *tfm =	1720	struct crypto_cipher *tfm =
1660	(struct crypto_cipher *)drbg->priv_data;	1721	(struct crypto_cipher *)drbg->priv_data;
1661		1722
1662	crypto_cipher_setkey(tfm, key, (drbg_keylen(drbg)));
1663	/* there is only component in in /	1723	/* there is only component in in /
1664	BUG_ON(in->len < drbg_blocklen(drbg));	1724	BUG_ON(in->len < drbg_blocklen(drbg));
1665	crypto_cipher_encrypt_one(tfm, outval, in->buf);	1725	crypto_cipher_encrypt_one(tfm, outval, in->buf);
1666	return 0;	1726	return 0;
1667	}	1727	}
		1728
		1729	static int drbg_kcapi_sym_ctr(struct drbg_state *drbg,
		1730	u8 *inbuf, u32 inlen,
		1731	u8 *outbuf, u32 outlen)
		1732	{
		1733	struct scatterlist sg_in;
		1734
		1735	sg_init_one(&sg_in, inbuf, inlen);
		1736
		1737	while (outlen) {
		1738	u32 cryptlen = min_t(u32, inlen, outlen);
		1739	struct scatterlist sg_out;
		1740	int ret;
		1741
		1742	sg_init_one(&sg_out, outbuf, cryptlen);
		1743	skcipher_request_set_crypt(drbg->ctr_req, &sg_in, &sg_out,
		1744	cryptlen, drbg->V);
		1745	ret = crypto_skcipher_encrypt(drbg->ctr_req);
		1746	switch (ret) {
		1747	case 0:
		1748	break;
		1749	case -EINPROGRESS:
		1750	case -EBUSY:
		1751	ret = wait_for_completion_interruptible(
		1752	&drbg->ctr_completion);
		1753	if (!ret && !drbg->ctr_async_err) {
		1754	reinit_completion(&drbg->ctr_completion);
		1755	break;
		1756	}
		1757	default:
		1758	return ret;
		1759	}
		1760	init_completion(&drbg->ctr_completion);
		1761
		1762	outlen -= cryptlen;
		1763	}
		1764
		1765	return 0;
		1766	}
1668	#endif /* CONFIG_CRYPTO_DRBG_CTR */	1767	#endif /* CONFIG_CRYPTO_DRBG_CTR */
1669		1768
1670	/***************************************************************	1769	/***************************************************************

crypto/ecc.c+1018−0

@@ -0,0 +1,1018 @@
	1	/*
	2	* Copyright (c) 2013, Kenneth MacKay
	3	* All rights reserved.
	4	*
	5	* Redistribution and use in source and binary forms, with or without
	6	* modification, are permitted provided that the following conditions are
	7	* met:
	8	* * Redistributions of source code must retain the above copyright
	9	* notice, this list of conditions and the following disclaimer.
	10	* * Redistributions in binary form must reproduce the above copyright
	11	* notice, this list of conditions and the following disclaimer in the
	12	* documentation and/or other materials provided with the distribution.
	13	*
	14	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	15	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	16	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	17	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	18	* HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	19	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	20	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	21	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	22	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	23	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	24	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	25	*/
	26
	27	#include <linux/random.h>
	28	#include <linux/slab.h>
	29	#include <linux/swab.h>
	30	#include <linux/fips.h>
	31	#include <crypto/ecdh.h>
	32
	33	#include "ecc.h"
	34	#include "ecc_curve_defs.h"
	35
	36	typedef struct {
	37	u64 m_low;
	38	u64 m_high;
	39	} uint128_t;
	40
	41	static inline const struct ecc_curve *ecc_get_curve(unsigned int curve_id)
	42	{
	43	switch (curve_id) {
	44	/* In FIPS mode only allow P256 and higher */
	45	case ECC_CURVE_NIST_P192:
	46	return fips_enabled ? NULL : &nist_p192;
	47	case ECC_CURVE_NIST_P256:
	48	return &nist_p256;
	49	default:
	50	return NULL;
	51	}
	52	}
	53
	54	static u64 *ecc_alloc_digits_space(unsigned int ndigits)
	55	{
	56	size_t len = ndigits * sizeof(u64);
	57
	58	if (!len)
	59	return NULL;
	60
	61	return kmalloc(len, GFP_KERNEL);
	62	}
	63
	64	static void ecc_free_digits_space(u64 *space)
	65	{
	66	kzfree(space);
	67	}
	68
	69	static struct ecc_point *ecc_alloc_point(unsigned int ndigits)
	70	{
	71	struct ecc_point p = kmalloc(sizeof(p), GFP_KERNEL);
	72
	73	if (!p)
	74	return NULL;
	75
	76	p->x = ecc_alloc_digits_space(ndigits);
	77	if (!p->x)
	78	goto err_alloc_x;
	79
	80	p->y = ecc_alloc_digits_space(ndigits);
	81	if (!p->y)
	82	goto err_alloc_y;
	83
	84	p->ndigits = ndigits;
	85
	86	return p;
	87
	88	err_alloc_y:
	89	ecc_free_digits_space(p->x);
	90	err_alloc_x:
	91	kfree(p);
	92	return NULL;
	93	}
	94
	95	static void ecc_free_point(struct ecc_point *p)
	96	{
	97	if (!p)
	98	return;
	99
	100	kzfree(p->x);
	101	kzfree(p->y);
	102	kzfree(p);
	103	}
	104
	105	static void vli_clear(u64 *vli, unsigned int ndigits)
	106	{
	107	int i;
	108
	109	for (i = 0; i < ndigits; i++)
	110	vli[i] = 0;
	111	}
	112
	113	/* Returns true if vli == 0, false otherwise. */
	114	static bool vli_is_zero(const u64 *vli, unsigned int ndigits)
	115	{
	116	int i;
	117
	118	for (i = 0; i < ndigits; i++) {
	119	if (vli[i])
	120	return false;
	121	}
	122
	123	return true;
	124	}
	125
	126	/* Returns nonzero if bit bit of vli is set. */
	127	static u64 vli_test_bit(const u64 *vli, unsigned int bit)
	128	{
	129	return (vli[bit / 64] & ((u64)1 << (bit % 64)));
	130	}
	131
	132	/* Counts the number of 64-bit "digits" in vli. */
	133	static unsigned int vli_num_digits(const u64 *vli, unsigned int ndigits)
	134	{
	135	int i;
	136
	137	/* Search from the end until we find a non-zero digit.
	138	* We do it in reverse because we expect that most digits will
	139	* be nonzero.
	140	*/
	141	for (i = ndigits - 1; i >= 0 && vli[i] == 0; i--);
	142
	143	return (i + 1);
	144	}
	145
	146	/* Counts the number of bits required for vli. */
	147	static unsigned int vli_num_bits(const u64 *vli, unsigned int ndigits)
	148	{
	149	unsigned int i, num_digits;
	150	u64 digit;
	151
	152	num_digits = vli_num_digits(vli, ndigits);
	153	if (num_digits == 0)
	154	return 0;
	155
	156	digit = vli[num_digits - 1];
	157	for (i = 0; digit; i++)
	158	digit >>= 1;
	159
	160	return ((num_digits - 1) * 64 + i);
	161	}
	162
	163	/* Sets dest = src. */
	164	static void vli_set(u64 dest, const u64 src, unsigned int ndigits)
	165	{
	166	int i;
	167
	168	for (i = 0; i < ndigits; i++)
	169	dest[i] = src[i];
	170	}
	171
	172	/* Returns sign of left - right. */
	173	static int vli_cmp(const u64 left, const u64 right, unsigned int ndigits)
	174	{
	175	int i;
	176
	177	for (i = ndigits - 1; i >= 0; i--) {
	178	if (left[i] > right[i])
	179	return 1;
	180	else if (left[i] < right[i])
	181	return -1;
	182	}
	183
	184	return 0;
	185	}
	186
	187	/* Computes result = in << c, returning carry. Can modify in place
	188	* (if result == in). 0 < shift < 64.
	189	*/
	190	static u64 vli_lshift(u64 result, const u64 in, unsigned int shift,
	191	unsigned int ndigits)
	192	{
	193	u64 carry = 0;
	194	int i;
	195
	196	for (i = 0; i < ndigits; i++) {
	197	u64 temp = in[i];
	198
	199	result[i] = (temp << shift) \| carry;
	200	carry = temp >> (64 - shift);
	201	}
	202
	203	return carry;
	204	}
	205
	206	/* Computes vli = vli >> 1. */
	207	static void vli_rshift1(u64 *vli, unsigned int ndigits)
	208	{
	209	u64 *end = vli;
	210	u64 carry = 0;
	211
	212	vli += ndigits;
	213
	214	while (vli-- > end) {
	215	u64 temp = *vli;
	216	*vli = (temp >> 1) \| carry;
	217	carry = temp << 63;
	218	}
	219	}
	220
	221	/* Computes result = left + right, returning carry. Can modify in place. */
	222	static u64 vli_add(u64 result, const u64 left, const u64 *right,
	223	unsigned int ndigits)
	224	{
	225	u64 carry = 0;
	226	int i;
	227
	228	for (i = 0; i < ndigits; i++) {
	229	u64 sum;
	230
	231	sum = left[i] + right[i] + carry;
	232	if (sum != left[i])
	233	carry = (sum < left[i]);
	234
	235	result[i] = sum;
	236	}
	237
	238	return carry;
	239	}
	240
	241	/* Computes result = left - right, returning borrow. Can modify in place. */
	242	static u64 vli_sub(u64 result, const u64 left, const u64 *right,
	243	unsigned int ndigits)
	244	{
	245	u64 borrow = 0;
	246	int i;
	247
	248	for (i = 0; i < ndigits; i++) {
	249	u64 diff;
	250
	251	diff = left[i] - right[i] - borrow;
	252	if (diff != left[i])
	253	borrow = (diff > left[i]);
	254
	255	result[i] = diff;
	256	}
	257
	258	return borrow;
	259	}
	260
	261	static uint128_t mul_64_64(u64 left, u64 right)
	262	{
	263	u64 a0 = left & 0xffffffffull;
	264	u64 a1 = left >> 32;
	265	u64 b0 = right & 0xffffffffull;
	266	u64 b1 = right >> 32;
	267	u64 m0 = a0 * b0;
	268	u64 m1 = a0 * b1;
	269	u64 m2 = a1 * b0;
	270	u64 m3 = a1 * b1;
	271	uint128_t result;
	272
	273	m2 += (m0 >> 32);
	274	m2 += m1;
	275
	276	/* Overflow */
	277	if (m2 < m1)
	278	m3 += 0x100000000ull;
	279
	280	result.m_low = (m0 & 0xffffffffull) \| (m2 << 32);
	281	result.m_high = m3 + (m2 >> 32);
	282
	283	return result;
	284	}
	285
	286	static uint128_t add_128_128(uint128_t a, uint128_t b)
	287	{
	288	uint128_t result;
	289
	290	result.m_low = a.m_low + b.m_low;
	291	result.m_high = a.m_high + b.m_high + (result.m_low < a.m_low);
	292
	293	return result;
	294	}
	295
	296	static void vli_mult(u64 result, const u64 left, const u64 *right,
	297	unsigned int ndigits)
	298	{
	299	uint128_t r01 = { 0, 0 };
	300	u64 r2 = 0;
	301	unsigned int i, k;
	302
	303	/* Compute each digit of result in sequence, maintaining the
	304	* carries.
	305	*/
	306	for (k = 0; k < ndigits * 2 - 1; k++) {
	307	unsigned int min;
	308
	309	if (k < ndigits)
	310	min = 0;
	311	else
	312	min = (k + 1) - ndigits;
	313
	314	for (i = min; i <= k && i < ndigits; i++) {
	315	uint128_t product;
	316
	317	product = mul_64_64(left[i], right[k - i]);
	318
	319	r01 = add_128_128(r01, product);
	320	r2 += (r01.m_high < product.m_high);
	321	}
	322
	323	result[k] = r01.m_low;
	324	r01.m_low = r01.m_high;
	325	r01.m_high = r2;
	326	r2 = 0;
	327	}
	328
	329	result[ndigits * 2 - 1] = r01.m_low;
	330	}
	331
	332	static void vli_square(u64 result, const u64 left, unsigned int ndigits)
	333	{
	334	uint128_t r01 = { 0, 0 };
	335	u64 r2 = 0;
	336	int i, k;
	337
	338	for (k = 0; k < ndigits * 2 - 1; k++) {
	339	unsigned int min;
	340
	341	if (k < ndigits)
	342	min = 0;
	343	else
	344	min = (k + 1) - ndigits;
	345
	346	for (i = min; i <= k && i <= k - i; i++) {
	347	uint128_t product;
	348
	349	product = mul_64_64(left[i], left[k - i]);
	350
	351	if (i < k - i) {
	352	r2 += product.m_high >> 63;
	353	product.m_high = (product.m_high << 1) \|
	354	(product.m_low >> 63);
	355	product.m_low <<= 1;
	356	}
	357
	358	r01 = add_128_128(r01, product);
	359	r2 += (r01.m_high < product.m_high);
	360	}
	361
	362	result[k] = r01.m_low;
	363	r01.m_low = r01.m_high;
	364	r01.m_high = r2;
	365	r2 = 0;
	366	}
	367
	368	result[ndigits * 2 - 1] = r01.m_low;
	369	}
	370
	371	/* Computes result = (left + right) % mod.
	372	* Assumes that left < mod and right < mod, result != mod.
	373	*/
	374	static void vli_mod_add(u64 result, const u64 left, const u64 *right,
	375	const u64 *mod, unsigned int ndigits)
	376	{
	377	u64 carry;
	378
	379	carry = vli_add(result, left, right, ndigits);
	380
	381	/* result > mod (result = mod + remainder), so subtract mod to
	382	* get remainder.
	383	*/
	384	if (carry \|\| vli_cmp(result, mod, ndigits) >= 0)
	385	vli_sub(result, result, mod, ndigits);
	386	}
	387
	388	/* Computes result = (left - right) % mod.
	389	* Assumes that left < mod and right < mod, result != mod.
	390	*/
	391	static void vli_mod_sub(u64 result, const u64 left, const u64 *right,
	392	const u64 *mod, unsigned int ndigits)
	393	{
	394	u64 borrow = vli_sub(result, left, right, ndigits);
	395
	396	/* In this case, p_result == -diff == (max int) - diff.
	397	* Since -x % d == d - x, we can get the correct result from
	398	* result + mod (with overflow).
	399	*/
	400	if (borrow)
	401	vli_add(result, result, mod, ndigits);
	402	}
	403
	404	/* Computes p_result = p_product % curve_p.
	405	* See algorithm 5 and 6 from
	406	* http://www.isys.uni-klu.ac.at/PDF/2001-0126-MT.pdf
	407	*/
	408	static void vli_mmod_fast_192(u64 result, const u64 product,
	409	const u64 curve_prime, u64 tmp)
	410	{
	411	const unsigned int ndigits = 3;
	412	int carry;
	413
	414	vli_set(result, product, ndigits);
	415
	416	vli_set(tmp, &product[3], ndigits);
	417	carry = vli_add(result, result, tmp, ndigits);
	418
	419	tmp[0] = 0;
	420	tmp[1] = product[3];
	421	tmp[2] = product[4];
	422	carry += vli_add(result, result, tmp, ndigits);
	423
	424	tmp[0] = tmp[1] = product[5];
	425	tmp[2] = 0;
	426	carry += vli_add(result, result, tmp, ndigits);
	427
	428	while (carry \|\| vli_cmp(curve_prime, result, ndigits) != 1)
	429	carry -= vli_sub(result, result, curve_prime, ndigits);
	430	}
	431
	432	/* Computes result = product % curve_prime
	433	* from http://www.nsa.gov/ia/_files/nist-routines.pdf
	434	*/
	435	static void vli_mmod_fast_256(u64 result, const u64 product,
	436	const u64 curve_prime, u64 tmp)
	437	{
	438	int carry;
	439	const unsigned int ndigits = 4;
	440
	441	/* t */
	442	vli_set(result, product, ndigits);
	443
	444	/* s1 */
	445	tmp[0] = 0;
	446	tmp[1] = product[5] & 0xffffffff00000000ull;
	447	tmp[2] = product[6];
	448	tmp[3] = product[7];
	449	carry = vli_lshift(tmp, tmp, 1, ndigits);
	450	carry += vli_add(result, result, tmp, ndigits);
	451
	452	/* s2 */
	453	tmp[1] = product[6] << 32;
	454	tmp[2] = (product[6] >> 32) \| (product[7] << 32);
	455	tmp[3] = product[7] >> 32;
	456	carry += vli_lshift(tmp, tmp, 1, ndigits);
	457	carry += vli_add(result, result, tmp, ndigits);
	458
	459	/* s3 */
	460	tmp[0] = product[4];
	461	tmp[1] = product[5] & 0xffffffff;
	462	tmp[2] = 0;
	463	tmp[3] = product[7];
	464	carry += vli_add(result, result, tmp, ndigits);
	465
	466	/* s4 */
	467	tmp[0] = (product[4] >> 32) \| (product[5] << 32);
	468	tmp[1] = (product[5] >> 32) \| (product[6] & 0xffffffff00000000ull);
	469	tmp[2] = product[7];
	470	tmp[3] = (product[6] >> 32) \| (product[4] << 32);
	471	carry += vli_add(result, result, tmp, ndigits);
	472
	473	/* d1 */
	474	tmp[0] = (product[5] >> 32) \| (product[6] << 32);
	475	tmp[1] = (product[6] >> 32);
	476	tmp[2] = 0;
	477	tmp[3] = (product[4] & 0xffffffff) \| (product[5] << 32);
	478	carry -= vli_sub(result, result, tmp, ndigits);
	479
	480	/* d2 */
	481	tmp[0] = product[6];
	482	tmp[1] = product[7];
	483	tmp[2] = 0;
	484	tmp[3] = (product[4] >> 32) \| (product[5] & 0xffffffff00000000ull);
	485	carry -= vli_sub(result, result, tmp, ndigits);
	486
	487	/* d3 */
	488	tmp[0] = (product[6] >> 32) \| (product[7] << 32);
	489	tmp[1] = (product[7] >> 32) \| (product[4] << 32);
	490	tmp[2] = (product[4] >> 32) \| (product[5] << 32);
	491	tmp[3] = (product[6] << 32);
	492	carry -= vli_sub(result, result, tmp, ndigits);
	493
	494	/* d4 */
	495	tmp[0] = product[7];
	496	tmp[1] = product[4] & 0xffffffff00000000ull;
	497	tmp[2] = product[5];
	498	tmp[3] = product[6] & 0xffffffff00000000ull;
	499	carry -= vli_sub(result, result, tmp, ndigits);
	500
	501	if (carry < 0) {
	502	do {
	503	carry += vli_add(result, result, curve_prime, ndigits);
	504	} while (carry < 0);
	505	} else {
	506	while (carry \|\| vli_cmp(curve_prime, result, ndigits) != 1)
	507	carry -= vli_sub(result, result, curve_prime, ndigits);
	508	}
	509	}
	510
	511	/* Computes result = product % curve_prime
	512	* from http://www.nsa.gov/ia/_files/nist-routines.pdf
	513	*/
	514	static bool vli_mmod_fast(u64 result, u64 product,
	515	const u64 *curve_prime, unsigned int ndigits)
	516	{
	517	u64 tmp[2 * ndigits];
	518
	519	switch (ndigits) {
	520	case 3:
	521	vli_mmod_fast_192(result, product, curve_prime, tmp);
	522	break;
	523	case 4:
	524	vli_mmod_fast_256(result, product, curve_prime, tmp);
	525	break;
	526	default:
	527	pr_err("unsupports digits size!\n");
	528	return false;
	529	}
	530
	531	return true;
	532	}
	533
	534	/* Computes result = (left * right) % curve_prime. */
	535	static void vli_mod_mult_fast(u64 result, const u64 left, const u64 *right,
	536	const u64 *curve_prime, unsigned int ndigits)
	537	{
	538	u64 product[2 * ndigits];
	539
	540	vli_mult(product, left, right, ndigits);
	541	vli_mmod_fast(result, product, curve_prime, ndigits);
	542	}
	543
	544	/* Computes result = left^2 % curve_prime. */
	545	static void vli_mod_square_fast(u64 result, const u64 left,
	546	const u64 *curve_prime, unsigned int ndigits)
	547	{
	548	u64 product[2 * ndigits];
	549
	550	vli_square(product, left, ndigits);
	551	vli_mmod_fast(result, product, curve_prime, ndigits);
	552	}
	553
	554	#define EVEN(vli) (!(vli[0] & 1))
	555	/* Computes result = (1 / p_input) % mod. All VLIs are the same size.
	556	* See "From Euclid's GCD to Montgomery Multiplication to the Great Divide"
	557	* https://labs.oracle.com/techrep/2001/smli_tr-2001-95.pdf
	558	*/
	559	static void vli_mod_inv(u64 result, const u64 input, const u64 *mod,
	560	unsigned int ndigits)
	561	{
	562	u64 a[ndigits], b[ndigits];
	563	u64 u[ndigits], v[ndigits];
	564	u64 carry;
	565	int cmp_result;
	566
	567	if (vli_is_zero(input, ndigits)) {
	568	vli_clear(result, ndigits);
	569	return;
	570	}
	571
	572	vli_set(a, input, ndigits);
	573	vli_set(b, mod, ndigits);
	574	vli_clear(u, ndigits);
	575	u[0] = 1;
	576	vli_clear(v, ndigits);
	577
	578	while ((cmp_result = vli_cmp(a, b, ndigits)) != 0) {
	579	carry = 0;
	580
	581	if (EVEN(a)) {
	582	vli_rshift1(a, ndigits);
	583
	584	if (!EVEN(u))
	585	carry = vli_add(u, u, mod, ndigits);
	586
	587	vli_rshift1(u, ndigits);
	588	if (carry)
	589	u[ndigits - 1] \|= 0x8000000000000000ull;
	590	} else if (EVEN(b)) {
	591	vli_rshift1(b, ndigits);
	592
	593	if (!EVEN(v))
	594	carry = vli_add(v, v, mod, ndigits);
	595
	596	vli_rshift1(v, ndigits);
	597	if (carry)
	598	v[ndigits - 1] \|= 0x8000000000000000ull;
	599	} else if (cmp_result > 0) {
	600	vli_sub(a, a, b, ndigits);
	601	vli_rshift1(a, ndigits);
	602
	603	if (vli_cmp(u, v, ndigits) < 0)
	604	vli_add(u, u, mod, ndigits);
	605
	606	vli_sub(u, u, v, ndigits);
	607	if (!EVEN(u))
	608	carry = vli_add(u, u, mod, ndigits);
	609
	610	vli_rshift1(u, ndigits);
	611	if (carry)
	612	u[ndigits - 1] \|= 0x8000000000000000ull;
	613	} else {
	614	vli_sub(b, b, a, ndigits);
	615	vli_rshift1(b, ndigits);
	616
	617	if (vli_cmp(v, u, ndigits) < 0)
	618	vli_add(v, v, mod, ndigits);
	619
	620	vli_sub(v, v, u, ndigits);
	621	if (!EVEN(v))
	622	carry = vli_add(v, v, mod, ndigits);
	623
	624	vli_rshift1(v, ndigits);
	625	if (carry)
	626	v[ndigits - 1] \|= 0x8000000000000000ull;
	627	}
	628	}
	629
	630	vli_set(result, u, ndigits);
	631	}
	632
	633	/* ------ Point operations ------ */
	634
	635	/* Returns true if p_point is the point at infinity, false otherwise. */
	636	static bool ecc_point_is_zero(const struct ecc_point *point)
	637	{
	638	return (vli_is_zero(point->x, point->ndigits) &&
	639	vli_is_zero(point->y, point->ndigits));
	640	}
	641
	642	/* Point multiplication algorithm using Montgomery's ladder with co-Z
	643	* coordinates. From http://eprint.iacr.org/2011/338.pdf
	644	*/
	645
	646	/* Double in place */
	647	static void ecc_point_double_jacobian(u64 x1, u64 y1, u64 *z1,
	648	u64 *curve_prime, unsigned int ndigits)
	649	{
	650	/* t1 = x, t2 = y, t3 = z */
	651	u64 t4[ndigits];
	652	u64 t5[ndigits];
	653
	654	if (vli_is_zero(z1, ndigits))
	655	return;
	656
	657	/* t4 = y1^2 */
	658	vli_mod_square_fast(t4, y1, curve_prime, ndigits);
	659	/* t5 = x1y1^2 = A /
	660	vli_mod_mult_fast(t5, x1, t4, curve_prime, ndigits);
	661	/* t4 = y1^4 */
	662	vli_mod_square_fast(t4, t4, curve_prime, ndigits);
	663	/* t2 = y1z1 = z3 /
	664	vli_mod_mult_fast(y1, y1, z1, curve_prime, ndigits);
	665	/* t3 = z1^2 */
	666	vli_mod_square_fast(z1, z1, curve_prime, ndigits);
	667
	668	/* t1 = x1 + z1^2 */
	669	vli_mod_add(x1, x1, z1, curve_prime, ndigits);
	670	/* t3 = 2z1^2 /
	671	vli_mod_add(z1, z1, z1, curve_prime, ndigits);
	672	/* t3 = x1 - z1^2 */
	673	vli_mod_sub(z1, x1, z1, curve_prime, ndigits);
	674	/* t1 = x1^2 - z1^4 */
	675	vli_mod_mult_fast(x1, x1, z1, curve_prime, ndigits);
	676
	677	/* t3 = 2(x1^2 - z1^4) /
	678	vli_mod_add(z1, x1, x1, curve_prime, ndigits);
	679	/* t1 = 3(x1^2 - z1^4) /
	680	vli_mod_add(x1, x1, z1, curve_prime, ndigits);
	681	if (vli_test_bit(x1, 0)) {
	682	u64 carry = vli_add(x1, x1, curve_prime, ndigits);
	683
	684	vli_rshift1(x1, ndigits);
	685	x1[ndigits - 1] \|= carry << 63;
	686	} else {
	687	vli_rshift1(x1, ndigits);
	688	}
	689	/* t1 = 3/2(x1^2 - z1^4) = B /
	690
	691	/* t3 = B^2 */
	692	vli_mod_square_fast(z1, x1, curve_prime, ndigits);
	693	/* t3 = B^2 - A */
	694	vli_mod_sub(z1, z1, t5, curve_prime, ndigits);
	695	/* t3 = B^2 - 2A = x3 */
	696	vli_mod_sub(z1, z1, t5, curve_prime, ndigits);
	697	/* t5 = A - x3 */
	698	vli_mod_sub(t5, t5, z1, curve_prime, ndigits);
	699	/* t1 = B * (A - x3) */
	700	vli_mod_mult_fast(x1, x1, t5, curve_prime, ndigits);
	701	/* t4 = B * (A - x3) - y1^4 = y3 */
	702	vli_mod_sub(t4, x1, t4, curve_prime, ndigits);
	703
	704	vli_set(x1, z1, ndigits);
	705	vli_set(z1, y1, ndigits);
	706	vli_set(y1, t4, ndigits);
	707	}
	708
	709	/* Modify (x1, y1) => (x1 * z^2, y1 * z^3) */
	710	static void apply_z(u64 x1, u64 y1, u64 z, u64 curve_prime,
	711	unsigned int ndigits)
	712	{
	713	u64 t1[ndigits];
	714
	715	vli_mod_square_fast(t1, z, curve_prime, ndigits); /* z^2 */
	716	vli_mod_mult_fast(x1, x1, t1, curve_prime, ndigits); /* x1 * z^2 */
	717	vli_mod_mult_fast(t1, t1, z, curve_prime, ndigits); /* z^3 */
	718	vli_mod_mult_fast(y1, y1, t1, curve_prime, ndigits); /* y1 * z^3 */
	719	}
	720
	721	/* P = (x1, y1) => 2P, (x2, y2) => P' */
	722	static void xycz_initial_double(u64 x1, u64 y1, u64 x2, u64 y2,
	723	u64 p_initial_z, u64 curve_prime,
	724	unsigned int ndigits)
	725	{
	726	u64 z[ndigits];
	727
	728	vli_set(x2, x1, ndigits);
	729	vli_set(y2, y1, ndigits);
	730
	731	vli_clear(z, ndigits);
	732	z[0] = 1;
	733
	734	if (p_initial_z)
	735	vli_set(z, p_initial_z, ndigits);
	736
	737	apply_z(x1, y1, z, curve_prime, ndigits);
	738
	739	ecc_point_double_jacobian(x1, y1, z, curve_prime, ndigits);
	740
	741	apply_z(x2, y2, z, curve_prime, ndigits);
	742	}
	743
	744	/* Input P = (x1, y1, Z), Q = (x2, y2, Z)
	745	* Output P' = (x1', y1', Z3), P + Q = (x3, y3, Z3)
	746	* or P => P', Q => P + Q
	747	*/
	748	static void xycz_add(u64 x1, u64 y1, u64 x2, u64 y2, u64 *curve_prime,
	749	unsigned int ndigits)
	750	{
	751	/* t1 = X1, t2 = Y1, t3 = X2, t4 = Y2 */
	752	u64 t5[ndigits];
	753
	754	/* t5 = x2 - x1 */
	755	vli_mod_sub(t5, x2, x1, curve_prime, ndigits);
	756	/* t5 = (x2 - x1)^2 = A */
	757	vli_mod_square_fast(t5, t5, curve_prime, ndigits);
	758	/* t1 = x1A = B /
	759	vli_mod_mult_fast(x1, x1, t5, curve_prime, ndigits);
	760	/* t3 = x2A = C /
	761	vli_mod_mult_fast(x2, x2, t5, curve_prime, ndigits);
	762	/* t4 = y2 - y1 */
	763	vli_mod_sub(y2, y2, y1, curve_prime, ndigits);
	764	/* t5 = (y2 - y1)^2 = D */
	765	vli_mod_square_fast(t5, y2, curve_prime, ndigits);
	766
	767	/* t5 = D - B */
	768	vli_mod_sub(t5, t5, x1, curve_prime, ndigits);
	769	/* t5 = D - B - C = x3 */
	770	vli_mod_sub(t5, t5, x2, curve_prime, ndigits);
	771	/* t3 = C - B */
	772	vli_mod_sub(x2, x2, x1, curve_prime, ndigits);
	773	/* t2 = y1(C - B) /
	774	vli_mod_mult_fast(y1, y1, x2, curve_prime, ndigits);
	775	/* t3 = B - x3 */
	776	vli_mod_sub(x2, x1, t5, curve_prime, ndigits);
	777	/* t4 = (y2 - y1)(B - x3) /
	778	vli_mod_mult_fast(y2, y2, x2, curve_prime, ndigits);
	779	/* t4 = y3 */
	780	vli_mod_sub(y2, y2, y1, curve_prime, ndigits);
	781
	782	vli_set(x2, t5, ndigits);
	783	}
	784
	785	/* Input P = (x1, y1, Z), Q = (x2, y2, Z)
	786	* Output P + Q = (x3, y3, Z3), P - Q = (x3', y3', Z3)
	787	* or P => P - Q, Q => P + Q
	788	*/
	789	static void xycz_add_c(u64 x1, u64 y1, u64 x2, u64 y2, u64 *curve_prime,
	790	unsigned int ndigits)
	791	{
	792	/* t1 = X1, t2 = Y1, t3 = X2, t4 = Y2 */
	793	u64 t5[ndigits];
	794	u64 t6[ndigits];
	795	u64 t7[ndigits];
	796
	797	/* t5 = x2 - x1 */
	798	vli_mod_sub(t5, x2, x1, curve_prime, ndigits);
	799	/* t5 = (x2 - x1)^2 = A */
	800	vli_mod_square_fast(t5, t5, curve_prime, ndigits);
	801	/* t1 = x1A = B /
	802	vli_mod_mult_fast(x1, x1, t5, curve_prime, ndigits);
	803	/* t3 = x2A = C /
	804	vli_mod_mult_fast(x2, x2, t5, curve_prime, ndigits);
	805	/* t4 = y2 + y1 */
	806	vli_mod_add(t5, y2, y1, curve_prime, ndigits);
	807	/* t4 = y2 - y1 */
	808	vli_mod_sub(y2, y2, y1, curve_prime, ndigits);
	809
	810	/* t6 = C - B */
	811	vli_mod_sub(t6, x2, x1, curve_prime, ndigits);
	812	/* t2 = y1 * (C - B) */
	813	vli_mod_mult_fast(y1, y1, t6, curve_prime, ndigits);
	814	/* t6 = B + C */
	815	vli_mod_add(t6, x1, x2, curve_prime, ndigits);
	816	/* t3 = (y2 - y1)^2 */
	817	vli_mod_square_fast(x2, y2, curve_prime, ndigits);
	818	/* t3 = x3 */
	819	vli_mod_sub(x2, x2, t6, curve_prime, ndigits);
	820
	821	/* t7 = B - x3 */
	822	vli_mod_sub(t7, x1, x2, curve_prime, ndigits);
	823	/* t4 = (y2 - y1)(B - x3) /
	824	vli_mod_mult_fast(y2, y2, t7, curve_prime, ndigits);
	825	/* t4 = y3 */
	826	vli_mod_sub(y2, y2, y1, curve_prime, ndigits);
	827
	828	/* t7 = (y2 + y1)^2 = F */
	829	vli_mod_square_fast(t7, t5, curve_prime, ndigits);
	830	/* t7 = x3' */
	831	vli_mod_sub(t7, t7, t6, curve_prime, ndigits);
	832	/* t6 = x3' - B */
	833	vli_mod_sub(t6, t7, x1, curve_prime, ndigits);
	834	/* t6 = (y2 + y1)(x3' - B) /
	835	vli_mod_mult_fast(t6, t6, t5, curve_prime, ndigits);
	836	/* t2 = y3' */
	837	vli_mod_sub(y1, t6, y1, curve_prime, ndigits);
	838
	839	vli_set(x1, t7, ndigits);
	840	}
	841
	842	static void ecc_point_mult(struct ecc_point *result,
	843	const struct ecc_point point, const u64 scalar,
	844	u64 initial_z, u64 curve_prime,
	845	unsigned int ndigits)
	846	{
	847	/* R0 and R1 */
	848	u64 rx[2][ndigits];
	849	u64 ry[2][ndigits];
	850	u64 z[ndigits];
	851	int i, nb;
	852	int num_bits = vli_num_bits(scalar, ndigits);
	853
	854	vli_set(rx[1], point->x, ndigits);
	855	vli_set(ry[1], point->y, ndigits);
	856
	857	xycz_initial_double(rx[1], ry[1], rx[0], ry[0], initial_z, curve_prime,
	858	ndigits);
	859
	860	for (i = num_bits - 2; i > 0; i--) {
	861	nb = !vli_test_bit(scalar, i);
	862	xycz_add_c(rx[1 - nb], ry[1 - nb], rx[nb], ry[nb], curve_prime,
	863	ndigits);
	864	xycz_add(rx[nb], ry[nb], rx[1 - nb], ry[1 - nb], curve_prime,
	865	ndigits);
	866	}
	867
	868	nb = !vli_test_bit(scalar, 0);
	869	xycz_add_c(rx[1 - nb], ry[1 - nb], rx[nb], ry[nb], curve_prime,
	870	ndigits);
	871
	872	/* Find final 1/Z value. */
	873	/* X1 - X0 */
	874	vli_mod_sub(z, rx[1], rx[0], curve_prime, ndigits);
	875	/* Yb * (X1 - X0) */
	876	vli_mod_mult_fast(z, z, ry[1 - nb], curve_prime, ndigits);
	877	/* xP * Yb * (X1 - X0) */
	878	vli_mod_mult_fast(z, z, point->x, curve_prime, ndigits);
	879
	880	/* 1 / (xP * Yb * (X1 - X0)) */
	881	vli_mod_inv(z, z, curve_prime, point->ndigits);
	882
	883	/* yP / (xP * Yb * (X1 - X0)) */
	884	vli_mod_mult_fast(z, z, point->y, curve_prime, ndigits);
	885	/* Xb * yP / (xP * Yb * (X1 - X0)) */
	886	vli_mod_mult_fast(z, z, rx[1 - nb], curve_prime, ndigits);
	887	/* End 1/Z calculation */
	888
	889	xycz_add(rx[nb], ry[nb], rx[1 - nb], ry[1 - nb], curve_prime, ndigits);
	890
	891	apply_z(rx[0], ry[0], z, curve_prime, ndigits);
	892
	893	vli_set(result->x, rx[0], ndigits);
	894	vli_set(result->y, ry[0], ndigits);
	895	}
	896
	897	static inline void ecc_swap_digits(const u64 in, u64 out,
	898	unsigned int ndigits)
	899	{
	900	int i;
	901
	902	for (i = 0; i < ndigits; i++)
	903	out[i] = __swab64(in[ndigits - 1 - i]);
	904	}
	905
	906	int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits,
	907	const u8 *private_key, unsigned int private_key_len)
	908	{
	909	int nbytes;
	910	const struct ecc_curve *curve = ecc_get_curve(curve_id);
	911
	912	if (!private_key)
	913	return -EINVAL;
	914
	915	nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
	916
	917	if (private_key_len != nbytes)
	918	return -EINVAL;
	919
	920	if (vli_is_zero((const u64 *)&private_key[0], ndigits))
	921	return -EINVAL;
	922
	923	/* Make sure the private key is in the range [1, n-1]. */
	924	if (vli_cmp(curve->n, (const u64 *)&private_key[0], ndigits) != 1)
	925	return -EINVAL;
	926
	927	return 0;
	928	}
	929
	930	int ecdh_make_pub_key(unsigned int curve_id, unsigned int ndigits,
	931	const u8 *private_key, unsigned int private_key_len,
	932	u8 *public_key, unsigned int public_key_len)
	933	{
	934	int ret = 0;
	935	struct ecc_point *pk;
	936	u64 priv[ndigits];
	937	unsigned int nbytes;
	938	const struct ecc_curve *curve = ecc_get_curve(curve_id);
	939
	940	if (!private_key \|\| !curve) {
	941	ret = -EINVAL;
	942	goto out;
	943	}
	944
	945	ecc_swap_digits((const u64 *)private_key, priv, ndigits);
	946
	947	pk = ecc_alloc_point(ndigits);
	948	if (!pk) {
	949	ret = -ENOMEM;
	950	goto out;
	951	}
	952
	953	ecc_point_mult(pk, &curve->g, priv, NULL, curve->p, ndigits);
	954	if (ecc_point_is_zero(pk)) {
	955	ret = -EAGAIN;
	956	goto err_free_point;
	957	}
	958
	959	nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
	960	ecc_swap_digits(pk->x, (u64 *)public_key, ndigits);
	961	ecc_swap_digits(pk->y, (u64 *)&public_key[nbytes], ndigits);
	962
	963	err_free_point:
	964	ecc_free_point(pk);
	965	out:
	966	return ret;
	967	}
	968
	969	int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits,
	970	const u8 *private_key, unsigned int private_key_len,
	971	const u8 *public_key, unsigned int public_key_len,
	972	u8 *secret, unsigned int secret_len)
	973	{
	974	int ret = 0;
	975	struct ecc_point product, pk;
	976	u64 priv[ndigits];
	977	u64 rand_z[ndigits];
	978	unsigned int nbytes;
	979	const struct ecc_curve *curve = ecc_get_curve(curve_id);
	980
	981	if (!private_key \|\| !public_key \|\| !curve) {
	982	ret = -EINVAL;
	983	goto out;
	984	}
	985
	986	nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
	987
	988	get_random_bytes(rand_z, nbytes);
	989
	990	pk = ecc_alloc_point(ndigits);
	991	if (!pk) {
	992	ret = -ENOMEM;
	993	goto out;
	994	}
	995
	996	product = ecc_alloc_point(ndigits);
	997	if (!product) {
	998	ret = -ENOMEM;
	999	goto err_alloc_product;
	1000	}
	1001
	1002	ecc_swap_digits((const u64 *)public_key, pk->x, ndigits);
	1003	ecc_swap_digits((const u64 *)&public_key[nbytes], pk->y, ndigits);
	1004	ecc_swap_digits((const u64 *)private_key, priv, ndigits);
	1005
	1006	ecc_point_mult(product, pk, priv, rand_z, curve->p, ndigits);
	1007
	1008	ecc_swap_digits(product->x, (u64 *)secret, ndigits);
	1009
	1010	if (ecc_point_is_zero(product))
	1011	ret = -EFAULT;
	1012
	1013	ecc_free_point(product);
	1014	err_alloc_product:
	1015	ecc_free_point(pk);
	1016	out:
	1017	return ret;
	1018	}

crypto/ecc.h+83−0

@@ -0,0 +1,83 @@
	1	/*
	2	* Copyright (c) 2013, Kenneth MacKay
	3	* All rights reserved.
	4	*
	5	* Redistribution and use in source and binary forms, with or without
	6	* modification, are permitted provided that the following conditions are
	7	* met:
	8	* * Redistributions of source code must retain the above copyright
	9	* notice, this list of conditions and the following disclaimer.
	10	* * Redistributions in binary form must reproduce the above copyright
	11	* notice, this list of conditions and the following disclaimer in the
	12	* documentation and/or other materials provided with the distribution.
	13	*
	14	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	15	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	16	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	17	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	18	* HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	19	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	20	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	21	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	22	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	23	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	24	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	25	*/
	26	#ifndef _CRYPTO_ECC_H
	27	#define _CRYPTO_ECC_H
	28
	29	#define ECC_MAX_DIGITS 4 /* 256 */
	30
	31	#define ECC_DIGITS_TO_BYTES_SHIFT 3
	32
	33	/**
	34	* ecc_is_key_valid() - Validate a given ECDH private key
	35	*
	36	* @curve_id: id representing the curve to use
	37	* @ndigits: curve number of digits
	38	* @private_key: private key to be used for the given curve
	39	* @private_key_len: private key len
	40	*
	41	* Returns 0 if the key is acceptable, a negative value otherwise
	42	*/
	43	int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits,
	44	const u8 *private_key, unsigned int private_key_len);
	45
	46	/**
	47	* ecdh_make_pub_key() - Compute an ECC public key
	48	*
	49	* @curve_id: id representing the curve to use
	50	* @private_key: pregenerated private key for the given curve
	51	* @private_key_len: length of private_key
	52	* @public_key: buffer for storing the public key generated
	53	* @public_key_len: length of the public_key buffer
	54	*
	55	* Returns 0 if the public key was generated successfully, a negative value
	56	* if an error occurred.
	57	*/
	58	int ecdh_make_pub_key(const unsigned int curve_id, unsigned int ndigits,
	59	const u8 *private_key, unsigned int private_key_len,
	60	u8 *public_key, unsigned int public_key_len);
	61
	62	/**
	63	* crypto_ecdh_shared_secret() - Compute a shared secret
	64	*
	65	* @curve_id: id representing the curve to use
	66	* @private_key: private key of part A
	67	* @private_key_len: length of private_key
	68	* @public_key: public key of counterpart B
	69	* @public_key_len: length of public_key
	70	* @secret: buffer for storing the calculated shared secret
	71	* @secret_len: length of the secret buffer
	72	*
	73	* Note: It is recommended that you hash the result of crypto_ecdh_shared_secret
	74	* before using it for symmetric encryption or HMAC.
	75	*
	76	* Returns 0 if the shared secret was generated successfully, a negative value
	77	* if an error occurred.
	78	*/
	79	int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits,
	80	const u8 *private_key, unsigned int private_key_len,
	81	const u8 *public_key, unsigned int public_key_len,
	82	u8 *secret, unsigned int secret_len);
	83	#endif

crypto/ecc_curve_defs.h+57−0

@@ -0,0 +1,57 @@
	1	#ifndef _CRYTO_ECC_CURVE_DEFS_H
	2	#define _CRYTO_ECC_CURVE_DEFS_H
	3
	4	struct ecc_point {
	5	u64 *x;
	6	u64 *y;
	7	u8 ndigits;
	8	};
	9
	10	struct ecc_curve {
	11	char *name;
	12	struct ecc_point g;
	13	u64 *p;
	14	u64 *n;
	15	};
	16
	17	/* NIST P-192 */
	18	static u64 nist_p192_g_x[] = { 0xF4FF0AFD82FF1012ull, 0x7CBF20EB43A18800ull,
	19	0x188DA80EB03090F6ull };
	20	static u64 nist_p192_g_y[] = { 0x73F977A11E794811ull, 0x631011ED6B24CDD5ull,
	21	0x07192B95FFC8DA78ull };
	22	static u64 nist_p192_p[] = { 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFEull,
	23	0xFFFFFFFFFFFFFFFFull };
	24	static u64 nist_p192_n[] = { 0x146BC9B1B4D22831ull, 0xFFFFFFFF99DEF836ull,
	25	0xFFFFFFFFFFFFFFFFull };
	26	static struct ecc_curve nist_p192 = {
	27	.name = "nist_192",
	28	.g = {
	29	.x = nist_p192_g_x,
	30	.y = nist_p192_g_y,
	31	.ndigits = 3,
	32	},
	33	.p = nist_p192_p,
	34	.n = nist_p192_n
	35	};
	36
	37	/* NIST P-256 */
	38	static u64 nist_p256_g_x[] = { 0xF4A13945D898C296ull, 0x77037D812DEB33A0ull,
	39	0xF8BCE6E563A440F2ull, 0x6B17D1F2E12C4247ull };
	40	static u64 nist_p256_g_y[] = { 0xCBB6406837BF51F5ull, 0x2BCE33576B315ECEull,
	41	0x8EE7EB4A7C0F9E16ull, 0x4FE342E2FE1A7F9Bull };
	42	static u64 nist_p256_p[] = { 0xFFFFFFFFFFFFFFFFull, 0x00000000FFFFFFFFull,
	43	0x0000000000000000ull, 0xFFFFFFFF00000001ull };
	44	static u64 nist_p256_n[] = { 0xF3B9CAC2FC632551ull, 0xBCE6FAADA7179E84ull,
	45	0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFF00000000ull };
	46	static struct ecc_curve nist_p256 = {
	47	.name = "nist_256",
	48	.g = {
	49	.x = nist_p256_g_x,
	50	.y = nist_p256_g_y,
	51	.ndigits = 4,
	52	},
	53	.p = nist_p256_p,
	54	.n = nist_p256_n
	55	};
	56
	57	#endif

crypto/ecdh.c+151−0

@@ -0,0 +1,151 @@
	1	/* ECDH key-agreement protocol
	2	*
	3	* Copyright (c) 2016, Intel Corporation
	4	* Authors: Salvator Benedetto <salvatore.benedetto@intel.com>
	5	*
	6	* This program is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU General Public Licence
	8	* as published by the Free Software Foundation; either version
	9	* 2 of the Licence, or (at your option) any later version.
	10	*/
	11
	12	#include <linux/module.h>
	13	#include <crypto/internal/kpp.h>
	14	#include <crypto/kpp.h>
	15	#include <crypto/ecdh.h>
	16	#include <linux/scatterlist.h>
	17	#include "ecc.h"
	18
	19	struct ecdh_ctx {
	20	unsigned int curve_id;
	21	unsigned int ndigits;
	22	u64 private_key[ECC_MAX_DIGITS];
	23	u64 public_key[2 * ECC_MAX_DIGITS];
	24	u64 shared_secret[ECC_MAX_DIGITS];
	25	};
	26
	27	static inline struct ecdh_ctx ecdh_get_ctx(struct crypto_kpp tfm)
	28	{
	29	return kpp_tfm_ctx(tfm);
	30	}
	31
	32	static unsigned int ecdh_supported_curve(unsigned int curve_id)
	33	{
	34	switch (curve_id) {
	35	case ECC_CURVE_NIST_P192: return 3;
	36	case ECC_CURVE_NIST_P256: return 4;
	37	default: return 0;
	38	}
	39	}
	40
	41	static int ecdh_set_secret(struct crypto_kpp tfm, void buf, unsigned int len)
	42	{
	43	struct ecdh_ctx *ctx = ecdh_get_ctx(tfm);
	44	struct ecdh params;
	45	unsigned int ndigits;
	46
	47	if (crypto_ecdh_decode_key(buf, len, &params) < 0)
	48	return -EINVAL;
	49
	50	ndigits = ecdh_supported_curve(params.curve_id);
	51	if (!ndigits)
	52	return -EINVAL;
	53
	54	ctx->curve_id = params.curve_id;
	55	ctx->ndigits = ndigits;
	56
	57	if (ecc_is_key_valid(ctx->curve_id, ctx->ndigits,
	58	(const u8 *)params.key, params.key_size) < 0)
	59	return -EINVAL;
	60
	61	memcpy(ctx->private_key, params.key, params.key_size);
	62
	63	return 0;
	64	}
	65
	66	static int ecdh_compute_value(struct kpp_request *req)
	67	{
	68	int ret = 0;
	69	struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
	70	struct ecdh_ctx *ctx = ecdh_get_ctx(tfm);
	71	size_t copied, nbytes;
	72	void *buf;
	73
	74	nbytes = ctx->ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
	75
	76	if (req->src) {
	77	copied = sg_copy_to_buffer(req->src, 1, ctx->public_key,
	78	2 * nbytes);
	79	if (copied != 2 * nbytes)
	80	return -EINVAL;
	81
	82	ret = crypto_ecdh_shared_secret(ctx->curve_id, ctx->ndigits,
	83	(const u8 *)ctx->private_key, nbytes,
	84	(const u8 )ctx->public_key, 2 nbytes,
	85	(u8 *)ctx->shared_secret, nbytes);
	86
	87	buf = ctx->shared_secret;
	88	} else {
	89	ret = ecdh_make_pub_key(ctx->curve_id, ctx->ndigits,
	90	(const u8 *)ctx->private_key, nbytes,
	91	(u8 *)ctx->public_key,
	92	sizeof(ctx->public_key));
	93	buf = ctx->public_key;
	94	/* Public part is a point thus it has both coordinates */
	95	nbytes *= 2;
	96	}
	97
	98	if (ret < 0)
	99	return ret;
	100
	101	copied = sg_copy_from_buffer(req->dst, 1, buf, nbytes);
	102	if (copied != nbytes)
	103	return -EINVAL;
	104
	105	return ret;
	106	}
	107
	108	static int ecdh_max_size(struct crypto_kpp *tfm)
	109	{
	110	struct ecdh_ctx *ctx = ecdh_get_ctx(tfm);
	111	int nbytes = ctx->ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
	112
	113	/* Public key is made of two coordinates */
	114	return 2 * nbytes;
	115	}
	116
	117	static void no_exit_tfm(struct crypto_kpp *tfm)
	118	{
	119	return;
	120	}
	121
	122	static struct kpp_alg ecdh = {
	123	.set_secret = ecdh_set_secret,
	124	.generate_public_key = ecdh_compute_value,
	125	.compute_shared_secret = ecdh_compute_value,
	126	.max_size = ecdh_max_size,
	127	.exit = no_exit_tfm,
	128	.base = {
	129	.cra_name = "ecdh",
	130	.cra_driver_name = "ecdh-generic",
	131	.cra_priority = 100,
	132	.cra_module = THIS_MODULE,
	133	.cra_ctxsize = sizeof(struct ecdh_ctx),
	134	},
	135	};
	136
	137	static int ecdh_init(void)
	138	{
	139	return crypto_register_kpp(&ecdh);
	140	}
	141
	142	static void ecdh_exit(void)
	143	{
	144	crypto_unregister_kpp(&ecdh);
	145	}
	146
	147	module_init(ecdh_init);
	148	module_exit(ecdh_exit);
	149	MODULE_ALIAS_CRYPTO("ecdh");
	150	MODULE_LICENSE("GPL");
	151	MODULE_DESCRIPTION("ECDH generic algorithm");

crypto/ecdh_helper.c+86−0

@@ -0,0 +1,86 @@
	1	/*
	2	* Copyright (c) 2016, Intel Corporation
	3	* Authors: Salvatore Benedetto <salvatore.benedetto@intel.com>
	4	*
	5	* This program is free software; you can redistribute it and/or
	6	* modify it under the terms of the GNU General Public Licence
	7	* as published by the Free Software Foundation; either version
	8	* 2 of the Licence, or (at your option) any later version.
	9	*/
	10	#include <linux/kernel.h>
	11	#include <linux/export.h>
	12	#include <linux/err.h>
	13	#include <linux/string.h>
	14	#include <crypto/ecdh.h>
	15	#include <crypto/kpp.h>
	16
	17	#define ECDH_KPP_SECRET_MIN_SIZE (sizeof(struct kpp_secret) + 2 * sizeof(short))
	18
	19	static inline u8 ecdh_pack_data(void dst, const void *src, size_t sz)
	20	{
	21	memcpy(dst, src, sz);
	22	return dst + sz;
	23	}
	24
	25	static inline const u8 ecdh_unpack_data(void dst, const void *src, size_t sz)
	26	{
	27	memcpy(dst, src, sz);
	28	return src + sz;
	29	}
	30
	31	int crypto_ecdh_key_len(const struct ecdh *params)
	32	{
	33	return ECDH_KPP_SECRET_MIN_SIZE + params->key_size;
	34	}
	35	EXPORT_SYMBOL_GPL(crypto_ecdh_key_len);
	36
	37	int crypto_ecdh_encode_key(char *buf, unsigned int len,
	38	const struct ecdh *params)
	39	{
	40	u8 *ptr = buf;
	41	struct kpp_secret secret = {
	42	.type = CRYPTO_KPP_SECRET_TYPE_ECDH,
	43	.len = len
	44	};
	45
	46	if (unlikely(!buf))
	47	return -EINVAL;
	48
	49	if (len != crypto_ecdh_key_len(params))
	50	return -EINVAL;
	51
	52	ptr = ecdh_pack_data(ptr, &secret, sizeof(secret));
	53	ptr = ecdh_pack_data(ptr, &params->curve_id, sizeof(params->curve_id));
	54	ptr = ecdh_pack_data(ptr, &params->key_size, sizeof(params->key_size));
	55	ecdh_pack_data(ptr, params->key, params->key_size);
	56
	57	return 0;
	58	}
	59	EXPORT_SYMBOL_GPL(crypto_ecdh_encode_key);
	60
	61	int crypto_ecdh_decode_key(const char *buf, unsigned int len,
	62	struct ecdh *params)
	63	{
	64	const u8 *ptr = buf;
	65	struct kpp_secret secret;
	66
	67	if (unlikely(!buf \|\| len < ECDH_KPP_SECRET_MIN_SIZE))
	68	return -EINVAL;
	69
	70	ptr = ecdh_unpack_data(&secret, ptr, sizeof(secret));
	71	if (secret.type != CRYPTO_KPP_SECRET_TYPE_ECDH)
	72	return -EINVAL;
	73
	74	ptr = ecdh_unpack_data(&params->curve_id, ptr, sizeof(params->curve_id));
	75	ptr = ecdh_unpack_data(&params->key_size, ptr, sizeof(params->key_size));
	76	if (secret.len != crypto_ecdh_key_len(params))
	77	return -EINVAL;
	78
	79	/* Don't allocate memory. Set pointer to data
	80	* within the given buffer
	81	*/
	82	params->key = (void *)ptr;
	83
	84	return 0;
	85	}
	86	EXPORT_SYMBOL_GPL(crypto_ecdh_decode_key);

crypto/echainiv.c+10−6

 #include <crypto/internal/geniv.h>
 #include <crypto/scatterwalk.h>
+#include <crypto/skcipher.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 	info = req->iv;
 	if (req->src != req->dst) {
-		struct blkcipher_desc desc = {
+		SKCIPHER_REQUEST_ON_STACK(nreq, ctx->sknull);
-			.tfm = ctx->null,
-		};
-		err = crypto_blkcipher_encrypt(
+		skcipher_request_set_tfm(nreq, ctx->sknull);
-			&desc, req->dst, req->src,
+		skcipher_request_set_callback(nreq, req->base.flags,
-			req->assoclen + req->cryptlen);
+					      NULL, NULL);
+		skcipher_request_set_crypt(nreq, req->src, req->dst,
+					   req->assoclen + req->cryptlen,
+					   NULL);
+		err = crypto_skcipher_encrypt(nreq);
 		if (err)
 			return err;
 	}

crypto/eseqiv.c+0−242

@@ -1,242 +0,0 @@
1	/*
2	* eseqiv: Encrypted Sequence Number IV Generator
3	*
4	* This generator generates an IV based on a sequence number by xoring it
5	* with a salt and then encrypting it with the same key as used to encrypt
6	* the plain text. This algorithm requires that the block size be equal
7	* to the IV size. It is mainly useful for CBC.
8	*
9	* Copyright (c) 2007 Herbert Xu <herbert@gondor.apana.org.au>
10	*
11	* This program is free software; you can redistribute it and/or modify it
12	* under the terms of the GNU General Public License as published by the Free
13	* Software Foundation; either version 2 of the License, or (at your option)
14	* any later version.
15	*
16	*/
17
18	#include <crypto/internal/skcipher.h>
19	#include <crypto/rng.h>
20	#include <crypto/scatterwalk.h>
21	#include <linux/err.h>
22	#include <linux/init.h>
23	#include <linux/kernel.h>
24	#include <linux/mm.h>
25	#include <linux/module.h>
26	#include <linux/scatterlist.h>
27	#include <linux/spinlock.h>
28	#include <linux/string.h>
29
30	struct eseqiv_request_ctx {
31	struct scatterlist src[2];
32	struct scatterlist dst[2];
33	char tail[];
34	};
35
36	struct eseqiv_ctx {
37	spinlock_t lock;
38	unsigned int reqoff;
39	char salt[];
40	};
41
42	static void eseqiv_complete2(struct skcipher_givcrypt_request *req)
43	{
44	struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
45	struct eseqiv_request_ctx *reqctx = skcipher_givcrypt_reqctx(req);
46
47	memcpy(req->giv, PTR_ALIGN((u8 *)reqctx->tail,
48	crypto_ablkcipher_alignmask(geniv) + 1),
49	crypto_ablkcipher_ivsize(geniv));
50	}
51
52	static void eseqiv_complete(struct crypto_async_request *base, int err)
53	{
54	struct skcipher_givcrypt_request *req = base->data;
55
56	if (err)
57	goto out;
58
59	eseqiv_complete2(req);
60
61	out:
62	skcipher_givcrypt_complete(req, err);
63	}
64
65	static int eseqiv_givencrypt(struct skcipher_givcrypt_request *req)
66	{
67	struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
68	struct eseqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
69	struct eseqiv_request_ctx *reqctx = skcipher_givcrypt_reqctx(req);
70	struct ablkcipher_request *subreq;
71	crypto_completion_t compl;
72	void *data;
73	struct scatterlist osrc, odst;
74	struct scatterlist *dst;
75	struct page *srcp;
76	struct page *dstp;
77	u8 *giv;
78	u8 *vsrc;
79	u8 *vdst;
80	__be64 seq;
81	unsigned int ivsize;
82	unsigned int len;
83	int err;
84
85	subreq = (void *)(reqctx->tail + ctx->reqoff);
86	ablkcipher_request_set_tfm(subreq, skcipher_geniv_cipher(geniv));
87
88	giv = req->giv;
89	compl = req->creq.base.complete;
90	data = req->creq.base.data;
91
92	osrc = req->creq.src;
93	odst = req->creq.dst;
94	srcp = sg_page(osrc);
95	dstp = sg_page(odst);
96	vsrc = PageHighMem(srcp) ? NULL : page_address(srcp) + osrc->offset;
97	vdst = PageHighMem(dstp) ? NULL : page_address(dstp) + odst->offset;
98
99	ivsize = crypto_ablkcipher_ivsize(geniv);
100
101	if (vsrc != giv + ivsize && vdst != giv + ivsize) {
102	giv = PTR_ALIGN((u8 *)reqctx->tail,
103	crypto_ablkcipher_alignmask(geniv) + 1);
104	compl = eseqiv_complete;
105	data = req;
106	}
107
108	ablkcipher_request_set_callback(subreq, req->creq.base.flags, compl,
109	data);
110
111	sg_init_table(reqctx->src, 2);
112	sg_set_buf(reqctx->src, giv, ivsize);
113	scatterwalk_crypto_chain(reqctx->src, osrc, vsrc == giv + ivsize, 2);
114
115	dst = reqctx->src;
116	if (osrc != odst) {
117	sg_init_table(reqctx->dst, 2);
118	sg_set_buf(reqctx->dst, giv, ivsize);
119	scatterwalk_crypto_chain(reqctx->dst, odst, vdst == giv + ivsize, 2);
120
121	dst = reqctx->dst;
122	}
123
124	ablkcipher_request_set_crypt(subreq, reqctx->src, dst,
125	req->creq.nbytes + ivsize,
126	req->creq.info);
127
128	memcpy(req->creq.info, ctx->salt, ivsize);
129
130	len = ivsize;
131	if (ivsize > sizeof(u64)) {
132	memset(req->giv, 0, ivsize - sizeof(u64));
133	len = sizeof(u64);
134	}
135	seq = cpu_to_be64(req->seq);
136	memcpy(req->giv + ivsize - len, &seq, len);
137
138	err = crypto_ablkcipher_encrypt(subreq);
139	if (err)
140	goto out;
141
142	if (giv != req->giv)
143	eseqiv_complete2(req);
144
145	out:
146	return err;
147	}
148
149	static int eseqiv_init(struct crypto_tfm *tfm)
150	{
151	struct crypto_ablkcipher *geniv = __crypto_ablkcipher_cast(tfm);
152	struct eseqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
153	unsigned long alignmask;
154	unsigned int reqsize;
155	int err;
156
157	spin_lock_init(&ctx->lock);
158
159	alignmask = crypto_tfm_ctx_alignment() - 1;
160	reqsize = sizeof(struct eseqiv_request_ctx);
161
162	if (alignmask & reqsize) {
163	alignmask &= reqsize;
164	alignmask--;
165	}
166
167	alignmask = ~alignmask;
168	alignmask &= crypto_ablkcipher_alignmask(geniv);
169
170	reqsize += alignmask;
171	reqsize += crypto_ablkcipher_ivsize(geniv);
172	reqsize = ALIGN(reqsize, crypto_tfm_ctx_alignment());
173
174	ctx->reqoff = reqsize - sizeof(struct eseqiv_request_ctx);
175
176	tfm->crt_ablkcipher.reqsize = reqsize +
177	sizeof(struct ablkcipher_request);
178
179	err = 0;
180	if (!crypto_get_default_rng()) {
181	crypto_ablkcipher_crt(geniv)->givencrypt = eseqiv_givencrypt;
182	err = crypto_rng_get_bytes(crypto_default_rng, ctx->salt,
183	crypto_ablkcipher_ivsize(geniv));
184	crypto_put_default_rng();
185	}
186
187	return err ?: skcipher_geniv_init(tfm);
188	}
189
190	static struct crypto_template eseqiv_tmpl;
191
192	static struct crypto_instance eseqiv_alloc(struct rtattr *tb)
193	{
194	struct crypto_instance *inst;
195	int err;
196
197	inst = skcipher_geniv_alloc(&eseqiv_tmpl, tb, 0, 0);
198	if (IS_ERR(inst))
199	goto out;
200
201	err = -EINVAL;
202	if (inst->alg.cra_ablkcipher.ivsize != inst->alg.cra_blocksize)
203	goto free_inst;
204
205	inst->alg.cra_init = eseqiv_init;
206	inst->alg.cra_exit = skcipher_geniv_exit;
207
208	inst->alg.cra_ctxsize = sizeof(struct eseqiv_ctx);
209	inst->alg.cra_ctxsize += inst->alg.cra_ablkcipher.ivsize;
210
211	out:
212	return inst;
213
214	free_inst:
215	skcipher_geniv_free(inst);
216	inst = ERR_PTR(err);
217	goto out;
218	}
219
220	static struct crypto_template eseqiv_tmpl = {
221	.name = "eseqiv",
222	.alloc = eseqiv_alloc,
223	.free = skcipher_geniv_free,
224	.module = THIS_MODULE,
225	};
226
227	static int __init eseqiv_module_init(void)
228	{
229	return crypto_register_template(&eseqiv_tmpl);
230	}
231
232	static void __exit eseqiv_module_exit(void)
233	{
234	crypto_unregister_template(&eseqiv_tmpl);
235	}
236
237	module_init(eseqiv_module_init);
238	module_exit(eseqiv_module_exit);
239
240	MODULE_LICENSE("GPL");
241	MODULE_DESCRIPTION("Encrypted Sequence Number IV Generator");
242	MODULE_ALIAS_CRYPTO("eseqiv");

crypto/gcm.c+61−54

 };
 struct crypto_gcm_ctx {
-	struct crypto_ablkcipher *ctr;
+	struct crypto_skcipher *ctr;
 	struct crypto_ahash *ghash;
 };
 struct crypto_rfc4543_ctx {
 	struct crypto_aead *child;
-	struct crypto_blkcipher *null;
+	struct crypto_skcipher *null;
 	u8 nonce[4];
 };
 	struct crypto_gcm_ghash_ctx ghash_ctx;
 	union {
 		struct ahash_request ahreq;
-		struct ablkcipher_request abreq;
+		struct skcipher_request skreq;
 	} u;
 };
 {
 	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
 	struct crypto_ahash *ghash = ctx->ghash;
-	struct crypto_ablkcipher *ctr = ctx->ctr;
+	struct crypto_skcipher *ctr = ctx->ctr;
 	struct {
 		be128 hash;
 		u8 iv[8];
 		struct crypto_gcm_setkey_result result;
 		struct scatterlist sg[1];
-		struct ablkcipher_request req;
+		struct skcipher_request req;
 	} *data;
 	int err;
-	crypto_ablkcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
+	crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
-	crypto_ablkcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
+	crypto_skcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
-					 CRYPTO_TFM_REQ_MASK);
+				       CRYPTO_TFM_REQ_MASK);
-	err = crypto_ablkcipher_setkey(ctr, key, keylen);
+	err = crypto_skcipher_setkey(ctr, key, keylen);
-	crypto_aead_set_flags(aead, crypto_ablkcipher_get_flags(ctr) &
+	crypto_aead_set_flags(aead, crypto_skcipher_get_flags(ctr) &
 				    CRYPTO_TFM_RES_MASK);
 	if (err)
 		return err;
-	data = kzalloc(sizeof(*data) + crypto_ablkcipher_reqsize(ctr),
+	data = kzalloc(sizeof(*data) + crypto_skcipher_reqsize(ctr),
 		       GFP_KERNEL);
 	if (!data)
 		return -ENOMEM;
 	init_completion(&data->result.completion);
 	sg_init_one(data->sg, &data->hash, sizeof(data->hash));
-	ablkcipher_request_set_tfm(&data->req, ctr);
+	skcipher_request_set_tfm(&data->req, ctr);
-	ablkcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |
+	skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |
-						    CRYPTO_TFM_REQ_MAY_BACKLOG,
+						  CRYPTO_TFM_REQ_MAY_BACKLOG,
-					crypto_gcm_setkey_done,
+				      crypto_gcm_setkey_done,
-					&data->result);
+				      &data->result);
-	ablkcipher_request_set_crypt(&data->req, data->sg, data->sg,
+	skcipher_request_set_crypt(&data->req, data->sg, data->sg,
-				     sizeof(data->hash), data->iv);
+				   sizeof(data->hash), data->iv);
-	err = crypto_ablkcipher_encrypt(&data->req);
+	err = crypto_skcipher_encrypt(&data->req);
 	if (err == -EINPROGRESS || err == -EBUSY) {
 		err = wait_for_completion_interruptible(
 			&data->result.completion);
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
 	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
-	struct ablkcipher_request *ablk_req = &pctx->u.abreq;
+	struct skcipher_request *skreq = &pctx->u.skreq;
 	struct scatterlist *dst;
 	dst = req->src == req->dst ? pctx->src : pctx->dst;
-	ablkcipher_request_set_tfm(ablk_req, ctx->ctr);
+	skcipher_request_set_tfm(skreq, ctx->ctr);
-	ablkcipher_request_set_crypt(ablk_req, pctx->src, dst,
+	skcipher_request_set_crypt(skreq, pctx->src, dst,
 				     cryptlen + sizeof(pctx->auth_tag),
 				     pctx->iv);
 }
 static int crypto_gcm_encrypt(struct aead_request *req)
 {
 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
-	struct ablkcipher_request *abreq = &pctx->u.abreq;
+	struct skcipher_request *skreq = &pctx->u.skreq;
 	u32 flags = aead_request_flags(req);
 	crypto_gcm_init_common(req);
 	crypto_gcm_init_crypt(req, req->cryptlen);
-	ablkcipher_request_set_callback(abreq, flags, gcm_encrypt_done, req);
+	skcipher_request_set_callback(skreq, flags, gcm_encrypt_done, req);
-	return crypto_ablkcipher_encrypt(abreq) ?:
+	return crypto_skcipher_encrypt(skreq) ?:
 	       gcm_encrypt_continue(req, flags);
 }
 static int gcm_dec_hash_continue(struct aead_request *req, u32 flags)
 {
 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
-	struct ablkcipher_request *abreq = &pctx->u.abreq;
+	struct skcipher_request *skreq = &pctx->u.skreq;
 	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
 	crypto_gcm_init_crypt(req, gctx->cryptlen);
-	ablkcipher_request_set_callback(abreq, flags, gcm_decrypt_done, req);
+	skcipher_request_set_callback(skreq, flags, gcm_decrypt_done, req);
-	return crypto_ablkcipher_decrypt(abreq) ?: crypto_gcm_verify(req);
+	return crypto_skcipher_decrypt(skreq) ?: crypto_gcm_verify(req);
 }
 static int crypto_gcm_decrypt(struct aead_request *req)
 	struct aead_instance *inst = aead_alg_instance(tfm);
 	struct gcm_instance_ctx *ictx = aead_instance_ctx(inst);
 	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm);
-	struct crypto_ablkcipher *ctr;
+	struct crypto_skcipher *ctr;
 	struct crypto_ahash *ghash;
 	unsigned long align;
 	int err;
 	if (IS_ERR(ghash))
 		return PTR_ERR(ghash);
-	ctr = crypto_spawn_skcipher(&ictx->ctr);
+	ctr = crypto_spawn_skcipher2(&ictx->ctr);
 	err = PTR_ERR(ctr);
 	if (IS_ERR(ctr))
 		goto err_free_hash;
 	align &= ~(crypto_tfm_ctx_alignment() - 1);
 	crypto_aead_set_reqsize(tfm,
 		align + offsetof(struct crypto_gcm_req_priv_ctx, u) +
-		max(sizeof(struct ablkcipher_request) +
+		max(sizeof(struct skcipher_request) +
-		    crypto_ablkcipher_reqsize(ctr),
+		    crypto_skcipher_reqsize(ctr),
 		    sizeof(struct ahash_request) +
 		    crypto_ahash_reqsize(ghash)));
 	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm);
 	crypto_free_ahash(ctx->ghash);
-	crypto_free_ablkcipher(ctx->ctr);
+	crypto_free_skcipher(ctx->ctr);
 }
 static void crypto_gcm_free(struct aead_instance *inst)
 {
 	struct crypto_attr_type *algt;
 	struct aead_instance *inst;
-	struct crypto_alg *ctr;
+	struct skcipher_alg *ctr;
 	struct crypto_alg *ghash_alg;
 	struct hash_alg_common *ghash;
 	struct gcm_instance_ctx *ctx;
 	ghash_alg = crypto_find_alg(ghash_name, &crypto_ahash_type,
 				    CRYPTO_ALG_TYPE_HASH,
-				    CRYPTO_ALG_TYPE_AHASH_MASK);
+				    CRYPTO_ALG_TYPE_AHASH_MASK |
+				    crypto_requires_sync(algt->type,
+							 algt->mask));
 	if (IS_ERR(ghash_alg))
 		return PTR_ERR(ghash_alg);
 		goto err_drop_ghash;
 	crypto_set_skcipher_spawn(&ctx->ctr, aead_crypto_instance(inst));
-	err = crypto_grab_skcipher(&ctx->ctr, ctr_name, 0,
+	err = crypto_grab_skcipher2(&ctx->ctr, ctr_name, 0,
-				   crypto_requires_sync(algt->type,
+				    crypto_requires_sync(algt->type,
-							algt->mask));
+							 algt->mask));
 	if (err)
 		goto err_drop_ghash;
-	ctr = crypto_skcipher_spawn_alg(&ctx->ctr);
+	ctr = crypto_spawn_skcipher_alg(&ctx->ctr);
 	/* We only support 16-byte blocks. */
-	if (ctr->cra_ablkcipher.ivsize != 16)
+	if (crypto_skcipher_alg_ivsize(ctr) != 16)
 		goto out_put_ctr;
 	/* Not a stream cipher? */
 	err = -EINVAL;
-	if (ctr->cra_blocksize != 1)
+	if (ctr->base.cra_blocksize != 1)
 		goto out_put_ctr;
 	err = -ENAMETOOLONG;
 	if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
-		     "gcm_base(%s,%s)", ctr->cra_driver_name,
+		     "gcm_base(%s,%s)", ctr->base.cra_driver_name,
 		     ghash_alg->cra_driver_name) >=
 	    CRYPTO_MAX_ALG_NAME)
 		goto out_put_ctr;
 	memcpy(inst->alg.base.cra_name, full_name, CRYPTO_MAX_ALG_NAME);
-	inst->alg.base.cra_flags = (ghash->base.cra_flags | ctr->cra_flags) &
+	inst->alg.base.cra_flags = (ghash->base.cra_flags |
-				   CRYPTO_ALG_ASYNC;
+				    ctr->base.cra_flags) & CRYPTO_ALG_ASYNC;
 	inst->alg.base.cra_priority = (ghash->base.cra_priority +
-				       ctr->cra_priority) / 2;
+				       ctr->base.cra_priority) / 2;
 	inst->alg.base.cra_blocksize = 1;
 	inst->alg.base.cra_alignmask = ghash->base.cra_alignmask |
-				       ctr->cra_alignmask;
+				       ctr->base.cra_alignmask;
 	inst->alg.base.cra_ctxsize = sizeof(struct crypto_gcm_ctx);
 	inst->alg.ivsize = 12;
+	inst->alg.chunksize = crypto_skcipher_alg_chunksize(ctr);
 	inst->alg.maxauthsize = 16;
 	inst->alg.init = crypto_gcm_init_tfm;
 	inst->alg.exit = crypto_gcm_exit_tfm;
 	inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4106_ctx);
 	inst->alg.ivsize = 8;
+	inst->alg.chunksize = crypto_aead_alg_chunksize(alg);
 	inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
 	inst->alg.init = crypto_rfc4106_init_tfm;
 	unsigned int authsize = crypto_aead_authsize(aead);
 	unsigned int nbytes = req->assoclen + req->cryptlen -
 			      (enc ? 0 : authsize);
-	struct blkcipher_desc desc = {
+	SKCIPHER_REQUEST_ON_STACK(nreq, ctx->null);
-		.tfm = ctx->null,
-	};
-	return crypto_blkcipher_encrypt(&desc, req->dst, req->src, nbytes);
+	skcipher_request_set_tfm(nreq, ctx->null);
+	skcipher_request_set_callback(nreq, req->base.flags, NULL, NULL);
+	skcipher_request_set_crypt(nreq, req->src, req->dst, nbytes, NULL);
+	return crypto_skcipher_encrypt(nreq);
 }
 static int crypto_rfc4543_encrypt(struct aead_request *req)
 	struct crypto_aead_spawn *spawn = &ictx->aead;
 	struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm);
 	struct crypto_aead *aead;
-	struct crypto_blkcipher *null;
+	struct crypto_skcipher *null;
 	unsigned long align;
 	int err = 0;
 	if (IS_ERR(aead))
 		return PTR_ERR(aead);
-	null = crypto_get_default_null_skcipher();
+	null = crypto_get_default_null_skcipher2();
 	err = PTR_ERR(null);
 	if (IS_ERR(null))
 		goto err_free_aead;
 	struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm);
 	crypto_free_aead(ctx->child);
-	crypto_put_default_null_skcipher();
+	crypto_put_default_null_skcipher2();
 }
 static void crypto_rfc4543_free(struct aead_instance *inst)
 	inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4543_ctx);
 	inst->alg.ivsize = 8;
+	inst->alg.chunksize = crypto_aead_alg_chunksize(alg);
 	inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
 	inst->alg.init = crypto_rfc4543_init_tfm;

crypto/jitterentropy-kcapi.c+13−9

 	memcpy(dest, src, n);
 }
+/*
+ * Obtain a high-resolution time stamp value. The time stamp is used to measure
+ * the execution time of a given code path and its variations. Hence, the time
+ * stamp must have a sufficiently high resolution.
+ *
+ * Note, if the function returns zero because a given architecture does not
+ * implement a high-resolution time stamp, the RNG code's runtime test
+ * will detect it and will not produce output.
+ */
 void jent_get_nstime(__u64 *out)
 {
-	struct timespec ts;
 	__u64 tmp = 0;
 	tmp = random_get_entropy();
 	/*
-	 * If random_get_entropy does not return a value (which is possible on,
+	 * If random_get_entropy does not return a value, i.e. it is not
-	 * for example, MIPS), invoke __getnstimeofday
+	 * implemented for a given architecture, use a clock source.
 	 * hoping that there are timers we can work with.
 	 */
-	if ((0 == tmp) &&
+	if (tmp == 0)
-	   (0 == __getnstimeofday(&ts))) {
+		tmp = ktime_get_ns();
-		tmp = ts.tv_sec;
-		tmp = tmp << 32;
-		tmp = tmp | ts.tv_nsec;
-	}
 	*out = tmp;
 }

crypto/kpp.c+123−0

@@ -0,0 +1,123 @@
	1	/*
	2	* Key-agreement Protocol Primitives (KPP)
	3	*
	4	* Copyright (c) 2016, Intel Corporation
	5	* Authors: Salvatore Benedetto <salvatore.benedetto@intel.com>
	6	*
	7	* This program is free software; you can redistribute it and/or modify it
	8	* under the terms of the GNU General Public License as published by the Free
	9	* Software Foundation; either version 2 of the License, or (at your option)
	10	* any later version.
	11	*
	12	*/
	13	#include <linux/errno.h>
	14	#include <linux/kernel.h>
	15	#include <linux/module.h>
	16	#include <linux/seq_file.h>
	17	#include <linux/slab.h>
	18	#include <linux/string.h>
	19	#include <linux/crypto.h>
	20	#include <crypto/algapi.h>
	21	#include <linux/cryptouser.h>
	22	#include <net/netlink.h>
	23	#include <crypto/kpp.h>
	24	#include <crypto/internal/kpp.h>
	25	#include "internal.h"
	26
	27	#ifdef CONFIG_NET
	28	static int crypto_kpp_report(struct sk_buff skb, struct crypto_alg alg)
	29	{
	30	struct crypto_report_kpp rkpp;
	31
	32	strncpy(rkpp.type, "kpp", sizeof(rkpp.type));
	33
	34	if (nla_put(skb, CRYPTOCFGA_REPORT_KPP,
	35	sizeof(struct crypto_report_kpp), &rkpp))
	36	goto nla_put_failure;
	37	return 0;
	38
	39	nla_put_failure:
	40	return -EMSGSIZE;
	41	}
	42	#else
	43	static int crypto_kpp_report(struct sk_buff skb, struct crypto_alg alg)
	44	{
	45	return -ENOSYS;
	46	}
	47	#endif
	48
	49	static void crypto_kpp_show(struct seq_file m, struct crypto_alg alg)
	50	__attribute__ ((unused));
	51
	52	static void crypto_kpp_show(struct seq_file m, struct crypto_alg alg)
	53	{
	54	seq_puts(m, "type : kpp\n");
	55	}
	56
	57	static void crypto_kpp_exit_tfm(struct crypto_tfm *tfm)
	58	{
	59	struct crypto_kpp *kpp = __crypto_kpp_tfm(tfm);
	60	struct kpp_alg *alg = crypto_kpp_alg(kpp);
	61
	62	alg->exit(kpp);
	63	}
	64
	65	static int crypto_kpp_init_tfm(struct crypto_tfm *tfm)
	66	{
	67	struct crypto_kpp *kpp = __crypto_kpp_tfm(tfm);
	68	struct kpp_alg *alg = crypto_kpp_alg(kpp);
	69
	70	if (alg->exit)
	71	kpp->base.exit = crypto_kpp_exit_tfm;
	72
	73	if (alg->init)
	74	return alg->init(kpp);
	75
	76	return 0;
	77	}
	78
	79	static const struct crypto_type crypto_kpp_type = {
	80	.extsize = crypto_alg_extsize,
	81	.init_tfm = crypto_kpp_init_tfm,
	82	#ifdef CONFIG_PROC_FS
	83	.show = crypto_kpp_show,
	84	#endif
	85	.report = crypto_kpp_report,
	86	.maskclear = ~CRYPTO_ALG_TYPE_MASK,
	87	.maskset = CRYPTO_ALG_TYPE_MASK,
	88	.type = CRYPTO_ALG_TYPE_KPP,
	89	.tfmsize = offsetof(struct crypto_kpp, base),
	90	};
	91
	92	struct crypto_kpp crypto_alloc_kpp(const char alg_name, u32 type, u32 mask)
	93	{
	94	return crypto_alloc_tfm(alg_name, &crypto_kpp_type, type, mask);
	95	}
	96	EXPORT_SYMBOL_GPL(crypto_alloc_kpp);
	97
	98	static void kpp_prepare_alg(struct kpp_alg *alg)
	99	{
	100	struct crypto_alg *base = &alg->base;
	101
	102	base->cra_type = &crypto_kpp_type;
	103	base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK;
	104	base->cra_flags \|= CRYPTO_ALG_TYPE_KPP;
	105	}
	106
	107	int crypto_register_kpp(struct kpp_alg *alg)
	108	{
	109	struct crypto_alg *base = &alg->base;
	110
	111	kpp_prepare_alg(alg);
	112	return crypto_register_alg(base);
	113	}
	114	EXPORT_SYMBOL_GPL(crypto_register_kpp);
	115
	116	void crypto_unregister_kpp(struct kpp_alg *alg)
	117	{
	118	crypto_unregister_alg(&alg->base);
	119	}
	120	EXPORT_SYMBOL_GPL(crypto_unregister_kpp);
	121
	122	MODULE_LICENSE("GPL");
	123	MODULE_DESCRIPTION("Key-agreement Protocol Primitives");

crypto/mcryptd.c+56−76

@@ -41,7 +41,7 @@ struct mcryptd_flush_list { · 40 unmodified lines
41	static struct mcryptd_flush_list __percpu *mcryptd_flist;	41	static struct mcryptd_flush_list __percpu *mcryptd_flist;
42		42
43	struct hashd_instance_ctx {	43	struct hashd_instance_ctx {
44	struct crypto_shash_spawn spawn;	44	struct crypto_ahash_spawn spawn;
45	struct mcryptd_queue *queue;	45	struct mcryptd_queue *queue;
46	};	46	};
47		47
@@ -272,18 +272,18 @@ static int mcryptd_hash_init_tfm(struct crypto_tfm *tfm) · 224 unmodified lines
272	{	272	{
273	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);	273	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
274	struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst);	274	struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst);
275	struct crypto_shash_spawn *spawn = &ictx->spawn;	275	struct crypto_ahash_spawn *spawn = &ictx->spawn;
276	struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);	276	struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
277	struct crypto_shash *hash;	277	struct crypto_ahash *hash;
278		278
279	hash = crypto_spawn_shash(spawn);	279	hash = crypto_spawn_ahash(spawn);
280	if (IS_ERR(hash))	280	if (IS_ERR(hash))
281	return PTR_ERR(hash);	281	return PTR_ERR(hash);
282		282
283	ctx->child = hash;	283	ctx->child = hash;
284	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),	284	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
285	sizeof(struct mcryptd_hash_request_ctx) +	285	sizeof(struct mcryptd_hash_request_ctx) +
286	crypto_shash_descsize(hash));	286	crypto_ahash_reqsize(hash));
287	return 0;	287	return 0;
288	}	288	}
289		289
@@ -291,21 +291,21 @@ static void mcryptd_hash_exit_tfm(struct crypto_tfm *tfm) · 1 unmodified line
291	{	291	{
292	struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);	292	struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
293		293
294	crypto_free_shash(ctx->child);	294	crypto_free_ahash(ctx->child);
295	}	295	}
296		296
297	static int mcryptd_hash_setkey(struct crypto_ahash *parent,	297	static int mcryptd_hash_setkey(struct crypto_ahash *parent,
298	const u8 *key, unsigned int keylen)	298	const u8 *key, unsigned int keylen)
299	{	299	{
300	struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(parent);	300	struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(parent);
301	struct crypto_shash *child = ctx->child;	301	struct crypto_ahash *child = ctx->child;
302	int err;	302	int err;
303		303
304	crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK);	304	crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
305	crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) &	305	crypto_ahash_set_flags(child, crypto_ahash_get_flags(parent) &
306	CRYPTO_TFM_REQ_MASK);	306	CRYPTO_TFM_REQ_MASK);
307	err = crypto_shash_setkey(child, key, keylen);	307	err = crypto_ahash_setkey(child, key, keylen);
308	crypto_ahash_set_flags(parent, crypto_shash_get_flags(child) &	308	crypto_ahash_set_flags(parent, crypto_ahash_get_flags(child) &
309	CRYPTO_TFM_RES_MASK);	309	CRYPTO_TFM_RES_MASK);
310	return err;	310	return err;
311	}	311	}
@@ -331,20 +331,20 @@ static int mcryptd_hash_enqueue(struct ahash_request *req, · 19 unmodified lines
331	static void mcryptd_hash_init(struct crypto_async_request *req_async, int err)	331	static void mcryptd_hash_init(struct crypto_async_request *req_async, int err)
332	{	332	{
333	struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);	333	struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
334	struct crypto_shash *child = ctx->child;	334	struct crypto_ahash *child = ctx->child;
335	struct ahash_request *req = ahash_request_cast(req_async);	335	struct ahash_request *req = ahash_request_cast(req_async);
336	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);	336	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
337	struct shash_desc *desc = &rctx->desc;	337	struct ahash_request *desc = &rctx->areq;
338		338
339	if (unlikely(err == -EINPROGRESS))	339	if (unlikely(err == -EINPROGRESS))
340	goto out;	340	goto out;
341		341
342	desc->tfm = child;	342	ahash_request_set_tfm(desc, child);
343	desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;	343	ahash_request_set_callback(desc, CRYPTO_TFM_REQ_MAY_SLEEP,
		344	rctx->complete, req_async);
344		345
345	err = crypto_shash_init(desc);	346	rctx->out = req->result;
346		347	err = crypto_ahash_init(desc);
347	req->base.complete = rctx->complete;
348		348
349	out:	349	out:
350	local_bh_disable();	350	local_bh_disable();
@@ -365,7 +365,8 @@ static void mcryptd_hash_update(struct crypto_async_request *req_async, int err) · 14 unmodified lines
365	if (unlikely(err == -EINPROGRESS))	365	if (unlikely(err == -EINPROGRESS))
366	goto out;	366	goto out;
367		367
368	err = shash_ahash_mcryptd_update(req, &rctx->desc);	368	rctx->out = req->result;
		369	err = ahash_mcryptd_update(&rctx->areq);
369	if (err) {	370	if (err) {
370	req->base.complete = rctx->complete;	371	req->base.complete = rctx->complete;
371	goto out;	372	goto out;
@@ -391,7 +392,8 @@ static void mcryptd_hash_final(struct crypto_async_request *req_async, int err) · 19 unmodified lines
391	if (unlikely(err == -EINPROGRESS))	392	if (unlikely(err == -EINPROGRESS))
392	goto out;	393	goto out;
393		394
394	err = shash_ahash_mcryptd_final(req, &rctx->desc);	395	rctx->out = req->result;
		396	err = ahash_mcryptd_final(&rctx->areq);
395	if (err) {	397	if (err) {
396	req->base.complete = rctx->complete;	398	req->base.complete = rctx->complete;
397	goto out;	399	goto out;
@@ -416,8 +418,8 @@ static void mcryptd_hash_finup(struct crypto_async_request *req_async, int err) · 18 unmodified lines
416		418
417	if (unlikely(err == -EINPROGRESS))	419	if (unlikely(err == -EINPROGRESS))
418	goto out;	420	goto out;
419		421	rctx->out = req->result;
420	err = shash_ahash_mcryptd_finup(req, &rctx->desc);	422	err = ahash_mcryptd_finup(&rctx->areq);
421		423
422	if (err) {	424	if (err) {
423	req->base.complete = rctx->complete;	425	req->base.complete = rctx->complete;
@@ -439,25 +441,21 @@ static int mcryptd_hash_finup_enqueue(struct ahash_request *req) · 15 unmodified lines
439	static void mcryptd_hash_digest(struct crypto_async_request *req_async, int err)	441	static void mcryptd_hash_digest(struct crypto_async_request *req_async, int err)
440	{	442	{
441	struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);	443	struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
442	struct crypto_shash *child = ctx->child;	444	struct crypto_ahash *child = ctx->child;
443	struct ahash_request *req = ahash_request_cast(req_async);	445	struct ahash_request *req = ahash_request_cast(req_async);
444	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);	446	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
445	struct shash_desc *desc = &rctx->desc;	447	struct ahash_request *desc = &rctx->areq;
446		448
447	if (unlikely(err == -EINPROGRESS))	449	if (unlikely(err == -EINPROGRESS))
448	goto out;	450	goto out;
449		451
450	desc->tfm = child;	452	ahash_request_set_tfm(desc, child);
451	desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; /* check this again */	453	ahash_request_set_callback(desc, CRYPTO_TFM_REQ_MAY_SLEEP,
452		454	rctx->complete, req_async);
453	err = shash_ahash_mcryptd_digest(req, desc);
454		455
455	if (err) {	456	rctx->out = req->result;
456	req->base.complete = rctx->complete;	457	err = ahash_mcryptd_digest(desc);
457	goto out;
458	}
459		458
460	return;
461	out:	459	out:
462	local_bh_disable();	460	local_bh_disable();
463	rctx->complete(&req->base, err);	461	rctx->complete(&req->base, err);
@@ -473,14 +471,14 @@ static int mcryptd_hash_export(struct ahash_request req, void out) · 9 unmodified lines
473	{	471	{
474	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);	472	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
475		473
476	return crypto_shash_export(&rctx->desc, out);	474	return crypto_ahash_export(&rctx->areq, out);
477	}	475	}
478		476
479	static int mcryptd_hash_import(struct ahash_request req, const void in)	477	static int mcryptd_hash_import(struct ahash_request req, const void in)
480	{	478	{
481	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);	479	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
482		480
483	return crypto_shash_import(&rctx->desc, in);	481	return crypto_ahash_import(&rctx->areq, in);
484	}	482	}
485		483
486	static int mcryptd_create_hash(struct crypto_template tmpl, struct rtattr *tb,	484	static int mcryptd_create_hash(struct crypto_template tmpl, struct rtattr *tb,
@@ -488,7 +486,7 @@ static int mcryptd_create_hash(struct crypto_template tmpl, struct rtattr *tb, · 1 unmodified line
488	{	486	{
489	struct hashd_instance_ctx *ctx;	487	struct hashd_instance_ctx *ctx;
490	struct ahash_instance *inst;	488	struct ahash_instance *inst;
491	struct shash_alg *salg;	489	struct hash_alg_common *halg;
492	struct crypto_alg *alg;	490	struct crypto_alg *alg;
493	u32 type = 0;	491	u32 type = 0;
494	u32 mask = 0;	492	u32 mask = 0;
@@ -496,11 +494,11 @@ static int mcryptd_create_hash(struct crypto_template tmpl, struct rtattr *tb, · 1 unmodified line
496		494
497	mcryptd_check_internal(tb, &type, &mask);	495	mcryptd_check_internal(tb, &type, &mask);
498		496
499	salg = shash_attr_alg(tb[1], type, mask);	497	halg = ahash_attr_alg(tb[1], type, mask);
500	if (IS_ERR(salg))	498	if (IS_ERR(halg))
501	return PTR_ERR(salg);	499	return PTR_ERR(halg);
502		500
503	alg = &salg->base;	501	alg = &halg->base;
504	pr_debug("crypto: mcryptd hash alg: %s\n", alg->cra_name);	502	pr_debug("crypto: mcryptd hash alg: %s\n", alg->cra_name);
505	inst = mcryptd_alloc_instance(alg, ahash_instance_headroom(),	503	inst = mcryptd_alloc_instance(alg, ahash_instance_headroom(),
506	sizeof(*ctx));	504	sizeof(*ctx));
@@ -511,7 +509,7 @@ static int mcryptd_create_hash(struct crypto_template tmpl, struct rtattr *tb, · 4 unmodified lines
511	ctx = ahash_instance_ctx(inst);	509	ctx = ahash_instance_ctx(inst);
512	ctx->queue = queue;	510	ctx->queue = queue;
513		511
514	err = crypto_init_shash_spawn(&ctx->spawn, salg,	512	err = crypto_init_ahash_spawn(&ctx->spawn, halg,
515	ahash_crypto_instance(inst));	513	ahash_crypto_instance(inst));
516	if (err)	514	if (err)
517	goto out_free_inst;	515	goto out_free_inst;
@@ -521,8 +519,8 @@ static int mcryptd_create_hash(struct crypto_template tmpl, struct rtattr *tb, · 3 unmodified lines
521	type \|= CRYPTO_ALG_INTERNAL;	519	type \|= CRYPTO_ALG_INTERNAL;
522	inst->alg.halg.base.cra_flags = type;	520	inst->alg.halg.base.cra_flags = type;
523		521
524	inst->alg.halg.digestsize = salg->digestsize;	522	inst->alg.halg.digestsize = halg->digestsize;
525	inst->alg.halg.statesize = salg->statesize;	523	inst->alg.halg.statesize = halg->statesize;
526	inst->alg.halg.base.cra_ctxsize = sizeof(struct mcryptd_hash_ctx);	524	inst->alg.halg.base.cra_ctxsize = sizeof(struct mcryptd_hash_ctx);
527		525
528	inst->alg.halg.base.cra_init = mcryptd_hash_init_tfm;	526	inst->alg.halg.base.cra_init = mcryptd_hash_init_tfm;
@@ -539,7 +537,7 @@ static int mcryptd_create_hash(struct crypto_template tmpl, struct rtattr *tb, · 10 unmodified lines
539		537
540	err = ahash_register_instance(tmpl, inst);	538	err = ahash_register_instance(tmpl, inst);
541	if (err) {	539	if (err) {
542	crypto_drop_shash(&ctx->spawn);	540	crypto_drop_ahash(&ctx->spawn);
543	out_free_inst:	541	out_free_inst:
544	kfree(inst);	542	kfree(inst);
545	}	543	}
@@ -575,7 +573,7 @@ static void mcryptd_free(struct crypto_instance *inst) · 29 unmodified lines
575		573
576	switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) {	574	switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) {
577	case CRYPTO_ALG_TYPE_AHASH:	575	case CRYPTO_ALG_TYPE_AHASH:
578	crypto_drop_shash(&hctx->spawn);	576	crypto_drop_ahash(&hctx->spawn);
579	kfree(ahash_instance(inst));	577	kfree(ahash_instance(inst));
580	return;	578	return;
581	default:	579	default:
@@ -612,55 +610,38 @@ struct mcryptd_ahash mcryptd_alloc_ahash(const char alg_name, · 30 unmodified lines
612	}	610	}
613	EXPORT_SYMBOL_GPL(mcryptd_alloc_ahash);	611	EXPORT_SYMBOL_GPL(mcryptd_alloc_ahash);
614		612
615	int shash_ahash_mcryptd_digest(struct ahash_request *req,	613	int ahash_mcryptd_digest(struct ahash_request *desc)
616	struct shash_desc *desc)
617	{	614	{
618	int err;	615	int err;
619		616
620	err = crypto_shash_init(desc) ?:	617	err = crypto_ahash_init(desc) ?:
621	shash_ahash_mcryptd_finup(req, desc);	618	ahash_mcryptd_finup(desc);
622		619
623	return err;	620	return err;
624	}	621	}
625	EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_digest);
626		622
627	int shash_ahash_mcryptd_update(struct ahash_request *req,	623	int ahash_mcryptd_update(struct ahash_request *desc)
628	struct shash_desc *desc)
629	{	624	{
630	struct crypto_shash *tfm = desc->tfm;
631	struct shash_alg *shash = crypto_shash_alg(tfm);
632
633	/* alignment is to be done by multi-buffer crypto algorithm if needed */	625	/* alignment is to be done by multi-buffer crypto algorithm if needed */
634		626
635	return shash->update(desc, NULL, 0);	627	return crypto_ahash_update(desc);
636	}	628	}
637	EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_update);
638		629
639	int shash_ahash_mcryptd_finup(struct ahash_request *req,	630	int ahash_mcryptd_finup(struct ahash_request *desc)
640	struct shash_desc *desc)
641	{	631	{
642	struct crypto_shash *tfm = desc->tfm;
643	struct shash_alg *shash = crypto_shash_alg(tfm);
644
645	/* alignment is to be done by multi-buffer crypto algorithm if needed */	632	/* alignment is to be done by multi-buffer crypto algorithm if needed */
646		633
647	return shash->finup(desc, NULL, 0, req->result);	634	return crypto_ahash_finup(desc);
648	}	635	}
649	EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_finup);
650		636
651	int shash_ahash_mcryptd_final(struct ahash_request *req,	637	int ahash_mcryptd_final(struct ahash_request *desc)
652	struct shash_desc *desc)
653	{	638	{
654	struct crypto_shash *tfm = desc->tfm;
655	struct shash_alg *shash = crypto_shash_alg(tfm);
656
657	/* alignment is to be done by multi-buffer crypto algorithm if needed */	639	/* alignment is to be done by multi-buffer crypto algorithm if needed */
658		640
659	return shash->final(desc, req->result);	641	return crypto_ahash_final(desc);
660	}	642	}
661	EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_final);
662		643
663	struct crypto_shash mcryptd_ahash_child(struct mcryptd_ahash tfm)	644	struct crypto_ahash mcryptd_ahash_child(struct mcryptd_ahash tfm)
664	{	645	{
665	struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);	646	struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
666		647
@@ -668,12 +649,12 @@ struct crypto_shash mcryptd_ahash_child(struct mcryptd_ahash tfm) · 1 unmodified line
668	}	649	}
669	EXPORT_SYMBOL_GPL(mcryptd_ahash_child);	650	EXPORT_SYMBOL_GPL(mcryptd_ahash_child);
670		651
671	struct shash_desc mcryptd_shash_desc(struct ahash_request req)	652	struct ahash_request mcryptd_ahash_desc(struct ahash_request req)
672	{	653	{
673	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);	654	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
674	return &rctx->desc;	655	return &rctx->areq;
675	}	656	}
676	EXPORT_SYMBOL_GPL(mcryptd_shash_desc);	657	EXPORT_SYMBOL_GPL(mcryptd_ahash_desc);
677		658
678	void mcryptd_free_ahash(struct mcryptd_ahash *tfm)	659	void mcryptd_free_ahash(struct mcryptd_ahash *tfm)
679	{	660	{
@@ -681,7 +662,6 @@ void mcryptd_free_ahash(struct mcryptd_ahash *tfm) · 1 unmodified line
681	}	662	}
682	EXPORT_SYMBOL_GPL(mcryptd_free_ahash);	663	EXPORT_SYMBOL_GPL(mcryptd_free_ahash);
683		664
684
685	static int __init mcryptd_init(void)	665	static int __init mcryptd_init(void)
686	{	666	{
687	int err, cpu;	667	int err, cpu;

crypto/rsa-pkcs1pad.c+131−194

@@ -92,19 +92,17 @@ static const struct rsa_asn1_template rsa_lookup_asn1(const char name) · 91 unmodified lines
92		92
93	struct pkcs1pad_ctx {	93	struct pkcs1pad_ctx {
94	struct crypto_akcipher *child;	94	struct crypto_akcipher *child;
95	const char *hash_name;
96	unsigned int key_size;	95	unsigned int key_size;
97	};	96	};
98		97
99	struct pkcs1pad_inst_ctx {	98	struct pkcs1pad_inst_ctx {
100	struct crypto_akcipher_spawn spawn;	99	struct crypto_akcipher_spawn spawn;
101	const char *hash_name;	100	const struct rsa_asn1_template *digest_info;
102	};	101	};
103		102
104	struct pkcs1pad_request {	103	struct pkcs1pad_request {
105	struct scatterlist in_sg[3], out_sg[2];	104	struct scatterlist in_sg[2], out_sg[1];
106	uint8_t in_buf, out_buf;	105	uint8_t in_buf, out_buf;
107
108	struct akcipher_request child_req;	106	struct akcipher_request child_req;
109	};	107	};
110		108
@@ -112,40 +110,48 @@ static int pkcs1pad_set_pub_key(struct crypto_akcipher tfm, const void key, · 1 unmodified line
112	unsigned int keylen)	110	unsigned int keylen)
113	{	111	{
114	struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm);	112	struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm);
115	int err, size;	113	int err;
		114
		115	ctx->key_size = 0;
116		116
117	err = crypto_akcipher_set_pub_key(ctx->child, key, keylen);	117	err = crypto_akcipher_set_pub_key(ctx->child, key, keylen);
		118	if (err)
		119	return err;
118		120
119	if (!err) {	121	/* Find out new modulus size from rsa implementation */
120	/* Find out new modulus size from rsa implementation */	122	err = crypto_akcipher_maxsize(ctx->child);
121	size = crypto_akcipher_maxsize(ctx->child);	123	if (err < 0)
		124	return err;
122		125
123	ctx->key_size = size > 0 ? size : 0;	126	if (err > PAGE_SIZE)
124	if (size <= 0)	127	return -ENOTSUPP;
125	err = size;
126	}
127		128
128	return err;	129	ctx->key_size = err;
		130	return 0;
129	}	131	}
130		132
131	static int pkcs1pad_set_priv_key(struct crypto_akcipher tfm, const void key,	133	static int pkcs1pad_set_priv_key(struct crypto_akcipher tfm, const void key,
132	unsigned int keylen)	134	unsigned int keylen)
133	{	135	{
134	struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm);	136	struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm);
135	int err, size;	137	int err;
		138
		139	ctx->key_size = 0;
136		140
137	err = crypto_akcipher_set_priv_key(ctx->child, key, keylen);	141	err = crypto_akcipher_set_priv_key(ctx->child, key, keylen);
		142	if (err)
		143	return err;
138		144
139	if (!err) {	145	/* Find out new modulus size from rsa implementation */
140	/* Find out new modulus size from rsa implementation */	146	err = crypto_akcipher_maxsize(ctx->child);
141	size = crypto_akcipher_maxsize(ctx->child);	147	if (err < 0)
		148	return err;
142		149
143	ctx->key_size = size > 0 ? size : 0;	150	if (err > PAGE_SIZE)
144	if (size <= 0)	151	return -ENOTSUPP;
145	err = size;
146	}
147		152
148	return err;	153	ctx->key_size = err;
		154	return 0;
149	}	155	}
150		156
151	static int pkcs1pad_get_max_size(struct crypto_akcipher *tfm)	157	static int pkcs1pad_get_max_size(struct crypto_akcipher *tfm)
@@ -164,19 +170,10 @@ static int pkcs1pad_get_max_size(struct crypto_akcipher *tfm) · 12 unmodified lines
164	static void pkcs1pad_sg_set_buf(struct scatterlist sg, void buf, size_t len,	170	static void pkcs1pad_sg_set_buf(struct scatterlist sg, void buf, size_t len,
165	struct scatterlist *next)	171	struct scatterlist *next)
166	{	172	{
167	int nsegs = next ? 1 : 0;	173	int nsegs = next ? 2 : 1;
168		174
169	if (offset_in_page(buf) + len <= PAGE_SIZE) {	175	sg_init_table(sg, nsegs);
170	nsegs += 1;	176	sg_set_buf(sg, buf, len);
171	sg_init_table(sg, nsegs);
172	sg_set_buf(sg, buf, len);
173	} else {
174	nsegs += 2;
175	sg_init_table(sg, nsegs);
176	sg_set_buf(sg + 0, buf, PAGE_SIZE - offset_in_page(buf));
177	sg_set_buf(sg + 1, buf + PAGE_SIZE - offset_in_page(buf),
178	offset_in_page(buf) + len - PAGE_SIZE);
179	}
180		177
181	if (next)	178	if (next)
182	sg_chain(sg, nsegs, next);	179	sg_chain(sg, nsegs, next);
@@ -187,37 +184,36 @@ static int pkcs1pad_encrypt_sign_complete(struct akcipher_request *req, int err) · 4 unmodified lines
187	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);	184	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
188	struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm);	185	struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm);
189	struct pkcs1pad_request *req_ctx = akcipher_request_ctx(req);	186	struct pkcs1pad_request *req_ctx = akcipher_request_ctx(req);
190	size_t pad_len = ctx->key_size - req_ctx->child_req.dst_len;	187	unsigned int pad_len;
191	size_t chunk_len, pad_left;	188	unsigned int len;
192	struct sg_mapping_iter miter;	189	u8 *out_buf;
193		190
194	if (!err) {	191	if (err)
195	if (pad_len) {	192	goto out;
196	sg_miter_start(&miter, req->dst,	193
197	sg_nents_for_len(req->dst, pad_len),	194	len = req_ctx->child_req.dst_len;
198	SG_MITER_ATOMIC \| SG_MITER_TO_SG);	195	pad_len = ctx->key_size - len;
199		196
200	pad_left = pad_len;	197	/* Four billion to one */
201	while (pad_left) {	198	if (likely(!pad_len))
202	sg_miter_next(&miter);	199	goto out;
203		200
204	chunk_len = min(miter.length, pad_left);	201	out_buf = kzalloc(ctx->key_size, GFP_ATOMIC);
205	memset(miter.addr, 0, chunk_len);	202	err = -ENOMEM;
206	pad_left -= chunk_len;	203	if (!out_buf)
207	}	204	goto out;
208		205
209	sg_miter_stop(&miter);	206	sg_copy_to_buffer(req->dst, sg_nents_for_len(req->dst, len),
210	}	207	out_buf + pad_len, len);
211		208	sg_copy_from_buffer(req->dst,
212	sg_pcopy_from_buffer(req->dst,	209	sg_nents_for_len(req->dst, ctx->key_size),
213	sg_nents_for_len(req->dst, ctx->key_size),	210	out_buf, ctx->key_size);
214	req_ctx->out_buf, req_ctx->child_req.dst_len,	211	kzfree(out_buf);
215	pad_len);	212
216	}	213	out:
217	req->dst_len = ctx->key_size;	214	req->dst_len = ctx->key_size;
218		215
219	kfree(req_ctx->in_buf);	216	kfree(req_ctx->in_buf);
220	kzfree(req_ctx->out_buf);
221		217
222	return err;	218	return err;
223	}	219	}
@@ -257,21 +253,8 @@ static int pkcs1pad_encrypt(struct akcipher_request *req) · 33 unmodified lines
257	return -EOVERFLOW;	253	return -EOVERFLOW;
258	}	254	}
259		255
260	if (ctx->key_size > PAGE_SIZE)
261	return -ENOTSUPP;
262
263	/*
264	* Replace both input and output to add the padding in the input and
265	* the potential missing leading zeros in the output.
266	*/
267	req_ctx->child_req.src = req_ctx->in_sg;
268	req_ctx->child_req.src_len = ctx->key_size - 1;
269	req_ctx->child_req.dst = req_ctx->out_sg;
270	req_ctx->child_req.dst_len = ctx->key_size;
271
272	req_ctx->in_buf = kmalloc(ctx->key_size - 1 - req->src_len,	256	req_ctx->in_buf = kmalloc(ctx->key_size - 1 - req->src_len,
273	(req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?	257	GFP_KERNEL);
274	GFP_KERNEL : GFP_ATOMIC);
275	if (!req_ctx->in_buf)	258	if (!req_ctx->in_buf)
276	return -ENOMEM;	259	return -ENOMEM;
277		260
@@ -284,9 +267,7 @@ static int pkcs1pad_encrypt(struct akcipher_request *req) · 6 unmodified lines
284	pkcs1pad_sg_set_buf(req_ctx->in_sg, req_ctx->in_buf,	267	pkcs1pad_sg_set_buf(req_ctx->in_sg, req_ctx->in_buf,
285	ctx->key_size - 1 - req->src_len, req->src);	268	ctx->key_size - 1 - req->src_len, req->src);
286		269
287	req_ctx->out_buf = kmalloc(ctx->key_size,	270	req_ctx->out_buf = kmalloc(ctx->key_size, GFP_KERNEL);
288	(req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
289	GFP_KERNEL : GFP_ATOMIC);
290	if (!req_ctx->out_buf) {	271	if (!req_ctx->out_buf) {
291	kfree(req_ctx->in_buf);	272	kfree(req_ctx->in_buf);
292	return -ENOMEM;	273	return -ENOMEM;
@@ -299,6 +280,10 @@ static int pkcs1pad_encrypt(struct akcipher_request *req) · 6 unmodified lines
299	akcipher_request_set_callback(&req_ctx->child_req, req->base.flags,	280	akcipher_request_set_callback(&req_ctx->child_req, req->base.flags,
300	pkcs1pad_encrypt_sign_complete_cb, req);	281	pkcs1pad_encrypt_sign_complete_cb, req);
301		282
		283	/* Reuse output buffer */
		284	akcipher_request_set_crypt(&req_ctx->child_req, req_ctx->in_sg,
		285	req->dst, ctx->key_size - 1, req->dst_len);
		286
302	err = crypto_akcipher_encrypt(&req_ctx->child_req);	287	err = crypto_akcipher_encrypt(&req_ctx->child_req);
303	if (err != -EINPROGRESS &&	288	if (err != -EINPROGRESS &&
304	(err != -EBUSY \|\|	289	(err != -EBUSY \|\|
@@ -380,18 +365,7 @@ static int pkcs1pad_decrypt(struct akcipher_request *req) · 75 unmodified lines
380	if (!ctx->key_size \|\| req->src_len != ctx->key_size)	365	if (!ctx->key_size \|\| req->src_len != ctx->key_size)
381	return -EINVAL;	366	return -EINVAL;
382		367
383	if (ctx->key_size > PAGE_SIZE)	368	req_ctx->out_buf = kmalloc(ctx->key_size, GFP_KERNEL);
384	return -ENOTSUPP;
385
386	/* Reuse input buffer, output to a new buffer */
387	req_ctx->child_req.src = req->src;
388	req_ctx->child_req.src_len = req->src_len;
389	req_ctx->child_req.dst = req_ctx->out_sg;
390	req_ctx->child_req.dst_len = ctx->key_size ;
391
392	req_ctx->out_buf = kmalloc(ctx->key_size,
393	(req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
394	GFP_KERNEL : GFP_ATOMIC);
395	if (!req_ctx->out_buf)	369	if (!req_ctx->out_buf)
396	return -ENOMEM;	370	return -ENOMEM;
397		371
@@ -402,6 +376,11 @@ static int pkcs1pad_decrypt(struct akcipher_request *req) · 4 unmodified lines
402	akcipher_request_set_callback(&req_ctx->child_req, req->base.flags,	376	akcipher_request_set_callback(&req_ctx->child_req, req->base.flags,
403	pkcs1pad_decrypt_complete_cb, req);	377	pkcs1pad_decrypt_complete_cb, req);
404		378
		379	/* Reuse input buffer, output to a new buffer */
		380	akcipher_request_set_crypt(&req_ctx->child_req, req->src,
		381	req_ctx->out_sg, req->src_len,
		382	ctx->key_size);
		383
405	err = crypto_akcipher_decrypt(&req_ctx->child_req);	384	err = crypto_akcipher_decrypt(&req_ctx->child_req);
406	if (err != -EINPROGRESS &&	385	if (err != -EINPROGRESS &&
407	(err != -EBUSY \|\|	386	(err != -EBUSY \|\|
@@ -416,20 +395,16 @@ static int pkcs1pad_sign(struct akcipher_request *req) · 8 unmodified lines
416	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);	395	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
417	struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm);	396	struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm);
418	struct pkcs1pad_request *req_ctx = akcipher_request_ctx(req);	397	struct pkcs1pad_request *req_ctx = akcipher_request_ctx(req);
419	const struct rsa_asn1_template *digest_info = NULL;	398	struct akcipher_instance *inst = akcipher_alg_instance(tfm);
		399	struct pkcs1pad_inst_ctx *ictx = akcipher_instance_ctx(inst);
		400	const struct rsa_asn1_template *digest_info = ictx->digest_info;
420	int err;	401	int err;
421	unsigned int ps_end, digest_size = 0;	402	unsigned int ps_end, digest_size = 0;
422		403
423	if (!ctx->key_size)	404	if (!ctx->key_size)
424	return -EINVAL;	405	return -EINVAL;
425		406
426	if (ctx->hash_name) {	407	digest_size = digest_info->size;
427	digest_info = rsa_lookup_asn1(ctx->hash_name);
428	if (!digest_info)
429	return -EINVAL;
430
431	digest_size = digest_info->size;
432	}
433		408
434	if (req->src_len + digest_size > ctx->key_size - 11)	409	if (req->src_len + digest_size > ctx->key_size - 11)
435	return -EOVERFLOW;	410	return -EOVERFLOW;
@@ -439,21 +414,8 @@ static int pkcs1pad_sign(struct akcipher_request *req) · 3 unmodified lines
439	return -EOVERFLOW;	414	return -EOVERFLOW;
440	}	415	}
441		416
442	if (ctx->key_size > PAGE_SIZE)
443	return -ENOTSUPP;
444
445	/*
446	* Replace both input and output to add the padding in the input and
447	* the potential missing leading zeros in the output.
448	*/
449	req_ctx->child_req.src = req_ctx->in_sg;
450	req_ctx->child_req.src_len = ctx->key_size - 1;
451	req_ctx->child_req.dst = req_ctx->out_sg;
452	req_ctx->child_req.dst_len = ctx->key_size;
453
454	req_ctx->in_buf = kmalloc(ctx->key_size - 1 - req->src_len,	417	req_ctx->in_buf = kmalloc(ctx->key_size - 1 - req->src_len,
455	(req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?	418	GFP_KERNEL);
456	GFP_KERNEL : GFP_ATOMIC);
457	if (!req_ctx->in_buf)	419	if (!req_ctx->in_buf)
458	return -ENOMEM;	420	return -ENOMEM;
459		421
@@ -462,29 +424,20 @@ static int pkcs1pad_sign(struct akcipher_request *req) · 2 unmodified lines
462	memset(req_ctx->in_buf + 1, 0xff, ps_end - 1);	424	memset(req_ctx->in_buf + 1, 0xff, ps_end - 1);
463	req_ctx->in_buf[ps_end] = 0x00;	425	req_ctx->in_buf[ps_end] = 0x00;
464		426
465	if (digest_info) {	427	memcpy(req_ctx->in_buf + ps_end + 1, digest_info->data,
466	memcpy(req_ctx->in_buf + ps_end + 1, digest_info->data,	428	digest_info->size);
467	digest_info->size);
468	}
469		429
470	pkcs1pad_sg_set_buf(req_ctx->in_sg, req_ctx->in_buf,	430	pkcs1pad_sg_set_buf(req_ctx->in_sg, req_ctx->in_buf,
471	ctx->key_size - 1 - req->src_len, req->src);	431	ctx->key_size - 1 - req->src_len, req->src);
472		432
473	req_ctx->out_buf = kmalloc(ctx->key_size,
474	(req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
475	GFP_KERNEL : GFP_ATOMIC);
476	if (!req_ctx->out_buf) {
477	kfree(req_ctx->in_buf);
478	return -ENOMEM;
479	}
480
481	pkcs1pad_sg_set_buf(req_ctx->out_sg, req_ctx->out_buf,
482	ctx->key_size, NULL);
483
484	akcipher_request_set_tfm(&req_ctx->child_req, ctx->child);	433	akcipher_request_set_tfm(&req_ctx->child_req, ctx->child);
485	akcipher_request_set_callback(&req_ctx->child_req, req->base.flags,	434	akcipher_request_set_callback(&req_ctx->child_req, req->base.flags,
486	pkcs1pad_encrypt_sign_complete_cb, req);	435	pkcs1pad_encrypt_sign_complete_cb, req);
487		436
		437	/* Reuse output buffer */
		438	akcipher_request_set_crypt(&req_ctx->child_req, req_ctx->in_sg,
		439	req->dst, ctx->key_size - 1, req->dst_len);
		440
488	err = crypto_akcipher_sign(&req_ctx->child_req);	441	err = crypto_akcipher_sign(&req_ctx->child_req);
489	if (err != -EINPROGRESS &&	442	if (err != -EINPROGRESS &&
490	(err != -EBUSY \|\|	443	(err != -EBUSY \|\|
@@ -499,56 +452,58 @@ static int pkcs1pad_verify_complete(struct akcipher_request *req, int err) · 8 unmodified lines
499	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);	452	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
500	struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm);	453	struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm);
501	struct pkcs1pad_request *req_ctx = akcipher_request_ctx(req);	454	struct pkcs1pad_request *req_ctx = akcipher_request_ctx(req);
502	const struct rsa_asn1_template *digest_info;	455	struct akcipher_instance *inst = akcipher_alg_instance(tfm);
		456	struct pkcs1pad_inst_ctx *ictx = akcipher_instance_ctx(inst);
		457	const struct rsa_asn1_template *digest_info = ictx->digest_info;
		458	unsigned int dst_len;
503	unsigned int pos;	459	unsigned int pos;
504		460	u8 *out_buf;
505	if (err == -EOVERFLOW)
506	/* Decrypted value had no leading 0 byte */
507	err = -EINVAL;
508		461
509	if (err)	462	if (err)
510	goto done;	463	goto done;
511		464
512	if (req_ctx->child_req.dst_len != ctx->key_size - 1) {	465	err = -EINVAL;
513	err = -EINVAL;	466	dst_len = req_ctx->child_req.dst_len;
		467	if (dst_len < ctx->key_size - 1)
514	goto done;	468	goto done;
		469
		470	out_buf = req_ctx->out_buf;
		471	if (dst_len == ctx->key_size) {
		472	if (out_buf[0] != 0x00)
		473	/* Decrypted value had no leading 0 byte */
		474	goto done;
		475
		476	dst_len--;
		477	out_buf++;
515	}	478	}
516		479
517	err = -EBADMSG;	480	err = -EBADMSG;
518	if (req_ctx->out_buf[0] != 0x01)	481	if (out_buf[0] != 0x01)
519	goto done;	482	goto done;
520		483
521	for (pos = 1; pos < req_ctx->child_req.dst_len; pos++)	484	for (pos = 1; pos < dst_len; pos++)
522	if (req_ctx->out_buf[pos] != 0xff)	485	if (out_buf[pos] != 0xff)
523	break;	486	break;
524		487
525	if (pos < 9 \|\| pos == req_ctx->child_req.dst_len \|\|	488	if (pos < 9 \|\| pos == dst_len \|\| out_buf[pos] != 0x00)
526	req_ctx->out_buf[pos] != 0x00)
527	goto done;	489	goto done;
528	pos++;	490	pos++;
529		491
530	if (ctx->hash_name) {	492	if (memcmp(out_buf + pos, digest_info->data, digest_info->size))
531	digest_info = rsa_lookup_asn1(ctx->hash_name);	493	goto done;
532	if (!digest_info)
533	goto done;
534
535	if (memcmp(req_ctx->out_buf + pos, digest_info->data,
536	digest_info->size))
537	goto done;
538		494
539	pos += digest_info->size;	495	pos += digest_info->size;
540	}
541		496
542	err = 0;	497	err = 0;
543		498
544	if (req->dst_len < req_ctx->child_req.dst_len - pos)	499	if (req->dst_len < dst_len - pos)
545	err = -EOVERFLOW;	500	err = -EOVERFLOW;
546	req->dst_len = req_ctx->child_req.dst_len - pos;	501	req->dst_len = dst_len - pos;
547		502
548	if (!err)	503	if (!err)
549	sg_copy_from_buffer(req->dst,	504	sg_copy_from_buffer(req->dst,
550	sg_nents_for_len(req->dst, req->dst_len),	505	sg_nents_for_len(req->dst, req->dst_len),
551	req_ctx->out_buf + pos, req->dst_len);	506	out_buf + pos, req->dst_len);
552	done:	507	done:
553	kzfree(req_ctx->out_buf);	508	kzfree(req_ctx->out_buf);
554		509
@@ -588,18 +543,7 @@ static int pkcs1pad_verify(struct akcipher_request *req) · 33 unmodified lines
588	if (!ctx->key_size \|\| req->src_len < ctx->key_size)	543	if (!ctx->key_size \|\| req->src_len < ctx->key_size)
589	return -EINVAL;	544	return -EINVAL;
590		545
591	if (ctx->key_size > PAGE_SIZE)	546	req_ctx->out_buf = kmalloc(ctx->key_size, GFP_KERNEL);
592	return -ENOTSUPP;
593
594	/* Reuse input buffer, output to a new buffer */
595	req_ctx->child_req.src = req->src;
596	req_ctx->child_req.src_len = req->src_len;
597	req_ctx->child_req.dst = req_ctx->out_sg;
598	req_ctx->child_req.dst_len = ctx->key_size;
599
600	req_ctx->out_buf = kmalloc(ctx->key_size,
601	(req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
602	GFP_KERNEL : GFP_ATOMIC);
603	if (!req_ctx->out_buf)	547	if (!req_ctx->out_buf)
604	return -ENOMEM;	548	return -ENOMEM;
605		549
@@ -610,6 +554,11 @@ static int pkcs1pad_verify(struct akcipher_request *req) · 4 unmodified lines
610	akcipher_request_set_callback(&req_ctx->child_req, req->base.flags,	554	akcipher_request_set_callback(&req_ctx->child_req, req->base.flags,
611	pkcs1pad_verify_complete_cb, req);	555	pkcs1pad_verify_complete_cb, req);
612		556
		557	/* Reuse input buffer, output to a new buffer */
		558	akcipher_request_set_crypt(&req_ctx->child_req, req->src,
		559	req_ctx->out_sg, req->src_len,
		560	ctx->key_size);
		561
613	err = crypto_akcipher_verify(&req_ctx->child_req);	562	err = crypto_akcipher_verify(&req_ctx->child_req);
614	if (err != -EINPROGRESS &&	563	if (err != -EINPROGRESS &&
615	(err != -EBUSY \|\|	564	(err != -EBUSY \|\|
@@ -626,12 +575,11 @@ static int pkcs1pad_init_tfm(struct crypto_akcipher *tfm) · 10 unmodified lines
626	struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm);	575	struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm);
627	struct crypto_akcipher *child_tfm;	576	struct crypto_akcipher *child_tfm;
628		577
629	child_tfm = crypto_spawn_akcipher(akcipher_instance_ctx(inst));	578	child_tfm = crypto_spawn_akcipher(&ictx->spawn);
630	if (IS_ERR(child_tfm))	579	if (IS_ERR(child_tfm))
631	return PTR_ERR(child_tfm);	580	return PTR_ERR(child_tfm);
632		581
633	ctx->child = child_tfm;	582	ctx->child = child_tfm;
634	ctx->hash_name = ictx->hash_name;
635	return 0;	583	return 0;
636	}	584	}
637		585
@@ -648,12 +596,12 @@ static void pkcs1pad_free(struct akcipher_instance *inst) · 10 unmodified lines
648	struct crypto_akcipher_spawn *spawn = &ctx->spawn;	596	struct crypto_akcipher_spawn *spawn = &ctx->spawn;
649		597
650	crypto_drop_akcipher(spawn);	598	crypto_drop_akcipher(spawn);
651	kfree(ctx->hash_name);
652	kfree(inst);	599	kfree(inst);
653	}	600	}
654		601
655	static int pkcs1pad_create(struct crypto_template tmpl, struct rtattr *tb)	602	static int pkcs1pad_create(struct crypto_template tmpl, struct rtattr *tb)
656	{	603	{
		604	const struct rsa_asn1_template *digest_info;
657	struct crypto_attr_type *algt;	605	struct crypto_attr_type *algt;
658	struct akcipher_instance *inst;	606	struct akcipher_instance *inst;
659	struct pkcs1pad_inst_ctx *ctx;	607	struct pkcs1pad_inst_ctx *ctx;
@@ -676,7 +624,11 @@ static int pkcs1pad_create(struct crypto_template tmpl, struct rtattr *tb) · 16 unmodified lines
676		624
677	hash_name = crypto_attr_alg_name(tb[2]);	625	hash_name = crypto_attr_alg_name(tb[2]);
678	if (IS_ERR(hash_name))	626	if (IS_ERR(hash_name))
679	hash_name = NULL;	627	return PTR_ERR(hash_name);
		628
		629	digest_info = rsa_lookup_asn1(hash_name);
		630	if (!digest_info)
		631	return -EINVAL;
680		632
681	inst = kzalloc(sizeof(inst) + sizeof(ctx), GFP_KERNEL);	633	inst = kzalloc(sizeof(inst) + sizeof(ctx), GFP_KERNEL);
682	if (!inst)	634	if (!inst)
@@ -684,7 +636,7 @@ static int pkcs1pad_create(struct crypto_template tmpl, struct rtattr *tb) · 1 unmodified line
684		636
685	ctx = akcipher_instance_ctx(inst);	637	ctx = akcipher_instance_ctx(inst);
686	spawn = &ctx->spawn;	638	spawn = &ctx->spawn;
687	ctx->hash_name = hash_name ? kstrdup(hash_name, GFP_KERNEL) : NULL;	639	ctx->digest_info = digest_info;
688		640
689	crypto_set_spawn(&spawn->base, akcipher_crypto_instance(inst));	641	crypto_set_spawn(&spawn->base, akcipher_crypto_instance(inst));
690	err = crypto_grab_akcipher(spawn, rsa_alg_name, 0,	642	err = crypto_grab_akcipher(spawn, rsa_alg_name, 0,
@@ -696,27 +648,14 @@ static int pkcs1pad_create(struct crypto_template tmpl, struct rtattr *tb) · 5 unmodified lines
696		648
697	err = -ENAMETOOLONG;	649	err = -ENAMETOOLONG;
698		650
699	if (!hash_name) {	651	if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
700	if (snprintf(inst->alg.base.cra_name,	652	"pkcs1pad(%s,%s)", rsa_alg->base.cra_name, hash_name) >=
701	CRYPTO_MAX_ALG_NAME, "pkcs1pad(%s)",	653	CRYPTO_MAX_ALG_NAME \|\|
702	rsa_alg->base.cra_name) >=	654	snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
703	CRYPTO_MAX_ALG_NAME \|\|	655	"pkcs1pad(%s,%s)",
704	snprintf(inst->alg.base.cra_driver_name,	656	rsa_alg->base.cra_driver_name, hash_name) >=
705	CRYPTO_MAX_ALG_NAME, "pkcs1pad(%s)",	657	CRYPTO_MAX_ALG_NAME)
706	rsa_alg->base.cra_driver_name) >=
707	CRYPTO_MAX_ALG_NAME)
708	goto out_drop_alg;	658	goto out_drop_alg;
709	} else {
710	if (snprintf(inst->alg.base.cra_name,
711	CRYPTO_MAX_ALG_NAME, "pkcs1pad(%s,%s)",
712	rsa_alg->base.cra_name, hash_name) >=
713	CRYPTO_MAX_ALG_NAME \|\|
714	snprintf(inst->alg.base.cra_driver_name,
715	CRYPTO_MAX_ALG_NAME, "pkcs1pad(%s,%s)",
716	rsa_alg->base.cra_driver_name, hash_name) >=
717	CRYPTO_MAX_ALG_NAME)
718	goto out_free_hash;
719	}
720		659
721	inst->alg.base.cra_flags = rsa_alg->base.cra_flags & CRYPTO_ALG_ASYNC;	660	inst->alg.base.cra_flags = rsa_alg->base.cra_flags & CRYPTO_ALG_ASYNC;
722	inst->alg.base.cra_priority = rsa_alg->base.cra_priority;	661	inst->alg.base.cra_priority = rsa_alg->base.cra_priority;
@@ -738,12 +677,10 @@ static int pkcs1pad_create(struct crypto_template tmpl, struct rtattr *tb) · 15 unmodified lines
738		677
739	err = akcipher_register_instance(tmpl, inst);	678	err = akcipher_register_instance(tmpl, inst);
740	if (err)	679	if (err)
741	goto out_free_hash;	680	goto out_drop_alg;
742		681
743	return 0;	682	return 0;
744		683
745	out_free_hash:
746	kfree(ctx->hash_name);
747	out_drop_alg:	684	out_drop_alg:
748	crypto_drop_akcipher(spawn);	685	crypto_drop_akcipher(spawn);
749	out_free_inst:	686	out_free_inst:

crypto/rsa.c+84−29

@@ -10,16 +10,23 @@ · 9 unmodified lines
10	*/	10	*/
11		11
12	#include <linux/module.h>	12	#include <linux/module.h>
		13	#include <linux/mpi.h>
13	#include <crypto/internal/rsa.h>	14	#include <crypto/internal/rsa.h>
14	#include <crypto/internal/akcipher.h>	15	#include <crypto/internal/akcipher.h>
15	#include <crypto/akcipher.h>	16	#include <crypto/akcipher.h>
16	#include <crypto/algapi.h>	17	#include <crypto/algapi.h>
17		18
		19	struct rsa_mpi_key {
		20	MPI n;
		21	MPI e;
		22	MPI d;
		23	};
		24
18	/*	25	/*
19	* RSAEP function [RFC3447 sec 5.1.1]	26	* RSAEP function [RFC3447 sec 5.1.1]
20	* c = m^e mod n;	27	* c = m^e mod n;
21	*/	28	*/
22	static int _rsa_enc(const struct rsa_key *key, MPI c, MPI m)	29	static int _rsa_enc(const struct rsa_mpi_key *key, MPI c, MPI m)
23	{	30	{
24	/* (1) Validate 0 <= m < n */	31	/* (1) Validate 0 <= m < n */
25	if (mpi_cmp_ui(m, 0) < 0 \|\| mpi_cmp(m, key->n) >= 0)	32	if (mpi_cmp_ui(m, 0) < 0 \|\| mpi_cmp(m, key->n) >= 0)
@@ -33,7 +40,7 @@ static int _rsa_enc(const struct rsa_key *key, MPI c, MPI m) · 7 unmodified lines
33	* RSADP function [RFC3447 sec 5.1.2]	40	* RSADP function [RFC3447 sec 5.1.2]
34	* m = c^d mod n;	41	* m = c^d mod n;
35	*/	42	*/
36	static int _rsa_dec(const struct rsa_key *key, MPI m, MPI c)	43	static int _rsa_dec(const struct rsa_mpi_key *key, MPI m, MPI c)
37	{	44	{
38	/* (1) Validate 0 <= c < n */	45	/* (1) Validate 0 <= c < n */
39	if (mpi_cmp_ui(c, 0) < 0 \|\| mpi_cmp(c, key->n) >= 0)	46	if (mpi_cmp_ui(c, 0) < 0 \|\| mpi_cmp(c, key->n) >= 0)
@@ -47,7 +54,7 @@ static int _rsa_dec(const struct rsa_key *key, MPI m, MPI c) · 7 unmodified lines
47	* RSASP1 function [RFC3447 sec 5.2.1]	54	* RSASP1 function [RFC3447 sec 5.2.1]
48	* s = m^d mod n	55	* s = m^d mod n
49	*/	56	*/
50	static int _rsa_sign(const struct rsa_key *key, MPI s, MPI m)	57	static int _rsa_sign(const struct rsa_mpi_key *key, MPI s, MPI m)
51	{	58	{
52	/* (1) Validate 0 <= m < n */	59	/* (1) Validate 0 <= m < n */
53	if (mpi_cmp_ui(m, 0) < 0 \|\| mpi_cmp(m, key->n) >= 0)	60	if (mpi_cmp_ui(m, 0) < 0 \|\| mpi_cmp(m, key->n) >= 0)
@@ -61,7 +68,7 @@ static int _rsa_sign(const struct rsa_key *key, MPI s, MPI m) · 7 unmodified lines
61	* RSAVP1 function [RFC3447 sec 5.2.2]	68	* RSAVP1 function [RFC3447 sec 5.2.2]
62	* m = s^e mod n;	69	* m = s^e mod n;
63	*/	70	*/
64	static int _rsa_verify(const struct rsa_key *key, MPI m, MPI s)	71	static int _rsa_verify(const struct rsa_mpi_key *key, MPI m, MPI s)
65	{	72	{
66	/* (1) Validate 0 <= s < n */	73	/* (1) Validate 0 <= s < n */
67	if (mpi_cmp_ui(s, 0) < 0 \|\| mpi_cmp(s, key->n) >= 0)	74	if (mpi_cmp_ui(s, 0) < 0 \|\| mpi_cmp(s, key->n) >= 0)
@@ -71,7 +78,7 @@ static int _rsa_verify(const struct rsa_key *key, MPI m, MPI s) · 3 unmodified lines
71	return mpi_powm(m, s, key->e, key->n);	78	return mpi_powm(m, s, key->e, key->n);
72	}	79	}
73		80
74	static inline struct rsa_key rsa_get_key(struct crypto_akcipher tfm)	81	static inline struct rsa_mpi_key rsa_get_key(struct crypto_akcipher tfm)
75	{	82	{
76	return akcipher_tfm_ctx(tfm);	83	return akcipher_tfm_ctx(tfm);
77	}	84	}
@@ -79,7 +86,7 @@ static inline struct rsa_key rsa_get_key(struct crypto_akcipher tfm) · 1 unmodified line
79	static int rsa_enc(struct akcipher_request *req)	86	static int rsa_enc(struct akcipher_request *req)
80	{	87	{
81	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);	88	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
82	const struct rsa_key *pkey = rsa_get_key(tfm);	89	const struct rsa_mpi_key *pkey = rsa_get_key(tfm);
83	MPI m, c = mpi_alloc(0);	90	MPI m, c = mpi_alloc(0);
84	int ret = 0;	91	int ret = 0;
85	int sign;	92	int sign;
@@ -101,7 +108,7 @@ static int rsa_enc(struct akcipher_request *req) · 15 unmodified lines
101	if (ret)	108	if (ret)
102	goto err_free_m;	109	goto err_free_m;
103		110
104	ret = mpi_write_to_sgl(c, req->dst, &req->dst_len, &sign);	111	ret = mpi_write_to_sgl(c, req->dst, req->dst_len, &sign);
105	if (ret)	112	if (ret)
106	goto err_free_m;	113	goto err_free_m;
107		114
@@ -118,7 +125,7 @@ err_free_c: · 10 unmodified lines
118	static int rsa_dec(struct akcipher_request *req)	125	static int rsa_dec(struct akcipher_request *req)
119	{	126	{
120	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);	127	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
121	const struct rsa_key *pkey = rsa_get_key(tfm);	128	const struct rsa_mpi_key *pkey = rsa_get_key(tfm);
122	MPI c, m = mpi_alloc(0);	129	MPI c, m = mpi_alloc(0);
123	int ret = 0;	130	int ret = 0;
124	int sign;	131	int sign;
@@ -140,7 +147,7 @@ static int rsa_dec(struct akcipher_request *req) · 15 unmodified lines
140	if (ret)	147	if (ret)
141	goto err_free_c;	148	goto err_free_c;
142		149
143	ret = mpi_write_to_sgl(m, req->dst, &req->dst_len, &sign);	150	ret = mpi_write_to_sgl(m, req->dst, req->dst_len, &sign);
144	if (ret)	151	if (ret)
145	goto err_free_c;	152	goto err_free_c;
146		153
@@ -156,7 +163,7 @@ err_free_m: · 9 unmodified lines
156	static int rsa_sign(struct akcipher_request *req)	163	static int rsa_sign(struct akcipher_request *req)
157	{	164	{
158	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);	165	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
159	const struct rsa_key *pkey = rsa_get_key(tfm);	166	const struct rsa_mpi_key *pkey = rsa_get_key(tfm);
160	MPI m, s = mpi_alloc(0);	167	MPI m, s = mpi_alloc(0);
161	int ret = 0;	168	int ret = 0;
162	int sign;	169	int sign;
@@ -178,7 +185,7 @@ static int rsa_sign(struct akcipher_request *req) · 15 unmodified lines
178	if (ret)	185	if (ret)
179	goto err_free_m;	186	goto err_free_m;
180		187
181	ret = mpi_write_to_sgl(s, req->dst, &req->dst_len, &sign);	188	ret = mpi_write_to_sgl(s, req->dst, req->dst_len, &sign);
182	if (ret)	189	if (ret)
183	goto err_free_m;	190	goto err_free_m;
184		191
@@ -195,7 +202,7 @@ err_free_s: · 10 unmodified lines
195	static int rsa_verify(struct akcipher_request *req)	202	static int rsa_verify(struct akcipher_request *req)
196	{	203	{
197	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);	204	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
198	const struct rsa_key *pkey = rsa_get_key(tfm);	205	const struct rsa_mpi_key *pkey = rsa_get_key(tfm);
199	MPI s, m = mpi_alloc(0);	206	MPI s, m = mpi_alloc(0);
200	int ret = 0;	207	int ret = 0;
201	int sign;	208	int sign;
@@ -219,7 +226,7 @@ static int rsa_verify(struct akcipher_request *req) · 17 unmodified lines
219	if (ret)	226	if (ret)
220	goto err_free_s;	227	goto err_free_s;
221		228
222	ret = mpi_write_to_sgl(m, req->dst, &req->dst_len, &sign);	229	ret = mpi_write_to_sgl(m, req->dst, req->dst_len, &sign);
223	if (ret)	230	if (ret)
224	goto err_free_s;	231	goto err_free_s;
225		232
@@ -233,6 +240,16 @@ err_free_m: · 7 unmodified lines
233	return ret;	240	return ret;
234	}	241	}
235		242
		243	static void rsa_free_mpi_key(struct rsa_mpi_key *key)
		244	{
		245	mpi_free(key->d);
		246	mpi_free(key->e);
		247	mpi_free(key->n);
		248	key->d = NULL;
		249	key->e = NULL;
		250	key->n = NULL;
		251	}
		252
236	static int rsa_check_key_length(unsigned int len)	253	static int rsa_check_key_length(unsigned int len)
237	{	254	{
238	switch (len) {	255	switch (len) {
@@ -251,49 +268,87 @@ static int rsa_check_key_length(unsigned int len) · 12 unmodified lines
251	static int rsa_set_pub_key(struct crypto_akcipher tfm, const void key,	268	static int rsa_set_pub_key(struct crypto_akcipher tfm, const void key,
252	unsigned int keylen)	269	unsigned int keylen)
253	{	270	{
254	struct rsa_key *pkey = akcipher_tfm_ctx(tfm);	271	struct rsa_mpi_key *mpi_key = akcipher_tfm_ctx(tfm);
		272	struct rsa_key raw_key = {0};
255	int ret;	273	int ret;
256		274
257	ret = rsa_parse_pub_key(pkey, key, keylen);	275	/* Free the old MPI key if any */
		276	rsa_free_mpi_key(mpi_key);
		277
		278	ret = rsa_parse_pub_key(&raw_key, key, keylen);
258	if (ret)	279	if (ret)
259	return ret;	280	return ret;
260		281
261	if (rsa_check_key_length(mpi_get_size(pkey->n) << 3)) {	282	mpi_key->e = mpi_read_raw_data(raw_key.e, raw_key.e_sz);
262	rsa_free_key(pkey);	283	if (!mpi_key->e)
263	ret = -EINVAL;	284	goto err;
		285
		286	mpi_key->n = mpi_read_raw_data(raw_key.n, raw_key.n_sz);
		287	if (!mpi_key->n)
		288	goto err;
		289
		290	if (rsa_check_key_length(mpi_get_size(mpi_key->n) << 3)) {
		291	rsa_free_mpi_key(mpi_key);
		292	return -EINVAL;
264	}	293	}
265	return ret;	294
		295	return 0;
		296
		297	err:
		298	rsa_free_mpi_key(mpi_key);
		299	return -ENOMEM;
266	}	300	}
267		301
268	static int rsa_set_priv_key(struct crypto_akcipher tfm, const void key,	302	static int rsa_set_priv_key(struct crypto_akcipher tfm, const void key,
269	unsigned int keylen)	303	unsigned int keylen)
270	{	304	{
271	struct rsa_key *pkey = akcipher_tfm_ctx(tfm);	305	struct rsa_mpi_key *mpi_key = akcipher_tfm_ctx(tfm);
		306	struct rsa_key raw_key = {0};
272	int ret;	307	int ret;
273		308
274	ret = rsa_parse_priv_key(pkey, key, keylen);	309	/* Free the old MPI key if any */
		310	rsa_free_mpi_key(mpi_key);
		311
		312	ret = rsa_parse_priv_key(&raw_key, key, keylen);
275	if (ret)	313	if (ret)
276	return ret;	314	return ret;
277		315
278	if (rsa_check_key_length(mpi_get_size(pkey->n) << 3)) {	316	mpi_key->d = mpi_read_raw_data(raw_key.d, raw_key.d_sz);
279	rsa_free_key(pkey);	317	if (!mpi_key->d)
280	ret = -EINVAL;	318	goto err;
		319
		320	mpi_key->e = mpi_read_raw_data(raw_key.e, raw_key.e_sz);
		321	if (!mpi_key->e)
		322	goto err;
		323
		324	mpi_key->n = mpi_read_raw_data(raw_key.n, raw_key.n_sz);
		325	if (!mpi_key->n)
		326	goto err;
		327
		328	if (rsa_check_key_length(mpi_get_size(mpi_key->n) << 3)) {
		329	rsa_free_mpi_key(mpi_key);
		330	return -EINVAL;
281	}	331	}
282	return ret;	332
		333	return 0;
		334
		335	err:
		336	rsa_free_mpi_key(mpi_key);
		337	return -ENOMEM;
283	}	338	}
284		339
285	static int rsa_max_size(struct crypto_akcipher *tfm)	340	static int rsa_max_size(struct crypto_akcipher *tfm)
286	{	341	{
287	struct rsa_key *pkey = akcipher_tfm_ctx(tfm);	342	struct rsa_mpi_key *pkey = akcipher_tfm_ctx(tfm);
288		343
289	return pkey->n ? mpi_get_size(pkey->n) : -EINVAL;	344	return pkey->n ? mpi_get_size(pkey->n) : -EINVAL;
290	}	345	}
291		346
292	static void rsa_exit_tfm(struct crypto_akcipher *tfm)	347	static void rsa_exit_tfm(struct crypto_akcipher *tfm)
293	{	348	{
294	struct rsa_key *pkey = akcipher_tfm_ctx(tfm);	349	struct rsa_mpi_key *pkey = akcipher_tfm_ctx(tfm);
295		350
296	rsa_free_key(pkey);	351	rsa_free_mpi_key(pkey);
297	}	352	}
298		353
299	static struct akcipher_alg rsa = {	354	static struct akcipher_alg rsa = {
@@ -310,7 +365,7 @@ static struct akcipher_alg rsa = { · 10 unmodified lines
310	.cra_driver_name = "rsa-generic",	365	.cra_driver_name = "rsa-generic",
311	.cra_priority = 100,	366	.cra_priority = 100,
312	.cra_module = THIS_MODULE,	367	.cra_module = THIS_MODULE,
313	.cra_ctxsize = sizeof(struct rsa_key),	368	.cra_ctxsize = sizeof(struct rsa_mpi_key),
314	},	369	},
315	};	370	};
316		371

crypto/rsa_helper.c+107−65

@@ -22,20 +22,29 @@ int rsa_get_n(void *context, size_t hdrlen, unsigned char tag, · 21 unmodified lines
22	const void *value, size_t vlen)	22	const void *value, size_t vlen)
23	{	23	{
24	struct rsa_key *key = context;	24	struct rsa_key *key = context;
		25	const u8 *ptr = value;
		26	size_t n_sz = vlen;
25		27
26	key->n = mpi_read_raw_data(value, vlen);	28	/* invalid key provided */
27		29	if (!value \|\| !vlen)
28	if (!key->n)
29	return -ENOMEM;
30
31	/* In FIPS mode only allow key size 2K & 3K */
32	if (fips_enabled && (mpi_get_size(key->n) != 256 &&
33	mpi_get_size(key->n) != 384)) {
34	pr_err("RSA: key size not allowed in FIPS mode\n");
35	mpi_free(key->n);
36	key->n = NULL;
37	return -EINVAL;	30	return -EINVAL;
		31
		32	if (fips_enabled) {
		33	while (!*ptr && n_sz) {
		34	ptr++;
		35	n_sz--;
		36	}
		37
		38	/* In FIPS mode only allow key size 2K & 3K */
		39	if (n_sz != 256 && n_sz != 384) {
		40	pr_err("RSA: key size not allowed in FIPS mode\n");
		41	return -EINVAL;
		42	}
38	}	43	}
		44
		45	key->n = value;
		46	key->n_sz = vlen;
		47
39	return 0;	48	return 0;
40	}	49	}
41		50
@@ -44,10 +53,12 @@ int rsa_get_e(void *context, size_t hdrlen, unsigned char tag, · 2 unmodified lines
44	{	53	{
45	struct rsa_key *key = context;	54	struct rsa_key *key = context;
46		55
47	key->e = mpi_read_raw_data(value, vlen);	56	/* invalid key provided */
		57	if (!value \|\| !key->n_sz \|\| !vlen \|\| vlen > key->n_sz)
		58	return -EINVAL;
48		59
49	if (!key->e)	60	key->e = value;
50	return -ENOMEM;	61	key->e_sz = vlen;
51		62
52	return 0;	63	return 0;
53	}	64	}
@@ -57,46 +68,95 @@ int rsa_get_d(void *context, size_t hdrlen, unsigned char tag, · 3 unmodified lines
57	{	68	{
58	struct rsa_key *key = context;	69	struct rsa_key *key = context;
59		70
60	key->d = mpi_read_raw_data(value, vlen);	71	/* invalid key provided */
		72	if (!value \|\| !key->n_sz \|\| !vlen \|\| vlen > key->n_sz)
		73	return -EINVAL;
61		74
62	if (!key->d)	75	key->d = value;
63	return -ENOMEM;	76	key->d_sz = vlen;
64		77
65	/* In FIPS mode only allow key size 2K & 3K */	78	return 0;
66	if (fips_enabled && (mpi_get_size(key->d) != 256 &&	79	}
67	mpi_get_size(key->d) != 384)) {	80
68	pr_err("RSA: key size not allowed in FIPS mode\n");	81	int rsa_get_p(void *context, size_t hdrlen, unsigned char tag,
69	mpi_free(key->d);	82	const void *value, size_t vlen)
70	key->d = NULL;	83	{
		84	struct rsa_key *key = context;
		85
		86	/* invalid key provided */
		87	if (!value \|\| !vlen \|\| vlen > key->n_sz)
71	return -EINVAL;	88	return -EINVAL;
72	}	89
		90	key->p = value;
		91	key->p_sz = vlen;
		92
73	return 0;	93	return 0;
74	}	94	}
75		95
76	static void free_mpis(struct rsa_key *key)	96	int rsa_get_q(void *context, size_t hdrlen, unsigned char tag,
		97	const void *value, size_t vlen)
77	{	98	{
78	mpi_free(key->n);	99	struct rsa_key *key = context;
79	mpi_free(key->e);	100
80	mpi_free(key->d);	101	/* invalid key provided */
81	key->n = NULL;	102	if (!value \|\| !vlen \|\| vlen > key->n_sz)
82	key->e = NULL;	103	return -EINVAL;
83	key->d = NULL;	104
		105	key->q = value;
		106	key->q_sz = vlen;
		107
		108	return 0;
84	}	109	}
85		110
86	/**	111	int rsa_get_dp(void *context, size_t hdrlen, unsigned char tag,
87	* rsa_free_key() - frees rsa key allocated by rsa_parse_key()	112	const void *value, size_t vlen)
88	*	113	{
89	* @rsa_key: struct rsa_key key representation	114	struct rsa_key *key = context;
90	*/	115
91	void rsa_free_key(struct rsa_key *key)	116	/* invalid key provided */
		117	if (!value \|\| !vlen \|\| vlen > key->n_sz)
		118	return -EINVAL;
		119
		120	key->dp = value;
		121	key->dp_sz = vlen;
		122
		123	return 0;
		124	}
		125
		126	int rsa_get_dq(void *context, size_t hdrlen, unsigned char tag,
		127	const void *value, size_t vlen)
92	{	128	{
93	free_mpis(key);	129	struct rsa_key *key = context;
		130
		131	/* invalid key provided */
		132	if (!value \|\| !vlen \|\| vlen > key->n_sz)
		133	return -EINVAL;
		134
		135	key->dq = value;
		136	key->dq_sz = vlen;
		137
		138	return 0;
		139	}
		140
		141	int rsa_get_qinv(void *context, size_t hdrlen, unsigned char tag,
		142	const void *value, size_t vlen)
		143	{
		144	struct rsa_key *key = context;
		145
		146	/* invalid key provided */
		147	if (!value \|\| !vlen \|\| vlen > key->n_sz)
		148	return -EINVAL;
		149
		150	key->qinv = value;
		151	key->qinv_sz = vlen;
		152
		153	return 0;
94	}	154	}
95	EXPORT_SYMBOL_GPL(rsa_free_key);
96		155
97	/**	156	/**
98	* rsa_parse_pub_key() - extracts an rsa public key from BER encoded buffer	157	* rsa_parse_pub_key() - decodes the BER encoded buffer and stores in the
99	* and stores it in the provided struct rsa_key	158	* provided struct rsa_key, pointers to the raw key as is,
		159	* so that the caller can copy it or MPI parse it, etc.
100	*	160	*
101	* @rsa_key: struct rsa_key key representation	161	* @rsa_key: struct rsa_key key representation
102	* @key: key in BER format	162	* @key: key in BER format
@@ -107,23 +167,15 @@ EXPORT_SYMBOL_GPL(rsa_free_key); · 4 unmodified lines
107	int rsa_parse_pub_key(struct rsa_key rsa_key, const void key,	167	int rsa_parse_pub_key(struct rsa_key rsa_key, const void key,
108	unsigned int key_len)	168	unsigned int key_len)
109	{	169	{
110	int ret;	170	return asn1_ber_decoder(&rsapubkey_decoder, rsa_key, key, key_len);
111
112	free_mpis(rsa_key);
113	ret = asn1_ber_decoder(&rsapubkey_decoder, rsa_key, key, key_len);
114	if (ret < 0)
115	goto error;
116
117	return 0;
118	error:
119	free_mpis(rsa_key);
120	return ret;
121	}	171	}
122	EXPORT_SYMBOL_GPL(rsa_parse_pub_key);	172	EXPORT_SYMBOL_GPL(rsa_parse_pub_key);
123		173
124	/**	174	/**
125	* rsa_parse_pub_key() - extracts an rsa private key from BER encoded buffer	175	* rsa_parse_priv_key() - decodes the BER encoded buffer and stores in the
126	* and stores it in the provided struct rsa_key	176	* provided struct rsa_key, pointers to the raw key
		177	* as is, so that the caller can copy it or MPI parse it,
		178	* etc.
127	*	179	*
128	* @rsa_key: struct rsa_key key representation	180	* @rsa_key: struct rsa_key key representation
129	* @key: key in BER format	181	* @key: key in BER format
@@ -134,16 +186,6 @@ EXPORT_SYMBOL_GPL(rsa_parse_pub_key); · 4 unmodified lines
134	int rsa_parse_priv_key(struct rsa_key rsa_key, const void key,	186	int rsa_parse_priv_key(struct rsa_key rsa_key, const void key,
135	unsigned int key_len)	187	unsigned int key_len)
136	{	188	{
137	int ret;	189	return asn1_ber_decoder(&rsaprivkey_decoder, rsa_key, key, key_len);
138
139	free_mpis(rsa_key);
140	ret = asn1_ber_decoder(&rsaprivkey_decoder, rsa_key, key, key_len);
141	if (ret < 0)
142	goto error;
143
144	return 0;
145	error:
146	free_mpis(rsa_key);
147	return ret;
148	}	190	}
149	EXPORT_SYMBOL_GPL(rsa_parse_priv_key);	191	EXPORT_SYMBOL_GPL(rsa_parse_priv_key);

crypto/rsaprivkey.asn1+5−5

@@ -3,9 +3,9 @@ RsaPrivKey ::= SEQUENCE { · 2 unmodified lines
3	n INTEGER ({ rsa_get_n }),	3	n INTEGER ({ rsa_get_n }),
4	e INTEGER ({ rsa_get_e }),	4	e INTEGER ({ rsa_get_e }),
5	d INTEGER ({ rsa_get_d }),	5	d INTEGER ({ rsa_get_d }),
6	prime1 INTEGER,	6	prime1 INTEGER ({ rsa_get_p }),
7	prime2 INTEGER,	7	prime2 INTEGER ({ rsa_get_q }),
8	exponent1 INTEGER,	8	exponent1 INTEGER ({ rsa_get_dp }),
9	exponent2 INTEGER,	9	exponent2 INTEGER ({ rsa_get_dq }),
10	coefficient INTEGER	10	coefficient INTEGER ({ rsa_get_qinv })
11	}	11	}

crypto/scatterwalk.c+6−75

@@ -18,8 +18,6 @@ · 17 unmodified lines
18	#include <linux/kernel.h>	18	#include <linux/kernel.h>
19	#include <linux/mm.h>	19	#include <linux/mm.h>
20	#include <linux/module.h>	20	#include <linux/module.h>
21	#include <linux/pagemap.h>
22	#include <linux/highmem.h>
23	#include <linux/scatterlist.h>	21	#include <linux/scatterlist.h>
24		22
25	static inline void memcpy_dir(void buf, void sgdata, size_t nbytes, int out)	23	static inline void memcpy_dir(void buf, void sgdata, size_t nbytes, int out)
@@ -30,53 +28,6 @@ static inline void memcpy_dir(void buf, void sgdata, size_t nbytes, int out) · 4 unmodified lines
30	memcpy(dst, src, nbytes);	28	memcpy(dst, src, nbytes);
31	}	29	}
32		30
33	void scatterwalk_start(struct scatter_walk walk, struct scatterlist sg)
34	{
35	walk->sg = sg;
36
37	BUG_ON(!sg->length);
38
39	walk->offset = sg->offset;
40	}
41	EXPORT_SYMBOL_GPL(scatterwalk_start);
42
43	void scatterwalk_map(struct scatter_walk walk)
44	{
45	return kmap_atomic(scatterwalk_page(walk)) +
46	offset_in_page(walk->offset);
47	}
48	EXPORT_SYMBOL_GPL(scatterwalk_map);
49
50	static void scatterwalk_pagedone(struct scatter_walk *walk, int out,
51	unsigned int more)
52	{
53	if (out) {
54	struct page *page;
55
56	page = sg_page(walk->sg) + ((walk->offset - 1) >> PAGE_SHIFT);
57	/* Test ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE first as
58	* PageSlab cannot be optimised away per se due to
59	* use of volatile pointer.
60	*/
61	if (ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE && !PageSlab(page))
62	flush_dcache_page(page);
63	}
64
65	if (more) {
66	walk->offset += PAGE_SIZE - 1;
67	walk->offset &= PAGE_MASK;
68	if (walk->offset >= walk->sg->offset + walk->sg->length)
69	scatterwalk_start(walk, sg_next(walk->sg));
70	}
71	}
72
73	void scatterwalk_done(struct scatter_walk *walk, int out, int more)
74	{
75	if (!(scatterwalk_pagelen(walk) & (PAGE_SIZE - 1)) \|\| !more)
76	scatterwalk_pagedone(walk, out, more);
77	}
78	EXPORT_SYMBOL_GPL(scatterwalk_done);
79
80	void scatterwalk_copychunks(void buf, struct scatter_walk walk,	31	void scatterwalk_copychunks(void buf, struct scatter_walk walk,
81	size_t nbytes, int out)	32	size_t nbytes, int out)
82	{	33	{
@@ -87,9 +38,11 @@ void scatterwalk_copychunks(void buf, struct scatter_walk walk, · 4 unmodified lines
87	if (len_this_page > nbytes)	38	if (len_this_page > nbytes)
88	len_this_page = nbytes;	39	len_this_page = nbytes;
89		40
90	vaddr = scatterwalk_map(walk);	41	if (out != 2) {
91	memcpy_dir(buf, vaddr, len_this_page, out);	42	vaddr = scatterwalk_map(walk);
92	scatterwalk_unmap(vaddr);	43	memcpy_dir(buf, vaddr, len_this_page, out);
		44	scatterwalk_unmap(vaddr);
		45	}
93		46
94	scatterwalk_advance(walk, len_this_page);	47	scatterwalk_advance(walk, len_this_page);
95		48
@@ -99,7 +52,7 @@ void scatterwalk_copychunks(void buf, struct scatter_walk walk, · 3 unmodified lines
99	buf += len_this_page;	52	buf += len_this_page;
100	nbytes -= len_this_page;	53	nbytes -= len_this_page;
101		54
102	scatterwalk_pagedone(walk, out, 1);	55	scatterwalk_pagedone(walk, out & 1, 1);
103	}	56	}
104	}	57	}
105	EXPORT_SYMBOL_GPL(scatterwalk_copychunks);	58	EXPORT_SYMBOL_GPL(scatterwalk_copychunks);
@@ -125,28 +78,6 @@ void scatterwalk_map_and_copy(void buf, struct scatterlist sg, · 19 unmodified lines
125	}	78	}
126	EXPORT_SYMBOL_GPL(scatterwalk_map_and_copy);	79	EXPORT_SYMBOL_GPL(scatterwalk_map_and_copy);
127		80
128	int scatterwalk_bytes_sglen(struct scatterlist *sg, int num_bytes)
129	{
130	int offset = 0, n = 0;
131
132	/* num_bytes is too small */
133	if (num_bytes < sg->length)
134	return -1;
135
136	do {
137	offset += sg->length;
138	n++;
139	sg = sg_next(sg);
140
141	/* num_bytes is too large */
142	if (unlikely(!sg && (num_bytes < offset)))
143	return -1;
144	} while (sg && (num_bytes > offset));
145
146	return n;
147	}
148	EXPORT_SYMBOL_GPL(scatterwalk_bytes_sglen);
149
150	struct scatterlist *scatterwalk_ffwd(struct scatterlist dst[2],	81	struct scatterlist *scatterwalk_ffwd(struct scatterlist dst[2],
151	struct scatterlist *src,	82	struct scatterlist *src,
152	unsigned int len)	83	unsigned int len)

crypto/seqiv.c+13−163

@@ -14,50 +14,17 @@ · 13 unmodified lines
14	*/	14	*/
15		15
16	#include <crypto/internal/geniv.h>	16	#include <crypto/internal/geniv.h>
17	#include <crypto/internal/skcipher.h>
18	#include <crypto/rng.h>
19	#include <crypto/scatterwalk.h>	17	#include <crypto/scatterwalk.h>
		18	#include <crypto/skcipher.h>
20	#include <linux/err.h>	19	#include <linux/err.h>
21	#include <linux/init.h>	20	#include <linux/init.h>
22	#include <linux/kernel.h>	21	#include <linux/kernel.h>
23	#include <linux/module.h>	22	#include <linux/module.h>
24	#include <linux/slab.h>	23	#include <linux/slab.h>
25	#include <linux/spinlock.h>
26	#include <linux/string.h>	24	#include <linux/string.h>
27		25
28	struct seqiv_ctx {
29	spinlock_t lock;
30	u8 salt[] __attribute__ ((aligned(__alignof__(u32))));
31	};
32
33	static void seqiv_free(struct crypto_instance *inst);	26	static void seqiv_free(struct crypto_instance *inst);
34		27
35	static void seqiv_complete2(struct skcipher_givcrypt_request *req, int err)
36	{
37	struct ablkcipher_request *subreq = skcipher_givcrypt_reqctx(req);
38	struct crypto_ablkcipher *geniv;
39
40	if (err == -EINPROGRESS)
41	return;
42
43	if (err)
44	goto out;
45
46	geniv = skcipher_givcrypt_reqtfm(req);
47	memcpy(req->creq.info, subreq->info, crypto_ablkcipher_ivsize(geniv));
48
49	out:
50	kfree(subreq->info);
51	}
52
53	static void seqiv_complete(struct crypto_async_request *base, int err)
54	{
55	struct skcipher_givcrypt_request *req = base->data;
56
57	seqiv_complete2(req, err);
58	skcipher_givcrypt_complete(req, err);
59	}
60
61	static void seqiv_aead_encrypt_complete2(struct aead_request *req, int err)	28	static void seqiv_aead_encrypt_complete2(struct aead_request *req, int err)
62	{	29	{
63	struct aead_request *subreq = aead_request_ctx(req);	30	struct aead_request *subreq = aead_request_ctx(req);
@@ -85,65 +52,6 @@ static void seqiv_aead_encrypt_complete(struct crypto_async_request *base, · 21 unmodified lines
85	aead_request_complete(req, err);	52	aead_request_complete(req, err);
86	}	53	}
87		54
88	static void seqiv_geniv(struct seqiv_ctx ctx, u8 info, u64 seq,
89	unsigned int ivsize)
90	{
91	unsigned int len = ivsize;
92
93	if (ivsize > sizeof(u64)) {
94	memset(info, 0, ivsize - sizeof(u64));
95	len = sizeof(u64);
96	}
97	seq = cpu_to_be64(seq);
98	memcpy(info + ivsize - len, &seq, len);
99	crypto_xor(info, ctx->salt, ivsize);
100	}
101
102	static int seqiv_givencrypt(struct skcipher_givcrypt_request *req)
103	{
104	struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
105	struct seqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
106	struct ablkcipher_request *subreq = skcipher_givcrypt_reqctx(req);
107	crypto_completion_t compl;
108	void *data;
109	u8 *info;
110	unsigned int ivsize;
111	int err;
112
113	ablkcipher_request_set_tfm(subreq, skcipher_geniv_cipher(geniv));
114
115	compl = req->creq.base.complete;
116	data = req->creq.base.data;
117	info = req->creq.info;
118
119	ivsize = crypto_ablkcipher_ivsize(geniv);
120
121	if (unlikely(!IS_ALIGNED((unsigned long)info,
122	crypto_ablkcipher_alignmask(geniv) + 1))) {
123	info = kmalloc(ivsize, req->creq.base.flags &
124	CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL:
125	GFP_ATOMIC);
126	if (!info)
127	return -ENOMEM;
128
129	compl = seqiv_complete;
130	data = req;
131	}
132
133	ablkcipher_request_set_callback(subreq, req->creq.base.flags, compl,
134	data);
135	ablkcipher_request_set_crypt(subreq, req->creq.src, req->creq.dst,
136	req->creq.nbytes, info);
137
138	seqiv_geniv(ctx, info, req->seq, ivsize);
139	memcpy(req->giv, info, ivsize);
140
141	err = crypto_ablkcipher_encrypt(subreq);
142	if (unlikely(info != req->creq.info))
143	seqiv_complete2(req, err);
144	return err;
145	}
146
147	static int seqiv_aead_encrypt(struct aead_request *req)	55	static int seqiv_aead_encrypt(struct aead_request *req)
148	{	56	{
149	struct crypto_aead *geniv = crypto_aead_reqtfm(req);	57	struct crypto_aead *geniv = crypto_aead_reqtfm(req);
@@ -165,12 +73,16 @@ static int seqiv_aead_encrypt(struct aead_request *req) · 15 unmodified lines
165	info = req->iv;	73	info = req->iv;
166		74
167	if (req->src != req->dst) {	75	if (req->src != req->dst) {
168	struct blkcipher_desc desc = {	76	SKCIPHER_REQUEST_ON_STACK(nreq, ctx->sknull);
169	.tfm = ctx->null,	77
170	};	78	skcipher_request_set_tfm(nreq, ctx->sknull);
		79	skcipher_request_set_callback(nreq, req->base.flags,
		80	NULL, NULL);
		81	skcipher_request_set_crypt(nreq, req->src, req->dst,
		82	req->assoclen + req->cryptlen,
		83	NULL);
171		84
172	err = crypto_blkcipher_encrypt(&desc, req->dst, req->src,	85	err = crypto_skcipher_encrypt(nreq);
173	req->assoclen + req->cryptlen);
174	if (err)	86	if (err)
175	return err;	87	return err;
176	}	88	}
@@ -229,62 +141,6 @@ static int seqiv_aead_decrypt(struct aead_request *req) · 52 unmodified lines
229	return crypto_aead_decrypt(subreq);	141	return crypto_aead_decrypt(subreq);
230	}	142	}
231		143
232	static int seqiv_init(struct crypto_tfm *tfm)
233	{
234	struct crypto_ablkcipher *geniv = __crypto_ablkcipher_cast(tfm);
235	struct seqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
236	int err;
237
238	spin_lock_init(&ctx->lock);
239
240	tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request);
241
242	err = 0;
243	if (!crypto_get_default_rng()) {
244	crypto_ablkcipher_crt(geniv)->givencrypt = seqiv_givencrypt;
245	err = crypto_rng_get_bytes(crypto_default_rng, ctx->salt,
246	crypto_ablkcipher_ivsize(geniv));
247	crypto_put_default_rng();
248	}
249
250	return err ?: skcipher_geniv_init(tfm);
251	}
252
253	static int seqiv_ablkcipher_create(struct crypto_template *tmpl,
254	struct rtattr **tb)
255	{
256	struct crypto_instance *inst;
257	int err;
258
259	inst = skcipher_geniv_alloc(tmpl, tb, 0, 0);
260
261	if (IS_ERR(inst))
262	return PTR_ERR(inst);
263
264	err = -EINVAL;
265	if (inst->alg.cra_ablkcipher.ivsize < sizeof(u64))
266	goto free_inst;
267
268	inst->alg.cra_init = seqiv_init;
269	inst->alg.cra_exit = skcipher_geniv_exit;
270
271	inst->alg.cra_ctxsize += inst->alg.cra_ablkcipher.ivsize;
272	inst->alg.cra_ctxsize += sizeof(struct seqiv_ctx);
273
274	inst->alg.cra_alignmask \|= __alignof__(u32) - 1;
275
276	err = crypto_register_instance(tmpl, inst);
277	if (err)
278	goto free_inst;
279
280	out:
281	return err;
282
283	free_inst:
284	skcipher_geniv_free(inst);
285	goto out;
286	}
287
288	static int seqiv_aead_create(struct crypto_template tmpl, struct rtattr *tb)	144	static int seqiv_aead_create(struct crypto_template tmpl, struct rtattr *tb)
289	{	145	{
290	struct aead_instance *inst;	146	struct aead_instance *inst;
@@ -330,26 +186,20 @@ free_inst: · 39 unmodified lines
330	static int seqiv_create(struct crypto_template tmpl, struct rtattr *tb)	186	static int seqiv_create(struct crypto_template tmpl, struct rtattr *tb)
331	{	187	{
332	struct crypto_attr_type *algt;	188	struct crypto_attr_type *algt;
333	int err;
334		189
335	algt = crypto_get_attr_type(tb);	190	algt = crypto_get_attr_type(tb);
336	if (IS_ERR(algt))	191	if (IS_ERR(algt))
337	return PTR_ERR(algt);	192	return PTR_ERR(algt);
338		193
339	if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & CRYPTO_ALG_TYPE_MASK)	194	if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & CRYPTO_ALG_TYPE_MASK)
340	err = seqiv_ablkcipher_create(tmpl, tb);	195	return -EINVAL;
341	else
342	err = seqiv_aead_create(tmpl, tb);
343		196
344	return err;	197	return seqiv_aead_create(tmpl, tb);
345	}	198	}
346		199
347	static void seqiv_free(struct crypto_instance *inst)	200	static void seqiv_free(struct crypto_instance *inst)
348	{	201	{
349	if ((inst->alg.cra_flags ^ CRYPTO_ALG_TYPE_AEAD) & CRYPTO_ALG_TYPE_MASK)	202	aead_geniv_free(aead_instance(inst));
350	skcipher_geniv_free(inst);
351	else
352	aead_geniv_free(aead_instance(inst));
353	}	203	}
354		204
355	static struct crypto_template seqiv_tmpl = {	205	static struct crypto_template seqiv_tmpl = {

crypto/sha3_generic.c+300−0

@@ -0,0 +1,300 @@
	1	/*
	2	* Cryptographic API.
	3	*
	4	* SHA-3, as specified in
	5	* http://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.202.pdf
	6	*
	7	* SHA-3 code by Jeff Garzik <jeff@garzik.org>
	8	*
	9	* This program is free software; you can redistribute it and/or modify it
	10	* under the terms of the GNU General Public License as published by the Free
	11	* Software Foundation; either version 2 of the License, or (at your option)•
	12	* any later version.
	13	*
	14	*/
	15	#include <crypto/internal/hash.h>
	16	#include <linux/init.h>
	17	#include <linux/module.h>
	18	#include <linux/types.h>
	19	#include <crypto/sha3.h>
	20	#include <asm/byteorder.h>
	21
	22	#define KECCAK_ROUNDS 24
	23
	24	#define ROTL64(x, y) (((x) << (y)) \| ((x) >> (64 - (y))))
	25
	26	static const u64 keccakf_rndc[24] = {
	27	0x0000000000000001, 0x0000000000008082, 0x800000000000808a,
	28	0x8000000080008000, 0x000000000000808b, 0x0000000080000001,
	29	0x8000000080008081, 0x8000000000008009, 0x000000000000008a,
	30	0x0000000000000088, 0x0000000080008009, 0x000000008000000a,
	31	0x000000008000808b, 0x800000000000008b, 0x8000000000008089,
	32	0x8000000000008003, 0x8000000000008002, 0x8000000000000080,
	33	0x000000000000800a, 0x800000008000000a, 0x8000000080008081,
	34	0x8000000000008080, 0x0000000080000001, 0x8000000080008008
	35	};
	36
	37	static const int keccakf_rotc[24] = {
	38	1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 2, 14,
	39	27, 41, 56, 8, 25, 43, 62, 18, 39, 61, 20, 44
	40	};
	41
	42	static const int keccakf_piln[24] = {
	43	10, 7, 11, 17, 18, 3, 5, 16, 8, 21, 24, 4,
	44	15, 23, 19, 13, 12, 2, 20, 14, 22, 9, 6, 1
	45	};
	46
	47	/* update the state with given number of rounds */
	48
	49	static void keccakf(u64 st[25])
	50	{
	51	int i, j, round;
	52	u64 t, bc[5];
	53
	54	for (round = 0; round < KECCAK_ROUNDS; round++) {
	55
	56	/* Theta */
	57	for (i = 0; i < 5; i++)
	58	bc[i] = st[i] ^ st[i + 5] ^ st[i + 10] ^ st[i + 15]
	59	^ st[i + 20];
	60
	61	for (i = 0; i < 5; i++) {
	62	t = bc[(i + 4) % 5] ^ ROTL64(bc[(i + 1) % 5], 1);
	63	for (j = 0; j < 25; j += 5)
	64	st[j + i] ^= t;
	65	}
	66
	67	/* Rho Pi */
	68	t = st[1];
	69	for (i = 0; i < 24; i++) {
	70	j = keccakf_piln[i];
	71	bc[0] = st[j];
	72	st[j] = ROTL64(t, keccakf_rotc[i]);
	73	t = bc[0];
	74	}
	75
	76	/* Chi */
	77	for (j = 0; j < 25; j += 5) {
	78	for (i = 0; i < 5; i++)
	79	bc[i] = st[j + i];
	80	for (i = 0; i < 5; i++)
	81	st[j + i] ^= (~bc[(i + 1) % 5]) &
	82	bc[(i + 2) % 5];
	83	}
	84
	85	/* Iota */
	86	st[0] ^= keccakf_rndc[round];
	87	}
	88	}
	89
	90	static void sha3_init(struct sha3_state *sctx, unsigned int digest_sz)
	91	{
	92	memset(sctx, 0, sizeof(*sctx));
	93	sctx->md_len = digest_sz;
	94	sctx->rsiz = 200 - 2 * digest_sz;
	95	sctx->rsizw = sctx->rsiz / 8;
	96	}
	97
	98	static int sha3_224_init(struct shash_desc *desc)
	99	{
	100	struct sha3_state *sctx = shash_desc_ctx(desc);
	101
	102	sha3_init(sctx, SHA3_224_DIGEST_SIZE);
	103	return 0;
	104	}
	105
	106	static int sha3_256_init(struct shash_desc *desc)
	107	{
	108	struct sha3_state *sctx = shash_desc_ctx(desc);
	109
	110	sha3_init(sctx, SHA3_256_DIGEST_SIZE);
	111	return 0;
	112	}
	113
	114	static int sha3_384_init(struct shash_desc *desc)
	115	{
	116	struct sha3_state *sctx = shash_desc_ctx(desc);
	117
	118	sha3_init(sctx, SHA3_384_DIGEST_SIZE);
	119	return 0;
	120	}
	121
	122	static int sha3_512_init(struct shash_desc *desc)
	123	{
	124	struct sha3_state *sctx = shash_desc_ctx(desc);
	125
	126	sha3_init(sctx, SHA3_512_DIGEST_SIZE);
	127	return 0;
	128	}
	129
	130	static int sha3_update(struct shash_desc desc, const u8 data,
	131	unsigned int len)
	132	{
	133	struct sha3_state *sctx = shash_desc_ctx(desc);
	134	unsigned int done;
	135	const u8 *src;
	136
	137	done = 0;
	138	src = data;
	139
	140	if ((sctx->partial + len) > (sctx->rsiz - 1)) {
	141	if (sctx->partial) {
	142	done = -sctx->partial;
	143	memcpy(sctx->buf + sctx->partial, data,
	144	done + sctx->rsiz);
	145	src = sctx->buf;
	146	}
	147
	148	do {
	149	unsigned int i;
	150
	151	for (i = 0; i < sctx->rsizw; i++)
	152	sctx->st[i] ^= ((u64 *) src)[i];
	153	keccakf(sctx->st);
	154
	155	done += sctx->rsiz;
	156	src = data + done;
	157	} while (done + (sctx->rsiz - 1) < len);
	158
	159	sctx->partial = 0;
	160	}
	161	memcpy(sctx->buf + sctx->partial, src, len - done);
	162	sctx->partial += (len - done);
	163
	164	return 0;
	165	}
	166
	167	static int sha3_final(struct shash_desc desc, u8 out)
	168	{
	169	struct sha3_state *sctx = shash_desc_ctx(desc);
	170	unsigned int i, inlen = sctx->partial;
	171
	172	sctx->buf[inlen++] = 0x06;
	173	memset(sctx->buf + inlen, 0, sctx->rsiz - inlen);
	174	sctx->buf[sctx->rsiz - 1] \|= 0x80;
	175
	176	for (i = 0; i < sctx->rsizw; i++)
	177	sctx->st[i] ^= ((u64 *) sctx->buf)[i];
	178
	179	keccakf(sctx->st);
	180
	181	for (i = 0; i < sctx->rsizw; i++)
	182	sctx->st[i] = cpu_to_le64(sctx->st[i]);
	183
	184	memcpy(out, sctx->st, sctx->md_len);
	185
	186	memset(sctx, 0, sizeof(*sctx));
	187	return 0;
	188	}
	189
	190	static struct shash_alg sha3_224 = {
	191	.digestsize = SHA3_224_DIGEST_SIZE,
	192	.init = sha3_224_init,
	193	.update = sha3_update,
	194	.final = sha3_final,
	195	.descsize = sizeof(struct sha3_state),
	196	.base = {
	197	.cra_name = "sha3-224",
	198	.cra_driver_name = "sha3-224-generic",
	199	.cra_flags = CRYPTO_ALG_TYPE_SHASH,
	200	.cra_blocksize = SHA3_224_BLOCK_SIZE,
	201	.cra_module = THIS_MODULE,
	202	}
	203	};
	204
	205	static struct shash_alg sha3_256 = {
	206	.digestsize = SHA3_256_DIGEST_SIZE,
	207	.init = sha3_256_init,
	208	.update = sha3_update,
	209	.final = sha3_final,
	210	.descsize = sizeof(struct sha3_state),
	211	.base = {
	212	.cra_name = "sha3-256",
	213	.cra_driver_name = "sha3-256-generic",
	214	.cra_flags = CRYPTO_ALG_TYPE_SHASH,
	215	.cra_blocksize = SHA3_256_BLOCK_SIZE,
	216	.cra_module = THIS_MODULE,
	217	}
	218	};
	219
	220	static struct shash_alg sha3_384 = {
	221	.digestsize = SHA3_384_DIGEST_SIZE,
	222	.init = sha3_384_init,
	223	.update = sha3_update,
	224	.final = sha3_final,
	225	.descsize = sizeof(struct sha3_state),
	226	.base = {
	227	.cra_name = "sha3-384",
	228	.cra_driver_name = "sha3-384-generic",
	229	.cra_flags = CRYPTO_ALG_TYPE_SHASH,
	230	.cra_blocksize = SHA3_384_BLOCK_SIZE,
	231	.cra_module = THIS_MODULE,
	232	}
	233	};
	234
	235	static struct shash_alg sha3_512 = {
	236	.digestsize = SHA3_512_DIGEST_SIZE,
	237	.init = sha3_512_init,
	238	.update = sha3_update,
	239	.final = sha3_final,
	240	.descsize = sizeof(struct sha3_state),
	241	.base = {
	242	.cra_name = "sha3-512",
	243	.cra_driver_name = "sha3-512-generic",
	244	.cra_flags = CRYPTO_ALG_TYPE_SHASH,
	245	.cra_blocksize = SHA3_512_BLOCK_SIZE,
	246	.cra_module = THIS_MODULE,
	247	}
	248	};
	249
	250	static int __init sha3_generic_mod_init(void)
	251	{
	252	int ret;
	253
	254	ret = crypto_register_shash(&sha3_224);
	255	if (ret < 0)
	256	goto err_out;
	257	ret = crypto_register_shash(&sha3_256);
	258	if (ret < 0)
	259	goto err_out_224;
	260	ret = crypto_register_shash(&sha3_384);
	261	if (ret < 0)
	262	goto err_out_256;
	263	ret = crypto_register_shash(&sha3_512);
	264	if (ret < 0)
	265	goto err_out_384;
	266
	267	return 0;
	268
	269	err_out_384:
	270	crypto_unregister_shash(&sha3_384);
	271	err_out_256:
	272	crypto_unregister_shash(&sha3_256);
	273	err_out_224:
	274	crypto_unregister_shash(&sha3_224);
	275	err_out:
	276	return ret;
	277	}
	278
	279	static void __exit sha3_generic_mod_fini(void)
	280	{
	281	crypto_unregister_shash(&sha3_224);
	282	crypto_unregister_shash(&sha3_256);
	283	crypto_unregister_shash(&sha3_384);
	284	crypto_unregister_shash(&sha3_512);
	285	}
	286
	287	module_init(sha3_generic_mod_init);
	288	module_exit(sha3_generic_mod_fini);
	289
	290	MODULE_LICENSE("GPL");
	291	MODULE_DESCRIPTION("SHA-3 Secure Hash Algorithm");
	292
	293	MODULE_ALIAS_CRYPTO("sha3-224");
	294	MODULE_ALIAS_CRYPTO("sha3-224-generic");
	295	MODULE_ALIAS_CRYPTO("sha3-256");
	296	MODULE_ALIAS_CRYPTO("sha3-256-generic");
	297	MODULE_ALIAS_CRYPTO("sha3-384");
	298	MODULE_ALIAS_CRYPTO("sha3-384-generic");
	299	MODULE_ALIAS_CRYPTO("sha3-512");
	300	MODULE_ALIAS_CRYPTO("sha3-512-generic");

crypto/skcipher.c+189−7

@@ -16,7 +16,11 @@ · 15 unmodified lines
16		16
17	#include <crypto/internal/skcipher.h>	17	#include <crypto/internal/skcipher.h>
18	#include <linux/bug.h>	18	#include <linux/bug.h>
		19	#include <linux/cryptouser.h>
19	#include <linux/module.h>	20	#include <linux/module.h>
		21	#include <linux/rtnetlink.h>
		22	#include <linux/seq_file.h>
		23	#include <net/netlink.h>
20		24
21	#include "internal.h"	25	#include "internal.h"
22		26
@@ -25,10 +29,11 @@ static unsigned int crypto_skcipher_extsize(struct crypto_alg *alg) · 2 unmodified lines
25	if (alg->cra_type == &crypto_blkcipher_type)	29	if (alg->cra_type == &crypto_blkcipher_type)
26	return sizeof(struct crypto_blkcipher *);	30	return sizeof(struct crypto_blkcipher *);
27		31
28	BUG_ON(alg->cra_type != &crypto_ablkcipher_type &&	32	if (alg->cra_type == &crypto_ablkcipher_type \|\|
29	alg->cra_type != &crypto_givcipher_type);	33	alg->cra_type == &crypto_givcipher_type)
		34	return sizeof(struct crypto_ablkcipher *);
30		35
31	return sizeof(struct crypto_ablkcipher *);	36	return crypto_alg_extsize(alg);
32	}	37	}
33		38
34	static int skcipher_setkey_blkcipher(struct crypto_skcipher *tfm,	39	static int skcipher_setkey_blkcipher(struct crypto_skcipher *tfm,
@@ -216,26 +221,118 @@ static int crypto_init_skcipher_ops_ablkcipher(struct crypto_tfm *tfm) · 181 unmodified lines
216	return 0;	221	return 0;
217	}	222	}
218		223
		224	static void crypto_skcipher_exit_tfm(struct crypto_tfm *tfm)
		225	{
		226	struct crypto_skcipher *skcipher = __crypto_skcipher_cast(tfm);
		227	struct skcipher_alg *alg = crypto_skcipher_alg(skcipher);
		228
		229	alg->exit(skcipher);
		230	}
		231
219	static int crypto_skcipher_init_tfm(struct crypto_tfm *tfm)	232	static int crypto_skcipher_init_tfm(struct crypto_tfm *tfm)
220	{	233	{
		234	struct crypto_skcipher *skcipher = __crypto_skcipher_cast(tfm);
		235	struct skcipher_alg *alg = crypto_skcipher_alg(skcipher);
		236
221	if (tfm->__crt_alg->cra_type == &crypto_blkcipher_type)	237	if (tfm->__crt_alg->cra_type == &crypto_blkcipher_type)
222	return crypto_init_skcipher_ops_blkcipher(tfm);	238	return crypto_init_skcipher_ops_blkcipher(tfm);
223		239
224	BUG_ON(tfm->__crt_alg->cra_type != &crypto_ablkcipher_type &&	240	if (tfm->__crt_alg->cra_type == &crypto_ablkcipher_type \|\|
225	tfm->__crt_alg->cra_type != &crypto_givcipher_type);	241	tfm->__crt_alg->cra_type == &crypto_givcipher_type)
		242	return crypto_init_skcipher_ops_ablkcipher(tfm);
		243
		244	skcipher->setkey = alg->setkey;
		245	skcipher->encrypt = alg->encrypt;
		246	skcipher->decrypt = alg->decrypt;
		247	skcipher->ivsize = alg->ivsize;
		248	skcipher->keysize = alg->max_keysize;
		249
		250	if (alg->exit)
		251	skcipher->base.exit = crypto_skcipher_exit_tfm;
226		252
227	return crypto_init_skcipher_ops_ablkcipher(tfm);	253	if (alg->init)
		254	return alg->init(skcipher);
		255
		256	return 0;
		257	}
		258
		259	static void crypto_skcipher_free_instance(struct crypto_instance *inst)
		260	{
		261	struct skcipher_instance *skcipher =
		262	container_of(inst, struct skcipher_instance, s.base);
		263
		264	skcipher->free(skcipher);
		265	}
		266
		267	static void crypto_skcipher_show(struct seq_file m, struct crypto_alg alg)
		268	__attribute__ ((unused));
		269	static void crypto_skcipher_show(struct seq_file m, struct crypto_alg alg)
		270	{
		271	struct skcipher_alg *skcipher = container_of(alg, struct skcipher_alg,
		272	base);
		273
		274	seq_printf(m, "type : skcipher\n");
		275	seq_printf(m, "async : %s\n",
		276	alg->cra_flags & CRYPTO_ALG_ASYNC ? "yes" : "no");
		277	seq_printf(m, "blocksize : %u\n", alg->cra_blocksize);
		278	seq_printf(m, "min keysize : %u\n", skcipher->min_keysize);
		279	seq_printf(m, "max keysize : %u\n", skcipher->max_keysize);
		280	seq_printf(m, "ivsize : %u\n", skcipher->ivsize);
		281	seq_printf(m, "chunksize : %u\n", skcipher->chunksize);
228	}	282	}
229		283
		284	#ifdef CONFIG_NET
		285	static int crypto_skcipher_report(struct sk_buff skb, struct crypto_alg alg)
		286	{
		287	struct crypto_report_blkcipher rblkcipher;
		288	struct skcipher_alg *skcipher = container_of(alg, struct skcipher_alg,
		289	base);
		290
		291	strncpy(rblkcipher.type, "skcipher", sizeof(rblkcipher.type));
		292	strncpy(rblkcipher.geniv, "<none>", sizeof(rblkcipher.geniv));
		293
		294	rblkcipher.blocksize = alg->cra_blocksize;
		295	rblkcipher.min_keysize = skcipher->min_keysize;
		296	rblkcipher.max_keysize = skcipher->max_keysize;
		297	rblkcipher.ivsize = skcipher->ivsize;
		298
		299	if (nla_put(skb, CRYPTOCFGA_REPORT_BLKCIPHER,
		300	sizeof(struct crypto_report_blkcipher), &rblkcipher))
		301	goto nla_put_failure;
		302	return 0;
		303
		304	nla_put_failure:
		305	return -EMSGSIZE;
		306	}
		307	#else
		308	static int crypto_skcipher_report(struct sk_buff skb, struct crypto_alg alg)
		309	{
		310	return -ENOSYS;
		311	}
		312	#endif
		313
230	static const struct crypto_type crypto_skcipher_type2 = {	314	static const struct crypto_type crypto_skcipher_type2 = {
231	.extsize = crypto_skcipher_extsize,	315	.extsize = crypto_skcipher_extsize,
232	.init_tfm = crypto_skcipher_init_tfm,	316	.init_tfm = crypto_skcipher_init_tfm,
		317	.free = crypto_skcipher_free_instance,
		318	#ifdef CONFIG_PROC_FS
		319	.show = crypto_skcipher_show,
		320	#endif
		321	.report = crypto_skcipher_report,
233	.maskclear = ~CRYPTO_ALG_TYPE_MASK,	322	.maskclear = ~CRYPTO_ALG_TYPE_MASK,
234	.maskset = CRYPTO_ALG_TYPE_BLKCIPHER_MASK,	323	.maskset = CRYPTO_ALG_TYPE_BLKCIPHER_MASK,
235	.type = CRYPTO_ALG_TYPE_BLKCIPHER,	324	.type = CRYPTO_ALG_TYPE_SKCIPHER,
236	.tfmsize = offsetof(struct crypto_skcipher, base),	325	.tfmsize = offsetof(struct crypto_skcipher, base),
237	};	326	};
238		327
		328	int crypto_grab_skcipher(struct crypto_skcipher_spawn *spawn,
		329	const char *name, u32 type, u32 mask)
		330	{
		331	spawn->base.frontend = &crypto_skcipher_type2;
		332	return crypto_grab_spawn(&spawn->base, name, type, mask);
		333	}
		334	EXPORT_SYMBOL_GPL(crypto_grab_skcipher);
		335
239	struct crypto_skcipher crypto_alloc_skcipher(const char alg_name,	336	struct crypto_skcipher crypto_alloc_skcipher(const char alg_name,
240	u32 type, u32 mask)	337	u32 type, u32 mask)
241	{	338	{
@@ -243,5 +340,90 @@ struct crypto_skcipher crypto_alloc_skcipher(const char alg_name, · 1 unmodified line
243	}	340	}
244	EXPORT_SYMBOL_GPL(crypto_alloc_skcipher);	341	EXPORT_SYMBOL_GPL(crypto_alloc_skcipher);
245		342
		343	int crypto_has_skcipher2(const char *alg_name, u32 type, u32 mask)
		344	{
		345	return crypto_type_has_alg(alg_name, &crypto_skcipher_type2,
		346	type, mask);
		347	}
		348	EXPORT_SYMBOL_GPL(crypto_has_skcipher2);
		349
		350	static int skcipher_prepare_alg(struct skcipher_alg *alg)
		351	{
		352	struct crypto_alg *base = &alg->base;
		353
		354	if (alg->ivsize > PAGE_SIZE / 8 \|\| alg->chunksize > PAGE_SIZE / 8)
		355	return -EINVAL;
		356
		357	if (!alg->chunksize)
		358	alg->chunksize = base->cra_blocksize;
		359
		360	base->cra_type = &crypto_skcipher_type2;
		361	base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK;
		362	base->cra_flags \|= CRYPTO_ALG_TYPE_SKCIPHER;
		363
		364	return 0;
		365	}
		366
		367	int crypto_register_skcipher(struct skcipher_alg *alg)
		368	{
		369	struct crypto_alg *base = &alg->base;
		370	int err;
		371
		372	err = skcipher_prepare_alg(alg);
		373	if (err)
		374	return err;
		375
		376	return crypto_register_alg(base);
		377	}
		378	EXPORT_SYMBOL_GPL(crypto_register_skcipher);
		379
		380	void crypto_unregister_skcipher(struct skcipher_alg *alg)
		381	{
		382	crypto_unregister_alg(&alg->base);
		383	}
		384	EXPORT_SYMBOL_GPL(crypto_unregister_skcipher);
		385
		386	int crypto_register_skciphers(struct skcipher_alg *algs, int count)
		387	{
		388	int i, ret;
		389
		390	for (i = 0; i < count; i++) {
		391	ret = crypto_register_skcipher(&algs[i]);
		392	if (ret)
		393	goto err;
		394	}
		395
		396	return 0;
		397
		398	err:
		399	for (--i; i >= 0; --i)
		400	crypto_unregister_skcipher(&algs[i]);
		401
		402	return ret;
		403	}
		404	EXPORT_SYMBOL_GPL(crypto_register_skciphers);
		405
		406	void crypto_unregister_skciphers(struct skcipher_alg *algs, int count)
		407	{
		408	int i;
		409
		410	for (i = count - 1; i >= 0; --i)
		411	crypto_unregister_skcipher(&algs[i]);
		412	}
		413	EXPORT_SYMBOL_GPL(crypto_unregister_skciphers);
		414
		415	int skcipher_register_instance(struct crypto_template *tmpl,
		416	struct skcipher_instance *inst)
		417	{
		418	int err;
		419
		420	err = skcipher_prepare_alg(&inst->alg);
		421	if (err)
		422	return err;
		423
		424	return crypto_register_instance(tmpl, skcipher_crypto_instance(inst));
		425	}
		426	EXPORT_SYMBOL_GPL(skcipher_register_instance);
		427
246	MODULE_LICENSE("GPL");	428	MODULE_LICENSE("GPL");
247	MODULE_DESCRIPTION("Symmetric key cipher type");	429	MODULE_DESCRIPTION("Symmetric key cipher type");

crypto/tcrypt.c+248−194

@@ -24,6 +24,7 @@ · 23 unmodified lines
24		24
25	#include <crypto/aead.h>	25	#include <crypto/aead.h>
26	#include <crypto/hash.h>	26	#include <crypto/hash.h>
		27	#include <crypto/skcipher.h>
27	#include <linux/err.h>	28	#include <linux/err.h>
28	#include <linux/fips.h>	29	#include <linux/fips.h>
29	#include <linux/init.h>	30	#include <linux/init.h>
@@ -72,7 +73,8 @@ static char *check[] = { · 42 unmodified lines
72	"cast6", "arc4", "michael_mic", "deflate", "crc32c", "tea", "xtea",	73	"cast6", "arc4", "michael_mic", "deflate", "crc32c", "tea", "xtea",
73	"khazad", "wp512", "wp384", "wp256", "tnepres", "xeta", "fcrypt",	74	"khazad", "wp512", "wp384", "wp256", "tnepres", "xeta", "fcrypt",
74	"camellia", "seed", "salsa20", "rmd128", "rmd160", "rmd256", "rmd320",	75	"camellia", "seed", "salsa20", "rmd128", "rmd160", "rmd256", "rmd320",
75	"lzo", "cts", "zlib", NULL	76	"lzo", "cts", "zlib", "sha3-224", "sha3-256", "sha3-384", "sha3-512",
		77	NULL
76	};	78	};
77		79
78	struct tcrypt_result {	80	struct tcrypt_result {
@@ -91,76 +93,6 @@ static void tcrypt_complete(struct crypto_async_request *req, int err) · 12 unmodified lines
91	complete(&res->completion);	93	complete(&res->completion);
92	}	94	}
93		95
94	static int test_cipher_jiffies(struct blkcipher_desc *desc, int enc,
95	struct scatterlist *sg, int blen, int secs)
96	{
97	unsigned long start, end;
98	int bcount;
99	int ret;
100
101	for (start = jiffies, end = start + secs * HZ, bcount = 0;
102	time_before(jiffies, end); bcount++) {
103	if (enc)
104	ret = crypto_blkcipher_encrypt(desc, sg, sg, blen);
105	else
106	ret = crypto_blkcipher_decrypt(desc, sg, sg, blen);
107
108	if (ret)
109	return ret;
110	}
111
112	printk("%d operations in %d seconds (%ld bytes)\n",
113	bcount, secs, (long)bcount * blen);
114	return 0;
115	}
116
117	static int test_cipher_cycles(struct blkcipher_desc *desc, int enc,
118	struct scatterlist *sg, int blen)
119	{
120	unsigned long cycles = 0;
121	int ret = 0;
122	int i;
123
124	local_irq_disable();
125
126	/* Warm-up run. */
127	for (i = 0; i < 4; i++) {
128	if (enc)
129	ret = crypto_blkcipher_encrypt(desc, sg, sg, blen);
130	else
131	ret = crypto_blkcipher_decrypt(desc, sg, sg, blen);
132
133	if (ret)
134	goto out;
135	}
136
137	/* The real thing. */
138	for (i = 0; i < 8; i++) {
139	cycles_t start, end;
140
141	start = get_cycles();
142	if (enc)
143	ret = crypto_blkcipher_encrypt(desc, sg, sg, blen);
144	else
145	ret = crypto_blkcipher_decrypt(desc, sg, sg, blen);
146	end = get_cycles();
147
148	if (ret)
149	goto out;
150
151	cycles += end - start;
152	}
153
154	out:
155	local_irq_enable();
156
157	if (ret == 0)
158	printk("1 operation in %lu cycles (%d bytes)\n",
159	(cycles + 4) / 8, blen);
160
161	return ret;
162	}
163
164	static inline int do_one_aead_op(struct aead_request *req, int ret)	96	static inline int do_one_aead_op(struct aead_request *req, int ret)
165	{	97	{
166	if (ret == -EINPROGRESS \|\| ret == -EBUSY) {	98	if (ret == -EINPROGRESS \|\| ret == -EBUSY) {
@@ -454,127 +386,148 @@ out_noxbuf: · 287 unmodified lines
454	return;	386	return;
455	}	387	}
456		388
457	static void test_cipher_speed(const char *algo, int enc, unsigned int secs,	389	static void test_hash_sg_init(struct scatterlist *sg)
458	struct cipher_speed_template *template,
459	unsigned int tcount, u8 *keysize)
460	{	390	{
461	unsigned int ret, i, j, iv_len;	391	int i;
462	const char *key;
463	char iv[128];
464	struct crypto_blkcipher *tfm;
465	struct blkcipher_desc desc;
466	const char *e;
467	u32 *b_size;
468		392
469	if (enc == ENCRYPT)	393	sg_init_table(sg, TVMEMSIZE);
470	e = "encryption";	394	for (i = 0; i < TVMEMSIZE; i++) {
471	else	395	sg_set_buf(sg + i, tvmem[i], PAGE_SIZE);
472	e = "decryption";	396	memset(tvmem[i], 0xff, PAGE_SIZE);
		397	}
		398	}
473		399
474	tfm = crypto_alloc_blkcipher(algo, 0, CRYPTO_ALG_ASYNC);	400	static inline int do_one_ahash_op(struct ahash_request *req, int ret)
		401	{
		402	if (ret == -EINPROGRESS \|\| ret == -EBUSY) {
		403	struct tcrypt_result *tr = req->base.data;
475		404
476	if (IS_ERR(tfm)) {	405	wait_for_completion(&tr->completion);
477	printk("failed to load transform for %s: %ld\n", algo,	406	reinit_completion(&tr->completion);
478	PTR_ERR(tfm));	407	ret = tr->err;
		408	}
		409	return ret;
		410	}
		411
		412	struct test_mb_ahash_data {
		413	struct scatterlist sg[TVMEMSIZE];
		414	char result[64];
		415	struct ahash_request *req;
		416	struct tcrypt_result tresult;
		417	char *xbuf[XBUFSIZE];
		418	};
		419
		420	static void test_mb_ahash_speed(const char *algo, unsigned int sec,
		421	struct hash_speed *speed)
		422	{
		423	struct test_mb_ahash_data *data;
		424	struct crypto_ahash *tfm;
		425	unsigned long start, end;
		426	unsigned long cycles;
		427	unsigned int i, j, k;
		428	int ret;
		429
		430	data = kzalloc(sizeof(data) 8, GFP_KERNEL);
		431	if (!data)
479	return;	432	return;
		433
		434	tfm = crypto_alloc_ahash(algo, 0, 0);
		435	if (IS_ERR(tfm)) {
		436	pr_err("failed to load transform for %s: %ld\n",
		437	algo, PTR_ERR(tfm));
		438	goto free_data;
480	}	439	}
481	desc.tfm = tfm;
482	desc.flags = 0;
483		440
484	printk(KERN_INFO "\ntesting speed of %s (%s) %s\n", algo,	441	for (i = 0; i < 8; ++i) {
485	get_driver_name(crypto_blkcipher, tfm), e);	442	if (testmgr_alloc_buf(data[i].xbuf))
		443	goto out;
486		444
487	i = 0;	445	init_completion(&data[i].tresult.completion);
488	do {
489		446
490	b_size = block_sizes;	447	data[i].req = ahash_request_alloc(tfm, GFP_KERNEL);
491	do {	448	if (!data[i].req) {
492	struct scatterlist sg[TVMEMSIZE];	449	pr_err("alg: hash: Failed to allocate request for %s\n",
		450	algo);
		451	goto out;
		452	}
493		453
494	if ((keysize + b_size) > TVMEMSIZE * PAGE_SIZE) {	454	ahash_request_set_callback(data[i].req, 0,
495	printk("template (%u) too big for "	455	tcrypt_complete, &data[i].tresult);
496	"tvmem (%lu)\n", keysize + b_size,	456	test_hash_sg_init(data[i].sg);
497	TVMEMSIZE * PAGE_SIZE);	457	}
498	goto out;
499	}
500		458
501	printk("test %u (%d bit key, %d byte blocks): ", i,	459	pr_info("\ntesting speed of multibuffer %s (%s)\n", algo,
502	keysize 8, *b_size);	460	get_driver_name(crypto_ahash, tfm));
503		461
504	memset(tvmem[0], 0xff, PAGE_SIZE);	462	for (i = 0; speed[i].blen != 0; i++) {
		463	/* For some reason this only tests digests. */
		464	if (speed[i].blen != speed[i].plen)
		465	continue;
505		466
506	/* set key, plain text and IV */	467	if (speed[i].blen > TVMEMSIZE * PAGE_SIZE) {
507	key = tvmem[0];	468	pr_err("template (%u) too big for tvmem (%lu)\n",
508	for (j = 0; j < tcount; j++) {	469	speed[i].blen, TVMEMSIZE * PAGE_SIZE);
509	if (template[j].klen == *keysize) {	470	goto out;
510	key = template[j].key;	471	}
511	break;
512	}
513	}
514		472
515	ret = crypto_blkcipher_setkey(tfm, key, *keysize);	473	if (speed[i].klen)
516	if (ret) {	474	crypto_ahash_setkey(tfm, tvmem[0], speed[i].klen);
517	printk("setkey() failed flags=%x\n",
518	crypto_blkcipher_get_flags(tfm));
519	goto out;
520	}
521		475
522	sg_init_table(sg, TVMEMSIZE);	476	for (k = 0; k < 8; k++)
523	sg_set_buf(sg, tvmem[0] + *keysize,	477	ahash_request_set_crypt(data[k].req, data[k].sg,
524	PAGE_SIZE - *keysize);	478	data[k].result, speed[i].blen);
525	for (j = 1; j < TVMEMSIZE; j++) {
526	sg_set_buf(sg + j, tvmem[j], PAGE_SIZE);
527	memset (tvmem[j], 0xff, PAGE_SIZE);
528	}
529		479
530	iv_len = crypto_blkcipher_ivsize(tfm);	480	pr_info("test%3u "
531	if (iv_len) {	481	"(%5u byte blocks,%5u bytes per update,%4u updates): ",
532	memset(&iv, 0xff, iv_len);	482	i, speed[i].blen, speed[i].plen,
533	crypto_blkcipher_set_iv(tfm, iv, iv_len);	483	speed[i].blen / speed[i].plen);
534	}
535		484
536	if (secs)	485	start = get_cycles();
537	ret = test_cipher_jiffies(&desc, enc, sg,
538	*b_size, secs);
539	else
540	ret = test_cipher_cycles(&desc, enc, sg,
541	*b_size);
542		486
543	if (ret) {	487	for (k = 0; k < 8; k++) {
544	printk("%s() failed flags=%x\n", e, desc.flags);	488	ret = crypto_ahash_digest(data[k].req);
545	break;	489	if (ret == -EINPROGRESS) {
		490	ret = 0;
		491	continue;
546	}	492	}
547	b_size++;
548	i++;
549	} while (*b_size);
550	keysize++;
551	} while (*keysize);
552		493
553	out:	494	if (ret)
554	crypto_free_blkcipher(tfm);	495	break;
555	}
556		496
557	static void test_hash_sg_init(struct scatterlist *sg)	497	complete(&data[k].tresult.completion);
558	{	498	data[k].tresult.err = 0;
559	int i;	499	}
560		500
561	sg_init_table(sg, TVMEMSIZE);	501	for (j = 0; j < k; j++) {
562	for (i = 0; i < TVMEMSIZE; i++) {	502	struct tcrypt_result *tr = &data[j].tresult;
563	sg_set_buf(sg + i, tvmem[i], PAGE_SIZE);
564	memset(tvmem[i], 0xff, PAGE_SIZE);
565	}
566	}
567		503
568	static inline int do_one_ahash_op(struct ahash_request *req, int ret)	504	wait_for_completion(&tr->completion);
569	{	505	if (tr->err)
570	if (ret == -EINPROGRESS \|\| ret == -EBUSY) {	506	ret = tr->err;
571	struct tcrypt_result *tr = req->base.data;	507	}
572		508
573	wait_for_completion(&tr->completion);	509	end = get_cycles();
574	reinit_completion(&tr->completion);	510	cycles = end - start;
575	ret = tr->err;	511	pr_cont("%6lu cycles/operation, %4lu cycles/byte\n",
		512	cycles, cycles / (8 * speed[i].blen));
		513
		514	if (ret) {
		515	pr_err("At least one hashing failed ret=%d\n", ret);
		516	break;
		517	}
576	}	518	}
577	return ret;	519
		520	out:
		521	for (k = 0; k < 8; ++k)
		522	ahash_request_free(data[k].req);
		523
		524	for (k = 0; k < 8; ++k)
		525	testmgr_free_buf(data[k].xbuf);
		526
		527	crypto_free_ahash(tfm);
		528
		529	free_data:
		530	kfree(data);
578	}	531	}
579		532
580	static int test_ahash_jiffies_digest(struct ahash_request *req, int blen,	533	static int test_ahash_jiffies_digest(struct ahash_request *req, int blen,
@@ -812,7 +765,7 @@ static void test_hash_speed(const char *algo, unsigned int secs, · 231 unmodified lines
812	return test_ahash_speed_common(algo, secs, speed, CRYPTO_ALG_ASYNC);	765	return test_ahash_speed_common(algo, secs, speed, CRYPTO_ALG_ASYNC);
813	}	766	}
814		767
815	static inline int do_one_acipher_op(struct ablkcipher_request *req, int ret)	768	static inline int do_one_acipher_op(struct skcipher_request *req, int ret)
816	{	769	{
817	if (ret == -EINPROGRESS \|\| ret == -EBUSY) {	770	if (ret == -EINPROGRESS \|\| ret == -EBUSY) {
818	struct tcrypt_result *tr = req->base.data;	771	struct tcrypt_result *tr = req->base.data;
@@ -825,7 +778,7 @@ static inline int do_one_acipher_op(struct ablkcipher_request *req, int ret) · 6 unmodified lines
825	return ret;	778	return ret;
826	}	779	}
827		780
828	static int test_acipher_jiffies(struct ablkcipher_request *req, int enc,	781	static int test_acipher_jiffies(struct skcipher_request *req, int enc,
829	int blen, int secs)	782	int blen, int secs)
830	{	783	{
831	unsigned long start, end;	784	unsigned long start, end;
@@ -836,10 +789,10 @@ static int test_acipher_jiffies(struct ablkcipher_request *req, int enc, · 4 unmodified lines
836	time_before(jiffies, end); bcount++) {	789	time_before(jiffies, end); bcount++) {
837	if (enc)	790	if (enc)
838	ret = do_one_acipher_op(req,	791	ret = do_one_acipher_op(req,
839	crypto_ablkcipher_encrypt(req));	792	crypto_skcipher_encrypt(req));
840	else	793	else
841	ret = do_one_acipher_op(req,	794	ret = do_one_acipher_op(req,
842	crypto_ablkcipher_decrypt(req));	795	crypto_skcipher_decrypt(req));
843		796
844	if (ret)	797	if (ret)
845	return ret;	798	return ret;
@@ -850,7 +803,7 @@ static int test_acipher_jiffies(struct ablkcipher_request *req, int enc, · 4 unmodified lines
850	return 0;	803	return 0;
851	}	804	}
852		805
853	static int test_acipher_cycles(struct ablkcipher_request *req, int enc,	806	static int test_acipher_cycles(struct skcipher_request *req, int enc,
854	int blen)	807	int blen)
855	{	808	{
856	unsigned long cycles = 0;	809	unsigned long cycles = 0;
@@ -861,10 +814,10 @@ static int test_acipher_cycles(struct ablkcipher_request *req, int enc, · 4 unmodified lines
861	for (i = 0; i < 4; i++) {	814	for (i = 0; i < 4; i++) {
862	if (enc)	815	if (enc)
863	ret = do_one_acipher_op(req,	816	ret = do_one_acipher_op(req,
864	crypto_ablkcipher_encrypt(req));	817	crypto_skcipher_encrypt(req));
865	else	818	else
866	ret = do_one_acipher_op(req,	819	ret = do_one_acipher_op(req,
867	crypto_ablkcipher_decrypt(req));	820	crypto_skcipher_decrypt(req));
868		821
869	if (ret)	822	if (ret)
870	goto out;	823	goto out;
@@ -877,10 +830,10 @@ static int test_acipher_cycles(struct ablkcipher_request *req, int enc, · 6 unmodified lines
877	start = get_cycles();	830	start = get_cycles();
878	if (enc)	831	if (enc)
879	ret = do_one_acipher_op(req,	832	ret = do_one_acipher_op(req,
880	crypto_ablkcipher_encrypt(req));	833	crypto_skcipher_encrypt(req));
881	else	834	else
882	ret = do_one_acipher_op(req,	835	ret = do_one_acipher_op(req,
883	crypto_ablkcipher_decrypt(req));	836	crypto_skcipher_decrypt(req));
884	end = get_cycles();	837	end = get_cycles();
885		838
886	if (ret)	839	if (ret)
@@ -897,16 +850,16 @@ out: · 10 unmodified lines
897	return ret;	850	return ret;
898	}	851	}
899		852
900	static void test_acipher_speed(const char *algo, int enc, unsigned int secs,	853	static void test_skcipher_speed(const char *algo, int enc, unsigned int secs,
901	struct cipher_speed_template *template,	854	struct cipher_speed_template *template,
902	unsigned int tcount, u8 *keysize)	855	unsigned int tcount, u8 *keysize, bool async)
903	{	856	{
904	unsigned int ret, i, j, k, iv_len;	857	unsigned int ret, i, j, k, iv_len;
905	struct tcrypt_result tresult;	858	struct tcrypt_result tresult;
906	const char *key;	859	const char *key;
907	char iv[128];	860	char iv[128];
908	struct ablkcipher_request *req;	861	struct skcipher_request *req;
909	struct crypto_ablkcipher *tfm;	862	struct crypto_skcipher *tfm;
910	const char *e;	863	const char *e;
911	u32 *b_size;	864	u32 *b_size;
912		865
@@ -917,7 +870,7 @@ static void test_acipher_speed(const char *algo, int enc, unsigned int secs, · 4 unmodified lines
917		870
918	init_completion(&tresult.completion);	871	init_completion(&tresult.completion);
919		872
920	tfm = crypto_alloc_ablkcipher(algo, 0, 0);	873	tfm = crypto_alloc_skcipher(algo, 0, async ? 0 : CRYPTO_ALG_ASYNC);
921		874
922	if (IS_ERR(tfm)) {	875	if (IS_ERR(tfm)) {
923	pr_err("failed to load transform for %s: %ld\n", algo,	876	pr_err("failed to load transform for %s: %ld\n", algo,
@@ -926,17 +879,17 @@ static void test_acipher_speed(const char *algo, int enc, unsigned int secs, · 2 unmodified lines
926	}	879	}
927		880
928	pr_info("\ntesting speed of async %s (%s) %s\n", algo,	881	pr_info("\ntesting speed of async %s (%s) %s\n", algo,
929	get_driver_name(crypto_ablkcipher, tfm), e);	882	get_driver_name(crypto_skcipher, tfm), e);
930		883
931	req = ablkcipher_request_alloc(tfm, GFP_KERNEL);	884	req = skcipher_request_alloc(tfm, GFP_KERNEL);
932	if (!req) {	885	if (!req) {
933	pr_err("tcrypt: skcipher: Failed to allocate request for %s\n",	886	pr_err("tcrypt: skcipher: Failed to allocate request for %s\n",
934	algo);	887	algo);
935	goto out;	888	goto out;
936	}	889	}
937		890
938	ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,	891	skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
939	tcrypt_complete, &tresult);	892	tcrypt_complete, &tresult);
940		893
941	i = 0;	894	i = 0;
942	do {	895	do {
@@ -966,12 +919,12 @@ static void test_acipher_speed(const char *algo, int enc, unsigned int secs, · 23 unmodified lines
966	}	919	}
967	}	920	}
968		921
969	crypto_ablkcipher_clear_flags(tfm, ~0);	922	crypto_skcipher_clear_flags(tfm, ~0);
970		923
971	ret = crypto_ablkcipher_setkey(tfm, key, *keysize);	924	ret = crypto_skcipher_setkey(tfm, key, *keysize);
972	if (ret) {	925	if (ret) {
973	pr_err("setkey() failed flags=%x\n",	926	pr_err("setkey() failed flags=%x\n",
974	crypto_ablkcipher_get_flags(tfm));	927	crypto_skcipher_get_flags(tfm));
975	goto out_free_req;	928	goto out_free_req;
976	}	929	}
977		930
@@ -995,11 +948,11 @@ static void test_acipher_speed(const char *algo, int enc, unsigned int secs, · 17 unmodified lines
995	sg_set_buf(sg, tvmem[0] + keysize, b_size);	948	sg_set_buf(sg, tvmem[0] + keysize, b_size);
996	}	949	}
997		950
998	iv_len = crypto_ablkcipher_ivsize(tfm);	951	iv_len = crypto_skcipher_ivsize(tfm);
999	if (iv_len)	952	if (iv_len)
1000	memset(&iv, 0xff, iv_len);	953	memset(&iv, 0xff, iv_len);
1001		954
1002	ablkcipher_request_set_crypt(req, sg, sg, *b_size, iv);	955	skcipher_request_set_crypt(req, sg, sg, *b_size, iv);
1003		956
1004	if (secs)	957	if (secs)
1005	ret = test_acipher_jiffies(req, enc,	958	ret = test_acipher_jiffies(req, enc,
@@ -1010,7 +963,7 @@ static void test_acipher_speed(const char *algo, int enc, unsigned int secs, · 4 unmodified lines
1010		963
1011	if (ret) {	964	if (ret) {
1012	pr_err("%s() failed flags=%x\n", e,	965	pr_err("%s() failed flags=%x\n", e,
1013	crypto_ablkcipher_get_flags(tfm));	966	crypto_skcipher_get_flags(tfm));
1014	break;	967	break;
1015	}	968	}
1016	b_size++;	969	b_size++;
@@ -1020,9 +973,25 @@ static void test_acipher_speed(const char *algo, int enc, unsigned int secs, · 3 unmodified lines
1020	} while (*keysize);	973	} while (*keysize);
1021		974
1022	out_free_req:	975	out_free_req:
1023	ablkcipher_request_free(req);	976	skcipher_request_free(req);
1024	out:	977	out:
1025	crypto_free_ablkcipher(tfm);	978	crypto_free_skcipher(tfm);
		979	}
		980
		981	static void test_acipher_speed(const char *algo, int enc, unsigned int secs,
		982	struct cipher_speed_template *template,
		983	unsigned int tcount, u8 *keysize)
		984	{
		985	return test_skcipher_speed(algo, enc, secs, template, tcount, keysize,
		986	true);
		987	}
		988
		989	static void test_cipher_speed(const char *algo, int enc, unsigned int secs,
		990	struct cipher_speed_template *template,
		991	unsigned int tcount, u8 *keysize)
		992	{
		993	return test_skcipher_speed(algo, enc, secs, template, tcount, keysize,
		994	false);
1026	}	995	}
1027		996
1028	static void test_available(void)	997	static void test_available(void)
@@ -1284,6 +1253,22 @@ static int do_test(const char *alg, u32 type, u32 mask, int m) · 255 unmodified lines
1284	ret += tcrypt_test("crct10dif");	1253	ret += tcrypt_test("crct10dif");
1285	break;	1254	break;
1286		1255
		1256	case 48:
		1257	ret += tcrypt_test("sha3-224");
		1258	break;
		1259
		1260	case 49:
		1261	ret += tcrypt_test("sha3-256");
		1262	break;
		1263
		1264	case 50:
		1265	ret += tcrypt_test("sha3-384");
		1266	break;
		1267
		1268	case 51:
		1269	ret += tcrypt_test("sha3-512");
		1270	break;
		1271
1287	case 100:	1272	case 100:
1288	ret += tcrypt_test("hmac(md5)");	1273	ret += tcrypt_test("hmac(md5)");
1289	break;	1274	break;
@@ -1328,6 +1313,22 @@ static int do_test(const char *alg, u32 type, u32 mask, int m) · 38 unmodified lines
1328	ret += tcrypt_test("hmac(crc32)");	1313	ret += tcrypt_test("hmac(crc32)");
1329	break;	1314	break;
1330		1315
		1316	case 111:
		1317	ret += tcrypt_test("hmac(sha3-224)");
		1318	break;
		1319
		1320	case 112:
		1321	ret += tcrypt_test("hmac(sha3-256)");
		1322	break;
		1323
		1324	case 113:
		1325	ret += tcrypt_test("hmac(sha3-384)");
		1326	break;
		1327
		1328	case 114:
		1329	ret += tcrypt_test("hmac(sha3-512)");
		1330	break;
		1331
1331	case 150:	1332	case 150:
1332	ret += tcrypt_test("ansi_cprng");	1333	ret += tcrypt_test("ansi_cprng");
1333	break;	1334	break;
@@ -1406,6 +1407,10 @@ static int do_test(const char *alg, u32 type, u32 mask, int m) · 72 unmodified lines
1406	speed_template_32_48_64);	1407	speed_template_32_48_64);
1407	test_cipher_speed("xts(aes)", DECRYPT, sec, NULL, 0,	1408	test_cipher_speed("xts(aes)", DECRYPT, sec, NULL, 0,
1408	speed_template_32_48_64);	1409	speed_template_32_48_64);
		1410	test_cipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0,
		1411	speed_template_16_24_32);
		1412	test_cipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0,
		1413	speed_template_16_24_32);
1409	test_cipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0,	1414	test_cipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0,
1410	speed_template_16_24_32);	1415	speed_template_16_24_32);
1411	test_cipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0,	1416	test_cipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0,
@@ -1691,6 +1696,22 @@ static int do_test(const char *alg, u32 type, u32 mask, int m) · 279 unmodified lines
1691	test_hash_speed("poly1305", sec, poly1305_speed_template);	1696	test_hash_speed("poly1305", sec, poly1305_speed_template);
1692	if (mode > 300 && mode < 400) break;	1697	if (mode > 300 && mode < 400) break;
1693		1698
		1699	case 322:
		1700	test_hash_speed("sha3-224", sec, generic_hash_speed_template);
		1701	if (mode > 300 && mode < 400) break;
		1702
		1703	case 323:
		1704	test_hash_speed("sha3-256", sec, generic_hash_speed_template);
		1705	if (mode > 300 && mode < 400) break;
		1706
		1707	case 324:
		1708	test_hash_speed("sha3-384", sec, generic_hash_speed_template);
		1709	if (mode > 300 && mode < 400) break;
		1710
		1711	case 325:
		1712	test_hash_speed("sha3-512", sec, generic_hash_speed_template);
		1713	if (mode > 300 && mode < 400) break;
		1714
1694	case 399:	1715	case 399:
1695	break;	1716	break;
1696		1717
@@ -1770,6 +1791,35 @@ static int do_test(const char *alg, u32 type, u32 mask, int m) · 73 unmodified lines
1770	test_ahash_speed("rmd320", sec, generic_hash_speed_template);	1791	test_ahash_speed("rmd320", sec, generic_hash_speed_template);
1771	if (mode > 400 && mode < 500) break;	1792	if (mode > 400 && mode < 500) break;
1772		1793
		1794	case 418:
		1795	test_ahash_speed("sha3-224", sec, generic_hash_speed_template);
		1796	if (mode > 400 && mode < 500) break;
		1797
		1798	case 419:
		1799	test_ahash_speed("sha3-256", sec, generic_hash_speed_template);
		1800	if (mode > 400 && mode < 500) break;
		1801
		1802	case 420:
		1803	test_ahash_speed("sha3-384", sec, generic_hash_speed_template);
		1804	if (mode > 400 && mode < 500) break;
		1805
		1806
		1807	case 421:
		1808	test_ahash_speed("sha3-512", sec, generic_hash_speed_template);
		1809	if (mode > 400 && mode < 500) break;
		1810
		1811	case 422:
		1812	test_mb_ahash_speed("sha1", sec, generic_hash_speed_template);
		1813	if (mode > 400 && mode < 500) break;
		1814
		1815	case 423:
		1816	test_mb_ahash_speed("sha256", sec, generic_hash_speed_template);
		1817	if (mode > 400 && mode < 500) break;
		1818
		1819	case 424:
		1820	test_mb_ahash_speed("sha512", sec, generic_hash_speed_template);
		1821	if (mode > 400 && mode < 500) break;
		1822
1773	case 499:	1823	case 499:
1774	break;	1824	break;
1775		1825
@@ -1790,6 +1840,10 @@ static int do_test(const char *alg, u32 type, u32 mask, int m) · 14 unmodified lines
1790	speed_template_32_48_64);	1840	speed_template_32_48_64);
1791	test_acipher_speed("xts(aes)", DECRYPT, sec, NULL, 0,	1841	test_acipher_speed("xts(aes)", DECRYPT, sec, NULL, 0,
1792	speed_template_32_48_64);	1842	speed_template_32_48_64);
		1843	test_acipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0,
		1844	speed_template_16_24_32);
		1845	test_acipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0,
		1846	speed_template_16_24_32);
1793	test_acipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0,	1847	test_acipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0,
1794	speed_template_16_24_32);	1848	speed_template_16_24_32);
1795	test_acipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0,	1849	test_acipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0,

crypto/testmgr.c+261−27

@@ -32,6 +32,7 @@ · 31 unmodified lines
32	#include <crypto/rng.h>	32	#include <crypto/rng.h>
33	#include <crypto/drbg.h>	33	#include <crypto/drbg.h>
34	#include <crypto/akcipher.h>	34	#include <crypto/akcipher.h>
		35	#include <crypto/kpp.h>
35		36
36	#include "internal.h"	37	#include "internal.h"
37		38
@@ -120,6 +121,11 @@ struct akcipher_test_suite { · 82 unmodified lines
120	unsigned int count;	121	unsigned int count;
121	};	122	};
122		123
		124	struct kpp_test_suite {
		125	struct kpp_testvec *vecs;
		126	unsigned int count;
		127	};
		128
123	struct alg_test_desc {	129	struct alg_test_desc {
124	const char *alg;	130	const char *alg;
125	int (test)(const struct alg_test_desc desc, const char *driver,	131	int (test)(const struct alg_test_desc desc, const char *driver,
@@ -134,6 +140,7 @@ struct alg_test_desc { · 8 unmodified lines
134	struct cprng_test_suite cprng;	140	struct cprng_test_suite cprng;
135	struct drbg_test_suite drbg;	141	struct drbg_test_suite drbg;
136	struct akcipher_test_suite akcipher;	142	struct akcipher_test_suite akcipher;
		143	struct kpp_test_suite kpp;
137	} suite;	144	} suite;
138	};	145	};
139		146
@@ -1777,8 +1784,135 @@ static int alg_test_drbg(const struct alg_test_desc desc, const char driver, · 1637 unmodified lines
1777		1784
1778	}	1785	}
1779		1786
1780	static int do_test_rsa(struct crypto_akcipher *tfm,	1787	static int do_test_kpp(struct crypto_kpp tfm, struct kpp_testvec vec,
1781	struct akcipher_testvec *vecs)	1788	const char *alg)
		1789	{
		1790	struct kpp_request *req;
		1791	void *input_buf = NULL;
		1792	void *output_buf = NULL;
		1793	struct tcrypt_result result;
		1794	unsigned int out_len_max;
		1795	int err = -ENOMEM;
		1796	struct scatterlist src, dst;
		1797
		1798	req = kpp_request_alloc(tfm, GFP_KERNEL);
		1799	if (!req)
		1800	return err;
		1801
		1802	init_completion(&result.completion);
		1803
		1804	err = crypto_kpp_set_secret(tfm, vec->secret, vec->secret_size);
		1805	if (err < 0)
		1806	goto free_req;
		1807
		1808	out_len_max = crypto_kpp_maxsize(tfm);
		1809	output_buf = kzalloc(out_len_max, GFP_KERNEL);
		1810	if (!output_buf) {
		1811	err = -ENOMEM;
		1812	goto free_req;
		1813	}
		1814
		1815	/* Use appropriate parameter as base */
		1816	kpp_request_set_input(req, NULL, 0);
		1817	sg_init_one(&dst, output_buf, out_len_max);
		1818	kpp_request_set_output(req, &dst, out_len_max);
		1819	kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
		1820	tcrypt_complete, &result);
		1821
		1822	/* Compute public key */
		1823	err = wait_async_op(&result, crypto_kpp_generate_public_key(req));
		1824	if (err) {
		1825	pr_err("alg: %s: generate public key test failed. err %d\n",
		1826	alg, err);
		1827	goto free_output;
		1828	}
		1829	/* Verify calculated public key */
		1830	if (memcmp(vec->expected_a_public, sg_virt(req->dst),
		1831	vec->expected_a_public_size)) {
		1832	pr_err("alg: %s: generate public key test failed. Invalid output\n",
		1833	alg);
		1834	err = -EINVAL;
		1835	goto free_output;
		1836	}
		1837
		1838	/* Calculate shared secret key by using counter part (b) public key. */
		1839	input_buf = kzalloc(vec->b_public_size, GFP_KERNEL);
		1840	if (!input_buf) {
		1841	err = -ENOMEM;
		1842	goto free_output;
		1843	}
		1844
		1845	memcpy(input_buf, vec->b_public, vec->b_public_size);
		1846	sg_init_one(&src, input_buf, vec->b_public_size);
		1847	sg_init_one(&dst, output_buf, out_len_max);
		1848	kpp_request_set_input(req, &src, vec->b_public_size);
		1849	kpp_request_set_output(req, &dst, out_len_max);
		1850	kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
		1851	tcrypt_complete, &result);
		1852	err = wait_async_op(&result, crypto_kpp_compute_shared_secret(req));
		1853	if (err) {
		1854	pr_err("alg: %s: compute shard secret test failed. err %d\n",
		1855	alg, err);
		1856	goto free_all;
		1857	}
		1858	/*
		1859	* verify shared secret from which the user will derive
		1860	* secret key by executing whatever hash it has chosen
		1861	*/
		1862	if (memcmp(vec->expected_ss, sg_virt(req->dst),
		1863	vec->expected_ss_size)) {
		1864	pr_err("alg: %s: compute shared secret test failed. Invalid output\n",
		1865	alg);
		1866	err = -EINVAL;
		1867	}
		1868
		1869	free_all:
		1870	kfree(input_buf);
		1871	free_output:
		1872	kfree(output_buf);
		1873	free_req:
		1874	kpp_request_free(req);
		1875	return err;
		1876	}
		1877
		1878	static int test_kpp(struct crypto_kpp tfm, const char alg,
		1879	struct kpp_testvec *vecs, unsigned int tcount)
		1880	{
		1881	int ret, i;
		1882
		1883	for (i = 0; i < tcount; i++) {
		1884	ret = do_test_kpp(tfm, vecs++, alg);
		1885	if (ret) {
		1886	pr_err("alg: %s: test failed on vector %d, err=%d\n",
		1887	alg, i + 1, ret);
		1888	return ret;
		1889	}
		1890	}
		1891	return 0;
		1892	}
		1893
		1894	static int alg_test_kpp(const struct alg_test_desc desc, const char driver,
		1895	u32 type, u32 mask)
		1896	{
		1897	struct crypto_kpp *tfm;
		1898	int err = 0;
		1899
		1900	tfm = crypto_alloc_kpp(driver, type \| CRYPTO_ALG_INTERNAL, mask);
		1901	if (IS_ERR(tfm)) {
		1902	pr_err("alg: kpp: Failed to load tfm for %s: %ld\n",
		1903	driver, PTR_ERR(tfm));
		1904	return PTR_ERR(tfm);
		1905	}
		1906	if (desc->suite.kpp.vecs)
		1907	err = test_kpp(tfm, desc->alg, desc->suite.kpp.vecs,
		1908	desc->suite.kpp.count);
		1909
		1910	crypto_free_kpp(tfm);
		1911	return err;
		1912	}
		1913
		1914	static int test_akcipher_one(struct crypto_akcipher *tfm,
		1915	struct akcipher_testvec *vecs)
1782	{	1916	{
1783	char *xbuf[XBUFSIZE];	1917	char *xbuf[XBUFSIZE];
1784	struct akcipher_request *req;	1918	struct akcipher_request *req;
@@ -1807,6 +1941,7 @@ static int do_test_rsa(struct crypto_akcipher *tfm, · 22 unmodified lines
1807	if (err)	1941	if (err)
1808	goto free_req;	1942	goto free_req;
1809		1943
		1944	err = -ENOMEM;
1810	out_len_max = crypto_akcipher_maxsize(tfm);	1945	out_len_max = crypto_akcipher_maxsize(tfm);
1811	outbuf_enc = kzalloc(out_len_max, GFP_KERNEL);	1946	outbuf_enc = kzalloc(out_len_max, GFP_KERNEL);
1812	if (!outbuf_enc)	1947	if (!outbuf_enc)
@@ -1829,17 +1964,18 @@ static int do_test_rsa(struct crypto_akcipher *tfm, · 16 unmodified lines
1829	/* Run RSA encrypt - c = m^e mod n;*/	1964	/* Run RSA encrypt - c = m^e mod n;*/
1830	err = wait_async_op(&result, crypto_akcipher_encrypt(req));	1965	err = wait_async_op(&result, crypto_akcipher_encrypt(req));
1831	if (err) {	1966	if (err) {
1832	pr_err("alg: rsa: encrypt test failed. err %d\n", err);	1967	pr_err("alg: akcipher: encrypt test failed. err %d\n", err);
1833	goto free_all;	1968	goto free_all;
1834	}	1969	}
1835	if (req->dst_len != vecs->c_size) {	1970	if (req->dst_len != vecs->c_size) {
1836	pr_err("alg: rsa: encrypt test failed. Invalid output len\n");	1971	pr_err("alg: akcipher: encrypt test failed. Invalid output len\n");
1837	err = -EINVAL;	1972	err = -EINVAL;
1838	goto free_all;	1973	goto free_all;
1839	}	1974	}
1840	/* verify that encrypted message is equal to expected */	1975	/* verify that encrypted message is equal to expected */
1841	if (memcmp(vecs->c, outbuf_enc, vecs->c_size)) {	1976	if (memcmp(vecs->c, outbuf_enc, vecs->c_size)) {
1842	pr_err("alg: rsa: encrypt test failed. Invalid output\n");	1977	pr_err("alg: akcipher: encrypt test failed. Invalid output\n");
		1978	hexdump(outbuf_enc, vecs->c_size);
1843	err = -EINVAL;	1979	err = -EINVAL;
1844	goto free_all;	1980	goto free_all;
1845	}	1981	}
@@ -1867,18 +2003,22 @@ static int do_test_rsa(struct crypto_akcipher *tfm, · 21 unmodified lines
1867	/* Run RSA decrypt - m = c^d mod n;*/	2003	/* Run RSA decrypt - m = c^d mod n;*/
1868	err = wait_async_op(&result, crypto_akcipher_decrypt(req));	2004	err = wait_async_op(&result, crypto_akcipher_decrypt(req));
1869	if (err) {	2005	if (err) {
1870	pr_err("alg: rsa: decrypt test failed. err %d\n", err);	2006	pr_err("alg: akcipher: decrypt test failed. err %d\n", err);
1871	goto free_all;	2007	goto free_all;
1872	}	2008	}
1873	out_len = req->dst_len;	2009	out_len = req->dst_len;
1874	if (out_len != vecs->m_size) {	2010	if (out_len < vecs->m_size) {
1875	pr_err("alg: rsa: decrypt test failed. Invalid output len\n");	2011	pr_err("alg: akcipher: decrypt test failed. "
		2012	"Invalid output len %u\n", out_len);
1876	err = -EINVAL;	2013	err = -EINVAL;
1877	goto free_all;	2014	goto free_all;
1878	}	2015	}
1879	/* verify that decrypted message is equal to the original msg */	2016	/* verify that decrypted message is equal to the original msg */
1880	if (memcmp(vecs->m, outbuf_dec, vecs->m_size)) {	2017	if (memchr_inv(outbuf_dec, 0, out_len - vecs->m_size) \|\|
1881	pr_err("alg: rsa: decrypt test failed. Invalid output\n");	2018	memcmp(vecs->m, outbuf_dec + out_len - vecs->m_size,
		2019	vecs->m_size)) {
		2020	pr_err("alg: akcipher: decrypt test failed. Invalid output\n");
		2021	hexdump(outbuf_dec, out_len);
1882	err = -EINVAL;	2022	err = -EINVAL;
1883	}	2023	}
1884	free_all:	2024	free_all:
@@ -1891,28 +2031,22 @@ free_xbuf: · 6 unmodified lines
1891	return err;	2031	return err;
1892	}	2032	}
1893		2033
1894	static int test_rsa(struct crypto_akcipher tfm, struct akcipher_testvec vecs,	2034	static int test_akcipher(struct crypto_akcipher tfm, const char alg,
1895	unsigned int tcount)	2035	struct akcipher_testvec *vecs, unsigned int tcount)
1896	{	2036	{
		2037	const char *algo =
		2038	crypto_tfm_alg_driver_name(crypto_akcipher_tfm(tfm));
1897	int ret, i;	2039	int ret, i;
1898		2040
1899	for (i = 0; i < tcount; i++) {	2041	for (i = 0; i < tcount; i++) {
1900	ret = do_test_rsa(tfm, vecs++);	2042	ret = test_akcipher_one(tfm, vecs++);
1901	if (ret) {	2043	if (!ret)
1902	pr_err("alg: rsa: test failed on vector %d, err=%d\n",	2044	continue;
1903	i + 1, ret);
1904	return ret;
1905	}
1906	}
1907	return 0;
1908	}
1909
1910	static int test_akcipher(struct crypto_akcipher tfm, const char alg,
1911	struct akcipher_testvec *vecs, unsigned int tcount)
1912	{
1913	if (strncmp(alg, "rsa", 3) == 0)
1914	return test_rsa(tfm, vecs, tcount);
1915		2045
		2046	pr_err("alg: akcipher: test %d failed for %s, err=%d\n",
		2047	i + 1, algo, ret);
		2048	return ret;
		2049	}
1916	return 0;	2050	return 0;
1917	}	2051	}
1918		2052
@@ -2728,6 +2862,16 @@ static const struct alg_test_desc alg_test_descs[] = { · 809 unmodified lines
2728	}	2862	}
2729	}	2863	}
2730	}	2864	}
		2865	}, {
		2866	.alg = "dh",
		2867	.test = alg_test_kpp,
		2868	.fips_allowed = 1,
		2869	.suite = {
		2870	.kpp = {
		2871	.vecs = dh_tv_template,
		2872	.count = DH_TEST_VECTORS
		2873	}
		2874	}
2731	}, {	2875	}, {
2732	.alg = "digest_null",	2876	.alg = "digest_null",
2733	.test = alg_test_null,	2877	.test = alg_test_null,
@@ -3156,6 +3300,16 @@ static const struct alg_test_desc alg_test_descs[] = { · 422 unmodified lines
3156	}	3300	}
3157	}	3301	}
3158	}	3302	}
		3303	}, {
		3304	.alg = "ecdh",
		3305	.test = alg_test_kpp,
		3306	.fips_allowed = 1,
		3307	.suite = {
		3308	.kpp = {
		3309	.vecs = ecdh_tv_template,
		3310	.count = ECDH_TEST_VECTORS
		3311	}
		3312	}
3159	}, {	3313	}, {
3160	.alg = "gcm(aes)",	3314	.alg = "gcm(aes)",
3161	.test = alg_test_aead,	3315	.test = alg_test_aead,
@@ -3248,6 +3402,46 @@ static const struct alg_test_desc alg_test_descs[] = { · 86 unmodified lines
3248	.count = HMAC_SHA256_TEST_VECTORS	3402	.count = HMAC_SHA256_TEST_VECTORS
3249	}	3403	}
3250	}	3404	}
		3405	}, {
		3406	.alg = "hmac(sha3-224)",
		3407	.test = alg_test_hash,
		3408	.fips_allowed = 1,
		3409	.suite = {
		3410	.hash = {
		3411	.vecs = hmac_sha3_224_tv_template,
		3412	.count = HMAC_SHA3_224_TEST_VECTORS
		3413	}
		3414	}
		3415	}, {
		3416	.alg = "hmac(sha3-256)",
		3417	.test = alg_test_hash,
		3418	.fips_allowed = 1,
		3419	.suite = {
		3420	.hash = {
		3421	.vecs = hmac_sha3_256_tv_template,
		3422	.count = HMAC_SHA3_256_TEST_VECTORS
		3423	}
		3424	}
		3425	}, {
		3426	.alg = "hmac(sha3-384)",
		3427	.test = alg_test_hash,
		3428	.fips_allowed = 1,
		3429	.suite = {
		3430	.hash = {
		3431	.vecs = hmac_sha3_384_tv_template,
		3432	.count = HMAC_SHA3_384_TEST_VECTORS
		3433	}
		3434	}
		3435	}, {
		3436	.alg = "hmac(sha3-512)",
		3437	.test = alg_test_hash,
		3438	.fips_allowed = 1,
		3439	.suite = {
		3440	.hash = {
		3441	.vecs = hmac_sha3_512_tv_template,
		3442	.count = HMAC_SHA3_512_TEST_VECTORS
		3443	}
		3444	}
3251	}, {	3445	}, {
3252	.alg = "hmac(sha384)",	3446	.alg = "hmac(sha384)",
3253	.test = alg_test_hash,	3447	.test = alg_test_hash,
@@ -3658,6 +3852,46 @@ static const struct alg_test_desc alg_test_descs[] = { · 404 unmodified lines
3658	.count = SHA256_TEST_VECTORS	3852	.count = SHA256_TEST_VECTORS
3659	}	3853	}
3660	}	3854	}
		3855	}, {
		3856	.alg = "sha3-224",
		3857	.test = alg_test_hash,
		3858	.fips_allowed = 1,
		3859	.suite = {
		3860	.hash = {
		3861	.vecs = sha3_224_tv_template,
		3862	.count = SHA3_224_TEST_VECTORS
		3863	}
		3864	}
		3865	}, {
		3866	.alg = "sha3-256",
		3867	.test = alg_test_hash,
		3868	.fips_allowed = 1,
		3869	.suite = {
		3870	.hash = {
		3871	.vecs = sha3_256_tv_template,
		3872	.count = SHA3_256_TEST_VECTORS
		3873	}
		3874	}
		3875	}, {
		3876	.alg = "sha3-384",
		3877	.test = alg_test_hash,
		3878	.fips_allowed = 1,
		3879	.suite = {
		3880	.hash = {
		3881	.vecs = sha3_384_tv_template,
		3882	.count = SHA3_384_TEST_VECTORS
		3883	}
		3884	}
		3885	}, {
		3886	.alg = "sha3-512",
		3887	.test = alg_test_hash,
		3888	.fips_allowed = 1,
		3889	.suite = {
		3890	.hash = {
		3891	.vecs = sha3_512_tv_template,
		3892	.count = SHA3_512_TEST_VECTORS
		3893	}
		3894	}
3661	}, {	3895	}, {
3662	.alg = "sha384",	3896	.alg = "sha384",
3663	.test = alg_test_hash,	3897	.test = alg_test_hash,

crypto/testmgr.h+1035−1

@@ -133,6 +133,17 @@ struct akcipher_testvec { · 132 unmodified lines
133	bool public_key_vec;	133	bool public_key_vec;
134	};	134	};
135		135
		136	struct kpp_testvec {
		137	unsigned char *secret;
		138	unsigned char *b_public;
		139	unsigned char *expected_a_public;
		140	unsigned char *expected_ss;
		141	unsigned short secret_size;
		142	unsigned short b_public_size;
		143	unsigned short expected_a_public_size;
		144	unsigned short expected_ss_size;
		145	};
		146
136	static char zeroed_string[48];	147	static char zeroed_string[48];
137		148
138	/*	149	/*
@@ -141,7 +152,7 @@ static char zeroed_string[48]; · 2 unmodified lines
141	#ifdef CONFIG_CRYPTO_FIPS	152	#ifdef CONFIG_CRYPTO_FIPS
142	#define RSA_TEST_VECTORS 2	153	#define RSA_TEST_VECTORS 2
143	#else	154	#else
144	#define RSA_TEST_VECTORS 4	155	#define RSA_TEST_VECTORS 5
145	#endif	156	#endif
146	static struct akcipher_testvec rsa_tv_template[] = {	157	static struct akcipher_testvec rsa_tv_template[] = {
147	{	158	{
@@ -327,6 +338,516 @@ static struct akcipher_testvec rsa_tv_template[] = { · 179 unmodified lines
327	.m_size = 8,	338	.m_size = 8,
328	.c_size = 256,	339	.c_size = 256,
329	.public_key_vec = true,	340	.public_key_vec = true,
		341	}, {
		342	.key =
		343	"\x30\x82\x09\x29" /* sequence of 2345 bytes */
		344	"\x02\x01\x00" /* version integer of 1 byte */
		345	"\x02\x82\x02\x01" /* modulus - integer of 513 bytes */
		346	"\x00\xC3\x8B\x55\x7B\x73\x4D\xFF\xE9\x9B\xC6\xDC\x67\x3C\xB4\x8E"
		347	"\xA0\x86\xED\xF2\xB9\x50\x5C\x54\x5C\xBA\xE4\xA1\xB2\xA7\xAE\x2F"
		348	"\x1B\x7D\xF1\xFB\xAC\x79\xC5\xDF\x1A\x00\xC9\xB2\xC1\x61\x25\x33"
		349	"\xE6\x9C\xE9\xCF\xD6\x27\xC4\x4E\x44\x30\x44\x5E\x08\xA1\x87\x52"
		350	"\xCC\x6B\x97\x70\x8C\xBC\xA5\x06\x31\x0C\xD4\x2F\xD5\x7D\x26\x24"
		351	"\xA2\xE2\xAC\x78\xF4\x53\x14\xCE\xF7\x19\x2E\xD7\xF7\xE6\x0C\xB9"
		352	"\x56\x7F\x0B\xF1\xB1\xE2\x43\x70\xBD\x86\x1D\xA1\xCC\x2B\x19\x08"
		353	"\x76\xEF\x91\xAC\xBF\x20\x24\x0D\x38\xC0\x89\xB8\x9A\x70\xB3\x64"
		354	"\xD9\x8F\x80\x41\x10\x5B\x9F\xB1\xCB\x76\x43\x00\x21\x25\x36\xD4"
		355	"\x19\xFC\x55\x95\x10\xE4\x26\x74\x98\x2C\xD9\xBD\x0B\x2B\x04\xC2"
		356	"\xAC\x82\x38\xB4\xDD\x4C\x04\x7E\x51\x36\x40\x1E\x0B\xC4\x7C\x25"
		357	"\xDD\x4B\xB2\xE7\x20\x0A\x57\xF9\xB4\x94\xC3\x08\x33\x22\x6F\x8B"
		358	"\x48\xDB\x03\x68\x5A\x5B\xBA\xAE\xF3\xAD\xCF\xC3\x6D\xBA\xF1\x28"
		359	"\x67\x7E\x6C\x79\x07\xDE\xFC\xED\xE7\x96\xE3\x6C\xE0\x2C\x87\xF8"
		360	"\x02\x01\x28\x38\x43\x21\x53\x84\x69\x75\x78\x15\x7E\xEE\xD2\x1B"
		361	"\xB9\x23\x40\xA8\x86\x1E\x38\x83\xB2\x73\x1D\x53\xFB\x9E\x2A\x8A"
		362	"\xB2\x75\x35\x01\xC3\xC3\xC4\x94\xE8\x84\x86\x64\x81\xF4\x42\xAA"
		363	"\x3C\x0E\xD6\x4F\xBC\x0A\x09\x2D\xE7\x1B\xD4\x10\xA8\x54\xEA\x89"
		364	"\x84\x8A\xCB\xF7\x5A\x3C\xCA\x76\x08\x29\x62\xB4\x6A\x22\xDF\x14"
		365	"\x95\x71\xFD\xB6\x86\x39\xB8\x8B\xF8\x91\x7F\x38\xAA\x14\xCD\xE5"
		366	"\xF5\x1D\xC2\x6D\x53\x69\x52\x84\x7F\xA3\x1A\x5E\x26\x04\x83\x06"
		367	"\x73\x52\x56\xCF\x76\x26\xC9\xDD\x75\xD7\xFC\xF4\x69\xD8\x7B\x55"
		368	"\xB7\x68\x13\x53\xB9\xE7\x89\xC3\xE8\xD6\x6E\xA7\x6D\xEA\x81\xFD"
		369	"\xC4\xB7\x05\x5A\xB7\x41\x0A\x23\x8E\x03\x8A\x1C\xAE\xD3\x1E\xCE"
		370	"\xE3\x5E\xFC\x19\x4A\xEE\x61\x9B\x8E\xE5\xE5\xDD\x85\xF9\x41\xEC"
		371	"\x14\x53\x92\xF7\xDD\x06\x85\x02\x91\xE3\xEB\x6C\x43\x03\xB1\x36"
		372	"\x7B\x89\x5A\xA8\xEB\xFC\xD5\xA8\x35\xDC\x81\xD9\x5C\xBD\xCA\xDC"
		373	"\x9B\x98\x0B\x06\x5D\x0C\x5B\xEE\xF3\xD5\xCC\x57\xC9\x71\x2F\x90"
		374	"\x3B\x3C\xF0\x8E\x4E\x35\x48\xAE\x63\x74\xA9\xFC\x72\x75\x8E\x34"
		375	"\xA8\xF2\x1F\xEA\xDF\x3A\x37\x2D\xE5\x39\x39\xF8\x57\x58\x3C\x04"
		376	"\xFE\x87\x06\x98\xBC\x7B\xD3\x21\x36\x60\x25\x54\xA7\x3D\xFA\x91"
		377	"\xCC\xA8\x0B\x92\x8E\xB4\xF7\x06\xFF\x1E\x95\xCB\x07\x76\x97\x3B"
		378	"\x9D"
		379	"\x02\x03\x01\x00\x01" /* public key integer of 3 bytes */
		380	"\x02\x82\x02\x00" /* private key integer of 512 bytes */
		381	"\x74\xA9\xE0\x6A\x32\xB4\xCA\x85\xD9\x86\x9F\x60\x88\x7B\x40\xCC"
		382	"\xCD\x33\x91\xA8\xB6\x25\x1F\xBF\xE3\x51\x1C\x97\xB6\x2A\xD9\xB8"
		383	"\x11\x40\x19\xE3\x21\x13\xC8\xB3\x7E\xDC\xD7\x65\x40\x4C\x2D\xD6"
		384	"\xDC\xAF\x32\x6C\x96\x75\x2C\x2C\xCA\x8F\x3F\x7A\xEE\xC4\x09\xC6"
		385	"\x24\x3A\xC9\xCF\x6D\x8D\x17\x50\x94\x52\xD3\xE7\x0F\x2F\x7E\x94"
		386	"\x1F\xA0\xBE\xD9\x25\xE8\x38\x42\x7C\x27\xD2\x79\xF8\x2A\x87\x38"
		387	"\xEF\xBB\x74\x8B\xA8\x6E\x8C\x08\xC6\xC7\x4F\x0C\xBC\x79\xC6\xEF"
		388	"\x0E\xA7\x5E\xE4\xF8\x8C\x09\xC7\x5E\x37\xCC\x87\x77\xCD\xCF\xD1"
		389	"\x6D\x28\x1B\xA9\x62\xC0\xB8\x16\xA7\x8B\xF9\xBB\xCC\xB4\x15\x7F"
		390	"\x1B\x69\x03\xF2\x7B\xEB\xE5\x8C\x14\xD6\x23\x4F\x52\x6F\x18\xA6"
		391	"\x4B\x5B\x01\xAD\x35\xF9\x48\x53\xB3\x86\x35\x66\xD7\xE7\x29\xC0"
		392	"\x09\xB5\xC6\xE6\xFA\xC4\xDA\x19\xBE\xD7\x4D\x41\x14\xBE\x6F\xDF"
		393	"\x1B\xAB\xC0\xCA\x88\x07\xAC\xF1\x7D\x35\x83\x67\x28\x2D\x50\xE9"
		394	"\xCE\x27\x71\x5E\x1C\xCF\xD2\x30\x65\x79\x72\x2F\x9C\xE1\xD2\x39"
		395	"\x7F\xEF\x3B\x01\xF2\x14\x1D\xDF\xBD\x51\xD3\xA1\x53\x62\xCF\x5F"
		396	"\x79\x84\xCE\x06\x96\x69\x29\x49\x82\x1C\x71\x4A\xA1\x66\xC8\x2F"
		397	"\xFD\x7B\x96\x7B\xFC\xC4\x26\x58\xC4\xFC\x7C\xAF\xB5\xE8\x95\x83"
		398	"\x87\xCB\x46\xDE\x97\xA7\xB3\xA2\x54\x5B\xD7\xAF\xAB\xEB\xC8\xF3"
		399	"\x55\x9D\x48\x2B\x30\x9C\xDC\x26\x4B\xC2\x89\x45\x13\xB2\x01\x9A"
		400	"\xA4\x65\xC3\xEC\x24\x2D\x26\x97\xEB\x80\x8A\x9D\x03\xBC\x59\x66"
		401	"\x9E\xE2\xBB\xBB\x63\x19\x64\x93\x11\x7B\x25\x65\x30\xCD\x5B\x4B"
		402	"\x2C\xFF\xDC\x2D\x30\x87\x1F\x3C\x88\x07\xD0\xFC\x48\xCC\x05\x8A"
		403	"\xA2\xC8\x39\x3E\xD5\x51\xBC\x0A\xBE\x6D\xA8\xA0\xF6\x88\x06\x79"
		404	"\x13\xFF\x1B\x45\xDA\x54\xC9\x24\x25\x8A\x75\x0A\x26\xD1\x69\x81"
		405	"\x14\x14\xD1\x79\x7D\x8E\x76\xF2\xE0\xEB\xDD\x0F\xDE\xC2\xEC\x80"
		406	"\xD7\xDC\x16\x99\x92\xBE\xCB\x40\x0C\xCE\x7C\x3B\x46\xA2\x5B\x5D"
		407	"\x0C\x45\xEB\xE1\x00\xDE\x72\x50\xB1\xA6\x0B\x76\xC5\x8D\xFC\x82"
		408	"\x38\x6D\x99\x14\x1D\x1A\x4A\xD3\x7C\x53\xB8\x12\x46\xA2\x30\x38"
		409	"\x82\xF4\x96\x6E\x8C\xCE\x47\x0D\xAF\x0A\x3B\x45\xB7\x43\x95\x43"
		410	"\x9E\x02\x2C\x44\x07\x6D\x1F\x3C\x66\x89\x09\xB6\x1F\x06\x30\xCC"
		411	"\xAD\xCE\x7D\x9A\xDE\x3E\xFB\x6C\xE4\x58\x43\xD2\x4F\xA5\x9E\x5E"
		412	"\xA7\x7B\xAE\x3A\xF6\x7E\xD9\xDB\xD3\xF5\xC5\x41\xAF\xE6\x9C\x91"
		413	"\x02\x82\x01\x01" /* prime1 - integer of 257 bytes */
		414	"\x00\xE0\xA6\x6C\xF0\xA2\xF8\x81\x85\x36\x43\xD0\x13\x0B\x33\x8B"
		415	"\x8F\x78\x3D\xAC\xC7\x5E\x46\x6A\x7F\x05\xAE\x3E\x26\x0A\xA6\xD0"
		416	"\x51\xF3\xC8\x61\xF5\x77\x22\x48\x10\x87\x4C\xD5\xA4\xD5\xAE\x2D"
		417	"\x4E\x7A\xFE\x1C\x31\xE7\x6B\xFF\xA4\x69\x20\xF9\x2A\x0B\x99\xBE"
		418	"\x7C\x32\x68\xAD\xB0\xC6\x94\x81\x41\x75\xDC\x06\x78\x0A\xB4\xCF"
		419	"\xCD\x1B\x2D\x31\xE4\x7B\xEA\xA8\x35\x99\x75\x57\xC6\x0E\xF6\x78"
		420	"\x4F\xA0\x92\x4A\x00\x1B\xE7\x96\xF2\x5B\xFD\x2C\x0A\x0A\x13\x81"
		421	"\xAF\xCB\x59\x87\x31\xD9\x83\x65\xF2\x22\x48\xD0\x03\x67\x39\xF6"
		422	"\xFF\xA8\x36\x07\x3A\x68\xE3\x7B\xA9\x64\xFD\x9C\xF7\xB1\x3D\xBF"
		423	"\x26\x5C\xCC\x7A\xFC\xA2\x8F\x51\xD1\xE1\xE2\x3C\xEC\x06\x75\x7C"
		424	"\x34\xF9\xA9\x33\x70\x11\xAD\x5A\xDC\x5F\xCF\x50\xF6\x23\x2F\x39"
		425	"\xAC\x92\x48\x53\x4D\x01\x96\x3C\xD8\xDC\x1F\x23\x23\x78\x80\x34"
		426	"\x54\x14\x76\x8B\xB6\xBB\xFB\x88\x78\x31\x59\x28\xD2\xB1\x75\x17"
		427	"\x88\x04\x4A\x78\x62\x18\x2E\xF5\xFB\x9B\xEF\x15\xD8\x16\x47\xC6"
		428	"\x42\xB1\x02\xDA\x9E\xE3\x84\x90\xB4\x2D\xC3\xCE\x13\xC9\x12\x7D"
		429	"\x3E\xCD\x39\x39\xC9\xAD\xA1\x1A\xE6\xD5\xAD\x5A\x09\x4D\x1B\x0C"
		430	"\xAB"
		431	"\x02\x82\x01\x01" /* prime 2 - integer of 257 bytes */
		432	"\x00\xDE\xD5\x1B\xF6\xCD\x83\xB1\xC6\x47\x7E\xB9\xC0\x6B\xA9\xB8"
		433	"\x02\xF3\xAE\x40\x5D\xFC\xD3\xE5\x4E\xF1\xE3\x39\x04\x52\x84\x89"
		434	"\x40\x37\xBB\xC2\xCD\x7F\x71\x77\x17\xDF\x6A\x4C\x31\x24\x7F\xB9"
		435	"\x7E\x7F\xC8\x43\x4A\x3C\xEB\x8D\x1B\x7F\x21\x51\x67\x45\x8F\xA0"
		436	"\x36\x29\x3A\x18\x45\xA5\x32\xEC\x74\x88\x3C\x98\x5D\x67\x3B\xD7"
		437	"\x51\x1F\xE9\xAE\x09\x01\xDE\xDE\x7C\xFB\x60\xD1\xA5\x6C\xE9\x6A"
		438	"\x93\x04\x02\x3A\xBB\x67\x02\xB9\xFD\x23\xF0\x02\x2B\x49\x85\xC9"
		439	"\x5B\xE7\x4B\xDF\xA3\xF4\xEE\x59\x4C\x45\xEF\x8B\xC1\x6B\xDE\xDE"
		440	"\xBC\x1A\xFC\xD2\x76\x3F\x33\x74\xA9\x8E\xA3\x7E\x0C\xC6\xCE\x70"
		441	"\xA1\x5B\xA6\x77\xEA\x76\xEB\x18\xCE\xB9\xD7\x78\x8D\xAE\x06\xBB"
		442	"\xD3\x1F\x16\x0D\x05\xAB\x4F\xC6\x52\xC8\x6B\x36\x51\x7D\x1D\x27"
		443	"\xAF\x88\x9A\x6F\xCC\x25\x2E\x74\x06\x72\xCE\x9E\xDB\xE0\x9D\x30"
		444	"\xEF\x55\xA5\x58\x21\xA7\x42\x12\x2C\x2C\x23\x87\xC1\x0F\xE8\x51"
		445	"\xDA\x53\xDA\xFC\x05\x36\xDF\x08\x0E\x08\x36\xBE\x5C\x86\x9E\xCA"
		446	"\x68\x90\x33\x12\x0B\x14\x82\xAB\x90\x1A\xD4\x49\x32\x9C\xBD\xAA"
		447	"\xAB\x4E\x38\xF1\xEE\xED\x3D\x3F\xE8\xBD\x48\x56\xA6\x64\xEE\xC8"
		448	"\xD7"
		449	"\x02\x82\x01\x01" /* exponent 1 - integer of 257 bytes */
		450	"\x00\x96\x5E\x6F\x8F\x06\xD6\xE6\x03\x1F\x96\x76\x81\x38\xBF\x30"
		451	"\xCC\x40\x84\xAF\xD0\xE7\x06\xA5\x24\x0E\xCE\x59\xA5\x26\xFE\x0F"
		452	"\x74\xBB\x83\xC6\x26\x02\xAF\x3C\xA3\x6B\x9C\xFF\x68\x0C\xEB\x40"
		453	"\x42\x46\xCB\x2E\x5E\x2C\xF4\x3A\x32\x77\x77\xED\xAF\xBA\x02\x17"
		454	"\xE1\x93\xF0\x43\x4A\x8F\x31\x39\xEF\x72\x0F\x6B\x79\x10\x59\x84"
		455	"\xBA\x5A\x55\x7F\x0E\xDB\xEE\xEE\xD6\xA9\xB8\x44\x9F\x3A\xC6\xB9"
		456	"\x33\x3B\x5C\x90\x11\xD0\x9B\xCC\x8A\xBF\x0E\x10\x5B\x4B\xF1\x50"
		457	"\x9E\x35\xB3\xE0\x6D\x7A\x95\x9C\x38\x5D\xC0\x75\x13\xC2\x15\xA7"
		458	"\x81\xEA\xBA\xF7\x4D\x9E\x85\x9D\xF1\x7D\xBA\xD0\x45\x6F\x2A\xD0"
		459	"\x76\xC2\x28\xD0\xAD\xA7\xB5\xDC\xE3\x6A\x99\xFF\x83\x50\xB3\x75"
		460	"\x07\x14\x91\xAF\xEF\x74\xB5\x9F\x9A\xE0\xBA\xA9\x0B\x87\xF3\x85"
		461	"\x5C\x40\xB2\x0E\xA7\xFD\xC6\xED\x45\x8E\xD9\x7C\xB0\xB2\x68\xC6"
		462	"\x1D\xFD\x70\x78\x06\x41\x7F\x95\x12\x36\x9D\xE2\x58\x5D\x15\xEE"
		463	"\x41\x49\xF5\xFA\xEC\x56\x19\xA0\xE6\xE0\xB2\x40\xE1\xD9\xD0\x03"
		464	"\x22\x02\xCF\xD1\x3C\x07\x38\x65\x8F\x65\x0E\xAA\x32\xCE\x25\x05"
		465	"\x16\x73\x51\xB9\x9F\x88\x0B\xCD\x30\xF3\x97\xCC\x2B\x6B\xA4\x0E"
		466	"\x6F"
		467	"\x02\x82\x01\x00" /* exponent 2 - integer of 256 bytes */
		468	"\x2A\x5F\x3F\xB8\x08\x90\x58\x47\xA9\xE4\xB1\x11\xA3\xE7\x5B\xF4"
		469	"\x43\xBE\x08\xC3\x56\x86\x3C\x7E\x6C\x84\x96\x9C\xF9\xCB\xF6\x05"
		470	"\x5E\x13\xB8\x11\x37\x80\xAD\xF2\xBE\x2B\x0A\x5D\xF5\xE0\xCB\xB7"
		471	"\x00\x39\x66\x82\x41\x5F\x51\x2F\xBF\x56\xE8\x91\xC8\xAA\x6C\xFE"
		472	"\x9F\x8C\x4A\x7D\x43\xD2\x91\x1F\xFF\x9F\xF6\x21\x1C\xB6\x46\x55"
		473	"\x48\xCA\x38\xAB\xC1\xCD\x4D\x65\x5A\xAF\xA8\x6D\xDA\x6D\xF0\x34"
		474	"\x10\x79\x14\x0D\xFA\xA2\x8C\x17\x54\xB4\x18\xD5\x7E\x5F\x90\x50"
		475	"\x87\x84\xE7\xFB\xD7\x61\x53\x5D\xAB\x96\xC7\x6E\x7A\x42\xA0\xFC"
		476	"\x07\xED\xB7\x5F\x80\xD9\x19\xFF\xFB\xFD\x9E\xC4\x73\x31\x62\x3D"
		477	"\x6C\x9E\x15\x03\x62\xA5\x85\xCC\x19\x8E\x9D\x7F\xE3\x6D\xA8\x5D"
		478	"\x96\xF5\xAC\x78\x3D\x81\x27\xE7\x29\xF1\x29\x1D\x09\xBB\x77\x86"
		479	"\x6B\x65\x62\x88\xE1\x31\x1A\x22\xF7\xC5\xCE\x73\x65\x1C\xBE\xE7"
		480	"\x63\xD3\xD3\x14\x63\x27\xAF\x28\xF3\x23\xB6\x76\xC1\xBD\x9D\x82"
		481	"\xF4\x9B\x19\x7D\x2C\x57\xF0\xC2\x2A\x51\xAE\x95\x0D\x8C\x38\x54"
		482	"\xF5\xC6\xA0\x51\xB7\x0E\xB9\xEC\xE7\x0D\x22\xF6\x1A\xD3\xFE\x16"
		483	"\x21\x03\xB7\x0D\x85\xD3\x35\xC9\xDD\xE4\x59\x85\xBE\x7F\xA1\x75"
		484	"\x02\x82\x01\x01" /* coefficient - integer of 257 bytes */
		485	"\x00\xB9\x48\xD2\x54\x2F\x19\x54\x64\xAE\x62\x80\x61\x89\x80\xB4"
		486	"\x48\x0B\x8D\x7E\x1B\x0F\x50\x08\x82\x3F\xED\x75\x84\xB7\x13\xE4"
		487	"\xF8\x8D\xA8\xBB\x54\x21\x4C\x5A\x54\x07\x16\x4B\xB4\xA4\x9E\x30"
		488	"\xBF\x7A\x30\x1B\x39\x60\xA3\x21\x53\xFB\xB0\xDC\x0F\x7C\x2C\xFB"
		489	"\xAA\x95\x7D\x51\x39\x28\x33\x1F\x25\x31\x53\xF5\xD2\x64\x2B\xF2"
		490	"\x1E\xB3\xC0\x6A\x0B\xC9\xA4\x42\x64\x5C\xFB\x15\xA3\xE8\x4C\x3A"
		491	"\x9C\x3C\xBE\xA3\x39\x83\x23\xE3\x6D\x18\xCC\xC2\xDC\x63\x8D\xBA"
		492	"\x98\xE0\xE0\x31\x4A\x2B\x37\x9C\x4D\x6B\xF3\x9F\x51\xE4\x43\x5C"
		493	"\x83\x5F\xBF\x5C\xFE\x92\x45\x01\xAF\xF5\xC2\xF4\xB7\x56\x93\xA5"
		494	"\xF4\xAA\x67\x3C\x48\x37\xBD\x9A\x3C\xFE\xA5\x9A\xB0\xD1\x6B\x85"
		495	"\xDD\x81\xD4\xFA\xAD\x31\x83\xA8\x22\x9B\xFD\xB4\x61\xDC\x7A\x51"
		496	"\x59\x62\x10\x1B\x7E\x44\xA3\xFE\x90\x51\x5A\x3E\x02\x87\xAD\xFA"
		497	"\xDD\x0B\x1F\x3D\x35\xAF\xEE\x13\x85\x51\xA7\x42\xC0\xEE\x9E\x20"
		498	"\xE9\xD0\x29\xB2\xE4\x21\xE4\x6D\x62\xB9\xF4\x48\x4A\xD8\x46\x8E"
		499	"\x61\xA6\x2C\x5D\xDF\x8F\x97\x2B\x3A\x75\x1D\x83\x17\x6F\xC6\xB0"
		500	"\xDE\xFC\x14\x25\x06\x5A\x60\xBB\xB8\x21\x89\xD1\xEF\x57\xF1\x71"
		501	"\x3D",
		502	.m = "\x54\x85\x9b\x34\x2c\x49\xea\x2a",
		503	.c =
		504	"\x5c\xce\x9c\xd7\x9a\x9e\xa1\xfe\x7a\x82\x3c\x68\x27\x98\xe3\x5d"
		505	"\xd5\xd7\x07\x29\xf5\xfb\xc3\x1a\x7f\x63\x1e\x62\x31\x3b\x19\x87"
		506	"\x79\x4f\xec\x7b\xf3\xcb\xea\x9b\x95\x52\x3a\x40\xe5\x87\x7b\x72"
		507	"\xd1\x72\xc9\xfb\x54\x63\xd8\xc9\xd7\x2c\xfc\x7b\xc3\x14\x1e\xbc"
		508	"\x18\xb4\x34\xa1\xbf\x14\xb1\x37\x31\x6e\xf0\x1b\x35\x19\x54\x07"
		509	"\xf7\x99\xec\x3e\x63\xe2\xcd\x61\x28\x65\xc3\xcd\xb1\x38\x36\xa5"
		510	"\xb2\xd7\xb0\xdc\x1f\xf5\xef\x19\xc7\x53\x32\x2d\x1c\x26\xda\xe4"
		511	"\x0d\xd6\x90\x7e\x28\xd8\xdc\xe4\x61\x05\xd2\x25\x90\x01\xd3\x96"
		512	"\x6d\xa6\xcf\x58\x20\xbb\x03\xf4\x01\xbc\x79\xb9\x18\xd8\xb8\xba"
		513	"\xbd\x93\xfc\xf2\x62\x5d\x8c\x66\x1e\x0e\x84\x59\x93\xdd\xe2\x93"
		514	"\xa2\x62\x7d\x08\x82\x7a\xdd\xfc\xb8\xbc\xc5\x4f\x9c\x4e\xbf\xb4"
		515	"\xfc\xf4\xc5\x01\xe8\x00\x70\x4d\x28\x26\xcc\x2e\xfe\x0e\x58\x41"
		516	"\x8b\xec\xaf\x7c\x4b\x54\xd0\xa0\x64\xf9\x32\xf4\x2e\x47\x65\x0a"
		517	"\x67\x88\x39\x3a\xdb\xb2\xdb\x7b\xb5\xf6\x17\xa8\xd9\xc6\x5e\x28"
		518	"\x13\x82\x8a\x99\xdb\x60\x08\xa5\x23\x37\xfa\x88\x90\x31\xc8\x9d"
		519	"\x8f\xec\xfb\x85\x9f\xb1\xce\xa6\x24\x50\x46\x44\x47\xcb\x65\xd1"
		520	"\xdf\xc0\xb1\x6c\x90\x1f\x99\x8e\x4d\xd5\x9e\x31\x07\x66\x87\xdf"
		521	"\x01\xaa\x56\x3c\x71\xe0\x2b\x6f\x67\x3b\x23\xed\xc2\xbd\x03\x30"
		522	"\x79\x76\x02\x10\x10\x98\x85\x8a\xff\xfd\x0b\xda\xa5\xd9\x32\x48"
		523	"\x02\xa0\x0b\xb9\x2a\x8a\x18\xca\xc6\x8f\x3f\xbb\x16\xb2\xaa\x98"
		524	"\x27\xe3\x60\x43\xed\x15\x70\xd4\x57\x15\xfe\x19\xd4\x9b\x13\x78"
		525	"\x8a\xf7\x21\xf1\xa2\xa2\x2d\xb3\x09\xcf\x44\x91\x6e\x08\x3a\x30"
		526	"\x81\x3e\x90\x93\x8a\x67\x33\x00\x59\x54\x9a\x25\xd3\x49\x8e\x9f"
		527	"\xc1\x4b\xe5\x86\xf3\x50\x4c\xbc\xc5\xd3\xf5\x3a\x54\xe1\x36\x3f"
		528	"\xe2\x5a\xb4\x37\xc0\xeb\x70\x35\xec\xf6\xb7\xe8\x44\x3b\x7b\xf3"
		529	"\xf1\xf2\x1e\xdb\x60\x7d\xd5\xbe\xf0\x71\x34\x90\x4c\xcb\xd4\x35"
		530	"\x51\xc7\xdd\xd8\xc9\x81\xf5\x5d\x57\x46\x2c\xb1\x7b\x9b\xaa\xcb"
		531	"\xd1\x22\x25\x49\x44\xa3\xd4\x6b\x29\x7b\xd8\xb2\x07\x93\xbf\x3d"
		532	"\x52\x49\x84\x79\xef\xb8\xe5\xc4\xad\xca\xa8\xc6\xf6\xa6\x76\x70"
		533	"\x5b\x0b\xe5\x83\xc6\x0e\xef\x55\xf2\xe7\xff\x04\xea\xe6\x13\xbe"
		534	"\x40\xe1\x40\x45\x48\x66\x75\x31\xae\x35\x64\x91\x11\x6f\xda\xee"
		535	"\x26\x86\x45\x6f\x0b\xd5\x9f\x03\xb1\x65\x5b\xdb\xa4\xe4\xf9\x45",
		536	.key_len = 2349,
		537	.m_size = 8,
		538	.c_size = 512,
		539	}
		540	};
		541
		542	#define DH_TEST_VECTORS 2
		543
		544	struct kpp_testvec dh_tv_template[] = {
		545	{
		546	.secret =
		547	#ifdef __LITTLE_ENDIAN
		548	"\x01\x00" /* type */
		549	"\x11\x02" /* len */
		550	"\x00\x01\x00\x00" /* key_size */
		551	"\x00\x01\x00\x00" /* p_size */
		552	"\x01\x00\x00\x00" /* g_size */
		553	#else
		554	"\x00\x01" /* type */
		555	"\x02\x11" /* len */
		556	"\x00\x00\x01\x00" /* key_size */
		557	"\x00\x00\x01\x00" /* p_size */
		558	"\x00\x00\x00\x01" /* g_size */
		559	#endif
		560	/* xa */
		561	"\x44\xc1\x48\x36\xa7\x2b\x6f\x4e\x43\x03\x68\xad\x31\x00\xda\xf3"
		562	"\x2a\x01\xa8\x32\x63\x5f\x89\x32\x1f\xdf\x4c\xa1\x6a\xbc\x10\x15"
		563	"\x90\x35\xc9\x26\x41\xdf\x7b\xaa\x56\x56\x3d\x85\x44\xb5\xc0\x8e"
		564	"\x37\x83\x06\x50\xb3\x5f\x0e\x28\x2c\xd5\x46\x15\xe3\xda\x7d\x74"
		565	"\x87\x13\x91\x4f\xd4\x2d\xf6\xc7\x5e\x14\x2c\x11\xc2\x26\xb4\x3a"
		566	"\xe3\xb2\x36\x20\x11\x3b\x22\xf2\x06\x65\x66\xe2\x57\x58\xf8\x22"
		567	"\x1a\x94\xbd\x2b\x0e\x8c\x55\xad\x61\x23\x45\x2b\x19\x1e\x63\x3a"
		568	"\x13\x61\xe3\xa0\x79\x70\x3e\x6d\x98\x32\xbc\x7f\x82\xc3\x11\xd8"
		569	"\xeb\x53\xb5\xfc\xb5\xd5\x3c\x4a\xea\x92\x3e\x01\xce\x15\x65\xd4"
		570	"\xaa\x85\xc1\x11\x90\x83\x31\x6e\xfe\xe7\x7f\x7d\xed\xab\xf9\x29"
		571	"\xf8\xc7\xf1\x68\xc6\xb7\xe4\x1f\x2f\x28\xa0\xc9\x1a\x50\x64\x29"
		572	"\x4b\x01\x6d\x1a\xda\x46\x63\x21\x07\x40\x8c\x8e\x4c\x6f\xb5\xe5"
		573	"\x12\xf3\xc2\x1b\x48\x27\x5e\x27\x01\xb1\xaa\xed\x68\x9b\x83\x18"
		574	"\x8f\xb1\xeb\x1f\x04\xd1\x3c\x79\xed\x4b\xf7\x0a\x33\xdc\xe0\xc6"
		575	"\xd8\x02\x51\x59\x00\x74\x30\x07\x4c\x2d\xac\xe4\x13\xf1\x80\xf0"
		576	"\xce\xfa\xff\xa9\xce\x29\x46\xdd\x9d\xad\xd1\xc3\xc6\x58\x1a\x63"
		577	/* p */
		578	"\xb9\x36\x3a\xf1\x82\x1f\x60\xd3\x22\x47\xb8\xbc\x2d\x22\x6b\x81"
		579	"\x7f\xe8\x20\x06\x09\x23\x73\x49\x9a\x59\x8b\x35\x25\xf8\x31\xbc"
		580	"\x7d\xa8\x1c\x9d\x56\x0d\x1a\xf7\x4b\x4f\x96\xa4\x35\x77\x6a\x89"
		581	"\xab\x42\x00\x49\x21\x71\xed\x28\x16\x1d\x87\x5a\x10\xa7\x9c\x64"
		582	"\x94\xd4\x87\x3d\x28\xef\x44\xfe\x4b\xe2\xb4\x15\x8c\x82\xa6\xf3"
		583	"\x50\x5f\xa8\xe8\xa2\x60\xe7\x00\x86\x78\x05\xd4\x78\x19\xa1\x98"
		584	"\x62\x4e\x4a\x00\x78\x56\x96\xe6\xcf\xd7\x10\x1b\x74\x5d\xd0\x26"
		585	"\x61\xdb\x6b\x32\x09\x51\xd8\xa5\xfd\x54\x16\x71\x01\xb3\x39\xe6"
		586	"\x4e\x69\xb1\xd7\x06\x8f\xd6\x1e\xdc\x72\x25\x26\x74\xc8\x41\x06"
		587	"\x5c\xd1\x26\x5c\xb0\x2f\xf9\x59\x13\xc1\x2a\x0f\x78\xea\x7b\xf7"
		588	"\xbd\x59\xa0\x90\x1d\xfc\x33\x5b\x4c\xbf\x05\x9c\x3a\x3f\x69\xa2"
		589	"\x45\x61\x4e\x10\x6a\xb3\x17\xc5\x68\x30\xfb\x07\x5f\x34\xc6\xfb"
		590	"\x73\x07\x3c\x70\xf6\xae\xe7\x72\x84\xc3\x18\x81\x8f\xe8\x11\x1f"
		591	"\x3d\x83\x83\x01\x2a\x14\x73\xbf\x32\x32\x2e\xc9\x4d\xdb\x2a\xca"
		592	"\xee\x71\xf9\xda\xad\xe8\x82\x0b\x4d\x0c\x1f\xb6\x1d\xef\x00\x67"
		593	"\x74\x3d\x95\xe0\xb7\xc4\x30\x8a\x24\x87\x12\x47\x27\x70\x0d\x73"
		594	/* g */
		595	"\x02",
		596	.b_public =
		597	"\x2a\x67\x5c\xfd\x63\x5d\xc0\x97\x0a\x8b\xa2\x1f\xf8\x8a\xcb\x54"
		598	"\xca\x2f\xd3\x49\x3f\x01\x8e\x87\xfe\xcc\x94\xa0\x3e\xd4\x26\x79"
		599	"\x9a\x94\x3c\x11\x81\x58\x5c\x60\x3d\xf5\x98\x90\x89\x64\x62\x1f"
		600	"\xbd\x05\x6d\x2b\xcd\x84\x40\x9b\x4a\x1f\xe0\x19\xf1\xca\x20\xb3"
		601	"\x4e\xa0\x4f\x15\xcc\xa5\xfe\xa5\xb4\xf5\x0b\x18\x7a\x5a\x37\xaa"
		602	"\x58\x00\x19\x7f\xe2\xa3\xd9\x1c\x44\x57\xcc\xde\x2e\xc1\x38\xea"
		603	"\xeb\xe3\x90\x40\xc4\x6c\xf7\xcd\xe9\x22\x50\x71\xf5\x7c\xdb\x37"
		604	"\x0e\x80\xc3\xed\x7e\xb1\x2b\x2f\xbe\x71\xa6\x11\xa5\x9d\xf5\x39"
		605	"\xf1\xa2\xe5\x85\xbc\x25\x91\x4e\x84\x8d\x26\x9f\x4f\xe6\x0f\xa6"
		606	"\x2b\x6b\xf9\x0d\xaf\x6f\xbb\xfa\x2d\x79\x15\x31\x57\xae\x19\x60"
		607	"\x22\x0a\xf5\xfd\x98\x0e\xbf\x5d\x49\x75\x58\x37\xbc\x7f\xf5\x21"
		608	"\x56\x1e\xd5\xb3\x50\x0b\xca\x96\xf3\xd1\x3f\xb3\x70\xa8\x6d\x63"
		609	"\x48\xfb\x3d\xd7\x29\x91\x45\xb5\x48\xcd\xb6\x78\x30\xf2\x3f\x1e"
		610	"\xd6\x22\xd6\x35\x9b\xf9\x1f\x85\xae\xab\x4b\xd7\xe0\xc7\x86\x67"
		611	"\x3f\x05\x7f\xa6\x0d\x2f\x0d\xbf\x53\x5f\x4d\x2c\x6d\x5e\x57\x40"
		612	"\x30\x3a\x23\x98\xf9\xb4\x32\xf5\x32\x83\xdd\x0b\xae\x33\x97\x2f",
		613	.expected_a_public =
		614	"\x5c\x24\xdf\xeb\x5b\x4b\xf8\xc5\xef\x39\x48\x82\xe0\x1e\x62\xee"
		615	"\x8a\xae\xdf\x93\x6c\x2b\x16\x95\x92\x16\x3f\x16\x7b\x75\x03\x85"
		616	"\xd9\xf1\x69\xc2\x14\x87\x45\xfc\xa4\x19\xf6\xf0\xa4\xf3\xec\xd4"
		617	"\x6c\x5c\x03\x3b\x94\xc2\x2f\x92\xe4\xce\xb3\xe4\x72\xe8\x17\xe6"
		618	"\x23\x7e\x00\x01\x09\x59\x13\xbf\xc1\x2f\x99\xa9\x07\xaa\x02\x23"
		619	"\x4a\xca\x39\x4f\xbc\xec\x0f\x27\x4f\x19\x93\x6c\xb9\x30\x52\xfd"
		620	"\x2b\x9d\x86\xf1\x06\x1e\xb6\x56\x27\x4a\xc9\x8a\xa7\x8a\x48\x5e"
		621	"\xb5\x60\xcb\xdf\xff\x03\x26\x10\xbf\x90\x8f\x46\x60\xeb\x9b\x9a"
		622	"\xd6\x6f\x44\x91\x03\x92\x18\x2c\x96\x5e\x40\x19\xfb\xf4\x4f\x3a"
		623	"\x02\x7b\xaf\xcc\x22\x20\x79\xb9\xf8\x9f\x8f\x85\x6b\xec\x44\xbb"
		624	"\xe6\xa8\x8e\xb1\xe8\x2c\xee\x64\xee\xf8\xbd\x00\xf3\xe2\x2b\x93"
		625	"\xcd\xe7\xc4\xdf\xc9\x19\x46\xfe\xb6\x07\x73\xc1\x8a\x64\x79\x26"
		626	"\xe7\x30\xad\x2a\xdf\xe6\x8f\x59\xf5\x81\xbf\x4a\x29\x91\xe7\xb7"
		627	"\xcf\x48\x13\x27\x75\x79\x40\xd9\xd6\x32\x52\x4e\x6a\x86\xae\x6f"
		628	"\xc2\xbf\xec\x1f\xc2\x69\xb2\xb6\x59\xe5\xa5\x17\xa4\x77\xb7\x62"
		629	"\x46\xde\xe8\xd2\x89\x78\x9a\xef\xa3\xb5\x8f\x26\xec\x80\xda\x39",
		630	.expected_ss =
		631	"\x8f\xf3\xac\xa2\xea\x22\x11\x5c\x45\x65\x1a\x77\x75\x2e\xcf\x46"
		632	"\x23\x14\x1e\x67\x53\x4d\x35\xb0\x38\x1d\x4e\xb9\x41\x9a\x21\x24"
		633	"\x6e\x9f\x40\xfe\x90\x51\xb1\x06\xa4\x7b\x87\x17\x2f\xe7\x5e\x22"
		634	"\xf0\x7b\x54\x84\x0a\xac\x0a\x90\xd2\xd7\xe8\x7f\xe7\xe3\x30\x75"
		635	"\x01\x1f\x24\x75\x56\xbe\xcc\x8d\x1e\x68\x0c\x41\x72\xd3\xfa\xbb"
		636	"\xe5\x9c\x60\xc7\x28\x77\x0c\xbe\x89\xab\x08\xd6\x21\xe7\x2e\x1a"
		637	"\x58\x7a\xca\x4f\x22\xf3\x2b\x30\xfd\xf4\x98\xc1\xa3\xf8\xf6\xcc"
		638	"\xa9\xe4\xdb\x5b\xee\xd5\x5c\x6f\x62\x4c\xd1\x1a\x02\x2a\x23\xe4"
		639	"\xb5\x57\xf3\xf9\xec\x04\x83\x54\xfe\x08\x5e\x35\xac\xfb\xa8\x09"
		640	"\x82\x32\x60\x11\xb2\x16\x62\x6b\xdf\xda\xde\x9c\xcb\x63\x44\x6c"
		641	"\x59\x26\x6a\x8f\xb0\x24\xcb\xa6\x72\x48\x1e\xeb\xe0\xe1\x09\x44"
		642	"\xdd\xee\x66\x6d\x84\xcf\xa5\xc1\xb8\x36\x74\xd3\x15\x96\xc3\xe4"
		643	"\xc6\x5a\x4d\x23\x97\x0c\x5c\xcb\xa9\xf5\x29\xc2\x0e\xff\x93\x82"
		644	"\xd3\x34\x49\xad\x64\xa6\xb1\xc0\x59\x28\x75\x60\xa7\x8a\xb0\x11"
		645	"\x56\x89\x42\x74\x11\xf5\xf6\x5e\x6f\x16\x54\x6a\xb1\x76\x4d\x50"
		646	"\x8a\x68\xc1\x5b\x82\xb9\x0d\x00\x32\x50\xed\x88\x87\x48\x92\x17",
		647	.secret_size = 529,
		648	.b_public_size = 256,
		649	.expected_a_public_size = 256,
		650	.expected_ss_size = 256,
		651	},
		652	{
		653	.secret =
		654	#ifdef __LITTLE_ENDIAN
		655	"\x01\x00" /* type */
		656	"\x11\x02" /* len */
		657	"\x00\x01\x00\x00" /* key_size */
		658	"\x00\x01\x00\x00" /* p_size */
		659	"\x01\x00\x00\x00" /* g_size */
		660	#else
		661	"\x00\x01" /* type */
		662	"\x02\x11" /* len */
		663	"\x00\x00\x01\x00" /* key_size */
		664	"\x00\x00\x01\x00" /* p_size */
		665	"\x00\x00\x00\x01" /* g_size */
		666	#endif
		667	/* xa */
		668	"\x4d\x75\xa8\x6e\xba\x23\x3a\x0c\x63\x56\xc8\xc9\x5a\xa7\xd6\x0e"
		669	"\xed\xae\x40\x78\x87\x47\x5f\xe0\xa7\x7b\xba\x84\x88\x67\x4e\xe5"
		670	"\x3c\xcc\x5c\x6a\xe7\x4a\x20\xec\xbe\xcb\xf5\x52\x62\x9f\x37\x80"
		671	"\x0c\x72\x7b\x83\x66\xa4\xf6\x7f\x95\x97\x1c\x6a\x5c\x7e\xf1\x67"
		672	"\x37\xb3\x93\x39\x3d\x0b\x55\x35\xd9\xe5\x22\x04\x9f\xf8\xc1\x04"
		673	"\xce\x13\xa5\xac\xe1\x75\x05\xd1\x2b\x53\xa2\x84\xef\xb1\x18\xf4"
		674	"\x66\xdd\xea\xe6\x24\x69\x5a\x49\xe0\x7a\xd8\xdf\x1b\xb7\xf1\x6d"
		675	"\x9b\x50\x2c\xc8\x1c\x1c\xa3\xb4\x37\xfb\x66\x3f\x67\x71\x73\xa9"
		676	"\xff\x5f\xd9\xa2\x25\x6e\x25\x1b\x26\x54\xbf\x0c\xc6\xdb\xea\x0a"
		677	"\x52\x6c\x16\x7c\x27\x68\x15\x71\x58\x73\x9d\xe6\xc2\x80\xaa\x97"
		678	"\x31\x66\xfb\xa6\xfb\xfd\xd0\x9c\x1d\xbe\x81\x48\xf5\x9a\x32\xf1"
		679	"\x69\x62\x18\x78\xae\x72\x36\xe6\x94\x27\xd1\xff\x18\x4f\x28\x6a"
		680	"\x16\xbd\x6a\x60\xee\xe5\xf9\x6d\x16\xe4\xb8\xa6\x41\x9b\x23\x7e"
		681	"\xf7\x9d\xd1\x1d\x03\x15\x66\x3a\xcf\xb6\x2c\x13\x96\x2c\x52\x21"
		682	"\xe4\x2d\x48\x7a\x8a\x5d\xb2\x88\xed\x98\x61\x79\x8b\x6a\x1e\x5f"
		683	"\xd0\x8a\x2d\x99\x5a\x2b\x0f\xbc\xef\x53\x8f\x32\xc1\xa2\x99\x26"
		684	/* p */
		685	"\xb9\x36\x3a\xf1\x82\x1f\x60\xd3\x22\x47\xb8\xbc\x2d\x22\x6b\x81"
		686	"\x7f\xe8\x20\x06\x09\x23\x73\x49\x9a\x59\x8b\x35\x25\xf8\x31\xbc"
		687	"\x7d\xa8\x1c\x9d\x56\x0d\x1a\xf7\x4b\x4f\x96\xa4\x35\x77\x6a\x89"
		688	"\xab\x42\x00\x49\x21\x71\xed\x28\x16\x1d\x87\x5a\x10\xa7\x9c\x64"
		689	"\x94\xd4\x87\x3d\x28\xef\x44\xfe\x4b\xe2\xb4\x15\x8c\x82\xa6\xf3"
		690	"\x50\x5f\xa8\xe8\xa2\x60\xe7\x00\x86\x78\x05\xd4\x78\x19\xa1\x98"
		691	"\x62\x4e\x4a\x00\x78\x56\x96\xe6\xcf\xd7\x10\x1b\x74\x5d\xd0\x26"
		692	"\x61\xdb\x6b\x32\x09\x51\xd8\xa5\xfd\x54\x16\x71\x01\xb3\x39\xe6"
		693	"\x4e\x69\xb1\xd7\x06\x8f\xd6\x1e\xdc\x72\x25\x26\x74\xc8\x41\x06"
		694	"\x5c\xd1\x26\x5c\xb0\x2f\xf9\x59\x13\xc1\x2a\x0f\x78\xea\x7b\xf7"
		695	"\xbd\x59\xa0\x90\x1d\xfc\x33\x5b\x4c\xbf\x05\x9c\x3a\x3f\x69\xa2"
		696	"\x45\x61\x4e\x10\x6a\xb3\x17\xc5\x68\x30\xfb\x07\x5f\x34\xc6\xfb"
		697	"\x73\x07\x3c\x70\xf6\xae\xe7\x72\x84\xc3\x18\x81\x8f\xe8\x11\x1f"
		698	"\x3d\x83\x83\x01\x2a\x14\x73\xbf\x32\x32\x2e\xc9\x4d\xdb\x2a\xca"
		699	"\xee\x71\xf9\xda\xad\xe8\x82\x0b\x4d\x0c\x1f\xb6\x1d\xef\x00\x67"
		700	"\x74\x3d\x95\xe0\xb7\xc4\x30\x8a\x24\x87\x12\x47\x27\x70\x0d\x73"
		701	/* g */
		702	"\x02",
		703	.b_public =
		704	"\x99\x4d\xd9\x01\x84\x8e\x4a\x5b\xb8\xa5\x64\x8c\x6c\x00\x5c\x0e"
		705	"\x1e\x1b\xee\x5d\x9f\x53\xe3\x16\x70\x01\xed\xbf\x4f\x14\x36\x6e"
		706	"\xe4\x43\x45\x43\x49\xcc\xb1\xb0\x2a\xc0\x6f\x22\x55\x42\x17\x94"
		707	"\x18\x83\xd7\x2a\x5c\x51\x54\xf8\x4e\x7c\x10\xda\x76\x68\x57\x77"
		708	"\x1e\x62\x03\x30\x04\x7b\x4c\x39\x9c\x54\x01\x54\xec\xef\xb3\x55"
		709	"\xa4\xc0\x24\x6d\x3d\xbd\xcc\x46\x5b\x00\x96\xc7\xea\x93\xd1\x3f"
		710	"\xf2\x6a\x72\xe3\xf2\xc1\x92\x24\x5b\xda\x48\x70\x2c\xa9\x59\x97"
		711	"\x19\xb1\xd6\x54\xb3\x9c\x2e\xb0\x63\x07\x9b\x5e\xac\xb5\xf2\xb1"
		712	"\x5b\xf8\xf3\xd7\x2d\x37\x9b\x68\x6c\xf8\x90\x07\xbc\x37\x9a\xa5"
		713	"\xe2\x91\x12\x25\x47\x77\xe3\x3d\xb2\x95\x69\x44\x0b\x91\x1e\xaf"
		714	"\x7c\x8c\x7c\x34\x41\x6a\xab\x60\x6e\xc6\x52\xec\x7e\x94\x0a\x37"
		715	"\xec\x98\x90\xdf\x3f\x02\xbd\x23\x52\xdd\xd9\xe5\x31\x80\x74\x25"
		716	"\xb6\xd2\xd3\xcc\xd5\xcc\x6d\xf9\x7e\x4d\x78\xab\x77\x51\xfa\x77"
		717	"\x19\x94\x49\x8c\x05\xd4\x75\xed\xd2\xb3\x64\x57\xe0\x52\x99\xc0"
		718	"\x83\xe3\xbb\x5e\x2b\xf1\xd2\xc0\xb1\x37\x36\x0b\x7c\xb5\x63\x96"
		719	"\x8e\xde\x04\x23\x11\x95\x62\x11\x9a\xce\x6f\x63\xc8\xd5\xd1\x8f",
		720	.expected_a_public =
		721	"\x90\x89\xe4\x82\xd6\x0a\xcf\x1a\xae\xce\x1b\x66\xa7\x19\x71\x18"
		722	"\x8f\x95\x4b\x5b\x80\x45\x4a\x5a\x43\x99\x4d\x37\xcf\xa3\xa7\x28"
		723	"\x9c\xc7\x73\xf1\xb2\x17\xf6\x99\xe3\x6b\x56\xcb\x3e\x35\x60\x7d"
		724	"\x65\xc7\x84\x6b\x3e\x60\xee\xcd\xd2\x70\xe7\xc9\x32\x1c\xf0\xb4"
		725	"\xf9\x52\xd9\x88\x75\xfd\x40\x2c\xa7\xbe\x19\x1c\x0a\xae\x93\xe1"
		726	"\x71\xc7\xcd\x4f\x33\x5c\x10\x7d\x39\x56\xfc\x73\x84\xb2\x67\xc3"
		727	"\x77\x26\x20\x97\x2b\xf8\x13\x43\x93\x9c\x9a\xa4\x08\xc7\x34\x83"
		728	"\xe6\x98\x61\xe7\x16\x30\x2c\xb1\xdb\x2a\xb2\xcc\xc3\x02\xa5\x3c"
		729	"\x71\x50\x14\x83\xc7\xbb\xa4\xbe\x98\x1b\xfe\xcb\x43\xe9\x97\x62"
		730	"\xd6\xf0\x8c\xcb\x1c\xba\x1e\xa8\xa6\xa6\x50\xfc\x85\x7d\x47\xbf"
		731	"\xf4\x3e\x23\xd3\x5f\xb2\x71\x3e\x40\x94\xaa\x87\x83\x2c\x6c\x8e"
		732	"\x60\xfd\xdd\xf7\xf4\x76\x03\xd3\x1d\xec\x18\x51\xa3\xf2\x44\x1a"
		733	"\x3f\xb4\x7c\x18\x0d\x68\x65\x92\x54\x0d\x2d\x81\x16\xf1\x84\x66"
		734	"\x89\x92\xd0\x1a\x5e\x1f\x42\x46\x5b\xe5\x83\x86\x80\xd9\xcd\x3a"
		735	"\x5a\x2f\xb9\x59\x9b\xe4\x43\x84\x64\xf3\x09\x1a\x0a\xa2\x64\x0f"
		736	"\x77\x4e\x8d\x8b\xe6\x88\xd1\xfc\xaf\x8f\xdf\x1d\xbc\x31\xb3\xbd",
		737	.expected_ss =
		738	"\x34\xc3\x35\x14\x88\x46\x26\x23\x97\xbb\xdd\x28\x5c\x94\xf6\x47"
		739	"\xca\xb3\x19\xaf\xca\x44\x9b\xc2\x7d\x89\xfd\x96\x14\xfd\x6d\x58"
		740	"\xd8\xc4\x6b\x61\x2a\x0d\xf2\x36\x45\xc8\xe4\xa4\xed\x81\x53\x81"
		741	"\x66\x1e\xe0\x5a\xb1\x78\x2d\x0b\x5c\xb4\xd1\xfc\x90\xc6\x9c\xdb"
		742	"\x5a\x30\x0b\x14\x7d\xbe\xb3\x7d\xb1\xb2\x76\x3c\x6c\xef\x74\x6b"
		743	"\xe7\x1f\x64\x0c\xab\x65\xe1\x76\x5c\x3d\x83\xb5\x8a\xfb\xaf\x0f"
		744	"\xf2\x06\x14\x8f\xa0\xf6\xc1\x89\x78\xf2\xba\x72\x73\x3c\xf7\x76"
		745	"\x21\x67\xbc\x24\x31\xb8\x09\x65\x0f\x0c\x02\x32\x4a\x98\x14\xfc"
		746	"\x72\x2c\x25\x60\x68\x5f\x2f\x30\x1e\x5b\xf0\x3b\xd1\xa2\x87\xa0"
		747	"\x54\xdf\xdb\xc0\xee\x0a\x0f\x47\xc9\x90\x20\x2c\xf9\xe3\x52\xad"
		748	"\x27\x65\x8d\x54\x8d\xa8\xa1\xf3\xed\x15\xd4\x94\x28\x90\x31\x93"
		749	"\x1b\xc0\x51\xbb\x43\x5d\x76\x3b\x1d\x2a\x71\x50\xea\x5d\x48\x94"
		750	"\x7f\x6f\xf1\x48\xdb\x30\xe5\xae\x64\x79\xd9\x7a\xdb\xc6\xff\xd8"
		751	"\x5e\x5a\x64\xbd\xf6\x85\x04\xe8\x28\x6a\xac\xef\xce\x19\x8e\x9a"
		752	"\xfe\x75\xc0\x27\x69\xe3\xb3\x7b\x21\xa7\xb1\x16\xa4\x85\x23\xee"
		753	"\xb0\x1b\x04\x6e\xbd\xab\x16\xde\xfd\x86\x6b\xa9\x95\xd7\x0b\xfd",
		754	.secret_size = 529,
		755	.b_public_size = 256,
		756	.expected_a_public_size = 256,
		757	.expected_ss_size = 256,
		758	}
		759	};
		760
		761	#ifdef CONFIG_CRYPTO_FIPS
		762	#define ECDH_TEST_VECTORS 1
		763	#else
		764	#define ECDH_TEST_VECTORS 2
		765	#endif
		766	struct kpp_testvec ecdh_tv_template[] = {
		767	{
		768	#ifndef CONFIG_CRYPTO_FIPS
		769	.secret =
		770	#ifdef __LITTLE_ENDIAN
		771	"\x02\x00" /* type */
		772	"\x20\x00" /* len */
		773	"\x01\x00" /* curve_id */
		774	"\x18\x00" /* key_size */
		775	#else
		776	"\x00\x02" /* type */
		777	"\x00\x20" /* len */
		778	"\x00\x01" /* curve_id */
		779	"\x00\x18" /* key_size */
		780	#endif
		781	"\xb5\x05\xb1\x71\x1e\xbf\x8c\xda"
		782	"\x4e\x19\x1e\x62\x1f\x23\x23\x31"
		783	"\x36\x1e\xd3\x84\x2f\xcc\x21\x72",
		784	.b_public =
		785	"\xc3\xba\x67\x4b\x71\xec\xd0\x76"
		786	"\x7a\x99\x75\x64\x36\x13\x9a\x94"
		787	"\x5d\x8b\xdc\x60\x90\x91\xfd\x3f"
		788	"\xb0\x1f\x8a\x0a\x68\xc6\x88\x6e"
		789	"\x83\x87\xdd\x67\x09\xf8\x8d\x96"
		790	"\x07\xd6\xbd\x1c\xe6\x8d\x9d\x67",
		791	.expected_a_public =
		792	"\x1a\x04\xdb\xa5\xe1\xdd\x4e\x79"
		793	"\xa3\xe6\xef\x0e\x5c\x80\x49\x85"
		794	"\xfa\x78\xb4\xef\x49\xbd\x4c\x7c"
		795	"\x22\x90\x21\x02\xf9\x1b\x81\x5d"
		796	"\x0c\x8a\xa8\x98\xd6\x27\x69\x88"
		797	"\x5e\xbc\x94\xd8\x15\x9e\x21\xce",
		798	.expected_ss =
		799	"\xf4\x57\xcc\x4f\x1f\x4e\x31\xcc"
		800	"\xe3\x40\x60\xc8\x06\x93\xc6\x2e"
		801	"\x99\x80\x81\x28\xaf\xc5\x51\x74",
		802	.secret_size = 32,
		803	.b_public_size = 48,
		804	.expected_a_public_size = 48,
		805	.expected_ss_size = 24
		806	}, {
		807	#endif
		808	.secret =
		809	#ifdef __LITTLE_ENDIAN
		810	"\x02\x00" /* type */
		811	"\x28\x00" /* len */
		812	"\x02\x00" /* curve_id */
		813	"\x20\x00" /* key_size */
		814	#else
		815	"\x00\x02" /* type */
		816	"\x00\x28" /* len */
		817	"\x00\x02" /* curve_id */
		818	"\x00\x20" /* key_size */
		819	#endif
		820	"\x24\xd1\x21\xeb\xe5\xcf\x2d\x83"
		821	"\xf6\x62\x1b\x6e\x43\x84\x3a\xa3"
		822	"\x8b\xe0\x86\xc3\x20\x19\xda\x92"
		823	"\x50\x53\x03\xe1\xc0\xea\xb8\x82",
		824	.expected_a_public =
		825	"\x1a\x7f\xeb\x52\x00\xbd\x3c\x31"
		826	"\x7d\xb6\x70\xc1\x86\xa6\xc7\xc4"
		827	"\x3b\xc5\x5f\x6c\x6f\x58\x3c\xf5"
		828	"\xb6\x63\x82\x77\x33\x24\xa1\x5f"
		829	"\x6a\xca\x43\x6f\xf7\x7e\xff\x02"
		830	"\x37\x08\xcc\x40\x5e\x7a\xfd\x6a"
		831	"\x6a\x02\x6e\x41\x87\x68\x38\x77"
		832	"\xfa\xa9\x44\x43\x2d\xef\x09\xdf",
		833	.expected_ss =
		834	"\xea\x17\x6f\x7e\x6e\x57\x26\x38"
		835	"\x8b\xfb\x41\xeb\xba\xc8\x6d\xa5"
		836	"\xa8\x72\xd1\xff\xc9\x47\x3d\xaa"
		837	"\x58\x43\x9f\x34\x0f\x8c\xf3\xc9",
		838	.b_public =
		839	"\xcc\xb4\xda\x74\xb1\x47\x3f\xea"
		840	"\x6c\x70\x9e\x38\x2d\xc7\xaa\xb7"
		841	"\x29\xb2\x47\x03\x19\xab\xdd\x34"
		842	"\xbd\xa8\x2c\x93\xe1\xa4\x74\xd9"
		843	"\x64\x63\xf7\x70\x20\x2f\xa4\xe6"
		844	"\x9f\x4a\x38\xcc\xc0\x2c\x49\x2f"
		845	"\xb1\x32\xbb\xaf\x22\x61\xda\xcb"
		846	"\x6f\xdb\xa9\xaa\xfc\x77\x81\xf3",
		847	.secret_size = 40,
		848	.b_public_size = 64,
		849	.expected_a_public_size = 64,
		850	.expected_ss_size = 32
330	}	851	}
331	};	852	};
332		853
@@ -376,6 +897,131 @@ static struct hash_testvec md4_tv_template [] = { · 43 unmodified lines
376	},	897	},
377	};	898	};
378		899
		900	#define SHA3_224_TEST_VECTORS 3
		901	static struct hash_testvec sha3_224_tv_template[] = {
		902	{
		903	.plaintext = "",
		904	.digest = "\x6b\x4e\x03\x42\x36\x67\xdb\xb7"
		905	"\x3b\x6e\x15\x45\x4f\x0e\xb1\xab"
		906	"\xd4\x59\x7f\x9a\x1b\x07\x8e\x3f"
		907	"\x5b\x5a\x6b\xc7",
		908	}, {
		909	.plaintext = "a",
		910	.psize = 1,
		911	.digest = "\x9e\x86\xff\x69\x55\x7c\xa9\x5f"
		912	"\x40\x5f\x08\x12\x69\x68\x5b\x38"
		913	"\xe3\xa8\x19\xb3\x09\xee\x94\x2f"
		914	"\x48\x2b\x6a\x8b",
		915	}, {
		916	.plaintext = "abcdbcdecdefdefgefghfghighijhijkijkl"
		917	"jklmklmnlmnomnopnopq",
		918	.psize = 56,
		919	.digest = "\x8a\x24\x10\x8b\x15\x4a\xda\x21"
		920	"\xc9\xfd\x55\x74\x49\x44\x79\xba"
		921	"\x5c\x7e\x7a\xb7\x6e\xf2\x64\xea"
		922	"\xd0\xfc\xce\x33",
		923	},
		924	};
		925
		926	#define SHA3_256_TEST_VECTORS 3
		927	static struct hash_testvec sha3_256_tv_template[] = {
		928	{
		929	.plaintext = "",
		930	.digest = "\xa7\xff\xc6\xf8\xbf\x1e\xd7\x66"
		931	"\x51\xc1\x47\x56\xa0\x61\xd6\x62"
		932	"\xf5\x80\xff\x4d\xe4\x3b\x49\xfa"
		933	"\x82\xd8\x0a\x4b\x80\xf8\x43\x4a",
		934	}, {
		935	.plaintext = "a",
		936	.psize = 1,
		937	.digest = "\x80\x08\x4b\xf2\xfb\xa0\x24\x75"
		938	"\x72\x6f\xeb\x2c\xab\x2d\x82\x15"
		939	"\xea\xb1\x4b\xc6\xbd\xd8\xbf\xb2"
		940	"\xc8\x15\x12\x57\x03\x2e\xcd\x8b",
		941	}, {
		942	.plaintext = "abcdbcdecdefdefgefghfghighijhijkijkl"
		943	"jklmklmnlmnomnopnopq",
		944	.psize = 56,
		945	.digest = "\x41\xc0\xdb\xa2\xa9\xd6\x24\x08"
		946	"\x49\x10\x03\x76\xa8\x23\x5e\x2c"
		947	"\x82\xe1\xb9\x99\x8a\x99\x9e\x21"
		948	"\xdb\x32\xdd\x97\x49\x6d\x33\x76",
		949	},
		950	};
		951
		952
		953	#define SHA3_384_TEST_VECTORS 3
		954	static struct hash_testvec sha3_384_tv_template[] = {
		955	{
		956	.plaintext = "",
		957	.digest = "\x0c\x63\xa7\x5b\x84\x5e\x4f\x7d"
		958	"\x01\x10\x7d\x85\x2e\x4c\x24\x85"
		959	"\xc5\x1a\x50\xaa\xaa\x94\xfc\x61"
		960	"\x99\x5e\x71\xbb\xee\x98\x3a\x2a"
		961	"\xc3\x71\x38\x31\x26\x4a\xdb\x47"
		962	"\xfb\x6b\xd1\xe0\x58\xd5\xf0\x04",
		963	}, {
		964	.plaintext = "a",
		965	.psize = 1,
		966	.digest = "\x18\x15\xf7\x74\xf3\x20\x49\x1b"
		967	"\x48\x56\x9e\xfe\xc7\x94\xd2\x49"
		968	"\xee\xb5\x9a\xae\x46\xd2\x2b\xf7"
		969	"\x7d\xaf\xe2\x5c\x5e\xdc\x28\xd7"
		970	"\xea\x44\xf9\x3e\xe1\x23\x4a\xa8"
		971	"\x8f\x61\xc9\x19\x12\xa4\xcc\xd9",
		972	}, {
		973	.plaintext = "abcdbcdecdefdefgefghfghighijhijkijkl"
		974	"jklmklmnlmnomnopnopq",
		975	.psize = 56,
		976	.digest = "\x99\x1c\x66\x57\x55\xeb\x3a\x4b"
		977	"\x6b\xbd\xfb\x75\xc7\x8a\x49\x2e"
		978	"\x8c\x56\xa2\x2c\x5c\x4d\x7e\x42"
		979	"\x9b\xfd\xbc\x32\xb9\xd4\xad\x5a"
		980	"\xa0\x4a\x1f\x07\x6e\x62\xfe\xa1"
		981	"\x9e\xef\x51\xac\xd0\x65\x7c\x22",
		982	},
		983	};
		984
		985
		986	#define SHA3_512_TEST_VECTORS 3
		987	static struct hash_testvec sha3_512_tv_template[] = {
		988	{
		989	.plaintext = "",
		990	.digest = "\xa6\x9f\x73\xcc\xa2\x3a\x9a\xc5"
		991	"\xc8\xb5\x67\xdc\x18\x5a\x75\x6e"
		992	"\x97\xc9\x82\x16\x4f\xe2\x58\x59"
		993	"\xe0\xd1\xdc\xc1\x47\x5c\x80\xa6"
		994	"\x15\xb2\x12\x3a\xf1\xf5\xf9\x4c"
		995	"\x11\xe3\xe9\x40\x2c\x3a\xc5\x58"
		996	"\xf5\x00\x19\x9d\x95\xb6\xd3\xe3"
		997	"\x01\x75\x85\x86\x28\x1d\xcd\x26",
		998	}, {
		999	.plaintext = "a",
		1000	.psize = 1,
		1001	.digest = "\x69\x7f\x2d\x85\x61\x72\xcb\x83"
		1002	"\x09\xd6\xb8\xb9\x7d\xac\x4d\xe3"
		1003	"\x44\xb5\x49\xd4\xde\xe6\x1e\xdf"
		1004	"\xb4\x96\x2d\x86\x98\xb7\xfa\x80"
		1005	"\x3f\x4f\x93\xff\x24\x39\x35\x86"
		1006	"\xe2\x8b\x5b\x95\x7a\xc3\xd1\xd3"
		1007	"\x69\x42\x0c\xe5\x33\x32\x71\x2f"
		1008	"\x99\x7b\xd3\x36\xd0\x9a\xb0\x2a",
		1009	}, {
		1010	.plaintext = "abcdbcdecdefdefgefghfghighijhijkijkl"
		1011	"jklmklmnlmnomnopnopq",
		1012	.psize = 56,
		1013	.digest = "\x04\xa3\x71\xe8\x4e\xcf\xb5\xb8"
		1014	"\xb7\x7c\xb4\x86\x10\xfc\xa8\x18"
		1015	"\x2d\xd4\x57\xce\x6f\x32\x6a\x0f"
		1016	"\xd3\xd7\xec\x2f\x1e\x91\x63\x6d"
		1017	"\xee\x69\x1f\xbe\x0c\x98\x53\x02"
		1018	"\xba\x1b\x0d\x8d\xc7\x8c\x08\x63"
		1019	"\x46\xb5\x33\xb4\x9c\x03\x0d\x99"
		1020	"\xa2\x7d\xaf\x11\x39\xd6\xe7\x5e",
		1021	},
		1022	};
		1023
		1024
379	/*	1025	/*
380	* MD5 test vectors from RFC1321	1026	* MD5 test vectors from RFC1321
381	*/	1027	*/
@@ -3246,6 +3892,394 @@ static struct hash_testvec hmac_sha512_tv_template[] = { · 2864 unmodified lines
3246	},	3892	},
3247	};	3893	};
3248		3894
		3895	#define HMAC_SHA3_224_TEST_VECTORS 4
		3896
		3897	static struct hash_testvec hmac_sha3_224_tv_template[] = {
		3898	{
		3899	.key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b"
		3900	"\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b"
		3901	"\x0b\x0b\x0b\x0b",
		3902	.ksize = 20,
		3903	.plaintext = "Hi There",
		3904	.psize = 8,
		3905	.digest = "\x3b\x16\x54\x6b\xbc\x7b\xe2\x70"
		3906	"\x6a\x03\x1d\xca\xfd\x56\x37\x3d"
		3907	"\x98\x84\x36\x76\x41\xd8\xc5\x9a"
		3908	"\xf3\xc8\x60\xf7",
		3909	}, {
		3910	.key = "Jefe",
		3911	.ksize = 4,
		3912	.plaintext = "what do ya want for nothing?",
		3913	.psize = 28,
		3914	.digest = "\x7f\xdb\x8d\xd8\x8b\xd2\xf6\x0d"
		3915	"\x1b\x79\x86\x34\xad\x38\x68\x11"
		3916	"\xc2\xcf\xc8\x5b\xfa\xf5\xd5\x2b"
		3917	"\xba\xce\x5e\x66",
		3918	.np = 4,
		3919	.tap = { 7, 7, 7, 7 }
		3920	}, {
		3921	.key = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3922	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3923	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3924	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3925	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3926	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3927	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3928	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3929	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3930	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3931	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3932	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3933	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3934	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3935	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3936	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3937	"\xaa\xaa\xaa",
		3938	.ksize = 131,
		3939	.plaintext = "Test Using Large"
		3940	"r Than Block-Siz"
		3941	"e Key - Hash Key"
		3942	" First",
		3943	.psize = 54,
		3944	.digest = "\xb4\xa1\xf0\x4c\x00\x28\x7a\x9b"
		3945	"\x7f\x60\x75\xb3\x13\xd2\x79\xb8"
		3946	"\x33\xbc\x8f\x75\x12\x43\x52\xd0"
		3947	"\x5f\xb9\x99\x5f",
		3948	}, {
		3949	.key = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3950	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3951	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3952	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3953	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3954	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3955	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3956	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3957	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3958	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3959	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3960	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3961	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3962	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3963	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3964	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		3965	"\xaa\xaa\xaa",
		3966	.ksize = 131,
		3967	.plaintext =
		3968	"This is a test u"
		3969	"sing a larger th"
		3970	"an block-size ke"
		3971	"y and a larger t"
		3972	"han block-size d"
		3973	"ata. The key nee"
		3974	"ds to be hashed "
		3975	"before being use"
		3976	"d by the HMAC al"
		3977	"gorithm.",
		3978	.psize = 152,
		3979	.digest = "\x05\xd8\xcd\x6d\x00\xfa\xea\x8d"
		3980	"\x1e\xb6\x8a\xde\x28\x73\x0b\xbd"
		3981	"\x3c\xba\xb6\x92\x9f\x0a\x08\x6b"
		3982	"\x29\xcd\x62\xa0",
		3983	},
		3984	};
		3985
		3986	#define HMAC_SHA3_256_TEST_VECTORS 4
		3987
		3988	static struct hash_testvec hmac_sha3_256_tv_template[] = {
		3989	{
		3990	.key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b"
		3991	"\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b"
		3992	"\x0b\x0b\x0b\x0b",
		3993	.ksize = 20,
		3994	.plaintext = "Hi There",
		3995	.psize = 8,
		3996	.digest = "\xba\x85\x19\x23\x10\xdf\xfa\x96"
		3997	"\xe2\xa3\xa4\x0e\x69\x77\x43\x51"
		3998	"\x14\x0b\xb7\x18\x5e\x12\x02\xcd"
		3999	"\xcc\x91\x75\x89\xf9\x5e\x16\xbb",
		4000	}, {
		4001	.key = "Jefe",
		4002	.ksize = 4,
		4003	.plaintext = "what do ya want for nothing?",
		4004	.psize = 28,
		4005	.digest = "\xc7\xd4\x07\x2e\x78\x88\x77\xae"
		4006	"\x35\x96\xbb\xb0\xda\x73\xb8\x87"
		4007	"\xc9\x17\x1f\x93\x09\x5b\x29\x4a"
		4008	"\xe8\x57\xfb\xe2\x64\x5e\x1b\xa5",
		4009	.np = 4,
		4010	.tap = { 7, 7, 7, 7 }
		4011	}, {
		4012	.key = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4013	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4014	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4015	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4016	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4017	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4018	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4019	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4020	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4021	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4022	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4023	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4024	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4025	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4026	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4027	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4028	"\xaa\xaa\xaa",
		4029	.ksize = 131,
		4030	.plaintext = "Test Using Large"
		4031	"r Than Block-Siz"
		4032	"e Key - Hash Key"
		4033	" First",
		4034	.psize = 54,
		4035	.digest = "\xed\x73\xa3\x74\xb9\x6c\x00\x52"
		4036	"\x35\xf9\x48\x03\x2f\x09\x67\x4a"
		4037	"\x58\xc0\xce\x55\x5c\xfc\x1f\x22"
		4038	"\x3b\x02\x35\x65\x60\x31\x2c\x3b",
		4039	}, {
		4040	.key = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4041	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4042	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4043	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4044	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4045	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4046	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4047	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4048	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4049	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4050	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4051	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4052	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4053	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4054	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4055	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4056	"\xaa\xaa\xaa",
		4057	.ksize = 131,
		4058	.plaintext =
		4059	"This is a test u"
		4060	"sing a larger th"
		4061	"an block-size ke"
		4062	"y and a larger t"
		4063	"han block-size d"
		4064	"ata. The key nee"
		4065	"ds to be hashed "
		4066	"before being use"
		4067	"d by the HMAC al"
		4068	"gorithm.",
		4069	.psize = 152,
		4070	.digest = "\x65\xc5\xb0\x6d\x4c\x3d\xe3\x2a"
		4071	"\x7a\xef\x87\x63\x26\x1e\x49\xad"
		4072	"\xb6\xe2\x29\x3e\xc8\xe7\xc6\x1e"
		4073	"\x8d\xe6\x17\x01\xfc\x63\xe1\x23",
		4074	},
		4075	};
		4076
		4077	#define HMAC_SHA3_384_TEST_VECTORS 4
		4078
		4079	static struct hash_testvec hmac_sha3_384_tv_template[] = {
		4080	{
		4081	.key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b"
		4082	"\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b"
		4083	"\x0b\x0b\x0b\x0b",
		4084	.ksize = 20,
		4085	.plaintext = "Hi There",
		4086	.psize = 8,
		4087	.digest = "\x68\xd2\xdc\xf7\xfd\x4d\xdd\x0a"
		4088	"\x22\x40\xc8\xa4\x37\x30\x5f\x61"
		4089	"\xfb\x73\x34\xcf\xb5\xd0\x22\x6e"
		4090	"\x1b\xc2\x7d\xc1\x0a\x2e\x72\x3a"
		4091	"\x20\xd3\x70\xb4\x77\x43\x13\x0e"
		4092	"\x26\xac\x7e\x3d\x53\x28\x86\xbd",
		4093	}, {
		4094	.key = "Jefe",
		4095	.ksize = 4,
		4096	.plaintext = "what do ya want for nothing?",
		4097	.psize = 28,
		4098	.digest = "\xf1\x10\x1f\x8c\xbf\x97\x66\xfd"
		4099	"\x67\x64\xd2\xed\x61\x90\x3f\x21"
		4100	"\xca\x9b\x18\xf5\x7c\xf3\xe1\xa2"
		4101	"\x3c\xa1\x35\x08\xa9\x32\x43\xce"
		4102	"\x48\xc0\x45\xdc\x00\x7f\x26\xa2"
		4103	"\x1b\x3f\x5e\x0e\x9d\xf4\xc2\x0a",
		4104	.np = 4,
		4105	.tap = { 7, 7, 7, 7 }
		4106	}, {
		4107	.key = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4108	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4109	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4110	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4111	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4112	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4113	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4114	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4115	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4116	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4117	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4118	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4119	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4120	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4121	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4122	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4123	"\xaa\xaa\xaa",
		4124	.ksize = 131,
		4125	.plaintext = "Test Using Large"
		4126	"r Than Block-Siz"
		4127	"e Key - Hash Key"
		4128	" First",
		4129	.psize = 54,
		4130	.digest = "\x0f\xc1\x95\x13\xbf\x6b\xd8\x78"
		4131	"\x03\x70\x16\x70\x6a\x0e\x57\xbc"
		4132	"\x52\x81\x39\x83\x6b\x9a\x42\xc3"
		4133	"\xd4\x19\xe4\x98\xe0\xe1\xfb\x96"
		4134	"\x16\xfd\x66\x91\x38\xd3\x3a\x11"
		4135	"\x05\xe0\x7c\x72\xb6\x95\x3b\xcc",
		4136	}, {
		4137	.key = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4138	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4139	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4140	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4141	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4142	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4143	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4144	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4145	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4146	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4147	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4148	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4149	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4150	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4151	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4152	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4153	"\xaa\xaa\xaa",
		4154	.ksize = 131,
		4155	.plaintext =
		4156	"This is a test u"
		4157	"sing a larger th"
		4158	"an block-size ke"
		4159	"y and a larger t"
		4160	"han block-size d"
		4161	"ata. The key nee"
		4162	"ds to be hashed "
		4163	"before being use"
		4164	"d by the HMAC al"
		4165	"gorithm.",
		4166	.psize = 152,
		4167	.digest = "\x02\x6f\xdf\x6b\x50\x74\x1e\x37"
		4168	"\x38\x99\xc9\xf7\xd5\x40\x6d\x4e"
		4169	"\xb0\x9f\xc6\x66\x56\x36\xfc\x1a"
		4170	"\x53\x00\x29\xdd\xf5\xcf\x3c\xa5"
		4171	"\xa9\x00\xed\xce\x01\xf5\xf6\x1e"
		4172	"\x2f\x40\x8c\xdf\x2f\xd3\xe7\xe8",
		4173	},
		4174	};
		4175
		4176	#define HMAC_SHA3_512_TEST_VECTORS 4
		4177
		4178	static struct hash_testvec hmac_sha3_512_tv_template[] = {
		4179	{
		4180	.key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b"
		4181	"\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b"
		4182	"\x0b\x0b\x0b\x0b",
		4183	.ksize = 20,
		4184	.plaintext = "Hi There",
		4185	.psize = 8,
		4186	.digest = "\xeb\x3f\xbd\x4b\x2e\xaa\xb8\xf5"
		4187	"\xc5\x04\xbd\x3a\x41\x46\x5a\xac"
		4188	"\xec\x15\x77\x0a\x7c\xab\xac\x53"
		4189	"\x1e\x48\x2f\x86\x0b\x5e\xc7\xba"
		4190	"\x47\xcc\xb2\xc6\xf2\xaf\xce\x8f"
		4191	"\x88\xd2\x2b\x6d\xc6\x13\x80\xf2"
		4192	"\x3a\x66\x8f\xd3\x88\x8b\xb8\x05"
		4193	"\x37\xc0\xa0\xb8\x64\x07\x68\x9e",
		4194	}, {
		4195	.key = "Jefe",
		4196	.ksize = 4,
		4197	.plaintext = "what do ya want for nothing?",
		4198	.psize = 28,
		4199	.digest = "\x5a\x4b\xfe\xab\x61\x66\x42\x7c"
		4200	"\x7a\x36\x47\xb7\x47\x29\x2b\x83"
		4201	"\x84\x53\x7c\xdb\x89\xaf\xb3\xbf"
		4202	"\x56\x65\xe4\xc5\xe7\x09\x35\x0b"
		4203	"\x28\x7b\xae\xc9\x21\xfd\x7c\xa0"
		4204	"\xee\x7a\x0c\x31\xd0\x22\xa9\x5e"
		4205	"\x1f\xc9\x2b\xa9\xd7\x7d\xf8\x83"
		4206	"\x96\x02\x75\xbe\xb4\xe6\x20\x24",
		4207	.np = 4,
		4208	.tap = { 7, 7, 7, 7 }
		4209	}, {
		4210	.key = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4211	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4212	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4213	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4214	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4215	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4216	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4217	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4218	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4219	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4220	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4221	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4222	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4223	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4224	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4225	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4226	"\xaa\xaa\xaa",
		4227	.ksize = 131,
		4228	.plaintext = "Test Using Large"
		4229	"r Than Block-Siz"
		4230	"e Key - Hash Key"
		4231	" First",
		4232	.psize = 54,
		4233	.digest = "\x00\xf7\x51\xa9\xe5\x06\x95\xb0"
		4234	"\x90\xed\x69\x11\xa4\xb6\x55\x24"
		4235	"\x95\x1c\xdc\x15\xa7\x3a\x5d\x58"
		4236	"\xbb\x55\x21\x5e\xa2\xcd\x83\x9a"
		4237	"\xc7\x9d\x2b\x44\xa3\x9b\xaf\xab"
		4238	"\x27\xe8\x3f\xde\x9e\x11\xf6\x34"
		4239	"\x0b\x11\xd9\x91\xb1\xb9\x1b\xf2"
		4240	"\xee\xe7\xfc\x87\x24\x26\xc3\xa4",
		4241	}, {
		4242	.key = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4243	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4244	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4245	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4246	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4247	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4248	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4249	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4250	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4251	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4252	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4253	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4254	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4255	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4256	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4257	"\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
		4258	"\xaa\xaa\xaa",
		4259	.ksize = 131,
		4260	.plaintext =
		4261	"This is a test u"
		4262	"sing a larger th"
		4263	"an block-size ke"
		4264	"y and a larger t"
		4265	"han block-size d"
		4266	"ata. The key nee"
		4267	"ds to be hashed "
		4268	"before being use"
		4269	"d by the HMAC al"
		4270	"gorithm.",
		4271	.psize = 152,
		4272	.digest = "\x38\xa4\x56\xa0\x04\xbd\x10\xd3"
		4273	"\x2c\x9a\xb8\x33\x66\x84\x11\x28"
		4274	"\x62\xc3\xdb\x61\xad\xcc\xa3\x18"
		4275	"\x29\x35\x5e\xaf\x46\xfd\x5c\x73"
		4276	"\xd0\x6a\x1f\x0d\x13\xfe\xc9\xa6"
		4277	"\x52\xfb\x38\x11\xb5\x77\xb1\xb1"
		4278	"\xd1\xb9\x78\x9f\x97\xae\x5b\x83"
		4279	"\xc6\xf4\x4d\xfc\xf1\xd6\x7e\xba",
		4280	},
		4281	};
		4282
3249	/*	4283	/*
3250	* Poly1305 test vectors from RFC7539 A.3.	4284	* Poly1305 test vectors from RFC7539 A.3.
3251	*/	4285	*/

drivers/char/hw_random/Kconfig+15−1

@@ -90,7 +90,7 @@ config HW_RANDOM_BCM63XX · 89 unmodified lines
90		90
91	config HW_RANDOM_BCM2835	91	config HW_RANDOM_BCM2835
92	tristate "Broadcom BCM2835 Random Number Generator support"	92	tristate "Broadcom BCM2835 Random Number Generator support"
93	depends on ARCH_BCM2835	93	depends on ARCH_BCM2835 \|\| ARCH_BCM_NSP \|\| ARCH_BCM_5301X
94	default HW_RANDOM	94	default HW_RANDOM
95	---help---	95	---help---
96	This driver provides kernel-side support for the Random Number	96	This driver provides kernel-side support for the Random Number
@@ -396,6 +396,20 @@ config HW_RANDOM_PIC32 · 299 unmodified lines
396		396
397	If unsure, say Y.	397	If unsure, say Y.
398		398
		399	config HW_RANDOM_MESON
		400	tristate "Amlogic Meson Random Number Generator support"
		401	depends on HW_RANDOM
		402	depends on ARCH_MESON \|\| COMPILE_TEST
		403	default y
		404	---help---
		405	This driver provides kernel-side support for the Random Number
		406	Generator hardware found on Amlogic Meson SoCs.
		407
		408	To compile this driver as a module, choose M here. the
		409	module will be called meson-rng.
		410
		411	If unsure, say Y.
		412
399	endif # HW_RANDOM	413	endif # HW_RANDOM
400		414
401	config UML_RANDOM	415	config UML_RANDOM

drivers/char/hw_random/Makefile+1−0

@@ -34,3 +34,4 @@ obj-$(CONFIG_HW_RANDOM_ST) += st-rng.o · 33 unmodified lines
34	obj-$(CONFIG_HW_RANDOM_XGENE) += xgene-rng.o	34	obj-$(CONFIG_HW_RANDOM_XGENE) += xgene-rng.o
35	obj-$(CONFIG_HW_RANDOM_STM32) += stm32-rng.o	35	obj-$(CONFIG_HW_RANDOM_STM32) += stm32-rng.o
36	obj-$(CONFIG_HW_RANDOM_PIC32) += pic32-rng.o	36	obj-$(CONFIG_HW_RANDOM_PIC32) += pic32-rng.o
		37	obj-$(CONFIG_HW_RANDOM_MESON) += meson-rng.o

drivers/char/hw_random/bcm2835-rng.c+41−6

 #define RNG_CTRL	0x0
 #define RNG_STATUS	0x4
 #define RNG_DATA	0x8
+#define RNG_INT_MASK	0x10
 /* enable rng */
 #define RNG_RBGEN	0x1
 /* the initial numbers generated are "less random" so will be discarded */
 #define RNG_WARMUP_COUNT 0x40000
+#define RNG_INT_OFF	0x1
+static void __init nsp_rng_init(void __iomem *base)
+{
+	u32 val;
+	/* mask the interrupt */
+	val = readl(base + RNG_INT_MASK);
+	val |= RNG_INT_OFF;
+	writel(val, base + RNG_INT_MASK);
+}
 static int bcm2835_rng_read(struct hwrng *rng, void *buf, size_t max,
 			       bool wait)
 {
 	void __iomem *rng_base = (void __iomem *)rng->priv;
+	u32 max_words = max / sizeof(u32);
+	u32 num_words, count;
 	while ((__raw_readl(rng_base + RNG_STATUS) >> 24) == 0) {
 		if (!wait)
 		cpu_relax();
 	}
-	*(u32 *)buf = __raw_readl(rng_base + RNG_DATA);
+	num_words = readl(rng_base + RNG_STATUS) >> 24;
-	return sizeof(u32);
+	if (num_words > max_words)
+		num_words = max_words;
+	for (count = 0; count < num_words; count++)
+		((u32 *)buf)[count] = readl(rng_base + RNG_DATA);
+	return num_words * sizeof(u32);
 }
 static struct hwrng bcm2835_rng_ops = {
 	.read	= bcm2835_rng_read,
 };
+static const struct of_device_id bcm2835_rng_of_match[] = {
+	{ .compatible = "brcm,bcm2835-rng"},
+	{ .compatible = "brcm,bcm-nsp-rng", .data = nsp_rng_init},
+	{ .compatible = "brcm,bcm5301x-rng", .data = nsp_rng_init},
+	{},
+};
 static int bcm2835_rng_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct device_node *np = dev->of_node;
+	void (*rng_setup)(void __iomem *base);
+	const struct of_device_id *rng_id;
 	void __iomem *rng_base;
 	int err;
 	}
 	bcm2835_rng_ops.priv = (unsigned long)rng_base;
+	rng_id = of_match_node(bcm2835_rng_of_match, np);
+	if (!rng_id)
+		return -EINVAL;
+	/* Check for rng init function, execute it */
+	rng_setup = rng_id->data;
+	if (rng_setup)
+		rng_setup(rng_base);
 	/* set warm-up count & enable */
 	__raw_writel(RNG_WARMUP_COUNT, rng_base + RNG_STATUS);
 	__raw_writel(RNG_RBGEN, rng_base + RNG_CTRL);
 	return 0;
 }
-static const struct of_device_id bcm2835_rng_of_match[] = {
-	{ .compatible = "brcm,bcm2835-rng", },
-	{},
-};
 MODULE_DEVICE_TABLE(of, bcm2835_rng_of_match);
 static struct platform_driver bcm2835_rng_driver = {

drivers/char/hw_random/exynos-rng.c+2−2

@@ -45,12 +45,12 @@ struct exynos_rng { · 44 unmodified lines
45		45
46	static u32 exynos_rng_readl(struct exynos_rng *rng, u32 offset)	46	static u32 exynos_rng_readl(struct exynos_rng *rng, u32 offset)
47	{	47	{
48	return __raw_readl(rng->mem + offset);	48	return readl_relaxed(rng->mem + offset);
49	}	49	}
50		50
51	static void exynos_rng_writel(struct exynos_rng *rng, u32 val, u32 offset)	51	static void exynos_rng_writel(struct exynos_rng *rng, u32 val, u32 offset)
52	{	52	{
53	__raw_writel(val, rng->mem + offset);	53	writel_relaxed(val, rng->mem + offset);
54	}	54	}
55		55
56	static int exynos_rng_configure(struct exynos_rng *exynos_rng)	56	static int exynos_rng_configure(struct exynos_rng *exynos_rng)

drivers/char/hw_random/meson-rng.c+131−0

@@ -0,0 +1,131 @@
	1	/*
	2	* This file is provided under a dual BSD/GPLv2 license. When using or
	3	* redistributing this file, you may do so under either license.
	4	*
	5	* GPL LICENSE SUMMARY
	6	*
	7	* Copyright (c) 2016 BayLibre, SAS.
	8	* Author: Neil Armstrong <narmstrong@baylibre.com>
	9	* Copyright (C) 2014 Amlogic, Inc.
	10	*
	11	* This program is free software; you can redistribute it and/or modify
	12	* it under the terms of version 2 of the GNU General Public License as
	13	* published by the Free Software Foundation.
	14	*
	15	* This program is distributed in the hope that it will be useful, but
	16	* WITHOUT ANY WARRANTY; without even the implied warranty of
	17	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	18	* General Public License for more details.
	19	*
	20	* You should have received a copy of the GNU General Public License
	21	* along with this program; if not, see <http://www.gnu.org/licenses/>.
	22	* The full GNU General Public License is included in this distribution
	23	* in the file called COPYING.
	24	*
	25	* BSD LICENSE
	26	*
	27	* Copyright (c) 2016 BayLibre, SAS.
	28	* Author: Neil Armstrong <narmstrong@baylibre.com>
	29	* Copyright (C) 2014 Amlogic, Inc.
	30	*
	31	* Redistribution and use in source and binary forms, with or without
	32	* modification, are permitted provided that the following conditions
	33	* are met:
	34	*
	35	* * Redistributions of source code must retain the above copyright
	36	* notice, this list of conditions and the following disclaimer.
	37	* * Redistributions in binary form must reproduce the above copyright
	38	* notice, this list of conditions and the following disclaimer in
	39	* the documentation and/or other materials provided with the
	40	* distribution.
	41	* * Neither the name of Intel Corporation nor the names of its
	42	* contributors may be used to endorse or promote products derived
	43	* from this software without specific prior written permission.
	44	*
	45	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	46	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	47	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	48	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	49	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	50	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	51	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	52	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	53	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	54	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	55	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	56	*/
	57	#include <linux/err.h>
	58	#include <linux/module.h>
	59	#include <linux/io.h>
	60	#include <linux/platform_device.h>
	61	#include <linux/hw_random.h>
	62	#include <linux/slab.h>
	63	#include <linux/types.h>
	64	#include <linux/of.h>
	65
	66	#define RNG_DATA 0x00
	67
	68	struct meson_rng_data {
	69	void __iomem *base;
	70	struct platform_device *pdev;
	71	struct hwrng rng;
	72	};
	73
	74	static int meson_rng_read(struct hwrng rng, void buf, size_t max, bool wait)
	75	{
	76	struct meson_rng_data *data =
	77	container_of(rng, struct meson_rng_data, rng);
	78
	79	if (max < sizeof(u32))
	80	return 0;
	81
	82	(u32 )buf = readl_relaxed(data->base + RNG_DATA);
	83
	84	return sizeof(u32);
	85	}
	86
	87	static int meson_rng_probe(struct platform_device *pdev)
	88	{
	89	struct device *dev = &pdev->dev;
	90	struct meson_rng_data *data;
	91	struct resource *res;
	92
	93	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
	94	if (!data)
	95	return -ENOMEM;
	96
	97	data->pdev = pdev;
	98
	99	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	100	data->base = devm_ioremap_resource(dev, res);
	101	if (IS_ERR(data->base))
	102	return PTR_ERR(data->base);
	103
	104	data->rng.name = pdev->name;
	105	data->rng.read = meson_rng_read;
	106
	107	platform_set_drvdata(pdev, data);
	108
	109	return devm_hwrng_register(dev, &data->rng);
	110	}
	111
	112	static const struct of_device_id meson_rng_of_match[] = {
	113	{ .compatible = "amlogic,meson-rng", },
	114	{},
	115	};
	116
	117	static struct platform_driver meson_rng_driver = {
	118	.probe = meson_rng_probe,
	119	.driver = {
	120	.name = "meson-rng",
	121	.of_match_table = meson_rng_of_match,
	122	},
	123	};
	124
	125	module_platform_driver(meson_rng_driver);
	126
	127	MODULE_ALIAS("platform:meson-rng");
	128	MODULE_DESCRIPTION("Meson H/W Random Number Generator driver");
	129	MODULE_AUTHOR("Lawrence Mok <lawrence.mok@amlogic.com>");
	130	MODULE_AUTHOR("Neil Armstrong <narmstrong@baylibre.com>");
	131	MODULE_LICENSE("Dual BSD/GPL");

drivers/char/hw_random/omap-rng.c+14−2

 	}
 	pm_runtime_enable(&pdev->dev);
-	pm_runtime_get_sync(&pdev->dev);
+	ret = pm_runtime_get_sync(&pdev->dev);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to runtime_get device: %d\n", ret);
+		pm_runtime_put_noidle(&pdev->dev);
+		goto err_ioremap;
+	}
 	ret = (dev->of_node) ? of_get_omap_rng_device_details(priv, pdev) :
 				get_omap_rng_device_details(priv);
 static int __maybe_unused omap_rng_resume(struct device *dev)
 {
 	struct omap_rng_dev *priv = dev_get_drvdata(dev);
+	int ret;
+	ret = pm_runtime_get_sync(dev);
+	if (ret) {
+		dev_err(dev, "Failed to runtime_get device: %d\n", ret);
+		pm_runtime_put_noidle(dev);
+		return ret;
+	}
-	pm_runtime_get_sync(dev);
 	priv->pdata->init(priv);
 	return 0;

drivers/char/hw_random/stm32-rng.c+5−5

 		}
 		/* If error detected or data not ready... */
-		if (sr != RNG_SR_DRDY)
+		if (sr != RNG_SR_DRDY) {
+			if (WARN_ONCE(sr & (RNG_SR_SEIS | RNG_SR_CEIS),
+					"bad RNG status - %x\n", sr))
+				writel_relaxed(0, priv->base + RNG_SR);
 			break;
+		}
 		*(u32 *)data = readl_relaxed(priv->base + RNG_DR);
 		max -= sizeof(u32);
 	}
-	if (WARN_ONCE(sr & (RNG_SR_SEIS | RNG_SR_CEIS),
-		      "bad RNG status - %x\n", sr))
-		writel_relaxed(0, priv->base + RNG_SR);
 	pm_runtime_mark_last_busy((struct device *) priv->rng.priv);
 	pm_runtime_put_sync_autosuspend((struct device *) priv->rng.priv);

drivers/crypto/bfin_crc.c+0−5

 	crypto_init_queue(&crc->queue, CRC_CCRYPTO_QUEUE_LENGTH);
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (res == NULL) {
-		dev_err(&pdev->dev, "Cannot get IORESOURCE_MEM\n");
-		return -ENOENT;
-	}
 	crc->regs = devm_ioremap_resource(dev, res);
 	if (IS_ERR((void *)crc->regs)) {
 		dev_err(&pdev->dev, "Cannot map CRC IO\n");

drivers/crypto/caam/Kconfig+13−5

 config CRYPTO_DEV_FSL_CAAM
 	tristate "Freescale CAAM-Multicore driver backend"
-	depends on FSL_SOC || ARCH_MXC
+	depends on FSL_SOC || ARCH_MXC || ARCH_LAYERSCAPE
 	help
 	  Enables the driver module for Freescale's Cryptographic Accelerator
 	  and Assurance Module (CAAM), also known as the SEC version 4 (SEC4).
 	  To compile this as a module, choose M here: the module
 	  will be called caamhash.
+config CRYPTO_DEV_FSL_CAAM_PKC_API
+        tristate "Register public key cryptography implementations with Crypto API"
+        depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR
+        default y
+        select CRYPTO_RSA
+        help
+          Selecting this will allow SEC Public key support for RSA.
+          Supported cryptographic primitives: encryption, decryption,
+          signature and verification.
+          To compile this as a module, choose M here: the module
+          will be called caam_pkc.
 config CRYPTO_DEV_FSL_CAAM_RNG_API
 	tristate "Register caam device for hwrng API"
 	depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR
 	def_bool SOC_IMX6 || SOC_IMX7D
 	depends on CRYPTO_DEV_FSL_CAAM
-config CRYPTO_DEV_FSL_CAAM_LE
-	def_bool CRYPTO_DEV_FSL_CAAM_IMX || SOC_LS1021A
-	depends on CRYPTO_DEV_FSL_CAAM
 config CRYPTO_DEV_FSL_CAAM_DEBUG
 	bool "Enable debug output in CAAM driver"
 	depends on CRYPTO_DEV_FSL_CAAM

drivers/crypto/caam/Makefile+3−1

 # Makefile for the CAAM backend and dependent components
 #
 ifeq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_DEBUG), y)
-	EXTRA_CFLAGS := -DDEBUG
+	ccflags-y := -DDEBUG
 endif
 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam.o
 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o
 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o
 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o
+obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API) += caam_pkc.o
 caam-objs := ctrl.o
 caam_jr-objs := jr.o key_gen.o error.o
+caam_pkc-y := caampkc.o pkc_desc.o

drivers/crypto/caam/caamhash.c+3−2

 							 *next_buflen, 0);
 		} else {
 			(edesc->sec4_sg + sec4_sg_src_index - 1)->len |=
-							SEC4_SG_LEN_FIN;
+				cpu_to_caam32(SEC4_SG_LEN_FIN);
 		}
 		state->current_buf = !state->current_buf;
 	state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1,
 						buf, state->buf_dma, buflen,
 						last_buflen);
-	(edesc->sec4_sg + sec4_sg_src_index - 1)->len |= SEC4_SG_LEN_FIN;
+	(edesc->sec4_sg + sec4_sg_src_index - 1)->len |=
+		cpu_to_caam32(SEC4_SG_LEN_FIN);
 	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
 					    sec4_sg_bytes, DMA_TO_DEVICE);

drivers/crypto/caam/caampkc.c+607−0

@@ -0,0 +1,607 @@
	1	/*
	2	* caam - Freescale FSL CAAM support for Public Key Cryptography
	3	*
	4	* Copyright 2016 Freescale Semiconductor, Inc.
	5	*
	6	* There is no Shared Descriptor for PKC so that the Job Descriptor must carry
	7	* all the desired key parameters, input and output pointers.
	8	*/
	9	#include "compat.h"
	10	#include "regs.h"
	11	#include "intern.h"
	12	#include "jr.h"
	13	#include "error.h"
	14	#include "desc_constr.h"
	15	#include "sg_sw_sec4.h"
	16	#include "caampkc.h"
	17
	18	#define DESC_RSA_PUB_LEN (2 * CAAM_CMD_SZ + sizeof(struct rsa_pub_pdb))
	19	#define DESC_RSA_PRIV_F1_LEN (2 * CAAM_CMD_SZ + \
	20	sizeof(struct rsa_priv_f1_pdb))
	21
	22	static void rsa_io_unmap(struct device dev, struct rsa_edesc edesc,
	23	struct akcipher_request *req)
	24	{
	25	dma_unmap_sg(dev, req->dst, edesc->dst_nents, DMA_FROM_DEVICE);
	26	dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
	27
	28	if (edesc->sec4_sg_bytes)
	29	dma_unmap_single(dev, edesc->sec4_sg_dma, edesc->sec4_sg_bytes,
	30	DMA_TO_DEVICE);
	31	}
	32
	33	static void rsa_pub_unmap(struct device dev, struct rsa_edesc edesc,
	34	struct akcipher_request *req)
	35	{
	36	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
	37	struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
	38	struct caam_rsa_key *key = &ctx->key;
	39	struct rsa_pub_pdb *pdb = &edesc->pdb.pub;
	40
	41	dma_unmap_single(dev, pdb->n_dma, key->n_sz, DMA_TO_DEVICE);
	42	dma_unmap_single(dev, pdb->e_dma, key->e_sz, DMA_TO_DEVICE);
	43	}
	44
	45	static void rsa_priv_f1_unmap(struct device dev, struct rsa_edesc edesc,
	46	struct akcipher_request *req)
	47	{
	48	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
	49	struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
	50	struct caam_rsa_key *key = &ctx->key;
	51	struct rsa_priv_f1_pdb *pdb = &edesc->pdb.priv_f1;
	52
	53	dma_unmap_single(dev, pdb->n_dma, key->n_sz, DMA_TO_DEVICE);
	54	dma_unmap_single(dev, pdb->d_dma, key->d_sz, DMA_TO_DEVICE);
	55	}
	56
	57	/* RSA Job Completion handler */
	58	static void rsa_pub_done(struct device dev, u32 desc, u32 err, void *context)
	59	{
	60	struct akcipher_request *req = context;
	61	struct rsa_edesc *edesc;
	62
	63	if (err)
	64	caam_jr_strstatus(dev, err);
	65
	66	edesc = container_of(desc, struct rsa_edesc, hw_desc[0]);
	67
	68	rsa_pub_unmap(dev, edesc, req);
	69	rsa_io_unmap(dev, edesc, req);
	70	kfree(edesc);
	71
	72	akcipher_request_complete(req, err);
	73	}
	74
	75	static void rsa_priv_f1_done(struct device dev, u32 desc, u32 err,
	76	void *context)
	77	{
	78	struct akcipher_request *req = context;
	79	struct rsa_edesc *edesc;
	80
	81	if (err)
	82	caam_jr_strstatus(dev, err);
	83
	84	edesc = container_of(desc, struct rsa_edesc, hw_desc[0]);
	85
	86	rsa_priv_f1_unmap(dev, edesc, req);
	87	rsa_io_unmap(dev, edesc, req);
	88	kfree(edesc);
	89
	90	akcipher_request_complete(req, err);
	91	}
	92
	93	static struct rsa_edesc rsa_edesc_alloc(struct akcipher_request req,
	94	size_t desclen)
	95	{
	96	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
	97	struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
	98	struct device *dev = ctx->dev;
	99	struct rsa_edesc *edesc;
	100	gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG \|
	101	CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
	102	int sgc;
	103	int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes;
	104	int src_nents, dst_nents;
	105
	106	src_nents = sg_nents_for_len(req->src, req->src_len);
	107	dst_nents = sg_nents_for_len(req->dst, req->dst_len);
	108
	109	if (src_nents > 1)
	110	sec4_sg_len = src_nents;
	111	if (dst_nents > 1)
	112	sec4_sg_len += dst_nents;
	113
	114	sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
	115
	116	/* allocate space for base edesc, hw desc commands and link tables */
	117	edesc = kzalloc(sizeof(*edesc) + desclen + sec4_sg_bytes,
	118	GFP_DMA \| flags);
	119	if (!edesc)
	120	return ERR_PTR(-ENOMEM);
	121
	122	sgc = dma_map_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
	123	if (unlikely(!sgc)) {
	124	dev_err(dev, "unable to map source\n");
	125	goto src_fail;
	126	}
	127
	128	sgc = dma_map_sg(dev, req->dst, dst_nents, DMA_FROM_DEVICE);
	129	if (unlikely(!sgc)) {
	130	dev_err(dev, "unable to map destination\n");
	131	goto dst_fail;
	132	}
	133
	134	edesc->sec4_sg = (void )edesc + sizeof(edesc) + desclen;
	135
	136	sec4_sg_index = 0;
	137	if (src_nents > 1) {
	138	sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0);
	139	sec4_sg_index += src_nents;
	140	}
	141	if (dst_nents > 1)
	142	sg_to_sec4_sg_last(req->dst, dst_nents,
	143	edesc->sec4_sg + sec4_sg_index, 0);
	144
	145	/* Save nents for later use in Job Descriptor */
	146	edesc->src_nents = src_nents;
	147	edesc->dst_nents = dst_nents;
	148
	149	if (!sec4_sg_bytes)
	150	return edesc;
	151
	152	edesc->sec4_sg_dma = dma_map_single(dev, edesc->sec4_sg,
	153	sec4_sg_bytes, DMA_TO_DEVICE);
	154	if (dma_mapping_error(dev, edesc->sec4_sg_dma)) {
	155	dev_err(dev, "unable to map S/G table\n");
	156	goto sec4_sg_fail;
	157	}
	158
	159	edesc->sec4_sg_bytes = sec4_sg_bytes;
	160
	161	return edesc;
	162
	163	sec4_sg_fail:
	164	dma_unmap_sg(dev, req->dst, dst_nents, DMA_FROM_DEVICE);
	165	dst_fail:
	166	dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
	167	src_fail:
	168	kfree(edesc);
	169	return ERR_PTR(-ENOMEM);
	170	}
	171
	172	static int set_rsa_pub_pdb(struct akcipher_request *req,
	173	struct rsa_edesc *edesc)
	174	{
	175	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
	176	struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
	177	struct caam_rsa_key *key = &ctx->key;
	178	struct device *dev = ctx->dev;
	179	struct rsa_pub_pdb *pdb = &edesc->pdb.pub;
	180	int sec4_sg_index = 0;
	181
	182	pdb->n_dma = dma_map_single(dev, key->n, key->n_sz, DMA_TO_DEVICE);
	183	if (dma_mapping_error(dev, pdb->n_dma)) {
	184	dev_err(dev, "Unable to map RSA modulus memory\n");
	185	return -ENOMEM;
	186	}
	187
	188	pdb->e_dma = dma_map_single(dev, key->e, key->e_sz, DMA_TO_DEVICE);
	189	if (dma_mapping_error(dev, pdb->e_dma)) {
	190	dev_err(dev, "Unable to map RSA public exponent memory\n");
	191	dma_unmap_single(dev, pdb->n_dma, key->n_sz, DMA_TO_DEVICE);
	192	return -ENOMEM;
	193	}
	194
	195	if (edesc->src_nents > 1) {
	196	pdb->sgf \|= RSA_PDB_SGF_F;
	197	pdb->f_dma = edesc->sec4_sg_dma;
	198	sec4_sg_index += edesc->src_nents;
	199	} else {
	200	pdb->f_dma = sg_dma_address(req->src);
	201	}
	202
	203	if (edesc->dst_nents > 1) {
	204	pdb->sgf \|= RSA_PDB_SGF_G;
	205	pdb->g_dma = edesc->sec4_sg_dma +
	206	sec4_sg_index * sizeof(struct sec4_sg_entry);
	207	} else {
	208	pdb->g_dma = sg_dma_address(req->dst);
	209	}
	210
	211	pdb->sgf \|= (key->e_sz << RSA_PDB_E_SHIFT) \| key->n_sz;
	212	pdb->f_len = req->src_len;
	213
	214	return 0;
	215	}
	216
	217	static int set_rsa_priv_f1_pdb(struct akcipher_request *req,
	218	struct rsa_edesc *edesc)
	219	{
	220	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
	221	struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
	222	struct caam_rsa_key *key = &ctx->key;
	223	struct device *dev = ctx->dev;
	224	struct rsa_priv_f1_pdb *pdb = &edesc->pdb.priv_f1;
	225	int sec4_sg_index = 0;
	226
	227	pdb->n_dma = dma_map_single(dev, key->n, key->n_sz, DMA_TO_DEVICE);
	228	if (dma_mapping_error(dev, pdb->n_dma)) {
	229	dev_err(dev, "Unable to map modulus memory\n");
	230	return -ENOMEM;
	231	}
	232
	233	pdb->d_dma = dma_map_single(dev, key->d, key->d_sz, DMA_TO_DEVICE);
	234	if (dma_mapping_error(dev, pdb->d_dma)) {
	235	dev_err(dev, "Unable to map RSA private exponent memory\n");
	236	dma_unmap_single(dev, pdb->n_dma, key->n_sz, DMA_TO_DEVICE);
	237	return -ENOMEM;
	238	}
	239
	240	if (edesc->src_nents > 1) {
	241	pdb->sgf \|= RSA_PRIV_PDB_SGF_G;
	242	pdb->g_dma = edesc->sec4_sg_dma;
	243	sec4_sg_index += edesc->src_nents;
	244	} else {
	245	pdb->g_dma = sg_dma_address(req->src);
	246	}
	247
	248	if (edesc->dst_nents > 1) {
	249	pdb->sgf \|= RSA_PRIV_PDB_SGF_F;
	250	pdb->f_dma = edesc->sec4_sg_dma +
	251	sec4_sg_index * sizeof(struct sec4_sg_entry);
	252	} else {
	253	pdb->f_dma = sg_dma_address(req->dst);
	254	}
	255
	256	pdb->sgf \|= (key->d_sz << RSA_PDB_D_SHIFT) \| key->n_sz;
	257
	258	return 0;
	259	}
	260
	261	static int caam_rsa_enc(struct akcipher_request *req)
	262	{
	263	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
	264	struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
	265	struct caam_rsa_key *key = &ctx->key;
	266	struct device *jrdev = ctx->dev;
	267	struct rsa_edesc *edesc;
	268	int ret;
	269
	270	if (unlikely(!key->n \|\| !key->e))
	271	return -EINVAL;
	272
	273	if (req->dst_len < key->n_sz) {
	274	req->dst_len = key->n_sz;
	275	dev_err(jrdev, "Output buffer length less than parameter n\n");
	276	return -EOVERFLOW;
	277	}
	278
	279	/* Allocate extended descriptor */
	280	edesc = rsa_edesc_alloc(req, DESC_RSA_PUB_LEN);
	281	if (IS_ERR(edesc))
	282	return PTR_ERR(edesc);
	283
	284	/* Set RSA Encrypt Protocol Data Block */
	285	ret = set_rsa_pub_pdb(req, edesc);
	286	if (ret)
	287	goto init_fail;
	288
	289	/* Initialize Job Descriptor */
	290	init_rsa_pub_desc(edesc->hw_desc, &edesc->pdb.pub);
	291
	292	ret = caam_jr_enqueue(jrdev, edesc->hw_desc, rsa_pub_done, req);
	293	if (!ret)
	294	return -EINPROGRESS;
	295
	296	rsa_pub_unmap(jrdev, edesc, req);
	297
	298	init_fail:
	299	rsa_io_unmap(jrdev, edesc, req);
	300	kfree(edesc);
	301	return ret;
	302	}
	303
	304	static int caam_rsa_dec(struct akcipher_request *req)
	305	{
	306	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
	307	struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
	308	struct caam_rsa_key *key = &ctx->key;
	309	struct device *jrdev = ctx->dev;
	310	struct rsa_edesc *edesc;
	311	int ret;
	312
	313	if (unlikely(!key->n \|\| !key->d))
	314	return -EINVAL;
	315
	316	if (req->dst_len < key->n_sz) {
	317	req->dst_len = key->n_sz;
	318	dev_err(jrdev, "Output buffer length less than parameter n\n");
	319	return -EOVERFLOW;
	320	}
	321
	322	/* Allocate extended descriptor */
	323	edesc = rsa_edesc_alloc(req, DESC_RSA_PRIV_F1_LEN);
	324	if (IS_ERR(edesc))
	325	return PTR_ERR(edesc);
	326
	327	/* Set RSA Decrypt Protocol Data Block - Private Key Form #1 */
	328	ret = set_rsa_priv_f1_pdb(req, edesc);
	329	if (ret)
	330	goto init_fail;
	331
	332	/* Initialize Job Descriptor */
	333	init_rsa_priv_f1_desc(edesc->hw_desc, &edesc->pdb.priv_f1);
	334
	335	ret = caam_jr_enqueue(jrdev, edesc->hw_desc, rsa_priv_f1_done, req);
	336	if (!ret)
	337	return -EINPROGRESS;
	338
	339	rsa_priv_f1_unmap(jrdev, edesc, req);
	340
	341	init_fail:
	342	rsa_io_unmap(jrdev, edesc, req);
	343	kfree(edesc);
	344	return ret;
	345	}
	346
	347	static void caam_rsa_free_key(struct caam_rsa_key *key)
	348	{
	349	kzfree(key->d);
	350	kfree(key->e);
	351	kfree(key->n);
	352	key->d = NULL;
	353	key->e = NULL;
	354	key->n = NULL;
	355	key->d_sz = 0;
	356	key->e_sz = 0;
	357	key->n_sz = 0;
	358	}
	359
	360	/**
	361	* caam_read_raw_data - Read a raw byte stream as a positive integer.
	362	* The function skips buffer's leading zeros, copies the remained data
	363	* to a buffer allocated in the GFP_DMA \| GFP_KERNEL zone and returns
	364	* the address of the new buffer.
	365	*
	366	* @buf : The data to read
	367	* @nbytes: The amount of data to read
	368	*/
	369	static inline u8 caam_read_raw_data(const u8 buf, size_t *nbytes)
	370	{
	371	u8 *val;
	372
	373	while (!buf && nbytes) {
	374	buf++;
	375	(*nbytes)--;
	376	}
	377
	378	val = kzalloc(*nbytes, GFP_DMA \| GFP_KERNEL);
	379	if (!val)
	380	return NULL;
	381
	382	memcpy(val, buf, *nbytes);
	383
	384	return val;
	385	}
	386
	387	static int caam_rsa_check_key_length(unsigned int len)
	388	{
	389	if (len > 4096)
	390	return -EINVAL;
	391	return 0;
	392	}
	393
	394	static int caam_rsa_set_pub_key(struct crypto_akcipher tfm, const void key,
	395	unsigned int keylen)
	396	{
	397	struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
	398	struct rsa_key raw_key = {0};
	399	struct caam_rsa_key *rsa_key = &ctx->key;
	400	int ret;
	401
	402	/* Free the old RSA key if any */
	403	caam_rsa_free_key(rsa_key);
	404
	405	ret = rsa_parse_pub_key(&raw_key, key, keylen);
	406	if (ret)
	407	return ret;
	408
	409	/* Copy key in DMA zone */
	410	rsa_key->e = kzalloc(raw_key.e_sz, GFP_DMA \| GFP_KERNEL);
	411	if (!rsa_key->e)
	412	goto err;
	413
	414	/*
	415	* Skip leading zeros and copy the positive integer to a buffer
	416	* allocated in the GFP_DMA \| GFP_KERNEL zone. The decryption descriptor
	417	* expects a positive integer for the RSA modulus and uses its length as
	418	* decryption output length.
	419	*/
	420	rsa_key->n = caam_read_raw_data(raw_key.n, &raw_key.n_sz);
	421	if (!rsa_key->n)
	422	goto err;
	423
	424	if (caam_rsa_check_key_length(raw_key.n_sz << 3)) {
	425	caam_rsa_free_key(rsa_key);
	426	return -EINVAL;
	427	}
	428
	429	rsa_key->e_sz = raw_key.e_sz;
	430	rsa_key->n_sz = raw_key.n_sz;
	431

176 more lines aren’t shown here.

drivers/crypto/caam/caampkc.h+70−0

This diff is large: its 71 lines aren’t shown here.

drivers/crypto/caam/compat.h+3−0

This diff is large: its 12 lines aren’t shown here.

drivers/crypto/caam/ctrl.c+80−45

This diff is large: its 242 lines aren’t shown here.

drivers/crypto/caam/desc.h+6−5

This diff is large: its 33 lines aren’t shown here.

drivers/crypto/caam/desc_constr.h+36−15

This diff is large: its 124 lines aren’t shown here.

drivers/crypto/caam/jr.c+11−11

This diff is large: its 85 lines aren’t shown here.

drivers/crypto/caam/pdb.h+157−31

This diff is large: its 276 lines aren’t shown here.

drivers/crypto/caam/pkc_desc.c+36−0

This diff is large: its 37 lines aren’t shown here.

drivers/crypto/caam/regs.h+101−50

This diff is large: its 192 lines aren’t shown here.

drivers/crypto/caam/sg_sw_sec4.h+9−8

This diff is large: its 47 lines aren’t shown here.

drivers/crypto/ccp/ccp-crypto-aes-xts.c+20−23

This diff is large: its 95 lines aren’t shown here.

drivers/crypto/ccp/ccp-crypto.h+1−2

This diff is large: its 17 lines aren’t shown here.

drivers/crypto/marvell/cesa.c+107−35

This diff is large: its 211 lines aren’t shown here.

drivers/crypto/marvell/cesa.h+86−34

This diff is large: its 274 lines aren’t shown here.

drivers/crypto/marvell/cipher.c+93−64

This diff is large: its 336 lines aren’t shown here.

drivers/crypto/marvell/hash.c+71−79

This diff is large: its 369 lines aren’t shown here.

drivers/crypto/marvell/tdma.c+124−6

This diff is large: its 181 lines aren’t shown here.

drivers/crypto/mxs-dcp.c+21−26

This diff is large: its 113 lines aren’t shown here.

drivers/crypto/nx/nx.c+1−1

This diff is large: its 9 lines aren’t shown here.

drivers/crypto/omap-aes.c+20−16

This diff is large: its 85 lines aren’t shown here.

drivers/crypto/omap-des.c+10−4

This diff is large: its 39 lines aren’t shown here.

drivers/crypto/omap-sham.c+28−19

This diff is large: its 180 lines aren’t shown here.

drivers/crypto/picoxcell_crypto.c+31−29

This diff is large: its 117 lines aren’t shown here.

drivers/crypto/qat/Kconfig+2−1

This diff is large: its 15 lines aren’t shown here.

drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c+1−0

This diff is large: its 8 lines aren’t shown here.

drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c+1−0

This diff is large: its 8 lines aren’t shown here.

drivers/crypto/qat/qat_common/Makefile+0−11

This diff is large: its 22 lines aren’t shown here.

drivers/crypto/qat/qat_common/adf_accel_devices.h+1−0

This diff is large: its 8 lines aren’t shown here.

drivers/crypto/qat/qat_common/adf_aer.c+39−10

This diff is large: its 82 lines aren’t shown here.

drivers/crypto/qat/qat_common/adf_common_drv.h+2−0

This diff is large: its 9 lines aren’t shown here.

drivers/crypto/qat/qat_common/adf_sriov.c+1−1

This diff is large: its 9 lines aren’t shown here.

drivers/crypto/qat/qat_common/adf_vf_isr.c+1−1

This diff is large: its 9 lines aren’t shown here.

drivers/crypto/qat/qat_common/qat_algs.c+4−4

This diff is large: its 18 lines aren’t shown here.

drivers/crypto/qat/qat_common/qat_asym_algs.c+738−134

This diff is large: its 1195 lines aren’t shown here.

drivers/crypto/qat/qat_common/qat_rsaprivkey.asn1+0−11

This diff is large: its 12 lines aren’t shown here.

drivers/crypto/qat/qat_common/qat_rsapubkey.asn1+0−4

This diff is large: its 5 lines aren’t shown here.

drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c+1−0

This diff is large: its 8 lines aren’t shown here.

drivers/crypto/qce/ablkcipher.c+16−11

This diff is large: its 63 lines aren’t shown here.

drivers/crypto/qce/cipher.h+1−1

This diff is large: its 9 lines aren’t shown here.

drivers/crypto/s5p-sss.c+40−40

This diff is large: its 138 lines aren’t shown here.

drivers/crypto/sahara.c+50−62

This diff is large: its 232 lines aren’t shown here.

drivers/crypto/talitos.c+426−246

This diff is large: its 1001 lines aren’t shown here.

drivers/crypto/ux500/cryp/Makefile+3−3

This diff is large: its 13 lines aren’t shown here.

drivers/crypto/ux500/hash/Makefile+1−1

This diff is large: its 9 lines aren’t shown here.

drivers/crypto/vmx/.gitignore+2−0

This diff is large: its 3 lines aren’t shown here.

drivers/crypto/vmx/Makefile+1−1

This diff is large: its 7 lines aren’t shown here.

drivers/crypto/vmx/aes_xts.c+190−0

This diff is large: its 191 lines aren’t shown here.

drivers/crypto/vmx/aesp8-ppc.h+4−0

This diff is large: its 8 lines aren’t shown here.

drivers/crypto/vmx/aesp8-ppc.pl+1861−2

This diff is large: its 1888 lines aren’t shown here.

drivers/crypto/vmx/vmx.c+2−0

This diff is large: its 13 lines aren’t shown here.

drivers/usb/misc/Kconfig+6−5

This diff is large: its 18 lines aren’t shown here.

drivers/usb/misc/chaoskey.c+19−2

This diff is large: its 68 lines aren’t shown here.

include/asm-generic/io.h+67−4

This diff is large: its 127 lines aren’t shown here.

include/asm-generic/iomap.h+8−0

This diff is large: its 21 lines aren’t shown here.

include/crypto/aead.h+7−5

This diff is large: its 26 lines aren’t shown here.

include/crypto/algapi.h+4−0

This diff is large: its 20 lines aren’t shown here.

include/crypto/cryptd.h+5−0

This diff is large: its 26 lines aren’t shown here.

include/crypto/dh.h+29−0

This diff is large: its 30 lines aren’t shown here.

include/crypto/drbg.h+12−0

This diff is large: its 34 lines aren’t shown here.

include/crypto/ecdh.h+30−0

This diff is large: its 31 lines aren’t shown here.

include/crypto/internal/aead.h+21−0

This diff is large: its 28 lines aren’t shown here.

include/crypto/internal/geniv.h+1−1

This diff is large: its 9 lines aren’t shown here.

include/crypto/internal/hash.h+4−8

This diff is large: its 19 lines aren’t shown here.

include/crypto/internal/kpp.h+64−0

This diff is large: its 65 lines aren’t shown here.

include/crypto/internal/rsa.h+36−6

This diff is large: its 57 lines aren’t shown here.

include/crypto/internal/skcipher.h+85−37

This diff is large: its 179 lines aren’t shown here.

include/crypto/kpp.h+330−0

This diff is large: its 331 lines aren’t shown here.

include/crypto/mcryptd.h+4−4

This diff is large: its 26 lines aren’t shown here.

include/crypto/null.h+11−1

This diff is large: its 19 lines aren’t shown here.

include/crypto/scatterwalk.h+40−8

This diff is large: its 73 lines aren’t shown here.

include/crypto/sha3.h+29−0

This diff is large: its 30 lines aren’t shown here.

include/crypto/skcipher.h+131−76

This diff is large: its 238 lines aren’t shown here.

include/linux/crypto.h+7−24

This diff is large: its 57 lines aren’t shown here.

include/linux/mpi.h+1−2

This diff is large: its 10 lines aren’t shown here.

include/uapi/linux/cryptouser.h+5−0

This diff is large: its 18 lines aren’t shown here.

lib/digsig.c+11−5

This diff is large: its 42 lines aren’t shown here.

lib/mpi/mpicoder.c+81−168

This diff is large: its 369 lines aren’t shown here.

security/keys/big_key.c+18−12

This diff is large: its 82 lines aren’t shown here.

Merge branch 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code