History

crypto: x86/sm4 - add AES-NI/AVX2/x86_64 implementation

Like the implementation of AESNI/AVX, this patch adds an accelerated implementation of AESNI/AVX2. In terms of code implementation, by reusing AESNI/AVX mode-related codes, the amount of code is greatly reduced. From the benchmark data, it can be seen that when the block size is 1024, compared to AVX acceleration, the performance achieved by AVX2 has increased by about 70%, it is also 7.7 times of the pure software implementation of sm4-generic. The main algorithm implementation comes from SM4 AES-NI work by libgcrypt and Markku-Juhani O. Saarinen at: https://github.com/mjosaarinen/sm4ni This optimization supports the four modes of SM4, ECB, CBC, CFB, and CTR. Since CBC and CFB do not support multiple block parallel encryption, the optimization effect is not obvious. Benchmark on Intel i5-6200U 2.30GHz, performance data of three implementation methods, pure software sm4-generic, aesni/avx acceleration, and aesni/avx2 acceleration, the data comes from the 218 mode and 518 mode of tcrypt. The abscissas are blocks of different lengths. The data is tabulated and the unit is Mb/s: block-size | 16 64 128 256 1024 1420 4096 sm4-generic ECB enc | 60.94 70.41 72.27 73.02 73.87 73.58 73.59 ECB dec | 61.87 70.53 72.15 73.09 73.89 73.92 73.86 CBC enc | 56.71 66.31 68.05 69.84 70.02 70.12 70.24 CBC dec | 54.54 65.91 68.22 69.51 70.63 70.79 70.82 CFB enc | 57.21 67.24 69.10 70.25 70.73 70.52 71.42 CFB dec | 57.22 64.74 66.31 67.24 67.40 67.64 67.58 CTR enc | 59.47 68.64 69.91 71.02 71.86 71.61 71.95 CTR dec | 59.94 68.77 69.95 71.00 71.84 71.55 71.95 sm4-aesni-avx ECB enc | 44.95 177.35 292.06 316.98 339.48 322.27 330.59 ECB dec | 45.28 178.66 292.31 317.52 339.59 322.52 331.16 CBC enc | 57.75 67.68 69.72 70.60 71.48 71.63 71.74 CBC dec | 44.32 176.83 284.32 307.24 328.61 312.61 325.82 CFB enc | 57.81 67.64 69.63 70.55 71.40 71.35 71.70 CFB dec | 43.14 167.78 282.03 307.20 328.35 318.24 325.95 CTR enc | 42.35 163.32 279.11 302.93 320.86 310.56 317.93 CTR dec | 42.39 162.81 278.49 302.37 321.11 310.33 318.37 sm4-aesni-avx2 ECB enc | 45.19 177.41 292.42 316.12 339.90 322.53 330.54 ECB dec | 44.83 178.90 291.45 317.31 339.85 322.55 331.07 CBC enc | 57.66 67.62 69.73 70.55 71.58 71.66 71.77 CBC dec | 44.34 176.86 286.10 501.68 559.58 483.87 527.46 CFB enc | 57.43 67.60 69.61 70.52 71.43 71.28 71.65 CFB dec | 43.12 167.75 268.09 499.33 558.35 490.36 524.73 CTR enc | 42.42 163.39 256.17 493.95 552.45 481.58 517.19 CTR dec | 42.49 163.11 256.36 493.34 552.62 481.49 516.83

Tianjia Zhang committed 5 years ago

Commit 5b2efa21 parent de79d9a4 files · +691 −0Browse files

Unified · Split

4 files
  1. arch/x86/crypto/Makefile +3 −0
  2. arch/x86/crypto/sm4-aesni-avx2-asm_64.S +497 −0
  3. arch/x86/crypto/sm4_aesni_avx2_glue.c +169 −0
  4. crypto/Kconfig +22 −0
arch/x86/crypto/Makefile+3−0
@@ -91,6 +91,9 @@ obj-$(CONFIG_CRYPTO_CURVE25519_X86) += curve25519-x86_64.o · 90 unmodified lines
9191 obj-$(CONFIG_CRYPTO_SM4_AESNI_AVX_X86_64) += sm4-aesni-avx-x86_64.o
9292 sm4-aesni-avx-x86_64-y := sm4-aesni-avx-asm_64.o sm4_aesni_avx_glue.o
9393
94+obj-$(CONFIG_CRYPTO_SM4_AESNI_AVX2_X86_64) += sm4-aesni-avx2-x86_64.o
95+sm4-aesni-avx2-x86_64-y := sm4-aesni-avx2-asm_64.o sm4_aesni_avx2_glue.o
96+
9497 quiet_cmd_perlasm = PERLASM $@
9598 cmd_perlasm = $(PERL) $< > $@
9699 $(obj)/%.S: $(src)/%.pl FORCE
arch/x86/crypto/sm4-aesni-avx2-asm_64.S+497−0
@@ -0,0 +1,497 @@
1+// SPDX-License-Identifier: GPL-2.0-or-later
2+/*
3+ * SM4 Cipher Algorithm, AES-NI/AVX2 optimized.
4+ * as specified in
5+ * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html
6+ *
7+ * Copyright (C) 2018 Markku-Juhani O. Saarinen <mjos@iki.fi>
8+ * Copyright (C) 2020 Jussi Kivilinna <jussi.kivilinna@iki.fi>
9+ * Copyright (c) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
10+ */
11+
12+/* Based on SM4 AES-NI work by libgcrypt and Markku-Juhani O. Saarinen at:
13+ * https://github.com/mjosaarinen/sm4ni
14+ */
15+
16+#include <linux/linkage.h>
17+#include <asm/frame.h>
18+
19+#define rRIP (%rip)
20+
21+/* vector registers */
22+#define RX0 %ymm0
23+#define RX1 %ymm1
24+#define MASK_4BIT %ymm2
25+#define RTMP0 %ymm3
26+#define RTMP1 %ymm4
27+#define RTMP2 %ymm5
28+#define RTMP3 %ymm6
29+#define RTMP4 %ymm7
30+
31+#define RA0 %ymm8
32+#define RA1 %ymm9
33+#define RA2 %ymm10
34+#define RA3 %ymm11
35+
36+#define RB0 %ymm12
37+#define RB1 %ymm13
38+#define RB2 %ymm14
39+#define RB3 %ymm15
40+
41+#define RNOT %ymm0
42+#define RBSWAP %ymm1
43+
44+#define RX0x %xmm0
45+#define RX1x %xmm1
46+#define MASK_4BITx %xmm2
47+
48+#define RNOTx %xmm0
49+#define RBSWAPx %xmm1
50+
51+#define RTMP0x %xmm3
52+#define RTMP1x %xmm4
53+#define RTMP2x %xmm5
54+#define RTMP3x %xmm6
55+#define RTMP4x %xmm7
56+
57+
58+/* helper macros */
59+
60+/* Transpose four 32-bit words between 128-bit vector lanes. */
61+#define transpose_4x4(x0, x1, x2, x3, t1, t2) \
62+ vpunpckhdq x1, x0, t2; \
63+ vpunpckldq x1, x0, x0; \
64+ \
65+ vpunpckldq x3, x2, t1; \
66+ vpunpckhdq x3, x2, x2; \
67+ \
68+ vpunpckhqdq t1, x0, x1; \
69+ vpunpcklqdq t1, x0, x0; \
70+ \
71+ vpunpckhqdq x2, t2, x3; \
72+ vpunpcklqdq x2, t2, x2;
73+
74+/* post-SubByte transform. */
75+#define transform_pre(x, lo_t, hi_t, mask4bit, tmp0) \
76+ vpand x, mask4bit, tmp0; \
77+ vpandn x, mask4bit, x; \
78+ vpsrld $4, x, x; \
79+ \
80+ vpshufb tmp0, lo_t, tmp0; \
81+ vpshufb x, hi_t, x; \
82+ vpxor tmp0, x, x;
83+
84+/* post-SubByte transform. Note: x has been XOR'ed with mask4bit by
85+ * 'vaeslastenc' instruction. */
86+#define transform_post(x, lo_t, hi_t, mask4bit, tmp0) \
87+ vpandn mask4bit, x, tmp0; \
88+ vpsrld $4, x, x; \
89+ vpand x, mask4bit, x; \
90+ \
91+ vpshufb tmp0, lo_t, tmp0; \
92+ vpshufb x, hi_t, x; \
93+ vpxor tmp0, x, x;
94+
95+
96+.section .rodata.cst164, "aM", @progbits, 164
97+.align 16
98+
99+/*
100+ * Following four affine transform look-up tables are from work by
101+ * Markku-Juhani O. Saarinen, at https://github.com/mjosaarinen/sm4ni
102+ *
103+ * These allow exposing SM4 S-Box from AES SubByte.
104+ */
105+
106+/* pre-SubByte affine transform, from SM4 field to AES field. */
107+.Lpre_tf_lo_s:
108+ .quad 0x9197E2E474720701, 0xC7C1B4B222245157
109+.Lpre_tf_hi_s:
110+ .quad 0xE240AB09EB49A200, 0xF052B91BF95BB012
111+
112+/* post-SubByte affine transform, from AES field to SM4 field. */
113+.Lpost_tf_lo_s:
114+ .quad 0x5B67F2CEA19D0834, 0xEDD14478172BBE82
115+.Lpost_tf_hi_s:
116+ .quad 0xAE7201DD73AFDC00, 0x11CDBE62CC1063BF
117+
118+/* For isolating SubBytes from AESENCLAST, inverse shift row */
119+.Linv_shift_row:
120+ .byte 0x00, 0x0d, 0x0a, 0x07, 0x04, 0x01, 0x0e, 0x0b
121+ .byte 0x08, 0x05, 0x02, 0x0f, 0x0c, 0x09, 0x06, 0x03
122+
123+/* Inverse shift row + Rotate left by 8 bits on 32-bit words with vpshufb */
124+.Linv_shift_row_rol_8:
125+ .byte 0x07, 0x00, 0x0d, 0x0a, 0x0b, 0x04, 0x01, 0x0e
126+ .byte 0x0f, 0x08, 0x05, 0x02, 0x03, 0x0c, 0x09, 0x06
127+
128+/* Inverse shift row + Rotate left by 16 bits on 32-bit words with vpshufb */
129+.Linv_shift_row_rol_16:
130+ .byte 0x0a, 0x07, 0x00, 0x0d, 0x0e, 0x0b, 0x04, 0x01
131+ .byte 0x02, 0x0f, 0x08, 0x05, 0x06, 0x03, 0x0c, 0x09
132+
133+/* Inverse shift row + Rotate left by 24 bits on 32-bit words with vpshufb */
134+.Linv_shift_row_rol_24:
135+ .byte 0x0d, 0x0a, 0x07, 0x00, 0x01, 0x0e, 0x0b, 0x04
136+ .byte 0x05, 0x02, 0x0f, 0x08, 0x09, 0x06, 0x03, 0x0c
137+
138+/* For CTR-mode IV byteswap */
139+.Lbswap128_mask:
140+ .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
141+
142+/* For input word byte-swap */
143+.Lbswap32_mask:
144+ .byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12
145+
146+.align 4
147+/* 4-bit mask */
148+.L0f0f0f0f:
149+ .long 0x0f0f0f0f
150+
151+.text
152+.align 16
153+
154+.align 8
155+SYM_FUNC_START_LOCAL(__sm4_crypt_blk16)
156+ /* input:
157+ * %rdi: round key array, CTX
158+ * RA0, RA1, RA2, RA3, RB0, RB1, RB2, RB3: sixteen parallel
159+ * plaintext blocks
160+ * output:
161+ * RA0, RA1, RA2, RA3, RB0, RB1, RB2, RB3: sixteen parallel
162+ * ciphertext blocks
163+ */
164+ FRAME_BEGIN
165+
166+ vbroadcasti128 .Lbswap32_mask rRIP, RTMP2;
167+ vpshufb RTMP2, RA0, RA0;
168+ vpshufb RTMP2, RA1, RA1;
169+ vpshufb RTMP2, RA2, RA2;
170+ vpshufb RTMP2, RA3, RA3;
171+ vpshufb RTMP2, RB0, RB0;
172+ vpshufb RTMP2, RB1, RB1;
173+ vpshufb RTMP2, RB2, RB2;
174+ vpshufb RTMP2, RB3, RB3;
175+
176+ vpbroadcastd .L0f0f0f0f rRIP, MASK_4BIT;
177+ transpose_4x4(RA0, RA1, RA2, RA3, RTMP0, RTMP1);
178+ transpose_4x4(RB0, RB1, RB2, RB3, RTMP0, RTMP1);
179+
180+#define ROUND(round, s0, s1, s2, s3, r0, r1, r2, r3) \
181+ vpbroadcastd (4*(round))(%rdi), RX0; \
182+ vbroadcasti128 .Lpre_tf_lo_s rRIP, RTMP4; \
183+ vbroadcasti128 .Lpre_tf_hi_s rRIP, RTMP1; \
184+ vmovdqa RX0, RX1; \
185+ vpxor s1, RX0, RX0; \
186+ vpxor s2, RX0, RX0; \
187+ vpxor s3, RX0, RX0; /* s1 ^ s2 ^ s3 ^ rk */ \
188+ vbroadcasti128 .Lpost_tf_lo_s rRIP, RTMP2; \
189+ vbroadcasti128 .Lpost_tf_hi_s rRIP, RTMP3; \
190+ vpxor r1, RX1, RX1; \
191+ vpxor r2, RX1, RX1; \
192+ vpxor r3, RX1, RX1; /* r1 ^ r2 ^ r3 ^ rk */ \
193+ \
194+ /* sbox, non-linear part */ \
195+ transform_pre(RX0, RTMP4, RTMP1, MASK_4BIT, RTMP0); \
196+ transform_pre(RX1, RTMP4, RTMP1, MASK_4BIT, RTMP0); \
197+ vextracti128 $1, RX0, RTMP4x; \
198+ vextracti128 $1, RX1, RTMP0x; \
199+ vaesenclast MASK_4BITx, RX0x, RX0x; \
200+ vaesenclast MASK_4BITx, RTMP4x, RTMP4x; \
201+ vaesenclast MASK_4BITx, RX1x, RX1x; \
202+ vaesenclast MASK_4BITx, RTMP0x, RTMP0x; \
203+ vinserti128 $1, RTMP4x, RX0, RX0; \
204+ vbroadcasti128 .Linv_shift_row rRIP, RTMP4; \
205+ vinserti128 $1, RTMP0x, RX1, RX1; \
206+ transform_post(RX0, RTMP2, RTMP3, MASK_4BIT, RTMP0); \
207+ transform_post(RX1, RTMP2, RTMP3, MASK_4BIT, RTMP0); \
208+ \
209+ /* linear part */ \
210+ vpshufb RTMP4, RX0, RTMP0; \
211+ vpxor RTMP0, s0, s0; /* s0 ^ x */ \
212+ vpshufb RTMP4, RX1, RTMP2; \
213+ vbroadcasti128 .Linv_shift_row_rol_8 rRIP, RTMP4; \
214+ vpxor RTMP2, r0, r0; /* r0 ^ x */ \
215+ vpshufb RTMP4, RX0, RTMP1; \
216+ vpxor RTMP1, RTMP0, RTMP0; /* x ^ rol(x,8) */ \
217+ vpshufb RTMP4, RX1, RTMP3; \
218+ vbroadcasti128 .Linv_shift_row_rol_16 rRIP, RTMP4; \
219+ vpxor RTMP3, RTMP2, RTMP2; /* x ^ rol(x,8) */ \
220+ vpshufb RTMP4, RX0, RTMP1; \
221+ vpxor RTMP1, RTMP0, RTMP0; /* x ^ rol(x,8) ^ rol(x,16) */ \
222+ vpshufb RTMP4, RX1, RTMP3; \
223+ vbroadcasti128 .Linv_shift_row_rol_24 rRIP, RTMP4; \
224+ vpxor RTMP3, RTMP2, RTMP2; /* x ^ rol(x,8) ^ rol(x,16) */ \
225+ vpshufb RTMP4, RX0, RTMP1; \
226+ vpxor RTMP1, s0, s0; /* s0 ^ x ^ rol(x,24) */ \
227+ vpslld $2, RTMP0, RTMP1; \
228+ vpsrld $30, RTMP0, RTMP0; \
229+ vpxor RTMP0, s0, s0; \
230+ /* s0 ^ x ^ rol(x,2) ^ rol(x,10) ^ rol(x,18) ^ rol(x,24) */ \
231+ vpxor RTMP1, s0, s0; \
232+ vpshufb RTMP4, RX1, RTMP3; \
233+ vpxor RTMP3, r0, r0; /* r0 ^ x ^ rol(x,24) */ \
234+ vpslld $2, RTMP2, RTMP3; \
235+ vpsrld $30, RTMP2, RTMP2; \
236+ vpxor RTMP2, r0, r0; \
237+ /* r0 ^ x ^ rol(x,2) ^ rol(x,10) ^ rol(x,18) ^ rol(x,24) */ \
238+ vpxor RTMP3, r0, r0;
239+
240+ leaq (32*4)(%rdi), %rax;
241+.align 16
242+.Lroundloop_blk8:
243+ ROUND(0, RA0, RA1, RA2, RA3, RB0, RB1, RB2, RB3);
244+ ROUND(1, RA1, RA2, RA3, RA0, RB1, RB2, RB3, RB0);
245+ ROUND(2, RA2, RA3, RA0, RA1, RB2, RB3, RB0, RB1);
246+ ROUND(3, RA3, RA0, RA1, RA2, RB3, RB0, RB1, RB2);
247+ leaq (4*4)(%rdi), %rdi;
248+ cmpq %rax, %rdi;
249+ jne .Lroundloop_blk8;
250+
251+#undef ROUND
252+
253+ vbroadcasti128 .Lbswap128_mask rRIP, RTMP2;
254+
255+ transpose_4x4(RA0, RA1, RA2, RA3, RTMP0, RTMP1);
256+ transpose_4x4(RB0, RB1, RB2, RB3, RTMP0, RTMP1);
257+ vpshufb RTMP2, RA0, RA0;
258+ vpshufb RTMP2, RA1, RA1;
259+ vpshufb RTMP2, RA2, RA2;
260+ vpshufb RTMP2, RA3, RA3;
261+ vpshufb RTMP2, RB0, RB0;
262+ vpshufb RTMP2, RB1, RB1;
263+ vpshufb RTMP2, RB2, RB2;
264+ vpshufb RTMP2, RB3, RB3;
265+
266+ FRAME_END
267+ ret;
268+SYM_FUNC_END(__sm4_crypt_blk16)
269+
270+#define inc_le128(x, minus_one, tmp) \
271+ vpcmpeqq minus_one, x, tmp; \
272+ vpsubq minus_one, x, x; \
273+ vpslldq $8, tmp, tmp; \
274+ vpsubq tmp, x, x;
275+
276+/*
277+ * void sm4_aesni_avx2_ctr_enc_blk16(const u32 *rk, u8 *dst,
278+ * const u8 *src, u8 *iv)
279+ */
280+.align 8
281+SYM_FUNC_START(sm4_aesni_avx2_ctr_enc_blk16)
282+ /* input:
283+ * %rdi: round key array, CTX
284+ * %rsi: dst (16 blocks)
285+ * %rdx: src (16 blocks)
286+ * %rcx: iv (big endian, 128bit)
287+ */
288+ FRAME_BEGIN
289+
290+ movq 8(%rcx), %rax;
291+ bswapq %rax;
292+
293+ vzeroupper;
294+
295+ vbroadcasti128 .Lbswap128_mask rRIP, RTMP3;
296+ vpcmpeqd RNOT, RNOT, RNOT;
297+ vpsrldq $8, RNOT, RNOT; /* ab: -1:0 ; cd: -1:0 */
298+ vpaddq RNOT, RNOT, RTMP2; /* ab: -2:0 ; cd: -2:0 */
299+
300+ /* load IV and byteswap */
301+ vmovdqu (%rcx), RTMP4x;
302+ vpshufb RTMP3x, RTMP4x, RTMP4x;
303+ vmovdqa RTMP4x, RTMP0x;
304+ inc_le128(RTMP4x, RNOTx, RTMP1x);
305+ vinserti128 $1, RTMP4x, RTMP0, RTMP0;
306+ vpshufb RTMP3, RTMP0, RA0; /* +1 ; +0 */
307+
308+ /* check need for handling 64-bit overflow and carry */
309+ cmpq $(0xffffffffffffffff - 16), %rax;
310+ ja .Lhandle_ctr_carry;
311+
312+ /* construct IVs */
313+ vpsubq RTMP2, RTMP0, RTMP0; /* +3 ; +2 */
314+ vpshufb RTMP3, RTMP0, RA1;
315+ vpsubq RTMP2, RTMP0, RTMP0; /* +5 ; +4 */
316+ vpshufb RTMP3, RTMP0, RA2;
317+ vpsubq RTMP2, RTMP0, RTMP0; /* +7 ; +6 */
318+ vpshufb RTMP3, RTMP0, RA3;
319+ vpsubq RTMP2, RTMP0, RTMP0; /* +9 ; +8 */
320+ vpshufb RTMP3, RTMP0, RB0;
321+ vpsubq RTMP2, RTMP0, RTMP0; /* +11 ; +10 */
322+ vpshufb RTMP3, RTMP0, RB1;
323+ vpsubq RTMP2, RTMP0, RTMP0; /* +13 ; +12 */
324+ vpshufb RTMP3, RTMP0, RB2;
325+ vpsubq RTMP2, RTMP0, RTMP0; /* +15 ; +14 */
326+ vpshufb RTMP3, RTMP0, RB3;
327+ vpsubq RTMP2, RTMP0, RTMP0; /* +16 */
328+ vpshufb RTMP3x, RTMP0x, RTMP0x;
329+
330+ jmp .Lctr_carry_done;
331+
332+.Lhandle_ctr_carry:
333+ /* construct IVs */
334+ inc_le128(RTMP0, RNOT, RTMP1);
335+ inc_le128(RTMP0, RNOT, RTMP1);
336+ vpshufb RTMP3, RTMP0, RA1; /* +3 ; +2 */
337+ inc_le128(RTMP0, RNOT, RTMP1);
338+ inc_le128(RTMP0, RNOT, RTMP1);
339+ vpshufb RTMP3, RTMP0, RA2; /* +5 ; +4 */
340+ inc_le128(RTMP0, RNOT, RTMP1);
341+ inc_le128(RTMP0, RNOT, RTMP1);
342+ vpshufb RTMP3, RTMP0, RA3; /* +7 ; +6 */
343+ inc_le128(RTMP0, RNOT, RTMP1);
344+ inc_le128(RTMP0, RNOT, RTMP1);
345+ vpshufb RTMP3, RTMP0, RB0; /* +9 ; +8 */
346+ inc_le128(RTMP0, RNOT, RTMP1);
347+ inc_le128(RTMP0, RNOT, RTMP1);
348+ vpshufb RTMP3, RTMP0, RB1; /* +11 ; +10 */
349+ inc_le128(RTMP0, RNOT, RTMP1);
350+ inc_le128(RTMP0, RNOT, RTMP1);
351+ vpshufb RTMP3, RTMP0, RB2; /* +13 ; +12 */
352+ inc_le128(RTMP0, RNOT, RTMP1);
353+ inc_le128(RTMP0, RNOT, RTMP1);
354+ vpshufb RTMP3, RTMP0, RB3; /* +15 ; +14 */
355+ inc_le128(RTMP0, RNOT, RTMP1);
356+ vextracti128 $1, RTMP0, RTMP0x;
357+ vpshufb RTMP3x, RTMP0x, RTMP0x; /* +16 */
358+
359+.align 4
360+.Lctr_carry_done:
361+ /* store new IV */
362+ vmovdqu RTMP0x, (%rcx);
363+
364+ call __sm4_crypt_blk16;
365+
366+ vpxor (0 * 32)(%rdx), RA0, RA0;
367+ vpxor (1 * 32)(%rdx), RA1, RA1;
368+ vpxor (2 * 32)(%rdx), RA2, RA2;
369+ vpxor (3 * 32)(%rdx), RA3, RA3;
370+ vpxor (4 * 32)(%rdx), RB0, RB0;
371+ vpxor (5 * 32)(%rdx), RB1, RB1;
372+ vpxor (6 * 32)(%rdx), RB2, RB2;
373+ vpxor (7 * 32)(%rdx), RB3, RB3;
374+
375+ vmovdqu RA0, (0 * 32)(%rsi);
376+ vmovdqu RA1, (1 * 32)(%rsi);
377+ vmovdqu RA2, (2 * 32)(%rsi);
378+ vmovdqu RA3, (3 * 32)(%rsi);
379+ vmovdqu RB0, (4 * 32)(%rsi);
380+ vmovdqu RB1, (5 * 32)(%rsi);
381+ vmovdqu RB2, (6 * 32)(%rsi);
382+ vmovdqu RB3, (7 * 32)(%rsi);
383+
384+ vzeroall;
385+ FRAME_END
386+ ret;
387+SYM_FUNC_END(sm4_aesni_avx2_ctr_enc_blk16)
388+
389+/*
390+ * void sm4_aesni_avx2_cbc_dec_blk16(const u32 *rk, u8 *dst,
391+ * const u8 *src, u8 *iv)
392+ */
393+.align 8
394+SYM_FUNC_START(sm4_aesni_avx2_cbc_dec_blk16)
395+ /* input:
396+ * %rdi: round key array, CTX
397+ * %rsi: dst (16 blocks)
398+ * %rdx: src (16 blocks)
399+ * %rcx: iv
400+ */
401+ FRAME_BEGIN
402+
403+ vzeroupper;
404+
405+ vmovdqu (0 * 32)(%rdx), RA0;
406+ vmovdqu (1 * 32)(%rdx), RA1;
407+ vmovdqu (2 * 32)(%rdx), RA2;
408+ vmovdqu (3 * 32)(%rdx), RA3;
409+ vmovdqu (4 * 32)(%rdx), RB0;
410+ vmovdqu (5 * 32)(%rdx), RB1;
411+ vmovdqu (6 * 32)(%rdx), RB2;
412+ vmovdqu (7 * 32)(%rdx), RB3;
413+
414+ call __sm4_crypt_blk16;
415+
416+ vmovdqu (%rcx), RNOTx;
417+ vinserti128 $1, (%rdx), RNOT, RNOT;
418+ vpxor RNOT, RA0, RA0;
419+ vpxor (0 * 32 + 16)(%rdx), RA1, RA1;
420+ vpxor (1 * 32 + 16)(%rdx), RA2, RA2;
421+ vpxor (2 * 32 + 16)(%rdx), RA3, RA3;
422+ vpxor (3 * 32 + 16)(%rdx), RB0, RB0;
423+ vpxor (4 * 32 + 16)(%rdx), RB1, RB1;
424+ vpxor (5 * 32 + 16)(%rdx), RB2, RB2;
425+ vpxor (6 * 32 + 16)(%rdx), RB3, RB3;
426+ vmovdqu (7 * 32 + 16)(%rdx), RNOTx;
427+ vmovdqu RNOTx, (%rcx); /* store new IV */
428+
429+ vmovdqu RA0, (0 * 32)(%rsi);
430+ vmovdqu RA1, (1 * 32)(%rsi);
431+ vmovdqu RA2, (2 * 32)(%rsi);
432+ vmovdqu RA3, (3 * 32)(%rsi);
433+ vmovdqu RB0, (4 * 32)(%rsi);
434+ vmovdqu RB1, (5 * 32)(%rsi);
435+ vmovdqu RB2, (6 * 32)(%rsi);
436+ vmovdqu RB3, (7 * 32)(%rsi);
437+
438+ vzeroall;
439+ FRAME_END
440+ ret;
441+SYM_FUNC_END(sm4_aesni_avx2_cbc_dec_blk16)
442+
443+/*
444+ * void sm4_aesni_avx2_cfb_dec_blk16(const u32 *rk, u8 *dst,
445+ * const u8 *src, u8 *iv)
446+ */
447+.align 8
448+SYM_FUNC_START(sm4_aesni_avx2_cfb_dec_blk16)
449+ /* input:
450+ * %rdi: round key array, CTX
451+ * %rsi: dst (16 blocks)
452+ * %rdx: src (16 blocks)
453+ * %rcx: iv
454+ */
455+ FRAME_BEGIN
456+
457+ vzeroupper;
458+
459+ /* Load input */
460+ vmovdqu (%rcx), RNOTx;
461+ vinserti128 $1, (%rdx), RNOT, RA0;
462+ vmovdqu (0 * 32 + 16)(%rdx), RA1;
463+ vmovdqu (1 * 32 + 16)(%rdx), RA2;
464+ vmovdqu (2 * 32 + 16)(%rdx), RA3;
465+ vmovdqu (3 * 32 + 16)(%rdx), RB0;
466+ vmovdqu (4 * 32 + 16)(%rdx), RB1;
467+ vmovdqu (5 * 32 + 16)(%rdx), RB2;
468+ vmovdqu (6 * 32 + 16)(%rdx), RB3;
469+
470+ /* Update IV */
471+ vmovdqu (7 * 32 + 16)(%rdx), RNOTx;
472+ vmovdqu RNOTx, (%rcx);
473+
474+ call __sm4_crypt_blk16;
475+
476+ vpxor (0 * 32)(%rdx), RA0, RA0;
477+ vpxor (1 * 32)(%rdx), RA1, RA1;
478+ vpxor (2 * 32)(%rdx), RA2, RA2;
479+ vpxor (3 * 32)(%rdx), RA3, RA3;
480+ vpxor (4 * 32)(%rdx), RB0, RB0;
481+ vpxor (5 * 32)(%rdx), RB1, RB1;
482+ vpxor (6 * 32)(%rdx), RB2, RB2;
483+ vpxor (7 * 32)(%rdx), RB3, RB3;
484+
485+ vmovdqu RA0, (0 * 32)(%rsi);
486+ vmovdqu RA1, (1 * 32)(%rsi);
487+ vmovdqu RA2, (2 * 32)(%rsi);
488+ vmovdqu RA3, (3 * 32)(%rsi);
489+ vmovdqu RB0, (4 * 32)(%rsi);
490+ vmovdqu RB1, (5 * 32)(%rsi);
491+ vmovdqu RB2, (6 * 32)(%rsi);
492+ vmovdqu RB3, (7 * 32)(%rsi);
493+
494+ vzeroall;
495+ FRAME_END
496+ ret;
497+SYM_FUNC_END(sm4_aesni_avx2_cfb_dec_blk16)
arch/x86/crypto/sm4_aesni_avx2_glue.c+169−0
@@ -0,0 +1,169 @@
1+/* SPDX-License-Identifier: GPL-2.0-or-later */
2+/*
3+ * SM4 Cipher Algorithm, AES-NI/AVX2 optimized.
4+ * as specified in
5+ * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html
6+ *
7+ * Copyright (c) 2021, Alibaba Group.
8+ * Copyright (c) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
9+ */
10+
11+#include <linux/module.h>
12+#include <linux/crypto.h>
13+#include <linux/kernel.h>
14+#include <asm/simd.h>
15+#include <crypto/internal/simd.h>
16+#include <crypto/internal/skcipher.h>
17+#include <crypto/sm4.h>
18+#include "sm4-avx.h"
19+
20+#define SM4_CRYPT16_BLOCK_SIZE (SM4_BLOCK_SIZE * 16)
21+
22+asmlinkage void sm4_aesni_avx2_ctr_enc_blk16(const u32 *rk, u8 *dst,
23+ const u8 *src, u8 *iv);
24+asmlinkage void sm4_aesni_avx2_cbc_dec_blk16(const u32 *rk, u8 *dst,
25+ const u8 *src, u8 *iv);
26+asmlinkage void sm4_aesni_avx2_cfb_dec_blk16(const u32 *rk, u8 *dst,
27+ const u8 *src, u8 *iv);
28+
29+static int sm4_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
30+ unsigned int key_len)
31+{
32+ struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
33+
34+ return sm4_expandkey(ctx, key, key_len);
35+}
36+
37+static int cbc_decrypt(struct skcipher_request *req)
38+{
39+ return sm4_avx_cbc_decrypt(req, SM4_CRYPT16_BLOCK_SIZE,
40+ sm4_aesni_avx2_cbc_dec_blk16);
41+}
42+
43+
44+static int cfb_decrypt(struct skcipher_request *req)
45+{
46+ return sm4_avx_cfb_decrypt(req, SM4_CRYPT16_BLOCK_SIZE,
47+ sm4_aesni_avx2_cfb_dec_blk16);
48+}
49+
50+static int ctr_crypt(struct skcipher_request *req)
51+{
52+ return sm4_avx_ctr_crypt(req, SM4_CRYPT16_BLOCK_SIZE,
53+ sm4_aesni_avx2_ctr_enc_blk16);
54+}
55+
56+static struct skcipher_alg sm4_aesni_avx2_skciphers[] = {
57+ {
58+ .base = {
59+ .cra_name = "__ecb(sm4)",
60+ .cra_driver_name = "__ecb-sm4-aesni-avx2",
61+ .cra_priority = 500,
62+ .cra_flags = CRYPTO_ALG_INTERNAL,
63+ .cra_blocksize = SM4_BLOCK_SIZE,
64+ .cra_ctxsize = sizeof(struct sm4_ctx),
65+ .cra_module = THIS_MODULE,
66+ },
67+ .min_keysize = SM4_KEY_SIZE,
68+ .max_keysize = SM4_KEY_SIZE,
69+ .walksize = 16 * SM4_BLOCK_SIZE,
70+ .setkey = sm4_skcipher_setkey,
71+ .encrypt = sm4_avx_ecb_encrypt,
72+ .decrypt = sm4_avx_ecb_decrypt,
73+ }, {
74+ .base = {
75+ .cra_name = "__cbc(sm4)",
76+ .cra_driver_name = "__cbc-sm4-aesni-avx2",
77+ .cra_priority = 500,
78+ .cra_flags = CRYPTO_ALG_INTERNAL,
79+ .cra_blocksize = SM4_BLOCK_SIZE,
80+ .cra_ctxsize = sizeof(struct sm4_ctx),
81+ .cra_module = THIS_MODULE,
82+ },
83+ .min_keysize = SM4_KEY_SIZE,
84+ .max_keysize = SM4_KEY_SIZE,
85+ .ivsize = SM4_BLOCK_SIZE,
86+ .walksize = 16 * SM4_BLOCK_SIZE,
87+ .setkey = sm4_skcipher_setkey,
88+ .encrypt = sm4_cbc_encrypt,
89+ .decrypt = cbc_decrypt,
90+ }, {
91+ .base = {
92+ .cra_name = "__cfb(sm4)",
93+ .cra_driver_name = "__cfb-sm4-aesni-avx2",
94+ .cra_priority = 500,
95+ .cra_flags = CRYPTO_ALG_INTERNAL,
96+ .cra_blocksize = 1,
97+ .cra_ctxsize = sizeof(struct sm4_ctx),
98+ .cra_module = THIS_MODULE,
99+ },
100+ .min_keysize = SM4_KEY_SIZE,
101+ .max_keysize = SM4_KEY_SIZE,
102+ .ivsize = SM4_BLOCK_SIZE,
103+ .chunksize = SM4_BLOCK_SIZE,
104+ .walksize = 16 * SM4_BLOCK_SIZE,
105+ .setkey = sm4_skcipher_setkey,
106+ .encrypt = sm4_cfb_encrypt,
107+ .decrypt = cfb_decrypt,
108+ }, {
109+ .base = {
110+ .cra_name = "__ctr(sm4)",
111+ .cra_driver_name = "__ctr-sm4-aesni-avx2",
112+ .cra_priority = 500,
113+ .cra_flags = CRYPTO_ALG_INTERNAL,
114+ .cra_blocksize = 1,
115+ .cra_ctxsize = sizeof(struct sm4_ctx),
116+ .cra_module = THIS_MODULE,
117+ },
118+ .min_keysize = SM4_KEY_SIZE,
119+ .max_keysize = SM4_KEY_SIZE,
120+ .ivsize = SM4_BLOCK_SIZE,
121+ .chunksize = SM4_BLOCK_SIZE,
122+ .walksize = 16 * SM4_BLOCK_SIZE,
123+ .setkey = sm4_skcipher_setkey,
124+ .encrypt = ctr_crypt,
125+ .decrypt = ctr_crypt,
126+ }
127+};
128+
129+static struct simd_skcipher_alg *
130+simd_sm4_aesni_avx2_skciphers[ARRAY_SIZE(sm4_aesni_avx2_skciphers)];
131+
132+static int __init sm4_init(void)
133+{
134+ const char *feature_name;
135+
136+ if (!boot_cpu_has(X86_FEATURE_AVX) ||
137+ !boot_cpu_has(X86_FEATURE_AVX2) ||
138+ !boot_cpu_has(X86_FEATURE_AES) ||
139+ !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
140+ pr_info("AVX2 or AES-NI instructions are not detected.\n");
141+ return -ENODEV;
142+ }
143+
144+ if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
145+ &feature_name)) {
146+ pr_info("CPU feature '%s' is not supported.\n", feature_name);
147+ return -ENODEV;
148+ }
149+
150+ return simd_register_skciphers_compat(sm4_aesni_avx2_skciphers,
151+ ARRAY_SIZE(sm4_aesni_avx2_skciphers),
152+ simd_sm4_aesni_avx2_skciphers);
153+}
154+
155+static void __exit sm4_exit(void)
156+{
157+ simd_unregister_skciphers(sm4_aesni_avx2_skciphers,
158+ ARRAY_SIZE(sm4_aesni_avx2_skciphers),
159+ simd_sm4_aesni_avx2_skciphers);
160+}
161+
162+module_init(sm4_init);
163+module_exit(sm4_exit);
164+
165+MODULE_LICENSE("GPL v2");
166+MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
167+MODULE_DESCRIPTION("SM4 Cipher Algorithm, AES-NI/AVX2 optimized");
168+MODULE_ALIAS_CRYPTO("sm4");
169+MODULE_ALIAS_CRYPTO("sm4-aesni-avx2");
crypto/Kconfig+22−0
@@ -1591,6 +1591,28 @@ config CRYPTO_SM4_AESNI_AVX_X86_64 · 1590 unmodified lines
15911591
15921592 If unsure, say N.
15931593
1594+config CRYPTO_SM4_AESNI_AVX2_X86_64
1595+ tristate "SM4 cipher algorithm (x86_64/AES-NI/AVX2)"
1596+ depends on X86 && 64BIT
1597+ select CRYPTO_SKCIPHER
1598+ select CRYPTO_SIMD
1599+ select CRYPTO_ALGAPI
1600+ select CRYPTO_LIB_SM4
1601+ select CRYPTO_SM4_AESNI_AVX_X86_64
1602+ help
1603+ SM4 cipher algorithms (OSCCA GB/T 32907-2016) (x86_64/AES-NI/AVX2).
1604+
1605+ SM4 (GBT.32907-2016) is a cryptographic standard issued by the
1606+ Organization of State Commercial Administration of China (OSCCA)
1607+ as an authorized cryptographic algorithms for the use within China.
1608+
1609+ This is SM4 optimized implementation using AES-NI/AVX2/x86_64
1610+ instruction set for block cipher. Through two affine transforms,
1611+ we can use the AES S-Box to simulate the SM4 S-Box to achieve the
1612+ effect of instruction acceleration.
1613+
1614+ If unsure, say N.
1615+
15941616 config CRYPTO_TEA
15951617 tristate "TEA, XTEA and XETA cipher algorithms"
15961618 depends on CRYPTO_USER_API_ENABLE_OBSOLETE

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t