History

crypto: x86/sm4 - add AES-NI/AVX2/x86_64 implementation

Like the implementation of AESNI/AVX, this patch adds an accelerated implementation of AESNI/AVX2. In terms of code implementation, by reusing AESNI/AVX mode-related codes, the amount of code is greatly reduced. From the benchmark data, it can be seen that when the block size is 1024, compared to AVX acceleration, the performance achieved by AVX2 has increased by about 70%, it is also 7.7 times of the pure software implementation of sm4-generic. The main algorithm implementation comes from SM4 AES-NI work by libgcrypt and Markku-Juhani O. Saarinen at: https://github.com/mjosaarinen/sm4ni This optimization supports the four modes of SM4, ECB, CBC, CFB, and CTR. Since CBC and CFB do not support multiple block parallel encryption, the optimization effect is not obvious. Benchmark on Intel i5-6200U 2.30GHz, performance data of three implementation methods, pure software sm4-generic, aesni/avx acceleration, and aesni/avx2 acceleration, the data comes from the 218 mode and 518 mode of tcrypt. The abscissas are blocks of different lengths. The data is tabulated and the unit is Mb/s: block-size | 16 64 128 256 1024 1420 4096 sm4-generic ECB enc | 60.94 70.41 72.27 73.02 73.87 73.58 73.59 ECB dec | 61.87 70.53 72.15 73.09 73.89 73.92 73.86 CBC enc | 56.71 66.31 68.05 69.84 70.02 70.12 70.24 CBC dec | 54.54 65.91 68.22 69.51 70.63 70.79 70.82 CFB enc | 57.21 67.24 69.10 70.25 70.73 70.52 71.42 CFB dec | 57.22 64.74 66.31 67.24 67.40 67.64 67.58 CTR enc | 59.47 68.64 69.91 71.02 71.86 71.61 71.95 CTR dec | 59.94 68.77 69.95 71.00 71.84 71.55 71.95 sm4-aesni-avx ECB enc | 44.95 177.35 292.06 316.98 339.48 322.27 330.59 ECB dec | 45.28 178.66 292.31 317.52 339.59 322.52 331.16 CBC enc | 57.75 67.68 69.72 70.60 71.48 71.63 71.74 CBC dec | 44.32 176.83 284.32 307.24 328.61 312.61 325.82 CFB enc | 57.81 67.64 69.63 70.55 71.40 71.35 71.70 CFB dec | 43.14 167.78 282.03 307.20 328.35 318.24 325.95 CTR enc | 42.35 163.32 279.11 302.93 320.86 310.56 317.93 CTR dec | 42.39 162.81 278.49 302.37 321.11 310.33 318.37 sm4-aesni-avx2 ECB enc | 45.19 177.41 292.42 316.12 339.90 322.53 330.54 ECB dec | 44.83 178.90 291.45 317.31 339.85 322.55 331.07 CBC enc | 57.66 67.62 69.73 70.55 71.58 71.66 71.77 CBC dec | 44.34 176.86 286.10 501.68 559.58 483.87 527.46 CFB enc | 57.43 67.60 69.61 70.52 71.43 71.28 71.65 CFB dec | 43.12 167.75 268.09 499.33 558.35 490.36 524.73 CTR enc | 42.42 163.39 256.17 493.95 552.45 481.58 517.19 CTR dec | 42.49 163.11 256.36 493.34 552.62 481.49 516.83

Tianjia Zhang committed 5 years ago

Commit 5b2efa21 parent de79d9a4 files · +691 −0Browse files

Unified · Split

4 files
  1. arch/x86/crypto/Makefile +3 −0
  2. arch/x86/crypto/sm4-aesni-avx2-asm_64.S +497 −0
  3. arch/x86/crypto/sm4_aesni_avx2_glue.c +169 −0
  4. crypto/Kconfig +22 −0
arch/x86/crypto/Makefile+3−0
@@ -91,6 +91,9 @@ obj-$(CONFIG_CRYPTO_CURVE25519_X86) += curve25519-x86_64.o · 90 unmodified lines
91obj-$(CONFIG_CRYPTO_SM4_AESNI_AVX_X86_64) += sm4-aesni-avx-x86_64.o91obj-$(CONFIG_CRYPTO_SM4_AESNI_AVX_X86_64) += sm4-aesni-avx-x86_64.o
92sm4-aesni-avx-x86_64-y := sm4-aesni-avx-asm_64.o sm4_aesni_avx_glue.o92sm4-aesni-avx-x86_64-y := sm4-aesni-avx-asm_64.o sm4_aesni_avx_glue.o
93 93
94obj-$(CONFIG_CRYPTO_SM4_AESNI_AVX2_X86_64) += sm4-aesni-avx2-x86_64.o
95sm4-aesni-avx2-x86_64-y := sm4-aesni-avx2-asm_64.o sm4_aesni_avx2_glue.o
96
94quiet_cmd_perlasm = PERLASM $@97quiet_cmd_perlasm = PERLASM $@
95 cmd_perlasm = $(PERL) $< > $@98 cmd_perlasm = $(PERL) $< > $@
96$(obj)/%.S: $(src)/%.pl FORCE99$(obj)/%.S: $(src)/%.pl FORCE
arch/x86/crypto/sm4-aesni-avx2-asm_64.S+497−0
@@ -0,0 +1,497 @@
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * SM4 Cipher Algorithm, AES-NI/AVX2 optimized.
4 * as specified in
5 * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html
6 *
7 * Copyright (C) 2018 Markku-Juhani O. Saarinen <mjos@iki.fi>
8 * Copyright (C) 2020 Jussi Kivilinna <jussi.kivilinna@iki.fi>
9 * Copyright (c) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
10 */
11
12/* Based on SM4 AES-NI work by libgcrypt and Markku-Juhani O. Saarinen at:
13 * https://github.com/mjosaarinen/sm4ni
14 */
15
16#include <linux/linkage.h>
17#include <asm/frame.h>
18
19#define rRIP (%rip)
20
21/* vector registers */
22#define RX0 %ymm0
23#define RX1 %ymm1
24#define MASK_4BIT %ymm2
25#define RTMP0 %ymm3
26#define RTMP1 %ymm4
27#define RTMP2 %ymm5
28#define RTMP3 %ymm6
29#define RTMP4 %ymm7
30
31#define RA0 %ymm8
32#define RA1 %ymm9
33#define RA2 %ymm10
34#define RA3 %ymm11
35
36#define RB0 %ymm12
37#define RB1 %ymm13
38#define RB2 %ymm14
39#define RB3 %ymm15
40
41#define RNOT %ymm0
42#define RBSWAP %ymm1
43
44#define RX0x %xmm0
45#define RX1x %xmm1
46#define MASK_4BITx %xmm2
47
48#define RNOTx %xmm0
49#define RBSWAPx %xmm1
50
51#define RTMP0x %xmm3
52#define RTMP1x %xmm4
53#define RTMP2x %xmm5
54#define RTMP3x %xmm6
55#define RTMP4x %xmm7
56
57
58/* helper macros */
59
60/* Transpose four 32-bit words between 128-bit vector lanes. */
61#define transpose_4x4(x0, x1, x2, x3, t1, t2) \
62 vpunpckhdq x1, x0, t2; \
63 vpunpckldq x1, x0, x0; \
64 \
65 vpunpckldq x3, x2, t1; \
66 vpunpckhdq x3, x2, x2; \
67 \
68 vpunpckhqdq t1, x0, x1; \
69 vpunpcklqdq t1, x0, x0; \
70 \
71 vpunpckhqdq x2, t2, x3; \
72 vpunpcklqdq x2, t2, x2;
73
74/* post-SubByte transform. */
75#define transform_pre(x, lo_t, hi_t, mask4bit, tmp0) \
76 vpand x, mask4bit, tmp0; \
77 vpandn x, mask4bit, x; \
78 vpsrld $4, x, x; \
79 \
80 vpshufb tmp0, lo_t, tmp0; \
81 vpshufb x, hi_t, x; \
82 vpxor tmp0, x, x;
83
84/* post-SubByte transform. Note: x has been XOR'ed with mask4bit by
85 * 'vaeslastenc' instruction. */
86#define transform_post(x, lo_t, hi_t, mask4bit, tmp0) \
87 vpandn mask4bit, x, tmp0; \
88 vpsrld $4, x, x; \
89 vpand x, mask4bit, x; \
90 \
91 vpshufb tmp0, lo_t, tmp0; \
92 vpshufb x, hi_t, x; \
93 vpxor tmp0, x, x;
94
95
96.section .rodata.cst164, "aM", @progbits, 164
97.align 16
98
99/*
100 * Following four affine transform look-up tables are from work by
101 * Markku-Juhani O. Saarinen, at https://github.com/mjosaarinen/sm4ni
102 *
103 * These allow exposing SM4 S-Box from AES SubByte.
104 */
105
106/* pre-SubByte affine transform, from SM4 field to AES field. */
107.Lpre_tf_lo_s:
108 .quad 0x9197E2E474720701, 0xC7C1B4B222245157
109.Lpre_tf_hi_s:
110 .quad 0xE240AB09EB49A200, 0xF052B91BF95BB012
111
112/* post-SubByte affine transform, from AES field to SM4 field. */
113.Lpost_tf_lo_s:
114 .quad 0x5B67F2CEA19D0834, 0xEDD14478172BBE82
115.Lpost_tf_hi_s:
116 .quad 0xAE7201DD73AFDC00, 0x11CDBE62CC1063BF
117
118/* For isolating SubBytes from AESENCLAST, inverse shift row */
119.Linv_shift_row:
120 .byte 0x00, 0x0d, 0x0a, 0x07, 0x04, 0x01, 0x0e, 0x0b
121 .byte 0x08, 0x05, 0x02, 0x0f, 0x0c, 0x09, 0x06, 0x03
122
123/* Inverse shift row + Rotate left by 8 bits on 32-bit words with vpshufb */
124.Linv_shift_row_rol_8:
125 .byte 0x07, 0x00, 0x0d, 0x0a, 0x0b, 0x04, 0x01, 0x0e
126 .byte 0x0f, 0x08, 0x05, 0x02, 0x03, 0x0c, 0x09, 0x06
127
128/* Inverse shift row + Rotate left by 16 bits on 32-bit words with vpshufb */
129.Linv_shift_row_rol_16:
130 .byte 0x0a, 0x07, 0x00, 0x0d, 0x0e, 0x0b, 0x04, 0x01
131 .byte 0x02, 0x0f, 0x08, 0x05, 0x06, 0x03, 0x0c, 0x09
132
133/* Inverse shift row + Rotate left by 24 bits on 32-bit words with vpshufb */
134.Linv_shift_row_rol_24:
135 .byte 0x0d, 0x0a, 0x07, 0x00, 0x01, 0x0e, 0x0b, 0x04
136 .byte 0x05, 0x02, 0x0f, 0x08, 0x09, 0x06, 0x03, 0x0c
137
138/* For CTR-mode IV byteswap */
139.Lbswap128_mask:
140 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
141
142/* For input word byte-swap */
143.Lbswap32_mask:
144 .byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12
145
146.align 4
147/* 4-bit mask */
148.L0f0f0f0f:
149 .long 0x0f0f0f0f
150
151.text
152.align 16
153
154.align 8
155SYM_FUNC_START_LOCAL(__sm4_crypt_blk16)
156 /* input:
157 * %rdi: round key array, CTX
158 * RA0, RA1, RA2, RA3, RB0, RB1, RB2, RB3: sixteen parallel
159 * plaintext blocks
160 * output:
161 * RA0, RA1, RA2, RA3, RB0, RB1, RB2, RB3: sixteen parallel
162 * ciphertext blocks
163 */
164 FRAME_BEGIN
165
166 vbroadcasti128 .Lbswap32_mask rRIP, RTMP2;
167 vpshufb RTMP2, RA0, RA0;
168 vpshufb RTMP2, RA1, RA1;
169 vpshufb RTMP2, RA2, RA2;
170 vpshufb RTMP2, RA3, RA3;
171 vpshufb RTMP2, RB0, RB0;
172 vpshufb RTMP2, RB1, RB1;
173 vpshufb RTMP2, RB2, RB2;
174 vpshufb RTMP2, RB3, RB3;
175
176 vpbroadcastd .L0f0f0f0f rRIP, MASK_4BIT;
177 transpose_4x4(RA0, RA1, RA2, RA3, RTMP0, RTMP1);
178 transpose_4x4(RB0, RB1, RB2, RB3, RTMP0, RTMP1);
179
180#define ROUND(round, s0, s1, s2, s3, r0, r1, r2, r3) \
181 vpbroadcastd (4*(round))(%rdi), RX0; \
182 vbroadcasti128 .Lpre_tf_lo_s rRIP, RTMP4; \
183 vbroadcasti128 .Lpre_tf_hi_s rRIP, RTMP1; \
184 vmovdqa RX0, RX1; \
185 vpxor s1, RX0, RX0; \
186 vpxor s2, RX0, RX0; \
187 vpxor s3, RX0, RX0; /* s1 ^ s2 ^ s3 ^ rk */ \
188 vbroadcasti128 .Lpost_tf_lo_s rRIP, RTMP2; \
189 vbroadcasti128 .Lpost_tf_hi_s rRIP, RTMP3; \
190 vpxor r1, RX1, RX1; \
191 vpxor r2, RX1, RX1; \
192 vpxor r3, RX1, RX1; /* r1 ^ r2 ^ r3 ^ rk */ \
193 \
194 /* sbox, non-linear part */ \
195 transform_pre(RX0, RTMP4, RTMP1, MASK_4BIT, RTMP0); \
196 transform_pre(RX1, RTMP4, RTMP1, MASK_4BIT, RTMP0); \
197 vextracti128 $1, RX0, RTMP4x; \
198 vextracti128 $1, RX1, RTMP0x; \
199 vaesenclast MASK_4BITx, RX0x, RX0x; \
200 vaesenclast MASK_4BITx, RTMP4x, RTMP4x; \
201 vaesenclast MASK_4BITx, RX1x, RX1x; \
202 vaesenclast MASK_4BITx, RTMP0x, RTMP0x; \
203 vinserti128 $1, RTMP4x, RX0, RX0; \
204 vbroadcasti128 .Linv_shift_row rRIP, RTMP4; \
205 vinserti128 $1, RTMP0x, RX1, RX1; \
206 transform_post(RX0, RTMP2, RTMP3, MASK_4BIT, RTMP0); \
207 transform_post(RX1, RTMP2, RTMP3, MASK_4BIT, RTMP0); \
208 \
209 /* linear part */ \
210 vpshufb RTMP4, RX0, RTMP0; \
211 vpxor RTMP0, s0, s0; /* s0 ^ x */ \
212 vpshufb RTMP4, RX1, RTMP2; \
213 vbroadcasti128 .Linv_shift_row_rol_8 rRIP, RTMP4; \
214 vpxor RTMP2, r0, r0; /* r0 ^ x */ \
215 vpshufb RTMP4, RX0, RTMP1; \
216 vpxor RTMP1, RTMP0, RTMP0; /* x ^ rol(x,8) */ \
217 vpshufb RTMP4, RX1, RTMP3; \
218 vbroadcasti128 .Linv_shift_row_rol_16 rRIP, RTMP4; \
219 vpxor RTMP3, RTMP2, RTMP2; /* x ^ rol(x,8) */ \
220 vpshufb RTMP4, RX0, RTMP1; \
221 vpxor RTMP1, RTMP0, RTMP0; /* x ^ rol(x,8) ^ rol(x,16) */ \
222 vpshufb RTMP4, RX1, RTMP3; \
223 vbroadcasti128 .Linv_shift_row_rol_24 rRIP, RTMP4; \
224 vpxor RTMP3, RTMP2, RTMP2; /* x ^ rol(x,8) ^ rol(x,16) */ \
225 vpshufb RTMP4, RX0, RTMP1; \
226 vpxor RTMP1, s0, s0; /* s0 ^ x ^ rol(x,24) */ \
227 vpslld $2, RTMP0, RTMP1; \
228 vpsrld $30, RTMP0, RTMP0; \
229 vpxor RTMP0, s0, s0; \
230 /* s0 ^ x ^ rol(x,2) ^ rol(x,10) ^ rol(x,18) ^ rol(x,24) */ \
231 vpxor RTMP1, s0, s0; \
232 vpshufb RTMP4, RX1, RTMP3; \
233 vpxor RTMP3, r0, r0; /* r0 ^ x ^ rol(x,24) */ \
234 vpslld $2, RTMP2, RTMP3; \
235 vpsrld $30, RTMP2, RTMP2; \
236 vpxor RTMP2, r0, r0; \
237 /* r0 ^ x ^ rol(x,2) ^ rol(x,10) ^ rol(x,18) ^ rol(x,24) */ \
238 vpxor RTMP3, r0, r0;
239
240 leaq (32*4)(%rdi), %rax;
241.align 16
242.Lroundloop_blk8:
243 ROUND(0, RA0, RA1, RA2, RA3, RB0, RB1, RB2, RB3);
244 ROUND(1, RA1, RA2, RA3, RA0, RB1, RB2, RB3, RB0);
245 ROUND(2, RA2, RA3, RA0, RA1, RB2, RB3, RB0, RB1);
246 ROUND(3, RA3, RA0, RA1, RA2, RB3, RB0, RB1, RB2);
247 leaq (4*4)(%rdi), %rdi;
248 cmpq %rax, %rdi;
249 jne .Lroundloop_blk8;
250
251#undef ROUND
252
253 vbroadcasti128 .Lbswap128_mask rRIP, RTMP2;
254
255 transpose_4x4(RA0, RA1, RA2, RA3, RTMP0, RTMP1);
256 transpose_4x4(RB0, RB1, RB2, RB3, RTMP0, RTMP1);
257 vpshufb RTMP2, RA0, RA0;
258 vpshufb RTMP2, RA1, RA1;
259 vpshufb RTMP2, RA2, RA2;
260 vpshufb RTMP2, RA3, RA3;
261 vpshufb RTMP2, RB0, RB0;
262 vpshufb RTMP2, RB1, RB1;
263 vpshufb RTMP2, RB2, RB2;
264 vpshufb RTMP2, RB3, RB3;
265
266 FRAME_END
267 ret;
268SYM_FUNC_END(__sm4_crypt_blk16)
269
270#define inc_le128(x, minus_one, tmp) \
271 vpcmpeqq minus_one, x, tmp; \
272 vpsubq minus_one, x, x; \
273 vpslldq $8, tmp, tmp; \
274 vpsubq tmp, x, x;
275
276/*
277 * void sm4_aesni_avx2_ctr_enc_blk16(const u32 *rk, u8 *dst,
278 * const u8 *src, u8 *iv)
279 */
280.align 8
281SYM_FUNC_START(sm4_aesni_avx2_ctr_enc_blk16)
282 /* input:
283 * %rdi: round key array, CTX
284 * %rsi: dst (16 blocks)
285 * %rdx: src (16 blocks)
286 * %rcx: iv (big endian, 128bit)
287 */
288 FRAME_BEGIN
289
290 movq 8(%rcx), %rax;
291 bswapq %rax;
292
293 vzeroupper;
294
295 vbroadcasti128 .Lbswap128_mask rRIP, RTMP3;
296 vpcmpeqd RNOT, RNOT, RNOT;
297 vpsrldq $8, RNOT, RNOT; /* ab: -1:0 ; cd: -1:0 */
298 vpaddq RNOT, RNOT, RTMP2; /* ab: -2:0 ; cd: -2:0 */
299
300 /* load IV and byteswap */
301 vmovdqu (%rcx), RTMP4x;
302 vpshufb RTMP3x, RTMP4x, RTMP4x;
303 vmovdqa RTMP4x, RTMP0x;
304 inc_le128(RTMP4x, RNOTx, RTMP1x);
305 vinserti128 $1, RTMP4x, RTMP0, RTMP0;
306 vpshufb RTMP3, RTMP0, RA0; /* +1 ; +0 */
307
308 /* check need for handling 64-bit overflow and carry */
309 cmpq $(0xffffffffffffffff - 16), %rax;
310 ja .Lhandle_ctr_carry;
311
312 /* construct IVs */
313 vpsubq RTMP2, RTMP0, RTMP0; /* +3 ; +2 */
314 vpshufb RTMP3, RTMP0, RA1;
315 vpsubq RTMP2, RTMP0, RTMP0; /* +5 ; +4 */
316 vpshufb RTMP3, RTMP0, RA2;
317 vpsubq RTMP2, RTMP0, RTMP0; /* +7 ; +6 */
318 vpshufb RTMP3, RTMP0, RA3;
319 vpsubq RTMP2, RTMP0, RTMP0; /* +9 ; +8 */
320 vpshufb RTMP3, RTMP0, RB0;
321 vpsubq RTMP2, RTMP0, RTMP0; /* +11 ; +10 */
322 vpshufb RTMP3, RTMP0, RB1;
323 vpsubq RTMP2, RTMP0, RTMP0; /* +13 ; +12 */
324 vpshufb RTMP3, RTMP0, RB2;
325 vpsubq RTMP2, RTMP0, RTMP0; /* +15 ; +14 */
326 vpshufb RTMP3, RTMP0, RB3;
327 vpsubq RTMP2, RTMP0, RTMP0; /* +16 */
328 vpshufb RTMP3x, RTMP0x, RTMP0x;
329
330 jmp .Lctr_carry_done;
331
332.Lhandle_ctr_carry:
333 /* construct IVs */
334 inc_le128(RTMP0, RNOT, RTMP1);
335 inc_le128(RTMP0, RNOT, RTMP1);
336 vpshufb RTMP3, RTMP0, RA1; /* +3 ; +2 */
337 inc_le128(RTMP0, RNOT, RTMP1);
338 inc_le128(RTMP0, RNOT, RTMP1);
339 vpshufb RTMP3, RTMP0, RA2; /* +5 ; +4 */
340 inc_le128(RTMP0, RNOT, RTMP1);
341 inc_le128(RTMP0, RNOT, RTMP1);
342 vpshufb RTMP3, RTMP0, RA3; /* +7 ; +6 */
343 inc_le128(RTMP0, RNOT, RTMP1);
344 inc_le128(RTMP0, RNOT, RTMP1);
345 vpshufb RTMP3, RTMP0, RB0; /* +9 ; +8 */
346 inc_le128(RTMP0, RNOT, RTMP1);
347 inc_le128(RTMP0, RNOT, RTMP1);
348 vpshufb RTMP3, RTMP0, RB1; /* +11 ; +10 */
349 inc_le128(RTMP0, RNOT, RTMP1);
350 inc_le128(RTMP0, RNOT, RTMP1);
351 vpshufb RTMP3, RTMP0, RB2; /* +13 ; +12 */
352 inc_le128(RTMP0, RNOT, RTMP1);
353 inc_le128(RTMP0, RNOT, RTMP1);
354 vpshufb RTMP3, RTMP0, RB3; /* +15 ; +14 */
355 inc_le128(RTMP0, RNOT, RTMP1);
356 vextracti128 $1, RTMP0, RTMP0x;
357 vpshufb RTMP3x, RTMP0x, RTMP0x; /* +16 */
358
359.align 4
360.Lctr_carry_done:
361 /* store new IV */
362 vmovdqu RTMP0x, (%rcx);
363
364 call __sm4_crypt_blk16;
365
366 vpxor (0 * 32)(%rdx), RA0, RA0;
367 vpxor (1 * 32)(%rdx), RA1, RA1;
368 vpxor (2 * 32)(%rdx), RA2, RA2;
369 vpxor (3 * 32)(%rdx), RA3, RA3;
370 vpxor (4 * 32)(%rdx), RB0, RB0;
371 vpxor (5 * 32)(%rdx), RB1, RB1;
372 vpxor (6 * 32)(%rdx), RB2, RB2;
373 vpxor (7 * 32)(%rdx), RB3, RB3;
374
375 vmovdqu RA0, (0 * 32)(%rsi);
376 vmovdqu RA1, (1 * 32)(%rsi);
377 vmovdqu RA2, (2 * 32)(%rsi);
378 vmovdqu RA3, (3 * 32)(%rsi);
379 vmovdqu RB0, (4 * 32)(%rsi);
380 vmovdqu RB1, (5 * 32)(%rsi);
381 vmovdqu RB2, (6 * 32)(%rsi);
382 vmovdqu RB3, (7 * 32)(%rsi);
383
384 vzeroall;
385 FRAME_END
386 ret;
387SYM_FUNC_END(sm4_aesni_avx2_ctr_enc_blk16)
388
389/*
390 * void sm4_aesni_avx2_cbc_dec_blk16(const u32 *rk, u8 *dst,
391 * const u8 *src, u8 *iv)
392 */
393.align 8
394SYM_FUNC_START(sm4_aesni_avx2_cbc_dec_blk16)
395 /* input:
396 * %rdi: round key array, CTX
397 * %rsi: dst (16 blocks)
398 * %rdx: src (16 blocks)
399 * %rcx: iv
400 */
401 FRAME_BEGIN
402
403 vzeroupper;
404
405 vmovdqu (0 * 32)(%rdx), RA0;
406 vmovdqu (1 * 32)(%rdx), RA1;
407 vmovdqu (2 * 32)(%rdx), RA2;
408 vmovdqu (3 * 32)(%rdx), RA3;
409 vmovdqu (4 * 32)(%rdx), RB0;
410 vmovdqu (5 * 32)(%rdx), RB1;
411 vmovdqu (6 * 32)(%rdx), RB2;
412 vmovdqu (7 * 32)(%rdx), RB3;
413
414 call __sm4_crypt_blk16;
415
416 vmovdqu (%rcx), RNOTx;
417 vinserti128 $1, (%rdx), RNOT, RNOT;
418 vpxor RNOT, RA0, RA0;
419 vpxor (0 * 32 + 16)(%rdx), RA1, RA1;
420 vpxor (1 * 32 + 16)(%rdx), RA2, RA2;
421 vpxor (2 * 32 + 16)(%rdx), RA3, RA3;
422 vpxor (3 * 32 + 16)(%rdx), RB0, RB0;
423 vpxor (4 * 32 + 16)(%rdx), RB1, RB1;
424 vpxor (5 * 32 + 16)(%rdx), RB2, RB2;
425 vpxor (6 * 32 + 16)(%rdx), RB3, RB3;
426 vmovdqu (7 * 32 + 16)(%rdx), RNOTx;
427 vmovdqu RNOTx, (%rcx); /* store new IV */
428
429 vmovdqu RA0, (0 * 32)(%rsi);
430 vmovdqu RA1, (1 * 32)(%rsi);
431 vmovdqu RA2, (2 * 32)(%rsi);
432 vmovdqu RA3, (3 * 32)(%rsi);
433 vmovdqu RB0, (4 * 32)(%rsi);
434 vmovdqu RB1, (5 * 32)(%rsi);
435 vmovdqu RB2, (6 * 32)(%rsi);
436 vmovdqu RB3, (7 * 32)(%rsi);
437
438 vzeroall;
439 FRAME_END
440 ret;
441SYM_FUNC_END(sm4_aesni_avx2_cbc_dec_blk16)
442
443/*
444 * void sm4_aesni_avx2_cfb_dec_blk16(const u32 *rk, u8 *dst,
445 * const u8 *src, u8 *iv)
446 */
447.align 8
448SYM_FUNC_START(sm4_aesni_avx2_cfb_dec_blk16)
449 /* input:
450 * %rdi: round key array, CTX
451 * %rsi: dst (16 blocks)
452 * %rdx: src (16 blocks)
453 * %rcx: iv
454 */
455 FRAME_BEGIN
456
457 vzeroupper;
458
459 /* Load input */
460 vmovdqu (%rcx), RNOTx;
461 vinserti128 $1, (%rdx), RNOT, RA0;
462 vmovdqu (0 * 32 + 16)(%rdx), RA1;
463 vmovdqu (1 * 32 + 16)(%rdx), RA2;
464 vmovdqu (2 * 32 + 16)(%rdx), RA3;
465 vmovdqu (3 * 32 + 16)(%rdx), RB0;
466 vmovdqu (4 * 32 + 16)(%rdx), RB1;
467 vmovdqu (5 * 32 + 16)(%rdx), RB2;
468 vmovdqu (6 * 32 + 16)(%rdx), RB3;
469
470 /* Update IV */
471 vmovdqu (7 * 32 + 16)(%rdx), RNOTx;
472 vmovdqu RNOTx, (%rcx);
473
474 call __sm4_crypt_blk16;
475
476 vpxor (0 * 32)(%rdx), RA0, RA0;
477 vpxor (1 * 32)(%rdx), RA1, RA1;
478 vpxor (2 * 32)(%rdx), RA2, RA2;
479 vpxor (3 * 32)(%rdx), RA3, RA3;
480 vpxor (4 * 32)(%rdx), RB0, RB0;
481 vpxor (5 * 32)(%rdx), RB1, RB1;
482 vpxor (6 * 32)(%rdx), RB2, RB2;
483 vpxor (7 * 32)(%rdx), RB3, RB3;
484
485 vmovdqu RA0, (0 * 32)(%rsi);
486 vmovdqu RA1, (1 * 32)(%rsi);
487 vmovdqu RA2, (2 * 32)(%rsi);
488 vmovdqu RA3, (3 * 32)(%rsi);
489 vmovdqu RB0, (4 * 32)(%rsi);
490 vmovdqu RB1, (5 * 32)(%rsi);
491 vmovdqu RB2, (6 * 32)(%rsi);
492 vmovdqu RB3, (7 * 32)(%rsi);
493
494 vzeroall;
495 FRAME_END
496 ret;
497SYM_FUNC_END(sm4_aesni_avx2_cfb_dec_blk16)
arch/x86/crypto/sm4_aesni_avx2_glue.c+169−0
@@ -0,0 +1,169 @@
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * SM4 Cipher Algorithm, AES-NI/AVX2 optimized.
4 * as specified in
5 * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html
6 *
7 * Copyright (c) 2021, Alibaba Group.
8 * Copyright (c) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
9 */
10
11#include <linux/module.h>
12#include <linux/crypto.h>
13#include <linux/kernel.h>
14#include <asm/simd.h>
15#include <crypto/internal/simd.h>
16#include <crypto/internal/skcipher.h>
17#include <crypto/sm4.h>
18#include "sm4-avx.h"
19
20#define SM4_CRYPT16_BLOCK_SIZE (SM4_BLOCK_SIZE * 16)
21
22asmlinkage void sm4_aesni_avx2_ctr_enc_blk16(const u32 *rk, u8 *dst,
23 const u8 *src, u8 *iv);
24asmlinkage void sm4_aesni_avx2_cbc_dec_blk16(const u32 *rk, u8 *dst,
25 const u8 *src, u8 *iv);
26asmlinkage void sm4_aesni_avx2_cfb_dec_blk16(const u32 *rk, u8 *dst,
27 const u8 *src, u8 *iv);
28
29static int sm4_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
30 unsigned int key_len)
31{
32 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
33
34 return sm4_expandkey(ctx, key, key_len);
35}
36
37static int cbc_decrypt(struct skcipher_request *req)
38{
39 return sm4_avx_cbc_decrypt(req, SM4_CRYPT16_BLOCK_SIZE,
40 sm4_aesni_avx2_cbc_dec_blk16);
41}
42
43
44static int cfb_decrypt(struct skcipher_request *req)
45{
46 return sm4_avx_cfb_decrypt(req, SM4_CRYPT16_BLOCK_SIZE,
47 sm4_aesni_avx2_cfb_dec_blk16);
48}
49
50static int ctr_crypt(struct skcipher_request *req)
51{
52 return sm4_avx_ctr_crypt(req, SM4_CRYPT16_BLOCK_SIZE,
53 sm4_aesni_avx2_ctr_enc_blk16);
54}
55
56static struct skcipher_alg sm4_aesni_avx2_skciphers[] = {
57 {
58 .base = {
59 .cra_name = "__ecb(sm4)",
60 .cra_driver_name = "__ecb-sm4-aesni-avx2",
61 .cra_priority = 500,
62 .cra_flags = CRYPTO_ALG_INTERNAL,
63 .cra_blocksize = SM4_BLOCK_SIZE,
64 .cra_ctxsize = sizeof(struct sm4_ctx),
65 .cra_module = THIS_MODULE,
66 },
67 .min_keysize = SM4_KEY_SIZE,
68 .max_keysize = SM4_KEY_SIZE,
69 .walksize = 16 * SM4_BLOCK_SIZE,
70 .setkey = sm4_skcipher_setkey,
71 .encrypt = sm4_avx_ecb_encrypt,
72 .decrypt = sm4_avx_ecb_decrypt,
73 }, {
74 .base = {
75 .cra_name = "__cbc(sm4)",
76 .cra_driver_name = "__cbc-sm4-aesni-avx2",
77 .cra_priority = 500,
78 .cra_flags = CRYPTO_ALG_INTERNAL,
79 .cra_blocksize = SM4_BLOCK_SIZE,
80 .cra_ctxsize = sizeof(struct sm4_ctx),
81 .cra_module = THIS_MODULE,
82 },
83 .min_keysize = SM4_KEY_SIZE,
84 .max_keysize = SM4_KEY_SIZE,
85 .ivsize = SM4_BLOCK_SIZE,
86 .walksize = 16 * SM4_BLOCK_SIZE,
87 .setkey = sm4_skcipher_setkey,
88 .encrypt = sm4_cbc_encrypt,
89 .decrypt = cbc_decrypt,
90 }, {
91 .base = {
92 .cra_name = "__cfb(sm4)",
93 .cra_driver_name = "__cfb-sm4-aesni-avx2",
94 .cra_priority = 500,
95 .cra_flags = CRYPTO_ALG_INTERNAL,
96 .cra_blocksize = 1,
97 .cra_ctxsize = sizeof(struct sm4_ctx),
98 .cra_module = THIS_MODULE,
99 },
100 .min_keysize = SM4_KEY_SIZE,
101 .max_keysize = SM4_KEY_SIZE,
102 .ivsize = SM4_BLOCK_SIZE,
103 .chunksize = SM4_BLOCK_SIZE,
104 .walksize = 16 * SM4_BLOCK_SIZE,
105 .setkey = sm4_skcipher_setkey,
106 .encrypt = sm4_cfb_encrypt,
107 .decrypt = cfb_decrypt,
108 }, {
109 .base = {
110 .cra_name = "__ctr(sm4)",
111 .cra_driver_name = "__ctr-sm4-aesni-avx2",
112 .cra_priority = 500,
113 .cra_flags = CRYPTO_ALG_INTERNAL,
114 .cra_blocksize = 1,
115 .cra_ctxsize = sizeof(struct sm4_ctx),
116 .cra_module = THIS_MODULE,
117 },
118 .min_keysize = SM4_KEY_SIZE,
119 .max_keysize = SM4_KEY_SIZE,
120 .ivsize = SM4_BLOCK_SIZE,
121 .chunksize = SM4_BLOCK_SIZE,
122 .walksize = 16 * SM4_BLOCK_SIZE,
123 .setkey = sm4_skcipher_setkey,
124 .encrypt = ctr_crypt,
125 .decrypt = ctr_crypt,
126 }
127};
128
129static struct simd_skcipher_alg *
130simd_sm4_aesni_avx2_skciphers[ARRAY_SIZE(sm4_aesni_avx2_skciphers)];
131
132static int __init sm4_init(void)
133{
134 const char *feature_name;
135
136 if (!boot_cpu_has(X86_FEATURE_AVX) ||
137 !boot_cpu_has(X86_FEATURE_AVX2) ||
138 !boot_cpu_has(X86_FEATURE_AES) ||
139 !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
140 pr_info("AVX2 or AES-NI instructions are not detected.\n");
141 return -ENODEV;
142 }
143
144 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
145 &feature_name)) {
146 pr_info("CPU feature '%s' is not supported.\n", feature_name);
147 return -ENODEV;
148 }
149
150 return simd_register_skciphers_compat(sm4_aesni_avx2_skciphers,
151 ARRAY_SIZE(sm4_aesni_avx2_skciphers),
152 simd_sm4_aesni_avx2_skciphers);
153}
154
155static void __exit sm4_exit(void)
156{
157 simd_unregister_skciphers(sm4_aesni_avx2_skciphers,
158 ARRAY_SIZE(sm4_aesni_avx2_skciphers),
159 simd_sm4_aesni_avx2_skciphers);
160}
161
162module_init(sm4_init);
163module_exit(sm4_exit);
164
165MODULE_LICENSE("GPL v2");
166MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
167MODULE_DESCRIPTION("SM4 Cipher Algorithm, AES-NI/AVX2 optimized");
168MODULE_ALIAS_CRYPTO("sm4");
169MODULE_ALIAS_CRYPTO("sm4-aesni-avx2");
crypto/Kconfig+22−0
@@ -1591,6 +1591,28 @@ config CRYPTO_SM4_AESNI_AVX_X86_64 · 1590 unmodified lines
1591 1591
1592 If unsure, say N.1592 If unsure, say N.
1593 1593
1594config CRYPTO_SM4_AESNI_AVX2_X86_64
1595 tristate "SM4 cipher algorithm (x86_64/AES-NI/AVX2)"
1596 depends on X86 && 64BIT
1597 select CRYPTO_SKCIPHER
1598 select CRYPTO_SIMD
1599 select CRYPTO_ALGAPI
1600 select CRYPTO_LIB_SM4
1601 select CRYPTO_SM4_AESNI_AVX_X86_64
1602 help
1603 SM4 cipher algorithms (OSCCA GB/T 32907-2016) (x86_64/AES-NI/AVX2).
1604
1605 SM4 (GBT.32907-2016) is a cryptographic standard issued by the
1606 Organization of State Commercial Administration of China (OSCCA)
1607 as an authorized cryptographic algorithms for the use within China.
1608
1609 This is SM4 optimized implementation using AES-NI/AVX2/x86_64
1610 instruction set for block cipher. Through two affine transforms,
1611 we can use the AES S-Box to simulate the SM4 S-Box to achieve the
1612 effect of instruction acceleration.
1613
1614 If unsure, say N.
1615
1594config CRYPTO_TEA1616config CRYPTO_TEA
1595 tristate "TEA, XTEA and XETA cipher algorithms"1617 tristate "TEA, XTEA and XETA cipher algorithms"
1596 depends on CRYPTO_USER_API_ENABLE_OBSOLETE1618 depends on CRYPTO_USER_API_ENABLE_OBSOLETE

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t