?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
arm64: dts: Revert "specify console via command line"
[linux/fpc-iii.git] / arch / x86 / crypto / aesni-intel_glue.c
blobbbbebbd35b5df6e0ea48a6d7f2b3a5a26b4d1815
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Support for Intel AES-NI instructions. This file contains glue
4 * code, the real AES implementation is in intel-aes_asm.S.
5 *
6 * Copyright (C) 2008, Intel Corp.
7 * Author: Huang Ying <ying.huang@intel.com>
8 *
9 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
10 * interface for 64-bit kernels.
11 * Authors: Adrian Hoban <adrian.hoban@intel.com>
12 * Gabriele Paoloni <gabriele.paoloni@intel.com>
13 * Tadeusz Struk (tadeusz.struk@intel.com)
14 * Aidan O'Mahony (aidan.o.mahony@intel.com)
15 * Copyright (c) 2010, Intel Corporation.
16 */
17
18 #include <linux/hardirq.h>
19 #include <linux/types.h>
20 #include <linux/module.h>
21 #include <linux/err.h>
22 #include <crypto/algapi.h>
23 #include <crypto/aes.h>
24 #include <crypto/ctr.h>
25 #include <crypto/b128ops.h>
26 #include <crypto/gcm.h>
27 #include <crypto/xts.h>
28 #include <asm/cpu_device_id.h>
29 #include <asm/simd.h>
30 #include <crypto/scatterwalk.h>
31 #include <crypto/internal/aead.h>
32 #include <crypto/internal/simd.h>
33 #include <crypto/internal/skcipher.h>
34 #include <linux/workqueue.h>
35 #include <linux/spinlock.h>
36 #ifdef CONFIG_X86_64
37 #include <asm/crypto/glue_helper.h>
38 #endif
39
40
41 #define AESNI_ALIGN 16
42 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
43 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE - 1))
44 #define RFC4106_HASH_SUBKEY_SIZE 16
45 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
46 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
47 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
48
49 /* This data is stored at the end of the crypto_tfm struct.
50 * It's a type of per "session" data storage location.
51 * This needs to be 16 byte aligned.
52 */
53 struct aesni_rfc4106_gcm_ctx {
54 u8 hash_subkey[16] AESNI_ALIGN_ATTR;
55 struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
56 u8 nonce[4];
57 };
58
59 struct generic_gcmaes_ctx {
60 u8 hash_subkey[16] AESNI_ALIGN_ATTR;
61 struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
62 };
63
64 struct aesni_xts_ctx {
65 u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
66 u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
67 };
68
69 #define GCM_BLOCK_LEN 16
70
71 struct gcm_context_data {
72 /* init, update and finalize context data */
73 u8 aad_hash[GCM_BLOCK_LEN];
74 u64 aad_length;
75 u64 in_length;
76 u8 partial_block_enc_key[GCM_BLOCK_LEN];
77 u8 orig_IV[GCM_BLOCK_LEN];
78 u8 current_counter[GCM_BLOCK_LEN];
79 u64 partial_block_len;
80 u64 unused;
81 u8 hash_keys[GCM_BLOCK_LEN * 16];
82 };
83
84 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
85 unsigned int key_len);
86 asmlinkage void aesni_enc(const void *ctx, u8 *out, const u8 *in);
87 asmlinkage void aesni_dec(const void *ctx, u8 *out, const u8 *in);
88 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
89 const u8 *in, unsigned int len);
90 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
91 const u8 *in, unsigned int len);
92 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
93 const u8 *in, unsigned int len, u8 *iv);
94 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
95 const u8 *in, unsigned int len, u8 *iv);
96
97 #define AVX_GEN2_OPTSIZE 640
98 #define AVX_GEN4_OPTSIZE 4096
99
100 #ifdef CONFIG_X86_64
101
102 static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out,
103 const u8 *in, unsigned int len, u8 *iv);
104 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
105 const u8 *in, unsigned int len, u8 *iv);
106
107 asmlinkage void aesni_xts_crypt8(const struct crypto_aes_ctx *ctx, u8 *out,
108 const u8 *in, bool enc, le128 *iv);
109
110 /* asmlinkage void aesni_gcm_enc()
111 * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
112 * struct gcm_context_data. May be uninitialized.
113 * u8 *out, Ciphertext output. Encrypt in-place is allowed.
114 * const u8 *in, Plaintext input
115 * unsigned long plaintext_len, Length of data in bytes for encryption.
116 * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
117 * 16-byte aligned pointer.
118 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
119 * const u8 *aad, Additional Authentication Data (AAD)
120 * unsigned long aad_len, Length of AAD in bytes.
121 * u8 *auth_tag, Authenticated Tag output.
122 * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
123 * Valid values are 16 (most likely), 12 or 8.
124 */
125 asmlinkage void aesni_gcm_enc(void *ctx,
126 struct gcm_context_data *gdata, u8 *out,
127 const u8 *in, unsigned long plaintext_len, u8 *iv,
128 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
129 u8 *auth_tag, unsigned long auth_tag_len);
130
131 /* asmlinkage void aesni_gcm_dec()
132 * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
133 * struct gcm_context_data. May be uninitialized.
134 * u8 *out, Plaintext output. Decrypt in-place is allowed.
135 * const u8 *in, Ciphertext input
136 * unsigned long ciphertext_len, Length of data in bytes for decryption.
137 * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
138 * 16-byte aligned pointer.
139 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
140 * const u8 *aad, Additional Authentication Data (AAD)
141 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
142 * to be 8 or 12 bytes
143 * u8 *auth_tag, Authenticated Tag output.
144 * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
145 * Valid values are 16 (most likely), 12 or 8.
146 */
147 asmlinkage void aesni_gcm_dec(void *ctx,
148 struct gcm_context_data *gdata, u8 *out,
149 const u8 *in, unsigned long ciphertext_len, u8 *iv,
150 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
151 u8 *auth_tag, unsigned long auth_tag_len);
152
153 /* Scatter / Gather routines, with args similar to above */
154 asmlinkage void aesni_gcm_init(void *ctx,
155 struct gcm_context_data *gdata,
156 u8 *iv,
157 u8 *hash_subkey, const u8 *aad,
158 unsigned long aad_len);
159 asmlinkage void aesni_gcm_enc_update(void *ctx,
160 struct gcm_context_data *gdata, u8 *out,
161 const u8 *in, unsigned long plaintext_len);
162 asmlinkage void aesni_gcm_dec_update(void *ctx,
163 struct gcm_context_data *gdata, u8 *out,
164 const u8 *in,
165 unsigned long ciphertext_len);
166 asmlinkage void aesni_gcm_finalize(void *ctx,
167 struct gcm_context_data *gdata,
168 u8 *auth_tag, unsigned long auth_tag_len);
169
170 static const struct aesni_gcm_tfm_s {
171 void (*init)(void *ctx, struct gcm_context_data *gdata, u8 *iv,
172 u8 *hash_subkey, const u8 *aad, unsigned long aad_len);
173 void (*enc_update)(void *ctx, struct gcm_context_data *gdata, u8 *out,
174 const u8 *in, unsigned long plaintext_len);
175 void (*dec_update)(void *ctx, struct gcm_context_data *gdata, u8 *out,
176 const u8 *in, unsigned long ciphertext_len);
177 void (*finalize)(void *ctx, struct gcm_context_data *gdata,
178 u8 *auth_tag, unsigned long auth_tag_len);
179 } *aesni_gcm_tfm;
180
181 static const struct aesni_gcm_tfm_s aesni_gcm_tfm_sse = {
182 .init = &aesni_gcm_init,
183 .enc_update = &aesni_gcm_enc_update,
184 .dec_update = &aesni_gcm_dec_update,
185 .finalize = &aesni_gcm_finalize,
186 };
187
188 #ifdef CONFIG_AS_AVX
189 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,
190 void *keys, u8 *out, unsigned int num_bytes);
191 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
192 void *keys, u8 *out, unsigned int num_bytes);
193 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
194 void *keys, u8 *out, unsigned int num_bytes);
195 /*
196 * asmlinkage void aesni_gcm_init_avx_gen2()
197 * gcm_data *my_ctx_data, context data
198 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
199 */
200 asmlinkage void aesni_gcm_init_avx_gen2(void *my_ctx_data,
201 struct gcm_context_data *gdata,
202 u8 *iv,
203 u8 *hash_subkey,
204 const u8 *aad,
205 unsigned long aad_len);
206
207 asmlinkage void aesni_gcm_enc_update_avx_gen2(void *ctx,
208 struct gcm_context_data *gdata, u8 *out,
209 const u8 *in, unsigned long plaintext_len);
210 asmlinkage void aesni_gcm_dec_update_avx_gen2(void *ctx,
211 struct gcm_context_data *gdata, u8 *out,
212 const u8 *in,
213 unsigned long ciphertext_len);
214 asmlinkage void aesni_gcm_finalize_avx_gen2(void *ctx,
215 struct gcm_context_data *gdata,
216 u8 *auth_tag, unsigned long auth_tag_len);
217
218 asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx,
219 struct gcm_context_data *gdata, u8 *out,
220 const u8 *in, unsigned long plaintext_len, u8 *iv,
221 const u8 *aad, unsigned long aad_len,
222 u8 *auth_tag, unsigned long auth_tag_len);
223
224 asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx,
225 struct gcm_context_data *gdata, u8 *out,
226 const u8 *in, unsigned long ciphertext_len, u8 *iv,
227 const u8 *aad, unsigned long aad_len,
228 u8 *auth_tag, unsigned long auth_tag_len);
229
230 static const struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen2 = {
231 .init = &aesni_gcm_init_avx_gen2,
232 .enc_update = &aesni_gcm_enc_update_avx_gen2,
233 .dec_update = &aesni_gcm_dec_update_avx_gen2,
234 .finalize = &aesni_gcm_finalize_avx_gen2,
235 };
236
237 #endif
238
239 #ifdef CONFIG_AS_AVX2
240 /*
241 * asmlinkage void aesni_gcm_init_avx_gen4()
242 * gcm_data *my_ctx_data, context data
243 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
244 */
245 asmlinkage void aesni_gcm_init_avx_gen4(void *my_ctx_data,
246 struct gcm_context_data *gdata,
247 u8 *iv,
248 u8 *hash_subkey,
249 const u8 *aad,
250 unsigned long aad_len);
251
252 asmlinkage void aesni_gcm_enc_update_avx_gen4(void *ctx,
253 struct gcm_context_data *gdata, u8 *out,
254 const u8 *in, unsigned long plaintext_len);
255 asmlinkage void aesni_gcm_dec_update_avx_gen4(void *ctx,
256 struct gcm_context_data *gdata, u8 *out,
257 const u8 *in,
258 unsigned long ciphertext_len);
259 asmlinkage void aesni_gcm_finalize_avx_gen4(void *ctx,
260 struct gcm_context_data *gdata,
261 u8 *auth_tag, unsigned long auth_tag_len);
262
263 asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx,
264 struct gcm_context_data *gdata, u8 *out,
265 const u8 *in, unsigned long plaintext_len, u8 *iv,
266 const u8 *aad, unsigned long aad_len,
267 u8 *auth_tag, unsigned long auth_tag_len);
268
269 asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx,
270 struct gcm_context_data *gdata, u8 *out,
271 const u8 *in, unsigned long ciphertext_len, u8 *iv,
272 const u8 *aad, unsigned long aad_len,
273 u8 *auth_tag, unsigned long auth_tag_len);
274
275 static const struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen4 = {
276 .init = &aesni_gcm_init_avx_gen4,
277 .enc_update = &aesni_gcm_enc_update_avx_gen4,
278 .dec_update = &aesni_gcm_dec_update_avx_gen4,
279 .finalize = &aesni_gcm_finalize_avx_gen4,
280 };
281
282 #endif
283
284 static inline struct
285 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
286 {
287 unsigned long align = AESNI_ALIGN;
288
289 if (align <= crypto_tfm_ctx_alignment())
290 align = 1;
291 return PTR_ALIGN(crypto_aead_ctx(tfm), align);
292 }
293
294 static inline struct
295 generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm)
296 {
297 unsigned long align = AESNI_ALIGN;
298
299 if (align <= crypto_tfm_ctx_alignment())
300 align = 1;
301 return PTR_ALIGN(crypto_aead_ctx(tfm), align);
302 }
303 #endif
304
305 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
306 {
307 unsigned long addr = (unsigned long)raw_ctx;
308 unsigned long align = AESNI_ALIGN;
309
310 if (align <= crypto_tfm_ctx_alignment())
311 align = 1;
312 return (struct crypto_aes_ctx *)ALIGN(addr, align);
313 }
314
315 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
316 const u8 *in_key, unsigned int key_len)
317 {
318 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
319 int err;
320
321 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
322 key_len != AES_KEYSIZE_256)
323 return -EINVAL;
324
325 if (!crypto_simd_usable())
326 err = aes_expandkey(ctx, in_key, key_len);
327 else {
328 kernel_fpu_begin();
329 err = aesni_set_key(ctx, in_key, key_len);
330 kernel_fpu_end();
331 }
332
333 return err;
334 }
335
336 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
337 unsigned int key_len)
338 {
339 return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
340 }
341
342 static void aesni_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
343 {
344 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
345
346 if (!crypto_simd_usable()) {
347 aes_encrypt(ctx, dst, src);
348 } else {
349 kernel_fpu_begin();
350 aesni_enc(ctx, dst, src);
351 kernel_fpu_end();
352 }
353 }
354
355 static void aesni_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
356 {
357 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
358
359 if (!crypto_simd_usable()) {
360 aes_decrypt(ctx, dst, src);
361 } else {
362 kernel_fpu_begin();
363 aesni_dec(ctx, dst, src);
364 kernel_fpu_end();
365 }
366 }
367
368 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
369 unsigned int len)
370 {
371 return aes_set_key_common(crypto_skcipher_tfm(tfm),
372 crypto_skcipher_ctx(tfm), key, len);
373 }
374
375 static int ecb_encrypt(struct skcipher_request *req)
376 {
377 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
378 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
379 struct skcipher_walk walk;
380 unsigned int nbytes;
381 int err;
382
383 err = skcipher_walk_virt(&walk, req, true);
384
385 kernel_fpu_begin();
386 while ((nbytes = walk.nbytes)) {
387 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
388 nbytes & AES_BLOCK_MASK);
389 nbytes &= AES_BLOCK_SIZE - 1;
390 err = skcipher_walk_done(&walk, nbytes);
391 }
392 kernel_fpu_end();
393
394 return err;
395 }
396
397 static int ecb_decrypt(struct skcipher_request *req)
398 {
399 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
400 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
401 struct skcipher_walk walk;
402 unsigned int nbytes;
403 int err;
404
405 err = skcipher_walk_virt(&walk, req, true);
406
407 kernel_fpu_begin();
408 while ((nbytes = walk.nbytes)) {
409 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
410 nbytes & AES_BLOCK_MASK);
411 nbytes &= AES_BLOCK_SIZE - 1;
412 err = skcipher_walk_done(&walk, nbytes);
413 }
414 kernel_fpu_end();
415
416 return err;
417 }
418
419 static int cbc_encrypt(struct skcipher_request *req)
420 {
421 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
422 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
423 struct skcipher_walk walk;
424 unsigned int nbytes;
425 int err;
426
427 err = skcipher_walk_virt(&walk, req, true);
428
429 kernel_fpu_begin();
430 while ((nbytes = walk.nbytes)) {
431 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
432 nbytes & AES_BLOCK_MASK, walk.iv);
433 nbytes &= AES_BLOCK_SIZE - 1;
434 err = skcipher_walk_done(&walk, nbytes);
435 }
436 kernel_fpu_end();
437
438 return err;
439 }
440
441 static int cbc_decrypt(struct skcipher_request *req)
442 {
443 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
444 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
445 struct skcipher_walk walk;
446 unsigned int nbytes;
447 int err;
448
449 err = skcipher_walk_virt(&walk, req, true);
450
451 kernel_fpu_begin();
452 while ((nbytes = walk.nbytes)) {
453 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
454 nbytes & AES_BLOCK_MASK, walk.iv);
455 nbytes &= AES_BLOCK_SIZE - 1;
456 err = skcipher_walk_done(&walk, nbytes);
457 }
458 kernel_fpu_end();
459
460 return err;
461 }
462
463 #ifdef CONFIG_X86_64
464 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
465 struct skcipher_walk *walk)
466 {
467 u8 *ctrblk = walk->iv;
468 u8 keystream[AES_BLOCK_SIZE];
469 u8 *src = walk->src.virt.addr;
470 u8 *dst = walk->dst.virt.addr;
471 unsigned int nbytes = walk->nbytes;
472
473 aesni_enc(ctx, keystream, ctrblk);
474 crypto_xor_cpy(dst, keystream, src, nbytes);
475
476 crypto_inc(ctrblk, AES_BLOCK_SIZE);
477 }
478
479 #ifdef CONFIG_AS_AVX
480 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
481 const u8 *in, unsigned int len, u8 *iv)
482 {
483 /*
484 * based on key length, override with the by8 version
485 * of ctr mode encryption/decryption for improved performance
486 * aes_set_key_common() ensures that key length is one of
487 * {128,192,256}
488 */
489 if (ctx->key_length == AES_KEYSIZE_128)
490 aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len);
491 else if (ctx->key_length == AES_KEYSIZE_192)
492 aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len);
493 else
494 aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);
495 }
496 #endif
497
498 static int ctr_crypt(struct skcipher_request *req)
499 {
500 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
501 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
502 struct skcipher_walk walk;
503 unsigned int nbytes;
504 int err;
505
506 err = skcipher_walk_virt(&walk, req, true);
507
508 kernel_fpu_begin();
509 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
510 aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr,
511 nbytes & AES_BLOCK_MASK, walk.iv);
512 nbytes &= AES_BLOCK_SIZE - 1;
513 err = skcipher_walk_done(&walk, nbytes);
514 }
515 if (walk.nbytes) {
516 ctr_crypt_final(ctx, &walk);
517 err = skcipher_walk_done(&walk, 0);
518 }
519 kernel_fpu_end();
520
521 return err;
522 }
523
524 static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key,
525 unsigned int keylen)
526 {
527 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
528 int err;
529
530 err = xts_verify_key(tfm, key, keylen);
531 if (err)
532 return err;
533
534 keylen /= 2;
535
536 /* first half of xts-key is for crypt */
537 err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx,
538 key, keylen);
539 if (err)
540 return err;
541
542 /* second half of xts-key is for tweak */
543 return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx,
544 key + keylen, keylen);
545 }
546
547
548 static void aesni_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
549 {
550 glue_xts_crypt_128bit_one(ctx, dst, src, iv, aesni_enc);
551 }
552
553 static void aesni_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
554 {
555 glue_xts_crypt_128bit_one(ctx, dst, src, iv, aesni_dec);
556 }
557
558 static void aesni_xts_enc8(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
559 {
560 aesni_xts_crypt8(ctx, dst, src, true, iv);
561 }
562
563 static void aesni_xts_dec8(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
564 {
565 aesni_xts_crypt8(ctx, dst, src, false, iv);
566 }
567
568 static const struct common_glue_ctx aesni_enc_xts = {
569 .num_funcs = 2,
570 .fpu_blocks_limit = 1,
571
572 .funcs = { {
573 .num_blocks = 8,
574 .fn_u = { .xts = aesni_xts_enc8 }
575 }, {
576 .num_blocks = 1,
577 .fn_u = { .xts = aesni_xts_enc }
578 } }
579 };
580
581 static const struct common_glue_ctx aesni_dec_xts = {
582 .num_funcs = 2,
583 .fpu_blocks_limit = 1,
584
585 .funcs = { {
586 .num_blocks = 8,
587 .fn_u = { .xts = aesni_xts_dec8 }
588 }, {
589 .num_blocks = 1,
590 .fn_u = { .xts = aesni_xts_dec }
591 } }
592 };
593
594 static int xts_encrypt(struct skcipher_request *req)
595 {
596 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
597 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
598
599 return glue_xts_req_128bit(&aesni_enc_xts, req, aesni_enc,
600 aes_ctx(ctx->raw_tweak_ctx),
601 aes_ctx(ctx->raw_crypt_ctx),
602 false);
603 }
604
605 static int xts_decrypt(struct skcipher_request *req)
606 {
607 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
608 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
609
610 return glue_xts_req_128bit(&aesni_dec_xts, req, aesni_enc,
611 aes_ctx(ctx->raw_tweak_ctx),
612 aes_ctx(ctx->raw_crypt_ctx),
613 true);
614 }
615
616 static int
617 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
618 {
619 struct crypto_aes_ctx ctx;
620 int ret;
621
622 ret = aes_expandkey(&ctx, key, key_len);
623 if (ret)
624 return ret;
625
626 /* Clear the data in the hash sub key container to zero.*/
627 /* We want to cipher all zeros to create the hash sub key. */
628 memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
629
630 aes_encrypt(&ctx, hash_subkey, hash_subkey);
631
632 memzero_explicit(&ctx, sizeof(ctx));
633 return 0;
634 }
635
636 static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
637 unsigned int key_len)
638 {
639 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead);
640
641 if (key_len < 4)
642 return -EINVAL;
643
644 /*Account for 4 byte nonce at the end.*/
645 key_len -= 4;
646
647 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
648
649 return aes_set_key_common(crypto_aead_tfm(aead),
650 &ctx->aes_key_expanded, key, key_len) ?:
651 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
652 }
653
654 /* This is the Integrity Check Value (aka the authentication tag) length and can
655 * be 8, 12 or 16 bytes long. */
656 static int common_rfc4106_set_authsize(struct crypto_aead *aead,
657 unsigned int authsize)
658 {
659 switch (authsize) {
660 case 8:
661 case 12:
662 case 16:
663 break;
664 default:
665 return -EINVAL;
666 }
667
668 return 0;
669 }
670
671 static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,
672 unsigned int authsize)
673 {
674 switch (authsize) {
675 case 4:
676 case 8:
677 case 12:
678 case 13:
679 case 14:
680 case 15:
681 case 16:
682 break;
683 default:
684 return -EINVAL;
685 }
686
687 return 0;
688 }
689
690 static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req,
691 unsigned int assoclen, u8 *hash_subkey,
692 u8 *iv, void *aes_ctx)
693 {
694 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
695 unsigned long auth_tag_len = crypto_aead_authsize(tfm);
696 const struct aesni_gcm_tfm_s *gcm_tfm = aesni_gcm_tfm;
697 struct gcm_context_data data AESNI_ALIGN_ATTR;
698 struct scatter_walk dst_sg_walk = {};
699 unsigned long left = req->cryptlen;
700 unsigned long len, srclen, dstlen;
701 struct scatter_walk assoc_sg_walk;
702 struct scatter_walk src_sg_walk;
703 struct scatterlist src_start[2];
704 struct scatterlist dst_start[2];
705 struct scatterlist *src_sg;
706 struct scatterlist *dst_sg;
707 u8 *src, *dst, *assoc;
708 u8 *assocmem = NULL;
709 u8 authTag[16];
710
711 if (!enc)
712 left -= auth_tag_len;
713
714 #ifdef CONFIG_AS_AVX2
715 if (left < AVX_GEN4_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen4)
716 gcm_tfm = &aesni_gcm_tfm_avx_gen2;
717 #endif
718 #ifdef CONFIG_AS_AVX
719 if (left < AVX_GEN2_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen2)
720 gcm_tfm = &aesni_gcm_tfm_sse;
721 #endif
722
723 /* Linearize assoc, if not already linear */
724 if (req->src->length >= assoclen && req->src->length &&
725 (!PageHighMem(sg_page(req->src)) ||
726 req->src->offset + req->src->length <= PAGE_SIZE)) {
727 scatterwalk_start(&assoc_sg_walk, req->src);
728 assoc = scatterwalk_map(&assoc_sg_walk);
729 } else {
730 /* assoc can be any length, so must be on heap */
731 assocmem = kmalloc(assoclen, GFP_ATOMIC);
732 if (unlikely(!assocmem))
733 return -ENOMEM;
734 assoc = assocmem;
735
736 scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0);
737 }
738
739 if (left) {
740 src_sg = scatterwalk_ffwd(src_start, req->src, req->assoclen);
741 scatterwalk_start(&src_sg_walk, src_sg);
742 if (req->src != req->dst) {
743 dst_sg = scatterwalk_ffwd(dst_start, req->dst,
744 req->assoclen);
745 scatterwalk_start(&dst_sg_walk, dst_sg);
746 }
747 }
748
749 kernel_fpu_begin();
750 gcm_tfm->init(aes_ctx, &data, iv,
751 hash_subkey, assoc, assoclen);
752 if (req->src != req->dst) {
753 while (left) {
754 src = scatterwalk_map(&src_sg_walk);
755 dst = scatterwalk_map(&dst_sg_walk);
756 srclen = scatterwalk_clamp(&src_sg_walk, left);
757 dstlen = scatterwalk_clamp(&dst_sg_walk, left);
758 len = min(srclen, dstlen);
759 if (len) {
760 if (enc)
761 gcm_tfm->enc_update(aes_ctx, &data,
762 dst, src, len);
763 else
764 gcm_tfm->dec_update(aes_ctx, &data,
765 dst, src, len);
766 }
767 left -= len;
768
769 scatterwalk_unmap(src);
770 scatterwalk_unmap(dst);
771 scatterwalk_advance(&src_sg_walk, len);
772 scatterwalk_advance(&dst_sg_walk, len);
773 scatterwalk_done(&src_sg_walk, 0, left);
774 scatterwalk_done(&dst_sg_walk, 1, left);
775 }
776 } else {
777 while (left) {
778 dst = src = scatterwalk_map(&src_sg_walk);
779 len = scatterwalk_clamp(&src_sg_walk, left);
780 if (len) {
781 if (enc)
782 gcm_tfm->enc_update(aes_ctx, &data,
783 src, src, len);
784 else
785 gcm_tfm->dec_update(aes_ctx, &data,
786 src, src, len);
787 }
788 left -= len;
789 scatterwalk_unmap(src);
790 scatterwalk_advance(&src_sg_walk, len);
791 scatterwalk_done(&src_sg_walk, 1, left);
792 }
793 }
794 gcm_tfm->finalize(aes_ctx, &data, authTag, auth_tag_len);
795 kernel_fpu_end();
796
797 if (!assocmem)
798 scatterwalk_unmap(assoc);
799 else
800 kfree(assocmem);
801
802 if (!enc) {
803 u8 authTagMsg[16];
804
805 /* Copy out original authTag */
806 scatterwalk_map_and_copy(authTagMsg, req->src,
807 req->assoclen + req->cryptlen -
808 auth_tag_len,
809 auth_tag_len, 0);
810
811 /* Compare generated tag with passed in tag. */
812 return crypto_memneq(authTagMsg, authTag, auth_tag_len) ?
813 -EBADMSG : 0;
814 }
815
816 /* Copy in the authTag */
817 scatterwalk_map_and_copy(authTag, req->dst,
818 req->assoclen + req->cryptlen,
819 auth_tag_len, 1);
820
821 return 0;
822 }
823
824 static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen,
825 u8 *hash_subkey, u8 *iv, void *aes_ctx)
826 {
827 return gcmaes_crypt_by_sg(true, req, assoclen, hash_subkey, iv,
828 aes_ctx);
829 }
830
831 static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen,
832 u8 *hash_subkey, u8 *iv, void *aes_ctx)
833 {
834 return gcmaes_crypt_by_sg(false, req, assoclen, hash_subkey, iv,
835 aes_ctx);
836 }
837
838 static int helper_rfc4106_encrypt(struct aead_request *req)
839 {
840 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
841 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
842 void *aes_ctx = &(ctx->aes_key_expanded);
843 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
844 unsigned int i;
845 __be32 counter = cpu_to_be32(1);
846
847 /* Assuming we are supporting rfc4106 64-bit extended */
848 /* sequence numbers We need to have the AAD length equal */
849 /* to 16 or 20 bytes */
850 if (unlikely(req->assoclen != 16 && req->assoclen != 20))
851 return -EINVAL;
852
853 /* IV below built */
854 for (i = 0; i < 4; i++)
855 *(iv+i) = ctx->nonce[i];
856 for (i = 0; i < 8; i++)
857 *(iv+4+i) = req->iv[i];
858 *((__be32 *)(iv+12)) = counter;
859
860 return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
861 aes_ctx);
862 }
863
864 static int helper_rfc4106_decrypt(struct aead_request *req)
865 {
866 __be32 counter = cpu_to_be32(1);
867 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
868 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
869 void *aes_ctx = &(ctx->aes_key_expanded);
870 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
871 unsigned int i;
872
873 if (unlikely(req->assoclen != 16 && req->assoclen != 20))
874 return -EINVAL;
875
876 /* Assuming we are supporting rfc4106 64-bit extended */
877 /* sequence numbers We need to have the AAD length */
878 /* equal to 16 or 20 bytes */
879
880 /* IV below built */
881 for (i = 0; i < 4; i++)
882 *(iv+i) = ctx->nonce[i];
883 for (i = 0; i < 8; i++)
884 *(iv+4+i) = req->iv[i];
885 *((__be32 *)(iv+12)) = counter;
886
887 return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
888 aes_ctx);
889 }
890 #endif
891
892 static struct crypto_alg aesni_cipher_alg = {
893 .cra_name = "aes",
894 .cra_driver_name = "aes-aesni",
895 .cra_priority = 300,
896 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
897 .cra_blocksize = AES_BLOCK_SIZE,
898 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
899 .cra_module = THIS_MODULE,
900 .cra_u = {
901 .cipher = {
902 .cia_min_keysize = AES_MIN_KEY_SIZE,
903 .cia_max_keysize = AES_MAX_KEY_SIZE,
904 .cia_setkey = aes_set_key,
905 .cia_encrypt = aesni_encrypt,
906 .cia_decrypt = aesni_decrypt
907 }
908 }
909 };
910
911 static struct skcipher_alg aesni_skciphers[] = {
912 {
913 .base = {
914 .cra_name = "__ecb(aes)",
915 .cra_driver_name = "__ecb-aes-aesni",
916 .cra_priority = 400,
917 .cra_flags = CRYPTO_ALG_INTERNAL,
918 .cra_blocksize = AES_BLOCK_SIZE,
919 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
920 .cra_module = THIS_MODULE,
921 },
922 .min_keysize = AES_MIN_KEY_SIZE,
923 .max_keysize = AES_MAX_KEY_SIZE,
924 .setkey = aesni_skcipher_setkey,
925 .encrypt = ecb_encrypt,
926 .decrypt = ecb_decrypt,
927 }, {
928 .base = {
929 .cra_name = "__cbc(aes)",
930 .cra_driver_name = "__cbc-aes-aesni",
931 .cra_priority = 400,
932 .cra_flags = CRYPTO_ALG_INTERNAL,
933 .cra_blocksize = AES_BLOCK_SIZE,
934 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
935 .cra_module = THIS_MODULE,
936 },
937 .min_keysize = AES_MIN_KEY_SIZE,
938 .max_keysize = AES_MAX_KEY_SIZE,
939 .ivsize = AES_BLOCK_SIZE,
940 .setkey = aesni_skcipher_setkey,
941 .encrypt = cbc_encrypt,
942 .decrypt = cbc_decrypt,
943 #ifdef CONFIG_X86_64
944 }, {
945 .base = {
946 .cra_name = "__ctr(aes)",
947 .cra_driver_name = "__ctr-aes-aesni",
948 .cra_priority = 400,
949 .cra_flags = CRYPTO_ALG_INTERNAL,
950 .cra_blocksize = 1,
951 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
952 .cra_module = THIS_MODULE,
953 },
954 .min_keysize = AES_MIN_KEY_SIZE,
955 .max_keysize = AES_MAX_KEY_SIZE,
956 .ivsize = AES_BLOCK_SIZE,
957 .chunksize = AES_BLOCK_SIZE,
958 .setkey = aesni_skcipher_setkey,
959 .encrypt = ctr_crypt,
960 .decrypt = ctr_crypt,
961 }, {
962 .base = {
963 .cra_name = "__xts(aes)",
964 .cra_driver_name = "__xts-aes-aesni",
965 .cra_priority = 401,
966 .cra_flags = CRYPTO_ALG_INTERNAL,
967 .cra_blocksize = AES_BLOCK_SIZE,
968 .cra_ctxsize = XTS_AES_CTX_SIZE,
969 .cra_module = THIS_MODULE,
970 },
971 .min_keysize = 2 * AES_MIN_KEY_SIZE,
972 .max_keysize = 2 * AES_MAX_KEY_SIZE,
973 .ivsize = AES_BLOCK_SIZE,
974 .setkey = xts_aesni_setkey,
975 .encrypt = xts_encrypt,
976 .decrypt = xts_decrypt,
977 #endif
978 }
979 };
980
981 static
982 struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)];
983
984 #ifdef CONFIG_X86_64
985 static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key,
986 unsigned int key_len)
987 {
988 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead);
989
990 return aes_set_key_common(crypto_aead_tfm(aead),
991 &ctx->aes_key_expanded, key, key_len) ?:
992 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
993 }
994
995 static int generic_gcmaes_encrypt(struct aead_request *req)
996 {
997 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
998 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
999 void *aes_ctx = &(ctx->aes_key_expanded);
1000 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
1001 __be32 counter = cpu_to_be32(1);
1002
1003 memcpy(iv, req->iv, 12);
1004 *((__be32 *)(iv+12)) = counter;
1005
1006 return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv,
1007 aes_ctx);
1008 }
1009
1010 static int generic_gcmaes_decrypt(struct aead_request *req)
1011 {
1012 __be32 counter = cpu_to_be32(1);
1013 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1014 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1015 void *aes_ctx = &(ctx->aes_key_expanded);
1016 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
1017
1018 memcpy(iv, req->iv, 12);
1019 *((__be32 *)(iv+12)) = counter;
1020
1021 return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv,
1022 aes_ctx);
1023 }
1024
1025 static struct aead_alg aesni_aeads[] = { {
1026 .setkey = common_rfc4106_set_key,
1027 .setauthsize = common_rfc4106_set_authsize,
1028 .encrypt = helper_rfc4106_encrypt,
1029 .decrypt = helper_rfc4106_decrypt,
1030 .ivsize = GCM_RFC4106_IV_SIZE,
1031 .maxauthsize = 16,
1032 .base = {
1033 .cra_name = "__rfc4106(gcm(aes))",
1034 .cra_driver_name = "__rfc4106-gcm-aesni",
1035 .cra_priority = 400,
1036 .cra_flags = CRYPTO_ALG_INTERNAL,
1037 .cra_blocksize = 1,
1038 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx),
1039 .cra_alignmask = AESNI_ALIGN - 1,
1040 .cra_module = THIS_MODULE,
1041 },
1042 }, {
1043 .setkey = generic_gcmaes_set_key,
1044 .setauthsize = generic_gcmaes_set_authsize,
1045 .encrypt = generic_gcmaes_encrypt,
1046 .decrypt = generic_gcmaes_decrypt,
1047 .ivsize = GCM_AES_IV_SIZE,
1048 .maxauthsize = 16,
1049 .base = {
1050 .cra_name = "__gcm(aes)",
1051 .cra_driver_name = "__generic-gcm-aesni",
1052 .cra_priority = 400,
1053 .cra_flags = CRYPTO_ALG_INTERNAL,
1054 .cra_blocksize = 1,
1055 .cra_ctxsize = sizeof(struct generic_gcmaes_ctx),
1056 .cra_alignmask = AESNI_ALIGN - 1,
1057 .cra_module = THIS_MODULE,
1058 },
1059 } };
1060 #else
1061 static struct aead_alg aesni_aeads[0];
1062 #endif
1063
1064 static struct simd_aead_alg *aesni_simd_aeads[ARRAY_SIZE(aesni_aeads)];
1065
1066 static const struct x86_cpu_id aesni_cpu_id[] = {
1067 X86_FEATURE_MATCH(X86_FEATURE_AES),
1068 {}
1069 };
1070 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1071
1072 static int __init aesni_init(void)
1073 {
1074 int err;
1075
1076 if (!x86_match_cpu(aesni_cpu_id))
1077 return -ENODEV;
1078 #ifdef CONFIG_X86_64
1079 #ifdef CONFIG_AS_AVX2
1080 if (boot_cpu_has(X86_FEATURE_AVX2)) {
1081 pr_info("AVX2 version of gcm_enc/dec engaged.\n");
1082 aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen4;
1083 } else
1084 #endif
1085 #ifdef CONFIG_AS_AVX
1086 if (boot_cpu_has(X86_FEATURE_AVX)) {
1087 pr_info("AVX version of gcm_enc/dec engaged.\n");
1088 aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen2;
1089 } else
1090 #endif
1091 {
1092 pr_info("SSE version of gcm_enc/dec engaged.\n");
1093 aesni_gcm_tfm = &aesni_gcm_tfm_sse;
1094 }
1095 aesni_ctr_enc_tfm = aesni_ctr_enc;
1096 #ifdef CONFIG_AS_AVX
1097 if (boot_cpu_has(X86_FEATURE_AVX)) {
1098 /* optimize performance of ctr mode encryption transform */
1099 aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm;
1100 pr_info("AES CTR mode by8 optimization enabled\n");
1101 }
1102 #endif
1103 #endif
1104
1105 err = crypto_register_alg(&aesni_cipher_alg);
1106 if (err)
1107 return err;
1108
1109 err = simd_register_skciphers_compat(aesni_skciphers,
1110 ARRAY_SIZE(aesni_skciphers),
1111 aesni_simd_skciphers);
1112 if (err)
1113 goto unregister_cipher;
1114
1115 err = simd_register_aeads_compat(aesni_aeads, ARRAY_SIZE(aesni_aeads),
1116 aesni_simd_aeads);
1117 if (err)
1118 goto unregister_skciphers;
1119
1120 return 0;
1121
1122 unregister_skciphers:
1123 simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
1124 aesni_simd_skciphers);
1125 unregister_cipher:
1126 crypto_unregister_alg(&aesni_cipher_alg);
1127 return err;
1128 }
1129
1130 static void __exit aesni_exit(void)
1131 {
1132 simd_unregister_aeads(aesni_aeads, ARRAY_SIZE(aesni_aeads),
1133 aesni_simd_aeads);
1134 simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
1135 aesni_simd_skciphers);
1136 crypto_unregister_alg(&aesni_cipher_alg);
1137 }
1138
1139 late_initcall(aesni_init);
1140 module_exit(aesni_exit);
1141
1142 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1143 MODULE_LICENSE("GPL");
1144 MODULE_ALIAS_CRYPTO("aes");
Linux with added FPC-III support