?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 4.13.16
[linux/fpc-iii.git] / arch / x86 / crypto / aesni-intel_glue.c
blob4a55cdcdc0082790bcf6457ab43e1a84caea7388
1 /*
2 * Support for Intel AES-NI instructions. This file contains glue
3 * code, the real AES implementation is in intel-aes_asm.S.
4 *
5 * Copyright (C) 2008, Intel Corp.
6 * Author: Huang Ying <ying.huang@intel.com>
7 *
8 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
9 * interface for 64-bit kernels.
10 * Authors: Adrian Hoban <adrian.hoban@intel.com>
11 * Gabriele Paoloni <gabriele.paoloni@intel.com>
12 * Tadeusz Struk (tadeusz.struk@intel.com)
13 * Aidan O'Mahony (aidan.o.mahony@intel.com)
14 * Copyright (c) 2010, Intel Corporation.
15 *
16 * This program is free software; you can redistribute it and/or modify
17 * it under the terms of the GNU General Public License as published by
18 * the Free Software Foundation; either version 2 of the License, or
19 * (at your option) any later version.
20 */
21
22 #include <linux/hardirq.h>
23 #include <linux/types.h>
24 #include <linux/module.h>
25 #include <linux/err.h>
26 #include <crypto/algapi.h>
27 #include <crypto/aes.h>
28 #include <crypto/cryptd.h>
29 #include <crypto/ctr.h>
30 #include <crypto/b128ops.h>
31 #include <crypto/xts.h>
32 #include <asm/cpu_device_id.h>
33 #include <asm/fpu/api.h>
34 #include <asm/crypto/aes.h>
35 #include <crypto/scatterwalk.h>
36 #include <crypto/internal/aead.h>
37 #include <crypto/internal/simd.h>
38 #include <crypto/internal/skcipher.h>
39 #include <linux/workqueue.h>
40 #include <linux/spinlock.h>
41 #ifdef CONFIG_X86_64
42 #include <asm/crypto/glue_helper.h>
43 #endif
44
45
46 #define AESNI_ALIGN 16
47 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
48 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE - 1))
49 #define RFC4106_HASH_SUBKEY_SIZE 16
50 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
51 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
52 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
53
54 /* This data is stored at the end of the crypto_tfm struct.
55 * It's a type of per "session" data storage location.
56 * This needs to be 16 byte aligned.
57 */
58 struct aesni_rfc4106_gcm_ctx {
59 u8 hash_subkey[16] AESNI_ALIGN_ATTR;
60 struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
61 u8 nonce[4];
62 };
63
64 struct generic_gcmaes_ctx {
65 u8 hash_subkey[16] AESNI_ALIGN_ATTR;
66 struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
67 };
68
69 struct aesni_xts_ctx {
70 u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
71 u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
72 };
73
74 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
75 unsigned int key_len);
76 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
77 const u8 *in);
78 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
79 const u8 *in);
80 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
81 const u8 *in, unsigned int len);
82 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
83 const u8 *in, unsigned int len);
84 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
85 const u8 *in, unsigned int len, u8 *iv);
86 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
87 const u8 *in, unsigned int len, u8 *iv);
88
89 int crypto_fpu_init(void);
90 void crypto_fpu_exit(void);
91
92 #define AVX_GEN2_OPTSIZE 640
93 #define AVX_GEN4_OPTSIZE 4096
94
95 #ifdef CONFIG_X86_64
96
97 static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out,
98 const u8 *in, unsigned int len, u8 *iv);
99 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
100 const u8 *in, unsigned int len, u8 *iv);
101
102 asmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out,
103 const u8 *in, bool enc, u8 *iv);
104
105 /* asmlinkage void aesni_gcm_enc()
106 * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
107 * u8 *out, Ciphertext output. Encrypt in-place is allowed.
108 * const u8 *in, Plaintext input
109 * unsigned long plaintext_len, Length of data in bytes for encryption.
110 * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
111 * 16-byte aligned pointer.
112 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
113 * const u8 *aad, Additional Authentication Data (AAD)
114 * unsigned long aad_len, Length of AAD in bytes.
115 * u8 *auth_tag, Authenticated Tag output.
116 * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
117 * Valid values are 16 (most likely), 12 or 8.
118 */
119 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out,
120 const u8 *in, unsigned long plaintext_len, u8 *iv,
121 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
122 u8 *auth_tag, unsigned long auth_tag_len);
123
124 /* asmlinkage void aesni_gcm_dec()
125 * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
126 * u8 *out, Plaintext output. Decrypt in-place is allowed.
127 * const u8 *in, Ciphertext input
128 * unsigned long ciphertext_len, Length of data in bytes for decryption.
129 * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
130 * 16-byte aligned pointer.
131 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
132 * const u8 *aad, Additional Authentication Data (AAD)
133 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
134 * to be 8 or 12 bytes
135 * u8 *auth_tag, Authenticated Tag output.
136 * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
137 * Valid values are 16 (most likely), 12 or 8.
138 */
139 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out,
140 const u8 *in, unsigned long ciphertext_len, u8 *iv,
141 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
142 u8 *auth_tag, unsigned long auth_tag_len);
143
144
145 #ifdef CONFIG_AS_AVX
146 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,
147 void *keys, u8 *out, unsigned int num_bytes);
148 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
149 void *keys, u8 *out, unsigned int num_bytes);
150 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
151 void *keys, u8 *out, unsigned int num_bytes);
152 /*
153 * asmlinkage void aesni_gcm_precomp_avx_gen2()
154 * gcm_data *my_ctx_data, context data
155 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
156 */
157 asmlinkage void aesni_gcm_precomp_avx_gen2(void *my_ctx_data, u8 *hash_subkey);
158
159 asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx, u8 *out,
160 const u8 *in, unsigned long plaintext_len, u8 *iv,
161 const u8 *aad, unsigned long aad_len,
162 u8 *auth_tag, unsigned long auth_tag_len);
163
164 asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx, u8 *out,
165 const u8 *in, unsigned long ciphertext_len, u8 *iv,
166 const u8 *aad, unsigned long aad_len,
167 u8 *auth_tag, unsigned long auth_tag_len);
168
169 static void aesni_gcm_enc_avx(void *ctx, u8 *out,
170 const u8 *in, unsigned long plaintext_len, u8 *iv,
171 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
172 u8 *auth_tag, unsigned long auth_tag_len)
173 {
174 struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
175 if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)){
176 aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad,
177 aad_len, auth_tag, auth_tag_len);
178 } else {
179 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
180 aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad,
181 aad_len, auth_tag, auth_tag_len);
182 }
183 }
184
185 static void aesni_gcm_dec_avx(void *ctx, u8 *out,
186 const u8 *in, unsigned long ciphertext_len, u8 *iv,
187 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
188 u8 *auth_tag, unsigned long auth_tag_len)
189 {
190 struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
191 if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) {
192 aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey, aad,
193 aad_len, auth_tag, auth_tag_len);
194 } else {
195 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
196 aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad,
197 aad_len, auth_tag, auth_tag_len);
198 }
199 }
200 #endif
201
202 #ifdef CONFIG_AS_AVX2
203 /*
204 * asmlinkage void aesni_gcm_precomp_avx_gen4()
205 * gcm_data *my_ctx_data, context data
206 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
207 */
208 asmlinkage void aesni_gcm_precomp_avx_gen4(void *my_ctx_data, u8 *hash_subkey);
209
210 asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx, u8 *out,
211 const u8 *in, unsigned long plaintext_len, u8 *iv,
212 const u8 *aad, unsigned long aad_len,
213 u8 *auth_tag, unsigned long auth_tag_len);
214
215 asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx, u8 *out,
216 const u8 *in, unsigned long ciphertext_len, u8 *iv,
217 const u8 *aad, unsigned long aad_len,
218 u8 *auth_tag, unsigned long auth_tag_len);
219
220 static void aesni_gcm_enc_avx2(void *ctx, u8 *out,
221 const u8 *in, unsigned long plaintext_len, u8 *iv,
222 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
223 u8 *auth_tag, unsigned long auth_tag_len)
224 {
225 struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
226 if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) {
227 aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad,
228 aad_len, auth_tag, auth_tag_len);
229 } else if (plaintext_len < AVX_GEN4_OPTSIZE) {
230 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
231 aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad,
232 aad_len, auth_tag, auth_tag_len);
233 } else {
234 aesni_gcm_precomp_avx_gen4(ctx, hash_subkey);
235 aesni_gcm_enc_avx_gen4(ctx, out, in, plaintext_len, iv, aad,
236 aad_len, auth_tag, auth_tag_len);
237 }
238 }
239
240 static void aesni_gcm_dec_avx2(void *ctx, u8 *out,
241 const u8 *in, unsigned long ciphertext_len, u8 *iv,
242 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
243 u8 *auth_tag, unsigned long auth_tag_len)
244 {
245 struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
246 if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) {
247 aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey,
248 aad, aad_len, auth_tag, auth_tag_len);
249 } else if (ciphertext_len < AVX_GEN4_OPTSIZE) {
250 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
251 aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad,
252 aad_len, auth_tag, auth_tag_len);
253 } else {
254 aesni_gcm_precomp_avx_gen4(ctx, hash_subkey);
255 aesni_gcm_dec_avx_gen4(ctx, out, in, ciphertext_len, iv, aad,
256 aad_len, auth_tag, auth_tag_len);
257 }
258 }
259 #endif
260
261 static void (*aesni_gcm_enc_tfm)(void *ctx, u8 *out,
262 const u8 *in, unsigned long plaintext_len, u8 *iv,
263 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
264 u8 *auth_tag, unsigned long auth_tag_len);
265
266 static void (*aesni_gcm_dec_tfm)(void *ctx, u8 *out,
267 const u8 *in, unsigned long ciphertext_len, u8 *iv,
268 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
269 u8 *auth_tag, unsigned long auth_tag_len);
270
271 static inline struct
272 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
273 {
274 unsigned long align = AESNI_ALIGN;
275
276 if (align <= crypto_tfm_ctx_alignment())
277 align = 1;
278 return PTR_ALIGN(crypto_aead_ctx(tfm), align);
279 }
280
281 static inline struct
282 generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm)
283 {
284 unsigned long align = AESNI_ALIGN;
285
286 if (align <= crypto_tfm_ctx_alignment())
287 align = 1;
288 return PTR_ALIGN(crypto_aead_ctx(tfm), align);
289 }
290 #endif
291
292 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
293 {
294 unsigned long addr = (unsigned long)raw_ctx;
295 unsigned long align = AESNI_ALIGN;
296
297 if (align <= crypto_tfm_ctx_alignment())
298 align = 1;
299 return (struct crypto_aes_ctx *)ALIGN(addr, align);
300 }
301
302 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
303 const u8 *in_key, unsigned int key_len)
304 {
305 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
306 u32 *flags = &tfm->crt_flags;
307 int err;
308
309 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
310 key_len != AES_KEYSIZE_256) {
311 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
312 return -EINVAL;
313 }
314
315 if (!irq_fpu_usable())
316 err = crypto_aes_expand_key(ctx, in_key, key_len);
317 else {
318 kernel_fpu_begin();
319 err = aesni_set_key(ctx, in_key, key_len);
320 kernel_fpu_end();
321 }
322
323 return err;
324 }
325
326 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
327 unsigned int key_len)
328 {
329 return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
330 }
331
332 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
333 {
334 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
335
336 if (!irq_fpu_usable())
337 crypto_aes_encrypt_x86(ctx, dst, src);
338 else {
339 kernel_fpu_begin();
340 aesni_enc(ctx, dst, src);
341 kernel_fpu_end();
342 }
343 }
344
345 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
346 {
347 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
348
349 if (!irq_fpu_usable())
350 crypto_aes_decrypt_x86(ctx, dst, src);
351 else {
352 kernel_fpu_begin();
353 aesni_dec(ctx, dst, src);
354 kernel_fpu_end();
355 }
356 }
357
358 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
359 {
360 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
361
362 aesni_enc(ctx, dst, src);
363 }
364
365 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
366 {
367 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
368
369 aesni_dec(ctx, dst, src);
370 }
371
372 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
373 unsigned int len)
374 {
375 return aes_set_key_common(crypto_skcipher_tfm(tfm),
376 crypto_skcipher_ctx(tfm), key, len);
377 }
378
379 static int ecb_encrypt(struct skcipher_request *req)
380 {
381 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
382 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
383 struct skcipher_walk walk;
384 unsigned int nbytes;
385 int err;
386
387 err = skcipher_walk_virt(&walk, req, true);
388
389 kernel_fpu_begin();
390 while ((nbytes = walk.nbytes)) {
391 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
392 nbytes & AES_BLOCK_MASK);
393 nbytes &= AES_BLOCK_SIZE - 1;
394 err = skcipher_walk_done(&walk, nbytes);
395 }
396 kernel_fpu_end();
397
398 return err;
399 }
400
401 static int ecb_decrypt(struct skcipher_request *req)
402 {
403 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
404 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
405 struct skcipher_walk walk;
406 unsigned int nbytes;
407 int err;
408
409 err = skcipher_walk_virt(&walk, req, true);
410
411 kernel_fpu_begin();
412 while ((nbytes = walk.nbytes)) {
413 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
414 nbytes & AES_BLOCK_MASK);
415 nbytes &= AES_BLOCK_SIZE - 1;
416 err = skcipher_walk_done(&walk, nbytes);
417 }
418 kernel_fpu_end();
419
420 return err;
421 }
422
423 static int cbc_encrypt(struct skcipher_request *req)
424 {
425 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
426 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
427 struct skcipher_walk walk;
428 unsigned int nbytes;
429 int err;
430
431 err = skcipher_walk_virt(&walk, req, true);
432
433 kernel_fpu_begin();
434 while ((nbytes = walk.nbytes)) {
435 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
436 nbytes & AES_BLOCK_MASK, walk.iv);
437 nbytes &= AES_BLOCK_SIZE - 1;
438 err = skcipher_walk_done(&walk, nbytes);
439 }
440 kernel_fpu_end();
441
442 return err;
443 }
444
445 static int cbc_decrypt(struct skcipher_request *req)
446 {
447 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
448 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
449 struct skcipher_walk walk;
450 unsigned int nbytes;
451 int err;
452
453 err = skcipher_walk_virt(&walk, req, true);
454
455 kernel_fpu_begin();
456 while ((nbytes = walk.nbytes)) {
457 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
458 nbytes & AES_BLOCK_MASK, walk.iv);
459 nbytes &= AES_BLOCK_SIZE - 1;
460 err = skcipher_walk_done(&walk, nbytes);
461 }
462 kernel_fpu_end();
463
464 return err;
465 }
466
467 #ifdef CONFIG_X86_64
468 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
469 struct skcipher_walk *walk)
470 {
471 u8 *ctrblk = walk->iv;
472 u8 keystream[AES_BLOCK_SIZE];
473 u8 *src = walk->src.virt.addr;
474 u8 *dst = walk->dst.virt.addr;
475 unsigned int nbytes = walk->nbytes;
476
477 aesni_enc(ctx, keystream, ctrblk);
478 crypto_xor(keystream, src, nbytes);
479 memcpy(dst, keystream, nbytes);
480 crypto_inc(ctrblk, AES_BLOCK_SIZE);
481 }
482
483 #ifdef CONFIG_AS_AVX
484 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
485 const u8 *in, unsigned int len, u8 *iv)
486 {
487 /*
488 * based on key length, override with the by8 version
489 * of ctr mode encryption/decryption for improved performance
490 * aes_set_key_common() ensures that key length is one of
491 * {128,192,256}
492 */
493 if (ctx->key_length == AES_KEYSIZE_128)
494 aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len);
495 else if (ctx->key_length == AES_KEYSIZE_192)
496 aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len);
497 else
498 aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);
499 }
500 #endif
501
502 static int ctr_crypt(struct skcipher_request *req)
503 {
504 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
505 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
506 struct skcipher_walk walk;
507 unsigned int nbytes;
508 int err;
509
510 err = skcipher_walk_virt(&walk, req, true);
511
512 kernel_fpu_begin();
513 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
514 aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr,
515 nbytes & AES_BLOCK_MASK, walk.iv);
516 nbytes &= AES_BLOCK_SIZE - 1;
517 err = skcipher_walk_done(&walk, nbytes);
518 }
519 if (walk.nbytes) {
520 ctr_crypt_final(ctx, &walk);
521 err = skcipher_walk_done(&walk, 0);
522 }
523 kernel_fpu_end();
524
525 return err;
526 }
527
528 static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key,
529 unsigned int keylen)
530 {
531 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
532 int err;
533
534 err = xts_verify_key(tfm, key, keylen);
535 if (err)
536 return err;
537
538 keylen /= 2;
539
540 /* first half of xts-key is for crypt */
541 err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx,
542 key, keylen);
543 if (err)
544 return err;
545
546 /* second half of xts-key is for tweak */
547 return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx,
548 key + keylen, keylen);
549 }
550
551
552 static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in)
553 {
554 aesni_enc(ctx, out, in);
555 }
556
557 static void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
558 {
559 glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc));
560 }
561
562 static void aesni_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
563 {
564 glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_dec));
565 }
566
567 static void aesni_xts_enc8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
568 {
569 aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, true, (u8 *)iv);
570 }
571
572 static void aesni_xts_dec8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
573 {
574 aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, false, (u8 *)iv);
575 }
576
577 static const struct common_glue_ctx aesni_enc_xts = {
578 .num_funcs = 2,
579 .fpu_blocks_limit = 1,
580
581 .funcs = { {
582 .num_blocks = 8,
583 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc8) }
584 }, {
585 .num_blocks = 1,
586 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc) }
587 } }
588 };
589
590 static const struct common_glue_ctx aesni_dec_xts = {
591 .num_funcs = 2,
592 .fpu_blocks_limit = 1,
593
594 .funcs = { {
595 .num_blocks = 8,
596 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec8) }
597 }, {
598 .num_blocks = 1,
599 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec) }
600 } }
601 };
602
603 static int xts_encrypt(struct skcipher_request *req)
604 {
605 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
606 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
607
608 return glue_xts_req_128bit(&aesni_enc_xts, req,
609 XTS_TWEAK_CAST(aesni_xts_tweak),
610 aes_ctx(ctx->raw_tweak_ctx),
611 aes_ctx(ctx->raw_crypt_ctx));
612 }
613
614 static int xts_decrypt(struct skcipher_request *req)
615 {
616 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
617 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
618
619 return glue_xts_req_128bit(&aesni_dec_xts, req,
620 XTS_TWEAK_CAST(aesni_xts_tweak),
621 aes_ctx(ctx->raw_tweak_ctx),
622 aes_ctx(ctx->raw_crypt_ctx));
623 }
624
625 static int rfc4106_init(struct crypto_aead *aead)
626 {
627 struct cryptd_aead *cryptd_tfm;
628 struct cryptd_aead **ctx = crypto_aead_ctx(aead);
629
630 cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni",
631 CRYPTO_ALG_INTERNAL,
632 CRYPTO_ALG_INTERNAL);
633 if (IS_ERR(cryptd_tfm))
634 return PTR_ERR(cryptd_tfm);
635
636 *ctx = cryptd_tfm;
637 crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
638 return 0;
639 }
640
641 static void rfc4106_exit(struct crypto_aead *aead)
642 {
643 struct cryptd_aead **ctx = crypto_aead_ctx(aead);
644
645 cryptd_free_aead(*ctx);
646 }
647
648 static int
649 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
650 {
651 struct crypto_cipher *tfm;
652 int ret;
653
654 tfm = crypto_alloc_cipher("aes", 0, 0);
655 if (IS_ERR(tfm))
656 return PTR_ERR(tfm);
657
658 ret = crypto_cipher_setkey(tfm, key, key_len);
659 if (ret)
660 goto out_free_cipher;
661
662 /* Clear the data in the hash sub key container to zero.*/
663 /* We want to cipher all zeros to create the hash sub key. */
664 memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
665
666 crypto_cipher_encrypt_one(tfm, hash_subkey, hash_subkey);
667
668 out_free_cipher:
669 crypto_free_cipher(tfm);
670 return ret;
671 }
672
673 static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
674 unsigned int key_len)
675 {
676 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead);
677
678 if (key_len < 4) {
679 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
680 return -EINVAL;
681 }
682 /*Account for 4 byte nonce at the end.*/
683 key_len -= 4;
684
685 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
686
687 return aes_set_key_common(crypto_aead_tfm(aead),
688 &ctx->aes_key_expanded, key, key_len) ?:
689 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
690 }
691
692 static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
693 unsigned int key_len)
694 {
695 struct cryptd_aead **ctx = crypto_aead_ctx(parent);
696 struct cryptd_aead *cryptd_tfm = *ctx;
697
698 return crypto_aead_setkey(&cryptd_tfm->base, key, key_len);
699 }
700
701 static int common_rfc4106_set_authsize(struct crypto_aead *aead,
702 unsigned int authsize)
703 {
704 switch (authsize) {
705 case 8:
706 case 12:
707 case 16:
708 break;
709 default:
710 return -EINVAL;
711 }
712
713 return 0;
714 }
715
716 /* This is the Integrity Check Value (aka the authentication tag length and can
717 * be 8, 12 or 16 bytes long. */
718 static int rfc4106_set_authsize(struct crypto_aead *parent,
719 unsigned int authsize)
720 {
721 struct cryptd_aead **ctx = crypto_aead_ctx(parent);
722 struct cryptd_aead *cryptd_tfm = *ctx;
723
724 return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
725 }
726
727 static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,
728 unsigned int authsize)
729 {
730 switch (authsize) {
731 case 4:
732 case 8:
733 case 12:
734 case 13:
735 case 14:
736 case 15:
737 case 16:
738 break;
739 default:
740 return -EINVAL;
741 }
742
743 return 0;
744 }
745
746 static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen,
747 u8 *hash_subkey, u8 *iv, void *aes_ctx)
748 {
749 u8 one_entry_in_sg = 0;
750 u8 *src, *dst, *assoc;
751 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
752 unsigned long auth_tag_len = crypto_aead_authsize(tfm);
753 struct scatter_walk src_sg_walk;
754 struct scatter_walk dst_sg_walk = {};
755
756 if (sg_is_last(req->src) &&
757 (!PageHighMem(sg_page(req->src)) ||
758 req->src->offset + req->src->length <= PAGE_SIZE) &&
759 sg_is_last(req->dst) &&
760 (!PageHighMem(sg_page(req->dst)) ||
761 req->dst->offset + req->dst->length <= PAGE_SIZE)) {
762 one_entry_in_sg = 1;
763 scatterwalk_start(&src_sg_walk, req->src);
764 assoc = scatterwalk_map(&src_sg_walk);
765 src = assoc + req->assoclen;
766 dst = src;
767 if (unlikely(req->src != req->dst)) {
768 scatterwalk_start(&dst_sg_walk, req->dst);
769 dst = scatterwalk_map(&dst_sg_walk) + req->assoclen;
770 }
771 } else {
772 /* Allocate memory for src, dst, assoc */
773 assoc = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
774 GFP_ATOMIC);
775 if (unlikely(!assoc))
776 return -ENOMEM;
777 scatterwalk_map_and_copy(assoc, req->src, 0,
778 req->assoclen + req->cryptlen, 0);
779 src = assoc + req->assoclen;
780 dst = src;
781 }
782
783 kernel_fpu_begin();
784 aesni_gcm_enc_tfm(aes_ctx, dst, src, req->cryptlen, iv,
785 hash_subkey, assoc, assoclen,
786 dst + req->cryptlen, auth_tag_len);
787 kernel_fpu_end();
788
789 /* The authTag (aka the Integrity Check Value) needs to be written
790 * back to the packet. */
791 if (one_entry_in_sg) {
792 if (unlikely(req->src != req->dst)) {
793 scatterwalk_unmap(dst - req->assoclen);
794 scatterwalk_advance(&dst_sg_walk, req->dst->length);
795 scatterwalk_done(&dst_sg_walk, 1, 0);
796 }
797 scatterwalk_unmap(assoc);
798 scatterwalk_advance(&src_sg_walk, req->src->length);
799 scatterwalk_done(&src_sg_walk, req->src == req->dst, 0);
800 } else {
801 scatterwalk_map_and_copy(dst, req->dst, req->assoclen,
802 req->cryptlen + auth_tag_len, 1);
803 kfree(assoc);
804 }
805 return 0;
806 }
807
808 static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen,
809 u8 *hash_subkey, u8 *iv, void *aes_ctx)
810 {
811 u8 one_entry_in_sg = 0;
812 u8 *src, *dst, *assoc;
813 unsigned long tempCipherLen = 0;
814 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
815 unsigned long auth_tag_len = crypto_aead_authsize(tfm);
816 u8 authTag[16];
817 struct scatter_walk src_sg_walk;
818 struct scatter_walk dst_sg_walk = {};
819 int retval = 0;
820
821 tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
822
823 if (sg_is_last(req->src) &&
824 (!PageHighMem(sg_page(req->src)) ||
825 req->src->offset + req->src->length <= PAGE_SIZE) &&
826 sg_is_last(req->dst) &&
827 (!PageHighMem(sg_page(req->dst)) ||
828 req->dst->offset + req->dst->length <= PAGE_SIZE)) {
829 one_entry_in_sg = 1;
830 scatterwalk_start(&src_sg_walk, req->src);
831 assoc = scatterwalk_map(&src_sg_walk);
832 src = assoc + req->assoclen;
833 dst = src;
834 if (unlikely(req->src != req->dst)) {
835 scatterwalk_start(&dst_sg_walk, req->dst);
836 dst = scatterwalk_map(&dst_sg_walk) + req->assoclen;
837 }
838 } else {
839 /* Allocate memory for src, dst, assoc */
840 assoc = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
841 if (!assoc)
842 return -ENOMEM;
843 scatterwalk_map_and_copy(assoc, req->src, 0,
844 req->assoclen + req->cryptlen, 0);
845 src = assoc + req->assoclen;
846 dst = src;
847 }
848
849
850 kernel_fpu_begin();
851 aesni_gcm_dec_tfm(aes_ctx, dst, src, tempCipherLen, iv,
852 hash_subkey, assoc, assoclen,
853 authTag, auth_tag_len);
854 kernel_fpu_end();
855
856 /* Compare generated tag with passed in tag. */
857 retval = crypto_memneq(src + tempCipherLen, authTag, auth_tag_len) ?
858 -EBADMSG : 0;
859
860 if (one_entry_in_sg) {
861 if (unlikely(req->src != req->dst)) {
862 scatterwalk_unmap(dst - req->assoclen);
863 scatterwalk_advance(&dst_sg_walk, req->dst->length);
864 scatterwalk_done(&dst_sg_walk, 1, 0);
865 }
866 scatterwalk_unmap(assoc);
867 scatterwalk_advance(&src_sg_walk, req->src->length);
868 scatterwalk_done(&src_sg_walk, req->src == req->dst, 0);
869 } else {
870 scatterwalk_map_and_copy(dst, req->dst, req->assoclen,
871 tempCipherLen, 1);
872 kfree(assoc);
873 }
874 return retval;
875
876 }
877
878 static int helper_rfc4106_encrypt(struct aead_request *req)
879 {
880 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
881 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
882 void *aes_ctx = &(ctx->aes_key_expanded);
883 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
884 unsigned int i;
885 __be32 counter = cpu_to_be32(1);
886
887 /* Assuming we are supporting rfc4106 64-bit extended */
888 /* sequence numbers We need to have the AAD length equal */
889 /* to 16 or 20 bytes */
890 if (unlikely(req->assoclen != 16 && req->assoclen != 20))
891 return -EINVAL;
892
893 /* IV below built */
894 for (i = 0; i < 4; i++)
895 *(iv+i) = ctx->nonce[i];
896 for (i = 0; i < 8; i++)
897 *(iv+4+i) = req->iv[i];
898 *((__be32 *)(iv+12)) = counter;
899
900 return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
901 aes_ctx);
902 }
903
904 static int helper_rfc4106_decrypt(struct aead_request *req)
905 {
906 __be32 counter = cpu_to_be32(1);
907 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
908 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
909 void *aes_ctx = &(ctx->aes_key_expanded);
910 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
911 unsigned int i;
912
913 if (unlikely(req->assoclen != 16 && req->assoclen != 20))
914 return -EINVAL;
915
916 /* Assuming we are supporting rfc4106 64-bit extended */
917 /* sequence numbers We need to have the AAD length */
918 /* equal to 16 or 20 bytes */
919
920 /* IV below built */
921 for (i = 0; i < 4; i++)
922 *(iv+i) = ctx->nonce[i];
923 for (i = 0; i < 8; i++)
924 *(iv+4+i) = req->iv[i];
925 *((__be32 *)(iv+12)) = counter;
926
927 return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
928 aes_ctx);
929 }
930
931 static int rfc4106_encrypt(struct aead_request *req)
932 {
933 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
934 struct cryptd_aead **ctx = crypto_aead_ctx(tfm);
935 struct cryptd_aead *cryptd_tfm = *ctx;
936
937 tfm = &cryptd_tfm->base;
938 if (irq_fpu_usable() && (!in_atomic() ||
939 !cryptd_aead_queued(cryptd_tfm)))
940 tfm = cryptd_aead_child(cryptd_tfm);
941
942 aead_request_set_tfm(req, tfm);
943
944 return crypto_aead_encrypt(req);
945 }
946
947 static int rfc4106_decrypt(struct aead_request *req)
948 {
949 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
950 struct cryptd_aead **ctx = crypto_aead_ctx(tfm);
951 struct cryptd_aead *cryptd_tfm = *ctx;
952
953 tfm = &cryptd_tfm->base;
954 if (irq_fpu_usable() && (!in_atomic() ||
955 !cryptd_aead_queued(cryptd_tfm)))
956 tfm = cryptd_aead_child(cryptd_tfm);
957
958 aead_request_set_tfm(req, tfm);
959
960 return crypto_aead_decrypt(req);
961 }
962 #endif
963
964 static struct crypto_alg aesni_algs[] = { {
965 .cra_name = "aes",
966 .cra_driver_name = "aes-aesni",
967 .cra_priority = 300,
968 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
969 .cra_blocksize = AES_BLOCK_SIZE,
970 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
971 .cra_module = THIS_MODULE,
972 .cra_u = {
973 .cipher = {
974 .cia_min_keysize = AES_MIN_KEY_SIZE,
975 .cia_max_keysize = AES_MAX_KEY_SIZE,
976 .cia_setkey = aes_set_key,
977 .cia_encrypt = aes_encrypt,
978 .cia_decrypt = aes_decrypt
979 }
980 }
981 }, {
982 .cra_name = "__aes",
983 .cra_driver_name = "__aes-aesni",
984 .cra_priority = 300,
985 .cra_flags = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_INTERNAL,
986 .cra_blocksize = AES_BLOCK_SIZE,
987 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
988 .cra_module = THIS_MODULE,
989 .cra_u = {
990 .cipher = {
991 .cia_min_keysize = AES_MIN_KEY_SIZE,
992 .cia_max_keysize = AES_MAX_KEY_SIZE,
993 .cia_setkey = aes_set_key,
994 .cia_encrypt = __aes_encrypt,
995 .cia_decrypt = __aes_decrypt
996 }
997 }
998 } };
999
1000 static struct skcipher_alg aesni_skciphers[] = {
1001 {
1002 .base = {
1003 .cra_name = "__ecb(aes)",
1004 .cra_driver_name = "__ecb-aes-aesni",
1005 .cra_priority = 400,
1006 .cra_flags = CRYPTO_ALG_INTERNAL,
1007 .cra_blocksize = AES_BLOCK_SIZE,
1008 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
1009 .cra_module = THIS_MODULE,
1010 },
1011 .min_keysize = AES_MIN_KEY_SIZE,
1012 .max_keysize = AES_MAX_KEY_SIZE,
1013 .setkey = aesni_skcipher_setkey,
1014 .encrypt = ecb_encrypt,
1015 .decrypt = ecb_decrypt,
1016 }, {
1017 .base = {
1018 .cra_name = "__cbc(aes)",
1019 .cra_driver_name = "__cbc-aes-aesni",
1020 .cra_priority = 400,
1021 .cra_flags = CRYPTO_ALG_INTERNAL,
1022 .cra_blocksize = AES_BLOCK_SIZE,
1023 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
1024 .cra_module = THIS_MODULE,
1025 },
1026 .min_keysize = AES_MIN_KEY_SIZE,
1027 .max_keysize = AES_MAX_KEY_SIZE,
1028 .ivsize = AES_BLOCK_SIZE,
1029 .setkey = aesni_skcipher_setkey,
1030 .encrypt = cbc_encrypt,
1031 .decrypt = cbc_decrypt,
1032 #ifdef CONFIG_X86_64
1033 }, {
1034 .base = {
1035 .cra_name = "__ctr(aes)",
1036 .cra_driver_name = "__ctr-aes-aesni",
1037 .cra_priority = 400,
1038 .cra_flags = CRYPTO_ALG_INTERNAL,
1039 .cra_blocksize = 1,
1040 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
1041 .cra_module = THIS_MODULE,
1042 },
1043 .min_keysize = AES_MIN_KEY_SIZE,
1044 .max_keysize = AES_MAX_KEY_SIZE,
1045 .ivsize = AES_BLOCK_SIZE,
1046 .chunksize = AES_BLOCK_SIZE,
1047 .setkey = aesni_skcipher_setkey,
1048 .encrypt = ctr_crypt,
1049 .decrypt = ctr_crypt,
1050 }, {
1051 .base = {
1052 .cra_name = "__xts(aes)",
1053 .cra_driver_name = "__xts-aes-aesni",
1054 .cra_priority = 401,
1055 .cra_flags = CRYPTO_ALG_INTERNAL,
1056 .cra_blocksize = AES_BLOCK_SIZE,
1057 .cra_ctxsize = XTS_AES_CTX_SIZE,
1058 .cra_module = THIS_MODULE,
1059 },
1060 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1061 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1062 .ivsize = AES_BLOCK_SIZE,
1063 .setkey = xts_aesni_setkey,
1064 .encrypt = xts_encrypt,
1065 .decrypt = xts_decrypt,
1066 #endif
1067 }
1068 };
1069
1070 struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)];
1071
1072 struct {
1073 const char *algname;
1074 const char *drvname;
1075 const char *basename;
1076 struct simd_skcipher_alg *simd;
1077 } aesni_simd_skciphers2[] = {
1078 #if (defined(MODULE) && IS_ENABLED(CONFIG_CRYPTO_PCBC)) || \
1079 IS_BUILTIN(CONFIG_CRYPTO_PCBC)
1080 {
1081 .algname = "pcbc(aes)",
1082 .drvname = "pcbc-aes-aesni",
1083 .basename = "fpu(pcbc(__aes-aesni))",
1084 },
1085 #endif
1086 };
1087
1088 #ifdef CONFIG_X86_64
1089 static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key,
1090 unsigned int key_len)
1091 {
1092 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead);
1093
1094 return aes_set_key_common(crypto_aead_tfm(aead),
1095 &ctx->aes_key_expanded, key, key_len) ?:
1096 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
1097 }
1098
1099 static int generic_gcmaes_encrypt(struct aead_request *req)
1100 {
1101 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1102 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1103 void *aes_ctx = &(ctx->aes_key_expanded);
1104 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
1105 __be32 counter = cpu_to_be32(1);
1106
1107 memcpy(iv, req->iv, 12);
1108 *((__be32 *)(iv+12)) = counter;
1109
1110 return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv,
1111 aes_ctx);
1112 }
1113
1114 static int generic_gcmaes_decrypt(struct aead_request *req)
1115 {
1116 __be32 counter = cpu_to_be32(1);
1117 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1118 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1119 void *aes_ctx = &(ctx->aes_key_expanded);
1120 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
1121
1122 memcpy(iv, req->iv, 12);
1123 *((__be32 *)(iv+12)) = counter;
1124
1125 return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv,
1126 aes_ctx);
1127 }
1128
1129 static struct aead_alg aesni_aead_algs[] = { {
1130 .setkey = common_rfc4106_set_key,
1131 .setauthsize = common_rfc4106_set_authsize,
1132 .encrypt = helper_rfc4106_encrypt,
1133 .decrypt = helper_rfc4106_decrypt,
1134 .ivsize = 8,
1135 .maxauthsize = 16,
1136 .base = {
1137 .cra_name = "__gcm-aes-aesni",
1138 .cra_driver_name = "__driver-gcm-aes-aesni",
1139 .cra_flags = CRYPTO_ALG_INTERNAL,
1140 .cra_blocksize = 1,
1141 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx),
1142 .cra_alignmask = AESNI_ALIGN - 1,
1143 .cra_module = THIS_MODULE,
1144 },
1145 }, {
1146 .init = rfc4106_init,
1147 .exit = rfc4106_exit,
1148 .setkey = rfc4106_set_key,
1149 .setauthsize = rfc4106_set_authsize,
1150 .encrypt = rfc4106_encrypt,
1151 .decrypt = rfc4106_decrypt,
1152 .ivsize = 8,
1153 .maxauthsize = 16,
1154 .base = {
1155 .cra_name = "rfc4106(gcm(aes))",
1156 .cra_driver_name = "rfc4106-gcm-aesni",
1157 .cra_priority = 400,
1158 .cra_flags = CRYPTO_ALG_ASYNC,
1159 .cra_blocksize = 1,
1160 .cra_ctxsize = sizeof(struct cryptd_aead *),
1161 .cra_module = THIS_MODULE,
1162 },
1163 }, {
1164 .setkey = generic_gcmaes_set_key,
1165 .setauthsize = generic_gcmaes_set_authsize,
1166 .encrypt = generic_gcmaes_encrypt,
1167 .decrypt = generic_gcmaes_decrypt,
1168 .ivsize = 12,
1169 .maxauthsize = 16,
1170 .base = {
1171 .cra_name = "gcm(aes)",
1172 .cra_driver_name = "generic-gcm-aesni",
1173 .cra_priority = 400,
1174 .cra_flags = CRYPTO_ALG_ASYNC,
1175 .cra_blocksize = 1,
1176 .cra_ctxsize = sizeof(struct generic_gcmaes_ctx),
1177 .cra_alignmask = AESNI_ALIGN - 1,
1178 .cra_module = THIS_MODULE,
1179 },
1180 } };
1181 #else
1182 static struct aead_alg aesni_aead_algs[0];
1183 #endif
1184
1185
1186 static const struct x86_cpu_id aesni_cpu_id[] = {
1187 X86_FEATURE_MATCH(X86_FEATURE_AES),
1188 {}
1189 };
1190 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1191
1192 static void aesni_free_simds(void)
1193 {
1194 int i;
1195
1196 for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers) &&
1197 aesni_simd_skciphers[i]; i++)
1198 simd_skcipher_free(aesni_simd_skciphers[i]);
1199
1200 for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers2); i++)
1201 if (aesni_simd_skciphers2[i].simd)
1202 simd_skcipher_free(aesni_simd_skciphers2[i].simd);
1203 }
1204
1205 static int __init aesni_init(void)
1206 {
1207 struct simd_skcipher_alg *simd;
1208 const char *basename;
1209 const char *algname;
1210 const char *drvname;
1211 int err;
1212 int i;
1213
1214 if (!x86_match_cpu(aesni_cpu_id))
1215 return -ENODEV;
1216 #ifdef CONFIG_X86_64
1217 #ifdef CONFIG_AS_AVX2
1218 if (boot_cpu_has(X86_FEATURE_AVX2)) {
1219 pr_info("AVX2 version of gcm_enc/dec engaged.\n");
1220 aesni_gcm_enc_tfm = aesni_gcm_enc_avx2;
1221 aesni_gcm_dec_tfm = aesni_gcm_dec_avx2;
1222 } else
1223 #endif
1224 #ifdef CONFIG_AS_AVX
1225 if (boot_cpu_has(X86_FEATURE_AVX)) {
1226 pr_info("AVX version of gcm_enc/dec engaged.\n");
1227 aesni_gcm_enc_tfm = aesni_gcm_enc_avx;
1228 aesni_gcm_dec_tfm = aesni_gcm_dec_avx;
1229 } else
1230 #endif
1231 {
1232 pr_info("SSE version of gcm_enc/dec engaged.\n");
1233 aesni_gcm_enc_tfm = aesni_gcm_enc;
1234 aesni_gcm_dec_tfm = aesni_gcm_dec;
1235 }
1236 aesni_ctr_enc_tfm = aesni_ctr_enc;
1237 #ifdef CONFIG_AS_AVX
1238 if (boot_cpu_has(X86_FEATURE_AVX)) {
1239 /* optimize performance of ctr mode encryption transform */
1240 aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm;
1241 pr_info("AES CTR mode by8 optimization enabled\n");
1242 }
1243 #endif
1244 #endif
1245
1246 err = crypto_fpu_init();
1247 if (err)
1248 return err;
1249
1250 err = crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1251 if (err)
1252 goto fpu_exit;
1253
1254 err = crypto_register_skciphers(aesni_skciphers,
1255 ARRAY_SIZE(aesni_skciphers));
1256 if (err)
1257 goto unregister_algs;
1258
1259 err = crypto_register_aeads(aesni_aead_algs,
1260 ARRAY_SIZE(aesni_aead_algs));
1261 if (err)
1262 goto unregister_skciphers;
1263
1264 for (i = 0; i < ARRAY_SIZE(aesni_skciphers); i++) {
1265 algname = aesni_skciphers[i].base.cra_name + 2;
1266 drvname = aesni_skciphers[i].base.cra_driver_name + 2;
1267 basename = aesni_skciphers[i].base.cra_driver_name;
1268 simd = simd_skcipher_create_compat(algname, drvname, basename);
1269 err = PTR_ERR(simd);
1270 if (IS_ERR(simd))
1271 goto unregister_simds;
1272
1273 aesni_simd_skciphers[i] = simd;
1274 }
1275
1276 for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers2); i++) {
1277 algname = aesni_simd_skciphers2[i].algname;
1278 drvname = aesni_simd_skciphers2[i].drvname;
1279 basename = aesni_simd_skciphers2[i].basename;
1280 simd = simd_skcipher_create_compat(algname, drvname, basename);
1281 err = PTR_ERR(simd);
1282 if (IS_ERR(simd))
1283 continue;
1284
1285 aesni_simd_skciphers2[i].simd = simd;
1286 }
1287
1288 return 0;
1289
1290 unregister_simds:
1291 aesni_free_simds();
1292 crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs));
1293 unregister_skciphers:
1294 crypto_unregister_skciphers(aesni_skciphers,
1295 ARRAY_SIZE(aesni_skciphers));
1296 unregister_algs:
1297 crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1298 fpu_exit:
1299 crypto_fpu_exit();
1300 return err;
1301 }
1302
1303 static void __exit aesni_exit(void)
1304 {
1305 aesni_free_simds();
1306 crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs));
1307 crypto_unregister_skciphers(aesni_skciphers,
1308 ARRAY_SIZE(aesni_skciphers));
1309 crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1310
1311 crypto_fpu_exit();
1312 }
1313
1314 late_initcall(aesni_init);
1315 module_exit(aesni_exit);
1316
1317 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1318 MODULE_LICENSE("GPL");
1319 MODULE_ALIAS_CRYPTO("aes");
Linux with added FPC-III support