?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost
[cris-mirror.git] / arch / x86 / crypto / aesni-intel_glue.c
blob34cf1c1f8c983367a0443d1343e718e9ded32c05
1 /*
2 * Support for Intel AES-NI instructions. This file contains glue
3 * code, the real AES implementation is in intel-aes_asm.S.
4 *
5 * Copyright (C) 2008, Intel Corp.
6 * Author: Huang Ying <ying.huang@intel.com>
7 *
8 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
9 * interface for 64-bit kernels.
10 * Authors: Adrian Hoban <adrian.hoban@intel.com>
11 * Gabriele Paoloni <gabriele.paoloni@intel.com>
12 * Tadeusz Struk (tadeusz.struk@intel.com)
13 * Aidan O'Mahony (aidan.o.mahony@intel.com)
14 * Copyright (c) 2010, Intel Corporation.
15 *
16 * This program is free software; you can redistribute it and/or modify
17 * it under the terms of the GNU General Public License as published by
18 * the Free Software Foundation; either version 2 of the License, or
19 * (at your option) any later version.
20 */
21
22 #include <linux/hardirq.h>
23 #include <linux/types.h>
24 #include <linux/module.h>
25 #include <linux/err.h>
26 #include <crypto/algapi.h>
27 #include <crypto/aes.h>
28 #include <crypto/cryptd.h>
29 #include <crypto/ctr.h>
30 #include <crypto/b128ops.h>
31 #include <crypto/gcm.h>
32 #include <crypto/xts.h>
33 #include <asm/cpu_device_id.h>
34 #include <asm/fpu/api.h>
35 #include <asm/crypto/aes.h>
36 #include <crypto/scatterwalk.h>
37 #include <crypto/internal/aead.h>
38 #include <crypto/internal/simd.h>
39 #include <crypto/internal/skcipher.h>
40 #include <linux/workqueue.h>
41 #include <linux/spinlock.h>
42 #ifdef CONFIG_X86_64
43 #include <asm/crypto/glue_helper.h>
44 #endif
45
46
47 #define AESNI_ALIGN 16
48 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
49 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE - 1))
50 #define RFC4106_HASH_SUBKEY_SIZE 16
51 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
52 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
53 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
54
55 /* This data is stored at the end of the crypto_tfm struct.
56 * It's a type of per "session" data storage location.
57 * This needs to be 16 byte aligned.
58 */
59 struct aesni_rfc4106_gcm_ctx {
60 u8 hash_subkey[16] AESNI_ALIGN_ATTR;
61 struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
62 u8 nonce[4];
63 };
64
65 struct generic_gcmaes_ctx {
66 u8 hash_subkey[16] AESNI_ALIGN_ATTR;
67 struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
68 };
69
70 struct aesni_xts_ctx {
71 u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
72 u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
73 };
74
75 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
76 unsigned int key_len);
77 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
78 const u8 *in);
79 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
80 const u8 *in);
81 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
82 const u8 *in, unsigned int len);
83 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
84 const u8 *in, unsigned int len);
85 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
86 const u8 *in, unsigned int len, u8 *iv);
87 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
88 const u8 *in, unsigned int len, u8 *iv);
89
90 int crypto_fpu_init(void);
91 void crypto_fpu_exit(void);
92
93 #define AVX_GEN2_OPTSIZE 640
94 #define AVX_GEN4_OPTSIZE 4096
95
96 #ifdef CONFIG_X86_64
97
98 static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out,
99 const u8 *in, unsigned int len, u8 *iv);
100 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
101 const u8 *in, unsigned int len, u8 *iv);
102
103 asmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out,
104 const u8 *in, bool enc, u8 *iv);
105
106 /* asmlinkage void aesni_gcm_enc()
107 * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
108 * u8 *out, Ciphertext output. Encrypt in-place is allowed.
109 * const u8 *in, Plaintext input
110 * unsigned long plaintext_len, Length of data in bytes for encryption.
111 * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
112 * 16-byte aligned pointer.
113 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
114 * const u8 *aad, Additional Authentication Data (AAD)
115 * unsigned long aad_len, Length of AAD in bytes.
116 * u8 *auth_tag, Authenticated Tag output.
117 * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
118 * Valid values are 16 (most likely), 12 or 8.
119 */
120 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out,
121 const u8 *in, unsigned long plaintext_len, u8 *iv,
122 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
123 u8 *auth_tag, unsigned long auth_tag_len);
124
125 /* asmlinkage void aesni_gcm_dec()
126 * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
127 * u8 *out, Plaintext output. Decrypt in-place is allowed.
128 * const u8 *in, Ciphertext input
129 * unsigned long ciphertext_len, Length of data in bytes for decryption.
130 * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
131 * 16-byte aligned pointer.
132 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
133 * const u8 *aad, Additional Authentication Data (AAD)
134 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
135 * to be 8 or 12 bytes
136 * u8 *auth_tag, Authenticated Tag output.
137 * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
138 * Valid values are 16 (most likely), 12 or 8.
139 */
140 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out,
141 const u8 *in, unsigned long ciphertext_len, u8 *iv,
142 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
143 u8 *auth_tag, unsigned long auth_tag_len);
144
145
146 #ifdef CONFIG_AS_AVX
147 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,
148 void *keys, u8 *out, unsigned int num_bytes);
149 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
150 void *keys, u8 *out, unsigned int num_bytes);
151 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
152 void *keys, u8 *out, unsigned int num_bytes);
153 /*
154 * asmlinkage void aesni_gcm_precomp_avx_gen2()
155 * gcm_data *my_ctx_data, context data
156 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
157 */
158 asmlinkage void aesni_gcm_precomp_avx_gen2(void *my_ctx_data, u8 *hash_subkey);
159
160 asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx, u8 *out,
161 const u8 *in, unsigned long plaintext_len, u8 *iv,
162 const u8 *aad, unsigned long aad_len,
163 u8 *auth_tag, unsigned long auth_tag_len);
164
165 asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx, u8 *out,
166 const u8 *in, unsigned long ciphertext_len, u8 *iv,
167 const u8 *aad, unsigned long aad_len,
168 u8 *auth_tag, unsigned long auth_tag_len);
169
170 static void aesni_gcm_enc_avx(void *ctx, u8 *out,
171 const u8 *in, unsigned long plaintext_len, u8 *iv,
172 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
173 u8 *auth_tag, unsigned long auth_tag_len)
174 {
175 struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
176 if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)){
177 aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad,
178 aad_len, auth_tag, auth_tag_len);
179 } else {
180 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
181 aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad,
182 aad_len, auth_tag, auth_tag_len);
183 }
184 }
185
186 static void aesni_gcm_dec_avx(void *ctx, u8 *out,
187 const u8 *in, unsigned long ciphertext_len, u8 *iv,
188 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
189 u8 *auth_tag, unsigned long auth_tag_len)
190 {
191 struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
192 if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) {
193 aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey, aad,
194 aad_len, auth_tag, auth_tag_len);
195 } else {
196 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
197 aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad,
198 aad_len, auth_tag, auth_tag_len);
199 }
200 }
201 #endif
202
203 #ifdef CONFIG_AS_AVX2
204 /*
205 * asmlinkage void aesni_gcm_precomp_avx_gen4()
206 * gcm_data *my_ctx_data, context data
207 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
208 */
209 asmlinkage void aesni_gcm_precomp_avx_gen4(void *my_ctx_data, u8 *hash_subkey);
210
211 asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx, u8 *out,
212 const u8 *in, unsigned long plaintext_len, u8 *iv,
213 const u8 *aad, unsigned long aad_len,
214 u8 *auth_tag, unsigned long auth_tag_len);
215
216 asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx, u8 *out,
217 const u8 *in, unsigned long ciphertext_len, u8 *iv,
218 const u8 *aad, unsigned long aad_len,
219 u8 *auth_tag, unsigned long auth_tag_len);
220
221 static void aesni_gcm_enc_avx2(void *ctx, u8 *out,
222 const u8 *in, unsigned long plaintext_len, u8 *iv,
223 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
224 u8 *auth_tag, unsigned long auth_tag_len)
225 {
226 struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
227 if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) {
228 aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad,
229 aad_len, auth_tag, auth_tag_len);
230 } else if (plaintext_len < AVX_GEN4_OPTSIZE) {
231 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
232 aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad,
233 aad_len, auth_tag, auth_tag_len);
234 } else {
235 aesni_gcm_precomp_avx_gen4(ctx, hash_subkey);
236 aesni_gcm_enc_avx_gen4(ctx, out, in, plaintext_len, iv, aad,
237 aad_len, auth_tag, auth_tag_len);
238 }
239 }
240
241 static void aesni_gcm_dec_avx2(void *ctx, u8 *out,
242 const u8 *in, unsigned long ciphertext_len, u8 *iv,
243 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
244 u8 *auth_tag, unsigned long auth_tag_len)
245 {
246 struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
247 if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) {
248 aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey,
249 aad, aad_len, auth_tag, auth_tag_len);
250 } else if (ciphertext_len < AVX_GEN4_OPTSIZE) {
251 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
252 aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad,
253 aad_len, auth_tag, auth_tag_len);
254 } else {
255 aesni_gcm_precomp_avx_gen4(ctx, hash_subkey);
256 aesni_gcm_dec_avx_gen4(ctx, out, in, ciphertext_len, iv, aad,
257 aad_len, auth_tag, auth_tag_len);
258 }
259 }
260 #endif
261
262 static void (*aesni_gcm_enc_tfm)(void *ctx, u8 *out,
263 const u8 *in, unsigned long plaintext_len, u8 *iv,
264 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
265 u8 *auth_tag, unsigned long auth_tag_len);
266
267 static void (*aesni_gcm_dec_tfm)(void *ctx, u8 *out,
268 const u8 *in, unsigned long ciphertext_len, u8 *iv,
269 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
270 u8 *auth_tag, unsigned long auth_tag_len);
271
272 static inline struct
273 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
274 {
275 unsigned long align = AESNI_ALIGN;
276
277 if (align <= crypto_tfm_ctx_alignment())
278 align = 1;
279 return PTR_ALIGN(crypto_aead_ctx(tfm), align);
280 }
281
282 static inline struct
283 generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm)
284 {
285 unsigned long align = AESNI_ALIGN;
286
287 if (align <= crypto_tfm_ctx_alignment())
288 align = 1;
289 return PTR_ALIGN(crypto_aead_ctx(tfm), align);
290 }
291 #endif
292
293 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
294 {
295 unsigned long addr = (unsigned long)raw_ctx;
296 unsigned long align = AESNI_ALIGN;
297
298 if (align <= crypto_tfm_ctx_alignment())
299 align = 1;
300 return (struct crypto_aes_ctx *)ALIGN(addr, align);
301 }
302
303 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
304 const u8 *in_key, unsigned int key_len)
305 {
306 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
307 u32 *flags = &tfm->crt_flags;
308 int err;
309
310 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
311 key_len != AES_KEYSIZE_256) {
312 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
313 return -EINVAL;
314 }
315
316 if (!irq_fpu_usable())
317 err = crypto_aes_expand_key(ctx, in_key, key_len);
318 else {
319 kernel_fpu_begin();
320 err = aesni_set_key(ctx, in_key, key_len);
321 kernel_fpu_end();
322 }
323
324 return err;
325 }
326
327 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
328 unsigned int key_len)
329 {
330 return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
331 }
332
333 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
334 {
335 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
336
337 if (!irq_fpu_usable())
338 crypto_aes_encrypt_x86(ctx, dst, src);
339 else {
340 kernel_fpu_begin();
341 aesni_enc(ctx, dst, src);
342 kernel_fpu_end();
343 }
344 }
345
346 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
347 {
348 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
349
350 if (!irq_fpu_usable())
351 crypto_aes_decrypt_x86(ctx, dst, src);
352 else {
353 kernel_fpu_begin();
354 aesni_dec(ctx, dst, src);
355 kernel_fpu_end();
356 }
357 }
358
359 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
360 {
361 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
362
363 aesni_enc(ctx, dst, src);
364 }
365
366 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
367 {
368 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
369
370 aesni_dec(ctx, dst, src);
371 }
372
373 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
374 unsigned int len)
375 {
376 return aes_set_key_common(crypto_skcipher_tfm(tfm),
377 crypto_skcipher_ctx(tfm), key, len);
378 }
379
380 static int ecb_encrypt(struct skcipher_request *req)
381 {
382 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
383 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
384 struct skcipher_walk walk;
385 unsigned int nbytes;
386 int err;
387
388 err = skcipher_walk_virt(&walk, req, true);
389
390 kernel_fpu_begin();
391 while ((nbytes = walk.nbytes)) {
392 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
393 nbytes & AES_BLOCK_MASK);
394 nbytes &= AES_BLOCK_SIZE - 1;
395 err = skcipher_walk_done(&walk, nbytes);
396 }
397 kernel_fpu_end();
398
399 return err;
400 }
401
402 static int ecb_decrypt(struct skcipher_request *req)
403 {
404 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
405 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
406 struct skcipher_walk walk;
407 unsigned int nbytes;
408 int err;
409
410 err = skcipher_walk_virt(&walk, req, true);
411
412 kernel_fpu_begin();
413 while ((nbytes = walk.nbytes)) {
414 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
415 nbytes & AES_BLOCK_MASK);
416 nbytes &= AES_BLOCK_SIZE - 1;
417 err = skcipher_walk_done(&walk, nbytes);
418 }
419 kernel_fpu_end();
420
421 return err;
422 }
423
424 static int cbc_encrypt(struct skcipher_request *req)
425 {
426 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
427 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
428 struct skcipher_walk walk;
429 unsigned int nbytes;
430 int err;
431
432 err = skcipher_walk_virt(&walk, req, true);
433
434 kernel_fpu_begin();
435 while ((nbytes = walk.nbytes)) {
436 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
437 nbytes & AES_BLOCK_MASK, walk.iv);
438 nbytes &= AES_BLOCK_SIZE - 1;
439 err = skcipher_walk_done(&walk, nbytes);
440 }
441 kernel_fpu_end();
442
443 return err;
444 }
445
446 static int cbc_decrypt(struct skcipher_request *req)
447 {
448 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
449 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
450 struct skcipher_walk walk;
451 unsigned int nbytes;
452 int err;
453
454 err = skcipher_walk_virt(&walk, req, true);
455
456 kernel_fpu_begin();
457 while ((nbytes = walk.nbytes)) {
458 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
459 nbytes & AES_BLOCK_MASK, walk.iv);
460 nbytes &= AES_BLOCK_SIZE - 1;
461 err = skcipher_walk_done(&walk, nbytes);
462 }
463 kernel_fpu_end();
464
465 return err;
466 }
467
468 #ifdef CONFIG_X86_64
469 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
470 struct skcipher_walk *walk)
471 {
472 u8 *ctrblk = walk->iv;
473 u8 keystream[AES_BLOCK_SIZE];
474 u8 *src = walk->src.virt.addr;
475 u8 *dst = walk->dst.virt.addr;
476 unsigned int nbytes = walk->nbytes;
477
478 aesni_enc(ctx, keystream, ctrblk);
479 crypto_xor_cpy(dst, keystream, src, nbytes);
480
481 crypto_inc(ctrblk, AES_BLOCK_SIZE);
482 }
483
484 #ifdef CONFIG_AS_AVX
485 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
486 const u8 *in, unsigned int len, u8 *iv)
487 {
488 /*
489 * based on key length, override with the by8 version
490 * of ctr mode encryption/decryption for improved performance
491 * aes_set_key_common() ensures that key length is one of
492 * {128,192,256}
493 */
494 if (ctx->key_length == AES_KEYSIZE_128)
495 aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len);
496 else if (ctx->key_length == AES_KEYSIZE_192)
497 aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len);
498 else
499 aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);
500 }
501 #endif
502
503 static int ctr_crypt(struct skcipher_request *req)
504 {
505 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
506 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
507 struct skcipher_walk walk;
508 unsigned int nbytes;
509 int err;
510
511 err = skcipher_walk_virt(&walk, req, true);
512
513 kernel_fpu_begin();
514 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
515 aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr,
516 nbytes & AES_BLOCK_MASK, walk.iv);
517 nbytes &= AES_BLOCK_SIZE - 1;
518 err = skcipher_walk_done(&walk, nbytes);
519 }
520 if (walk.nbytes) {
521 ctr_crypt_final(ctx, &walk);
522 err = skcipher_walk_done(&walk, 0);
523 }
524 kernel_fpu_end();
525
526 return err;
527 }
528
529 static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key,
530 unsigned int keylen)
531 {
532 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
533 int err;
534
535 err = xts_verify_key(tfm, key, keylen);
536 if (err)
537 return err;
538
539 keylen /= 2;
540
541 /* first half of xts-key is for crypt */
542 err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx,
543 key, keylen);
544 if (err)
545 return err;
546
547 /* second half of xts-key is for tweak */
548 return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx,
549 key + keylen, keylen);
550 }
551
552
553 static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in)
554 {
555 aesni_enc(ctx, out, in);
556 }
557
558 static void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
559 {
560 glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc));
561 }
562
563 static void aesni_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
564 {
565 glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_dec));
566 }
567
568 static void aesni_xts_enc8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
569 {
570 aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, true, (u8 *)iv);
571 }
572
573 static void aesni_xts_dec8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
574 {
575 aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, false, (u8 *)iv);
576 }
577
578 static const struct common_glue_ctx aesni_enc_xts = {
579 .num_funcs = 2,
580 .fpu_blocks_limit = 1,
581
582 .funcs = { {
583 .num_blocks = 8,
584 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc8) }
585 }, {
586 .num_blocks = 1,
587 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc) }
588 } }
589 };
590
591 static const struct common_glue_ctx aesni_dec_xts = {
592 .num_funcs = 2,
593 .fpu_blocks_limit = 1,
594
595 .funcs = { {
596 .num_blocks = 8,
597 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec8) }
598 }, {
599 .num_blocks = 1,
600 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec) }
601 } }
602 };
603
604 static int xts_encrypt(struct skcipher_request *req)
605 {
606 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
607 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
608
609 return glue_xts_req_128bit(&aesni_enc_xts, req,
610 XTS_TWEAK_CAST(aesni_xts_tweak),
611 aes_ctx(ctx->raw_tweak_ctx),
612 aes_ctx(ctx->raw_crypt_ctx));
613 }
614
615 static int xts_decrypt(struct skcipher_request *req)
616 {
617 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
618 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
619
620 return glue_xts_req_128bit(&aesni_dec_xts, req,
621 XTS_TWEAK_CAST(aesni_xts_tweak),
622 aes_ctx(ctx->raw_tweak_ctx),
623 aes_ctx(ctx->raw_crypt_ctx));
624 }
625
626 static int rfc4106_init(struct crypto_aead *aead)
627 {
628 struct cryptd_aead *cryptd_tfm;
629 struct cryptd_aead **ctx = crypto_aead_ctx(aead);
630
631 cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni",
632 CRYPTO_ALG_INTERNAL,
633 CRYPTO_ALG_INTERNAL);
634 if (IS_ERR(cryptd_tfm))
635 return PTR_ERR(cryptd_tfm);
636
637 *ctx = cryptd_tfm;
638 crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
639 return 0;
640 }
641
642 static void rfc4106_exit(struct crypto_aead *aead)
643 {
644 struct cryptd_aead **ctx = crypto_aead_ctx(aead);
645
646 cryptd_free_aead(*ctx);
647 }
648
649 static int
650 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
651 {
652 struct crypto_cipher *tfm;
653 int ret;
654
655 tfm = crypto_alloc_cipher("aes", 0, 0);
656 if (IS_ERR(tfm))
657 return PTR_ERR(tfm);
658
659 ret = crypto_cipher_setkey(tfm, key, key_len);
660 if (ret)
661 goto out_free_cipher;
662
663 /* Clear the data in the hash sub key container to zero.*/
664 /* We want to cipher all zeros to create the hash sub key. */
665 memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
666
667 crypto_cipher_encrypt_one(tfm, hash_subkey, hash_subkey);
668
669 out_free_cipher:
670 crypto_free_cipher(tfm);
671 return ret;
672 }
673
674 static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
675 unsigned int key_len)
676 {
677 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead);
678
679 if (key_len < 4) {
680 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
681 return -EINVAL;
682 }
683 /*Account for 4 byte nonce at the end.*/
684 key_len -= 4;
685
686 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
687
688 return aes_set_key_common(crypto_aead_tfm(aead),
689 &ctx->aes_key_expanded, key, key_len) ?:
690 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
691 }
692
693 static int gcmaes_wrapper_set_key(struct crypto_aead *parent, const u8 *key,
694 unsigned int key_len)
695 {
696 struct cryptd_aead **ctx = crypto_aead_ctx(parent);
697 struct cryptd_aead *cryptd_tfm = *ctx;
698
699 return crypto_aead_setkey(&cryptd_tfm->base, key, key_len);
700 }
701
702 static int common_rfc4106_set_authsize(struct crypto_aead *aead,
703 unsigned int authsize)
704 {
705 switch (authsize) {
706 case 8:
707 case 12:
708 case 16:
709 break;
710 default:
711 return -EINVAL;
712 }
713
714 return 0;
715 }
716
717 /* This is the Integrity Check Value (aka the authentication tag length and can
718 * be 8, 12 or 16 bytes long. */
719 static int gcmaes_wrapper_set_authsize(struct crypto_aead *parent,
720 unsigned int authsize)
721 {
722 struct cryptd_aead **ctx = crypto_aead_ctx(parent);
723 struct cryptd_aead *cryptd_tfm = *ctx;
724
725 return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
726 }
727
728 static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,
729 unsigned int authsize)
730 {
731 switch (authsize) {
732 case 4:
733 case 8:
734 case 12:
735 case 13:
736 case 14:
737 case 15:
738 case 16:
739 break;
740 default:
741 return -EINVAL;
742 }
743
744 return 0;
745 }
746
747 static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen,
748 u8 *hash_subkey, u8 *iv, void *aes_ctx)
749 {
750 u8 one_entry_in_sg = 0;
751 u8 *src, *dst, *assoc;
752 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
753 unsigned long auth_tag_len = crypto_aead_authsize(tfm);
754 struct scatter_walk src_sg_walk;
755 struct scatter_walk dst_sg_walk = {};
756
757 if (sg_is_last(req->src) &&
758 (!PageHighMem(sg_page(req->src)) ||
759 req->src->offset + req->src->length <= PAGE_SIZE) &&
760 sg_is_last(req->dst) &&
761 (!PageHighMem(sg_page(req->dst)) ||
762 req->dst->offset + req->dst->length <= PAGE_SIZE)) {
763 one_entry_in_sg = 1;
764 scatterwalk_start(&src_sg_walk, req->src);
765 assoc = scatterwalk_map(&src_sg_walk);
766 src = assoc + req->assoclen;
767 dst = src;
768 if (unlikely(req->src != req->dst)) {
769 scatterwalk_start(&dst_sg_walk, req->dst);
770 dst = scatterwalk_map(&dst_sg_walk) + req->assoclen;
771 }
772 } else {
773 /* Allocate memory for src, dst, assoc */
774 assoc = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
775 GFP_ATOMIC);
776 if (unlikely(!assoc))
777 return -ENOMEM;
778 scatterwalk_map_and_copy(assoc, req->src, 0,
779 req->assoclen + req->cryptlen, 0);
780 src = assoc + req->assoclen;
781 dst = src;
782 }
783
784 kernel_fpu_begin();
785 aesni_gcm_enc_tfm(aes_ctx, dst, src, req->cryptlen, iv,
786 hash_subkey, assoc, assoclen,
787 dst + req->cryptlen, auth_tag_len);
788 kernel_fpu_end();
789
790 /* The authTag (aka the Integrity Check Value) needs to be written
791 * back to the packet. */
792 if (one_entry_in_sg) {
793 if (unlikely(req->src != req->dst)) {
794 scatterwalk_unmap(dst - req->assoclen);
795 scatterwalk_advance(&dst_sg_walk, req->dst->length);
796 scatterwalk_done(&dst_sg_walk, 1, 0);
797 }
798 scatterwalk_unmap(assoc);
799 scatterwalk_advance(&src_sg_walk, req->src->length);
800 scatterwalk_done(&src_sg_walk, req->src == req->dst, 0);
801 } else {
802 scatterwalk_map_and_copy(dst, req->dst, req->assoclen,
803 req->cryptlen + auth_tag_len, 1);
804 kfree(assoc);
805 }
806 return 0;
807 }
808
809 static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen,
810 u8 *hash_subkey, u8 *iv, void *aes_ctx)
811 {
812 u8 one_entry_in_sg = 0;
813 u8 *src, *dst, *assoc;
814 unsigned long tempCipherLen = 0;
815 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
816 unsigned long auth_tag_len = crypto_aead_authsize(tfm);
817 u8 authTag[16];
818 struct scatter_walk src_sg_walk;
819 struct scatter_walk dst_sg_walk = {};
820 int retval = 0;
821
822 tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
823
824 if (sg_is_last(req->src) &&
825 (!PageHighMem(sg_page(req->src)) ||
826 req->src->offset + req->src->length <= PAGE_SIZE) &&
827 sg_is_last(req->dst) && req->dst->length &&
828 (!PageHighMem(sg_page(req->dst)) ||
829 req->dst->offset + req->dst->length <= PAGE_SIZE)) {
830 one_entry_in_sg = 1;
831 scatterwalk_start(&src_sg_walk, req->src);
832 assoc = scatterwalk_map(&src_sg_walk);
833 src = assoc + req->assoclen;
834 dst = src;
835 if (unlikely(req->src != req->dst)) {
836 scatterwalk_start(&dst_sg_walk, req->dst);
837 dst = scatterwalk_map(&dst_sg_walk) + req->assoclen;
838 }
839 } else {
840 /* Allocate memory for src, dst, assoc */
841 assoc = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
842 if (!assoc)
843 return -ENOMEM;
844 scatterwalk_map_and_copy(assoc, req->src, 0,
845 req->assoclen + req->cryptlen, 0);
846 src = assoc + req->assoclen;
847 dst = src;
848 }
849
850
851 kernel_fpu_begin();
852 aesni_gcm_dec_tfm(aes_ctx, dst, src, tempCipherLen, iv,
853 hash_subkey, assoc, assoclen,
854 authTag, auth_tag_len);
855 kernel_fpu_end();
856
857 /* Compare generated tag with passed in tag. */
858 retval = crypto_memneq(src + tempCipherLen, authTag, auth_tag_len) ?
859 -EBADMSG : 0;
860
861 if (one_entry_in_sg) {
862 if (unlikely(req->src != req->dst)) {
863 scatterwalk_unmap(dst - req->assoclen);
864 scatterwalk_advance(&dst_sg_walk, req->dst->length);
865 scatterwalk_done(&dst_sg_walk, 1, 0);
866 }
867 scatterwalk_unmap(assoc);
868 scatterwalk_advance(&src_sg_walk, req->src->length);
869 scatterwalk_done(&src_sg_walk, req->src == req->dst, 0);
870 } else {
871 scatterwalk_map_and_copy(dst, req->dst, req->assoclen,
872 tempCipherLen, 1);
873 kfree(assoc);
874 }
875 return retval;
876
877 }
878
879 static int helper_rfc4106_encrypt(struct aead_request *req)
880 {
881 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
882 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
883 void *aes_ctx = &(ctx->aes_key_expanded);
884 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
885 unsigned int i;
886 __be32 counter = cpu_to_be32(1);
887
888 /* Assuming we are supporting rfc4106 64-bit extended */
889 /* sequence numbers We need to have the AAD length equal */
890 /* to 16 or 20 bytes */
891 if (unlikely(req->assoclen != 16 && req->assoclen != 20))
892 return -EINVAL;
893
894 /* IV below built */
895 for (i = 0; i < 4; i++)
896 *(iv+i) = ctx->nonce[i];
897 for (i = 0; i < 8; i++)
898 *(iv+4+i) = req->iv[i];
899 *((__be32 *)(iv+12)) = counter;
900
901 return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
902 aes_ctx);
903 }
904
905 static int helper_rfc4106_decrypt(struct aead_request *req)
906 {
907 __be32 counter = cpu_to_be32(1);
908 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
909 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
910 void *aes_ctx = &(ctx->aes_key_expanded);
911 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
912 unsigned int i;
913
914 if (unlikely(req->assoclen != 16 && req->assoclen != 20))
915 return -EINVAL;
916
917 /* Assuming we are supporting rfc4106 64-bit extended */
918 /* sequence numbers We need to have the AAD length */
919 /* equal to 16 or 20 bytes */
920
921 /* IV below built */
922 for (i = 0; i < 4; i++)
923 *(iv+i) = ctx->nonce[i];
924 for (i = 0; i < 8; i++)
925 *(iv+4+i) = req->iv[i];
926 *((__be32 *)(iv+12)) = counter;
927
928 return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
929 aes_ctx);
930 }
931
932 static int gcmaes_wrapper_encrypt(struct aead_request *req)
933 {
934 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
935 struct cryptd_aead **ctx = crypto_aead_ctx(tfm);
936 struct cryptd_aead *cryptd_tfm = *ctx;
937
938 tfm = &cryptd_tfm->base;
939 if (irq_fpu_usable() && (!in_atomic() ||
940 !cryptd_aead_queued(cryptd_tfm)))
941 tfm = cryptd_aead_child(cryptd_tfm);
942
943 aead_request_set_tfm(req, tfm);
944
945 return crypto_aead_encrypt(req);
946 }
947
948 static int gcmaes_wrapper_decrypt(struct aead_request *req)
949 {
950 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
951 struct cryptd_aead **ctx = crypto_aead_ctx(tfm);
952 struct cryptd_aead *cryptd_tfm = *ctx;
953
954 tfm = &cryptd_tfm->base;
955 if (irq_fpu_usable() && (!in_atomic() ||
956 !cryptd_aead_queued(cryptd_tfm)))
957 tfm = cryptd_aead_child(cryptd_tfm);
958
959 aead_request_set_tfm(req, tfm);
960
961 return crypto_aead_decrypt(req);
962 }
963 #endif
964
965 static struct crypto_alg aesni_algs[] = { {
966 .cra_name = "aes",
967 .cra_driver_name = "aes-aesni",
968 .cra_priority = 300,
969 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
970 .cra_blocksize = AES_BLOCK_SIZE,
971 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
972 .cra_module = THIS_MODULE,
973 .cra_u = {
974 .cipher = {
975 .cia_min_keysize = AES_MIN_KEY_SIZE,
976 .cia_max_keysize = AES_MAX_KEY_SIZE,
977 .cia_setkey = aes_set_key,
978 .cia_encrypt = aes_encrypt,
979 .cia_decrypt = aes_decrypt
980 }
981 }
982 }, {
983 .cra_name = "__aes",
984 .cra_driver_name = "__aes-aesni",
985 .cra_priority = 300,
986 .cra_flags = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_INTERNAL,
987 .cra_blocksize = AES_BLOCK_SIZE,
988 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
989 .cra_module = THIS_MODULE,
990 .cra_u = {
991 .cipher = {
992 .cia_min_keysize = AES_MIN_KEY_SIZE,
993 .cia_max_keysize = AES_MAX_KEY_SIZE,
994 .cia_setkey = aes_set_key,
995 .cia_encrypt = __aes_encrypt,
996 .cia_decrypt = __aes_decrypt
997 }
998 }
999 } };
1000
1001 static struct skcipher_alg aesni_skciphers[] = {
1002 {
1003 .base = {
1004 .cra_name = "__ecb(aes)",
1005 .cra_driver_name = "__ecb-aes-aesni",
1006 .cra_priority = 400,
1007 .cra_flags = CRYPTO_ALG_INTERNAL,
1008 .cra_blocksize = AES_BLOCK_SIZE,
1009 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
1010 .cra_module = THIS_MODULE,
1011 },
1012 .min_keysize = AES_MIN_KEY_SIZE,
1013 .max_keysize = AES_MAX_KEY_SIZE,
1014 .setkey = aesni_skcipher_setkey,
1015 .encrypt = ecb_encrypt,
1016 .decrypt = ecb_decrypt,
1017 }, {
1018 .base = {
1019 .cra_name = "__cbc(aes)",
1020 .cra_driver_name = "__cbc-aes-aesni",
1021 .cra_priority = 400,
1022 .cra_flags = CRYPTO_ALG_INTERNAL,
1023 .cra_blocksize = AES_BLOCK_SIZE,
1024 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
1025 .cra_module = THIS_MODULE,
1026 },
1027 .min_keysize = AES_MIN_KEY_SIZE,
1028 .max_keysize = AES_MAX_KEY_SIZE,
1029 .ivsize = AES_BLOCK_SIZE,
1030 .setkey = aesni_skcipher_setkey,
1031 .encrypt = cbc_encrypt,
1032 .decrypt = cbc_decrypt,
1033 #ifdef CONFIG_X86_64
1034 }, {
1035 .base = {
1036 .cra_name = "__ctr(aes)",
1037 .cra_driver_name = "__ctr-aes-aesni",
1038 .cra_priority = 400,
1039 .cra_flags = CRYPTO_ALG_INTERNAL,
1040 .cra_blocksize = 1,
1041 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
1042 .cra_module = THIS_MODULE,
1043 },
1044 .min_keysize = AES_MIN_KEY_SIZE,
1045 .max_keysize = AES_MAX_KEY_SIZE,
1046 .ivsize = AES_BLOCK_SIZE,
1047 .chunksize = AES_BLOCK_SIZE,
1048 .setkey = aesni_skcipher_setkey,
1049 .encrypt = ctr_crypt,
1050 .decrypt = ctr_crypt,
1051 }, {
1052 .base = {
1053 .cra_name = "__xts(aes)",
1054 .cra_driver_name = "__xts-aes-aesni",
1055 .cra_priority = 401,
1056 .cra_flags = CRYPTO_ALG_INTERNAL,
1057 .cra_blocksize = AES_BLOCK_SIZE,
1058 .cra_ctxsize = XTS_AES_CTX_SIZE,
1059 .cra_module = THIS_MODULE,
1060 },
1061 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1062 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1063 .ivsize = AES_BLOCK_SIZE,
1064 .setkey = xts_aesni_setkey,
1065 .encrypt = xts_encrypt,
1066 .decrypt = xts_decrypt,
1067 #endif
1068 }
1069 };
1070
1071 static
1072 struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)];
1073
1074 static struct {
1075 const char *algname;
1076 const char *drvname;
1077 const char *basename;
1078 struct simd_skcipher_alg *simd;
1079 } aesni_simd_skciphers2[] = {
1080 #if (defined(MODULE) && IS_ENABLED(CONFIG_CRYPTO_PCBC)) || \
1081 IS_BUILTIN(CONFIG_CRYPTO_PCBC)
1082 {
1083 .algname = "pcbc(aes)",
1084 .drvname = "pcbc-aes-aesni",
1085 .basename = "fpu(pcbc(__aes-aesni))",
1086 },
1087 #endif
1088 };
1089
1090 #ifdef CONFIG_X86_64
1091 static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key,
1092 unsigned int key_len)
1093 {
1094 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead);
1095
1096 return aes_set_key_common(crypto_aead_tfm(aead),
1097 &ctx->aes_key_expanded, key, key_len) ?:
1098 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
1099 }
1100
1101 static int generic_gcmaes_encrypt(struct aead_request *req)
1102 {
1103 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1104 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1105 void *aes_ctx = &(ctx->aes_key_expanded);
1106 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
1107 __be32 counter = cpu_to_be32(1);
1108
1109 memcpy(iv, req->iv, 12);
1110 *((__be32 *)(iv+12)) = counter;
1111
1112 return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv,
1113 aes_ctx);
1114 }
1115
1116 static int generic_gcmaes_decrypt(struct aead_request *req)
1117 {
1118 __be32 counter = cpu_to_be32(1);
1119 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1120 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1121 void *aes_ctx = &(ctx->aes_key_expanded);
1122 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
1123
1124 memcpy(iv, req->iv, 12);
1125 *((__be32 *)(iv+12)) = counter;
1126
1127 return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv,
1128 aes_ctx);
1129 }
1130
1131 static int generic_gcmaes_init(struct crypto_aead *aead)
1132 {
1133 struct cryptd_aead *cryptd_tfm;
1134 struct cryptd_aead **ctx = crypto_aead_ctx(aead);
1135
1136 cryptd_tfm = cryptd_alloc_aead("__driver-generic-gcm-aes-aesni",
1137 CRYPTO_ALG_INTERNAL,
1138 CRYPTO_ALG_INTERNAL);
1139 if (IS_ERR(cryptd_tfm))
1140 return PTR_ERR(cryptd_tfm);
1141
1142 *ctx = cryptd_tfm;
1143 crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
1144
1145 return 0;
1146 }
1147
1148 static void generic_gcmaes_exit(struct crypto_aead *aead)
1149 {
1150 struct cryptd_aead **ctx = crypto_aead_ctx(aead);
1151
1152 cryptd_free_aead(*ctx);
1153 }
1154
1155 static struct aead_alg aesni_aead_algs[] = { {
1156 .setkey = common_rfc4106_set_key,
1157 .setauthsize = common_rfc4106_set_authsize,
1158 .encrypt = helper_rfc4106_encrypt,
1159 .decrypt = helper_rfc4106_decrypt,
1160 .ivsize = GCM_RFC4106_IV_SIZE,
1161 .maxauthsize = 16,
1162 .base = {
1163 .cra_name = "__gcm-aes-aesni",
1164 .cra_driver_name = "__driver-gcm-aes-aesni",
1165 .cra_flags = CRYPTO_ALG_INTERNAL,
1166 .cra_blocksize = 1,
1167 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx),
1168 .cra_alignmask = AESNI_ALIGN - 1,
1169 .cra_module = THIS_MODULE,
1170 },
1171 }, {
1172 .init = rfc4106_init,
1173 .exit = rfc4106_exit,
1174 .setkey = gcmaes_wrapper_set_key,
1175 .setauthsize = gcmaes_wrapper_set_authsize,
1176 .encrypt = gcmaes_wrapper_encrypt,
1177 .decrypt = gcmaes_wrapper_decrypt,
1178 .ivsize = GCM_RFC4106_IV_SIZE,
1179 .maxauthsize = 16,
1180 .base = {
1181 .cra_name = "rfc4106(gcm(aes))",
1182 .cra_driver_name = "rfc4106-gcm-aesni",
1183 .cra_priority = 400,
1184 .cra_flags = CRYPTO_ALG_ASYNC,
1185 .cra_blocksize = 1,
1186 .cra_ctxsize = sizeof(struct cryptd_aead *),
1187 .cra_module = THIS_MODULE,
1188 },
1189 }, {
1190 .setkey = generic_gcmaes_set_key,
1191 .setauthsize = generic_gcmaes_set_authsize,
1192 .encrypt = generic_gcmaes_encrypt,
1193 .decrypt = generic_gcmaes_decrypt,
1194 .ivsize = GCM_AES_IV_SIZE,
1195 .maxauthsize = 16,
1196 .base = {
1197 .cra_name = "__generic-gcm-aes-aesni",
1198 .cra_driver_name = "__driver-generic-gcm-aes-aesni",
1199 .cra_priority = 0,
1200 .cra_flags = CRYPTO_ALG_INTERNAL,
1201 .cra_blocksize = 1,
1202 .cra_ctxsize = sizeof(struct generic_gcmaes_ctx),
1203 .cra_alignmask = AESNI_ALIGN - 1,
1204 .cra_module = THIS_MODULE,
1205 },
1206 }, {
1207 .init = generic_gcmaes_init,
1208 .exit = generic_gcmaes_exit,
1209 .setkey = gcmaes_wrapper_set_key,
1210 .setauthsize = gcmaes_wrapper_set_authsize,
1211 .encrypt = gcmaes_wrapper_encrypt,
1212 .decrypt = gcmaes_wrapper_decrypt,
1213 .ivsize = GCM_AES_IV_SIZE,
1214 .maxauthsize = 16,
1215 .base = {
1216 .cra_name = "gcm(aes)",
1217 .cra_driver_name = "generic-gcm-aesni",
1218 .cra_priority = 400,
1219 .cra_flags = CRYPTO_ALG_ASYNC,
1220 .cra_blocksize = 1,
1221 .cra_ctxsize = sizeof(struct cryptd_aead *),
1222 .cra_module = THIS_MODULE,
1223 },
1224 } };
1225 #else
1226 static struct aead_alg aesni_aead_algs[0];
1227 #endif
1228
1229
1230 static const struct x86_cpu_id aesni_cpu_id[] = {
1231 X86_FEATURE_MATCH(X86_FEATURE_AES),
1232 {}
1233 };
1234 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1235
1236 static void aesni_free_simds(void)
1237 {
1238 int i;
1239
1240 for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers) &&
1241 aesni_simd_skciphers[i]; i++)
1242 simd_skcipher_free(aesni_simd_skciphers[i]);
1243
1244 for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers2); i++)
1245 if (aesni_simd_skciphers2[i].simd)
1246 simd_skcipher_free(aesni_simd_skciphers2[i].simd);
1247 }
1248
1249 static int __init aesni_init(void)
1250 {
1251 struct simd_skcipher_alg *simd;
1252 const char *basename;
1253 const char *algname;
1254 const char *drvname;
1255 int err;
1256 int i;
1257
1258 if (!x86_match_cpu(aesni_cpu_id))
1259 return -ENODEV;
1260 #ifdef CONFIG_X86_64
1261 #ifdef CONFIG_AS_AVX2
1262 if (boot_cpu_has(X86_FEATURE_AVX2)) {
1263 pr_info("AVX2 version of gcm_enc/dec engaged.\n");
1264 aesni_gcm_enc_tfm = aesni_gcm_enc_avx2;
1265 aesni_gcm_dec_tfm = aesni_gcm_dec_avx2;
1266 } else
1267 #endif
1268 #ifdef CONFIG_AS_AVX
1269 if (boot_cpu_has(X86_FEATURE_AVX)) {
1270 pr_info("AVX version of gcm_enc/dec engaged.\n");
1271 aesni_gcm_enc_tfm = aesni_gcm_enc_avx;
1272 aesni_gcm_dec_tfm = aesni_gcm_dec_avx;
1273 } else
1274 #endif
1275 {
1276 pr_info("SSE version of gcm_enc/dec engaged.\n");
1277 aesni_gcm_enc_tfm = aesni_gcm_enc;
1278 aesni_gcm_dec_tfm = aesni_gcm_dec;
1279 }
1280 aesni_ctr_enc_tfm = aesni_ctr_enc;
1281 #ifdef CONFIG_AS_AVX
1282 if (boot_cpu_has(X86_FEATURE_AVX)) {
1283 /* optimize performance of ctr mode encryption transform */
1284 aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm;
1285 pr_info("AES CTR mode by8 optimization enabled\n");
1286 }
1287 #endif
1288 #endif
1289
1290 err = crypto_fpu_init();
1291 if (err)
1292 return err;
1293
1294 err = crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1295 if (err)
1296 goto fpu_exit;
1297
1298 err = crypto_register_skciphers(aesni_skciphers,
1299 ARRAY_SIZE(aesni_skciphers));
1300 if (err)
1301 goto unregister_algs;
1302
1303 err = crypto_register_aeads(aesni_aead_algs,
1304 ARRAY_SIZE(aesni_aead_algs));
1305 if (err)
1306 goto unregister_skciphers;
1307
1308 for (i = 0; i < ARRAY_SIZE(aesni_skciphers); i++) {
1309 algname = aesni_skciphers[i].base.cra_name + 2;
1310 drvname = aesni_skciphers[i].base.cra_driver_name + 2;
1311 basename = aesni_skciphers[i].base.cra_driver_name;
1312 simd = simd_skcipher_create_compat(algname, drvname, basename);
1313 err = PTR_ERR(simd);
1314 if (IS_ERR(simd))
1315 goto unregister_simds;
1316
1317 aesni_simd_skciphers[i] = simd;
1318 }
1319
1320 for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers2); i++) {
1321 algname = aesni_simd_skciphers2[i].algname;
1322 drvname = aesni_simd_skciphers2[i].drvname;
1323 basename = aesni_simd_skciphers2[i].basename;
1324 simd = simd_skcipher_create_compat(algname, drvname, basename);
1325 err = PTR_ERR(simd);
1326 if (IS_ERR(simd))
1327 continue;
1328
1329 aesni_simd_skciphers2[i].simd = simd;
1330 }
1331
1332 return 0;
1333
1334 unregister_simds:
1335 aesni_free_simds();
1336 crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs));
1337 unregister_skciphers:
1338 crypto_unregister_skciphers(aesni_skciphers,
1339 ARRAY_SIZE(aesni_skciphers));
1340 unregister_algs:
1341 crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1342 fpu_exit:
1343 crypto_fpu_exit();
1344 return err;
1345 }
1346
1347 static void __exit aesni_exit(void)
1348 {
1349 aesni_free_simds();
1350 crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs));
1351 crypto_unregister_skciphers(aesni_skciphers,
1352 ARRAY_SIZE(aesni_skciphers));
1353 crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1354
1355 crypto_fpu_exit();
1356 }
1357
1358 late_initcall(aesni_init);
1359 module_exit(aesni_exit);
1360
1361 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1362 MODULE_LICENSE("GPL");
1363 MODULE_ALIAS_CRYPTO("aes");
CRIS linux kernel tree