?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
spi-topcliff-pch: Fix issue for transmitting over 4KByte
[zen-stable.git] / arch / x86 / crypto / aesni-intel_glue.c
blob545d0ce5981839583d1ed6f185d6df64a9c13c1b
1 /*
2 * Support for Intel AES-NI instructions. This file contains glue
3 * code, the real AES implementation is in intel-aes_asm.S.
4 *
5 * Copyright (C) 2008, Intel Corp.
6 * Author: Huang Ying <ying.huang@intel.com>
7 *
8 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
9 * interface for 64-bit kernels.
10 * Authors: Adrian Hoban <adrian.hoban@intel.com>
11 * Gabriele Paoloni <gabriele.paoloni@intel.com>
12 * Tadeusz Struk (tadeusz.struk@intel.com)
13 * Aidan O'Mahony (aidan.o.mahony@intel.com)
14 * Copyright (c) 2010, Intel Corporation.
15 *
16 * This program is free software; you can redistribute it and/or modify
17 * it under the terms of the GNU General Public License as published by
18 * the Free Software Foundation; either version 2 of the License, or
19 * (at your option) any later version.
20 */
21
22 #include <linux/hardirq.h>
23 #include <linux/types.h>
24 #include <linux/crypto.h>
25 #include <linux/module.h>
26 #include <linux/err.h>
27 #include <crypto/algapi.h>
28 #include <crypto/aes.h>
29 #include <crypto/cryptd.h>
30 #include <crypto/ctr.h>
31 #include <asm/i387.h>
32 #include <asm/aes.h>
33 #include <crypto/scatterwalk.h>
34 #include <crypto/internal/aead.h>
35 #include <linux/workqueue.h>
36 #include <linux/spinlock.h>
37
38 #if defined(CONFIG_CRYPTO_CTR) || defined(CONFIG_CRYPTO_CTR_MODULE)
39 #define HAS_CTR
40 #endif
41
42 #if defined(CONFIG_CRYPTO_LRW) || defined(CONFIG_CRYPTO_LRW_MODULE)
43 #define HAS_LRW
44 #endif
45
46 #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE)
47 #define HAS_PCBC
48 #endif
49
50 #if defined(CONFIG_CRYPTO_XTS) || defined(CONFIG_CRYPTO_XTS_MODULE)
51 #define HAS_XTS
52 #endif
53
54 struct async_aes_ctx {
55 struct cryptd_ablkcipher *cryptd_tfm;
56 };
57
58 /* This data is stored at the end of the crypto_tfm struct.
59 * It's a type of per "session" data storage location.
60 * This needs to be 16 byte aligned.
61 */
62 struct aesni_rfc4106_gcm_ctx {
63 u8 hash_subkey[16];
64 struct crypto_aes_ctx aes_key_expanded;
65 u8 nonce[4];
66 struct cryptd_aead *cryptd_tfm;
67 };
68
69 struct aesni_gcm_set_hash_subkey_result {
70 int err;
71 struct completion completion;
72 };
73
74 struct aesni_hash_subkey_req_data {
75 u8 iv[16];
76 struct aesni_gcm_set_hash_subkey_result result;
77 struct scatterlist sg;
78 };
79
80 #define AESNI_ALIGN (16)
81 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1))
82 #define RFC4106_HASH_SUBKEY_SIZE 16
83
84 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
85 unsigned int key_len);
86 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
87 const u8 *in);
88 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
89 const u8 *in);
90 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
91 const u8 *in, unsigned int len);
92 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
93 const u8 *in, unsigned int len);
94 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
95 const u8 *in, unsigned int len, u8 *iv);
96 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
97 const u8 *in, unsigned int len, u8 *iv);
98
99 int crypto_fpu_init(void);
100 void crypto_fpu_exit(void);
101
102 #ifdef CONFIG_X86_64
103 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
104 const u8 *in, unsigned int len, u8 *iv);
105
106 /* asmlinkage void aesni_gcm_enc()
107 * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
108 * u8 *out, Ciphertext output. Encrypt in-place is allowed.
109 * const u8 *in, Plaintext input
110 * unsigned long plaintext_len, Length of data in bytes for encryption.
111 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
112 * concatenated with 8 byte Initialisation Vector (from IPSec ESP
113 * Payload) concatenated with 0x00000001. 16-byte aligned pointer.
114 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
115 * const u8 *aad, Additional Authentication Data (AAD)
116 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this
117 * is going to be 8 or 12 bytes
118 * u8 *auth_tag, Authenticated Tag output.
119 * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
120 * Valid values are 16 (most likely), 12 or 8.
121 */
122 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out,
123 const u8 *in, unsigned long plaintext_len, u8 *iv,
124 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
125 u8 *auth_tag, unsigned long auth_tag_len);
126
127 /* asmlinkage void aesni_gcm_dec()
128 * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
129 * u8 *out, Plaintext output. Decrypt in-place is allowed.
130 * const u8 *in, Ciphertext input
131 * unsigned long ciphertext_len, Length of data in bytes for decryption.
132 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
133 * concatenated with 8 byte Initialisation Vector (from IPSec ESP
134 * Payload) concatenated with 0x00000001. 16-byte aligned pointer.
135 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
136 * const u8 *aad, Additional Authentication Data (AAD)
137 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
138 * to be 8 or 12 bytes
139 * u8 *auth_tag, Authenticated Tag output.
140 * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
141 * Valid values are 16 (most likely), 12 or 8.
142 */
143 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out,
144 const u8 *in, unsigned long ciphertext_len, u8 *iv,
145 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
146 u8 *auth_tag, unsigned long auth_tag_len);
147
148 static inline struct
149 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
150 {
151 return
152 (struct aesni_rfc4106_gcm_ctx *)
153 PTR_ALIGN((u8 *)
154 crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN);
155 }
156 #endif
157
158 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
159 {
160 unsigned long addr = (unsigned long)raw_ctx;
161 unsigned long align = AESNI_ALIGN;
162
163 if (align <= crypto_tfm_ctx_alignment())
164 align = 1;
165 return (struct crypto_aes_ctx *)ALIGN(addr, align);
166 }
167
168 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
169 const u8 *in_key, unsigned int key_len)
170 {
171 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
172 u32 *flags = &tfm->crt_flags;
173 int err;
174
175 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
176 key_len != AES_KEYSIZE_256) {
177 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
178 return -EINVAL;
179 }
180
181 if (!irq_fpu_usable())
182 err = crypto_aes_expand_key(ctx, in_key, key_len);
183 else {
184 kernel_fpu_begin();
185 err = aesni_set_key(ctx, in_key, key_len);
186 kernel_fpu_end();
187 }
188
189 return err;
190 }
191
192 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
193 unsigned int key_len)
194 {
195 return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
196 }
197
198 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
199 {
200 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
201
202 if (!irq_fpu_usable())
203 crypto_aes_encrypt_x86(ctx, dst, src);
204 else {
205 kernel_fpu_begin();
206 aesni_enc(ctx, dst, src);
207 kernel_fpu_end();
208 }
209 }
210
211 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
212 {
213 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
214
215 if (!irq_fpu_usable())
216 crypto_aes_decrypt_x86(ctx, dst, src);
217 else {
218 kernel_fpu_begin();
219 aesni_dec(ctx, dst, src);
220 kernel_fpu_end();
221 }
222 }
223
224 static struct crypto_alg aesni_alg = {
225 .cra_name = "aes",
226 .cra_driver_name = "aes-aesni",
227 .cra_priority = 300,
228 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
229 .cra_blocksize = AES_BLOCK_SIZE,
230 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
231 .cra_alignmask = 0,
232 .cra_module = THIS_MODULE,
233 .cra_list = LIST_HEAD_INIT(aesni_alg.cra_list),
234 .cra_u = {
235 .cipher = {
236 .cia_min_keysize = AES_MIN_KEY_SIZE,
237 .cia_max_keysize = AES_MAX_KEY_SIZE,
238 .cia_setkey = aes_set_key,
239 .cia_encrypt = aes_encrypt,
240 .cia_decrypt = aes_decrypt
241 }
242 }
243 };
244
245 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
246 {
247 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
248
249 aesni_enc(ctx, dst, src);
250 }
251
252 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
253 {
254 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
255
256 aesni_dec(ctx, dst, src);
257 }
258
259 static struct crypto_alg __aesni_alg = {
260 .cra_name = "__aes-aesni",
261 .cra_driver_name = "__driver-aes-aesni",
262 .cra_priority = 0,
263 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
264 .cra_blocksize = AES_BLOCK_SIZE,
265 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
266 .cra_alignmask = 0,
267 .cra_module = THIS_MODULE,
268 .cra_list = LIST_HEAD_INIT(__aesni_alg.cra_list),
269 .cra_u = {
270 .cipher = {
271 .cia_min_keysize = AES_MIN_KEY_SIZE,
272 .cia_max_keysize = AES_MAX_KEY_SIZE,
273 .cia_setkey = aes_set_key,
274 .cia_encrypt = __aes_encrypt,
275 .cia_decrypt = __aes_decrypt
276 }
277 }
278 };
279
280 static int ecb_encrypt(struct blkcipher_desc *desc,
281 struct scatterlist *dst, struct scatterlist *src,
282 unsigned int nbytes)
283 {
284 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
285 struct blkcipher_walk walk;
286 int err;
287
288 blkcipher_walk_init(&walk, dst, src, nbytes);
289 err = blkcipher_walk_virt(desc, &walk);
290 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
291
292 kernel_fpu_begin();
293 while ((nbytes = walk.nbytes)) {
294 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
295 nbytes & AES_BLOCK_MASK);
296 nbytes &= AES_BLOCK_SIZE - 1;
297 err = blkcipher_walk_done(desc, &walk, nbytes);
298 }
299 kernel_fpu_end();
300
301 return err;
302 }
303
304 static int ecb_decrypt(struct blkcipher_desc *desc,
305 struct scatterlist *dst, struct scatterlist *src,
306 unsigned int nbytes)
307 {
308 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
309 struct blkcipher_walk walk;
310 int err;
311
312 blkcipher_walk_init(&walk, dst, src, nbytes);
313 err = blkcipher_walk_virt(desc, &walk);
314 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
315
316 kernel_fpu_begin();
317 while ((nbytes = walk.nbytes)) {
318 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
319 nbytes & AES_BLOCK_MASK);
320 nbytes &= AES_BLOCK_SIZE - 1;
321 err = blkcipher_walk_done(desc, &walk, nbytes);
322 }
323 kernel_fpu_end();
324
325 return err;
326 }
327
328 static struct crypto_alg blk_ecb_alg = {
329 .cra_name = "__ecb-aes-aesni",
330 .cra_driver_name = "__driver-ecb-aes-aesni",
331 .cra_priority = 0,
332 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
333 .cra_blocksize = AES_BLOCK_SIZE,
334 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
335 .cra_alignmask = 0,
336 .cra_type = &crypto_blkcipher_type,
337 .cra_module = THIS_MODULE,
338 .cra_list = LIST_HEAD_INIT(blk_ecb_alg.cra_list),
339 .cra_u = {
340 .blkcipher = {
341 .min_keysize = AES_MIN_KEY_SIZE,
342 .max_keysize = AES_MAX_KEY_SIZE,
343 .setkey = aes_set_key,
344 .encrypt = ecb_encrypt,
345 .decrypt = ecb_decrypt,
346 },
347 },
348 };
349
350 static int cbc_encrypt(struct blkcipher_desc *desc,
351 struct scatterlist *dst, struct scatterlist *src,
352 unsigned int nbytes)
353 {
354 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
355 struct blkcipher_walk walk;
356 int err;
357
358 blkcipher_walk_init(&walk, dst, src, nbytes);
359 err = blkcipher_walk_virt(desc, &walk);
360 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
361
362 kernel_fpu_begin();
363 while ((nbytes = walk.nbytes)) {
364 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
365 nbytes & AES_BLOCK_MASK, walk.iv);
366 nbytes &= AES_BLOCK_SIZE - 1;
367 err = blkcipher_walk_done(desc, &walk, nbytes);
368 }
369 kernel_fpu_end();
370
371 return err;
372 }
373
374 static int cbc_decrypt(struct blkcipher_desc *desc,
375 struct scatterlist *dst, struct scatterlist *src,
376 unsigned int nbytes)
377 {
378 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
379 struct blkcipher_walk walk;
380 int err;
381
382 blkcipher_walk_init(&walk, dst, src, nbytes);
383 err = blkcipher_walk_virt(desc, &walk);
384 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
385
386 kernel_fpu_begin();
387 while ((nbytes = walk.nbytes)) {
388 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
389 nbytes & AES_BLOCK_MASK, walk.iv);
390 nbytes &= AES_BLOCK_SIZE - 1;
391 err = blkcipher_walk_done(desc, &walk, nbytes);
392 }
393 kernel_fpu_end();
394
395 return err;
396 }
397
398 static struct crypto_alg blk_cbc_alg = {
399 .cra_name = "__cbc-aes-aesni",
400 .cra_driver_name = "__driver-cbc-aes-aesni",
401 .cra_priority = 0,
402 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
403 .cra_blocksize = AES_BLOCK_SIZE,
404 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
405 .cra_alignmask = 0,
406 .cra_type = &crypto_blkcipher_type,
407 .cra_module = THIS_MODULE,
408 .cra_list = LIST_HEAD_INIT(blk_cbc_alg.cra_list),
409 .cra_u = {
410 .blkcipher = {
411 .min_keysize = AES_MIN_KEY_SIZE,
412 .max_keysize = AES_MAX_KEY_SIZE,
413 .setkey = aes_set_key,
414 .encrypt = cbc_encrypt,
415 .decrypt = cbc_decrypt,
416 },
417 },
418 };
419
420 #ifdef CONFIG_X86_64
421 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
422 struct blkcipher_walk *walk)
423 {
424 u8 *ctrblk = walk->iv;
425 u8 keystream[AES_BLOCK_SIZE];
426 u8 *src = walk->src.virt.addr;
427 u8 *dst = walk->dst.virt.addr;
428 unsigned int nbytes = walk->nbytes;
429
430 aesni_enc(ctx, keystream, ctrblk);
431 crypto_xor(keystream, src, nbytes);
432 memcpy(dst, keystream, nbytes);
433 crypto_inc(ctrblk, AES_BLOCK_SIZE);
434 }
435
436 static int ctr_crypt(struct blkcipher_desc *desc,
437 struct scatterlist *dst, struct scatterlist *src,
438 unsigned int nbytes)
439 {
440 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
441 struct blkcipher_walk walk;
442 int err;
443
444 blkcipher_walk_init(&walk, dst, src, nbytes);
445 err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
446 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
447
448 kernel_fpu_begin();
449 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
450 aesni_ctr_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
451 nbytes & AES_BLOCK_MASK, walk.iv);
452 nbytes &= AES_BLOCK_SIZE - 1;
453 err = blkcipher_walk_done(desc, &walk, nbytes);
454 }
455 if (walk.nbytes) {
456 ctr_crypt_final(ctx, &walk);
457 err = blkcipher_walk_done(desc, &walk, 0);
458 }
459 kernel_fpu_end();
460
461 return err;
462 }
463
464 static struct crypto_alg blk_ctr_alg = {
465 .cra_name = "__ctr-aes-aesni",
466 .cra_driver_name = "__driver-ctr-aes-aesni",
467 .cra_priority = 0,
468 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
469 .cra_blocksize = 1,
470 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
471 .cra_alignmask = 0,
472 .cra_type = &crypto_blkcipher_type,
473 .cra_module = THIS_MODULE,
474 .cra_list = LIST_HEAD_INIT(blk_ctr_alg.cra_list),
475 .cra_u = {
476 .blkcipher = {
477 .min_keysize = AES_MIN_KEY_SIZE,
478 .max_keysize = AES_MAX_KEY_SIZE,
479 .ivsize = AES_BLOCK_SIZE,
480 .setkey = aes_set_key,
481 .encrypt = ctr_crypt,
482 .decrypt = ctr_crypt,
483 },
484 },
485 };
486 #endif
487
488 static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
489 unsigned int key_len)
490 {
491 struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
492 struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base;
493 int err;
494
495 crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
496 crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm)
497 & CRYPTO_TFM_REQ_MASK);
498 err = crypto_ablkcipher_setkey(child, key, key_len);
499 crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child)
500 & CRYPTO_TFM_RES_MASK);
501 return err;
502 }
503
504 static int ablk_encrypt(struct ablkcipher_request *req)
505 {
506 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
507 struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
508
509 if (!irq_fpu_usable()) {
510 struct ablkcipher_request *cryptd_req =
511 ablkcipher_request_ctx(req);
512 memcpy(cryptd_req, req, sizeof(*req));
513 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
514 return crypto_ablkcipher_encrypt(cryptd_req);
515 } else {
516 struct blkcipher_desc desc;
517 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
518 desc.info = req->info;
519 desc.flags = 0;
520 return crypto_blkcipher_crt(desc.tfm)->encrypt(
521 &desc, req->dst, req->src, req->nbytes);
522 }
523 }
524
525 static int ablk_decrypt(struct ablkcipher_request *req)
526 {
527 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
528 struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
529
530 if (!irq_fpu_usable()) {
531 struct ablkcipher_request *cryptd_req =
532 ablkcipher_request_ctx(req);
533 memcpy(cryptd_req, req, sizeof(*req));
534 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
535 return crypto_ablkcipher_decrypt(cryptd_req);
536 } else {
537 struct blkcipher_desc desc;
538 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
539 desc.info = req->info;
540 desc.flags = 0;
541 return crypto_blkcipher_crt(desc.tfm)->decrypt(
542 &desc, req->dst, req->src, req->nbytes);
543 }
544 }
545
546 static void ablk_exit(struct crypto_tfm *tfm)
547 {
548 struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
549
550 cryptd_free_ablkcipher(ctx->cryptd_tfm);
551 }
552
553 static void ablk_init_common(struct crypto_tfm *tfm,
554 struct cryptd_ablkcipher *cryptd_tfm)
555 {
556 struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
557
558 ctx->cryptd_tfm = cryptd_tfm;
559 tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
560 crypto_ablkcipher_reqsize(&cryptd_tfm->base);
561 }
562
563 static int ablk_ecb_init(struct crypto_tfm *tfm)
564 {
565 struct cryptd_ablkcipher *cryptd_tfm;
566
567 cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ecb-aes-aesni", 0, 0);
568 if (IS_ERR(cryptd_tfm))
569 return PTR_ERR(cryptd_tfm);
570 ablk_init_common(tfm, cryptd_tfm);
571 return 0;
572 }
573
574 static struct crypto_alg ablk_ecb_alg = {
575 .cra_name = "ecb(aes)",
576 .cra_driver_name = "ecb-aes-aesni",
577 .cra_priority = 400,
578 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
579 .cra_blocksize = AES_BLOCK_SIZE,
580 .cra_ctxsize = sizeof(struct async_aes_ctx),
581 .cra_alignmask = 0,
582 .cra_type = &crypto_ablkcipher_type,
583 .cra_module = THIS_MODULE,
584 .cra_list = LIST_HEAD_INIT(ablk_ecb_alg.cra_list),
585 .cra_init = ablk_ecb_init,
586 .cra_exit = ablk_exit,
587 .cra_u = {
588 .ablkcipher = {
589 .min_keysize = AES_MIN_KEY_SIZE,
590 .max_keysize = AES_MAX_KEY_SIZE,
591 .setkey = ablk_set_key,
592 .encrypt = ablk_encrypt,
593 .decrypt = ablk_decrypt,
594 },
595 },
596 };
597
598 static int ablk_cbc_init(struct crypto_tfm *tfm)
599 {
600 struct cryptd_ablkcipher *cryptd_tfm;
601
602 cryptd_tfm = cryptd_alloc_ablkcipher("__driver-cbc-aes-aesni", 0, 0);
603 if (IS_ERR(cryptd_tfm))
604 return PTR_ERR(cryptd_tfm);
605 ablk_init_common(tfm, cryptd_tfm);
606 return 0;
607 }
608
609 static struct crypto_alg ablk_cbc_alg = {
610 .cra_name = "cbc(aes)",
611 .cra_driver_name = "cbc-aes-aesni",
612 .cra_priority = 400,
613 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
614 .cra_blocksize = AES_BLOCK_SIZE,
615 .cra_ctxsize = sizeof(struct async_aes_ctx),
616 .cra_alignmask = 0,
617 .cra_type = &crypto_ablkcipher_type,
618 .cra_module = THIS_MODULE,
619 .cra_list = LIST_HEAD_INIT(ablk_cbc_alg.cra_list),
620 .cra_init = ablk_cbc_init,
621 .cra_exit = ablk_exit,
622 .cra_u = {
623 .ablkcipher = {
624 .min_keysize = AES_MIN_KEY_SIZE,
625 .max_keysize = AES_MAX_KEY_SIZE,
626 .ivsize = AES_BLOCK_SIZE,
627 .setkey = ablk_set_key,
628 .encrypt = ablk_encrypt,
629 .decrypt = ablk_decrypt,
630 },
631 },
632 };
633
634 #ifdef CONFIG_X86_64
635 static int ablk_ctr_init(struct crypto_tfm *tfm)
636 {
637 struct cryptd_ablkcipher *cryptd_tfm;
638
639 cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ctr-aes-aesni", 0, 0);
640 if (IS_ERR(cryptd_tfm))
641 return PTR_ERR(cryptd_tfm);
642 ablk_init_common(tfm, cryptd_tfm);
643 return 0;
644 }
645
646 static struct crypto_alg ablk_ctr_alg = {
647 .cra_name = "ctr(aes)",
648 .cra_driver_name = "ctr-aes-aesni",
649 .cra_priority = 400,
650 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
651 .cra_blocksize = 1,
652 .cra_ctxsize = sizeof(struct async_aes_ctx),
653 .cra_alignmask = 0,
654 .cra_type = &crypto_ablkcipher_type,
655 .cra_module = THIS_MODULE,
656 .cra_list = LIST_HEAD_INIT(ablk_ctr_alg.cra_list),
657 .cra_init = ablk_ctr_init,
658 .cra_exit = ablk_exit,
659 .cra_u = {
660 .ablkcipher = {
661 .min_keysize = AES_MIN_KEY_SIZE,
662 .max_keysize = AES_MAX_KEY_SIZE,
663 .ivsize = AES_BLOCK_SIZE,
664 .setkey = ablk_set_key,
665 .encrypt = ablk_encrypt,
666 .decrypt = ablk_encrypt,
667 .geniv = "chainiv",
668 },
669 },
670 };
671
672 #ifdef HAS_CTR
673 static int ablk_rfc3686_ctr_init(struct crypto_tfm *tfm)
674 {
675 struct cryptd_ablkcipher *cryptd_tfm;
676
677 cryptd_tfm = cryptd_alloc_ablkcipher(
678 "rfc3686(__driver-ctr-aes-aesni)", 0, 0);
679 if (IS_ERR(cryptd_tfm))
680 return PTR_ERR(cryptd_tfm);
681 ablk_init_common(tfm, cryptd_tfm);
682 return 0;
683 }
684
685 static struct crypto_alg ablk_rfc3686_ctr_alg = {
686 .cra_name = "rfc3686(ctr(aes))",
687 .cra_driver_name = "rfc3686-ctr-aes-aesni",
688 .cra_priority = 400,
689 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
690 .cra_blocksize = 1,
691 .cra_ctxsize = sizeof(struct async_aes_ctx),
692 .cra_alignmask = 0,
693 .cra_type = &crypto_ablkcipher_type,
694 .cra_module = THIS_MODULE,
695 .cra_list = LIST_HEAD_INIT(ablk_rfc3686_ctr_alg.cra_list),
696 .cra_init = ablk_rfc3686_ctr_init,
697 .cra_exit = ablk_exit,
698 .cra_u = {
699 .ablkcipher = {
700 .min_keysize = AES_MIN_KEY_SIZE+CTR_RFC3686_NONCE_SIZE,
701 .max_keysize = AES_MAX_KEY_SIZE+CTR_RFC3686_NONCE_SIZE,
702 .ivsize = CTR_RFC3686_IV_SIZE,
703 .setkey = ablk_set_key,
704 .encrypt = ablk_encrypt,
705 .decrypt = ablk_decrypt,
706 .geniv = "seqiv",
707 },
708 },
709 };
710 #endif
711 #endif
712
713 #ifdef HAS_LRW
714 static int ablk_lrw_init(struct crypto_tfm *tfm)
715 {
716 struct cryptd_ablkcipher *cryptd_tfm;
717
718 cryptd_tfm = cryptd_alloc_ablkcipher("fpu(lrw(__driver-aes-aesni))",
719 0, 0);
720 if (IS_ERR(cryptd_tfm))
721 return PTR_ERR(cryptd_tfm);
722 ablk_init_common(tfm, cryptd_tfm);
723 return 0;
724 }
725
726 static struct crypto_alg ablk_lrw_alg = {
727 .cra_name = "lrw(aes)",
728 .cra_driver_name = "lrw-aes-aesni",
729 .cra_priority = 400,
730 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
731 .cra_blocksize = AES_BLOCK_SIZE,
732 .cra_ctxsize = sizeof(struct async_aes_ctx),
733 .cra_alignmask = 0,
734 .cra_type = &crypto_ablkcipher_type,
735 .cra_module = THIS_MODULE,
736 .cra_list = LIST_HEAD_INIT(ablk_lrw_alg.cra_list),
737 .cra_init = ablk_lrw_init,
738 .cra_exit = ablk_exit,
739 .cra_u = {
740 .ablkcipher = {
741 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
742 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
743 .ivsize = AES_BLOCK_SIZE,
744 .setkey = ablk_set_key,
745 .encrypt = ablk_encrypt,
746 .decrypt = ablk_decrypt,
747 },
748 },
749 };
750 #endif
751
752 #ifdef HAS_PCBC
753 static int ablk_pcbc_init(struct crypto_tfm *tfm)
754 {
755 struct cryptd_ablkcipher *cryptd_tfm;
756
757 cryptd_tfm = cryptd_alloc_ablkcipher("fpu(pcbc(__driver-aes-aesni))",
758 0, 0);
759 if (IS_ERR(cryptd_tfm))
760 return PTR_ERR(cryptd_tfm);
761 ablk_init_common(tfm, cryptd_tfm);
762 return 0;
763 }
764
765 static struct crypto_alg ablk_pcbc_alg = {
766 .cra_name = "pcbc(aes)",
767 .cra_driver_name = "pcbc-aes-aesni",
768 .cra_priority = 400,
769 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
770 .cra_blocksize = AES_BLOCK_SIZE,
771 .cra_ctxsize = sizeof(struct async_aes_ctx),
772 .cra_alignmask = 0,
773 .cra_type = &crypto_ablkcipher_type,
774 .cra_module = THIS_MODULE,
775 .cra_list = LIST_HEAD_INIT(ablk_pcbc_alg.cra_list),
776 .cra_init = ablk_pcbc_init,
777 .cra_exit = ablk_exit,
778 .cra_u = {
779 .ablkcipher = {
780 .min_keysize = AES_MIN_KEY_SIZE,
781 .max_keysize = AES_MAX_KEY_SIZE,
782 .ivsize = AES_BLOCK_SIZE,
783 .setkey = ablk_set_key,
784 .encrypt = ablk_encrypt,
785 .decrypt = ablk_decrypt,
786 },
787 },
788 };
789 #endif
790
791 #ifdef HAS_XTS
792 static int ablk_xts_init(struct crypto_tfm *tfm)
793 {
794 struct cryptd_ablkcipher *cryptd_tfm;
795
796 cryptd_tfm = cryptd_alloc_ablkcipher("fpu(xts(__driver-aes-aesni))",
797 0, 0);
798 if (IS_ERR(cryptd_tfm))
799 return PTR_ERR(cryptd_tfm);
800 ablk_init_common(tfm, cryptd_tfm);
801 return 0;
802 }
803
804 static struct crypto_alg ablk_xts_alg = {
805 .cra_name = "xts(aes)",
806 .cra_driver_name = "xts-aes-aesni",
807 .cra_priority = 400,
808 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
809 .cra_blocksize = AES_BLOCK_SIZE,
810 .cra_ctxsize = sizeof(struct async_aes_ctx),
811 .cra_alignmask = 0,
812 .cra_type = &crypto_ablkcipher_type,
813 .cra_module = THIS_MODULE,
814 .cra_list = LIST_HEAD_INIT(ablk_xts_alg.cra_list),
815 .cra_init = ablk_xts_init,
816 .cra_exit = ablk_exit,
817 .cra_u = {
818 .ablkcipher = {
819 .min_keysize = 2 * AES_MIN_KEY_SIZE,
820 .max_keysize = 2 * AES_MAX_KEY_SIZE,
821 .ivsize = AES_BLOCK_SIZE,
822 .setkey = ablk_set_key,
823 .encrypt = ablk_encrypt,
824 .decrypt = ablk_decrypt,
825 },
826 },
827 };
828 #endif
829
830 #ifdef CONFIG_X86_64
831 static int rfc4106_init(struct crypto_tfm *tfm)
832 {
833 struct cryptd_aead *cryptd_tfm;
834 struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *)
835 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
836 struct crypto_aead *cryptd_child;
837 struct aesni_rfc4106_gcm_ctx *child_ctx;
838 cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0);
839 if (IS_ERR(cryptd_tfm))
840 return PTR_ERR(cryptd_tfm);
841
842 cryptd_child = cryptd_aead_child(cryptd_tfm);
843 child_ctx = aesni_rfc4106_gcm_ctx_get(cryptd_child);
844 memcpy(child_ctx, ctx, sizeof(*ctx));
845 ctx->cryptd_tfm = cryptd_tfm;
846 tfm->crt_aead.reqsize = sizeof(struct aead_request)
847 + crypto_aead_reqsize(&cryptd_tfm->base);
848 return 0;
849 }
850
851 static void rfc4106_exit(struct crypto_tfm *tfm)
852 {
853 struct aesni_rfc4106_gcm_ctx *ctx =
854 (struct aesni_rfc4106_gcm_ctx *)
855 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
856 if (!IS_ERR(ctx->cryptd_tfm))
857 cryptd_free_aead(ctx->cryptd_tfm);
858 return;
859 }
860
861 static void
862 rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err)
863 {
864 struct aesni_gcm_set_hash_subkey_result *result = req->data;
865
866 if (err == -EINPROGRESS)
867 return;
868 result->err = err;
869 complete(&result->completion);
870 }
871
872 static int
873 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
874 {
875 struct crypto_ablkcipher *ctr_tfm;
876 struct ablkcipher_request *req;
877 int ret = -EINVAL;
878 struct aesni_hash_subkey_req_data *req_data;
879
880 ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0);
881 if (IS_ERR(ctr_tfm))
882 return PTR_ERR(ctr_tfm);
883
884 crypto_ablkcipher_clear_flags(ctr_tfm, ~0);
885
886 ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len);
887 if (ret)
888 goto out_free_ablkcipher;
889
890 ret = -ENOMEM;
891 req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL);
892 if (!req)
893 goto out_free_ablkcipher;
894
895 req_data = kmalloc(sizeof(*req_data), GFP_KERNEL);
896 if (!req_data)
897 goto out_free_request;
898
899 memset(req_data->iv, 0, sizeof(req_data->iv));
900
901 /* Clear the data in the hash sub key container to zero.*/
902 /* We want to cipher all zeros to create the hash sub key. */
903 memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
904
905 init_completion(&req_data->result.completion);
906 sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE);
907 ablkcipher_request_set_tfm(req, ctr_tfm);
908 ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |
909 CRYPTO_TFM_REQ_MAY_BACKLOG,
910 rfc4106_set_hash_subkey_done,
911 &req_data->result);
912
913 ablkcipher_request_set_crypt(req, &req_data->sg,
914 &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv);
915
916 ret = crypto_ablkcipher_encrypt(req);
917 if (ret == -EINPROGRESS || ret == -EBUSY) {
918 ret = wait_for_completion_interruptible
919 (&req_data->result.completion);
920 if (!ret)
921 ret = req_data->result.err;
922 }
923 kfree(req_data);
924 out_free_request:
925 ablkcipher_request_free(req);
926 out_free_ablkcipher:
927 crypto_free_ablkcipher(ctr_tfm);
928 return ret;
929 }
930
931 static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
932 unsigned int key_len)
933 {
934 int ret = 0;
935 struct crypto_tfm *tfm = crypto_aead_tfm(parent);
936 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
937 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
938 struct aesni_rfc4106_gcm_ctx *child_ctx =
939 aesni_rfc4106_gcm_ctx_get(cryptd_child);
940 u8 *new_key_mem = NULL;
941
942 if (key_len < 4) {
943 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
944 return -EINVAL;
945 }
946 /*Account for 4 byte nonce at the end.*/
947 key_len -= 4;
948 if (key_len != AES_KEYSIZE_128) {
949 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
950 return -EINVAL;
951 }
952
953 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
954 /*This must be on a 16 byte boundary!*/
955 if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN)
956 return -EINVAL;
957
958 if ((unsigned long)key % AESNI_ALIGN) {
959 /*key is not aligned: use an auxuliar aligned pointer*/
960 new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL);
961 if (!new_key_mem)
962 return -ENOMEM;
963
964 new_key_mem = PTR_ALIGN(new_key_mem, AESNI_ALIGN);
965 memcpy(new_key_mem, key, key_len);
966 key = new_key_mem;
967 }
968
969 if (!irq_fpu_usable())
970 ret = crypto_aes_expand_key(&(ctx->aes_key_expanded),
971 key, key_len);
972 else {
973 kernel_fpu_begin();
974 ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len);
975 kernel_fpu_end();
976 }
977 /*This must be on a 16 byte boundary!*/
978 if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) {
979 ret = -EINVAL;
980 goto exit;
981 }
982 ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
983 memcpy(child_ctx, ctx, sizeof(*ctx));
984 exit:
985 kfree(new_key_mem);
986 return ret;
987 }
988
989 /* This is the Integrity Check Value (aka the authentication tag length and can
990 * be 8, 12 or 16 bytes long. */
991 static int rfc4106_set_authsize(struct crypto_aead *parent,
992 unsigned int authsize)
993 {
994 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
995 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
996
997 switch (authsize) {
998 case 8:
999 case 12:
1000 case 16:
1001 break;
1002 default:
1003 return -EINVAL;
1004 }
1005 crypto_aead_crt(parent)->authsize = authsize;
1006 crypto_aead_crt(cryptd_child)->authsize = authsize;
1007 return 0;
1008 }
1009
1010 static int rfc4106_encrypt(struct aead_request *req)
1011 {
1012 int ret;
1013 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1014 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1015
1016 if (!irq_fpu_usable()) {
1017 struct aead_request *cryptd_req =
1018 (struct aead_request *) aead_request_ctx(req);
1019 memcpy(cryptd_req, req, sizeof(*req));
1020 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
1021 return crypto_aead_encrypt(cryptd_req);
1022 } else {
1023 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
1024 kernel_fpu_begin();
1025 ret = cryptd_child->base.crt_aead.encrypt(req);
1026 kernel_fpu_end();
1027 return ret;
1028 }
1029 }
1030
1031 static int rfc4106_decrypt(struct aead_request *req)
1032 {
1033 int ret;
1034 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1035 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1036
1037 if (!irq_fpu_usable()) {
1038 struct aead_request *cryptd_req =
1039 (struct aead_request *) aead_request_ctx(req);
1040 memcpy(cryptd_req, req, sizeof(*req));
1041 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
1042 return crypto_aead_decrypt(cryptd_req);
1043 } else {
1044 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
1045 kernel_fpu_begin();
1046 ret = cryptd_child->base.crt_aead.decrypt(req);
1047 kernel_fpu_end();
1048 return ret;
1049 }
1050 }
1051
1052 static struct crypto_alg rfc4106_alg = {
1053 .cra_name = "rfc4106(gcm(aes))",
1054 .cra_driver_name = "rfc4106-gcm-aesni",
1055 .cra_priority = 400,
1056 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1057 .cra_blocksize = 1,
1058 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN,
1059 .cra_alignmask = 0,
1060 .cra_type = &crypto_nivaead_type,
1061 .cra_module = THIS_MODULE,
1062 .cra_list = LIST_HEAD_INIT(rfc4106_alg.cra_list),
1063 .cra_init = rfc4106_init,
1064 .cra_exit = rfc4106_exit,
1065 .cra_u = {
1066 .aead = {
1067 .setkey = rfc4106_set_key,
1068 .setauthsize = rfc4106_set_authsize,
1069 .encrypt = rfc4106_encrypt,
1070 .decrypt = rfc4106_decrypt,
1071 .geniv = "seqiv",
1072 .ivsize = 8,
1073 .maxauthsize = 16,
1074 },
1075 },
1076 };
1077
1078 static int __driver_rfc4106_encrypt(struct aead_request *req)
1079 {
1080 u8 one_entry_in_sg = 0;
1081 u8 *src, *dst, *assoc;
1082 __be32 counter = cpu_to_be32(1);
1083 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1084 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1085 void *aes_ctx = &(ctx->aes_key_expanded);
1086 unsigned long auth_tag_len = crypto_aead_authsize(tfm);
1087 u8 iv_tab[16+AESNI_ALIGN];
1088 u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN);
1089 struct scatter_walk src_sg_walk;
1090 struct scatter_walk assoc_sg_walk;
1091 struct scatter_walk dst_sg_walk;
1092 unsigned int i;
1093
1094 /* Assuming we are supporting rfc4106 64-bit extended */
1095 /* sequence numbers We need to have the AAD length equal */
1096 /* to 8 or 12 bytes */
1097 if (unlikely(req->assoclen != 8 && req->assoclen != 12))
1098 return -EINVAL;
1099 /* IV below built */
1100 for (i = 0; i < 4; i++)
1101 *(iv+i) = ctx->nonce[i];
1102 for (i = 0; i < 8; i++)
1103 *(iv+4+i) = req->iv[i];
1104 *((__be32 *)(iv+12)) = counter;
1105
1106 if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
1107 one_entry_in_sg = 1;
1108 scatterwalk_start(&src_sg_walk, req->src);
1109 scatterwalk_start(&assoc_sg_walk, req->assoc);
1110 src = scatterwalk_map(&src_sg_walk, 0);
1111 assoc = scatterwalk_map(&assoc_sg_walk, 0);
1112 dst = src;
1113 if (unlikely(req->src != req->dst)) {
1114 scatterwalk_start(&dst_sg_walk, req->dst);
1115 dst = scatterwalk_map(&dst_sg_walk, 0);
1116 }
1117
1118 } else {
1119 /* Allocate memory for src, dst, assoc */
1120 src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
1121 GFP_ATOMIC);
1122 if (unlikely(!src))
1123 return -ENOMEM;
1124 assoc = (src + req->cryptlen + auth_tag_len);
1125 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
1126 scatterwalk_map_and_copy(assoc, req->assoc, 0,
1127 req->assoclen, 0);
1128 dst = src;
1129 }
1130
1131 aesni_gcm_enc(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv,
1132 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst
1133 + ((unsigned long)req->cryptlen), auth_tag_len);
1134
1135 /* The authTag (aka the Integrity Check Value) needs to be written
1136 * back to the packet. */
1137 if (one_entry_in_sg) {
1138 if (unlikely(req->src != req->dst)) {
1139 scatterwalk_unmap(dst, 0);
1140 scatterwalk_done(&dst_sg_walk, 0, 0);
1141 }
1142 scatterwalk_unmap(src, 0);
1143 scatterwalk_unmap(assoc, 0);
1144 scatterwalk_done(&src_sg_walk, 0, 0);
1145 scatterwalk_done(&assoc_sg_walk, 0, 0);
1146 } else {
1147 scatterwalk_map_and_copy(dst, req->dst, 0,
1148 req->cryptlen + auth_tag_len, 1);
1149 kfree(src);
1150 }
1151 return 0;
1152 }
1153
1154 static int __driver_rfc4106_decrypt(struct aead_request *req)
1155 {
1156 u8 one_entry_in_sg = 0;
1157 u8 *src, *dst, *assoc;
1158 unsigned long tempCipherLen = 0;
1159 __be32 counter = cpu_to_be32(1);
1160 int retval = 0;
1161 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1162 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1163 void *aes_ctx = &(ctx->aes_key_expanded);
1164 unsigned long auth_tag_len = crypto_aead_authsize(tfm);
1165 u8 iv_and_authTag[32+AESNI_ALIGN];
1166 u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN);
1167 u8 *authTag = iv + 16;
1168 struct scatter_walk src_sg_walk;
1169 struct scatter_walk assoc_sg_walk;
1170 struct scatter_walk dst_sg_walk;
1171 unsigned int i;
1172
1173 if (unlikely((req->cryptlen < auth_tag_len) ||
1174 (req->assoclen != 8 && req->assoclen != 12)))
1175 return -EINVAL;
1176 /* Assuming we are supporting rfc4106 64-bit extended */
1177 /* sequence numbers We need to have the AAD length */
1178 /* equal to 8 or 12 bytes */
1179
1180 tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
1181 /* IV below built */
1182 for (i = 0; i < 4; i++)
1183 *(iv+i) = ctx->nonce[i];
1184 for (i = 0; i < 8; i++)
1185 *(iv+4+i) = req->iv[i];
1186 *((__be32 *)(iv+12)) = counter;
1187
1188 if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
1189 one_entry_in_sg = 1;
1190 scatterwalk_start(&src_sg_walk, req->src);
1191 scatterwalk_start(&assoc_sg_walk, req->assoc);
1192 src = scatterwalk_map(&src_sg_walk, 0);
1193 assoc = scatterwalk_map(&assoc_sg_walk, 0);
1194 dst = src;
1195 if (unlikely(req->src != req->dst)) {
1196 scatterwalk_start(&dst_sg_walk, req->dst);
1197 dst = scatterwalk_map(&dst_sg_walk, 0);
1198 }
1199
1200 } else {
1201 /* Allocate memory for src, dst, assoc */
1202 src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
1203 if (!src)
1204 return -ENOMEM;
1205 assoc = (src + req->cryptlen + auth_tag_len);
1206 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
1207 scatterwalk_map_and_copy(assoc, req->assoc, 0,
1208 req->assoclen, 0);
1209 dst = src;
1210 }
1211
1212 aesni_gcm_dec(aes_ctx, dst, src, tempCipherLen, iv,
1213 ctx->hash_subkey, assoc, (unsigned long)req->assoclen,
1214 authTag, auth_tag_len);
1215
1216 /* Compare generated tag with passed in tag. */
1217 retval = memcmp(src + tempCipherLen, authTag, auth_tag_len) ?
1218 -EBADMSG : 0;
1219
1220 if (one_entry_in_sg) {
1221 if (unlikely(req->src != req->dst)) {
1222 scatterwalk_unmap(dst, 0);
1223 scatterwalk_done(&dst_sg_walk, 0, 0);
1224 }
1225 scatterwalk_unmap(src, 0);
1226 scatterwalk_unmap(assoc, 0);
1227 scatterwalk_done(&src_sg_walk, 0, 0);
1228 scatterwalk_done(&assoc_sg_walk, 0, 0);
1229 } else {
1230 scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1);
1231 kfree(src);
1232 }
1233 return retval;
1234 }
1235
1236 static struct crypto_alg __rfc4106_alg = {
1237 .cra_name = "__gcm-aes-aesni",
1238 .cra_driver_name = "__driver-gcm-aes-aesni",
1239 .cra_priority = 0,
1240 .cra_flags = CRYPTO_ALG_TYPE_AEAD,
1241 .cra_blocksize = 1,
1242 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN,
1243 .cra_alignmask = 0,
1244 .cra_type = &crypto_aead_type,
1245 .cra_module = THIS_MODULE,
1246 .cra_list = LIST_HEAD_INIT(__rfc4106_alg.cra_list),
1247 .cra_u = {
1248 .aead = {
1249 .encrypt = __driver_rfc4106_encrypt,
1250 .decrypt = __driver_rfc4106_decrypt,
1251 },
1252 },
1253 };
1254 #endif
1255
1256 static int __init aesni_init(void)
1257 {
1258 int err;
1259
1260 if (!cpu_has_aes) {
1261 printk(KERN_INFO "Intel AES-NI instructions are not detected.\n");
1262 return -ENODEV;
1263 }
1264
1265 if ((err = crypto_fpu_init()))
1266 goto fpu_err;
1267 if ((err = crypto_register_alg(&aesni_alg)))
1268 goto aes_err;
1269 if ((err = crypto_register_alg(&__aesni_alg)))
1270 goto __aes_err;
1271 if ((err = crypto_register_alg(&blk_ecb_alg)))
1272 goto blk_ecb_err;
1273 if ((err = crypto_register_alg(&blk_cbc_alg)))
1274 goto blk_cbc_err;
1275 if ((err = crypto_register_alg(&ablk_ecb_alg)))
1276 goto ablk_ecb_err;
1277 if ((err = crypto_register_alg(&ablk_cbc_alg)))
1278 goto ablk_cbc_err;
1279 #ifdef CONFIG_X86_64
1280 if ((err = crypto_register_alg(&blk_ctr_alg)))
1281 goto blk_ctr_err;
1282 if ((err = crypto_register_alg(&ablk_ctr_alg)))
1283 goto ablk_ctr_err;
1284 if ((err = crypto_register_alg(&__rfc4106_alg)))
1285 goto __aead_gcm_err;
1286 if ((err = crypto_register_alg(&rfc4106_alg)))
1287 goto aead_gcm_err;
1288 #ifdef HAS_CTR
1289 if ((err = crypto_register_alg(&ablk_rfc3686_ctr_alg)))
1290 goto ablk_rfc3686_ctr_err;
1291 #endif
1292 #endif
1293 #ifdef HAS_LRW
1294 if ((err = crypto_register_alg(&ablk_lrw_alg)))
1295 goto ablk_lrw_err;
1296 #endif
1297 #ifdef HAS_PCBC
1298 if ((err = crypto_register_alg(&ablk_pcbc_alg)))
1299 goto ablk_pcbc_err;
1300 #endif
1301 #ifdef HAS_XTS
1302 if ((err = crypto_register_alg(&ablk_xts_alg)))
1303 goto ablk_xts_err;
1304 #endif
1305 return err;
1306
1307 #ifdef HAS_XTS
1308 ablk_xts_err:
1309 #endif
1310 #ifdef HAS_PCBC
1311 crypto_unregister_alg(&ablk_pcbc_alg);
1312 ablk_pcbc_err:
1313 #endif
1314 #ifdef HAS_LRW
1315 crypto_unregister_alg(&ablk_lrw_alg);
1316 ablk_lrw_err:
1317 #endif
1318 #ifdef CONFIG_X86_64
1319 #ifdef HAS_CTR
1320 crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
1321 ablk_rfc3686_ctr_err:
1322 #endif
1323 crypto_unregister_alg(&rfc4106_alg);
1324 aead_gcm_err:
1325 crypto_unregister_alg(&__rfc4106_alg);
1326 __aead_gcm_err:
1327 crypto_unregister_alg(&ablk_ctr_alg);
1328 ablk_ctr_err:
1329 crypto_unregister_alg(&blk_ctr_alg);
1330 blk_ctr_err:
1331 #endif
1332 crypto_unregister_alg(&ablk_cbc_alg);
1333 ablk_cbc_err:
1334 crypto_unregister_alg(&ablk_ecb_alg);
1335 ablk_ecb_err:
1336 crypto_unregister_alg(&blk_cbc_alg);
1337 blk_cbc_err:
1338 crypto_unregister_alg(&blk_ecb_alg);
1339 blk_ecb_err:
1340 crypto_unregister_alg(&__aesni_alg);
1341 __aes_err:
1342 crypto_unregister_alg(&aesni_alg);
1343 aes_err:
1344 fpu_err:
1345 return err;
1346 }
1347
1348 static void __exit aesni_exit(void)
1349 {
1350 #ifdef HAS_XTS
1351 crypto_unregister_alg(&ablk_xts_alg);
1352 #endif
1353 #ifdef HAS_PCBC
1354 crypto_unregister_alg(&ablk_pcbc_alg);
1355 #endif
1356 #ifdef HAS_LRW
1357 crypto_unregister_alg(&ablk_lrw_alg);
1358 #endif
1359 #ifdef CONFIG_X86_64
1360 #ifdef HAS_CTR
1361 crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
1362 #endif
1363 crypto_unregister_alg(&rfc4106_alg);
1364 crypto_unregister_alg(&__rfc4106_alg);
1365 crypto_unregister_alg(&ablk_ctr_alg);
1366 crypto_unregister_alg(&blk_ctr_alg);
1367 #endif
1368 crypto_unregister_alg(&ablk_cbc_alg);
1369 crypto_unregister_alg(&ablk_ecb_alg);
1370 crypto_unregister_alg(&blk_cbc_alg);
1371 crypto_unregister_alg(&blk_ecb_alg);
1372 crypto_unregister_alg(&__aesni_alg);
1373 crypto_unregister_alg(&aesni_alg);
1374
1375 crypto_fpu_exit();
1376 }
1377
1378 module_init(aesni_init);
1379 module_exit(aesni_exit);
1380
1381 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1382 MODULE_LICENSE("GPL");
1383 MODULE_ALIAS("aes");