?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
IPVS: netns, svc counters moved in ip_vs_ctl,c
[linux/fpc-iii.git] / arch / x86 / crypto / aesni-intel_glue.c
blob2cb3dcc4490ae0f7e17690b467c534a32777531a
1 /*
2 * Support for Intel AES-NI instructions. This file contains glue
3 * code, the real AES implementation is in intel-aes_asm.S.
4 *
5 * Copyright (C) 2008, Intel Corp.
6 * Author: Huang Ying <ying.huang@intel.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 */
13
14 #include <linux/hardirq.h>
15 #include <linux/types.h>
16 #include <linux/crypto.h>
17 #include <linux/err.h>
18 #include <crypto/algapi.h>
19 #include <crypto/aes.h>
20 #include <crypto/cryptd.h>
21 #include <crypto/ctr.h>
22 #include <asm/i387.h>
23 #include <asm/aes.h>
24
25 #if defined(CONFIG_CRYPTO_CTR) || defined(CONFIG_CRYPTO_CTR_MODULE)
26 #define HAS_CTR
27 #endif
28
29 #if defined(CONFIG_CRYPTO_LRW) || defined(CONFIG_CRYPTO_LRW_MODULE)
30 #define HAS_LRW
31 #endif
32
33 #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE)
34 #define HAS_PCBC
35 #endif
36
37 #if defined(CONFIG_CRYPTO_XTS) || defined(CONFIG_CRYPTO_XTS_MODULE)
38 #define HAS_XTS
39 #endif
40
41 struct async_aes_ctx {
42 struct cryptd_ablkcipher *cryptd_tfm;
43 };
44
45 #define AESNI_ALIGN 16
46 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1))
47
48 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
49 unsigned int key_len);
50 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
51 const u8 *in);
52 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
53 const u8 *in);
54 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
55 const u8 *in, unsigned int len);
56 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
57 const u8 *in, unsigned int len);
58 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
59 const u8 *in, unsigned int len, u8 *iv);
60 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
61 const u8 *in, unsigned int len, u8 *iv);
62 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
63 const u8 *in, unsigned int len, u8 *iv);
64
65 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
66 {
67 unsigned long addr = (unsigned long)raw_ctx;
68 unsigned long align = AESNI_ALIGN;
69
70 if (align <= crypto_tfm_ctx_alignment())
71 align = 1;
72 return (struct crypto_aes_ctx *)ALIGN(addr, align);
73 }
74
75 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
76 const u8 *in_key, unsigned int key_len)
77 {
78 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
79 u32 *flags = &tfm->crt_flags;
80 int err;
81
82 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
83 key_len != AES_KEYSIZE_256) {
84 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
85 return -EINVAL;
86 }
87
88 if (!irq_fpu_usable())
89 err = crypto_aes_expand_key(ctx, in_key, key_len);
90 else {
91 kernel_fpu_begin();
92 err = aesni_set_key(ctx, in_key, key_len);
93 kernel_fpu_end();
94 }
95
96 return err;
97 }
98
99 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
100 unsigned int key_len)
101 {
102 return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
103 }
104
105 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
106 {
107 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
108
109 if (!irq_fpu_usable())
110 crypto_aes_encrypt_x86(ctx, dst, src);
111 else {
112 kernel_fpu_begin();
113 aesni_enc(ctx, dst, src);
114 kernel_fpu_end();
115 }
116 }
117
118 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
119 {
120 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
121
122 if (!irq_fpu_usable())
123 crypto_aes_decrypt_x86(ctx, dst, src);
124 else {
125 kernel_fpu_begin();
126 aesni_dec(ctx, dst, src);
127 kernel_fpu_end();
128 }
129 }
130
131 static struct crypto_alg aesni_alg = {
132 .cra_name = "aes",
133 .cra_driver_name = "aes-aesni",
134 .cra_priority = 300,
135 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
136 .cra_blocksize = AES_BLOCK_SIZE,
137 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
138 .cra_alignmask = 0,
139 .cra_module = THIS_MODULE,
140 .cra_list = LIST_HEAD_INIT(aesni_alg.cra_list),
141 .cra_u = {
142 .cipher = {
143 .cia_min_keysize = AES_MIN_KEY_SIZE,
144 .cia_max_keysize = AES_MAX_KEY_SIZE,
145 .cia_setkey = aes_set_key,
146 .cia_encrypt = aes_encrypt,
147 .cia_decrypt = aes_decrypt
148 }
149 }
150 };
151
152 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
153 {
154 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
155
156 aesni_enc(ctx, dst, src);
157 }
158
159 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
160 {
161 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
162
163 aesni_dec(ctx, dst, src);
164 }
165
166 static struct crypto_alg __aesni_alg = {
167 .cra_name = "__aes-aesni",
168 .cra_driver_name = "__driver-aes-aesni",
169 .cra_priority = 0,
170 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
171 .cra_blocksize = AES_BLOCK_SIZE,
172 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
173 .cra_alignmask = 0,
174 .cra_module = THIS_MODULE,
175 .cra_list = LIST_HEAD_INIT(__aesni_alg.cra_list),
176 .cra_u = {
177 .cipher = {
178 .cia_min_keysize = AES_MIN_KEY_SIZE,
179 .cia_max_keysize = AES_MAX_KEY_SIZE,
180 .cia_setkey = aes_set_key,
181 .cia_encrypt = __aes_encrypt,
182 .cia_decrypt = __aes_decrypt
183 }
184 }
185 };
186
187 static int ecb_encrypt(struct blkcipher_desc *desc,
188 struct scatterlist *dst, struct scatterlist *src,
189 unsigned int nbytes)
190 {
191 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
192 struct blkcipher_walk walk;
193 int err;
194
195 blkcipher_walk_init(&walk, dst, src, nbytes);
196 err = blkcipher_walk_virt(desc, &walk);
197 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
198
199 kernel_fpu_begin();
200 while ((nbytes = walk.nbytes)) {
201 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
202 nbytes & AES_BLOCK_MASK);
203 nbytes &= AES_BLOCK_SIZE - 1;
204 err = blkcipher_walk_done(desc, &walk, nbytes);
205 }
206 kernel_fpu_end();
207
208 return err;
209 }
210
211 static int ecb_decrypt(struct blkcipher_desc *desc,
212 struct scatterlist *dst, struct scatterlist *src,
213 unsigned int nbytes)
214 {
215 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
216 struct blkcipher_walk walk;
217 int err;
218
219 blkcipher_walk_init(&walk, dst, src, nbytes);
220 err = blkcipher_walk_virt(desc, &walk);
221 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
222
223 kernel_fpu_begin();
224 while ((nbytes = walk.nbytes)) {
225 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
226 nbytes & AES_BLOCK_MASK);
227 nbytes &= AES_BLOCK_SIZE - 1;
228 err = blkcipher_walk_done(desc, &walk, nbytes);
229 }
230 kernel_fpu_end();
231
232 return err;
233 }
234
235 static struct crypto_alg blk_ecb_alg = {
236 .cra_name = "__ecb-aes-aesni",
237 .cra_driver_name = "__driver-ecb-aes-aesni",
238 .cra_priority = 0,
239 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
240 .cra_blocksize = AES_BLOCK_SIZE,
241 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
242 .cra_alignmask = 0,
243 .cra_type = &crypto_blkcipher_type,
244 .cra_module = THIS_MODULE,
245 .cra_list = LIST_HEAD_INIT(blk_ecb_alg.cra_list),
246 .cra_u = {
247 .blkcipher = {
248 .min_keysize = AES_MIN_KEY_SIZE,
249 .max_keysize = AES_MAX_KEY_SIZE,
250 .setkey = aes_set_key,
251 .encrypt = ecb_encrypt,
252 .decrypt = ecb_decrypt,
253 },
254 },
255 };
256
257 static int cbc_encrypt(struct blkcipher_desc *desc,
258 struct scatterlist *dst, struct scatterlist *src,
259 unsigned int nbytes)
260 {
261 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
262 struct blkcipher_walk walk;
263 int err;
264
265 blkcipher_walk_init(&walk, dst, src, nbytes);
266 err = blkcipher_walk_virt(desc, &walk);
267 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
268
269 kernel_fpu_begin();
270 while ((nbytes = walk.nbytes)) {
271 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
272 nbytes & AES_BLOCK_MASK, walk.iv);
273 nbytes &= AES_BLOCK_SIZE - 1;
274 err = blkcipher_walk_done(desc, &walk, nbytes);
275 }
276 kernel_fpu_end();
277
278 return err;
279 }
280
281 static int cbc_decrypt(struct blkcipher_desc *desc,
282 struct scatterlist *dst, struct scatterlist *src,
283 unsigned int nbytes)
284 {
285 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
286 struct blkcipher_walk walk;
287 int err;
288
289 blkcipher_walk_init(&walk, dst, src, nbytes);
290 err = blkcipher_walk_virt(desc, &walk);
291 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
292
293 kernel_fpu_begin();
294 while ((nbytes = walk.nbytes)) {
295 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
296 nbytes & AES_BLOCK_MASK, walk.iv);
297 nbytes &= AES_BLOCK_SIZE - 1;
298 err = blkcipher_walk_done(desc, &walk, nbytes);
299 }
300 kernel_fpu_end();
301
302 return err;
303 }
304
305 static struct crypto_alg blk_cbc_alg = {
306 .cra_name = "__cbc-aes-aesni",
307 .cra_driver_name = "__driver-cbc-aes-aesni",
308 .cra_priority = 0,
309 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
310 .cra_blocksize = AES_BLOCK_SIZE,
311 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
312 .cra_alignmask = 0,
313 .cra_type = &crypto_blkcipher_type,
314 .cra_module = THIS_MODULE,
315 .cra_list = LIST_HEAD_INIT(blk_cbc_alg.cra_list),
316 .cra_u = {
317 .blkcipher = {
318 .min_keysize = AES_MIN_KEY_SIZE,
319 .max_keysize = AES_MAX_KEY_SIZE,
320 .setkey = aes_set_key,
321 .encrypt = cbc_encrypt,
322 .decrypt = cbc_decrypt,
323 },
324 },
325 };
326
327 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
328 struct blkcipher_walk *walk)
329 {
330 u8 *ctrblk = walk->iv;
331 u8 keystream[AES_BLOCK_SIZE];
332 u8 *src = walk->src.virt.addr;
333 u8 *dst = walk->dst.virt.addr;
334 unsigned int nbytes = walk->nbytes;
335
336 aesni_enc(ctx, keystream, ctrblk);
337 crypto_xor(keystream, src, nbytes);
338 memcpy(dst, keystream, nbytes);
339 crypto_inc(ctrblk, AES_BLOCK_SIZE);
340 }
341
342 static int ctr_crypt(struct blkcipher_desc *desc,
343 struct scatterlist *dst, struct scatterlist *src,
344 unsigned int nbytes)
345 {
346 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
347 struct blkcipher_walk walk;
348 int err;
349
350 blkcipher_walk_init(&walk, dst, src, nbytes);
351 err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
352 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
353
354 kernel_fpu_begin();
355 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
356 aesni_ctr_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
357 nbytes & AES_BLOCK_MASK, walk.iv);
358 nbytes &= AES_BLOCK_SIZE - 1;
359 err = blkcipher_walk_done(desc, &walk, nbytes);
360 }
361 if (walk.nbytes) {
362 ctr_crypt_final(ctx, &walk);
363 err = blkcipher_walk_done(desc, &walk, 0);
364 }
365 kernel_fpu_end();
366
367 return err;
368 }
369
370 static struct crypto_alg blk_ctr_alg = {
371 .cra_name = "__ctr-aes-aesni",
372 .cra_driver_name = "__driver-ctr-aes-aesni",
373 .cra_priority = 0,
374 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
375 .cra_blocksize = 1,
376 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
377 .cra_alignmask = 0,
378 .cra_type = &crypto_blkcipher_type,
379 .cra_module = THIS_MODULE,
380 .cra_list = LIST_HEAD_INIT(blk_ctr_alg.cra_list),
381 .cra_u = {
382 .blkcipher = {
383 .min_keysize = AES_MIN_KEY_SIZE,
384 .max_keysize = AES_MAX_KEY_SIZE,
385 .ivsize = AES_BLOCK_SIZE,
386 .setkey = aes_set_key,
387 .encrypt = ctr_crypt,
388 .decrypt = ctr_crypt,
389 },
390 },
391 };
392
393 static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
394 unsigned int key_len)
395 {
396 struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
397 struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base;
398 int err;
399
400 crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
401 crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm)
402 & CRYPTO_TFM_REQ_MASK);
403 err = crypto_ablkcipher_setkey(child, key, key_len);
404 crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child)
405 & CRYPTO_TFM_RES_MASK);
406 return err;
407 }
408
409 static int ablk_encrypt(struct ablkcipher_request *req)
410 {
411 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
412 struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
413
414 if (!irq_fpu_usable()) {
415 struct ablkcipher_request *cryptd_req =
416 ablkcipher_request_ctx(req);
417 memcpy(cryptd_req, req, sizeof(*req));
418 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
419 return crypto_ablkcipher_encrypt(cryptd_req);
420 } else {
421 struct blkcipher_desc desc;
422 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
423 desc.info = req->info;
424 desc.flags = 0;
425 return crypto_blkcipher_crt(desc.tfm)->encrypt(
426 &desc, req->dst, req->src, req->nbytes);
427 }
428 }
429
430 static int ablk_decrypt(struct ablkcipher_request *req)
431 {
432 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
433 struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
434
435 if (!irq_fpu_usable()) {
436 struct ablkcipher_request *cryptd_req =
437 ablkcipher_request_ctx(req);
438 memcpy(cryptd_req, req, sizeof(*req));
439 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
440 return crypto_ablkcipher_decrypt(cryptd_req);
441 } else {
442 struct blkcipher_desc desc;
443 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
444 desc.info = req->info;
445 desc.flags = 0;
446 return crypto_blkcipher_crt(desc.tfm)->decrypt(
447 &desc, req->dst, req->src, req->nbytes);
448 }
449 }
450
451 static void ablk_exit(struct crypto_tfm *tfm)
452 {
453 struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
454
455 cryptd_free_ablkcipher(ctx->cryptd_tfm);
456 }
457
458 static void ablk_init_common(struct crypto_tfm *tfm,
459 struct cryptd_ablkcipher *cryptd_tfm)
460 {
461 struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
462
463 ctx->cryptd_tfm = cryptd_tfm;
464 tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
465 crypto_ablkcipher_reqsize(&cryptd_tfm->base);
466 }
467
468 static int ablk_ecb_init(struct crypto_tfm *tfm)
469 {
470 struct cryptd_ablkcipher *cryptd_tfm;
471
472 cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ecb-aes-aesni", 0, 0);
473 if (IS_ERR(cryptd_tfm))
474 return PTR_ERR(cryptd_tfm);
475 ablk_init_common(tfm, cryptd_tfm);
476 return 0;
477 }
478
479 static struct crypto_alg ablk_ecb_alg = {
480 .cra_name = "ecb(aes)",
481 .cra_driver_name = "ecb-aes-aesni",
482 .cra_priority = 400,
483 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
484 .cra_blocksize = AES_BLOCK_SIZE,
485 .cra_ctxsize = sizeof(struct async_aes_ctx),
486 .cra_alignmask = 0,
487 .cra_type = &crypto_ablkcipher_type,
488 .cra_module = THIS_MODULE,
489 .cra_list = LIST_HEAD_INIT(ablk_ecb_alg.cra_list),
490 .cra_init = ablk_ecb_init,
491 .cra_exit = ablk_exit,
492 .cra_u = {
493 .ablkcipher = {
494 .min_keysize = AES_MIN_KEY_SIZE,
495 .max_keysize = AES_MAX_KEY_SIZE,
496 .setkey = ablk_set_key,
497 .encrypt = ablk_encrypt,
498 .decrypt = ablk_decrypt,
499 },
500 },
501 };
502
503 static int ablk_cbc_init(struct crypto_tfm *tfm)
504 {
505 struct cryptd_ablkcipher *cryptd_tfm;
506
507 cryptd_tfm = cryptd_alloc_ablkcipher("__driver-cbc-aes-aesni", 0, 0);
508 if (IS_ERR(cryptd_tfm))
509 return PTR_ERR(cryptd_tfm);
510 ablk_init_common(tfm, cryptd_tfm);
511 return 0;
512 }
513
514 static struct crypto_alg ablk_cbc_alg = {
515 .cra_name = "cbc(aes)",
516 .cra_driver_name = "cbc-aes-aesni",
517 .cra_priority = 400,
518 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
519 .cra_blocksize = AES_BLOCK_SIZE,
520 .cra_ctxsize = sizeof(struct async_aes_ctx),
521 .cra_alignmask = 0,
522 .cra_type = &crypto_ablkcipher_type,
523 .cra_module = THIS_MODULE,
524 .cra_list = LIST_HEAD_INIT(ablk_cbc_alg.cra_list),
525 .cra_init = ablk_cbc_init,
526 .cra_exit = ablk_exit,
527 .cra_u = {
528 .ablkcipher = {
529 .min_keysize = AES_MIN_KEY_SIZE,
530 .max_keysize = AES_MAX_KEY_SIZE,
531 .ivsize = AES_BLOCK_SIZE,
532 .setkey = ablk_set_key,
533 .encrypt = ablk_encrypt,
534 .decrypt = ablk_decrypt,
535 },
536 },
537 };
538
539 static int ablk_ctr_init(struct crypto_tfm *tfm)
540 {
541 struct cryptd_ablkcipher *cryptd_tfm;
542
543 cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ctr-aes-aesni", 0, 0);
544 if (IS_ERR(cryptd_tfm))
545 return PTR_ERR(cryptd_tfm);
546 ablk_init_common(tfm, cryptd_tfm);
547 return 0;
548 }
549
550 static struct crypto_alg ablk_ctr_alg = {
551 .cra_name = "ctr(aes)",
552 .cra_driver_name = "ctr-aes-aesni",
553 .cra_priority = 400,
554 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
555 .cra_blocksize = 1,
556 .cra_ctxsize = sizeof(struct async_aes_ctx),
557 .cra_alignmask = 0,
558 .cra_type = &crypto_ablkcipher_type,
559 .cra_module = THIS_MODULE,
560 .cra_list = LIST_HEAD_INIT(ablk_ctr_alg.cra_list),
561 .cra_init = ablk_ctr_init,
562 .cra_exit = ablk_exit,
563 .cra_u = {
564 .ablkcipher = {
565 .min_keysize = AES_MIN_KEY_SIZE,
566 .max_keysize = AES_MAX_KEY_SIZE,
567 .ivsize = AES_BLOCK_SIZE,
568 .setkey = ablk_set_key,
569 .encrypt = ablk_encrypt,
570 .decrypt = ablk_encrypt,
571 .geniv = "chainiv",
572 },
573 },
574 };
575
576 #ifdef HAS_CTR
577 static int ablk_rfc3686_ctr_init(struct crypto_tfm *tfm)
578 {
579 struct cryptd_ablkcipher *cryptd_tfm;
580
581 cryptd_tfm = cryptd_alloc_ablkcipher(
582 "rfc3686(__driver-ctr-aes-aesni)", 0, 0);
583 if (IS_ERR(cryptd_tfm))
584 return PTR_ERR(cryptd_tfm);
585 ablk_init_common(tfm, cryptd_tfm);
586 return 0;
587 }
588
589 static struct crypto_alg ablk_rfc3686_ctr_alg = {
590 .cra_name = "rfc3686(ctr(aes))",
591 .cra_driver_name = "rfc3686-ctr-aes-aesni",
592 .cra_priority = 400,
593 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
594 .cra_blocksize = 1,
595 .cra_ctxsize = sizeof(struct async_aes_ctx),
596 .cra_alignmask = 0,
597 .cra_type = &crypto_ablkcipher_type,
598 .cra_module = THIS_MODULE,
599 .cra_list = LIST_HEAD_INIT(ablk_rfc3686_ctr_alg.cra_list),
600 .cra_init = ablk_rfc3686_ctr_init,
601 .cra_exit = ablk_exit,
602 .cra_u = {
603 .ablkcipher = {
604 .min_keysize = AES_MIN_KEY_SIZE+CTR_RFC3686_NONCE_SIZE,
605 .max_keysize = AES_MAX_KEY_SIZE+CTR_RFC3686_NONCE_SIZE,
606 .ivsize = CTR_RFC3686_IV_SIZE,
607 .setkey = ablk_set_key,
608 .encrypt = ablk_encrypt,
609 .decrypt = ablk_decrypt,
610 .geniv = "seqiv",
611 },
612 },
613 };
614 #endif
615
616 #ifdef HAS_LRW
617 static int ablk_lrw_init(struct crypto_tfm *tfm)
618 {
619 struct cryptd_ablkcipher *cryptd_tfm;
620
621 cryptd_tfm = cryptd_alloc_ablkcipher("fpu(lrw(__driver-aes-aesni))",
622 0, 0);
623 if (IS_ERR(cryptd_tfm))
624 return PTR_ERR(cryptd_tfm);
625 ablk_init_common(tfm, cryptd_tfm);
626 return 0;
627 }
628
629 static struct crypto_alg ablk_lrw_alg = {
630 .cra_name = "lrw(aes)",
631 .cra_driver_name = "lrw-aes-aesni",
632 .cra_priority = 400,
633 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
634 .cra_blocksize = AES_BLOCK_SIZE,
635 .cra_ctxsize = sizeof(struct async_aes_ctx),
636 .cra_alignmask = 0,
637 .cra_type = &crypto_ablkcipher_type,
638 .cra_module = THIS_MODULE,
639 .cra_list = LIST_HEAD_INIT(ablk_lrw_alg.cra_list),
640 .cra_init = ablk_lrw_init,
641 .cra_exit = ablk_exit,
642 .cra_u = {
643 .ablkcipher = {
644 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
645 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
646 .ivsize = AES_BLOCK_SIZE,
647 .setkey = ablk_set_key,
648 .encrypt = ablk_encrypt,
649 .decrypt = ablk_decrypt,
650 },
651 },
652 };
653 #endif
654
655 #ifdef HAS_PCBC
656 static int ablk_pcbc_init(struct crypto_tfm *tfm)
657 {
658 struct cryptd_ablkcipher *cryptd_tfm;
659
660 cryptd_tfm = cryptd_alloc_ablkcipher("fpu(pcbc(__driver-aes-aesni))",
661 0, 0);
662 if (IS_ERR(cryptd_tfm))
663 return PTR_ERR(cryptd_tfm);
664 ablk_init_common(tfm, cryptd_tfm);
665 return 0;
666 }
667
668 static struct crypto_alg ablk_pcbc_alg = {
669 .cra_name = "pcbc(aes)",
670 .cra_driver_name = "pcbc-aes-aesni",
671 .cra_priority = 400,
672 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
673 .cra_blocksize = AES_BLOCK_SIZE,
674 .cra_ctxsize = sizeof(struct async_aes_ctx),
675 .cra_alignmask = 0,
676 .cra_type = &crypto_ablkcipher_type,
677 .cra_module = THIS_MODULE,
678 .cra_list = LIST_HEAD_INIT(ablk_pcbc_alg.cra_list),
679 .cra_init = ablk_pcbc_init,
680 .cra_exit = ablk_exit,
681 .cra_u = {
682 .ablkcipher = {
683 .min_keysize = AES_MIN_KEY_SIZE,
684 .max_keysize = AES_MAX_KEY_SIZE,
685 .ivsize = AES_BLOCK_SIZE,
686 .setkey = ablk_set_key,
687 .encrypt = ablk_encrypt,
688 .decrypt = ablk_decrypt,
689 },
690 },
691 };
692 #endif
693
694 #ifdef HAS_XTS
695 static int ablk_xts_init(struct crypto_tfm *tfm)
696 {
697 struct cryptd_ablkcipher *cryptd_tfm;
698
699 cryptd_tfm = cryptd_alloc_ablkcipher("fpu(xts(__driver-aes-aesni))",
700 0, 0);
701 if (IS_ERR(cryptd_tfm))
702 return PTR_ERR(cryptd_tfm);
703 ablk_init_common(tfm, cryptd_tfm);
704 return 0;
705 }
706
707 static struct crypto_alg ablk_xts_alg = {
708 .cra_name = "xts(aes)",
709 .cra_driver_name = "xts-aes-aesni",
710 .cra_priority = 400,
711 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
712 .cra_blocksize = AES_BLOCK_SIZE,
713 .cra_ctxsize = sizeof(struct async_aes_ctx),
714 .cra_alignmask = 0,
715 .cra_type = &crypto_ablkcipher_type,
716 .cra_module = THIS_MODULE,
717 .cra_list = LIST_HEAD_INIT(ablk_xts_alg.cra_list),
718 .cra_init = ablk_xts_init,
719 .cra_exit = ablk_exit,
720 .cra_u = {
721 .ablkcipher = {
722 .min_keysize = 2 * AES_MIN_KEY_SIZE,
723 .max_keysize = 2 * AES_MAX_KEY_SIZE,
724 .ivsize = AES_BLOCK_SIZE,
725 .setkey = ablk_set_key,
726 .encrypt = ablk_encrypt,
727 .decrypt = ablk_decrypt,
728 },
729 },
730 };
731 #endif
732
733 static int __init aesni_init(void)
734 {
735 int err;
736
737 if (!cpu_has_aes) {
738 printk(KERN_INFO "Intel AES-NI instructions are not detected.\n");
739 return -ENODEV;
740 }
741 if ((err = crypto_register_alg(&aesni_alg)))
742 goto aes_err;
743 if ((err = crypto_register_alg(&__aesni_alg)))
744 goto __aes_err;
745 if ((err = crypto_register_alg(&blk_ecb_alg)))
746 goto blk_ecb_err;
747 if ((err = crypto_register_alg(&blk_cbc_alg)))
748 goto blk_cbc_err;
749 if ((err = crypto_register_alg(&blk_ctr_alg)))
750 goto blk_ctr_err;
751 if ((err = crypto_register_alg(&ablk_ecb_alg)))
752 goto ablk_ecb_err;
753 if ((err = crypto_register_alg(&ablk_cbc_alg)))
754 goto ablk_cbc_err;
755 if ((err = crypto_register_alg(&ablk_ctr_alg)))
756 goto ablk_ctr_err;
757 #ifdef HAS_CTR
758 if ((err = crypto_register_alg(&ablk_rfc3686_ctr_alg)))
759 goto ablk_rfc3686_ctr_err;
760 #endif
761 #ifdef HAS_LRW
762 if ((err = crypto_register_alg(&ablk_lrw_alg)))
763 goto ablk_lrw_err;
764 #endif
765 #ifdef HAS_PCBC
766 if ((err = crypto_register_alg(&ablk_pcbc_alg)))
767 goto ablk_pcbc_err;
768 #endif
769 #ifdef HAS_XTS
770 if ((err = crypto_register_alg(&ablk_xts_alg)))
771 goto ablk_xts_err;
772 #endif
773
774 return err;
775
776 #ifdef HAS_XTS
777 ablk_xts_err:
778 #endif
779 #ifdef HAS_PCBC
780 crypto_unregister_alg(&ablk_pcbc_alg);
781 ablk_pcbc_err:
782 #endif
783 #ifdef HAS_LRW
784 crypto_unregister_alg(&ablk_lrw_alg);
785 ablk_lrw_err:
786 #endif
787 #ifdef HAS_CTR
788 crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
789 ablk_rfc3686_ctr_err:
790 #endif
791 crypto_unregister_alg(&ablk_ctr_alg);
792 ablk_ctr_err:
793 crypto_unregister_alg(&ablk_cbc_alg);
794 ablk_cbc_err:
795 crypto_unregister_alg(&ablk_ecb_alg);
796 ablk_ecb_err:
797 crypto_unregister_alg(&blk_ctr_alg);
798 blk_ctr_err:
799 crypto_unregister_alg(&blk_cbc_alg);
800 blk_cbc_err:
801 crypto_unregister_alg(&blk_ecb_alg);
802 blk_ecb_err:
803 crypto_unregister_alg(&__aesni_alg);
804 __aes_err:
805 crypto_unregister_alg(&aesni_alg);
806 aes_err:
807 return err;
808 }
809
810 static void __exit aesni_exit(void)
811 {
812 #ifdef HAS_XTS
813 crypto_unregister_alg(&ablk_xts_alg);
814 #endif
815 #ifdef HAS_PCBC
816 crypto_unregister_alg(&ablk_pcbc_alg);
817 #endif
818 #ifdef HAS_LRW
819 crypto_unregister_alg(&ablk_lrw_alg);
820 #endif
821 #ifdef HAS_CTR
822 crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
823 #endif
824 crypto_unregister_alg(&ablk_ctr_alg);
825 crypto_unregister_alg(&ablk_cbc_alg);
826 crypto_unregister_alg(&ablk_ecb_alg);
827 crypto_unregister_alg(&blk_ctr_alg);
828 crypto_unregister_alg(&blk_cbc_alg);
829 crypto_unregister_alg(&blk_ecb_alg);
830 crypto_unregister_alg(&__aesni_alg);
831 crypto_unregister_alg(&aesni_alg);
832 }
833
834 module_init(aesni_init);
835 module_exit(aesni_exit);
836
837 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
838 MODULE_LICENSE("GPL");
839 MODULE_ALIAS("aes");
Linux with added FPC-III support