?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
hugetlb: introduce generic version of hugetlb_free_pgd_range
[linux/fpc-iii.git] / arch / x86 / crypto / aesni-intel_glue.c
blob661f7daf43da947acccfffe5b820ef7cebe7009c
1 /*
2 * Support for Intel AES-NI instructions. This file contains glue
3 * code, the real AES implementation is in intel-aes_asm.S.
4 *
5 * Copyright (C) 2008, Intel Corp.
6 * Author: Huang Ying <ying.huang@intel.com>
7 *
8 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
9 * interface for 64-bit kernels.
10 * Authors: Adrian Hoban <adrian.hoban@intel.com>
11 * Gabriele Paoloni <gabriele.paoloni@intel.com>
12 * Tadeusz Struk (tadeusz.struk@intel.com)
13 * Aidan O'Mahony (aidan.o.mahony@intel.com)
14 * Copyright (c) 2010, Intel Corporation.
15 *
16 * This program is free software; you can redistribute it and/or modify
17 * it under the terms of the GNU General Public License as published by
18 * the Free Software Foundation; either version 2 of the License, or
19 * (at your option) any later version.
20 */
21
22 #include <linux/hardirq.h>
23 #include <linux/types.h>
24 #include <linux/module.h>
25 #include <linux/err.h>
26 #include <crypto/algapi.h>
27 #include <crypto/aes.h>
28 #include <crypto/cryptd.h>
29 #include <crypto/ctr.h>
30 #include <crypto/b128ops.h>
31 #include <crypto/gcm.h>
32 #include <crypto/xts.h>
33 #include <asm/cpu_device_id.h>
34 #include <asm/fpu/api.h>
35 #include <asm/crypto/aes.h>
36 #include <crypto/scatterwalk.h>
37 #include <crypto/internal/aead.h>
38 #include <crypto/internal/simd.h>
39 #include <crypto/internal/skcipher.h>
40 #include <linux/workqueue.h>
41 #include <linux/spinlock.h>
42 #ifdef CONFIG_X86_64
43 #include <asm/crypto/glue_helper.h>
44 #endif
45
46
47 #define AESNI_ALIGN 16
48 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
49 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE - 1))
50 #define RFC4106_HASH_SUBKEY_SIZE 16
51 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
52 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
53 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
54
55 /* This data is stored at the end of the crypto_tfm struct.
56 * It's a type of per "session" data storage location.
57 * This needs to be 16 byte aligned.
58 */
59 struct aesni_rfc4106_gcm_ctx {
60 u8 hash_subkey[16] AESNI_ALIGN_ATTR;
61 struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
62 u8 nonce[4];
63 };
64
65 struct generic_gcmaes_ctx {
66 u8 hash_subkey[16] AESNI_ALIGN_ATTR;
67 struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
68 };
69
70 struct aesni_xts_ctx {
71 u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
72 u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
73 };
74
75 #define GCM_BLOCK_LEN 16
76
77 struct gcm_context_data {
78 /* init, update and finalize context data */
79 u8 aad_hash[GCM_BLOCK_LEN];
80 u64 aad_length;
81 u64 in_length;
82 u8 partial_block_enc_key[GCM_BLOCK_LEN];
83 u8 orig_IV[GCM_BLOCK_LEN];
84 u8 current_counter[GCM_BLOCK_LEN];
85 u64 partial_block_len;
86 u64 unused;
87 u8 hash_keys[GCM_BLOCK_LEN * 8];
88 };
89
90 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
91 unsigned int key_len);
92 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
93 const u8 *in);
94 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
95 const u8 *in);
96 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
97 const u8 *in, unsigned int len);
98 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
99 const u8 *in, unsigned int len);
100 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
101 const u8 *in, unsigned int len, u8 *iv);
102 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
103 const u8 *in, unsigned int len, u8 *iv);
104
105 #define AVX_GEN2_OPTSIZE 640
106 #define AVX_GEN4_OPTSIZE 4096
107
108 #ifdef CONFIG_X86_64
109
110 static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out,
111 const u8 *in, unsigned int len, u8 *iv);
112 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
113 const u8 *in, unsigned int len, u8 *iv);
114
115 asmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out,
116 const u8 *in, bool enc, u8 *iv);
117
118 /* asmlinkage void aesni_gcm_enc()
119 * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
120 * struct gcm_context_data. May be uninitialized.
121 * u8 *out, Ciphertext output. Encrypt in-place is allowed.
122 * const u8 *in, Plaintext input
123 * unsigned long plaintext_len, Length of data in bytes for encryption.
124 * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
125 * 16-byte aligned pointer.
126 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
127 * const u8 *aad, Additional Authentication Data (AAD)
128 * unsigned long aad_len, Length of AAD in bytes.
129 * u8 *auth_tag, Authenticated Tag output.
130 * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
131 * Valid values are 16 (most likely), 12 or 8.
132 */
133 asmlinkage void aesni_gcm_enc(void *ctx,
134 struct gcm_context_data *gdata, u8 *out,
135 const u8 *in, unsigned long plaintext_len, u8 *iv,
136 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
137 u8 *auth_tag, unsigned long auth_tag_len);
138
139 /* asmlinkage void aesni_gcm_dec()
140 * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
141 * struct gcm_context_data. May be uninitialized.
142 * u8 *out, Plaintext output. Decrypt in-place is allowed.
143 * const u8 *in, Ciphertext input
144 * unsigned long ciphertext_len, Length of data in bytes for decryption.
145 * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
146 * 16-byte aligned pointer.
147 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
148 * const u8 *aad, Additional Authentication Data (AAD)
149 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
150 * to be 8 or 12 bytes
151 * u8 *auth_tag, Authenticated Tag output.
152 * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
153 * Valid values are 16 (most likely), 12 or 8.
154 */
155 asmlinkage void aesni_gcm_dec(void *ctx,
156 struct gcm_context_data *gdata, u8 *out,
157 const u8 *in, unsigned long ciphertext_len, u8 *iv,
158 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
159 u8 *auth_tag, unsigned long auth_tag_len);
160
161 /* Scatter / Gather routines, with args similar to above */
162 asmlinkage void aesni_gcm_init(void *ctx,
163 struct gcm_context_data *gdata,
164 u8 *iv,
165 u8 *hash_subkey, const u8 *aad,
166 unsigned long aad_len);
167 asmlinkage void aesni_gcm_enc_update(void *ctx,
168 struct gcm_context_data *gdata, u8 *out,
169 const u8 *in, unsigned long plaintext_len);
170 asmlinkage void aesni_gcm_dec_update(void *ctx,
171 struct gcm_context_data *gdata, u8 *out,
172 const u8 *in,
173 unsigned long ciphertext_len);
174 asmlinkage void aesni_gcm_finalize(void *ctx,
175 struct gcm_context_data *gdata,
176 u8 *auth_tag, unsigned long auth_tag_len);
177
178 #ifdef CONFIG_AS_AVX
179 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,
180 void *keys, u8 *out, unsigned int num_bytes);
181 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
182 void *keys, u8 *out, unsigned int num_bytes);
183 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
184 void *keys, u8 *out, unsigned int num_bytes);
185 /*
186 * asmlinkage void aesni_gcm_precomp_avx_gen2()
187 * gcm_data *my_ctx_data, context data
188 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
189 */
190 asmlinkage void aesni_gcm_precomp_avx_gen2(void *my_ctx_data, u8 *hash_subkey);
191
192 asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx, u8 *out,
193 const u8 *in, unsigned long plaintext_len, u8 *iv,
194 const u8 *aad, unsigned long aad_len,
195 u8 *auth_tag, unsigned long auth_tag_len);
196
197 asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx, u8 *out,
198 const u8 *in, unsigned long ciphertext_len, u8 *iv,
199 const u8 *aad, unsigned long aad_len,
200 u8 *auth_tag, unsigned long auth_tag_len);
201
202 static void aesni_gcm_enc_avx(void *ctx,
203 struct gcm_context_data *data, u8 *out,
204 const u8 *in, unsigned long plaintext_len, u8 *iv,
205 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
206 u8 *auth_tag, unsigned long auth_tag_len)
207 {
208 struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
209 if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)){
210 aesni_gcm_enc(ctx, data, out, in,
211 plaintext_len, iv, hash_subkey, aad,
212 aad_len, auth_tag, auth_tag_len);
213 } else {
214 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
215 aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad,
216 aad_len, auth_tag, auth_tag_len);
217 }
218 }
219
220 static void aesni_gcm_dec_avx(void *ctx,
221 struct gcm_context_data *data, u8 *out,
222 const u8 *in, unsigned long ciphertext_len, u8 *iv,
223 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
224 u8 *auth_tag, unsigned long auth_tag_len)
225 {
226 struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
227 if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) {
228 aesni_gcm_dec(ctx, data, out, in,
229 ciphertext_len, iv, hash_subkey, aad,
230 aad_len, auth_tag, auth_tag_len);
231 } else {
232 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
233 aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad,
234 aad_len, auth_tag, auth_tag_len);
235 }
236 }
237 #endif
238
239 #ifdef CONFIG_AS_AVX2
240 /*
241 * asmlinkage void aesni_gcm_precomp_avx_gen4()
242 * gcm_data *my_ctx_data, context data
243 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
244 */
245 asmlinkage void aesni_gcm_precomp_avx_gen4(void *my_ctx_data, u8 *hash_subkey);
246
247 asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx, u8 *out,
248 const u8 *in, unsigned long plaintext_len, u8 *iv,
249 const u8 *aad, unsigned long aad_len,
250 u8 *auth_tag, unsigned long auth_tag_len);
251
252 asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx, u8 *out,
253 const u8 *in, unsigned long ciphertext_len, u8 *iv,
254 const u8 *aad, unsigned long aad_len,
255 u8 *auth_tag, unsigned long auth_tag_len);
256
257 static void aesni_gcm_enc_avx2(void *ctx,
258 struct gcm_context_data *data, u8 *out,
259 const u8 *in, unsigned long plaintext_len, u8 *iv,
260 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
261 u8 *auth_tag, unsigned long auth_tag_len)
262 {
263 struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
264 if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) {
265 aesni_gcm_enc(ctx, data, out, in,
266 plaintext_len, iv, hash_subkey, aad,
267 aad_len, auth_tag, auth_tag_len);
268 } else if (plaintext_len < AVX_GEN4_OPTSIZE) {
269 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
270 aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad,
271 aad_len, auth_tag, auth_tag_len);
272 } else {
273 aesni_gcm_precomp_avx_gen4(ctx, hash_subkey);
274 aesni_gcm_enc_avx_gen4(ctx, out, in, plaintext_len, iv, aad,
275 aad_len, auth_tag, auth_tag_len);
276 }
277 }
278
279 static void aesni_gcm_dec_avx2(void *ctx,
280 struct gcm_context_data *data, u8 *out,
281 const u8 *in, unsigned long ciphertext_len, u8 *iv,
282 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
283 u8 *auth_tag, unsigned long auth_tag_len)
284 {
285 struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
286 if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) {
287 aesni_gcm_dec(ctx, data, out, in,
288 ciphertext_len, iv, hash_subkey,
289 aad, aad_len, auth_tag, auth_tag_len);
290 } else if (ciphertext_len < AVX_GEN4_OPTSIZE) {
291 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
292 aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad,
293 aad_len, auth_tag, auth_tag_len);
294 } else {
295 aesni_gcm_precomp_avx_gen4(ctx, hash_subkey);
296 aesni_gcm_dec_avx_gen4(ctx, out, in, ciphertext_len, iv, aad,
297 aad_len, auth_tag, auth_tag_len);
298 }
299 }
300 #endif
301
302 static void (*aesni_gcm_enc_tfm)(void *ctx,
303 struct gcm_context_data *data, u8 *out,
304 const u8 *in, unsigned long plaintext_len,
305 u8 *iv, u8 *hash_subkey, const u8 *aad,
306 unsigned long aad_len, u8 *auth_tag,
307 unsigned long auth_tag_len);
308
309 static void (*aesni_gcm_dec_tfm)(void *ctx,
310 struct gcm_context_data *data, u8 *out,
311 const u8 *in, unsigned long ciphertext_len,
312 u8 *iv, u8 *hash_subkey, const u8 *aad,
313 unsigned long aad_len, u8 *auth_tag,
314 unsigned long auth_tag_len);
315
316 static inline struct
317 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
318 {
319 unsigned long align = AESNI_ALIGN;
320
321 if (align <= crypto_tfm_ctx_alignment())
322 align = 1;
323 return PTR_ALIGN(crypto_aead_ctx(tfm), align);
324 }
325
326 static inline struct
327 generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm)
328 {
329 unsigned long align = AESNI_ALIGN;
330
331 if (align <= crypto_tfm_ctx_alignment())
332 align = 1;
333 return PTR_ALIGN(crypto_aead_ctx(tfm), align);
334 }
335 #endif
336
337 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
338 {
339 unsigned long addr = (unsigned long)raw_ctx;
340 unsigned long align = AESNI_ALIGN;
341
342 if (align <= crypto_tfm_ctx_alignment())
343 align = 1;
344 return (struct crypto_aes_ctx *)ALIGN(addr, align);
345 }
346
347 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
348 const u8 *in_key, unsigned int key_len)
349 {
350 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
351 u32 *flags = &tfm->crt_flags;
352 int err;
353
354 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
355 key_len != AES_KEYSIZE_256) {
356 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
357 return -EINVAL;
358 }
359
360 if (!irq_fpu_usable())
361 err = crypto_aes_expand_key(ctx, in_key, key_len);
362 else {
363 kernel_fpu_begin();
364 err = aesni_set_key(ctx, in_key, key_len);
365 kernel_fpu_end();
366 }
367
368 return err;
369 }
370
371 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
372 unsigned int key_len)
373 {
374 return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
375 }
376
377 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
378 {
379 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
380
381 if (!irq_fpu_usable())
382 crypto_aes_encrypt_x86(ctx, dst, src);
383 else {
384 kernel_fpu_begin();
385 aesni_enc(ctx, dst, src);
386 kernel_fpu_end();
387 }
388 }
389
390 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
391 {
392 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
393
394 if (!irq_fpu_usable())
395 crypto_aes_decrypt_x86(ctx, dst, src);
396 else {
397 kernel_fpu_begin();
398 aesni_dec(ctx, dst, src);
399 kernel_fpu_end();
400 }
401 }
402
403 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
404 {
405 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
406
407 aesni_enc(ctx, dst, src);
408 }
409
410 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
411 {
412 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
413
414 aesni_dec(ctx, dst, src);
415 }
416
417 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
418 unsigned int len)
419 {
420 return aes_set_key_common(crypto_skcipher_tfm(tfm),
421 crypto_skcipher_ctx(tfm), key, len);
422 }
423
424 static int ecb_encrypt(struct skcipher_request *req)
425 {
426 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
427 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
428 struct skcipher_walk walk;
429 unsigned int nbytes;
430 int err;
431
432 err = skcipher_walk_virt(&walk, req, true);
433
434 kernel_fpu_begin();
435 while ((nbytes = walk.nbytes)) {
436 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
437 nbytes & AES_BLOCK_MASK);
438 nbytes &= AES_BLOCK_SIZE - 1;
439 err = skcipher_walk_done(&walk, nbytes);
440 }
441 kernel_fpu_end();
442
443 return err;
444 }
445
446 static int ecb_decrypt(struct skcipher_request *req)
447 {
448 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
449 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
450 struct skcipher_walk walk;
451 unsigned int nbytes;
452 int err;
453
454 err = skcipher_walk_virt(&walk, req, true);
455
456 kernel_fpu_begin();
457 while ((nbytes = walk.nbytes)) {
458 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
459 nbytes & AES_BLOCK_MASK);
460 nbytes &= AES_BLOCK_SIZE - 1;
461 err = skcipher_walk_done(&walk, nbytes);
462 }
463 kernel_fpu_end();
464
465 return err;
466 }
467
468 static int cbc_encrypt(struct skcipher_request *req)
469 {
470 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
471 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
472 struct skcipher_walk walk;
473 unsigned int nbytes;
474 int err;
475
476 err = skcipher_walk_virt(&walk, req, true);
477
478 kernel_fpu_begin();
479 while ((nbytes = walk.nbytes)) {
480 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
481 nbytes & AES_BLOCK_MASK, walk.iv);
482 nbytes &= AES_BLOCK_SIZE - 1;
483 err = skcipher_walk_done(&walk, nbytes);
484 }
485 kernel_fpu_end();
486
487 return err;
488 }
489
490 static int cbc_decrypt(struct skcipher_request *req)
491 {
492 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
493 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
494 struct skcipher_walk walk;
495 unsigned int nbytes;
496 int err;
497
498 err = skcipher_walk_virt(&walk, req, true);
499
500 kernel_fpu_begin();
501 while ((nbytes = walk.nbytes)) {
502 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
503 nbytes & AES_BLOCK_MASK, walk.iv);
504 nbytes &= AES_BLOCK_SIZE - 1;
505 err = skcipher_walk_done(&walk, nbytes);
506 }
507 kernel_fpu_end();
508
509 return err;
510 }
511
512 #ifdef CONFIG_X86_64
513 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
514 struct skcipher_walk *walk)
515 {
516 u8 *ctrblk = walk->iv;
517 u8 keystream[AES_BLOCK_SIZE];
518 u8 *src = walk->src.virt.addr;
519 u8 *dst = walk->dst.virt.addr;
520 unsigned int nbytes = walk->nbytes;
521
522 aesni_enc(ctx, keystream, ctrblk);
523 crypto_xor_cpy(dst, keystream, src, nbytes);
524
525 crypto_inc(ctrblk, AES_BLOCK_SIZE);
526 }
527
528 #ifdef CONFIG_AS_AVX
529 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
530 const u8 *in, unsigned int len, u8 *iv)
531 {
532 /*
533 * based on key length, override with the by8 version
534 * of ctr mode encryption/decryption for improved performance
535 * aes_set_key_common() ensures that key length is one of
536 * {128,192,256}
537 */
538 if (ctx->key_length == AES_KEYSIZE_128)
539 aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len);
540 else if (ctx->key_length == AES_KEYSIZE_192)
541 aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len);
542 else
543 aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);
544 }
545 #endif
546
547 static int ctr_crypt(struct skcipher_request *req)
548 {
549 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
550 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
551 struct skcipher_walk walk;
552 unsigned int nbytes;
553 int err;
554
555 err = skcipher_walk_virt(&walk, req, true);
556
557 kernel_fpu_begin();
558 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
559 aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr,
560 nbytes & AES_BLOCK_MASK, walk.iv);
561 nbytes &= AES_BLOCK_SIZE - 1;
562 err = skcipher_walk_done(&walk, nbytes);
563 }
564 if (walk.nbytes) {
565 ctr_crypt_final(ctx, &walk);
566 err = skcipher_walk_done(&walk, 0);
567 }
568 kernel_fpu_end();
569
570 return err;
571 }
572
573 static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key,
574 unsigned int keylen)
575 {
576 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
577 int err;
578
579 err = xts_verify_key(tfm, key, keylen);
580 if (err)
581 return err;
582
583 keylen /= 2;
584
585 /* first half of xts-key is for crypt */
586 err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx,
587 key, keylen);
588 if (err)
589 return err;
590
591 /* second half of xts-key is for tweak */
592 return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx,
593 key + keylen, keylen);
594 }
595
596
597 static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in)
598 {
599 aesni_enc(ctx, out, in);
600 }
601
602 static void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
603 {
604 glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc));
605 }
606
607 static void aesni_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
608 {
609 glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_dec));
610 }
611
612 static void aesni_xts_enc8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
613 {
614 aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, true, (u8 *)iv);
615 }
616
617 static void aesni_xts_dec8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
618 {
619 aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, false, (u8 *)iv);
620 }
621
622 static const struct common_glue_ctx aesni_enc_xts = {
623 .num_funcs = 2,
624 .fpu_blocks_limit = 1,
625
626 .funcs = { {
627 .num_blocks = 8,
628 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc8) }
629 }, {
630 .num_blocks = 1,
631 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc) }
632 } }
633 };
634
635 static const struct common_glue_ctx aesni_dec_xts = {
636 .num_funcs = 2,
637 .fpu_blocks_limit = 1,
638
639 .funcs = { {
640 .num_blocks = 8,
641 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec8) }
642 }, {
643 .num_blocks = 1,
644 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec) }
645 } }
646 };
647
648 static int xts_encrypt(struct skcipher_request *req)
649 {
650 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
651 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
652
653 return glue_xts_req_128bit(&aesni_enc_xts, req,
654 XTS_TWEAK_CAST(aesni_xts_tweak),
655 aes_ctx(ctx->raw_tweak_ctx),
656 aes_ctx(ctx->raw_crypt_ctx));
657 }
658
659 static int xts_decrypt(struct skcipher_request *req)
660 {
661 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
662 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
663
664 return glue_xts_req_128bit(&aesni_dec_xts, req,
665 XTS_TWEAK_CAST(aesni_xts_tweak),
666 aes_ctx(ctx->raw_tweak_ctx),
667 aes_ctx(ctx->raw_crypt_ctx));
668 }
669
670 static int rfc4106_init(struct crypto_aead *aead)
671 {
672 struct cryptd_aead *cryptd_tfm;
673 struct cryptd_aead **ctx = crypto_aead_ctx(aead);
674
675 cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni",
676 CRYPTO_ALG_INTERNAL,
677 CRYPTO_ALG_INTERNAL);
678 if (IS_ERR(cryptd_tfm))
679 return PTR_ERR(cryptd_tfm);
680
681 *ctx = cryptd_tfm;
682 crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
683 return 0;
684 }
685
686 static void rfc4106_exit(struct crypto_aead *aead)
687 {
688 struct cryptd_aead **ctx = crypto_aead_ctx(aead);
689
690 cryptd_free_aead(*ctx);
691 }
692
693 static int
694 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
695 {
696 struct crypto_cipher *tfm;
697 int ret;
698
699 tfm = crypto_alloc_cipher("aes", 0, 0);
700 if (IS_ERR(tfm))
701 return PTR_ERR(tfm);
702
703 ret = crypto_cipher_setkey(tfm, key, key_len);
704 if (ret)
705 goto out_free_cipher;
706
707 /* Clear the data in the hash sub key container to zero.*/
708 /* We want to cipher all zeros to create the hash sub key. */
709 memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
710
711 crypto_cipher_encrypt_one(tfm, hash_subkey, hash_subkey);
712
713 out_free_cipher:
714 crypto_free_cipher(tfm);
715 return ret;
716 }
717
718 static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
719 unsigned int key_len)
720 {
721 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead);
722
723 if (key_len < 4) {
724 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
725 return -EINVAL;
726 }
727 /*Account for 4 byte nonce at the end.*/
728 key_len -= 4;
729
730 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
731
732 return aes_set_key_common(crypto_aead_tfm(aead),
733 &ctx->aes_key_expanded, key, key_len) ?:
734 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
735 }
736
737 static int gcmaes_wrapper_set_key(struct crypto_aead *parent, const u8 *key,
738 unsigned int key_len)
739 {
740 struct cryptd_aead **ctx = crypto_aead_ctx(parent);
741 struct cryptd_aead *cryptd_tfm = *ctx;
742
743 return crypto_aead_setkey(&cryptd_tfm->base, key, key_len);
744 }
745
746 static int common_rfc4106_set_authsize(struct crypto_aead *aead,
747 unsigned int authsize)
748 {
749 switch (authsize) {
750 case 8:
751 case 12:
752 case 16:
753 break;
754 default:
755 return -EINVAL;
756 }
757
758 return 0;
759 }
760
761 /* This is the Integrity Check Value (aka the authentication tag length and can
762 * be 8, 12 or 16 bytes long. */
763 static int gcmaes_wrapper_set_authsize(struct crypto_aead *parent,
764 unsigned int authsize)
765 {
766 struct cryptd_aead **ctx = crypto_aead_ctx(parent);
767 struct cryptd_aead *cryptd_tfm = *ctx;
768
769 return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
770 }
771
772 static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,
773 unsigned int authsize)
774 {
775 switch (authsize) {
776 case 4:
777 case 8:
778 case 12:
779 case 13:
780 case 14:
781 case 15:
782 case 16:
783 break;
784 default:
785 return -EINVAL;
786 }
787
788 return 0;
789 }
790
791 static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req,
792 unsigned int assoclen, u8 *hash_subkey,
793 u8 *iv, void *aes_ctx)
794 {
795 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
796 unsigned long auth_tag_len = crypto_aead_authsize(tfm);
797 struct gcm_context_data data AESNI_ALIGN_ATTR;
798 struct scatter_walk dst_sg_walk = {};
799 unsigned long left = req->cryptlen;
800 unsigned long len, srclen, dstlen;
801 struct scatter_walk assoc_sg_walk;
802 struct scatter_walk src_sg_walk;
803 struct scatterlist src_start[2];
804 struct scatterlist dst_start[2];
805 struct scatterlist *src_sg;
806 struct scatterlist *dst_sg;
807 u8 *src, *dst, *assoc;
808 u8 *assocmem = NULL;
809 u8 authTag[16];
810
811 if (!enc)
812 left -= auth_tag_len;
813
814 /* Linearize assoc, if not already linear */
815 if (req->src->length >= assoclen && req->src->length &&
816 (!PageHighMem(sg_page(req->src)) ||
817 req->src->offset + req->src->length <= PAGE_SIZE)) {
818 scatterwalk_start(&assoc_sg_walk, req->src);
819 assoc = scatterwalk_map(&assoc_sg_walk);
820 } else {
821 /* assoc can be any length, so must be on heap */
822 assocmem = kmalloc(assoclen, GFP_ATOMIC);
823 if (unlikely(!assocmem))
824 return -ENOMEM;
825 assoc = assocmem;
826
827 scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0);
828 }
829
830 src_sg = scatterwalk_ffwd(src_start, req->src, req->assoclen);
831 scatterwalk_start(&src_sg_walk, src_sg);
832 if (req->src != req->dst) {
833 dst_sg = scatterwalk_ffwd(dst_start, req->dst, req->assoclen);
834 scatterwalk_start(&dst_sg_walk, dst_sg);
835 }
836
837 kernel_fpu_begin();
838 aesni_gcm_init(aes_ctx, &data, iv,
839 hash_subkey, assoc, assoclen);
840 if (req->src != req->dst) {
841 while (left) {
842 src = scatterwalk_map(&src_sg_walk);
843 dst = scatterwalk_map(&dst_sg_walk);
844 srclen = scatterwalk_clamp(&src_sg_walk, left);
845 dstlen = scatterwalk_clamp(&dst_sg_walk, left);
846 len = min(srclen, dstlen);
847 if (len) {
848 if (enc)
849 aesni_gcm_enc_update(aes_ctx, &data,
850 dst, src, len);
851 else
852 aesni_gcm_dec_update(aes_ctx, &data,
853 dst, src, len);
854 }
855 left -= len;
856
857 scatterwalk_unmap(src);
858 scatterwalk_unmap(dst);
859 scatterwalk_advance(&src_sg_walk, len);
860 scatterwalk_advance(&dst_sg_walk, len);
861 scatterwalk_done(&src_sg_walk, 0, left);
862 scatterwalk_done(&dst_sg_walk, 1, left);
863 }
864 } else {
865 while (left) {
866 dst = src = scatterwalk_map(&src_sg_walk);
867 len = scatterwalk_clamp(&src_sg_walk, left);
868 if (len) {
869 if (enc)
870 aesni_gcm_enc_update(aes_ctx, &data,
871 src, src, len);
872 else
873 aesni_gcm_dec_update(aes_ctx, &data,
874 src, src, len);
875 }
876 left -= len;
877 scatterwalk_unmap(src);
878 scatterwalk_advance(&src_sg_walk, len);
879 scatterwalk_done(&src_sg_walk, 1, left);
880 }
881 }
882 aesni_gcm_finalize(aes_ctx, &data, authTag, auth_tag_len);
883 kernel_fpu_end();
884
885 if (!assocmem)
886 scatterwalk_unmap(assoc);
887 else
888 kfree(assocmem);
889
890 if (!enc) {
891 u8 authTagMsg[16];
892
893 /* Copy out original authTag */
894 scatterwalk_map_and_copy(authTagMsg, req->src,
895 req->assoclen + req->cryptlen -
896 auth_tag_len,
897 auth_tag_len, 0);
898
899 /* Compare generated tag with passed in tag. */
900 return crypto_memneq(authTagMsg, authTag, auth_tag_len) ?
901 -EBADMSG : 0;
902 }
903
904 /* Copy in the authTag */
905 scatterwalk_map_and_copy(authTag, req->dst,
906 req->assoclen + req->cryptlen,
907 auth_tag_len, 1);
908
909 return 0;
910 }
911
912 static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen,
913 u8 *hash_subkey, u8 *iv, void *aes_ctx)
914 {
915 u8 one_entry_in_sg = 0;
916 u8 *src, *dst, *assoc;
917 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
918 unsigned long auth_tag_len = crypto_aead_authsize(tfm);
919 struct scatter_walk src_sg_walk;
920 struct scatter_walk dst_sg_walk = {};
921 struct gcm_context_data data AESNI_ALIGN_ATTR;
922
923 if (((struct crypto_aes_ctx *)aes_ctx)->key_length != AES_KEYSIZE_128 ||
924 aesni_gcm_enc_tfm == aesni_gcm_enc ||
925 req->cryptlen < AVX_GEN2_OPTSIZE) {
926 return gcmaes_crypt_by_sg(true, req, assoclen, hash_subkey, iv,
927 aes_ctx);
928 }
929 if (sg_is_last(req->src) &&
930 (!PageHighMem(sg_page(req->src)) ||
931 req->src->offset + req->src->length <= PAGE_SIZE) &&
932 sg_is_last(req->dst) &&
933 (!PageHighMem(sg_page(req->dst)) ||
934 req->dst->offset + req->dst->length <= PAGE_SIZE)) {
935 one_entry_in_sg = 1;
936 scatterwalk_start(&src_sg_walk, req->src);
937 assoc = scatterwalk_map(&src_sg_walk);
938 src = assoc + req->assoclen;
939 dst = src;
940 if (unlikely(req->src != req->dst)) {
941 scatterwalk_start(&dst_sg_walk, req->dst);
942 dst = scatterwalk_map(&dst_sg_walk) + req->assoclen;
943 }
944 } else {
945 /* Allocate memory for src, dst, assoc */
946 assoc = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
947 GFP_ATOMIC);
948 if (unlikely(!assoc))
949 return -ENOMEM;
950 scatterwalk_map_and_copy(assoc, req->src, 0,
951 req->assoclen + req->cryptlen, 0);
952 src = assoc + req->assoclen;
953 dst = src;
954 }
955
956 kernel_fpu_begin();
957 aesni_gcm_enc_tfm(aes_ctx, &data, dst, src, req->cryptlen, iv,
958 hash_subkey, assoc, assoclen,
959 dst + req->cryptlen, auth_tag_len);
960 kernel_fpu_end();
961
962 /* The authTag (aka the Integrity Check Value) needs to be written
963 * back to the packet. */
964 if (one_entry_in_sg) {
965 if (unlikely(req->src != req->dst)) {
966 scatterwalk_unmap(dst - req->assoclen);
967 scatterwalk_advance(&dst_sg_walk, req->dst->length);
968 scatterwalk_done(&dst_sg_walk, 1, 0);
969 }
970 scatterwalk_unmap(assoc);
971 scatterwalk_advance(&src_sg_walk, req->src->length);
972 scatterwalk_done(&src_sg_walk, req->src == req->dst, 0);
973 } else {
974 scatterwalk_map_and_copy(dst, req->dst, req->assoclen,
975 req->cryptlen + auth_tag_len, 1);
976 kfree(assoc);
977 }
978 return 0;
979 }
980
981 static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen,
982 u8 *hash_subkey, u8 *iv, void *aes_ctx)
983 {
984 u8 one_entry_in_sg = 0;
985 u8 *src, *dst, *assoc;
986 unsigned long tempCipherLen = 0;
987 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
988 unsigned long auth_tag_len = crypto_aead_authsize(tfm);
989 u8 authTag[16];
990 struct scatter_walk src_sg_walk;
991 struct scatter_walk dst_sg_walk = {};
992 struct gcm_context_data data AESNI_ALIGN_ATTR;
993 int retval = 0;
994
995 if (((struct crypto_aes_ctx *)aes_ctx)->key_length != AES_KEYSIZE_128 ||
996 aesni_gcm_enc_tfm == aesni_gcm_enc ||
997 req->cryptlen < AVX_GEN2_OPTSIZE) {
998 return gcmaes_crypt_by_sg(false, req, assoclen, hash_subkey, iv,
999 aes_ctx);
1000 }
1001 tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
1002
1003 if (sg_is_last(req->src) &&
1004 (!PageHighMem(sg_page(req->src)) ||
1005 req->src->offset + req->src->length <= PAGE_SIZE) &&
1006 sg_is_last(req->dst) && req->dst->length &&
1007 (!PageHighMem(sg_page(req->dst)) ||
1008 req->dst->offset + req->dst->length <= PAGE_SIZE)) {
1009 one_entry_in_sg = 1;
1010 scatterwalk_start(&src_sg_walk, req->src);
1011 assoc = scatterwalk_map(&src_sg_walk);
1012 src = assoc + req->assoclen;
1013 dst = src;
1014 if (unlikely(req->src != req->dst)) {
1015 scatterwalk_start(&dst_sg_walk, req->dst);
1016 dst = scatterwalk_map(&dst_sg_walk) + req->assoclen;
1017 }
1018 } else {
1019 /* Allocate memory for src, dst, assoc */
1020 assoc = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
1021 if (!assoc)
1022 return -ENOMEM;
1023 scatterwalk_map_and_copy(assoc, req->src, 0,
1024 req->assoclen + req->cryptlen, 0);
1025 src = assoc + req->assoclen;
1026 dst = src;
1027 }
1028
1029
1030 kernel_fpu_begin();
1031 aesni_gcm_dec_tfm(aes_ctx, &data, dst, src, tempCipherLen, iv,
1032 hash_subkey, assoc, assoclen,
1033 authTag, auth_tag_len);
1034 kernel_fpu_end();
1035
1036 /* Compare generated tag with passed in tag. */
1037 retval = crypto_memneq(src + tempCipherLen, authTag, auth_tag_len) ?
1038 -EBADMSG : 0;
1039
1040 if (one_entry_in_sg) {
1041 if (unlikely(req->src != req->dst)) {
1042 scatterwalk_unmap(dst - req->assoclen);
1043 scatterwalk_advance(&dst_sg_walk, req->dst->length);
1044 scatterwalk_done(&dst_sg_walk, 1, 0);
1045 }
1046 scatterwalk_unmap(assoc);
1047 scatterwalk_advance(&src_sg_walk, req->src->length);
1048 scatterwalk_done(&src_sg_walk, req->src == req->dst, 0);
1049 } else {
1050 scatterwalk_map_and_copy(dst, req->dst, req->assoclen,
1051 tempCipherLen, 1);
1052 kfree(assoc);
1053 }
1054 return retval;
1055
1056 }
1057
1058 static int helper_rfc4106_encrypt(struct aead_request *req)
1059 {
1060 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1061 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1062 void *aes_ctx = &(ctx->aes_key_expanded);
1063 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
1064 unsigned int i;
1065 __be32 counter = cpu_to_be32(1);
1066
1067 /* Assuming we are supporting rfc4106 64-bit extended */
1068 /* sequence numbers We need to have the AAD length equal */
1069 /* to 16 or 20 bytes */
1070 if (unlikely(req->assoclen != 16 && req->assoclen != 20))
1071 return -EINVAL;
1072
1073 /* IV below built */
1074 for (i = 0; i < 4; i++)
1075 *(iv+i) = ctx->nonce[i];
1076 for (i = 0; i < 8; i++)
1077 *(iv+4+i) = req->iv[i];
1078 *((__be32 *)(iv+12)) = counter;
1079
1080 return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
1081 aes_ctx);
1082 }
1083
1084 static int helper_rfc4106_decrypt(struct aead_request *req)
1085 {
1086 __be32 counter = cpu_to_be32(1);
1087 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1088 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1089 void *aes_ctx = &(ctx->aes_key_expanded);
1090 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
1091 unsigned int i;
1092
1093 if (unlikely(req->assoclen != 16 && req->assoclen != 20))
1094 return -EINVAL;
1095
1096 /* Assuming we are supporting rfc4106 64-bit extended */
1097 /* sequence numbers We need to have the AAD length */
1098 /* equal to 16 or 20 bytes */
1099
1100 /* IV below built */
1101 for (i = 0; i < 4; i++)
1102 *(iv+i) = ctx->nonce[i];
1103 for (i = 0; i < 8; i++)
1104 *(iv+4+i) = req->iv[i];
1105 *((__be32 *)(iv+12)) = counter;
1106
1107 return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
1108 aes_ctx);
1109 }
1110
1111 static int gcmaes_wrapper_encrypt(struct aead_request *req)
1112 {
1113 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1114 struct cryptd_aead **ctx = crypto_aead_ctx(tfm);
1115 struct cryptd_aead *cryptd_tfm = *ctx;
1116
1117 tfm = &cryptd_tfm->base;
1118 if (irq_fpu_usable() && (!in_atomic() ||
1119 !cryptd_aead_queued(cryptd_tfm)))
1120 tfm = cryptd_aead_child(cryptd_tfm);
1121
1122 aead_request_set_tfm(req, tfm);
1123
1124 return crypto_aead_encrypt(req);
1125 }
1126
1127 static int gcmaes_wrapper_decrypt(struct aead_request *req)
1128 {
1129 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1130 struct cryptd_aead **ctx = crypto_aead_ctx(tfm);
1131 struct cryptd_aead *cryptd_tfm = *ctx;
1132
1133 tfm = &cryptd_tfm->base;
1134 if (irq_fpu_usable() && (!in_atomic() ||
1135 !cryptd_aead_queued(cryptd_tfm)))
1136 tfm = cryptd_aead_child(cryptd_tfm);
1137
1138 aead_request_set_tfm(req, tfm);
1139
1140 return crypto_aead_decrypt(req);
1141 }
1142 #endif
1143
1144 static struct crypto_alg aesni_algs[] = { {
1145 .cra_name = "aes",
1146 .cra_driver_name = "aes-aesni",
1147 .cra_priority = 300,
1148 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
1149 .cra_blocksize = AES_BLOCK_SIZE,
1150 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
1151 .cra_module = THIS_MODULE,
1152 .cra_u = {
1153 .cipher = {
1154 .cia_min_keysize = AES_MIN_KEY_SIZE,
1155 .cia_max_keysize = AES_MAX_KEY_SIZE,
1156 .cia_setkey = aes_set_key,
1157 .cia_encrypt = aes_encrypt,
1158 .cia_decrypt = aes_decrypt
1159 }
1160 }
1161 }, {
1162 .cra_name = "__aes",
1163 .cra_driver_name = "__aes-aesni",
1164 .cra_priority = 300,
1165 .cra_flags = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_INTERNAL,
1166 .cra_blocksize = AES_BLOCK_SIZE,
1167 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
1168 .cra_module = THIS_MODULE,
1169 .cra_u = {
1170 .cipher = {
1171 .cia_min_keysize = AES_MIN_KEY_SIZE,
1172 .cia_max_keysize = AES_MAX_KEY_SIZE,
1173 .cia_setkey = aes_set_key,
1174 .cia_encrypt = __aes_encrypt,
1175 .cia_decrypt = __aes_decrypt
1176 }
1177 }
1178 } };
1179
1180 static struct skcipher_alg aesni_skciphers[] = {
1181 {
1182 .base = {
1183 .cra_name = "__ecb(aes)",
1184 .cra_driver_name = "__ecb-aes-aesni",
1185 .cra_priority = 400,
1186 .cra_flags = CRYPTO_ALG_INTERNAL,
1187 .cra_blocksize = AES_BLOCK_SIZE,
1188 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
1189 .cra_module = THIS_MODULE,
1190 },
1191 .min_keysize = AES_MIN_KEY_SIZE,
1192 .max_keysize = AES_MAX_KEY_SIZE,
1193 .setkey = aesni_skcipher_setkey,
1194 .encrypt = ecb_encrypt,
1195 .decrypt = ecb_decrypt,
1196 }, {
1197 .base = {
1198 .cra_name = "__cbc(aes)",
1199 .cra_driver_name = "__cbc-aes-aesni",
1200 .cra_priority = 400,
1201 .cra_flags = CRYPTO_ALG_INTERNAL,
1202 .cra_blocksize = AES_BLOCK_SIZE,
1203 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
1204 .cra_module = THIS_MODULE,
1205 },
1206 .min_keysize = AES_MIN_KEY_SIZE,
1207 .max_keysize = AES_MAX_KEY_SIZE,
1208 .ivsize = AES_BLOCK_SIZE,
1209 .setkey = aesni_skcipher_setkey,
1210 .encrypt = cbc_encrypt,
1211 .decrypt = cbc_decrypt,
1212 #ifdef CONFIG_X86_64
1213 }, {
1214 .base = {
1215 .cra_name = "__ctr(aes)",
1216 .cra_driver_name = "__ctr-aes-aesni",
1217 .cra_priority = 400,
1218 .cra_flags = CRYPTO_ALG_INTERNAL,
1219 .cra_blocksize = 1,
1220 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
1221 .cra_module = THIS_MODULE,
1222 },
1223 .min_keysize = AES_MIN_KEY_SIZE,
1224 .max_keysize = AES_MAX_KEY_SIZE,
1225 .ivsize = AES_BLOCK_SIZE,
1226 .chunksize = AES_BLOCK_SIZE,
1227 .setkey = aesni_skcipher_setkey,
1228 .encrypt = ctr_crypt,
1229 .decrypt = ctr_crypt,
1230 }, {
1231 .base = {
1232 .cra_name = "__xts(aes)",
1233 .cra_driver_name = "__xts-aes-aesni",
1234 .cra_priority = 401,
1235 .cra_flags = CRYPTO_ALG_INTERNAL,
1236 .cra_blocksize = AES_BLOCK_SIZE,
1237 .cra_ctxsize = XTS_AES_CTX_SIZE,
1238 .cra_module = THIS_MODULE,
1239 },
1240 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1241 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1242 .ivsize = AES_BLOCK_SIZE,
1243 .setkey = xts_aesni_setkey,
1244 .encrypt = xts_encrypt,
1245 .decrypt = xts_decrypt,
1246 #endif
1247 }
1248 };
1249
1250 static
1251 struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)];
1252
1253 #ifdef CONFIG_X86_64
1254 static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key,
1255 unsigned int key_len)
1256 {
1257 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead);
1258
1259 return aes_set_key_common(crypto_aead_tfm(aead),
1260 &ctx->aes_key_expanded, key, key_len) ?:
1261 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
1262 }
1263
1264 static int generic_gcmaes_encrypt(struct aead_request *req)
1265 {
1266 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1267 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1268 void *aes_ctx = &(ctx->aes_key_expanded);
1269 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
1270 __be32 counter = cpu_to_be32(1);
1271
1272 memcpy(iv, req->iv, 12);
1273 *((__be32 *)(iv+12)) = counter;
1274
1275 return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv,
1276 aes_ctx);
1277 }
1278
1279 static int generic_gcmaes_decrypt(struct aead_request *req)
1280 {
1281 __be32 counter = cpu_to_be32(1);
1282 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1283 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1284 void *aes_ctx = &(ctx->aes_key_expanded);
1285 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
1286
1287 memcpy(iv, req->iv, 12);
1288 *((__be32 *)(iv+12)) = counter;
1289
1290 return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv,
1291 aes_ctx);
1292 }
1293
1294 static int generic_gcmaes_init(struct crypto_aead *aead)
1295 {
1296 struct cryptd_aead *cryptd_tfm;
1297 struct cryptd_aead **ctx = crypto_aead_ctx(aead);
1298
1299 cryptd_tfm = cryptd_alloc_aead("__driver-generic-gcm-aes-aesni",
1300 CRYPTO_ALG_INTERNAL,
1301 CRYPTO_ALG_INTERNAL);
1302 if (IS_ERR(cryptd_tfm))
1303 return PTR_ERR(cryptd_tfm);
1304
1305 *ctx = cryptd_tfm;
1306 crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
1307
1308 return 0;
1309 }
1310
1311 static void generic_gcmaes_exit(struct crypto_aead *aead)
1312 {
1313 struct cryptd_aead **ctx = crypto_aead_ctx(aead);
1314
1315 cryptd_free_aead(*ctx);
1316 }
1317
1318 static struct aead_alg aesni_aead_algs[] = { {
1319 .setkey = common_rfc4106_set_key,
1320 .setauthsize = common_rfc4106_set_authsize,
1321 .encrypt = helper_rfc4106_encrypt,
1322 .decrypt = helper_rfc4106_decrypt,
1323 .ivsize = GCM_RFC4106_IV_SIZE,
1324 .maxauthsize = 16,
1325 .base = {
1326 .cra_name = "__gcm-aes-aesni",
1327 .cra_driver_name = "__driver-gcm-aes-aesni",
1328 .cra_flags = CRYPTO_ALG_INTERNAL,
1329 .cra_blocksize = 1,
1330 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx),
1331 .cra_alignmask = AESNI_ALIGN - 1,
1332 .cra_module = THIS_MODULE,
1333 },
1334 }, {
1335 .init = rfc4106_init,
1336 .exit = rfc4106_exit,
1337 .setkey = gcmaes_wrapper_set_key,
1338 .setauthsize = gcmaes_wrapper_set_authsize,
1339 .encrypt = gcmaes_wrapper_encrypt,
1340 .decrypt = gcmaes_wrapper_decrypt,
1341 .ivsize = GCM_RFC4106_IV_SIZE,
1342 .maxauthsize = 16,
1343 .base = {
1344 .cra_name = "rfc4106(gcm(aes))",
1345 .cra_driver_name = "rfc4106-gcm-aesni",
1346 .cra_priority = 400,
1347 .cra_flags = CRYPTO_ALG_ASYNC,
1348 .cra_blocksize = 1,
1349 .cra_ctxsize = sizeof(struct cryptd_aead *),
1350 .cra_module = THIS_MODULE,
1351 },
1352 }, {
1353 .setkey = generic_gcmaes_set_key,
1354 .setauthsize = generic_gcmaes_set_authsize,
1355 .encrypt = generic_gcmaes_encrypt,
1356 .decrypt = generic_gcmaes_decrypt,
1357 .ivsize = GCM_AES_IV_SIZE,
1358 .maxauthsize = 16,
1359 .base = {
1360 .cra_name = "__generic-gcm-aes-aesni",
1361 .cra_driver_name = "__driver-generic-gcm-aes-aesni",
1362 .cra_priority = 0,
1363 .cra_flags = CRYPTO_ALG_INTERNAL,
1364 .cra_blocksize = 1,
1365 .cra_ctxsize = sizeof(struct generic_gcmaes_ctx),
1366 .cra_alignmask = AESNI_ALIGN - 1,
1367 .cra_module = THIS_MODULE,
1368 },
1369 }, {
1370 .init = generic_gcmaes_init,
1371 .exit = generic_gcmaes_exit,
1372 .setkey = gcmaes_wrapper_set_key,
1373 .setauthsize = gcmaes_wrapper_set_authsize,
1374 .encrypt = gcmaes_wrapper_encrypt,
1375 .decrypt = gcmaes_wrapper_decrypt,
1376 .ivsize = GCM_AES_IV_SIZE,
1377 .maxauthsize = 16,
1378 .base = {
1379 .cra_name = "gcm(aes)",
1380 .cra_driver_name = "generic-gcm-aesni",
1381 .cra_priority = 400,
1382 .cra_flags = CRYPTO_ALG_ASYNC,
1383 .cra_blocksize = 1,
1384 .cra_ctxsize = sizeof(struct cryptd_aead *),
1385 .cra_module = THIS_MODULE,
1386 },
1387 } };
1388 #else
1389 static struct aead_alg aesni_aead_algs[0];
1390 #endif
1391
1392
1393 static const struct x86_cpu_id aesni_cpu_id[] = {
1394 X86_FEATURE_MATCH(X86_FEATURE_AES),
1395 {}
1396 };
1397 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1398
1399 static void aesni_free_simds(void)
1400 {
1401 int i;
1402
1403 for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers) &&
1404 aesni_simd_skciphers[i]; i++)
1405 simd_skcipher_free(aesni_simd_skciphers[i]);
1406 }
1407
1408 static int __init aesni_init(void)
1409 {
1410 struct simd_skcipher_alg *simd;
1411 const char *basename;
1412 const char *algname;
1413 const char *drvname;
1414 int err;
1415 int i;
1416
1417 if (!x86_match_cpu(aesni_cpu_id))
1418 return -ENODEV;
1419 #ifdef CONFIG_X86_64
1420 #ifdef CONFIG_AS_AVX2
1421 if (boot_cpu_has(X86_FEATURE_AVX2)) {
1422 pr_info("AVX2 version of gcm_enc/dec engaged.\n");
1423 aesni_gcm_enc_tfm = aesni_gcm_enc_avx2;
1424 aesni_gcm_dec_tfm = aesni_gcm_dec_avx2;
1425 } else
1426 #endif
1427 #ifdef CONFIG_AS_AVX
1428 if (boot_cpu_has(X86_FEATURE_AVX)) {
1429 pr_info("AVX version of gcm_enc/dec engaged.\n");
1430 aesni_gcm_enc_tfm = aesni_gcm_enc_avx;
1431 aesni_gcm_dec_tfm = aesni_gcm_dec_avx;
1432 } else
1433 #endif
1434 {
1435 pr_info("SSE version of gcm_enc/dec engaged.\n");
1436 aesni_gcm_enc_tfm = aesni_gcm_enc;
1437 aesni_gcm_dec_tfm = aesni_gcm_dec;
1438 }
1439 aesni_ctr_enc_tfm = aesni_ctr_enc;
1440 #ifdef CONFIG_AS_AVX
1441 if (boot_cpu_has(X86_FEATURE_AVX)) {
1442 /* optimize performance of ctr mode encryption transform */
1443 aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm;
1444 pr_info("AES CTR mode by8 optimization enabled\n");
1445 }
1446 #endif
1447 #endif
1448
1449 err = crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1450 if (err)
1451 return err;
1452
1453 err = crypto_register_skciphers(aesni_skciphers,
1454 ARRAY_SIZE(aesni_skciphers));
1455 if (err)
1456 goto unregister_algs;
1457
1458 err = crypto_register_aeads(aesni_aead_algs,
1459 ARRAY_SIZE(aesni_aead_algs));
1460 if (err)
1461 goto unregister_skciphers;
1462
1463 for (i = 0; i < ARRAY_SIZE(aesni_skciphers); i++) {
1464 algname = aesni_skciphers[i].base.cra_name + 2;
1465 drvname = aesni_skciphers[i].base.cra_driver_name + 2;
1466 basename = aesni_skciphers[i].base.cra_driver_name;
1467 simd = simd_skcipher_create_compat(algname, drvname, basename);
1468 err = PTR_ERR(simd);
1469 if (IS_ERR(simd))
1470 goto unregister_simds;
1471
1472 aesni_simd_skciphers[i] = simd;
1473 }
1474
1475 return 0;
1476
1477 unregister_simds:
1478 aesni_free_simds();
1479 crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs));
1480 unregister_skciphers:
1481 crypto_unregister_skciphers(aesni_skciphers,
1482 ARRAY_SIZE(aesni_skciphers));
1483 unregister_algs:
1484 crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1485 return err;
1486 }
1487
1488 static void __exit aesni_exit(void)
1489 {
1490 aesni_free_simds();
1491 crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs));
1492 crypto_unregister_skciphers(aesni_skciphers,
1493 ARRAY_SIZE(aesni_skciphers));
1494 crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1495 }
1496
1497 late_initcall(aesni_init);
1498 module_exit(aesni_exit);
1499
1500 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1501 MODULE_LICENSE("GPL");
1502 MODULE_ALIAS_CRYPTO("aes");
Linux with added FPC-III support