?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 4.1.18
[linux/fpc-iii.git] / arch / x86 / crypto / camellia_aesni_avx2_glue.c
blobbaf0ac21ace5664835ed9435207ddabc3dc0cc7a
1 /*
2 * Glue Code for x86_64/AVX2/AES-NI assembler optimized version of Camellia
3 *
4 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 */
12
13 #include <linux/module.h>
14 #include <linux/types.h>
15 #include <linux/crypto.h>
16 #include <linux/err.h>
17 #include <crypto/ablk_helper.h>
18 #include <crypto/algapi.h>
19 #include <crypto/ctr.h>
20 #include <crypto/lrw.h>
21 #include <crypto/xts.h>
22 #include <asm/xcr.h>
23 #include <asm/xsave.h>
24 #include <asm/crypto/camellia.h>
25 #include <asm/crypto/glue_helper.h>
26
27 #define CAMELLIA_AESNI_PARALLEL_BLOCKS 16
28 #define CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS 32
29
30 /* 32-way AVX2/AES-NI parallel cipher functions */
31 asmlinkage void camellia_ecb_enc_32way(struct camellia_ctx *ctx, u8 *dst,
32 const u8 *src);
33 asmlinkage void camellia_ecb_dec_32way(struct camellia_ctx *ctx, u8 *dst,
34 const u8 *src);
35
36 asmlinkage void camellia_cbc_dec_32way(struct camellia_ctx *ctx, u8 *dst,
37 const u8 *src);
38 asmlinkage void camellia_ctr_32way(struct camellia_ctx *ctx, u8 *dst,
39 const u8 *src, le128 *iv);
40
41 asmlinkage void camellia_xts_enc_32way(struct camellia_ctx *ctx, u8 *dst,
42 const u8 *src, le128 *iv);
43 asmlinkage void camellia_xts_dec_32way(struct camellia_ctx *ctx, u8 *dst,
44 const u8 *src, le128 *iv);
45
46 static const struct common_glue_ctx camellia_enc = {
47 .num_funcs = 4,
48 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
49
50 .funcs = { {
51 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
52 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_enc_32way) }
53 }, {
54 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
55 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_enc_16way) }
56 }, {
57 .num_blocks = 2,
58 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk_2way) }
59 }, {
60 .num_blocks = 1,
61 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk) }
62 } }
63 };
64
65 static const struct common_glue_ctx camellia_ctr = {
66 .num_funcs = 4,
67 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
68
69 .funcs = { {
70 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
71 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_ctr_32way) }
72 }, {
73 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
74 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_ctr_16way) }
75 }, {
76 .num_blocks = 2,
77 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr_2way) }
78 }, {
79 .num_blocks = 1,
80 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr) }
81 } }
82 };
83
84 static const struct common_glue_ctx camellia_enc_xts = {
85 .num_funcs = 3,
86 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
87
88 .funcs = { {
89 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
90 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc_32way) }
91 }, {
92 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
93 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc_16way) }
94 }, {
95 .num_blocks = 1,
96 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc) }
97 } }
98 };
99
100 static const struct common_glue_ctx camellia_dec = {
101 .num_funcs = 4,
102 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
103
104 .funcs = { {
105 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
106 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_dec_32way) }
107 }, {
108 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
109 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_dec_16way) }
110 }, {
111 .num_blocks = 2,
112 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk_2way) }
113 }, {
114 .num_blocks = 1,
115 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk) }
116 } }
117 };
118
119 static const struct common_glue_ctx camellia_dec_cbc = {
120 .num_funcs = 4,
121 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
122
123 .funcs = { {
124 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
125 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_cbc_dec_32way) }
126 }, {
127 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
128 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_cbc_dec_16way) }
129 }, {
130 .num_blocks = 2,
131 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_decrypt_cbc_2way) }
132 }, {
133 .num_blocks = 1,
134 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_dec_blk) }
135 } }
136 };
137
138 static const struct common_glue_ctx camellia_dec_xts = {
139 .num_funcs = 3,
140 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
141
142 .funcs = { {
143 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
144 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec_32way) }
145 }, {
146 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
147 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec_16way) }
148 }, {
149 .num_blocks = 1,
150 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec) }
151 } }
152 };
153
154 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
155 struct scatterlist *src, unsigned int nbytes)
156 {
157 return glue_ecb_crypt_128bit(&camellia_enc, desc, dst, src, nbytes);
158 }
159
160 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
161 struct scatterlist *src, unsigned int nbytes)
162 {
163 return glue_ecb_crypt_128bit(&camellia_dec, desc, dst, src, nbytes);
164 }
165
166 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
167 struct scatterlist *src, unsigned int nbytes)
168 {
169 return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(camellia_enc_blk), desc,
170 dst, src, nbytes);
171 }
172
173 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
174 struct scatterlist *src, unsigned int nbytes)
175 {
176 return glue_cbc_decrypt_128bit(&camellia_dec_cbc, desc, dst, src,
177 nbytes);
178 }
179
180 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
181 struct scatterlist *src, unsigned int nbytes)
182 {
183 return glue_ctr_crypt_128bit(&camellia_ctr, desc, dst, src, nbytes);
184 }
185
186 static inline bool camellia_fpu_begin(bool fpu_enabled, unsigned int nbytes)
187 {
188 return glue_fpu_begin(CAMELLIA_BLOCK_SIZE,
189 CAMELLIA_AESNI_PARALLEL_BLOCKS, NULL, fpu_enabled,
190 nbytes);
191 }
192
193 static inline void camellia_fpu_end(bool fpu_enabled)
194 {
195 glue_fpu_end(fpu_enabled);
196 }
197
198 static int camellia_setkey(struct crypto_tfm *tfm, const u8 *in_key,
199 unsigned int key_len)
200 {
201 return __camellia_setkey(crypto_tfm_ctx(tfm), in_key, key_len,
202 &tfm->crt_flags);
203 }
204
205 struct crypt_priv {
206 struct camellia_ctx *ctx;
207 bool fpu_enabled;
208 };
209
210 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
211 {
212 const unsigned int bsize = CAMELLIA_BLOCK_SIZE;
213 struct crypt_priv *ctx = priv;
214 int i;
215
216 ctx->fpu_enabled = camellia_fpu_begin(ctx->fpu_enabled, nbytes);
217
218 if (nbytes >= CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS * bsize) {
219 camellia_ecb_enc_32way(ctx->ctx, srcdst, srcdst);
220 srcdst += bsize * CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS;
221 nbytes -= bsize * CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS;
222 }
223
224 if (nbytes >= CAMELLIA_AESNI_PARALLEL_BLOCKS * bsize) {
225 camellia_ecb_enc_16way(ctx->ctx, srcdst, srcdst);
226 srcdst += bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS;
227 nbytes -= bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS;
228 }
229
230 while (nbytes >= CAMELLIA_PARALLEL_BLOCKS * bsize) {
231 camellia_enc_blk_2way(ctx->ctx, srcdst, srcdst);
232 srcdst += bsize * CAMELLIA_PARALLEL_BLOCKS;
233 nbytes -= bsize * CAMELLIA_PARALLEL_BLOCKS;
234 }
235
236 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
237 camellia_enc_blk(ctx->ctx, srcdst, srcdst);
238 }
239
240 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
241 {
242 const unsigned int bsize = CAMELLIA_BLOCK_SIZE;
243 struct crypt_priv *ctx = priv;
244 int i;
245
246 ctx->fpu_enabled = camellia_fpu_begin(ctx->fpu_enabled, nbytes);
247
248 if (nbytes >= CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS * bsize) {
249 camellia_ecb_dec_32way(ctx->ctx, srcdst, srcdst);
250 srcdst += bsize * CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS;
251 nbytes -= bsize * CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS;
252 }
253
254 if (nbytes >= CAMELLIA_AESNI_PARALLEL_BLOCKS * bsize) {
255 camellia_ecb_dec_16way(ctx->ctx, srcdst, srcdst);
256 srcdst += bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS;
257 nbytes -= bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS;
258 }
259
260 while (nbytes >= CAMELLIA_PARALLEL_BLOCKS * bsize) {
261 camellia_dec_blk_2way(ctx->ctx, srcdst, srcdst);
262 srcdst += bsize * CAMELLIA_PARALLEL_BLOCKS;
263 nbytes -= bsize * CAMELLIA_PARALLEL_BLOCKS;
264 }
265
266 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
267 camellia_dec_blk(ctx->ctx, srcdst, srcdst);
268 }
269
270 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
271 struct scatterlist *src, unsigned int nbytes)
272 {
273 struct camellia_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
274 be128 buf[CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS];
275 struct crypt_priv crypt_ctx = {
276 .ctx = &ctx->camellia_ctx,
277 .fpu_enabled = false,
278 };
279 struct lrw_crypt_req req = {
280 .tbuf = buf,
281 .tbuflen = sizeof(buf),
282
283 .table_ctx = &ctx->lrw_table,
284 .crypt_ctx = &crypt_ctx,
285 .crypt_fn = encrypt_callback,
286 };
287 int ret;
288
289 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
290 ret = lrw_crypt(desc, dst, src, nbytes, &req);
291 camellia_fpu_end(crypt_ctx.fpu_enabled);
292
293 return ret;
294 }
295
296 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
297 struct scatterlist *src, unsigned int nbytes)
298 {
299 struct camellia_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
300 be128 buf[CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS];
301 struct crypt_priv crypt_ctx = {
302 .ctx = &ctx->camellia_ctx,
303 .fpu_enabled = false,
304 };
305 struct lrw_crypt_req req = {
306 .tbuf = buf,
307 .tbuflen = sizeof(buf),
308
309 .table_ctx = &ctx->lrw_table,
310 .crypt_ctx = &crypt_ctx,
311 .crypt_fn = decrypt_callback,
312 };
313 int ret;
314
315 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
316 ret = lrw_crypt(desc, dst, src, nbytes, &req);
317 camellia_fpu_end(crypt_ctx.fpu_enabled);
318
319 return ret;
320 }
321
322 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
323 struct scatterlist *src, unsigned int nbytes)
324 {
325 struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
326
327 return glue_xts_crypt_128bit(&camellia_enc_xts, desc, dst, src, nbytes,
328 XTS_TWEAK_CAST(camellia_enc_blk),
329 &ctx->tweak_ctx, &ctx->crypt_ctx);
330 }
331
332 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
333 struct scatterlist *src, unsigned int nbytes)
334 {
335 struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
336
337 return glue_xts_crypt_128bit(&camellia_dec_xts, desc, dst, src, nbytes,
338 XTS_TWEAK_CAST(camellia_enc_blk),
339 &ctx->tweak_ctx, &ctx->crypt_ctx);
340 }
341
342 static struct crypto_alg cmll_algs[10] = { {
343 .cra_name = "__ecb-camellia-aesni-avx2",
344 .cra_driver_name = "__driver-ecb-camellia-aesni-avx2",
345 .cra_priority = 0,
346 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
347 CRYPTO_ALG_INTERNAL,
348 .cra_blocksize = CAMELLIA_BLOCK_SIZE,
349 .cra_ctxsize = sizeof(struct camellia_ctx),
350 .cra_alignmask = 0,
351 .cra_type = &crypto_blkcipher_type,
352 .cra_module = THIS_MODULE,
353 .cra_u = {
354 .blkcipher = {
355 .min_keysize = CAMELLIA_MIN_KEY_SIZE,
356 .max_keysize = CAMELLIA_MAX_KEY_SIZE,
357 .setkey = camellia_setkey,
358 .encrypt = ecb_encrypt,
359 .decrypt = ecb_decrypt,
360 },
361 },
362 }, {
363 .cra_name = "__cbc-camellia-aesni-avx2",
364 .cra_driver_name = "__driver-cbc-camellia-aesni-avx2",
365 .cra_priority = 0,
366 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
367 CRYPTO_ALG_INTERNAL,
368 .cra_blocksize = CAMELLIA_BLOCK_SIZE,
369 .cra_ctxsize = sizeof(struct camellia_ctx),
370 .cra_alignmask = 0,
371 .cra_type = &crypto_blkcipher_type,
372 .cra_module = THIS_MODULE,
373 .cra_u = {
374 .blkcipher = {
375 .min_keysize = CAMELLIA_MIN_KEY_SIZE,
376 .max_keysize = CAMELLIA_MAX_KEY_SIZE,
377 .setkey = camellia_setkey,
378 .encrypt = cbc_encrypt,
379 .decrypt = cbc_decrypt,
380 },
381 },
382 }, {
383 .cra_name = "__ctr-camellia-aesni-avx2",
384 .cra_driver_name = "__driver-ctr-camellia-aesni-avx2",
385 .cra_priority = 0,
386 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
387 CRYPTO_ALG_INTERNAL,
388 .cra_blocksize = 1,
389 .cra_ctxsize = sizeof(struct camellia_ctx),
390 .cra_alignmask = 0,
391 .cra_type = &crypto_blkcipher_type,
392 .cra_module = THIS_MODULE,
393 .cra_u = {
394 .blkcipher = {
395 .min_keysize = CAMELLIA_MIN_KEY_SIZE,
396 .max_keysize = CAMELLIA_MAX_KEY_SIZE,
397 .ivsize = CAMELLIA_BLOCK_SIZE,
398 .setkey = camellia_setkey,
399 .encrypt = ctr_crypt,
400 .decrypt = ctr_crypt,
401 },
402 },
403 }, {
404 .cra_name = "__lrw-camellia-aesni-avx2",
405 .cra_driver_name = "__driver-lrw-camellia-aesni-avx2",
406 .cra_priority = 0,
407 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
408 CRYPTO_ALG_INTERNAL,
409 .cra_blocksize = CAMELLIA_BLOCK_SIZE,
410 .cra_ctxsize = sizeof(struct camellia_lrw_ctx),
411 .cra_alignmask = 0,
412 .cra_type = &crypto_blkcipher_type,
413 .cra_module = THIS_MODULE,
414 .cra_exit = lrw_camellia_exit_tfm,
415 .cra_u = {
416 .blkcipher = {
417 .min_keysize = CAMELLIA_MIN_KEY_SIZE +
418 CAMELLIA_BLOCK_SIZE,
419 .max_keysize = CAMELLIA_MAX_KEY_SIZE +
420 CAMELLIA_BLOCK_SIZE,
421 .ivsize = CAMELLIA_BLOCK_SIZE,
422 .setkey = lrw_camellia_setkey,
423 .encrypt = lrw_encrypt,
424 .decrypt = lrw_decrypt,
425 },
426 },
427 }, {
428 .cra_name = "__xts-camellia-aesni-avx2",
429 .cra_driver_name = "__driver-xts-camellia-aesni-avx2",
430 .cra_priority = 0,
431 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
432 CRYPTO_ALG_INTERNAL,
433 .cra_blocksize = CAMELLIA_BLOCK_SIZE,
434 .cra_ctxsize = sizeof(struct camellia_xts_ctx),
435 .cra_alignmask = 0,
436 .cra_type = &crypto_blkcipher_type,
437 .cra_module = THIS_MODULE,
438 .cra_u = {
439 .blkcipher = {
440 .min_keysize = CAMELLIA_MIN_KEY_SIZE * 2,
441 .max_keysize = CAMELLIA_MAX_KEY_SIZE * 2,
442 .ivsize = CAMELLIA_BLOCK_SIZE,
443 .setkey = xts_camellia_setkey,
444 .encrypt = xts_encrypt,
445 .decrypt = xts_decrypt,
446 },
447 },
448 }, {
449 .cra_name = "ecb(camellia)",
450 .cra_driver_name = "ecb-camellia-aesni-avx2",
451 .cra_priority = 500,
452 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
453 .cra_blocksize = CAMELLIA_BLOCK_SIZE,
454 .cra_ctxsize = sizeof(struct async_helper_ctx),
455 .cra_alignmask = 0,
456 .cra_type = &crypto_ablkcipher_type,
457 .cra_module = THIS_MODULE,
458 .cra_init = ablk_init,
459 .cra_exit = ablk_exit,
460 .cra_u = {
461 .ablkcipher = {
462 .min_keysize = CAMELLIA_MIN_KEY_SIZE,
463 .max_keysize = CAMELLIA_MAX_KEY_SIZE,
464 .setkey = ablk_set_key,
465 .encrypt = ablk_encrypt,
466 .decrypt = ablk_decrypt,
467 },
468 },
469 }, {
470 .cra_name = "cbc(camellia)",
471 .cra_driver_name = "cbc-camellia-aesni-avx2",
472 .cra_priority = 500,
473 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
474 .cra_blocksize = CAMELLIA_BLOCK_SIZE,
475 .cra_ctxsize = sizeof(struct async_helper_ctx),
476 .cra_alignmask = 0,
477 .cra_type = &crypto_ablkcipher_type,
478 .cra_module = THIS_MODULE,
479 .cra_init = ablk_init,
480 .cra_exit = ablk_exit,
481 .cra_u = {
482 .ablkcipher = {
483 .min_keysize = CAMELLIA_MIN_KEY_SIZE,
484 .max_keysize = CAMELLIA_MAX_KEY_SIZE,
485 .ivsize = CAMELLIA_BLOCK_SIZE,
486 .setkey = ablk_set_key,
487 .encrypt = __ablk_encrypt,
488 .decrypt = ablk_decrypt,
489 },
490 },
491 }, {
492 .cra_name = "ctr(camellia)",
493 .cra_driver_name = "ctr-camellia-aesni-avx2",
494 .cra_priority = 500,
495 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
496 .cra_blocksize = 1,
497 .cra_ctxsize = sizeof(struct async_helper_ctx),
498 .cra_alignmask = 0,
499 .cra_type = &crypto_ablkcipher_type,
500 .cra_module = THIS_MODULE,
501 .cra_init = ablk_init,
502 .cra_exit = ablk_exit,
503 .cra_u = {
504 .ablkcipher = {
505 .min_keysize = CAMELLIA_MIN_KEY_SIZE,
506 .max_keysize = CAMELLIA_MAX_KEY_SIZE,
507 .ivsize = CAMELLIA_BLOCK_SIZE,
508 .setkey = ablk_set_key,
509 .encrypt = ablk_encrypt,
510 .decrypt = ablk_encrypt,
511 .geniv = "chainiv",
512 },
513 },
514 }, {
515 .cra_name = "lrw(camellia)",
516 .cra_driver_name = "lrw-camellia-aesni-avx2",
517 .cra_priority = 500,
518 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
519 .cra_blocksize = CAMELLIA_BLOCK_SIZE,
520 .cra_ctxsize = sizeof(struct async_helper_ctx),
521 .cra_alignmask = 0,
522 .cra_type = &crypto_ablkcipher_type,
523 .cra_module = THIS_MODULE,
524 .cra_init = ablk_init,
525 .cra_exit = ablk_exit,
526 .cra_u = {
527 .ablkcipher = {
528 .min_keysize = CAMELLIA_MIN_KEY_SIZE +
529 CAMELLIA_BLOCK_SIZE,
530 .max_keysize = CAMELLIA_MAX_KEY_SIZE +
531 CAMELLIA_BLOCK_SIZE,
532 .ivsize = CAMELLIA_BLOCK_SIZE,
533 .setkey = ablk_set_key,
534 .encrypt = ablk_encrypt,
535 .decrypt = ablk_decrypt,
536 },
537 },
538 }, {
539 .cra_name = "xts(camellia)",
540 .cra_driver_name = "xts-camellia-aesni-avx2",
541 .cra_priority = 500,
542 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
543 .cra_blocksize = CAMELLIA_BLOCK_SIZE,
544 .cra_ctxsize = sizeof(struct async_helper_ctx),
545 .cra_alignmask = 0,
546 .cra_type = &crypto_ablkcipher_type,
547 .cra_module = THIS_MODULE,
548 .cra_init = ablk_init,
549 .cra_exit = ablk_exit,
550 .cra_u = {
551 .ablkcipher = {
552 .min_keysize = CAMELLIA_MIN_KEY_SIZE * 2,
553 .max_keysize = CAMELLIA_MAX_KEY_SIZE * 2,
554 .ivsize = CAMELLIA_BLOCK_SIZE,
555 .setkey = ablk_set_key,
556 .encrypt = ablk_encrypt,
557 .decrypt = ablk_decrypt,
558 },
559 },
560 } };
561
562 static int __init camellia_aesni_init(void)
563 {
564 u64 xcr0;
565
566 if (!cpu_has_avx2 || !cpu_has_avx || !cpu_has_aes || !cpu_has_osxsave) {
567 pr_info("AVX2 or AES-NI instructions are not detected.\n");
568 return -ENODEV;
569 }
570
571 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
572 if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
573 pr_info("AVX2 detected but unusable.\n");
574 return -ENODEV;
575 }
576
577 return crypto_register_algs(cmll_algs, ARRAY_SIZE(cmll_algs));
578 }
579
580 static void __exit camellia_aesni_fini(void)
581 {
582 crypto_unregister_algs(cmll_algs, ARRAY_SIZE(cmll_algs));
583 }
584
585 module_init(camellia_aesni_init);
586 module_exit(camellia_aesni_fini);
587
588 MODULE_LICENSE("GPL");
589 MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX2 optimized");
590 MODULE_ALIAS_CRYPTO("camellia");
591 MODULE_ALIAS_CRYPTO("camellia-asm");
Linux with added FPC-III support