arch/x86/crypto/camellia_aesni_avx2_glue.c

   1 /*
   2  * Glue Code for x86_64/AVX2/AES-NI assembler optimized version of Camellia
   3  *
   4  * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  */
  12
  13 #include <linux/module.h>
  14 #include <linux/types.h>
  15 #include <linux/crypto.h>
  16 #include <linux/err.h>
  17 #include <crypto/ablk_helper.h>
  18 #include <crypto/algapi.h>
  19 #include <crypto/ctr.h>
  20 #include <crypto/lrw.h>
  21 #include <crypto/xts.h>
  22 #include <asm/fpu/api.h>
  23 #include <asm/crypto/camellia.h>
  24 #include <asm/crypto/glue_helper.h>
  25
  26 #define CAMELLIA_AESNI_PARALLEL_BLOCKS 16
  27 #define CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS 32
  28
  29 /* 32-way AVX2/AES-NI parallel cipher functions */
  30 asmlinkage void camellia_ecb_enc_32way(struct camellia_ctx *ctx, u8 *dst,
  31                                        const u8 *src);
  32 asmlinkage void camellia_ecb_dec_32way(struct camellia_ctx *ctx, u8 *dst,
  33                                        const u8 *src);
  34
  35 asmlinkage void camellia_cbc_dec_32way(struct camellia_ctx *ctx, u8 *dst,
  36                                        const u8 *src);
  37 asmlinkage void camellia_ctr_32way(struct camellia_ctx *ctx, u8 *dst,
  38                                    const u8 *src, le128 *iv);
  39
  40 asmlinkage void camellia_xts_enc_32way(struct camellia_ctx *ctx, u8 *dst,
  41                                        const u8 *src, le128 *iv);
  42 asmlinkage void camellia_xts_dec_32way(struct camellia_ctx *ctx, u8 *dst,
  43                                        const u8 *src, le128 *iv);
  44
  45 static const struct common_glue_ctx camellia_enc = {
  46         .num_funcs = 4,
  47         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  48
  49         .funcs = { {
  50                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
  51                 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_enc_32way) }
  52         }, {
  53                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  54                 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_enc_16way) }
  55         }, {
  56                 .num_blocks = 2,
  57                 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk_2way) }
  58         }, {
  59                 .num_blocks = 1,
  60                 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk) }
  61         } }
  62 };
  63
  64 static const struct common_glue_ctx camellia_ctr = {
  65         .num_funcs = 4,
  66         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  67
  68         .funcs = { {
  69                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
  70                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_ctr_32way) }
  71         }, {
  72                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  73                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_ctr_16way) }
  74         }, {
  75                 .num_blocks = 2,
  76                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr_2way) }
  77         }, {
  78                 .num_blocks = 1,
  79                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr) }
  80         } }
  81 };
  82
  83 static const struct common_glue_ctx camellia_enc_xts = {
  84         .num_funcs = 3,
  85         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  86
  87         .funcs = { {
  88                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
  89                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc_32way) }
  90         }, {
  91                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  92                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc_16way) }
  93         }, {
  94                 .num_blocks = 1,
  95                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc) }
  96         } }
  97 };
  98
  99 static const struct common_glue_ctx camellia_dec = {
 100         .num_funcs = 4,
 101         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 102
 103         .funcs = { {
 104                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
 105                 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_dec_32way) }
 106         }, {
 107                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 108                 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_dec_16way) }
 109         }, {
 110                 .num_blocks = 2,
 111                 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk_2way) }
 112         }, {
 113                 .num_blocks = 1,
 114                 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk) }
 115         } }
 116 };
 117
 118 static const struct common_glue_ctx camellia_dec_cbc = {
 119         .num_funcs = 4,
 120         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 121
 122         .funcs = { {
 123                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
 124                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_cbc_dec_32way) }
 125         }, {
 126                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 127                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_cbc_dec_16way) }
 128         }, {
 129                 .num_blocks = 2,
 130                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_decrypt_cbc_2way) }
 131         }, {
 132                 .num_blocks = 1,
 133                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_dec_blk) }
 134         } }
 135 };
 136
 137 static const struct common_glue_ctx camellia_dec_xts = {
 138         .num_funcs = 3,
 139         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 140
 141         .funcs = { {
 142                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
 143                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec_32way) }
 144         }, {
 145                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 146                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec_16way) }
 147         }, {
 148                 .num_blocks = 1,
 149                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec) }
 150         } }
 151 };
 152
 153 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 154                        struct scatterlist *src, unsigned int nbytes)
 155 {
 156         return glue_ecb_crypt_128bit(&camellia_enc, desc, dst, src, nbytes);
 157 }
 158
 159 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 160                        struct scatterlist *src, unsigned int nbytes)
 161 {
 162         return glue_ecb_crypt_128bit(&camellia_dec, desc, dst, src, nbytes);
 163 }
 164
 165 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 166                        struct scatterlist *src, unsigned int nbytes)
 167 {
 168         return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(camellia_enc_blk), desc,
 169                                        dst, src, nbytes);
 170 }
 171
 172 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 173                        struct scatterlist *src, unsigned int nbytes)
 174 {
 175         return glue_cbc_decrypt_128bit(&camellia_dec_cbc, desc, dst, src,
 176                                        nbytes);
 177 }
 178
 179 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 180                      struct scatterlist *src, unsigned int nbytes)
 181 {
 182         return glue_ctr_crypt_128bit(&camellia_ctr, desc, dst, src, nbytes);
 183 }
 184
 185 static inline bool camellia_fpu_begin(bool fpu_enabled, unsigned int nbytes)
 186 {
 187         return glue_fpu_begin(CAMELLIA_BLOCK_SIZE,
 188                               CAMELLIA_AESNI_PARALLEL_BLOCKS, NULL, fpu_enabled,
 189                               nbytes);
 190 }
 191
 192 static inline void camellia_fpu_end(bool fpu_enabled)
 193 {
 194         glue_fpu_end(fpu_enabled);
 195 }
 196
 197 static int camellia_setkey(struct crypto_tfm *tfm, const u8 *in_key,
 198                            unsigned int key_len)
 199 {
 200         return __camellia_setkey(crypto_tfm_ctx(tfm), in_key, key_len,
 201                                  &tfm->crt_flags);
 202 }
 203
 204 struct crypt_priv {
 205         struct camellia_ctx *ctx;
 206         bool fpu_enabled;
 207 };
 208
 209 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
 210 {
 211         const unsigned int bsize = CAMELLIA_BLOCK_SIZE;
 212         struct crypt_priv *ctx = priv;
 213         int i;
 214
 215         ctx->fpu_enabled = camellia_fpu_begin(ctx->fpu_enabled, nbytes);
 216
 217         if (nbytes >= CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS * bsize) {
 218                 camellia_ecb_enc_32way(ctx->ctx, srcdst, srcdst);
 219                 srcdst += bsize * CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS;
 220                 nbytes -= bsize * CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS;
 221         }
 222
 223         if (nbytes >= CAMELLIA_AESNI_PARALLEL_BLOCKS * bsize) {
 224                 camellia_ecb_enc_16way(ctx->ctx, srcdst, srcdst);
 225                 srcdst += bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS;
 226                 nbytes -= bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS;
 227         }
 228
 229         while (nbytes >= CAMELLIA_PARALLEL_BLOCKS * bsize) {
 230                 camellia_enc_blk_2way(ctx->ctx, srcdst, srcdst);
 231                 srcdst += bsize * CAMELLIA_PARALLEL_BLOCKS;
 232                 nbytes -= bsize * CAMELLIA_PARALLEL_BLOCKS;
 233         }
 234
 235         for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
 236                 camellia_enc_blk(ctx->ctx, srcdst, srcdst);
 237 }
 238
 239 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
 240 {
 241         const unsigned int bsize = CAMELLIA_BLOCK_SIZE;
 242         struct crypt_priv *ctx = priv;
 243         int i;
 244
 245         ctx->fpu_enabled = camellia_fpu_begin(ctx->fpu_enabled, nbytes);
 246
 247         if (nbytes >= CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS * bsize) {
 248                 camellia_ecb_dec_32way(ctx->ctx, srcdst, srcdst);
 249                 srcdst += bsize * CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS;
 250                 nbytes -= bsize * CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS;
 251         }
 252
 253         if (nbytes >= CAMELLIA_AESNI_PARALLEL_BLOCKS * bsize) {
 254                 camellia_ecb_dec_16way(ctx->ctx, srcdst, srcdst);
 255                 srcdst += bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS;
 256                 nbytes -= bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS;
 257         }
 258
 259         while (nbytes >= CAMELLIA_PARALLEL_BLOCKS * bsize) {
 260                 camellia_dec_blk_2way(ctx->ctx, srcdst, srcdst);
 261                 srcdst += bsize * CAMELLIA_PARALLEL_BLOCKS;
 262                 nbytes -= bsize * CAMELLIA_PARALLEL_BLOCKS;
 263         }
 264
 265         for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
 266                 camellia_dec_blk(ctx->ctx, srcdst, srcdst);
 267 }
 268
 269 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 270                        struct scatterlist *src, unsigned int nbytes)
 271 {
 272         struct camellia_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 273         be128 buf[CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS];
 274         struct crypt_priv crypt_ctx = {
 275                 .ctx = &ctx->camellia_ctx,
 276                 .fpu_enabled = false,
 277         };
 278         struct lrw_crypt_req req = {
 279                 .tbuf = buf,
 280                 .tbuflen = sizeof(buf),
 281
 282                 .table_ctx = &ctx->lrw_table,
 283                 .crypt_ctx = &crypt_ctx,
 284                 .crypt_fn = encrypt_callback,
 285         };
 286         int ret;
 287
 288         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 289         ret = lrw_crypt(desc, dst, src, nbytes, &req);
 290         camellia_fpu_end(crypt_ctx.fpu_enabled);
 291
 292         return ret;
 293 }
 294
 295 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 296                        struct scatterlist *src, unsigned int nbytes)
 297 {
 298         struct camellia_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 299         be128 buf[CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS];
 300         struct crypt_priv crypt_ctx = {
 301                 .ctx = &ctx->camellia_ctx,
 302                 .fpu_enabled = false,
 303         };
 304         struct lrw_crypt_req req = {
 305                 .tbuf = buf,
 306                 .tbuflen = sizeof(buf),
 307
 308                 .table_ctx = &ctx->lrw_table,
 309                 .crypt_ctx = &crypt_ctx,
 310                 .crypt_fn = decrypt_callback,
 311         };
 312         int ret;
 313
 314         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 315         ret = lrw_crypt(desc, dst, src, nbytes, &req);
 316         camellia_fpu_end(crypt_ctx.fpu_enabled);
 317
 318         return ret;
 319 }
 320
 321 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 322                        struct scatterlist *src, unsigned int nbytes)
 323 {
 324         struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 325
 326         return glue_xts_crypt_128bit(&camellia_enc_xts, desc, dst, src, nbytes,
 327                                      XTS_TWEAK_CAST(camellia_enc_blk),
 328                                      &ctx->tweak_ctx, &ctx->crypt_ctx);
 329 }
 330
 331 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 332                        struct scatterlist *src, unsigned int nbytes)
 333 {
 334         struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 335
 336         return glue_xts_crypt_128bit(&camellia_dec_xts, desc, dst, src, nbytes,
 337                                      XTS_TWEAK_CAST(camellia_enc_blk),
 338                                      &ctx->tweak_ctx, &ctx->crypt_ctx);
 339 }
 340
 341 static struct crypto_alg cmll_algs[10] = { {
 342         .cra_name               = "__ecb-camellia-aesni-avx2",
 343         .cra_driver_name        = "__driver-ecb-camellia-aesni-avx2",
 344         .cra_priority           = 0,
 345         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 346                                   CRYPTO_ALG_INTERNAL,
 347         .cra_blocksize          = CAMELLIA_BLOCK_SIZE,
 348         .cra_ctxsize            = sizeof(struct camellia_ctx),
 349         .cra_alignmask          = 0,
 350         .cra_type               = &crypto_blkcipher_type,
 351         .cra_module             = THIS_MODULE,
 352         .cra_u = {
 353                 .blkcipher = {
 354                         .min_keysize    = CAMELLIA_MIN_KEY_SIZE,
 355                         .max_keysize    = CAMELLIA_MAX_KEY_SIZE,
 356                         .setkey         = camellia_setkey,
 357                         .encrypt        = ecb_encrypt,
 358                         .decrypt        = ecb_decrypt,
 359                 },
 360         },
 361 }, {
 362         .cra_name               = "__cbc-camellia-aesni-avx2",
 363         .cra_driver_name        = "__driver-cbc-camellia-aesni-avx2",
 364         .cra_priority           = 0,
 365         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 366                                   CRYPTO_ALG_INTERNAL,
 367         .cra_blocksize          = CAMELLIA_BLOCK_SIZE,
 368         .cra_ctxsize            = sizeof(struct camellia_ctx),
 369         .cra_alignmask          = 0,
 370         .cra_type               = &crypto_blkcipher_type,
 371         .cra_module             = THIS_MODULE,
 372         .cra_u = {
 373                 .blkcipher = {
 374                         .min_keysize    = CAMELLIA_MIN_KEY_SIZE,
 375                         .max_keysize    = CAMELLIA_MAX_KEY_SIZE,
 376                         .setkey         = camellia_setkey,
 377                         .encrypt        = cbc_encrypt,
 378                         .decrypt        = cbc_decrypt,
 379                 },
 380         },
 381 }, {
 382         .cra_name               = "__ctr-camellia-aesni-avx2",
 383         .cra_driver_name        = "__driver-ctr-camellia-aesni-avx2",
 384         .cra_priority           = 0,
 385         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 386                                   CRYPTO_ALG_INTERNAL,
 387         .cra_blocksize          = 1,
 388         .cra_ctxsize            = sizeof(struct camellia_ctx),
 389         .cra_alignmask          = 0,
 390         .cra_type               = &crypto_blkcipher_type,
 391         .cra_module             = THIS_MODULE,
 392         .cra_u = {
 393                 .blkcipher = {
 394                         .min_keysize    = CAMELLIA_MIN_KEY_SIZE,
 395                         .max_keysize    = CAMELLIA_MAX_KEY_SIZE,
 396                         .ivsize         = CAMELLIA_BLOCK_SIZE,
 397                         .setkey         = camellia_setkey,
 398                         .encrypt        = ctr_crypt,
 399                         .decrypt        = ctr_crypt,
 400                 },
 401         },
 402 }, {
 403         .cra_name               = "__lrw-camellia-aesni-avx2",
 404         .cra_driver_name        = "__driver-lrw-camellia-aesni-avx2",
 405         .cra_priority           = 0,
 406         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 407                                   CRYPTO_ALG_INTERNAL,
 408         .cra_blocksize          = CAMELLIA_BLOCK_SIZE,
 409         .cra_ctxsize            = sizeof(struct camellia_lrw_ctx),
 410         .cra_alignmask          = 0,
 411         .cra_type               = &crypto_blkcipher_type,
 412         .cra_module             = THIS_MODULE,
 413         .cra_exit               = lrw_camellia_exit_tfm,
 414         .cra_u = {
 415                 .blkcipher = {
 416                         .min_keysize    = CAMELLIA_MIN_KEY_SIZE +
 417                                           CAMELLIA_BLOCK_SIZE,
 418                         .max_keysize    = CAMELLIA_MAX_KEY_SIZE +
 419                                           CAMELLIA_BLOCK_SIZE,
 420                         .ivsize         = CAMELLIA_BLOCK_SIZE,
 421                         .setkey         = lrw_camellia_setkey,
 422                         .encrypt        = lrw_encrypt,
 423                         .decrypt        = lrw_decrypt,
 424                 },
 425         },
 426 }, {
 427         .cra_name               = "__xts-camellia-aesni-avx2",
 428         .cra_driver_name        = "__driver-xts-camellia-aesni-avx2",
 429         .cra_priority           = 0,
 430         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 431                                   CRYPTO_ALG_INTERNAL,
 432         .cra_blocksize          = CAMELLIA_BLOCK_SIZE,
 433         .cra_ctxsize            = sizeof(struct camellia_xts_ctx),
 434         .cra_alignmask          = 0,
 435         .cra_type               = &crypto_blkcipher_type,
 436         .cra_module             = THIS_MODULE,
 437         .cra_u = {
 438                 .blkcipher = {
 439                         .min_keysize    = CAMELLIA_MIN_KEY_SIZE * 2,
 440                         .max_keysize    = CAMELLIA_MAX_KEY_SIZE * 2,
 441                         .ivsize         = CAMELLIA_BLOCK_SIZE,
 442                         .setkey         = xts_camellia_setkey,
 443                         .encrypt        = xts_encrypt,
 444                         .decrypt        = xts_decrypt,
 445                 },
 446         },
 447 }, {
 448         .cra_name               = "ecb(camellia)",
 449         .cra_driver_name        = "ecb-camellia-aesni-avx2",
 450         .cra_priority           = 500,
 451         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 452         .cra_blocksize          = CAMELLIA_BLOCK_SIZE,
 453         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 454         .cra_alignmask          = 0,
 455         .cra_type               = &crypto_ablkcipher_type,
 456         .cra_module             = THIS_MODULE,
 457         .cra_init               = ablk_init,
 458         .cra_exit               = ablk_exit,
 459         .cra_u = {
 460                 .ablkcipher = {
 461                         .min_keysize    = CAMELLIA_MIN_KEY_SIZE,
 462                         .max_keysize    = CAMELLIA_MAX_KEY_SIZE,
 463                         .setkey         = ablk_set_key,
 464                         .encrypt        = ablk_encrypt,
 465                         .decrypt        = ablk_decrypt,
 466                 },
 467         },
 468 }, {
 469         .cra_name               = "cbc(camellia)",
 470         .cra_driver_name        = "cbc-camellia-aesni-avx2",
 471         .cra_priority           = 500,
 472         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 473         .cra_blocksize          = CAMELLIA_BLOCK_SIZE,
 474         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 475         .cra_alignmask          = 0,
 476         .cra_type               = &crypto_ablkcipher_type,
 477         .cra_module             = THIS_MODULE,
 478         .cra_init               = ablk_init,
 479         .cra_exit               = ablk_exit,
 480         .cra_u = {
 481                 .ablkcipher = {
 482                         .min_keysize    = CAMELLIA_MIN_KEY_SIZE,
 483                         .max_keysize    = CAMELLIA_MAX_KEY_SIZE,
 484                         .ivsize         = CAMELLIA_BLOCK_SIZE,
 485                         .setkey         = ablk_set_key,
 486                         .encrypt        = __ablk_encrypt,
 487                         .decrypt        = ablk_decrypt,
 488                 },
 489         },
 490 }, {
 491         .cra_name               = "ctr(camellia)",
 492         .cra_driver_name        = "ctr-camellia-aesni-avx2",
 493         .cra_priority           = 500,
 494         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 495         .cra_blocksize          = 1,
 496         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 497         .cra_alignmask          = 0,
 498         .cra_type               = &crypto_ablkcipher_type,
 499         .cra_module             = THIS_MODULE,
 500         .cra_init               = ablk_init,
 501         .cra_exit               = ablk_exit,
 502         .cra_u = {
 503                 .ablkcipher = {
 504                         .min_keysize    = CAMELLIA_MIN_KEY_SIZE,
 505                         .max_keysize    = CAMELLIA_MAX_KEY_SIZE,
 506                         .ivsize         = CAMELLIA_BLOCK_SIZE,
 507                         .setkey         = ablk_set_key,
 508                         .encrypt        = ablk_encrypt,
 509                         .decrypt        = ablk_encrypt,
 510                         .geniv          = "chainiv",
 511                 },
 512         },
 513 }, {
 514         .cra_name               = "lrw(camellia)",
 515         .cra_driver_name        = "lrw-camellia-aesni-avx2",
 516         .cra_priority           = 500,
 517         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 518         .cra_blocksize          = CAMELLIA_BLOCK_SIZE,
 519         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 520         .cra_alignmask          = 0,
 521         .cra_type               = &crypto_ablkcipher_type,
 522         .cra_module             = THIS_MODULE,
 523         .cra_init               = ablk_init,
 524         .cra_exit               = ablk_exit,
 525         .cra_u = {
 526                 .ablkcipher = {
 527                         .min_keysize    = CAMELLIA_MIN_KEY_SIZE +
 528                                           CAMELLIA_BLOCK_SIZE,
 529                         .max_keysize    = CAMELLIA_MAX_KEY_SIZE +
 530                                           CAMELLIA_BLOCK_SIZE,
 531                         .ivsize         = CAMELLIA_BLOCK_SIZE,
 532                         .setkey         = ablk_set_key,
 533                         .encrypt        = ablk_encrypt,
 534                         .decrypt        = ablk_decrypt,
 535                 },
 536         },
 537 }, {
 538         .cra_name               = "xts(camellia)",
 539         .cra_driver_name        = "xts-camellia-aesni-avx2",
 540         .cra_priority           = 500,
 541         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 542         .cra_blocksize          = CAMELLIA_BLOCK_SIZE,
 543         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 544         .cra_alignmask          = 0,
 545         .cra_type               = &crypto_ablkcipher_type,
 546         .cra_module             = THIS_MODULE,
 547         .cra_init               = ablk_init,
 548         .cra_exit               = ablk_exit,
 549         .cra_u = {
 550                 .ablkcipher = {
 551                         .min_keysize    = CAMELLIA_MIN_KEY_SIZE * 2,
 552                         .max_keysize    = CAMELLIA_MAX_KEY_SIZE * 2,
 553                         .ivsize         = CAMELLIA_BLOCK_SIZE,
 554                         .setkey         = ablk_set_key,
 555                         .encrypt        = ablk_encrypt,
 556                         .decrypt        = ablk_decrypt,
 557                 },
 558         },
 559 } };
 560
 561 static int __init camellia_aesni_init(void)
 562 {
 563         const char *feature_name;
 564
 565         if (!boot_cpu_has(X86_FEATURE_AVX) ||
 566             !boot_cpu_has(X86_FEATURE_AVX2) ||
 567             !boot_cpu_has(X86_FEATURE_AES) ||
 568             !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
 569                 pr_info("AVX2 or AES-NI instructions are not detected.\n");
 570                 return -ENODEV;
 571         }
 572
 573         if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
 574                                 &feature_name)) {
 575                 pr_info("CPU feature '%s' is not supported.\n", feature_name);
 576                 return -ENODEV;
 577         }
 578
 579         return crypto_register_algs(cmll_algs, ARRAY_SIZE(cmll_algs));
 580 }
 581
 582 static void __exit camellia_aesni_fini(void)
 583 {
 584         crypto_unregister_algs(cmll_algs, ARRAY_SIZE(cmll_algs));
 585 }
 586
 587 module_init(camellia_aesni_init);
 588 module_exit(camellia_aesni_fini);
 589
 590 MODULE_LICENSE("GPL");
 591 MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX2 optimized");
 592 MODULE_ALIAS_CRYPTO("camellia");
 593 MODULE_ALIAS_CRYPTO("camellia-asm");