arch/x86/crypto/twofish_avx_glue.c

   1 /*
   2  * Glue Code for AVX assembler version of Twofish Cipher
   3  *
   4  * Copyright (C) 2012 Johannes Goetzfried
   5  *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
   6  *
   7  * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program; if not, write to the Free Software
  21  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
  22  * USA
  23  *
  24  */
  25
  26 #include <linux/module.h>
  27 #include <linux/hardirq.h>
  28 #include <linux/types.h>
  29 #include <linux/crypto.h>
  30 #include <linux/err.h>
  31 #include <crypto/ablk_helper.h>
  32 #include <crypto/algapi.h>
  33 #include <crypto/twofish.h>
  34 #include <crypto/cryptd.h>
  35 #include <crypto/b128ops.h>
  36 #include <crypto/ctr.h>
  37 #include <crypto/lrw.h>
  38 #include <crypto/xts.h>
  39 #include <asm/i387.h>
  40 #include <asm/xcr.h>
  41 #include <asm/xsave.h>
  42 #include <asm/crypto/twofish.h>
  43 #include <asm/crypto/glue_helper.h>
  44 #include <crypto/scatterwalk.h>
  45 #include <linux/workqueue.h>
  46 #include <linux/spinlock.h>
  47
  48 #define TWOFISH_PARALLEL_BLOCKS 8
  49
  50 /* 8-way parallel cipher functions */
  51 asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst,
  52                                      const u8 *src);
  53 asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst,
  54                                      const u8 *src);
  55
  56 asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst,
  57                                      const u8 *src);
  58 asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst,
  59                                  const u8 *src, le128 *iv);
  60
  61 asmlinkage void twofish_xts_enc_8way(struct twofish_ctx *ctx, u8 *dst,
  62                                      const u8 *src, le128 *iv);
  63 asmlinkage void twofish_xts_dec_8way(struct twofish_ctx *ctx, u8 *dst,
  64                                      const u8 *src, le128 *iv);
  65
  66 static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
  67                                         const u8 *src)
  68 {
  69         __twofish_enc_blk_3way(ctx, dst, src, false);
  70 }
  71
  72 static void twofish_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
  73 {
  74         glue_xts_crypt_128bit_one(ctx, dst, src, iv,
  75                                   GLUE_FUNC_CAST(twofish_enc_blk));
  76 }
  77
  78 static void twofish_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
  79 {
  80         glue_xts_crypt_128bit_one(ctx, dst, src, iv,
  81                                   GLUE_FUNC_CAST(twofish_dec_blk));
  82 }
  83
  84
  85 static const struct common_glue_ctx twofish_enc = {
  86         .num_funcs = 3,
  87         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
  88
  89         .funcs = { {
  90                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
  91                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) }
  92         }, {
  93                 .num_blocks = 3,
  94                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
  95         }, {
  96                 .num_blocks = 1,
  97                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
  98         } }
  99 };
 100
 101 static const struct common_glue_ctx twofish_ctr = {
 102         .num_funcs = 3,
 103         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
 104
 105         .funcs = { {
 106                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
 107                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) }
 108         }, {
 109                 .num_blocks = 3,
 110                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) }
 111         }, {
 112                 .num_blocks = 1,
 113                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) }
 114         } }
 115 };
 116
 117 static const struct common_glue_ctx twofish_enc_xts = {
 118         .num_funcs = 2,
 119         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
 120
 121         .funcs = { {
 122                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
 123                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc_8way) }
 124         }, {
 125                 .num_blocks = 1,
 126                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc) }
 127         } }
 128 };
 129
 130 static const struct common_glue_ctx twofish_dec = {
 131         .num_funcs = 3,
 132         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
 133
 134         .funcs = { {
 135                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
 136                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) }
 137         }, {
 138                 .num_blocks = 3,
 139                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
 140         }, {
 141                 .num_blocks = 1,
 142                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
 143         } }
 144 };
 145
 146 static const struct common_glue_ctx twofish_dec_cbc = {
 147         .num_funcs = 3,
 148         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
 149
 150         .funcs = { {
 151                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
 152                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) }
 153         }, {
 154                 .num_blocks = 3,
 155                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
 156         }, {
 157                 .num_blocks = 1,
 158                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
 159         } }
 160 };
 161
 162 static const struct common_glue_ctx twofish_dec_xts = {
 163         .num_funcs = 2,
 164         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
 165
 166         .funcs = { {
 167                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
 168                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec_8way) }
 169         }, {
 170                 .num_blocks = 1,
 171                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec) }
 172         } }
 173 };
 174
 175 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 176                        struct scatterlist *src, unsigned int nbytes)
 177 {
 178         return glue_ecb_crypt_128bit(&twofish_enc, desc, dst, src, nbytes);
 179 }
 180
 181 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 182                        struct scatterlist *src, unsigned int nbytes)
 183 {
 184         return glue_ecb_crypt_128bit(&twofish_dec, desc, dst, src, nbytes);
 185 }
 186
 187 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 188                        struct scatterlist *src, unsigned int nbytes)
 189 {
 190         return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk), desc,
 191                                        dst, src, nbytes);
 192 }
 193
 194 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 195                        struct scatterlist *src, unsigned int nbytes)
 196 {
 197         return glue_cbc_decrypt_128bit(&twofish_dec_cbc, desc, dst, src,
 198                                        nbytes);
 199 }
 200
 201 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 202                      struct scatterlist *src, unsigned int nbytes)
 203 {
 204         return glue_ctr_crypt_128bit(&twofish_ctr, desc, dst, src, nbytes);
 205 }
 206
 207 static inline bool twofish_fpu_begin(bool fpu_enabled, unsigned int nbytes)
 208 {
 209         return glue_fpu_begin(TF_BLOCK_SIZE, TWOFISH_PARALLEL_BLOCKS, NULL,
 210                               fpu_enabled, nbytes);
 211 }
 212
 213 static inline void twofish_fpu_end(bool fpu_enabled)
 214 {
 215         glue_fpu_end(fpu_enabled);
 216 }
 217
 218 struct crypt_priv {
 219         struct twofish_ctx *ctx;
 220         bool fpu_enabled;
 221 };
 222
 223 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
 224 {
 225         const unsigned int bsize = TF_BLOCK_SIZE;
 226         struct crypt_priv *ctx = priv;
 227         int i;
 228
 229         ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
 230
 231         if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
 232                 twofish_ecb_enc_8way(ctx->ctx, srcdst, srcdst);
 233                 return;
 234         }
 235
 236         for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
 237                 twofish_enc_blk_3way(ctx->ctx, srcdst, srcdst);
 238
 239         nbytes %= bsize * 3;
 240
 241         for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
 242                 twofish_enc_blk(ctx->ctx, srcdst, srcdst);
 243 }
 244
 245 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
 246 {
 247         const unsigned int bsize = TF_BLOCK_SIZE;
 248         struct crypt_priv *ctx = priv;
 249         int i;
 250
 251         ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
 252
 253         if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
 254                 twofish_ecb_dec_8way(ctx->ctx, srcdst, srcdst);
 255                 return;
 256         }
 257
 258         for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
 259                 twofish_dec_blk_3way(ctx->ctx, srcdst, srcdst);
 260
 261         nbytes %= bsize * 3;
 262
 263         for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
 264                 twofish_dec_blk(ctx->ctx, srcdst, srcdst);
 265 }
 266
 267 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 268                        struct scatterlist *src, unsigned int nbytes)
 269 {
 270         struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 271         be128 buf[TWOFISH_PARALLEL_BLOCKS];
 272         struct crypt_priv crypt_ctx = {
 273                 .ctx = &ctx->twofish_ctx,
 274                 .fpu_enabled = false,
 275         };
 276         struct lrw_crypt_req req = {
 277                 .tbuf = buf,
 278                 .tbuflen = sizeof(buf),
 279
 280                 .table_ctx = &ctx->lrw_table,
 281                 .crypt_ctx = &crypt_ctx,
 282                 .crypt_fn = encrypt_callback,
 283         };
 284         int ret;
 285
 286         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 287         ret = lrw_crypt(desc, dst, src, nbytes, &req);
 288         twofish_fpu_end(crypt_ctx.fpu_enabled);
 289
 290         return ret;
 291 }
 292
 293 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 294                        struct scatterlist *src, unsigned int nbytes)
 295 {
 296         struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 297         be128 buf[TWOFISH_PARALLEL_BLOCKS];
 298         struct crypt_priv crypt_ctx = {
 299                 .ctx = &ctx->twofish_ctx,
 300                 .fpu_enabled = false,
 301         };
 302         struct lrw_crypt_req req = {
 303                 .tbuf = buf,
 304                 .tbuflen = sizeof(buf),
 305
 306                 .table_ctx = &ctx->lrw_table,
 307                 .crypt_ctx = &crypt_ctx,
 308                 .crypt_fn = decrypt_callback,
 309         };
 310         int ret;
 311
 312         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 313         ret = lrw_crypt(desc, dst, src, nbytes, &req);
 314         twofish_fpu_end(crypt_ctx.fpu_enabled);
 315
 316         return ret;
 317 }
 318
 319 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 320                        struct scatterlist *src, unsigned int nbytes)
 321 {
 322         struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 323
 324         return glue_xts_crypt_128bit(&twofish_enc_xts, desc, dst, src, nbytes,
 325                                      XTS_TWEAK_CAST(twofish_enc_blk),
 326                                      &ctx->tweak_ctx, &ctx->crypt_ctx);
 327 }
 328
 329 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 330                        struct scatterlist *src, unsigned int nbytes)
 331 {
 332         struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 333
 334         return glue_xts_crypt_128bit(&twofish_dec_xts, desc, dst, src, nbytes,
 335                                      XTS_TWEAK_CAST(twofish_enc_blk),
 336                                      &ctx->tweak_ctx, &ctx->crypt_ctx);
 337 }
 338
 339 static struct crypto_alg twofish_algs[10] = { {
 340         .cra_name               = "__ecb-twofish-avx",
 341         .cra_driver_name        = "__driver-ecb-twofish-avx",
 342         .cra_priority           = 0,
 343         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 344                                   CRYPTO_ALG_INTERNAL,
 345         .cra_blocksize          = TF_BLOCK_SIZE,
 346         .cra_ctxsize            = sizeof(struct twofish_ctx),
 347         .cra_alignmask          = 0,
 348         .cra_type               = &crypto_blkcipher_type,
 349         .cra_module             = THIS_MODULE,
 350         .cra_u = {
 351                 .blkcipher = {
 352                         .min_keysize    = TF_MIN_KEY_SIZE,
 353                         .max_keysize    = TF_MAX_KEY_SIZE,
 354                         .setkey         = twofish_setkey,
 355                         .encrypt        = ecb_encrypt,
 356                         .decrypt        = ecb_decrypt,
 357                 },
 358         },
 359 }, {
 360         .cra_name               = "__cbc-twofish-avx",
 361         .cra_driver_name        = "__driver-cbc-twofish-avx",
 362         .cra_priority           = 0,
 363         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 364                                   CRYPTO_ALG_INTERNAL,
 365         .cra_blocksize          = TF_BLOCK_SIZE,
 366         .cra_ctxsize            = sizeof(struct twofish_ctx),
 367         .cra_alignmask          = 0,
 368         .cra_type               = &crypto_blkcipher_type,
 369         .cra_module             = THIS_MODULE,
 370         .cra_u = {
 371                 .blkcipher = {
 372                         .min_keysize    = TF_MIN_KEY_SIZE,
 373                         .max_keysize    = TF_MAX_KEY_SIZE,
 374                         .setkey         = twofish_setkey,
 375                         .encrypt        = cbc_encrypt,
 376                         .decrypt        = cbc_decrypt,
 377                 },
 378         },
 379 }, {
 380         .cra_name               = "__ctr-twofish-avx",
 381         .cra_driver_name        = "__driver-ctr-twofish-avx",
 382         .cra_priority           = 0,
 383         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 384                                   CRYPTO_ALG_INTERNAL,
 385         .cra_blocksize          = 1,
 386         .cra_ctxsize            = sizeof(struct twofish_ctx),
 387         .cra_alignmask          = 0,
 388         .cra_type               = &crypto_blkcipher_type,
 389         .cra_module             = THIS_MODULE,
 390         .cra_u = {
 391                 .blkcipher = {
 392                         .min_keysize    = TF_MIN_KEY_SIZE,
 393                         .max_keysize    = TF_MAX_KEY_SIZE,
 394                         .ivsize         = TF_BLOCK_SIZE,
 395                         .setkey         = twofish_setkey,
 396                         .encrypt        = ctr_crypt,
 397                         .decrypt        = ctr_crypt,
 398                 },
 399         },
 400 }, {
 401         .cra_name               = "__lrw-twofish-avx",
 402         .cra_driver_name        = "__driver-lrw-twofish-avx",
 403         .cra_priority           = 0,
 404         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 405                                   CRYPTO_ALG_INTERNAL,
 406         .cra_blocksize          = TF_BLOCK_SIZE,
 407         .cra_ctxsize            = sizeof(struct twofish_lrw_ctx),
 408         .cra_alignmask          = 0,
 409         .cra_type               = &crypto_blkcipher_type,
 410         .cra_module             = THIS_MODULE,
 411         .cra_exit               = lrw_twofish_exit_tfm,
 412         .cra_u = {
 413                 .blkcipher = {
 414                         .min_keysize    = TF_MIN_KEY_SIZE +
 415                                           TF_BLOCK_SIZE,
 416                         .max_keysize    = TF_MAX_KEY_SIZE +
 417                                           TF_BLOCK_SIZE,
 418                         .ivsize         = TF_BLOCK_SIZE,
 419                         .setkey         = lrw_twofish_setkey,
 420                         .encrypt        = lrw_encrypt,
 421                         .decrypt        = lrw_decrypt,
 422                 },
 423         },
 424 }, {
 425         .cra_name               = "__xts-twofish-avx",
 426         .cra_driver_name        = "__driver-xts-twofish-avx",
 427         .cra_priority           = 0,
 428         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 429                                   CRYPTO_ALG_INTERNAL,
 430         .cra_blocksize          = TF_BLOCK_SIZE,
 431         .cra_ctxsize            = sizeof(struct twofish_xts_ctx),
 432         .cra_alignmask          = 0,
 433         .cra_type               = &crypto_blkcipher_type,
 434         .cra_module             = THIS_MODULE,
 435         .cra_u = {
 436                 .blkcipher = {
 437                         .min_keysize    = TF_MIN_KEY_SIZE * 2,
 438                         .max_keysize    = TF_MAX_KEY_SIZE * 2,
 439                         .ivsize         = TF_BLOCK_SIZE,
 440                         .setkey         = xts_twofish_setkey,
 441                         .encrypt        = xts_encrypt,
 442                         .decrypt        = xts_decrypt,
 443                 },
 444         },
 445 }, {
 446         .cra_name               = "ecb(twofish)",
 447         .cra_driver_name        = "ecb-twofish-avx",
 448         .cra_priority           = 400,
 449         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 450         .cra_blocksize          = TF_BLOCK_SIZE,
 451         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 452         .cra_alignmask          = 0,
 453         .cra_type               = &crypto_ablkcipher_type,
 454         .cra_module             = THIS_MODULE,
 455         .cra_init               = ablk_init,
 456         .cra_exit               = ablk_exit,
 457         .cra_u = {
 458                 .ablkcipher = {
 459                         .min_keysize    = TF_MIN_KEY_SIZE,
 460                         .max_keysize    = TF_MAX_KEY_SIZE,
 461                         .setkey         = ablk_set_key,
 462                         .encrypt        = ablk_encrypt,
 463                         .decrypt        = ablk_decrypt,
 464                 },
 465         },
 466 }, {
 467         .cra_name               = "cbc(twofish)",
 468         .cra_driver_name        = "cbc-twofish-avx",
 469         .cra_priority           = 400,
 470         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 471         .cra_blocksize          = TF_BLOCK_SIZE,
 472         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 473         .cra_alignmask          = 0,
 474         .cra_type               = &crypto_ablkcipher_type,
 475         .cra_module             = THIS_MODULE,
 476         .cra_init               = ablk_init,
 477         .cra_exit               = ablk_exit,
 478         .cra_u = {
 479                 .ablkcipher = {
 480                         .min_keysize    = TF_MIN_KEY_SIZE,
 481                         .max_keysize    = TF_MAX_KEY_SIZE,
 482                         .ivsize         = TF_BLOCK_SIZE,
 483                         .setkey         = ablk_set_key,
 484                         .encrypt        = __ablk_encrypt,
 485                         .decrypt        = ablk_decrypt,
 486                 },
 487         },
 488 }, {
 489         .cra_name               = "ctr(twofish)",
 490         .cra_driver_name        = "ctr-twofish-avx",
 491         .cra_priority           = 400,
 492         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 493         .cra_blocksize          = 1,
 494         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 495         .cra_alignmask          = 0,
 496         .cra_type               = &crypto_ablkcipher_type,
 497         .cra_module             = THIS_MODULE,
 498         .cra_init               = ablk_init,
 499         .cra_exit               = ablk_exit,
 500         .cra_u = {
 501                 .ablkcipher = {
 502                         .min_keysize    = TF_MIN_KEY_SIZE,
 503                         .max_keysize    = TF_MAX_KEY_SIZE,
 504                         .ivsize         = TF_BLOCK_SIZE,
 505                         .setkey         = ablk_set_key,
 506                         .encrypt        = ablk_encrypt,
 507                         .decrypt        = ablk_encrypt,
 508                         .geniv          = "chainiv",
 509                 },
 510         },
 511 }, {
 512         .cra_name               = "lrw(twofish)",
 513         .cra_driver_name        = "lrw-twofish-avx",
 514         .cra_priority           = 400,
 515         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 516         .cra_blocksize          = TF_BLOCK_SIZE,
 517         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 518         .cra_alignmask          = 0,
 519         .cra_type               = &crypto_ablkcipher_type,
 520         .cra_module             = THIS_MODULE,
 521         .cra_init               = ablk_init,
 522         .cra_exit               = ablk_exit,
 523         .cra_u = {
 524                 .ablkcipher = {
 525                         .min_keysize    = TF_MIN_KEY_SIZE +
 526                                           TF_BLOCK_SIZE,
 527                         .max_keysize    = TF_MAX_KEY_SIZE +
 528                                           TF_BLOCK_SIZE,
 529                         .ivsize         = TF_BLOCK_SIZE,
 530                         .setkey         = ablk_set_key,
 531                         .encrypt        = ablk_encrypt,
 532                         .decrypt        = ablk_decrypt,
 533                 },
 534         },
 535 }, {
 536         .cra_name               = "xts(twofish)",
 537         .cra_driver_name        = "xts-twofish-avx",
 538         .cra_priority           = 400,
 539         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 540         .cra_blocksize          = TF_BLOCK_SIZE,
 541         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 542         .cra_alignmask          = 0,
 543         .cra_type               = &crypto_ablkcipher_type,
 544         .cra_module             = THIS_MODULE,
 545         .cra_init               = ablk_init,
 546         .cra_exit               = ablk_exit,
 547         .cra_u = {
 548                 .ablkcipher = {
 549                         .min_keysize    = TF_MIN_KEY_SIZE * 2,
 550                         .max_keysize    = TF_MAX_KEY_SIZE * 2,
 551                         .ivsize         = TF_BLOCK_SIZE,
 552                         .setkey         = ablk_set_key,
 553                         .encrypt        = ablk_encrypt,
 554                         .decrypt        = ablk_decrypt,
 555                 },
 556         },
 557 } };
 558
 559 static int __init twofish_init(void)
 560 {
 561         u64 xcr0;
 562
 563         if (!cpu_has_avx || !cpu_has_osxsave) {
 564                 printk(KERN_INFO "AVX instructions are not detected.\n");
 565                 return -ENODEV;
 566         }
 567
 568         xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
 569         if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
 570                 printk(KERN_INFO "AVX detected but unusable.\n");
 571                 return -ENODEV;
 572         }
 573
 574         return crypto_register_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
 575 }
 576
 577 static void __exit twofish_exit(void)
 578 {
 579         crypto_unregister_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
 580 }
 581
 582 module_init(twofish_init);
 583 module_exit(twofish_exit);
 584
 585 MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
 586 MODULE_LICENSE("GPL");
 587 MODULE_ALIAS_CRYPTO("twofish");