arch/x86/crypto/serpent_avx2_glue.c

   1 /*
   2  * Glue Code for x86_64/AVX2 assembler optimized version of Serpent
   3  *
   4  * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  */
  12
  13 #include <linux/module.h>
  14 #include <linux/types.h>
  15 #include <linux/crypto.h>
  16 #include <linux/err.h>
  17 #include <crypto/ablk_helper.h>
  18 #include <crypto/algapi.h>
  19 #include <crypto/ctr.h>
  20 #include <crypto/lrw.h>
  21 #include <crypto/xts.h>
  22 #include <crypto/serpent.h>
  23 #include <asm/fpu/api.h>
  24 #include <asm/crypto/serpent-avx.h>
  25 #include <asm/crypto/glue_helper.h>
  26
  27 #define SERPENT_AVX2_PARALLEL_BLOCKS 16
  28
  29 /* 16-way AVX2 parallel cipher functions */
  30 asmlinkage void serpent_ecb_enc_16way(struct serpent_ctx *ctx, u8 *dst,
  31                                       const u8 *src);
  32 asmlinkage void serpent_ecb_dec_16way(struct serpent_ctx *ctx, u8 *dst,
  33                                       const u8 *src);
  34 asmlinkage void serpent_cbc_dec_16way(void *ctx, u128 *dst, const u128 *src);
  35
  36 asmlinkage void serpent_ctr_16way(void *ctx, u128 *dst, const u128 *src,
  37                                   le128 *iv);
  38 asmlinkage void serpent_xts_enc_16way(struct serpent_ctx *ctx, u8 *dst,
  39                                       const u8 *src, le128 *iv);
  40 asmlinkage void serpent_xts_dec_16way(struct serpent_ctx *ctx, u8 *dst,
  41                                       const u8 *src, le128 *iv);
  42
  43 static const struct common_glue_ctx serpent_enc = {
  44         .num_funcs = 3,
  45         .fpu_blocks_limit = 8,
  46
  47         .funcs = { {
  48                 .num_blocks = 16,
  49                 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_16way) }
  50         }, {
  51                 .num_blocks = 8,
  52                 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_8way_avx) }
  53         }, {
  54                 .num_blocks = 1,
  55                 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
  56         } }
  57 };
  58
  59 static const struct common_glue_ctx serpent_ctr = {
  60         .num_funcs = 3,
  61         .fpu_blocks_limit = 8,
  62
  63         .funcs = { {
  64                 .num_blocks = 16,
  65                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_16way) }
  66         },  {
  67                 .num_blocks = 8,
  68                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_8way_avx) }
  69         }, {
  70                 .num_blocks = 1,
  71                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(__serpent_crypt_ctr) }
  72         } }
  73 };
  74
  75 static const struct common_glue_ctx serpent_enc_xts = {
  76         .num_funcs = 3,
  77         .fpu_blocks_limit = 8,
  78
  79         .funcs = { {
  80                 .num_blocks = 16,
  81                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_16way) }
  82         }, {
  83                 .num_blocks = 8,
  84                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_8way_avx) }
  85         }, {
  86                 .num_blocks = 1,
  87                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc) }
  88         } }
  89 };
  90
  91 static const struct common_glue_ctx serpent_dec = {
  92         .num_funcs = 3,
  93         .fpu_blocks_limit = 8,
  94
  95         .funcs = { {
  96                 .num_blocks = 16,
  97                 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_16way) }
  98         }, {
  99                 .num_blocks = 8,
 100                 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_8way_avx) }
 101         }, {
 102                 .num_blocks = 1,
 103                 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
 104         } }
 105 };
 106
 107 static const struct common_glue_ctx serpent_dec_cbc = {
 108         .num_funcs = 3,
 109         .fpu_blocks_limit = 8,
 110
 111         .funcs = { {
 112                 .num_blocks = 16,
 113                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_16way) }
 114         }, {
 115                 .num_blocks = 8,
 116                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_8way_avx) }
 117         }, {
 118                 .num_blocks = 1,
 119                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
 120         } }
 121 };
 122
 123 static const struct common_glue_ctx serpent_dec_xts = {
 124         .num_funcs = 3,
 125         .fpu_blocks_limit = 8,
 126
 127         .funcs = { {
 128                 .num_blocks = 16,
 129                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_16way) }
 130         }, {
 131                 .num_blocks = 8,
 132                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_8way_avx) }
 133         }, {
 134                 .num_blocks = 1,
 135                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec) }
 136         } }
 137 };
 138
 139 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 140                        struct scatterlist *src, unsigned int nbytes)
 141 {
 142         return glue_ecb_crypt_128bit(&serpent_enc, desc, dst, src, nbytes);
 143 }
 144
 145 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 146                        struct scatterlist *src, unsigned int nbytes)
 147 {
 148         return glue_ecb_crypt_128bit(&serpent_dec, desc, dst, src, nbytes);
 149 }
 150
 151 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 152                        struct scatterlist *src, unsigned int nbytes)
 153 {
 154         return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__serpent_encrypt), desc,
 155                                        dst, src, nbytes);
 156 }
 157
 158 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 159                        struct scatterlist *src, unsigned int nbytes)
 160 {
 161         return glue_cbc_decrypt_128bit(&serpent_dec_cbc, desc, dst, src,
 162                                        nbytes);
 163 }
 164
 165 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 166                      struct scatterlist *src, unsigned int nbytes)
 167 {
 168         return glue_ctr_crypt_128bit(&serpent_ctr, desc, dst, src, nbytes);
 169 }
 170
 171 static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes)
 172 {
 173         /* since reusing AVX functions, starts using FPU at 8 parallel blocks */
 174         return glue_fpu_begin(SERPENT_BLOCK_SIZE, 8, NULL, fpu_enabled, nbytes);
 175 }
 176
 177 static inline void serpent_fpu_end(bool fpu_enabled)
 178 {
 179         glue_fpu_end(fpu_enabled);
 180 }
 181
 182 struct crypt_priv {
 183         struct serpent_ctx *ctx;
 184         bool fpu_enabled;
 185 };
 186
 187 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
 188 {
 189         const unsigned int bsize = SERPENT_BLOCK_SIZE;
 190         struct crypt_priv *ctx = priv;
 191         int i;
 192
 193         ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
 194
 195         if (nbytes >= SERPENT_AVX2_PARALLEL_BLOCKS * bsize) {
 196                 serpent_ecb_enc_16way(ctx->ctx, srcdst, srcdst);
 197                 srcdst += bsize * SERPENT_AVX2_PARALLEL_BLOCKS;
 198                 nbytes -= bsize * SERPENT_AVX2_PARALLEL_BLOCKS;
 199         }
 200
 201         while (nbytes >= SERPENT_PARALLEL_BLOCKS * bsize) {
 202                 serpent_ecb_enc_8way_avx(ctx->ctx, srcdst, srcdst);
 203                 srcdst += bsize * SERPENT_PARALLEL_BLOCKS;
 204                 nbytes -= bsize * SERPENT_PARALLEL_BLOCKS;
 205         }
 206
 207         for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
 208                 __serpent_encrypt(ctx->ctx, srcdst, srcdst);
 209 }
 210
 211 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
 212 {
 213         const unsigned int bsize = SERPENT_BLOCK_SIZE;
 214         struct crypt_priv *ctx = priv;
 215         int i;
 216
 217         ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
 218
 219         if (nbytes >= SERPENT_AVX2_PARALLEL_BLOCKS * bsize) {
 220                 serpent_ecb_dec_16way(ctx->ctx, srcdst, srcdst);
 221                 srcdst += bsize * SERPENT_AVX2_PARALLEL_BLOCKS;
 222                 nbytes -= bsize * SERPENT_AVX2_PARALLEL_BLOCKS;
 223         }
 224
 225         while (nbytes >= SERPENT_PARALLEL_BLOCKS * bsize) {
 226                 serpent_ecb_dec_8way_avx(ctx->ctx, srcdst, srcdst);
 227                 srcdst += bsize * SERPENT_PARALLEL_BLOCKS;
 228                 nbytes -= bsize * SERPENT_PARALLEL_BLOCKS;
 229         }
 230
 231         for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
 232                 __serpent_decrypt(ctx->ctx, srcdst, srcdst);
 233 }
 234
 235 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 236                        struct scatterlist *src, unsigned int nbytes)
 237 {
 238         struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 239         be128 buf[SERPENT_AVX2_PARALLEL_BLOCKS];
 240         struct crypt_priv crypt_ctx = {
 241                 .ctx = &ctx->serpent_ctx,
 242                 .fpu_enabled = false,
 243         };
 244         struct lrw_crypt_req req = {
 245                 .tbuf = buf,
 246                 .tbuflen = sizeof(buf),
 247
 248                 .table_ctx = &ctx->lrw_table,
 249                 .crypt_ctx = &crypt_ctx,
 250                 .crypt_fn = encrypt_callback,
 251         };
 252         int ret;
 253
 254         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 255         ret = lrw_crypt(desc, dst, src, nbytes, &req);
 256         serpent_fpu_end(crypt_ctx.fpu_enabled);
 257
 258         return ret;
 259 }
 260
 261 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 262                        struct scatterlist *src, unsigned int nbytes)
 263 {
 264         struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 265         be128 buf[SERPENT_AVX2_PARALLEL_BLOCKS];
 266         struct crypt_priv crypt_ctx = {
 267                 .ctx = &ctx->serpent_ctx,
 268                 .fpu_enabled = false,
 269         };
 270         struct lrw_crypt_req req = {
 271                 .tbuf = buf,
 272                 .tbuflen = sizeof(buf),
 273
 274                 .table_ctx = &ctx->lrw_table,
 275                 .crypt_ctx = &crypt_ctx,
 276                 .crypt_fn = decrypt_callback,
 277         };
 278         int ret;
 279
 280         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 281         ret = lrw_crypt(desc, dst, src, nbytes, &req);
 282         serpent_fpu_end(crypt_ctx.fpu_enabled);
 283
 284         return ret;
 285 }
 286
 287 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 288                        struct scatterlist *src, unsigned int nbytes)
 289 {
 290         struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 291
 292         return glue_xts_crypt_128bit(&serpent_enc_xts, desc, dst, src, nbytes,
 293                                      XTS_TWEAK_CAST(__serpent_encrypt),
 294                                      &ctx->tweak_ctx, &ctx->crypt_ctx);
 295 }
 296
 297 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 298                        struct scatterlist *src, unsigned int nbytes)
 299 {
 300         struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 301
 302         return glue_xts_crypt_128bit(&serpent_dec_xts, desc, dst, src, nbytes,
 303                                      XTS_TWEAK_CAST(__serpent_encrypt),
 304                                      &ctx->tweak_ctx, &ctx->crypt_ctx);
 305 }
 306
 307 static struct crypto_alg srp_algs[10] = { {
 308         .cra_name               = "__ecb-serpent-avx2",
 309         .cra_driver_name        = "__driver-ecb-serpent-avx2",
 310         .cra_priority           = 0,
 311         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 312                                   CRYPTO_ALG_INTERNAL,
 313         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 314         .cra_ctxsize            = sizeof(struct serpent_ctx),
 315         .cra_alignmask          = 0,
 316         .cra_type               = &crypto_blkcipher_type,
 317         .cra_module             = THIS_MODULE,
 318         .cra_list               = LIST_HEAD_INIT(srp_algs[0].cra_list),
 319         .cra_u = {
 320                 .blkcipher = {
 321                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 322                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 323                         .setkey         = serpent_setkey,
 324                         .encrypt        = ecb_encrypt,
 325                         .decrypt        = ecb_decrypt,
 326                 },
 327         },
 328 }, {
 329         .cra_name               = "__cbc-serpent-avx2",
 330         .cra_driver_name        = "__driver-cbc-serpent-avx2",
 331         .cra_priority           = 0,
 332         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 333                                   CRYPTO_ALG_INTERNAL,
 334         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 335         .cra_ctxsize            = sizeof(struct serpent_ctx),
 336         .cra_alignmask          = 0,
 337         .cra_type               = &crypto_blkcipher_type,
 338         .cra_module             = THIS_MODULE,
 339         .cra_list               = LIST_HEAD_INIT(srp_algs[1].cra_list),
 340         .cra_u = {
 341                 .blkcipher = {
 342                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 343                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 344                         .setkey         = serpent_setkey,
 345                         .encrypt        = cbc_encrypt,
 346                         .decrypt        = cbc_decrypt,
 347                 },
 348         },
 349 }, {
 350         .cra_name               = "__ctr-serpent-avx2",
 351         .cra_driver_name        = "__driver-ctr-serpent-avx2",
 352         .cra_priority           = 0,
 353         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 354                                   CRYPTO_ALG_INTERNAL,
 355         .cra_blocksize          = 1,
 356         .cra_ctxsize            = sizeof(struct serpent_ctx),
 357         .cra_alignmask          = 0,
 358         .cra_type               = &crypto_blkcipher_type,
 359         .cra_module             = THIS_MODULE,
 360         .cra_list               = LIST_HEAD_INIT(srp_algs[2].cra_list),
 361         .cra_u = {
 362                 .blkcipher = {
 363                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 364                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 365                         .ivsize         = SERPENT_BLOCK_SIZE,
 366                         .setkey         = serpent_setkey,
 367                         .encrypt        = ctr_crypt,
 368                         .decrypt        = ctr_crypt,
 369                 },
 370         },
 371 }, {
 372         .cra_name               = "__lrw-serpent-avx2",
 373         .cra_driver_name        = "__driver-lrw-serpent-avx2",
 374         .cra_priority           = 0,
 375         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 376                                   CRYPTO_ALG_INTERNAL,
 377         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 378         .cra_ctxsize            = sizeof(struct serpent_lrw_ctx),
 379         .cra_alignmask          = 0,
 380         .cra_type               = &crypto_blkcipher_type,
 381         .cra_module             = THIS_MODULE,
 382         .cra_list               = LIST_HEAD_INIT(srp_algs[3].cra_list),
 383         .cra_exit               = lrw_serpent_exit_tfm,
 384         .cra_u = {
 385                 .blkcipher = {
 386                         .min_keysize    = SERPENT_MIN_KEY_SIZE +
 387                                           SERPENT_BLOCK_SIZE,
 388                         .max_keysize    = SERPENT_MAX_KEY_SIZE +
 389                                           SERPENT_BLOCK_SIZE,
 390                         .ivsize         = SERPENT_BLOCK_SIZE,
 391                         .setkey         = lrw_serpent_setkey,
 392                         .encrypt        = lrw_encrypt,
 393                         .decrypt        = lrw_decrypt,
 394                 },
 395         },
 396 }, {
 397         .cra_name               = "__xts-serpent-avx2",
 398         .cra_driver_name        = "__driver-xts-serpent-avx2",
 399         .cra_priority           = 0,
 400         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 401                                   CRYPTO_ALG_INTERNAL,
 402         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 403         .cra_ctxsize            = sizeof(struct serpent_xts_ctx),
 404         .cra_alignmask          = 0,
 405         .cra_type               = &crypto_blkcipher_type,
 406         .cra_module             = THIS_MODULE,
 407         .cra_list               = LIST_HEAD_INIT(srp_algs[4].cra_list),
 408         .cra_u = {
 409                 .blkcipher = {
 410                         .min_keysize    = SERPENT_MIN_KEY_SIZE * 2,
 411                         .max_keysize    = SERPENT_MAX_KEY_SIZE * 2,
 412                         .ivsize         = SERPENT_BLOCK_SIZE,
 413                         .setkey         = xts_serpent_setkey,
 414                         .encrypt        = xts_encrypt,
 415                         .decrypt        = xts_decrypt,
 416                 },
 417         },
 418 }, {
 419         .cra_name               = "ecb(serpent)",
 420         .cra_driver_name        = "ecb-serpent-avx2",
 421         .cra_priority           = 600,
 422         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 423         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 424         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 425         .cra_alignmask          = 0,
 426         .cra_type               = &crypto_ablkcipher_type,
 427         .cra_module             = THIS_MODULE,
 428         .cra_list               = LIST_HEAD_INIT(srp_algs[5].cra_list),
 429         .cra_init               = ablk_init,
 430         .cra_exit               = ablk_exit,
 431         .cra_u = {
 432                 .ablkcipher = {
 433                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 434                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 435                         .setkey         = ablk_set_key,
 436                         .encrypt        = ablk_encrypt,
 437                         .decrypt        = ablk_decrypt,
 438                 },
 439         },
 440 }, {
 441         .cra_name               = "cbc(serpent)",
 442         .cra_driver_name        = "cbc-serpent-avx2",
 443         .cra_priority           = 600,
 444         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 445         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 446         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 447         .cra_alignmask          = 0,
 448         .cra_type               = &crypto_ablkcipher_type,
 449         .cra_module             = THIS_MODULE,
 450         .cra_list               = LIST_HEAD_INIT(srp_algs[6].cra_list),
 451         .cra_init               = ablk_init,
 452         .cra_exit               = ablk_exit,
 453         .cra_u = {
 454                 .ablkcipher = {
 455                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 456                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 457                         .ivsize         = SERPENT_BLOCK_SIZE,
 458                         .setkey         = ablk_set_key,
 459                         .encrypt        = __ablk_encrypt,
 460                         .decrypt        = ablk_decrypt,
 461                 },
 462         },
 463 }, {
 464         .cra_name               = "ctr(serpent)",
 465         .cra_driver_name        = "ctr-serpent-avx2",
 466         .cra_priority           = 600,
 467         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 468         .cra_blocksize          = 1,
 469         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 470         .cra_alignmask          = 0,
 471         .cra_type               = &crypto_ablkcipher_type,
 472         .cra_module             = THIS_MODULE,
 473         .cra_list               = LIST_HEAD_INIT(srp_algs[7].cra_list),
 474         .cra_init               = ablk_init,
 475         .cra_exit               = ablk_exit,
 476         .cra_u = {
 477                 .ablkcipher = {
 478                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 479                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 480                         .ivsize         = SERPENT_BLOCK_SIZE,
 481                         .setkey         = ablk_set_key,
 482                         .encrypt        = ablk_encrypt,
 483                         .decrypt        = ablk_encrypt,
 484                         .geniv          = "chainiv",
 485                 },
 486         },
 487 }, {
 488         .cra_name               = "lrw(serpent)",
 489         .cra_driver_name        = "lrw-serpent-avx2",
 490         .cra_priority           = 600,
 491         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 492         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 493         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 494         .cra_alignmask          = 0,
 495         .cra_type               = &crypto_ablkcipher_type,
 496         .cra_module             = THIS_MODULE,
 497         .cra_list               = LIST_HEAD_INIT(srp_algs[8].cra_list),
 498         .cra_init               = ablk_init,
 499         .cra_exit               = ablk_exit,
 500         .cra_u = {
 501                 .ablkcipher = {
 502                         .min_keysize    = SERPENT_MIN_KEY_SIZE +
 503                                           SERPENT_BLOCK_SIZE,
 504                         .max_keysize    = SERPENT_MAX_KEY_SIZE +
 505                                           SERPENT_BLOCK_SIZE,
 506                         .ivsize         = SERPENT_BLOCK_SIZE,
 507                         .setkey         = ablk_set_key,
 508                         .encrypt        = ablk_encrypt,
 509                         .decrypt        = ablk_decrypt,
 510                 },
 511         },
 512 }, {
 513         .cra_name               = "xts(serpent)",
 514         .cra_driver_name        = "xts-serpent-avx2",
 515         .cra_priority           = 600,
 516         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 517         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 518         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 519         .cra_alignmask          = 0,
 520         .cra_type               = &crypto_ablkcipher_type,
 521         .cra_module             = THIS_MODULE,
 522         .cra_list               = LIST_HEAD_INIT(srp_algs[9].cra_list),
 523         .cra_init               = ablk_init,
 524         .cra_exit               = ablk_exit,
 525         .cra_u = {
 526                 .ablkcipher = {
 527                         .min_keysize    = SERPENT_MIN_KEY_SIZE * 2,
 528                         .max_keysize    = SERPENT_MAX_KEY_SIZE * 2,
 529                         .ivsize         = SERPENT_BLOCK_SIZE,
 530                         .setkey         = ablk_set_key,
 531                         .encrypt        = ablk_encrypt,
 532                         .decrypt        = ablk_decrypt,
 533                 },
 534         },
 535 } };
 536
 537 static int __init init(void)
 538 {
 539         const char *feature_name;
 540
 541         if (!cpu_has_avx2 || !cpu_has_osxsave) {
 542                 pr_info("AVX2 instructions are not detected.\n");
 543                 return -ENODEV;
 544         }
 545         if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
 546                                 &feature_name)) {
 547                 pr_info("CPU feature '%s' is not supported.\n", feature_name);
 548                 return -ENODEV;
 549         }
 550
 551         return crypto_register_algs(srp_algs, ARRAY_SIZE(srp_algs));
 552 }
 553
 554 static void __exit fini(void)
 555 {
 556         crypto_unregister_algs(srp_algs, ARRAY_SIZE(srp_algs));
 557 }
 558
 559 module_init(init);
 560 module_exit(fini);
 561
 562 MODULE_LICENSE("GPL");
 563 MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX2 optimized");
 564 MODULE_ALIAS_CRYPTO("serpent");
 565 MODULE_ALIAS_CRYPTO("serpent-asm");