arch/x86/crypto/twofish_glue_3way.c

   1 /*
   2  * Glue Code for 3-way parallel assembler optimized version of Twofish
   3  *
   4  * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
  19  * USA
  20  *
  21  */
  22
  23 #include <asm/processor.h>
  24 #include <linux/crypto.h>
  25 #include <linux/init.h>
  26 #include <linux/module.h>
  27 #include <linux/types.h>
  28 #include <crypto/algapi.h>
  29 #include <crypto/twofish.h>
  30 #include <crypto/b128ops.h>
  31 #include <asm/crypto/twofish.h>
  32 #include <asm/crypto/glue_helper.h>
  33 #include <crypto/lrw.h>
  34 #include <crypto/xts.h>
  35
  36 EXPORT_SYMBOL_GPL(__twofish_enc_blk_3way);
  37 EXPORT_SYMBOL_GPL(twofish_dec_blk_3way);
  38
  39 static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
  40                                         const u8 *src)
  41 {
  42         __twofish_enc_blk_3way(ctx, dst, src, false);
  43 }
  44
  45 static inline void twofish_enc_blk_xor_3way(struct twofish_ctx *ctx, u8 *dst,
  46                                             const u8 *src)
  47 {
  48         __twofish_enc_blk_3way(ctx, dst, src, true);
  49 }
  50
  51 void twofish_dec_blk_cbc_3way(void *ctx, u128 *dst, const u128 *src)
  52 {
  53         u128 ivs[2];
  54
  55         ivs[0] = src[0];
  56         ivs[1] = src[1];
  57
  58         twofish_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src);
  59
  60         u128_xor(&dst[1], &dst[1], &ivs[0]);
  61         u128_xor(&dst[2], &dst[2], &ivs[1]);
  62 }
  63 EXPORT_SYMBOL_GPL(twofish_dec_blk_cbc_3way);
  64
  65 void twofish_enc_blk_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
  66 {
  67         be128 ctrblk;
  68
  69         if (dst != src)
  70                 *dst = *src;
  71
  72         le128_to_be128(&ctrblk, iv);
  73         le128_inc(iv);
  74
  75         twofish_enc_blk(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
  76         u128_xor(dst, dst, (u128 *)&ctrblk);
  77 }
  78 EXPORT_SYMBOL_GPL(twofish_enc_blk_ctr);
  79
  80 void twofish_enc_blk_ctr_3way(void *ctx, u128 *dst, const u128 *src,
  81                               le128 *iv)
  82 {
  83         be128 ctrblks[3];
  84
  85         if (dst != src) {
  86                 dst[0] = src[0];
  87                 dst[1] = src[1];
  88                 dst[2] = src[2];
  89         }
  90
  91         le128_to_be128(&ctrblks[0], iv);
  92         le128_inc(iv);
  93         le128_to_be128(&ctrblks[1], iv);
  94         le128_inc(iv);
  95         le128_to_be128(&ctrblks[2], iv);
  96         le128_inc(iv);
  97
  98         twofish_enc_blk_xor_3way(ctx, (u8 *)dst, (u8 *)ctrblks);
  99 }
 100 EXPORT_SYMBOL_GPL(twofish_enc_blk_ctr_3way);
 101
 102 static const struct common_glue_ctx twofish_enc = {
 103         .num_funcs = 2,
 104         .fpu_blocks_limit = -1,
 105
 106         .funcs = { {
 107                 .num_blocks = 3,
 108                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
 109         }, {
 110                 .num_blocks = 1,
 111                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
 112         } }
 113 };
 114
 115 static const struct common_glue_ctx twofish_ctr = {
 116         .num_funcs = 2,
 117         .fpu_blocks_limit = -1,
 118
 119         .funcs = { {
 120                 .num_blocks = 3,
 121                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_ctr_3way) }
 122         }, {
 123                 .num_blocks = 1,
 124                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_ctr) }
 125         } }
 126 };
 127
 128 static const struct common_glue_ctx twofish_dec = {
 129         .num_funcs = 2,
 130         .fpu_blocks_limit = -1,
 131
 132         .funcs = { {
 133                 .num_blocks = 3,
 134                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
 135         }, {
 136                 .num_blocks = 1,
 137                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
 138         } }
 139 };
 140
 141 static const struct common_glue_ctx twofish_dec_cbc = {
 142         .num_funcs = 2,
 143         .fpu_blocks_limit = -1,
 144
 145         .funcs = { {
 146                 .num_blocks = 3,
 147                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
 148         }, {
 149                 .num_blocks = 1,
 150                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
 151         } }
 152 };
 153
 154 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 155                        struct scatterlist *src, unsigned int nbytes)
 156 {
 157         return glue_ecb_crypt_128bit(&twofish_enc, desc, dst, src, nbytes);
 158 }
 159
 160 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 161                        struct scatterlist *src, unsigned int nbytes)
 162 {
 163         return glue_ecb_crypt_128bit(&twofish_dec, desc, dst, src, nbytes);
 164 }
 165
 166 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 167                        struct scatterlist *src, unsigned int nbytes)
 168 {
 169         return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk), desc,
 170                                        dst, src, nbytes);
 171 }
 172
 173 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 174                        struct scatterlist *src, unsigned int nbytes)
 175 {
 176         return glue_cbc_decrypt_128bit(&twofish_dec_cbc, desc, dst, src,
 177                                        nbytes);
 178 }
 179
 180 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 181                      struct scatterlist *src, unsigned int nbytes)
 182 {
 183         return glue_ctr_crypt_128bit(&twofish_ctr, desc, dst, src, nbytes);
 184 }
 185
 186 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
 187 {
 188         const unsigned int bsize = TF_BLOCK_SIZE;
 189         struct twofish_ctx *ctx = priv;
 190         int i;
 191
 192         if (nbytes == 3 * bsize) {
 193                 twofish_enc_blk_3way(ctx, srcdst, srcdst);
 194                 return;
 195         }
 196
 197         for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
 198                 twofish_enc_blk(ctx, srcdst, srcdst);
 199 }
 200
 201 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
 202 {
 203         const unsigned int bsize = TF_BLOCK_SIZE;
 204         struct twofish_ctx *ctx = priv;
 205         int i;
 206
 207         if (nbytes == 3 * bsize) {
 208                 twofish_dec_blk_3way(ctx, srcdst, srcdst);
 209                 return;
 210         }
 211
 212         for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
 213                 twofish_dec_blk(ctx, srcdst, srcdst);
 214 }
 215
 216 int lrw_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
 217                        unsigned int keylen)
 218 {
 219         struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
 220         int err;
 221
 222         err = __twofish_setkey(&ctx->twofish_ctx, key, keylen - TF_BLOCK_SIZE,
 223                                &tfm->crt_flags);
 224         if (err)
 225                 return err;
 226
 227         return lrw_init_table(&ctx->lrw_table, key + keylen - TF_BLOCK_SIZE);
 228 }
 229 EXPORT_SYMBOL_GPL(lrw_twofish_setkey);
 230
 231 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 232                        struct scatterlist *src, unsigned int nbytes)
 233 {
 234         struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 235         be128 buf[3];
 236         struct lrw_crypt_req req = {
 237                 .tbuf = buf,
 238                 .tbuflen = sizeof(buf),
 239
 240                 .table_ctx = &ctx->lrw_table,
 241                 .crypt_ctx = &ctx->twofish_ctx,
 242                 .crypt_fn = encrypt_callback,
 243         };
 244
 245         return lrw_crypt(desc, dst, src, nbytes, &req);
 246 }
 247
 248 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 249                        struct scatterlist *src, unsigned int nbytes)
 250 {
 251         struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 252         be128 buf[3];
 253         struct lrw_crypt_req req = {
 254                 .tbuf = buf,
 255                 .tbuflen = sizeof(buf),
 256
 257                 .table_ctx = &ctx->lrw_table,
 258                 .crypt_ctx = &ctx->twofish_ctx,
 259                 .crypt_fn = decrypt_callback,
 260         };
 261
 262         return lrw_crypt(desc, dst, src, nbytes, &req);
 263 }
 264
 265 void lrw_twofish_exit_tfm(struct crypto_tfm *tfm)
 266 {
 267         struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
 268
 269         lrw_free_table(&ctx->lrw_table);
 270 }
 271 EXPORT_SYMBOL_GPL(lrw_twofish_exit_tfm);
 272
 273 int xts_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
 274                        unsigned int keylen)
 275 {
 276         struct twofish_xts_ctx *ctx = crypto_tfm_ctx(tfm);
 277         u32 *flags = &tfm->crt_flags;
 278         int err;
 279
 280         err = xts_check_key(tfm, key, keylen);
 281         if (err)
 282                 return err;
 283
 284         /* first half of xts-key is for crypt */
 285         err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2, flags);
 286         if (err)
 287                 return err;
 288
 289         /* second half of xts-key is for tweak */
 290         return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
 291                                 flags);
 292 }
 293 EXPORT_SYMBOL_GPL(xts_twofish_setkey);
 294
 295 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 296                        struct scatterlist *src, unsigned int nbytes)
 297 {
 298         struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 299         be128 buf[3];
 300         struct xts_crypt_req req = {
 301                 .tbuf = buf,
 302                 .tbuflen = sizeof(buf),
 303
 304                 .tweak_ctx = &ctx->tweak_ctx,
 305                 .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk),
 306                 .crypt_ctx = &ctx->crypt_ctx,
 307                 .crypt_fn = encrypt_callback,
 308         };
 309
 310         return xts_crypt(desc, dst, src, nbytes, &req);
 311 }
 312
 313 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 314                        struct scatterlist *src, unsigned int nbytes)
 315 {
 316         struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 317         be128 buf[3];
 318         struct xts_crypt_req req = {
 319                 .tbuf = buf,
 320                 .tbuflen = sizeof(buf),
 321
 322                 .tweak_ctx = &ctx->tweak_ctx,
 323                 .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk),
 324                 .crypt_ctx = &ctx->crypt_ctx,
 325                 .crypt_fn = decrypt_callback,
 326         };
 327
 328         return xts_crypt(desc, dst, src, nbytes, &req);
 329 }
 330
 331 static struct crypto_alg tf_algs[5] = { {
 332         .cra_name               = "ecb(twofish)",
 333         .cra_driver_name        = "ecb-twofish-3way",
 334         .cra_priority           = 300,
 335         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 336         .cra_blocksize          = TF_BLOCK_SIZE,
 337         .cra_ctxsize            = sizeof(struct twofish_ctx),
 338         .cra_alignmask          = 0,
 339         .cra_type               = &crypto_blkcipher_type,
 340         .cra_module             = THIS_MODULE,
 341         .cra_u = {
 342                 .blkcipher = {
 343                         .min_keysize    = TF_MIN_KEY_SIZE,
 344                         .max_keysize    = TF_MAX_KEY_SIZE,
 345                         .setkey         = twofish_setkey,
 346                         .encrypt        = ecb_encrypt,
 347                         .decrypt        = ecb_decrypt,
 348                 },
 349         },
 350 }, {
 351         .cra_name               = "cbc(twofish)",
 352         .cra_driver_name        = "cbc-twofish-3way",
 353         .cra_priority           = 300,
 354         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 355         .cra_blocksize          = TF_BLOCK_SIZE,
 356         .cra_ctxsize            = sizeof(struct twofish_ctx),
 357         .cra_alignmask          = 0,
 358         .cra_type               = &crypto_blkcipher_type,
 359         .cra_module             = THIS_MODULE,
 360         .cra_u = {
 361                 .blkcipher = {
 362                         .min_keysize    = TF_MIN_KEY_SIZE,
 363                         .max_keysize    = TF_MAX_KEY_SIZE,
 364                         .ivsize         = TF_BLOCK_SIZE,
 365                         .setkey         = twofish_setkey,
 366                         .encrypt        = cbc_encrypt,
 367                         .decrypt        = cbc_decrypt,
 368                 },
 369         },
 370 }, {
 371         .cra_name               = "ctr(twofish)",
 372         .cra_driver_name        = "ctr-twofish-3way",
 373         .cra_priority           = 300,
 374         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 375         .cra_blocksize          = 1,
 376         .cra_ctxsize            = sizeof(struct twofish_ctx),
 377         .cra_alignmask          = 0,
 378         .cra_type               = &crypto_blkcipher_type,
 379         .cra_module             = THIS_MODULE,
 380         .cra_u = {
 381                 .blkcipher = {
 382                         .min_keysize    = TF_MIN_KEY_SIZE,
 383                         .max_keysize    = TF_MAX_KEY_SIZE,
 384                         .ivsize         = TF_BLOCK_SIZE,
 385                         .setkey         = twofish_setkey,
 386                         .encrypt        = ctr_crypt,
 387                         .decrypt        = ctr_crypt,
 388                 },
 389         },
 390 }, {
 391         .cra_name               = "lrw(twofish)",
 392         .cra_driver_name        = "lrw-twofish-3way",
 393         .cra_priority           = 300,
 394         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 395         .cra_blocksize          = TF_BLOCK_SIZE,
 396         .cra_ctxsize            = sizeof(struct twofish_lrw_ctx),
 397         .cra_alignmask          = 0,
 398         .cra_type               = &crypto_blkcipher_type,
 399         .cra_module             = THIS_MODULE,
 400         .cra_exit               = lrw_twofish_exit_tfm,
 401         .cra_u = {
 402                 .blkcipher = {
 403                         .min_keysize    = TF_MIN_KEY_SIZE + TF_BLOCK_SIZE,
 404                         .max_keysize    = TF_MAX_KEY_SIZE + TF_BLOCK_SIZE,
 405                         .ivsize         = TF_BLOCK_SIZE,
 406                         .setkey         = lrw_twofish_setkey,
 407                         .encrypt        = lrw_encrypt,
 408                         .decrypt        = lrw_decrypt,
 409                 },
 410         },
 411 }, {
 412         .cra_name               = "xts(twofish)",
 413         .cra_driver_name        = "xts-twofish-3way",
 414         .cra_priority           = 300,
 415         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 416         .cra_blocksize          = TF_BLOCK_SIZE,
 417         .cra_ctxsize            = sizeof(struct twofish_xts_ctx),
 418         .cra_alignmask          = 0,
 419         .cra_type               = &crypto_blkcipher_type,
 420         .cra_module             = THIS_MODULE,
 421         .cra_u = {
 422                 .blkcipher = {
 423                         .min_keysize    = TF_MIN_KEY_SIZE * 2,
 424                         .max_keysize    = TF_MAX_KEY_SIZE * 2,
 425                         .ivsize         = TF_BLOCK_SIZE,
 426                         .setkey         = xts_twofish_setkey,
 427                         .encrypt        = xts_encrypt,
 428                         .decrypt        = xts_decrypt,
 429                 },
 430         },
 431 } };
 432
 433 static bool is_blacklisted_cpu(void)
 434 {
 435         if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
 436                 return false;
 437
 438         if (boot_cpu_data.x86 == 0x06 &&
 439                 (boot_cpu_data.x86_model == 0x1c ||
 440                  boot_cpu_data.x86_model == 0x26 ||
 441                  boot_cpu_data.x86_model == 0x36)) {
 442                 /*
 443                  * On Atom, twofish-3way is slower than original assembler
 444                  * implementation. Twofish-3way trades off some performance in
 445                  * storing blocks in 64bit registers to allow three blocks to
 446                  * be processed parallel. Parallel operation then allows gaining
 447                  * more performance than was trade off, on out-of-order CPUs.
 448                  * However Atom does not benefit from this parallellism and
 449                  * should be blacklisted.
 450                  */
 451                 return true;
 452         }
 453
 454         if (boot_cpu_data.x86 == 0x0f) {
 455                 /*
 456                  * On Pentium 4, twofish-3way is slower than original assembler
 457                  * implementation because excessive uses of 64bit rotate and
 458                  * left-shifts (which are really slow on P4) needed to store and
 459                  * handle 128bit block in two 64bit registers.
 460                  */
 461                 return true;
 462         }
 463
 464         return false;
 465 }
 466
 467 static int force;
 468 module_param(force, int, 0);
 469 MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist");
 470
 471 static int __init init(void)
 472 {
 473         if (!force && is_blacklisted_cpu()) {
 474                 printk(KERN_INFO
 475                         "twofish-x86_64-3way: performance on this CPU "
 476                         "would be suboptimal: disabling "
 477                         "twofish-x86_64-3way.\n");
 478                 return -ENODEV;
 479         }
 480
 481         return crypto_register_algs(tf_algs, ARRAY_SIZE(tf_algs));
 482 }
 483
 484 static void __exit fini(void)
 485 {
 486         crypto_unregister_algs(tf_algs, ARRAY_SIZE(tf_algs));
 487 }
 488
 489 module_init(init);
 490 module_exit(fini);
 491
 492 MODULE_LICENSE("GPL");
 493 MODULE_DESCRIPTION("Twofish Cipher Algorithm, 3-way parallel asm optimized");
 494 MODULE_ALIAS_CRYPTO("twofish");
 495 MODULE_ALIAS_CRYPTO("twofish-asm");