arch/x86/crypto/twofish_glue_3way.c

   1 /*
   2  * Glue Code for 3-way parallel assembler optimized version of Twofish
   3  *
   4  * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
  19  * USA
  20  *
  21  */
  22
  23 #include <asm/processor.h>
  24 #include <linux/crypto.h>
  25 #include <linux/init.h>
  26 #include <linux/module.h>
  27 #include <linux/types.h>
  28 #include <crypto/algapi.h>
  29 #include <crypto/twofish.h>
  30 #include <crypto/b128ops.h>
  31 #include <asm/crypto/twofish.h>
  32 #include <asm/crypto/glue_helper.h>
  33 #include <crypto/lrw.h>
  34 #include <crypto/xts.h>
  35
  36 EXPORT_SYMBOL_GPL(__twofish_enc_blk_3way);
  37 EXPORT_SYMBOL_GPL(twofish_dec_blk_3way);
  38
  39 static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
  40                                         const u8 *src)
  41 {
  42         __twofish_enc_blk_3way(ctx, dst, src, false);
  43 }
  44
  45 static inline void twofish_enc_blk_xor_3way(struct twofish_ctx *ctx, u8 *dst,
  46                                             const u8 *src)
  47 {
  48         __twofish_enc_blk_3way(ctx, dst, src, true);
  49 }
  50
  51 void twofish_dec_blk_cbc_3way(void *ctx, u128 *dst, const u128 *src)
  52 {
  53         u128 ivs[2];
  54
  55         ivs[0] = src[0];
  56         ivs[1] = src[1];
  57
  58         twofish_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src);
  59
  60         u128_xor(&dst[1], &dst[1], &ivs[0]);
  61         u128_xor(&dst[2], &dst[2], &ivs[1]);
  62 }
  63 EXPORT_SYMBOL_GPL(twofish_dec_blk_cbc_3way);
  64
  65 void twofish_enc_blk_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
  66 {
  67         be128 ctrblk;
  68
  69         if (dst != src)
  70                 *dst = *src;
  71
  72         le128_to_be128(&ctrblk, iv);
  73         le128_inc(iv);
  74
  75         twofish_enc_blk(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
  76         u128_xor(dst, dst, (u128 *)&ctrblk);
  77 }
  78 EXPORT_SYMBOL_GPL(twofish_enc_blk_ctr);
  79
  80 void twofish_enc_blk_ctr_3way(void *ctx, u128 *dst, const u128 *src,
  81                               le128 *iv)
  82 {
  83         be128 ctrblks[3];
  84
  85         if (dst != src) {
  86                 dst[0] = src[0];
  87                 dst[1] = src[1];
  88                 dst[2] = src[2];
  89         }
  90
  91         le128_to_be128(&ctrblks[0], iv);
  92         le128_inc(iv);
  93         le128_to_be128(&ctrblks[1], iv);
  94         le128_inc(iv);
  95         le128_to_be128(&ctrblks[2], iv);
  96         le128_inc(iv);
  97
  98         twofish_enc_blk_xor_3way(ctx, (u8 *)dst, (u8 *)ctrblks);
  99 }
 100 EXPORT_SYMBOL_GPL(twofish_enc_blk_ctr_3way);
 101
 102 static const struct common_glue_ctx twofish_enc = {
 103         .num_funcs = 2,
 104         .fpu_blocks_limit = -1,
 105
 106         .funcs = { {
 107                 .num_blocks = 3,
 108                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
 109         }, {
 110                 .num_blocks = 1,
 111                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
 112         } }
 113 };
 114
 115 static const struct common_glue_ctx twofish_ctr = {
 116         .num_funcs = 2,
 117         .fpu_blocks_limit = -1,
 118
 119         .funcs = { {
 120                 .num_blocks = 3,
 121                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_ctr_3way) }
 122         }, {
 123                 .num_blocks = 1,
 124                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_ctr) }
 125         } }
 126 };
 127
 128 static const struct common_glue_ctx twofish_dec = {
 129         .num_funcs = 2,
 130         .fpu_blocks_limit = -1,
 131
 132         .funcs = { {
 133                 .num_blocks = 3,
 134                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
 135         }, {
 136                 .num_blocks = 1,
 137                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
 138         } }
 139 };
 140
 141 static const struct common_glue_ctx twofish_dec_cbc = {
 142         .num_funcs = 2,
 143         .fpu_blocks_limit = -1,
 144
 145         .funcs = { {
 146                 .num_blocks = 3,
 147                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
 148         }, {
 149                 .num_blocks = 1,
 150                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
 151         } }
 152 };
 153
 154 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 155                        struct scatterlist *src, unsigned int nbytes)
 156 {
 157         return glue_ecb_crypt_128bit(&twofish_enc, desc, dst, src, nbytes);
 158 }
 159
 160 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 161                        struct scatterlist *src, unsigned int nbytes)
 162 {
 163         return glue_ecb_crypt_128bit(&twofish_dec, desc, dst, src, nbytes);
 164 }
 165
 166 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 167                        struct scatterlist *src, unsigned int nbytes)
 168 {
 169         return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk), desc,
 170                                        dst, src, nbytes);
 171 }
 172
 173 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 174                        struct scatterlist *src, unsigned int nbytes)
 175 {
 176         return glue_cbc_decrypt_128bit(&twofish_dec_cbc, desc, dst, src,
 177                                        nbytes);
 178 }
 179
 180 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 181                      struct scatterlist *src, unsigned int nbytes)
 182 {
 183         return glue_ctr_crypt_128bit(&twofish_ctr, desc, dst, src, nbytes);
 184 }
 185
 186 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
 187 {
 188         const unsigned int bsize = TF_BLOCK_SIZE;
 189         struct twofish_ctx *ctx = priv;
 190         int i;
 191
 192         if (nbytes == 3 * bsize) {
 193                 twofish_enc_blk_3way(ctx, srcdst, srcdst);
 194                 return;
 195         }
 196
 197         for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
 198                 twofish_enc_blk(ctx, srcdst, srcdst);
 199 }
 200
 201 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
 202 {
 203         const unsigned int bsize = TF_BLOCK_SIZE;
 204         struct twofish_ctx *ctx = priv;
 205         int i;
 206
 207         if (nbytes == 3 * bsize) {
 208                 twofish_dec_blk_3way(ctx, srcdst, srcdst);
 209                 return;
 210         }
 211
 212         for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
 213                 twofish_dec_blk(ctx, srcdst, srcdst);
 214 }
 215
 216 int lrw_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
 217                        unsigned int keylen)
 218 {
 219         struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
 220         int err;
 221
 222         err = __twofish_setkey(&ctx->twofish_ctx, key, keylen - TF_BLOCK_SIZE,
 223                                &tfm->crt_flags);
 224         if (err)
 225                 return err;
 226
 227         return lrw_init_table(&ctx->lrw_table, key + keylen - TF_BLOCK_SIZE);
 228 }
 229 EXPORT_SYMBOL_GPL(lrw_twofish_setkey);
 230
 231 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 232                        struct scatterlist *src, unsigned int nbytes)
 233 {
 234         struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 235         be128 buf[3];
 236         struct lrw_crypt_req req = {
 237                 .tbuf = buf,
 238                 .tbuflen = sizeof(buf),
 239
 240                 .table_ctx = &ctx->lrw_table,
 241                 .crypt_ctx = &ctx->twofish_ctx,
 242                 .crypt_fn = encrypt_callback,
 243         };
 244
 245         return lrw_crypt(desc, dst, src, nbytes, &req);
 246 }
 247
 248 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 249                        struct scatterlist *src, unsigned int nbytes)
 250 {
 251         struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 252         be128 buf[3];
 253         struct lrw_crypt_req req = {
 254                 .tbuf = buf,
 255                 .tbuflen = sizeof(buf),
 256
 257                 .table_ctx = &ctx->lrw_table,
 258                 .crypt_ctx = &ctx->twofish_ctx,
 259                 .crypt_fn = decrypt_callback,
 260         };
 261
 262         return lrw_crypt(desc, dst, src, nbytes, &req);
 263 }
 264
 265 void lrw_twofish_exit_tfm(struct crypto_tfm *tfm)
 266 {
 267         struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
 268
 269         lrw_free_table(&ctx->lrw_table);
 270 }
 271 EXPORT_SYMBOL_GPL(lrw_twofish_exit_tfm);
 272
 273 int xts_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
 274                        unsigned int keylen)
 275 {
 276         struct twofish_xts_ctx *ctx = crypto_tfm_ctx(tfm);
 277         u32 *flags = &tfm->crt_flags;
 278         int err;
 279
 280         /* key consists of keys of equal size concatenated, therefore
 281          * the length must be even
 282          */
 283         if (keylen % 2) {
 284                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 285                 return -EINVAL;
 286         }
 287
 288         /* first half of xts-key is for crypt */
 289         err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2, flags);
 290         if (err)
 291                 return err;
 292
 293         /* second half of xts-key is for tweak */
 294         return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
 295                                 flags);
 296 }
 297 EXPORT_SYMBOL_GPL(xts_twofish_setkey);
 298
 299 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 300                        struct scatterlist *src, unsigned int nbytes)
 301 {
 302         struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 303         be128 buf[3];
 304         struct xts_crypt_req req = {
 305                 .tbuf = buf,
 306                 .tbuflen = sizeof(buf),
 307
 308                 .tweak_ctx = &ctx->tweak_ctx,
 309                 .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk),
 310                 .crypt_ctx = &ctx->crypt_ctx,
 311                 .crypt_fn = encrypt_callback,
 312         };
 313
 314         return xts_crypt(desc, dst, src, nbytes, &req);
 315 }
 316
 317 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 318                        struct scatterlist *src, unsigned int nbytes)
 319 {
 320         struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 321         be128 buf[3];
 322         struct xts_crypt_req req = {
 323                 .tbuf = buf,
 324                 .tbuflen = sizeof(buf),
 325
 326                 .tweak_ctx = &ctx->tweak_ctx,
 327                 .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk),
 328                 .crypt_ctx = &ctx->crypt_ctx,
 329                 .crypt_fn = decrypt_callback,
 330         };
 331
 332         return xts_crypt(desc, dst, src, nbytes, &req);
 333 }
 334
 335 static struct crypto_alg tf_algs[5] = { {
 336         .cra_name               = "ecb(twofish)",
 337         .cra_driver_name        = "ecb-twofish-3way",
 338         .cra_priority           = 300,
 339         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 340         .cra_blocksize          = TF_BLOCK_SIZE,
 341         .cra_ctxsize            = sizeof(struct twofish_ctx),
 342         .cra_alignmask          = 0,
 343         .cra_type               = &crypto_blkcipher_type,
 344         .cra_module             = THIS_MODULE,
 345         .cra_u = {
 346                 .blkcipher = {
 347                         .min_keysize    = TF_MIN_KEY_SIZE,
 348                         .max_keysize    = TF_MAX_KEY_SIZE,
 349                         .setkey         = twofish_setkey,
 350                         .encrypt        = ecb_encrypt,
 351                         .decrypt        = ecb_decrypt,
 352                 },
 353         },
 354 }, {
 355         .cra_name               = "cbc(twofish)",
 356         .cra_driver_name        = "cbc-twofish-3way",
 357         .cra_priority           = 300,
 358         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 359         .cra_blocksize          = TF_BLOCK_SIZE,
 360         .cra_ctxsize            = sizeof(struct twofish_ctx),
 361         .cra_alignmask          = 0,
 362         .cra_type               = &crypto_blkcipher_type,
 363         .cra_module             = THIS_MODULE,
 364         .cra_u = {
 365                 .blkcipher = {
 366                         .min_keysize    = TF_MIN_KEY_SIZE,
 367                         .max_keysize    = TF_MAX_KEY_SIZE,
 368                         .ivsize         = TF_BLOCK_SIZE,
 369                         .setkey         = twofish_setkey,
 370                         .encrypt        = cbc_encrypt,
 371                         .decrypt        = cbc_decrypt,
 372                 },
 373         },
 374 }, {
 375         .cra_name               = "ctr(twofish)",
 376         .cra_driver_name        = "ctr-twofish-3way",
 377         .cra_priority           = 300,
 378         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 379         .cra_blocksize          = 1,
 380         .cra_ctxsize            = sizeof(struct twofish_ctx),
 381         .cra_alignmask          = 0,
 382         .cra_type               = &crypto_blkcipher_type,
 383         .cra_module             = THIS_MODULE,
 384         .cra_u = {
 385                 .blkcipher = {
 386                         .min_keysize    = TF_MIN_KEY_SIZE,
 387                         .max_keysize    = TF_MAX_KEY_SIZE,
 388                         .ivsize         = TF_BLOCK_SIZE,
 389                         .setkey         = twofish_setkey,
 390                         .encrypt        = ctr_crypt,
 391                         .decrypt        = ctr_crypt,
 392                 },
 393         },
 394 }, {
 395         .cra_name               = "lrw(twofish)",
 396         .cra_driver_name        = "lrw-twofish-3way",
 397         .cra_priority           = 300,
 398         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 399         .cra_blocksize          = TF_BLOCK_SIZE,
 400         .cra_ctxsize            = sizeof(struct twofish_lrw_ctx),
 401         .cra_alignmask          = 0,
 402         .cra_type               = &crypto_blkcipher_type,
 403         .cra_module             = THIS_MODULE,
 404         .cra_exit               = lrw_twofish_exit_tfm,
 405         .cra_u = {
 406                 .blkcipher = {
 407                         .min_keysize    = TF_MIN_KEY_SIZE + TF_BLOCK_SIZE,
 408                         .max_keysize    = TF_MAX_KEY_SIZE + TF_BLOCK_SIZE,
 409                         .ivsize         = TF_BLOCK_SIZE,
 410                         .setkey         = lrw_twofish_setkey,
 411                         .encrypt        = lrw_encrypt,
 412                         .decrypt        = lrw_decrypt,
 413                 },
 414         },
 415 }, {
 416         .cra_name               = "xts(twofish)",
 417         .cra_driver_name        = "xts-twofish-3way",
 418         .cra_priority           = 300,
 419         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 420         .cra_blocksize          = TF_BLOCK_SIZE,
 421         .cra_ctxsize            = sizeof(struct twofish_xts_ctx),
 422         .cra_alignmask          = 0,
 423         .cra_type               = &crypto_blkcipher_type,
 424         .cra_module             = THIS_MODULE,
 425         .cra_u = {
 426                 .blkcipher = {
 427                         .min_keysize    = TF_MIN_KEY_SIZE * 2,
 428                         .max_keysize    = TF_MAX_KEY_SIZE * 2,
 429                         .ivsize         = TF_BLOCK_SIZE,
 430                         .setkey         = xts_twofish_setkey,
 431                         .encrypt        = xts_encrypt,
 432                         .decrypt        = xts_decrypt,
 433                 },
 434         },
 435 } };
 436
 437 static bool is_blacklisted_cpu(void)
 438 {
 439         if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
 440                 return false;
 441
 442         if (boot_cpu_data.x86 == 0x06 &&
 443                 (boot_cpu_data.x86_model == 0x1c ||
 444                  boot_cpu_data.x86_model == 0x26 ||
 445                  boot_cpu_data.x86_model == 0x36)) {
 446                 /*
 447                  * On Atom, twofish-3way is slower than original assembler
 448                  * implementation. Twofish-3way trades off some performance in
 449                  * storing blocks in 64bit registers to allow three blocks to
 450                  * be processed parallel. Parallel operation then allows gaining
 451                  * more performance than was trade off, on out-of-order CPUs.
 452                  * However Atom does not benefit from this parallellism and
 453                  * should be blacklisted.
 454                  */
 455                 return true;
 456         }
 457
 458         if (boot_cpu_data.x86 == 0x0f) {
 459                 /*
 460                  * On Pentium 4, twofish-3way is slower than original assembler
 461                  * implementation because excessive uses of 64bit rotate and
 462                  * left-shifts (which are really slow on P4) needed to store and
 463                  * handle 128bit block in two 64bit registers.
 464                  */
 465                 return true;
 466         }
 467
 468         return false;
 469 }
 470
 471 static int force;
 472 module_param(force, int, 0);
 473 MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist");
 474
 475 static int __init init(void)
 476 {
 477         if (!force && is_blacklisted_cpu()) {
 478                 printk(KERN_INFO
 479                         "twofish-x86_64-3way: performance on this CPU "
 480                         "would be suboptimal: disabling "
 481                         "twofish-x86_64-3way.\n");
 482                 return -ENODEV;
 483         }
 484
 485         return crypto_register_algs(tf_algs, ARRAY_SIZE(tf_algs));
 486 }
 487
 488 static void __exit fini(void)
 489 {
 490         crypto_unregister_algs(tf_algs, ARRAY_SIZE(tf_algs));
 491 }
 492
 493 module_init(init);
 494 module_exit(fini);
 495
 496 MODULE_LICENSE("GPL");
 497 MODULE_DESCRIPTION("Twofish Cipher Algorithm, 3-way parallel asm optimized");
 498 MODULE_ALIAS("twofish");
 499 MODULE_ALIAS("twofish-asm");