arch/x86/crypto/blowfish_glue.c

   1 /*
   2  * Glue Code for assembler optimized version of Blowfish
   3  *
   4  * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
   5  *
   6  * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
   7  *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
   8  * CTR part based on code (crypto/ctr.c) by:
   9  *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
  10  *
  11  * This program is free software; you can redistribute it and/or modify
  12  * it under the terms of the GNU General Public License as published by
  13  * the Free Software Foundation; either version 2 of the License, or
  14  * (at your option) any later version.
  15  *
  16  * This program is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19  * GNU General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU General Public License
  22  * along with this program; if not, write to the Free Software
  23  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
  24  * USA
  25  *
  26  */
  27
  28 #include <crypto/blowfish.h>
  29 #include <linux/crypto.h>
  30 #include <linux/init.h>
  31 #include <linux/module.h>
  32 #include <linux/types.h>
  33 #include <crypto/algapi.h>
  34
  35 /* regular block cipher functions */
  36 asmlinkage void __blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src,
  37                                    bool xor);
  38 asmlinkage void blowfish_dec_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src);
  39
  40 /* 4-way parallel cipher functions */
  41 asmlinkage void __blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst,
  42                                         const u8 *src, bool xor);
  43 asmlinkage void blowfish_dec_blk_4way(struct bf_ctx *ctx, u8 *dst,
  44                                       const u8 *src);
  45
  46 static inline void blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src)
  47 {
  48         __blowfish_enc_blk(ctx, dst, src, false);
  49 }
  50
  51 static inline void blowfish_enc_blk_xor(struct bf_ctx *ctx, u8 *dst,
  52                                         const u8 *src)
  53 {
  54         __blowfish_enc_blk(ctx, dst, src, true);
  55 }
  56
  57 static inline void blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst,
  58                                          const u8 *src)
  59 {
  60         __blowfish_enc_blk_4way(ctx, dst, src, false);
  61 }
  62
  63 static inline void blowfish_enc_blk_xor_4way(struct bf_ctx *ctx, u8 *dst,
  64                                       const u8 *src)
  65 {
  66         __blowfish_enc_blk_4way(ctx, dst, src, true);
  67 }
  68
  69 static void blowfish_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
  70 {
  71         blowfish_enc_blk(crypto_tfm_ctx(tfm), dst, src);
  72 }
  73
  74 static void blowfish_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
  75 {
  76         blowfish_dec_blk(crypto_tfm_ctx(tfm), dst, src);
  77 }
  78
  79 static struct crypto_alg bf_alg = {
  80         .cra_name               =       "blowfish",
  81         .cra_driver_name        =       "blowfish-asm",
  82         .cra_priority           =       200,
  83         .cra_flags              =       CRYPTO_ALG_TYPE_CIPHER,
  84         .cra_blocksize          =       BF_BLOCK_SIZE,
  85         .cra_ctxsize            =       sizeof(struct bf_ctx),
  86         .cra_alignmask          =       3,
  87         .cra_module             =       THIS_MODULE,
  88         .cra_list               =       LIST_HEAD_INIT(bf_alg.cra_list),
  89         .cra_u                  =       {
  90                 .cipher = {
  91                         .cia_min_keysize        =       BF_MIN_KEY_SIZE,
  92                         .cia_max_keysize        =       BF_MAX_KEY_SIZE,
  93                         .cia_setkey             =       blowfish_setkey,
  94                         .cia_encrypt            =       blowfish_encrypt,
  95                         .cia_decrypt            =       blowfish_decrypt,
  96                 }
  97         }
  98 };
  99
 100 static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
 101                      void (*fn)(struct bf_ctx *, u8 *, const u8 *),
 102                      void (*fn_4way)(struct bf_ctx *, u8 *, const u8 *))
 103 {
 104         struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 105         unsigned int bsize = BF_BLOCK_SIZE;
 106         unsigned int nbytes;
 107         int err;
 108
 109         err = blkcipher_walk_virt(desc, walk);
 110
 111         while ((nbytes = walk->nbytes)) {
 112                 u8 *wsrc = walk->src.virt.addr;
 113                 u8 *wdst = walk->dst.virt.addr;
 114
 115                 /* Process four block batch */
 116                 if (nbytes >= bsize * 4) {
 117                         do {
 118                                 fn_4way(ctx, wdst, wsrc);
 119
 120                                 wsrc += bsize * 4;
 121                                 wdst += bsize * 4;
 122                                 nbytes -= bsize * 4;
 123                         } while (nbytes >= bsize * 4);
 124
 125                         if (nbytes < bsize)
 126                                 goto done;
 127                 }
 128
 129                 /* Handle leftovers */
 130                 do {
 131                         fn(ctx, wdst, wsrc);
 132
 133                         wsrc += bsize;
 134                         wdst += bsize;
 135                         nbytes -= bsize;
 136                 } while (nbytes >= bsize);
 137
 138 done:
 139                 err = blkcipher_walk_done(desc, walk, nbytes);
 140         }
 141
 142         return err;
 143 }
 144
 145 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 146                        struct scatterlist *src, unsigned int nbytes)
 147 {
 148         struct blkcipher_walk walk;
 149
 150         blkcipher_walk_init(&walk, dst, src, nbytes);
 151         return ecb_crypt(desc, &walk, blowfish_enc_blk, blowfish_enc_blk_4way);
 152 }
 153
 154 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 155                        struct scatterlist *src, unsigned int nbytes)
 156 {
 157         struct blkcipher_walk walk;
 158
 159         blkcipher_walk_init(&walk, dst, src, nbytes);
 160         return ecb_crypt(desc, &walk, blowfish_dec_blk, blowfish_dec_blk_4way);
 161 }
 162
 163 static struct crypto_alg blk_ecb_alg = {
 164         .cra_name               = "ecb(blowfish)",
 165         .cra_driver_name        = "ecb-blowfish-asm",
 166         .cra_priority           = 300,
 167         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 168         .cra_blocksize          = BF_BLOCK_SIZE,
 169         .cra_ctxsize            = sizeof(struct bf_ctx),
 170         .cra_alignmask          = 0,
 171         .cra_type               = &crypto_blkcipher_type,
 172         .cra_module             = THIS_MODULE,
 173         .cra_list               = LIST_HEAD_INIT(blk_ecb_alg.cra_list),
 174         .cra_u = {
 175                 .blkcipher = {
 176                         .min_keysize    = BF_MIN_KEY_SIZE,
 177                         .max_keysize    = BF_MAX_KEY_SIZE,
 178                         .setkey         = blowfish_setkey,
 179                         .encrypt        = ecb_encrypt,
 180                         .decrypt        = ecb_decrypt,
 181                 },
 182         },
 183 };
 184
 185 static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,
 186                                   struct blkcipher_walk *walk)
 187 {
 188         struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 189         unsigned int bsize = BF_BLOCK_SIZE;
 190         unsigned int nbytes = walk->nbytes;
 191         u64 *src = (u64 *)walk->src.virt.addr;
 192         u64 *dst = (u64 *)walk->dst.virt.addr;
 193         u64 *iv = (u64 *)walk->iv;
 194
 195         do {
 196                 *dst = *src ^ *iv;
 197                 blowfish_enc_blk(ctx, (u8 *)dst, (u8 *)dst);
 198                 iv = dst;
 199
 200                 src += 1;
 201                 dst += 1;
 202                 nbytes -= bsize;
 203         } while (nbytes >= bsize);
 204
 205         *(u64 *)walk->iv = *iv;
 206         return nbytes;
 207 }
 208
 209 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 210                        struct scatterlist *src, unsigned int nbytes)
 211 {
 212         struct blkcipher_walk walk;
 213         int err;
 214
 215         blkcipher_walk_init(&walk, dst, src, nbytes);
 216         err = blkcipher_walk_virt(desc, &walk);
 217
 218         while ((nbytes = walk.nbytes)) {
 219                 nbytes = __cbc_encrypt(desc, &walk);
 220                 err = blkcipher_walk_done(desc, &walk, nbytes);
 221         }
 222
 223         return err;
 224 }
 225
 226 static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
 227                                   struct blkcipher_walk *walk)
 228 {
 229         struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 230         unsigned int bsize = BF_BLOCK_SIZE;
 231         unsigned int nbytes = walk->nbytes;
 232         u64 *src = (u64 *)walk->src.virt.addr;
 233         u64 *dst = (u64 *)walk->dst.virt.addr;
 234         u64 ivs[4 - 1];
 235         u64 last_iv;
 236
 237         /* Start of the last block. */
 238         src += nbytes / bsize - 1;
 239         dst += nbytes / bsize - 1;
 240
 241         last_iv = *src;
 242
 243         /* Process four block batch */
 244         if (nbytes >= bsize * 4) {
 245                 do {
 246                         nbytes -= bsize * 4 - bsize;
 247                         src -= 4 - 1;
 248                         dst -= 4 - 1;
 249
 250                         ivs[0] = src[0];
 251                         ivs[1] = src[1];
 252                         ivs[2] = src[2];
 253
 254                         blowfish_dec_blk_4way(ctx, (u8 *)dst, (u8 *)src);
 255
 256                         dst[1] ^= ivs[0];
 257                         dst[2] ^= ivs[1];
 258                         dst[3] ^= ivs[2];
 259
 260                         nbytes -= bsize;
 261                         if (nbytes < bsize)
 262                                 goto done;
 263
 264                         *dst ^= *(src - 1);
 265                         src -= 1;
 266                         dst -= 1;
 267                 } while (nbytes >= bsize * 4);
 268
 269                 if (nbytes < bsize)
 270                         goto done;
 271         }
 272
 273         /* Handle leftovers */
 274         for (;;) {
 275                 blowfish_dec_blk(ctx, (u8 *)dst, (u8 *)src);
 276
 277                 nbytes -= bsize;
 278                 if (nbytes < bsize)
 279                         break;
 280
 281                 *dst ^= *(src - 1);
 282                 src -= 1;
 283                 dst -= 1;
 284         }
 285
 286 done:
 287         *dst ^= *(u64 *)walk->iv;
 288         *(u64 *)walk->iv = last_iv;
 289
 290         return nbytes;
 291 }
 292
 293 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 294                        struct scatterlist *src, unsigned int nbytes)
 295 {
 296         struct blkcipher_walk walk;
 297         int err;
 298
 299         blkcipher_walk_init(&walk, dst, src, nbytes);
 300         err = blkcipher_walk_virt(desc, &walk);
 301
 302         while ((nbytes = walk.nbytes)) {
 303                 nbytes = __cbc_decrypt(desc, &walk);
 304                 err = blkcipher_walk_done(desc, &walk, nbytes);
 305         }
 306
 307         return err;
 308 }
 309
 310 static struct crypto_alg blk_cbc_alg = {
 311         .cra_name               = "cbc(blowfish)",
 312         .cra_driver_name        = "cbc-blowfish-asm",
 313         .cra_priority           = 300,
 314         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 315         .cra_blocksize          = BF_BLOCK_SIZE,
 316         .cra_ctxsize            = sizeof(struct bf_ctx),
 317         .cra_alignmask          = 0,
 318         .cra_type               = &crypto_blkcipher_type,
 319         .cra_module             = THIS_MODULE,
 320         .cra_list               = LIST_HEAD_INIT(blk_cbc_alg.cra_list),
 321         .cra_u = {
 322                 .blkcipher = {
 323                         .min_keysize    = BF_MIN_KEY_SIZE,
 324                         .max_keysize    = BF_MAX_KEY_SIZE,
 325                         .ivsize         = BF_BLOCK_SIZE,
 326                         .setkey         = blowfish_setkey,
 327                         .encrypt        = cbc_encrypt,
 328                         .decrypt        = cbc_decrypt,
 329                 },
 330         },
 331 };
 332
 333 static void ctr_crypt_final(struct bf_ctx *ctx, struct blkcipher_walk *walk)
 334 {
 335         u8 *ctrblk = walk->iv;
 336         u8 keystream[BF_BLOCK_SIZE];
 337         u8 *src = walk->src.virt.addr;
 338         u8 *dst = walk->dst.virt.addr;
 339         unsigned int nbytes = walk->nbytes;
 340
 341         blowfish_enc_blk(ctx, keystream, ctrblk);
 342         crypto_xor(keystream, src, nbytes);
 343         memcpy(dst, keystream, nbytes);
 344
 345         crypto_inc(ctrblk, BF_BLOCK_SIZE);
 346 }
 347
 348 static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
 349                                 struct blkcipher_walk *walk)
 350 {
 351         struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 352         unsigned int bsize = BF_BLOCK_SIZE;
 353         unsigned int nbytes = walk->nbytes;
 354         u64 *src = (u64 *)walk->src.virt.addr;
 355         u64 *dst = (u64 *)walk->dst.virt.addr;
 356         u64 ctrblk = be64_to_cpu(*(__be64 *)walk->iv);
 357         __be64 ctrblocks[4];
 358
 359         /* Process four block batch */
 360         if (nbytes >= bsize * 4) {
 361                 do {
 362                         if (dst != src) {
 363                                 dst[0] = src[0];
 364                                 dst[1] = src[1];
 365                                 dst[2] = src[2];
 366                                 dst[3] = src[3];
 367                         }
 368
 369                         /* create ctrblks for parallel encrypt */
 370                         ctrblocks[0] = cpu_to_be64(ctrblk++);
 371                         ctrblocks[1] = cpu_to_be64(ctrblk++);
 372                         ctrblocks[2] = cpu_to_be64(ctrblk++);
 373                         ctrblocks[3] = cpu_to_be64(ctrblk++);
 374
 375                         blowfish_enc_blk_xor_4way(ctx, (u8 *)dst,
 376                                                   (u8 *)ctrblocks);
 377
 378                         src += 4;
 379                         dst += 4;
 380                 } while ((nbytes -= bsize * 4) >= bsize * 4);
 381
 382                 if (nbytes < bsize)
 383                         goto done;
 384         }
 385
 386         /* Handle leftovers */
 387         do {
 388                 if (dst != src)
 389                         *dst = *src;
 390
 391                 ctrblocks[0] = cpu_to_be64(ctrblk++);
 392
 393                 blowfish_enc_blk_xor(ctx, (u8 *)dst, (u8 *)ctrblocks);
 394
 395                 src += 1;
 396                 dst += 1;
 397         } while ((nbytes -= bsize) >= bsize);
 398
 399 done:
 400         *(__be64 *)walk->iv = cpu_to_be64(ctrblk);
 401         return nbytes;
 402 }
 403
 404 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 405                      struct scatterlist *src, unsigned int nbytes)
 406 {
 407         struct blkcipher_walk walk;
 408         int err;
 409
 410         blkcipher_walk_init(&walk, dst, src, nbytes);
 411         err = blkcipher_walk_virt_block(desc, &walk, BF_BLOCK_SIZE);
 412
 413         while ((nbytes = walk.nbytes) >= BF_BLOCK_SIZE) {
 414                 nbytes = __ctr_crypt(desc, &walk);
 415                 err = blkcipher_walk_done(desc, &walk, nbytes);
 416         }
 417
 418         if (walk.nbytes) {
 419                 ctr_crypt_final(crypto_blkcipher_ctx(desc->tfm), &walk);
 420                 err = blkcipher_walk_done(desc, &walk, 0);
 421         }
 422
 423         return err;
 424 }
 425
 426 static struct crypto_alg blk_ctr_alg = {
 427         .cra_name               = "ctr(blowfish)",
 428         .cra_driver_name        = "ctr-blowfish-asm",
 429         .cra_priority           = 300,
 430         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 431         .cra_blocksize          = 1,
 432         .cra_ctxsize            = sizeof(struct bf_ctx),
 433         .cra_alignmask          = 0,
 434         .cra_type               = &crypto_blkcipher_type,
 435         .cra_module             = THIS_MODULE,
 436         .cra_list               = LIST_HEAD_INIT(blk_ctr_alg.cra_list),
 437         .cra_u = {
 438                 .blkcipher = {
 439                         .min_keysize    = BF_MIN_KEY_SIZE,
 440                         .max_keysize    = BF_MAX_KEY_SIZE,
 441                         .ivsize         = BF_BLOCK_SIZE,
 442                         .setkey         = blowfish_setkey,
 443                         .encrypt        = ctr_crypt,
 444                         .decrypt        = ctr_crypt,
 445                 },
 446         },
 447 };
 448
 449 static int __init init(void)
 450 {
 451         int err;
 452
 453         err = crypto_register_alg(&bf_alg);
 454         if (err)
 455                 goto bf_err;
 456         err = crypto_register_alg(&blk_ecb_alg);
 457         if (err)
 458                 goto ecb_err;
 459         err = crypto_register_alg(&blk_cbc_alg);
 460         if (err)
 461                 goto cbc_err;
 462         err = crypto_register_alg(&blk_ctr_alg);
 463         if (err)
 464                 goto ctr_err;
 465
 466         return 0;
 467
 468 ctr_err:
 469         crypto_unregister_alg(&blk_cbc_alg);
 470 cbc_err:
 471         crypto_unregister_alg(&blk_ecb_alg);
 472 ecb_err:
 473         crypto_unregister_alg(&bf_alg);
 474 bf_err:
 475         return err;
 476 }
 477
 478 static void __exit fini(void)
 479 {
 480         crypto_unregister_alg(&blk_ctr_alg);
 481         crypto_unregister_alg(&blk_cbc_alg);
 482         crypto_unregister_alg(&blk_ecb_alg);
 483         crypto_unregister_alg(&bf_alg);
 484 }
 485
 486 module_init(init);
 487 module_exit(fini);
 488
 489 MODULE_LICENSE("GPL");
 490 MODULE_DESCRIPTION("Blowfish Cipher Algorithm, asm optimized");
 491 MODULE_ALIAS("blowfish");
 492 MODULE_ALIAS("blowfish-asm");