arch/sparc/crypto/des_glue.c

   1 /* Glue code for DES encryption optimized for sparc64 crypto opcodes.
   2  *
   3  * Copyright (C) 2012 David S. Miller <davem@davemloft.net>
   4  */
   5
   6 #define pr_fmt(fmt)     KBUILD_MODNAME ": " fmt
   7
   8 #include <linux/crypto.h>
   9 #include <linux/init.h>
  10 #include <linux/module.h>
  11 #include <linux/mm.h>
  12 #include <linux/types.h>
  13 #include <crypto/algapi.h>
  14 #include <crypto/des.h>
  15
  16 #include <asm/fpumacro.h>
  17 #include <asm/pstate.h>
  18 #include <asm/elf.h>
  19
  20 #include "opcodes.h"
  21
  22 struct des_sparc64_ctx {
  23         u64 encrypt_expkey[DES_EXPKEY_WORDS / 2];
  24         u64 decrypt_expkey[DES_EXPKEY_WORDS / 2];
  25 };
  26
  27 struct des3_ede_sparc64_ctx {
  28         u64 encrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
  29         u64 decrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
  30 };
  31
  32 static void encrypt_to_decrypt(u64 *d, const u64 *e)
  33 {
  34         const u64 *s = e + (DES_EXPKEY_WORDS / 2) - 1;
  35         int i;
  36
  37         for (i = 0; i < DES_EXPKEY_WORDS / 2; i++)
  38                 *d++ = *s--;
  39 }
  40
  41 extern void des_sparc64_key_expand(const u32 *input_key, u64 *key);
  42
  43 static int des_set_key(struct crypto_tfm *tfm, const u8 *key,
  44                        unsigned int keylen)
  45 {
  46         struct des_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
  47         u32 *flags = &tfm->crt_flags;
  48         u32 tmp[DES_EXPKEY_WORDS];
  49         int ret;
  50
  51         /* Even though we have special instructions for key expansion,
  52          * we call des_ekey() so that we don't have to write our own
  53          * weak key detection code.
  54          */
  55         ret = des_ekey(tmp, key);
  56         if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
  57                 *flags |= CRYPTO_TFM_RES_WEAK_KEY;
  58                 return -EINVAL;
  59         }
  60
  61         des_sparc64_key_expand((const u32 *) key, &dctx->encrypt_expkey[0]);
  62         encrypt_to_decrypt(&dctx->decrypt_expkey[0], &dctx->encrypt_expkey[0]);
  63
  64         return 0;
  65 }
  66
  67 extern void des_sparc64_crypt(const u64 *key, const u64 *input,
  68                               u64 *output);
  69
  70 static void des_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
  71 {
  72         struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
  73         const u64 *K = ctx->encrypt_expkey;
  74
  75         des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
  76 }
  77
  78 static void des_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
  79 {
  80         struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
  81         const u64 *K = ctx->decrypt_expkey;
  82
  83         des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
  84 }
  85
  86 extern void des_sparc64_load_keys(const u64 *key);
  87
  88 extern void des_sparc64_ecb_crypt(const u64 *input, u64 *output,
  89                                   unsigned int len);
  90
  91 #define DES_BLOCK_MASK  (~(DES_BLOCK_SIZE - 1))
  92
  93 static int __ecb_crypt(struct blkcipher_desc *desc,
  94                        struct scatterlist *dst, struct scatterlist *src,
  95                        unsigned int nbytes, bool encrypt)
  96 {
  97         struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
  98         struct blkcipher_walk walk;
  99         int err;
 100
 101         blkcipher_walk_init(&walk, dst, src, nbytes);
 102         err = blkcipher_walk_virt(desc, &walk);
 103         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 104
 105         if (encrypt)
 106                 des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
 107         else
 108                 des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
 109         while ((nbytes = walk.nbytes)) {
 110                 unsigned int block_len = nbytes & DES_BLOCK_MASK;
 111
 112                 if (likely(block_len)) {
 113                         des_sparc64_ecb_crypt((const u64 *)walk.src.virt.addr,
 114                                               (u64 *) walk.dst.virt.addr,
 115                                               block_len);
 116                 }
 117                 nbytes &= DES_BLOCK_SIZE - 1;
 118                 err = blkcipher_walk_done(desc, &walk, nbytes);
 119         }
 120         fprs_write(0);
 121         return err;
 122 }
 123
 124 static int ecb_encrypt(struct blkcipher_desc *desc,
 125                        struct scatterlist *dst, struct scatterlist *src,
 126                        unsigned int nbytes)
 127 {
 128         return __ecb_crypt(desc, dst, src, nbytes, true);
 129 }
 130
 131 static int ecb_decrypt(struct blkcipher_desc *desc,
 132                        struct scatterlist *dst, struct scatterlist *src,
 133                        unsigned int nbytes)
 134 {
 135         return __ecb_crypt(desc, dst, src, nbytes, false);
 136 }
 137
 138 extern void des_sparc64_cbc_encrypt(const u64 *input, u64 *output,
 139                                     unsigned int len, u64 *iv);
 140
 141 static int cbc_encrypt(struct blkcipher_desc *desc,
 142                        struct scatterlist *dst, struct scatterlist *src,
 143                        unsigned int nbytes)
 144 {
 145         struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 146         struct blkcipher_walk walk;
 147         int err;
 148
 149         blkcipher_walk_init(&walk, dst, src, nbytes);
 150         err = blkcipher_walk_virt(desc, &walk);
 151         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 152
 153         des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
 154         while ((nbytes = walk.nbytes)) {
 155                 unsigned int block_len = nbytes & DES_BLOCK_MASK;
 156
 157                 if (likely(block_len)) {
 158                         des_sparc64_cbc_encrypt((const u64 *)walk.src.virt.addr,
 159                                                 (u64 *) walk.dst.virt.addr,
 160                                                 block_len, (u64 *) walk.iv);
 161                 }
 162                 nbytes &= DES_BLOCK_SIZE - 1;
 163                 err = blkcipher_walk_done(desc, &walk, nbytes);
 164         }
 165         fprs_write(0);
 166         return err;
 167 }
 168
 169 extern void des_sparc64_cbc_decrypt(const u64 *input, u64 *output,
 170                                     unsigned int len, u64 *iv);
 171
 172 static int cbc_decrypt(struct blkcipher_desc *desc,
 173                        struct scatterlist *dst, struct scatterlist *src,
 174                        unsigned int nbytes)
 175 {
 176         struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 177         struct blkcipher_walk walk;
 178         int err;
 179
 180         blkcipher_walk_init(&walk, dst, src, nbytes);
 181         err = blkcipher_walk_virt(desc, &walk);
 182         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 183
 184         des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
 185         while ((nbytes = walk.nbytes)) {
 186                 unsigned int block_len = nbytes & DES_BLOCK_MASK;
 187
 188                 if (likely(block_len)) {
 189                         des_sparc64_cbc_decrypt((const u64 *)walk.src.virt.addr,
 190                                                 (u64 *) walk.dst.virt.addr,
 191                                                 block_len, (u64 *) walk.iv);
 192                 }
 193                 nbytes &= DES_BLOCK_SIZE - 1;
 194                 err = blkcipher_walk_done(desc, &walk, nbytes);
 195         }
 196         fprs_write(0);
 197         return err;
 198 }
 199
 200 static int des3_ede_set_key(struct crypto_tfm *tfm, const u8 *key,
 201                             unsigned int keylen)
 202 {
 203         struct des3_ede_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
 204         const u32 *K = (const u32 *)key;
 205         u32 *flags = &tfm->crt_flags;
 206         u64 k1[DES_EXPKEY_WORDS / 2];
 207         u64 k2[DES_EXPKEY_WORDS / 2];
 208         u64 k3[DES_EXPKEY_WORDS / 2];
 209
 210         if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) ||
 211                      !((K[2] ^ K[4]) | (K[3] ^ K[5]))) &&
 212                      (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
 213                 *flags |= CRYPTO_TFM_RES_WEAK_KEY;
 214                 return -EINVAL;
 215         }
 216
 217         des_sparc64_key_expand((const u32 *)key, k1);
 218         key += DES_KEY_SIZE;
 219         des_sparc64_key_expand((const u32 *)key, k2);
 220         key += DES_KEY_SIZE;
 221         des_sparc64_key_expand((const u32 *)key, k3);
 222
 223         memcpy(&dctx->encrypt_expkey[0], &k1[0], sizeof(k1));
 224         encrypt_to_decrypt(&dctx->encrypt_expkey[DES_EXPKEY_WORDS / 2], &k2[0]);
 225         memcpy(&dctx->encrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
 226                &k3[0], sizeof(k3));
 227
 228         encrypt_to_decrypt(&dctx->decrypt_expkey[0], &k3[0]);
 229         memcpy(&dctx->decrypt_expkey[DES_EXPKEY_WORDS / 2],
 230                &k2[0], sizeof(k2));
 231         encrypt_to_decrypt(&dctx->decrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
 232                            &k1[0]);
 233
 234         return 0;
 235 }
 236
 237 extern void des3_ede_sparc64_crypt(const u64 *key, const u64 *input,
 238                                    u64 *output);
 239
 240 static void des3_ede_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 241 {
 242         struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
 243         const u64 *K = ctx->encrypt_expkey;
 244
 245         des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
 246 }
 247
 248 static void des3_ede_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 249 {
 250         struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
 251         const u64 *K = ctx->decrypt_expkey;
 252
 253         des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
 254 }
 255
 256 extern void des3_ede_sparc64_load_keys(const u64 *key);
 257
 258 extern void des3_ede_sparc64_ecb_crypt(const u64 *expkey, const u64 *input,
 259                                        u64 *output, unsigned int len);
 260
 261 static int __ecb3_crypt(struct blkcipher_desc *desc,
 262                         struct scatterlist *dst, struct scatterlist *src,
 263                         unsigned int nbytes, bool encrypt)
 264 {
 265         struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 266         struct blkcipher_walk walk;
 267         const u64 *K;
 268         int err;
 269
 270         blkcipher_walk_init(&walk, dst, src, nbytes);
 271         err = blkcipher_walk_virt(desc, &walk);
 272         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 273
 274         if (encrypt)
 275                 K = &ctx->encrypt_expkey[0];
 276         else
 277                 K = &ctx->decrypt_expkey[0];
 278         des3_ede_sparc64_load_keys(K);
 279         while ((nbytes = walk.nbytes)) {
 280                 unsigned int block_len = nbytes & DES_BLOCK_MASK;
 281
 282                 if (likely(block_len)) {
 283                         const u64 *src64 = (const u64 *)walk.src.virt.addr;
 284                         des3_ede_sparc64_ecb_crypt(K, src64,
 285                                                    (u64 *) walk.dst.virt.addr,
 286                                                    block_len);
 287                 }
 288                 nbytes &= DES_BLOCK_SIZE - 1;
 289                 err = blkcipher_walk_done(desc, &walk, nbytes);
 290         }
 291         fprs_write(0);
 292         return err;
 293 }
 294
 295 static int ecb3_encrypt(struct blkcipher_desc *desc,
 296                        struct scatterlist *dst, struct scatterlist *src,
 297                        unsigned int nbytes)
 298 {
 299         return __ecb3_crypt(desc, dst, src, nbytes, true);
 300 }
 301
 302 static int ecb3_decrypt(struct blkcipher_desc *desc,
 303                        struct scatterlist *dst, struct scatterlist *src,
 304                        unsigned int nbytes)
 305 {
 306         return __ecb3_crypt(desc, dst, src, nbytes, false);
 307 }
 308
 309 extern void des3_ede_sparc64_cbc_encrypt(const u64 *expkey, const u64 *input,
 310                                          u64 *output, unsigned int len,
 311                                          u64 *iv);
 312
 313 static int cbc3_encrypt(struct blkcipher_desc *desc,
 314                         struct scatterlist *dst, struct scatterlist *src,
 315                         unsigned int nbytes)
 316 {
 317         struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 318         struct blkcipher_walk walk;
 319         const u64 *K;
 320         int err;
 321
 322         blkcipher_walk_init(&walk, dst, src, nbytes);
 323         err = blkcipher_walk_virt(desc, &walk);
 324         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 325
 326         K = &ctx->encrypt_expkey[0];
 327         des3_ede_sparc64_load_keys(K);
 328         while ((nbytes = walk.nbytes)) {
 329                 unsigned int block_len = nbytes & DES_BLOCK_MASK;
 330
 331                 if (likely(block_len)) {
 332                         const u64 *src64 = (const u64 *)walk.src.virt.addr;
 333                         des3_ede_sparc64_cbc_encrypt(K, src64,
 334                                                      (u64 *) walk.dst.virt.addr,
 335                                                      block_len,
 336                                                      (u64 *) walk.iv);
 337                 }
 338                 nbytes &= DES_BLOCK_SIZE - 1;
 339                 err = blkcipher_walk_done(desc, &walk, nbytes);
 340         }
 341         fprs_write(0);
 342         return err;
 343 }
 344
 345 extern void des3_ede_sparc64_cbc_decrypt(const u64 *expkey, const u64 *input,
 346                                          u64 *output, unsigned int len,
 347                                          u64 *iv);
 348
 349 static int cbc3_decrypt(struct blkcipher_desc *desc,
 350                         struct scatterlist *dst, struct scatterlist *src,
 351                         unsigned int nbytes)
 352 {
 353         struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 354         struct blkcipher_walk walk;
 355         const u64 *K;
 356         int err;
 357
 358         blkcipher_walk_init(&walk, dst, src, nbytes);
 359         err = blkcipher_walk_virt(desc, &walk);
 360         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 361
 362         K = &ctx->decrypt_expkey[0];
 363         des3_ede_sparc64_load_keys(K);
 364         while ((nbytes = walk.nbytes)) {
 365                 unsigned int block_len = nbytes & DES_BLOCK_MASK;
 366
 367                 if (likely(block_len)) {
 368                         const u64 *src64 = (const u64 *)walk.src.virt.addr;
 369                         des3_ede_sparc64_cbc_decrypt(K, src64,
 370                                                      (u64 *) walk.dst.virt.addr,
 371                                                      block_len,
 372                                                      (u64 *) walk.iv);
 373                 }
 374                 nbytes &= DES_BLOCK_SIZE - 1;
 375                 err = blkcipher_walk_done(desc, &walk, nbytes);
 376         }
 377         fprs_write(0);
 378         return err;
 379 }
 380
 381 static struct crypto_alg algs[] = { {
 382         .cra_name               = "des",
 383         .cra_driver_name        = "des-sparc64",
 384         .cra_priority           = SPARC_CR_OPCODE_PRIORITY,
 385         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
 386         .cra_blocksize          = DES_BLOCK_SIZE,
 387         .cra_ctxsize            = sizeof(struct des_sparc64_ctx),
 388         .cra_alignmask          = 7,
 389         .cra_module             = THIS_MODULE,
 390         .cra_u  = {
 391                 .cipher = {
 392                         .cia_min_keysize        = DES_KEY_SIZE,
 393                         .cia_max_keysize        = DES_KEY_SIZE,
 394                         .cia_setkey             = des_set_key,
 395                         .cia_encrypt            = des_encrypt,
 396                         .cia_decrypt            = des_decrypt
 397                 }
 398         }
 399 }, {
 400         .cra_name               = "ecb(des)",
 401         .cra_driver_name        = "ecb-des-sparc64",
 402         .cra_priority           = SPARC_CR_OPCODE_PRIORITY,
 403         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 404         .cra_blocksize          = DES_BLOCK_SIZE,
 405         .cra_ctxsize            = sizeof(struct des_sparc64_ctx),
 406         .cra_alignmask          = 7,
 407         .cra_type               = &crypto_blkcipher_type,
 408         .cra_module             = THIS_MODULE,
 409         .cra_u = {
 410                 .blkcipher = {
 411                         .min_keysize    = DES_KEY_SIZE,
 412                         .max_keysize    = DES_KEY_SIZE,
 413                         .setkey         = des_set_key,
 414                         .encrypt        = ecb_encrypt,
 415                         .decrypt        = ecb_decrypt,
 416                 },
 417         },
 418 }, {
 419         .cra_name               = "cbc(des)",
 420         .cra_driver_name        = "cbc-des-sparc64",
 421         .cra_priority           = SPARC_CR_OPCODE_PRIORITY,
 422         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 423         .cra_blocksize          = DES_BLOCK_SIZE,
 424         .cra_ctxsize            = sizeof(struct des_sparc64_ctx),
 425         .cra_alignmask          = 7,
 426         .cra_type               = &crypto_blkcipher_type,
 427         .cra_module             = THIS_MODULE,
 428         .cra_u = {
 429                 .blkcipher = {
 430                         .min_keysize    = DES_KEY_SIZE,
 431                         .max_keysize    = DES_KEY_SIZE,
 432                         .ivsize         = DES_BLOCK_SIZE,
 433                         .setkey         = des_set_key,
 434                         .encrypt        = cbc_encrypt,
 435                         .decrypt        = cbc_decrypt,
 436                 },
 437         },
 438 }, {
 439         .cra_name               = "des3_ede",
 440         .cra_driver_name        = "des3_ede-sparc64",
 441         .cra_priority           = SPARC_CR_OPCODE_PRIORITY,
 442         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
 443         .cra_blocksize          = DES3_EDE_BLOCK_SIZE,
 444         .cra_ctxsize            = sizeof(struct des3_ede_sparc64_ctx),
 445         .cra_alignmask          = 7,
 446         .cra_module             = THIS_MODULE,
 447         .cra_u  = {
 448                 .cipher = {
 449                         .cia_min_keysize        = DES3_EDE_KEY_SIZE,
 450                         .cia_max_keysize        = DES3_EDE_KEY_SIZE,
 451                         .cia_setkey             = des3_ede_set_key,
 452                         .cia_encrypt            = des3_ede_encrypt,
 453                         .cia_decrypt            = des3_ede_decrypt
 454                 }
 455         }
 456 }, {
 457         .cra_name               = "ecb(des3_ede)",
 458         .cra_driver_name        = "ecb-des3_ede-sparc64",
 459         .cra_priority           = SPARC_CR_OPCODE_PRIORITY,
 460         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 461         .cra_blocksize          = DES3_EDE_BLOCK_SIZE,
 462         .cra_ctxsize            = sizeof(struct des3_ede_sparc64_ctx),
 463         .cra_alignmask          = 7,
 464         .cra_type               = &crypto_blkcipher_type,
 465         .cra_module             = THIS_MODULE,
 466         .cra_u = {
 467                 .blkcipher = {
 468                         .min_keysize    = DES3_EDE_KEY_SIZE,
 469                         .max_keysize    = DES3_EDE_KEY_SIZE,
 470                         .setkey         = des3_ede_set_key,
 471                         .encrypt        = ecb3_encrypt,
 472                         .decrypt        = ecb3_decrypt,
 473                 },
 474         },
 475 }, {
 476         .cra_name               = "cbc(des3_ede)",
 477         .cra_driver_name        = "cbc-des3_ede-sparc64",
 478         .cra_priority           = SPARC_CR_OPCODE_PRIORITY,
 479         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 480         .cra_blocksize          = DES3_EDE_BLOCK_SIZE,
 481         .cra_ctxsize            = sizeof(struct des3_ede_sparc64_ctx),
 482         .cra_alignmask          = 7,
 483         .cra_type               = &crypto_blkcipher_type,
 484         .cra_module             = THIS_MODULE,
 485         .cra_u = {
 486                 .blkcipher = {
 487                         .min_keysize    = DES3_EDE_KEY_SIZE,
 488                         .max_keysize    = DES3_EDE_KEY_SIZE,
 489                         .ivsize         = DES3_EDE_BLOCK_SIZE,
 490                         .setkey         = des3_ede_set_key,
 491                         .encrypt        = cbc3_encrypt,
 492                         .decrypt        = cbc3_decrypt,
 493                 },
 494         },
 495 } };
 496
 497 static bool __init sparc64_has_des_opcode(void)
 498 {
 499         unsigned long cfr;
 500
 501         if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
 502                 return false;
 503
 504         __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
 505         if (!(cfr & CFR_DES))
 506                 return false;
 507
 508         return true;
 509 }
 510
 511 static int __init des_sparc64_mod_init(void)
 512 {
 513         int i;
 514
 515         for (i = 0; i < ARRAY_SIZE(algs); i++)
 516                 INIT_LIST_HEAD(&algs[i].cra_list);
 517
 518         if (sparc64_has_des_opcode()) {
 519                 pr_info("Using sparc64 des opcodes optimized DES implementation\n");
 520                 return crypto_register_algs(algs, ARRAY_SIZE(algs));
 521         }
 522         pr_info("sparc64 des opcodes not available.\n");
 523         return -ENODEV;
 524 }
 525
 526 static void __exit des_sparc64_mod_fini(void)
 527 {
 528         crypto_unregister_algs(algs, ARRAY_SIZE(algs));
 529 }
 530
 531 module_init(des_sparc64_mod_init);
 532 module_exit(des_sparc64_mod_fini);
 533
 534 MODULE_LICENSE("GPL");
 535 MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated");
 536
 537 MODULE_ALIAS_CRYPTO("des");
 538 MODULE_ALIAS_CRYPTO("des3_ede");
 539
 540 #include "crop_devid.c"