arch/sparc/crypto/aes_glue.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Glue code for AES encryption optimized for sparc64 crypto opcodes.
   3  *
   4  * This is based largely upon arch/x86/crypto/aesni-intel_glue.c
   5  *
   6  * Copyright (C) 2008, Intel Corp.
   7  *    Author: Huang Ying <ying.huang@intel.com>
   8  *
   9  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
  10  * interface for 64-bit kernels.
  11  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
  12  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
  13  *             Tadeusz Struk (tadeusz.struk@intel.com)
  14  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
  15  *    Copyright (c) 2010, Intel Corporation.
  16  */
  17
  18 #define pr_fmt(fmt)     KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/crypto.h>
  21 #include <linux/init.h>
  22 #include <linux/module.h>
  23 #include <linux/mm.h>
  24 #include <linux/types.h>
  25 #include <crypto/algapi.h>
  26 #include <crypto/aes.h>
  27 #include <crypto/internal/skcipher.h>
  28
  29 #include <asm/fpumacro.h>
  30 #include <asm/pstate.h>
  31 #include <asm/elf.h>
  32
  33 #include "opcodes.h"
  34
  35 struct aes_ops {
  36         void (*encrypt)(const u64 *key, const u32 *input, u32 *output);
  37         void (*decrypt)(const u64 *key, const u32 *input, u32 *output);
  38         void (*load_encrypt_keys)(const u64 *key);
  39         void (*load_decrypt_keys)(const u64 *key);
  40         void (*ecb_encrypt)(const u64 *key, const u64 *input, u64 *output,
  41                             unsigned int len);
  42         void (*ecb_decrypt)(const u64 *key, const u64 *input, u64 *output,
  43                             unsigned int len);
  44         void (*cbc_encrypt)(const u64 *key, const u64 *input, u64 *output,
  45                             unsigned int len, u64 *iv);
  46         void (*cbc_decrypt)(const u64 *key, const u64 *input, u64 *output,
  47                             unsigned int len, u64 *iv);
  48         void (*ctr_crypt)(const u64 *key, const u64 *input, u64 *output,
  49                           unsigned int len, u64 *iv);
  50 };
  51
  52 struct crypto_sparc64_aes_ctx {
  53         struct aes_ops *ops;
  54         u64 key[AES_MAX_KEYLENGTH / sizeof(u64)];
  55         u32 key_length;
  56         u32 expanded_key_length;
  57 };
  58
  59 extern void aes_sparc64_encrypt_128(const u64 *key, const u32 *input,
  60                                     u32 *output);
  61 extern void aes_sparc64_encrypt_192(const u64 *key, const u32 *input,
  62                                     u32 *output);
  63 extern void aes_sparc64_encrypt_256(const u64 *key, const u32 *input,
  64                                     u32 *output);
  65
  66 extern void aes_sparc64_decrypt_128(const u64 *key, const u32 *input,
  67                                     u32 *output);
  68 extern void aes_sparc64_decrypt_192(const u64 *key, const u32 *input,
  69                                     u32 *output);
  70 extern void aes_sparc64_decrypt_256(const u64 *key, const u32 *input,
  71                                     u32 *output);
  72
  73 extern void aes_sparc64_load_encrypt_keys_128(const u64 *key);
  74 extern void aes_sparc64_load_encrypt_keys_192(const u64 *key);
  75 extern void aes_sparc64_load_encrypt_keys_256(const u64 *key);
  76
  77 extern void aes_sparc64_load_decrypt_keys_128(const u64 *key);
  78 extern void aes_sparc64_load_decrypt_keys_192(const u64 *key);
  79 extern void aes_sparc64_load_decrypt_keys_256(const u64 *key);
  80
  81 extern void aes_sparc64_ecb_encrypt_128(const u64 *key, const u64 *input,
  82                                         u64 *output, unsigned int len);
  83 extern void aes_sparc64_ecb_encrypt_192(const u64 *key, const u64 *input,
  84                                         u64 *output, unsigned int len);
  85 extern void aes_sparc64_ecb_encrypt_256(const u64 *key, const u64 *input,
  86                                         u64 *output, unsigned int len);
  87
  88 extern void aes_sparc64_ecb_decrypt_128(const u64 *key, const u64 *input,
  89                                         u64 *output, unsigned int len);
  90 extern void aes_sparc64_ecb_decrypt_192(const u64 *key, const u64 *input,
  91                                         u64 *output, unsigned int len);
  92 extern void aes_sparc64_ecb_decrypt_256(const u64 *key, const u64 *input,
  93                                         u64 *output, unsigned int len);
  94
  95 extern void aes_sparc64_cbc_encrypt_128(const u64 *key, const u64 *input,
  96                                         u64 *output, unsigned int len,
  97                                         u64 *iv);
  98
  99 extern void aes_sparc64_cbc_encrypt_192(const u64 *key, const u64 *input,
 100                                         u64 *output, unsigned int len,
 101                                         u64 *iv);
 102
 103 extern void aes_sparc64_cbc_encrypt_256(const u64 *key, const u64 *input,
 104                                         u64 *output, unsigned int len,
 105                                         u64 *iv);
 106
 107 extern void aes_sparc64_cbc_decrypt_128(const u64 *key, const u64 *input,
 108                                         u64 *output, unsigned int len,
 109                                         u64 *iv);
 110
 111 extern void aes_sparc64_cbc_decrypt_192(const u64 *key, const u64 *input,
 112                                         u64 *output, unsigned int len,
 113                                         u64 *iv);
 114
 115 extern void aes_sparc64_cbc_decrypt_256(const u64 *key, const u64 *input,
 116                                         u64 *output, unsigned int len,
 117                                         u64 *iv);
 118
 119 extern void aes_sparc64_ctr_crypt_128(const u64 *key, const u64 *input,
 120                                       u64 *output, unsigned int len,
 121                                       u64 *iv);
 122 extern void aes_sparc64_ctr_crypt_192(const u64 *key, const u64 *input,
 123                                       u64 *output, unsigned int len,
 124                                       u64 *iv);
 125 extern void aes_sparc64_ctr_crypt_256(const u64 *key, const u64 *input,
 126                                       u64 *output, unsigned int len,
 127                                       u64 *iv);
 128
 129 static struct aes_ops aes128_ops = {
 130         .encrypt                = aes_sparc64_encrypt_128,
 131         .decrypt                = aes_sparc64_decrypt_128,
 132         .load_encrypt_keys      = aes_sparc64_load_encrypt_keys_128,
 133         .load_decrypt_keys      = aes_sparc64_load_decrypt_keys_128,
 134         .ecb_encrypt            = aes_sparc64_ecb_encrypt_128,
 135         .ecb_decrypt            = aes_sparc64_ecb_decrypt_128,
 136         .cbc_encrypt            = aes_sparc64_cbc_encrypt_128,
 137         .cbc_decrypt            = aes_sparc64_cbc_decrypt_128,
 138         .ctr_crypt              = aes_sparc64_ctr_crypt_128,
 139 };
 140
 141 static struct aes_ops aes192_ops = {
 142         .encrypt                = aes_sparc64_encrypt_192,
 143         .decrypt                = aes_sparc64_decrypt_192,
 144         .load_encrypt_keys      = aes_sparc64_load_encrypt_keys_192,
 145         .load_decrypt_keys      = aes_sparc64_load_decrypt_keys_192,
 146         .ecb_encrypt            = aes_sparc64_ecb_encrypt_192,
 147         .ecb_decrypt            = aes_sparc64_ecb_decrypt_192,
 148         .cbc_encrypt            = aes_sparc64_cbc_encrypt_192,
 149         .cbc_decrypt            = aes_sparc64_cbc_decrypt_192,
 150         .ctr_crypt              = aes_sparc64_ctr_crypt_192,
 151 };
 152
 153 static struct aes_ops aes256_ops = {
 154         .encrypt                = aes_sparc64_encrypt_256,
 155         .decrypt                = aes_sparc64_decrypt_256,
 156         .load_encrypt_keys      = aes_sparc64_load_encrypt_keys_256,
 157         .load_decrypt_keys      = aes_sparc64_load_decrypt_keys_256,
 158         .ecb_encrypt            = aes_sparc64_ecb_encrypt_256,
 159         .ecb_decrypt            = aes_sparc64_ecb_decrypt_256,
 160         .cbc_encrypt            = aes_sparc64_cbc_encrypt_256,
 161         .cbc_decrypt            = aes_sparc64_cbc_decrypt_256,
 162         .ctr_crypt              = aes_sparc64_ctr_crypt_256,
 163 };
 164
 165 extern void aes_sparc64_key_expand(const u32 *in_key, u64 *output_key,
 166                                    unsigned int key_len);
 167
 168 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 169                        unsigned int key_len)
 170 {
 171         struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 172
 173         switch (key_len) {
 174         case AES_KEYSIZE_128:
 175                 ctx->expanded_key_length = 0xb0;
 176                 ctx->ops = &aes128_ops;
 177                 break;
 178
 179         case AES_KEYSIZE_192:
 180                 ctx->expanded_key_length = 0xd0;
 181                 ctx->ops = &aes192_ops;
 182                 break;
 183
 184         case AES_KEYSIZE_256:
 185                 ctx->expanded_key_length = 0xf0;
 186                 ctx->ops = &aes256_ops;
 187                 break;
 188
 189         default:
 190                 return -EINVAL;
 191         }
 192
 193         aes_sparc64_key_expand((const u32 *)in_key, &ctx->key[0], key_len);
 194         ctx->key_length = key_len;
 195
 196         return 0;
 197 }
 198
 199 static int aes_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key,
 200                                 unsigned int key_len)
 201 {
 202         return aes_set_key(crypto_skcipher_tfm(tfm), in_key, key_len);
 203 }
 204
 205 static void crypto_aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 206 {
 207         struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 208
 209         ctx->ops->encrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst);
 210 }
 211
 212 static void crypto_aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 213 {
 214         struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 215
 216         ctx->ops->decrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst);
 217 }
 218
 219 static int ecb_encrypt(struct skcipher_request *req)
 220 {
 221         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 222         const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 223         struct skcipher_walk walk;
 224         unsigned int nbytes;
 225         int err;
 226
 227         err = skcipher_walk_virt(&walk, req, true);
 228         if (err)
 229                 return err;
 230
 231         ctx->ops->load_encrypt_keys(&ctx->key[0]);
 232         while ((nbytes = walk.nbytes) != 0) {
 233                 ctx->ops->ecb_encrypt(&ctx->key[0], walk.src.virt.addr,
 234                                       walk.dst.virt.addr,
 235                                       round_down(nbytes, AES_BLOCK_SIZE));
 236                 err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
 237         }
 238         fprs_write(0);
 239         return err;
 240 }
 241
 242 static int ecb_decrypt(struct skcipher_request *req)
 243 {
 244         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 245         const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 246         const u64 *key_end;
 247         struct skcipher_walk walk;
 248         unsigned int nbytes;
 249         int err;
 250
 251         err = skcipher_walk_virt(&walk, req, true);
 252         if (err)
 253                 return err;
 254
 255         ctx->ops->load_decrypt_keys(&ctx->key[0]);
 256         key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];
 257         while ((nbytes = walk.nbytes) != 0) {
 258                 ctx->ops->ecb_decrypt(key_end, walk.src.virt.addr,
 259                                       walk.dst.virt.addr,
 260                                       round_down(nbytes, AES_BLOCK_SIZE));
 261                 err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
 262         }
 263         fprs_write(0);
 264
 265         return err;
 266 }
 267
 268 static int cbc_encrypt(struct skcipher_request *req)
 269 {
 270         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 271         const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 272         struct skcipher_walk walk;
 273         unsigned int nbytes;
 274         int err;
 275
 276         err = skcipher_walk_virt(&walk, req, true);
 277         if (err)
 278                 return err;
 279
 280         ctx->ops->load_encrypt_keys(&ctx->key[0]);
 281         while ((nbytes = walk.nbytes) != 0) {
 282                 ctx->ops->cbc_encrypt(&ctx->key[0], walk.src.virt.addr,
 283                                       walk.dst.virt.addr,
 284                                       round_down(nbytes, AES_BLOCK_SIZE),
 285                                       walk.iv);
 286                 err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
 287         }
 288         fprs_write(0);
 289         return err;
 290 }
 291
 292 static int cbc_decrypt(struct skcipher_request *req)
 293 {
 294         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 295         const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 296         const u64 *key_end;
 297         struct skcipher_walk walk;
 298         unsigned int nbytes;
 299         int err;
 300
 301         err = skcipher_walk_virt(&walk, req, true);
 302         if (err)
 303                 return err;
 304
 305         ctx->ops->load_decrypt_keys(&ctx->key[0]);
 306         key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];
 307         while ((nbytes = walk.nbytes) != 0) {
 308                 ctx->ops->cbc_decrypt(key_end, walk.src.virt.addr,
 309                                       walk.dst.virt.addr,
 310                                       round_down(nbytes, AES_BLOCK_SIZE),
 311                                       walk.iv);
 312                 err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
 313         }
 314         fprs_write(0);
 315
 316         return err;
 317 }
 318
 319 static void ctr_crypt_final(const struct crypto_sparc64_aes_ctx *ctx,
 320                             struct skcipher_walk *walk)
 321 {
 322         u8 *ctrblk = walk->iv;
 323         u64 keystream[AES_BLOCK_SIZE / sizeof(u64)];
 324         u8 *src = walk->src.virt.addr;
 325         u8 *dst = walk->dst.virt.addr;
 326         unsigned int nbytes = walk->nbytes;
 327
 328         ctx->ops->ecb_encrypt(&ctx->key[0], (const u64 *)ctrblk,
 329                               keystream, AES_BLOCK_SIZE);
 330         crypto_xor_cpy(dst, (u8 *) keystream, src, nbytes);
 331         crypto_inc(ctrblk, AES_BLOCK_SIZE);
 332 }
 333
 334 static int ctr_crypt(struct skcipher_request *req)
 335 {
 336         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 337         const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 338         struct skcipher_walk walk;
 339         unsigned int nbytes;
 340         int err;
 341
 342         err = skcipher_walk_virt(&walk, req, true);
 343         if (err)
 344                 return err;
 345
 346         ctx->ops->load_encrypt_keys(&ctx->key[0]);
 347         while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
 348                 ctx->ops->ctr_crypt(&ctx->key[0], walk.src.virt.addr,
 349                                     walk.dst.virt.addr,
 350                                     round_down(nbytes, AES_BLOCK_SIZE),
 351                                     walk.iv);
 352                 err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
 353         }
 354         if (walk.nbytes) {
 355                 ctr_crypt_final(ctx, &walk);
 356                 err = skcipher_walk_done(&walk, 0);
 357         }
 358         fprs_write(0);
 359         return err;
 360 }
 361
 362 static struct crypto_alg cipher_alg = {
 363         .cra_name               = "aes",
 364         .cra_driver_name        = "aes-sparc64",
 365         .cra_priority           = SPARC_CR_OPCODE_PRIORITY,
 366         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
 367         .cra_blocksize          = AES_BLOCK_SIZE,
 368         .cra_ctxsize            = sizeof(struct crypto_sparc64_aes_ctx),
 369         .cra_alignmask          = 3,
 370         .cra_module             = THIS_MODULE,
 371         .cra_u  = {
 372                 .cipher = {
 373                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
 374                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
 375                         .cia_setkey             = aes_set_key,
 376                         .cia_encrypt            = crypto_aes_encrypt,
 377                         .cia_decrypt            = crypto_aes_decrypt
 378                 }
 379         }
 380 };
 381
 382 static struct skcipher_alg skcipher_algs[] = {
 383         {
 384                 .base.cra_name          = "ecb(aes)",
 385                 .base.cra_driver_name   = "ecb-aes-sparc64",
 386                 .base.cra_priority      = SPARC_CR_OPCODE_PRIORITY,
 387                 .base.cra_blocksize     = AES_BLOCK_SIZE,
 388                 .base.cra_ctxsize       = sizeof(struct crypto_sparc64_aes_ctx),
 389                 .base.cra_alignmask     = 7,
 390                 .base.cra_module        = THIS_MODULE,
 391                 .min_keysize            = AES_MIN_KEY_SIZE,
 392                 .max_keysize            = AES_MAX_KEY_SIZE,
 393                 .setkey                 = aes_set_key_skcipher,
 394                 .encrypt                = ecb_encrypt,
 395                 .decrypt                = ecb_decrypt,
 396         }, {
 397                 .base.cra_name          = "cbc(aes)",
 398                 .base.cra_driver_name   = "cbc-aes-sparc64",
 399                 .base.cra_priority      = SPARC_CR_OPCODE_PRIORITY,
 400                 .base.cra_blocksize     = AES_BLOCK_SIZE,
 401                 .base.cra_ctxsize       = sizeof(struct crypto_sparc64_aes_ctx),
 402                 .base.cra_alignmask     = 7,
 403                 .base.cra_module        = THIS_MODULE,
 404                 .min_keysize            = AES_MIN_KEY_SIZE,
 405                 .max_keysize            = AES_MAX_KEY_SIZE,
 406                 .ivsize                 = AES_BLOCK_SIZE,
 407                 .setkey                 = aes_set_key_skcipher,
 408                 .encrypt                = cbc_encrypt,
 409                 .decrypt                = cbc_decrypt,
 410         }, {
 411                 .base.cra_name          = "ctr(aes)",
 412                 .base.cra_driver_name   = "ctr-aes-sparc64",
 413                 .base.cra_priority      = SPARC_CR_OPCODE_PRIORITY,
 414                 .base.cra_blocksize     = 1,
 415                 .base.cra_ctxsize       = sizeof(struct crypto_sparc64_aes_ctx),
 416                 .base.cra_alignmask     = 7,
 417                 .base.cra_module        = THIS_MODULE,
 418                 .min_keysize            = AES_MIN_KEY_SIZE,
 419                 .max_keysize            = AES_MAX_KEY_SIZE,
 420                 .ivsize                 = AES_BLOCK_SIZE,
 421                 .setkey                 = aes_set_key_skcipher,
 422                 .encrypt                = ctr_crypt,
 423                 .decrypt                = ctr_crypt,
 424                 .chunksize              = AES_BLOCK_SIZE,
 425         }
 426 };
 427
 428 static bool __init sparc64_has_aes_opcode(void)
 429 {
 430         unsigned long cfr;
 431
 432         if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
 433                 return false;
 434
 435         __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
 436         if (!(cfr & CFR_AES))
 437                 return false;
 438
 439         return true;
 440 }
 441
 442 static int __init aes_sparc64_mod_init(void)
 443 {
 444         int err;
 445
 446         if (!sparc64_has_aes_opcode()) {
 447                 pr_info("sparc64 aes opcodes not available.\n");
 448                 return -ENODEV;
 449         }
 450         pr_info("Using sparc64 aes opcodes optimized AES implementation\n");
 451         err = crypto_register_alg(&cipher_alg);
 452         if (err)
 453                 return err;
 454         err = crypto_register_skciphers(skcipher_algs,
 455                                         ARRAY_SIZE(skcipher_algs));
 456         if (err)
 457                 crypto_unregister_alg(&cipher_alg);
 458         return err;
 459 }
 460
 461 static void __exit aes_sparc64_mod_fini(void)
 462 {
 463         crypto_unregister_alg(&cipher_alg);
 464         crypto_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs));
 465 }
 466
 467 module_init(aes_sparc64_mod_init);
 468 module_exit(aes_sparc64_mod_fini);
 469
 470 MODULE_LICENSE("GPL");
 471 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, sparc64 aes opcode accelerated");
 472
 473 MODULE_ALIAS_CRYPTO("aes");
 474
 475 #include "crop_devid.c"