arch/arm64/crypto/aes-glue.c

   1 /*
   2  * linux/arch/arm64/crypto/aes-glue.c - wrapper code for ARMv8 AES
   3  *
   4  * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10
  11 #include <asm/neon.h>
  12 #include <asm/hwcap.h>
  13 #include <crypto/aes.h>
  14 #include <crypto/ablk_helper.h>
  15 #include <crypto/algapi.h>
  16 #include <linux/module.h>
  17 #include <linux/cpufeature.h>
  18
  19 #ifdef USE_V8_CRYPTO_EXTENSIONS
  20 #define MODE                    "ce"
  21 #define PRIO                    300
  22 #define aes_ecb_encrypt         ce_aes_ecb_encrypt
  23 #define aes_ecb_decrypt         ce_aes_ecb_decrypt
  24 #define aes_cbc_encrypt         ce_aes_cbc_encrypt
  25 #define aes_cbc_decrypt         ce_aes_cbc_decrypt
  26 #define aes_ctr_encrypt         ce_aes_ctr_encrypt
  27 #define aes_xts_encrypt         ce_aes_xts_encrypt
  28 #define aes_xts_decrypt         ce_aes_xts_decrypt
  29 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
  30 #else
  31 #define MODE                    "neon"
  32 #define PRIO                    200
  33 #define aes_ecb_encrypt         neon_aes_ecb_encrypt
  34 #define aes_ecb_decrypt         neon_aes_ecb_decrypt
  35 #define aes_cbc_encrypt         neon_aes_cbc_encrypt
  36 #define aes_cbc_decrypt         neon_aes_cbc_decrypt
  37 #define aes_ctr_encrypt         neon_aes_ctr_encrypt
  38 #define aes_xts_encrypt         neon_aes_xts_encrypt
  39 #define aes_xts_decrypt         neon_aes_xts_decrypt
  40 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 NEON");
  41 MODULE_ALIAS("ecb(aes)");
  42 MODULE_ALIAS("cbc(aes)");
  43 MODULE_ALIAS("ctr(aes)");
  44 MODULE_ALIAS("xts(aes)");
  45 #endif
  46
  47 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
  48 MODULE_LICENSE("GPL v2");
  49
  50 /* defined in aes-modes.S */
  51 asmlinkage void aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[],
  52                                 int rounds, int blocks, int first);
  53 asmlinkage void aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[],
  54                                 int rounds, int blocks, int first);
  55
  56 asmlinkage void aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[],
  57                                 int rounds, int blocks, u8 iv[], int first);
  58 asmlinkage void aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[],
  59                                 int rounds, int blocks, u8 iv[], int first);
  60
  61 asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
  62                                 int rounds, int blocks, u8 ctr[], int first);
  63
  64 asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[],
  65                                 int rounds, int blocks, u8 const rk2[], u8 iv[],
  66                                 int first);
  67 asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[],
  68                                 int rounds, int blocks, u8 const rk2[], u8 iv[],
  69                                 int first);
  70
  71 struct crypto_aes_xts_ctx {
  72         struct crypto_aes_ctx key1;
  73         struct crypto_aes_ctx __aligned(8) key2;
  74 };
  75
  76 static int xts_set_key(struct crypto_tfm *tfm, const u8 *in_key,
  77                        unsigned int key_len)
  78 {
  79         struct crypto_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
  80         int ret;
  81
  82         ret = crypto_aes_expand_key(&ctx->key1, in_key, key_len / 2);
  83         if (!ret)
  84                 ret = crypto_aes_expand_key(&ctx->key2, &in_key[key_len / 2],
  85                                             key_len / 2);
  86         if (!ret)
  87                 return 0;
  88
  89         tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
  90         return -EINVAL;
  91 }
  92
  93 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
  94                        struct scatterlist *src, unsigned int nbytes)
  95 {
  96         struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
  97         int err, first, rounds = 6 + ctx->key_length / 4;
  98         struct blkcipher_walk walk;
  99         unsigned int blocks;
 100
 101         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 102         blkcipher_walk_init(&walk, dst, src, nbytes);
 103         err = blkcipher_walk_virt(desc, &walk);
 104
 105         kernel_neon_begin();
 106         for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 107                 aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 108                                 (u8 *)ctx->key_enc, rounds, blocks, first);
 109                 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
 110         }
 111         kernel_neon_end();
 112         return err;
 113 }
 114
 115 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 116                        struct scatterlist *src, unsigned int nbytes)
 117 {
 118         struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 119         int err, first, rounds = 6 + ctx->key_length / 4;
 120         struct blkcipher_walk walk;
 121         unsigned int blocks;
 122
 123         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 124         blkcipher_walk_init(&walk, dst, src, nbytes);
 125         err = blkcipher_walk_virt(desc, &walk);
 126
 127         kernel_neon_begin();
 128         for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 129                 aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 130                                 (u8 *)ctx->key_dec, rounds, blocks, first);
 131                 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
 132         }
 133         kernel_neon_end();
 134         return err;
 135 }
 136
 137 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 138                        struct scatterlist *src, unsigned int nbytes)
 139 {
 140         struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 141         int err, first, rounds = 6 + ctx->key_length / 4;
 142         struct blkcipher_walk walk;
 143         unsigned int blocks;
 144
 145         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 146         blkcipher_walk_init(&walk, dst, src, nbytes);
 147         err = blkcipher_walk_virt(desc, &walk);
 148
 149         kernel_neon_begin();
 150         for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 151                 aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 152                                 (u8 *)ctx->key_enc, rounds, blocks, walk.iv,
 153                                 first);
 154                 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
 155         }
 156         kernel_neon_end();
 157         return err;
 158 }
 159
 160 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 161                        struct scatterlist *src, unsigned int nbytes)
 162 {
 163         struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 164         int err, first, rounds = 6 + ctx->key_length / 4;
 165         struct blkcipher_walk walk;
 166         unsigned int blocks;
 167
 168         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 169         blkcipher_walk_init(&walk, dst, src, nbytes);
 170         err = blkcipher_walk_virt(desc, &walk);
 171
 172         kernel_neon_begin();
 173         for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 174                 aes_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 175                                 (u8 *)ctx->key_dec, rounds, blocks, walk.iv,
 176                                 first);
 177                 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
 178         }
 179         kernel_neon_end();
 180         return err;
 181 }
 182
 183 static int ctr_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 184                        struct scatterlist *src, unsigned int nbytes)
 185 {
 186         struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 187         int err, first, rounds = 6 + ctx->key_length / 4;
 188         struct blkcipher_walk walk;
 189         int blocks;
 190
 191         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 192         blkcipher_walk_init(&walk, dst, src, nbytes);
 193         err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
 194
 195         first = 1;
 196         kernel_neon_begin();
 197         while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
 198                 aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 199                                 (u8 *)ctx->key_enc, rounds, blocks, walk.iv,
 200                                 first);
 201                 first = 0;
 202                 nbytes -= blocks * AES_BLOCK_SIZE;
 203                 if (nbytes && nbytes == walk.nbytes % AES_BLOCK_SIZE)
 204                         break;
 205                 err = blkcipher_walk_done(desc, &walk,
 206                                           walk.nbytes % AES_BLOCK_SIZE);
 207         }
 208         if (nbytes) {
 209                 u8 *tdst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE;
 210                 u8 *tsrc = walk.src.virt.addr + blocks * AES_BLOCK_SIZE;
 211                 u8 __aligned(8) tail[AES_BLOCK_SIZE];
 212
 213                 /*
 214                  * Minimum alignment is 8 bytes, so if nbytes is <= 8, we need
 215                  * to tell aes_ctr_encrypt() to only read half a block.
 216                  */
 217                 blocks = (nbytes <= 8) ? -1 : 1;
 218
 219                 aes_ctr_encrypt(tail, tsrc, (u8 *)ctx->key_enc, rounds,
 220                                 blocks, walk.iv, first);
 221                 memcpy(tdst, tail, nbytes);
 222                 err = blkcipher_walk_done(desc, &walk, 0);
 223         }
 224         kernel_neon_end();
 225
 226         return err;
 227 }
 228
 229 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 230                        struct scatterlist *src, unsigned int nbytes)
 231 {
 232         struct crypto_aes_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 233         int err, first, rounds = 6 + ctx->key1.key_length / 4;
 234         struct blkcipher_walk walk;
 235         unsigned int blocks;
 236
 237         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 238         blkcipher_walk_init(&walk, dst, src, nbytes);
 239         err = blkcipher_walk_virt(desc, &walk);
 240
 241         kernel_neon_begin();
 242         for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 243                 aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 244                                 (u8 *)ctx->key1.key_enc, rounds, blocks,
 245                                 (u8 *)ctx->key2.key_enc, walk.iv, first);
 246                 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
 247         }
 248         kernel_neon_end();
 249
 250         return err;
 251 }
 252
 253 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 254                        struct scatterlist *src, unsigned int nbytes)
 255 {
 256         struct crypto_aes_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 257         int err, first, rounds = 6 + ctx->key1.key_length / 4;
 258         struct blkcipher_walk walk;
 259         unsigned int blocks;
 260
 261         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 262         blkcipher_walk_init(&walk, dst, src, nbytes);
 263         err = blkcipher_walk_virt(desc, &walk);
 264
 265         kernel_neon_begin();
 266         for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 267                 aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 268                                 (u8 *)ctx->key1.key_dec, rounds, blocks,
 269                                 (u8 *)ctx->key2.key_enc, walk.iv, first);
 270                 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
 271         }
 272         kernel_neon_end();
 273
 274         return err;
 275 }
 276
 277 static struct crypto_alg aes_algs[] = { {
 278         .cra_name               = "__ecb-aes-" MODE,
 279         .cra_driver_name        = "__driver-ecb-aes-" MODE,
 280         .cra_priority           = 0,
 281         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 282         .cra_blocksize          = AES_BLOCK_SIZE,
 283         .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
 284         .cra_alignmask          = 7,
 285         .cra_type               = &crypto_blkcipher_type,
 286         .cra_module             = THIS_MODULE,
 287         .cra_blkcipher = {
 288                 .min_keysize    = AES_MIN_KEY_SIZE,
 289                 .max_keysize    = AES_MAX_KEY_SIZE,
 290                 .ivsize         = AES_BLOCK_SIZE,
 291                 .setkey         = crypto_aes_set_key,
 292                 .encrypt        = ecb_encrypt,
 293                 .decrypt        = ecb_decrypt,
 294         },
 295 }, {
 296         .cra_name               = "__cbc-aes-" MODE,
 297         .cra_driver_name        = "__driver-cbc-aes-" MODE,
 298         .cra_priority           = 0,
 299         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 300         .cra_blocksize          = AES_BLOCK_SIZE,
 301         .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
 302         .cra_alignmask          = 7,
 303         .cra_type               = &crypto_blkcipher_type,
 304         .cra_module             = THIS_MODULE,
 305         .cra_blkcipher = {
 306                 .min_keysize    = AES_MIN_KEY_SIZE,
 307                 .max_keysize    = AES_MAX_KEY_SIZE,
 308                 .ivsize         = AES_BLOCK_SIZE,
 309                 .setkey         = crypto_aes_set_key,
 310                 .encrypt        = cbc_encrypt,
 311                 .decrypt        = cbc_decrypt,
 312         },
 313 }, {
 314         .cra_name               = "__ctr-aes-" MODE,
 315         .cra_driver_name        = "__driver-ctr-aes-" MODE,
 316         .cra_priority           = 0,
 317         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 318         .cra_blocksize          = 1,
 319         .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
 320         .cra_alignmask          = 7,
 321         .cra_type               = &crypto_blkcipher_type,
 322         .cra_module             = THIS_MODULE,
 323         .cra_blkcipher = {
 324                 .min_keysize    = AES_MIN_KEY_SIZE,
 325                 .max_keysize    = AES_MAX_KEY_SIZE,
 326                 .ivsize         = AES_BLOCK_SIZE,
 327                 .setkey         = crypto_aes_set_key,
 328                 .encrypt        = ctr_encrypt,
 329                 .decrypt        = ctr_encrypt,
 330         },
 331 }, {
 332         .cra_name               = "__xts-aes-" MODE,
 333         .cra_driver_name        = "__driver-xts-aes-" MODE,
 334         .cra_priority           = 0,
 335         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 336         .cra_blocksize          = AES_BLOCK_SIZE,
 337         .cra_ctxsize            = sizeof(struct crypto_aes_xts_ctx),
 338         .cra_alignmask          = 7,
 339         .cra_type               = &crypto_blkcipher_type,
 340         .cra_module             = THIS_MODULE,
 341         .cra_blkcipher = {
 342                 .min_keysize    = 2 * AES_MIN_KEY_SIZE,
 343                 .max_keysize    = 2 * AES_MAX_KEY_SIZE,
 344                 .ivsize         = AES_BLOCK_SIZE,
 345                 .setkey         = xts_set_key,
 346                 .encrypt        = xts_encrypt,
 347                 .decrypt        = xts_decrypt,
 348         },
 349 }, {
 350         .cra_name               = "ecb(aes)",
 351         .cra_driver_name        = "ecb-aes-" MODE,
 352         .cra_priority           = PRIO,
 353         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 354         .cra_blocksize          = AES_BLOCK_SIZE,
 355         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 356         .cra_alignmask          = 7,
 357         .cra_type               = &crypto_ablkcipher_type,
 358         .cra_module             = THIS_MODULE,
 359         .cra_init               = ablk_init,
 360         .cra_exit               = ablk_exit,
 361         .cra_ablkcipher = {
 362                 .min_keysize    = AES_MIN_KEY_SIZE,
 363                 .max_keysize    = AES_MAX_KEY_SIZE,
 364                 .ivsize         = AES_BLOCK_SIZE,
 365                 .setkey         = ablk_set_key,
 366                 .encrypt        = ablk_encrypt,
 367                 .decrypt        = ablk_decrypt,
 368         }
 369 }, {
 370         .cra_name               = "cbc(aes)",
 371         .cra_driver_name        = "cbc-aes-" MODE,
 372         .cra_priority           = PRIO,
 373         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 374         .cra_blocksize          = AES_BLOCK_SIZE,
 375         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 376         .cra_alignmask          = 7,
 377         .cra_type               = &crypto_ablkcipher_type,
 378         .cra_module             = THIS_MODULE,
 379         .cra_init               = ablk_init,
 380         .cra_exit               = ablk_exit,
 381         .cra_ablkcipher = {
 382                 .min_keysize    = AES_MIN_KEY_SIZE,
 383                 .max_keysize    = AES_MAX_KEY_SIZE,
 384                 .ivsize         = AES_BLOCK_SIZE,
 385                 .setkey         = ablk_set_key,
 386                 .encrypt        = ablk_encrypt,
 387                 .decrypt        = ablk_decrypt,
 388         }
 389 }, {
 390         .cra_name               = "ctr(aes)",
 391         .cra_driver_name        = "ctr-aes-" MODE,
 392         .cra_priority           = PRIO,
 393         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 394         .cra_blocksize          = 1,
 395         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 396         .cra_alignmask          = 7,
 397         .cra_type               = &crypto_ablkcipher_type,
 398         .cra_module             = THIS_MODULE,
 399         .cra_init               = ablk_init,
 400         .cra_exit               = ablk_exit,
 401         .cra_ablkcipher = {
 402                 .min_keysize    = AES_MIN_KEY_SIZE,
 403                 .max_keysize    = AES_MAX_KEY_SIZE,
 404                 .ivsize         = AES_BLOCK_SIZE,
 405                 .setkey         = ablk_set_key,
 406                 .encrypt        = ablk_encrypt,
 407                 .decrypt        = ablk_decrypt,
 408         }
 409 }, {
 410         .cra_name               = "xts(aes)",
 411         .cra_driver_name        = "xts-aes-" MODE,
 412         .cra_priority           = PRIO,
 413         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 414         .cra_blocksize          = AES_BLOCK_SIZE,
 415         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 416         .cra_alignmask          = 7,
 417         .cra_type               = &crypto_ablkcipher_type,
 418         .cra_module             = THIS_MODULE,
 419         .cra_init               = ablk_init,
 420         .cra_exit               = ablk_exit,
 421         .cra_ablkcipher = {
 422                 .min_keysize    = 2 * AES_MIN_KEY_SIZE,
 423                 .max_keysize    = 2 * AES_MAX_KEY_SIZE,
 424                 .ivsize         = AES_BLOCK_SIZE,
 425                 .setkey         = ablk_set_key,
 426                 .encrypt        = ablk_encrypt,
 427                 .decrypt        = ablk_decrypt,
 428         }
 429 } };
 430
 431 static int __init aes_init(void)
 432 {
 433         return crypto_register_algs(aes_algs, ARRAY_SIZE(aes_algs));
 434 }
 435
 436 static void __exit aes_exit(void)
 437 {
 438         crypto_unregister_algs(aes_algs, ARRAY_SIZE(aes_algs));
 439 }
 440
 441 #ifdef USE_V8_CRYPTO_EXTENSIONS
 442 module_cpu_feature_match(AES, aes_init);
 443 #else
 444 module_init(aes_init);
 445 #endif
 446 module_exit(aes_exit);