arch/arm64/crypto/aes-glue.c

   1 /*
   2  * linux/arch/arm64/crypto/aes-glue.c - wrapper code for ARMv8 AES
   3  *
   4  * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10
  11 #include <asm/neon.h>
  12 #include <asm/hwcap.h>
  13 #include <crypto/aes.h>
  14 #include <crypto/ablk_helper.h>
  15 #include <crypto/algapi.h>
  16 #include <linux/module.h>
  17 #include <linux/cpufeature.h>
  18
  19 #include "aes-ce-setkey.h"
  20
  21 #ifdef USE_V8_CRYPTO_EXTENSIONS
  22 #define MODE                    "ce"
  23 #define PRIO                    300
  24 #define aes_setkey              ce_aes_setkey
  25 #define aes_expandkey           ce_aes_expandkey
  26 #define aes_ecb_encrypt         ce_aes_ecb_encrypt
  27 #define aes_ecb_decrypt         ce_aes_ecb_decrypt
  28 #define aes_cbc_encrypt         ce_aes_cbc_encrypt
  29 #define aes_cbc_decrypt         ce_aes_cbc_decrypt
  30 #define aes_ctr_encrypt         ce_aes_ctr_encrypt
  31 #define aes_xts_encrypt         ce_aes_xts_encrypt
  32 #define aes_xts_decrypt         ce_aes_xts_decrypt
  33 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
  34 #else
  35 #define MODE                    "neon"
  36 #define PRIO                    200
  37 #define aes_setkey              crypto_aes_set_key
  38 #define aes_expandkey           crypto_aes_expand_key
  39 #define aes_ecb_encrypt         neon_aes_ecb_encrypt
  40 #define aes_ecb_decrypt         neon_aes_ecb_decrypt
  41 #define aes_cbc_encrypt         neon_aes_cbc_encrypt
  42 #define aes_cbc_decrypt         neon_aes_cbc_decrypt
  43 #define aes_ctr_encrypt         neon_aes_ctr_encrypt
  44 #define aes_xts_encrypt         neon_aes_xts_encrypt
  45 #define aes_xts_decrypt         neon_aes_xts_decrypt
  46 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 NEON");
  47 MODULE_ALIAS_CRYPTO("ecb(aes)");
  48 MODULE_ALIAS_CRYPTO("cbc(aes)");
  49 MODULE_ALIAS_CRYPTO("ctr(aes)");
  50 MODULE_ALIAS_CRYPTO("xts(aes)");
  51 #endif
  52
  53 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
  54 MODULE_LICENSE("GPL v2");
  55
  56 /* defined in aes-modes.S */
  57 asmlinkage void aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[],
  58                                 int rounds, int blocks, int first);
  59 asmlinkage void aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[],
  60                                 int rounds, int blocks, int first);
  61
  62 asmlinkage void aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[],
  63                                 int rounds, int blocks, u8 iv[], int first);
  64 asmlinkage void aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[],
  65                                 int rounds, int blocks, u8 iv[], int first);
  66
  67 asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
  68                                 int rounds, int blocks, u8 ctr[], int first);
  69
  70 asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[],
  71                                 int rounds, int blocks, u8 const rk2[], u8 iv[],
  72                                 int first);
  73 asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[],
  74                                 int rounds, int blocks, u8 const rk2[], u8 iv[],
  75                                 int first);
  76
  77 struct crypto_aes_xts_ctx {
  78         struct crypto_aes_ctx key1;
  79         struct crypto_aes_ctx __aligned(8) key2;
  80 };
  81
  82 static int xts_set_key(struct crypto_tfm *tfm, const u8 *in_key,
  83                        unsigned int key_len)
  84 {
  85         struct crypto_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
  86         int ret;
  87
  88         ret = aes_expandkey(&ctx->key1, in_key, key_len / 2);
  89         if (!ret)
  90                 ret = aes_expandkey(&ctx->key2, &in_key[key_len / 2],
  91                                     key_len / 2);
  92         if (!ret)
  93                 return 0;
  94
  95         tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
  96         return -EINVAL;
  97 }
  98
  99 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 100                        struct scatterlist *src, unsigned int nbytes)
 101 {
 102         struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 103         int err, first, rounds = 6 + ctx->key_length / 4;
 104         struct blkcipher_walk walk;
 105         unsigned int blocks;
 106
 107         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 108         blkcipher_walk_init(&walk, dst, src, nbytes);
 109         err = blkcipher_walk_virt(desc, &walk);
 110
 111         kernel_neon_begin();
 112         for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 113                 aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 114                                 (u8 *)ctx->key_enc, rounds, blocks, first);
 115                 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
 116         }
 117         kernel_neon_end();
 118         return err;
 119 }
 120
 121 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 122                        struct scatterlist *src, unsigned int nbytes)
 123 {
 124         struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 125         int err, first, rounds = 6 + ctx->key_length / 4;
 126         struct blkcipher_walk walk;
 127         unsigned int blocks;
 128
 129         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 130         blkcipher_walk_init(&walk, dst, src, nbytes);
 131         err = blkcipher_walk_virt(desc, &walk);
 132
 133         kernel_neon_begin();
 134         for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 135                 aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 136                                 (u8 *)ctx->key_dec, rounds, blocks, first);
 137                 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
 138         }
 139         kernel_neon_end();
 140         return err;
 141 }
 142
 143 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 144                        struct scatterlist *src, unsigned int nbytes)
 145 {
 146         struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 147         int err, first, rounds = 6 + ctx->key_length / 4;
 148         struct blkcipher_walk walk;
 149         unsigned int blocks;
 150
 151         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 152         blkcipher_walk_init(&walk, dst, src, nbytes);
 153         err = blkcipher_walk_virt(desc, &walk);
 154
 155         kernel_neon_begin();
 156         for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 157                 aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 158                                 (u8 *)ctx->key_enc, rounds, blocks, walk.iv,
 159                                 first);
 160                 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
 161         }
 162         kernel_neon_end();
 163         return err;
 164 }
 165
 166 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 167                        struct scatterlist *src, unsigned int nbytes)
 168 {
 169         struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 170         int err, first, rounds = 6 + ctx->key_length / 4;
 171         struct blkcipher_walk walk;
 172         unsigned int blocks;
 173
 174         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 175         blkcipher_walk_init(&walk, dst, src, nbytes);
 176         err = blkcipher_walk_virt(desc, &walk);
 177
 178         kernel_neon_begin();
 179         for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 180                 aes_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 181                                 (u8 *)ctx->key_dec, rounds, blocks, walk.iv,
 182                                 first);
 183                 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
 184         }
 185         kernel_neon_end();
 186         return err;
 187 }
 188
 189 static int ctr_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 190                        struct scatterlist *src, unsigned int nbytes)
 191 {
 192         struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 193         int err, first, rounds = 6 + ctx->key_length / 4;
 194         struct blkcipher_walk walk;
 195         int blocks;
 196
 197         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 198         blkcipher_walk_init(&walk, dst, src, nbytes);
 199         err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
 200
 201         first = 1;
 202         kernel_neon_begin();
 203         while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
 204                 aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 205                                 (u8 *)ctx->key_enc, rounds, blocks, walk.iv,
 206                                 first);
 207                 first = 0;
 208                 nbytes -= blocks * AES_BLOCK_SIZE;
 209                 if (nbytes && nbytes == walk.nbytes % AES_BLOCK_SIZE)
 210                         break;
 211                 err = blkcipher_walk_done(desc, &walk,
 212                                           walk.nbytes % AES_BLOCK_SIZE);
 213         }
 214         if (nbytes) {
 215                 u8 *tdst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE;
 216                 u8 *tsrc = walk.src.virt.addr + blocks * AES_BLOCK_SIZE;
 217                 u8 __aligned(8) tail[AES_BLOCK_SIZE];
 218
 219                 /*
 220                  * Minimum alignment is 8 bytes, so if nbytes is <= 8, we need
 221                  * to tell aes_ctr_encrypt() to only read half a block.
 222                  */
 223                 blocks = (nbytes <= 8) ? -1 : 1;
 224
 225                 aes_ctr_encrypt(tail, tsrc, (u8 *)ctx->key_enc, rounds,
 226                                 blocks, walk.iv, first);
 227                 memcpy(tdst, tail, nbytes);
 228                 err = blkcipher_walk_done(desc, &walk, 0);
 229         }
 230         kernel_neon_end();
 231
 232         return err;
 233 }
 234
 235 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 236                        struct scatterlist *src, unsigned int nbytes)
 237 {
 238         struct crypto_aes_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 239         int err, first, rounds = 6 + ctx->key1.key_length / 4;
 240         struct blkcipher_walk walk;
 241         unsigned int blocks;
 242
 243         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 244         blkcipher_walk_init(&walk, dst, src, nbytes);
 245         err = blkcipher_walk_virt(desc, &walk);
 246
 247         kernel_neon_begin();
 248         for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 249                 aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 250                                 (u8 *)ctx->key1.key_enc, rounds, blocks,
 251                                 (u8 *)ctx->key2.key_enc, walk.iv, first);
 252                 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
 253         }
 254         kernel_neon_end();
 255
 256         return err;
 257 }
 258
 259 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 260                        struct scatterlist *src, unsigned int nbytes)
 261 {
 262         struct crypto_aes_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 263         int err, first, rounds = 6 + ctx->key1.key_length / 4;
 264         struct blkcipher_walk walk;
 265         unsigned int blocks;
 266
 267         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 268         blkcipher_walk_init(&walk, dst, src, nbytes);
 269         err = blkcipher_walk_virt(desc, &walk);
 270
 271         kernel_neon_begin();
 272         for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 273                 aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 274                                 (u8 *)ctx->key1.key_dec, rounds, blocks,
 275                                 (u8 *)ctx->key2.key_enc, walk.iv, first);
 276                 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
 277         }
 278         kernel_neon_end();
 279
 280         return err;
 281 }
 282
 283 static struct crypto_alg aes_algs[] = { {
 284         .cra_name               = "__ecb-aes-" MODE,
 285         .cra_driver_name        = "__driver-ecb-aes-" MODE,
 286         .cra_priority           = 0,
 287         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 288                                   CRYPTO_ALG_INTERNAL,
 289         .cra_blocksize          = AES_BLOCK_SIZE,
 290         .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
 291         .cra_alignmask          = 7,
 292         .cra_type               = &crypto_blkcipher_type,
 293         .cra_module             = THIS_MODULE,
 294         .cra_blkcipher = {
 295                 .min_keysize    = AES_MIN_KEY_SIZE,
 296                 .max_keysize    = AES_MAX_KEY_SIZE,
 297                 .ivsize         = AES_BLOCK_SIZE,
 298                 .setkey         = aes_setkey,
 299                 .encrypt        = ecb_encrypt,
 300                 .decrypt        = ecb_decrypt,
 301         },
 302 }, {
 303         .cra_name               = "__cbc-aes-" MODE,
 304         .cra_driver_name        = "__driver-cbc-aes-" MODE,
 305         .cra_priority           = 0,
 306         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 307                                   CRYPTO_ALG_INTERNAL,
 308         .cra_blocksize          = AES_BLOCK_SIZE,
 309         .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
 310         .cra_alignmask          = 7,
 311         .cra_type               = &crypto_blkcipher_type,
 312         .cra_module             = THIS_MODULE,
 313         .cra_blkcipher = {
 314                 .min_keysize    = AES_MIN_KEY_SIZE,
 315                 .max_keysize    = AES_MAX_KEY_SIZE,
 316                 .ivsize         = AES_BLOCK_SIZE,
 317                 .setkey         = aes_setkey,
 318                 .encrypt        = cbc_encrypt,
 319                 .decrypt        = cbc_decrypt,
 320         },
 321 }, {
 322         .cra_name               = "__ctr-aes-" MODE,
 323         .cra_driver_name        = "__driver-ctr-aes-" MODE,
 324         .cra_priority           = 0,
 325         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 326                                   CRYPTO_ALG_INTERNAL,
 327         .cra_blocksize          = 1,
 328         .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
 329         .cra_alignmask          = 7,
 330         .cra_type               = &crypto_blkcipher_type,
 331         .cra_module             = THIS_MODULE,
 332         .cra_blkcipher = {
 333                 .min_keysize    = AES_MIN_KEY_SIZE,
 334                 .max_keysize    = AES_MAX_KEY_SIZE,
 335                 .ivsize         = AES_BLOCK_SIZE,
 336                 .setkey         = aes_setkey,
 337                 .encrypt        = ctr_encrypt,
 338                 .decrypt        = ctr_encrypt,
 339         },
 340 }, {
 341         .cra_name               = "__xts-aes-" MODE,
 342         .cra_driver_name        = "__driver-xts-aes-" MODE,
 343         .cra_priority           = 0,
 344         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 345                                   CRYPTO_ALG_INTERNAL,
 346         .cra_blocksize          = AES_BLOCK_SIZE,
 347         .cra_ctxsize            = sizeof(struct crypto_aes_xts_ctx),
 348         .cra_alignmask          = 7,
 349         .cra_type               = &crypto_blkcipher_type,
 350         .cra_module             = THIS_MODULE,
 351         .cra_blkcipher = {
 352                 .min_keysize    = 2 * AES_MIN_KEY_SIZE,
 353                 .max_keysize    = 2 * AES_MAX_KEY_SIZE,
 354                 .ivsize         = AES_BLOCK_SIZE,
 355                 .setkey         = xts_set_key,
 356                 .encrypt        = xts_encrypt,
 357                 .decrypt        = xts_decrypt,
 358         },
 359 }, {
 360         .cra_name               = "ecb(aes)",
 361         .cra_driver_name        = "ecb-aes-" MODE,
 362         .cra_priority           = PRIO,
 363         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 364         .cra_blocksize          = AES_BLOCK_SIZE,
 365         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 366         .cra_alignmask          = 7,
 367         .cra_type               = &crypto_ablkcipher_type,
 368         .cra_module             = THIS_MODULE,
 369         .cra_init               = ablk_init,
 370         .cra_exit               = ablk_exit,
 371         .cra_ablkcipher = {
 372                 .min_keysize    = AES_MIN_KEY_SIZE,
 373                 .max_keysize    = AES_MAX_KEY_SIZE,
 374                 .ivsize         = AES_BLOCK_SIZE,
 375                 .setkey         = ablk_set_key,
 376                 .encrypt        = ablk_encrypt,
 377                 .decrypt        = ablk_decrypt,
 378         }
 379 }, {
 380         .cra_name               = "cbc(aes)",
 381         .cra_driver_name        = "cbc-aes-" MODE,
 382         .cra_priority           = PRIO,
 383         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 384         .cra_blocksize          = AES_BLOCK_SIZE,
 385         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 386         .cra_alignmask          = 7,
 387         .cra_type               = &crypto_ablkcipher_type,
 388         .cra_module             = THIS_MODULE,
 389         .cra_init               = ablk_init,
 390         .cra_exit               = ablk_exit,
 391         .cra_ablkcipher = {
 392                 .min_keysize    = AES_MIN_KEY_SIZE,
 393                 .max_keysize    = AES_MAX_KEY_SIZE,
 394                 .ivsize         = AES_BLOCK_SIZE,
 395                 .setkey         = ablk_set_key,
 396                 .encrypt        = ablk_encrypt,
 397                 .decrypt        = ablk_decrypt,
 398         }
 399 }, {
 400         .cra_name               = "ctr(aes)",
 401         .cra_driver_name        = "ctr-aes-" MODE,
 402         .cra_priority           = PRIO,
 403         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 404         .cra_blocksize          = 1,
 405         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 406         .cra_alignmask          = 7,
 407         .cra_type               = &crypto_ablkcipher_type,
 408         .cra_module             = THIS_MODULE,
 409         .cra_init               = ablk_init,
 410         .cra_exit               = ablk_exit,
 411         .cra_ablkcipher = {
 412                 .min_keysize    = AES_MIN_KEY_SIZE,
 413                 .max_keysize    = AES_MAX_KEY_SIZE,
 414                 .ivsize         = AES_BLOCK_SIZE,
 415                 .setkey         = ablk_set_key,
 416                 .encrypt        = ablk_encrypt,
 417                 .decrypt        = ablk_decrypt,
 418         }
 419 }, {
 420         .cra_name               = "xts(aes)",
 421         .cra_driver_name        = "xts-aes-" MODE,
 422         .cra_priority           = PRIO,
 423         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 424         .cra_blocksize          = AES_BLOCK_SIZE,
 425         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 426         .cra_alignmask          = 7,
 427         .cra_type               = &crypto_ablkcipher_type,
 428         .cra_module             = THIS_MODULE,
 429         .cra_init               = ablk_init,
 430         .cra_exit               = ablk_exit,
 431         .cra_ablkcipher = {
 432                 .min_keysize    = 2 * AES_MIN_KEY_SIZE,
 433                 .max_keysize    = 2 * AES_MAX_KEY_SIZE,
 434                 .ivsize         = AES_BLOCK_SIZE,
 435                 .setkey         = ablk_set_key,
 436                 .encrypt        = ablk_encrypt,
 437                 .decrypt        = ablk_decrypt,
 438         }
 439 } };
 440
 441 static int __init aes_init(void)
 442 {
 443         return crypto_register_algs(aes_algs, ARRAY_SIZE(aes_algs));
 444 }
 445
 446 static void __exit aes_exit(void)
 447 {
 448         crypto_unregister_algs(aes_algs, ARRAY_SIZE(aes_algs));
 449 }
 450
 451 #ifdef USE_V8_CRYPTO_EXTENSIONS
 452 module_cpu_feature_match(AES, aes_init);
 453 #else
 454 module_init(aes_init);
 455 #endif
 456 module_exit(aes_exit);