drivers/crypto/amcc/crypto4xx_alg.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /**
   3  * AMCC SoC PPC4xx Crypto Driver
   4  *
   5  * Copyright (c) 2008 Applied Micro Circuits Corporation.
   6  * All rights reserved. James Hsiao <jhsiao@amcc.com>
   7  *
   8  * This file implements the Linux crypto algorithms.
   9  */
  10
  11 #include <linux/kernel.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/spinlock_types.h>
  14 #include <linux/scatterlist.h>
  15 #include <linux/crypto.h>
  16 #include <linux/hash.h>
  17 #include <crypto/internal/hash.h>
  18 #include <linux/dma-mapping.h>
  19 #include <crypto/algapi.h>
  20 #include <crypto/aead.h>
  21 #include <crypto/aes.h>
  22 #include <crypto/gcm.h>
  23 #include <crypto/sha.h>
  24 #include <crypto/ctr.h>
  25 #include <crypto/skcipher.h>
  26 #include "crypto4xx_reg_def.h"
  27 #include "crypto4xx_core.h"
  28 #include "crypto4xx_sa.h"
  29
  30 static void set_dynamic_sa_command_0(struct dynamic_sa_ctl *sa, u32 save_h,
  31                                      u32 save_iv, u32 ld_h, u32 ld_iv,
  32                                      u32 hdr_proc, u32 h, u32 c, u32 pad_type,
  33                                      u32 op_grp, u32 op, u32 dir)
  34 {
  35         sa->sa_command_0.w = 0;
  36         sa->sa_command_0.bf.save_hash_state = save_h;
  37         sa->sa_command_0.bf.save_iv = save_iv;
  38         sa->sa_command_0.bf.load_hash_state = ld_h;
  39         sa->sa_command_0.bf.load_iv = ld_iv;
  40         sa->sa_command_0.bf.hdr_proc = hdr_proc;
  41         sa->sa_command_0.bf.hash_alg = h;
  42         sa->sa_command_0.bf.cipher_alg = c;
  43         sa->sa_command_0.bf.pad_type = pad_type & 3;
  44         sa->sa_command_0.bf.extend_pad = pad_type >> 2;
  45         sa->sa_command_0.bf.op_group = op_grp;
  46         sa->sa_command_0.bf.opcode = op;
  47         sa->sa_command_0.bf.dir = dir;
  48 }
  49
  50 static void set_dynamic_sa_command_1(struct dynamic_sa_ctl *sa, u32 cm,
  51                                      u32 hmac_mc, u32 cfb, u32 esn,
  52                                      u32 sn_mask, u32 mute, u32 cp_pad,
  53                                      u32 cp_pay, u32 cp_hdr)
  54 {
  55         sa->sa_command_1.w = 0;
  56         sa->sa_command_1.bf.crypto_mode31 = (cm & 4) >> 2;
  57         sa->sa_command_1.bf.crypto_mode9_8 = cm & 3;
  58         sa->sa_command_1.bf.feedback_mode = cfb,
  59         sa->sa_command_1.bf.sa_rev = 1;
  60         sa->sa_command_1.bf.hmac_muting = hmac_mc;
  61         sa->sa_command_1.bf.extended_seq_num = esn;
  62         sa->sa_command_1.bf.seq_num_mask = sn_mask;
  63         sa->sa_command_1.bf.mutable_bit_proc = mute;
  64         sa->sa_command_1.bf.copy_pad = cp_pad;
  65         sa->sa_command_1.bf.copy_payload = cp_pay;
  66         sa->sa_command_1.bf.copy_hdr = cp_hdr;
  67 }
  68
  69 static inline int crypto4xx_crypt(struct skcipher_request *req,
  70                                   const unsigned int ivlen, bool decrypt,
  71                                   bool check_blocksize)
  72 {
  73         struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
  74         struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher);
  75         __le32 iv[AES_IV_SIZE];
  76
  77         if (check_blocksize && !IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE))
  78                 return -EINVAL;
  79
  80         if (ivlen)
  81                 crypto4xx_memcpy_to_le32(iv, req->iv, ivlen);
  82
  83         return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst,
  84                 req->cryptlen, iv, ivlen, decrypt ? ctx->sa_in : ctx->sa_out,
  85                 ctx->sa_len, 0, NULL);
  86 }
  87
  88 int crypto4xx_encrypt_noiv_block(struct skcipher_request *req)
  89 {
  90         return crypto4xx_crypt(req, 0, false, true);
  91 }
  92
  93 int crypto4xx_encrypt_iv_stream(struct skcipher_request *req)
  94 {
  95         return crypto4xx_crypt(req, AES_IV_SIZE, false, false);
  96 }
  97
  98 int crypto4xx_decrypt_noiv_block(struct skcipher_request *req)
  99 {
 100         return crypto4xx_crypt(req, 0, true, true);
 101 }
 102
 103 int crypto4xx_decrypt_iv_stream(struct skcipher_request *req)
 104 {
 105         return crypto4xx_crypt(req, AES_IV_SIZE, true, false);
 106 }
 107
 108 int crypto4xx_encrypt_iv_block(struct skcipher_request *req)
 109 {
 110         return crypto4xx_crypt(req, AES_IV_SIZE, false, true);
 111 }
 112
 113 int crypto4xx_decrypt_iv_block(struct skcipher_request *req)
 114 {
 115         return crypto4xx_crypt(req, AES_IV_SIZE, true, true);
 116 }
 117
 118 /**
 119  * AES Functions
 120  */
 121 static int crypto4xx_setkey_aes(struct crypto_skcipher *cipher,
 122                                 const u8 *key,
 123                                 unsigned int keylen,
 124                                 unsigned char cm,
 125                                 u8 fb)
 126 {
 127         struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher);
 128         struct dynamic_sa_ctl *sa;
 129         int    rc;
 130
 131         if (keylen != AES_KEYSIZE_256 && keylen != AES_KEYSIZE_192 &&
 132             keylen != AES_KEYSIZE_128)
 133                 return -EINVAL;
 134
 135         /* Create SA */
 136         if (ctx->sa_in || ctx->sa_out)
 137                 crypto4xx_free_sa(ctx);
 138
 139         rc = crypto4xx_alloc_sa(ctx, SA_AES128_LEN + (keylen-16) / 4);
 140         if (rc)
 141                 return rc;
 142
 143         /* Setup SA */
 144         sa = ctx->sa_in;
 145
 146         set_dynamic_sa_command_0(sa, SA_NOT_SAVE_HASH, (cm == CRYPTO_MODE_ECB ?
 147                                  SA_NOT_SAVE_IV : SA_SAVE_IV),
 148                                  SA_NOT_LOAD_HASH, (cm == CRYPTO_MODE_ECB ?
 149                                  SA_LOAD_IV_FROM_SA : SA_LOAD_IV_FROM_STATE),
 150                                  SA_NO_HEADER_PROC, SA_HASH_ALG_NULL,
 151                                  SA_CIPHER_ALG_AES, SA_PAD_TYPE_ZERO,
 152                                  SA_OP_GROUP_BASIC, SA_OPCODE_DECRYPT,
 153                                  DIR_INBOUND);
 154
 155         set_dynamic_sa_command_1(sa, cm, SA_HASH_MODE_HASH,
 156                                  fb, SA_EXTENDED_SN_OFF,
 157                                  SA_SEQ_MASK_OFF, SA_MC_ENABLE,
 158                                  SA_NOT_COPY_PAD, SA_NOT_COPY_PAYLOAD,
 159                                  SA_NOT_COPY_HDR);
 160         crypto4xx_memcpy_to_le32(get_dynamic_sa_key_field(sa),
 161                                  key, keylen);
 162         sa->sa_contents.w = SA_AES_CONTENTS | (keylen << 2);
 163         sa->sa_command_1.bf.key_len = keylen >> 3;
 164
 165         memcpy(ctx->sa_out, ctx->sa_in, ctx->sa_len * 4);
 166         sa = ctx->sa_out;
 167         sa->sa_command_0.bf.dir = DIR_OUTBOUND;
 168         /*
 169          * SA_OPCODE_ENCRYPT is the same value as SA_OPCODE_DECRYPT.
 170          * it's the DIR_(IN|OUT)BOUND that matters
 171          */
 172         sa->sa_command_0.bf.opcode = SA_OPCODE_ENCRYPT;
 173
 174         return 0;
 175 }
 176
 177 int crypto4xx_setkey_aes_cbc(struct crypto_skcipher *cipher,
 178                              const u8 *key, unsigned int keylen)
 179 {
 180         return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_CBC,
 181                                     CRYPTO_FEEDBACK_MODE_NO_FB);
 182 }
 183
 184 int crypto4xx_setkey_aes_cfb(struct crypto_skcipher *cipher,
 185                              const u8 *key, unsigned int keylen)
 186 {
 187         return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_CFB,
 188                                     CRYPTO_FEEDBACK_MODE_128BIT_CFB);
 189 }
 190
 191 int crypto4xx_setkey_aes_ecb(struct crypto_skcipher *cipher,
 192                              const u8 *key, unsigned int keylen)
 193 {
 194         return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_ECB,
 195                                     CRYPTO_FEEDBACK_MODE_NO_FB);
 196 }
 197
 198 int crypto4xx_setkey_aes_ofb(struct crypto_skcipher *cipher,
 199                              const u8 *key, unsigned int keylen)
 200 {
 201         return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_OFB,
 202                                     CRYPTO_FEEDBACK_MODE_64BIT_OFB);
 203 }
 204
 205 int crypto4xx_setkey_rfc3686(struct crypto_skcipher *cipher,
 206                              const u8 *key, unsigned int keylen)
 207 {
 208         struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher);
 209         int rc;
 210
 211         rc = crypto4xx_setkey_aes(cipher, key, keylen - CTR_RFC3686_NONCE_SIZE,
 212                 CRYPTO_MODE_CTR, CRYPTO_FEEDBACK_MODE_NO_FB);
 213         if (rc)
 214                 return rc;
 215
 216         ctx->iv_nonce = cpu_to_le32p((u32 *)&key[keylen -
 217                                                  CTR_RFC3686_NONCE_SIZE]);
 218
 219         return 0;
 220 }
 221
 222 int crypto4xx_rfc3686_encrypt(struct skcipher_request *req)
 223 {
 224         struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
 225         struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher);
 226         __le32 iv[AES_IV_SIZE / 4] = {
 227                 ctx->iv_nonce,
 228                 cpu_to_le32p((u32 *) req->iv),
 229                 cpu_to_le32p((u32 *) (req->iv + 4)),
 230                 cpu_to_le32(1) };
 231
 232         return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst,
 233                                   req->cryptlen, iv, AES_IV_SIZE,
 234                                   ctx->sa_out, ctx->sa_len, 0, NULL);
 235 }
 236
 237 int crypto4xx_rfc3686_decrypt(struct skcipher_request *req)
 238 {
 239         struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
 240         struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher);
 241         __le32 iv[AES_IV_SIZE / 4] = {
 242                 ctx->iv_nonce,
 243                 cpu_to_le32p((u32 *) req->iv),
 244                 cpu_to_le32p((u32 *) (req->iv + 4)),
 245                 cpu_to_le32(1) };
 246
 247         return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst,
 248                                   req->cryptlen, iv, AES_IV_SIZE,
 249                                   ctx->sa_out, ctx->sa_len, 0, NULL);
 250 }
 251
 252 static int
 253 crypto4xx_ctr_crypt(struct skcipher_request *req, bool encrypt)
 254 {
 255         struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
 256         struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher);
 257         size_t iv_len = crypto_skcipher_ivsize(cipher);
 258         unsigned int counter = be32_to_cpup((__be32 *)(req->iv + iv_len - 4));
 259         unsigned int nblks = ALIGN(req->cryptlen, AES_BLOCK_SIZE) /
 260                         AES_BLOCK_SIZE;
 261
 262         /*
 263          * The hardware uses only the last 32-bits as the counter while the
 264          * kernel tests (aes_ctr_enc_tv_template[4] for example) expect that
 265          * the whole IV is a counter.  So fallback if the counter is going to
 266          * overlow.
 267          */
 268         if (counter + nblks < counter) {
 269                 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->sw_cipher.cipher);
 270                 int ret;
 271
 272                 skcipher_request_set_sync_tfm(subreq, ctx->sw_cipher.cipher);
 273                 skcipher_request_set_callback(subreq, req->base.flags,
 274                         NULL, NULL);
 275                 skcipher_request_set_crypt(subreq, req->src, req->dst,
 276                         req->cryptlen, req->iv);
 277                 ret = encrypt ? crypto_skcipher_encrypt(subreq)
 278                         : crypto_skcipher_decrypt(subreq);
 279                 skcipher_request_zero(subreq);
 280                 return ret;
 281         }
 282
 283         return encrypt ? crypto4xx_encrypt_iv_stream(req)
 284                        : crypto4xx_decrypt_iv_stream(req);
 285 }
 286
 287 static int crypto4xx_sk_setup_fallback(struct crypto4xx_ctx *ctx,
 288                                        struct crypto_skcipher *cipher,
 289                                        const u8 *key,
 290                                        unsigned int keylen)
 291 {
 292         crypto_sync_skcipher_clear_flags(ctx->sw_cipher.cipher,
 293                                     CRYPTO_TFM_REQ_MASK);
 294         crypto_sync_skcipher_set_flags(ctx->sw_cipher.cipher,
 295                 crypto_skcipher_get_flags(cipher) & CRYPTO_TFM_REQ_MASK);
 296         return crypto_sync_skcipher_setkey(ctx->sw_cipher.cipher, key, keylen);
 297 }
 298
 299 int crypto4xx_setkey_aes_ctr(struct crypto_skcipher *cipher,
 300                              const u8 *key, unsigned int keylen)
 301 {
 302         struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher);
 303         int rc;
 304
 305         rc = crypto4xx_sk_setup_fallback(ctx, cipher, key, keylen);
 306         if (rc)
 307                 return rc;
 308
 309         return crypto4xx_setkey_aes(cipher, key, keylen,
 310                 CRYPTO_MODE_CTR, CRYPTO_FEEDBACK_MODE_NO_FB);
 311 }
 312
 313 int crypto4xx_encrypt_ctr(struct skcipher_request *req)
 314 {
 315         return crypto4xx_ctr_crypt(req, true);
 316 }
 317
 318 int crypto4xx_decrypt_ctr(struct skcipher_request *req)
 319 {
 320         return crypto4xx_ctr_crypt(req, false);
 321 }
 322
 323 static inline bool crypto4xx_aead_need_fallback(struct aead_request *req,
 324                                                 unsigned int len,
 325                                                 bool is_ccm, bool decrypt)
 326 {
 327         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 328
 329         /* authsize has to be a multiple of 4 */
 330         if (aead->authsize & 3)
 331                 return true;
 332
 333         /*
 334          * hardware does not handle cases where plaintext
 335          * is less than a block.
 336          */
 337         if (len < AES_BLOCK_SIZE)
 338                 return true;
 339
 340         /* assoc len needs to be a multiple of 4 and <= 1020 */
 341         if (req->assoclen & 0x3 || req->assoclen > 1020)
 342                 return true;
 343
 344         /* CCM supports only counter field length of 2 and 4 bytes */
 345         if (is_ccm && !(req->iv[0] == 1 || req->iv[0] == 3))
 346                 return true;
 347
 348         return false;
 349 }
 350
 351 static int crypto4xx_aead_fallback(struct aead_request *req,
 352         struct crypto4xx_ctx *ctx, bool do_decrypt)
 353 {
 354         struct aead_request *subreq = aead_request_ctx(req);
 355
 356         aead_request_set_tfm(subreq, ctx->sw_cipher.aead);
 357         aead_request_set_callback(subreq, req->base.flags,
 358                                   req->base.complete, req->base.data);
 359         aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
 360                                req->iv);
 361         aead_request_set_ad(subreq, req->assoclen);
 362         return do_decrypt ? crypto_aead_decrypt(subreq) :
 363                             crypto_aead_encrypt(subreq);
 364 }
 365
 366 static int crypto4xx_aead_setup_fallback(struct crypto4xx_ctx *ctx,
 367                                          struct crypto_aead *cipher,
 368                                          const u8 *key,
 369                                          unsigned int keylen)
 370 {
 371         crypto_aead_clear_flags(ctx->sw_cipher.aead, CRYPTO_TFM_REQ_MASK);
 372         crypto_aead_set_flags(ctx->sw_cipher.aead,
 373                 crypto_aead_get_flags(cipher) & CRYPTO_TFM_REQ_MASK);
 374         return crypto_aead_setkey(ctx->sw_cipher.aead, key, keylen);
 375 }
 376
 377 /**
 378  * AES-CCM Functions
 379  */
 380
 381 int crypto4xx_setkey_aes_ccm(struct crypto_aead *cipher, const u8 *key,
 382                              unsigned int keylen)
 383 {
 384         struct crypto_tfm *tfm = crypto_aead_tfm(cipher);
 385         struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm);
 386         struct dynamic_sa_ctl *sa;
 387         int rc = 0;
 388
 389         rc = crypto4xx_aead_setup_fallback(ctx, cipher, key, keylen);
 390         if (rc)
 391                 return rc;
 392
 393         if (ctx->sa_in || ctx->sa_out)
 394                 crypto4xx_free_sa(ctx);
 395
 396         rc = crypto4xx_alloc_sa(ctx, SA_AES128_CCM_LEN + (keylen - 16) / 4);
 397         if (rc)
 398                 return rc;
 399
 400         /* Setup SA */
 401         sa = (struct dynamic_sa_ctl *) ctx->sa_in;
 402         sa->sa_contents.w = SA_AES_CCM_CONTENTS | (keylen << 2);
 403
 404         set_dynamic_sa_command_0(sa, SA_SAVE_HASH, SA_NOT_SAVE_IV,
 405                                  SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE,
 406                                  SA_NO_HEADER_PROC, SA_HASH_ALG_CBC_MAC,
 407                                  SA_CIPHER_ALG_AES,
 408                                  SA_PAD_TYPE_ZERO, SA_OP_GROUP_BASIC,
 409                                  SA_OPCODE_HASH_DECRYPT, DIR_INBOUND);
 410
 411         set_dynamic_sa_command_1(sa, CRYPTO_MODE_CTR, SA_HASH_MODE_HASH,
 412                                  CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF,
 413                                  SA_SEQ_MASK_OFF, SA_MC_ENABLE,
 414                                  SA_NOT_COPY_PAD, SA_COPY_PAYLOAD,
 415                                  SA_NOT_COPY_HDR);
 416
 417         sa->sa_command_1.bf.key_len = keylen >> 3;
 418
 419         crypto4xx_memcpy_to_le32(get_dynamic_sa_key_field(sa), key, keylen);
 420
 421         memcpy(ctx->sa_out, ctx->sa_in, ctx->sa_len * 4);
 422         sa = (struct dynamic_sa_ctl *) ctx->sa_out;
 423
 424         set_dynamic_sa_command_0(sa, SA_SAVE_HASH, SA_NOT_SAVE_IV,
 425                                  SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE,
 426                                  SA_NO_HEADER_PROC, SA_HASH_ALG_CBC_MAC,
 427                                  SA_CIPHER_ALG_AES,
 428                                  SA_PAD_TYPE_ZERO, SA_OP_GROUP_BASIC,
 429                                  SA_OPCODE_ENCRYPT_HASH, DIR_OUTBOUND);
 430
 431         set_dynamic_sa_command_1(sa, CRYPTO_MODE_CTR, SA_HASH_MODE_HASH,
 432                                  CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF,
 433                                  SA_SEQ_MASK_OFF, SA_MC_ENABLE,
 434                                  SA_COPY_PAD, SA_COPY_PAYLOAD,
 435                                  SA_NOT_COPY_HDR);
 436
 437         sa->sa_command_1.bf.key_len = keylen >> 3;
 438         return 0;
 439 }
 440
 441 static int crypto4xx_crypt_aes_ccm(struct aead_request *req, bool decrypt)
 442 {
 443         struct crypto4xx_ctx *ctx  = crypto_tfm_ctx(req->base.tfm);
 444         struct crypto4xx_aead_reqctx *rctx = aead_request_ctx(req);
 445         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 446         __le32 iv[16];
 447         u32 tmp_sa[SA_AES128_CCM_LEN + 4];
 448         struct dynamic_sa_ctl *sa = (struct dynamic_sa_ctl *)tmp_sa;
 449         unsigned int len = req->cryptlen;
 450
 451         if (decrypt)
 452                 len -= crypto_aead_authsize(aead);
 453
 454         if (crypto4xx_aead_need_fallback(req, len, true, decrypt))
 455                 return crypto4xx_aead_fallback(req, ctx, decrypt);
 456
 457         memcpy(tmp_sa, decrypt ? ctx->sa_in : ctx->sa_out, ctx->sa_len * 4);
 458         sa->sa_command_0.bf.digest_len = crypto_aead_authsize(aead) >> 2;
 459
 460         if (req->iv[0] == 1) {
 461                 /* CRYPTO_MODE_AES_ICM */
 462                 sa->sa_command_1.bf.crypto_mode9_8 = 1;
 463         }
 464
 465         iv[3] = cpu_to_le32(0);
 466         crypto4xx_memcpy_to_le32(iv, req->iv, 16 - (req->iv[0] + 1));
 467
 468         return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst,
 469                                   len, iv, sizeof(iv),
 470                                   sa, ctx->sa_len, req->assoclen, rctx->dst);
 471 }
 472
 473 int crypto4xx_encrypt_aes_ccm(struct aead_request *req)
 474 {
 475         return crypto4xx_crypt_aes_ccm(req, false);
 476 }
 477
 478 int crypto4xx_decrypt_aes_ccm(struct aead_request *req)
 479 {
 480         return crypto4xx_crypt_aes_ccm(req, true);
 481 }
 482
 483 int crypto4xx_setauthsize_aead(struct crypto_aead *cipher,
 484                                unsigned int authsize)
 485 {
 486         struct crypto_tfm *tfm = crypto_aead_tfm(cipher);
 487         struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm);
 488
 489         return crypto_aead_setauthsize(ctx->sw_cipher.aead, authsize);
 490 }
 491
 492 /**
 493  * AES-GCM Functions
 494  */
 495
 496 static int crypto4xx_aes_gcm_validate_keylen(unsigned int keylen)
 497 {
 498         switch (keylen) {
 499         case 16:
 500         case 24:
 501         case 32:
 502                 return 0;
 503         default:
 504                 return -EINVAL;
 505         }
 506 }
 507
 508 static int crypto4xx_compute_gcm_hash_key_sw(__le32 *hash_start, const u8 *key,
 509                                              unsigned int keylen)
 510 {
 511         struct crypto_aes_ctx ctx;
 512         uint8_t src[16] = { 0 };
 513         int rc;
 514
 515         rc = aes_expandkey(&ctx, key, keylen);
 516         if (rc) {
 517                 pr_err("aes_expandkey() failed: %d\n", rc);
 518                 return rc;
 519         }
 520
 521         aes_encrypt(&ctx, src, src);
 522         crypto4xx_memcpy_to_le32(hash_start, src, 16);
 523         memzero_explicit(&ctx, sizeof(ctx));
 524         return 0;
 525 }
 526
 527 int crypto4xx_setkey_aes_gcm(struct crypto_aead *cipher,
 528                              const u8 *key, unsigned int keylen)
 529 {
 530         struct crypto_tfm *tfm = crypto_aead_tfm(cipher);
 531         struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm);
 532         struct dynamic_sa_ctl *sa;
 533         int    rc = 0;
 534
 535         if (crypto4xx_aes_gcm_validate_keylen(keylen) != 0)
 536                 return -EINVAL;
 537
 538         rc = crypto4xx_aead_setup_fallback(ctx, cipher, key, keylen);
 539         if (rc)
 540                 return rc;
 541
 542         if (ctx->sa_in || ctx->sa_out)
 543                 crypto4xx_free_sa(ctx);
 544
 545         rc = crypto4xx_alloc_sa(ctx, SA_AES128_GCM_LEN + (keylen - 16) / 4);
 546         if (rc)
 547                 return rc;
 548
 549         sa  = (struct dynamic_sa_ctl *) ctx->sa_in;
 550
 551         sa->sa_contents.w = SA_AES_GCM_CONTENTS | (keylen << 2);
 552         set_dynamic_sa_command_0(sa, SA_SAVE_HASH, SA_NOT_SAVE_IV,
 553                                  SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE,
 554                                  SA_NO_HEADER_PROC, SA_HASH_ALG_GHASH,
 555                                  SA_CIPHER_ALG_AES, SA_PAD_TYPE_ZERO,
 556                                  SA_OP_GROUP_BASIC, SA_OPCODE_HASH_DECRYPT,
 557                                  DIR_INBOUND);
 558         set_dynamic_sa_command_1(sa, CRYPTO_MODE_CTR, SA_HASH_MODE_HASH,
 559                                  CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF,
 560                                  SA_SEQ_MASK_ON, SA_MC_DISABLE,
 561                                  SA_NOT_COPY_PAD, SA_COPY_PAYLOAD,
 562                                  SA_NOT_COPY_HDR);
 563
 564         sa->sa_command_1.bf.key_len = keylen >> 3;
 565
 566         crypto4xx_memcpy_to_le32(get_dynamic_sa_key_field(sa),
 567                                  key, keylen);
 568
 569         rc = crypto4xx_compute_gcm_hash_key_sw(get_dynamic_sa_inner_digest(sa),
 570                 key, keylen);
 571         if (rc) {
 572                 pr_err("GCM hash key setting failed = %d\n", rc);
 573                 goto err;
 574         }
 575
 576         memcpy(ctx->sa_out, ctx->sa_in, ctx->sa_len * 4);
 577         sa = (struct dynamic_sa_ctl *) ctx->sa_out;
 578         sa->sa_command_0.bf.dir = DIR_OUTBOUND;
 579         sa->sa_command_0.bf.opcode = SA_OPCODE_ENCRYPT_HASH;
 580
 581         return 0;
 582 err:
 583         crypto4xx_free_sa(ctx);
 584         return rc;
 585 }
 586
 587 static inline int crypto4xx_crypt_aes_gcm(struct aead_request *req,
 588                                           bool decrypt)
 589 {
 590         struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
 591         struct crypto4xx_aead_reqctx *rctx = aead_request_ctx(req);
 592         __le32 iv[4];
 593         unsigned int len = req->cryptlen;
 594
 595         if (decrypt)
 596                 len -= crypto_aead_authsize(crypto_aead_reqtfm(req));
 597
 598         if (crypto4xx_aead_need_fallback(req, len, false, decrypt))
 599                 return crypto4xx_aead_fallback(req, ctx, decrypt);
 600
 601         crypto4xx_memcpy_to_le32(iv, req->iv, GCM_AES_IV_SIZE);
 602         iv[3] = cpu_to_le32(1);
 603
 604         return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst,
 605                                   len, iv, sizeof(iv),
 606                                   decrypt ? ctx->sa_in : ctx->sa_out,
 607                                   ctx->sa_len, req->assoclen, rctx->dst);
 608 }
 609
 610 int crypto4xx_encrypt_aes_gcm(struct aead_request *req)
 611 {
 612         return crypto4xx_crypt_aes_gcm(req, false);
 613 }
 614
 615 int crypto4xx_decrypt_aes_gcm(struct aead_request *req)
 616 {
 617         return crypto4xx_crypt_aes_gcm(req, true);
 618 }
 619
 620 /**
 621  * HASH SHA1 Functions
 622  */
 623 static int crypto4xx_hash_alg_init(struct crypto_tfm *tfm,
 624                                    unsigned int sa_len,
 625                                    unsigned char ha,
 626                                    unsigned char hm)
 627 {
 628         struct crypto_alg *alg = tfm->__crt_alg;
 629         struct crypto4xx_alg *my_alg;
 630         struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm);
 631         struct dynamic_sa_hash160 *sa;
 632         int rc;
 633
 634         my_alg = container_of(__crypto_ahash_alg(alg), struct crypto4xx_alg,
 635                               alg.u.hash);
 636         ctx->dev   = my_alg->dev;
 637
 638         /* Create SA */
 639         if (ctx->sa_in || ctx->sa_out)
 640                 crypto4xx_free_sa(ctx);
 641
 642         rc = crypto4xx_alloc_sa(ctx, sa_len);
 643         if (rc)
 644                 return rc;
 645
 646         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
 647                                  sizeof(struct crypto4xx_ctx));
 648         sa = (struct dynamic_sa_hash160 *)ctx->sa_in;
 649         set_dynamic_sa_command_0(&sa->ctrl, SA_SAVE_HASH, SA_NOT_SAVE_IV,
 650                                  SA_NOT_LOAD_HASH, SA_LOAD_IV_FROM_SA,
 651                                  SA_NO_HEADER_PROC, ha, SA_CIPHER_ALG_NULL,
 652                                  SA_PAD_TYPE_ZERO, SA_OP_GROUP_BASIC,
 653                                  SA_OPCODE_HASH, DIR_INBOUND);
 654         set_dynamic_sa_command_1(&sa->ctrl, 0, SA_HASH_MODE_HASH,
 655                                  CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF,
 656                                  SA_SEQ_MASK_OFF, SA_MC_ENABLE,
 657                                  SA_NOT_COPY_PAD, SA_NOT_COPY_PAYLOAD,
 658                                  SA_NOT_COPY_HDR);
 659         /* Need to zero hash digest in SA */
 660         memset(sa->inner_digest, 0, sizeof(sa->inner_digest));
 661         memset(sa->outer_digest, 0, sizeof(sa->outer_digest));
 662
 663         return 0;
 664 }
 665
 666 int crypto4xx_hash_init(struct ahash_request *req)
 667 {
 668         struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
 669         int ds;
 670         struct dynamic_sa_ctl *sa;
 671
 672         sa = ctx->sa_in;
 673         ds = crypto_ahash_digestsize(
 674                         __crypto_ahash_cast(req->base.tfm));
 675         sa->sa_command_0.bf.digest_len = ds >> 2;
 676         sa->sa_command_0.bf.load_hash_state = SA_LOAD_HASH_FROM_SA;
 677
 678         return 0;
 679 }
 680
 681 int crypto4xx_hash_update(struct ahash_request *req)
 682 {
 683         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 684         struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
 685         struct scatterlist dst;
 686         unsigned int ds = crypto_ahash_digestsize(ahash);
 687
 688         sg_init_one(&dst, req->result, ds);
 689
 690         return crypto4xx_build_pd(&req->base, ctx, req->src, &dst,
 691                                   req->nbytes, NULL, 0, ctx->sa_in,
 692                                   ctx->sa_len, 0, NULL);
 693 }
 694
 695 int crypto4xx_hash_final(struct ahash_request *req)
 696 {
 697         return 0;
 698 }
 699
 700 int crypto4xx_hash_digest(struct ahash_request *req)
 701 {
 702         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 703         struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
 704         struct scatterlist dst;
 705         unsigned int ds = crypto_ahash_digestsize(ahash);
 706
 707         sg_init_one(&dst, req->result, ds);
 708
 709         return crypto4xx_build_pd(&req->base, ctx, req->src, &dst,
 710                                   req->nbytes, NULL, 0, ctx->sa_in,
 711                                   ctx->sa_len, 0, NULL);
 712 }
 713
 714 /**
 715  * SHA1 Algorithm
 716  */
 717 int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm)
 718 {
 719         return crypto4xx_hash_alg_init(tfm, SA_HASH160_LEN, SA_HASH_ALG_SHA1,
 720                                        SA_HASH_MODE_HASH);
 721 }