drivers/crypto/nx/nx-aes-ccm.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /**
   3  * AES CCM routines supporting the Power 7+ Nest Accelerators driver
   4  *
   5  * Copyright (C) 2012 International Business Machines Inc.
   6  *
   7  * Author: Kent Yoder <yoder1@us.ibm.com>
   8  */
   9
  10 #include <crypto/internal/aead.h>
  11 #include <crypto/aes.h>
  12 #include <crypto/algapi.h>
  13 #include <crypto/scatterwalk.h>
  14 #include <linux/module.h>
  15 #include <linux/types.h>
  16 #include <linux/crypto.h>
  17 #include <asm/vio.h>
  18
  19 #include "nx_csbcpb.h"
  20 #include "nx.h"
  21
  22
  23 static int ccm_aes_nx_set_key(struct crypto_aead *tfm,
  24                               const u8           *in_key,
  25                               unsigned int        key_len)
  26 {
  27         struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base);
  28         struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
  29         struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead;
  30
  31         nx_ctx_init(nx_ctx, HCOP_FC_AES);
  32
  33         switch (key_len) {
  34         case AES_KEYSIZE_128:
  35                 NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128);
  36                 NX_CPB_SET_KEY_SIZE(csbcpb_aead, NX_KS_AES_128);
  37                 nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128];
  38                 break;
  39         default:
  40                 return -EINVAL;
  41         }
  42
  43         csbcpb->cpb.hdr.mode = NX_MODE_AES_CCM;
  44         memcpy(csbcpb->cpb.aes_ccm.key, in_key, key_len);
  45
  46         csbcpb_aead->cpb.hdr.mode = NX_MODE_AES_CCA;
  47         memcpy(csbcpb_aead->cpb.aes_cca.key, in_key, key_len);
  48
  49         return 0;
  50
  51 }
  52
  53 static int ccm4309_aes_nx_set_key(struct crypto_aead *tfm,
  54                                   const u8           *in_key,
  55                                   unsigned int        key_len)
  56 {
  57         struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base);
  58
  59         if (key_len < 3)
  60                 return -EINVAL;
  61
  62         key_len -= 3;
  63
  64         memcpy(nx_ctx->priv.ccm.nonce, in_key + key_len, 3);
  65
  66         return ccm_aes_nx_set_key(tfm, in_key, key_len);
  67 }
  68
  69 static int ccm_aes_nx_setauthsize(struct crypto_aead *tfm,
  70                                   unsigned int authsize)
  71 {
  72         switch (authsize) {
  73         case 4:
  74         case 6:
  75         case 8:
  76         case 10:
  77         case 12:
  78         case 14:
  79         case 16:
  80                 break;
  81         default:
  82                 return -EINVAL;
  83         }
  84
  85         return 0;
  86 }
  87
  88 static int ccm4309_aes_nx_setauthsize(struct crypto_aead *tfm,
  89                                       unsigned int authsize)
  90 {
  91         switch (authsize) {
  92         case 8:
  93         case 12:
  94         case 16:
  95                 break;
  96         default:
  97                 return -EINVAL;
  98         }
  99
 100         return 0;
 101 }
 102
 103 /* taken from crypto/ccm.c */
 104 static int set_msg_len(u8 *block, unsigned int msglen, int csize)
 105 {
 106         __be32 data;
 107
 108         memset(block, 0, csize);
 109         block += csize;
 110
 111         if (csize >= 4)
 112                 csize = 4;
 113         else if (msglen > (unsigned int)(1 << (8 * csize)))
 114                 return -EOVERFLOW;
 115
 116         data = cpu_to_be32(msglen);
 117         memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
 118
 119         return 0;
 120 }
 121
 122 /* taken from crypto/ccm.c */
 123 static inline int crypto_ccm_check_iv(const u8 *iv)
 124 {
 125         /* 2 <= L <= 8, so 1 <= L' <= 7. */
 126         if (1 > iv[0] || iv[0] > 7)
 127                 return -EINVAL;
 128
 129         return 0;
 130 }
 131
 132 /* based on code from crypto/ccm.c */
 133 static int generate_b0(u8 *iv, unsigned int assoclen, unsigned int authsize,
 134                        unsigned int cryptlen, u8 *b0)
 135 {
 136         unsigned int l, lp, m = authsize;
 137         int rc;
 138
 139         memcpy(b0, iv, 16);
 140
 141         lp = b0[0];
 142         l = lp + 1;
 143
 144         /* set m, bits 3-5 */
 145         *b0 |= (8 * ((m - 2) / 2));
 146
 147         /* set adata, bit 6, if associated data is used */
 148         if (assoclen)
 149                 *b0 |= 64;
 150
 151         rc = set_msg_len(b0 + 16 - l, cryptlen, l);
 152
 153         return rc;
 154 }
 155
 156 static int generate_pat(u8                   *iv,
 157                         struct aead_request  *req,
 158                         struct nx_crypto_ctx *nx_ctx,
 159                         unsigned int          authsize,
 160                         unsigned int          nbytes,
 161                         unsigned int          assoclen,
 162                         u8                   *out)
 163 {
 164         struct nx_sg *nx_insg = nx_ctx->in_sg;
 165         struct nx_sg *nx_outsg = nx_ctx->out_sg;
 166         unsigned int iauth_len = 0;
 167         u8 tmp[16], *b1 = NULL, *b0 = NULL, *result = NULL;
 168         int rc;
 169         unsigned int max_sg_len;
 170
 171         /* zero the ctr value */
 172         memset(iv + 15 - iv[0], 0, iv[0] + 1);
 173
 174         /* page 78 of nx_wb.pdf has,
 175          * Note: RFC3610 allows the AAD data to be up to 2^64 -1 bytes
 176          * in length. If a full message is used, the AES CCA implementation
 177          * restricts the maximum AAD length to 2^32 -1 bytes.
 178          * If partial messages are used, the implementation supports
 179          * 2^64 -1 bytes maximum AAD length.
 180          *
 181          * However, in the cryptoapi's aead_request structure,
 182          * assoclen is an unsigned int, thus it cannot hold a length
 183          * value greater than 2^32 - 1.
 184          * Thus the AAD is further constrained by this and is never
 185          * greater than 2^32.
 186          */
 187
 188         if (!assoclen) {
 189                 b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0;
 190         } else if (assoclen <= 14) {
 191                 /* if associated data is 14 bytes or less, we do 1 GCM
 192                  * operation on 2 AES blocks, B0 (stored in the csbcpb) and B1,
 193                  * which is fed in through the source buffers here */
 194                 b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0;
 195                 b1 = nx_ctx->priv.ccm.iauth_tag;
 196                 iauth_len = assoclen;
 197         } else if (assoclen <= 65280) {
 198                 /* if associated data is less than (2^16 - 2^8), we construct
 199                  * B1 differently and feed in the associated data to a CCA
 200                  * operation */
 201                 b0 = nx_ctx->csbcpb_aead->cpb.aes_cca.b0;
 202                 b1 = nx_ctx->csbcpb_aead->cpb.aes_cca.b1;
 203                 iauth_len = 14;
 204         } else {
 205                 b0 = nx_ctx->csbcpb_aead->cpb.aes_cca.b0;
 206                 b1 = nx_ctx->csbcpb_aead->cpb.aes_cca.b1;
 207                 iauth_len = 10;
 208         }
 209
 210         /* generate B0 */
 211         rc = generate_b0(iv, assoclen, authsize, nbytes, b0);
 212         if (rc)
 213                 return rc;
 214
 215         /* generate B1:
 216          * add control info for associated data
 217          * RFC 3610 and NIST Special Publication 800-38C
 218          */
 219         if (b1) {
 220                 memset(b1, 0, 16);
 221                 if (assoclen <= 65280) {
 222                         *(u16 *)b1 = assoclen;
 223                         scatterwalk_map_and_copy(b1 + 2, req->src, 0,
 224                                          iauth_len, SCATTERWALK_FROM_SG);
 225                 } else {
 226                         *(u16 *)b1 = (u16)(0xfffe);
 227                         *(u32 *)&b1[2] = assoclen;
 228                         scatterwalk_map_and_copy(b1 + 6, req->src, 0,
 229                                          iauth_len, SCATTERWALK_FROM_SG);
 230                 }
 231         }
 232
 233         /* now copy any remaining AAD to scatterlist and call nx... */
 234         if (!assoclen) {
 235                 return rc;
 236         } else if (assoclen <= 14) {
 237                 unsigned int len = 16;
 238
 239                 nx_insg = nx_build_sg_list(nx_insg, b1, &len, nx_ctx->ap->sglen);
 240
 241                 if (len != 16)
 242                         return -EINVAL;
 243
 244                 nx_outsg = nx_build_sg_list(nx_outsg, tmp, &len,
 245                                             nx_ctx->ap->sglen);
 246
 247                 if (len != 16)
 248                         return -EINVAL;
 249
 250                 /* inlen should be negative, indicating to phyp that its a
 251                  * pointer to an sg list */
 252                 nx_ctx->op.inlen = (nx_ctx->in_sg - nx_insg) *
 253                                         sizeof(struct nx_sg);
 254                 nx_ctx->op.outlen = (nx_ctx->out_sg - nx_outsg) *
 255                                         sizeof(struct nx_sg);
 256
 257                 NX_CPB_FDM(nx_ctx->csbcpb) |= NX_FDM_ENDE_ENCRYPT;
 258                 NX_CPB_FDM(nx_ctx->csbcpb) |= NX_FDM_INTERMEDIATE;
 259
 260                 result = nx_ctx->csbcpb->cpb.aes_ccm.out_pat_or_mac;
 261
 262                 rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
 263                                    req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
 264                 if (rc)
 265                         return rc;
 266
 267                 atomic_inc(&(nx_ctx->stats->aes_ops));
 268                 atomic64_add(assoclen, &nx_ctx->stats->aes_bytes);
 269
 270         } else {
 271                 unsigned int processed = 0, to_process;
 272
 273                 processed += iauth_len;
 274
 275                 /* page_limit: number of sg entries that fit on one page */
 276                 max_sg_len = min_t(u64, nx_ctx->ap->sglen,
 277                                 nx_driver.of.max_sg_len/sizeof(struct nx_sg));
 278                 max_sg_len = min_t(u64, max_sg_len,
 279                                 nx_ctx->ap->databytelen/NX_PAGE_SIZE);
 280
 281                 do {
 282                         to_process = min_t(u32, assoclen - processed,
 283                                            nx_ctx->ap->databytelen);
 284
 285                         nx_insg = nx_walk_and_build(nx_ctx->in_sg,
 286                                                     nx_ctx->ap->sglen,
 287                                                     req->src, processed,
 288                                                     &to_process);
 289
 290                         if ((to_process + processed) < assoclen) {
 291                                 NX_CPB_FDM(nx_ctx->csbcpb_aead) |=
 292                                         NX_FDM_INTERMEDIATE;
 293                         } else {
 294                                 NX_CPB_FDM(nx_ctx->csbcpb_aead) &=
 295                                         ~NX_FDM_INTERMEDIATE;
 296                         }
 297
 298
 299                         nx_ctx->op_aead.inlen = (nx_ctx->in_sg - nx_insg) *
 300                                                 sizeof(struct nx_sg);
 301
 302                         result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0;
 303
 304                         rc = nx_hcall_sync(nx_ctx, &nx_ctx->op_aead,
 305                                    req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
 306                         if (rc)
 307                                 return rc;
 308
 309                         memcpy(nx_ctx->csbcpb_aead->cpb.aes_cca.b0,
 310                                 nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0,
 311                                 AES_BLOCK_SIZE);
 312
 313                         NX_CPB_FDM(nx_ctx->csbcpb_aead) |= NX_FDM_CONTINUATION;
 314
 315                         atomic_inc(&(nx_ctx->stats->aes_ops));
 316                         atomic64_add(assoclen, &nx_ctx->stats->aes_bytes);
 317
 318                         processed += to_process;
 319                 } while (processed < assoclen);
 320
 321                 result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0;
 322         }
 323
 324         memcpy(out, result, AES_BLOCK_SIZE);
 325
 326         return rc;
 327 }
 328
 329 static int ccm_nx_decrypt(struct aead_request   *req,
 330                           u8                    *iv,
 331                           unsigned int assoclen)
 332 {
 333         struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
 334         struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
 335         unsigned int nbytes = req->cryptlen;
 336         unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
 337         struct nx_ccm_priv *priv = &nx_ctx->priv.ccm;
 338         unsigned long irq_flags;
 339         unsigned int processed = 0, to_process;
 340         int rc = -1;
 341
 342         spin_lock_irqsave(&nx_ctx->lock, irq_flags);
 343
 344         nbytes -= authsize;
 345
 346         /* copy out the auth tag to compare with later */
 347         scatterwalk_map_and_copy(priv->oauth_tag,
 348                                  req->src, nbytes + req->assoclen, authsize,
 349                                  SCATTERWALK_FROM_SG);
 350
 351         rc = generate_pat(iv, req, nx_ctx, authsize, nbytes, assoclen,
 352                           csbcpb->cpb.aes_ccm.in_pat_or_b0);
 353         if (rc)
 354                 goto out;
 355
 356         do {
 357
 358                 /* to_process: the AES_BLOCK_SIZE data chunk to process in this
 359                  * update. This value is bound by sg list limits.
 360                  */
 361                 to_process = nbytes - processed;
 362
 363                 if ((to_process + processed) < nbytes)
 364                         NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
 365                 else
 366                         NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
 367
 368                 NX_CPB_FDM(nx_ctx->csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
 369
 370                 rc = nx_build_sg_lists(nx_ctx, iv, req->dst, req->src,
 371                                        &to_process, processed + req->assoclen,
 372                                        csbcpb->cpb.aes_ccm.iv_or_ctr);
 373                 if (rc)
 374                         goto out;
 375
 376                 rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
 377                            req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
 378                 if (rc)
 379                         goto out;
 380
 381                 /* for partial completion, copy following for next
 382                  * entry into loop...
 383                  */
 384                 memcpy(iv, csbcpb->cpb.aes_ccm.out_ctr, AES_BLOCK_SIZE);
 385                 memcpy(csbcpb->cpb.aes_ccm.in_pat_or_b0,
 386                         csbcpb->cpb.aes_ccm.out_pat_or_mac, AES_BLOCK_SIZE);
 387                 memcpy(csbcpb->cpb.aes_ccm.in_s0,
 388                         csbcpb->cpb.aes_ccm.out_s0, AES_BLOCK_SIZE);
 389
 390                 NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
 391
 392                 /* update stats */
 393                 atomic_inc(&(nx_ctx->stats->aes_ops));
 394                 atomic64_add(csbcpb->csb.processed_byte_count,
 395                              &(nx_ctx->stats->aes_bytes));
 396
 397                 processed += to_process;
 398         } while (processed < nbytes);
 399
 400         rc = crypto_memneq(csbcpb->cpb.aes_ccm.out_pat_or_mac, priv->oauth_tag,
 401                     authsize) ? -EBADMSG : 0;
 402 out:
 403         spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
 404         return rc;
 405 }
 406
 407 static int ccm_nx_encrypt(struct aead_request   *req,
 408                           u8                    *iv,
 409                           unsigned int assoclen)
 410 {
 411         struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
 412         struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
 413         unsigned int nbytes = req->cryptlen;
 414         unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
 415         unsigned long irq_flags;
 416         unsigned int processed = 0, to_process;
 417         int rc = -1;
 418
 419         spin_lock_irqsave(&nx_ctx->lock, irq_flags);
 420
 421         rc = generate_pat(iv, req, nx_ctx, authsize, nbytes, assoclen,
 422                           csbcpb->cpb.aes_ccm.in_pat_or_b0);
 423         if (rc)
 424                 goto out;
 425
 426         do {
 427                 /* to process: the AES_BLOCK_SIZE data chunk to process in this
 428                  * update. This value is bound by sg list limits.
 429                  */
 430                 to_process = nbytes - processed;
 431
 432                 if ((to_process + processed) < nbytes)
 433                         NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
 434                 else
 435                         NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
 436
 437                 NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
 438
 439                 rc = nx_build_sg_lists(nx_ctx, iv, req->dst, req->src,
 440                                        &to_process, processed + req->assoclen,
 441                                        csbcpb->cpb.aes_ccm.iv_or_ctr);
 442                 if (rc)
 443                         goto out;
 444
 445                 rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
 446                                    req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
 447                 if (rc)
 448                         goto out;
 449
 450                 /* for partial completion, copy following for next
 451                  * entry into loop...
 452                  */
 453                 memcpy(iv, csbcpb->cpb.aes_ccm.out_ctr, AES_BLOCK_SIZE);
 454                 memcpy(csbcpb->cpb.aes_ccm.in_pat_or_b0,
 455                         csbcpb->cpb.aes_ccm.out_pat_or_mac, AES_BLOCK_SIZE);
 456                 memcpy(csbcpb->cpb.aes_ccm.in_s0,
 457                         csbcpb->cpb.aes_ccm.out_s0, AES_BLOCK_SIZE);
 458
 459                 NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
 460
 461                 /* update stats */
 462                 atomic_inc(&(nx_ctx->stats->aes_ops));
 463                 atomic64_add(csbcpb->csb.processed_byte_count,
 464                              &(nx_ctx->stats->aes_bytes));
 465
 466                 processed += to_process;
 467
 468         } while (processed < nbytes);
 469
 470         /* copy out the auth tag */
 471         scatterwalk_map_and_copy(csbcpb->cpb.aes_ccm.out_pat_or_mac,
 472                                  req->dst, nbytes + req->assoclen, authsize,
 473                                  SCATTERWALK_TO_SG);
 474
 475 out:
 476         spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
 477         return rc;
 478 }
 479
 480 static int ccm4309_aes_nx_encrypt(struct aead_request *req)
 481 {
 482         struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
 483         struct nx_gcm_rctx *rctx = aead_request_ctx(req);
 484         u8 *iv = rctx->iv;
 485
 486         iv[0] = 3;
 487         memcpy(iv + 1, nx_ctx->priv.ccm.nonce, 3);
 488         memcpy(iv + 4, req->iv, 8);
 489
 490         return ccm_nx_encrypt(req, iv, req->assoclen - 8);
 491 }
 492
 493 static int ccm_aes_nx_encrypt(struct aead_request *req)
 494 {
 495         int rc;
 496
 497         rc = crypto_ccm_check_iv(req->iv);
 498         if (rc)
 499                 return rc;
 500
 501         return ccm_nx_encrypt(req, req->iv, req->assoclen);
 502 }
 503
 504 static int ccm4309_aes_nx_decrypt(struct aead_request *req)
 505 {
 506         struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
 507         struct nx_gcm_rctx *rctx = aead_request_ctx(req);
 508         u8 *iv = rctx->iv;
 509
 510         iv[0] = 3;
 511         memcpy(iv + 1, nx_ctx->priv.ccm.nonce, 3);
 512         memcpy(iv + 4, req->iv, 8);
 513
 514         return ccm_nx_decrypt(req, iv, req->assoclen - 8);
 515 }
 516
 517 static int ccm_aes_nx_decrypt(struct aead_request *req)
 518 {
 519         int rc;
 520
 521         rc = crypto_ccm_check_iv(req->iv);
 522         if (rc)
 523                 return rc;
 524
 525         return ccm_nx_decrypt(req, req->iv, req->assoclen);
 526 }
 527
 528 struct aead_alg nx_ccm_aes_alg = {
 529         .base = {
 530                 .cra_name        = "ccm(aes)",
 531                 .cra_driver_name = "ccm-aes-nx",
 532                 .cra_priority    = 300,
 533                 .cra_flags       = CRYPTO_ALG_NEED_FALLBACK,
 534                 .cra_blocksize   = 1,
 535                 .cra_ctxsize     = sizeof(struct nx_crypto_ctx),
 536                 .cra_module      = THIS_MODULE,
 537         },
 538         .init        = nx_crypto_ctx_aes_ccm_init,
 539         .exit        = nx_crypto_ctx_aead_exit,
 540         .ivsize      = AES_BLOCK_SIZE,
 541         .maxauthsize = AES_BLOCK_SIZE,
 542         .setkey      = ccm_aes_nx_set_key,
 543         .setauthsize = ccm_aes_nx_setauthsize,
 544         .encrypt     = ccm_aes_nx_encrypt,
 545         .decrypt     = ccm_aes_nx_decrypt,
 546 };
 547
 548 struct aead_alg nx_ccm4309_aes_alg = {
 549         .base = {
 550                 .cra_name        = "rfc4309(ccm(aes))",
 551                 .cra_driver_name = "rfc4309-ccm-aes-nx",
 552                 .cra_priority    = 300,
 553                 .cra_flags       = CRYPTO_ALG_NEED_FALLBACK,
 554                 .cra_blocksize   = 1,
 555                 .cra_ctxsize     = sizeof(struct nx_crypto_ctx),
 556                 .cra_module      = THIS_MODULE,
 557         },
 558         .init        = nx_crypto_ctx_aes_ccm_init,
 559         .exit        = nx_crypto_ctx_aead_exit,
 560         .ivsize      = 8,
 561         .maxauthsize = AES_BLOCK_SIZE,
 562         .setkey      = ccm4309_aes_nx_set_key,
 563         .setauthsize = ccm4309_aes_nx_setauthsize,
 564         .encrypt     = ccm4309_aes_nx_encrypt,
 565         .decrypt     = ccm4309_aes_nx_decrypt,
 566 };