usr/src/common/crypto/rsa/rsa_impl.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  24  */
  25
  26 /*
  27  * This file contains RSA helper routines common to
  28  * the PKCS11 soft token code and the kernel RSA code.
  29  */
  30
  31 #include <sys/types.h>
  32 #include <bignum.h>
  33
  34 #ifdef _KERNEL
  35 #include <sys/param.h>
  36 #else
  37 #include <strings.h>
  38 #include <cryptoutil.h>
  39 #endif
  40
  41 #include <sys/crypto/common.h>
  42 #include "rsa_impl.h"
  43
  44 /*
  45  * DER encoding T of the DigestInfo values for MD5, SHA1, and SHA2
  46  * from PKCS#1 v2.1: RSA Cryptography Standard Section 9.2 Note 1
  47  *
  48  * MD5:     (0x)30 20 30 0c 06 08 2a 86 48 86 f7 0d 02 05 05 00 04 10 || H
  49  * SHA-1:   (0x)30 21 30 09 06 05 2b 0e 03 02 1a 05 00 04 14 || H
  50  * SHA-256: (0x)30 31 30 0d 06 09 60 86 48 01 65 03 04 02 01 05 00 04 20 || H.
  51  * SHA-384: (0x)30 41 30 0d 06 09 60 86 48 01 65 03 04 02 02 05 00 04 30 || H.
  52  * SHA-512: (0x)30 51 30 0d 06 09 60 86 48 01 65 03 04 02 03 05 00 04 40 || H.
  53  *
  54  * Where H is the digested output from MD5 or SHA1. We define the constant
  55  * byte array (the prefix) here and use it rather than doing the DER
  56  * encoding of the OID in a separate routine.
  57  */
  58 const CK_BYTE MD5_DER_PREFIX[MD5_DER_PREFIX_Len] = {0x30, 0x20, 0x30, 0x0c,
  59     0x06, 0x08, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x02, 0x05, 0x05, 0x00,
  60     0x04, 0x10};
  61
  62 const CK_BYTE SHA1_DER_PREFIX[SHA1_DER_PREFIX_Len] = {0x30, 0x21, 0x30,
  63     0x09, 0x06, 0x05, 0x2b, 0x0e, 0x03, 0x02, 0x1a, 0x05, 0x00, 0x04, 0x14};
  64
  65 const CK_BYTE SHA1_DER_PREFIX_OID[SHA1_DER_PREFIX_OID_Len] = {0x30, 0x1f, 0x30,
  66     0x07, 0x06, 0x05, 0x2b, 0x0e, 0x03, 0x02, 0x1a, 0x04, 0x14};
  67
  68 const CK_BYTE SHA256_DER_PREFIX[SHA2_DER_PREFIX_Len] = {0x30, 0x31, 0x30, 0x0d,
  69     0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05,
  70     0x00, 0x04, 0x20};
  71
  72 const CK_BYTE SHA384_DER_PREFIX[SHA2_DER_PREFIX_Len] = {0x30, 0x41, 0x30, 0x0d,
  73     0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x02, 0x05,
  74     0x00, 0x04, 0x30};
  75
  76 const CK_BYTE SHA512_DER_PREFIX[SHA2_DER_PREFIX_Len] = {0x30, 0x51, 0x30, 0x0d,
  77     0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05,
  78     0x00, 0x04, 0x40};
  79
  80 const CK_BYTE DEFAULT_PUB_EXPO[DEFAULT_PUB_EXPO_Len] = { 0x01, 0x00, 0x01 };
  81
  82
  83 static CK_RV
  84 convert_rv(BIG_ERR_CODE err)
  85 {
  86         switch (err) {
  87
  88         case BIG_OK:
  89                 return (CKR_OK);
  90
  91         case BIG_NO_MEM:
  92                 return (CKR_HOST_MEMORY);
  93
  94         case BIG_NO_RANDOM:
  95                 return (CKR_DEVICE_ERROR);
  96
  97         case BIG_INVALID_ARGS:
  98                 return (CKR_ARGUMENTS_BAD);
  99
 100         case BIG_DIV_BY_0:
 101         default:
 102                 return (CKR_GENERAL_ERROR);
 103         }
 104 }
 105
 106 /* psize and qsize are in bits */
 107 static BIG_ERR_CODE
 108 RSA_key_init(RSAkey *key, int psize, int qsize)
 109 {
 110         BIG_ERR_CODE err = BIG_OK;
 111
 112         int plen, qlen, nlen;
 113
 114         plen = BITLEN2BIGNUMLEN(psize);
 115         qlen = BITLEN2BIGNUMLEN(qsize);
 116         nlen = plen + qlen;
 117         key->size = psize + qsize;
 118         if ((err = big_init(&(key->p), plen)) != BIG_OK)
 119                 return (err);
 120         if ((err = big_init(&(key->q), qlen)) != BIG_OK)
 121                 goto ret1;
 122         if ((err = big_init(&(key->n), nlen)) != BIG_OK)
 123                 goto ret2;
 124         if ((err = big_init(&(key->d), nlen)) != BIG_OK)
 125                 goto ret3;
 126         if ((err = big_init(&(key->e), nlen)) != BIG_OK)
 127                 goto ret4;
 128         if ((err = big_init(&(key->dmodpminus1), plen)) != BIG_OK)
 129                 goto ret5;
 130         if ((err = big_init(&(key->dmodqminus1), qlen)) != BIG_OK)
 131                 goto ret6;
 132         if ((err = big_init(&(key->pinvmodq), qlen)) != BIG_OK)
 133                 goto ret7;
 134         if ((err = big_init(&(key->p_rr), plen)) != BIG_OK)
 135                 goto ret8;
 136         if ((err = big_init(&(key->q_rr), qlen)) != BIG_OK)
 137                 goto ret9;
 138         if ((err = big_init(&(key->n_rr), nlen)) != BIG_OK)
 139                 goto ret10;
 140
 141         return (BIG_OK);
 142
 143 ret10:
 144         big_finish(&(key->q_rr));
 145 ret9:
 146         big_finish(&(key->p_rr));
 147 ret8:
 148         big_finish(&(key->pinvmodq));
 149 ret7:
 150         big_finish(&(key->dmodqminus1));
 151 ret6:
 152         big_finish(&(key->dmodpminus1));
 153 ret5:
 154         big_finish(&(key->e));
 155 ret4:
 156         big_finish(&(key->d));
 157 ret3:
 158         big_finish(&(key->n));
 159 ret2:
 160         big_finish(&(key->q));
 161 ret1:
 162         big_finish(&(key->p));
 163
 164         return (err);
 165 }
 166
 167 static void
 168 RSA_key_finish(RSAkey *key)
 169 {
 170         big_finish(&(key->n_rr));
 171         big_finish(&(key->q_rr));
 172         big_finish(&(key->p_rr));
 173         big_finish(&(key->pinvmodq));
 174         big_finish(&(key->dmodqminus1));
 175         big_finish(&(key->dmodpminus1));
 176         big_finish(&(key->e));
 177         big_finish(&(key->d));
 178         big_finish(&(key->n));
 179         big_finish(&(key->q));
 180         big_finish(&(key->p));
 181 }
 182
 183 /*
 184  * Generate RSA key
 185  */
 186 static CK_RV
 187 generate_rsa_key(RSAkey *key, int psize, int qsize, BIGNUM *pubexp,
 188     int (*rfunc)(void *, size_t))
 189 {
 190         CK_RV           rv = CKR_OK;
 191
 192         int             (*rf)(void *, size_t);
 193         BIGNUM          a, b, c, d, e, f, g, h;
 194         int             len, keylen, size;
 195         BIG_ERR_CODE    brv = BIG_OK;
 196
 197         size = psize + qsize;
 198         keylen = BITLEN2BIGNUMLEN(size);
 199         len = keylen * 2 + 1;
 200         key->size = size;
 201
 202         /*
 203          * Note: It is not really necessary to compute e, it is in pubexp:
 204          *      (void) big_copy(&(key->e), pubexp);
 205          */
 206
 207         a.malloced = 0;
 208         b.malloced = 0;
 209         c.malloced = 0;
 210         d.malloced = 0;
 211         e.malloced = 0;
 212         f.malloced = 0;
 213         g.malloced = 0;
 214         h.malloced = 0;
 215
 216         if ((big_init(&a, len) != BIG_OK) ||
 217             (big_init(&b, len) != BIG_OK) ||
 218             (big_init(&c, len) != BIG_OK) ||
 219             (big_init(&d, len) != BIG_OK) ||
 220             (big_init(&e, len) != BIG_OK) ||
 221             (big_init(&f, len) != BIG_OK) ||
 222             (big_init(&g, len) != BIG_OK) ||
 223             (big_init(&h, len) != BIG_OK)) {
 224                 big_finish(&h);
 225                 big_finish(&g);
 226                 big_finish(&f);
 227                 big_finish(&e);
 228                 big_finish(&d);
 229                 big_finish(&c);
 230                 big_finish(&b);
 231                 big_finish(&a);
 232
 233                 return (CKR_HOST_MEMORY);
 234         }
 235
 236         rf = rfunc;
 237         if (rf == NULL) {
 238 #ifdef _KERNEL
 239                 rf = (int (*)(void *, size_t))random_get_pseudo_bytes;
 240 #else
 241                 rf = pkcs11_get_urandom;
 242 #endif
 243         }
 244
 245 nextp:
 246         if ((brv = big_random(&a, psize, rf)) != BIG_OK) {
 247                 goto ret;
 248         }
 249
 250         if ((brv = big_nextprime_pos(&b, &a)) != BIG_OK) {
 251                 goto ret;
 252         }
 253         /* b now contains the potential prime p */
 254
 255         (void) big_sub_pos(&a, &b, &big_One);
 256         if ((brv = big_ext_gcd_pos(&f, &d, &g, pubexp, &a)) != BIG_OK) {
 257                 goto ret;
 258         }
 259         if (big_cmp_abs(&f, &big_One) != 0) {
 260                 goto nextp;
 261         }
 262
 263         if ((brv = big_random(&c, qsize, rf)) != BIG_OK) {
 264                 goto ret;
 265         }
 266
 267 nextq:
 268         (void) big_add(&a, &c, &big_Two);
 269
 270         if (big_bitlength(&a) != qsize) {
 271                 goto nextp;
 272         }
 273         if (big_cmp_abs(&a, &b) == 0) {
 274                 goto nextp;
 275         }
 276         if ((brv = big_nextprime_pos(&c, &a)) != BIG_OK) {
 277                 goto ret;
 278         }
 279         /* c now contains the potential prime q */
 280
 281         if ((brv = big_mul(&g, &b, &c)) != BIG_OK) {
 282                 goto ret;
 283         }
 284         if (big_bitlength(&g) != size) {
 285                 goto nextp;
 286         }
 287         /* g now contains the potential modulus n */
 288
 289         (void) big_sub_pos(&a, &b, &big_One);
 290         (void) big_sub_pos(&d, &c, &big_One);
 291
 292         if ((brv = big_mul(&a, &a, &d)) != BIG_OK) {
 293                 goto ret;
 294         }
 295         if ((brv = big_ext_gcd_pos(&f, &d, &h, pubexp, &a)) != BIG_OK) {
 296                 goto ret;
 297         }
 298         if (big_cmp_abs(&f, &big_One) != 0) {
 299                 goto nextq;
 300         } else {
 301                 (void) big_copy(&e, pubexp);
 302         }
 303         if (d.sign == -1) {
 304                 if ((brv = big_add(&d, &d, &a)) != BIG_OK) {
 305                         goto ret;
 306                 }
 307         }
 308         (void) big_copy(&(key->p), &b);
 309         (void) big_copy(&(key->q), &c);
 310         (void) big_copy(&(key->n), &g);
 311         (void) big_copy(&(key->d), &d);
 312         (void) big_copy(&(key->e), &e);
 313
 314         if ((brv = big_ext_gcd_pos(&a, &f, &h, &b, &c)) != BIG_OK) {
 315                 goto ret;
 316         }
 317         if (f.sign == -1) {
 318                 if ((brv = big_add(&f, &f, &c)) != BIG_OK) {
 319                         goto ret;
 320                 }
 321         }
 322         (void) big_copy(&(key->pinvmodq), &f);
 323
 324         (void) big_sub(&a, &b, &big_One);
 325         if ((brv = big_div_pos(&a, &f, &d, &a)) != BIG_OK) {
 326                 goto ret;
 327         }
 328         (void) big_copy(&(key->dmodpminus1), &f);
 329         (void) big_sub(&a, &c, &big_One);
 330         if ((brv = big_div_pos(&a, &f, &d, &a)) != BIG_OK) {
 331                 goto ret;
 332         }
 333         (void) big_copy(&(key->dmodqminus1), &f);
 334
 335         /* pairwise consistency check:  decrypt and encrypt restores value */
 336         if ((brv = big_random(&h, size, rf)) != BIG_OK) {
 337                 goto ret;
 338         }
 339         if ((brv = big_div_pos(&a, &h, &h, &g)) != BIG_OK) {
 340                 goto ret;
 341         }
 342         if ((brv = big_modexp(&a, &h, &d, &g, NULL)) != BIG_OK) {
 343                 goto ret;
 344         }
 345
 346         if ((brv = big_modexp(&b, &a, &e, &g, NULL)) != BIG_OK) {
 347                 goto ret;
 348         }
 349
 350         if (big_cmp_abs(&b, &h) != 0) {
 351                 /* this should not happen */
 352                 rv = generate_rsa_key(key, psize, qsize, pubexp, rf);
 353                 goto ret1;
 354         } else {
 355                 brv = BIG_OK;
 356         }
 357
 358 ret:
 359         rv = convert_rv(brv);
 360 ret1:
 361         big_finish(&h);
 362         big_finish(&g);
 363         big_finish(&f);
 364         big_finish(&e);
 365         big_finish(&d);
 366         big_finish(&c);
 367         big_finish(&b);
 368         big_finish(&a);
 369
 370         return (rv);
 371 }
 372
 373 CK_RV
 374 rsa_genkey_pair(RSAbytekey *bkey)
 375 {
 376         /*
 377          * NOTE:  Whomever originally wrote this function swapped p and q.
 378          * This table shows the mapping between name convention used here
 379          * versus what is used in most texts that describe RSA key generation.
 380          *      This function:                  Standard convention:
 381          *      --------------                  --------------------
 382          *      modulus, n                      -same-
 383          *      prime 1, q                      prime 1, p
 384          *      prime 2, p                      prime 2, q
 385          *      private exponent, d             -same-
 386          *      public exponent, e              -same-
 387          *      exponent 1, d mod (q-1)         d mod (p-1)
 388          *      exponent 2, d mod (p-1)         d mod (q-1)
 389          *      coefficient, p^-1 mod q         q^-1 mod p
 390          *
 391          * Also notice the struct member for coefficient is named .pinvmodq
 392          * rather than .qinvmodp, reflecting the switch.
 393          *
 394          * The code here wasn't unswapped, because "it works".  Further,
 395          * p and q are interchangeable as long as exponent 1 and 2 and
 396          * the coefficient are kept straight too.  This note is here to
 397          * make the reader aware of the switcheroo.
 398          */
 399         CK_RV   rv = CKR_OK;
 400
 401         BIGNUM  public_exponent = {0};
 402         RSAkey  rsakey;
 403         uint32_t modulus_bytes;
 404
 405         if (bkey == NULL)
 406                 return (CKR_ARGUMENTS_BAD);
 407
 408         /* Must have modulus bits set */
 409         if (bkey->modulus_bits == 0)
 410                 return (CKR_ARGUMENTS_BAD);
 411
 412         /* Must have public exponent set */
 413         if (bkey->pubexpo_bytes == 0 || bkey->pubexpo == NULL)
 414                 return (CKR_ARGUMENTS_BAD);
 415
 416         /* Note: modulus_bits may not be same as (8 * sizeof (modulus)) */
 417         modulus_bytes = CRYPTO_BITS2BYTES(bkey->modulus_bits);
 418
 419         /* Modulus length needs to be between min key size and max key size. */
 420         if ((modulus_bytes < MIN_RSA_KEYLENGTH_IN_BYTES) ||
 421             (modulus_bytes > MAX_RSA_KEYLENGTH_IN_BYTES)) {
 422                 return (CKR_KEY_SIZE_RANGE);
 423         }
 424
 425         /*
 426          * Initialize the RSA key.
 427          */
 428         if (RSA_key_init(&rsakey, modulus_bytes * 4, modulus_bytes * 4) !=
 429             BIG_OK) {
 430                 return (CKR_HOST_MEMORY);
 431         }
 432
 433         /* Create a public exponent in bignum format. */
 434         if (big_init(&public_exponent,
 435             CHARLEN2BIGNUMLEN(bkey->pubexpo_bytes)) != BIG_OK) {
 436                 rv = CKR_HOST_MEMORY;
 437                 goto clean1;
 438         }
 439         bytestring2bignum(&public_exponent, bkey->pubexpo, bkey->pubexpo_bytes);
 440
 441         /* Generate RSA key pair. */
 442         if ((rv = generate_rsa_key(&rsakey,
 443             modulus_bytes * 4, modulus_bytes * 4, &public_exponent,
 444             bkey->rfunc)) != CKR_OK) {
 445                 big_finish(&public_exponent);
 446                 goto clean1;
 447         }
 448         big_finish(&public_exponent);
 449
 450         /* modulus_bytes = rsakey.n.len * (int)sizeof (BIG_CHUNK_TYPE); */
 451         bignum2bytestring(bkey->modulus, &(rsakey.n), modulus_bytes);
 452
 453         bkey->privexpo_bytes = rsakey.d.len * (int)sizeof (BIG_CHUNK_TYPE);
 454         bignum2bytestring(bkey->privexpo, &(rsakey.d), bkey->privexpo_bytes);
 455
 456         bkey->pubexpo_bytes = rsakey.e.len * (int)sizeof (BIG_CHUNK_TYPE);
 457         bignum2bytestring(bkey->pubexpo, &(rsakey.e), bkey->pubexpo_bytes);
 458
 459         bkey->prime1_bytes = rsakey.q.len * (int)sizeof (BIG_CHUNK_TYPE);
 460         bignum2bytestring(bkey->prime1, &(rsakey.q), bkey->prime1_bytes);
 461
 462         bkey->prime2_bytes = rsakey.p.len * (int)sizeof (BIG_CHUNK_TYPE);
 463         bignum2bytestring(bkey->prime2, &(rsakey.p), bkey->prime2_bytes);
 464
 465         bkey->expo1_bytes =
 466             rsakey.dmodqminus1.len * (int)sizeof (BIG_CHUNK_TYPE);
 467         bignum2bytestring(bkey->expo1, &(rsakey.dmodqminus1),
 468             bkey->expo1_bytes);
 469
 470         bkey->expo2_bytes =
 471             rsakey.dmodpminus1.len * (int)sizeof (BIG_CHUNK_TYPE);
 472         bignum2bytestring(bkey->expo2,
 473             &(rsakey.dmodpminus1), bkey->expo2_bytes);
 474
 475         bkey->coeff_bytes =
 476             rsakey.pinvmodq.len * (int)sizeof (BIG_CHUNK_TYPE);
 477         bignum2bytestring(bkey->coeff, &(rsakey.pinvmodq), bkey->coeff_bytes);
 478
 479 clean1:
 480         RSA_key_finish(&rsakey);
 481
 482         return (rv);
 483 }
 484
 485 /*
 486  * RSA encrypt operation
 487  */
 488 CK_RV
 489 rsa_encrypt(RSAbytekey *bkey, uchar_t *in, uint32_t in_len, uchar_t *out)
 490 {
 491         CK_RV rv = CKR_OK;
 492
 493         BIGNUM msg;
 494         RSAkey rsakey;
 495         uint32_t modulus_bytes;
 496
 497         if (bkey == NULL)
 498                 return (CKR_ARGUMENTS_BAD);
 499
 500         /* Must have modulus and public exponent set */
 501         if (bkey->modulus_bits == 0 || bkey->modulus == NULL ||
 502             bkey->pubexpo_bytes == 0 || bkey->pubexpo == NULL)
 503                 return (CKR_ARGUMENTS_BAD);
 504
 505         /* Note: modulus_bits may not be same as (8 * sizeof (modulus)) */
 506         modulus_bytes = CRYPTO_BITS2BYTES(bkey->modulus_bits);
 507
 508         if (bkey->pubexpo_bytes > modulus_bytes) {
 509                 return (CKR_KEY_SIZE_RANGE);
 510         }
 511
 512         /* psize and qsize for RSA_key_init is in bits. */
 513         if (RSA_key_init(&rsakey, modulus_bytes * 4, modulus_bytes * 4) !=
 514             BIG_OK) {
 515                 return (CKR_HOST_MEMORY);
 516         }
 517
 518         /* Size for big_init is in BIG_CHUNK_TYPE words. */
 519         if (big_init(&msg, CHARLEN2BIGNUMLEN(in_len)) != BIG_OK) {
 520                 rv = CKR_HOST_MEMORY;
 521                 goto clean2;
 522         }
 523         bytestring2bignum(&msg, in, in_len);
 524
 525         /* Convert public exponent and modulus to big integer format. */
 526         bytestring2bignum(&(rsakey.e), bkey->pubexpo, bkey->pubexpo_bytes);
 527         bytestring2bignum(&(rsakey.n), bkey->modulus, modulus_bytes);
 528
 529         if (big_cmp_abs(&msg, &(rsakey.n)) > 0) {
 530                 rv = CKR_DATA_LEN_RANGE;
 531                 goto clean3;
 532         }
 533
 534         /* Perform RSA computation on big integer input data. */
 535         if (big_modexp(&msg, &msg, &(rsakey.e), &(rsakey.n), NULL) !=
 536             BIG_OK) {
 537                 rv = CKR_HOST_MEMORY;
 538                 goto clean3;
 539         }
 540
 541         /* Convert the big integer output data to octet string. */
 542         bignum2bytestring(out, &msg, modulus_bytes);
 543
 544 clean3:
 545         big_finish(&msg);
 546 clean2:
 547         RSA_key_finish(&rsakey);
 548
 549         return (rv);
 550 }
 551
 552 /*
 553  * RSA decrypt operation
 554  */
 555 CK_RV
 556 rsa_decrypt(RSAbytekey *bkey, uchar_t *in, uint32_t in_len, uchar_t *out)
 557 {
 558         CK_RV rv = CKR_OK;
 559
 560         BIGNUM msg;
 561         RSAkey rsakey;
 562         uint32_t modulus_bytes;
 563
 564         if (bkey == NULL)
 565                 return (CKR_ARGUMENTS_BAD);
 566
 567         /* Must have modulus, prime1, prime2, expo1, expo2, and coeff set */
 568         if (bkey->modulus_bits == 0 || bkey->modulus == NULL ||
 569             bkey->prime1_bytes == 0 || bkey->prime1 == NULL ||
 570             bkey->prime2_bytes == 0 || bkey->prime2 == NULL ||
 571             bkey->expo1_bytes == 0 || bkey->expo1 == NULL ||
 572             bkey->expo2_bytes == 0 || bkey->expo2 == NULL ||
 573             bkey->coeff_bytes == 0 || bkey->coeff == NULL)
 574                 return (CKR_ARGUMENTS_BAD);
 575
 576         /* Note: modulus_bits may not be same as (8 * sizeof (modulus)) */
 577         modulus_bytes = CRYPTO_BITS2BYTES(bkey->modulus_bits);
 578
 579         /* psize and qsize for RSA_key_init is in bits. */
 580         if (RSA_key_init(&rsakey, CRYPTO_BYTES2BITS(bkey->prime2_bytes),
 581             CRYPTO_BYTES2BITS(bkey->prime1_bytes)) != BIG_OK) {
 582                 return (CKR_HOST_MEMORY);
 583         }
 584
 585         /* Size for big_init is in BIG_CHUNK_TYPE words. */
 586         if (big_init(&msg, CHARLEN2BIGNUMLEN(in_len)) != BIG_OK) {
 587                 rv = CKR_HOST_MEMORY;
 588                 goto clean3;
 589         }
 590         /* Convert octet string input data to big integer format. */
 591         bytestring2bignum(&msg, in, in_len);
 592
 593         /* Convert octet string modulus to big integer format. */
 594         bytestring2bignum(&(rsakey.n), bkey->modulus, modulus_bytes);
 595
 596         if (big_cmp_abs(&msg, &(rsakey.n)) > 0) {
 597                 rv = CKR_DATA_LEN_RANGE;
 598                 goto clean4;
 599         }
 600
 601         /* Convert the rest of private key attributes to big integer format. */
 602         bytestring2bignum(&(rsakey.q), bkey->prime1, bkey->prime1_bytes);
 603         bytestring2bignum(&(rsakey.p), bkey->prime2, bkey->prime2_bytes);
 604         bytestring2bignum(&(rsakey.dmodqminus1),
 605             bkey->expo1, bkey->expo1_bytes);
 606         bytestring2bignum(&(rsakey.dmodpminus1),
 607             bkey->expo2, bkey->expo2_bytes);
 608         bytestring2bignum(&(rsakey.pinvmodq),
 609             bkey->coeff, bkey->coeff_bytes);
 610
 611         if ((big_cmp_abs(&(rsakey.dmodpminus1), &(rsakey.p)) > 0) ||
 612             (big_cmp_abs(&(rsakey.dmodqminus1), &(rsakey.q)) > 0) ||
 613             (big_cmp_abs(&(rsakey.pinvmodq), &(rsakey.q)) > 0)) {
 614                 rv = CKR_KEY_SIZE_RANGE;
 615                 goto clean4;
 616         }
 617
 618         /* Perform RSA computation on big integer input data. */
 619         if (big_modexp_crt(&msg, &msg, &(rsakey.dmodpminus1),
 620             &(rsakey.dmodqminus1), &(rsakey.p), &(rsakey.q),
 621             &(rsakey.pinvmodq), NULL, NULL) != BIG_OK) {
 622                 rv = CKR_HOST_MEMORY;
 623                 goto clean4;
 624         }
 625
 626         /* Convert the big integer output data to octet string. */
 627         bignum2bytestring(out, &msg, modulus_bytes);
 628
 629 clean4:
 630         big_finish(&msg);
 631 clean3:
 632         RSA_key_finish(&rsakey);
 633
 634         return (rv);
 635 }