security/nss/lib/softoken/rsawrapr.c

   1 /*
   2  * PKCS#1 encoding and decoding functions.
   3  * This file is believed to contain no code licensed from other parties.
   4  *
   5  * ***** BEGIN LICENSE BLOCK *****
   6  * Version: MPL 1.1/GPL 2.0/LGPL 2.1
   7  *
   8  * The contents of this file are subject to the Mozilla Public License Version
   9  * 1.1 (the "License"); you may not use this file except in compliance with
  10  * the License. You may obtain a copy of the License at
  11  * http://www.mozilla.org/MPL/
  12  *
  13  * Software distributed under the License is distributed on an "AS IS" basis,
  14  * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  15  * for the specific language governing rights and limitations under the
  16  * License.
  17  *
  18  * The Original Code is the Netscape security libraries.
  19  *
  20  * The Initial Developer of the Original Code is
  21  * Netscape Communications Corporation.
  22  * Portions created by the Initial Developer are Copyright (C) 1994-2000
  23  * the Initial Developer. All Rights Reserved.
  24  *
  25  * Contributor(s):
  26  *
  27  * Alternatively, the contents of this file may be used under the terms of
  28  * either the GNU General Public License Version 2 or later (the "GPL"), or
  29  * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  30  * in which case the provisions of the GPL or the LGPL are applicable instead
  31  * of those above. If you wish to allow use of your version of this file only
  32  * under the terms of either the GPL or the LGPL, and not to allow others to
  33  * use your version of this file under the terms of the MPL, indicate your
  34  * decision by deleting the provisions above and replace them with the notice
  35  * and other provisions required by the GPL or the LGPL. If you do not delete
  36  * the provisions above, a recipient may use your version of this file under
  37  * the terms of any one of the MPL, the GPL or the LGPL.
  38  *
  39  * ***** END LICENSE BLOCK ***** */
  40 /* $Id: rsawrapr.c,v 1.11 2006/10/23 21:24:38 wtchang%redhat.com Exp $ */
  41
  42 #include "blapi.h"
  43 #include "softoken.h"
  44 #include "sechash.h"
  45
  46 #include "lowkeyi.h"
  47 #include "secerr.h"
  48
  49 #define RSA_BLOCK_MIN_PAD_LEN           8
  50 #define RSA_BLOCK_FIRST_OCTET           0x00
  51 #define RSA_BLOCK_PRIVATE0_PAD_OCTET    0x00
  52 #define RSA_BLOCK_PRIVATE_PAD_OCTET     0xff
  53 #define RSA_BLOCK_AFTER_PAD_OCTET       0x00
  54
  55 #define OAEP_SALT_LEN           8
  56 #define OAEP_PAD_LEN            8
  57 #define OAEP_PAD_OCTET          0x00
  58
  59 #define FLAT_BUFSIZE 512        /* bytes to hold flattened SHA1Context. */
  60
  61 static SHA1Context *
  62 SHA1_CloneContext(SHA1Context *original)
  63 {
  64     SHA1Context *  clone        = NULL;
  65     unsigned char *pBuf;
  66     int            sha1ContextSize = SHA1_FlattenSize(original);
  67     SECStatus      frv;
  68     unsigned char  buf[FLAT_BUFSIZE];
  69
  70     PORT_Assert(sizeof buf >= sha1ContextSize);
  71     if (sizeof buf >= sha1ContextSize) {
  72         pBuf = buf;
  73     } else {
  74         pBuf = PORT_Alloc(sha1ContextSize);
  75         if (!pBuf)
  76             goto done;
  77     }
  78
  79     frv = SHA1_Flatten(original, pBuf);
  80     if (frv == SECSuccess) {
  81         clone = SHA1_Resurrect(pBuf, NULL);
  82         memset(pBuf, 0, sha1ContextSize);
  83     }
  84 done:
  85     if (pBuf != buf)
  86         PORT_Free(pBuf);
  87     return clone;
  88 }
  89
  90 /*
  91  * Modify data by XORing it with a special hash of salt.
  92  */
  93 static SECStatus
  94 oaep_xor_with_h1(unsigned char *data, unsigned int datalen,
  95                  unsigned char *salt, unsigned int saltlen)
  96 {
  97     SHA1Context *sha1cx;
  98     unsigned char *dp, *dataend;
  99     unsigned char end_octet;
 100
 101     sha1cx = SHA1_NewContext();
 102     if (sha1cx == NULL) {
 103         return SECFailure;
 104     }
 105
 106     /*
 107      * Get a hash of salt started; we will use it several times,
 108      * adding in a different end octet (x00, x01, x02, ...).
 109      */
 110     SHA1_Begin (sha1cx);
 111     SHA1_Update (sha1cx, salt, saltlen);
 112     end_octet = 0;
 113
 114     dp = data;
 115     dataend = data + datalen;
 116
 117     while (dp < dataend) {
 118         SHA1Context *sha1cx_h1;
 119         unsigned int sha1len, sha1off;
 120         unsigned char sha1[SHA1_LENGTH];
 121
 122         /*
 123          * Create hash of (salt || end_octet)
 124          */
 125         sha1cx_h1 = SHA1_CloneContext (sha1cx);
 126         SHA1_Update (sha1cx_h1, &end_octet, 1);
 127         SHA1_End (sha1cx_h1, sha1, &sha1len, sizeof(sha1));
 128         SHA1_DestroyContext (sha1cx_h1, PR_TRUE);
 129         PORT_Assert (sha1len == SHA1_LENGTH);
 130
 131         /*
 132          * XOR that hash with the data.
 133          * When we have fewer than SHA1_LENGTH octets of data
 134          * left to xor, use just the low-order ones of the hash.
 135          */
 136         sha1off = 0;
 137         if ((dataend - dp) < SHA1_LENGTH)
 138             sha1off = SHA1_LENGTH - (dataend - dp);
 139         while (sha1off < SHA1_LENGTH)
 140             *dp++ ^= sha1[sha1off++];
 141
 142         /*
 143          * Bump for next hash chunk.
 144          */
 145         end_octet++;
 146     }
 147
 148     SHA1_DestroyContext (sha1cx, PR_TRUE);
 149     return SECSuccess;
 150 }
 151
 152 /*
 153  * Modify salt by XORing it with a special hash of data.
 154  */
 155 static SECStatus
 156 oaep_xor_with_h2(unsigned char *salt, unsigned int saltlen,
 157                  unsigned char *data, unsigned int datalen)
 158 {
 159     unsigned char sha1[SHA1_LENGTH];
 160     unsigned char *psalt, *psha1, *saltend;
 161     SECStatus rv;
 162
 163     /*
 164      * Create a hash of data.
 165      */
 166     rv = SHA1_HashBuf (sha1, data, datalen);
 167     if (rv != SECSuccess) {
 168         return rv;
 169     }
 170
 171     /*
 172      * XOR the low-order octets of that hash with salt.
 173      */
 174     PORT_Assert (saltlen <= SHA1_LENGTH);
 175     saltend = salt + saltlen;
 176     psalt = salt;
 177     psha1 = sha1 + SHA1_LENGTH - saltlen;
 178     while (psalt < saltend) {
 179         *psalt++ ^= *psha1++;
 180     }
 181
 182     return SECSuccess;
 183 }
 184
 185 /*
 186  * Format one block of data for public/private key encryption using
 187  * the rules defined in PKCS #1.
 188  */
 189 static unsigned char *
 190 rsa_FormatOneBlock(unsigned modulusLen, RSA_BlockType blockType,
 191                    SECItem *data)
 192 {
 193     unsigned char *block;
 194     unsigned char *bp;
 195     int padLen;
 196     int i;
 197     SECStatus rv;
 198
 199     block = (unsigned char *) PORT_Alloc(modulusLen);
 200     if (block == NULL)
 201         return NULL;
 202
 203     bp = block;
 204
 205     /*
 206      * All RSA blocks start with two octets:
 207      *  0x00 || BlockType
 208      */
 209     *bp++ = RSA_BLOCK_FIRST_OCTET;
 210     *bp++ = (unsigned char) blockType;
 211
 212     switch (blockType) {
 213
 214       /*
 215        * Blocks intended for private-key operation.
 216        */
 217       case RSA_BlockPrivate0: /* essentially unused */
 218       case RSA_BlockPrivate:     /* preferred method */
 219         /*
 220          * 0x00 || BT || Pad || 0x00 || ActualData
 221          *   1      1   padLen    1      data->len
 222          * Pad is either all 0x00 or all 0xff bytes, depending on blockType.
 223          */
 224         padLen = modulusLen - data->len - 3;
 225         PORT_Assert (padLen >= RSA_BLOCK_MIN_PAD_LEN);
 226         if (padLen < RSA_BLOCK_MIN_PAD_LEN) {
 227             PORT_Free (block);
 228             return NULL;
 229         }
 230         PORT_Memset (bp,
 231                    blockType == RSA_BlockPrivate0
 232                         ? RSA_BLOCK_PRIVATE0_PAD_OCTET
 233                         : RSA_BLOCK_PRIVATE_PAD_OCTET,
 234                    padLen);
 235         bp += padLen;
 236         *bp++ = RSA_BLOCK_AFTER_PAD_OCTET;
 237         PORT_Memcpy (bp, data->data, data->len);
 238         break;
 239
 240       /*
 241        * Blocks intended for public-key operation.
 242        */
 243       case RSA_BlockPublic:
 244
 245         /*
 246          * 0x00 || BT || Pad || 0x00 || ActualData
 247          *   1      1   padLen    1      data->len
 248          * Pad is all non-zero random bytes.
 249          */
 250         padLen = modulusLen - data->len - 3;
 251         PORT_Assert (padLen >= RSA_BLOCK_MIN_PAD_LEN);
 252         if (padLen < RSA_BLOCK_MIN_PAD_LEN) {
 253             PORT_Free (block);
 254             return NULL;
 255         }
 256         for (i = 0; i < padLen; i++) {
 257             /* Pad with non-zero random data. */
 258             do {
 259                 rv = RNG_GenerateGlobalRandomBytes(bp + i, 1);
 260             } while (rv == SECSuccess && bp[i] == RSA_BLOCK_AFTER_PAD_OCTET);
 261             if (rv != SECSuccess) {
 262                 sftk_fatalError = PR_TRUE;
 263                 PORT_Free (block);
 264                 return NULL;
 265             }
 266         }
 267         bp += padLen;
 268         *bp++ = RSA_BLOCK_AFTER_PAD_OCTET;
 269         PORT_Memcpy (bp, data->data, data->len);
 270
 271         break;
 272
 273       /*
 274        * Blocks intended for public-key operation, using
 275        * Optimal Asymmetric Encryption Padding (OAEP).
 276        */
 277       case RSA_BlockOAEP:
 278         /*
 279          * 0x00 || BT || Modified2(Salt) || Modified1(PaddedData)
 280          *   1      1     OAEP_SALT_LEN     OAEP_PAD_LEN + data->len [+ N]
 281          *
 282          * where:
 283          *   PaddedData is "Pad1 || ActualData [|| Pad2]"
 284          *   Salt is random data.
 285          *   Pad1 is all zeros.
 286          *   Pad2, if present, is random data.
 287          *   (The "modified" fields are all the same length as the original
 288          * unmodified values; they are just xor'd with other values.)
 289          *
 290          *   Modified1 is an XOR of PaddedData with a special octet
 291          * string constructed of iterated hashing of Salt (see below).
 292          *   Modified2 is an XOR of Salt with the low-order octets of
 293          * the hash of Modified1 (see farther below ;-).
 294          *
 295          * Whew!
 296          */
 297
 298
 299         /*
 300          * Salt
 301          */
 302         rv = RNG_GenerateGlobalRandomBytes(bp, OAEP_SALT_LEN);
 303         if (rv != SECSuccess) {
 304             sftk_fatalError = PR_TRUE;
 305             PORT_Free (block);
 306             return NULL;
 307         }
 308         bp += OAEP_SALT_LEN;
 309
 310         /*
 311          * Pad1
 312          */
 313         PORT_Memset (bp, OAEP_PAD_OCTET, OAEP_PAD_LEN);
 314         bp += OAEP_PAD_LEN;
 315
 316         /*
 317          * Data
 318          */
 319         PORT_Memcpy (bp, data->data, data->len);
 320         bp += data->len;
 321
 322         /*
 323          * Pad2
 324          */
 325         if (bp < (block + modulusLen)) {
 326             rv = RNG_GenerateGlobalRandomBytes(bp, block - bp + modulusLen);
 327             if (rv != SECSuccess) {
 328                 sftk_fatalError = PR_TRUE;
 329                 PORT_Free (block);
 330                 return NULL;
 331             }
 332         }
 333
 334         /*
 335          * Now we have the following:
 336          * 0x00 || BT || Salt || PaddedData
 337          * (From this point on, "Pad1 || Data [|| Pad2]" is treated
 338          * as the one entity PaddedData.)
 339          *
 340          * We need to turn PaddedData into Modified1.
 341          */
 342         if (oaep_xor_with_h1(block + 2 + OAEP_SALT_LEN,
 343                              modulusLen - 2 - OAEP_SALT_LEN,
 344                              block + 2, OAEP_SALT_LEN) != SECSuccess) {
 345             PORT_Free (block);
 346             return NULL;
 347         }
 348
 349         /*
 350          * Now we have:
 351          * 0x00 || BT || Salt || Modified1(PaddedData)
 352          *
 353          * The remaining task is to turn Salt into Modified2.
 354          */
 355         if (oaep_xor_with_h2(block + 2, OAEP_SALT_LEN,
 356                              block + 2 + OAEP_SALT_LEN,
 357                              modulusLen - 2 - OAEP_SALT_LEN) != SECSuccess) {
 358             PORT_Free (block);
 359             return NULL;
 360         }
 361
 362         break;
 363
 364       default:
 365         PORT_Assert (0);
 366         PORT_Free (block);
 367         return NULL;
 368     }
 369
 370     return block;
 371 }
 372
 373 static SECStatus
 374 rsa_FormatBlock(SECItem *result, unsigned modulusLen,
 375                 RSA_BlockType blockType, SECItem *data)
 376 {
 377     /*
 378      * XXX For now assume that the data length fits in a single
 379      * XXX encryption block; the ASSERTs below force this.
 380      * XXX To fix it, each case will have to loop over chunks whose
 381      * XXX lengths satisfy the assertions, until all data is handled.
 382      * XXX (Unless RSA has more to say about how to handle data
 383      * XXX which does not fit in a single encryption block?)
 384      * XXX And I do not know what the result is supposed to be,
 385      * XXX so the interface to this function may need to change
 386      * XXX to allow for returning multiple blocks, if they are
 387      * XXX not wanted simply concatenated one after the other.
 388      */
 389
 390     switch (blockType) {
 391       case RSA_BlockPrivate0:
 392       case RSA_BlockPrivate:
 393       case RSA_BlockPublic:
 394         /*
 395          * 0x00 || BT || Pad || 0x00 || ActualData
 396          *
 397          * The "3" below is the first octet + the second octet + the 0x00
 398          * octet that always comes just before the ActualData.
 399          */
 400         PORT_Assert (data->len <= (modulusLen - (3 + RSA_BLOCK_MIN_PAD_LEN)));
 401
 402         result->data = rsa_FormatOneBlock(modulusLen, blockType, data);
 403         if (result->data == NULL) {
 404             result->len = 0;
 405             return SECFailure;
 406         }
 407         result->len = modulusLen;
 408
 409         break;
 410
 411       case RSA_BlockOAEP:
 412         /*
 413          * 0x00 || BT || M1(Salt) || M2(Pad1||ActualData[||Pad2])
 414          *
 415          * The "2" below is the first octet + the second octet.
 416          * (The other fields do not contain the clear values, but are
 417          * the same length as the clear values.)
 418          */
 419         PORT_Assert (data->len <= (modulusLen - (2 + OAEP_SALT_LEN
 420                                                  + OAEP_PAD_LEN)));
 421
 422         result->data = rsa_FormatOneBlock(modulusLen, blockType, data);
 423         if (result->data == NULL) {
 424             result->len = 0;
 425             return SECFailure;
 426         }
 427         result->len = modulusLen;
 428
 429         break;
 430
 431       case RSA_BlockRaw:
 432         /*
 433          * Pad || ActualData
 434          * Pad is zeros. The application is responsible for recovering
 435          * the actual data.
 436          */
 437         if (data->len > modulusLen ) {
 438             return SECFailure;
 439         }
 440         result->data = (unsigned char*)PORT_ZAlloc(modulusLen);
 441         result->len = modulusLen;
 442         PORT_Memcpy(result->data+(modulusLen-data->len),data->data,data->len);
 443         break;
 444
 445       default:
 446         PORT_Assert (0);
 447         result->data = NULL;
 448         result->len = 0;
 449         return SECFailure;
 450     }
 451
 452     return SECSuccess;
 453 }
 454
 455 /* XXX Doesn't set error code */
 456 SECStatus
 457 RSA_Sign(NSSLOWKEYPrivateKey *key,
 458          unsigned char *      output,
 459          unsigned int *       output_len,
 460          unsigned int         maxOutputLen,
 461          unsigned char *      input,
 462          unsigned int         input_len)
 463 {
 464     SECStatus     rv          = SECSuccess;
 465     unsigned int  modulus_len = nsslowkey_PrivateModulusLen(key);
 466     SECItem       formatted;
 467     SECItem       unformatted;
 468
 469     if (maxOutputLen < modulus_len)
 470         return SECFailure;
 471     PORT_Assert(key->keyType == NSSLOWKEYRSAKey);
 472     if (key->keyType != NSSLOWKEYRSAKey)
 473         return SECFailure;
 474
 475     unformatted.len  = input_len;
 476     unformatted.data = input;
 477     formatted.data   = NULL;
 478     rv = rsa_FormatBlock(&formatted, modulus_len, RSA_BlockPrivate,
 479                          &unformatted);
 480     if (rv != SECSuccess)
 481         goto done;
 482
 483     rv = RSA_PrivateKeyOpDoubleChecked(&key->u.rsa, output, formatted.data);
 484     if (rv != SECSuccess && PORT_GetError() == SEC_ERROR_LIBRARY_FAILURE) {
 485         sftk_fatalError = PR_TRUE;
 486     }
 487     *output_len = modulus_len;
 488
 489     goto done;
 490
 491 done:
 492     if (formatted.data != NULL)
 493         PORT_ZFree(formatted.data, modulus_len);
 494     return rv;
 495 }
 496
 497 /* XXX Doesn't set error code */
 498 SECStatus
 499 RSA_CheckSign(NSSLOWKEYPublicKey *key,
 500               unsigned char *     sign,
 501               unsigned int        sign_len,
 502               unsigned char *     hash,
 503               unsigned int        hash_len)
 504 {
 505     SECStatus       rv;
 506     unsigned int    modulus_len = nsslowkey_PublicModulusLen(key);
 507     unsigned int    i;
 508     unsigned char * buffer;
 509
 510     modulus_len = nsslowkey_PublicModulusLen(key);
 511     if (sign_len != modulus_len)
 512         goto failure;
 513     /*
 514      * 0x00 || BT || Pad || 0x00 || ActualData
 515      *
 516      * The "3" below is the first octet + the second octet + the 0x00
 517      * octet that always comes just before the ActualData.
 518      */
 519     if (hash_len > modulus_len - (3 + RSA_BLOCK_MIN_PAD_LEN))
 520         goto failure;
 521     PORT_Assert(key->keyType == NSSLOWKEYRSAKey);
 522     if (key->keyType != NSSLOWKEYRSAKey)
 523         goto failure;
 524
 525     buffer = (unsigned char *)PORT_Alloc(modulus_len + 1);
 526     if (!buffer)
 527         goto failure;
 528
 529     rv = RSA_PublicKeyOp(&key->u.rsa, buffer, sign);
 530     if (rv != SECSuccess)
 531         goto loser;
 532
 533     /*
 534      * check the padding that was used
 535      */
 536     if (buffer[0] != 0 || buffer[1] != 1)
 537         goto loser;
 538     for (i = 2; i < modulus_len - hash_len - 1; i++) {
 539         if (buffer[i] != 0xff)
 540             goto loser;
 541     }
 542     if (buffer[i] != 0)
 543         goto loser;
 544
 545     /*
 546      * make sure we get the same results
 547      */
 548     if (PORT_Memcmp(buffer + modulus_len - hash_len, hash, hash_len) != 0)
 549         goto loser;
 550
 551     PORT_Free(buffer);
 552     return SECSuccess;
 553
 554 loser:
 555     PORT_Free(buffer);
 556 failure:
 557     return SECFailure;
 558 }
 559
 560 /* XXX Doesn't set error code */
 561 SECStatus
 562 RSA_CheckSignRecover(NSSLOWKEYPublicKey *key,
 563                      unsigned char *     data,
 564                      unsigned int *      data_len,
 565                      unsigned int        max_output_len,
 566                      unsigned char *     sign,
 567                      unsigned int        sign_len)
 568 {
 569     SECStatus       rv;
 570     unsigned int    modulus_len = nsslowkey_PublicModulusLen(key);
 571     unsigned int    i;
 572     unsigned char * buffer;
 573
 574     if (sign_len != modulus_len)
 575         goto failure;
 576     PORT_Assert(key->keyType == NSSLOWKEYRSAKey);
 577     if (key->keyType != NSSLOWKEYRSAKey)
 578         goto failure;
 579
 580     buffer = (unsigned char *)PORT_Alloc(modulus_len + 1);
 581     if (!buffer)
 582         goto failure;
 583
 584     rv = RSA_PublicKeyOp(&key->u.rsa, buffer, sign);
 585     if (rv != SECSuccess)
 586         goto loser;
 587     *data_len = 0;
 588
 589     /*
 590      * check the padding that was used
 591      */
 592     if (buffer[0] != 0 || buffer[1] != 1)
 593         goto loser;
 594     for (i = 2; i < modulus_len; i++) {
 595         if (buffer[i] == 0) {
 596             *data_len = modulus_len - i - 1;
 597             break;
 598         }
 599         if (buffer[i] != 0xff)
 600             goto loser;
 601     }
 602     if (*data_len == 0)
 603         goto loser;
 604     if (*data_len > max_output_len)
 605         goto loser;
 606
 607     /*
 608      * make sure we get the same results
 609      */
 610     PORT_Memcpy(data,buffer + modulus_len - *data_len, *data_len);
 611
 612     PORT_Free(buffer);
 613     return SECSuccess;
 614
 615 loser:
 616     PORT_Free(buffer);
 617 failure:
 618     return SECFailure;
 619 }
 620
 621 /* XXX Doesn't set error code */
 622 SECStatus
 623 RSA_EncryptBlock(NSSLOWKEYPublicKey *key,
 624                  unsigned char *     output,
 625                  unsigned int *      output_len,
 626                  unsigned int        max_output_len,
 627                  unsigned char *     input,
 628                  unsigned int        input_len)
 629 {
 630     SECStatus     rv;
 631     unsigned int  modulus_len = nsslowkey_PublicModulusLen(key);
 632     SECItem       formatted;
 633     SECItem       unformatted;
 634
 635     formatted.data = NULL;
 636     if (max_output_len < modulus_len)
 637         goto failure;
 638     PORT_Assert(key->keyType == NSSLOWKEYRSAKey);
 639     if (key->keyType != NSSLOWKEYRSAKey)
 640         goto failure;
 641
 642     unformatted.len  = input_len;
 643     unformatted.data = input;
 644     formatted.data   = NULL;
 645     rv = rsa_FormatBlock(&formatted, modulus_len, RSA_BlockPublic,
 646                          &unformatted);
 647     if (rv != SECSuccess)
 648         goto failure;
 649
 650     rv = RSA_PublicKeyOp(&key->u.rsa, output, formatted.data);
 651     if (rv != SECSuccess)
 652         goto failure;
 653
 654     PORT_ZFree(formatted.data, modulus_len);
 655     *output_len = modulus_len;
 656     return SECSuccess;
 657
 658 failure:
 659     if (formatted.data != NULL)
 660         PORT_ZFree(formatted.data, modulus_len);
 661     return SECFailure;
 662 }
 663
 664 /* XXX Doesn't set error code */
 665 SECStatus
 666 RSA_DecryptBlock(NSSLOWKEYPrivateKey *key,
 667                  unsigned char *      output,
 668                  unsigned int *       output_len,
 669                  unsigned int         max_output_len,
 670                  unsigned char *      input,
 671                  unsigned int         input_len)
 672 {
 673     SECStatus       rv;
 674     unsigned int    modulus_len = nsslowkey_PrivateModulusLen(key);
 675     unsigned int    i;
 676     unsigned char * buffer;
 677
 678     PORT_Assert(key->keyType == NSSLOWKEYRSAKey);
 679     if (key->keyType != NSSLOWKEYRSAKey)
 680         goto failure;
 681     if (input_len != modulus_len)
 682         goto failure;
 683
 684     buffer = (unsigned char *)PORT_Alloc(modulus_len + 1);
 685     if (!buffer)
 686         goto failure;
 687
 688     rv = RSA_PrivateKeyOp(&key->u.rsa, buffer, input);
 689     if (rv != SECSuccess) {
 690         if (PORT_GetError() == SEC_ERROR_LIBRARY_FAILURE) {
 691             sftk_fatalError = PR_TRUE;
 692         }
 693         goto loser;
 694     }
 695
 696     if (buffer[0] != 0 || buffer[1] != 2)
 697         goto loser;
 698     *output_len = 0;
 699     for (i = 2; i < modulus_len; i++) {
 700         if (buffer[i] == 0) {
 701             *output_len = modulus_len - i - 1;
 702             break;
 703         }
 704     }
 705     if (*output_len == 0)
 706         goto loser;
 707     if (*output_len > max_output_len)
 708         goto loser;
 709
 710     PORT_Memcpy(output, buffer + modulus_len - *output_len, *output_len);
 711
 712     PORT_Free(buffer);
 713     return SECSuccess;
 714
 715 loser:
 716     PORT_Free(buffer);
 717 failure:
 718     return SECFailure;
 719 }
 720
 721 /* XXX Doesn't set error code */
 722 /*
 723  * added to make pkcs #11 happy
 724  *   RAW is RSA_X_509
 725  */
 726 SECStatus
 727 RSA_SignRaw(NSSLOWKEYPrivateKey *key,
 728             unsigned char *      output,
 729             unsigned int *       output_len,
 730             unsigned int         maxOutputLen,
 731             unsigned char *      input,
 732             unsigned int         input_len)
 733 {
 734     SECStatus    rv          = SECSuccess;
 735     unsigned int modulus_len = nsslowkey_PrivateModulusLen(key);
 736     SECItem      formatted;
 737     SECItem      unformatted;
 738
 739     if (maxOutputLen < modulus_len)
 740         return SECFailure;
 741     PORT_Assert(key->keyType == NSSLOWKEYRSAKey);
 742     if (key->keyType != NSSLOWKEYRSAKey)
 743         return SECFailure;
 744
 745     unformatted.len  = input_len;
 746     unformatted.data = input;
 747     formatted.data   = NULL;
 748     rv = rsa_FormatBlock(&formatted, modulus_len, RSA_BlockRaw, &unformatted);
 749     if (rv != SECSuccess)
 750         goto done;
 751
 752     rv = RSA_PrivateKeyOpDoubleChecked(&key->u.rsa, output, formatted.data);
 753     if (rv != SECSuccess && PORT_GetError() == SEC_ERROR_LIBRARY_FAILURE) {
 754         sftk_fatalError = PR_TRUE;
 755     }
 756     *output_len = modulus_len;
 757
 758 done:
 759     if (formatted.data != NULL)
 760         PORT_ZFree(formatted.data, modulus_len);
 761     return rv;
 762 }
 763
 764 /* XXX Doesn't set error code */
 765 SECStatus
 766 RSA_CheckSignRaw(NSSLOWKEYPublicKey *key,
 767                  unsigned char *     sign,
 768                  unsigned int        sign_len,
 769                  unsigned char *     hash,
 770                  unsigned int        hash_len)
 771 {
 772     SECStatus       rv;
 773     unsigned int    modulus_len = nsslowkey_PublicModulusLen(key);
 774     unsigned char * buffer;
 775
 776     if (sign_len != modulus_len)
 777         goto failure;
 778     if (hash_len > modulus_len)
 779         goto failure;
 780     PORT_Assert(key->keyType == NSSLOWKEYRSAKey);
 781     if (key->keyType != NSSLOWKEYRSAKey)
 782         goto failure;
 783
 784     buffer = (unsigned char *)PORT_Alloc(modulus_len + 1);
 785     if (!buffer)
 786         goto failure;
 787
 788     rv = RSA_PublicKeyOp(&key->u.rsa, buffer, sign);
 789     if (rv != SECSuccess)
 790         goto loser;
 791
 792     /*
 793      * make sure we get the same results
 794      */
 795     /* NOTE: should we verify the leading zeros? */
 796     if (PORT_Memcmp(buffer + (modulus_len-hash_len), hash, hash_len) != 0)
 797         goto loser;
 798
 799     PORT_Free(buffer);
 800     return SECSuccess;
 801
 802 loser:
 803     PORT_Free(buffer);
 804 failure:
 805     return SECFailure;
 806 }
 807
 808 /* XXX Doesn't set error code */
 809 SECStatus
 810 RSA_CheckSignRecoverRaw(NSSLOWKEYPublicKey *key,
 811                         unsigned char *     data,
 812                         unsigned int *      data_len,
 813                         unsigned int        max_output_len,
 814                         unsigned char *     sign,
 815                         unsigned int        sign_len)
 816 {
 817     SECStatus      rv;
 818     unsigned int   modulus_len = nsslowkey_PublicModulusLen(key);
 819
 820     if (sign_len != modulus_len)
 821         goto failure;
 822     if (max_output_len < modulus_len)
 823         goto failure;
 824     PORT_Assert(key->keyType == NSSLOWKEYRSAKey);
 825     if (key->keyType != NSSLOWKEYRSAKey)
 826         goto failure;
 827
 828     rv = RSA_PublicKeyOp(&key->u.rsa, data, sign);
 829     if (rv != SECSuccess)
 830         goto failure;
 831
 832     *data_len = modulus_len;
 833     return SECSuccess;
 834
 835 failure:
 836     return SECFailure;
 837 }
 838
 839
 840 /* XXX Doesn't set error code */
 841 SECStatus
 842 RSA_EncryptRaw(NSSLOWKEYPublicKey *key,
 843                unsigned char *     output,
 844                unsigned int *      output_len,
 845                unsigned int        max_output_len,
 846                unsigned char *     input,
 847                unsigned int        input_len)
 848 {
 849     SECStatus rv;
 850     unsigned int  modulus_len = nsslowkey_PublicModulusLen(key);
 851     SECItem       formatted;
 852     SECItem       unformatted;
 853
 854     formatted.data = NULL;
 855     if (max_output_len < modulus_len)
 856         goto failure;
 857     PORT_Assert(key->keyType == NSSLOWKEYRSAKey);
 858     if (key->keyType != NSSLOWKEYRSAKey)
 859         goto failure;
 860
 861     unformatted.len  = input_len;
 862     unformatted.data = input;
 863     formatted.data   = NULL;
 864     rv = rsa_FormatBlock(&formatted, modulus_len, RSA_BlockRaw, &unformatted);
 865     if (rv != SECSuccess)
 866         goto failure;
 867
 868     rv = RSA_PublicKeyOp(&key->u.rsa, output, formatted.data);
 869     if (rv != SECSuccess)
 870         goto failure;
 871
 872     PORT_ZFree(formatted.data, modulus_len);
 873     *output_len = modulus_len;
 874     return SECSuccess;
 875
 876 failure:
 877     if (formatted.data != NULL)
 878         PORT_ZFree(formatted.data, modulus_len);
 879     return SECFailure;
 880 }
 881
 882 /* XXX Doesn't set error code */
 883 SECStatus
 884 RSA_DecryptRaw(NSSLOWKEYPrivateKey *key,
 885                unsigned char *      output,
 886                unsigned int *       output_len,
 887                unsigned int         max_output_len,
 888                unsigned char *      input,
 889                unsigned int         input_len)
 890 {
 891     SECStatus     rv;
 892     unsigned int  modulus_len = nsslowkey_PrivateModulusLen(key);
 893
 894     if (modulus_len <= 0)
 895         goto failure;
 896     if (modulus_len > max_output_len)
 897         goto failure;
 898     PORT_Assert(key->keyType == NSSLOWKEYRSAKey);
 899     if (key->keyType != NSSLOWKEYRSAKey)
 900         goto failure;
 901     if (input_len != modulus_len)
 902         goto failure;
 903
 904     rv = RSA_PrivateKeyOp(&key->u.rsa, output, input);
 905     if (rv != SECSuccess) {
 906         if (PORT_GetError() == SEC_ERROR_LIBRARY_FAILURE) {
 907             sftk_fatalError = PR_TRUE;
 908         }
 909         goto failure;
 910     }
 911
 912     *output_len = modulus_len;
 913     return SECSuccess;
 914
 915 failure:
 916     return SECFailure;
 917 }