crypto/dist/heimdal/lib/hcrypto/rsa.c

   1 /*
   2  * Copyright (c) 2006 - 2008 Kungliga Tekniska Högskolan
   3  * (Royal Institute of Technology, Stockholm, Sweden).
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  *
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  *
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * 3. Neither the name of the Institute nor the names of its contributors
  18  *    may be used to endorse or promote products derived from this software
  19  *    without specific prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND
  22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24  * ARE DISCLAIMED.  IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE
  25  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  31  * SUCH DAMAGE.
  32  */
  33
  34 #ifdef HAVE_CONFIG_H
  35 #include <config.h>
  36 #endif
  37
  38 __RCSID("$Heimdal: rsa.c 22422 2008-01-13 09:43:59Z lha $"
  39         "$NetBSD$");
  40
  41 #include <stdio.h>
  42 #include <stdlib.h>
  43 #include <krb5-types.h>
  44 #include <rfc2459_asn1.h>
  45
  46 #include <rsa.h>
  47
  48 #include <roken.h>
  49
  50 /**
  51  * @page page_rsa RSA - public-key cryptography
  52  *
  53  * RSA is named by its inventors (Ron Rivest, Adi Shamir, and Leonard
  54  * Adleman) (published in 1977), patented expired in 21 September 2000.
  55  *
  56  * See the library functions here: @ref hcrypto_rsa
  57  */
  58
  59 /**
  60  * Same as RSA_new_method() using NULL as engine.
  61  *
  62  * @return a newly allocated RSA object. Free with RSA_free().
  63  *
  64  * @ingroup hcrypto_rsa
  65  */
  66
  67 RSA *
  68 RSA_new(void)
  69 {
  70     return RSA_new_method(NULL);
  71 }
  72
  73 /**
  74  * Allocate a new RSA object using the engine, if NULL is specified as
  75  * the engine, use the default RSA engine as returned by
  76  * ENGINE_get_default_RSA().
  77  *
  78  * @param engine Specific what ENGINE RSA provider should be used.
  79  *
  80  * @return a newly allocated RSA object. Free with RSA_free().
  81  *
  82  * @ingroup hcrypto_rsa
  83  */
  84
  85 RSA *
  86 RSA_new_method(ENGINE *engine)
  87 {
  88     RSA *rsa;
  89
  90     rsa = calloc(1, sizeof(*rsa));
  91     if (rsa == NULL)
  92         return NULL;
  93
  94     rsa->references = 1;
  95
  96     if (engine) {
  97         ENGINE_up_ref(engine);
  98         rsa->engine = engine;
  99     } else {
 100         rsa->engine = ENGINE_get_default_RSA();
 101     }
 102
 103     if (rsa->engine) {
 104         rsa->meth = ENGINE_get_RSA(rsa->engine);
 105         if (rsa->meth == NULL) {
 106             ENGINE_finish(engine);
 107             free(rsa);
 108             return 0;
 109         }
 110     }
 111
 112     if (rsa->meth == NULL)
 113         rsa->meth = rk_UNCONST(RSA_get_default_method());
 114
 115     (*rsa->meth->init)(rsa);
 116
 117     return rsa;
 118 }
 119
 120 /**
 121  * Free an allocation RSA object.
 122  *
 123  * @param rsa the RSA object to free.
 124  * @ingroup hcrypto_rsa
 125  */
 126
 127 void
 128 RSA_free(RSA *rsa)
 129 {
 130     if (rsa->references <= 0)
 131         abort();
 132
 133     if (--rsa->references > 0)
 134         return;
 135
 136     (*rsa->meth->finish)(rsa);
 137
 138     if (rsa->engine)
 139         ENGINE_finish(rsa->engine);
 140
 141 #define free_if(f) if (f) { BN_free(f); }
 142     free_if(rsa->n);
 143     free_if(rsa->e);
 144     free_if(rsa->d);
 145     free_if(rsa->p);
 146     free_if(rsa->q);
 147     free_if(rsa->dmp1);
 148     free_if(rsa->dmq1);
 149     free_if(rsa->iqmp);
 150 #undef free_if
 151
 152     memset(rsa, 0, sizeof(*rsa));
 153     free(rsa);
 154 }
 155
 156 /**
 157  * Add an extra reference to the RSA object. The object should be free
 158  * with RSA_free() to drop the reference.
 159  *
 160  * @param rsa the object to add reference counting too.
 161  *
 162  * @return the current reference count, can't safely be used except
 163  * for debug printing.
 164  *
 165  * @ingroup hcrypto_rsa
 166  */
 167
 168 int
 169 RSA_up_ref(RSA *rsa)
 170 {
 171     return ++rsa->references;
 172 }
 173
 174 /**
 175  * Return the RSA_METHOD used for this RSA object.
 176  *
 177  * @param rsa the object to get the method from.
 178  *
 179  * @return the method used for this RSA object.
 180  *
 181  * @ingroup hcrypto_rsa
 182  */
 183
 184 const RSA_METHOD *
 185 RSA_get_method(const RSA *rsa)
 186 {
 187     return rsa->meth;
 188 }
 189
 190 /**
 191  * Set a new method for the RSA keypair.
 192  *
 193  * @param rsa rsa parameter.
 194  * @param method the new method for the RSA parameter.
 195  *
 196  * @return 1 on success.
 197  *
 198  * @ingroup hcrypto_rsa
 199  */
 200
 201 int
 202 RSA_set_method(RSA *rsa, const RSA_METHOD *method)
 203 {
 204     (*rsa->meth->finish)(rsa);
 205
 206     if (rsa->engine) {
 207         ENGINE_finish(rsa->engine);
 208         rsa->engine = NULL;
 209     }
 210
 211     rsa->meth = method;
 212     (*rsa->meth->init)(rsa);
 213     return 1;
 214 }
 215
 216 /**
 217  * Set the application data for the RSA object.
 218  *
 219  * @param rsa the rsa object to set the parameter for
 220  * @param arg the data object to store
 221  *
 222  * @return 1 on success.
 223  *
 224  * @ingroup hcrypto_rsa
 225  */
 226
 227 int
 228 RSA_set_app_data(RSA *rsa, void *arg)
 229 {
 230     rsa->ex_data.sk = arg;
 231     return 1;
 232 }
 233
 234 /**
 235  * Get the application data for the RSA object.
 236  *
 237  * @param rsa the rsa object to get the parameter for
 238  *
 239  * @return the data object
 240  *
 241  * @ingroup hcrypto_rsa
 242  */
 243
 244 void *
 245 RSA_get_app_data(RSA *rsa)
 246 {
 247     return rsa->ex_data.sk;
 248 }
 249
 250 int
 251 RSA_check_key(const RSA *key)
 252 {
 253     static const unsigned char inbuf[] = "hello, world!";
 254     RSA *rsa = rk_UNCONST(key);
 255     void *buffer;
 256     int ret;
 257
 258     /*
 259      * XXX I have no clue how to implement this w/o a bignum library.
 260      * Well, when we have a RSA key pair, we can try to encrypt/sign
 261      * and then decrypt/verify.
 262      */
 263
 264     if ((rsa->d == NULL || rsa->n == NULL) &&
 265         (rsa->p == NULL || rsa->q || rsa->dmp1 == NULL || rsa->dmq1 == NULL || rsa->iqmp == NULL))
 266         return 0;
 267
 268     buffer = malloc(RSA_size(rsa));
 269     if (buffer == NULL)
 270         return 0;
 271
 272     ret = RSA_private_encrypt(sizeof(inbuf), inbuf, buffer,
 273                              rsa, RSA_PKCS1_PADDING);
 274     if (ret == -1) {
 275         free(buffer);
 276         return 0;
 277     }
 278
 279     ret = RSA_public_decrypt(ret, buffer, buffer,
 280                               rsa, RSA_PKCS1_PADDING);
 281     if (ret == -1) {
 282         free(buffer);
 283         return 0;
 284     }
 285
 286     if (ret == sizeof(inbuf) && memcmp(buffer, inbuf, sizeof(inbuf)) == 0) {
 287         free(buffer);
 288         return 1;
 289     }
 290     free(buffer);
 291     return 0;
 292 }
 293
 294 int
 295 RSA_size(const RSA *rsa)
 296 {
 297     return BN_num_bytes(rsa->n);
 298 }
 299
 300 #define RSAFUNC(name, body) \
 301 int \
 302 name(int flen,const unsigned char* f, unsigned char* t, RSA* r, int p){\
 303     return body; \
 304 }
 305
 306 RSAFUNC(RSA_public_encrypt, (r)->meth->rsa_pub_enc(flen, f, t, r, p))
 307 RSAFUNC(RSA_public_decrypt, (r)->meth->rsa_pub_dec(flen, f, t, r, p))
 308 RSAFUNC(RSA_private_encrypt, (r)->meth->rsa_priv_enc(flen, f, t, r, p))
 309 RSAFUNC(RSA_private_decrypt, (r)->meth->rsa_priv_dec(flen, f, t, r, p))
 310
 311 /* XXX */
 312 int
 313 RSA_sign(int type, const unsigned char *from, unsigned int flen,
 314          unsigned char *to, unsigned int *tlen, RSA *rsa)
 315 {
 316     return -1;
 317 }
 318
 319 int
 320 RSA_verify(int type, const unsigned char *from, unsigned int flen,
 321            unsigned char *to, unsigned int tlen, RSA *rsa)
 322 {
 323     return -1;
 324 }
 325
 326 /*
 327  * A NULL RSA_METHOD that returns failure for all operations. This is
 328  * used as the default RSA method if we don't have any native
 329  * support.
 330  */
 331
 332 static RSAFUNC(null_rsa_public_encrypt, -1)
 333 static RSAFUNC(null_rsa_public_decrypt, -1)
 334 static RSAFUNC(null_rsa_private_encrypt, -1)
 335 static RSAFUNC(null_rsa_private_decrypt, -1)
 336
 337 /*
 338  *
 339  */
 340
 341 int
 342 RSA_generate_key_ex(RSA *r, int bits, BIGNUM *e, BN_GENCB *cb)
 343 {
 344     if (r->meth->rsa_keygen)
 345         return (*r->meth->rsa_keygen)(r, bits, e, cb);
 346     return 0;
 347 }
 348
 349
 350 /*
 351  *
 352  */
 353
 354 static int
 355 null_rsa_init(RSA *rsa)
 356 {
 357     return 1;
 358 }
 359
 360 static int
 361 null_rsa_finish(RSA *rsa)
 362 {
 363     return 1;
 364 }
 365
 366 static const RSA_METHOD rsa_null_method = {
 367     "hcrypto null RSA",
 368     null_rsa_public_encrypt,
 369     null_rsa_public_decrypt,
 370     null_rsa_private_encrypt,
 371     null_rsa_private_decrypt,
 372     NULL,
 373     NULL,
 374     null_rsa_init,
 375     null_rsa_finish,
 376     0,
 377     NULL,
 378     NULL,
 379     NULL
 380 };
 381
 382 const RSA_METHOD *
 383 RSA_null_method(void)
 384 {
 385     return &rsa_null_method;
 386 }
 387
 388 extern const RSA_METHOD hc_rsa_imath_method;
 389 #ifdef HAVE_GMP
 390 static const RSA_METHOD *default_rsa_method = &hc_rsa_gmp_method;
 391 #else
 392 static const RSA_METHOD *default_rsa_method = &hc_rsa_imath_method;
 393 #endif
 394
 395 const RSA_METHOD *
 396 RSA_get_default_method(void)
 397 {
 398     return default_rsa_method;
 399 }
 400
 401 void
 402 RSA_set_default_method(const RSA_METHOD *meth)
 403 {
 404     default_rsa_method = meth;
 405 }
 406
 407 /*
 408  *
 409  */
 410
 411 static BIGNUM *
 412 heim_int2BN(const heim_integer *i)
 413 {
 414     BIGNUM *bn;
 415
 416     bn = BN_bin2bn(i->data, i->length, NULL);
 417     if (bn)
 418         BN_set_negative(bn, i->negative);
 419     return bn;
 420 }
 421
 422 static int
 423 bn2heim_int(BIGNUM *bn, heim_integer *integer)
 424 {
 425     integer->length = BN_num_bytes(bn);
 426     integer->data = malloc(integer->length);
 427     if (integer->data == NULL) {
 428         integer->length = 0;
 429         return ENOMEM;
 430     }
 431     BN_bn2bin(bn, integer->data);
 432     integer->negative = BN_is_negative(bn);
 433     return 0;
 434 }
 435
 436
 437 RSA *
 438 d2i_RSAPrivateKey(RSA *rsa, const unsigned char **pp, size_t len)
 439 {
 440     RSAPrivateKey data;
 441     RSA *k = rsa;
 442     size_t size;
 443     int ret;
 444
 445     ret = decode_RSAPrivateKey(*pp, len, &data, &size);
 446     if (ret)
 447         return NULL;
 448
 449     *pp += size;
 450
 451     if (k == NULL) {
 452         k = RSA_new();
 453         if (k == NULL) {
 454             free_RSAPrivateKey(&data);
 455             return NULL;
 456         }
 457     }
 458
 459     k->n = heim_int2BN(&data.modulus);
 460     k->e = heim_int2BN(&data.publicExponent);
 461     k->d = heim_int2BN(&data.privateExponent);
 462     k->p = heim_int2BN(&data.prime1);
 463     k->q = heim_int2BN(&data.prime2);
 464     k->dmp1 = heim_int2BN(&data.exponent1);
 465     k->dmq1 = heim_int2BN(&data.exponent2);
 466     k->iqmp = heim_int2BN(&data.coefficient);
 467     free_RSAPrivateKey(&data);
 468
 469     if (k->n == NULL || k->e == NULL || k->d == NULL || k->p == NULL ||
 470         k->q == NULL || k->dmp1 == NULL || k->dmq1 == NULL || k->iqmp == NULL)
 471     {
 472         RSA_free(k);
 473         return NULL;
 474     }
 475
 476     return k;
 477 }
 478
 479 int
 480 i2d_RSAPrivateKey(RSA *rsa, unsigned char **pp)
 481 {
 482     RSAPrivateKey data;
 483     size_t size;
 484     int ret;
 485
 486     if (rsa->n == NULL || rsa->e == NULL || rsa->d == NULL || rsa->p == NULL ||
 487         rsa->q == NULL || rsa->dmp1 == NULL || rsa->dmq1 == NULL ||
 488         rsa->iqmp == NULL)
 489         return -1;
 490
 491     memset(&data, 0, sizeof(data));
 492
 493     ret  = bn2heim_int(rsa->n, &data.modulus);
 494     ret |= bn2heim_int(rsa->e, &data.publicExponent);
 495     ret |= bn2heim_int(rsa->d, &data.privateExponent);
 496     ret |= bn2heim_int(rsa->p, &data.prime1);
 497     ret |= bn2heim_int(rsa->q, &data.prime2);
 498     ret |= bn2heim_int(rsa->dmp1, &data.exponent1);
 499     ret |= bn2heim_int(rsa->dmq1, &data.exponent2);
 500     ret |= bn2heim_int(rsa->iqmp, &data.coefficient);
 501     if (ret) {
 502         free_RSAPrivateKey(&data);
 503         return -1;
 504     }
 505
 506     if (pp == NULL) {
 507         size = length_RSAPrivateKey(&data);
 508         free_RSAPrivateKey(&data);
 509     } else {
 510         void *p;
 511         size_t len;
 512
 513         ASN1_MALLOC_ENCODE(RSAPrivateKey, p, len, &data, &size, ret);
 514         free_RSAPrivateKey(&data);
 515         if (ret)
 516             return -1;
 517         if (len != size)
 518             abort();
 519
 520         memcpy(*pp, p, size);
 521         free(p);
 522
 523         *pp += size;
 524
 525     }
 526     return size;
 527 }
 528
 529 int
 530 i2d_RSAPublicKey(RSA *rsa, unsigned char **pp)
 531 {
 532     RSAPublicKey data;
 533     size_t size;
 534     int ret;
 535
 536     memset(&data, 0, sizeof(data));
 537
 538     if (bn2heim_int(rsa->n, &data.modulus) ||
 539         bn2heim_int(rsa->e, &data.publicExponent))
 540     {
 541         free_RSAPublicKey(&data);
 542         return -1;
 543     }
 544
 545     if (pp == NULL) {
 546         size = length_RSAPublicKey(&data);
 547         free_RSAPublicKey(&data);
 548     } else {
 549         void *p;
 550         size_t len;
 551
 552         ASN1_MALLOC_ENCODE(RSAPublicKey, p, len, &data, &size, ret);
 553         free_RSAPublicKey(&data);
 554         if (ret)
 555             return -1;
 556         if (len != size)
 557             abort();
 558
 559         memcpy(*pp, p, size);
 560         free(p);
 561
 562         *pp += size;
 563     }
 564
 565     return size;
 566 }