usr/src/cmd/keyserv/xcrypt.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License, Version 1.0 only
   6  * (the "License").  You may not use this file except in compliance
   7  * with the License.
   8  *
   9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  10  * or http://www.opensolaris.org/os/licensing.
  11  * See the License for the specific language governing permissions
  12  * and limitations under the License.
  13  *
  14  * When distributing Covered Code, include this CDDL HEADER in each
  15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  16  * If applicable, add the following below this CDDL HEADER, with the
  17  * fields enclosed by brackets "[]" replaced with your own identifying
  18  * information: Portions Copyright [yyyy] [name of copyright owner]
  19  *
  20  * CDDL HEADER END
  21  */
  22
  23 /*
  24  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  25  * Use is subject to license terms.
  26  */
  27
  28 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
  29 /* All Rights Reserved */
  30
  31 /*
  32  * Portions of this source code were derived from Berkeley 4.3 BSD
  33  * under license from the Regents of the University of California.
  34  */
  35
  36 /*
  37  * Hex encryption/decryption and utility routines
  38  */
  39
  40 #include <stdio.h>
  41 #include <stdlib.h>
  42 #include <sys/types.h>
  43 #include <rpc/rpc.h>
  44 #include <rpc/key_prot.h>   /* for KEYCHECKSUMSIZE */
  45 #include <rpc/des_crypt.h>
  46 #include <string.h>
  47 #include <rpcsvc/nis_dhext.h>
  48 #include <md5.h>
  49
  50 #define MD5HEXSIZE      32
  51
  52 static int bin2hex(int len, unsigned char *binnum, char *hexnum);
  53 static int hex2bin(int len, char *hexnum, char *binnum);
  54 static char hex[];      /* forward */
  55 static char hexval();
  56
  57 int passwd2des(char *, char *);
  58 static int weak_DES_key(des_block);
  59
  60 /*
  61  * For export control reasons, we want to limit the maximum size of
  62  * data that can be encrypted or decrypted.  We limit this to 1024
  63  * bits of key data, which amounts to 128 bytes.
  64  *
  65  * For the extended DH project, we have increased it to
  66  * 144 bytes (128key + 16checksum) to accomadate all the 128 bytes
  67  * being used by the new 1024bit keys plus 16 bytes MD5 checksum.
  68  * We discussed this with Sun's export control office and lawyers
  69  * and we have reason to believe this is ok for export.
  70  */
  71 #define MAX_KEY_CRYPT_LEN       144
  72
  73 /*
  74  * Encrypt a secret key given passwd
  75  * The secret key is passed and returned in hex notation.
  76  * Its length must be a multiple of 16 hex digits (64 bits).
  77  */
  78 int
  79 xencrypt(secret, passwd)
  80         char *secret;
  81         char *passwd;
  82 {
  83         char key[8];
  84         char ivec[8];
  85         char *buf;
  86         int err;
  87         int len;
  88
  89         len = (int)strlen(secret) / 2;
  90         if (len > MAX_KEY_CRYPT_LEN)
  91                 return (0);
  92         buf = malloc((unsigned)len);
  93         (void) hex2bin(len, secret, buf);
  94         (void) passwd2des(passwd, key);
  95         (void) memset(ivec, 0, 8);
  96
  97         err = cbc_crypt(key, buf, len, DES_ENCRYPT | DES_HW, ivec);
  98         if (DES_FAILED(err)) {
  99                 free(buf);
 100                 return (0);
 101         }
 102         (void) bin2hex(len, (unsigned char *) buf, secret);
 103         free(buf);
 104         return (1);
 105 }
 106
 107 /*
 108  * Decrypt secret key using passwd
 109  * The secret key is passed and returned in hex notation.
 110  * Once again, the length is a multiple of 16 hex digits
 111  */
 112 int
 113 xdecrypt(secret, passwd)
 114         char *secret;
 115         char *passwd;
 116 {
 117         char key[8];
 118         char ivec[8];
 119         char *buf;
 120         int err;
 121         int len;
 122
 123         len = (int)strlen(secret) / 2;
 124         if (len > MAX_KEY_CRYPT_LEN)
 125                 return (0);
 126         buf = malloc((unsigned)len);
 127
 128         (void) hex2bin(len, secret, buf);
 129         (void) passwd2des(passwd, key);
 130         (void) memset(ivec, 0, 8);
 131
 132         err = cbc_crypt(key, buf, len, DES_DECRYPT | DES_HW, ivec);
 133         if (DES_FAILED(err)) {
 134                 free(buf);
 135                 return (0);
 136         }
 137         (void) bin2hex(len, (unsigned char *) buf, secret);
 138         free(buf);
 139         return (1);
 140 }
 141
 142 /*
 143  * Turn password into DES key
 144  */
 145 int
 146 passwd2des(pw, key)
 147         char *pw;
 148         char *key;
 149 {
 150         int i;
 151
 152         (void) memset(key, 0, 8);
 153         for (i = 0; *pw; i = (i+1) % 8) {
 154                 key[i] ^= *pw++ << 1;
 155         }
 156         des_setparity(key);
 157         return (1);
 158 }
 159
 160
 161 /*
 162  * Hex to binary conversion
 163  */
 164 static int
 165 hex2bin(len, hexnum, binnum)
 166         int len;
 167         char *hexnum;
 168         char *binnum;
 169 {
 170         int i;
 171
 172         for (i = 0; i < len; i++) {
 173                 *binnum++ = 16 * hexval(hexnum[2 * i]) +
 174                                         hexval(hexnum[2 * i + 1]);
 175         }
 176         return (1);
 177 }
 178
 179 /*
 180  * Binary to hex conversion
 181  */
 182 static int
 183 bin2hex(len, binnum, hexnum)
 184         int len;
 185         unsigned char *binnum;
 186         char *hexnum;
 187 {
 188         int i;
 189         unsigned val;
 190
 191         for (i = 0; i < len; i++) {
 192                 val = binnum[i];
 193                 hexnum[i*2] = hex[val >> 4];
 194                 hexnum[i*2+1] = hex[val & 0xf];
 195         }
 196         hexnum[len*2] = 0;
 197         return (1);
 198 }
 199
 200 static char hex[16] = {
 201         '0', '1', '2', '3', '4', '5', '6', '7',
 202         '8', '9', 'a', 'b', 'c', 'd', 'e', 'f',
 203 };
 204
 205 static char
 206 hexval(c)
 207         char c;
 208 {
 209         if (c >= '0' && c <= '9') {
 210                 return (c - '0');
 211         } else if (c >= 'a' && c <= 'z') {
 212                 return (c - 'a' + 10);
 213         } else if (c >= 'A' && c <= 'Z') {
 214                 return (c - 'A' + 10);
 215         } else {
 216                 return (-1);
 217         }
 218 }
 219
 220 /*
 221  * Generic key length/algorithm version of xencrypt().
 222  *
 223  * Encrypt a secret key given passwd.
 224  * The secret key is passed in hex notation.
 225  * Arg encrypted_secret will be set to point to the encrypted
 226  * secret key (NUL term, hex notation).
 227  *
 228  * Its length must be a multiple of 16 hex digits (64 bits).
 229  *
 230  * For 192-0 (AUTH_DES), then encrypt using the same method as xencrypt().
 231  *
 232  * If arg do_chksum is TRUE, append the checksum before the encrypt.
 233  * For 192-0, the checksum is done the same as in xencrypt().  For
 234  * bigger keys, MD5 is used.
 235  *
 236  * Arg netname can be NULL for 192-0.
 237  */
 238 int
 239 xencrypt_g(
 240         char *secret,                   /* in  */
 241         keylen_t keylen,                /* in  */
 242         algtype_t algtype,              /* in  */
 243         const char *passwd,             /* in  */
 244         const char netname[],           /* in  */
 245         char **encrypted_secret,        /* out */
 246         bool_t do_chksum)               /* in  */
 247 {
 248         des_block key;
 249         char ivec[8];
 250         char *binkeybuf;
 251         int err;
 252         const int classic_des = keylen == 192 && algtype == 0;
 253         const int hexkeybytes = BITS2NIBBLES(keylen);
 254         const int keychecksumsize = classic_des ? KEYCHECKSUMSIZE : MD5HEXSIZE;
 255         const int binkeybytes = do_chksum ? keylen/8 + keychecksumsize/2 :
 256                 keylen/8;
 257         const int bufsize = do_chksum ? hexkeybytes + keychecksumsize + 1 :
 258                 hexkeybytes + 1;
 259         char *hexkeybuf;
 260
 261         if (!secret || !keylen || !passwd || !encrypted_secret)
 262                 return (0);
 263
 264         if ((hexkeybuf = malloc(bufsize)) == 0)
 265                 return (0);
 266
 267         (void) memcpy(hexkeybuf, secret, hexkeybytes);
 268         if (do_chksum)
 269                 if (classic_des) {
 270                         (void) memcpy(hexkeybuf + hexkeybytes, secret,
 271                                         keychecksumsize);
 272                 } else {
 273                         MD5_CTX md5_ctx;
 274                         char md5hexbuf[MD5HEXSIZE + 1] = {0};
 275                         uint8_t digest[MD5HEXSIZE/2];
 276
 277                         MD5Init(&md5_ctx);
 278                         MD5Update(&md5_ctx, (unsigned char *)hexkeybuf,
 279                                         hexkeybytes);
 280                         MD5Final(digest, &md5_ctx);
 281
 282                         /* convert md5 binary digest to hex */
 283                         (void) bin2hex(MD5HEXSIZE/2, digest, md5hexbuf);
 284
 285                         /* append the hex md5 string to the end of the key */
 286                         (void) memcpy(hexkeybuf + hexkeybytes,
 287                                         (void *)md5hexbuf, MD5HEXSIZE);
 288                 }
 289         hexkeybuf[bufsize - 1] = 0;
 290
 291         if (binkeybytes > MAX_KEY_CRYPT_LEN) {
 292                 free(hexkeybuf);
 293                 return (0);
 294         }
 295         if ((binkeybuf = malloc((unsigned)binkeybytes)) == 0) {
 296                 free(hexkeybuf);
 297                 return (0);
 298         }
 299
 300         (void) hex2bin(binkeybytes, hexkeybuf, binkeybuf);
 301         if (classic_des)
 302                 (void) passwd2des((char *)passwd, key.c);
 303         else
 304                 if (netname)
 305                         (void) passwd2des_g(passwd, netname,
 306                                         (int)strlen(netname), &key, FALSE);
 307                 else {
 308                         free(hexkeybuf);
 309                         return (0);
 310                 }
 311
 312         (void) memset(ivec, 0, 8);
 313
 314         err = cbc_crypt(key.c, binkeybuf, binkeybytes, DES_ENCRYPT | DES_HW,
 315                         ivec);
 316         if (DES_FAILED(err)) {
 317                 free(hexkeybuf);
 318                 free(binkeybuf);
 319                 return (0);
 320         }
 321         (void) bin2hex(binkeybytes, (unsigned char *) binkeybuf, hexkeybuf);
 322         free(binkeybuf);
 323         *encrypted_secret = hexkeybuf;
 324         return (1);
 325 }
 326
 327 /*
 328  * Generic key len and alg type for version of xdecrypt.
 329  *
 330  * Decrypt secret key using passwd.  The decrypted secret key
 331  * *overwrites* the supplied encrypted secret key.
 332  * The secret key is passed and returned in hex notation.
 333  * Once again, the length is a multiple of 16 hex digits.
 334  *
 335  * If 'do_chksum' is TRUE, the 'secret' buffer is assumed to contain
 336  * a checksum calculated by a call to xencrypt_g().
 337  *
 338  * If keylen is 192 and algtype is 0, then decrypt the same way
 339  * as xdecrypt().
 340  *
 341  * Arg netname can be NULL for 192-0.
 342  */
 343 int
 344 xdecrypt_g(
 345         char *secret,           /* out  */
 346         int keylen,             /* in  */
 347         int algtype,            /* in  */
 348         const char *passwd,     /* in  */
 349         const char netname[],   /* in  */
 350         bool_t do_chksum)       /* in  */
 351 {
 352         des_block key;
 353         char ivec[8];
 354         char *buf;
 355         int err;
 356         int len;
 357         const int classic_des = keylen == 192 && algtype == 0;
 358         const int hexkeybytes = BITS2NIBBLES(keylen);
 359         const int keychecksumsize = classic_des ? KEYCHECKSUMSIZE : MD5HEXSIZE;
 360
 361         len = (int)strlen(secret) / 2;
 362         if (len > MAX_KEY_CRYPT_LEN)
 363                 return (0);
 364         if ((buf = malloc((unsigned)len)) == 0)
 365                 return (0);
 366
 367         (void) hex2bin(len, secret, buf);
 368         if (classic_des)
 369                 (void) passwd2des((char *)passwd, key.c);
 370         else
 371                 if (netname)
 372                         (void) passwd2des_g(passwd, netname,
 373                                         (int)strlen(netname), &key, FALSE);
 374                 else {
 375                         free(buf);
 376                         return (0);
 377                 }
 378         (void) memset(ivec, 0, 8);
 379
 380         err = cbc_crypt(key.c, buf, len, DES_DECRYPT | DES_HW, ivec);
 381         if (DES_FAILED(err)) {
 382                 free(buf);
 383                 return (0);
 384         }
 385         (void) bin2hex(len, (unsigned char *) buf, secret);
 386         free(buf);
 387
 388         if (do_chksum)
 389                 if (classic_des) {
 390                         if (memcmp(secret, &(secret[hexkeybytes]),
 391                                         keychecksumsize) != 0) {
 392                                 secret[0] = 0;
 393                                 return (0);
 394                         }
 395                 } else {
 396                         MD5_CTX md5_ctx;
 397                         char md5hexbuf[MD5HEXSIZE + 1] = {0};
 398                         uint8_t digest[MD5HEXSIZE/2];
 399
 400                         MD5Init(&md5_ctx);
 401                         MD5Update(&md5_ctx, (unsigned char *)secret,
 402                                         hexkeybytes);
 403                         MD5Final(digest, &md5_ctx);
 404
 405                         /* convert md5 binary digest to hex */
 406                         (void) bin2hex(MD5HEXSIZE/2, digest, md5hexbuf);
 407
 408                         /* does the digest match the appended one? */
 409                         if (memcmp(&(secret[hexkeybytes]),
 410                                         md5hexbuf, MD5HEXSIZE) != 0) {
 411                                 secret[0] = 0;
 412                                 return (0);
 413                         }
 414                 }
 415
 416         secret[hexkeybytes] = '\0';
 417
 418         return (1);
 419 }
 420
 421
 422 /*
 423  * Modified version of passwd2des(). passwd2des_g() uses the Kerberos
 424  * RFC 1510 algorithm to generate a DES key from a user password
 425  * and mix-in string. The mix-in is expected to be the netname.
 426  * This function to be used only for extended Diffie-Hellman keys.
 427  *
 428  * If altarg is TRUE, reverse the concat of passwd and mix-in.
 429  */
 430 int
 431 passwd2des_g(
 432         const char *pw,
 433         const char *mixin,
 434         int len,
 435         des_block *key, /* out */
 436         bool_t altalg)
 437 {
 438
 439         int  i, j, incr = 1;
 440         des_block ivec, tkey;
 441         char *text;
 442         int  plen, tlen;
 443
 444         (void) memset(tkey.c, 0, 8);
 445         (void) memset(ivec.c, 0, 8);
 446
 447
 448 /*
 449  * Concatentate the password and the mix-in string, fan-fold and XOR them
 450  * to the required eight byte initial DES key. Since passwords can be
 451  * expected to use mostly seven bit ASCII, left shift the password one
 452  * bit in order to preserve as much key space as possible.
 453  */
 454
 455 #define KEYLEN sizeof (tkey.c)
 456         plen = strlen(pw);
 457         tlen = ((plen + len + (KEYLEN-1))/KEYLEN)*KEYLEN;
 458         if ((text = malloc(tlen)) == NULL) {
 459                 return (0);
 460         }
 461
 462         (void) memset(text, 0, tlen);
 463
 464         if (!altalg) {
 465
 466 /*
 467  * Concatenate the password and the mix-in string, fan-fold and XOR them
 468  * to the required eight byte initial DES key. Since passwords can be
 469  * expected to use mostly seven bit ASCII, left shift the password one
 470  * bit in order to preserve as much key space as possible.
 471  */
 472                 (void) memcpy(text, pw, plen);
 473                 (void) memcpy(&text[plen], mixin, len);
 474
 475                 for (i = 0, j = 0; pw[j]; j++) {
 476                         tkey.c[i] ^= pw[j] << 1;
 477                         i += incr;
 478                         if (i == 8) {
 479                                 i = 7;
 480                                 incr = -incr;
 481                         } else if (i == -1) {
 482                                 i = 0;
 483                                 incr = -incr;
 484                         }
 485                 }
 486
 487                 for (j = 0; j < len; j++) {
 488                         tkey.c[i] ^= mixin[j];
 489                         i += incr;
 490                         if (i == 8) {
 491                                 i = 7;
 492                                 incr = -incr;
 493                         } else if (i == -1) {
 494                                 i = 0;
 495                                 incr = -incr;
 496                         }
 497                 }
 498         } else {  /* use alternative algorithm */
 499                 (void) memcpy(text, mixin, len);
 500                 (void) memcpy(&text[len], pw, plen);
 501
 502                 for (i = 0, j = 0; j < len; j++) {
 503                         tkey.c[i] ^= mixin[j];
 504                         i += incr;
 505                         if (i == 8) {
 506                                 i = 7;
 507                                 incr = -incr;
 508                         } else if (i == -1) {
 509                                 i = 0;
 510                                 incr = -incr;
 511                         }
 512                 }
 513
 514                 for (j = 0; pw[j]; j++) {
 515                         tkey.c[i] ^= pw[j] << 1;
 516                         i += incr;
 517                         if (i == 8) {
 518                                 i = 7;
 519                                 incr = -incr;
 520                         } else if (i == -1) {
 521                                 i = 0;
 522                                 incr = -incr;
 523                         }
 524                 }
 525         }
 526         des_setparity_g(&tkey);
 527
 528         /*
 529          * Use the temporary key to produce a DES CBC checksum for the text
 530          * string; cbc_crypt returns the checksum in the ivec.
 531          */
 532         (void) cbc_crypt(tkey.c, text, tlen, DES_ENCRYPT|DES_HW, ivec.c);
 533         des_setparity_g(&ivec);
 534         free(text);
 535
 536         if (weak_DES_key(ivec)) {
 537                 ivec.c[7] ^= 0xf0;
 538                 /*
 539                  *  XORing with 0xf0 preserves parity, so no need to check
 540                  *  that again.
 541                  */
 542         }
 543
 544         (void) memcpy((*key).c, ivec.c, sizeof (ivec.c));
 545
 546         return (1);
 547
 548 }
 549
 550 struct DESkey {
 551         uint32_t h1;
 552         uint32_t h2;
 553 };
 554
 555 /*
 556  * Weak and semiweak keys from "Applied Cryptography", second edition,
 557  * by Bruce Schneier, Wiley 1996.
 558  */
 559 static struct DESkey weakDESkeys[] = {
 560         /* Weak keys */
 561         {0x01010101, 0x01010101},
 562         {0x1f1f1f1f, 0x1f1f1f1f},
 563         {0xe0e0e0e0, 0xe0e0e0e0},
 564         {0xfefefefe, 0xfefefefe},
 565         /* Semiweak keys */
 566         {0x01fe01fe, 0x01fe01fe},
 567         {0x1fe01fe0, 0x0ef10ef1},
 568         {0x01e001e0, 0x01f101f1},
 569         {0x1ffe1ffe, 0x0efe0efe},
 570         {0x011f011f, 0x010e010e},
 571         {0xe0fee0fe, 0xf1fef1fe},
 572         {0xfe01fe01, 0xfe01fe01},
 573         {0xe01fe01f, 0xf10ef10e},
 574         {0xe001e001, 0xf101f101},
 575         {0xfe1ffe1f, 0xfe0efe0e},
 576         {0x1f011f01, 0x0e010e01},
 577         {0xfee0fee0, 0xfef1fef1}
 578 };
 579
 580 static int
 581 weak_DES_key(des_block db)
 582 {
 583         int i;
 584
 585         for (i = 0; i < sizeof (weakDESkeys)/sizeof (struct DESkey); i++) {
 586                 if (weakDESkeys[i].h1 == db.key.high &&
 587                         weakDESkeys[i].h2 == db.key.low)
 588                         return (1);
 589         }
 590
 591         return (0);
 592 }