usr/src/lib/smbsrv/libsmb/common/smb_crypt.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  * Copyright 2012 Nexenta Systems, Inc.  All rights reserved.
  26  */
  27
  28 #include <sys/md4.h>
  29 #include <sys/types.h>
  30 #include <string.h>
  31 #include <security/cryptoki.h>
  32 #include <security/pkcs11.h>
  33 #include <cryptoutil.h>
  34 #include <smbsrv/libsmb.h>
  35
  36 static void smb_initlmkey(unsigned char *keyin, unsigned char *keyout);
  37
  38 /*
  39  * randomize
  40  *
  41  * Randomize the contents of the specified buffer.
  42  */
  43 void
  44 randomize(char *data, unsigned len)
  45 {
  46         char *p = data;
  47
  48         if (pkcs11_get_random(data, len) == 0)
  49                 return;
  50
  51         /*
  52          * Implement a "fall back", because current callers
  53          * don't expect an error from this.  In practice,
  54          * we never use this fall back.
  55          */
  56         while (len--) {
  57                 *p++ = (random() >> 24);
  58         }
  59 }
  60
  61 /*
  62  * smb_auth_md4
  63  *
  64  * Compute an MD4 digest.
  65  */
  66 int
  67 smb_auth_md4(unsigned char *result, unsigned char *input, int length)
  68 {
  69         MD4_CTX md4_context;
  70
  71         MD4Init(&md4_context);
  72         MD4Update(&md4_context, input, length);
  73         MD4Final(result, &md4_context);
  74         return (SMBAUTH_SUCCESS);
  75 }
  76
  77 int
  78 smb_auth_hmac_md5(unsigned char *data,
  79         int data_len,
  80         unsigned char *key,
  81         int key_len,
  82         unsigned char *digest)
  83 {
  84         CK_RV rv;
  85         CK_MECHANISM mechanism;
  86         CK_OBJECT_HANDLE hKey;
  87         CK_SESSION_HANDLE hSession;
  88         CK_ULONG diglen = MD_DIGEST_LEN;
  89
  90         mechanism.mechanism = CKM_MD5_HMAC;
  91         mechanism.pParameter = 0;
  92         mechanism.ulParameterLen = 0;
  93         rv = SUNW_C_GetMechSession(mechanism.mechanism, &hSession);
  94         if (rv != CKR_OK) {
  95                 return (SMBAUTH_FAILURE);
  96         }
  97
  98         rv = SUNW_C_KeyToObject(hSession, mechanism.mechanism,
  99             key, key_len, &hKey);
 100         if (rv != CKR_OK) {
 101                 (void) C_CloseSession(hSession);
 102                 return (SMBAUTH_FAILURE);
 103         }
 104
 105         /* Initialize the digest operation in the session */
 106         rv = C_SignInit(hSession, &mechanism, hKey);
 107         if (rv != CKR_OK) {
 108                 (void) C_DestroyObject(hSession, hKey);
 109                 (void) C_CloseSession(hSession);
 110                 return (SMBAUTH_FAILURE);
 111         }
 112         rv = C_SignUpdate(hSession, (CK_BYTE_PTR)data, data_len);
 113         if (rv != CKR_OK) {
 114                 (void) C_DestroyObject(hSession, hKey);
 115                 (void) C_CloseSession(hSession);
 116                 return (SMBAUTH_FAILURE);
 117         }
 118         rv = C_SignFinal(hSession, (CK_BYTE_PTR)digest, &diglen);
 119         if (rv != CKR_OK) {
 120                 (void) C_DestroyObject(hSession, hKey);
 121                 (void) C_CloseSession(hSession);
 122                 return (SMBAUTH_FAILURE);
 123         }
 124         (void) C_DestroyObject(hSession, hKey);
 125         (void) C_CloseSession(hSession);
 126         if (diglen != MD_DIGEST_LEN) {
 127                 return (SMBAUTH_FAILURE);
 128         }
 129         return (SMBAUTH_SUCCESS);
 130 }
 131
 132 int
 133 smb_auth_DES(unsigned char *Result, int ResultLen,
 134     unsigned char *Key, int KeyLen,
 135     unsigned char *Data, int DataLen)
 136 {
 137         CK_RV rv;
 138         CK_MECHANISM mechanism;
 139         CK_OBJECT_HANDLE hKey;
 140         CK_SESSION_HANDLE hSession;
 141         CK_ULONG ciphertext_len;
 142         uchar_t des_key[8];
 143         int error = 0;
 144         int K, D;
 145         int k, d;
 146
 147         /*
 148          * Calculate proper number of iterations.
 149          * Known call cases include:
 150          *   ResultLen=16, KeyLen=14, DataLen=8
 151          *   ResultLen=24, KeyLen=21, DataLen=8
 152          *   ResultLen=16, KeyLen=14, DataLen=16
 153          */
 154         K = KeyLen / 7;
 155         D = DataLen / 8;
 156         if ((KeyLen % 7) || (DataLen % 8))
 157                 return (EINVAL);
 158         if (K == 0 || D == 0)
 159                 return (EINVAL);
 160         if (ResultLen < (K * 8))
 161                 return (EINVAL);
 162
 163         /*
 164          * Use SUNW convenience function to initialize the cryptoki
 165          * library, and open a session with a slot that supports
 166          * the mechanism we plan on using.
 167          */
 168         mechanism.mechanism = CKM_DES_ECB;
 169         mechanism.pParameter = NULL;
 170         mechanism.ulParameterLen = 0;
 171         rv = SUNW_C_GetMechSession(mechanism.mechanism, &hSession);
 172         if (rv != CKR_OK) {
 173                 return (SMBAUTH_FAILURE);
 174         }
 175
 176         for (d = k = 0; k < K; k++, d++) {
 177                 /* Cycle the input again, as necessary. */
 178                 if (d == D)
 179                         d = 0;
 180                 smb_initlmkey(&Key[k * 7], des_key);
 181                 rv = SUNW_C_KeyToObject(hSession, mechanism.mechanism,
 182                     des_key, 8, &hKey);
 183                 if (rv != CKR_OK) {
 184                         error = 1;
 185                         goto exit_session;
 186                 }
 187                 /* Initialize the encryption operation in the session */
 188                 rv = C_EncryptInit(hSession, &mechanism, hKey);
 189                 if (rv != CKR_OK) {
 190                         error = 1;
 191                         goto exit_encrypt;
 192                 }
 193                 ciphertext_len = 8;
 194
 195                 /* Read in the data and encrypt this portion */
 196                 rv = C_EncryptUpdate(hSession,
 197                     (CK_BYTE_PTR)Data + (d * 8), 8,
 198                     (CK_BYTE_PTR)Result + (k * 8),
 199                     &ciphertext_len);
 200                 if (rv != CKR_OK) {
 201                         error = 1;
 202                         goto exit_encrypt;
 203                 }
 204
 205                 (void) C_DestroyObject(hSession, hKey);
 206         }
 207         goto exit_session;
 208
 209 exit_encrypt:
 210         (void) C_DestroyObject(hSession, hKey);
 211 exit_session:
 212         (void) C_CloseSession(hSession);
 213
 214         if (error)
 215                 return (SMBAUTH_FAILURE);
 216
 217         return (SMBAUTH_SUCCESS);
 218 }
 219
 220 /*
 221  * See "Netlogon Credential Computation" section of MS-NRPC document.
 222  */
 223 static void
 224 smb_initlmkey(unsigned char *keyin, unsigned char *keyout)
 225 {
 226         int i;
 227
 228         keyout[0] = keyin[0] >> 0x01;
 229         keyout[1] = ((keyin[0] & 0x01) << 6) | (keyin[1] >> 2);
 230         keyout[2] = ((keyin[1] & 0x03) << 5) | (keyin[2] >> 3);
 231         keyout[3] = ((keyin[2] & 0x07) << 4) | (keyin[3] >> 4);
 232         keyout[4] = ((keyin[3] & 0x0f) << 3) | (keyin[4] >> 5);
 233         keyout[5] = ((keyin[4] & 0x1f) << 2) | (keyin[5] >> 6);
 234         keyout[6] = ((keyin[5] & 0x3f) << 1) | (keyin[6] >> 7);
 235         keyout[7] = keyin[6] & 0x7f;
 236
 237         for (i = 0; i < 8; i++)
 238                 keyout[i] = (keyout[i] << 1) & 0xfe;
 239 }
 240
 241 /*
 242  * CKM_RC4
 243  */
 244 int
 245 smb_auth_RC4(uchar_t *Result, int ResultLen,
 246         uchar_t *Key, int KeyLen,
 247         uchar_t *Data, int DataLen)
 248 {
 249         CK_RV rv;
 250         CK_MECHANISM mechanism;
 251         CK_OBJECT_HANDLE hKey;
 252         CK_SESSION_HANDLE hSession;
 253         CK_ULONG ciphertext_len;
 254         int error = SMBAUTH_FAILURE;
 255
 256         /*
 257          * Use SUNW convenience function to initialize the cryptoki
 258          * library, and open a session with a slot that supports
 259          * the mechanism we plan on using.
 260          */
 261         mechanism.mechanism = CKM_RC4;
 262         mechanism.pParameter = NULL;
 263         mechanism.ulParameterLen = 0;
 264         rv = SUNW_C_GetMechSession(mechanism.mechanism, &hSession);
 265         if (rv != CKR_OK) {
 266                 return (SMBAUTH_FAILURE);
 267         }
 268
 269         rv = SUNW_C_KeyToObject(hSession, mechanism.mechanism,
 270             Key, KeyLen, &hKey);
 271         if (rv != CKR_OK)
 272                 goto exit_session;
 273
 274         /* Initialize the encryption operation in the session */
 275         rv = C_EncryptInit(hSession, &mechanism, hKey);
 276         if (rv != CKR_OK)
 277                 goto exit_encrypt;
 278
 279         ciphertext_len = ResultLen;
 280         rv = C_EncryptUpdate(hSession,
 281             (CK_BYTE_PTR)Data, DataLen,
 282             (CK_BYTE_PTR)Result, &ciphertext_len);
 283         if (rv == CKR_OK)
 284                 error = 0;
 285
 286 exit_encrypt:
 287         (void) C_DestroyObject(hSession, hKey);
 288 exit_session:
 289         (void) C_CloseSession(hSession);
 290
 291         return (error);
 292 }