Fix spelling
[heimdal.git] / lib / gssapi / ntlm / crypto.c
blobd2cfddf6d5434b5bead94462804650b7baac5c14
1 /*
2 * Copyright (c) 2006-2016 Kungliga Tekniska Högskolan
3 * (Royal Institute of Technology, Stockholm, Sweden).
4 * All rights reserved.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * 3. Neither the name of the Institute nor the names of its contributors
18 * may be used to endorse or promote products derived from this software
19 * without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND
22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE
25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * SUCH DAMAGE.
34 #include "ntlm.h"
35 struct hx509_certs_data;
36 struct krb5_pk_identity;
37 struct krb5_pk_cert;
38 struct ContentInfo;
39 struct AlgorithmIdentifier;
40 struct _krb5_krb_auth_data;
41 struct krb5_dh_moduli;
42 struct _krb5_key_data;
43 struct _krb5_encryption_type;
44 struct _krb5_key_type;
45 #include "krb5_locl.h"
51 const char a2i_signmagic[] =
52 "session key to server-to-client signing key magic constant";
53 const char a2i_sealmagic[] =
54 "session key to server-to-client sealing key magic constant";
55 const char i2a_signmagic[] =
56 "session key to client-to-server signing key magic constant";
57 const char i2a_sealmagic[] =
58 "session key to client-to-server sealing key magic constant";
61 void
62 _gss_ntlm_set_key(struct ntlmv2_key *key, int acceptor, int sealsign,
63 unsigned char *data, size_t len)
65 unsigned char out[16];
66 EVP_MD_CTX *ctx;
67 const char *signmagic;
68 const char *sealmagic;
70 if (acceptor) {
71 signmagic = a2i_signmagic;
72 sealmagic = a2i_sealmagic;
73 } else {
74 signmagic = i2a_signmagic;
75 sealmagic = i2a_sealmagic;
78 key->seq = 0;
80 ctx = EVP_MD_CTX_create();
81 EVP_DigestInit_ex(ctx, EVP_md5(), NULL);
82 EVP_DigestUpdate(ctx, data, len);
83 EVP_DigestUpdate(ctx, signmagic, strlen(signmagic) + 1);
84 EVP_DigestFinal_ex(ctx, key->signkey, NULL);
86 EVP_DigestInit_ex(ctx, EVP_md5(), NULL);
87 EVP_DigestUpdate(ctx, data, len);
88 EVP_DigestUpdate(ctx, sealmagic, strlen(sealmagic) + 1);
89 EVP_DigestFinal_ex(ctx, out, NULL);
90 EVP_MD_CTX_destroy(ctx);
92 RC4_set_key(&key->sealkey, 16, out);
93 if (sealsign)
94 key->signsealkey = &key->sealkey;
98 * Set (or reset) keys
101 void
102 _gss_ntlm_set_keys(ntlm_ctx ctx)
104 int acceptor;
106 if (ctx->sessionkey.length == 0)
107 return;
109 acceptor = !(ctx->status & STATUS_CLIENT);
111 ctx->status |= STATUS_SESSIONKEY;
113 if (ctx->flags & NTLM_NEG_NTLM2_SESSION) {
114 _gss_ntlm_set_key(&ctx->u.v2.send, acceptor,
115 (ctx->flags & NTLM_NEG_KEYEX),
116 ctx->sessionkey.data,
117 ctx->sessionkey.length);
118 _gss_ntlm_set_key(&ctx->u.v2.recv, !acceptor,
119 (ctx->flags & NTLM_NEG_KEYEX),
120 ctx->sessionkey.data,
121 ctx->sessionkey.length);
122 } else {
123 ctx->u.v1.crypto_send.seq = 0;
124 RC4_set_key(&ctx->u.v1.crypto_send.key,
125 ctx->sessionkey.length,
126 ctx->sessionkey.data);
127 ctx->u.v1.crypto_recv.seq = 0;
128 RC4_set_key(&ctx->u.v1.crypto_recv.key,
129 ctx->sessionkey.length,
130 ctx->sessionkey.data);
138 static OM_uint32
139 v1_sign_message(gss_buffer_t in,
140 RC4_KEY *signkey,
141 uint32_t seq,
142 unsigned char out[16])
144 unsigned char sigature[12];
145 uint32_t crc;
147 _krb5_crc_init_table();
148 crc = _krb5_crc_update(in->value, in->length, 0);
150 _gss_mg_encode_le_uint32(0, &sigature[0]);
151 _gss_mg_encode_le_uint32(crc, &sigature[4]);
152 _gss_mg_encode_le_uint32(seq, &sigature[8]);
154 _gss_mg_encode_le_uint32(1, out); /* version */
155 RC4(signkey, sizeof(sigature), sigature, out + 4);
157 if (RAND_bytes(out + 4, 4) != 1)
158 return GSS_S_UNAVAILABLE;
160 return 0;
164 static OM_uint32
165 v2_sign_message(gss_buffer_t in,
166 unsigned char signkey[16],
167 RC4_KEY *sealkey,
168 uint32_t seq,
169 unsigned char out[16])
171 unsigned char hmac[16];
172 unsigned int hmaclen;
173 HMAC_CTX c;
175 HMAC_CTX_init(&c);
176 if (HMAC_Init_ex(&c, signkey, 16, EVP_md5(), NULL) == 0) {
177 HMAC_CTX_cleanup(&c);
178 return GSS_S_FAILURE;
181 _gss_mg_encode_le_uint32(seq, hmac);
182 HMAC_Update(&c, hmac, 4);
183 HMAC_Update(&c, in->value, in->length);
184 HMAC_Final(&c, hmac, &hmaclen);
185 HMAC_CTX_cleanup(&c);
187 _gss_mg_encode_le_uint32(1, &out[0]);
188 if (sealkey)
189 RC4(sealkey, 8, hmac, &out[4]);
190 else
191 memcpy(&out[4], hmac, 8);
193 memset(&out[12], 0, 4);
195 return GSS_S_COMPLETE;
198 static OM_uint32
199 v2_verify_message(gss_buffer_t in,
200 unsigned char signkey[16],
201 RC4_KEY *sealkey,
202 uint32_t seq,
203 const unsigned char checksum[16])
205 OM_uint32 ret;
206 unsigned char out[16];
208 ret = v2_sign_message(in, signkey, sealkey, seq, out);
209 if (ret)
210 return ret;
212 if (ct_memcmp(checksum, out, 16) != 0)
213 return GSS_S_BAD_MIC;
215 return GSS_S_COMPLETE;
218 static OM_uint32
219 v2_seal_message(const gss_buffer_t in,
220 unsigned char signkey[16],
221 uint32_t seq,
222 RC4_KEY *sealkey,
223 gss_buffer_t out)
225 unsigned char *p;
226 OM_uint32 ret;
228 if (in->length + 16 < in->length)
229 return EINVAL;
231 p = malloc(in->length + 16);
232 if (p == NULL)
233 return ENOMEM;
235 RC4(sealkey, in->length, in->value, p);
237 ret = v2_sign_message(in, signkey, sealkey, seq, &p[in->length]);
238 if (ret) {
239 free(p);
240 return ret;
243 out->value = p;
244 out->length = in->length + 16;
246 return 0;
249 static OM_uint32
250 v2_unseal_message(gss_buffer_t in,
251 unsigned char signkey[16],
252 uint32_t seq,
253 RC4_KEY *sealkey,
254 gss_buffer_t out)
256 OM_uint32 ret;
258 if (in->length < 16)
259 return GSS_S_BAD_MIC;
261 out->length = in->length - 16;
262 out->value = malloc(out->length);
263 if (out->value == NULL)
264 return GSS_S_BAD_MIC;
266 RC4(sealkey, out->length, in->value, out->value);
268 ret = v2_verify_message(out, signkey, sealkey, seq,
269 ((const unsigned char *)in->value) + out->length);
270 if (ret) {
271 OM_uint32 junk;
272 gss_release_buffer(&junk, out);
274 return ret;
281 #define CTX_FLAGS_ISSET(_ctx,_flags) \
282 (((_ctx)->flags & (_flags)) == (_flags))
288 OM_uint32 GSSAPI_CALLCONV
289 _gss_ntlm_get_mic
290 (OM_uint32 * minor_status,
291 gss_const_ctx_id_t context_handle,
292 gss_qop_t qop_req,
293 const gss_buffer_t message_buffer,
294 gss_buffer_t message_token
297 ntlm_ctx ctx = (ntlm_ctx)context_handle;
298 OM_uint32 junk;
300 *minor_status = 0;
302 message_token->value = malloc(16);
303 message_token->length = 16;
304 if (message_token->value == NULL) {
305 *minor_status = ENOMEM;
306 return GSS_S_FAILURE;
309 if (CTX_FLAGS_ISSET(ctx, NTLM_NEG_SIGN|NTLM_NEG_NTLM2_SESSION)) {
310 OM_uint32 ret;
312 if ((ctx->status & STATUS_SESSIONKEY) == 0) {
313 gss_release_buffer(&junk, message_token);
314 return GSS_S_UNAVAILABLE;
317 ret = v2_sign_message(message_buffer,
318 ctx->u.v2.send.signkey,
319 ctx->u.v2.send.signsealkey,
320 ctx->u.v2.send.seq++,
321 message_token->value);
322 if (ret)
323 gss_release_buffer(&junk, message_token);
324 return ret;
326 } else if (CTX_FLAGS_ISSET(ctx, NTLM_NEG_SIGN)) {
327 OM_uint32 ret;
329 if ((ctx->status & STATUS_SESSIONKEY) == 0) {
330 gss_release_buffer(&junk, message_token);
331 return GSS_S_UNAVAILABLE;
334 ret = v1_sign_message(message_buffer,
335 &ctx->u.v1.crypto_send.key,
336 ctx->u.v1.crypto_send.seq++,
337 message_token->value);
338 if (ret)
339 gss_release_buffer(&junk, message_token);
340 return ret;
342 } else if (CTX_FLAGS_ISSET(ctx, NTLM_NEG_ALWAYS_SIGN)) {
343 unsigned char *sigature;
345 sigature = message_token->value;
347 _gss_mg_encode_le_uint32(1, &sigature[0]); /* version */
348 _gss_mg_encode_le_uint32(0, &sigature[4]);
349 _gss_mg_encode_le_uint32(0, &sigature[8]);
350 _gss_mg_encode_le_uint32(0, &sigature[12]);
352 return GSS_S_COMPLETE;
354 gss_release_buffer(&junk, message_token);
356 return GSS_S_UNAVAILABLE;
363 OM_uint32 GSSAPI_CALLCONV
364 _gss_ntlm_verify_mic
365 (OM_uint32 * minor_status,
366 gss_const_ctx_id_t context_handle,
367 const gss_buffer_t message_buffer,
368 const gss_buffer_t token_buffer,
369 gss_qop_t * qop_state
372 ntlm_ctx ctx = (ntlm_ctx)context_handle;
374 if (qop_state != NULL)
375 *qop_state = GSS_C_QOP_DEFAULT;
376 *minor_status = 0;
378 if (token_buffer->length != 16)
379 return GSS_S_BAD_MIC;
381 if (CTX_FLAGS_ISSET(ctx, NTLM_NEG_SIGN|NTLM_NEG_NTLM2_SESSION)) {
382 OM_uint32 ret;
384 if ((ctx->status & STATUS_SESSIONKEY) == 0)
385 return GSS_S_UNAVAILABLE;
387 ret = v2_verify_message(message_buffer,
388 ctx->u.v2.recv.signkey,
389 ctx->u.v2.recv.signsealkey,
390 ctx->u.v2.recv.seq++,
391 token_buffer->value);
392 if (ret)
393 return ret;
395 return GSS_S_COMPLETE;
396 } else if (CTX_FLAGS_ISSET(ctx, NTLM_NEG_SIGN)) {
398 unsigned char sigature[12];
399 uint32_t crc, num;
401 if ((ctx->status & STATUS_SESSIONKEY) == 0)
402 return GSS_S_UNAVAILABLE;
404 _gss_mg_decode_le_uint32(token_buffer->value, &num);
405 if (num != 1)
406 return GSS_S_BAD_MIC;
408 RC4(&ctx->u.v1.crypto_recv.key, sizeof(sigature),
409 ((unsigned char *)token_buffer->value) + 4, sigature);
411 _krb5_crc_init_table();
412 crc = _krb5_crc_update(message_buffer->value,
413 message_buffer->length, 0);
414 /* skip first 4 bytes in the encrypted checksum */
415 _gss_mg_decode_le_uint32(&sigature[4], &num);
416 if (num != crc)
417 return GSS_S_BAD_MIC;
418 _gss_mg_decode_le_uint32(&sigature[8], &num);
419 if (ctx->u.v1.crypto_recv.seq != num)
420 return GSS_S_BAD_MIC;
421 ctx->u.v1.crypto_recv.seq++;
423 return GSS_S_COMPLETE;
424 } else if (ctx->flags & NTLM_NEG_ALWAYS_SIGN) {
425 uint32_t num;
426 unsigned char *p;
428 p = (unsigned char*)(token_buffer->value);
430 _gss_mg_decode_le_uint32(&p[0], &num); /* version */
431 if (num != 1) return GSS_S_BAD_MIC;
432 _gss_mg_decode_le_uint32(&p[4], &num);
433 if (num != 0) return GSS_S_BAD_MIC;
434 _gss_mg_decode_le_uint32(&p[8], &num);
435 if (num != 0) return GSS_S_BAD_MIC;
436 _gss_mg_decode_le_uint32(&p[12], &num);
437 if (num != 0) return GSS_S_BAD_MIC;
439 return GSS_S_COMPLETE;
442 return GSS_S_UNAVAILABLE;
449 OM_uint32 GSSAPI_CALLCONV
450 _gss_ntlm_wrap_size_limit (
451 OM_uint32 * minor_status,
452 gss_const_ctx_id_t context_handle,
453 int conf_req_flag,
454 gss_qop_t qop_req,
455 OM_uint32 req_output_size,
456 OM_uint32 * max_input_size
459 ntlm_ctx ctx = (ntlm_ctx)context_handle;
461 *minor_status = 0;
463 if(ctx->flags & NTLM_NEG_SEAL) {
465 if (req_output_size < 16)
466 *max_input_size = 0;
467 else
468 *max_input_size = req_output_size - 16;
470 return GSS_S_COMPLETE;
473 return GSS_S_UNAVAILABLE;
480 OM_uint32 GSSAPI_CALLCONV
481 _gss_ntlm_wrap
482 (OM_uint32 * minor_status,
483 gss_const_ctx_id_t context_handle,
484 int conf_req_flag,
485 gss_qop_t qop_req,
486 const gss_buffer_t input_message_buffer,
487 int * conf_state,
488 gss_buffer_t output_message_buffer
491 ntlm_ctx ctx = (ntlm_ctx)context_handle;
492 OM_uint32 ret;
494 *minor_status = 0;
495 if (conf_state)
496 *conf_state = 0;
497 if (output_message_buffer == GSS_C_NO_BUFFER)
498 return GSS_S_FAILURE;
501 if (CTX_FLAGS_ISSET(ctx, NTLM_NEG_SEAL|NTLM_NEG_NTLM2_SESSION)) {
503 return v2_seal_message(input_message_buffer,
504 ctx->u.v2.send.signkey,
505 ctx->u.v2.send.seq++,
506 &ctx->u.v2.send.sealkey,
507 output_message_buffer);
509 } else if (CTX_FLAGS_ISSET(ctx, NTLM_NEG_SEAL)) {
510 gss_buffer_desc trailer;
511 OM_uint32 junk;
513 output_message_buffer->length = input_message_buffer->length + 16;
514 output_message_buffer->value = malloc(output_message_buffer->length);
515 if (output_message_buffer->value == NULL) {
516 output_message_buffer->length = 0;
517 return GSS_S_FAILURE;
521 RC4(&ctx->u.v1.crypto_send.key, input_message_buffer->length,
522 input_message_buffer->value, output_message_buffer->value);
524 ret = _gss_ntlm_get_mic(minor_status, context_handle,
525 0, input_message_buffer,
526 &trailer);
527 if (ret) {
528 gss_release_buffer(&junk, output_message_buffer);
529 return ret;
531 if (trailer.length != 16) {
532 gss_release_buffer(&junk, output_message_buffer);
533 gss_release_buffer(&junk, &trailer);
534 return GSS_S_FAILURE;
536 memcpy(((unsigned char *)output_message_buffer->value) +
537 input_message_buffer->length,
538 trailer.value, trailer.length);
539 gss_release_buffer(&junk, &trailer);
541 return GSS_S_COMPLETE;
544 return GSS_S_UNAVAILABLE;
551 OM_uint32 GSSAPI_CALLCONV
552 _gss_ntlm_unwrap
553 (OM_uint32 * minor_status,
554 gss_const_ctx_id_t context_handle,
555 const gss_buffer_t input_message_buffer,
556 gss_buffer_t output_message_buffer,
557 int * conf_state,
558 gss_qop_t * qop_state
561 ntlm_ctx ctx = (ntlm_ctx)context_handle;
562 OM_uint32 ret;
564 *minor_status = 0;
565 output_message_buffer->value = NULL;
566 output_message_buffer->length = 0;
568 if (conf_state)
569 *conf_state = 0;
570 if (qop_state)
571 *qop_state = 0;
573 if (CTX_FLAGS_ISSET(ctx, NTLM_NEG_SEAL|NTLM_NEG_NTLM2_SESSION)) {
575 return v2_unseal_message(input_message_buffer,
576 ctx->u.v2.recv.signkey,
577 ctx->u.v2.recv.seq++,
578 &ctx->u.v2.recv.sealkey,
579 output_message_buffer);
581 } else if (CTX_FLAGS_ISSET(ctx, NTLM_NEG_SEAL)) {
583 gss_buffer_desc trailer;
584 OM_uint32 junk;
586 if (input_message_buffer->length < 16)
587 return GSS_S_BAD_MIC;
589 output_message_buffer->length = input_message_buffer->length - 16;
590 output_message_buffer->value = malloc(output_message_buffer->length);
591 if (output_message_buffer->value == NULL) {
592 output_message_buffer->length = 0;
593 return GSS_S_FAILURE;
596 RC4(&ctx->u.v1.crypto_recv.key, output_message_buffer->length,
597 input_message_buffer->value, output_message_buffer->value);
599 trailer.value = ((unsigned char *)input_message_buffer->value) +
600 output_message_buffer->length;
601 trailer.length = 16;
603 ret = _gss_ntlm_verify_mic(minor_status, context_handle,
604 output_message_buffer,
605 &trailer, NULL);
606 if (ret) {
607 gss_release_buffer(&junk, output_message_buffer);
608 return ret;
611 return GSS_S_COMPLETE;
614 return GSS_S_UNAVAILABLE;