Expand PMF_FN_* macros.
[netbsd-mini2440.git] / lib / libtelnet / enc_des.c
blob1e7d3f88661e78a8a6800db20c8be37742c2903b
1 /* $NetBSD: enc_des.c,v 1.12 2005/02/06 05:53:07 perry Exp $ */
3 /*-
4 * Copyright (c) 1991, 1993
5 * The Regents of the University of California. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. Neither the name of the University nor the names of its contributors
16 * may be used to endorse or promote products derived from this software
17 * without specific prior written permission.
19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
32 #include <sys/cdefs.h>
33 #ifndef lint
34 #if 0
35 static char sccsid[] = "@(#)enc_des.c 8.3 (Berkeley) 5/30/95"; */
36 #else
37 __RCSID("$NetBSD: enc_des.c,v 1.12 2005/02/06 05:53:07 perry Exp $");
38 #endif
39 #endif /* not lint */
41 #ifdef ENCRYPTION
42 # ifdef AUTHENTICATION
43 # ifdef DES_ENCRYPTION
44 #include <arpa/telnet.h>
45 #include <stdio.h>
46 #include <string.h>
47 #include <stdlib.h>
49 #include <des.h>
50 #include "encrypt.h"
51 #include "key-proto.h"
52 #include "misc-proto.h"
54 #define CFB 0
55 #define OFB 1
57 #define NO_SEND_IV 1
58 #define NO_RECV_IV 2
59 #define NO_KEYID 4
60 #define IN_PROGRESS (NO_SEND_IV|NO_RECV_IV|NO_KEYID)
61 #define SUCCESS 0
62 #define FAILED -1
65 struct fb {
66 Block krbdes_key;
67 Schedule krbdes_sched;
68 Block temp_feed;
69 unsigned char fb_feed[64];
70 int need_start;
71 int state[2];
72 int keyid[2];
73 int once;
74 struct stinfo {
75 Block str_output;
76 Block str_feed;
77 Block str_iv;
78 Block str_ikey;
79 Schedule str_sched;
80 int str_index;
81 int str_flagshift;
82 } streams[2];
85 static struct fb fb[2];
87 struct keyidlist {
88 char *keyid;
89 int keyidlen;
90 char *key;
91 int keylen;
92 int flags;
93 } keyidlist [] = {
94 { "\0", 1, 0, 0, 0 }, /* default key of zero */
95 { 0, 0, 0, 0, 0 }
98 #define KEYFLAG_MASK 03
100 #define KEYFLAG_NOINIT 00
101 #define KEYFLAG_INIT 01
102 #define KEYFLAG_OK 02
103 #define KEYFLAG_BAD 03
105 #define KEYFLAG_SHIFT 2
107 #define SHIFT_VAL(a,b) (KEYFLAG_SHIFT*((a)+((b)*2)))
109 #define FB64_IV 1
110 #define FB64_IV_OK 2
111 #define FB64_IV_BAD 3
114 void fb64_stream_iv(Block, struct stinfo *);
115 void fb64_init(struct fb *);
116 static int fb64_start(struct fb *, int, int);
117 int fb64_is(unsigned char *, int, struct fb *);
118 int fb64_reply(unsigned char *, int, struct fb *);
119 static void fb64_session(Session_Key *, int, struct fb *);
120 void fb64_stream_key(Block *, struct stinfo *);
121 int fb64_keyid(int, unsigned char *, int *, struct fb *);
123 void
124 cfb64_init(server)
125 int server;
127 fb64_init(&fb[CFB]);
128 fb[CFB].fb_feed[4] = ENCTYPE_DES_CFB64;
129 fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, CFB);
130 fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, CFB);
133 void
134 ofb64_init(server)
135 int server;
137 fb64_init(&fb[OFB]);
138 fb[OFB].fb_feed[4] = ENCTYPE_DES_OFB64;
139 fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, OFB);
140 fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, OFB);
143 void
144 fb64_init(fbp)
145 register struct fb *fbp;
147 memset((void *)fbp, 0, sizeof(*fbp));
148 fbp->state[0] = fbp->state[1] = FAILED;
149 fbp->fb_feed[0] = IAC;
150 fbp->fb_feed[1] = SB;
151 fbp->fb_feed[2] = TELOPT_ENCRYPT;
152 fbp->fb_feed[3] = ENCRYPT_IS;
156 * Returns:
157 * -1: some error. Negotiation is done, encryption not ready.
158 * 0: Successful, initial negotiation all done.
159 * 1: successful, negotiation not done yet.
160 * 2: Not yet. Other things (like getting the key from
161 * Kerberos) have to happen before we can continue.
164 cfb64_start(dir, server)
165 int dir;
166 int server;
168 return(fb64_start(&fb[CFB], dir, server));
171 ofb64_start(dir, server)
172 int dir;
173 int server;
175 return(fb64_start(&fb[OFB], dir, server));
178 static int
179 fb64_start(fbp, dir, server)
180 struct fb *fbp;
181 int dir;
182 int server;
184 int x;
185 unsigned char *p;
186 register int state;
188 switch (dir) {
189 case DIR_DECRYPT:
191 * This is simply a request to have the other side
192 * start output (our input). He will negotiate an
193 * IV so we need not look for it.
195 state = fbp->state[dir-1];
196 if (state == FAILED)
197 state = IN_PROGRESS;
198 break;
200 case DIR_ENCRYPT:
201 state = fbp->state[dir-1];
202 if (state == FAILED)
203 state = IN_PROGRESS;
204 else if ((state & NO_SEND_IV) == 0)
205 break;
207 if (!VALIDKEY(fbp->krbdes_key)) {
208 fbp->need_start = 1;
209 break;
211 state &= ~NO_SEND_IV;
212 state |= NO_RECV_IV;
213 if (encrypt_debug_mode)
214 printf("Creating new feed\r\n");
216 * Create a random feed and send it over.
218 des_new_random_key(&fbp->temp_feed);
219 des_ecb_encrypt(&fbp->temp_feed, &fbp->temp_feed,
220 fbp->krbdes_sched, 1);
221 p = fbp->fb_feed + 3;
222 *p++ = ENCRYPT_IS;
223 p++;
224 *p++ = FB64_IV;
225 for (x = 0; x < sizeof(Block); ++x) {
226 if ((*p++ = fbp->temp_feed[x]) == IAC)
227 *p++ = IAC;
229 *p++ = IAC;
230 *p++ = SE;
231 printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
232 telnet_net_write(fbp->fb_feed, p - fbp->fb_feed);
233 break;
234 default:
235 return(FAILED);
237 return(fbp->state[dir-1] = state);
241 * Returns:
242 * -1: some error. Negotiation is done, encryption not ready.
243 * 0: Successful, initial negotiation all done.
244 * 1: successful, negotiation not done yet.
247 cfb64_is(data, cnt)
248 unsigned char *data;
249 int cnt;
251 return(fb64_is(data, cnt, &fb[CFB]));
254 ofb64_is(data, cnt)
255 unsigned char *data;
256 int cnt;
258 return(fb64_is(data, cnt, &fb[OFB]));
262 fb64_is(data, cnt, fbp)
263 unsigned char *data;
264 int cnt;
265 struct fb *fbp;
267 unsigned char *p;
268 register int state = fbp->state[DIR_DECRYPT-1];
270 if (cnt-- < 1)
271 goto failure;
273 switch (*data++) {
274 case FB64_IV:
275 if (cnt != sizeof(Block)) {
276 if (encrypt_debug_mode)
277 printf("CFB64: initial vector failed on size\r\n");
278 state = FAILED;
279 goto failure;
282 if (encrypt_debug_mode)
283 printf("CFB64: initial vector received\r\n");
285 if (encrypt_debug_mode)
286 printf("Initializing Decrypt stream\r\n");
288 fb64_stream_iv((void *)data, &fbp->streams[DIR_DECRYPT-1]);
290 p = fbp->fb_feed + 3;
291 *p++ = ENCRYPT_REPLY;
292 p++;
293 *p++ = FB64_IV_OK;
294 *p++ = IAC;
295 *p++ = SE;
296 printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
297 telnet_net_write(fbp->fb_feed, p - fbp->fb_feed);
299 state = fbp->state[DIR_DECRYPT-1] = IN_PROGRESS;
300 break;
302 default:
303 if (encrypt_debug_mode) {
304 printf("Unknown option type: %d\r\n", *(data-1));
305 printd(data, cnt);
306 printf("\r\n");
308 /* FALL THROUGH */
309 failure:
311 * We failed. Send an FB64_IV_BAD option
312 * to the other side so it will know that
313 * things failed.
315 p = fbp->fb_feed + 3;
316 *p++ = ENCRYPT_REPLY;
317 p++;
318 *p++ = FB64_IV_BAD;
319 *p++ = IAC;
320 *p++ = SE;
321 printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
322 telnet_net_write(fbp->fb_feed, p - fbp->fb_feed);
324 break;
326 return(fbp->state[DIR_DECRYPT-1] = state);
330 * Returns:
331 * -1: some error. Negotiation is done, encryption not ready.
332 * 0: Successful, initial negotiation all done.
333 * 1: successful, negotiation not done yet.
336 cfb64_reply(data, cnt)
337 unsigned char *data;
338 int cnt;
340 return(fb64_reply(data, cnt, &fb[CFB]));
343 ofb64_reply(data, cnt)
344 unsigned char *data;
345 int cnt;
347 return(fb64_reply(data, cnt, &fb[OFB]));
352 fb64_reply(data, cnt, fbp)
353 unsigned char *data;
354 int cnt;
355 struct fb *fbp;
357 register int state = fbp->state[DIR_ENCRYPT-1];
359 if (cnt-- < 1)
360 goto failure;
362 switch (*data++) {
363 case FB64_IV_OK:
364 fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
365 if (state == FAILED)
366 state = IN_PROGRESS;
367 state &= ~NO_RECV_IV;
368 encrypt_send_keyid(DIR_ENCRYPT, (unsigned char *)"\0", 1, 1);
369 break;
371 case FB64_IV_BAD:
372 memset(fbp->temp_feed, 0, sizeof(Block));
373 fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
374 state = FAILED;
375 break;
377 default:
378 if (encrypt_debug_mode) {
379 printf("Unknown option type: %d\r\n", data[-1]);
380 printd(data, cnt);
381 printf("\r\n");
383 /* FALL THROUGH */
384 failure:
385 state = FAILED;
386 break;
388 return(fbp->state[DIR_ENCRYPT-1] = state);
391 void
392 cfb64_session(key, server)
393 Session_Key *key;
394 int server;
396 fb64_session(key, server, &fb[CFB]);
399 void
400 ofb64_session(key, server)
401 Session_Key *key;
402 int server;
404 fb64_session(key, server, &fb[OFB]);
407 static void
408 fb64_session(key, server, fbp)
409 Session_Key *key;
410 int server;
411 struct fb *fbp;
414 if (!key || key->type != SK_DES) {
415 if (encrypt_debug_mode)
416 printf("Can't set krbdes's session key (%d != %d)\r\n",
417 key ? key->type : -1, SK_DES);
418 return;
420 memmove((void *)fbp->krbdes_key, (void *)key->data, sizeof(Block));
422 fb64_stream_key(&fbp->krbdes_key, &fbp->streams[DIR_ENCRYPT-1]);
423 fb64_stream_key(&fbp->krbdes_key, &fbp->streams[DIR_DECRYPT-1]);
425 if (fbp->once == 0) {
426 des_init_random_number_generator(&fbp->krbdes_key);
427 fbp->once = 1;
429 des_key_sched(&fbp->krbdes_key, fbp->krbdes_sched);
431 * Now look to see if krbdes_start() was was waiting for
432 * the key to show up. If so, go ahead an call it now
433 * that we have the key.
435 if (fbp->need_start) {
436 fbp->need_start = 0;
437 fb64_start(fbp, DIR_ENCRYPT, server);
442 * We only accept a keyid of 0. If we get a keyid of
443 * 0, then mark the state as SUCCESS.
446 cfb64_keyid(dir, kp, lenp)
447 int dir, *lenp;
448 unsigned char *kp;
450 return(fb64_keyid(dir, kp, lenp, &fb[CFB]));
454 ofb64_keyid(dir, kp, lenp)
455 int dir, *lenp;
456 unsigned char *kp;
458 return(fb64_keyid(dir, kp, lenp, &fb[OFB]));
462 fb64_keyid(dir, kp, lenp, fbp)
463 int dir, *lenp;
464 unsigned char *kp;
465 struct fb *fbp;
467 register int state = fbp->state[dir-1];
469 if (*lenp != 1 || (*kp != '\0')) {
470 *lenp = 0;
471 return(state);
474 if (state == FAILED)
475 state = IN_PROGRESS;
477 state &= ~NO_KEYID;
479 return(fbp->state[dir-1] = state);
482 void
483 fb64_printsub(data, cnt, buf, buflen, type)
484 unsigned char *data, *buf, *type;
485 int cnt, buflen;
487 char lbuf[32];
488 register int i;
489 char *cp;
491 buf[buflen-1] = '\0'; /* make sure it's NULL terminated */
492 buflen -= 1;
494 switch(data[2]) {
495 case FB64_IV:
496 snprintf(lbuf, sizeof(lbuf), "%s_IV", type);
497 cp = lbuf;
498 goto common;
500 case FB64_IV_OK:
501 snprintf(lbuf, sizeof(lbuf), "%s_IV_OK", type);
502 cp = lbuf;
503 goto common;
505 case FB64_IV_BAD:
506 snprintf(lbuf, sizeof(lbuf), "%s_IV_BAD", type);
507 cp = lbuf;
508 goto common;
510 default:
511 snprintf(lbuf, sizeof(lbuf), " %d (unknown)", data[2]);
512 cp = lbuf;
513 common:
514 for (; (buflen > 0) && (*buf = *cp++); buf++)
515 buflen--;
516 for (i = 3; i < cnt; i++) {
517 snprintf(lbuf, sizeof(lbuf), " %d", data[i]);
518 for (cp = lbuf; (buflen > 0) && (*buf = *cp++); buf++)
519 buflen--;
521 break;
525 void
526 cfb64_printsub(data, cnt, buf, buflen)
527 unsigned char *data, *buf;
528 int cnt, buflen;
530 fb64_printsub(data, cnt, buf, buflen, "CFB64");
533 void
534 ofb64_printsub(data, cnt, buf, buflen)
535 unsigned char *data, *buf;
536 int cnt, buflen;
538 fb64_printsub(data, cnt, buf, buflen, "OFB64");
541 void
542 fb64_stream_iv(seed, stp)
543 Block seed;
544 register struct stinfo *stp;
547 memmove((void *)stp->str_iv, (void *)seed, sizeof(Block));
548 memmove((void *)stp->str_output, (void *)seed, sizeof(Block));
550 des_key_sched(&stp->str_ikey, stp->str_sched);
552 stp->str_index = sizeof(Block);
555 void
556 fb64_stream_key(key, stp)
557 Block *key;
558 register struct stinfo *stp;
560 memmove((void *)stp->str_ikey, (void *)key, sizeof(Block));
561 des_key_sched(key, stp->str_sched);
563 memmove((void *)stp->str_output, (void *)stp->str_iv, sizeof(Block));
565 stp->str_index = sizeof(Block);
569 * DES 64 bit Cipher Feedback
571 * key --->+-----+
572 * +->| DES |--+
573 * | +-----+ |
574 * | v
575 * INPUT --(--------->(+)+---> DATA
576 * | |
577 * +-------------+
580 * Given:
581 * iV: Initial vector, 64 bits (8 bytes) long.
582 * Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
583 * On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
585 * V0 = DES(iV, key)
586 * On = Dn ^ Vn
587 * V(n+1) = DES(On, key)
590 void
591 cfb64_encrypt(s, c)
592 register unsigned char *s;
593 int c;
595 register struct stinfo *stp = &fb[CFB].streams[DIR_ENCRYPT-1];
596 register int idx;
598 idx = stp->str_index;
599 while (c-- > 0) {
600 if (idx == sizeof(Block)) {
601 Block b;
602 des_ecb_encrypt(&stp->str_output, &b, stp->str_sched, 1);
603 memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
604 idx = 0;
607 /* On encryption, we store (feed ^ data) which is cypher */
608 *s = stp->str_output[idx] = (stp->str_feed[idx] ^ *s);
609 s++;
610 idx++;
612 stp->str_index = idx;
616 cfb64_decrypt(data)
617 int data;
619 register struct stinfo *stp = &fb[CFB].streams[DIR_DECRYPT-1];
620 int idx;
622 if (data == -1) {
624 * Back up one byte. It is assumed that we will
625 * never back up more than one byte. If we do, this
626 * may or may not work.
628 if (stp->str_index)
629 --stp->str_index;
630 return(0);
633 idx = stp->str_index++;
634 if (idx == sizeof(Block)) {
635 Block b;
636 des_ecb_encrypt(&stp->str_output, &b, stp->str_sched, 1);
637 memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
638 stp->str_index = 1; /* Next time will be 1 */
639 idx = 0; /* But now use 0 */
642 /* On decryption we store (data) which is cypher. */
643 stp->str_output[idx] = data;
644 return(data ^ stp->str_feed[idx]);
648 * DES 64 bit Output Feedback
650 * key --->+-----+
651 * +->| DES |--+
652 * | +-----+ |
653 * +-----------+
655 * INPUT -------->(+) ----> DATA
657 * Given:
658 * iV: Initial vector, 64 bits (8 bytes) long.
659 * Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
660 * On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
662 * V0 = DES(iV, key)
663 * V(n+1) = DES(Vn, key)
664 * On = Dn ^ Vn
666 void
667 ofb64_encrypt(s, c)
668 register unsigned char *s;
669 int c;
671 register struct stinfo *stp = &fb[OFB].streams[DIR_ENCRYPT-1];
672 register int idx;
674 idx = stp->str_index;
675 while (c-- > 0) {
676 if (idx == sizeof(Block)) {
677 Block b;
678 des_ecb_encrypt(&stp->str_feed, &b, stp->str_sched, 1);
679 memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
680 idx = 0;
682 *s++ ^= stp->str_feed[idx];
683 idx++;
685 stp->str_index = idx;
689 ofb64_decrypt(data)
690 int data;
692 register struct stinfo *stp = &fb[OFB].streams[DIR_DECRYPT-1];
693 int idx;
695 if (data == -1) {
697 * Back up one byte. It is assumed that we will
698 * never back up more than one byte. If we do, this
699 * may or may not work.
701 if (stp->str_index)
702 --stp->str_index;
703 return(0);
706 idx = stp->str_index++;
707 if (idx == sizeof(Block)) {
708 Block b;
709 des_ecb_encrypt(&stp->str_feed, &b, stp->str_sched, 1);
710 memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
711 stp->str_index = 1; /* Next time will be 1 */
712 idx = 0; /* But now use 0 */
715 return(data ^ stp->str_feed[idx]);
717 # endif /* DES_ENCRYPTION */
718 # endif /* AUTHENTICATION */
719 #endif /* ENCRYPTION */