libs/iovm/source/IoLexer.c

   1 /*
   2 Copyright (c) 2003, Steve Dekorte
   3 All rights reserved. See _BSDLicense.txt.
   4
   5 Aug 2004 - removed {} from op chars
   6 - changed identifier to stop after 1 colon
   7 */
   8
   9 #include "IoLexer.h"
  10 #include <stdlib.h>
  11 #include <string.h>
  12 #include <ctype.h>
  13 #include <stddef.h>
  14
  15 //#define LEXER_DEBUG
  16 //#define LEXER_DEBUG_TOKENS
  17 #define TEST_INLINE inline
  18
  19 static IoToken *IoLexer_currentToken(IoLexer *self)
  20 {
  21         return List_top(self->tokenStream);
  22 }
  23
  24 IoLexer *IoLexer_new(void)
  25 {
  26         IoLexer *self = (IoLexer *)io_calloc(1, sizeof(IoLexer));
  27         self->s = (char *)io_calloc(1, 1);
  28         self->s[0] = 0;
  29         self->posStack = Stack_new();
  30         self->tokenStack = Stack_new();
  31         self->tokenStream = List_new();
  32         self->charLineIndex = List_new();
  33         return self;
  34 }
  35
  36 void IoLexer_free(IoLexer *self)
  37 {
  38         IoLexer_clear(self);
  39         io_free(self->s);
  40         Stack_free(self->posStack);
  41         Stack_free(self->tokenStack);
  42         List_free(self->tokenStream);
  43         List_free(self->charLineIndex);
  44         if(self->errorDescription) io_free(self->errorDescription);
  45         io_free(self);
  46 }
  47
  48 char *IoLexer_errorDescription(IoLexer *self)
  49 {
  50         IoToken *et = IoLexer_errorToken(self);
  51
  52         if (!self->errorDescription)
  53         {
  54                 self->errorDescription = io_calloc(1, 1024);
  55                 self->errorDescription[0] = 0;
  56         }
  57
  58         if (et)
  59         {
  60                 sprintf(self->errorDescription,
  61                         "\"%s\" on line %i character %i",
  62                         et->error,
  63                         IoToken_lineNumber(et),
  64                         IoToken_charNumber(et));
  65         }
  66
  67         return self->errorDescription;
  68 }
  69
  70
  71 void IoLexer_buildLineIndex(IoLexer *self)
  72 {
  73         char *s = self->s;
  74
  75         List_removeAll(self->charLineIndex);
  76
  77         List_append_(self->charLineIndex, s);
  78
  79         while (*s)
  80         {
  81                 if (*s == '\n')
  82                 {
  83                         List_append_(self->charLineIndex, s);
  84                 }
  85                 s ++;
  86         }
  87
  88         List_append_(self->charLineIndex, s);
  89         self->lineHint = 0;
  90 }
  91
  92 // next/prev character ------------------------
  93
  94 #define UTF8_SEQLEN(c) (    \
  95     (c) < 0x80 ? 1 :        \
  96     (c) < 0xe0 ? 2 :        \
  97     (c) < 0xf0 ? 3 :        \
  98     (c) < 0xf8 ? 4 :        \
  99     (c) < 0xfc ? 5 :        \
 100     (c) < 0xfe ? 6 : 1      \
 101     )
 102 #define INVALID_CHAR 0xfffe
 103
 104 static uchar_t _IoLexer_DecodeUTF8(const unsigned char *s)
 105 {
 106     if (*s < 0x80)
 107         return *s;
 108     else if (*s < 0xc2)
 109         return INVALID_CHAR;
 110     else if (*s < 0xe0)
 111     {
 112         if (!((s[1] ^ 0x80) < 0x40))
 113             return INVALID_CHAR;
 114         return ((uchar_t)(s[0] & 0x1f) << 6) | (uchar_t)(s[1] ^ 0x80);
 115     }
 116     else if (*s < 0xf0)
 117     {
 118         if (!((s[1] ^ 0x80) < 0x40 && (s[2] ^ 0x80) < 0x40 && (s[0] >= 0xe1 || s[1] >= 0xa0)))
 119             return INVALID_CHAR;
 120         return ((uchar_t)(s[0] & 0x0f) << 12) | ((uchar_t)(s[1] ^ 0x80) << 6) | (uchar_t)(s[2] ^ 0x80);
 121     }
 122     else if (*s < 0xf8)
 123     {
 124         if (!((s[1] ^ 0x80) < 0x40 && (s[2] ^ 0x80) < 0x40 && (s[3] ^ 0x80) < 0x40 && (s[0] >= 0xf1 || s[1] >= 0x90)))
 125             return INVALID_CHAR;
 126         return ((uchar_t)(s[0] & 0x07) << 18) | ((uchar_t)(s[1] ^ 0x80) << 12) | ((uchar_t)(s[2] ^ 0x80) << 6) | (uchar_t)(s[3] ^ 0x80);
 127     }
 128     else if (*s < 0xfc)
 129     {
 130         if (!((s[1] ^ 0x80) < 0x40 && (s[2] ^ 0x80) < 0x40 && (s[3] ^ 0x80) < 0x40 && (s[4] ^ 0x80) < 0x40 && (s[0] >= 0xf9 || s[1] >= 0x88)))
 131             return INVALID_CHAR;
 132         return ((uchar_t)(s[0] & 0x03) << 24) | ((uchar_t)(s[1] ^ 0x80) << 18) | ((uchar_t)(s[2] ^ 0x80) << 12) | ((uchar_t)(s[3] ^ 0x80) << 6) | (uchar_t)(s[4] ^ 0x80);
 133     }
 134     else if (*s < 0xfe)
 135     {
 136         if (!((s[1] ^ 0x80) < 0x40 && (s[2] ^ 0x80) < 0x40 && (s[3] ^ 0x80) < 0x40 && (s[4] ^ 0x80) < 0x40 && (s[5] ^ 0x80) < 0x40 && (s[0] >= 0xfd || s[1] >= 0x84)))
 137             return INVALID_CHAR;
 138         return ((uchar_t)(s[0] & 0x01) << 30) | ((uchar_t)(s[1] ^ 0x80) << 24) | ((uchar_t)(s[2] ^ 0x80) << 18) | ((uchar_t)(s[3] ^ 0x80) << 12) | ((uchar_t)(s[4] ^ 0x80) << 6) | (uchar_t)(s[5] ^ 0x80);
 139     }
 140     else
 141         return INVALID_CHAR;
 142 }
 143
 144 TEST_INLINE uchar_t IoLexer_nextChar(IoLexer *self)
 145 {
 146     unsigned char c = (unsigned char) * (self->current);
 147     int seqlen, i;
 148     uchar_t uch;
 149
 150     if (c == 0)
 151     {
 152         return 0;
 153         }
 154     else if (c < 0x80)
 155     {
 156         self->current++;
 157         return c;
 158     }
 159
 160     seqlen = UTF8_SEQLEN(c);
 161
 162     for (i = 0; i < seqlen; i++)
 163     {
 164         if (self->current[i] == 0)
 165         {
 166             // XXX: invalid or incomplete sequence
 167             return 0;
 168         }
 169     }
 170
 171     uch = _IoLexer_DecodeUTF8((unsigned char*)self->current);
 172
 173     if (uch == INVALID_CHAR)
 174     {
 175         return 0;
 176         }
 177
 178     self->current += seqlen;
 179     return uch;
 180 }
 181
 182 TEST_INLINE uchar_t IoLexer_prevChar(IoLexer *self)
 183 {
 184     uchar_t uch;
 185     int len;
 186
 187     for (len = 1; len <= 6 && self->current - len > self->s; len++)
 188     {
 189         unsigned char c = *(unsigned char *)(self->current - len);
 190         if (c < 0x80 || c >= 0xc2)
 191             break;
 192     }
 193
 194     self->current -= len;
 195     uch = _IoLexer_DecodeUTF8((unsigned char*)self->current);
 196     if (uch == INVALID_CHAR)
 197         return 0;
 198
 199     return uch;
 200 }
 201
 202 TEST_INLINE char *IoLexer_current(IoLexer *self)
 203 {
 204         return self->current;
 205 }
 206
 207 TEST_INLINE int IoLexer_onNULL(IoLexer *self)
 208 {
 209         return (*(self->current) == 0);
 210 }
 211
 212 // ------------------------------------------
 213
 214 size_t IoLexer_currentLineNumberOld(IoLexer *self)
 215 {
 216         size_t lineNumber = 1;
 217         char *s = self->s;
 218
 219         while (s < self->current)
 220         {
 221                 if (*s == '\n')
 222                 {
 223                         lineNumber ++;
 224                 }
 225
 226                 s ++;
 227         }
 228
 229         return lineNumber;
 230 }
 231
 232 TEST_INLINE size_t IoLexer_currentLineNumber(IoLexer *self)
 233 {
 234         // this should be even faster than a binary search
 235         // since almost all results are very close to the last
 236
 237         List *index = self->charLineIndex;
 238         size_t line = self->lineHint;
 239         size_t numLines = List_size(index);
 240         void *current = (void *)self->current;
 241
 242         if (current < List_at_(index, line))
 243         {
 244                 // walk down lines until char is bigger than one
 245                 while (line > 0 && !(current > List_at_(index, line)))
 246                 {
 247                         line --;
 248                 }
 249                 line ++;
 250         }
 251         else
 252         {
 253                 // walk up lines until char is less than or equal to one
 254                 while (line < numLines && !(current <= List_at_(index, line)))
 255                 {
 256                         line ++;
 257                 }
 258         }
 259
 260
 261         self->lineHint = line;
 262
 263         /*
 264          {
 265                  size_t realLine = IoLexer_currentLineNumberOld(self);
 266
 267                  if (line != realLine)
 268                  {
 269                          printf("mismatch on currentLine %i != %i\n", (int)line, (int)realLine);
 270                  }
 271          }
 272          */
 273         return line;
 274 }
 275
 276 void IoLexer_clear(IoLexer *self)
 277 {
 278         LIST_FOREACH(self->tokenStream, i, t, IoToken_free((IoToken *)t) );
 279         List_removeAll(self->tokenStream);
 280
 281         Stack_clear(self->posStack);
 282         Stack_clear(self->tokenStack);
 283
 284         self->current = self->s;
 285         self->resultIndex = 0;
 286         self->maxChar = 0;
 287         self->errorToken = NULL;
 288 }
 289
 290 IoToken *IoLexer_errorToken(IoLexer *self)
 291 {
 292         return self->errorToken;
 293 }
 294
 295 // lexing -------------------------------------
 296
 297 void IoLexer_string_(IoLexer *self, const char *string)
 298 {
 299         self->s = strcpy((char *)io_realloc(self->s, strlen(string) + 1), string);
 300         self->current = self->s;
 301         IoLexer_buildLineIndex(self);
 302 }
 303
 304 void IoLexer_printLast_(IoLexer *self, int max)
 305 {
 306         char *s = self->s + self->maxChar;
 307         int i;
 308
 309         for (i = 0; i < max && s[i]; i ++)
 310         {
 311                 putchar(s[i]);
 312         }
 313 }
 314
 315 // --- token and character position stacks ---
 316
 317 char *IoLexer_lastPos(IoLexer *self)
 318 {
 319         return Stack_top(self->posStack);
 320 }
 321
 322 TEST_INLINE void IoLexer_pushPos(IoLexer *self)
 323 {
 324         intptr_t index = self->current - self->s;
 325
 326         if (index > (intptr_t)self->maxChar)
 327         {
 328                 self->maxChar = index;
 329         }
 330
 331         Stack_push_(self->tokenStack, (void *)(intptr_t)(List_size(self->tokenStream) - 1));
 332         Stack_push_(self->posStack, self->current);
 333
 334 #ifdef LEXER_DEBUG
 335         printf("push: ");
 336         IoLexer_print(self);
 337 #endif
 338 }
 339
 340 TEST_INLINE void IoLexer_popPos(IoLexer *self)
 341 {
 342         Stack_pop(self->tokenStack);
 343         Stack_pop(self->posStack);
 344 #ifdef LEXER_DEBUG
 345         printf("pop:    ");
 346         IoLexer_print(self);
 347 #endif
 348 }
 349
 350 TEST_INLINE void IoLexer_popPosBack(IoLexer *self)
 351 {
 352         intptr_t i = (intptr_t)Stack_pop(self->tokenStack);
 353         intptr_t topIndex = (intptr_t)Stack_top(self->tokenStack);
 354
 355         if (i > -1)
 356         {
 357                 List_setSize_(self->tokenStream, i + 1);
 358
 359                 if (i != topIndex) // ok to io_free token
 360                 {
 361                         IoToken *parent = IoLexer_currentToken(self);
 362
 363                         if (parent)
 364                         {
 365                                 IoToken_nextToken_(parent, NULL);
 366                         }
 367                 }
 368         }
 369
 370         self->current = Stack_pop(self->posStack);
 371 #ifdef LEXER_DEBUG
 372         printf("back: "); IoLexer_print(self);
 373 #endif
 374 }
 375
 376 // ------------------------------------------
 377
 378 int IoLexer_lex(IoLexer *self)
 379 {
 380         IoLexer_clear(self);
 381         IoLexer_pushPos(self);
 382
 383         IoLexer_messageChain(self);
 384
 385         if (*(self->current))
 386         {
 387                 //printf("Lexing error after: ");
 388                 //IoLexer_printLast_(self, 30);
 389                 //printf("\n");
 390
 391                 if (!self->errorToken)
 392                 {
 393                         if (List_size(self->tokenStream))
 394                         {
 395                                 self->errorToken = IoLexer_currentToken(self);
 396                         }
 397                         else
 398                         {
 399                                 self->errorToken = IoLexer_addTokenString_length_type_(self, self->current, 30, NO_TOKEN);
 400                         }
 401
 402                         IoToken_error_(self->errorToken, "Syntax error near this location");
 403                 }
 404                 return -1;
 405         }
 406         return 0;
 407 }
 408
 409 // getting results --------------------------------
 410
 411 IoToken *IoLexer_top(IoLexer *self)
 412 {
 413         return List_at_(self->tokenStream, self->resultIndex);
 414 }
 415
 416 IoTokenType IoLexer_topType(IoLexer *self)
 417 {
 418         if (!IoLexer_top(self))
 419         {
 420                 return 0;
 421         }
 422
 423         return IoLexer_top(self)->type;
 424 }
 425
 426 IoToken *IoLexer_pop(IoLexer *self)
 427 {
 428         IoToken *t = IoLexer_top(self);
 429         self->resultIndex ++;
 430         return t;
 431 }
 432
 433 // stack management --------------------------------
 434
 435 void IoLexer_print(IoLexer *self)
 436 {
 437         IoToken *first = List_first(self->tokenStream);
 438
 439         if (first)
 440         {
 441                 IoToken_print(first);
 442         }
 443
 444         printf("\n");
 445 }
 446
 447 void IoLexer_printTokens(IoLexer *self)
 448 {
 449         int i;
 450
 451         for (i = 0; i < List_size(self->tokenStream); i ++)
 452         {
 453                 IoToken *t = List_at_(self->tokenStream, i);
 454
 455                 printf("'%s'", t->name);
 456                 printf(" %s ", IoToken_typeName(t));
 457
 458                 if (i < List_size(self->tokenStream) - 1)
 459                 {
 460                         printf(", ");
 461                 }
 462         }
 463
 464         printf("\n");
 465 }
 466
 467 // grabbing ---------------------------------------------
 468
 469 int IoLexer_grabLength(IoLexer *self)
 470 {
 471         char *s1 = IoLexer_lastPos(self);
 472         char *s2 = IoLexer_current(self);
 473
 474         return s2 - s1;
 475 }
 476
 477 void IoLexer_grabTokenType_(IoLexer *self, IoTokenType type)
 478 {
 479         char *s1 = IoLexer_lastPos(self);
 480         char *s2 = IoLexer_current(self);
 481         size_t len = (s2 - s1);
 482
 483         if (!len)
 484         {
 485                 printf("IoLexer fatal error: empty token\n");
 486                 exit(1);
 487         }
 488
 489         IoLexer_addTokenString_length_type_(self, s1, len, type);
 490 }
 491
 492 IoToken *IoLexer_addTokenString_length_type_(IoLexer *self, const char *s1, size_t len, IoTokenType type)
 493 {
 494         IoToken *top = IoLexer_currentToken(self);
 495         IoToken *t = IoToken_new();
 496
 497         t->lineNumber = IoLexer_currentLineNumber(self);
 498         //t->charNumber = (int)(s1 - self->s);
 499         t->charNumber = (int)(self->current - self->s);
 500
 501         if (t->charNumber < 0)
 502         {
 503                 printf("bad t->charNumber = %i\n", t->charNumber);
 504         }
 505
 506         IoToken_name_length_(t, s1, len);
 507         IoToken_type_(t, type);
 508
 509         if (top)
 510         {
 511                 IoToken_nextToken_(top, t);
 512         }
 513
 514         List_push_(self->tokenStream, t);
 515 #ifdef LEXER_DEBUG_TOKENS
 516         printf("token '%s' %s\n", t->name, IoToken_typeName(t));
 517 #endif
 518
 519         return t;
 520 }
 521
 522 // reading ------------------------------------
 523
 524 void IoLexer_messageChain(IoLexer *self)
 525 {
 526         do
 527         {
 528                 while ( IoLexer_readTerminator(self) ||
 529                                 IoLexer_readSeparator(self) ||
 530                                 IoLexer_readComment(self))
 531                 {}
 532         } while ( IoLexer_readMessage(self));
 533 }
 534
 535 // message -------------------------------
 536
 537 static void IoLexer_readMessage_error(IoLexer *self, const char *name)
 538 {
 539         IoLexer_popPosBack(self);
 540         self->errorToken = IoLexer_currentToken(self);
 541         IoToken_error_(self->errorToken, name);
 542 }
 543
 544 int IoLexer_readTokenChars_type_(IoLexer *self, const char *chars, IoTokenType type)
 545 {
 546         while (*chars)
 547         {
 548                 if (IoLexer_readTokenChar_type_(self, *chars, type)) return 1;
 549                 chars ++;
 550         }
 551
 552         return 0;
 553 }
 554
 555 const char *IoLexer_nameForGroupChar_(IoLexer *self, char groupChar)
 556 {
 557         switch (groupChar)
 558         {
 559                 case '(': return "";
 560                 case '[': return "squareBrackets";
 561                 case '{': return "curlyBrackets";
 562         }
 563
 564         printf("IoLexer: fatal error - invalid group char %c\n", groupChar);
 565         exit(1);
 566 }
 567
 568 static char *specialChars = ":._";
 569
 570 int IoLexer_readMessage(IoLexer *self)
 571 {
 572         char foundSymbol;
 573
 574         IoLexer_pushPos(self);
 575         IoLexer_readPadding(self);
 576
 577         foundSymbol = IoLexer_readSymbol(self);
 578
 579
 580         {
 581                 char groupChar;
 582                 while (IoLexer_readSeparator(self) || IoLexer_readComment(self))
 583                 {}
 584
 585                 groupChar = *IoLexer_current(self);
 586
 587                 if (groupChar && (strchr("[{", groupChar) || (!foundSymbol && groupChar == '(')))
 588                 {
 589                         char *groupName = (char *)IoLexer_nameForGroupChar_(self, groupChar);
 590                         IoLexer_addTokenString_length_type_(self, groupName, strlen(groupName), IDENTIFIER_TOKEN);
 591                 }
 592
 593                 if (IoLexer_readTokenChars_type_(self, "([{", OPENPAREN_TOKEN))
 594                 {
 595                         IoLexer_readPadding(self);
 596                         do {
 597                                 IoTokenType type = IoLexer_currentToken(self)->type;
 598
 599                                 IoLexer_readPadding(self);
 600                                 // Empty argument: (... ,)
 601                                 if (COMMA_TOKEN == type)
 602                                 {
 603                                         char c = *IoLexer_current(self);
 604
 605                                         if (',' == c || strchr(")]}", c))
 606                                         {
 607                                                 IoLexer_readMessage_error(self, "missing argument in argument list");
 608                                                 return 0;
 609                                         }
 610                                 }
 611
 612                                 if (groupChar == '[') specialChars = "._";
 613                                 IoLexer_messageChain(self);
 614                                 if (groupChar == '[') specialChars = ":._";
 615                                 IoLexer_readPadding(self);
 616
 617                         } while (IoLexer_readTokenChar_type_(self, ',', COMMA_TOKEN));
 618
 619                         if (!IoLexer_readTokenChars_type_(self, ")]}", CLOSEPAREN_TOKEN))
 620                         {
 621                                 /*
 622                                 char c = *IoLexer_current(self);
 623
 624                                 if (strchr("([{", c))
 625                                 {
 626                                         IoLexer_readMessage_error(self, "expected a message but instead found a open group character");
 627                                 }
 628                                 else
 629                                 {
 630                                         IoLexer_readMessage_error(self, "missing closing group character for argument list");
 631                                 }
 632                                 */
 633                                 if (groupChar == '(')
 634                                 {
 635                                         IoLexer_readMessage_error(self, "unmatched ()s");
 636                                 }
 637                                 else if (groupChar == '[')
 638                                 {
 639                                         IoLexer_readMessage_error(self, "unmatched []s");
 640                                 }
 641                                 else if (groupChar == '{')
 642                                 {
 643                                         IoLexer_readMessage_error(self, "unmatched {}s");
 644                                 }
 645                                 //printf("Token %p error: %s - %s\n", t, t->error, IoToken_error(t));
 646                                 return 0;
 647                         }
 648
 649                         IoLexer_popPos(self);
 650                         return 1;
 651                 }
 652
 653                 if (foundSymbol)
 654                 {
 655                         IoLexer_popPos(self);
 656                         return 1;
 657                 }
 658         }
 659         IoLexer_popPosBack(self);
 660         return 0;
 661 }
 662
 663 int IoLexer_readPadding(IoLexer *self)
 664 {
 665         int r = 0;
 666
 667         while (IoLexer_readWhitespace(self) || IoLexer_readComment(self))
 668         {
 669                 r = 1;
 670         }
 671
 672         return r;
 673 }
 674
 675 // symbols ------------------------------------------
 676
 677 int IoLexer_readSymbol(IoLexer *self)
 678 {
 679         if (    IoLexer_readNumber(self) ||
 680                    IoLexer_readOperator(self) ||
 681                    IoLexer_readIdentifier(self) ||
 682                    IoLexer_readQuote(self)) return 1;
 683         return 0;
 684 }
 685
 686 int IoLexer_readIdentifier(IoLexer *self)
 687 {
 688         IoLexer_pushPos(self);
 689
 690         while ( IoLexer_readLetter(self) ||
 691                    IoLexer_readDigit(self) ||
 692                    IoLexer_readSpecialChar(self))
 693         {}
 694
 695         if (IoLexer_grabLength(self))
 696         {
 697                 // avoid grabing : on last character if followed by =
 698
 699                 char *current = IoLexer_current(self);
 700
 701                 if (*(current - 1) == ':' && *current == '=')
 702                 {
 703                         IoLexer_prevChar(self);
 704                 }
 705
 706
 707                 IoLexer_grabTokenType_(self, IDENTIFIER_TOKEN);
 708                 IoLexer_popPos(self);
 709                 return 1;
 710         }
 711
 712         IoLexer_popPosBack(self);
 713
 714         return 0;
 715 }
 716
 717 int IoLexer_readOperator(IoLexer *self)
 718 {
 719         uchar_t c;
 720         IoLexer_pushPos(self);
 721         // ok if first character is a colon
 722         c = IoLexer_nextChar(self);
 723         //printf("IoLexer_nextChar(self) = %c %i\n", c, c);
 724
 725         if (c == 0)
 726         {
 727                 IoLexer_popPosBack(self);
 728                 return 0;
 729         }
 730         else
 731         {
 732                 IoLexer_prevChar(self);
 733         }
 734         /*
 735         if (c != ':')
 736         {
 737                 IoLexer_prevChar(self);
 738         }
 739         */
 740
 741         while (IoLexer_readOpChar(self))
 742         { }
 743
 744         if (IoLexer_grabLength(self))
 745         {
 746                 IoLexer_grabTokenType_(self, IDENTIFIER_TOKEN);
 747                 IoLexer_popPos(self);
 748                 return 1;
 749         }
 750
 751         IoLexer_popPosBack(self);
 752         return 0;
 753 }
 754
 755 // comments ------------------------------------------
 756
 757 int IoLexer_readComment(IoLexer *self)
 758 {
 759         return (IoLexer_readSlashStarComment(self) ||
 760             IoLexer_readSlashSlashComment(self) ||
 761             IoLexer_readPoundComment(self));
 762 }
 763
 764 int IoLexer_readSlashStarComment(IoLexer *self)
 765 {
 766         IoLexer_pushPos(self);
 767
 768         if (IoLexer_readString_(self, "/*"))
 769         {
 770                 unsigned int nesting = 1;
 771
 772                 while (nesting > 0)
 773                 {
 774                         if (IoLexer_readString_(self, "/*"))
 775                         {
 776                                 IoLexer_nextChar(self);
 777                                 nesting++;
 778                         }
 779                         else if (IoLexer_readString_(self, "*/"))
 780                         {
 781                                 // otherwise we end up trimming the last char
 782                                 if (nesting > 1) IoLexer_nextChar(self);
 783                                 nesting--;
 784                         }
 785                         else
 786                                 IoLexer_nextChar(self);
 787                 }
 788                 IoLexer_popPos(self);
 789                 return 1;
 790          }
 791
 792         IoLexer_popPosBack(self);
 793         return 0;
 794 }
 795
 796 int IoLexer_readSlashSlashComment(IoLexer *self)
 797 {
 798         IoLexer_pushPos(self);
 799
 800         if (IoLexer_nextChar(self) == '/')
 801         {
 802                 if (IoLexer_nextChar(self) == '/')
 803                 {
 804                         while (IoLexer_readNonReturn(self)) { }
 805                         //IoLexer_grabTokenType_(self, COMMENT_TOKEN);
 806                         IoLexer_popPos(self);
 807                         return 1;
 808                 }
 809         }
 810
 811         IoLexer_popPosBack(self);
 812         return 0;
 813 }
 814
 815 int IoLexer_readPoundComment(IoLexer *self)
 816 {
 817         IoLexer_pushPos(self);
 818
 819         if (IoLexer_nextChar(self) == '#')
 820         {
 821                 while (IoLexer_readNonReturn(self))
 822                 {
 823                 }
 824                 //IoLexer_grabTokenType_(self, COMMENT_TOKEN);
 825                 IoLexer_popPos(self);
 826                 return 1;
 827         }
 828
 829         IoLexer_popPosBack(self);
 830         return 0;
 831 }
 832
 833 // quotes -----------------------------------------
 834
 835 int IoLexer_readQuote(IoLexer *self)
 836 {
 837         return (IoLexer_readTriQuote(self) || IoLexer_readMonoQuote(self));
 838 }
 839
 840 int IoLexer_readMonoQuote(IoLexer *self)
 841 {
 842         int mbskip = 0; // multi-byte character length
 843
 844         IoLexer_pushPos(self);
 845
 846         if (IoLexer_nextChar(self) == '"')
 847         {
 848                 for (;;)
 849                 {
 850                         uchar_t c = IoLexer_nextChar(self);
 851
 852                         if (mbskip <= 0 && ismbchar(c))
 853                         {
 854                                 mbskip = mbcharlen(c);
 855                         }
 856
 857                         if (mbskip-- > 0)
 858                         {
 859                                 continue;
 860                         }
 861
 862                         if (c == '"')
 863                         {
 864                                 break;
 865                         }
 866
 867                         if (c == '\\')
 868                         {
 869                                 IoLexer_nextChar(self);
 870                                 continue;
 871                         }
 872
 873                         if (c == 0)
 874                         {
 875                                 self->errorToken = IoLexer_currentToken(self);
 876
 877                                 if (self->errorToken)
 878                                 {
 879                                         IoToken_error_(self->errorToken, "unterminated quote");
 880                                 }
 881
 882                                 IoLexer_popPosBack(self);
 883                                 return 0;
 884                         }
 885                 }
 886
 887                 IoLexer_grabTokenType_(self, MONOQUOTE_TOKEN);
 888                 IoLexer_popPos(self);
 889                 return 1;
 890         }
 891
 892         IoLexer_popPosBack(self);
 893         return 0;
 894 }
 895
 896 int IoLexer_readTriQuote(IoLexer *self)
 897 {
 898         IoLexer_pushPos(self);
 899
 900         if (IoLexer_readString_(self, "\"\"\""))
 901         {
 902                 while (!IoLexer_readString_(self, "\"\"\""))
 903                 {
 904                         uchar_t c = IoLexer_nextChar(self);
 905
 906                         if (c == 0)
 907                         {
 908                                 IoLexer_popPosBack(self);
 909                                 return 0;
 910                         }
 911                 }
 912
 913                 IoLexer_grabTokenType_(self, TRIQUOTE_TOKEN);
 914                 IoLexer_popPos(self);
 915                 return 1;
 916         }
 917
 918         IoLexer_popPosBack(self);
 919         return 0;
 920 }
 921
 922 // helpers ----------------------------
 923
 924 int IoLexer_readTokenChar_type_(IoLexer *self, char c, IoTokenType type)
 925 {
 926         IoLexer_pushPos(self);
 927
 928         if (IoLexer_readChar_(self, c))
 929         {
 930                 IoLexer_grabTokenType_(self, type);
 931                 IoLexer_popPos(self);
 932                 return 1;
 933         }
 934
 935         IoLexer_popPosBack(self);
 936         return 0;
 937 }
 938
 939 int IoLexer_readTokenString_(IoLexer *self, const char *s)
 940 {
 941         IoLexer_pushPos(self);
 942
 943         if (IoLexer_readString_(self, s))
 944         {
 945                 IoLexer_grabTokenType_(self, IDENTIFIER_TOKEN);
 946                 IoLexer_popPos(self);
 947                 return 1;
 948         }
 949
 950         IoLexer_popPosBack(self);
 951         return 0;
 952 }
 953
 954
 955 int IoLexer_readString_(IoLexer *self, const char *s)
 956 {
 957         int len = strlen(s);
 958
 959         if (IoLexer_onNULL(self))
 960         {
 961                 return 0;
 962         }
 963
 964         if (strncmp(self->current, s, len) == 0)
 965         {
 966                 self->current += len;
 967                 return 1;
 968         }
 969
 970         return 0;
 971 }
 972
 973 TEST_INLINE int IoLexer_readCharIn_(IoLexer *self, const char *s)
 974 {
 975         if (!IoLexer_onNULL(self))
 976         {
 977                 uchar_t c = IoLexer_nextChar(self);
 978
 979                 if (c < 0x80 && strchr(s, c))
 980                 {
 981                         return 1;
 982                 }
 983
 984                 IoLexer_prevChar(self);
 985         }
 986         return 0;
 987 }
 988
 989 TEST_INLINE int IoLexer_readCharInRange_(IoLexer *self, uchar_t first, uchar_t last)
 990 {
 991         if (!IoLexer_onNULL(self))
 992         {
 993                 uchar_t c = IoLexer_nextChar(self);
 994
 995                 if (c >= first && c <= last)
 996                 {
 997                         return 1;
 998                 }
 999
1000                 IoLexer_prevChar(self);
1001         }
1002         return 0;
1003 }
1004
1005 int IoLexer_readChar_(IoLexer *self, char c)
1006 {
1007         if (!IoLexer_onNULL(self))
1008         {
1009                 uchar_t nc = IoLexer_nextChar(self);
1010
1011                 if (nc && nc == c)
1012                 {
1013                         return 1;
1014                 }
1015
1016                 IoLexer_prevChar(self);
1017         }
1018         return 0;
1019 }
1020
1021 int IoLexer_readCharAnyCase_(IoLexer *self, char c)
1022 {
1023         if (!IoLexer_onNULL(self))
1024         {
1025                 uchar_t nc = IoLexer_nextChar(self);
1026
1027                 if (nc && tolower(nc) == tolower(c))
1028                 {
1029                         return 1;
1030                 }
1031
1032                 IoLexer_prevChar(self);
1033         }
1034         return 0;
1035 }
1036
1037 int IoLexer_readNonASCIIChar_(IoLexer *self)
1038 {
1039     if (!IoLexer_onNULL(self))
1040     {
1041         uchar_t nc = IoLexer_nextChar(self);
1042
1043         if (nc >= 0x80)
1044             return 1;
1045
1046         IoLexer_prevChar(self);
1047     }
1048     return 0;
1049 }
1050
1051 int IoLexer_readNonReturn(IoLexer *self)
1052 {
1053         if (IoLexer_onNULL(self)) return 0;
1054         if (IoLexer_nextChar(self) != '\n') return 1;
1055         IoLexer_prevChar(self);
1056         return 0;
1057 }
1058
1059 int IoLexer_readNonQuote(IoLexer *self)
1060 {
1061         if (IoLexer_onNULL(self)) return 0;
1062         if (IoLexer_nextChar(self) != '"') return 1;
1063         IoLexer_prevChar(self);
1064         return 0;
1065 }
1066
1067 // character definitions ----------------------------
1068
1069 int IoLexer_readCharacters(IoLexer *self)
1070 {
1071         int read = 0;
1072
1073         while (IoLexer_readCharacter(self))
1074         {
1075                 read = 1;
1076         }
1077
1078         return read;
1079 }
1080
1081 int IoLexer_readCharacter(IoLexer *self)
1082 {
1083         return (
1084                    IoLexer_readLetter(self) ||
1085                    IoLexer_readDigit(self) ||
1086                    IoLexer_readSpecialChar(self) ||
1087                    IoLexer_readOpChar(self)
1088                    );
1089 }
1090
1091 int IoLexer_readOpChar(IoLexer *self)
1092 {
1093         return IoLexer_readCharIn_(self, ":'~!@$%^&*-+=|\\<>?/");
1094 }
1095
1096 int IoLexer_readSpecialChar(IoLexer *self)
1097 {
1098         return IoLexer_readCharIn_(self, specialChars);
1099 }
1100
1101 int IoLexer_readDigit(IoLexer *self)
1102 {
1103         return IoLexer_readCharInRange_(self, '0', '9');
1104 }
1105
1106 int IoLexer_readLetter(IoLexer *self)
1107 {
1108         return IoLexer_readCharInRange_(self, 'A', 'Z') ||
1109             IoLexer_readCharInRange_(self, 'a', 'z') ||
1110         IoLexer_readNonASCIIChar_(self);
1111 }
1112
1113 // terminator -------------------------------
1114
1115 int IoLexer_readTerminator(IoLexer *self)
1116 {
1117         int terminated = 0;
1118         IoLexer_pushPos(self);
1119         IoLexer_readSeparator(self);
1120
1121         while (IoLexer_readTerminatorChar(self))
1122         {
1123                 terminated = 1;
1124                 IoLexer_readSeparator(self);
1125         }
1126
1127         if (terminated)
1128         {
1129                 IoToken *top = IoLexer_currentToken(self);
1130
1131                 // avoid double terminators
1132                 if (top && IoToken_type(top) == TERMINATOR_TOKEN)
1133                 {
1134                         return 1;
1135                 }
1136
1137                 IoLexer_addTokenString_length_type_(self, ";", 1, TERMINATOR_TOKEN);
1138                 IoLexer_popPos(self);
1139                 return 1;
1140         }
1141
1142         IoLexer_popPosBack(self);
1143         return 0;
1144 }
1145
1146 int IoLexer_readTerminatorChar(IoLexer *self)
1147 {
1148         return IoLexer_readCharIn_(self, ";\n");
1149 }
1150
1151 // separator --------------------------------
1152
1153 int IoLexer_readSeparator(IoLexer *self)
1154 {
1155         IoLexer_pushPos(self);
1156
1157         while (IoLexer_readSeparatorChar(self))
1158         {
1159         }
1160
1161         if (IoLexer_grabLength(self))
1162         {
1163                 //IoLexer_grabTokenType_(self, SEPERATOR_TOKEN);
1164                 IoLexer_popPos(self);
1165                 return 1;
1166         }
1167
1168         IoLexer_popPosBack(self);
1169         return 0;
1170 }
1171
1172 int IoLexer_readSeparatorChar(IoLexer *self)
1173 {
1174         if (IoLexer_readCharIn_(self, " \f\r\t\v"))
1175         {
1176                 return 1;
1177         }
1178         else
1179         {
1180                 IoLexer_pushPos(self);
1181                 if (IoLexer_readCharIn_(self, "\\"))
1182                 {
1183                         while (IoLexer_readCharIn_(self, " \f\r\t\v"))
1184                         {
1185                         }
1186
1187                         if (IoLexer_readCharIn_(self, "\n"))
1188                         {
1189                                 IoLexer_popPos(self);
1190                                 return 1;
1191                         }
1192                 }
1193                 IoLexer_popPosBack(self);
1194                 return 0;
1195         }
1196 }
1197
1198 // whitespace -----------------------------------
1199
1200 int IoLexer_readWhitespace(IoLexer *self)
1201 {
1202         IoLexer_pushPos(self);
1203
1204         while (IoLexer_readWhitespaceChar(self))
1205         {
1206         }
1207
1208         if (IoLexer_grabLength(self))
1209         {
1210                 //IoLexer_grabTokenType_(self, WHITESPACE_TOKEN);
1211                 IoLexer_popPos(self);
1212                 return 1;
1213         }
1214
1215         IoLexer_popPosBack(self);
1216         return 0;
1217 }
1218
1219 int IoLexer_readWhitespaceChar(IoLexer *self)
1220 {
1221         return IoLexer_readCharIn_(self, " \f\r\t\v\n");
1222 }
1223
1224 int IoLexer_readDigits(IoLexer *self)
1225 {
1226         int read = 0;
1227
1228         IoLexer_pushPos(self);
1229
1230         while (IoLexer_readDigit(self))
1231         {
1232                 read = 1;
1233         }
1234
1235         if (!read)
1236         {
1237                 IoLexer_popPosBack(self);
1238                 return 0;
1239         }
1240
1241         IoLexer_popPos(self);
1242         return read;
1243 }
1244
1245 int IoLexer_readNumber(IoLexer *self)
1246 {
1247         return (IoLexer_readHexNumber(self) || IoLexer_readDecimal(self));
1248 }
1249
1250 int IoLexer_readExponent(IoLexer *self)
1251 {
1252         if (IoLexer_readCharAnyCase_(self, 'e'))
1253         {
1254                 IoLexer_readChar_(self, '-') || IoLexer_readChar_(self, '+');
1255
1256                 if (!IoLexer_readDigits(self))
1257                 {
1258                         return -1;
1259                 }
1260
1261                 return 1;
1262         }
1263         return 0;
1264 }
1265
1266 int IoLexer_readDecimalPlaces(IoLexer *self)
1267 {
1268         if (IoLexer_readChar_(self, '.'))
1269         {
1270                 if (!IoLexer_readDigits(self))
1271                 {
1272                         return -1;
1273                 }
1274
1275                 return 1;
1276         }
1277         return 0;
1278 }
1279
1280 int IoLexer_readDecimal(IoLexer *self)
1281 {
1282         IoLexer_pushPos(self);
1283
1284         if (IoLexer_readDigits(self))
1285         {
1286                 if (IoLexer_readDecimalPlaces(self) == -1)
1287                 {
1288                         goto error;
1289                 }
1290         }
1291         else
1292         {
1293                 if (IoLexer_readDecimalPlaces(self) != 1)
1294                 {
1295                         goto error;
1296                 }
1297         }
1298
1299         if (IoLexer_readExponent(self) == -1)
1300         {
1301                 goto error;
1302         }
1303
1304         if (IoLexer_grabLength(self))
1305         {
1306                 IoLexer_grabTokenType_(self, NUMBER_TOKEN);
1307                 IoLexer_popPos(self);
1308                 return 1;
1309         }
1310 error:
1311                 IoLexer_popPosBack(self);
1312         return 0;
1313 }
1314
1315 int IoLexer_readHexNumber(IoLexer *self)
1316 {
1317         int read = 0;
1318
1319         IoLexer_pushPos(self);
1320
1321         if (IoLexer_readChar_(self, '0') && IoLexer_readCharAnyCase_(self, 'x'))
1322         {
1323                 while (IoLexer_readDigits(self) || IoLexer_readCharacters(self))
1324                 {
1325                         read ++;
1326                 }
1327         }
1328
1329         if (read && IoLexer_grabLength(self))
1330         {
1331                 IoLexer_grabTokenType_(self, HEXNUMBER_TOKEN);
1332                 IoLexer_popPos(self);
1333                 return 1;
1334         }
1335
1336         IoLexer_popPosBack(self);
1337         return 0;
1338 }