vala/valageniescanner.vala

   1 /* valageniescanner.vala
   2  *
   3  * Copyright (C) 2008  Jamie McCracken, Jürg Billeter
   4  * Based on code by Jürg Billeter
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
  19  *
  20  * Author:
  21  *      Jamie McCracken jamiemcc gnome org
  22  */
  23
  24 using GLib;
  25 using Gee;
  26
  27 /**
  28  * Lexical scanner for Genie source files.
  29  */
  30 public class Vala.Genie.Scanner {
  31         public SourceFile source_file { get; private set; }
  32
  33         public int indent_spaces { get; set;}
  34
  35         char* begin;
  36         char* current;
  37         char* end;
  38
  39         int line;
  40         int column;
  41
  42         int current_indent_level;
  43         int indent_level;
  44         int pending_dedents;
  45
  46         TokenType last_token;
  47         bool parse_started;
  48
  49         string _comment;
  50
  51         public Scanner (SourceFile source_file) {
  52                 this.source_file = source_file;
  53
  54                 begin = source_file.get_mapped_contents ();
  55                 end = begin + source_file.get_mapped_length ();
  56
  57                 current = begin;
  58
  59                 _indent_spaces = 0;
  60                 line = 1;
  61                 column = 1;
  62                 current_indent_level = 0;
  63                 indent_level = 0;
  64                 pending_dedents = 0;
  65
  66                 parse_started = false;
  67                 last_token = TokenType.NONE;
  68
  69         }
  70
  71         bool is_ident_char (char c) {
  72                 return (c.isalnum () || c == '_');
  73         }
  74
  75         TokenType get_identifier_or_keyword (char* begin, int len) {
  76                 switch (len) {
  77                 case 2:
  78                         switch (begin[0]) {
  79                         case 'a':
  80                                 if (matches (begin, "as")) return TokenType.AS;
  81                                 break;
  82                         case 'd':
  83                                 if (matches (begin, "do")) return TokenType.DO;
  84                                 break;
  85                         case 'i':
  86                                 switch (begin[1]) {
  87                                 case 'f':
  88                                         return TokenType.IF;
  89                                 case 'n':
  90                                         return TokenType.IN;
  91                                 case 's':
  92                                         return TokenType.IS;
  93                                 }
  94                                 break;
  95                         case 'o':
  96                                 if (matches (begin, "of")) return TokenType.OF;
  97
  98                                 if (matches (begin, "or")) return TokenType.OP_OR;
  99                                 break;
 100                         case 't':
 101                                 if (matches (begin, "to")) return TokenType.TO;
 102                                 break;
 103                         }
 104                         break;
 105                 case 3:
 106                         switch (begin[0]) {
 107                         case 'a':
 108                                 if (matches (begin, "and")) return TokenType.OP_AND;
 109                                 break;
 110                         case 'd':
 111                                 if (matches (begin, "def")) return TokenType.DEF;
 112                                 break;
 113                         case 'f':
 114                                 if (matches (begin, "for")) return TokenType.FOR;
 115                                 break;
 116                         case 'g':
 117                                 if (matches (begin, "get")) return TokenType.GET;
 118                                 break;
 119                         case 'i':
 120                                 if (matches (begin, "isa")) return TokenType.ISA;
 121                                 break;
 122                         case 'n':
 123                                 switch (begin[1]) {
 124                                 case 'e':
 125                                         if (matches (begin, "new")) return TokenType.NEW;
 126                                         break;
 127                                 case 'o':
 128                                         if (matches (begin, "not")) return TokenType.OP_NEG;
 129                                         break;
 130                                 }
 131                                 break;
 132                         case 'o':
 133                                 if (matches (begin, "out")) return TokenType.OUT;
 134                                 break;
 135                         case 'r':
 136                                 if (matches (begin, "ref")) return TokenType.REF;
 137                                 break;
 138                         case 's':
 139                                 if (matches (begin, "set")) return TokenType.SET;
 140                                 break;
 141                         case 't':
 142                                 if (matches (begin, "try")) return TokenType.TRY;
 143                                 break;
 144                         case 'v':
 145                                 if (matches (begin, "var")) return TokenType.VAR;
 146                                 break;
 147                         }
 148                         break;
 149                 case 4:
 150                         switch (begin[0]) {
 151                         case 'c':
 152                                 if (matches (begin, "case")) return TokenType.CASE;
 153                                 break;
 154                         case 'd':
 155                                 if (matches (begin, "dict")) return TokenType.DICT;
 156                                 break;
 157                         case 'e':
 158                                 switch (begin[1]) {
 159                                 case 'l':
 160                                         if (matches (begin, "else")) return TokenType.ELSE;
 161                                         break;
 162                                 case 'n':
 163                                         if (matches (begin, "enum")) return TokenType.ENUM;
 164                                         break;
 165                                 }
 166                                 break;
 167                         case 'i':
 168                                 if (matches (begin, "init")) return TokenType.INIT;
 169                                 break;
 170                         case 'l':
 171                                 switch (begin[1]) {
 172                                 case 'i':
 173                                         if (matches (begin, "list")) return TokenType.LIST;
 174                                         break;
 175                                 case 'o':
 176                                         if (matches (begin, "lock")) return TokenType.LOCK;
 177                                         break;
 178                                 }
 179                                 break;
 180
 181                         case 'n':
 182                                 if (matches (begin, "null")) return TokenType.NULL;
 183                                 break;
 184                         case 'p':
 185                                 switch (begin[1]) {
 186                                 case 'a':
 187                                         if (matches (begin, "pass")) return TokenType.PASS;
 188                                         break;
 189                                 case 'r':
 190                                         if (matches (begin, "prop")) return TokenType.PROP;
 191                                         break;
 192                                 }
 193                                 break;
 194                         case 's':
 195                                 if (matches (begin, "self")) return TokenType.THIS;
 196                                 break;
 197                         case 't':
 198                                 if (matches (begin, "true")) return TokenType.TRUE;
 199                                 break;
 200                         case 'u':
 201                                 if (matches (begin, "uses")) return TokenType.USES;
 202                                 break;
 203                         case 'v':
 204                                 if (matches (begin, "void")) return TokenType.VOID;
 205                                 break;
 206                         case 'w':
 207                                 switch (begin[1]) {
 208                                 case 'e':
 209                                         if (matches (begin, "weak")) return TokenType.WEAK;
 210                                         break;
 211                                 case 'h':
 212                                         if (matches (begin, "when")) return TokenType.WHEN;
 213                                         break;
 214                                 }
 215                                 break;
 216                         }
 217                         break;
 218                 case 5:
 219                         switch (begin[0]) {
 220                         case 'a':
 221                                 if (matches (begin, "array")) return TokenType.ARRAY;
 222                                 break;
 223                         case 'b':
 224                                 if (matches (begin, "break")) return TokenType.BREAK;
 225                                 break;
 226                         case 'c':
 227                                 switch (begin[1]) {
 228                                 case 'l':
 229                                         if (matches (begin, "class")) return TokenType.CLASS;
 230                                         break;
 231                                 case 'o':
 232                                         if (matches (begin, "const")) return TokenType.CONST;
 233                                         break;
 234                                 }
 235                                 break;
 236                         case 'e':
 237                                 if (matches (begin, "event")) return TokenType.EVENT;
 238                                 break;
 239                         case 'f':
 240                                 switch (begin[1]) {
 241                                 case 'a':
 242                                         if (matches (begin, "false")) return TokenType.FALSE;
 243                                         break;
 244                                 case 'i':
 245                                         if (matches (begin, "final")) return TokenType.FINAL;
 246                                         break;
 247                                 }
 248                                 break;
 249                         case 'p':
 250                                 if (matches (begin, "print")) return TokenType.PRINT;
 251                                 break;
 252                         case 's':
 253                                 if (matches (begin, "super")) return TokenType.SUPER;
 254                                 break;
 255                         case 'r':
 256                                 if (matches (begin, "raise")) return TokenType.RAISE;
 257                                 break;
 258                         case 'w':
 259                                 if (matches (begin, "while")) return TokenType.WHILE;
 260                                 break;
 261                         }
 262                         break;
 263                 case 6:
 264                         switch (begin[0]) {
 265                         case 'a':
 266                                 if (matches (begin, "assert")) return TokenType.ASSERT;
 267                                 break;
 268                         case 'd':
 269                                 switch (begin[1]) {
 270                                 case 'e':
 271                                         if (matches (begin, "delete")) return TokenType.DELETE;
 272                                         break;
 273                                 case 'o':
 274                                         if (matches (begin, "downto")) return TokenType.DOWNTO;
 275                                         break;
 276                                 }
 277                                 break;
 278                         case 'e':
 279                                 switch (begin[1]) {
 280                                 case 'x':
 281                                         switch (begin[2]) {
 282                                         case 'c':
 283                                                 if (matches (begin, "except")) return TokenType.EXCEPT;
 284                                                 break;
 285                                         case 't':
 286                                                 if (matches (begin, "extern")) return TokenType.EXTERN;
 287                                                 break;
 288                                         }
 289                                         break;
 290                                 }
 291                                 break;
 292                         case 'i':
 293                                 if (matches (begin, "inline")) return TokenType.INLINE;
 294                                 break;
 295                         case 'p':
 296                                 if (matches (begin, "public")) return TokenType.PUBLIC;
 297                                 break;
 298                         case 'r':
 299                                 switch (begin[1]) {
 300                                 case 'a':
 301                                         if (matches (begin, "raises")) return TokenType.RAISES;
 302                                         break;
 303                                 case 'e':
 304                                         if (matches (begin, "return")) return TokenType.RETURN;
 305                                         break;
 306                                 }
 307                                 break;
 308                         case 's':
 309                                 switch (begin[1]) {
 310                                 case 'i':
 311                                         if (matches (begin, "sizeof")) return TokenType.SIZEOF;
 312                                         break;
 313                                 case 't':
 314                                         switch (begin[2]) {
 315                                         case 'a':
 316                                                 if (matches (begin, "static")) return TokenType.STATIC;
 317                                                 break;
 318                                         case 'r':
 319                                                 if (matches (begin, "struct")) return TokenType.STRUCT;
 320                                                 break;
 321                                         }
 322                                         break;
 323                                 }
 324                                 break;
 325                         case 't':
 326                                 if (matches (begin, "typeof")) return TokenType.TYPEOF;
 327                                 break;
 328                         }
 329                         break;
 330                 case 7:
 331                         switch (begin[0]) {
 332                         case 'd':
 333                                 switch (begin[1]) {
 334                                 case 'e':
 335                                         if (matches (begin, "default")) return TokenType.DEFAULT;
 336                                         break;
 337                                 case 'y':
 338                                         if (matches (begin, "dynamic")) return TokenType.DYNAMIC;
 339                                         break;
 340                                 }
 341                                 break;
 342                         case 'e':
 343                                 if (matches (begin, "ensures")) return TokenType.ENSURES;
 344                                 break;
 345                         case 'f':
 346                                 switch (begin[1]) {
 347                                 case 'i':
 348                                         if (matches (begin, "finally")) return TokenType.FINALLY;
 349                                         break;
 350                                 case 'o':
 351                                         if (matches (begin, "foreach")) return TokenType.FOREACH;
 352                                         break;
 353                                 }
 354                                 break;
 355                         case 'p':
 356                                 if (matches (begin, "private")) return TokenType.PRIVATE;
 357                                 break;
 358                         case 'v':
 359                                 if (matches (begin, "virtual")) return TokenType.VIRTUAL;
 360                                 break;
 361                         }
 362                         break;
 363                 case 8:
 364                         switch (begin[0]) {
 365                         case 'a':
 366                                 if (matches (begin, "abstract")) return TokenType.ABSTRACT;
 367                                 break;
 368                         case 'c':
 369                                 if (matches (begin, "continue")) return TokenType.CONTINUE;
 370                                 break;
 371                         case 'd':
 372                                 if (matches (begin, "delegate")) return TokenType.DELEGATE;
 373                                 break;
 374                         case 'o':
 375                                 if (matches (begin, "override")) return TokenType.OVERRIDE;
 376                                 break;
 377                         case 'r':
 378                                 switch (begin[2]) {
 379                                 case 'a':
 380                                         if (matches (begin, "readonly")) return TokenType.READONLY;
 381                                         break;
 382                                 case 'q':
 383                                         if (matches (begin, "requires")) return TokenType.REQUIRES;
 384                                         break;
 385                                 }
 386                                 break;
 387                         case 'v':
 388                                 if (matches (begin, "volatile")) return TokenType.VOLATILE;
 389                                 break;
 390                         }
 391                         break;
 392                 case 9:
 393                         switch (begin[0]) {
 394                         case 'c':
 395                                 if (matches (begin, "construct")) return TokenType.CONSTRUCT;
 396                                 break;
 397                         case 'e':
 398                                 if (matches (begin, "exception")) return TokenType.ERRORDOMAIN;
 399                                 break;
 400                         case 'i':
 401                                 if (matches (begin, "interface")) return TokenType.INTERFACE;
 402                                 break;
 403                         case 'n':
 404                                 if (matches (begin, "namespace")) return TokenType.NAMESPACE;
 405                                 break;
 406                         case 'p':
 407                                 if (matches (begin, "protected")) return TokenType.PROTECTED;
 408                                 break;
 409                         case 'w':
 410                                 if (matches (begin, "writeonly")) return TokenType.WRITEONLY;
 411                                 break;
 412                         }
 413                         break;
 414                 case 10:
 415                         switch (begin[0]) {
 416                         case 'i':
 417                                 if (matches (begin, "implements")) return TokenType.IMPLEMENTS;
 418                                 break;
 419                         }
 420                         break;
 421                 }
 422                 return TokenType.IDENTIFIER;
 423         }
 424
 425         public TokenType read_token (out SourceLocation token_begin, out SourceLocation token_end) {
 426                 /* emit dedents if outstanding before checking any other chars */
 427
 428                 if (pending_dedents > 0) {
 429                         pending_dedents--;
 430                         indent_level--;
 431
 432
 433                         token_begin.pos = current;
 434                         token_begin.line = line;
 435                         token_begin.column = column;
 436
 437                         token_end.pos = current;
 438                         token_end.line = line;
 439                         token_end.column = column;
 440
 441                         last_token = TokenType.DEDENT;
 442
 443                         return TokenType.DEDENT;
 444                 }
 445
 446
 447                 if ((_indent_spaces == 0 ) || (last_token != TokenType.EOL)) {
 448                         /* scrub whitespace (excluding newlines) and comments */
 449                         space ();
 450                 }
 451
 452                 /* handle line continuation (lines ending with \) */
 453                 while (current < end && current[0] == '\\' && current[1] == '\n') {
 454                         current += 2;
 455                         line++;
 456                         skip_space_tabs ();
 457                 }
 458
 459                 /* handle non-consecutive new line once parsing is underway - EOL */
 460                 if (newline () && parse_started && last_token != TokenType.EOL && last_token != TokenType.SEMICOLON) {
 461                         token_begin.pos = current;
 462                         token_begin.line = line;
 463                         token_begin.column = column;
 464
 465                         token_end.pos = current;
 466                         token_end.line = line;
 467                         token_end.column = column;
 468
 469                         last_token = TokenType.EOL;
 470
 471                         return TokenType.EOL;
 472                 }
 473
 474
 475                 while (skip_newlines ()) {
 476                         token_begin.pos = current;
 477                         token_begin.line = line;
 478                         token_begin.column = column;
 479
 480                         current_indent_level = count_tabs ();
 481
 482                         /* if its an empty new line then ignore */
 483                         if (current_indent_level == -1)  {
 484                                 continue;
 485                         }
 486
 487                         if (current_indent_level > indent_level) {
 488                                 indent_level = current_indent_level;
 489
 490                                 token_end.pos = current;
 491                                 token_end.line = line;
 492                                 token_end.column = column;
 493
 494                                 last_token = TokenType.INDENT;
 495
 496                                 return TokenType.INDENT;
 497                         } else if (current_indent_level < indent_level) {
 498                                 indent_level--;
 499
 500                                 pending_dedents = (indent_level - current_indent_level);
 501
 502                                 token_end.pos = current;
 503                                 token_end.line = line;
 504                                 token_end.column = column;
 505
 506                                 last_token = TokenType.DEDENT;
 507
 508                                 return TokenType.DEDENT;
 509                         }
 510                 }
 511
 512                 TokenType type;
 513                 char* begin = current;
 514                 token_begin.pos = begin;
 515                 token_begin.line = line;
 516                 token_begin.column = column;
 517
 518                 int token_length_in_chars = -1;
 519
 520                 parse_started = true;
 521
 522                 if (current >= end) {
 523                         if (indent_level > 0) {
 524                                 indent_level--;
 525
 526                                 pending_dedents = indent_level;
 527
 528                                 type = TokenType.DEDENT;
 529                         } else {
 530                                 type = TokenType.EOF;
 531                         }
 532                 } else if (current[0].isalpha () || current[0] == '_') {
 533                         int len = 0;
 534                         while (current < end && is_ident_char (current[0])) {
 535                                 current++;
 536                                 len++;
 537                         }
 538                         type = get_identifier_or_keyword (begin, len);
 539                 } else if (current[0] == '@') {
 540                         int len = 0;
 541                         if (current[1] == '@') {
 542                                 token_begin.pos += 2; // @@ is not part of the identifier
 543                                 current += 2;
 544                         } else {
 545                                 current++;
 546                                 len = 1;
 547                         }
 548                         while (current < end && is_ident_char (current[0])) {
 549                                 current++;
 550                                 len++;
 551                         }
 552                         type = TokenType.IDENTIFIER;
 553                 } else if (current[0].isdigit ()) {
 554                         while (current < end && current[0].isdigit ()) {
 555                                 current++;
 556                         }
 557                         type = TokenType.INTEGER_LITERAL;
 558                         if (current < end && current[0].tolower () == 'l') {
 559                                 current++;
 560                                 if (current < end && current[0].tolower () == 'l') {
 561                                         current++;
 562                                 }
 563                         } else if (current < end && current[0].tolower () == 'u') {
 564                                 current++;
 565                                 if (current < end && current[0].tolower () == 'l') {
 566                                         current++;
 567                                         if (current < end && current[0].tolower () == 'l') {
 568                                                 current++;
 569                                         }
 570                                 }
 571                         } else if (current < end - 1 && current[0] == '.' && current[1].isdigit ()) {
 572                                 current++;
 573                                 while (current < end && current[0].isdigit ()) {
 574                                         current++;
 575                                 }
 576                                 if (current < end && current[0].tolower () == 'e') {
 577                                         current++;
 578                                         if (current < end && (current[0] == '+' || current[0] == '-')) {
 579                                                 current++;
 580                                         }
 581                                         while (current < end && current[0].isdigit ()) {
 582                                                 current++;
 583                                         }
 584                                 }
 585                                 if (current < end && current[0].tolower () == 'f') {
 586                                         current++;
 587                                 }
 588                                 type = TokenType.REAL_LITERAL;
 589                         } else if (current < end && current == begin + 1
 590                                    && begin[0] == '0' && begin[1] == 'x' && begin[2].isxdigit ()) {
 591                                 // hexadecimal integer literal
 592                                 current++;
 593                                 while (current < end && current[0].isxdigit ()) {
 594                                         current++;
 595                                 }
 596                         } else if (current < end && is_ident_char (current[0])) {
 597                                 // allow identifiers to start with a digit
 598                                 // as long as they contain at least one char
 599                                 while (current < end && is_ident_char (current[0])) {
 600                                         current++;
 601                                 }
 602                                 type = TokenType.IDENTIFIER;
 603                         }
 604                 } else {
 605                         switch (current[0]) {
 606                         case '{':
 607                                 type = TokenType.OPEN_BRACE;
 608                                 current++;
 609                                 break;
 610                         case '}':
 611                                 type = TokenType.CLOSE_BRACE;
 612                                 current++;
 613                                 break;
 614                         case '(':
 615                                 type = TokenType.OPEN_PARENS;
 616                                 current++;
 617                                 break;
 618                         case ')':
 619                                 type = TokenType.CLOSE_PARENS;
 620                                 current++;
 621                                 break;
 622                         case '[':
 623                                 type = TokenType.OPEN_BRACKET;
 624                                 current++;
 625                                 break;
 626                         case ']':
 627                                 type = TokenType.CLOSE_BRACKET;
 628                                 current++;
 629                                 break;
 630                         case '.':
 631                                 type = TokenType.DOT;
 632                                 current++;
 633                                 if (current < end - 1) {
 634                                         if (current[0] == '.' && current[1] == '.') {
 635                                                 type = TokenType.ELLIPSIS;
 636                                                 current += 2;
 637                                         }
 638                                 }
 639                                 break;
 640                         case ':':
 641                                 type = TokenType.COLON;
 642                                 current++;
 643                                 break;
 644                         case ',':
 645                                 type = TokenType.COMMA;
 646                                 current++;
 647                                 break;
 648                         case ';':
 649                                 type = TokenType.SEMICOLON;
 650                                 current++;
 651                                 break;
 652                         case '#':
 653                                 type = TokenType.HASH;
 654                                 current++;
 655                                 break;
 656                         case '?':
 657                                 type = TokenType.INTERR;
 658                                 current++;
 659                                 break;
 660                         case '|':
 661                                 type = TokenType.BITWISE_OR;
 662                                 current++;
 663                                 if (current < end) {
 664                                         switch (current[0]) {
 665                                         case '=':
 666                                                 type = TokenType.ASSIGN_BITWISE_OR;
 667                                                 current++;
 668                                                 break;
 669                                         case '|':
 670                                                 type = TokenType.OP_OR;
 671                                                 current++;
 672                                                 break;
 673                                         }
 674                                 }
 675                                 break;
 676                         case '&':
 677                                 type = TokenType.BITWISE_AND;
 678                                 current++;
 679                                 if (current < end) {
 680                                         switch (current[0]) {
 681                                         case '=':
 682                                                 type = TokenType.ASSIGN_BITWISE_AND;
 683                                                 current++;
 684                                                 break;
 685                                         case '&':
 686                                                 type = TokenType.OP_AND;
 687                                                 current++;
 688                                                 break;
 689                                         }
 690                                 }
 691                                 break;
 692                         case '^':
 693                                 type = TokenType.CARRET;
 694                                 current++;
 695                                 if (current < end && current[0] == '=') {
 696                                         type = TokenType.ASSIGN_BITWISE_XOR;
 697                                         current++;
 698                                 }
 699                                 break;
 700                         case '~':
 701                                 type = TokenType.TILDE;
 702                                 current++;
 703                                 break;
 704                         case '=':
 705                                 type = TokenType.ASSIGN;
 706                                 current++;
 707                                 if (current < end) {
 708                                         switch (current[0]) {
 709                                         case '=':
 710                                                 type = TokenType.OP_EQ;
 711                                                 current++;
 712                                                 break;
 713                                         case '>':
 714                                                 type = TokenType.LAMBDA;
 715                                                 current++;
 716                                                 break;
 717                                         }
 718                                 }
 719                                 break;
 720                         case '<':
 721                                 type = TokenType.OP_LT;
 722                                 current++;
 723                                 if (current < end) {
 724                                         switch (current[0]) {
 725                                         case '=':
 726                                                 type = TokenType.OP_LE;
 727                                                 current++;
 728                                                 break;
 729                                         case '<':
 730                                                 type = TokenType.OP_SHIFT_LEFT;
 731                                                 current++;
 732                                                 if (current < end && current[0] == '=') {
 733                                                         type = TokenType.ASSIGN_SHIFT_LEFT;
 734                                                         current++;
 735                                                 }
 736                                                 break;
 737                                         }
 738                                 }
 739                                 break;
 740                         case '>':
 741                                 type = TokenType.OP_GT;
 742                                 current++;
 743                                 if (current < end && current[0] == '=') {
 744                                         type = TokenType.OP_GE;
 745                                         current++;
 746                                 }
 747                                 break;
 748                         case '!':
 749                                 type = TokenType.OP_NEG;
 750                                 current++;
 751                                 if (current < end && current[0] == '=') {
 752                                         type = TokenType.OP_NE;
 753                                         current++;
 754                                 }
 755                                 break;
 756                         case '+':
 757                                 type = TokenType.PLUS;
 758                                 current++;
 759                                 if (current < end) {
 760                                         switch (current[0]) {
 761                                         case '=':
 762                                                 type = TokenType.ASSIGN_ADD;
 763                                                 current++;
 764                                                 break;
 765                                         case '+':
 766                                                 type = TokenType.OP_INC;
 767                                                 current++;
 768                                                 break;
 769                                         }
 770                                 }
 771                                 break;
 772                         case '-':
 773                                 type = TokenType.MINUS;
 774                                 current++;
 775                                 if (current < end) {
 776                                         switch (current[0]) {
 777                                         case '=':
 778                                                 type = TokenType.ASSIGN_SUB;
 779                                                 current++;
 780                                                 break;
 781                                         case '-':
 782                                                 type = TokenType.OP_DEC;
 783                                                 current++;
 784                                                 break;
 785                                         case '>':
 786                                                 type = TokenType.OP_PTR;
 787                                                 current++;
 788                                                 break;
 789                                         }
 790                                 }
 791                                 break;
 792                         case '*':
 793                                 type = TokenType.STAR;
 794                                 current++;
 795                                 if (current < end && current[0] == '=') {
 796                                         type = TokenType.ASSIGN_MUL;
 797                                         current++;
 798                                 }
 799                                 break;
 800                         case '/':
 801                                 type = TokenType.DIV;
 802                                 current++;
 803                                 if (current < end && current[0] == '=') {
 804                                         type = TokenType.ASSIGN_DIV;
 805                                         current++;
 806                                 }
 807                                 break;
 808                         case '%':
 809                                 type = TokenType.PERCENT;
 810                                 current++;
 811                                 if (current < end && current[0] == '=') {
 812                                         type = TokenType.ASSIGN_PERCENT;
 813                                         current++;
 814                                 }
 815                                 break;
 816                         case '\'':
 817                         case '"':
 818                                 if (begin[0] == '\'') {
 819                                         type = TokenType.CHARACTER_LITERAL;
 820                                 } else {
 821                                         type = TokenType.STRING_LITERAL;
 822                                 }
 823                                 token_length_in_chars = 2;
 824                                 current++;
 825                                 while (current < end && current[0] != begin[0]) {
 826                                         if (current[0] == '\\') {
 827                                                 current++;
 828                                                 token_length_in_chars++;
 829                                                 if (current < end && current[0] == 'x') {
 830                                                         // hexadecimal escape character
 831                                                         current++;
 832                                                         token_length_in_chars++;
 833                                                         while (current < end && current[0].isxdigit ()) {
 834                                                                 current++;
 835                                                                 token_length_in_chars++;
 836                                                         }
 837                                                 } else {
 838                                                         current++;
 839                                                         token_length_in_chars++;
 840                                                 }
 841                                         } else if (current[0] == '\n') {
 842                                                 break;
 843                                         } else {
 844                                                 unichar u = ((string) current).get_char_validated ((long) (end - current));
 845                                                 if (u != (unichar) (-1)) {
 846                                                         current += u.to_utf8 (null);
 847                                                         token_length_in_chars++;
 848                                                 } else {
 849                                                         Report.error (new SourceReference (source_file, line, column + token_length_in_chars, line, column + token_length_in_chars), "invalid UTF-8 character");
 850                                                 }
 851                                         }
 852                                 }
 853                                 if (current < end && current[0] != '\n') {
 854                                         current++;
 855                                 } else {
 856                                         Report.error (new SourceReference (source_file, line, column + token_length_in_chars, line, column + token_length_in_chars), "syntax error, expected %c".printf (begin[0]));
 857                                 }
 858                                 break;
 859                         default:
 860                                 unichar u = ((string) current).get_char_validated ((long) (end - current));
 861                                 if (u != (unichar) (-1)) {
 862                                         current += u.to_utf8 (null);
 863                                         Report.error (new SourceReference (source_file, line, column, line, column), "syntax error, unexpected character");
 864                                 } else {
 865                                         current++;
 866                                         Report.error (new SourceReference (source_file, line, column, line, column), "invalid UTF-8 character");
 867                                 }
 868                                 column++;
 869                                 last_token = TokenType.STRING_LITERAL;
 870                                 return read_token (out token_begin, out token_end);
 871                         }
 872                 }
 873
 874                 if (token_length_in_chars < 0) {
 875                         column += (int) (current - begin);
 876                 } else {
 877                         column += token_length_in_chars;
 878                 }
 879
 880                 token_end.pos = current;
 881                 token_end.line = line;
 882                 token_end.column = column - 1;
 883
 884                 last_token = type;
 885
 886                 return type;
 887         }
 888
 889         int count_tabs ()
 890         {
 891
 892                 int tab_count = 0;
 893
 894
 895                 if (_indent_spaces == 0) {
 896                         while (current < end && current[0] == '\t') {
 897                                 current++;
 898                                 column++;
 899                                 tab_count++;
 900                         }
 901                 } else {
 902                         int space_count = 0;
 903                         while (current < end && current[0] == ' ') {
 904                                 current++;
 905                                 column++;
 906                                 space_count++;
 907                         }
 908
 909                         tab_count = space_count / _indent_spaces;
 910
 911                 }
 912
 913                 /* ignore comments and whitspace and other lines that contain no code */
 914
 915                 space ();
 916
 917                 if ((current < end) && (current[0] == '\n')) return -1;
 918
 919                 return tab_count;
 920         }
 921
 922         bool matches (char* begin, string keyword) {
 923                 char* keyword_array = keyword;
 924                 long len = keyword.len ();
 925                 for (int i = 0; i < len; i++) {
 926                         if (begin[i] != keyword_array[i]) {
 927                                 return false;
 928                         }
 929                 }
 930                 return true;
 931         }
 932
 933         bool whitespace () {
 934                 bool found = false;
 935                 while (current < end && current[0].isspace () && current[0] != '\n' ) {
 936
 937                         found = true;
 938                         current++;
 939                         column++;
 940                 }
 941                 return found;
 942         }
 943
 944         inline bool newline () {
 945                 if (current[0] == '\n') {
 946                         return true;
 947                 }
 948
 949                 return false;
 950         }
 951
 952         bool skip_newlines () {
 953                 bool new_lines = false;
 954
 955                 while (newline ()) {
 956                         current++;
 957
 958                         line++;
 959                         column = 1;
 960                         current_indent_level = 0;
 961
 962                         new_lines = true;
 963                 }
 964
 965                 return new_lines;
 966         }
 967
 968         bool comment () {
 969                 if (current > end - 2
 970                     || current[0] != '/'
 971                     || (current[1] != '/' && current[1] != '*')) {
 972                         return false;
 973                 }
 974
 975                 if (current[1] == '/') {
 976                         // single-line comment
 977                         current += 2;
 978                         char* begin = current;
 979                         // skip until end of line or end of file
 980                         while (current < end && current[0] != '\n') {
 981                                 current++;
 982                         }
 983                         push_comment (((string) begin).ndup ((long) (current - begin)), line == 1);
 984
 985                         if (current[0] == '\n') {
 986                                 current++;
 987                                 line++;
 988                                 column = 1;
 989                                 current_indent_level = 0;
 990                         }
 991                 } else {
 992                         // delimited comment
 993                         current += 2;
 994                         char* begin = current;
 995                         int begin_line = line;
 996                         while (current < end - 1
 997                                && (current[0] != '*' || current[1] != '/')) {
 998                                 if (current[0] == '\n') {
 999                                         line++;
1000                                         column = 0;
1001                                 }
1002                                 current++;
1003                                 column++;
1004                         }
1005                         if (current == end - 1) {
1006                                 Report.error (new SourceReference (source_file, line, column, line, column), "syntax error, expected */");
1007                                 return true;
1008                         }
1009                         push_comment (((string) begin).ndup ((long) (current - begin)), begin_line == 1);
1010                         current += 2;
1011                         column += 2;
1012                 }
1013
1014                 return true;
1015         }
1016
1017         bool skip_tabs () {
1018                 bool found = false;
1019                 while (current < end && current[0] == '\t' ) {
1020                         current++;
1021                         column++;
1022                         found = true;
1023                 }
1024
1025                 return found;
1026         }
1027
1028         void skip_space_tabs () {
1029                 while (whitespace () || skip_tabs () || comment () ) {
1030                 }
1031
1032         }
1033
1034         void space () {
1035                 while (whitespace () || comment ()) {
1036                 }
1037         }
1038
1039
1040
1041
1042         void push_comment (string comment_item, bool file_comment) {
1043                 if (_comment == null) {
1044                         _comment = comment_item;
1045                 } else {
1046                         _comment = "%s\n%s".printf (_comment, comment_item);
1047                 }
1048                 if (file_comment) {
1049                         source_file.comment = _comment;
1050                         _comment = null;
1051                 }
1052         }
1053
1054         /**
1055          * Clears and returns the content of the comment stack.
1056          *
1057          * @return saved comment
1058          */
1059         public string? pop_comment () {
1060                 if (_comment == null) {
1061                         return null;
1062                 }
1063
1064                 var result = new StringBuilder (_comment);
1065                 _comment = null;
1066
1067                 weak string index;
1068                 while ((index = result.str.chr (-1, '\t')) != null) {
1069                         result.erase (result.str.pointer_to_offset (index), 1);
1070                 }
1071
1072                 return result.str;
1073         }
1074 }
1075