src/lexer.cpp

   1
   2 #include "private.h"
   3
   4 namespace TinyJS
   5 {
   6     Lexer::Lexer(const std::string& input)
   7     {
   8         m_data = strncopy(input.c_str(), input.size());
   9         m_dataOwned = true;
  10         m_dataStart = 0;
  11         m_dataEnd = int(input.size());
  12         reset();
  13     }
  14
  15     Lexer::Lexer(Lexer* owner, int startChar, int endChar)
  16     {
  17         m_data = owner->m_data;
  18         m_dataOwned = false;
  19         m_dataStart = startChar;
  20         m_dataEnd = endChar;
  21         reset();
  22     }
  23
  24     Lexer::~Lexer(void)
  25     {
  26         if(m_dataOwned)
  27         {
  28             free((void*)m_data);
  29         }
  30     }
  31
  32     void Lexer::reset()
  33     {
  34         m_dataPos = m_dataStart;
  35         m_tokenStart = 0;
  36         m_tokenEnd = 0;
  37         m_tokenLastEnd = 0;
  38         m_tk = 0;
  39         m_tkStr = "";
  40         getNextCh();
  41         getNextCh();
  42         getNextToken();
  43     }
  44
  45     void Lexer::match(int expected_tk)
  46     {
  47         if(m_tk != expected_tk)
  48         {
  49             std::stringstream errorString;
  50             errorString << "Got " << getTokenStr(m_tk) << " expected " << getTokenStr(expected_tk)
  51                         << " at " << getPosition(m_tokenStart);
  52             throw new RuntimeError(errorString.str());
  53         }
  54         getNextToken();
  55     }
  56
  57     std::string Lexer::getTokenStr(int token)
  58     {
  59         if((token > 32) && (token < 128))
  60         {
  61             char buf[4] = "' '";
  62             buf[1] = (char)token;
  63             return buf;
  64         }
  65         switch(token)
  66         {
  67             case LEX_EOF :
  68                 return "EOF";
  69             case LEX_ID :
  70                 return "ID";
  71             case LEX_INT :
  72                 return "INT";
  73             case LEX_FLOAT :
  74                 return "FLOAT";
  75             case LEX_STR :
  76                 return "STRING";
  77             case LEX_EQUAL :
  78                 return "==";
  79             case LEX_TYPEEQUAL :
  80                 return "===";
  81             case LEX_NEQUAL :
  82                 return "!=";
  83             case LEX_NTYPEEQUAL :
  84                 return "!==";
  85             case LEX_LEQUAL :
  86                 return "<=";
  87             case LEX_LSHIFT :
  88                 return "<<";
  89             case LEX_LSHIFTEQUAL :
  90                 return "<<=";
  91             case LEX_GEQUAL :
  92                 return ">=";
  93             case LEX_RSHIFT :
  94                 return ">>";
  95             case LEX_RSHIFTUNSIGNED :
  96                 return ">>";
  97             case LEX_RSHIFTEQUAL :
  98                 return ">>=";
  99             case LEX_PLUSEQUAL :
 100                 return "+=";
 101             case LEX_MINUSEQUAL :
 102                 return "-=";
 103             case LEX_PLUSPLUS :
 104                 return "++";
 105             case LEX_MINUSMINUS :
 106                 return "--";
 107             case LEX_ANDEQUAL :
 108                 return "&=";
 109             case LEX_ANDAND :
 110                 return "&&";
 111             case LEX_OREQUAL :
 112                 return "|=";
 113             case LEX_OROR :
 114                 return "||";
 115             case LEX_XOREQUAL :
 116                 return "^=";
 117                 // reserved words
 118             case LEX_R_IF :
 119                 return "if";
 120             case LEX_R_ELSE :
 121                 return "else";
 122             case LEX_R_DO :
 123                 return "do";
 124             case LEX_R_WHILE :
 125                 return "while";
 126             case LEX_R_FOR :
 127                 return "for";
 128             case LEX_R_BREAK :
 129                 return "break";
 130             case LEX_R_CONTINUE :
 131                 return "continue";
 132             case LEX_R_FUNCTION :
 133                 return "function";
 134             case LEX_R_RETURN :
 135                 return "return";
 136             case LEX_R_VAR :
 137                 return "var";
 138             case LEX_R_TRUE :
 139                 return "true";
 140             case LEX_R_FALSE :
 141                 return "false";
 142             case LEX_R_NULL :
 143                 return "null";
 144             case LEX_R_UNDEFINED :
 145                 return "undefined";
 146             case LEX_R_NEW :
 147                 return "new";
 148         }
 149         std::stringstream msg;
 150         msg << "?[" << token << "]";
 151         return msg.str();
 152     }
 153
 154     void Lexer::getNextCh()
 155     {
 156         m_currCh = m_nextCh;
 157         if(m_dataPos < m_dataEnd)
 158         {
 159             m_nextCh = m_data[m_dataPos];
 160         }
 161         else
 162         {
 163             m_nextCh = 0;
 164         }
 165         m_dataPos++;
 166     }
 167
 168     void Lexer::getNextToken()
 169     {
 170         m_tk = LEX_EOF;
 171         m_tkStr.clear();
 172         while(m_currCh && isWhitespace(m_currCh))
 173         {
 174             getNextCh();
 175         }
 176         // newline comments
 177         if((m_currCh == '/') && (m_nextCh == '/'))
 178         {
 179             while(m_currCh && (m_currCh != '\n'))
 180             {
 181                 getNextCh();
 182             }
 183             getNextCh();
 184             getNextToken();
 185             return;
 186         }
 187         // block comments
 188         if((m_currCh == '/') && (m_nextCh == '*'))
 189         {
 190             while(m_currCh && ((m_currCh != '*') || (m_nextCh != '/')))
 191             {
 192                 getNextCh();
 193             }
 194             getNextCh();
 195             getNextCh();
 196             getNextToken();
 197             return;
 198         }
 199         // record beginning of this token
 200         m_tokenStart = (m_dataPos - 2);
 201         // tokens
 202         if(isAlpha(m_currCh))    //  IDs
 203         {
 204             while(isAlpha(m_currCh) || isNumeric(m_currCh))
 205             {
 206                 m_tkStr += m_currCh;
 207                 getNextCh();
 208             }
 209             m_tk = LEX_ID;
 210             if(m_tkStr == "if")
 211             {
 212                 m_tk = LEX_R_IF;
 213             }
 214             else if(m_tkStr == "else")
 215             {
 216                 m_tk = LEX_R_ELSE;
 217             }
 218             else if(m_tkStr == "do")
 219             {
 220                 m_tk = LEX_R_DO;
 221             }
 222             else if(m_tkStr == "while")
 223             {
 224                 m_tk = LEX_R_WHILE;
 225             }
 226             else if(m_tkStr == "for")
 227             {
 228                 m_tk = LEX_R_FOR;
 229             }
 230             else if(m_tkStr == "break")
 231             {
 232                 m_tk = LEX_R_BREAK;
 233             }
 234             else if(m_tkStr == "continue")
 235             {
 236                 m_tk = LEX_R_CONTINUE;
 237             }
 238             else if(m_tkStr == "function")
 239             {
 240                 m_tk = LEX_R_FUNCTION;
 241             }
 242             else if(m_tkStr == "return")
 243             {
 244                 m_tk = LEX_R_RETURN;
 245             }
 246             else if(m_tkStr == "var")
 247             {
 248                 m_tk = LEX_R_VAR;
 249             }
 250             else if(m_tkStr == "true")
 251             {
 252                 m_tk = LEX_R_TRUE;
 253             }
 254             else if(m_tkStr == "false")
 255             {
 256                 m_tk = LEX_R_FALSE;
 257             }
 258             else if(m_tkStr == "null")
 259             {
 260                 m_tk = LEX_R_NULL;
 261             }
 262             else if(m_tkStr == "undefined")
 263             {
 264                 m_tk = LEX_R_UNDEFINED;
 265             }
 266             else if(m_tkStr == "new")
 267             {
 268                 m_tk = LEX_R_NEW;
 269             }
 270         }
 271         else if(isNumeric(m_currCh))      // Numbers
 272         {
 273             bool isHex = false;
 274             if(m_currCh == '0')
 275             {
 276                 m_tkStr += m_currCh;
 277                 getNextCh();
 278             }
 279             if(m_currCh == 'x')
 280             {
 281                 isHex = true;
 282                 m_tkStr += m_currCh;
 283                 getNextCh();
 284             }
 285             m_tk = LEX_INT;
 286             while(isNumeric(m_currCh) || (isHex && isHexadecimal(m_currCh)))
 287             {
 288                 m_tkStr += m_currCh;
 289                 getNextCh();
 290             }
 291             if(!isHex && (m_currCh == '.'))
 292             {
 293                 m_tk = LEX_FLOAT;
 294                 m_tkStr += '.';
 295                 getNextCh();
 296                 while(isNumeric(m_currCh))
 297                 {
 298                     m_tkStr += m_currCh;
 299                     getNextCh();
 300                 }
 301             }
 302             // do fancy e-style floating point
 303             if(!isHex && ((m_currCh == 'e') || (m_currCh == 'E')))
 304             {
 305                 m_tk = LEX_FLOAT;
 306                 m_tkStr += m_currCh;
 307                 getNextCh();
 308                 if(m_currCh == '-')
 309                 {
 310                     m_tkStr += m_currCh;
 311                     getNextCh();
 312                 }
 313                 while(isNumeric(m_currCh))
 314                 {
 315                     m_tkStr += m_currCh;
 316                     getNextCh();
 317                 }
 318             }
 319         }
 320         else if(m_currCh == '"')
 321         {
 322             // strings...
 323             getNextCh();
 324             while(m_currCh && (m_currCh != '"'))
 325             {
 326                 if(m_currCh == '\\')
 327                 {
 328                     getNextCh();
 329                     switch(m_currCh)
 330                     {
 331                         case 'n' :
 332                             m_tkStr += '\n';
 333                             break;
 334                         case '"' :
 335                             m_tkStr += '"';
 336                             break;
 337                         case '\\' :
 338                             m_tkStr += '\\';
 339                             break;
 340                         default:
 341                             m_tkStr += m_currCh;
 342                     }
 343                 }
 344                 else
 345                 {
 346                     m_tkStr += m_currCh;
 347                 }
 348                 getNextCh();
 349             }
 350             getNextCh();
 351             m_tk = LEX_STR;
 352         }
 353         else if(m_currCh == '\'')
 354         {
 355             // strings again...
 356             getNextCh();
 357             while(m_currCh && (m_currCh != '\''))
 358             {
 359                 if(m_currCh == '\\')
 360                 {
 361                     getNextCh();
 362                     switch(m_currCh)
 363                     {
 364                         case 'n' :
 365                             m_tkStr += '\n';
 366                             break;
 367                         case 'a' :
 368                             m_tkStr += '\a';
 369                             break;
 370                         case 'r' :
 371                             m_tkStr += '\r';
 372                             break;
 373                         case 't' :
 374                             m_tkStr += '\t';
 375                             break;
 376                         case '\'' :
 377                             m_tkStr += '\'';
 378                             break;
 379                         case '\\' :
 380                             m_tkStr += '\\';
 381                             break;
 382                         case 'x' :   // hex digits
 383                             {
 384                                 char buf[3] = "??";
 385                                 getNextCh();
 386                                 buf[0] = m_currCh;
 387                                 getNextCh();
 388                                 buf[1] = m_currCh;
 389                                 m_tkStr += (char)strtol(buf,0,16);
 390                             }
 391                             break;
 392                         default:
 393                             if((m_currCh >= '0') && (m_currCh <= '7'))
 394                             {
 395                                 // octal digits
 396                                 char buf[4] = "???";
 397                                 buf[0] = m_currCh;
 398                                 getNextCh();
 399                                 buf[1] = m_currCh;
 400                                 getNextCh();
 401                                 buf[2] = m_currCh;
 402                                 m_tkStr += (char)strtol(buf,0,8);
 403                             }
 404                             else
 405                             {
 406                                 m_tkStr += m_currCh;
 407                             }
 408                     }
 409                 }
 410                 else
 411                 {
 412                     m_tkStr += m_currCh;
 413                 }
 414                 getNextCh();
 415             }
 416             getNextCh();
 417             m_tk = LEX_STR;
 418         }
 419         else
 420         {
 421             // single chars
 422             m_tk = m_currCh;
 423             if(m_currCh)
 424             {
 425                 getNextCh();
 426             }
 427             if((m_tk == '=') && (m_currCh == '='))    // ==
 428             {
 429                 m_tk = LEX_EQUAL;
 430                 getNextCh();
 431                 if(m_currCh == '=')    // ===
 432                 {
 433                     m_tk = LEX_TYPEEQUAL;
 434                     getNextCh();
 435                 }
 436             }
 437             else if((m_tk == '!') && (m_currCh == '='))      // !=
 438             {
 439                 m_tk = LEX_NEQUAL;
 440                 getNextCh();
 441                 if(m_currCh == '=')    // !==
 442                 {
 443                     m_tk = LEX_NTYPEEQUAL;
 444                     getNextCh();
 445                 }
 446             }
 447             else if((m_tk == '<') && (m_currCh == '='))
 448             {
 449                 m_tk = LEX_LEQUAL;
 450                 getNextCh();
 451             }
 452             else if((m_tk == '<') && (m_currCh == '<'))
 453             {
 454                 m_tk = LEX_LSHIFT;
 455                 getNextCh();
 456                 if(m_currCh == '=')    // <<=
 457                 {
 458                     m_tk = LEX_LSHIFTEQUAL;
 459                     getNextCh();
 460                 }
 461             }
 462             else if((m_tk == '>') && (m_currCh == '='))
 463             {
 464                 m_tk = LEX_GEQUAL;
 465                 getNextCh();
 466             }
 467             else if((m_tk == '>') && (m_currCh == '>'))
 468             {
 469                 m_tk = LEX_RSHIFT;
 470                 getNextCh();
 471                 if(m_currCh == '=')    // >>=
 472                 {
 473                     m_tk = LEX_RSHIFTEQUAL;
 474                     getNextCh();
 475                 }
 476                 else if(m_currCh == '>')      // >>>
 477                 {
 478                     m_tk = LEX_RSHIFTUNSIGNED;
 479                     getNextCh();
 480                 }
 481             }
 482             else if((m_tk == '+') && (m_currCh == '='))
 483             {
 484                 m_tk = LEX_PLUSEQUAL;
 485                 getNextCh();
 486             }
 487             else if((m_tk == '-') && (m_currCh == '='))
 488             {
 489                 m_tk = LEX_MINUSEQUAL;
 490                 getNextCh();
 491             }
 492             else if((m_tk == '+') && (m_currCh == '+'))
 493             {
 494                 m_tk = LEX_PLUSPLUS;
 495                 getNextCh();
 496             }
 497             else if((m_tk == '-') && (m_currCh == '-'))
 498             {
 499                 m_tk = LEX_MINUSMINUS;
 500                 getNextCh();
 501             }
 502             else if((m_tk == '&') && (m_currCh == '='))
 503             {
 504                 m_tk = LEX_ANDEQUAL;
 505                 getNextCh();
 506             }
 507             else if((m_tk == '&') && (m_currCh == '&'))
 508             {
 509                 m_tk = LEX_ANDAND;
 510                 getNextCh();
 511             }
 512             else if((m_tk == '|') && (m_currCh == '='))
 513             {
 514                 m_tk = LEX_OREQUAL;
 515                 getNextCh();
 516             }
 517             else if((m_tk == '|') && (m_currCh == '|'))
 518             {
 519                 m_tk = LEX_OROR;
 520                 getNextCh();
 521             }
 522             else if((m_tk == '^') && (m_currCh == '='))
 523             {
 524                 m_tk = LEX_XOREQUAL;
 525                 getNextCh();
 526             }
 527         }
 528         /* This isn't quite right yet */
 529         m_tokenLastEnd = m_tokenEnd;
 530         m_tokenEnd = (m_dataPos - 3);
 531     }
 532
 533     std::string Lexer::getSubString(int lastPosition)
 534     {
 535         int lastCharIdx = (m_tokenLastEnd + 1);
 536         if(lastCharIdx < m_dataEnd)
 537         {
 538             /* save a memory alloc by using our data array to create the string */
 539             char old = m_data[lastCharIdx];
 540             m_data[lastCharIdx] = 0;
 541             std::string value = &m_data[lastPosition];
 542             m_data[lastCharIdx] = old;
 543             return value;
 544         }
 545         else
 546         {
 547             return std::string(&m_data[lastPosition]);
 548         }
 549     }
 550
 551     Lexer* Lexer::getSubLex(int lastPosition)
 552     {
 553         int lastCharIdx = (m_tokenLastEnd + 1);
 554         if(lastCharIdx < m_dataEnd)
 555         {
 556             return new Lexer(this, lastPosition, lastCharIdx);
 557         }
 558         else
 559         {
 560             return new Lexer(this, lastPosition, m_dataEnd);
 561         }
 562     }
 563
 564     std::string Lexer::getPosition(int pos)
 565     {
 566         int line;
 567         int col;
 568         if(pos < 0)
 569         {
 570             pos = m_tokenLastEnd;
 571         }
 572         line = 1;
 573         col = 1;
 574         for(int i=0; i<pos; i++)
 575         {
 576             char ch;
 577             if(i < m_dataEnd)
 578             {
 579                 ch = m_data[i];
 580             }
 581             else
 582             {
 583                 ch = 0;
 584             }
 585             col++;
 586             if(ch == '\n')
 587             {
 588                 line++;
 589                 col = 0;
 590             }
 591         }
 592         char buf[256];
 593         sprintf_s(buf, 256, "(line: %d, col: %d)", line, col);
 594         return buf;
 595     }
 596 }