src/llex.c

   1 /*
   2 ** $Id: llex.c,v 2.20.1.1 2007/12/27 13:02:25 roberto Exp $
   3 ** Lexical Analyzer
   4 ** See Copyright Notice in lua.h
   5 */
   6
   7
   8 #include <ctype.h>
   9 #include <locale.h>
  10 #include <string.h>
  11 #include <stdio.h>
  12
  13 #define llex_c
  14 #define LUA_CORE
  15
  16 #include "lua.h"
  17 #include "lauxlib.h"
  18
  19 #include "ldo.h"
  20 #include "llex.h"
  21 #include "lobject.h"
  22 #include "lparser.h"
  23 #include "lstate.h"
  24 #include "lstring.h"
  25 #include "ltable.h"
  26 #include "lzio.h"
  27
  28 #define _HEX2INT_(v) ( (v) <= '9' ? (v)-'0' : tolower(v)-'a'+10 )
  29
  30
  31 #define next(ls) (ls->current = zgetc(ls->z))
  32
  33
  34
  35
  36 #define currIsNewline(ls)       (ls->current == '\n' || ls->current == '\r')
  37
  38
  39 /* ORDER RESERVED */
  40 const char *const luaX_tokens [] = {
  41     "and", "break", "continue", "do", "else", "elseif",
  42     "end", "false", "for", "function", "if",
  43     "in", "local", "nil", "not", "or", "repeat",
  44     "return", "then", "true", "until", "while",
  45     "..", "...", "==", ">=", "<=", "~=",
  46     "<number>", "<name>", "<string>", "<eof>",
  47     "<<", ">>", "^^",
  48     NULL
  49 };
  50
  51
  52 #define save_and_next(ls) (save(ls, ls->current), next(ls))
  53
  54
  55 static void save (LexState *ls, int c) {
  56   Mbuffer *b = ls->buff;
  57   if (b->n + 1 > b->buffsize) {
  58     size_t newsize;
  59     if (b->buffsize >= MAX_SIZET/2)
  60       luaX_lexerror(ls, "lexical element too long", 0);
  61     newsize = b->buffsize * 2;
  62     luaZ_resizebuffer(ls->L, b, newsize);
  63   }
  64   b->buffer[b->n++] = cast(char, c);
  65 }
  66
  67
  68 void luaX_init (lua_State *L) {
  69   int i;
  70   for (i=0; i<NUM_RESERVED; i++) {
  71     TString *ts = luaS_new(L, luaX_tokens[i]);
  72     luaS_fix(ts);  /* reserved words are never collected */
  73     lua_assert(strlen(luaX_tokens[i])+1 <= TOKEN_LEN);
  74     ts->tsv.reserved = cast_byte(i+1);  /* reserved word */
  75   }
  76 }
  77
  78
  79 #define MAXSRC          80
  80
  81
  82 const char *luaX_token2str (LexState *ls, int token) {
  83   if (token < FIRST_RESERVED) {
  84     lua_assert(token == cast(unsigned char, token));
  85     return (iscntrl(token)) ? luaO_pushfstring(ls->L, "char(%d)", token) :
  86                               luaO_pushfstring(ls->L, "%c", token);
  87   }
  88   else
  89     return luaX_tokens[token-FIRST_RESERVED];
  90 }
  91
  92
  93 static const char *txtToken (LexState *ls, int token) {
  94   switch (token) {
  95     case TK_NAME:
  96     case TK_STRING:
  97     case TK_NUMBER:
  98       save(ls, '\0');
  99       return luaZ_buffer(ls->buff);
 100     default:
 101       return luaX_token2str(ls, token);
 102   }
 103 }
 104
 105
 106 void luaX_lexerror (LexState *ls, const char *msg, int token) {
 107   char buff[MAXSRC];
 108   luaO_chunkid(buff, getstr(ls->source), MAXSRC);
 109   msg = luaO_pushfstring(ls->L, "%s:%d: %s", buff, ls->linenumber, msg);
 110   if (token)
 111     luaO_pushfstring(ls->L, "%s near " LUA_QS, msg, txtToken(ls, token));
 112   luaD_throw(ls->L, LUA_ERRSYNTAX);
 113 }
 114
 115
 116 void luaX_syntaxerror (LexState *ls, const char *msg) {
 117   luaX_lexerror(ls, msg, ls->t.token);
 118 }
 119
 120
 121 TString *luaX_newstring (LexState *ls, const char *str, size_t l) {
 122   lua_State *L = ls->L;
 123   TString *ts = luaS_newlstr(L, str, l);
 124   TValue *o = luaH_setstr(L, ls->fs->h, ts);  /* entry for `str' */
 125   if (ttisnil(o))
 126     setbvalue(o, 1);  /* make sure `str' will not be collected */
 127   return ts;
 128 }
 129
 130
 131 static void inclinenumber (LexState *ls) {
 132   int old = ls->current;
 133   lua_assert(currIsNewline(ls));
 134   next(ls);  /* skip `\n' or `\r' */
 135   if (currIsNewline(ls) && ls->current != old)
 136     next(ls);  /* skip `\n\r' or `\r\n' */
 137   if (++ls->linenumber >= MAX_INT)
 138     luaX_syntaxerror(ls, "chunk has too many lines");
 139 }
 140
 141
 142 void luaX_setinput(lua_State *L, LexState *ls, ZIO *z, TString *source)
 143 {
 144   ls->decpoint = '.';
 145   ls->L = L;
 146   ls->lookahead.token = TK_EOS;  /* no look-ahead token */
 147   ls->z = z;
 148   ls->fs = NULL;
 149   ls->linenumber = 1;
 150   ls->lastline = 1;
 151   ls->source = source;
 152   ls->refstr = 0;
 153   luaZ_resizebuffer(ls->L, ls->buff, LUA_MINBUFFER);  /* initialize buffer */
 154   next(ls);  /* read first char */
 155 }
 156
 157
 158
 159 /*
 160 ** =======================================================
 161 ** LEXICAL ANALYZER
 162 ** =======================================================
 163 */
 164
 165
 166
 167 static int check_next (LexState *ls, const char *set) {
 168   if (!strchr(set, ls->current))
 169     return 0;
 170   save_and_next(ls);
 171   return 1;
 172 }
 173
 174
 175 static void buffreplace (LexState *ls, char from, char to) {
 176   size_t n = luaZ_bufflen(ls->buff);
 177   char *p = luaZ_buffer(ls->buff);
 178   while (n--)
 179     if (p[n] == from) p[n] = to;
 180 }
 181
 182
 183 static void trydecpoint (LexState *ls, SemInfo *seminfo) {
 184   /* format error: try to update decimal point separator */
 185   struct lconv *cv = localeconv();
 186   char old = ls->decpoint;
 187   ls->decpoint = (cv ? cv->decimal_point[0] : '.');
 188   buffreplace(ls, old, ls->decpoint);  /* try updated decimal separator */
 189   if (!luaO_str2d(luaZ_buffer(ls->buff), &seminfo->r)) {
 190     /* format error with correct decimal point: no more options */
 191     buffreplace(ls, ls->decpoint, '.');  /* undo change (for error message) */
 192     luaX_lexerror(ls, "malformed number", TK_NUMBER);
 193   }
 194 }
 195
 196
 197 /* LUA_NUMBER */
 198 static void read_numeral (LexState *ls, SemInfo *seminfo) {
 199   lua_assert(isdigit(ls->current));
 200   do {
 201     save_and_next(ls);
 202   } while (isdigit(ls->current) || ls->current == '.');
 203   if (check_next(ls, "Ee"))  /* `E'? */
 204     check_next(ls, "+-");  /* optional exponent sign */
 205   while (isalnum(ls->current) || ls->current == '_')
 206     save_and_next(ls);
 207   save(ls, '\0');
 208   buffreplace(ls, '.', ls->decpoint);  /* follow locale for decimal point */
 209   if (!luaO_str2d(luaZ_buffer(ls->buff), &seminfo->r))  /* format error? */
 210     trydecpoint(ls, seminfo); /* try to update decimal point separator */
 211 }
 212
 213
 214 static int skip_sep (LexState *ls) {
 215   int count = 0;
 216   int s = ls->current;
 217   lua_assert(s == '[' || s == ']');
 218   save_and_next(ls);
 219   while (ls->current == '=') {
 220     save_and_next(ls);
 221     count++;
 222   }
 223   return (ls->current == s) ? count : (-count) - 1;
 224 }
 225
 226
 227 static void read_long_string (LexState *ls, SemInfo *seminfo, int sep) {
 228   int cont = 0;
 229   (void)(cont);  /* avoid warnings when `cont' is not used */
 230   save_and_next(ls);  /* skip 2nd `[' */
 231   if (currIsNewline(ls))  /* string starts with a newline? */
 232     inclinenumber(ls);  /* skip it */
 233   for (;;) {
 234     switch (ls->current) {
 235       case EOZ:
 236         luaX_lexerror(ls, (seminfo) ? "unfinished long string" :
 237                                    "unfinished long comment", TK_EOS);
 238         break;  /* to avoid warnings */
 239 #if defined(LUA_COMPAT_LSTR)
 240       case '[': {
 241         if (skip_sep(ls) == sep) {
 242           save_and_next(ls);  /* skip 2nd `[' */
 243           cont++;
 244 #if LUA_COMPAT_LSTR == 1
 245           if (sep == 0)
 246             luaX_lexerror(ls, "nesting of [[...]] is deprecated", '[');
 247 #endif
 248         }
 249         break;
 250       }
 251 #endif
 252       case ']': {
 253         if (skip_sep(ls) == sep) {
 254           save_and_next(ls);  /* skip 2nd `]' */
 255 #if defined(LUA_COMPAT_LSTR) && LUA_COMPAT_LSTR == 2
 256           cont--;
 257           if (sep == 0 && cont >= 0) break;
 258 #endif
 259           goto endloop;
 260         }
 261         break;
 262       }
 263       case '\n':
 264       case '\r': {
 265         save(ls, '\n');
 266         inclinenumber(ls);
 267         if (!seminfo) luaZ_resetbuffer(ls->buff);  /* avoid wasting space */
 268         break;
 269       }
 270       default: {
 271         if (seminfo) save_and_next(ls);
 272         else next(ls);
 273       }
 274     }
 275   } endloop:
 276   if (seminfo)
 277     seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + (2 + sep),
 278                                      luaZ_bufflen(ls->buff) - 2*(2 + sep));
 279 }
 280
 281
 282 static void read_string (LexState *ls, int del, SemInfo *seminfo) {
 283   save_and_next(ls);
 284   while (ls->current != del) {
 285     switch (ls->current) {
 286       case EOZ:
 287         luaX_lexerror(ls, "unfinished string", TK_EOS);
 288         continue;  /* to avoid warnings */
 289       case '\n':
 290       case '\r':
 291         luaX_lexerror(ls, "unfinished string", TK_STRING);
 292         continue;  /* to avoid warnings */
 293       case '\\': {
 294         int c;
 295         next(ls);  /* do not save the `\' */
 296         switch (ls->current) {
 297           case 'a': c = '\a'; break;
 298           case 'b': c = '\b'; break;
 299           case 'f': c = '\f'; break;
 300           case 'n': c = '\n'; break;
 301           case 'r': c = '\r'; break;
 302           case 't': c = '\t'; break;
 303           case 'v': c = '\v'; break;
 304           case '\n':  /* go through */
 305           case '\r': save(ls, '\n'); inclinenumber(ls); continue;
 306           case EOZ: continue;  /* will raise an error next loop */
 307           case '$':
 308             if (del == '"') {
 309               save_and_next(ls); /* skip $ */
 310               seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + 1,
 311                             luaZ_bufflen(ls->buff) - 2);
 312               ls->refstr++; /* expect '\' anytime soon */
 313               lua_assert(ls->lookahead.token == TK_EOS);
 314               ls->lookahead.token = TK_CONCAT;
 315               return;
 316             }
 317           default: {
 318             if (!isdigit(ls->current))
 319               save_and_next(ls);  /* handles \\, \", \', and \? */
 320             else {  /* \xxx */
 321               int i = 0;
 322               c = 0;
 323               do {
 324                 c = 10*c + (ls->current-'0');
 325                 next(ls);
 326               } while (++i<3 && isdigit(ls->current));
 327               if (c > UCHAR_MAX)
 328                 luaX_lexerror(ls, "escape sequence too large", TK_STRING);
 329               save(ls, c);
 330             }
 331             continue;
 332           }
 333             // "\x00".."\xFFFF": raw hex literals
 334             //
 335           case 'x': {
 336               int i;
 337               c = 0;
 338               next(ls);   // skip 'x'
 339
 340               for( i=0; i<5 && isxdigit(ls->current); i++ ) {
 341                 c = (c<<4) | _HEX2INT_(ls->current);
 342                 next(ls);
 343               };
 344
 345             switch (i) {
 346                 case 4: save( ls, (c>>8) & 0xff );  // pass-through..
 347                 case 2: save( ls, c&0xff );
 348                         break;
 349                 case 5:
 350                 default:
 351                     luaX_lexerror(ls, "escape sequence wrong size (2 or 4 digits)", TK_STRING);
 352             }
 353           }
 354           continue;
 355
 356             // "\u0000".."\x10FFFF": UTF-8 encoded Unicode
 357             //
 358             // Note that although codes are entered like this (must have min. four digit,
 359             // just to tease you!) the actual outcome in the string will vary between
 360             // 1..4 bytes, depending on the value range.
 361             //
 362           case 'u': {
 363               int i;
 364               c = 0;
 365               next(ls);   // skip 'x'
 366
 367               for ( i=0; i<7 && isxdigit(ls->current); i++ ) {
 368                 c = (c<<4) | _HEX2INT_(ls->current);
 369                 next(ls);
 370               };
 371
 372               if ((i!=4) && (i!=6)) {
 373                 luaX_lexerror(ls, "escape sequence wrong size (4 or 6 digits)", TK_STRING);
 374               }
 375
 376     /* http://www.cl.cam.ac.uk/~mgk25/unicode.html#utf-8
 377         U-00000000 - U-0000007F:        0xxxxxxx
 378         U-00000080 - U-000007FF:        110xxxxx 10xxxxxx
 379         U-00000800 - U-0000FFFF:        1110xxxx 10xxxxxx 10xxxxxx
 380         U-00010000 - U-001FFFFF:        11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
 381         (U-00200000 - U-03FFFFFF:       111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx)
 382         (U-04000000 - U-7FFFFFFF:       1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx)
 383     */
 384               if ( c <= 0x007f ) {  // 1-byte
 385                 save( ls, c );
 386               } else if ( c <= 0x000007ff ) {   // 2-byte
 387                 save( ls, 0xc0 | ((c>>6)&0x1f) );
 388                 save( ls, 0x80 | (c&0x3f) );
 389               } else if ( c <= 0x0000ffff ) {   // 3-byte
 390                 save( ls, 0xe0 | ((c>>12)&0x0f) );
 391                 save( ls, 0x80 | ((c>>6)&0x3f) );
 392                 save( ls, 0x80 | (c&0x3f) );
 393               } else if ( c <= 0x001fffff ) {   // 4-byte
 394                 save( ls, 0xf0 | ((c>>18)&0x07) );
 395                 save( ls, 0x80 | ((c>>12)&0x3f) );
 396                 save( ls, 0x80 | ((c>>6)&0x3f) );
 397                 save( ls, 0x80 | (c&0x3f) );
 398               } else {
 399                 luaX_lexerror(ls, "escape sequence too large", TK_STRING);
 400               }
 401             }
 402             continue;
 403         }
 404         save(ls, c);
 405         next(ls);
 406         continue;
 407       }
 408       default:
 409         save_and_next(ls);
 410     }
 411   }
 412   save_and_next(ls);  /* skip delimiter */
 413   seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + 1,
 414                                    luaZ_bufflen(ls->buff) - 2);
 415 }
 416
 417 static void pre_lua(LexState *ls)
 418 {
 419         char mstr[128];
 420         luaZ_resetbuffer(ls->buff);
 421         sprintf(mstr, "= macro at %d", ls->linenumber);
 422         while (1) {
 423                 //printf("a%d\n", ls->current);
 424                 next(ls);
 425                 //printf("b%d\n", ls->current);
 426                 /* save line till newline */
 427                 while (ls->current != '\r' && ls->current != '\n' && ls->current != EOZ) {
 428                         save_and_next(ls);
 429                 }
 430                 if (ls->current != EOZ) {
 431                         inclinenumber(ls);
 432                         save(ls, '\n');
 433                 }
 434 reentry:
 435                 //printf("%d\n", ls->current);
 436                 if (ls->current == EOZ || ls->current != '#') break;
 437                 //printf("ja%d\n", ls->current);
 438                 next(ls);
 439                 //printf("jb%d\n", ls->current);
 440                 switch (ls->current) {
 441                         case ' ':
 442                         case '\t':
 443                                 /* skip comment */
 444                                 while (!currIsNewline(ls) && ls->current != EOZ)
 445                                         next(ls);
 446                                 inclinenumber(ls);
 447                                 if (ls->current == '#')
 448                                         goto reentry;
 449                                 //printf(" x%d\n", ls->current);
 450                                 break;
 451                         /* 2nd '#' */
 452                         case '#':
 453                                 //printf("cont\n");
 454                                 continue;
 455                         default:
 456                                 luaX_lexerror(ls, "invalid pragma", ls->current);
 457                 }
 458                 break;
 459         } while (0);
 460 //      printf("compiling %s %d\n",luaZ_buffer(ls->buff),luaZ_bufflen(ls->buff));
 461         luaL_loadbuffer(ls->L, luaZ_buffer(ls->buff), luaZ_bufflen(ls->buff), mstr);
 462         luaZ_resetbuffer(ls->buff);
 463         /* call the buffer, expect one return value */
 464         lua_call(ls->L, 0, 1);
 465         if (!lua_isnil(ls->L, -1)) {
 466                 size_t l;
 467                 const char *s = luaL_checklstring(ls->L, -1, &l);
 468                 //printf("prepending %s %d %d\n", s, strlen(s), l);
 469                 luaZ_prepend(ls->z, s, l);
 470                 next(ls);
 471         }
 472         lua_pop(ls->L, 1);
 473 }
 474
 475 static int llex (LexState *ls, SemInfo *seminfo) {
 476   luaZ_resetbuffer(ls->buff);
 477   for (;;) {
 478     switch (ls->current) {
 479       case '\n':
 480       case '\r': {
 481         inclinenumber(ls);
 482         /* "# " is comment, ## is lua preprocessing */
 483         if (ls->current == '#') {
 484                 next(ls);
 485                 switch (ls->current) {
 486                         case '#':
 487                                 pre_lua(ls);
 488                                 continue;
 489                         case ' ':
 490                         case '\t':
 491                                 while (!currIsNewline(ls) && ls->current != EOZ)
 492                                         next(ls);
 493                                 continue;
 494                         default:
 495                                 luaX_lexerror(ls, "invalid pragma", '#');
 496                 }
 497         }
 498         continue;
 499       }
 500       case '-': {
 501         next(ls);
 502         if (ls->current != '-') return '-';
 503         /* else is a comment */
 504         next(ls);
 505         if (ls->current == '[') {
 506           int sep = skip_sep(ls);
 507           luaZ_resetbuffer(ls->buff);  /* `skip_sep' may dirty the buffer */
 508           if (sep >= 0) {
 509             read_long_string(ls, NULL, sep);  /* long comment */
 510             luaZ_resetbuffer(ls->buff);
 511             continue;
 512           }
 513         }
 514         /* else short comment */
 515         while (!currIsNewline(ls) && ls->current != EOZ)
 516           next(ls);
 517         continue;
 518       }
 519       case '[': {
 520         int sep = skip_sep(ls);
 521         if (sep >= 0) {
 522           read_long_string(ls, seminfo, sep);
 523           return TK_STRING;
 524         }
 525         else if (sep == -1) return '[';
 526         else luaX_lexerror(ls, "invalid long string delimiter", TK_STRING);
 527       }
 528       case '=': {
 529         next(ls);
 530         if (ls->current != '=') return '=';
 531         else { next(ls); return TK_EQ; }
 532       }
 533       case '<': {
 534         next(ls);
 535         if (ls->current == '<') { next(ls); return TK_SHL; }
 536         if (ls->current != '=') return '<';
 537         else { next(ls); return TK_LE; }
 538       }
 539       case '>': {
 540         next(ls);
 541         if (ls->current == '>') { next(ls); return TK_SHR; }
 542         if (ls->current != '=') return '>';
 543         else { next(ls); return TK_GE; }
 544       }
 545       case '^': {
 546         next(ls);
 547         if (ls->current == '^') { next(ls); return TK_XOR; }
 548         else { return '^'; }
 549       }
 550       case '!': {
 551         next(ls);
 552         if (ls->current != '=') return '~';
 553         else { next(ls); return TK_NE; }
 554       }
 555       case '~': {
 556         next(ls);
 557         if (ls->current != '=') return '~';
 558         else { next(ls); return TK_NE; }
 559       }
 560       case '"':
 561       case '\'': {
 562         read_string(ls, ls->current, seminfo);
 563         return TK_STRING;
 564       }
 565       case '.': {
 566         save_and_next(ls);
 567         if (check_next(ls, ".")) {
 568           if (check_next(ls, "."))
 569             return TK_DOTS;   /* ... */
 570           else return TK_CONCAT;   /* .. */
 571         }
 572         else if (!isdigit(ls->current)) return '.';
 573         else {
 574           read_numeral(ls, seminfo);
 575           return TK_NUMBER;
 576         }
 577       }
 578       case '$': {
 579         int i = 0;
 580         next(ls);
 581         while (isdigit(ls->current)) {
 582           i = 10*i + (ls->current-'0');
 583           next(ls);
 584         }
 585         seminfo->r = i;
 586         return '$';
 587       }
 588       case EOZ: {
 589         return TK_EOS;
 590       }
 591       case '\\': if (ls->refstr) {
 592         ls->refstr--;
 593         ls->current = '"'; /* whacky! */
 594         return TK_CONCAT;
 595       }
 596       default: {
 597         if (isspace(ls->current)) {
 598           lua_assert(!currIsNewline(ls));
 599           next(ls);
 600           continue;
 601         }
 602         else if (isdigit(ls->current)) {
 603           read_numeral(ls, seminfo);
 604           return TK_NUMBER;
 605         }
 606         else if (isalpha(ls->current) || ls->current == '_') {
 607           /* identifier or reserved word */
 608           TString *ts;
 609           do {
 610             save_and_next(ls);
 611           } while (isalnum(ls->current) || ls->current == '_');
 612           ts = luaX_newstring(ls, luaZ_buffer(ls->buff),
 613                                   luaZ_bufflen(ls->buff));
 614           if (ts->tsv.reserved > 0)  /* reserved word? */
 615             return ts->tsv.reserved - 1 + FIRST_RESERVED;
 616           else {
 617             seminfo->ts = ts;
 618             return TK_NAME;
 619           }
 620         }
 621         else {
 622           int c = ls->current;
 623           next(ls);
 624           return c;  /* single-char tokens (+ - / ...) */
 625         }
 626       }
 627     }
 628   }
 629 }
 630
 631 #include "ltokenf.c"
 632
 633 void luaX_next (LexState *ls) {
 634   ls->lastline = ls->linenumber;
 635   if (ls->lookahead.token != TK_EOS) {  /* is there a look-ahead token? */
 636     ls->t = ls->lookahead;  /* use this one */
 637     ls->lookahead.token = TK_EOS;  /* and discharge it */
 638   }
 639   else
 640     ls->t.token = llex(ls, &ls->t.seminfo);  /* read next token */
 641 }
 642
 643
 644 void luaX_lookahead (LexState *ls) {
 645   lua_assert(ls->lookahead.token == TK_EOS);
 646   ls->lookahead.token = llex(ls, &ls->lookahead.seminfo);
 647 }
 648