com32/lua/src/llex.c

   1 /*
   2 ** $Id: llex.c,v 2.20.1.2 2009/11/23 14:58:22 roberto Exp $
   3 ** Lexical Analyzer
   4 ** See Copyright Notice in lua.h
   5 */
   6
   7
   8 #include <ctype.h>
   9 #ifndef SYSLINUX
  10 #include <locale.h>
  11 #endif
  12 #include <string.h>
  13
  14 #define llex_c
  15 #define LUA_CORE
  16
  17 #include "lua.h"
  18
  19 #include "ldo.h"
  20 #include "llex.h"
  21 #include "lobject.h"
  22 #include "lparser.h"
  23 #include "lstate.h"
  24 #include "lstring.h"
  25 #include "ltable.h"
  26 #include "lzio.h"
  27
  28
  29
  30 #define next(ls) (ls->current = zgetc(ls->z))
  31
  32
  33
  34
  35 #define currIsNewline(ls)       (ls->current == '\n' || ls->current == '\r')
  36
  37
  38 /* ORDER RESERVED */
  39 const char *const luaX_tokens [] = {
  40     "and", "break", "do", "else", "elseif",
  41     "end", "false", "for", "function", "if",
  42     "in", "local", "nil", "not", "or", "repeat",
  43     "return", "then", "true", "until", "while",
  44     "..", "...", "==", ">=", "<=", "~=",
  45     "<number>", "<name>", "<string>", "<eof>",
  46     NULL
  47 };
  48
  49
  50 #define save_and_next(ls) (save(ls, ls->current), next(ls))
  51
  52
  53 static void save (LexState *ls, int c) {
  54   Mbuffer *b = ls->buff;
  55   if (b->n + 1 > b->buffsize) {
  56     size_t newsize;
  57     if (b->buffsize >= MAX_SIZET/2)
  58       luaX_lexerror(ls, "lexical element too long", 0);
  59     newsize = b->buffsize * 2;
  60     luaZ_resizebuffer(ls->L, b, newsize);
  61   }
  62   b->buffer[b->n++] = cast(char, c);
  63 }
  64
  65
  66 void luaX_init (lua_State *L) {
  67   int i;
  68   for (i=0; i<NUM_RESERVED; i++) {
  69     TString *ts = luaS_new(L, luaX_tokens[i]);
  70     luaS_fix(ts);  /* reserved words are never collected */
  71     lua_assert(strlen(luaX_tokens[i])+1 <= TOKEN_LEN);
  72     ts->tsv.reserved = cast_byte(i+1);  /* reserved word */
  73   }
  74 }
  75
  76
  77 #define MAXSRC          80
  78
  79
  80 const char *luaX_token2str (LexState *ls, int token) {
  81   if (token < FIRST_RESERVED) {
  82     lua_assert(token == cast(unsigned char, token));
  83     return (iscntrl(token)) ? luaO_pushfstring(ls->L, "char(%d)", token) :
  84                               luaO_pushfstring(ls->L, "%c", token);
  85   }
  86   else
  87     return luaX_tokens[token-FIRST_RESERVED];
  88 }
  89
  90
  91 static const char *txtToken (LexState *ls, int token) {
  92   switch (token) {
  93     case TK_NAME:
  94     case TK_STRING:
  95     case TK_NUMBER:
  96       save(ls, '\0');
  97       return luaZ_buffer(ls->buff);
  98     default:
  99       return luaX_token2str(ls, token);
 100   }
 101 }
 102
 103
 104 void luaX_lexerror (LexState *ls, const char *msg, int token) {
 105   char buff[MAXSRC];
 106   luaO_chunkid(buff, getstr(ls->source), MAXSRC);
 107   msg = luaO_pushfstring(ls->L, "%s:%d: %s", buff, ls->linenumber, msg);
 108   if (token)
 109     luaO_pushfstring(ls->L, "%s near " LUA_QS, msg, txtToken(ls, token));
 110   luaD_throw(ls->L, LUA_ERRSYNTAX);
 111 }
 112
 113
 114 void luaX_syntaxerror (LexState *ls, const char *msg) {
 115   luaX_lexerror(ls, msg, ls->t.token);
 116 }
 117
 118
 119 TString *luaX_newstring (LexState *ls, const char *str, size_t l) {
 120   lua_State *L = ls->L;
 121   TString *ts = luaS_newlstr(L, str, l);
 122   TValue *o = luaH_setstr(L, ls->fs->h, ts);  /* entry for `str' */
 123   if (ttisnil(o)) {
 124     setbvalue(o, 1);  /* make sure `str' will not be collected */
 125     luaC_checkGC(L);
 126   }
 127   return ts;
 128 }
 129
 130
 131 static void inclinenumber (LexState *ls) {
 132   int old = ls->current;
 133   lua_assert(currIsNewline(ls));
 134   next(ls);  /* skip `\n' or `\r' */
 135   if (currIsNewline(ls) && ls->current != old)
 136     next(ls);  /* skip `\n\r' or `\r\n' */
 137   if (++ls->linenumber >= MAX_INT)
 138     luaX_syntaxerror(ls, "chunk has too many lines");
 139 }
 140
 141
 142 void luaX_setinput (lua_State *L, LexState *ls, ZIO *z, TString *source) {
 143   ls->decpoint = '.';
 144   ls->L = L;
 145   ls->lookahead.token = TK_EOS;  /* no look-ahead token */
 146   ls->z = z;
 147   ls->fs = NULL;
 148   ls->linenumber = 1;
 149   ls->lastline = 1;
 150   ls->source = source;
 151   luaZ_resizebuffer(ls->L, ls->buff, LUA_MINBUFFER);  /* initialize buffer */
 152   next(ls);  /* read first char */
 153 }
 154
 155
 156
 157 /*
 158 ** =======================================================
 159 ** LEXICAL ANALYZER
 160 ** =======================================================
 161 */
 162
 163
 164
 165 static int check_next (LexState *ls, const char *set) {
 166   if (!strchr(set, ls->current))
 167     return 0;
 168   save_and_next(ls);
 169   return 1;
 170 }
 171
 172
 173 static void buffreplace (LexState *ls, char from, char to) {
 174   size_t n = luaZ_bufflen(ls->buff);
 175   char *p = luaZ_buffer(ls->buff);
 176   while (n--)
 177     if (p[n] == from) p[n] = to;
 178 }
 179
 180
 181 static void trydecpoint (LexState *ls, SemInfo *seminfo) {
 182   /* format error: try to update decimal point separator */
 183   char old = ls->decpoint;
 184 #ifndef SYSLINUX
 185   struct lconv *cv = localeconv();
 186   ls->decpoint = (cv ? cv->decimal_point[0] : '.');
 187 #endif
 188   buffreplace(ls, old, ls->decpoint);  /* try updated decimal separator */
 189   if (!luaO_str2d(luaZ_buffer(ls->buff), &seminfo->r)) {
 190     /* format error with correct decimal point: no more options */
 191     buffreplace(ls, ls->decpoint, '.');  /* undo change (for error message) */
 192     luaX_lexerror(ls, "malformed number", TK_NUMBER);
 193   }
 194 }
 195
 196
 197 /* LUA_NUMBER */
 198 static void read_numeral (LexState *ls, SemInfo *seminfo) {
 199   lua_assert(isdigit(ls->current));
 200   do {
 201     save_and_next(ls);
 202   } while (isdigit(ls->current) || ls->current == '.');
 203   if (check_next(ls, "Ee"))  /* `E'? */
 204     check_next(ls, "+-");  /* optional exponent sign */
 205   while (isalnum(ls->current) || ls->current == '_')
 206     save_and_next(ls);
 207   save(ls, '\0');
 208   buffreplace(ls, '.', ls->decpoint);  /* follow locale for decimal point */
 209   if (!luaO_str2d(luaZ_buffer(ls->buff), &seminfo->r))  /* format error? */
 210     trydecpoint(ls, seminfo); /* try to update decimal point separator */
 211 }
 212
 213
 214 static int skip_sep (LexState *ls) {
 215   int count = 0;
 216   int s = ls->current;
 217   lua_assert(s == '[' || s == ']');
 218   save_and_next(ls);
 219   while (ls->current == '=') {
 220     save_and_next(ls);
 221     count++;
 222   }
 223   return (ls->current == s) ? count : (-count) - 1;
 224 }
 225
 226
 227 static void read_long_string (LexState *ls, SemInfo *seminfo, int sep) {
 228   int cont = 0;
 229   (void)(cont);  /* avoid warnings when `cont' is not used */
 230   save_and_next(ls);  /* skip 2nd `[' */
 231   if (currIsNewline(ls))  /* string starts with a newline? */
 232     inclinenumber(ls);  /* skip it */
 233   for (;;) {
 234     switch (ls->current) {
 235       case EOZ:
 236         luaX_lexerror(ls, (seminfo) ? "unfinished long string" :
 237                                    "unfinished long comment", TK_EOS);
 238         break;  /* to avoid warnings */
 239 #if defined(LUA_COMPAT_LSTR)
 240       case '[': {
 241         if (skip_sep(ls) == sep) {
 242           save_and_next(ls);  /* skip 2nd `[' */
 243           cont++;
 244 #if LUA_COMPAT_LSTR == 1
 245           if (sep == 0)
 246             luaX_lexerror(ls, "nesting of [[...]] is deprecated", '[');
 247 #endif
 248         }
 249         break;
 250       }
 251 #endif
 252       case ']': {
 253         if (skip_sep(ls) == sep) {
 254           save_and_next(ls);  /* skip 2nd `]' */
 255 #if defined(LUA_COMPAT_LSTR) && LUA_COMPAT_LSTR == 2
 256           cont--;
 257           if (sep == 0 && cont >= 0) break;
 258 #endif
 259           goto endloop;
 260         }
 261         break;
 262       }
 263       case '\n':
 264       case '\r': {
 265         save(ls, '\n');
 266         inclinenumber(ls);
 267         if (!seminfo) luaZ_resetbuffer(ls->buff);  /* avoid wasting space */
 268         break;
 269       }
 270       default: {
 271         if (seminfo) save_and_next(ls);
 272         else next(ls);
 273       }
 274     }
 275   } endloop:
 276   if (seminfo)
 277     seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + (2 + sep),
 278                                      luaZ_bufflen(ls->buff) - 2*(2 + sep));
 279 }
 280
 281
 282 static void read_string (LexState *ls, int del, SemInfo *seminfo) {
 283   save_and_next(ls);
 284   while (ls->current != del) {
 285     switch (ls->current) {
 286       case EOZ:
 287         luaX_lexerror(ls, "unfinished string", TK_EOS);
 288         continue;  /* to avoid warnings */
 289       case '\n':
 290       case '\r':
 291         luaX_lexerror(ls, "unfinished string", TK_STRING);
 292         continue;  /* to avoid warnings */
 293       case '\\': {
 294         int c;
 295         next(ls);  /* do not save the `\' */
 296         switch (ls->current) {
 297           case 'a': c = '\a'; break;
 298           case 'b': c = '\b'; break;
 299           case 'f': c = '\f'; break;
 300           case 'n': c = '\n'; break;
 301           case 'r': c = '\r'; break;
 302           case 't': c = '\t'; break;
 303           case 'v': c = '\v'; break;
 304           case '\n':  /* go through */
 305           case '\r': save(ls, '\n'); inclinenumber(ls); continue;
 306           case EOZ: continue;  /* will raise an error next loop */
 307           default: {
 308             if (!isdigit(ls->current))
 309               save_and_next(ls);  /* handles \\, \", \', and \? */
 310             else {  /* \xxx */
 311               int i = 0;
 312               c = 0;
 313               do {
 314                 c = 10*c + (ls->current-'0');
 315                 next(ls);
 316               } while (++i<3 && isdigit(ls->current));
 317               if (c > UCHAR_MAX)
 318                 luaX_lexerror(ls, "escape sequence too large", TK_STRING);
 319               save(ls, c);
 320             }
 321             continue;
 322           }
 323         }
 324         save(ls, c);
 325         next(ls);
 326         continue;
 327       }
 328       default:
 329         save_and_next(ls);
 330     }
 331   }
 332   save_and_next(ls);  /* skip delimiter */
 333   seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + 1,
 334                                    luaZ_bufflen(ls->buff) - 2);
 335 }
 336
 337
 338 static int llex (LexState *ls, SemInfo *seminfo) {
 339   luaZ_resetbuffer(ls->buff);
 340   for (;;) {
 341     switch (ls->current) {
 342       case '\n':
 343       case '\r': {
 344         inclinenumber(ls);
 345         continue;
 346       }
 347       case '-': {
 348         next(ls);
 349         if (ls->current != '-') return '-';
 350         /* else is a comment */
 351         next(ls);
 352         if (ls->current == '[') {
 353           int sep = skip_sep(ls);
 354           luaZ_resetbuffer(ls->buff);  /* `skip_sep' may dirty the buffer */
 355           if (sep >= 0) {
 356             read_long_string(ls, NULL, sep);  /* long comment */
 357             luaZ_resetbuffer(ls->buff);
 358             continue;
 359           }
 360         }
 361         /* else short comment */
 362         while (!currIsNewline(ls) && ls->current != EOZ)
 363           next(ls);
 364         continue;
 365       }
 366       case '[': {
 367         int sep = skip_sep(ls);
 368         if (sep >= 0) {
 369           read_long_string(ls, seminfo, sep);
 370           return TK_STRING;
 371         }
 372         else if (sep == -1) return '[';
 373         else luaX_lexerror(ls, "invalid long string delimiter", TK_STRING);
 374       }
 375       case '=': {
 376         next(ls);
 377         if (ls->current != '=') return '=';
 378         else { next(ls); return TK_EQ; }
 379       }
 380       case '<': {
 381         next(ls);
 382         if (ls->current != '=') return '<';
 383         else { next(ls); return TK_LE; }
 384       }
 385       case '>': {
 386         next(ls);
 387         if (ls->current != '=') return '>';
 388         else { next(ls); return TK_GE; }
 389       }
 390       case '~': {
 391         next(ls);
 392         if (ls->current != '=') return '~';
 393         else { next(ls); return TK_NE; }
 394       }
 395       case '"':
 396       case '\'': {
 397         read_string(ls, ls->current, seminfo);
 398         return TK_STRING;
 399       }
 400       case '.': {
 401         save_and_next(ls);
 402         if (check_next(ls, ".")) {
 403           if (check_next(ls, "."))
 404             return TK_DOTS;   /* ... */
 405           else return TK_CONCAT;   /* .. */
 406         }
 407         else if (!isdigit(ls->current)) return '.';
 408         else {
 409           read_numeral(ls, seminfo);
 410           return TK_NUMBER;
 411         }
 412       }
 413       case EOZ: {
 414         return TK_EOS;
 415       }
 416       default: {
 417         if (isspace(ls->current)) {
 418           lua_assert(!currIsNewline(ls));
 419           next(ls);
 420           continue;
 421         }
 422         else if (isdigit(ls->current)) {
 423           read_numeral(ls, seminfo);
 424           return TK_NUMBER;
 425         }
 426         else if (isalpha(ls->current) || ls->current == '_') {
 427           /* identifier or reserved word */
 428           TString *ts;
 429           do {
 430             save_and_next(ls);
 431           } while (isalnum(ls->current) || ls->current == '_');
 432           ts = luaX_newstring(ls, luaZ_buffer(ls->buff),
 433                                   luaZ_bufflen(ls->buff));
 434           if (ts->tsv.reserved > 0)  /* reserved word? */
 435             return ts->tsv.reserved - 1 + FIRST_RESERVED;
 436           else {
 437             seminfo->ts = ts;
 438             return TK_NAME;
 439           }
 440         }
 441         else {
 442           int c = ls->current;
 443           next(ls);
 444           return c;  /* single-char tokens (+ - / ...) */
 445         }
 446       }
 447     }
 448   }
 449 }
 450
 451
 452 void luaX_next (LexState *ls) {
 453   ls->lastline = ls->linenumber;
 454   if (ls->lookahead.token != TK_EOS) {  /* is there a look-ahead token? */
 455     ls->t = ls->lookahead;  /* use this one */
 456     ls->lookahead.token = TK_EOS;  /* and discharge it */
 457   }
 458   else
 459     ls->t.token = llex(ls, &ls->t.seminfo);  /* read next token */
 460 }
 461
 462
 463 void luaX_lookahead (LexState *ls) {
 464   lua_assert(ls->lookahead.token == TK_EOS);
 465   ls->lookahead.token = llex(ls, &ls->lookahead.seminfo);
 466 }
 467