lua/llex.c

   1 /*
   2 ** $Id: llex.c,v 2.20.1.1 2007/12/27 13:02:25 roberto Exp $
   3 ** Lexical Analyzer
   4 ** See Copyright Notice in lua.h
   5 */
   6
   7 #if 0
   8 #include <ctype.h>
   9 #include <locale.h>
  10 #include <string.h>
  11 #endif
  12
  13 #define llex_c
  14 #define LUA_CORE
  15
  16 #include "lua.h"
  17
  18 #include "ldo.h"
  19 #include "llex.h"
  20 #include "lobject.h"
  21 #include "lparser.h"
  22 #include "lstate.h"
  23 #include "lstring.h"
  24 #include "ltable.h"
  25 #include "lzio.h"
  26
  27
  28
  29 #define next(ls) (ls->current = zgetc(ls->z))
  30
  31
  32
  33
  34 #define currIsNewline(ls)       (ls->current == '\n' || ls->current == '\r')
  35
  36
  37 /* ORDER RESERVED */
  38 const char *const luaX_tokens [] = {
  39     "and", "break", "do", "else", "elseif",
  40     "end", "false", "for", "function", "if",
  41     "in", "local", "nil", "not", "or", "repeat",
  42     "return", "then", "true", "until", "while",
  43     "..", "...", "==", ">=", "<=", "~=",
  44     "<number>", "<name>", "<string>", "<eof>",
  45     NULL
  46 };
  47
  48
  49 #define save_and_next(ls) (save(ls, ls->current), next(ls))
  50
  51
  52 static void save (LexState *ls, int c) {
  53   Mbuffer *b = ls->buff;
  54   if (b->n + 1 > b->buffsize) {
  55     size_t newsize;
  56     if (b->buffsize >= MAX_SIZET/2)
  57       luaX_lexerror(ls, "lexical element too long", 0);
  58     newsize = b->buffsize * 2;
  59     luaZ_resizebuffer(ls->L, b, newsize);
  60   }
  61   b->buffer[b->n++] = cast(char, c);
  62 }
  63
  64
  65 void luaX_init (lua_State *L) {
  66   int i;
  67   for (i=0; i<NUM_RESERVED; i++) {
  68     TString *ts = luaS_new(L, luaX_tokens[i]);
  69     luaS_fix(ts);  /* reserved words are never collected */
  70     lua_assert(strlen(luaX_tokens[i])+1 <= TOKEN_LEN);
  71     ts->tsv.reserved = cast_byte(i+1);  /* reserved word */
  72   }
  73 }
  74
  75
  76 #define MAXSRC          80
  77
  78
  79 const char *luaX_token2str (LexState *ls, int token) {
  80   if (token < FIRST_RESERVED) {
  81     lua_assert(token == cast(unsigned char, token));
  82     return (iscntrl(token)) ? luaO_pushfstring(ls->L, "char(%d)", token) :
  83                               luaO_pushfstring(ls->L, "%c", token);
  84   }
  85   else
  86     return luaX_tokens[token-FIRST_RESERVED];
  87 }
  88
  89
  90 static const char *txtToken (LexState *ls, int token) {
  91   switch (token) {
  92     case TK_NAME:
  93     case TK_STRING:
  94     case TK_NUMBER:
  95       save(ls, '\0');
  96       return luaZ_buffer(ls->buff);
  97     default:
  98       return luaX_token2str(ls, token);
  99   }
 100 }
 101
 102
 103 void luaX_lexerror (LexState *ls, const char *msg, int token) {
 104   char buff[MAXSRC];
 105   luaO_chunkid(buff, getstr(ls->source), MAXSRC);
 106   msg = luaO_pushfstring(ls->L, "%s:%d: %s", buff, ls->linenumber, msg);
 107   if (token)
 108     luaO_pushfstring(ls->L, "%s near " LUA_QS, msg, txtToken(ls, token));
 109   luaD_throw(ls->L, LUA_ERRSYNTAX);
 110 }
 111
 112
 113 void luaX_syntaxerror (LexState *ls, const char *msg) {
 114   luaX_lexerror(ls, msg, ls->t.token);
 115 }
 116
 117
 118 TString *luaX_newstring (LexState *ls, const char *str, size_t l) {
 119   lua_State *L = ls->L;
 120   TString *ts = luaS_newlstr(L, str, l);
 121   TValue *o = luaH_setstr(L, ls->fs->h, ts);  /* entry for `str' */
 122   if (ttisnil(o))
 123     setbvalue(o, 1);  /* make sure `str' will not be collected */
 124   return ts;
 125 }
 126
 127
 128 static void inclinenumber (LexState *ls) {
 129   int old = ls->current;
 130   lua_assert(currIsNewline(ls));
 131   next(ls);  /* skip `\n' or `\r' */
 132   if (currIsNewline(ls) && ls->current != old)
 133     next(ls);  /* skip `\n\r' or `\r\n' */
 134   if (++ls->linenumber >= MAX_INT)
 135     luaX_syntaxerror(ls, "chunk has too many lines");
 136 }
 137
 138
 139 void luaX_setinput (lua_State *L, LexState *ls, ZIO *z, TString *source) {
 140   ls->decpoint = '.';
 141   ls->L = L;
 142   ls->lookahead.token = TK_EOS;  /* no look-ahead token */
 143   ls->z = z;
 144   ls->fs = NULL;
 145   ls->linenumber = 1;
 146   ls->lastline = 1;
 147   ls->source = source;
 148   luaZ_resizebuffer(ls->L, ls->buff, LUA_MINBUFFER);  /* initialize buffer */
 149   next(ls);  /* read first char */
 150 }
 151
 152
 153
 154 /*
 155 ** =======================================================
 156 ** LEXICAL ANALYZER
 157 ** =======================================================
 158 */
 159
 160
 161
 162 static int check_next (LexState *ls, const char *set) {
 163   if (!strchr(set, ls->current))
 164     return 0;
 165   save_and_next(ls);
 166   return 1;
 167 }
 168
 169
 170 static void buffreplace (LexState *ls, char from, char to) {
 171   size_t n = luaZ_bufflen(ls->buff);
 172   char *p = luaZ_buffer(ls->buff);
 173   while (n--)
 174     if (p[n] == from) p[n] = to;
 175 }
 176
 177
 178 static void trydecpoint (LexState *ls, SemInfo *seminfo) {
 179   /* format error: try to update decimal point separator */
 180 #if 0
 181   struct lconv *cv = localeconv();
 182   char old = ls->decpoint;
 183   ls->decpoint = (cv ? cv->decimal_point[0] : '.');
 184 #else
 185   char old = ls->decpoint;
 186   ls->decpoint = '.';
 187 #endif
 188   buffreplace(ls, old, ls->decpoint);  /* try updated decimal separator */
 189   if (!luaO_str2d(luaZ_buffer(ls->buff), &seminfo->r)) {
 190     /* format error with correct decimal point: no more options */
 191     buffreplace(ls, ls->decpoint, '.');  /* undo change (for error message) */
 192     luaX_lexerror(ls, "malformed number", TK_NUMBER);
 193   }
 194 }
 195
 196
 197 /* LUA_NUMBER */
 198 static void read_numeral (LexState *ls, SemInfo *seminfo) {
 199   lua_assert(isdigit(ls->current));
 200   do {
 201     save_and_next(ls);
 202   } while (isdigit(ls->current) || ls->current == '.');
 203   if (check_next(ls, "Ee"))  /* `E'? */
 204     check_next(ls, "+-");  /* optional exponent sign */
 205   while (isalnum(ls->current) || ls->current == '_')
 206     save_and_next(ls);
 207   save(ls, '\0');
 208   buffreplace(ls, '.', ls->decpoint);  /* follow locale for decimal point */
 209   if (!luaO_str2d(luaZ_buffer(ls->buff), &seminfo->r))  /* format error? */
 210     trydecpoint(ls, seminfo); /* try to update decimal point separator */
 211 }
 212
 213
 214 static int skip_sep (LexState *ls) {
 215   int count = 0;
 216   int s = ls->current;
 217   lua_assert(s == '[' || s == ']');
 218   save_and_next(ls);
 219   while (ls->current == '=') {
 220     save_and_next(ls);
 221     count++;
 222   }
 223   return (ls->current == s) ? count : (-count) - 1;
 224 }
 225
 226
 227 static void read_long_string (LexState *ls, SemInfo *seminfo, int sep) {
 228   int cont = 0;
 229   (void)(cont);  /* avoid warnings when `cont' is not used */
 230   save_and_next(ls);  /* skip 2nd `[' */
 231   if (currIsNewline(ls))  /* string starts with a newline? */
 232     inclinenumber(ls);  /* skip it */
 233   for (;;) {
 234     switch (ls->current) {
 235       case EOZ:
 236         luaX_lexerror(ls, (seminfo) ? "unfinished long string" :
 237                                    "unfinished long comment", TK_EOS);
 238         break;  /* to avoid warnings */
 239 #if defined(LUA_COMPAT_LSTR)
 240       case '[': {
 241         if (skip_sep(ls) == sep) {
 242           save_and_next(ls);  /* skip 2nd `[' */
 243           cont++;
 244 #if LUA_COMPAT_LSTR == 1
 245           if (sep == 0)
 246             luaX_lexerror(ls, "nesting of [[...]] is deprecated", '[');
 247 #endif
 248         }
 249         break;
 250       }
 251 #endif
 252       case ']': {
 253         if (skip_sep(ls) == sep) {
 254           save_and_next(ls);  /* skip 2nd `]' */
 255 #if defined(LUA_COMPAT_LSTR) && LUA_COMPAT_LSTR == 2
 256           cont--;
 257           if (sep == 0 && cont >= 0) break;
 258 #endif
 259           goto endloop;
 260         }
 261         break;
 262       }
 263       case '\n':
 264       case '\r': {
 265         save(ls, '\n');
 266         inclinenumber(ls);
 267         if (!seminfo) luaZ_resetbuffer(ls->buff);  /* avoid wasting space */
 268         break;
 269       }
 270       default: {
 271         if (seminfo) save_and_next(ls);
 272         else next(ls);
 273       }
 274     }
 275   } endloop:
 276   if (seminfo)
 277     seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + (2 + sep),
 278                                      luaZ_bufflen(ls->buff) - 2*(2 + sep));
 279 }
 280
 281
 282 static void read_string (LexState *ls, int del, SemInfo *seminfo) {
 283   save_and_next(ls);
 284   while (ls->current != del) {
 285     switch (ls->current) {
 286       case EOZ:
 287         luaX_lexerror(ls, "unfinished string", TK_EOS);
 288         continue;  /* to avoid warnings */
 289       case '\n':
 290       case '\r':
 291         luaX_lexerror(ls, "unfinished string", TK_STRING);
 292         continue;  /* to avoid warnings */
 293       case '\\': {
 294         int c;
 295         next(ls);  /* do not save the `\' */
 296         switch (ls->current) {
 297           case 'a': c = '\a'; break;
 298           case 'b': c = '\b'; break;
 299           case 'f': c = '\f'; break;
 300           case 'n': c = '\n'; break;
 301           case 'r': c = '\r'; break;
 302           case 't': c = '\t'; break;
 303           case 'v': c = '\v'; break;
 304           case '\n':  /* go through */
 305           case '\r': save(ls, '\n'); inclinenumber(ls); continue;
 306           case EOZ: continue;  /* will raise an error next loop */
 307           default: {
 308             if (!isdigit(ls->current))
 309               save_and_next(ls);  /* handles \\, \", \', and \? */
 310             else {  /* \xxx */
 311               int i = 0;
 312               c = 0;
 313               do {
 314                 c = 10*c + (ls->current-'0');
 315                 next(ls);
 316               } while (++i<3 && isdigit(ls->current));
 317               if (c > UCHAR_MAX)
 318                 luaX_lexerror(ls, "escape sequence too large", TK_STRING);
 319               save(ls, c);
 320             }
 321             continue;
 322           }
 323         }
 324         save(ls, c);
 325         next(ls);
 326         continue;
 327       }
 328       default:
 329         save_and_next(ls);
 330     }
 331   }
 332   save_and_next(ls);  /* skip delimiter */
 333   seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + 1,
 334                                    luaZ_bufflen(ls->buff) - 2);
 335 }
 336
 337
 338 static int llex (LexState *ls, SemInfo *seminfo) {
 339   luaZ_resetbuffer(ls->buff);
 340   for (;;) {
 341     switch (ls->current) {
 342       case '\n':
 343       case '\r': {
 344         inclinenumber(ls);
 345         continue;
 346       }
 347       case '-': {
 348         next(ls);
 349         if (ls->current != '-') return '-';
 350         /* else is a comment */
 351         next(ls);
 352         if (ls->current == '[') {
 353           int sep = skip_sep(ls);
 354           luaZ_resetbuffer(ls->buff);  /* `skip_sep' may dirty the buffer */
 355           if (sep >= 0) {
 356             read_long_string(ls, NULL, sep);  /* long comment */
 357             luaZ_resetbuffer(ls->buff);
 358             continue;
 359           }
 360         }
 361         /* else short comment */
 362         while (!currIsNewline(ls) && ls->current != EOZ)
 363           next(ls);
 364         continue;
 365       }
 366       case '[': {
 367         int sep = skip_sep(ls);
 368         if (sep >= 0) {
 369           read_long_string(ls, seminfo, sep);
 370           return TK_STRING;
 371         }
 372         else if (sep == -1) return '[';
 373         else luaX_lexerror(ls, "invalid long string delimiter", TK_STRING);
 374       }
 375       case '=': {
 376         next(ls);
 377         if (ls->current != '=') return '=';
 378         else { next(ls); return TK_EQ; }
 379       }
 380       case '<': {
 381         next(ls);
 382         if (ls->current != '=') return '<';
 383         else { next(ls); return TK_LE; }
 384       }
 385       case '>': {
 386         next(ls);
 387         if (ls->current != '=') return '>';
 388         else { next(ls); return TK_GE; }
 389       }
 390       case '~': {
 391         next(ls);
 392         if (ls->current != '=') return '~';
 393         else { next(ls); return TK_NE; }
 394       }
 395       case '"':
 396       case '\'': {
 397         read_string(ls, ls->current, seminfo);
 398         return TK_STRING;
 399       }
 400       case '.': {
 401         save_and_next(ls);
 402         if (check_next(ls, ".")) {
 403           if (check_next(ls, "."))
 404             return TK_DOTS;   /* ... */
 405           else return TK_CONCAT;   /* .. */
 406         }
 407         else if (!isdigit(ls->current)) return '.';
 408         else {
 409           read_numeral(ls, seminfo);
 410           return TK_NUMBER;
 411         }
 412       }
 413       case EOZ: {
 414         return TK_EOS;
 415       }
 416       default: {
 417         if (isspace(ls->current)) {
 418           lua_assert(!currIsNewline(ls));
 419           next(ls);
 420           continue;
 421         }
 422         else if (isdigit(ls->current)) {
 423           read_numeral(ls, seminfo);
 424           return TK_NUMBER;
 425         }
 426         else if (isalpha(ls->current) || ls->current == '_') {
 427           /* identifier or reserved word */
 428           TString *ts;
 429           do {
 430             save_and_next(ls);
 431           } while (isalnum(ls->current) || ls->current == '_');
 432           ts = luaX_newstring(ls, luaZ_buffer(ls->buff),
 433                                   luaZ_bufflen(ls->buff));
 434           if (ts->tsv.reserved > 0)  /* reserved word? */
 435             return ts->tsv.reserved - 1 + FIRST_RESERVED;
 436           else {
 437             seminfo->ts = ts;
 438             return TK_NAME;
 439           }
 440         }
 441         else {
 442           int c = ls->current;
 443           next(ls);
 444           return c;  /* single-char tokens (+ - / ...) */
 445         }
 446       }
 447     }
 448   }
 449 }
 450
 451
 452 void luaX_next (LexState *ls) {
 453   ls->lastline = ls->linenumber;
 454   if (ls->lookahead.token != TK_EOS) {  /* is there a look-ahead token? */
 455     ls->t = ls->lookahead;  /* use this one */
 456     ls->lookahead.token = TK_EOS;  /* and discharge it */
 457   }
 458   else
 459     ls->t.token = llex(ls, &ls->t.seminfo);  /* read next token */
 460 }
 461
 462
 463 void luaX_lookahead (LexState *ls) {
 464   lua_assert(ls->lookahead.token == TK_EOS);
 465   ls->lookahead.token = llex(ls, &ls->lookahead.seminfo);
 466 }
 467