js/src/jsscan.h

   1 /* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
   2  *
   3  * ***** BEGIN LICENSE BLOCK *****
   4  * Version: MPL 1.1/GPL 2.0/LGPL 2.1
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version
   7  * 1.1 (the "License"); you may not use this file except in compliance with
   8  * the License. You may obtain a copy of the License at
   9  * http://www.mozilla.org/MPL/
  10  *
  11  * Software distributed under the License is distributed on an "AS IS" basis,
  12  * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  13  * for the specific language governing rights and limitations under the
  14  * License.
  15  *
  16  * The Original Code is Mozilla Communicator client code, released
  17  * March 31, 1998.
  18  *
  19  * The Initial Developer of the Original Code is
  20  * Netscape Communications Corporation.
  21  * Portions created by the Initial Developer are Copyright (C) 1998
  22  * the Initial Developer. All Rights Reserved.
  23  *
  24  * Contributor(s):
  25  *
  26  * Alternatively, the contents of this file may be used under the terms of
  27  * either of the GNU General Public License Version 2 or later (the "GPL"),
  28  * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  29  * in which case the provisions of the GPL or the LGPL are applicable instead
  30  * of those above. If you wish to allow use of your version of this file only
  31  * under the terms of either the GPL or the LGPL, and not to allow others to
  32  * use your version of this file under the terms of the MPL, indicate your
  33  * decision by deleting the provisions above and replace them with the notice
  34  * and other provisions required by the GPL or the LGPL. If you do not delete
  35  * the provisions above, a recipient may use your version of this file under
  36  * the terms of any one of the MPL, the GPL or the LGPL.
  37  *
  38  * ***** END LICENSE BLOCK ***** */
  39
  40 #ifndef jsscan_h___
  41 #define jsscan_h___
  42 /*
  43  * JS lexical scanner interface.
  44  */
  45 #include <stddef.h>
  46 #include <stdio.h>
  47 #include "jsversion.h"
  48 #include "jsopcode.h"
  49 #include "jsprvtd.h"
  50 #include "jspubtd.h"
  51
  52 JS_BEGIN_EXTERN_C
  53
  54 #define JS_KEYWORD(keyword, type, op, version) \
  55     extern const char js_##keyword##_str[];
  56 #include "jskeyword.tbl"
  57 #undef JS_KEYWORD
  58
  59 typedef enum JSTokenType {
  60     TOK_ERROR = -1,                     /* well-known as the only code < EOF */
  61     TOK_EOF = 0,                        /* end of file */
  62     TOK_EOL = 1,                        /* end of line */
  63     TOK_SEMI = 2,                       /* semicolon */
  64     TOK_COMMA = 3,                      /* comma operator */
  65     TOK_ASSIGN = 4,                     /* assignment ops (= += -= etc.) */
  66     TOK_HOOK = 5, TOK_COLON = 6,        /* conditional (?:) */
  67     TOK_OR = 7,                         /* logical or (||) */
  68     TOK_AND = 8,                        /* logical and (&&) */
  69     TOK_BITOR = 9,                      /* bitwise-or (|) */
  70     TOK_BITXOR = 10,                    /* bitwise-xor (^) */
  71     TOK_BITAND = 11,                    /* bitwise-and (&) */
  72     TOK_EQOP = 12,                      /* equality ops (== !=) */
  73     TOK_RELOP = 13,                     /* relational ops (< <= > >=) */
  74     TOK_SHOP = 14,                      /* shift ops (<< >> >>>) */
  75     TOK_PLUS = 15,                      /* plus */
  76     TOK_MINUS = 16,                     /* minus */
  77     TOK_STAR = 17, TOK_DIVOP = 18,      /* multiply/divide ops (* / %) */
  78     TOK_UNARYOP = 19,                   /* unary prefix operator */
  79     TOK_INC = 20, TOK_DEC = 21,         /* increment/decrement (++ --) */
  80     TOK_DOT = 22,                       /* member operator (.) */
  81     TOK_LB = 23, TOK_RB = 24,           /* left and right brackets */
  82     TOK_LC = 25, TOK_RC = 26,           /* left and right curlies (braces) */
  83     TOK_LP = 27, TOK_RP = 28,           /* left and right parentheses */
  84     TOK_NAME = 29,                      /* identifier */
  85     TOK_NUMBER = 30,                    /* numeric constant */
  86     TOK_STRING = 31,                    /* string constant */
  87     TOK_REGEXP = 32,                    /* RegExp constant */
  88     TOK_PRIMARY = 33,                   /* true, false, null, this, super */
  89     TOK_FUNCTION = 34,                  /* function keyword */
  90     TOK_IF = 35,                        /* if keyword */
  91     TOK_ELSE = 36,                      /* else keyword */
  92     TOK_SWITCH = 37,                    /* switch keyword */
  93     TOK_CASE = 38,                      /* case keyword */
  94     TOK_DEFAULT = 39,                   /* default keyword */
  95     TOK_WHILE = 40,                     /* while keyword */
  96     TOK_DO = 41,                        /* do keyword */
  97     TOK_FOR = 42,                       /* for keyword */
  98     TOK_BREAK = 43,                     /* break keyword */
  99     TOK_CONTINUE = 44,                  /* continue keyword */
 100     TOK_IN = 45,                        /* in keyword */
 101     TOK_VAR = 46,                       /* var keyword */
 102     TOK_WITH = 47,                      /* with keyword */
 103     TOK_RETURN = 48,                    /* return keyword */
 104     TOK_NEW = 49,                       /* new keyword */
 105     TOK_DELETE = 50,                    /* delete keyword */
 106     TOK_DEFSHARP = 51,                  /* #n= for object/array initializers */
 107     TOK_USESHARP = 52,                  /* #n# for object/array initializers */
 108     TOK_TRY = 53,                       /* try keyword */
 109     TOK_CATCH = 54,                     /* catch keyword */
 110     TOK_FINALLY = 55,                   /* finally keyword */
 111     TOK_THROW = 56,                     /* throw keyword */
 112     TOK_INSTANCEOF = 57,                /* instanceof keyword */
 113     TOK_DEBUGGER = 58,                  /* debugger keyword */
 114     TOK_XMLSTAGO = 59,                  /* XML start tag open (<) */
 115     TOK_XMLETAGO = 60,                  /* XML end tag open (</) */
 116     TOK_XMLPTAGC = 61,                  /* XML point tag close (/>) */
 117     TOK_XMLTAGC = 62,                   /* XML start or end tag close (>) */
 118     TOK_XMLNAME = 63,                   /* XML start-tag non-final fragment */
 119     TOK_XMLATTR = 64,                   /* XML quoted attribute value */
 120     TOK_XMLSPACE = 65,                  /* XML whitespace */
 121     TOK_XMLTEXT = 66,                   /* XML text */
 122     TOK_XMLCOMMENT = 67,                /* XML comment */
 123     TOK_XMLCDATA = 68,                  /* XML CDATA section */
 124     TOK_XMLPI = 69,                     /* XML processing instruction */
 125     TOK_AT = 70,                        /* XML attribute op (@) */
 126     TOK_DBLCOLON = 71,                  /* namespace qualified name op (::) */
 127     TOK_ANYNAME = 72,                   /* XML AnyName singleton (*) */
 128     TOK_DBLDOT = 73,                    /* XML descendant op (..) */
 129     TOK_FILTER = 74,                    /* XML filtering predicate op (.()) */
 130     TOK_XMLELEM = 75,                   /* XML element node type (no token) */
 131     TOK_XMLLIST = 76,                   /* XML list node type (no token) */
 132     TOK_YIELD = 77,                     /* yield from generator function */
 133     TOK_ARRAYCOMP = 78,                 /* array comprehension initialiser */
 134     TOK_ARRAYPUSH = 79,                 /* array push within comprehension */
 135     TOK_LEXICALSCOPE = 80,              /* block scope AST node label */
 136     TOK_LET = 81,                       /* let keyword */
 137     TOK_SEQ = 82,                       /* synthetic sequence of statements,
 138                                            not a block */
 139     TOK_RESERVED,                       /* reserved keywords */
 140     TOK_LIMIT                           /* domain size */
 141 } JSTokenType;
 142
 143 #define IS_PRIMARY_TOKEN(tt) \
 144     ((uintN)((tt) - TOK_NAME) <= (uintN)(TOK_PRIMARY - TOK_NAME))
 145
 146 #define TOKEN_TYPE_IS_XML(tt) \
 147     (tt == TOK_AT || tt == TOK_DBLCOLON || tt == TOK_ANYNAME)
 148
 149 #if JS_HAS_BLOCK_SCOPE
 150 # define TOKEN_TYPE_IS_DECL(tt) ((tt) == TOK_VAR || (tt) == TOK_LET)
 151 #else
 152 # define TOKEN_TYPE_IS_DECL(tt) ((tt) == TOK_VAR)
 153 #endif
 154
 155 struct JSStringBuffer {
 156     jschar      *base;
 157     jschar      *limit;         /* length limit for quick bounds check */
 158     jschar      *ptr;           /* slot for next non-NUL char to store */
 159     void        *data;
 160     JSBool      (*grow)(JSStringBuffer *sb, size_t newlength);
 161     void        (*free)(JSStringBuffer *sb);
 162 };
 163
 164 #define STRING_BUFFER_ERROR_BASE        ((jschar *) 1)
 165 #define STRING_BUFFER_OK(sb)            ((sb)->base != STRING_BUFFER_ERROR_BASE)
 166 #define STRING_BUFFER_OFFSET(sb)        ((sb)->ptr -(sb)->base)
 167
 168 extern void
 169 js_InitStringBuffer(JSStringBuffer *sb);
 170
 171 extern void
 172 js_FinishStringBuffer(JSStringBuffer *sb);
 173
 174 extern void
 175 js_AppendChar(JSStringBuffer *sb, jschar c);
 176
 177 extern void
 178 js_RepeatChar(JSStringBuffer *sb, jschar c, uintN count);
 179
 180 extern void
 181 js_AppendCString(JSStringBuffer *sb, const char *asciiz);
 182
 183 extern void
 184 js_AppendJSString(JSStringBuffer *sb, JSString *str);
 185
 186 struct JSTokenPtr {
 187     uint16              index;          /* index of char in physical line */
 188     uint16              lineno;         /* physical line number */
 189 };
 190
 191 struct JSTokenPos {
 192     JSTokenPtr          begin;          /* first character and line of token */
 193     JSTokenPtr          end;            /* index 1 past last char, last line */
 194 };
 195
 196 struct JSToken {
 197     JSTokenType         type;           /* char value or above enumerator */
 198     JSTokenPos          pos;            /* token position in file */
 199     jschar              *ptr;           /* beginning of token in line buffer */
 200     union {
 201         struct {                        /* name or string literal */
 202             JSOp        op;             /* operator, for minimal parser */
 203             JSAtom      *atom;          /* atom table entry */
 204         } s;
 205         uintN           reflags;        /* regexp flags, use tokenbuf to access
 206                                            regexp chars */
 207         struct {                        /* atom pair, for XML PIs */
 208             JSAtom      *atom2;         /* auxiliary atom table entry */
 209             JSAtom      *atom;          /* main atom table entry */
 210         } p;
 211         jsdouble        dval;           /* floating point number */
 212     } u;
 213 };
 214
 215 #define t_op            u.s.op
 216 #define t_reflags       u.reflags
 217 #define t_atom          u.s.atom
 218 #define t_atom2         u.p.atom2
 219 #define t_dval          u.dval
 220
 221 typedef struct JSTokenBuf {
 222     jschar              *base;          /* base of line or stream buffer */
 223     jschar              *limit;         /* limit for quick bounds check */
 224     jschar              *ptr;           /* next char to get, or slot to use */
 225 } JSTokenBuf;
 226
 227 #define JS_LINE_LIMIT   256             /* logical line buffer size limit --
 228                                            physical line length is unlimited */
 229 #define NTOKENS         4               /* 1 current + 2 lookahead, rounded */
 230 #define NTOKENS_MASK    (NTOKENS-1)     /* to power of 2 to avoid divmod by 3 */
 231
 232 struct JSTokenStream {
 233     JSToken             tokens[NTOKENS];/* circular token buffer */
 234     uintN               cursor;         /* index of last parsed token */
 235     uintN               lookahead;      /* count of lookahead tokens */
 236     uintN               lineno;         /* current line number */
 237     uintN               ungetpos;       /* next free char slot in ungetbuf */
 238     jschar              ungetbuf[6];    /* at most 6, for \uXXXX lookahead */
 239     uintN               flags;          /* flags -- see below */
 240     ptrdiff_t           linelen;        /* physical linebuf segment length */
 241     ptrdiff_t           linepos;        /* linebuf offset in physical line */
 242     JSTokenBuf          linebuf;        /* line buffer for diagnostics */
 243     JSTokenBuf          userbuf;        /* user input buffer if !file */
 244     JSStringBuffer      tokenbuf;       /* current token string buffer */
 245     const char          *filename;      /* input filename or null */
 246     FILE                *file;          /* stdio stream if reading from file */
 247     JSSourceHandler     listener;       /* callback for source; eg debugger */
 248     void                *listenerData;  /* listener 'this' data */
 249     void                *listenerTSData;/* listener data for this TokenStream */
 250     jschar              *saveEOL;       /* save next end of line in userbuf, to
 251                                            optimize for very long lines */
 252 };
 253
 254 #define CURRENT_TOKEN(ts)       ((ts)->tokens[(ts)->cursor])
 255 #define ON_CURRENT_LINE(ts,pos) ((uint16)(ts)->lineno == (pos).end.lineno)
 256
 257 /* JSTokenStream flags */
 258 #define TSF_ERROR       0x01            /* fatal error while compiling */
 259 #define TSF_EOF         0x02            /* hit end of file */
 260 #define TSF_NEWLINES    0x04            /* tokenize newlines */
 261 #define TSF_OPERAND     0x08            /* looking for operand, not operator */
 262 #define TSF_NLFLAG      0x20            /* last linebuf ended with \n */
 263 #define TSF_CRFLAG      0x40            /* linebuf would have ended with \r */
 264 #define TSF_DIRTYLINE   0x80            /* non-whitespace since start of line */
 265 #define TSF_OWNFILENAME 0x100           /* ts->filename is malloc'd */
 266 #define TSF_XMLTAGMODE  0x200           /* scanning within an XML tag in E4X */
 267 #define TSF_XMLTEXTMODE 0x400           /* scanning XMLText terminal from E4X */
 268 #define TSF_XMLONLYMODE 0x800           /* don't scan {expr} within text/tag */
 269
 270 /* Flag indicating unexpected end of input, i.e. TOK_EOF not at top-level. */
 271 #define TSF_UNEXPECTED_EOF 0x1000
 272
 273 /*
 274  * To handle the hard case of contiguous HTML comments, we want to clear the
 275  * TSF_DIRTYINPUT flag at the end of each such comment.  But we'd rather not
 276  * scan for --> within every //-style comment unless we have to.  So we set
 277  * TSF_IN_HTML_COMMENT when a <!-- is scanned as an HTML begin-comment, and
 278  * clear it (and TSF_DIRTYINPUT) when we scan --> either on a clean line, or
 279  * only if (ts->flags & TSF_IN_HTML_COMMENT), in a //-style comment.
 280  *
 281  * This still works as before given a malformed comment hiding hack such as:
 282  *
 283  *    <script>
 284  *      <!-- comment hiding hack #1
 285  *      code goes here
 286  *      // --> oops, markup for script-unaware browsers goes here!
 287  *    </script>
 288  *
 289  * It does not cope with malformed comment hiding hacks where --> is hidden
 290  * by C-style comments, or on a dirty line.  Such cases are already broken.
 291  */
 292 #define TSF_IN_HTML_COMMENT 0x2000
 293
 294 /* Ignore keywords and return TOK_NAME instead to the parser. */
 295 #define TSF_KEYWORD_IS_NAME 0x4000
 296
 297 /* Unicode separators that are treated as line terminators, in addition to \n, \r */
 298 #define LINE_SEPARATOR  0x2028
 299 #define PARA_SEPARATOR  0x2029
 300
 301 /*
 302  * Create a new token stream, either from an input buffer or from a file.
 303  * Return null on file-open or memory-allocation failure.
 304  *
 305  * The function uses JSContext.tempPool to allocate internal buffers. The
 306  * caller should release them using JS_ARENA_RELEASE after it has finished
 307  * with the token stream and has called js_CloseTokenStream.
 308  */
 309 extern JSBool
 310 js_InitTokenStream(JSContext *cx, JSTokenStream *ts,
 311                    const jschar *base, size_t length,
 312                    FILE *fp, const char *filename, uintN lineno);
 313
 314 extern void
 315 js_CloseTokenStream(JSContext *cx, JSTokenStream *ts);
 316
 317 extern JS_FRIEND_API(int)
 318 js_fgets(char *buf, int size, FILE *file);
 319
 320 /*
 321  * If the given char array forms JavaScript keyword, return corresponding
 322  * token. Otherwise return TOK_EOF.
 323  */
 324 extern JSTokenType
 325 js_CheckKeyword(const jschar *chars, size_t length);
 326
 327 /*
 328  * Friend-exported API entry point to call a mapping function on each reserved
 329  * identifier in the scanner's keyword table.
 330  */
 331 extern JS_FRIEND_API(void)
 332 js_MapKeywords(void (*mapfun)(const char *));
 333
 334 /*
 335  * Check that str forms a valid JS identifier name. The function does not
 336  * check if str is a JS keyword.
 337  */
 338 extern JSBool
 339 js_IsIdentifier(JSString *str);
 340
 341 /*
 342  * Report a compile-time error by its number. Return true for a warning, false
 343  * for an error. When pn is not null, use it to report error's location.
 344  * Otherwise use ts, which must not be null.
 345  */
 346 JSBool
 347 js_ReportCompileErrorNumber(JSContext *cx, JSTokenStream *ts, JSParseNode *pn,
 348                             uintN flags, uintN errorNumber, ...);
 349
 350 /*
 351  * Steal one JSREPORT_* bit (see jsapi.h) to tell that arguments to the error
 352  * message have const jschar* type, not const char*.
 353  */
 354 #define JSREPORT_UC 0x100
 355
 356 /*
 357  * Look ahead one token and return its type.
 358  */
 359 extern JSTokenType
 360 js_PeekToken(JSContext *cx, JSTokenStream *ts);
 361
 362 extern JSTokenType
 363 js_PeekTokenSameLine(JSContext *cx, JSTokenStream *ts);
 364
 365 /*
 366  * Get the next token from ts.
 367  */
 368 extern JSTokenType
 369 js_GetToken(JSContext *cx, JSTokenStream *ts);
 370
 371 /*
 372  * Push back the last scanned token onto ts.
 373  */
 374 extern void
 375 js_UngetToken(JSTokenStream *ts);
 376
 377 /*
 378  * Get the next token from ts if its type is tt.
 379  */
 380 extern JSBool
 381 js_MatchToken(JSContext *cx, JSTokenStream *ts, JSTokenType tt);
 382
 383 JS_END_EXTERN_C
 384
 385 #endif /* jsscan_h___ */