Parser/tokenizer.h

   1 #ifndef Py_TOKENIZER_H
   2 #define Py_TOKENIZER_H
   3 #ifdef __cplusplus
   4 extern "C" {
   5 #endif
   6
   7 /***********************************************************
   8 Copyright 1991-1995 by Stichting Mathematisch Centrum, Amsterdam,
   9 The Netherlands.
  10
  11                         All Rights Reserved
  12
  13 Permission to use, copy, modify, and distribute this software and its
  14 documentation for any purpose and without fee is hereby granted,
  15 provided that the above copyright notice appear in all copies and that
  16 both that copyright notice and this permission notice appear in
  17 supporting documentation, and that the names of Stichting Mathematisch
  18 Centrum or CWI or Corporation for National Research Initiatives or
  19 CNRI not be used in advertising or publicity pertaining to
  20 distribution of the software without specific, written prior
  21 permission.
  22
  23 While CWI is the initial source for this software, a modified version
  24 is made available by the Corporation for National Research Initiatives
  25 (CNRI) at the Internet address ftp://ftp.python.org.
  26
  27 STICHTING MATHEMATISCH CENTRUM AND CNRI DISCLAIM ALL WARRANTIES WITH
  28 REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF
  29 MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH
  30 CENTRUM OR CNRI BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL
  31 DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
  32 PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
  33 TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
  34 PERFORMANCE OF THIS SOFTWARE.
  35
  36 ******************************************************************/
  37
  38 /* Tokenizer interface */
  39
  40 #include "token.h"      /* For token types */
  41
  42 #define MAXINDENT 100   /* Max indentation level */
  43
  44 /* Tokenizer state */
  45 struct tok_state {
  46         /* Input state; buf <= cur <= inp <= end */
  47         /* NB an entire line is held in the buffer */
  48         char *buf;      /* Input buffer, or NULL; malloc'ed if fp != NULL */
  49         char *cur;      /* Next character in buffer */
  50         char *inp;      /* End of data in buffer */
  51         char *end;      /* End of input buffer if buf != NULL */
  52         char *start;    /* Start of current token if not NULL */
  53         int done;       /* E_OK normally, E_EOF at EOF, otherwise error code */
  54         /* NB If done != E_OK, cur must be == inp!!! */
  55         FILE *fp;       /* Rest of input; NULL if tokenizing a string */
  56         int tabsize;    /* Tab spacing */
  57         int indent;     /* Current indentation index */
  58         int indstack[MAXINDENT];        /* Stack of indents */
  59         int atbol;      /* Nonzero if at begin of new line */
  60         int pendin;     /* Pending indents (if > 0) or dedents (if < 0) */
  61         char *prompt, *nextprompt;      /* For interactive prompting */
  62         int lineno;     /* Current line number */
  63         int level;      /* () [] {} Parentheses nesting level */
  64                         /* Used to allow free continuations inside them */
  65         /* Stuff for checking on different tab sizes */
  66         char *filename; /* For error messages */
  67         int altwarning; /* Issue warning if alternate tabs don't match */
  68         int alterror;   /* Issue error if alternate tabs don't match */
  69         int alttabsize; /* Alternate tab spacing */
  70         int altindstack[MAXINDENT];     /* Stack of alternate indents */
  71 };
  72
  73 extern struct tok_state *PyTokenizer_FromString Py_PROTO((char *));
  74 extern struct tok_state *PyTokenizer_FromFile
  75         Py_PROTO((FILE *, char *, char *));
  76 extern void PyTokenizer_Free Py_PROTO((struct tok_state *));
  77 extern int PyTokenizer_Get Py_PROTO((struct tok_state *, char **, char **));
  78
  79 #ifdef __cplusplus
  80 }
  81 #endif
  82 #endif /* !Py_TOKENIZER_H */