Parser/tokenizer.h

   1 #ifndef Py_TOKENIZER_H
   2 #define Py_TOKENIZER_H
   3 #ifdef __cplusplus
   4 extern "C" {
   5 #endif
   6
   7
   8 /* Tokenizer interface */
   9
  10 #include "token.h"      /* For token types */
  11
  12 #define MAXINDENT 100   /* Max indentation level */
  13
  14 /* Tokenizer state */
  15 struct tok_state {
  16         /* Input state; buf <= cur <= inp <= end */
  17         /* NB an entire line is held in the buffer */
  18         char *buf;      /* Input buffer, or NULL; malloc'ed if fp != NULL */
  19         char *cur;      /* Next character in buffer */
  20         char *inp;      /* End of data in buffer */
  21         char *end;      /* End of input buffer if buf != NULL */
  22         char *start;    /* Start of current token if not NULL */
  23         int done;       /* E_OK normally, E_EOF at EOF, otherwise error code */
  24         /* NB If done != E_OK, cur must be == inp!!! */
  25         FILE *fp;       /* Rest of input; NULL if tokenizing a string */
  26         int tabsize;    /* Tab spacing */
  27         int indent;     /* Current indentation index */
  28         int indstack[MAXINDENT];        /* Stack of indents */
  29         int atbol;      /* Nonzero if at begin of new line */
  30         int pendin;     /* Pending indents (if > 0) or dedents (if < 0) */
  31         char *prompt, *nextprompt;      /* For interactive prompting */
  32         int lineno;     /* Current line number */
  33         int level;      /* () [] {} Parentheses nesting level */
  34                         /* Used to allow free continuations inside them */
  35         /* Stuff for checking on different tab sizes */
  36         char *filename; /* For error messages */
  37         int altwarning; /* Issue warning if alternate tabs don't match */
  38         int alterror;   /* Issue error if alternate tabs don't match */
  39         int alttabsize; /* Alternate tab spacing */
  40         int altindstack[MAXINDENT];     /* Stack of alternate indents */
  41 };
  42
  43 extern struct tok_state *PyTokenizer_FromString(char *);
  44 extern struct tok_state *PyTokenizer_FromFile(FILE *, char *, char *);
  45 extern void PyTokenizer_Free(struct tok_state *);
  46 extern int PyTokenizer_Get(struct tok_state *, char **, char **);
  47
  48 #ifdef __cplusplus
  49 }
  50 #endif
  51 #endif /* !Py_TOKENIZER_H */