Parser/tokenizer.h

   1 #ifndef Py_TOKENIZER_H
   2 #define Py_TOKENIZER_H
   3 #ifdef __cplusplus
   4 extern "C" {
   5 #endif
   6
   7 /***********************************************************
   8 Copyright (c) 2000, BeOpen.com.
   9 Copyright (c) 1995-2000, Corporation for National Research Initiatives.
  10 Copyright (c) 1990-1995, Stichting Mathematisch Centrum.
  11 All rights reserved.
  12
  13 See the file "Misc/COPYRIGHT" for information on usage and
  14 redistribution of this file, and for a DISCLAIMER OF ALL WARRANTIES.
  15 ******************************************************************/
  16
  17 /* Tokenizer interface */
  18
  19 #include "token.h"      /* For token types */
  20
  21 #define MAXINDENT 100   /* Max indentation level */
  22
  23 /* Tokenizer state */
  24 struct tok_state {
  25         /* Input state; buf <= cur <= inp <= end */
  26         /* NB an entire line is held in the buffer */
  27         char *buf;      /* Input buffer, or NULL; malloc'ed if fp != NULL */
  28         char *cur;      /* Next character in buffer */
  29         char *inp;      /* End of data in buffer */
  30         char *end;      /* End of input buffer if buf != NULL */
  31         char *start;    /* Start of current token if not NULL */
  32         int done;       /* E_OK normally, E_EOF at EOF, otherwise error code */
  33         /* NB If done != E_OK, cur must be == inp!!! */
  34         FILE *fp;       /* Rest of input; NULL if tokenizing a string */
  35         int tabsize;    /* Tab spacing */
  36         int indent;     /* Current indentation index */
  37         int indstack[MAXINDENT];        /* Stack of indents */
  38         int atbol;      /* Nonzero if at begin of new line */
  39         int pendin;     /* Pending indents (if > 0) or dedents (if < 0) */
  40         char *prompt, *nextprompt;      /* For interactive prompting */
  41         int lineno;     /* Current line number */
  42         int level;      /* () [] {} Parentheses nesting level */
  43                         /* Used to allow free continuations inside them */
  44         /* Stuff for checking on different tab sizes */
  45         char *filename; /* For error messages */
  46         int altwarning; /* Issue warning if alternate tabs don't match */
  47         int alterror;   /* Issue error if alternate tabs don't match */
  48         int alttabsize; /* Alternate tab spacing */
  49         int altindstack[MAXINDENT];     /* Stack of alternate indents */
  50 };
  51
  52 extern struct tok_state *PyTokenizer_FromString(char *);
  53 extern struct tok_state *PyTokenizer_FromFile(FILE *, char *, char *);
  54 extern void PyTokenizer_Free(struct tok_state *);
  55 extern int PyTokenizer_Get(struct tok_state *, char **, char **);
  56
  57 #ifdef __cplusplus
  58 }
  59 #endif
  60 #endif /* !Py_TOKENIZER_H */