src/lexer.h

   1 #ifndef LEXER_H
   2 #define LEXER_H
   3
   4 #include "prelexer.h"
   5
   6 /* Structure to hold single token. */
   7 typedef struct
   8 {
   9     gchar* string;                      /* Poniter to local copy of token string. */
  10     guint start_index;                  /* Start index in original stream. */
  11     guint end_index;                    /* End index in original stream. */
  12     LexerTokenType token_type;          /* Type of token. */
  13 } LexerToken;
  14
  15 /* Structure to hold lexer state and all the tokens. */
  16 typedef struct
  17 {
  18     PreLexerState *prelexer;            /* Pre-lexer state. Pre-lexer is part of lexer. */
  19     LexerToken *tokens;                 /* Pointer to the dynamic array of LexerTokens. */
  20     guint token_count;                  /* Count of tokens in array. */
  21     guint next_token;                   /* Index of next, to be sent, token. */
  22     struct parser_state *parent;        /* Pointer to the parent parser. */
  23 } LexerState;
  24
  25 /* Create a new LexerState object and fill the dynamic array with tokens. */
  26 LexerState* l_create_lexer(const gchar*, struct parser_state*);
  27
  28 /* Destroy LexerState object and free up space. */
  29 void l_destroy_lexer(LexerState*);
  30
  31 /* Tokanize complete string. */
  32 void l_insert_all_tokens(LexerState*);
  33
  34 /* Return next, to be sent, token. */
  35 LexerToken* l_get_next_token(LexerState*);
  36
  37 /* Roll back one token. */
  38 void l_roll_back(LexerState*);
  39
  40 #endif /* LEXER_H */