libtu/tokenizer.h

   1 /*
   2  * libtu/tokenizer.h
   3  *
   4  * Copyright (c) Tuomo Valkonen 1999-2002.
   5  *
   6  * You may distribute and modify this library under the terms of either
   7  * the Clarified Artistic License or the GNU LGPL, version 2.1 or later.
   8  */
   9
  10 #ifndef LIBTU_TOKENIZER_H
  11 #define LIBTU_TOKENIZER_H
  12
  13 #include <stdio.h>
  14 #include "types.h"
  15
  16
  17 #define TOK_SET_BOOL(TOK, VAL)         {(TOK)->type=TOK_BOOL; (TOK)->u.bval=VAL;}
  18 #define TOK_SET_LONG(TOK, VAL)         {(TOK)->type=TOK_LONG; (TOK)->u.lval=VAL;}
  19 #define TOK_SET_DOUBLE(TOK, VAL)     {(TOK)->type=TOK_DOUBLE; (TOK)->u.dval=VAL;}
  20 #define TOK_SET_CHAR(TOK, VAL)         {(TOK)->type=TOK_CHAR; (TOK)->u.cval=VAL;}
  21 #define TOK_SET_STRING(TOK, VAL)     {(TOK)->type=TOK_STRING; (TOK)->u.sval=VAL;}
  22 #define TOK_SET_IDENT(TOK, VAL)     {(TOK)->type=TOK_IDENT; (TOK)->u.sval=VAL;}
  23 #define TOK_SET_COMMENT(TOK, VAL)     {(TOK)->type=TOK_COMMENT; (TOK)->u.sval=VAL;}
  24 #define TOK_SET_OP(TOK, VAL)         {(TOK)->type=TOK_OP; (TOK)->u.opval=VAL;}
  25
  26 #define TOK_TYPE(TOK)                ((TOK)->type)
  27 #define TOK_BOOL_VAL(TOK)            ((TOK)->u.bval)
  28 #define TOK_LONG_VAL(TOK)            ((TOK)->u.lval)
  29 #define TOK_DOUBLE_VAL(TOK)            ((TOK)->u.dval)
  30 #define TOK_CHAR_VAL(TOK)            ((TOK)->u.cval)
  31 #define TOK_STRING_VAL(TOK)            ((TOK)->u.sval)
  32 #define TOK_IDENT_VAL(TOK)            ((TOK)->u.sval)
  33 #define TOK_COMMENT_VAL(TOK)        ((TOK)->u.sval)
  34 #define TOK_OP_VAL(TOK)                ((TOK)->u.opval)
  35
  36 #define TOK_IS_INVALID(TOK)            ((TOK)->type==TOK_INVALID)
  37 #define TOK_IS_BOOL(TOK)            ((TOK)->type==TOK_BOOL)
  38 #define TOK_IS_LONG(TOK)            ((TOK)->type==TOK_LONG)
  39 #define TOK_IS_DOUBLE(TOK)            ((TOK)->type==TOK_DOUBLE)
  40 #define TOK_IS_CHAR(TOK)            ((TOK)->type==TOK_CHAR)
  41 #define TOK_IS_STRING(TOK)            ((TOK)->type==TOK_STRING)
  42 #define TOK_IS_IDENT(TOK)            ((TOK)->type==TOK_IDENT)
  43 #define TOK_IS_COMMENT(TOK)            ((TOK)->type==TOK_COMMENT)
  44 #define TOK_IS_OP(TOK)                ((TOK)->type==TOK_OP)
  45
  46 #define TOK_OP_IS(TOK, OP)            ((TOK)->type==TOK_OP && (TOK)->u.opval==(OP))
  47
  48 #define TOK_TAKE_STRING_VAL(TOK)    ((TOK)->type=TOK_INVALID, (TOK)->u.sval)
  49 #define TOK_TAKE_IDENT_VAL(TOK)        ((TOK)->type=TOK_INVALID, (TOK)->u.sval)
  50 #define TOK_TAKE_COMMENT_VAL(TOK)    ((TOK)->type=TOK_INVALID, (TOK)->u.sval)
  51
  52
  53 enum{
  54     TOK_INVALID=0,
  55     TOK_LONG='l',
  56     TOK_DOUBLE='d',
  57     TOK_CHAR='c',
  58     TOK_STRING='s',
  59     TOK_IDENT='i',
  60     TOK_BOOL='b',
  61     TOK_COMMENT='#',
  62     TOK_OP='+'
  63 };
  64
  65
  66 enum{
  67 #define OP2(X,Y)   ((X)|((Y)<<8))
  68 #define OP3(X,Y,Z) ((X)|((Y)<<8)|((Z)<<16))
  69
  70     OP_L_PAR=    '(', OP_R_PAR=    ')', OP_L_BRK=    '[', OP_R_BRK=    ']',
  71     OP_L_BRC=    '{', OP_R_BRC=    '}', OP_COMMA=    ',', OP_SCOLON=    ';',
  72
  73     OP_PLUS=    '+', OP_MINUS=    '-', OP_MUL=    '*', OP_DIV=    '/',
  74     OP_MOD=        '%', OP_POW=    '^', OP_OR=     '|', OP_AND=    '&',
  75     /*OP_NOT=    '~',*/ OP_NOT=    '!', OP_ASGN=    '=', OP_LT=        '<',
  76     OP_GT=        '>', OP_DOT=    '.', OP_COLON=    ':', OP_QMARK=    '?',
  77     OP_AT=        '@',
  78     OP_NEXTLINE='\n',OP_EOF=    -1,
  79
  80     OP_INC=        OP2('+','+'),         OP_DEC=    OP2('-','-'),
  81     OP_LSHIFT=    OP2('<','<'),          OP_RSHIFT=    OP2('>','>'),
  82     OP_AS_INC=    OP2('+','='),          OP_AS_DEC= OP2('-','='),
  83     OP_AS_MUL=    OP2('*','='),          OP_AS_DIV= OP2('/','='),
  84     OP_AS_MOD=    OP2('%','='),          OP_AS_POW= OP2('^','='),
  85
  86 /*    AS_OR=        OP2('|','='),         AS_AND=    OP2('&','='), */
  87     OP_EQ=        OP2('=','='),          OP_NE=        OP2('!','='),
  88     OP_LE=        OP2('<','='),          OP_GE=        OP2('>','=')
  89
  90 /*    L_AND=        OP2('&','&'), L_OR=        OP2('|','|'),
  91     L_XOR=        OP2('^','^'), */
  92
  93 /*    AsLShift=    OP3('<','<','='),
  94     AsRShift=    OP3('>','>','='), */
  95
  96 #undef OP2
  97 #undef OP3
  98 };
  99
 100
 101 typedef struct{
 102     int type;
 103     int line;
 104     union{
 105         bool bval;
 106         long lval;
 107         double dval;
 108         char cval;
 109         char *sval;
 110         int opval;
 111     } u;
 112 } Token;
 113
 114 #define TOK_INIT {0, 0, {0}}
 115
 116
 117 extern void tok_free(Token*tok);
 118 extern void tok_init(Token*tok);
 119
 120
 121 /* */
 122
 123
 124 enum{
 125     TOKZ_IGNORE_NEXTLINE=0x1,
 126     TOKZ_READ_COMMENTS=0x2,
 127     TOKZ_PARSER_INDENT_MODE=0x04,
 128     TOKZ_ERROR_TOLERANT=0x8,
 129     TOKZ_READ_FROM_BUFFER=0x10,
 130     TOKZ_DEFAULT_OPTION=0x20
 131 };
 132
 133
 134 enum{
 135     E_TOKZ_UNEXPECTED_EOF=1,
 136     E_TOKZ_UNEXPECTED_EOL,
 137     E_TOKZ_EOL_EXPECTED,
 138     E_TOKZ_INVALID_CHAR,
 139     E_TOKZ_TOOBIG,
 140     E_TOKZ_NUMFMT,
 141     E_TOKZ_NUM_JUNK,
 142     E_TOKZ_NOTINT,
 143     E_TOKZ_RANGE,
 144     E_TOKZ_MULTICHAR,
 145
 146     E_TOKZ_TOKEN_LIMIT,
 147     E_TOKZ_UNKNOWN_OPTION,
 148     E_TOKZ_SYNTAX,
 149     E_TOKZ_INVALID_ARGUMENT,
 150     E_TOKZ_EOS_EXPECTED,
 151     E_TOKZ_TOO_FEW_ARGS,
 152     E_TOKZ_TOO_MANY_ARGS,
 153     E_TOKZ_MAX_NEST,
 154     E_TOKZ_IDENTIFIER_EXPECTED,
 155
 156     E_TOKZ_LBRACE_EXPECTED
 157 };
 158
 159
 160 struct _ConfOpt;
 161
 162 typedef struct _Tokenizer_FInfo{
 163     FILE *file;
 164     char *name;
 165     int line;
 166     int ungetc;
 167     Token ungettok;
 168 } Tokenizer_FInfo;
 169
 170 typedef struct _Tokenizer_Buffer{
 171         char *data;
 172         int len;
 173         int pos;
 174 } Tokenizer_Buffer;
 175
 176 typedef struct _Tokenizer{
 177     FILE *file;
 178     char *name;
 179     int line;
 180     int ungetc;
 181     Token ungettok;
 182
 183     Tokenizer_Buffer buffer;
 184
 185     int flags;
 186     const struct _ConfOpt **optstack;
 187     int nest_lvl;
 188     void *user_data;
 189
 190     int filestack_n;
 191     Tokenizer_FInfo *filestack;
 192
 193     char **includepaths;
 194 } Tokenizer;
 195
 196
 197 extern Tokenizer *tokz_open(const char *fname);
 198 extern Tokenizer *tokz_open_file(FILE *file, const char *fname);
 199 extern Tokenizer *tokz_prepare_buffer(char *buffer, int len);
 200 extern void tokz_close(Tokenizer *tokz);
 201 extern bool tokz_get_token(Tokenizer *tokz, Token *tok);
 202 extern void tokz_unget_token(Tokenizer *tokz, Token *tok);
 203 extern void tokz_warn_error(const Tokenizer *tokz, int line, int e);
 204 extern void tokz_warn(const Tokenizer *tokz, int line, const char *fmt, ...);
 205
 206 extern bool tokz_pushf(Tokenizer *tokz, const char *fname);
 207 extern bool tokz_pushf_file(Tokenizer *tokz, FILE *file, const char *fname);
 208 extern bool tokz_popf(Tokenizer *tokz);
 209
 210 extern void tokz_set_includepaths(Tokenizer *tokz, char **paths);
 211
 212 #endif /* LIBTU_TOKENIZER_H */