src/util/scanner.h

   1 #ifndef EL__UTIL_SCANNER_H
   2 #define EL__UTIL_SCANNER_H
   3
   4 #include "util/error.h"
   5
   6 /* Define if you want a talking scanner */
   7 /* #define DEBUG_SCANNER */
   8
   9 /* The {struct scanner_token} describes one scanner state. There are two kinds
  10  * of tokens: char and non-char tokens. Char tokens contains only one char and
  11  * simply have their char value as type. They are tokens having special control
  12  * meaning in the code, like ':', ';', '{', '}' and '*'. Non char tokens has
  13  * one or more chars and contain stuff like number or indentifier strings.  */
  14 struct scanner_token {
  15         /* The type the token */
  16         int type;
  17
  18         /* Some precedence value */
  19         int precedence;
  20
  21         /* The start of the token string and the token length */
  22         unsigned char *string;
  23         int length;
  24 };
  25
  26 /* The naming of these two macros is a bit odd .. we compare often with
  27  * "static" strings (I don't have a better word) so the macro name should
  28  * be short. --jonas */
  29
  30 /* Compare the string of @token with @string */
  31 #define scanner_token_strlcasecmp(token, str, len) \
  32         ((token) && !strlcasecmp((token)->string, (token)->length, str, len))
  33
  34 /* Also compares the token string but using a "static" string */
  35 #define scanner_token_contains(token, str) \
  36         scanner_token_strlcasecmp(token, str, sizeof(str) - 1)
  37
  38
  39 struct scan_table_info {
  40         enum { SCAN_RANGE, SCAN_STRING, SCAN_END } type;
  41         union scan_table_data {
  42                 struct { unsigned char *source; long length; } string;
  43                 struct { unsigned char *start; long end; } range;
  44         } data;
  45         int bits;
  46 };
  47
  48 #define SCAN_TABLE_SIZE 256
  49
  50 #define SCAN_TABLE_INFO(type, data1, data2, bits) \
  51         { (type), { { (data1), (data2) } }, (bits) }
  52
  53 #define SCAN_TABLE_RANGE(from, to, bits) SCAN_TABLE_INFO(SCAN_RANGE, from, to, bits)
  54 #define SCAN_TABLE_STRING(str, bits)     SCAN_TABLE_INFO(SCAN_STRING, str, sizeof(str) - 1, bits)
  55 #define SCAN_TABLE_END                   SCAN_TABLE_INFO(SCAN_END, 0, 0, 0)
  56
  57 struct scanner_string_mapping {
  58         unsigned char *name;
  59         int type;
  60         int base_type;
  61 };
  62
  63 struct scanner;
  64
  65 struct scanner_info {
  66         /* Table containing how to map strings to token types */
  67         const struct scanner_string_mapping *mappings;
  68
  69         /* Information for how to initialize the scanner table */
  70         const struct scan_table_info *scan_table_info;
  71
  72         /* Fills the scanner with tokens. Already scanned tokens which have not
  73          * been requested remain and are moved to the start of the scanners
  74          * token table. */
  75         /* Returns the current token or NULL if there are none. */
  76         struct scanner_token *(*scan)(struct scanner *scanner);
  77
  78         /* The scanner table */
  79         /* Contains bitmaps for the various characters groups.
  80          * Idea sync'ed from mozilla browser. */
  81         int scan_table[SCAN_TABLE_SIZE];
  82
  83         /* Has the scanner info been initialized? */
  84         unsigned int initialized:1;
  85 };
  86
  87
  88 /* Initializes the scanner. */
  89 void init_scanner(struct scanner *scanner, struct scanner_info *scanner_info,
  90                   unsigned char *string, unsigned char *end);
  91
  92 /* The number of tokens in the scanners token table:
  93  * At best it should be big enough to contain properties with space separated
  94  * values and function calls with up to 3 variables like rgb(). At worst it
  95  * should be no less than 2 in order to be able to peek at the next token in
  96  * the scanner. */
  97 #define SCANNER_TOKENS 10
  98
  99 /* The {struct scanner} describes the current state of the scanner. */
 100 struct scanner {
 101         /* The very start of the scanned string, the position in the string
 102          * where to scan next and the end of the string. If position is NULL it
 103          * means that no more tokens can be retrieved from the string. */
 104         unsigned char *string, *position, *end;
 105
 106         /* The current token and number of scanned tokens in the table.
 107          * If the number of scanned tokens is less than SCANNER_TOKENS
 108          * it is because there are no more tokens in the string. */
 109         struct scanner_token *current;
 110         int tokens;
 111
 112         /* The 'meta' scanner information */
 113         struct scanner_info *info;
 114
 115 #ifdef DEBUG_SCANNER
 116         /* Debug info about the caller. */
 117         unsigned char *file;
 118         int line;
 119 #endif
 120
 121         /* Some state indicator only meaningful to the scanner internals */
 122         int state;
 123
 124         /* The table contain already scanned tokens. It is maintained in
 125          * order to optimize the scanning a bit and make it possible to look
 126          * ahead at the next token. You should always use the accessors
 127          * (defined below) for getting tokens from the scanner. */
 128         struct scanner_token table[SCANNER_TOKENS];
 129 };
 130
 131 #define scanner_has_tokens(scanner) \
 132         ((scanner)->tokens > 0 && (scanner)->current < (scanner)->table + (scanner)->tokens)
 133
 134 /* This macro checks if the current scanner state is valid. Meaning if the
 135  * scanners table is full the last token skipping or get_next_scanner_token()
 136  * call made it possible to get the type of the next token. */
 137 #define check_scanner(scanner) \
 138         (scanner->tokens < SCANNER_TOKENS \
 139          || scanner->current + 1 < scanner->table + scanner->tokens)
 140
 141
 142 /* Scanner table accessors and mutators */
 143
 144 /* Checks the type of the next token */
 145 #define check_next_scanner_token(scanner, token_type)                           \
 146         (scanner_has_tokens(scanner)                                    \
 147          && ((scanner)->current + 1 < (scanner)->table + (scanner)->tokens)     \
 148          && (scanner)->current[1].type == (token_type))
 149
 150 /* Access current and next token. Getting the next token might cause
 151  * a rescan so any token pointers that has been stored in a local variable
 152  * might not be valid after the call. */
 153 static inline struct scanner_token *
 154 get_scanner_token(struct scanner *scanner)
 155 {
 156         return scanner_has_tokens(scanner) ? scanner->current : NULL;
 157 }
 158
 159 /* Do a scanning if we do not have also have access to next token. */
 160 static inline struct scanner_token *
 161 get_next_scanner_token(struct scanner *scanner)
 162 {
 163         return (scanner_has_tokens(scanner)
 164                 && (++scanner->current + 1 >= scanner->table + scanner->tokens)
 165                 ? scanner->info->scan(scanner) : get_scanner_token(scanner));
 166 }
 167
 168 /* This should just make the code more understandable .. hopefully */
 169 #define skip_scanner_token(scanner) get_next_scanner_token(scanner)
 170
 171 /* Removes tokens from the scanner until it meets a token of the given type.
 172  * This token will then also be skipped. */
 173 struct scanner_token *
 174 skip_scanner_tokens(struct scanner *scanner, int skipto, int precedence);
 175
 176 /* Looks up the string from @ident to @end to in the scanners string mapping
 177  * table */
 178 int
 179 map_scanner_string(struct scanner *scanner,
 180                    unsigned char *ident, unsigned char *end, int base_type);
 181
 182 #ifdef DEBUG_SCANNER
 183 void dump_scanner(struct scanner *scanner);
 184 #endif
 185
 186 /* The begin_token_scanning() and end_token_scanning() functions provide the
 187  * basic setup and teardown for the rescan function made public via the
 188  * scanner_info->scan member. */
 189
 190 /* Returns NULL if it is not necessary to try to scan for more tokens */
 191 static inline struct scanner_token *
 192 begin_token_scanning(struct scanner *scanner)
 193 {
 194         struct scanner_token *table = scanner->table;
 195         struct scanner_token *table_end = table + scanner->tokens;
 196         int move_to_front = int_max(table_end - scanner->current, 0);
 197         struct scanner_token *current = move_to_front ? scanner->current : table;
 198         size_t moved_size = 0;
 199
 200         assert(scanner->current);
 201
 202         /* Move any untouched tokens */
 203         if (move_to_front) {
 204                 moved_size = move_to_front * sizeof(*table);
 205                 memmove(table, current, moved_size);
 206                 current = &table[move_to_front];
 207         }
 208
 209         /* Clear all unused tokens */
 210         memset(current, 0, sizeof(*table) * SCANNER_TOKENS - moved_size);
 211
 212         if (!scanner->position) {
 213                 scanner->tokens = move_to_front ? move_to_front : -1;
 214                 scanner->current = table;
 215                 assert(check_scanner(scanner));
 216                 return NULL;
 217         }
 218
 219         scanner->tokens = move_to_front;
 220
 221         return table;
 222 }
 223
 224 /* Updates the @scanner struct after scanning has been done. The position
 225  * _after_ the last valid token is taken as the @end argument. */
 226 /* It is ok for @end to be < scanner->table since scanner->tokens will become
 227  * <= 0 anyway. */
 228 static inline struct scanner_token *
 229 end_token_scanning(struct scanner *scanner, struct scanner_token *end)
 230 {
 231         assert(end <= scanner->table + SCANNER_TOKENS);
 232
 233         scanner->tokens = (end - scanner->table);
 234         scanner->current = scanner->table;
 235         if (scanner->position >= scanner->end)
 236                 scanner->position = NULL;
 237
 238         assert(check_scanner(scanner));
 239
 240         return get_scanner_token(scanner);
 241 }
 242
 243 #endif