src/util/scanner.h

   1 #ifndef EL__UTIL_SCANNER_H
   2 #define EL__UTIL_SCANNER_H
   3
   4 #include "util/error.h"
   5 #include "util/string.h"
   6
   7 /* Define if you want a talking scanner */
   8 /* #define DEBUG_SCANNER */
   9
  10 /** The struct scanner_token describes one scanner state. There are two kinds
  11  * of tokens: char and non-char tokens. Char tokens contains only one char and
  12  * simply have their char value as type. They are tokens having special control
  13  * meaning in the code, like ':', ';', '{', '}' and '*'. Non char tokens has
  14  * one or more chars and contain stuff like number or indentifier strings.  */
  15 struct scanner_token {
  16         /** The type of the token */
  17         int type;
  18
  19         /** Some precedence value */
  20         int precedence;
  21
  22         /** The start of the token string and the token length */
  23         const unsigned char *string;
  24         int length;
  25 };
  26
  27 /* The naming of these two macros is a bit odd .. we compare often with
  28  * "static" strings (I don't have a better word) so the macro name should
  29  * be short. --jonas */
  30
  31 /** Compare the string of @a token with @a str */
  32 #define scanner_token_strlcasecmp(token, str, len) \
  33         ((token) && !c_strlcasecmp((token)->string, (token)->length, str, len))
  34
  35 /** Also compares the token string but using a "static" string */
  36 #define scanner_token_contains(token, str) \
  37         scanner_token_strlcasecmp(token, str, sizeof(str) - 1)
  38
  39
  40 struct scan_table_info {
  41         enum { SCAN_RANGE, SCAN_STRING, SCAN_END } type;
  42         union scan_table_data {
  43                 struct { unsigned char *source; long length; } string;
  44                 struct { unsigned char *start; long end; } range;
  45         } data;
  46         int bits;
  47 };
  48
  49 #define SCAN_TABLE_SIZE 256
  50
  51 #define SCAN_TABLE_INFO(type, data1, data2, bits) \
  52         { (type), { { (data1), (data2) } }, (bits) }
  53
  54 #define SCAN_TABLE_RANGE(from, to, bits) SCAN_TABLE_INFO(SCAN_RANGE, from, to, bits)
  55 #define SCAN_TABLE_STRING(str, bits)     SCAN_TABLE_INFO(SCAN_STRING, str, sizeof(str) - 1, bits)
  56 #define SCAN_TABLE_END                   SCAN_TABLE_INFO(SCAN_END, 0, 0, 0)
  57
  58 struct scanner_string_mapping {
  59         unsigned char *name;
  60         int type;
  61         int base_type;
  62 };
  63
  64 struct scanner;
  65
  66 struct scanner_info {
  67         /** Table containing how to map strings to token types */
  68         const struct scanner_string_mapping *mappings;
  69
  70         /** Information for how to initialize the scanner table */
  71         const struct scan_table_info *scan_table_info;
  72
  73         /** Fills the scanner with tokens. Already scanned tokens
  74          * which have not been requested remain and are moved to the
  75          * start of the scanners token table.
  76          * @returns the current token or NULL if there are none. */
  77         struct scanner_token *(*scan)(struct scanner *scanner);
  78
  79         /** The scanner table.  Contains bitmaps for the various
  80          * characters groups.  Idea sync'ed from mozilla browser. */
  81         int scan_table[SCAN_TABLE_SIZE];
  82
  83         /** Has the scanner info been initialized? */
  84         unsigned int initialized:1;
  85 };
  86
  87
  88 /** Initializes the scanner.
  89  * @relates scanner */
  90 void init_scanner(struct scanner *scanner, struct scanner_info *scanner_info,
  91                   const unsigned char *string, const unsigned char *end);
  92
  93 /** The number of tokens in the scanners token table:
  94  * At best it should be big enough to contain properties with space separated
  95  * values and function calls with up to 3 variables like rgb(). At worst it
  96  * should be no less than 2 in order to be able to peek at the next token in
  97  * the scanner. */
  98 #define SCANNER_TOKENS 10
  99
 100 /** The struct scanner describes the current state of the scanner. */
 101 struct scanner {
 102         /** The very start of the scanned string, the position in the string
 103          * where to scan next and the end of the string. If #position is NULL
 104          * it means that no more tokens can be retrieved from the string. */
 105         const unsigned char *string, *position, *end;
 106
 107         /** The current token and number of scanned tokens in the table.
 108          * If the number of scanned tokens is less than ::SCANNER_TOKENS
 109          * it is because there are no more tokens in the string. */
 110         struct scanner_token *current;
 111         int tokens;
 112
 113         /** The 'meta' scanner information */
 114         struct scanner_info *info;
 115
 116 #ifdef DEBUG_SCANNER
 117         /** @name Debug info about the caller.
 118          * @{ */
 119         unsigned char *file;
 120         int line;
 121         /** @} */
 122 #endif
 123
 124         /** Some state indicator only meaningful to the scanner internals */
 125         int state;
 126
 127         /** The table contain already scanned tokens. It is maintained in
 128          * order to optimize the scanning a bit and make it possible to look
 129          * ahead at the next token. You should always use the accessors
 130          * (defined below) for getting tokens from the scanner. */
 131         struct scanner_token table[SCANNER_TOKENS];
 132 };
 133
 134 /** @relates scanner  */
 135 #define scanner_has_tokens(scanner) \
 136         ((scanner)->tokens > 0 && (scanner)->current < (scanner)->table + (scanner)->tokens)
 137
 138 /** This macro checks if the current scanner state is valid. Meaning if the
 139  * scanners table is full the last token skipping or get_next_scanner_token()
 140  * call made it possible to get the type of the next token.
 141  * @relates scanner */
 142 #define check_scanner(scanner) \
 143         (scanner->tokens < SCANNER_TOKENS \
 144          || scanner->current + 1 < scanner->table + scanner->tokens)
 145
 146
 147 /** @name Scanner table accessors and mutators
 148  * @{ */
 149
 150 /** Checks the type of the next token
 151  * @relates scanner */
 152 #define check_next_scanner_token(scanner, token_type)                           \
 153         (scanner_has_tokens(scanner)                                    \
 154          && ((scanner)->current + 1 < (scanner)->table + (scanner)->tokens)     \
 155          && (scanner)->current[1].type == (token_type))
 156
 157 /** Access current and next token. Getting the next token might cause
 158  * a rescan so any token pointers that has been stored in a local variable
 159  * might not be valid after the call.
 160  * @relates scanner */
 161 static inline struct scanner_token *
 162 get_scanner_token(struct scanner *scanner)
 163 {
 164         return scanner_has_tokens(scanner) ? scanner->current : NULL;
 165 }
 166
 167 /** Do a scanning if we do not have also have access to next token.
 168  * @relates scanner */
 169 static inline struct scanner_token *
 170 get_next_scanner_token(struct scanner *scanner)
 171 {
 172         return (scanner_has_tokens(scanner)
 173                 && (++scanner->current + 1 >= scanner->table + scanner->tokens)
 174                 ? scanner->info->scan(scanner) : get_scanner_token(scanner));
 175 }
 176
 177 /** This should just make the code more understandable .. hopefully
 178  * @relates scanner */
 179 #define skip_scanner_token(scanner) get_next_scanner_token(scanner)
 180
 181 /** Removes tokens from the scanner until it meets a token of the given type.
 182  * This token will then also be skipped.
 183  * @relates scanner */
 184 struct scanner_token *
 185 skip_scanner_tokens(struct scanner *scanner, int skipto, int precedence);
 186
 187 /** @} */
 188
 189 /** Looks up the string from @a ident to @a end to in the scanners
 190  * string mapping table
 191  * @relates scanner */
 192 int
 193 map_scanner_string(struct scanner *scanner,
 194                    const unsigned char *ident, const unsigned char *end,
 195                    int base_type);
 196
 197 #ifdef DEBUG_SCANNER
 198 /** @relates scanner */
 199 void dump_scanner(struct scanner *scanner);
 200 #endif
 201
 202 /* The begin_token_scanning() and end_token_scanning() functions provide the
 203  * basic setup and teardown for the rescan function made public via the
 204  * scanner_info->scan member.
 205  * @returns NULL if it is not necessary to try to scan for more tokens
 206  * @relates scanner */
 207 static inline struct scanner_token *
 208 begin_token_scanning(struct scanner *scanner)
 209 {
 210         struct scanner_token *table = scanner->table;
 211         struct scanner_token *table_end = table + scanner->tokens;
 212         int move_to_front = int_max(table_end - scanner->current, 0);
 213         struct scanner_token *current = move_to_front ? scanner->current : table;
 214         size_t moved_size = 0;
 215
 216         assert(scanner->current);
 217
 218         /* Move any untouched tokens */
 219         if (move_to_front) {
 220                 moved_size = move_to_front * sizeof(*table);
 221                 memmove(table, current, moved_size);
 222                 current = &table[move_to_front];
 223         }
 224
 225         /* Clear all unused tokens */
 226         memset(current, 0, sizeof(*table) * SCANNER_TOKENS - moved_size);
 227
 228         if (!scanner->position) {
 229                 scanner->tokens = move_to_front ? move_to_front : -1;
 230                 scanner->current = table;
 231                 assert(check_scanner(scanner));
 232                 return NULL;
 233         }
 234
 235         scanner->tokens = move_to_front;
 236
 237         return table;
 238 }
 239
 240 /* Updates the @a scanner struct after scanning has been done. The position
 241  * _after_ the last valid token is taken as the @a end argument.
 242  *
 243  * It is ok for @a end to be < scanner->table since scanner->tokens
 244  * will become <= 0 anyway.
 245  * @relates scanner */
 246 static inline struct scanner_token *
 247 end_token_scanning(struct scanner *scanner, struct scanner_token *end)
 248 {
 249         assert(end <= scanner->table + SCANNER_TOKENS);
 250
 251         scanner->tokens = (end - scanner->table);
 252         scanner->current = scanner->table;
 253         if (scanner->position >= scanner->end)
 254                 scanner->position = NULL;
 255
 256         assert(check_scanner(scanner));
 257
 258         return get_scanner_token(scanner);
 259 }
 260
 261 #endif