tok.c

   1 /* neatcc tokenizer */
   2 #include <ctype.h>
   3 #include <unistd.h>
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include "gen.h"
   7 #include "mem.h"
   8 #include "ncc.h"
   9 #include "tok.h"
  10
  11 static struct mem tok_mem;      /* the data read via cpp_read() so far */
  12 static struct mem str;          /* the last tok_str() string */
  13 static char *buf;
  14 static int len;
  15 static int cur;
  16 static char name[NAMELEN];
  17 static int next = -1;
  18 static int pre;
  19
  20 static struct {
  21         char *name;
  22         unsigned id;
  23 } kwds[] = {
  24         {"void", TOK_VOID},
  25         {"static", TOK_STATIC},
  26         {"extern", TOK_EXTERN},
  27         {"return", TOK_RETURN},
  28         {"unsigned", TOK_UNSIGNED},
  29         {"signed", TOK_SIGNED},
  30         {"short", TOK_SHORT},
  31         {"long", TOK_LONG},
  32         {"int", TOK_INT},
  33         {"char", TOK_CHAR},
  34         {"struct", TOK_STRUCT},
  35         {"union", TOK_UNION},
  36         {"enum", TOK_ENUM},
  37         {"typedef", TOK_TYPEDEF},
  38         {"if", TOK_IF},
  39         {"else", TOK_ELSE},
  40         {"for", TOK_FOR},
  41         {"while", TOK_WHILE},
  42         {"do", TOK_DO},
  43         {"switch", TOK_SWITCH},
  44         {"case", TOK_CASE},
  45         {"sizeof", TOK_SIZEOF},
  46         {"break", TOK_BREAK},
  47         {"continue", TOK_CONTINUE},
  48         {"default", TOK_DEFAULT},
  49         {"goto", TOK_GOTO},
  50 };
  51
  52 static char *tok3[] = {
  53         "<<=", ">>=", "...", "<<", ">>", "++", "--", "+=", "-=", "*=", "/=",
  54         "%=", "|=", "&=", "^=", "&&", "||", "==", "!=", "<=", ">=", "->"
  55 };
  56
  57 static int get_tok3(int num)
  58 {
  59         int i;
  60         for (i = 0; i < LEN(tok3); i++)
  61                 if (num == TOK3(tok3[i]))
  62                         return num;
  63         return 0;
  64 }
  65
  66 static char *esc_code = "abefnrtv";
  67 static char *esc = "\a\b\e\f\n\r\t\v";
  68 static char *digs = "0123456789abcdef";
  69
  70 static int esc_char(int *c, char *s)
  71 {
  72         if (*s != '\\') {
  73                 *c = (unsigned char) *s;
  74                 return 1;
  75         }
  76         if (strchr(esc_code, s[1])) {
  77                 *c = esc[strchr(esc_code, s[1]) - esc_code];
  78                 return 2;
  79         }
  80         if (isdigit(s[1]) || s[1] == 'x') {
  81                 int ret = 0;
  82                 int base = 8;
  83                 int i = 1;
  84                 char *d;
  85                 if (s[1] == 'x') {
  86                         base = 16;
  87                         i++;
  88                 }
  89                 while ((d = memchr(digs, s[i], base))) {
  90                         ret *= base;
  91                         ret += d - digs;
  92                         i++;
  93                 }
  94                 *c = ret;
  95                 return i;
  96         }
  97         *c = (unsigned char) s[1];
  98         return 2;
  99 }
 100
 101 static long num;
 102 static int num_bt;
 103
 104 int tok_num(long *n)
 105 {
 106         *n = num;
 107         return num_bt;
 108 }
 109
 110 static void readnum(void)
 111 {
 112         int base = 10;
 113         num_bt = 4 | BT_SIGNED;
 114         if (buf[cur] == '0' && tolower(buf[cur + 1]) == 'x') {
 115                 num_bt &= ~BT_SIGNED;
 116                 base = 16;
 117                 cur += 2;
 118         }
 119         if (strchr(digs, tolower(buf[cur]))) {
 120                 long result = 0;
 121                 char *c;
 122                 if (base == 10 && buf[cur] == '0')
 123                         base = 8;
 124                 while (cur < len && (c = strchr(digs, tolower(buf[cur])))) {
 125                         result *= base;
 126                         result += c - digs;
 127                         cur++;
 128                 }
 129                 num = result;
 130                 while (cur < len) {
 131                         int c = tolower(buf[cur]);
 132                         if (c != 'u' && c != 'l')
 133                                 break;
 134                         if (c == 'u')
 135                                 num_bt &= ~BT_SIGNED;
 136                         if (c == 'l')
 137                                 num_bt = (num_bt & BT_SIGNED) | LONGSZ;
 138                         cur++;
 139                 }
 140                 return;
 141         }
 142         if (buf[cur] == '\'') {
 143                 int ret;
 144                 cur += 2 + esc_char(&ret, buf + cur + 1);
 145                 num = ret;
 146                 return;
 147         }
 148         num = -1;
 149 }
 150
 151 void tok_str(char **buf, int *len)
 152 {
 153         if (len)
 154                 *len = mem_len(&str) + 1;
 155         if (buf)
 156                 *buf = mem_buf(&str);
 157 }
 158
 159 static void readstr(struct mem *mem)
 160 {
 161         char *s = buf + cur;
 162         char *e = buf + len;
 163         int c;
 164         s++;
 165         while (s < e && *s != '"') {
 166                 if (*s == '\\') {
 167                         s += esc_char(&c, s);
 168                         mem_putc(mem, c);
 169                 } else {
 170                         mem_putc(mem, (unsigned char) *s++);
 171                 }
 172         }
 173         cur = s - buf + 1;
 174 }
 175
 176 static int id_char(int c)
 177 {
 178         return isalnum(c) || c == '_';
 179 }
 180
 181 static int skipws(void)
 182 {
 183         int clen;
 184         char *cbuf;
 185         while (1) {
 186                 if (cur == len) {
 187                         clen = 0;
 188                         while (!clen)
 189                                 if (cpp_read(&cbuf, &clen))
 190                                         return 1;
 191                         mem_put(&tok_mem, cbuf, clen);
 192                         buf = mem_buf(&tok_mem);
 193                         len = mem_len(&tok_mem);
 194                 }
 195                 while (cur < len && isspace(buf[cur]))
 196                         cur++;
 197                 if (cur == len)
 198                         continue;
 199                 if (buf[cur] == '\\' && buf[cur + 1] == '\n') {
 200                         cur += 2;
 201                         continue;
 202                 }
 203                 if (buf[cur] == '/' && buf[cur + 1] == '/') {
 204                         while (++cur < len && buf[cur] != '\n')
 205                                 if (buf[cur] == '\\')
 206                                         cur++;
 207                         continue;
 208                 }
 209                 if (buf[cur] == '/' && buf[cur + 1] == '*') {
 210                         while (++cur < len) {
 211                                 if (buf[cur] == '*' && buf[cur + 1] == '/') {
 212                                         cur += 2;
 213                                         break;
 214                                 }
 215                         }
 216                         continue;
 217                 }
 218                 break;
 219         }
 220         return 0;
 221 }
 222
 223 int tok_get(void)
 224 {
 225         int num;
 226         if (next != -1) {
 227                 int tok = next;
 228                 next = -1;
 229                 return tok;
 230         }
 231         pre = cur;
 232         if (skipws())
 233                 return TOK_EOF;
 234         if (buf[cur] == '"') {
 235                 mem_cut(&str, 0);
 236                 while (buf[cur] == '"') {
 237                         readstr(&str);
 238                         if (skipws())
 239                                 return TOK_EOF;
 240                 }
 241                 return TOK_STR;
 242         }
 243         if (isdigit(buf[cur]) || buf[cur] == '\'') {
 244                 readnum();
 245                 return TOK_NUM;
 246         }
 247         if (id_char(buf[cur])) {
 248                 char *s = name;
 249                 int i;
 250                 while (cur < len && id_char(buf[cur]))
 251                         *s++ = buf[cur++];
 252                 *s = '\0';
 253                 for (i = 0; i < LEN(kwds); i++)
 254                         if (!strcmp(kwds[i].name, name))
 255                                 return kwds[i].id;
 256                 return TOK_NAME;
 257         }
 258         if (cur + 3 <= len && (num = get_tok3(TOK3(buf + cur)))) {
 259                 cur += 3;
 260                 return num;
 261         }
 262         if ((num = get_tok3(TOK2(buf + cur)))) {
 263                 cur += 2;
 264                 return num;
 265         }
 266         if (strchr(";,{}()[]<>*&!=+-/%?:|^~.", buf[cur]))
 267                 return buf[cur++];
 268         return -1;
 269 }
 270
 271 int tok_see(void)
 272 {
 273         if (next == -1)
 274                 next = tok_get();
 275         return next;
 276 }
 277
 278 char *tok_id(void)
 279 {
 280         return name;
 281 }
 282
 283 long tok_addr(void)
 284 {
 285         return next == -1 ? cur : pre;
 286 }
 287
 288 void tok_jump(long addr)
 289 {
 290         cur = addr;
 291         pre = cur - 1;
 292         next = -1;
 293 }