utils/TableGen/TGLexer.cpp

   1 //===- TGLexer.cpp - Lexer for TableGen -----------------------------------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // Implement the Lexer for TableGen.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "TGLexer.h"
  15 #include "TGSourceMgr.h"
  16 #include "llvm/Support/Streams.h"
  17 #include "llvm/Support/MemoryBuffer.h"
  18 #include <ostream>
  19 #include "llvm/Config/config.h"
  20 #include <cctype>
  21 #include <cstdio>
  22 #include <cstdlib>
  23 #include <cstring>
  24 #include <cerrno>
  25 using namespace llvm;
  26
  27 TGLexer::TGLexer(TGSourceMgr &SM) : SrcMgr(SM) {
  28   CurBuffer = 0;
  29   CurBuf = SrcMgr.getMemoryBuffer(CurBuffer);
  30   CurPtr = CurBuf->getBufferStart();
  31   TokStart = 0;
  32 }
  33
  34 TGLoc TGLexer::getLoc() const {
  35   return TGLoc::getFromPointer(TokStart);
  36 }
  37
  38
  39 /// ReturnError - Set the error to the specified string at the specified
  40 /// location.  This is defined to always return tgtok::Error.
  41 tgtok::TokKind TGLexer::ReturnError(const char *Loc, const std::string &Msg) {
  42   PrintError(Loc, Msg);
  43   return tgtok::Error;
  44 }
  45
  46
  47 void TGLexer::PrintError(const char *Loc, const std::string &Msg) const {
  48   SrcMgr.PrintError(TGLoc::getFromPointer(Loc), Msg);
  49 }
  50
  51 void TGLexer::PrintError(TGLoc Loc, const std::string &Msg) const {
  52   SrcMgr.PrintError(Loc, Msg);
  53 }
  54
  55
  56 int TGLexer::getNextChar() {
  57   char CurChar = *CurPtr++;
  58   switch (CurChar) {
  59   default:
  60     return (unsigned char)CurChar;
  61   case 0: {
  62     // A nul character in the stream is either the end of the current buffer or
  63     // a random nul in the file.  Disambiguate that here.
  64     if (CurPtr-1 != CurBuf->getBufferEnd())
  65       return 0;  // Just whitespace.
  66
  67     // If this is the end of an included file, pop the parent file off the
  68     // include stack.
  69     TGLoc ParentIncludeLoc = SrcMgr.getParentIncludeLoc(CurBuffer);
  70     if (ParentIncludeLoc != TGLoc()) {
  71       CurBuffer = SrcMgr.FindBufferContainingLoc(ParentIncludeLoc);
  72       CurBuf = SrcMgr.getMemoryBuffer(CurBuffer);
  73       CurPtr = ParentIncludeLoc.getPointer();
  74       return getNextChar();
  75     }
  76
  77     // Otherwise, return end of file.
  78     --CurPtr;  // Another call to lex will return EOF again.
  79     return EOF;
  80   }
  81   case '\n':
  82   case '\r':
  83     // Handle the newline character by ignoring it and incrementing the line
  84     // count.  However, be careful about 'dos style' files with \n\r in them.
  85     // Only treat a \n\r or \r\n as a single line.
  86     if ((*CurPtr == '\n' || (*CurPtr == '\r')) &&
  87         *CurPtr != CurChar)
  88       ++CurPtr;  // Eat the two char newline sequence.
  89     return '\n';
  90   }
  91 }
  92
  93 tgtok::TokKind TGLexer::LexToken() {
  94   TokStart = CurPtr;
  95   // This always consumes at least one character.
  96   int CurChar = getNextChar();
  97
  98   switch (CurChar) {
  99   default:
 100     // Handle letters: [a-zA-Z_]
 101     if (isalpha(CurChar) || CurChar == '_' || CurChar == '#')
 102       return LexIdentifier();
 103
 104     // Unknown character, emit an error.
 105     return ReturnError(TokStart, "Unexpected character");
 106   case EOF: return tgtok::Eof;
 107   case ':': return tgtok::colon;
 108   case ';': return tgtok::semi;
 109   case '.': return tgtok::period;
 110   case ',': return tgtok::comma;
 111   case '<': return tgtok::less;
 112   case '>': return tgtok::greater;
 113   case ']': return tgtok::r_square;
 114   case '{': return tgtok::l_brace;
 115   case '}': return tgtok::r_brace;
 116   case '(': return tgtok::l_paren;
 117   case ')': return tgtok::r_paren;
 118   case '=': return tgtok::equal;
 119   case '?': return tgtok::question;
 120
 121   case 0:
 122   case ' ':
 123   case '\t':
 124   case '\n':
 125   case '\r':
 126     // Ignore whitespace.
 127     return LexToken();
 128   case '/':
 129     // If this is the start of a // comment, skip until the end of the line or
 130     // the end of the buffer.
 131     if (*CurPtr == '/')
 132       SkipBCPLComment();
 133     else if (*CurPtr == '*') {
 134       if (SkipCComment())
 135         return tgtok::Error;
 136     } else // Otherwise, this is an error.
 137       return ReturnError(TokStart, "Unexpected character");
 138     return LexToken();
 139   case '-': case '+':
 140   case '0': case '1': case '2': case '3': case '4': case '5': case '6':
 141   case '7': case '8': case '9':
 142     return LexNumber();
 143   case '"': return LexString();
 144   case '$': return LexVarName();
 145   case '[': return LexBracket();
 146   case '!': return LexExclaim();
 147   }
 148 }
 149
 150 /// LexString - Lex "[^"]*"
 151 tgtok::TokKind TGLexer::LexString() {
 152   const char *StrStart = CurPtr;
 153
 154   CurStrVal = "";
 155
 156   while (*CurPtr != '"') {
 157     // If we hit the end of the buffer, report an error.
 158     if (*CurPtr == 0 && CurPtr == CurBuf->getBufferEnd())
 159       return ReturnError(StrStart, "End of file in string literal");
 160
 161     if (*CurPtr == '\n' || *CurPtr == '\r')
 162       return ReturnError(StrStart, "End of line in string literal");
 163
 164     if (*CurPtr != '\\') {
 165       CurStrVal += *CurPtr++;
 166       continue;
 167     }
 168
 169     ++CurPtr;
 170
 171     switch (*CurPtr) {
 172     case '\\': case '\'': case '"':
 173       // These turn into their literal character.
 174       CurStrVal += *CurPtr++;
 175       break;
 176     case 't':
 177       CurStrVal += '\t';
 178       ++CurPtr;
 179       break;
 180     case 'n':
 181       CurStrVal += '\n';
 182       ++CurPtr;
 183       break;
 184
 185     case '\n':
 186     case '\r':
 187       return ReturnError(CurPtr, "escaped newlines not supported in tblgen");
 188
 189     // If we hit the end of the buffer, report an error.
 190     case '\0':
 191       if (CurPtr == CurBuf->getBufferEnd())
 192         return ReturnError(StrStart, "End of file in string literal");
 193       // FALL THROUGH
 194     default:
 195       return ReturnError(CurPtr, "invalid escape in string literal");
 196     }
 197   }
 198
 199   ++CurPtr;
 200   return tgtok::StrVal;
 201 }
 202
 203 tgtok::TokKind TGLexer::LexVarName() {
 204   if (!isalpha(CurPtr[0]) && CurPtr[0] != '_')
 205     return ReturnError(TokStart, "Invalid variable name");
 206
 207   // Otherwise, we're ok, consume the rest of the characters.
 208   const char *VarNameStart = CurPtr++;
 209
 210   while (isalpha(*CurPtr) || isdigit(*CurPtr) || *CurPtr == '_')
 211     ++CurPtr;
 212
 213   CurStrVal.assign(VarNameStart, CurPtr);
 214   return tgtok::VarName;
 215 }
 216
 217
 218 tgtok::TokKind TGLexer::LexIdentifier() {
 219   // The first letter is [a-zA-Z_].
 220   const char *IdentStart = TokStart;
 221
 222   // Match the rest of the identifier regex: [0-9a-zA-Z_]*
 223   while (isalpha(*CurPtr) || isdigit(*CurPtr) || *CurPtr == '_'
 224          || *CurPtr == '#') {
 225     // If this contains a '#', make sure it's value
 226     if (*CurPtr == '#') {
 227       if (strncmp(CurPtr, "#NAME#", 6) != 0) {
 228         return tgtok::Error;
 229       }
 230       CurPtr += 6;
 231     }
 232     else {
 233       ++CurPtr;
 234     }
 235   }
 236
 237
 238   // Check to see if this identifier is a keyword.
 239   unsigned Len = CurPtr-IdentStart;
 240
 241   if (Len == 3 && !memcmp(IdentStart, "int", 3)) return tgtok::Int;
 242   if (Len == 3 && !memcmp(IdentStart, "bit", 3)) return tgtok::Bit;
 243   if (Len == 4 && !memcmp(IdentStart, "bits", 4)) return tgtok::Bits;
 244   if (Len == 6 && !memcmp(IdentStart, "string", 6)) return tgtok::String;
 245   if (Len == 4 && !memcmp(IdentStart, "list", 4)) return tgtok::List;
 246   if (Len == 4 && !memcmp(IdentStart, "code", 4)) return tgtok::Code;
 247   if (Len == 3 && !memcmp(IdentStart, "dag", 3)) return tgtok::Dag;
 248
 249   if (Len == 5 && !memcmp(IdentStart, "class", 5)) return tgtok::Class;
 250   if (Len == 3 && !memcmp(IdentStart, "def", 3)) return tgtok::Def;
 251   if (Len == 4 && !memcmp(IdentStart, "defm", 4)) return tgtok::Defm;
 252   if (Len == 10 && !memcmp(IdentStart, "multiclass", 10))
 253     return tgtok::MultiClass;
 254   if (Len == 5 && !memcmp(IdentStart, "field", 5)) return tgtok::Field;
 255   if (Len == 3 && !memcmp(IdentStart, "let", 3)) return tgtok::Let;
 256   if (Len == 2 && !memcmp(IdentStart, "in", 2)) return tgtok::In;
 257
 258   if (Len == 7 && !memcmp(IdentStart, "include", 7)) {
 259     if (LexInclude()) return tgtok::Error;
 260     return Lex();
 261   }
 262
 263   CurStrVal.assign(IdentStart, CurPtr);
 264   return tgtok::Id;
 265 }
 266
 267 /// LexInclude - We just read the "include" token.  Get the string token that
 268 /// comes next and enter the include.
 269 bool TGLexer::LexInclude() {
 270   // The token after the include must be a string.
 271   tgtok::TokKind Tok = LexToken();
 272   if (Tok == tgtok::Error) return true;
 273   if (Tok != tgtok::StrVal) {
 274     PrintError(getLoc(), "Expected filename after include");
 275     return true;
 276   }
 277
 278   // Get the string.
 279   std::string Filename = CurStrVal;
 280
 281   // Try to find the file.
 282   MemoryBuffer *NewBuf = MemoryBuffer::getFile(Filename.c_str());
 283
 284   // If the file didn't exist directly, see if it's in an include path.
 285   for (unsigned i = 0, e = IncludeDirectories.size(); i != e && !NewBuf; ++i) {
 286     std::string IncFile = IncludeDirectories[i] + "/" + Filename;
 287     NewBuf = MemoryBuffer::getFile(IncFile.c_str());
 288   }
 289
 290   if (NewBuf == 0) {
 291     PrintError(getLoc(), "Could not find include file '" + Filename + "'");
 292     return true;
 293   }
 294
 295   // Save the line number and lex buffer of the includer.
 296   CurBuffer = SrcMgr.AddNewSourceBuffer(NewBuf, TGLoc::getFromPointer(CurPtr));
 297
 298   CurBuf = NewBuf;
 299   CurPtr = CurBuf->getBufferStart();
 300   return false;
 301 }
 302
 303 void TGLexer::SkipBCPLComment() {
 304   ++CurPtr;  // skip the second slash.
 305   while (1) {
 306     switch (*CurPtr) {
 307     case '\n':
 308     case '\r':
 309       return;  // Newline is end of comment.
 310     case 0:
 311       // If this is the end of the buffer, end the comment.
 312       if (CurPtr == CurBuf->getBufferEnd())
 313         return;
 314       break;
 315     }
 316     // Otherwise, skip the character.
 317     ++CurPtr;
 318   }
 319 }
 320
 321 /// SkipCComment - This skips C-style /**/ comments.  The only difference from C
 322 /// is that we allow nesting.
 323 bool TGLexer::SkipCComment() {
 324   ++CurPtr;  // skip the star.
 325   unsigned CommentDepth = 1;
 326
 327   while (1) {
 328     int CurChar = getNextChar();
 329     switch (CurChar) {
 330     case EOF:
 331       PrintError(TokStart, "Unterminated comment!");
 332       return true;
 333     case '*':
 334       // End of the comment?
 335       if (CurPtr[0] != '/') break;
 336
 337       ++CurPtr;   // End the */.
 338       if (--CommentDepth == 0)
 339         return false;
 340       break;
 341     case '/':
 342       // Start of a nested comment?
 343       if (CurPtr[0] != '*') break;
 344       ++CurPtr;
 345       ++CommentDepth;
 346       break;
 347     }
 348   }
 349 }
 350
 351 /// LexNumber - Lex:
 352 ///    [-+]?[0-9]+
 353 ///    0x[0-9a-fA-F]+
 354 ///    0b[01]+
 355 tgtok::TokKind TGLexer::LexNumber() {
 356   if (CurPtr[-1] == '0') {
 357     if (CurPtr[0] == 'x') {
 358       ++CurPtr;
 359       const char *NumStart = CurPtr;
 360       while (isxdigit(CurPtr[0]))
 361         ++CurPtr;
 362
 363       // Requires at least one hex digit.
 364       if (CurPtr == NumStart)
 365         return ReturnError(CurPtr-2, "Invalid hexadecimal number");
 366
 367       errno = 0;
 368       CurIntVal = strtoll(NumStart, 0, 16);
 369       if (errno == EINVAL)
 370         return ReturnError(CurPtr-2, "Invalid hexadecimal number");
 371       if (errno == ERANGE) {
 372         errno = 0;
 373         CurIntVal = (int64_t)strtoull(NumStart, 0, 16);
 374         if (errno == EINVAL)
 375           return ReturnError(CurPtr-2, "Invalid hexadecimal number");
 376         if (errno == ERANGE)
 377           return ReturnError(CurPtr-2, "Hexadecimal number out of range");
 378       }
 379       return tgtok::IntVal;
 380     } else if (CurPtr[0] == 'b') {
 381       ++CurPtr;
 382       const char *NumStart = CurPtr;
 383       while (CurPtr[0] == '0' || CurPtr[0] == '1')
 384         ++CurPtr;
 385
 386       // Requires at least one binary digit.
 387       if (CurPtr == NumStart)
 388         return ReturnError(CurPtr-2, "Invalid binary number");
 389       CurIntVal = strtoll(NumStart, 0, 2);
 390       return tgtok::IntVal;
 391     }
 392   }
 393
 394   // Check for a sign without a digit.
 395   if (!isdigit(CurPtr[0])) {
 396     if (CurPtr[-1] == '-')
 397       return tgtok::minus;
 398     else if (CurPtr[-1] == '+')
 399       return tgtok::plus;
 400   }
 401
 402   while (isdigit(CurPtr[0]))
 403     ++CurPtr;
 404   CurIntVal = strtoll(TokStart, 0, 10);
 405   return tgtok::IntVal;
 406 }
 407
 408 /// LexBracket - We just read '['.  If this is a code block, return it,
 409 /// otherwise return the bracket.  Match: '[' and '[{ ( [^}]+ | }[^]] )* }]'
 410 tgtok::TokKind TGLexer::LexBracket() {
 411   if (CurPtr[0] != '{')
 412     return tgtok::l_square;
 413   ++CurPtr;
 414   const char *CodeStart = CurPtr;
 415   while (1) {
 416     int Char = getNextChar();
 417     if (Char == EOF) break;
 418
 419     if (Char != '}') continue;
 420
 421     Char = getNextChar();
 422     if (Char == EOF) break;
 423     if (Char == ']') {
 424       CurStrVal.assign(CodeStart, CurPtr-2);
 425       return tgtok::CodeFragment;
 426     }
 427   }
 428
 429   return ReturnError(CodeStart-2, "Unterminated Code Block");
 430 }
 431
 432 /// LexExclaim - Lex '!' and '![a-zA-Z]+'.
 433 tgtok::TokKind TGLexer::LexExclaim() {
 434   if (!isalpha(*CurPtr))
 435     return ReturnError(CurPtr-1, "Invalid \"!operator\"");
 436
 437   const char *Start = CurPtr++;
 438   while (isalpha(*CurPtr))
 439     ++CurPtr;
 440
 441   // Check to see which operator this is.
 442   unsigned Len = CurPtr-Start;
 443
 444   if (Len == 3  && !memcmp(Start, "con", 3)) return tgtok::XConcat;
 445   if (Len == 3  && !memcmp(Start, "sra", 3)) return tgtok::XSRA;
 446   if (Len == 3  && !memcmp(Start, "srl", 3)) return tgtok::XSRL;
 447   if (Len == 3  && !memcmp(Start, "shl", 3)) return tgtok::XSHL;
 448   if (Len == 9  && !memcmp(Start, "strconcat", 9))   return tgtok::XStrConcat;
 449   if (Len == 10 && !memcmp(Start, "nameconcat", 10)) return tgtok::XNameConcat;
 450   if (Len == 5 && !memcmp(Start, "subst", 5)) return tgtok::XSubst;
 451   if (Len == 7 && !memcmp(Start, "foreach", 7)) return tgtok::XForEach;
 452   if (Len == 4 && !memcmp(Start, "cast", 4)) return tgtok::XCast;
 453   if (Len == 3 && !memcmp(Start, "car", 3)) return tgtok::XCar;
 454   if (Len == 3 && !memcmp(Start, "cdr", 3)) return tgtok::XCdr;
 455   if (Len == 4 && !memcmp(Start, "null", 4)) return tgtok::XNull;
 456   if (Len == 2 && !memcmp(Start, "if", 2)) return tgtok::XIf;
 457
 458   return ReturnError(Start-1, "Unknown operator");
 459 }
 460