tools/generators/Tokenizer.cs

   1 /*
   2  * Tokenizer.cs.
   3  *
   4  * Contact:
   5  *   Moonlight List (moonlight-list@lists.ximian.com)
   6  *
   7  * Copyright 2008 Novell, Inc. (http://www.novell.com)
   8  *
   9  * See the LICENSE file included with the distribution for details.
  10  *
  11  */
  12
  13 using System;
  14 using System.Text;
  15 using System.Collections.Generic;
  16 using System.IO;
  17
  18 public enum Token2Type {
  19         Identifier,
  20         Literal,
  21         Punctuation,
  22         CommentProperty
  23 }
  24
  25 public class Token2 {
  26         public Token2Type type;
  27         public string value;
  28
  29         public Token2 (Token2Type type, string value)
  30         {
  31                 //Console.WriteLine ("Token2: {0}, '{1}'", type, value);
  32                 this.type = type;
  33                 this.value = value;
  34         }
  35         public Token2 (Token2Type type, char value)
  36         {
  37                 //Console.WriteLine ("Token2: {0}, '{1}'", type, value);
  38                 this.type = type;
  39                 this.value = value.ToString ();
  40         }
  41
  42         public override string ToString ()
  43         {
  44                 return string.Format ("{0} '{1}'", type, value);
  45         }
  46
  47 }
  48
  49 public class Tokenizer {
  50         private Queue <string> files;
  51         private StreamReader current_stream;
  52         private Token2 current_token;
  53         private string current_file;
  54         private List <char> chars = new List <char> ();
  55         private int current_line;
  56
  57         public int CurrentLine {
  58                 get { return current_line; }
  59         }
  60
  61         public Token2 CurrentToken {
  62                 get { return current_token; }
  63         }
  64
  65         public string CurrentFile {
  66                 get { return current_file; }
  67         }
  68
  69         public Tokenizer (string [] files)
  70         {
  71                 this.files = new Queue <string> (files);
  72         }
  73
  74         public char ReadNextChar ()
  75         {
  76                 StringBuilder line = new StringBuilder ();
  77
  78                 do {
  79                         line.Length = 0;
  80
  81                         while (current_stream == null || current_stream.EndOfStream) {
  82                                 if (current_stream != null) {
  83                                         current_stream.Close ();
  84                                         current_stream = null;
  85                                 }
  86                                 if (files.Count == 0)
  87                                         return char.MinValue;;
  88                                 current_file = files.Dequeue ();
  89                                 current_stream = new StreamReader (current_file);
  90                                 current_line = 0;
  91                                 //Console.WriteLine ("Parsing {0}...", current_file);
  92                         }
  93
  94                         do {
  95                                 line.Append (current_stream.ReadLine ());
  96                                 current_line++;
  97
  98                                 if (line == null || line.Length == 0)
  99                                         break;
 100
 101                                 if (line [line.Length - 1] != '\\')
 102                                         break;
 103
 104                                 line.Length--;
 105                         } while (true);
 106
 107                         //Console.WriteLine ("ReadNextChar: Read line: '{0}'", line);
 108
 109                         if (line.Length == 0) {
 110                                 //Console.WriteLine ("ReadNextChar: Empty line");
 111                                 continue;
 112                         }
 113
 114                         if (line [0] == '#') {
 115                                 //Console.WriteLine ("ReadNextChar: Skipped preprocessor line: '{0}'", line);
 116                                 continue;
 117                         }
 118                         break;
 119                 } while (true);
 120
 121                 for (int i = 0; i < line.Length; i++)
 122                         chars.Add (line [i]);
 123                 chars.Add ('\n');
 124                 return '\n';
 125         }
 126
 127         public char GetNextChar ()
 128         {
 129                 char result;
 130
 131                 if (chars.Count != 0) {
 132                         result = chars [0];
 133                         chars.RemoveAt (0);
 134                         //Console.WriteLine ("GetNextChar (): popped '{0}'", result);
 135                         return result;
 136                 }
 137
 138                 result = ReadNextChar ();
 139                 //Console.WriteLine ("GetNextChar (): read '{0}'", result);
 140
 141                 // All newlines are passed as only one '\n' to the rest of the code
 142                 if (result == '\n') {
 143                         if (PeekChar (1) == '\r')
 144                                 ReadNextChar ();
 145                 } else if (result == '\r') {
 146                         if (PeekChar (1) == '\n')
 147                                 result = ReadNextChar ();
 148                         else
 149                                 result = '\n';
 150                 }
 151                 return result;
 152         }
 153
 154         public void PutBackChar (char v)
 155         {
 156                 chars.Add (v);
 157         }
 158
 159         public char PeekChar (int positions)
 160         {
 161                 while (chars.Count < positions) {
 162                         char c = ReadNextChar ();
 163                         if (c == char.MinValue)
 164                                 return c;
 165                         PutBackChar (c);
 166                 }
 167
 168                 //Console.WriteLine ("PeekChar ({0}): peeked '{1}'", positions, chars [positions - 1]);
 169                 return chars [positions - 1];
 170         }
 171
 172         public bool Advance (bool throw_on_end)
 173         {
 174                 bool result;
 175                 try {
 176                         do {
 177                                 result = AdvanceInternal (throw_on_end);
 178                                 if (CurrentToken.value == "G_BEGIN_DECLS") {
 179                                         continue;
 180                                 } else if (CurrentToken.value == "G_END_DECLS") {
 181                                         continue;
 182                                 }
 183                                 break;
 184                         } while (true);
 185                         return result;
 186                 } catch (Exception ex) {
 187                         throw new Exception (string.Format ("{0}({1}): {2}", current_file, current_line, ex.Message), ex);
 188                 }
 189         }
 190
 191         private bool AdvanceInternal (bool throw_on_end)
 192         {
 193                 char current;
 194                 StringBuilder builder = new StringBuilder ();
 195
 196          startagain:
 197
 198                 builder.Length = 0;
 199
 200                 do {
 201                         current = GetNextChar ();
 202                 } while (IsWhiteSpace (current));
 203
 204                 if (current == '/') {
 205                         current = GetNextChar ();
 206                         if (current == '*') { // Found a comment
 207                                 // Skip any whitespace
 208                                 do {
 209                                         current = GetNextChar ();
 210                                 } while (IsWhiteSpace (current));
 211
 212                                 // Check for a comment property
 213                                 if (current == '@') {
 214                                         current = GetNextChar ();
 215                                         while (true) {
 216                                                 if (current == char.MinValue)
 217                                                         throw new Exception ("Unexpected end of code.");
 218                                                 if (current == '*' && PeekChar (1) == '/')
 219                                                         break;
 220                                                 builder.Append (current);
 221                                                 current = GetNextChar ();
 222                                         }
 223                                         while (builder.Length > 0 && builder [builder.Length - 1] == ' ')
 224                                                 builder.Length--;
 225
 226                                         if (builder.Length == 0)
 227                                                 throw new Exception ("Empty comment property.");
 228                                 }
 229
 230                                 while (true) {
 231                                         if (current == char.MinValue)
 232                                                 throw new Exception ("Unexpected end of code.");
 233                                         if (current == '*' && PeekChar (1) == '/')
 234                                                 break;
 235
 236                                         current = GetNextChar ();
 237                                 }
 238
 239                                 if (current != '*')
 240                                         throw new Exception (string.Format ("Expected '*', got '{0}'", current));
 241
 242                                 current = GetNextChar ();
 243                                 if (current != '/')
 244                                         throw new Exception (string.Format ("Expected '/', got '{0}'", current));
 245
 246                                 if (builder.Length != 0) {
 247                                         current_token = new Token2 (Token2Type.CommentProperty, builder.ToString ());
 248                                         return true;
 249                                 }
 250                                 // We've skipped the comment, start again
 251                                 goto startagain;
 252                         } else if (current == '/') { // Found a comment
 253                                 do {
 254                                         current = GetNextChar ();
 255                                 } while (current != '\r' && current != '\n' && current != char.MinValue);
 256                                 if (current == '\r') {
 257                                 } else if (current == '\n') {
 258                                 } else {
 259                                         throw new Exception ("Expected end of line.");
 260                                 }
 261                                 // We've skipped the comment, start again
 262                                 goto startagain;
 263                         } else {
 264                                 PutBackChar (current);
 265                                 current_token = new Token2 (Token2Type.Punctuation, "/");
 266                                 return true;
 267                         }
 268                 }
 269
 270                 if (IsPunctuation (current)) {
 271                         current_token = new Token2 (Token2Type.Punctuation, current);
 272                         return true;
 273                 }
 274
 275                 if (current == '\'') {
 276                         current_token = new Token2 (Token2Type.Punctuation, current);
 277                         return true;
 278                 }
 279
 280                 if (current == '"') {
 281                         do {
 282                                 current = GetNextChar ();
 283                                 if (current == '\\') {
 284                                         current = GetNextChar ();
 285                                         builder.Append (current); // We don't care much about special characters like \n, \t, etc.
 286                                 } else if (current != '"') {
 287                                         builder.Append (current);
 288                                 } else if (current == char.MinValue) {
 289                                         throw new Exception ("Unexpected end of code in string literal.");
 290                                 } else if (current == '"') {
 291                                         GetNextChar (); // Skip the "
 292                                         break;
 293                                 } else {
 294                                         throw new Exception (string.Format ("Got unexpected character: {0}", current));
 295                                 }
 296                         } while (true);
 297                         current_token = new Token2 (Token2Type.Literal, builder.ToString ());
 298                         return true;
 299                 }
 300
 301                 if (IsPunctuation (current)) {
 302                         current_token = new Token2 (Token2Type.Literal, current);
 303                         return true;
 304                 }
 305
 306                 if (IsIdentifier (current)) {
 307                         builder.Append (current);
 308                         while (IsIdentifier (PeekChar (1))) {
 309                                 builder.Append (GetNextChar ());
 310                         }
 311                         current_token = new Token2 (Token2Type.Identifier, builder.ToString ());
 312                         return true;
 313                 }
 314
 315                 if (current == char.MinValue) {
 316                         if (throw_on_end)
 317                                 throw new Exception ("Unexpected end of code");
 318                         return false;
 319                 }
 320
 321                 throw new Exception (string.Format ("Unexpected character: {0}", current));
 322         }
 323
 324         public void FindStartBrace ()
 325         {
 326                 while (CurrentToken.value != "{") {
 327                         Advance (true);
 328                 }
 329         }
 330
 331         public void SyncWithEndBrace ()
 332         {
 333                 int braces = 0;
 334
 335                 AcceptOrThrow (Token2Type.Punctuation, "{");
 336
 337                 do {
 338                         if (Accept (Token2Type.Punctuation, "{")) {
 339                                 braces++;
 340                         } else if (Accept (Token2Type.Punctuation, "}")) {
 341                                 if (braces == 0)
 342                                         return;
 343                                 braces--;
 344                         } else {
 345                                 Advance (true);
 346                         }
 347                 } while (true);
 348         }
 349
 350         public string GetIdentifier ()
 351         {
 352                 string result;
 353                 VerifyIdentifier ();
 354                 result = CurrentToken.value;
 355                 Advance (true);
 356                 return result;
 357         }
 358
 359         public void VerifyIdentifier ()
 360         {
 361                 VerifyType (Token2Type.Identifier);
 362         }
 363
 364         public void VerifyType (Token2Type type)
 365         {
 366                 if (CurrentToken.type != type)
 367                         throw new Exception (string.Format ("Expected {0}, got {1}", type, CurrentToken));
 368         }
 369
 370         public bool Accept (Token2Type type, string value)
 371         {
 372                 if (CurrentToken.type != type)
 373                         return false;
 374
 375                 if (CurrentToken.value != value)
 376                         return false;
 377
 378                 Advance (true);
 379                 return true;
 380         }
 381
 382         public void AcceptOrThrow (Token2Type type, string value)
 383         {
 384                 if (CurrentToken.type != type)
 385                         throw new Exception (string.Format ("Expected {0} '{2}', got {1}", type, CurrentToken, value));
 386
 387                 if (CurrentToken.value != value)
 388                         throw new Exception (string.Format ("Expected '{0}', not '{1}'", value, CurrentToken.value));
 389
 390                 Advance (true);
 391         }
 392
 393         public static bool IsIdentifier(string str)
 394         {
 395                 return IsIdentifier (str [0]);
 396         }
 397
 398         public static bool IsIdentifier (char c)
 399         {
 400                 return char.IsLetterOrDigit (c) || c == '_';
 401         }
 402
 403         public static bool IsPunctuation (char c)
 404         {
 405                 switch (c) {
 406                 case '~':
 407                 case '=':
 408                 case '<':
 409                 case '>':
 410                 case ':':
 411                 case ';':
 412                 case '!':
 413                 case '.':
 414                 case ',':
 415                 case '|':
 416                 case '^':
 417                 case '&':
 418                 case '{':
 419                 case '}':
 420                 case '(':
 421                 case ')':
 422                 case '[':
 423                 case ']':
 424                 case '*':
 425                 case '-':
 426                 case '+':
 427                 case '?':
 428                 case '\\':
 429                 case '/':
 430                         return true;
 431                 default:
 432                         return false;
 433                 }
 434         }
 435
 436         public static bool IsWhiteSpace (char c)
 437         {
 438                 return char.IsWhiteSpace (c);
 439         }
 440 }