Lucene.Net/Analysis/Standard/Token.cs

   1 using System;
   2
   3 namespace Lucene.Net.Analysis.Standard
   4 {
   5         /// <summary>
   6         /// Describes the input token stream.
   7         /// </summary>
   8         public class Token
   9         {
  10                 /// <summary>
  11                 /// An integer that describes the kind of this token.  This numbering
  12                 /// system is determined by JavaCCParser, and a table of these numbers is
  13                 /// stored in the file ...Constants.cs.
  14                 /// </summary>
  15                 public int kind;
  16
  17                 /// <summary>
  18                 /// beginLine and beginColumn describe the position of the first character
  19                 /// of this token; endLine and endColumn describe the position of the
  20                 /// last character of this token.
  21                 /// </summary>
  22                 public int beginLine, beginColumn, endLine, endColumn;
  23
  24                 /// <summary>
  25                 /// The string image of the token.
  26                 /// </summary>
  27                 public String image;
  28
  29                 /// <summary>
  30                 /// A reference to the next regular (non-special) token from the input
  31                 /// stream.  If this is the last token from the input stream, or if the
  32                 /// token manager has not read tokens beyond this one, this field is
  33                 /// set to null.  This is true only if this token is also a regular
  34                 /// token.  Otherwise, see below for a description of the contents of
  35                 /// this field.
  36                 /// </summary>
  37                 public Token next;
  38
  39                 /// <summary>
  40                 /// This field is used to access special tokens that occur prior to this
  41                 /// token, but after the immediately preceding regular (non-special) token.
  42                 /// If there are no such special tokens, this field is set to null.
  43                 /// When there are more than one such special token, this field refers
  44                 /// to the last of these special tokens, which in turn refers to the next
  45                 /// previous special token through its specialToken field, and so on
  46                 /// until the first special token (whose specialToken field is null).
  47                 /// The next fields of special tokens refer to other special tokens that
  48                 /// immediately follow it (without an intervening regular token).  If there
  49                 /// is no such token, this field is null.
  50                 /// </summary>
  51                 public Token specialToken;
  52
  53                 /// <summary>
  54                 /// Returns the image.
  55                 /// </summary>
  56                 /// <returns></returns>
  57                 public override String ToString()
  58                 {
  59                         return image;
  60                 }
  61
  62                 /// <summary>
  63                 /// Returns a new Token object, by default. However, if you want, you
  64                 /// can create and return subclass objects based on the value of ofKind.
  65                 /// Simply add the cases to the switch for all those special cases.
  66                 /// For example, if you have a subclass of Token called IDToken that
  67                 /// you want to create if ofKind is ID, simlpy add something like :
  68                 ///
  69                 ///    case MyParserConstants.ID : return new IDToken();
  70                 ///
  71                 /// to the following switch statement. Then you can cast matchedToken
  72                 /// variable to the appropriate type and use it in your lexical actions.
  73                 /// </summary>
  74                 /// <param name="ofKind"></param>
  75                 /// <returns></returns>
  76                 public static Token NewToken(int ofKind)
  77                 {
  78                         switch(ofKind)
  79                         {
  80                                 default : return new Token();
  81                         }
  82                 }
  83         }
  84 }