beagled/Lucene.Net/Analysis/Standard/StandardFilter.cs

   1 /*
   2  * Copyright 2004 The Apache Software Foundation
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 using System;
  17 using Lucene.Net.Analysis;
  18 namespace Lucene.Net.Analysis.Standard
  19 {
  20
  21         /// <summary>Normalizes tokens extracted with {@link StandardTokenizer}. </summary>
  22
  23         public sealed class StandardFilter : TokenFilter
  24         {
  25
  26
  27                 /// <summary>Construct filtering <i>in</i>. </summary>
  28                 public StandardFilter(TokenStream in_Renamed) : base(in_Renamed)
  29                 {
  30                 }
  31
  32                 private static readonly System.String APOSTROPHE_TYPE = Lucene.Net.Analysis.Standard.StandardTokenizerConstants.tokenImage[Lucene.Net.Analysis.Standard.StandardTokenizerConstants.APOSTROPHE];
  33                 private static readonly System.String ACRONYM_TYPE = Lucene.Net.Analysis.Standard.StandardTokenizerConstants.tokenImage[Lucene.Net.Analysis.Standard.StandardTokenizerConstants.ACRONYM];
  34
  35                 /// <summary>Returns the next token in the stream, or null at EOS.
  36                 /// <p>Removes <tt>'s</tt> from the end of words.
  37                 /// <p>Removes dots from acronyms.
  38                 /// </summary>
  39                 public override Lucene.Net.Analysis.Token Next()
  40                 {
  41                         Lucene.Net.Analysis.Token t = input.Next();
  42
  43                         if (t == null)
  44                                 return null;
  45
  46                         System.String text = t.TermText();
  47                         System.String type = t.Type();
  48
  49                         if ((System.Object) type == (System.Object) APOSTROPHE_TYPE && (text.EndsWith("'s") || text.EndsWith("'S")))
  50                         {
  51                                 return new Lucene.Net.Analysis.Token(text.Substring(0, (text.Length - 2) - (0)), t.StartOffset(), t.EndOffset(), type);
  52                         }
  53                         else if ((System.Object) type == (System.Object) ACRONYM_TYPE)
  54                         {
  55                                 // remove dots
  56                                 System.Text.StringBuilder trimmed = new System.Text.StringBuilder();
  57                                 for (int i = 0; i < text.Length; i++)
  58                                 {
  59                                         char c = text[i];
  60                                         if (c != '.')
  61                                                 trimmed.Append(c);
  62                                 }
  63                                 return new Lucene.Net.Analysis.Token(trimmed.ToString(), t.StartOffset(), t.EndOffset(), type);
  64                         }
  65                         else
  66                         {
  67                                 return t;
  68                         }
  69                 }
  70         }
  71 }