runtime/nepomuk/services/storage/cluceneanalyzer.cpp

   1 /*
   2  * Modified version of StandardAnalyzer.cpp for Nepomuk mostly to optimize for filename indexing
   3  * Copyright (C) 2008 Sebastian Trueg <trueg@kde.org>
   4  *
   5  * Based on StandardAnalyzer.cpp from the CLucene package.
   6  * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team
   7  *
   8  * This library is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Library General Public
  10  * License as published by the Free Software Foundation; either
  11  * version 2 of the License, or (at your option) any later version.
  12  *
  13  * This library is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * Library General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Library General Public License
  19  * along with this library; see the file COPYING.LIB.  If not, write to
  20  * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  21  * Boston, MA 02110-1301, USA.
  22  */
  23
  24 #include "cluceneanalyzer.h"
  25 #include "clucenetokenizer.h"
  26 #include "clucenefilter.h"
  27
  28 #include <CLucene/StdHeader.h>
  29 #include <CLucene/util/VoidMap.h>
  30 #include <CLucene/util/Reader.h>
  31 #include <CLucene/analysis/AnalysisHeader.h>
  32 #include <CLucene/analysis/Analyzers.h>
  33
  34 CL_NS_USE(util)
  35 CL_NS_USE(analysis)
  36
  37 namespace Nepomuk {
  38
  39         CLuceneAnalyzer::CLuceneAnalyzer()
  40         : stopSet(false)
  41         {
  42         CL_NS(analysis)::StopFilter::fillStopTable( &stopSet,CL_NS(analysis)::StopAnalyzer::ENGLISH_STOP_WORDS);
  43         }
  44
  45         CLuceneAnalyzer::CLuceneAnalyzer( const TCHAR** stopWords):
  46                 stopSet(false)
  47         {
  48                 CL_NS(analysis)::StopFilter::fillStopTable( &stopSet,stopWords );
  49         }
  50
  51         CLuceneAnalyzer::~CLuceneAnalyzer()
  52     {
  53         }
  54
  55
  56         CL_NS(analysis)::TokenStream* CLuceneAnalyzer::tokenStream(const TCHAR* fieldName, Reader* reader)
  57         {
  58                 CL_NS(analysis)::TokenStream* ret = _CLNEW CLuceneTokenizer(reader);
  59                 ret = _CLNEW CLuceneFilter(ret,true);
  60                 ret = _CLNEW CL_NS(analysis)::LowerCaseFilter(ret,true);
  61                 ret = _CLNEW CL_NS(analysis)::StopFilter(ret,true, &stopSet);
  62                 return ret;
  63         }
  64 }