search/Zend/Search/Lucene/Analysis/Analyzer.php

   1 <?php
   2 /**
   3  * Zend Framework
   4  *
   5  * LICENSE
   6  *
   7  * This source file is subject to the new BSD license that is bundled
   8  * with this package in the file LICENSE.txt.
   9  * It is also available through the world-wide-web at this URL:
  10  * http://framework.zend.com/license/new-bsd
  11  * If you did not receive a copy of the license and are unable to
  12  * obtain it through the world-wide-web, please send an email
  13  * to license@zend.com so we can send you a copy immediately.
  14  *
  15  * @category   Zend
  16  * @package    Zend_Search_Lucene
  17  * @subpackage Analysis
  18  * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  19  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  20  */
  21
  22
  23 /** Zend_Search_Lucene_Analysis_Token */
  24 require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Token.php';
  25
  26 /** Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8 */
  27 require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8.php';
  28
  29 /** Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num */
  30 require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8Num.php';
  31
  32 /** Zend_Search_Lucene_Analysis_Analyzer_Common_Text */
  33 require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php';
  34
  35 /** Zend_Search_Lucene_Analysis_Analyzer_Common_Text_CaseInsensitive */
  36 require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text/CaseInsensitive.php';
  37
  38 /** Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum */
  39 require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum.php';
  40
  41 /** Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum_CaseInsensitive */
  42 require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum/CaseInsensitive.php';
  43
  44 /** Zend_Search_Lucene_Analysis_TokenFilter_StopWords */
  45 require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter/StopWords.php';
  46
  47 /** Zend_Search_Lucene_Analysis_TokenFilter_ShortWords */
  48 require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter/ShortWords.php';
  49
  50
  51 /**
  52  * An Analyzer is used to analyze text.
  53  * It thus represents a policy for extracting index terms from text.
  54  *
  55  * Note:
  56  * Lucene Java implementation is oriented to streams. It provides effective work
  57  * with a huge documents (more then 20Mb).
  58  * But engine itself is not oriented such documents.
  59  * Thus Zend_Search_Lucene analysis API works with data strings and sets (arrays).
  60  *
  61  * @category   Zend
  62  * @package    Zend_Search_Lucene
  63  * @subpackage Analysis
  64  * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  65  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  66  */
  67
  68 abstract class Zend_Search_Lucene_Analysis_Analyzer
  69 {
  70     /**
  71      * The Analyzer implementation used by default.
  72      *
  73      * @var Zend_Search_Lucene_Analysis_Analyzer
  74      */
  75     private static $_defaultImpl;
  76
  77     /**
  78      * Input string
  79      *
  80      * @var string
  81      */
  82     protected $_input = null;
  83
  84     /**
  85      * Input string encoding
  86      *
  87      * @var string
  88      */
  89     protected $_encoding = '';
  90
  91     /**
  92      * Tokenize text to a terms
  93      * Returns array of Zend_Search_Lucene_Analysis_Token objects
  94      *
  95      * Tokens are returned in UTF-8 (internal Zend_Search_Lucene encoding)
  96      *
  97      * @param string $data
  98      * @return array
  99      */
 100     public function tokenize($data, $encoding = '')
 101     {
 102         $this->setInput($data, $encoding);
 103
 104         $tokenList = array();
 105         while (($nextToken = $this->nextToken()) !== null) {
 106             $tokenList[] = $nextToken;
 107         }
 108
 109         return $tokenList;
 110     }
 111
 112
 113     /**
 114      * Tokenization stream API
 115      * Set input
 116      *
 117      * @param string $data
 118      */
 119     public function setInput($data, $encoding = '')
 120     {
 121         $this->_input    = $data;
 122         $this->_encoding = $encoding;
 123         $this->reset();
 124     }
 125
 126     /**
 127      * Reset token stream
 128      */
 129     abstract public function reset();
 130
 131     /**
 132      * Tokenization stream API
 133      * Get next token
 134      * Returns null at the end of stream
 135      *
 136      * Tokens are returned in UTF-8 (internal Zend_Search_Lucene encoding)
 137      *
 138      * @return Zend_Search_Lucene_Analysis_Token|null
 139      */
 140     abstract public function nextToken();
 141
 142
 143
 144
 145     /**
 146      * Set the default Analyzer implementation used by indexing code.
 147      *
 148      * @param Zend_Search_Lucene_Analysis_Analyzer $similarity
 149      */
 150     public static function setDefault(Zend_Search_Lucene_Analysis_Analyzer $analyzer)
 151     {
 152         self::$_defaultImpl = $analyzer;
 153     }
 154
 155
 156     /**
 157      * Return the default Analyzer implementation used by indexing code.
 158      *
 159      * @return Zend_Search_Lucene_Analysis_Analyzer
 160      */
 161     public static function getDefault()
 162     {
 163         if (!self::$_defaultImpl instanceof Zend_Search_Lucene_Analysis_Analyzer) {
 164             self::$_defaultImpl = new Zend_Search_Lucene_Analysis_Analyzer_Common_Text_CaseInsensitive();
 165         }
 166
 167         return self::$_defaultImpl;
 168     }
 169 }
 170