search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php

   1 <?php
   2 /**
   3  * Zend Framework
   4  *
   5  * LICENSE
   6  *
   7  * This source file is subject to the new BSD license that is bundled
   8  * with this package in the file LICENSE.txt.
   9  * It is also available through the world-wide-web at this URL:
  10  * http://framework.zend.com/license/new-bsd
  11  * If you did not receive a copy of the license and are unable to
  12  * obtain it through the world-wide-web, please send an email
  13  * to license@zend.com so we can send you a copy immediately.
  14  *
  15  * @category   Zend
  16  * @package    Zend_Search_Lucene
  17  * @subpackage Analysis
  18  * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  19  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  20  */
  21
  22
  23 /** Zend_Search_Lucene_Analysis_Analyzer_Common */
  24 require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common.php';
  25
  26
  27 /**
  28  * @category   Zend
  29  * @package    Zend_Search_Lucene
  30  * @subpackage Analysis
  31  * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  32  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  33  */
  34
  35 class Zend_Search_Lucene_Analysis_Analyzer_Common_Text extends Zend_Search_Lucene_Analysis_Analyzer_Common
  36 {
  37     /**
  38      * Current position in a stream
  39      *
  40      * @var integer
  41      */
  42     private $_position;
  43
  44     /**
  45      * Reset token stream
  46      */
  47     public function reset()
  48     {
  49         $this->_position = 0;
  50
  51         if ($this->_input === null) {
  52             return;
  53         }
  54
  55         // convert input into ascii
  56         $this->_input = iconv($this->_encoding, 'ASCII//TRANSLIT', $this->_input);
  57         $this->_encoding = 'ASCII';
  58     }
  59
  60     /**
  61      * Tokenization stream API
  62      * Get next token
  63      * Returns null at the end of stream
  64      *
  65      * @return Zend_Search_Lucene_Analysis_Token|null
  66      */
  67     public function nextToken()
  68     {
  69         if ($this->_input === null) {
  70             return null;
  71         }
  72
  73
  74         do {
  75             if (! preg_match('/[a-zA-Z]+/', $this->_input, $match, PREG_OFFSET_CAPTURE, $this->_position)) {
  76                 // It covers both cases a) there are no matches (preg_match(...) === 0)
  77                 // b) error occured (preg_match(...) === FALSE)
  78                 return null;
  79             }
  80
  81             $str = $match[0][0];
  82             $pos = $match[0][1];
  83             $endpos = $pos + strlen($str);
  84
  85             $this->_position = $endpos;
  86
  87             $token = $this->normalize(new Zend_Search_Lucene_Analysis_Token($str, $pos, $endpos));
  88         } while ($token === null); // try again if token is skipped
  89
  90         return $token;
  91     }
  92 }
  93