library/Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php

   1 <?php
   2 /**
   3  * Zend Framework
   4  *
   5  * LICENSE
   6  *
   7  * This source file is subject to the new BSD license that is bundled
   8  * with this package in the file LICENSE.txt.
   9  * It is also available through the world-wide-web at this URL:
  10  * http://framework.zend.com/license/new-bsd
  11  * If you did not receive a copy of the license and are unable to
  12  * obtain it through the world-wide-web, please send an email
  13  * to license@zend.com so we can send you a copy immediately.
  14  *
  15  * @category   Zend
  16  * @package    Zend_Search_Lucene
  17  * @subpackage Analysis
  18  * @copyright  Copyright (c) 2005-2009 Zend Technologies USA Inc. (http://www.zend.com)
  19  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  20  * @version    $Id: Text.php 16541 2009-07-07 06:59:03Z bkarwin $
  21  */
  22
  23
  24 /** Zend_Search_Lucene_Analysis_Analyzer_Common */
  25 require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common.php';
  26
  27
  28 /**
  29  * @category   Zend
  30  * @package    Zend_Search_Lucene
  31  * @subpackage Analysis
  32  * @copyright  Copyright (c) 2005-2009 Zend Technologies USA Inc. (http://www.zend.com)
  33  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  34  */
  35
  36 class Zend_Search_Lucene_Analysis_Analyzer_Common_Text extends Zend_Search_Lucene_Analysis_Analyzer_Common
  37 {
  38     /**
  39      * Current position in a stream
  40      *
  41      * @var integer
  42      */
  43     private $_position;
  44
  45     /**
  46      * Reset token stream
  47      */
  48     public function reset()
  49     {
  50         $this->_position = 0;
  51
  52         if ($this->_input === null) {
  53             return;
  54         }
  55
  56         // convert input into ascii
  57         if (PHP_OS != 'AIX') {
  58             $this->_input = iconv($this->_encoding, 'ASCII//TRANSLIT', $this->_input);
  59         }
  60         $this->_encoding = 'ASCII';
  61     }
  62
  63     /**
  64      * Tokenization stream API
  65      * Get next token
  66      * Returns null at the end of stream
  67      *
  68      * @return Zend_Search_Lucene_Analysis_Token|null
  69      */
  70     public function nextToken()
  71     {
  72         if ($this->_input === null) {
  73             return null;
  74         }
  75
  76
  77         do {
  78             if (! preg_match('/[a-zA-Z]+/', $this->_input, $match, PREG_OFFSET_CAPTURE, $this->_position)) {
  79                 // It covers both cases a) there are no matches (preg_match(...) === 0)
  80                 // b) error occured (preg_match(...) === FALSE)
  81                 return null;
  82             }
  83
  84             $str = $match[0][0];
  85             $pos = $match[0][1];
  86             $endpos = $pos + strlen($str);
  87
  88             $this->_position = $endpos;
  89
  90             $token = $this->normalize(new Zend_Search_Lucene_Analysis_Token($str, $pos, $endpos));
  91         } while ($token === null); // try again if token is skipped
  92
  93         return $token;
  94     }
  95 }
  96