includes/Tokenizer.php

   1 <?php
   2 class Tokenizer {
   3         /* private */ var $mText,               # Text to be processed by the tokenizer
   4                           $mPos,                # current position of tokenizer in text
   5                           $mTextLength,         # Length of $mText
   6                           $mCount,              # token count, computed in preParse
   7                           $mMatch,              # matches of tokenizer regex, computed in preParse
   8                           $mMatchPos;           # current token position of tokenizer. Each match can
   9                                                 # be up to two tokens: A matched token and the text after it.
  10
  11         /* private */ function Tokenizer()
  12         {
  13                 $this->mPos=0;
  14         }
  15
  16         # factory function
  17         function newFromString( $s )
  18         {
  19                 $t = new Tokenizer();
  20                 $t->mText = $s;
  21                 $t->preParse();
  22                 $t->mTextLength = strlen( $s );
  23                 return $t;
  24         }
  25
  26         function preParse()
  27         {
  28                 $this->mCount = preg_match_all( "/(\[\[|\]\]|\'\'\'\'\'|\'\'\'|\'\')/",
  29                                                 $this->mText, $this->mMatch,
  30                                                 PREG_PATTERN_ORDER|PREG_OFFSET_CAPTURE);
  31                 $this->mMatchPos=0;
  32         }
  33
  34         function nextToken()
  35         {
  36                 $token = $this->previewToken();
  37                 if ( $token ) {
  38                         if ( $token["type"] == "text" ) {
  39                                 $this->mPos = $token["mPos"];
  40                         } else {
  41                                 $this->mMatchPos = $token["mMatchPos"];
  42                                 $this->mPos = $token["mPos"];
  43                         }
  44                 }
  45                 return $token;
  46         }
  47
  48
  49         function previewToken()
  50         {
  51                 if ( $this->mMatchPos <= $this->mCount  ) {
  52                         $token["pos"] = $this->mPos;
  53                         if ( $this->mPos < $this->mMatch[0][$this->mMatchPos][1] ) {
  54                                 $token["type"] = "text";
  55                                 $token["text"] = substr( $this->mText, $this->mPos,
  56                                                          $this->mMatch[0][$this->mMatchPos][1] - $this->mPos );
  57                                 $token["mPos"] = $this->mMatch[0][$this->mMatchPos][1];
  58                         } else {
  59                                 $token["type"] = $this->mMatch[0][$this->mMatchPos][0];
  60                                 $token["mPos"] = $this->mPos + strlen($token["type"]);
  61                                 $token["mMatchPos"] = $this->mMatchPos + 1;
  62                         }
  63                 } elseif ( $this->mPos < $this->mTextLength ) {
  64                         $token["type"] = "text";
  65                         $token["text"] = substr( $this->mText, $this->mPos );
  66                         $token["mPos"] = $this->mTextLength;
  67                 } else {
  68                         $token = FALSE;
  69                 }
  70                 return $token;
  71         }
  72
  73
  74 }
  75