includes/Tokenizer.php

   1 <?php
   2 class Tokenizer {
   3         /* private */ var $mText, $mPos, $mTextLength;
   4         /* private */ var $mCount, $mM, $mMPos; # What is M meant to stand for?
   5
   6         /* private */ function Tokenizer()
   7         {
   8                 $this->mPos=0;
   9         }
  10
  11         # factory function
  12         function newFromString( $s )
  13         {
  14                 $t = new Tokenizer();
  15                 $t->mText = $s;
  16                 $t->preParse();
  17                 $t->mTextLength = strlen( $s );
  18                 return $t;
  19         }
  20
  21         function preParse()
  22         {
  23                 $this->mCount = preg_match_all( "/(\[\[|\]\]|\'\'\'\'\'|\'\'\'|\'\')/",
  24                                                 $this->mText, $this->mM,
  25                                                 PREG_PATTERN_ORDER|PREG_OFFSET_CAPTURE);
  26                 $this->mMPos=0;
  27         }
  28
  29         function nextToken()
  30         {
  31                 $token = $this->previewToken();
  32                 if ( $token ) {
  33                         if ( $token["type"] == "text" ) {
  34                                 $this->mPos = $token["mPos"];
  35                         } else {
  36                                 $this->mMPos = $token["mMPos"];
  37                                 $this->mPos = $token["mPos"];
  38                         }
  39                 }
  40                 return $token;
  41         }
  42
  43
  44         function previewToken()
  45         {
  46                 if ( $this->mMPos <= $this->mCount  ) {
  47                         $token["pos"] = $this->mPos;
  48                         if ( $this->mPos < $this->mM[0][$this->mMPos][1] ) {
  49                                 $token["type"] = "text";
  50                                 $token["text"] = substr( $this->mText, $this->mPos,
  51                                                          $this->mM[0][$this->mMPos][1] - $this->mPos );
  52                                 $token["mPos"] = $this->mM[0][$this->mMPos][1];
  53                         } else {
  54                                 $token["type"] = $this->mM[0][$this->mMPos][0];
  55                                 $token["mPos"] = $this->mPos + strlen($token["type"]);
  56                                 $token["mMPos"] = $this->mMPos + 1;
  57                         }
  58                 } elseif ( $this->mPos < $this->mTextLength ) {
  59                         $token["type"] = "text";
  60                         $token["text"] = substr( $this->mText, $this->mPos );
  61                         $token["mPos"] = $this->mTextLength;
  62                 } else {
  63                         $token = FALSE;
  64                 }
  65                 return $token;
  66         }
  67
  68
  69 }
  70