includes/Tokenizer.php

   1 <?php
   2 class Tokenizer {
   3         /* private */ var $mText,               # Text to be processed by the tokenizer
   4                           $mPos,                # current position of tokenizer in text
   5                           $mTextLength,         # Length of $mText
   6                           $mCount,              # token count, computed in preParse
   7                           $mMatch,              # matches of tokenizer regex, computed in preParse
   8                           $mMatchPos;           # current token position of tokenizer. Each match can
   9                                                 # be up to two tokens: A matched token and the text after it.
  10
  11         /* private */ function Tokenizer()
  12         {
  13                 $this->mPos=0;
  14         }
  15
  16         # factory function
  17         function newFromString( $s )
  18         {
  19                 $t = new Tokenizer();
  20                 $t->mText = $s;
  21                 $t->preParse();
  22                 $t->mTextLength = strlen( $s );
  23                 return $t;
  24         }
  25
  26         function preParse()
  27         {
  28                 global $wgLang;
  29                 if (  $wgLang->linkPrefixExtension() ) {
  30                         $regex = "/(([a-zA-Z\x80-\xff]+)\[\[|\]\]|\'\'\'\'\'|\'\'\'|\'\')/";
  31                         #          000000000000000000000000000000000000000000000000000000
  32                         #           1111111111111111111111111111111111111111111111111111
  33                         #            222222222222222222
  34                         # which $this->mMatch[...] will contain the match.
  35                 } else {
  36                         $regex = "/(\[\[|\]\]|\'\'\'\'\'|\'\'\'|\'\')/";
  37                 }
  38
  39                 $this->mCount = preg_match_all( $regex, $this->mText, $this->mMatch,
  40                                                 PREG_PATTERN_ORDER|PREG_OFFSET_CAPTURE);
  41                 $this->mMatchPos=0;
  42                 # print( "<pre>" );
  43                 # print_r( $this->mMatch );
  44                 # print( "</pre>" );
  45         }
  46
  47         function nextToken()
  48         {
  49                 $token = $this->previewToken();
  50                 if ( $token ) {
  51                         $this->mMatchPos = $token["mMatchPos"];
  52                         $this->mPos = $token["mPos"];
  53                 }
  54                 return $token;
  55         }
  56
  57
  58         function previewToken()
  59         {
  60                 if ( $this->mMatchPos <= $this->mCount  ) {
  61                         $token["pos"] = $this->mPos;
  62                         if ( $this->mPos < $this->mMatch[0][$this->mMatchPos][1] ) {
  63                                 $token["type"] = "text";
  64                                 $token["text"] = substr( $this->mText, $this->mPos,
  65                                                          $this->mMatch[0][$this->mMatchPos][1] - $this->mPos );
  66                                 # What the pointers would change to if this would not just be a preview
  67                                 $token["mMatchPos"] = $this->mMatchPos;
  68                                 $token["mPos"] = $this->mMatch[0][$this->mMatchPos][1];
  69                         } else {
  70                                 # If linkPrefixExtension is set,  $this->mMatch[2][$this->mMatchPos][0]
  71                                 # contains the link prefix, or is null if no link prefix exist.
  72                                 if ( $this->mMatch[2][$this->mMatchPos][0] )
  73                                 {
  74                                         # prefixed link open tag, [0] is "prefix[["
  75                                         $token["type"] = "[[";
  76                                         $token["text"] = $this->mMatch[2][$this->mMatchPos][0]; # the prefix
  77                                 } else {
  78                                         $token["type"] = $this->mMatch[0][$this->mMatchPos][0];
  79                                 }
  80                                 # What the pointers would change to if this would not just be a preview
  81                                 $token["mPos"] = $this->mPos + strlen( $this->mMatch[0][$this->mMatchPos][0] );
  82                                 $token["mMatchPos"] = $this->mMatchPos + 1;
  83                         }
  84                 } elseif ( $this->mPos < $this->mTextLength ) {
  85                         $token["type"] = "text";
  86                         $token["text"] = substr( $this->mText, $this->mPos );
  87                         # What the pointers would change to if this would not just be a preview
  88                         $token["mPos"] = $this->mTextLength;
  89                         $token["mMatchPos"] = $this->mMatchPos;
  90                 } else {
  91                         $token = FALSE;
  92                 }
  93                 return $token;
  94         }
  95
  96
  97 }
  98