3 /* private */ var $mText, # Text to be processed by the tokenizer
4 $mPos, # current position of tokenizer in text
5 $mTextLength, # Length of $mText
6 $mCount, # token count, computed in preParse
7 $mMatch, # matches of tokenizer regex, computed in preParse
8 $mMatchPos; # current token position of tokenizer. Each match can
9 # be up to two tokens: A matched token and the text after it.
11 /* private */ function Tokenizer()
17 function newFromString( $s )
22 $t->mTextLength
= strlen( $s );
28 $this->mCount
= preg_match_all( "/(\[\[|\]\]|\'\'\'\'\'|\'\'\'|\'\')/",
29 $this->mText
, $this->mMatch
,
30 PREG_PATTERN_ORDER|PREG_OFFSET_CAPTURE
);
36 $token = $this->previewToken();
38 if ( $token["type"] == "text" ) {
39 $this->mPos
= $token["mPos"];
41 $this->mMatchPos
= $token["mMatchPos"];
42 $this->mPos
= $token["mPos"];
49 function previewToken()
51 if ( $this->mMatchPos
<= $this->mCount
) {
52 $token["pos"] = $this->mPos
;
53 if ( $this->mPos
< $this->mMatch
[0][$this->mMatchPos
][1] ) {
54 $token["type"] = "text";
55 $token["text"] = substr( $this->mText
, $this->mPos
,
56 $this->mMatch
[0][$this->mMatchPos
][1] - $this->mPos
);
57 $token["mPos"] = $this->mMatch
[0][$this->mMatchPos
][1];
59 $token["type"] = $this->mMatch
[0][$this->mMatchPos
][0];
60 $token["mPos"] = $this->mPos +
strlen($token["type"]);
61 $token["mMatchPos"] = $this->mMatchPos +
1;
63 } elseif ( $this->mPos
< $this->mTextLength
) {
64 $token["type"] = "text";
65 $token["text"] = substr( $this->mText
, $this->mPos
);
66 $token["mPos"] = $this->mTextLength
;