3 /* private */ var $mText, # Text to be processed by the tokenizer
4 $mPos, # current position of tokenizer in text
5 $mTextLength, # Length of $mText
6 $mCount, # token count, computed in preParse
7 $mMatch, # matches of tokenizer regex, computed in preParse
8 $mMatchPos; # current token position of tokenizer. Each match can
9 # be up to two tokens: A matched token and the text after it.
11 /* private */ function Tokenizer()
17 function newFromString( $s )
22 $t->mTextLength
= strlen( $s );
29 if ( $wgLang->linkPrefixExtension() ) {
30 $regex = "/(([a-zA-Z\x80-\xff]+)\[\[|\]\]|\'\'\'\'\'|\'\'\'|\'\')/";
31 # 000000000000000000000000000000000000000000000000000000
32 # 1111111111111111111111111111111111111111111111111111
34 # which $this->mMatch[...] will contain the match.
36 $regex = "/(\[\[|\]\]|\'\'\'\'\'|\'\'\'|\'\')/";
39 $this->mCount
= preg_match_all( $regex, $this->mText
, $this->mMatch
,
40 PREG_PATTERN_ORDER|PREG_OFFSET_CAPTURE
);
43 # print_r( $this->mMatch );
49 $token = $this->previewToken();
51 $this->mMatchPos
= $token["mMatchPos"];
52 $this->mPos
= $token["mPos"];
58 function previewToken()
60 if ( $this->mMatchPos
<= $this->mCount
) {
61 $token["pos"] = $this->mPos
;
62 if ( $this->mPos
< $this->mMatch
[0][$this->mMatchPos
][1] ) {
63 $token["type"] = "text";
64 $token["text"] = substr( $this->mText
, $this->mPos
,
65 $this->mMatch
[0][$this->mMatchPos
][1] - $this->mPos
);
66 # What the pointers would change to if this would not just be a preview
67 $token["mMatchPos"] = $this->mMatchPos
;
68 $token["mPos"] = $this->mMatch
[0][$this->mMatchPos
][1];
70 # If linkPrefixExtension is set, $this->mMatch[2][$this->mMatchPos][0]
71 # contains the link prefix, or is null if no link prefix exist.
72 if ( $this->mMatch
[2][$this->mMatchPos
][0] )
74 # prefixed link open tag, [0] is "prefix[["
75 $token["type"] = "[[";
76 $token["text"] = $this->mMatch
[2][$this->mMatchPos
][0]; # the prefix
78 $token["type"] = $this->mMatch
[0][$this->mMatchPos
][0];
80 # What the pointers would change to if this would not just be a preview
81 $token["mPos"] = $this->mPos +
strlen( $this->mMatch
[0][$this->mMatchPos
][0] );
82 $token["mMatchPos"] = $this->mMatchPos +
1;
84 } elseif ( $this->mPos
< $this->mTextLength
) {
85 $token["type"] = "text";
86 $token["text"] = substr( $this->mText
, $this->mPos
);
87 # What the pointers would change to if this would not just be a preview
88 $token["mPos"] = $this->mTextLength
;
89 $token["mMatchPos"] = $this->mMatchPos
;