3 /* private */ var $mText, # Text to be processed by the tokenizer
4 $mPos, # current position of tokenizer in text
5 $mTextLength, # Length of $mText
6 $mQueuedToken; # Tokens that were already found, but not
9 /* private */ function Tokenizer()
14 $this->mTokenQueue
=array();
15 $this->linkPrefixExtension
= $wgLang->linkPrefixExtension();
19 function newFromString( $s )
21 $fname = "Tokenizer::newFromString";
22 wfProfileIn( $fname );
26 $t->mTextLength
= strlen( $s );
28 wfProfileOut( $fname );
33 // Return the next token, but do not increase the pointer. The next call
34 // to previewToken or nextToken will return the same token again.
35 // Actually, the pointer is increased, but the token is queued. The next
36 // call to previewToken or nextToken will check the queue and return
38 function previewToken()
40 $fname = "Tokenizer::previewToken";
41 wfProfileIn( $fname );
43 if ( count( $this->mQueuedToken
) != 0 ) {
44 // still one token from the last round around. Return that one first.
45 $token = $this->mQueuedToken
[0];
47 $token = $this->nextToken();
48 array_unshift( $this->mQueuedToken
, $token );
51 wfProfileOut( $fname );
57 // proceeds character by character through the text, looking for characters needing
58 // special attention. Those are currently: I, R, ', [, ], newline
60 // TODO: handling of French blanks not yet implemented
63 $fname = "Tokenizer::nextToken";
64 wfProfileIn( $fname );
66 if ( count( $this->mQueuedToken
) != 0 ) {
67 // still one token from the last round around. Return that one first.
68 $token = array_shift( $this->mQueuedToken
);
69 } else if ( $this->mPos
> $this->mTextLength
) {
70 // If no text is left, return "false".
75 $token["type"]="text";
77 while ( $this->mPos
<= $this->mTextLength
) {
78 switch ( @$ch = $this->mText
[$this->mPos
] ) {
79 case 'R': // for "RFC "
80 if ( $this->continues("FC ") ) {
81 $queueToken["type"] = $queueToken["text"] = "RFC ";
82 $this->mQueuedToken
[] = $queueToken;
84 break 2; // switch + while
87 case 'I': // for "ISBN "
88 if ( $this->continues("SBN ") ) {
89 $queueToken["type"] = $queueToken["text"] = "ISBN ";
90 $this->mQueuedToken
[] = $queueToken;
92 break 2; // switch + while
95 case "[": // for links "[["
96 if ( $this->continues("[[") ) {
97 $queueToken["type"] = "[[[";
98 $queueToken["text"] = "";
99 $this->mQueuedToken
[] = $queueToken;
101 break 2; // switch + while
102 } else if ( $this->continues("[") ) {
103 $queueToken["type"] = "[[";
104 $queueToken["text"] = "";
105 // Check for a "prefixed link", e.g. Al[[Khazar]]
106 // Mostly for arabic wikipedia
107 if ( $this->linkPrefixExtension
) {
108 while ( $this->linkPrefixExtension
109 && ($len = strlen( $token["text"] ) ) > 0
110 && !ctype_space( $token["text"][$len-1] ) )
112 //prepend the character to the link's open tag
113 $queueToken["text"] = $token["text"][$len-1] . $queueToken["text"];
114 //remove character from the end of the text token
115 $token["text"] = substr( $token["text"], 0, -1);
118 $this->mQueuedToken
[] = $queueToken;
120 break 2; // switch + while
123 case "]": // for end of links "]]"
124 if ( $this->continues("]") ) {
125 $queueToken["type"] = "]]";
126 $queueToken["text"] = "";
127 $this->mQueuedToken
[] = $queueToken;
129 break 2; // switch + while
132 case "'": // for all kind of em's and strong's
133 if ( $this->continues("'") ) {
134 $queueToken["type"] = "'";
135 $queueToken["text"] = "";
136 while( ($this->mPos+
1 < $this->mTextLength
)
137 && $this->mText
[$this->mPos+
1] == "'" )
139 $queueToken["type"] .= "'";
143 $this->mQueuedToken
[] = $queueToken;
145 break 2; // switch + while
148 case "\n": // for block levels, actually, only "----" is handled.
150 if ( $this->continues( "----" ) )
152 $queueToken["type"] = "----";
153 $queueToken["text"] = "";
154 $this->mQueuedToken
[] = $queueToken;
156 while ( $this->mPos
<$this->mTextLength
157 and $this->mText
[$this->mPos
] == "-" )
164 case "!": // French spacing rules have a space before exclamation
165 case "?": // and question marks. Those have to become
166 case ":": // And colons, Hashar says ...
167 if ( $this->preceeded( " " ) )
169 // strip blank from Token
170 $token["text"] = substr( $token["text"], 0, -1 );
171 $queueToken["type"] = "blank";
172 $queueToken["text"] = " {$ch}";
173 $this->mQueuedToken
[] = $queueToken;
175 break 2; // switch + while
178 case "0": // A space between two numbers is used to ease reading
179 case "1": // of big numbers, e.g. 1 000 000. Those spaces need
180 case "2": // to be unbreakable
188 if ( ($this->mTextLength
>= $this->mPos +
2)
189 && ($this->mText
[$this->mPos+
1] == " ")
190 && ctype_digit( $this->mText
[$this->mPos+
2] ) )
192 $queueToken["type"] = "blank";
193 $queueToken["text"] = $ch . " ";
194 $this->mQueuedToken
[] = $queueToken;
196 break 2; // switch + while
199 case "\302": // first byte of UTF-8 Character Guillemet-left
200 if ( $this->continues( "\253 ") ) // second byte and a blank
202 $queueToken["type"] = "blank";
203 $queueToken["text"] = "\302\253 ";
204 $this->mQueuedToken
[] = $queueToken;
206 break 2; // switch + while
209 case "\273": //last byte of UTF-8 Character Guillemet-right
210 if ( $this->preceeded( " \302" ) )
212 $queueToken["type"] = "blank";
213 $queueToken["text"] = " \302\273";
214 $token["text"] = substr( $token["text"], 0, -2 );
215 $this->mQueuedToken
[] = $queueToken;
217 break 2; // switch + while
224 // echo $this->mPos . "<br>\n";
226 } /* if (nothing left in queue) */
228 wfProfileOut( $fname );
232 // function continues
233 // checks whether the mText continues with $cont from mPos+1
234 /* private */ function continues( $cont )
236 // If string is not long enough to contain $cont, return false
237 if ( $this->mTextLength
< $this->mPos +
strlen( $cont ) )
239 for ( $i=0; $i < strlen( $cont ); $i++
)
241 if ( $this->mText
[$this->mPos+
1+
$i] != $cont[$i] )
247 // function preceeded
248 // checks whether the mText is preceeded by $prec at position mPos
249 /* private */ function preceeded( $prec )
251 $len = strlen( $prec );
252 // if $prec is longer than the text up to mPos, return false
253 if ( $this->mPos
< $len )
255 return ( 0 == strcmp( $prec, substr($this->mText
, $this->mPos
-$len, $len) ) );