3 * Tests for UtfNormal::cleanUp() function.
5 * Copyright © 2004 Brion Vibber <brion@pobox.com>
6 * http://www.mediawiki.org/
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 * http://www.gnu.org/copyleft/gpl.html
27 * Additional tests for UtfNormal::cleanUp() function, inclusion
28 * regression checks for known problems.
33 class CleanUpTest
extends MediaWikiTestCase
{
35 function testAscii() {
36 $text = 'This is plain ASCII text.';
37 $this->assertEquals( $text, UtfNormal
::cleanUp( $text ) );
42 $text = "a \x00 null";
43 $expect = "a \xef\xbf\xbd null";
46 bin2hex( UtfNormal
::cleanUp( $text ) ) );
50 function testLatin() {
51 $text = "L'\xc3\xa9cole";
52 $this->assertEquals( $text, UtfNormal
::cleanUp( $text ) );
56 function testLatinNormal() {
57 $text = "L'e\xcc\x81cole";
58 $expect = "L'\xc3\xa9cole";
59 $this->assertEquals( $expect, UtfNormal
::cleanUp( $text ) );
63 * This test is *very* expensive!
66 function XtestAllChars() {
67 $rep = UTF8_REPLACEMENT
;
68 for( $i = 0x0; $i < UNICODE_MAX
; $i++
) {
69 $char = codepointToUtf8( $i );
70 $clean = UtfNormal
::cleanUp( $char );
71 $x = sprintf( "%04X", $i );
72 if( $i %
0x1000 == 0 ) echo "U+$x\n";
76 ($i > 0x001f && $i < UNICODE_SURROGATE_FIRST
) ||
77 ($i > UNICODE_SURROGATE_LAST
&& $i < 0xfffe ) ||
78 ($i > 0xffff && $i <= UNICODE_MAX
) ) {
79 if( isset( UtfNormal
::$utfCanonicalComp[$char] ) ||
isset( UtfNormal
::$utfCanonicalDecomp[$char] ) ) {
80 $comp = UtfNormal
::NFC( $char );
84 "U+$x should be decomposed" );
89 "U+$x should be intact" );
92 $this->assertEquals( bin2hex( $rep ), bin2hex( $clean ), $x );
98 function testAllBytes() {
99 $this->doTestBytes( '', '' );
100 $this->doTestBytes( 'x', '' );
101 $this->doTestBytes( '', 'x' );
102 $this->doTestBytes( 'x', 'x' );
105 /** @todo document */
106 function doTestBytes( $head, $tail ) {
107 for( $i = 0x0; $i < 256; $i++
) {
108 $char = $head . chr( $i ) . $tail;
109 $clean = UtfNormal
::cleanUp( $char );
110 $x = sprintf( "%02X", $i );
114 ($i > 0x001f && $i < 0x80) ) {
118 "ASCII byte $x should be intact" );
119 if( $char != $clean ) return;
121 $norm = $head . UTF8_REPLACEMENT
. $tail;
125 "Forbidden byte $x should be rejected" );
126 if( $norm != $clean ) return;
131 /** @todo document */
132 function testDoubleBytes() {
133 $this->doTestDoubleBytes( '', '' );
134 $this->doTestDoubleBytes( 'x', '' );
135 $this->doTestDoubleBytes( '', 'x' );
136 $this->doTestDoubleBytes( 'x', 'x' );
142 function doTestDoubleBytes( $head, $tail ) {
143 for( $first = 0xc0; $first < 0x100; $first+
=2 ) {
144 for( $second = 0x80; $second < 0x100; $second+
=2 ) {
145 $char = $head . chr( $first ) . chr( $second ) . $tail;
146 $clean = UtfNormal
::cleanUp( $char );
147 $x = sprintf( "%02X,%02X", $first, $second );
151 $norm = UtfNormal
::NFC( $char );
155 "Pair $x should be intact" );
156 if( $norm != $clean ) return;
157 } elseif( $first > 0xfd ||
$second > 0xbf ) {
158 # fe and ff are not legal head bytes -- expect two replacement chars
159 $norm = $head . UTF8_REPLACEMENT
. UTF8_REPLACEMENT
. $tail;
163 "Forbidden pair $x should be rejected" );
164 if( $norm != $clean ) return;
166 $norm = $head . UTF8_REPLACEMENT
. $tail;
170 "Forbidden pair $x should be rejected" );
171 if( $norm != $clean ) return;
177 /** @todo document */
178 function testTripleBytes() {
179 $this->doTestTripleBytes( '', '' );
180 $this->doTestTripleBytes( 'x', '' );
181 $this->doTestTripleBytes( '', 'x' );
182 $this->doTestTripleBytes( 'x', 'x' );
185 /** @todo document */
186 function doTestTripleBytes( $head, $tail ) {
187 for( $first = 0xc0; $first < 0x100; $first+
=2 ) {
188 for( $second = 0x80; $second < 0x100; $second+
=2 ) {
189 #for( $third = 0x80; $third < 0x100; $third++ ) {
190 for( $third = 0x80; $third < 0x81; $third++
) {
191 $char = $head . chr( $first ) . chr( $second ) . chr( $third ) . $tail;
192 $clean = UtfNormal
::cleanUp( $char );
193 $x = sprintf( "%02X,%02X,%02X", $first, $second, $third );
194 if( $first >= 0xe0 &&
198 if( $first == 0xe0 && $second < 0xa0 ) {
200 bin2hex( $head . UTF8_REPLACEMENT
. $tail ),
202 "Overlong triplet $x should be rejected" );
203 } elseif( $first == 0xed &&
204 ( chr( $first ) . chr( $second ) . chr( $third )) >= UTF8_SURROGATE_FIRST
) {
206 bin2hex( $head . UTF8_REPLACEMENT
. $tail ),
208 "Surrogate triplet $x should be rejected" );
211 bin2hex( UtfNormal
::NFC( $char ) ),
213 "Triplet $x should be intact" );
215 } elseif( $first > 0xc1 && $first < 0xe0 && $second < 0xc0 ) {
217 bin2hex( UtfNormal
::NFC( $head . chr( $first ) . chr( $second ) ) . UTF8_REPLACEMENT
. $tail ),
219 "Valid 2-byte $x + broken tail" );
220 } elseif( $second > 0xc1 && $second < 0xe0 && $third < 0xc0 ) {
222 bin2hex( $head . UTF8_REPLACEMENT
. UtfNormal
::NFC( chr( $second ) . chr( $third ) . $tail ) ),
224 "Broken head + valid 2-byte $x" );
225 } elseif( ( $first > 0xfd ||
$second > 0xfd ) &&
226 ( ( $second > 0xbf && $third > 0xbf ) ||
227 ( $second < 0xc0 && $third < 0xc0 ) ||
228 ( $second > 0xfd ) ||
229 ( $third > 0xfd ) ) ) {
230 # fe and ff are not legal head bytes -- expect three replacement chars
232 bin2hex( $head . UTF8_REPLACEMENT
. UTF8_REPLACEMENT
. UTF8_REPLACEMENT
. $tail ),
234 "Forbidden triplet $x should be rejected" );
235 } elseif( $first > 0xc2 && $second < 0xc0 && $third < 0xc0 ) {
237 bin2hex( $head . UTF8_REPLACEMENT
. $tail ),
239 "Forbidden triplet $x should be rejected" );
242 bin2hex( $head . UTF8_REPLACEMENT
. UTF8_REPLACEMENT
. $tail ),
244 "Forbidden triplet $x should be rejected" );
251 /** @todo document */
252 function testChunkRegression() {
253 # Check for regression against a chunking bug
254 $text = "\x46\x55\xb8" .
261 $expect = "\x46\x55\xef\xbf\xbd" .
271 bin2hex( UtfNormal
::cleanUp( $text ) ) );
274 /** @todo document */
275 function testInterposeRegression() {
290 $expect = "\x4e\x30" .
306 bin2hex( UtfNormal
::cleanUp( $text ) ) );
309 /** @todo document */
310 function testOverlongRegression() {
312 "\x1a" . # forbidden ascii
314 "\xc1\xa6" . # overlong sequence
316 "\x1c" . # forbidden ascii
331 bin2hex( UtfNormal
::cleanUp( $text ) ) );
334 /** @todo document */
335 function testSurrogateRegression() {
336 $text = "\xed\xb4\x96" . # surrogate 0xDD16
340 $expect = "\xef\xbf\xbd" .
346 bin2hex( UtfNormal
::cleanUp( $text ) ) );
349 /** @todo document */
350 function testBomRegression() {
351 $text = "\xef\xbf\xbe" . # U+FFFE, illegal char
355 $expect = "\xef\xbf\xbd" .
361 bin2hex( UtfNormal
::cleanUp( $text ) ) );
364 /** @todo document */
365 function testForbiddenRegression() {
366 $text = "\xef\xbf\xbf"; # U+FFFF, illegal char
367 $expect = "\xef\xbf\xbd";
370 bin2hex( UtfNormal
::cleanUp( $text ) ) );
373 /** @todo document */
374 function testHangulRegression() {
375 $text = "\xed\x9c\xaf" . # Hangul char
376 "\xe1\x87\x81"; # followed by another final jamo
377 $expect = $text; # Should *not* change.
380 bin2hex( UtfNormal
::cleanUp( $text ) ) );