3 * Tests for UtfNormal::cleanUp() function.
5 * Copyright © 2004 Brion Vibber <brion@pobox.com>
6 * http://www.mediawiki.org/
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 * http://www.gnu.org/copyleft/gpl.html
28 if( php_sapi_name() != 'cli' ) {
29 die( "Run me from the command line please.\n" );
33 if( isset( $_SERVER['argv'] ) && in_array( '--icu', $_SERVER['argv'] ) ) {
34 dl( 'php_utfnormal.so' );
37 #ini_set( 'memory_limit', '40M' );
39 require_once( 'PHPUnit/Runner/Version.php' );
40 if( version_compare( PHPUnit_Runner_Version
::id(), '3.5.0', '>=' ) ) {
41 # PHPUnit 3.5.0 introduced a nice autoloader based on class name
42 require_once( 'PHPUnit/Autoload.php' );
44 # Keep the old pre PHPUnit 3.5.0 behaviour for compatibility
45 require_once 'PHPUnit/Framework.php';
47 require_once 'PHPUnit/TextUI/TestRunner.php';
49 require_once 'UtfNormal.php';
52 * Additional tests for UtfNormal::cleanUp() function, inclusion
53 * regression checks for known problems.
59 class CleanUpTest
extends PHPUnit_Framework_TestCase
{
69 function testAscii() {
70 $text = 'This is plain ASCII text.';
71 $this->assertEquals( $text, UtfNormal
::cleanUp( $text ) );
76 $text = "a \x00 null";
77 $expect = "a \xef\xbf\xbd null";
80 bin2hex( UtfNormal
::cleanUp( $text ) ) );
84 function testLatin() {
85 $text = "L'\xc3\xa9cole";
86 $this->assertEquals( $text, UtfNormal
::cleanUp( $text ) );
90 function testLatinNormal() {
91 $text = "L'e\xcc\x81cole";
92 $expect = "L'\xc3\xa9cole";
93 $this->assertEquals( $expect, UtfNormal
::cleanUp( $text ) );
97 * This test is *very* expensive!
100 function XtestAllChars() {
101 $rep = UTF8_REPLACEMENT
;
102 for( $i = 0x0; $i < UNICODE_MAX
; $i++
) {
103 $char = codepointToUtf8( $i );
104 $clean = UtfNormal
::cleanUp( $char );
105 $x = sprintf( "%04X", $i );
106 if( $i %
0x1000 == 0 ) echo "U+$x\n";
110 ($i > 0x001f && $i < UNICODE_SURROGATE_FIRST
) ||
111 ($i > UNICODE_SURROGATE_LAST
&& $i < 0xfffe ) ||
112 ($i > 0xffff && $i <= UNICODE_MAX
) ) {
113 if( isset( UtfNormal
::$utfCanonicalComp[$char] ) ||
isset( UtfNormal
::$utfCanonicalDecomp[$char] ) ) {
114 $comp = UtfNormal
::NFC( $char );
118 "U+$x should be decomposed" );
123 "U+$x should be intact" );
126 $this->assertEquals( bin2hex( $rep ), bin2hex( $clean ), $x );
131 /** @todo document */
132 function testAllBytes() {
133 $this->doTestBytes( '', '' );
134 $this->doTestBytes( 'x', '' );
135 $this->doTestBytes( '', 'x' );
136 $this->doTestBytes( 'x', 'x' );
139 /** @todo document */
140 function doTestBytes( $head, $tail ) {
141 for( $i = 0x0; $i < 256; $i++
) {
142 $char = $head . chr( $i ) . $tail;
143 $clean = UtfNormal
::cleanUp( $char );
144 $x = sprintf( "%02X", $i );
148 ($i > 0x001f && $i < 0x80) ) {
152 "ASCII byte $x should be intact" );
153 if( $char != $clean ) return;
155 $norm = $head . UTF8_REPLACEMENT
. $tail;
159 "Forbidden byte $x should be rejected" );
160 if( $norm != $clean ) return;
165 /** @todo document */
166 function testDoubleBytes() {
167 $this->doTestDoubleBytes( '', '' );
168 $this->doTestDoubleBytes( 'x', '' );
169 $this->doTestDoubleBytes( '', 'x' );
170 $this->doTestDoubleBytes( 'x', 'x' );
176 function doTestDoubleBytes( $head, $tail ) {
177 for( $first = 0xc0; $first < 0x100; $first++
) {
178 for( $second = 0x80; $second < 0x100; $second++
) {
179 $char = $head . chr( $first ) . chr( $second ) . $tail;
180 $clean = UtfNormal
::cleanUp( $char );
181 $x = sprintf( "%02X,%02X", $first, $second );
185 $norm = UtfNormal
::NFC( $char );
189 "Pair $x should be intact" );
190 if( $norm != $clean ) return;
191 } elseif( $first > 0xfd ||
$second > 0xbf ) {
192 # fe and ff are not legal head bytes -- expect two replacement chars
193 $norm = $head . UTF8_REPLACEMENT
. UTF8_REPLACEMENT
. $tail;
197 "Forbidden pair $x should be rejected" );
198 if( $norm != $clean ) return;
200 $norm = $head . UTF8_REPLACEMENT
. $tail;
204 "Forbidden pair $x should be rejected" );
205 if( $norm != $clean ) return;
211 /** @todo document */
212 function testTripleBytes() {
213 $this->doTestTripleBytes( '', '' );
214 $this->doTestTripleBytes( 'x', '' );
215 $this->doTestTripleBytes( '', 'x' );
216 $this->doTestTripleBytes( 'x', 'x' );
219 /** @todo document */
220 function doTestTripleBytes( $head, $tail ) {
221 for( $first = 0xc0; $first < 0x100; $first++
) {
222 for( $second = 0x80; $second < 0x100; $second++
) {
223 #for( $third = 0x80; $third < 0x100; $third++ ) {
224 for( $third = 0x80; $third < 0x81; $third++
) {
225 $char = $head . chr( $first ) . chr( $second ) . chr( $third ) . $tail;
226 $clean = UtfNormal
::cleanUp( $char );
227 $x = sprintf( "%02X,%02X,%02X", $first, $second, $third );
228 if( $first >= 0xe0 &&
232 if( $first == 0xe0 && $second < 0xa0 ) {
234 bin2hex( $head . UTF8_REPLACEMENT
. $tail ),
236 "Overlong triplet $x should be rejected" );
237 } elseif( $first == 0xed &&
238 ( chr( $first ) . chr( $second ) . chr( $third )) >= UTF8_SURROGATE_FIRST
) {
240 bin2hex( $head . UTF8_REPLACEMENT
. $tail ),
242 "Surrogate triplet $x should be rejected" );
245 bin2hex( UtfNormal
::NFC( $char ) ),
247 "Triplet $x should be intact" );
249 } elseif( $first > 0xc1 && $first < 0xe0 && $second < 0xc0 ) {
251 bin2hex( UtfNormal
::NFC( $head . chr( $first ) . chr( $second ) ) . UTF8_REPLACEMENT
. $tail ),
253 "Valid 2-byte $x + broken tail" );
254 } elseif( $second > 0xc1 && $second < 0xe0 && $third < 0xc0 ) {
256 bin2hex( $head . UTF8_REPLACEMENT
. UtfNormal
::NFC( chr( $second ) . chr( $third ) . $tail ) ),
258 "Broken head + valid 2-byte $x" );
259 } elseif( ( $first > 0xfd ||
$second > 0xfd ) &&
260 ( ( $second > 0xbf && $third > 0xbf ) ||
261 ( $second < 0xc0 && $third < 0xc0 ) ||
262 ( $second > 0xfd ) ||
263 ( $third > 0xfd ) ) ) {
264 # fe and ff are not legal head bytes -- expect three replacement chars
266 bin2hex( $head . UTF8_REPLACEMENT
. UTF8_REPLACEMENT
. UTF8_REPLACEMENT
. $tail ),
268 "Forbidden triplet $x should be rejected" );
269 } elseif( $first > 0xc2 && $second < 0xc0 && $third < 0xc0 ) {
271 bin2hex( $head . UTF8_REPLACEMENT
. $tail ),
273 "Forbidden triplet $x should be rejected" );
276 bin2hex( $head . UTF8_REPLACEMENT
. UTF8_REPLACEMENT
. $tail ),
278 "Forbidden triplet $x should be rejected" );
285 /** @todo document */
286 function testChunkRegression() {
287 # Check for regression against a chunking bug
288 $text = "\x46\x55\xb8" .
295 $expect = "\x46\x55\xef\xbf\xbd" .
305 bin2hex( UtfNormal
::cleanUp( $text ) ) );
308 /** @todo document */
309 function testInterposeRegression() {
324 $expect = "\x4e\x30" .
340 bin2hex( UtfNormal
::cleanUp( $text ) ) );
343 /** @todo document */
344 function testOverlongRegression() {
346 "\x1a" . # forbidden ascii
348 "\xc1\xa6" . # overlong sequence
350 "\x1c" . # forbidden ascii
365 bin2hex( UtfNormal
::cleanUp( $text ) ) );
368 /** @todo document */
369 function testSurrogateRegression() {
370 $text = "\xed\xb4\x96" . # surrogate 0xDD16
374 $expect = "\xef\xbf\xbd" .
380 bin2hex( UtfNormal
::cleanUp( $text ) ) );
383 /** @todo document */
384 function testBomRegression() {
385 $text = "\xef\xbf\xbe" . # U+FFFE, illegal char
389 $expect = "\xef\xbf\xbd" .
395 bin2hex( UtfNormal
::cleanUp( $text ) ) );
398 /** @todo document */
399 function testForbiddenRegression() {
400 $text = "\xef\xbf\xbf"; # U+FFFF, illegal char
401 $expect = "\xef\xbf\xbd";
404 bin2hex( UtfNormal
::cleanUp( $text ) ) );
407 /** @todo document */
408 function testHangulRegression() {
409 $text = "\xed\x9c\xaf" . # Hangul char
410 "\xe1\x87\x81"; # followed by another final jamo
411 $expect = $text; # Should *not* change.
414 bin2hex( UtfNormal
::cleanUp( $text ) ) );
419 $suite = new PHPUnit_Framework_TestSuite( 'CleanUpTest' );
420 $result = PHPUnit_TextUI_TestRunner
::run( $suite );
422 if( !$result->wasSuccessful() ) {