2 # Copyright (C) 2004 Brion Vibber <brion@pobox.com>
3 # http://www.mediawiki.org/
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 2 of the License, or
8 # (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License along
16 # with this program; if not, write to the Free Software Foundation, Inc.,
17 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 # http://www.gnu.org/copyleft/gpl.html
21 * Additional tests for UtfNormal::cleanUp() function, inclusion
22 * regression checks for known problems.
30 if( php_sapi_name() != 'cli' ) {
31 die( "Run me from the command line please.\n" );
35 if( isset( $_SERVER['argv'] ) && in_array( '--icu', $_SERVER['argv'] ) ) {
36 dl( 'php_utfnormal.so' );
39 #ini_set( 'memory_limit', '40M' );
41 require_once( 'PHPUnit.php' );
42 require_once( 'UtfNormal.php' );
47 class CleanUpTest
extends PHPUnit_TestCase
{
49 * @param $name String: FIXME
51 function CleanUpTest( $name ) {
52 $this->PHPUnit_TestCase( $name );
64 function testAscii() {
65 $text = 'This is plain ASCII text.';
66 $this->assertEquals( $text, UtfNormal
::cleanUp( $text ) );
71 $text = "a \x00 null";
72 $expect = "a \xef\xbf\xbd null";
75 bin2hex( UtfNormal
::cleanUp( $text ) ) );
79 function testLatin() {
80 $text = "L'\xc3\xa9cole";
81 $this->assertEquals( $text, UtfNormal
::cleanUp( $text ) );
85 function testLatinNormal() {
86 $text = "L'e\xcc\x81cole";
87 $expect = "L'\xc3\xa9cole";
88 $this->assertEquals( $expect, UtfNormal
::cleanUp( $text ) );
92 * This test is *very* expensive!
95 function XtestAllChars() {
96 $rep = UTF8_REPLACEMENT
;
97 global $utfCanonicalComp, $utfCanonicalDecomp;
98 for( $i = 0x0; $i < UNICODE_MAX
; $i++
) {
99 $char = codepointToUtf8( $i );
100 $clean = UtfNormal
::cleanUp( $char );
101 $x = sprintf( "%04X", $i );
102 if( $i %
0x1000 == 0 ) echo "U+$x\n";
106 ($i > 0x001f && $i < UNICODE_SURROGATE_FIRST
) ||
107 ($i > UNICODE_SURROGATE_LAST
&& $i < 0xfffe ) ||
108 ($i > 0xffff && $i <= UNICODE_MAX
) ) {
109 if( isset( $utfCanonicalComp[$char] ) ||
isset( $utfCanonicalDecomp[$char] ) ) {
110 $comp = UtfNormal
::NFC( $char );
114 "U+$x should be decomposed" );
119 "U+$x should be intact" );
122 $this->assertEquals( bin2hex( $rep ), bin2hex( $clean ), $x );
127 /** @todo document */
128 function testAllBytes() {
129 $this->doTestBytes( '', '' );
130 $this->doTestBytes( 'x', '' );
131 $this->doTestBytes( '', 'x' );
132 $this->doTestBytes( 'x', 'x' );
135 /** @todo document */
136 function doTestBytes( $head, $tail ) {
137 for( $i = 0x0; $i < 256; $i++
) {
138 $char = $head . chr( $i ) . $tail;
139 $clean = UtfNormal
::cleanUp( $char );
140 $x = sprintf( "%02X", $i );
144 ($i > 0x001f && $i < 0x80) ) {
148 "ASCII byte $x should be intact" );
149 if( $char != $clean ) return;
151 $norm = $head . UTF8_REPLACEMENT
. $tail;
155 "Forbidden byte $x should be rejected" );
156 if( $norm != $clean ) return;
161 /** @todo document */
162 function testDoubleBytes() {
163 $this->doTestDoubleBytes( '', '' );
164 $this->doTestDoubleBytes( 'x', '' );
165 $this->doTestDoubleBytes( '', 'x' );
166 $this->doTestDoubleBytes( 'x', 'x' );
172 function doTestDoubleBytes( $head, $tail ) {
173 for( $first = 0xc0; $first < 0x100; $first++
) {
174 for( $second = 0x80; $second < 0x100; $second++
) {
175 $char = $head . chr( $first ) . chr( $second ) . $tail;
176 $clean = UtfNormal
::cleanUp( $char );
177 $x = sprintf( "%02X,%02X", $first, $second );
181 $norm = UtfNormal
::NFC( $char );
185 "Pair $x should be intact" );
186 if( $norm != $clean ) return;
187 } elseif( $first > 0xfd ||
$second > 0xbf ) {
188 # fe and ff are not legal head bytes -- expect two replacement chars
189 $norm = $head . UTF8_REPLACEMENT
. UTF8_REPLACEMENT
. $tail;
193 "Forbidden pair $x should be rejected" );
194 if( $norm != $clean ) return;
196 $norm = $head . UTF8_REPLACEMENT
. $tail;
200 "Forbidden pair $x should be rejected" );
201 if( $norm != $clean ) return;
207 /** @todo document */
208 function testTripleBytes() {
209 $this->doTestTripleBytes( '', '' );
210 $this->doTestTripleBytes( 'x', '' );
211 $this->doTestTripleBytes( '', 'x' );
212 $this->doTestTripleBytes( 'x', 'x' );
215 /** @todo document */
216 function doTestTripleBytes( $head, $tail ) {
217 for( $first = 0xc0; $first < 0x100; $first++
) {
218 for( $second = 0x80; $second < 0x100; $second++
) {
219 #for( $third = 0x80; $third < 0x100; $third++ ) {
220 for( $third = 0x80; $third < 0x81; $third++
) {
221 $char = $head . chr( $first ) . chr( $second ) . chr( $third ) . $tail;
222 $clean = UtfNormal
::cleanUp( $char );
223 $x = sprintf( "%02X,%02X,%02X", $first, $second, $third );
224 if( $first >= 0xe0 &&
228 if( $first == 0xe0 && $second < 0xa0 ) {
230 bin2hex( $head . UTF8_REPLACEMENT
. $tail ),
232 "Overlong triplet $x should be rejected" );
233 } elseif( $first == 0xed &&
234 ( chr( $first ) . chr( $second ) . chr( $third )) >= UTF8_SURROGATE_FIRST
) {
236 bin2hex( $head . UTF8_REPLACEMENT
. $tail ),
238 "Surrogate triplet $x should be rejected" );
241 bin2hex( UtfNormal
::NFC( $char ) ),
243 "Triplet $x should be intact" );
245 } elseif( $first > 0xc1 && $first < 0xe0 && $second < 0xc0 ) {
247 bin2hex( UtfNormal
::NFC( $head . chr( $first ) . chr( $second ) ) . UTF8_REPLACEMENT
. $tail ),
249 "Valid 2-byte $x + broken tail" );
250 } elseif( $second > 0xc1 && $second < 0xe0 && $third < 0xc0 ) {
252 bin2hex( $head . UTF8_REPLACEMENT
. UtfNormal
::NFC( chr( $second ) . chr( $third ) . $tail ) ),
254 "Broken head + valid 2-byte $x" );
255 } elseif( ( $first > 0xfd ||
$second > 0xfd ) &&
256 ( ( $second > 0xbf && $third > 0xbf ) ||
257 ( $second < 0xc0 && $third < 0xc0 ) ||
258 ( $second > 0xfd ) ||
259 ( $third > 0xfd ) ) ) {
260 # fe and ff are not legal head bytes -- expect three replacement chars
262 bin2hex( $head . UTF8_REPLACEMENT
. UTF8_REPLACEMENT
. UTF8_REPLACEMENT
. $tail ),
264 "Forbidden triplet $x should be rejected" );
265 } elseif( $first > 0xc2 && $second < 0xc0 && $third < 0xc0 ) {
267 bin2hex( $head . UTF8_REPLACEMENT
. $tail ),
269 "Forbidden triplet $x should be rejected" );
272 bin2hex( $head . UTF8_REPLACEMENT
. UTF8_REPLACEMENT
. $tail ),
274 "Forbidden triplet $x should be rejected" );
281 /** @todo document */
282 function testChunkRegression() {
283 # Check for regression against a chunking bug
284 $text = "\x46\x55\xb8" .
291 $expect = "\x46\x55\xef\xbf\xbd" .
301 bin2hex( UtfNormal
::cleanUp( $text ) ) );
304 /** @todo document */
305 function testInterposeRegression() {
320 $expect = "\x4e\x30" .
336 bin2hex( UtfNormal
::cleanUp( $text ) ) );
339 /** @todo document */
340 function testOverlongRegression() {
342 "\x1a" . # forbidden ascii
344 "\xc1\xa6" . # overlong sequence
346 "\x1c" . # forbidden ascii
361 bin2hex( UtfNormal
::cleanUp( $text ) ) );
364 /** @todo document */
365 function testSurrogateRegression() {
366 $text = "\xed\xb4\x96" . # surrogate 0xDD16
370 $expect = "\xef\xbf\xbd" .
376 bin2hex( UtfNormal
::cleanUp( $text ) ) );
379 /** @todo document */
380 function testBomRegression() {
381 $text = "\xef\xbf\xbe" . # U+FFFE, illegal char
385 $expect = "\xef\xbf\xbd" .
391 bin2hex( UtfNormal
::cleanUp( $text ) ) );
394 /** @todo document */
395 function testForbiddenRegression() {
396 $text = "\xef\xbf\xbf"; # U+FFFF, illegal char
397 $expect = "\xef\xbf\xbd";
400 bin2hex( UtfNormal
::cleanUp( $text ) ) );
403 /** @todo document */
404 function testHangulRegression() {
405 $text = "\xed\x9c\xaf" . # Hangul char
406 "\xe1\x87\x81"; # followed by another final jamo
407 $expect = $text; # Should *not* change.
410 bin2hex( UtfNormal
::cleanUp( $text ) ) );
415 $suite = new PHPUnit_TestSuite( 'CleanUpTest' );
416 $result = PHPUnit
::run( $suite );
417 echo $result->toString();
419 if( !$result->wasSuccessful() ) {