2 # Copyright (C) 2004 Brion Vibber <brion@pobox.com>
3 # http://www.mediawiki.org/
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 2 of the License, or
8 # (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License along
16 # with this program; if not, write to the Free Software Foundation, Inc.,
17 # 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 # http://www.gnu.org/copyleft/gpl.html
21 * This script generates UniNormalData.inc from the Unicode Character Database
22 * and supplementary files.
28 require_once 'UtfNormalUtil.php';
30 $in = fopen("DerivedNormalizationProps.txt", "rt" );
32 print "Can't open DerivedNormalizationProps.txt for reading.\n";
33 print "If necessary, fetch this file from the internet:\n";
34 print "http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt\n";
37 print "Initializing normalization quick check tables...\n";
39 while( false !== ($line = fgets( $in ) ) ) {
40 if( preg_match( '/^([0-9A-F]+)(?:..([0-9A-F]+))?\s*;\s*(NFC_QC)\s*;\s*([MN])/', $line, $matches ) ) {
41 list( $junk, $first, $last, $prop, $value ) = $matches;
42 #print "$first $last $prop $value\n";
43 if( !$last ) $last = $first;
44 for( $i = hexdec( $first ); $i <= hexdec( $last ); $i++
) {
45 $char = codepointToUtf8( $i );
46 $checkNFC[$char] = $value;
52 $in = fopen("CompositionExclusions.txt", "rt" );
54 print "Can't open CompositionExclusions.txt for reading.\n";
55 print "If necessary, fetch this file from the internet:\n";
56 print "http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt\n";
60 while( false !== ($line = fgets( $in ) ) ) {
61 if( preg_match( '/^([0-9A-F]+)/i', $line, $matches ) ) {
62 $codepoint = $matches[1];
63 $source = codepointToUtf8( hexdec( $codepoint ) );
64 $exclude[$source] = true;
69 $in = fopen("UnicodeData.txt", "rt" );
71 print "Can't open UnicodeData.txt for reading.\n";
72 print "If necessary, fetch this file from the internet:\n";
73 print "http://www.unicode.org/Public/UNIDATA/UnicodeData.txt\n";
77 $compatibilityDecomp = array();
78 $canonicalDecomp = array();
79 $canonicalComp = array();
80 $combiningClass = array();
85 print "Reading character definitions...\n";
86 while( false !== ($line = fgets( $in ) ) ) {
87 $columns = split(';', $line);
88 $codepoint = $columns[0];
90 $canonicalCombiningClass = $columns[3];
91 $decompositionMapping = $columns[5];
93 $source = codepointToUtf8( hexdec( $codepoint ) );
95 if( $canonicalCombiningClass != 0 ) {
96 $combiningClass[$source] = IntVal( $canonicalCombiningClass );
99 if( $decompositionMapping === '' ) continue;
100 if( preg_match( '/^<(.+)> (.*)$/', $decompositionMapping, $matches ) ) {
101 # Compatibility decomposition
103 $decompositionMapping = $matches[2];
110 $dest = hexSequenceToUtf8( $decompositionMapping );
112 $compatibilityDecomp[$source] = $dest;
114 $canonicalDecomp[$source] = $dest;
115 if( empty( $exclude[$source] ) ) {
116 $canonicalComp[$dest] = $source;
119 #print "$codepoint | $canonicalCombiningClasses | $decompositionMapping\n";
123 print "Recursively expanding canonical mappings...\n";
126 while( $changed > 0 ) {
127 print "pass $pass\n";
129 foreach( $canonicalDecomp as $source => $dest ) {
130 $newDest = preg_replace_callback(
131 '/([\xc0-\xff][\x80-\xbf]+)/',
134 if( $newDest === $dest ) continue;
136 $canonicalDecomp[$source] = $newDest;
141 print "Recursively expanding compatibility mappings...\n";
144 while( $changed > 0 ) {
145 print "pass $pass\n";
147 foreach( $compatibilityDecomp as $source => $dest ) {
148 $newDest = preg_replace_callback(
149 '/([\xc0-\xff][\x80-\xbf]+)/',
152 if( $newDest === $dest ) continue;
154 $compatibilityDecomp[$source] = $newDest;
159 print "$total decomposition mappings ($canon canonical, $compat compatibility)\n";
161 $out = fopen("UtfNormalData.inc", "wt");
163 $serCombining = escapeSingleString( serialize( $combiningClass ) );
164 $serComp = escapeSingleString( serialize( $canonicalComp ) );
165 $serCanon = escapeSingleString( serialize( $canonicalDecomp ) );
166 $serCheckNFC = escapeSingleString( serialize( $checkNFC ) );
167 $outdata = "<" . "?php
169 * This file was automatically generated -- do not edit!
170 * Run UtfNormalGenerate.php to create this file again (make clean && make)
174 global \$utfCombiningClass, \$utfCanonicalComp, \$utfCanonicalDecomp;
175 \$utfCombiningClass = unserialize( '$serCombining' );
176 \$utfCanonicalComp = unserialize( '$serComp' );
177 \$utfCanonicalDecomp = unserialize( '$serCanon' );
178 \$utfCheckNFC = unserialize( '$serCheckNFC' );
180 fputs( $out, $outdata );
182 print "Wrote out UtfNormalData.inc\n";
184 print "Can't create file UtfNormalData.inc\n";
189 $out = fopen("UtfNormalDataK.inc", "wt");
191 $serCompat = escapeSingleString( serialize( $compatibilityDecomp ) );
192 $outdata = "<" . "?php
194 * This file was automatically generated -- do not edit!
195 * Run UtfNormalGenerate.php to create this file again (make clean && make)
199 global \$utfCompatibilityDecomp;
200 \$utfCompatibilityDecomp = unserialize( '$serCompat' );
202 fputs( $out, $outdata );
204 print "Wrote out UtfNormalDataK.inc\n";
207 print "Can't create file UtfNormalDataK.inc\n";
213 function callbackCanonical( $matches ) {
214 global $canonicalDecomp;
215 if( isset( $canonicalDecomp[$matches[1]] ) ) {
216 return $canonicalDecomp[$matches[1]];
221 function callbackCompat( $matches ) {
222 global $compatibilityDecomp;
223 if( isset( $compatibilityDecomp[$matches[1]] ) ) {
224 return $compatibilityDecomp[$matches[1]];