includes/normal/UtfNormalGenerate.php

   1 <?php
   2 # Copyright (C) 2004 Brion Vibber <brion@pobox.com>
   3 # http://www.mediawiki.org/
   4 #
   5 # This program is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 2 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along
  16 # with this program; if not, write to the Free Software Foundation, Inc.,
  17 # 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18 # http://www.gnu.org/copyleft/gpl.html
  19
  20 /**
  21  * This script generates UniNormalData.inc from the Unicode Character Database
  22  * and supplementary files.
  23  *
  24  * @package UtfNormal
  25  * @access private
  26  */
  27
  28 /** */
  29 require_once 'UtfNormalUtil.php';
  30
  31 $in = fopen("DerivedNormalizationProps.txt", "rt" );
  32 if( !$in ) {
  33         print "Can't open DerivedNormalizationProps.txt for reading.\n";
  34         print "If necessary, fetch this file from the internet:\n";
  35         print "http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt\n";
  36         exit(-1);
  37 }
  38 print "Initializing normalization quick check tables...\n";
  39 $checkNFC = array();
  40 while( false !== ($line = fgets( $in ) ) ) {
  41         if( preg_match( '/^([0-9A-F]+)(?:..([0-9A-F]+))?\s*;\s*(NFC_QC)\s*;\s*([MN])/', $line, $matches ) ) {
  42                 list( $junk, $first, $last, $prop, $value ) = $matches;
  43                 #print "$first $last $prop $value\n";
  44                 if( !$last ) $last = $first;
  45                 for( $i = hexdec( $first ); $i <= hexdec( $last ); $i++) {
  46                         $char = codepointToUtf8( $i );
  47                         $checkNFC[$char] = $value;
  48                 }
  49         }
  50 }
  51 fclose( $in );
  52
  53 $in = fopen("CompositionExclusions.txt", "rt" );
  54 if( !$in ) {
  55         print "Can't open CompositionExclusions.txt for reading.\n";
  56         print "If necessary, fetch this file from the internet:\n";
  57         print "http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt\n";
  58         exit(-1);
  59 }
  60 $exclude = array();
  61 while( false !== ($line = fgets( $in ) ) ) {
  62         if( preg_match( '/^([0-9A-F]+)/i', $line, $matches ) ) {
  63                 $codepoint = $matches[1];
  64                 $source = codepointToUtf8( hexdec( $codepoint ) );
  65                 $exclude[$source] = true;
  66         }
  67 }
  68 fclose($in);
  69
  70 $in = fopen("UnicodeData.txt", "rt" );
  71 if( !$in ) {
  72         print "Can't open UnicodeData.txt for reading.\n";
  73         print "If necessary, fetch this file from the internet:\n";
  74         print "http://www.unicode.org/Public/UNIDATA/UnicodeData.txt\n";
  75         exit(-1);
  76 }
  77
  78 $compatibilityDecomp = array();
  79 $canonicalDecomp = array();
  80 $canonicalComp = array();
  81 $combiningClass = array();
  82 $total = 0;
  83 $compat = 0;
  84 $canon = 0;
  85
  86 print "Reading character definitions...\n";
  87 while( false !== ($line = fgets( $in ) ) ) {
  88         $columns = split(';', $line);
  89         $codepoint = $columns[0];
  90         $name = $columns[1];
  91         $canonicalCombiningClass = $columns[3];
  92         $decompositionMapping = $columns[5];
  93
  94         $source = codepointToUtf8( hexdec( $codepoint ) );
  95
  96         if( $canonicalCombiningClass != 0 ) {
  97                 $combiningClass[$source] = IntVal( $canonicalCombiningClass );
  98         }
  99
 100         if( $decompositionMapping === '' ) continue;
 101         if( preg_match( '/^<(.+)> (.*)$/', $decompositionMapping, $matches ) ) {
 102                 # Compatibility decomposition
 103                 $canonical = false;
 104                 $decompositionMapping = $matches[2];
 105                 $compat++;
 106         } else {
 107                 $canonical = true;
 108                 $canon++;
 109         }
 110         $total++;
 111         $dest = hexSequenceToUtf8( $decompositionMapping );
 112
 113         $compatibilityDecomp[$source] = $dest;
 114         if( $canonical ) {
 115                 $canonicalDecomp[$source] = $dest;
 116                 if( empty( $exclude[$source] ) ) {
 117                         $canonicalComp[$dest] = $source;
 118                 }
 119         }
 120         #print "$codepoint | $canonicalCombiningClasses | $decompositionMapping\n";
 121 }
 122 fclose( $in );
 123
 124 print "Recursively expanding canonical mappings...\n";
 125 $changed = 42;
 126 $pass = 1;
 127 while( $changed > 0 ) {
 128         print "pass $pass\n";
 129         $changed = 0;
 130         foreach( $canonicalDecomp as $source => $dest ) {
 131                 $newDest = preg_replace_callback(
 132                         '/([\xc0-\xff][\x80-\xbf]+)/',
 133                         'callbackCanonical',
 134                         $dest);
 135                 if( $newDest === $dest ) continue;
 136                 $changed++;
 137                 $canonicalDecomp[$source] = $newDest;
 138         }
 139         $pass++;
 140 }
 141
 142 print "Recursively expanding compatibility mappings...\n";
 143 $changed = 42;
 144 $pass = 1;
 145 while( $changed > 0 ) {
 146         print "pass $pass\n";
 147         $changed = 0;
 148         foreach( $compatibilityDecomp as $source => $dest ) {
 149                 $newDest = preg_replace_callback(
 150                         '/([\xc0-\xff][\x80-\xbf]+)/',
 151                         'callbackCompat',
 152                         $dest);
 153                 if( $newDest === $dest ) continue;
 154                 $changed++;
 155                 $compatibilityDecomp[$source] = $newDest;
 156         }
 157         $pass++;
 158 }
 159
 160 print "$total decomposition mappings ($canon canonical, $compat compatibility)\n";
 161
 162 $out = fopen("UtfNormalData.inc", "wt");
 163 if( $out ) {
 164         $serCombining = escapeSingleString( serialize( $combiningClass ) );
 165         $serComp = escapeSingleString( serialize( $canonicalComp ) );
 166         $serCanon = escapeSingleString( serialize( $canonicalDecomp ) );
 167         $serCheckNFC = escapeSingleString( serialize( $checkNFC ) );
 168         $outdata = "<" . "?php
 169 /**
 170  * This file was automatically generated -- do not edit!
 171  * Run UtfNormalGenerate.php to create this file again (make clean && make)
 172  * @package MediaWiki
 173  */
 174 /** */
 175 global \$utfCombiningClass, \$utfCanonicalComp, \$utfCanonicalDecomp, \$utfCheckNFC;
 176 \$utfCombiningClass = unserialize( '$serCombining' );
 177 \$utfCanonicalComp = unserialize( '$serComp' );
 178 \$utfCanonicalDecomp = unserialize( '$serCanon' );
 179 \$utfCheckNFC = unserialize( '$serCheckNFC' );
 180 ?" . ">\n";
 181         fputs( $out, $outdata );
 182         fclose( $out );
 183         print "Wrote out UtfNormalData.inc\n";
 184 } else {
 185         print "Can't create file UtfNormalData.inc\n";
 186         exit(-1);
 187 }
 188
 189
 190 $out = fopen("UtfNormalDataK.inc", "wt");
 191 if( $out ) {
 192         $serCompat = escapeSingleString( serialize( $compatibilityDecomp ) );
 193         $outdata = "<" . "?php
 194 /**
 195  * This file was automatically generated -- do not edit!
 196  * Run UtfNormalGenerate.php to create this file again (make clean && make)
 197  * @package MediaWiki
 198  */
 199 /** */
 200 global \$utfCompatibilityDecomp;
 201 \$utfCompatibilityDecomp = unserialize( '$serCompat' );
 202 ?" . ">\n";
 203         fputs( $out, $outdata );
 204         fclose( $out );
 205         print "Wrote out UtfNormalDataK.inc\n";
 206         exit(0);
 207 } else {
 208         print "Can't create file UtfNormalDataK.inc\n";
 209         exit(-1);
 210 }
 211
 212 # ---------------
 213
 214 function callbackCanonical( $matches ) {
 215         global $canonicalDecomp;
 216         if( isset( $canonicalDecomp[$matches[1]] ) ) {
 217                 return $canonicalDecomp[$matches[1]];
 218         }
 219         return $matches[1];
 220 }
 221
 222 function callbackCompat( $matches ) {
 223         global $compatibilityDecomp;
 224         if( isset( $compatibilityDecomp[$matches[1]] ) ) {
 225                 return $compatibilityDecomp[$matches[1]];
 226         }
 227         return $matches[1];
 228 }
 229
 230 ?>