intl/unicharutil/tools/gencasetable.pl

   1 #!/usr/bin/perl
   2 #
   3 # ***** BEGIN LICENSE BLOCK *****
   4 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   5 #
   6 # The contents of this file are subject to the Mozilla Public License Version
   7 # 1.1 (the "License"); you may not use this file except in compliance with
   8 # the License. You may obtain a copy of the License at
   9 # http://www.mozilla.org/MPL/
  10 #
  11 # Software distributed under the License is distributed on an "AS IS" basis,
  12 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  13 # for the specific language governing rights and limitations under the
  14 # License.
  15 #
  16 # The Original Code is mozilla.org code.
  17 #
  18 # The Initial Developer of the Original Code is
  19 # Netscape Communications Corporation.
  20 # Portions created by the Initial Developer are Copyright (C) 1999
  21 # the Initial Developer. All Rights Reserved.
  22 #
  23 # Contributor(s):
  24 #
  25 # Alternatively, the contents of this file may be used under the terms of
  26 # either the GNU General Public License Version 2 or later (the "GPL"), or
  27 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  28 # in which case the provisions of the GPL or the LGPL are applicable instead
  29 # of those above. If you wish to allow use of your version of this file only
  30 # under the terms of either the GPL or the LGPL, and not to allow others to
  31 # use your version of this file under the terms of the MPL, indicate your
  32 # decision by deleting the provisions above and replace them with the notice
  33 # and other provisions required by the GPL or the LGPL. If you do not delete
  34 # the provisions above, a recipient may use your version of this file under
  35 # the terms of any one of the MPL, the GPL or the LGPL.
  36 #
  37 # ***** END LICENSE BLOCK *****
  38
  39 # If you run this script because the Unicode standard has been updated,
  40 # check xpcom/string/public/nsCharTraits.h to see whether
  41 # nsCharTraits<PRUnichar>::ASCIIToLower needs to be updated. It only
  42 # needs to be update if the Unicode consortium adds (or removes)
  43 # a Unicode character whose lowercase form is an ASCII character.
  44 # Currently there are only two such characters: KELVIN SIGN and
  45 # LATIN CAPITAL LETTER I WITH DOT ABOVE.
  46
  47 ######################################################################
  48 #
  49 # Initial global variable
  50 #
  51 ######################################################################
  52 %utot = ();
  53 $ui=0;
  54 $li=0;
  55
  56 ######################################################################
  57 #
  58 # Open the unicode database file
  59 #
  60 ######################################################################
  61 open ( UNICODATA , "< UnicodeData-Latest.txt")
  62    || die "cannot find UnicodeData-Latest.txt";
  63
  64 ######################################################################
  65 #
  66 # Open the output file
  67 #
  68 ######################################################################
  69 open ( OUT , "> ../src/casetable.h")
  70   || die "cannot open output ../src/casetable.h file";
  71
  72 ######################################################################
  73 #
  74 # Generate license and header
  75 #
  76 ######################################################################
  77 $npl = <<END_OF_NPL;
  78 /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
  79 /* ***** BEGIN LICENSE BLOCK *****
  80  * Version: MPL 1.1/GPL 2.0/LGPL 2.1
  81  *
  82  * The contents of this file are subject to the Mozilla Public License Version
  83  * 1.1 (the "License"); you may not use this file except in compliance with
  84  * the License. You may obtain a copy of the License at
  85  * http://www.mozilla.org/MPL/
  86  *
  87  * Software distributed under the License is distributed on an "AS IS" basis,
  88  * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  89  * for the specific language governing rights and limitations under the
  90  * License.
  91  *
  92  * The Original Code is mozilla.org code.
  93  *
  94  * The Initial Developer of the Original Code is
  95  * Netscape Communications Corporation.
  96  * Portions created by the Initial Developer are Copyright (C) 1999
  97  * the Initial Developer. All Rights Reserved.
  98  *
  99  * Contributor(s):
 100  *
 101  * Alternatively, the contents of this file may be used under the terms of
 102  * either the GNU General Public License Version 2 or later (the "GPL"), or
 103  * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
 104  * in which case the provisions of the GPL or the LGPL are applicable instead
 105  * of those above. If you wish to allow use of your version of this file only
 106  * under the terms of either the GPL or the LGPL, and not to allow others to
 107  * use your version of this file under the terms of the MPL, indicate your
 108  * decision by deleting the provisions above and replace them with the notice
 109  * and other provisions required by the GPL or the LGPL. If you do not delete
 110  * the provisions above, a recipient may use your version of this file under
 111  * the terms of any one of the MPL, the GPL or the LGPL.
 112  *
 113  * ***** END LICENSE BLOCK ***** */
 114 /*
 115     DO NOT EDIT THIS DOCUMENT !!! THIS DOCUMENT IS GENERATED BY
 116     mozilla/intl/unicharutil/tools/gencasetable.pl
 117  */
 118 END_OF_NPL
 119 print OUT $npl;
 120
 121 print OUT "#include \"nscore.h\" \n\n";
 122
 123 ######################################################################
 124 #
 125 # Process the file line by line
 126 #
 127 ######################################################################
 128 while(<UNICODATA>) {
 129    chop;
 130    ######################################################################
 131    #
 132    # Get value from fields
 133    #
 134    ######################################################################
 135    @f = split(/;/ , $_);
 136    $c = $f[0];   # The unicode value
 137    $u = $f[12];  # The upper case
 138    $l = $f[13];  # The lower case
 139    $t = $f[14];  # The title case
 140
 141    #
 142    # print $c . " | " . $u . " | " . $l . " | " . $t . "\n";
 143    #
 144
 145    ######################################################################
 146    #
 147    # Process title case for this entry
 148    #
 149    ######################################################################
 150    #
 151    # if upper case is not equal to title case , store into
 152    # %utot hash
 153    #
 154
 155    if(($t ne "") && ($u ne "") && ( $u ne $t )) {
 156      #
 157      # print $c . " | " . $u . " | " . $l . " | " . $t . "\n";
 158      #
 159      $utot{$u} = $t;
 160    }
 161
 162    $cv = hex($c);     # convert the Unicode value into integer
 163
 164    ######################################################################
 165    #
 166    # Process upper case for this entry
 167    #
 168    ######################################################################
 169    if( $u ne "") {    # if upper case exist
 170       $uvalue = hex($u); # convert the upper case value into integer
 171
 172       ######################################################################
 173       # store related information into arrays
 174       # @ucv     - unicode value
 175       # @uv      - upper case value (debug only)
 176       # @ud      - difference between unicode and upper case
 177       # @ulastd  - difference between unicode and last unicode in the entry
 178       ######################################################################
 179
 180       $ucv[$ui] = $cv;
 181       $uv[$ui] = $uvalue;
 182       $ud[$ui] = 0x0000FFFF & ($uvalue - $cv);
 183
 184       if( $ui ne 0) {
 185           $ulastd[$ui] = $cv - $ucv[$ui-1];
 186       }
 187       $ui++;
 188    }
 189
 190    ######################################################################
 191    #
 192    # Process lower case for this entry
 193    #
 194    ######################################################################
 195    if( $l ne "") {    # if lower case exist
 196       $lvalue = hex($l); # convert the lower case value into integer
 197
 198       ######################################################################
 199       # store related information into arrays
 200       # @lcv     - unicode value
 201       # @lv      - lower case value (debug only)
 202       # @ld      - difference between unicode and lower case
 203       # @llastd  - difference between unicode and last unicode in the entry
 204       ######################################################################
 205
 206       $lcv[$li] = $cv;
 207       $lv[$li] = $lvalue;
 208       $ld[$li] = 0x0000FFFF & ($lvalue - $cv);
 209
 210       if( $li ne 0) {
 211           $llastd[$li] = $cv - $lcv[$li-1];
 212       }
 213       $li++;
 214    }
 215    if(( $l ne "") || ($u ne "")) {    # if lower or upper case exist
 216       $idx = $cv >> 13;
 217       $bits = 1 << (($cv >> 8) & 0x1F) ;
 218       $blk[$idx] |= $bits;
 219    }
 220
 221 }
 222
 223
 224 ######################################################################
 225 #
 226 # Print out all the tables
 227 #
 228 ######################################################################
 229
 230 ######################################################################
 231 #
 232 # Print out upper to title case mapping
 233 #
 234 ######################################################################
 235
 236 $ttotal = 0;
 237 print OUT "static const PRUnichar gUpperToTitle[] = { \n";
 238 while(($upper, $title) = each(%utot)) {
 239    print OUT "   0x" . $upper . ",    0x" . $utot{$upper} . ", \n";
 240    $ttotal++;
 241 }
 242 print OUT "};\n\n";
 243 print OUT "static const PRUint32 gUpperToTitleItems = $ttotal;\n\n";
 244
 245 ######################################################################
 246 #
 247 # Print out gToUpper table
 248 #
 249 ######################################################################
 250 print OUT "static const PRUint16 gToUpper[] = \n";
 251 print OUT "{ /*   From    To             Every   Diff   */ \n";
 252 $utotal=0;
 253 $ufrom = 0;   # remember the start of the output item
 254 for ($i = 0; $i <= $#ucv; $i++)
 255 {
 256   if(0 eq $i) {
 257      ######################################################################
 258      #
 259      # Print the first item in the array
 260      #
 261      ######################################################################
 262      $ufrom = $ucv[0];
 263      printf OUT "       0x%04x, " ,  $ucv[0];
 264   } else {
 265      ######################################################################
 266      #
 267      # Print all the item except the first and last one
 268      #    only print if the upper case difference is different from the
 269      #    and the difference between last entry changed
 270      #
 271      ######################################################################
 272      if(($ud[$i] ne $ud[$i-1])  ||
 273         (($ufrom ne $ucv[$i-1]) && ($ulastd[$i] ne $ulastd[$i-1]))) {
 274
 275          $every = 0;
 276          if($ufrom ne $ucv[$i-1])
 277          {
 278              $every = $ulastd[$i-1];
 279          }
 280
 281          printf OUT "((0x%02x << 8) | 0x%02x), 0x%04x  ,\n",
 282                 ($ucv[$i-1] - $ufrom), $every, $ud[$i-1];
 283
 284          if((($ucv[$i-1] - $ufrom) > 255) || ($every > 255)) {
 285              print "WARNING!!! cannot handle block > 255 chars (Upper)\n\n";
 286              printf "0x%04X, 0x%04x,  0x%04x), 0x%04x \n",
 287                 $ufrom, $ucv[$i-1], $every, $ud[$i-1];
 288          }
 289
 290          $ufrom = $ucv[$i];                  # update the start of the item
 291          printf OUT "       0x%04x, " , $ufrom;
 292          $utotal++;
 293      }
 294   }
 295   if( $i eq $#ucv) {
 296      ######################################################################
 297      #
 298      # Print the last item in the array
 299      #
 300      ######################################################################
 301      printf OUT "((0x%02x << 8) | 0x%02x), 0x%04x   \n};\n\n",
 302             ($ucv[$i] - $ufrom), $ulastd[$i], $ud[$i];
 303      $utotal++;
 304      print OUT "static const PRUint32 gToUpperItems = $utotal;\n\n";
 305   }
 306   #
 307   #  printf "%4x - %4x - %4x - %4x\n", $ucv[$i], $uv[$i], $ud[$i], $ulastd[$i];
 308   #
 309 }
 310
 311 ######################################################################
 312 #
 313 # Print out gToLower table
 314 #
 315 ######################################################################
 316 print OUT "static const PRUint16 gToLower[] = \n";
 317 print OUT "{ /*   From    To             Every   Diff   */ \n";
 318 $ltotal=0;
 319 $lfrom = 0;   # remember the start of the output item
 320 for ($i = 0; $i <= $#lcv; $i++)
 321 {
 322   if(0 eq $i) {
 323      ######################################################################
 324      #
 325      # Print the first item in the array
 326      #
 327      ######################################################################
 328      $lfrom = $lcv[0];
 329      printf OUT "       0x%04x, " ,  $lcv[0];
 330   } else {
 331      ######################################################################
 332      #
 333      # Print all the item except the first and last one
 334      #    only print if the lower case difference is different from the
 335      #    and the difference between last entry changed
 336      #
 337      ######################################################################
 338      if(($ld[$i] ne $ld[$i-1])  ||
 339         (($lfrom ne $lcv[$i-1]) && ($llastd[$i] ne $llastd[$i-1]))) {
 340
 341          $every = 0;
 342          if($lfrom ne $lcv[$i-1])
 343          {
 344              $every = $llastd[$i-1];
 345          }
 346
 347          printf OUT "((0x%02x << 8) | 0x%02x), 0x%04x  ,\n",
 348                 ($lcv[$i-1] - $lfrom) , $every, $ld[$i-1];
 349
 350          if((($lcv[$i-1] - $lfrom) > 255) || ($every > 255)) {
 351              print "WARNING!!! cannot handle block > 255 chars (Lower)\n\n";
 352              printf "0x%04X, 0x%04x,  0x%04x, 0x%04x \n",
 353                 $lfrom, $lcv[$i-1], $every, $ld[$i-1];
 354          }
 355          $lfrom = $lcv[$i];                  # update the start of the item
 356          printf OUT "       0x%04x, " , $lfrom;
 357          $ltotal++;
 358      }
 359   }
 360   if( $i eq $#lcv) {
 361      ######################################################################
 362      #
 363      # Print the last item in the array
 364      #
 365      ######################################################################
 366      printf OUT "((0x%02x << 8) | 0x%02x), 0x%04x   \n};\n\n",
 367             ($lcv[$i] - $lfrom), $llastd[$i], $ld[$i];
 368      $ltotal++;
 369      print OUT "static const PRUint32 gToLowerItems = $ltotal;\n\n";
 370   }
 371   #
 372   #  printf "%4x - %4x - %4x - %4x\n", $lcv[$i], $lv[$i], $ld[$i], $llastd[$i];
 373   #
 374 }
 375
 376 print OUT "static const PRUint32 gCaseBlocks [8] = {\n";
 377 for($idx=0;$idx<8;$idx++)
 378 {
 379    printf OUT "0x%08X", $blk[$idx];
 380    if($idx != 7) {
 381       printf OUT ",\n";
 382    } else {
 383       printf OUT "\n";
 384    }
 385 }
 386 print OUT "};\n";
 387
 388
 389 ######################################################################
 390 #
 391 # Close files
 392 #
 393 ######################################################################
 394 close(UNIDATA);
 395 close(OUT);
 396