sal/textenc/generate/gb180302000.pl

   1 #!/usr/bin/perl
   2 #
   3 # This file is part of the LibreOffice project.
   4 #
   5 # This Source Code Form is subject to the terms of the Mozilla Public
   6 # License, v. 2.0. If a copy of the MPL was not distributed with this
   7 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   8 #
   9 # This file incorporates work covered by the following license notice:
  10 #
  11 #   Licensed to the Apache Software Foundation (ASF) under one or more
  12 #   contributor license agreements. See the NOTICE file distributed
  13 #   with this work for additional information regarding copyright
  14 #   ownership. The ASF licenses this file to you under the Apache
  15 #   License, Version 2.0 (the "License"); you may not use this file
  16 #   except in compliance with the License. You may obtain a copy of
  17 #   the License at http://www.apache.org/licenses/LICENSE-2.0 .
  18 #
  19
  20 # The following files must be available in a ./input subdir:
  21
  22 # <http://oss.software.ibm.com/cvs/icu/~checkout~/charset/data/xml/
  23 # gb-18030-2000.xml?rev=1.4&content-type=text/plain>:
  24 #  "modified version="3" date="2001-02-21""
  25
  26 $id = "Gb180302000";
  27
  28 sub printUtf32
  29 {
  30     my $utf32 = $_[0];
  31     return sprintf("U+%04X", $utf32);
  32 }
  33
  34 sub printGb
  35 {
  36     if (defined($_[2]))
  37     {
  38         return sprintf("%02X%02X%02X%02X", $_[0], $_[1], $_[2], $_[3]);
  39     }
  40     elsif (defined($_[1]))
  41     {
  42         return sprintf("%02X%02X", $_[0], $_[1]);
  43     }
  44     else
  45     {
  46         return sprintf("%02X", $_[0]);
  47     }
  48 }
  49
  50 $gb_map_2_count = 0;
  51 $gb_map_4_count = 0;
  52 $gb_map_4_ranges = 0;
  53 $gb_map_4_max = 0;
  54 $uni_map_count = 0;
  55
  56 $range_count = 0;
  57
  58 if (1)
  59 {
  60     $filename = "gb-18030-2000.xml";
  61     open IN, ("input/" . $filename) or die "Cannot read " . $filename;
  62     while (<IN>)
  63     {
  64         if (/^[ \t]*<a +u=\"([0-9A-F]+)\" +b=\"([0-7][0-9A-F])\"\/>$/)
  65         {
  66             $utf32 = oct("0x" . $1);
  67             $gb1 = oct("0x" . $2);
  68             ($utf32 == $gb1)
  69                 or die "Bad " . printUtf32($utf32) . " to " . printGb($gb1);
  70         }
  71         elsif (/^[ \t]*<a +u=\"([0-9A-F]+)\" +b=\"([89A-F][0-9A-F]) ([4-789A-F][0-9A-F])\"\/>$/)
  72         {
  73             $utf32 = oct("0x" . $1);
  74             $gb1 = oct("0x" . $2);
  75             $gb2 = oct("0x" . $3);
  76             $gb_code = ($gb1 - 0x81) * 190
  77                            + ($gb2 <= 0x7E ? $gb2 - 0x40 : $gb2 - 0x80 + 63);
  78             !defined($gb_map_2[$gb_code])
  79                 or die "Redefined " . printGb($gb1, $gb2);
  80             $gb_map_2[$gb_code] = $utf32;
  81             ++$gb_map_2_count;
  82
  83             !defined($uni_map[$utf32]) or die "Double Unicode mapping";
  84             $uni_map[$utf32] = $gb1 << 8 | $gb2;
  85             ++$uni_map_count;
  86         }
  87         elsif (/^[ \t]*<a +u=\"([0-9A-F]+)\" +b=\"([89A-F][0-9A-F]) (3[0-9]) ([89A-F][0-9A-F]) (3[0-9])\"\/>$/)
  88         {
  89             $utf32 = oct("0x" . $1);
  90             $gb1 = oct("0x" . $2);
  91             $gb2 = oct("0x" . $3);
  92             $gb3 = oct("0x" . $4);
  93             $gb4 = oct("0x" . $5);
  94             $gb_code = ($gb1 - 0x81) * 12600
  95                            + ($gb2 - 0x30) * 1260
  96                            + ($gb3 - 0x81) * 10
  97                            + ($gb4 - 0x30);
  98             !defined($gb_map_4[$gb_code])
  99                 or die "Redefined " . printGb($gb1, $gb2, $gb3, $gb4);
 100             $gb_map_4[$gb_code] = $utf32;
 101             ++$gb_map_4_count;
 102             $gb_map_4_max = $gb_code if ($gb_code > $gb_map_4_max);
 103
 104             !defined($uni_map[$utf32]) or die "Double Unicode mapping";
 105             $uni_map[$utf32] = $gb1 << 24 | $gb2 << 16 | $gb3 << 8 | $gb4;
 106             ++$uni_map_count;
 107         }
 108         elsif (/<a /)
 109         {
 110             die "Bad format";
 111         }
 112         elsif (/^[ \t]*<range +uFirst=\"([0-9A-F]+)\" +uLast=\"([0-9A-F]+)\" +bFirst=\"([89A-F][0-9A-F]) (3[0-9]) ([89A-F][0-9A-F]) (3[0-9])\" +bLast=\"([89A-F][0-9A-F]) (3[0-9]) ([89A-F][0-9A-F]) (3[0-9])\" +bMin=\"81 30 81 30\" +bMax=\"FE 39 FE 39\"\/>$/)
 113         {
 114             $utf32_first = oct("0x" . $1);
 115             $utf32_last = oct("0x" . $2);
 116             $gb1_first = oct("0x" . $3);
 117             $gb2_first = oct("0x" . $4);
 118             $gb3_first = oct("0x" . $5);
 119             $gb4_first = oct("0x" . $6);
 120             $gb1_last = oct("0x" . $7);
 121             $gb2_last = oct("0x" . $8);
 122             $gb3_last = oct("0x" . $9);
 123             $gb4_last = oct("0x" . $10);
 124             $linear_first
 125                 = ($gb1_first - 0x81) * 12600
 126                     + ($gb2_first - 0x30) * 1260
 127                         + ($gb3_first - 0x81) * 10
 128                             + ($gb4_first - 0x30);
 129             $linear_last
 130                 = ($gb1_last - 0x81) * 12600
 131                     + ($gb2_last - 0x30) * 1260
 132                         + ($gb3_last - 0x81) * 10
 133                             + ($gb4_last - 0x30);
 134             ($utf32_last - $utf32_first == $linear_last - $linear_first)
 135                 or die "Bad range";
 136             if ($linear_first != 189000 || $linear_last != 1237575)
 137             {
 138                 $range_uni_first[$range_count] = $utf32_first;
 139                 $range_uni_last[$range_count]
 140                     = ($utf32_last == 0xD7FF ? 0xDFFF : $utf32_last);
 141                 $range_linear_first[$range_count] = $linear_first;
 142                 $range_linear_last[$range_count] = $linear_last;
 143                 ++$range_count;
 144                 $gb_map_4_ranges += $linear_last - $linear_first + 1;
 145                 $gb_map_4_max = $linear_last
 146                     if ($linear_last > $gb_map_4_max);
 147             }
 148         }
 149         elsif (/<range /)
 150         {
 151             die "Bad format";
 152         }
 153     }
 154     close IN;
 155 }
 156
 157 print "gb_map_2_count = ", $gb_map_2_count,
 158       ", gb_map_4_count = ", $gb_map_4_count,
 159       ", gb_map_4_ranges = ", $gb_map_4_ranges,
 160       ", gb_map_4_max = ", $gb_map_4_max,
 161       ", uni_map_count = ", $uni_map_count, "\n";
 162 ($gb_map_2_count == 23940) or die "Bad gb_map_2_count != 23940";
 163 ($gb_map_4_max == $gb_map_4_count + $gb_map_4_ranges - 1)
 164     or die "Bad gb_map_4_max != gb_map_4_count + gb_map_4_ranges";
 165 ($uni_map_count + $gb_map_4_ranges == 0x10000 - (0xE000 - 0xD800) - 0x80)
 166     or die "Bad uni_map_count";
 167
 168 $range_index = 0;
 169 $gb_nonrangedataindex[$range_index] = $gb_map_2_count;
 170 for ($gb_code = 0; $gb_code < $gb_map_4_max; ++$gb_code)
 171 {
 172     if (defined($gb_map_4[$gb_code]))
 173     {
 174         $gb_map_2[$gb_map_2_count++] = $gb_map_4[$gb_code];
 175     }
 176     else
 177     {
 178         ($gb_code == $range_linear_first[$range_index]) or die "Bad input";
 179         $gb_code = $range_linear_last[$range_index];
 180         ++$range_index;
 181         $gb_nonrangedataindex[$range_index] = $gb_map_2_count;
 182     }
 183 }
 184 ($range_index == $range_count) or die "Bad input";
 185
 186 $filename = lc($id) . ".tab";
 187 open OUT, ("> " . $filename) or die "Cannot write " . $filename;
 188
 189 {
 190     $filename = lc($id). ".pl";
 191     open IN, $filename or die "Cannot read ". $filename;
 192     $first = 1;
 193     while (<IN>)
 194     {
 195         if (/^\#!.*$/)
 196         {
 197         }
 198         elsif (/^\#(\*.*)$/)
 199         {
 200             if ($first == 1)
 201             {
 202                 print OUT "/", $1, "\n";
 203                 $first = 0;
 204             }
 205             else
 206             {
 207                 print OUT " ", substr($1, 0, length($1) - 1), "/\n";
 208             }
 209         }
 210         elsif (/^\# (.*)$/)
 211         {
 212             print OUT " *", $1, "\n";
 213         }
 214         elsif (/^\#(.*)$/)
 215         {
 216             print OUT " *", $1, "\n";
 217         }
 218         else
 219         {
 220             goto done;
 221         }
 222     }
 223   done:
 224 }
 225
 226 print OUT "\n",
 227           "#include \"convertgb18030.h\"\n",
 228           "\n",
 229           "#include \"sal/types.h\"\n",
 230           "\n";
 231
 232 print OUT "static sal_Unicode const aImpl", $id, "ToUnicodeData[] = {\n  ";
 233 for ($gb_code = 0; $gb_code < $gb_map_2_count; ++$gb_code)
 234 {
 235     printf OUT "0x%04X,", $gb_map_2[$gb_code];
 236     if ($gb_code % 8 == 7 && $gb_code != $gb_map_2_count - 1)
 237     {
 238         print OUT "\n  ";
 239     }
 240 }
 241 print OUT "\n};\n\n";
 242
 243 print OUT "static ImplGb180302000ToUnicodeRange const\n    aImpl",
 244           $id,
 245           "ToUnicodeRanges[] = {\n";
 246 for ($range_index = 0; $range_index < $range_count; ++$range_index)
 247 {
 248     printf OUT "  { %d, %d, %d, 0x%04X },\n",
 249                $gb_nonrangedataindex[$range_index],
 250                $range_linear_first[$range_index],
 251                $range_linear_last[$range_index] + 1,
 252                $range_uni_first[$range_index];
 253 }
 254 print OUT "  { -1, 0, 0, 0 }\n};\n\n";
 255
 256 print OUT "static sal_uInt32 const aImplUnicodeTo", $id, "Data[] = {\n  ";
 257 $index = 0;
 258 $range_index = 0;
 259 $uni_nonrangedataindex[$range_index] = $index;
 260 for ($utf32 = 0x80; $utf32 <= 0xFFFF; ++$utf32)
 261 {
 262     if (defined($uni_map[$utf32]))
 263     {
 264         if ($index > 0 && ($index - 1) % 6 == 5)
 265         {
 266             print OUT "\n  ";
 267         }
 268         $bytes = $uni_map[$utf32];
 269         printf OUT ($bytes <= 0xFFFF ? "    0x%04X," : "0x%08X,"), $bytes;
 270         ++$index;
 271     }
 272     else
 273     {
 274         ($utf32 == $range_uni_first[$range_index]) or die "Bad input";
 275         $utf32 = $range_uni_last[$range_index];
 276         ++$range_index;
 277         $uni_nonrangedataindex[$range_index] = $index;
 278     }
 279 }
 280 ($range_index == $range_count) or die "Bad input";
 281 print OUT "\n};\n\n";
 282
 283 print OUT "static ImplUnicodeToGb180302000Range const\n    aImplUnicodeTo",
 284           $id,
 285           "Ranges[] = {\n";
 286 for ($range_index = 0; $range_index < $range_count; ++$range_index)
 287 {
 288     printf OUT "  { %d, 0x%04X, 0x%04X, %d },\n",
 289                $uni_nonrangedataindex[$range_index],
 290                $range_uni_first[$range_index],
 291                $range_uni_last[$range_index],
 292                $range_linear_first[$range_index];
 293 }
 294 print OUT "};\n";
 295
 296 close OUT;