i18npool/source/textconversion/genconv_dict.cxx

   1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
   2 /*
   3  * This file is part of the LibreOffice project.
   4  *
   5  * This Source Code Form is subject to the terms of the Mozilla Public
   6  * License, v. 2.0. If a copy of the MPL was not distributed with this
   7  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
   8  *
   9  * This file incorporates work covered by the following license notice:
  10  *
  11  *   Licensed to the Apache Software Foundation (ASF) under one or more
  12  *   contributor license agreements. See the NOTICE file distributed
  13  *   with this work for additional information regarding copyright
  14  *   ownership. The ASF licenses this file to you under the Apache
  15  *   License, Version 2.0 (the "License"); you may not use this file
  16  *   except in compliance with the License. You may obtain a copy of
  17  *   the License at http://www.apache.org/licenses/LICENSE-2.0 .
  18  */
  19
  20
  21 #include <stdio.h>
  22 #include <string.h>
  23 #include <stdlib.h>
  24 #include <errno.h>
  25 #include <sal/main.h>
  26 #include <sal/types.h>
  27 #include <rtl/strbuf.hxx>
  28 #include <rtl/ustring.hxx>
  29
  30 #include <vector>
  31
  32 void make_hhc_char(FILE *sfp, FILE *cfp);
  33 void make_stc_char(FILE *sfp, FILE *cfp);
  34 void make_stc_word(FILE *sfp, FILE *cfp);
  35
  36 /* Main Procedure */
  37
  38 SAL_IMPLEMENT_MAIN_WITH_ARGS(argc, argv)
  39 {
  40     FILE *sfp, *cfp;
  41
  42     if (argc < 4) exit(-1);
  43
  44
  45     sfp = fopen(argv[2], "rb"); // open the source file for read;
  46     if (sfp == nullptr)
  47     {
  48         fprintf(stderr, "Opening the dictionary source file %s for reading failed: %s\n", argv[1], strerror(errno));
  49         exit(1);
  50     }
  51
  52     // create the C source file to write
  53     cfp = fopen(argv[3], "wb");
  54     if (cfp == nullptr) {
  55         fclose(sfp);
  56         fprintf(stderr, "Opening %s for writing failed: %s\n", argv[3], strerror(errno));
  57         exit(1);
  58     }
  59
  60     fprintf(cfp, "/*\n");
  61     fprintf(cfp, " * Copyright(c) 1999 - 2000, Sun Microsystems, Inc.\n");
  62     fprintf(cfp, " * All Rights Reserved.\n");
  63     fprintf(cfp, " */\n\n");
  64     fprintf(cfp, "/* !!!The file is generated automatically. DONOT edit the file manually!!! */\n\n");
  65     fprintf(cfp, "#include <sal/types.h>\n");
  66     fprintf(cfp, "#include <textconversion.hxx>\n");
  67     fprintf(cfp, "\nextern \"C\" {\n");
  68
  69     if (strcmp(argv[1], "hhc_char") == 0)
  70         make_hhc_char(sfp, cfp);
  71     else if (strcmp(argv[1], "stc_char") == 0)
  72         make_stc_char(sfp, cfp);
  73     else if (strcmp(argv[1], "stc_word") == 0)
  74         make_stc_word(sfp, cfp);
  75
  76     fprintf (cfp, "}\n");
  77
  78     fclose(sfp);
  79     fclose(cfp);
  80
  81     return 0;
  82 } // end of main
  83
  84 // Hangul/Hanja character conversion
  85 void make_hhc_char(FILE *sfp, FILE *cfp)
  86 {
  87     sal_Int32 count, address, i, j, k;
  88     sal_Unicode Hanja2HangulData[0x10000];
  89     for (i = 0; i < 0x10000; i++) {
  90         Hanja2HangulData[i] = 0;
  91     }
  92     sal_uInt16 Hangul2HanjaData[10000][3];
  93
  94     // generate main dict. data array
  95     fprintf(cfp, "\nstatic const sal_Unicode Hangul2HanjaData[] = {");
  96
  97     sal_Char Cstr[1024];
  98     count = 0;
  99     address = 0;
 100     while (fgets(Cstr, 1024, sfp)) {
 101         // input file is in UTF-8 encoding (Hangul:Hanja)
 102         // don't convert last new line character to Ostr.
 103         OUString Ostr(Cstr, strlen(Cstr) - 1, RTL_TEXTENCODING_UTF8);
 104         sal_Int32  len = Ostr.getLength();
 105
 106         Hangul2HanjaData[count][0] = Ostr[0];
 107         Hangul2HanjaData[count][1] = sal::static_int_cast<sal_uInt16>( address );
 108         Hangul2HanjaData[count][2] = sal::static_int_cast<sal_uInt16>( len - 2 );
 109         count++;
 110
 111         for (i = 2; i < len; i++) {
 112             Hanja2HangulData[Ostr[i]] = Ostr[0];
 113             if (address++ % 16 == 0)
 114                 fprintf(cfp, "\n\t");
 115             fprintf(cfp, "0x%04x, ", Ostr[i]);
 116         }
 117     }
 118     fprintf(cfp, "\n};\n");
 119
 120     fprintf(cfp, "\nstatic const i18npool::Hangul_Index Hangul2HanjaIndex[] = {\n");
 121     for (i = 0; i < count; i++)
 122         fprintf(cfp, "\t{ 0x%04x, 0x%04x, 0x%02x },\n",
 123                         Hangul2HanjaData[i][0],
 124                         Hangul2HanjaData[i][1],
 125                         Hangul2HanjaData[i][2]);
 126     fprintf(cfp, "};\n");
 127
 128     fprintf(cfp, "\nstatic const sal_uInt16 Hanja2HangulIndex[] = {");
 129
 130     address=0;
 131     for (i = 0; i < 0x10; i++) {
 132         fprintf(cfp, "\n\t");
 133         for (j = 0; j < 0x10; j++) {
 134             for (k = 0; k < 0x100; k++) {
 135                 if (Hanja2HangulData[((i*0x10)+j)*0x100+k] != 0)
 136                     break;
 137             }
 138             fprintf(
 139                 cfp, "0x%04lx, ",
 140                 sal::static_int_cast< unsigned long >(
 141                     k < 0x100 ? (address++)*0x100 : 0xFFFF));
 142         }
 143     }
 144     fprintf(cfp, "\n};\n");
 145
 146     fprintf(cfp, "\nstatic const sal_Unicode Hanja2HangulData[] = {");
 147
 148     for (i = 0; i < 0x100; i++) {
 149         for (j = 0; j < 0x100; j++) {
 150             if (Hanja2HangulData[i*0x100+j] != 0)
 151                 break;
 152         }
 153         if (j < 0x100) {
 154             for (j = 0; j < 0x10; j++) {
 155                 fprintf(cfp, "\n\t");
 156                 for (k = 0; k < 0x10; k++) {
 157                     sal_Unicode c = Hanja2HangulData[((i*0x10+j)*0x10)+k];
 158                     fprintf(cfp, "0x%04x, ", c ? c : 0xFFFF);
 159                 }
 160             }
 161         }
 162     }
 163     fprintf(cfp, "\n};\n");
 164
 165     // create function to return arrays
 166     fprintf (cfp, "\tconst sal_Unicode* getHangul2HanjaData() { return Hangul2HanjaData; }\n");
 167     fprintf (cfp, "\tconst i18npool::Hangul_Index* getHangul2HanjaIndex() { return Hangul2HanjaIndex; }\n");
 168     fprintf (cfp, "\tsal_Int16 getHangul2HanjaIndexCount() { return sizeof(Hangul2HanjaIndex) / sizeof(i18npool::Hangul_Index); }\n");
 169     fprintf (cfp, "\tconst sal_uInt16* getHanja2HangulIndex() { return Hanja2HangulIndex; }\n");
 170     fprintf (cfp, "\tconst sal_Unicode* getHanja2HangulData() { return Hanja2HangulData; }\n");
 171 }
 172
 173 // Simplified/Traditional Chinese character conversion
 174 void make_stc_char(FILE *sfp, FILE *cfp)
 175 {
 176     sal_Int32 address, i, j, k;
 177     sal_Unicode SChinese2TChineseData[0x10000];
 178     sal_Unicode SChinese2VChineseData[0x10000];
 179     sal_Unicode TChinese2SChineseData[0x10000];
 180     for (i = 0; i < 0x10000; i++) {
 181         SChinese2TChineseData[i] = 0;
 182         SChinese2VChineseData[i] = 0;
 183         TChinese2SChineseData[i] = 0;
 184     }
 185
 186     sal_Char Cstr[1024];
 187     while (fgets(Cstr, 1024, sfp)) {
 188         // input file is in UTF-8 encoding (SChinese:TChinese)
 189         // don't convert last new line character to Ostr.
 190         OUString Ostr(Cstr, strlen(Cstr) - 1, RTL_TEXTENCODING_UTF8);
 191         sal_Int32  len = Ostr.getLength();
 192         if (Ostr[1] == 'v')
 193             SChinese2VChineseData[Ostr[0]] = Ostr[2];
 194         else {
 195             SChinese2TChineseData[Ostr[0]] = Ostr[2];
 196             if (SChinese2VChineseData[Ostr[0]] == 0)
 197                 SChinese2VChineseData[Ostr[0]] = Ostr[2];
 198         }
 199         for (i = 2; i < len; i++)
 200             TChinese2SChineseData[Ostr[i]] = Ostr[0];
 201     }
 202
 203     fprintf(cfp, "\nstatic const sal_uInt16 STC_CharIndex_S2T[] = {");
 204
 205     address=0;
 206     for (i = 0; i < 0x10; i++) {
 207         fprintf(cfp, "\n\t");
 208         for (j = 0; j < 0x10; j++) {
 209             for (k = 0; k < 0x100; k++) {
 210                 if (SChinese2TChineseData[((i*0x10)+j)*0x100+k] != 0)
 211                     break;
 212             }
 213             fprintf(
 214                 cfp, "0x%04lx, ",
 215                 sal::static_int_cast< unsigned long >(
 216                     k < 0x100 ? (address++)*0x100 : 0xFFFF));
 217         }
 218     }
 219     fprintf(cfp, "\n};\n");
 220
 221     fprintf(cfp, "\nstatic const sal_Unicode STC_CharData_S2T[] = {");
 222
 223     for (i = 0; i < 0x100; i++) {
 224         for (j = 0; j < 0x100; j++) {
 225             if (SChinese2TChineseData[i*0x100+j] != 0)
 226                 break;
 227         }
 228         if (j < 0x100) {
 229             for (j = 0; j < 0x10; j++) {
 230                 fprintf(cfp, "\n\t");
 231                 for (k = 0; k < 0x10; k++) {
 232                     sal_Unicode c = SChinese2TChineseData[((i*0x10+j)*0x10)+k];
 233                     fprintf(cfp, "0x%04x, ", c ? c : 0xFFFF);
 234                 }
 235             }
 236         }
 237     }
 238     fprintf(cfp, "\n};\n");
 239
 240     fprintf(cfp, "\nstatic const sal_uInt16 STC_CharIndex_S2V[] = {");
 241
 242     address=0;
 243     for (i = 0; i < 0x10; i++) {
 244         fprintf(cfp, "\n\t");
 245         for (j = 0; j < 0x10; j++) {
 246             for (k = 0; k < 0x100; k++) {
 247                 if (SChinese2VChineseData[((i*0x10)+j)*0x100+k] != 0)
 248                     break;
 249             }
 250             fprintf(
 251                 cfp, "0x%04lx, ",
 252                 sal::static_int_cast< unsigned long >(
 253                     k < 0x100 ? (address++)*0x100 : 0xFFFF));
 254         }
 255     }
 256     fprintf(cfp, "\n};\n");
 257
 258     fprintf(cfp, "\nstatic const sal_Unicode STC_CharData_S2V[] = {");
 259
 260     for (i = 0; i < 0x100; i++) {
 261         for (j = 0; j < 0x100; j++) {
 262             if (SChinese2VChineseData[i*0x100+j] != 0)
 263                 break;
 264         }
 265         if (j < 0x100) {
 266             for (j = 0; j < 0x10; j++) {
 267                 fprintf(cfp, "\n\t");
 268                 for (k = 0; k < 0x10; k++) {
 269                     sal_Unicode c = SChinese2VChineseData[((i*0x10+j)*0x10)+k];
 270                     fprintf(cfp, "0x%04x, ", c ? c : 0xFFFF);
 271                 }
 272             }
 273         }
 274     }
 275     fprintf(cfp, "\n};\n");
 276
 277     fprintf(cfp, "\nstatic const sal_uInt16 STC_CharIndex_T2S[] = {");
 278
 279     address=0;
 280     for (i = 0; i < 0x10; i++) {
 281         fprintf(cfp, "\n\t");
 282         for (j = 0; j < 0x10; j++) {
 283             for (k = 0; k < 0x100; k++) {
 284                 if (TChinese2SChineseData[((i*0x10)+j)*0x100+k] != 0)
 285                     break;
 286             }
 287             fprintf(
 288                 cfp, "0x%04lx, ",
 289                 sal::static_int_cast< unsigned long >(
 290                     k < 0x100 ? (address++)*0x100 : 0xFFFF));
 291         }
 292     }
 293     fprintf(cfp, "\n};\n");
 294
 295     fprintf(cfp, "\nstatic const sal_Unicode STC_CharData_T2S[] = {");
 296
 297     for (i = 0; i < 0x100; i++) {
 298         for (j = 0; j < 0x100; j++) {
 299             if (TChinese2SChineseData[i*0x100+j] != 0)
 300                 break;
 301         }
 302         if (j < 0x100) {
 303             for (j = 0; j < 0x10; j++) {
 304                 fprintf(cfp, "\n\t");
 305                 for (k = 0; k < 0x10; k++) {
 306                     sal_Unicode c = TChinese2SChineseData[((i*0x10+j)*0x10)+k];
 307                     fprintf(cfp, "0x%04x, ", c ? c : 0xFFFF);
 308                 }
 309             }
 310         }
 311     }
 312     fprintf(cfp, "\n};\n");
 313
 314     // create function to return arrays
 315     fprintf (cfp, "\tconst sal_uInt16* getSTC_CharIndex_S2T() { return STC_CharIndex_S2T; }\n");
 316     fprintf (cfp, "\tconst sal_Unicode* getSTC_CharData_S2T() { return STC_CharData_S2T; }\n");
 317     fprintf (cfp, "\tconst sal_uInt16* getSTC_CharIndex_S2V() { return STC_CharIndex_S2V; }\n");
 318     fprintf (cfp, "\tconst sal_Unicode* getSTC_CharData_S2V() { return STC_CharData_S2V; }\n");
 319     fprintf (cfp, "\tconst sal_uInt16* getSTC_CharIndex_T2S() { return STC_CharIndex_T2S; }\n");
 320     fprintf (cfp, "\tconst sal_Unicode* getSTC_CharData_T2S() { return STC_CharData_T2S; }\n");
 321 }
 322
 323
 324 typedef struct {
 325     sal_uInt16 address;
 326     sal_Int32 len;
 327     sal_Unicode *data;
 328 } Index;
 329
 330 extern "C" {
 331 int Index_comp(const void* s1, const void* s2)
 332 {
 333     Index const *p1 = static_cast<Index const *>(s1), *p2 = static_cast<Index const *>(s2);
 334     int result = p1->len - p2->len;
 335     for (int i = 0; result == 0 && i < p1->len; i++)
 336         result = *(p1->data+i) - *(p2->data+i);
 337     return result;
 338 }
 339 }
 340
 341 // Simplified/Traditional Chinese word conversion
 342 void make_stc_word(FILE *sfp, FILE *cfp)
 343 {
 344     sal_Int32 count, i, length;
 345     sal_Unicode STC_WordData[0x10000];
 346     std::vector<Index> STC_WordEntry_S2T(0x10000);
 347     std::vector<Index> STC_WordEntry_T2S(0x10000);
 348     sal_Int32 count_S2T = 0, count_T2S = 0;
 349     sal_Int32 line = 0, char_total = 0;
 350     sal_Char Cstr[1024];
 351
 352     while (fgets(Cstr, 1024, sfp)) {
 353         // input file is in UTF-8 encoding (SChinese:TChinese)
 354         // don't convert last new line character to Ostr.
 355         OUString Ostr(Cstr, strlen(Cstr) - 1, RTL_TEXTENCODING_UTF8);
 356         sal_Int32  len = Ostr.getLength();
 357         if (char_total + len + 1 > 0xFFFF) {
 358             fprintf(stderr, "Word Dictionary stc_word.dic is too big (line %ld)", sal::static_int_cast< long >(line));
 359             return;
 360         }
 361         sal_Int32 sep=-1, eq=-1, gt=-1, lt=-1;
 362         if (((sep = eq = Ostr.indexOf('=')) > 0) ||
 363             ((sep = gt = Ostr.indexOf('>')) > 0) ||
 364             ((sep = lt = Ostr.indexOf('<')) > 0)) {
 365
 366             if (eq > 0 || gt > 0) {
 367                 STC_WordEntry_S2T[count_S2T].address = sal::static_int_cast<sal_uInt16>( char_total );
 368                 STC_WordEntry_S2T[count_S2T].len = sep;
 369                 STC_WordEntry_S2T[count_S2T++].data = &STC_WordData[char_total];
 370             }
 371             if (eq > 0 || lt > 0) {
 372                 STC_WordEntry_T2S[count_T2S].address = sal::static_int_cast<sal_uInt16>( char_total + sep + 1 );
 373                 STC_WordEntry_T2S[count_T2S].len = len - sep - 1;
 374                 STC_WordEntry_T2S[count_T2S++].data = &STC_WordData[char_total + sep + 1];
 375             }
 376             for (i = 0; i < len; i++)
 377                 STC_WordData[char_total++] = (i == sep) ? 0 : Ostr[i];
 378             STC_WordData[char_total++] = 0;
 379         } else {
 380             fprintf(stderr, "Invalid entry in stc_word.dic (line %ld)", sal::static_int_cast< long >(line));
 381             return;
 382         }
 383         line++;
 384     }
 385
 386     if (char_total > 0) {
 387         fprintf(cfp, "\nstatic const sal_Unicode STC_WordData[] = {");
 388         for (i = 0; i < char_total; i++) {
 389             if (i % 32 == 0) fprintf(cfp, "\n\t");
 390             fprintf(cfp, "0x%04x, ", STC_WordData[i]);
 391         }
 392         fprintf(cfp, "\n};\n");
 393
 394         fprintf(cfp, "\nstatic sal_Int32 STC_WordData_Count = %ld;\n", sal::static_int_cast< long >(char_total));
 395
 396         // create function to return arrays
 397         fprintf (cfp, "\tconst sal_Unicode* getSTC_WordData(sal_Int32& count) { count = STC_WordData_Count; return STC_WordData; }\n");
 398     } else {
 399         fprintf (cfp, "\tconst sal_Unicode* getSTC_WordData(sal_Int32& count) { count = 0; return NULL; }\n");
 400     }
 401
 402     sal_uInt16 STC_WordIndex[0x100];
 403
 404     if (count_S2T > 0) {
 405         qsort(&STC_WordEntry_S2T[0], count_S2T, sizeof(Index), Index_comp);
 406
 407         fprintf(cfp, "\nstatic const sal_uInt16 STC_WordEntry_S2T[] = {");
 408         count = 0;
 409         length = 0;
 410         for (i = 0; i < count_S2T; i++) {
 411             if (i % 32 == 0) fprintf(cfp, "\n\t");
 412             fprintf(cfp, "0x%04x, ", STC_WordEntry_S2T[i].address);
 413             if (STC_WordEntry_S2T[i].len != length) {
 414                 length = STC_WordEntry_S2T[i].len;
 415                 while (count <= length)
 416                     STC_WordIndex[count++] = sal::static_int_cast<sal_uInt16>(i);
 417             }
 418         }
 419         fprintf(cfp, "\n};\n");
 420         STC_WordIndex[count++] = sal::static_int_cast<sal_uInt16>(i);
 421
 422         fprintf(cfp, "\nstatic const sal_uInt16 STC_WordIndex_S2T[] = {");
 423         for (i = 0; i < count; i++) {
 424             if (i % 16 == 0) fprintf(cfp, "\n\t");
 425             fprintf(cfp, "0x%04x, ", STC_WordIndex[i]);
 426         }
 427         fprintf(cfp, "\n};\n");
 428
 429         fprintf(cfp, "\nstatic sal_Int32 STC_WordIndex_S2T_Count = %ld;\n", sal::static_int_cast< long >(length));
 430         fprintf (cfp, "\tconst sal_uInt16* getSTC_WordEntry_S2T() { return STC_WordEntry_S2T; }\n");
 431         fprintf (cfp, "\tconst sal_uInt16* getSTC_WordIndex_S2T(sal_Int32& count) { count = STC_WordIndex_S2T_Count; return STC_WordIndex_S2T; }\n");
 432     } else {
 433         fprintf (cfp, "\tconst sal_uInt16* getSTC_WordEntry_S2T() { return NULL; }\n");
 434         fprintf (cfp, "\tconst sal_uInt16* getSTC_WordIndex_S2T(sal_Int32& count) { count = 0; return NULL; }\n");
 435     }
 436
 437     if (count_T2S > 0) {
 438         qsort(&STC_WordEntry_T2S[0], count_T2S, sizeof(Index), Index_comp);
 439
 440         fprintf(cfp, "\nstatic const sal_uInt16 STC_WordEntry_T2S[] = {");
 441         count = 0;
 442         length = 0;
 443         for (i = 0; i < count_T2S; i++) {
 444             if (i % 32 == 0) fprintf(cfp, "\n\t");
 445             fprintf(cfp, "0x%04x, ", STC_WordEntry_T2S[i].address);
 446             if (STC_WordEntry_T2S[i].len != length) {
 447                 length = STC_WordEntry_T2S[i].len;
 448                 while (count <= length)
 449                     STC_WordIndex[count++] = sal::static_int_cast<sal_uInt16>(i);
 450             }
 451         }
 452         STC_WordIndex[count++] = sal::static_int_cast<sal_uInt16>(i);
 453         fprintf(cfp, "\n};\n");
 454
 455         fprintf(cfp, "\nstatic const sal_uInt16 STC_WordIndex_T2S[] = {");
 456         for (i = 0; i < count; i++) {
 457             if (i % 16 == 0) fprintf(cfp, "\n\t");
 458             fprintf(cfp, "0x%04x, ",  STC_WordIndex[i]);
 459         }
 460         fprintf(cfp, "\n};\n");
 461
 462         fprintf(cfp, "\nstatic sal_Int32 STC_WordIndex_T2S_Count = %ld;\n\n", sal::static_int_cast< long >(length));
 463         fprintf (cfp, "\tconst sal_uInt16* getSTC_WordEntry_T2S() { return STC_WordEntry_T2S; }\n");
 464         fprintf (cfp, "\tconst sal_uInt16* getSTC_WordIndex_T2S(sal_Int32& count) { count = STC_WordIndex_T2S_Count; return STC_WordIndex_T2S; }\n");
 465     } else {
 466         fprintf (cfp, "\tconst sal_uInt16* getSTC_WordEntry_T2S() { return NULL; }\n");
 467         fprintf (cfp, "\tconst sal_uInt16* getSTC_WordIndex_T2S(sal_Int32& count) { count = 0; return NULL; }\n");
 468     }
 469 }
 470
 471 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */