src/strgen/strgen.cpp

   1 /* $Id$ */
   2
   3 /*
   4  * This file is part of OpenTTD.
   5  * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
   6  * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
   7  * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
   8  */
   9
  10 /** @file strgen.cpp Tool to create computer readable (stand-alone) translation files. */
  11
  12 #include "../stdafx.h"
  13 #include "../core/endian_func.hpp"
  14 #include "../string_func.h"
  15 #include "../strings_type.h"
  16 #include "../misc/getoptdata.h"
  17 #include "../table/control_codes.h"
  18
  19 #include "strgen.h"
  20
  21 #include <stdarg.h>
  22 #include <exception>
  23
  24 #if (!defined(WIN32) && !defined(WIN64)) || defined(__CYGWIN__)
  25 #include <unistd.h>
  26 #include <sys/stat.h>
  27 #endif
  28
  29 #if defined WIN32 || defined __WATCOMC__
  30 #include <direct.h>
  31 #endif /* WIN32 || __WATCOMC__ */
  32
  33 #ifdef __MORPHOS__
  34 #ifdef stderr
  35 #undef stderr
  36 #endif
  37 #define stderr stdout
  38 #endif /* __MORPHOS__ */
  39
  40 #include "../table/strgen_tables.h"
  41
  42 #include "../safeguards.h"
  43
  44
  45 #ifdef _MSC_VER
  46 # define LINE_NUM_FMT(s) "%s (%d): warning: %s (" s ")\n"
  47 #else
  48 # define LINE_NUM_FMT(s) "%s:%d: " s ": %s\n"
  49 #endif
  50
  51 void CDECL strgen_warning(const char *s, ...)
  52 {
  53         char buf[1024];
  54         va_list va;
  55         va_start(va, s);
  56         vseprintf(buf, lastof(buf), s, va);
  57         va_end(va);
  58         fprintf(stderr, LINE_NUM_FMT("warning"), _file, _cur_line, buf);
  59         _warnings++;
  60 }
  61
  62 void CDECL strgen_error(const char *s, ...)
  63 {
  64         char buf[1024];
  65         va_list va;
  66         va_start(va, s);
  67         vseprintf(buf, lastof(buf), s, va);
  68         va_end(va);
  69         fprintf(stderr, LINE_NUM_FMT("error"), _file, _cur_line, buf);
  70         _errors++;
  71 }
  72
  73 void NORETURN CDECL strgen_fatal(const char *s, ...)
  74 {
  75         char buf[1024];
  76         va_list va;
  77         va_start(va, s);
  78         vseprintf(buf, lastof(buf), s, va);
  79         va_end(va);
  80         fprintf(stderr, LINE_NUM_FMT("FATAL"), _file, _cur_line, buf);
  81 #ifdef _MSC_VER
  82         fprintf(stderr, LINE_NUM_FMT("warning"), _file, _cur_line, "language is not compiled");
  83 #endif
  84         throw std::exception();
  85 }
  86
  87 void NORETURN CDECL error(const char *s, ...)
  88 {
  89         char buf[1024];
  90         va_list va;
  91         va_start(va, s);
  92         vseprintf(buf, lastof(buf), s, va);
  93         va_end(va);
  94         fprintf(stderr, LINE_NUM_FMT("FATAL"), _file, _cur_line, buf);
  95 #ifdef _MSC_VER
  96         fprintf(stderr, LINE_NUM_FMT("warning"), _file, _cur_line, "language is not compiled");
  97 #endif
  98         exit(2);
  99 }
 100
 101 /** A reader that simply reads using fopen. */
 102 struct FileStringReader : StringReader {
 103         FILE *fh; ///< The file we are reading.
 104
 105         /**
 106          * Create the reader.
 107          * @param data        The data to fill during reading.
 108          * @param file        The file we are reading.
 109          * @param master      Are we reading the master file?
 110          * @param translation Are we reading a translation?
 111          */
 112         FileStringReader(StringData &data, const char *file, bool master, bool translation) :
 113                         StringReader(data, file, master, translation)
 114         {
 115                 this->fh = fopen(file, "rb");
 116                 if (this->fh == NULL) error("Could not open %s", file);
 117         }
 118
 119         /** Free/close the file. */
 120         virtual ~FileStringReader()
 121         {
 122                 fclose(this->fh);
 123         }
 124
 125         /* virtual */ char *ReadLine(char *buffer, const char *last)
 126         {
 127                 return fgets(buffer, ClampToU16(last - buffer + 1), this->fh);
 128         }
 129
 130         /* virtual */ void HandlePragma(char *str);
 131
 132         /* virtual */ void ParseFile()
 133         {
 134                 this->StringReader::ParseFile();
 135
 136                 if (StrEmpty(_lang.name) || StrEmpty(_lang.own_name) || StrEmpty(_lang.isocode)) {
 137                         error("Language must include ##name, ##ownname and ##isocode");
 138                 }
 139         }
 140 };
 141
 142 void FileStringReader::HandlePragma(char *str)
 143 {
 144         if (!memcmp(str, "id ", 3)) {
 145                 this->data.next_string_id = strtoul(str + 3, NULL, 0);
 146         } else if (!memcmp(str, "name ", 5)) {
 147                 strecpy(_lang.name, str + 5, lastof(_lang.name));
 148         } else if (!memcmp(str, "ownname ", 8)) {
 149                 strecpy(_lang.own_name, str + 8, lastof(_lang.own_name));
 150         } else if (!memcmp(str, "isocode ", 8)) {
 151                 strecpy(_lang.isocode, str + 8, lastof(_lang.isocode));
 152         } else if (!memcmp(str, "textdir ", 8)) {
 153                 if (!memcmp(str + 8, "ltr", 3)) {
 154                         _lang.text_dir = TD_LTR;
 155                 } else if (!memcmp(str + 8, "rtl", 3)) {
 156                         _lang.text_dir = TD_RTL;
 157                 } else {
 158                         error("Invalid textdir %s", str + 8);
 159                 }
 160         } else if (!memcmp(str, "digitsep ", 9)) {
 161                 str += 9;
 162                 strecpy(_lang.digit_group_separator, strcmp(str, "{NBSP}") == 0 ? NBSP : str, lastof(_lang.digit_group_separator));
 163         } else if (!memcmp(str, "digitsepcur ", 12)) {
 164                 str += 12;
 165                 strecpy(_lang.digit_group_separator_currency, strcmp(str, "{NBSP}") == 0 ? NBSP : str, lastof(_lang.digit_group_separator_currency));
 166         } else if (!memcmp(str, "decimalsep ", 11)) {
 167                 str += 11;
 168                 strecpy(_lang.digit_decimal_separator, strcmp(str, "{NBSP}") == 0 ? NBSP : str, lastof(_lang.digit_decimal_separator));
 169         } else if (!memcmp(str, "winlangid ", 10)) {
 170                 const char *buf = str + 10;
 171                 long langid = strtol(buf, NULL, 16);
 172                 if (langid > (long)UINT16_MAX || langid < 0) {
 173                         error("Invalid winlangid %s", buf);
 174                 }
 175                 _lang.winlangid = (uint16)langid;
 176         } else if (!memcmp(str, "grflangid ", 10)) {
 177                 const char *buf = str + 10;
 178                 long langid = strtol(buf, NULL, 16);
 179                 if (langid >= 0x7F || langid < 0) {
 180                         error("Invalid grflangid %s", buf);
 181                 }
 182                 _lang.newgrflangid = (uint8)langid;
 183         } else if (!memcmp(str, "gender ", 7)) {
 184                 if (this->master) error("Genders are not allowed in the base translation.");
 185                 char *buf = str + 7;
 186
 187                 for (;;) {
 188                         const char *s = ParseWord(&buf);
 189
 190                         if (s == NULL) break;
 191                         if (_lang.num_genders >= MAX_NUM_GENDERS) error("Too many genders, max %d", MAX_NUM_GENDERS);
 192                         strecpy(_lang.genders[_lang.num_genders], s, lastof(_lang.genders[_lang.num_genders]));
 193                         _lang.num_genders++;
 194                 }
 195         } else if (!memcmp(str, "case ", 5)) {
 196                 if (this->master) error("Cases are not allowed in the base translation.");
 197                 char *buf = str + 5;
 198
 199                 for (;;) {
 200                         const char *s = ParseWord(&buf);
 201
 202                         if (s == NULL) break;
 203                         if (_lang.num_cases >= MAX_NUM_CASES) error("Too many cases, max %d", MAX_NUM_CASES);
 204                         strecpy(_lang.cases[_lang.num_cases], s, lastof(_lang.cases[_lang.num_cases]));
 205                         _lang.num_cases++;
 206                 }
 207         } else {
 208                 StringReader::HandlePragma(str);
 209         }
 210 }
 211
 212 bool CompareFiles(const char *n1, const char *n2)
 213 {
 214         FILE *f2 = fopen(n2, "rb");
 215         if (f2 == NULL) return false;
 216
 217         FILE *f1 = fopen(n1, "rb");
 218         if (f1 == NULL) error("can't open %s", n1);
 219
 220         size_t l1, l2;
 221         do {
 222                 char b1[4096];
 223                 char b2[4096];
 224                 l1 = fread(b1, 1, sizeof(b1), f1);
 225                 l2 = fread(b2, 1, sizeof(b2), f2);
 226
 227                 if (l1 != l2 || memcmp(b1, b2, l1)) {
 228                         fclose(f2);
 229                         fclose(f1);
 230                         return false;
 231                 }
 232         } while (l1 != 0);
 233
 234         fclose(f2);
 235         fclose(f1);
 236         return true;
 237 }
 238
 239 /** Base class for writing data to disk. */
 240 struct FileWriter {
 241         FILE *fh;             ///< The file handle we're writing to.
 242         const char *filename; ///< The file name we're writing to.
 243
 244         /**
 245          * Open a file to write to.
 246          * @param filename The file to open.
 247          */
 248         FileWriter(const char *filename)
 249         {
 250                 this->filename = stredup(filename);
 251                 this->fh = fopen(this->filename, "wb");
 252
 253                 if (this->fh == NULL) {
 254                         error("Could not open %s", this->filename);
 255                 }
 256         }
 257
 258         /** Finalise the writing. */
 259         void Finalise()
 260         {
 261                 fclose(this->fh);
 262                 this->fh = NULL;
 263         }
 264
 265         /** Make sure the file is closed. */
 266         virtual ~FileWriter()
 267         {
 268                 /* If we weren't closed an exception was thrown, so remove the temporary file. */
 269                 if (fh != NULL) {
 270                         fclose(this->fh);
 271                         unlink(this->filename);
 272                 }
 273                 free(this->filename);
 274         }
 275 };
 276
 277 struct HeaderFileWriter : HeaderWriter, FileWriter {
 278         /** The real file name we eventually want to write to. */
 279         const char *real_filename;
 280         /** The previous string ID that was printed. */
 281         int prev;
 282
 283         /**
 284          * Open a file to write to.
 285          * @param filename The file to open.
 286          */
 287         HeaderFileWriter(const char *filename) : FileWriter("tmp.xxx"),
 288                 real_filename(stredup(filename)), prev(0)
 289         {
 290                 fprintf(this->fh, "/* This file is automatically generated. Do not modify */\n\n");
 291                 fprintf(this->fh, "#ifndef TABLE_STRINGS_H\n");
 292                 fprintf(this->fh, "#define TABLE_STRINGS_H\n");
 293         }
 294
 295         /** Free the filename. */
 296         ~HeaderFileWriter()
 297         {
 298                 free(real_filename);
 299         }
 300
 301         void WriteStringID(const char *name, int stringid)
 302         {
 303                 if (prev + 1 != stringid) fprintf(this->fh, "\n");
 304                 fprintf(this->fh, "static const StringID %s = 0x%X;\n", name, stringid);
 305                 prev = stringid;
 306         }
 307
 308         void Finalise(const StringData &data)
 309         {
 310                 /* Find the plural form with the most amount of cases. */
 311                 int max_plural_forms = 0;
 312                 for (uint i = 0; i < lengthof(_plural_forms); i++) {
 313                         max_plural_forms = max(max_plural_forms, _plural_forms[i].plural_count);
 314                 }
 315
 316                 fprintf(this->fh,
 317                         "\n"
 318                         "static const uint LANGUAGE_PACK_VERSION     = 0x%X;\n"
 319                         "static const uint LANGUAGE_MAX_PLURAL       = %d;\n"
 320                         "static const uint LANGUAGE_MAX_PLURAL_FORMS = %d;\n\n",
 321                         (uint)data.Version(), (uint)lengthof(_plural_forms), max_plural_forms
 322                 );
 323
 324                 fprintf(this->fh, "#endif /* TABLE_STRINGS_H */\n");
 325
 326                 this->FileWriter::Finalise();
 327
 328                 if (CompareFiles(this->filename, this->real_filename)) {
 329                         /* files are equal. tmp.xxx is not needed */
 330                         unlink(this->filename);
 331                 } else {
 332                         /* else rename tmp.xxx into filename */
 333         #if defined(WIN32) || defined(WIN64)
 334                         unlink(this->real_filename);
 335         #endif
 336                         if (rename(this->filename, this->real_filename) == -1) error("rename() failed");
 337                 }
 338         }
 339 };
 340
 341 /** Class for writing a language to disk. */
 342 struct LanguageFileWriter : LanguageWriter, FileWriter {
 343         /**
 344          * Open a file to write to.
 345          * @param filename The file to open.
 346          */
 347         LanguageFileWriter(const char *filename) : FileWriter(filename)
 348         {
 349         }
 350
 351         void WriteHeader(const LanguagePackHeader *header)
 352         {
 353                 this->Write((const byte *)header, sizeof(*header));
 354         }
 355
 356         void Finalise()
 357         {
 358                 if (fputc(0, this->fh) == EOF) {
 359                         error("Could not write to %s", this->filename);
 360                 }
 361                 this->FileWriter::Finalise();
 362         }
 363
 364         void Write(const byte *buffer, size_t length)
 365         {
 366                 if (fwrite(buffer, sizeof(*buffer), length, this->fh) != length) {
 367                         error("Could not write to %s", this->filename);
 368                 }
 369         }
 370 };
 371
 372 /** Multi-OS mkdirectory function */
 373 static inline void ottd_mkdir(const char *directory)
 374 {
 375         /* Ignore directory creation errors; they'll surface later on, and most
 376          * of the time they are 'directory already exists' errors anyhow. */
 377 #if defined(WIN32) || defined(__WATCOMC__)
 378         mkdir(directory);
 379 #else
 380         mkdir(directory, 0755);
 381 #endif
 382 }
 383
 384 /**
 385  * Create a path consisting of an already existing path, a possible
 386  * path separator and the filename. The separator is only appended if the path
 387  * does not already end with a separator
 388  */
 389 static inline char *mkpath(char *buf, const char *last, const char *path, const char *file)
 390 {
 391         strecpy(buf, path, last); // copy directory into buffer
 392
 393         char *p = strchr(buf, '\0'); // add path separator if necessary
 394         if (p[-1] != PATHSEPCHAR && p != last) *p++ = PATHSEPCHAR;
 395         strecpy(p, file, last); // concatenate filename at end of buffer
 396         return buf;
 397 }
 398
 399 #if defined(__MINGW32__)
 400 /**
 401  * On MingW, it is common that both / as \ are accepted in the
 402  * params. To go with those flow, we rewrite all incoming /
 403  * simply to \, so internally we can safely assume \.
 404  */
 405 static inline char *replace_pathsep(char *s)
 406 {
 407         for (char *c = s; *c != '\0'; c++) if (*c == '/') *c = '\\';
 408         return s;
 409 }
 410 #else
 411 static inline char *replace_pathsep(char *s) { return s; }
 412 #endif
 413
 414 /** Options of strgen. */
 415 static const OptionData _opts[] = {
 416           GETOPT_NOVAL(     'v',  "--version"),
 417         GETOPT_GENERAL('C', '\0', "-export-commands", ODF_NO_VALUE),
 418         GETOPT_GENERAL('L', '\0', "-export-plurals",  ODF_NO_VALUE),
 419         GETOPT_GENERAL('P', '\0', "-export-pragmas",  ODF_NO_VALUE),
 420           GETOPT_NOVAL(     't',  "--todo"),
 421           GETOPT_NOVAL(     'w',  "--warning"),
 422           GETOPT_NOVAL(     'h',  "--help"),
 423         GETOPT_GENERAL('h', '?',  NULL,               ODF_NO_VALUE),
 424           GETOPT_VALUE(     's',  "--source_dir"),
 425           GETOPT_VALUE(     'd',  "--dest_dir"),
 426         GETOPT_END(),
 427 };
 428
 429 int CDECL main(int argc, char *argv[])
 430 {
 431         char pathbuf[MAX_PATH];
 432         const char *src_dir = ".";
 433         const char *dest_dir = NULL;
 434
 435         GetOptData mgo(argc - 1, argv + 1, _opts);
 436         for (;;) {
 437                 int i = mgo.GetOpt();
 438                 if (i == -1) break;
 439
 440                 switch (i) {
 441                         case 'v':
 442                                 puts("$Revision$");
 443                                 return 0;
 444
 445                         case 'C':
 446                                 printf("args\tflags\tcommand\treplacement\n");
 447                                 for (const CmdStruct *cs = _cmd_structs; cs < endof(_cmd_structs); cs++) {
 448                                         char flags;
 449                                         if (cs->proc == EmitGender) {
 450                                                 flags = 'g'; // Command needs number of parameters defined by number of genders
 451                                         } else if (cs->proc == EmitPlural) {
 452                                                 flags = 'p'; // Command needs number of parameters defined by plural value
 453                                         } else if (cs->flags & C_DONTCOUNT) {
 454                                                 flags = 'i'; // Command may be in the translation when it is not in base
 455                                         } else {
 456                                                 flags = '0'; // Command needs no parameters
 457                                         }
 458                                         printf("%i\t%c\t\"%s\"\t\"%s\"\n", cs->consumes, flags, cs->cmd, strstr(cs->cmd, "STRING") ? "STRING" : cs->cmd);
 459                                 }
 460                                 return 0;
 461
 462                         case 'L':
 463                                 printf("count\tdescription\tnames\n");
 464                                 for (const PluralForm *pf = _plural_forms; pf < endof(_plural_forms); pf++) {
 465                                         printf("%i\t\"%s\"\t%s\n", pf->plural_count, pf->description, pf->names);
 466                                 }
 467                                 return 0;
 468
 469                         case 'P':
 470                                 printf("name\tflags\tdefault\tdescription\n");
 471                                 for (size_t i = 0; i < lengthof(_pragmas); i++) {
 472                                         printf("\"%s\"\t%s\t\"%s\"\t\"%s\"\n",
 473                                                         _pragmas[i][0], _pragmas[i][1], _pragmas[i][2], _pragmas[i][3]);
 474                                 }
 475                                 return 0;
 476
 477                         case 't':
 478                                 _show_todo |= 1;
 479                                 break;
 480
 481                         case 'w':
 482                                 _show_todo |= 2;
 483                                 break;
 484
 485                         case 'h':
 486                                 puts(
 487                                         "strgen - $Revision$\n"
 488                                         " -v | --version    print version information and exit\n"
 489                                         " -t | --todo       replace any untranslated strings with '<TODO>'\n"
 490                                         " -w | --warning    print a warning for any untranslated strings\n"
 491                                         " -h | -? | --help  print this help message and exit\n"
 492                                         " -s | --source_dir search for english.txt in the specified directory\n"
 493                                         " -d | --dest_dir   put output file in the specified directory, create if needed\n"
 494                                         " -export-commands  export all commands and exit\n"
 495                                         " -export-plurals   export all plural forms and exit\n"
 496                                         " -export-pragmas   export all pragmas and exit\n"
 497                                         " Run without parameters and strgen will search for english.txt and parse it,\n"
 498                                         " creating strings.h. Passing an argument, strgen will translate that language\n"
 499                                         " file using english.txt as a reference and output <language>.lng."
 500                                 );
 501                                 return 0;
 502
 503                         case 's':
 504                                 src_dir = replace_pathsep(mgo.opt);
 505                                 break;
 506
 507                         case 'd':
 508                                 dest_dir = replace_pathsep(mgo.opt);
 509                                 break;
 510
 511                         case -2:
 512                                 fprintf(stderr, "Invalid arguments\n");
 513                                 return 0;
 514                 }
 515         }
 516
 517         if (dest_dir == NULL) dest_dir = src_dir; // if dest_dir is not specified, it equals src_dir
 518
 519         try {
 520                 /* strgen has two modes of operation. If no (free) arguments are passed
 521                  * strgen generates strings.h to the destination directory. If it is supplied
 522                  * with a (free) parameter the program will translate that language to destination
 523                  * directory. As input english.txt is parsed from the source directory */
 524                 if (mgo.numleft == 0) {
 525                         mkpath(pathbuf, lastof(pathbuf), src_dir, "english.txt");
 526
 527                         /* parse master file */
 528                         StringData data(TEXT_TAB_END);
 529                         FileStringReader master_reader(data, pathbuf, true, false);
 530                         master_reader.ParseFile();
 531                         if (_errors != 0) return 1;
 532
 533                         /* write strings.h */
 534                         ottd_mkdir(dest_dir);
 535                         mkpath(pathbuf, lastof(pathbuf), dest_dir, "strings.h");
 536
 537                         HeaderFileWriter writer(pathbuf);
 538                         writer.WriteHeader(data);
 539                         writer.Finalise(data);
 540                 } else if (mgo.numleft >= 1) {
 541                         char *r;
 542
 543                         mkpath(pathbuf, lastof(pathbuf), src_dir, "english.txt");
 544
 545                         StringData data(TEXT_TAB_END);
 546                         /* parse master file and check if target file is correct */
 547                         FileStringReader master_reader(data, pathbuf, true, false);
 548                         master_reader.ParseFile();
 549
 550                         for (int i = 0; i < mgo.numleft; i++) {
 551                                 data.FreeTranslation();
 552
 553                                 const char *translation = replace_pathsep(mgo.argv[i]);
 554                                 const char *file = strrchr(translation, PATHSEPCHAR);
 555                                 FileStringReader translation_reader(data, translation, false, file == NULL || strcmp(file + 1, "english.txt") != 0);
 556                                 translation_reader.ParseFile(); // target file
 557                                 if (_errors != 0) return 1;
 558
 559                                 /* get the targetfile, strip any directories and append to destination path */
 560                                 r = strrchr(mgo.argv[i], PATHSEPCHAR);
 561                                 mkpath(pathbuf, lastof(pathbuf), dest_dir, (r != NULL) ? &r[1] : mgo.argv[i]);
 562
 563                                 /* rename the .txt (input-extension) to .lng */
 564                                 r = strrchr(pathbuf, '.');
 565                                 if (r == NULL || strcmp(r, ".txt") != 0) r = strchr(pathbuf, '\0');
 566                                 strecpy(r, ".lng", lastof(pathbuf));
 567
 568                                 LanguageFileWriter writer(pathbuf);
 569                                 writer.WriteLang(data);
 570                                 writer.Finalise();
 571
 572                                 /* if showing warnings, print a summary of the language */
 573                                 if ((_show_todo & 2) != 0) {
 574                                         fprintf(stdout, "%d warnings and %d errors for %s\n", _warnings, _errors, pathbuf);
 575                                 }
 576                         }
 577                 }
 578         } catch (...) {
 579                 return 2;
 580         }
 581
 582         return 0;
 583 }