src/strgen/strgen.cpp

   1 /*
   2  * This file is part of OpenTTD.
   3  * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
   4  * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
   5  * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
   6  */
   7
   8 /** @file strgen.cpp Tool to create computer readable (stand-alone) translation files. */
   9
  10 #include "../stdafx.h"
  11 #include "../core/endian_func.hpp"
  12 #include "../string_func.h"
  13 #include "../strings_type.h"
  14 #include "../misc/getoptdata.h"
  15 #include "../table/control_codes.h"
  16
  17 #include "strgen.h"
  18
  19 #include <stdarg.h>
  20 #include <exception>
  21
  22 #if !defined(_WIN32) || defined(__CYGWIN__)
  23 #include <unistd.h>
  24 #include <sys/stat.h>
  25 #endif
  26
  27 #if defined(_WIN32) || defined(__WATCOMC__)
  28 #include <direct.h>
  29 #endif /* _WIN32 || __WATCOMC__ */
  30
  31 #include "../table/strgen_tables.h"
  32
  33 #include "../safeguards.h"
  34
  35
  36 #ifdef _MSC_VER
  37 # define LINE_NUM_FMT(s) "%s (%d): warning: %s (" s ")\n"
  38 #else
  39 # define LINE_NUM_FMT(s) "%s:%d: " s ": %s\n"
  40 #endif
  41
  42 void CDECL strgen_warning(const char *s, ...)
  43 {
  44         char buf[1024];
  45         va_list va;
  46         va_start(va, s);
  47         vseprintf(buf, lastof(buf), s, va);
  48         va_end(va);
  49         fprintf(stderr, LINE_NUM_FMT("warning"), _file, _cur_line, buf);
  50         _warnings++;
  51 }
  52
  53 void CDECL strgen_error(const char *s, ...)
  54 {
  55         char buf[1024];
  56         va_list va;
  57         va_start(va, s);
  58         vseprintf(buf, lastof(buf), s, va);
  59         va_end(va);
  60         fprintf(stderr, LINE_NUM_FMT("error"), _file, _cur_line, buf);
  61         _errors++;
  62 }
  63
  64 void NORETURN CDECL strgen_fatal(const char *s, ...)
  65 {
  66         char buf[1024];
  67         va_list va;
  68         va_start(va, s);
  69         vseprintf(buf, lastof(buf), s, va);
  70         va_end(va);
  71         fprintf(stderr, LINE_NUM_FMT("FATAL"), _file, _cur_line, buf);
  72 #ifdef _MSC_VER
  73         fprintf(stderr, LINE_NUM_FMT("warning"), _file, _cur_line, "language is not compiled");
  74 #endif
  75         throw std::exception();
  76 }
  77
  78 void NORETURN CDECL error(const char *s, ...)
  79 {
  80         char buf[1024];
  81         va_list va;
  82         va_start(va, s);
  83         vseprintf(buf, lastof(buf), s, va);
  84         va_end(va);
  85         fprintf(stderr, LINE_NUM_FMT("FATAL"), _file, _cur_line, buf);
  86 #ifdef _MSC_VER
  87         fprintf(stderr, LINE_NUM_FMT("warning"), _file, _cur_line, "language is not compiled");
  88 #endif
  89         exit(2);
  90 }
  91
  92 /** A reader that simply reads using fopen. */
  93 struct FileStringReader : StringReader {
  94         FILE *fh; ///< The file we are reading.
  95
  96         /**
  97          * Create the reader.
  98          * @param data        The data to fill during reading.
  99          * @param file        The file we are reading.
 100          * @param master      Are we reading the master file?
 101          * @param translation Are we reading a translation?
 102          */
 103         FileStringReader(StringData &data, const char *file, bool master, bool translation) :
 104                         StringReader(data, file, master, translation)
 105         {
 106                 this->fh = fopen(file, "rb");
 107                 if (this->fh == nullptr) error("Could not open %s", file);
 108         }
 109
 110         /** Free/close the file. */
 111         virtual ~FileStringReader()
 112         {
 113                 fclose(this->fh);
 114         }
 115
 116         char *ReadLine(char *buffer, const char *last) override
 117         {
 118                 return fgets(buffer, ClampToU16(last - buffer + 1), this->fh);
 119         }
 120
 121         void HandlePragma(char *str) override;
 122
 123         void ParseFile() override
 124         {
 125                 this->StringReader::ParseFile();
 126
 127                 if (StrEmpty(_lang.name) || StrEmpty(_lang.own_name) || StrEmpty(_lang.isocode)) {
 128                         error("Language must include ##name, ##ownname and ##isocode");
 129                 }
 130         }
 131 };
 132
 133 void FileStringReader::HandlePragma(char *str)
 134 {
 135         if (!memcmp(str, "id ", 3)) {
 136                 this->data.next_string_id = strtoul(str + 3, nullptr, 0);
 137         } else if (!memcmp(str, "name ", 5)) {
 138                 strecpy(_lang.name, str + 5, lastof(_lang.name));
 139         } else if (!memcmp(str, "ownname ", 8)) {
 140                 strecpy(_lang.own_name, str + 8, lastof(_lang.own_name));
 141         } else if (!memcmp(str, "isocode ", 8)) {
 142                 strecpy(_lang.isocode, str + 8, lastof(_lang.isocode));
 143         } else if (!memcmp(str, "textdir ", 8)) {
 144                 if (!memcmp(str + 8, "ltr", 3)) {
 145                         _lang.text_dir = TD_LTR;
 146                 } else if (!memcmp(str + 8, "rtl", 3)) {
 147                         _lang.text_dir = TD_RTL;
 148                 } else {
 149                         error("Invalid textdir %s", str + 8);
 150                 }
 151         } else if (!memcmp(str, "digitsep ", 9)) {
 152                 str += 9;
 153                 strecpy(_lang.digit_group_separator, strcmp(str, "{NBSP}") == 0 ? NBSP : str, lastof(_lang.digit_group_separator));
 154         } else if (!memcmp(str, "digitsepcur ", 12)) {
 155                 str += 12;
 156                 strecpy(_lang.digit_group_separator_currency, strcmp(str, "{NBSP}") == 0 ? NBSP : str, lastof(_lang.digit_group_separator_currency));
 157         } else if (!memcmp(str, "decimalsep ", 11)) {
 158                 str += 11;
 159                 strecpy(_lang.digit_decimal_separator, strcmp(str, "{NBSP}") == 0 ? NBSP : str, lastof(_lang.digit_decimal_separator));
 160         } else if (!memcmp(str, "winlangid ", 10)) {
 161                 const char *buf = str + 10;
 162                 long langid = strtol(buf, nullptr, 16);
 163                 if (langid > (long)UINT16_MAX || langid < 0) {
 164                         error("Invalid winlangid %s", buf);
 165                 }
 166                 _lang.winlangid = (uint16)langid;
 167         } else if (!memcmp(str, "grflangid ", 10)) {
 168                 const char *buf = str + 10;
 169                 long langid = strtol(buf, nullptr, 16);
 170                 if (langid >= 0x7F || langid < 0) {
 171                         error("Invalid grflangid %s", buf);
 172                 }
 173                 _lang.newgrflangid = (uint8)langid;
 174         } else if (!memcmp(str, "gender ", 7)) {
 175                 if (this->master) error("Genders are not allowed in the base translation.");
 176                 char *buf = str + 7;
 177
 178                 for (;;) {
 179                         const char *s = ParseWord(&buf);
 180
 181                         if (s == nullptr) break;
 182                         if (_lang.num_genders >= MAX_NUM_GENDERS) error("Too many genders, max %d", MAX_NUM_GENDERS);
 183                         strecpy(_lang.genders[_lang.num_genders], s, lastof(_lang.genders[_lang.num_genders]));
 184                         _lang.num_genders++;
 185                 }
 186         } else if (!memcmp(str, "case ", 5)) {
 187                 if (this->master) error("Cases are not allowed in the base translation.");
 188                 char *buf = str + 5;
 189
 190                 for (;;) {
 191                         const char *s = ParseWord(&buf);
 192
 193                         if (s == nullptr) break;
 194                         if (_lang.num_cases >= MAX_NUM_CASES) error("Too many cases, max %d", MAX_NUM_CASES);
 195                         strecpy(_lang.cases[_lang.num_cases], s, lastof(_lang.cases[_lang.num_cases]));
 196                         _lang.num_cases++;
 197                 }
 198         } else {
 199                 StringReader::HandlePragma(str);
 200         }
 201 }
 202
 203 bool CompareFiles(const char *n1, const char *n2)
 204 {
 205         FILE *f2 = fopen(n2, "rb");
 206         if (f2 == nullptr) return false;
 207
 208         FILE *f1 = fopen(n1, "rb");
 209         if (f1 == nullptr) {
 210                 fclose(f2);
 211                 error("can't open %s", n1);
 212         }
 213
 214         size_t l1, l2;
 215         do {
 216                 char b1[4096];
 217                 char b2[4096];
 218                 l1 = fread(b1, 1, sizeof(b1), f1);
 219                 l2 = fread(b2, 1, sizeof(b2), f2);
 220
 221                 if (l1 != l2 || memcmp(b1, b2, l1)) {
 222                         fclose(f2);
 223                         fclose(f1);
 224                         return false;
 225                 }
 226         } while (l1 != 0);
 227
 228         fclose(f2);
 229         fclose(f1);
 230         return true;
 231 }
 232
 233 /** Base class for writing data to disk. */
 234 struct FileWriter {
 235         FILE *fh;             ///< The file handle we're writing to.
 236         const char *filename; ///< The file name we're writing to.
 237
 238         /**
 239          * Open a file to write to.
 240          * @param filename The file to open.
 241          */
 242         FileWriter(const char *filename)
 243         {
 244                 this->filename = stredup(filename);
 245                 this->fh = fopen(this->filename, "wb");
 246
 247                 if (this->fh == nullptr) {
 248                         error("Could not open %s", this->filename);
 249                 }
 250         }
 251
 252         /** Finalise the writing. */
 253         void Finalise()
 254         {
 255                 fclose(this->fh);
 256                 this->fh = nullptr;
 257         }
 258
 259         /** Make sure the file is closed. */
 260         virtual ~FileWriter()
 261         {
 262                 /* If we weren't closed an exception was thrown, so remove the temporary file. */
 263                 if (fh != nullptr) {
 264                         fclose(this->fh);
 265                         unlink(this->filename);
 266                 }
 267                 free(this->filename);
 268         }
 269 };
 270
 271 struct HeaderFileWriter : HeaderWriter, FileWriter {
 272         /** The real file name we eventually want to write to. */
 273         const char *real_filename;
 274         /** The previous string ID that was printed. */
 275         int prev;
 276
 277         /**
 278          * Open a file to write to.
 279          * @param filename The file to open.
 280          */
 281         HeaderFileWriter(const char *filename) : FileWriter("tmp.xxx"),
 282                 real_filename(stredup(filename)), prev(0)
 283         {
 284                 fprintf(this->fh, "/* This file is automatically generated. Do not modify */\n\n");
 285                 fprintf(this->fh, "#ifndef TABLE_STRINGS_H\n");
 286                 fprintf(this->fh, "#define TABLE_STRINGS_H\n");
 287         }
 288
 289         /** Free the filename. */
 290         ~HeaderFileWriter()
 291         {
 292                 free(real_filename);
 293         }
 294
 295         void WriteStringID(const char *name, int stringid)
 296         {
 297                 if (prev + 1 != stringid) fprintf(this->fh, "\n");
 298                 fprintf(this->fh, "static const StringID %s = 0x%X;\n", name, stringid);
 299                 prev = stringid;
 300         }
 301
 302         void Finalise(const StringData &data)
 303         {
 304                 /* Find the plural form with the most amount of cases. */
 305                 int max_plural_forms = 0;
 306                 for (uint i = 0; i < lengthof(_plural_forms); i++) {
 307                         max_plural_forms = max(max_plural_forms, _plural_forms[i].plural_count);
 308                 }
 309
 310                 fprintf(this->fh,
 311                         "\n"
 312                         "static const uint LANGUAGE_PACK_VERSION     = 0x%X;\n"
 313                         "static const uint LANGUAGE_MAX_PLURAL       = %u;\n"
 314                         "static const uint LANGUAGE_MAX_PLURAL_FORMS = %d;\n\n",
 315                         (uint)data.Version(), (uint)lengthof(_plural_forms), max_plural_forms
 316                 );
 317
 318                 fprintf(this->fh, "#endif /* TABLE_STRINGS_H */\n");
 319
 320                 this->FileWriter::Finalise();
 321
 322                 if (CompareFiles(this->filename, this->real_filename)) {
 323                         /* files are equal. tmp.xxx is not needed */
 324                         unlink(this->filename);
 325                 } else {
 326                         /* else rename tmp.xxx into filename */
 327 #       if defined(_WIN32)
 328                         unlink(this->real_filename);
 329 #       endif
 330                         if (rename(this->filename, this->real_filename) == -1) error("rename() failed");
 331                 }
 332         }
 333 };
 334
 335 /** Class for writing a language to disk. */
 336 struct LanguageFileWriter : LanguageWriter, FileWriter {
 337         /**
 338          * Open a file to write to.
 339          * @param filename The file to open.
 340          */
 341         LanguageFileWriter(const char *filename) : FileWriter(filename)
 342         {
 343         }
 344
 345         void WriteHeader(const LanguagePackHeader *header)
 346         {
 347                 this->Write((const byte *)header, sizeof(*header));
 348         }
 349
 350         void Finalise()
 351         {
 352                 if (fputc(0, this->fh) == EOF) {
 353                         error("Could not write to %s", this->filename);
 354                 }
 355                 this->FileWriter::Finalise();
 356         }
 357
 358         void Write(const byte *buffer, size_t length)
 359         {
 360                 if (fwrite(buffer, sizeof(*buffer), length, this->fh) != length) {
 361                         error("Could not write to %s", this->filename);
 362                 }
 363         }
 364 };
 365
 366 /** Multi-OS mkdirectory function */
 367 static inline void ottd_mkdir(const char *directory)
 368 {
 369         /* Ignore directory creation errors; they'll surface later on, and most
 370          * of the time they are 'directory already exists' errors anyhow. */
 371 #if defined(_WIN32) || defined(__WATCOMC__)
 372         mkdir(directory);
 373 #else
 374         mkdir(directory, 0755);
 375 #endif
 376 }
 377
 378 /**
 379  * Create a path consisting of an already existing path, a possible
 380  * path separator and the filename. The separator is only appended if the path
 381  * does not already end with a separator
 382  */
 383 static inline char *mkpath(char *buf, const char *last, const char *path, const char *file)
 384 {
 385         strecpy(buf, path, last); // copy directory into buffer
 386
 387         char *p = strchr(buf, '\0'); // add path separator if necessary
 388         if (p[-1] != PATHSEPCHAR && p != last) *p++ = PATHSEPCHAR;
 389         strecpy(p, file, last); // concatenate filename at end of buffer
 390         return buf;
 391 }
 392
 393 #if defined(_WIN32)
 394 /**
 395  * On MingW, it is common that both / as \ are accepted in the
 396  * params. To go with those flow, we rewrite all incoming /
 397  * simply to \, so internally we can safely assume \, and do
 398  * this for all Windows machines to keep identical behaviour,
 399  * no matter what your compiler was.
 400  */
 401 static inline char *replace_pathsep(char *s)
 402 {
 403         for (char *c = s; *c != '\0'; c++) if (*c == '/') *c = '\\';
 404         return s;
 405 }
 406 #else
 407 static inline char *replace_pathsep(char *s) { return s; }
 408 #endif
 409
 410 /** Options of strgen. */
 411 static const OptionData _opts[] = {
 412           GETOPT_NOVAL(     'v',  "--version"),
 413         GETOPT_GENERAL('C', '\0', "-export-commands", ODF_NO_VALUE),
 414         GETOPT_GENERAL('L', '\0', "-export-plurals",  ODF_NO_VALUE),
 415         GETOPT_GENERAL('P', '\0', "-export-pragmas",  ODF_NO_VALUE),
 416           GETOPT_NOVAL(     't',  "--todo"),
 417           GETOPT_NOVAL(     'w',  "--warning"),
 418           GETOPT_NOVAL(     'h',  "--help"),
 419         GETOPT_GENERAL('h', '?',  nullptr,            ODF_NO_VALUE),
 420           GETOPT_VALUE(     's',  "--source_dir"),
 421           GETOPT_VALUE(     'd',  "--dest_dir"),
 422         GETOPT_END(),
 423 };
 424
 425 int CDECL main(int argc, char *argv[])
 426 {
 427         char pathbuf[MAX_PATH];
 428         const char *src_dir = ".";
 429         const char *dest_dir = nullptr;
 430
 431         GetOptData mgo(argc - 1, argv + 1, _opts);
 432         for (;;) {
 433                 int i = mgo.GetOpt();
 434                 if (i == -1) break;
 435
 436                 switch (i) {
 437                         case 'v':
 438                                 puts("$Revision$");
 439                                 return 0;
 440
 441                         case 'C':
 442                                 printf("args\tflags\tcommand\treplacement\n");
 443                                 for (const CmdStruct *cs = _cmd_structs; cs < endof(_cmd_structs); cs++) {
 444                                         char flags;
 445                                         if (cs->proc == EmitGender) {
 446                                                 flags = 'g'; // Command needs number of parameters defined by number of genders
 447                                         } else if (cs->proc == EmitPlural) {
 448                                                 flags = 'p'; // Command needs number of parameters defined by plural value
 449                                         } else if (cs->flags & C_DONTCOUNT) {
 450                                                 flags = 'i'; // Command may be in the translation when it is not in base
 451                                         } else {
 452                                                 flags = '0'; // Command needs no parameters
 453                                         }
 454                                         printf("%i\t%c\t\"%s\"\t\"%s\"\n", cs->consumes, flags, cs->cmd, strstr(cs->cmd, "STRING") ? "STRING" : cs->cmd);
 455                                 }
 456                                 return 0;
 457
 458                         case 'L':
 459                                 printf("count\tdescription\tnames\n");
 460                                 for (const PluralForm *pf = _plural_forms; pf < endof(_plural_forms); pf++) {
 461                                         printf("%i\t\"%s\"\t%s\n", pf->plural_count, pf->description, pf->names);
 462                                 }
 463                                 return 0;
 464
 465                         case 'P':
 466                                 printf("name\tflags\tdefault\tdescription\n");
 467                                 for (size_t i = 0; i < lengthof(_pragmas); i++) {
 468                                         printf("\"%s\"\t%s\t\"%s\"\t\"%s\"\n",
 469                                                         _pragmas[i][0], _pragmas[i][1], _pragmas[i][2], _pragmas[i][3]);
 470                                 }
 471                                 return 0;
 472
 473                         case 't':
 474                                 _show_todo |= 1;
 475                                 break;
 476
 477                         case 'w':
 478                                 _show_todo |= 2;
 479                                 break;
 480
 481                         case 'h':
 482                                 puts(
 483                                         "strgen - $Revision$\n"
 484                                         " -v | --version    print version information and exit\n"
 485                                         " -t | --todo       replace any untranslated strings with '<TODO>'\n"
 486                                         " -w | --warning    print a warning for any untranslated strings\n"
 487                                         " -h | -? | --help  print this help message and exit\n"
 488                                         " -s | --source_dir search for english.txt in the specified directory\n"
 489                                         " -d | --dest_dir   put output file in the specified directory, create if needed\n"
 490                                         " -export-commands  export all commands and exit\n"
 491                                         " -export-plurals   export all plural forms and exit\n"
 492                                         " -export-pragmas   export all pragmas and exit\n"
 493                                         " Run without parameters and strgen will search for english.txt and parse it,\n"
 494                                         " creating strings.h. Passing an argument, strgen will translate that language\n"
 495                                         " file using english.txt as a reference and output <language>.lng."
 496                                 );
 497                                 return 0;
 498
 499                         case 's':
 500                                 src_dir = replace_pathsep(mgo.opt);
 501                                 break;
 502
 503                         case 'd':
 504                                 dest_dir = replace_pathsep(mgo.opt);
 505                                 break;
 506
 507                         case -2:
 508                                 fprintf(stderr, "Invalid arguments\n");
 509                                 return 0;
 510                 }
 511         }
 512
 513         if (dest_dir == nullptr) dest_dir = src_dir; // if dest_dir is not specified, it equals src_dir
 514
 515         try {
 516                 /* strgen has two modes of operation. If no (free) arguments are passed
 517                  * strgen generates strings.h to the destination directory. If it is supplied
 518                  * with a (free) parameter the program will translate that language to destination
 519                  * directory. As input english.txt is parsed from the source directory */
 520                 if (mgo.numleft == 0) {
 521                         mkpath(pathbuf, lastof(pathbuf), src_dir, "english.txt");
 522
 523                         /* parse master file */
 524                         StringData data(TEXT_TAB_END);
 525                         FileStringReader master_reader(data, pathbuf, true, false);
 526                         master_reader.ParseFile();
 527                         if (_errors != 0) return 1;
 528
 529                         /* write strings.h */
 530                         ottd_mkdir(dest_dir);
 531                         mkpath(pathbuf, lastof(pathbuf), dest_dir, "strings.h");
 532
 533                         HeaderFileWriter writer(pathbuf);
 534                         writer.WriteHeader(data);
 535                         writer.Finalise(data);
 536                         if (_errors != 0) return 1;
 537                 } else if (mgo.numleft >= 1) {
 538                         char *r;
 539
 540                         mkpath(pathbuf, lastof(pathbuf), src_dir, "english.txt");
 541
 542                         StringData data(TEXT_TAB_END);
 543                         /* parse master file and check if target file is correct */
 544                         FileStringReader master_reader(data, pathbuf, true, false);
 545                         master_reader.ParseFile();
 546
 547                         for (int i = 0; i < mgo.numleft; i++) {
 548                                 data.FreeTranslation();
 549
 550                                 const char *translation = replace_pathsep(mgo.argv[i]);
 551                                 const char *file = strrchr(translation, PATHSEPCHAR);
 552                                 FileStringReader translation_reader(data, translation, false, file == nullptr || strcmp(file + 1, "english.txt") != 0);
 553                                 translation_reader.ParseFile(); // target file
 554                                 if (_errors != 0) return 1;
 555
 556                                 /* get the targetfile, strip any directories and append to destination path */
 557                                 r = strrchr(mgo.argv[i], PATHSEPCHAR);
 558                                 mkpath(pathbuf, lastof(pathbuf), dest_dir, (r != nullptr) ? &r[1] : mgo.argv[i]);
 559
 560                                 /* rename the .txt (input-extension) to .lng */
 561                                 r = strrchr(pathbuf, '.');
 562                                 if (r == nullptr || strcmp(r, ".txt") != 0) r = strchr(pathbuf, '\0');
 563                                 strecpy(r, ".lng", lastof(pathbuf));
 564
 565                                 LanguageFileWriter writer(pathbuf);
 566                                 writer.WriteLang(data);
 567                                 writer.Finalise();
 568
 569                                 /* if showing warnings, print a summary of the language */
 570                                 if ((_show_todo & 2) != 0) {
 571                                         fprintf(stdout, "%d warnings and %d errors for %s\n", _warnings, _errors, pathbuf);
 572                                 }
 573                         }
 574                 }
 575         } catch (...) {
 576                 return 2;
 577         }
 578
 579         return 0;
 580 }