src/strgen/strgen.cpp

   1 /*
   2  * This file is part of OpenTTD.
   3  * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
   4  * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
   5  * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
   6  */
   7
   8 /** @file strgen.cpp Tool to create computer readable (stand-alone) translation files. */
   9
  10 #include "../stdafx.h"
  11 #include "../core/endian_func.hpp"
  12 #include "../core/mem_func.hpp"
  13 #include "../error_func.h"
  14 #include "../string_func.h"
  15 #include "../strings_type.h"
  16 #include "../misc/getoptdata.h"
  17 #include "../table/control_codes.h"
  18 #include "../3rdparty/fmt/std.h"
  19
  20 #include "strgen.h"
  21
  22 #include <filesystem>
  23 #include <fstream>
  24
  25 #include "../table/strgen_tables.h"
  26
  27 #include "../safeguards.h"
  28
  29
  30 #ifdef _MSC_VER
  31 # define LINE_NUM_FMT(s) "{} ({}): warning: {} (" s ")\n"
  32 #else
  33 # define LINE_NUM_FMT(s) "{}:{}: " s ": {}\n"
  34 #endif
  35
  36 void StrgenWarningI(const std::string &msg)
  37 {
  38         if (_show_todo > 0) {
  39                 fmt::print(stderr, LINE_NUM_FMT("warning"), _file, _cur_line, msg);
  40         } else {
  41                 fmt::print(stderr, LINE_NUM_FMT("info"), _file, _cur_line, msg);
  42         }
  43         _warnings++;
  44 }
  45
  46 void StrgenErrorI(const std::string &msg)
  47 {
  48         fmt::print(stderr, LINE_NUM_FMT("error"), _file, _cur_line, msg);
  49         _errors++;
  50 }
  51
  52 [[noreturn]] void StrgenFatalI(const std::string &msg)
  53 {
  54         fmt::print(stderr, LINE_NUM_FMT("FATAL"), _file, _cur_line, msg);
  55 #ifdef _MSC_VER
  56         fmt::print(stderr, LINE_NUM_FMT("warning"), _file, _cur_line, "language is not compiled");
  57 #endif
  58         throw std::exception();
  59 }
  60
  61 [[noreturn]] void FatalErrorI(const std::string &msg)
  62 {
  63         fmt::print(stderr, LINE_NUM_FMT("FATAL"), _file, _cur_line, msg);
  64 #ifdef _MSC_VER
  65         fmt::print(stderr, LINE_NUM_FMT("warning"), _file, _cur_line, "language is not compiled");
  66 #endif
  67         exit(2);
  68 }
  69
  70 /** A reader that simply reads using fopen. */
  71 struct FileStringReader : StringReader {
  72         std::ifstream input_stream;
  73
  74         /**
  75          * Create the reader.
  76          * @param data        The data to fill during reading.
  77          * @param file        The file we are reading.
  78          * @param master      Are we reading the master file?
  79          * @param translation Are we reading a translation?
  80          */
  81         FileStringReader(StringData &data, const std::filesystem::path &file, bool master, bool translation) :
  82                         StringReader(data, file.generic_string(), master, translation)
  83         {
  84                 this->input_stream.open(file, std::ifstream::binary);
  85         }
  86
  87         std::optional<std::string> ReadLine() override
  88         {
  89                 std::string result;
  90                 if (!std::getline(this->input_stream, result)) return std::nullopt;
  91                 return result;
  92         }
  93
  94         void HandlePragma(char *str) override;
  95
  96         void ParseFile() override
  97         {
  98                 this->StringReader::ParseFile();
  99
 100                 if (StrEmpty(_lang.name) || StrEmpty(_lang.own_name) || StrEmpty(_lang.isocode)) {
 101                         FatalError("Language must include ##name, ##ownname and ##isocode");
 102                 }
 103         }
 104 };
 105
 106 void FileStringReader::HandlePragma(char *str)
 107 {
 108         if (!memcmp(str, "id ", 3)) {
 109                 this->data.next_string_id = std::strtoul(str + 3, nullptr, 0);
 110         } else if (!memcmp(str, "name ", 5)) {
 111                 strecpy(_lang.name, str + 5);
 112         } else if (!memcmp(str, "ownname ", 8)) {
 113                 strecpy(_lang.own_name, str + 8);
 114         } else if (!memcmp(str, "isocode ", 8)) {
 115                 strecpy(_lang.isocode, str + 8);
 116         } else if (!memcmp(str, "textdir ", 8)) {
 117                 if (!memcmp(str + 8, "ltr", 3)) {
 118                         _lang.text_dir = TD_LTR;
 119                 } else if (!memcmp(str + 8, "rtl", 3)) {
 120                         _lang.text_dir = TD_RTL;
 121                 } else {
 122                         FatalError("Invalid textdir {}", str + 8);
 123                 }
 124         } else if (!memcmp(str, "digitsep ", 9)) {
 125                 str += 9;
 126                 strecpy(_lang.digit_group_separator, strcmp(str, "{NBSP}") == 0 ? NBSP : str);
 127         } else if (!memcmp(str, "digitsepcur ", 12)) {
 128                 str += 12;
 129                 strecpy(_lang.digit_group_separator_currency, strcmp(str, "{NBSP}") == 0 ? NBSP : str);
 130         } else if (!memcmp(str, "decimalsep ", 11)) {
 131                 str += 11;
 132                 strecpy(_lang.digit_decimal_separator, strcmp(str, "{NBSP}") == 0 ? NBSP : str);
 133         } else if (!memcmp(str, "winlangid ", 10)) {
 134                 const char *buf = str + 10;
 135                 long langid = std::strtol(buf, nullptr, 16);
 136                 if (langid > (long)UINT16_MAX || langid < 0) {
 137                         FatalError("Invalid winlangid {}", buf);
 138                 }
 139                 _lang.winlangid = (uint16_t)langid;
 140         } else if (!memcmp(str, "grflangid ", 10)) {
 141                 const char *buf = str + 10;
 142                 long langid = std::strtol(buf, nullptr, 16);
 143                 if (langid >= 0x7F || langid < 0) {
 144                         FatalError("Invalid grflangid {}", buf);
 145                 }
 146                 _lang.newgrflangid = (uint8_t)langid;
 147         } else if (!memcmp(str, "gender ", 7)) {
 148                 if (this->master) FatalError("Genders are not allowed in the base translation.");
 149                 char *buf = str + 7;
 150
 151                 for (;;) {
 152                         const char *s = ParseWord(&buf);
 153
 154                         if (s == nullptr) break;
 155                         if (_lang.num_genders >= MAX_NUM_GENDERS) FatalError("Too many genders, max {}", MAX_NUM_GENDERS);
 156                         strecpy(_lang.genders[_lang.num_genders], s);
 157                         _lang.num_genders++;
 158                 }
 159         } else if (!memcmp(str, "case ", 5)) {
 160                 if (this->master) FatalError("Cases are not allowed in the base translation.");
 161                 char *buf = str + 5;
 162
 163                 for (;;) {
 164                         const char *s = ParseWord(&buf);
 165
 166                         if (s == nullptr) break;
 167                         if (_lang.num_cases >= MAX_NUM_CASES) FatalError("Too many cases, max {}", MAX_NUM_CASES);
 168                         strecpy(_lang.cases[_lang.num_cases], s);
 169                         _lang.num_cases++;
 170                 }
 171         } else {
 172                 StringReader::HandlePragma(str);
 173         }
 174 }
 175
 176 bool CompareFiles(const std::filesystem::path &path1, const std::filesystem::path &path2)
 177 {
 178         /* Check for equal size, but ignore the error code for cases when a file does not exist. */
 179         std::error_code error_code;
 180         if (std::filesystem::file_size(path1, error_code) != std::filesystem::file_size(path2, error_code)) return false;
 181
 182         std::ifstream stream1(path1, std::ifstream::binary);
 183         std::ifstream stream2(path2, std::ifstream::binary);
 184
 185         return std::equal(std::istreambuf_iterator<char>(stream1.rdbuf()),
 186                         std::istreambuf_iterator<char>(),
 187                         std::istreambuf_iterator<char>(stream2.rdbuf()));
 188 }
 189
 190 /** Base class for writing data to disk. */
 191 struct FileWriter {
 192         std::ofstream output_stream; ///< The stream to write all the output to.
 193         const std::filesystem::path path; ///< The file name we're writing to.
 194
 195         /**
 196          * Open a file to write to.
 197          * @param path The path to the file to open.
 198          * @param openmode The openmode flags for opening the file.
 199          */
 200         FileWriter(const std::filesystem::path &path, std::ios_base::openmode openmode) : path(path)
 201         {
 202                 this->output_stream.open(path, openmode);
 203         }
 204
 205         /** Finalise the writing. */
 206         void Finalise()
 207         {
 208                 this->output_stream.close();
 209         }
 210
 211         /** Make sure the file is closed. */
 212         virtual ~FileWriter()
 213         {
 214                 /* If we weren't closed an exception was thrown, so remove the temporary file. */
 215                 if (this->output_stream.is_open()) {
 216                         this->output_stream.close();
 217                         std::filesystem::remove(this->path);
 218                 }
 219         }
 220 };
 221
 222 struct HeaderFileWriter : HeaderWriter, FileWriter {
 223         /** The real path we eventually want to write to. */
 224         const std::filesystem::path real_path;
 225         /** The previous string ID that was printed. */
 226         int prev;
 227         uint total_strings;
 228
 229         /**
 230          * Open a file to write to.
 231          * @param path The path to the file to open.
 232          */
 233         HeaderFileWriter(const std::filesystem::path &path) : FileWriter("tmp.xxx", std::ofstream::out),
 234                 real_path(path), prev(0), total_strings(0)
 235         {
 236                 this->output_stream << "/* This file is automatically generated. Do not modify */\n\n";
 237                 this->output_stream << "#ifndef TABLE_STRINGS_H\n";
 238                 this->output_stream << "#define TABLE_STRINGS_H\n";
 239         }
 240
 241         void WriteStringID(const std::string &name, int stringid) override
 242         {
 243                 if (prev + 1 != stringid) this->output_stream << "\n";
 244                 fmt::print(this->output_stream, "static const StringID {} = 0x{:X};\n", name, stringid);
 245                 prev = stringid;
 246                 total_strings++;
 247         }
 248
 249         void Finalise(const StringData &data) override
 250         {
 251                 /* Find the plural form with the most amount of cases. */
 252                 int max_plural_forms = 0;
 253                 for (const auto &pf : _plural_forms) {
 254                         max_plural_forms = std::max(max_plural_forms, pf.plural_count);
 255                 }
 256
 257                 fmt::print(this->output_stream,
 258                         "\n"
 259                         "static const uint LANGUAGE_PACK_VERSION     = 0x{:X};\n"
 260                         "static const uint LANGUAGE_MAX_PLURAL       = {};\n"
 261                         "static const uint LANGUAGE_MAX_PLURAL_FORMS = {};\n"
 262                         "static const uint LANGUAGE_TOTAL_STRINGS    = {};\n"
 263                         "\n",
 264                         data.Version(), std::size(_plural_forms), max_plural_forms, total_strings
 265                 );
 266
 267                 this->output_stream << "#endif /* TABLE_STRINGS_H */\n";
 268
 269                 this->FileWriter::Finalise();
 270
 271                 std::error_code error_code;
 272                 if (CompareFiles(this->path, this->real_path)) {
 273                         /* files are equal. tmp.xxx is not needed */
 274                         std::filesystem::remove(this->path, error_code); // Just ignore the error
 275                 } else {
 276                         /* else rename tmp.xxx into filename */
 277                         std::filesystem::rename(this->path, this->real_path, error_code);
 278                         if (error_code) FatalError("rename({}, {}) failed: {}", this->path, this->real_path, error_code.message());
 279                 }
 280         }
 281 };
 282
 283 /** Class for writing a language to disk. */
 284 struct LanguageFileWriter : LanguageWriter, FileWriter {
 285         /**
 286          * Open a file to write to.
 287          * @param path The path to the file to open.
 288          */
 289         LanguageFileWriter(const std::filesystem::path &path) : FileWriter(path, std::ofstream::binary | std::ofstream::out)
 290         {
 291         }
 292
 293         void WriteHeader(const LanguagePackHeader *header) override
 294         {
 295                 this->Write((const uint8_t *)header, sizeof(*header));
 296         }
 297
 298         void Finalise() override
 299         {
 300                 this->output_stream.put(0);
 301                 this->FileWriter::Finalise();
 302         }
 303
 304         void Write(const uint8_t *buffer, size_t length) override
 305         {
 306                 this->output_stream.write((const char *)buffer, length);
 307         }
 308 };
 309
 310 /** Options of strgen. */
 311 static const OptionData _opts[] = {
 312         { .type = ODF_NO_VALUE, .id = 'C', .longname = "-export-commands" },
 313         { .type = ODF_NO_VALUE, .id = 'L', .longname = "-export-plurals" },
 314         { .type = ODF_NO_VALUE, .id = 'P', .longname = "-export-pragmas" },
 315         { .type = ODF_NO_VALUE, .id = 't', .shortname = 't', .longname = "--todo" },
 316         { .type = ODF_NO_VALUE, .id = 'w', .shortname = 'w', .longname = "--warning" },
 317         { .type = ODF_NO_VALUE, .id = 'h', .shortname = 'h', .longname = "--help" },
 318         { .type = ODF_NO_VALUE, .id = 'h', .shortname = '?' },
 319         { .type = ODF_HAS_VALUE, .id = 's', .shortname = 's', .longname = "--source_dir" },
 320         { .type = ODF_HAS_VALUE, .id = 'd', .shortname = 'd', .longname = "--dest_dir" },
 321 };
 322
 323 int CDECL main(int argc, char *argv[])
 324 {
 325         std::filesystem::path src_dir(".");
 326         std::filesystem::path dest_dir;
 327
 328         GetOptData mgo(std::span(argv + 1, argc - 1), _opts);
 329         for (;;) {
 330                 int i = mgo.GetOpt();
 331                 if (i == -1) break;
 332
 333                 switch (i) {
 334                         case 'C':
 335                                 fmt::print("args\tflags\tcommand\treplacement\n");
 336                                 for (const auto &cs : _cmd_structs) {
 337                                         char flags;
 338                                         if (cs.proc == EmitGender) {
 339                                                 flags = 'g'; // Command needs number of parameters defined by number of genders
 340                                         } else if (cs.proc == EmitPlural) {
 341                                                 flags = 'p'; // Command needs number of parameters defined by plural value
 342                                         } else if (cs.flags & C_DONTCOUNT) {
 343                                                 flags = 'i'; // Command may be in the translation when it is not in base
 344                                         } else {
 345                                                 flags = '0'; // Command needs no parameters
 346                                         }
 347                                         fmt::print("{}\t{:c}\t\"{}\"\t\"{}\"\n", cs.consumes, flags, cs.cmd, strstr(cs.cmd, "STRING") ? "STRING" : cs.cmd);
 348                                 }
 349                                 return 0;
 350
 351                         case 'L':
 352                                 fmt::print("count\tdescription\tnames\n");
 353                                 for (const auto &pf : _plural_forms) {
 354                                         fmt::print("{}\t\"{}\"\t{}\n", pf.plural_count, pf.description, pf.names);
 355                                 }
 356                                 return 0;
 357
 358                         case 'P':
 359                                 fmt::print("name\tflags\tdefault\tdescription\n");
 360                                 for (const auto &pragma : _pragmas) {
 361                                         fmt::print("\"{}\"\t{}\t\"{}\"\t\"{}\"\n",
 362                                                         pragma[0], pragma[1], pragma[2], pragma[3]);
 363                                 }
 364                                 return 0;
 365
 366                         case 't':
 367                                 _show_todo |= 1;
 368                                 break;
 369
 370                         case 'w':
 371                                 _show_todo |= 2;
 372                                 break;
 373
 374                         case 'h':
 375                                 fmt::print(
 376                                         "strgen\n"
 377                                         " -t | --todo       replace any untranslated strings with '<TODO>'\n"
 378                                         " -w | --warning    print a warning for any untranslated strings\n"
 379                                         " -h | -? | --help  print this help message and exit\n"
 380                                         " -s | --source_dir search for english.txt in the specified directory\n"
 381                                         " -d | --dest_dir   put output file in the specified directory, create if needed\n"
 382                                         " -export-commands  export all commands and exit\n"
 383                                         " -export-plurals   export all plural forms and exit\n"
 384                                         " -export-pragmas   export all pragmas and exit\n"
 385                                         " Run without parameters and strgen will search for english.txt and parse it,\n"
 386                                         " creating strings.h. Passing an argument, strgen will translate that language\n"
 387                                         " file using english.txt as a reference and output <language>.lng.\n"
 388                                 );
 389                                 return 0;
 390
 391                         case 's':
 392                                 src_dir = mgo.opt;
 393                                 break;
 394
 395                         case 'd':
 396                                 dest_dir = mgo.opt;
 397                                 break;
 398
 399                         case -2:
 400                                 fmt::print(stderr, "Invalid arguments\n");
 401                                 return 0;
 402                 }
 403         }
 404
 405         if (dest_dir.empty()) dest_dir = src_dir; // if dest_dir is not specified, it equals src_dir
 406
 407         try {
 408                 /* strgen has two modes of operation. If no (free) arguments are passed
 409                  * strgen generates strings.h to the destination directory. If it is supplied
 410                  * with a (free) parameter the program will translate that language to destination
 411                  * directory. As input english.txt is parsed from the source directory */
 412                 if (mgo.arguments.empty()) {
 413                         std::filesystem::path input_path = src_dir;
 414                         input_path /= "english.txt";
 415
 416                         /* parse master file */
 417                         StringData data(TEXT_TAB_END);
 418                         FileStringReader master_reader(data, input_path, true, false);
 419                         master_reader.ParseFile();
 420                         if (_errors != 0) return 1;
 421
 422                         /* write strings.h */
 423                         std::filesystem::path output_path = dest_dir;
 424                         std::filesystem::create_directories(dest_dir);
 425                         output_path /= "strings.h";
 426
 427                         HeaderFileWriter writer(output_path);
 428                         writer.WriteHeader(data);
 429                         writer.Finalise(data);
 430                         if (_errors != 0) return 1;
 431                 } else {
 432                         std::filesystem::path input_path = src_dir;
 433                         input_path /= "english.txt";
 434
 435                         StringData data(TEXT_TAB_END);
 436                         /* parse master file and check if target file is correct */
 437                         FileStringReader master_reader(data, input_path, true, false);
 438                         master_reader.ParseFile();
 439
 440                         for (auto &argument: mgo.arguments) {
 441                                 data.FreeTranslation();
 442
 443                                 std::filesystem::path lang_file = argument;
 444                                 FileStringReader translation_reader(data, lang_file, false, lang_file.filename() != "english.txt");
 445                                 translation_reader.ParseFile(); // target file
 446                                 if (_errors != 0) return 1;
 447
 448                                 /* get the targetfile, strip any directories and append to destination path */
 449                                 std::filesystem::path output_file = dest_dir;
 450                                 output_file /= lang_file.filename();
 451                                 output_file.replace_extension("lng");
 452
 453                                 LanguageFileWriter writer(output_file);
 454                                 writer.WriteLang(data);
 455                                 writer.Finalise();
 456
 457                                 /* if showing warnings, print a summary of the language */
 458                                 if ((_show_todo & 2) != 0) {
 459                                         fmt::print("{} warnings and {} errors for {}\n", _warnings, _errors, output_file);
 460                                 }
 461                         }
 462                 }
 463         } catch (...) {
 464                 return 2;
 465         }
 466
 467         return 0;
 468 }