l10ntools/source/localize.cxx

   1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
   2 /*
   3  * This file is part of the LibreOffice project.
   4  *
   5  * This Source Code Form is subject to the terms of the Mozilla Public
   6  * License, v. 2.0. If a copy of the MPL was not distributed with this
   7  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
   8  *
   9  * This file incorporates work covered by the following license notice:
  10  *
  11  *   Licensed to the Apache Software Foundation (ASF) under one or more
  12  *   contributor license agreements. See the NOTICE file distributed
  13  *   with this work for additional information regarding copyright
  14  *   ownership. The ASF licenses this file to you under the Apache
  15  *   License, Version 2.0 (the "License"); you may not use this file
  16  *   except in compliance with the License. You may obtain a copy of
  17  *   the License at http://www.apache.org/licenses/LICENSE-2.0 .
  18  */
  19
  20 #include <sal/config.h>
  21
  22 #include <cassert>
  23 #include <cstddef>
  24 #include <cstdlib>
  25 #include <iostream>
  26 #include <string>
  27 #include <string_view>
  28 #include <map>
  29 #include <vector>
  30 #include <algorithm>
  31
  32 #include <o3tl/string_view.hxx>
  33 #include <osl/file.h>
  34 #include <osl/file.hxx>
  35 #include <osl/thread.h>
  36 #include <rtl/string.h>
  37 #include <rtl/string.hxx>
  38 #include <rtl/textcvt.h>
  39 #include <rtl/strbuf.hxx>
  40 #include <rtl/ustring.h>
  41 #include <rtl/ustring.hxx>
  42 #include <sal/macros.h>
  43 #include <sal/main.h>
  44 #include <sal/types.h>
  45
  46 #include <po.hxx>
  47
  48 namespace {
  49
  50 bool matchList(
  51     std::u16string_view rUrl, const std::u16string_view* pList, size_t nLength)
  52 {
  53     for (size_t i = 0; i != nLength; ++i) {
  54         if (o3tl::ends_with(rUrl, pList[i])) {
  55             return true;
  56         }
  57     }
  58     return false;
  59 }
  60
  61 bool passesNegativeList(std::u16string_view rUrl) {
  62     static const std::u16string_view list[] = {
  63         u"/desktop/test/deployment/passive/help/en/help.tree",
  64         u"/desktop/test/deployment/passive/help/en/main.xhp",
  65         u"/dictionaries.xcu",
  66         u"/dictionaries/da_DK/help/da/help.tree",
  67         (u"/dictionaries/da_DK/help/da/"
  68          "org.openoffice.da.hunspell.dictionaries/page1.xhp"),
  69         (u"/dictionaries/da_DK/help/da/"
  70          "org.openoffice.da.hunspell.dictionaries/page2.xhp"),
  71         u"/dictionaries/hu_HU/help/hu/help.tree",
  72         (u"/dictionaries/hu_HU/help/hu/"
  73          "org.openoffice.hu.hunspell.dictionaries/page1.xhp"),
  74         u"/officecfg/registry/data/org/openoffice/Office/Accelerators.xcu"
  75     };
  76     return !matchList(rUrl, list, SAL_N_ELEMENTS(list));
  77 }
  78
  79 bool passesPositiveList(std::u16string_view rUrl) {
  80     static const std::u16string_view list[] = {
  81         u"/description.xml"
  82     };
  83     return matchList(rUrl, list, SAL_N_ELEMENTS(list));
  84 }
  85
  86 void handleCommand(
  87     std::string_view rInPath, std::string_view rOutPath,
  88     const std::string& rExecutable)
  89 {
  90     OStringBuffer buf;
  91     if (rExecutable == "uiex" || rExecutable == "hrcex")
  92     {
  93         auto const env = getenv("SRC_ROOT");
  94         assert(env != nullptr);
  95         buf.append(env);
  96         buf.append("/solenv/bin/");
  97     }
  98     else
  99     {
 100         auto const env = getenv("WORKDIR_FOR_BUILD");
 101         assert(env != nullptr);
 102         buf.append(env);
 103         buf.append("/LinkTarget/Executable/");
 104     }
 105     buf.append(rExecutable.data());
 106     buf.append(" -i ");
 107     buf.append(rInPath);
 108     buf.append(" -o ");
 109     buf.append(rOutPath);
 110
 111     const OString cmd = buf.makeStringAndClear();
 112     if (system(cmd.getStr()) != 0)
 113     {
 114         std::cerr << "Error: Failed to execute " << cmd << '\n';
 115         throw false; //TODO
 116     }
 117 }
 118
 119 void InitPoFile(
 120     std::string_view rProject, const OString& rInPath,
 121     const OString& rPotDir, const OString& rOutPath )
 122 {
 123     //Create directory for po file
 124     {
 125         OUString outDir =
 126             OStringToOUString(
 127                 rPotDir.subView(0,rPotDir.lastIndexOf('/')), RTL_TEXTENCODING_UTF8);
 128         OUString outDirUrl;
 129         if (osl::FileBase::getFileURLFromSystemPath(outDir, outDirUrl)
 130             != osl::FileBase::E_None)
 131         {
 132             std::cerr
 133                 << ("Error: Cannot convert pathname to URL in " __FILE__
 134                     ", in line ")
 135                 << __LINE__ << "\n       outDir: "
 136                 << outDir
 137                 << "\n";
 138             throw false; //TODO
 139         }
 140         osl::Directory::createPath(outDirUrl);
 141     }
 142
 143     //Add header to the po file
 144     PoOfstream aPoOutPut;
 145     aPoOutPut.open(rOutPath.getStr());
 146     if (!aPoOutPut.isOpen())
 147     {
 148         std::cerr
 149             << "Error: Cannot open po file "
 150             << rOutPath << "\n";
 151         throw false; //TODO
 152     }
 153
 154     const sal_Int32 nProjectInd = rInPath.indexOf(rProject);
 155     const OString relativPath =
 156         rInPath.copy(nProjectInd, rInPath.lastIndexOf('/')- nProjectInd);
 157
 158     PoHeader aTmp(relativPath);
 159     aPoOutPut.writeHeader(aTmp);
 160     aPoOutPut.close();
 161 }
 162
 163 bool fileExists(const OString& fileName)
 164 {
 165     FILE *f = fopen(fileName.getStr(), "r");
 166
 167     if (f != nullptr)
 168     {
 169         fclose(f);
 170         return true;
 171     }
 172
 173     return false;
 174 }
 175
 176 OString gDestRoot;
 177
 178 bool handleFile(std::string_view rProject, const OUString& rUrl, const OString& rPotDir)
 179 {
 180     struct Command {
 181         std::u16string_view extension;
 182         std::string executable;
 183         bool positive;
 184     };
 185     static Command const commands[] = {
 186         { std::u16string_view(u".hrc"), "hrcex", false },
 187         { std::u16string_view(u".ulf"), "ulfex", false },
 188         { std::u16string_view(u".xcu"), "cfgex", false },
 189         { std::u16string_view(u".xrm"), "xrmex", false },
 190         { std::u16string_view(u"description.xml"), "xrmex", true },
 191         { std::u16string_view(u".xhp"), "helpex", false },
 192         { std::u16string_view(u".properties"), "propex", false },
 193         { std::u16string_view(u".ui"), "uiex", false },
 194         { std::u16string_view(u".tree"), "treex", false } };
 195     for (size_t i = 0; i != SAL_N_ELEMENTS(commands); ++i)
 196     {
 197         if (rUrl.endsWith(commands[i].extension) &&
 198             (commands[i].executable != "propex" || rUrl.indexOf("en_US") != -1))
 199         {
 200             if (commands[i].positive ? passesPositiveList(rUrl) : passesNegativeList(rUrl))
 201             {
 202                 //Get input file path
 203                 OString sInPath;
 204                 {
 205                     OUString sInPathTmp;
 206                     if (osl::FileBase::getSystemPathFromFileURL(rUrl, sInPathTmp) !=
 207                         osl::FileBase::E_None)
 208                     {
 209                         std::cerr << "osl::FileBase::getSystemPathFromFileURL(" << rUrl << ") failed\n";
 210                         throw false; //TODO
 211                     }
 212                     sInPath = OUStringToOString( sInPathTmp, RTL_TEXTENCODING_UTF8 );
 213                 }
 214                 OString sOutPath;
 215                 bool bCreatedFile = false;
 216                 bool bSimpleModuleCase = commands[i].executable == "uiex" || commands[i].executable == "hrcex";
 217                 if (bSimpleModuleCase)
 218                     sOutPath = gDestRoot + "/" + rProject + "/messages.pot";
 219                 else
 220                     sOutPath = rPotDir + ".pot";
 221
 222                 if (!fileExists(sOutPath))
 223                 {
 224                     InitPoFile(rProject, sInPath, rPotDir, sOutPath);
 225                     bCreatedFile = true;
 226                 }
 227                 handleCommand(sInPath, sOutPath, commands[i].executable);
 228
 229                 {
 230                     //Delete pot file if it contain only the header
 231                     PoIfstream aPOStream(sOutPath);
 232                     PoEntry aPO;
 233                     aPOStream.readEntry( aPO );
 234                     bool bDel = aPOStream.eof();
 235                     aPOStream.close();
 236
 237                     if (bDel)
 238                     {
 239                         if ( system(OString("rm " + sOutPath).getStr()) != 0 )
 240                         {
 241                             std::cerr
 242                                 << "Error: Cannot remove entryless pot file: "
 243                                 << sOutPath << "\n";
 244                             throw false; //TODO
 245                         }
 246                     }
 247                     else if (bCreatedFile && bSimpleModuleCase)
 248                     {
 249                         // add one stock Add, Cancel, Close, Help, No, OK, Yes entry to each module.po
 250                         // and duplicates in .ui files then filtered out by solenv/bin/uiex
 251
 252                         std::ofstream aOutPut;
 253                         aOutPut.open(sOutPath.getStr(), std::ios_base::out | std::ios_base::app);
 254
 255                         aOutPut << "#. wH3TZ\nmsgctxt \"stock\"\nmsgid \"_Add\"\nmsgstr \"\"\n\n";
 256                         aOutPut << "#. S9dsC\nmsgctxt \"stock\"\nmsgid \"_Apply\"\nmsgstr \"\"\n\n";
 257                         aOutPut << "#. TMo6G\nmsgctxt \"stock\"\nmsgid \"_Cancel\"\nmsgstr \"\"\n\n";
 258                         aOutPut << "#. MRCkv\nmsgctxt \"stock\"\nmsgid \"_Close\"\nmsgstr \"\"\n\n";
 259                         aOutPut << "#. nvx5t\nmsgctxt \"stock\"\nmsgid \"_Delete\"\nmsgstr \"\"\n\n";
 260                         aOutPut << "#. YspCj\nmsgctxt \"stock\"\nmsgid \"_Edit\"\nmsgstr \"\"\n\n";
 261                         aOutPut << "#. imQxr\nmsgctxt \"stock\"\nmsgid \"_Help\"\nmsgstr \"\"\n\n";
 262                         aOutPut << "#. RbjyB\nmsgctxt \"stock\"\nmsgid \"_New\"\nmsgstr \"\"\n\n";
 263                         aOutPut << "#. dx2yy\nmsgctxt \"stock\"\nmsgid \"_No\"\nmsgstr \"\"\n\n";
 264                         aOutPut << "#. M9DsL\nmsgctxt \"stock\"\nmsgid \"_OK\"\nmsgstr \"\"\n\n";
 265                         aOutPut << "#. VtJS9\nmsgctxt \"stock\"\nmsgid \"_Remove\"\nmsgstr \"\"\n\n";
 266                         aOutPut << "#. C69Fy\nmsgctxt \"stock\"\nmsgid \"_Reset\"\nmsgstr \"\"\n\n";
 267                         aOutPut << "#. mgpxh\nmsgctxt \"stock\"\nmsgid \"_Yes\"\nmsgstr \"\"\n";
 268
 269                         aOutPut.close();
 270                     }
 271                 }
 272
 273
 274                 return true;
 275             }
 276             break;
 277         }
 278     }
 279     return false;
 280 }
 281
 282 void handleFilesOfDir(
 283     std::vector<OUString>& aFiles, std::string_view rProject,
 284     const OString& rPotDir )
 285 {
 286     ///Handle files in lexical order
 287     std::sort(aFiles.begin(), aFiles.end());
 288
 289     for (auto const& elem : aFiles)
 290         handleFile(rProject, elem, rPotDir);
 291 }
 292
 293 bool includeProject(std::string_view rProject) {
 294     static const char *projects[] = {
 295         "include",
 296         "accessibility",
 297         "avmedia",
 298         "basctl",
 299         "basic",
 300         "chart2",
 301         "connectivity",
 302         "cui",
 303         "dbaccess",
 304         "desktop",
 305         "dictionaries",
 306         "editeng",
 307         "extensions",
 308         "extras",
 309         "filter",
 310         "forms",
 311         "formula",
 312         "fpicker",
 313         "framework",
 314         "helpcontent2",
 315         "instsetoo_native",
 316         "librelogo",
 317         "mysqlc",
 318         "nlpsolver",
 319         "officecfg",
 320         "oox",
 321         "readlicense_oo",
 322         "reportbuilder",
 323         "reportdesign",
 324         "sc",
 325         "scaddins",
 326         "sccomp",
 327         "scp2",
 328         "sd",
 329         "sdext",
 330         "setup_native",
 331         "sfx2",
 332         "shell",
 333         "starmath",
 334         "svl",
 335         "svtools",
 336         "svx",
 337         "sw",
 338         "swext",
 339         "sysui",
 340         "uui",
 341         "vcl",
 342         "wizards",
 343         "writerperfect",
 344         "xmlsecurity" };
 345     for (size_t i = 0; i != SAL_N_ELEMENTS(projects); ++i) {
 346         if (rProject == projects[i]) {
 347             return true;
 348         }
 349     }
 350     return false;
 351 }
 352
 353 /// Handle one directory in the hierarchy.
 354 ///
 355 /// Ignores symlinks and instead explicitly descends into clone/* or src/*,
 356 /// as the Cygwin symlinks are not supported by osl::Directory on Windows.
 357 ///
 358 /// @param rUrl the absolute file URL of this directory
 359 ///
 360 /// @param nLevel 0 if this is the root directory (core repository)
 361 /// that contains the individual modules. 1 if it is a toplevel module and
 362 /// larger values for the subdirectories.
 363 ///
 364 /// @param rProject the name of the project (empty and ignored if nLevel <= 0)
 365 /// @param rPotDir the path of pot directory
 366 void handleDirectory(
 367     const OUString& rUrl, int nLevel,
 368     const OString& rProject, const OString& rPotDir)
 369 {
 370     osl::Directory dir(rUrl);
 371     if (dir.open() != osl::FileBase::E_None) {
 372         std::cerr
 373             << "Error: Cannot open directory: " << rUrl << '\n';
 374         throw false; //TODO
 375     }
 376     std::vector<OUString> aFileNames;
 377     std::map<OUString, std::map<OString, OString>> aSubDirs;
 378     for (;;) {
 379         osl::DirectoryItem item;
 380         osl::FileBase::RC e = dir.getNextItem(item);
 381         if (e == osl::FileBase::E_NOENT) {
 382             break;
 383         }
 384         if (e != osl::FileBase::E_None) {
 385             std::cerr << "Error: Cannot read directory\n";
 386             throw false; //TODO
 387         }
 388         osl::FileStatus stat(
 389             osl_FileStatus_Mask_Type | osl_FileStatus_Mask_FileName
 390             | osl_FileStatus_Mask_FileURL);
 391         if (item.getFileStatus(stat) != osl::FileBase::E_None) {
 392             std::cerr << "Error: Cannot get file status\n";
 393             throw false; //TODO
 394         }
 395         const OString sDirName =
 396             OUStringToOString(stat.getFileName(),RTL_TEXTENCODING_UTF8);
 397         switch (nLevel)
 398         {
 399             case 0: // a root directory
 400                 if (stat.getFileType() == osl::FileStatus::Directory && includeProject(sDirName))
 401                     aSubDirs[stat.getFileURL()][sDirName] = rPotDir + "/" + sDirName;
 402                 break;
 403             default:
 404                 if (stat.getFileType() == osl::FileStatus::Directory)
 405                     aSubDirs[stat.getFileURL()][rProject] = rPotDir + "/" + sDirName;
 406                 else
 407                     aFileNames.push_back(stat.getFileURL());
 408                 break;
 409         }
 410     }
 411
 412     OString aPotDir(rPotDir);
 413     if( !aFileNames.empty() )
 414     {
 415         OString aProject(rProject);
 416         if (aProject == "include" && nLevel > 1)
 417         {
 418             aProject = aPotDir.copy(aPotDir.lastIndexOf('/') + 1);
 419             aPotDir = aPotDir.subView(0, aPotDir.lastIndexOf("include")) + aProject + "/messages";
 420         }
 421         if (aProject != "include")
 422         {
 423             handleFilesOfDir(aFileNames, aProject, aPotDir);
 424         }
 425     }
 426
 427     if (dir.close() != osl::FileBase::E_None) {
 428         std::cerr << "Error: Cannot close directory\n";
 429         throw false; //TODO
 430     }
 431
 432     for (auto const& elem : aSubDirs)
 433         handleDirectory(elem.first, nLevel + 1, elem.second.begin()->first,
 434                         elem.second.begin()->second);
 435
 436     //Remove empty pot directory
 437     OUString sPoPath =
 438         OStringToOUString(
 439             aPotDir.subView(0,aPotDir.lastIndexOf('/')), RTL_TEXTENCODING_UTF8);
 440     OUString sPoUrl;
 441     if (osl::FileBase::getFileURLFromSystemPath(sPoPath, sPoUrl)
 442         != osl::FileBase::E_None)
 443     {
 444         std::cerr
 445             << ("Error: Cannot convert pathname to URL in " __FILE__
 446                 ", in line ")
 447             << __LINE__ << "\n"
 448             << sPoPath
 449             << "\n";
 450         throw false; //TODO
 451     }
 452     osl::Directory::remove(sPoUrl);
 453 }
 454
 455 void handleProjects(char const * sSourceRoot, char const * sDestRoot)
 456 {
 457     OUString root16;
 458     if (!rtl_convertStringToUString(
 459             &root16.pData, sSourceRoot, rtl_str_getLength(sSourceRoot),
 460             osl_getThreadTextEncoding(),
 461             (RTL_TEXTTOUNICODE_FLAGS_UNDEFINED_ERROR
 462              | RTL_TEXTTOUNICODE_FLAGS_MBUNDEFINED_ERROR
 463              | RTL_TEXTTOUNICODE_FLAGS_INVALID_ERROR)))
 464     {
 465         std::cerr << "Error: Cannot convert pathname to UTF-16\n";
 466         throw false; //TODO
 467     }
 468     OUString rootUrl;
 469     if (osl::FileBase::getFileURLFromSystemPath(root16, rootUrl)
 470         != osl::FileBase::E_None)
 471     {
 472         std::cerr
 473             << ("Error: Cannot convert pathname to URL in " __FILE__
 474                 ", in line ")
 475             << __LINE__ << "\n       root16: "
 476             << root16
 477             << "\n";
 478         throw false; //TODO
 479     }
 480     gDestRoot = OString(sDestRoot);
 481     handleDirectory(rootUrl, 0, OString(), gDestRoot);
 482 }
 483 }
 484
 485 SAL_IMPLEMENT_MAIN_WITH_ARGS(argc, argv)
 486 {
 487     try
 488     {
 489         if (argc != 3)
 490         {
 491             std::cerr
 492                 << ("localize (c)2001 by Sun Microsystems\n\n"
 493                     "As part of the L10N framework, localize extracts en-US\n"
 494                     "strings for translation out of the toplevel modules defined\n"
 495                     "in projects array in l10ntools/source/localize.cxx.\n\n"
 496                     "Syntax: localize <source-root> <outfile>\n");
 497             exit(EXIT_FAILURE);
 498         }
 499         handleProjects(argv[1],argv[2]);
 500     }
 501     catch (std::exception& e)
 502     {
 503         std::cerr << "exception: " << e.what() << std::endl;
 504         return EXIT_FAILURE;
 505     }
 506     catch (bool) //TODO
 507     {
 508         return EXIT_FAILURE;
 509     }
 510     return EXIT_SUCCESS;
 511 }
 512
 513 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */