l10ntools/source/localize.cxx

   1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
   2 /*
   3  * This file is part of the LibreOffice project.
   4  *
   5  * This Source Code Form is subject to the terms of the Mozilla Public
   6  * License, v. 2.0. If a copy of the MPL was not distributed with this
   7  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
   8  *
   9  * This file incorporates work covered by the following license notice:
  10  *
  11  *   Licensed to the Apache Software Foundation (ASF) under one or more
  12  *   contributor license agreements. See the NOTICE file distributed
  13  *   with this work for additional information regarding copyright
  14  *   ownership. The ASF licenses this file to you under the Apache
  15  *   License, Version 2.0 (the "License"); you may not use this file
  16  *   except in compliance with the License. You may obtain a copy of
  17  *   the License at http://www.apache.org/licenses/LICENSE-2.0 .
  18  */
  19
  20 #include <sal/config.h>
  21
  22 #include <cassert>
  23 #include <cstdlib>
  24 #include <iostream>
  25 #include <string>
  26 #include <string_view>
  27 #include <map>
  28 #include <vector>
  29 #include <algorithm>
  30
  31 #include <o3tl/string_view.hxx>
  32 #include <osl/file.h>
  33 #include <osl/file.hxx>
  34 #include <osl/thread.h>
  35 #include <rtl/string.h>
  36 #include <rtl/string.hxx>
  37 #include <rtl/textcvt.h>
  38 #include <rtl/strbuf.hxx>
  39 #include <rtl/ustring.h>
  40 #include <rtl/ustring.hxx>
  41 #include <sal/macros.h>
  42 #include <sal/main.h>
  43 #include <sal/types.h>
  44
  45 #include <po.hxx>
  46
  47 namespace {
  48
  49 OString libraryPathEnvVarOverride;
  50
  51 bool matchList(
  52     std::u16string_view rUrl, const std::u16string_view* pList, size_t nLength)
  53 {
  54     for (size_t i = 0; i != nLength; ++i) {
  55         if (o3tl::ends_with(rUrl, pList[i])) {
  56             return true;
  57         }
  58     }
  59     return false;
  60 }
  61
  62 bool passesNegativeList(std::u16string_view rUrl) {
  63     static const std::u16string_view list[] = {
  64         u"/desktop/test/deployment/passive/help/en/help.tree",
  65         u"/desktop/test/deployment/passive/help/en/main.xhp",
  66         u"/dictionaries.xcu",
  67         u"/dictionaries/da_DK/help/da/help.tree",
  68         (u"/dictionaries/da_DK/help/da/"
  69          "org.openoffice.da.hunspell.dictionaries/page1.xhp"),
  70         (u"/dictionaries/da_DK/help/da/"
  71          "org.openoffice.da.hunspell.dictionaries/page2.xhp"),
  72         u"/dictionaries/hu_HU/help/hu/help.tree",
  73         (u"/dictionaries/hu_HU/help/hu/"
  74          "org.openoffice.hu.hunspell.dictionaries/page1.xhp"),
  75         u"/officecfg/registry/data/org/openoffice/Office/Accelerators.xcu"
  76     };
  77     return !matchList(rUrl, list, SAL_N_ELEMENTS(list));
  78 }
  79
  80 bool passesPositiveList(std::u16string_view rUrl) {
  81     static const std::u16string_view list[] = {
  82         u"/description.xml"
  83     };
  84     return matchList(rUrl, list, SAL_N_ELEMENTS(list));
  85 }
  86
  87 void handleCommand(
  88     std::string_view rInPath, std::string_view rOutPath,
  89     const std::string& rExecutable)
  90 {
  91     OStringBuffer buf;
  92     if (rExecutable == "uiex" || rExecutable == "hrcex")
  93     {
  94 #if !defined _WIN32
  95         // For now, this is only needed by some Linux ASan builds, so keep it simply and disable it
  96         // on  Windows (which doesn't support the relevant shell syntax for (un-)setting environment
  97         // variables).
  98         auto const n = libraryPathEnvVarOverride.indexOf('=');
  99         if (n == -1) {
 100             buf.append("unset -v " + libraryPathEnvVarOverride + " && ");
 101         } else {
 102             buf.append(libraryPathEnvVarOverride + " ");
 103         }
 104 #endif
 105         auto const env = getenv("SRC_ROOT");
 106         assert(env != nullptr);
 107         buf.append(OString::Concat(env) + "/solenv/bin/");
 108     }
 109     else
 110     {
 111 #if defined MACOSX
 112         if (auto const env = getenv("DYLD_LIBRARY_PATH")) {
 113             buf.append(OString::Concat("DYLD_LIBRARY_PATH=") + env + " ");
 114         }
 115 #endif
 116         auto const env = getenv("WORKDIR_FOR_BUILD");
 117         assert(env != nullptr);
 118         buf.append(OString::Concat(env) + "/LinkTarget/Executable/");
 119     }
 120     buf.append(OString::Concat(std::string_view(rExecutable))
 121         + " -i " + rInPath + " -o " + rOutPath);
 122
 123     if (system(buf.getStr()) != 0)
 124     {
 125         std::cerr << "Error: Failed to execute " << buf.getStr() << '\n';
 126         throw false; //TODO
 127     }
 128 }
 129
 130 void InitPoFile(
 131     std::string_view rProject, std::string_view rInPath,
 132     std::string_view rPotDir, const OString& rOutPath )
 133 {
 134     //Create directory for po file
 135     {
 136         OUString outDir =
 137             OStringToOUString(
 138                 rPotDir.substr(0,rPotDir.rfind('/')), RTL_TEXTENCODING_UTF8);
 139         OUString outDirUrl;
 140         if (osl::FileBase::getFileURLFromSystemPath(outDir, outDirUrl)
 141             != osl::FileBase::E_None)
 142         {
 143             std::cerr
 144                 << ("Error: Cannot convert pathname to URL in " __FILE__
 145                     ", in line ")
 146                 << __LINE__ << "\n       outDir: "
 147                 << outDir
 148                 << "\n";
 149             throw false; //TODO
 150         }
 151         osl::Directory::createPath(outDirUrl);
 152     }
 153
 154     //Add header to the po file
 155     PoOfstream aPoOutPut;
 156     aPoOutPut.open(rOutPath);
 157     if (!aPoOutPut.isOpen())
 158     {
 159         std::cerr
 160             << "Error: Cannot open po file "
 161             << rOutPath << "\n";
 162         throw false; //TODO
 163     }
 164
 165     const size_t nProjectInd = rInPath.find(rProject);
 166     const std::string_view relativPath =
 167         rInPath.substr(nProjectInd, rInPath.rfind('/')- nProjectInd);
 168
 169     PoHeader aTmp(relativPath);
 170     aPoOutPut.writeHeader(aTmp);
 171     aPoOutPut.close();
 172 }
 173
 174 bool fileExists(const OString& fileName)
 175 {
 176     FILE *f = fopen(fileName.getStr(), "r");
 177
 178     if (f != nullptr)
 179     {
 180         fclose(f);
 181         return true;
 182     }
 183
 184     return false;
 185 }
 186
 187 OString gDestRoot;
 188
 189 bool handleFile(std::string_view rProject, const OUString& rUrl, std::string_view rPotDir)
 190 {
 191     struct Command {
 192         std::u16string_view extension;
 193         std::string executable;
 194         bool positive;
 195     };
 196     static Command const commands[] = {
 197         { std::u16string_view(u".hrc"), "hrcex", false },
 198         { std::u16string_view(u".ulf"), "ulfex", false },
 199         { std::u16string_view(u".xcu"), "cfgex", false },
 200         { std::u16string_view(u".xrm"), "xrmex", false },
 201         { std::u16string_view(u"description.xml"), "xrmex", true },
 202         { std::u16string_view(u".xhp"), "helpex", false },
 203         { std::u16string_view(u".properties"), "propex", false },
 204         { std::u16string_view(u".ui"), "uiex", false },
 205         { std::u16string_view(u".tree"), "treex", false } };
 206     for (size_t i = 0; i != std::size(commands); ++i)
 207     {
 208         if (rUrl.endsWith(commands[i].extension) &&
 209             (commands[i].executable != "propex" || rUrl.indexOf("en_US") != -1))
 210         {
 211             if (commands[i].positive ? passesPositiveList(rUrl) : passesNegativeList(rUrl))
 212             {
 213                 //Get input file path
 214                 OString sInPath;
 215                 {
 216                     OUString sInPathTmp;
 217                     if (osl::FileBase::getSystemPathFromFileURL(rUrl, sInPathTmp) !=
 218                         osl::FileBase::E_None)
 219                     {
 220                         std::cerr << "osl::FileBase::getSystemPathFromFileURL(" << rUrl << ") failed\n";
 221                         throw false; //TODO
 222                     }
 223                     sInPath = OUStringToOString( sInPathTmp, RTL_TEXTENCODING_UTF8 );
 224                 }
 225                 OString sOutPath;
 226                 bool bCreatedFile = false;
 227                 bool bSimpleModuleCase = commands[i].executable == "uiex" || commands[i].executable == "hrcex";
 228                 if (bSimpleModuleCase)
 229                     sOutPath = gDestRoot + "/" + rProject + "/messages.pot";
 230                 else
 231                     sOutPath = OString::Concat(rPotDir) + ".pot";
 232
 233                 if (!fileExists(sOutPath))
 234                 {
 235                     InitPoFile(rProject, sInPath, rPotDir, sOutPath);
 236                     bCreatedFile = true;
 237                 }
 238                 handleCommand(sInPath, sOutPath, commands[i].executable);
 239
 240                 {
 241                     //Delete pot file if it contain only the header
 242                     PoIfstream aPOStream(sOutPath);
 243                     PoEntry aPO;
 244                     aPOStream.readEntry( aPO );
 245                     bool bDel = aPOStream.eof();
 246                     aPOStream.close();
 247
 248                     if (bDel)
 249                     {
 250                         if ( system(OString("rm " + sOutPath).getStr()) != 0 )
 251                         {
 252                             std::cerr
 253                                 << "Error: Cannot remove entryless pot file: "
 254                                 << sOutPath << "\n";
 255                             throw false; //TODO
 256                         }
 257                     }
 258                     else if (bCreatedFile && bSimpleModuleCase)
 259                     {
 260                         // add one stock Add, Cancel, Close, Help, No, OK, Yes entry to each module.po
 261                         // and duplicates in .ui files then filtered out by solenv/bin/uiex
 262
 263                         std::ofstream aOutPut;
 264                         aOutPut.open(sOutPath.getStr(), std::ios_base::out | std::ios_base::app);
 265
 266                         aOutPut << "#. wH3TZ\nmsgctxt \"stock\"\nmsgid \"_Add\"\nmsgstr \"\"\n\n";
 267                         aOutPut << "#. S9dsC\nmsgctxt \"stock\"\nmsgid \"_Apply\"\nmsgstr \"\"\n\n";
 268                         aOutPut << "#. TMo6G\nmsgctxt \"stock\"\nmsgid \"_Cancel\"\nmsgstr \"\"\n\n";
 269                         aOutPut << "#. MRCkv\nmsgctxt \"stock\"\nmsgid \"_Close\"\nmsgstr \"\"\n\n";
 270                         aOutPut << "#. nvx5t\nmsgctxt \"stock\"\nmsgid \"_Delete\"\nmsgstr \"\"\n\n";
 271                         aOutPut << "#. YspCj\nmsgctxt \"stock\"\nmsgid \"_Edit\"\nmsgstr \"\"\n\n";
 272                         aOutPut << "#. imQxr\nmsgctxt \"stock\"\nmsgid \"_Help\"\nmsgstr \"\"\n\n";
 273                         aOutPut << "#. RbjyB\nmsgctxt \"stock\"\nmsgid \"_New\"\nmsgstr \"\"\n\n";
 274                         aOutPut << "#. dx2yy\nmsgctxt \"stock\"\nmsgid \"_No\"\nmsgstr \"\"\n\n";
 275                         aOutPut << "#. M9DsL\nmsgctxt \"stock\"\nmsgid \"_OK\"\nmsgstr \"\"\n\n";
 276                         aOutPut << "#. VtJS9\nmsgctxt \"stock\"\nmsgid \"_Remove\"\nmsgstr \"\"\n\n";
 277                         aOutPut << "#. C69Fy\nmsgctxt \"stock\"\nmsgid \"_Reset\"\nmsgstr \"\"\n\n";
 278                         aOutPut << "#. mgpxh\nmsgctxt \"stock\"\nmsgid \"_Yes\"\nmsgstr \"\"\n";
 279
 280                         aOutPut.close();
 281                     }
 282                 }
 283
 284
 285                 return true;
 286             }
 287             break;
 288         }
 289     }
 290     return false;
 291 }
 292
 293 void handleFilesOfDir(
 294     std::vector<OUString>& aFiles, std::string_view rProject,
 295     std::string_view rPotDir )
 296 {
 297     ///Handle files in lexical order
 298     std::sort(aFiles.begin(), aFiles.end());
 299
 300     for (auto const& elem : aFiles)
 301         handleFile(rProject, elem, rPotDir);
 302 }
 303
 304 bool includeProject(std::string_view rProject) {
 305     static const char *projects[] = {
 306         "include",
 307         "accessibility",
 308         "avmedia",
 309         "basctl",
 310         "basic",
 311         "chart2",
 312         "connectivity",
 313         "cui",
 314         "dbaccess",
 315         "desktop",
 316         "dictionaries",
 317         "editeng",
 318         "extensions",
 319         "extras",
 320         "filter",
 321         "forms",
 322         "formula",
 323         "fpicker",
 324         "framework",
 325         "helpcontent2",
 326         "instsetoo_native",
 327         "librelogo",
 328         "mysqlc",
 329         "nlpsolver",
 330         "officecfg",
 331         "oox",
 332         "readlicense_oo",
 333         "reportbuilder",
 334         "reportdesign",
 335         "sc",
 336         "scaddins",
 337         "sccomp",
 338         "scp2",
 339         "sd",
 340         "sdext",
 341         "setup_native",
 342         "sfx2",
 343         "shell",
 344         "starmath",
 345         "svl",
 346         "svtools",
 347         "svx",
 348         "sw",
 349         "swext",
 350         "sysui",
 351         "uui",
 352         "vcl",
 353         "wizards",
 354         "writerperfect",
 355         "xmlsecurity" };
 356     for (size_t i = 0; i != SAL_N_ELEMENTS(projects); ++i) {
 357         if (rProject == projects[i]) {
 358             return true;
 359         }
 360     }
 361     return false;
 362 }
 363
 364 /// Handle one directory in the hierarchy.
 365 ///
 366 /// Ignores symlinks and instead explicitly descends into clone/* or src/*,
 367 /// as the Cygwin symlinks are not supported by osl::Directory on Windows.
 368 ///
 369 /// @param rUrl the absolute file URL of this directory
 370 ///
 371 /// @param nLevel 0 if this is the root directory (core repository)
 372 /// that contains the individual modules. 1 if it is a toplevel module and
 373 /// larger values for the subdirectories.
 374 ///
 375 /// @param rProject the name of the project (empty and ignored if nLevel <= 0)
 376 /// @param rPotDir the path of pot directory
 377 void handleDirectory(
 378     const OUString& rUrl, int nLevel,
 379     const OString& rProject, const OString& rPotDir)
 380 {
 381     osl::Directory dir(rUrl);
 382     if (dir.open() != osl::FileBase::E_None) {
 383         std::cerr
 384             << "Error: Cannot open directory: " << rUrl << '\n';
 385         throw false; //TODO
 386     }
 387     std::vector<OUString> aFileNames;
 388     std::map<OUString, std::map<OString, OString>> aSubDirs;
 389     for (;;) {
 390         osl::DirectoryItem item;
 391         osl::FileBase::RC e = dir.getNextItem(item);
 392         if (e == osl::FileBase::E_NOENT) {
 393             break;
 394         }
 395         if (e != osl::FileBase::E_None) {
 396             std::cerr << "Error: Cannot read directory\n";
 397             throw false; //TODO
 398         }
 399         osl::FileStatus stat(
 400             osl_FileStatus_Mask_Type | osl_FileStatus_Mask_FileName
 401             | osl_FileStatus_Mask_FileURL);
 402         if (item.getFileStatus(stat) != osl::FileBase::E_None) {
 403             std::cerr << "Error: Cannot get file status\n";
 404             throw false; //TODO
 405         }
 406         const OString sDirName =
 407             OUStringToOString(stat.getFileName(),RTL_TEXTENCODING_UTF8);
 408         switch (nLevel)
 409         {
 410             case 0: // a root directory
 411                 if (stat.getFileType() == osl::FileStatus::Directory && includeProject(sDirName))
 412                     aSubDirs[stat.getFileURL()][sDirName] = rPotDir + "/" + sDirName;
 413                 break;
 414             default:
 415                 if (stat.getFileType() == osl::FileStatus::Directory)
 416                     aSubDirs[stat.getFileURL()][rProject] = rPotDir + "/" + sDirName;
 417                 else
 418                     aFileNames.push_back(stat.getFileURL());
 419                 break;
 420         }
 421     }
 422
 423     OString aPotDir(rPotDir);
 424     if( !aFileNames.empty() )
 425     {
 426         OString aProject(rProject);
 427         if (aProject == "include" && nLevel > 1)
 428         {
 429             aProject = aPotDir.copy(aPotDir.lastIndexOf('/') + 1);
 430             aPotDir = aPotDir.subView(0, aPotDir.lastIndexOf("include")) + aProject + "/messages";
 431         }
 432         if (aProject != "include")
 433         {
 434             handleFilesOfDir(aFileNames, aProject, aPotDir);
 435         }
 436     }
 437
 438     if (dir.close() != osl::FileBase::E_None) {
 439         std::cerr << "Error: Cannot close directory\n";
 440         throw false; //TODO
 441     }
 442
 443     for (auto const& elem : aSubDirs)
 444         handleDirectory(elem.first, nLevel + 1, elem.second.begin()->first,
 445                         elem.second.begin()->second);
 446
 447     //Remove empty pot directory
 448     OUString sPoPath =
 449         OStringToOUString(
 450             aPotDir.subView(0,aPotDir.lastIndexOf('/')), RTL_TEXTENCODING_UTF8);
 451     OUString sPoUrl;
 452     if (osl::FileBase::getFileURLFromSystemPath(sPoPath, sPoUrl)
 453         != osl::FileBase::E_None)
 454     {
 455         std::cerr
 456             << ("Error: Cannot convert pathname to URL in " __FILE__
 457                 ", in line ")
 458             << __LINE__ << "\n"
 459             << sPoPath
 460             << "\n";
 461         throw false; //TODO
 462     }
 463     osl::Directory::remove(sPoUrl);
 464 }
 465
 466 void handleProjects(char const * sSourceRoot, char const * sDestRoot)
 467 {
 468     OUString root16;
 469     if (!rtl_convertStringToUString(
 470             &root16.pData, sSourceRoot, rtl_str_getLength(sSourceRoot),
 471             osl_getThreadTextEncoding(),
 472             (RTL_TEXTTOUNICODE_FLAGS_UNDEFINED_ERROR
 473              | RTL_TEXTTOUNICODE_FLAGS_MBUNDEFINED_ERROR
 474              | RTL_TEXTTOUNICODE_FLAGS_INVALID_ERROR)))
 475     {
 476         std::cerr << "Error: Cannot convert pathname to UTF-16\n";
 477         throw false; //TODO
 478     }
 479     OUString rootUrl;
 480     if (osl::FileBase::getFileURLFromSystemPath(root16, rootUrl)
 481         != osl::FileBase::E_None)
 482     {
 483         std::cerr
 484             << ("Error: Cannot convert pathname to URL in " __FILE__
 485                 ", in line ")
 486             << __LINE__ << "\n       root16: "
 487             << root16
 488             << "\n";
 489         throw false; //TODO
 490     }
 491     gDestRoot = OString(sDestRoot);
 492     handleDirectory(rootUrl, 0, OString(), gDestRoot);
 493 }
 494 }
 495
 496 SAL_IMPLEMENT_MAIN_WITH_ARGS(argc, argv)
 497 {
 498     try
 499     {
 500         if (argc != 4)
 501         {
 502             std::cerr
 503                 << ("localize (c)2001 by Sun Microsystems\n\n"
 504                     "As part of the L10N framework, localize extracts en-US\n"
 505                     "strings for translation out of the toplevel modules defined\n"
 506                     "in projects array in l10ntools/source/localize.cxx.\n\n"
 507                     "Syntax: localize <source-root> <outfile> <library-path-env-var-override>\n");
 508             exit(EXIT_FAILURE);
 509         }
 510         libraryPathEnvVarOverride = argv[3];
 511         handleProjects(argv[1],argv[2]);
 512     }
 513     catch (std::exception& e)
 514     {
 515         std::cerr << "exception: " << e.what() << std::endl;
 516         return EXIT_FAILURE;
 517     }
 518     catch (bool) //TODO
 519     {
 520         return EXIT_FAILURE;
 521     }
 522     return EXIT_SUCCESS;
 523 }
 524
 525 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */