shell/source/tools/lngconvex/lngconvex.cxx

   1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
   2 /*
   3  * This file is part of the LibreOffice project.
   4  *
   5  * This Source Code Form is subject to the terms of the Mozilla Public
   6  * License, v. 2.0. If a copy of the MPL was not distributed with this
   7  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
   8  *
   9  * This file incorporates work covered by the following license notice:
  10  *
  11  *   Licensed to the Apache Software Foundation (ASF) under one or more
  12  *   contributor license agreements. See the NOTICE file distributed
  13  *   with this work for additional information regarding copyright
  14  *   ownership. The ASF licenses this file to you under the Apache
  15  *   License, Version 2.0 (the "License"); you may not use this file
  16  *   except in compliance with the License. You may obtain a copy of
  17  *   the License at http://www.apache.org/licenses/LICENSE-2.0 .
  18  */
  19
  20 #ifdef _WIN32
  21 #include <prewin.h>
  22 #include <postwin.h>
  23 #else
  24 // From MinGW
  25 typedef unsigned short WORD;
  26 #define PRIMARYLANGID(lgid) (static_cast<WORD>(lgid) & 0x3ff)
  27 #define SUBLANGID(lgid) (static_cast<WORD>(lgid) >> 10)
  28 #define LANG_SPANISH 0x0a
  29 #define SUBLANG_NEUTRAL 0x00
  30 #define SUBLANG_SPANISH 0x01
  31 #endif
  32
  33 #include "cmdline.hxx"
  34
  35 #include <comphelper/string.hxx>
  36 #include <osl/thread.h>
  37 #include <osl/process.h>
  38 #include <osl/file.hxx>
  39 #include <sal/main.h>
  40
  41 #include <tools/config.hxx>
  42 #include <i18nlangtag/languagetag.hxx>
  43
  44 #include <iostream>
  45 #include <fstream>
  46 #include <map>
  47 #include <iterator>
  48 #include <string>
  49 #include <string_view>
  50
  51 #ifndef _WIN32
  52 #include <cstring>
  53 #endif
  54
  55 namespace /* private */
  56 {
  57
  58
  59 void ShowUsage()
  60 {
  61     std::cout << "Usage: -ulf ulf_file -rc rc_output_file -rct rc_template_file -rch rch_file -rcf rcf_file" << std::endl;
  62     std::cout << "-ulf Name of the ulf file" << std::endl;
  63     std::cout << "-rc  Name of the resulting resource file" << std::endl;
  64     std::cout << "-rct Name of the resource template file" << std::endl;
  65     std::cout << "-rch Name of the resource file header" << std::endl;
  66     std::cout << "-rcf Name of the resource file footer" << std::endl;
  67 }
  68
  69 OUString OStringToOUString(std::string_view str)
  70 { return rtl::OStringToOUString(str, osl_getThreadTextEncoding()); }
  71
  72 OString OUStringToOString(std::u16string_view str)
  73 { return rtl::OUStringToOString(str, osl_getThreadTextEncoding()); }
  74
  75 /** Get the directory where the module
  76     is located as system directory, the
  77     returned directory has a trailing '\'  */
  78 OUString get_module_path()
  79 {
  80     OUString cwd_url;
  81     OUString module_path;
  82     if (osl_Process_E_None == osl_getProcessWorkingDir(&cwd_url.pData))
  83         osl::FileBase::getSystemPathFromFileURL(cwd_url, module_path);
  84
  85     return module_path;
  86 }
  87
  88 /** Make the absolute directory of a base and
  89     a relative directory, if the relative
  90     directory is absolute the relative
  91     directory will be returned unchanged.
  92     Base and relative directory should be
  93     system paths the returned directory is
  94     a system path too */
  95 OUString get_absolute_path(
  96     const OUString& BaseDir, const OUString& RelDir)
  97 {
  98     OUString base_url;
  99     OUString rel_url;
 100
 101     osl::FileBase::getFileURLFromSystemPath(BaseDir, base_url);
 102     osl::FileBase::getFileURLFromSystemPath(RelDir, rel_url);
 103
 104     OUString abs_url;
 105     (void)osl::FileBase::getAbsoluteFileURL(base_url, rel_url, abs_url);
 106
 107     OUString abs_sys_path;
 108     osl::FileBase::getSystemPathFromFileURL(abs_url, abs_sys_path);
 109
 110     return abs_sys_path;
 111 }
 112
 113 std::string make_absolute(const std::string& file_name)
 114 {
 115     OUString fp = get_absolute_path(
 116         get_module_path(), OStringToOUString(file_name.c_str()));
 117     return std::string(OUStringToOString(fp));
 118 }
 119
 120 /** A helper class, enables stream exceptions
 121     on construction, restores the old exception
 122     state on destruction */
 123 class StreamExceptionsEnabler
 124 {
 125 public:
 126     explicit StreamExceptionsEnabler(
 127         std::ios& iostrm ) :
 128         m_IoStrm(iostrm),
 129         m_OldIos(m_IoStrm.exceptions())
 130     {
 131         m_IoStrm.exceptions(std::ios::failbit | std::ios::badbit);
 132     }
 133
 134     ~StreamExceptionsEnabler()
 135     {
 136         m_IoStrm.exceptions(m_OldIos);
 137     }
 138 private:
 139     std::ios& m_IoStrm;
 140     std::ios::iostate m_OldIos;
 141 };
 142
 143 class iso_lang_identifier
 144 {
 145 public:
 146     iso_lang_identifier() {};
 147
 148     explicit iso_lang_identifier(const OString& str) :
 149         maBcp47(str)
 150     { }
 151
 152     explicit iso_lang_identifier(const std::string& str) :
 153         maBcp47(str.c_str())
 154     { }
 155
 156     OUString make_OUString() const
 157     { return OStringToOUString( maBcp47, RTL_TEXTENCODING_ASCII_US); }
 158
 159     std::string make_std_string() const
 160     { return maBcp47.getStr(); }
 161
 162 private:
 163     OString maBcp47;
 164 };
 165
 166 /** Convert an OUString to the MS resource
 167     file format string e.g.
 168     OUString -> L"\x1A00\x2200\x3400" */
 169 std::string make_winrc_unicode_string(const OUString& str)
 170 {
 171     std::ostringstream oss;
 172     oss << "L\"";
 173
 174     size_t length = str.getLength();
 175     const sal_Unicode* pchr = str.getStr();
 176
 177     for (size_t i = 0; i < length; i++)
 178         oss << "\\x" << std::hex << static_cast<int>(*pchr++);
 179
 180     oss << "\"";
 181     return oss.str();
 182 }
 183
 184 std::string make_winrc_unicode_string(const std::string& str)
 185 {
 186     return make_winrc_unicode_string(
 187         OUString::createFromAscii(str.c_str()));
 188 }
 189
 190 /** A replacement table contains pairs of
 191     placeholders and the appropriate substitute */
 192 class Substitutor
 193 {
 194 private:
 195     typedef std::map<std::string, std::string> replacement_table_t;
 196     typedef std::map<std::string, replacement_table_t> iso_lang_replacement_table_t;
 197
 198 public:
 199     typedef iso_lang_replacement_table_t::iterator iterator;
 200     typedef iso_lang_replacement_table_t::const_iterator const_iterator;
 201
 202     iterator begin()
 203     { return iso_lang_replacement_table_.begin(); }
 204
 205     iterator end()
 206     { return iso_lang_replacement_table_.end(); }
 207
 208 public:
 209
 210     Substitutor() {};
 211
 212     void set_language(const iso_lang_identifier& iso_lang)
 213     {
 214         active_iso_lang_ = iso_lang;
 215     }
 216
 217     // If Text is a placeholder substitute it with
 218     //its substitute else leave it unchanged
 219     void substitute(std::string& Text)
 220     {
 221         replacement_table_t& prt = get_replacement_table(active_iso_lang_.make_std_string());
 222         replacement_table_t::iterator iter = prt.find(Text);
 223         if (iter != prt.end())
 224             Text = iter->second;
 225     }
 226
 227     void add_substitution(
 228         const std::string& Placeholder, const std::string& Substitute)
 229     {
 230         replacement_table_t& prt = get_replacement_table(active_iso_lang_.make_std_string());
 231         prt.insert(std::make_pair(Placeholder, Substitute));
 232     }
 233
 234
 235 private:
 236     // Return the replacement table for the iso lang id
 237     // create a new one if not already present
 238     replacement_table_t& get_replacement_table(const std::string& iso_lang)
 239     {
 240         return iso_lang_replacement_table_[iso_lang];
 241     }
 242
 243 private:
 244     iso_lang_replacement_table_t iso_lang_replacement_table_;
 245     iso_lang_identifier active_iso_lang_;
 246 };
 247
 248 void add_group_entries(
 249     Config& aConfig,
 250     const OString& GroupName,
 251     Substitutor& Substitutor)
 252 {
 253     OSL_ASSERT(aConfig.HasGroup(GroupName));
 254
 255     aConfig.SetGroup(GroupName);
 256     size_t key_count = aConfig.GetKeyCount();
 257     std::map< LanguageType, std::string > map;
 258
 259     for (size_t i = 0; i < key_count; i++)
 260     {
 261         OString iso_lang = aConfig.GetKeyName(sal::static_int_cast<sal_uInt16>(i));
 262         OString key_value_utf8 = aConfig.ReadKey(sal::static_int_cast<sal_uInt16>(i));
 263         iso_lang_identifier myiso_lang( iso_lang );
 264         LanguageType ltype = LanguageTag( myiso_lang.make_OUString()).makeFallback().getLanguageType();
 265         if(  ( static_cast<sal_uInt16>(ltype) & 0x0200 ) == 0 && map[ ltype ].empty()  )
 266         {
 267             Substitutor.set_language(iso_lang_identifier(iso_lang));
 268
 269             key_value_utf8 = comphelper::string::strip(key_value_utf8, '\"');
 270
 271             OUString key_value_utf16 =
 272                 OStringToOUString(key_value_utf8, RTL_TEXTENCODING_UTF8);
 273
 274             Substitutor.add_substitution(
 275                 GroupName.getStr(), make_winrc_unicode_string(key_value_utf16));
 276             map[ ltype ] = std::string( iso_lang.getStr() );
 277         }
 278         else
 279         {
 280             if( !map[ ltype ].empty() )
 281             {
 282                 printf("ERROR: Duplicated ms id %d found for the languages %s and %s !!!! This does not work in microsoft resources\nPlease remove one!\n", static_cast<sal_uInt16>(ltype) , map[ ltype ].c_str() , iso_lang.getStr());
 283                 exit( -1 );
 284             }
 285         }
 286     }
 287 }
 288
 289 void read_ulf_file(const std::string& FileName, Substitutor& Substitutor)
 290 {
 291     // work-around for #i32420#
 292
 293     // as the Config class is currently not able to deal correctly with
 294     // UTF8 files starting with a byte-order-mark we create a copy of the
 295     // original file without the byte-order-mark
 296     OUString tmpfile_url;
 297     osl_createTempFile(nullptr, nullptr, &tmpfile_url.pData);
 298
 299     OUString tmpfile_sys;
 300     osl::FileBase::getSystemPathFromFileURL(tmpfile_url, tmpfile_sys);
 301
 302     std::ifstream in(FileName.c_str());
 303     std::ofstream out(OUStringToOString(tmpfile_sys).getStr());
 304
 305     try
 306     {
 307         StreamExceptionsEnabler sexc_out(out);
 308         StreamExceptionsEnabler sexc_in(in);
 309
 310         //skip the byte-order-mark 0xEF 0xBB 0xBF, identifying UTF8 files
 311         unsigned char const BOM[3] = {0xEF, 0xBB, 0xBF};
 312         char buff[3];
 313         in.read(&buff[0], 3);
 314
 315         if (memcmp(buff, BOM, 3) != 0)
 316             in.seekg(0);
 317
 318         std::string line;
 319         while (std::getline(in, line))
 320             out << line << std::endl;
 321     }
 322     catch (const std::ios::failure&)
 323     {
 324         if (!in.eof())
 325             throw;
 326     }
 327
 328
 329     // end work-around for #i32420#
 330
 331     Config config(tmpfile_url);
 332     size_t grpcnt = config.GetGroupCount();
 333     for (size_t i = 0; i < grpcnt; i++)
 334         add_group_entries(config, config.GetGroupName(sal::static_int_cast<sal_uInt16>(i)), Substitutor);
 335 }
 336
 337 void read_file(
 338     const std::string& fname,
 339     std::vector<std::string>& string_container)
 340 {
 341     std::ifstream file(fname.c_str());
 342     StreamExceptionsEnabler sexc(file);
 343
 344     try
 345     {
 346         std::string line;
 347         while (std::getline(file, line))
 348             string_container.push_back(line);
 349     }
 350     catch(const std::ios::failure&)
 351     {
 352         if (!file.eof())
 353             throw;
 354     }
 355 }
 356
 357 /** A simple helper function that happens the
 358     content of one file to another one  */
 359 void concatenate_files(std::ostream& os, std::istream& is)
 360 {
 361     StreamExceptionsEnabler os_sexc(os);
 362     StreamExceptionsEnabler is_sexc(is);
 363
 364     try
 365     {
 366         std::string line;
 367         while (std::getline(is, line))
 368             os << line << std::endl;
 369     }
 370     catch(const std::ios::failure&)
 371     {
 372         if (!is.eof())
 373             throw;
 374     }
 375 }
 376
 377 bool is_placeholder(const std::string& str)
 378 {
 379     return ((str.length() > 1) &&
 380             ('%' == str[0]) &&
 381             ('%' == str[str.length() - 1]));
 382 }
 383
 384 void start_language_section(
 385     std::ostream_iterator<std::string>& ostream_iter, const iso_lang_identifier& iso_lang)
 386 {
 387     ostream_iter = std::string();
 388
 389     std::string lang_section("LANGUAGE ");
 390
 391     LanguageType ltype = LanguageTag( iso_lang.make_OUString()).makeFallback().getLanguageType();
 392
 393     char buff[10];
 394     int primLangID = PRIMARYLANGID(ltype);
 395     int subLangID = SUBLANGID(ltype);
 396     // Our resources are normally not sub language dependent.
 397     // Esp. for spanish we don't want to distinguish between trad.
 398     // and international sorting (which leads to two different sub languages)
 399     // Setting the sub language to neutral allows us to use one
 400     // stringlist for all spanish variants
 401     if ( ( primLangID == LANG_SPANISH ) &&
 402          ( subLangID == SUBLANG_SPANISH ) )
 403         subLangID = SUBLANG_NEUTRAL;
 404
 405 #ifdef _WIN32
 406     _itoa(primLangID, buff, 16);
 407 #else
 408     sprintf(buff, "%x", primLangID);
 409 #endif
 410     lang_section += std::string("0x") + std::string(buff);
 411
 412     lang_section += std::string(" , ");
 413
 414 #ifdef _WIN32
 415     _itoa(subLangID, buff, 16);
 416 #else
 417     sprintf(buff, "%x", subLangID);
 418 #endif
 419     lang_section += std::string("0x") + std::string(buff);
 420     ostream_iter = lang_section;
 421 }
 422
 423 /** Iterate all languages in the substitutor,
 424     replace the all placeholder and append the
 425     result to the output file */
 426 void inflate_rc_template_to_file(
 427     std::ostream& os, const std::vector<std::string>& rctmpl, Substitutor& substitutor)
 428 {
 429     StreamExceptionsEnabler sexc(os);
 430
 431     Substitutor::const_iterator iter = substitutor.begin();
 432     Substitutor::const_iterator iter_end = substitutor.end();
 433
 434     std::ostream_iterator<std::string> oi(os, "\n");
 435
 436     for ( /**/ ;iter != iter_end; ++iter)
 437     {
 438         substitutor.set_language(iso_lang_identifier(iter->first));
 439
 440         if (!rctmpl.empty())
 441             start_language_section(oi, iso_lang_identifier(iter->first));
 442
 443         for ( auto& rct : rctmpl)
 444         {
 445             std::istringstream iss(rct);
 446             std::string line;
 447
 448             while (iss)
 449             {
 450                 std::string token;
 451                 iss >> token;
 452                 substitutor.substitute(token);
 453
 454                 // HACK for partially merged
 455                 // *.lng files where some strings have
 456                 // a particular language that others
 457                 // don't have in order to keep the
 458                 // build
 459                 // coverity[tainted_data] - trusted data source
 460                 if (is_placeholder(token))
 461                     token = make_winrc_unicode_string(token);
 462
 463                 line += token;
 464                 line += " ";
 465             }
 466             oi = line;
 467         }
 468     }
 469 }
 470
 471 } // namespace /* private */
 472
 473 /* MAIN
 474    The file names provided via command line should be
 475    absolute or relative to the directory of this module.
 476
 477    Algo:
 478    1. read the ulf file and initialize the substitutor
 479    2. read the resource template file
 480    3. create the output file and append the header
 481    4. inflate the resource template to the output file
 482       for every language using the substitutor
 483    5. append the footer
 484 */
 485
 486 SAL_IMPLEMENT_MAIN_WITH_ARGS(argc, argv)
 487 {
 488     try
 489     {
 490         CommandLine cmdline(argc, argv);
 491
 492         Substitutor substitutor;
 493         read_ulf_file(make_absolute(cmdline.get_arg("-ulf")), substitutor);
 494
 495         std::vector<std::string> rc_tmpl;
 496         read_file(make_absolute(cmdline.get_arg("-rct")), rc_tmpl);
 497
 498         std::ofstream rc_file(make_absolute(cmdline.get_arg("-rc")));
 499         std::ifstream in_header(make_absolute(cmdline.get_arg("-rch")));
 500         concatenate_files(rc_file, in_header);
 501
 502         inflate_rc_template_to_file(rc_file, rc_tmpl, substitutor);
 503
 504         std::ifstream in_footer(make_absolute(cmdline.get_arg("-rcf")));
 505         concatenate_files(rc_file, in_footer);
 506     }
 507     catch(const std::ios::failure& ex)
 508     {
 509         std::cout << ex.what() << std::endl;
 510         return 1;
 511     }
 512     catch(const std::exception& ex)
 513     {
 514         std::cout << ex.what() << std::endl;
 515         ShowUsage();
 516         return 1;
 517     }
 518     catch(...)
 519     {
 520         std::cout << "Unexpected error..." << std::endl;
 521         return 1;
 522     }
 523     return 0;
 524 }
 525
 526 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */