shell/source/tools/lngconvex/lngconvex.cxx

   1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
   2 /*
   3  * This file is part of the LibreOffice project.
   4  *
   5  * This Source Code Form is subject to the terms of the Mozilla Public
   6  * License, v. 2.0. If a copy of the MPL was not distributed with this
   7  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
   8  *
   9  * This file incorporates work covered by the following license notice:
  10  *
  11  *   Licensed to the Apache Software Foundation (ASF) under one or more
  12  *   contributor license agreements. See the NOTICE file distributed
  13  *   with this work for additional information regarding copyright
  14  *   ownership. The ASF licenses this file to you under the Apache
  15  *   License, Version 2.0 (the "License"); you may not use this file
  16  *   except in compliance with the License. You may obtain a copy of
  17  *   the License at http://www.apache.org/licenses/LICENSE-2.0 .
  18  */
  19
  20 #ifdef _WIN32
  21 #include <prewin.h>
  22 #include <postwin.h>
  23 #else
  24 // From MinGW
  25 typedef unsigned short WORD;
  26 #define PRIMARYLANGID(lgid) (static_cast<WORD>(lgid) & 0x3ff)
  27 #define SUBLANGID(lgid) (static_cast<WORD>(lgid) >> 10)
  28 #define LANG_SPANISH 0x0a
  29 #define SUBLANG_NEUTRAL 0x00
  30 #define SUBLANG_SPANISH 0x01
  31 #endif
  32
  33 #include "cmdline.hxx"
  34
  35 #include <comphelper/string.hxx>
  36 #include <osl/thread.h>
  37 #include <osl/process.h>
  38 #include <osl/diagnose.h>
  39 #include <osl/file.hxx>
  40 #include <sal/main.h>
  41
  42 #include <tools/config.hxx>
  43 #include <i18nlangtag/languagetag.hxx>
  44
  45 #include <iostream>
  46 #include <fstream>
  47 #include <map>
  48 #include <iterator>
  49 #include <string>
  50 #include <string_view>
  51
  52 #ifndef _WIN32
  53 #include <cstring>
  54 #endif
  55
  56 namespace /* private */
  57 {
  58
  59
  60 void ShowUsage()
  61 {
  62     std::cout << "Usage: -ulf ulf_file -rc rc_output_file -rct rc_template_file -rch rch_file -rcf rcf_file" << std::endl;
  63     std::cout << "-ulf Name of the ulf file" << std::endl;
  64     std::cout << "-rc  Name of the resulting resource file" << std::endl;
  65     std::cout << "-rct Name of the resource template file" << std::endl;
  66     std::cout << "-rch Name of the resource file header" << std::endl;
  67     std::cout << "-rcf Name of the resource file footer" << std::endl;
  68 }
  69
  70 OUString OStringToOUString(std::string_view str)
  71 { return rtl::OStringToOUString(str, osl_getThreadTextEncoding()); }
  72
  73 OString OUStringToOString(std::u16string_view str)
  74 { return rtl::OUStringToOString(str, osl_getThreadTextEncoding()); }
  75
  76 /** Get the directory where the module
  77     is located as system directory, the
  78     returned directory has a trailing '\'  */
  79 OUString get_module_path()
  80 {
  81     OUString cwd_url;
  82     OUString module_path;
  83     if (osl_Process_E_None == osl_getProcessWorkingDir(&cwd_url.pData))
  84         osl::FileBase::getSystemPathFromFileURL(cwd_url, module_path);
  85
  86     return module_path;
  87 }
  88
  89 /** Make the absolute directory of a base and
  90     a relative directory, if the relative
  91     directory is absolute the relative
  92     directory will be returned unchanged.
  93     Base and relative directory should be
  94     system paths the returned directory is
  95     a system path too */
  96 OUString get_absolute_path(
  97     const OUString& BaseDir, const OUString& RelDir)
  98 {
  99     OUString base_url;
 100     OUString rel_url;
 101
 102     osl::FileBase::getFileURLFromSystemPath(BaseDir, base_url);
 103     osl::FileBase::getFileURLFromSystemPath(RelDir, rel_url);
 104
 105     OUString abs_url;
 106     (void)osl::FileBase::getAbsoluteFileURL(base_url, rel_url, abs_url);
 107
 108     OUString abs_sys_path;
 109     osl::FileBase::getSystemPathFromFileURL(abs_url, abs_sys_path);
 110
 111     return abs_sys_path;
 112 }
 113
 114 std::string make_absolute(const std::string& file_name)
 115 {
 116     OUString fp = get_absolute_path(
 117         get_module_path(), OStringToOUString(file_name));
 118     return std::string(OUStringToOString(fp));
 119 }
 120
 121 /** A helper class, enables stream exceptions
 122     on construction, restores the old exception
 123     state on destruction */
 124 class StreamExceptionsEnabler
 125 {
 126 public:
 127     explicit StreamExceptionsEnabler(
 128         std::ios& iostrm ) :
 129         m_IoStrm(iostrm),
 130         m_OldIos(m_IoStrm.exceptions())
 131     {
 132         m_IoStrm.exceptions(std::ios::failbit | std::ios::badbit);
 133     }
 134
 135     ~StreamExceptionsEnabler()
 136     {
 137         m_IoStrm.exceptions(m_OldIos);
 138     }
 139 private:
 140     std::ios& m_IoStrm;
 141     std::ios::iostate m_OldIos;
 142 };
 143
 144 class iso_lang_identifier
 145 {
 146 public:
 147     iso_lang_identifier() {};
 148
 149     explicit iso_lang_identifier(const OString& str) :
 150         maBcp47(str)
 151     { }
 152
 153     explicit iso_lang_identifier(const std::string& str) :
 154         maBcp47(str.c_str())
 155     { }
 156
 157     OUString make_OUString() const
 158     { return OStringToOUString( maBcp47, RTL_TEXTENCODING_ASCII_US); }
 159
 160     std::string make_std_string() const
 161     { return std::string(maBcp47); }
 162
 163 private:
 164     OString maBcp47;
 165 };
 166
 167 /** Convert an OUString to the MS resource
 168     file format string e.g.
 169     OUString -> L"\x1A00\x2200\x3400" */
 170 std::string make_winrc_unicode_string(const OUString& str)
 171 {
 172     std::ostringstream oss;
 173     oss << "L\"";
 174
 175     size_t length = str.getLength();
 176     const sal_Unicode* pchr = str.getStr();
 177
 178     for (size_t i = 0; i < length; i++)
 179         oss << "\\x" << std::hex << static_cast<int>(*pchr++);
 180
 181     oss << "\"";
 182     return oss.str();
 183 }
 184
 185 std::string make_winrc_unicode_string(const std::string& str)
 186 {
 187     return make_winrc_unicode_string(
 188         OUString::createFromAscii(str));
 189 }
 190
 191 /** A replacement table contains pairs of
 192     placeholders and the appropriate substitute */
 193 class Substitutor
 194 {
 195 private:
 196     typedef std::map<std::string, std::string> replacement_table_t;
 197     typedef std::map<std::string, replacement_table_t> iso_lang_replacement_table_t;
 198
 199 public:
 200     typedef iso_lang_replacement_table_t::iterator iterator;
 201     typedef iso_lang_replacement_table_t::const_iterator const_iterator;
 202
 203     iterator begin()
 204     { return iso_lang_replacement_table_.begin(); }
 205
 206     iterator end()
 207     { return iso_lang_replacement_table_.end(); }
 208
 209 public:
 210
 211     Substitutor() {};
 212
 213     void set_language(const iso_lang_identifier& iso_lang)
 214     {
 215         active_iso_lang_ = iso_lang;
 216     }
 217
 218     // If Text is a placeholder substitute it with
 219     //its substitute else leave it unchanged
 220     void substitute(std::string& Text)
 221     {
 222         replacement_table_t& prt = get_replacement_table(active_iso_lang_.make_std_string());
 223         replacement_table_t::iterator iter = prt.find(Text);
 224         if (iter != prt.end())
 225             Text = iter->second;
 226     }
 227
 228     void add_substitution(
 229         const std::string& Placeholder, const std::string& Substitute)
 230     {
 231         replacement_table_t& prt = get_replacement_table(active_iso_lang_.make_std_string());
 232         prt.insert(std::make_pair(Placeholder, Substitute));
 233     }
 234
 235
 236 private:
 237     // Return the replacement table for the iso lang id
 238     // create a new one if not already present
 239     replacement_table_t& get_replacement_table(const std::string& iso_lang)
 240     {
 241         return iso_lang_replacement_table_[iso_lang];
 242     }
 243
 244 private:
 245     iso_lang_replacement_table_t iso_lang_replacement_table_;
 246     iso_lang_identifier active_iso_lang_;
 247 };
 248
 249 void add_group_entries(
 250     Config& aConfig,
 251     const OString& GroupName,
 252     Substitutor& Substitutor)
 253 {
 254     OSL_ASSERT(aConfig.HasGroup(GroupName));
 255
 256     aConfig.SetGroup(GroupName);
 257     size_t key_count = aConfig.GetKeyCount();
 258     std::map< LanguageType, std::string > map;
 259
 260     for (size_t i = 0; i < key_count; i++)
 261     {
 262         OString iso_lang = aConfig.GetKeyName(sal::static_int_cast<sal_uInt16>(i));
 263         OString key_value_utf8 = aConfig.ReadKey(sal::static_int_cast<sal_uInt16>(i));
 264         iso_lang_identifier myiso_lang( iso_lang );
 265         LanguageType ltype = LanguageTag( myiso_lang.make_OUString()).makeFallback().getLanguageType();
 266         if(  ( static_cast<sal_uInt16>(ltype) & 0x0200 ) == 0 && map[ ltype ].empty()  )
 267         {
 268             Substitutor.set_language(iso_lang_identifier(iso_lang));
 269
 270             key_value_utf8 = comphelper::string::strip(key_value_utf8, '\"');
 271
 272             OUString key_value_utf16 =
 273                 OStringToOUString(key_value_utf8, RTL_TEXTENCODING_UTF8);
 274
 275             Substitutor.add_substitution(
 276                 std::string(GroupName), make_winrc_unicode_string(key_value_utf16));
 277             map[ ltype ] = std::string( iso_lang );
 278         }
 279         else
 280         {
 281             if( !map[ ltype ].empty() )
 282             {
 283                 printf("ERROR: Duplicated ms id %d found for the languages %s and %s !!!! This does not work in microsoft resources\nPlease remove one!\n", static_cast<sal_uInt16>(ltype) , map[ ltype ].c_str() , iso_lang.getStr());
 284                 exit( -1 );
 285             }
 286         }
 287     }
 288 }
 289
 290 void read_ulf_file(const std::string& FileName, Substitutor& Substitutor)
 291 {
 292     // work-around for #i32420#
 293
 294     // as the Config class is currently not able to deal correctly with
 295     // UTF8 files starting with a byte-order-mark we create a copy of the
 296     // original file without the byte-order-mark
 297     OUString tmpfile_url;
 298     osl_createTempFile(nullptr, nullptr, &tmpfile_url.pData);
 299
 300     OUString tmpfile_sys;
 301     osl::FileBase::getSystemPathFromFileURL(tmpfile_url, tmpfile_sys);
 302
 303     std::ifstream in(FileName.c_str());
 304     std::ofstream out(OUStringToOString(tmpfile_sys).getStr());
 305
 306     try
 307     {
 308         StreamExceptionsEnabler sexc_out(out);
 309         StreamExceptionsEnabler sexc_in(in);
 310
 311         //skip the byte-order-mark 0xEF 0xBB 0xBF, identifying UTF8 files
 312         unsigned char const BOM[3] = {0xEF, 0xBB, 0xBF};
 313         char buff[3];
 314         in.read(&buff[0], 3);
 315
 316         if (memcmp(buff, BOM, 3) != 0)
 317             in.seekg(0);
 318
 319         std::string line;
 320         while (std::getline(in, line))
 321             out << line << std::endl;
 322     }
 323     catch (const std::ios::failure&)
 324     {
 325         if (!in.eof())
 326             throw;
 327     }
 328
 329
 330     // end work-around for #i32420#
 331
 332     Config config(tmpfile_url);
 333     size_t grpcnt = config.GetGroupCount();
 334     for (size_t i = 0; i < grpcnt; i++)
 335         add_group_entries(config, config.GetGroupName(sal::static_int_cast<sal_uInt16>(i)), Substitutor);
 336 }
 337
 338 void read_file(
 339     const std::string& fname,
 340     std::vector<std::string>& string_container)
 341 {
 342     std::ifstream file(fname.c_str());
 343     StreamExceptionsEnabler sexc(file);
 344
 345     try
 346     {
 347         std::string line;
 348         while (std::getline(file, line))
 349             string_container.push_back(line);
 350     }
 351     catch(const std::ios::failure&)
 352     {
 353         if (!file.eof())
 354             throw;
 355     }
 356 }
 357
 358 /** A simple helper function that happens the
 359     content of one file to another one  */
 360 void concatenate_files(std::ostream& os, std::istream& is)
 361 {
 362     StreamExceptionsEnabler os_sexc(os);
 363     StreamExceptionsEnabler is_sexc(is);
 364
 365     try
 366     {
 367         std::string line;
 368         while (std::getline(is, line))
 369             os << line << std::endl;
 370     }
 371     catch(const std::ios::failure&)
 372     {
 373         if (!is.eof())
 374             throw;
 375     }
 376 }
 377
 378 bool is_placeholder(const std::string& str)
 379 {
 380     return ((str.length() > 1) &&
 381             ('%' == str[0]) &&
 382             ('%' == str[str.length() - 1]));
 383 }
 384
 385 void start_language_section(
 386     std::ostream_iterator<std::string>& ostream_iter, const iso_lang_identifier& iso_lang)
 387 {
 388     ostream_iter = std::string();
 389
 390     std::string lang_section("LANGUAGE ");
 391
 392     LanguageType ltype = LanguageTag( iso_lang.make_OUString()).makeFallback().getLanguageType();
 393
 394     char buff[10];
 395     int primLangID = PRIMARYLANGID(ltype);
 396     int subLangID = SUBLANGID(ltype);
 397     // Our resources are normally not sub language dependent.
 398     // Esp. for spanish we don't want to distinguish between trad.
 399     // and international sorting (which leads to two different sub languages)
 400     // Setting the sub language to neutral allows us to use one
 401     // stringlist for all spanish variants
 402     if ( ( primLangID == LANG_SPANISH ) &&
 403          ( subLangID == SUBLANG_SPANISH ) )
 404         subLangID = SUBLANG_NEUTRAL;
 405
 406 #ifdef _WIN32
 407     _itoa(primLangID, buff, 16);
 408 #else
 409     sprintf(buff, "%x", primLangID);
 410 #endif
 411     lang_section += std::string("0x") + std::string(buff);
 412
 413     lang_section += std::string(" , ");
 414
 415 #ifdef _WIN32
 416     _itoa(subLangID, buff, 16);
 417 #else
 418     sprintf(buff, "%x", subLangID);
 419 #endif
 420     lang_section += std::string("0x") + std::string(buff);
 421     ostream_iter = lang_section;
 422 }
 423
 424 /** Iterate all languages in the substitutor,
 425     replace the all placeholder and append the
 426     result to the output file */
 427 void inflate_rc_template_to_file(
 428     std::ostream& os, const std::vector<std::string>& rctmpl, Substitutor& substitutor)
 429 {
 430     StreamExceptionsEnabler sexc(os);
 431
 432     Substitutor::const_iterator iter = substitutor.begin();
 433     Substitutor::const_iterator iter_end = substitutor.end();
 434
 435     std::ostream_iterator<std::string> oi(os, "\n");
 436
 437     for ( /**/ ;iter != iter_end; ++iter)
 438     {
 439         substitutor.set_language(iso_lang_identifier(iter->first));
 440
 441         if (!rctmpl.empty())
 442             start_language_section(oi, iso_lang_identifier(iter->first));
 443
 444         for ( auto& rct : rctmpl)
 445         {
 446             std::istringstream iss(rct);
 447             std::string line;
 448
 449             while (iss)
 450             {
 451                 std::string token;
 452                 iss >> token;
 453                 substitutor.substitute(token);
 454
 455                 // HACK for partially merged
 456                 // *.lng files where some strings have
 457                 // a particular language that others
 458                 // don't have in order to keep the
 459                 // build
 460                 // coverity[tainted_data] - trusted data source
 461                 if (is_placeholder(token))
 462                     token = make_winrc_unicode_string(token);
 463
 464                 line += token;
 465                 line += " ";
 466             }
 467             oi = line;
 468         }
 469     }
 470 }
 471
 472 } // namespace /* private */
 473
 474 /* MAIN
 475    The file names provided via command line should be
 476    absolute or relative to the directory of this module.
 477
 478    Algo:
 479    1. read the ulf file and initialize the substitutor
 480    2. read the resource template file
 481    3. create the output file and append the header
 482    4. inflate the resource template to the output file
 483       for every language using the substitutor
 484    5. append the footer
 485 */
 486
 487 SAL_IMPLEMENT_MAIN_WITH_ARGS(argc, argv)
 488 {
 489     try
 490     {
 491         CommandLine cmdline(argc, argv);
 492
 493         Substitutor substitutor;
 494         read_ulf_file(make_absolute(cmdline.get_arg("-ulf")), substitutor);
 495
 496         std::vector<std::string> rc_tmpl;
 497         read_file(make_absolute(cmdline.get_arg("-rct")), rc_tmpl);
 498
 499         std::ofstream rc_file(make_absolute(cmdline.get_arg("-rc")));
 500         std::ifstream in_header(make_absolute(cmdline.get_arg("-rch")));
 501         concatenate_files(rc_file, in_header);
 502
 503         inflate_rc_template_to_file(rc_file, rc_tmpl, substitutor);
 504
 505         std::ifstream in_footer(make_absolute(cmdline.get_arg("-rcf")));
 506         concatenate_files(rc_file, in_footer);
 507     }
 508     catch(const std::ios::failure& ex)
 509     {
 510         std::cout << ex.what() << std::endl;
 511         return 1;
 512     }
 513     catch(const std::exception& ex)
 514     {
 515         std::cout << ex.what() << std::endl;
 516         ShowUsage();
 517         return 1;
 518     }
 519     catch(...)
 520     {
 521         std::cout << "Unexpected error..." << std::endl;
 522         return 1;
 523     }
 524     return 0;
 525 }
 526
 527 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */