shell/source/tools/lngconvex/lngconvex.cxx

   1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
   2 /*
   3  * This file is part of the LibreOffice project.
   4  *
   5  * This Source Code Form is subject to the terms of the Mozilla Public
   6  * License, v. 2.0. If a copy of the MPL was not distributed with this
   7  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
   8  *
   9  * This file incorporates work covered by the following license notice:
  10  *
  11  *   Licensed to the Apache Software Foundation (ASF) under one or more
  12  *   contributor license agreements. See the NOTICE file distributed
  13  *   with this work for additional information regarding copyright
  14  *   ownership. The ASF licenses this file to you under the Apache
  15  *   License, Version 2.0 (the "License"); you may not use this file
  16  *   except in compliance with the License. You may obtain a copy of
  17  *   the License at http://www.apache.org/licenses/LICENSE-2.0 .
  18  */
  19
  20 #ifdef AIX
  21 #    undef _THREAD_SAFE
  22 #endif
  23
  24
  25 #ifdef WNT
  26 #include <prewin.h>
  27 #include <postwin.h>
  28 #else
  29 // From MinGW
  30 typedef unsigned short WORD;
  31 #define PRIMARYLANGID(lgid) ((WORD)(lgid) & 0x3ff)
  32 #define SUBLANGID(lgid) ((WORD)(lgid) >> 10)
  33 #define LANG_SPANISH 0x0a
  34 #define SUBLANG_NEUTRAL 0x00
  35 #define SUBLANG_SPANISH 0x01
  36 #endif
  37
  38 #include "cmdline.hxx"
  39
  40 #include <comphelper/string.hxx>
  41 #include "osl/thread.h"
  42 #include "osl/process.h"
  43 #include "osl/file.hxx"
  44 #include "sal/main.h"
  45
  46 #include "tools/config.hxx"
  47 #include "i18nlangtag/languagetag.hxx"
  48
  49 #include <iostream>
  50 #include <fstream>
  51 #include <map>
  52 #include <sstream>
  53 #include <iterator>
  54 #include <algorithm>
  55 #include <string>
  56
  57 #ifndef WNT
  58 #include <cstring>
  59 #endif
  60
  61 namespace /* private */
  62 {
  63
  64
  65 void ShowUsage()
  66 {
  67     std::cout << "Usage: -ulf ulf_file -rc rc_output_file -rct rc_template_file -rch rch_file -rcf rcf_file" << std::endl;
  68     std::cout << "-ulf Name of the ulf file" << std::endl;
  69     std::cout << "-rc  Name of the resulting resource file" << std::endl;
  70     std::cout << "-rct Name of the resource template file" << std::endl;
  71     std::cout << "-rch Name of the resource file header" << std::endl;
  72     std::cout << "-rcf Name of the resource file footer" << std::endl;
  73 }
  74
  75 inline OUString OStringToOUString(const OString& str)
  76 { return OStringToOUString(str, osl_getThreadTextEncoding()); }
  77
  78 inline OString OUStringToOString(const OUString& str)
  79 { return OUStringToOString(str, osl_getThreadTextEncoding()); }
  80
  81 /** Get the directory where the module
  82     is located as system directory, the
  83     returned directory has a trailing '\'  */
  84 OUString get_module_path()
  85 {
  86     OUString cwd_url;
  87     OUString module_path;
  88     if (osl_Process_E_None == osl_getProcessWorkingDir(&cwd_url.pData))
  89         osl::FileBase::getSystemPathFromFileURL(cwd_url, module_path);
  90
  91     return module_path;
  92 }
  93
  94 /** Make the absolute directory of a base and
  95     a relative directory, if the relative
  96     directory is absolute the relative
  97     directory will be returned unchanged.
  98     Base and relative directory should be
  99     system paths the returned directory is
 100     a system path too */
 101 OUString get_absolute_path(
 102     const OUString& BaseDir, const OUString& RelDir)
 103 {
 104     OUString base_url;
 105     OUString rel_url;
 106
 107     osl::FileBase::getFileURLFromSystemPath(BaseDir, base_url);
 108     osl::FileBase::getFileURLFromSystemPath(RelDir, rel_url);
 109
 110     OUString abs_url;
 111     osl::FileBase::getAbsoluteFileURL(base_url, rel_url, abs_url);
 112
 113     OUString abs_sys_path;
 114     osl::FileBase::getSystemPathFromFileURL(abs_url, abs_sys_path);
 115
 116     return abs_sys_path;
 117 }
 118
 119 OString get_absolute_file_path(const std::string& file_name)
 120 {
 121     OUString fp = get_absolute_path(
 122         get_module_path(), OStringToOUString(file_name.c_str()));
 123     return OUStringToOString(fp);
 124 }
 125
 126 /** A helper class, enables stream exceptions
 127     on construction, restors the old exception
 128     state on destruction */
 129 class StreamExceptionsEnabler
 130 {
 131 public:
 132     explicit StreamExceptionsEnabler(
 133         std::ios& iostrm,
 134         std::ios::iostate NewIos = std::ios::failbit | std::ios::badbit) :
 135         m_IoStrm(iostrm),
 136         m_OldIos(m_IoStrm.exceptions())
 137     {
 138         m_IoStrm.exceptions(NewIos);
 139     }
 140
 141     ~StreamExceptionsEnabler()
 142     {
 143         m_IoStrm.exceptions(m_OldIos);
 144     }
 145 private:
 146     std::ios& m_IoStrm;
 147     std::ios::iostate m_OldIos;
 148 };
 149
 150 typedef std::vector<std::string> string_container_t;
 151
 152 class iso_lang_identifier
 153 {
 154 public:
 155     iso_lang_identifier() {};
 156
 157     iso_lang_identifier(const OString& str) :
 158         maBcp47(str)
 159     { }
 160
 161     iso_lang_identifier(const std::string& str) :
 162         maBcp47(str.c_str())
 163     { }
 164
 165     OUString make_OUString() const
 166     { return OStringToOUString( maBcp47, RTL_TEXTENCODING_ASCII_US); }
 167
 168     std::string make_std_string() const
 169     { return maBcp47.getStr(); }
 170
 171 private:
 172     OString maBcp47;
 173 };
 174
 175 /** Convert a OUString to the MS resource
 176     file format string e.g.
 177     OUString -> L"\x1A00\x2200\x3400" */
 178 std::string make_winrc_unicode_string(const OUString& str)
 179 {
 180     std::ostringstream oss;
 181     oss << "L\"";
 182
 183     size_t length = str.getLength();
 184     const sal_Unicode* pchr = str.getStr();
 185
 186     for (size_t i = 0; i < length; i++)
 187         oss << "\\x" << std::hex << (int)*pchr++;
 188
 189     oss << "\"";
 190     return oss.str();
 191 }
 192
 193 std::string make_winrc_unicode_string(const std::string& str)
 194 {
 195     return make_winrc_unicode_string(
 196         OUString::createFromAscii(str.c_str()));
 197 }
 198
 199 /** A replacement table contains pairs of
 200     placeholders and the appropriate substitute */
 201 class Substitutor
 202 {
 203 private:
 204     typedef std::map<std::string, std::string> replacement_table_t;
 205     typedef std::map<std::string, replacement_table_t*> iso_lang_replacement_table_t;
 206
 207 public:
 208     typedef iso_lang_replacement_table_t::iterator iterator;
 209     typedef iso_lang_replacement_table_t::const_iterator const_iterator;
 210
 211     iterator begin()
 212     { return iso_lang_replacement_table_.begin(); }
 213
 214     iterator end()
 215     { return iso_lang_replacement_table_.end(); }
 216
 217 public:
 218
 219     Substitutor() {};
 220
 221     ~Substitutor()
 222     {
 223         iso_lang_replacement_table_t::iterator iter_end = iso_lang_replacement_table_.end();
 224         iso_lang_replacement_table_t::iterator iter = iso_lang_replacement_table_.begin();
 225
 226         for( /* no init */; iter != iter_end; ++iter)
 227             delete iter->second;
 228
 229         iso_lang_replacement_table_.clear();
 230     }
 231
 232     void set_language(const iso_lang_identifier& iso_lang)
 233     {
 234         active_iso_lang_ = iso_lang;
 235     }
 236
 237     // If Text is a placeholder substitute it with
 238     //its substitute else leave it unchanged
 239     void substitute(std::string& Text)
 240     {
 241         replacement_table_t* prt = get_replacement_table(active_iso_lang_.make_std_string());
 242         OSL_ASSERT(prt);
 243         replacement_table_t::iterator iter = prt->find(Text);
 244         if (iter != prt->end())
 245             Text = iter->second;
 246     }
 247
 248     void add_substitution(
 249         const std::string& Placeholder, const std::string& Substitute)
 250     {
 251         replacement_table_t* prt = get_replacement_table(active_iso_lang_.make_std_string());
 252         OSL_ASSERT(prt);
 253         prt->insert(std::make_pair(Placeholder, Substitute));
 254     }
 255
 256
 257 private:
 258     // Return the replacement table for the iso lang id
 259     // create a new one if not already present
 260     replacement_table_t* get_replacement_table(const std::string& iso_lang)
 261     {
 262         iso_lang_replacement_table_t::iterator iter =
 263             iso_lang_replacement_table_.find(iso_lang);
 264
 265         replacement_table_t* prt = NULL;
 266
 267         if (iso_lang_replacement_table_.end() == iter)
 268         {
 269             prt = new replacement_table_t();
 270             iso_lang_replacement_table_.insert(std::make_pair(iso_lang, prt));
 271         }
 272         else
 273         {
 274             prt = iter->second;
 275         }
 276         return prt;
 277     }
 278
 279 private:
 280     iso_lang_replacement_table_t iso_lang_replacement_table_;
 281     iso_lang_identifier active_iso_lang_;
 282 };
 283
 284 typedef std::map< unsigned short , std::string , std::less< unsigned short > > shortmap;
 285
 286 void add_group_entries(
 287     Config& aConfig,
 288     const OString& GroupName,
 289     Substitutor& Substitutor)
 290 {
 291     OSL_ASSERT(aConfig.HasGroup(GroupName));
 292
 293     aConfig.SetGroup(GroupName);
 294     size_t key_count = aConfig.GetKeyCount();
 295     shortmap map;
 296
 297     for (size_t i = 0; i < key_count; i++)
 298     {
 299         OString iso_lang = aConfig.GetKeyName(sal::static_int_cast<sal_uInt16>(i));
 300         OString key_value_utf8 = aConfig.ReadKey(sal::static_int_cast<sal_uInt16>(i));
 301         iso_lang_identifier myiso_lang( iso_lang );
 302         LanguageType ltype = LanguageTag( myiso_lang.make_OUString()).makeFallback().getLanguageType();
 303         if(  ( ltype & 0x0200 ) == 0 && map[ ltype ].empty()  )
 304         {
 305             Substitutor.set_language(iso_lang_identifier(iso_lang));
 306
 307             key_value_utf8 = comphelper::string::strip(key_value_utf8, '\"');
 308
 309             OUString key_value_utf16 =
 310                 OStringToOUString(key_value_utf8, RTL_TEXTENCODING_UTF8);
 311
 312             Substitutor.add_substitution(
 313                 GroupName.getStr(), make_winrc_unicode_string(key_value_utf16));
 314             map[ static_cast<unsigned short>(ltype) ] = std::string( iso_lang.getStr() );
 315         }
 316         else
 317         {
 318             if( !map[ ltype ].empty() )
 319             {
 320                 printf("ERROR: Duplicated ms id %d found for the languages %s and %s !!!! This does not work in microsoft resources\nPlease remove one!\n", ltype , map[ ltype ].c_str() , iso_lang.getStr());
 321                 exit( -1 );
 322             }
 323         }
 324     }
 325 }
 326
 327 void read_ulf_file(const std::string& FileName, Substitutor& Substitutor)
 328 {
 329     // work-around for #i32420#
 330
 331     // as the Config class is currently not able to deal correctly with
 332     // UTF8 files starting with a byte-order-mark we create a copy of the
 333     // original file without the byte-order-mark
 334     OUString tmpfile_url;
 335     osl_createTempFile(NULL, NULL, &tmpfile_url.pData);
 336
 337     OUString tmpfile_sys;
 338     osl::FileBase::getSystemPathFromFileURL(tmpfile_url, tmpfile_sys);
 339
 340     std::ifstream in(FileName.c_str());
 341     std::ofstream out(OUStringToOString(tmpfile_sys).getStr());
 342
 343     try
 344     {
 345         StreamExceptionsEnabler sexc_out(out);
 346         StreamExceptionsEnabler sexc_in(in);
 347
 348         //skip the byte-order-mark 0xEF 0xBB 0xBF, identifying UTF8 files
 349         unsigned char BOM[3] = {0xEF, 0xBB, 0xBF};
 350         char buff[3];
 351         in.read(&buff[0], 3);
 352
 353         if (memcmp(buff, BOM, 3) != 0)
 354             in.seekg(0);
 355
 356         std::string line;
 357         while (std::getline(in, line))
 358             out << line << std::endl;
 359     }
 360     catch (const std::ios::failure&)
 361     {
 362         if (!in.eof())
 363             throw;
 364     }
 365
 366
 367     // end work-around for #i32420#
 368
 369     Config config(tmpfile_url.getStr());
 370     size_t grpcnt = config.GetGroupCount();
 371     for (size_t i = 0; i < grpcnt; i++)
 372         add_group_entries(config, config.GetGroupName(sal::static_int_cast<sal_uInt16>(i)), Substitutor);
 373 }
 374
 375 void read_file(
 376     const std::string& fname,
 377     string_container_t& string_container)
 378 {
 379     std::ifstream file(fname.c_str());
 380     StreamExceptionsEnabler sexc(file);
 381
 382     try
 383     {
 384         std::string line;
 385         while (std::getline(file, line))
 386             string_container.push_back(line);
 387     }
 388     catch(const std::ios::failure&)
 389     {
 390         if (!file.eof())
 391             throw;
 392     }
 393 }
 394
 395 /** A simple helper function that appens the
 396     content of one file to another one  */
 397 void concatenate_files(std::ostream& os, std::istream& is)
 398 {
 399     StreamExceptionsEnabler os_sexc(os);
 400     StreamExceptionsEnabler is_sexc(is);
 401
 402     try
 403     {
 404         std::string line;
 405         while (std::getline(is, line))
 406             os << line << std::endl;
 407     }
 408     catch(const std::ios::failure&)
 409     {
 410         if (!is.eof())
 411             throw;
 412     }
 413 }
 414
 415 bool is_placeholder(const std::string& str)
 416 {
 417     return ((str.length() > 1) &&
 418             ('%' == str[0]) &&
 419             ('%' == str[str.length() - 1]));
 420 }
 421
 422 void start_language_section(
 423     std::ostream_iterator<std::string>& ostream_iter, const iso_lang_identifier& iso_lang)
 424 {
 425     ostream_iter = std::string();
 426
 427     std::string lang_section("LANGUAGE ");
 428
 429     LanguageType ltype = LanguageTag( iso_lang.make_OUString()).makeFallback().getLanguageType();
 430
 431     char buff[10];
 432     int primLangID = PRIMARYLANGID(ltype);
 433     int subLangID = SUBLANGID(ltype);
 434     // Our resources are normally not sub language dependent.
 435     // Esp. for spanish we don't want to distinguish between trad.
 436     // and internatinal sorting ( which leads to two different sub languages )
 437     // Setting the sub language to neutral allows us to use one
 438     // stringlist for all spanish variants
 439     if ( ( primLangID == LANG_SPANISH ) &&
 440          ( subLangID == SUBLANG_SPANISH ) )
 441         subLangID = SUBLANG_NEUTRAL;
 442
 443 #ifdef WNT
 444     _itoa(primLangID, buff, 16);
 445 #else
 446     sprintf(buff, "%x", primLangID);
 447 #endif
 448     lang_section += std::string("0x") + std::string(buff);
 449
 450     lang_section += std::string(" , ");
 451
 452 #ifdef WNT
 453     _itoa(subLangID, buff, 16);
 454 #else
 455     sprintf(buff, "%x", subLangID);
 456 #endif
 457     lang_section += std::string("0x") + std::string(buff);
 458     ostream_iter = lang_section;
 459 }
 460
 461 /** Iterate all languages in the substitutor,
 462     replace the all placeholder and append the
 463     result to the output file */
 464 void inflate_rc_template_to_file(
 465     std::ostream& os, const string_container_t& rctmpl, Substitutor& substitutor)
 466 {
 467     StreamExceptionsEnabler sexc(os);
 468
 469     Substitutor::const_iterator iter = substitutor.begin();
 470     Substitutor::const_iterator iter_end = substitutor.end();
 471
 472     std::ostream_iterator<std::string> oi(os, "\n");
 473
 474     for ( /**/ ;iter != iter_end; ++iter)
 475     {
 476         substitutor.set_language(iso_lang_identifier(iter->first));
 477
 478         string_container_t::const_iterator rct_iter = rctmpl.begin();
 479         string_container_t::const_iterator rct_iter_end = rctmpl.end();
 480
 481         if (!rctmpl.empty())
 482             start_language_section(oi, iter->first);
 483
 484         for ( /**/ ;rct_iter != rct_iter_end; ++rct_iter)
 485         {
 486             std::istringstream iss(*rct_iter);
 487             std::string line;
 488
 489             while (iss)
 490             {
 491                 std::string token;
 492                 iss >> token;
 493                 substitutor.substitute(token);
 494
 495                 // HACK for partially merged
 496                 // *.lng files where some strings have
 497                 // a particular language that others
 498                 // don't have in order to keep the
 499                 // build
 500                 if (is_placeholder(token))
 501                     token = make_winrc_unicode_string(token);
 502
 503                 line += token;
 504                 line += " ";
 505             }
 506             oi = line;
 507         }
 508     }
 509 }
 510
 511 } // namespace /* private */
 512
 513 /* MAIN
 514    The file names provided via command line should be
 515    absolute or relative to the directory of this module.
 516
 517    Algo:
 518    1. read the ulf file and initialize the substitutor
 519    2. read the resource template file
 520    3. create the output file and append the header
 521    4. inflate the resource template to the output file
 522       for every language using the substitutor
 523    5. append the footer
 524 */
 525 #define MAKE_ABSOLUTE(s) (get_absolute_file_path((s)).getStr())
 526 #define ULF_FILE(c)    MAKE_ABSOLUTE((c).get_arg("-ulf"))
 527 #define RC_TEMPLATE(c) MAKE_ABSOLUTE((c).get_arg("-rct"))
 528 #define RC_FILE(c)     MAKE_ABSOLUTE((c).get_arg("-rc"))
 529 #define RC_HEADER(c)   MAKE_ABSOLUTE((c).get_arg("-rch"))
 530 #define RC_FOOTER(c)   MAKE_ABSOLUTE((c).get_arg("-rcf"))
 531
 532 SAL_IMPLEMENT_MAIN_WITH_ARGS(argc, argv)
 533 {
 534     try
 535     {
 536         CommandLine cmdline(argc, argv);
 537
 538         Substitutor substitutor;
 539         read_ulf_file(ULF_FILE(cmdline), substitutor);
 540
 541         string_container_t rc_tmpl;
 542         read_file(RC_TEMPLATE(cmdline), rc_tmpl);
 543
 544         std::ofstream rc_file(RC_FILE(cmdline));
 545         std::ifstream in_header(RC_HEADER(cmdline));
 546         concatenate_files(rc_file, in_header);
 547
 548         inflate_rc_template_to_file(rc_file, rc_tmpl, substitutor);
 549
 550         std::ifstream in_footer(RC_FOOTER(cmdline));
 551         concatenate_files(rc_file, in_footer);
 552     }
 553     catch(const std::ios::failure& ex)
 554     {
 555         std::cout << ex.what() << std::endl;
 556         return 1;
 557     }
 558     catch(const std::exception& ex)
 559     {
 560         std::cout << ex.what() << std::endl;
 561         ShowUsage();
 562         return 1;
 563     }
 564     catch(...)
 565     {
 566         std::cout << "Unexpected error..." << std::endl;
 567         return 1;
 568     }
 569     return 0;
 570 }
 571
 572 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */