l10ntools/source/po.cxx

   1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
   2 /*
   3  * This file is part of the LibreOffice project.
   4  *
   5  * This Source Code Form is subject to the terms of the Mozilla Public
   6  * License, v. 2.0. If a copy of the MPL was not distributed with this
   7  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
   8  */
   9
  10 #include <rtl/ustring.hxx>
  11 #include <rtl/crc.h>
  12 #include <sal/log.hxx>
  13
  14 #include <cstring>
  15 #include <ctime>
  16 #include <cassert>
  17
  18 #include <vector>
  19 #include <string>
  20
  21 #include <po.hxx>
  22 #include <helper.hxx>
  23
  24 /** Container of po entry
  25
  26     Provide all file operations related to LibreOffice specific
  27     po entry and store it's attributes.
  28 */
  29 class GenPoEntry
  30 {
  31 private:
  32     OStringBuffer m_sExtractCom;
  33     std::vector<OString>    m_sReferences;
  34     OString    m_sMsgCtxt;
  35     OString    m_sMsgId;
  36     OString    m_sMsgIdPlural;
  37     OString    m_sMsgStr;
  38     std::vector<OString>    m_sMsgStrPlural;
  39     bool       m_bFuzzy;
  40     bool       m_bCFormat;
  41     bool       m_bNull;
  42
  43 public:
  44     GenPoEntry();
  45
  46     const std::vector<OString>& getReference() const    { return m_sReferences; }
  47     const OString& getMsgCtxt() const      { return m_sMsgCtxt; }
  48     const OString& getMsgId() const        { return m_sMsgId; }
  49     const OString& getMsgStr() const       { return m_sMsgStr; }
  50     bool        isFuzzy() const         { return m_bFuzzy; }
  51     bool        isNull() const          { return m_bNull; }
  52
  53     void        setExtractCom(const OString& rExtractCom)
  54                         {
  55                             m_sExtractCom = rExtractCom;
  56                         }
  57     void        setReference(const OString& rReference)
  58                         {
  59                             m_sReferences.push_back(rReference);
  60                         }
  61     void        setMsgCtxt(const OString& rMsgCtxt)
  62                         {
  63                             m_sMsgCtxt = rMsgCtxt;
  64                         }
  65     void        setMsgId(const OString& rMsgId)
  66                         {
  67                             m_sMsgId = rMsgId;
  68                         }
  69     void        setMsgStr(const OString& rMsgStr)
  70                         {
  71                             m_sMsgStr = rMsgStr;
  72                         }
  73
  74     void        writeToFile(std::ofstream& rOFStream) const;
  75     void        readFromFile(std::ifstream& rIFStream);
  76 };
  77
  78 namespace
  79 {
  80     // Convert a normal string to msg/po output string
  81     OString lcl_GenMsgString(const OString& rString)
  82     {
  83         if ( rString.isEmpty() )
  84             return "\"\"";
  85
  86         OString sResult =
  87             "\"" +
  88             helper::escapeAll(rString,"\n""\t""\r""\\""\"","\\n""\\t""\\r""\\\\""\\\"") +
  89             "\"";
  90         sal_Int32 nIndex = 0;
  91         while((nIndex=sResult.indexOf("\\n",nIndex))!=-1)
  92         {
  93             if( !sResult.match("\\\\n", nIndex-1) &&
  94                 nIndex!=sResult.getLength()-3)
  95             {
  96                sResult = sResult.replaceAt(nIndex,2,"\\n\"\n\"");
  97             }
  98             ++nIndex;
  99         }
 100
 101         if ( sResult.indexOf('\n') != -1 )
 102             return "\"\"\n" +  sResult;
 103
 104         return sResult;
 105     }
 106
 107     // Convert msg string to normal form
 108     OString lcl_GenNormString(const OString& rString)
 109     {
 110         return
 111             helper::unEscapeAll(
 112                 rString.copy(1,rString.getLength()-2),
 113                 "\\n""\\t""\\r""\\\\""\\\"",
 114                 "\n""\t""\r""\\""\"");
 115     }
 116 }
 117
 118 GenPoEntry::GenPoEntry()
 119     : m_sExtractCom( OString() )
 120     , m_sReferences( std::vector<OString>() )
 121     , m_sMsgCtxt( OString() )
 122     , m_sMsgId( OString() )
 123     , m_sMsgIdPlural( OString() )
 124     , m_sMsgStr( OString() )
 125     , m_sMsgStrPlural( std::vector<OString>() )
 126     , m_bFuzzy( false )
 127     , m_bCFormat( false )
 128     , m_bNull( false )
 129 {
 130 }
 131
 132 void GenPoEntry::writeToFile(std::ofstream& rOFStream) const
 133 {
 134     if ( rOFStream.tellp() != std::ofstream::pos_type( 0 ))
 135         rOFStream << std::endl;
 136     if ( !m_sExtractCom.isEmpty() )
 137         rOFStream
 138             << "#. "
 139             << m_sExtractCom.toString().replaceAll("\n","\n#. ") << std::endl;
 140     for(const auto& rReference : m_sReferences)
 141         rOFStream << "#: " << rReference << std::endl;
 142     if ( m_bFuzzy )
 143         rOFStream << "#, fuzzy" << std::endl;
 144     if ( m_bCFormat )
 145         rOFStream << "#, c-format" << std::endl;
 146     if ( !m_sMsgCtxt.isEmpty() )
 147         rOFStream << "msgctxt "
 148                   << lcl_GenMsgString(m_sMsgCtxt)
 149                   << std::endl;
 150     rOFStream << "msgid "
 151               << lcl_GenMsgString(m_sMsgId) << std::endl;
 152     if ( !m_sMsgIdPlural.isEmpty() )
 153         rOFStream << "msgid_plural "
 154                   << lcl_GenMsgString(m_sMsgIdPlural)
 155                   << std::endl;
 156     if ( !m_sMsgStrPlural.empty() )
 157         for(auto & line : m_sMsgStrPlural)
 158             rOFStream << line.copy(0,10) << lcl_GenMsgString(line.copy(10)) << std::endl;
 159     else
 160         rOFStream << "msgstr "
 161                   << lcl_GenMsgString(m_sMsgStr) << std::endl;
 162 }
 163
 164 void GenPoEntry::readFromFile(std::ifstream& rIFStream)
 165 {
 166     *this = GenPoEntry();
 167     OString* pLastMsg = nullptr;
 168     std::string sTemp;
 169     getline(rIFStream,sTemp);
 170     if( rIFStream.eof() || sTemp.empty() )
 171     {
 172         m_bNull = true;
 173         return;
 174     }
 175     while(!rIFStream.eof())
 176     {
 177         OString sLine(sTemp.data(),sTemp.length());
 178         if (sLine.startsWith("#. "))
 179         {
 180             if( !m_sExtractCom.isEmpty() )
 181             {
 182                 m_sExtractCom.append("\n");
 183             }
 184             m_sExtractCom.append(sLine.copy(3));
 185         }
 186         else if (sLine.startsWith("#: "))
 187         {
 188             m_sReferences.push_back(sLine.copy(3));
 189         }
 190         else if (sLine.startsWith("#, fuzzy"))
 191         {
 192             m_bFuzzy = true;
 193         }
 194         else if (sLine.startsWith("#, c-format"))
 195         {
 196             m_bCFormat = true;
 197         }
 198         else if (sLine.startsWith("msgctxt "))
 199         {
 200             m_sMsgCtxt = lcl_GenNormString(sLine.copy(8));
 201             pLastMsg = &m_sMsgCtxt;
 202         }
 203         else if (sLine.startsWith("msgid "))
 204         {
 205             m_sMsgId = lcl_GenNormString(sLine.copy(6));
 206             pLastMsg = &m_sMsgId;
 207         }
 208         else if (sLine.startsWith("msgid_plural "))
 209         {
 210             m_sMsgIdPlural = lcl_GenNormString(sLine.copy(13));
 211             pLastMsg = &m_sMsgIdPlural;
 212         }
 213         else if (sLine.startsWith("msgstr "))
 214         {
 215             m_sMsgStr = lcl_GenNormString(sLine.copy(7));
 216             pLastMsg = &m_sMsgStr;
 217         }
 218         else if (sLine.startsWith("msgstr["))
 219         {
 220             // assume there are no more than 10 plural forms...
 221             // and that plural strings are never split to multi-line in po
 222             m_sMsgStrPlural.push_back(sLine.subView(0,10) + lcl_GenNormString(sLine.copy(10)));
 223         }
 224         else if (sLine.startsWith("\"") && pLastMsg)
 225         {
 226             OString sReference;
 227             if (!m_sReferences.empty())
 228             {
 229                 sReference = m_sReferences.front();
 230             }
 231             if (pLastMsg != &m_sMsgCtxt || sLine != "\"" + sReference + "\\n\"")
 232             {
 233                 *pLastMsg += lcl_GenNormString(sLine);
 234             }
 235         }
 236         else
 237             break;
 238         getline(rIFStream,sTemp);
 239     }
 240  }
 241
 242 PoEntry::PoEntry()
 243     : m_bIsInitialized( false )
 244 {
 245 }
 246
 247 PoEntry::PoEntry(
 248     const OString& rSourceFile, const OString& rResType, const OString& rGroupId,
 249     const OString& rLocalId, const OString& rHelpText,
 250     const OString& rText, const TYPE eType )
 251     : m_bIsInitialized( false )
 252 {
 253     if( rSourceFile.isEmpty() )
 254         throw NOSOURCFILE;
 255     else if ( rResType.isEmpty() )
 256         throw NORESTYPE;
 257     else if ( rGroupId.isEmpty() )
 258         throw NOGROUPID;
 259     else if ( rText.isEmpty() )
 260         throw NOSTRING;
 261     else if ( rHelpText.getLength() == 5 )
 262         throw WRONGHELPTEXT;
 263
 264     m_pGenPo.reset( new GenPoEntry() );
 265     OString sReference = rSourceFile.copy(rSourceFile.lastIndexOf('/')+1);
 266     m_pGenPo->setReference(sReference);
 267
 268     OString sMsgCtxt =
 269         sReference + "\n" +
 270         rGroupId + "\n" +
 271         (rLocalId.isEmpty() ? OString() : rLocalId + "\n") +
 272         rResType;
 273     switch(eType){
 274     case TTEXT:
 275         sMsgCtxt += ".text"; break;
 276     case TQUICKHELPTEXT:
 277         sMsgCtxt += ".quickhelptext"; break;
 278     case TTITLE:
 279         sMsgCtxt += ".title"; break;
 280     // Default case is unneeded because the type of eType has only three element
 281     }
 282     m_pGenPo->setMsgCtxt(sMsgCtxt);
 283     m_pGenPo->setMsgId(rText);
 284     m_pGenPo->setExtractCom(
 285         ( !rHelpText.isEmpty() ?  rHelpText + "\n" : OString()) +
 286         genKeyId( m_pGenPo->getReference().front() + rGroupId + rLocalId + rResType + rText ) );
 287     m_bIsInitialized = true;
 288 }
 289
 290 PoEntry::~PoEntry()
 291 {
 292 }
 293
 294 PoEntry::PoEntry( const PoEntry& rPo )
 295     : m_pGenPo( rPo.m_pGenPo ? new GenPoEntry( *(rPo.m_pGenPo) ) : nullptr )
 296     , m_bIsInitialized( rPo.m_bIsInitialized )
 297 {
 298 }
 299
 300 PoEntry& PoEntry::operator=(const PoEntry& rPo)
 301 {
 302     if( this == &rPo )
 303     {
 304         return *this;
 305     }
 306     if( rPo.m_pGenPo )
 307     {
 308         if( m_pGenPo )
 309         {
 310             *m_pGenPo = *(rPo.m_pGenPo);
 311         }
 312         else
 313         {
 314             m_pGenPo.reset( new GenPoEntry( *(rPo.m_pGenPo) ) );
 315         }
 316     }
 317     else
 318     {
 319         m_pGenPo.reset();
 320     }
 321     m_bIsInitialized = rPo.m_bIsInitialized;
 322     return *this;
 323 }
 324
 325 PoEntry& PoEntry::operator=(PoEntry&& rPo) noexcept
 326 {
 327     m_pGenPo = std::move(rPo.m_pGenPo);
 328     m_bIsInitialized = std::move(rPo.m_bIsInitialized);
 329     return *this;
 330 }
 331
 332 OString const & PoEntry::getSourceFile() const
 333 {
 334     assert( m_bIsInitialized );
 335     return m_pGenPo->getReference().front();
 336 }
 337
 338 OString PoEntry::getGroupId() const
 339 {
 340     assert( m_bIsInitialized );
 341     return m_pGenPo->getMsgCtxt().getToken(0,'\n');
 342 }
 343
 344 OString PoEntry::getLocalId() const
 345 {
 346     assert( m_bIsInitialized );
 347     const OString sMsgCtxt = m_pGenPo->getMsgCtxt();
 348     if (sMsgCtxt.indexOf('\n')==sMsgCtxt.lastIndexOf('\n'))
 349         return OString();
 350     else
 351         return sMsgCtxt.getToken(1,'\n');
 352 }
 353
 354 OString PoEntry::getResourceType() const
 355 {
 356     assert( m_bIsInitialized );
 357     const OString sMsgCtxt = m_pGenPo->getMsgCtxt();
 358     if (sMsgCtxt.indexOf('\n')==sMsgCtxt.lastIndexOf('\n'))
 359         return sMsgCtxt.getToken(1,'\n').getToken(0,'.');
 360     else
 361         return sMsgCtxt.getToken(2,'\n').getToken(0,'.');
 362 }
 363
 364 PoEntry::TYPE PoEntry::getType() const
 365 {
 366     assert( m_bIsInitialized );
 367     const OString sMsgCtxt = m_pGenPo->getMsgCtxt();
 368     const OString sType = sMsgCtxt.copy( sMsgCtxt.lastIndexOf('.') + 1 );
 369     assert(
 370         (sType == "text" || sType == "quickhelptext" || sType == "title") );
 371     if ( sType == "text" )
 372         return TTEXT;
 373     else if ( sType == "quickhelptext" )
 374         return TQUICKHELPTEXT;
 375     else
 376         return TTITLE;
 377 }
 378
 379 bool PoEntry::isFuzzy() const
 380 {
 381     assert( m_bIsInitialized );
 382     return m_pGenPo->isFuzzy();
 383 }
 384
 385 // Get message context
 386 const OString& PoEntry::getMsgCtxt() const
 387 {
 388     assert( m_bIsInitialized );
 389     return m_pGenPo->getMsgCtxt();
 390
 391 }
 392
 393 // Get translation string in merge format
 394 OString const & PoEntry::getMsgId() const
 395 {
 396     assert( m_bIsInitialized );
 397     return m_pGenPo->getMsgId();
 398 }
 399
 400 // Get translated string in merge format
 401 const OString& PoEntry::getMsgStr() const
 402 {
 403     assert( m_bIsInitialized );
 404     return m_pGenPo->getMsgStr();
 405
 406 }
 407
 408 bool PoEntry::IsInSameComp(const PoEntry& rPo1,const PoEntry& rPo2)
 409 {
 410     assert( rPo1.m_bIsInitialized && rPo2.m_bIsInitialized );
 411     return ( rPo1.getSourceFile() == rPo2.getSourceFile() &&
 412              rPo1.getGroupId() == rPo2.getGroupId() &&
 413              rPo1.getLocalId() == rPo2.getLocalId() &&
 414              rPo1.getResourceType() == rPo2.getResourceType() );
 415 }
 416
 417 OString PoEntry::genKeyId(const OString& rGenerator)
 418 {
 419     sal_uInt32 nCRC = rtl_crc32(0, rGenerator.getStr(), rGenerator.getLength());
 420     // Use simple ASCII characters, exclude I, l, 1 and O, 0 to avoid confusing IDs
 421     static const char sSymbols[] =
 422         "ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz23456789";
 423     char sKeyId[6];
 424     for( short nKeyInd = 0; nKeyInd < 5; ++nKeyInd )
 425     {
 426         sKeyId[nKeyInd] = sSymbols[(nCRC & 63) % strlen(sSymbols)];
 427         nCRC >>= 6;
 428     }
 429     sKeyId[5] = '\0';
 430     return sKeyId;
 431 }
 432
 433 namespace
 434 {
 435     // Get actual time in "YEAR-MO-DA HO:MI+ZONE" form
 436     OString lcl_GetTime()
 437     {
 438         time_t aNow = time(nullptr);
 439         struct tm* pNow = localtime(&aNow);
 440         char pBuff[50];
 441         strftime( pBuff, sizeof pBuff, "%Y-%m-%d %H:%M%z", pNow );
 442         return pBuff;
 443     }
 444 }
 445
 446 // when updating existing files (pocheck), reuse provided po-header
 447 PoHeader::PoHeader( const OString& rExtSrc, const OString& rPoHeaderMsgStr )
 448     : m_pGenPo( new GenPoEntry() )
 449     , m_bIsInitialized( false )
 450 {
 451     m_pGenPo->setExtractCom("extracted from " + rExtSrc);
 452     m_pGenPo->setMsgStr(rPoHeaderMsgStr);
 453     m_bIsInitialized = true;
 454 }
 455
 456 PoHeader::PoHeader( const OString& rExtSrc )
 457     : m_pGenPo( new GenPoEntry() )
 458     , m_bIsInitialized( false )
 459 {
 460     m_pGenPo->setExtractCom("extracted from " + rExtSrc);
 461     m_pGenPo->setMsgStr(
 462         "Project-Id-Version: PACKAGE VERSION\n"
 463         "Report-Msgid-Bugs-To: https://bugs.libreoffice.org/enter_bug.cgi?"
 464         "product=LibreOffice&bug_status=UNCONFIRMED&component=UI\n"
 465         "POT-Creation-Date: " + lcl_GetTime() +
 466         "\nPO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 467         "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 468         "Language-Team: LANGUAGE <LL@li.org>\n"
 469         "MIME-Version: 1.0\n"
 470         "Content-Type: text/plain; charset=UTF-8\n"
 471         "Content-Transfer-Encoding: 8bit\n"
 472         "X-Accelerator-Marker: ~\n"
 473         "X-Generator: LibreOffice\n");
 474     m_bIsInitialized = true;
 475 }
 476
 477 PoHeader::~PoHeader()
 478 {
 479 }
 480
 481 PoOfstream::PoOfstream()
 482     : m_aOutPut()
 483     , m_bIsAfterHeader( false )
 484 {
 485 }
 486
 487 PoOfstream::PoOfstream(const OString& rFileName, OpenMode aMode )
 488     : m_aOutPut()
 489     , m_bIsAfterHeader( false )
 490 {
 491     open( rFileName, aMode );
 492 }
 493
 494 PoOfstream::~PoOfstream()
 495 {
 496     if( isOpen() )
 497     {
 498        close();
 499     }
 500 }
 501
 502 void PoOfstream::open(const OString& rFileName, OpenMode aMode )
 503 {
 504     assert( !isOpen() );
 505     if( aMode == TRUNC )
 506     {
 507         m_aOutPut.open( rFileName.getStr(),
 508             std::ios_base::out | std::ios_base::trunc );
 509         m_bIsAfterHeader = false;
 510     }
 511     else if( aMode == APP )
 512     {
 513         m_aOutPut.open( rFileName.getStr(),
 514             std::ios_base::out | std::ios_base::app );
 515         m_bIsAfterHeader = m_aOutPut.tellp() != std::ofstream::pos_type( 0 );
 516     }
 517 }
 518
 519 void PoOfstream::close()
 520 {
 521     assert( isOpen() );
 522     m_aOutPut.close();
 523 }
 524
 525 void PoOfstream::writeHeader(const PoHeader& rPoHeader)
 526 {
 527     assert( isOpen() && !m_bIsAfterHeader && rPoHeader.m_bIsInitialized );
 528     rPoHeader.m_pGenPo->writeToFile( m_aOutPut );
 529     m_bIsAfterHeader = true;
 530 }
 531
 532 void PoOfstream::writeEntry( const PoEntry& rPoEntry )
 533 {
 534     assert( isOpen() && m_bIsAfterHeader && rPoEntry.m_bIsInitialized );
 535     rPoEntry.m_pGenPo->writeToFile( m_aOutPut );
 536 }
 537
 538 namespace
 539 {
 540
 541 // Check the validity of read entry
 542 bool lcl_CheckInputEntry(const GenPoEntry& rEntry)
 543 {
 544     return !rEntry.getReference().empty() &&
 545            !rEntry.getMsgCtxt().isEmpty() &&
 546            !rEntry.getMsgId().isEmpty();
 547 }
 548
 549 }
 550
 551 PoIfstream::PoIfstream()
 552     : m_aInPut()
 553     , m_bEof( false )
 554 {
 555 }
 556
 557 PoIfstream::PoIfstream(const OString& rFileName)
 558     : m_aInPut()
 559     , m_bEof( false )
 560 {
 561     open( rFileName );
 562 }
 563
 564 PoIfstream::~PoIfstream()
 565 {
 566     if( isOpen() )
 567     {
 568        close();
 569     }
 570 }
 571
 572 void PoIfstream::open( const OString& rFileName, OString& rPoHeader )
 573 {
 574     assert( !isOpen() );
 575     m_aInPut.open( rFileName.getStr(), std::ios_base::in );
 576
 577     // capture header, updating timestamp and generator
 578     std::string sTemp;
 579     std::getline(m_aInPut,sTemp);
 580     while( !sTemp.empty() && !m_aInPut.eof() )
 581     {
 582         std::getline(m_aInPut,sTemp);
 583         OString sLine(sTemp.data(),sTemp.length());
 584         if (sLine.startsWith("\"PO-Revision-Date"))
 585             rPoHeader += "PO-Revision-Date: " + lcl_GetTime() + "\n";
 586         else if (sLine.startsWith("\"X-Generator"))
 587             rPoHeader += "X-Generator: LibreOffice\n";
 588         else if (sLine.startsWith("\""))
 589             rPoHeader += lcl_GenNormString(sLine);
 590     }
 591     m_bEof = false;
 592 }
 593
 594 void PoIfstream::open( const OString& rFileName )
 595 {
 596     assert( !isOpen() );
 597     m_aInPut.open( rFileName.getStr(), std::ios_base::in );
 598
 599     // Skip header
 600     std::string sTemp;
 601     std::getline(m_aInPut,sTemp);
 602     while( !sTemp.empty() && !m_aInPut.eof() )
 603     {
 604         std::getline(m_aInPut,sTemp);
 605     }
 606     m_bEof = false;
 607 }
 608
 609 void PoIfstream::close()
 610 {
 611     assert( isOpen() );
 612     m_aInPut.close();
 613 }
 614
 615 void PoIfstream::readEntry( PoEntry& rPoEntry )
 616 {
 617     assert( isOpen() && !eof() );
 618     GenPoEntry aGenPo;
 619     aGenPo.readFromFile( m_aInPut );
 620     if( aGenPo.isNull() )
 621     {
 622         m_bEof = true;
 623         rPoEntry = PoEntry();
 624     }
 625     else
 626     {
 627         if( lcl_CheckInputEntry(aGenPo) )
 628         {
 629             if( rPoEntry.m_pGenPo )
 630             {
 631                 *(rPoEntry.m_pGenPo) = aGenPo;
 632             }
 633             else
 634             {
 635                 rPoEntry.m_pGenPo.reset( new GenPoEntry( aGenPo ) );
 636             }
 637             rPoEntry.m_bIsInitialized = true;
 638         }
 639         else
 640         {
 641             SAL_WARN("l10ntools", "Parse problem with entry: " << aGenPo.getMsgStr());
 642             throw PoIfstream::Exception();
 643         }
 644     }
 645 }
 646
 647 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */