l10ntools/source/po.cxx

   1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
   2 /*
   3  * This file is part of the LibreOffice project.
   4  *
   5  * This Source Code Form is subject to the terms of the Mozilla Public
   6  * License, v. 2.0. If a copy of the MPL was not distributed with this
   7  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
   8  */
   9
  10 #include <rtl/ustring.hxx>
  11 #include <rtl/strbuf.hxx>
  12 #include <rtl/crc.h>
  13 #include <sal/log.hxx>
  14
  15 #include <cstring>
  16 #include <ctime>
  17 #include <cassert>
  18
  19 #include <vector>
  20 #include <string>
  21 #include <string_view>
  22
  23 #include <po.hxx>
  24 #include <helper.hxx>
  25
  26 /** Container of po entry
  27
  28     Provide all file operations related to LibreOffice specific
  29     po entry and store it's attributes.
  30 */
  31 class GenPoEntry
  32 {
  33 private:
  34     OStringBuffer m_sExtractCom;
  35     std::vector<OString>    m_sReferences;
  36     OString    m_sMsgCtxt;
  37     OString    m_sMsgId;
  38     OString    m_sMsgIdPlural;
  39     OString    m_sMsgStr;
  40     std::vector<OString>    m_sMsgStrPlural;
  41     bool       m_bFuzzy;
  42     bool       m_bCFormat;
  43     bool       m_bNull;
  44
  45 public:
  46     GenPoEntry();
  47
  48     const std::vector<OString>& getReference() const    { return m_sReferences; }
  49     const OString& getMsgCtxt() const      { return m_sMsgCtxt; }
  50     const OString& getMsgId() const        { return m_sMsgId; }
  51     const OString& getMsgStr() const       { return m_sMsgStr; }
  52     bool        isFuzzy() const         { return m_bFuzzy; }
  53     bool        isNull() const          { return m_bNull; }
  54
  55     void        setExtractCom(std::string_view rExtractCom)
  56                         {
  57                             m_sExtractCom = rExtractCom;
  58                         }
  59     void        setReference(const OString& rReference)
  60                         {
  61                             m_sReferences.push_back(rReference);
  62                         }
  63     void        setMsgCtxt(const OString& rMsgCtxt)
  64                         {
  65                             m_sMsgCtxt = rMsgCtxt;
  66                         }
  67     void        setMsgId(const OString& rMsgId)
  68                         {
  69                             m_sMsgId = rMsgId;
  70                         }
  71     void        setMsgStr(const OString& rMsgStr)
  72                         {
  73                             m_sMsgStr = rMsgStr;
  74                         }
  75
  76     void        writeToFile(std::ofstream& rOFStream) const;
  77     void        readFromFile(std::ifstream& rIFStream);
  78 };
  79
  80 namespace
  81 {
  82     // Convert a normal string to msg/po output string
  83     OString lcl_GenMsgString(std::string_view rString)
  84     {
  85         if ( rString.empty() )
  86             return "\"\""_ostr;
  87
  88         OString sResult =
  89             "\"" +
  90             helper::escapeAll(rString,"\n""\t""\r""\\""\"","\\n""\\t""\\r""\\\\""\\\"") +
  91             "\"";
  92         sal_Int32 nIndex = 0;
  93         while((nIndex=sResult.indexOf("\\n",nIndex))!=-1)
  94         {
  95             if( !sResult.match("\\\\n", nIndex-1) &&
  96                 nIndex!=sResult.getLength()-3)
  97             {
  98                sResult = sResult.replaceAt(nIndex,2,"\\n\"\n\"");
  99             }
 100             ++nIndex;
 101         }
 102
 103         if ( sResult.indexOf('\n') != -1 )
 104             return "\"\"\n" +  sResult;
 105
 106         return sResult;
 107     }
 108
 109     // Convert msg string to normal form
 110     OString lcl_GenNormString(std::string_view rString)
 111     {
 112         return
 113             helper::unEscapeAll(
 114                 rString.substr(1,rString.size()-2),
 115                 "\\n""\\t""\\r""\\\\""\\\"",
 116                 "\n""\t""\r""\\""\"");
 117     }
 118 }
 119
 120 GenPoEntry::GenPoEntry()
 121     : m_bFuzzy( false )
 122     , m_bCFormat( false )
 123     , m_bNull( false )
 124 {
 125 }
 126
 127 void GenPoEntry::writeToFile(std::ofstream& rOFStream) const
 128 {
 129     if ( rOFStream.tellp() != std::ofstream::pos_type( 0 ))
 130         rOFStream << std::endl;
 131     if ( !m_sExtractCom.isEmpty() )
 132         rOFStream
 133             << "#. "
 134             << m_sExtractCom.toString().replaceAll("\n"_ostr,"\n#. "_ostr) << std::endl;
 135     for(const auto& rReference : m_sReferences)
 136         rOFStream << "#: " << rReference << std::endl;
 137     if ( m_bFuzzy )
 138         rOFStream << "#, fuzzy" << std::endl;
 139     if ( m_bCFormat )
 140         rOFStream << "#, c-format" << std::endl;
 141     if ( !m_sMsgCtxt.isEmpty() )
 142         rOFStream << "msgctxt "
 143                   << lcl_GenMsgString(m_sMsgCtxt)
 144                   << std::endl;
 145     rOFStream << "msgid "
 146               << lcl_GenMsgString(m_sMsgId) << std::endl;
 147     if ( !m_sMsgIdPlural.isEmpty() )
 148         rOFStream << "msgid_plural "
 149                   << lcl_GenMsgString(m_sMsgIdPlural)
 150                   << std::endl;
 151     if ( !m_sMsgStrPlural.empty() )
 152         for(auto & line : m_sMsgStrPlural)
 153             rOFStream << line.copy(0,10) << lcl_GenMsgString(line.subView(10)) << std::endl;
 154     else
 155         rOFStream << "msgstr "
 156                   << lcl_GenMsgString(m_sMsgStr) << std::endl;
 157 }
 158
 159 void GenPoEntry::readFromFile(std::ifstream& rIFStream)
 160 {
 161     *this = GenPoEntry();
 162     OString* pLastMsg = nullptr;
 163     std::string sTemp;
 164     getline(rIFStream,sTemp);
 165     if( rIFStream.eof() || sTemp.empty() )
 166     {
 167         m_bNull = true;
 168         return;
 169     }
 170     while(!rIFStream.eof())
 171     {
 172         OString sLine(sTemp.data(),sTemp.length());
 173         if (sLine.startsWith("#. "))
 174         {
 175             if( !m_sExtractCom.isEmpty() )
 176             {
 177                 m_sExtractCom.append("\n");
 178             }
 179             m_sExtractCom.append(sLine.subView(3));
 180         }
 181         else if (sLine.startsWith("#: "))
 182         {
 183             m_sReferences.push_back(sLine.copy(3));
 184         }
 185         else if (sLine.startsWith("#, fuzzy"))
 186         {
 187             m_bFuzzy = true;
 188         }
 189         else if (sLine.startsWith("#, c-format"))
 190         {
 191             m_bCFormat = true;
 192         }
 193         else if (sLine.startsWith("msgctxt "))
 194         {
 195             m_sMsgCtxt = lcl_GenNormString(sLine.subView(8));
 196             pLastMsg = &m_sMsgCtxt;
 197         }
 198         else if (sLine.startsWith("msgid "))
 199         {
 200             m_sMsgId = lcl_GenNormString(sLine.subView(6));
 201             pLastMsg = &m_sMsgId;
 202         }
 203         else if (sLine.startsWith("msgid_plural "))
 204         {
 205             m_sMsgIdPlural = lcl_GenNormString(sLine.subView(13));
 206             pLastMsg = &m_sMsgIdPlural;
 207         }
 208         else if (sLine.startsWith("msgstr "))
 209         {
 210             m_sMsgStr = lcl_GenNormString(sLine.subView(7));
 211             pLastMsg = &m_sMsgStr;
 212         }
 213         else if (sLine.startsWith("msgstr["))
 214         {
 215             // assume there are no more than 10 plural forms...
 216             // and that plural strings are never split to multi-line in po
 217             m_sMsgStrPlural.push_back(sLine.subView(0,10) + lcl_GenNormString(sLine.subView(10)));
 218         }
 219         else if (sLine.startsWith("\"") && pLastMsg)
 220         {
 221             OString sReference;
 222             if (!m_sReferences.empty())
 223             {
 224                 sReference = m_sReferences.front();
 225             }
 226             if (pLastMsg != &m_sMsgCtxt || sLine != Concat2View("\"" + sReference + "\\n\""))
 227             {
 228                 *pLastMsg += lcl_GenNormString(sLine);
 229             }
 230         }
 231         else
 232             break;
 233         getline(rIFStream,sTemp);
 234     }
 235  }
 236
 237 PoEntry::PoEntry()
 238     : m_bIsInitialized( false )
 239 {
 240 }
 241
 242 PoEntry::PoEntry(
 243     std::string_view rSourceFile, std::string_view rResType, std::string_view rGroupId,
 244     std::string_view rLocalId, std::string_view rHelpText,
 245     const OString& rText, const TYPE eType )
 246     : m_bIsInitialized( false )
 247 {
 248     if( rSourceFile.empty() )
 249         throw NOSOURCFILE;
 250     else if ( rResType.empty() )
 251         throw NORESTYPE;
 252     else if ( rGroupId.empty() )
 253         throw NOGROUPID;
 254     else if ( rText.isEmpty() )
 255         throw NOSTRING;
 256     else if ( rHelpText.size() == 5 )
 257         throw WRONGHELPTEXT;
 258
 259     m_pGenPo.reset( new GenPoEntry() );
 260     size_t idx = rSourceFile.rfind('/');
 261     if (idx == std::string_view::npos)
 262         idx = 0;
 263     OString sReference(rSourceFile.substr(idx+1));
 264     m_pGenPo->setReference(sReference);
 265
 266     OString sMsgCtxt =
 267         sReference + "\n" +
 268         rGroupId + "\n" +
 269         (rLocalId.empty() ? OString() : OString::Concat(rLocalId) + "\n") +
 270         rResType;
 271     switch(eType){
 272     case TTEXT:
 273         sMsgCtxt += ".text"; break;
 274     case TQUICKHELPTEXT:
 275         sMsgCtxt += ".quickhelptext"; break;
 276     case TTITLE:
 277         sMsgCtxt += ".title"; break;
 278     // Default case is unneeded because the type of eType has only three element
 279     }
 280     m_pGenPo->setMsgCtxt(sMsgCtxt);
 281     m_pGenPo->setMsgId(rText);
 282     m_pGenPo->setExtractCom(Concat2View(
 283         ( !rHelpText.empty() ?  OString::Concat(rHelpText) + "\n" : OString()) +
 284         genKeyId( m_pGenPo->getReference().front() + rGroupId + rLocalId + rResType + rText ) ));
 285     m_bIsInitialized = true;
 286 }
 287
 288 PoEntry::~PoEntry()
 289 {
 290 }
 291
 292 PoEntry::PoEntry( const PoEntry& rPo )
 293     : m_pGenPo( rPo.m_pGenPo ? new GenPoEntry( *(rPo.m_pGenPo) ) : nullptr )
 294     , m_bIsInitialized( rPo.m_bIsInitialized )
 295 {
 296 }
 297
 298 PoEntry& PoEntry::operator=(const PoEntry& rPo)
 299 {
 300     if( this == &rPo )
 301     {
 302         return *this;
 303     }
 304     if( rPo.m_pGenPo )
 305     {
 306         if( m_pGenPo )
 307         {
 308             *m_pGenPo = *(rPo.m_pGenPo);
 309         }
 310         else
 311         {
 312             m_pGenPo.reset( new GenPoEntry( *(rPo.m_pGenPo) ) );
 313         }
 314     }
 315     else
 316     {
 317         m_pGenPo.reset();
 318     }
 319     m_bIsInitialized = rPo.m_bIsInitialized;
 320     return *this;
 321 }
 322
 323 PoEntry& PoEntry::operator=(PoEntry&& rPo) noexcept
 324 {
 325     m_pGenPo = std::move(rPo.m_pGenPo);
 326     m_bIsInitialized = std::move(rPo.m_bIsInitialized);
 327     return *this;
 328 }
 329
 330 OString const & PoEntry::getSourceFile() const
 331 {
 332     assert( m_bIsInitialized );
 333     return m_pGenPo->getReference().front();
 334 }
 335
 336 OString PoEntry::getGroupId() const
 337 {
 338     assert( m_bIsInitialized );
 339     return m_pGenPo->getMsgCtxt().getToken(0,'\n');
 340 }
 341
 342 OString PoEntry::getLocalId() const
 343 {
 344     assert( m_bIsInitialized );
 345     const OString sMsgCtxt = m_pGenPo->getMsgCtxt();
 346     if (sMsgCtxt.indexOf('\n')==sMsgCtxt.lastIndexOf('\n'))
 347         return OString();
 348     else
 349         return sMsgCtxt.getToken(1,'\n');
 350 }
 351
 352 OString PoEntry::getResourceType() const
 353 {
 354     assert( m_bIsInitialized );
 355     const OString sMsgCtxt = m_pGenPo->getMsgCtxt();
 356     if (sMsgCtxt.indexOf('\n')==sMsgCtxt.lastIndexOf('\n'))
 357         return sMsgCtxt.getToken(1,'\n').getToken(0,'.');
 358     else
 359         return sMsgCtxt.getToken(2,'\n').getToken(0,'.');
 360 }
 361
 362 PoEntry::TYPE PoEntry::getType() const
 363 {
 364     assert( m_bIsInitialized );
 365     const OString sMsgCtxt = m_pGenPo->getMsgCtxt();
 366     const OString sType = sMsgCtxt.copy( sMsgCtxt.lastIndexOf('.') + 1 );
 367     assert(
 368         (sType == "text" || sType == "quickhelptext" || sType == "title") );
 369     if ( sType == "text" )
 370         return TTEXT;
 371     else if ( sType == "quickhelptext" )
 372         return TQUICKHELPTEXT;
 373     else
 374         return TTITLE;
 375 }
 376
 377 bool PoEntry::isFuzzy() const
 378 {
 379     assert( m_bIsInitialized );
 380     return m_pGenPo->isFuzzy();
 381 }
 382
 383 // Get message context
 384 const OString& PoEntry::getMsgCtxt() const
 385 {
 386     assert( m_bIsInitialized );
 387     return m_pGenPo->getMsgCtxt();
 388
 389 }
 390
 391 // Get translation string in merge format
 392 OString const & PoEntry::getMsgId() const
 393 {
 394     assert( m_bIsInitialized );
 395     return m_pGenPo->getMsgId();
 396 }
 397
 398 // Get translated string in merge format
 399 const OString& PoEntry::getMsgStr() const
 400 {
 401     assert( m_bIsInitialized );
 402     return m_pGenPo->getMsgStr();
 403
 404 }
 405
 406 bool PoEntry::IsInSameComp(const PoEntry& rPo1,const PoEntry& rPo2)
 407 {
 408     assert( rPo1.m_bIsInitialized && rPo2.m_bIsInitialized );
 409     return ( rPo1.getSourceFile() == rPo2.getSourceFile() &&
 410              rPo1.getGroupId() == rPo2.getGroupId() &&
 411              rPo1.getLocalId() == rPo2.getLocalId() &&
 412              rPo1.getResourceType() == rPo2.getResourceType() );
 413 }
 414
 415 OString PoEntry::genKeyId(const OString& rGenerator)
 416 {
 417     sal_uInt32 nCRC = rtl_crc32(0, rGenerator.getStr(), rGenerator.getLength());
 418     // Use simple ASCII characters, exclude I, l, 1 and O, 0 to avoid confusing IDs
 419     static const char sSymbols[] =
 420         "ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz23456789";
 421     char sKeyId[6];
 422     for( short nKeyInd = 0; nKeyInd < 5; ++nKeyInd )
 423     {
 424         sKeyId[nKeyInd] = sSymbols[(nCRC & 63) % strlen(sSymbols)];
 425         nCRC >>= 6;
 426     }
 427     sKeyId[5] = '\0';
 428     return sKeyId;
 429 }
 430
 431 namespace
 432 {
 433     // Get actual time in "YEAR-MO-DA HO:MI+ZONE" form
 434     OString lcl_GetTime()
 435     {
 436         time_t aNow = time(nullptr);
 437         struct tm* pNow = localtime(&aNow);
 438         char pBuff[50];
 439         strftime( pBuff, sizeof pBuff, "%Y-%m-%d %H:%M%z", pNow );
 440         return pBuff;
 441     }
 442 }
 443
 444 // when updating existing files (pocheck), reuse provided po-header
 445 PoHeader::PoHeader( std::string_view rExtSrc, const OString& rPoHeaderMsgStr )
 446     : m_pGenPo( new GenPoEntry() )
 447     , m_bIsInitialized( false )
 448 {
 449     m_pGenPo->setExtractCom(Concat2View(OString::Concat("extracted from ") + rExtSrc));
 450     m_pGenPo->setMsgStr(rPoHeaderMsgStr);
 451     m_bIsInitialized = true;
 452 }
 453
 454 PoHeader::PoHeader( std::string_view rExtSrc )
 455     : m_pGenPo( new GenPoEntry() )
 456     , m_bIsInitialized( false )
 457 {
 458     m_pGenPo->setExtractCom(Concat2View(OString::Concat("extracted from ") + rExtSrc));
 459     m_pGenPo->setMsgStr(
 460         "Project-Id-Version: PACKAGE VERSION\n"
 461         "Report-Msgid-Bugs-To: https://bugs.libreoffice.org/enter_bug.cgi?"
 462         "product=LibreOffice&bug_status=UNCONFIRMED&component=UI\n"
 463         "POT-Creation-Date: " + lcl_GetTime() +
 464         "\nPO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 465         "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 466         "Language-Team: LANGUAGE <LL@li.org>\n"
 467         "MIME-Version: 1.0\n"
 468         "Content-Type: text/plain; charset=UTF-8\n"
 469         "Content-Transfer-Encoding: 8bit\n"
 470         "X-Accelerator-Marker: ~\n"
 471         "X-Generator: LibreOffice\n");
 472     m_bIsInitialized = true;
 473 }
 474
 475 PoHeader::~PoHeader()
 476 {
 477 }
 478
 479 PoOfstream::PoOfstream()
 480     : m_bIsAfterHeader( false )
 481 {
 482 }
 483
 484 PoOfstream::PoOfstream(const OString& rFileName, OpenMode aMode )
 485     : m_bIsAfterHeader( false )
 486 {
 487     open( rFileName, aMode );
 488 }
 489
 490 PoOfstream::~PoOfstream()
 491 {
 492     if( isOpen() )
 493     {
 494        close();
 495     }
 496 }
 497
 498 void PoOfstream::open(const OString& rFileName, OpenMode aMode )
 499 {
 500     assert( !isOpen() );
 501     if( aMode == TRUNC )
 502     {
 503         m_aOutPut.open( rFileName.getStr(),
 504             std::ios_base::out | std::ios_base::trunc );
 505         m_bIsAfterHeader = false;
 506     }
 507     else if( aMode == APP )
 508     {
 509         m_aOutPut.open( rFileName.getStr(),
 510             std::ios_base::out | std::ios_base::app );
 511         m_bIsAfterHeader = m_aOutPut.tellp() != std::ofstream::pos_type( 0 );
 512     }
 513 }
 514
 515 void PoOfstream::close()
 516 {
 517     assert( isOpen() );
 518     m_aOutPut.close();
 519 }
 520
 521 void PoOfstream::writeHeader(const PoHeader& rPoHeader)
 522 {
 523     assert( isOpen() && !m_bIsAfterHeader && rPoHeader.m_bIsInitialized );
 524     rPoHeader.m_pGenPo->writeToFile( m_aOutPut );
 525     m_bIsAfterHeader = true;
 526 }
 527
 528 void PoOfstream::writeEntry( const PoEntry& rPoEntry )
 529 {
 530     assert( isOpen() && m_bIsAfterHeader && rPoEntry.m_bIsInitialized );
 531     rPoEntry.m_pGenPo->writeToFile( m_aOutPut );
 532 }
 533
 534 namespace
 535 {
 536
 537 // Check the validity of read entry
 538 bool lcl_CheckInputEntry(const GenPoEntry& rEntry)
 539 {
 540     // stock button labels don't have a reference/sourcefile - they are not extracted from ui files
 541     // (explicitly skipped by solenv/bin/uiex) but instead inserted by l10ntools/source/localize.cxx
 542     // into all module templates (see d5d905b480c2a9b1db982f2867e87b5c230d1ab9)
 543     return !rEntry.getMsgCtxt().isEmpty() &&
 544            (rEntry.getMsgCtxt() == "stock" || !rEntry.getReference().empty()) &&
 545            !rEntry.getMsgId().isEmpty();
 546 }
 547
 548 }
 549
 550 PoIfstream::PoIfstream()
 551     : m_bEof( false )
 552 {
 553 }
 554
 555 PoIfstream::PoIfstream(const OString& rFileName)
 556     : m_bEof( false )
 557 {
 558     open( rFileName );
 559 }
 560
 561 PoIfstream::~PoIfstream()
 562 {
 563     if( isOpen() )
 564     {
 565        close();
 566     }
 567 }
 568
 569 void PoIfstream::open( const OString& rFileName, OString& rPoHeader )
 570 {
 571     assert( !isOpen() );
 572     m_aInPut.open( rFileName.getStr(), std::ios_base::in );
 573
 574     // capture header, updating timestamp and generator
 575     std::string sTemp;
 576     std::getline(m_aInPut,sTemp);
 577     while( !sTemp.empty() && !m_aInPut.eof() )
 578     {
 579         std::getline(m_aInPut,sTemp);
 580         OString sLine(sTemp.data(),sTemp.length());
 581         if (sLine.startsWith("\"PO-Revision-Date"))
 582             rPoHeader += "PO-Revision-Date: " + lcl_GetTime() + "\n";
 583         else if (sLine.startsWith("\"X-Generator"))
 584             rPoHeader += "X-Generator: LibreOffice\n";
 585         else if (sLine.startsWith("\""))
 586             rPoHeader += lcl_GenNormString(sLine);
 587     }
 588     m_bEof = false;
 589 }
 590
 591 void PoIfstream::open( const OString& rFileName )
 592 {
 593     assert( !isOpen() );
 594     m_aInPut.open( rFileName.getStr(), std::ios_base::in );
 595
 596     // Skip header
 597     std::string sTemp;
 598     std::getline(m_aInPut,sTemp);
 599     while( !sTemp.empty() && !m_aInPut.eof() )
 600     {
 601         std::getline(m_aInPut,sTemp);
 602     }
 603     m_bEof = false;
 604 }
 605
 606 void PoIfstream::close()
 607 {
 608     assert( isOpen() );
 609     m_aInPut.close();
 610 }
 611
 612 void PoIfstream::readEntry( PoEntry& rPoEntry )
 613 {
 614     assert( isOpen() && !eof() );
 615     GenPoEntry aGenPo;
 616     aGenPo.readFromFile( m_aInPut );
 617     if( aGenPo.isNull() )
 618     {
 619         m_bEof = true;
 620         rPoEntry = PoEntry();
 621     }
 622     else
 623     {
 624         if( lcl_CheckInputEntry(aGenPo) )
 625         {
 626             if( rPoEntry.m_pGenPo )
 627             {
 628                 *(rPoEntry.m_pGenPo) = std::move(aGenPo);
 629             }
 630             else
 631             {
 632                 rPoEntry.m_pGenPo.reset( new GenPoEntry(std::move(aGenPo)) );
 633             }
 634             rPoEntry.m_bIsInitialized = true;
 635         }
 636         else
 637         {
 638             SAL_WARN("l10ntools", "Parse problem with entry: " << aGenPo.getMsgStr());
 639             throw PoIfstream::Exception();
 640         }
 641     }
 642 }
 643
 644 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */