l10ntools/inc/export.hxx

   1 /*************************************************************************
   2  *
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * Copyright 2000, 2010 Oracle and/or its affiliates.
   6  *
   7  * OpenOffice.org - a multi-platform office productivity suite
   8  *
   9  * This file is part of OpenOffice.org.
  10  *
  11  * OpenOffice.org is free software: you can redistribute it and/or modify
  12  * it under the terms of the GNU Lesser General Public License version 3
  13  * only, as published by the Free Software Foundation.
  14  *
  15  * OpenOffice.org is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU Lesser General Public License version 3 for more details
  19  * (a copy is included in the LICENSE file that accompanied this code).
  20  *
  21  * You should have received a copy of the GNU Lesser General Public License
  22  * version 3 along with OpenOffice.org.  If not, see
  23  * <http://www.openoffice.org/license.html>
  24  * for a copy of the LGPLv3 License.
  25  *
  26  ************************************************************************/
  27
  28 #ifndef _EXPORT_HXX
  29 #define _EXPORT_HXX
  30
  31 #ifndef L10NTOOLS_DIRECTORY_HXX
  32 #define L10NTOOLS_DIRECTORY_HXX
  33 #include <l10ntools/directory.hxx>
  34 #endif
  35
  36
  37 // #define MERGE_SOURCE_LANGUAGES <- To merge en-US and de resource
  38
  39 #include <tools/string.hxx>
  40 #include <tools/list.hxx>
  41 #include <tools/stream.hxx>
  42 #include <tools/fsys.hxx>
  43 #include <osl/file.hxx>
  44 #include <osl/file.h>
  45
  46 #include <hash_map> /* std::hashmap*/
  47 #include <iterator> /* std::iterator*/
  48 #include <set>      /* std::set*/
  49 #include <vector>   /* std::vector*/
  50 #include <queue>
  51 #include <string>
  52
  53 #include <unistd.h>
  54 #ifdef WNT
  55 #include <direct.h>
  56 #endif
  57
  58 #define NO_TRANSLATE_ISO        "x-no-translate"
  59
  60 #define JAPANESE_ISO "ja"
  61
  62
  63 struct eqstr{
  64   BOOL operator()(const char* s1, const char* s2) const{
  65     return strcmp(s1,s2)==0;
  66   }
  67 };
  68
  69 struct equalByteString{
  70         bool operator()( const ByteString& rKey1, const ByteString& rKey2 ) const {
  71             return rKey1.CompareTo( rKey2 )==COMPARE_EQUAL;
  72     }
  73 };
  74 struct lessByteString{
  75         bool operator()( const ByteString& rKey1, const ByteString& rKey2 ) const {
  76             return rKey1.CompareTo( rKey2 )==COMPARE_LESS;
  77     }
  78 };
  79
  80 struct hashByteString{
  81     size_t operator()( const ByteString& rName ) const{
  82                 std::hash< const char* > myHash;
  83                 return myHash( rName.GetBuffer() );
  84     }
  85 };
  86
  87 class PFormEntrys;
  88 class MergeData;
  89 typedef std::set<ByteString , lessByteString > ByteStringSet;
  90
  91 typedef std::hash_map<ByteString , ByteString , hashByteString,equalByteString>
  92                                 ByteStringHashMap;
  93
  94 typedef std::hash_map<ByteString , bool , hashByteString,equalByteString>
  95                                 ByteStringBoolHashMap;
  96
  97 typedef std::hash_map<ByteString , PFormEntrys* , hashByteString,equalByteString>
  98                                 PFormEntrysHashMap;
  99
 100 typedef std::hash_map<ByteString , MergeData* , hashByteString,equalByteString>
 101                                 MergeDataHashMap;
 102
 103 #define SOURCE_LANGUAGE ByteString("en-US")
 104 #define LIST_REFID  "LIST_REFID"
 105
 106 typedef ByteStringHashMap ExportListEntry;
 107
 108 DECLARE_LIST( ExportListBase, ExportListEntry * )
 109
 110 //
 111 // class ExportList
 112 //
 113
 114 class ExportList : public ExportListBase
 115 {
 116 private:
 117     ULONG nSourceLanguageListEntryCount;
 118
 119 public:
 120     ExportList() : ExportListBase() { nSourceLanguageListEntryCount = 0; }
 121     ULONG GetSourceLanguageListEntryCount() { return nSourceLanguageListEntryCount; }
 122     void NewSourceLanguageListEntry() { nSourceLanguageListEntryCount++; }
 123 };
 124
 125 #define REFID_NONE 0xFFFF
 126
 127 //
 128 // struct ResData
 129 //
 130
 131 /******************************************************************************
 132 * Purpose: holds mandatory data to export a single res (used with ResStack)
 133 ******************************************************************************/
 134
 135 #define ID_LEVEL_NULL       0x0000
 136 #define ID_LEVEL_AUTOID     0x0001
 137 #define ID_LEVEL_TEXT       0x0002
 138 #define ID_LEVEL_FIELDNAME  0x0003
 139 #define ID_LEVEL_ACCESSPATH 0x0004
 140 #define ID_LEVEL_IDENTIFIER 0x0005
 141 #define ID_LEVEL_LISTINDEX  0x0006
 142
 143 class ResData
 144 {
 145 public:
 146     ~ResData();
 147     BOOL SetId( const ByteString &rId, USHORT nLevel );
 148
 149     USHORT nWidth;
 150     USHORT nChildIndex;
 151     USHORT nIdLevel;
 152     BOOL bChild;
 153     BOOL bChildWithText;
 154
 155     BOOL bText;
 156     BOOL bHelpText;
 157     BOOL bQuickHelpText;
 158     BOOL bTitle;
 159     BOOL bList;
 160
 161     BOOL bRestMerged;
 162
 163     ByteString sResTyp;
 164     ByteString sId;
 165     ByteString sGId;
 166     ByteString sHelpId;
 167     ByteString sFilename;
 168
 169     ByteStringHashMap sText;
 170     USHORT nTextRefId;
 171
 172     ByteStringHashMap sHelpText;
 173     USHORT nHelpTextRefId;
 174
 175     ByteStringHashMap sQuickHelpText;
 176     USHORT nQuickHelpTextRefId;
 177
 178     ByteStringHashMap sTitle;
 179     USHORT nTitleRefId;
 180
 181     ByteString sTextTyp;
 182     ByteStringHashMap aFallbackData;
 183     ByteStringHashMap aMergedLanguages;
 184
 185     ExportList  *pStringList;
 186     ExportList  *pUIEntries;
 187     ExportList  *pItemList;
 188     ExportList  *pFilterList;
 189     ExportList  *pPairedList;
 190
 191     ByteString sPForm;
 192
 193     void Dump();
 194     void addFallbackData( ByteString& sId , const ByteString& sText );
 195     bool getFallbackData( ByteString& sId , ByteString& sText);
 196
 197     void addMergedLanguage( ByteString& sLang );
 198     bool isMerged( ByteString& sLang );
 199     ResData( const ByteString &rPF, const ByteString &rGId )
 200             :
 201             nWidth( 0 ),
 202             nChildIndex( 0 ),
 203             nIdLevel( ID_LEVEL_NULL ),
 204             bChild( FALSE ),
 205             bChildWithText( FALSE ),
 206             bText( FALSE ),
 207             bHelpText( FALSE ),
 208             bQuickHelpText( FALSE ),
 209             bTitle( FALSE ),
 210             bList( FALSE ),
 211             bRestMerged( FALSE ),
 212             sGId( rGId ),
 213             nTextRefId( REFID_NONE ),
 214             nHelpTextRefId( REFID_NONE ),
 215             nQuickHelpTextRefId( REFID_NONE ),
 216             nTitleRefId( REFID_NONE ),
 217             sTextTyp( "Text" ),
 218             pStringList( NULL ),
 219             pUIEntries( NULL ),
 220             pItemList( NULL ),
 221             pFilterList( NULL ),
 222             pPairedList( NULL ),
 223             sPForm( rPF )
 224     {
 225         sGId.EraseAllChars( '\r' );
 226         sPForm.EraseAllChars( '\r' );
 227     };
 228     ResData( const ByteString &rPF, const ByteString &rGId , const ByteString &rFilename )
 229             :
 230             nChildIndex( 0 ),
 231             nIdLevel( ID_LEVEL_NULL ),
 232             bChild( FALSE ),
 233             bChildWithText( FALSE ),
 234             bText( FALSE ),
 235             bHelpText( FALSE ),
 236             bQuickHelpText( FALSE ),
 237             bTitle( FALSE ),
 238             bList( FALSE ),
 239             bRestMerged( FALSE ),
 240             sGId( rGId ),
 241             sFilename( rFilename ),
 242             nTextRefId( REFID_NONE ),
 243             nHelpTextRefId( REFID_NONE ),
 244             nQuickHelpTextRefId( REFID_NONE ),
 245             nTitleRefId( REFID_NONE ),
 246             sTextTyp( "Text" ),
 247             pStringList( NULL ),
 248             pUIEntries( NULL ),
 249             pItemList( NULL ),
 250             pFilterList( NULL ),
 251             pPairedList( NULL ),
 252             sPForm( rPF )
 253
 254     {
 255         sGId.EraseAllChars( '\r' );
 256         sPForm.EraseAllChars( '\r' );
 257     };
 258
 259
 260 };
 261
 262
 263 //
 264 // class Export
 265 //
 266
 267 /******************************************************************************
 268 * Purpose: syntax check and export of *.src, called from lexer
 269 ******************************************************************************/
 270
 271 #define LIST_NON                    0x0000
 272 #define LIST_STRING                 0x0001
 273 #define LIST_FILTER                 0x0002
 274 #define LIST_ITEM                   0x0004
 275 #define LIST_PAIRED                 0x0005
 276 #define LIST_UIENTRIES              0x0008
 277 #define STRING_TYP_TEXT             0x0010
 278 #define STRING_TYP_HELPTEXT         0x0020
 279 #define STRING_TYP_QUICKHELPTEXT    0x0040
 280 #define STRING_TYP_TITLE            0x0080
 281
 282 #define MERGE_MODE_NORMAL           0x0000
 283 #define MERGE_MODE_LIST             0x0001
 284
 285 DECLARE_LIST( ResStack, ResData * )
 286 // forwards
 287 class WordTransformer;
 288 class ParserQueue;
 289
 290 class Export
 291 {
 292 private:
 293     WordTransformer *pWordTransformer;
 294
 295     CharSet aCharSet;                   // used charset in src
 296
 297     SvFileStream aOutput;
 298
 299     ResStack aResStack;                 // stack for parsing recursive
 300
 301     ByteString sActPForm;               // hold cur. system
 302
 303     BOOL bDefine;                       // cur. res. in a define?
 304     BOOL bNextMustBeDefineEOL;          // define but no \ at lineend
 305     ULONG nLevel;                       // res. recursiv? how deep?
 306     USHORT nList;                       // cur. res. is String- or FilterList
 307     ByteString nListLang;
 308     ULONG nListIndex;
 309     ULONG nListLevel;
 310     bool bSkipFile;
 311     ByteString sProject;
 312     ByteString sRoot;
 313     BOOL bEnableExport;
 314     BOOL bMergeMode;
 315     ByteString sMergeSrc;
 316     ByteString sLastListLine;
 317     BOOL bError;                        // any errors while export?
 318     BOOL bReadOver;
 319     BOOL bDontWriteOutput;
 320     ByteString sLastTextTyp;
 321     static bool isInitialized;
 322     ByteString sFilename;
 323
 324
 325 public:
 326     ParserQueue* pParseQueue; // public ?
 327     static ByteString sLanguages; // public ?
 328     static ByteString sForcedLanguages; // public ?
 329
 330
 331     static bool skipProject( ByteString sPrj ) ;
 332     static void InitLanguages( bool bMergeMode = false );
 333     static void InitForcedLanguages( bool bMergeMode = false );
 334     static std::vector<ByteString> GetLanguages();
 335     static std::vector<ByteString> GetForcedLanguages();
 336
 337     static void SetLanguages( std::vector<ByteString> val );
 338     static void RemoveUTF8ByteOrderMarker( ByteString &rString );
 339     static bool hasUTF8ByteOrderMarker( const ByteString &rString );
 340     static void RemoveUTF8ByteOrderMarkerFromFile( const ByteString &rFilename );
 341     static bool fileHasUTF8ByteOrderMarker( const ByteString &rString );
 342     static ByteString GetIsoLangByIndex( USHORT nIndex );
 343     static void QuotHTML( ByteString &rString );
 344     static bool CopyFile( const ByteString& source , const ByteString& dest );
 345
 346     static void QuotHTMLXRM( ByteString &rString );
 347     static void UnquotHTML( ByteString &rString );
 348
 349     static const char* GetEnv( const char *pVar );
 350     static int getCurrentDirectory( rtl::OUString& base_fqurl , rtl::OUString& base );
 351
 352     static bool isSourceLanguage( const ByteString &sLanguage );
 353     static bool isAllowed( const ByteString &sLanguage );
 354
 355     static bool LanguageAllowed( const ByteString &nLanguage );
 356     static void Languages( std::vector<ByteString>::const_iterator& begin , std::vector<ByteString>::const_iterator& end );
 357     static void getRandomName( const ByteString& sPrefix , ByteString& sRandStr , const ByteString& sPostfix  );
 358     static void getRandomName( ByteString& sRandStr );
 359     static void getCurrentDir( std::string& dir );
 360
 361     static void replaceEncoding( ByteString& rString );
 362
 363     static ByteString GetFallbackLanguage( const ByteString nLanguage );
 364     static void FillInFallbacks( ResData *pResData );
 365     static void FillInListFallbacks( ExportList *pList, const ByteString &nSource, const ByteString &nFallback );
 366     static ByteString GetTimeStamp();
 367     static BOOL ConvertLineEnds( ByteString sSource, ByteString sDestination );
 368     static ByteString GetNativeFile( ByteString sSource );
 369     static DirEntry GetTempFile();
 370
 371     static void DumpExportList( ByteString& sListName , ExportList& aList );
 372     static ByteString DumpMap( ByteString& sMapName , ByteStringHashMap& aMap );
 373
 374 private:
 375     static std::vector<ByteString> aLanguages;
 376     static std::vector<ByteString> aForcedLanguages;
 377
 378     BOOL ListExists( ResData *pResData, USHORT nLst );
 379
 380     BOOL WriteData( ResData *pResData, BOOL bCreateNew = FALSE );// called befor dest. cur ResData
 381     BOOL WriteExportList( ResData *pResData, ExportList *pExportList,
 382                         const ByteString &rTyp, BOOL bCreateNew = FALSE );
 383
 384     ByteString MergePairedList( ByteString& sLine , ByteString& sText );
 385
 386     ByteString FullId();                    // creates cur. GID
 387
 388     bool PairedListFallback( ByteString& sText , ResData& aResData );
 389
 390     ByteString GetPairedListID      ( const ByteString& sText );
 391     ByteString GetPairedListString  ( const ByteString& sText );
 392     ByteString StripList    ( const ByteString& sText );
 393
 394     void UnmergeUTF8( ByteString& sOrig );
 395     void InsertListEntry( const ByteString &rText, const ByteString &rLine );
 396     void CleanValue( ByteString &rValue );
 397     ByteString GetText( const ByteString &rSource, int nToken );
 398
 399     BOOL PrepareTextToMerge( ByteString &rText, USHORT nTyp,
 400         ByteString &nLangIndex, ResData *pResData );
 401
 402     void MergeRest( ResData *pResData, USHORT nMode = MERGE_MODE_NORMAL );
 403     void ConvertMergeContent( ByteString &rText );
 404
 405       void WriteToMerged( const ByteString &rText , bool bSDFContent );
 406     void SetChildWithText();
 407
 408     void CutComment( ByteString &rText );
 409
 410 public:
 411     Export( const ByteString &rOutput, BOOL bWrite,
 412             const ByteString &rPrj, const ByteString &rPrjRoot , const ByteString& rFile );
 413     Export( const ByteString &rOutput, BOOL bWrite,
 414             const ByteString &rPrj, const ByteString &rPrjRoot,
 415             const ByteString &rMergeSource , const ByteString& rFile );
 416     ~Export();
 417
 418     void Init();
 419     int Execute( int nToken, const char * pToken ); // called from lexer
 420     void SetError() { bError = TRUE; }
 421     BOOL GetError() { return bError; }
 422 };
 423
 424
 425 //
 426 // class PFormEntrys
 427 //
 428
 429 /******************************************************************************
 430 * Purpose: holds information of data to merge (one pform)
 431 ******************************************************************************/
 432
 433 class PFormEntrys : public ByteString
 434 {
 435 friend class MergeDataFile;
 436 private:
 437     ByteString sHelpText; // empty string
 438     ByteStringHashMap sText;
 439     ByteStringBoolHashMap bTextFirst;
 440     ByteStringHashMap sQuickHelpText;
 441     ByteStringBoolHashMap bQuickHelpTextFirst;
 442     ByteStringHashMap sTitle;
 443     ByteStringBoolHashMap bTitleFirst;
 444
 445 public:
 446     PFormEntrys( const ByteString &rPForm ) : ByteString( rPForm ) {};
 447     ByteString Dump();
 448     void InsertEntry(
 449                     const ByteString &nId ,
 450                     const ByteString &rText,
 451                     const ByteString &rQuickHelpText,
 452                     const ByteString &rTitle
 453                     )
 454         {
 455
 456             sText[ nId ] = rText;
 457             bTextFirst[ nId ] = true;
 458             sQuickHelpText[ nId ] = rQuickHelpText;
 459             bQuickHelpTextFirst[ nId ] = true;
 460             sTitle[ nId ] = rTitle;
 461             bTitleFirst[ nId ] = true;
 462         }
 463      BOOL GetText( ByteString &rReturn, USHORT nTyp, const ByteString &nLangIndex, BOOL bDel = FALSE );
 464      BOOL GetTransex3Text( ByteString &rReturn, USHORT nTyp, const ByteString &nLangIndex, BOOL bDel = FALSE );
 465
 466 };
 467
 468 //
 469 // class MergeData
 470 //
 471
 472 /******************************************************************************
 473 * Purpose: holds information of data to merge (one ressource)
 474 ******************************************************************************/
 475
 476 class MergeDataFile;
 477
 478 class MergeData
 479 {
 480 friend class MergeDataFile;
 481 private:
 482     ByteString sTyp;
 483     ByteString sGID;
 484     ByteString sLID;
 485     ByteString sFilename;
 486     PFormEntrysHashMap aMap;
 487 public:
 488     MergeData( const ByteString &rTyp, const ByteString &rGID, const ByteString &rLID , const ByteString &rFilename )
 489             : sTyp( rTyp ), sGID( rGID ), sLID( rLID ) , sFilename( rFilename ) {};
 490     ~MergeData();
 491     PFormEntrys* InsertEntry( const ByteString &rPForm );
 492     PFormEntrys* GetPFormEntrys( ResData *pResData );
 493
 494     void Insert( const ByteString& rPFO , PFormEntrys* pfEntrys );
 495     PFormEntrys* GetPFObject( const ByteString& rPFO );
 496
 497     ByteString Dump();
 498     BOOL operator==( ResData *pData );
 499 };
 500
 501 //
 502 // class MergeDataFile
 503 //
 504
 505 /******************************************************************************
 506 * Purpose: holds information of data to merge
 507 ******************************************************************************/
 508
 509 class MergeDataFile
 510 {
 511 private:
 512     BOOL bErrorLog;
 513     ByteString sErrorLog;
 514     SvFileStream aErrLog;
 515     ByteStringSet aLanguageSet;
 516     MergeDataHashMap aMap;
 517     ByteStringHashMap aLanguageMap;
 518     std::vector<ByteString> aLanguageList;
 519     ByteStringHashMap aFilenames;
 520
 521
 522 public:
 523     MergeDataFile( const ByteString &rFileName, const ByteString& rFile , BOOL bErrLog, CharSet aCharSet, bool bCaseSensitive = false );
 524     ~MergeDataFile();
 525
 526
 527     std::vector<ByteString> GetLanguages();
 528     MergeData *GetMergeData( ResData *pResData , bool bCaseSensitve = false );
 529
 530     PFormEntrys *GetPFormEntrys( ResData *pResData );
 531     PFormEntrys *GetPFormEntrysCaseSensitive( ResData *pResData );
 532
 533     void InsertEntry( const ByteString &rTYP, const ByteString &rGID, const ByteString &rLID,
 534                 const ByteString &rPFO,
 535                 const ByteString &nLang , const ByteString &rTEXT,
 536                 const ByteString &rQHTEXT, const ByteString &rTITLE ,
 537                 const ByteString &sFilename , bool bCaseSensitive
 538                 );
 539     static USHORT GetLangIndex( USHORT nId );
 540     static ByteString CreateKey( const ByteString& rTYP , const ByteString& rGID , const ByteString& rLID , const ByteString& rFilename , bool bCaseSensitive = false );
 541
 542     ByteString Dump();
 543     void WriteError( const ByteString &rLine );
 544 };
 545
 546
 547 class QueueEntry
 548 {
 549 public:
 550     QueueEntry( int nTypVal , ByteString sLineVal ): nTyp( nTypVal ) , sLine( sLineVal ){};
 551     int nTyp;
 552     ByteString sLine;
 553 };
 554
 555 class ParserQueue
 556 {
 557 public:
 558
 559     ParserQueue( Export& aExportObj );
 560     ~ParserQueue();
 561
 562     inline void Push( const QueueEntry& aEntry );
 563     bool bCurrentIsM;  // public ?
 564     bool bNextIsM;   // public ?
 565     bool bLastWasM;   // public ?
 566     bool bMflag;   // public ?
 567
 568     void Close();
 569 private:
 570     // Future / Next
 571     std::queue<QueueEntry>* aQueueNext;
 572     // Current
 573     std::queue<QueueEntry>* aQueueCur;
 574     // Ref
 575     std::queue<QueueEntry>* aQref;
 576
 577     Export& aExport;
 578     bool bStart;
 579     bool bStartNext;
 580
 581     inline void Pop( std::queue<QueueEntry>& aQueue );
 582
 583 };
 584 #endif
 585