transex3/inc/export.hxx

   1 /*************************************************************************
   2  *
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * Copyright 2008 by Sun Microsystems, Inc.
   6  *
   7  * OpenOffice.org - a multi-platform office productivity suite
   8  *
   9  * $RCSfile: export.hxx,v $
  10  * $Revision: 1.27 $
  11  *
  12  * This file is part of OpenOffice.org.
  13  *
  14  * OpenOffice.org is free software: you can redistribute it and/or modify
  15  * it under the terms of the GNU Lesser General Public License version 3
  16  * only, as published by the Free Software Foundation.
  17  *
  18  * OpenOffice.org is distributed in the hope that it will be useful,
  19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21  * GNU Lesser General Public License version 3 for more details
  22  * (a copy is included in the LICENSE file that accompanied this code).
  23  *
  24  * You should have received a copy of the GNU Lesser General Public License
  25  * version 3 along with OpenOffice.org.  If not, see
  26  * <http://www.openoffice.org/license.html>
  27  * for a copy of the LGPLv3 License.
  28  *
  29  ************************************************************************/
  30
  31 #ifndef _EXPORT_HXX
  32 #define _EXPORT_HXX
  33
  34 #ifndef TRANSEX_DIRECTORY_HXX
  35 #define TRANSEX_DIRECTORY_HXX
  36 #include <transex3/directory.hxx>
  37 #endif
  38
  39
  40 // #define MERGE_SOURCE_LANGUAGES <- To merge en-US and de resource
  41
  42 #include <tools/string.hxx>
  43 #include <tools/list.hxx>
  44 #include <tools/stream.hxx>
  45 #include <tools/fsys.hxx>
  46 #include <osl/file.hxx>
  47 #include <osl/file.h>
  48
  49 #include <hash_map> /* std::hashmap*/
  50 #include <iterator> /* std::iterator*/
  51 #include <set>      /* std::set*/
  52 #include <vector>   /* std::vector*/
  53 #include <queue>
  54
  55 #define NO_TRANSLATE_ISO                "x-no-translate"
  56
  57 #define JAPANESE_ISO "ja"
  58
  59
  60 struct eqstr{
  61   BOOL operator()(const char* s1, const char* s2) const{
  62     return strcmp(s1,s2)==0;
  63   }
  64 };
  65
  66 struct equalByteString{
  67         bool operator()( const ByteString& rKey1, const ByteString& rKey2 ) const {
  68             return rKey1.CompareTo( rKey2 )==COMPARE_EQUAL;
  69     }
  70 };
  71 struct lessByteString{
  72         bool operator()( const ByteString& rKey1, const ByteString& rKey2 ) const {
  73             return rKey1.CompareTo( rKey2 )==COMPARE_LESS;
  74     }
  75 };
  76
  77 struct hashByteString{
  78     size_t operator()( const ByteString& rName ) const{
  79                 std::hash< const char* > myHash;
  80                 return myHash( rName.GetBuffer() );
  81     }
  82 };
  83
  84 class PFormEntrys;
  85 class MergeData;
  86 typedef std::set<ByteString , lessByteString > ByteStringSet;
  87
  88 typedef std::hash_map<ByteString , ByteString , hashByteString,equalByteString>
  89                                 ByteStringHashMap;
  90
  91 typedef std::hash_map<ByteString , bool , hashByteString,equalByteString>
  92                                 ByteStringBoolHashMap;
  93
  94 typedef std::hash_map<ByteString , PFormEntrys* , hashByteString,equalByteString>
  95                                 PFormEntrysHashMap;
  96
  97 typedef std::hash_map<ByteString , MergeData* , hashByteString,equalByteString>
  98                                 MergeDataHashMap;
  99
 100 #define SOURCE_LANGUAGE ByteString("en-US")
 101 #define LIST_REFID  "LIST_REFID"
 102
 103 typedef ByteStringHashMap ExportListEntry;
 104
 105 DECLARE_LIST( ExportListBase, ExportListEntry * )
 106
 107 //
 108 // class ExportList
 109 //
 110
 111 class ExportList : public ExportListBase
 112 {
 113 private:
 114     ULONG nSourceLanguageListEntryCount;
 115
 116 public:
 117     ExportList() : ExportListBase() { nSourceLanguageListEntryCount = 0; }
 118     ULONG GetSourceLanguageListEntryCount() { return nSourceLanguageListEntryCount; }
 119     void NewSourceLanguageListEntry() { nSourceLanguageListEntryCount++; }
 120 };
 121
 122 #define REFID_NONE 0xFFFF
 123
 124 //
 125 // struct ResData
 126 //
 127
 128 /******************************************************************************
 129 * Purpose: holds mandatory data to export a single res (used with ResStack)
 130 ******************************************************************************/
 131
 132 #define ID_LEVEL_NULL           0x0000
 133 #define ID_LEVEL_AUTOID         0x0001
 134 #define ID_LEVEL_TEXT           0x0002
 135 #define ID_LEVEL_FIELDNAME      0x0003
 136 #define ID_LEVEL_ACCESSPATH     0x0004
 137 #define ID_LEVEL_IDENTIFIER 0x0005
 138 #define ID_LEVEL_LISTINDEX      0x0006
 139
 140 class ResData
 141 {
 142 public:
 143     ~ResData();
 144     BOOL SetId( const ByteString &rId, USHORT nLevel );
 145
 146     USHORT nWidth;
 147     USHORT nChildIndex;
 148     USHORT nIdLevel;
 149     BOOL bChild;
 150     BOOL bChildWithText;
 151
 152     BOOL bText;
 153     BOOL bHelpText;
 154     BOOL bQuickHelpText;
 155     BOOL bTitle;
 156     BOOL bList;
 157
 158     BOOL bRestMerged;
 159
 160     ByteString sResTyp;
 161     ByteString sId;
 162     ByteString sGId;
 163     ByteString sHelpId;
 164     ByteString sFilename;
 165
 166     ByteStringHashMap sText;
 167     USHORT nTextRefId;
 168
 169     ByteStringHashMap sHelpText;
 170     USHORT nHelpTextRefId;
 171
 172     ByteStringHashMap sQuickHelpText;
 173     USHORT nQuickHelpTextRefId;
 174
 175     ByteStringHashMap sTitle;
 176     USHORT nTitleRefId;
 177
 178     ByteString sTextTyp;
 179     ByteStringHashMap aFallbackData;
 180     ByteStringHashMap aMergedLanguages;
 181
 182     ExportList  *pStringList;
 183     ExportList  *pUIEntries;
 184     ExportList  *pItemList;
 185     ExportList  *pFilterList;
 186     ExportList  *pPairedList;
 187
 188     ByteString sPForm;
 189
 190     void Dump();
 191     void addFallbackData( ByteString& sId , const ByteString& sText );
 192     bool getFallbackData( ByteString& sId , ByteString& sText);
 193
 194     void addMergedLanguage( ByteString& sLang );
 195     bool isMerged( ByteString& sLang );
 196     ResData( const ByteString &rPF, const ByteString &rGId )
 197             :
 198             nWidth( 0 ),
 199             nChildIndex( 0 ),
 200             nIdLevel( ID_LEVEL_NULL ),
 201             bChild( FALSE ),
 202             bChildWithText( FALSE ),
 203             bText( FALSE ),
 204             bHelpText( FALSE ),
 205             bQuickHelpText( FALSE ),
 206             bTitle( FALSE ),
 207             bList( FALSE ),
 208             bRestMerged( FALSE ),
 209             sGId( rGId ),
 210             nTextRefId( REFID_NONE ),
 211             nHelpTextRefId( REFID_NONE ),
 212             nQuickHelpTextRefId( REFID_NONE ),
 213             nTitleRefId( REFID_NONE ),
 214             sTextTyp( "Text" ),
 215             pStringList( NULL ),
 216             pUIEntries( NULL ),
 217             pItemList( NULL ),
 218             pFilterList( NULL ),
 219             pPairedList( NULL ),
 220             sPForm( rPF )
 221     {
 222         sGId.EraseAllChars( '\r' );
 223         sPForm.EraseAllChars( '\r' );
 224     };
 225     ResData( const ByteString &rPF, const ByteString &rGId , const ByteString &rFilename )
 226             :
 227             nChildIndex( 0 ),
 228             nIdLevel( ID_LEVEL_NULL ),
 229             bChild( FALSE ),
 230             bChildWithText( FALSE ),
 231             bText( FALSE ),
 232             bHelpText( FALSE ),
 233             bQuickHelpText( FALSE ),
 234             bTitle( FALSE ),
 235             bList( FALSE ),
 236             bRestMerged( FALSE ),
 237             sGId( rGId ),
 238             sFilename( rFilename ),
 239             nTextRefId( REFID_NONE ),
 240             nHelpTextRefId( REFID_NONE ),
 241             nQuickHelpTextRefId( REFID_NONE ),
 242             nTitleRefId( REFID_NONE ),
 243             sTextTyp( "Text" ),
 244             pStringList( NULL ),
 245             pUIEntries( NULL ),
 246             pItemList( NULL ),
 247             pFilterList( NULL ),
 248             pPairedList( NULL ),
 249             sPForm( rPF )
 250
 251     {
 252         sGId.EraseAllChars( '\r' );
 253         sPForm.EraseAllChars( '\r' );
 254     };
 255
 256
 257 };
 258
 259
 260 //
 261 // class Export
 262 //
 263
 264 /******************************************************************************
 265 * Purpose: syntax check and export of *.src, called from lexer
 266 ******************************************************************************/
 267
 268 #define LIST_NON                                        0x0000
 269 #define LIST_STRING                             0x0001
 270 #define LIST_FILTER                                     0x0002
 271 #define LIST_ITEM                                       0x0004
 272 #define LIST_PAIRED                 0x0005
 273 #define LIST_UIENTRIES                          0x0008
 274 #define STRING_TYP_TEXT                         0x0010
 275 #define STRING_TYP_HELPTEXT                     0x0020
 276 #define STRING_TYP_QUICKHELPTEXT        0x0040
 277 #define STRING_TYP_TITLE                        0x0080
 278
 279 #define MERGE_MODE_NORMAL                       0x0000
 280 #define MERGE_MODE_LIST                         0x0001
 281
 282 DECLARE_LIST( ResStack, ResData * )
 283 // forwards
 284 class WordTransformer;
 285 class ParserQueue;
 286
 287 class Export
 288 {
 289 private:
 290     WordTransformer     *pWordTransformer;
 291
 292     CharSet     aCharSet;                                       // used charset in src
 293
 294     SvFileStream aOutput;
 295
 296     ResStack aResStack;                                 // stack for parsing recursive
 297
 298     ByteString sActPForm;                               // hold cur. system
 299
 300     BOOL bDefine;                                               // cur. res. in a define?
 301     BOOL bNextMustBeDefineEOL;                  // define but no \ at lineend
 302     ULONG nLevel;                                               // res. recursiv? how deep?
 303     USHORT nList;                                               // cur. res. is String- or FilterList
 304     ByteString nListLang;
 305     ULONG nListIndex;
 306     ULONG nListLevel;
 307     bool bSkipFile;
 308     ByteString sProject;
 309     ByteString sRoot;
 310     BOOL bEnableExport;
 311     BOOL bMergeMode;
 312     ByteString sMergeSrc;
 313     ByteString sLastListLine;
 314     BOOL bError;                                                // any errors while export?
 315     BOOL bReadOver;
 316     BOOL bDontWriteOutput;
 317     ByteString sLastTextTyp;
 318     static bool isInitialized;
 319     ByteString sFilename;
 320
 321
 322 public:
 323     ParserQueue* pParseQueue; // public ?
 324     static ByteString sLanguages; // public ?
 325     static ByteString sForcedLanguages; // public ?
 326
 327
 328     static bool skipProject( ByteString sPrj ) ;
 329     static ByteString sIsoCode99;
 330     static void InitLanguages( bool bMergeMode = false );
 331     static void InitForcedLanguages( bool bMergeMode = false );
 332     static std::vector<ByteString> GetLanguages();
 333     static std::vector<ByteString> GetForcedLanguages();
 334
 335     static void SetLanguages( std::vector<ByteString> val );
 336     static void RemoveUTF8ByteOrderMarker( ByteString &rString );
 337     static bool hasUTF8ByteOrderMarker( const ByteString &rString );
 338     static void RemoveUTF8ByteOrderMarkerFromFile( const ByteString &rFilename );
 339     static bool fileHasUTF8ByteOrderMarker( const ByteString &rString );
 340     static ByteString GetIsoLangByIndex( USHORT nIndex );
 341     static void QuotHTML( ByteString &rString );
 342     static bool CopyFile( const ByteString& source , const ByteString& dest );
 343
 344     static void QuotHTMLXRM( ByteString &rString );
 345     static void UnquotHTML( ByteString &rString );
 346
 347     static const char* GetEnv( const char *pVar );
 348     static int getCurrentDirectory( rtl::OUString& base_fqurl , rtl::OUString& base );
 349
 350     static bool isSourceLanguage( const ByteString &sLanguage );
 351     static bool isAllowed( const ByteString &sLanguage );
 352     //static bool isMergingGermanAllowed( const ByteString& rPrj );
 353
 354     static bool LanguageAllowed( const ByteString &nLanguage );
 355     static void Languages( std::vector<ByteString>::const_iterator& begin , std::vector<ByteString>::const_iterator& end );
 356     static void getRandomName( const ByteString& sPrefix , ByteString& sRandStr , const ByteString& sPostfix  );
 357     static void getRandomName( ByteString& sRandStr );
 358
 359     static void replaceEncoding( ByteString& rString );
 360
 361     static ByteString GetFallbackLanguage( const ByteString nLanguage );
 362     static void FillInFallbacks( ResData *pResData );
 363     static void FillInListFallbacks( ExportList *pList, const ByteString &nSource, const ByteString &nFallback );
 364     static ByteString GetTimeStamp();
 365     static BOOL ConvertLineEnds( ByteString sSource, ByteString sDestination );
 366     static ByteString GetNativeFile( ByteString sSource );
 367     static DirEntry GetTempFile();
 368
 369     static void DumpExportList( ByteString& sListName , ExportList& aList );
 370     static ByteString DumpMap( ByteString& sMapName , ByteStringHashMap& aMap );
 371
 372 private:
 373     static std::vector<ByteString> aLanguages;
 374     static std::vector<ByteString> aForcedLanguages;
 375
 376     BOOL ListExists( ResData *pResData, USHORT nLst );
 377
 378     BOOL WriteData( ResData *pResData, BOOL bCreateNew = FALSE );// called befor dest. cur ResData
 379     BOOL WriteExportList( ResData *pResData, ExportList *pExportList,
 380                         const ByteString &rTyp, BOOL bCreateNew = FALSE );
 381
 382     ByteString MergePairedList( ByteString& sLine , ByteString& sText );
 383
 384     ByteString FullId();                                        // creates cur. GID
 385
 386     bool PairedListFallback( ByteString& sText , ResData& aResData );
 387
 388     ByteString GetPairedListID          ( const ByteString& sText );
 389     ByteString GetPairedListString      ( const ByteString& sText );
 390     ByteString StripList        ( const ByteString& sText );
 391
 392     void UnmergeUTF8( ByteString& sOrig );
 393     void InsertListEntry( const ByteString &rText, const ByteString &rLine );
 394     void CleanValue( ByteString &rValue );
 395     ByteString GetText( const ByteString &rSource, int nToken );
 396
 397     BOOL PrepareTextToMerge( ByteString &rText, USHORT nTyp,
 398         ByteString &nLangIndex, ResData *pResData );
 399
 400     void MergeRest( ResData *pResData, USHORT nMode = MERGE_MODE_NORMAL );
 401     void ConvertMergeContent( ByteString &rText );
 402
 403       void WriteToMerged( const ByteString &rText , bool bSDFContent );
 404     void SetChildWithText();
 405
 406     void CutComment( ByteString &rText );
 407
 408 public:
 409     Export( const ByteString &rOutput, BOOL bWrite,
 410             const ByteString &rPrj, const ByteString &rPrjRoot , const ByteString& rFile );
 411     Export( const ByteString &rOutput, BOOL bWrite,
 412             const ByteString &rPrj, const ByteString &rPrjRoot,
 413             const ByteString &rMergeSource , const ByteString& rFile );
 414     ~Export();
 415
 416     void Init();
 417     int Execute( int nToken, const char * pToken );     // called from lexer
 418     void SetError() { bError = TRUE; }
 419     BOOL GetError() { return bError; }
 420 };
 421
 422
 423 //
 424 // class PFormEntrys
 425 //
 426
 427 /******************************************************************************
 428 * Purpose: holds information of data to merge (one pform)
 429 ******************************************************************************/
 430
 431 class PFormEntrys : public ByteString
 432 {
 433 friend class MergeDataFile;
 434 private:
 435     ByteString sHelpText; // empty string
 436     ByteStringHashMap sText;
 437     ByteStringBoolHashMap bTextFirst;
 438     ByteStringHashMap sQuickHelpText;
 439     ByteStringBoolHashMap bQuickHelpTextFirst;
 440     ByteStringHashMap sTitle;
 441     ByteStringBoolHashMap bTitleFirst;
 442
 443 public:
 444     PFormEntrys( const ByteString &rPForm ) : ByteString( rPForm ) {};
 445     ByteString Dump();
 446     void InsertEntry(
 447                     const ByteString &nId ,
 448                     const ByteString &rText,
 449                     const ByteString &rQuickHelpText,
 450                     const ByteString &rTitle
 451                     )
 452         {
 453
 454             sText[ nId ] = rText;
 455             bTextFirst[ nId ] = true;
 456             sQuickHelpText[ nId ] = rQuickHelpText;
 457             bQuickHelpTextFirst[ nId ] = true;
 458             sTitle[ nId ] = rTitle;
 459             bTitleFirst[ nId ] = true;
 460         }
 461      BOOL GetText( ByteString &rReturn, USHORT nTyp, const ByteString &nLangIndex, BOOL bDel = FALSE );
 462 };
 463
 464 //
 465 // class MergeData
 466 //
 467
 468 /******************************************************************************
 469 * Purpose: holds information of data to merge (one ressource)
 470 ******************************************************************************/
 471
 472 class MergeDataFile;
 473
 474 class MergeData
 475 {
 476 friend class MergeDataFile;
 477 private:
 478     ByteString sTyp;
 479     ByteString sGID;
 480     ByteString sLID;
 481     ByteString sFilename;
 482     PFormEntrysHashMap aMap;
 483 public:
 484     MergeData( const ByteString &rTyp, const ByteString &rGID, const ByteString &rLID , const ByteString &rFilename )
 485             : sTyp( rTyp ), sGID( rGID ), sLID( rLID ) , sFilename( rFilename ) {};
 486     ~MergeData();
 487     PFormEntrys* InsertEntry( const ByteString &rPForm );
 488     PFormEntrys* GetPFormEntrys( ResData *pResData );
 489
 490     void Insert( const ByteString& rPFO , PFormEntrys* pfEntrys );
 491     PFormEntrys* GetPFObject( const ByteString& rPFO );
 492
 493     ByteString Dump();
 494     BOOL operator==( ResData *pData );
 495 };
 496
 497 //
 498 // class MergeDataFile
 499 //
 500
 501 /******************************************************************************
 502 * Purpose: holds information of data to merge
 503 ******************************************************************************/
 504
 505 class MergeDataFile
 506 {
 507 private:
 508     BOOL bErrorLog;
 509     ByteString sErrorLog;
 510     SvFileStream aErrLog;
 511     ByteStringSet aLanguageSet;
 512     MergeDataHashMap aMap;
 513     ByteStringHashMap aLanguageMap;
 514     std::vector<ByteString> aLanguageList;
 515     ByteStringHashMap aFilenames;
 516
 517
 518 public:
 519     MergeDataFile( const ByteString &rFileName, const ByteString& rFile , BOOL bErrLog, CharSet aCharSet, bool bCaseSensitive = false );
 520 //    MergeDataFile( const ByteString &rFileName, const ByteString& rFile , BOOL bErrLog, CharSet aCharSet
 521 //            );
 522     ~MergeDataFile();
 523
 524
 525     std::vector<ByteString> GetLanguages();
 526     MergeData *GetMergeData( ResData *pResData , bool bCaseSensitve = false );
 527
 528     PFormEntrys *GetPFormEntrys( ResData *pResData );
 529     PFormEntrys *GetPFormEntrysCaseSensitive( ResData *pResData );
 530
 531     void InsertEntry( const ByteString &rTYP, const ByteString &rGID, const ByteString &rLID,
 532                 const ByteString &rPFO,
 533                 const ByteString &nLang , const ByteString &rTEXT,
 534                 const ByteString &rQHTEXT, const ByteString &rTITLE ,
 535                 const ByteString &sFilename , bool bCaseSensitive
 536                 );
 537     static USHORT GetLangIndex( USHORT nId );
 538     static ByteString CreateKey( const ByteString& rTYP , const ByteString& rGID , const ByteString& rLID , const ByteString& rFilename , bool bCaseSensitive = false );
 539
 540     ByteString Dump();
 541 //      void WriteErrorLog( const ByteString &rFileName );
 542     void WriteError( const ByteString &rLine );
 543 };
 544
 545
 546 class QueueEntry
 547 {
 548 public:
 549     QueueEntry( int nTypVal , ByteString sLineVal ): nTyp( nTypVal ) , sLine( sLineVal ){};
 550     int nTyp;
 551     ByteString sLine;
 552 };
 553
 554 class ParserQueue
 555 {
 556 public:
 557
 558     ParserQueue( Export& aExportObj );
 559     ~ParserQueue();
 560
 561     inline void Push( const QueueEntry& aEntry );
 562     bool bCurrentIsM;  // public ?
 563     bool bNextIsM;   // public ?
 564     bool bLastWasM;   // public ?
 565     bool bMflag;   // public ?
 566
 567     void Close();
 568 private:
 569     // Future / Next
 570     std::queue<QueueEntry>* aQueueNext;
 571     // Current
 572     std::queue<QueueEntry>* aQueueCur;
 573     // Ref
 574     std::queue<QueueEntry>* aQref;
 575
 576     Export& aExport;
 577     bool bStart;
 578     bool bStartNext;
 579
 580     inline void Pop( std::queue<QueueEntry>& aQueue );
 581
 582 };
 583 #endif
 584