ryzom/common/src/game_share/persistent_data.h

   1 // Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
   2 // Copyright (C) 2010  Winch Gate Property Limited
   3 //
   4 // This program is free software: you can redistribute it and/or modify
   5 // it under the terms of the GNU Affero General Public License as
   6 // published by the Free Software Foundation, either version 3 of the
   7 // License, or (at your option) any later version.
   8 //
   9 // This program is distributed in the hope that it will be useful,
  10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 // GNU Affero General Public License for more details.
  13 //
  14 // You should have received a copy of the GNU Affero General Public License
  15 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 /**
  18   This file contains a system for loading and saving persistent data in a robust manor
  19   The system is hierarchical and all data elements are tagged.
  20
  21   The system supports saving to text files and to binary files
  22
  23   NOTE: The text file format is similar to XML but is not currently XML feature-complete
  24
  25 */
  26
  27 #ifndef PERSISTENT_DATA_H
  28 #define PERSISTENT_DATA_H
  29
  30 //-----------------------------------------------------------------------------
  31 // Includes
  32 //-----------------------------------------------------------------------------
  33
  34 #include "nel/misc/types_nl.h"
  35 #include "nel/misc/stream.h"
  36 #include "nel/misc/mem_stream.h"
  37 #include "nel/misc/entity_id.h"
  38 #include "nel/misc/sheet_id.h"
  39 #include "nel/misc/sstring.h"
  40 #include "nel/misc/common.h"
  41 #include "utils.h"
  42
  43 #include <vector>
  44 #include <map>
  45
  46
  47 //-----------------------------------------------------------------------------
  48 // Global Macros
  49 //-----------------------------------------------------------------------------
  50
  51 // The following macros define methods that are instantiated using the
  52 // 'persistent_data_template.h' header file
  53
  54 #define DECLARE_PERSISTENCE_APPLY_METHOD\
  55         void apply(CPersistentDataRecord &pdr);
  56
  57 #define DECLARE_VIRTUAL_PERSISTENCE_APPLY_METHOD\
  58         virtual void apply(CPersistentDataRecord &pdr);
  59
  60 #define DECLARE_PURE_VIRTUAL_PERSISTENCE_APPLY_METHOD\
  61         virtual void apply(CPersistentDataRecord &pdr) =0;
  62
  63
  64 #define DECLARE_PERSISTENCE_STORE_METHOD\
  65         void store(CPersistentDataRecord &pdr) const;
  66
  67 #define DECLARE_VIRTUAL_PERSISTENCE_STORE_METHOD\
  68         virtual void store(CPersistentDataRecord &pdr) const;
  69
  70 #define DECLARE_PURE_VIRTUAL_PERSISTENCE_STORE_METHOD\
  71         virtual void store(CPersistentDataRecord &pdr) const =0;
  72
  73
  74 #define DECLARE_PERSISTENCE_METHODS\
  75         DECLARE_PERSISTENCE_APPLY_METHOD\
  76         DECLARE_PERSISTENCE_STORE_METHOD
  77
  78 #define DECLARE_VIRTUAL_PERSISTENCE_METHODS\
  79         DECLARE_VIRTUAL_PERSISTENCE_APPLY_METHOD\
  80         DECLARE_VIRTUAL_PERSISTENCE_STORE_METHOD
  81
  82 #define DECLARE_PURE_VIRTUAL_PERSISTENCE_METHODS\
  83         DECLARE_PURE_VIRTUAL_PERSISTENCE_APPLY_METHOD\
  84         DECLARE_PURE_VIRTUAL_PERSISTENCE_STORE_METHOD
  85
  86
  87 #define DECLARE_PERSISTENCE_METHODS_WITH_STORE_ARG(arg)\
  88         void store(CPersistentDataRecord &pdr,arg) const;\
  89         void apply(CPersistentDataRecord &pdr);
  90
  91 #define DECLARE_PERSISTENCE_METHODS_WITH_APPLY_ARG(arg)\
  92         void store(CPersistentDataRecord &pdr) const;\
  93         void apply(CPersistentDataRecord &pdr,arg);
  94
  95 #define DECLARE_PERSISTENCE_METHODS_WITH_ARG(arg)\
  96         void store(CPersistentDataRecord &pdr,arg) const;\
  97         void apply(CPersistentDataRecord &pdr,arg);
  98
  99
 100 #define DECLARE_PERSISTENCE_METHODS_WITH_TARGET(arg)\
 101         void store(CPersistentDataRecord &pdr,const arg) const;\
 102         void apply(CPersistentDataRecord &pdr,arg);
 103
 104
 105 // A friendly macro to place before each inclusion of persistent_data_template.h to ease debug
 106 #define PERSISTENT_GENERATION_MESSAGE \
 107         NL_LOC_MSG "generating persistence code for " NL_MACRO_TO_STR(PERSISTENT_CLASS)
 108
 109
 110 //-----------------------------------------------------------------------------
 111 // forward declarations
 112 //-----------------------------------------------------------------------------
 113
 114 class CPersistentDataRecord;
 115 class CPDRLookupTbl;
 116
 117 #pragma pack(push , 4)
 118
 119
 120 //-----------------------------------------------------------------------------
 121 // class CPersistentDataRecord
 122 //-----------------------------------------------------------------------------
 123
 124 class CPersistentDataRecord
 125 {
 126 public:
 127         //-------------------------------------------------------------------------
 128         // public data structures
 129         //-------------------------------------------------------------------------
 130
 131         typedef uint16 TToken;
 132         typedef std::vector<NLMISC::CSString> TStringTable;
 133
 134
 135 private:
 136         //-------------------------------------------------------------------------
 137         // private data structures
 138         //-------------------------------------------------------------------------
 139
 140         struct CArg
 141         {
 142                 /*** NOTE: Tokens are coded on 3 bits so there should never be more than 8 ***/
 143                 enum
 144                 {
 145                         BEGIN_TOKEN,
 146                         END_TOKEN,
 147                         SINT_TOKEN,
 148                         UINT_TOKEN,
 149                         FLOAT_TOKEN,
 150                         STRING_TOKEN,
 151                         FLAG_TOKEN,
 152                         EXTEND_TOKEN
 153                 };
 154
 155                 enum TType
 156                 {
 157                         STRUCT_BEGIN,
 158                         STRUCT_END,
 159                         FLAG,
 160                         SINT32,
 161                         UINT32,
 162                         FLOAT32,
 163                         STRING,
 164                         SINT64,
 165                         UINT64,
 166                         FLOAT64,
 167                         EXTEND_TYPE,
 168                         NB_TYPE
 169                 } _Type;
 170
 171                 enum TExtendType
 172                 {
 173                         ET_SHEET_ID,
 174
 175                         ET_64_BIT_EXTENDED_TYPES = 0x80000000,
 176                         ET_ENTITY_ID = ET_64_BIT_EXTENDED_TYPES
 177                 };
 178
 179                 union
 180                 {
 181                         struct
 182                         {
 183                                 uint32  i32_1;
 184                                 uint32  i32_2;
 185                         };
 186
 187                         sint32  i32;
 188                         sint64  i64;
 189                         float   f32;
 190                         double  f64;
 191
 192                         struct
 193                         {
 194                                 uint32  ExType;
 195                                 union
 196                                 {
 197                                         struct
 198                                         {
 199                                                 uint32  ex32_1;
 200                                                 uint32  ex32_2;
 201                                         };
 202
 203                                         uint32  ExData32;
 204                                         uint64  ExData64;
 205                                 };
 206                         };
 207                 } _Value;
 208                 NLMISC::CSString _String;
 209
 210
 211                 CArg();
 212                 CArg(const std::string& type,const std::string& value,CPersistentDataRecord& pdr);
 213
 214                 uint64 asUint() const;
 215                 sint64 asSint() const;
 216                 float asFloat() const;
 217                 double asDouble() const;
 218                 NLMISC::CSheetId asSheetId() const;
 219                 NLMISC::CEntityId asEntityId() const;
 220                 NLMISC::CSString asString() const;
 221                 ucstring asUCString() const;
 222                 NLMISC::CSString typeName() const;
 223
 224                 bool setType(const std::string &typeName);
 225                 void setType(CArg::TType value);
 226                 bool isExtended() const;
 227                 bool isFlag() const;
 228
 229                 void push(TToken token, std::vector<TToken>& tokenTable, std::vector<uint32>& argTable) const;
 230
 231                 static CArg EntityId(NLMISC::CEntityId val);
 232                 static CArg SheetId(NLMISC::CSheetId val);
 233                 static CArg Int32(sint32 val);
 234                 static CArg Int32(uint32 val);
 235                 static CArg Int64(sint64 val);
 236                 static CArg Int64(uint64 val);
 237                 static CArg Float32(float val);
 238                 static CArg Float64(double val);
 239                 static CArg String(const std::string& value,CPersistentDataRecord& pdr);
 240                 static CArg UCString(const ucstring& value,CPersistentDataRecord& pdr);
 241                 static CArg Flag();
 242
 243                 static TType  token2Type(uint32 token,bool extend=false);
 244                 static TToken type2Token(uint32 type);
 245                 static bool   isTypeExtended(uint32 type);
 246         };
 247
 248 public:
 249         //-------------------------------------------------------------------------
 250         // class basics
 251         //-------------------------------------------------------------------------
 252
 253         // ctor
 254         CPersistentDataRecord(const std::string& tokenFamily=std::string());
 255
 256
 257         //-------------------------------------------------------------------------
 258         // set of methods for storing data in a CPersistentDataRecord
 259         //-------------------------------------------------------------------------
 260
 261         // clear out all data from the record
 262         void clear();
 263
 264         // accessors for the string table
 265         // note that the second variant of addString() is often faster as it checks
 266         // whether the value of 'result' is already correct before looking further...
 267         // note that the tokenFamily is used to setup the initial state of the string table (at init time)
 268         uint16 addString(const std::string& name);
 269         void addString(const std::string& name,uint16 &result);
 270         void addString(const char* name,uint16 &result);
 271         const NLMISC::CSString& lookupString(uint32 idx) const;
 272         const NLMISC::CSString& getTokenFamily() const;
 273
 274         // deal with the next data element (checking that the token matches)
 275
 276         void push(TToken token,const CArg& arg);
 277         void push(TToken token,bool val);
 278
 279         void push(TToken token,sint8 val);
 280         void push(TToken token,sint16 val);
 281         void push(TToken token,sint32 val);
 282         void push(TToken token,sint64 val);
 283
 284         void push(TToken token,uint8 val);
 285         void push(TToken token,uint16 val);
 286         void push(TToken token,uint32 val);
 287         void push(TToken token,uint64 val);
 288
 289         void push(TToken token,float val);
 290         void push(TToken token,double val);
 291         void push(TToken token,const std::string& val);
 292         void push(TToken token,const ucstring& val);
 293
 294         void push(TToken token,NLMISC::CSheetId val);
 295         void push(TToken token,const NLMISC::CEntityId& val);
 296
 297         void push(TToken token);        // for flag token that takes no args
 298
 299
 300         // deal with start and end of a 'structure' data block
 301
 302         void pushStructBegin(TToken token);
 303         void pushStructEnd(TToken token);
 304
 305
 306         //-------------------------------------------------------------------------
 307         // set of methods for retrieving data from a CPersistentDataRecord
 308         //-------------------------------------------------------------------------
 309
 310         // reset the read pointer to the start of the input data
 311         void rewind();
 312
 313         bool isEndOfData() const;
 314         bool isEndOfStruct() const;
 315         bool isStartOfStruct() const;
 316         bool isTokenWithNoData() const;
 317
 318         TToken peekNextToken() const;
 319         const NLMISC::CSString& peekNextTokenName() const;
 320         CArg::TType peekNextTokenType() const;
 321
 322         void peekNextArg(CArg& result) const;
 323         const CArg& peekNextArg() const;
 324
 325         void popNextArg(TToken token,CArg& result);
 326         const CArg& popNextArg(TToken token);
 327
 328         void pop(TToken token,bool& result);
 329
 330         void pop(TToken token,sint8& result);
 331         void pop(TToken token,sint16& result);
 332         void pop(TToken token,sint32& result);
 333         void pop(TToken token,sint64& result);
 334
 335         void pop(TToken token,uint8& result);
 336         void pop(TToken token,uint16& result);
 337         void pop(TToken token,uint32& result);
 338         void pop(TToken token,uint64& result);
 339
 340         void pop(TToken token,float& result);
 341         void pop(TToken token,double& result);
 342         void pop(TToken token,std::string& result);
 343         void pop(TToken token,ucstring& result);
 344
 345         void pop(TToken token,NLMISC::CSheetId& result);
 346         void pop(TToken token,NLMISC::CEntityId& result);
 347
 348         void pop(TToken token); // for flag token that takes no args
 349
 350         void popStructBegin(TToken token);
 351         void popStructEnd(TToken token);
 352         void skipStruct();                                      // Added by Ben, will skip a whole struct in record
 353         void skipData();                                        // Added by Ben, will skip next record, a struct as well as a simple token
 354
 355         // accessors for the _LookupTbls container
 356         CPDRLookupTbl* getLookupTbl(uint32 id) const;
 357         void setLookupTbl(uint32 id, CPDRLookupTbl* tbl);
 358
 359         // compare two persistent data records (verify that their contents are eqivalent)
 360         bool operator==(const CPersistentDataRecord& other) const;
 361
 362
 363         //-------------------------------------------------------------------------
 364         // debug methods for retrieving info from pdr records
 365         //-------------------------------------------------------------------------
 366
 367         // get info as a single text string including field names and values in a man-readable format
 368         NLMISC::CSString getInfo() const;
 369         // get info as a string of comma separated values
 370         NLMISC::CSString getInfoAsCSV() const;
 371         // get the header row that correspond to the comma separated values in getInfoAsCSV() - note that this method is static
 372         static const NLMISC::CSString& getCSVHeaderLine();
 373         // get the number of values in a pdr (a value is a string, a uint32, uint64, CEntitiyId, flag or other such primitive)
 374         uint32 getNumValues() const;
 375
 376
 377         //-------------------------------------------------------------------------
 378         // API enums
 379         //-------------------------------------------------------------------------
 380
 381         enum TFileFormat { BINARY_FILE, XML_FILE, LINES_FILE, ANY_FILE };
 382         enum TStringFormat { XML_STRING=XML_FILE, LINES_STRING=LINES_FILE };
 383
 384
 385         //-------------------------------------------------------------------------
 386         // set of methods for storing a data record to various destinations
 387         //-------------------------------------------------------------------------
 388
 389         uint32 totalDataSize() const;
 390         uint32 stringDataSize() const;
 391
 392         // the following routines save the data in different ways
 393         // NOTE: they are all inclined to rewind the 'read pointer'
 394
 395         // write the contents of the pdr to a stream
 396         bool toStream(NLMISC::IStream& dest);
 397
 398         // write data to a binary buffer - the size required for the buffer is totalDataSize()
 399         bool toBuffer(char *dest,uint32 sizeLimit);
 400
 401         // write data to a text buffer in one of several txt formats
 402         bool toString(std::string& result,TStringFormat stringFormat=XML_STRING);
 403
 404         // write data to a text buffer in an xml format
 405         bool toXML(std::string& result);
 406
 407         // write data to a text buffer in line-based txt format
 408         bool toLines(std::string& result);
 409
 410         // perform a toBuffer() and write the result to a binary file
 411         bool writeToBinFile(const std::string &fileName);
 412
 413         // perform a toString() and write the result to a text file
 414         bool writeToTxtFile(const std::string &fileName, TStringFormat stringFormat=XML_STRING);
 415
 416         // if the format is set to 'ANY_FILE' then use the extension provided in the 'fileName' argument to
 417         // determine the file type. In this case 'txt' and 'xml' have specific meanings
 418         // returns writeToTxtFile(...) or writeToBinFile(...) depending on the file format
 419         bool writeToFile(const std::string &fileName, TFileFormat fileFormat=ANY_FILE);
 420
 421
 422         //-------------------------------------------------------------------------
 423         // set of methods for retrieving a data record from various sources
 424         //-------------------------------------------------------------------------
 425
 426         // read from a CMemStream (maybe either binary or text data)
 427         bool fromBuffer(NLMISC::IStream& stream);
 428
 429         // read from a binary data buffer
 430         // if the input data looks like text then calls fromString()
 431         bool fromBuffer(const char *src, uint32 bufferSize);
 432
 433         // read from a text string (either xml or lines)
 434         // note 1: This routine is not at all optimised
 435         // note 2: The content of s is destroyed by the routine
 436         bool fromString(const std::string& s);
 437
 438         // read from an xml string
 439         // note 1: This routine is not at all optimised
 440         // note 2: The content of s is destroyed by the routine
 441         bool fromXML(const std::string& s);
 442
 443         // read from a lines string
 444         // note 1: This routine is not at all optimised
 445         // note 2: The content of s is destroyed by the routine
 446         bool fromLines(const std::string& s);
 447
 448         // read from a binary file
 449         bool readFromBinFile(const std::string &fileName);
 450
 451         // read from a text file
 452         bool readFromTxtFile(const std::string &fileName);
 453
 454         // read a file and determine whether it's a binary or text file from it's
 455         // content - then behave like readFromBinFile() or readFromTxtFile()
 456         bool readFromFile(const std::string &fileName);
 457
 458
 459 private:
 460
 461         bool    fromStream(NLMISC::IStream& stream, uint32 size);
 462
 463         //-------------------------------------------------------------------------
 464         // private persistent data
 465         //-------------------------------------------------------------------------
 466
 467         TStringTable _StringTable;
 468         std::vector<uint32> _ArgTable;
 469         std::vector<TToken> _TokenTable;
 470
 471
 472         //-------------------------------------------------------------------------
 473         // private work data - for writing
 474         //-------------------------------------------------------------------------
 475
 476         NLMISC::CSString        _TokenFamily;
 477         std::vector<TToken> _WritingStructStack;
 478
 479
 480         //-------------------------------------------------------------------------
 481         // private work data - for reading
 482         //-------------------------------------------------------------------------
 483
 484         mutable uint32 _ArgOffset;
 485         mutable uint32 _TokenOffset;
 486
 487         typedef std::vector<TToken> TReadingStructStack;
 488         mutable TReadingStructStack _ReadingStructStack;
 489
 490         typedef std::vector< NLMISC::CSmartPtr<CPDRLookupTbl> > TLookupTbls;
 491         TLookupTbls _LookupTbls;
 492
 493
 494         //-------------------------------------------------------------------------
 495         // globals
 496         //-------------------------------------------------------------------------
 497         static CArg TempArg;
 498 };
 499
 500
 501 //-----------------------------------------------------------------------------
 502 // class CPersistentDataRecordRyzomStore
 503 //-----------------------------------------------------------------------------
 504 // This is just a specialisation of the class that register it into the "RyzomTokenFamily"
 505 // Use it only to store (else useless copy of default string table at clear())
 506
 507 class CPersistentDataRecordRyzomStore : public CPersistentDataRecord
 508 {
 509 public:
 510         CPersistentDataRecordRyzomStore() : CPersistentDataRecord("RyzomTokenFamily") {}
 511 };
 512
 513
 514 //-----------------------------------------------------------------------------
 515 // class CPDRLookupTbl
 516 //-----------------------------------------------------------------------------
 517
 518 class CPDRLookupTbl: public NLMISC::CRefCount
 519 {
 520 public:
 521         // the data type used to represent enum values
 522         typedef sint16 TEnumValue;
 523
 524         // the [] operator is used to lookup entries in the lookup table
 525         // in debug mode if the index 'idx' falls outside the table we assert
 526         // in release mode if the index 'idx' falls outside the table we return -1; (This is the 'Bad String' value)
 527         TEnumValue operator[](uint32 idx) const;
 528
 529         // get the number of lookup table classes registered so far
 530         static uint32 getNumLookupTableClasses();
 531
 532 protected:
 533         // a static used by derived classes to setup unique ids
 534         static uint32 _NextLookupTblClassId;
 535
 536         // table mapping from pdr string table indices to enum values
 537         typedef std::vector<TEnumValue> TTheTbl;
 538         TTheTbl _TheTbl;
 539 };
 540
 541
 542 //-----------------------------------------------------------------------------
 543 // class CPdrTokenRegistry
 544 //-----------------------------------------------------------------------------
 545 // This is a singleton that can be used to setup families of tokens that
 546 // are copied directly into a new pdr's string table at init time
 547 // This class
 548
 549 class CPdrTokenRegistry
 550 {
 551 public:
 552         // singleton instantiation
 553         static CPdrTokenRegistry* getInstance();
 554
 555         // release memory
 556         static void releaseInstance();
 557
 558         // adding a token to the string table
 559         uint16 addToken(const std::string& family,const std::string& token);
 560
 561         // get hold of the string table for a given family
 562         const CPersistentDataRecord::TStringTable& getStringTable(const std::string& family);
 563
 564 private:
 565         // private data
 566         typedef std::map<NLMISC::CSString,CPersistentDataRecord::TStringTable> TRegistry;
 567         TRegistry _Registry;
 568
 569 private:
 570         static CPdrTokenRegistry *_Instance;
 571         // this is a singleton so prohibit creation
 572         CPdrTokenRegistry();
 573 };
 574
 575
 576 #pragma pack(pop)
 577
 578 #include "persistent_data_inline.h"
 579
 580 //-----------------------------------------------------------------------------
 581 #endif