ryzom/server/src/pd_lib/pds_table_buffer.h

   1 // Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
   2 // Copyright (C) 2010  Winch Gate Property Limited
   3 //
   4 // This program is free software: you can redistribute it and/or modify
   5 // it under the terms of the GNU Affero General Public License as
   6 // published by the Free Software Foundation, either version 3 of the
   7 // License, or (at your option) any later version.
   8 //
   9 // This program is distributed in the hope that it will be useful,
  10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 // GNU Affero General Public License for more details.
  13 //
  14 // You should have received a copy of the GNU Affero General Public License
  15 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 #ifndef NL_PDS_TABLE_BUFFER_H
  18 #define NL_PDS_TABLE_BUFFER_H
  19
  20 #include <nel/misc/types_nl.h>
  21 #include <nel/misc/path.h>
  22
  23 #include "../pd_lib/pd_utils.h"
  24 #include "../pd_lib/pd_server_utils.h"
  25 #include "../pd_lib/db_reference_file.h"
  26 #include "../pd_lib/db_delta_file.h"
  27
  28
  29 class IRowProcessor;
  30
  31 /**
  32  * A memory cache for table
  33  * \author Benjamin Legros
  34  * \author Nevrax France
  35  * \date 2003
  36  */
  37 class CTableBuffer : public CPDSLogger
  38 {
  39 public:
  40
  41         /// Constructor
  42         CTableBuffer();
  43
  44         /// Destructor
  45         ~CTableBuffer();
  46
  47         /// Clear all
  48         void                            clear();
  49
  50         /// Init
  51         void                            init(uint32 tableId, uint32 rowSize, bool mapped);
  52
  53         /// Setup Ref
  54         void                            setupRef(CRefIndex& ref);
  55
  56
  57
  58
  59         /// Init, only for reference builder
  60         void                            init(uint32 tableId, const std::string& refRootPath, const std::string& refPath); //CRefIndex& ref);
  61
  62
  63
  64         /// Row Data Type
  65         typedef uint8*          TRowData;
  66
  67         /// Map of Rows
  68         typedef CHashMap<RY_PDS::TRowIndex, TRowData>   TRowMap;
  69
  70         /**
  71          * Row Accessor, embeds any row access
  72          */
  73         class CAccessor
  74         {
  75         public:
  76
  77                 /// Get row index
  78                 RY_PDS::TRowIndex       row() const                             { return (*_MapIt).first; }
  79
  80                 /// Get row data
  81                 TRowData                        data()                                  { return (*_MapIt).second + (_Mapped ? sizeof(CMappedHeader) : sizeof(CHeader)); }
  82
  83                 /// Get row data
  84                 TRowData                        data() const                    { return (*_MapIt).second + (_Mapped ? sizeof(CMappedHeader) : sizeof(CHeader)); }
  85
  86                 /// Default Constructor, should never be used
  87                 explicit CAccessor()                                            { }
  88
  89
  90
  91                 /// Is Row Allocated
  92                 bool                            allocated() const               { return ((CHeader*)fullRow())->allocated(); }
  93
  94                 /// Is Row Mapped
  95                 bool                            mapped() const                  { return _Mapped && ((CMappedHeader*)fullRow())->mapped(); }
  96
  97                 /// Is Row Dirty
  98                 bool                            dirty() const                   { return ((CHeader*)fullRow())->dirty(); }
  99
 100                 /// Get Dirt stamp
 101                 uint32                          dirtyStamp() const              { return ((CHeader*)fullRow())->getDirtStamp(); }
 102
 103                 /// Get Map Key
 104                 uint64                          key() const                             { return mapped() ? ((CMappedHeader*)fullRow())->getKey() : 0; }
 105
 106                 /// Get Full Row Data
 107                 TRowData                        fullRow() const                 { return (*_MapIt).second; }
 108
 109
 110                 /// Equals
 111                 bool                            operator == (const CAccessor& access) const     { return _Mapped == access._Mapped && _MapIt == access._MapIt; }
 112
 113
 114                 /**
 115                  * Copy header info from another accessor
 116                  * Warning, this is only a local copy of header, data won't been updated in parent CTableBuffer
 117                  */
 118                 void                            copyHeader(const CAccessor& access);
 119
 120         private:
 121
 122                 friend class CTableBuffer;
 123
 124                 /// Accessor in table map
 125                 typedef TRowMap::iterator       TMapIt;
 126                 TMapIt                          _MapIt;
 127
 128                 bool                            _Mapped;
 129
 130                 /// Constructor for CTableBuffer
 131                 CAccessor(TMapIt it, bool mapped) : _MapIt(it), _Mapped(mapped)         { }
 132
 133                 /// Get Full Row Data
 134                 TRowData                        fullRow()                               { return (*_MapIt).second; }
 135         };
 136
 137
 138
 139         /**
 140          * Return a pointer to straight row data
 141          * If row is not loaded, data are loaded from table file
 142          */
 143         CAccessor                       getRow(RY_PDS::TRowIndex row);
 144
 145         /**
 146          * Return a pointer to straight new row data
 147          * Row is filled with blank, not loaded from file
 148          */
 149         CAccessor                       getEmptyRow(RY_PDS::TRowIndex row);
 150
 151         /**
 152          * Acquire a Row
 153          * Row is marked as non releasable until it is released enough time
 154          */
 155         bool                            acquireRow(CAccessor accessor);
 156
 157         /**
 158          * Release a Row
 159          * Row is marked as purgeable, and will be purged as soon as possible,
 160          * unless it is reactivated before purge
 161          */
 162         bool                            releaseRow(CAccessor accessor);
 163
 164         /**
 165          * Release a Row
 166          * Row is marked as purgeable, and will be purged as soon as possible,
 167          * unless it is reactivated before purge
 168          */
 169         bool                            releaseRow(RY_PDS::TRowIndex row);
 170
 171         /**
 172          * Release all rows
 173          * To be called when client disconnects, so all loaded rows may be released without
 174          * client releasing them manually
 175          */
 176         bool                            releaseAll();
 177
 178
 179
 180         /**
 181          * Mark a Row as being dirty for later delta save
 182          */
 183         bool                            dirtyRow(CAccessor accessor);
 184
 185
 186
 187         /**
 188          * Build Delta
 189          */
 190         bool                            buildDelta(const CTimestamp& starttimestamp, const CTimestamp& endtimestamp);
 191
 192         /**
 193          * Apply delta changes from a file
 194          */
 195         bool                            applyDeltaChanges(const std::string& filename);
 196
 197         /**
 198          * Flush Released Rows from memory
 199          */
 200         void                            flushReleased();
 201
 202         /**
 203          * Reset dirty tags
 204          */
 205         void                            resetDirty();
 206
 207         /**
 208          * Flush Reference files
 209          */
 210         void                            flushRefFiles();
 211
 212         /**
 213          * Purge all references
 214          * Flush loaded reference files, and close them all.
 215          */
 216         bool                            purgeReferences();
 217
 218         /**
 219          * Open all reference files in reference directory to update
 220          */
 221         bool                            openAllRefFilesWrite();
 222
 223         /**
 224          * Open all reference files in reference directory to read
 225          */
 226         bool                            openAllRefFilesRead();
 227
 228         /**
 229          * Update delta ids for all references
 230          */
 231         bool                            updateDeltaIds(uint32 start, uint32 end);
 232
 233
 234
 235
 236         /**
 237          * Allocate a row
 238          * \param row is the row index to allocate
 239          * Return true if succeded
 240          */
 241         bool                            allocate(RY_PDS::TRowIndex row, CAccessor& accessor);
 242
 243         /**
 244          * Deallocate a row
 245          * \param row is the row index to deallocate
 246          * Return true if succeded
 247          */
 248         bool                            deallocate(RY_PDS::TRowIndex row);
 249
 250         /**
 251          * Tells if a row is allocated
 252          * \param row is the row to check
 253          */
 254         bool                            isAllocated(RY_PDS::TRowIndex row) const                { return _RowMapper.allocated(row); }
 255
 256         /**
 257          * Get number of actually allocated rows
 258          */
 259         uint32                          numAllocated() const                                                    { return _RowMapper.numAllocated(); }
 260
 261         /**
 262          * Get Max row index used
 263          */
 264         RY_PDS::TRowIndex       maxRowIndex() const                                                             { return _RowMapper.maxRowIndex(); }
 265
 266
 267         /**
 268          * Get Next Unalloocated row
 269          */
 270         RY_PDS::TRowIndex       nextUnallocatedRow() const                                              { return _RowMapper.nextUnallocatedRow(); }
 271
 272
 273         /**
 274          * Map a row
 275          * \param index is the index to map
 276          * \param key is the 64 bits row key
 277          * Return true if succeded
 278          */
 279         bool                            mapRow(const RY_PDS::CObjectIndex &index, uint64 key);
 280
 281         /**
 282          * Unmap a row in a table
 283          * \param key is the 64 bits row key
 284          * Return true if succeded
 285          */
 286         bool                            unmapRow(const RY_PDS::CObjectIndex &index, uint64 key);
 287
 288         /**
 289          * Get a mapped row
 290          * \param key is the 64 bits row key
 291          * Return a valid CObjectIndex if success
 292          */
 293         RY_PDS::CObjectIndex    getMappedRow(uint64 key) const;
 294
 295         /**
 296          * Get number of actually allocated rows
 297          */
 298         uint32                          numMapped() const                                                               { return _RowMapper.numMapped(); }
 299
 300
 301         /**
 302          * Get the number of actually loaded rows
 303          */
 304         uint32                          getLoadedRows() const                                                   { return (uint32)_RowMap.size(); }
 305
 306         /**
 307          * Get memory load
 308          */
 309         uint32                          getMemoryLoad() const                                                   { return (uint32)_RowMap.size()*_InternalRowSize; }
 310
 311
 312
 313         /**
 314          * Build RowMapper
 315          */
 316         bool                            buildRowMapper();
 317
 318         /**
 319          * Process Rows, apply external row processing after rows are loaded
 320          */
 321         bool                            processRows(IRowProcessor* processor);
 322
 323         /**
 324          * Link RowMapper
 325          */
 326         void                            linkRowMapper(CTableBuffer* link)                               { _RowMapper.link(&(link->_RowMapper)); }
 327
 328
 329
 330         /**
 331          * Get Size of Row Header (depends on whether table is mapped or not)
 332          */
 333         uint                            getHeaderSize() const                                                   { return _Mapped ? sizeof(CMappedHeader) : sizeof(CHeader); }
 334
 335         /**
 336          * Get Internal Row Size
 337          */
 338         uint                            getInternalRowSize() const                                              { return _InternalRowSize; }
 339
 340         /**
 341          * Get User Row Size
 342          */
 343         uint                            getRowSize() const                                                              { return _InternalRowSize; }
 344
 345
 346
 347         /**
 348          * Update a Row
 349          * Update in reference file the whole row
 350          */
 351         bool                            updateRow(CAccessor accessor);
 352
 353         /**
 354          * Get current Timestamp
 355          */
 356         static uint32           getCommonStamp()                                                                { return _CommonStamp; }
 357
 358         /**
 359          * Update common Timestamp
 360          */
 361         static void                     updateCommonStamp();
 362
 363
 364
 365
 366         /// \name Debug
 367         // @{
 368
 369         /**
 370          * Setup debug delta file
 371          */
 372         bool                            setupDebugDeltaFile(const std::string& filename, CDBDeltaFile& delta) const;
 373
 374         /**
 375          * Get Delta file Row
 376          */
 377         uint8*                          getDeltaRow(uint32& row, CDBDeltaFile& delta) const;
 378
 379         // @}
 380
 381
 382
 383 protected:
 384
 385         virtual std::string     getLoggerIdentifier() const                                             { return "buffer"; }
 386
 387 private:
 388
 389         /// Initialised
 390         bool                                                            _Init;
 391
 392         /// Table Id
 393         uint32                                                          _TableId;
 394
 395         /// Row Size
 396         uint32                                                          _RowSize;
 397
 398         /// Internal Row Size
 399         uint32                                                          _InternalRowSize;
 400
 401         /// Ref Root Path
 402         std::string                                                     _RefRootPath;
 403
 404         /// Ref Path
 405         std::string                                                     _RefPath;
 406
 407         /// Rows per ref file
 408         uint32                                                          _RowsPerFile;
 409
 410         /// Maximum ref file size
 411         static uint32                                           _MaxRefFileSize;
 412
 413         /// Row Map
 414         TRowMap                                                         _RowMap;
 415
 416         /// Map of purgeable rows
 417         typedef std::set<RY_PDS::TRowIndex>     TReleaseSet;
 418
 419         /// Release Map
 420         TReleaseSet                                                     _ReleaseSet;
 421
 422         /// List of dirty rows
 423         typedef std::vector<CAccessor>          TDirtyList;
 424
 425         /// Dirty List
 426         TDirtyList                                                      _DirtyList;
 427
 428         /// List of ref files -- a vector or pointer, to avoid realloc issues
 429         typedef std::vector<CDBReferenceFile*>  TRefFileMap;
 430
 431         /// Reference files
 432         TRefFileMap                                                     _RefFileMap;
 433
 434         /// Index and Allocation Map
 435         CRowMapper                                                      _RowMapper;
 436
 437         /// Rows are mapped?
 438         bool                                                            _Mapped;
 439
 440         /// Current Delta Id
 441         uint32                                                          _CurrentDeltaId;
 442
 443
 444         /// Current reference stamp
 445         uint32                                                          _ReferenceStamp;
 446
 447         /// Common stamp
 448         static uint32                                           _CommonStamp;
 449
 450
 451         /// Load a row from the appropriate reference file
 452         bool                            loadRow(RY_PDS::TRowIndex row, TRowData rowData);
 453
 454         /// Init Number of Rows per File
 455         bool                            initRowsPerFile();
 456
 457         /// Check reference file is ready
 458         bool                            checkRef(uint32 refFile);
 459
 460         /// Acquire a row, internal version
 461         bool                            acquireRow(TRowMap::iterator it);
 462
 463         /// Release a row, internal version
 464         bool                            releaseRow(TRowMap::iterator it, bool forceNotAcquired = false);
 465
 466         /// Update a Row -- rowdata is _InternalByteSize long, this is a full buffer row with header
 467         bool                            updateRow(RY_PDS::TRowIndex row, const TRowData rowData, bool forceWriteToDisk);
 468
 469         /// Get Reference Files list
 470         void                            getReferenceFilesList(std::vector<std::string> &result);
 471
 472         /// Process row for RowMapper
 473         bool                            processRow(CAccessor& accessor);
 474
 475         /// Header of a row
 476         class CHeader
 477         {
 478         public:
 479
 480                 enum
 481                 {
 482                         Dirty                   = 1,
 483                         Allocated               = 2,
 484                 };
 485
 486                 /// Clear
 487                 void    clear()                                                         { _Flags = 0; _Acquire = 0; _DirtStamp = 0; }
 488
 489                 /// GetFlags
 490                 uint8   getFlags() const                                        { return _Flags; }
 491
 492                 /// Is Row Dirty
 493                 bool    dirty() const                                           { return checkFlags(Dirty); }
 494
 495
 496                 /// Is Row Allocated
 497                 bool    allocated() const                                       { return checkFlags(Allocated); }
 498
 499
 500                 /// Set flags
 501                 void    setFlags(uint8 flags)                           { _Flags |= flags; }
 502
 503                 /// Set flags
 504                 void    clearFlags(uint8 flags)                         { _Flags &= (~flags); }
 505
 506                 /// Check Flags
 507                 bool    checkFlags(uint8 testflags) const       { return (_Flags & testflags) != 0; }
 508
 509
 510
 511                 /// Set Row As Dirty
 512                 void    setDirty();
 513
 514                 /// Clear DirtStamp (set initial dirt stamp as 'no dirt stamp' value)
 515                 void    clearDirtStamp()                                        { _DirtStamp = 0; clearFlags(Dirty); }
 516
 517                 /// Get DirtStamp
 518                 uint32  getDirtStamp() const                            { return _DirtStamp; }
 519
 520
 521
 522                 /// Acquire, return true the first time it is acquired
 523                 bool    acquire()
 524                 {
 525                         ++_Acquire;
 526                         return _Acquire == 1;
 527                 }
 528
 529                 /// Release, return true if row can be release effectively
 530                 bool    unacquire()
 531                 {
 532                         --_Acquire;
 533                         if (_Acquire < 0)
 534                                 _Acquire = 0;
 535                         return !isAcquired();
 536                 }
 537
 538                 /// Clear Acquire count
 539                 void    clearAcquireCount()                                     { _Acquire = 0; }
 540
 541                 /// Is acquired
 542                 bool    isAcquired() const                                      { return _Acquire > 0; }
 543
 544         protected:
 545
 546                 /// Row flags
 547                 uint8   _Flags;
 548
 549                 /// Acquire counter
 550                 sint16  _Acquire;
 551
 552                 /// Row Dirty Timestamp
 553                 uint32  _DirtStamp;
 554
 555         };
 556
 557         /// Header of a mapped row
 558         class CMappedHeader : public CHeader
 559         {
 560         public:
 561
 562                 /// Clear up header
 563                 void                    clear()                                         { CHeader::clear(); _Key = 0; }
 564
 565                 /// Set Row Key
 566                 void                    setKey(uint64 key)                      { _Key = key; }
 567
 568                 /// Get Row Key
 569                 uint64                  getKey() const                          { return _Key; }
 570
 571                 /// Is Row Mapped?
 572                 bool                    mapped() const                          { return _Key != 0; }
 573
 574         protected:
 575
 576                 /**
 577                  * Row map key
 578                  * The key of the row in table map, 0 means this row is not mapped
 579                  */
 580                 uint64                          _Key;
 581
 582         };
 583
 584         friend class CAccessor;
 585         friend class CDatabaseAdapter;
 586 };
 587
 588 /**
 589  * A callback interface for row processing at loading
 590  * \author Benjamin Legros
 591  * \author Nevrax France
 592  * \date 2003
 593  */
 594 class IRowProcessor
 595 {
 596 public:
 597
 598         virtual bool    processRow(RY_PDS::TTableIndex table, CTableBuffer::CAccessor& accessor) = 0;
 599 };
 600
 601
 602 #include "pds_table_buffer_inline.h"
 603
 604
 605 #endif // NL_PDS_TABLE_BUFFER_H
 606
 607 /* End of pds_table_buffer.h */