ryzom/server/src/persistant_data_service/pds_database.cpp

   1 // Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
   2 // Copyright (C) 2010  Winch Gate Property Limited
   3 //
   4 // This program is free software: you can redistribute it and/or modify
   5 // it under the terms of the GNU Affero General Public License as
   6 // published by the Free Software Foundation, either version 3 of the
   7 // License, or (at your option) any later version.
   8 //
   9 // This program is distributed in the hope that it will be useful,
  10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 // GNU Affero General Public License for more details.
  13 //
  14 // You should have received a copy of the GNU Affero General Public License
  15 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 #include "../pd_lib/pds_common.h"
  18
  19 #include "pds_database.h"
  20 #include "pds_table.h"
  21 #include "pds_type.h"
  22 #include "database_adapter.h"
  23
  24 #include "db_manager.h"
  25
  26 #include "pd_lib/pd_server_utils.h"
  27 #include "pd_lib/reference_builder.h"
  28 #include "pd_lib/delta_builder.h"
  29
  30 #include <nel/misc/debug.h>
  31 #include <nel/misc/path.h>
  32 #include <nel/misc/config_file.h>
  33 #include <nel/misc/file.h>
  34 #include <nel/misc/variable.h>
  35 #include <nel/misc/o_xml.h>
  36 #include <nel/misc/i_xml.h>
  37
  38 #include <nel/net/service.h>
  39
  40 using namespace std;
  41 using namespace NLMISC;
  42 using namespace NLNET;
  43
  44 // Hosted by db_manager.cpp
  45 extern NLMISC::CVariable<uint>  DeltaUpdateRate;
  46
  47 /*
  48  * Constructor
  49  */
  50 CDatabase::CDatabase(uint32 id) : _Reference(id)
  51 {
  52         clear();
  53
  54         _State.Id = id;
  55 }
  56
  57 /*
  58  * Destructor
  59  */
  60 CDatabase::~CDatabase()
  61 {
  62         PDS_FULL_DEBUG("delete()");
  63
  64         clear();
  65 }
  66
  67
  68 /*
  69  * Massive Database clear
  70  */
  71 void    CDatabase::clear()
  72 {
  73         uint    i;
  74
  75         for (i=0; i<_Types.size(); ++i)
  76         {
  77                 if (_Types[i] != NULL)
  78                         delete _Types[i];
  79                 _Types[i] = NULL;
  80         }
  81
  82         _Types.clear();
  83
  84         for (i=0; i<_Tables.size(); ++i)
  85         {
  86                 if (_Tables[i] != NULL)
  87                         delete _Tables[i];
  88                 _Tables[i] = NULL;
  89         }
  90
  91         _Tables.clear();
  92
  93         _Init = false;
  94         //_ObjectList.clear();
  95         _SetMap.clear();
  96         _State.Name.clear();
  97         _ServiceId.set(0xffff);
  98 }
  99
 100
 101 /*
 102  * Init database
 103  * \param xmlStream is the xml description of the database
 104  */
 105 bool    CDatabase::init()
 106 {
 107         PDS_DEBUG("init()");
 108
 109         // clear all before doing anything
 110         clear();
 111
 112         const CDatabaseNode&    db = _Description.getDatabaseNode();
 113
 114         _State.Name = db.Name;
 115
 116         uint    i;
 117         for (i=0; i<db.Types.size(); ++i)
 118         {
 119                 CType*  type = new CType;
 120
 121                 if (!type->init(this, db.Types[i]))
 122                 {
 123                         PDS_WARNING("init(): failed to init type '%d'", i);
 124                         return false;
 125                 }
 126
 127                 if (type->getId() != i)
 128                 {
 129                         PDS_WARNING("init(): failed to init type '%d'", i);
 130                         return false;
 131                 }
 132
 133                 _Types.push_back(type);
 134         }
 135
 136         for (i=0; i<db.Tables.size(); ++i)
 137         {
 138                 CTable* table = new CTable();
 139
 140                 if (!table->init(this, db.Tables[i]) || table->getId() != i)
 141                 {
 142                         PDS_WARNING("init(): failed to init table '%d'", i);
 143                         return false;
 144                 }
 145
 146                 _Tables.push_back(table);
 147         }
 148
 149         for (i=0; i<_Tables.size(); ++i)
 150         {
 151                 CTable* table = _Tables[i];
 152
 153                 if (!table->buildColumns())
 154                 {
 155                         PDS_WARNING("init(): failed to build table '%d' columns", i);
 156                         return false;
 157                 }
 158         }
 159
 160         for (i=0; i<_Tables.size(); ++i)
 161         {
 162                 _Tables[i]->postInit();
 163         }
 164
 165         _Init = true;
 166
 167         if (!notifyNewReference(false))
 168         {
 169                 _Init = false;
 170                 return false;
 171         }
 172
 173         // save description in log dir
 174         std::string     logDir = RY_PDS::CPDSLib::getLogDirectory(_State.Id);
 175         if (!CFile::isExists(logDir) || !CFile::isDirectory(logDir))
 176         {
 177                 if (!CFile::createDirectoryTree(logDir))
 178                 {
 179                         PDS_WARNING("init(): failed to create log root directory '%s'", logDir.c_str());
 180                 }
 181
 182                 if (!CFile::setRWAccess(logDir))
 183                 {
 184                         PDS_WARNING("init(): failed, can't set RW access to directory '%s'", logDir.c_str());
 185                 }
 186         }
 187
 188 //      CTimestamp      initDate;
 189 //      initDate.setToCurrent();
 190 //      _Description.saveDescription(logDir + initDate.toString() + ".description");
 191
 192         PDS_DEBUG("init() successful");
 193
 194         return true;
 195 }
 196
 197
 198
 199
 200 /*
 201  * Checkup database
 202  */
 203 bool    CDatabase::checkup()
 204 {
 205         PDS_DEBUG("checkup()");
 206
 207         if (!initialised())
 208         {
 209                 PDS_WARNING("checkup(): database not initialised");
 210                 return false;
 211         }
 212
 213         uint    i;
 214         bool    dbOk = true;
 215
 216         for (i=0; i<_Types.size(); ++i)
 217         {
 218                 if (_Types[i] == NULL || !_Types[i]->initialised())
 219                 {
 220                         PDS_WARNING("checkup(): type '%d' not initialised", i);
 221                         continue;
 222                 }
 223         }
 224
 225         for (i=0; i<_Tables.size(); ++i)
 226         {
 227                 if (_Tables[i] == NULL || !_Tables[i]->initialised())
 228                 {
 229                         PDS_WARNING("checkup(): table '%d' not initialised", i);
 230                         continue;
 231                 }
 232
 233                 CTable* table = _Tables[i];
 234
 235                 const std::vector<CColumn>              &columns = table->getColumns();
 236                 const std::vector<CAttribute*>  &attributes = table->getAttributes();
 237
 238                 uint    j;
 239
 240                 // walk through attributes
 241                 for (j=0; j<attributes.size(); ++j)
 242                 {
 243                         if (attributes[j] == NULL || !attributes[j]->initialised())
 244                         {
 245                                 PDS_WARNING("checkup(): attribute '%s:%d' not initialised", table->getName().c_str(), i);
 246                                 dbOk = false;
 247                                 continue;
 248                         }
 249
 250                         const CAttribute*       attribute = attributes[j];
 251
 252                         if (attribute->getParent() != table)
 253                         {
 254                                 PDS_WARNING("checkup(): attribute '%s:%s' doesn't point table '%d'", table->getName().c_str(), attribute->getName().c_str(), i);
 255                                 dbOk = false;
 256                         }
 257
 258                         uint    k;
 259
 260                         for (k=attribute->getOffset(); k<attribute->getOffset()+attribute->getColumns(); ++k)
 261                         {
 262                                 if (k >= columns.size() || !columns[k].initialised())
 263                                 {
 264                                         PDS_WARNING("checkup(): attribute '%s:%s' points to not initialised column '%d'", table->getName().c_str(), attribute->getName().c_str(), k);
 265                                         dbOk = false;
 266                                 }
 267                                 else if (columns[k].getParent() != attribute)
 268                                 {
 269                                         PDS_WARNING("checkup(): attribute '%s:%s' points to column '%s' that doesn't point back", table->getName().c_str(), attribute->getName().c_str(), columns[k].getName().c_str());
 270                                         dbOk = false;
 271                                 }
 272                                 else
 273                                 {
 274                                         const CColumn   &column = columns[k];
 275
 276                                 }
 277                         }
 278                 }
 279
 280                 // walk through columns
 281                 for (j=0; j<columns.size(); ++j)
 282                 {
 283                         if (!columns[j].initialised())
 284                         {
 285                                 PDS_WARNING("checkup(): column '%s:%d' not initialised", table->getName().c_str(), j);
 286                                 dbOk = false;
 287                                 continue;
 288                         }
 289                 }
 290         }
 291
 292         if (dbOk)
 293         {
 294                 PDS_DEBUG("checkup(): Database is correct");
 295         }
 296
 297         PDS_DEBUG("checkup() finished");
 298
 299         return true;
 300 }
 301
 302 /*
 303  * Initialise internal timestamps
 304  */
 305 void    CDatabase::initTimestamps()
 306 {
 307         // initialise update and creation timestamps to now
 308         _CreationTimestamp.setToCurrent();
 309         _MinuteUpdateTimestamp = _CreationTimestamp;
 310         _HourUpdateTimestamp = _CreationTimestamp;
 311         _DayUpdateTimestamp = _CreationTimestamp;
 312 }
 313
 314
 315
 316 /*
 317  * Get value as a string
 318  * \param path is of the form '[tableindex|tablename].[$key|index].attrib1.attrib2'
 319  */
 320 string  CDatabase::getValue(const CLocatePath::TLocatePath &path)
 321 {
 322         if (path.size() < 3)
 323         {
 324                 PDS_WARNING("getValue(): path is too short");
 325                 return "";
 326         }
 327
 328         uint    node = 0;
 329
 330         // select table
 331         const CTable*   table;
 332         uint                    tableId;
 333         if (sscanf(path[node].Name.c_str(), "%d", &tableId) == 1)
 334                 table = getTable(tableId);
 335         else
 336                 table = getTable(path[node].Name);
 337
 338         if (table == NULL)
 339         {
 340                 PDS_WARNING("getValue(): unable to select table '%s'", path[node].Name.c_str());
 341                 return "";
 342         }
 343
 344         ++node;
 345
 346         // select row
 347         RY_PDS::CObjectIndex    object;
 348
 349         if (path[node].Name[0] == '$')
 350         {
 351                 uint64  key;
 352                 if (sscanf(path[node].Name.c_str()+1, "%" NL_I64 "X", &key) != 1)
 353                 {
 354                         PDS_WARNING("getValue(): unable to select mapped row '%s'", path[node].Name.c_str());
 355                         return "";
 356                 }
 357
 358                 object = table->getMappedRow(key);
 359         }
 360         else
 361         {
 362                 RY_PDS::TRowIndex       row;
 363
 364                 if (sscanf(path[node].Name.c_str(), "%u", &row) != 1)
 365                 {
 366                         PDS_WARNING("getValue(): unable to select row '%s'", path[node].Name.c_str());
 367                         return "";
 368                 }
 369
 370                 object = RY_PDS::CObjectIndex((RY_PDS::TTableIndex)table->getId(), row);
 371         }
 372
 373         if (!object.isValid() || !isAllocated(object))
 374         {
 375                 PDS_WARNING("getValue(): object '%s' not accessible", path[node].Name.c_str());
 376                 return "";
 377         }
 378
 379         ++node;
 380
 381         const CTable*   subTable = table;
 382
 383         // browse through row
 384         while (node < path.size())
 385         {
 386                 const CAttribute*       attribute = subTable->getAttribute(path[node].Name);
 387
 388                 if (attribute == NULL)
 389                 {
 390                         PDS_WARNING("getValue(): '%s' is not an attribute of '%s'", path[node].Name.c_str(), subTable->getName().c_str());
 391                         return "";
 392                 }
 393
 394                 ++node;
 395         }
 396
 397         return "";
 398 }
 399
 400
 401
 402
 403
 404
 405 /*
 406  * Get Table
 407  */
 408 const CTable*   CDatabase::getTable(const std::string &name) const
 409 {
 410         uint    i;
 411         for (i=0; i<_Tables.size(); ++i)
 412                 if (_Tables[i] != NULL && _Tables[i]->getName() == name)
 413                         return _Tables[i];
 414
 415         return NULL;
 416 }
 417
 418 /*
 419  * Get Type
 420  */
 421 const CType*    CDatabase::getType(const std::string &name) const
 422 {
 423         uint    i;
 424         for (i=0; i<_Types.size(); ++i)
 425                 if (_Types[i] != NULL && _Types[i]->getName() == name)
 426                         return _Types[i];
 427
 428         return NULL;
 429 }
 430
 431
 432 /*
 433  * Get Attribute
 434  */
 435 const CAttribute*       CDatabase::getAttribute(uint32 tableId, uint32 attributeId) const
 436 {
 437         const CTable*   table = getTable(tableId);
 438
 439         if (table == NULL)
 440                 return NULL;
 441
 442         return table->getAttribute(attributeId);
 443 }
 444
 445 /*
 446  * Get Column
 447  */
 448 const CColumn*  CDatabase::getColumn(uint32 tableId, uint32 columnId) const
 449 {
 450         const CTable*   table = getTable(tableId);
 451
 452         if (table == NULL)
 453                 return NULL;
 454
 455         return table->getColumn(columnId);
 456 }
 457
 458
 459
 460
 461 /*
 462  * Allocate a row in a table
 463  * \param index is the table/row to allocate
 464  * Return true if succeded
 465  */
 466 bool    CDatabase::allocate(const RY_PDS::CObjectIndex &index)
 467 {
 468         H_AUTO(PDS_Database_allocate);
 469
 470         if (!initialised())
 471         {
 472                 PDS_WARNING("allocate(): database not initialised");
 473                 return false;
 474         }
 475
 476         if (!index.isValid())
 477         {
 478                 PDS_WARNING("allocate(): index '%s' is not valid", index.toString().c_str());
 479                 return false;
 480         }
 481
 482         CTable  *table = getNonConstTable(index.table());
 483
 484         if (table == NULL || !table->initialised())
 485         {
 486                 PDS_WARNING("allocate(): table '%d' is not initialised", index.table());
 487                 return false;
 488         }
 489
 490         bool    success = table->allocate(index.row());
 491
 492         if (success)
 493                 PDS_FULL_DEBUG("allocated '%s' successfully", index.toString().c_str());
 494
 495         return success;
 496 }
 497
 498 /*
 499  * Deallocate a row in a table
 500  * \param index is the table/row to deallocate
 501  * Return true if succeded
 502  */
 503 bool    CDatabase::deallocate(const RY_PDS::CObjectIndex &index)
 504 {
 505         H_AUTO(PDS_Database_deallocate);
 506
 507         if (!initialised())
 508         {
 509                 PDS_WARNING("deallocate(): database not initialised");
 510                 return false;
 511         }
 512
 513         if (!index.isValid())
 514         {
 515                 PDS_WARNING("deallocate(): index '%s' is not valid", index.toString().c_str());
 516                 return false;
 517         }
 518
 519         CTable  *table = getNonConstTable(index.table());
 520
 521         if (table == NULL || !table->initialised())
 522         {
 523                 PDS_WARNING("deallocate(): table '%d' is not initialised", index.table());
 524                 return false;
 525         }
 526
 527         bool    success = table->deallocate(index.row());
 528
 529         if (success)
 530                 PDS_FULL_DEBUG("deallocated '%s' successfully", index.toString().c_str());
 531
 532         return success;
 533 }
 534
 535 /*
 536  * Tells if an object is allocated
 537  * \param object is the object index to test
 538  */
 539 bool    CDatabase::isAllocated(const RY_PDS::CObjectIndex &index) const
 540 {
 541         const CTable*   table = getTable(index.table());
 542
 543         return table != NULL && table->isAllocated(index.row());
 544 }
 545
 546
 547 /*
 548  * Map a row in a table
 549  * \param index is the table/row to allocate
 550  * \param key is the 64 bits row key
 551  * Return true if succeded
 552  */
 553 bool    CDatabase::mapRow(const RY_PDS::CObjectIndex &index, uint64 key)
 554 {
 555         if (!initialised())
 556         {
 557                 PDS_WARNING("mapRow(): database not initialised");
 558                 return false;
 559         }
 560
 561         if (!index.isValid())
 562         {
 563                 PDS_WARNING("mapRow(): index '%s' is not valid", index.toString().c_str());
 564                 return false;
 565         }
 566
 567         CTable  *table = getNonConstTable(index.table());
 568
 569         if (table == NULL || !table->initialised())
 570         {
 571                 PDS_WARNING("deallocate(): table '%d' is not initialised", index.table());
 572                 return false;
 573         }
 574
 575         bool    success = table->mapRow(index, key);
 576
 577         if (success)
 578                 PDS_FULL_DEBUG("mapped '%016" NL_I64 "X' to '%s' successfully", key, index.toString().c_str());
 579
 580         return success;
 581 }
 582
 583 /*
 584  * Unmap a row in a table
 585  * \param tableIndex is the table to find row
 586  * \param key is the 64 bits row key
 587  * Return true if succeded
 588  */
 589 bool    CDatabase::unmapRow(RY_PDS::TTableIndex tableIndex, uint64 key)
 590 {
 591         if (!initialised())
 592         {
 593                 PDS_WARNING("unmapRow(): database not initialised");
 594                 return false;
 595         }
 596
 597         CTable  *table = getNonConstTable(tableIndex);
 598
 599         if (table == NULL || !table->initialised())
 600         {
 601                 PDS_WARNING("deallocate(): table '%d' is not initialised", tableIndex);
 602                 return false;
 603         }
 604
 605         bool    success = table->unmapRow(key);
 606
 607         if (success)
 608                 PDS_FULL_DEBUG("unmapped '%016" NL_I64 "X' successfully", key);
 609
 610         return success;
 611 }
 612
 613 /**
 614  * Get a mapped row
 615  * \param tableIndex is the table in which the row is mapped
 616  * \param key is the 64 bits row key
 617  * Return a valid CObjectIndex if success
 618  */
 619 RY_PDS::CObjectIndex    CDatabase::getMappedRow(RY_PDS::TTableIndex tableIndex, uint64 key) const
 620 {
 621         if (!initialised())
 622         {
 623                 PDS_WARNING("getMappedRow(): database not initialised");
 624                 return RY_PDS::CObjectIndex();
 625         }
 626
 627         const CTable    *table = getTable(tableIndex);
 628
 629         if (table == NULL || !table->initialised())
 630         {
 631                 PDS_WARNING("getMappedRow(): table '%d' is not initialised", tableIndex);
 632                 return RY_PDS::CObjectIndex();
 633         }
 634
 635         return table->getMappedRow(key);
 636 }
 637
 638 /**
 639  * Release a row in a table
 640  * \param index is the table/row to release
 641  * Return true if succeded
 642  */
 643 bool    CDatabase::release(const RY_PDS::CObjectIndex &index)
 644 {
 645         if (!initialised())
 646         {
 647                 PDS_WARNING("release(): database not initialised");
 648                 return false;
 649         }
 650
 651         if (!index.isValid())
 652         {
 653                 PDS_WARNING("release(): index '%s' is not valid", index.toString().c_str());
 654                 return false;
 655         }
 656
 657         CTable  *table = getNonConstTable(index.table());
 658
 659         if (table == NULL || !table->initialised())
 660         {
 661                 PDS_WARNING("release(): table '%d' is not initialised", index.table());
 662                 return false;
 663         }
 664
 665         bool    success = table->release(index.row());
 666
 667         if (success)
 668                 PDS_FULL_DEBUG("released '%s' successfully", index.toString().c_str());
 669
 670         return success;
 671 }
 672
 673 /*
 674  * Release all rows in all table
 675  * Typically, the client disconnected, there is no need to keep rows
 676  */
 677 bool    CDatabase::releaseAll()
 678 {
 679         if (!initialised())
 680         {
 681                 PDS_WARNING("releaseAll(): database not initialised");
 682                 return false;
 683         }
 684
 685         uint    i;
 686         for (i=0; i<_Tables.size(); ++i)
 687         {
 688                 if (_Tables[i] != NULL && _Tables[i]->initialised())
 689                 {
 690                         _Tables[i]->releaseAll();
 691                 }
 692         }
 693
 694         return true;
 695 }
 696
 697
 698 //
 699 // Set method
 700 //
 701 bool    CDatabase::set(RY_PDS::TTableIndex table, RY_PDS::TRowIndex row, RY_PDS::TColumnIndex column, uint datasize, const void* dataptr)
 702 {
 703         H_AUTO(PDS_Database_set);
 704
 705         if (!initialised())
 706         {
 707                 PDS_WARNING("set(): database not initialised");
 708                 return false;
 709         }
 710
 711         CTable*         _table = getNonConstTable(table);
 712
 713         if (_table == NULL)
 714         {
 715                 PDS_WARNING("set(): table '%d' is NULL", table);
 716                 return false;
 717         }
 718
 719         bool    success = _table->set(row, column, datasize, dataptr);
 720
 721         if (success)
 722                 PDS_FULL_DEBUG("set '%s' column '%d' successfully", RY_PDS::CObjectIndex(table, row).toString().c_str(), column);
 723
 724         return success;
 725 }
 726
 727
 728 /*
 729  * Set an object parent
 730  */
 731 bool    CDatabase::setParent(RY_PDS::TTableIndex table, RY_PDS::TRowIndex row, RY_PDS::TColumnIndex column, const RY_PDS::CObjectIndex &parent)
 732 {
 733         H_AUTO(PDS_Database_setParent);
 734
 735         if (!initialised())
 736         {
 737                 PDS_WARNING("set(): database not initialised");
 738                 return false;
 739         }
 740
 741         CTable*         _table = getNonConstTable(table);
 742
 743         if (_table == NULL)
 744         {
 745                 PDS_WARNING("set(): table '%d' is NULL", table);
 746                 return false;
 747         }
 748
 749         bool    success = _table->setParent(row, column, parent);
 750
 751         if (success)
 752                 PDS_FULL_DEBUG("set '%s' column '%d' successfully", RY_PDS::CObjectIndex(table, row).toString().c_str(), column);
 753
 754         return success;
 755 }
 756
 757
 758 //
 759 // Get method
 760 //
 761 bool    CDatabase::get(RY_PDS::TTableIndex table, RY_PDS::TRowIndex row, RY_PDS::TColumnIndex column, uint& datasize, void* dataptr, TDataType &type)
 762 {
 763         if (!initialised())
 764         {
 765                 PDS_WARNING("set(): database not initialised");
 766                 return false;
 767         }
 768
 769         CTable*         _table = getNonConstTable(table);
 770
 771         if (_table == NULL)
 772         {
 773                 PDS_WARNING("set(): table '%d' is NULL", table);
 774                 return false;
 775         }
 776
 777         bool    success = _table->get(row, column, datasize, dataptr, type);
 778
 779         if (success)
 780                 PDS_FULL_DEBUG("get '%s' column '%d' successfully", RY_PDS::CObjectIndex(table, row).toString().c_str(), column);
 781
 782         return success;
 783 }
 784
 785
 786
 787
 788 /*
 789  * Display database
 790  */
 791 void    CDatabase::display(NLMISC::CLog* log, bool displayHeader) const
 792 {
 793         if (!initialised())
 794         {
 795                 log->displayNL("Database not initialised");
 796         }
 797
 798         log->displayNL("Database '%s'", _State.Name.c_str());
 799         log->displayNL("%d types, %d tables", _Types.size(), _Tables.size());
 800
 801         uint    i;
 802
 803         log->displayNL("     %-36s | %-15s | %-2s | %s", "TypeId/Name", "DataTypeId/Name", "Sz", "IsIndex");
 804         for (i=0; i<_Types.size(); ++i)
 805         {
 806                 if (_Types[i] == NULL)
 807                         log->display("** Type %d not initialised", i);
 808                 else
 809                         _Types[i]->display(log);
 810         }
 811
 812         bool    first = true;
 813
 814         for (i=0; i<_Tables.size(); ++i)
 815         {
 816                 if (_Tables[i] == NULL)
 817                 {
 818                         log->display("** Table %d not initialised", i);
 819                 }
 820                 else
 821                 {
 822                         _Tables[i]->display(log, false, first);
 823                         first = false;
 824                 }
 825         }
 826 }
 827
 828 /*
 829  * Dump database content and info of an object to xml
 830  */
 831 void    CDatabase::dumpToXml(const RY_PDS::CObjectIndex& index, NLMISC::IStream& xml, sint expandDepth)
 832 {
 833         if (xml.isReading() || !initialised())
 834                 return;
 835
 836         CTable* table = getNonConstTable(index.table());
 837
 838         if (table != NULL)
 839                 table->dumpToXml(index.row(), xml, expandDepth);
 840 }
 841
 842
 843
 844 /*
 845  * Set value with human readable parameters
 846  */
 847 bool    CDatabase::set(RY_PDS::TTableIndex table, RY_PDS::TRowIndex row, RY_PDS::TColumnIndex column, const std::string& type, const std::string &value)
 848 {
 849         TDataType       datatype = getDataTypeFromName(type);
 850
 851         if (!checkDataType(datatype))
 852                 return false;
 853
 854         switch (datatype)
 855         {
 856         case PDS_bool:
 857         case PDS_char:
 858         case PDS_uint8:
 859         case PDS_sint8:
 860                 {
 861                         uint8   data;
 862                         NLMISC::fromString(value, data);
 863                         return set(table, row, column, sizeof(data), &data);
 864                 }
 865                 break;
 866
 867         case PDS_ucchar:
 868         case PDS_uint16:
 869         case PDS_sint16:
 870                 {
 871                         uint16  data;
 872                         NLMISC::fromString(value, data);
 873                         return set(table, row, column, sizeof(data), &data);
 874                 }
 875                 break;
 876
 877         case PDS_uint32:
 878         case PDS_sint32:
 879         case PDS_enum:
 880         case PDS_CSheetId:
 881         case PDS_CNodeId:
 882                 {
 883                         uint32  data;
 884                         NLMISC::fromString(value, data);
 885                         return set(table, row, column, sizeof(data), &data);
 886                 }
 887                 break;
 888
 889         case PDS_uint64:
 890         case PDS_sint64:
 891                 {
 892                         uint64  data;
 893                         sscanf(value.c_str(), "%016" NL_I64 "d", &data);
 894                         return set(table, row, column, sizeof(data), &data);
 895                 }
 896                 break;
 897
 898         case PDS_CEntityId:
 899                 {
 900                         CEntityId       data;
 901                         data.fromString(value.c_str());
 902                         return set(table, row, column, sizeof(data), &data);
 903                 }
 904                 break;
 905
 906         case PDS_float:
 907                 {
 908                         float data;
 909                         NLMISC::fromString(value, data);
 910                         return set(table, row, column, sizeof(data), &data);
 911                 }
 912                 break;
 913
 914         case PDS_double:
 915                 {
 916                         double data;
 917                         NLMISC::fromString(value, data);
 918                         return set(table, row, column, sizeof(data), &data);
 919                 }
 920                 break;
 921
 922         case PDS_Index:
 923                 {
 924                         RY_PDS::CObjectIndex    data;
 925                         data.fromString(value.c_str());
 926                         return set(table, row, column, sizeof(data), &data);
 927                 }
 928                 break;
 929
 930         case PDS_dimension:
 931                 {
 932                         uint32  data;
 933                         NLMISC::fromString(value, data);
 934                         return set(table, row, column, sizeof(data), &data);
 935                 }
 936                 break;
 937         }
 938
 939         return false;
 940 }
 941
 942
 943 /*
 944  * Fetch data
 945  */
 946 bool    CDatabase::fetch(const RY_PDS::CObjectIndex& index, RY_PDS::CPData &data, bool fetchIndex)
 947 {
 948         H_AUTO(PDS_Database_fetch);
 949
 950         if (!initialised())
 951         {
 952                 PDS_WARNING("set(): database not initialised");
 953                 return false;
 954         }
 955
 956         CTable*         table = getNonConstTable(index.table());
 957
 958         if (table == NULL)
 959         {
 960                 PDS_WARNING("fetch(): unable to get table '%d'", index.table());
 961                 return false;
 962         }
 963
 964         bool    success = table->fetch(index.row(), data, fetchIndex);
 965
 966         if (success)
 967                 PDS_FULL_DEBUG("fetch '%s' successfully", index.toString().c_str());
 968
 969         return success;
 970 }
 971
 972
 973
 974
 975 /*
 976  * Build index allocators
 977  * One per table
 978  */
 979 bool    CDatabase::buildIndexAllocators(std::vector<RY_PDS::CIndexAllocator> &allocators)
 980 {
 981         PDS_DEBUG("buildIndexAllocators()");
 982
 983         if (!initialised())
 984         {
 985                 PDS_WARNING("buildIndexAllocators(): database not initialised");
 986                 return false;
 987         }
 988
 989         allocators.clear();
 990         allocators.resize(_Tables.size());
 991
 992         uint    i;
 993         for (i=0; i<_Tables.size(); ++i)
 994                 if (_Tables[i] != NULL && _Tables[i]->initialised())
 995                         _Tables[i]->buildIndexAllocator(allocators[i]);
 996
 997         return true;
 998 }
 999
1000
1001
1002
1003
1004
1005 /*
1006  * Rebuild forwardrefs from backrefs
1007  */
1008 bool    CDatabase::rebuildForwardRefs()
1009 {
1010         if (!initialised())
1011         {
1012                 PDS_WARNING("rebuildForwardRefs(): database not initialised");
1013                 return false;
1014         }
1015
1016         // clear up object list
1017         _SetMap.clear();
1018
1019         // the rebuild forwardrefs
1020         uint    i;
1021         for (i=0; i<_Tables.size(); ++i)
1022         {
1023                 if (_Tables[i] != NULL && _Tables[i]->initialised())
1024                 {
1025                         if (!_Tables[i]->resetForwardRefs())
1026                         {
1027                                 PDS_WARNING("rebuildForwardRefs(): failed to resetForwardRefs(), abort");
1028                                 return false;
1029                         }
1030                 }
1031         }
1032
1033         for (i=0; i<_Tables.size(); ++i)
1034         {
1035                 if (_Tables[i] != NULL && _Tables[i]->initialised())
1036                 {
1037                         if (!_Tables[i]->rebuildForwardRefs())
1038                         {
1039                                 PDS_WARNING("rebuildForwardRefs(): failed to rebuildForwardRefs(), abort");
1040                                 return false;
1041                         }
1042                 }
1043         }
1044
1045         for (i=0; i<_Tables.size(); ++i)
1046         {
1047                 if (_Tables[i] != NULL && _Tables[i]->initialised())
1048                 {
1049                         if (!_Tables[i]->fixForwardRefs())
1050                         {
1051                                 PDS_WARNING("rebuildForwardRefs(): failed to fixForwardRefs(), abort");
1052                                 return false;
1053                         }
1054                 }
1055         }
1056
1057         return true;
1058 }
1059
1060
1061 /*
1062  * Rebuild table maps
1063  */
1064 bool    CDatabase::rebuildTableMaps()
1065 {
1066         if (!initialised())
1067         {
1068                 PDS_WARNING("rebuildTableMaps(): database not initialised");
1069                 return false;
1070         }
1071
1072         uint    i;
1073         for (i=0; i<_Tables.size(); ++i)
1074         {
1075                 if (_Tables[i] != NULL && _Tables[i]->initialised())
1076                 {
1077                         _Tables[i]->buildRowMapper();
1078                 }
1079         }
1080
1081         return true;
1082 }
1083
1084 /*
1085  * Reset dirty lists
1086  */
1087 bool    CDatabase::resetDirtyTags()
1088 {
1089         if (!initialised())
1090         {
1091                 PDS_WARNING("resetDirtyLists(): database not initialised");
1092                 return false;
1093         }
1094
1095         uint    i;
1096         for (i=0; i<_Tables.size(); ++i)
1097         {
1098                 if (_Tables[i] != NULL && _Tables[i]->initialised())
1099                 {
1100                         _Tables[i]->resetDirtyTags();
1101                 }
1102         }
1103
1104         return true;
1105 }
1106
1107 /*
1108  * Reset and rebuild all maps, references, lists...
1109  */
1110 bool    CDatabase::rebuildVolatileData()
1111 {
1112         PDS_DEBUG("rebuildVolatileData()");
1113
1114         if (!initialised())
1115         {
1116                 PDS_WARNING("rebuildVolatileData(): database not initialised");
1117                 return false;
1118         }
1119
1120         // force database to notify a new reference is up to date
1121         notifyNewReference();
1122
1123         if (!rebuildTableMaps())
1124         {
1125                 PDS_WARNING("rebuildVolatileData(): failed to rebuildTableMaps()");
1126                 return false;
1127         }
1128
1129         if (!rebuildForwardRefs())
1130         {
1131                 PDS_WARNING("rebuildVolatileData(): failed to rebuildForwardRefs()");
1132                 return false;
1133         }
1134
1135         /*
1136         if (!resetDirtyTags())
1137         {
1138                 PDS_WARNING("rebuildVolatileData(): failed to resetDirtyTags()");
1139                 return false;
1140         }
1141         */
1142
1143         // preload data from reference files
1144         uint    i;
1145         for (i=0; i<_Tables.size(); ++i)
1146         {
1147                 if (_Tables[i] != NULL && _Tables[i]->initialised())
1148                 {
1149                         if (!_Tables[i]->preloadRefFiles())
1150                         {
1151                                 PDS_WARNING("rebuildVolatileData(): failed to preloadRefFiles() for table '%d'", i);
1152                                 return false;
1153                         }
1154                 }
1155         }
1156
1157         // load string manager
1158 //      if (!_StringManager.load(_Reference.getPath()))
1159 //      {
1160 //              PDS_WARNING("rebuildVolatileData(): failed to load string manager");
1161 //      }
1162
1163         return true;
1164 }
1165
1166
1167
1168
1169
1170 /*
1171  * Adapt database to new description
1172  * \param description is the latest xml description of the database
1173  * \returns true is adaptation succeded
1174  */
1175 CDatabase*      CDatabase::adapt(const string& description)
1176 {
1177         PDS_DEBUG("adapt()");
1178
1179         if (!initialised())
1180         {
1181                 PDS_WARNING("adapt(): failed, database not initialised");
1182                 return NULL;
1183         }
1184
1185         // get 'From' HashKey
1186         CHashKey        hash1 = _Description.getHashKey();
1187         // get 'Into' HashKey
1188         CHashKey        hash2 = getSHA1((const uint8*)(description.c_str()), (uint32)description.size());
1189
1190         // same hash, ok go on
1191         if (hash1 == hash2)
1192                 return this;
1193
1194         // build a clean reference if needed
1195         if (!isReferenceUpToDate())
1196         {
1197                 if (!buildReference())
1198                 {
1199                         PDS_WARNING("adapt(): failed to buildReference()");
1200                         return NULL;
1201                 }
1202         }
1203
1204         // backup old reference
1205         if (!_Reference.save(_Reference.getPath()+"ref"))
1206         {
1207                 PDS_WARNING("adapt(): failed to backup reference index");
1208         }
1209
1210         // create a new destination database
1211         CDatabase*      into = new CDatabase(_State.Id);
1212
1213         if (!into->createFromScratch(description))
1214         {
1215                 PDS_WARNING("adapt(): failed to create new reference");
1216                 delete into;
1217                 return NULL;
1218         }
1219
1220         CDatabaseAdapter        adapter;
1221
1222         if (!adapter.build(this, into))
1223         {
1224                 PDS_WARNING("adapt(): failed to build() adapter");
1225                 delete into;
1226                 return NULL;
1227         }
1228
1229         if (!adapter.buildContent())
1230         {
1231                 PDS_WARNING("adapt(): adapter failed to buildContent()");
1232                 delete into;
1233                 return NULL;
1234         }
1235
1236         // rebuild volatile data
1237         // that is all non persistant data that can be found from persistant data
1238         if (!into->rebuildVolatileData())
1239         {
1240                 PDS_WARNING("adapt(): failed to rebuildVolatileData()");
1241                 delete into;
1242                 return NULL;
1243         }
1244
1245         // init timestamps as database is ready to run
1246         into->initTimestamps();
1247
1248         return into;
1249 }
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262 /*
1263  * Load previous database state
1264  */
1265 bool    CDatabase::loadState()
1266 {
1267         PDS_DEBUG("loadState()");
1268
1269         if (initialised())
1270         {
1271                 PDS_WARNING("loadState(): cannot loadState(), database is already initialised");
1272                 return false;
1273         }
1274
1275         // check directory exists
1276         std::string     directory = _Reference.getNominalRootPath();
1277         if (!CFile::isExists(directory) || !CFile::isDirectory(directory))
1278         {
1279                 return false;
1280         }
1281
1282         // load database current state (or previous state if current state corrupted)
1283         if (!_State.load(_Reference) && !_State.load(_Reference, true))
1284         {
1285                 return false;
1286         }
1287
1288         // get last valid reference
1289         if (!_Reference.load())
1290         {
1291                 return false;
1292         }
1293
1294         if (_Reference.Index != _State.CurrentIndex)
1295         {
1296                 PDS_WARNING("loadState(): failed, Reference and State files have different current index", (_Reference.getPath()+"description.xml").c_str());
1297                 return false;
1298         }
1299
1300         // load description
1301         if (!_Description.loadDescriptionFile(_Reference.getPath()+"description.xml"))
1302         {
1303                 PDS_WARNING("loadState(): failed to load description file '%s'", (_Reference.getPath()+"description.xml").c_str());
1304                 return false;
1305         }
1306
1307         // and init!
1308         if (!init())
1309         {
1310                 PDS_WARNING("loadState(): failed to init db '%d'", _State.Id);
1311                 return false;
1312         }
1313
1314         PDS_DEBUG("loadState(): database '%d' init ok", _State.Id);
1315
1316         // check reference is up to date
1317         if (!isReferenceUpToDate())
1318         {
1319                 if (!buildReference())
1320                 {
1321                         PDS_WARNING("loadState(): failed to buildReference()");
1322                         return false;
1323                 }
1324         }
1325
1326         // rebuild volatile data
1327         // that is all non persistant data that can be found from persistant data
1328         if (!rebuildVolatileData())
1329         {
1330                 PDS_WARNING("loadState(): failed to rebuildVolatileData()");
1331                 return false;
1332         }
1333
1334         // init timestamps as database is ready to run
1335         initTimestamps();
1336
1337         return true;
1338 }
1339
1340
1341 /*
1342  * Check if reference is still the same
1343  * Reinit reference if reference changed
1344  */
1345 bool    CDatabase::checkReferenceChange()
1346 {
1347         CRefIndex       ref(_State.Id);
1348
1349         // load current reference
1350         if (!ref.load())
1351         {
1352                 PDS_WARNING("checkReferenceChange(): failed to load reference for database '%d', no change assumed", _State.Id);
1353                 return false;
1354         }
1355
1356         // check current database reference and current up to date reference
1357         if (ref.Index != _Reference.Index)
1358         {
1359                 // check nothing went wrong
1360                 if (ref.Index < _Reference.Index || ref.Timestamp <= _Reference.Timestamp)
1361                 {
1362                         PDS_WARNING("checkReferenceChange(): index changed (new index %d), but is lower than previous index (%d), no change assumed", ref.Index, _Reference.Index);
1363                         return false;
1364                 }
1365
1366                 // affect new reference
1367                 _Reference = ref;
1368
1369                 notifyNewReference(false);
1370                 return true;
1371         }
1372
1373         return false;
1374 }
1375
1376
1377 /*
1378  * Check if reference is up to date
1379  * Returns true if reference is the latest valid database image
1380  */
1381 bool    CDatabase::isReferenceUpToDate()
1382 {
1383         PDS_DEBUG("isReferenceUpToDate()");
1384
1385         if (!initialised())
1386         {
1387                 PDS_WARNING("isReferenceUpToDate(): failed, database is not initialised");
1388                 return false;
1389         }
1390
1391         vector<string>  files;
1392
1393         NLMISC::CPath::getPathContent(_Reference.getRootPath()+"hours", false, false, true, files);
1394         NLMISC::CPath::getPathContent(_Reference.getRootPath()+"minutes", false, false, true, files);
1395         NLMISC::CPath::getPathContent(_Reference.getRootPath()+"seconds", false, false, true, files);
1396
1397         bool    upToDate = true;
1398
1399         uint    i;
1400         for (i=0; i<files.size(); ++i)
1401         {
1402                 uint32          tableId;
1403                 CTimestamp      timestamp;
1404
1405                 bool            deltaFile = CDBDeltaFile::isDeltaFileName(files[i], tableId, timestamp);
1406 //              bool            stringLogFile = RY_PDS::CPDStringManager::isLogFileName(files[i], timestamp);
1407
1408                 if (deltaFile /*|| stringLogFile*/)
1409                 {
1410                         if (timestamp > _State.EndTimestamp)
1411                         {
1412                                 CFile::moveFile(files[i]+".disabled", files[i]);
1413                                 continue;
1414                         }
1415
1416                         // compare file timestamp (in name) to current reference timestamp
1417                         if (_Reference.Timestamp <= timestamp && timestamp <= _State.EndTimestamp)
1418                                 upToDate = false;
1419                 }
1420         }
1421
1422         if (upToDate)
1423                 PDS_DEBUG("isReferenceUpToDate(): database is up to date");
1424
1425         return upToDate;
1426 }
1427
1428 /*
1429  * Build a up to date reference
1430  */
1431 bool    CDatabase::buildReference()
1432 {
1433         PDS_DEBUG("buildReference()");
1434
1435         CRefIndex       prev = _Reference;
1436         CRefIndex       next(_State.Id);
1437
1438         if (!next.buildNext())
1439         {
1440                 PDS_WARNING("buildReference(): failed to build next ref");
1441                 return false;
1442         }
1443
1444         PDS_DEBUG("buildReference(): building from reference '%08X' to reference '%08X'", prev.Index, next.Index);
1445
1446         // build reference but clamp to last valid state (
1447         next.Timestamp = _State.EndTimestamp;
1448
1449         _Reference = next;
1450
1451         // setup reference
1452         // this will release all references owned by the internal table buffers
1453         // so we can copy/modify reference files without problem
1454
1455         if (!CReferenceBuilder::build(prev, next))
1456         {
1457                 PDS_WARNING("buildReference(): failed to build next reference");
1458                 return false;
1459         }
1460
1461         notifyNewReference();
1462
1463         PDS_DEBUG("buildReference(): new reference is up to date");
1464
1465         return true;
1466 }
1467
1468
1469
1470
1471
1472 /*
1473  * Create new database from scratch, setup everything needed (references, etc.)
1474  * \param description is the xml database description
1475  */
1476 bool    CDatabase::createFromScratch(const string& description)
1477 {
1478         PDS_DEBUG("createFromScratch()");
1479
1480         // In order to create a new database from scratch, directory must no exist
1481         if (CFile::isExists(_Reference.getNominalRootPath()))
1482         {
1483                 PDS_WARNING("createFromScratch(): failed, directory '%s' already exists for database '%d', manual cleanup is needed to proceed", _Reference.getRootPath().c_str(), _Reference.DatabaseId);
1484                 return false;
1485         }
1486
1487         // build a brand new reference
1488         if (!_Reference.buildNext())
1489         {
1490                 PDS_WARNING("createFromScratch(): failed, unable to init reference");
1491                 return false;
1492         }
1493
1494         if (initialised())
1495         {
1496                 PDS_WARNING("createFromScratch(): failed, database is initialised");
1497                 return false;
1498         }
1499
1500         if (!_Description.loadDescription((uint8*)description.c_str()))
1501         {
1502                 PDS_WARNING("createFromScratch(): failed to load description");
1503                 return false;
1504         }
1505
1506         if (!init())
1507         {
1508                 PDS_WARNING("createFromScratch(): failed to init() database");
1509                 return false;
1510         }
1511
1512 //      // save description
1513 //      if (!_Description.saveDescription(_Reference.getPath() + "description.xml"))
1514 //      {
1515 //              PDS_WARNING("createFromScratch(): failed to save description");
1516 //              return false;
1517 //      }
1518
1519         // init timestamps as database is ready to run
1520         initTimestamps();
1521
1522         // a new reference is valid
1523         notifyNewReference();
1524
1525         return true;
1526 }
1527
1528 /*
1529  * Build the delta files and purge all dirty rows in tables
1530  */
1531 bool    CDatabase::buildDelta(const CTimestamp& starttime, const CTimestamp& endtime)
1532 {
1533         H_AUTO(PDS_Database_buildDelta);
1534
1535         if (!initialised())
1536         {
1537                 PDS_WARNING("buildDelta(): failed, database is not initialised");
1538                 return false;
1539         }
1540
1541         uint    i;
1542         for (i=0; i<_Tables.size(); ++i)
1543         {
1544                 if (!_Tables[i]->buildDelta(starttime, endtime))
1545                 {
1546                         PDS_WARNING("buildDelta(): failed to buildDelta() for table '%d' '%s'", i, _Tables[i]->getName().c_str());
1547                         return false;
1548                 }
1549         }
1550
1551 //      std::string     logDir = RY_PDS::CPDSLib::getLogDirectory(_State.Id);
1552 //      if (!CFile::isExists(logDir) || !CFile::isDirectory(logDir))
1553 //      {
1554 //              if (!CFile::createDirectoryTree(logDir))
1555 //              {
1556 //                      PDS_WARNING("buildDelta(): failed to create log root directory '%s'", logDir.c_str());
1557 //              }
1558 //
1559 //              if (!CFile::setRWAccess(logDir))
1560 //              {
1561 //                      PDS_WARNING("buildDelta(): failed, can't set RW access to directory '%s'", logDir.c_str());
1562 //              }
1563 //      }
1564
1565 //      // save string manager logs
1566 //      if (!_StringManager.logEmpty())
1567 //      {
1568 //              COFile          smf;
1569 //              COXml           smxml;
1570 //              std::string     smfilename = logDir + endtime.toString()+".string_log";
1571 //              if (!smf.open(smfilename) || !smxml.init(&smf) || !_StringManager.storeLog(smxml))
1572 //              {
1573 //                      PDS_WARNING("buildDelta(): failed to build string manager log file '%s'", smfilename.c_str());
1574 //              }
1575 //      }
1576
1577         // save straight logs
1578         if (!_LogQueue.empty())
1579         {
1580 //              std::string     logfilename = logDir + endtime.toString()+"_0000.pd_log";
1581 //              COFile          logf;
1582 //              if (logf.open(logfilename))
1583 //              {
1584 //                      try
1585 //                      {
1586 //                              logf.serialCont(_LogQueue);
1587 //                      }
1588 //                      catch (const Exception& e)
1589 //                      {
1590 //                              PDS_WARNING("buildDelta(): exception occurred while saving straight log : %s", e.what());
1591 //                      }
1592 //              }
1593 //              else
1594 //              {
1595 //                      PDS_WARNING("buildDelta(): failed to build log file '%s'", logfilename.c_str());
1596 //              }
1597 //
1598                 _LogQueue.clear();
1599         }
1600
1601         // State file swapping
1602         std::string     statePath = _Reference.getRootPath();
1603         std::string     stateName = CDatabaseState::fileName();
1604         if (CFile::fileExists(statePath+stateName) &&
1605                 !CFile::copyFile(statePath+"previous_"+stateName, statePath+stateName, false))
1606         {
1607                 PDS_WARNING("buildDelta(): failed copy state file to backup previous_state");
1608         }
1609
1610         // setup state timestamp
1611         _State.EndTimestamp = endtime;
1612
1613         _State.save(_Reference);
1614
1615         return true;
1616 }
1617
1618 /*
1619  * Flush database from released rows
1620  */
1621 bool    CDatabase::flushReleased()
1622 {
1623         if (!initialised())
1624         {
1625                 PDS_WARNING("flushReleased(): failed, database is not initialised");
1626                 return false;
1627         }
1628
1629         uint    i;
1630         for (i=0; i<_Tables.size(); ++i)
1631         {
1632                 if (!_Tables[i]->flushReleased())
1633                 {
1634                         PDS_WARNING("flushReleased(): failed to flushReleased() for table '%d' '%s'", i, _Tables[i]->getName().c_str());
1635                         return false;
1636                 }
1637         }
1638
1639         return true;
1640 }
1641
1642
1643
1644 /*
1645  * Notify a new reference is ready
1646  */
1647 bool    CDatabase::notifyNewReference(bool validateRef)
1648 {
1649         if (!initialised())
1650         {
1651                 PDS_WARNING("notifyNewReference(): failed, database is not initialised");
1652                 return false;
1653         }
1654
1655         if (validateRef)
1656                 _Reference.setAsValidRef();
1657
1658         _State.CurrentIndex = _Reference.Index;
1659
1660         uint    i;
1661         for (i=0; i<_Tables.size(); ++i)
1662                 _Tables[i]->notifyNewReference(_Reference);
1663
1664         return true;
1665 }
1666
1667
1668
1669
1670 /*
1671  * Receive update
1672  */
1673 void    CDatabase::receiveUpdate(uint32 id)
1674 {
1675         // add to acknowledged updates
1676         _ReceivedUpdates.push_back(id);
1677
1678         _State.LastUpdateId = id;
1679 }
1680
1681 /*
1682  * Flush updates
1683  */
1684 void    CDatabase::flushUpdates(std::vector<uint32>& acknowledged)
1685 {
1686         // copy acknowledge and flush
1687         acknowledged = _ReceivedUpdates;
1688         _ReceivedUpdates.clear();
1689 }
1690
1691
1692 /**
1693  * Get Update Queue for id
1694  * May return NULL if message was already received
1695  */
1696 RY_PDS::CDbMessageQueue*        CDatabase::getUpdateMessageQueue(uint32 updateId)
1697 {
1698         if (updateId != 0 && updateId <= _State.LastUpdateId)
1699                 return NULL;
1700
1701         _LogQueue.push_back(RY_PDS::CUpdateLog());
1702
1703         RY_PDS::CUpdateLog&     ulog = _LogQueue.back();
1704
1705         ulog.UpdateId = updateId;
1706         ulog.createUpdates();
1707         ulog.StartStamp.setToCurrent();
1708         ulog.EndStamp.setToCurrent();
1709
1710         return ulog.getUpdates();
1711 }
1712
1713
1714
1715
1716 CVariable<uint> MinuteUpdateRate("pds", "MinuteUpdateRate", "Number of seconds between two minute updates", 60, 0, true);
1717 CVariable<uint> HourUpdateRate("pds", "HourUpdateRate", "Number of seconds between two hour updates", 3600, 0, true);
1718 CVariable<uint> ReferenceUpdateRate("pds", "ReferenceUpdateRate", "Number of seconds between two reference builds", 86400, 0, true);
1719
1720 CVariable<uint> KeepSecondsAtMinuteUpdate("pds", "KeepSecondsAtMinuteUpdate", "Number of seconds to keep (in number of minute updates)", 2, 0, true);
1721 CVariable<uint> KeepMinutesAtHourUpdate("pds", "KeepMinutesAtHourUpdate", "Number of minutes to keep (in number of hours updates)", 2, 0, true);
1722 CVariable<uint> KeepHoursAtReferenceUpdate("pds", "KeepHoursAtReferenceUpdate", "Number of hours to keep (in number of daily updates)", 2, 0, true);
1723
1724
1725 /*
1726  * Send Delta/Reference build commands
1727  */
1728 bool    CDatabase::sendBuildCommands(const CTimestamp& current)
1729 {
1730         checkUpdateRates();
1731
1732         while (current - _MinuteUpdateTimestamp >= MinuteUpdateRate)
1733         {
1734                 CTimestamp      start = _MinuteUpdateTimestamp;
1735                 CTimestamp      end = start + MinuteUpdateRate;
1736                 CTimestamp      keep = end - MinuteUpdateRate*KeepSecondsAtMinuteUpdate;
1737
1738                 string                                  outputPath = _Reference.getMinutesUpdatePath();
1739                 string                                  hoursUpdatePath = _Reference.getHoursUpdatePath();
1740                 string                                  minutesUpdatePath = _Reference.getMinutesUpdatePath();
1741                 string                                  secondsUpdatePath = _Reference.getSecondsUpdatePath();
1742                 string                                  mintimestamp = start.toString();
1743                 string                                  maxtimestamp = end.toString();
1744                 CDeltaBuilder::TDelta   type = CDeltaBuilder::Minute;
1745                 string                                  keeptimestamp = keep.toString();
1746
1747                 CMessage&                               msgdelta = CDbManager::addTask("RB_GEN_DELTA", NULL, NULL);
1748                 msgdelta.serial(outputPath);
1749                 msgdelta.serial(hoursUpdatePath);
1750                 msgdelta.serial(minutesUpdatePath);
1751                 msgdelta.serial(secondsUpdatePath);
1752                 msgdelta.serial(mintimestamp);
1753                 msgdelta.serial(maxtimestamp);
1754                 msgdelta.serialEnum(type);
1755                 msgdelta.serial(keeptimestamp);
1756
1757                 _MinuteUpdateTimestamp = end;
1758         }
1759
1760
1761         while (current - _HourUpdateTimestamp >= HourUpdateRate)
1762         {
1763                 CTimestamp      start = _HourUpdateTimestamp;
1764                 CTimestamp      end = start + HourUpdateRate;
1765                 CTimestamp      keep = end - HourUpdateRate*KeepMinutesAtHourUpdate;
1766
1767                 string                                  outputPath = _Reference.getHoursUpdatePath();
1768                 string                                  hoursUpdatePath = _Reference.getHoursUpdatePath();
1769                 string                                  minutesUpdatePath = _Reference.getMinutesUpdatePath();
1770                 string                                  secondsUpdatePath = _Reference.getSecondsUpdatePath();
1771                 string                                  mintimestamp = start.toString();
1772                 string                                  maxtimestamp = end.toString();
1773                 CDeltaBuilder::TDelta   type = CDeltaBuilder::Hour;
1774                 string                                  keeptimestamp = keep.toString();
1775
1776                 CMessage&                               msgdelta = CDbManager::addTask("RB_GEN_DELTA", NULL, NULL);
1777                 msgdelta.serial(outputPath);
1778                 msgdelta.serial(hoursUpdatePath);
1779                 msgdelta.serial(minutesUpdatePath);
1780                 msgdelta.serial(secondsUpdatePath);
1781                 msgdelta.serial(mintimestamp);
1782                 msgdelta.serial(maxtimestamp);
1783                 msgdelta.serialEnum(type);
1784                 msgdelta.serial(keeptimestamp);
1785
1786                 _HourUpdateTimestamp = end;
1787         }
1788
1789         while (current - _DayUpdateTimestamp >= ReferenceUpdateRate)
1790         {
1791                 CTimestamp      start = _DayUpdateTimestamp;
1792                 CTimestamp      end = start + ReferenceUpdateRate;
1793                 CTimestamp      keep = end - ReferenceUpdateRate*KeepHoursAtReferenceUpdate;
1794
1795                 CRefIndex*      next = new CRefIndex(_State.Id);
1796                 *next = _Reference;
1797                 next->buildNext();
1798
1799                 string          rootRefPath = _Reference.getRootPath();
1800                 string          previousReferencePath = _Reference.getPath();
1801                 string          nextReferencePath = next->getPath();
1802
1803                 string          logUpdatePath = _Reference.getLogPath();
1804                 string          hoursUpdatePath = _Reference.getHoursUpdatePath();
1805                 string          minutesUpdatePath = _Reference.getMinutesUpdatePath();
1806                 string          secondsUpdatePath = _Reference.getSecondsUpdatePath();
1807
1808                 string          mintimestamp = start.toString();
1809                 string          maxtimestamp = end.toString();
1810                 string          keeptimestamp = keep.toString();
1811
1812                 CMessage&       msgref = CDbManager::addTask("RB_GEN_REF", this, (void*)next);
1813                 msgref.serial(rootRefPath);
1814                 msgref.serial(previousReferencePath);
1815                 msgref.serial(nextReferencePath);
1816                 msgref.serial(hoursUpdatePath);
1817                 msgref.serial(minutesUpdatePath);
1818                 msgref.serial(secondsUpdatePath);
1819                 msgref.serial(logUpdatePath);
1820                 msgref.serial(mintimestamp);
1821                 msgref.serial(maxtimestamp);
1822                 msgref.serial(keeptimestamp);
1823
1824                 _DayUpdateTimestamp = end;
1825         }
1826
1827         return true;
1828 }
1829
1830 /*
1831  * Task ran successfully
1832  */
1833 void    CDatabase::taskSuccessful(void* arg)
1834 {
1835         // when reference is up to date, notify new reference
1836         CRefIndex*      ref = (CRefIndex*)arg;
1837
1838         _Reference = *ref;
1839         notifyNewReference(true);
1840
1841         delete ref;
1842 }
1843
1844 /*
1845  * Task failed!
1846  */
1847 void    CDatabase::taskFailed(void* arg)
1848 {
1849         // ok, nothing to do in this case...
1850 }
1851
1852 /*
1853  * Static Check for update rates
1854  */
1855 void    CDatabase::checkUpdateRates()
1856 {
1857         uint    deltaRate = DeltaUpdateRate;
1858         uint    minuteRate = MinuteUpdateRate;
1859
1860         // minute rate must be a multiple of delta rate
1861         if ((minuteRate % deltaRate) != 0)
1862         {
1863                 minuteRate = deltaRate*(minuteRate/deltaRate + 1);
1864                 nlwarning("CDatabase::checkUpdateRate(): MinuteUpdateRate is not a multiple of DeltaUpdateRate! Rounded to %d seconds", minuteRate);
1865                 MinuteUpdateRate = minuteRate;
1866         }
1867
1868         uint    hourRate = HourUpdateRate;
1869
1870         // hour rate must be a multiple of minute rate
1871         if ((hourRate % minuteRate) != 0)
1872         {
1873                 hourRate = minuteRate*(hourRate/minuteRate + 1);
1874                 nlwarning("CDatabase::checkUpdateRate(): HourUpdateRate is not a multiple of MinuteUpdateRate! Rounded to %d seconds", hourRate);
1875                 HourUpdateRate = hourRate;
1876         }
1877
1878         uint    referenceRate = ReferenceUpdateRate;
1879
1880         // reference rate must be a multiple of hour rate
1881         if ((referenceRate % hourRate) != 0)
1882         {
1883                 referenceRate = hourRate*(referenceRate/hourRate + 1);
1884                 nlwarning("CDatabase::checkUpdateRate(): ReferenceUpdateRate is not a multiple of HourUpdateRate! Rounded to %d seconds", referenceRate);
1885                 ReferenceUpdateRate = referenceRate;
1886         }
1887 }
1888
1889
1890
1891 /*
1892  * Serialise SheetId String Mapper
1893  */
1894 void    CDatabase::serialSheetIdStringMapper(NLMISC::IStream& f)
1895 {
1896         // serial mapper
1897         _SheetIdStringMapper.serial(f);
1898
1899         if (f.isReading())
1900         {
1901                 // if mapper is read, save it now in reference path
1902                 std::string             refPath = _Reference.getPath();
1903
1904                 COFile  ofile;
1905                 if (!ofile.open(refPath+"sheetid_map.bin"))
1906                 {
1907                         PDS_WARNING("serialSheetIdStringMapper(): failed to open reference sheetid_map file '%s' for save", (refPath+"sheetid_map.bin").c_str());
1908                         return;
1909                 }
1910
1911                 _SheetIdStringMapper.serial(ofile);
1912         }
1913 }
1914
1915
1916
1917 /*
1918  * Get Table Index from name
1919  */
1920 RY_PDS::TTableIndex     CDatabase::getTableIndex(const std::string& tableName) const
1921 {
1922         const CTable*   table = getTable(tableName);
1923         return (table == NULL) ? RY_PDS::INVALID_TABLE_INDEX : (RY_PDS::TTableIndex)table->getId();
1924 }
1925
1926 /*
1927  * Get Table Index from name
1928  */
1929 std::string     CDatabase::getTableName(RY_PDS::TTableIndex index) const
1930 {
1931         if (index >= _Tables.size() || _Tables[index] == NULL)
1932                 return "invalid";
1933
1934         return _Tables[index]->getName();
1935 }
1936
1937
1938
1939
1940 /*
1941  * Search object in database using its key
1942  * \param key is the 64 bits row key to search through all tables
1943  */
1944 bool    CDatabase::searchObjectIndex(uint64 key, std::set<RY_PDS::CObjectIndex>& indexes) const
1945 {
1946         indexes.clear();
1947
1948         uint    i;
1949         for (i=0; i<_Tables.size(); ++i)
1950         {
1951                 if (_Tables[i] == NULL || !_Tables[i]->initialised() || !_Tables[i]->isMapped())
1952                         continue;
1953
1954                 RY_PDS::CObjectIndex    index = _Tables[i]->getMappedRow(key);
1955
1956                 if (index.isValid())
1957                         indexes.insert(index);
1958         }
1959
1960         return !indexes.empty();
1961 }