1 // Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
2 // Copyright (C) 2010 Winch Gate Property Limited
4 // This source file has been modified by the following contributors:
5 // Copyright (C) 2013 Laszlo KIS-ADAM (dfighter) <dfighter1985@gmail.com>
6 // Copyright (C) 2013-2020 Jan BOON (Kaetemi) <jan.boon@kaetemi.be>
8 // This program is free software: you can redistribute it and/or modify
9 // it under the terms of the GNU Affero General Public License as
10 // published by the Free Software Foundation, either version 3 of the
11 // License, or (at your option) any later version.
13 // This program is distributed in the hope that it will be useful,
14 // but WITHOUT ANY WARRANTY; without even the implied warranty of
15 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 // GNU Affero General Public License for more details.
18 // You should have received a copy of the GNU Affero General Public License
19 // along with this program. If not, see <http://www.gnu.org/licenses/>.
23 //#define TRACE_READ_DELTA
24 //#define TRACE_WRITE_DELTA
25 //#define TRACE_SET_VALUE
33 #include "nel/misc/cdb_branch.h"
34 #include "nel/misc/cdb_leaf.h"
35 #include "nel/misc/xml_auto_ptr.h"
36 //#include <iostream.h>
44 #include "nel/misc/types_nl.h"
45 #include "nel/misc/debug.h"
46 #include "nel/misc/file.h"
47 #include "nel/misc/i_xml.h"
48 #include "nel/misc/progress_callback.h"
49 #include "nel/misc/bit_mem_stream.h"
50 #include "nel/misc/bit_set.h"
51 #include "nel/misc/cdb_bank_handler.h"
53 #include <libxml/parser.h>
69 //-----------------------------------------------
72 //-----------------------------------------------
73 static /*inline*/ void addNode( ICDBNode
*newNode
, std::string newName
, CCDBNodeBranch
* parent
,
74 std::vector
<ICDBNode
*> &nodes
, std::vector
<ICDBNode
*> &nodesSorted
,
75 xmlNodePtr
&child
, const string
& bankName
,
76 bool atomBranch
, bool clientOnly
,
77 IProgressCallback
&progressCallBack
,
78 bool mapBanks
, CCDBBankHandler
*bankHandler
= NULL
)
80 nodesSorted
.push_back(newNode
);
81 nodes
.push_back(newNode
);
82 nodes
.back()->setParent(parent
);
83 nodes
.back()->setAtomic( parent
->isAtomic() || atomBranch
);
84 nodes
.back()->init(child
, progressCallBack
);
86 // Setup bank mapping for first-level node
87 if ( mapBanks
&& (parent
->getParent() == NULL
) )
89 if ( ! bankName
.empty() )
91 bankHandler
->mapNodeByBank( bankName
);
92 //nldebug( "CDB: Mapping %s for %s (node %u)", newName.c_str(), bankName.c_str(), nodes.size()-1 );
96 nlerror( "Missing bank for first-level node %s", newName
.c_str() );
101 void CCDBNodeBranch::init( xmlNodePtr node
, IProgressCallback
&progressCallBack
, bool mapBanks
, CCDBBankHandler
*bankHandler
)
106 // look for other branches within this branch
107 uint countNode
= CIXml::countChildren (node
, "branch") + CIXml::countChildren (node
, "leaf");
109 for (child
= CIXml::getFirstChildNode (node
, "branch"); child
; child
= CIXml::getNextChildNode (child
, "branch"))
112 progressCallBack
.progress ((float)nodeId
/(float)countNode
);
113 progressCallBack
.pushCropedValues ((float)nodeId
/(float)countNode
, (float)(nodeId
+1)/(float)countNode
);
115 CXMLAutoPtr
name((const char*)xmlGetProp (child
, (xmlChar
*)"name"));
116 CXMLAutoPtr
count((const char*)xmlGetProp (child
, (xmlChar
*)"count"));
117 CXMLAutoPtr
bank((const char*)xmlGetProp (child
, (xmlChar
*)"bank"));
118 CXMLAutoPtr
atom((const char*)xmlGetProp (child
, (xmlChar
*)"atom"));
119 CXMLAutoPtr
clientonly((const char*)xmlGetProp (child
, (xmlChar
*)"clientonly"));
121 string sBank
, sAtom
, sClientonly
;
122 if ( bank
) sBank
= bank
.getDatas();
123 if ( atom
) sAtom
= (const char*)atom
;
124 if ( clientonly
) sClientonly
= clientonly
.getDatas();
125 nlassert((const char *) name
!= NULL
);
126 if ((const char *) count
!= NULL
)
128 // dealing with an array of entries
130 fromString((const char*) count
, countAsInt
);
132 for (uint i
=0;i
<countAsInt
;i
++)
135 progressCallBack
.progress ((float)i
/(float)countAsInt
);
136 progressCallBack
.pushCropedValues ((float)i
/(float)countAsInt
, (float)(i
+1)/(float)countAsInt
);
138 // nlinfo("+ %s%d",name,i);
139 string newName
= string(name
.getDatas())+toString(i
);
140 addNode( new CCDBNodeBranch(newName
), newName
, this, _Nodes
, _NodesByName
, child
, sBank
, sAtom
=="1", sClientonly
=="1", progressCallBack
, mapBanks
, bankHandler
);
144 progressCallBack
.popCropedValues ();
149 // dealing with a single entry
150 // nlinfo("+ %s",name);
151 string newName
= string(name
.getDatas());
152 addNode( new CCDBNodeBranch(newName
), newName
, this, _Nodes
, _NodesByName
, child
, sBank
, sAtom
=="1", sClientonly
=="1", progressCallBack
, mapBanks
, bankHandler
);
157 progressCallBack
.popCropedValues ();
161 // look for leaves of this branch
162 for (child
= CIXml::getFirstChildNode (node
, "leaf"); child
; child
= CIXml::getNextChildNode (child
, "leaf"))
165 progressCallBack
.progress ((float)nodeId
/(float)countNode
);
166 progressCallBack
.pushCropedValues ((float)nodeId
/(float)countNode
, (float)(nodeId
+1)/(float)countNode
);
168 CXMLAutoPtr
name((const char*)xmlGetProp (child
, (xmlChar
*)"name"));
169 CXMLAutoPtr
count((const char*)xmlGetProp (child
, (xmlChar
*)"count"));
170 CXMLAutoPtr
bank((const char*)xmlGetProp (child
, (xmlChar
*)"bank"));
173 if ( bank
) sBank
= bank
.getDatas();
174 nlassert((const char *) name
!= NULL
);
175 if ((const char *) count
!= NULL
)
177 // dealing with an array of entries
179 fromString((const char *) count
, countAsInt
);
181 for (uint i
=0;i
<countAsInt
;i
++)
184 progressCallBack
.progress ((float)i
/(float)countAsInt
);
185 progressCallBack
.pushCropedValues ((float)i
/(float)countAsInt
, (float)(i
+1)/(float)countAsInt
);
187 // nlinfo(" %s%d",name,i);
188 string newName
= string(name
.getDatas())+toString(i
);
189 addNode( new CCDBNodeLeaf(newName
), newName
, this, _Nodes
, _NodesByName
, child
, sBank
, false, false, progressCallBack
, mapBanks
, bankHandler
);
192 progressCallBack
.popCropedValues ();
197 // nlinfo(" %s",name);
198 string newName
= string(name
.getDatas());
199 addNode( new CCDBNodeLeaf(newName
), newName
, this, _Nodes
, _NodesByName
, child
, sBank
, false, false, progressCallBack
, mapBanks
, bankHandler
);
203 progressCallBack
.popCropedValues ();
207 // count number of bits required to store the id
208 if ( (mapBanks
) && (getParent() == NULL
) )
210 nlassert( bankHandler
!= NULL
);
211 nlassertex( bankHandler
->getUnifiedIndexToBankSize() == countNode
, ("Mapped: %u Nodes: %u", bankHandler
->getUnifiedIndexToBankSize(), countNode
) );
212 bankHandler
->calcIdBitsByBank();
218 for ( _IdBits
=1; _Nodes
.size() > ((size_t)1 <<_IdBits
) ; _IdBits
++ ) {}
227 //-----------------------------------------------
230 //-----------------------------------------------
231 void CCDBNodeBranch::attachChild( ICDBNode
* node
, string nodeName
)
233 nlassert(_Parent
==NULL
);
237 node
->setParent(this);
238 _Nodes
.push_back( node
);
239 //nldebug ( "CDB: Attaching node" );
240 _NodesByName
.push_back( node
);
242 #if NL_CDB_OPTIMIZE_PREDICT
249 //-----------------------------------------------
252 //-----------------------------------------------
253 CCDBNodeLeaf
*CCDBNodeBranch::getLeaf( const char *id
, bool bCreate
)
255 // get the last name piece
256 const char *last
= strrchr( id
, ':' );
259 ICDBNode
*pNode
= find( &last
[1] );
260 if( !pNode
&& bCreate
)
262 pNode
= new CCDBNodeLeaf( id
);
263 _Nodes
.push_back( pNode
);
264 _NodesByName
.push_back( pNode
);
266 pNode
->setParent(this);
268 return dynamic_cast<CCDBNodeLeaf
*>(pNode
);
271 //-----------------------------------------------
274 //-----------------------------------------------
275 ICDBNode
* CCDBNodeBranch::getNode (const CTextId
& id
, bool bCreate
)
277 // lookup next element from textid in my index => idx
278 const string
&str
= id
.readNext();
280 ICDBNode
*pNode
= find(str
);
281 // If the node do not exists
286 // Yoyo: must not be SERVER or LOCAL, cause definied through xml.
287 // This may cause some important crash error
288 //nlassert(!id.empty());
289 //nlassert(id.getElement(0)!="SERVER");
290 //nlassert(id.getElement(0)!="LOCAL");
292 if (id
.getCurrentIndex() == id
.size() )
293 newNode
= new CCDBNodeLeaf (str
);
295 newNode
= new CCDBNodeBranch (str
);
297 _Nodes
.push_back( newNode
);
298 _NodesByName
.push_back( newNode
);
300 newNode
->setParent(this);
309 // get property from child
310 if (!id
.hasElements())
313 return pNode
->getNode( id
, bCreate
);
318 //-----------------------------------------------
321 //-----------------------------------------------
322 ICDBNode
* CCDBNodeBranch::getNode( uint16 idx
)
324 if ( idx
< _Nodes
.size() )
332 //-----------------------------------------------
335 //-----------------------------------------------
336 void CCDBNodeBranch::write( CTextId
& id
, FILE * f
)
339 for( i
= 0; i
< _Nodes
.size(); i
++ )
341 id
.push (*_Nodes
[i
]->getName());
342 _Nodes
[i
]->write(id
,f
);
348 //-----------------------------------------------
351 //-----------------------------------------------
352 sint64
CCDBNodeBranch::getProp( CTextId
& id
)
354 // lookup next element from textid in my index => idx
355 const string
&str
= id
.readNext();
356 ICDBNode
*pNode
= find( str
);
357 nlassert( pNode
!= NULL
);
359 // get property from child
360 return pNode
->getProp( id
);
366 //-----------------------------------------------
368 // Set the value of a property (the update flag is set to true)
369 // \param id is the text id of the property/grp
370 // \param name is the name of the property
371 // \param value is the value of the property
372 // \return bool : 'true' if property found.
373 //-----------------------------------------------
374 bool CCDBNodeBranch::setProp( CTextId
& id
, sint64 value
)
377 // lookup next element from textid in my index => idx
378 const string
&str
= id
.readNext();
379 ICDBNode
*pNode
= find( str
);
381 // Property not found.
384 nlwarning("Property %s not found in %s", str
.c_str(), id
.toString().c_str());
388 // set property in child
389 pNode
->setProp(id
,value
);
396 * Update the database from the delta, but map the first level with the bank mapping (see _CDBBankToUnifiedIndexMapping)
398 void CCDBNodeBranch::readAndMapDelta( TGameCycle gc
, CBitMemStream
& s
, uint bank
, CCDBBankHandler
*bankHandler
)
400 nlassert( ! isAtomic() ); // root node mustn't be atomic
404 s
.serial( idx
, bankHandler
->getFirstLevelIdBits( bank
) );
406 // Translate bank index -> unified index
407 idx
= bankHandler
->getServerToClientUIDMapping( bank
, idx
);
408 if (idx
>= _Nodes
.size())
410 throw Exception ("idx %d > _Nodes.size() %d ", idx
, _Nodes
.size());
413 // Display the Name if we are in verbose mode
414 if ( verboseDatabase
)
416 string displayStr
= string("Reading: ") + *(_Nodes
[idx
]->getName());
417 //CInterfaceManager::getInstance()->getChatOutput()->addTextChild( ucstring( displayStr ),CRGBA(255,255,255,255));
418 nlinfo( "CDB: %s%s %u/%d", (!getParent())?"[root] ":"-", displayStr
.c_str(), idx
, _IdBits
);
421 // Apply delta to children nodes
422 _Nodes
[idx
]->readDelta( gc
, s
);
426 //-----------------------------------------------
429 //-----------------------------------------------
430 void CCDBNodeBranch::readDelta( TGameCycle gc
, CBitMemStream
& f
)
434 // Read the atom bitfield
435 uint nbAtomElements
= countLeaves();
437 nlinfo( "CDB/ATOM: %u leaves", nbAtomElements
);
438 CBitSet
bitfield( nbAtomElements
);
439 f
.readBits( bitfield
);
440 if ( ! bitfield
.getVector().empty() )
444 nldebug( "CDB/ATOM: Bitfield: %s LastBits:", bitfield
.toString().c_str() );
445 f
.displayLastBits( bitfield
.size() );
449 // Set each modified property
451 for ( uint i
=0; i
!=bitfield
.size(); ++i
)
457 nldebug( "CDB/ATOM: Reading prop[%u] of atom", i
);
461 CCDBNodeLeaf
*leaf
= findLeafAtCount( atomIndex
);
463 leaf
->readDelta( gc
, f
);
465 nlwarning( "CDB: Can't find leaf with index %u in atom branch %s", i
, getParent()?getName()->c_str():"(root)" );
473 f
.serial(idx
,_IdBits
);
475 if (idx
>= _Nodes
.size())
477 throw Exception ("idx %d > _Nodes.size() %d ", idx
, _Nodes
.size());
480 // Display the Name if we are in verbose mode
481 if ( verboseDatabase
)
483 string displayStr
= string("Reading: ") + *(_Nodes
[idx
]->getName());
484 //CInterfaceManager::getInstance()->getChatOutput()->addTextChild( ucstring( displayStr ),CRGBA(255,255,255,255));
485 nlinfo( "CDB: %s%s %u/%d", (!getParent())?"[root] ":"-", displayStr
.c_str(), idx
, _IdBits
);
488 _Nodes
[idx
]->readDelta(gc
, f
);
494 //-----------------------------------------------
497 //-----------------------------------------------
498 // For old debug of random crash (let it in case it come back)
499 //static bool AllowTestYoyoWarning= true;
500 void CCDBNodeBranch::clear()
502 // TestYoyo. Track the random crash at exit
503 /*if(AllowTestYoyoWarning)
505 std::string name= getFullName();
506 nlinfo("** clear: %s", name.c_str());
509 vector
<ICDBNode
*>::iterator itNode
;
510 for( itNode
= _Nodes
.begin(); itNode
!= _Nodes
.end(); ++itNode
)
514 //AllowTestYoyoWarning= false;
516 //AllowTestYoyoWarning= true;
519 _NodesByName
.clear();
520 // must remove all branch observers, to avoid any problem in subsequent flushObserversCalls()
521 removeAllBranchObserver();
526 * Find the leaf which count is specified (if found, the returned value is non-null and count is 0)
528 CCDBNodeLeaf
*CCDBNodeBranch::findLeafAtCount( uint
& count
)
530 vector
<ICDBNode
*>::const_iterator itNode
;
531 for ( itNode
= _Nodes
.begin(); itNode
!= _Nodes
.end(); ++itNode
)
533 CCDBNodeLeaf
*leaf
= (*itNode
)->findLeafAtCount( count
);
543 uint
CCDBNodeBranch::countLeaves() const
546 vector
<ICDBNode
*>::const_iterator itNode
;
547 for ( itNode
= _Nodes
.begin(); itNode
!= _Nodes
.end(); ++itNode
)
549 n
+= (*itNode
)->countLeaves();
555 void CCDBNodeBranch::display (const std::string
&prefix
)
557 nlinfo("%sB %s", prefix
.c_str(), _DBSM
->localUnmap(_Name
).c_str());
558 string newPrefix
= " " + prefix
;
559 vector
<ICDBNode
*>::const_iterator itNode
;
560 for ( itNode
= _Nodes
.begin(); itNode
!= _Nodes
.end(); ++itNode
)
562 (*itNode
)->display(newPrefix
);
566 void CCDBNodeBranch::removeNode (const CTextId
& id
)
569 CCDBNodeBranch
*pNode
= dynamic_cast<CCDBNodeBranch
*>(getNode(id
,false));
572 nlwarning("node %s not found", id
.toString().c_str());
575 CCDBNodeBranch
*pParent
= pNode
->_Parent
;
578 nlwarning("parent node not found");
581 // search index node unsorted
583 for (indexNode
= 0; indexNode
< pParent
->_Nodes
.size(); ++indexNode
)
584 if (pParent
->_Nodes
[indexNode
] == pNode
)
586 if (indexNode
== pParent
->_Nodes
.size())
588 nlwarning("node not found");
591 // search index node sorted
593 for (indexSorted
= 0; indexSorted
< pParent
->_NodesByName
.size(); ++indexSorted
)
594 if (pParent
->_NodesByName
[indexSorted
] == pNode
)
596 if (indexSorted
== pParent
->_NodesByName
.size())
598 nlwarning("node not found");
602 // Remove node from parent
603 pParent
->_Nodes
.erase (pParent
->_Nodes
.begin()+indexNode
);
604 pParent
->_NodesByName
.erase (pParent
->_NodesByName
.begin()+indexSorted
);
605 pParent
->_Sorted
= false;
612 void CCDBNodeBranch::onLeafChanged( NLMISC::TStringId leafName
)
614 for( TObserverHandleList::iterator itr
= observerHandles
.begin(); itr
!= observerHandles
.end(); ++itr
)
615 if( (*itr
)->observesLeaf( *leafName
) )
616 (*itr
)->addToFlushableList();
618 if( _Parent
!= NULL
)
619 _Parent
->onLeafChanged( leafName
);
623 //-----------------------------------------------
626 //-----------------------------------------------
627 bool CCDBNodeBranch::addObserver(IPropertyObserver
* observer
,CTextId
& id
)
629 //test if this node is the desired one, if yes, add the observer to all the children nodes
630 if ( id
.getCurrentIndex() == id
.size() )
632 for (uint i
= 0; i
< _Nodes
.size(); ++i
)
634 if (!_Nodes
[i
]->addObserver(observer
,id
))
640 // lookup next element from textid in my index => idx
641 const string
&str
= id
.readNext();
642 ICDBNode
*pNode
= find( str
);
643 // Property not found.
646 nlwarning(" Property %s not found", id
.toString().c_str());
650 // set property in child
651 pNode
->addObserver(observer
,id
);
656 //-----------------------------------------------
659 //-----------------------------------------------
660 bool CCDBNodeBranch::removeObserver(IPropertyObserver
* observer
, CTextId
& id
)
662 //test if this node is the desired one, if yes, remove the observer to all the children nodes
663 if ( id
.getCurrentIndex() == id
.size() )
665 for (uint i
= 0; i
< _Nodes
.size(); ++i
)
667 if (!_Nodes
[i
]->removeObserver(observer
, id
))
673 // lookup next element from textid in my index => idx
674 const string
&str
= id
.readNext();
675 ICDBNode
*pNode
= find( str
);
676 // Property not found.
679 nlwarning(" Property %s not found", id
.toString().c_str());
683 // remove observer in child
684 pNode
->removeObserver(observer
,id
);
687 } // removeObserver //
691 //-----------------------------------------------
692 void CCDBNodeBranch::addBranchObserver( ICDBDBBranchObserverHandle
*handle
, const std::vector
<std::string
>& positiveLeafNameFilter
)
694 CCDBNodeBranch::TObserverHandleList::iterator itr
695 = std::find( observerHandles
.begin(), observerHandles
.end(), handle
);
697 if( itr
!= observerHandles
.end() ){
702 observerHandles
.push_back( handle
);
705 //-----------------------------------------------
706 void CCDBNodeBranch::addBranchObserver( ICDBDBBranchObserverHandle
*handle
, const char *dbPathFromThisNode
, const char **positiveLeafNameFilter
, uint positiveLeafNameFilterSize
)
708 CCDBNodeBranch
*branchNode
;
709 if (dbPathFromThisNode
[0] == '\0') // empty string
715 branchNode
= safe_cast
<CCDBNodeBranch
*>(getNode(ICDBNode::CTextId(dbPathFromThisNode
), false));
716 if( branchNode
== NULL
){
717 std::string msg
= *getName();
719 msg
+= dbPathFromThisNode
;
720 msg
+= " branch missing in DB";
722 nlerror( msg
.c_str() );
727 std::vector
<std::string
> leavesToMonitor(positiveLeafNameFilterSize
);
728 for (uint i
=0; i
!=positiveLeafNameFilterSize
; ++i
)
730 leavesToMonitor
[i
] = string(positiveLeafNameFilter
[i
]);
732 branchNode
->addBranchObserver(handle
, leavesToMonitor
);
735 //-----------------------------------------------
736 void CCDBNodeBranch::removeBranchObserver(const char *dbPathFromThisNode
, ICDBNode::IPropertyObserver
& observer
)
738 CCDBNodeBranch
*branchNode
= safe_cast
<CCDBNodeBranch
*>(getNode(ICDBNode::CTextId(dbPathFromThisNode
), false));
739 if( branchNode
== NULL
){
740 std::string msg
= *getName();
742 msg
+= dbPathFromThisNode
;
743 msg
+= " branch missing in DB";
744 nlerror( msg
.c_str() );
747 branchNode
->removeBranchObserver(&observer
);
751 //-----------------------------------------------
752 bool CCDBNodeBranch::removeBranchObserver(IPropertyObserver
* observer
)
756 TObserverHandleList::iterator itr
= observerHandles
.begin();
757 while( itr
!= observerHandles
.end() )
759 if( (*itr
)->observer() == observer
)
761 (*itr
)->removeFromFlushableList();
763 itr
= observerHandles
.erase( itr
);
775 //-----------------------------------------------
776 void CCDBNodeBranch::removeAllBranchObserver()
778 for( TObserverHandleList::iterator itr
= observerHandles
.begin();
779 itr
!= observerHandles
.end(); ++itr
){
780 (*itr
)->removeFromFlushableList();
784 observerHandles
.clear();
787 //-----------------------------------------------
789 //-----------------------------------------------
790 class CCDBNodeBranchComp
: public std::binary_function
<ICDBNode
*, ICDBNode
*, bool>
793 bool operator()(const ICDBNode
* x
, const ICDBNode
* y
) const
795 return *(x
->getName()) < *(y
->getName());
799 class CCDBNodeBranchComp2
: public std::binary_function
<ICDBNode
*, const string
&, bool>
802 bool operator()(const ICDBNode
* x
, const string
& y
) const
804 return *(x
->getName()) < y
;
809 //-----------------------------------------------
810 ICDBNode
*CCDBNodeBranch::find(const std::string
&nodeName
)
812 #if NL_CDB_OPTIMIZE_PREDICT
813 ICDBNode
*predictNode
= _PredictNode
;
816 if (predictNode
->getParent() == this
817 && *predictNode
->getName() == nodeName
)
827 sort(_NodesByName
.begin(), _NodesByName
.end(), CCDBNodeBranchComp());
830 CCDBNodeLeaf
tmp(nodeName
);
831 vector
<ICDBNode
*>::iterator it
= lower_bound(_NodesByName
.begin(), _NodesByName
.end(), &tmp
, CCDBNodeBranchComp());
832 if (it
== _NodesByName
.end())
836 if (*(*it
)->getName() == nodeName
)
838 #if NL_CDB_OPTIMIZE_PREDICT
839 ICDBNode
*node
= *it
;
853 #ifdef TRACE_READ_DELTA
854 #undef TRACE_READ_DELTA
857 #ifdef TRACE_WRITE_DELTA
858 #undef TRACE_WRITE_DELTA
861 #ifdef TRACE_SET_VALUE
862 #undef TRACE_SET_VALUE