ryzom/server/src/patchman_service/file_repository.cpp

   1 // Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
   2 // Copyright (C) 2010  Winch Gate Property Limited
   3 //
   4 // This program is free software: you can redistribute it and/or modify
   5 // it under the terms of the GNU Affero General Public License as
   6 // published by the Free Software Foundation, either version 3 of the
   7 // License, or (at your option) any later version.
   8 //
   9 // This program is distributed in the hope that it will be useful,
  10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 // GNU Affero General Public License for more details.
  13 //
  14 // You should have received a copy of the GNU Affero General Public License
  15 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 //-----------------------------------------------------------------------------
  18 // includes
  19 //-----------------------------------------------------------------------------
  20
  21 // nel
  22 #include "nel/misc/debug.h"
  23 #include "nel/misc/singleton.h"
  24 #include "nel/misc/variable.h"
  25
  26 // game share
  27 #include "game_share/utils.h"
  28
  29 // local
  30 #include "file_manager.h"
  31 #include "file_repository.h"
  32 #include "patchman_constants.h"
  33
  34
  35 //-------------------------------------------------------------------------------------------------
  36 // namespaces
  37 //-------------------------------------------------------------------------------------------------
  38
  39 using namespace std;
  40 using namespace NLMISC;
  41 using namespace NLNET;
  42 using namespace PATCHMAN;
  43
  44
  45 //-------------------------------------------------------------------------------------------------
  46 // namespace PATCHMAN
  47 //-------------------------------------------------------------------------------------------------
  48
  49 namespace PATCHMAN
  50 {
  51
  52         //-----------------------------------------------------------------------------
  53         // methods CFileRepository
  54         //-----------------------------------------------------------------------------
  55
  56 //      std::string CFileRepository::buildModuleManifest() const
  57 //      {
  58 //              // make sure we've been initialised
  59 ////            nlassert(_Parent!=NULL);
  60 //
  61 //              return "isFileRepository";
  62 //      }
  63         CFileRepository::CFileRepository()
  64         {
  65                 _FileRequestCount= 0;
  66                 _FileInfoCount= 0;
  67
  68                 _MaxHistorySize= 10;
  69                 _FileRequestHistorySize= 0;
  70                 _FileInfoHistorySize= 0;
  71         }
  72
  73         void CFileRepository::init(NLNET::IModule* parent,const NLMISC::CSString& rootDirectory)
  74         {
  75                 CFileRepositorySkel::init(parent);
  76                 _Interceptor.init(this, parent);
  77                 _Parent= parent;
  78                 _Directory= CFileManager::getInstance().getRepositoryDirectory(rootDirectory);
  79                 _AdministeredModuleWrapper.init(dynamic_cast<CAdministeredModuleBase*>(parent));
  80         }
  81
  82         void CFileRepository::onModuleUp(IModuleProxy *module)
  83         {
  84                 // make sure we've been initialised
  85                 nlassert(_Parent!=NULL);
  86
  87                 if (NLMISC::CSString(module->getModuleManifest()).contains(ManifestEntryIsFileReceiver))
  88                 {
  89                         CFileReceiverProxy rr(module);
  90                         rr.setupSubscriptions(_Parent);
  91                         _AdministeredModuleWrapper.registerProgress("Receiver up: "+module->getModuleName());
  92                 }
  93         }
  94
  95         void CFileRepository::onModuleDown(IModuleProxy *module)
  96         {
  97                 // make sure we've been initialised
  98                 nlassert(_Parent!=NULL);
  99
 100                 // if the module has subscribed to listen to stuff then end its subscriptions
 101                 onFileRepositoryModuleDown(module);
 102         }
 103
 104         void CFileRepository::onFileRepositoryModuleDown(IModuleProxy *module)
 105         {
 106                 // make sure we've been initialised
 107                 nlassert(_Parent!=NULL);
 108
 109                 // if the module has subscribed to listen to stuff then end its subscriptions
 110                 unsubscribeAll(module);
 111         }
 112
 113         void CFileRepository::onModuleUpdate()
 114         {
 115                 H_AUTO(CFileRepository_onModuleUpdate);
 116
 117                 // make sure we've been initialised
 118                 nlassert(_Parent!=NULL);
 119
 120                 // update subscriptions
 121                 _broadcastFileInfoChanges(_FileInfoChanges);
 122
 123                 // clear out the changes container (now that it's been treated)
 124                 if (!_FileInfoChanges.empty())
 125                 {
 126                         _AdministeredModuleWrapper.registerProgress(NLMISC::toString("updated %d files",_FileInfoChanges.size()));
 127                         _FileInfoChanges.clear();
 128                 }
 129         }
 130
 131         std::string CFileRepository::buildModuleManifest() const
 132         {
 133                 // make sure we've been initialised
 134                 nlassert(_Parent!=NULL);
 135
 136                 return ManifestEntryIsFileRepository;
 137         }
 138
 139         void CFileRepository::rescanFull()
 140         {
 141                 // make sure we've been initialised
 142                 nlassert(_Parent!=NULL);
 143
 144                 // delegate to the _Directory object
 145                 _Directory->rescanFull(this);
 146         }
 147
 148         void CFileRepository::rescanPartial()
 149         {
 150                 // make sure we've been initialised
 151                 nlassert(_Parent!=NULL);
 152
 153                 // delegate to the _Directory object
 154                 _Directory->rescanPartial(this);
 155         }
 156
 157         void CFileRepository::updateFile(const NLMISC::CSString& fileName)
 158         {
 159                 // make sure we've been initialised
 160                 nlassert(_Parent!=NULL);
 161
 162                 // delegate to the _Directory object
 163                 _Directory->updateFile(fileName,SFileInfo::RECALCULATE_IF_CHANGED,this);
 164         }
 165
 166         void CFileRepository::getFileInfo(const NLMISC::CSString& fileSpec,TFileInfoVector& result,const NLNET::IModuleProxy *sender) const
 167         {
 168                 // make sure we've been initialised
 169                 nlassert(_Parent!=NULL);
 170
 171                 // delegate to the _Directory object
 172                 _Directory->getFileInfo(fileSpec,result,const_cast<CFileRepository*>(this),sender);
 173         }
 174
 175         TRepositoryDirectoryPtr CFileRepository::getRepositoryDirectory()
 176         {
 177                 return _Directory;
 178         }
 179
 180         void CFileRepository::getFile(const NLMISC::CSString& fileName,NLMISC::CSString& resultData,const NLNET::IModuleProxy* sender) const
 181         {
 182                 // make sure we've been initialised
 183                 nlassert(_Parent!=NULL);
 184
 185                 // clear out the result container before we begin...
 186                 resultData.clear();
 187
 188                 // delegate to the _Directory object
 189                 _Directory->getFile(fileName,resultData,const_cast<CFileRepository*>(this),sender);
 190         }
 191
 192         void CFileRepository::dump(NLMISC::CLog& log)
 193         {
 194                 // make sure we've been initialised
 195                 nlassert(_Parent!=NULL);
 196
 197                 log.displayNL("-----------------------------------");
 198                 log.displayNL("Recent Info requests (%u in all)",_FileInfoCount);
 199                 log.displayNL("-----------------------------------");
 200                 for (THistory::iterator it=_FileInfoHistory.begin(); it!=_FileInfoHistory.end();++it)
 201                 {
 202                         log.displayNL("  '%s'",it->c_str());
 203                 }
 204                 log.displayNL("-----------------------------------");
 205                 log.displayNL("Recent Download requests (%u in all)",_FileRequestCount);
 206                 log.displayNL("-----------------------------------");
 207                 for (THistory::iterator it=_FileRequestHistory.begin(); it!=_FileRequestHistory.end();++it)
 208                 {
 209                         log.displayNL("  '%s'",it->c_str());
 210                 }
 211                 log.displayNL("-----------------------------------");
 212                 log.displayNL("Active Subscriptions");
 213                 log.displayNL("-----------------------------------");
 214                 for (TSubscribers::iterator it=_Subscribers.begin(); it!=_Subscribers.end();++it)
 215                 {
 216                         log.displayNL("  '%s'",it->first.c_str());
 217                 }
 218         }
 219
 220         void CFileRepository::setMaxHistorySize(uint32 maxHistorySize)
 221         {
 222                 // make sure we've been initialised
 223                 nlassert(_Parent!=NULL);
 224
 225                 _MaxHistorySize= maxHistorySize;
 226
 227                 // if the new limit is bigger than the request history size then prune down the history list
 228                 while (_FileRequestHistorySize > _MaxHistorySize)
 229                 {
 230                         _FileRequestHistory.pop_back();
 231                         --_FileRequestHistorySize;
 232                 }
 233
 234                 // if the new limit is bigger than the info history size then prune down the history list
 235                 while (_FileInfoHistorySize > _MaxHistorySize)
 236                 {
 237                         _FileInfoHistory.pop_front();
 238                         --_FileInfoHistorySize;
 239                 }
 240         }
 241
 242         uint32 CFileRepository::getMaxHistorySize() const
 243         {
 244                 // make sure we've been initialised
 245                 nlassert(_Parent!=NULL);
 246
 247                 return _MaxHistorySize;
 248         }
 249
 250         void CFileRepository::_broadcastFileInfoChanges(const TFileInfoVector& fileInfoChanges)
 251         {
 252                 // make sure we've been initialised
 253                 nlassert(_Parent!=NULL);
 254
 255                 // update subscriptions
 256                 for (TSubscribers::iterator sit=_Subscribers.begin(); sit!=_Subscribers.end();++sit)
 257                 {
 258                         // setup a filespec object for this subscriber
 259                         CFileSpec fileSpec(sit->first.splitFrom('@').strip());
 260
 261                         // build a vector of files that matches the subscription's request
 262                         TFileInfoVector infoVector;
 263
 264                         // run through the changes that we've accumulated since the last update...
 265                         for (TFileInfoVector::const_iterator cit=fileInfoChanges.begin(); cit!=fileInfoChanges.end();++cit)
 266                         {
 267                                 // if this change is for a file that matches the subscriber's filespec then addit to our result vector
 268                                 if (fileSpec.matches(cit->FileName) && cbValidateFileInfoRequest(sit->second,cit->FileName))
 269                                 {
 270                                         infoVector.push_back(*cit);
 271                                 }
 272                         }
 273
 274                         // if we found changes that interest this subscriber then dispatch them
 275                         if (!infoVector.empty())
 276                         {
 277                                 CFileReceiverProxy client(sit->second);
 278                                 client.cbFileInfo(_Parent,infoVector);
 279                         }
 280                 }
 281         }
 282
 283         void CFileRepository::requestFileInfo(NLNET::IModuleProxy *sender,const NLMISC::CSString& fileSpec)
 284         {
 285                 // make sure we've been initialised
 286                 nlassert(_Parent!=NULL);
 287
 288                 // setup an object to receive the file info and fill it in
 289                 TFileInfoVector result;
 290                 getFileInfo(fileSpec,result,sender);
 291
 292                 // get hold of a proxy for the module who sent the request and return the file info to them
 293                 CFileReceiverProxy fr(sender);
 294                 fr.cbFileInfo(_Parent,result);
 295
 296                 // update our stats
 297                 ++_FileInfoCount;
 298
 299                 // add our new history record
 300                 _FileInfoHistory.push_front((sender==NULL?"<local> ":"<"+sender->getModuleName()+"> ")+fileSpec);
 301                 ++_FileInfoHistorySize;
 302
 303                 // if we're grown too big then prune the oldest entry
 304                 if (_FileInfoHistorySize>_MaxHistorySize)
 305                 {
 306                         _FileInfoHistory.pop_back();
 307                         --_FileInfoHistorySize;
 308                 }
 309
 310                 // deal with progress / state info
 311                 _AdministeredModuleWrapper.registerProgress("File info request: "+sender->getModuleName()+" "+fileSpec);
 312                 _AdministeredModuleWrapper.setStateVariable("InfoReq",NLMISC::toString(_FileInfoCount));
 313         }
 314
 315         void CFileRepository::requestFileData(NLNET::IModuleProxy *sender, const NLMISC::CSString &fileName, uint32 startOffset, uint32 numBytes)
 316         {
 317                 // make sure we've been initialised
 318                 nlassert(_Parent!=NULL);
 319
 320                 // load the file chunk that's being requested
 321                 CSString result;
 322                 bool ok;
 323
 324                 // allow the overloadable validation callback a chance to prohibit read
 325                 ok= cbValidateDownloadRequest(sender,fileName);
 326
 327                 // load the file (if validation was favorable)
 328                 CSString fullFileName= _Directory->getRootDirectory()+fileName;
 329                 ok&= CFileManager::getInstance().load(fullFileName,startOffset,numBytes,result);
 330
 331                 // get hold of a proxy for the module who sent the request and return the file info to them
 332                 CFileReceiverProxy rr(sender);
 333                 if (ok && !result.empty())
 334                 {
 335                         rr.cbFileData(_Parent,fileName,startOffset,NLNET::TBinBuffer((const uint8 *)&result[0],(uint32)result.size()));
 336                 }
 337                 else
 338                 {
 339                         rr.cbFileDataFailure(_Parent,fileName);
 340                 }
 341
 342                 // update our stats
 343                 ++_FileRequestCount;
 344
 345                 // add our new history record
 346                 _FileRequestHistory.push_front((sender==NULL?"<local> ":"<"+sender->getModuleName()+"> ")+fileName);
 347                 ++_FileRequestHistorySize;
 348
 349                 // if we're grown too big then prune the oldest entry
 350                 if (_FileRequestHistorySize>_MaxHistorySize)
 351                 {
 352                         _FileRequestHistory.pop_back();
 353                         --_FileRequestHistorySize;
 354                 }
 355
 356                 // deal with progress / state info
 357                 uint32 fileLen=CFileManager::getInstance().getFileSize(fullFileName);
 358                 _AdministeredModuleWrapper.registerProgress("File data: "+sender->getModuleName()+" "+fileName+NLMISC::toString("(%d..%d/%d)",startOffset,startOffset+numBytes,fileLen));
 359                 _AdministeredModuleWrapper.setStateVariable("DataReq",NLMISC::toString(_FileRequestCount));
 360
 361                 // set state variables to reflect 'end of file reached'
 362                 if (fileLen<startOffset+numBytes)
 363                 {
 364                         _AdministeredModuleWrapper.registerError("Invalid request from: "+sender->getModuleName()+" for: '"+fileName+"'");
 365                         _AdministeredModuleWrapper.registerProgress("Failed file: "+fileName);
 366                 }
 367                 else if (fileLen==startOffset+numBytes)
 368                 {
 369                         _AdministeredModuleWrapper.registerProgress("Finished file: "+fileName);
 370                 }
 371         }
 372
 373         void CFileRepository::subscribe(NLNET::IModuleProxy *sender, const NLMISC::CSString &fileSpec)
 374         {
 375                 // make sure we've been initialised
 376                 nlassert(_Parent!=NULL);
 377
 378                 // build the string that we'll use to represent this subscription
 379                 NLMISC::CSString subscriptionString= sender->getModuleName()+" @ "+fileSpec;
 380
 381                 // make sure the entry doesn't exist in our subscribers set
 382                 DROP_IF(_Subscribers.find(subscriptionString)!=_Subscribers.end(),"Ignoring dumplicate request for the same subscription: '"+subscriptionString+'\'',return);
 383
 384                 // display a fancy message
 385                 _AdministeredModuleWrapper.registerProgress("Subscribe '"+subscriptionString+"'");
 386
 387                 // get hold of the info on the requested files as it stands right now
 388                 TFileInfoVector fileInfoVector;
 389                 getFileInfo(fileSpec,fileInfoVector,sender);
 390
 391                 // dispatch the info to the sender
 392                 CFileReceiverProxy client(sender);
 393                 client.cbFileInfo(_Parent,fileInfoVector);
 394
 395                 // add this entry to our subscriptions set
 396                 _Subscribers[subscriptionString]= sender;
 397         }
 398
 399         void CFileRepository::unsubscribe(NLNET::IModuleProxy *sender, const NLMISC::CSString &fileSpec)
 400         {
 401                 // make sure we've been initialised
 402                 nlassert(_Parent!=NULL);
 403
 404                 // build the string that we'll use to represent this subscription
 405                 NLMISC::CSString subscriptionString= sender->getModuleName()+" @ "+fileSpec;
 406
 407                 // display a fancy message
 408                 _AdministeredModuleWrapper.registerProgress("Unsubscribe '"+subscriptionString+"'");
 409
 410                 // make sure that this subscription really exists
 411                 DROP_IF(_Subscribers.find(subscriptionString)==_Subscribers.end(),"Ignoring unsubscribe for unknown subscription: '"+subscriptionString+'\'',return);
 412
 413                 // remove the subscription
 414                 _Subscribers.erase(subscriptionString);
 415         }
 416
 417         void CFileRepository::unsubscribeAll(NLNET::IModuleProxy *sender)
 418         {
 419                 // make sure we've been initialised
 420                 nlassert(_Parent!=NULL);
 421
 422                 // build the string that this module's subscriptions will contain
 423                 NLMISC::CSString subscriptionString= sender->getModuleName()+" @ ";
 424
 425                 // iterate over our subscriptions looking for matches to erase
 426                 TSubscribers::iterator last;
 427                 TSubscribers::iterator it;
 428                 for (it=_Subscribers.begin();it!=_Subscribers.end();)
 429                 {
 430                         // take a copy of the iterator and increment to point at the next element to be processed
 431                         last=it;
 432                         ++it;
 433
 434                         // see if the last element needs to be erased
 435                         if (last->second==sender)
 436                         {
 437                                 // delegate to standard 'unsubscribe' to do the work
 438                                 unsubscribe(sender,last->first.splitFrom('@').strip());
 439                         }
 440                 }
 441         }
 442
 443         void CFileRepository::getInfo(NLNET::IModuleProxy *sender, const NLMISC::CSString &fileSpec)
 444         {
 445                 // make sure we've been initialised
 446                 nlassert(_Parent!=NULL);
 447
 448                 _AdministeredModuleWrapper.registerProgress("getInfo "+sender->getModuleName()+" "+fileSpec);
 449                 subscribe(sender,fileSpec);
 450                 unsubscribe(sender,fileSpec);
 451         }
 452
 453         void CFileRepository::cbFileInfoUpdate(const SFileInfo& fileInfo)
 454         {
 455                 // make sure we've been initialised
 456                 nlassert(_Parent!=NULL);
 457
 458                 _AdministeredModuleWrapper.registerProgress(NLMISC::toString("cbFileInfoUpdate: %s (%d bytes)",fileInfo.FileName.c_str(),fileInfo.FileSize));
 459
 460                 // add an entry to our changes list
 461                 _FileInfoChanges.push_back(fileInfo);
 462         }
 463
 464         void CFileRepository::cbFileInfoErased(const NLMISC::CSString& fileName)
 465         {
 466                 // make sure we've been initialised
 467                 nlassert(_Parent!=NULL);
 468
 469                 _AdministeredModuleWrapper.registerProgress("cbFileInfoErased: "+fileName);
 470
 471                 // add an entry to our changes list (this is an empty entry with just the filename set to represent a deleted file)
 472                 SFileInfo fileInfo;
 473                 fileInfo.FileName= fileName;
 474                 _FileInfoChanges.push_back(fileInfo);
 475         }
 476
 477         NLMISC_CLASS_COMMAND_IMPL(CFileRepository, incRescan)
 478         {
 479                 // make sure we've been initialised
 480                 nlassert(_Parent!=NULL);
 481
 482                 if (args.size()!=0)
 483                         return false;
 484
 485                 CFileRepository::rescanPartial();
 486
 487                 return true;
 488         }
 489
 490         NLMISC_CLASS_COMMAND_IMPL(CFileRepository, fullRescan)
 491         {
 492                 // make sure we've been initialised
 493                 nlassert(_Parent!=NULL);
 494
 495                 if (args.size()!=0)
 496                         return false;
 497
 498                 CFileRepository::rescanFull();
 499
 500                 return true;
 501         }
 502
 503         NLMISC_CLASS_COMMAND_IMPL(CFileRepository, getFile)
 504         {
 505                 // make sure we've been initialised
 506                 nlassert(_Parent!=NULL);
 507
 508                 if (args.size()!=1)
 509                         return true;
 510
 511                 NLMISC::CSString data;
 512                 CFileRepository::getFile(args[0],data,NULL);
 513                 log.displayNL("Retrieved %u bytes of data for file %s, starting: %s",data.size(), args[0].c_str(), data.left(20).quote().c_str());
 514
 515                 return true;
 516         }
 517
 518         NLMISC_CLASS_COMMAND_IMPL(CFileRepository, getFileInfo)
 519         {
 520                 // make sure we've been initialised
 521                 nlassert(_Parent!=NULL);
 522
 523                 // make sure a filespec was given
 524                 if (args.size()!=1)
 525                         return true;
 526
 527                 // lookup the set of files
 528                 TFileInfoVector fileInfoVector;
 529                 getFileInfo(args[0],fileInfoVector,NULL);
 530
 531                 // display a summary info message
 532                 log.displayNL("Result of info request '%s': %d matches",args[0].c_str(),fileInfoVector.size());
 533                 log.displayNL("- %-32s %10s %10s %s","checksum","time","size","name");
 534
 535                 // iterate over results, displaying the info
 536                 for (TFileInfoVector::iterator it= fileInfoVector.begin(); it!= fileInfoVector.end(); ++it)
 537                 {
 538                         log.displayNL("- %-32s %10u %10u %s",it->Checksum.toString().c_str(),it->FileTime,it->FileSize,it->FileName.c_str());
 539                 }
 540                 return true;
 541         }
 542
 543         NLMISC_CLASS_COMMAND_IMPL(CFileRepository, updateFile)
 544         {
 545                 // make sure we've been initialised
 546                 nlassert(_Parent!=NULL);
 547
 548                 if (args.size()!=1)
 549                         return true;
 550
 551                 CFileRepository::updateFile(args[0]);
 552
 553                 return true;
 554         }
 555
 556         NLMISC_CLASS_COMMAND_IMPL(CFileRepository, MaxHistorySize)
 557         {
 558                 // make sure we've been initialised
 559                 nlassert(_Parent!=NULL);
 560
 561                 switch(args.size())
 562                 {
 563                 case 1:
 564                         {
 565                                 uint32 newVal= NLMISC::CSString(args[0]).atoui();
 566                                 if (newVal==0 && args[0]!="0")
 567                                         break;
 568
 569                                 CFileRepository::setMaxHistorySize(newVal);
 570                         }
 571                         // drop through...
 572
 573                 case 0:
 574                         log.displayNL("MaxHistorySize %u",_MaxHistorySize);
 575                         return true;
 576                 }
 577
 578                 return false;
 579         }
 580
 581 } // end of namespace