ryzom/server/src/patchman_service/file_manager.cpp

   1 // Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
   2 // Copyright (C) 2010  Winch Gate Property Limited
   3 //
   4 // This program is free software: you can redistribute it and/or modify
   5 // it under the terms of the GNU Affero General Public License as
   6 // published by the Free Software Foundation, either version 3 of the
   7 // License, or (at your option) any later version.
   8 //
   9 // This program is distributed in the hope that it will be useful,
  10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 // GNU Affero General Public License for more details.
  13 //
  14 // You should have received a copy of the GNU Affero General Public License
  15 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 //-----------------------------------------------------------------------------
  18 // includes
  19 //-----------------------------------------------------------------------------
  20
  21 // nel
  22 #include "nel/misc/variable.h"
  23 #include "nel/misc/file.h"
  24 #include "nel/misc/common.h"
  25
  26 // game share
  27 #include "game_share/utils.h"
  28
  29 // local
  30 #include "file_manager.h"
  31
  32
  33 //-------------------------------------------------------------------------------------------------
  34 // namespaces
  35 //-------------------------------------------------------------------------------------------------
  36
  37 using namespace std;
  38 using namespace NLMISC;
  39 //using namespace NLNET;
  40 using namespace PATCHMAN;
  41
  42
  43 //-----------------------------------------------------------------------------
  44 // some NLMISC Variable
  45 //-----------------------------------------------------------------------------
  46
  47 NLMISC::CVariable<uint32> FileCacheSize("patchman","FileCacheSize","Minimum size for re file cache",100*1024*1024,0,true);
  48
  49
  50 namespace PATCHMAN
  51 {
  52         //-----------------------------------------------------------------------------
  53         // handy utils
  54         //-----------------------------------------------------------------------------
  55
  56         static NLMISC::CSString getTempFileName(const NLMISC::CSString& fileName)
  57         {
  58                 return fileName+".download.tmp";
  59         }
  60
  61         static NLMISC::CSString getIndexFileName(const NLMISC::CSString& rootDirectory)
  62         {
  63                 return rootDirectory+".patchman.file_index";
  64         }
  65
  66
  67         //-----------------------------------------------------------------------------
  68         // methods SFileInfo
  69         //-----------------------------------------------------------------------------
  70
  71         bool SFileInfo::updateFileInfo(const NLMISC::CSString& fileName,const NLMISC::CSString& fullFileName, SFileInfo::TUpdateMethod updateMethod, IFileInfoUpdateListener* updateListener)
  72         {
  73                 // if file doesn't exist then just drop out
  74                 if (!NLMISC::CFile::fileExists(fullFileName))
  75                 {
  76                         clear();
  77                         return false;
  78                 }
  79
  80                 // give ourselves 5 attempts in case the file is being accessed...
  81                 for(uint32 i=0;i<5;++i)
  82                 {
  83                         try
  84                         {
  85                                 uint32 newFileSize= NLMISC::CFile::getFileSize(fullFileName);
  86                                 uint32 newFileTime= NLMISC::CFile::getFileModificationDate(fullFileName);
  87
  88                                 // work out whether the record has changed (based on size and time stamp)
  89                                 bool changed= ( (FileTime!=newFileTime) || (FileSize!=newFileSize) );
  90
  91                                 // note: it is possible to hit an exception in the checksum calculation if the file is being accessed
  92                                 // by someone else
  93                                 if (updateMethod==FORCE_RECALCULATE || (updateMethod==RECALCULATE_IF_CHANGED  && (newFileSize!=FileSize || newFileTime!=FileTime)))
  94                                 {
  95                                         // call the updateListener object's callback so that they can write a log, etc
  96                                         if (updateListener!=NULL)
  97                                         {
  98                                                 updateListener->cbFileInfoRescanning(fileName,newFileSize);
  99                                         }
 100
 101                                         // workout the new checksum and update the 'changed' flag accordingly
 102                                         CHashKeyMD5 newChecksum= NLMISC::getMD5(fullFileName);
 103                                         changed|= (Checksum!=newChecksum);
 104                                         Checksum= newChecksum;
 105                                 }
 106
 107                                 // update the fields in our new record
 108                                 FileName= fileName;
 109                                 FileSize= newFileSize;
 110                                 FileTime= newFileTime;
 111
 112                                 // if there's an update listener object, then let them know about the file update
 113                                 if (changed && updateListener!=NULL)
 114                                 {
 115                                         updateListener->cbFileInfoUpdate(*this);
 116                                 }
 117
 118                                 // return true if we are unchanged, otherwise false
 119                                 return !changed;
 120                         }
 121                         catch(...)
 122                         {
 123                                 nlwarning("Exception thrown in getMD5(\"%s\") ... will try again in a few seconds",fullFileName.c_str());
 124                                 nlSleep(5);
 125                         }
 126                 }
 127
 128                 nlwarning("Failed to get info on file: %s",fullFileName.c_str());
 129                 clear();
 130                 return false;
 131         }
 132
 133
 134         //-----------------------------------------------------------------------------
 135         // methods CFileSpec
 136         //-----------------------------------------------------------------------------
 137
 138         CFileSpec::CFileSpec()
 139         {
 140         }
 141
 142         CFileSpec::CFileSpec(const NLMISC::CSString& fileSpec)
 143         {
 144                 _NameSpec= NLMISC::CFile::getFilename(fileSpec);
 145                 _PathSpec= NLMISC::CFile::getPath(fileSpec);
 146
 147                 _NameIsWild= _NameSpec.contains('*') || _NameSpec.contains('?');
 148                 _PathIsWild= _PathSpec.contains('*') || _PathSpec.contains('?');
 149
 150                 _AcceptAllNames= (_NameSpec=="*");
 151                 _AcceptAllPaths= (_PathSpec=="*/");
 152         }
 153
 154         bool CFileSpec::matches(const NLMISC::CSString& fullFileName) const
 155         {
 156                 CSString name= NLMISC::CFile::getFilename(fullFileName);
 157                 CSString path= NLMISC::CFile::getPath(fullFileName);
 158
 159                 return (_AcceptAllPaths || pathMatches(path))  && (_AcceptAllNames || nameMatches(name));
 160         }
 161
 162         bool CFileSpec::nameMatches(const NLMISC::CSString& fileName) const
 163         {
 164                 return _AcceptAllNames || ( _NameIsWild ? testWildCard(fileName,_NameSpec) : (fileName==_NameSpec) );
 165         }
 166
 167         bool CFileSpec::pathMatches(const NLMISC::CSString& path) const
 168         {
 169                 // treat the special case where the file that we're set to accept all paths
 170                 if (_AcceptAllPaths)
 171                         return true;
 172
 173                 // treat the special case where the file that we're testing has no path and we
 174                 // are looking for files in the root directory only
 175                 if (path.empty() && (_PathSpec=="./") )
 176                         return true;
 177
 178                 return _PathIsWild ? testWildCard(path,_PathSpec) : (path==_PathSpec);
 179         }
 180
 181         NLMISC::CSString CFileSpec::toString() const
 182         {
 183                 return _PathSpec+_NameSpec;
 184         }
 185
 186         const CSString& CFileSpec::nameSpec() const
 187         {
 188                 return _NameSpec;
 189         }
 190
 191         const CSString& CFileSpec::pathSpec() const
 192         {
 193                 return _PathSpec;
 194         }
 195
 196         bool CFileSpec::isWild() const
 197         {
 198                 return _NameIsWild || _PathIsWild;
 199         }
 200
 201         bool CFileSpec::nameIsWild() const
 202         {
 203                 return _NameIsWild;
 204         }
 205
 206         bool CFileSpec::pathIsWild() const
 207         {
 208                 return _PathIsWild;
 209         }
 210
 211
 212
 213         //-----------------------------------------------------------------------------
 214         // methods CRepositoryDirectory
 215         //-----------------------------------------------------------------------------
 216
 217         CRepositoryDirectory::CRepositoryDirectory(const NLMISC::CSString& path)
 218         {
 219                 _Root= NLMISC::CPath::getFullPath(path);
 220                 _IndexFileIsUpToDate= false;
 221         }
 222
 223         void CRepositoryDirectory::clear()
 224         {
 225                 // clear out our directories map
 226                 _DirectoryTree.clear();
 227         }
 228
 229         void CRepositoryDirectory::rescanFull(IFileInfoUpdateListener* updateListener)
 230         {
 231                 // rescan our directories recursively, starting at the root
 232                 _rescanDirectory("",true,updateListener);
 233
 234                 // update the index file as required
 235                 if (!_IndexFileIsUpToDate)
 236                         writeIndex();
 237         }
 238
 239         void CRepositoryDirectory::rescanPartial(IFileInfoUpdateListener* updateListener)
 240         {
 241                 // if the directory tree is empty then start with the rooot directory...
 242                 if (_DirectoryTree.empty())
 243                 {
 244                         _rescanDirectory("",false,updateListener);
 245                         return;
 246                 }
 247
 248                 // try to get hold of a ref to the last directory that we rescanned
 249                 TDirectoryTree::iterator it= _DirectoryTree.find(_LastRescan);
 250                 if (it==_DirectoryTree.end())
 251                 {
 252                         // we failed to get a ref to the last directory so wrap back round to the start
 253                         it= _DirectoryTree.begin();
 254                 }
 255                 else
 256                 {
 257                         // increement our iterator to get hold of a ref to the next directory in the map
 258                         ++it;
 259                         // if we reach end of map then wrap back to the start
 260                         if (it==_DirectoryTree.end())
 261                         {
 262                                 it= _DirectoryTree.begin();
 263                         }
 264                 }
 265
 266                 // scan the next directory in our map (this is not recursive
 267                 _rescanDirectory(it->first,false,updateListener);
 268                 _LastRescan= it->first;
 269
 270                 // update the index file as required
 271                 if (!_IndexFileIsUpToDate)
 272                         writeIndex();
 273         }
 274
 275         void CRepositoryDirectory::updateFile(const NLMISC::CSString& fileName,SFileInfo::TUpdateMethod updateMethod, IFileInfoUpdateListener* updateListener)
 276         {
 277                 // if the file doesn't exist then give up
 278                 if (!NLMISC::CFile::fileExists(_Root+fileName))
 279                         return;
 280
 281                 // split the file name into path and fileName
 282                 const NLMISC::CSString path= NLMISC::CFile::getPath(fileName);
 283
 284                 // get hold of the directory object for this file (or create a new one if need be)
 285                 TFileInfoMap& directory= _DirectoryTree[path];
 286
 287                 // get hold of the file info object for this (or create a new one if need be)
 288                 SFileInfo& fileInfo= directory[fileName];
 289
 290                 // update the file info for the given file
 291                 _IndexFileIsUpToDate&= fileInfo.updateFileInfo(fileName,_Root+fileName,updateMethod,updateListener);
 292         }
 293
 294         // query methods
 295         void CRepositoryDirectory::getFileInfo(const NLMISC::CSString& fileSpec,TFileInfoVector& result,IFileRequestValidator* validator,const NLNET::IModuleProxy *sender) const
 296         {
 297                 // split into directory and file name
 298                 const CFileSpec spec(fileSpec);
 299
 300                 // setup a set of paths to scan
 301                 std::vector<TDirectoryTree::const_iterator> paths;
 302
 303                 // if the path has wirlcards in it then do a search for wildcard matches otherwise just do a lookup
 304                 if (spec.pathIsWild())
 305                 {
 306                         for (TDirectoryTree::const_iterator it=_DirectoryTree.begin();  it!=_DirectoryTree.end(); ++it)
 307                         {
 308                                 if (spec.pathMatches(it->first))
 309                                         paths.push_back(it);
 310                         }
 311                 }
 312                 else
 313                 {
 314                         TDirectoryTree::const_iterator it= _DirectoryTree.find(spec.pathSpec());
 315                         if (it != _DirectoryTree.end())
 316                                 paths.push_back(it);
 317                 }
 318
 319                 // run through the selected paths looking for files
 320                 for (uint32 i=0;i<paths.size();++i)
 321                 {
 322                         // compose the file name that we're supposed to match
 323                         NLMISC::CSString fullFilePattern= paths[i]->first+spec.nameSpec();
 324
 325                         // if the filename doesn't have wildcards then just do a straight lookup...
 326                         if (!spec.nameIsWild())
 327                         {
 328                                 // see whether we have a match...
 329                                 TFileInfoMap::const_iterator fit= paths[i]->second.find(fullFilePattern);
 330                                 if (fit!=paths[i]->second.end())
 331                                 {
 332                                         // call the overloadable validation callback before adding the file info record to the result container
 333                                         if (validator==NULL || validator->cbValidateFileInfoRequest(sender,fit->second.FileName))
 334                                         {
 335                                                 result.push_back(fit->second);
 336                                         }
 337                                 }
 338                                 continue;
 339                         }
 340
 341                         // run through the files in the given path
 342                         for (TFileInfoMap::const_iterator fit= paths[i]->second.begin(); fit!= paths[i]->second.end(); ++fit)
 343                         {
 344                                 // get hold of the file name for this map entry
 345                                 const NLMISC::CSString& name=fit->second.FileName;
 346
 347                                 // do either a wildcard compare or a quick and dirty string compare
 348                                 if (testWildCard(name,fullFilePattern))
 349                                 {
 350                                         // call the overloadable validation callback before adding the file info record to the result container
 351                                         if (validator==NULL || validator->cbValidateFileInfoRequest(sender,name))
 352                                         {
 353                                                 result.push_back(fit->second);
 354                                         }
 355                                 }
 356                         }
 357                 }
 358         }
 359
 360         void CRepositoryDirectory::getFile(const NLMISC::CSString& fileName,NLMISC::CSString& resultData,IFileRequestValidator* validator,const NLNET::IModuleProxy *sender) const
 361         {
 362                 // start by clearing out the result container...
 363                 resultData.clear();
 364
 365                 // allow the overloadable validation callback a chance to prohibit read
 366                 if (validator!=NULL && !validator->cbValidateDownloadRequest(sender,fileName))
 367                         return;
 368
 369                 // if the file exists then go ahead and read it
 370                 NLMISC::CSString fullFileName= _Root+fileName;
 371                 if (NLMISC::CFile::fileExists(fullFileName))
 372                 {
 373                         resultData.readFromFile(fullFileName);
 374                 }
 375         }
 376
 377         const NLMISC::CSString &CRepositoryDirectory::getRootDirectory() const
 378         {
 379                 return _Root;
 380         }
 381
 382         bool CRepositoryDirectory::readIndex()
 383         {
 384                 // start by clearing out our containers
 385                 clear();
 386
 387                 // make sure the file exists (return false if not found)
 388                 NLMISC::CSString indexFileName= getIndexFileName(_Root);
 389                 if (!NLMISC::CFile::fileExists(indexFileName))
 390                         return false;
 391
 392                 // read the file contents
 393                 NLMISC::CSString index;
 394                 index.readFromFile(indexFileName);
 395                 DROP_IF(index.empty(),"Failed to read data from index file: "+indexFileName,return false);
 396
 397                 nlinfo("CRepositoryDirectory_Reading index file: %s",indexFileName.c_str());
 398                 NLMISC::CVectorSString lines;
 399                 index.splitLines(lines);
 400                 for (uint32 i=0;i<lines.size();++i)
 401                 {
 402                         // get hold of the line and strip off comments and spurious blanks
 403                         NLMISC::CSString line= lines[i].splitTo("//").strip();
 404                         if (line.empty()) continue;
 405
 406                         // break the line down into constituent parts
 407                         SFileInfo fileInfo;
 408                         fileInfo.FileSize= line.strtok(",").strip().atoi();
 409                         fileInfo.FileTime= line.strtok(",").strip().atoi();
 410                         fileInfo.Checksum.fromString(line.strtok(",").strip());
 411                         fileInfo.FileName= line.strip();
 412
 413                         // make sure that the text in the line was valid and that the file on disk looks like the one in our record
 414                         DROP_IF(fileInfo.FileName.empty(),"Skipping line due to parse error: "+lines[i],continue);
 415
 416                         // add the result to one of our maps of files
 417                         NLMISC::CSString path= NLMISC::CFile::getPath(fileInfo.FileName);
 418                         _DirectoryTree[path][fileInfo.FileName]=fileInfo;
 419                 }
 420                 nlinfo("%d entries read from index file",lines.size());
 421                 _IndexFileIsUpToDate= true;
 422                 return true;
 423         }
 424
 425         bool CRepositoryDirectory::writeIndex() const
 426         {
 427                 // get hold of the file name for the index file
 428                 NLMISC::CSString indexFileName= getIndexFileName(_Root);
 429 //              nldebug("Flushing changes to index file: %s",indexFileName.c_str());
 430
 431                 // setup a text buffer to build our output file in
 432                 NLMISC::CSString outputText= "// Index file for patchman service\n// *FORMAT*: size, time, checksum, filename\n\n";
 433
 434                 // iterate over our directories and their files, building entries
 435                 for (TDirectoryTree::const_iterator dit= _DirectoryTree.begin(); dit!= _DirectoryTree.end(); ++dit)
 436                 {
 437                         const TFileInfoMap& theDirectory= dit->second;
 438                         for (TFileInfoMap::const_iterator fit= theDirectory.begin(); fit!=theDirectory.end(); ++fit)
 439                         {
 440                                 const SFileInfo& theInfo= fit->second;
 441                                 outputText+= NLMISC::toString("%10u,%12u, %s, %s\n",theInfo.FileSize,theInfo.FileTime,theInfo.Checksum.toString().c_str(),theInfo.FileName.c_str());
 442                         }
 443
 444                 }
 445
 446                 // write the resulting buffer to disk
 447                 _IndexFileIsUpToDate= outputText.writeToFile(indexFileName);
 448                 return _IndexFileIsUpToDate;
 449         }
 450
 451         void CRepositoryDirectory::_rescanDirectory(const NLMISC::CSString& directoryName, bool recurse, IFileInfoUpdateListener* updateListener)
 452         {
 453 //              nldebug("VERBOSE_Scanning directory: root=%s directory=%s",_Root.c_str(),directoryName.c_str());
 454
 455                 // make sure we exist in the '_DirectoryTree' map (and get a handle to it)
 456                 TFileInfoMap& theDirectory= _DirectoryTree[directoryName];
 457
 458                 // first scan for directories
 459                 std::vector<std::string> pathContents;
 460                 NLMISC::CPath::getPathContent(_Root+directoryName,false,true,false,pathContents);
 461
 462                 // run through the directories we found...
 463                 for (uint32 i=(uint32)pathContents.size();i--;)
 464                 {
 465                         NLMISC::CSString childDirectoryName= NLMISC::CSString(pathContents[i]).leftCrop((uint32)_Root.size());
 466
 467                         // make sure they exist in the '_DirectoryTree' map
 468                         _DirectoryTree[childDirectoryName];
 469
 470                         // if we're recursing then go for it
 471                         if (recurse)
 472                         {
 473                                 _rescanDirectory(childDirectoryName,recurse,updateListener);
 474                         }
 475                 }
 476
 477                 // run through all of the files in our map flagging them as 'not updated'
 478                 for (TFileInfoMap::iterator fit= theDirectory.begin(); fit!= theDirectory.end(); ++fit)
 479                 {
 480                         fit->second.FileName.clear();
 481                 }
 482
 483                 // now scan for files
 484                 pathContents.clear();
 485                 NLMISC::CPath::getPathContent(_Root+directoryName,false,false,true,pathContents);
 486
 487                 // run through the files adding them to ourself
 488                 for (uint32 i=(uint32)pathContents.size();i--;)
 489                 {
 490                         // if the file is system file then skip it
 491                         if (pathContents[i].find("/.")!=std::string::npos)
 492                                 continue;
 493
 494                         // construct the file name
 495                         NLMISC::CSString fileName= NLMISC::CSString(pathContents[i]).leftCrop((uint32)_Root.size());
 496                         // get hold of the directory entry for this file (or create a new one if not exist) and update it
 497                         _IndexFileIsUpToDate&= _DirectoryTree[directoryName][fileName].updateFileInfo(fileName,pathContents[i],SFileInfo::RECALCULATE_IF_CHANGED,updateListener);
 498                 }
 499
 500                 // run through all of the files in our map looking for files that are not updated and that need erasing
 501                 TFileInfoMap::iterator fit= theDirectory.begin();
 502                 while (fit!=theDirectory.end())
 503                 {
 504                         TFileInfoMap::iterator thisIt= fit;
 505                         ++fit;
 506                         if (thisIt->second.FileName.empty())
 507                         {
 508                                 // if there's an update listener object, then let them know about the file update
 509                                 if (updateListener!=NULL)
 510                                 {
 511                                         updateListener->cbFileInfoErased(thisIt->first);
 512                                 }
 513
 514                                 // erase the entry in our files map and flag the index file as out of date
 515                                 theDirectory.erase(thisIt);
 516                                 _IndexFileIsUpToDate= false;
 517                         }
 518                 }
 519         }
 520
 521
 522         //-----------------------------------------------------------------------------
 523         // methods CFileManager
 524         //-----------------------------------------------------------------------------
 525
 526         bool CFileManager::load(const NLMISC::CSString& fileName, uint32 startOffset, uint32 numBytes, NLMISC::CSString& result)
 527         {
 528                 // clear out the return value before we begin
 529                 result.clear();
 530
 531                 // make sure the file exists
 532                 if (!NLMISC::CFile::fileExists(fileName))
 533                         return false;
 534
 535 //              nldebug("Loading file data for: %s",fileName.c_str());
 536
 537                 // get the file's vital statistics from disk
 538                 uint32 fileSize= NLMISC::CFile::getFileSize(fileName);
 539                 uint32 fileTime= NLMISC::CFile::getFileModificationDate(fileName);
 540
 541                 // run through the files to see if the one we're after is here
 542                 TCacheFiles::iterator it= _CacheFiles.begin();
 543                 for (; it!=_CacheFiles.end();++it)
 544                 {
 545                         // if we've found the file we're after then break out here
 546                         if (it->FileName==fileName && fileSize==it->FileSize && fileTime==it->FileTime)
 547                         {
 548 //                              nldebug("- Found data in Ram @ offset: %d",it->StartOffset);
 549                                 break;
 550                         }
 551                 }
 552
 553                 // if we didn't find the file then we have to load it
 554                 if (it==_CacheFiles.end())
 555                 {
 556 //                      nldebug("- Found data NOT already in Ram");
 557
 558                         // setup a data block for this file
 559                         SCacheFileEntry newFileEntry;
 560                         newFileEntry.FileName= fileName;
 561                         newFileEntry.FileSize= fileSize;
 562                         newFileEntry.FileTime= fileTime;
 563                         newFileEntry.StartOffset= ~0u;
 564
 565                         // if the buffer is too small to load this file then reallocate it and clear out the _CacheFiles vector
 566                         if (_CacheBuffer.size()<fileSize || _CacheFiles.empty())
 567                         {
 568                                 _CacheFiles.clear();
 569                                 _CacheBuffer.clear();
 570                                 _CacheBuffer.resize(max(uint32(fileSize*2),uint32(FileCacheSize)));
 571                                 newFileEntry.StartOffset= 0;
 572 //                              nldebug("- Grew buffer to %d bytes",_CacheBuffer.size());
 573                         }
 574                         else
 575                         {
 576                                 newFileEntry.StartOffset= _CacheFiles.back().StartOffset+_CacheFiles.back().FileSize;
 577                                 uint32 requiredEndOffset= newFileEntry.StartOffset+ fileSize;
 578                                 // see whether we'll fit in between the 'back file and end of buffer
 579                                 if (requiredEndOffset>_CacheBuffer.size())
 580                                 {
 581                                         // clear out all remaining files between us and the end of buffer
 582                                         while (!_CacheFiles.empty() && _CacheFiles.front().StartOffset>=newFileEntry.StartOffset)
 583                                         {
 584 //                                              nldebug("- Ditching cache entry: %s",_CacheFiles.front().FileName.c_str());
 585                                                 _CacheFiles.pop_front();
 586                                                 nlassert(!_CacheFiles.empty());
 587                                         }
 588                                         // not enough space at end of file so we'll need to spin round to the start
 589                                         newFileEntry.StartOffset= 0;
 590                                         requiredEndOffset= fileSize;
 591                                 }
 592                                 // our start offset is now OK, so make a bit of room for our data as required
 593                                 while (!_CacheFiles.empty() && _CacheFiles.front().StartOffset>=newFileEntry.StartOffset && _CacheFiles.front().StartOffset<requiredEndOffset)
 594                                 {
 595 //                                      nldebug("- Ditching cache entry: %s",_CacheFiles.front().FileName.c_str());
 596                                         _CacheFiles.pop_front();
 597                                 }
 598                         }
 599
 600 //                      nldebug("- Reading file data @offset: %d (%d bytes)",newFileEntry.StartOffset,fileSize);
 601
 602                         // read in the file
 603                         FILE* inf = nlfopen(fileName, "rb");
 604                         BOMB_IF(inf==NULL,"Failed to open input file for reading: "+fileName,return false);
 605                         uint32 bytesRead=(uint32)fread(&_CacheBuffer[newFileEntry.StartOffset],1,fileSize,inf);
 606                         fclose(inf);
 607                         BOMB_IF(bytesRead!=fileSize,"Failed to read data from input file: "+fileName,return false);
 608
 609                         // add our new file descriptioon block to the _CacheFiles container
 610                         _CacheFiles.push_back(newFileEntry);
 611                         it=_CacheFiles.end();
 612                         it--;
 613                 }
 614
 615                 // make sure the requested file segment is valid
 616                 uint32 endOffset= startOffset+numBytes;
 617                 DROP_IF(endOffset>fileSize,"Ignoring request for data where end offset > file size: "+fileName,return false);
 618
 619 //              nldebug("- Retrieving data from buffer @offset: %d (%d bytes)",it->StartOffset+startOffset,numBytes);
 620
 621                 // copy out the data chunk that we're after
 622                 result.resize(numBytes);
 623                 memcpy(&result[0],&_CacheBuffer[it->StartOffset+startOffset],numBytes);
 624
 625                 // we succeeded so return true
 626                 return true;
 627         }
 628
 629         TRepositoryDirectoryPtr CFileManager::getRepositoryDirectory(const NLMISC::CSString& path)
 630         {
 631                 // get hold of a ref to the map entry pointing at the directory we want (create a new map entry if need be)
 632                 TRepositoryDirectoryRefPtr& thePtr=_RepositoryDirectories[path];
 633
 634                 // if the map entry is null then create a new one
 635                 if (thePtr==NULL)
 636                 {
 637                         thePtr= new CRepositoryDirectory(path);
 638                         thePtr->readIndex();
 639                 }
 640
 641                 return &*thePtr;
 642         }
 643
 644         bool CFileManager::save(const NLMISC::CSString& fileName, const NLMISC::CMemStream& data)
 645         {
 646                 NLMISC::CSString tmpFileName= fileName+"__patchman__.sav";
 647
 648                 // make sure the destination file is deleted before we begin
 649                 if (NLMISC::CFile::fileExists(fileName))
 650                 {
 651                         NLMISC::CFile::deleteFile(fileName);
 652                 }
 653
 654                 // try to write the tmp file
 655                 try
 656                 {
 657                         // make sure that the directory structure exists foe the file
 658                         NLMISC::CSString path= NLMISC::CFile::getPath(fileName);
 659                         if (!path.empty())
 660                         {
 661                                 NLMISC::CFile::createDirectoryTree(path);
 662                         }
 663
 664                         // go ahead and write the data to disk...
 665                         COFile outputFile(tmpFileName);
 666                         outputFile.serialBuffer(const_cast<uint8*>(data.buffer()),data.size());
 667                 }
 668                 catch(...)
 669                 {
 670                 }
 671
 672                 // make sure that the file write succeeded
 673                 if (NLMISC::CFile::getFileSize(tmpFileName)!=data.size())
 674                 {
 675                         nlwarning("Failed to save file '%s' because failed to save tmp file: %s",fileName.c_str(),tmpFileName.c_str());
 676                         NLMISC::CFile::deleteFile(tmpFileName);
 677                         return false;
 678                 }
 679
 680                 // write succeeded so rename the tmp file to the correct file name
 681                 bool ok= NLMISC::CFile::moveFile(fileName, tmpFileName);
 682                 DROP_IF(!ok,"Failed to save file '"+fileName+"' because failed to rename tmp file: '"+tmpFileName+"'",return false);
 683
 684                 return true;
 685         }
 686
 687         uint32 CFileManager::getFileSize(const NLMISC::CSString& fileName)
 688         {
 689                 try
 690                 {
 691                         return NLMISC::CFile::getFileSize(fileName);
 692                 }
 693                 catch(...)
 694                 {
 695                         return 0;
 696                 }
 697         }
 698
 699 } // end of namespace
 700
 701
 702 //-----------------------------------------------------------------------------
 703 // NLMISC_COMMANDS for testing the singleton interface
 704 //-----------------------------------------------------------------------------
 705
 706 NLMISC_CATEGORISED_COMMAND(patchman,fileManagerLoad,"Load a file segment via the file manager","<file name> <start offset> <num bytes>")
 707 {
 708         if (args.size()!=3)
 709                 return false;
 710
 711         CSString fileName= args[0];
 712         CSString startOffset=args[1];
 713         CSString numBytes=args[2];
 714
 715         CSString data;
 716         bool ok= CFileManager::getInstance().load(fileName,startOffset.atoui(),numBytes.atoui(),data);
 717         if (ok)
 718         {
 719                 log.displayNL("Loaded %d bytes from file: %s[%d] (requested %d) starting: %s",data.size(),fileName.c_str(),startOffset.atoui(),numBytes.atoui(),data.left(20).quote().c_str());
 720         }
 721         else
 722         {
 723                 log.displayNL("Load failed for file: %s (from offset %d to %d)",fileName.c_str(),startOffset.atoui(),startOffset.atoui()+numBytes.atoui()-1);
 724         }
 725
 726         return true;
 727 }
 728
 729 NLMISC_CATEGORISED_COMMAND(patchman,fileManagerSave,"Save a file via the file manager","<file name> <text to save>")
 730 {
 731         if (args.size()!=2)
 732                 return false;
 733
 734         CSString fileName= args[0];
 735         CMemStream data;
 736         data.serialBuffer((uint8*)(&args[1][0]),(uint32)args[1].size());
 737         CFileManager::getInstance().save(fileName,data);
 738
 739         return true;
 740 }
 741
 742 NLMISC_CATEGORISED_COMMAND(patchman,testCFileSpec,"test the CFileSpec class","")
 743 {
 744         if (args.size()!=0)
 745                 return false;
 746
 747         {
 748                 nlinfo("test a");
 749                 CFileSpec fsa("foo/bar");
 750                 nlassert(fsa.matches("foo/bar"));
 751                 nlassert(!fsa.matches("foo/bard"));
 752                 nlassert(!fsa.matches("food/bar"));
 753                 nlassert(!fsa.matches("foo/d/bar"));
 754                 nlassert(!fsa.nameIsWild());
 755                 nlassert(!fsa.pathIsWild());
 756                 nlassert(fsa.nameMatches("bar"));
 757                 nlassert(fsa.pathMatches("foo/"));
 758                 nlassert(!fsa.nameMatches("bard"));
 759                 nlassert(!fsa.pathMatches("food/"));
 760                 nlassert(!fsa.pathMatches("foo/d/"));
 761                 nlassert(fsa.nameSpec()=="bar");
 762                 nlassert(fsa.pathSpec()=="foo/");
 763                 nlassert(fsa.toString()=="foo/bar");
 764         }
 765
 766         {
 767                 nlinfo("test b");
 768                 CFileSpec fsb("foo/bar*");
 769                 nlassert(fsb.matches("foo/bar"));
 770                 nlassert(fsb.matches("foo/bard"));
 771                 nlassert(!fsb.matches("food/bar"));
 772                 nlassert(!fsb.matches("foo/d/bar"));
 773                 nlassert(fsb.nameIsWild());
 774                 nlassert(!fsb.pathIsWild());
 775                 nlassert(fsb.nameMatches("bard"));
 776                 nlassert(fsb.pathMatches("foo/"));
 777                 nlassert(fsb.nameMatches("bard"));
 778                 nlassert(!fsb.pathMatches("food/"));
 779                 nlassert(!fsb.pathMatches("foo/d/"));
 780                 nlassert(fsb.nameSpec()=="bar*");
 781                 nlassert(fsb.pathSpec()=="foo/");
 782                 nlassert(fsb.toString()=="foo/bar*");
 783         }
 784
 785         {
 786                 nlinfo("test c");
 787                 CFileSpec fsc("foo*/bar");
 788                 nlassert(fsc.matches("foo/bar"));
 789                 nlassert(!fsc.matches("foo/bard"));
 790                 nlassert(fsc.matches("food/bar"));
 791                 nlassert(fsc.matches("foo/d/bar"));
 792                 nlassert(!fsc.nameIsWild());
 793                 nlassert(fsc.pathIsWild());
 794                 nlassert(fsc.nameMatches("bar"));
 795                 nlassert(fsc.pathMatches("foo/"));
 796                 nlassert(!fsc.nameMatches("bard"));
 797                 nlassert(fsc.pathMatches("food/"));
 798                 nlassert(fsc.pathMatches("foo/d/"));
 799                 nlassert(fsc.nameSpec()=="bar");
 800                 nlassert(fsc.pathSpec()=="foo*/");
 801                 nlassert(fsc.toString()=="foo*/bar");
 802         }
 803
 804         {
 805                 nlinfo("test d");
 806                 CFileSpec fsd("foo*/bar*");
 807                 nlassert(fsd.matches("foo/bar"));
 808                 nlassert(fsd.matches("foo/bard"));
 809                 nlassert(fsd.matches("food/bar"));
 810                 nlassert(fsd.matches("foo/d/bar"));
 811                 nlassert(fsd.nameIsWild());
 812                 nlassert(fsd.pathIsWild());
 813                 nlassert(fsd.nameMatches("bar"));
 814                 nlassert(fsd.pathMatches("foo/"));
 815                 nlassert(fsd.nameMatches("bard"));
 816                 nlassert(fsd.pathMatches("food/"));
 817                 nlassert(fsd.pathMatches("foo/d/"));
 818                 nlassert(fsd.nameSpec()=="bar*");
 819                 nlassert(fsd.pathSpec()=="foo*/");
 820                 nlassert(fsd.toString()=="foo*/bar*");
 821         }
 822
 823         return true;
 824 }
 825
 826         class CFileSpec
 827         {
 828         public:
 829                 // ctors
 830                 CFileSpec();
 831                 CFileSpec(const NLMISC::CSString& fileSpec);
 832
 833                 // test a complete match (filename and path)
 834                 bool matches(const NLMISC::CSString& fullFileName) const;
 835
 836                 // test whether the given file name matches the filename part of the filespec
 837                 // note - the supplied filename should already have been stripped of its path
 838                 bool nameMatches(const NLMISC::CSString& fileName) const;
 839
 840                 // test whether the given path matches the path part of the filespec
 841                 // note - the supplied path should not have an attached file name
 842                 bool pathMatches(const NLMISC::CSString& path) const;
 843
 844                 // retrieve the filespec as a single string (for serialising, etc)
 845                 NLMISC::CSString toString() const;
 846
 847                 // accessors
 848                 const NLMISC::CSString& nameSpec() const;
 849                 const NLMISC::CSString& pathSpec() const;
 850                 bool nameIsWild() const;
 851                 bool pathIsWild() const;
 852
 853         private:
 854                 NLMISC::CSString _NameSpec;
 855                 NLMISC::CSString _PathSpec;
 856                 bool _NameIsWild;                       // true if _NameSpec contains wildcards ('*' or '?')
 857                 bool _PathIsWild;                       // true if _PathSpec contains wildcards ('*' or '?')
 858         };