ryzom/server/src/general_utilities_service/repository.cpp

   1 // Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
   2 // Copyright (C) 2010  Winch Gate Property Limited
   3 //
   4 // This program is free software: you can redistribute it and/or modify
   5 // it under the terms of the GNU Affero General Public License as
   6 // published by the Free Software Foundation, either version 3 of the
   7 // License, or (at your option) any later version.
   8 //
   9 // This program is distributed in the hope that it will be useful,
  10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 // GNU Affero General Public License for more details.
  13 //
  14 // You should have received a copy of the GNU Affero General Public License
  15 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 //-----------------------------------------------------------------------------
  18 // includes
  19 //-----------------------------------------------------------------------------
  20
  21 // game share
  22 #include "game_share/utils.h"
  23 #include "game_share/file_description_container.h"
  24
  25 // local
  26 #include "repository.h"
  27
  28
  29 //-------------------------------------------------------------------------------------------------
  30 // namespaces
  31 //-------------------------------------------------------------------------------------------------
  32
  33 using namespace std;
  34 using namespace NLMISC;
  35
  36
  37 //-------------------------------------------------------------------------------------------------
  38 // constants & utilities
  39 //-------------------------------------------------------------------------------------------------
  40
  41 NLMISC::CSString getRepositoryIndexFileName(const NLMISC::CSString& repositoryName)
  42 {
  43         return "repository_"+repositoryName+".idx";
  44 }
  45
  46
  47 //-----------------------------------------------------------------------------
  48 // methods CRepository
  49 //-----------------------------------------------------------------------------
  50
  51 bool CRepository::init(const NLMISC::CSString& name,const NLMISC::CSString& directory,const NLMISC::CSString& filespec,const NLMISC::CSString& lockFilespecs)
  52 {
  53         _Name= name.unquoteIfQuoted();
  54         _TargetDirectory= NLMISC::CPath::standardizePath(directory.unquoteIfQuoted());
  55         filespec.splitBySeparator('|',_Filespec);
  56         if (_Filespec.empty()) _Filespec.push_back("*");
  57         lockFilespecs.splitBySeparator('|',_LockFilespecs);
  58
  59         nldebug("Repository %s: %s",_Name.c_str(),_TargetDirectory.c_str());
  60         nldebug("- filespec: %s",NLMISC::CSString().join(_Filespec,';').c_str());
  61         nldebug("- avoid filespec: %s",NLMISC::CSString().join(_LockFilespecs,';').c_str());
  62
  63         // check whether the target directory exists
  64         if (!NLMISC::CFile::isDirectory(_TargetDirectory))
  65         {
  66                 // the directory didn't exist so try to create it...
  67                 NLMISC::CFile::createDirectoryTree(_TargetDirectory);
  68                 DROP_IF(!NLMISC::CFile::isDirectory(_TargetDirectory),"Failed to create target directory: \""+_TargetDirectory+"\"",return false);
  69         }
  70
  71         // if we have a saved file that gives timestamp / size / checksum correspondances then load it
  72         NLMISC::CSString index;
  73         if (NLMISC::CFile::fileExists(_TargetDirectory+getRepositoryIndexFileName(_Name)))
  74         {
  75                 index.readFromFile(_TargetDirectory+getRepositoryIndexFileName(_Name));
  76         }
  77         if (!index.empty())
  78         {
  79                 nlinfo("GUSREP_Reading index file: %s",(_TargetDirectory+getRepositoryIndexFileName(_Name)).c_str());
  80                 NLMISC::CVectorSString lines;
  81                 index.splitLines(lines);
  82                 for (uint32 i=0;i<lines.size();++i)
  83                 {
  84                         // get hold of the line and strip off comments and spurious blanks
  85                         NLMISC::CSString line= lines[i].splitTo("//").strip();
  86                         if (line.empty()) continue;
  87
  88                         // break the line down into constituent parts
  89                         uint32 fileSize= line.strtok(" \t").atoi();
  90                         uint32 fileTime= line.strtok(" \t").atoi();
  91                         NLMISC::CHashKeyMD5 checksum;
  92                         checksum.fromString(line.strtok(" \t"));
  93                         NLMISC::CSString fileName= line.strip();
  94
  95                         // make sure that the text in the line was valid and that the file name existed ok
  96                         DROP_IF(fileName.empty(),"Skipping line due to parse error: "+lines[i],continue);
  97                         DROP_IF(_Files.find(fileName)!=_Files.end(),"Skipping line due to repeated file name: "+lines[i],continue);
  98                         if (!NLMISC::CFile::fileExists(_TargetDirectory+fileName))      continue;
  99
 100                         // add the result to our map of files
 101                         _Files[fileName].set(fileSize,fileTime,checksum);
 102                 }
 103         }
 104
 105         // scan the target directory looking for updates
 106         update();
 107
 108         // housekeeping and return with success
 109         return true;
 110 }
 111
 112 void CRepository::updateFile(NLMISC::CSString fileName)
 113 {
 114         nldebug(("Updating repository entry for file: '"+fileName+"'").c_str());
 115
 116         // if the name of the file that has changed contains the target directory name then crop it
 117         if (fileName.left(_TargetDirectory.size())==_TargetDirectory)
 118         {
 119                 fileName=fileName.leftCrop(_TargetDirectory.size());
 120         }
 121
 122         // lookup the file in the map
 123         TFiles::iterator fileIt= _Files.find(fileName);
 124         BOMB_IF(fileIt== _Files.end(),"Failed to IDENTIFY the file that I have been asked to update: '"+fileName+"'",return);
 125
 126         // make sure the file exists on the disk
 127         BOMB_IF(!NLMISC::CFile::fileExists(_TargetDirectory+fileName),"Failed to LOCATE the file that I have been asked to update: '"+fileName+"'",return);
 128
 129         fileIt->second.FileSize= NLMISC::CFile::getFileSize(_TargetDirectory+fileName);
 130         fileIt->second.FileTime= NLMISC::CFile::getFileModificationDate(_TargetDirectory+fileName);
 131         fileIt->second.Checksum= NLMISC::getMD5(_TargetDirectory+fileName);
 132 }
 133
 134 void CRepository::addFileStub(NLMISC::CSString fileName)
 135 {
 136         nldebug(("Adding repository stub for file: '"+fileName+"'").c_str());
 137
 138         // if the name of the file that has changed contains the target directory name then crop it
 139         if (fileName.left(_TargetDirectory.size())==_TargetDirectory)
 140         {
 141                 fileName=fileName.leftCrop(_TargetDirectory.size());
 142         }
 143
 144         // make sure the file didn't already exist in the map
 145         TFiles::iterator fileIt= _Files.end();
 146         fileIt=_Files.find(fileName);
 147         BOMB_IF(fileIt!= _Files.end(),"Failed to add stub for file that already exists: '"+fileName+"'",return);
 148
 149         // create the new map entry and set properties to 0
 150         _Files[fileName].FileSize= 0;
 151         _Files[fileName].FileTime= 0;
 152 }
 153
 154 uint32 CRepository::update()
 155 {
 156         // setup a variable to hold our return value
 157         uint32 result= 0;
 158
 159         // make sure there are no 'avoid' files about
 160         CFileDescriptionContainer fdc;
 161         for (uint32 i=0;i<_LockFilespecs.size();++i)
 162         {
 163                 fdc.addFileSpec(_LockFilespecs[i]);
 164         }
 165         if (!fdc.empty())
 166         {
 167                 // some avoid files have been found so build a list of them
 168                 NLMISC::CSString s;
 169                 for (uint32 i=0;i<fdc.size();++i)
 170                 {
 171                         if (!s.empty()) s+= " | ";
 172                         s+= fdc[i].FileName;
 173                 }
 174                 if (s!=_BlockingFiles)
 175                 {
 176                         // the avoid list has changed so re-display it and record it for safe keeping
 177                         nldebug("%s: Waiting for the following files to be removed: %s",_Name.c_str(),s.c_str());
 178                         _BlockingFiles= s;
 179                 }
 180                 return 0;
 181         }
 182
 183         // clear out the blocking files string because there clearly are none if we made it this far.
 184         _BlockingFiles.clear();
 185
 186         // scan the target directory to determine the files that it contains...
 187 //      nlinfo("GUSREP_Scanning for files in directory: %s",_TargetDirectory.c_str());
 188         fdc.clear();
 189         fdc.addFiles(_TargetDirectory,_Filespec,true);
 190
 191         // update the file index from the files found in the directory...
 192 //      nlinfo("GUSREP_Checking index for updates",_TargetDirectory.c_str());
 193         for (uint32 i=0;i<fdc.size();++i)
 194         {
 195                 // get a refference to the file description for this iteration
 196                 CFileDescription& theFile= fdc[i];
 197
 198                 // get hold of the file name for the next file
 199 //              CSString fileName= NLMISC::CFile::getFilename(theFile.FileName);
 200                 CSString fileName= theFile.FileName.leftCrop(_TargetDirectory.size());
 201
 202                 // if this is the index file then skip it
 203                 if (fileName==getRepositoryIndexFileName(_Name))
 204                         continue;
 205
 206                 // get hold of a refference to this file's entry in the index (create a new entry if need be)
 207                 CFilesMapEntry& mapEntry= _Files[fileName];
 208
 209                 // check whether the info in the file description corresponds to the index entry...
 210                 if (mapEntry.FileSize!=theFile.FileSize || mapEntry.FileTime!=theFile.FileTimeStamp)
 211                 {
 212                         // the file index entry is not up to date so update it
 213                         nlinfo("GUSREP_Updating file index entry for file: %s",fileName.c_str());
 214                         mapEntry.FileSize= theFile.FileSize;
 215                         mapEntry.FileTime= theFile.FileTimeStamp;
 216                         mapEntry.Checksum= NLMISC::getMD5(theFile.FileName);
 217
 218                         // write the file index back to disk with this new record
 219                         writeIndexFile();
 220                         ++result;
 221                 }
 222         }
 223
 224         return result;
 225 }
 226
 227 void CRepository::writeIndexFile()
 228 {
 229         nlinfo("GUSREP_Writing index: %s",(_TargetDirectory+getRepositoryIndexFileName(_Name)).c_str());
 230         NLMISC::CSString fileIndex;
 231         iterator it= _Files.begin();
 232         iterator itEnd= _Files.end();
 233         for(;it!=itEnd;++it)
 234         {
 235                 fileIndex+= NLMISC::toString("%10d %10d %16s %s\n",it->second.FileSize,it->second.FileTime,it->second.Checksum.toString().c_str(),it->first.c_str());
 236         }
 237         fileIndex.writeToFile(_TargetDirectory+getRepositoryIndexFileName(_Name));
 238 }
 239
 240 uint32 CRepository::size() const
 241 {
 242         return _Files.size();
 243 }
 244
 245 const CRepository::CFilesMapEntry& CRepository::operator[](const NLMISC::CSString& key) const
 246 {
 247         return const_cast<CRepository*>(this)->_Files[key];
 248 }
 249
 250 CRepository::iterator CRepository::find(const NLMISC::CSString& key)
 251 {
 252         return _Files.find(key);
 253 }
 254
 255 CRepository::const_iterator CRepository::find(const NLMISC::CSString& key) const
 256 {
 257         return _Files.find(key);
 258 }
 259
 260 CRepository::iterator CRepository::begin()
 261 {
 262         return _Files.begin();
 263 }
 264
 265 CRepository::const_iterator CRepository::begin() const
 266 {
 267         return _Files.begin();
 268 }
 269
 270 CRepository::iterator CRepository::end()
 271 {
 272         return _Files.end();
 273 }
 274
 275 CRepository::const_iterator CRepository::end() const
 276 {
 277         return _Files.end();
 278 }
 279
 280 void CRepository::fillShortList(std::vector<GUS_SCM::TFileRecord>       &files) const
 281 {
 282         // start by clearing out any previous contents in the files vector
 283         files.clear();
 284
 285         // iterate over the repository adding files to the files vector
 286         const_iterator it= _Files.begin();
 287         const_iterator itEnd= _Files.end();
 288         for (;it!=itEnd;++it)
 289         {
 290                 // append a new entry to the vector
 291                 vectAppend(files);
 292                 // setup data for the (new) back vector entry
 293                 files.back().setFileName(it->first);
 294                 files.back().setChecksum(it->second.Checksum);
 295                 nlinfo("sending info on file: '%s'",files.back().getFileName().c_str());
 296         }
 297 }