ryzom/server/src/patchman_service/repository.cpp

   1 // Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
   2 // Copyright (C) 2010  Winch Gate Property Limited
   3 //
   4 // This program is free software: you can redistribute it and/or modify
   5 // it under the terms of the GNU Affero General Public License as
   6 // published by the Free Software Foundation, either version 3 of the
   7 // License, or (at your option) any later version.
   8 //
   9 // This program is distributed in the hope that it will be useful,
  10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 // GNU Affero General Public License for more details.
  13 //
  14 // You should have received a copy of the GNU Affero General Public License
  15 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 //-----------------------------------------------------------------------------
  18 // includes
  19 //-----------------------------------------------------------------------------
  20
  21 // nel
  22 #include "nel/misc/common.h"
  23
  24 // game share
  25 #include "game_share/utils.h"
  26 #include "game_share/file_description_container.h"
  27
  28 // local
  29 #include "repository.h"
  30
  31
  32 //-------------------------------------------------------------------------------------------------
  33 // namespaces
  34 //-------------------------------------------------------------------------------------------------
  35
  36 using namespace std;
  37 using namespace NLMISC;
  38
  39 //-------------------------------------------------------------------------------------------------
  40 // constants & utilities
  41 //-------------------------------------------------------------------------------------------------
  42
  43 // From spa_server_patch_applier.cpp
  44 extern void writeVersionFile(const NLMISC::CSString& fileName, uint32 version);
  45 extern uint32 readVersionFile(const NLMISC::CSString& fileName);
  46
  47 NLMISC::CSString getRepositoryIndexFileName(const NLMISC::CSString& repositoryName)
  48 {
  49         return "repository_"+repositoryName+".idx";
  50 }
  51
  52 uint32 getFileVersion(const NLMISC::CSString& fileName)
  53 {
  54         // start at the back of the file name and scan forwards until we find a '/' or '\\' or ':' or a digit
  55         uint32 i= (uint32)fileName.size();
  56         while (i--)
  57         {
  58                 char c= fileName[i];
  59
  60                 // if we've hit a directory name separator then we haven't found a version number so drop out
  61                 if (c=='/' || c=='\\' || c==':')
  62                         return ~0u;
  63
  64                 // if we've found a digit then construct the rest of the version number and return
  65                 if (isdigit(c))
  66                 {
  67                         uint32 firstDigit= i;
  68                         while (firstDigit!=0 && isdigit(fileName[firstDigit-1]))
  69                         {
  70                                 --firstDigit;
  71                         }
  72                         return fileName.leftCrop(firstDigit).left(i-firstDigit+1).atoui();
  73                 }
  74         }
  75
  76         // default to our 'invalid' value
  77         return ~0u;
  78 }
  79
  80 CHashKeyMD5 safeGetMD5(const NLMISC::CSString& fileName)
  81 {
  82         while (true)
  83         {
  84                 try
  85                 {
  86                         CHashKeyMD5 result= NLMISC::getMD5(fileName);
  87                         return result;
  88                 }
  89                 catch(...)
  90                 {
  91                         nlwarning("Exception thrown in getMD5(\"%s\") ... will try again in a few seconds",fileName.c_str());
  92                         nlSleep(3);
  93                 }
  94         }
  95 }
  96
  97
  98 //-----------------------------------------------------------------------------
  99 // methods CRepository
 100 //-----------------------------------------------------------------------------
 101
 102 CRepository::CRepository()
 103 {
 104         _Version= 0;
 105 }
 106
 107 bool CRepository::init(const NLMISC::CSString& name,const NLMISC::CSString& directory)
 108 {
 109         _Name= name.unquoteIfQuoted();
 110         _TargetDirectory= NLMISC::CPath::standardizePath(directory.unquoteIfQuoted());
 111
 112         nldebug("Repository %s: %s",_Name.c_str(),_TargetDirectory.c_str());
 113
 114         // check whether the target directory exists
 115         if (!NLMISC::CFile::isDirectory(_TargetDirectory))
 116         {
 117                 // the directory didn't exist so try to create it...
 118                 NLMISC::CFile::createDirectoryTree(_TargetDirectory);
 119                 DROP_IF(!NLMISC::CFile::isDirectory(_TargetDirectory),"Failed to create target directory: \""+_TargetDirectory+"\"",return false);
 120         }
 121
 122         // if we have a saved file that gives timestamp / size / checksum correspondances then load it
 123         NLMISC::CSString index;
 124         NLMISC::CSString indexFileName= _TargetDirectory+getRepositoryIndexFileName(_Name);
 125         if (NLMISC::CFile::fileExists(indexFileName))
 126         {
 127                 index.readFromFile(indexFileName);
 128         }
 129         if (!index.empty())
 130         {
 131                 nlinfo("GUSREP_Reading index file: %s",indexFileName.c_str());
 132                 NLMISC::CVectorSString lines;
 133                 index.splitLines(lines);
 134                 for (uint32 i=0;i<lines.size();++i)
 135                 {
 136                         // get hold of the line and strip off comments and spurious blanks
 137                         NLMISC::CSString line= lines[i].splitTo("//").strip();
 138                         if (line.empty()) continue;
 139
 140                         // see if this is a special line of some sort
 141                         if (line.left(1)=="*")
 142                         {
 143                                 line=line.leftCrop(1).strip();
 144                                 NLMISC::CSString keyword= line.firstWord(true);
 145                                 if (keyword=="PatchVersion")
 146                                 {
 147                                         uint32 version= line.strip().atoui();
 148                                         DROP_IF(version==0,NLMISC::toString("Skipping line %d because expected to find version number but found: '%s'",i,line.c_str()),continue);
 149                                         _Version= version;
 150                                         nlinfo("Repository - Version number found in save file: %u",version);
 151                                 }
 152                                 continue;
 153                         }
 154
 155                         // break the line down into constituent parts
 156                         uint32 fileSize= line.strtok(" \t").atoi();
 157                         uint32 fileTime= line.strtok(" \t").atoi();
 158                         NLMISC::CHashKeyMD5 checksum;
 159                         checksum.fromString(line.strtok(" \t"));
 160                         NLMISC::CSString fileName= line.strip();
 161
 162                         // make sure that the text in the line was valid and that the file on disk looks like the one in our record
 163                         DROP_IF(fileName.empty(),"Skipping line due to parse error: "+lines[i],continue);
 164                         DROP_IF(_Files.find(fileName)!=_Files.end(),"Skipping line due to repeated file name: "+lines[i],continue);
 165                         if (!NLMISC::CFile::fileExists(_TargetDirectory+fileName))      continue;
 166                         if (NLMISC::CFile::getFileSize(_TargetDirectory+fileName)!=fileSize)    continue;
 167                         if (NLMISC::CFile::getFileModificationDate(_TargetDirectory+fileName)!=fileTime)        continue;
 168
 169                         // add the result to our map of files
 170                         _Files[fileName].set(fileSize,fileTime,checksum);
 171                 }
 172                 nlinfo("%d lines read from file, %d retained as pertinent",lines.size(),_Files.size());
 173         }
 174
 175         // scan the target directory looking for updates
 176         update();
 177
 178         // housekeeping and return with success
 179         return true;
 180 }
 181
 182 void CRepository::updateFile(NLMISC::CSString fileName)
 183 {
 184         nldebug(("GUSREP_Updating repository entry for file: '"+fileName+"'").c_str());
 185
 186         // if the name of the file that has changed contains the target directory name then crop it
 187         if (fileName.left((uint32)_TargetDirectory.size())==_TargetDirectory)
 188         {
 189                 fileName=fileName.leftCrop((uint32)_TargetDirectory.size());
 190         }
 191
 192         // lookup the file in the map
 193         TFiles::iterator fileIt= _Files.find(fileName);
 194         BOMB_IF(fileIt== _Files.end(),"Failed to IDENTIFY the file that I have been asked to update: '"+fileName+"' in my map",return);
 195
 196         // make sure the file exists on the disk
 197         BOMB_IF(!NLMISC::CFile::fileExists(_TargetDirectory+fileName),"Failed to LOCATE the file that I have been asked to update: '"+fileName+"' on the disk",return);
 198
 199         for (bool done=false;!done;)
 200         {
 201                 NLMISC::CSString fullFileName= _TargetDirectory+fileName;
 202                 try
 203                 {
 204                         fileIt->second.FileSize= NLMISC::CFile::getFileSize(fullFileName);
 205                         fileIt->second.FileTime= NLMISC::CFile::getFileModificationDate(fullFileName);
 206                         fileIt->second.Checksum= safeGetMD5(fullFileName);
 207                         done= true;
 208                 }
 209                 catch(...)
 210                 {
 211                         nlwarning("Failed to update file '%s' ... waiting a few seconds before trying again ...",fullFileName.c_str());
 212                         nlSleep(2000);
 213                 }
 214         }
 215 }
 216
 217 void CRepository::addFileStub(NLMISC::CSString fileName)
 218 {
 219         nldebug(("GUSREP_Adding repository stub for file: '"+fileName+"'").c_str());
 220
 221         // if the name of the file that has changed contains the target directory name then crop it
 222         if (fileName.left((uint32)_TargetDirectory.size())==_TargetDirectory)
 223         {
 224                 fileName=fileName.leftCrop((uint32)_TargetDirectory.size());
 225         }
 226
 227         // make sure the file didn't already exist in the map
 228         TFiles::iterator fileIt= _Files.end();
 229         fileIt=_Files.find(fileName);
 230         BOMB_IF(fileIt!= _Files.end(),"Failed to add stub for file that already exists: '"+fileName+"'",return);
 231
 232         // create the new map entry and set properties to 0
 233         _Files[fileName].FileSize= 0;
 234         _Files[fileName].FileTime= 0;
 235 }
 236
 237 uint32 CRepository::update()
 238 {
 239         // setup a variable to hold our return value
 240         uint32 result= 0;
 241
 242         // read the version number from the 'version' file and drop out immediately if the version is unchanged
 243         uint32 versionOnDisk= readVersionFile(_TargetDirectory+"version");
 244         if (versionOnDisk==_Version)
 245                 return result;
 246
 247         // scan the target directory to determine the files that it contains...
 248         nlinfo("GUSREP_Scanning for files in directory: %s",_TargetDirectory.c_str());
 249         CFileDescriptionContainer fdc;
 250         fdc.addFileSpec(_TargetDirectory+"*",true);
 251
 252         // update the file index from the files found in the directory...
 253         nldebug("GUSREP_Checking index for updates",_TargetDirectory.c_str());
 254         for (uint32 i=0;i<fdc.size();++i)
 255         {
 256                 // get a refference to the file description for this iteration
 257                 CFileDescription& theFile= fdc[i];
 258
 259                 // get hold of the file name for the next file
 260 //              CSString fileName= NLMISC::CFile::getFilename(theFile.FileName);
 261                 CSString fileName= theFile.FileName.leftCrop((uint32)_TargetDirectory.size());
 262
 263                 // extract the version number from the file name and skip the file if it's too recent or the version number was invalid
 264                 uint32 fileVersion= getFileVersion(fileName);
 265                 if (fileVersion>versionOnDisk)
 266                         continue;
 267
 268                 // check whether this is a file that should already be in my index
 269                 if (fileVersion<=_Version)
 270                 {
 271                         // make sure we already had an entry for this file
 272                         BOMB_IF(_Files.find(fileName)==_Files.end(),"Big nasty problem - The following file was not in my index but exists on the disk!: "+fileName,return result);
 273
 274                         // get hold of a refference to this file's entry in the index
 275                         CFilesMapEntry& mapEntry= _Files[fileName];
 276
 277                         // make sure this file hasn't changed size...
 278                         BOMB_IF(mapEntry.FileSize!=theFile.FileSize,"Big nasty problem - The following file was already in my index but size has changed!: "+fileName,return result);
 279
 280                         // if the file's timestamp has changed then we need to make sure the checksum hasn't
 281                         if (mapEntry.FileTime!=theFile.FileTimeStamp)
 282                         {
 283                                 BOMB_IF(safeGetMD5(theFile.FileName)!=mapEntry.Checksum,"Big nasty problem - The following file was already in my index but checksum has changed!: "+fileName,return result);
 284                         }
 285
 286                         // the file matches OK so just ignore it
 287                         continue;
 288                 }
 289
 290                 // get hold of a refference to this file's entry in the index (create a new entry if need be)
 291                 CFilesMapEntry& mapEntry= _Files[fileName];
 292
 293                 // check whether the info in the file description corresponds to the index entry...
 294                 if (mapEntry.FileSize!=theFile.FileSize || mapEntry.FileTime!=theFile.FileTimeStamp)
 295                 {
 296                         // the file index entry is not up to date so update it
 297                         nldebug("GUSREP_Updating file index entry for file: %s",fileName.c_str());
 298                         mapEntry.FileSize= theFile.FileSize;
 299                         mapEntry.FileTime= theFile.FileTimeStamp;
 300                         mapEntry.Checksum= safeGetMD5(theFile.FileName);
 301
 302                         // write the file index back to disk with this new record
 303                         writeIndexFile();
 304                         ++result;
 305                 }
 306         }
 307
 308         _Version= versionOnDisk;
 309         writeIndexFile();
 310         return result;
 311 }
 312
 313 void CRepository::writeIndexFile()
 314 {
 315 //      nldebug("GUSREP_Writing index: %s",(_TargetDirectory+getRepositoryIndexFileName(_Name)).c_str());
 316         NLMISC::CSString fileIndex;
 317         iterator it= _Files.begin();
 318         iterator itEnd= _Files.end();
 319         fileIndex+= NLMISC::toString("*PatchVersion %d\n",_Version);
 320         for(;it!=itEnd;++it)
 321         {
 322                 fileIndex+= NLMISC::toString("%10d %10d %16s %s\n",it->second.FileSize,it->second.FileTime,it->second.Checksum.toString().c_str(),it->first.c_str());
 323         }
 324         fileIndex.writeToFile(_TargetDirectory+getRepositoryIndexFileName(_Name));
 325 }
 326
 327 // display info about the repository (version number, directory names, etc)
 328 void CRepository::display(NLMISC::CLog& log) const
 329 {
 330         log.displayNL("Directory: %s (%u files found)",_TargetDirectory.c_str(),_Files.size());
 331         log.displayNL("Index file name: %s",(_TargetDirectory+getRepositoryIndexFileName(_Name)).c_str());
 332         log.displayNL("Version: %u",_Version);
 333 }
 334
 335 uint32 CRepository::getVersion() const
 336 {
 337         return _Version;
 338 }
 339
 340 void CRepository::setVersion(uint32 version)
 341 {
 342         writeVersionFile(_TargetDirectory+"version", version);
 343         _Version= version;
 344 }
 345
 346 uint32 CRepository::size() const
 347 {
 348         return (uint32)_Files.size();
 349 }
 350
 351 const CRepository::CFilesMapEntry& CRepository::operator[](const NLMISC::CSString& key) const
 352 {
 353         return const_cast<CRepository*>(this)->_Files[key];
 354 }
 355
 356 CRepository::iterator CRepository::find(const NLMISC::CSString& key)
 357 {
 358         return _Files.find(key);
 359 }
 360
 361 CRepository::const_iterator CRepository::find(const NLMISC::CSString& key) const
 362 {
 363         return _Files.find(key);
 364 }
 365
 366 CRepository::iterator CRepository::begin()
 367 {
 368         return _Files.begin();
 369 }
 370
 371 CRepository::const_iterator CRepository::begin() const
 372 {
 373         return _Files.begin();
 374 }
 375
 376 CRepository::iterator CRepository::end()
 377 {
 378         return _Files.end();
 379 }
 380
 381 CRepository::const_iterator CRepository::end() const
 382 {
 383         return _Files.end();
 384 }
 385
 386 void CRepository::fillShortList(PATCHMAN::TFileInfoVector       &files) const
 387 {
 388         // start by clearing out any previous contents in the files vector
 389         files.clear();
 390
 391         // iterate over the repository adding files to the files vector
 392         const_iterator it= _Files.begin();
 393         const_iterator itEnd= _Files.end();
 394         for (;it!=itEnd;++it)
 395         {
 396                 // skip files that we haven't finished building info on
 397                 if (it->second.FileSize==0)
 398                         continue;
 399
 400                 // append a new entry to the vector
 401                 vectAppend(files);
 402                 // setup data for the (new) back vector entry
 403                 files.back().FileName=it->first;
 404                 files.back().Checksum=it->second.Checksum;
 405                 nldebug("GUSREP_sending info on file: '%s'",files.back().FileName.c_str());
 406         }
 407 }