ryzom/tools/client/client_data_check/data_scan.cpp

   1 // Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
   2 // Copyright (C) 2010  Winch Gate Property Limited
   3 //
   4 // This program is free software: you can redistribute it and/or modify
   5 // it under the terms of the GNU Affero General Public License as
   6 // published by the Free Software Foundation, either version 3 of the
   7 // License, or (at your option) any later version.
   8 //
   9 // This program is distributed in the hope that it will be useful,
  10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 // GNU Affero General Public License for more details.
  13 //
  14 // You should have received a copy of the GNU Affero General Public License
  15 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 //
  18 // Includes
  19 //
  20
  21 #include "nel/misc/debug.h"
  22 #include "nel/misc/path.h"
  23 #include "nel/misc/thread.h"
  24 #include "nel/misc/sha1.h"
  25 #include "nel/misc/big_file.h"
  26 #include "nel/misc/i18n.h"
  27
  28 #include "data_scan.h"
  29
  30
  31 //
  32 // Namespaces
  33 //
  34
  35 using namespace std;
  36 using namespace NLMISC;
  37
  38
  39 // ***************************************************************************
  40 static  ucstring        dummyI18N(const std::string &s)
  41 {
  42         return s;
  43 }
  44
  45
  46 // ****************************************************************************
  47 // ****************************************************************************
  48 // ****************************************************************************
  49 // CPatchManager
  50 // ****************************************************************************
  51 // ****************************************************************************
  52 // ****************************************************************************
  53
  54 struct EPatchDownloadException : public Exception
  55 {
  56         EPatchDownloadException() : Exception( "Download Error" ) {}
  57         EPatchDownloadException( const std::string& str ) : Exception( str ) {}
  58         virtual ~EPatchDownloadException() throw() {}
  59 };
  60
  61 CPatchManager *CPatchManager::_Instance = NULL;
  62
  63 // ****************************************************************************
  64 CPatchManager::CPatchManager() : State("t_state"), DataScanState("t_data_scan_state")
  65 {
  66         DescFilename = "ryzom_xxxxx.idx";
  67
  68         ClientPatchPath = "./unpack/";
  69         ClientDataPath = "./data/";
  70
  71
  72         VerboseLog = true;
  73
  74         ScanDataThread = NULL;
  75         thread = NULL;
  76
  77         ValidDescFile = false;
  78 }
  79
  80 // ****************************************************************************
  81 void CPatchManager::init()
  82 {
  83         // retrieve the current client version, according to .idx
  84         readClientVersionAndDescFile();
  85 }
  86
  87 // ***************************************************************************
  88 void CPatchManager::readClientVersionAndDescFile()
  89 {
  90         try
  91         {
  92                 ValidDescFile = false;
  93                 vector<string> vFiles;
  94                 CPath::getPathContent(ClientPatchPath, false, false, true, vFiles);
  95                 uint32 nVersion = 0xFFFFFFFF;
  96                 for (uint32 i = 0; i < vFiles.size(); ++i)
  97                 {
  98                         string sName = CFile::getFilename(vFiles[i]);
  99                         string sExt = CFile::getExtension(sName);
 100                         string sBase = sName.substr(0, sName.rfind('_'));
 101                         if ((sExt == "idx") && (sBase == "ryzom"))
 102                         {
 103                                 string val = sName.substr(sName.rfind('_')+1, 5);
 104                                 uint32 nNewVersion = atoi(val.c_str());
 105                                 if ((nNewVersion > nVersion) || (nVersion == 0xFFFFFFFF))
 106                                         nVersion = nNewVersion;
 107                         }
 108                 }
 109                 if (nVersion != 0xFFFFFFFF)
 110                         readDescFile(nVersion);
 111                 else
 112                         DescFilename = "unknown";
 113                 ValidDescFile = true;
 114         }
 115         catch(Exception &)
 116         {
 117                 // Not important that there is no desc file
 118         }
 119 }
 120
 121 // ****************************************************************************
 122 // Called in main thread
 123 bool CPatchManager::getThreadState (ucstring &stateOut, vector<ucstring> &stateLogOut)
 124 {
 125         if (ScanDataThread==NULL)
 126                 return false;
 127
 128         // clear output
 129         stateOut.clear();
 130         stateLogOut.clear();
 131
 132         // Get access to the state
 133         bool    changed= false;
 134         {
 135                 CSynchronized<CState>::CAccessor as(&State);
 136                 CState  &rState= as.value();
 137                 if (rState.StateChanged)
 138                 {
 139                         // and retrieve info
 140                         changed= true;
 141                         stateOut = rState.State;
 142                         stateLogOut = rState.StateLog;
 143                         // clear state
 144                         rState.StateLog.clear();
 145                         rState.StateChanged= false;
 146                 }
 147         }
 148
 149         // verbose log
 150         if (isVerboseLog() && !stateLogOut.empty())
 151                 for (uint32 i = 0; i < stateLogOut.size(); ++i)
 152                         nlinfo("%s", stateLogOut[i].toString().c_str());
 153
 154         return changed;
 155 }
 156
 157 // ****************************************************************************
 158 int CPatchManager::getTotalFilesToGet()
 159 {
 160         if (ScanDataThread != NULL)
 161                 return ScanDataThread->TotalFileToScan;
 162
 163         return 1;
 164 }
 165
 166 // ****************************************************************************
 167 int CPatchManager::getCurrentFilesToGet()
 168 {
 169         if (ScanDataThread != NULL)
 170                 return ScanDataThread->CurrentFileScanned;
 171
 172         return 1;
 173 }
 174
 175 // ****************************************************************************
 176 // Take care this function is called by the thread
 177 void CPatchManager::setState (bool bOutputToLog, const ucstring &ucsNewState)
 178 {
 179         {
 180                 CSynchronized<CState>::CAccessor as(&State);
 181                 CState  &rState= as.value();
 182                 rState.State= ucsNewState;
 183                 if(bOutputToLog)
 184                         rState.StateLog.push_back(ucsNewState);
 185                 rState.StateChanged= true;
 186         }
 187 }
 188
 189 // ****************************************************************************
 190 string CPatchManager::getClientVersion()
 191 {
 192         if (!ValidDescFile)
 193                 return "";
 194
 195         return toString("%05d", DescFile.getFiles().getVersionNumber());
 196 }
 197
 198 // ****************************************************************************
 199 void CPatchManager::readDescFile(sint32 nVersion)
 200 {
 201         DescFilename = toString("ryzom_%05d.idx", nVersion);
 202         string srcName = ClientPatchPath + DescFilename;
 203         DescFile.clear();
 204         if (!DescFile.load(srcName))
 205                 throw Exception ("Can't open file '%s'", srcName.c_str ());
 206 }
 207
 208 // ****************************************************************************
 209 // Get all the patches that need to be applied to a file from the description of this file given by the server
 210 void CPatchManager::getPatchFromDesc(SFileToPatch &ftpOut, const CBNPFile &fIn, bool forceCheckSumTest)
 211 {
 212         uint32 j;
 213         const CBNPFile rFile = fIn;
 214         const string &rFilename = rFile.getFileName();
 215         // Does the BNP exists ?
 216         string sFilePath = CPath::lookup(rFilename);
 217         if (sFilePath.empty())
 218         {
 219                 if (NLMISC::CFile::fileExists(ClientPatchPath + rFilename))
 220                         sFilePath = ClientPatchPath + rFilename;
 221         }
 222
 223         // if file not found anywhere
 224         if (sFilePath.empty())
 225         {
 226                 ftpOut.FileName = rFilename;
 227                 ftpOut.LocalFileToDelete = false;
 228                 ftpOut.LocalFileExists = false;
 229                 // It happens some time (maybe a bug) that the versionCount is 0... =>
 230                 // it happens if the BNP file is empty (8 bytes)
 231                 ftpOut.FinalFileSize = EmptyBnpFileSize;
 232                 // BNP does not exists : get all the patches version
 233                 for (j = 0; j < rFile.versionCount(); ++j)
 234                 {
 235                         ftpOut.Patches.push_back(rFile.getVersion(j).getVersionNumber());
 236                         ftpOut.PatcheSizes.push_back(rFile.getVersion(j).getPatchSize());
 237                         ftpOut.LastFileDate = rFile.getVersion(j).getTimeStamp();
 238                         ftpOut.FinalFileSize = rFile.getVersion(j).getFileSize();
 239                 }
 240         }
 241         else
 242         {
 243                 // The local BNP file exists : find its version
 244                 uint32 nLocalSize = CFile::getFileSize(sFilePath);
 245                 uint32 nLocalTime = CFile::getFileModificationDate(sFilePath);
 246                 // From the couple time, size look the version of the file
 247                 uint32 nVersionFound = 0xFFFFFFFF;
 248                 // If forceChecksum is wanted (slow), then don't do the test with filesize/date
 249                 if(!forceCheckSumTest)
 250                 {
 251                         for (j = 0; j < rFile.versionCount(); ++j)
 252                         {
 253                                 const CBNPFileVersion &rVersion = rFile.getVersion(j);
 254                                 uint32 nServerSize = rVersion.getFileSize();
 255                                 uint32 nServerTime = rVersion.getTimeStamp();
 256                                 // Does the time and size match a version ?
 257                                 if ((nServerSize == nLocalSize) && (abs((sint32)(nServerTime - nLocalTime)) <= 2) )
 258                                 {
 259                                         nVersionFound = rVersion.getVersionNumber();
 260                                         // break; // ace -> get the last good version (if more than one version of the same file exists)
 261                                 }
 262                         }
 263                 }
 264
 265                 // If the version cannot be found with size and time try with sha1
 266                 if (nVersionFound == 0xFFFFFFFF)
 267                 {
 268                         ucstring sTranslate = dummyI18N("Checking Integrity :") + " " + rFilename;
 269                         setState(true, sTranslate);
 270                         CHashKey hkLocalSHA1 = getSHA1(sFilePath);
 271                         for (j = 0; j < rFile.versionCount(); ++j)
 272                         {
 273                                 const CBNPFileVersion &rVersion = rFile.getVersion(j);
 274                                 CHashKey hkServerSHA1 = rVersion.getHashKey();
 275                                 // Does the sha1 match a version ?
 276                                 if (hkServerSHA1 == hkLocalSHA1)
 277                                 {
 278                                         nVersionFound = rVersion.getVersionNumber();
 279                                         // break; // ace -> same as above
 280                                 }
 281                         }
 282                 }
 283
 284                 // No version available found
 285                 if (nVersionFound == 0xFFFFFFFF)
 286                 {
 287                         ucstring sTranslate = dummyI18N("No Version Found");
 288                         setState(true, sTranslate);
 289                         // Get all patches from beginning (first patch is reference file)
 290                         ftpOut.FileName = rFilename;
 291                         ftpOut.LocalFileToDelete = true;
 292                         ftpOut.LocalFileExists = true;
 293                         // It happens some time (maybe a bug) that the versionCount is 0... =>
 294                         // it happens if the BNP file is empty (8 bytes)
 295                         ftpOut.FinalFileSize = EmptyBnpFileSize;
 296                         // Get all the patches version
 297                         for (j = 0; j < rFile.versionCount(); ++j)
 298                         {
 299                                 ftpOut.Patches.push_back(rFile.getVersion(j).getVersionNumber());
 300                                 ftpOut.PatcheSizes.push_back(rFile.getVersion(j).getPatchSize());
 301                                 ftpOut.LastFileDate = rFile.getVersion(j).getTimeStamp();
 302                                 ftpOut.FinalFileSize = rFile.getVersion(j).getFileSize();
 303                         }
 304                 }
 305                 else // A version of the file has been found
 306                 {
 307                         ucstring sTranslate = dummyI18N("Version Found :") + " " + toString(nVersionFound);
 308                         setState(true, sTranslate);
 309                         // Get All patches from this version !
 310                         ftpOut.FileName = rFilename;
 311                         ftpOut.LocalFileToDelete = false;
 312                         ftpOut.LocalFileExists = true;
 313                         // Go to the version
 314                         for (j = 0; j < rFile.versionCount(); ++j)
 315                                 if (rFile.getVersion(j).getVersionNumber() == nVersionFound)
 316                                         break;
 317
 318                         nlassert(j != rFile.versionCount()); // Not normal if we cant find the version we found previously
 319
 320                         // Point on the next version
 321                         j++;
 322                         // If there are newer versions
 323                         if (j != rFile.versionCount())
 324                         {
 325                                 // Add all version until the last one
 326                                 for (; j < rFile.versionCount(); ++j)
 327                                 {
 328                                         ftpOut.Patches.push_back(rFile.getVersion(j).getVersionNumber());
 329                                         ftpOut.PatcheSizes.push_back(rFile.getVersion(j).getPatchSize());
 330                                         ftpOut.LastFileDate = rFile.getVersion(j).getTimeStamp();
 331                                 }
 332                         }
 333                         // Else this file is up to date !
 334
 335                         // For info, get its final file size
 336                         ftpOut.FinalFileSize= rFile.getVersion(rFile.versionCount()-1).getFileSize();
 337                 }
 338         } // end of else local BNP file exists
 339 }
 340
 341
 342 // ***************************************************************************
 343 void CPatchManager::startScanDataThread()
 344 {
 345         if (ScanDataThread != NULL)
 346         {
 347                 nlwarning ("scan data thread is already running");
 348                 return;
 349         }
 350         if (thread != NULL)
 351         {
 352                 nlwarning ("a thread is already running");
 353                 return;
 354         }
 355
 356         // Reset result
 357         clearDataScanLog();
 358
 359         // Read now the client version and Desc File.
 360         readClientVersionAndDescFile();
 361
 362         // start thread
 363         ScanDataThread = new CScanDataThread();
 364         nlassert (ScanDataThread != NULL);
 365
 366         thread = IThread::create (ScanDataThread);
 367         nlassert (thread != NULL);
 368         thread->start ();
 369 }
 370
 371 // ****************************************************************************
 372 bool CPatchManager::isScanDataThreadEnded(bool &ok)
 373 {
 374         if (ScanDataThread == NULL)
 375         {
 376                 ok = false;
 377                 return true;
 378         }
 379
 380         bool end = ScanDataThread->Ended;
 381         if (end)
 382         {
 383                 ok = ScanDataThread->CheckOk;
 384                 stopScanDataThread();
 385         }
 386
 387         return end;
 388 }
 389
 390 // ****************************************************************************
 391 void CPatchManager::stopScanDataThread()
 392 {
 393         if(ScanDataThread && thread)
 394         {
 395                 thread->wait();
 396                 delete thread;
 397                 thread = NULL;
 398                 delete ScanDataThread;
 399                 ScanDataThread = NULL;
 400         }
 401 }
 402
 403 // ***************************************************************************
 404 void CPatchManager::askForStopScanDataThread()
 405 {
 406         if(!ScanDataThread)
 407                 return;
 408
 409         ScanDataThread->AskForCancel= true;
 410 }
 411
 412 // ***************************************************************************
 413 bool CPatchManager::getDataScanLog(ucstring &text)
 414 {
 415         text.clear();
 416         bool    changed= false;
 417         {
 418                 TSyncDataScanState::CAccessor   ac(&DataScanState);
 419                 CDataScanState  &val= ac.value();
 420                 changed= val.Changed;
 421                 // if changed, build the log
 422                 if(changed)
 423                 {
 424                         for(uint i=0;i<val.FilesWithScanDataError.size();i++)
 425                         {
 426                                 ucstring        str;
 427                                 getCorruptedFileInfo(val.FilesWithScanDataError[i], str);
 428                                 text+= str + "\n";
 429                         }
 430                 }
 431                 // then reset
 432                 val.Changed= false;
 433         }
 434
 435         return changed;
 436 }
 437
 438 // ***************************************************************************
 439 void CPatchManager::addDataScanLogCorruptedFile(const SFileToPatch &ftp)
 440 {
 441         {
 442                 TSyncDataScanState::CAccessor   ac(&DataScanState);
 443                 CDataScanState  &val= ac.value();
 444                 val.FilesWithScanDataError.push_back(ftp);
 445                 val.Changed= true;
 446         }
 447 }
 448
 449 // ***************************************************************************
 450 void CPatchManager::clearDataScanLog()
 451 {
 452         {
 453                 TSyncDataScanState::CAccessor   ac(&DataScanState);
 454                 CDataScanState  &val= ac.value();
 455                 val.FilesWithScanDataError.clear();
 456                 val.Changed= true;
 457         }
 458 }
 459
 460 // ***************************************************************************
 461 void CPatchManager::getCorruptedFileInfo(const SFileToPatch &ftp, ucstring &sTranslate)
 462 {
 463         sTranslate = dummyI18N("Corrupted File: ") + ftp.FileName + " (" +
 464                 toString("%.1f ", (float)ftp.FinalFileSize/1000000.f) + dummyI18N("Mb") + ")";
 465 }
 466
 467
 468 // ****************************************************************************
 469 // ****************************************************************************
 470 // ****************************************************************************
 471 // CScanDataThread
 472 // ****************************************************************************
 473 // ****************************************************************************
 474 // ****************************************************************************
 475
 476 // ****************************************************************************
 477 CScanDataThread::CScanDataThread()
 478 {
 479         AskForCancel= false;
 480         Ended = false;
 481         CheckOk = false;
 482         TotalFileToScan = 1;
 483         CurrentFileScanned = 1;
 484 }
 485
 486 // ****************************************************************************
 487 void CScanDataThread::run ()
 488 {
 489         CPatchManager *pPM = CPatchManager::getInstance();
 490         try
 491         {
 492                 uint32 i;
 493                 // Check if the client version is the same as the server version
 494                 string sClientVersion = pPM->getClientVersion();
 495                 ucstring sTranslate = dummyI18N("Client Version") + " (" + sClientVersion + ") ";
 496                 pPM->setState(true, sTranslate);
 497
 498                 // For all bnp in the description file get all patches to apply
 499                 // depending on the version of the client bnp files
 500                 const CBNPFileSet &rDescFiles = pPM->DescFile.getFiles();
 501                 TotalFileToScan = rDescFiles.fileCount();
 502                 for (i = 0; i < rDescFiles.fileCount(); ++i)
 503                 {
 504                         sTranslate = dummyI18N("Checking File") + " " + rDescFiles.getFile(i).getFileName();
 505                         pPM->setState(true, sTranslate);
 506
 507                         // get list of file to apply to this patch, performing a full checksum test (slow...)
 508                         CPatchManager::SFileToPatch ftp;
 509                         pPM->getPatchFromDesc(ftp, rDescFiles.getFile(i), true);
 510                         // if the file has been found but don't correspond to any local version (SHA1)
 511                         if (ftp.LocalFileExists && ftp.LocalFileToDelete)
 512                         {
 513                                 pPM->addDataScanLogCorruptedFile(ftp);
 514                                 CPatchManager::getCorruptedFileInfo(ftp, sTranslate);
 515                                 pPM->setState(true, sTranslate);
 516                         }
 517                         CurrentFileScanned = i;
 518
 519                         // if the user ask to cancel the thread, stop now
 520                         if(AskForCancel)
 521                                 break;
 522                 }
 523
 524                 sTranslate = dummyI18N("Checking file ended with no error");
 525                 pPM->setState(true, sTranslate);
 526                 CheckOk = true;
 527                 Ended = true;
 528         }
 529         catch (Exception &e)
 530         {
 531                 ucstring sTranslate = dummyI18N("Checking file ended with errors :") + " " + e.what();
 532                 pPM->setState(true, sTranslate);
 533                 CheckOk = false;
 534                 Ended = true;
 535         }
 536 }
 537