nel/tools/misc/snp_make/main.cpp

   1 // NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
   2 // Copyright (C) 2010  Winch Gate Property Limited
   3 //
   4 // This source file has been modified by the following contributors:
   5 // Copyright (C) 2014-2020  Jan BOON (Kaetemi) <jan.boon@kaetemi.be>
   6 //
   7 // This program is free software: you can redistribute it and/or modify
   8 // it under the terms of the GNU Affero General Public License as
   9 // published by the Free Software Foundation, either version 3 of the
  10 // License, or (at your option) any later version.
  11 //
  12 // This program is distributed in the hope that it will be useful,
  13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 // GNU Affero General Public License for more details.
  16 //
  17 // You should have received a copy of the GNU Affero General Public License
  18 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 #include "nel/misc/types_nl.h"
  21
  22 #include <stdio.h>
  23 #include <stdlib.h>
  24
  25 #ifdef NL_OS_WINDOWS
  26 #       include <io.h>
  27 #       include <direct.h>
  28 #endif
  29
  30 #include <vector>
  31 #include <string>
  32
  33 #include "nel/misc/debug.h"
  34 #include "nel/misc/file.h"
  35 #include "nel/misc/path.h"
  36 #include "nel/misc/algo.h"
  37 #include "nel/misc/common.h"
  38 #include "nel/misc/streamed_package.h"
  39 #include "nel/misc/seven_zip.h"
  40
  41 using namespace std;
  42 using namespace NLMISC;
  43
  44 // ---------------------------------------------------------------------------
  45
  46 class CWildCard
  47 {
  48 public:
  49         string          Expression;
  50         bool            Not;
  51 };
  52 std::vector<CWildCard>  WildCards;
  53
  54 std::string SourceDirectory;
  55 std::string PackageFileName;
  56 std::string StreamDirectory;
  57
  58 CStreamedPackage Package;
  59
  60 // ---------------------------------------------------------------------------
  61
  62 bool keepFile (const char *fileName)
  63 {
  64         uint i;
  65         bool ifPresent = false;
  66         bool ifTrue = false;
  67         string file = toLowerAscii(CFile::getFilename (fileName));
  68         for (i=0; i<WildCards.size(); i++)
  69         {
  70                 if (WildCards[i].Not)
  71                 {
  72                         // One ifnot condition met and the file is not added
  73                         if (testWildCard(file.c_str(), WildCards[i].Expression.c_str()))
  74                                 return false;
  75                 }
  76                 else
  77                 {
  78                         ifPresent = true;
  79                         ifTrue |= testWildCard(file.c_str(), WildCards[i].Expression.c_str());
  80                 }
  81         }
  82
  83         return !ifPresent || ifTrue;
  84 }
  85
  86 // ---------------------------------------------------------------------------
  87 void usage()
  88 {
  89         printf ("USAGE : \n");
  90         printf ("   snp_make -p <directory_name> <package_file> <stream_directory> [option] ... [option]\n");
  91         printf ("   option : \n");
  92         printf ("      -if wildcard : add the file if it matches the wilcard (at least one 'if' conditions must be met for a file to be adding)\n");
  93         printf ("      -ifnot wildcard : add the file if it doesn't match the wilcard (all the 'ifnot' conditions must be met for a file to be adding)\n");
  94         printf (" Pack the directory to a snp file\n");
  95         printf ("   snp_make -l <package_file>\n");
  96         printf (" List the files contained in the snp file\n");
  97 }
  98
  99 // ---------------------------------------------------------------------------
 100
 101 void generateLZMA(const std::string &sourceFile, const std::string &outputFile)
 102 {
 103         NLMISC::packLZMA(sourceFile, outputFile);
 104
 105         /*
 106         std::string cmd="lzma e ";
 107         cmd+=" "+sourceFile+" "+outputFile;
 108         nlinfo("executing system command: %s",cmd.c_str());
 109 #ifdef NL_OS_WINDOWS
 110         _spawnlp(_P_WAIT, "lzma.exe","lzma.exe", "e", sourceFile.c_str(), outputFile.c_str(), NULL);
 111 #else // NL_OS_WINDOWS
 112         sint error = system (cmd.c_str());
 113         if (error)
 114                 nlwarning("'%s' failed with error code %d", cmd.c_str(), error);
 115 #endif // NL_OS_WINDOWS
 116         */
 117 }
 118
 119 // ---------------------------------------------------------------------------
 120
 121 uint readOptions (int nNbArg, char **ppArgs)
 122 {
 123         uint i;
 124         uint optionCount = 0;
 125         for (i=0; i<(uint)nNbArg; i++)
 126         {
 127                 // If ?
 128                 if ((strcmp (ppArgs[i], "-if") == 0) && ((i+1)<(uint)nNbArg))
 129                 {
 130                         CWildCard card;
 131                         card.Expression = toLower(string(ppArgs[i+1]));
 132                         card.Not = false;
 133                         WildCards.push_back (card);
 134                         optionCount += 2;
 135                 }
 136                 // If not ?
 137                 if ((strcmp (ppArgs[i], "-ifnot") == 0) && ((i+1)<(uint)nNbArg))
 138                 {
 139                         CWildCard card;
 140                         card.Expression = toLower(string(ppArgs[i+1]));
 141                         card.Not = true;
 142                         WildCards.push_back (card);
 143                         optionCount += 2;
 144                 }
 145         }
 146         return optionCount;
 147 }
 148
 149 // ---------------------------------------------------------------------------
 150 int main (int nNbArg, char **ppArgs)
 151 {
 152         NLMISC::CApplicationContext myApplicationContext;
 153
 154         if (nNbArg < 3)
 155         {
 156                 usage();
 157                 return -1;
 158         }
 159
 160         if ((strcmp(ppArgs[1], "/p") == 0) || (strcmp(ppArgs[1], "/P") == 0) ||
 161                 (strcmp(ppArgs[1], "-p") == 0) || (strcmp(ppArgs[1], "-P") == 0))
 162         {
 163                 if (nNbArg < 5)
 164                 {
 165                         usage();
 166                         return -1;
 167                 }
 168
 169                 SourceDirectory = ppArgs[2];
 170                 PackageFileName = ppArgs[3];
 171                 StreamDirectory = ppArgs[4];
 172                 readOptions(nNbArg, ppArgs);
 173
 174                 nldebug("Make streamed package: '%s'", PackageFileName.c_str());
 175
 176                 if (CFile::fileExists(PackageFileName))
 177                 {
 178                         nldebug("Update existing package");
 179                         try
 180                         {
 181                                 CIFile fi;
 182                                 fi.open(PackageFileName);
 183                                 fi.serial(Package);
 184                         }
 185                         catch (Exception &e)
 186                         {
 187                                 nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what());
 188                                 return -1;
 189                         }
 190                 }
 191                 else
 192                 {
 193                         nldebug("New package");
 194                 }
 195
 196                 std::vector<std::string> pathContent; // contains full pathnames
 197                 std::vector<std::string> nameContent; // only filename
 198                 CPath::getPathContent(SourceDirectory, true, false, true, pathContent);
 199                 nameContent.reserve(pathContent.size());
 200                 for (std::vector<std::string>::size_type i = 0; i < pathContent.size(); ++i)
 201                 {
 202                         const std::string &file = pathContent[i];
 203                         if (keepFile(file.c_str()))
 204                         {
 205                                 std::string fileName = NLMISC::toLower(CFile::getFilename(file));
 206                                 // nldebug("File: '%s' ('%s')", file.c_str(), fileName.c_str());
 207                                 nameContent.push_back(fileName);
 208                                 nlassert(nameContent.size() == (i + 1));
 209                         }
 210                         else
 211                         {
 212                                 // Not included in this package
 213                                 pathContent.erase(pathContent.begin() + i);
 214                                 --i;
 215                         }
 216                 }
 217
 218                 std::vector<sint> packageIndex; // index of file in package
 219                 packageIndex.resize(pathContent.size(), -1);
 220
 221                 for (CStreamedPackage::TEntries::size_type i = 0; i < Package.Entries.size(); ++i)
 222                 {
 223                         const CStreamedPackage::CEntry &entry = Package.Entries[i];
 224
 225                         sint foundIndex = -1; // find index in found file list
 226                         for (std::vector<std::string>::size_type j = 0; j < pathContent.size(); ++j)
 227                         {
 228                                 if (nameContent[j] == entry.Name)
 229                                 {
 230                                         foundIndex = j;
 231                                         break;
 232                                 }
 233                         }
 234
 235                         if (foundIndex < 0)
 236                         {
 237                                 nlinfo("File no longer exists: '%s'", entry.Name.c_str());
 238                                 Package.Entries.erase(Package.Entries.begin() + i);
 239                                 --i;
 240                         }
 241                         else
 242                         {
 243                                 // File still exists, map it
 244                                 packageIndex[foundIndex] = i;
 245                         }
 246                 }
 247
 248                 for (std::vector<std::string>::size_type i = 0; i < pathContent.size(); ++i)
 249                 {
 250                         sint pidx = packageIndex[i];
 251                         const std::string &name = nameContent[i];
 252                         const std::string &path = pathContent[i];
 253
 254                         if (pidx < 0)
 255                         {
 256                                 nlinfo("File added: '%s'", name.c_str());
 257                                 pidx = Package.Entries.size();
 258                                 Package.Entries.push_back(CStreamedPackage::CEntry());
 259                                 Package.Entries[pidx].Name = name;
 260                                 Package.Entries[pidx].LastModified = 0;
 261                                 Package.Entries[pidx].Size = 0;
 262                         }
 263                         else
 264                         {
 265                                 nlinfo("File check for changes: '%s'", name.c_str());
 266                         }
 267
 268                         CStreamedPackage::CEntry &entry = Package.Entries[pidx];
 269
 270                         std::string targetLzmaOld; // in case lzma wasn't made make sure it exists a second run
 271                         CStreamedPackage::makePath(targetLzmaOld, entry.Hash);
 272                         targetLzmaOld = StreamDirectory + targetLzmaOld + ".lzma";
 273
 274                         uint32 lastModified = CFile::getFileModificationDate(path);
 275                         uint32 fileSize = CFile::getFileSize(path);
 276                         if (lastModified > entry.LastModified || fileSize != entry.Size || !CFile::fileExists(targetLzmaOld))
 277                         {
 278                                 entry.LastModified = lastModified;
 279
 280                                 nlinfo("Calculate file hash");
 281                                 CHashKey hash = getSHA1(path, true);
 282                                 /*nldebug("%s", hash.toString().c_str());
 283                                 std::string hashPath;
 284                                 CStreamedPackage::makePath(hashPath, hash);
 285                                 nldebug("%s", hashPath.c_str());*/
 286
 287                                 if (hash == entry.Hash && fileSize == entry.Size)
 288                                 {
 289                                         // File has not changed
 290                                 }
 291                                 else
 292                                 {
 293                                         nlinfo("File changed");
 294                                         entry.Hash = hash;
 295                                         entry.Size = fileSize;
 296                                 }
 297
 298                                 std::string targetLzma; // in case lzma wasn't made make sure it exists a second run
 299                                 CStreamedPackage::makePath(targetLzma, entry.Hash);
 300                                 targetLzma = StreamDirectory + targetLzma + ".lzma";
 301
 302                                 if (!CFile::fileExists(targetLzma))
 303                                 {
 304                                         // make the compressed file
 305                                         nlinfo("%s -> %s", path.c_str(), targetLzma.c_str());
 306                                         CFile::createDirectoryTree(CFile::getPath(targetLzma));
 307                                         generateLZMA(path, targetLzma);
 308                                 }
 309                         }
 310                 }
 311
 312                 try
 313                 {
 314                         nldebug("Store package '%s'", PackageFileName.c_str());
 315                         COFile fo;
 316                         fo.open(PackageFileName);
 317                         fo.serial(Package);
 318                 }
 319                 catch (Exception &e)
 320                 {
 321                         nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what());
 322                         return -1;
 323                 }
 324
 325                 return 0;
 326         }
 327
 328         if ((strcmp(ppArgs[1], "/l") == 0) || (strcmp(ppArgs[1], "/L") == 0) ||
 329                 (strcmp(ppArgs[1], "-l") == 0) || (strcmp(ppArgs[1], "-L") == 0))
 330         {
 331                 PackageFileName = ppArgs[2];
 332                 if (!CFile::fileExists(PackageFileName))
 333                 {
 334                         nlwarning("ERROR (snp_make) : package doesn't exist: '%s'", PackageFileName.c_str());
 335                         return -1;
 336                 }
 337
 338                 try
 339                 {
 340                         CIFile fi;
 341                         fi.open(PackageFileName);
 342                         fi.serial(Package);
 343                 }
 344                 catch (Exception &e)
 345                 {
 346                         nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what());
 347                         return -1;
 348                 }
 349
 350                 for (CStreamedPackage::TEntries::const_iterator it(Package.Entries.begin()), end(Package.Entries.end()); it != end; ++it)
 351                 {
 352                         const CStreamedPackage::CEntry &entry = (*it);
 353
 354                         printf("List files in '%s'", PackageFileName.c_str());
 355                         printf("%s { Hash: '%s', Size: '%u', LastModified: '%u' }", entry.Name.c_str(), entry.Hash.toString().c_str(), entry.Size, entry.LastModified);
 356                 }
 357
 358                 return 0;
 359         }
 360
 361         usage ();
 362         return -1;
 363 }