nel/src/misc/seven_zip.cpp

   1 // Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
   2 // Copyright (C) 2010  Winch Gate Property Limited
   3 //
   4 // This source file has been modified by the following contributors:
   5 // Copyright (C) 2019-2020  Jan BOON (Kaetemi) <jan.boon@kaetemi.be>
   6 //
   7 // This program is free software: you can redistribute it and/or modify
   8 // it under the terms of the GNU Affero General Public License as
   9 // published by the Free Software Foundation, either version 3 of the
  10 // License, or (at your option) any later version.
  11 //
  12 // This program is distributed in the hope that it will be useful,
  13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 // GNU Affero General Public License for more details.
  16 //
  17 // You should have received a copy of the GNU Affero General Public License
  18 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 #include <nel/misc/seven_zip.h>
  21
  22 #include <nel/misc/types_nl.h>
  23 #include <nel/misc/file.h>
  24 #include <nel/misc/path.h>
  25
  26 #include <seven_zip/7z.h>
  27 #include <seven_zip/7zAlloc.h>
  28 #include <seven_zip/7zCrc.h>
  29 #include <seven_zip/7zVersion.h>
  30 #include <seven_zip/LzmaLib.h>
  31 #include <seven_zip/LzmaDec.h>
  32
  33 #include <memory>
  34
  35 //
  36 // Namespaces
  37 //
  38
  39 using namespace std;
  40 using namespace NLMISC;
  41
  42 #ifdef DEBUG_NEW
  43 #define new DEBUG_NEW
  44 #endif
  45
  46 namespace NLMISC {
  47
  48 /// Input stream class for 7zip archive
  49 class CNel7ZipInStream : public ISeekInStream
  50 {
  51         NLMISC::IStream         *_Stream;
  52
  53 public:
  54         /// Constructor, only allow file stream because 7zip will 'seek' in the stream
  55         CNel7ZipInStream(NLMISC::IStream *s)
  56                 :       _Stream(s)
  57         {
  58                 Read = readFunc;
  59                 Seek = seekFunc;
  60         }
  61
  62         // the read function called by 7zip to read data
  63         static SRes readFunc(const ISeekInStream *object, void *buffer, size_t *size)
  64         {
  65                 try
  66                 {
  67                         CNel7ZipInStream *me = (CNel7ZipInStream*)object;
  68                         uint len = (uint)*size;
  69                         me->_Stream->serialBuffer((uint8*)buffer, len);
  70                         return SZ_OK;
  71                 }
  72                 catch (...)
  73                 {
  74                         return SZ_ERROR_READ;
  75                 }
  76         }
  77
  78         // the seek function called by seven zip to seek inside stream
  79         static SRes seekFunc(const ISeekInStream *object, Int64 *pos, ESzSeek origin)
  80         {
  81                 try
  82                 {
  83                         CNel7ZipInStream *me = (CNel7ZipInStream*)object;
  84                         sint32 offset = (sint32)*pos;
  85                         bool ret = me->_Stream->seek(offset, (NLMISC::IStream::TSeekOrigin)origin);
  86
  87                         if (ret)
  88                         {
  89                                 *pos = (Int64)me->_Stream->getPos();
  90                                 return SZ_OK;
  91                         }
  92                 }
  93                 catch (...)
  94                 {
  95                 }
  96
  97                 return SZ_ERROR_READ;
  98         }
  99 };
 100
 101 bool unpack7Zip(const std::string &sevenZipFile, const std::string &destFileName)
 102 {
 103         nlinfo("Uncompressing 7zip archive '%s' to '%s'", sevenZipFile.c_str(), destFileName.c_str());
 104
 105         // init seven zip
 106         ISzAlloc allocImp;
 107         allocImp.Alloc = SzAlloc;
 108         allocImp.Free = SzFree;
 109
 110         ISzAlloc allocTempImp;
 111         allocTempImp.Alloc = SzAllocTemp;
 112         allocTempImp.Free = SzFreeTemp;
 113
 114         // wrap file in a CIFile
 115         CIFile input(sevenZipFile);
 116         CNel7ZipInStream inStr(&input);
 117
 118         CLookToRead2 lookStream;
 119         lookStream.realStream = &inStr;
 120         LookToRead2_CreateVTable(&lookStream, False);
 121
 122         size_t bufferSize = 1024;
 123
 124         {
 125                 lookStream.buf = (Byte*)ISzAlloc_Alloc(&allocImp, bufferSize);
 126
 127                 if (!lookStream.buf)
 128                 {
 129                         nlerror("Unable to allocate %zu bytes", bufferSize);
 130                         return false;
 131                 }
 132
 133                 lookStream.bufSize = bufferSize;
 134                 lookStream.realStream = &inStr;
 135                 LookToRead2_Init(&lookStream);
 136         }
 137
 138         CrcGenerateTable();
 139
 140         CSzArEx db;
 141         SzArEx_Init(&db);
 142
 143         // unpack the file using the 7zip API
 144         SRes res = SzArEx_Open(&db, &lookStream.vt, &allocImp, &allocTempImp);
 145
 146         if (res != SZ_OK)
 147         {
 148                 nlerror("Failed to open archive file %s", sevenZipFile.c_str());
 149                 return false;
 150         }
 151
 152         if (db.NumFiles != 1)
 153         {
 154                 nlerror("Seven zip archive with more than 1 file are unsupported");
 155                 return false;
 156         }
 157
 158         UInt32 blockIndex = 0xFFFFFFFF; /* it can have any value before first call (if outBuffer = 0) */
 159         Byte *outBuffer = 0; /* it must be 0 before first call for each new archive. */
 160         size_t outBufferSize = 0; /* it can have any value before first call (if outBuffer = 0) */
 161
 162         size_t offset;
 163         size_t outSizeProcessed = 0;
 164
 165         // get the first file
 166         res = SzArEx_Extract(&db, &lookStream.vt, 0, &blockIndex, &outBuffer, &outBufferSize, &offset, &outSizeProcessed, &allocImp, &allocTempImp);
 167
 168         // get the length of first file
 169         size_t nameLen = SzArEx_GetFileNameUtf16(&db, 0, NULL);
 170
 171         ucstring filename;
 172         filename.resize(nameLen);
 173
 174         // write filename into ucstring
 175         SzArEx_GetFileNameUtf16(&db, 0, reinterpret_cast<UInt16 *>(&filename[0]));
 176
 177         // write the extracted file
 178         FILE *outputHandle = nlfopen(destFileName, "wb+");
 179
 180         if (outputHandle == 0)
 181         {
 182                 nlerror("Can not open output file '%s'", destFileName.c_str());
 183                 return false;
 184         }
 185
 186         UInt32 processedSize = (UInt32)fwrite(outBuffer + offset, 1, outSizeProcessed, outputHandle);
 187
 188         if (processedSize != outSizeProcessed)
 189         {
 190                 nlerror("Failed to write %u char to output file '%s'", outSizeProcessed-processedSize, destFileName.c_str());
 191                 return false;
 192         }
 193
 194         fclose(outputHandle);
 195
 196         IAlloc_Free(&allocImp, outBuffer);
 197
 198         // free 7z context
 199         SzArEx_Free(&db, &allocImp);
 200
 201         // ok, all is fine, file is unpacked
 202         return true;
 203 }
 204
 205 bool unpackLZMA(const std::string &lzmaFile, const std::string &destFileName)
 206 {
 207         nldebug("unpackLZMA: decompress LZMA file '%s' to '%s", lzmaFile.c_str(), destFileName.c_str());
 208
 209         // open input file
 210         CIFile inStream(lzmaFile);
 211         uint32 inSize = inStream.getFileSize();
 212
 213         if (inSize < LZMA_PROPS_SIZE + 8)
 214         {
 215                 nlwarning("unpackLZMA: Invalid file size, too small file '%s'", lzmaFile.c_str());
 216                 return false;
 217         }
 218
 219         // allocate input buffer for props
 220         std::vector<uint8> propsBuffer(LZMA_PROPS_SIZE);
 221
 222         // size of LZMA content
 223         inSize -= LZMA_PROPS_SIZE + 8;
 224
 225         // allocate input buffer for lzma data
 226         std::vector<uint8> inBuffer(inSize);
 227
 228         uint64 fileSize = 0;
 229
 230         try
 231         {
 232                 // read props
 233                 inStream.serialBuffer(&propsBuffer[0], LZMA_PROPS_SIZE);
 234
 235                 // read uncompressed size
 236                 inStream.serial(fileSize);
 237
 238                 // read lzma content
 239                 inStream.serialBuffer(&inBuffer[0], inSize);
 240         }
 241         catch(const EReadError &e)
 242         {
 243                 nlwarning("unpackLZMA: Error while reading '%s': %s", lzmaFile.c_str(), e.what());
 244                 return false;
 245         }
 246
 247         // allocate the output buffer
 248         std::vector<uint8> outBuffer(fileSize);
 249
 250         // in and out file sizes
 251         SizeT outProcessed = (SizeT)fileSize;
 252         SizeT inProcessed = (SizeT)inSize;
 253
 254         // decompress the file in memory
 255         sint res = LzmaUncompress(&outBuffer[0], &outProcessed, &inBuffer[0], &inProcessed, &propsBuffer[0], LZMA_PROPS_SIZE);
 256
 257         if (res != 0 || outProcessed != fileSize)
 258         {
 259                 nlwarning("unpackLZMA: Failed to decode lzma file '%s' with status %d", lzmaFile.c_str(), res);
 260                 return false;
 261         }
 262
 263         // store on output buffer
 264         COFile outStream(destFileName);
 265
 266         try
 267         {
 268                 // write content
 269                 outStream.serialBuffer(&outBuffer[0], (uint)fileSize);
 270         }
 271         catch(const EFile &e)
 272         {
 273                 nlwarning("unpackLZMA: Error while writing '%s': %s", destFileName.c_str(), e.what());
 274                 CFile::deleteFile(destFileName);
 275                 return false;
 276         }
 277
 278         return true;
 279 }
 280
 281 bool unpackLZMA(const std::string &lzmaFile, const std::string &destFileName, CHashKey &sha1)
 282 {
 283         nldebug("unpackLZMA: decompress LZMA file '%s' to '%s", lzmaFile.c_str(), destFileName.c_str());
 284
 285         // open input file
 286         CIFile inStream(lzmaFile);
 287         uint32 inSize = inStream.getFileSize();
 288
 289         if (inSize < LZMA_PROPS_SIZE + 8)
 290         {
 291                 nlwarning("unpackLZMA: Invalid file size, too small file '%s'", lzmaFile.c_str());
 292                 return false;
 293         }
 294
 295         // allocate input buffer for props
 296         std::vector<uint8> propsBuffer(LZMA_PROPS_SIZE);
 297
 298         // size of LZMA content
 299         inSize -= LZMA_PROPS_SIZE + 8;
 300
 301         // allocate input buffer for lzma data
 302         std::vector<uint8> inBuffer(inSize);
 303
 304         uint64 fileSize = 0;
 305
 306         try
 307         {
 308                 // read props
 309                 inStream.serialBuffer(&propsBuffer[0], LZMA_PROPS_SIZE);
 310
 311                 // read uncompressed size
 312                 inStream.serial(fileSize);
 313
 314                 // read lzma content
 315                 inStream.serialBuffer(&inBuffer[0], inSize);
 316         }
 317         catch(const EReadError &e)
 318         {
 319                 nlwarning("unpackLZMA: Error while reading '%s': %s", lzmaFile.c_str(), e.what());
 320                 return false;
 321         }
 322
 323         // allocate the output buffer
 324         std::vector<uint8> outBuffer(fileSize);
 325
 326         // in and out file sizes
 327         SizeT outProcessed = (SizeT)fileSize;
 328         SizeT inProcessed = (SizeT)inSize;
 329
 330         // decompress the file in memory
 331         sint res = LzmaUncompress(&outBuffer[0], &outProcessed, &inBuffer[0], &inProcessed, &propsBuffer[0], LZMA_PROPS_SIZE);
 332
 333         if (res != 0 || outProcessed != fileSize)
 334         {
 335                 nlwarning("unpackLZMA: Failed to decode lzma file '%s' with status %d", lzmaFile.c_str(), res);
 336                 return false;
 337         }
 338
 339         // get hash
 340         sha1 = getSHA1(&outBuffer[0], outBuffer.size());
 341
 342         // store on output buffer
 343         COFile outStream(destFileName);
 344
 345         try
 346         {
 347                 // write content
 348                 outStream.serialBuffer(&outBuffer[0], (uint)fileSize);
 349         }
 350         catch(const EFile &e)
 351         {
 352                 nlwarning("unpackLZMA: Error while writing '%s': %s", destFileName.c_str(), e.what());
 353                 CFile::deleteFile(destFileName);
 354                 return false;
 355         }
 356
 357         return true;
 358 }
 359
 360 bool packLZMA(const std::string &srcFileName, const std::string &lzmaFileName)
 361 {
 362         nldebug("packLZMA: compress '%s' to LZMA file '%s", srcFileName.c_str(), lzmaFileName.c_str());
 363
 364         // open file
 365         CIFile inStream(srcFileName);
 366         size_t inSize = inStream.getFileSize();
 367
 368         // file empty
 369         if (!inSize)
 370         {
 371                 nlwarning("packLZMA: File '%s' not found or empty", srcFileName.c_str());
 372                 return false;
 373         }
 374
 375         // allocate input buffer
 376         std::vector<uint8> inBuffer(inSize);
 377
 378         try
 379         {
 380                 // read file in buffer
 381                 inStream.serialBuffer(&inBuffer[0], inSize);
 382         }
 383         catch(const EReadError &e)
 384         {
 385                 nlwarning("packLZMA: Error while reading '%s': %s", srcFileName.c_str(), e.what());
 386                 return false;
 387         }
 388
 389         // allocate output buffer
 390         size_t outSize = (11 * inSize / 10) + 65536; // worst case = 1.1 * size + 64K
 391         std::vector<uint8> outBuffer(outSize);
 392
 393         // allocate buffer for props
 394         size_t outPropsSize = LZMA_PROPS_SIZE;
 395         std::vector<uint8> outProps(outPropsSize);
 396
 397         // compress with best compression and other default settings
 398         sint res = LzmaCompress(&outBuffer[0], &outSize, &inBuffer[0], inSize, &outProps[0], &outPropsSize, 9, 1 << 27, -1, -1, -1, -1, 1);
 399
 400         switch(res)
 401         {
 402                 case SZ_OK:
 403                 {
 404                         // store on output buffer
 405                         COFile outStream(lzmaFileName);
 406
 407                         // unable to create file
 408                         if (!outStream.isOpen())
 409                         {
 410                                 nlwarning("packLZMA: Unable to create '%s'", srcFileName.c_str());
 411                                 return false;
 412                         }
 413
 414                         try
 415                         {
 416                                 // write props
 417                                 outStream.serialBuffer(&outProps[0], (uint)outPropsSize);
 418
 419                                 // write uncompressed size
 420                                 uint64 uncompressSize = inSize;
 421                                 outStream.serial(uncompressSize);
 422
 423                                 // write content
 424                                 outStream.serialBuffer(&outBuffer[0], (uint)outSize);
 425                         }
 426                         catch(const EFile &e)
 427                         {
 428                                 nlwarning("packLZMA: Error while writing '%s': %s", lzmaFileName.c_str(), e.what());
 429                                 CFile::deleteFile(lzmaFileName);
 430                                 return false;
 431                         }
 432
 433                         return true;
 434                 }
 435
 436                 case SZ_ERROR_MEM:
 437                 nlwarning("packLZMA: Memory allocation error while compressing '%s' (input buffer size: %u, output buffer size: %u)", srcFileName.c_str(), (uint)inSize, (uint)outSize);
 438                 break;
 439
 440                 case SZ_ERROR_PARAM:
 441                 nlwarning("packLZMA: Incorrect parameter while compressing '%s'", srcFileName.c_str());
 442                 break;
 443
 444                 case SZ_ERROR_OUTPUT_EOF:
 445                 nlwarning("packLZMA: Output buffer overflow while compressing '%s' (input buffer size: %u, output buffer size: %u)", srcFileName.c_str(), (uint)inSize, (uint)outSize);
 446                 break;
 447
 448                 case SZ_ERROR_THREAD:
 449                 nlwarning("packLZMA: Errors in multithreading functions (only for Mt version)");
 450                 break;
 451
 452                 default:
 453                 nlwarning("packLZMA: Unknown error (%d) while compressing '%s'", res, srcFileName.c_str());
 454         }
 455
 456         return false;
 457 }
 458
 459 }