1 // Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
2 // Copyright (C) 2010 Winch Gate Property Limited
4 // This program is free software: you can redistribute it and/or modify
5 // it under the terms of the GNU Affero General Public License as
6 // published by the Free Software Foundation, either version 3 of the
7 // License, or (at your option) any later version.
9 // This program is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 // GNU Affero General Public License for more details.
14 // You should have received a copy of the GNU Affero General Public License
15 // along with this program. If not, see <http://www.gnu.org/licenses/>.
18 #include "filesextractor.h"
19 #include "operation.h"
22 #include "nel/misc/big_file.h"
23 #include "nel/misc/callback.h"
24 #include "nel/misc/file.h"
25 #include "nel/misc/path.h"
32 #include "qzipreader.h"
36 #ifndef FILE_ATTRIBUTE_READONLY
37 #define FILE_ATTRIBUTE_READONLY 0x1
39 #ifndef FILE_ATTRIBUTE_HIDDEN
40 #define FILE_ATTRIBUTE_HIDDEN 0x2
43 #ifndef FILE_ATTRIBUTE_SYSTEM
44 #define FILE_ATTRIBUTE_SYSTEM 0x4
47 #ifndef FILE_ATTRIBUTE_DIRECTORY
48 #define FILE_ATTRIBUTE_DIRECTORY 0x10
51 #ifndef FILE_ATTRIBUTE_ARCHIVE
52 #define FILE_ATTRIBUTE_ARCHIVE 0x20
55 #ifndef FILE_ATTRIBUTE_DEVICE
56 #define FILE_ATTRIBUTE_DEVICE 0x40
59 #ifndef FILE_ATTRIBUTE_NORMAL
60 #define FILE_ATTRIBUTE_NORMAL 0x80
63 #ifndef FILE_ATTRIBUTE_TEMPORARY
64 #define FILE_ATTRIBUTE_TEMPORARY 0x100
67 #ifndef FILE_ATTRIBUTE_SPARSE_FILE
68 #define FILE_ATTRIBUTE_SPARSE_FILE 0x200
71 #ifndef FILE_ATTRIBUTE_REPARSE_POINT
72 #define FILE_ATTRIBUTE_REPARSE_POINT 0x400
75 #ifndef FILE_ATTRIBUTE_COMPRESSED
76 #define FILE_ATTRIBUTE_COMPRESSED 0x800
79 #ifndef FILE_ATTRIBUTE_OFFLINE
80 #define FILE_ATTRIBUTE_OFFLINE 0x1000
83 #ifndef FILE_ATTRIBUTE_ENCRYPTED
84 #define FILE_ATTRIBUTE_ENCRYPTED 0x4000
87 #define FILE_ATTRIBUTE_UNIX_EXTENSION 0x8000 /* trick for Unix */
89 #define FILE_ATTRIBUTE_WINDOWS 0x7fff
90 #define FILE_ATTRIBUTE_UNIX 0xffff0000
92 #define kInputBufSize ((size_t)1 << 18)
94 bool Set7zFileAttrib(const QString
&filename
, uint32 fileAttributes
)
96 if (filename
.isEmpty()) return false;
98 bool attrReadOnly
= (fileAttributes
& FILE_ATTRIBUTE_READONLY
) != 0;
99 bool attrHidden
= (fileAttributes
& FILE_ATTRIBUTE_HIDDEN
) != 0;
100 bool attrSystem
= (fileAttributes
& FILE_ATTRIBUTE_SYSTEM
) != 0;
101 bool attrDir
= (fileAttributes
& FILE_ATTRIBUTE_DIRECTORY
) != 0;
102 bool attrArchive
= (fileAttributes
& FILE_ATTRIBUTE_ARCHIVE
) != 0;
103 bool attrDevice
= (fileAttributes
& FILE_ATTRIBUTE_DEVICE
) != 0;
104 bool attrNormal
= (fileAttributes
& FILE_ATTRIBUTE_NORMAL
) != 0;
105 bool attrTemp
= (fileAttributes
& FILE_ATTRIBUTE_TEMPORARY
) != 0;
106 bool attrSparceFile
= (fileAttributes
& FILE_ATTRIBUTE_SPARSE_FILE
) != 0;
107 bool attrReparsePoint
= (fileAttributes
& FILE_ATTRIBUTE_REPARSE_POINT
) != 0;
108 bool attrCompressed
= (fileAttributes
& FILE_ATTRIBUTE_COMPRESSED
) != 0;
109 bool attrOffline
= (fileAttributes
& FILE_ATTRIBUTE_OFFLINE
) != 0;
110 bool attrEncrypted
= (fileAttributes
& FILE_ATTRIBUTE_ENCRYPTED
) != 0;
111 bool attrUnix
= (fileAttributes
& FILE_ATTRIBUTE_UNIX_EXTENSION
) != 0;
113 uint32 unixAttributes
= (fileAttributes
& FILE_ATTRIBUTE_UNIX
) >> 16;
114 uint32 windowsAttributes
= fileAttributes
& FILE_ATTRIBUTE_WINDOWS
;
116 // qDebug() << "attribs" << QByteArray::fromRawData((const char*)&fileAttributes, 4).toHex();
119 SetFileAttributesW((wchar_t*)filename
.utf16(), windowsAttributes
);
121 std::string name
= filename
.toUtf8().constData();
123 mode_t current_umask
= umask(0); // get and set the umask
124 umask(current_umask
); // restore the umask
125 mode_t mask
= 0777 & (~current_umask
);
127 struct stat stat_info
;
129 if (lstat(name
.c_str(), &stat_info
) != 0)
131 nlwarning("Unable to get file attributes for %s", name
.c_str());
137 stat_info
.st_mode
= unixAttributes
;
139 // ignore symbolic links
140 if (!S_ISLNK(stat_info
.st_mode
))
142 if (S_ISREG(stat_info
.st_mode
))
144 chmod(name
.c_str(), stat_info
.st_mode
& mask
);
146 else if (S_ISDIR(stat_info
.st_mode
))
148 // user/7za must be able to create files in this directory
149 stat_info
.st_mode
|= (S_IRUSR
| S_IWUSR
| S_IXUSR
);
150 chmod(name
.c_str(), stat_info
.st_mode
& mask
);
154 else if (!S_ISLNK(stat_info
.st_mode
) && !S_ISDIR(stat_info
.st_mode
) && attrReadOnly
)
156 // do not use chmod on a link
157 // Remark : FILE_ATTRIBUTE_READONLY ignored for directory.
158 // Only Windows Attributes
160 // octal!, clear write permission bits
161 stat_info
.st_mode
&= ~0222;
163 chmod(name
.c_str(), stat_info
.st_mode
& mask
);
171 #define new DEBUG_NEW
174 #define SZ_ERROR_INTERRUPTED 18
176 class Q7zFile
: public ISeekInStream
181 Q7zFile(const QString
&filename
):m_file(filename
)
193 return m_file
.open(QFile::ReadOnly
);
196 // the read function called by 7zip to read data
197 static SRes
readFunc(const ISeekInStream
*object
, void *buffer
, size_t *size
)
199 Q7zFile
*me
= (Q7zFile
*)object
;
201 len
= me
->m_file
.read((char*)buffer
, len
);
210 return SZ_ERROR_READ
;
214 // the seek function called by seven zip to seek inside stream
215 static SRes
seekFunc(const ISeekInStream
*object
, Int64
*pos
, ESzSeek origin
)
217 Q7zFile
*me
= (Q7zFile
*)object
;
222 case SZ_SEEK_SET
: newPos
= *pos
; break;
223 case SZ_SEEK_CUR
: newPos
= me
->m_file
.pos() + *pos
; break;
224 case SZ_SEEK_END
: newPos
= me
->m_file
.size() - *pos
; break;
227 if (me
->m_file
.seek(newPos
))
234 return SZ_ERROR_READ
;
239 CFilesExtractor::CFilesExtractor(IOperationProgressListener
*listener
):m_listener(listener
)
243 CFilesExtractor::~CFilesExtractor()
247 void CFilesExtractor::setSourceFile(const QString
&src
)
252 void CFilesExtractor::setDestinationDirectory(const QString
&dst
)
254 m_destinationDirectory
= dst
;
257 bool CFilesExtractor::exec()
259 if (m_sourceFile
.isEmpty() || m_destinationDirectory
.isEmpty()) return false;
261 if (m_listener
) m_listener
->operationPrepare();
263 QFile
file(m_sourceFile
);
265 // open archive file to check format
266 if (!file
.open(QFile::ReadOnly
)) return false;
268 // read 2 first bytes
269 QByteArray header
= file
.read(2);
274 // create destination directory
276 dir
.mkpath(m_destinationDirectory
);
278 // compare to supported formats and call the appropriate decompressor
289 if (QFileInfo(m_sourceFile
).suffix().toLower() == "bnp")
294 nlwarning("Unsupported format for file %s", Q2C(m_sourceFile
));
298 static uint32
convertWindowsFileTimeToUnixTimestamp(const CNtfsFileTime
&nt
)
300 // first, convert it into second since jan1, 1601
301 uint64 t
= nt
.Low
| (uint64(nt
.High
) << 32);
303 // offset to convert Windows file times to UNIX timestamp
304 uint64 offset
= UINT64_CONSTANT(116444736000000000);
306 // adjust time base to unix epoch base
309 // convert the resulting time into seconds
311 t
/= 1000; // millisec
314 // return the resulting time
318 bool CFilesExtractor::extract7z()
320 Q7zFile
inFile(m_sourceFile
);
324 nlwarning("Unable to open %s", Q2C(m_sourceFile
));
326 if (m_listener
) m_listener
->operationFail(QApplication::tr("Unable to open %1").arg(m_sourceFile
));
333 // register allocators
335 allocImp
.Alloc
= SzAlloc
;
336 allocImp
.Free
= SzFree
;
338 ISzAlloc allocTempImp
;
339 allocTempImp
.Alloc
= SzAllocTemp
;
340 allocTempImp
.Free
= SzFreeTemp
;
342 // register the files read handlers
343 CLookToRead2 lookStream
;
344 LookToRead2_CreateVTable(&lookStream
, False
);
345 lookStream
.buf
= (Byte
*)ISzAlloc_Alloc(&allocImp
, kInputBufSize
);
349 nlwarning("Unable to allocate %u bytes", (uint32
)kInputBufSize
);
351 if (m_listener
) m_listener
->operationFail(QApplication::tr("Unable to allocate %1 bytes").arg(kInputBufSize
));
355 LookToRead2_Init(&lookStream
);
357 lookStream
.bufSize
= kInputBufSize
;
358 lookStream
.realStream
= &inFile
;
359 LookToRead2_Init(&lookStream
);
368 qint64 total
= 0, totalUncompressed
= 0;
372 SRes res
= SzArEx_Open(&db
, &lookStream
.vt
, &allocImp
, &allocTempImp
);
376 // process each file in archive
377 for (UInt32 i
= 0; i
< db
.NumFiles
; ++i
)
379 bool isDir
= SzArEx_IsDir(&db
, i
) != 0;
381 if (!isDir
) total
+= SzArEx_GetFileSize(&db
, i
);
386 m_listener
->operationInit(0, total
);
387 m_listener
->operationStart();
390 // variables used for decompression
391 UInt32 blockIndex
= 0xFFFFFFFF;
392 Byte
*outBuffer
= NULL
;
393 size_t outBufferSize
= 0;
395 // process each file in archive
396 for (UInt32 i
= 0; i
< db
.NumFiles
; ++i
)
398 if (m_listener
&& m_listener
->operationShouldStop())
400 res
= SZ_ERROR_INTERRUPTED
;
405 size_t outSizeProcessed
= 0;
407 bool isDir
= SzArEx_IsDir(&db
, i
) != 0;
409 size_t len
= SzArEx_GetFileNameUtf16(&db
, i
, NULL
);
415 temp
= (UInt16
*)SzAlloc(NULL
, tempSize
* sizeof(temp
[0]));
423 SzArEx_GetFileNameUtf16(&db
, i
, temp
);
425 QString path
= QString::fromUtf16(temp
);
426 QString filename
= QFileInfo(path
).fileName();
428 QString destPath
= m_destinationDirectory
+ '/' + path
;
430 // get uncompressed size
431 quint64 uncompressedSize
= SzArEx_GetFileSize(&db
, i
);
433 // get modification time
434 quint32 modificationTime
= 0, creationTime
= 0;
436 if (SzBitWithVals_Check(&db
.MTime
, i
))
438 modificationTime
= convertWindowsFileTimeToUnixTimestamp(db
.MTime
.Vals
[i
]);
441 if (SzBitWithVals_Check(&db
.CTime
, i
))
443 creationTime
= convertWindowsFileTimeToUnixTimestamp(db
.CTime
.Vals
[i
]);
448 QDir().mkpath(destPath
);
452 // check if file exists
453 if (QFile::exists(destPath
))
455 QFileInfo
currentFileInfo(destPath
);
457 // skip file if same size and same modification date
458 if (currentFileInfo
.lastModified().toTime_t() == modificationTime
&& currentFileInfo
.size() == uncompressedSize
)
461 totalUncompressed
+= uncompressedSize
;
463 if (m_listener
) m_listener
->operationProgress(totalUncompressed
, filename
);
469 if (m_listener
) m_listener
->operationProgress(totalUncompressed
, filename
);
471 res
= SzArEx_Extract(&db
, &lookStream
.vt
, i
, &blockIndex
, &outBuffer
, &outBufferSize
,
472 &offset
, &outSizeProcessed
, &allocImp
, &allocTempImp
);
474 if (res
!= SZ_OK
) break;
476 QString destSubPath
= QFileInfo(destPath
).absolutePath();
478 // create file directory
479 if (!QDir().mkpath(destSubPath
))
481 nlwarning("Unable to create directory %s", Q2C(destSubPath
));
485 QSaveFile
outFile(destPath
);
487 if (!outFile
.open(QFile::WriteOnly
))
489 nlwarning("Unable to open file %s", Q2C(destPath
));
491 error
= QApplication::tr("Unable to open output file %1").arg(destPath
);
496 qint64 currentSizeToProcess
= outSizeProcessed
;
500 qint64 currentProcessedSize
= outFile
.write((const char*)(outBuffer
+ offset
), currentSizeToProcess
);
502 // errors only occur when returned size is -1
503 if (currentProcessedSize
< 0) break;
505 offset
+= currentProcessedSize
;
506 currentSizeToProcess
-= currentProcessedSize
;
508 while (currentSizeToProcess
> 0);
510 if (offset
!= outSizeProcessed
)
512 nlwarning("Unable to write output file %s (%u bytes written but expecting %u bytes)", Q2C(destPath
), (uint32
)offset
, (uint32
)outSizeProcessed
);
514 error
= QApplication::tr("Unable to write output file %1 (%2 bytes written but expecting %3 bytes)").arg(destPath
).arg(offset
).arg(outSizeProcessed
);
521 totalUncompressed
+= uncompressedSize
;
523 if (m_listener
) m_listener
->operationProgress(totalUncompressed
, filename
);
526 if (SzBitWithVals_Check(&db
.Attribs
, i
))
528 Set7zFileAttrib(destPath
, db
.Attribs
.Vals
[i
]);
531 // set modification time
532 if (!NLMISC::CFile::setFileModificationDate(qToUtf8(destPath
), modificationTime
))
534 nlwarning("Unable to change date of %s", Q2C(destPath
));
538 IAlloc_Free(&allocImp
, outBuffer
);
542 SzArEx_Free(&db
, &allocImp
);
543 ISzAlloc_Free(&allocImp
, lookStream
.buf
);
550 m_listener
->operationSuccess(totalUncompressed
);
554 case SZ_ERROR_INTERRUPTED
:
555 if (m_listener
) m_listener
->operationStop();
558 case SZ_ERROR_UNSUPPORTED
:
559 error
= QApplication::tr("7zip decoder doesn't support this archive");
563 error
= QApplication::tr("Unable to allocate memory");
567 error
= QApplication::tr("7zip decoder doesn't support this archive");
570 case SZ_ERROR_INPUT_EOF
:
571 error
= QApplication::tr("File %1 is corrupted, unable to uncompress it").arg(m_sourceFile
);
575 // error already defined
579 error
= QApplication::tr("Error %1").arg(res
);
582 if (m_listener
) m_listener
->operationFail(error
);
587 bool CFilesExtractor::extractZip()
589 QZipReader
reader(m_sourceFile
);
591 QDir
baseDir(m_destinationDirectory
);
593 // create directories first
594 QList
<QZipReader::FileInfo
> allFiles
= reader
.fileInfoList();
596 qint64 totalSize
= 0, currentSize
= 0;
598 foreach (const QZipReader::FileInfo
&fi
, allFiles
)
602 const QString absPath
= m_destinationDirectory
+ QDir::separator() + fi
.filePath
;
604 if (!baseDir
.mkpath(fi
.filePath
))
606 nlwarning("Unable to create directory %s", Q2C(fi
.filePath
));
608 if (m_listener
) m_listener
->operationFail(QApplication::tr("Unable to create directory %1").arg(fi
.filePath
));
612 if (!QFile::setPermissions(absPath
, fi
.permissions
))
614 nlwarning("Unable to change permissions of %s", Q2C(absPath
));
616 if (m_listener
) m_listener
->operationFail(QApplication::tr("Unable to set permissions of %1").arg(absPath
));
621 totalSize
+= fi
.size
;
626 m_listener
->operationInit(0, totalSize
);
627 m_listener
->operationStart();
630 // client won't use symbolic links so don't process them
632 foreach (const QZipReader::FileInfo
&fi
, allFiles
)
634 const QString absPath
= m_destinationDirectory
+ QDir::separator() + fi
.filePath
;
638 if (m_listener
&& m_listener
->operationShouldStop())
640 m_listener
->operationStop();
644 QSaveFile
f(absPath
);
646 if (!f
.open(QIODevice::WriteOnly
))
648 nlwarning("Unable to open %s", Q2C(absPath
));
650 if (m_listener
) m_listener
->operationFail(QApplication::tr("Unable to open %1").arg(absPath
));
654 currentSize
+= f
.write(reader
.fileData(fi
.filePath
));
656 if (!f
.setPermissions(fi
.permissions
))
658 nlwarning("Unable to change permissions of %s", Q2C(absPath
));
663 // set the right modification date
664 if (!NLMISC::CFile::setFileModificationDate(qToUtf8(absPath
), fi
.lastModified
.toTime_t()))
666 nlwarning("Unable to change date of %s", Q2C(absPath
));
669 if (m_listener
) m_listener
->operationProgress(currentSize
, QFileInfo(absPath
).fileName());
675 m_listener
->operationSuccess(totalSize
);
681 bool CFilesExtractor::progress(const std::string
&filename
, uint32 currentSize
, uint32 totalSize
)
683 if (m_listener
&& m_listener
->operationShouldStop())
685 m_listener
->operationStop();
690 if (currentSize
== 0)
694 m_listener
->operationInit(0, (qint64
)totalSize
);
695 m_listener
->operationStart();
699 if (m_listener
) m_listener
->operationProgress((qint64
)currentSize
, qFromUtf8(filename
));
701 if (currentSize
== totalSize
)
705 m_listener
->operationSuccess((qint64
)totalSize
);
712 bool CFilesExtractor::extractBnp()
716 NLMISC::CBigFile::TUnpackProgressCallback cbMethod
= NLMISC::CBigFile::TUnpackProgressCallback(this, &CFilesExtractor::progress
);
720 if (NLMISC::CBigFile::unpack(qToUtf8(m_sourceFile
), qToUtf8(m_destinationDirectory
), &cbMethod
))
725 if (m_listener
&& m_listener
->operationShouldStop())
728 m_listener
->operationStop();
735 catch(const NLMISC::EDiskFullError
&e
)
737 nlwarning("Disk full when extracting %s to %s", Q2C(m_sourceFile
), Q2C(m_destinationDirectory
));
739 error
= QApplication::tr("disk full");
741 catch(const NLMISC::EWriteError
&e
)
743 nlwarning("Write error when extracting %s to %s", Q2C(m_sourceFile
), Q2C(m_destinationDirectory
));
745 error
= QApplication::tr("unable to write %1").arg(qFromUtf8(e
.Filename
));
747 catch(const NLMISC::EReadError
&e
)
749 nlwarning("Read error when extracting %s to %s", Q2C(m_sourceFile
), Q2C(m_destinationDirectory
));
751 error
= QApplication::tr("unable to read %1").arg(qFromUtf8(e
.Filename
));
753 catch(const std::exception
&e
)
755 nlwarning("Unknown exception when extracting %s to %s", Q2C(m_sourceFile
), Q2C(m_destinationDirectory
));
757 error
= QApplication::tr("failed (%1)").arg(qFromUtf8(e
.what()));
760 if (m_listener
) m_listener
->operationFail(QApplication::tr("Unable to unpack %1 to %2: %3").arg(m_sourceFile
).arg(m_destinationDirectory
).arg(error
));