Merge branch 'ryzom/ark-features' into main/gingo-test
[ryzomcore.git] / nel / src / misc / big_file.cpp
blob7d68c18c216829f6945ad526d059d4e090d4a2fe
1 // NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
2 // Copyright (C) 2010 Winch Gate Property Limited
3 //
4 // This source file has been modified by the following contributors:
5 // Copyright (C) 2020 Jan BOON (Kaetemi) <jan.boon@kaetemi.be>
6 //
7 // This program is free software: you can redistribute it and/or modify
8 // it under the terms of the GNU Affero General Public License as
9 // published by the Free Software Foundation, either version 3 of the
10 // License, or (at your option) any later version.
12 // This program is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 // GNU Affero General Public License for more details.
17 // You should have received a copy of the GNU Affero General Public License
18 // along with this program. If not, see <http://www.gnu.org/licenses/>.
20 #include "stdmisc.h"
22 #include "nel/misc/file.h"
23 #include "nel/misc/big_file.h"
24 #include "nel/misc/path.h"
26 using namespace std;
27 using namespace NLMISC;
29 #ifdef DEBUG_NEW
30 #define new DEBUG_NEW
31 #endif
33 namespace NLMISC {
35 //CBigFile *CBigFile::_Singleton = NULL;
36 NLMISC_SAFE_SINGLETON_IMPL(CBigFile);
38 // ***************************************************************************
39 void CBigFile::releaseInstance()
41 if (_Instance)
43 NLMISC::INelContext::getInstance().releaseSingletonPointer("CBigFile", _Instance);
44 delete _Instance;
45 _Instance = NULL;
48 // ***************************************************************************
49 CBigFile::CThreadFileArray::CThreadFileArray()
51 _CurrentId = 0;
54 // ***************************************************************************
55 CBigFile::CThreadFileArray::~CThreadFileArray()
57 vector<CHandleFile> *ptr = (vector<CHandleFile>*)_TDS.getPointer();
58 if (ptr) delete ptr;
61 // ***************************************************************************
62 uint32 CBigFile::CThreadFileArray::allocate()
64 return _CurrentId++;
67 // ***************************************************************************
68 CBigFile::CHandleFile &CBigFile::CThreadFileArray::get(uint32 index)
70 // If the thread struct ptr is NULL, must allocate it.
71 vector<CHandleFile> *ptr= (vector<CHandleFile>*)_TDS.getPointer();
72 if(ptr==NULL)
74 ptr= new vector<CHandleFile>;
75 _TDS.setPointer(ptr);
78 // if the vector is not allocated, allocate it (empty entries filled with NULL => not opened FILE* in this thread)
79 if(index>=ptr->size())
81 ptr->resize((ptrdiff_t)index + 1);
84 return (*ptr)[index];
88 // ***************************************************************************
89 void CBigFile::currentThreadFinished()
91 _ThreadFileArray.currentThreadFinished();
94 // ***************************************************************************
95 void CBigFile::CThreadFileArray::currentThreadFinished()
97 vector<CHandleFile> *ptr= (vector<CHandleFile>*)_TDS.getPointer();
98 if (ptr==NULL) return;
99 for (uint k = 0; k < ptr->size(); ++k)
101 if ((*ptr)[k].File)
103 fclose((*ptr)[k].File);
104 (*ptr)[k].File = NULL;
107 delete ptr;
108 _TDS.setPointer(NULL);
112 // ***************************************************************************
113 //CBigFile::CBigFile ()
117 //// ***************************************************************************
118 //CBigFile &CBigFile::getInstance ()
120 // if (_Singleton == NULL)
121 // {
122 // _Singleton = new CBigFile();
123 // }
124 // return *_Singleton;
127 // ***************************************************************************
128 bool CBigFile::add (const std::string &sBigFileName, uint32 nOptions)
130 // Is already the same bigfile name ?
131 string bigfilenamealone = toLowerAscii(CFile::getFilename (sBigFileName));
132 if (_BNPs.find(bigfilenamealone) != _BNPs.end())
134 nlwarning ("CBigFile::add : bigfile %s already added.", bigfilenamealone.c_str());
135 return false;
138 // Create the new bnp entry
139 BNP &bnp = _BNPs[bigfilenamealone];
141 bnp.BigFileName= sBigFileName;
143 // Allocate a new ThreadSafe FileId for this bnp.
144 bnp.ThreadFileId= _ThreadFileArray.allocate();
146 // Get a ThreadSafe handle on the file
147 CHandleFile &handle= _ThreadFileArray.get(bnp.ThreadFileId);
149 // Open the big file.
150 handle.File = nlfopen (sBigFileName, "rb");
151 if (handle.File == NULL)
152 return false;
154 // Used internally by CBigFile, use optimizations and lower case of filenames
155 bnp.InternalUse = true;
157 // read BNP header
158 if (!bnp.readHeader(handle.File))
160 fclose (handle.File);
161 handle.File = NULL;
162 return false;
164 if (nOptions&BF_CACHE_FILE_ON_OPEN)
165 bnp.CacheFileOnOpen = true;
166 else
167 bnp.CacheFileOnOpen = false;
169 if (!(nOptions&BF_ALWAYS_OPENED))
171 fclose (handle.File);
172 handle.File = NULL;
173 bnp.AlwaysOpened = false;
175 else
177 bnp.AlwaysOpened = true;
180 //nldebug("BigFile : added bnp '%s' to the collection", bigfilenamealone.c_str());
182 return true;
185 // ***************************************************************************
186 void CBigFile::remove (const std::string &sBigFileName)
188 if (_BNPs.find (sBigFileName) != _BNPs.end())
190 map<string, BNP>::iterator it = _BNPs.find (sBigFileName);
191 BNP &rbnp = it->second;
192 // Get a ThreadSafe handle on the file
193 CHandleFile &handle= _ThreadFileArray.get(rbnp.ThreadFileId);
194 // close it if needed
195 if (handle.File != NULL)
197 fclose (handle.File);
198 handle.File= NULL;
201 _BNPs.erase (it);
205 CBigFile::BNP::BNP() : FileNames(NULL), ThreadFileId(0), CacheFileOnOpen(false), AlwaysOpened(false), InternalUse(false), OffsetFromBeginning(0)
209 CBigFile::BNP::~BNP()
211 if (FileNames)
213 delete[] FileNames;
214 FileNames = NULL;
218 //// ***************************************************************************
219 bool CBigFile::BNP::readHeader()
221 // Only external use
222 if (InternalUse || BigFileName.empty()) return false;
224 FILE *f = nlfopen (BigFileName, "rb");
225 if (f == NULL) return false;
227 bool res = readHeader(f);
228 fclose (f);
230 return res;
233 //// ***************************************************************************
234 bool CBigFile::BNP::readHeader(FILE *file)
236 if (file == NULL) return false;
238 uint32 nFileSize=CFile::getFileSize (file);
240 // Result
241 if (nlfseek64 (file, nFileSize-4, SEEK_SET) != 0)
243 return false;
246 if (fread (&OffsetFromBeginning, sizeof(uint32), 1, file) != 1)
248 return false;
251 #ifdef NL_BIG_ENDIAN
252 NLMISC_BSWAP32(OffsetFromBeginning);
253 #endif
255 if (nlfseek64 (file, OffsetFromBeginning, SEEK_SET) != 0)
257 return false;
260 // Read the file count
261 uint32 nNbFile;
262 if (fread (&nNbFile, sizeof(uint32), 1, file) != 1)
264 return false;
267 #ifdef NL_BIG_ENDIAN
268 NLMISC_BSWAP32(nNbFile);
269 #endif
271 map<string, BNPFile> tempMap;
273 if (!InternalUse) SFiles.clear();
275 for (uint32 i = 0; i < nNbFile; ++i)
277 uint8 nStringSize;
278 if (fread (&nStringSize, 1, 1, file) != 1)
280 return false;
283 char sFileName[256];
284 if (nStringSize)
286 if (fread(sFileName, 1, nStringSize, file) != nStringSize)
288 return false;
291 sFileName[nStringSize] = 0;
293 uint32 nFileSize2;
294 if (fread (&nFileSize2, sizeof(uint32), 1, file) != 1)
296 return false;
299 #ifdef NL_BIG_ENDIAN
300 NLMISC_BSWAP32(nFileSize2);
301 #endif
303 uint32 nFilePos;
304 if (fread (&nFilePos, sizeof(uint32), 1, file) != 1)
306 return false;
309 #ifdef NL_BIG_ENDIAN
310 NLMISC_BSWAP32(nFilePos);
311 #endif
313 if (InternalUse)
315 BNPFile bnpfTmp;
316 bnpfTmp.Pos = nFilePos;
317 bnpfTmp.Size = nFileSize2;
318 tempMap.insert (make_pair(toLowerAscii(string(sFileName)), bnpfTmp));
320 else
322 SBNPFile bnpfTmp;
323 bnpfTmp.Name = sFileName;
324 bnpfTmp.Pos = nFilePos;
325 bnpfTmp.Size = nFileSize2;
326 SFiles.push_back(bnpfTmp);
330 if (nlfseek64 (file, 0, SEEK_SET) != 0)
332 return false;
335 // Convert temp map
336 if (InternalUse && nNbFile > 0)
338 uint nSize = 0, nNb = 0;
339 map<string,BNPFile>::iterator it = tempMap.begin();
340 while (it != tempMap.end())
342 nSize += (uint)it->first.size() + 1;
343 nNb++;
344 it++;
347 if (FileNames)
348 delete[] FileNames;
350 FileNames = new char[nSize];
351 memset(FileNames, 0, nSize);
352 Files.resize(nNb);
354 it = tempMap.begin();
355 nSize = 0;
356 nNb = 0;
357 while (it != tempMap.end())
359 strcpy(FileNames+nSize, it->first.c_str());
361 Files[nNb].Name = FileNames+nSize;
362 Files[nNb].Size = it->second.Size;
363 Files[nNb].Pos = it->second.Pos;
365 nSize += (uint)it->first.size() + 1;
366 nNb++;
367 it++;
370 // End of temp map conversion
372 return true;
375 bool CBigFile::BNP::appendHeader()
377 // Only external use
378 if (InternalUse || BigFileName.empty()) return false;
380 FILE *f = nlfopen (BigFileName, "ab");
381 if (f == NULL) return false;
383 uint32 nNbFile = (uint32)SFiles.size();
385 // value to be serialized
386 uint32 nNbFile2 = nNbFile;
388 #ifdef NL_BIG_ENDIAN
389 NLMISC_BSWAP32(nNbFile2);
390 #endif
392 if (fwrite (&nNbFile2, sizeof(uint32), 1, f) != 1)
394 fclose(f);
395 return false;
398 for (uint32 i = 0; i < nNbFile; ++i)
400 uint8 nStringSize = (uint8)SFiles[i].Name.length();
401 if (fwrite (&nStringSize, 1, 1, f) != 1)
403 fclose(f);
404 return false;
407 if (fwrite (SFiles[i].Name.c_str(), 1, nStringSize, f) != nStringSize)
409 fclose(f);
410 return false;
413 uint32 nFileSize = SFiles[i].Size;
415 #ifdef NL_BIG_ENDIAN
416 NLMISC_BSWAP32(nFileSize);
417 #endif
419 if (fwrite (&nFileSize, sizeof(uint32), 1, f) != 1)
421 fclose(f);
422 return false;
425 uint32 nFilePos = SFiles[i].Pos;
427 #ifdef NL_BIG_ENDIAN
428 NLMISC_BSWAP32(nFilePos);
429 #endif
431 if (fwrite (&nFilePos, sizeof(uint32), 1, f) != 1)
433 fclose(f);
434 return false;
438 uint32 nOffsetFromBeginning = OffsetFromBeginning;
440 #ifdef NL_BIG_ENDIAN
441 NLMISC_BSWAP32(nOffsetFromBeginning);
442 #endif
444 if (fwrite (&nOffsetFromBeginning, sizeof(uint32), 1, f) != 1)
446 fclose(f);
447 return false;
450 fclose (f);
451 return true;
454 // ***************************************************************************
455 bool CBigFile::BNP::appendFile(const std::string &filename)
457 // Only external use
458 if (InternalUse || BigFileName.empty()) return false;
460 // Check if we can read the source file
461 if (!CFile::fileExists(filename)) return false;
463 SBNPFile ftmp;
464 ftmp.Name = CFile::getFilename(filename);
465 ftmp.Size = CFile::getFileSize(filename);
466 ftmp.Pos = OffsetFromBeginning;
467 SFiles.push_back(ftmp);
468 OffsetFromBeginning += ftmp.Size;
470 FILE *f1 = nlfopen(BigFileName, "ab");
471 if (f1 == NULL) return false;
473 FILE *f2 = nlfopen(filename, "rb");
474 if (f2 == NULL)
476 fclose(f1);
477 return false;
480 uint8 *ptr = new uint8[ftmp.Size];
482 if (fread (ptr, ftmp.Size, 1, f2) != 1)
484 nlwarning("%s read error", filename.c_str());
486 else if (fwrite (ptr, ftmp.Size, 1, f1) != 1)
488 nlwarning("%s write error", BigFileName.c_str());
491 delete [] ptr;
493 fclose(f2);
494 fclose(f1);
496 return true;
499 // ***************************************************************************
500 bool CBigFile::BNP::unpack(const std::string &sDestDir, TUnpackProgressCallback *callback)
502 // Only external use
503 if (InternalUse || BigFileName.empty()) return false;
505 FILE *bnp = nlfopen (BigFileName, "rb");
506 if (bnp == NULL)
507 return false;
509 // only read header is not already read
510 if (SFiles.empty() && !readHeader(bnp))
512 fclose (bnp);
513 return false;
516 CFile::createDirectory(sDestDir);
518 uint32 totalUncompressed = 0, total = 0;
520 for (uint32 i = 0; i < SFiles.size(); ++i)
522 total += SFiles[i].Size;
525 FILE *out = NULL;
527 for (uint32 i = 0; i < SFiles.size(); ++i)
529 const SBNPFile &rBNPFile = SFiles[i];
530 string filename = CPath::standardizePath(sDestDir) + rBNPFile.Name;
532 if (callback && !(*callback)(filename, totalUncompressed, total))
534 fclose (bnp);
535 return false;
538 out = nlfopen (filename, "wb");
539 if (out != NULL)
541 nlfseek64 (bnp, rBNPFile.Pos, SEEK_SET);
542 uint8 *ptr = new uint8[rBNPFile.Size];
543 bool readError = fread (ptr, rBNPFile.Size, 1, bnp) != 1;
544 if (readError)
546 nlwarning("%s read error errno = %d: %s", filename.c_str(), errno, strerror(errno));
548 bool writeError = fwrite (ptr, rBNPFile.Size, 1, out) != 1;
549 if (writeError)
551 nlwarning("%s write error errno = %d: %s", filename.c_str(), errno, strerror(errno));
553 bool diskFull = ferror(out) && errno == 28 /* ENOSPC*/;
554 fclose (out);
555 delete [] ptr;
556 if (diskFull)
558 fclose (bnp);
559 throw NLMISC::EDiskFullError(filename);
561 if (writeError)
563 fclose (bnp);
564 throw NLMISC::EWriteError(filename);
566 if (readError)
568 fclose (bnp);
569 throw NLMISC::EReadError(filename);
573 totalUncompressed += rBNPFile.Size;
575 if (callback && !(*callback)(filename, totalUncompressed, total))
577 fclose (bnp);
578 return false;
582 fclose (bnp);
583 return true;
586 // ***************************************************************************
587 bool CBigFile::isBigFileAdded(const std::string &sBigFileName) const
589 // Is already the same bigfile name ?
590 string bigfilenamealone = CFile::getFilename (sBigFileName);
591 return _BNPs.find(bigfilenamealone) != _BNPs.end();
594 // ***************************************************************************
595 std::string CBigFile::getBigFileName(const std::string &sBigFileName) const
597 string bigfilenamealone = CFile::getFilename (sBigFileName);
598 map<string, BNP>::const_iterator it = _BNPs.find(bigfilenamealone);
599 if (it != _BNPs.end())
600 return it->second.BigFileName;
601 else
602 return "";
606 // ***************************************************************************
607 void CBigFile::list (const std::string &sBigFileName, std::vector<std::string> &vAllFiles)
609 string lwrFileName = toLowerAscii(sBigFileName);
610 if (_BNPs.find (lwrFileName) == _BNPs.end())
611 return;
612 vAllFiles.clear ();
613 BNP &rbnp = _BNPs.find (lwrFileName)->second;
614 vector<BNPFile>::iterator it = rbnp.Files.begin();
615 while (it != rbnp.Files.end())
617 vAllFiles.push_back (string(it->Name)); // Add the name of the file to the return vector
618 ++it;
622 // ***************************************************************************
623 void CBigFile::removeAll ()
625 while (_BNPs.begin() != _BNPs.end())
627 remove (_BNPs.begin()->first);
631 struct CBNPFileComp
633 bool operator()(const CBigFile::BNPFile &f, const CBigFile::BNPFile &s )
635 return strcmp( f.Name, s.Name ) < 0;
639 // ***************************************************************************
640 bool CBigFile::getFileInternal (const std::string &sFileName, BNP *&zeBnp, BNPFile *&zeBnpFile)
642 string zeFileName, zeBigFileName, lwrFileName = toLowerAscii(sFileName);
643 string::size_type i, nPos = sFileName.find ('@');
644 if (nPos == string::npos)
646 return false;
649 for (i = 0; i < nPos; ++i)
650 zeBigFileName += lwrFileName[i];
651 ++i; // Skip @
652 for (; i < lwrFileName.size(); ++i)
653 zeFileName += lwrFileName[i];
655 if (_BNPs.find (zeBigFileName) == _BNPs.end())
657 return false;
660 BNP &rbnp = _BNPs.find (zeBigFileName)->second;
661 if (rbnp.Files.empty())
663 return false;
666 vector<BNPFile>::iterator itNBPFile;
668 BNPFile temp_bnp_file;
669 temp_bnp_file.Name = (char*)zeFileName.c_str();
670 itNBPFile = lower_bound(rbnp.Files.begin(), rbnp.Files.end(), temp_bnp_file, CBNPFileComp());
672 if (itNBPFile != rbnp.Files.end())
674 if (strcmp(itNBPFile->Name, zeFileName.c_str()) != 0)
676 return false;
679 else
681 return false;
684 BNPFile &rbnpfile = *itNBPFile;
686 // set ptr on found bnp/bnpFile
687 zeBnp= &rbnp;
688 zeBnpFile= &rbnpfile;
690 return true;
693 // ***************************************************************************
694 FILE* CBigFile::getFile (const std::string &sFileName, uint32 &rFileSize,
695 uint32 &rBigFileOffset, bool &rCacheFileOnOpen, bool &rAlwaysOpened)
697 BNP *bnp= NULL;
698 BNPFile *bnpFile= NULL;
699 if(!getFileInternal(sFileName, bnp, bnpFile))
701 nlwarning ("BF: Couldn't load '%s'", sFileName.c_str());
702 return NULL;
704 nlassert(bnp && bnpFile);
706 // Get a ThreadSafe handle on the file
707 CHandleFile &handle= _ThreadFileArray.get(bnp->ThreadFileId);
708 /* If not opened, open it now. There is 2 reason for it to be not opened:
709 rbnp.AlwaysOpened==false, or it is a new thread which use it for the first time.
711 if(handle.File== NULL)
713 handle.File = nlfopen (bnp->BigFileName, "rb");
714 if (handle.File == NULL)
716 nlwarning ("bnp: can't fopen big file '%s' error %d '%s'", bnp->BigFileName.c_str(), errno, strerror(errno));
717 return NULL;
721 rCacheFileOnOpen = bnp->CacheFileOnOpen;
722 rAlwaysOpened = bnp->AlwaysOpened;
723 rBigFileOffset = bnpFile->Pos;
724 rFileSize = bnpFile->Size;
725 return handle.File;
728 // ***************************************************************************
729 bool CBigFile::getFileInfo (const std::string &sFileName, uint32 &rFileSize, uint32 &rBigFileOffset)
731 BNP *bnp= NULL;
732 BNPFile *bnpFile= NULL;
733 if(!getFileInternal(sFileName, bnp, bnpFile))
735 nlwarning ("BF: Couldn't find '%s' for info", sFileName.c_str());
736 return false;
738 nlassert(bnp && bnpFile);
740 // get infos
741 rBigFileOffset = bnpFile->Pos;
742 rFileSize = bnpFile->Size;
743 return true;
746 // ***************************************************************************
747 char *CBigFile::getFileNamePtr(const std::string &sFileName, const std::string &sBigFileName)
749 string bigfilenamealone = CFile::getFilename (sBigFileName);
750 if (_BNPs.find(bigfilenamealone) != _BNPs.end())
752 BNP &rbnp = _BNPs.find (bigfilenamealone)->second;
753 vector<BNPFile>::iterator itNBPFile;
754 if (rbnp.Files.empty())
755 return NULL;
756 string lwrFileName = toLowerAscii(sFileName);
758 BNPFile temp_bnp_file;
759 temp_bnp_file.Name = (char*)lwrFileName.c_str();
760 itNBPFile = lower_bound(rbnp.Files.begin(), rbnp.Files.end(), temp_bnp_file, CBNPFileComp());
762 if (itNBPFile != rbnp.Files.end())
764 if (strcmp(itNBPFile->Name, lwrFileName.c_str()) == 0)
766 return itNBPFile->Name;
771 return NULL;
774 // ***************************************************************************
775 void CBigFile::getBigFilePaths(std::vector<std::string> &bigFilePaths)
777 bigFilePaths.clear();
778 for(std::map<std::string, BNP>::iterator it = _BNPs.begin(); it != _BNPs.end(); ++it)
780 bigFilePaths.push_back(it->second.BigFileName);
784 // ***************************************************************************
785 bool CBigFile::unpack(const std::string &sBigFileName, const std::string &sDestDir, TUnpackProgressCallback *callback)
787 BNP bnpFile;
788 bnpFile.BigFileName = sBigFileName;
789 return bnpFile.unpack(sDestDir, callback);
792 } // namespace NLMISC