1 // NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
2 // Copyright (C) 2010 Winch Gate Property Limited
4 // This source file has been modified by the following contributors:
5 // Copyright (C) 2020 Jan BOON (Kaetemi) <jan.boon@kaetemi.be>
7 // This program is free software: you can redistribute it and/or modify
8 // it under the terms of the GNU Affero General Public License as
9 // published by the Free Software Foundation, either version 3 of the
10 // License, or (at your option) any later version.
12 // This program is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 // GNU Affero General Public License for more details.
17 // You should have received a copy of the GNU Affero General Public License
18 // along with this program. If not, see <http://www.gnu.org/licenses/>.
22 #include "nel/misc/file.h"
23 #include "nel/misc/big_file.h"
24 #include "nel/misc/path.h"
27 using namespace NLMISC
;
35 //CBigFile *CBigFile::_Singleton = NULL;
36 NLMISC_SAFE_SINGLETON_IMPL(CBigFile
);
38 // ***************************************************************************
39 void CBigFile::releaseInstance()
43 NLMISC::INelContext::getInstance().releaseSingletonPointer("CBigFile", _Instance
);
48 // ***************************************************************************
49 CBigFile::CThreadFileArray::CThreadFileArray()
54 // ***************************************************************************
55 CBigFile::CThreadFileArray::~CThreadFileArray()
57 vector
<CHandleFile
> *ptr
= (vector
<CHandleFile
>*)_TDS
.getPointer();
61 // ***************************************************************************
62 uint32
CBigFile::CThreadFileArray::allocate()
67 // ***************************************************************************
68 CBigFile::CHandleFile
&CBigFile::CThreadFileArray::get(uint32 index
)
70 // If the thread struct ptr is NULL, must allocate it.
71 vector
<CHandleFile
> *ptr
= (vector
<CHandleFile
>*)_TDS
.getPointer();
74 ptr
= new vector
<CHandleFile
>;
78 // if the vector is not allocated, allocate it (empty entries filled with NULL => not opened FILE* in this thread)
79 if(index
>=ptr
->size())
81 ptr
->resize((ptrdiff_t)index
+ 1);
88 // ***************************************************************************
89 void CBigFile::currentThreadFinished()
91 _ThreadFileArray
.currentThreadFinished();
94 // ***************************************************************************
95 void CBigFile::CThreadFileArray::currentThreadFinished()
97 vector
<CHandleFile
> *ptr
= (vector
<CHandleFile
>*)_TDS
.getPointer();
98 if (ptr
==NULL
) return;
99 for (uint k
= 0; k
< ptr
->size(); ++k
)
103 fclose((*ptr
)[k
].File
);
104 (*ptr
)[k
].File
= NULL
;
108 _TDS
.setPointer(NULL
);
112 // ***************************************************************************
113 //CBigFile::CBigFile ()
117 //// ***************************************************************************
118 //CBigFile &CBigFile::getInstance ()
120 // if (_Singleton == NULL)
122 // _Singleton = new CBigFile();
124 // return *_Singleton;
127 // ***************************************************************************
128 bool CBigFile::add (const std::string
&sBigFileName
, uint32 nOptions
)
130 // Is already the same bigfile name ?
131 string bigfilenamealone
= toLowerAscii(CFile::getFilename (sBigFileName
));
132 if (_BNPs
.find(bigfilenamealone
) != _BNPs
.end())
134 nlwarning ("CBigFile::add : bigfile %s already added.", bigfilenamealone
.c_str());
138 // Create the new bnp entry
139 BNP
&bnp
= _BNPs
[bigfilenamealone
];
141 bnp
.BigFileName
= sBigFileName
;
143 // Allocate a new ThreadSafe FileId for this bnp.
144 bnp
.ThreadFileId
= _ThreadFileArray
.allocate();
146 // Get a ThreadSafe handle on the file
147 CHandleFile
&handle
= _ThreadFileArray
.get(bnp
.ThreadFileId
);
149 // Open the big file.
150 handle
.File
= nlfopen (sBigFileName
, "rb");
151 if (handle
.File
== NULL
)
154 // Used internally by CBigFile, use optimizations and lower case of filenames
155 bnp
.InternalUse
= true;
158 if (!bnp
.readHeader(handle
.File
))
160 fclose (handle
.File
);
164 if (nOptions
&BF_CACHE_FILE_ON_OPEN
)
165 bnp
.CacheFileOnOpen
= true;
167 bnp
.CacheFileOnOpen
= false;
169 if (!(nOptions
&BF_ALWAYS_OPENED
))
171 fclose (handle
.File
);
173 bnp
.AlwaysOpened
= false;
177 bnp
.AlwaysOpened
= true;
180 //nldebug("BigFile : added bnp '%s' to the collection", bigfilenamealone.c_str());
185 // ***************************************************************************
186 void CBigFile::remove (const std::string
&sBigFileName
)
188 if (_BNPs
.find (sBigFileName
) != _BNPs
.end())
190 map
<string
, BNP
>::iterator it
= _BNPs
.find (sBigFileName
);
191 BNP
&rbnp
= it
->second
;
192 // Get a ThreadSafe handle on the file
193 CHandleFile
&handle
= _ThreadFileArray
.get(rbnp
.ThreadFileId
);
194 // close it if needed
195 if (handle
.File
!= NULL
)
197 fclose (handle
.File
);
205 CBigFile::BNP::BNP() : FileNames(NULL
), ThreadFileId(0), CacheFileOnOpen(false), AlwaysOpened(false), InternalUse(false), OffsetFromBeginning(0)
209 CBigFile::BNP::~BNP()
218 //// ***************************************************************************
219 bool CBigFile::BNP::readHeader()
222 if (InternalUse
|| BigFileName
.empty()) return false;
224 FILE *f
= nlfopen (BigFileName
, "rb");
225 if (f
== NULL
) return false;
227 bool res
= readHeader(f
);
233 //// ***************************************************************************
234 bool CBigFile::BNP::readHeader(FILE *file
)
236 if (file
== NULL
) return false;
238 uint32 nFileSize
=CFile::getFileSize (file
);
241 if (nlfseek64 (file
, nFileSize
-4, SEEK_SET
) != 0)
246 if (fread (&OffsetFromBeginning
, sizeof(uint32
), 1, file
) != 1)
252 NLMISC_BSWAP32(OffsetFromBeginning
);
255 if (nlfseek64 (file
, OffsetFromBeginning
, SEEK_SET
) != 0)
260 // Read the file count
262 if (fread (&nNbFile
, sizeof(uint32
), 1, file
) != 1)
268 NLMISC_BSWAP32(nNbFile
);
271 map
<string
, BNPFile
> tempMap
;
273 if (!InternalUse
) SFiles
.clear();
275 for (uint32 i
= 0; i
< nNbFile
; ++i
)
278 if (fread (&nStringSize
, 1, 1, file
) != 1)
286 if (fread(sFileName
, 1, nStringSize
, file
) != nStringSize
)
291 sFileName
[nStringSize
] = 0;
294 if (fread (&nFileSize2
, sizeof(uint32
), 1, file
) != 1)
300 NLMISC_BSWAP32(nFileSize2
);
304 if (fread (&nFilePos
, sizeof(uint32
), 1, file
) != 1)
310 NLMISC_BSWAP32(nFilePos
);
316 bnpfTmp
.Pos
= nFilePos
;
317 bnpfTmp
.Size
= nFileSize2
;
318 tempMap
.insert (make_pair(toLowerAscii(string(sFileName
)), bnpfTmp
));
323 bnpfTmp
.Name
= sFileName
;
324 bnpfTmp
.Pos
= nFilePos
;
325 bnpfTmp
.Size
= nFileSize2
;
326 SFiles
.push_back(bnpfTmp
);
330 if (nlfseek64 (file
, 0, SEEK_SET
) != 0)
336 if (InternalUse
&& nNbFile
> 0)
338 uint nSize
= 0, nNb
= 0;
339 map
<string
,BNPFile
>::iterator it
= tempMap
.begin();
340 while (it
!= tempMap
.end())
342 nSize
+= (uint
)it
->first
.size() + 1;
350 FileNames
= new char[nSize
];
351 memset(FileNames
, 0, nSize
);
354 it
= tempMap
.begin();
357 while (it
!= tempMap
.end())
359 strcpy(FileNames
+nSize
, it
->first
.c_str());
361 Files
[nNb
].Name
= FileNames
+nSize
;
362 Files
[nNb
].Size
= it
->second
.Size
;
363 Files
[nNb
].Pos
= it
->second
.Pos
;
365 nSize
+= (uint
)it
->first
.size() + 1;
370 // End of temp map conversion
375 bool CBigFile::BNP::appendHeader()
378 if (InternalUse
|| BigFileName
.empty()) return false;
380 FILE *f
= nlfopen (BigFileName
, "ab");
381 if (f
== NULL
) return false;
383 uint32 nNbFile
= (uint32
)SFiles
.size();
385 // value to be serialized
386 uint32 nNbFile2
= nNbFile
;
389 NLMISC_BSWAP32(nNbFile2
);
392 if (fwrite (&nNbFile2
, sizeof(uint32
), 1, f
) != 1)
398 for (uint32 i
= 0; i
< nNbFile
; ++i
)
400 uint8 nStringSize
= (uint8
)SFiles
[i
].Name
.length();
401 if (fwrite (&nStringSize
, 1, 1, f
) != 1)
407 if (fwrite (SFiles
[i
].Name
.c_str(), 1, nStringSize
, f
) != nStringSize
)
413 uint32 nFileSize
= SFiles
[i
].Size
;
416 NLMISC_BSWAP32(nFileSize
);
419 if (fwrite (&nFileSize
, sizeof(uint32
), 1, f
) != 1)
425 uint32 nFilePos
= SFiles
[i
].Pos
;
428 NLMISC_BSWAP32(nFilePos
);
431 if (fwrite (&nFilePos
, sizeof(uint32
), 1, f
) != 1)
438 uint32 nOffsetFromBeginning
= OffsetFromBeginning
;
441 NLMISC_BSWAP32(nOffsetFromBeginning
);
444 if (fwrite (&nOffsetFromBeginning
, sizeof(uint32
), 1, f
) != 1)
454 // ***************************************************************************
455 bool CBigFile::BNP::appendFile(const std::string
&filename
)
458 if (InternalUse
|| BigFileName
.empty()) return false;
460 // Check if we can read the source file
461 if (!CFile::fileExists(filename
)) return false;
464 ftmp
.Name
= CFile::getFilename(filename
);
465 ftmp
.Size
= CFile::getFileSize(filename
);
466 ftmp
.Pos
= OffsetFromBeginning
;
467 SFiles
.push_back(ftmp
);
468 OffsetFromBeginning
+= ftmp
.Size
;
470 FILE *f1
= nlfopen(BigFileName
, "ab");
471 if (f1
== NULL
) return false;
473 FILE *f2
= nlfopen(filename
, "rb");
480 uint8
*ptr
= new uint8
[ftmp
.Size
];
482 if (fread (ptr
, ftmp
.Size
, 1, f2
) != 1)
484 nlwarning("%s read error", filename
.c_str());
486 else if (fwrite (ptr
, ftmp
.Size
, 1, f1
) != 1)
488 nlwarning("%s write error", BigFileName
.c_str());
499 // ***************************************************************************
500 bool CBigFile::BNP::unpack(const std::string
&sDestDir
, TUnpackProgressCallback
*callback
)
503 if (InternalUse
|| BigFileName
.empty()) return false;
505 FILE *bnp
= nlfopen (BigFileName
, "rb");
509 // only read header is not already read
510 if (SFiles
.empty() && !readHeader(bnp
))
516 CFile::createDirectory(sDestDir
);
518 uint32 totalUncompressed
= 0, total
= 0;
520 for (uint32 i
= 0; i
< SFiles
.size(); ++i
)
522 total
+= SFiles
[i
].Size
;
527 for (uint32 i
= 0; i
< SFiles
.size(); ++i
)
529 const SBNPFile
&rBNPFile
= SFiles
[i
];
530 string filename
= CPath::standardizePath(sDestDir
) + rBNPFile
.Name
;
532 if (callback
&& !(*callback
)(filename
, totalUncompressed
, total
))
538 out
= nlfopen (filename
, "wb");
541 nlfseek64 (bnp
, rBNPFile
.Pos
, SEEK_SET
);
542 uint8
*ptr
= new uint8
[rBNPFile
.Size
];
543 bool readError
= fread (ptr
, rBNPFile
.Size
, 1, bnp
) != 1;
546 nlwarning("%s read error errno = %d: %s", filename
.c_str(), errno
, strerror(errno
));
548 bool writeError
= fwrite (ptr
, rBNPFile
.Size
, 1, out
) != 1;
551 nlwarning("%s write error errno = %d: %s", filename
.c_str(), errno
, strerror(errno
));
553 bool diskFull
= ferror(out
) && errno
== 28 /* ENOSPC*/;
559 throw NLMISC::EDiskFullError(filename
);
564 throw NLMISC::EWriteError(filename
);
569 throw NLMISC::EReadError(filename
);
573 totalUncompressed
+= rBNPFile
.Size
;
575 if (callback
&& !(*callback
)(filename
, totalUncompressed
, total
))
586 // ***************************************************************************
587 bool CBigFile::isBigFileAdded(const std::string
&sBigFileName
) const
589 // Is already the same bigfile name ?
590 string bigfilenamealone
= CFile::getFilename (sBigFileName
);
591 return _BNPs
.find(bigfilenamealone
) != _BNPs
.end();
594 // ***************************************************************************
595 std::string
CBigFile::getBigFileName(const std::string
&sBigFileName
) const
597 string bigfilenamealone
= CFile::getFilename (sBigFileName
);
598 map
<string
, BNP
>::const_iterator it
= _BNPs
.find(bigfilenamealone
);
599 if (it
!= _BNPs
.end())
600 return it
->second
.BigFileName
;
606 // ***************************************************************************
607 void CBigFile::list (const std::string
&sBigFileName
, std::vector
<std::string
> &vAllFiles
)
609 string lwrFileName
= toLowerAscii(sBigFileName
);
610 if (_BNPs
.find (lwrFileName
) == _BNPs
.end())
613 BNP
&rbnp
= _BNPs
.find (lwrFileName
)->second
;
614 vector
<BNPFile
>::iterator it
= rbnp
.Files
.begin();
615 while (it
!= rbnp
.Files
.end())
617 vAllFiles
.push_back (string(it
->Name
)); // Add the name of the file to the return vector
622 // ***************************************************************************
623 void CBigFile::removeAll ()
625 while (_BNPs
.begin() != _BNPs
.end())
627 remove (_BNPs
.begin()->first
);
633 bool operator()(const CBigFile::BNPFile
&f
, const CBigFile::BNPFile
&s
)
635 return strcmp( f
.Name
, s
.Name
) < 0;
639 // ***************************************************************************
640 bool CBigFile::getFileInternal (const std::string
&sFileName
, BNP
*&zeBnp
, BNPFile
*&zeBnpFile
)
642 string zeFileName
, zeBigFileName
, lwrFileName
= toLowerAscii(sFileName
);
643 string::size_type i
, nPos
= sFileName
.find ('@');
644 if (nPos
== string::npos
)
649 for (i
= 0; i
< nPos
; ++i
)
650 zeBigFileName
+= lwrFileName
[i
];
652 for (; i
< lwrFileName
.size(); ++i
)
653 zeFileName
+= lwrFileName
[i
];
655 if (_BNPs
.find (zeBigFileName
) == _BNPs
.end())
660 BNP
&rbnp
= _BNPs
.find (zeBigFileName
)->second
;
661 if (rbnp
.Files
.empty())
666 vector
<BNPFile
>::iterator itNBPFile
;
668 BNPFile temp_bnp_file
;
669 temp_bnp_file
.Name
= (char*)zeFileName
.c_str();
670 itNBPFile
= lower_bound(rbnp
.Files
.begin(), rbnp
.Files
.end(), temp_bnp_file
, CBNPFileComp());
672 if (itNBPFile
!= rbnp
.Files
.end())
674 if (strcmp(itNBPFile
->Name
, zeFileName
.c_str()) != 0)
684 BNPFile
&rbnpfile
= *itNBPFile
;
686 // set ptr on found bnp/bnpFile
688 zeBnpFile
= &rbnpfile
;
693 // ***************************************************************************
694 FILE* CBigFile::getFile (const std::string
&sFileName
, uint32
&rFileSize
,
695 uint32
&rBigFileOffset
, bool &rCacheFileOnOpen
, bool &rAlwaysOpened
)
698 BNPFile
*bnpFile
= NULL
;
699 if(!getFileInternal(sFileName
, bnp
, bnpFile
))
701 nlwarning ("BF: Couldn't load '%s'", sFileName
.c_str());
704 nlassert(bnp
&& bnpFile
);
706 // Get a ThreadSafe handle on the file
707 CHandleFile
&handle
= _ThreadFileArray
.get(bnp
->ThreadFileId
);
708 /* If not opened, open it now. There is 2 reason for it to be not opened:
709 rbnp.AlwaysOpened==false, or it is a new thread which use it for the first time.
711 if(handle
.File
== NULL
)
713 handle
.File
= nlfopen (bnp
->BigFileName
, "rb");
714 if (handle
.File
== NULL
)
716 nlwarning ("bnp: can't fopen big file '%s' error %d '%s'", bnp
->BigFileName
.c_str(), errno
, strerror(errno
));
721 rCacheFileOnOpen
= bnp
->CacheFileOnOpen
;
722 rAlwaysOpened
= bnp
->AlwaysOpened
;
723 rBigFileOffset
= bnpFile
->Pos
;
724 rFileSize
= bnpFile
->Size
;
728 // ***************************************************************************
729 bool CBigFile::getFileInfo (const std::string
&sFileName
, uint32
&rFileSize
, uint32
&rBigFileOffset
)
732 BNPFile
*bnpFile
= NULL
;
733 if(!getFileInternal(sFileName
, bnp
, bnpFile
))
735 nlwarning ("BF: Couldn't find '%s' for info", sFileName
.c_str());
738 nlassert(bnp
&& bnpFile
);
741 rBigFileOffset
= bnpFile
->Pos
;
742 rFileSize
= bnpFile
->Size
;
746 // ***************************************************************************
747 char *CBigFile::getFileNamePtr(const std::string
&sFileName
, const std::string
&sBigFileName
)
749 string bigfilenamealone
= CFile::getFilename (sBigFileName
);
750 if (_BNPs
.find(bigfilenamealone
) != _BNPs
.end())
752 BNP
&rbnp
= _BNPs
.find (bigfilenamealone
)->second
;
753 vector
<BNPFile
>::iterator itNBPFile
;
754 if (rbnp
.Files
.empty())
756 string lwrFileName
= toLowerAscii(sFileName
);
758 BNPFile temp_bnp_file
;
759 temp_bnp_file
.Name
= (char*)lwrFileName
.c_str();
760 itNBPFile
= lower_bound(rbnp
.Files
.begin(), rbnp
.Files
.end(), temp_bnp_file
, CBNPFileComp());
762 if (itNBPFile
!= rbnp
.Files
.end())
764 if (strcmp(itNBPFile
->Name
, lwrFileName
.c_str()) == 0)
766 return itNBPFile
->Name
;
774 // ***************************************************************************
775 void CBigFile::getBigFilePaths(std::vector
<std::string
> &bigFilePaths
)
777 bigFilePaths
.clear();
778 for(std::map
<std::string
, BNP
>::iterator it
= _BNPs
.begin(); it
!= _BNPs
.end(); ++it
)
780 bigFilePaths
.push_back(it
->second
.BigFileName
);
784 // ***************************************************************************
785 bool CBigFile::unpack(const std::string
&sBigFileName
, const std::string
&sDestDir
, TUnpackProgressCallback
*callback
)
788 bnpFile
.BigFileName
= sBigFileName
;
789 return bnpFile
.unpack(sDestDir
, callback
);
792 } // namespace NLMISC