Merge branch 'fixes' into main/rendor-staging
[ryzomcore.git] / nel / src / misc / file.cpp
blobdc0863c631b21536d63374b838ece3642e4d7b93
1 // NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
2 // Copyright (C) 2010-2019 Winch Gate Property Limited
3 //
4 // This source file has been modified by the following contributors:
5 // Copyright (C) 2014-2020 Jan BOON (Kaetemi) <jan.boon@kaetemi.be>
6 //
7 // This program is free software: you can redistribute it and/or modify
8 // it under the terms of the GNU Affero General Public License as
9 // published by the Free Software Foundation, either version 3 of the
10 // License, or (at your option) any later version.
12 // This program is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 // GNU Affero General Public License for more details.
17 // You should have received a copy of the GNU Affero General Public License
18 // along with this program. If not, see <http://www.gnu.org/licenses/>.
20 #include "stdmisc.h"
22 #include "nel/misc/file.h"
23 #include "nel/misc/debug.h"
24 #include "nel/misc/big_file.h"
25 #include "nel/misc/path.h"
26 #include "nel/misc/command.h"
27 #include "nel/misc/sstring.h"
28 #include "nel/misc/xml_pack.h"
29 #include "nel/misc/streamed_package_manager.h"
31 #ifndef NL_OS_WINDOWS
32 #include <errno.h>
33 #endif
35 using namespace std;
37 #define NLMISC_DONE_FILE_OPENED 40
39 #ifdef DEBUG_NEW
40 #define new DEBUG_NEW
41 #endif
43 namespace NLMISC
46 typedef std::list<uint64> TFileAccessTimes; // list of times at which a given file is opened for reading
47 typedef CHashMap<std::string,TFileAccessTimes> TFileAccessLog; // map from file name to read access times
48 typedef NLMISC::CSynchronized<TFileAccessLog> TSynchronizedFileAccessLog;
50 static TSynchronizedFileAccessLog IFileAccessLog("IFileAccessLog");
51 static bool IFileAccessLoggingEnabled= false;
52 static uint64 IFileAccessLogStartTime= 0;
54 uint32 CIFile::_NbBytesSerialized = 0;
55 uint32 CIFile::_NbBytesLoaded = 0;
56 uint32 CIFile::_ReadFromFile = 0;
57 uint32 CIFile::_ReadingFromFile = 0;
58 uint32 CIFile::_FileOpened = 0;
59 uint32 CIFile::_FileRead = 0;
60 CSynchronized<std::deque<std::string> > CIFile::_OpenedFiles("");
62 // ======================================================================================================
63 CIFile::CIFile() : IStream(true)
65 _F = NULL;
66 _Cache = NULL;
67 _ReadPos = 0;
68 _FileSize = 0;
69 _BigFileOffset = 0;
70 _IsInBigFile = false;
71 _IsInXMLPackFile = false;
72 _CacheFileOnOpen = false;
73 _IsAsyncLoading = false;
74 _AllowBNPCacheFileOnOpen= true;
77 // ======================================================================================================
78 CIFile::CIFile(const std::string &path, bool text) : IStream(true)
80 _F=NULL;
81 _Cache = NULL;
82 _ReadPos = 0;
83 _FileSize = 0;
84 _BigFileOffset = 0;
85 _IsInBigFile = false;
86 _IsInXMLPackFile = false;
87 _CacheFileOnOpen = false;
88 _IsAsyncLoading = false;
89 _AllowBNPCacheFileOnOpen= true;
90 open(path, text);
93 // ======================================================================================================
94 CIFile::~CIFile()
96 close();
100 // ======================================================================================================
101 void CIFile::loadIntoCache()
103 const uint32 READPACKETSIZE = 64 * 1024;
104 const uint32 INTERPACKETSLEEP = 5;
106 _Cache = new uint8[_FileSize];
107 if(!_IsAsyncLoading)
109 _ReadingFromFile += _FileSize;
110 int read = (int)fread (_Cache, _FileSize, 1, _F);
111 _FileRead++;
112 _ReadingFromFile -= _FileSize;
113 _ReadFromFile += read * _FileSize;
115 else
117 uint index= 0;
118 while(index<_FileSize)
120 if( _NbBytesLoaded + (_FileSize-index) > READPACKETSIZE )
122 sint n= READPACKETSIZE-_NbBytesLoaded;
123 n= max(n, 1);
124 _ReadingFromFile += n;
125 int read = (int)fread (_Cache+index, n, 1, _F);
126 _FileRead++;
127 _ReadingFromFile -= n;
128 _ReadFromFile += read * n;
129 index+= n;
131 nlSleep (INTERPACKETSLEEP);
132 _NbBytesLoaded= 0;
134 else
136 uint n= _FileSize-index;
137 _ReadingFromFile += n;
138 int read = (int)fread (_Cache+index, n, 1, _F);
139 _FileRead++;
140 _ReadingFromFile -= n;
141 _ReadFromFile += read * n;
142 _NbBytesLoaded+= n;
143 index+= n;
150 // ======================================================================================================
151 bool CIFile::open(const std::string &path, bool text)
153 // Log opened files
155 CSynchronized<deque<string> >::CAccessor fileOpened(&_OpenedFiles);
156 fileOpened.value().push_front (path);
157 if (fileOpened.value().size () > NLMISC_DONE_FILE_OPENED)
158 fileOpened.value().resize (NLMISC_DONE_FILE_OPENED);
159 _FileOpened++;
162 close();
164 if ((_IsInBigFile || _IsInXMLPackFile) && path.find('@') == string::npos)
166 // CIFile can be reused to load file from bnp and from regular files.
167 // Last open happened to be inside bnp and close() may not set _F to NULL.
168 // Opening regular file will fail as _F points to bnp file.
169 _F = NULL;
172 // can't open empty filename
173 if(path.empty ())
174 return false;
176 // IFile Access Log management
177 if (IFileAccessLoggingEnabled)
179 // get the current time
180 uint64 timeNow = NLMISC::CTime::getPerformanceTime();
182 // get a handle for the container
183 TSynchronizedFileAccessLog::CAccessor synchronizedFileAccesLog(&IFileAccessLog);
184 TFileAccessLog& fileAccessLog= synchronizedFileAccesLog.value();
186 // add the current time to the container entry for the given path (creating a new container entry if required)
187 fileAccessLog[path].push_back(timeNow);
190 char mode[3];
191 mode[0] = 'r';
192 mode[1] = 'b'; // No more reading in text mode
193 mode[2] = '\0';
195 _FileName = path;
196 _ReadPos = 0;
198 // Bigfile or xml pack access requested ?
199 string::size_type pos;
200 if ((pos = path.find('@')) != string::npos)
202 // check for a double @ to identify XML pack file
203 if (pos+1 < path.size() && path[pos+1] == '@')
205 // xml pack file
206 _IsInXMLPackFile = true;
208 if(_AllowBNPCacheFileOnOpen)
210 _F = CXMLPack::getInstance().getFile(path, _FileSize, _BigFileOffset, _CacheFileOnOpen, _AlwaysOpened);
212 else
214 bool dummy;
215 _F = CXMLPack::getInstance().getFile (path, _FileSize, _BigFileOffset, dummy, _AlwaysOpened);
218 else if (pos > 3 && path[pos-3] == 's' && path[pos-2] == 'n' && path[pos-1] == 'p')
220 // nldebug("Opening a streamed package file");
222 _IsInXMLPackFile = false;
223 _IsInBigFile = false;
224 _BigFileOffset = 0;
225 _AlwaysOpened = false;
226 std::string filePath;
227 if (CStreamedPackageManager::getInstance().getFile (filePath, path.substr(pos+1)))
229 _F = fopen (filePath.c_str(), mode);
230 if (_F != NULL)
232 _FileSize=CFile::getFileSize(_F);
233 if (_FileSize == 0)
235 nlwarning("FILE: Size of file '%s' is 0", path.c_str());
236 fclose(_F);
237 _F = NULL;
240 else
242 nlwarning("Failed to open file '%s', error %u : %s", path.c_str(), errno, strerror(errno));
243 _FileSize = 0;
246 else
248 // TEMPORARY ERROR
249 // nlerror("File '%s' not in streamed package", path.c_str());
252 else
254 // bnp file
255 _IsInBigFile = true;
256 if(_AllowBNPCacheFileOnOpen)
258 _F = CBigFile::getInstance().getFile (path, _FileSize, _BigFileOffset, _CacheFileOnOpen, _AlwaysOpened);
260 else
262 bool dummy;
263 _F = CBigFile::getInstance().getFile (path, _FileSize, _BigFileOffset, dummy, _AlwaysOpened);
266 if(_F != NULL)
268 // Start to load the bigfile or xml file at the file offset.
269 nlfseek64 (_F, _BigFileOffset, SEEK_SET);
271 // Load into cache ?
272 if (_CacheFileOnOpen)
274 // load file in the cache
275 loadIntoCache();
277 if (!_AlwaysOpened)
279 fclose (_F);
280 _F = NULL;
282 return (_Cache != NULL);
287 // not in bnp, but may have '@' in the name
288 if (_F == NULL)
290 _IsInBigFile = false;
291 _IsInXMLPackFile = false;
292 _BigFileOffset = 0;
293 _AlwaysOpened = false;
294 _F = nlfopen (path, mode);
295 if (_F != NULL)
298 THIS CODE REPLACED BY SADGE BECAUSE SOMETIMES
299 ftell() RETRUNS 0 FOR NO GOOD REASON - LEADING TO CLIENT CRASH
301 nlfseek64 (_F, 0, SEEK_END);
302 _FileSize = ftell(_F);
303 nlfseek64 (_F, 0, SEEK_SET);
304 nlassert(_FileSize==filelength(fileno(_F)));
306 THE FOLLOWING WORKS BUT IS NOT PORTABLE
307 _FileSize=filelength(fileno(_F));
309 _FileSize=CFile::getFileSize (_F);
310 if (_FileSize == 0)
312 nlwarning ("FILE: Size of file '%s' is 0", path.c_str());
313 fclose (_F);
314 _F = NULL;
317 else
319 nlwarning("Failed to open file '%s', error %u : %s", path.c_str(), errno, strerror(errno));
320 _FileSize = 0;
323 if ((_CacheFileOnOpen) && (_F != NULL))
325 // load file in the cache
326 loadIntoCache();
328 fclose (_F);
329 _F = NULL;
330 return (_Cache != NULL);
334 return (_F != NULL);
337 // ======================================================================================================
338 void CIFile::setCacheFileOnOpen (bool newState)
340 _CacheFileOnOpen = newState;
343 // ======================================================================================================
344 void CIFile::setAsyncLoading (bool newState)
346 _IsAsyncLoading = true;
350 // ======================================================================================================
351 void CIFile::close()
353 if (_CacheFileOnOpen)
355 if (_Cache)
357 delete[] _Cache;
358 _Cache = NULL;
361 else
363 if (_IsInBigFile || _IsInXMLPackFile)
365 if (!_AlwaysOpened)
367 if (_F)
369 fclose (_F);
370 _F = NULL;
374 else
376 if (_F)
378 fclose (_F);
379 _F = NULL;
383 nlassert(_Cache == NULL);
384 resetPtrTable();
387 // ======================================================================================================
388 void CIFile::flush()
390 if (_CacheFileOnOpen)
393 else
395 if (_F)
397 fflush (_F);
402 // ======================================================================================================
403 bool CIFile::readAll(std::string &buffer)
407 uint32 remaining = _FileSize;
409 buffer.clear();
410 buffer.reserve(_FileSize);
411 while(!eof() && remaining > 0)
413 const static uint bufsize = 1024;
414 char buf[bufsize];
415 uint32 readnow = bufsize;
416 if (readnow > remaining)
417 readnow = remaining;
419 serialBuffer((uint8 *)&buf[0], readnow);
420 buffer.append(buf, readnow);
421 remaining -= readnow;
424 catch (const EFile &)
426 // buffer state is unknown
427 return false;
430 return true;
433 // ======================================================================================================
434 void CIFile::getline (char *buffer, uint32 bufferSize)
436 if (bufferSize == 0)
437 return;
439 uint read = 0;
440 for(;;)
442 if (read == bufferSize - 1)
444 *buffer = '\0';
445 return;
450 // read one byte
451 serialBuffer ((uint8 *)buffer, 1);
453 catch (const EFile &)
455 *buffer = '\0';
456 return;
459 if (*buffer == '\n')
461 *buffer = '\0';
462 return;
465 // skip '\r' char
466 if (*buffer != '\r')
468 buffer++;
469 read++;
476 // ======================================================================================================
477 bool CIFile::eof ()
479 return _ReadPos >= (sint32)_FileSize;
482 // ======================================================================================================
483 void CIFile::serialBuffer(uint8 *buf, uint len)
485 if (len == 0)
486 return;
487 // Check the read pos
488 if ((_ReadPos < 0) || ((_ReadPos+len) > _FileSize))
489 throw EReadError (_FileName);
490 if ((_CacheFileOnOpen) && (_Cache == NULL))
491 throw EFileNotOpened (_FileName);
492 if ((!_CacheFileOnOpen) && (_F == NULL))
493 throw EFileNotOpened (_FileName);
495 if (_IsAsyncLoading)
497 _NbBytesSerialized += len;
498 if (_NbBytesSerialized > 64 * 1024)
500 // give up time slice
501 nlSleep (0);
502 _NbBytesSerialized = 0;
506 if (_CacheFileOnOpen)
508 memcpy (buf, _Cache + _ReadPos, len);
509 _ReadPos += len;
511 else
513 int read;
514 _ReadingFromFile += len;
515 read=(int)fread(buf, len, 1, _F);
516 _FileRead++;
517 _ReadingFromFile -= len;
518 _ReadFromFile += /*read **/ len;
519 if (read != 1 /*< (int)len*/)
520 throw EReadError(_FileName);
521 _ReadPos += len;
525 // ======================================================================================================
526 void CIFile::serialBit(bool &bit)
528 // Simple for now.
529 uint8 v=bit;
530 serialBuffer(&v, 1);
531 bit=(v!=0);
534 // ======================================================================================================
535 bool CIFile::seek (sint32 offset, IStream::TSeekOrigin origin) const
537 if ((_CacheFileOnOpen) && (_Cache == NULL))
538 return false;
539 if ((!_CacheFileOnOpen) && (_F == NULL))
540 return false;
542 switch (origin)
544 case IStream::begin:
545 _ReadPos = offset;
546 break;
547 case IStream::current:
548 _ReadPos = _ReadPos + offset;
549 break;
550 case IStream::end:
551 _ReadPos = _FileSize + offset;
552 break;
553 default:
554 nlstop;
557 if (_CacheFileOnOpen)
558 return true;
560 // seek in the file. NB: if not in bigfile, _BigFileOffset==0.
561 if (nlfseek64(_F, (sint64)_BigFileOffset + _ReadPos, SEEK_SET) != 0)
562 return false;
563 return true;
566 // ======================================================================================================
567 sint32 CIFile::getPos () const
569 return _ReadPos;
573 // ======================================================================================================
574 std::string CIFile::getStreamName() const
576 return _FileName;
580 // ======================================================================================================
581 void CIFile::allowBNPCacheFileOnOpen(bool newState)
583 _AllowBNPCacheFileOnOpen= newState;
587 // ======================================================================================================
588 void CIFile::dump (std::vector<std::string> &result)
590 CSynchronized<deque<string> >::CAccessor acces(&_OpenedFiles);
592 const deque<string> &openedFile = acces.value();
594 // Resize the destination array
595 result.clear ();
596 result.reserve (openedFile.size ());
598 // Add the waiting strings
599 deque<string>::const_reverse_iterator ite = openedFile.rbegin ();
600 while (ite != openedFile.rend ())
602 result.push_back (*ite);
604 // Next task
605 ite++;
609 // ======================================================================================================
610 void CIFile::clearDump ()
612 CSynchronized<deque<string> >::CAccessor acces(&_OpenedFiles);
613 acces.value().clear();
616 // ======================================================================================================
617 uint CIFile::getDbgStreamSize() const
619 return getFileSize();
623 // ======================================================================================================
624 // ======================================================================================================
625 // ======================================================================================================
628 // ======================================================================================================
629 COFile::COFile() : IStream(false)
631 _F=NULL;
634 // ======================================================================================================
635 COFile::COFile(const std::string &path, bool append, bool text, bool useTempFile) : IStream(false)
637 _F=NULL;
638 open(path, append, text, useTempFile);
641 // ======================================================================================================
642 COFile::~COFile()
644 internalClose(false);
646 // ======================================================================================================
647 bool COFile::open(const std::string &path, bool append, bool text, bool useTempFile)
649 close();
651 // can't open empty filename
652 if(path.empty ())
653 return false;
655 _FileName = path;
656 _TempFileName.clear();
658 char mode[3];
659 mode[0] = (append)?'a':'w';
660 // ACE: NEVER SAVE IN TEXT MODE!!! mode[1] = (text)?'\0':'b';
661 mode[1] = 'b';
662 mode[2] = '\0';
664 string fileToOpen = path;
665 if (useTempFile)
667 CFile::getTemporaryOutputFilename (path, _TempFileName);
668 fileToOpen = _TempFileName;
671 // if appending to file and using a temporary file, copycat temporary file from original...
672 if (append && useTempFile && CFile::fileExists(_FileName))
674 // open fails if can't copy original content
675 if (!CFile::copyFile(_TempFileName, _FileName))
676 return false;
679 _F = nlfopen(fileToOpen, mode);
681 return _F!=NULL;
683 // ======================================================================================================
684 void COFile::close()
686 internalClose(true);
688 // ======================================================================================================
689 void COFile::internalClose(bool success)
691 if(_F)
693 fclose(_F);
695 // Temporary filename ?
696 if (!_TempFileName.empty())
698 // Delete old
699 if (success)
701 // Bug under windows, sometimes the file is not deleted
702 uint retry = 1000;
703 while (--retry)
705 if (CFile::fileExists(_FileName))
706 CFile::deleteFile (_FileName);
708 if (CFile::moveFile(_FileName, _TempFileName))
709 break;
710 nlSleep (0);
712 if (!retry)
713 throw ERenameError (_FileName, _TempFileName);
715 else
716 CFile::deleteFile (_TempFileName);
719 _F=NULL;
721 resetPtrTable();
723 // ======================================================================================================
724 void COFile::flush()
726 if(_F)
728 fflush(_F);
733 // ======================================================================================================
734 void COFile::serialBuffer(uint8 *buf, uint len)
736 if(!_F)
737 throw EFileNotOpened(_FileName);
738 if(fwrite(buf, len, 1, _F) != 1)
739 // if(fwrite(buf, 1, len, _F) != len)
741 if (ferror(_F) && errno == 28 /*ENOSPC*/)
743 throw EDiskFullError(_FileName);
745 throw EWriteError(_FileName);
748 // ======================================================================================================
749 void COFile::serialBit(bool &bit)
751 // Simple for now.
752 uint8 v=bit;
753 serialBuffer(&v, 1);
755 // ======================================================================================================
756 bool COFile::seek (sint32 offset, IStream::TSeekOrigin origin) const
758 if (_F)
760 int origin_c = SEEK_SET;
761 switch (origin)
763 case IStream::begin:
764 origin_c=SEEK_SET;
765 break;
766 case IStream::current:
767 origin_c=SEEK_CUR;
768 break;
769 case IStream::end:
770 origin_c=SEEK_END;
771 break;
772 default:
773 nlstop;
776 if (nlfseek64 (_F, offset, origin_c)!=0)
777 return false;
778 return true;
780 return false;
782 // ======================================================================================================
783 sint32 COFile::getPos () const
785 if (_F)
787 return ftell (_F);
789 return 0;
792 // ======================================================================================================
793 std::string COFile::getStreamName() const
795 return _FileName;
800 // ======================================================================================================
801 // ======================================================================================================
802 // ======================================================================================================
805 // ======================================================================================================
806 NLMISC_CATEGORISED_COMMAND(nel, iFileAccessLogStart, "Start file access logging", "")
808 if (!args.empty())
809 return false;
811 IFileAccessLoggingEnabled= true;
812 if (IFileAccessLogStartTime==0)
814 uint64 timeNow = NLMISC::CTime::getPerformanceTime();
815 IFileAccessLogStartTime= timeNow;
818 return true;
821 // ======================================================================================================
822 NLMISC_CATEGORISED_COMMAND(nel, iFileAccessLogStop, "Stop file access logging", "")
824 if (!args.empty())
825 return false;
827 IFileAccessLoggingEnabled= false;
829 return true;
832 // ======================================================================================================
833 NLMISC_CATEGORISED_COMMAND(nel, iFileAccessLogClear, "Clear file access logs", "")
835 if (!args.empty())
836 return false;
838 TSynchronizedFileAccessLog::CAccessor(&IFileAccessLog).value().clear();
840 return true;
843 // ======================================================================================================
844 NLMISC_CATEGORISED_COMMAND(nel, iFileAccessLogDisplay, "Display file access logs", "")
846 if (!args.empty())
847 return false;
849 log.displayNL("-- FILE ACCESS LOG BEGIN --");
851 TSynchronizedFileAccessLog::CAccessor fileAccesLog(&IFileAccessLog);
852 TFileAccessLog::const_iterator it= fileAccesLog.value().begin();
853 TFileAccessLog::const_iterator itEnd= fileAccesLog.value().end();
854 uint32 count=0;
855 while (it!=itEnd)
857 uint32 numTimes= (uint32)it->second.size();
858 CSString fileName= it->first;
859 if (fileName.contains("@"))
861 log.display("%d,%s,%s,",numTimes,fileName.splitTo("@").c_str(),fileName.splitFrom("@").c_str());
863 else
865 log.display("%d,,%s,",numTimes,fileName.c_str());
867 TFileAccessTimes::const_iterator atIt= it->second.begin();
868 TFileAccessTimes::const_iterator atItEnd=it->second.end();
869 while (atIt!=atItEnd)
871 uint64 delta= (*atIt-IFileAccessLogStartTime);
872 log.display("%" NL_I64 "u,",delta);
873 ++atIt;
875 log.displayNL("");
876 ++count;
877 ++it;
880 log.displayNL("-- FILE ACCESS LOG END (%d Unique Files Accessed) --",count);
882 return true;