1 // NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
2 // Copyright (C) 2010-2019 Winch Gate Property Limited
4 // This source file has been modified by the following contributors:
5 // Copyright (C) 2014-2020 Jan BOON (Kaetemi) <jan.boon@kaetemi.be>
7 // This program is free software: you can redistribute it and/or modify
8 // it under the terms of the GNU Affero General Public License as
9 // published by the Free Software Foundation, either version 3 of the
10 // License, or (at your option) any later version.
12 // This program is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 // GNU Affero General Public License for more details.
17 // You should have received a copy of the GNU Affero General Public License
18 // along with this program. If not, see <http://www.gnu.org/licenses/>.
22 #include "nel/misc/file.h"
23 #include "nel/misc/debug.h"
24 #include "nel/misc/big_file.h"
25 #include "nel/misc/path.h"
26 #include "nel/misc/command.h"
27 #include "nel/misc/sstring.h"
28 #include "nel/misc/xml_pack.h"
29 #include "nel/misc/streamed_package_manager.h"
37 #define NLMISC_DONE_FILE_OPENED 40
46 typedef std::list
<uint64
> TFileAccessTimes
; // list of times at which a given file is opened for reading
47 typedef CHashMap
<std::string
,TFileAccessTimes
> TFileAccessLog
; // map from file name to read access times
48 typedef NLMISC::CSynchronized
<TFileAccessLog
> TSynchronizedFileAccessLog
;
50 static TSynchronizedFileAccessLog
IFileAccessLog("IFileAccessLog");
51 static bool IFileAccessLoggingEnabled
= false;
52 static uint64 IFileAccessLogStartTime
= 0;
54 uint32
CIFile::_NbBytesSerialized
= 0;
55 uint32
CIFile::_NbBytesLoaded
= 0;
56 uint32
CIFile::_ReadFromFile
= 0;
57 uint32
CIFile::_ReadingFromFile
= 0;
58 uint32
CIFile::_FileOpened
= 0;
59 uint32
CIFile::_FileRead
= 0;
60 CSynchronized
<std::deque
<std::string
> > CIFile::_OpenedFiles("");
62 // ======================================================================================================
63 CIFile::CIFile() : IStream(true)
71 _IsInXMLPackFile
= false;
72 _CacheFileOnOpen
= false;
73 _IsAsyncLoading
= false;
74 _AllowBNPCacheFileOnOpen
= true;
77 // ======================================================================================================
78 CIFile::CIFile(const std::string
&path
, bool text
) : IStream(true)
86 _IsInXMLPackFile
= false;
87 _CacheFileOnOpen
= false;
88 _IsAsyncLoading
= false;
89 _AllowBNPCacheFileOnOpen
= true;
93 // ======================================================================================================
100 // ======================================================================================================
101 void CIFile::loadIntoCache()
103 const uint32 READPACKETSIZE
= 64 * 1024;
104 const uint32 INTERPACKETSLEEP
= 5;
106 _Cache
= new uint8
[_FileSize
];
109 _ReadingFromFile
+= _FileSize
;
110 int read
= (int)fread (_Cache
, _FileSize
, 1, _F
);
112 _ReadingFromFile
-= _FileSize
;
113 _ReadFromFile
+= read
* _FileSize
;
118 while(index
<_FileSize
)
120 if( _NbBytesLoaded
+ (_FileSize
-index
) > READPACKETSIZE
)
122 sint n
= READPACKETSIZE
-_NbBytesLoaded
;
124 _ReadingFromFile
+= n
;
125 int read
= (int)fread (_Cache
+index
, n
, 1, _F
);
127 _ReadingFromFile
-= n
;
128 _ReadFromFile
+= read
* n
;
131 nlSleep (INTERPACKETSLEEP
);
136 uint n
= _FileSize
-index
;
137 _ReadingFromFile
+= n
;
138 int read
= (int)fread (_Cache
+index
, n
, 1, _F
);
140 _ReadingFromFile
-= n
;
141 _ReadFromFile
+= read
* n
;
150 // ======================================================================================================
151 bool CIFile::open(const std::string
&path
, bool text
)
155 CSynchronized
<deque
<string
> >::CAccessor
fileOpened(&_OpenedFiles
);
156 fileOpened
.value().push_front (path
);
157 if (fileOpened
.value().size () > NLMISC_DONE_FILE_OPENED
)
158 fileOpened
.value().resize (NLMISC_DONE_FILE_OPENED
);
164 if ((_IsInBigFile
|| _IsInXMLPackFile
) && path
.find('@') == string::npos
)
166 // CIFile can be reused to load file from bnp and from regular files.
167 // Last open happened to be inside bnp and close() may not set _F to NULL.
168 // Opening regular file will fail as _F points to bnp file.
172 // can't open empty filename
176 // IFile Access Log management
177 if (IFileAccessLoggingEnabled
)
179 // get the current time
180 uint64 timeNow
= NLMISC::CTime::getPerformanceTime();
182 // get a handle for the container
183 TSynchronizedFileAccessLog::CAccessor
synchronizedFileAccesLog(&IFileAccessLog
);
184 TFileAccessLog
& fileAccessLog
= synchronizedFileAccesLog
.value();
186 // add the current time to the container entry for the given path (creating a new container entry if required)
187 fileAccessLog
[path
].push_back(timeNow
);
192 mode
[1] = 'b'; // No more reading in text mode
198 // Bigfile or xml pack access requested ?
199 string::size_type pos
;
200 if ((pos
= path
.find('@')) != string::npos
)
202 // check for a double @ to identify XML pack file
203 if (pos
+1 < path
.size() && path
[pos
+1] == '@')
206 _IsInXMLPackFile
= true;
208 if(_AllowBNPCacheFileOnOpen
)
210 _F
= CXMLPack::getInstance().getFile(path
, _FileSize
, _BigFileOffset
, _CacheFileOnOpen
, _AlwaysOpened
);
215 _F
= CXMLPack::getInstance().getFile (path
, _FileSize
, _BigFileOffset
, dummy
, _AlwaysOpened
);
218 else if (pos
> 3 && path
[pos
-3] == 's' && path
[pos
-2] == 'n' && path
[pos
-1] == 'p')
220 // nldebug("Opening a streamed package file");
222 _IsInXMLPackFile
= false;
223 _IsInBigFile
= false;
225 _AlwaysOpened
= false;
226 std::string filePath
;
227 if (CStreamedPackageManager::getInstance().getFile (filePath
, path
.substr(pos
+1)))
229 _F
= fopen (filePath
.c_str(), mode
);
232 _FileSize
=CFile::getFileSize(_F
);
235 nlwarning("FILE: Size of file '%s' is 0", path
.c_str());
242 nlwarning("Failed to open file '%s', error %u : %s", path
.c_str(), errno
, strerror(errno
));
249 // nlerror("File '%s' not in streamed package", path.c_str());
256 if(_AllowBNPCacheFileOnOpen
)
258 _F
= CBigFile::getInstance().getFile (path
, _FileSize
, _BigFileOffset
, _CacheFileOnOpen
, _AlwaysOpened
);
263 _F
= CBigFile::getInstance().getFile (path
, _FileSize
, _BigFileOffset
, dummy
, _AlwaysOpened
);
268 // Start to load the bigfile or xml file at the file offset.
269 nlfseek64 (_F
, _BigFileOffset
, SEEK_SET
);
272 if (_CacheFileOnOpen
)
274 // load file in the cache
282 return (_Cache
!= NULL
);
287 // not in bnp, but may have '@' in the name
290 _IsInBigFile
= false;
291 _IsInXMLPackFile
= false;
293 _AlwaysOpened
= false;
294 _F
= nlfopen (path
, mode
);
298 THIS CODE REPLACED BY SADGE BECAUSE SOMETIMES
299 ftell() RETRUNS 0 FOR NO GOOD REASON - LEADING TO CLIENT CRASH
301 nlfseek64 (_F, 0, SEEK_END);
302 _FileSize = ftell(_F);
303 nlfseek64 (_F, 0, SEEK_SET);
304 nlassert(_FileSize==filelength(fileno(_F)));
306 THE FOLLOWING WORKS BUT IS NOT PORTABLE
307 _FileSize=filelength(fileno(_F));
309 _FileSize
=CFile::getFileSize (_F
);
312 nlwarning ("FILE: Size of file '%s' is 0", path
.c_str());
319 nlwarning("Failed to open file '%s', error %u : %s", path
.c_str(), errno
, strerror(errno
));
323 if ((_CacheFileOnOpen
) && (_F
!= NULL
))
325 // load file in the cache
330 return (_Cache
!= NULL
);
337 // ======================================================================================================
338 void CIFile::setCacheFileOnOpen (bool newState
)
340 _CacheFileOnOpen
= newState
;
343 // ======================================================================================================
344 void CIFile::setAsyncLoading (bool newState
)
346 _IsAsyncLoading
= true;
350 // ======================================================================================================
353 if (_CacheFileOnOpen
)
363 if (_IsInBigFile
|| _IsInXMLPackFile
)
383 nlassert(_Cache
== NULL
);
387 // ======================================================================================================
390 if (_CacheFileOnOpen
)
402 // ======================================================================================================
403 bool CIFile::readAll(std::string
&buffer
)
407 uint32 remaining
= _FileSize
;
410 buffer
.reserve(_FileSize
);
411 while(!eof() && remaining
> 0)
413 const static uint bufsize
= 1024;
415 uint32 readnow
= bufsize
;
416 if (readnow
> remaining
)
419 serialBuffer((uint8
*)&buf
[0], readnow
);
420 buffer
.append(buf
, readnow
);
421 remaining
-= readnow
;
424 catch (const EFile
&)
426 // buffer state is unknown
433 // ======================================================================================================
434 void CIFile::getline (char *buffer
, uint32 bufferSize
)
442 if (read
== bufferSize
- 1)
451 serialBuffer ((uint8
*)buffer
, 1);
453 catch (const EFile
&)
476 // ======================================================================================================
479 return _ReadPos
>= (sint32
)_FileSize
;
482 // ======================================================================================================
483 void CIFile::serialBuffer(uint8
*buf
, uint len
)
487 // Check the read pos
488 if ((_ReadPos
< 0) || ((_ReadPos
+len
) > _FileSize
))
489 throw EReadError (_FileName
);
490 if ((_CacheFileOnOpen
) && (_Cache
== NULL
))
491 throw EFileNotOpened (_FileName
);
492 if ((!_CacheFileOnOpen
) && (_F
== NULL
))
493 throw EFileNotOpened (_FileName
);
497 _NbBytesSerialized
+= len
;
498 if (_NbBytesSerialized
> 64 * 1024)
500 // give up time slice
502 _NbBytesSerialized
= 0;
506 if (_CacheFileOnOpen
)
508 memcpy (buf
, _Cache
+ _ReadPos
, len
);
514 _ReadingFromFile
+= len
;
515 read
=(int)fread(buf
, len
, 1, _F
);
517 _ReadingFromFile
-= len
;
518 _ReadFromFile
+= /*read **/ len
;
519 if (read
!= 1 /*< (int)len*/)
520 throw EReadError(_FileName
);
525 // ======================================================================================================
526 void CIFile::serialBit(bool &bit
)
534 // ======================================================================================================
535 bool CIFile::seek (sint32 offset
, IStream::TSeekOrigin origin
) const
537 if ((_CacheFileOnOpen
) && (_Cache
== NULL
))
539 if ((!_CacheFileOnOpen
) && (_F
== NULL
))
547 case IStream::current
:
548 _ReadPos
= _ReadPos
+ offset
;
551 _ReadPos
= _FileSize
+ offset
;
557 if (_CacheFileOnOpen
)
560 // seek in the file. NB: if not in bigfile, _BigFileOffset==0.
561 if (nlfseek64(_F
, (sint64
)_BigFileOffset
+ _ReadPos
, SEEK_SET
) != 0)
566 // ======================================================================================================
567 sint32
CIFile::getPos () const
573 // ======================================================================================================
574 std::string
CIFile::getStreamName() const
580 // ======================================================================================================
581 void CIFile::allowBNPCacheFileOnOpen(bool newState
)
583 _AllowBNPCacheFileOnOpen
= newState
;
587 // ======================================================================================================
588 void CIFile::dump (std::vector
<std::string
> &result
)
590 CSynchronized
<deque
<string
> >::CAccessor
acces(&_OpenedFiles
);
592 const deque
<string
> &openedFile
= acces
.value();
594 // Resize the destination array
596 result
.reserve (openedFile
.size ());
598 // Add the waiting strings
599 deque
<string
>::const_reverse_iterator ite
= openedFile
.rbegin ();
600 while (ite
!= openedFile
.rend ())
602 result
.push_back (*ite
);
609 // ======================================================================================================
610 void CIFile::clearDump ()
612 CSynchronized
<deque
<string
> >::CAccessor
acces(&_OpenedFiles
);
613 acces
.value().clear();
616 // ======================================================================================================
617 uint
CIFile::getDbgStreamSize() const
619 return getFileSize();
623 // ======================================================================================================
624 // ======================================================================================================
625 // ======================================================================================================
628 // ======================================================================================================
629 COFile::COFile() : IStream(false)
634 // ======================================================================================================
635 COFile::COFile(const std::string
&path
, bool append
, bool text
, bool useTempFile
) : IStream(false)
638 open(path
, append
, text
, useTempFile
);
641 // ======================================================================================================
644 internalClose(false);
646 // ======================================================================================================
647 bool COFile::open(const std::string
&path
, bool append
, bool text
, bool useTempFile
)
651 // can't open empty filename
656 _TempFileName
.clear();
659 mode
[0] = (append
)?'a':'w';
660 // ACE: NEVER SAVE IN TEXT MODE!!! mode[1] = (text)?'\0':'b';
664 string fileToOpen
= path
;
667 CFile::getTemporaryOutputFilename (path
, _TempFileName
);
668 fileToOpen
= _TempFileName
;
671 // if appending to file and using a temporary file, copycat temporary file from original...
672 if (append
&& useTempFile
&& CFile::fileExists(_FileName
))
674 // open fails if can't copy original content
675 if (!CFile::copyFile(_TempFileName
, _FileName
))
679 _F
= nlfopen(fileToOpen
, mode
);
683 // ======================================================================================================
688 // ======================================================================================================
689 void COFile::internalClose(bool success
)
695 // Temporary filename ?
696 if (!_TempFileName
.empty())
701 // Bug under windows, sometimes the file is not deleted
705 if (CFile::fileExists(_FileName
))
706 CFile::deleteFile (_FileName
);
708 if (CFile::moveFile(_FileName
, _TempFileName
))
713 throw ERenameError (_FileName
, _TempFileName
);
716 CFile::deleteFile (_TempFileName
);
723 // ======================================================================================================
733 // ======================================================================================================
734 void COFile::serialBuffer(uint8
*buf
, uint len
)
737 throw EFileNotOpened(_FileName
);
738 if(fwrite(buf
, len
, 1, _F
) != 1)
739 // if(fwrite(buf, 1, len, _F) != len)
741 if (ferror(_F
) && errno
== 28 /*ENOSPC*/)
743 throw EDiskFullError(_FileName
);
745 throw EWriteError(_FileName
);
748 // ======================================================================================================
749 void COFile::serialBit(bool &bit
)
755 // ======================================================================================================
756 bool COFile::seek (sint32 offset
, IStream::TSeekOrigin origin
) const
760 int origin_c
= SEEK_SET
;
766 case IStream::current
:
776 if (nlfseek64 (_F
, offset
, origin_c
)!=0)
782 // ======================================================================================================
783 sint32
COFile::getPos () const
792 // ======================================================================================================
793 std::string
COFile::getStreamName() const
800 // ======================================================================================================
801 // ======================================================================================================
802 // ======================================================================================================
805 // ======================================================================================================
806 NLMISC_CATEGORISED_COMMAND(nel
, iFileAccessLogStart
, "Start file access logging", "")
811 IFileAccessLoggingEnabled
= true;
812 if (IFileAccessLogStartTime
==0)
814 uint64 timeNow
= NLMISC::CTime::getPerformanceTime();
815 IFileAccessLogStartTime
= timeNow
;
821 // ======================================================================================================
822 NLMISC_CATEGORISED_COMMAND(nel
, iFileAccessLogStop
, "Stop file access logging", "")
827 IFileAccessLoggingEnabled
= false;
832 // ======================================================================================================
833 NLMISC_CATEGORISED_COMMAND(nel
, iFileAccessLogClear
, "Clear file access logs", "")
838 TSynchronizedFileAccessLog::CAccessor(&IFileAccessLog
).value().clear();
843 // ======================================================================================================
844 NLMISC_CATEGORISED_COMMAND(nel
, iFileAccessLogDisplay
, "Display file access logs", "")
849 log
.displayNL("-- FILE ACCESS LOG BEGIN --");
851 TSynchronizedFileAccessLog::CAccessor
fileAccesLog(&IFileAccessLog
);
852 TFileAccessLog::const_iterator it
= fileAccesLog
.value().begin();
853 TFileAccessLog::const_iterator itEnd
= fileAccesLog
.value().end();
857 uint32 numTimes
= (uint32
)it
->second
.size();
858 CSString fileName
= it
->first
;
859 if (fileName
.contains("@"))
861 log
.display("%d,%s,%s,",numTimes
,fileName
.splitTo("@").c_str(),fileName
.splitFrom("@").c_str());
865 log
.display("%d,,%s,",numTimes
,fileName
.c_str());
867 TFileAccessTimes::const_iterator atIt
= it
->second
.begin();
868 TFileAccessTimes::const_iterator atItEnd
=it
->second
.end();
869 while (atIt
!=atItEnd
)
871 uint64 delta
= (*atIt
-IFileAccessLogStartTime
);
872 log
.display("%" NL_I64
"u,",delta
);
880 log
.displayNL("-- FILE ACCESS LOG END (%d Unique Files Accessed) --",count
);