3 Copyright (C) 2005 and later Cockos Incorporated
5 This software is provided 'as-is', without any express or implied
6 warranty. In no event will the authors be held liable for any damages
7 arising from the use of this software.
9 Permission is granted to anyone to use this software for any purpose,
10 including commercial applications, and to alter it and redistribute it
11 freely, subject to the following restrictions:
13 1. The origin of this software must not be misrepresented; you must not
14 claim that you wrote the original software. If you use this software
15 in a product, an acknowledgment in the product documentation would be
16 appreciated but is not required.
17 2. Altered source versions must be plainly marked as such, and must not be
18 misrepresented as being the original software.
19 3. This notice may not be removed or altered from any source distribution.
22 This file provides the WDL_FileWrite object, which can be used to create/write files.
23 On windows systems it supports writing synchronously, asynchronously, and asynchronously without buffering.
24 On windows systems it supports files larger than 4gb.
25 On non-windows systems it acts as a wrapper for fopen()/etc.
31 #ifndef _WDL_FILEWRITE_H_
32 #define _WDL_FILEWRITE_H_
34 #ifndef WDL_FILEWRITE_ON_ERROR
35 #define WDL_FILEWRITE_ON_ERROR(is_full)
42 #if defined(_WIN32) && !defined(WDL_NO_WIN32_FILEWRITE)
43 #ifndef WDL_WIN32_NATIVE_WRITE
44 #define WDL_WIN32_NATIVE_WRITE
47 #ifdef WDL_WIN32_NATIVE_WRITE
48 #undef WDL_WIN32_NATIVE_WRITE
50 #if !defined(WDL_NO_POSIX_FILEWRITE)
51 #include <sys/fcntl.h>
54 #include <sys/errno.h>
55 #define WDL_POSIX_NATIVE_WRITE
56 extern struct stat wdl_stat_chk
;
57 // if this fails on linux, use CFLAGS += -D_FILE_OFFSET_BITS=64
58 typedef char wdl_filewrite_assert_failed_stat_not_64
[sizeof(wdl_stat_chk
.st_size
)!=8 ? -1 : 1];
59 typedef char wdl_filewrite_assert_failed_off_t_64
[sizeof(off_t
)!=8 ? -1 : 1];
66 #define WDL_FILEWRITE_POSTYPE __int64
68 #define WDL_FILEWRITE_POSTYPE long long
73 #ifdef WDL_WIN32_NATIVE_WRITE
75 class WDL_FileWrite__WriteEnt
78 WDL_FileWrite__WriteEnt(int sz
)
83 m_bufptr
= (char *)__buf
.Resize(sz
+4095);
84 int a
=((int)(INT_PTR
)m_bufptr
)&4095;
85 if (a
) m_bufptr
+= 4096-a
;
87 memset(&m_ol
,0,sizeof(m_ol
));
88 m_ol
.hEvent
=CreateEvent(NULL
,TRUE
,TRUE
,NULL
);
90 ~WDL_FileWrite__WriteEnt()
92 CloseHandle(m_ol
.hEvent
);
95 WDL_FILEWRITE_POSTYPE m_last_writepos
;
97 int m_bufused
,m_bufsz
;
100 WDL_TypedBuf
<char> __buf
;
105 #if defined(_WIN32) && !defined(WDL_NO_SUPPORT_UTF8)
106 BOOL
HasUTF8(const char *_str
)
108 const unsigned char *str
= (const unsigned char *)_str
;
109 if (!str
) return FALSE
;
112 unsigned char c
= *str
;
115 if (c
<= 0xDF && str
[1] >=0x80 && str
[1] <= 0xBF) return TRUE
;
116 else if (c
<= 0xEF && str
[1] >=0x80 && str
[1] <= 0xBF && str
[2] >=0x80 && str
[2] <= 0xBF) return TRUE
;
117 else if (c
<= 0xF4 && str
[1] >=0x80 && str
[1] <= 0xBF && str
[2] >=0x80 && str
[2] <= 0xBF) return TRUE
;
120 if (((const char *)str
-_str
) >= 256) return TRUE
; // long filenames get converted to wide
128 // async==2 is write-through
129 // async==3 is non-buffered (win32-only)
130 WDL_FileWrite(const char *filename
, int allow_async
=1, int bufsize
=8192, int minbufs
=16, int maxbufs
=16, bool wantAppendTo
=false, bool noFileLocking
=false)
133 m_file_max_position
=0;
136 #ifdef WDL_WIN32_NATIVE_WRITE
137 m_fh
= INVALID_HANDLE_VALUE
;
139 #elif defined(WDL_POSIX_NATIVE_WRITE)
140 m_filedes_locked
=false;
149 #ifdef WDL_WIN32_NATIVE_WRITE
150 #ifdef WDL_SUPPORT_WIN9X
151 const bool isNT
= (GetVersion()<0x80000000);
153 const bool isNT
= true;
155 m_async
= allow_async
&& isNT
? 1 : 0;
156 if (m_async
&& allow_async
== 3 && !wantAppendTo
)
159 bufsize
= (bufsize
+4095)&~4095;
160 if (bufsize
<4096) bufsize
=4096;
163 int rwflag
= GENERIC_WRITE
;
164 int createFlag
= wantAppendTo
?OPEN_ALWAYS
:CREATE_ALWAYS
;
165 int shareFlag
= noFileLocking
? (FILE_SHARE_READ
|FILE_SHARE_WRITE
) : FILE_SHARE_READ
;
166 int flag
= FILE_ATTRIBUTE_NORMAL
;
170 rwflag
|= GENERIC_READ
;
172 flag
|= FILE_FLAG_OVERLAPPED
|FILE_FLAG_NO_BUFFERING
|FILE_FLAG_WRITE_THROUGH
;
174 flag
|= FILE_FLAG_OVERLAPPED
|(allow_async
>1 ? FILE_FLAG_WRITE_THROUGH
: 0);
178 #ifndef WDL_NO_SUPPORT_UTF8
179 bool allow_ansi
= true;
180 m_fh
=INVALID_HANDLE_VALUE
;
181 if (isNT
&& HasUTF8(filename
))
183 int szreq
=MultiByteToWideChar(CP_UTF8
,MB_ERR_INVALID_CHARS
,filename
,-1,NULL
,0);
186 WDL_TypedBuf
<WCHAR
> wfilename
;
187 wfilename
.Resize(szreq
+20);
188 if (MultiByteToWideChar(CP_UTF8
,MB_ERR_INVALID_CHARS
,filename
,-1,wfilename
.Get(),wfilename
.GetSize()-10))
190 correctlongpath(wfilename
.Get());
192 m_fh
= CreateFileW(wfilename
.Get(),rwflag
,shareFlag
,NULL
,createFlag
,flag
,NULL
);
197 WCHAR wfilename
[1024];
198 if (MultiByteToWideChar(CP_UTF8
,MB_ERR_INVALID_CHARS
,filename
,-1,wfilename
,1024-10))
200 correctlongpath(wfilename
);
202 m_fh
= CreateFileW(wfilename
,rwflag
,shareFlag
,NULL
,createFlag
,flag
,NULL
);
207 if (m_fh
== INVALID_HANDLE_VALUE
&& allow_ansi
)
209 m_fh
= CreateFileA(filename
,rwflag
,shareFlag
,NULL
,createFlag
,flag
,NULL
);
212 if (m_async
&& m_fh
!= INVALID_HANDLE_VALUE
)
214 m_async_bufsize
=bufsize
;
215 m_async_maxbufs
=maxbufs
;
216 m_async_minbufs
=minbufs
;
218 for (x
= 0; x
< m_async_minbufs
; x
++)
220 WDL_FileWrite__WriteEnt
*t
=new WDL_FileWrite__WriteEnt(m_async_bufsize
);
225 if (m_fh
!= INVALID_HANDLE_VALUE
&& wantAppendTo
)
226 SetPosition(GetSize());
228 #elif defined(WDL_POSIX_NATIVE_WRITE)
230 m_filedes_locked
=false;
231 m_filedes
=open(filename
,O_WRONLY
|O_CREAT
232 // todo: use fcntl() for platforms when O_CLOEXEC is not available (if we ever need to support them)
233 // (currently the only platform that meets this criteria is macOS w/ old SDK, but we don't use execve()
244 m_filedes_locked
= !flock(m_filedes
,LOCK_EX
|LOCK_NB
);
245 if (!m_filedes_locked
)
247 // this check might not be necessary, it might be sufficient to just fail and close if no exclusive lock possible
248 if (errno
== EWOULDBLOCK
)
250 // FAILED exclusive locking because someone else has a lock
254 else // failed for some other reason, try to keep a shared lock at least
256 m_filedes_locked
= !flock(m_filedes
,LOCK_SH
|LOCK_NB
);
265 if (ftruncate(m_filedes
,0) < 0)
267 WDL_ASSERT( false /* ftruncate() failed in WDL_FileWrite */ );
273 if (!fstat(m_filedes
,&st
)) SetPosition(st
.st_size
);
279 if (m_filedes
>= 0 && allow_async
>1) fcntl(m_filedes
,F_NOCACHE
,1);
282 if (minbufs
* bufsize
>= 16384) m_bufspace
.Resize((minbufs
*bufsize
+4095)&~4095);
284 m_fp
=fopen(filename
,wantAppendTo
? "a+b" : "wb");
285 if (wantAppendTo
&& m_fp
)
286 fseek(m_fp
,0,SEEK_END
);
292 #ifdef WDL_WIN32_NATIVE_WRITE
293 // todo, async close stuff?
294 if (m_fh
!= INVALID_HANDLE_VALUE
&& m_async
)
299 m_empties
.Empty(true);
300 m_pending
.Empty(true);
302 if (m_fh
!= INVALID_HANDLE_VALUE
) CloseHandle(m_fh
);
303 m_fh
=INVALID_HANDLE_VALUE
;
304 #elif defined(WDL_POSIX_NATIVE_WRITE)
307 if (m_bufspace
.GetSize() > 0 && m_bufspace_used
>0)
309 int v
=(int)pwrite(m_filedes
,m_bufspace
.Get(),m_bufspace_used
,m_file_position
);
310 if (v
>0) m_file_position
+=v
;
311 if (m_file_position
> m_file_max_position
) m_file_max_position
=m_file_position
;
314 if (m_filedes_locked
) flock(m_filedes
,LOCK_UN
);
320 if (m_fp
) fclose(m_fp
);
328 #ifdef WDL_WIN32_NATIVE_WRITE
329 return (m_fh
!= INVALID_HANDLE_VALUE
);
330 #elif defined(WDL_POSIX_NATIVE_WRITE)
331 return m_filedes
>= 0;
338 int Write(const void *buf
, int len
)
340 #ifdef WDL_WIN32_NATIVE_WRITE
341 if (m_fh
== INVALID_HANDLE_VALUE
) return 0;
348 if (!m_empties
.GetSize())
350 WDL_FileWrite__WriteEnt
*ent
=m_pending
.Get(0);
356 if (GetOverlappedResult(m_fh
,&ent
->m_ol
,&s
,FALSE
)||
357 (wasabort
=((err
=GetLastError())==ERROR_OPERATION_ABORTED
)))
363 if (!RunAsyncWrite(ent
,false)) m_empties
.Add(ent
);
371 else if (err
!= ERROR_IO_PENDING
&& err
!= ERROR_IO_INCOMPLETE
)
373 WDL_FILEWRITE_ON_ERROR(err
== ERROR_DISK_FULL
)
379 WDL_FileWrite__WriteEnt
*ent
=m_empties
.Get(0);
382 if (m_pending
.GetSize()>=m_async_maxbufs
)
387 if (!(ent
=m_empties
.Get(0)))
388 m_empties
.Add(ent
= new WDL_FileWrite__WriteEnt(m_async_bufsize
)); // new buffer
393 int ml
=ent
->m_bufsz
-ent
->m_bufused
;
395 memcpy(ent
->m_bufptr
+ent
->m_bufused
,(const char *)buf
+ rdpos
,ml
);
401 if (ent
->m_bufused
>= ent
->m_bufsz
)
403 if (RunAsyncWrite(ent
,true)) m_empties
.Delete(0); // if queued remove from list
411 if (!WriteFile(m_fh
,buf
,len
,&dw
,NULL
))
413 WDL_FILEWRITE_ON_ERROR(GetLastError() == ERROR_DISK_FULL
)
416 if (m_file_position
>m_file_max_position
) m_file_max_position
=m_file_position
;
419 #elif defined(WDL_POSIX_NATIVE_WRITE)
420 if (m_bufspace
.GetSize()>0)
422 char *rdptr
= (char *)buf
;
426 int amt
= m_bufspace
.GetSize() - m_bufspace_used
;
429 if (amt
>rdlen
) amt
=rdlen
;
430 memcpy((char *)m_bufspace
.Get()+m_bufspace_used
,rdptr
,amt
);
431 m_bufspace_used
+= amt
;
435 if (m_file_position
+m_bufspace_used
> m_file_max_position
) m_file_max_position
=m_file_position
+ m_bufspace_used
;
437 if (m_bufspace_used
>= m_bufspace
.GetSize())
439 int v
=(int)pwrite(m_filedes
,m_bufspace
.Get(),m_bufspace_used
,m_file_position
);
440 if (v
!= m_bufspace_used
) { WDL_FILEWRITE_ON_ERROR(v
>=0 || errno
== EDQUOT
|| errno
== ENOSPC
) }
441 if (v
>0) m_file_position
+=v
;
449 int v
=(int)pwrite(m_filedes
,buf
,len
,m_file_position
);
450 if (v
!= len
) { WDL_FILEWRITE_ON_ERROR(v
>=0 || errno
== EDQUOT
|| errno
== ENOSPC
) }
451 if (v
>0) m_file_position
+=v
;
452 if (m_file_position
> m_file_max_position
) m_file_max_position
=m_file_position
;
456 int written
= (int)fwrite(buf
,1,len
,m_fp
);
457 if (written
!= len
) { WDL_FILEWRITE_ON_ERROR(false) }
464 WDL_FILEWRITE_POSTYPE
GetSize()
466 #ifdef WDL_WIN32_NATIVE_WRITE
467 if (m_fh
== INVALID_HANDLE_VALUE
) return 0;
469 DWORD l
=GetFileSize(m_fh
,&h
);
470 WDL_FILEWRITE_POSTYPE tmp
=(((WDL_FILEWRITE_POSTYPE
)h
)<<32)|l
;
471 WDL_FILEWRITE_POSTYPE tmp2
=GetPosition();
472 if (tmp
<m_file_max_position
) return m_file_max_position
;
473 if (tmp
<tmp2
) return tmp2
;
476 #elif defined(WDL_POSIX_NATIVE_WRITE)
477 if (m_filedes
< 0) return -1;
478 return m_file_max_position
;
480 if (!m_fp
) return -1;
481 int opos
=ftell(m_fp
);
482 fseek(m_fp
,0,SEEK_END
);
484 fseek(m_fp
,opos
,SEEK_SET
);
490 WDL_FILEWRITE_POSTYPE
GetPosition()
492 #ifdef WDL_WIN32_NATIVE_WRITE
493 if (m_fh
== INVALID_HANDLE_VALUE
) return -1;
495 WDL_FILEWRITE_POSTYPE pos
=m_file_position
;
498 WDL_FileWrite__WriteEnt
*ent
=m_empties
.Get(0);
499 if (ent
) pos
+=ent
->m_bufused
;
502 #elif defined(WDL_POSIX_NATIVE_WRITE)
503 if (m_filedes
< 0) return -1;
504 return m_file_position
+ m_bufspace_used
;
506 if (!m_fp
) return -1;
512 #ifdef WDL_WIN32_NATIVE_WRITE
514 bool RunAsyncWrite(WDL_FileWrite__WriteEnt
*ent
, bool updatePosition
) // returns true if ent is added to pending
516 if (ent
&& ent
->m_bufused
>0)
520 ent
->m_last_writepos
= m_file_position
;
521 m_file_position
+= ent
->m_bufused
;
522 if (m_file_position
>m_file_max_position
) m_file_max_position
=m_file_position
;
525 if (m_async
== 3 && (ent
->m_bufused
&4095))
527 int offs
=(ent
->m_bufused
&4095);
531 *(WDL_FILEWRITE_POSTYPE
*)&ent
->m_ol
.Offset
= ent
->m_last_writepos
+ ent
->m_bufused
- offs
;
532 ResetEvent(ent
->m_ol
.hEvent
);
535 if (!ReadFile(m_fh
,tmp
,4096,&dw
,&ent
->m_ol
))
537 if (GetLastError() == ERROR_IO_PENDING
)
538 WaitForSingleObject(ent
->m_ol
.hEvent
,INFINITE
);
540 memcpy(ent
->m_bufptr
+ent
->m_bufused
,tmp
+offs
,4096-offs
);
542 ent
->m_bufused
+= 4096-offs
;
547 *(WDL_FILEWRITE_POSTYPE
*)&ent
->m_ol
.Offset
= ent
->m_last_writepos
;
549 ResetEvent(ent
->m_ol
.hEvent
);
551 if (!WriteFile(m_fh
,ent
->m_bufptr
,ent
->m_bufused
,&d
,&ent
->m_ol
))
553 if (GetLastError()==ERROR_IO_PENDING
)
558 else { WDL_FILEWRITE_ON_ERROR(GetLastError()==ERROR_DISK_FULL
) }
565 void SyncOutput(bool syncall
)
569 if (RunAsyncWrite(m_empties
.Get(0),true)) m_empties
.Delete(0);
573 WDL_FileWrite__WriteEnt
*ent
=m_pending
.Get(0);
577 BOOL ok
= GetOverlappedResult(m_fh
,&ent
->m_ol
,&s
,TRUE
);
579 if (!ok
&& (errcode
=GetLastError())==ERROR_OPERATION_ABORTED
)
582 if (!RunAsyncWrite(ent
,false)) m_empties
.Add(ent
);
586 if (!ok
) { WDL_FILEWRITE_ON_ERROR(errcode
==ERROR_DISK_FULL
) }
597 bool SetPosition(WDL_FILEWRITE_POSTYPE pos
) // returns 0 on success
599 #ifdef WDL_WIN32_NATIVE_WRITE
600 if (m_fh
== INVALID_HANDLE_VALUE
) return true;
605 if (m_file_position
>m_file_max_position
) m_file_max_position
=m_file_position
;
607 if (m_async
==3 && (m_file_position
&4095))
609 WDL_FileWrite__WriteEnt
*ent
=m_empties
.Get(0);
612 int psz
=(int) (m_file_position
&4095);
614 m_file_position
-= psz
;
615 *(WDL_FILEWRITE_POSTYPE
*)&ent
->m_ol
.Offset
= m_file_position
;
616 ResetEvent(ent
->m_ol
.hEvent
);
619 if (!ReadFile(m_fh
,ent
->m_bufptr
,4096,&dwo
,&ent
->m_ol
))
621 if (GetLastError() == ERROR_IO_PENDING
)
622 WaitForSingleObject(ent
->m_ol
.hEvent
,INFINITE
);
624 ent
->m_bufused
=(int)psz
;
631 if (m_file_position
>m_file_max_position
) m_file_max_position
=m_file_position
;
633 LONG high
=(LONG
) (m_file_position
>>32);
634 return SetFilePointer(m_fh
,(LONG
)(m_file_position
&((WDL_FILEWRITE_POSTYPE
)0xFFFFFFFF)),&high
,FILE_BEGIN
)==0xFFFFFFFF && GetLastError() != NO_ERROR
;
635 #elif defined(WDL_POSIX_NATIVE_WRITE)
637 if (m_filedes
< 0) return true;
638 if (m_bufspace
.GetSize() > 0 && m_bufspace_used
>0)
640 int v
=(int)pwrite(m_filedes
,m_bufspace
.Get(),m_bufspace_used
,m_file_position
);
641 if (v
>0) m_file_position
+=v
;
642 if (m_file_position
> m_file_max_position
) m_file_max_position
=m_file_position
;
646 m_file_position
= pos
; // seek!
647 if (m_file_position
>m_file_max_position
) m_file_max_position
=m_file_position
;
650 if (!m_fp
) return true;
651 return !!fseek(m_fp
,pos
,SEEK_SET
);
655 WDL_FILEWRITE_POSTYPE m_file_position
, m_file_max_position
;
657 #ifdef WDL_WIN32_NATIVE_WRITE
658 HANDLE
GetHandle() { return m_fh
; }
660 int m_async
; // 3 = unbuffered
662 int m_async_bufsize
, m_async_minbufs
, m_async_maxbufs
;
664 WDL_PtrList
<WDL_FileWrite__WriteEnt
> m_empties
;
665 WDL_PtrList
<WDL_FileWrite__WriteEnt
> m_pending
;
667 #elif defined(WDL_POSIX_NATIVE_WRITE)
668 int GetHandle() { return m_filedes
; }
670 WDL_HeapBuf m_bufspace
;
674 bool m_filedes_locked
;
677 int GetHandle() { return fileno(m_fp
); }
683 static void correctlongpath(WCHAR
*buf
) // this also exists as wdl_utf8_correctlongpath
688 if (!buf
|| !buf
[0] || wcslen(buf
) < 256) return;
689 if (buf
[1] == ':') insert
=L
"\\\\?\\";
690 else if (buf
[0] == '\\' && buf
[1] == '\\') { insert
= L
"\\\\?\\UNC\\"; skip
=2; }
693 wr
= buf
+ wcslen(insert
);
694 memmove(wr
, buf
+ skip
, (wcslen(buf
+skip
)+1)*2);
695 memmove(buf
,insert
,wcslen(insert
)*2);
698 if (*wr
== '/') *wr
= '\\';