langpackedit: sorting fixes, 0.015 -- from 8f06f769
[wdl.git] / WDL / filewrite.h
blob21e711b3df4006ffb8e01c1ddd441b3572255b13
1 /*
2 WDL - filewrite.h
3 Copyright (C) 2005 and later Cockos Incorporated
5 This software is provided 'as-is', without any express or implied
6 warranty. In no event will the authors be held liable for any damages
7 arising from the use of this software.
9 Permission is granted to anyone to use this software for any purpose,
10 including commercial applications, and to alter it and redistribute it
11 freely, subject to the following restrictions:
13 1. The origin of this software must not be misrepresented; you must not
14 claim that you wrote the original software. If you use this software
15 in a product, an acknowledgment in the product documentation would be
16 appreciated but is not required.
17 2. Altered source versions must be plainly marked as such, and must not be
18 misrepresented as being the original software.
19 3. This notice may not be removed or altered from any source distribution.
22 This file provides the WDL_FileWrite object, which can be used to create/write files.
23 On windows systems it supports writing synchronously, asynchronously, and asynchronously without buffering.
24 On windows systems it supports files larger than 4gb.
25 On non-windows systems it acts as a wrapper for fopen()/etc.
31 #ifndef _WDL_FILEWRITE_H_
32 #define _WDL_FILEWRITE_H_
34 #ifndef WDL_FILEWRITE_ON_ERROR
35 #define WDL_FILEWRITE_ON_ERROR(is_full)
36 #endif
38 #include "ptrlist.h"
42 #if defined(_WIN32) && !defined(WDL_NO_WIN32_FILEWRITE)
43 #ifndef WDL_WIN32_NATIVE_WRITE
44 #define WDL_WIN32_NATIVE_WRITE
45 #endif
46 #else
47 #ifdef WDL_WIN32_NATIVE_WRITE
48 #undef WDL_WIN32_NATIVE_WRITE
49 #endif
50 #if !defined(WDL_NO_POSIX_FILEWRITE)
51 #include <sys/fcntl.h>
52 #include <sys/file.h>
53 #include <sys/stat.h>
54 #include <sys/errno.h>
55 #define WDL_POSIX_NATIVE_WRITE
56 extern struct stat wdl_stat_chk;
57 // if this fails on linux, use CFLAGS += -D_FILE_OFFSET_BITS=64
58 typedef char wdl_filewrite_assert_failed_stat_not_64[sizeof(wdl_stat_chk.st_size)!=8 ? -1 : 1];
59 typedef char wdl_filewrite_assert_failed_off_t_64[sizeof(off_t)!=8 ? -1 : 1];
60 #endif
61 #endif
65 #ifdef _MSC_VER
66 #define WDL_FILEWRITE_POSTYPE __int64
67 #else
68 #define WDL_FILEWRITE_POSTYPE long long
69 #endif
71 class WDL_FileWrite
73 #ifdef WDL_WIN32_NATIVE_WRITE
75 class WDL_FileWrite__WriteEnt
77 public:
78 WDL_FileWrite__WriteEnt(int sz)
80 m_last_writepos=0;
81 m_bufused=0;
82 m_bufsz=sz;
83 m_bufptr = (char *)__buf.Resize(sz+4095);
84 int a=((int)(INT_PTR)m_bufptr)&4095;
85 if (a) m_bufptr += 4096-a;
87 memset(&m_ol,0,sizeof(m_ol));
88 m_ol.hEvent=CreateEvent(NULL,TRUE,TRUE,NULL);
90 ~WDL_FileWrite__WriteEnt()
92 CloseHandle(m_ol.hEvent);
95 WDL_FILEWRITE_POSTYPE m_last_writepos;
97 int m_bufused,m_bufsz;
98 OVERLAPPED m_ol;
99 char *m_bufptr;
100 WDL_TypedBuf<char> __buf;
103 #endif
105 #if defined(_WIN32) && !defined(WDL_NO_SUPPORT_UTF8)
106 BOOL HasUTF8(const char *_str)
108 const unsigned char *str = (const unsigned char *)_str;
109 if (!str) return FALSE;
110 while (*str)
112 unsigned char c = *str;
113 if (c >= 0xC2)
115 if (c <= 0xDF && str[1] >=0x80 && str[1] <= 0xBF) return TRUE;
116 else if (c <= 0xEF && str[1] >=0x80 && str[1] <= 0xBF && str[2] >=0x80 && str[2] <= 0xBF) return TRUE;
117 else if (c <= 0xF4 && str[1] >=0x80 && str[1] <= 0xBF && str[2] >=0x80 && str[2] <= 0xBF) return TRUE;
119 str++;
120 if (((const char *)str-_str) >= 256) return TRUE; // long filenames get converted to wide
122 return FALSE;
124 #endif
127 public:
128 // async==2 is write-through
129 // async==3 is non-buffered (win32-only)
130 WDL_FileWrite(const char *filename, int allow_async=1, int bufsize=8192, int minbufs=16, int maxbufs=16, bool wantAppendTo=false, bool noFileLocking=false)
132 m_file_position=0;
133 m_file_max_position=0;
134 if(!filename)
136 #ifdef WDL_WIN32_NATIVE_WRITE
137 m_fh = INVALID_HANDLE_VALUE;
138 m_async = 0;
139 #elif defined(WDL_POSIX_NATIVE_WRITE)
140 m_filedes_locked=false;
141 m_filedes=-1;
142 m_bufspace_used=0;
143 #else
144 m_fp = NULL;
145 #endif
146 return;
149 #ifdef WDL_WIN32_NATIVE_WRITE
150 #ifdef WDL_SUPPORT_WIN9X
151 const bool isNT = (GetVersion()<0x80000000);
152 #else
153 const bool isNT = true;
154 #endif
155 m_async = allow_async && isNT ? 1 : 0;
156 if (m_async && allow_async == 3 && !wantAppendTo)
158 m_async = 3;
159 bufsize = (bufsize+4095)&~4095;
160 if (bufsize<4096) bufsize=4096;
163 int rwflag = GENERIC_WRITE;
164 int createFlag= wantAppendTo?OPEN_ALWAYS:CREATE_ALWAYS;
165 int shareFlag = noFileLocking ? (FILE_SHARE_READ|FILE_SHARE_WRITE) : FILE_SHARE_READ;
166 int flag = FILE_ATTRIBUTE_NORMAL;
168 if (m_async)
170 rwflag |= GENERIC_READ;
171 if (m_async == 3)
172 flag |= FILE_FLAG_OVERLAPPED|FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH;
173 else
174 flag |= FILE_FLAG_OVERLAPPED|(allow_async>1 ? FILE_FLAG_WRITE_THROUGH: 0);
178 #ifndef WDL_NO_SUPPORT_UTF8
179 bool allow_ansi = true;
180 m_fh=INVALID_HANDLE_VALUE;
181 if (isNT && HasUTF8(filename))
183 int szreq=MultiByteToWideChar(CP_UTF8,MB_ERR_INVALID_CHARS,filename,-1,NULL,0);
184 if (szreq > 1000)
186 WDL_TypedBuf<WCHAR> wfilename;
187 wfilename.Resize(szreq+20);
188 if (MultiByteToWideChar(CP_UTF8,MB_ERR_INVALID_CHARS,filename,-1,wfilename.Get(),wfilename.GetSize()-10))
190 correctlongpath(wfilename.Get());
191 allow_ansi = false;
192 m_fh = CreateFileW(wfilename.Get(),rwflag,shareFlag,NULL,createFlag,flag,NULL);
195 else
197 WCHAR wfilename[1024];
198 if (MultiByteToWideChar(CP_UTF8,MB_ERR_INVALID_CHARS,filename,-1,wfilename,1024-10))
200 correctlongpath(wfilename);
201 allow_ansi = false;
202 m_fh = CreateFileW(wfilename,rwflag,shareFlag,NULL,createFlag,flag,NULL);
207 if (m_fh == INVALID_HANDLE_VALUE && allow_ansi)
208 #endif
209 m_fh = CreateFileA(filename,rwflag,shareFlag,NULL,createFlag,flag,NULL);
212 if (m_async && m_fh != INVALID_HANDLE_VALUE)
214 m_async_bufsize=bufsize;
215 m_async_maxbufs=maxbufs;
216 m_async_minbufs=minbufs;
217 int x;
218 for (x = 0; x < m_async_minbufs; x ++)
220 WDL_FileWrite__WriteEnt *t=new WDL_FileWrite__WriteEnt(m_async_bufsize);
221 m_empties.Add(t);
225 if (m_fh != INVALID_HANDLE_VALUE && wantAppendTo)
226 SetPosition(GetSize());
228 #elif defined(WDL_POSIX_NATIVE_WRITE)
229 m_bufspace_used=0;
230 m_filedes_locked=false;
231 m_filedes=open(filename,O_WRONLY|O_CREAT
232 // todo: use fcntl() for platforms when O_CLOEXEC is not available (if we ever need to support them)
233 // (currently the only platform that meets this criteria is macOS w/ old SDK, but we don't use execve()
234 // there
235 #ifdef O_CLOEXEC
236 | O_CLOEXEC
237 #endif
238 ,0644);
239 if (m_filedes>=0)
242 if (!noFileLocking)
244 m_filedes_locked = !flock(m_filedes,LOCK_EX|LOCK_NB);
245 if (!m_filedes_locked)
247 // this check might not be necessary, it might be sufficient to just fail and close if no exclusive lock possible
248 if (errno == EWOULDBLOCK)
250 // FAILED exclusive locking because someone else has a lock
251 close(m_filedes);
252 m_filedes=-1;
254 else // failed for some other reason, try to keep a shared lock at least
256 m_filedes_locked = !flock(m_filedes,LOCK_SH|LOCK_NB);
261 if (m_filedes>=0)
263 if (!wantAppendTo)
265 if (ftruncate(m_filedes,0) < 0)
267 WDL_ASSERT( false /* ftruncate() failed in WDL_FileWrite */ );
270 else
272 struct stat st;
273 if (!fstat(m_filedes,&st)) SetPosition(st.st_size);
278 #ifdef __APPLE__
279 if (m_filedes >= 0 && allow_async>1) fcntl(m_filedes,F_NOCACHE,1);
280 #endif
282 if (minbufs * bufsize >= 16384) m_bufspace.Resize((minbufs*bufsize+4095)&~4095);
283 #else
284 m_fp=fopen(filename,wantAppendTo ? "a+b" : "wb");
285 if (wantAppendTo && m_fp)
286 fseek(m_fp,0,SEEK_END);
287 #endif
290 ~WDL_FileWrite()
292 #ifdef WDL_WIN32_NATIVE_WRITE
293 // todo, async close stuff?
294 if (m_fh != INVALID_HANDLE_VALUE && m_async)
296 SyncOutput(true);
299 m_empties.Empty(true);
300 m_pending.Empty(true);
302 if (m_fh != INVALID_HANDLE_VALUE) CloseHandle(m_fh);
303 m_fh=INVALID_HANDLE_VALUE;
304 #elif defined(WDL_POSIX_NATIVE_WRITE)
305 if (m_filedes >= 0)
307 if (m_bufspace.GetSize() > 0 && m_bufspace_used>0)
309 int v=(int)pwrite(m_filedes,m_bufspace.Get(),m_bufspace_used,m_file_position);
310 if (v>0) m_file_position+=v;
311 if (m_file_position > m_file_max_position) m_file_max_position=m_file_position;
312 m_bufspace_used=0;
314 if (m_filedes_locked) flock(m_filedes,LOCK_UN);
315 close(m_filedes);
317 m_filedes=-1;
319 #else
320 if (m_fp) fclose(m_fp);
321 m_fp=0;
322 #endif
326 bool IsOpen()
328 #ifdef WDL_WIN32_NATIVE_WRITE
329 return (m_fh != INVALID_HANDLE_VALUE);
330 #elif defined(WDL_POSIX_NATIVE_WRITE)
331 return m_filedes >= 0;
332 #else
333 return m_fp != NULL;
334 #endif
338 int Write(const void *buf, int len)
340 #ifdef WDL_WIN32_NATIVE_WRITE
341 if (m_fh == INVALID_HANDLE_VALUE) return 0;
343 if (m_async)
345 int rdpos = 0;
346 while (len > 0)
348 if (!m_empties.GetSize())
350 WDL_FileWrite__WriteEnt *ent=m_pending.Get(0);
351 DWORD s=0;
352 if (ent)
354 bool wasabort=false;
355 DWORD err;
356 if (GetOverlappedResult(m_fh,&ent->m_ol,&s,FALSE)||
357 (wasabort=((err=GetLastError())==ERROR_OPERATION_ABORTED)))
359 m_pending.Delete(0);
361 if (wasabort)
363 if (!RunAsyncWrite(ent,false)) m_empties.Add(ent);
365 else
367 m_empties.Add(ent);
368 ent->m_bufused=0;
371 else if (err != ERROR_IO_PENDING && err != ERROR_IO_INCOMPLETE)
373 WDL_FILEWRITE_ON_ERROR(err == ERROR_DISK_FULL)
379 WDL_FileWrite__WriteEnt *ent=m_empties.Get(0);
380 if (!ent)
382 if (m_pending.GetSize()>=m_async_maxbufs)
384 SyncOutput(false);
387 if (!(ent=m_empties.Get(0)))
388 m_empties.Add(ent = new WDL_FileWrite__WriteEnt(m_async_bufsize)); // new buffer
393 int ml=ent->m_bufsz-ent->m_bufused;
394 if (ml>len) ml=len;
395 memcpy(ent->m_bufptr+ent->m_bufused,(const char *)buf + rdpos,ml);
397 ent->m_bufused+=ml;
398 len-=ml;
399 rdpos+=ml;
401 if (ent->m_bufused >= ent->m_bufsz)
403 if (RunAsyncWrite(ent,true)) m_empties.Delete(0); // if queued remove from list
406 return rdpos;
408 else
410 DWORD dw=0;
411 if (!WriteFile(m_fh,buf,len,&dw,NULL))
413 WDL_FILEWRITE_ON_ERROR(GetLastError() == ERROR_DISK_FULL)
415 m_file_position+=dw;
416 if (m_file_position>m_file_max_position) m_file_max_position=m_file_position;
417 return dw;
419 #elif defined(WDL_POSIX_NATIVE_WRITE)
420 if (m_bufspace.GetSize()>0)
422 char *rdptr = (char *)buf;
423 int rdlen = len;
424 while (rdlen>0)
426 int amt = m_bufspace.GetSize() - m_bufspace_used;
427 if (amt>0)
429 if (amt>rdlen) amt=rdlen;
430 memcpy((char *)m_bufspace.Get()+m_bufspace_used,rdptr,amt);
431 m_bufspace_used += amt;
432 rdptr+=amt;
433 rdlen -= amt;
435 if (m_file_position+m_bufspace_used > m_file_max_position) m_file_max_position=m_file_position + m_bufspace_used;
437 if (m_bufspace_used >= m_bufspace.GetSize())
439 int v=(int)pwrite(m_filedes,m_bufspace.Get(),m_bufspace_used,m_file_position);
440 if (v != m_bufspace_used) { WDL_FILEWRITE_ON_ERROR(v>=0 || errno == EDQUOT || errno == ENOSPC) }
441 if (v>0) m_file_position+=v;
442 m_bufspace_used=0;
445 return len;
447 else
449 int v=(int)pwrite(m_filedes,buf,len,m_file_position);
450 if (v != len) { WDL_FILEWRITE_ON_ERROR(v>=0 || errno == EDQUOT || errno == ENOSPC) }
451 if (v>0) m_file_position+=v;
452 if (m_file_position > m_file_max_position) m_file_max_position=m_file_position;
453 return v;
455 #else
456 int written = (int)fwrite(buf,1,len,m_fp);
457 if (written != len) { WDL_FILEWRITE_ON_ERROR(false) }
458 return written;
459 #endif
464 WDL_FILEWRITE_POSTYPE GetSize()
466 #ifdef WDL_WIN32_NATIVE_WRITE
467 if (m_fh == INVALID_HANDLE_VALUE) return 0;
468 DWORD h=0;
469 DWORD l=GetFileSize(m_fh,&h);
470 WDL_FILEWRITE_POSTYPE tmp=(((WDL_FILEWRITE_POSTYPE)h)<<32)|l;
471 WDL_FILEWRITE_POSTYPE tmp2=GetPosition();
472 if (tmp<m_file_max_position) return m_file_max_position;
473 if (tmp<tmp2) return tmp2;
475 return tmp;
476 #elif defined(WDL_POSIX_NATIVE_WRITE)
477 if (m_filedes < 0) return -1;
478 return m_file_max_position;
479 #else
480 if (!m_fp) return -1;
481 int opos=ftell(m_fp);
482 fseek(m_fp,0,SEEK_END);
483 int a=ftell(m_fp);
484 fseek(m_fp,opos,SEEK_SET);
485 return a;
487 #endif
490 WDL_FILEWRITE_POSTYPE GetPosition()
492 #ifdef WDL_WIN32_NATIVE_WRITE
493 if (m_fh == INVALID_HANDLE_VALUE) return -1;
495 WDL_FILEWRITE_POSTYPE pos=m_file_position;
496 if (m_async)
498 WDL_FileWrite__WriteEnt *ent=m_empties.Get(0);
499 if (ent) pos+=ent->m_bufused;
501 return pos;
502 #elif defined(WDL_POSIX_NATIVE_WRITE)
503 if (m_filedes < 0) return -1;
504 return m_file_position + m_bufspace_used;
505 #else
506 if (!m_fp) return -1;
507 return ftell(m_fp);
509 #endif
512 #ifdef WDL_WIN32_NATIVE_WRITE
514 bool RunAsyncWrite(WDL_FileWrite__WriteEnt *ent, bool updatePosition) // returns true if ent is added to pending
516 if (ent && ent->m_bufused>0)
518 if (updatePosition)
520 ent->m_last_writepos = m_file_position;
521 m_file_position += ent->m_bufused;
522 if (m_file_position>m_file_max_position) m_file_max_position=m_file_position;
525 if (m_async == 3 && (ent->m_bufused&4095))
527 int offs=(ent->m_bufused&4095);
528 char tmp[4096];
529 memset(tmp,0,4096);
531 *(WDL_FILEWRITE_POSTYPE *)&ent->m_ol.Offset = ent->m_last_writepos + ent->m_bufused - offs;
532 ResetEvent(ent->m_ol.hEvent);
534 DWORD dw=0;
535 if (!ReadFile(m_fh,tmp,4096,&dw,&ent->m_ol))
537 if (GetLastError() == ERROR_IO_PENDING)
538 WaitForSingleObject(ent->m_ol.hEvent,INFINITE);
540 memcpy(ent->m_bufptr+ent->m_bufused,tmp+offs,4096-offs);
542 ent->m_bufused += 4096-offs;
545 DWORD d=0;
547 *(WDL_FILEWRITE_POSTYPE *)&ent->m_ol.Offset = ent->m_last_writepos;
549 ResetEvent(ent->m_ol.hEvent);
551 if (!WriteFile(m_fh,ent->m_bufptr,ent->m_bufused,&d,&ent->m_ol))
553 if (GetLastError()==ERROR_IO_PENDING)
555 m_pending.Add(ent);
556 return true;
558 else { WDL_FILEWRITE_ON_ERROR(GetLastError()==ERROR_DISK_FULL) }
560 ent->m_bufused=0;
562 return false;
565 void SyncOutput(bool syncall)
567 if (syncall)
569 if (RunAsyncWrite(m_empties.Get(0),true)) m_empties.Delete(0);
571 for (;;)
573 WDL_FileWrite__WriteEnt *ent=m_pending.Get(0);
574 if (!ent) break;
575 DWORD s=0;
576 m_pending.Delete(0);
577 BOOL ok = GetOverlappedResult(m_fh,&ent->m_ol,&s,TRUE);
578 int errcode;
579 if (!ok && (errcode=GetLastError())==ERROR_OPERATION_ABORTED)
581 // rewrite this one
582 if (!RunAsyncWrite(ent,false)) m_empties.Add(ent);
584 else
586 if (!ok) { WDL_FILEWRITE_ON_ERROR(errcode==ERROR_DISK_FULL) }
587 m_empties.Add(ent);
588 ent->m_bufused=0;
589 if (!syncall) break;
594 #endif
597 bool SetPosition(WDL_FILEWRITE_POSTYPE pos) // returns 0 on success
599 #ifdef WDL_WIN32_NATIVE_WRITE
600 if (m_fh == INVALID_HANDLE_VALUE) return true;
601 if (m_async)
603 SyncOutput(true);
604 m_file_position=pos;
605 if (m_file_position>m_file_max_position) m_file_max_position=m_file_position;
607 if (m_async==3 && (m_file_position&4095))
609 WDL_FileWrite__WriteEnt *ent=m_empties.Get(0);
610 if (ent)
612 int psz=(int) (m_file_position&4095);
614 m_file_position -= psz;
615 *(WDL_FILEWRITE_POSTYPE *)&ent->m_ol.Offset = m_file_position;
616 ResetEvent(ent->m_ol.hEvent);
618 DWORD dwo=0;
619 if (!ReadFile(m_fh,ent->m_bufptr,4096,&dwo,&ent->m_ol))
621 if (GetLastError() == ERROR_IO_PENDING)
622 WaitForSingleObject(ent->m_ol.hEvent,INFINITE);
624 ent->m_bufused=(int)psz;
627 return false;
630 m_file_position=pos;
631 if (m_file_position>m_file_max_position) m_file_max_position=m_file_position;
633 LONG high=(LONG) (m_file_position>>32);
634 return SetFilePointer(m_fh,(LONG)(m_file_position&((WDL_FILEWRITE_POSTYPE)0xFFFFFFFF)),&high,FILE_BEGIN)==0xFFFFFFFF && GetLastError() != NO_ERROR;
635 #elif defined(WDL_POSIX_NATIVE_WRITE)
637 if (m_filedes < 0) return true;
638 if (m_bufspace.GetSize() > 0 && m_bufspace_used>0)
640 int v=(int)pwrite(m_filedes,m_bufspace.Get(),m_bufspace_used,m_file_position);
641 if (v>0) m_file_position+=v;
642 if (m_file_position > m_file_max_position) m_file_max_position=m_file_position;
643 m_bufspace_used=0;
646 m_file_position = pos; // seek!
647 if (m_file_position>m_file_max_position) m_file_max_position=m_file_position;
648 return false;
649 #else
650 if (!m_fp) return true;
651 return !!fseek(m_fp,pos,SEEK_SET);
652 #endif
655 WDL_FILEWRITE_POSTYPE m_file_position, m_file_max_position;
657 #ifdef WDL_WIN32_NATIVE_WRITE
658 HANDLE GetHandle() { return m_fh; }
659 HANDLE m_fh;
660 int m_async; // 3 = unbuffered
662 int m_async_bufsize, m_async_minbufs, m_async_maxbufs;
664 WDL_PtrList<WDL_FileWrite__WriteEnt> m_empties;
665 WDL_PtrList<WDL_FileWrite__WriteEnt> m_pending;
667 #elif defined(WDL_POSIX_NATIVE_WRITE)
668 int GetHandle() { return m_filedes; }
670 WDL_HeapBuf m_bufspace;
671 int m_bufspace_used;
672 int m_filedes;
674 bool m_filedes_locked;
676 #else
677 int GetHandle() { return fileno(m_fp); }
679 FILE *m_fp;
680 #endif
682 #ifdef _WIN32
683 static void correctlongpath(WCHAR *buf) // this also exists as wdl_utf8_correctlongpath
685 const WCHAR *insert;
686 WCHAR *wr;
687 int skip = 0;
688 if (!buf || !buf[0] || wcslen(buf) < 256) return;
689 if (buf[1] == ':') insert=L"\\\\?\\";
690 else if (buf[0] == '\\' && buf[1] == '\\') { insert = L"\\\\?\\UNC\\"; skip=2; }
691 else return;
693 wr = buf + wcslen(insert);
694 memmove(wr, buf + skip, (wcslen(buf+skip)+1)*2);
695 memmove(buf,insert,wcslen(insert)*2);
696 while (*wr)
698 if (*wr == '/') *wr = '\\';
699 wr++;
702 #endif
703 } WDL_FIXALIGN;
710 #endif