WDL/filewrite.h

   1 /*
   2   WDL - filewrite.h
   3   Copyright (C) 2005 and later Cockos Incorporated
   4
   5   This software is provided 'as-is', without any express or implied
   6   warranty.  In no event will the authors be held liable for any damages
   7   arising from the use of this software.
   8
   9   Permission is granted to anyone to use this software for any purpose,
  10   including commercial applications, and to alter it and redistribute it
  11   freely, subject to the following restrictions:
  12
  13   1. The origin of this software must not be misrepresented; you must not
  14      claim that you wrote the original software. If you use this software
  15      in a product, an acknowledgment in the product documentation would be
  16      appreciated but is not required.
  17   2. Altered source versions must be plainly marked as such, and must not be
  18      misrepresented as being the original software.
  19   3. This notice may not be removed or altered from any source distribution.
  20
  21
  22   This file provides the WDL_FileWrite object, which can be used to create/write files.
  23   On windows systems it supports writing synchronously, asynchronously, and asynchronously without buffering.
  24   On windows systems it supports files larger than 4gb.
  25   On non-windows systems it acts as a wrapper for fopen()/etc.
  26
  27
  28 */
  29
  30
  31 #ifndef _WDL_FILEWRITE_H_
  32 #define _WDL_FILEWRITE_H_
  33
  34
  35
  36
  37 #include "ptrlist.h"
  38
  39
  40
  41 #if defined(_WIN32) && !defined(WDL_NO_WIN32_FILEWRITE)
  42   #ifndef WDL_WIN32_NATIVE_WRITE
  43     #define WDL_WIN32_NATIVE_WRITE
  44   #endif
  45 #else
  46   #ifdef WDL_WIN32_NATIVE_WRITE
  47     #undef WDL_WIN32_NATIVE_WRITE
  48   #endif
  49   #if !defined(WDL_NO_POSIX_FILEWRITE)
  50     #include <sys/fcntl.h>
  51     #include <sys/file.h>
  52     #include <sys/stat.h>
  53     #include <sys/errno.h>
  54     #define WDL_POSIX_NATIVE_WRITE
  55   #endif
  56 #endif
  57
  58
  59
  60 #ifdef _MSC_VER
  61 #define WDL_FILEWRITE_POSTYPE __int64
  62 #else
  63 #define WDL_FILEWRITE_POSTYPE long long
  64 #endif
  65
  66 //#define WIN32_ASYNC_NOBUF_WRITE // this doesnt seem to give much perf increase (writethrough with buffering is fine, since ultimately writes get deferred anyway)
  67
  68 class WDL_FileWrite
  69 {
  70 #ifdef WDL_WIN32_NATIVE_WRITE
  71
  72 class WDL_FileWrite__WriteEnt
  73 {
  74 public:
  75   WDL_FileWrite__WriteEnt(int sz)
  76   {
  77     m_last_writepos=0;
  78     m_bufused=0;
  79     m_bufsz=sz;
  80     m_bufptr = (char *)__buf.Resize(sz+4095);
  81     int a=((int)(INT_PTR)m_bufptr)&4095;
  82     if (a) m_bufptr += 4096-a;
  83
  84     memset(&m_ol,0,sizeof(m_ol));
  85     m_ol.hEvent=CreateEvent(NULL,TRUE,TRUE,NULL);
  86   }
  87   ~WDL_FileWrite__WriteEnt()
  88   {
  89     CloseHandle(m_ol.hEvent);
  90   }
  91
  92   WDL_FILEWRITE_POSTYPE m_last_writepos;
  93
  94   int m_bufused,m_bufsz;
  95   OVERLAPPED m_ol;
  96   char *m_bufptr;
  97   WDL_TypedBuf<char> __buf;
  98 };
  99
 100 #endif
 101
 102 #if defined(_WIN32) && !defined(WDL_NO_SUPPORT_UTF8)
 103   BOOL HasUTF8(const char *_str)
 104   {
 105     const unsigned char *str = (const unsigned char *)_str;
 106     if (!str) return FALSE;
 107     while (*str)
 108     {
 109       unsigned char c = *str;
 110       if (c >= 0xC2)
 111       {
 112         if (c <= 0xDF && str[1] >=0x80 && str[1] <= 0xBF) return TRUE;
 113         else if (c <= 0xEF && str[1] >=0x80 && str[1] <= 0xBF && str[2] >=0x80 && str[2] <= 0xBF) return TRUE;
 114         else if (c <= 0xF4 && str[1] >=0x80 && str[1] <= 0xBF && str[2] >=0x80 && str[2] <= 0xBF) return TRUE;
 115       }
 116       str++;
 117     }
 118     return FALSE;
 119   }
 120 #endif
 121
 122
 123 public:
 124   WDL_FileWrite(const char *filename, int allow_async=1, int bufsize=8192, int minbufs=16, int maxbufs=16, bool wantAppendTo=false, bool noFileLocking=false) // async==2 is unbuffered
 125   {
 126     m_file_position=0;
 127     m_file_max_position=0;
 128     if(!filename)
 129     {
 130 #ifdef WDL_WIN32_NATIVE_WRITE
 131       m_fh = INVALID_HANDLE_VALUE;
 132       m_async = 0;
 133 #elif defined(WDL_POSIX_NATIVE_WRITE)
 134       m_filedes_locked=false;
 135       m_filedes=-1;
 136       m_bufspace_used=0;
 137 #else
 138       m_fp = NULL;
 139 #endif
 140       return;
 141     }
 142
 143 #ifdef WDL_WIN32_NATIVE_WRITE
 144     #ifdef WDL_SUPPORT_WIN9X
 145     const bool isNT = (GetVersion()<0x80000000);
 146     #else
 147     const bool isNT = true;
 148     #endif
 149     m_async = allow_async && isNT;
 150 #ifdef WIN32_ASYNC_NOBUF_WRITE
 151     bufsize = (bufsize+4095)&~4095;
 152     if (bufsize<4096) bufsize=4096;
 153 #endif
 154
 155     int rwflag = GENERIC_WRITE;
 156     int createFlag= wantAppendTo?OPEN_ALWAYS:CREATE_ALWAYS;
 157     int shareFlag = noFileLocking ? (FILE_SHARE_READ|FILE_SHARE_WRITE) : FILE_SHARE_READ;
 158     int flag = FILE_ATTRIBUTE_NORMAL;
 159
 160     if (m_async)
 161     {
 162       rwflag |= GENERIC_READ;
 163 #ifdef WIN32_ASYNC_NOBUF_WRITE
 164       flag |= FILE_FLAG_OVERLAPPED|FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH;
 165 #else
 166       flag |= FILE_FLAG_OVERLAPPED|(allow_async>1 ? FILE_FLAG_WRITE_THROUGH: 0);
 167 #endif
 168     }
 169
 170     {
 171 #ifndef WDL_NO_SUPPORT_UTF8
 172       m_fh=INVALID_HANDLE_VALUE;
 173       if (isNT && HasUTF8(filename))
 174       {
 175         int szreq=MultiByteToWideChar(CP_UTF8,MB_ERR_INVALID_CHARS,filename,-1,NULL,0);
 176         if (szreq > 1000)
 177         {
 178           WDL_TypedBuf<WCHAR> wfilename;
 179           wfilename.Resize(szreq+10);
 180           if (MultiByteToWideChar(CP_UTF8,MB_ERR_INVALID_CHARS,filename,-1,wfilename.Get(),wfilename.GetSize()))
 181             m_fh = CreateFileW(wfilename.Get(),rwflag,shareFlag,NULL,createFlag,flag,NULL);
 182         }
 183         else
 184         {
 185           WCHAR wfilename[1024];
 186           if (MultiByteToWideChar(CP_UTF8,MB_ERR_INVALID_CHARS,filename,-1,wfilename,1024))
 187             m_fh = CreateFileW(wfilename,rwflag,shareFlag,NULL,createFlag,flag,NULL);
 188         }
 189       }
 190
 191       if (m_fh == INVALID_HANDLE_VALUE)
 192 #endif
 193         m_fh = CreateFileA(filename,rwflag,shareFlag,NULL,createFlag,flag,NULL);
 194     }
 195
 196     if (m_async && m_fh != INVALID_HANDLE_VALUE)
 197     {
 198       m_async_bufsize=bufsize;
 199       m_async_maxbufs=maxbufs;
 200       m_async_minbufs=minbufs;
 201       int x;
 202       for (x = 0; x < m_async_minbufs; x ++)
 203       {
 204         WDL_FileWrite__WriteEnt *t=new WDL_FileWrite__WriteEnt(m_async_bufsize);
 205         m_empties.Add(t);
 206       }
 207     }
 208
 209     if (m_fh != INVALID_HANDLE_VALUE && wantAppendTo)
 210       SetPosition(GetSize());
 211
 212 #elif defined(WDL_POSIX_NATIVE_WRITE)
 213     m_bufspace_used=0;
 214     m_filedes_locked=false;
 215     m_filedes=open(filename,O_WRONLY|O_CREAT,0644);
 216     if (m_filedes>=0)
 217     {
 218
 219       if (!noFileLocking)
 220       {
 221         m_filedes_locked = !flock(m_filedes,LOCK_EX|LOCK_NB);
 222         if (!m_filedes_locked)
 223         {
 224           // this check might not be necessary, it might be sufficient to just fail and close if no exclusive lock possible
 225           if (errno == EWOULDBLOCK)
 226           {
 227             // FAILED exclusive locking because someone else has a lock
 228             close(m_filedes);
 229             m_filedes=-1;
 230           }
 231           else  // failed for some other reason, try to keep a shared lock at least
 232           {
 233             m_filedes_locked = !flock(m_filedes,LOCK_SH|LOCK_NB);
 234           }
 235         }
 236       }
 237
 238       if (m_filedes>=0)
 239       {
 240         if (!wantAppendTo) ftruncate(m_filedes,0);
 241         else
 242         {
 243           struct stat64 st;
 244           if (!fstat64(m_filedes,&st))  SetPosition(st.st_size);
 245         }
 246       }
 247
 248
 249 #ifdef __APPLE__
 250       if (m_filedes >= 0 && allow_async>1) fcntl(m_filedes,F_NOCACHE,1);
 251 #endif
 252     }
 253     if (minbufs * bufsize >= 16384) m_bufspace.Resize((minbufs*bufsize+4095)&~4095);
 254 #else
 255     m_fp=fopen(filename,wantAppendTo ? "a+b" : "wb");
 256     if (wantAppendTo && m_fp)
 257       fseek(m_fp,0,SEEK_END);
 258 #endif
 259   }
 260
 261   ~WDL_FileWrite()
 262   {
 263 #ifdef WDL_WIN32_NATIVE_WRITE
 264     // todo, async close stuff?
 265     if (m_fh != INVALID_HANDLE_VALUE && m_async)
 266     {
 267       SyncOutput(true);
 268     }
 269
 270     m_empties.Empty(true);
 271     m_pending.Empty(true);
 272
 273     if (m_fh != INVALID_HANDLE_VALUE) CloseHandle(m_fh);
 274     m_fh=INVALID_HANDLE_VALUE;
 275 #elif defined(WDL_POSIX_NATIVE_WRITE)
 276    if (m_filedes >= 0)
 277    {
 278      if (m_bufspace.GetSize() > 0 && m_bufspace_used>0)
 279      {
 280        int v=(int)pwrite(m_filedes,m_bufspace.Get(),m_bufspace_used,m_file_position);
 281        if (v>0) m_file_position+=v;
 282        if (m_file_position > m_file_max_position) m_file_max_position=m_file_position;
 283        m_bufspace_used=0;
 284      }
 285      if (m_filedes_locked) flock(m_filedes,LOCK_UN);
 286      close(m_filedes);
 287    }
 288    m_filedes=-1;
 289
 290 #else
 291     if (m_fp) fclose(m_fp);
 292     m_fp=0;
 293 #endif
 294
 295   }
 296
 297   bool IsOpen()
 298   {
 299 #ifdef WDL_WIN32_NATIVE_WRITE
 300     return (m_fh != INVALID_HANDLE_VALUE);
 301 #elif defined(WDL_POSIX_NATIVE_WRITE)
 302     return m_filedes >= 0;
 303 #else
 304     return m_fp != NULL;
 305 #endif
 306   }
 307
 308
 309   int Write(const void *buf, int len)
 310   {
 311 #ifdef WDL_WIN32_NATIVE_WRITE
 312     if (m_fh == INVALID_HANDLE_VALUE) return 0;
 313
 314     if (m_async)
 315     {
 316       int rdpos = 0;
 317       while (len > 0)
 318       {
 319         if (!m_empties.GetSize())
 320         {
 321           WDL_FileWrite__WriteEnt *ent=m_pending.Get(0);
 322           DWORD s=0;
 323           if (ent)
 324           {
 325             bool wasabort=false;
 326             if (GetOverlappedResult(m_fh,&ent->m_ol,&s,FALSE)||
 327                 (wasabort=(GetLastError()==ERROR_OPERATION_ABORTED)))
 328             {
 329               m_pending.Delete(0);
 330
 331               if (wasabort)
 332               {
 333                 if (!RunAsyncWrite(ent,false)) m_empties.Add(ent);
 334               }
 335               else
 336               {
 337                 m_empties.Add(ent);
 338                 ent->m_bufused=0;
 339               }
 340             }
 341           }
 342         }
 343
 344
 345         WDL_FileWrite__WriteEnt *ent=m_empties.Get(0);
 346         if (!ent)
 347         {
 348           if (m_pending.GetSize()>=m_async_maxbufs)
 349           {
 350             SyncOutput(false);
 351           }
 352
 353           if (!(ent=m_empties.Get(0)))
 354             m_empties.Add(ent = new WDL_FileWrite__WriteEnt(m_async_bufsize)); // new buffer
 355
 356
 357         }
 358
 359         int ml=ent->m_bufsz-ent->m_bufused;
 360         if (ml>len) ml=len;
 361         memcpy(ent->m_bufptr+ent->m_bufused,(const char *)buf + rdpos,ml);
 362
 363         ent->m_bufused+=ml;
 364         len-=ml;
 365         rdpos+=ml;
 366
 367         if (ent->m_bufused >= ent->m_bufsz)
 368         {
 369           if (RunAsyncWrite(ent,true)) m_empties.Delete(0); // if queued remove from list
 370         }
 371       }
 372       return rdpos;
 373     }
 374     else
 375     {
 376       DWORD dw=0;
 377       WriteFile(m_fh,buf,len,&dw,NULL);
 378       m_file_position+=dw;
 379       if (m_file_position>m_file_max_position) m_file_max_position=m_file_position;
 380       return dw;
 381     }
 382 #elif defined(WDL_POSIX_NATIVE_WRITE)
 383    if (m_bufspace.GetSize()>0)
 384    {
 385      char *rdptr = (char *)buf;
 386      int rdlen = len;
 387      while (rdlen>0)
 388      {
 389        int amt = m_bufspace.GetSize() - m_bufspace_used;
 390        if (amt>0)
 391        {
 392          if (amt>rdlen) amt=rdlen;
 393          memcpy((char *)m_bufspace.Get()+m_bufspace_used,rdptr,amt);
 394          m_bufspace_used += amt;
 395          rdptr+=amt;
 396          rdlen -= amt;
 397
 398          if (m_file_position+m_bufspace_used > m_file_max_position) m_file_max_position=m_file_position + m_bufspace_used;
 399        }
 400        if (m_bufspace_used >= m_bufspace.GetSize())
 401        {
 402          int v=(int)pwrite(m_filedes,m_bufspace.Get(),m_bufspace_used,m_file_position);
 403          if (v>0) m_file_position+=v;
 404          m_bufspace_used=0;
 405        }
 406      }
 407      return len;
 408    }
 409    else
 410    {
 411      int v=(int)pwrite(m_filedes,buf,len,m_file_position);
 412      if (v>0) m_file_position+=v;
 413      if (m_file_position > m_file_max_position) m_file_max_position=m_file_position;
 414      return v;
 415    }
 416 #else
 417     return fwrite(buf,1,len,m_fp);
 418 #endif
 419
 420
 421   }
 422
 423   WDL_FILEWRITE_POSTYPE GetSize()
 424   {
 425 #ifdef WDL_WIN32_NATIVE_WRITE
 426     if (m_fh == INVALID_HANDLE_VALUE) return 0;
 427     DWORD h=0;
 428     DWORD l=GetFileSize(m_fh,&h);
 429     WDL_FILEWRITE_POSTYPE tmp=(((WDL_FILEWRITE_POSTYPE)h)<<32)|l;
 430     WDL_FILEWRITE_POSTYPE tmp2=GetPosition();
 431     if (tmp<m_file_max_position) return m_file_max_position;
 432     if (tmp<tmp2) return tmp2;
 433
 434     return tmp;
 435 #elif defined(WDL_POSIX_NATIVE_WRITE)
 436     if (m_filedes < 0) return -1;
 437     return m_file_max_position;
 438 #else
 439     if (!m_fp) return -1;
 440     int opos=ftell(m_fp);
 441     fseek(m_fp,0,SEEK_END);
 442     int a=ftell(m_fp);
 443     fseek(m_fp,opos,SEEK_SET);
 444     return a;
 445
 446 #endif
 447   }
 448
 449   WDL_FILEWRITE_POSTYPE GetPosition()
 450   {
 451 #ifdef WDL_WIN32_NATIVE_WRITE
 452     if (m_fh == INVALID_HANDLE_VALUE) return -1;
 453
 454     WDL_FILEWRITE_POSTYPE pos=m_file_position;
 455     if (m_async)
 456     {
 457       WDL_FileWrite__WriteEnt *ent=m_empties.Get(0);
 458       if (ent) pos+=ent->m_bufused;
 459     }
 460     return pos;
 461 #elif defined(WDL_POSIX_NATIVE_WRITE)
 462     if (m_filedes < 0) return -1;
 463     return m_file_position + m_bufspace_used;
 464 #else
 465     if (!m_fp) return -1;
 466     return ftell(m_fp);
 467
 468 #endif
 469   }
 470
 471 #ifdef WDL_WIN32_NATIVE_WRITE
 472
 473   bool RunAsyncWrite(WDL_FileWrite__WriteEnt *ent, bool updatePosition) // returns true if ent is added to pending
 474   {
 475     if (ent && ent->m_bufused>0)
 476     {
 477       if (updatePosition)
 478       {
 479         ent->m_last_writepos = m_file_position;
 480         m_file_position += ent->m_bufused;
 481         if (m_file_position>m_file_max_position) m_file_max_position=m_file_position;
 482       }
 483
 484 #ifdef WIN32_ASYNC_NOBUF_WRITE
 485       if (ent->m_bufused&4095)
 486       {
 487         int offs=(ent->m_bufused&4095);
 488         char tmp[4096];
 489         memset(tmp,0,4096);
 490
 491         *(WDL_FILEWRITE_POSTYPE *)&ent->m_ol.Offset = ent->m_last_writepos + ent->m_bufused - offs;
 492         ResetEvent(ent->m_ol.hEvent);
 493
 494         DWORD dw=0;
 495         if (!ReadFile(m_fh,tmp,4096,&dw,&ent->m_ol))
 496         {
 497           if (GetLastError() == ERROR_IO_PENDING)
 498             WaitForSingleObject(ent->m_ol.hEvent,INFINITE);
 499         }
 500         memcpy(ent->m_bufptr+ent->m_bufused,tmp+offs,4096-offs);
 501
 502         ent->m_bufused += 4096-offs;
 503       }
 504 #endif
 505       DWORD d=0;
 506
 507       *(WDL_FILEWRITE_POSTYPE *)&ent->m_ol.Offset = ent->m_last_writepos;
 508
 509       ResetEvent(ent->m_ol.hEvent);
 510
 511       if (!WriteFile(m_fh,ent->m_bufptr,ent->m_bufused,&d,&ent->m_ol))
 512       {
 513         if (GetLastError()==ERROR_IO_PENDING)
 514         {
 515           m_pending.Add(ent);
 516           return true;
 517         }
 518       }
 519       ent->m_bufused=0;
 520     }
 521     return false;
 522   }
 523
 524   void SyncOutput(bool syncall)
 525   {
 526     if (syncall)
 527     {
 528       if (RunAsyncWrite(m_empties.Get(0),true)) m_empties.Delete(0);
 529     }
 530     for (;;)
 531     {
 532       WDL_FileWrite__WriteEnt *ent=m_pending.Get(0);
 533       if (!ent) break;
 534       DWORD s=0;
 535       m_pending.Delete(0);
 536       if (!GetOverlappedResult(m_fh,&ent->m_ol,&s,TRUE) && GetLastError()==ERROR_OPERATION_ABORTED)
 537       {
 538         // rewrite this one
 539         if (!RunAsyncWrite(ent,false)) m_empties.Add(ent);
 540       }
 541       else
 542       {
 543         m_empties.Add(ent);
 544         ent->m_bufused=0;
 545         if (!syncall) break;
 546       }
 547     }
 548   }
 549
 550 #endif
 551
 552
 553   bool SetPosition(WDL_FILEWRITE_POSTYPE pos) // returns 0 on success
 554   {
 555 #ifdef WDL_WIN32_NATIVE_WRITE
 556     if (m_fh == INVALID_HANDLE_VALUE) return true;
 557     if (m_async)
 558     {
 559       SyncOutput(true);
 560       m_file_position=pos;
 561       if (m_file_position>m_file_max_position) m_file_max_position=m_file_position;
 562
 563 #ifdef WIN32_ASYNC_NOBUF_WRITE
 564       if (m_file_position&4095)
 565       {
 566         WDL_FileWrite__WriteEnt *ent=m_empties.Get(0);
 567         if (ent)
 568         {
 569           int psz=(int) (m_file_position&4095);
 570
 571           m_file_position -= psz;
 572           *(WDL_FILEWRITE_POSTYPE *)&ent->m_ol.Offset = m_file_position;
 573           ResetEvent(ent->m_ol.hEvent);
 574
 575           DWORD dwo=0;
 576           if (!ReadFile(m_fh,ent->m_bufptr,4096,&dwo,&ent->m_ol))
 577           {
 578             if (GetLastError() == ERROR_IO_PENDING)
 579               WaitForSingleObject(ent->m_ol.hEvent,INFINITE);
 580           }
 581           ent->m_bufused=(int)psz;
 582         }
 583       }
 584 #endif
 585       return false;
 586     }
 587
 588     m_file_position=pos;
 589     if (m_file_position>m_file_max_position) m_file_max_position=m_file_position;
 590
 591     LONG high=(LONG) (m_file_position>>32);
 592     return SetFilePointer(m_fh,(LONG)(m_file_position&((WDL_FILEWRITE_POSTYPE)0xFFFFFFFF)),&high,FILE_BEGIN)==0xFFFFFFFF && GetLastError() != NO_ERROR;
 593 #elif defined(WDL_POSIX_NATIVE_WRITE)
 594
 595     if (m_filedes < 0) return true;
 596     if (m_bufspace.GetSize() > 0 && m_bufspace_used>0)
 597     {
 598       int v=(int)pwrite(m_filedes,m_bufspace.Get(),m_bufspace_used,m_file_position);
 599       if (v>0) m_file_position+=v;
 600       if (m_file_position > m_file_max_position) m_file_max_position=m_file_position;
 601       m_bufspace_used=0;
 602     }
 603
 604     m_file_position = pos; // seek!
 605     if (m_file_position>m_file_max_position) m_file_max_position=m_file_position;
 606     return false;
 607 #else
 608     if (!m_fp) return true;
 609     return !!fseek(m_fp,pos,SEEK_SET);
 610 #endif
 611   }
 612
 613   WDL_FILEWRITE_POSTYPE m_file_position, m_file_max_position;
 614
 615 #ifdef WDL_WIN32_NATIVE_WRITE
 616   HANDLE GetHandle() { return m_fh; }
 617   HANDLE m_fh;
 618   bool m_async;
 619
 620   int m_async_bufsize, m_async_minbufs, m_async_maxbufs;
 621
 622   WDL_PtrList<WDL_FileWrite__WriteEnt> m_empties;
 623   WDL_PtrList<WDL_FileWrite__WriteEnt> m_pending;
 624
 625 #elif defined(WDL_POSIX_NATIVE_WRITE)
 626   int GetHandle() { return m_filedes; }
 627
 628   WDL_HeapBuf m_bufspace;
 629   int m_bufspace_used;
 630   int m_filedes;
 631
 632   bool m_filedes_locked;
 633
 634 #else
 635   int GetHandle() { return fileno(m_fp); }
 636
 637   FILE *m_fp;
 638 #endif
 639 } WDL_FIXALIGN;
 640
 641
 642
 643
 644
 645
 646 #endif