WDL/projectcontext.cpp

   1 #ifdef _WIN32
   2 #include <windows.h>
   3 #else
   4 #include <unistd.h>
   5 #include <string.h>
   6 #include <sys/stat.h>
   7 #endif
   8 #include <stdlib.h>
   9 #include <stdio.h>
  10 #include <stdarg.h>
  11
  12 #include "projectcontext.h"
  13
  14 #include "fileread.h"
  15 #include "filewrite.h"
  16 #include "heapbuf.h"
  17 #include "wdlstring.h"
  18 #include "wdlcstring.h"
  19 #include "fastqueue.h"
  20 #include "lineparse.h"
  21
  22
  23 //#define WDL_MEMPROJECTCONTEXT_USE_ZLIB 1
  24
  25 #ifdef WDL_MEMPROJECTCONTEXT_USE_ZLIB
  26
  27 #define WDL_MEMPROJECTCONTEXT_ZLIB_CHUNKSIZE 65536
  28 #include "zlib/zlib.h"
  29
  30 #endif
  31
  32 #include "denormal.h"
  33
  34
  35 char *projectcontext_fastDoubleToString(double value, char *bufOut, int prec_digits)
  36 {
  37   value = denormal_filter_double2(value);
  38
  39   if (value<0.0)
  40   {
  41     value=-value;
  42     *bufOut++ = '-';
  43   }
  44
  45   if (value < 1e-20)
  46   {
  47     *bufOut++ = '0';
  48     *bufOut = 0;
  49     return bufOut;
  50   }
  51
  52   if (value > 2147483647.0)
  53   {
  54     if (value >= 1.0e40) sprintf(bufOut, "%e", value);
  55     else sprintf(bufOut, "%.*f", wdl_min(prec_digits,8), value);
  56     while (*bufOut) bufOut++;
  57     return bufOut;
  58   }
  59
  60   unsigned int value_i, frac, frac2;
  61   int prec_digits2 = 0;
  62
  63   if (prec_digits>0)
  64   {
  65     static const unsigned int scales[9] =
  66     {
  67       10,
  68       100,
  69       1000,
  70       10000,
  71       100000,
  72       1000000,
  73       10000000,
  74       100000000,
  75       1000000000
  76     };
  77
  78     value_i = (unsigned int)value;
  79     const int value_digits =
  80        value_i >= 10000 ? (
  81          value_i >= 1000000 ?
  82            (value_i >= 100000000 ?
  83             (value_i >= 1000000000 ? 10 : 9) :
  84              (value_i >= 10000000 ? 8 : 7)) :
  85             (value_i >= 100000 ? 6 : 5)
  86        )
  87        :
  88        (
  89          value_i >= 100 ?
  90            (value_i >= 1000 ? 4 : 3) :
  91            (value_i >= 10 ? 2 : 1)
  92        );
  93
  94            // double precision is limited to about 17 decimal digits of meaningful values
  95     if (prec_digits + value_digits > 17) prec_digits = 17-value_digits;
  96
  97     if (prec_digits > 9)
  98     {
  99       prec_digits2 = prec_digits - 9;
 100       prec_digits = 9;
 101       if (prec_digits2 > 9) prec_digits2 = 9;
 102     }
 103
 104     const unsigned int prec_scale = scales[prec_digits-1];
 105     const double dfrac = (value - value_i) * prec_scale;
 106     if (prec_digits2 > 0)
 107     {
 108       const unsigned int prec_scale2 = scales[prec_digits2-1];
 109       frac = (unsigned int) dfrac;
 110
 111       double dfrac2 = (dfrac - frac) * prec_scale2;
 112       frac2 = (unsigned int) (dfrac2 + 0.5);
 113
 114       const int prec_scale2_small = wdl_min(prec_scale2/1024,10);
 115
 116       if (frac2 <= prec_scale2_small) frac2=0;
 117       else if (frac2 >= prec_scale2 - prec_scale2_small - 1) frac2=prec_scale2;
 118
 119       if (frac2 >= prec_scale2)
 120       {
 121         frac2 -= prec_scale2;
 122         frac++;
 123       }
 124     }
 125     else
 126     {
 127       frac = (unsigned int) (dfrac + 0.5);
 128       frac2 = 0;
 129       const int prec_scale_small = wdl_min(prec_scale/1024,10);
 130       if (frac <= prec_scale_small) frac=0;
 131       else if (frac>=prec_scale-prec_scale_small - 1) frac=prec_scale;
 132     }
 133
 134     if (frac >= prec_scale) // round up to next integer
 135     {
 136       frac -= prec_scale;
 137       value_i++;
 138     }
 139   }
 140   else // round to int
 141   {
 142     value_i = (unsigned int)(value+0.5);
 143     frac2 = frac = 0;
 144   }
 145
 146   char digs[32];
 147
 148   if (value_i)
 149   {
 150     int tmp=value_i;
 151     int x = 0;
 152     do {
 153       const int a = (tmp%10);
 154       tmp/=10;
 155       digs[x++]='0' + a;
 156     } while (tmp);
 157
 158     while (x>0) *bufOut++ = digs[--x];
 159   }
 160   else
 161   {
 162     *bufOut++ = '0';
 163   }
 164
 165
 166   if (frac || frac2)
 167   {
 168     int x = 0;
 169     int tmp=frac;
 170     int dleft = prec_digits;
 171     *bufOut++='.';
 172
 173     if (frac) do
 174     {
 175       const int a = (tmp%10);
 176       tmp /= 10;
 177       if (x || a || frac2) digs[x++] = '0'+a;
 178     } while (dleft-- > 0 && tmp);
 179
 180     while (dleft-->0) *bufOut++ = '0';
 181     while (x>0) *bufOut++ = digs[--x];
 182     // x is 0
 183
 184     if (frac2)
 185     {
 186       tmp=frac2;
 187       dleft = prec_digits2;
 188       do
 189       {
 190         const int a = (tmp%10);
 191         tmp /= 10;
 192         if (x || a) digs[x++] = '0'+a;
 193       } while (dleft-- > 0 && tmp);
 194
 195       while (dleft-->0) *bufOut++ = '0';
 196       while (x>0) *bufOut++ = digs[--x];
 197     }
 198   }
 199
 200   *bufOut = 0;
 201   return bufOut;
 202 }
 203
 204 int ProjectContextFormatString(char *outbuf, size_t outbuf_size, const char *fmt, va_list va)
 205 {
 206   int wroffs=0;
 207
 208   while (*fmt && outbuf_size > 1)
 209   {
 210     char c = *fmt++;
 211     if (c != '%')
 212     {
 213       outbuf[wroffs++] = c != '\n' ? c : ' ';
 214       outbuf_size--;
 215       continue;
 216     }
 217
 218     if (*fmt == '%')
 219     {
 220       outbuf[wroffs++] = '%';
 221       outbuf_size--;
 222       fmt++;
 223       continue;
 224     }
 225
 226
 227     const char *ofmt = fmt-1;
 228     bool want_abort=false;
 229
 230     int has_prec=0;
 231     int prec=0;
 232     if (*fmt == '.')
 233     {
 234       has_prec=1;
 235       fmt++;
 236       while (*fmt >= '0' && *fmt <= '9') prec = prec*10 + (*fmt++-'0');
 237       if (*fmt != 'f' || prec < 0 || prec>20)
 238       {
 239         want_abort=true;
 240       }
 241     }
 242     else if (*fmt == '0')
 243     {
 244       has_prec=2;
 245       fmt++;
 246       while (*fmt >= '0' && *fmt <= '9') prec = prec*10 + (*fmt++-'0');
 247       if (*fmt != 'x' && *fmt != 'X' && *fmt != 'd' && *fmt != 'u')
 248       {
 249         want_abort=true;
 250       }
 251     }
 252
 253     c = *fmt++;
 254     if (!want_abort) switch (c)
 255     {
 256       case '@':
 257       case 'p':
 258       case 's':
 259       {
 260         const char *str=va_arg(va,const char *);
 261         bool prefer_quoteless = true;
 262         if (c == 'p' && *fmt == '~') // %p~ to bias towards "string" (legacy)
 263         {
 264           prefer_quoteless = false;
 265           fmt++;
 266         }
 267         const char qc = outbuf_size >= 3 && c != 's' ? getConfigStringQuoteChar(str, prefer_quoteless) : ' ';
 268
 269         if (qc != ' ')
 270         {
 271           outbuf[wroffs++] = qc ? qc : '`';
 272           outbuf_size-=2; // will add trailing quote below
 273         }
 274
 275         if (str) while (outbuf_size > 1 && *str)
 276         {
 277           char v = *str++;
 278           if (!qc && v == '`') v = '\'';
 279           outbuf[wroffs++] = v != '\n' && v != '\r' ? v : ' ';
 280           outbuf_size--;
 281         }
 282
 283         if (qc != ' ')
 284         {
 285           outbuf[wroffs++] = qc ? qc : '`';
 286           // outbuf_size already decreased above
 287         }
 288       }
 289       break;
 290       case 'c':
 291       {
 292         int v = (va_arg(va,int)) & 0xff;
 293         outbuf[wroffs++] = v != '\n' ? v : ' ';
 294         outbuf_size--;
 295       }
 296       break;
 297       case 'd':
 298       {
 299         int v = va_arg(va,int);
 300         if (v<0)
 301         {
 302           outbuf[wroffs++] = '-';
 303           outbuf_size--;
 304           v=-v; // this won't handle -2147483648 right, todo special case?
 305         }
 306
 307         char tab[32];
 308         int x=0;
 309         do
 310         {
 311           tab[x++] = v%10;
 312           v/=10;
 313         }
 314         while (v);
 315         if (has_prec == 2) while (x<prec) { tab[x++] = 0; }
 316
 317         while (--x >= 0 && outbuf_size>1)
 318         {
 319           outbuf[wroffs++] = '0' + tab[x];
 320           outbuf_size--;
 321         }
 322       }
 323       break;
 324       case 'u':
 325       {
 326         unsigned int v = va_arg(va,unsigned int);
 327
 328         char tab[32];
 329         int x=0;
 330         do
 331         {
 332           tab[x++] = v%10;
 333           v/=10;
 334         }
 335         while (v);
 336         if (has_prec == 2) while (x<prec) { tab[x++] = 0; }
 337
 338         while (--x >= 0 && outbuf_size>1)
 339         {
 340           outbuf[wroffs++] = '0' + tab[x];
 341           outbuf_size--;
 342         }
 343       }
 344       break;
 345       case 'x':
 346       case 'X':
 347       {
 348         const char base = (c - 'x') + 'a';
 349         unsigned int v = va_arg(va,unsigned int);
 350
 351         char tab[32];
 352         int x=0;
 353         do
 354         {
 355           tab[x++] = v&0xf;
 356           v>>=4;
 357         }
 358         while (v);
 359
 360         if (has_prec == 2) while (x<prec) { tab[x++] = 0; }
 361
 362         while (--x >= 0 && outbuf_size>1)
 363         {
 364           if (tab[x] < 10)
 365             outbuf[wroffs++] = '0' + tab[x];
 366           else
 367             outbuf[wroffs++] = base + tab[x] - 10;
 368
 369           outbuf_size--;
 370         }
 371       }
 372       break;
 373       case 'f':
 374       {
 375         double v = va_arg(va,double);
 376         if (outbuf_size<64)
 377         {
 378           char tmp[64];
 379           projectcontext_fastDoubleToString(v,tmp,has_prec?prec:6);
 380           const char *str = tmp;
 381           while (outbuf_size > 1 && *str)
 382           {
 383             outbuf[wroffs++] = *str++;
 384             outbuf_size--;
 385           }
 386         }
 387         else
 388         {
 389           const char *p=projectcontext_fastDoubleToString(v,outbuf+wroffs,has_prec?prec:6);
 390           int amt = (int) (p-(outbuf+wroffs));
 391           wroffs += amt;
 392           outbuf_size-=amt;
 393         }
 394       }
 395       break;
 396       default:
 397         want_abort=true;
 398       break;
 399     }
 400     if (want_abort)
 401     {
 402       fmt=ofmt;
 403       break;
 404     }
 405   }
 406
 407   outbuf += wroffs;
 408   outbuf[0] = 0;
 409   if (outbuf_size<2||!*fmt)
 410     return wroffs;
 411
 412 #if defined(_WIN32) && defined(_MSC_VER)
 413    // _vsnprintf() does not always null terminate (see below)
 414   _vsnprintf(outbuf,outbuf_size,fmt,va);
 415 #else
 416   // vsnprintf() on non-win32, always null terminates
 417   vsnprintf(outbuf,outbuf_size,fmt,va);
 418 #endif
 419
 420   int l;
 421   outbuf_size--;
 422   for (l = 0; l < outbuf_size && outbuf[l]; l ++) if (outbuf[l] == '\n') outbuf[l] = ' ';
 423
 424 #if defined(_WIN32) && defined(_MSC_VER)
 425    // nul terminate for _vsnprintf()
 426   outbuf[l]=0;
 427 #endif
 428
 429   return wroffs+l;
 430 }
 431
 432
 433
 434 class ProjectStateContext_Mem : public ProjectStateContext
 435 {
 436 public:
 437
 438   ProjectStateContext_Mem(WDL_HeapBuf *hb, int rwflags)
 439   {
 440     m_rwflags=rwflags;
 441     m_heapbuf=hb;
 442     m_pos=0;
 443     m_tmpflag=0;
 444 #ifdef WDL_MEMPROJECTCONTEXT_USE_ZLIB
 445     memset(&m_compstream,0,sizeof(m_compstream));
 446     m_usecomp=0;
 447 #endif
 448   }
 449
 450   virtual ~ProjectStateContext_Mem()
 451   {
 452     #ifdef WDL_MEMPROJECTCONTEXT_USE_ZLIB
 453       if (m_usecomp==1)
 454       {
 455         FlushComp(true);
 456         deflateEnd(&m_compstream);
 457       }
 458       else if (m_usecomp==2)
 459       {
 460         inflateEnd(&m_compstream);
 461       }
 462     #endif
 463   };
 464
 465   virtual void WDL_VARARG_WARN(printf,2,3) AddLine(const char *fmt, ...);
 466   virtual int GetLine(char *buf, int buflen); // returns -1 on eof
 467
 468   virtual WDL_INT64 GetOutputSize() { return m_heapbuf ? m_heapbuf->GetSize() : 0; }
 469
 470   virtual int GetTempFlag() { return m_tmpflag; }
 471   virtual void SetTempFlag(int flag) { m_tmpflag=flag; }
 472
 473   int m_pos;
 474   WDL_HeapBuf *m_heapbuf;
 475   int m_tmpflag;
 476   int m_rwflags;
 477
 478 #ifdef WDL_MEMPROJECTCONTEXT_USE_ZLIB
 479   int DecompressData()
 480   {
 481     if (m_pos >= m_heapbuf->GetSize()) return 1;
 482
 483     m_compstream.next_in = (unsigned char *)m_heapbuf->Get() + m_pos;
 484     m_compstream.avail_in = m_heapbuf->GetSize()-m_pos;
 485     m_compstream.total_in = 0;
 486
 487     int outchunk = 65536;
 488     m_compstream.next_out = (unsigned char *)m_compdatabuf.Add(NULL,outchunk);
 489     m_compstream.avail_out = outchunk;
 490     m_compstream.total_out = 0;
 491
 492     int e = inflate(&m_compstream,Z_NO_FLUSH);
 493
 494     m_pos += m_compstream.total_in;
 495     m_compdatabuf.Add(NULL,m_compstream.total_out - outchunk); // rewind
 496
 497     return e != Z_OK;
 498   }
 499
 500   void FlushComp(bool eof)
 501   {
 502     while (m_compdatabuf.Available()>=WDL_MEMPROJECTCONTEXT_ZLIB_CHUNKSIZE || eof)
 503     {
 504       if (!m_heapbuf->GetSize()) m_heapbuf->SetGranul(256*1024);
 505       m_compstream.next_in = (unsigned char *)m_compdatabuf.Get();
 506       m_compstream.avail_in = m_compdatabuf.Available();
 507       m_compstream.total_in = 0;
 508
 509       int osz = m_heapbuf->GetSize();
 510
 511       int newsz=osz + wdl_max(m_compstream.avail_in,8192) + 256;
 512       m_compstream.next_out = (unsigned char *)m_heapbuf->Resize(newsz, false) + osz;
 513       if (m_heapbuf->GetSize()!=newsz) return; // ERROR
 514       m_compstream.avail_out = newsz-osz;
 515       m_compstream.total_out=0;
 516
 517       deflate(&m_compstream,eof?Z_SYNC_FLUSH:Z_NO_FLUSH);
 518
 519       m_heapbuf->Resize(osz+m_compstream.total_out,false);
 520       m_compdatabuf.Advance(m_compstream.total_in);
 521       if (m_compstream.avail_out) break; // no need to process anymore
 522
 523     }
 524     m_compdatabuf.Compact();
 525   }
 526
 527   // these will be used for either decompression or compression, fear
 528   int m_usecomp; // 0=?, -1 = fail, 1=yes
 529   WDL_Queue m_compdatabuf;
 530   z_stream m_compstream;
 531 #endif
 532
 533 };
 534
 535 // returns length, modifies ptr to point to tmp if newline needed to be filtered
 536 static int filter_newline_buf(const char **ptr, char *tmp, int tmpsz)
 537 {
 538   const char *use_buf = *ptr;
 539   if (!use_buf) return -1;
 540
 541   int l;
 542   for (l=0; use_buf[l] && use_buf[l] != '\n'; l++);
 543
 544   if (!use_buf[l]) return l;
 545
 546   lstrcpyn_safe(tmp,use_buf,tmpsz);
 547   *ptr=tmp;
 548
 549   if (l >= tmpsz) return tmpsz-1;
 550
 551   for (;tmp[l]; l++) if (tmp[l] == '\n') tmp[l] = ' '; // replace any newlines with spaces
 552   return l;
 553 }
 554
 555
 556 void ProjectStateContext_Mem::AddLine(const char *fmt, ...)
 557 {
 558   if (!m_heapbuf || !(m_rwflags&2)) return;
 559
 560   char tmp[8192];
 561
 562   const char *use_buf;
 563   int l;
 564
 565   va_list va;
 566   va_start(va,fmt);
 567
 568   if (fmt && fmt[0] == '%' && (fmt[1] == 's' || fmt[1] == 'S') && !fmt[2])
 569   {
 570     use_buf = va_arg(va,const char *);
 571     l = filter_newline_buf(&use_buf,tmp,(int)sizeof(tmp)) + 1;
 572   }
 573   else
 574   {
 575     l = ProjectContextFormatString(tmp,sizeof(tmp),fmt, va) + 1;
 576     use_buf = tmp;
 577   }
 578   va_end(va);
 579
 580   if (l < 1) return;
 581
 582
 583 #ifdef WDL_MEMPROJECTCONTEXT_USE_ZLIB
 584   if (!m_usecomp)
 585   {
 586     if (deflateInit(&m_compstream,WDL_MEMPROJECTCONTEXT_USE_ZLIB)==Z_OK) m_usecomp=1;
 587     else m_usecomp=-1;
 588   }
 589
 590   if (m_usecomp==1)
 591   {
 592     m_compdatabuf.Add(use_buf,(int)l);
 593     FlushComp(false);
 594     return;
 595   }
 596 #endif
 597
 598
 599   const int sz=m_heapbuf->GetSize();
 600   if (!sz && m_heapbuf->GetGranul() < 256*1024)
 601   {
 602     m_heapbuf->SetGranul(256*1024);
 603   }
 604
 605   char *p = (char *)m_heapbuf->ResizeOK(sz+l);
 606   if (!p)
 607   {
 608     // ERROR, resize to 0 and return
 609     m_heapbuf->Resize(0);
 610     m_heapbuf=0;
 611     return;
 612   }
 613   memcpy(p+sz,use_buf,l);
 614 }
 615
 616 int ProjectStateContext_Mem::GetLine(char *buf, int buflen) // returns -1 on eof
 617 {
 618   if (!m_heapbuf || !(m_rwflags&1)) return -1;
 619
 620   buf[0]=0;
 621
 622
 623 #ifdef WDL_MEMPROJECTCONTEXT_USE_ZLIB
 624   if (!m_usecomp)
 625   {
 626     unsigned char hdr[]={0x78,0x01};
 627     if (m_heapbuf->GetSize()>2 && !memcmp(hdr,m_heapbuf->Get(),4) && inflateInit(&m_compstream)==Z_OK) m_usecomp=2;
 628     else m_usecomp=-1;
 629   }
 630   if (m_usecomp==2)
 631   {
 632     int x=0;
 633     for (;;)
 634     {
 635       const char *p = (const char*) m_compdatabuf.Get();
 636       for (x = 0; x < m_compdatabuf.Available() && p[x] && p[x] != '\r' && p[x] != '\n'; x ++);
 637       while (x >= m_compdatabuf.Available())
 638       {
 639         int err = DecompressData();
 640         p = (const char *)m_compdatabuf.Get();
 641         for (; x < m_compdatabuf.Available() && p[x] && p[x] != '\r' && p[x] != '\n'; x ++);
 642
 643         if (err) break;
 644       }
 645
 646       if (x||!m_compdatabuf.Available()) break;
 647
 648       m_compdatabuf.Advance(1); // skip over nul or newline char
 649     }
 650
 651     if (!x) return -1;
 652
 653     if (buflen > 0 && buf)
 654     {
 655       int l = wdl_min(buflen-1,x);
 656       memcpy(buf,m_compdatabuf.Get(),l);
 657       buf[l]=0;
 658     }
 659
 660     m_compdatabuf.Advance(x+1);
 661     m_compdatabuf.Compact();
 662     return 0;
 663   }
 664 #endif
 665
 666
 667   int avail = m_heapbuf->GetSize() - m_pos;
 668   const char *p=(const char *)m_heapbuf->Get() + m_pos;
 669   while (avail > 0 && (p[0] =='\r'||p[0]=='\n'||!p[0]||p[0] == ' ' || p[0] == '\t'))
 670   {
 671     p++;
 672     m_pos++;
 673     avail--;
 674   }
 675   if (avail <= 0) return -1;
 676
 677   int x;
 678   for (x = 0; x < avail && p[x] && p[x] != '\n'; x ++);
 679   m_pos += x+1;
 680
 681   if (buflen > 0&&buf)
 682   {
 683     int l = buflen-1;
 684     if (l>x) l=x;
 685     memcpy(buf,p,l);
 686     if (l>0 && buf[l-1]=='\r') l--;
 687     buf[l]=0;
 688   }
 689   return 0;
 690 }
 691
 692 class ProjectStateContext_File : public ProjectStateContext
 693 {
 694 public:
 695
 696   ProjectStateContext_File(WDL_FileRead *rd, WDL_FileWrite *wr)
 697   {
 698     m_rd=rd;
 699     m_wr=wr;
 700     m_indent=0;
 701     m_bytesOut=0;
 702     m_errcnt=false;
 703     m_tmpflag=0;
 704     _rdbuf_pos = _rdbuf_valid = 0;
 705   }
 706   virtual ~ProjectStateContext_File(){ delete m_rd; delete m_wr; };
 707
 708   virtual void WDL_VARARG_WARN(printf,2,3) AddLine(const char *fmt, ...);
 709   virtual int GetLine(char *buf, int buflen); // returns -1 on eof
 710
 711   virtual WDL_INT64 GetOutputSize() { return m_bytesOut; }
 712
 713   virtual int GetTempFlag() { return m_tmpflag; }
 714   virtual void SetTempFlag(int flag) { m_tmpflag=flag; }
 715
 716   bool HasError() { return m_errcnt; }
 717
 718   WDL_INT64 m_bytesOut WDL_FIXALIGN;
 719
 720   WDL_FileRead *m_rd;
 721   WDL_FileWrite *m_wr;
 722
 723   char rdbuf[4096];
 724   int _rdbuf_pos, _rdbuf_valid;
 725
 726   int m_indent;
 727   int m_tmpflag;
 728   bool m_errcnt;
 729
 730 };
 731
 732 int ProjectStateContext_File::GetLine(char *buf, int buflen)
 733 {
 734   if (!m_rd||buflen<3) return -1;
 735
 736   char * const buf_orig=buf;
 737   int rdpos = _rdbuf_pos;
 738   int rdvalid = _rdbuf_valid;
 739   buflen -= 2;
 740
 741   for (;;)
 742   {
 743     while (rdpos < rdvalid)
 744     {
 745       char c=rdbuf[rdpos++];
 746       switch (c)
 747       {
 748         case ' ': case '\r': case '\n': case '\t': break;
 749         default:
 750           *buf++=c;
 751
 752           do
 753           {
 754             int mxl = rdvalid - rdpos;
 755             if (mxl > buflen) mxl=buflen;
 756             while (mxl-->0)
 757             {
 758               char c2 = rdbuf[rdpos++];
 759               if (c2=='\n') goto finished;
 760
 761               *buf++ = c2;
 762               buflen--;
 763             }
 764             if (rdpos>=rdvalid)
 765             {
 766               rdpos = 0;
 767               rdvalid = m_rd->Read(rdbuf, sizeof(rdbuf));
 768               if (rdvalid<1) break;
 769             }
 770           }
 771           while (buflen > 0);
 772
 773         finished:
 774           _rdbuf_pos=rdpos;
 775           _rdbuf_valid=rdvalid;
 776
 777           if (buf > buf_orig && buf[-1] == '\r') buf--;
 778           *buf=0;
 779         return 0;
 780       }
 781     }
 782
 783     rdpos = 0;
 784     rdvalid = m_rd->Read(rdbuf, sizeof(rdbuf));
 785     if (rdvalid<1)
 786     {
 787       buf[0]=0;
 788       return -1;
 789     }
 790   }
 791 }
 792
 793 void ProjectStateContext_File::AddLine(const char *fmt, ...)
 794 {
 795   if (m_wr && !m_errcnt)
 796   {
 797     int err=0;
 798
 799     char tmp[8192];
 800     const char *use_buf;
 801     va_list va;
 802     va_start(va,fmt);
 803     int l;
 804
 805     if (fmt && fmt[0] == '%' && (fmt[1] == 's' || fmt[1] == 'S') && !fmt[2])
 806     {
 807       // special case "%s" passed, directly use it
 808       use_buf = va_arg(va,const char *);
 809       l = filter_newline_buf(&use_buf,tmp,(int)sizeof(tmp));
 810     }
 811     else
 812     {
 813       l = ProjectContextFormatString(tmp,sizeof(tmp),fmt, va);
 814       use_buf = tmp;
 815     }
 816
 817     va_end(va);
 818     if (l < 0) return;
 819
 820
 821     int a=m_indent;
 822     if (use_buf[0] == '<') m_indent+=2;
 823     else if (use_buf[0] == '>') a=(m_indent-=2);
 824
 825     if (a>0)
 826     {
 827       m_bytesOut+=a;
 828       char tb[128];
 829       memset(tb,' ',a < (int)sizeof(tb) ? a : (int)sizeof(tb));
 830       while (a>0)
 831       {
 832         const int tl = a < (int)sizeof(tb) ? a : (int)sizeof(tb);
 833         a-=tl;
 834         m_wr->Write(tb,tl);
 835       }
 836     }
 837
 838     err |= m_wr->Write(use_buf,l) != l;
 839     err |= m_wr->Write("\r\n",2) != 2;
 840     m_bytesOut += 2 + l;
 841
 842     if (err) m_errcnt=true;
 843   }
 844 }
 845
 846
 847
 848 ProjectStateContext *ProjectCreateFileRead(const char *fn)
 849 {
 850   WDL_FileRead *rd = new WDL_FileRead(fn,0,65536,1);
 851   if (!rd || !rd->IsOpen())
 852   {
 853     delete rd;
 854     return NULL;
 855   }
 856   return new ProjectStateContext_File(rd,NULL);
 857 }
 858 ProjectStateContext *ProjectCreateFileWrite(const char *fn)
 859 {
 860   WDL_FileWrite *wr = new WDL_FileWrite(fn);
 861   if (!wr || !wr->IsOpen())
 862   {
 863     delete wr;
 864     return NULL;
 865   }
 866   return new ProjectStateContext_File(NULL,wr);
 867 }
 868
 869
 870 ProjectStateContext *ProjectCreateMemCtx(WDL_HeapBuf *hb)
 871 {
 872   return new ProjectStateContext_Mem(hb,3);
 873 }
 874
 875 ProjectStateContext *ProjectCreateMemCtx_Read(const WDL_HeapBuf *hb)
 876 {
 877   return new ProjectStateContext_Mem((WDL_HeapBuf *)hb,1);
 878 }
 879 ProjectStateContext *ProjectCreateMemCtx_Write(WDL_HeapBuf *hb)
 880 {
 881   return new ProjectStateContext_Mem(hb,2);
 882 }
 883
 884
 885
 886
 887 class ProjectStateContext_FastQueue : public ProjectStateContext
 888 {
 889 public:
 890
 891   ProjectStateContext_FastQueue(WDL_FastQueue *fq)
 892   {
 893     m_fq = fq;
 894     m_tmpflag=0;
 895   }
 896
 897   virtual ~ProjectStateContext_FastQueue()
 898   {
 899   };
 900
 901   virtual void WDL_VARARG_WARN(printf,2,3) AddLine(const char *fmt, ...);
 902   virtual int GetLine(char *buf, int buflen) { return -1; }//unsup
 903
 904   virtual WDL_INT64 GetOutputSize() { return m_fq ? m_fq->Available() : 0; }
 905
 906   virtual int GetTempFlag() { return m_tmpflag; }
 907   virtual void SetTempFlag(int flag) { m_tmpflag=flag; }
 908
 909   WDL_FastQueue *m_fq;
 910   int m_tmpflag;
 911
 912
 913 };
 914
 915 void ProjectStateContext_FastQueue::AddLine(const char *fmt, ...)
 916 {
 917   if (!m_fq) return;
 918
 919   va_list va;
 920   va_start(va,fmt);
 921
 922   char tmp[8192];
 923   if (fmt && fmt[0] == '%' && (fmt[1] == 's' || fmt[1] == 'S') && !fmt[2])
 924   {
 925     const char *use_buf = va_arg(va,const char *);
 926     const int l = filter_newline_buf(&use_buf,tmp,(int)sizeof(tmp));
 927     if (use_buf) m_fq->Add(use_buf, l + 1);
 928   }
 929   else
 930   {
 931     const int l = ProjectContextFormatString(tmp,sizeof(tmp),fmt, va);
 932     if (l>0) m_fq->Add(tmp, l+1);
 933   }
 934   va_end(va);
 935 }
 936
 937
 938
 939
 940
 941
 942
 943 ProjectStateContext *ProjectCreateMemWriteFastQueue(WDL_FastQueue *fq) // only write!
 944 {
 945   return new ProjectStateContext_FastQueue(fq);
 946 }
 947
 948 bool ProjectContext_GetNextLine(ProjectStateContext *ctx, LineParser *lpOut)
 949 {
 950   for (;;)
 951   {
 952     char linebuf[4096];
 953     if (ctx->GetLine(linebuf,sizeof(linebuf)))
 954     {
 955       lpOut->parse("");
 956       return false;
 957     }
 958
 959     if (lpOut->parse(linebuf)||lpOut->getnumtokens()<=0) continue;
 960
 961     return true; // success!
 962
 963   }
 964 }
 965
 966
 967 bool ProjectContext_EatCurrentBlock(ProjectStateContext *ctx, ProjectStateContext *ctxOut)
 968 {
 969   int child_count=1;
 970   if (ctx) for (;;)
 971   {
 972     char linebuf[4096];
 973     if (ctx->GetLine(linebuf,sizeof(linebuf))) break;
 974     const char *sp = linebuf;
 975     while (*sp == ' ' || *sp == '\t') sp++;
 976
 977     const char *p = sp;
 978     if (*p == '\'' || *p == '"' || *p == '`') p++; // skip a quote if any
 979     if (p[0] == '>')  if (--child_count < 1) return true;
 980
 981     if (ctxOut) ctxOut->AddLine("%s",sp);
 982
 983     if (p[0] == '<') child_count++;
 984   }
 985
 986   return false;
 987 }
 988
 989
 990 #include "wdl_base64.h"
 991
 992 int cfg_decode_binary(ProjectStateContext *ctx, WDL_HeapBuf *hb) // 0 on success, doesnt clear hb
 993 {
 994   int child_count=1;
 995   for (;;)
 996   {
 997     char linebuf[4096];
 998     if (ctx->GetLine(linebuf,sizeof(linebuf))) break;
 999
1000     const char *p = linebuf;
1001     while (*p == ' ' || *p == '\t') p++;
1002     if (*p == '\'' || *p == '"' || *p == '`') p++; // skip a quote if any
1003
1004     if (p[0] == '<') child_count++;
1005     else if (p[0] == '>') { if (child_count-- == 1) return 0; }
1006     else if (child_count == 1 && p[0])
1007     {
1008       unsigned char buf[3200];
1009       const int buf_l=wdl_base64decode(p,buf,sizeof(buf));
1010       if (buf_l)
1011       {
1012         const int os=hb->GetSize();
1013         char *dest = (char*)hb->ResizeOK(os+buf_l);
1014         if (dest) memcpy(dest+os,buf,buf_l);
1015       }
1016     }
1017   }
1018   return -1;
1019 }
1020
1021 void cfg_encode_binary(ProjectStateContext *ctx, const void *ptr, int len)
1022 {
1023   if (!ctx || len < 1) return;
1024
1025   const unsigned char *p=(const unsigned char *)ptr;
1026   if (len > 128 && len < (1<<30))
1027   {
1028     // we could (probably should) use dynamic_cast<> here, but as we span modules this
1029     // raises all kinds of questions (especially with VC having the option to disable RTTI).
1030     // for now, we assume that the first void * in an object is the vtable pointer. with
1031     // testing, of course.
1032     WDL_FastQueue *fq = NULL;
1033     WDL_HeapBuf *hb = NULL;
1034 #ifndef WDL_MEMPROJECTCONTEXT_USE_ZLIB
1035     static const ProjectStateContext_Mem hb_def(NULL,0);
1036 #endif
1037     static const ProjectStateContext_FastQueue fq_def(NULL);
1038     if (*(void **)ctx == *(void **)&fq_def)
1039     {
1040       fq=((ProjectStateContext_FastQueue*)ctx)->m_fq;
1041     }
1042 #ifndef WDL_MEMPROJECTCONTEXT_USE_ZLIB
1043     else if (*(void **)ctx == *(void **)&hb_def)
1044     {
1045       hb=((ProjectStateContext_Mem*)ctx)->m_heapbuf;
1046     }
1047 #endif
1048
1049     if (fq||hb)
1050     {
1051       const int linelen8 = 280/8;
1052
1053       const int enc_len = ((len+2)/3)*4; // every 3 characters end up as 4
1054       const int lines = (enc_len + linelen8*8 - 1) / (linelen8*8);
1055
1056       char *wr = NULL;
1057       if (fq)
1058       {
1059         wr = (char*)fq->Add(WDL_FASTQUEUE_ADD_NOZEROBUF,enc_len + lines);
1060       }
1061       else if (hb)
1062       {
1063         const int oldsz=hb->GetSize();
1064         wr=(char*)hb->ResizeOK(oldsz + enc_len + lines,false);
1065         if (wr) wr+=oldsz;
1066       }
1067
1068       if (wr)
1069       {
1070         #ifdef _DEBUG
1071         char * const wr_end=wr + enc_len + lines;
1072         #endif
1073
1074         int lpos = 0;
1075
1076         while (len >= 6)
1077         {
1078           const int accum = (p[0] << 16) + (p[1] << 8) + p[2];
1079           const int accum2 = (p[3] << 16) + (p[4] << 8) + p[5];
1080           wr[0] = wdl_base64_alphabet[(accum >> 18) & 0x3F];
1081           wr[1] = wdl_base64_alphabet[(accum >> 12) & 0x3F];
1082           wr[2] = wdl_base64_alphabet[(accum >> 6) & 0x3F];
1083           wr[3] = wdl_base64_alphabet[accum & 0x3F];
1084           wr[4] = wdl_base64_alphabet[(accum2 >> 18) & 0x3F];
1085           wr[5] = wdl_base64_alphabet[(accum2 >> 12) & 0x3F];
1086           wr[6] = wdl_base64_alphabet[(accum2 >> 6) & 0x3F];
1087           wr[7] = wdl_base64_alphabet[accum2 & 0x3F];
1088           wr+=8;
1089           p+=6;
1090           len-=6;
1091
1092           if (++lpos >= linelen8) { *wr++= 0; lpos=0; }
1093         }
1094
1095         if (len >= 3)
1096         {
1097           const int accum = (p[0] << 16) + (p[1] << 8) + p[2];
1098           wr[0] = wdl_base64_alphabet[(accum >> 18) & 0x3F];
1099           wr[1] = wdl_base64_alphabet[(accum >> 12) & 0x3F];
1100           wr[2] = wdl_base64_alphabet[(accum >> 6) & 0x3F];
1101           wr[3] = wdl_base64_alphabet[accum & 0x3F];
1102           wr+=4;
1103           p+=3;
1104           len-=3;
1105           lpos+=3;
1106         }
1107
1108         if (len>0)
1109         {
1110           lpos += len;
1111           if (len == 2)
1112           {
1113             const int accum = (p[0] << 8) | p[1];
1114             wr[0] = wdl_base64_alphabet[(accum >> 10) & 0x3F];
1115             wr[1] = wdl_base64_alphabet[(accum >> 4) & 0x3F];
1116             wr[2] = wdl_base64_alphabet[(accum & 0xF)<<2];
1117           }
1118           else
1119           {
1120             const int accum = p[0];
1121             wr[0] = wdl_base64_alphabet[(accum >> 2) & 0x3F];
1122             wr[1] = wdl_base64_alphabet[(accum & 0x3)<<4];
1123             wr[2] = '=';
1124           }
1125           wr[3] = '=';
1126           wr+=4;
1127         }
1128         if (lpos>0) *wr++=0;
1129
1130         #ifdef _DEBUG
1131           if (wr != wr_end) wdl_log("cfg_encode_binary: block mode size mismatch %d!\n", (int)(wr-wr_end));
1132         #endif
1133         return;
1134       }
1135     }
1136   }
1137
1138   do
1139   {
1140     char buf[256];
1141     int thiss=len;
1142     if (thiss > 96) thiss=96;
1143     wdl_base64encode(p,buf,thiss);
1144
1145     ctx->AddLine("%s",buf);
1146     p+=thiss;
1147     len-=thiss;
1148   }
1149   while (len>0);
1150
1151 }
1152
1153
1154 int cfg_decode_textblock(ProjectStateContext *ctx, WDL_FastString *str) // 0 on success, appends to str
1155 {
1156   int child_count=1;
1157   bool did_firstline=!!str->Get()[0];
1158   for (;;)
1159   {
1160     char linebuf[4096];
1161     if (ctx->GetLine(linebuf,sizeof(linebuf))) break;
1162
1163     const char *p = linebuf;
1164     while (*p == ' ' || *p == '\t') p++;
1165     if (*p == '\'' || *p == '"' || *p == '`') p++; // skip a quote if any
1166
1167     if (!p[0]) continue;
1168     else if (p[0] == '<') child_count++;
1169     else if (p[0] == '>') { if (child_count-- == 1) return 0; }
1170     else if (child_count == 1)
1171     {
1172       const char *prefix = did_firstline ? "\r\n" : "";
1173       // lines can have a prefix immediately before | to specify the line ending of the previous line
1174       switch (p[0])
1175       {
1176         case 'c': prefix=""; p++; break;
1177         case 'n': prefix="\n"; p++; break;
1178         case 'r': prefix="\r"; p++; break;
1179         case 'R': prefix="\n\r"; p++; break;
1180       }
1181
1182       if (p[0] == '|')
1183       {
1184         if (*prefix) str->Append(prefix);
1185         str->Append(++p);
1186         did_firstline=true;
1187       }
1188     }
1189   }
1190   return -1;
1191
1192 }
1193
1194 void cfg_encode_textblock(ProjectStateContext *ctx, const char *txt) // splits long lines
1195 {
1196   while (*txt)
1197   {
1198     int l = 0;
1199     while (txt[l] && l < 4000 && txt[l] != '\r' && txt[l] != '\n') l++;
1200     ctx->AddLine("|%.*s",l,txt);
1201     txt += l;
1202     if (*txt == '\r')
1203     {
1204       if (*++txt== '\n') txt++;
1205     }
1206     else if (*txt == '\n')
1207     {
1208       if (*++txt == '\r') txt++;
1209     }
1210   }
1211 }
1212
1213 void cfg_encode_textblock2(ProjectStateContext *ctx, const char *txt) // preserves long lines, line endings
1214 {
1215   char prefix = ' ';
1216   while (*txt)
1217   {
1218     int l = 0;
1219     while (txt[l] && l < 2000 && txt[l] != '\r' && txt[l] != '\n') l++;
1220     ctx->AddLine("%c|%.*s",prefix,l,txt);
1221     txt += l;
1222     if (*txt == '\r')
1223     {
1224       prefix = 'r';
1225       if (*++txt== '\n')
1226       {
1227         txt++;
1228         prefix = ' ';
1229       }
1230     }
1231     else if (*txt == '\n')
1232     {
1233       prefix = 'n';
1234       if (*++txt == '\r')
1235       {
1236         txt++;
1237         prefix = 'R';
1238       }
1239     }
1240     else if (*txt) prefix = 'c';
1241     else break;
1242
1243     if (!*txt)
1244       ctx->AddLine("%c|",prefix);
1245   }
1246 }
1247
1248 char getConfigStringQuoteChar(const char *p, bool prefer_quoteless)
1249 {
1250   if (!p || !*p) return '"';
1251
1252   char fc = *p;
1253   int flags=0;
1254   while (*p && flags!=15)
1255   {
1256     char c=*p++;
1257     if (c=='"') flags|=1;
1258     else if (c=='\'') flags|=2;
1259     else if (c=='`') flags|=4;
1260     else if (c == ' ' || c == '\t' || c == '\n' || c == '\r') flags |= 8;
1261   }
1262   if (prefer_quoteless || flags == 7)
1263   {
1264     if (!(flags & 8) && fc != '"' && fc != '\'' && fc != '`' && fc != '#' && fc != ';') return ' ';
1265   }
1266
1267   if (!(flags & 1)) return '"';
1268   if (!(flags & 2)) return '\'';
1269   if (!(flags & 4)) return '`';
1270   return 0;
1271 }
1272
1273 bool configStringWantsBlockEncoding(const char *in) // returns true if over 1k long, has newlines, or contains all quote chars
1274 {
1275   int maxl = 1024, flags = 0;
1276   while (--maxl)
1277   {
1278     switch (*in++)
1279     {
1280       case 0: return false;
1281       case '\n': return true;
1282       case '"': if ((flags|=1)==7) return true; break;
1283       case '`': if ((flags|=2)==7) return true; break;
1284       case '\'': if ((flags|=4)==7) return true; break;
1285     }
1286   }
1287   return true;
1288 }
1289
1290 void makeEscapedConfigString(const char *in, WDL_FastString *out)
1291 {
1292   char c;
1293   if (!in || !*in) out->Set("\"\"");
1294   else if ((c = getConfigStringQuoteChar(in)))
1295   {
1296     if (c == ' ')
1297     {
1298       out->Set(in);
1299     }
1300     else
1301     {
1302       out->Set(&c,1);
1303       out->Append(in);
1304       out->Append(&c,1);
1305     }
1306   }
1307   else  // ick, change ` into '
1308   {
1309     out->Set("`");
1310     out->Append(in);
1311     out->Append("`");
1312     char *p=(char *)out->Get()+1;
1313     while (*p && p[1])
1314     {
1315       if (*p == '`') *p='\'';
1316       else if (*p == '\r' || *p == '\n') *p=' ';
1317       p++;
1318     }
1319   }
1320 }