libuuu/buffer.cpp

   1 /*
   2 * Copyright 2018-2022 NXP.
   3 *
   4 * Redistribution and use in source and binary forms, with or without modification,
   5 * are permitted provided that the following conditions are met:
   6 *
   7 * Redistributions of source code must retain the above copyright notice, this
   8 * list of conditions and the following disclaimer.
   9 *
  10 * Redistributions in binary form must reproduce the above copyright notice, this
  11 * list of conditions and the following disclaimer in the documentation and/or
  12 * other materials provided with the distribution.
  13 *
  14 * Neither the name of the NXP Semiconductor nor the names of its
  15 * contributors may be used to endorse or promote products derived from this
  16 * software without specific prior written permission.
  17 *
  18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  19 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
  22 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  28 * POSSIBILITY OF SUCH DAMAGE.
  29 *
  30 */
  31
  32 #include <map>
  33 #include "buffer.h"
  34 #include <sys/stat.h>
  35 #include "liberror.h"
  36 #include <iostream>
  37 #include <fstream>
  38 #include "libcomm.h"
  39 #include "libuuu.h"
  40 #include "zip.h"
  41 #include "fat.h"
  42 #include "tar.h"
  43 #include <string.h>
  44 #include "bzlib.h"
  45 #include "stdio.h"
  46 #include <limits>
  47 #include "http.h"
  48 #include "zstd.h"
  49 #include "libusb.h"
  50
  51 #ifdef WIN32
  52 #define stat_os _stat64
  53 #elif defined(__APPLE__)
  54 #define stat_os stat
  55 #include "dirent.h"
  56 #else
  57 #define stat_os stat64
  58 #include "dirent.h"
  59 #endif
  60
  61 static map<string, shared_ptr<FileBuffer>> g_filebuffer_map;
  62 static mutex g_mutex_map;
  63 static bool g_small_memory = true;
  64
  65 #define MAGIC_PATH '>'
  66
  67 string g_current_dir = ">";
  68
  69 void set_current_dir(const string &dir)
  70 {
  71         g_current_dir = MAGIC_PATH;
  72         g_current_dir += dir;
  73 }
  74
  75
  76 int DataBuffer:: resize(size_t sz)
  77 {
  78         if (m_allocate_way != ALLOCATION_WAYS::MALLOC)
  79         {
  80                 set_last_err_string("data buffer ref can't resize");
  81                 assert(false);
  82                 return -1;
  83         }
  84
  85         if (sz > m_MemSize)
  86         {
  87                 m_pDatabuffer = (uint8_t*)realloc(m_pDatabuffer, sz);
  88                 if (!m_pDatabuffer)
  89                 {
  90                         set_last_err_string("fail alloc memory");
  91                         return -1;
  92                 }
  93                 m_MemSize = sz;
  94         }
  95
  96         m_DataSize = sz;
  97         return 0;
  98 }
  99
 100 int DataBuffer::ref_other_buffer(std::shared_ptr<FileBuffer> p, size_t offset, size_t size)
 101 {
 102         if (!p->IsRefable())
 103                 return -1;
 104
 105         if (p->m_allocate_way == FileBuffer::ALLOCATION_WAYS::SEGMENT)
 106         {
 107                 shared_ptr<FragmentBlock> blk;
 108                 blk = p->get_map_it(offset);
 109                 if (offset + size < blk->m_output_offset + blk->m_actual_size)
 110                 {
 111                         m_pDatabuffer = blk->m_data.data() + offset - blk->m_output_offset;
 112                 }
 113                 else
 114                 {
 115                         return -1;
 116                 }
 117         }
 118         else
 119         {
 120                 m_pDatabuffer = p->data() + offset;
 121         }
 122
 123         m_DataSize = size;
 124
 125         m_allocate_way = ALLOCATION_WAYS::REF;
 126         m_ref = p;
 127         return 0;
 128 };
 129
 130 class FSBasic
 131 {
 132 public:
 133         friend class DataBuffer;
 134         friend class FileBuffer;
 135         virtual int get_file_timesample(const string &filename, uint64_t *ptime) = 0;
 136         virtual int load(const string &backfile, const string &filename, shared_ptr<FileBuffer> p) = 0;
 137         virtual bool exist(const string &backfile, const string &filename) = 0;
 138         virtual int for_each_ls(uuu_ls_file fn, const string &backfile, const string &filename, void *p) = 0;
 139
 140         virtual int Decompress(const string& /*backfifle*/, shared_ptr<FileBuffer> /*outp*/) { return 0; };
 141         virtual bool seekable(const string& /*backfile*/) { return false; }
 142         virtual std::shared_ptr<FragmentBlock> ScanCompressblock(const string& /*backfile*/, size_t& /*input_offset*/, size_t& /*output_offset*/) { return NULL; };
 143         virtual int PreloadWorkThread(shared_ptr<FileBuffer>outp);
 144
 145         virtual int split(const string &filename, string *outbackfile, string *outfilename, bool dir=false)
 146         {
 147                 string path = str_to_upper(filename);
 148                 if (m_ext == nullptr || strlen(m_ext) == 0)
 149                 {
 150                         if(dir)
 151                         {
 152                                 size_t pos = path.rfind("/");
 153                                 if(pos == string::npos)
 154                                 {
 155                                         *outbackfile = MAGIC_PATH;
 156                                         *outbackfile += "./";
 157                                         *outfilename = filename;
 158                                 } else {
 159                                         *outbackfile = filename.substr(0, pos);
 160                                         if(filename.size() >= pos + 1)
 161                                                 *outfilename = filename.substr(pos + 1);
 162                                         else
 163                                                 outfilename->clear();
 164                                 }
 165                         }else
 166                         {
 167                                 *outbackfile = filename;
 168                         }
 169                         return 0;
 170                 }
 171
 172                 string ext = m_ext;
 173                 if(!dir)
 174                         ext += "/";
 175                 size_t pos = path.rfind(ext);
 176                 if (pos == string::npos)
 177                 {
 178                         string err = "can't find ext name in path: ";
 179                         err += filename;
 180                         set_last_err_string(err);
 181                         return -1;
 182                 }
 183
 184                 *outbackfile = filename.substr(0, pos + strlen(m_ext));
 185
 186                 if(filename.size() >= pos + strlen(m_ext) + 1)
 187                         *outfilename = filename.substr(pos + strlen(m_ext) + 1);
 188                 else
 189                         outfilename->clear();
 190                 return 0;
 191         }
 192
 193 protected:
 194         const char * m_ext = nullptr;
 195         const char * m_Prefix = nullptr;
 196 public:
 197         bool    m_small_pool = false;
 198 };
 199
 200 static class FSFlat: public FSBasic
 201 {
 202 public:
 203         FSFlat() { m_ext = ""; }
 204         int get_file_timesample(const string &filename, uint64_t *ptime) override
 205         {
 206                 struct stat_os st;
 207                 if (stat_os(filename.c_str() + 1, &st))
 208                 {
 209                         set_last_err_string("stat_os failure");
 210                         return -1;
 211                 }
 212
 213                 *ptime = st.st_mtime;
 214
 215                 return 0;
 216         }
 217
 218         bool exist(const string &backfile, const string & /*filename*/) override
 219         {
 220                 struct stat_os st;
 221                 int off = 1;
 222
 223                 if (backfile[0] != MAGIC_PATH)
 224                         off = 0;
 225
 226                 return stat_os(backfile.c_str() + off, &st) == 0 && ((st.st_mode & S_IFDIR) == 0);
 227         }
 228
 229         int load(const string &backfile, const string &/*filename*/, shared_ptr<FileBuffer> p) override
 230         {
 231                 struct stat_os st;
 232                 if (stat_os(backfile.c_str() + 1, &st))
 233                 {
 234                         set_last_err_string("stat_os failure");
 235                         return -1;
 236                 }
 237                 p->unmapfile();
 238
 239                 if (p->mapfile(backfile.substr(1), st.st_size))
 240                         return -1;
 241
 242                 p->m_available_size = st.st_size;
 243
 244                 atomic_fetch_or(&p->m_dataflags, FILEBUFFER_FLAG_LOADED | FILEBUFFER_FLAG_NEVER_FREE);
 245                 p->m_request_cv.notify_all();
 246
 247                 return 0;
 248         }
 249
 250         int for_each_ls(uuu_ls_file fn, const string &backfile, const string &filename, void *p) override
 251         {
 252                 struct stat_os st;
 253
 254                 if(stat_os(backfile.c_str() + 1, &st))
 255                 {
 256                         return -1;
 257                 }
 258
 259                 if(st.st_mode & S_IFDIR)
 260                 {
 261 #ifdef WIN32
 262                         string str = backfile.substr(1);
 263                         if (filename.empty())
 264                                 str += "/*";
 265                         else
 266                                 str += "/" + filename;
 267
 268                         WIN32_FIND_DATA fd;
 269                         HANDLE handle = FindFirstFile(str.c_str(), &fd);
 270                         BOOL b = false;
 271                         do
 272                         {
 273                                 if (handle == INVALID_HANDLE_VALUE)
 274                                         break;
 275                                 string path = backfile + "/" + fd.cFileName;
 276                                 if(fd.dwFileAttributes&FILE_ATTRIBUTE_DIRECTORY)
 277                                         path += "/";
 278                                 fn(path.c_str() + 1, p);
 279                         } while (FindNextFile(handle, &fd));
 280                         CloseHandle(handle);
 281                         return 0;
 282 #else
 283                         DIR *dir;
 284                         dir = opendir(backfile.c_str() + 1);
 285                         struct dirent *dp;
 286                         while ((dp=readdir(dir)) != nullptr)
 287                         {
 288                                 string name = dp->d_name;
 289                                 if(name.substr(0, filename.size()) == filename || filename.empty())
 290                                 {
 291                                         string path = backfile + "/" + name;
 292                                         if(dp->d_type == DT_DIR)
 293                                                 path += "/";
 294                                         fn(path.c_str() + 1, p);
 295                                 }
 296                         }
 297                         closedir(dir);
 298                         return 0;
 299 #endif
 300                 }else
 301                 {
 302                         return fn(backfile.c_str() + 1, p);
 303                 }
 304         }
 305
 306 } g_fsflat;
 307
 308 class FSNetwork : public FSBasic
 309 {
 310 protected:
 311         int m_Port;
 312
 313 public:
 314         int split(const string &filename, string *outbackfile, string *outfilename, bool /*dir = false*/) override
 315         {
 316                 if (m_Prefix == nullptr)
 317                         return -1;
 318
 319                 if (filename.size() < strlen(m_Prefix))
 320                         return -1;
 321
 322                 string path = str_to_upper(filename);
 323                 if (path.compare(1, strlen(m_Prefix), m_Prefix) == 0)
 324                 {
 325                         size_t pos;
 326                         pos = filename.find('/', 1 + strlen(m_Prefix));
 327
 328                         *outbackfile = filename.substr(1 + strlen(m_Prefix), pos - 1 - strlen(m_Prefix));
 329
 330                         size_t cpos;
 331                         cpos = outbackfile->find(':');
 332                         if (cpos != string::npos)
 333                         {
 334                                 m_Port = str_to_uint32(outbackfile->substr(cpos + 1));
 335
 336                                 *outbackfile = outbackfile->substr(0, cpos);
 337                         }
 338                         *outfilename = filename.substr(pos);
 339
 340                         return 0;
 341                 }
 342
 343                 return -1;
 344         }
 345 };
 346
 347 static class FSHttp : public FSNetwork
 348 {
 349 public:
 350         FSHttp() { m_Prefix = "HTTP://"; m_Port = 80; }
 351         int load(const string &backfile, const string &filename, shared_ptr<FileBuffer> p) override;
 352         virtual bool exist(const string &backfile, const string &filename) override
 353         {
 354                 shared_ptr<HttpStream> http = make_shared<HttpStream>();
 355
 356                 if (http->HttpGetHeader(backfile, filename, m_Port, typeid(*this) != typeid(FSHttp)))
 357                         return false;
 358
 359                 return true;
 360         };
 361         int for_each_ls(uuu_ls_file /*fn*/, const string &/*backfile*/, const string &/*filename*/, void * /*p*/) override { return 0; };
 362         int get_file_timesample(const string &/*filename*/, uint64_t * /*ptime*/) override { return 0; };
 363         int http_load(shared_ptr<HttpStream> http, shared_ptr<FileBuffer> p, string filename);
 364 }g_fshttp;
 365
 366 static class FSHttps : public FSHttp
 367 {
 368 public:
 369         FSHttps() { m_Prefix = "HTTPS://"; m_Port = 443; }
 370 }g_fshttps;
 371
 372 int FSHttp::http_load(shared_ptr<HttpStream> http, shared_ptr<FileBuffer> p, string filename)
 373 {
 374         size_t max = 0x10000;
 375
 376         uuu_notify ut;
 377         ut.type = uuu_notify::NOTIFY_DOWNLOAD_START;
 378         ut.str = (char*)filename.c_str();
 379         call_notify(ut);
 380
 381         ut.type = uuu_notify::NOTIFY_TRANS_SIZE;
 382         ut.total = p->size();
 383         call_notify(ut);
 384
 385         for (size_t i = 0; i < p->size() && !p->m_reset_stream; i += max)
 386         {
 387                 size_t sz = p->size() - i;
 388                 if (sz > max)
 389                         sz = max;
 390                 if (http->HttpDownload((char*)(p->data() + i), sz) < 0)
 391                 {
 392                         atomic_fetch_or(&p->m_dataflags, FILEBUFFER_FLAG_ERROR_BIT);
 393                         p->m_request_cv.notify_all();
 394                         return -1;
 395                 }
 396                 p->m_available_size = i + sz;
 397                 p->m_request_cv.notify_all();
 398
 399                 ut.type = uuu_notify::NOTIFY_TRANS_POS;
 400                 ut.total = i + sz;
 401                 call_notify(ut);
 402         }
 403
 404         atomic_fetch_or(&p->m_dataflags, FILEBUFFER_FLAG_LOADED | FILEBUFFER_FLAG_NEVER_FREE);
 405         p->m_request_cv.notify_all();
 406
 407         ut.type = uuu_notify::NOTIFY_DOWNLOAD_END;
 408         ut.str = (char*)filename.c_str();
 409         call_notify(ut);
 410         return 0;
 411 }
 412
 413 class FSBackFile : public FSBasic
 414 {
 415 public:
 416         int get_file_timesample(const string &filename, uint64_t *ptime) override;
 417
 418 };
 419
 420 static class FSZip : public FSBackFile
 421 {
 422 public:
 423         FSZip() { m_ext = ".ZIP"; };
 424         int load(const string &backfile, const string &filename, shared_ptr<FileBuffer> p) override;
 425         bool exist(const string &backfile, const string &filename) override;
 426         int for_each_ls(uuu_ls_file fn, const string &backfile, const string &filename, void *p) override;
 427 }g_fszip;
 428
 429 static class FSTar: public FSBackFile
 430 {
 431 public:
 432         FSTar() {m_ext = ".TAR"; };
 433         int load(const string &backfile, const string &filename, shared_ptr<FileBuffer> p) override;
 434         bool exist(const string &backfile, const string &filename) override;
 435         int for_each_ls(uuu_ls_file fn, const string &backfile, const string &filename, void *p) override;
 436 }g_fstar;
 437
 438
 439 static class FSFat : public FSBackFile
 440 {
 441 public:
 442         FSFat() { m_ext = ".SDCARD"; };
 443         int load(const string &backfile, const string &filename, shared_ptr<FileBuffer> p) override;
 444         bool exist(const string &backfile, const string &filename) override;
 445         int for_each_ls(uuu_ls_file fn, const string &backfile, const string &filename, void *p) override;
 446 }g_fsfat;
 447
 448
 449 class CommonStream
 450 {
 451 public:
 452         virtual int set_input_buff(void* p, size_t sz) = 0;
 453         virtual int set_output_buff(void* p, size_t sz) = 0;
 454         virtual size_t get_input_pos() = 0;
 455         virtual size_t get_output_pos() = 0;
 456         virtual int decompress() = 0;
 457
 458         virtual size_t get_default_input_size() { return 0x1000; }
 459         virtual size_t decompress_size(const string& /*backfile*/) { return 0; }
 460 };
 461
 462 class FSCompressStream : public FSBackFile
 463 {
 464 public:
 465         FSCompressStream() { m_small_pool = g_small_memory; }
 466         int load(const string& backfile, const string& filename, shared_ptr<FileBuffer>outp) override;
 467         bool exist(const string& backfile, const string& filename) override;
 468         int for_each_ls(uuu_ls_file fn, const string &backfile, const string &filename, void *p) override;
 469         int Decompress(const string& backfile, shared_ptr<FileBuffer>outp) override;
 470         virtual std::shared_ptr<CommonStream> create_stream() { return nullptr; };
 471 };
 472
 473
 474 class Bz2stream : public CommonStream
 475 {
 476         bz_stream m_strm;
 477         size_t m_in_size = 0;
 478         size_t m_out_size = 0;
 479
 480 public:
 481         Bz2stream() { memset(&m_strm, 0, sizeof(m_strm));  BZ2_bzDecompressInit(&m_strm, 0, 0); }
 482         virtual ~Bz2stream()
 483         {
 484                 BZ2_bzDecompressEnd(&m_strm);
 485         }
 486         virtual int set_input_buff(void* p, size_t sz) override
 487         {
 488                 m_strm.next_in = (char*)p;
 489                 m_strm.avail_in = m_in_size = sz;
 490                 return 0;
 491         };
 492         virtual int set_output_buff(void* p, size_t sz) override
 493         {
 494                 m_strm.next_out = (char*)p;
 495                 m_strm.avail_out = m_out_size = sz;
 496                 return 0;
 497         };
 498         virtual size_t get_input_pos() override
 499         {
 500                 return m_in_size - m_strm.avail_in;
 501         };
 502         virtual size_t get_output_pos() override
 503         {
 504                 return m_out_size - m_strm.avail_out;
 505         };
 506         virtual int decompress() override
 507         {
 508                 return BZ2_bzDecompress(&m_strm);
 509         };
 510
 511         virtual size_t get_default_input_size() override { return 0x10000; }
 512 };
 513
 514 static class FSBz2 : public FSCompressStream
 515 {
 516 public:
 517         FSBz2() { m_ext = ".BZ2"; };
 518         virtual bool seekable(const string& backfile) override;
 519         virtual std::shared_ptr<CommonStream> create_stream() override { return std::make_shared<Bz2stream>(); }
 520         virtual std::shared_ptr<FragmentBlock> ScanCompressblock(const string& backfile, size_t& input_offset, size_t& output_offset) override;
 521
 522 }g_fsbz2;
 523
 524 class Gzstream : public CommonStream
 525 {
 526         z_stream m_strm;
 527         size_t m_in_size = 0;
 528         size_t m_out_size = 0;
 529
 530 public:
 531         Gzstream()
 532         {
 533                 memset(&m_strm, 0, sizeof(m_strm));
 534                 inflateInit2(&m_strm, 15 + 16);
 535         }
 536         virtual ~Gzstream()
 537         {
 538                 deflateEnd(&m_strm);
 539         }
 540         virtual int set_input_buff(void* p, size_t sz) override
 541         {
 542                 m_strm.next_in = (Bytef*)p;
 543                 m_strm.avail_in = m_in_size = sz;
 544                 return 0;
 545         };
 546         virtual int set_output_buff(void* p, size_t sz) override
 547         {
 548                 m_strm.next_out = (Bytef*)p;
 549                 m_strm.avail_out = m_out_size = sz;
 550                 return 0;
 551         };
 552         virtual size_t get_input_pos() override
 553         {
 554                 return m_in_size - m_strm.avail_in;
 555         };
 556         virtual size_t get_output_pos() override
 557         {
 558                 return m_out_size - m_strm.avail_out;
 559         };
 560         virtual int decompress() override
 561         {
 562                 return inflate(&m_strm, Z_SYNC_FLUSH);
 563         };
 564
 565         virtual size_t get_default_input_size() override { return 0x10000; }
 566 };
 567
 568 static class FSGz : public FSCompressStream
 569 {
 570 public:
 571         FSGz() { m_ext = ".GZ"; };
 572         virtual std::shared_ptr<CommonStream> create_stream() { return std::make_shared<Gzstream>(); }
 573 }g_fsgz;
 574
 575 class ZstdStream:public CommonStream
 576 {
 577         ZSTD_DCtx* m_dctx;
 578         ZSTD_outBuffer m_output = { 0, 0, 0 };
 579         ZSTD_inBuffer m_input = { 0, 0, 0 };
 580
 581 public:
 582         virtual int set_input_buff(void* p, size_t sz) override
 583         {
 584                 m_input.src = p;
 585                 m_input.pos = 0;
 586                 m_input.size = sz;
 587                 return 0;
 588         };
 589         virtual int set_output_buff(void* p, size_t sz) override
 590         {
 591                 m_output.dst = p;
 592                 m_output.pos = 0;
 593                 m_output.size = sz;
 594                 return 0;
 595         };
 596         virtual size_t get_input_pos() override
 597         {
 598                 return m_input.pos;
 599         };
 600         virtual size_t get_output_pos() override
 601         {
 602                 return m_output.pos;
 603         };
 604
 605         virtual int decompress() override
 606         {
 607                 return ZSTD_decompressStream(m_dctx, &m_output, &m_input);
 608         };
 609
 610         virtual size_t get_default_input_size() override
 611         {
 612                 return ZSTD_DStreamInSize();
 613         }
 614
 615         size_t decompress_size(const string& backfile) override
 616         {
 617                 shared_ptr<FileBuffer> inp = get_file_buffer(backfile, true);
 618                 if (inp == nullptr)
 619                 {
 620                         return 0;
 621                 }
 622                 size_t sz = ZSTD_DStreamInSize();
 623                 shared_ptr<DataBuffer> pb = inp->request_data(0, sz);
 624                 if (!pb)
 625                         return 0;
 626                 size_t decompressed_sz = ZSTD_getFrameContentSize(pb->data(), sz);
 627
 628                 return decompressed_sz;
 629         }
 630         ZstdStream()
 631         {
 632                 m_dctx = ZSTD_createDCtx();
 633         };
 634         virtual ~ZstdStream()
 635         {
 636                 ZSTD_freeDCtx(m_dctx);
 637         }
 638 };
 639
 640 static class FSzstd : public FSCompressStream
 641 {
 642 public:
 643         FSzstd() { m_ext = ".ZST"; };
 644         virtual std::shared_ptr<CommonStream> create_stream() { return make_shared<ZstdStream>(); };
 645 }g_FSzstd;
 646
 647 static class FS_DATA
 648 {
 649 public:
 650         vector<FSBasic *> m_pFs;
 651         FS_DATA()
 652         {
 653                 m_pFs.push_back(&g_fsflat);
 654                 m_pFs.push_back(&g_fszip);
 655                 m_pFs.push_back(&g_fstar);
 656                 m_pFs.push_back(&g_fsbz2);
 657                 m_pFs.push_back(&g_fsfat);
 658                 m_pFs.push_back(&g_fsgz);
 659                 m_pFs.push_back(&g_FSzstd);
 660                 m_pFs.push_back(&g_fshttps);
 661                 m_pFs.push_back(&g_fshttp);
 662         }
 663
 664         int get_file_timesample(const string &filename, uint64_t *ptimesample)
 665         {
 666                 if (ptimesample == nullptr)
 667                 {
 668                         set_last_err_string("ptimesample is null\n");
 669                         return -1;
 670                 }
 671
 672                 for (size_t i = 0; i < m_pFs.size(); i++)
 673                 {
 674                         if (!m_pFs[i]->get_file_timesample(filename, ptimesample))
 675                                 return 0;
 676                 }
 677
 678                 return -1;
 679         }
 680
 681         int for_each_ls(uuu_ls_file fn, string path, void *p)
 682         {
 683                 for (int i = m_pFs.size() -1; i >= 0; i--)
 684                 {
 685                         string back, filename;
 686                         if (m_pFs[i]->split(path, &back, &filename, true) == 0)
 687                                 if(m_pFs[i]->for_each_ls(fn, back, filename, p)==0)
 688                                 {
 689                                         return 0;
 690                                 }
 691         }
 692                 return 0;
 693         }
 694
 695         bool exist(const string &filename)
 696         {
 697                 for (size_t i = 0; i < m_pFs.size(); i++)
 698                 {
 699                         string back, fn;
 700                         if (m_pFs[i]->split(filename, &back, &fn) == 0)
 701                                 if (m_pFs[i]->exist(back, fn))
 702                                         return true;
 703                 }
 704                 return false;
 705         }
 706
 707         bool need_small_mem(const string& filename)
 708         {
 709                 for (size_t i = 0; i < m_pFs.size(); i++)
 710                 {
 711                         string back, fn;
 712                         if (m_pFs[i]->split(filename, &back, &fn) == 0)
 713                                 if (m_pFs[i]->exist(back, fn))
 714                                         return m_pFs[i]->m_small_pool;
 715                 }
 716                 return false;
 717         }
 718         int load(const string &filename, shared_ptr<FileBuffer> p)
 719         {
 720                 for (size_t i = 0; i < m_pFs.size(); i++)
 721                 {
 722                         string back, fn;
 723                         if (m_pFs[i]->split(filename, &back, &fn) == 0) {
 724                                 if (m_pFs[i]->load(back, fn, p) == 0)
 725                                         return 0;
 726                         }
 727                 }
 728
 729                 string err;
 730                 err = "fail open file: ";
 731                 err += filename;
 732                 set_last_err_string(err);
 733                 return -1;
 734         }
 735 }g_fs_data;
 736
 737 int FSBackFile::get_file_timesample(const string &filename, uint64_t *ptime)
 738 {
 739         string back, file;
 740         if (split(filename, &back, &file))
 741                 return -1;
 742
 743         return g_fs_data.get_file_timesample(back, ptime);
 744 }
 745
 746 bool FSZip::exist(const string &backfile, const string &filename)
 747 {
 748         Zip zip;
 749         if (zip.Open(backfile))
 750                 return false;
 751
 752         return zip.check_file_exist(filename);
 753 }
 754
 755 int FSZip::for_each_ls(uuu_ls_file fn, const string &backfile, const string &filename, void *p)
 756 {
 757         Zip zip;
 758
 759         if (zip.Open(backfile))
 760                 return -1;
 761
 762         for(auto it = zip.m_filemap.begin(); it!=zip.m_filemap.end(); ++it)
 763         {
 764                 if(it->first.substr(0, filename.size()) == filename || filename.empty())
 765                 {
 766                         string name = backfile;
 767                         name += "/";
 768                         name += it->first;
 769                         fn(name.c_str()+1, p);
 770                 }
 771         }
 772
 773         return 0;
 774 }
 775
 776 int zip_async_load(string zipfile, string fn, shared_ptr<FileBuffer> buff)
 777 {
 778         std::lock_guard<mutex> lock(buff->m_async_mutex);
 779
 780         Zip zip;
 781         if (zip.Open(zipfile))
 782                 return -1;
 783
 784         if(zip.get_file_buff(fn, buff))
 785                 return -1;
 786
 787         buff->m_available_size = buff->m_DataSize;
 788         atomic_fetch_or(&buff->m_dataflags, FILEBUFFER_FLAG_LOADED);
 789
 790         buff->m_request_cv.notify_all();
 791         return 0;
 792 }
 793
 794 int FSZip::load(const string &backfile, const string &filename, shared_ptr<FileBuffer> p)
 795 {
 796         Zip zip;
 797
 798         if (zip.Open(backfile))
 799                 return -1;
 800
 801         if (!zip.check_file_exist(filename))
 802                 return -1;
 803
 804         if(zip.get_file_buff(filename, p))
 805                 return -1;
 806
 807         atomic_fetch_or(&p->m_dataflags, FILEBUFFER_FLAG_LOADED);
 808         p->m_request_cv.notify_all();
 809
 810         return 0;
 811 }
 812
 813 bool FSTar::exist(const string &backfile, const string &filename)
 814 {
 815         Tar tar;
 816         if (tar.Open(backfile))
 817                 return false;
 818
 819         return tar.check_file_exist(filename);
 820 }
 821
 822
 823 int FSTar::for_each_ls(uuu_ls_file fn, const string &backfile, const string &filename, void *p)
 824 {
 825         Tar tar;
 826
 827         if (tar.Open(backfile))
 828                 return -1;
 829
 830         for(auto it = tar.m_filemap.begin(); it!=tar.m_filemap.end(); ++it)
 831         {
 832                 if(it->first.substr(0, filename.size()) == filename || filename.empty())
 833                 {
 834                         string name = backfile;
 835                         name += "/";
 836                         name += it->first;
 837                         fn(name.c_str()+1, p);
 838                 }
 839         }
 840
 841         return 0;
 842 }
 843
 844 int FSTar::load(const string &backfile, const string &filename, shared_ptr<FileBuffer> p)
 845 {
 846         Tar tar;
 847         if (tar.Open(backfile))
 848                 return -1;
 849
 850         if (!tar.check_file_exist(filename))
 851                 return -1;
 852
 853         if(tar.get_file_buff(filename, p))
 854                 return -1;
 855         p->m_available_size = p->m_DataSize;
 856         atomic_fetch_or(&p->m_dataflags, FILEBUFFER_FLAG_LOADED);
 857         p->m_request_cv.notify_all();
 858         return 0;
 859 }
 860
 861 bool FSFat::exist(const string &backfile, const string &filename)
 862 {
 863         Fat fat;
 864         if (fat.Open(backfile))
 865         {
 866                 return false;
 867         }
 868         return fat.m_filemap.find(filename) != fat.m_filemap.end();
 869 }
 870
 871 int FSFat::load(const string &backfile, const string &filename, shared_ptr<FileBuffer> p)
 872 {
 873         Fat fat;
 874         if (fat.Open(backfile))
 875         {
 876                 return -1;
 877         }
 878
 879         if(fat.get_file_buff(filename, p))
 880                 return -1;
 881
 882         atomic_fetch_or(&p->m_dataflags, FILEBUFFER_FLAG_LOADED);
 883         p->m_request_cv.notify_all();
 884
 885         return 0;
 886 }
 887
 888 int FSFat::for_each_ls(uuu_ls_file fn, const string &backfile, const string &filename, void *p)
 889 {
 890         Fat fat;
 891         if (fat.Open(backfile))
 892         {
 893                 return -1;
 894         }
 895
 896         for(auto it = fat.m_filemap.begin(); it != fat.m_filemap.end(); ++it)
 897         {
 898                 if(it->first.substr(0, filename.size()) == filename || filename.empty())
 899                 {
 900                         string name = backfile;
 901                         name += "/";
 902                         name += it->first;
 903                         fn(name.c_str()+1, p);
 904                 }
 905         }
 906         return 0;
 907 }
 908
 909 bool FSCompressStream::exist(const string &backfile, const string &filename)
 910 {
 911
 912         if (!g_fs_data.exist(backfile))
 913                 return false;
 914
 915         if (filename == "*")
 916                 return true;
 917
 918         return false;
 919 }
 920
 921 int FSCompressStream::for_each_ls(uuu_ls_file fn, const string &backfile, const string &/*filename*/, void *p)
 922 {
 923
 924         if(!g_fs_data.exist(backfile))
 925                 return -1;
 926
 927         string str;
 928         str = backfile + "/*";
 929
 930         fn(str.c_str() + 1, p);
 931         return 0;
 932 }
 933
 934 class Bz2FragmentBlock: public FragmentBlock
 935 {
 936 public:
 937         virtual ~Bz2FragmentBlock() {}
 938         int DataConvert() override
 939         {
 940                 std::lock_guard<mutex> lock(m_mutex);
 941
 942                 m_actual_size = m_output_size;
 943                 m_data.resize(m_output_size);
 944
 945                 shared_ptr<DataBuffer> input = m_input->request_data(m_input_offset, m_input_sz);
 946                 if (!input)
 947                         return -1;
 948                 unsigned int len = m_output_size;
 949                 m_ret = BZ2_bzBuffToBuffDecompress((char*)m_data.data(),
 950                         &len,
 951                         (char*)input->data(),
 952                         m_input_sz,
 953                         0,
 954                         0);
 955
 956                 m_actual_size = len;
 957                 m_data.resize(m_actual_size);
 958
 959                 assert(m_output_size >= m_actual_size);
 960
 961                 atomic_fetch_or(&m_dataflags, (int)CONVERT_DONE);
 962                 return m_ret;
 963         }
 964 };
 965
 966 shared_ptr<FragmentBlock> FSBz2::ScanCompressblock(const string& backfile, size_t& input_offset, size_t& output_offset)
 967 {
 968         shared_ptr<FileBuffer> pbz;
 969
 970         pbz = get_file_buffer(backfile, true);
 971         if (pbz == nullptr) {
 972                 return NULL;
 973         }
 974
 975         size_t request_size = 1 * 1000 * 1000;
 976         shared_ptr<DataBuffer> pd = pbz->request_data(input_offset, request_size);
 977         if (!pd)
 978                 return NULL;
 979
 980         uint8_t* p1 = pd->data();
 981
 982         size_t sz = min(request_size - 10, pd->size());
 983
 984         for (size_t i = 0; i < sz; i++)
 985         {
 986                 uint16_t* header = (uint16_t*)p1++;
 987                 if (*header == 0x5a42) //"BZ"
 988                 {
 989                         uint32_t* magic1 = (uint32_t*)&pd->at(i + 4);
 990                         if (*magic1 == 0x26594131 && pd->at(i + 2) == 'h') //PI 3.1415926
 991                         {
 992                                 uint16_t* magic2 = (uint16_t*)&pd->at(i + 8);
 993                                 if (*magic2 == 0x5953)
 994                                 {
 995                                         shared_ptr<FragmentBlock> p = shared_ptr<FragmentBlock>(new Bz2FragmentBlock);
 996
 997                                         p->m_input = pbz;
 998                                         p->m_actual_size = 0;
 999                                         p->m_dataflags = 0;
1000                                         p->m_input_offset = input_offset + i;
1001                                         p->m_output_offset = output_offset;
1002                                         p->m_output_size = (pd->at(i + 3) - '0') * 100 * 1000; /* not l024 for bz2 */
1003                                         p->m_input_sz = request_size;
1004
1005                                         input_offset += i + 8;
1006
1007                                         output_offset += p->m_output_size;
1008                                         return p;
1009                                 }
1010                         }
1011                 }
1012         }
1013
1014         return NULL;
1015 }
1016
1017 bool FSBz2::seekable(const string& backfile)
1018 {
1019         shared_ptr<FileBuffer> file = get_file_buffer(backfile, true);
1020         shared_ptr<DataBuffer> p = file->request_data(0, 1024*1024);
1021
1022         if (!p)
1023                 return false;
1024
1025         int header_num = 0;
1026         uint8_t* ptr = p->data();
1027
1028         for (size_t i = 0; i < p->size(); i++)
1029         {
1030                 if (ptr[0] == 'B' && ptr[1] == 'Z' && ptr[2] == 'h' && ptr[4] == '1' && ptr[5] == 'A' && ptr[6] == 'Y' && ptr[7] == '&' && ptr[8] == 'S' && ptr[9] == 'Y')
1031                 {
1032                         header_num++;
1033                 }
1034                 ptr++;
1035
1036                 if (header_num > 1)
1037                         return true;
1038         }
1039
1040         return false;
1041 }
1042
1043 int FSCompressStream::Decompress(const string& backfile, shared_ptr<FileBuffer>outp)
1044 {
1045         shared_ptr<CommonStream> cs = create_stream();
1046         if (!cs)
1047                 return -1;
1048
1049         ssize_t lastRet = 0;
1050         size_t outOffset = 0;
1051         shared_ptr<FileBuffer> inp = get_file_buffer(backfile, true);
1052         if (inp == nullptr)
1053         {
1054                 return -1;
1055         }
1056
1057         size_t sz = cs->decompress_size(backfile);
1058         if (sz)
1059         {
1060                 outp->resize(sz);
1061                 atomic_fetch_or(&outp->m_dataflags, FILEBUFFER_FLAG_KNOWN_SIZE);
1062         }
1063
1064         std::shared_ptr<DataBuffer> buff;
1065         buff = inp->request_data(0, 0x1000);
1066         if (!buff)
1067                 return -1;
1068
1069         size_t offset = 0;
1070         uuu_notify ut;
1071         ut.type = uuu_notify::NOTIFY_DECOMPRESS_START;
1072         ut.str = (char*)backfile.c_str();
1073         call_notify(ut);
1074
1075         shared_ptr<FragmentBlock> blk;
1076         blk = outp->get_map_it(0, true);
1077         if (!blk)
1078                 blk = outp->request_new_blk();
1079
1080         {
1081                 lock_guard<mutex> l(outp->m_seg_map_mutex);
1082                 outp->m_last_db = blk;
1083         }
1084
1085         cs->set_output_buff(blk->data(), blk->m_output_size);
1086
1087         while ((buff = inp->request_data(offset, cs->get_default_input_size())))
1088         {
1089                 if (!buff)
1090                         return -1;
1091
1092                 //ZSTD_inBuffer input = { buff->data(), buff->size(), 0 };
1093                 cs->set_input_buff(buff->data(), buff->size());
1094                 /* Given a valid frame, zstd won't consume the last byte of the frame
1095                  * until it has flushed all of the decompressed data of the frame.
1096                  * Therefore, instead of checking if the return code is 0, we can
1097                  * decompress just check if input.pos < input.size.
1098                  */
1099                 while (cs->get_input_pos() < min(buff->size(), inp->m_DataSize - offset))
1100                 {
1101                         /* The return code is zero if the frame is complete, but there may
1102                          * be multiple frames concatenated together. Zstd will automatically
1103                          * reset the context when a frame is complete. Still, calling
1104                          * ZSTD_DCtx_reset() can be useful to reset the context to a clean
1105                          * state, for instance if the last decompression call returned an
1106                          * error.
1107                          */
1108                         size_t old = cs->get_output_pos();
1109                         ssize_t const ret = cs->decompress();  //ZSTD_decompressStream(dctx, &output, &input);
1110                         lastRet = min(lastRet, ret);
1111                         outOffset += cs->get_output_pos() - old;
1112
1113                         if (ret < 0) {
1114                                 blk->m_ret = ret;
1115                                 set_last_err_string("decompress error");
1116                                 outp->m_request_cv.notify_all();
1117                                 return -1;
1118                         }
1119
1120                         blk->m_ret = 0;
1121                         blk->m_actual_size = cs->get_output_pos();
1122
1123                         atomic_fetch_or(&blk->m_dataflags, (int)FragmentBlock::CONVERT_PARTIAL);
1124
1125                         ut.type = uuu_notify::NOTIFY_DECOMPRESS_POS;
1126                         ut.index = outOffset;
1127                         call_notify(ut);
1128                         outp->m_available_size = outOffset;
1129                         outp->m_request_cv.notify_all();
1130
1131
1132                         if (cs->get_output_pos() == blk->m_output_size)
1133                         {
1134                                 atomic_fetch_or(&blk->m_dataflags, (int)FragmentBlock::CONVERT_DONE);
1135                                 if (!(cs->get_input_pos() == buff->size() &&
1136                                         buff->size() == (inp->size() - offset)))
1137                                 {
1138                                         blk = outp->get_map_it(outOffset, true);
1139                                         if(!blk)
1140                                                 blk = outp->request_new_blk();
1141                                         cs->set_output_buff(blk->data(), blk->m_output_size);
1142
1143                                         {
1144                                                 lock_guard<mutex> l(outp->m_seg_map_mutex);
1145                                                 outp->m_last_db = blk;
1146                                         }
1147                                 }
1148                         }
1149
1150                         if (outp->m_reset_stream)
1151                         {
1152                                 outp->m_reset_stream = false;
1153                                 return -1;
1154                         }
1155                 }
1156
1157                 offset += cs->get_default_input_size();
1158         }
1159         outp->resize(outOffset);
1160
1161         atomic_fetch_or(&blk->m_dataflags, (int)FragmentBlock::CONVERT_DONE);
1162         atomic_fetch_or(&outp->m_dataflags, FILEBUFFER_FLAG_LOADED);
1163
1164         outp->m_request_cv.notify_all();
1165         if (lastRet < 0)
1166                 return -1;
1167         return 0;
1168 }
1169
1170 uint64_t get_file_timesample(string filename)
1171 {
1172         uint64_t time=0;
1173         g_fs_data.get_file_timesample(filename, &time);
1174         return time;
1175 }
1176
1177 shared_ptr<FileBuffer> get_file_buffer(string filename, bool async)
1178 {
1179         filename = remove_quota(filename);
1180
1181         if (!filename.empty() && filename[0] != MAGIC_PATH)
1182         {
1183                 if (filename == "..")
1184                         filename = g_current_dir.substr(0, g_current_dir.size() - 1);
1185                 else
1186                         filename = g_current_dir + filename;
1187         }
1188
1189         string_ex path;
1190         path += filename;
1191
1192         path.replace('\\', '/');
1193
1194         filename = path;
1195
1196         bool find;
1197         {
1198                 std::lock_guard<mutex> lock(g_mutex_map);
1199                 find = (g_filebuffer_map.find(filename) == g_filebuffer_map.end());
1200         }
1201
1202         if (find)
1203         {
1204                 shared_ptr<FileBuffer> p(new FileBuffer);
1205
1206                 if (p->reload(filename, async))
1207                         return nullptr;
1208
1209                 {
1210                         std::lock_guard<mutex> lock(g_mutex_map);
1211                         g_filebuffer_map[filename] = p;
1212                 }
1213                 return p;
1214         }
1215         else
1216         {
1217                 shared_ptr<FileBuffer> p;
1218                 {
1219                         std::lock_guard<mutex> lock(g_mutex_map);
1220                         p= g_filebuffer_map[filename];
1221                 }
1222                 if (p->m_timesample != get_file_timesample(filename))
1223                         if (p->reload(filename, async))
1224                         {
1225                                 return nullptr;
1226                         }
1227
1228                 if (!p->IsLoaded() && !async)
1229                 {
1230                         std::lock_guard<mutex> lock(p->m_async_mutex);
1231
1232                         if(p->m_async_thread.joinable())
1233                                 p->m_async_thread.join();
1234
1235                         if(!p->IsLoaded())
1236                         {
1237                                 return nullptr;
1238                         }
1239                 }
1240
1241                 return p;
1242         }
1243 }
1244
1245 FileBuffer::FileBuffer()
1246 {
1247         m_pDatabuffer = nullptr;
1248         m_DataSize = 0;
1249         m_MemSize = 0;
1250         m_dataflags = 0;
1251         m_available_size = 0;
1252 }
1253
1254 FileBuffer::FileBuffer(void *p, size_t sz)
1255 {
1256         m_pDatabuffer = nullptr;
1257         m_DataSize = 0;
1258         m_MemSize = 0;
1259
1260         m_pDatabuffer = (uint8_t*)malloc(sz);
1261         m_MemSize = m_DataSize = sz;
1262
1263         memcpy(m_pDatabuffer, p, sz);
1264         m_dataflags = 0;
1265
1266         atomic_fetch_or(&m_dataflags, FILEBUFFER_FLAG_LOADED);
1267 }
1268
1269 FileBuffer::~FileBuffer()
1270 {
1271         m_reset_stream = true;
1272
1273         if(m_async_thread.joinable())
1274                 m_async_thread.join();
1275
1276         if (m_pDatabuffer)
1277         {
1278                 if(m_allocate_way == ALLOCATION_WAYS::MMAP)
1279                         unmapfile();
1280                 if(m_allocate_way == ALLOCATION_WAYS::MALLOC)
1281                         free(m_pDatabuffer);
1282         }
1283 }
1284
1285 int FileBuffer::mapfile(const string &filename, size_t sz)
1286 {
1287 #ifdef _MSC_VER
1288
1289                 m_Request.StructureVersion = REQUEST_OPLOCK_CURRENT_VERSION;
1290                 m_Request.StructureLength = sizeof(REQUEST_OPLOCK_INPUT_BUFFER);
1291                 m_Request.RequestedOplockLevel = (OPLOCK_LEVEL_CACHE_READ | OPLOCK_LEVEL_CACHE_HANDLE);
1292                 m_Request.Flags = REQUEST_OPLOCK_INPUT_FLAG_REQUEST;
1293
1294                 REQUEST_OPLOCK_OUTPUT_BUFFER Response;
1295
1296                 m_OverLapped.hEvent = CreateEvent(nullptr, TRUE, FALSE, nullptr);
1297                 ResetEvent(m_OverLapped.hEvent);
1298
1299                 m_file_handle = CreateFile(filename.c_str(),
1300                         GENERIC_READ,
1301                         FILE_SHARE_READ | FILE_SHARE_WRITE,
1302                         nullptr,
1303                         OPEN_EXISTING,
1304                         FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS | FILE_FLAG_OVERLAPPED,
1305                         nullptr);
1306
1307                 if (m_file_handle == INVALID_HANDLE_VALUE)
1308                 {
1309                         string err = "Create File Failure ";
1310                         err += filename;
1311                         set_last_err_string(err);
1312                         return -1;
1313                 }
1314
1315                 BOOL bSuccess = DeviceIoControl(m_file_handle,
1316                         FSCTL_REQUEST_OPLOCK,
1317                         &m_Request,
1318                         sizeof(m_Request),
1319                         &Response,
1320                         sizeof(Response),
1321                         nullptr,
1322                         &m_OverLapped);
1323
1324                 if (bSuccess || GetLastError() == ERROR_IO_PENDING)
1325                 {
1326                         m_file_monitor = thread(file_overwrite_monitor, filename, this);
1327                 }
1328
1329                 m_file_map = CreateFileMapping(m_file_handle,
1330                         nullptr, PAGE_READONLY, 0, 0, nullptr);
1331
1332                 if (m_file_map == INVALID_HANDLE_VALUE)
1333                 {
1334                         set_last_err_string("Fail create Map");
1335                         return -1;
1336                 }
1337
1338                 m_pDatabuffer = (uint8_t *)MapViewOfFile(m_file_map, FILE_MAP_READ, 0, 0, sz);
1339                 m_DataSize = sz;
1340                 m_MemSize = sz;
1341                 m_allocate_way = ALLOCATION_WAYS::MMAP;
1342
1343 #else
1344                 int fd = open(filename.c_str(), O_RDONLY);
1345                 if (fd == -1)
1346                 {
1347                         string err;
1348                         err += "xx Failure open file: ";
1349                         err + filename;
1350                         set_last_err_string(err);
1351                         return -1;
1352                 }
1353
1354                 m_pDatabuffer = (uint8_t *)mmap64(0, sz, PROT_READ, MAP_SHARED, fd, 0);
1355                 if (m_pDatabuffer == MAP_FAILED) {
1356                         m_pDatabuffer = nullptr;
1357                         set_last_err_string("mmap failure\n");
1358                         return -1;
1359                 }
1360                 m_DataSize = sz;
1361                 m_MemSize = sz;
1362                 m_allocate_way = ALLOCATION_WAYS::MMAP;
1363                 close(fd);
1364 #endif
1365
1366                 if (m_pDatabuffer)
1367                         return 0;
1368
1369                 set_last_err_string("mmap file failure");
1370                 return -1;
1371 }
1372
1373 int FileBuffer::ref_other_buffer(shared_ptr<FileBuffer> p, size_t offset, size_t size)
1374 {
1375         m_pDatabuffer = p->data() + offset;
1376         m_DataSize = m_MemSize = size;
1377         m_available_size = m_DataSize;
1378         m_allocate_way = ALLOCATION_WAYS::REF;
1379         m_ref = p;
1380
1381         atomic_fetch_or(&m_dataflags, FILEBUFFER_FLAG_LOADED);
1382         return 0;
1383 }
1384
1385 int FileBuffer::reload(string filename, bool async)
1386 {
1387         if(async) {
1388                 if(!g_fs_data.exist(filename))
1389                         return - 1;
1390
1391                 if (g_fs_data.need_small_mem(filename))
1392                         m_allocate_way = ALLOCATION_WAYS::SEGMENT;
1393
1394                 m_async_thread = thread(&FS_DATA::load, &g_fs_data, filename, shared_from_this());
1395         }
1396         else
1397         {
1398                 if(g_fs_data.load(filename, shared_from_this()))
1399                         return - 1;
1400         }
1401         m_timesample = get_file_timesample(filename);
1402         m_filename = filename;
1403
1404         return 0;
1405 }
1406
1407 int FileBuffer::request_data(std::vector<uint8_t> &data, size_t offset, size_t sz)
1408 {
1409         int64_t ret;
1410         ret = request_data(data.data(), offset, sz);
1411         if (ret < 0)
1412         {
1413                 data.clear();
1414                 return -1;
1415         }
1416
1417         data.resize(ret);
1418         return 0;
1419 }
1420
1421 void FileBuffer::truncate_old_data_in_pool()
1422 {
1423         if (!g_small_memory)
1424                 return;
1425
1426
1427         std::unique_lock<std::mutex> lock(this->m_seg_map_mutex);
1428
1429         if (m_last_request_offset < m_total_buffer_size/2)
1430                 return;
1431
1432         size_t off = m_last_request_offset - m_total_buffer_size/2;
1433
1434         for (auto it= m_seg_map.lower_bound(off); it != m_seg_map.end(); it++)
1435         {
1436                 auto blk = it->second;
1437                 std::unique_lock<std::mutex> lock(blk->m_mutex);
1438
1439                 if ((blk->m_dataflags & FragmentBlock::CONVERT_DONE)
1440                         /* && !(blk->m_dataflags & FragmentBlock::USING)*/
1441                         )
1442                 {
1443                         blk->m_dataflags = 0;
1444                         blk->m_actual_size = 0;
1445                         vector<uint8_t> v;
1446                         blk->m_data.swap(v);
1447                 }
1448         }
1449 }
1450
1451 int64_t FileBuffer::request_data_from_segment(void *data, size_t offset, size_t sz)
1452 {
1453         size_t return_sz = 0;
1454
1455         do
1456         {
1457                 m_last_request_offset = offset;
1458                 std::unique_lock<std::mutex> lck(m_request_cv_mutex);
1459
1460                 shared_ptr<FragmentBlock> blk;
1461
1462                 m_pool_load_cv.notify_all();
1463
1464                 while (!(blk = get_map_it(offset)))
1465                 {
1466                         if (IsKnownSize())
1467                         {
1468                                 if(offset >= this->m_DataSize)
1469                                         return -1;
1470                         }
1471                         auto now = std::chrono::system_clock::now();
1472                         m_request_cv.wait_until(lck, now + 500ms);
1473                 }
1474                 do
1475                 {
1476                         shared_ptr<FragmentBlock> last_decompress_db;
1477                         {
1478                                 std::unique_lock<std::mutex> lock(m_seg_map_mutex);
1479                                 last_decompress_db = m_last_db;
1480                         }
1481
1482                         {   /*lock hold*/
1483                                 std::unique_lock<std::mutex> lock(blk->m_mutex);
1484
1485                                 if (blk->m_actual_size >= (offset + sz - blk->m_output_offset))
1486                                         break;
1487
1488                                 if (!(m_dataflags & FILEBUFFER_FLAG_PARTIAL_RELOADABLE))
1489                                 {
1490                                         if (last_decompress_db)
1491                                         {
1492                                                 if (offset < last_decompress_db->m_output_offset && !(blk->m_dataflags & FragmentBlock::CONVERT_DONE))
1493                                                 {
1494                                                         m_reset_stream = true;
1495                                                         break;
1496                                                 }
1497                                         }
1498                                 }
1499
1500                                 if (blk->m_ret)
1501                                         return -1;
1502
1503                                 if ((blk->m_dataflags & FragmentBlock::CONVERT_DONE))
1504                                 {
1505                                         atomic_fetch_or(&blk->m_dataflags, (int)FragmentBlock::USING);
1506                                         break;
1507                                 }
1508                         }
1509                         auto now = std::chrono::system_clock::now();
1510                         m_request_cv.wait_until(lck, now + 500ms);
1511                 } while (1);
1512
1513                 if (m_reset_stream)
1514                 {
1515                         m_dataflags = 0;
1516                         m_available_size = 0;
1517                         this->m_async_thread.join();
1518                         m_reset_stream = false;
1519
1520                         this->reload(m_filename, true);
1521                         continue;
1522                 }
1523
1524                 {   /*hold lock*/
1525                         std::unique_lock<std::mutex> lock(blk->m_mutex);
1526
1527                         size_t off = offset - blk->m_output_offset;
1528
1529                         assert(offset >= blk->m_output_offset);
1530
1531                         size_t item_sz = blk->m_actual_size - off;
1532
1533                         if (off > blk->m_actual_size)
1534                                 return -1;
1535
1536                         if (item_sz >= sz)
1537                         {
1538                                 memcpy(data, blk->data() + off, sz);
1539                                 atomic_fetch_and(&blk->m_dataflags, ~FragmentBlock::USING);
1540
1541                                 return_sz += sz;
1542                                 return return_sz;
1543                         }
1544                         else if (item_sz == 0)
1545                         {
1546                                 return return_sz;
1547                         }
1548                         else
1549                         {
1550                                 memcpy(data, blk->m_data.data() + off, item_sz);
1551                                 data = ((uint8_t*)data) + item_sz;
1552                                 sz -= item_sz;
1553                                 offset += item_sz;
1554
1555                                 return_sz += item_sz;
1556                         }
1557                 }
1558         } while (1);
1559
1560         return -1;
1561 }
1562
1563 int64_t FileBuffer::request_data(void *data, size_t offset, size_t sz)
1564 {
1565         bool needlock = false;
1566         int ret = 0;
1567
1568         if (IsLoaded())
1569         {
1570                 if (offset >= this->size())
1571                 {
1572                         set_last_err_string("request offset exceed memory size");
1573                         return -1;
1574                 }
1575
1576                 if (this->m_allocate_way == FileBuffer::ALLOCATION_WAYS::SEGMENT)
1577                         return request_data_from_segment(data, offset, sz);
1578
1579         }
1580         else
1581         {
1582                 if (this->m_allocate_way == FileBuffer::ALLOCATION_WAYS::SEGMENT)
1583                         return request_data_from_segment(data, offset, sz);
1584
1585                 std::unique_lock<std::mutex> lck(m_request_cv_mutex);
1586                 while ((offset + sz > m_available_size) && !IsLoaded())
1587                 {
1588                         if (IsError())
1589                         {
1590                                 set_last_err_string("Async request data error");
1591                                 return -1;
1592                         }
1593                         m_request_cv.wait(lck);
1594                 }
1595
1596                 if (IsLoaded())
1597                 {
1598                         if (offset > m_available_size)
1599                         {
1600                                 set_last_err_string("request offset execeed memory size");
1601                                 return -1;
1602                         }
1603                 }
1604                 needlock = true;
1605         }
1606
1607         size_t size = sz;
1608         if (offset + size >= m_available_size)
1609                 size = m_available_size - offset;
1610
1611         if (needlock) m_data_mutex.lock();
1612
1613         if (this->data())
1614         {
1615                 memcpy(data, this->data() + offset, size);
1616                 ret = size;
1617         }
1618         else
1619         {
1620                 set_last_err_string("Out of memory");
1621                 ret = ERR_OUT_MEMORY;
1622         }
1623         if (needlock) m_data_mutex.unlock();
1624
1625         return ret;
1626 }
1627
1628 std::shared_ptr<FragmentBlock> FileBuffer::request_new_blk()
1629 {
1630         if (m_allocate_way == ALLOCATION_WAYS::SEGMENT)
1631         {
1632                 if (m_seg_map.empty())
1633                 {
1634                         std::shared_ptr<FragmentBlock> p(new FragmentBlock);
1635                         lock_guard<mutex> lock(m_seg_map_mutex);
1636                         p->m_output_size = m_seg_blk_size;
1637                         p->m_data.resize(m_seg_blk_size);
1638                         m_seg_map[0] = p;
1639                         return p;
1640                 }
1641
1642                 size_t offset;
1643
1644                 if (g_small_memory)
1645                 {
1646                         truncate_old_data_in_pool();
1647
1648                         {
1649                                 lock_guard<mutex> lock(m_seg_map_mutex);
1650                                 shared_ptr <FragmentBlock> p = m_seg_map.begin()->second;
1651                                 offset = p->m_output_offset;
1652                         }
1653
1654                         while (offset > m_last_request_offset + m_total_buffer_size)
1655                         {
1656                                 if (m_reset_stream)
1657                                         return NULL;
1658                                 std::unique_lock<std::mutex> lck(m_pool_load_cv_mutex);
1659                                 m_pool_load_cv.wait(lck);
1660                         }
1661                 }
1662
1663                 {
1664                         lock_guard<mutex> lock(m_seg_map_mutex);
1665                         shared_ptr <FragmentBlock> p = m_seg_map.begin()->second;
1666                         offset = p->m_output_offset;
1667                 }
1668
1669                 offset += m_seg_blk_size;
1670
1671                 std::shared_ptr<FragmentBlock> p(new FragmentBlock);
1672                 p->m_output_size = m_seg_blk_size;
1673                 p->m_output_offset = offset;
1674                 p->m_data.resize(m_seg_blk_size);
1675
1676                 {
1677                         lock_guard<mutex> lock(m_seg_map_mutex);
1678                         m_seg_map[offset] = p;
1679                 }
1680
1681                 return p;
1682         }
1683         else
1684         {
1685                 std::shared_ptr<FragmentBlock> p(new FragmentBlock);
1686                 p->m_pData = this->m_pDatabuffer;
1687                 p->m_output_size = this->m_MemSize;
1688                 return p;
1689         }
1690 }
1691
1692 std::shared_ptr<DataBuffer> FileBuffer::request_data(size_t offset, size_t sz)
1693 {
1694         shared_ptr<DataBuffer> p(new DataBuffer);
1695
1696         if (IsLoaded() && IsRefable())
1697         {
1698                 if (offset >= this->size())
1699                 {
1700                         set_last_err_string("request offset bigger than file size");
1701                         return nullptr;
1702                 }
1703
1704                 size_t size = sz;
1705                 if (offset + sz > this->size())
1706                         size = this->size() - offset;
1707
1708                 if (!p->ref_other_buffer(shared_from_this(), offset, size))
1709                         return p;
1710         }
1711
1712         if (sz == SIZE_MAX)
1713                 sz = size() - offset;
1714
1715         p->resize(sz);
1716         int64_t ret = request_data(p->m_pDatabuffer, offset, sz);
1717         if (ret < 0)
1718                 return nullptr;
1719
1720         p->resize(ret);
1721         return p;
1722 }
1723
1724 int FileBuffer::reserve(size_t sz)
1725 {
1726         assert(m_allocate_way == ALLOCATION_WAYS::MALLOC);
1727
1728         if (sz > m_MemSize)
1729         {
1730                 m_pDatabuffer = (uint8_t*)realloc(m_pDatabuffer, sz);
1731                 m_MemSize = sz;
1732
1733                 if (m_pDatabuffer == nullptr)
1734                 {
1735                         set_last_err_string("Out of memory\n");
1736                         return -1;
1737                 }
1738         }
1739
1740         return 0;
1741 }
1742
1743 int FileBuffer::resize(size_t sz)
1744 {
1745         if (this->m_allocate_way == ALLOCATION_WAYS::SEGMENT)
1746         {
1747                 m_DataSize = sz;
1748                 return 0;
1749         }
1750
1751         if (this->m_allocate_way == ALLOCATION_WAYS::REF)
1752         {
1753                 if (sz > m_DataSize)
1754                 {
1755                         assert(true);
1756                         return 0;
1757                 }
1758                 m_DataSize = sz;
1759                 return m_DataSize;
1760         }
1761         int ret = reserve(sz);
1762
1763         m_DataSize = sz;
1764         return ret;
1765 }
1766
1767 int FileBuffer::swap(FileBuffer &a)
1768 {
1769         std::swap(m_pDatabuffer, a.m_pDatabuffer);
1770         std::swap(m_DataSize, a.m_DataSize);
1771         std::swap(m_MemSize, a.m_MemSize);
1772         std::swap(m_allocate_way, a.m_allocate_way);
1773
1774         return 0;
1775 }
1776
1777 int FileBuffer::unmapfile()
1778 {
1779         if (m_pDatabuffer)
1780         {
1781 #ifdef _MSC_VER
1782                 UnmapViewOfFile(m_pDatabuffer);
1783                 m_pDatabuffer = nullptr;
1784                 CloseHandle(m_file_map);
1785                 CloseHandle(m_file_handle);
1786                 SetEvent(m_OverLapped.hEvent);
1787
1788                 if (m_file_monitor.joinable())
1789                         m_file_monitor.join();
1790
1791                 CloseHandle(m_OverLapped.hEvent);
1792                 m_OverLapped.hEvent = m_file_map = m_file_handle = INVALID_HANDLE_VALUE;
1793 #else
1794                 munmap(m_pDatabuffer, m_DataSize);
1795 #endif
1796                 m_pDatabuffer = nullptr;
1797         }
1798         return 0;
1799 }
1800
1801 bool check_file_exist(string filename, bool /*start_async_load*/)
1802 {
1803         string_ex fn;
1804         fn += remove_quota(filename);
1805         string_ex path;
1806         if (!fn.empty() && fn[0] != MAGIC_PATH)
1807         {
1808                 if (fn == "..")
1809                         path += g_current_dir.substr(0, g_current_dir.size() - 1);
1810                 else
1811                         path += g_current_dir + fn;
1812         }
1813         else {
1814                 path = fn;
1815         }
1816
1817         path.replace('\\', '/');
1818
1819         if (path.empty())
1820                 path += "./";
1821         return g_fs_data.exist(path);
1822 }
1823
1824 #ifdef WIN32
1825
1826 int file_overwrite_monitor(string filename, FileBuffer *p)
1827 {
1828         WaitForSingleObject(p->m_OverLapped.hEvent, INFINITE);
1829
1830         string str;
1831         str = ">";
1832         str += filename;
1833
1834         if(p->m_pDatabuffer && p->get_m_allocate_way() == FileBuffer::ALLOCATION_WAYS::MMAP)
1835         {
1836                 std::lock_guard<mutex> lock(g_mutex_map);
1837                 p->m_file_monitor.detach(); /*Detach itself, erase will delete p*/
1838                 if(g_filebuffer_map.find(str) != g_filebuffer_map.end())
1839                         g_filebuffer_map.erase(str);
1840         }
1841
1842         return 0;
1843 }
1844 #endif
1845
1846 int uuu_for_each_ls_file(uuu_ls_file fn, const char *file_path, void *p)
1847 {
1848         string_ex path;
1849         path +=">";
1850
1851         string f = file_path;
1852
1853         if(f.size() == 0)
1854         {
1855                 path += "./";
1856         }else if( f[0] == '/')
1857         {
1858                 path += "//";
1859         }else
1860         {
1861                 path += "./";
1862         }
1863
1864         path+=file_path;
1865         path.replace('\\', '/');
1866
1867         f = path;
1868         return g_fs_data.for_each_ls(fn, f, p);
1869 }
1870
1871 int FSCompressStream::load(const string& backfile, const string& filename, shared_ptr<FileBuffer>outp)
1872 {
1873         if (!g_fs_data.exist(backfile))
1874         {
1875                 string str;
1876                 str = "Failure open file:";
1877                 str += backfile;
1878                 set_last_err_string(str);
1879                 return -1;
1880         }
1881         if (filename != "*")
1882         {
1883                 string star("/*");
1884                 string decompressed_name = backfile + star;
1885                 shared_ptr<FileBuffer> decompressed_file = get_file_buffer(decompressed_name);
1886                 Tar tar;
1887                 tar.Open(decompressed_name);
1888                 if (tar.get_file_buff(filename, outp))
1889                         return -1;
1890                 outp->m_available_size = outp->m_DataSize;
1891         }
1892
1893         if (seekable(backfile))
1894         {
1895                 size_t offset = 0;
1896                 size_t decompress_off = 0;
1897                 shared_ptr<FragmentBlock> p;
1898                 size_t total_size = 0;
1899
1900                 atomic_fetch_or(&outp->m_dataflags, FILEBUFFER_FLAG_PARTIAL_RELOADABLE);
1901
1902                 int nthread = thread::hardware_concurrency();
1903
1904                 vector<thread> threads;
1905
1906                 for (int i = 0; i < nthread; i++)
1907                 {
1908                         threads.push_back(thread(&FSCompressStream::PreloadWorkThread, this, outp));
1909                 }
1910
1911                 while ((p = ScanCompressblock(backfile, offset, decompress_off)))
1912                 {
1913                         if (!p)
1914                                 return 0;
1915
1916                         {
1917                                 lock_guard<mutex> lock(outp->m_seg_map_mutex);
1918                                 outp->m_seg_map[p->m_output_offset] = p;
1919                         }
1920
1921                         outp->m_request_cv.notify_all();
1922                         outp->m_pool_load_cv.notify_all();
1923
1924                         total_size = p->m_output_offset + p->m_output_size;
1925
1926                         if (outp->m_reset_stream)
1927                                 return -1;
1928                 }
1929
1930                 outp->m_DataSize = total_size;
1931
1932                 atomic_fetch_or(&outp->m_dataflags, FILEBUFFER_FLAG_KNOWN_SIZE | FILEBUFFER_FLAG_SEG_DONE);
1933                 outp->m_request_cv.notify_one();
1934
1935                 for (int i = 0; i < nthread; i++)
1936                 {
1937                         threads[i].join();
1938                 }
1939                 return 0;
1940         }
1941
1942         return Decompress(backfile, outp);
1943 }
1944
1945 int FSBasic::PreloadWorkThread(shared_ptr<FileBuffer>outp)
1946 {
1947         while (!outp->m_reset_stream)
1948         {
1949                 size_t request_offset = outp->m_last_request_offset;
1950                 {
1951                         lock_guard<mutex> lock(outp->m_seg_map_mutex);
1952                         if (!outp->m_offset_request.empty())
1953                         {
1954                                 request_offset = outp->m_offset_request.front();
1955                                 outp->m_offset_request.pop();
1956                         }
1957                 }
1958
1959                 shared_ptr<FragmentBlock> blk;
1960
1961                 {
1962                         lock_guard<mutex> lock(outp->m_seg_map_mutex);
1963
1964                         auto low = outp->m_seg_map.begin();
1965
1966                         low = outp->m_seg_map.lower_bound(request_offset);
1967
1968                         int count = 0;
1969                         while (low != outp->m_seg_map.end() )
1970                         {
1971                                 if (!(low->second->m_dataflags & (FragmentBlock::CONVERT_START)))
1972                                         break;
1973
1974                                 low--;
1975                                 count++;
1976                                 if (g_small_memory && count >= 5)
1977                                         break;
1978                         }
1979
1980                         if (low != outp->m_seg_map.end()) {
1981                                 if (!(low->second->m_dataflags & FragmentBlock::CONVERT_START))
1982                                 {
1983                                         atomic_fetch_or(&low->second->m_dataflags, (int)FragmentBlock::CONVERT_START);
1984                                         blk = low->second;
1985                                 }
1986                         }
1987                 }
1988
1989                 if (!blk ||
1990                         (blk && (blk->m_dataflags & FragmentBlock::CONVERT_DONE))
1991                         )
1992                 {
1993                         std::unique_lock<std::mutex> lck(outp->m_pool_load_cv_mutex);
1994                         outp->m_pool_load_cv.wait(lck);
1995                         continue;
1996                 }
1997
1998                 outp->truncate_old_data_in_pool();
1999
2000                 if (blk->DataConvert() < 0)
2001                 {
2002                                 // todo error handle;
2003                                 continue;
2004                 }
2005
2006                 atomic_fetch_or(&blk->m_dataflags, (int)FragmentBlock::CONVERT_DONE);
2007                 outp->m_request_cv.notify_all();
2008         }
2009         return 0;
2010 }
2011
2012 int FSHttp::load(const string& backfile, const string& filename, shared_ptr<FileBuffer> p)
2013 {
2014         shared_ptr<HttpStream> http = make_shared<HttpStream>();
2015
2016         if (http->HttpGetHeader(backfile, filename, m_Port, typeid(*this) == typeid(FSHttps)))
2017                 return -1;
2018
2019         size_t sz = http->HttpGetFileSize();
2020
2021         p->resize(sz);
2022
2023         atomic_fetch_or(&p->m_dataflags, FILEBUFFER_FLAG_KNOWN_SIZE);
2024         p->m_request_cv.notify_all();
2025
2026         return http_load(http, p, backfile);
2027 }
2028
2029 void uuu_set_small_mem(uint32_t val)
2030 {
2031         g_small_memory = !!val;
2032 }
2033
2034 void clean_up_filemap()
2035 {
2036         for (auto it : g_filebuffer_map)
2037         {
2038                 it.second->m_reset_stream = true;
2039                 it.second->m_pool_load_cv.notify_all();
2040                 if (it.second->m_async_thread.joinable())
2041                         it.second->m_async_thread.join();
2042         }
2043         g_filebuffer_map.clear();
2044 }