libuuu/buffer.h

   1 /*
   2 * Copyright 2018 NXP.
   3 *
   4 * Redistribution and use in source and binary forms, with or without modification,
   5 * are permitted provided that the following conditions are met:
   6 *
   7 * Redistributions of source code must retain the above copyright notice, this
   8 * list of conditions and the following disclaimer.
   9 *
  10 * Redistributions in binary form must reproduce the above copyright notice, this
  11 * list of conditions and the following disclaimer in the documentation and/or
  12 * other materials provided with the distribution.
  13 *
  14 * Neither the name of the NXP Semiconductor nor the names of its
  15 * contributors may be used to endorse or promote products derived from this
  16 * software without specific prior written permission.
  17 *
  18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  19 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
  22 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  28 * POSSIBILITY OF SUCH DAMAGE.
  29 *
  30 */
  31
  32 #pragma once
  33
  34 #include <atomic>
  35 #include <cassert>
  36 #include <condition_variable>
  37 #include <memory>
  38 #include <mutex>
  39 #include <thread>
  40 #include <vector>
  41 #include <map>
  42 #include <queue>
  43 #include "liberror.h"
  44 #include <cstring>
  45 #ifdef _MSC_VER
  46 #include <Windows.h>
  47 #else
  48 #include <sys/mman.h>
  49 #include <fcntl.h>
  50 #include <unistd.h>
  51 #endif
  52
  53 #ifdef __APPLE__
  54 #define mmap64 mmap
  55 #endif
  56
  57 #ifdef WIN32
  58 class FileBuffer;
  59 int file_overwrite_monitor(std::string filename, FileBuffer *p);
  60 #endif
  61
  62 //bit 0, data loaded
  63 //bit 1, data total size known
  64 #define FILEBUFFER_FLAG_LOADED_BIT              0x1
  65 #define FILEBUFFER_FLAG_KNOWN_SIZE_BIT  0x2
  66 #define FILEBUFFER_FLAG_ERROR_BIT               0x4
  67 #define FILEBUFFER_FLAG_NEVER_FREE              0x8
  68 #define FILEBUFFER_FLAG_PARTIAL_RELOADABLE 0x10
  69 #define FILEBUFFER_FLAG_SEG_DONE                0x20
  70
  71 #define FILEBUFFER_FLAG_LOADED          (FILEBUFFER_FLAG_LOADED_BIT|FILEBUFFER_FLAG_KNOWN_SIZE_BIT) // LOADED must be knownsize
  72 #define FILEBUFFER_FLAG_KNOWN_SIZE      FILEBUFFER_FLAG_KNOWN_SIZE_BIT
  73
  74 class FileBuffer;
  75 class FSBasic;
  76
  77 class FragmentBlock
  78 {
  79 public:
  80         enum
  81         {
  82                 CONVERT_DONE = 0x1,
  83                 USING = 0x2,
  84                 CONVERT_START = 0x4,
  85                 CONVERT_PARTIAL = 0x8,
  86         };
  87         size_t m_input_offset = 0;
  88         size_t m_input_sz = 0;
  89         std::shared_ptr<FileBuffer> m_input;
  90         size_t m_ret = 0;
  91         size_t m_actual_size = 0;
  92         size_t m_output_size = 0;
  93         size_t m_output_offset = 0;
  94         virtual int DataConvert() { return -1; };
  95         std::vector<uint8_t> m_data;
  96         std::mutex m_mutex;
  97         std::atomic_int m_dataflags{0};
  98         uint8_t* m_pData = NULL;
  99         uint8_t* data()
 100         {
 101                 if (m_pData)
 102                         return m_pData;
 103                 return m_data.data();
 104         }
 105 };
 106
 107
 108 class DataBuffer : public std::enable_shared_from_this<DataBuffer>
 109 {
 110         enum class ALLOCATION_WAYS
 111         {
 112                 MALLOC,
 113                 REF,
 114         };
 115
 116 protected:
 117
 118         ALLOCATION_WAYS get_m_allocate_way() const noexcept { return m_allocate_way; }
 119         uint8_t* m_pDatabuffer = NULL;
 120         size_t m_DataSize = 0;
 121         size_t m_MemSize = 0;
 122         std::shared_ptr<FileBuffer> m_ref;
 123         ALLOCATION_WAYS m_allocate_way = ALLOCATION_WAYS::MALLOC;
 124
 125 public:
 126         DataBuffer()
 127         {
 128                 m_allocate_way = ALLOCATION_WAYS::MALLOC;
 129         }
 130         DataBuffer(void* p, size_t sz)
 131         {
 132                 m_allocate_way = ALLOCATION_WAYS::MALLOC;
 133                 resize(sz);
 134                 memcpy(data(), p, sz);
 135         }
 136         uint8_t* data() { return m_pDatabuffer; }
 137         size_t size() { return m_DataSize; }
 138         int resize(size_t sz);
 139         int ref_other_buffer(std::shared_ptr<FileBuffer> p, size_t offset, size_t size);
 140         uint8_t& operator[] (size_t index)
 141         {
 142                 assert(m_pDatabuffer);
 143                 assert(index < m_DataSize);
 144
 145                 return *(m_pDatabuffer + index);
 146         }
 147         uint8_t& at(size_t index)
 148         {
 149                 return (*this)[index];
 150         }
 151         ~DataBuffer()
 152         {
 153                 if (m_allocate_way == ALLOCATION_WAYS::MALLOC)
 154                 {
 155                         free(m_pDatabuffer);
 156                 }
 157         }
 158         friend class FileBuffer;
 159 };
 160
 161 class FileBuffer: public std::enable_shared_from_this<FileBuffer>
 162 {
 163 public:
 164         friend class DataBuffer;
 165         friend class FSBase;
 166         friend class FSFlat;
 167         friend class FSHttps;
 168         friend class FSHttp;
 169         friend class FSGz;
 170         friend class FSzstd;
 171         friend class FSCompressStream;
 172         friend class Fat;
 173         friend class Tar;
 174         friend class Zip;
 175         friend class Zip_file_Info;
 176         enum class ALLOCATION_WAYS
 177         {
 178                 MALLOC,
 179                 MMAP,
 180                 REF,
 181                 VMALLOC,
 182                 SEGMENT,
 183         };
 184
 185         std::mutex m_data_mutex;
 186
 187         uint8_t *m_pDatabuffer;
 188         size_t m_DataSize;
 189         size_t m_MemSize;
 190
 191         std::shared_ptr<FileBuffer> m_ref;
 192
 193         int ref_other_buffer(std::shared_ptr<FileBuffer> p, size_t offset, size_t size);
 194
 195         std::mutex m_async_mutex;
 196
 197         std::map<size_t, std::shared_ptr<FragmentBlock>, std::greater<size_t>> m_seg_map;
 198         std::mutex m_seg_map_mutex;
 199         std::queue<size_t> m_offset_request;
 200         size_t m_last_request_offset = 0;
 201         std::condition_variable m_pool_load_cv;
 202         std::mutex m_pool_load_cv_mutex;
 203         std::shared_ptr<FragmentBlock> m_last_db;
 204         size_t m_seg_blk_size = 0x800000;
 205         size_t m_totall_buffer_size = 8 * m_seg_blk_size;
 206         std::atomic_bool m_reset_stream { false };
 207
 208         //used for continue decompress\loading only
 209         std::shared_ptr<FragmentBlock> request_new_blk();
 210         bool check_offset_in_seg(size_t offset, std::shared_ptr<FragmentBlock> blk)
 211         {
 212                 if (offset >= blk->m_output_offset
 213                         && offset < blk->m_output_offset + blk->m_output_size)
 214                         return true;
 215
 216                 return false;
 217         }
 218         std::shared_ptr<FragmentBlock> get_map_it(size_t offset, bool alloc = false)
 219         {
 220                 {
 221                         std::lock_guard<std::mutex> lock(m_seg_map_mutex);
 222                         auto it = m_seg_map.lower_bound(offset);
 223                         if ( it == m_seg_map.end())
 224                                 return NULL;
 225
 226                         auto blk = it->second;
 227                         if (check_offset_in_seg(offset, blk))
 228                         {
 229                                 if (alloc)
 230                                 {
 231                                         std::lock_guard<std::mutex> lck(blk->m_mutex);
 232                                         blk->m_data.resize(blk->m_output_size);
 233                                 }
 234                                 return blk;
 235                         }
 236                         return NULL;
 237                 }
 238         }
 239         void truncate_old_data_in_pool();
 240
 241         std::atomic_int m_dataflags;
 242
 243         std::thread m_aync_thread;
 244
 245         std::atomic_size_t m_avaible_size;
 246         std::condition_variable m_request_cv;
 247         std::mutex m_requext_cv_mutex;
 248
 249 #ifdef WIN32
 250         OVERLAPPED m_OverLapped;
 251         REQUEST_OPLOCK_INPUT_BUFFER m_Request;
 252         HANDLE m_file_handle;
 253         HANDLE m_file_map;
 254         std::thread m_file_monitor;
 255 #endif
 256
 257         uint64_t m_timesample;
 258
 259         FileBuffer();
 260         FileBuffer(void*p, size_t sz);
 261         ~FileBuffer();
 262
 263         ALLOCATION_WAYS get_m_allocate_way() const noexcept { return m_allocate_way; }
 264
 265         int64_t request_data(void * data, size_t offset, size_t sz);
 266         int request_data(std::vector<uint8_t> &data, size_t offset, size_t sz);
 267         std::shared_ptr<DataBuffer> request_data(size_t offset, size_t sz);
 268
 269         bool IsLoaded() const noexcept
 270         {
 271                 return m_dataflags & FILEBUFFER_FLAG_LOADED_BIT;
 272         }
 273
 274         bool IsRefable() const noexcept
 275         {
 276                 return m_dataflags & FILEBUFFER_FLAG_NEVER_FREE;
 277         }
 278
 279         bool IsKnownSize() const noexcept
 280         {
 281                 return m_dataflags & FILEBUFFER_FLAG_KNOWN_SIZE_BIT;
 282         }
 283
 284         bool IsError() const noexcept
 285         {
 286                 return m_dataflags & FILEBUFFER_FLAG_ERROR_BIT;
 287         }
 288
 289         int reload(std::string filename, bool async = false);
 290
 291         size_t size()
 292         {
 293                 if (IsKnownSize())
 294                         return m_DataSize;
 295
 296                 std::unique_lock<std::mutex> lck(m_requext_cv_mutex);
 297                 while (!(m_dataflags & FILEBUFFER_FLAG_KNOWN_SIZE_BIT))
 298                         m_request_cv.wait(lck);
 299
 300                 return m_DataSize;
 301         }
 302
 303 protected:
 304         uint8_t * data() noexcept
 305         {
 306                 return m_pDatabuffer ;
 307         }
 308
 309         uint8_t & operator[] (size_t index)
 310         {
 311                 assert(m_pDatabuffer);
 312                 assert(index < m_DataSize);
 313
 314                 return *(m_pDatabuffer + index);
 315         }
 316
 317         uint8_t & at(size_t index)
 318         {
 319                 return (*this)[index];
 320         }
 321         int resize(size_t sz);
 322
 323         int reserve(size_t sz);
 324
 325         int swap(FileBuffer & a);
 326
 327         int mapfile(const std::string &filename, size_t sz);
 328
 329         int unmapfile();
 330         //Read write lock;
 331
 332 protected:
 333         int64_t request_data_from_segment(void* data, size_t offset, size_t sz);
 334         int m_pool_size = 10;
 335         std::string m_filename;
 336 private:
 337         ALLOCATION_WAYS m_allocate_way = ALLOCATION_WAYS::MALLOC;
 338 };
 339
 340 std::shared_ptr<FileBuffer> get_file_buffer(std::string filename, bool aysnc=false);
 341 bool check_file_exist(std::string filename, bool start_async_load=true);
 342
 343 void set_current_dir(const std::string &dir);
 344
 345 bool IsMBR(std::shared_ptr<DataBuffer> p);
 346 size_t ScanTerm(std::shared_ptr<DataBuffer> p, size_t& pos, size_t offset = 512, size_t limited = 0x800000);
 347 void clean_up_filemap();