libuuu/buffer.h

   1 /*
   2 * Copyright 2018 NXP.
   3 *
   4 * Redistribution and use in source and binary forms, with or without modification,
   5 * are permitted provided that the following conditions are met:
   6 *
   7 * Redistributions of source code must retain the above copyright notice, this
   8 * list of conditions and the following disclaimer.
   9 *
  10 * Redistributions in binary form must reproduce the above copyright notice, this
  11 * list of conditions and the following disclaimer in the documentation and/or
  12 * other materials provided with the distribution.
  13 *
  14 * Neither the name of the NXP Semiconductor nor the names of its
  15 * contributors may be used to endorse or promote products derived from this
  16 * software without specific prior written permission.
  17 *
  18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  19 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
  22 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  28 * POSSIBILITY OF SUCH DAMAGE.
  29 *
  30 */
  31
  32 #pragma once
  33
  34 #include <atomic>
  35 #include <cassert>
  36 #include <condition_variable>
  37 #include <memory>
  38 #include <mutex>
  39 #include <thread>
  40 #include <vector>
  41 #include <map>
  42 #include <queue>
  43 #include "liberror.h"
  44 #include <cstring>
  45 #ifdef _MSC_VER
  46 #include <Windows.h>
  47 #else
  48 #include <sys/mman.h>
  49 #include <fcntl.h>
  50 #include <unistd.h>
  51 #endif
  52
  53 #ifdef __APPLE__
  54 #define mmap64 mmap
  55 #endif
  56
  57 #ifdef WIN32
  58 class FileBuffer;
  59 int file_overwrite_monitor(std::string filename, FileBuffer *p);
  60 #endif
  61
  62 //bit 0, data loaded
  63 //bit 1, data total size known
  64 #define FILEBUFFER_FLAG_LOADED_BIT              0x1
  65 #define FILEBUFFER_FLAG_KNOWN_SIZE_BIT  0x2
  66 #define FILEBUFFER_FLAG_ERROR_BIT               0x4
  67 #define FILEBUFFER_FLAG_NEVER_FREE              0x8
  68 #define FILEBUFFER_FLAG_PARTIAL_RELOADABLE 0x10
  69 #define FILEBUFFER_FLAG_SEG_DONE                0x20
  70
  71 #define FILEBUFFER_FLAG_LOADED          (FILEBUFFER_FLAG_LOADED_BIT|FILEBUFFER_FLAG_KNOWN_SIZE_BIT) // LOADED must be known size
  72 #define FILEBUFFER_FLAG_KNOWN_SIZE      FILEBUFFER_FLAG_KNOWN_SIZE_BIT
  73
  74 class FileBuffer;
  75 class FSBasic;
  76
  77 class FragmentBlock
  78 {
  79 public:
  80         enum
  81         {
  82                 CONVERT_DONE = 0x1,
  83                 USING = 0x2,
  84                 CONVERT_START = 0x4,
  85                 CONVERT_PARTIAL = 0x8,
  86         };
  87         size_t m_input_offset = 0;
  88         size_t m_input_sz = 0;
  89         std::shared_ptr<FileBuffer> m_input;
  90         size_t m_ret = 0;
  91         size_t m_actual_size = 0;
  92         size_t m_output_size = 0;
  93         size_t m_output_offset = 0;
  94         virtual int DataConvert() { return -1; };
  95         std::vector<uint8_t> m_data;
  96         std::mutex m_mutex;
  97         std::atomic_int m_dataflags{0};
  98         uint8_t* m_pData = NULL;
  99         uint8_t* data()
 100         {
 101                 if (m_pData)
 102                         return m_pData;
 103                 return m_data.data();
 104         }
 105
 106         virtual ~FragmentBlock() {}
 107 };
 108
 109
 110 class DataBuffer : public std::enable_shared_from_this<DataBuffer>
 111 {
 112         enum class ALLOCATION_WAYS
 113         {
 114                 MALLOC,
 115                 REF,
 116         };
 117
 118 protected:
 119
 120         ALLOCATION_WAYS get_m_allocate_way() const noexcept { return m_allocate_way; }
 121         uint8_t* m_pDatabuffer = NULL;
 122         size_t m_DataSize = 0;
 123         size_t m_MemSize = 0;
 124         std::shared_ptr<FileBuffer> m_ref;
 125         ALLOCATION_WAYS m_allocate_way = ALLOCATION_WAYS::MALLOC;
 126
 127 public:
 128         DataBuffer()
 129         {
 130                 m_allocate_way = ALLOCATION_WAYS::MALLOC;
 131         }
 132         DataBuffer(void* p, size_t sz)
 133         {
 134                 m_allocate_way = ALLOCATION_WAYS::MALLOC;
 135                 resize(sz);
 136                 memcpy(data(), p, sz);
 137         }
 138         uint8_t* data() { return m_pDatabuffer; }
 139         size_t size() { return m_DataSize; }
 140         int resize(size_t sz);
 141         int ref_other_buffer(std::shared_ptr<FileBuffer> p, size_t offset, size_t size);
 142         uint8_t& operator[] (size_t index)
 143         {
 144                 assert(m_pDatabuffer);
 145                 assert(index < m_DataSize);
 146
 147                 return *(m_pDatabuffer + index);
 148         }
 149         uint8_t& at(size_t index)
 150         {
 151                 return (*this)[index];
 152         }
 153         virtual ~DataBuffer()
 154         {
 155                 if (m_allocate_way == ALLOCATION_WAYS::MALLOC)
 156                 {
 157                         free(m_pDatabuffer);
 158                 }
 159         }
 160         friend class FileBuffer;
 161 };
 162
 163 class FileBuffer: public std::enable_shared_from_this<FileBuffer>
 164 {
 165 public:
 166         friend class DataBuffer;
 167         friend class FSBase;
 168         friend class FSFlat;
 169         friend class FSHttps;
 170         friend class FSHttp;
 171         friend class FSGz;
 172         friend class FSzstd;
 173         friend class FSCompressStream;
 174         friend class Fat;
 175         friend class Tar;
 176         friend class Zip;
 177         friend class Zip_file_Info;
 178         enum class ALLOCATION_WAYS
 179         {
 180                 MALLOC,
 181                 MMAP,
 182                 REF,
 183                 VMALLOC,
 184                 SEGMENT,
 185         };
 186
 187         std::mutex m_data_mutex;
 188
 189         uint8_t *m_pDatabuffer;
 190         size_t m_DataSize;
 191         size_t m_MemSize;
 192
 193         std::shared_ptr<FileBuffer> m_ref;
 194
 195         int ref_other_buffer(std::shared_ptr<FileBuffer> p, size_t offset, size_t size);
 196
 197         std::mutex m_async_mutex;
 198
 199         std::map<size_t, std::shared_ptr<FragmentBlock>, std::greater<size_t>> m_seg_map;
 200         std::mutex m_seg_map_mutex;
 201         std::queue<size_t> m_offset_request;
 202         size_t m_last_request_offset = 0;
 203         std::condition_variable m_pool_load_cv;
 204         std::mutex m_pool_load_cv_mutex;
 205         std::shared_ptr<FragmentBlock> m_last_db;
 206         size_t m_seg_blk_size = 0x800000;
 207         size_t m_total_buffer_size = 8 * m_seg_blk_size;
 208         std::atomic_bool m_reset_stream { false };
 209
 210         //used for continue decompress\loading only
 211         std::shared_ptr<FragmentBlock> request_new_blk();
 212         bool check_offset_in_seg(size_t offset, std::shared_ptr<FragmentBlock> blk)
 213         {
 214                 if (offset >= blk->m_output_offset
 215                         && offset < blk->m_output_offset + blk->m_output_size)
 216                         return true;
 217
 218                 return false;
 219         }
 220         std::shared_ptr<FragmentBlock> get_map_it(size_t offset, bool alloc = false)
 221         {
 222                 {
 223                         std::lock_guard<std::mutex> lock(m_seg_map_mutex);
 224                         auto it = m_seg_map.lower_bound(offset);
 225                         if ( it == m_seg_map.end())
 226                                 return NULL;
 227
 228                         auto blk = it->second;
 229                         if (check_offset_in_seg(offset, blk))
 230                         {
 231                                 if (alloc)
 232                                 {
 233                                         std::lock_guard<std::mutex> lck(blk->m_mutex);
 234                                         blk->m_data.resize(blk->m_output_size);
 235                                 }
 236                                 return blk;
 237                         }
 238                         return NULL;
 239                 }
 240         }
 241         void truncate_old_data_in_pool();
 242
 243         std::atomic_int m_dataflags;
 244
 245         std::thread m_async_thread;
 246
 247         std::atomic_size_t m_available_size;
 248         std::condition_variable m_request_cv;
 249         std::mutex m_request_cv_mutex;
 250
 251 #ifdef WIN32
 252         OVERLAPPED m_OverLapped;
 253         REQUEST_OPLOCK_INPUT_BUFFER m_Request;
 254         HANDLE m_file_handle;
 255         HANDLE m_file_map;
 256         std::thread m_file_monitor;
 257 #endif
 258
 259         uint64_t m_timesample;
 260
 261         FileBuffer();
 262         FileBuffer(void*p, size_t sz);
 263         ~FileBuffer();
 264
 265         ALLOCATION_WAYS get_m_allocate_way() const noexcept { return m_allocate_way; }
 266
 267         int64_t request_data(void * data, size_t offset, size_t sz);
 268         int request_data(std::vector<uint8_t> &data, size_t offset, size_t sz);
 269         std::shared_ptr<DataBuffer> request_data(size_t offset, size_t sz);
 270
 271         bool IsLoaded() const noexcept
 272         {
 273                 return m_dataflags & FILEBUFFER_FLAG_LOADED_BIT;
 274         }
 275
 276         bool IsRefable() const noexcept
 277         {
 278                 return m_dataflags & FILEBUFFER_FLAG_NEVER_FREE;
 279         }
 280
 281         bool IsKnownSize() const noexcept
 282         {
 283                 return m_dataflags & FILEBUFFER_FLAG_KNOWN_SIZE_BIT;
 284         }
 285
 286         bool IsError() const noexcept
 287         {
 288                 return m_dataflags & FILEBUFFER_FLAG_ERROR_BIT;
 289         }
 290
 291         int reload(std::string filename, bool async = false);
 292
 293         size_t size()
 294         {
 295                 if (IsKnownSize())
 296                         return m_DataSize;
 297
 298                 std::unique_lock<std::mutex> lck(m_request_cv_mutex);
 299                 while (!(m_dataflags & FILEBUFFER_FLAG_KNOWN_SIZE_BIT))
 300                         m_request_cv.wait(lck);
 301
 302                 return m_DataSize;
 303         }
 304
 305 protected:
 306         uint8_t * data() noexcept
 307         {
 308                 return m_pDatabuffer ;
 309         }
 310
 311         uint8_t & operator[] (size_t index)
 312         {
 313                 assert(m_pDatabuffer);
 314                 assert(index < m_DataSize);
 315
 316                 return *(m_pDatabuffer + index);
 317         }
 318
 319         uint8_t & at(size_t index)
 320         {
 321                 return (*this)[index];
 322         }
 323         int resize(size_t sz);
 324
 325         int reserve(size_t sz);
 326
 327         int swap(FileBuffer & a);
 328
 329         int mapfile(const std::string &filename, size_t sz);
 330
 331         int unmapfile();
 332         //Read write lock;
 333
 334 protected:
 335         int64_t request_data_from_segment(void* data, size_t offset, size_t sz);
 336         int m_pool_size = 10;
 337         std::string m_filename;
 338 private:
 339         ALLOCATION_WAYS m_allocate_way = ALLOCATION_WAYS::MALLOC;
 340 };
 341
 342 std::shared_ptr<FileBuffer> get_file_buffer(std::string filename, bool async=false);
 343 bool check_file_exist(std::string filename, bool start_async_load=true);
 344
 345 void set_current_dir(const std::string &dir);
 346
 347 bool IsMBR(std::shared_ptr<DataBuffer> p);
 348 size_t ScanTerm(std::shared_ptr<DataBuffer> p, size_t& pos, size_t offset = 512, size_t limited = 0x800000);
 349 void clean_up_filemap();