From 5aacea6cd9e04c4432a9ddcea3f528e1d2be969a Mon Sep 17 00:00:00 2001 From: Frank Li Date: Tue, 23 Jul 2019 17:37:10 -0500 Subject: [PATCH] Change FileBuffer memory management Use reallocate to allocate memory because it will fast when allocate big memory in windows platform Signed-off-by: Frank Li --- libuuu/buffer.cpp | 6 +-- libuuu/buffer.h | 108 ++++++++++++++++++++++++++++++++++++------------------ 2 files changed, 75 insertions(+), 39 deletions(-) diff --git a/libuuu/buffer.cpp b/libuuu/buffer.cpp index 7d42dc6..66000d7 100644 --- a/libuuu/buffer.cpp +++ b/libuuu/buffer.cpp @@ -581,8 +581,6 @@ int bz2_decompress(shared_ptr pbz, FileBuffer *p, bz2_blks * pblk) return 0; } - - int bz_async_load(string filename, FileBuffer *p) { shared_ptr pbz; @@ -621,7 +619,7 @@ int bz_async_load(string filename, FileBuffer *p) std::this_thread::sleep_for(std::chrono::milliseconds(1000)); - p->resize(pbz->size() * 5); //estimate uncompressed memory size; + p->reserve(pbz->size() * 5); //estimate uncompressed memory size; for (size_t i = 0; i < pbz->size() - 10; i++) { @@ -884,7 +882,7 @@ int file_overwrite_monitor(string filename, FileBuffer *p) str = ">"; str += filename; - if(p->m_pMapbuffer) + if(p->m_pDatabuffer && p->m_allocate_way == FileBuffer::ALLOCATE_MMAP) { std::lock_guard lock(g_mutex_map); p->m_file_monitor.detach(); /*Detach itself, erase will delete p*/ diff --git a/libuuu/buffer.h b/libuuu/buffer.h index 26fa8b5..db0c59b 100644 --- a/libuuu/buffer.h +++ b/libuuu/buffer.h @@ -64,11 +64,19 @@ int file_overwrite_monitor(string filename, FileBuffer *p); class FileBuffer { public: - vector m_data; mutex m_data_mutex; - uint8_t *m_pMapbuffer; - size_t m_MapSize; + uint8_t *m_pDatabuffer; + size_t m_DataSize; + size_t m_MemSize; + enum + { + ALLOCATE_MALLOC, + ALLOCATE_MMAP, + ALLOCATE_VMALLOC, + }m_allocate_way; + + mutex m_async_mutex; atomic_bool m_loaded; thread m_aync_thread; @@ -87,25 +95,39 @@ public: FileBuffer() { - m_pMapbuffer = NULL; - m_MapSize = 0; + m_pDatabuffer = NULL; + m_DataSize = 0; m_loaded = false; + m_MemSize = 0; + m_allocate_way = ALLOCATE_MALLOC; } FileBuffer(void*p, size_t sz) { - m_pMapbuffer = NULL; - m_MapSize = 0; + m_pDatabuffer = NULL; + m_DataSize = 0; m_loaded = true; - m_data.resize(sz); - memcpy(m_data.data(), p, sz); + m_allocate_way = ALLOCATE_MALLOC; + m_MemSize = 0; + + m_pDatabuffer = (uint8_t*)malloc(sz); + m_MemSize = m_DataSize = sz; + + memcpy(m_pDatabuffer, p, sz); } ~FileBuffer() { if(m_aync_thread.joinable()) m_aync_thread.join(); - unmapfile(); + + if (m_pDatabuffer) + { + if(m_allocate_way == ALLOCATE_MMAP) + unmapfile(); + if(m_allocate_way == ALLOCATE_MALLOC) + free(m_pDatabuffer); + } } int request_data(vector &data, size_t offset, size_t sz); @@ -113,12 +135,12 @@ public: uint8_t * data() { - return m_pMapbuffer ? m_pMapbuffer : m_data.data(); + return m_pDatabuffer ; } size_t size() { - return m_pMapbuffer ? m_MapSize : m_data.size(); + return m_DataSize; } uint8_t & operator[] (size_t index) @@ -126,13 +148,10 @@ public: if (!m_loaded) m_aync_thread.join(); - if (m_pMapbuffer) { - assert(index < m_MapSize); - return *(m_pMapbuffer + index); - } - else { - return m_data[index]; - } + assert(m_pDatabuffer); + assert(index < m_DataSize); + + return *(m_pDatabuffer + index); } uint8_t & at(size_t index) @@ -141,13 +160,28 @@ public: } void resize(size_t sz) { - return m_data.resize(sz); + reserve(sz); + + m_DataSize = sz; + } + + void reserve(size_t sz) + { + assert(m_allocate_way != ALLOCATE_MMAP); + + if (sz > m_MemSize) + { + m_pDatabuffer = (uint8_t*)realloc(m_pDatabuffer, sz); + m_MemSize = sz; + } } int swap(FileBuffer & a) { - m_data.swap(a.m_data); - std::swap(m_pMapbuffer, a.m_pMapbuffer); - std::swap(m_MapSize, a.m_MapSize); + std::swap(m_pDatabuffer, a.m_pDatabuffer); + std::swap(m_DataSize, a.m_DataSize); + std::swap(m_MemSize, a.m_MemSize); + std::swap(m_allocate_way, a.m_allocate_way); + return 0; } @@ -204,8 +238,10 @@ public: return -1; } - m_pMapbuffer = (uint8_t *)MapViewOfFile(m_file_map, FILE_MAP_READ, 0, 0, sz); - m_MapSize = sz; + m_pDatabuffer = (uint8_t *)MapViewOfFile(m_file_map, FILE_MAP_READ, 0, 0, sz); + m_DataSize = sz; + m_MemSize = sz; + m_allocate_way = ALLOCATE_MMAP; #else int fd = open(filename.c_str(), O_RDONLY); @@ -218,16 +254,18 @@ public: return -1; } - m_pMapbuffer = (uint8_t *)mmap64(0, sz, PROT_READ, MAP_SHARED, fd, 0); - if (m_pMapbuffer == MAP_FAILED) { - m_pMapbuffer = NULL; + m_pDatabuffer = (uint8_t *)mmap64(0, sz, PROT_READ, MAP_SHARED, fd, 0); + if (m_pDatabuffer == MAP_FAILED) { + m_pDatabuffer = NULL; set_last_err_string("mmap failure\n"); return -1; } - m_MapSize = sz; + m_DataSize = sz; + m_MemSize = sz; + m_allocate_way = ALLOCATE_MMAP; close(fd); #endif - if (m_pMapbuffer) + if (m_pDatabuffer) return 0; set_last_err_string("mmap file failure"); @@ -236,11 +274,11 @@ public: int unmapfile() { - if (m_pMapbuffer) + if (m_pDatabuffer) { #ifdef _MSC_VER - UnmapViewOfFile(m_pMapbuffer); - m_pMapbuffer = NULL; + UnmapViewOfFile(m_pDatabuffer); + m_pDatabuffer = NULL; CloseHandle(m_file_map); CloseHandle(m_file_handle); SetEvent(m_OverLapped.hEvent); @@ -251,9 +289,9 @@ public: CloseHandle(m_OverLapped.hEvent); m_OverLapped.hEvent = m_file_map = m_file_handle = INVALID_HANDLE_VALUE; #else - munmap(m_pMapbuffer, m_MapSize); + munmap(m_pDatabuffer, m_DataSize); #endif - m_pMapbuffer = NULL; + m_pDatabuffer = NULL; } return 0; } -- 2.11.4.GIT