uuu
uuu (Universal Update Utility), mfgtools 3.0
Loading...
Searching...
No Matches
buffer.h
Go to the documentation of this file.
1/*
2* Copyright 2018 NXP.
3*
4* Redistribution and use in source and binary forms, with or without modification,
5* are permitted provided that the following conditions are met:
6*
7* Redistributions of source code must retain the above copyright notice, this
8* list of conditions and the following disclaimer.
9*
10* Redistributions in binary form must reproduce the above copyright notice, this
11* list of conditions and the following disclaimer in the documentation and/or
12* other materials provided with the distribution.
13*
14* Neither the name of the NXP Semiconductor nor the names of its
15* contributors may be used to endorse or promote products derived from this
16* software without specific prior written permission.
17*
18* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28* POSSIBILITY OF SUCH DAMAGE.
29*
30*/
31
32#pragma once
33
34#include <atomic>
35#include <cassert>
36#include <condition_variable>
37#include <memory>
38#include <mutex>
39#include <thread>
40#include <vector>
41#include <map>
42#include <queue>
43#include "liberror.h"
44#include <cstring>
45#ifdef _MSC_VER
46#include <Windows.h>
47#else
48#include <sys/mman.h>
49#include <fcntl.h>
50#include <unistd.h>
51#endif
52
53#ifdef __APPLE__
54#define mmap64 mmap
55#endif
56
57#ifdef WIN32
58class FileBuffer;
59int file_overwrite_monitor(std::string filename, FileBuffer *p);
60#endif
61
62//bit 0, data loaded
63//bit 1, data total size known
64#define FILEBUFFER_FLAG_LOADED_BIT 0x1
65#define FILEBUFFER_FLAG_KNOWN_SIZE_BIT 0x2
66#define FILEBUFFER_FLAG_ERROR_BIT 0x4
67#define FILEBUFFER_FLAG_NEVER_FREE 0x8
68#define FILEBUFFER_FLAG_PARTIAL_RELOADABLE 0x10
69#define FILEBUFFER_FLAG_SEG_DONE 0x20
70
71#define FILEBUFFER_FLAG_LOADED (FILEBUFFER_FLAG_LOADED_BIT|FILEBUFFER_FLAG_KNOWN_SIZE_BIT) // LOADED must be known size
72#define FILEBUFFER_FLAG_KNOWN_SIZE FILEBUFFER_FLAG_KNOWN_SIZE_BIT
73
74class FileBuffer;
75class FSBasic;
76
78{
79public:
80 enum
81 {
83 USING = 0x2,
86 };
87 size_t m_input_offset = 0;
88 size_t m_input_sz = 0;
89 std::shared_ptr<FileBuffer> m_input;
90 size_t m_ret = 0;
91 size_t m_actual_size = 0;
92 size_t m_output_size = 0;
93 size_t m_output_offset = 0;
94 virtual int DataConvert() { return -1; };
95 std::vector<uint8_t> m_data;
96 std::mutex m_mutex;
97 std::atomic_int m_dataflags{0};
98 uint8_t* m_pData = NULL;
99 uint8_t* data()
100 {
101 if (m_pData)
102 return m_pData;
103 return m_data.data();
104 }
105
106 virtual ~FragmentBlock() {}
107};
108
109
110class DataBuffer : public std::enable_shared_from_this<DataBuffer>
111{
113 {
116 };
117
118protected:
119
121 uint8_t* m_pDatabuffer = NULL;
122 size_t m_DataSize = 0;
123 size_t m_MemSize = 0;
124 std::shared_ptr<FileBuffer> m_ref;
126
127public:
132 DataBuffer(void* p, size_t sz)
133 {
135 resize(sz);
136 memcpy(data(), p, sz);
137 }
138 uint8_t* data() { return m_pDatabuffer; }
139 size_t size() { return m_DataSize; }
140 int resize(size_t sz);
141 int ref_other_buffer(std::shared_ptr<FileBuffer> p, size_t offset, size_t size);
142 uint8_t& operator[] (size_t index)
143 {
144 assert(m_pDatabuffer);
145 assert(index < m_DataSize);
146
147 return *(m_pDatabuffer + index);
148 }
149 uint8_t& at(size_t index)
150 {
151 return (*this)[index];
152 }
153 virtual ~DataBuffer()
154 {
156 {
157 free(m_pDatabuffer);
158 }
159 }
160 friend class FileBuffer;
161};
162
163class FileBuffer: public std::enable_shared_from_this<FileBuffer>
164{
165public:
166 friend class DataBuffer;
167 friend class FSBase;
168 friend class FSFlat;
169 friend class FSHttps;
170 friend class FSHttp;
171 friend class FSGz;
172 friend class FSzstd;
173 friend class FSCompressStream;
174 friend class Fat;
175 friend class Tar;
176 friend class Zip;
177 friend class Zip_file_Info;
186
187 std::mutex m_data_mutex;
188
191 size_t m_MemSize;
192
193 std::shared_ptr<FileBuffer> m_ref;
194
195 int ref_other_buffer(std::shared_ptr<FileBuffer> p, size_t offset, size_t size);
196
197 std::mutex m_async_mutex;
198
199 std::map<size_t, std::shared_ptr<FragmentBlock>, std::greater<size_t>> m_seg_map;
200 std::mutex m_seg_map_mutex;
201 std::queue<size_t> m_offset_request;
203 std::condition_variable m_pool_load_cv;
205 std::shared_ptr<FragmentBlock> m_last_db;
206 size_t m_seg_blk_size = 0x800000;
208 std::atomic_bool m_reset_stream { false };
209
210 //used for continue decompress\loading only
211 std::shared_ptr<FragmentBlock> request_new_blk();
212 bool check_offset_in_seg(size_t offset, std::shared_ptr<FragmentBlock> blk)
213 {
214 if (offset >= blk->m_output_offset
215 && offset < blk->m_output_offset + blk->m_output_size)
216 return true;
217
218 return false;
219 }
220 std::shared_ptr<FragmentBlock> get_map_it(size_t offset, bool alloc = false)
221 {
222 {
223 std::lock_guard<std::mutex> lock(m_seg_map_mutex);
224 auto it = m_seg_map.lower_bound(offset);
225 if ( it == m_seg_map.end())
226 return NULL;
227
228 auto blk = it->second;
229 if (check_offset_in_seg(offset, blk))
230 {
231 if (alloc)
232 {
233 std::lock_guard<std::mutex> lck(blk->m_mutex);
234 blk->m_data.resize(blk->m_output_size);
235 }
236 return blk;
237 }
238 return NULL;
239 }
240 }
242
243 std::atomic_int m_dataflags;
244
245 std::thread m_async_thread;
246
247 std::atomic_size_t m_available_size;
248 std::condition_variable m_request_cv;
250
251#ifdef WIN32
252 OVERLAPPED m_OverLapped;
253 REQUEST_OPLOCK_INPUT_BUFFER m_Request;
254 HANDLE m_file_handle;
255 HANDLE m_file_map;
256 std::thread m_file_monitor;
257#endif
258
259 uint64_t m_timesample;
260
261 FileBuffer();
262 FileBuffer(void*p, size_t sz);
263 ~FileBuffer();
264
266
267 int64_t request_data(void * data, size_t offset, size_t sz);
268 int request_data(std::vector<uint8_t> &data, size_t offset, size_t sz);
269 std::shared_ptr<DataBuffer> request_data(size_t offset, size_t sz);
270
271 bool IsLoaded() const noexcept
272 {
274 }
275
276 bool IsRefable() const noexcept
277 {
279 }
280
281 bool IsKnownSize() const noexcept
282 {
284 }
285
286 bool IsError() const noexcept
287 {
289 }
290
291 int reload(std::string filename, bool async = false);
292
293 size_t size()
294 {
295 if (IsKnownSize())
296 return m_DataSize;
297
298 std::unique_lock<std::mutex> lck(m_request_cv_mutex);
300 m_request_cv.wait(lck);
301
302 return m_DataSize;
303 }
304
305protected:
306 uint8_t * data() noexcept
307 {
308 return m_pDatabuffer ;
309 }
310
311 uint8_t & operator[] (size_t index)
312 {
313 assert(m_pDatabuffer);
314 assert(index < m_DataSize);
315
316 return *(m_pDatabuffer + index);
317 }
318
319 uint8_t & at(size_t index)
320 {
321 return (*this)[index];
322 }
323 int resize(size_t sz);
324
325 int reserve(size_t sz);
326
327 int swap(FileBuffer & a);
328
329 int mapfile(const std::string &filename, size_t sz);
330
331 int unmapfile();
332 //Read write lock;
333
334protected:
335 int64_t request_data_from_segment(void* data, size_t offset, size_t sz);
336 int m_pool_size = 10;
337 std::string m_filename;
338private:
340};
341
342std::shared_ptr<FileBuffer> get_file_buffer(std::string filename, bool async=false);
343bool check_file_exist(std::string filename, bool start_async_load=true);
344
345void set_current_dir(const std::string &dir);
346
347bool IsMBR(std::shared_ptr<DataBuffer> p);
348size_t ScanTerm(std::shared_ptr<DataBuffer> p, size_t& pos, size_t offset = 512, size_t limited = 0x800000);
349void clean_up_filemap();
std::shared_ptr< FileBuffer > get_file_buffer(std::string filename, bool async=false)
Definition buffer.cpp:1178
bool check_file_exist(std::string filename, bool start_async_load=true)
Definition buffer.cpp:1806
#define FILEBUFFER_FLAG_NEVER_FREE
Definition buffer.h:67
#define FILEBUFFER_FLAG_ERROR_BIT
Definition buffer.h:66
void clean_up_filemap()
Definition buffer.cpp:2039
#define FILEBUFFER_FLAG_LOADED_BIT
Definition buffer.h:64
bool IsMBR(std::shared_ptr< DataBuffer > p)
size_t ScanTerm(std::shared_ptr< DataBuffer > p, size_t &pos, size_t offset=512, size_t limited=0x800000)
Definition rominfo.cpp:246
void set_current_dir(const std::string &dir)
Definition buffer.cpp:70
#define FILEBUFFER_FLAG_KNOWN_SIZE_BIT
Definition buffer.h:65
int ref_other_buffer(std::shared_ptr< FileBuffer > p, size_t offset, size_t size)
Definition buffer.cpp:101
DataBuffer(void *p, size_t sz)
Definition buffer.h:132
uint8_t * data()
Definition buffer.h:138
ALLOCATION_WAYS m_allocate_way
Definition buffer.h:125
int resize(size_t sz)
Definition buffer.cpp:77
DataBuffer()
Definition buffer.h:128
uint8_t * m_pDatabuffer
Definition buffer.h:121
ALLOCATION_WAYS
Definition buffer.h:113
@ MALLOC
Definition buffer.h:114
@ REF
Definition buffer.h:115
ALLOCATION_WAYS get_m_allocate_way() const noexcept
Definition buffer.h:120
size_t m_MemSize
Definition buffer.h:123
friend class FileBuffer
Definition buffer.h:160
std::shared_ptr< FileBuffer > m_ref
Definition buffer.h:124
uint8_t & at(size_t index)
Definition buffer.h:149
size_t m_DataSize
Definition buffer.h:122
size_t size()
Definition buffer.h:139
virtual ~DataBuffer()
Definition buffer.h:153
uint8_t & operator[](size_t index)
Definition buffer.h:142
Definition buffer.cpp:132
Definition buffer.h:164
std::atomic_int m_dataflags
Definition buffer.h:243
friend class DataBuffer
Definition buffer.h:166
size_t m_DataSize
Definition buffer.h:190
std::mutex m_async_mutex
Definition buffer.h:197
friend class FSFlat
Definition buffer.h:168
std::string m_filename
Definition buffer.h:337
ALLOCATION_WAYS get_m_allocate_way() const noexcept
Definition buffer.h:265
std::map< size_t, std::shared_ptr< FragmentBlock >, std::greater< size_t > > m_seg_map
Definition buffer.h:199
size_t size()
Definition buffer.h:293
bool IsError() const noexcept
Definition buffer.h:286
uint8_t * data() noexcept
Definition buffer.h:306
friend class Zip_file_Info
Definition buffer.h:177
friend class FSHttp
Definition buffer.h:170
int64_t request_data(void *data, size_t offset, size_t sz)
Definition buffer.cpp:1568
size_t m_MemSize
Definition buffer.h:191
std::shared_ptr< FragmentBlock > request_new_blk()
Definition buffer.cpp:1633
int unmapfile()
Definition buffer.cpp:1782
std::atomic_bool m_reset_stream
Definition buffer.h:208
size_t m_seg_blk_size
Definition buffer.h:206
std::shared_ptr< FileBuffer > m_ref
Definition buffer.h:193
friend class Tar
Definition buffer.h:175
friend class Zip
Definition buffer.h:176
size_t m_last_request_offset
Definition buffer.h:202
int mapfile(const std::string &filename, size_t sz)
Definition buffer.cpp:1286
friend class Fat
Definition buffer.h:174
int resize(size_t sz)
Definition buffer.cpp:1748
std::mutex m_pool_load_cv_mutex
Definition buffer.h:204
ALLOCATION_WAYS m_allocate_way
Definition buffer.h:339
int m_pool_size
Definition buffer.h:336
friend class FSzstd
Definition buffer.h:172
std::condition_variable m_pool_load_cv
Definition buffer.h:203
friend class FSCompressStream
Definition buffer.h:173
friend class FSGz
Definition buffer.h:171
uint64_t m_timesample
Definition buffer.h:259
std::shared_ptr< FragmentBlock > get_map_it(size_t offset, bool alloc=false)
Definition buffer.h:220
std::thread m_async_thread
Definition buffer.h:245
int64_t request_data_from_segment(void *data, size_t offset, size_t sz)
Definition buffer.cpp:1456
int ref_other_buffer(std::shared_ptr< FileBuffer > p, size_t offset, size_t size)
Definition buffer.cpp:1374
uint8_t & at(size_t index)
Definition buffer.h:319
int reload(std::string filename, bool async=false)
Definition buffer.cpp:1386
std::mutex m_seg_map_mutex
Definition buffer.h:200
friend class FSHttps
Definition buffer.h:169
bool IsLoaded() const noexcept
Definition buffer.h:271
bool IsKnownSize() const noexcept
Definition buffer.h:281
FileBuffer()
Definition buffer.cpp:1246
uint8_t & operator[](size_t index)
Definition buffer.h:311
std::mutex m_data_mutex
Definition buffer.h:187
std::shared_ptr< FragmentBlock > m_last_db
Definition buffer.h:205
~FileBuffer()
Definition buffer.cpp:1270
size_t m_total_buffer_size
Definition buffer.h:207
std::condition_variable m_request_cv
Definition buffer.h:248
ALLOCATION_WAYS
Definition buffer.h:179
@ MMAP
Definition buffer.h:181
@ MALLOC
Definition buffer.h:180
@ SEGMENT
Definition buffer.h:184
@ REF
Definition buffer.h:182
@ VMALLOC
Definition buffer.h:183
bool IsRefable() const noexcept
Definition buffer.h:276
friend class FSBase
Definition buffer.h:167
bool check_offset_in_seg(size_t offset, std::shared_ptr< FragmentBlock > blk)
Definition buffer.h:212
void truncate_old_data_in_pool()
Definition buffer.cpp:1426
uint8_t * m_pDatabuffer
Definition buffer.h:189
std::queue< size_t > m_offset_request
Definition buffer.h:201
std::mutex m_request_cv_mutex
Definition buffer.h:249
int reserve(size_t sz)
Definition buffer.cpp:1729
std::atomic_size_t m_available_size
Definition buffer.h:247
int swap(FileBuffer &a)
Definition buffer.cpp:1772
Definition buffer.h:78
uint8_t * data()
Definition buffer.h:99
virtual ~FragmentBlock()
Definition buffer.h:106
size_t m_input_offset
Definition buffer.h:87
std::shared_ptr< FileBuffer > m_input
Definition buffer.h:89
size_t m_ret
Definition buffer.h:90
std::vector< uint8_t > m_data
Definition buffer.h:95
size_t m_input_sz
Definition buffer.h:88
std::mutex m_mutex
Definition buffer.h:96
@ USING
Definition buffer.h:83
@ CONVERT_PARTIAL
Definition buffer.h:85
@ CONVERT_START
Definition buffer.h:84
@ CONVERT_DONE
Definition buffer.h:82
std::atomic_int m_dataflags
Definition buffer.h:97
virtual int DataConvert()
Definition buffer.h:94
size_t m_actual_size
Definition buffer.h:91
size_t m_output_offset
Definition buffer.h:93
uint8_t * m_pData
Definition buffer.h:98
size_t m_output_size
Definition buffer.h:92
auto pos
Definition usbhotplug.cpp:159